Add hierarchical schedulers

ChangeLog

@@ -44,6 +44,10 @@ New features:
     let starpu commute write accesses.
   * Out-of-core support, through registration of disk areas as additional memory
     nodes.
+  * New hierarchical schedulers which allow the user to easily build
+    its own scheduler, by coding itself each "box" it wants, or by
+    combining existing boxes in StarPU to build it. Hierarchical
+    schedulers have very interesting scalability properties.
   * Add STARPU_CUDA_ASYNC and STARPU_OPENCL_ASYNC flags to allow asynchronous
     CUDA and OpenCL kernel execution.
   * Add STARPU_CUDA_PIPELINE and STARPU_OPENCL_PIPELINE to specify how

Makefile.am

@@ -74,6 +74,7 @@ versinclude_HEADERS = 				\
 	include/starpu_profiling.h		\
 	include/starpu_bound.h			\
 	include/starpu_scheduler.h		\
+	include/starpu_sched_component.h		\
 	include/starpu_sched_ctx.h		\
 	include/starpu_sched_ctx_hypervisor.h	\
 	include/starpu_top.h			\

configure.ac

@@ -1059,7 +1059,7 @@ AC_DEFINE_UNQUOTED(STARPU_MAXMICDEVS, [$nmaxmicdev],
 AC_MSG_CHECKING(maximum number of MIC threads)
 AC_ARG_ENABLE(maxmicthreads, [AS_HELP_STRING([--enable-maxmicthreads=<number>],
 			[maximum number of MIC threads])],
-			nmaxmicthreads=$enableval, nmaxmicthreads=480)
+			nmaxmicthreads=$enableval, nmaxmicthreads=120)
 AC_MSG_RESULT($nmaxmicthread)
 
 AC_DEFINE_UNQUOTED(STARPU_MAXMICCORES, [$nmaxmicthreads],

doc/doxygen/Makefile.am

@@ -52,6 +52,7 @@ chapters =	\
 	chapters/45files.doxy \
 	chapters/50scaling-vector-example.doxy \
 	chapters/51fdl-1.3.doxy \
+	chapters/modularized_scheduler.doxy \
 	chapters/code/hello_pragma2.c \
 	chapters/code/hello_pragma.c \
 	chapters/code/scal_pragma.cu \
@@ -107,7 +108,8 @@ chapters =	\
 	chapters/api/tree.doxy \
 	chapters/api/toolbox.doxy \
 	chapters/api/sc_hypervisor/sc_hypervisor.doxy \
-	chapters/api/sc_hypervisor/sc_hypervisor_usage.doxy
+	chapters/api/sc_hypervisor/sc_hypervisor_usage.doxy \
+	chapters/api/modularized_scheduler.doxy
 
 starpu_config.h: $(top_srcdir)/include/starpu_config.h.in
 	@$(SED) 's/#undef \(.*\)/#define \1 1/' $< > $@

doc/doxygen/chapters/api/modularized_scheduler.doxy

@@ -0,0 +1,350 @@
+/*! \defgroup API_Modularized_Scheduler Modularized Scheduler Interface
+
+\struct starpu_sched_component
+\ingroup API_Modularized_Scheduler
+This structure represent a scheduler module.
+
+\var starpu_sched_component::push_task
+	push a task in the scheduler module.
+\var starpu_sched_component::pop_task
+	pop a task from the scheduler module, the task returned by this function is executable by the caller
+	It should use starpu_sched_component::fathers[sched_ctx_id] to perform a recursive call.
+
+\var starpu_sched_component::estimated_load
+	is an heuristic to compute load of scheduler module. Basically the number of tasks divided by the sum
+	of relatives speedup of workers available in context.
+\var starpu_sched_component::estimated_end
+	return the time when a worker will enter in starvation. This function is relevant only if the task->predicted
+	member has been set.
+\var starpu_sched_component::nchildren
+	the number of modules downstairs
+\var starpu_sched_component::children
+	modules downstairs
+\var starpu_sched_component::workers
+	this member contain the set of underlying workers
+\var starpu_sched_component::workers_in_ctx
+	this member contain the subset of starpu_sched_component::workers that is currently available in the context
+	The push method should take this member into account.
+\var starpu_sched_component::properties
+	flags starpu_sched_component_properties for things
+\var starpu_sched_component::data
+	data used by the scheduler module
+\var starpu_sched_component::add_child
+	add a child to component
+\var starpu_sched_component::remove_child
+	remove a child from component
+
+\var starpu_sched_component::fathers
+	the array of scheduler module above indexed by scheduling context index
+
+\var starpu_sched_component::notify_change_workers
+	this function is called when starpu_sched_component::workers_in_ctx or starpu_sched_component::workers is changed
+\var starpu_sched_component::deinit_data
+	called by starpu_sched_component_destroy. Should free data allocated during creation
+\var starpu_sched_component::obj
+	the hwloc object associated to scheduler module
+
+\enum starpu_sched_component_properties
+\ingroup API_Modularized_Scheduler
+flags for starpu_sched_component::properties
+\var STARPU_SCHED_component_HOMOGENEOUS
+     indicate that all workers have the same starpu_worker_archtype
+\var STARPU_SCHED_component_SINGLE_MEMORY_component
+     indicate that all workers have the same memory component
+
+\def STARPU_SCHED_component_IS_HOMOGENEOUS
+\ingroup API_Modularized_Scheduler
+     indicate if component is homogeneous
+\def STARPU_SCHED_component_IS_SINGLE_MEMORY_component
+\ingroup API_Modularized_Scheduler
+     indicate if all workers have the same memory component
+
+
+\struct starpu_sched_tree
+\ingroup API_Modularized_Scheduler
+The actual scheduler
+\var starpu_sched_tree::root
+	this is the entry module of the scheduler
+\var starpu_sched_tree::workers
+	this is the set of workers available in this context, this value is used to mask workers in modules
+\var starpu_sched_tree::lock
+	this lock protect concurrent access from the hypervisor and the application.
+\var starpu_sched_tree::sched_ctx_id
+	the context id of the scheduler
+
+\fn struct starpu_sched_component * starpu_sched_component_create(void)
+\ingroup API_Modularized_Scheduler
+	 allocate and initialize component field with defaults values :
+	.pop_task make recursive call on father
+	.estimated_load compute relative speedup and tasks in sub tree
+	.estimated_end return the average of recursive call on children
+	.add_child is starpu_sched_component_add_child
+	.remove_child is starpu_sched_component_remove_child
+	.notify_change_workers does nothing
+	.deinit_data does nothing
+
+
+\fn void starpu_sched_component_destroy(struct starpu_sched_component * component)
+\ingroup API_Modularized_Scheduler
+	 free data allocated by starpu_sched_component_create and call component->deinit_data(component)
+	 set to null the member starpu_sched_component::fathers[sched_ctx_id] of all child if its equal to \p component
+
+\fn void starpu_sched_component_set_father(struct starpu_sched_component * component, struct starpu_sched_component * father_component, unsigned sched_ctx_id)
+\ingroup API_Modularized_Scheduler
+	 set component->fathers[sched_ctx_id] to father_component
+
+\fn void starpu_sched_component_add_child(struct starpu_sched_component* component, struct starpu_sched_component * child)
+\ingroup API_Modularized_Scheduler
+	 add child to component->children and increment nchildren as well.
+	 and do not modify child->fathers
+	 \p child must not be already in starpu_sched_component::children of \p component
+
+\fn void starpu_sched_component_remove_child(struct starpu_sched_component * component, struct starpu_sched_component * child)
+\ingroup API_Modularized_Scheduler
+	 remove child from component->children and decrements nchildren
+	 \p child must be in starpu_sched_component::child of \p component
+
+
+\fn int starpu_sched_component_can_execute_task(struct starpu_sched_component * component, struct starpu_task * task)
+\ingroup API_Modularized_Scheduler
+	 return true iff \p component can execute \p task, this function take into account the workers available in the scheduling context
+
+\fn int STARPU_WARN_UNUSED_RESULT starpu_sched_component_execute_preds(struct starpu_sched_component * component, struct starpu_task * task, double * length);
+\ingroup API_Modularized_Scheduler
+	 return a non null value if \p component can execute \p task.
+	 write the execution prediction length for the best implementation of the best worker available and write this at \p length address.
+	 this result is more relevant if starpu_sched_component::is_homogeneous is non null.
+	 if a worker need to be calibrated for an implementation, nan is set to \p length.
+
+\fn double starpu_sched_component_transfer_length(struct starpu_sched_component * component, struct starpu_task * task);
+\ingroup API_Modularized_Scheduler
+	 return the average time to transfer \p task data to underlying \p component workers.
+
+\fn struct starpu_sched_component * starpu_sched_component_worker_get(int workerid)
+\ingroup API_Modularized_Scheduler
+	 return the struct starpu_sched_component corresponding to \p workerid. Undefined if \p workerid is not a valid workerid
+
+
+\fn int starpu_sched_component_is_worker(struct starpu_sched_component * component)
+\ingroup API_Modularized_Scheduler
+	 return true iff \p component is a worker component
+\fn int starpu_sched_component_is_simple_worker(struct starpu_sched_component * component)
+\ingroup API_Modularized_Scheduler
+	 return true iff \p component is a simple worker component
+\fn int starpu_sched_component_is_combined_worker(struct starpu_sched_component * component)
+\ingroup API_Modularized_Scheduler
+	 return true iff \p component is a combined worker component
+\fn int starpu_sched_component_worker_get_workerid(struct starpu_sched_component * worker_component)
+\ingroup API_Modularized_Scheduler
+	 return the workerid of \p worker_component, undefined if starpu_sched_component_is_worker(worker_component) == 0
+
+
+\fn struct starpu_sched_component * starpu_sched_component_fifo_create(void * arg STARPU_ATTRIBUTE_UNUSED)
+\ingroup API_Modularized_Scheduler
+	 Return a struct starpu_sched_component with a fifo. A stable sort is performed according to tasks priorities.
+	 A push_task call on this component does not perform recursive calls, underlying components will have to call pop_task to get it.
+	 starpu_sched_component::estimated_end function compute the estimated length by dividing the sequential length by the number of underlying workers. Do not take into account tasks that are currently executed.
+
+\fn int starpu_sched_component_is_fifo(struct starpu_sched_component * component)
+\ingroup API_Modularized_Scheduler
+	 return true iff \p component is a fifo component
+
+\fn struct starpu_sched_component * starpu_sched_component_work_stealing_create(void * arg STARPU_ATTRIBUTE_UNUSED)
+\ingroup API_Modularized_Scheduler
+	 return a component that perform a work stealing scheduling. Tasks are pushed in a round robin way. estimated_end return the average of expected length of fifos, starting at the average of the expected_end of his children. When a worker have to steal a task, it steal a task in a round robin way, and get the last pushed task of the higher priority.
+
+
+\fn int starpu_sched_tree_work_stealing_push_task(struct starpu_task *task)
+\ingroup API_Modularized_Scheduler
+	 undefined if there is no work stealing component in the scheduler. If any, \p task is pushed in a default way if the caller is the application, and in the caller's fifo if its a worker.
+
+\fn int starpu_sched_component_is_work_stealing(struct starpu_sched_component * component)
+\ingroup API_Modularized_Scheduler
+	 return true iff \p component is a work stealing component
+
+\fn struct starpu_sched_component * starpu_sched_component_random_create(void * arg STARPU_ATTRIBUTE_UNUSED)
+\ingroup API_Modularized_Scheduler
+	 create a component that perform a random scheduling
+
+\fn int starpu_sched_component_is_random(struct starpu_sched_component *);
+\ingroup API_Modularized_Scheduler
+	 return true iff \p component is a random component
+
+\fn struct starpu_sched_component * starpu_sched_component_heft_create(struct starpu_heft_data * heft_data)
+\ingroup API_Modularized_Scheduler
+	 this component perform a heft scheduling
+
+\fn int starpu_sched_component_is_heft(struct starpu_sched_component * component)
+\ingroup API_Modularized_Scheduler
+	 return true iff \p component is a heft component
+
+\struct starpu_heft_data
+\ingroup API_Modularized_Scheduler
+starpu_sched_component_heft_create parameters
+\var starpu_heft_data::alpha
+	 coefficient applied to computation length
+\var starpu_heft_data::beta
+	 coefficient applied to communication length
+\var starpu_heft_data::gamma
+	 coefficient applied to power consumption
+\var starpu_heft_data::idle_power
+	 idle consumption of the machine
+\var starpu_heft_data::no_perf_model_component_create
+	 called to create the component to push task for whom no perf model is available
+\var starpu_heft_data::arg_no_perf_model
+	 argument passed to starpu_heft_data::no_perf_model_component_create
+\var starpu_heft_data::calibrating_component_create
+	 idem for tasks with an non calibrated implementation
+\var starpu_heft_data::arg_calibrating_component
+	 argument passed to starpu_heft_data::calibrating_component_create
+
+
+\fn struct starpu_sched_component * starpu_sched_component_best_implementation_create(void * arg STARPU_ATTRIBUTE_UNUSED);
+\ingroup API_Modularized_Scheduler
+	 Select the implementation that offer the shortest computation length for the first worker that can execute the task.
+	 Or an implementation that need to be calibrated.
+	 Also set starpu_task::predicted and starpu_task::predicted_transfer for memory component of the first suitable workerid.
+	 If starpu_sched_component::push method is called and starpu_sched_component::nchild > 1 the result is undefined.
+
+
+
+\fn struct starpu_sched_tree * starpu_sched_tree_create(void)
+\ingroup API_Modularized_Scheduler
+	 create a empty initialized starpu_sched_tree
+\fn void starpu_sched_tree_destroy(struct starpu_sched_tree * tree, unsigned sched_ctx_id)
+\ingroup API_Modularized_Scheduler
+	 destroy tree and free all non shared component in it.
+\fn void starpu_sched_component_destroy_rec (struct starpu_sched_component *component, unsigned sched_ctx_id)
+\ingroup API_Modularized_Scheduler
+	 recursively destroy non shared parts of a \p component 's tree
+
+
+\fn starpu_sched_component_available(struct starpu_sched_component * component)
+\ingroup API_Modularized_Scheduler
+	 notify all component's underlying workers that a task is available to pop
+
+\fn int starpu_sched_tree_push_task(struct starpu_task * task)
+\ingroup API_Modularized_Scheduler
+	 compatibility with starpu_sched_policy interface
+\fn struct starpu_task * starpu_sched_tree_pop_task(unsigned sched_ctx_id)
+\ingroup API_Modularized_Scheduler
+	 compatibility with starpu_sched_policy interface
+\fn void starpu_sched_tree_add_workers(unsigned sched_ctx_id, int *workerids, unsigned nworkers)
+\ingroup API_Modularized_Scheduler
+	 compatibility with starpu_sched_policy interface
+\fn void starpu_sched_tree_remove_workers(unsigned sched_ctx_id, int *workerids, unsigned nworkers)
+\ingroup API_Modularized_Scheduler
+	 compatibility with starpu_sched_policy interface
+\fn void starpu_sched_component_worker_pre_exec_hook(struct starpu_task * task)
+\ingroup API_Modularized_Scheduler
+	 compatibility with starpu_sched_policy interface
+	 update predictions for workers
+\fn void starpu_sched_component_worker_post_exec_hook(struct starpu_task * task)
+\ingroup API_Modularized_Scheduler
+	 compatibility with starpu_sched_policy interface
+
+
+\fn void starpu_sched_tree_update_workers(struct starpu_sched_tree * t)
+\ingroup API_Modularized_Scheduler
+	 recursively set all starpu_sched_component::workers, do not take into account shared parts (except workers).
+
+\fn void starpu_sched_tree_update_workers_in_ctx(struct starpu_sched_tree * t)
+\ingroup API_Modularized_Scheduler
+	 recursively set all starpu_sched_component::workers_in_ctx, do not take into account shared parts (except workers)
+
+\struct starpu_bitmap;
+\ingroup API_Modularized_Scheduler
+	 implement a simple bitmap
+\fn struct starpu_bitmap * starpu_bitmap_create(void)
+\ingroup API_Modularized_Scheduler
+	 create a empty starpu_bitmap
+\fn void starpu_bitmap_destroy(struct starpu_bitmap *)
+\ingroup API_Modularized_Scheduler
+	 free a starpu_bitmap
+
+\fn void starpu_bitmap_set(struct starpu_bitmap * bitmap, int e)
+\ingroup API_Modularized_Scheduler
+	 set bit \p e in \p bitmap
+\fn void starpu_bitmap_unset(struct starpu_bitmap * bitmap, int e)
+\ingroup API_Modularized_Scheduler
+	 unset bit \p e in \p bitmap
+
+\fn void starpu_bitmap_unset_all(struct starpu_bitmap * bitmap)
+\ingroup API_Modularized_Scheduler
+	 unset all bits in \b bitmap
+
+\fn int starpu_bitmap_get(struct starpu_bitmap * bitmap, int e);
+\ingroup API_Modularized_Scheduler
+	 return true iff bit \p e is set in \p bitmap
+
+\fn int starpu_bitmap_cardinal(struct starpu_bitmap * bitmap);
+\ingroup API_Modularized_Scheduler
+	 return the number of set bits in \p bitmap
+
+
+\fn int starpu_bitmap_first(struct starpu_bitmap * bitmap)
+\ingroup API_Modularized_Scheduler
+	 return the position of the first set bit of \p bitmap, -1 if none
+\fn int starpu_bitmap_last(struct starpu_bitmap * bitmap)
+\ingroup API_Modularized_Scheduler
+	 return the position of the last set bit of \p bitmap, -1 if none
+
+\fn int starpu_bitmap_next(struct starpu_bitmap *, int e)
+\ingroup API_Modularized_Scheduler
+	 return the position of set bit right after \p e in \p bitmap, -1 if none
+
+
+\struct starpu_sched_component_composed_recipe;
+\ingroup API_Modularized_Scheduler
+	parameters for starpu_sched_component_composed_component_create
+
+\fn struct starpu_sched_component_composed_recipe * starpu_sched_component_create_recipe(void)
+\ingroup API_Modularized_Scheduler
+	 return an empty recipe for a composed component, it should not be used without modification
+
+
+\fn struct starpu_sched_component_composed_recipe * starpu_sched_component_create_recipe_singleton(struct starpu_sched_component *(*create_component)(void * arg), void * arg)
+\ingroup API_Modularized_Scheduler
+	 return a recipe to build a composed component with a \p create_component
+
+\fn void starpu_sched_recipe_add_component(struct starpu_sched_component_composed_recipe * recipe, struct starpu_sched_component *(*create_component)(void * arg), void * arg)
+\ingroup API_Modularized_Scheduler
+	 add \p create_component under all previous components in recipe
+
+\fn void starpu_destroy_composed_sched_component_recipe(struct starpu_sched_component_composed_recipe *)
+\ingroup API_Modularized_Scheduler
+	 destroy composed_sched_component, this should be done after starpu_sched_component_composed_component_create was called
+
+\fn struct starpu_sched_component * starpu_sched_component_composed_component_create(struct starpu_sched_component_composed_recipe * recipe)
+\ingroup API_Modularized_Scheduler
+	 create a component that behave as all component of recipe where linked. Except that you cant use starpu_sched_component_is_foo function
+	 if recipe contain a single create_foo arg_foo pair, create_foo(arg_foo) is returned instead of a composed component
+
+
+\struct starpu_sched_specs
+\ingroup API_Modularized_Scheduler
+	 Define how build a scheduler according to topology. Each level (except for hwloc_machine_composed_sched_component) can be NULL, then
+	 the level is just skipped. Bugs everywhere, do not rely on.
+\var starpu_sched_specs::hwloc_machine_composed_sched_component
+     the composed component to put on the top of the scheduler
+     this member must not be NULL
+\var starpu_sched_specs::hwloc_component_composed_sched_component
+     the composed component to put for each memory component
+\var starpu_sched_specs::hwloc_socket_composed_sched_component
+     the composed component to put for each socket
+\var starpu_sched_specs::hwloc_cache_composed_sched_component
+     the composed component to put for each cache
+     
+\var starpu_sched_specs::worker_composed_sched_component
+     a function that return a starpu_sched_component_composed_recipe to put on top of a worker of type \p archtype.
+     NULL is a valid return value, then no component will be added on top
+\var starpu_sched_specs::mix_heterogeneous_workers
+     this flag is a dirty hack because of the poor expressivity of this interface. As example, if you want to build
+     a heft component with a fifo component per numa component, and you also have GPUs, if this flag is set, GPUs will share those fifos.
+     If this flag is not set, a new fifo will be built for each of them (if they have the same starpu_perf_arch and the same
+     numa component it will be shared
+
+\fn struct starpu_sched_tree * starpu_sched_component_make_scheduler(unsigned sched_ctx_id, struct starpu_sched_specs s);
+\ingroup API_Modularized_Scheduler
+	 this function build a scheduler for \p sched_ctx_id according to \p s and the hwloc topology of the machine.
+*/

doc/doxygen/chapters/modularized_scheduler.doxy

@@ -0,0 +1,353 @@
+/*
+ * This file is part of the StarPU Handbook.
+ * Copyright (C) 2013 Simon Archipoff
+ * Copyright (C) 2013 INRIA
+ * See the file version.doxy for copying conditions.
+ */
+
+/*! \page ModularizedScheduler Modularized Schedulers
+
+\section Introduction
+
+StarPU's Modularized Schedulers are made of individual Scheduling Components 
+Modularizedly assembled as a Scheduling Tree. Each Scheduling Component has an 
+unique purpose, such as prioritizing tasks or mapping tasks over resources.
+A typical Scheduling Tree is shown below.
+
+<pre>
+                                  |
+              starpu_push_task    |
+                                  |
+                                  v
+                            Fifo_Component
+                                |  ^
+                                |  |        
+                                v  |
+                           Eager_Component
+                                |  ^
+                                |  |    
+                                v  |
+                 --------><--------------><--------
+                 |  ^                          |  ^
+                 |  |                          |  |        
+                 v  |                          v  |
+             Fifo_Component                 Fifo_Component
+                 |  ^                          |  ^
+                 |  |                          |  |        
+                 v  |                          v  |
+            Worker_Component               Worker_Component
+</pre>
+
+When a task is pushed by StarPU in a Modularized Scheduler, the task moves from
+a Scheduling Component to an other, following the hierarchy of the
+Scheduling Tree, and is stored in one of the Scheduling Components of the 
+strategy.
+When a worker wants to pop a task from the Modularized Scheduler, the
+corresponding Worker Component of the Scheduling Tree tries to pull a task from 
+its parents, following the hierarchy, and gives it to the worker if it succeded 
+to get one.
+
+
+\section UsingModularizedSchedulers Using Modularized Schedulers
+
+\subsection ExistingModularizedSchedulers Existing Modularized Schedulers
+
+StarPU is currently shipped with the following pre-defined Modularized 
+Schedulers :
+
+- Eager-based Schedulers (with/without prefetching) : \n
+Naive scheduler, which tries to map a task on the first available resource
+it finds.
+
+- Prio-based Schedulers (with/without prefetching) : \n
+Similar to Eager-Based Schedulers. Can handle tasks which have a defined 
+priority and schedule them accordingly.
+
+- Random-based Schedulers (with/without prefetching) : \n
+Selects randomly a resource to be mapped on for each task. 
+
+- HEFT Scheduler : \n
+Heterogeneous Earliest Finish Time Scheduler.
+This scheduler needs that every task submitted to StarPU have a
+defined performance model (\ref PerformanceModelCalibration)
+to work efficiently, but can handle tasks without a performance
+model.
+
+It is currently needed to set the environment variable \ref STARPU_SCHED 
+to use those Schedulers. Modularized Schedulers' naming is tree-*
+
+\subsection ExampleTreeEagerPrefetchingStrategy An Example : The Tree-Eager-Prefetching Strategy
+
+<pre>
+                                 |
+             starpu_push_task    |
+                                 |
+                                 v
+                           Fifo_Component
+                                |  ^
+                        Push    |  |    Can_Push
+                                v  |
+                          Eager_Component
+                                |  ^
+                                |  |    
+                                v  |
+              --------><-------------------><---------
+              |  ^                                |  ^
+      Push    |  |    Can_Push            Push    |  |    Can_Push
+              v  |                                v  |
+         Fifo_Component                       Fifo_Component
+              |  ^                                |  ^
+      Pull    |  |    Can_Pull            Pull    |  |    Can_Pull
+              v  |                                v  |
+        Worker_Component                     Worker_Component
+</pre>
+
+\subsection Interface
+
+Each Scheduling Component must follow the following pre-defined Interface 
+to be able to interact with other Scheduling Components.
+
+	- Push (Caller_Component, Child_Component, Task) \n
+	The calling Scheduling Component transfers a task to its 
+	Child Component. When the Push function returns, the task no longer 
+	belongs to the calling Component. The Modularized Schedulers' 
+	model relies on this function to perform prefetching.
+
+	- Pull (Caller_Component, Parent_Component)  ->  Task \n
+	The calling Scheduling Component requests a task from
+	its Parent Component. When the Pull function ends, the returned 
+	task belongs to the calling Component.
+
+	- Can_Push (Caller_Component, Parent_Component) \n
+	The calling Scheduling Component notifies its Parent Component that 
+	it is ready to accept new tasks.
+
+	- Can_Pull (Caller_Component, Child_Component) \n
+	The calling Scheduling Component notifies its Child Component
+	that it is ready to give new tasks.
+
+
+\section BuildAModularizedScheduler Build a Modularized Scheduler
+
+\subsection PreImplementedComponents Pre-implemented Components
+
+StarPU is currently shipped with the following four Scheduling Components : 
+
+	- Flow-control Components : Fifo, Prio \n 
+	Components which store tasks. They can also prioritize them if
+	they have a defined priority. It is possible to define a threshold
+	for those Components following two criterias : the number of tasks
+	stored in the Component, or the sum of the expected length of all
+	tasks stored in the Component.
+
+	- Resource-Mapping Components : Mct, Heft, Eager, Random, Work-Stealing \n
+	"Core" of the Scheduling Strategy, those Components are the
+	ones who make scheduling choices.
+
+	- Worker Components : Worker \n
+	Each Worker Component modelize a concrete worker.
+
+	- Special-Purpose Components : Perfmodel_Select, Best_Implementation \n
+	Components dedicated to original purposes. The Perfmodel_Select 
+	Component decides which Resource-Mapping Component should be used to 
+	schedule a task. The Best_Implementation Component chooses which
+	implementation of a task should be used on the choosen resource.
+
+\subsection ProgressionAndValidationRules Progression And Validation Rules
+
+Some rules must be followed to ensure the correctness of a Modularized 
+Scheduler :
+
+	- At least one Flow-control Component without threshold per Worker Component 
+	is needed in a Modularized Scheduler, to store incoming tasks from StarPU 
+	and to give tasks to Worker Components who asks for it. It is possible to 
+	use one Flow-control Component per Worker Component, or one for all Worker
+	Components, depending on how the Scheduling Tree is defined.
+
+	- At least one Resource-Mapping Component is needed in a Modularized
+	Scheduler. Resource-Mapping Components are the only ones who can make
+	scheduling choices, and so the only ones who can have several child.
+
+\subsection ImplementAModularizedScheduler Implement a Modularized Scheduler
+
+The following code shows how the Tree-Eager-Prefetching Scheduler
+shown in Section \ref ExampleTreeEagerPrefetchingStrategy is implemented :
+
+\code{.c}
+#define _STARPU_SCHED_NTASKS_THRESHOLD_DEFAULT 2
+#define _STARPU_SCHED_EXP_LEN_THRESHOLD_DEFAULT 1000000000.0
+
+static void initialize_eager_prefetching_center_policy(unsigned sched_ctx_id)
+{
+  unsigned ntasks_threshold = _STARPU_SCHED_NTASKS_THRESHOLD_DEFAULT;
+  double exp_len_threshold = _STARPU_SCHED_EXP_LEN_THRESHOLD_DEFAULT;
+
+  [...]
+
+  starpu_sched_ctx_create_worker_collection
+    (sched_ctx_id, STARPU_WORKER_LIST);
+
+  /* Create the Scheduling Tree */
+  struct starpu_sched_tree * t = 
+    starpu_sched_tree_create(sched_ctx_id);
+
+  /* The Root Component is a Flow-control Fifo Component */
+   t->root = starpu_sched_component_fifo_create(NULL);
+
+  /* The Resource-mapping Component of the strategy is an Eager Component
+   */
+  struct starpu_sched_component * eager_component =
+    starpu_sched_component_eager_create(NULL);
+
+  /* Create links between Components : the Eager Component is the child 
+   * of the Root Component */
+  t->root->add_child
+    (t->root, eager_component);
+  eager_component->add_father
+    (eager_component, t->root);
+
+  /* A task threshold is set for the Flow-control Components which will 
+   * be connected to Worker Components. By doing so, this Modularized 
+   * Scheduler will be able to perform some prefetching on the resources 
+   */
+  struct starpu_fifo_data fifo_data =
+  {
+    .ntasks_threshold = ntasks_threshold,
+    .exp_len_threshold = exp_len_threshold,
+  };
+
+  unsigned i;
+  for(i = 0;
+    i < starpu_worker_get_count() + 
+    starpu_combined_worker_get_count();
+    i++)
+  {
+    /* Each Worker Component has a Flow-control Fifo Component as 
+     * father */
+    struct starpu_sched_component * worker_component =
+	  starpu_sched_component_worker_get(i);
+    struct starpu_sched_component * fifo_component =
+	  starpu_sched_component_fifo_create(&fifo_data);
+    fifo_component->add_child
+      (fifo_component, worker_component);
+    worker_component->add_father
+      (worker_component, fifo_component);
+
+    /* Each Flow-control Fifo Component associated to a Worker 
+     * Component is linked to the Eager Component as one of its 
+     * children */
+    eager_component->add_child
+      (eager_component, fifo_component);
+    fifo_component->add_father
+      (fifo_component, eager_component);
+  }
+
+  starpu_sched_tree_update_workers(t);
+  starpu_sched_ctx_set_policy_data
+    (sched_ctx_id, (void*)t);
+}
+
+/* Properly destroy the Scheduling Tree and all its Components */
+static void deinitialize_eager_prefetching_center_policy(unsigned sched_ctx_id)
+{
+  struct starpu_sched_tree * tree =
+  	(struct starpu_sched_tree*)starpu_sched_ctx_get_policy_data(sched_ctx_id);
+  starpu_sched_tree_destroy(tree);
+  starpu_sched_ctx_delete_worker_collection
+    (sched_ctx_id);
+}
+
+/* Initializing the starpu_sched_policy struct associated to the Modularized
+ * Scheduler : only the init_sched and deinit_sched needs to be defined to
+ * implement a Modularized Scheduler */
+struct starpu_sched_policy _starpu_sched_tree_eager_prefetching_policy =
+{
+  .init_sched = initialize_eager_prefetching_center_policy,
+  .deinit_sched = deinitialize_eager_prefetching_center_policy,
+  .add_workers = starpu_sched_tree_add_workers,
+  .remove_workers = starpu_sched_tree_remove_workers,
+  .push_task = starpu_sched_tree_push_task,
+  .pop_task = starpu_sched_tree_pop_task,
+  .pre_exec_hook = starpu_sched_component_worker_pre_exec_hook,
+  .post_exec_hook = starpu_sched_component_worker_post_exec_hook,
+  .pop_every_task = NULL,
+  .policy_name = "tree-eager-prefetching",
+  .policy_description = "eager with prefetching tree policy"
+};
+\endcode
+
+\section WriteASchedulingComponent Write a Scheduling Component
+
+\subsection GenericSchedulingComponent Generic Scheduling Component
+
+Each Scheduling Component is instantiated from a Generic Scheduling Component,
+which implements a generic version of the Interface. The generic implementation
+of Pull, Can_Pull and Can_Push functions are recursive calls to their parents
+(respectively to their children). However, as a Generic Scheduling Component do
+not know how much children it will have when it will be instantiated, it does 
+not implement the Push function.
+
+\subsection InstantiationRedefineInterface Instantiation : Redefine the Interface
+
+A Scheduling Component must implement all the functions of the Interface. It is
+so necessary to implement a Push function to instantiate a Scheduling Component.
+The implemented Push function is the "fingerprint" of a Scheduling Component.
+Depending on how functionalities or properties the programmer wants to give
+to the Scheduling Component he is implementing, it is possible to reimplement
+all the functions of the Interface. For example, a Flow-control Component
+reimplements the Pull and the Can_Push functions of the Interface, allowing him
+to catch the generic recursive calls of these functions. The Pull function of
+a Flow-control Component can, for example, pop a task from the local storage 
+queue of the Component, and give it to the calling Component which asks for it.
+
+\subsection DetailedProgressionAndValidationRules Detailed Progression and Validation Rules
+
+	- A Reservoir is a Scheduling Component which redefines a Push and a Pull
+	function, in order to store tasks into it. A Reservoir delimit Scheduling
+	Areas in the Scheduling Tree.
+
+	- A Pump is the engine source of the Scheduler : it pushes/pulls tasks
+	to/from a Scheduling Component to an other. Native Pumps of a Scheduling 
+	Tree are located at the root of the Tree (incoming Push calls from StarPU), 
+	and at the leafs of the Tree (Pop calls coming from StarPU Workers). 
+	Pre-implemented Scheduling Components currently shipped with Pumps are 
+	Flow-Control Components and the Resource-Mapping Component Heft, within 
+	their defined Can_Push functions.
+
+	- A correct Scheduling Tree requires a Pump per Scheduling Area and per 
+	Execution Flow. 
+
+
+The Tree-Eager-Prefetching Scheduler shown in Section 
+\ref ExampleTreeEagerPrefetchingStrategy follows the previous assumptions :
+
+<pre>
+                                  starpu_push_task
+                                       <b>Pump</b>
+                                         |
+ Area 1                                  |
+                                         |
+                                         v
+            -----------------------Fifo_Component-----------------------------
+                                       <b>Pump</b>
+                                        |  ^
+                                Push    |  |    Can_Push
+                                        v  |
+ Area 2                           Eager_Component
+                                        |  ^
+                                        |  |    
+                                        v  |
+                      --------><-------------------><---------
+                      |  ^                                |  ^
+              Push    |  |    Can_Push            Push    |  |    Can_Push
+                      v  |                                v  |
+            -----Fifo_Component-----------------------Fifo_Component----------
+                      |  ^                                |  ^
+              Pull    |  |    Can_Pull            Pull    |  |    Can_Pull
+ Area 3               v  |                                v  |
+                     <b>Pump</b>                               <b>Pump</b>
+                Worker_Component                     Worker_Component
+</pre>
+
+*/
+

doc/doxygen/doxygen-config.cfg.in

@@ -56,6 +56,7 @@ INPUT                  = @top_srcdir@/doc/doxygen/chapters \
 			 @top_srcdir@/include/starpu_tree.h \
 			 @top_srcdir@/include/starpu_util.h \
 			 @top_srcdir@/include/starpu_worker.h \
+			 @top_srcdir@/include/starpu_sched_component.h \
 			 @top_srcdir@/mpi/include/ \
 			 @top_srcdir@/starpufft/include/starpufft.h \
 			 @top_srcdir@/sc_hypervisor/include

doc/doxygen/refman.tex

@@ -97,6 +97,11 @@ Documentation License”.
 \hypertarget{SchedulingContextHypervisor}{}
 \input{SchedulingContextHypervisor}
 
+\chapter{Modularized Scheduler}
+\label{ModularizedScheduler}
+\hypertarget{ModularizedScheduler}{}
+\input{ModularizedScheduler}
+
 \chapter{Debugging Tools}
 \label{DebuggingTools}
 \hypertarget{DebuggingTools}{}
@@ -217,6 +222,7 @@ Documentation License”.
 \input{group__API__Tree}
 \input{group__API__SC__Hypervisor__usage}
 \input{group__API__SC__Hypervisor}
+\input{group__API__Modularized__Scheduler}
 
 \chapter{File Index}
 \input{files}

include/starpu_sched_component.h

@@ -0,0 +1,356 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2013  Simon Archipoff
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#ifndef __STARPU_SCHED_COMPONENT_H__
+#define __STARPU_SCHED_COMPONENT_H__
+#include <starpu.h>
+#include <common/starpu_spinlock.h>
+#include <common/fxt.h>
+#include <starpu_bitmap.h>
+
+#ifdef STARPU_HAVE_HWLOC
+#include <hwloc.h>
+#endif
+
+enum starpu_sched_component_properties
+{
+	STARPU_SCHED_COMPONENT_HOMOGENEOUS = (1<<0),
+	STARPU_SCHED_COMPONENT_SINGLE_MEMORY_NODE = (1<<1)
+};
+
+#define STARPU_SCHED_COMPONENT_IS_HOMOGENEOUS(component) ((component)->properties & STARPU_SCHED_COMPONENT_HOMOGENEOUS)
+#define STARPU_SCHED_COMPONENT_IS_SINGLE_MEMORY_NODE(component) ((component)->properties & STARPU_SCHED_COMPONENT_SINGLE_MEMORY_NODE)
+
+/* struct starpu_sched_component are scheduler modules, a scheduler is a tree-like
+ * structure of them, some parts of scheduler can be shared by several contexes
+ * to perform some local optimisations, so, for all components, a list of parent is
+ * defined indexed by sched_ctx_id
+ *
+ * they embed there specialised method in a pseudo object-style, so calls are like component->push_task(component,task)
+ *
+ */
+struct starpu_sched_component
+{
+	/* the set of workers in the component's subtree
+	 */
+	struct starpu_bitmap * workers;
+	/* the workers available in context
+	 * this member is set with : 
+	 * component->workers UNION tree->workers UNION
+	 * component->child[i]->workers_in_ctx iff exist x such as component->children[i]->parents[x] == component
+	 */
+	struct starpu_bitmap * workers_in_ctx;
+	
+	/* component's private data, no restriction on use
+	 */
+	void * data;
+
+	/* the numbers of component's children
+	 */
+	int nchildren;
+	/* the vector of component's children
+	 */
+	struct starpu_sched_component ** children;
+	/* the numbers of component's parents
+	 */
+	int nparents;
+	/* may be shared by several contexts
+	 * so we need several parents
+	 */
+	struct starpu_sched_component ** parents;
+
+	void (*add_child)(struct starpu_sched_component * component, struct starpu_sched_component * child);
+	void (*remove_child)(struct starpu_sched_component * component, struct starpu_sched_component * child);
+	void (*add_parent)(struct starpu_sched_component * component, struct starpu_sched_component * parent);
+	void (*remove_parent)(struct starpu_sched_component * component, struct starpu_sched_component * parent);
+
+	/* component->push_task(component, task)
+	 * this function is called to push a task on component subtree, this can either
+	 * perform a recursive call on a child or store the task in the component, then
+	 * it will be returned by a further pull_task call
+	 *
+	 * the caller must ensure that component is able to execute task
+	 */
+	int (*push_task)(struct starpu_sched_component *,
+			 struct starpu_task *);
+	/* this function is called by workers to get a task on them parents
+	 * this function should first return a localy stored task or perform
+	 * a recursive call on parent
+	 *
+	 * a default implementation simply do a recursive call on parent
+	 */
+	struct starpu_task * (*pull_task)(struct starpu_sched_component *);
+
+	/* This function is called by a component which implements a queue, allowing it to
+	 * signify to its parents that an empty slot is available in its queue.
+	 * The basic implementation of this function is a recursive call to its
+	 * parents, the user have to specify a personally-made function to catch those
+	 * calls.
+	 */ 
+	int (*can_push)(struct starpu_sched_component * component);
+	/* This function allow a component to wake up a worker.
+	 * It is currently called by component which implements a queue, to signify to
+	 * its children that a task have been pushed in its local queue, and is
+	 * available to been popped by a worker, for example.
+	 * The basic implementation of this function is a recursive call to
+	 * its children, until at least one worker have been woken up.
+	 */
+	void (*can_pull)(struct starpu_sched_component * component);
+
+	/* this function is an heuristic that compute load of subtree, basicaly
+	 * it compute
+	 * estimated_load(component) = sum(estimated_load(component_children)) +
+	 *          nb_local_tasks / average(relative_speedup(underlying_worker))
+	 */
+	double (*estimated_load)(struct starpu_sched_component * component);
+	double (*estimated_end)(struct starpu_sched_component * component);
+
+	/* this function is called by starpu_sched_component_destroy just before freeing component
+	 */
+	void (*deinit_data)(struct starpu_sched_component * component);
+	/* this function is called for each component when workers are added or removed from a context
+	 */
+	void (*notify_change_workers)(struct starpu_sched_component * component);
+
+	/* is_homogeneous is 0 if workers in the component's subtree are heterogeneous,
+	 * this field is set and updated automaticaly, you shouldn't write on it
+	 */
+	int properties;
+
+#ifdef STARPU_HAVE_HWLOC
+	/* in case of a modularized scheduler, this is set to the part of
+	 * topology that is binded to this component, eg: a numa node for a ws
+	 * component that would balance load between underlying sockets
+	 */
+	hwloc_obj_t obj;
+#endif
+};
+
+struct starpu_sched_tree
+{
+	struct starpu_sched_component * root;
+	struct starpu_bitmap * workers;
+	unsigned sched_ctx_id;
+	/* this lock is used to protect the scheduler,
+	 * it is taken in read mode pushing a task
+	 * and in write mode for adding or removing workers
+	 */
+	starpu_pthread_mutex_t lock;
+};
+
+/*******************************************************************************
+ *							Scheduling Tree's Interface 					   *
+ ******************************************************************************/
+
+/* create an empty tree
+ */
+struct starpu_sched_tree * starpu_sched_tree_create(unsigned sched_ctx_id);
+void starpu_sched_tree_destroy(struct starpu_sched_tree * tree);
+
+/* destroy component and all his child
+ * except if they are shared between several contexts
+ */
+void starpu_sched_component_destroy_rec(struct starpu_sched_component * component);
+
+/* update all the component->workers member recursively
+ */
+void starpu_sched_tree_update_workers(struct starpu_sched_tree * t);
+/* idem for workers_in_ctx 
+ */
+void starpu_sched_tree_update_workers_in_ctx(struct starpu_sched_tree * t);
+
+int starpu_sched_tree_push_task(struct starpu_task * task);
+struct starpu_task * starpu_sched_tree_pop_task();
+
+void starpu_sched_tree_add_workers(unsigned sched_ctx_id, int *workerids, unsigned nworkers);
+void starpu_sched_tree_remove_workers(unsigned sched_ctx_id, int *workerids, unsigned nworkers);
+
+/*******************************************************************************
+ *					Generic Scheduling Component's Interface 				   *
+ ******************************************************************************/
+
+struct starpu_sched_component * starpu_sched_component_create(void);
+void starpu_sched_component_destroy(struct starpu_sched_component * component);
+
+int starpu_sched_component_can_execute_task(struct starpu_sched_component * component, struct starpu_task * task);
+int STARPU_WARN_UNUSED_RESULT starpu_sched_component_execute_preds(struct starpu_sched_component * component, struct starpu_task * task, double * length);
+double starpu_sched_component_transfer_length(struct starpu_sched_component * component, struct starpu_task * task);
+void starpu_sched_component_prefetch_on_node(struct starpu_sched_component * component, struct starpu_task * task);
+
+/*******************************************************************************
+ *							Worker Component's Interface 				   	   *
+ ******************************************************************************/
+
+/* no public create function for workers because we dont want to have several component_worker for a single workerid */
+struct starpu_sched_component * starpu_sched_component_worker_get(int workerid);
+int starpu_sched_component_worker_get_workerid(struct starpu_sched_component * worker_component);
+
+/* this function compare the available function of the component with the standard available for worker components*/
+int starpu_sched_component_is_worker(struct starpu_sched_component * component);
+int starpu_sched_component_is_simple_worker(struct starpu_sched_component * component);
+int starpu_sched_component_is_combined_worker(struct starpu_sched_component * component);
+
+void starpu_sched_component_worker_pre_exec_hook(struct starpu_task * task);
+void starpu_sched_component_worker_post_exec_hook(struct starpu_task * task);
+
+/*******************************************************************************
+ *					Flow-control Fifo Component's Interface 				   *
+ ******************************************************************************/
+
+struct starpu_fifo_data
+{
+	unsigned ntasks_threshold;
+	double exp_len_threshold;
+};
+
+struct starpu_sched_component * starpu_sched_component_fifo_create(struct starpu_fifo_data * fifo_data);
+int starpu_sched_component_is_fifo(struct starpu_sched_component * component);
+
+/*******************************************************************************
+ *					Flow-control Prio Component's Interface 				   *
+ ******************************************************************************/
+
+struct starpu_prio_data
+{
+	unsigned ntasks_threshold;
+	double exp_len_threshold;
+};
+
+struct starpu_sched_component * starpu_sched_component_prio_create(struct starpu_prio_data * prio_data);
+int starpu_sched_component_is_prio(struct starpu_sched_component * component);
+
+/*******************************************************************************
+ *			Resource-mapping Work-Stealing Component's Interface 			   *
+ ******************************************************************************/
+
+struct starpu_sched_component * starpu_sched_component_work_stealing_create(void * arg STARPU_ATTRIBUTE_UNUSED);
+int starpu_sched_component_is_work_stealing(struct starpu_sched_component * component);
+int starpu_sched_tree_work_stealing_push_task(struct starpu_task *task);
+
+/*******************************************************************************
+ *				Resource-mapping Random Component's Interface 			   	   *
+ ******************************************************************************/
+
+struct starpu_sched_component * starpu_sched_component_random_create(void * arg STARPU_ATTRIBUTE_UNUSED);
+int starpu_sched_component_is_random(struct starpu_sched_component *);
+
+/*******************************************************************************
+ *				Resource-mapping Eager Component's Interface 				   *
+ ******************************************************************************/
+
+struct starpu_sched_component * starpu_sched_component_eager_create(void * arg STARPU_ATTRIBUTE_UNUSED);
+int starpu_sched_component_is_eager(struct starpu_sched_component *);
+
+/*******************************************************************************
+ *			Resource-mapping Eager-Calibration Component's Interface 		   *
+ ******************************************************************************/
+
+struct starpu_sched_component * starpu_sched_component_eager_calibration_create(void * arg STARPU_ATTRIBUTE_UNUSED);
+int starpu_sched_component_is_eager_calibration(struct starpu_sched_component *);
+
+/*******************************************************************************
+ *				Resource-mapping MCT Component's Interface 					   *
+ ******************************************************************************/
+
+struct starpu_mct_data
+{
+	double alpha;
+	double beta;
+	double gamma;
+	double idle_power;
+};
+
+/* create a component with mct_data paremeters
+   a copy the struct starpu_mct_data * given is performed during the init_data call
+   the mct component doesnt do anything but pushing tasks on no_perf_model_component and calibrating_component
+*/
+struct starpu_sched_component * starpu_sched_component_mct_create(struct starpu_mct_data * mct_data);
+int starpu_sched_component_is_mct(struct starpu_sched_component * component);
+
+/*******************************************************************************
+ *				Resource-mapping HEFT Component's Interface 				   *
+ ******************************************************************************/
+
+struct starpu_sched_component * starpu_sched_component_heft_create(struct starpu_mct_data * mct_data);
+int starpu_sched_component_is_heft(struct starpu_sched_component * component);
+
+/*******************************************************************************
+ *		Special-purpose Best_Implementation Component's Interface 			   *
+ ******************************************************************************/
+
+/* this component select the best implementation for the first worker in context that can execute task.
+ * and fill task->predicted and task->predicted_transfer
+ * cannot have several child if push_task is called
+ */
+struct starpu_sched_component * starpu_sched_component_best_implementation_create(void * arg STARPU_ATTRIBUTE_UNUSED);
+
+struct starpu_perfmodel_select_data
+{
+	struct starpu_sched_component * calibrator_component;
+	struct starpu_sched_component * no_perfmodel_component;
+	struct starpu_sched_component * perfmodel_component;
+};
+
+/*******************************************************************************
+ *			Special-purpose Perfmodel_Select Component's Interface	 		   *
+ ******************************************************************************/
+
+struct starpu_sched_component * starpu_sched_component_perfmodel_select_create(struct starpu_perfmodel_select_data * perfmodel_select_data);
+int starpu_sched_component_is_perfmodel_select(struct starpu_sched_component * component);
+
+/*******************************************************************************
+ *						Recipe Component's Interface	 					   *
+ ******************************************************************************/
+
+struct starpu_sched_component_composed_recipe;
+
+/* create empty recipe */
+struct starpu_sched_component_composed_recipe * starpu_sched_component_create_recipe(void);
+struct starpu_sched_component_composed_recipe * starpu_sched_component_create_recipe_singleton(struct starpu_sched_component *(*create_component)(void * arg), void * arg);
+
+/* add a function creation component to recipe */
+void starpu_sched_recipe_add_component(struct starpu_sched_component_composed_recipe * recipe, struct starpu_sched_component *(*create_component)(void * arg), void * arg);
+
+void starpu_destroy_composed_sched_component_recipe(struct starpu_sched_component_composed_recipe *);
+struct starpu_sched_component * starpu_sched_component_composed_component_create(struct starpu_sched_component_composed_recipe * recipe);
+
+
+#ifdef STARPU_HAVE_HWLOC
+/* null pointer mean to ignore a level L of hierarchy, then components of levels > L become children of level L - 1 */
+struct starpu_sched_specs
+{
+	/* hw_loc_machine_composed_sched_component must be set as its the root of the topology */
+	struct starpu_sched_component_composed_recipe * hwloc_machine_composed_sched_component;
+	struct starpu_sched_component_composed_recipe * hwloc_component_composed_sched_component;
+	struct starpu_sched_component_composed_recipe * hwloc_socket_composed_sched_component;
+	struct starpu_sched_component_composed_recipe * hwloc_cache_composed_sched_component;
+
+	/* this member should return a new allocated starpu_sched_component_composed_recipe or NULL
+	 * the starpu_sched_component_composed_recipe_t must not include the worker component
+	 */
+	struct starpu_sched_component_composed_recipe * (*worker_composed_sched_component)(enum starpu_worker_archtype archtype);
+ 
+	/* this flag indicate if heterogenous workers should be brothers or cousins,
+	 * as example, if a gpu and a cpu should share or not there numa node
+	 */
+	int mix_heterogeneous_workers;
+};
+
+struct starpu_sched_tree * starpu_sched_component_make_scheduler(unsigned sched_ctx_id, struct starpu_sched_specs);
+#endif /* STARPU_HAVE_HWLOC */
+
+
+#endif

include/starpu_task.h

@@ -1,7 +1,7 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
  * Copyright (C) 2010-2014  Université de Bordeaux
- * Copyright (C) 2010, 2011, 2012, 2013  Centre National de la Recherche Scientifique
+ * Copyright (C) 2010, 2011, 2012, 2013, 2014  Centre National de la Recherche Scientifique
  * Copyright (C) 2011  Télécom-SudParis
  * Copyright (C) 2011, 2014  INRIA
  *
@@ -196,6 +196,7 @@ struct starpu_task
 	struct starpu_task *prev;
 	struct starpu_task *next;
 	void *starpu_private;
+	unsigned prefetched;
 #ifdef STARPU_OPENMP
 	struct starpu_omp_task *omp_task;
 #endif
@@ -227,6 +228,7 @@ struct starpu_task
 	.hypervisor_tag = 0,				\
 	.flops = 0.0,					\
 	.scheduled = 0,					\
+	.prefetched = 0,					\
 	.dyn_handles = NULL,				\
 	.dyn_interfaces = NULL,				\
 	.dyn_modes = NULL,				\

src/Makefile.am

@@ -79,6 +79,7 @@ noinst_HEADERS = 						\
 	core/simgrid.h						\
 	core/task_bundle.h					\
 	core/detect_combined_workers.h				\
+	sched_policies/helper_mct.h				\
 	sched_policies/fifo_queues.h				\
 	sched_policies/deque_queues.h				\
 	sched_policies/stack_queues.h				\
@@ -133,12 +134,14 @@ noinst_HEADERS = 						\
 	starpu_parameters.h					\
 	top/starpu_top_message_queue.h				\
 	top/starpu_top_connection.h				\
-	top/starpu_top_core.h					
+	top/starpu_top_core.h					\
+	sched_policies/prio_deque.h				\
+	sched_policies/sched_component.h
 
 libstarpu_@STARPU_EFFECTIVE_VERSION@_la_SOURCES = 		\
 	common/barrier.c					\
-	common/bitmap.c						\
 	common/barrier_counter.c				\
+	common/bitmap.c						\
 	common/hash.c 						\
 	common/rwlock.c						\
 	common/starpu_spinlock.c				\
@@ -244,9 +247,31 @@ libstarpu_@STARPU_EFFECTIVE_VERSION@_la_SOURCES = 		\
 	top/starpu_top.c					\
 	top/starpu_top_task.c					\
 	top/starpu_top_message_queue.c				\
-	top/starpu_top_connection.c                          	\
+	top/starpu_top_connection.c                        	\
 	worker_collection/worker_list.c				\
-	worker_collection/worker_tree.c				
+	worker_collection/worker_tree.c				\
+	sched_policies/component_worker.c				\
+	sched_policies/component_sched.c				\
+	sched_policies/component_fifo.c 				\
+	sched_policies/prio_deque.c				\
+	sched_policies/helper_mct.c				\
+	sched_policies/component_prio.c 				\
+	sched_policies/component_random.c				\
+	sched_policies/component_eager.c				\
+	sched_policies/component_eager_calibration.c				\
+	sched_policies/component_mct.c				\
+	sched_policies/component_heft.c				\
+	sched_policies/component_best_implementation.c		\
+	sched_policies/component_perfmodel_select.c				\
+	sched_policies/component_composed.c				\
+	sched_policies/modular_eager.c				\
+	sched_policies/modular_eager_prefetching.c				\
+	sched_policies/modular_prio.c				\
+	sched_policies/modular_prio_prefetching.c				\
+	sched_policies/modular_random.c				\
+	sched_policies/modular_random_prefetching.c			\
+	sched_policies/modular_heft.c				\
+	sched_policies/modular_heft2.c
 
 
 if STARPU_HAVE_LEVELDB

src/common/fxt.h

@@ -153,6 +153,9 @@
 #define	_STARPU_FUT_START_WRITEBACK	0x5158
 #define	_STARPU_FUT_END_WRITEBACK	0x5159
 
+#define _STARPU_FUT_SCHED_COMPONENT_PUSH_PRIO 	0x515a
+#define _STARPU_FUT_SCHED_COMPONENT_POP_PRIO 	0x515b
+
 #define	_STARPU_FUT_HYPERVISOR_BEGIN    0x5160
 #define	_STARPU_FUT_HYPERVISOR_END	0x5161
 
@@ -803,6 +806,12 @@ do {										\
 #define _STARPU_TRACE_END_UNPARTITION(handle, memnode)		\
 	FUT_DO_PROBE3(_STARPU_FUT_END_UNPARTITION, memnode, _starpu_gettid(), handle);
 
+#define _STARPU_TRACE_SCHED_COMPONENT_PUSH_PRIO(workerid, ntasks, exp_len)		\
+	FUT_DO_PROBE4(_STARPU_FUT_SCHED_COMPONENT_PUSH_PRIO, _starpu_gettid(), workerid, ntasks, exp_len);
+
+#define _STARPU_TRACE_SCHED_COMPONENT_POP_PRIO(workerid, ntasks, exp_len)		\
+	FUT_DO_PROBE4(_STARPU_FUT_SCHED_COMPONENT_POP_PRIO, _starpu_gettid(), workerid, ntasks, exp_len);
+
 #else // !STARPU_USE_FXT
 
 /* Dummy macros in case FxT is disabled */
@@ -885,6 +894,8 @@ do {										\
 #define _STARPU_TRACE_DATA_LOAD(workerid,size)		do {} while(0)
 #define _STARPU_TRACE_START_UNPARTITION(handle, memnode)	do {} while(0)
 #define _STARPU_TRACE_END_UNPARTITION(handle, memnode)		do {} while(0)
+#define _STARPU_TRACE_SCHED_COMPONENT_PUSH_PRIO(workerid, ntasks, exp_len)	do {} while(0)
+#define _STARPU_TRACE_SCHED_COMPONENT_POP_PRIO(workerid, ntasks, exp_len)	do {} while(0)
 #define _STARPU_TRACE_HYPERVISOR_BEGIN()        do {} while(0)
 #define _STARPU_TRACE_HYPERVISOR_END()                  do {} while(0)
 

src/core/combined_workers.c

@@ -82,7 +82,7 @@ int starpu_combined_worker_assign_workerid(int nworkers, int workerid_array[])
 	}
 
 	/* Get an id for that combined worker. Note that this is not thread
-	 * safe because thhis method should only be called when the scheduler
+	 * safe because this method should only be called when the scheduler
 	 * is being initialized. */
 	new_workerid = basic_worker_count + combined_worker_id;
 	config->topology.ncombinedworkers++;
@@ -95,6 +95,8 @@ int starpu_combined_worker_assign_workerid(int nworkers, int workerid_array[])
 	}
 	fprintf(stderr, "into worker %d\n", new_workerid);
 #endif
+	for(i = 0; i < nworkers; i++)
+		_starpu_get_worker_struct(workerid_array[i])->combined_workerid = new_workerid;
 
 	struct _starpu_combined_worker *combined_worker =
 		&config->combined_workers[combined_worker_id];

src/core/errorcheck.c

@@ -1,6 +1,6 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2009, 2010  Université de Bordeaux
+ * Copyright (C) 2009, 2010, 2014  Université de Bordeaux 1
  * Copyright (C) 2010, 2011  Centre National de la Recherche Scientifique
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -18,6 +18,16 @@
 #include <core/errorcheck.h>
 #include <core/workers.h>
 
+void _starpu_set_worker_status(struct _starpu_worker *worker, enum _starpu_worker_status st)
+{
+	starpu_pthread_mutex_t *sched_mutex;
+	starpu_pthread_cond_t *sched_cond;
+	starpu_worker_get_sched_condition(worker->workerid, &sched_mutex, &sched_cond);
+	STARPU_PTHREAD_MUTEX_LOCK(sched_mutex);
+	worker->status = st;
+	STARPU_PTHREAD_MUTEX_UNLOCK(sched_mutex);
+}
+
 void _starpu_set_local_worker_status(enum _starpu_worker_status st)
 {
 	struct _starpu_worker *worker = _starpu_get_local_worker_key();
@@ -26,7 +36,7 @@ void _starpu_set_local_worker_status(enum _starpu_worker_status st)
 	 * thereforce outside a worker), for instance if we are executing the
 	 * callback function of a task with a "NULL" codelet. */
 	if (worker)
-		worker->status = st;
+		_starpu_set_worker_status(worker, st);
 }
 
 enum _starpu_worker_status _starpu_get_local_worker_status(void)

src/core/errorcheck.h

@@ -42,9 +42,11 @@ enum _starpu_worker_status
 	STATUS_WAKING_UP
 };
 
+struct _starpu_worker;
 /* Specify what the local worker is currently doing (eg. executing a callback).
  * This permits to detect if this is legal to do a blocking call for instance.
  * */
+void _starpu_set_worker_status(struct _starpu_worker *worker, enum _starpu_worker_status st);
 void _starpu_set_local_worker_status(enum _starpu_worker_status st);
 
 /* Indicate what type of operation the worker is currently doing. */

src/core/sched_policy.c

@@ -35,6 +35,17 @@ int starpu_get_prefetch_flag(void)
 
 static struct starpu_sched_policy *predefined_policies[] =
 {
+	&_starpu_sched_modular_eager_policy,
+	&_starpu_sched_modular_eager_prefetching_policy,
+	&_starpu_sched_modular_prio_policy,
+	&_starpu_sched_modular_prio_prefetching_policy,
+	&_starpu_sched_modular_random_policy,
+	&_starpu_sched_modular_random_prio_policy,
+	&_starpu_sched_modular_random_prefetching_policy,
+	&_starpu_sched_modular_random_prio_prefetching_policy,
+	//&_starpu_sched_modular_ws_policy,
+	&_starpu_sched_modular_heft_policy,
+	&_starpu_sched_modular_heft2_policy,
 	&_starpu_sched_eager_policy,
 	&_starpu_sched_prio_policy,
 	&_starpu_sched_random_policy,

src/core/sched_policy.h

@@ -69,6 +69,15 @@ extern struct starpu_sched_policy _starpu_sched_dmda_sorted_policy;
 extern struct starpu_sched_policy _starpu_sched_eager_policy;
 extern struct starpu_sched_policy _starpu_sched_parallel_heft_policy;
 extern struct starpu_sched_policy _starpu_sched_peager_policy;
-
-
+extern struct starpu_sched_policy _starpu_sched_modular_eager_policy;
+extern struct starpu_sched_policy _starpu_sched_modular_eager_prefetching_policy;
+extern struct starpu_sched_policy _starpu_sched_modular_prio_policy;
+extern struct starpu_sched_policy _starpu_sched_modular_prio_prefetching_policy;
+extern struct starpu_sched_policy _starpu_sched_modular_random_policy;
+extern struct starpu_sched_policy _starpu_sched_modular_random_prio_policy;
+extern struct starpu_sched_policy _starpu_sched_modular_random_prefetching_policy;
+extern struct starpu_sched_policy _starpu_sched_modular_random_prio_prefetching_policy;
+//extern struct starpu_sched_policy _starpu_sched_modular_ws_policy;
+extern struct starpu_sched_policy _starpu_sched_modular_heft_policy;
+extern struct starpu_sched_policy _starpu_sched_modular_heft2_policy;
 #endif // __SCHED_POLICY_H__

src/core/task.c

@@ -82,6 +82,7 @@ void starpu_task_init(struct starpu_task *task)
 	task->flops = 0.0;
 
 	task->scheduled = 0;
+	task->prefetched = 0;
 
 	task->dyn_handles = NULL;
 	task->dyn_interfaces = NULL;

src/core/workers.c

@@ -30,6 +30,7 @@
 #include <datawizard/malloc.h>
 #include <profiling/profiling.h>
 #include <starpu_task_list.h>
+#include <sched_policies/sched_component.h>
 #include <drivers/mp_common/sink_common.h>
 #include <drivers/scc/driver_scc_common.h>
 
@@ -1447,6 +1448,7 @@ void starpu_shutdown(void)
 	}
 
 	_starpu_delete_all_sched_ctxs();
+	_starpu_sched_component_workers_destroy();
 
 	_starpu_disk_unregister();
 #ifdef STARPU_HAVE_HWLOC

src/datawizard/coherency.c

@@ -729,6 +729,7 @@ static void _starpu_set_data_requested_flag_if_needed(starpu_data_handle_t handl
 
 int starpu_prefetch_task_input_on_node(struct starpu_task *task, unsigned node)
 {
+	STARPU_ASSERT(!task->prefetched);
 	unsigned nbuffers = STARPU_TASK_GET_NBUFFERS(task);
 	unsigned index;
 

src/debug/traces/starpu_fxt.c

@@ -2048,6 +2048,12 @@ void starpu_fxt_parse_new_file(char *filename_in, struct starpu_fxt_options *opt
 			case _STARPU_FUT_MEMORY_FULL:
 				break;
 
+			case _STARPU_FUT_SCHED_COMPONENT_POP_PRIO:
+				break;
+
+			case _STARPU_FUT_SCHED_COMPONENT_PUSH_PRIO:
+				break;
+
 			case _STARPU_FUT_HYPERVISOR_BEGIN:
 				handle_hypervisor_begin(&ev, options);
 				break;

src/drivers/driver_common/driver_common.c

@@ -52,7 +52,7 @@ void _starpu_driver_start_job(struct _starpu_worker *worker, struct _starpu_job
 	if (j->task_size == 1)
 		_starpu_sched_pre_exec_hook(task);
 
-	worker->status = STATUS_EXECUTING;
+	_starpu_set_worker_status(worker, STATUS_EXECUTING);
 	task->status = STARPU_TASK_RUNNING;
 
 	if (rank == 0)
@@ -103,7 +103,7 @@ void _starpu_driver_end_job(struct _starpu_worker *worker, struct _starpu_job *j
 	if (starpu_top)
 		_starpu_top_task_ended(task,workerid,codelet_end);
 
-	worker->status = STATUS_UNKNOWN;
+	_starpu_set_worker_status(worker, STATUS_UNKNOWN);
 }
 void _starpu_driver_update_job_feedback(struct _starpu_job *j, struct _starpu_worker *worker,
 					struct starpu_perfmodel_arch* perf_arch,

src/sched_policies/README

@@ -0,0 +1,76 @@
+# StarPU --- Runtime system for heterogeneous multicore architectures.
+#
+# Copyright (C) 2013 Simon Archipoff
+#
+# StarPU is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 2.1 of the License, or (at
+# your option) any later version.
+#
+# StarPU is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See the GNU Lesser General Public License in COPYING.LGPL for more details.
+
+
+
+Mutex policy
+
+scheduler have to be protected when the hypervisor is modifying it.
+there is a mutex in struct starpu_sched_tree wich should be taken by
+the application to push a task
+and one mutex per worker wich should be taken by workers when they pop
+or push a task.
+The hypervisor must take all of them to modifying the scheduler.
+
+
+
+Creation/Destruction
+
+All the struct starpu_sched_component * starpu_sched_component_foo_create()
+function return a initialized struct starpu_sched_component.
+
+The void starpu_sched_component_destroy(struct starpu_sched_component * component)
+function call component->deinit_data(component) to free data allocated during
+creation
+
+Workers components are particulars, there is no creation function, only
+accessor to guaranty uniqueness of worker components. worker_component->workers and
+worker_component->workers_in_ctx should not be modified.
+
+
+
+Add/Remove workers
+
+I see 2 way for adding/removing workers of the scheduler
+The hypervisor block all the scheduling and modify the scheduler in
+the way it wants, and then update all component->workers_in_ctx bitmaps, and
+all component->push_task should respect it.
+
+And the second one may be done in an atomic way. The struct
+starpu_sched_tree hold a struct starpu_bitmap * that represent
+available workers in context. All component can make a call to struct starpu_bitmap
+* starpu_sched_component_get_worker_mask(unsigned sched_ctx_id) to see
+where they can push a task according to available workers.
+But with this way we have a problem for component->estimated_end, in case
+of fifo, we have to know how many workers are available to the fifo
+component. We also have a problem for shared object. The first way seems to
+be better.
+
+
+Hierarchical construction
+
+Bugs everywhere, works only in simple and particulars cases.
+Its difficult to guess where we should plug accelerators because we cant rely on
+hwloc topology. Hierarchical heft seems to work on simple machines with numa components
+and GPUs
+this fail if hwloc_socket_composed_sched_component or hwloc_cache_composed_sched_component is not
+NULL
+
+
+Various things
+
+In several place realloc is used (in prio_deque and for
+starpu_sched_component_add_child), because we should not have a lot
+different priority level nor adding too many childs.

src/sched_policies/component_best_implementation.c

@@ -0,0 +1,113 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2013  INRIA
+ * Copyright (C) 2013  Simon Archipoff
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <starpu_sched_component.h>
+#include <starpu_scheduler.h>
+
+#include <float.h>
+
+/* return true if workerid can execute task, and fill task->predicted and task->predicted_transfer
+ *  according to best implementation predictions
+ */
+static int find_best_impl(struct starpu_task * task, int workerid)
+{
+	double len = DBL_MAX;
+	int best_impl = -1;
+	int impl;
+	for(impl = 0; impl < STARPU_MAXIMPLEMENTATIONS; impl++)
+	{
+		if(starpu_worker_can_execute_task(workerid, task, impl))
+		{
+			struct starpu_perfmodel_arch* archtype = starpu_worker_get_perf_archtype(workerid);
+			double d = starpu_task_expected_length(task, archtype, impl);
+			if(isnan(d))
+			{
+				best_impl = impl;
+				len = 0.0;
+				break;
+			}
+			if(d < len)
+			{
+				len = d;
+				best_impl = impl;
+			}
+		}
+	}
+	if(best_impl == -1)
+		return 0;
+
+	int memory_node = starpu_worker_get_memory_node(workerid);
+	task->predicted = len;
+	task->predicted_transfer = starpu_task_expected_data_transfer_time(memory_node, task);
+	starpu_task_set_implementation(task, best_impl);
+	return 1;
+}
+
+
+/* set implementation, task->predicted and task->predicted_transfer with the first worker of workers that can execute that task
+ * or have to be calibrated
+ */
+static void select_best_implementation_and_set_preds(struct starpu_bitmap * workers, struct starpu_task * task)
+{
+	int workerid;
+	for(workerid = starpu_bitmap_first(workers);
+	    -1 != workerid;
+	    workerid = starpu_bitmap_next(workers, workerid))
+		if(find_best_impl(task, workerid))
+			break;
+}
+
+static int best_implementation_push_task(struct starpu_sched_component * component, struct starpu_task * task)
+{
+	STARPU_ASSERT(component->nchildren == 1);
+	select_best_implementation_and_set_preds(component->workers_in_ctx, task);
+	return component->children[0]->push_task(component->children[0],task);
+}
+
+int starpu_sched_component_is_best_implementation(struct starpu_sched_component * component)
+{
+	return component->push_task == best_implementation_push_task;
+}
+
+static struct starpu_task * best_implementation_pull_task(struct starpu_sched_component * component)
+{
+	struct starpu_task * task = NULL;
+	int i;
+	for(i=0; i < component->nparents; i++)
+	{
+		if(component->parents[i] == NULL)
+			continue;
+		else
+		{
+			task = component->parents[i]->pull_task(component->parents[i]);
+			if(task)
+				break;
+		}
+	}
+	if(task)
+		/* this worker can execute this task as it was returned by a pop*/
+		(void)find_best_impl(task, starpu_worker_get_id());
+	return task;
+}
+
+struct starpu_sched_component * starpu_sched_component_best_implementation_create(void * ARG STARPU_ATTRIBUTE_UNUSED)
+{
+	struct starpu_sched_component * component = starpu_sched_component_create();
+	component->push_task = best_implementation_push_task;
+	component->pull_task = best_implementation_pull_task;
+	return component;
+}

src/sched_policies/component_composed.c

@@ -0,0 +1,233 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2013  INRIA
+ * Copyright (C) 2013  Simon Archipoff
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <starpu_sched_component.h>
+#include <common/list.h>
+
+
+/* a composed component is parametred by a list of pair
+ * (create_component_function(arg), arg)
+ */
+LIST_TYPE(fun_create_component,
+	  struct starpu_sched_component *(*create_component)(void * arg);
+	  void * arg;
+);
+
+
+struct starpu_sched_component_composed_recipe
+{
+	struct fun_create_component_list * list;
+};
+
+
+struct starpu_sched_component_composed_recipe * starpu_sched_component_create_recipe(void)
+{
+	struct starpu_sched_component_composed_recipe * recipe = malloc(sizeof(*recipe));
+	recipe->list = fun_create_component_list_new();
+	return recipe;
+}
+
+void starpu_sched_recipe_add_component(struct starpu_sched_component_composed_recipe * recipe,
+				  struct starpu_sched_component *(*create_component)(void * arg),
+				  void * arg)
+{
+	struct fun_create_component * e = fun_create_component_new();
+	e->create_component = create_component;
+	e->arg = arg;
+	fun_create_component_list_push_back(recipe->list, e);
+}
+struct starpu_sched_component_composed_recipe * starpu_sched_component_create_recipe_singleton(struct starpu_sched_component *(*create_component)(void * arg),
+										      void * arg)
+{
+	struct starpu_sched_component_composed_recipe * r = starpu_sched_component_create_recipe();
+	starpu_sched_recipe_add_component(r, create_component, arg);
+	return r;
+}
+void starpu_destroy_composed_sched_component_recipe(struct starpu_sched_component_composed_recipe * recipe)
+{
+	if(!recipe)
+		return;
+	while(!fun_create_component_list_empty(recipe->list))
+		fun_create_component_delete(fun_create_component_list_pop_back(recipe->list));
+	fun_create_component_list_delete(recipe->list);
+	free(recipe);
+}
+
+
+
+struct composed_component
+{
+	struct starpu_sched_component *top,*bottom;
+};
+
+/* this function actualy build the composed component data by changing the list of
+ * (component_create_fun, arg_create_fun) into a tree where all components have 1 childs
+ */
+struct composed_component create_composed_component(struct starpu_sched_component_composed_recipe * recipe
+#ifdef STARPU_HAVE_HWLOC
+					  ,hwloc_obj_t obj
+#endif
+	)
+{
+	struct composed_component c;
+	STARPU_ASSERT(recipe);
+
+	struct fun_create_component_list * list = recipe->list;
+	struct fun_create_component * i = fun_create_component_list_begin(list);
+	STARPU_ASSERT(i);
+	STARPU_ASSERT(i->create_component(i->arg));
+	c.top = c.bottom = i->create_component(i->arg);
+#ifdef STARPU_HAVE_HWLOC
+	c.top->obj = obj;
+#endif
+	for(i  = fun_create_component_list_next(i);
+	    i != fun_create_component_list_end(list);
+	    i  = fun_create_component_list_next(i))
+	{
+		STARPU_ASSERT(i->create_component(i->arg));
+		struct starpu_sched_component * component = i->create_component(i->arg);
+#ifdef STARPU_HAVE_HWLOC
+		component->obj = obj;
+#endif
+		c.bottom->add_child(c.bottom, component);
+
+		/* we want to be able to traverse scheduler bottom up for all sched ctxs
+		 * when a worker call pop()
+		 */
+		unsigned j;
+		for(j = 0; j < STARPU_NMAX_SCHED_CTXS; j++)
+			component->add_parent(component, c.bottom);
+		c.bottom = component;
+	}
+	STARPU_ASSERT(!starpu_sched_component_is_worker(c.bottom));
+	return c;
+}
+
+
+static int composed_component_push_task(struct starpu_sched_component * component, struct starpu_task * task)
+{
+	struct composed_component *c = component->data;
+	return c->top->push_task(c->top,task);
+}
+struct starpu_task * composed_component_pull_task(struct starpu_sched_component *component)
+{
+	struct composed_component *c = component->data;
+	struct starpu_task * task = NULL;
+	
+	task = c->bottom->pull_task(c->bottom);
+	if(task)
+		return task;
+
+	int i;
+	for(i=0; i < component->nparents; i++)
+	{
+		if(component->parents[i] == NULL)
+			continue;
+		else
+		{
+			task = component->parents[i]->pull_task(component->parents[i]);
+			if(task)
+				break;
+		}
+	}
+	return task;
+}
+
+double composed_component_estimated_load(struct starpu_sched_component * component)
+{
+	struct composed_component * c = component->data;
+	return c->top->estimated_load(c->top);
+}
+
+static void composed_component_add_child(struct starpu_sched_component * component, struct starpu_sched_component * child)
+{
+	struct composed_component * c = component->data;
+	component->add_child(component, child);
+	c->bottom->add_child(c->bottom, child);
+}
+static void composed_component_remove_child(struct starpu_sched_component * component, struct starpu_sched_component * child)
+{
+	struct composed_component * c = component->data;
+	component->remove_child(component, child);
+	c->bottom->remove_child(c->bottom, child);
+}
+
+static void composed_component_notify_change_workers(struct starpu_sched_component * component)
+{
+	struct composed_component * c = component->data;
+	struct starpu_bitmap * workers = component->workers;
+	struct starpu_bitmap * workers_in_ctx = component->workers_in_ctx;
+	struct starpu_sched_component * n;
+	for(n = c->top; ;n = n->children[0])
+	{
+		starpu_bitmap_unset_all(n->workers);
+		starpu_bitmap_or(n->workers, workers);
+
+		starpu_bitmap_unset_all(n->workers_in_ctx);
+		starpu_bitmap_or(n->workers_in_ctx, workers_in_ctx);
+
+		n->properties = component->properties;
+		if(n == c->bottom)
+			break;
+	}
+}
+
+void composed_component_deinit_data(struct starpu_sched_component * _component)
+{
+	struct composed_component *c = _component->data;
+	c->bottom->children = NULL;
+	c->bottom->nchildren = 0;
+	struct starpu_sched_component * component = c->top;
+	struct starpu_sched_component * next = NULL;
+	do
+	{
+		component->workers = NULL;
+		next = component->children ? component->children[0] : NULL;
+		starpu_sched_component_destroy(component);
+	}while(next);
+	free(c);
+	_component->data = NULL;
+}
+
+struct starpu_sched_component * starpu_sched_component_composed_component_create(struct starpu_sched_component_composed_recipe * recipe)
+{
+	STARPU_ASSERT(!fun_create_component_list_empty(recipe->list));
+	struct fun_create_component_list * l = recipe->list;
+	if(l->_head == l->_tail)
+		return l->_head->create_component(l->_head->arg);
+	struct starpu_sched_component * component = starpu_sched_component_create();
+
+	struct composed_component * c = malloc(sizeof(struct composed_component));
+	*c = create_composed_component(recipe
+#ifdef STARPU_HAVE_HWLOC
+				  ,component->obj
+#endif
+);
+	c->bottom->nchildren = component->nchildren;
+	c->bottom->children = component->children;
+	c->bottom->nparents = component->nparents;
+	c->bottom->parents = component->parents;
+
+	component->data = c;
+	component->push_task = composed_component_push_task;
+	component->pull_task = composed_component_pull_task;
+	component->estimated_load = composed_component_estimated_load;
+	component->add_child = composed_component_add_child;
+	component->remove_child = composed_component_remove_child;
+	component->notify_change_workers = composed_component_notify_change_workers;
+	return component;
+}

src/sched_policies/component_eager.c

@@ -0,0 +1,73 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2013  INRIA
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <starpu_sched_component.h>
+#include <starpu_scheduler.h>
+
+static int eager_push_task(struct starpu_sched_component * component, struct starpu_task * task)
+{
+	STARPU_ASSERT(component && task && starpu_sched_component_is_eager(component));
+	STARPU_ASSERT(starpu_sched_component_can_execute_task(component,task));
+	
+	int workerid;
+	for(workerid = starpu_bitmap_first(component->workers_in_ctx);
+	    workerid != -1;
+	    workerid = starpu_bitmap_next(component->workers_in_ctx, workerid))
+	{
+		int nimpl;
+		for(nimpl = 0; nimpl < STARPU_MAXIMPLEMENTATIONS; nimpl++)
+		{
+			if(starpu_worker_can_execute_task(workerid,task,nimpl)
+			   || starpu_combined_worker_can_execute_task(workerid, task, nimpl))
+			{
+				int i;
+				for (i = 0; i < component->nchildren; i++)
+				{
+					int idworker;
+					for(idworker = starpu_bitmap_first(component->children[i]->workers);
+						idworker != -1;
+						idworker = starpu_bitmap_next(component->children[i]->workers, idworker))
+					{
+						if (idworker == workerid)
+						{
+							if(starpu_sched_component_is_worker(component->children[i]))
+							{
+								component->children[i]->can_pull(component->children[i]);
+								return 1;
+							}
+							else
+								return component->children[i]->push_task(component->children[i],task);
+						}
+					}
+				}
+			}
+		}
+	}
+	return 1;
+}
+
+int starpu_sched_component_is_eager(struct starpu_sched_component * component)
+{
+	return component->push_task == eager_push_task;
+}
+
+struct starpu_sched_component * starpu_sched_component_eager_create(void * ARG STARPU_ATTRIBUTE_UNUSED)
+{
+	struct starpu_sched_component * component = starpu_sched_component_create();
+	component->push_task = eager_push_task;
+
+	return component;
+}

src/sched_policies/component_eager_calibration.c

@@ -0,0 +1,86 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2013  INRIA
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <starpu_sched_component.h>
+#include <starpu_scheduler.h>
+
+static int eager_calibration_push_task(struct starpu_sched_component * component, struct starpu_task * task)
+{
+	STARPU_ASSERT(component && task && starpu_sched_component_is_eager_calibration(component));
+	STARPU_ASSERT(starpu_sched_component_can_execute_task(component,task));
+	
+	starpu_task_bundle_t bundle = task->bundle;
+
+	int workerid;
+	for(workerid = starpu_bitmap_first(component->workers_in_ctx);
+	    workerid != -1;
+	    workerid = starpu_bitmap_next(component->workers_in_ctx, workerid))
+	{
+		struct starpu_perfmodel_arch* archtype = starpu_worker_get_perf_archtype(workerid);
+		int nimpl;
+		for(nimpl = 0; nimpl < STARPU_MAXIMPLEMENTATIONS; nimpl++)
+		{
+			if(starpu_worker_can_execute_task(workerid,task,nimpl)
+			   || starpu_combined_worker_can_execute_task(workerid, task, nimpl))
+			{
+				double d;
+
+				if(bundle)
+					d = starpu_task_bundle_expected_length(bundle, archtype, nimpl);
+				else
+					d = starpu_task_expected_length(task, archtype, nimpl);
+
+				if(isnan(d))
+				{
+					int i;
+					for (i = 0; i < component->nchildren; i++)
+					{
+						int idworker;
+						for(idworker = starpu_bitmap_first(component->children[i]->workers);
+							idworker != -1;
+							idworker = starpu_bitmap_next(component->children[i]->workers, idworker))
+						{
+							if (idworker == workerid)
+							{
+								if(starpu_sched_component_is_worker(component->children[i]))
+								{
+									component->children[i]->can_pull(component->children[i]);
+									return 1;
+								}
+								else
+									return component->children[i]->push_task(component->children[i],task);
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+	return 1;
+}
+
+int starpu_sched_component_is_eager_calibration(struct starpu_sched_component * component)
+{
+	return component->push_task == eager_calibration_push_task;
+}
+
+struct starpu_sched_component * starpu_sched_component_eager_calibration_create(void * ARG STARPU_ATTRIBUTE_UNUSED)
+{
+	struct starpu_sched_component * component = starpu_sched_component_create();
+	component->push_task = eager_calibration_push_task;
+
+	return component;
+}

src/sched_policies/component_fifo.c

@@ -0,0 +1,266 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2013  INRIA
+ * Copyright (C) 2013  Simon Archipoff
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <starpu_sched_component.h>
+#include <starpu_scheduler.h>
+
+#include "fifo_queues.h"
+#include "sched_component.h"
+
+struct _starpu_fifo_data
+{
+	struct _starpu_fifo_taskq * fifo;
+	starpu_pthread_mutex_t mutex;
+	unsigned ntasks_threshold;
+	double exp_len_threshold;
+};
+
+static void fifo_component_deinit_data(struct starpu_sched_component * component)
+{
+	STARPU_ASSERT(component && component->data);
+	struct _starpu_fifo_data * f = component->data;
+	_starpu_destroy_fifo(f->fifo);
+	STARPU_PTHREAD_MUTEX_DESTROY(&f->mutex);
+	free(f);
+}
+
+static double fifo_estimated_end(struct starpu_sched_component * component)
+{
+	STARPU_ASSERT(component && component->data);
+	struct _starpu_fifo_data * data = component->data;
+	struct _starpu_fifo_taskq * fifo = data->fifo;
+	starpu_pthread_mutex_t * mutex = &data->mutex;
+	int card = starpu_bitmap_cardinal(component->workers_in_ctx);
+	STARPU_ASSERT(card != 0);
+	STARPU_PTHREAD_MUTEX_LOCK(mutex);
+	fifo->exp_start = STARPU_MAX(fifo->exp_start, starpu_timing_now());
+	double estimated_end = fifo->exp_start + fifo->exp_len / card;
+	STARPU_PTHREAD_MUTEX_UNLOCK(mutex);
+
+	return estimated_end;
+}
+
+static double fifo_estimated_load(struct starpu_sched_component * component)
+{
+	STARPU_ASSERT(component && component->data);
+	STARPU_ASSERT(starpu_bitmap_cardinal(component->workers_in_ctx) != 0);
+	struct _starpu_fifo_data * data = component->data;
+	struct _starpu_fifo_taskq * fifo = data->fifo;
+	starpu_pthread_mutex_t * mutex = &data->mutex;
+	double relative_speedup = 0.0;
+	double load;
+	if(STARPU_SCHED_COMPONENT_IS_HOMOGENEOUS(component))
+	{		
+		int first_worker = starpu_bitmap_first(component->workers_in_ctx);
+		relative_speedup = starpu_worker_get_relative_speedup(starpu_worker_get_perf_archtype(first_worker));
+		STARPU_PTHREAD_MUTEX_LOCK(mutex);
+		load = fifo->ntasks / relative_speedup;
+		STARPU_PTHREAD_MUTEX_UNLOCK(mutex);
+		return load;
+	}
+	else
+	{
+		int i;
+		for(i = starpu_bitmap_first(component->workers_in_ctx);
+		    i != -1;
+		    i = starpu_bitmap_next(component->workers_in_ctx, i))
+			relative_speedup += starpu_worker_get_relative_speedup(starpu_worker_get_perf_archtype(i));
+		relative_speedup /= starpu_bitmap_cardinal(component->workers_in_ctx);
+		STARPU_ASSERT(!_STARPU_IS_ZERO(relative_speedup));
+		STARPU_PTHREAD_MUTEX_LOCK(mutex);
+		load = fifo->ntasks / relative_speedup;
+		STARPU_PTHREAD_MUTEX_UNLOCK(mutex);
+	}
+	int i;
+	for(i = 0; i < component->nchildren; i++)
+	{
+		struct starpu_sched_component * c = component->children[i];
+		load += c->estimated_load(c);
+	}
+	return load;
+}
+
+static int fifo_push_local_task(struct starpu_sched_component * component, struct starpu_task * task, unsigned is_pushback)
+{
+	STARPU_ASSERT(component && component->data && task);
+	STARPU_ASSERT(starpu_sched_component_can_execute_task(component,task));
+	struct _starpu_fifo_data * data = component->data;
+	struct _starpu_fifo_taskq * fifo = data->fifo;
+	starpu_pthread_mutex_t * mutex = &data->mutex;
+	int ret = 0;
+
+	STARPU_PTHREAD_MUTEX_LOCK(mutex);
+	double exp_len;
+	if(!isnan(task->predicted))
+		exp_len = fifo->exp_len + task->predicted;
+	else
+		exp_len = fifo->exp_len;
+
+	if((data->ntasks_threshold != 0) && (data->exp_len_threshold != 0.0) && 
+			((fifo->ntasks >= data->ntasks_threshold) || (exp_len >= data->exp_len_threshold)))
+	{
+		static int warned;
+		if(task->predicted > data->exp_len_threshold && !warned)
+		{
+			_STARPU_DISP("Warning : a predicted task length (%lf) exceeds the expected length threshold (%lf) of a prio component queue, you should reconsider the value of this threshold. This message will not be printed again for further thresholds exceeding.\n",task->predicted,data->exp_len_threshold);
+			warned = 1;
+		}
+		STARPU_ASSERT(!is_pushback);
+		ret = 1;
+		STARPU_PTHREAD_MUTEX_UNLOCK(mutex);
+	}
+	else
+	{
+		if(is_pushback)
+			ret = _starpu_fifo_push_back_task(fifo,task);
+		else
+		{
+			ret = _starpu_fifo_push_task(fifo,task);
+			starpu_sched_component_prefetch_on_node(component, task);
+		}
+
+		if(!isnan(task->predicted))
+		{
+			fifo->exp_len = exp_len;
+			fifo->exp_end = fifo->exp_start + fifo->exp_len;
+		}
+		STARPU_ASSERT(!isnan(fifo->exp_end));
+		STARPU_ASSERT(!isnan(fifo->exp_len));
+		STARPU_ASSERT(!isnan(fifo->exp_start));
+		
+		if(!is_pushback)
+			component->can_pull(component);
+		STARPU_PTHREAD_MUTEX_UNLOCK(mutex);
+	}
+	
+	return ret;
+}
+
+static int fifo_push_task(struct starpu_sched_component * component, struct starpu_task * task)
+{
+	return fifo_push_local_task(component, task, 0);
+}
+
+static struct starpu_task * fifo_pull_task(struct starpu_sched_component * component)
+{
+	STARPU_ASSERT(component && component->data);
+	struct _starpu_fifo_data * data = component->data;
+	struct _starpu_fifo_taskq * fifo = data->fifo;
+	starpu_pthread_mutex_t * mutex = &data->mutex;
+	STARPU_PTHREAD_MUTEX_LOCK(mutex);
+	struct starpu_task * task = _starpu_fifo_pop_task(fifo, starpu_worker_get_id());
+	if(task)
+	{
+		if(!isnan(task->predicted))
+		{
+			fifo->exp_start = starpu_timing_now() + task->predicted;
+			fifo->exp_len -= task->predicted;
+		}
+		fifo->exp_end = fifo->exp_start + fifo->exp_len;
+		if(fifo->ntasks == 0)
+			fifo->exp_len = 0.0;
+	}
+	STARPU_ASSERT(!isnan(fifo->exp_end));
+	STARPU_ASSERT(!isnan(fifo->exp_len));
+	STARPU_ASSERT(!isnan(fifo->exp_start));
+	STARPU_PTHREAD_MUTEX_UNLOCK(mutex);
+
+	// When a pop is called, a can_push is called for pushing tasks onto
+	// the empty place of the queue left by the popped task.
+	int i,ret;
+	for(i=0; i < component->nparents; i++)
+	{
+		if(component->parents[i] == NULL)
+			continue;
+		else
+		{
+			ret = component->parents[i]->can_push(component->parents[i]);
+			if(ret)
+				break;
+		}
+	}
+	
+	if(task)
+		return task;
+
+	return NULL;
+}
+
+/* When a can_push is caught by this function, we try to pop and push
+ * tasks from our local queue as much as possible, until a
+ * push fails, which means that the worker fifo_components are
+ * currently "full".
+ */
+static int fifo_can_push(struct starpu_sched_component * component)
+{
+	STARPU_ASSERT(component && starpu_sched_component_is_fifo(component));
+	int ret = 0;
+	int res = 0;
+
+	STARPU_ASSERT(component->nchildren == 1);
+	struct starpu_sched_component * child = component->children[0];
+
+	struct starpu_task * task = component->pull_task(component);
+	if(task)
+		ret = child->push_task(child,task);	
+	while(task && !ret) 
+	{
+		if(!res)
+			res = 1;
+
+		task = component->pull_task(component);
+		if(task)
+			ret = child->push_task(child,task);	
+	} 
+	if(task && ret)
+		fifo_push_local_task(component,task,1); 
+
+	return res;
+}
+
+int starpu_sched_component_is_fifo(struct starpu_sched_component * component)
+{
+	return component->push_task == fifo_push_task;
+}
+
+struct starpu_sched_component * starpu_sched_component_fifo_create(struct starpu_fifo_data * params)
+{
+	struct starpu_sched_component * component = starpu_sched_component_create();
+	struct _starpu_fifo_data * data = malloc(sizeof(*data));
+	data->fifo = _starpu_create_fifo();
+	STARPU_PTHREAD_MUTEX_INIT(&data->mutex,NULL);
+	component->data = data;
+	component->estimated_end = fifo_estimated_end;
+	component->estimated_load = fifo_estimated_load;
+	component->push_task = fifo_push_task;
+	component->pull_task = fifo_pull_task;
+	component->can_push = fifo_can_push;
+	component->deinit_data = fifo_component_deinit_data;
+
+	if(params)
+	{
+		data->ntasks_threshold=params->ntasks_threshold;
+		data->exp_len_threshold=params->exp_len_threshold;
+	}
+	else
+	{
+		data->ntasks_threshold=0;
+		data->exp_len_threshold=0.0;
+	}
+
+	return component;
+}

src/sched_policies/component_heft.c

@@ -0,0 +1,242 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2013-2014  Université de Bordeaux 1
+ * Copyright (C) 2013  INRIA
+ * Copyright (C) 2013  Simon Archipoff
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+/* HEFT variant which tries to schedule a given number of tasks instead of just
+ * the first of its scheduling window, and actually schedule the task for which
+ * the most benefit is achieved.  */
+
+#include <starpu_sched_component.h>
+#include "prio_deque.h"
+#include "sched_component.h"
+#include <starpu_perfmodel.h>
+#include "helper_mct.h"
+#include <float.h>
+
+#define NTASKS 5
+
+struct _starpu_heft_data
+{
+	struct _starpu_prio_deque prio;
+	starpu_pthread_mutex_t mutex;
+	struct _starpu_mct_data *mct_data;
+};
+
+static int heft_progress_one(struct starpu_sched_component *component)
+{
+	struct _starpu_heft_data * data = component->data;
+	starpu_pthread_mutex_t * mutex = &data->mutex;
+	struct _starpu_prio_deque * prio = &data->prio;
+	struct starpu_task * (tasks[NTASKS]);
+
+	unsigned ntasks, n, i;
+
+	STARPU_PTHREAD_MUTEX_LOCK(mutex);
+	/* Try to look at NTASKS from the queue */
+	for (ntasks = 0; ntasks < NTASKS; ntasks++)
+	{
+		tasks[ntasks] = _starpu_prio_deque_pop_task(prio);
+		if (!tasks[ntasks])
+			break;
+	}
+	STARPU_PTHREAD_MUTEX_UNLOCK(mutex);
+
+	if (!ntasks) {
+		return 1;
+	}
+
+	{
+		struct _starpu_mct_data * d = data->mct_data;
+		struct starpu_sched_component * best_component = NULL;
+
+		/* Estimated task duration for each child */
+		double estimated_lengths[component->nchildren * ntasks];
+		/* Estimated transfer duration for each child */
+		double estimated_transfer_length[component->nchildren * ntasks];
+		/* Estimated transfer+task termination for each child */
+		double estimated_ends_with_task[component->nchildren * ntasks];
+
+		/* Minimum transfer+task termination on all children */
+		double min_exp_end_with_task[ntasks];
+		/* Maximum transfer+task termination on all children */
+		double max_exp_end_with_task[ntasks];
+
+		int suitable_components[component->nchildren * ntasks];
+
+		unsigned nsuitable_components[ntasks];
+
+		/* Estimate durations */
+		for (n = 0; n < ntasks; n++)
+		{
+			int offset = component->nchildren * n;
+
+			min_exp_end_with_task[n] = DBL_MAX;
+			max_exp_end_with_task[n] = 0.0;
+
+			nsuitable_components[n] = starpu_mct_compute_expected_times(component, tasks[n],
+					estimated_lengths + offset,
+					estimated_transfer_length + offset,
+					estimated_ends_with_task + offset,
+					&min_exp_end_with_task[n], &max_exp_end_with_task[n],
+					suitable_components + offset);
+		}
+
+		int best_task = 0;
+		double max_benefit = 0;
+
+		/* Find the task which provides the most computation time benefit */
+		for (n = 1; n < ntasks; n++)
+		{
+			double benefit = max_exp_end_with_task[n] - min_exp_end_with_task[n];
+			if (max_benefit < benefit)
+			{
+				max_benefit = benefit;
+				best_task = n;
+			}
+		}
+
+		double best_fitness = DBL_MAX;
+		int best_icomponent = -1;
+
+		/* Push back the other tasks */
+		STARPU_PTHREAD_MUTEX_LOCK(mutex);
+		for (n = ntasks - 1; n < ntasks; n--)
+			if ((int) n != best_task)
+				_starpu_prio_deque_push_back_task(prio, tasks[n]);
+		STARPU_PTHREAD_MUTEX_UNLOCK(mutex);
+
+		/* And now find out which worker suits best for this task,
+		 * including data transfer */
+		for(i = 0; i < nsuitable_components[best_task]; i++)
+		{
+			int offset = component->nchildren * best_task;
+			int icomponent = suitable_components[offset + i];
+#ifdef STARPU_DEVEL
+#warning FIXME: take power consumption into account
+#endif
+			double tmp = starpu_mct_compute_fitness(d,
+						     estimated_ends_with_task[offset + icomponent],
+						     min_exp_end_with_task[best_task],
+						     max_exp_end_with_task[best_task],
+						     estimated_transfer_length[offset + icomponent],
+						     0.0);
+
+			if(tmp < best_fitness)
+			{
+				best_fitness = tmp;
+				best_icomponent = icomponent;
+			}
+		}
+
+		STARPU_ASSERT(best_icomponent != -1);
+		STARPU_ASSERT(best_task >= 0);
+		best_component = component->children[best_icomponent];
+
+		if(starpu_sched_component_is_worker(best_component))
+		{
+			best_component->can_pull(best_component);
+			return 1;
+		}
+
+		int ret = best_component->push_task(best_component, tasks[best_task]);
+
+		if (ret)
+		{
+			/* Could not push to child actually, push that one back too */
+			STARPU_PTHREAD_MUTEX_LOCK(mutex);
+			_starpu_prio_deque_push_back_task(prio, tasks[best_task]);
+			STARPU_PTHREAD_MUTEX_UNLOCK(mutex);
+			return 1;
+		}
+		else
+			return 0;
+	}
+}
+
+/* Try to push some tasks below */
+static void heft_progress(struct starpu_sched_component *component)
+{
+	STARPU_ASSERT(component && starpu_sched_component_is_heft(component));
+	while (!heft_progress_one(component))
+		;
+}
+
+static int heft_push_task(struct starpu_sched_component * component, struct starpu_task * task)
+{
+	STARPU_ASSERT(component && task && starpu_sched_component_is_heft(component));
+	struct _starpu_heft_data * data = component->data;
+	struct _starpu_prio_deque * prio = &data->prio;
+	starpu_pthread_mutex_t * mutex = &data->mutex;
+
+	STARPU_PTHREAD_MUTEX_LOCK(mutex);
+	_starpu_prio_deque_push_task(prio,task);
+	STARPU_PTHREAD_MUTEX_UNLOCK(mutex);
+
+	heft_progress(component);
+
+	return 0;
+}
+
+static int heft_can_push(struct starpu_sched_component *component)
+{
+	heft_progress(component);
+	int ret, j;
+	for(j=0; j < component->nparents; j++)
+	{
+		if(component->parents[j] == NULL)
+			continue;
+		else
+		{
+			ret = component->parents[j]->can_push(component->parents[j]);
+			if(ret)
+				break;
+		}
+	}
+	return ret;
+}
+
+static void heft_component_deinit_data(struct starpu_sched_component * component)
+{
+	STARPU_ASSERT(starpu_sched_component_is_heft(component));
+	struct _starpu_heft_data * d = component->data;
+	struct _starpu_mct_data * mct_d = d->mct_data;
+	free(mct_d);
+	free(d);
+}
+
+int starpu_sched_component_is_heft(struct starpu_sched_component * component)
+{
+	return component->push_task == heft_push_task;
+}
+
+struct starpu_sched_component * starpu_sched_component_heft_create(struct starpu_mct_data * params)
+{
+	struct starpu_sched_component * component = starpu_sched_component_create();
+	struct _starpu_mct_data *mct_data = starpu_mct_init_parameters(params);
+	struct _starpu_heft_data *data = malloc(sizeof(*data));
+
+	_starpu_prio_deque_init(&data->prio);
+	STARPU_PTHREAD_MUTEX_INIT(&data->mutex,NULL);
+	data->mct_data = mct_data;
+	component->data = data;
+
+	component->push_task = heft_push_task;
+	component->can_push = heft_can_push;
+	component->deinit_data = heft_component_deinit_data;
+
+	return component;
+}

src/sched_policies/component_mct.c

@@ -0,0 +1,129 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2013  Université de Bordeaux 1
+ * Copyright (C) 2013  INRIA
+ * Copyright (C) 2013  Simon Archipoff
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <starpu_sched_component.h>
+#include "sched_component.h"
+#include <starpu_perfmodel.h>
+#include "helper_mct.h"
+#include <float.h>
+
+static int mct_push_task(struct starpu_sched_component * component, struct starpu_task * task)
+{
+	STARPU_ASSERT(component && task && starpu_sched_component_is_mct(component));
+	struct _starpu_mct_data * d = component->data;	
+	struct starpu_sched_component * best_component = NULL;
+
+	/* Estimated task duration for each child */
+	double estimated_lengths[component->nchildren];
+	/* Estimated transfer duration for each child */
+	double estimated_transfer_length[component->nchildren];
+	/* Estimated transfer+task termination for each child */
+	double estimated_ends_with_task[component->nchildren];
+
+	int i;
+	for(i=0; i < component->nchildren; i++)
+	{
+		estimated_lengths[i] = 0.0;
+		estimated_transfer_length[i] = 0.0;
+		estimated_ends_with_task[i] = 0.0;
+
+	}
+	/* Minimum transfer+task termination on all children */
+	double min_exp_end_with_task = DBL_MAX;
+	/* Maximum transfer+task termination on all children */
+	double max_exp_end_with_task = 0.0;
+
+	int suitable_components[component->nchildren];
+	int nsuitable_components = 0;
+
+	nsuitable_components = starpu_mct_compute_expected_times(component, task,
+			estimated_lengths, estimated_transfer_length, estimated_ends_with_task,
+			&min_exp_end_with_task, &max_exp_end_with_task, suitable_components);
+
+	/* If no suitable components were found, it means that the perfmodel of
+	 * the task had been purged since it has been pushed on the mct component.
+	 * We should send a push_fail message to its parent so that it will
+	 * be able to reschedule the task properly. */
+	if(nsuitable_components == 0)
+		return 1;
+
+	double best_fitness = DBL_MAX;
+	int best_icomponent = -1;
+	for(i = 0; i < nsuitable_components; i++)
+	{
+		int icomponent = suitable_components[i];
+#ifdef STARPU_DEVEL
+#warning FIXME: take power consumption into account
+#endif
+		double tmp = starpu_mct_compute_fitness(d,
+					     estimated_ends_with_task[icomponent],
+					     min_exp_end_with_task,
+					     max_exp_end_with_task,
+					     estimated_transfer_length[icomponent],
+					     0.0);
+
+		if(tmp < best_fitness)
+		{
+			best_fitness = tmp;
+			best_icomponent = icomponent;
+		}
+	}
+
+	/* If no best component is found, it means that the perfmodel of
+	 * the task had been purged since it has been pushed on the mct component.
+	 * We should send a push_fail message to its parent so that it will
+	 * be able to reschedule the task properly. */
+	if(best_icomponent == -1)
+		return 1;
+
+	best_component = component->children[best_icomponent];
+
+	if(starpu_sched_component_is_worker(best_component))
+	{
+		best_component->can_pull(best_component);
+		return 1;
+	}
+
+	int ret = best_component->push_task(best_component, task);
+	return ret;
+}
+
+static void mct_component_deinit_data(struct starpu_sched_component * component)
+{
+	STARPU_ASSERT(starpu_sched_component_is_mct(component));
+	struct _starpu_mct_data * d = component->data;
+	free(d);
+}
+
+int starpu_sched_component_is_mct(struct starpu_sched_component * component)
+{
+	return component->push_task == mct_push_task;
+}
+
+struct starpu_sched_component * starpu_sched_component_mct_create(struct starpu_mct_data * params)
+{
+	struct starpu_sched_component * component = starpu_sched_component_create();
+	struct _starpu_mct_data *data = starpu_mct_init_parameters(params);
+
+	component->data = data;
+
+	component->push_task = mct_push_task;
+	component->deinit_data = mct_component_deinit_data;
+
+	return component;
+}

src/sched_policies/component_perfmodel_select.c

@@ -0,0 +1,85 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2013  INRIA
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <starpu_sched_component.h>
+#include <starpu_scheduler.h>
+
+/* The decision component takes care of the scheduling of tasks which are not
+ * calibrated, or tasks which don't have a performance model, because the scheduling
+ * architecture of this scheduler for tasks with no performance model is exactly
+ * the same as the tree-prio scheduler.
+ * Tasks with a perfmodel are pushed to the perfmodel_component, which takes care of the
+ * scheduling of those tasks on the correct worker_component.
+ */
+
+struct _starpu_perfmodel_select_data
+{
+	struct starpu_sched_component * calibrator_component;
+	struct starpu_sched_component * no_perfmodel_component;
+	struct starpu_sched_component * perfmodel_component;
+};
+
+static int perfmodel_select_push_task(struct starpu_sched_component * component, struct starpu_task * task)
+{
+	STARPU_ASSERT(component && component->data && task && starpu_sched_component_is_perfmodel_select(component));
+	STARPU_ASSERT(starpu_sched_component_can_execute_task(component,task));
+
+	struct _starpu_perfmodel_select_data * data = component->data;
+	double length;
+	int can_execute = starpu_sched_component_execute_preds(component,task,&length);
+	
+	if(can_execute)
+	{
+		if(isnan(length))
+			return data->calibrator_component->push_task(data->calibrator_component,task);
+		if(_STARPU_IS_ZERO(length))
+			return data->no_perfmodel_component->push_task(data->no_perfmodel_component,task);
+		return data->perfmodel_component->push_task(data->perfmodel_component,task);
+	}
+	else
+		return 1;
+
+}
+
+static void perfmodel_select_component_deinit_data(struct starpu_sched_component * component)
+{
+	STARPU_ASSERT(component && component->data);
+	struct _starpu_perfmodel_select_data * d = component->data;
+	free(d);
+}
+
+int starpu_sched_component_is_perfmodel_select(struct starpu_sched_component * component)
+{
+	return component->push_task == perfmodel_select_push_task;
+}
+
+struct starpu_sched_component * starpu_sched_component_perfmodel_select_create(struct starpu_perfmodel_select_data * params)
+{
+	STARPU_ASSERT(params);
+	STARPU_ASSERT(params->calibrator_component && params->no_perfmodel_component && params->perfmodel_component);
+	struct starpu_sched_component * component = starpu_sched_component_create();
+
+	struct _starpu_perfmodel_select_data * data = malloc(sizeof(*data));
+	data->calibrator_component = params->calibrator_component;
+	data->no_perfmodel_component = params->no_perfmodel_component;
+	data->perfmodel_component = params->perfmodel_component;
+	
+	component->data = data;
+	component->push_task = perfmodel_select_push_task;
+	component->deinit_data = perfmodel_select_component_deinit_data;
+
+	return component;
+}

src/sched_policies/component_prio.c

@@ -0,0 +1,282 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2013  INRIA
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <starpu_sched_component.h>
+#include <starpu_scheduler.h>
+
+#include "prio_deque.h"
+#include "sched_component.h"
+
+#define STARPU_TRACE_SCHED_COMPONENT_PUSH_PRIO(component,ntasks,exp_len) do {                                 \
+	int workerid = STARPU_NMAXWORKERS + 1;									\
+	if((component->nchildren == 1) && starpu_sched_component_is_worker(component->children[0])) \
+		workerid = starpu_sched_component_worker_get_workerid(component->children[0]); \
+	_STARPU_TRACE_SCHED_COMPONENT_PUSH_PRIO(workerid, ntasks, exp_len); \
+} while (0)
+
+#define STARPU_TRACE_SCHED_COMPONENT_POP_PRIO(component,ntasks,exp_len) do {                                 \
+	int workerid = STARPU_NMAXWORKERS + 1;									\
+	if((component->nchildren == 1) && starpu_sched_component_is_worker(component->children[0])) \
+		workerid = starpu_sched_component_worker_get_workerid(component->children[0]); \
+	_STARPU_TRACE_SCHED_COMPONENT_POP_PRIO(workerid, ntasks, exp_len); \
+} while (0)
+
+struct _starpu_prio_data
+{
+	struct _starpu_prio_deque prio;
+	starpu_pthread_mutex_t mutex;
+	unsigned ntasks_threshold;
+	double exp_len_threshold;
+};
+
+static void prio_component_deinit_data(struct starpu_sched_component * component)
+{
+	STARPU_ASSERT(component && component->data);
+	struct _starpu_prio_data * f = component->data;
+	_starpu_prio_deque_destroy(&f->prio);
+	STARPU_PTHREAD_MUTEX_DESTROY(&f->mutex);
+	free(f);
+}
+
+static double prio_estimated_end(struct starpu_sched_component * component)
+{
+	STARPU_ASSERT(component && component->data);
+	struct _starpu_prio_data * data = component->data;
+	struct _starpu_prio_deque * prio = &data->prio;
+	starpu_pthread_mutex_t * mutex = &data->mutex;
+	int card = starpu_bitmap_cardinal(component->workers_in_ctx);
+	STARPU_ASSERT(card != 0);
+	STARPU_PTHREAD_MUTEX_LOCK(mutex);
+	prio->exp_start = STARPU_MAX(prio->exp_start, starpu_timing_now());
+	double estimated_end = prio->exp_start + prio->exp_len / card;
+	STARPU_PTHREAD_MUTEX_UNLOCK(mutex);
+
+	return estimated_end;
+}
+
+static double prio_estimated_load(struct starpu_sched_component * component)
+{
+	STARPU_ASSERT(component && component->data);
+	STARPU_ASSERT(starpu_bitmap_cardinal(component->workers_in_ctx) != 0);
+	struct _starpu_prio_data * data = component->data;
+	struct _starpu_prio_deque * prio = &data->prio;
+	starpu_pthread_mutex_t * mutex = &data->mutex;
+	double relative_speedup = 0.0;
+	double load;
+	if(STARPU_SCHED_COMPONENT_IS_HOMOGENEOUS(component))
+	{		
+		int first_worker = starpu_bitmap_first(component->workers_in_ctx);
+		relative_speedup = starpu_worker_get_relative_speedup(starpu_worker_get_perf_archtype(first_worker));
+		STARPU_PTHREAD_MUTEX_LOCK(mutex);
+		load = prio->ntasks / relative_speedup;
+		STARPU_PTHREAD_MUTEX_UNLOCK(mutex);
+		return load;
+	}
+	else
+	{
+		int i;
+		for(i = starpu_bitmap_first(component->workers_in_ctx);
+		    i != -1;
+		    i = starpu_bitmap_next(component->workers_in_ctx, i))
+			relative_speedup += starpu_worker_get_relative_speedup(starpu_worker_get_perf_archtype(i));
+		relative_speedup /= starpu_bitmap_cardinal(component->workers_in_ctx);
+		STARPU_ASSERT(!_STARPU_IS_ZERO(relative_speedup));
+		STARPU_PTHREAD_MUTEX_LOCK(mutex);
+		load = prio->ntasks / relative_speedup;
+		STARPU_PTHREAD_MUTEX_UNLOCK(mutex);
+	}
+	int i;
+	for(i = 0; i < component->nchildren; i++)
+	{
+		struct starpu_sched_component * c = component->children[i];
+		load += c->estimated_load(c);
+	}
+	return load;
+}
+
+static int prio_push_local_task(struct starpu_sched_component * component, struct starpu_task * task, unsigned is_pushback)
+{
+	STARPU_ASSERT(component && component->data && task);
+	STARPU_ASSERT(starpu_sched_component_can_execute_task(component,task));
+	struct _starpu_prio_data * data = component->data;
+	struct _starpu_prio_deque * prio = &data->prio;
+	starpu_pthread_mutex_t * mutex = &data->mutex;
+	int ret;
+	
+	STARPU_PTHREAD_MUTEX_LOCK(mutex);
+	double exp_len;
+	if(!isnan(task->predicted))
+		exp_len = prio->exp_len + task->predicted;
+	else
+		exp_len = prio->exp_len;
+
+	if((data->ntasks_threshold != 0) && (data->exp_len_threshold != 0.0) && 
+			((prio->ntasks >= data->ntasks_threshold) || (exp_len >= data->exp_len_threshold)))
+	{
+		static int warned;
+		if(task->predicted > data->exp_len_threshold && !warned)
+		{
+			_STARPU_DISP("Warning : a predicted task length (%lf) exceeds the expected length threshold (%lf) of a prio component queue, you should reconsider the value of this threshold. This message will not be printed again for further thresholds exceeding.\n",task->predicted,data->exp_len_threshold);
+			warned = 1;
+		}
+		ret = 1;
+		STARPU_PTHREAD_MUTEX_UNLOCK(mutex);
+	}
+	else
+	{
+		if(is_pushback)
+			ret = _starpu_prio_deque_push_back_task(prio,task);
+		else
+		{
+			ret = _starpu_prio_deque_push_task(prio,task);
+			starpu_sched_component_prefetch_on_node(component, task);
+			STARPU_TRACE_SCHED_COMPONENT_PUSH_PRIO(component, prio->ntasks, exp_len);
+		}
+
+		if(!isnan(task->predicted))
+		{
+			prio->exp_len = exp_len;
+			prio->exp_end = prio->exp_start + prio->exp_len;
+		}
+		STARPU_ASSERT(!isnan(prio->exp_end));
+		STARPU_ASSERT(!isnan(prio->exp_len));
+		STARPU_ASSERT(!isnan(prio->exp_start));
+		
+		if(!is_pushback)
+			component->can_pull(component);
+		STARPU_PTHREAD_MUTEX_UNLOCK(mutex);
+	}
+
+	return ret;
+}
+
+static int prio_push_task(struct starpu_sched_component * component, struct starpu_task * task)
+{
+	int ret = prio_push_local_task(component, task, 0);
+	return ret;
+}
+
+static struct starpu_task * prio_pull_task(struct starpu_sched_component * component)
+{
+	STARPU_ASSERT(component && component->data);
+	struct _starpu_prio_data * data = component->data;
+	struct _starpu_prio_deque * prio = &data->prio;
+	starpu_pthread_mutex_t * mutex = &data->mutex;
+	STARPU_PTHREAD_MUTEX_LOCK(mutex);
+	struct starpu_task * task = _starpu_prio_deque_pop_task(prio);
+	if(task)
+	{
+		if(!isnan(task->predicted))
+		{
+			prio->exp_start = starpu_timing_now() + task->predicted;
+			prio->exp_len -= task->predicted;
+		}
+		prio->exp_end = prio->exp_start + prio->exp_len;
+		if(prio->ntasks == 0)
+			prio->exp_len = 0.0;
+		
+		STARPU_TRACE_SCHED_COMPONENT_POP_PRIO(component, prio->ntasks, prio->exp_len);
+	}
+	STARPU_ASSERT(!isnan(prio->exp_end));
+	STARPU_ASSERT(!isnan(prio->exp_len));
+	STARPU_ASSERT(!isnan(prio->exp_start));
+	STARPU_PTHREAD_MUTEX_UNLOCK(mutex);
+
+	// When a pop is called, a can_push is called for pushing tasks onto
+	// the empty place of the queue left by the popped task.
+	int i,ret;
+	for(i=0; i < component->nparents; i++)
+	{
+		if(component->parents[i] == NULL)
+			continue;
+		else
+		{
+			ret = component->parents[i]->can_push(component->parents[i]);
+			if(ret)
+				break;
+		}
+	}
+	
+	if(task)
+		return task;
+
+	return NULL;
+}
+
+/* When a can_push is caught by this function, we try to pop and push
+ * tasks from our local queue as much as possible, until a
+ * push fails, which means that the worker prio_components are
+ * currently "full".
+ */
+static int prio_can_push(struct starpu_sched_component * component)
+{
+	STARPU_ASSERT(component && starpu_sched_component_is_prio(component));
+	int ret = 0;
+	int res = 0;
+
+	STARPU_ASSERT(component->nchildren == 1);
+	struct starpu_sched_component * child = component->children[0];
+
+	struct starpu_task * task = component->pull_task(component);
+	if(task)
+		ret = child->push_task(child,task);	
+	while(task && !ret) 
+	{
+		if(!res)
+			res = 1;
+
+		task = component->pull_task(component);
+		if(task)
+			ret = child->push_task(child,task);	
+	} 
+	if(task && ret)
+		prio_push_local_task(component,task,1); 
+
+	return res;
+}
+
+int starpu_sched_component_is_prio(struct starpu_sched_component * component)
+{
+	return component->push_task == prio_push_task;
+}
+
+struct starpu_sched_component * starpu_sched_component_prio_create(struct starpu_prio_data * params)
+{
+	struct starpu_sched_component * component = starpu_sched_component_create();
+	struct _starpu_prio_data * data = malloc(sizeof(*data));
+	_starpu_prio_deque_init(&data->prio);
+	STARPU_PTHREAD_MUTEX_INIT(&data->mutex,NULL);
+	component->data = data;
+	component->estimated_end = prio_estimated_end;
+	component->estimated_load = prio_estimated_load;
+	component->push_task = prio_push_task;
+	component->pull_task = prio_pull_task;
+	component->can_push = prio_can_push;
+	component->deinit_data = prio_component_deinit_data;
+
+	if(params)
+	{
+		data->ntasks_threshold=params->ntasks_threshold;
+		data->exp_len_threshold=params->exp_len_threshold;
+	}
+	else
+	{
+		data->ntasks_threshold=0;
+		data->exp_len_threshold=0.0;
+	}
+
+	return component;
+}

src/sched_policies/component_random.c

@@ -0,0 +1,120 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2013  INRIA
+ * Copyright (C) 2013  Simon Archipoff
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <starpu_sched_component.h>
+#include <core/workers.h>
+
+static double compute_relative_speedup(struct starpu_sched_component * component)
+{
+	double sum = 0.0;
+	int id;
+	for(id = starpu_bitmap_first(component->workers_in_ctx);
+	    id != -1;
+	    id = starpu_bitmap_next(component->workers_in_ctx, id))
+	{
+		struct starpu_perfmodel_arch* perf_arch = starpu_worker_get_perf_archtype(id);
+		sum += starpu_worker_get_relative_speedup(perf_arch);
+
+	}
+	STARPU_ASSERT(sum != 0.0);
+	return sum;
+}
+
+static int random_push_task(struct starpu_sched_component * component, struct starpu_task * task)
+{
+	STARPU_ASSERT(component->nchildren > 0);
+
+	/* indexes_components and size are used to memoize component that can execute tasks
+	 * during the first phase of algorithm, it contain the size indexes of the components
+	 * that can execute task.
+	 */
+	int indexes_components[component->nchildren];
+	int size=0;
+
+	/* speedup[i] is revelant only if i is in the size firsts elements of
+	 * indexes_components
+	 */
+	double speedup[component->nchildren];
+
+	double alpha_sum = 0.0;
+
+	int i;
+	for(i = 0; i < component->nchildren ; i++)
+	{
+		if(starpu_sched_component_can_execute_task(component->children[i],task))
+		{
+			speedup[size] = compute_relative_speedup(component->children[i]);
+			alpha_sum += speedup[size];
+			indexes_components[size] = i;
+			size++;
+		}
+	}
+	if(size == 0)
+		return -ENODEV;
+
+	/* not fully sure that this code is correct
+	 * because of bad properties of double arithmetic
+	 */
+	double random = starpu_drand48()*alpha_sum;
+	double alpha = 0.0;
+	struct starpu_sched_component * select  = NULL;
+	
+	for(i = 0; i < size ; i++)
+	{
+		int index = indexes_components[i];
+		if(alpha + speedup[i] >= random)
+		{	
+			select = component->children[index];
+			break;
+		}
+		alpha += speedup[i];
+	}
+	STARPU_ASSERT(select != NULL);
+	if(starpu_sched_component_is_worker(select))
+	{
+		select->can_pull(select);
+		return 1;
+	}
+
+	int ret_val = select->push_task(select,task);
+	return ret_val;
+}
+
+/* taking the min of estimated_end not seems to be a good value to return here
+ * as random scheduler balance between childs very poorly
+ */
+static double random_estimated_end(struct starpu_sched_component * component)
+{
+	double sum = 0.0;
+	int i;
+	for(i = 0; i < component->nchildren; i++)
+		sum += component->children[i]->estimated_end(component->children[i]);
+	return sum / component->nchildren;
+}
+
+int starpu_sched_component_is_random(struct starpu_sched_component *component)
+{
+	return component->push_task == random_push_task;
+}
+
+struct starpu_sched_component * starpu_sched_component_random_create(void * arg STARPU_ATTRIBUTE_UNUSED)
+{
+	struct starpu_sched_component * component = starpu_sched_component_create();
+	component->estimated_end = random_estimated_end;
+	component->push_task = random_push_task;
+	return component;
+}

src/sched_policies/component_sched.c

@@ -0,0 +1,574 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2013  INRIA
+ * Copyright (C) 2013  Simon Archipoff
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <core/jobs.h>
+#include <core/workers.h>
+#include <starpu_sched_component.h>
+#include <starpu_thread_util.h>
+
+#include <float.h>
+
+#include "sched_component.h"
+
+
+
+/******************************************************************************
+ *				Generic Scheduling Components' helper functions        		  *
+ ******************************************************************************/
+
+
+
+/* this function find the best implementation or an implementation that need to be calibrated for a worker available
+ * and set prediction in *length. nan if a implementation need to be calibrated, 0.0 if no perf model are available
+ * return false if no worker on the component can execute that task
+ */
+int starpu_sched_component_execute_preds(struct starpu_sched_component * component, struct starpu_task * task, double * length)
+{
+	STARPU_ASSERT(component && task);
+	int can_execute = 0;
+	starpu_task_bundle_t bundle = task->bundle;
+	double len = DBL_MAX;
+	
+
+	int workerid;
+	for(workerid = starpu_bitmap_first(component->workers_in_ctx);
+	    workerid != -1;
+	    workerid = starpu_bitmap_next(component->workers_in_ctx, workerid))
+	{
+		struct starpu_perfmodel_arch* archtype = starpu_worker_get_perf_archtype(workerid);
+		int nimpl;
+		for(nimpl = 0; nimpl < STARPU_MAXIMPLEMENTATIONS; nimpl++)
+		{
+			if(starpu_worker_can_execute_task(workerid,task,nimpl)
+			   || starpu_combined_worker_can_execute_task(workerid, task, nimpl))
+			{
+				double d;
+				can_execute = 1;
+				if(bundle)
+					d = starpu_task_bundle_expected_length(bundle, archtype, nimpl);
+				else
+					d = starpu_task_expected_length(task, archtype, nimpl);
+				if(isnan(d))
+				{
+					*length = d;
+					return can_execute;
+						
+				}
+				if(_STARPU_IS_ZERO(d) && !can_execute)
+				{
+					can_execute = 1;
+					continue;
+				}
+				if(d < len)
+				{
+					len = d;
+				}
+			}
+		}
+		if(STARPU_SCHED_COMPONENT_IS_HOMOGENEOUS(component))
+			break;
+	}
+
+	if(len == DBL_MAX) /* we dont have perf model */
+		len = 0.0; 
+	if(length)
+		*length = len;
+	return can_execute;
+}
+
+/* very similar function that dont compute prediction */
+int starpu_sched_component_can_execute_task(struct starpu_sched_component * component, struct starpu_task * task)
+{
+	STARPU_ASSERT(task);
+	STARPU_ASSERT(component);
+	unsigned nimpl;
+	int worker;
+	for (nimpl = 0; nimpl < STARPU_MAXIMPLEMENTATIONS; nimpl++)
+		for(worker = starpu_bitmap_first(component->workers_in_ctx);
+		    -1 != worker;
+		    worker = starpu_bitmap_next(component->workers_in_ctx, worker))
+			if (starpu_worker_can_execute_task(worker, task, nimpl)
+			     || starpu_combined_worker_can_execute_task(worker, task, nimpl))
+			    return 1;
+	return 0;
+}
+
+/* compute the average of transfer length for tasks on all workers
+ * maybe this should be optimised if all workers are under the same numa component
+ */
+double starpu_sched_component_transfer_length(struct starpu_sched_component * component, struct starpu_task * task)
+{
+	STARPU_ASSERT(component && task);
+	int nworkers = starpu_bitmap_cardinal(component->workers_in_ctx);
+	double sum = 0.0;
+	int worker;
+	if(STARPU_SCHED_COMPONENT_IS_SINGLE_MEMORY_NODE(component))
+	{
+		unsigned memory_node  = starpu_worker_get_memory_node(starpu_bitmap_first(component->workers_in_ctx));
+		if(task->bundle)
+			return starpu_task_bundle_expected_data_transfer_time(task->bundle,memory_node);
+		else
+			return starpu_task_expected_data_transfer_time(memory_node, task);
+	}
+
+	for(worker = starpu_bitmap_first(component->workers_in_ctx);
+	    worker != -1;
+	    worker = starpu_bitmap_next(component->workers_in_ctx, worker))
+	{
+		unsigned memory_node  = starpu_worker_get_memory_node(worker);
+		if(task->bundle)
+		{
+			sum += starpu_task_bundle_expected_data_transfer_time(task->bundle,memory_node);
+		}
+		else
+		{
+			sum += starpu_task_expected_data_transfer_time(memory_node, task);
+			/* sum += starpu_task_expected_conversion_time(task, starpu_worker_get_perf_archtype(worker), impl ?)
+			 * I dont know what to do as we dont know what implementation would be used here...
+			 */
+		}
+	}
+	return sum / nworkers;
+}
+
+/* This function can be called by components when they think that a prefetching request can be submitted.
+ * For example, it is currently used by the MCT component to begin the prefetching on accelerators 
+ * on which it pushed tasks as soon as possible.
+ */
+void starpu_sched_component_prefetch_on_node(struct starpu_sched_component * component, struct starpu_task * task)
+{
+	if (starpu_get_prefetch_flag() && (!task->prefetched)
+		&& (component->properties >= STARPU_SCHED_COMPONENT_SINGLE_MEMORY_NODE))
+	{
+		int worker = starpu_bitmap_first(component->workers_in_ctx);
+		unsigned memory_node = starpu_worker_get_memory_node(worker);
+		starpu_prefetch_task_input_on_node(task, memory_node);
+		task->prefetched = 1;
+	}
+}
+
+/* remove all child
+ * for all child of component, if child->parents[x] == component, set child->parents[x] to null 
+ * call component->deinit_data
+ */
+void starpu_sched_component_destroy(struct starpu_sched_component *component)
+{
+	STARPU_ASSERT(component);
+	int i,j;
+	for(i = 0; i < component->nchildren; i++)
+	{
+		struct starpu_sched_component * child = component->children[i];
+		for(j = 0; j < child->nparents; j++)
+			if(child->parents[j] == component)
+				child->remove_parent(child,component);
+
+	}
+	while(component->nchildren != 0)
+		component->remove_child(component, component->children[0]);
+	for(i = 0; i < component->nparents; i++)
+	{
+		struct starpu_sched_component * parent = component->parents[i];
+		for(j = 0; j < parent->nchildren; j++)
+			if(parent->children[j] == component)
+				parent->remove_child(parent,component);
+
+	}
+	while(component->nparents != 0)
+		component->remove_parent(component, component->parents[0]);
+	component->deinit_data(component);
+	free(component->children);
+	free(component->parents);
+	starpu_bitmap_destroy(component->workers);
+	starpu_bitmap_destroy(component->workers_in_ctx);
+	free(component);
+}
+
+void starpu_sched_component_destroy_rec(struct starpu_sched_component * component)
+{
+	if(component == NULL)
+		return;
+
+	int i = 0;
+	while(i < component->nchildren)
+	{
+		if (starpu_sched_component_is_worker(component->children[i]))
+			i++;
+		else
+			starpu_sched_component_destroy_rec(component->children[i]);
+	}
+
+	if (!starpu_sched_component_is_worker(component))
+		starpu_sched_component_destroy(component);
+}
+
+void set_properties(struct starpu_sched_component * component)
+{
+	STARPU_ASSERT(component);
+	component->properties = 0;
+
+	int worker = starpu_bitmap_first(component->workers_in_ctx);
+	if (worker == -1)
+		return;
+	uint32_t first_worker = _starpu_get_worker_struct(worker)->worker_mask;
+	unsigned first_memory_node = _starpu_get_worker_struct(worker)->memory_node;
+	int is_homogeneous = 1;
+	int is_all_same_component = 1;
+	for(;
+	    worker != -1;
+	    worker = starpu_bitmap_next(component->workers_in_ctx, worker))		
+	{
+		if(first_worker != _starpu_get_worker_struct(worker)->worker_mask)
+			is_homogeneous = 0;
+		if(first_memory_node != _starpu_get_worker_struct(worker)->memory_node)
+			is_all_same_component = 0;
+	}
+	
+
+	if(is_homogeneous)
+		component->properties |= STARPU_SCHED_COMPONENT_HOMOGENEOUS;
+	if(is_all_same_component)
+		component->properties |= STARPU_SCHED_COMPONENT_SINGLE_MEMORY_NODE;
+}
+
+
+/* recursively set the component->workers member of component's subtree
+ */
+void _starpu_sched_component_update_workers(struct starpu_sched_component * component)
+{
+	STARPU_ASSERT(component);
+	if(starpu_sched_component_is_worker(component))
+		return;
+	starpu_bitmap_unset_all(component->workers);
+	int i;
+	for(i = 0; i < component->nchildren; i++)
+	{
+		_starpu_sched_component_update_workers(component->children[i]);
+		starpu_bitmap_or(component->workers, component->children[i]->workers);
+		component->notify_change_workers(component);
+	}
+}
+
+/* recursively set the component->workers_in_ctx in component's subtree
+ */
+void _starpu_sched_component_update_workers_in_ctx(struct starpu_sched_component * component, unsigned sched_ctx_id)
+{
+	STARPU_ASSERT(component);
+	if(starpu_sched_component_is_worker(component))
+		return;
+	struct starpu_bitmap * workers_in_ctx = _starpu_get_worker_mask(sched_ctx_id);
+	starpu_bitmap_unset_and(component->workers_in_ctx,component->workers, workers_in_ctx);
+	int i,j;
+	for(i = 0; i < component->nchildren; i++)
+	{
+		struct starpu_sched_component * child = component->children[i];
+		_starpu_sched_component_update_workers_in_ctx(child, sched_ctx_id);
+		for(j = 0; j < STARPU_NMAX_SCHED_CTXS; j++)
+			if(child->parents[j] == component)
+			{
+				starpu_bitmap_or(component->workers_in_ctx, child->workers_in_ctx);
+				break;
+			}
+	}
+	set_properties(component);
+	component->notify_change_workers(component);
+}
+
+
+
+/******************************************************************************
+ *          			Scheduling Trees' helper functions        			  *
+ ******************************************************************************/
+
+
+
+struct starpu_bitmap * _starpu_get_worker_mask(unsigned sched_ctx_id)
+{
+	STARPU_ASSERT(sched_ctx_id < STARPU_NMAX_SCHED_CTXS);
+	struct starpu_sched_tree * t = starpu_sched_ctx_get_policy_data(sched_ctx_id);
+	STARPU_ASSERT(t);
+	return t->workers;
+}
+
+void starpu_sched_tree_update_workers_in_ctx(struct starpu_sched_tree * t)
+{
+	STARPU_ASSERT(t);
+	_starpu_sched_component_update_workers_in_ctx(t->root, t->sched_ctx_id);
+}
+
+void starpu_sched_tree_update_workers(struct starpu_sched_tree * t)
+{
+	STARPU_ASSERT(t);
+	_starpu_sched_component_update_workers(t->root);
+}
+
+
+
+/******************************************************************************
+ *          			Scheduling Trees' Functions                			  *
+ *  	Most of them are used to define the starpu_sched_policy interface     *
+ ******************************************************************************/
+
+
+
+int starpu_sched_tree_push_task(struct starpu_task * task)
+{
+	STARPU_ASSERT(task);
+	unsigned sched_ctx_id = task->sched_ctx;
+	struct starpu_sched_tree *tree = starpu_sched_ctx_get_policy_data(sched_ctx_id);
+
+	int ret_val = tree->root->push_task(tree->root,task);
+	
+	return ret_val;
+}
+
+struct starpu_task * starpu_sched_tree_pop_task(unsigned sched_ctx STARPU_ATTRIBUTE_UNUSED)
+{
+	int workerid = starpu_worker_get_id();
+	struct starpu_sched_component * component = starpu_sched_component_worker_get(workerid);
+
+	/* _starpu_sched_component_lock_worker(workerid) is called by component->pull_task()
+	 */
+	struct starpu_task * task = component->pull_task(component);
+	return task;
+}
+
+void starpu_sched_tree_add_workers(unsigned sched_ctx_id, int *workerids, unsigned nworkers)
+{
+	STARPU_ASSERT(sched_ctx_id < STARPU_NMAX_SCHED_CTXS);
+	STARPU_ASSERT(workerids);
+	struct starpu_sched_tree * t = starpu_sched_ctx_get_policy_data(sched_ctx_id);
+
+	STARPU_PTHREAD_MUTEX_LOCK(&t->lock);
+	_starpu_sched_component_lock_all_workers();
+
+	unsigned i;
+	for(i = 0; i < nworkers; i++)
+		starpu_bitmap_set(t->workers, workerids[i]);
+
+	starpu_sched_tree_update_workers_in_ctx(t);
+
+	_starpu_sched_component_unlock_all_workers();
+	STARPU_PTHREAD_MUTEX_UNLOCK(&t->lock);
+}
+
+void starpu_sched_tree_remove_workers(unsigned sched_ctx_id, int *workerids, unsigned nworkers)
+{
+	STARPU_ASSERT(sched_ctx_id < STARPU_NMAX_SCHED_CTXS);
+	STARPU_ASSERT(workerids);
+	struct starpu_sched_tree * t = starpu_sched_ctx_get_policy_data(sched_ctx_id);
+
+	STARPU_PTHREAD_MUTEX_LOCK(&t->lock);
+	_starpu_sched_component_lock_all_workers();
+
+	unsigned i;
+	for(i = 0; i < nworkers; i++)
+		starpu_bitmap_unset(t->workers, workerids[i]);
+
+	starpu_sched_tree_update_workers_in_ctx(t);
+
+	_starpu_sched_component_unlock_all_workers();
+	STARPU_PTHREAD_MUTEX_UNLOCK(&t->lock);
+}
+
+struct starpu_sched_tree * starpu_sched_tree_create(unsigned sched_ctx_id)
+{
+	STARPU_ASSERT(sched_ctx_id < STARPU_NMAX_SCHED_CTXS);
+	struct starpu_sched_tree * t = malloc(sizeof(*t));
+	memset(t, 0, sizeof(*t));
+	t->sched_ctx_id = sched_ctx_id;
+	t->workers = starpu_bitmap_create();
+	STARPU_PTHREAD_MUTEX_INIT(&t->lock,NULL);
+	return t;
+}
+
+void starpu_sched_tree_destroy(struct starpu_sched_tree * tree)
+{
+	STARPU_ASSERT(tree);
+	if(tree->root)
+		starpu_sched_component_destroy_rec(tree->root);
+	starpu_bitmap_destroy(tree->workers);
+	STARPU_PTHREAD_MUTEX_DESTROY(&tree->lock);
+	free(tree);
+}
+
+
+
+/******************************************************************************
+ *          Interface Functions for Generic Scheduling Components             *
+ ******************************************************************************/
+
+
+
+static void starpu_sched_component_add_child(struct starpu_sched_component* component, struct starpu_sched_component * child)
+{
+	STARPU_ASSERT(component && child);
+	STARPU_ASSERT(!starpu_sched_component_is_worker(component));
+	int i;
+	for(i = 0; i < component->nchildren; i++){
+		STARPU_ASSERT(component->children[i] != component);
+		STARPU_ASSERT(component->children[i] != NULL);
+	}
+
+	component->children = realloc(component->children, sizeof(struct starpu_sched_component *) * (component->nchildren + 1));
+	component->children[component->nchildren] = child;
+	component->nchildren++;
+}
+
+static void starpu_sched_component_remove_child(struct starpu_sched_component * component, struct starpu_sched_component * child)
+{
+	STARPU_ASSERT(component && child);
+	STARPU_ASSERT(!starpu_sched_component_is_worker(component));
+	int pos;
+	for(pos = 0; pos < component->nchildren; pos++)
+		if(component->children[pos] == child)
+			break;
+	STARPU_ASSERT(pos != component->nchildren);
+	component->children[pos] = component->children[--component->nchildren];
+}
+
+static void starpu_sched_component_add_parent(struct starpu_sched_component* component, struct starpu_sched_component * parent)
+{
+	STARPU_ASSERT(component && parent);
+	int i;
+	for(i = 0; i < component->nparents; i++){
+		STARPU_ASSERT(component->parents[i] != component);
+		STARPU_ASSERT(component->parents[i] != NULL);
+	}
+
+	component->parents = realloc(component->parents, sizeof(struct starpu_sched_component *) * (component->nparents + 1));
+	component->parents[component->nparents] = parent;
+	component->nparents++;
+}
+
+static void starpu_sched_component_remove_parent(struct starpu_sched_component * component, struct starpu_sched_component * parent)
+{
+	STARPU_ASSERT(component && parent);
+	int pos;
+	for(pos = 0; pos < component->nparents; pos++)
+		if(component->parents[pos] == parent)
+			break;
+	STARPU_ASSERT(pos != component->nparents);
+	component->parents[pos] = component->parents[--component->nparents];
+}
+
+/* default implementation for component->pull_task()
+ * just perform a recursive call on parent
+ */
+static struct starpu_task * starpu_sched_component_pull_task(struct starpu_sched_component * component)
+{
+	STARPU_ASSERT(component);
+	struct starpu_task * task = NULL;
+	int i;
+	for(i=0; i < component->nparents; i++)
+	{
+		if(component->parents[i] == NULL)
+			continue;
+		else
+		{
+			task = component->parents[i]->pull_task(component->parents[i]);
+			if(task)
+				break;
+		}
+	}
+	return task;
+}
+
+/* The default implementation of the can_push function is a recursive call to its parents.
+ * A personally-made can_push in a component (like in prio components) is necessary to catch
+ * this recursive call somewhere, if the user wants to exploit it.
+ */
+static int starpu_sched_component_can_push(struct starpu_sched_component * component)
+{
+	STARPU_ASSERT(component);
+	int ret = 0;
+	if(component->nparents > 0)
+	{
+		int i;
+		for(i=0; i < component->nparents; i++)
+		{
+			struct starpu_sched_component * parent = component->parents[i];
+			if(parent != NULL)
+				ret = parent->can_push(parent);
+			if(ret)
+				break;
+		}
+	}
+	return ret;
+}
+
+/* A can_pull call will try to wake up one worker associated to the childs of the
+ * component. It is currenly called by components which holds a queue (like fifo and prio
+ * components) to signify its childs that a task has been pushed on its local queue.
+ */
+static void starpu_sched_component_can_pull(struct starpu_sched_component * component)
+{
+	STARPU_ASSERT(component);
+	STARPU_ASSERT(!starpu_sched_component_is_worker(component));
+	int i;
+	for(i = 0; i < component->nchildren; i++)
+		component->children[i]->can_pull(component->children[i]);
+}
+
+static double starpu_sched_component_estimated_load(struct starpu_sched_component * component)
+{
+	double sum = 0.0;
+	int i;
+	for( i = 0; i < component->nchildren; i++)
+	{
+		struct starpu_sched_component * c = component->children[i];
+		sum += c->estimated_load(c);
+	}
+	return sum;
+}
+
+static double starpu_sched_component_estimated_end_min(struct starpu_sched_component * component)
+{
+	STARPU_ASSERT(component);
+	double min = DBL_MAX;
+	int i;
+	for(i = 0; i < component->nchildren; i++)
+	{
+		double tmp = component->children[i]->estimated_end(component->children[i]);
+		if(tmp < min)
+			min = tmp;
+	}
+	return min;
+}
+
+static void take_component_and_does_nothing(struct starpu_sched_component * component STARPU_ATTRIBUTE_UNUSED)
+{
+}
+
+struct starpu_sched_component * starpu_sched_component_create(void)
+{
+	struct starpu_sched_component * component = malloc(sizeof(*component));
+	memset(component,0,sizeof(*component));
+	component->workers = starpu_bitmap_create();
+	component->workers_in_ctx = starpu_bitmap_create();
+	component->add_child = starpu_sched_component_add_child;
+	component->remove_child = starpu_sched_component_remove_child;
+	component->add_parent = starpu_sched_component_add_parent;
+	component->remove_parent = starpu_sched_component_remove_parent;
+	component->pull_task = starpu_sched_component_pull_task;
+	component->can_push = starpu_sched_component_can_push;
+	component->can_pull = starpu_sched_component_can_pull;
+	component->estimated_load = starpu_sched_component_estimated_load;
+	component->estimated_end = starpu_sched_component_estimated_end_min;
+	component->deinit_data = take_component_and_does_nothing;
+	component->notify_change_workers = take_component_and_does_nothing;
+	return component;
+}

src/sched_policies/component_work_stealing.c

@@ -0,0 +1,352 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2013  INRIA
+ * Copyright (C) 2013  Simon Archipoff
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <starpu_sched_component.h>
+#include <starpu_scheduler.h>
+#include <starpu.h>
+
+#include <float.h>
+
+#include "prio_deque.h"
+
+struct _starpu_work_stealing_data
+{
+/* keep track of the work performed from the beginning of the algorithm to make
+ * better decisions about which queue to child when stealing or deferring work
+ */
+	unsigned performed_total, last_pop_child, last_push_child;
+
+	struct _starpu_prio_deque ** fifos;	
+	starpu_pthread_mutex_t ** mutexes;
+	int size;
+};
+
+
+/**
+ * steal a task in a round robin way
+ * return NULL if none available
+ */
+static struct starpu_task *  steal_task_round_robin(struct starpu_sched_component *component, int workerid)
+{
+	struct _starpu_work_stealing_data *wsd = component->data;
+	unsigned i = wsd->last_pop_child;
+	wsd->last_pop_child = (wsd->last_pop_child + 1) % component->nchildren;
+	/* If the worker's queue have no suitable tasks, let's try
+	 * the next ones */
+	struct starpu_task * task = NULL;
+	while (1)
+	{
+		struct _starpu_prio_deque * fifo = wsd->fifos[i];
+
+		STARPU_PTHREAD_MUTEX_LOCK(wsd->mutexes[i]);
+		task = _starpu_prio_deque_deque_task_for_worker(fifo, workerid);
+		if(task && !isnan(task->predicted))
+		{
+			fifo->exp_len -= task->predicted;
+			fifo->nprocessed--;
+		}
+		STARPU_PTHREAD_MUTEX_UNLOCK(wsd->mutexes[i]);
+		if(task)
+			break;
+
+		if (i == wsd->last_pop_child)
+		{
+			/* We got back to the first worker,
+			 * don't go in infinite loop */
+			return NULL;
+		}
+		i = (i + 1) % component->nchildren;
+	
+	}
+	return task;
+}
+
+/**
+ * Return a worker to whom add a task.
+ * Selecting a worker is done in a round-robin fashion.
+ */
+static unsigned select_worker_round_robin(struct starpu_sched_component * component)
+{
+	struct _starpu_work_stealing_data *ws = (struct _starpu_work_stealing_data*)component->data;
+	unsigned i = (ws->last_push_child + 1) % component->nchildren ;
+	ws->last_push_child = i;
+	return i;
+}
+
+
+/**
+ * Return a worker from which a task can be stolen.
+ * This is a phony function used to call the right
+ * function depending on the value of USE_OVERLOAD.
+ */
+static inline struct starpu_task * steal_task(struct starpu_sched_component * component, int workerid)
+{
+	return steal_task_round_robin(component, workerid);
+}
+
+/**
+ * Return a worker from which a task can be stolen.
+ * This is a phony function used to call the right
+ * function depending on the value of USE_OVERLOAD.
+ */
+static inline unsigned select_worker(struct starpu_sched_component * component)
+{
+	return select_worker_round_robin(component);
+}
+
+
+static int is_worker_of_component(struct starpu_sched_component * component, int workerid)
+{
+	return starpu_bitmap_get(component->workers, workerid);
+}
+
+
+
+static struct starpu_task * pull_task(struct starpu_sched_component * component)
+{
+	int workerid = starpu_worker_get_id();
+	int i;
+	for(i = 0; i < component->nchildren; i++)
+	{
+		if(is_worker_of_component(component->children[i], workerid))
+			break;
+	}
+	STARPU_ASSERT(i < component->nchildren);
+	struct _starpu_work_stealing_data * wsd = component->data;
+	STARPU_PTHREAD_MUTEX_LOCK(wsd->mutexes[i]);
+	struct starpu_task * task = _starpu_prio_deque_pop_task(wsd->fifos[i]);
+	if(task)
+	{
+		if(!isnan(task->predicted))
+		{
+			wsd->fifos[i]->exp_len -= task->predicted;
+			wsd->fifos[i]->exp_start = starpu_timing_now() + task->predicted;
+		}
+	}
+	else
+		wsd->fifos[i]->exp_len = 0.0;
+
+	STARPU_PTHREAD_MUTEX_UNLOCK(wsd->mutexes[i]);
+	if(task)
+	{
+		return task;
+	}
+	
+	task  = steal_task(component, workerid);
+	if(task)
+	{
+		STARPU_PTHREAD_MUTEX_LOCK(wsd->mutexes[i]);
+		wsd->fifos[i]->nprocessed++;
+		STARPU_PTHREAD_MUTEX_UNLOCK(wsd->mutexes[i]);
+
+		return task;
+	}
+	for(i=0; i < component->nparents; i++)
+	{
+		if(component->parents[i] == NULL)
+			continue;
+		else
+		{
+			task = component->parents[i]->pull_task(component->parents[i]);
+			if(task)
+				break;
+		}
+	}
+	if(task)
+		return task;
+	else
+		return NULL;
+}
+
+double _ws_estimated_end(struct starpu_sched_component * component)
+{
+	STARPU_ASSERT(starpu_sched_component_is_work_stealing(component));
+	struct _starpu_work_stealing_data * wsd = component->data;
+	double sum_len = 0.0;
+	double sum_start = 0.0;
+	int i;
+	for(i = 0; i < component->nchildren; i++)
+	{
+		STARPU_PTHREAD_MUTEX_LOCK(wsd->mutexes[i]);
+		sum_len += wsd->fifos[i]->exp_len;
+		wsd->fifos[i]->exp_start = STARPU_MAX(starpu_timing_now(), wsd->fifos[i]->exp_start);
+		sum_start += wsd->fifos[i]->exp_start;
+		STARPU_PTHREAD_MUTEX_UNLOCK(wsd->mutexes[i]);
+
+	}
+	int nb_workers = starpu_bitmap_cardinal(component->workers_in_ctx);
+
+	return (sum_start + sum_len) / nb_workers;
+}
+
+double _ws_estimated_load(struct starpu_sched_component * component)
+{
+	STARPU_ASSERT(starpu_sched_component_is_work_stealing(component));
+	struct _starpu_work_stealing_data * wsd = component->data;
+	int ntasks = 0;
+	int i;
+	for(i = 0; i < component->nchildren; i++)
+	{
+		STARPU_PTHREAD_MUTEX_LOCK(wsd->mutexes[i]);
+		ntasks += wsd->fifos[i]->ntasks;
+		STARPU_PTHREAD_MUTEX_UNLOCK(wsd->mutexes[i]);
+	}
+	double speedup = 0.0;
+	int workerid;
+	for(workerid = starpu_bitmap_first(component->workers_in_ctx);
+	    -1 != workerid;
+	    workerid = starpu_bitmap_next(component->workers_in_ctx, workerid))
+	{
+		speedup += starpu_worker_get_relative_speedup(starpu_worker_get_perf_archtype(workerid));
+	}
+	
+	return ntasks / speedup;
+}
+
+static int push_task(struct starpu_sched_component * component, struct starpu_task * task)
+{
+	struct _starpu_work_stealing_data * wsd = component->data;
+	int ret = -1;
+	int i = wsd->last_push_child;
+	i = (i+1)%component->nchildren;
+
+	STARPU_PTHREAD_MUTEX_LOCK(wsd->mutexes[i]);
+	ret = _starpu_prio_deque_push_task(wsd->fifos[i], task);
+	STARPU_PTHREAD_MUTEX_UNLOCK(wsd->mutexes[i]);
+
+	wsd->last_push_child = i;
+	component->can_pull(component);
+	return ret;
+}
+
+
+//this function is special, when a worker call it, we want to push the task in his fifo
+int starpu_sched_tree_work_stealing_push_task(struct starpu_task *task)
+{
+	int workerid = starpu_worker_get_id();
+	if(workerid == -1)
+		return starpu_sched_tree_push_task(task);
+
+	unsigned sched_ctx_id = task->sched_ctx;
+	struct starpu_sched_component * component =starpu_sched_component_worker_get(workerid);
+	while(component->parents[sched_ctx_id] != NULL)
+	{
+		component = component->parents[sched_ctx_id];
+		if(starpu_sched_component_is_work_stealing(component))
+		{
+			if(!starpu_sched_component_can_execute_task(component, task))
+				return starpu_sched_tree_push_task(task);
+
+			int i;
+			for(i = 0; i < component->nchildren; i++)
+				if(is_worker_of_component(component->children[i], workerid))
+					break;
+			STARPU_ASSERT(i < component->nchildren);
+			
+			struct _starpu_work_stealing_data * wsd = component->data;
+			STARPU_PTHREAD_MUTEX_LOCK(wsd->mutexes[i]);
+			int ret = _starpu_prio_deque_push_task(wsd->fifos[i] , task);
+			if(ret == 0 && !isnan(task->predicted))
+				wsd->fifos[i]->exp_len += task->predicted;
+			STARPU_PTHREAD_MUTEX_UNLOCK(wsd->mutexes[i]);
+			
+			//we need to wake all workers
+			component->can_pull(component);
+			return ret;
+		}
+	}
+
+	/* this should not be reached */
+	return starpu_sched_tree_push_task(task);
+}
+
+
+void _ws_add_child(struct starpu_sched_component * component, struct starpu_sched_component * child)
+{
+	struct _starpu_work_stealing_data * wsd = component->data;
+	component->add_child(component, child);
+	if(wsd->size < component->nchildren)
+	{
+		STARPU_ASSERT(wsd->size == component->nchildren - 1);
+		wsd->fifos = realloc(wsd->fifos, component->nchildren * sizeof(*wsd->fifos));
+		wsd->mutexes = realloc(wsd->mutexes, component->nchildren * sizeof(*wsd->mutexes));
+		wsd->size = component->nchildren;
+	}
+
+	struct _starpu_prio_deque * fifo = malloc(sizeof(*fifo));
+	_starpu_prio_deque_init(fifo);
+	wsd->fifos[component->nchildren - 1] = fifo;
+
+	starpu_pthread_mutex_t * mutex = malloc(sizeof(*mutex));
+	STARPU_PTHREAD_MUTEX_INIT(mutex,NULL);
+	wsd->mutexes[component->nchildren - 1] = mutex;
+}
+
+void _ws_remove_child(struct starpu_sched_component * component, struct starpu_sched_component * child)
+{
+	struct _starpu_work_stealing_data * wsd = component->data;
+
+	STARPU_PTHREAD_MUTEX_DESTROY(wsd->mutexes[component->nchildren - 1]);
+	free(wsd->mutexes[component->nchildren - 1]);
+
+	int i_component;
+	for(i_component = 0; i_component < component->nchildren; i_component++)
+	{
+		if(component->children[i_component] == child)
+			break;
+	}
+	STARPU_ASSERT(i_component != component->nchildren);
+	struct _starpu_prio_deque * tmp_fifo = wsd->fifos[i_component];
+	wsd->fifos[i_component] = wsd->fifos[component->nchildren - 1];
+
+	
+	component->children[i_component] = component->children[component->nchildren - 1];
+	component->nchildren--;
+	struct starpu_task * task;
+	while((task = _starpu_prio_deque_pop_task(tmp_fifo)))
+	{
+		component->push_task(component, task);
+	}
+	_starpu_prio_deque_destroy(tmp_fifo);
+	free(tmp_fifo);
+}
+
+void _work_stealing_component_deinit_data(struct starpu_sched_component * component)
+{
+	free(component->data);
+}
+
+int starpu_sched_component_is_work_stealing(struct starpu_sched_component * component)
+{
+	return component->push_task == push_task;
+}
+
+struct starpu_sched_component * starpu_sched_component_work_stealing_create(void * arg STARPU_ATTRIBUTE_UNUSED)
+{
+	struct starpu_sched_component * component = starpu_sched_component_create();
+	struct _starpu_work_stealing_data * wsd = malloc(sizeof(*wsd));
+	memset(wsd, 0, sizeof(*wsd));
+	component->pull_task = pull_task;
+	component->push_task = push_task;
+	component->add_child = _ws_add_child;
+	component->remove_child = _ws_remove_child;
+	component->estimated_end = _ws_estimated_end;
+	component->estimated_load = _ws_estimated_load;
+	component->deinit_data = _work_stealing_component_deinit_data;
+	component->data = wsd;
+	return  component;
+}

src/sched_policies/component_worker.c

@@ -0,0 +1,901 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2010-2014  Université de Bordeaux 1
+ * Copyright (C) 2010, 2011, 2012, 2013  Centre National de la Recherche Scientifique
+ * Copyright (C) 2011  Télécom-SudParis
+ * Copyright (C) 2011-2013  INRIA
+ * Copyright (C) 2013  Simon Archipoff
+
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <starpu_sched_component.h>
+#include <core/workers.h>
+
+#include <float.h>
+
+/* data structure for worker's queue look like this :
+ * W = worker
+ * T = simple task
+ * P = parallel task
+ *
+ *
+ *         P--P  T
+ *         |  | \|
+ *   P--P  T  T  P  T
+ *   |  |  |  |  |  |
+ *   T  T  P--P--P  T
+ *   |  |  |  |  |  |
+ *   W  W  W  W  W  W
+ *
+ *
+ *
+ * its possible that a _starpu_task_grid wont have task, because it have been
+ * poped by a worker.
+ *
+ * N = no task
+ *
+ *   T  T  T
+ *   |  |  |
+ *   P--N--N
+ *   |  |  |
+ *   W  W  W
+ *
+ *
+ * this API is a little asymmetric : struct _starpu_task_grid are allocated by the caller and freed by the data structure
+ *
+ */
+
+
+
+/******************************************************************************
+ *					  Worker Components' Data Structures					  *
+ *****************************************************************************/
+
+
+
+struct _starpu_task_grid
+{
+	/* this member may be NULL if a worker have poped it but its a
+	 * parallel task and we dont want mad pointers
+	 */
+	struct starpu_task * task;
+
+	struct _starpu_task_grid *up, *down, *left, *right;
+
+	/* this is used to count the number of task to be poped by a worker
+	 * the leftist _starpu_task_grid maintain the ntasks counter (ie .left == NULL),
+	 * all the others use the pntasks that point to it
+	 *
+	 * when the counter reach 0, all the left and right member are set to NULL,
+	 * that mean that we will free that components.
+	 */
+	union
+	{
+		int ntasks;
+		int * pntasks;
+	};
+};
+
+
+/* list->exp_start, list->exp_len, list-exp_end and list->ntasks
+ * are updated by starpu_sched_component_worker_push_task(component, task) and pre_exec_hook
+ */
+struct _starpu_worker_task_list
+{
+	double exp_start, exp_len, exp_end;
+	struct _starpu_task_grid *first, *last;
+	unsigned ntasks;
+	starpu_pthread_mutex_t mutex;
+};
+
+enum _starpu_worker_component_status
+{
+	COMPONENT_STATUS_SLEEPING,
+	COMPONENT_STATUS_RESET,
+	COMPONENT_STATUS_CHANGED
+};
+
+struct _starpu_worker_component_data
+{
+	union 
+	{
+		struct
+		{
+			struct _starpu_worker * worker;
+			starpu_pthread_mutex_t lock;
+		};	
+		struct _starpu_combined_worker * combined_worker;
+	};
+	struct _starpu_worker_task_list * list;
+	enum _starpu_worker_component_status status;
+};
+
+/* this array store worker components */
+static struct starpu_sched_component * _worker_components[STARPU_NMAXWORKERS];
+	
+struct starpu_sched_component * starpu_sched_component_worker_get(int workerid);
+
+
+/******************************************************************************
+ *				Worker Components' Task List and Grid Functions				  *
+ *****************************************************************************/
+
+
+
+static struct _starpu_worker_task_list * _starpu_worker_task_list_create(void)
+{
+	struct _starpu_worker_task_list * l = malloc(sizeof(*l));
+	memset(l, 0, sizeof(*l));
+	l->exp_len = 0.0;
+	l->exp_start = l->exp_end = starpu_timing_now();
+	STARPU_PTHREAD_MUTEX_INIT(&l->mutex,NULL);
+	return l;
+}
+
+static struct _starpu_task_grid * _starpu_task_grid_create(void)
+{
+	struct _starpu_task_grid * t = malloc(sizeof(*t));
+	memset(t, 0, sizeof(*t));
+	return t;
+}
+
+static struct _starpu_worker_task_list * _worker_get_list(void)
+{
+	int workerid = starpu_worker_get_id();
+	STARPU_ASSERT(0 <= workerid && workerid < (int) starpu_worker_get_count());
+	struct _starpu_worker_component_data * d = starpu_sched_component_worker_get(workerid)->data;
+	return d->list;
+}
+
+static void _starpu_task_grid_destroy(struct _starpu_task_grid * t)
+{
+	free(t);
+}
+
+static void _starpu_worker_task_list_destroy(struct _starpu_worker_task_list * l)
+{
+	if(l)
+	{
+		STARPU_PTHREAD_MUTEX_DESTROY(&l->mutex);
+		free(l);
+	}
+}
+
+static inline void _starpu_worker_task_list_push(struct _starpu_worker_task_list * l, struct _starpu_task_grid * t)
+{
+/* the task, ntasks, pntasks, left and right members of t are set by the caller */
+	STARPU_ASSERT(t->task);
+	if(l->first == NULL)
+		l->first = l->last = t;
+	t->down = l->last;
+	l->last->up = t;
+	t->up = NULL;
+	l->last = t;
+	l->ntasks++;
+
+	double predicted = t->task->predicted;
+	double predicted_transfer = t->task->predicted_transfer;
+
+	/* Sometimes workers didn't take the tasks as early as we expected */
+	l->exp_start = STARPU_MAX(l->exp_start, starpu_timing_now());
+	l->exp_end = l->exp_start + l->exp_len;
+
+	if (starpu_timing_now() + predicted_transfer < l->exp_end)
+	{
+		/* We may hope that the transfer will be finished by
+		 * the start of the task. */
+		predicted_transfer = 0.0;
+	}
+	else
+	{
+		/* The transfer will not be finished by then, take the
+		 * remainder into account */
+		predicted_transfer = (starpu_timing_now() + predicted_transfer) - l->exp_end;
+	}
+
+	if(!isnan(predicted_transfer))
+	{
+		l->exp_end += predicted_transfer;
+		l->exp_len += predicted_transfer;
+	}
+
+	if(!isnan(predicted))
+	{
+		l->exp_end += predicted;
+		l->exp_len += predicted;
+	}
+
+	t->task->predicted = predicted;
+	t->task->predicted_transfer = predicted_transfer;
+}
+
+/* recursively set left and right pointers to NULL */
+static inline void _starpu_task_grid_unset_left_right_member(struct _starpu_task_grid * t)
+{
+	STARPU_ASSERT(t->task == NULL);
+	struct _starpu_task_grid * t_left = t->left;
+	struct _starpu_task_grid * t_right = t->right;
+	t->left = t->right = NULL;
+	while(t_left)
+	{
+		STARPU_ASSERT(t_left->task == NULL);
+		t = t_left;
+		t_left = t_left->left;
+		t->left = NULL;
+		t->right = NULL;
+	}
+	while(t_right)
+	{
+		STARPU_ASSERT(t_right->task == NULL);
+		t = t_right;
+		t_right = t_right->right;
+		t->left = NULL;
+		t->right = NULL;
+	}
+}
+
+static inline struct starpu_task * _starpu_worker_task_list_pop(struct _starpu_worker_task_list * l)
+{
+ 	if(!l->first)
+	{
+		l->exp_start = l->exp_end = starpu_timing_now();
+		l->exp_len = 0;
+		return NULL;
+	}
+	struct _starpu_task_grid * t = l->first;
+
+	/* if there is no task there is no tasks linked to this, then we can free it */
+	if(t->task == NULL && t->right == NULL && t->left == NULL)
+	{
+		l->first = t->up;
+		if(l->first)
+			l->first->down = NULL;
+		if(l->last == t)
+			l->last = NULL;
+		_starpu_task_grid_destroy(t);
+		return _starpu_worker_task_list_pop(l);
+	}
+
+	while(t)
+	{
+		if(t->task)
+		{
+			struct starpu_task * task = t->task;
+			t->task = NULL;
+			/* the leftist thing hold the number of tasks, other have a pointer to it */
+			int * p = t->left ? t->pntasks : &t->ntasks;
+			
+			/* the worker who pop the last task allow the rope to be freed */
+			if(STARPU_ATOMIC_ADD(p, -1) == 0) 
+				_starpu_task_grid_unset_left_right_member(t);
+
+			l->ntasks--;
+
+			if(!isnan(task->predicted))
+			{
+				l->exp_len -= task->predicted_transfer;
+				l->exp_end = l->exp_start + l->exp_len;
+			}
+
+			return task;
+		}
+		t = t->up;
+	}
+
+	return NULL;
+}
+
+
+
+/******************************************************************************
+ *			Worker Components' Public Helper Functions (Part 1)		     	  *
+ *****************************************************************************/
+
+
+
+struct _starpu_worker * _starpu_sched_component_worker_get_worker(struct starpu_sched_component * worker_component)
+{
+	STARPU_ASSERT(starpu_sched_component_is_simple_worker(worker_component));
+	struct _starpu_worker_component_data * data = worker_component->data;
+	return data->worker;
+}
+struct _starpu_combined_worker * _starpu_sched_component_combined_worker_get_combined_worker(struct starpu_sched_component * worker_component)
+{
+	STARPU_ASSERT(starpu_sched_component_is_combined_worker(worker_component));
+	struct _starpu_worker_component_data * data = worker_component->data;
+	return data->combined_worker;
+}
+
+void _starpu_sched_component_lock_worker(int workerid)
+{
+	STARPU_ASSERT(0 <= workerid && workerid < (int) starpu_worker_get_count());
+	struct _starpu_worker_component_data * data = starpu_sched_component_worker_get(workerid)->data;
+	STARPU_PTHREAD_MUTEX_LOCK(&data->lock);
+}
+void _starpu_sched_component_unlock_worker(int workerid)
+{
+	STARPU_ASSERT(0 <= workerid && workerid < (int)starpu_worker_get_count());
+	struct _starpu_worker_component_data * data = starpu_sched_component_worker_get(workerid)->data;
+	STARPU_PTHREAD_MUTEX_UNLOCK(&data->lock);
+}
+
+
+
+/******************************************************************************
+ *				Worker Components' Private Helper Functions			      	  *
+ *****************************************************************************/
+
+
+
+/* Allows a worker to lock/unlock scheduling mutexes. Currently used in 
+ * self-defined can_push calls to allow can_pull calls to take those mutexes while the 
+ * current worker is pushing tasks on other workers (or itself). 
+ */
+static void _starpu_sched_component_worker_lock_scheduling(void)
+{
+	int workerid = starpu_worker_get_id();
+	starpu_pthread_mutex_t *sched_mutex;
+	starpu_pthread_cond_t *sched_cond;
+	starpu_worker_get_sched_condition(workerid, &sched_mutex, &sched_cond);
+	_starpu_sched_component_lock_worker(workerid);	
+	STARPU_PTHREAD_MUTEX_LOCK(sched_mutex);
+}
+
+static void _starpu_sched_component_worker_unlock_scheduling(void)
+{
+	int workerid = starpu_worker_get_id();
+	starpu_pthread_mutex_t *sched_mutex;
+	starpu_pthread_cond_t *sched_cond;
+	starpu_worker_get_sched_condition(workerid, &sched_mutex, &sched_cond);
+	STARPU_PTHREAD_MUTEX_UNLOCK(sched_mutex);
+	_starpu_sched_component_unlock_worker(workerid);	
+}
+
+static void _starpu_sched_component_worker_set_sleep_status(struct starpu_sched_component * worker_component)
+{
+	STARPU_ASSERT(starpu_sched_component_is_worker(worker_component));
+	struct _starpu_worker_component_data * data = worker_component->data;
+	data->status = COMPONENT_STATUS_SLEEPING;
+}
+static void _starpu_sched_component_worker_set_changed_status(struct starpu_sched_component * worker_component)
+{
+	STARPU_ASSERT(starpu_sched_component_is_worker(worker_component));
+	struct _starpu_worker_component_data * data = worker_component->data;
+	data->status = COMPONENT_STATUS_CHANGED;
+}
+static void _starpu_sched_component_worker_reset_status(struct starpu_sched_component * worker_component)
+{
+	STARPU_ASSERT(starpu_sched_component_is_worker(worker_component));
+	struct _starpu_worker_component_data * data = worker_component->data;
+	data->status = COMPONENT_STATUS_RESET;
+}
+
+static int _starpu_sched_component_worker_is_reset_status(struct starpu_sched_component * worker_component)
+{
+	STARPU_ASSERT(starpu_sched_component_is_worker(worker_component));
+	struct _starpu_worker_component_data * data = worker_component->data;
+	return (data->status == COMPONENT_STATUS_RESET);
+}
+static int _starpu_sched_component_worker_is_changed_status(struct starpu_sched_component * worker_component)
+{
+	STARPU_ASSERT(starpu_sched_component_is_worker(worker_component));
+	struct _starpu_worker_component_data * data = worker_component->data;
+	return (data->status == COMPONENT_STATUS_CHANGED);
+}
+static int _starpu_sched_component_worker_is_sleeping_status(struct starpu_sched_component * worker_component)
+{
+	STARPU_ASSERT(starpu_sched_component_is_worker(worker_component));
+	struct _starpu_worker_component_data * data = worker_component->data;
+	return (data->status == COMPONENT_STATUS_SLEEPING);
+}
+
+#ifndef STARPU_NO_ASSERT
+static int _worker_consistant(struct starpu_sched_component * component)
+{
+	int is_a_worker = 0;
+	int i;
+	for(i = 0; i<STARPU_NMAXWORKERS; i++)
+		if(_worker_components[i] == component)
+			is_a_worker = 1;
+	if(!is_a_worker)
+		return 0;
+	struct _starpu_worker_component_data * data = component->data;
+	if(data->worker)
+	{
+		int id = data->worker->workerid;
+		return  (_worker_components[id] == component)
+			&&  component->nchildren == 0;
+	}
+	return 1;
+}
+#endif
+
+
+
+/******************************************************************************
+ *				Simple Worker Components' Interface Functions			      *
+ *****************************************************************************/
+
+
+
+static void simple_worker_can_pull(struct starpu_sched_component * worker_component)
+{
+	(void) worker_component;
+	struct _starpu_worker * w = _starpu_sched_component_worker_get_worker(worker_component);
+	_starpu_sched_component_lock_worker(w->workerid);
+	if(_starpu_sched_component_worker_is_reset_status(worker_component))
+		_starpu_sched_component_worker_set_changed_status(worker_component);
+
+	if(w->workerid == starpu_worker_get_id())
+	{
+		_starpu_sched_component_unlock_worker(w->workerid);
+		return;
+	}
+	if(_starpu_sched_component_worker_is_sleeping_status(worker_component))
+	{
+		starpu_pthread_mutex_t *sched_mutex;
+		starpu_pthread_cond_t *sched_cond;
+		starpu_worker_get_sched_condition(w->workerid, &sched_mutex, &sched_cond);
+		_starpu_sched_component_unlock_worker(w->workerid);
+		starpu_wakeup_worker(w->workerid, sched_cond, sched_mutex);
+	}
+	else
+		_starpu_sched_component_unlock_worker(w->workerid);
+}
+
+static int simple_worker_push_task(struct starpu_sched_component * component, struct starpu_task *task)
+{
+	STARPU_ASSERT(starpu_sched_component_is_worker(component));
+	/*this function take the worker's mutex */
+	struct _starpu_worker_component_data * data = component->data;
+	struct _starpu_task_grid * t = _starpu_task_grid_create();
+	t->task = task;
+	t->ntasks = 1;
+
+	task->workerid = starpu_bitmap_first(component->workers);
+#if 1 /* dead lock problem? */
+	if (starpu_get_prefetch_flag())
+	{
+		unsigned memory_node = starpu_worker_get_memory_node(task->workerid);
+		starpu_prefetch_task_input_on_node(task, memory_node);
+	}
+#endif
+	STARPU_PTHREAD_MUTEX_LOCK(&data->list->mutex);
+	_starpu_worker_task_list_push(data->list, t);
+	STARPU_PTHREAD_MUTEX_UNLOCK(&data->list->mutex);
+	simple_worker_can_pull(component);	
+	return 0;
+}
+
+static struct starpu_task * simple_worker_pull_task(struct starpu_sched_component *component)
+{
+	int workerid = starpu_worker_get_id();
+	struct _starpu_worker_component_data * data = component->data;
+	struct _starpu_worker_task_list * list = data->list;
+	STARPU_PTHREAD_MUTEX_LOCK(&list->mutex);
+	struct starpu_task * task =  _starpu_worker_task_list_pop(list);
+	STARPU_PTHREAD_MUTEX_UNLOCK(&list->mutex);
+	if(task)
+	{
+		starpu_push_task_end(task);
+		return task;
+	}
+	_starpu_sched_component_lock_worker(workerid);	
+	int i;
+	do {
+		_starpu_sched_component_worker_reset_status(component);
+		for(i=0; i < component->nparents; i++)
+		{
+			if(component->parents[i] == NULL)
+				continue;
+			else
+			{
+				_starpu_sched_component_worker_unlock_scheduling();
+				task = component->parents[i]->pull_task(component->parents[i]);
+				_starpu_sched_component_worker_lock_scheduling();
+				if(task)
+					break;
+			}
+		}
+	} while((!task) && _starpu_sched_component_worker_is_changed_status(component));
+	_starpu_sched_component_worker_set_sleep_status(component);
+	_starpu_sched_component_unlock_worker(workerid);	
+	if(!task)
+		return NULL;
+	if(task->cl->type == STARPU_SPMD)
+	{
+		int workerid = starpu_worker_get_id();
+		if(!starpu_worker_is_combined_worker(workerid))
+		{
+			starpu_push_task_end(task);
+			return task;
+		}
+		struct starpu_sched_component * combined_worker_component = starpu_sched_component_worker_get(workerid);
+		(void)combined_worker_component->push_task(combined_worker_component, task);
+		/* we have pushed a task in queue, so can make a recursive call */
+		return simple_worker_pull_task(component);
+
+	}
+	if(task)
+		starpu_push_task_end(task);
+	return task;
+}
+
+static double simple_worker_estimated_end(struct starpu_sched_component * component)
+{
+	struct _starpu_worker_component_data * data = component->data;
+	STARPU_PTHREAD_MUTEX_LOCK(&data->list->mutex);
+	data->list->exp_start = STARPU_MAX(starpu_timing_now(), data->list->exp_start);
+	double tmp = data->list->exp_end = data->list->exp_start + data->list->exp_len;
+	STARPU_PTHREAD_MUTEX_UNLOCK(&data->list->mutex);
+	return tmp;
+}
+
+static double simple_worker_estimated_load(struct starpu_sched_component * component)
+{
+	struct _starpu_worker * worker = _starpu_sched_component_worker_get_worker(component);
+	int nb_task = 0;
+	STARPU_PTHREAD_MUTEX_LOCK(&worker->mutex);
+	struct starpu_task_list list = worker->local_tasks;
+	struct starpu_task * task;
+	for(task = starpu_task_list_front(&list);
+	    task != starpu_task_list_end(&list);
+	    task = starpu_task_list_next(task))
+		nb_task++;
+	STARPU_PTHREAD_MUTEX_UNLOCK(&worker->mutex);
+	struct _starpu_worker_component_data * d = component->data;
+	struct _starpu_worker_task_list * l = d->list;
+	int ntasks_in_fifo = l ? l->ntasks : 0;
+	return (double) (nb_task + ntasks_in_fifo)
+		/ starpu_worker_get_relative_speedup(
+				starpu_worker_get_perf_archtype(starpu_bitmap_first(component->workers)));
+}
+
+static void _worker_component_deinit_data(struct starpu_sched_component * component)
+{
+	struct _starpu_worker_component_data * d = component->data;
+	_starpu_worker_task_list_destroy(d->list);
+	if(starpu_sched_component_is_simple_worker(component))
+		STARPU_PTHREAD_MUTEX_DESTROY(&d->lock);
+	int i;
+	for(i = 0; i < STARPU_NMAXWORKERS; i++)
+		if(_worker_components[i] == component)
+		{
+			_worker_components[i] = NULL;
+			break;
+		}
+	free(d);
+}
+
+static struct starpu_sched_component * starpu_sched_component_worker_create(int workerid)
+{
+	STARPU_ASSERT(0 <=  workerid && workerid < (int) starpu_worker_get_count());
+
+	if(_worker_components[workerid])
+		return _worker_components[workerid];
+
+	struct _starpu_worker * worker = _starpu_get_worker_struct(workerid);
+	if(worker == NULL)
+		return NULL;
+	struct starpu_sched_component * component = starpu_sched_component_create();
+	struct _starpu_worker_component_data * data = malloc(sizeof(*data));
+	memset(data, 0, sizeof(*data));
+
+	data->worker = worker;
+	STARPU_PTHREAD_MUTEX_INIT(&data->lock,NULL);
+	data->status = COMPONENT_STATUS_SLEEPING;
+	data->list = _starpu_worker_task_list_create();
+	component->data = data;
+
+	component->push_task = simple_worker_push_task;
+	component->pull_task = simple_worker_pull_task;
+	component->can_pull = simple_worker_can_pull;
+	component->estimated_end = simple_worker_estimated_end;
+	component->estimated_load = simple_worker_estimated_load;
+	component->deinit_data = _worker_component_deinit_data;
+	starpu_bitmap_set(component->workers, workerid);
+	starpu_bitmap_or(component->workers_in_ctx, component->workers);
+	_worker_components[workerid] = component;
+
+	/*
+#ifdef STARPU_HAVE_HWLOC
+	struct _starpu_machine_config *config = _starpu_get_machine_config();
+	struct _starpu_machine_topology *topology = &config->topology;
+	hwloc_obj_t obj = hwloc_get_obj_by_depth(topology->hwtopology, config->cpu_depth, worker->bindid);
+	STARPU_ASSERT(obj);
+	component->obj = obj;
+#endif
+	*/
+
+	return component;
+}
+
+
+
+/******************************************************************************
+ *				Combined Worker Components' Interface Functions			      *
+ *****************************************************************************/
+
+
+
+static void combined_worker_can_pull(struct starpu_sched_component * component)
+{
+	(void) component;
+	STARPU_ASSERT(starpu_sched_component_is_combined_worker(component));
+	struct _starpu_worker_component_data * data = component->data;
+	int workerid = starpu_worker_get_id();
+	int i;
+	for(i = 0; i < data->combined_worker->worker_size; i++)
+	{
+		if(i == workerid)
+			continue;
+		int worker = data->combined_worker->combined_workerid[i];
+		_starpu_sched_component_lock_worker(worker);
+		if(_starpu_sched_component_worker_is_sleeping_status(component))
+		{
+			starpu_pthread_mutex_t *sched_mutex;
+			starpu_pthread_cond_t *sched_cond;
+			starpu_worker_get_sched_condition(worker, &sched_mutex, &sched_cond);
+			starpu_wakeup_worker(worker, sched_cond, sched_mutex);
+		}
+		if(_starpu_sched_component_worker_is_reset_status(component))
+			_starpu_sched_component_worker_set_changed_status(component);
+
+		_starpu_sched_component_unlock_worker(worker);
+	}
+}
+
+static int combined_worker_push_task(struct starpu_sched_component * component, struct starpu_task *task)
+{
+	STARPU_ASSERT(starpu_sched_component_is_combined_worker(component));
+	struct _starpu_worker_component_data * data = component->data;
+	STARPU_ASSERT(data->combined_worker && !data->worker);
+	struct _starpu_combined_worker  * combined_worker = data->combined_worker;
+	STARPU_ASSERT(combined_worker->worker_size >= 1);
+	struct _starpu_task_grid * task_alias[combined_worker->worker_size];
+	starpu_parallel_task_barrier_init(task, starpu_bitmap_first(component->workers));
+	task_alias[0] = _starpu_task_grid_create();
+	task_alias[0]->task = starpu_task_dup(task);
+	task_alias[0]->task->workerid = combined_worker->combined_workerid[0];
+	task_alias[0]->left = NULL;
+	task_alias[0]->ntasks = combined_worker->worker_size;
+	int i;
+	for(i = 1; i < combined_worker->worker_size; i++)
+	{
+		task_alias[i] = _starpu_task_grid_create();
+		task_alias[i]->task = starpu_task_dup(task);
+		task_alias[i]->task->workerid = combined_worker->combined_workerid[i];
+		task_alias[i]->left = task_alias[i-1];
+		task_alias[i - 1]->right = task_alias[i];
+		task_alias[i]->pntasks = &task_alias[0]->ntasks;
+	}
+
+	starpu_pthread_mutex_t * mutex_to_unlock = NULL;
+	i = 0;
+	do
+	{
+		struct starpu_sched_component * worker_component = starpu_sched_component_worker_get(combined_worker->combined_workerid[i]);
+		struct _starpu_worker_component_data * worker_data = worker_component->data;
+		struct _starpu_worker_task_list * list = worker_data->list;
+		STARPU_PTHREAD_MUTEX_LOCK(&list->mutex);
+		if(mutex_to_unlock)
+			STARPU_PTHREAD_MUTEX_UNLOCK(mutex_to_unlock);
+		mutex_to_unlock = &list->mutex;
+
+		_starpu_worker_task_list_push(list, task_alias[i]);
+		i++;
+	}
+	while(i < combined_worker->worker_size);
+	
+	STARPU_PTHREAD_MUTEX_UNLOCK(mutex_to_unlock);
+
+	int workerid = starpu_worker_get_id();
+	if(-1 == workerid)
+	{
+		combined_worker_can_pull(component);
+	}
+	else
+	{
+		starpu_pthread_mutex_t *worker_sched_mutex;
+		starpu_pthread_cond_t *worker_sched_cond;
+		starpu_worker_get_sched_condition(workerid, &worker_sched_mutex, &worker_sched_cond);
+		STARPU_PTHREAD_MUTEX_UNLOCK(worker_sched_mutex);
+
+		/* wake up all other workers of combined worker */
+		for(i = 0; i < combined_worker->worker_size; i++)
+		{
+			struct starpu_sched_component * worker_component = starpu_sched_component_worker_get(combined_worker->combined_workerid[i]);
+			simple_worker_can_pull(worker_component);
+		}
+
+		combined_worker_can_pull(component);
+
+		STARPU_PTHREAD_MUTEX_LOCK(worker_sched_mutex);
+	}
+
+	return 0;
+}
+
+static double combined_worker_estimated_end(struct starpu_sched_component * component)
+{
+	STARPU_ASSERT(starpu_sched_component_is_combined_worker(component));
+	struct _starpu_worker_component_data * data = component->data;
+	struct _starpu_combined_worker * combined_worker = data->combined_worker;
+	double max = 0.0;
+	int i;
+	for(i = 0; i < combined_worker->worker_size; i++)
+	{
+		data = _worker_components[combined_worker->combined_workerid[i]]->data;
+		STARPU_PTHREAD_MUTEX_LOCK(&data->list->mutex);
+		double tmp = data->list->exp_end;
+		STARPU_PTHREAD_MUTEX_UNLOCK(&data->list->mutex);
+		max = tmp > max ? tmp : max;
+	}
+	return max;
+}
+
+static double combined_worker_estimated_load(struct starpu_sched_component * component)
+{
+	struct _starpu_worker_component_data * d = component->data;
+	struct _starpu_combined_worker * c = d->combined_worker;
+	double load = 0;
+	int i;
+	for(i = 0; i < c->worker_size; i++)
+	{
+		struct starpu_sched_component * n = starpu_sched_component_worker_get(c->combined_workerid[i]);
+		load += n->estimated_load(n);
+	}
+	return load;
+}
+
+static struct starpu_sched_component  * starpu_sched_component_combined_worker_create(int workerid)
+{
+	STARPU_ASSERT(0 <= workerid && workerid <  STARPU_NMAXWORKERS);
+
+	if(_worker_components[workerid])
+		return _worker_components[workerid];
+
+	struct _starpu_combined_worker * combined_worker = _starpu_get_combined_worker_struct(workerid);
+	if(combined_worker == NULL)
+		return NULL;
+	struct starpu_sched_component * component = starpu_sched_component_create();
+	struct _starpu_worker_component_data * data = malloc(sizeof(*data));
+	memset(data, 0, sizeof(*data));
+	data->combined_worker = combined_worker;
+	data->status = COMPONENT_STATUS_SLEEPING;
+
+	component->data = data;
+	component->push_task = combined_worker_push_task;
+	component->pull_task = NULL;
+	component->estimated_end = combined_worker_estimated_end;
+	component->estimated_load = combined_worker_estimated_load;
+	component->can_pull = combined_worker_can_pull;
+	component->deinit_data = _worker_component_deinit_data;
+	starpu_bitmap_set(component->workers, workerid);
+	starpu_bitmap_or(component->workers_in_ctx, component->workers);
+	_worker_components[workerid] = component;
+
+#ifdef STARPU_HAVE_HWLOC
+	struct _starpu_machine_config *config = _starpu_get_machine_config();
+	struct _starpu_machine_topology *topology = &config->topology;
+	hwloc_obj_t obj = hwloc_get_obj_by_depth(topology->hwtopology, config->cpu_depth, combined_worker->combined_workerid[0]);
+	STARPU_ASSERT(obj);
+	component->obj = obj;
+#endif
+	return component;
+}
+
+
+
+/******************************************************************************
+ *			Worker Components' Public Helper Functions (Part 2)			      *
+ *****************************************************************************/
+
+
+
+void _starpu_sched_component_lock_all_workers(void)
+{
+	unsigned i;
+	for(i = 0; i < starpu_worker_get_count(); i++)
+		_starpu_sched_component_lock_worker(i);
+}
+void _starpu_sched_component_unlock_all_workers(void)
+{
+	unsigned i;
+	for(i = 0; i < starpu_worker_get_count(); i++)
+		_starpu_sched_component_unlock_worker(i);
+}
+
+void _starpu_sched_component_workers_destroy(void)
+{
+	int i;
+	for(i = 0; i < STARPU_NMAXWORKERS; i++)
+		if (_worker_components[i])
+			starpu_sched_component_destroy(_worker_components[i]);
+}
+
+int starpu_sched_component_worker_get_workerid(struct starpu_sched_component * worker_component)
+{
+#ifndef STARPU_NO_ASSERT
+	STARPU_ASSERT(_worker_consistant(worker_component));
+#endif
+	STARPU_ASSERT(1 == starpu_bitmap_cardinal(worker_component->workers));
+	return starpu_bitmap_first(worker_component->workers);
+}
+
+void starpu_sched_component_worker_pre_exec_hook(struct starpu_task * task)
+{
+	if(!isnan(task->predicted))
+	{
+		struct _starpu_worker_task_list * list = _worker_get_list();
+		STARPU_PTHREAD_MUTEX_LOCK(&list->mutex);
+
+		list->exp_start = starpu_timing_now() + task->predicted;
+
+		if(list->ntasks == 0)
+		{
+			list->exp_end = list->exp_start;
+			list->exp_len = 0.0;
+		}
+		else
+			list->exp_end = list->exp_start + list->exp_len;
+		STARPU_PTHREAD_MUTEX_UNLOCK(&list->mutex);
+	}
+}
+
+void starpu_sched_component_worker_post_exec_hook(struct starpu_task * task)
+{
+	if(task->execute_on_a_specific_worker)
+		return;
+	struct _starpu_worker_task_list * list = _worker_get_list();
+	STARPU_PTHREAD_MUTEX_LOCK(&list->mutex);
+	list->exp_start = starpu_timing_now();
+	list->exp_end = list->exp_start + list->exp_len;
+	STARPU_PTHREAD_MUTEX_UNLOCK(&list->mutex);
+}
+
+int starpu_sched_component_is_simple_worker(struct starpu_sched_component * component)
+{
+	return component->push_task == simple_worker_push_task;
+}
+int starpu_sched_component_is_combined_worker(struct starpu_sched_component * component)
+{
+	return component->push_task == combined_worker_push_task;
+}
+
+int starpu_sched_component_is_worker(struct starpu_sched_component * component)
+{
+	return starpu_sched_component_is_simple_worker(component)
+		|| starpu_sched_component_is_combined_worker(component);
+}
+
+/* As Worker Components' creating functions are protected, this function allows
+ * the user to get a Worker Component from a worker id */
+struct starpu_sched_component * starpu_sched_component_worker_get(int workerid)
+{
+	STARPU_ASSERT(workerid >= 0 && workerid < STARPU_NMAXWORKERS);
+	/* we may need to take a mutex here */
+	if(_worker_components[workerid])
+		return _worker_components[workerid];
+	else
+	{
+		struct starpu_sched_component * component;
+		if(workerid < (int) starpu_worker_get_count())
+			component = starpu_sched_component_worker_create(workerid);
+		else
+			component = starpu_sched_component_combined_worker_create(workerid);
+		_worker_components[workerid] = component;
+		return component;
+	}
+}

src/sched_policies/fifo_queues.c

@@ -22,6 +22,24 @@
 
 #include <sched_policies/fifo_queues.h>
 #include <common/fxt.h>
+/*
+static int is_sorted_task_list(struct starpu_task * task)
+{
+	if(!task)
+		return 1;
+	struct starpu_task * next = task->next;
+	if(!next)
+		return 1;
+	while(next)
+	{
+		if(task->priority < next->priority)
+			return 0;
+		task = next;
+		next = next->next;
+	}
+	return 1;
+}
+*/
 
 struct _starpu_fifo_taskq *_starpu_create_fifo(void)
 {
@@ -130,6 +148,23 @@ int _starpu_fifo_push_task(struct _starpu_fifo_taskq *fifo_queue, struct starpu_
 	return 0;
 }
 
+int _starpu_fifo_push_back_task(struct _starpu_fifo_taskq *fifo_queue, struct starpu_task *task)
+{
+
+	if (task->priority > 0)
+	{
+		_starpu_fifo_push_sorted_task(fifo_queue, task);
+	}
+	else
+	{
+		starpu_task_list_push_front(&fifo_queue->taskq, task);
+
+		fifo_queue->ntasks++;
+	}
+
+	return 0;
+}
+
 struct starpu_task *_starpu_fifo_pop_task(struct _starpu_fifo_taskq *fifo_queue, int workerid)
 {
 	struct starpu_task *task;

src/sched_policies/fifo_queues.h

@@ -48,6 +48,7 @@ int _starpu_fifo_empty(struct _starpu_fifo_taskq *fifo);
 int _starpu_fifo_push_sorted_task(struct _starpu_fifo_taskq *fifo_queue, struct starpu_task *task);
 
 int _starpu_fifo_push_task(struct _starpu_fifo_taskq *fifo, struct starpu_task *task);
+int _starpu_fifo_push_back_task(struct _starpu_fifo_taskq *fifo_queue, struct starpu_task *task);
 
 struct starpu_task *_starpu_fifo_pop_task(struct _starpu_fifo_taskq *fifo, int workerid);
 struct starpu_task *_starpu_fifo_pop_local_task(struct _starpu_fifo_taskq *fifo);

src/sched_policies/helper_mct.c

@@ -0,0 +1,176 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2013  Université de Bordeaux 1
+ * Copyright (C) 2013  INRIA
+ * Copyright (C) 2013  Simon Archipoff
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <starpu_sched_component.h>
+#include "helper_mct.h"
+
+/* Alpha, Beta and Gamma are MCT-specific values, which allows the
+ * user to set more precisely the weight of each computing value.
+ * Beta, for example, controls the weight of communications between
+ * memories for the computation of the best component to choose. 
+ */
+#define _STARPU_SCHED_ALPHA_DEFAULT 1.0
+#define _STARPU_SCHED_BETA_DEFAULT 1.0
+#define _STARPU_SCHED_GAMMA_DEFAULT 1000.0
+
+#ifdef STARPU_USE_TOP
+static void param_modified(struct starpu_top_param* d)
+{
+	/* Just to show parameter modification. */
+	fprintf(stderr, "%s has been modified : %f\n",
+			d->name, *(double*) d->value);
+}
+#endif /* !STARPU_USE_TOP */
+
+#ifdef STARPU_USE_TOP
+static const float alpha_minimum=0;
+static const float alpha_maximum=10.0;
+static const float beta_minimum=0;
+static const float beta_maximum=10.0;
+static const float gamma_minimum=0;
+static const float gamma_maximum=10000.0;
+static const float idle_power_minimum=0;
+static const float idle_power_maximum=10000.0;
+#endif /* !STARPU_USE_TOP */
+
+struct _starpu_mct_data *starpu_mct_init_parameters(struct starpu_mct_data *params)
+{
+	struct _starpu_mct_data * data = malloc(sizeof(*data));
+	if (params)
+	{
+		data->alpha = params->alpha;
+		data->beta = params->beta;
+		data->gamma = params->gamma;
+		data->idle_power = params->idle_power;
+	}
+	else
+	{
+		double alpha = _STARPU_SCHED_ALPHA_DEFAULT,
+		       beta = _STARPU_SCHED_BETA_DEFAULT,
+		       _gamma = _STARPU_SCHED_GAMMA_DEFAULT,
+		       idle_power = 0.0;
+
+		const char *strval_alpha = getenv("STARPU_SCHED_ALPHA");
+		if (strval_alpha)
+			alpha = atof(strval_alpha);
+
+		const char *strval_beta = getenv("STARPU_SCHED_BETA");
+		if (strval_beta)
+			beta = atof(strval_beta);
+
+		const char *strval_gamma = getenv("STARPU_SCHED_GAMMA");
+		if (strval_gamma)
+			_gamma = atof(strval_gamma);
+
+		const char *strval_idle_power = getenv("STARPU_IDLE_POWER");
+		if (strval_idle_power)
+			idle_power = atof(strval_idle_power);
+
+		data->alpha = alpha;
+		data->beta = beta;
+		data->gamma = _gamma;
+		data->idle_power = idle_power;
+	}
+
+#ifdef STARPU_USE_TOP
+	starpu_top_register_parameter_float("MCT_ALPHA", &data->alpha,
+					    alpha_minimum, alpha_maximum, param_modified);
+	starpu_top_register_parameter_float("MCT_BETA", &data->beta,
+					    beta_minimum, beta_maximum, param_modified);
+	starpu_top_register_parameter_float("MCT_GAMMA", &data->gamma,
+					    gamma_minimum, gamma_maximum, param_modified);
+	starpu_top_register_parameter_float("MCT_IDLE_POWER", &data->idle_power,
+					    idle_power_minimum, idle_power_maximum, param_modified);
+#endif /* !STARPU_USE_TOP */
+
+	return data;
+}
+
+/* compute predicted_end by taking into account the case of the predicted transfer and the predicted_end overlap
+ */
+static double compute_expected_time(double now, double predicted_end, double predicted_length, double predicted_transfer)
+{
+	STARPU_ASSERT(!isnan(now + predicted_end + predicted_length + predicted_transfer));
+	STARPU_ASSERT(now >= 0.0 && predicted_end >= 0.0 && predicted_length >= 0.0 && predicted_transfer >= 0.0);
+
+	/* TODO: actually schedule transfers */
+	if (now + predicted_transfer < predicted_end)
+	{
+		/* We may hope that the transfer will be finished by
+		 * the start of the task. */
+		predicted_transfer = 0;
+	}
+	else
+	{
+		/* The transfer will not be finished by then, take the
+		 * remainder into account */
+		predicted_transfer -= (predicted_end - now);
+	}
+
+	predicted_end += predicted_transfer;
+	predicted_end += predicted_length;
+
+	return predicted_end;
+}
+
+double starpu_mct_compute_fitness(struct _starpu_mct_data * d, double exp_end, double min_exp_end, double max_exp_end, double transfer_len, double local_power)
+{
+	/* Note: the expected end includes the data transfer duration, which we want to be able to tune separately */
+
+	return d->alpha * (exp_end - min_exp_end)
+		+ d->beta * transfer_len
+		+ d->gamma * local_power
+		+ d->gamma * d->idle_power * (exp_end - max_exp_end);
+}
+
+int starpu_mct_compute_expected_times(struct starpu_sched_component *component, struct starpu_task *task,
+		double *estimated_lengths, double *estimated_transfer_length, double *estimated_ends_with_task,
+		double *min_exp_end_with_task, double *max_exp_end_with_task, int *suitable_components)
+{
+	int nsuitable_components = 0;
+
+	int i;
+	for(i = 0; i < component->nchildren; i++)
+	{
+		struct starpu_sched_component * c = component->children[i];
+		if(starpu_sched_component_execute_preds(c, task, estimated_lengths + i))
+		{
+			if(isnan(estimated_lengths[i]))
+				/* The perfmodel had been purged since the task was pushed
+				 * onto the mct component. */
+				continue;
+
+			/* Estimated availability of worker */
+			double estimated_end = c->estimated_end(c);
+			double now = starpu_timing_now();
+			if (estimated_end < now)
+				estimated_end = now;
+			estimated_transfer_length[i] = starpu_sched_component_transfer_length(c, task);
+			estimated_ends_with_task[i] = compute_expected_time(now,
+									    estimated_end,
+									    estimated_lengths[i],
+									    estimated_transfer_length[i]);
+			if(estimated_ends_with_task[i] < *min_exp_end_with_task)
+				*min_exp_end_with_task = estimated_ends_with_task[i];
+			if(estimated_ends_with_task[i] > *max_exp_end_with_task)
+				*max_exp_end_with_task = estimated_ends_with_task[i];
+			suitable_components[nsuitable_components++] = i;
+		}
+	}
+	return nsuitable_components;
+}

src/sched_policies/helper_mct.h

@@ -0,0 +1,31 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2013  Université de Bordeaux 1
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+struct _starpu_mct_data
+{
+	double alpha;
+	double beta;
+	double gamma;
+	double idle_power;
+};
+
+struct _starpu_mct_data *starpu_mct_init_parameters(struct starpu_mct_data *params);
+
+int starpu_mct_compute_expected_times(struct starpu_sched_component *component, struct starpu_task *task,
+		double *estimated_lengths, double *estimated_transfer_length, double *estimated_ends_with_task,
+		double *min_exp_end_with_task, double *max_exp_end_with_task, int *suitable_components);
+
+double starpu_mct_compute_fitness(struct _starpu_mct_data * d, double exp_end, double min_exp_end, double max_exp_end, double transfer_len, double local_power);

src/sched_policies/hierarchical_heft.c

@@ -0,0 +1,78 @@
+#include <starpu_sched_component.h>
+#include <core/workers.h>
+
+static struct  starpu_sched_component_composed_recipe *  recipe_for_worker(enum starpu_worker_archtype a STARPU_ATTRIBUTE_UNUSED)
+{
+	struct starpu_sched_component_composed_recipe * r = starpu_sched_component_create_recipe();
+	starpu_sched_recipe_add_component(r, starpu_sched_component_best_implementation_create, NULL);
+	starpu_sched_recipe_add_component(r, starpu_sched_component_fifo_create, NULL);
+	return r;
+}
+
+
+
+
+static void initialize_heft_center_policy(unsigned sched_ctx_id)
+{
+	starpu_sched_ctx_create_worker_collection(sched_ctx_id, STARPU_WORKER_LIST);
+
+	struct starpu_sched_specs specs;
+	memset(&specs,0,sizeof(specs));
+
+	struct starpu_heft_data heft_data =
+	{
+		.alpha = 1.0,
+		.beta = 2.0,
+		.gamma = 0.0,
+		.idle_power = 0.0,
+		.no_perf_model_component_create = starpu_sched_component_random_create,
+		.arg_no_perf_model = NULL,
+		.calibrating_component_create = starpu_sched_component_random_create,
+		.arg_calibrating_component = NULL,
+	};
+	struct starpu_sched_component_composed_recipe * r = starpu_sched_component_create_recipe();
+	starpu_sched_recipe_add_component(r,(struct starpu_sched_component * (*)(void*))starpu_sched_component_heft_create,&heft_data);
+	specs.hwloc_machine_composed_sched_component = r;
+
+	r = starpu_sched_component_create_recipe();
+	starpu_sched_recipe_add_component(r, starpu_sched_component_best_implementation_create, NULL);
+	starpu_sched_recipe_add_component(r, starpu_sched_component_fifo_create ,NULL);
+
+	specs.hwloc_component_composed_sched_component = r;
+	specs.worker_composed_sched_component = recipe_for_worker;
+
+	struct starpu_sched_tree *t = starpu_sched_component_make_scheduler(sched_ctx_id, specs);
+
+	starpu_destroy_composed_sched_component_recipe(specs.hwloc_machine_composed_sched_component);
+
+
+	starpu_sched_tree_update_workers(t);
+	starpu_sched_ctx_set_policy_data(sched_ctx_id, (void*)t);
+}
+
+static void deinitialize_heft_center_policy(unsigned sched_ctx_id)
+{
+	struct starpu_sched_tree *t = (struct starpu_sched_tree*)starpu_sched_ctx_get_policy_data(sched_ctx_id);
+
+	starpu_sched_tree_destroy(t);
+	starpu_sched_ctx_delete_worker_collection(sched_ctx_id);
+}
+
+
+
+
+
+struct starpu_sched_policy _starpu_sched_tree_heft_hierarchical_policy =
+{
+	.init_sched = initialize_heft_center_policy,
+	.deinit_sched = deinitialize_heft_center_policy,
+	.add_workers = starpu_sched_tree_add_workers,
+	.remove_workers = starpu_sched_tree_remove_workers,
+	.push_task = starpu_sched_tree_push_task,
+	.pop_task = starpu_sched_tree_pop_task,
+	.pre_exec_hook = NULL,
+	.post_exec_hook = NULL,
+	.pop_every_task = NULL,
+	.policy_name = "tree-heft-hierarchical",
+	.policy_description = "hierarchical heft tree policy"
+};

src/sched_policies/modular_eager.c

@@ -0,0 +1,65 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2013  INRIA
+ * Copyright (C) 2013  Simon Archipoff
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <starpu_sched_component.h>
+#include <starpu_scheduler.h>
+
+static void initialize_eager_center_policy(unsigned sched_ctx_id)
+{
+	_STARPU_DISP("Warning: you are running the default modular-eager scheduler, which is not very smart. Make sure to read the StarPU documentation about adding performance models in order to be able to use the modular-heft scheduler instead.\n");
+
+	starpu_sched_ctx_create_worker_collection(sched_ctx_id, STARPU_WORKER_LIST);
+	struct starpu_sched_tree *t = starpu_sched_tree_create(sched_ctx_id);
+ 	t->root = starpu_sched_component_fifo_create(NULL);
+	struct starpu_sched_component * eager_component = starpu_sched_component_eager_create(NULL);
+	t->root->add_child(t->root, eager_component);
+	eager_component->add_parent(eager_component, t->root);
+
+	unsigned i;
+	for(i = 0; i < starpu_worker_get_count() + starpu_combined_worker_get_count(); i++)
+	{
+		struct starpu_sched_component * worker_component = starpu_sched_component_worker_get(i);
+		STARPU_ASSERT(worker_component);
+
+		eager_component->add_child(eager_component, worker_component);
+		worker_component->add_parent(worker_component, eager_component);
+	}
+	starpu_sched_tree_update_workers(t);
+	starpu_sched_ctx_set_policy_data(sched_ctx_id, (void*)t);
+}
+
+static void deinitialize_eager_center_policy(unsigned sched_ctx_id)
+{
+	struct starpu_sched_tree *tree = (struct starpu_sched_tree*)starpu_sched_ctx_get_policy_data(sched_ctx_id);
+	starpu_sched_tree_destroy(tree);
+	starpu_sched_ctx_delete_worker_collection(sched_ctx_id);
+}
+
+struct starpu_sched_policy _starpu_sched_modular_eager_policy =
+{
+	.init_sched = initialize_eager_center_policy,
+	.deinit_sched = deinitialize_eager_center_policy,
+	.add_workers = starpu_sched_tree_add_workers,
+	.remove_workers = starpu_sched_tree_remove_workers,
+	.push_task = starpu_sched_tree_push_task,
+	.pop_task = starpu_sched_tree_pop_task,
+	.pre_exec_hook = starpu_sched_component_worker_pre_exec_hook,
+	.post_exec_hook = starpu_sched_component_worker_post_exec_hook,
+	.pop_every_task = NULL,
+	.policy_name = "modular-eager",
+	.policy_description = "eager modular policy"
+};

src/sched_policies/modular_eager_prefetching.c

@@ -0,0 +1,89 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2013  INRIA
+ * Copyright (C) 2013  Simon Archipoff
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <starpu_sched_component.h>
+#include <starpu_scheduler.h>
+
+#define _STARPU_SCHED_NTASKS_THRESHOLD_DEFAULT 2
+#define _STARPU_SCHED_EXP_LEN_THRESHOLD_DEFAULT 1000000000.0
+
+static void initialize_eager_prefetching_center_policy(unsigned sched_ctx_id)
+{
+	unsigned ntasks_threshold = _STARPU_SCHED_NTASKS_THRESHOLD_DEFAULT;
+	double exp_len_threshold = _STARPU_SCHED_EXP_LEN_THRESHOLD_DEFAULT;
+
+	_STARPU_DISP("Warning: you are running the default modular-eager-prefetching scheduler, which is not very smart. Make sure to read the StarPU documentation about adding performance models in order to be able to use the modular-heft scheduler instead.\n");
+
+	const char *strval_ntasks_threshold = getenv("STARPU_NTASKS_THRESHOLD");
+	if (strval_ntasks_threshold)
+		ntasks_threshold = atof(strval_ntasks_threshold);
+
+	const char *strval_exp_len_threshold = getenv("STARPU_EXP_LEN_THRESHOLD");
+	if (strval_exp_len_threshold)
+		exp_len_threshold = atof(strval_exp_len_threshold);
+
+	starpu_sched_ctx_create_worker_collection(sched_ctx_id, STARPU_WORKER_LIST);
+	struct starpu_sched_tree *t = starpu_sched_tree_create(sched_ctx_id);
+ 	t->root = starpu_sched_component_fifo_create(NULL);
+	struct starpu_sched_component * eager_component = starpu_sched_component_eager_create(NULL);
+	t->root->add_child(t->root, eager_component);
+	eager_component->add_parent(eager_component, t->root);
+
+	struct starpu_fifo_data fifo_data =
+		{
+			.ntasks_threshold = ntasks_threshold,
+			.exp_len_threshold = exp_len_threshold,
+		};
+
+	unsigned i;
+	for(i = 0; i < starpu_worker_get_count() + starpu_combined_worker_get_count(); i++)
+	{
+		struct starpu_sched_component * worker_component = starpu_sched_component_worker_get(i);
+		STARPU_ASSERT(worker_component);
+
+		struct starpu_sched_component * fifo_component = starpu_sched_component_fifo_create(&fifo_data);
+		fifo_component->add_child(fifo_component, worker_component);
+		worker_component->add_parent(worker_component, fifo_component);
+
+		eager_component->add_child(eager_component, fifo_component);
+		fifo_component->add_parent(fifo_component, eager_component);
+	}
+	starpu_sched_tree_update_workers(t);
+	starpu_sched_ctx_set_policy_data(sched_ctx_id, (void*)t);
+}
+
+static void deinitialize_eager_prefetching_center_policy(unsigned sched_ctx_id)
+{
+	struct starpu_sched_tree *tree = (struct starpu_sched_tree*)starpu_sched_ctx_get_policy_data(sched_ctx_id);
+	starpu_sched_tree_destroy(tree);
+	starpu_sched_ctx_delete_worker_collection(sched_ctx_id);
+}
+
+struct starpu_sched_policy _starpu_sched_modular_eager_prefetching_policy =
+{
+	.init_sched = initialize_eager_prefetching_center_policy,
+	.deinit_sched = deinitialize_eager_prefetching_center_policy,
+	.add_workers = starpu_sched_tree_add_workers,
+	.remove_workers = starpu_sched_tree_remove_workers,
+	.push_task = starpu_sched_tree_push_task,
+	.pop_task = starpu_sched_tree_pop_task,
+	.pre_exec_hook = starpu_sched_component_worker_pre_exec_hook,
+	.post_exec_hook = starpu_sched_component_worker_post_exec_hook,
+	.pop_every_task = NULL,
+	.policy_name = "modular-eager-prefetching",
+	.policy_description = "eager with prefetching modular policy"
+};

src/sched_policies/modular_heft.c

@@ -0,0 +1,161 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2013-2014  Université de Bordeaux 1
+ * Copyright (C) 2013  INRIA
+ * Copyright (C) 2013  Simon Archipoff
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <starpu_sched_component.h>
+#include <starpu_scheduler.h>
+#include <float.h>
+#include <limits.h>
+
+/* The two thresolds concerns the prio components, which contains queues
+ * who can handle the priority of StarPU tasks. You can tune your
+ * scheduling by benching those values and choose which one is the
+ * best for your current application. 
+ * The current value of the ntasks_threshold is the best we found
+ * so far across several types of applications (cholesky, LU, stencil).
+ */
+#define _STARPU_SCHED_NTASKS_THRESHOLD_DEFAULT 30
+#define _STARPU_SCHED_EXP_LEN_THRESHOLD_DEFAULT 1000000000.0
+
+static void initialize_heft_center_policy(unsigned sched_ctx_id)
+{
+	starpu_sched_ctx_create_worker_collection(sched_ctx_id, STARPU_WORKER_LIST);
+
+	/* The application may use any integer */
+	if (starpu_sched_ctx_min_priority_is_set(sched_ctx_id) == 0)
+		starpu_sched_ctx_set_min_priority(sched_ctx_id, INT_MIN);
+	if (starpu_sched_ctx_max_priority_is_set(sched_ctx_id) == 0)
+		starpu_sched_ctx_set_max_priority(sched_ctx_id, INT_MAX);
+
+	unsigned ntasks_threshold = _STARPU_SCHED_NTASKS_THRESHOLD_DEFAULT;
+	double exp_len_threshold = _STARPU_SCHED_EXP_LEN_THRESHOLD_DEFAULT;
+
+	const char *strval_ntasks_threshold = getenv("STARPU_NTASKS_THRESHOLD");
+	if (strval_ntasks_threshold)
+		ntasks_threshold = atof(strval_ntasks_threshold);
+
+	const char *strval_exp_len_threshold = getenv("STARPU_EXP_LEN_THRESHOLD");
+	if (strval_exp_len_threshold)
+		exp_len_threshold = atof(strval_exp_len_threshold);
+
+
+/* The scheduling strategy look like this :
+ *
+ *                                    |
+ *                              window_component
+ *                                    |
+ * perfmodel_component <--push-- perfmodel_select_component --push--> eager_component
+ *          |                                                    |
+ *          |                                                    |
+ *          >----------------------------------------------------<
+ *                    |                                |
+ *              best_impl_component                    best_impl_component
+ *                    |                                |
+ *                prio_component                        prio_component
+ *                    |                                |
+ *               worker_component                   worker_component
+ *
+ * A window contain the tasks that failed to be pushed, so as when the prio_components reclaim
+ * tasks by calling can_push to their parent (classically, just after a successful pop have
+ * been made by its associated worker_component), this call goes up to the window_component which
+ * pops a task from its local queue and try to schedule it by pushing it to the
+ * decision_component. 
+ * Finally, the task will be pushed to the prio_component which is the direct
+ * parent in the tree of the worker_component the task has been scheduled on. This
+ * component will push the task on its local queue if no one of the two thresholds
+ * have been reached for it, or send a push_error signal to its parent.
+ */
+	struct starpu_sched_tree * t = starpu_sched_tree_create(sched_ctx_id);
+
+	struct starpu_sched_component * window_component = starpu_sched_component_prio_create(NULL);
+	t->root = window_component;
+
+	struct starpu_sched_component * perfmodel_component = starpu_sched_component_mct_create(NULL);
+	struct starpu_sched_component * no_perfmodel_component = starpu_sched_component_eager_create(NULL);
+	struct starpu_sched_component * calibrator_component = starpu_sched_component_eager_calibration_create(NULL);
+	
+	struct starpu_perfmodel_select_data perfmodel_select_data =
+		{
+			.calibrator_component = calibrator_component,
+			.no_perfmodel_component = no_perfmodel_component,
+			.perfmodel_component = perfmodel_component,
+		};
+
+	struct starpu_sched_component * perfmodel_select_component = starpu_sched_component_perfmodel_select_create(&perfmodel_select_data);
+	window_component->add_child(window_component, perfmodel_select_component);
+	perfmodel_select_component->add_parent(perfmodel_select_component, window_component);
+
+	perfmodel_select_component->add_child(perfmodel_select_component, calibrator_component);
+	calibrator_component->add_parent(calibrator_component, perfmodel_select_component);
+	perfmodel_select_component->add_child(perfmodel_select_component, perfmodel_component);
+	perfmodel_component->add_parent(perfmodel_component, perfmodel_select_component);
+	perfmodel_select_component->add_child(perfmodel_select_component, no_perfmodel_component);
+	no_perfmodel_component->add_parent(no_perfmodel_component, perfmodel_select_component);
+
+	struct starpu_prio_data prio_data =
+		{
+			.ntasks_threshold = ntasks_threshold,
+			.exp_len_threshold = exp_len_threshold,
+		};
+
+	unsigned i;
+	for(i = 0; i < starpu_worker_get_count() + starpu_combined_worker_get_count(); i++)
+	{
+		struct starpu_sched_component * worker_component = starpu_sched_component_worker_get(i);
+		STARPU_ASSERT(worker_component);
+
+		struct starpu_sched_component * prio_component = starpu_sched_component_prio_create(&prio_data);
+		prio_component->add_child(prio_component, worker_component);
+		worker_component->add_parent(worker_component, prio_component);
+
+		struct starpu_sched_component * impl_component = starpu_sched_component_best_implementation_create(NULL);
+		impl_component->add_child(impl_component, prio_component);
+		prio_component->add_parent(prio_component, impl_component);
+
+		perfmodel_component->add_child(perfmodel_component, impl_component);
+		impl_component->add_parent(impl_component, perfmodel_component);
+		no_perfmodel_component->add_child(no_perfmodel_component, impl_component);
+		impl_component->add_parent(impl_component, no_perfmodel_component);
+		calibrator_component->add_child(calibrator_component, impl_component);
+		impl_component->add_parent(impl_component, calibrator_component);
+	}
+
+	starpu_sched_tree_update_workers(t);
+	starpu_sched_ctx_set_policy_data(sched_ctx_id, (void*)t);
+}
+
+static void deinitialize_heft_center_policy(unsigned sched_ctx_id)
+{
+	struct starpu_sched_tree *t = (struct starpu_sched_tree*)starpu_sched_ctx_get_policy_data(sched_ctx_id);
+	starpu_sched_tree_destroy(t);
+	starpu_sched_ctx_delete_worker_collection(sched_ctx_id);
+}
+
+struct starpu_sched_policy _starpu_sched_modular_heft_policy =
+{
+	.init_sched = initialize_heft_center_policy,
+	.deinit_sched = deinitialize_heft_center_policy,
+	.add_workers = starpu_sched_tree_add_workers,
+	.remove_workers = starpu_sched_tree_remove_workers,
+	.push_task = starpu_sched_tree_push_task,
+	.pop_task = starpu_sched_tree_pop_task,
+	.pre_exec_hook = starpu_sched_component_worker_pre_exec_hook,
+	.post_exec_hook = starpu_sched_component_worker_post_exec_hook,
+	.pop_every_task = NULL,
+	.policy_name = "modular-heft",
+	.policy_description = "heft modular policy"
+};

src/sched_policies/modular_heft2.c

@@ -0,0 +1,128 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2013  Université de Bordeaux 1
+ * Copyright (C) 2013  INRIA
+ * Copyright (C) 2013  Simon Archipoff
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <starpu_sched_component.h>
+#include <starpu_scheduler.h>
+#include <float.h>
+
+/* The two thresolds concerns the prio components, which contains queues
+ * who can handle the priority of StarPU tasks. You can tune your
+ * scheduling by benching those values and choose which one is the
+ * best for your current application. 
+ * The current value of the ntasks_threshold is the best we found
+ * so far across several types of applications (cholesky, LU, stencil).
+ */
+#define _STARPU_SCHED_NTASKS_THRESHOLD_DEFAULT 30
+#define _STARPU_SCHED_EXP_LEN_THRESHOLD_DEFAULT 1000000000.0
+
+static void initialize_heft2_center_policy(unsigned sched_ctx_id)
+{
+	starpu_sched_ctx_create_worker_collection(sched_ctx_id, STARPU_WORKER_LIST);
+
+	unsigned ntasks_threshold = _STARPU_SCHED_NTASKS_THRESHOLD_DEFAULT;
+	double exp_len_threshold = _STARPU_SCHED_EXP_LEN_THRESHOLD_DEFAULT;
+
+	const char *strval_ntasks_threshold = getenv("STARPU_NTASKS_THRESHOLD");
+	if (strval_ntasks_threshold)
+		ntasks_threshold = atof(strval_ntasks_threshold);
+
+	const char *strval_exp_len_threshold = getenv("STARPU_EXP_LEN_THRESHOLD");
+	if (strval_exp_len_threshold)
+		exp_len_threshold = atof(strval_exp_len_threshold);
+
+
+	struct starpu_sched_tree * t = starpu_sched_tree_create(sched_ctx_id);
+
+	struct starpu_sched_component * perfmodel_component = starpu_sched_component_heft_create(NULL);
+	struct starpu_sched_component * no_perfmodel_component = starpu_sched_component_eager_create(NULL);
+	struct starpu_sched_component * calibrator_component = starpu_sched_component_eager_create(NULL);
+	
+	struct starpu_perfmodel_select_data perfmodel_select_data =
+		{
+			.calibrator_component = calibrator_component,
+			.no_perfmodel_component = no_perfmodel_component,
+			.perfmodel_component = perfmodel_component,
+		};
+
+	struct starpu_sched_component * window_component = starpu_sched_component_prio_create(NULL);
+	t->root = window_component;
+
+	struct starpu_sched_component * perfmodel_select_component = starpu_sched_component_perfmodel_select_create(&perfmodel_select_data);
+	window_component->add_child(window_component, perfmodel_select_component);
+	perfmodel_select_component->add_parent(perfmodel_select_component, window_component);
+
+	perfmodel_select_component->add_child(perfmodel_select_component, calibrator_component);
+	calibrator_component->add_parent(calibrator_component, perfmodel_select_component);
+	perfmodel_select_component->add_child(perfmodel_select_component, perfmodel_component);
+	perfmodel_component->add_parent(perfmodel_component, perfmodel_select_component);
+	perfmodel_select_component->add_child(perfmodel_select_component, no_perfmodel_component);
+	no_perfmodel_component->add_parent(no_perfmodel_component, perfmodel_select_component);
+
+	struct starpu_prio_data prio_data =
+		{
+			.ntasks_threshold = ntasks_threshold,
+			.exp_len_threshold = exp_len_threshold,
+		};
+
+	unsigned i;
+	for(i = 0; i < starpu_worker_get_count() + starpu_combined_worker_get_count(); i++)
+	{
+		struct starpu_sched_component * worker_component = starpu_sched_component_worker_get(i);
+		STARPU_ASSERT(worker_component);
+
+		struct starpu_sched_component * prio_component = starpu_sched_component_prio_create(&prio_data);
+		prio_component->add_child(prio_component, worker_component);
+		worker_component->add_parent(worker_component, prio_component);
+
+		struct starpu_sched_component * impl_component = starpu_sched_component_best_implementation_create(NULL);
+		impl_component->add_child(impl_component, prio_component);
+		prio_component->add_parent(prio_component, impl_component);
+
+		perfmodel_component->add_child(perfmodel_component, impl_component);
+		impl_component->add_parent(impl_component, perfmodel_component);
+		no_perfmodel_component->add_child(no_perfmodel_component, impl_component);
+		impl_component->add_parent(impl_component, no_perfmodel_component);
+		calibrator_component->add_child(calibrator_component, impl_component);
+		impl_component->add_parent(impl_component, calibrator_component);
+	}
+
+	starpu_sched_tree_update_workers(t);
+	starpu_sched_ctx_set_policy_data(sched_ctx_id, (void*)t);
+}
+
+static void deinitialize_heft2_center_policy(unsigned sched_ctx_id)
+{
+	struct starpu_sched_tree *t = (struct starpu_sched_tree*)starpu_sched_ctx_get_policy_data(sched_ctx_id);
+	starpu_sched_tree_destroy(t);
+	starpu_sched_ctx_delete_worker_collection(sched_ctx_id);
+}
+
+struct starpu_sched_policy _starpu_sched_modular_heft2_policy =
+{
+	.init_sched = initialize_heft2_center_policy,
+	.deinit_sched = deinitialize_heft2_center_policy,
+	.add_workers = starpu_sched_tree_add_workers,
+	.remove_workers = starpu_sched_tree_remove_workers,
+	.push_task = starpu_sched_tree_push_task,
+	.pop_task = starpu_sched_tree_pop_task,
+	.pre_exec_hook = starpu_sched_component_worker_pre_exec_hook,
+	.post_exec_hook = starpu_sched_component_worker_post_exec_hook,
+	.pop_every_task = NULL,
+	.policy_name = "modular-heft2",
+	.policy_description = "heft modular2 policy"
+};

src/sched_policies/modular_prio.c

@@ -0,0 +1,62 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2013  INRIA
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <starpu_sched_component.h>
+#include <starpu_scheduler.h>
+
+static void initialize_prio_center_policy(unsigned sched_ctx_id)
+{
+	starpu_sched_ctx_create_worker_collection(sched_ctx_id, STARPU_WORKER_LIST);
+	struct starpu_sched_tree *t = starpu_sched_tree_create(sched_ctx_id);
+ 	t->root = starpu_sched_component_prio_create(NULL);
+	struct starpu_sched_component * eager_component = starpu_sched_component_eager_create(NULL);
+	t->root->add_child(t->root, eager_component);
+	eager_component->add_parent(eager_component, t->root);
+
+	unsigned i;
+	for(i = 0; i < starpu_worker_get_count() + starpu_combined_worker_get_count(); i++)
+	{
+		struct starpu_sched_component * worker_component = starpu_sched_component_worker_get(i);
+		STARPU_ASSERT(worker_component);
+
+		eager_component->add_child(eager_component, worker_component);
+		worker_component->add_parent(worker_component, eager_component);
+	}
+	starpu_sched_tree_update_workers(t);
+	starpu_sched_ctx_set_policy_data(sched_ctx_id, (void*)t);
+}
+
+static void deinitialize_prio_center_policy(unsigned sched_ctx_id)
+{
+	struct starpu_sched_tree *tree = (struct starpu_sched_tree*)starpu_sched_ctx_get_policy_data(sched_ctx_id);
+	starpu_sched_tree_destroy(tree);
+	starpu_sched_ctx_delete_worker_collection(sched_ctx_id);
+}
+
+struct starpu_sched_policy _starpu_sched_modular_prio_policy =
+{
+	.init_sched = initialize_prio_center_policy,
+	.deinit_sched = deinitialize_prio_center_policy,
+	.add_workers = starpu_sched_tree_add_workers,
+	.remove_workers = starpu_sched_tree_remove_workers,
+	.push_task = starpu_sched_tree_push_task,
+	.pop_task = starpu_sched_tree_pop_task,
+	.pre_exec_hook = starpu_sched_component_worker_pre_exec_hook,
+	.post_exec_hook = starpu_sched_component_worker_post_exec_hook,
+	.pop_every_task = NULL,
+	.policy_name = "modular-prio",
+	.policy_description = "prio modular policy"
+};

src/sched_policies/modular_prio_prefetching.c

@@ -0,0 +1,86 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2013  INRIA
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <starpu_sched_component.h>
+#include <starpu_scheduler.h>
+
+#define _STARPU_SCHED_NTASKS_THRESHOLD_DEFAULT 4
+#define _STARPU_SCHED_EXP_LEN_THRESHOLD_DEFAULT 1000000000.0
+
+static void initialize_prio_prefetching_center_policy(unsigned sched_ctx_id)
+{
+	unsigned ntasks_threshold = _STARPU_SCHED_NTASKS_THRESHOLD_DEFAULT;
+	double exp_len_threshold = _STARPU_SCHED_EXP_LEN_THRESHOLD_DEFAULT;
+
+	const char *strval_ntasks_threshold = getenv("STARPU_NTASKS_THRESHOLD");
+	if (strval_ntasks_threshold)
+		ntasks_threshold = atof(strval_ntasks_threshold);
+
+	const char *strval_exp_len_threshold = getenv("STARPU_EXP_LEN_THRESHOLD");
+	if (strval_exp_len_threshold)
+		exp_len_threshold = atof(strval_exp_len_threshold);
+
+	starpu_sched_ctx_create_worker_collection(sched_ctx_id, STARPU_WORKER_LIST);
+	struct starpu_sched_tree *t = starpu_sched_tree_create(sched_ctx_id);
+ 	t->root = starpu_sched_component_prio_create(NULL);
+	struct starpu_sched_component * eager_component = starpu_sched_component_eager_create(NULL);
+	t->root->add_child(t->root, eager_component);
+	eager_component->add_parent(eager_component, t->root);
+
+	struct starpu_prio_data prio_data =
+		{
+			.ntasks_threshold = ntasks_threshold,
+			.exp_len_threshold = exp_len_threshold,
+		};
+
+	unsigned i;
+	for(i = 0; i < starpu_worker_get_count() + starpu_combined_worker_get_count(); i++)
+	{
+		struct starpu_sched_component * worker_component = starpu_sched_component_worker_get(i);
+		STARPU_ASSERT(worker_component);
+
+		struct starpu_sched_component * prio_component = starpu_sched_component_prio_create(&prio_data);
+		prio_component->add_child(prio_component, worker_component);
+		worker_component->add_parent(worker_component, prio_component);
+
+		eager_component->add_child(eager_component, prio_component);
+		prio_component->add_parent(prio_component, eager_component);
+	}
+	starpu_sched_tree_update_workers(t);
+	starpu_sched_ctx_set_policy_data(sched_ctx_id, (void*)t);
+}
+
+static void deinitialize_prio_prefetching_center_policy(unsigned sched_ctx_id)
+{
+	struct starpu_sched_tree *tree = (struct starpu_sched_tree*)starpu_sched_ctx_get_policy_data(sched_ctx_id);
+	starpu_sched_tree_destroy(tree);
+	starpu_sched_ctx_delete_worker_collection(sched_ctx_id);
+}
+
+struct starpu_sched_policy _starpu_sched_modular_prio_prefetching_policy =
+{
+	.init_sched = initialize_prio_prefetching_center_policy,
+	.deinit_sched = deinitialize_prio_prefetching_center_policy,
+	.add_workers = starpu_sched_tree_add_workers,
+	.remove_workers = starpu_sched_tree_remove_workers,
+	.push_task = starpu_sched_tree_push_task,
+	.pop_task = starpu_sched_tree_pop_task,
+	.pre_exec_hook = starpu_sched_component_worker_pre_exec_hook,
+	.post_exec_hook = starpu_sched_component_worker_post_exec_hook,
+	.pop_every_task = NULL,
+	.policy_name = "modular-prio-prefetching",
+	.policy_description = "prio prefetching modular policy"
+};

src/sched_policies/modular_random.c

@@ -0,0 +1,111 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2013  INRIA
+ * Copyright (C) 2013  Simon Archipoff
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <starpu_sched_component.h>
+#include <starpu_scheduler.h>
+
+/* Random scheduler with a fifo queue for its scheduling window */
+
+static void initialize_random_fifo_center_policy(unsigned sched_ctx_id)
+{
+	starpu_sched_ctx_create_worker_collection(sched_ctx_id, STARPU_WORKER_LIST);
+	struct starpu_sched_tree *t = starpu_sched_tree_create(sched_ctx_id);
+ 	t->root = starpu_sched_component_fifo_create(NULL);
+	struct starpu_sched_component * random_component = starpu_sched_component_random_create(NULL);
+	t->root->add_child(t->root, random_component);
+	random_component->add_parent(random_component, t->root);
+
+	unsigned i;
+	for(i = 0; i < starpu_worker_get_count() + starpu_combined_worker_get_count(); i++)
+	{
+		struct starpu_sched_component * worker_component = starpu_sched_component_worker_get(i);
+		STARPU_ASSERT(worker_component);
+
+		random_component->add_child(random_component, worker_component);
+		worker_component->add_parent(worker_component, random_component);
+	}
+	starpu_sched_tree_update_workers(t);
+	starpu_sched_ctx_set_policy_data(sched_ctx_id, (void*)t);
+}
+
+static void deinitialize_random_fifo_center_policy(unsigned sched_ctx_id)
+{
+	struct starpu_sched_tree *tree = (struct starpu_sched_tree*)starpu_sched_ctx_get_policy_data(sched_ctx_id);
+	starpu_sched_tree_destroy(tree);
+	starpu_sched_ctx_delete_worker_collection(sched_ctx_id);
+}
+
+struct starpu_sched_policy _starpu_sched_modular_random_policy =
+{
+	.init_sched = initialize_random_fifo_center_policy,
+	.deinit_sched = deinitialize_random_fifo_center_policy,
+	.add_workers = starpu_sched_tree_add_workers,
+	.remove_workers = starpu_sched_tree_remove_workers,
+	.push_task = starpu_sched_tree_push_task,
+	.pop_task = starpu_sched_tree_pop_task,
+	.pre_exec_hook = NULL,
+	.post_exec_hook = NULL,
+	.pop_every_task = NULL,
+	.policy_name = "modular-random",
+	.policy_description = "random modular policy"
+};
+
+/* Random scheduler with a priority queue for its scheduling window */
+
+static void initialize_random_prio_center_policy(unsigned sched_ctx_id)
+{
+	starpu_sched_ctx_create_worker_collection(sched_ctx_id, STARPU_WORKER_LIST);
+	struct starpu_sched_tree *t = starpu_sched_tree_create(sched_ctx_id);
+ 	t->root = starpu_sched_component_prio_create(NULL);
+	struct starpu_sched_component * random_component = starpu_sched_component_random_create(NULL);
+	t->root->add_child(t->root, random_component);
+	random_component->add_parent(random_component, t->root);
+
+	unsigned i;
+	for(i = 0; i < starpu_worker_get_count() + starpu_combined_worker_get_count(); i++)
+	{
+		struct starpu_sched_component * worker_component = starpu_sched_component_worker_get(i);
+		STARPU_ASSERT(worker_component);
+
+		random_component->add_child(random_component, worker_component);
+		worker_component->add_parent(worker_component, random_component);
+	}
+	starpu_sched_tree_update_workers(t);
+	starpu_sched_ctx_set_policy_data(sched_ctx_id, (void*)t);
+}
+
+static void deinitialize_random_prio_center_policy(unsigned sched_ctx_id)
+{
+	struct starpu_sched_tree *tree = (struct starpu_sched_tree*)starpu_sched_ctx_get_policy_data(sched_ctx_id);
+	starpu_sched_tree_destroy(tree);
+	starpu_sched_ctx_delete_worker_collection(sched_ctx_id);
+}
+
+struct starpu_sched_policy _starpu_sched_modular_random_prio_policy =
+{
+	.init_sched = initialize_random_prio_center_policy,
+	.deinit_sched = deinitialize_random_prio_center_policy,
+	.add_workers = starpu_sched_tree_add_workers,
+	.remove_workers = starpu_sched_tree_remove_workers,
+	.push_task = starpu_sched_tree_push_task,
+	.pop_task = starpu_sched_tree_pop_task,
+	.pre_exec_hook = NULL,
+	.post_exec_hook = NULL,
+	.pop_every_task = NULL,
+	.policy_name = "modular-random-prio",
+	.policy_description = "random-prio modular policy"
+};

src/sched_policies/modular_random_prefetching.c

@@ -0,0 +1,156 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2013  INRIA
+ * Copyright (C) 2013  Simon Archipoff
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <starpu_sched_component.h>
+#include <starpu_scheduler.h>
+
+#define _STARPU_SCHED_NTASKS_THRESHOLD_DEFAULT 2
+#define _STARPU_SCHED_EXP_LEN_THRESHOLD_DEFAULT 1000000000.0
+
+/* Random scheduler with fifo queues for its scheduling window and its workers. */
+
+static void initialize_random_fifo_prefetching_center_policy(unsigned sched_ctx_id)
+{
+	unsigned ntasks_threshold = _STARPU_SCHED_NTASKS_THRESHOLD_DEFAULT;
+	double exp_len_threshold = _STARPU_SCHED_EXP_LEN_THRESHOLD_DEFAULT;
+
+	const char *strval_ntasks_threshold = getenv("STARPU_NTASKS_THRESHOLD");
+	if (strval_ntasks_threshold)
+		ntasks_threshold = atof(strval_ntasks_threshold);
+
+	const char *strval_exp_len_threshold = getenv("STARPU_EXP_LEN_THRESHOLD");
+	if (strval_exp_len_threshold)
+		exp_len_threshold = atof(strval_exp_len_threshold);
+
+	starpu_sched_ctx_create_worker_collection(sched_ctx_id, STARPU_WORKER_LIST);
+	struct starpu_sched_tree *t = starpu_sched_tree_create(sched_ctx_id);
+ 	t->root = starpu_sched_component_fifo_create(NULL);
+	struct starpu_sched_component * random_component = starpu_sched_component_random_create(NULL);
+	t->root->add_child(t->root, random_component);
+	random_component->add_parent(random_component, t->root);
+
+	struct starpu_fifo_data fifo_data =
+		{
+			.ntasks_threshold = ntasks_threshold,
+			.exp_len_threshold = exp_len_threshold,
+		};
+
+	unsigned i;
+	for(i = 0; i < starpu_worker_get_count() + starpu_combined_worker_get_count(); i++)
+	{
+		struct starpu_sched_component * worker_component = starpu_sched_component_worker_get(i);
+		STARPU_ASSERT(worker_component);
+
+		struct starpu_sched_component * fifo_component = starpu_sched_component_fifo_create(&fifo_data);
+		fifo_component->add_child(fifo_component, worker_component);
+		worker_component->add_parent(worker_component, fifo_component);
+
+		random_component->add_child(random_component, fifo_component);
+		fifo_component->add_parent(fifo_component, random_component);
+	}
+	starpu_sched_tree_update_workers(t);
+	starpu_sched_ctx_set_policy_data(sched_ctx_id, (void*)t);
+}
+
+static void deinitialize_random_fifo_prefetching_center_policy(unsigned sched_ctx_id)
+{
+	struct starpu_sched_tree *tree = (struct starpu_sched_tree*)starpu_sched_ctx_get_policy_data(sched_ctx_id);
+	starpu_sched_tree_destroy(tree);
+	starpu_sched_ctx_delete_worker_collection(sched_ctx_id);
+}
+
+struct starpu_sched_policy _starpu_sched_modular_random_prefetching_policy =
+{
+	.init_sched = initialize_random_fifo_prefetching_center_policy,
+	.deinit_sched = deinitialize_random_fifo_prefetching_center_policy,
+	.add_workers = starpu_sched_tree_add_workers,
+	.remove_workers = starpu_sched_tree_remove_workers,
+	.push_task = starpu_sched_tree_push_task,
+	.pop_task = starpu_sched_tree_pop_task,
+	.pre_exec_hook = NULL,
+	.post_exec_hook = NULL,
+	.pop_every_task = NULL,
+	.policy_name = "modular-random-prefetching",
+	.policy_description = "random prefetching modular policy"
+};
+
+/* Random scheduler with priority queues for its scheduling window and its workers. */
+
+static void initialize_random_prio_prefetching_center_policy(unsigned sched_ctx_id)
+{
+	unsigned ntasks_threshold = _STARPU_SCHED_NTASKS_THRESHOLD_DEFAULT;
+	double exp_len_threshold = _STARPU_SCHED_EXP_LEN_THRESHOLD_DEFAULT;
+
+	const char *strval_ntasks_threshold = getenv("STARPU_NTASKS_THRESHOLD");
+	if (strval_ntasks_threshold)
+		ntasks_threshold = atof(strval_ntasks_threshold);
+
+	const char *strval_exp_len_threshold = getenv("STARPU_EXP_LEN_THRESHOLD");
+	if (strval_exp_len_threshold)
+		exp_len_threshold = atof(strval_exp_len_threshold);
+
+	starpu_sched_ctx_create_worker_collection(sched_ctx_id, STARPU_WORKER_LIST);
+	struct starpu_sched_tree *t = starpu_sched_tree_create(sched_ctx_id);
+ 	t->root = starpu_sched_component_prio_create(NULL);
+	struct starpu_sched_component * random_component = starpu_sched_component_random_create(NULL);
+	t->root->add_child(t->root, random_component);
+	random_component->add_parent(random_component, t->root);
+
+	struct starpu_prio_data prio_data =
+		{
+			.ntasks_threshold = ntasks_threshold,
+			.exp_len_threshold = exp_len_threshold,
+		};
+
+	unsigned i;
+	for(i = 0; i < starpu_worker_get_count() + starpu_combined_worker_get_count(); i++)
+	{
+		struct starpu_sched_component * worker_component = starpu_sched_component_worker_get(i);
+		STARPU_ASSERT(worker_component);
+
+		struct starpu_sched_component * prio_component = starpu_sched_component_prio_create(&prio_data);
+		prio_component->add_child(prio_component, worker_component);
+		worker_component->add_parent(worker_component, prio_component);
+
+		random_component->add_child(random_component, prio_component);
+		prio_component->add_parent(prio_component, random_component);
+	}
+	starpu_sched_tree_update_workers(t);
+	starpu_sched_ctx_set_policy_data(sched_ctx_id, (void*)t);
+}
+
+static void deinitialize_random_prio_prefetching_center_policy(unsigned sched_ctx_id)
+{
+	struct starpu_sched_tree *tree = (struct starpu_sched_tree*)starpu_sched_ctx_get_policy_data(sched_ctx_id);
+	starpu_sched_tree_destroy(tree);
+	starpu_sched_ctx_delete_worker_collection(sched_ctx_id);
+}
+
+struct starpu_sched_policy _starpu_sched_modular_random_prio_prefetching_policy =
+{
+	.init_sched = initialize_random_prio_prefetching_center_policy,
+	.deinit_sched = deinitialize_random_prio_prefetching_center_policy,
+	.add_workers = starpu_sched_tree_add_workers,
+	.remove_workers = starpu_sched_tree_remove_workers,
+	.push_task = starpu_sched_tree_push_task,
+	.pop_task = starpu_sched_tree_pop_task,
+	.pre_exec_hook = NULL,
+	.post_exec_hook = NULL,
+	.pop_every_task = NULL,
+	.policy_name = "modular-random-prio-prefetching",
+	.policy_description = "random-prio prefetching modular policy"
+};

src/sched_policies/modular_ws.c

@@ -0,0 +1,61 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2013  INRIA
+ * Copyright (C) 2013  Simon Archipoff
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <starpu_sched_component.h>
+#include <starpu_scheduler.h>
+
+
+static void initialize_ws_center_policy(unsigned sched_ctx_id)
+{
+	starpu_sched_ctx_create_worker_collection(sched_ctx_id, STARPU_WORKER_LIST);
+	struct starpu_sched_tree *t = starpu_sched_tree_create(sched_ctx_id);
+ 	t->root = starpu_sched_component_work_stealing_create(NULL);
+	unsigned i;
+	for(i = 0; i < starpu_worker_get_count() + starpu_combined_worker_get_count(); i++)
+	{
+		struct starpu_sched_component * worker_component = starpu_sched_component_worker_get(i);
+		STARPU_ASSERT(worker_component);
+
+		t->root->add_child(t->root, worker_component);
+		worker_component->add_parent(worker_component, t->root);
+	}
+	starpu_sched_tree_update_workers(t);
+	starpu_sched_ctx_set_policy_data(sched_ctx_id, (void*)t);
+}
+
+static void deinitialize_ws_center_policy(unsigned sched_ctx_id)
+{
+	struct starpu_sched_tree *t = (struct starpu_sched_tree*)starpu_sched_ctx_get_policy_data(sched_ctx_id);
+	starpu_sched_tree_destroy(t);
+	starpu_sched_ctx_delete_worker_collection(sched_ctx_id);
+}
+
+
+struct starpu_sched_policy _starpu_sched_modular_ws_policy =
+{
+	.init_sched = initialize_ws_center_policy,
+	.deinit_sched = deinitialize_ws_center_policy,
+	.add_workers = starpu_sched_tree_add_workers,
+	.remove_workers = starpu_sched_tree_remove_workers,
+	.push_task = starpu_sched_tree_work_stealing_push_task,
+	.pop_task = starpu_sched_tree_pop_task,
+	.pre_exec_hook = NULL,
+	.post_exec_hook = NULL,
+	.pop_every_task = NULL,
+	.policy_name = "modular-ws",
+	.policy_description = "work stealing modular policy"
+};

src/sched_policies/prio_deque.c

@@ -0,0 +1,156 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2013  Simon Archipoff
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <core/workers.h>
+
+#include "prio_deque.h"
+
+
+void _starpu_prio_deque_init(struct _starpu_prio_deque * pdeque)
+{
+	STARPU_ASSERT(pdeque);
+	memset(pdeque,0,sizeof(*pdeque));
+}
+void _starpu_prio_deque_destroy(struct _starpu_prio_deque * pdeque)
+{
+	STARPU_ASSERT(pdeque);
+	int i;
+	for(i = 0; i < pdeque->size_array; i++)
+	{
+		STARPU_ASSERT(starpu_task_list_empty(&pdeque->array[i].list));
+	}
+	free(pdeque->array);
+}
+
+int _starpu_prio_deque_is_empty(struct _starpu_prio_deque * pdeque)
+{
+	STARPU_ASSERT(pdeque);
+	return pdeque->ntasks == 0;
+}
+
+
+/* return the struct starpu_prio_list * of prio,
+ * create it and return it if none exist yet
+ */
+static struct starpu_task_list * get_prio(struct _starpu_prio_deque * pdeque, int prio)
+{
+	STARPU_ASSERT(pdeque);
+	int i;
+	for(i = 0; i < pdeque->size_array; i++)
+	{
+		if(pdeque->array[i].prio == prio)
+		{
+			return &pdeque->array[i].list;
+		}
+		else
+			if(pdeque->array[i].prio < prio)
+				break;
+	}
+	pdeque->size_array++;
+	pdeque->array = realloc(pdeque->array, sizeof(struct _starpu_prio_list) * (pdeque->size_array));
+	memmove(pdeque->array + i + 1,
+		pdeque->array + i,
+		(pdeque->size_array - i - 1) * sizeof(struct _starpu_prio_list));
+	pdeque->array[i].prio = prio;
+	starpu_task_list_init(&pdeque->array[i].list);
+	return &pdeque->array[i].list;
+}
+
+
+
+int _starpu_prio_deque_push_task(struct _starpu_prio_deque * pdeque, struct starpu_task * task)
+{
+	STARPU_ASSERT(pdeque && task);
+	struct starpu_task_list * list = get_prio(pdeque, task->priority);
+	starpu_task_list_push_back(list, task);
+	pdeque->ntasks++;
+	return 0;
+}
+
+int _starpu_prio_deque_push_back_task(struct _starpu_prio_deque * pdeque, struct starpu_task * task)
+{
+	STARPU_ASSERT(pdeque && task);
+	struct starpu_task_list * list = get_prio(pdeque, task->priority);
+	starpu_task_list_push_front(list, task);
+	pdeque->ntasks++;
+	return 0;
+}
+
+
+
+
+/* a little dirty code factorization */
+
+static inline int pred_true(struct starpu_task * t STARPU_ATTRIBUTE_UNUSED, void * v STARPU_ATTRIBUTE_UNUSED)
+{
+	return 1;
+}
+
+static inline int pred_can_execute(struct starpu_task * t, void * pworkerid)
+{
+	int i;
+	for(i = 0; i < STARPU_MAXIMPLEMENTATIONS; i++)
+		if(starpu_worker_can_execute_task(*(int*)pworkerid, t,i))
+			return 1;
+	return 0;
+}
+
+#define REMOVE_TASK(pdeque, first_task_field, next_task_field, predicate, parg)	\
+	{								\
+		int i;							\
+		struct starpu_task * t = NULL;				\
+		for(i = 0; i < pdeque->size_array; i++)			\
+		{							\
+			t = pdeque->array[i].list.first_task_field;	\
+			while(t && !predicate(t,parg))			\
+				t = t->next_task_field;			\
+			if(t)						\
+			{						\
+				starpu_task_list_erase(&pdeque->array[i].list, t); \
+				pdeque->ntasks--;			\
+				return t;				\
+			}						\
+		}							\
+		return NULL;						\
+	}
+
+/* deque a task of the higher priority available */
+
+/* From the front of the list for the highest priority */
+struct starpu_task * _starpu_prio_deque_pop_task(struct _starpu_prio_deque * pdeque)
+{
+	REMOVE_TASK(pdeque, head, prev, pred_true, STARPU_POISON_PTR);
+}
+struct starpu_task * _starpu_prio_deque_pop_task_for_worker(struct _starpu_prio_deque * pdeque, int workerid)
+{
+	STARPU_ASSERT(pdeque);
+	STARPU_ASSERT(0 <= workerid && (unsigned) workerid < starpu_worker_get_count());
+	REMOVE_TASK(pdeque, head, prev, pred_can_execute, &workerid);
+}
+
+/* From the back of the list for the highest priority */
+struct starpu_task * _starpu_prio_deque_deque_task(struct _starpu_prio_deque * pdeque)
+{
+	STARPU_ASSERT(pdeque);
+	REMOVE_TASK(pdeque, tail, next, pred_true, STARPU_POISON_PTR);
+}
+
+struct starpu_task * _starpu_prio_deque_deque_task_for_worker(struct _starpu_prio_deque * pdeque, int workerid)
+{
+	STARPU_ASSERT(pdeque);
+	STARPU_ASSERT(0 <= workerid && (unsigned) workerid < starpu_worker_get_count());
+	REMOVE_TASK(pdeque, tail, next, pred_can_execute, &workerid);
+}

src/sched_policies/prio_deque.h

@@ -0,0 +1,64 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2013  Simon Archipoff
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+#ifndef __PRIO_DEQUE_H__
+#define __PRIO_DEQUE_H__
+#include <starpu.h>
+#include <starpu_task_list.h>
+
+
+struct _starpu_prio_list
+{
+	int prio;
+	struct starpu_task_list list;
+};
+
+struct _starpu_prio_deque
+{
+	struct _starpu_prio_list * array;
+	int size_array;
+	unsigned ntasks;
+	unsigned nprocessed;
+	double exp_start, exp_end, exp_len;
+};
+
+void _starpu_prio_deque_init(struct _starpu_prio_deque *);
+void _starpu_prio_deque_destroy(struct _starpu_prio_deque *);
+
+/* return 0 iff the struct _starpu_prio_deque is not empty */
+int _starpu_prio_deque_is_empty(struct _starpu_prio_deque *);
+
+/* push a task in O(nb priorities) */
+int _starpu_prio_deque_push_task(struct _starpu_prio_deque *, struct starpu_task *);
+int _starpu_prio_deque_push_back_task(struct _starpu_prio_deque *, struct starpu_task *);
+
+
+/* all _starpu_prio_deque_pop/deque_task function return a task or a NULL pointer if none are available
+ * in O(nb priorities)
+ */
+
+struct starpu_task * _starpu_prio_deque_pop_task(struct _starpu_prio_deque *);
+
+/* return a task that can be executed by workerid
+ */
+struct starpu_task * _starpu_prio_deque_pop_task_for_worker(struct _starpu_prio_deque *, int workerid);
+
+/* deque a task of the higher priority available */
+struct starpu_task * _starpu_prio_deque_deque_task(struct _starpu_prio_deque *);
+/* return a task that can be executed by workerid
+ */
+struct starpu_task * _starpu_prio_deque_deque_task_for_worker(struct _starpu_prio_deque *, int workerid);
+
+#endif /* __PRIO_DEQUE_H__ */

src/sched_policies/sched_component.h

@@ -0,0 +1,40 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2013  Simon Archipoff
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#ifndef __SCHED_COMPONENT_H__
+#define __SCHED_COMPONENT_H__
+
+#include <starpu_sched_component.h>
+
+
+/* lock and unlock drivers for modifying schedulers */
+void _starpu_sched_component_lock_all_workers(void);
+void _starpu_sched_component_unlock_all_workers(void);
+void _starpu_sched_component_lock_worker(int workerid);
+void _starpu_sched_component_unlock_worker(int workerid);
+
+void _starpu_sched_component_workers_destroy(void);
+
+struct _starpu_worker * _starpu_sched_component_worker_get_worker(struct starpu_sched_component *);
+struct _starpu_combined_worker * _starpu_sched_component_combined_worker_get_combined_worker(struct starpu_sched_component * worker_component);
+
+struct starpu_bitmap * _starpu_get_worker_mask(unsigned sched_ctx_id);
+
+
+void _starpu_sched_component_lock_scheduling(void);
+void _starpu_sched_component_unlock_scheduling(void);
+
+#endif

src/sched_policies/scheduler_maker.c

@@ -0,0 +1,275 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2013  Simon Archipoff
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <starpu_sched_component.h>
+#include <common/list.h>
+#include <core/workers.h>
+
+#include "sched_component.h"
+
+
+
+/* The scheduler is built by a recursive function called on the hwloc topology with a starpu_sched_specs structure,
+ * each call return a set of starpu_sched_component, not a single one, because you may have a topology like that :
+ * MACHINE -- MEMORY NODE -- SOCKET
+ *                        \- SOCKET
+ * and you have defined a component for MACHINE, and a component for SOCKET, but not for MEMORY NODE then the recursive call
+ * on MEMORY NODE will return 2 starpu_sched_component for those 2 sockets
+ *
+ *
+ */
+
+struct sched_component_list
+{
+	struct starpu_sched_component ** arr;
+	unsigned size;
+};
+
+static void init_list(struct sched_component_list * list)
+{
+	memset(list,0,sizeof(*list));
+}
+static void destroy_list(struct sched_component_list * list)
+{
+	free(list->arr);
+}
+static void add_component(struct sched_component_list *list, struct starpu_sched_component * component)
+{
+	list->arr = realloc(list->arr,sizeof(*list->arr) * (list->size + 1));
+	list->arr[list->size] = component;
+	list->size++;
+}
+/* this is the function that actualy built the scheduler, but without workers */
+static struct sched_component_list helper_make_scheduler(hwloc_obj_t obj, struct starpu_sched_specs specs, unsigned sched_ctx_id)
+{
+	STARPU_ASSERT(obj);
+
+	struct starpu_sched_component * component = NULL;
+
+	/*set components for this obj */
+#define CASE(ENUM,spec_member)						\
+		case ENUM:						\
+			if(specs.spec_member)				\
+				component = starpu_sched_component_composed_component_create(specs.spec_member); \
+			break
+	switch(obj->type)
+	{
+		CASE(HWLOC_OBJ_MACHINE,hwloc_machine_composed_sched_component);
+		CASE(HWLOC_OBJ_NODE,hwloc_component_composed_sched_component);
+		CASE(HWLOC_OBJ_SOCKET,hwloc_socket_composed_sched_component);
+		CASE(HWLOC_OBJ_CACHE,hwloc_cache_composed_sched_component);
+	default:
+		break;
+	}
+
+	struct sched_component_list l;
+	init_list(&l);
+	unsigned i;
+	/* collect childs component's */
+	for(i = 0; i < obj->arity; i++)
+	{
+		struct sched_component_list lc = helper_make_scheduler(obj->children[i],specs, sched_ctx_id);
+		unsigned j;
+		for(j = 0; j < lc.size; j++)
+			add_component(&l, lc.arr[j]);
+		destroy_list(&lc);
+	}
+	if(!component)
+		return l;
+	for(i = 0; i < l.size; i++)
+	{
+		component->add_child(component, l.arr[i]);
+		starpu_sched_component_add_parent(l.arr[i],component);
+	}
+	destroy_list(&l);
+	init_list(&l);
+	component->obj = obj;
+	add_component(&l, component);
+	return l;
+}
+/* return the firt component in prefix order such as component->obj == obj, or NULL */
+struct starpu_sched_component * _find_sched_component_with_obj(struct starpu_sched_component * component, hwloc_obj_t obj)
+{
+	if(component == NULL)
+		return NULL;
+	if(component->obj == obj)
+		return component;
+	int i;
+	for(i = 0; i < component->nchildren; i++)
+	{
+		struct starpu_sched_component * tmp = _find_sched_component_with_obj(component->children[i], obj);
+		if(tmp)
+			return tmp;
+	}
+	return NULL;
+}
+
+/* return true if all workers in the tree have the same perf_arch as w_ref,
+ * if there is no worker it return true
+ */
+static int is_same_kind_of_all(struct starpu_sched_component * root, struct _starpu_worker * w_ref)
+{
+	if(starpu_sched_component_is_worker(root))
+	{
+		struct _starpu_worker * w = root->data;
+		return w->perf_arch.type == w_ref->perf_arch.type;
+	}
+	
+	int i;
+	for(i = 0;i < root->nchildren; i++)
+		if(!is_same_kind_of_all(root->children[i], w_ref))
+			return 0;
+	return 1;
+}
+/* buggy function
+ * return the starpu_sched_component linked to the supposed memory component of worker_component
+ */
+static struct starpu_sched_component * find_mem_component(struct starpu_sched_component * root, struct starpu_sched_component * worker_component)
+{
+	struct starpu_sched_component * component = worker_component;
+	while(component->obj->type != HWLOC_OBJ_NODE
+	      && component->obj->type != HWLOC_OBJ_MACHINE)
+	{
+		hwloc_obj_t tmp = component->obj;
+		do
+		{
+			component = _find_sched_component_with_obj(root,tmp);
+			tmp = tmp->parent;
+		}
+		while(!component);
+		
+	}
+	return component;
+}
+
+static struct starpu_sched_component * where_should_we_plug_this(struct starpu_sched_component *root, struct starpu_sched_component * worker_component, struct starpu_sched_specs specs, unsigned sched_ctx_id)
+{
+	struct starpu_sched_component * mem = find_mem_component(root ,worker_component);
+	if(specs.mix_heterogeneous_workers || mem->parents[sched_ctx_id] == NULL)
+		return mem;
+	hwloc_obj_t obj = mem->obj;
+	struct starpu_sched_component * parent = mem->parents[sched_ctx_id];
+	int i;
+	for(i = 0; i < parent->nchildren; i++)
+	{
+		if(parent->children[i]->obj == obj
+		   && is_same_kind_of_all(parent->children[i], worker_component->data))
+			return parent->children[i];
+	}
+	if(obj->type == HWLOC_OBJ_NODE)
+	{	
+		struct starpu_sched_component * component = starpu_sched_component_composed_component_create(specs.hwloc_component_composed_sched_component);
+		component->obj = obj;
+		parent->add_child(parent, component);
+		starpu_sched_component_add_parent(component, parent);
+		return component;
+	}
+	return parent;
+}
+
+static void set_worker_leaf(struct starpu_sched_component * root, struct starpu_sched_component * worker_component, unsigned sched_ctx_id,
+			    struct starpu_sched_specs specs)
+{
+	struct _starpu_worker * worker = worker_component->data;
+	struct starpu_sched_component * component = where_should_we_plug_this(root,worker_component,specs, sched_ctx_id);
+	struct starpu_sched_component_composed_recipe * recipe = specs.worker_composed_sched_component ?
+		specs.worker_composed_sched_component(worker->arch):NULL;
+	STARPU_ASSERT(component);
+	if(recipe)
+	{
+		struct starpu_sched_component * tmp = starpu_sched_component_composed_component_create(recipe);
+#ifdef STARPU_DEVEL
+#warning FIXME component->obj is set to worker_component->obj even for accelerators workers
+#endif
+		tmp->obj = worker_component->obj;
+		starpu_sched_component_add_parent(tmp, component);
+		component->add_child(component, tmp);
+		component = tmp;
+		
+	}
+	starpu_destroy_composed_sched_component_recipe(recipe);
+	starpu_sched_component_add_parent(worker_component, component);
+	component->add_child(component, worker_component);
+}
+
+#ifdef STARPU_DEVEL
+static const char * name_hwloc_component(struct starpu_sched_component * component)
+{
+	return hwloc_obj_type_string(component->obj->type);
+}
+static const char * name_sched_component(struct starpu_sched_component * component)
+{
+	if(starpu_sched_component_is_fifo(component))
+		return "fifo component";
+	if(starpu_sched_component_is_heft(component))
+		return "heft component";
+	if(starpu_sched_component_is_random(component))
+		return "random component";
+	if(starpu_sched_component_is_worker(component))
+	{
+		struct _starpu_worker * w = _starpu_sched_component_worker_get_worker(component);
+#define SIZE 256
+		static char output[SIZE];
+		snprintf(output, SIZE,"component worker %d %s",w->workerid,w->name);
+		return output;
+	}
+	if(starpu_sched_component_is_work_stealing(component))
+		return "work stealing component";
+
+	return "unknown";
+}
+static void helper_display_scheduler(FILE* out, unsigned depth, struct starpu_sched_component * component)
+{
+	if(!component)
+		return;
+	fprintf(out,"%*s-> %s : %s\n", depth * 2 , "", name_sched_component(component), name_hwloc_component(component));
+	int i;
+	for(i = 0; i < component->nchildren; i++)
+		helper_display_scheduler(out, depth + 1, component->children[i]);
+}
+#endif //STARPU_DEVEL
+struct starpu_sched_tree * starpu_sched_component_make_scheduler(unsigned sched_ctx_id, struct starpu_sched_specs specs)
+{
+	struct starpu_sched_tree * tree = starpu_sched_tree_create(sched_ctx_id);
+	
+	struct _starpu_machine_config *config = _starpu_get_machine_config();
+	hwloc_topology_t topology = config->topology.hwtopology;
+
+	struct sched_component_list list = helper_make_scheduler(hwloc_get_root_obj(topology), specs, sched_ctx_id);
+	STARPU_ASSERT(list.size == 1);
+
+	tree->root = list.arr[0];
+	destroy_list(&list);
+	
+	unsigned i;
+	for(i = 0; i < starpu_worker_get_count(); i++)
+	{
+		struct _starpu_worker * worker = _starpu_get_worker_struct(i);
+		struct starpu_sched_component * worker_component = starpu_sched_component_worker_get(i);
+		STARPU_ASSERT(worker);
+		set_worker_leaf(tree->root,worker_component, sched_ctx_id, specs);
+	}
+
+
+	starpu_sched_tree_update_workers(tree);
+#ifdef STARPU_DEVEL
+	fprintf(stderr, "scheduler created :\n");
+	helper_display_scheduler(stderr, 0, tree->root);
+#endif
+
+	return tree;
+
+}

tools/valgrind/starpu.suppr

@@ -70,3 +70,22 @@
    fun:_starpu_load_bus_performance_files
    ...
 }
+
+{
+   TODO1: This is temporary. It perhaps does not pose problem because only the worker takes this mutex. Fixing this will require changing the scheduler interface, so that the schedulers themselves take the scheduling lock, not the caller. Filter it out for now, so we can see other races more easily.
+   Helgrind:LockOrder
+   fun:pthread_mutex_lock
+   fun:starpu_pthread_mutex_lock
+   fun:simple_worker_pull_task
+   ...
+}
+
+{
+   TODO2: This is temporary. It perhaps does not pose problem because only the worker takes this mutex. Fixing this will require changing the scheduler interface, so that the schedulers themselves take the scheduling lock, not the caller. Filter it out for now, so we can see other races more easily.
+   Helgrind:LockOrder
+   fun:pthread_mutex_lock
+   fun:starpu_pthread_mutex_lock
+   fun:_starpu_sched_component_lock_worker
+   fun:simple_worker_pull_task
+   ...
+}