finish api's doc

clean interface
bug fixes, typos, and comments everywhere

doc/doxygen/chapters/api/modularized_scheduler.doxy

@@ -7,91 +7,106 @@ This structure represent a scheduler module.
 \var starpu_sched_node::push_task
 	push a task in the scheduler module.
 \var starpu_sched_node::pop_task
-	pop a task from the scheduler module, the task returned by this function is executable by the caller if its a worker
-\var starpu_sched_node::available
-	notify workers downstairs that a task is waiting for a pop, this member dont seems to be necessary
+	pop a task from the scheduler module, the task returned by this function is executable by the caller
+	It should use starpu_sched_node::fathers[sched_ctx_id] to perform a recursive call.
+
 \var starpu_sched_node::estimated_load
-	is an heuristic to compute load of scheduler module
-\var starpu_sched_node::estimated_execute_preds
-	compute executions prediction for a task
+	is an heuristic to compute load of scheduler module. Basically the number of tasks divided by the sum
+	of relatives speedup of workers available in context.
+\var starpu_sched_node::estimated_end
+	return the time when a worker will enter in starvation. This function is relevant only if the task->predicted
+	member has been set.
 \var starpu_sched_node::nchilds
 	the number of modules downstairs
 \var starpu_sched_node::childs
 	modules downstairs
 \var starpu_sched_node::workers
-	this member contain the set of underlaying workers
+	this member contain the set of underlying workers
+\var starpu_sched_node::workers_in_ctx
+	this member contain the subset of starpu_sched_node::workers that is currently available in the context
+	The push method should take this member into account.
 \var starpu_sched_node::is_homogeneous
-	this is set to true iff all underlaying workers are the same
+	this is set to true iff all underlying workers are the same starpu_worker_archtype.
 \var starpu_sched_node::data
 	data used by the scheduler module
+\var starpu_sched_node::add_child
+	add a child to node
+\var starpu_sched_node::remove_child
+	remove a child from node
+
 \var starpu_sched_node::fathers
 	the array of scheduler module above indexed by scheduling context index
-\var starpu_sched_node::init_data
-	is called after all the scheduler is created and should init data member
-	you can store things in node->data while calling _sched_node_create(arg)
-	and use it with init_data
+
+\var starpu_sched_node::notify_change_workers
+	this function is called when starpu_sched_node::workers_in_ctx or starpu_sched_node::workers is changed
 \var starpu_sched_node::deinit_data
-	is called just before starpu_sched_node_destroy
+	called by starpu_sched_node_destroy. Should free data allocated during creation
 \var starpu_sched_node::obj
-	the hwloc object associed to scheduler module
-
-\struct starpu_task_execute_preds
-\ingroup API_Modularized_Scheduler
-this structure containt predictions for a task and is filled by starpu_sched_node::estimated_execute_preds
-\var starpu_task_execute_preds::state
-	indicate status of prediction, if several status are possible, in order of priority it will be : CALIBRATING, PERF_MODEL, NO_PERF_MODEL, CANNOT_EXECUTE
-\var starpu_task_execute_preds::archtype
-\var starpu_task_execute_preds::impl
-	those members are revelant is state is PERF_MODEL or CALIBRATING and is set to best or uncalibrated archtype and implementation, respectively, or suitable values if state is NO_PERF_MODEL
-\var starpu_task_execute_preds::expected_finish_time
-	expected finish time of task
-\var starpu_task_execute_preds::expected_length
-	expected compute time of task
-\var starpu_task_execute_preds::expected_transfer_length
-	expected time for transfering data to worker's memory node
-\var starpu_task_execute_preds::expected_power
-	expected power consumption for task
+	the hwloc object associated to scheduler module
 
 \struct starpu_sched_tree
 \ingroup API_Modularized_Scheduler
+The actual scheduler
 \var starpu_sched_tree::root
 	this is the entry module of the scheduler
 \var starpu_sched_tree::workers
 	this is the set of workers available in this context, this value is used to mask workers in modules
-\var lock
-	this lock protect the worker member
+\var starpu_sched_tree::lock
+	this lock protect concurrent access from the hypervisor and the application.
+\var starpu_sched_tree::sched_ctx_id
+	the context id of the scheduler
 
 \fn struct starpu_sched_node * starpu_sched_node_create(void)
 \ingroup API_Modularized_Scheduler
-        allocate and initalise node field with defaults values :
+	 allocate and initialize node field with defaults values :
 	.pop_task make recursive call on father
-	.available make a recursive call on childrens
-	.estimated_load compute relative speedup and tasks in subtree
-	.estimated_execute_preds return the average of recursive call on childs
+	.estimated_load compute relative speedup and tasks in sub tree
+	.estimated_end return the average of recursive call on childs
+	.add_child is starpu_sched_node_add_child
+	.remove_child is starpu_sched_node_remove_child
+	.notify_change_workers does nothing
+	.deinit_data does nothing
 
 
 \fn void starpu_sched_node_destroy(struct starpu_sched_node * node)
 \ingroup API_Modularized_Scheduler
-    	 free data allocated by starpu_sched_node_create, but dont call node->deinit_data(node)
+    	 free data allocated by starpu_sched_node_create and call node->deinit_data(node)
+	 set to null the member starpu_sched_node::fathers[sched_ctx_id] of all child if its equal to \p node
+
 \fn void starpu_sched_node_set_father(struct starpu_sched_node * node, struct starpu_sched_node * father_node, unsigned sched_ctx_id)
 \ingroup API_Modularized_Scheduler
-    	 set node->fathers[sched_ctx_id] to father_node
+	 set node->fathers[sched_ctx_id] to father_node
+
 \fn void starpu_sched_node_add_child(struct starpu_sched_node* node, struct starpu_sched_node * child)
 \ingroup API_Modularized_Scheduler
-    	 add child to node->childs and increment nchilds as well
+	 add child to node->childs and increment nchilds as well.
 	 and dont modify child->fathers
+	 \p child must not be already in starpu_sched_node::childs of \p node
+
 \fn void starpu_sched_node_remove_child(struct starpu_sched_node * node, struct starpu_sched_node * child)
 \ingroup API_Modularized_Scheduler
-    	 remove child from node->childs and decrement nchilds
+	 remove child from node->childs and decrements nchilds
+	 \p child must be in starpu_sched_node::child of \p node
 
 
 \fn int starpu_sched_node_can_execute_task(struct starpu_sched_node * node, struct starpu_task * task)
 \ingroup API_Modularized_Scheduler
 	 return true iff \p node can execute \p task, this function take into account the workers available in the scheduling context
 
+\fn int STARPU_WARN_UNUSED_RESULT starpu_sched_node_execute_preds(struct starpu_sched_node * node, struct starpu_task * task, double * length);
+\ingroup API_Modularized_Scheduler
+	 return a non null value if \p node can execute \p task.
+	 write the execution prediction length for the best implementation of the best worker available and write this at \p length address.
+	 this result is more relevant if starpu_sched_node::is_homogeneous is non null.
+	 if a worker need to be calibrated for an implementation, nan is set to \p length.
+
+\fn double starpu_sched_node_transfer_length(struct starpu_sched_node * node, struct starpu_task * task);
+\ingroup API_Modularized_Scheduler
+	 return the average time to transfer \p task data to underlying \p node workers.
+
 \fn struct starpu_sched_node * starpu_sched_node_worker_get(int workerid)
 \ingroup API_Modularized_Scheduler
-	 return the struct starpu_sched_node corresponding to \p workerid
+	 return the struct starpu_sched_node corresponding to \p workerid. Undefined if \p workerid is not a valid workerid
 
 
 \fn int starpu_sched_node_is_worker(struct starpu_sched_node * node)
@@ -110,47 +125,72 @@ this structure containt predictions for a task and is filled by starpu_sched_nod
 
 \fn struct starpu_sched_node * starpu_sched_node_fifo_create(void * arg STARPU_ATTRIBUTE_UNUSED)
 \ingroup API_Modularized_Scheduler
-	 return a struct starpu_sched_node with a fifo. A stable sort is performed according to tasks priorities.
-	 a push_task call on this node does not perform recursive calls, underlaying nodes will have to call pop_task to get it.
-	 estimated_execute_preds function compute the estimated length by dividing the sequencial length by the number of underlaying workers
+	 Return a struct starpu_sched_node with a fifo. A stable sort is performed according to tasks priorities.
+	 A push_task call on this node does not perform recursive calls, underlaying nodes will have to call pop_task to get it.
+	 starpu_sched_node::estimated_end function compute the estimated length by dividing the sequential length by the number of underlying workers. Do not take into account tasks that are currently executed.
 
 \fn int starpu_sched_node_is_fifo(struct starpu_sched_node * node)
 \ingroup API_Modularized_Scheduler
 	 return true iff \p node is a fifo node
 
+\fn struct starpu_sched_node * starpu_sched_node_work_stealing_create(void * arg STARPU_ATTRIBUTE_UNUSED)
+\ingroup API_Modularized_Scheduler
+	 return a node that perform a work stealing scheduling. Tasks are pushed in a round robin way. estimated_end return the average of expected length of fifos, starting at the average of the expected_end of his childs. When a worker have to steal a task, it steal a task in a round robin way, and get the last pushed task of the higher priority.
+
 
-\fn struct starpu_sched_node * starpu_sched_node_work_stealing_create(void)
+\fn int starpu_sched_tree_work_stealing_push_task(struct starpu_task *task)
 \ingroup API_Modularized_Scheduler
-	 return a node that perform a work stealing scheduling,
-	 a special function may be used instead of starpu_sched_tree_push_task that push tasks in the good fifo
-	  
+	 undefined if there is no work stealing node in the scheduler. If any, \p task is pushed in a default way if the caller is the application, and in the caller's fifo if its a worker.
+
 \fn int starpu_sched_node_is_work_stealing(struct starpu_sched_node * node)
 \ingroup API_Modularized_Scheduler
 	 return true iff \p node is a work stealing node
 
 \fn struct starpu_sched_node * starpu_sched_node_random_create(void * arg STARPU_ATTRIBUTE_UNUSED)
 \ingroup API_Modularized_Scheduler
-	 create a node that perform a random scheduling
+	 create a node that perform a random scheduling. 
 
 \fn int starpu_sched_node_is_random(struct starpu_sched_node *);
 \ingroup API_Modularized_Scheduler
 	 return true iff \p node is a random node
 
-
-\fn struct starpu_sched_node * starpu_sched_node_heft_create(void * arg STARPU_ATTRIBUTE_UNUSED)
-\ingroup API_Modularized_Scheduler
-	 this node perform a heft scheduling using data provided by estimated_execute_preds, if no pred_model are available a random node is used to push tasks
-
-\fn void starpu_sched_node_heft_set_no_model_node(struct starpu_sched_node * heft_node, struct starpu_sched_node * (*create_no_model_node)(void * arg), void * arg)
+\fn struct starpu_sched_node * starpu_sched_node_heft_create(struct starpu_heft_data * heft_data)
 \ingroup API_Modularized_Scheduler
-	 this should disapear and use arg to pass that kind of parameters
+	 this node perform a heft scheduling using data provided by estimated_execute_preds, if no pred_model are available a random node is used to push tasks.
 
 \fn int starpu_sched_node_is_heft(struct starpu_sched_node * node)
 \ingroup API_Modularized_Scheduler
 	 return true iff \p node is a heft node
-\fn double starpu_sched_compute_expected_time(double now, double predicted_end, double predicted_length, double predicted_transfer)
+
+\struct starpu_heft_data
+\ingroup API_Modularized_Scheduler
+starpu_sched_node_heft_create parameters
+\var starpu_heft_data::alpha
+	 coefficient applied to computation length 
+\var starpu_heft_data::beta
+	 coefficient applied to communication length
+\var starpu_heft_data::gamma
+	 coefficient applied to power consumption
+\var starpu_heft_data::idle_power
+	 idle consumption of the machine
+\var starpu_heft_data::no_perf_model_node_create
+	 called to create the node to push task for whom no perf model is available
+\var starpu_heft_data::arg_no_perf_model
+	 argument passed to starpu_heft_data::no_perf_model_node_create
+\var starpu_heft_data::calibrating_node_create
+	 idem for tasks with an non calibrated implementation
+\var starpu_heft_data::arg_calibrating_node
+	 argument passed to starpu_heft_data::calibrating_node_create
+
+
+\fn struct starpu_sched_node * starpu_sched_node_best_implementation_create(void * arg STARPU_ATTRIBUTE_UNUSED);
 \ingroup API_Modularized_Scheduler
-	 compute expected end of a fifo with a task of \p predicted_length by taking in acount the fact that data may not be ready at \p predicted_end
+	 Select the implementation that offer the shortest computation length for the first worker that can execute the task.
+	 Or an implementation that need to be calibrated.
+	 Also set starpu_task::predicted and starpu_task::predicted_transfer for memory node of the first suitable workerid.
+	 If starpu_sched_node::push method is called and starpu_sched_node::nchild > 1 the result is undefined.
+
+
 
 \fn struct starpu_sched_tree * starpu_sched_tree_create(void)
 \ingroup API_Modularized_Scheduler
@@ -183,11 +223,14 @@ this structure containt predictions for a task and is filled by starpu_sched_nod
 \ingroup API_Modularized_Scheduler
 	 compatibility with starpu_sched_policy interface
 
-\fn struct starpu_bitmap * _starpu_get_worker_mask(struct starpu_task *task)
-\ingroup API_Modularized_Scheduler
-	 return a ref to bitmap that give available worker to execute \p task
 
+\fn void starpu_sched_tree_update_workers(struct starpu_sched_tree * t)
+\ingroup API_Modularized_Scheduler
+	 recursively set all starpu_sched_node::workers, do not take into account shared parts (except workers).
 
+\fn void starpu_sched_tree_update_workers_in_ctx(struct starpu_sched_tree * t)
+\ingroup API_Modularized_Scheduler
+	 recursively set all starpu_sched_node::workers_in_ctx, do not take into account shared parts (except workers)
 
 \struct starpu_bitmap;
 \ingroup API_Modularized_Scheduler
@@ -231,4 +274,3 @@ this structure containt predictions for a task and is filled by starpu_sched_nod
 	 return the position of set bit right after \p e in \p bitmap, -1 if none
 
 */
-

doc/doxygen/chapters/modularized_scheduler.doxy

@@ -23,15 +23,17 @@ should call pop.
 \section Initialization
 Scheduler node are created with the starpu_sched_node_foo_create() functions
 and then must be assembled using them starpu_sched_node::add_child and
-starpu_sched_node::remove_child function.
-A father can be set in order to allow him to be reacheable by a starpu_sched_node::pop_task
+starpu_sched_node::remove_child functions.
+A father can be set to allow him to be reacheable by a starpu_sched_node::pop_task
 call.
 
 Underlayings workers are memoized in starpu_sched_node::workers. Hence the
 function starpu_sched_tree_update_workers should be called when the scheduler is
 finished, or modified.
 
-\section Adding and removing workers
+\section AddingAndRemovingWorkers Adding and removing workers
+The hypervisor can balance load between contexts by adding or removing workers from a scheduler.
+
 
 
 

include/starpu_sched_node.h

@@ -105,9 +105,6 @@ struct starpu_sched_tree
 
 
 
-int STARPU_WARN_UNUSED_RESULT starpu_sched_node_execute_preds(struct starpu_sched_node * node, struct starpu_task * task, double * length);
-double starpu_sched_node_transfer_length(struct starpu_sched_node * node, struct starpu_task * task);
-
 struct starpu_sched_node * starpu_sched_node_create(void);
 
 void starpu_sched_node_destroy(struct starpu_sched_node * node);
@@ -117,6 +114,9 @@ void starpu_sched_node_remove_child(struct starpu_sched_node * node, struct star
 
 
 int starpu_sched_node_can_execute_task(struct starpu_sched_node * node, struct starpu_task * task);
+int STARPU_WARN_UNUSED_RESULT starpu_sched_node_execute_preds(struct starpu_sched_node * node, struct starpu_task * task, double * length);
+double starpu_sched_node_transfer_length(struct starpu_sched_node * node, struct starpu_task * task);
+
 
 /* no public create function for workers because we dont want to have several node_worker for a single workerid */
 struct starpu_sched_node * starpu_sched_node_worker_get(int workerid);
@@ -130,10 +130,10 @@ int starpu_sched_node_worker_get_workerid(struct starpu_sched_node * worker_node
 
 struct starpu_sched_node * starpu_sched_node_fifo_create(void * arg STARPU_ATTRIBUTE_UNUSED);
 int starpu_sched_node_is_fifo(struct starpu_sched_node * node);
-//struct starpu_task_list  starpu_sched_node_fifo_get_non_executable_tasks(struct starpu_sched_node * fifo_node);
 
 struct starpu_sched_node * starpu_sched_node_work_stealing_create(void * arg STARPU_ATTRIBUTE_UNUSED);
 int starpu_sched_node_is_work_stealing(struct starpu_sched_node * node);
+int starpu_sched_tree_work_stealing_push_task(struct starpu_task *task);
 
 struct starpu_sched_node * starpu_sched_node_random_create(void * arg STARPU_ATTRIBUTE_UNUSED);
 int starpu_sched_node_is_random(struct starpu_sched_node *);
@@ -164,10 +164,7 @@ int starpu_sched_node_is_heft(struct starpu_sched_node * node);
  * cannot have several childs if push_task is called
  */
 struct starpu_sched_node * starpu_sched_node_best_implementation_create(void * arg STARPU_ATTRIBUTE_UNUSED);
-/* this node select an implementation that need to be calibrated.
- * cannot have several childs if push_task is called.
- */
-struct starpu_sched_node * starpu_sched_node_calibration_create(void * arg STARPU_ATTRIBUTE_UNUSED);
+
 /*create an empty tree
  */
 struct starpu_sched_tree * starpu_sched_tree_create(unsigned sched_ctx_id);
@@ -181,8 +178,7 @@ void starpu_sched_node_destroy_rec(struct starpu_sched_node * node, unsigned sch
 /* update all the node->workers member recursively
  */
 void starpu_sched_tree_update_workers(struct starpu_sched_tree * t);
-/* 
- *
+/* idem for workers_in_ctx 
  */
 void starpu_sched_tree_update_workers_in_ctx(struct starpu_sched_tree * t);
 /* wake up underlaying workers of node
@@ -196,23 +192,6 @@ void starpu_sched_tree_remove_workers(unsigned sched_ctx_id, int *workerids, uns
 void starpu_sched_node_worker_pre_exec_hook(struct starpu_task * task);
 void starpu_sched_node_worker_post_exec_hook(struct starpu_task * task);
 
-/* return the bitmap of worker that are allowed to use in this scheduling context
- */
-struct starpu_bitmap * _starpu_get_worker_mask(unsigned sched_ctx_id);
-
-/* this function is called to initialize a scheduler tree
- */
-void starpu_sched_node_init_rec(struct starpu_sched_node * node);
-/* this function fill all the node->workers members
- */
-void _starpu_set_workers_bitmaps(void);
-/* this function call init data on all nodes in postfix order
- */
-void starpu_sched_tree_call_init_data(struct starpu_sched_tree * t);
-
-/* push task of list lower as possible in the tree, a non null value is returned if some task couldn't be pushed
- */
-int starpu_sched_node_push_tasks_to_firsts_suitable_parent(struct starpu_sched_node * node, struct starpu_task_list * list, int sched_ctx_id);
 
 
 struct starpu_bitmap * starpu_bitmap_create(void);

src/sched_policies/README

@@ -23,7 +23,7 @@ function call node->deinit_data(node) to free data allocated during
 creation
 
 Workers nodes are particulars, there is no creation function, only
-accessor to garanty unicity of worker nodes. worker_node->workers and
+accessor to guaranty uniqueness of worker nodes. worker_node->workers and
 worker_node->workers_in_ctx should not be modified.
 
 
@@ -48,9 +48,11 @@ node. We also have a problem for shared object. The first way seems to
 be better.
 
 
-
-
+Various things
 
 In several place realloc is used (in prio_deque and for
 starpu_sched_node_add_child), because we should not have a lot
-different priority level nor adding too many childs.
+different priority level nor adding too many childs.
+
+Estimated_end is defined differently for work stealing and fifo.
+I dont know which approach is better.

src/sched_policies/node_best_implementation.c

@@ -64,26 +64,6 @@ static struct starpu_task * select_best_implementation_pop_task(struct starpu_sc
 	return t;
 }
 
-
-
-static int select_calibration_push_task(struct starpu_sched_node * node, struct starpu_task * task)
-{
-	STARPU_ASSERT(node->nchilds == 1);
-	select_best_implementation_and_set_preds(node->workers_in_ctx, task);
-	return node->childs[0]->push_task(node->childs[0],task);
-}
-
-static struct starpu_task * select_calibration_pop_task(struct starpu_sched_node * node, unsigned sched_ctx_id)
-{
-	struct starpu_task * t;
-	if(!node->fathers[sched_ctx_id])
-		return NULL;
-	t = node->fathers[sched_ctx_id]->pop_task(node->fathers[sched_ctx_id], sched_ctx_id);
-	if(t)
-		select_best_implementation_and_set_preds(node->workers_in_ctx, t);
-	return t;
-}
-
 struct starpu_sched_node * starpu_sched_node_best_implementation_create(void * ARG STARPU_ATTRIBUTE_UNUSED)
 {
 	struct starpu_sched_node * node = starpu_sched_node_create();
@@ -93,10 +73,7 @@ struct starpu_sched_node * starpu_sched_node_best_implementation_create(void * A
 }
 
 
-struct starpu_sched_node * starpu_sched_node_calibration_create(void * arg STARPU_ATTRIBUTE_UNUSED)
-{
-	struct starpu_sched_node * node = starpu_sched_node_create();
-	node->push_task = select_calibration_push_task;
-	node->pop_task = select_calibration_pop_task;
-	return node;
-}
+
+
+
+

src/sched_policies/node_heft.c

@@ -99,12 +99,16 @@ static int heft_push_task(struct starpu_sched_node * node, struct starpu_task *
 	for(i = 0; i < nsuitable_nodes; i++)
 	{
 		int inode = suitable_nodes[i];
+#ifdef STARPU_DEVEL
+#warning FIXME: take power consumption into account
+#endif
 		double tmp = compute_fitness(d,
 					     estimated_ends_with_task[inode],
 					     best_exp_end_with_task,
 					     max_exp_end_with_task,
 					     estimated_transfer_length[inode],
 					     0.0);
+
 		if(tmp < best_fitness)
 		{
 			best_fitness = tmp;

src/sched_policies/node_sched.c

@@ -255,7 +255,6 @@ void starpu_sched_node_remove_child(struct starpu_sched_node * node, struct star
 {
 	STARPU_ASSERT(node && child);
 	STARPU_ASSERT(!starpu_sched_node_is_worker(node));
-	STARPU_ASSERT(node && child);
 	int pos;
 	for(pos = 0; pos < node->nchilds; pos++)
 		if(node->childs[pos] == child)
@@ -417,11 +416,11 @@ struct starpu_sched_node * starpu_sched_node_create(void)
 	node->workers_in_ctx = starpu_bitmap_create();
 	node->add_child = starpu_sched_node_add_child;
 	node->remove_child = starpu_sched_node_remove_child;
-	node->notify_change_workers = take_node_and_does_nothing;
 	node->pop_task = pop_task_node;
 	node->estimated_load = estimated_load;
 	node->estimated_end = _starpu_sched_node_estimated_end_min;
 	node->deinit_data = take_node_and_does_nothing;
+	node->notify_change_workers = take_node_and_does_nothing;
 	return node;
 }
 

src/sched_policies/node_work_stealing.c

@@ -200,7 +200,7 @@ static int push_task(struct starpu_sched_node * node, struct starpu_task * task)
 
 
 //this function is special, when a worker call it, we want to push the task in his fifo
-int _starpu_ws_push_task(struct starpu_task *task)
+int starpu_sched_tree_work_stealing_push_task(struct starpu_task *task)
 {
 	int workerid = starpu_worker_get_id();
 	if(workerid == -1)
@@ -362,7 +362,7 @@ struct starpu_sched_policy _starpu_sched_tree_ws_policy =
 	.deinit_sched = deinitialize_ws_center_policy,
 	.add_workers = starpu_sched_tree_add_workers,
 	.remove_workers = starpu_sched_tree_remove_workers,
-	.push_task = _starpu_ws_push_task,
+	.push_task = starpu_sched_tree_work_stealing_push_task,
 	.pop_task = starpu_sched_tree_pop_task,
 	.pre_exec_hook = NULL,
 	.post_exec_hook = NULL,

src/sched_policies/node_worker.c

@@ -220,8 +220,7 @@ static inline struct starpu_task * _starpu_worker_task_list_pop(struct _starpu_w
 			struct starpu_task * task = t->task;
 			t->task = NULL;
 			int * p = t->left ? t->pntasks : &t->ntasks;
-			(void) STARPU_ATOMIC_ADD(p, -1);
-			if(*p == 0)
+			if(STARPU_ATOMIC_ADD(p, -1) == 0)
 				_starpu_task_grid_unset_left_right_member(t);
 			l->ntasks--;
 			if(!isnan(task->predicted))

src/sched_policies/sched_node.h

@@ -10,4 +10,6 @@ void _starpu_sched_node_unlock_worker(int workerid);
 struct _starpu_worker * _starpu_sched_node_worker_get_worker(struct starpu_sched_node *);
 struct _starpu_combined_worker * _starpu_sched_node_combined_worker_get_combined_worker(struct starpu_sched_node * worker_node);
 
+struct starpu_bitmap * _starpu_get_worker_mask(unsigned sched_ctx_id);
+
 #endif