use sched_ctx id instead of reference in user mode + small fixes

examples/cholesky_and_lu/cholesky/cholesky.h

@@ -71,10 +71,7 @@ void chol_cublas_codelet_update_u21(void *descr[], void *_args);
 void chol_cublas_codelet_update_u22(void *descr[], void *_args);
 #endif
 
-int run_cholesky_grain_tag(struct starpu_sched_ctx *sched_ctx, int argc, char **argv);
-double run_cholesky_implicit(struct starpu_sched_ctx *sched_ctx, int argc, char **argv, double *timing, pthread_barrier_t *barrier);
-int run_cholesky_tag(struct starpu_sched_ctx *sched_ctx, int argc, char **argv);
-double run_cholesky_tile_tag(struct starpu_sched_ctx *sched_ctx, int argc, char **argv);
+double run_cholesky_implicit(int sched_ctx, int argc, char **argv, double *timing, pthread_barrier_t *barrier);
 
 extern struct starpu_perfmodel_t chol_model_11;
 extern struct starpu_perfmodel_t chol_model_21;

examples/cholesky_and_lu/cholesky/cholesky_implicit.c

@@ -69,7 +69,7 @@ static void callback_turn_spmd_on(void *arg __attribute__ ((unused)))
 	cl22.type = STARPU_SPMD;
 }
 
-static double _cholesky(starpu_data_handle dataA, unsigned nblocks, struct starpu_sched_ctx *sched_ctx, double *timing)
+static double _cholesky(starpu_data_handle dataA, unsigned nblocks, int sched_ctx, double *timing)
 {
 	struct timeval start;
 	struct timeval end;
@@ -85,12 +85,13 @@ static double _cholesky(starpu_data_handle dataA, unsigned nblocks, struct starp
 	{
                 starpu_data_handle sdatakk = starpu_data_get_sub_data(dataA, 2, k, k);
 
-		if(sched_ctx != NULL)
-			starpu_insert_task_to_ctx(sched_ctx, &cl11,
-						  STARPU_PRIORITY, prio_level,
-						  STARPU_RW, sdatakk,
-						  STARPU_CALLBACK, (k == 3*nblocks/4)?callback_turn_spmd_on:NULL,
-						  0);
+		if(sched_ctx != -1)
+			starpu_insert_task(&cl11,
+					   STARPU_PRIORITY, prio_level,
+					   STARPU_RW, sdatakk,
+					   STARPU_CALLBACK, (k == 3*nblocks/4)?callback_turn_spmd_on:NULL,
+					   STARPU_CTX, sched_ctx,
+					   0);
 		else
 			starpu_insert_task(&cl11,
 					   STARPU_PRIORITY, prio_level,
@@ -101,12 +102,13 @@ static double _cholesky(starpu_data_handle dataA, unsigned nblocks, struct starp
 		for (j = k+1; j<nblocks; j++)
 		{
                         starpu_data_handle sdatakj = starpu_data_get_sub_data(dataA, 2, k, j);
-			if(sched_ctx != NULL)
-				starpu_insert_task_to_ctx(sched_ctx, &cl21,
-							  STARPU_PRIORITY, (j == k+1)?prio_level:STARPU_DEFAULT_PRIO,
-							  STARPU_R, sdatakk,
-							  STARPU_RW, sdatakj,
-							  0);
+			if(sched_ctx != -1)
+				starpu_insert_task(&cl21,
+						   STARPU_PRIORITY, (j == k+1)?prio_level:STARPU_DEFAULT_PRIO,
+						   STARPU_R, sdatakk,
+						   STARPU_RW, sdatakj,
+						   STARPU_CTX, sched_ctx,
+						   0);
 			else
 				starpu_insert_task(&cl21,
 						   STARPU_PRIORITY, (j == k+1)?prio_level:STARPU_DEFAULT_PRIO,
@@ -120,13 +122,14 @@ static double _cholesky(starpu_data_handle dataA, unsigned nblocks, struct starp
                                 {
 					starpu_data_handle sdataki = starpu_data_get_sub_data(dataA, 2, k, i);
 					starpu_data_handle sdataij = starpu_data_get_sub_data(dataA, 2, i, j);
-					if(sched_ctx != NULL)
-						starpu_insert_task_to_ctx(sched_ctx, &cl22,
-									  STARPU_PRIORITY, ((i == k+1) && (j == k+1))?prio_level:STARPU_DEFAULT_PRIO,
-									  STARPU_R, sdataki,
-									  STARPU_R, sdatakj,
-									  STARPU_RW, sdataij,
-									  0);
+					if(sched_ctx != -1)
+						starpu_insert_task(&cl22,
+								   STARPU_PRIORITY, ((i == k+1) && (j == k+1))?prio_level:STARPU_DEFAULT_PRIO,
+								   STARPU_R, sdataki,
+								   STARPU_R, sdatakj,
+								   STARPU_RW, sdataij,
+								   STARPU_CTX, sched_ctx,
+								   0);
 					else 
 						starpu_insert_task(&cl22,
 								   STARPU_PRIORITY, ((i == k+1) && (j == k+1))?prio_level:STARPU_DEFAULT_PRIO,
@@ -139,7 +142,7 @@ static double _cholesky(starpu_data_handle dataA, unsigned nblocks, struct starp
 		}
 	}
 
-	if(sched_ctx != NULL)
+	if(sched_ctx != -1)
 		starpu_wait_for_all_tasks_of_sched_ctx(sched_ctx);
 	else
 		starpu_task_wait_for_all();
@@ -157,7 +160,7 @@ static double _cholesky(starpu_data_handle dataA, unsigned nblocks, struct starp
 	return (flop/(*timing)/1000.0f);
 }
 
-static double cholesky(float *matA, unsigned size, unsigned ld, unsigned nblocks, struct starpu_sched_ctx *sched_ctx, double *timing)
+static double cholesky(float *matA, unsigned size, unsigned ld, unsigned nblocks, int sched_ctx, double *timing)
 {
 	starpu_data_handle dataA;
 
@@ -181,7 +184,7 @@ static double cholesky(float *matA, unsigned size, unsigned ld, unsigned nblocks
 	return _cholesky(dataA, nblocks, sched_ctx, timing);
 }
 
-double run_cholesky_implicit(struct starpu_sched_ctx *sched_ctx, int argc, char **argv, double *timing, pthread_barrier_t *barrier)
+double run_cholesky_implicit(int sched_ctx, int argc, char **argv, double *timing, pthread_barrier_t *barrier)
 {
 	/* create a simple definite positive symetric matrix example
 	 *

examples/cholesky_and_lu/cholesky_and_lu.c

@@ -4,6 +4,7 @@
 typedef struct {
   int argc;
   char **argv;
+  int ctx;
 } params;
 
 typedef struct {
@@ -13,14 +14,11 @@ typedef struct {
 
 #define NSAMPLES 1
 
-struct starpu_sched_ctx sched_ctx;
-struct starpu_sched_ctx sched_ctx2;
-struct starpu_sched_ctx sched_ctx3;
-struct starpu_sched_ctx sched_ctx4;
 pthread_barrier_t barrier;
 
 void* func_cholesky(void *val){
   params *p = (params*)val;
+  int sched_ctx = p->ctx;
 
   int i;
   retvals *rv  = (retvals*)malloc(sizeof(retvals));
@@ -29,7 +27,7 @@ void* func_cholesky(void *val){
   double timing = 0;
   for(i = 0; i < NSAMPLES; i++)
     {
-      rv->flops += run_cholesky_implicit(&sched_ctx, p->argc, p->argv, &timing, &barrier);
+      rv->flops += run_cholesky_implicit(sched_ctx, p->argc, p->argv, &timing, (sched_ctx == -1 ? NULL : &barrier));
       rv->avg_timing += timing;
     }
 
@@ -38,63 +36,22 @@ void* func_cholesky(void *val){
   return (void*)rv;
 }
 
-void* func_cholesky2(void *val){
-  params *p = (params*)val;
-
-  int i;
-  retvals *rv  = (retvals*)malloc(sizeof(retvals));
-  rv->flops = 0;
-  rv->avg_timing = 0;
-  double timing = 0;
-
-  for(i = 0; i < NSAMPLES; i++)
-    {
-      rv->flops += run_cholesky_implicit(&sched_ctx2, p->argc, p->argv, &timing, &barrier);
-      rv->avg_timing += timing;
-    }
-
-  rv->flops /= NSAMPLES;
-  rv->avg_timing /= NSAMPLES;
-  return (void*)rv;
-}
-
-void* func_cholesky3(void *val){
-  params *p = (params*)val;
-
-  int i;
-  retvals *rv  = (retvals*)malloc(sizeof(retvals));
-  rv->flops = 0;
-  rv->avg_timing = 0;
-  double timing = 0;
-
-  for(i = 0; i < NSAMPLES; i++)
-    {
-      rv->flops += run_cholesky_implicit(NULL, p->argc, p->argv, &timing, NULL);
-      rv->avg_timing += timing;
-    }
-
-  rv->flops /= NSAMPLES;
-  rv->avg_timing /= NSAMPLES;
-
-  return (void*)rv;
-}
-
-void cholesky_vs_cholesky(params *p){
+void cholesky_vs_cholesky(params *p1, params *p2, params *p3){
   /* 2 cholesky in different ctxs */
   starpu_init(NULL);
   starpu_helper_cublas_init();
 
   int procs[] = {1, 2, 3, 4, 5, 6};
-  starpu_create_sched_ctx(&sched_ctx, "heft", procs, 6, "cholesky1");
+  p1->ctx = starpu_create_sched_ctx("heft", procs, 6, "cholesky1");
 
   int procs2[] = {0, 7, 8, 9, 10, 11};
-  starpu_create_sched_ctx(&sched_ctx2, "heft", procs2, 6, "cholesky2");
+  p2->ctx = starpu_create_sched_ctx("heft", procs2, 6, "cholesky2");
 
   pthread_t tid[2];
   pthread_barrier_init(&barrier, NULL, 2);
 
-  pthread_create(&tid[0], NULL, (void*)func_cholesky, (void*)p);
-  pthread_create(&tid[1], NULL, (void*)func_cholesky2, (void*)p);
+  pthread_create(&tid[0], NULL, (void*)func_cholesky, (void*)p1);
+  pthread_create(&tid[1], NULL, (void*)func_cholesky, (void*)p2);
 
   void *gflops_cholesky1;
   void *gflops_cholesky2;
@@ -109,7 +66,7 @@ void cholesky_vs_cholesky(params *p){
   /* starpu_init(NULL); */
   /* starpu_helper_cublas_init(); */
 
-  /* void *gflops_cholesky3 = func_cholesky3(p); */
+  /* void *gflops_cholesky3 = func_cholesky((void*)p3); */
 
   /* starpu_helper_cublas_shutdown(); */
   /* starpu_shutdown(); */
@@ -121,8 +78,8 @@ void cholesky_vs_cholesky(params *p){
 
   /* pthread_t tid2[2]; */
 
-  /* pthread_create(&tid2[0], NULL, (void*)func_cholesky3, (void*)p); */
-  /* pthread_create(&tid2[1], NULL, (void*)func_cholesky3, (void*)p); */
+  /* pthread_create(&tid2[0], NULL, (void*)func_cholesky, (void*)p3); */
+  /* pthread_create(&tid2[1], NULL, (void*)func_cholesky, (void*)p3); */
 
   /* void *gflops_cholesky4; */
   /* void *gflops_cholesky5; */
@@ -146,11 +103,19 @@ void cholesky_vs_cholesky(params *p){
 
 int main(int argc, char **argv)
 {
-  params p;
-  p.argc = argc;
-  p.argv = argv;
-
-  cholesky_vs_cholesky(&p);
+  params p1;
+  p1.argc = argc;
+  p1.argv = argv;
+
+  params p2;
+  p2.argc = argc;
+  p2.argv = argv;
+
+  params p3;
+  p3.argc = argc;
+  p3.argv = argv;
+  p3.ctx = -1;
+  cholesky_vs_cholesky(&p1, &p2,&p3);
 
   return 0;
 }

include/starpu_scheduler.h

@@ -32,6 +32,7 @@ struct starpu_machine_topology_s {
 
 	unsigned ncombinedworkers;
 
+	unsigned nsched_ctxs;
 #ifdef STARPU_HAVE_HWLOC
 	hwloc_topology_t hwtopology;
 #else
@@ -63,13 +64,13 @@ struct starpu_machine_topology_s {
  * field of the starpu_conf structure passed to the starpu_init function. */
 struct starpu_sched_policy_s {
 	/* Initialize the scheduling policy. */
-	void (*init_sched)(struct starpu_sched_ctx *);
+	void (*init_sched)(int);
 
 	/* Cleanup the scheduling policy. */
-	void (*deinit_sched)(struct starpu_sched_ctx *);
+	void (*deinit_sched)(int);
 
 	/* Insert a task into the scheduler. */
-        int (*push_task)(struct starpu_task *, struct starpu_sched_ctx *);
+        int (*push_task)(struct starpu_task *, int);
 	/* Notify the scheduler that a task was pushed on the worker. This
 	 * method is called when a task that was explicitely assigned to a
 	 * worker is scheduled. This method therefore permits to keep the state
@@ -78,7 +79,7 @@ struct starpu_sched_policy_s {
 	void (*push_task_notify)(struct starpu_task *, int workerid);
 
 	/* Insert a priority task into the scheduler. */
-        int (*push_prio_task)(struct starpu_task *, struct starpu_sched_ctx *);
+        int (*push_prio_task)(struct starpu_task *, int);
 
 	/* Get a task from the scheduler. The mutex associated to the worker is
 	 * already taken when this method is called. */
@@ -100,24 +101,13 @@ struct starpu_sched_policy_s {
 	const char *policy_description;
 };
 
-struct starpu_sched_ctx {
-	struct starpu_sched_policy_s *sched_policy; /* policy of the contex */
-	int workerid[STARPU_NMAXWORKERS]; /* list of indices of workers */
-	int nworkers_in_ctx; /* number of threads in contex */
-	unsigned is_initial_sched; /* we keep an initial sched which we never delete */
-	pthread_cond_t submitted_cond; /* cond used for no of submitted tasks to a sched_ctx */
-	pthread_mutex_t submitted_mutex; /* mut used for no of submitted tasks to a sched_ctx */
-	int nsubmitted;	 /* counter used for no of submitted tasks to a sched_ctx */
-	const char *sched_name;
-};
-
-void starpu_create_sched_ctx(struct starpu_sched_ctx *sched_ctx, const char *policy_name, int *workerids_in_ctx, int nworkerids_in_ctx, const char *sched_name);
+int starpu_create_sched_ctx(const char *policy_name, int *workerids_in_ctx, int nworkerids_in_ctx, const char *sched_name);
 
-void starpu_delete_sched_ctx(struct starpu_sched_ctx *sched_ctx);
+void starpu_delete_sched_ctx(int sched_ctx);
 
-void starpu_add_workers_to_sched_ctx(int *workerids_in_ctx, int nworkerids_in_ctx, struct starpu_sched_ctx *sched_ctx);
+void starpu_add_workers_to_sched_ctx(int *workerids_in_ctx, int nworkerids_in_ctx, int sched_ctx);
 
-void starpu_remove_workers_from_sched_ctx(int *workerids_in_ctx, int nworkerids_in_ctx, struct starpu_sched_ctx *sched_ctx);
+void starpu_remove_workers_from_sched_ctx(int *workerids_in_ctx, int nworkerids_in_ctx, int sched_ctx);
 
 /* When there is no available task for a worker, StarPU blocks this worker on a
 condition variable. This function specifies which condition variable (and the
@@ -149,7 +139,7 @@ int starpu_push_local_task(int workerid, struct starpu_task *task, int back);
 /* By convention, the default priority level should be 0 so that we can
  * statically allocate tasks with a default priority. */
 #define STARPU_DEFAULT_PRIO	0
-
+#define STARPU_NMAX_SCHED_CTXS 10
 int starpu_sched_get_min_priority(void);
 int starpu_sched_get_max_priority(void);
 

include/starpu_task.h

@@ -158,7 +158,7 @@ struct starpu_task {
 	 * NULL. */
 	void *starpu_private;
      
-        struct starpu_sched_ctx *sched_ctx;
+        int sched_ctx;
   
 };
 
@@ -183,7 +183,8 @@ struct starpu_task {
 	.status = STARPU_TASK_INVALID,			\
 	.profiling_info = NULL,				\
 	.predicted = -1.0,				\
-	.starpu_private = NULL				\
+	.starpu_private = NULL,				\
+	.sched_ctx = -1					\
 };
 
 /*
@@ -246,7 +247,7 @@ struct starpu_task *starpu_task_create(void);
  * allocated task results in an undefined behaviour. */
 void starpu_task_destroy(struct starpu_task *task);
 int starpu_task_submit(struct starpu_task *task);
-int starpu_task_submit_to_ctx(struct starpu_task *task, struct starpu_sched_ctx *sched_ctx);
+int starpu_task_submit_to_ctx(struct starpu_task *task, int sched_ctx);
 
 /* This function blocks until the task was executed. It is not possible to
  * synchronize with a task more than once. It is not possible to wait
@@ -261,7 +262,7 @@ int starpu_task_wait_for_all(void);
 
 /* This function waits until all the tasks that were already submitted to a specific
  * context have been executed. */
-int starpu_wait_for_all_tasks_of_sched_ctx(struct starpu_sched_ctx *sched_ctx);
+int starpu_wait_for_all_tasks_of_sched_ctx(int sched_ctx);
 
 void starpu_display_codelet_stats(struct starpu_codelet_t *cl);
 

include/starpu_util.h

@@ -176,10 +176,10 @@ int starpu_data_cpy(starpu_data_handle dst_handle, starpu_data_handle src_handle
 #define STARPU_CALLBACK_ARG	(1<<6)	/* Argument of the callback function (of type void *) */
 #define STARPU_PRIORITY		(1<<7)	/* Priority associated to the task */
 #define STARPU_EXECUTE		(1<<8)	/* Used by MPI to define which task is going to execute the codelet */
+#define STARPU_CTX		(1<<9)	/* Used to define which ctx will execute the */
 
 /* Wrapper to create a task. */
 void starpu_insert_task(starpu_codelet *cl, ...);
-void starpu_insert_task_to_ctx(struct starpu_sched_ctx *sched_ctx,starpu_codelet *cl, ...);
 
 /* Retrieve the arguments of type STARPU_VALUE associated to a task
  * automatically created using starpu_insert_task. */
@@ -193,6 +193,5 @@ void starpu_pack_cl_args(char **arg_buffer, size_t *arg_buffer_size, ...);
 }
 #endif
 
-#define STARPU_NMAXSCHEDCTXS 32
 
 #endif // __STARPU_UTIL_H__

src/core/sched_ctx.c

@@ -9,11 +9,15 @@ static pthread_cond_t wakeup_ths_cond = PTHREAD_COND_INITIALIZER;
 static pthread_mutex_t blocking_ths_mutex = PTHREAD_MUTEX_INITIALIZER;
 static int nblocked_ths = 0;
 
-void _starpu_create_sched_ctx(struct starpu_sched_ctx *sched_ctx, const char *policy_name, int
-			      *workerids_in_ctx, int nworkerids_in_ctx, unsigned is_initial_sched,
-			      const char *sched_name)
+int _starpu_create_sched_ctx(const char *policy_name, int *workerids_in_ctx, 
+			     int nworkerids_in_ctx, unsigned is_initial_sched,
+			     const char *sched_name)
 {
 	struct starpu_machine_config_s *config = (struct starpu_machine_config_s *)_starpu_get_machine_config();
+	STARPU_ASSERT(config->topology.nsched_ctxs < STARPU_NMAX_SCHED_CTXS - 1);
+
+	struct starpu_sched_ctx *sched_ctx = &config->sched_ctxs[config->topology.nsched_ctxs];
+
 	int nworkers = config->topology.nworkers;
 	
 	STARPU_ASSERT(nworkerids_in_ctx <= nworkers);
@@ -61,7 +65,10 @@ void _starpu_create_sched_ctx(struct starpu_sched_ctx *sched_ctx, const char *po
 
 	_starpu_init_sched_policy(config, sched_ctx, policy_name);
 	
-	return;
+	sched_ctx->sched_ctx_id = config->topology.nsched_ctxs;
+	config->topology.nsched_ctxs++;
+
+ 	return sched_ctx->sched_ctx_id;
 }
 
 void _starpu_decrement_nblocked_ths(void)
@@ -107,8 +114,7 @@ static int _starpu_wait_for_all_threads_to_wake_up(void)
 	return 0;
 }
 
-static int set_changing_ctx_flag(starpu_worker_status changing_ctx, int nworkerids_in_ctx, int
-				 *workerids_in_ctx)
+static int set_changing_ctx_flag(starpu_worker_status changing_ctx, int nworkerids_in_ctx, int *workerids_in_ctx)
 {
 	struct starpu_machine_config_s *config = _starpu_get_machine_config();
 
@@ -154,16 +160,17 @@ static int set_changing_ctx_flag(starpu_worker_status changing_ctx, int nworkeri
   return 0;
 }
 
-void starpu_create_sched_ctx(struct starpu_sched_ctx *sched_ctx, const char *policy_name, int
-			     *workerids_in_ctx, int nworkerids_in_ctx, const char *sched_name)
+int starpu_create_sched_ctx(const char *policy_name, int *workerids_in_ctx, 
+			    int nworkerids_in_ctx, const char *sched_name)
 {
+	int ret;
 	/* block the workers until the contex is switched */
 	set_changing_ctx_flag(STATUS_CHANGING_CTX, nworkerids_in_ctx, workerids_in_ctx);
-	_starpu_create_sched_ctx(sched_ctx, policy_name, workerids_in_ctx, nworkerids_in_ctx, 0, sched_name);
+	ret = _starpu_create_sched_ctx(policy_name, workerids_in_ctx, nworkerids_in_ctx, 0, sched_name);
 	/* also wait the workers to wake up before using the context */
 	set_changing_ctx_flag(STATUS_UNKNOWN, nworkerids_in_ctx, workerids_in_ctx);
-
-	return;
+	
+	return ret;
 }
 
 static unsigned _starpu_worker_belongs_to_ctx(struct starpu_worker_s *workerarg, struct starpu_sched_ctx *sched_ctx)
@@ -178,7 +185,7 @@ static unsigned _starpu_worker_belongs_to_ctx(struct starpu_worker_s *workerarg,
 
 static void _starpu_remove_sched_ctx_from_worker(struct starpu_worker_s *workerarg, struct starpu_sched_ctx *sched_ctx)
 {
-	int i;
+	unsigned i;
 	unsigned to_remove = 0;
 	for(i = 0; i < workerarg->nctxs; i++)
 	  {
@@ -196,10 +203,11 @@ static void _starpu_remove_sched_ctx_from_worker(struct starpu_worker_s *workera
 	return;
 }
 
-void starpu_delete_sched_ctx(struct starpu_sched_ctx *sched_ctx)
+void starpu_delete_sched_ctx(int sched_ctx_id)
 {
-  if(!starpu_wait_for_all_tasks_of_sched_ctx(sched_ctx))
+	if(!starpu_wait_for_all_tasks_of_sched_ctx(sched_ctx_id))
 	  {
+		struct starpu_sched_ctx *sched_ctx = _starpu_get_sched_ctx(sched_ctx_id);
 
 		int nworkers = sched_ctx->nworkers_in_ctx;
 		int workerid;
@@ -296,7 +304,6 @@ void _starpu_decrement_nsubmitted_tasks_of_worker(int workerid)
 
 void _starpu_increment_nsubmitted_tasks_of_worker(int workerid)
 {
-  //  printf("task submitted to %d\n", workerid);
 	struct starpu_worker_s *worker = _starpu_get_worker_struct(workerid);
 
 	PTHREAD_MUTEX_LOCK(&worker->submitted_mutex);
@@ -356,8 +363,10 @@ static void _starpu_add_workers_to_sched_ctx(int *workerids_in_ctx, int nworkeri
 }
 
 void starpu_add_workers_to_sched_ctx(int *workerids_in_ctx, int nworkerids_in_ctx,
-				     struct starpu_sched_ctx *sched_ctx)
+				     int sched_ctx_id)
 {
+	struct starpu_sched_ctx *sched_ctx = _starpu_get_sched_ctx(sched_ctx_id);
+
 	/* block the workers until the contex is switched */
 	set_changing_ctx_flag(STATUS_CHANGING_CTX, nworkerids_in_ctx, workerids_in_ctx);
 	_starpu_add_workers_to_sched_ctx(workerids_in_ctx, nworkerids_in_ctx, sched_ctx);
@@ -451,12 +460,14 @@ static void _starpu_remove_workers_from_sched_ctx(int *workerids_in_ctx, int nwo
 }
 
 void starpu_remove_workers_from_sched_ctx(int *workerids_in_ctx, int nworkerids_in_ctx, 
-					  struct starpu_sched_ctx *sched_ctx)
+					  int sched_ctx_id)
 {
 	  /* wait for the workers concerned by the change of contex                       
 	   * to finish their work in the previous context */
 	if(!starpu_wait_for_all_tasks_of_workers(workerids_in_ctx, nworkerids_in_ctx))
 	  {
+		struct starpu_sched_ctx *sched_ctx = _starpu_get_sched_ctx(sched_ctx_id);
+
 		/* block the workers until the contex is switched */
 		set_changing_ctx_flag(STATUS_CHANGING_CTX, nworkerids_in_ctx, workerids_in_ctx);
 		_starpu_remove_workers_from_sched_ctx(workerids_in_ctx, nworkerids_in_ctx, sched_ctx);
@@ -467,20 +478,22 @@ void starpu_remove_workers_from_sched_ctx(int *workerids_in_ctx, int nworkerids_
 
 }
 
-int starpu_wait_for_all_tasks_of_sched_ctx(struct starpu_sched_ctx *sched_ctx)
+int starpu_wait_for_all_tasks_of_sched_ctx(int sched_ctx_id)
 {
-  if (STARPU_UNLIKELY(!_starpu_worker_may_perform_blocking_calls()))
-    return -EDEADLK;
-
-  PTHREAD_MUTEX_LOCK(&sched_ctx->submitted_mutex);
-
-
-  while (sched_ctx->nsubmitted > 0)
-    PTHREAD_COND_WAIT(&sched_ctx->submitted_cond, &sched_ctx->submitted_mutex);
-
-  PTHREAD_MUTEX_UNLOCK(&sched_ctx->submitted_mutex);
-
-  return 0;
+	struct starpu_sched_ctx *sched_ctx = _starpu_get_sched_ctx(sched_ctx_id);
+	
+	if (STARPU_UNLIKELY(!_starpu_worker_may_perform_blocking_calls()))
+	  return -EDEADLK;
+	
+	PTHREAD_MUTEX_LOCK(&sched_ctx->submitted_mutex);
+	
+	
+	while (sched_ctx->nsubmitted > 0)
+	  PTHREAD_COND_WAIT(&sched_ctx->submitted_cond, &sched_ctx->submitted_mutex);
+	
+	PTHREAD_MUTEX_UNLOCK(&sched_ctx->submitted_mutex);
+	
+	return 0;
 }
 
 void _starpu_decrement_nsubmitted_tasks_of_sched_ctx(struct starpu_sched_ctx *sched_ctx)

src/core/sched_ctx.h

@@ -1,4 +1,4 @@
- /* StarPU --- Runtime system for heterogeneous multicore architectures.
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
  * Copyright (C) 2010  Université de Bordeaux 1
  *
@@ -18,11 +18,21 @@
 #define __SCHED_CONTEXT_H__
 
 #include <starpu.h>
-#include <core/workers.h>
 #include <starpu_scheduler.h>
 
-
-void _starpu_create_sched_ctx(struct starpu_sched_ctx *sched_ctx, const char *policy_name, int *workerid, int nworkerids, unsigned is_init_sched, const char *sched_name);
+struct starpu_sched_ctx {
+	int sched_ctx_id;
+	struct starpu_sched_policy_s *sched_policy; /* policy of the contex */
+	int workerid[STARPU_NMAXWORKERS]; /* list of indices of workers */
+	int nworkers_in_ctx; /* number of threads in contex */
+	unsigned is_initial_sched; /* we keep an initial sched which we never delete */
+	pthread_cond_t submitted_cond; /* cond used for no of submitted tasks to a sched_ctx */
+	pthread_mutex_t submitted_mutex; /* mut used for no of submitted tasks to a sched_ctx */
+	int nsubmitted;	 /* counter used for no of submitted tasks to a sched_ctx */
+	const char *sched_name;
+};
+
+int _starpu_create_sched_ctx(const char *policy_name, int *workerid, int nworkerids, unsigned is_init_sched, const char *sched_name);
 
 void _starpu_delete_all_sched_ctxs();
 

src/core/sched_policy.c

@@ -206,14 +206,14 @@ void _starpu_init_sched_policy(struct starpu_machine_config_s *config, struct st
 
 	load_sched_policy(selected_policy, sched_ctx);
 
-	sched_ctx->sched_policy->init_sched(sched_ctx);
+	sched_ctx->sched_policy->init_sched(sched_ctx->sched_ctx_id);
 }
 
 void _starpu_deinit_sched_policy(struct starpu_machine_config_s *config, struct starpu_sched_ctx *sched_ctx)
 {
         struct starpu_sched_policy_s *policy = sched_ctx->sched_policy;
 	if (policy->deinit_sched)
-		policy->deinit_sched(sched_ctx);
+		policy->deinit_sched(sched_ctx->sched_ctx_id);
 }
 
 /* Enqueue a task into the list of tasks explicitely attached to a worker. In
@@ -309,9 +309,9 @@ int _starpu_push_task(starpu_job_t j, unsigned job_is_already_locked)
 		ret = _starpu_push_task_on_specific_worker(task, task->workerid);
 	}
 	else {
-	        struct starpu_sched_ctx *sched_ctx = task->sched_ctx;
+		struct starpu_sched_ctx *sched_ctx = _starpu_get_sched_ctx(task->sched_ctx);
 		STARPU_ASSERT(sched_ctx->sched_policy->push_task);
-		ret = sched_ctx->sched_policy->push_task(task, sched_ctx);
+		ret = sched_ctx->sched_policy->push_task(task, sched_ctx->sched_ctx_id);
 	}
 
 	_starpu_profiling_set_task_push_end_time(task);
@@ -380,8 +380,9 @@ struct starpu_task *_starpu_pop_every_task(struct starpu_sched_ctx *sched_ctx)
 
 void _starpu_sched_post_exec_hook(struct starpu_task *task)
 {
-	if (task->sched_ctx->sched_policy->post_exec_hook)
-		task->sched_ctx->sched_policy->post_exec_hook(task);
+	struct starpu_sched_ctx *sched_ctx = _starpu_get_sched_ctx(task->sched_ctx);
+	if (sched_ctx->sched_policy->post_exec_hook)
+		sched_ctx->sched_policy->post_exec_hook(task);
 }
 
 void _starpu_wait_on_sched_event(void)

src/core/sched_policy.h

@@ -19,7 +19,7 @@
 
 #include <starpu.h>
 #include <core/workers.h>
-
+#include <core/sched_ctx.h>
 #include <starpu_scheduler.h>
 
 struct starpu_machine_config_s;

src/core/task.c

@@ -19,6 +19,7 @@
 #include <starpu_profiling.h>
 #include <starpu_task_bundle.h>
 #include <core/workers.h>
+#include <core/sched_ctx.h>
 #include <core/jobs.h>
 #include <core/task.h>
 #include <common/config.h>
@@ -77,7 +78,7 @@ void starpu_task_init(struct starpu_task *task)
 
 	task->starpu_private = NULL;
 
-	task->sched_ctx = NULL;
+	task->sched_ctx = -1;
 }
 
 /* Free all the ressources allocated for a task, without deallocating the task
@@ -200,7 +201,7 @@ int _starpu_submit_job(starpu_job_t j, unsigned do_not_increment_nsubmitted)
 
 	if (!do_not_increment_nsubmitted){
 		_starpu_increment_nsubmitted_tasks();
-		_starpu_increment_nsubmitted_tasks_of_sched_ctx(j->task->sched_ctx);
+		_starpu_increment_nsubmitted_tasks_of_sched_ctx(_starpu_get_sched_ctx(j->task->sched_ctx));
 	}
 
 	PTHREAD_MUTEX_LOCK(&j->sync_mutex);
@@ -216,9 +217,9 @@ int _starpu_submit_job(starpu_job_t j, unsigned do_not_increment_nsubmitted)
 }
 
 /* application should submit new tasks to StarPU through this function */
-int starpu_task_submit_to_ctx(struct starpu_task *task, struct starpu_sched_ctx *sched_ctx)
+int starpu_task_submit_to_ctx(struct starpu_task *task, int sched_ctx)
 {
-        if(task->sched_ctx == NULL && sched_ctx != NULL)
+        if(task->sched_ctx == -1 && sched_ctx != -1)
 	  task->sched_ctx = sched_ctx;
 
 	int ret;
@@ -287,7 +288,8 @@ int starpu_task_submit_to_ctx(struct starpu_task *task, struct starpu_sched_ctx
 }
 
 int starpu_task_submit(struct starpu_task *task){
-   return  starpu_task_submit_to_ctx(task, _starpu_get_initial_sched_ctx());
+  struct starpu_sched_ctx *sched_ctx = _starpu_get_initial_sched_ctx();
+   return  starpu_task_submit_to_ctx(task, sched_ctx->sched_ctx_id);
 }
 
 void starpu_display_codelet_stats(struct starpu_codelet_t *cl)

src/core/topology.c

@@ -280,6 +280,7 @@ static int _starpu_init_machine_config(struct starpu_machine_config_s *config,
 
 	topology->nworkers = 0;
 	topology->ncombinedworkers = 0;
+	topology->nsched_ctxs = 0;
 	_starpu_init_topology(config);
 
 	_starpu_initialize_workers_bindid(config);

src/core/workers.c

@@ -40,8 +40,6 @@ static pthread_key_t worker_key;
 
 static struct starpu_machine_config_s config;
 
-static struct starpu_sched_ctx sched_ctx;
-
 struct starpu_machine_config_s *_starpu_get_machine_config(void)
 {
 	return &config;
@@ -356,9 +354,9 @@ int starpu_init(struct starpu_conf *user_conf)
 	/* initialize the scheduling policy */
 
 	if(user_conf == NULL)
-	  _starpu_create_sched_ctx(&sched_ctx, NULL, NULL, -1, 1, "init");
+	  _starpu_create_sched_ctx(NULL, NULL, -1, 1, "init");
 	else
-	  _starpu_create_sched_ctx(&sched_ctx, user_conf->sched_policy_name, NULL, -1, 1, "init");
+	  _starpu_create_sched_ctx(user_conf->sched_policy_name, NULL, -1, 1, "init");
 
 	//_starpu_init_sched_policy(&config, &sched_ctx);
 
@@ -621,6 +619,11 @@ struct starpu_worker_s *_starpu_get_worker_struct(unsigned id)
 	return &config.workers[id];
 }
 
+struct starpu_sched_ctx *_starpu_get_sched_ctx(int id)
+{
+	return &config.sched_ctxs[id];
+}
+
 struct starpu_combined_worker_s *_starpu_get_combined_worker_struct(unsigned id)
 {
 	unsigned basic_worker_count = starpu_worker_get_count();
@@ -661,5 +664,5 @@ void starpu_worker_set_sched_condition(int workerid, pthread_cond_t *sched_cond,
 }
 
 struct starpu_sched_ctx* _starpu_get_initial_sched_ctx(void){
-	return &sched_ctx;
+	return &config.sched_ctxs[0];
 }

src/core/workers.h

@@ -29,7 +29,7 @@
 #include <core/sched_policy.h>
 #include <core/topology.h>
 #include <core/errorcheck.h>
-
+#include <core/sched_ctx.h>
 
 #ifdef STARPU_HAVE_HWLOC
 #include <hwloc.h>
@@ -78,7 +78,7 @@ struct starpu_worker_s {
 	starpu_worker_status status; /* what is the worker doing now ? (eg. CALLBACK) */
 	char name[32];
 
-	struct starpu_sched_ctx *sched_ctx[STARPU_NMAXSCHEDCTXS];
+	struct starpu_sched_ctx *sched_ctx[STARPU_NMAX_SCHED_CTXS];
 	unsigned nctxs; /* the no of contexts a worker belongs to*/
 	unsigned changing_ctx;
 	pthread_mutex_t changing_ctx_mutex;
@@ -164,6 +164,9 @@ struct starpu_machine_config_s {
 
 	/* this flag is set until the runtime is stopped */
 	unsigned running;
+	
+	/* all the sched ctx of the current instance of starpu */
+	struct starpu_sched_ctx sched_ctxs[STARPU_NMAX_SCHED_CTXS];
 };
 
 /* Has starpu_shutdown already been called ? */
@@ -203,6 +206,11 @@ struct starpu_worker_s *_starpu_get_local_worker_key(void);
  * specified worker. */
 struct starpu_worker_s *_starpu_get_worker_struct(unsigned id);
 
+/* Returns the starpu_sched_ctx structure that descriebes the state of the 
+ * specified ctx */
+struct starpu_sched_ctx *_starpu_get_sched_ctx(int id);
+
+
 struct starpu_combined_worker_s *_starpu_get_combined_worker_struct(unsigned id);
 
 /* Returns the structure that describes the overall machine configuration (eg.

src/drivers/cpu/driver_cpu.c

@@ -227,7 +227,7 @@ void *_starpu_cpu_worker(void *arg)
 
 		_starpu_set_current_task(j->task);
 
-		struct starpu_sched_ctx *local_sched_ctx = j->task->sched_ctx;
+		struct starpu_sched_ctx *local_sched_ctx = _starpu_get_sched_ctx(j->task->sched_ctx);
 
                 res = execute_job_on_cpu(j, cpu_arg, is_parallel_task, rank, perf_arch);
 

src/drivers/cuda/driver_cuda.c

@@ -326,7 +326,7 @@ void *_starpu_cuda_worker(void *arg)
 
 		_starpu_set_current_task(task);
 
-		struct starpu_sched_ctx *local_sched_ctx = j->task->sched_ctx;
+		struct starpu_sched_ctx *local_sched_ctx = _starpu_get_sched_ctx(j->task->sched_ctx);
 
 		res = execute_job_on_cuda(j, args);
 

src/sched_policies/heft.c

@@ -20,6 +20,7 @@
 #include <float.h>
 
 #include <core/workers.h>
+#include <core/sched_ctx.h>
 #include <core/perfmodel/perfmodel.h>
 #include <starpu_parameters.h>
 #include <starpu_task_bundle.h>
@@ -39,8 +40,9 @@ static double exp_end[STARPU_NMAXWORKERS];
 static double exp_len[STARPU_NMAXWORKERS];
 static double ntasks[STARPU_NMAXWORKERS];
 
-static void heft_init(struct starpu_sched_ctx *sched_ctx)
+static void heft_init(int sched_ctx_id)
 {
+	struct starpu_sched_ctx *sched_ctx = _starpu_get_sched_ctx(sched_ctx_id);
 	unsigned nworkers = sched_ctx->nworkers_in_ctx;
 
 	const char *strval_alpha = getenv("STARPU_SCHED_ALPHA");
@@ -144,7 +146,7 @@ static void compute_all_performance_predictions(struct starpu_task *task,
 						double *local_data_penalty,
 						double *local_power, int *forced_best,
 						struct starpu_task_bundle *bundle,
-						struct starpu_sched_ctx *sched_ctx)
+						 struct starpu_sched_ctx *sched_ctx )
 {
   int calibrating = 0;
   double max_exp_end = DBL_MIN;
@@ -230,8 +232,9 @@ static void compute_all_performance_predictions(struct starpu_task *task,
   *max_exp_endp = max_exp_end;
 }
 
-static int _heft_push_task(struct starpu_task *task, unsigned prio, struct starpu_sched_ctx *sched_ctx)
+static int _heft_push_task(struct starpu_task *task, unsigned prio, int sched_ctx_id)
 {
+	struct starpu_sched_ctx *sched_ctx = _starpu_get_sched_ctx(sched_ctx_id);
 	unsigned worker, worker_in_ctx;
 	int best = -1, best_id_in_ctx = -1;
 	
@@ -341,21 +344,22 @@ static int _heft_push_task(struct starpu_task *task, unsigned prio, struct starp
 	return push_task_on_best_worker(task, best, model_best, prio);
 }
 
-static int heft_push_prio_task(struct starpu_task *task, struct starpu_sched_ctx *sched_ctx)
+static int heft_push_prio_task(struct starpu_task *task, int sched_ctx_id)
 {
-        return _heft_push_task(task, 1, sched_ctx);
+        return _heft_push_task(task, 1, sched_ctx_id);
 }
 
-static int heft_push_task(struct starpu_task *task, struct starpu_sched_ctx *sched_ctx)
+static int heft_push_task(struct starpu_task *task, int sched_ctx_id)
 {
 	if (task->priority > 0)
-        	  return _heft_push_task(task, 1, sched_ctx);
+        	  return _heft_push_task(task, 1, sched_ctx_id);
 
-	return _heft_push_task(task, 0, sched_ctx);
+	return _heft_push_task(task, 0, sched_ctx_id);
 }
 
-static void heft_deinit(struct starpu_sched_ctx *sched_ctx) 
+static void heft_deinit(int sched_ctx_id) 
 {
+	struct starpu_sched_ctx *sched_ctx = _starpu_get_sched_ctx(sched_ctx_id);
         unsigned workerid;
 	int workerid_in_ctx;
 	unsigned nworkers = sched_ctx->nworkers_in_ctx;

src/util/starpu_insert_task.c

@@ -78,24 +78,4 @@ void starpu_insert_task(starpu_codelet *cl, ...)
 	va_start(varg_list, cl);
         struct starpu_task *task = starpu_task_create();
         _starpu_insert_task_create_and_submit(arg_buffer, cl, &task, varg_list);
-
-}
-
-void starpu_insert_task_to_ctx(struct starpu_sched_ctx *sched_ctx, starpu_codelet *cl, ...)
-{
-	va_list varg_list;
-
-	/* Compute the size */
-	size_t arg_buffer_size = 0;
-	va_start(varg_list, cl);
-        arg_buffer_size = _starpu_insert_task_get_arg_size(varg_list);
-
-	va_start(varg_list, cl);
-	char *arg_buffer;
-	_starpu_pack_cl_args(arg_buffer_size, &arg_buffer, varg_list);
-
-	va_start(varg_list, cl);
-        struct starpu_task *task = starpu_task_create();
-        _starpu_insert_task_create_and_submit_to_ctx(arg_buffer, cl, &task, varg_list, sched_ctx);
-
 }

src/util/starpu_insert_task_utils.c

@@ -131,7 +131,7 @@ int _starpu_pack_cl_args(size_t arg_buffer_size, char **arg_buffer, va_list varg
 	va_end(varg_list);
 	return 0;
 }
-static void _starpu_prepare_task(char *arg_buffer, starpu_codelet *cl, struct starpu_task **task, va_list varg_list) {
+static void _starpu_prepare_task(char *arg_buffer, starpu_codelet *cl, struct starpu_task **task, va_list varg_list, int *ctx) {
         int arg_type;
 	unsigned current_buffer = 0;
 
@@ -179,6 +179,10 @@ static void _starpu_prepare_task(char *arg_buffer, starpu_codelet *cl, struct st
 		else if (arg_type==STARPU_EXECUTE) {
 			va_arg(varg_list, int);
 		}
+		else if (arg_type==STARPU_CTX) {
+			*ctx = va_arg(varg_list, int);
+		}
+
 	}
 
 	va_end(varg_list);
@@ -192,24 +196,13 @@ static void _starpu_prepare_task(char *arg_buffer, starpu_codelet *cl, struct st
 	 * application's callback, if any. */
 	(*task)->callback_func = starpu_task_insert_callback_wrapper;
 	(*task)->callback_arg = cl_arg_wrapper;
-
 }
-int _starpu_insert_task_create_and_submit(char *arg_buffer, starpu_codelet *cl, struct starpu_task **task, va_list varg_list) {
-	 _starpu_prepare_task(arg_buffer, cl, task, varg_list);
-	
-	 int ret = starpu_task_submit(*task);
 
-	if (STARPU_UNLIKELY(ret == -ENODEV))
-          fprintf(stderr, "No one can execute task %p wih cl %p (symbol %s)\n", *task, (*task)->cl, ((*task)->cl->model && (*task)->cl->model->symbol)?(*task)->cl->model->symbol:"none");
-
-	STARPU_ASSERT(!ret);
-        return ret;
-}
-
-int _starpu_insert_task_create_and_submit_to_ctx(char *arg_buffer, starpu_codelet *cl, struct starpu_task **task, va_list varg_list, struct starpu_sched_ctx *sched_ctx) {
-	 _starpu_prepare_task(arg_buffer, cl, task, varg_list);
+int _starpu_insert_task_create_and_submit(char *arg_buffer, starpu_codelet *cl, struct starpu_task **task, va_list varg_list) {
+	int ctx = -1;
+	_starpu_prepare_task(arg_buffer, cl, task, varg_list, &ctx);
 	
-	 int ret = starpu_task_submit_to_ctx(*task, sched_ctx);
+	 int ret = ctx == -1 ? starpu_task_submit(*task) : starpu_task_submit_to_ctx(*task, ctx);
 
 	if (STARPU_UNLIKELY(ret == -ENODEV))
           fprintf(stderr, "No one can execute task %p wih cl %p (symbol %s)\n", *task, (*task)->cl, ((*task)->cl->model && (*task)->cl->model->symbol)?(*task)->cl->model->symbol:"none");

src/util/starpu_insert_task_utils.h

@@ -24,7 +24,6 @@
 size_t _starpu_insert_task_get_arg_size(va_list varg_list);
 int _starpu_pack_cl_args(size_t arg_buffer_size, char **arg_buffer, va_list varg_list);
 int _starpu_insert_task_create_and_submit(char *arg_buffer, starpu_codelet *cl, struct starpu_task **task, va_list varg_list);
-int _starpu_insert_task_create_and_submit_to_ctx(char *arg_buffer, starpu_codelet *cl, struct starpu_task **task, va_list varg_list, struct starpu_sched_ctx *sched_ctx);
 
 #endif // __STARPU_INSERT_TASK_UTILS_H__