simgrid: taking into account scheduling cost

doc/doxygen/chapters/470_simgrid.doxy

@@ -141,8 +141,8 @@ be extended as well), change the available GPU memory size, PCI memory bandwidth
 The simulation can be tweaked, to be able to tune it between a very accurate
 simulation and a very simple simulation (which is thus close to scheduling
 theory results), see the \ref STARPU_SIMGRID_CUDA_MALLOC_COST,
-\ref STARPU_SIMGRID_CUDA_QUEUE_COST, \ref STARPU_SIMGRID_TASK_SUBMIT_COST
-and \ref STARPU_SIMGRID_FETCHING_INPUT_COST environment variables.
+\ref STARPU_SIMGRID_CUDA_QUEUE_COST, \ref STARPU_SIMGRID_TASK_SUBMIT_COST,
+\ref STARPU_SIMGRID_FETCHING_INPUT_COST and STARPU_SIMGRID_SCHED_COST environment variables.
 
 \section SimulationMPIApplications MPI Applications
 

doc/doxygen/chapters/501_environment_variables.doxy

@@ -638,6 +638,15 @@ account in simgrid mode. This provides more accurate simgrid predictions,
 especially regarding data transfers.
 </dd>
 
+<dt>STARPU_SIMGRID_SCHED_COST</dt>
+<dd>
+\anchor STARPU_SIMGRID_SCHED_COST
+\addindex __env__STARPU_SIMGRID_SCHED_COST
+When set to 1 (which is the default), scheduling costs are taken into
+account in simgrid mode. This provides more accurate simgrid predictions,
+and allows studying scheduling overhead of the runtime system.
+</dd>
+
 </dl>
 
 \section MiscellaneousAndDebug Miscellaneous And Debug

src/core/sched_ctx.c

@@ -74,9 +74,9 @@ void _starpu_worker_gets_out_of_ctx(unsigned sched_ctx_id, struct _starpu_worker
 		/* struct _starpu_sched_ctx *sched_ctx = _starpu_get_sched_ctx_struct(sched_ctx_id); */
 		/* if(sched_ctx && sched_ctx->sched_policy && sched_ctx->sched_policy->remove_workers) */
 		/* { */
-		/* 	_STARPU_TRACE_WORKER_SCHEDULING_PUSH; */
+		/* 	_STARPU_SCHED_BEGIN; */
 		/* 	sched_ctx->sched_policy->remove_workers(sched_ctx_id, &worker->workerid, 1); */
-		/* 	_STARPU_TRACE_WORKER_SCHEDULING_POP; */
+		/* 	_STARPU_SCHED_END; */
 		/* } */
 		if (!_starpu_sched_ctx_list_remove(&worker->sched_ctx_list, sched_ctx_id))
 			worker->nsched_ctxs--;
@@ -310,7 +310,7 @@ static void _starpu_add_workers_to_sched_ctx(struct _starpu_sched_ctx *sched_ctx
 
 	if(sched_ctx->sched_policy && sched_ctx->sched_policy->add_workers)
 	{
-		_STARPU_TRACE_WORKER_SCHEDULING_PUSH;
+		_STARPU_SCHED_BEGIN;
 		if(added_workers)
 		{
 			if(*n_added_workers > 0)
@@ -320,7 +320,7 @@ static void _starpu_add_workers_to_sched_ctx(struct _starpu_sched_ctx *sched_ctx
 		{
 			sched_ctx->sched_policy->add_workers(sched_ctx->id, workers_to_add, nworkers_to_add);
 		}
-		_STARPU_TRACE_WORKER_SCHEDULING_POP;
+		_STARPU_SCHED_END;
 	}
 	return;
 }
@@ -407,9 +407,9 @@ static void _starpu_sched_ctx_free_scheduling_data(struct _starpu_sched_ctx *sch
 
 		if(nworkers_ctx > 0)
 		{
-			_STARPU_TRACE_WORKER_SCHEDULING_PUSH;
+			_STARPU_SCHED_BEGIN;
 			sched_ctx->sched_policy->remove_workers(sched_ctx->id, workerids, nworkers_ctx);
-			_STARPU_TRACE_WORKER_SCHEDULING_POP;
+			_STARPU_SCHED_END;
 		}
 
 		free(workerids);

src/core/sched_policy.c

@@ -222,9 +222,9 @@ void _starpu_init_sched_policy(struct _starpu_machine_config *config, struct _st
 	starpu_sched_ctx_create_worker_collection(sched_ctx->id,
 						  sched_ctx->sched_policy->worker_type);
 
-	_STARPU_TRACE_WORKER_SCHEDULING_PUSH;
+	_STARPU_SCHED_BEGIN;
 	sched_ctx->sched_policy->init_sched(sched_ctx->id);
-	_STARPU_TRACE_WORKER_SCHEDULING_POP;
+	_STARPU_SCHED_END;
 }
 
 void _starpu_deinit_sched_policy(struct _starpu_sched_ctx *sched_ctx)
@@ -232,9 +232,9 @@ void _starpu_deinit_sched_policy(struct _starpu_sched_ctx *sched_ctx)
 	struct starpu_sched_policy *policy = sched_ctx->sched_policy;
 	if (policy->deinit_sched)
 	{
-		_STARPU_TRACE_WORKER_SCHEDULING_PUSH;
+		_STARPU_SCHED_BEGIN;
 		policy->deinit_sched(sched_ctx->id);
-		_STARPU_TRACE_WORKER_SCHEDULING_POP;
+		_STARPU_SCHED_END;
 	}
 	starpu_sched_ctx_delete_worker_collection(sched_ctx->id);
 }
@@ -246,9 +246,9 @@ void _starpu_sched_task_submit(struct starpu_task *task)
 		return;
 	if (!sched_ctx->sched_policy->submit_hook)
 		return;
-	_STARPU_TRACE_WORKER_SCHEDULING_PUSH;
+	_STARPU_SCHED_BEGIN;
 	sched_ctx->sched_policy->submit_hook(task);
-	_STARPU_TRACE_WORKER_SCHEDULING_POP;
+	_STARPU_SCHED_END;
 }
 
 void _starpu_sched_do_schedule(unsigned sched_ctx_id)
@@ -258,9 +258,9 @@ void _starpu_sched_do_schedule(unsigned sched_ctx_id)
 		return;
 	if (!sched_ctx->sched_policy->do_schedule)
 		return;
-	_STARPU_TRACE_WORKER_SCHEDULING_PUSH;
+	_STARPU_SCHED_BEGIN;
 	sched_ctx->sched_policy->do_schedule(sched_ctx_id);
-	_STARPU_TRACE_WORKER_SCHEDULING_POP;
+	_STARPU_SCHED_END;
 }
 
 static void _starpu_push_task_on_specific_worker_notify_sched(struct starpu_task *task, struct _starpu_worker *worker, int workerid, int perf_workerid)
@@ -275,9 +275,9 @@ static void _starpu_push_task_on_specific_worker_notify_sched(struct starpu_task
 		struct _starpu_sched_ctx *sched_ctx = _starpu_get_sched_ctx_struct(e->sched_ctx);
 		if (sched_ctx->sched_policy != NULL && sched_ctx->sched_policy->push_task_notify)
 		{
-			_STARPU_TRACE_WORKER_SCHEDULING_PUSH;
+			_STARPU_SCHED_BEGIN;
 			sched_ctx->sched_policy->push_task_notify(task, workerid, perf_workerid, sched_ctx->id);
-			_STARPU_TRACE_WORKER_SCHEDULING_POP;
+			_STARPU_SCHED_END;
 		}
 	}
 }
@@ -597,9 +597,9 @@ int _starpu_push_task_to_workers(struct starpu_task *task)
 			else
 			{
 				_STARPU_TASK_BREAK_ON(task, push);
-				_STARPU_TRACE_WORKER_SCHEDULING_PUSH;
+				_STARPU_SCHED_BEGIN;
 				ret = sched_ctx->sched_policy->push_task(task);
-				_STARPU_TRACE_WORKER_SCHEDULING_POP;
+				_STARPU_SCHED_END;
 			}
 			STARPU_PTHREAD_RWLOCK_UNLOCK(changing_ctx_mutex);
 		}
@@ -1016,9 +1016,9 @@ struct starpu_task *_starpu_pop_every_task(struct _starpu_sched_ctx *sched_ctx)
 		/* TODO set profiling info */
 		if(sched_ctx->sched_policy->pop_every_task)
 		{
-			_STARPU_TRACE_WORKER_SCHEDULING_PUSH;
+			_STARPU_SCHED_BEGIN;
 			task = sched_ctx->sched_policy->pop_every_task(sched_ctx->id);
-			_STARPU_TRACE_WORKER_SCHEDULING_POP;
+			_STARPU_SCHED_END;
 		}
 	}
 	return task;
@@ -1030,9 +1030,9 @@ void _starpu_sched_pre_exec_hook(struct starpu_task *task)
 	struct _starpu_sched_ctx *sched_ctx = _starpu_get_sched_ctx_struct(sched_ctx_id);
 	if (sched_ctx->sched_policy && sched_ctx->sched_policy->pre_exec_hook)
 	{
-		_STARPU_TRACE_WORKER_SCHEDULING_PUSH;
+		_STARPU_SCHED_BEGIN;
 		sched_ctx->sched_policy->pre_exec_hook(task, sched_ctx_id);
-		_STARPU_TRACE_WORKER_SCHEDULING_POP;
+		_STARPU_SCHED_END;
 	}
 
 	if(!sched_ctx->sched_policy)
@@ -1052,9 +1052,9 @@ void _starpu_sched_pre_exec_hook(struct starpu_task *task)
 			    other_sched_ctx->sched_policy != NULL && 
 			    other_sched_ctx->sched_policy->pre_exec_hook)
 			{
-				_STARPU_TRACE_WORKER_SCHEDULING_PUSH;
+				_STARPU_SCHED_BEGIN;
 				other_sched_ctx->sched_policy->pre_exec_hook(task, other_sched_ctx->id);
-				_STARPU_TRACE_WORKER_SCHEDULING_POP;
+				_STARPU_SCHED_END;
 			}
 		}
 	}
@@ -1067,9 +1067,9 @@ void _starpu_sched_post_exec_hook(struct starpu_task *task)
 	struct _starpu_sched_ctx *sched_ctx = _starpu_get_sched_ctx_struct(sched_ctx_id);
 	if (sched_ctx->sched_policy && sched_ctx->sched_policy->post_exec_hook)
 	{
-		_STARPU_TRACE_WORKER_SCHEDULING_PUSH;
+		_STARPU_SCHED_BEGIN;
 		sched_ctx->sched_policy->post_exec_hook(task, sched_ctx_id);
-		_STARPU_TRACE_WORKER_SCHEDULING_POP;
+		_STARPU_SCHED_END;
 	}
 	if(!sched_ctx->sched_policy)
 	{
@@ -1088,9 +1088,9 @@ void _starpu_sched_post_exec_hook(struct starpu_task *task)
 			    other_sched_ctx->sched_policy != NULL && 
 			    other_sched_ctx->sched_policy->post_exec_hook)
 			{
-				_STARPU_TRACE_WORKER_SCHEDULING_PUSH;
+				_STARPU_SCHED_BEGIN;
 				other_sched_ctx->sched_policy->post_exec_hook(task, other_sched_ctx->id);
-				_STARPU_TRACE_WORKER_SCHEDULING_POP;
+				_STARPU_SCHED_END;
 			}
 		}
 	}

src/core/sched_policy.h

@@ -24,6 +24,15 @@
 #include <core/sched_ctx.h>
 #include <starpu_scheduler.h>
 
+#include <core/simgrid.h>
+
+#define _STARPU_SCHED_BEGIN \
+	_STARPU_TRACE_WORKER_SCHEDULING_PUSH;	\
+	_SIMGRID_TIMER_BEGIN
+#define _STARPU_SCHED_END \
+	_STARPU_TRACE_WORKER_SCHEDULING_POP;	\
+	_SIMGRID_TIMER_END
+
 void _starpu_sched_init(void);
 
 struct starpu_machine_config;

src/core/simgrid.h

@@ -26,6 +26,7 @@
 #endif
 
 #include <datawizard/data_request.h>
+#include <xbt/xbt_os_time.h>
 
 struct _starpu_pthread_args
 {
@@ -67,6 +68,7 @@ starpu_pthread_queue_t _starpu_simgrid_task_queue[STARPU_NMAXWORKERS];
 #define _starpu_simgrid_queue_malloc_cost() starpu_get_env_number_default("STARPU_SIMGRID_QUEUE_MALLOC_COST", 1)
 #define _starpu_simgrid_task_submit_cost() starpu_get_env_number_default("STARPU_SIMGRID_TASK_SUBMIT_COST", 1)
 #define _starpu_simgrid_fetching_input_cost() starpu_get_env_number_default("STARPU_SIMGRID_FETCHING_INPUT_COST", 1)
+#define _starpu_simgrid_sched_cost() starpu_get_env_number_default("STARPU_SIMGRID_SCHED_COST", 1)
 
 /* Called at initialization to count how many GPUs are interfering with each
  * bus */
@@ -74,6 +76,25 @@ void _starpu_simgrid_count_ngpus(void);
 
 void _starpu_simgrid_xbt_thread_create(const char *name, void_f_pvoid_t code,
 				       void *param);
+
+#define _SIMGRID_TIMER_BEGIN		\
+	{		\
+		xbt_os_timer_t __timer = NULL;		\
+		if (_starpu_simgrid_sched_cost()) {		\
+		  __timer = xbt_os_timer_new();		\
+		  xbt_os_threadtimer_start(__timer);	\
+		}
+#define _SIMGRID_TIMER_END		\
+		if (__timer) {		\
+			xbt_os_threadtimer_stop(__timer);		\
+			MSG_process_sleep(xbt_os_timer_elapsed(__timer));\
+			xbt_os_timer_free(__timer);		\
+		}	\
+	}
+
+#else // !STARPU_SIMGRID
+#define _SIMGRID_TIMER_BEGIN {
+#define _SIMGRID_TIMER_END }
 #endif
 
 #endif // __SIMGRID_H__