Advertise workers more comprehensively. Also fix links with poti-generated traces.

src/common/fxt.h

@@ -1,6 +1,6 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2009-2012  Université de Bordeaux 1
+ * Copyright (C) 2009-2013  Université de Bordeaux 1
  * Copyright (C) 2010, 2011, 2012  Centre National de la Recherche Scientifique
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -194,8 +194,8 @@ do {									\
 #define _STARPU_TRACE_NEW_MEM_NODE(nodeid)			\
 	FUT_DO_PROBE2(_STARPU_FUT_NEW_MEM_NODE, nodeid, _starpu_gettid());
 
-#define _STARPU_TRACE_WORKER_INIT_START(workerkind, devid, memnode)	\
-	FUT_DO_PROBE4(_STARPU_FUT_WORKER_INIT_START, workerkind, devid, memnode, _starpu_gettid());
+#define _STARPU_TRACE_WORKER_INIT_START(workerkind, workerid, devid, memnode)	\
+	FUT_DO_PROBE5(_STARPU_FUT_WORKER_INIT_START, workerkind, workerid, devid, memnode, _starpu_gettid());
 
 #define _STARPU_TRACE_WORKER_INIT_END				\
 	FUT_DO_PROBE1(_STARPU_FUT_WORKER_INIT_END, _starpu_gettid());

src/core/workers.c

@@ -337,7 +337,7 @@ void _starpu_worker_init(struct _starpu_worker *worker, unsigned fut_key)
 	_starpu_fxt_register_thread(worker->bindid);
 
 	unsigned memnode = worker->memory_node;
-	_STARPU_TRACE_WORKER_INIT_START(fut_key, devid, memnode);
+	_STARPU_TRACE_WORKER_INIT_START(fut_key, worker->workerid, devid, memnode);
 #endif
 
 	_starpu_bind_thread_on_cpu(worker->config, worker->bindid);
@@ -351,6 +351,11 @@ void _starpu_worker_init(struct _starpu_worker *worker, unsigned fut_key)
 
 	_starpu_set_local_worker_key(worker);
 
+	_STARPU_PTHREAD_MUTEX_LOCK(&worker->mutex);
+	worker->worker_is_running = 1;
+	_STARPU_PTHREAD_COND_SIGNAL(&worker->started_cond);
+	_STARPU_PTHREAD_MUTEX_UNLOCK(&worker->mutex);
+
 }
 
 static void _starpu_launch_drivers(struct _starpu_machine_config *config)
@@ -387,6 +392,7 @@ static void _starpu_launch_drivers(struct _starpu_machine_config *config)
 		_starpu_barrier_counter_init(&workerarg->tasks_barrier, 0);
 
 		_STARPU_PTHREAD_MUTEX_INIT(&workerarg->mutex, NULL);
+		_STARPU_PTHREAD_COND_INIT(&workerarg->started_cond, NULL);
 		_STARPU_PTHREAD_COND_INIT(&workerarg->ready_cond, NULL);
 
 		workerarg->worker_size = 1;
@@ -397,6 +403,8 @@ static void _starpu_launch_drivers(struct _starpu_machine_config *config)
 		/* we have a single local list */
 		/* afterwards there would be a mutex + cond for the list of each strategy */
 		workerarg->run_by_starpu = 1;
+		workerarg->worker_is_running = 0;
+		workerarg->worker_is_initialized = 0;
 
 		_STARPU_PTHREAD_MUTEX_INIT(&workerarg->sched_mutex, NULL);
 		_STARPU_PTHREAD_COND_INIT(&workerarg->sched_cond, NULL);
@@ -423,7 +431,6 @@ static void _starpu_launch_drivers(struct _starpu_machine_config *config)
 #ifdef STARPU_USE_CPU
 			case STARPU_CPU_WORKER:
 				workerarg->set = NULL;
-				workerarg->worker_is_initialized = 0;
 				driver.id.cpu_id = cpu;
 				if (_starpu_may_launch_driver(config->conf, &driver))
 				{
@@ -434,6 +441,10 @@ static void _starpu_launch_drivers(struct _starpu_machine_config *config)
 						_starpu_cpu_worker,
 						workerarg,
 						worker+1);
+					_STARPU_PTHREAD_MUTEX_LOCK(&workerarg->mutex);
+					while (!workerarg->worker_is_running)
+						_STARPU_PTHREAD_COND_WAIT(&workerarg->started_cond, &workerarg->mutex);
+					_STARPU_PTHREAD_MUTEX_UNLOCK(&workerarg->mutex);
 				}
 				else
 				{
@@ -445,7 +456,6 @@ static void _starpu_launch_drivers(struct _starpu_machine_config *config)
 #ifdef STARPU_USE_CUDA
 			case STARPU_CUDA_WORKER:
 				workerarg->set = NULL;
-				workerarg->worker_is_initialized = 0;
 				driver.id.cuda_id = cuda;
 				if (_starpu_may_launch_driver(config->conf, &driver))
 				{
@@ -456,6 +466,10 @@ static void _starpu_launch_drivers(struct _starpu_machine_config *config)
 						_starpu_cuda_worker,
 						workerarg,
 						worker+1);
+					_STARPU_PTHREAD_MUTEX_LOCK(&workerarg->mutex);
+					while (!workerarg->worker_is_running)
+						_STARPU_PTHREAD_COND_WAIT(&workerarg->started_cond, &workerarg->mutex);
+					_STARPU_PTHREAD_MUTEX_UNLOCK(&workerarg->mutex);
 				}
 				else
 				{
@@ -473,7 +487,6 @@ static void _starpu_launch_drivers(struct _starpu_machine_config *config)
 					break;
 				}
 				workerarg->set = NULL;
-				workerarg->worker_is_initialized = 0;
 				_STARPU_PTHREAD_CREATE_ON(
 					workerarg->name,
 					&workerarg->worker_thread,
@@ -481,6 +494,10 @@ static void _starpu_launch_drivers(struct _starpu_machine_config *config)
 					_starpu_opencl_worker,
 					workerarg,
 					worker+1);
+				_STARPU_PTHREAD_MUTEX_LOCK(&workerarg->mutex);
+				while (!workerarg->worker_is_running)
+					_STARPU_PTHREAD_COND_WAIT(&workerarg->started_cond, &workerarg->mutex);
+				_STARPU_PTHREAD_MUTEX_UNLOCK(&workerarg->mutex);
 				break;
 #endif
 #ifdef STARPU_USE_GORDON

src/core/workers.h

@@ -1,6 +1,6 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2009-2012  Université de Bordeaux 1
+ * Copyright (C) 2009-2013  Université de Bordeaux 1
  * Copyright (C) 2010, 2011, 2012  Centre National de la Recherche Scientifique
  * Copyright (C) 2011  INRIA
  *
@@ -68,6 +68,7 @@ struct _starpu_worker
 	int combined_workerid; /* combined worker currently using this worker */
 	int current_rank; /* current rank in case the worker is used in a parallel fashion */
 	int worker_size; /* size of the worker in case we use a combined worker */
+	_starpu_pthread_cond_t started_cond; /* indicate when the worker is ready */
 	_starpu_pthread_cond_t ready_cond; /* indicate when the worker is ready */
 	unsigned memory_node; /* which memory node is the worker associated with ? */
 	_starpu_pthread_cond_t sched_cond; /* condition variable used when the worker waits for tasks. */

src/debug/traces/starpu_fxt.c

@@ -1,6 +1,6 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2009-2012  Université de Bordeaux 1
+ * Copyright (C) 2009-2013  Université de Bordeaux 1
  *
  * StarPU is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
@@ -137,9 +137,9 @@ struct worker_entry
 	int workerid;
 } *worker_ids;
 
-static int register_worker_id(unsigned long tid)
+static void register_worker_id(unsigned long tid, int workerid)
 {
-	int workerid = nworkers++;
+	nworkers++;
 	struct worker_entry *entry;
 
 	HASH_FIND(hh, worker_ids, &tid, sizeof(tid), entry);
@@ -152,8 +152,6 @@ static int register_worker_id(unsigned long tid)
 	entry->workerid = workerid;
 
 	HASH_ADD(hh, worker_ids, tid, sizeof(tid), entry);
-
-	return workerid;
 }
 
 static int find_worker_id(unsigned long tid)
@@ -197,9 +195,15 @@ static char *memnode_container_alias(char *output, int len, const char *prefix,
 	return output;
 }
 
+static char *thread_container_alias(char *output, int len, const char *prefix, long unsigned int threadid)
+{
+	snprintf(output, len, "t%s%"PRIu64"", prefix, threadid);
+	return output;
+}
+
 static char *worker_container_alias(char *output, int len, const char *prefix, long unsigned int workerid)
 {
-	snprintf(output, len, "t%s%"PRIu64"", prefix, workerid);
+	snprintf(output, len, "w%s%"PRIu64"", prefix, workerid);
 	return output;
 }
 
@@ -223,7 +227,7 @@ static void memnode_set_state(double time, const char *prefix, unsigned int memn
 	memnode_container_alias(container, STARPU_POTI_STR_LEN, prefix, memnodeid);
 	poti_SetState(time, container, "MS", name);
 #else
-	fprintf(out_paje_file, "10	%.9f	%smn%u	MS	%s\n", time, prefix, memnodeid, name);
+	fprintf(out_paje_file, "10	%.9f	mn%s%u	MS	%s\n", time, prefix, memnodeid, name);
 #endif
 }
 
@@ -231,7 +235,7 @@ static void worker_set_state(double time, const char *prefix, long unsigned int
 {
 #ifdef STARPU_HAVE_POTI
 	char container[STARPU_POTI_STR_LEN];
-	worker_container_alias(container, STARPU_POTI_STR_LEN, prefix, workerid);
+	thread_container_alias(container, STARPU_POTI_STR_LEN, prefix, workerid);
 	poti_SetState(time, container, "S", name);
 #else
 	fprintf(out_paje_file, "10	%.9f	t%s%"PRIu64"	S	%s\n", time, prefix, workerid, name);
@@ -254,17 +258,18 @@ static void handle_new_mem_node(struct fxt_ev_64 *ev, struct starpu_fxt_options
 		program_container_alias(program_container, STARPU_POTI_STR_LEN, prefix);
 		char new_memnode_container_alias[STARPU_POTI_STR_LEN], new_memnode_container_name[STARPU_POTI_STR_LEN];
 		memnode_container_alias (new_memnode_container_alias, STARPU_POTI_STR_LEN, prefix, ev->param[0]);
-		snprintf(new_memnode_container_name, STARPU_POTI_STR_LEN, "%sMEMNODE%"PRIu64"", prefix, ev->param[0]);
+		/* TODO: ramkind */
+		snprintf(new_memnode_container_name, STARPU_POTI_STR_LEN, "MEMNODE%s%"PRIu64"", prefix, ev->param[0]);
 		poti_CreateContainer(get_event_time_stamp(ev, options), new_memnode_container_alias, "Mn", program_container, new_memnode_container_name);
 #else
-		fprintf(out_paje_file, "7	%.9f	mn%"PRIu64"	Mn	%sp	%sMEMNODE%"PRIu64"\n", get_event_time_stamp(ev, options), ev->param[0], prefix, options->file_prefix, ev->param[0]);
+		fprintf(out_paje_file, "7	%.9f	mn%"PRIu64"	Mn	%sp	MEMNODE%s%"PRIu64"\n", get_event_time_stamp(ev, options), ev->param[0], prefix, options->file_prefix, ev->param[0]);
 #endif
 
 		if (!options->no_bus)
 #ifdef STARPU_HAVE_POTI
 			poti_SetVariable(get_event_time_stamp(ev, options), new_memnode_container_alias, "bw", 0.0);
 #else
-			fprintf(out_paje_file, "13	%.9f	%smn%"PRIu64"	bw	0.0\n", 0.0f, prefix, ev->param[0]);
+			fprintf(out_paje_file, "13	%.9f	mn%s%"PRIu64"	bw	0.0\n", 0.0f, prefix, ev->param[0]);
 #endif
 	}
 }
@@ -278,24 +283,12 @@ static void handle_worker_init_start(struct fxt_ev_64 *ev, struct starpu_fxt_opt
 	*/
 	char *prefix = options->file_prefix;
 
-	if (out_paje_file)
-	{
-#ifdef STARPU_HAVE_POTI
-		char new_worker_container_alias[STARPU_POTI_STR_LEN];
-		worker_container_alias (new_worker_container_alias, STARPU_POTI_STR_LEN, prefix, ev->param[3]);
-		char memnode_container[STARPU_POTI_STR_LEN];
-		memnode_container_alias(memnode_container, STARPU_POTI_STR_LEN, prefix, ev->param[2]);
-		char new_worker_container_name[STARPU_POTI_STR_LEN];
-		snprintf(new_worker_container_name, STARPU_POTI_STR_LEN, "%s%"PRIu64"", prefix, ev->param[3]);
-		poti_CreateContainer(get_event_time_stamp(ev, options), new_worker_container_alias, "T", memnode_container, new_worker_container_name);
-#else
-		fprintf(out_paje_file, "7	%.9f	t%s%"PRIu64"	T	%smn%"PRIu64"	%s%"PRIu64"\n",
-			get_event_time_stamp(ev, options), prefix, ev->param[3], prefix, ev->param[2], prefix, ev->param[3]);
-#endif
-	}
+	int devid = ev->param[2];
+	int workerid = ev->param[1];
+	int nodeid = ev->param[3];
+	int threadid = ev->param[4];
 
-	int devid = ev->param[1];
-	int workerid = register_worker_id(ev->param[3]);
+	register_worker_id(threadid, workerid);
 
 	char *kindstr = "";
 	enum starpu_perf_archtype archtype = 0;
@@ -304,30 +297,53 @@ static void handle_worker_init_start(struct fxt_ev_64 *ev, struct starpu_fxt_opt
 	{
 		case _STARPU_FUT_APPS_KEY:
 			set_next_other_worker_color(workerid);
-			kindstr = "apps";
+			kindstr = "APPS";
 			break;
 		case _STARPU_FUT_CPU_KEY:
 			set_next_cpu_worker_color(workerid);
-			kindstr = "cpu";
+			kindstr = "CPU";
 			archtype = STARPU_CPU_DEFAULT;
 			break;
 		case _STARPU_FUT_CUDA_KEY:
 			set_next_cuda_worker_color(workerid);
-			kindstr = "cuda";
+			kindstr = "CUDA";
 			archtype = STARPU_CUDA_DEFAULT + devid;
 			break;
 		case _STARPU_FUT_OPENCL_KEY:
 			set_next_opencl_worker_color(workerid);
-			kindstr = "opencl";
+			kindstr = "OPENCL";
 			archtype = STARPU_OPENCL_DEFAULT + devid;
 			break;
 		default:
 			STARPU_ABORT();
 	}
 
+	if (out_paje_file)
+	{
+#ifdef STARPU_HAVE_POTI
+		char new_thread_container_alias[STARPU_POTI_STR_LEN];
+		thread_container_alias (new_thread_container_alias, STARPU_POTI_STR_LEN, prefix, threadid);
+		char new_worker_container_alias[STARPU_POTI_STR_LEN];
+		worker_container_alias (new_worker_container_alias, STARPU_POTI_STR_LEN, prefix, workerid);
+		char memnode_container[STARPU_POTI_STR_LEN];
+		memnode_container_alias(memnode_container, STARPU_POTI_STR_LEN, prefix, nodeid);
+		char new_thread_container_name[STARPU_POTI_STR_LEN];
+		snprintf(new_thread_container_name, STARPU_POTI_STR_LEN, "%s%d", prefix, threadid);
+		char new_worker_container_name[STARPU_POTI_STR_LEN];
+		snprintf(new_worker_container_name, STARPU_POTI_STR_LEN, "%s%s%d", kindstr, prefix, devid);
+		poti_CreateContainer(get_event_time_stamp(ev, options), new_thread_container_alias, "T", memnode_container, new_thread_container_name);
+		poti_CreateContainer(get_event_time_stamp(ev, options), new_worker_container_alias, "W", new_thread_container_alias, new_worker_container_name);
+#else
+		fprintf(out_paje_file, "7	%.9f	t%s%d	T	mn%s%d	%s%d\n",
+			get_event_time_stamp(ev, options), prefix, threadid, prefix, nodeid, prefix, threadid);
+		fprintf(out_paje_file, "7	%.9f	w%s%d	W	t%s%d	%s%s%d\n",
+			get_event_time_stamp(ev, options), prefix, workerid, prefix, threadid, kindstr, prefix, devid);
+#endif
+	}
+
 	/* start initialization */
 	if (out_paje_file)
-		worker_set_state(get_event_time_stamp(ev, options), prefix, ev->param[3], "I");
+		worker_set_state(get_event_time_stamp(ev, options), prefix, threadid, "I");
 
 	if (activity_file)
 	fprintf(activity_file, "name\t%d\t%s %d\n", workerid, kindstr, devid);
@@ -366,7 +382,7 @@ static void handle_worker_deinit_end(struct fxt_ev_64 *ev, struct starpu_fxt_opt
 	{
 #ifdef STARPU_HAVE_POTI
 		char worker_container[STARPU_POTI_STR_LEN];
-		worker_container_alias(worker_container, STARPU_POTI_STR_LEN, prefix, ev->param[1]);
+		thread_container_alias(worker_container, STARPU_POTI_STR_LEN, prefix, ev->param[1]);
 		poti_DestroyContainer(get_event_time_stamp(ev, options), "T", worker_container);
 #else
 		fprintf(out_paje_file, "8	%.9f	t%s%"PRIu64"	T\n",
@@ -489,7 +505,7 @@ static void handle_start_codelet_body(struct fxt_ev_64 *ev, struct starpu_fxt_op
 			char container[STARPU_POTI_STR_LEN];
 			char ctx[6];
 			snprintf(ctx, sizeof(ctx), "Ctx%d", sched_ctx);
-			worker_container_alias(container, STARPU_POTI_STR_LEN, prefix, ev->param[2]);
+			thread_container_alias(container, STARPU_POTI_STR_LEN, prefix, ev->param[2]);
 			poti_SetState(start_codelet_time, container, ctx, name);
 #else
 			fprintf(out_paje_file, "10	%.9f	t%s%"PRIu64"	Ctx%d	%s\n", start_codelet_time, prefix, ev->param[2], sched_ctx, name);
@@ -568,7 +584,7 @@ static void handle_user_event(struct fxt_ev_64 *ev, struct starpu_fxt_options *o
 	{
 		if (out_paje_file)
 #ifdef STARPU_HAVE_POTI
-			worker_container_alias (container, STARPU_POTI_STR_LEN, prefix, ev->param[1]);
+			thread_container_alias (container, STARPU_POTI_STR_LEN, prefix, ev->param[1]);
 #else
 			fprintf(out_paje_file, "9	%.9f	event	t%s%"PRIu64"	%lu\n", get_event_time_stamp(ev, options), prefix, ev->param[1], code);
 #endif
@@ -671,7 +687,7 @@ static void handle_start_driver_copy(struct fxt_ev_64 *ev, struct starpu_fxt_opt
 			memnode_container_alias(src_memnode_container, STARPU_POTI_STR_LEN, prefix, src);
 			poti_StartLink(time, program_container, "L", src_memnode_container, paje_value, paje_key);
 #else
-			fprintf(out_paje_file, "18	%.9f	L	%sp	%u	%smn%u	com_%u\n", time, prefix, size, prefix, src, comid);
+			fprintf(out_paje_file, "18	%.9f	L	%sp	%u	mn%s%u	com_%u\n", time, prefix, size, prefix, src, comid);
 #endif
 		}
 
@@ -705,12 +721,13 @@ static void handle_end_driver_copy(struct fxt_ev_64 *ev, struct starpu_fxt_optio
 #ifdef STARPU_HAVE_POTI
 			char paje_value[STARPU_POTI_STR_LEN], paje_key[STARPU_POTI_STR_LEN];
 			char dst_memnode_container[STARPU_POTI_STR_LEN], program_container[STARPU_POTI_STR_LEN];
-			program_container_alias(program_container, STARPU_POTI_STR_LEN, prefix);
 			snprintf(paje_value, STARPU_POTI_STR_LEN, "%u", size);
 			snprintf(paje_key, STARPU_POTI_STR_LEN, "com_%u", comid);
+			program_container_alias(program_container, STARPU_POTI_STR_LEN, prefix);
+			memnode_container_alias(dst_memnode_container, STARPU_POTI_STR_LEN, prefix, dst);
 			poti_EndLink(time, program_container, "L", dst_memnode_container, paje_value, paje_key);
 #else
-			fprintf(out_paje_file, "19	%.9f	L	%sp	%u	%smn%u	com_%u\n", time, prefix, size, prefix, dst, comid);
+			fprintf(out_paje_file, "19	%.9f	L	%sp	%u	mn%s%u	com_%u\n", time, prefix, size, prefix, dst, comid);
 #endif
 		}
 
@@ -995,7 +1012,7 @@ void _starpu_fxt_display_bandwidth(struct starpu_fxt_options *options)
 			memnode_container_alias(src_memnode_container, STARPU_POTI_STR_LEN, prefix, itor->src_node);
 			poti_SetVariable(itor->comm_start, src_memnode_container, "bw", current_bandwidth_per_node[itor->src_node]);
 #else
-			fprintf(out_paje_file, "13	%.9f	%smn%u	bw	%f\n",
+			fprintf(out_paje_file, "13	%.9f	mn%s%u	bw	%f\n",
 				itor->comm_start, prefix, itor->src_node, current_bandwidth_per_node[itor->src_node]);
 #endif
 		}
@@ -1008,7 +1025,7 @@ void _starpu_fxt_display_bandwidth(struct starpu_fxt_options *options)
 			memnode_container_alias(dst_memnode_container, STARPU_POTI_STR_LEN, prefix, itor->dst_node);
 			poti_SetVariable(itor->comm_start, dst_memnode_container, "bw", current_bandwidth_per_node[itor->dst_node]);
 #else
-			fprintf(out_paje_file, "13	%.9f	%smn%u	bw	%f\n",
+			fprintf(out_paje_file, "13	%.9f	mn%s%u	bw	%f\n",
 				itor->comm_start, prefix, itor->dst_node, current_bandwidth_per_node[itor->dst_node]);
 #endif
 		}

src/debug/traces/starpu_paje.c

@@ -1,6 +1,6 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2010-2012  Université de Bordeaux 1
+ * Copyright (C) 2010-2013  Université de Bordeaux 1
  *
  * StarPU is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
@@ -136,7 +136,8 @@ void _starpu_fxt_write_paje_header(FILE *file)
 	poti_DefineContainerType("MPIP", "0", "MPI Program");
 	poti_DefineContainerType("P", "MPIP", "Program");
 	poti_DefineContainerType("Mn", "P", "Memory Node");
-	poti_DefineContainerType("T", "Mn", "Worker");
+	poti_DefineContainerType("T", "Mn", "Thread");
+	poti_DefineContainerType("W", "T", "Worker");
 	poti_DefineContainerType("Sc", "P", "Scheduler");
 
 	/* Types for the memory node */
@@ -191,7 +192,8 @@ void _starpu_fxt_write_paje_header(FILE *file)
 1       MPIP      0       \"MPI Program\"                      	\n\
 1       P      MPIP       \"Program\"                      	\n\
 1       Mn      P       \"Memory Node\"                         \n\
-1       T      Mn       \"Worker\"                               \n\
+1       T      Mn       \"Thread\"                               \n\
+1       W      T       \"Worker\"                               \n\
 1       Sc       P       \"Scheduler State\"                        \n\
 2       event   T       \"event type\"				\n\
 3       S       T       \"Thread State\"                        \n");