Benefit from MSG_process_attach

configure.ac

@@ -1075,7 +1075,7 @@ if test x$enable_simgrid = xyes ; then
 		]
 	)
 	AC_CHECK_HEADERS([simgrid/msg.h], [AC_DEFINE([STARPU_HAVE_SIMGRID_MSG_H], [1], [Define to 1 if you have msg.h in simgrid/.])])
-   	AC_CHECK_FUNCS([MSG_process_join MSG_get_as_by_name MSG_environment_get_routing_root MSG_host_get_speed xbt_mutex_try_acquire smpi_process_set_user_data])
+   	AC_CHECK_FUNCS([MSG_process_join MSG_process_attach MSG_get_as_by_name MSG_environment_get_routing_root MSG_host_get_speed xbt_mutex_try_acquire smpi_process_set_user_data])
 	AC_CHECK_FUNCS([xbt_barrier_init], [AC_DEFINE([STARPU_SIMGRID_HAVE_XBT_BARRIER_INIT], [1], [Define to 1 if you have the `xbt_barrier_init' function.])])
 	AC_CHECK_DECLS([smpi_process_set_user_data], [], [], [[#include <smpi/smpi.h>]])
 	AC_CHECK_FUNCS([SIMIX_process_get_code], [AC_DEFINE([STARPU_SIMGRID_HAVE_SIMIX_PROCESS_GET_CODE], [1], [Define to 1 if you have the `SIMIX_process_get_code' function.])])

src/core/simgrid.c

@@ -34,24 +34,11 @@ extern int smpi_main(int (*realmain) (int argc, char *argv[]), int argc, char *a
 #pragma weak _starpu_mpi_simgrid_init
 extern int _starpu_mpi_simgrid_init(int argc, char *argv[]);
 
-static int main_ret;
+static int simgrid_started;
 
 starpu_pthread_queue_t _starpu_simgrid_transfer_queue[STARPU_MAXNODES];
 starpu_pthread_queue_t _starpu_simgrid_task_queue[STARPU_NMAXWORKERS];
 
-struct main_args
-{
-	int argc;
-	char **argv;
-};
-
-int do_starpu_main(int argc STARPU_ATTRIBUTE_UNUSED, char *argv[])
-{
-	struct main_args *args = (void*) argv;
-	main_ret = starpu_main(args->argc, args->argv);
-	return main_ret;
-}
-
 /* In case the MPI application didn't use smpicc to build the file containing
  * main(), try to cope by calling starpu_main */
 #pragma weak smpi_simulated_main_
@@ -193,27 +180,18 @@ msg_host_t _starpu_simgrid_get_host_by_worker(struct _starpu_worker *worker)
 	return host;
 }
 
-#ifdef STARPU_DEVEL
-#warning TODO: use another way to start main, when simgrid provides it, and then include the application-provided configuration for platform numbers
-#endif
-#undef main
-int main(int argc, char **argv)
+static void start_simgrid(int *argc, char **argv)
 {
 	char path[256];
 
-	if (!starpu_main && !(smpi_main && smpi_simulated_main_))
-	{
-		_STARPU_ERROR("In simgrid mode, the file containing the main() function of this application needs to be compiled with starpu.h included, to properly rename it into starpu_main\n");
-	}
+	simgrid_started = 1;
 
-	if (_starpu_simgrid_running_smpi())
+	if (!starpu_main && !(smpi_main && smpi_simulated_main_))
 	{
-		/* Oops, we are running SMPI, let it start Simgrid, and we'll
-		 * take back hand in _starpu_simgrid_init from starpu_init() */
-		return smpi_main(_starpu_mpi_simgrid_init, argc, argv);
+		_STARPU_ERROR("In simgrid mode, the file containing the main() function of this application needs to be compiled with starpu.h or starpu_simgrid_wrap.h included, to properly rename it into starpu_main\n");
 	}
 
-	MSG_init(&argc, argv);
+	MSG_init(argc, argv);
 #if SIMGRID_VERSION_MAJOR < 3 || (SIMGRID_VERSION_MAJOR == 3 && SIMGRID_VERSION_MINOR < 9)
 	/* Versions earlier than 3.9 didn't support our communication tasks */
 	MSG_config("workstation/model", "ptask_L07");
@@ -238,22 +216,75 @@ int main(int argc, char **argv)
 	_starpu_simgrid_get_platform_path(4, path, sizeof(path));
 #endif
 	MSG_create_environment(path);
+}
+
+struct main_args
+{
+	int argc;
+	char **argv;
+};
+static int main_ret;
 
+int do_starpu_main(int argc STARPU_ATTRIBUTE_UNUSED, char *argv[])
+{
+	struct main_args *args = (void*) argv;
+	main_ret = starpu_main(args->argc, args->argv);
+	return main_ret;
+}
+
+#undef main
+#pragma weak main
+int main(int argc, char **argv)
+{
+	if (_starpu_simgrid_running_smpi())
+	{
+		/* Oops, we are running SMPI, let it start Simgrid, and we'll
+		 * take back hand in _starpu_simgrid_init from starpu_init() */
+		return smpi_main(_starpu_mpi_simgrid_init, argc, argv);
+	}
+
+	/* Managed to catch application's main, initialize simgrid first */
+	start_simgrid(&argc, argv);
+
+	/* Create a simgrid process for main */
 	struct main_args *args = malloc(sizeof(*args));
 	args->argc = argc;
 	args->argv = argv;
 	MSG_process_create_with_arguments("main", &do_starpu_main, calloc(MAX_TSD, sizeof(void*)), MSG_get_host_by_name("MAIN"), 0, (char**) args);
 
+	/* And run maestro in main thread */
 	MSG_main();
 	return main_ret;
 }
 
-void _starpu_simgrid_init()
+static void maestro(void *data STARPU_ATTRIBUTE_UNUSED)
+{
+	MSG_main();
+}
+
+void _starpu_simgrid_init(int *argc, char ***argv)
 {
+#ifdef HAVE_MSG_PROCESS_ATTACH
+	if (!simgrid_started)
+	{
+#if SIMGRID_VERSION_MAJOR < 3 || (SIMGRID_VERSION_MAJOR == 3 && SIMGRID_VERSION_MINOR < 14)
+		_STARPU_DISP("In simgrid mode, the file containing the main() function of this application should to be compiled with starpu.h or starpu_simgrid_wrap.h included, to properly rename it into starpu_main to avoid having to use --cfg=contexts/factory:thread which reduces performance\n");
+#endif
+		/* We didn't catch application's main. */
+		/* Start maestro as a separate thread */
+		SIMIX_set_maestro(maestro, NULL);
+		/* Initialize simgrid */
+		start_simgrid(argc, *argv);
+		/* And attach the main thread to the main simgrid process */
+		MSG_process_attach("main", calloc(MAX_TSD, sizeof(void*)), MSG_get_host_by_name("MAIN"), NULL);
+		simgrid_started = 2;
+	}
+#endif
+
 	unsigned i;
-	if (!starpu_main && !(smpi_main && smpi_simulated_main_))
+	if (!simgrid_started && !starpu_main && !(smpi_main && smpi_simulated_main_))
 	{
-		_STARPU_ERROR("In simgrid mode, the file containing the main() function of this application needs to be compiled with starpu.h included, to properly rename it into starpu_main\n");
+		_STARPU_ERROR("In simgrid mode, the file containing the main() function of this application needs to be compiled with starpu.h or starpu_simgrid_wrap.h included, to properly rename it into starpu_main\n");
 	}
 	if (_starpu_simgrid_running_smpi())
 	{
@@ -268,6 +299,16 @@ void _starpu_simgrid_init()
 		starpu_pthread_queue_init(&_starpu_simgrid_task_queue[i]);
 }
 
+void _starpu_simgrid_deinit(void)
+{
+	if (simgrid_started == 2)
+	{
+		/* Started with MSG_process_attach, now detach */
+		MSG_process_detach();
+		simgrid_started = 0;
+	}
+}
+
 /*
  * Tasks
  */

src/core/simgrid.h

@@ -37,7 +37,8 @@ struct _starpu_pthread_args
 #define STARPU_MPI_AS_PREFIX "StarPU-MPI"
 #define _starpu_simgrid_running_smpi() (getenv("SMPI_GLOBAL_SIZE") != NULL)
 
-void _starpu_simgrid_init(void);
+void _starpu_simgrid_init(int *argc, char ***argv);
+void _starpu_simgrid_deinit(void);
 void _starpu_simgrid_wait_tasks(int workerid);
 void _starpu_simgrid_submit_job(int workerid, struct _starpu_job *job, struct starpu_perfmodel_arch* perf_arch, double length, unsigned *finished, starpu_pthread_mutex_t *mutex, starpu_pthread_cond_t *cond);
 int _starpu_simgrid_transfer(size_t size, unsigned src_node, unsigned dst_node, struct _starpu_data_request *req);

src/core/workers.c

@@ -1048,7 +1048,7 @@ int starpu_initialize(struct starpu_conf *user_conf, int *argc, char ***argv)
 
 #ifdef STARPU_SIMGRID
 	/* This initializes the simgrid thread library, thus needs to be early */
-	_starpu_simgrid_init();
+	_starpu_simgrid_init(argc, argv);
 #endif
 
 	STARPU_PTHREAD_MUTEX_LOCK(&init_mutex);
@@ -1565,6 +1565,11 @@ void starpu_shutdown(void)
 #endif
 	_starpu_print_idle_time();
 	_STARPU_DEBUG("Shutdown finished\n");
+
+#ifdef STARPU_SIMGRID
+	/* This finalizes the simgrid thread library, thus needs to be late */
+	_starpu_simgrid_deinit();
+#endif
 }