Extend simgrid initialization to distinguish between main redirection and MSG_process_attach. When using openmp, prevent simgrid from cleaning at exit since we still need it in destructors

src/core/simgrid.c

@@ -47,6 +47,8 @@ extern int smpi_main(int (*realmain) (int argc, char *argv[]), int argc, char *a
 #pragma weak _starpu_mpi_simgrid_init
 extern int _starpu_mpi_simgrid_init(int argc, char *argv[]);
 
+/* 1 when MSG_init was done, 2 when initialized through redirected main, 3 when
+ * initialized through MSG_process_attach */
 static int simgrid_started;
 
 static int runners_running;
@@ -298,9 +300,17 @@ int main(int argc, char **argv)
 #endif
 	}
 
+        /* Already initialized?  It probably has been done through a
+         * constructor and MSG_process_attach, directly jump to real main */
+	if (simgrid_started == 3) {
+		return do_starpu_main(argc, argv);
+	}
+
 	/* Managed to catch application's main, initialize simgrid first */
 	_starpu_start_simgrid(&argc, argv);
 
+	simgrid_started = 2;
+
 	/* Create a simgrid process for main */
 	char **argv_cpy;
 	_STARPU_MALLOC(argv_cpy, argc * sizeof(char*));
@@ -309,6 +319,7 @@ int main(int argc, char **argv)
 		argv_cpy[i] = strdup(argv[i]);
 	void **tsd;
 	_STARPU_CALLOC(tsd, MAX_TSD+1, sizeof(void*));
+
 	/* Run the application in a separate thread */
 	MSG_process_create_with_arguments("main", &do_starpu_main, tsd, MSG_get_host_by_name("MAIN"), argc, argv_cpy);
 
@@ -328,7 +339,7 @@ static void maestro(void *data STARPU_ATTRIBUTE_UNUSED)
 void _starpu_simgrid_init_early(int *argc STARPU_ATTRIBUTE_UNUSED, char ***argv STARPU_ATTRIBUTE_UNUSED)
 {
 #ifdef HAVE_MSG_PROCESS_ATTACH
-	if (!simgrid_started && !_starpu_simgrid_running_smpi())
+	if (simgrid_started < 2 && !_starpu_simgrid_running_smpi())
 	{
 		/* "Cannot create_maestro with this ContextFactory.
 		 * Try using --cfg=contexts/factory:thread instead."
@@ -346,7 +357,8 @@ void _starpu_simgrid_init_early(int *argc STARPU_ATTRIBUTE_UNUSED, char ***argv
 		void **tsd;
 		_STARPU_CALLOC(tsd, MAX_TSD+1, sizeof(void*));
 		MSG_process_attach("main", tsd, MSG_get_host_by_name("MAIN"), NULL);
-		simgrid_started = 2;
+		/* We initialized through MSG_process_attach */
+		simgrid_started = 3;
 	}
 #endif
 
@@ -389,6 +401,18 @@ void _starpu_simgrid_init(void)
 	}
 }
 
+void _starpu_simgrid_deinit_late(void)
+{
+#ifdef HAVE_MSG_PROCESS_ATTACH
+	if (simgrid_started == 3)
+	{
+		/* Started with MSG_process_attach, now detach */
+		MSG_process_detach();
+		simgrid_started = 0;
+	}
+#endif
+}
+
 void _starpu_simgrid_deinit(void)
 {
 	unsigned i, j;
@@ -428,14 +452,19 @@ void _starpu_simgrid_deinit(void)
 		MSG_sem_destroy(w->sem);
 		starpu_pthread_queue_destroy(&_starpu_simgrid_task_queue[i]);
 	}
-#ifdef HAVE_MSG_PROCESS_ATTACH
-	if (simgrid_started == 2)
-	{
-		/* Started with MSG_process_attach, now detach */
-		MSG_process_detach();
-		simgrid_started = 0;
-	}
+
+#if SIMGRID_VERSION_MAJOR < 3 || (SIMGRID_VERSION_MAJOR == 3 && SIMGRID_VERSION_MINOR < 13)
+	extern xbt_cfg_t _sg_cfg_set;
+#endif
+	if (
+#if SIMGRID_VERSION_MAJOR < 3 || (SIMGRID_VERSION_MAJOR == 3 && SIMGRID_VERSION_MINOR < 13)
+		xbt_cfg_get_boolean(_sg_cfg_set, "clean-atexit")
+#else
+		xbt_cfg_get_boolean("clean-atexit")
 #endif
+		) {
+		_starpu_simgrid_deinit_late();
+	}
 }
 
 /*

src/core/simgrid.h

@@ -47,6 +47,7 @@ void _starpu_start_simgrid(int *argc, char **argv);
 void _starpu_simgrid_init_early(int *argc, char ***argv);
 void _starpu_simgrid_init(void);
 void _starpu_simgrid_deinit(void);
+void _starpu_simgrid_deinit_late(void);
 void _starpu_simgrid_wait_tasks(int workerid);
 struct _starpu_job;
 void _starpu_simgrid_submit_job(int workerid, struct _starpu_job *job, struct starpu_perfmodel_arch* perf_arch, double length, unsigned *finished);

src/util/openmp_runtime_support.c

@@ -921,10 +921,13 @@ void _starpu_omp_dummy_shutdown(void)
 int starpu_omp_init(void)
 {
 #ifdef STARPU_SIMGRID
-	/* XXX: ideally we'd pass the real argc/argv */
-	int argc;
-	char *argv[] = { NULL };
-	_starpu_start_simgrid(&argc, &argv);
+	/* XXX: ideally we'd pass the real argc/argv.  */
+	/* We have to tell simgrid to avoid cleaning up at exit, since that's before our destructor :/ */
+	char *argv[] = { "program", "--cfg=clean-atexit:0", NULL };
+	int argc = sizeof(argv) / sizeof(argv[0]) - 1;
+	char **_argv = argv;
+	/* Initialize simgrid before anything else.  */
+	_starpu_simgrid_init_early(&argc, &_argv);
 #endif
 
 	_starpu_omp_global_state = &_global_state;
@@ -998,6 +1001,7 @@ void starpu_omp_shutdown(void)
 	_starpu_omp_environment_exit();
 	STARPU_PTHREAD_KEY_DELETE(omp_task_key);
 	STARPU_PTHREAD_KEY_DELETE(omp_thread_key);
+	_starpu_simgrid_deinit_late();
 }
 
 static void implicit_task__destroy_callback(void *_task)

tests/Makefile.am

@@ -170,6 +170,7 @@ myPROGRAMS +=					\
 	sched_ctx/sched_ctx_list		\
 	sched_ctx/sched_ctx_policy_data		\
 	openmp/init_exit_01			\
+	openmp/init_exit_02			\
 	perfmodels/value_nan
 
 if STARPU_SIMGRID
@@ -303,7 +304,6 @@ myPROGRAMS +=				\
 	microbenchs/prefetch_data_on_node 	\
 	microbenchs/redundant_buffer		\
 	microbenchs/matrix_as_vector		\
-	openmp/init_exit_02			\
 	openmp/environment			\
 	openmp/api_01				\
 	openmp/parallel_01			\