Rearrange simgrid initialization: simgrid 3.16 and below use dlopen in smpi, which simplifies things a lot

src/core/simgrid.c

@@ -37,8 +37,10 @@
 
 #pragma weak starpu_main
 extern int starpu_main(int argc, char *argv[]);
+#if SIMGRID_VERSION_MAJOR < 3 || (SIMGRID_VERSION_MAJOR == 3 && SIMGRID_VERSION_MINOR < 16)
 #pragma weak smpi_main
 extern int smpi_main(int (*realmain) (int argc, char *argv[]), int argc, char *argv[]);
+#endif
 #pragma weak _starpu_mpi_simgrid_init
 extern int _starpu_mpi_simgrid_init(int argc, char *argv[]);
 
@@ -63,19 +65,6 @@ static struct worker_runner
 } worker_runner[STARPU_NMAXWORKERS];
 static int task_execute(int argc STARPU_ATTRIBUTE_UNUSED, char *argv[] STARPU_ATTRIBUTE_UNUSED);
 
-/* In case the MPI application didn't use smpicc to build the file containing
- * main(), try to cope by calling starpu_main */
-int _starpu_smpi_simulated_main_(int argc, char *argv[])
-{
-	if (!starpu_main)
-	{
-		_STARPU_ERROR("In simgrid mode, the file containing the main() function of this application needs to be compiled with starpu.h or starpu_simgrid_wrap.h included, to properly rename it into starpu_main\n");
-	}
-
-	return starpu_main(argc, argv);
-}
-int smpi_simulated_main_(int argc, char *argv[]) __attribute__((weak, alias("_starpu_smpi_simulated_main_")));
-
 #ifdef HAVE_MSG_ENVIRONMENT_GET_ROUTING_ROOT
 #ifdef HAVE_MSG_GET_AS_BY_NAME
 msg_as_t _starpu_simgrid_get_as_by_name(const char *name)
@@ -204,17 +193,29 @@ msg_host_t _starpu_simgrid_get_host_by_worker(struct _starpu_worker *worker)
 	return host;
 }
 
+/* Simgrid up to 3.15 would rename main into smpi_simulated_main_, and call that
+ * from SMPI initialization
+ * In case the MPI application didn't use smpicc to build the file containing
+ * main(), but included our #define main starpu_main, try to cope by calling
+ * starpu_main */
+int _starpu_smpi_simulated_main_(int argc, char *argv[])
+{
+	if (!starpu_main)
+	{
+		_STARPU_ERROR("In simgrid mode, the file containing the main() function of this application needs to be compiled with starpu.h or starpu_simgrid_wrap.h included, to properly rename it into starpu_main\n");
+	}
+
+	return starpu_main(argc, argv);
+}
+int smpi_simulated_main_(int argc, char *argv[]) __attribute__((weak, alias("_starpu_smpi_simulated_main_")));
+
+/* This is used to start a non-MPI simgrid environment */
 static void start_simgrid(int *argc, char **argv)
 {
 	char path[256];
 
 	simgrid_started = 1;
 
-	if (!starpu_main && !(smpi_main && smpi_simulated_main_ != _starpu_smpi_simulated_main_))
-	{
-		_STARPU_ERROR("In simgrid mode, the file containing the main() function of this application needs to be compiled with starpu.h or starpu_simgrid_wrap.h included, to properly rename it into starpu_main\n");
-	}
-
 	MSG_init(argc, argv);
 #if SIMGRID_VERSION_MAJOR < 3 || (SIMGRID_VERSION_MAJOR == 3 && SIMGRID_VERSION_MINOR < 9)
 	/* Versions earlier than 3.9 didn't support our communication tasks */
@@ -264,6 +265,7 @@ int do_starpu_main(int argc, char *argv[])
 #pragma weak smpi_process_get_user_data
 extern void *smpi_process_get_user_data();
 
+/* This is hopefully called before the application and simgrid */
 #undef main
 #pragma weak main
 int main(int argc, char **argv)
@@ -275,9 +277,16 @@ int main(int argc, char **argv)
 			fprintf(stderr,"Your version of simgrid does not provide smpi_process_get_user_data, we can not continue without it\n");
 			exit(1);
 		}
-		/* Oops, we are running SMPI, let it start Simgrid, and we'll
+
+#if SIMGRID_VERSION_MAJOR > 3 || (SIMGRID_VERSION_MAJOR == 3 && SIMGRID_VERSION_MINOR >= 16)
+		/* Recent versions of simgrid dlopen() us, so we don't need to
+		 * do circumvolutions, just init MPI early and run the application's main */
+		return _starpu_mpi_simgrid_init(argc, argv);
+#else
+		/* Oops, we are running old SMPI, let it start Simgrid, and we'll
 		 * take back hand in _starpu_simgrid_init from starpu_init() */
 		return smpi_main(_starpu_mpi_simgrid_init, argc, argv);
+#endif
 	}
 
 	/* Managed to catch application's main, initialize simgrid first */
@@ -291,9 +300,10 @@ int main(int argc, char **argv)
 		argv_cpy[i] = strdup(argv[i]);
 	void **tsd;
 	_STARPU_CALLOC(tsd, MAX_TSD+1, sizeof(void*));
+	/* Run the application in a separate thread */
 	MSG_process_create_with_arguments("main", &do_starpu_main, tsd, MSG_get_host_by_name("MAIN"), argc, argv_cpy);
 
-	/* And run maestro in main thread */
+	/* And run maestro in the main thread */
 	MSG_main();
 	return main_ret;
 }
@@ -305,16 +315,17 @@ static void maestro(void *data STARPU_ATTRIBUTE_UNUSED)
 }
 #endif
 
+/* This is called early from starpu_init, so thread functions etc. can work */
 void _starpu_simgrid_init_early(int *argc STARPU_ATTRIBUTE_UNUSED, char ***argv STARPU_ATTRIBUTE_UNUSED)
 {
 #ifdef HAVE_MSG_PROCESS_ATTACH
-	if (!simgrid_started && !(smpi_main && smpi_simulated_main_ != _starpu_smpi_simulated_main_))
+	if (!simgrid_started && !_starpu_simgrid_running_smpi())
 	{
-		_STARPU_DISP("Warning: In simgrid mode, the file containing the main() function of this application should to be compiled with starpu.h or starpu_simgrid_wrap.h included, to properly rename it into starpu_main to avoid having to use --cfg=contexts/factory:thread which reduces performance\n");
-#if SIMGRID_VERSION_MAJOR > 3 || (SIMGRID_VERSION_MAJOR == 3 && SIMGRID_VERSION_MINOR >= 14) /* Only recent versions of simgrid support setting xbt_cfg_set_string before starting simgrid */
 		/* "Cannot create_maestro with this ContextFactory.
 		 * Try using --cfg=contexts/factory:thread instead."
 		 * See https://github.com/simgrid/simgrid/issues/141 */
+		_STARPU_DISP("Warning: In simgrid mode, the file containing the main() function of this application should to be compiled with starpu.h or starpu_simgrid_wrap.h included, to properly rename it into starpu_main to avoid having to use --cfg=contexts/factory:thread which reduces performance\n");
+#if SIMGRID_VERSION_MAJOR > 3 || (SIMGRID_VERSION_MAJOR == 3 && SIMGRID_VERSION_MINOR >= 14) /* Only recent versions of simgrid support setting xbt_cfg_set_string before starting simgrid */
 		xbt_cfg_set_string("contexts/factory", "thread");
 #endif
 		/* We didn't catch application's main. */
@@ -330,8 +341,10 @@ void _starpu_simgrid_init_early(int *argc STARPU_ATTRIBUTE_UNUSED, char ***argv
 	}
 #endif
 
-	if (!simgrid_started && !starpu_main && !(smpi_main && smpi_simulated_main_ != _starpu_smpi_simulated_main_))
+	if (!simgrid_started && !starpu_main && !_starpu_simgrid_running_smpi())
 	{
+                /* Oops, we don't have MSG_process_attach and didn't catch the
+                 * 'main' symbol, there is no way for us */
 		_STARPU_ERROR("In simgrid mode, the file containing the main() function of this application needs to be compiled with starpu.h or starpu_simgrid_wrap.h included, to properly rename it into starpu_main\n");
 	}
 	if (_starpu_simgrid_running_smpi())
@@ -350,6 +363,7 @@ void _starpu_simgrid_init_early(int *argc STARPU_ATTRIBUTE_UNUSED, char ***argv
 		starpu_pthread_queue_init(&_starpu_simgrid_task_queue[i]);
 }
 
+/* This is called late from starpu_init, to start task executors */
 void _starpu_simgrid_init(void)
 {
 	unsigned i;