MPI: New feature to display communication statistics, i.e amount of data transfers between pair of MPI nodes

  * configure.ac: new option --enable-comm-stats to enable the new feature
  * mpi/starpu_mpi_stats.* : implement the new feature
  * mpi/starpu_mpi.c: invoke the new feature

configure.ac

@@ -1081,6 +1081,18 @@ if test x$use_mpi = xyes; then
 	AC_DEFINE(STARPU_USE_MPI,[],[whether the StarPU MPI library is available])
 fi
 
+AC_MSG_CHECKING(whether communication statistics should be generated)
+AC_ARG_ENABLE(comm-stats, [AS_HELP_STRING([--enable-comm-stats],
+			[enable communication statistics (only valid with the StarPU MPI library])],
+			enable_comm_stats=$enableval, enable_comm_stats=no)
+AC_MSG_RESULT($enable_comm_stats)
+AC_SUBST(STATS, $enable_comm_stats)
+AC_SUBST(STARPU_COMM_STATS, $enable_comm_stats)
+
+if test x$enable_comm_stats = xyes; then
+        AC_DEFINE(STARPU_COMM_STATS, [1], [enable communication statistics])
+fi
+
 ###############################################################################
 #                                                                             #
 #                               StarPU-Top                                    #

mpi/Makefile.am

@@ -76,7 +76,8 @@ libstarpumpi_@STARPU_EFFECTIVE_VERSION@_la_LDFLAGS = $(ldflags) -no-undefined
 noinst_HEADERS =					\
 	starpu_mpi_private.h				\
 	starpu_mpi_fxt.h				\
-	starpu_mpi_insert_task_cache.h
+	starpu_mpi_insert_task_cache.h			\
+	starpu_mpi_stats.h
 
 versincludedir = $(includedir)/starpu/$(STARPU_EFFECTIVE_VERSION)
 versinclude_HEADERS = 				\
@@ -89,7 +90,8 @@ libstarpumpi_@STARPU_EFFECTIVE_VERSION@_la_SOURCES =	\
 	starpu_mpi_datatype.c				\
 	starpu_mpi_insert_task.c			\
 	starpu_mpi_insert_task_cache.c			\
-	starpu_mpi_collective.c
+	starpu_mpi_collective.c				\
+	starpu_mpi_stats.c
 
 ###################
 # Stencil example #

mpi/starpu_mpi.c

@@ -1,7 +1,7 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
  * Copyright (C) 2009, 2010-2012  Université de Bordeaux 1
- * Copyright (C) 2010, 2011  Centre National de la Recherche Scientifique
+ * Copyright (C) 2010, 2011, 2012  Centre National de la Recherche Scientifique
  *
  * StarPU is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
@@ -21,6 +21,7 @@
 //#define STARPU_MPI_VERBOSE	1
 #include <starpu_mpi_private.h>
 #include <starpu_profiling.h>
+#include <starpu_mpi_stats.h>
 
 /* TODO find a better way to select the polling method (perhaps during the
  * configuration) */
@@ -63,6 +64,11 @@ static void starpu_mpi_isend_func(struct _starpu_mpi_req *req)
 
 	starpu_mpi_handle_to_datatype(req->data_handle, &req->datatype);
 
+#ifdef STARPU_DEVEL
+#  warning give the real size of the data
+#endif
+	_starpu_mpi_comm_amounts_inc(req->comm, req->srcdst, 1);//req->data_handle, req->mpi_tag);
+
         req->ret = MPI_Isend(ptr, 1, req->datatype, req->srcdst, req->mpi_tag, req->comm, &req->request);
         STARPU_ASSERT(req->ret == MPI_SUCCESS);
 
@@ -765,6 +771,10 @@ static void _starpu_mpi_add_sync_point_in_fxt(void)
 static
 int _starpu_mpi_initialize(int initialize_mpi, int *rank, int *world_size)
 {
+#ifdef STARPU_COMM_STATS
+	if (!getenv("STARPU_SILENT")) fprintf(stderr,"Warning: StarPU was configured with --enable-comm-stats, which slows down a bit\n");
+#endif
+
 	_STARPU_PTHREAD_MUTEX_INIT(&mutex, NULL);
 	_STARPU_PTHREAD_COND_INIT(&cond_progression, NULL);
 	_STARPU_PTHREAD_COND_INIT(&cond_finished, NULL);
@@ -800,7 +810,7 @@ int _starpu_mpi_initialize(int initialize_mpi, int *rank, int *world_size)
 #endif
 
 	_starpu_mpi_add_sync_point_in_fxt();
-
+	_starpu_mpi_comm_amounts_init();
 	return 0;
 }
 
@@ -834,6 +844,9 @@ int starpu_mpi_shutdown(void)
 	_starpu_mpi_req_list_delete(detached_requests);
 	_starpu_mpi_req_list_delete(new_requests);
 
+	_starpu_mpi_comm_amounts_display();
+	_starpu_mpi_comm_amounts_free();
+
 	return 0;
 }
 

mpi/starpu_mpi_stats.c

@@ -0,0 +1,97 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2012  Centre National de la Recherche Scientifique
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <starpu_mpi_stats.h>
+#include <common/config.h>
+#include <stdio.h>
+
+/* measure the amount of data transfers between each pair of MPI nodes */
+#ifdef STARPU_COMM_STATS
+static size_t **comm_amount;
+static int world_size;
+#endif /* STARPU_COMM_STATS */
+
+void _starpu_mpi_comm_amounts_init()
+{
+#ifdef STARPU_COMM_STATS
+	int i;
+
+	MPI_Comm_size(MPI_COMM_WORLD, &world_size);
+
+	comm_amount = (size_t **) calloc(1, world_size * sizeof(size_t *));
+	for(i=0 ; i<world_size ; i++)
+	{
+		comm_amount[i] = (size_t *) calloc(1, world_size * sizeof(size_t));
+	}
+#endif /* STARPU_COMM_STATS */
+}
+
+void _starpu_mpi_comm_amounts_free()
+{
+#ifdef STARPU_COMM_STATS
+	int i;
+	for(i=0 ; i<world_size ; i++)
+	{
+		free(comm_amount[i]);
+	}
+	free(comm_amount);
+#endif /* STARPU_COMM_STATS */
+}
+
+void _starpu_mpi_comm_amounts_inc(MPI_Comm comm  __attribute__ ((unused)),
+				  unsigned dst  __attribute__ ((unused)), size_t size  __attribute__ ((unused)))
+{
+#ifdef STARPU_COMM_STATS
+	int src;
+	MPI_Comm_rank(comm, &src);
+	comm_amount[src][dst] += size;
+#endif /* STARPU_COMM_STATS */
+}
+
+void _starpu_mpi_comm_amounts_display()
+{
+#ifdef STARPU_COMM_STATS
+	unsigned src, dst;
+
+	size_t sum = 0;
+
+	for (dst = 0; dst < world_size; dst++)
+		for (src = 0; src < world_size; src++)
+		{
+			sum += comm_amount[src][dst];
+		}
+
+	fprintf(stderr, "\nCommunication transfers stats:\nTOTAL transfers %f B\t%f MB\n", (float)sum, (float)sum/1024/1024);
+
+	for (dst = 0; dst < world_size; dst++)
+		for (src = 0; src < world_size; src++)
+		{
+			if (comm_amount[src][dst])
+			{
+				fprintf(stderr, "\t%d <-> %d\t%f B\t%f MB\n",
+					src, dst, (float)comm_amount[src][dst] + (float)comm_amount[dst][src],
+					((float)comm_amount[src][dst] + (float)comm_amount[dst][src])/(1024*1024));
+				fprintf(stderr, "\t\t%d -> %d\t%f B\t%f MB\n",
+					src, dst, (float)comm_amount[src][dst],
+					((float)comm_amount[src][dst])/(1024*1024));
+				fprintf(stderr, "\t\t%d -> %d\t%f B\t%f MB\n",
+					dst, src, (float)comm_amount[dst][src],
+					((float)comm_amount[dst][src])/(1024*1024));
+			}
+		}
+#endif /* STARPU_COMM_STATS */
+}
+

mpi/starpu_mpi_stats.h

@@ -0,0 +1,24 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2012  Centre National de la Recherche Scientifique
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <stdlib.h>
+#include <mpi.h>
+
+void _starpu_mpi_comm_amounts_init();
+void _starpu_mpi_comm_amounts_free();
+void _starpu_mpi_comm_amounts_inc(MPI_Comm comm, unsigned dst, size_t size);
+void _starpu_mpi_comm_amounts_display();
+