mpi: new configure option --enable-mpi-pedantic-isend (disabled by default) to acquire data in STARPU_RW (instead of STARPU_R) before performing MPI_Isend call

ChangeLog

@@ -1,7 +1,7 @@
 # StarPU --- Runtime system for heterogeneous multicore architectures.
 #
 # Copyright (C) 2009-2017  Université de Bordeaux
-# Copyright (C) 2010, 2011, 2012, 2013, 2014, 2015, 2016  CNRS
+# Copyright (C) 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017  CNRS
 # Copyright (C) 2014, 2016, 2017  INRIA
 #
 # StarPU is free software; you can redistribute it and/or modify
@@ -32,6 +32,9 @@ Small features:
   * Add STARPU_SIMGRID_TASK_SUBMIT_COST to simulate the cost of task submission
     in simgrid mode. This provides more accurate simgrid predictions, especially
     for the beginning of the execution.
+  * New configure option --enable-mpi-pedantic-isend (disabled by
+    default) to acquire data in STARPU_RW (instead of STARPU_R) before
+    performing MPI_Isend call
 
 Changes:
   * Vastly improve simgrid simulation time.

configure.ac

@@ -298,7 +298,7 @@ AC_ARG_WITH(mpicxx, [AS_HELP_STRING([--with-mpicxx[=<path to mpicxx>]],
        fi
        # nothing was specified: default value is used
        AC_PATH_PROG(mpicxx_path, $DEFAULT_MPICXX, [no], [$simgrid_dir/bin:$PATH])
-       
+
        # try with mpic++ if mpicxx was not found
        if test x$mpicxx_path = xno ; then
             DEFAULT_MPICXX=mpic++
@@ -341,6 +341,13 @@ if  test x$enable_mpi_progression_hook = xyes; then
 	AC_DEFINE(STARPU_MPI_ACTIVITY, [1], [enable StarPU MPI activity polling method])
 fi
 
+AC_ARG_ENABLE(mpi-pedantic-isend, [AS_HELP_STRING([--enable-mpi-pedantic-isend],
+				   [Enable StarPU MPI pedantic isend])],
+				   enable_mpi_pedantic_isend=$enableval, enable_mpi_pedantic_isend=no)
+if  test x$enable_mpi_pedantic_isend = xyes; then
+	AC_DEFINE(STARPU_MPI_PEDANTIC_ISEND, [1], [enable StarPU MPI pedantic isend])
+fi
+
 #We can only build MPI Master Slave if User wants it and MPI is available
 if test x$use_mpi_master_slave = xyes -a x$use_mpi = xyes -a x$use_mpicxx = xyes; then
     build_mpi_master_slave=yes
@@ -355,10 +362,10 @@ fi
 
 if test x$build_mpi_master_slave = xyes; then
     AC_DEFINE(STARPU_USE_MPI_MASTER_SLAVE, [1], [MPI Master Slave support is enabled])
-    CC=$mpicc_path    
-    CCLD=$mpicc_path      
-    CXX=$mpicxx_path      
-    CXXLD=mpicxx_path    
+    CC=$mpicc_path
+    CCLD=$mpicc_path
+    CXX=$mpicxx_path
+    CXXLD=mpicxx_path
 fi
 
 AC_ARG_WITH(mpi-master-slave-multiple-thread, [AS_HELP_STRING([--with-mpi-master-slave-multiple-thread])],
@@ -603,7 +610,7 @@ then
     INCLUDE_PTHREAD_H='#include <pthread.h>'
 fi
 
-AC_CHECK_TYPE([struct timespec], 
+AC_CHECK_TYPE([struct timespec],
 	       AC_DEFINE(STARPU_HAVE_STRUCT_TIMESPEC,[1],[struct timespec is defined]),
 	       [], [
 #include <sys/types.h>
@@ -1330,7 +1337,7 @@ if test x$enable_opencl = xyes -o x$enable_opencl = xmaybe; then
           [AC_MSG_RESULT(no)
              enable_opencl=no])
           LIBS=$SAVED_LIBS
-          ;;        
+          ;;
         *)
 	  STARPU_LOOK_FOR_OPENCL()
 	  # in case OpenCL was explicitely required, but is not available, this is an error
@@ -2136,7 +2143,7 @@ AC_MSG_RESULT($nmaxworkers)
 AC_DEFINE_UNQUOTED(STARPU_NMAXWORKERS, [$nmaxworkers], [Maximum number of workers])
 
 # Computes the maximun number of combined worker
-nmaxcombinedworkers=`expr $maxcpus + $nmaxmicthreads`  
+nmaxcombinedworkers=`expr $maxcpus + $nmaxmicthreads`
 AC_MSG_CHECKING(Maximum number of workers combinations)
 AC_MSG_RESULT($nmaxcombinedworkers)
 AC_DEFINE_UNQUOTED(STARPU_NMAX_COMBINEDWORKERS,
@@ -2445,9 +2452,9 @@ fi
 if test x$build_mpi_master_slave = xyes; then
     #Check if we can compile fortran cases
     if test x$use_mpi_fort = xyes ; then
-        F77LD=$mpifort_path    
+        F77LD=$mpifort_path
         FCLD=$mpifort_path
-        F77=$mpifort_path    
+        F77=$mpifort_path
         FC=$mpifort_path
     else
         enable_build_fortran=no
@@ -2810,7 +2817,7 @@ if test x$enable_mlr = xyes -a "$starpu_windows" != "yes" ; then
 		LDFLAGS="-llapack $LDFLAGS"
 	else
 		if test x$blas_lib = xmkl; then
-		   	AC_DEFINE(STARPU_MLR_MODEL, [1], [use mkl library])			
+		   	AC_DEFINE(STARPU_MLR_MODEL, [1], [use mkl library])
 		else
 			AC_MSG_CHECKING(whether min-dgels is linked)
 			if test x"$DGELS_LIBS" != x; then

doc/doxygen/chapters/410_mpi_support.doxy

@@ -1,7 +1,7 @@
 /*
  * This file is part of the StarPU Handbook.
  * Copyright (C) 2009--2011  Universit@'e de Bordeaux
- * Copyright (C) 2010, 2011, 2012, 2013, 2014, 2015, 2016  CNRS
+ * Copyright (C) 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017  CNRS
  * Copyright (C) 2011, 2012, 2017  INRIA
  * See the file version.doxy for copying conditions.
  */
@@ -208,6 +208,12 @@ automatically released. This mechanism is similar to the pthread
 detach state attribute which determines whether a thread will be
 created in a joinable or a detached state.
 
+For send communications, data is acquired with the mode ::STARPU_R.
+When using the configure option
+\ref enable-mpi-pedantic-isend "--enable-mpi-pedantic-isend", the mode
+::STARPU_RW is used to make sure there is no more than 1 concurrent
+MPI_Isend call accessing a data.
+
 Internally, all communication are divided in 2 communications, a first
 message is used to exchange an envelope describing the data (i.e its
 tag and its size), the data itself is sent in a second message. All

doc/doxygen/chapters/510_configure_options.doxy

@@ -1,7 +1,7 @@
 /*
  * This file is part of the StarPU Handbook.
  * Copyright (C) 2009--2011  Universit@'e de Bordeaux
- * Copyright (C) 2010, 2011, 2012, 2013, 2014, 2015, 2016  CNRS
+ * Copyright (C) 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017  CNRS
  * Copyright (C) 2011, 2012, 2017  INRIA
  * See the file version.doxy for copying conditions.
  */
@@ -327,6 +327,34 @@ used by StarPU data structures.
 Disable the build of libstarpumpi. By default, it is enabled when MPI is found.
 </dd>
 
+<dt>--with-mpicc=<c>path</c></dt>
+<dd>
+\anchor with-mpicc
+\addindex __configure__--with-mpicc
+Use the compiler <c>mpicc</c> at <c>path</c>, for StarPU-MPI.
+(\ref MPISupport).
+</dd>
+
+<dt>--enable-mpi-progression-hook</dt>
+<dd>
+\anchor enable-mpi-progression-hook
+\addindex __configure__--enable-mpi-progression-hook
+Enable the activity polling method for StarPU-MPI. This is however experimental,
+do not enable it unless you know what you are doing.
+</dd>
+
+<dt>--enable-mpi-pedantic-isend</dt>
+<dd>
+\anchor enable-mpi-pedantic-isend
+\addindex __configure__--enable-mpi-pedantic-isend
+Before performing any MPI communication, StarPU-MPI waits for the data
+to be available in the main memory of the node submitting the request.
+For send communications, data is acquired with the mode ::STARPU_R.
+When enabling the pedantic mode, data are instead acquired with the
+::STARPU_RW which thus ensures that there is not more than 1
+concurrent MPI_Isend calls accessing the data.
+</dd>
+
 <dt>--enable-mpi-master-slave</dt>
 <dd>
 \anchor enable-mpi-master-slave
@@ -373,22 +401,6 @@ Disable the GCC plug-in (\ref cExtensions).  By default, it is
 enabled when the GCC compiler provides a plug-in support.
 </dd>
 
-<dt>--with-mpicc=<c>path</c></dt>
-<dd>
-\anchor with-mpicc
-\addindex __configure__--with-mpicc
-Use the compiler <c>mpicc</c> at <c>path</c>, for StarPU-MPI.
-(\ref MPISupport).
-</dd>
-
-<dt>--enable-mpi-progression-hook</dt>
-<dd>
-\anchor enable-mpi-progression-hook
-\addindex __configure__--enable-mpi-progression-hook
-Enable the activity polling method for StarPU-MPI. This is however experimental,
-do not enable it unless you know what you are doing.
-</dd>
-
 <dt>--with-coi-dir</dt>
 <dd>
 \anchor with-coi-dir

mpi/src/starpu_mpi.c

@@ -533,7 +533,13 @@ static struct _starpu_mpi_req *_starpu_mpi_isend_common(starpu_data_handle_t dat
 							unsigned detached, unsigned sync, void (*callback)(void *), void *arg,
 							int sequential_consistency)
 {
-	return _starpu_mpi_isend_irecv_common(data_handle, dest, data_tag, comm, detached, sync, callback, arg, SEND_REQ, _starpu_mpi_isend_size_func, STARPU_R, sequential_consistency, 0, 0);
+	return _starpu_mpi_isend_irecv_common(data_handle, dest, data_tag, comm, detached, sync, callback, arg, SEND_REQ, _starpu_mpi_isend_size_func,
+#ifdef STARPU_MPI_PEDANTIC_ISEND
+					      STARPU_RW,
+#else
+					      STARPU_R,
+#endif
+					      sequential_consistency, 0, 0);
 }
 
 int starpu_mpi_isend(starpu_data_handle_t data_handle, starpu_mpi_req *public_req, int dest, int data_tag, MPI_Comm comm)