MPI: Improve starpu_mpi_insert_task

    - Rename property STARPU_EXECUTE into STARPU_EXECUTE_ON_NODE to specify the node executing the codelet.
    - Add the property STARPU_EXECUTE_ON_DATA which should be followed by a data handle to specify the codelet is to be executed on the node owning the data.

doc/starpu.texi

@@ -2160,15 +2160,19 @@ arguments.  The argument list must be zero-terminated.
 
 The arguments following the codelets are the same types as for the
 function @code{starpu_insert_task} defined in @ref{Insert Task
-Utility}. The extra argument @code{STARPU_EXECUTE} followed by an
-integer allows to specify the node to execute the codelet.
+Utility}. The extra argument @code{STARPU_EXECUTE_ON_NODE} followed by an
+integer allows to specify the node to execute the codelet. It is also
+possible to specify that the node owning a specific data will execute
+the codelet, by using @code{STARPU_EXECUTE_ON_DATA} followed by a data
+handle.
 
 The algorithm is as follows:
 @enumerate
 @item Find out whether we are to execute the codelet because we own the
 data to be written to. If different tasks own data to be written to,
-the argument @code{STARPU_EXECUTE} should be used to specify the
-executing task @code{ET}.
+the argument @code{STARPU_EXECUTE_ON_NODE} or
+@code{STARPU_EXECUTE_ON_DATA} should be used to specify the executing
+task @code{ET}.
 @item Send and receive data as requested. Tasks owning data which need
 to be read by the executing task @code{ET} are sending them to @code{ET}.
 @item Execute the codelet. This is done by the task selected in the

include/starpu_util.h

@@ -236,7 +236,8 @@ int starpu_data_cpy(starpu_data_handle dst_handle, starpu_data_handle src_handle
 #define STARPU_CALLBACK		(1<<5)	/* Callback function */
 #define STARPU_CALLBACK_ARG	(1<<6)	/* Argument of the callback function (of type void *) */
 #define STARPU_PRIORITY		(1<<7)	/* Priority associated to the task */
-#define STARPU_EXECUTE		(1<<8)	/* Used by MPI to define which task is going to execute the codelet */
+#define STARPU_EXECUTE_ON_NODE	(1<<8)	/* Used by MPI to define which task is going to execute the codelet */
+#define STARPU_EXECUTE_ON_DATA	(1<<9)	/* Used by MPI to define which task is going to execute the codelet */
 
 /* Wrapper to create a task. */
 int starpu_insert_task(starpu_codelet *cl, ...);

mpi/Makefile.am

@@ -162,6 +162,7 @@ check_PROGRAMS +=					\
 	tests/insert_task_block				\
 	tests/insert_task_owner				\
 	tests/insert_task_owner2			\
+	tests/insert_task_owner_data			\
 	tests/multiple_send
 
 testbin_PROGRAMS =				\
@@ -182,6 +183,7 @@ testbin_PROGRAMS =				\
 	tests/insert_task_block				\
 	tests/insert_task_owner				\
 	tests/insert_task_owner2			\
+	tests/insert_task_owner_data			\
 	tests/multiple_send
 
 tests_mpi_isend_LDADD =					\
@@ -286,6 +288,9 @@ tests_insert_task_owner2_LDADD =			\
 tests_insert_task_owner2_SOURCES =			\
 	tests/insert_task_owner2.c
 
+tests_insert_task_owner_data_LDADD =			\
+	libstarpumpi.la
+
 tests_multiple_send_LDADD =				\
 	libstarpumpi.la
 

mpi/starpu_mpi_insert_task.c

@@ -129,13 +129,17 @@ int starpu_mpi_insert_task(MPI_Comm comm, starpu_codelet *codelet, ...)
 	va_start(varg_list, codelet);
 	_starpu_pack_cl_args(arg_buffer_size, &arg_buffer, varg_list);
 
-        /* Finds out if the property STARPU_EXECUTE is specified */
+        /* Finds out if the property STARPU_EXECUTE_ON_NODE or STARPU_EXECUTE_ON_DATA is specified */
         execute = -1;
 	va_start(varg_list, codelet);
 	while ((arg_type = va_arg(varg_list, int)) != 0) {
-		if (arg_type==STARPU_EXECUTE) {
+		if (arg_type==STARPU_EXECUTE_ON_NODE) {
                         execute = va_arg(varg_list, int);
                 }
+		else if (arg_type==STARPU_EXECUTE_ON_DATA) {
+			starpu_data_handle data = va_arg(varg_list, starpu_data_handle);
+                        execute = starpu_data_get_rank(data);
+                }
 		else if (arg_type==STARPU_R || arg_type==STARPU_W || arg_type==STARPU_RW || arg_type == STARPU_SCRATCH) {
                         va_arg(varg_list, starpu_data_handle);
                 }
@@ -209,7 +213,10 @@ int starpu_mpi_insert_task(MPI_Comm comm, starpu_codelet *codelet, ...)
 		else if (arg_type==STARPU_PRIORITY) {
 			va_arg(varg_list, int);
 		}
-		else if (arg_type==STARPU_EXECUTE) {
+		else if (arg_type==STARPU_EXECUTE_ON_NODE) {
+			va_arg(varg_list, int);
+		}
+		else if (arg_type==STARPU_EXECUTE_ON_DATA) {
 			va_arg(varg_list, int);
 		}
 	}
@@ -227,7 +234,7 @@ int starpu_mpi_insert_task(MPI_Comm comm, starpu_codelet *codelet, ...)
                 }
         }
         else if (execute != -1) {
-                _STARPU_MPI_DEBUG("Property STARPU_EXECUTE ignored as W data are all owned by the same task\n");
+                _STARPU_MPI_DEBUG("Property STARPU_EXECUTE_ON_NODE or STARPU_EXECUTE_ON_DATA ignored as W data are all owned by the same task\n");
         }
 
         /* Send and receive data as requested */
@@ -289,9 +296,12 @@ int starpu_mpi_insert_task(MPI_Comm comm, starpu_codelet *codelet, ...)
 		else if (arg_type==STARPU_PRIORITY) {
 			va_arg(varg_list, int);
 		}
-		else if (arg_type==STARPU_EXECUTE) {
+		else if (arg_type==STARPU_EXECUTE_ON_NODE) {
 			va_arg(varg_list, int);
 		}
+		else if (arg_type==STARPU_EXECUTE_ON_DATA) {
+			va_arg(varg_list, starpu_data_handle);
+		}
         }
 	va_end(varg_list);
 
@@ -336,9 +346,12 @@ int starpu_mpi_insert_task(MPI_Comm comm, starpu_codelet *codelet, ...)
                         else if (arg_type==STARPU_PRIORITY) {
                                 va_arg(varg_list, int);
                         }
-                        else if (arg_type==STARPU_EXECUTE) {
+                        else if (arg_type==STARPU_EXECUTE_ON_NODE) {
                                 va_arg(varg_list, int);
                         }
+                        else if (arg_type==STARPU_EXECUTE_ON_DATA) {
+                                va_arg(varg_list, starpu_data_handle);
+                        }
                 }
                 va_end(varg_list);
         }
@@ -396,9 +409,12 @@ int starpu_mpi_insert_task(MPI_Comm comm, starpu_codelet *codelet, ...)
 		else if (arg_type==STARPU_PRIORITY) {
 			va_arg(varg_list, int);
 		}
-		else if (arg_type==STARPU_EXECUTE) {
+		else if (arg_type==STARPU_EXECUTE_ON_NODE) {
 			va_arg(varg_list, int);
 		}
+		else if (arg_type==STARPU_EXECUTE_ON_DATA) {
+			va_arg(varg_list, starpu_data_handle);
+		}
         }
 	va_end(varg_list);
         _STARPU_MPI_LOG_OUT();

mpi/tests/insert_task_owner.c

@@ -95,34 +95,34 @@ int main(int argc, char **argv)
         CHECK_RESULT;
         RELEASE_DATA;
 
-        err = starpu_mpi_insert_task(MPI_COMM_WORLD, &mycodelet, STARPU_RW, data_handlesx0, STARPU_RW, data_handlesx1, STARPU_EXECUTE, 1, 0);
+        err = starpu_mpi_insert_task(MPI_COMM_WORLD, &mycodelet, STARPU_RW, data_handlesx0, STARPU_RW, data_handlesx1, STARPU_EXECUTE_ON_NODE, 1, 0);
         assert(err == 0);
         ACQUIRE_DATA;
         vx0[0] ++ ; vx1[1] ++;
         CHECK_RESULT;
         RELEASE_DATA;
 
-        err = starpu_mpi_insert_task(MPI_COMM_WORLD, &mycodelet, STARPU_RW, data_handlesx0, STARPU_RW, data_handlesx1, STARPU_EXECUTE, 0, 0);
+        err = starpu_mpi_insert_task(MPI_COMM_WORLD, &mycodelet, STARPU_RW, data_handlesx0, STARPU_RW, data_handlesx1, STARPU_EXECUTE_ON_NODE, 0, 0);
         assert(err == 0);
         ACQUIRE_DATA;
         vx0[0] ++ ; vx1[1] ++;
         CHECK_RESULT;
         RELEASE_DATA;
 
-        /* Here the value specified by the property STARPU_EXECUTE is
+        /* Here the value specified by the property STARPU_EXECUTE_ON_NODE is
            going to be ignored as the data model clearly specifies
            which task is going to execute the codelet */
-        err = starpu_mpi_insert_task(MPI_COMM_WORLD, &mycodelet, STARPU_R, data_handlesx0, STARPU_W, data_handlesx1, STARPU_EXECUTE, 12, 0);
+        err = starpu_mpi_insert_task(MPI_COMM_WORLD, &mycodelet, STARPU_R, data_handlesx0, STARPU_W, data_handlesx1, STARPU_EXECUTE_ON_NODE, 12, 0);
         assert(err == 0);
         ACQUIRE_DATA;
         vx1[1] ++;
         CHECK_RESULT;
         RELEASE_DATA;
 
-        /* Here the value specified by the property STARPU_EXECUTE is
+        /* Here the value specified by the property STARPU_EXECUTE_ON_NODE is
            going to be ignored as the data model clearly specifies
            which task is going to execute the codelet */
-        err = starpu_mpi_insert_task(MPI_COMM_WORLD, &mycodelet, STARPU_W, data_handlesx0, STARPU_R, data_handlesx1, STARPU_EXECUTE, 11, 0);
+        err = starpu_mpi_insert_task(MPI_COMM_WORLD, &mycodelet, STARPU_W, data_handlesx0, STARPU_R, data_handlesx1, STARPU_EXECUTE_ON_NODE, 11, 0);
         assert(err == 0);
         ACQUIRE_DATA;
         vx0[0] ++;

mpi/tests/insert_task_owner2.c

@@ -86,7 +86,7 @@ int main(int argc, char **argv)
                                      STARPU_R, data_handles[0], STARPU_RW, data_handles[1],
                                      STARPU_W, data_handles[2],
                                      STARPU_W, data_handles[3],
-                                     STARPU_EXECUTE, 1, 0);
+                                     STARPU_EXECUTE_ON_NODE, 1, 0);
         assert(err == 0);
         starpu_task_wait_for_all();
 

src/util/starpu_insert_task_utils.c

@@ -69,9 +69,12 @@ size_t _starpu_insert_task_get_arg_size(va_list varg_list)
 		else if (arg_type==STARPU_PRIORITY) {
 			va_arg(varg_list, int);
 		}
-		else if (arg_type==STARPU_EXECUTE) {
+		else if (arg_type==STARPU_EXECUTE_ON_NODE) {
 			va_arg(varg_list, int);
 		}
+		else if (arg_type==STARPU_EXECUTE_ON_DATA) {
+			va_arg(varg_list, starpu_data_handle);
+		}
 	}
 
 	va_end(varg_list);
@@ -123,9 +126,12 @@ int _starpu_pack_cl_args(size_t arg_buffer_size, char **arg_buffer, va_list varg
 		{
 			va_arg(varg_list, int);
 		}
-		else if (arg_type==STARPU_EXECUTE) {
+		else if (arg_type==STARPU_EXECUTE_ON_NODE) {
 			va_arg(varg_list, int);
 		}
+		else if (arg_type==STARPU_EXECUTE_ON_DATA) {
+			va_arg(varg_list, starpu_data_handle);
+		}
 	}
 
 	(*arg_buffer)[0] = nargs;
@@ -178,9 +184,12 @@ int _starpu_insert_task_create_and_submit(char *arg_buffer, starpu_codelet *cl,
 			int prio = va_arg(varg_list, int); 
 			(*task)->priority = prio;
 		}
-		else if (arg_type==STARPU_EXECUTE) {
+		else if (arg_type==STARPU_EXECUTE_ON_NODE) {
 			va_arg(varg_list, int);
 		}
+		else if (arg_type==STARPU_EXECUTE_ON_DATA) {
+			va_arg(varg_list, starpu_data_handle);
+		}
 	}
 
 	va_end(varg_list);