MPI: The tag used in mpi communications is set by the data handle and no longer internally by the starpumpi library

mpi/examples/cholesky/mpi_cholesky.c

@@ -96,7 +96,10 @@ static void dw_cholesky(float ***matA, unsigned size, unsigned ld, unsigned nblo
                                                             ld, size/nblocks, size/nblocks, sizeof(float));
                         }
                         if (data_handles[x][y])
+			{
                                 starpu_data_set_rank(data_handles[x][y], mpi_rank);
+                                starpu_data_set_tag(data_handles[x][y], (y*nblocks)+x);
+			}
                 }
         }
 

mpi/examples/cholesky/mpi_cholesky_distributed.c

@@ -96,7 +96,10 @@ static void dw_cholesky(float ***matA, unsigned size, unsigned ld, unsigned nblo
                                                             ld, size/nblocks, size/nblocks, sizeof(float));
                         }
                         if (data_handles[x][y])
+			{
                                 starpu_data_set_rank(data_handles[x][y], mpi_rank);
+                                starpu_data_set_tag(data_handles[x][y], (y*nblocks)+x);
+			}
                 }
         }
 

mpi/examples/scatter_gather/mpi_scatter_gather.c

@@ -143,6 +143,7 @@ int main(int argc, char **argv)
                         if (data_handles[x+y*nblocks])
 			{
                                 starpu_data_set_rank(data_handles[x+y*nblocks], mpi_rank);
+                                starpu_data_set_tag(data_handles[x+y*nblocks], (y*nblocks)+x);
 			}
                 }
         }

mpi/examples/stencil/stencil5.c

@@ -102,7 +102,10 @@ int main(int argc, char **argv)
                                 data_handles[x][y] = NULL;
                         }
                         if (data_handles[x][y])
+			{
                                 starpu_data_set_rank(data_handles[x][y], mpi_rank);
+                                starpu_data_set_tag(data_handles[x][y], (y*X)+x);
+			}
                 }
         }
 

mpi/starpu_mpi_collective.c

@@ -22,7 +22,6 @@ int starpu_mpi_scatter_detached(starpu_data_handle *data_handles, int count, int
 {
 	int rank;
 	int x;
-	int mpi_tag = 0;
 
 	MPI_Comm_rank(comm, &rank);
 
@@ -31,6 +30,8 @@ int starpu_mpi_scatter_detached(starpu_data_handle *data_handles, int count, int
 		if (data_handles[x])
 		{
 			int owner = starpu_data_get_rank(data_handles[x]);
+			int mpi_tag = starpu_data_get_tag(data_handles[x]);
+			STARPU_ASSERT(mpi_tag >= 0);
 			if ((rank == root) && (owner != root))
 			{
 				//fprintf(stderr, "[%d] Sending data[%d] to %d\n", rank, x, owner);
@@ -39,11 +40,9 @@ int starpu_mpi_scatter_detached(starpu_data_handle *data_handles, int count, int
 			if ((rank != root) && (owner == rank))
 			{
 				//fprintf(stderr, "[%d] Receiving data[%d] from %d\n", rank, x, root);
-				//MPI_Status status;
 				starpu_mpi_irecv_detached(data_handles[x], root, mpi_tag, comm, NULL, NULL);
 			}
 		}
-		mpi_tag++;
 	}
 	return 0;
 }
@@ -52,7 +51,6 @@ int starpu_mpi_gather_detached(starpu_data_handle *data_handles, int count, int
 {
 	int rank;
 	int x;
-	int mpi_tag = 0;
 
 	MPI_Comm_rank(comm, &rank);
 
@@ -61,10 +59,11 @@ int starpu_mpi_gather_detached(starpu_data_handle *data_handles, int count, int
 		if (data_handles[x])
 		{
 			int owner = starpu_data_get_rank(data_handles[x]);
+			int mpi_tag = starpu_data_get_tag(data_handles[x]);
+			STARPU_ASSERT(mpi_tag >= 0);
 			if ((rank == root) && (owner != root))
 			{
 				//fprintf(stderr, "[%d] Receiving data[%d] from %d\n", rank, x, owner);
-				//MPI_Status status;
 				starpu_mpi_irecv_detached(data_handles[x], owner, mpi_tag, comm, NULL, NULL);
 			}
 			if ((rank != root) && (owner == rank))
@@ -73,7 +72,6 @@ int starpu_mpi_gather_detached(starpu_data_handle *data_handles, int count, int
 				starpu_mpi_isend_detached(data_handles[x], root, mpi_tag, comm, NULL, NULL);
 			}
 		}
-		mpi_tag ++;
 	}
 	return 0;
 }

mpi/starpu_mpi_insert_task.c

@@ -108,7 +108,6 @@ int starpu_mpi_insert_task(MPI_Comm comm, starpu_codelet *codelet, ...)
 	size_t arg_buffer_size = 0;
 	char *arg_buffer;
         int dest=0, execute, inconsistent_execute;
-        int mpi_tag = 100;
 
         _STARPU_MPI_LOG_IN();
 
@@ -244,6 +243,8 @@ int starpu_mpi_insert_task(MPI_Comm comm, starpu_codelet *codelet, ...)
                         starpu_data_handle data = va_arg(varg_list, starpu_data_handle);
                         if (data && arg_type & STARPU_R) {
                                 int mpi_rank = starpu_data_get_rank(data);
+				int mpi_tag = starpu_data_get_tag(data);
+				STARPU_ASSERT(mpi_tag >= 0);
                                 /* The task needs to read this data */
                                 if (do_execute && mpi_rank != me && mpi_rank != -1) {
                                         /* I will have to execute but I don't have the data, receive */
@@ -258,10 +259,10 @@ int starpu_mpi_insert_task(MPI_Comm comm, starpu_codelet *codelet, ...)
                                         }
                                         if (!already_received)
 #endif
-                                                {
-                                                        _STARPU_MPI_DEBUG("Receive data %p from %d\n", data, mpi_rank);
-                                                        starpu_mpi_irecv_detached(data, mpi_rank, mpi_tag, comm, NULL, NULL);
-                                                }
+					{
+						_STARPU_MPI_DEBUG("Receive data %p from %d\n", data, mpi_rank);
+						starpu_mpi_irecv_detached(data, mpi_rank, mpi_tag, comm, NULL, NULL);
+					}
                                 }
                                 if (!do_execute && mpi_rank == me) {
                                         /* Somebody else will execute it, and I have the data, send it. */
@@ -276,12 +277,11 @@ int starpu_mpi_insert_task(MPI_Comm comm, starpu_codelet *codelet, ...)
                                         }
                                         if (!already_sent)
 #endif
-                                                {
-                                                        _STARPU_MPI_DEBUG("Send data %p to %d\n", data, dest);
-                                                        starpu_mpi_isend_detached(data, dest, mpi_tag, comm, NULL, NULL);
-                                                }
+					{
+						_STARPU_MPI_DEBUG("Send data %p to %d\n", data, dest);
+						starpu_mpi_isend_detached(data, dest, mpi_tag, comm, NULL, NULL);
+					}
                                 }
-                                mpi_tag++;
                         }
                 }
 		else if (arg_type==STARPU_VALUE) {
@@ -321,6 +321,8 @@ int starpu_mpi_insert_task(MPI_Comm comm, starpu_codelet *codelet, ...)
                                 starpu_data_handle data = va_arg(varg_list, starpu_data_handle);
                                 if (arg_type & STARPU_W) {
                                         int mpi_rank = starpu_data_get_rank(data);
+					int mpi_tag = starpu_data_get_tag(data);
+					STARPU_ASSERT(mpi_tag >= 0);
                                         if (mpi_rank == me) {
                                                 if (execute != -1 && me != execute) {
                                                         _STARPU_MPI_DEBUG("Receive data %p back from the task %d which executed the codelet ...\n", data, dest);
@@ -331,7 +333,6 @@ int starpu_mpi_insert_task(MPI_Comm comm, starpu_codelet *codelet, ...)
                                                 _STARPU_MPI_DEBUG("Send data %p back to its owner %d...\n", data, mpi_rank);
                                                 starpu_mpi_isend_detached(data, mpi_rank, mpi_tag, comm, NULL, NULL);
                                         }
-                                        mpi_tag ++;
                                 }
                         }
                         else if (arg_type==STARPU_VALUE) {

mpi/tests/insert_task.c

@@ -84,7 +84,10 @@ int main(int argc, char **argv)
                                 data_handles[x][y] = NULL;
                         }
                         if (data_handles[x][y])
+			{
                                 starpu_data_set_rank(data_handles[x][y], mpi_rank);
+                                starpu_data_set_tag(data_handles[x][y], (y*X)+x);
+			}
                 }
         }
 

mpi/tests/insert_task_block.c

@@ -99,7 +99,10 @@ int main(int argc, char **argv)
                                 data_handles[x][y] = NULL;
                         }
                         if (data_handles[x][y])
+			{
                                 starpu_data_set_rank(data_handles[x][y], mpi_rank);
+                                starpu_data_set_tag(data_handles[x][y], (y*BLOCKS)+x);
+			}
                 }
         }
 

mpi/tests/insert_task_cache.c

@@ -84,7 +84,10 @@ int main(int argc, char **argv)
                                 data_handles[x][y] = NULL;
                         }
                         if (data_handles[x][y])
+			{
                                 starpu_data_set_rank(data_handles[x][y], mpi_rank);
+                                starpu_data_set_tag(data_handles[x][y], (y*X)+x);
+			}
                 }
         }
 

mpi/tests/insert_task_owner.c

@@ -65,14 +65,18 @@ int main(int argc, char **argv)
         if (rank == 0) {
                 starpu_variable_data_register(&data_handlesx0, 0, (uintptr_t)&x0, sizeof(x0));
                 starpu_data_set_rank(data_handlesx0, rank);
+		starpu_data_set_tag(data_handlesx0, 0);
                 starpu_variable_data_register(&data_handlesx1, -1, (uintptr_t)NULL, sizeof(int));
                 starpu_data_set_rank(data_handlesx1, 1);
+		starpu_data_set_tag(data_handlesx1, 1);
         }
         else if (rank == 1) {
                 starpu_variable_data_register(&data_handlesx1, 0, (uintptr_t)&x1, sizeof(x1));
                 starpu_data_set_rank(data_handlesx1, rank);
+		starpu_data_set_tag(data_handlesx1, 1);
                 starpu_variable_data_register(&data_handlesx0, -1, (uintptr_t)NULL, sizeof(int));
                 starpu_data_set_rank(data_handlesx0, 0);
+		starpu_data_set_tag(data_handlesx0, 0);
         }
 
         err = starpu_mpi_insert_task(MPI_COMM_WORLD, &mycodelet, STARPU_R, data_handlesx0, STARPU_W, data_handlesx1, 0);

mpi/tests/insert_task_owner2.c

@@ -65,20 +65,24 @@ int main(int argc, char **argv)
                         x[i] = 10*(i+1);
                         starpu_variable_data_register(&data_handles[i], 0, (uintptr_t)&x[i], sizeof(x[i]));
                         starpu_data_set_rank(data_handles[i], rank);
+			starpu_data_set_tag(data_handles[i], i);
                 }
                 y = -1;
                 starpu_variable_data_register(&data_handles[3], -1, (uintptr_t)NULL, sizeof(int));
                 starpu_data_set_rank(data_handles[3], 1);
+		starpu_data_set_tag(data_handles[3], 3);
         }
         else if (rank == 1) {
                 for(i=0 ; i<3 ; i++) {
                         x[i] = -1;
                         starpu_variable_data_register(&data_handles[i], -1, (uintptr_t)NULL, sizeof(int));
                         starpu_data_set_rank(data_handles[i], 0);
+			starpu_data_set_tag(data_handles[i], i);
                 }
                 y=200;
                 starpu_variable_data_register(&data_handles[3], 0, (uintptr_t)&y, sizeof(int));
                 starpu_data_set_rank(data_handles[3], rank);
+		starpu_data_set_tag(data_handles[3], 3);
         }
         fprintf(stderr, "[%d][init] VALUES: %d %d %d %d\n", rank, x[0], x[1], x[2], y);
 

mpi/tests/insert_task_owner_data.c

@@ -54,15 +54,19 @@ int main(int argc, char **argv)
 		x[0] = 11;
 		starpu_variable_data_register(&data_handles[0], 0, (uintptr_t)&x[0], sizeof(x[0]));
 		starpu_data_set_rank(data_handles[0], 0);
+		starpu_data_set_tag(data_handles[0], 0);
 		starpu_variable_data_register(&data_handles[1], -1, (uintptr_t)NULL, sizeof(x[1]));
 		starpu_data_set_rank(data_handles[1], 1);
+		starpu_data_set_tag(data_handles[1],10);
         }
         else if (rank == 1) {
 		x[1] = 12;
 		starpu_variable_data_register(&data_handles[0], -1, (uintptr_t)NULL, sizeof(x[0]));
 		starpu_data_set_rank(data_handles[0], 0);
+		starpu_data_set_tag(data_handles[0], 0);
 		starpu_variable_data_register(&data_handles[1], 0, (uintptr_t)&x[1], sizeof(x[1]));
 		starpu_data_set_rank(data_handles[1], 1);
+		starpu_data_set_tag(data_handles[1], 1);
         }
 
         err = starpu_mpi_insert_task(MPI_COMM_WORLD, &mycodelet,