mpi: add possibility to exchange user-defined data interface (this still needs to be documented)

mpi/starpu_mpi.c

@@ -57,19 +57,22 @@ static int posted_requests = 0, newer_requests, barrier_running = 0;
 
 static void starpu_mpi_isend_func(struct _starpu_mpi_req *req)
 {
-	int count=1, ret;
+	int count;
 
         _STARPU_MPI_LOG_IN();
 
-	void *ptr = starpu_mpi_handle_to_ptr(req->data_handle);
+	req->needs_unpacking = starpu_mpi_handle_to_datatype(req->data_handle, &req->datatype, &count);
+	if (req->needs_unpacking)
+		starpu_handle_pack_data(req->data_handle, &req->ptr);
+	else
+		req->ptr = starpu_handle_get_local_ptr(req->data_handle);
+	STARPU_ASSERT(req->ptr);
 
-        _STARPU_MPI_DEBUG("post MPI isend tag %d dst %d ptr %p req %p\n", req->mpi_tag, req->srcdst, ptr, &req->request);
-
-	starpu_mpi_handle_to_datatype(req->data_handle, &req->datatype);
+        _STARPU_MPI_DEBUG("post MPI isend tag %d dst %d ptr %p datatype %p count %d req %p\n", req->mpi_tag, req->srcdst, req->ptr, req->datatype, count, &req->request);
 
 	_starpu_mpi_comm_amounts_inc(req->comm, req->srcdst, req->datatype, count);
 
-        req->ret = MPI_Isend(ptr, count, req->datatype, req->srcdst, req->mpi_tag, req->comm, &req->request);
+        req->ret = MPI_Isend(req->ptr, count, req->datatype, req->srcdst, req->mpi_tag, req->comm, &req->request);
         STARPU_ASSERT(req->ret == MPI_SUCCESS);
 
 	TRACE_MPI_ISEND(req->srcdst, req->mpi_tag, 0);
@@ -155,16 +158,20 @@ int starpu_mpi_isend_detached(starpu_data_handle_t data_handle,
 
 static void starpu_mpi_irecv_func(struct _starpu_mpi_req *req)
 {
+	int count;
+
         _STARPU_MPI_LOG_IN();
-	int count=1;
-	void *ptr = starpu_mpi_handle_to_ptr(req->data_handle);
-	STARPU_ASSERT(ptr);
 
-	starpu_mpi_handle_to_datatype(req->data_handle, &req->datatype);
+	req->needs_unpacking = starpu_mpi_handle_to_datatype(req->data_handle, &req->datatype, &count);
+	if (req->needs_unpacking == 1)
+		req->ptr = malloc(count);
+	else
+		req->ptr = starpu_handle_get_local_ptr(req->data_handle);
+	STARPU_ASSERT(req->ptr);
 
-	_STARPU_MPI_DEBUG("post MPI irecv tag %d src %d data %p ptr %p req %p datatype %d\n", req->mpi_tag, req->srcdst, req->data_handle, ptr, &req->request, (int)req->datatype);
+	_STARPU_MPI_DEBUG("post MPI irecv tag %d src %d data %p ptr %p req %p datatype %p\n", req->mpi_tag, req->srcdst, req->data_handle, req->ptr, &req->request, req->datatype);
 
-        req->ret = MPI_Irecv(ptr, count, req->datatype, req->srcdst, req->mpi_tag, req->comm, &req->request);
+        req->ret = MPI_Irecv(req->ptr, count, req->datatype, req->srcdst, req->mpi_tag, req->comm, &req->request);
         STARPU_ASSERT(req->ret == MPI_SUCCESS);
 
 	/* somebody is perhaps waiting for the MPI request to be posted */
@@ -514,11 +521,13 @@ static void handle_request_termination(struct _starpu_mpi_req *req)
 
 	_STARPU_MPI_DEBUG("complete MPI (%s %d) data %p req %p - tag %d\n", starpu_mpi_request_type(req->request_type), req->srcdst, req->data_handle, &req->request, req->mpi_tag);
         if (req->request_type != BARRIER_REQ) {
-                MPI_Type_free(&req->datatype);
+		if (req->needs_unpacking)
+			starpu_handle_unpack_data(req->data_handle, req->ptr);
+		else
+			MPI_Type_free(&req->datatype);
                 starpu_data_release(req->data_handle);
         }
 
-
 	if (req->request_type == RECV_REQ)
 	{
 		TRACE_MPI_IRECV_END(req->srcdst, req->mpi_tag);

mpi/starpu_mpi_datatype.c

@@ -17,13 +17,7 @@
 
 #include <starpu_mpi_datatype.h>
 
-/*
- *	MPI_* functions usually requires both a pointer to the first element of
- *	a datatype and the datatype itself, so we need to provide both.
- */
-
 typedef int (*handle_to_datatype_func)(starpu_data_handle_t, MPI_Datatype *);
-typedef void *(*handle_to_ptr_func)(starpu_data_handle_t);
 
 /*
  * 	Matrix
@@ -133,21 +127,24 @@ static handle_to_datatype_func handle_to_datatype_funcs[STARPU_MAX_INTERFACE_ID]
 	[STARPU_MULTIFORMAT_INTERFACE_ID] = NULL,
 };
 
-
-int starpu_mpi_handle_to_datatype(starpu_data_handle_t data_handle, MPI_Datatype *datatype)
+int starpu_mpi_handle_to_datatype(starpu_data_handle_t data_handle, MPI_Datatype *datatype, int *count)
 {
 	enum starpu_data_interface_id id = starpu_handle_get_interface_id(data_handle);
 
-	STARPU_ASSERT_MSG(id <= STARPU_MULTIFORMAT_INTERFACE_ID, "Unknown data interface");
-
-	handle_to_datatype_func func = handle_to_datatype_funcs[id];
-
-	STARPU_ASSERT(func);
-
-	return func(data_handle, datatype);
-}
-
-void *starpu_mpi_handle_to_ptr(starpu_data_handle_t data_handle)
-{
-	return (void*) starpu_handle_get_local_ptr(data_handle);
+	if (id <= STARPU_MULTIFORMAT_INTERFACE_ID)
+	{
+		handle_to_datatype_func func = handle_to_datatype_funcs[id];
+		STARPU_ASSERT(func);
+		func(data_handle, datatype);
+		*count = 1;
+		return 0;
+	}
+	else
+	{
+		/* The datatype is not predefined by StarPU */
+		*count = starpu_handle_get_size(data_handle);
+		*datatype = MPI_BYTE;
+		return 1;
+	}
+	return 0;
 }

mpi/starpu_mpi_datatype.h

@@ -1,7 +1,7 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
  * Copyright (C) 2009-2011  Université de Bordeaux 1
- * Copyright (C) 2010  Centre National de la Recherche Scientifique
+ * Copyright (C) 2010, 2012  Centre National de la Recherche Scientifique
  *
  * StarPU is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
@@ -24,8 +24,7 @@
 extern "C" {
 #endif
 
-int starpu_mpi_handle_to_datatype(starpu_data_handle_t data_handle, MPI_Datatype *datatype);
-void *starpu_mpi_handle_to_ptr(starpu_data_handle_t data_handle);
+int starpu_mpi_handle_to_datatype(starpu_data_handle_t data_handle, MPI_Datatype *datatype, int *count);
 
 #ifdef __cplusplus
 }

mpi/starpu_mpi_private.h

@@ -63,6 +63,8 @@ LIST_TYPE(_starpu_mpi_req,
 
 	/* description of the data to be sent/received */
 	MPI_Datatype datatype;
+	void *ptr;
+	int needs_unpacking;
 
 	/* who are we talking to ? */
 	int srcdst;