Introduce the MPI helper lib (enabled with --with-mpi at configure time).

Makefile.am

@@ -16,7 +16,11 @@
 
 CLEANFILES = *.gcno *.gcda *.linkinfo
 
-SUBDIRS = src tools examples tests doc 
+SUBDIRS = src tools examples tests doc
+
+if USE_MPI
+SUBDIRS += mpi
+endif
 
 pkgconfigdir = $(libdir)/pkgconfig
 pkgconfig_DATA = libstarpu.pc

configure.ac

@@ -409,6 +409,39 @@ fi
 
 ###############################################################################
 #                                                                             #
+#                                    MPI                                      #
+#                                                                             #
+###############################################################################
+
+AC_ARG_WITH(mpi, [AS_HELP_STRING([--with-mpi[=<dir>]],
+			[Generate a library to use StarPU with MPI])],
+	[
+		if test x$withval != xno; then
+			use_mpi=yes
+			if test x$withval = xyes; then
+				# mpi option is enabled explicitely, but no dir
+				# is specified
+				use_default_mpi=yes
+			else
+				use_default_mpi=no
+				mpidir=$withval
+				AC_SUBST(MPIDIR, $mpidir)
+			fi
+		else
+			# explicitly without mpi
+			use_mpi=no
+		fi
+	],
+	[
+		use_mpi=no
+	])
+AC_MSG_CHECKING(whether mpi lib should be generated)
+AC_MSG_RESULT($use_mpi)
+AC_SUBST(USE_MPI, $use_mpi)
+AM_CONDITIONAL(USE_MPI, test x$use_mpi = xyes)
+
+###############################################################################
+#                                                                             #
 #                                  Examples                                   #
 #                                                                             #
 ###############################################################################
@@ -562,4 +595,5 @@ AC_OUTPUT([
 	examples/starpufft/Makefile
 	tests/Makefile
 	doc/Makefile
+	mpi/Makefile
 ])

mpi/Makefile.am

@@ -0,0 +1,45 @@
+#
+# StarPU
+# Copyright (C) INRIA 2008-2009 (see AUTHORS file)
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 2.1 of the License, or (at
+# your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See the GNU Lesser General Public License in COPYING.LGPL for more details.
+#
+
+CC=mpicc
+
+LIBS = $(top_builddir)/src/libstarpu.la @LIBS@
+AM_CPPFLAGS = -I$(top_srcdir)/include/ -I$(top_srcdir)/mpi/ -I$(top_srcdir)/src/
+
+lib_LTLIBRARIES = libstarpumpi.la
+
+libstarpumpi_la_SOURCES = starpu_mpi.c
+libstarpumpi_la_LIBADD = $(top_builddir)/src/libstarpu.la
+
+TESTS = $(check_PROGRAMS)
+
+check_PROGRAMS =
+
+check_PROGRAMS +=					\
+	tests/pingpong					\
+	tests/ring
+
+tests_pingpong_LDADD =					\
+	libstarpumpi.la
+
+tests_pingpong_SOURCES =				\
+	tests/pingpong.c
+
+tests_ring_LDADD =					\
+	libstarpumpi.la
+
+tests_ring_SOURCES =					\
+	tests/ring.c

mpi/starpu_mpi.c

@@ -0,0 +1,106 @@
+/*
+ * StarPU
+ * Copyright (C) INRIA 2008-2009 (see AUTHORS file)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <starpu_mpi.h>
+
+pthread_cond_t cond;
+pthread_mutex_t mutex;
+pthread_t progress_thread;
+
+static int handle_to_datatype(starpu_data_handle data_handle, MPI_Datatype *datatype)
+{
+	/* TODO we assume that we have a vector yet ! */
+
+	unsigned nx = starpu_get_vector_nx(data_handle);
+	size_t elemsize = starpu_get_vector_elemsize(data_handle);
+
+	MPI_Type_contiguous(nx*elemsize, MPI_BYTE, datatype);
+	MPI_Type_commit(datatype);
+
+	return 0;
+}
+
+int starpu_mpi_isend(starpu_data_handle data_handle, starpu_mpi_req_t *req,
+		int dest, int mpi_tag, MPI_Comm comm,
+		void (*callback)(void *))
+{
+	return 0;
+}
+
+int starpu_mpi_irecv(starpu_data_handle data_handle, starpu_mpi_req_t *req,
+		int source, int mpi_tag, MPI_Comm comm,
+		void (*callback)(void *))
+{
+	return 0;
+}
+
+int starpu_mpi_recv(starpu_data_handle data_handle,
+		int source, int mpi_tag, MPI_Comm comm)
+{
+	/* TODO test if we are blocking in a callback .. */
+
+	starpu_sync_data_with_mem(data_handle, STARPU_W);
+
+	void *ptr = (void *)starpu_get_vector_local_ptr(data_handle);
+	
+	MPI_Status status;
+	MPI_Datatype datatype;
+	handle_to_datatype(data_handle, &datatype);
+	MPI_Recv(ptr, 1, datatype, source, mpi_tag, comm, &status);
+
+	starpu_release_data_from_mem(data_handle);
+
+	return 0;
+}
+
+int starpu_mpi_send(starpu_data_handle data_handle,
+		int dest, int mpi_tag, MPI_Comm comm)
+{
+	/* TODO test if we are blocking in a callback .. */
+
+	starpu_sync_data_with_mem(data_handle, STARPU_R);
+
+	void *ptr = (void *)starpu_get_vector_local_ptr(data_handle);
+	
+	MPI_Status status;
+	MPI_Datatype datatype;
+	handle_to_datatype(data_handle, &datatype);
+	MPI_Send(ptr, 1, datatype, dest,  mpi_tag, comm);
+
+	starpu_release_data_from_mem(data_handle);
+
+	return 0;
+}
+
+int starpu_mpi_wait(starpu_mpi_req_t *req)
+{
+	return 0;
+}
+
+int starpu_mpi_test(starpu_mpi_req_t *req, int *flag)
+{
+	return 0;
+}
+
+int starpu_mpi_initialize(void)
+{
+	return 0;
+}
+
+int starpu_mpi_shutdown(void)
+{
+	return 0;
+}

mpi/starpu_mpi.h

@@ -0,0 +1,46 @@
+/*
+ * StarPU
+ * Copyright (C) INRIA 2008-2009 (see AUTHORS file)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#ifndef __STARPU_MPI_H__
+#define __STARPU_MPI_H__
+
+#include <starpu.h>
+#include <mpi.h>
+#include <common/list.h>
+#include <pthread.h>
+
+LIST_TYPE(starpu_mpi_req,
+	void *ptr;
+	MPI_Datatype datatype;
+	MPI_Request request;
+);
+
+int starpu_mpi_isend(starpu_data_handle data_handle, starpu_mpi_req_t *req,
+		int dest, int mpi_tag, MPI_Comm comm,
+		void (*callback)(void *));
+int starpu_mpi_irecv(starpu_data_handle data_handle, starpu_mpi_req_t *req,
+		int source, int mpi_tag, MPI_Comm comm,
+		void (*callback)(void *));
+int starpu_mpi_send(starpu_data_handle data_handle,
+		int dest, int mpi_tag, MPI_Comm comm);
+int starpu_mpi_recv(starpu_data_handle data_handle,
+		int source, int mpi_tag, MPI_Comm comm);
+int starpu_mpi_wait(starpu_mpi_req_t *req);
+int starpu_mpi_test(starpu_mpi_req_t *req, int *flag);
+int starpu_mpi_initialize(void);
+int starpu_mpi_shutdown(void);
+
+#endif // __STARPU_MPI_H__

mpi/tests/pingpong.c

@@ -0,0 +1,72 @@
+/*
+ * StarPU
+ * Copyright (C) INRIA 2008-2009 (see AUTHORS file)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <starpu_mpi.h>
+
+#define NITER	2048
+#define SIZE	16
+
+float *tab;
+starpu_data_handle tab_handle;
+
+int main(int argc, char **argv)
+{
+	MPI_Init(NULL, NULL);
+
+	int rank, size;
+
+	MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+	MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+	if (size != 2)
+	{
+		if (rank == 0)
+			fprintf(stderr, "We need exactly 2 processes.\n");
+
+		MPI_Finalize();
+		return 0;
+	}
+
+	starpu_init(NULL);
+	starpu_mpi_initialize();
+
+	tab = malloc(SIZE*sizeof(float));
+
+	starpu_register_vector_data(&tab_handle, 0, (uintptr_t)tab, SIZE, sizeof(float));
+
+	unsigned nloops = NITER;
+	unsigned loop;
+
+	int other_rank = (rank + 1)%2;
+
+	for (loop = 0; loop < nloops; loop++)
+	{
+		if ((loop % 2) == rank)
+		{
+			starpu_mpi_send(tab_handle, other_rank, loop, MPI_COMM_WORLD);
+		}
+		else {
+			starpu_mpi_recv(tab_handle, other_rank, loop, MPI_COMM_WORLD);
+		}
+	}
+	
+	starpu_mpi_shutdown();
+	starpu_shutdown();
+
+	MPI_Finalize();
+
+	return 0;
+}

mpi/tests/ring.c

@@ -0,0 +1,89 @@
+/*
+ * StarPU
+ * Copyright (C) INRIA 2008-2009 (see AUTHORS file)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <starpu_mpi.h>
+
+#define NITER	2048
+
+unsigned token = 42;
+starpu_data_handle token_handle;
+
+int main(int argc, char **argv)
+{
+	MPI_Init(NULL, NULL);
+
+	int rank, size;
+
+	MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+	MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+	if (size < 2)
+	{
+		if (rank == 0)
+			fprintf(stderr, "We need at least 2 processes.\n");
+
+		MPI_Finalize();
+		return 0;
+	}
+
+	starpu_init(NULL);
+	starpu_mpi_initialize();
+
+	starpu_register_vector_data(&token_handle, 0, (uintptr_t)&token, 1, sizeof(unsigned));
+
+	unsigned nloops = NITER;
+	unsigned loop;
+
+	unsigned last_loop = nloops - 1;
+	unsigned last_rank = size - 1;
+
+	for (loop = 0; loop < nloops; loop++)
+	{
+		int tag = loop*size + rank;
+
+		if (!((loop == 0) && (rank == 0)))
+		{
+			token = 0;
+			starpu_mpi_recv(token_handle, (rank+size-1)%size, tag, MPI_COMM_WORLD);
+		}
+		else {
+			token = 0;
+			fprintf(stdout, "Start with token value %d\n", token);
+		}
+
+		token += 1;
+		
+		if (!((loop == last_loop) && (rank == last_rank)))
+		{
+			starpu_mpi_send(token_handle, (rank+1)%size, tag+1, MPI_COMM_WORLD);
+		}
+		else {
+			fprintf(stdout, "Finished : token value %d\n", token);
+		}
+	}
+	
+	starpu_mpi_shutdown();
+	starpu_shutdown();
+
+	MPI_Finalize();
+
+	if (rank == last_rank)
+	{
+		STARPU_ASSERT(token == nloops*size);
+	}
+
+	return 0;
+}