mpi/examples/user_datatype: fix example, data was not managed properly

- make 2 distinct struct for the user data type and for its starpu data interface
- thanks to Philippe Swartvagher for his help in finding&fixing the bug

mpi/examples/user_datatype/my_interface.c

@@ -19,40 +19,40 @@
 
 #include "my_interface.h"
 
-void starpu_my_interface_display_codelet_cpu(void *descr[], void *_args)
+void starpu_my_data_display_codelet_cpu(void *descr[], void *_args)
 {
-	char c = STARPU_MY_INTERFACE_GET_CHAR(descr[0]);
-	int d = STARPU_MY_INTERFACE_GET_INT(descr[0]);
-	char msg[100];
+	char c = STARPU_MY_DATA_GET_CHAR(descr[0]);
+	int d = STARPU_MY_DATA_GET_INT(descr[0]);
+	char msg[100]="";
 
 	if (_args)
 		starpu_codelet_unpack_args(_args, &msg);
 
-	fprintf(stderr, "[%s] My value = '%c' %d\n", _args?msg:NULL, c, d);
+	fprintf(stderr, "[%s] My value = '%c' %d\n", msg, c, d);
 }
 
-void starpu_my_interface_compare_codelet_cpu(void *descr[], void *_args)
+void starpu_my_data_compare_codelet_cpu(void *descr[], void *_args)
 {
 	int *compare;
 
 	starpu_codelet_unpack_args(_args, &compare);
 
-	int d0 = STARPU_MY_INTERFACE_GET_INT(descr[0]);
-	char c0 = STARPU_MY_INTERFACE_GET_CHAR(descr[0]);
-	int d1 = STARPU_MY_INTERFACE_GET_INT(descr[1]);
-	char c1 = STARPU_MY_INTERFACE_GET_CHAR(descr[1]);
+	int d0 = STARPU_MY_DATA_GET_INT(descr[0]);
+	char c0 = STARPU_MY_DATA_GET_CHAR(descr[0]);
+	int d1 = STARPU_MY_DATA_GET_INT(descr[1]);
+	char c1 = STARPU_MY_DATA_GET_CHAR(descr[1]);
 
 	*compare = (d0 == d1 && c0 == c1);
 }
 
-void _starpu_my_interface_datatype_allocate(MPI_Datatype *mpi_datatype)
+void _starpu_my_data_datatype_allocate(MPI_Datatype *mpi_datatype)
 {
 	int ret;
 	int blocklengths[2] = {1, 1};
 	MPI_Aint displacements[2];
 	MPI_Datatype types[2] = {MPI_INT, MPI_CHAR};
-	struct starpu_my_interface *myinterface;
-	myinterface = malloc(sizeof(struct starpu_my_interface));
+	struct starpu_my_data *myinterface;
+	myinterface = malloc(sizeof(struct starpu_my_data));
 
 	MPI_Address(myinterface, displacements);
 	MPI_Address(&myinterface[0].c, displacements+1);
@@ -68,67 +68,98 @@ void _starpu_my_interface_datatype_allocate(MPI_Datatype *mpi_datatype)
 	free(myinterface);
 }
 
-void starpu_my_interface_datatype_allocate(starpu_data_handle_t handle, MPI_Datatype *mpi_datatype)
+void starpu_my_data_datatype_allocate(starpu_data_handle_t handle, MPI_Datatype *mpi_datatype)
 {
 	(void)handle;
-	_starpu_my_interface_datatype_allocate(mpi_datatype);
+	_starpu_my_data_datatype_allocate(mpi_datatype);
 }
 
-void starpu_my_interface_datatype_free(MPI_Datatype *mpi_datatype)
+void starpu_my_data_datatype_free(MPI_Datatype *mpi_datatype)
 {
 	MPI_Type_free(mpi_datatype);
 }
 
-int starpu_my_interface_get_int(starpu_data_handle_t handle)
+char starpu_my_data_interface_get_char(void *interface)
 {
-	struct starpu_my_interface *my_interface =
-		(struct starpu_my_interface *) starpu_data_get_interface_on_node(handle, STARPU_MAIN_RAM);
+	struct starpu_my_data_interface *my_data = (struct starpu_my_data_interface *) interface;
+	struct starpu_my_data *data = (struct starpu_my_data *)my_data->ptr;
+	return data->c;
+}
 
-	return my_interface->d;
+int starpu_my_data_interface_get_int(void *interface)
+{
+	struct starpu_my_data_interface *my_data = (struct starpu_my_data_interface *) interface;
+	struct starpu_my_data *data = (struct starpu_my_data *)my_data->ptr;
+	return data->d;
 }
 
-char starpu_my_interface_get_char(starpu_data_handle_t handle)
+int starpu_my_data_get_int(starpu_data_handle_t handle)
 {
-	struct starpu_my_interface *my_interface =
-		(struct starpu_my_interface *) starpu_data_get_interface_on_node(handle, STARPU_MAIN_RAM);
+	struct starpu_my_data_interface *my_data = (struct starpu_my_data_interface *) starpu_data_get_interface_on_node(handle, STARPU_MAIN_RAM);
+	struct starpu_my_data *data = (struct starpu_my_data *)my_data->ptr;
+	return data->d;
+}
 
-	return my_interface->c;
+char starpu_my_data_get_char(starpu_data_handle_t handle)
+{
+	struct starpu_my_data_interface *my_data = (struct starpu_my_data_interface *) starpu_data_get_interface_on_node(handle, STARPU_MAIN_RAM);
+	struct starpu_my_data *data = (struct starpu_my_data *)my_data->ptr;
+	return data->c;
 }
 
 static void data_register_data_handle(starpu_data_handle_t handle, unsigned home_node, void *data_interface)
 {
-	struct starpu_my_interface *my_interface = (struct starpu_my_interface *) data_interface;
+	struct starpu_my_data_interface *my_data_interface = (struct starpu_my_data_interface *) data_interface;
+
+	struct starpu_my_data *data = (struct starpu_my_data *)my_data_interface->ptr;
 
 	unsigned node;
 	for (node = 0; node < STARPU_MAXNODES; node++)
 	{
-		struct starpu_my_interface *local_interface = (struct starpu_my_interface *)
-			starpu_data_get_interface_on_node(handle, node);
+		struct starpu_my_data_interface *local_interface =
+			(struct starpu_my_data_interface *) starpu_data_get_interface_on_node(handle, node);
 
 		if (node == home_node)
 		{
-			local_interface->d = my_interface->d;
-			local_interface->c = my_interface->c;
+			local_interface->ptr = my_data_interface->ptr;
+                        local_interface->dev_handle = my_data_interface->dev_handle;
+                        local_interface->offset = my_data_interface->offset;
 		}
 		else
 		{
-			local_interface->d = 0;
-			local_interface->c = 0;
+			local_interface->ptr = 0;
+                        local_interface->dev_handle = 0;
+                        local_interface->offset = 0;
 		}
 	}
 }
 
 static starpu_ssize_t data_allocate_data_on_node(void *data_interface, unsigned node)
 {
-	(void)data_interface;
-	(void)node;
-	return 0;
+	uintptr_t addr = 0, handle;
+
+	struct starpu_my_data_interface *my_data_interface = (struct starpu_my_data_interface *) data_interface;
+
+	starpu_ssize_t allocated_memory = sizeof(int)+sizeof(char);
+	handle = starpu_malloc_on_node(node, allocated_memory);
+	if (!handle)
+		return -ENOMEM;
+
+	if (starpu_node_get_kind(node) != STARPU_OPENCL_RAM)
+		addr = handle;
+
+	/* update the data properly in consequence */
+	my_data_interface->ptr = addr;
+	my_data_interface->dev_handle = handle;
+        my_data_interface->offset = 0;
+
+	return allocated_memory;
 }
 
 static void data_free_data_on_node(void *data_interface, unsigned node)
 {
-	(void)data_interface;
-	(void)node;
+	struct starpu_my_data_interface *my_data_interface = (struct starpu_my_data_interface *) data_interface;
+	starpu_free_on_node(node, my_data_interface->dev_handle, sizeof(int)+sizeof(char));
 }
 
 static size_t data_get_size(starpu_data_handle_t handle)
@@ -137,9 +168,16 @@ static size_t data_get_size(starpu_data_handle_t handle)
 	return sizeof(int) + sizeof(char);
 }
 
+static size_t data_get_alloc_size(starpu_data_handle_t handle)
+{
+	(void)handle;
+	return sizeof(int) + sizeof(char);
+}
+
 static uint32_t data_footprint(starpu_data_handle_t handle)
 {
-	return starpu_hash_crc32c_be(starpu_my_interface_get_int(handle), 0);
+	struct starpu_my_data_interface *my_data = (struct starpu_my_data_interface *) starpu_data_get_interface_on_node(handle, STARPU_MAIN_RAM);
+	return starpu_hash_crc32c_be(my_data->ptr, 0);
 }
 
 static int data_pack_data(starpu_data_handle_t handle, unsigned node, void **ptr, starpu_ssize_t *count)
@@ -164,26 +202,30 @@ static int data_unpack_data(starpu_data_handle_t handle, unsigned node, void *pt
 
 static starpu_ssize_t data_describe(void *data_interface, char *buf, size_t size)
 {
-	struct starpu_my_interface *my_interface = (struct starpu_my_interface *) data_interface;
-	return snprintf(buf, size, "Data%d-%c", my_interface->d, my_interface->c);
+	struct starpu_my_data_interface *my_data = (struct starpu_my_data_interface *) data_interface;
+	struct starpu_my_data *data = (struct starpu_my_data *)my_data->ptr;
+	return snprintf(buf, size, "Data%d-%c", data->d, data->c);
 }
 
 static void *data_to_pointer(void *data_interface, unsigned node)
 {
 	(void) node;
-	struct starpu_my_interface *my_interface = data_interface;
+	struct starpu_my_data_interface *my_data_interface = data_interface;
 
-	return (void*) &my_interface->d;
+	return (void*) my_data_interface->ptr;
 }
 
 static int copy_any_to_any(void *src_interface, unsigned src_node,
 			   void *dst_interface, unsigned dst_node,
 			   void *async_data)
 {
-	struct starpu_my_interface *src_data = src_interface;
-	struct starpu_my_interface *dst_data = dst_interface;
+	assert(0);
+	struct starpu_my_data *src_data = src_interface;
+	struct starpu_my_data *dst_data = dst_interface;
 	int ret = 0;
 
+	fprintf(stderr, "copying data src_data.d=%d src_data.c %c\n", src_data->d, src_data->c);
+
 	if (starpu_interface_copy((uintptr_t) src_data->d, 0, src_node,
 				  (uintptr_t) dst_data->d, 0, dst_node,
 				  sizeof(src_data->d), async_data))
@@ -206,23 +248,32 @@ static struct starpu_data_interface_ops interface_data_ops =
 	.register_data_handle = data_register_data_handle,
 	.allocate_data_on_node = data_allocate_data_on_node,
 	.free_data_on_node = data_free_data_on_node,
-	.copy_methods = &data_copy_methods,
+	//	.copy_methods = &data_copy_methods,
 	.get_size = data_get_size,
+	.get_alloc_size = data_get_alloc_size,
 	.footprint = data_footprint,
 	.interfaceid = STARPU_UNKNOWN_INTERFACE_ID,
-	.interface_size = sizeof(struct starpu_my_interface),
+	.interface_size = sizeof(struct starpu_my_data_interface),
 	.to_pointer = data_to_pointer,
 	.pack_data = data_pack_data,
 	.unpack_data = data_unpack_data,
 	.describe = data_describe
 };
 
-void starpu_my_interface_data_register(starpu_data_handle_t *handleptr, unsigned home_node, struct starpu_my_interface *xc)
+void starpu_my_data_register(starpu_data_handle_t *handleptr, unsigned home_node, struct starpu_my_data *xc)
 {
 	if (interface_data_ops.interfaceid == STARPU_UNKNOWN_INTERFACE_ID)
 	{
 		interface_data_ops.interfaceid = starpu_data_interface_get_next_id();
 	}
 
-	starpu_data_register(handleptr, home_node, xc, &interface_data_ops);
+	struct starpu_my_data_interface data =
+	{
+	 	.id = interface_data_ops.interfaceid,
+		.ptr = (uintptr_t) xc,
+		.dev_handle = (uintptr_t) xc,
+		.offset = 0,
+	};
+
+	starpu_data_register(handleptr, home_node, &data, &interface_data_ops);
 }

mpi/examples/user_datatype/my_interface.h

@@ -1,6 +1,6 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2015,2017                                CNRS
+ * Copyright (C) 2015,2017,2019                           CNRS
  *
  * StarPU is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
@@ -20,43 +20,55 @@
 #ifndef __DATA_INTERFACE_H
 #define __DATA_INTERFACE_H
 
-struct starpu_my_interface
+struct starpu_my_data_interface
+{
+	enum starpu_data_interface_id id; /**< Identifier of the interface */
+
+	uintptr_t ptr;                    /**< local pointer of the data */
+	uintptr_t dev_handle;             /**< device handle of the data. */
+	size_t offset;                    /**< offset in the data */
+};
+
+struct starpu_my_data
 {
 	int d;
 	char c;
 };
 
-void starpu_my_interface_data_register(starpu_data_handle_t *handle, unsigned home_node, struct starpu_my_interface *xc);
+void starpu_my_data_register(starpu_data_handle_t *handle, unsigned home_node, struct starpu_my_data *xc);
+
+char starpu_my_data_get_char(starpu_data_handle_t handle);
+int starpu_my_data_get_int(starpu_data_handle_t handle);
 
-char starpu_my_interface_get_char(starpu_data_handle_t handle);
-int starpu_my_interface_get_int(starpu_data_handle_t handle);
+char starpu_my_data_interface_get_char(void *interface);
+int starpu_my_data_interface_get_int(void *interface);
 
-#define STARPU_MY_INTERFACE_GET_CHAR(interface)	(((struct starpu_my_interface *)(interface))->c)
-#define STARPU_MY_INTERFACE_GET_INT(interface)	(((struct starpu_my_interface *)(interface))->d)
+#define STARPU_MY_DATA_GET_CHAR(interface)	starpu_my_data_interface_get_char(interface)
+#define STARPU_MY_DATA_GET_INT(interface)	starpu_my_data_interface_get_int(interface)
 
-void _starpu_my_interface_datatype_allocate(MPI_Datatype *mpi_datatype);
-void starpu_my_interface_datatype_allocate(starpu_data_handle_t handle, MPI_Datatype *mpi_datatype);
-void starpu_my_interface_datatype_free(MPI_Datatype *mpi_datatype);
+void _starpu_my_data_datatype_allocate(MPI_Datatype *mpi_datatype);
+void starpu_my_data_datatype_allocate(starpu_data_handle_t handle, MPI_Datatype *mpi_datatype);
+void starpu_my_data_datatype_free(MPI_Datatype *mpi_datatype);
 
-void starpu_my_interface_display_codelet_cpu(void *descr[], void *_args);
-void starpu_my_interface_compare_codelet_cpu(void *descr[], void *_args);
+void starpu_my_data_display_codelet_cpu(void *descr[], void *_args);
+void starpu_my_data_compare_codelet_cpu(void *descr[], void *_args);
 
-static struct starpu_codelet starpu_my_interface_display_codelet =
+static struct starpu_codelet starpu_my_data_display_codelet =
 {
-	.cpu_funcs = {starpu_my_interface_display_codelet_cpu},
-	.cpu_funcs_name = {"starpu_my_interface_display_codelet_cpu"},
+	.cpu_funcs = {starpu_my_data_display_codelet_cpu},
+	.cpu_funcs_name = {"starpu_my_data_display_codelet_cpu"},
 	.nbuffers = 1,
 	.modes = {STARPU_R},
-	.name = "starpu_my_interface_display_codelet"
+	.name = "starpu_my_data_display_codelet"
 };
 
-static struct starpu_codelet starpu_my_interface_compare_codelet =
+static struct starpu_codelet starpu_my_data_compare_codelet =
 {
-	.cpu_funcs = {starpu_my_interface_compare_codelet_cpu},
-	.cpu_funcs_name = {"starpu_my_interface_compare_codelet_cpu"},
+	.cpu_funcs = {starpu_my_data_compare_codelet_cpu},
+	.cpu_funcs_name = {"starpu_my_data_compare_codelet_cpu"},
 	.nbuffers = 2,
 	.modes = {STARPU_R, STARPU_R},
-	.name = "starpu_my_interface_compare_codelet"
+	.name = "starpu_my_data_compare_codelet"
 };
 
 #endif /* __MY_INTERFACE_H */

mpi/examples/user_datatype/user_datatype.c

@@ -1,6 +1,6 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2015-2017                                CNRS
+ * Copyright (C) 2015-2017, 2019                          CNRS
  * Copyright (C) 2018                                     Université de Bordeaux
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -26,12 +26,6 @@ int main(int argc, char **argv)
 	int ret=0;
 	int compare=0;
 
-	struct starpu_my_interface my1 = {.d = 98 , .c = 'z'};
-	struct starpu_my_interface my0 = {.d = 42 , .c = 'n'};
-
-	starpu_data_handle_t handle0;
-	starpu_data_handle_t handle1;
-
 	ret = starpu_mpi_init_conf(&argc, &argv, 1, MPI_COMM_WORLD, NULL);
 	STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init_conf");
 	starpu_mpi_comm_rank(MPI_COMM_WORLD, &rank);
@@ -50,50 +44,79 @@ int main(int argc, char **argv)
 		return 77;
 	}
 
+	struct starpu_my_data my0 = {.d = 42 , .c = 'n'};
+	struct starpu_my_data my1 = {.d = 98 , .c = 'z'};
+
+	starpu_data_handle_t handle0;
+	starpu_data_handle_t handle1;
+
 	if (rank == 1)
 	{
 		my0.d = 0;
 		my0.c = 'z';
 	}
-	starpu_my_interface_data_register(&handle0, STARPU_MAIN_RAM, &my0);
-	starpu_my_interface_data_register(&handle1, -1, &my1);
-	starpu_mpi_datatype_register(handle1, starpu_my_interface_datatype_allocate, starpu_my_interface_datatype_free);
+
+	starpu_my_data_register(&handle0, STARPU_MAIN_RAM, &my0);
+	starpu_my_data_register(&handle1, -1, &my1);
+	starpu_mpi_datatype_register(handle1, starpu_my_data_datatype_allocate, starpu_my_data_datatype_free);
 
 	starpu_mpi_barrier(MPI_COMM_WORLD);
 
+	// Send data directly with MPI
 	if (rank == 0)
 	{
 		MPI_Datatype mpi_datatype;
-		_starpu_my_interface_datatype_allocate(&mpi_datatype);
+		_starpu_my_data_datatype_allocate(&mpi_datatype);
 		MPI_Send(&my0, 1, mpi_datatype, 1, 42, MPI_COMM_WORLD);
-		starpu_my_interface_datatype_free(&mpi_datatype);
+		starpu_my_data_datatype_free(&mpi_datatype);
 	}
 	else if (rank == 1)
 	{
 		MPI_Datatype mpi_datatype;
 		MPI_Status status;
-		_starpu_my_interface_datatype_allocate(&mpi_datatype);
-		MPI_Recv(&my0, 1, mpi_datatype, 0, 42, MPI_COMM_WORLD, &status);
-		FPRINTF(stderr, "Received value: '%c' %d\n", my0.c, my0.d);
-		starpu_my_interface_datatype_free(&mpi_datatype);
+		struct starpu_my_data myx;
+		_starpu_my_data_datatype_allocate(&mpi_datatype);
+		MPI_Recv(&myx, 1, mpi_datatype, 0, 42, MPI_COMM_WORLD, &status);
+		FPRINTF(stderr, "[mpi] Received value: '%c' %d\n", myx.c, myx.d);
+		starpu_my_data_datatype_free(&mpi_datatype);
+		STARPU_ASSERT_MSG(myx.d == 42 && myx.c == 'n', "Incorrect received value\n");
+	}
+
+	if (rank == 0)
+	{
+		struct starpu_my_data myx = {.d = 98 , .c = 'z'};
+		starpu_data_handle_t handlex;
+		starpu_my_data_register(&handlex, STARPU_MAIN_RAM, &myx);
+		starpu_mpi_send(handlex, 1, 10, MPI_COMM_WORLD);
+		starpu_data_unregister(handlex);
+	}
+	else if (rank == 1)
+	{
+		struct starpu_my_data myx = {.d = 11 , .c = 'a'};
+		starpu_data_handle_t handlex;
+		starpu_my_data_register(&handlex, STARPU_MAIN_RAM, &myx);
+		starpu_mpi_recv(handlex, 0, 10, MPI_COMM_WORLD, NULL);
+		starpu_data_unregister(handlex);
+		FPRINTF(stderr, "[starpu mpi] myx.d=%d myx.c=%c\n", myx.d, myx.c);
+		STARPU_ASSERT_MSG(myx.d == 98 && myx.c == 'z', "Incorrect received value\n");
 	}
 
 	if (rank == 0)
 	{
 		int *compare_ptr = &compare;
 
-		starpu_task_insert(&starpu_my_interface_display_codelet, STARPU_VALUE, "node0 initial value", strlen("node0 initial value")+1, STARPU_R, handle0, 0);
+		starpu_task_insert(&starpu_my_data_display_codelet, STARPU_VALUE, "node0 initial value", strlen("node0 initial value")+1, STARPU_R, handle0, 0);
 		starpu_mpi_isend_detached(handle0, 1, 10, MPI_COMM_WORLD, NULL, NULL);
 		starpu_mpi_irecv_detached(handle1, 1, 20, MPI_COMM_WORLD, NULL, NULL);
 
-		starpu_task_insert(&starpu_my_interface_display_codelet, STARPU_VALUE, "node0 received value", strlen("node0 received value")+1, STARPU_R, handle1, 0);
-		starpu_task_insert(&starpu_my_interface_compare_codelet, STARPU_R, handle0, STARPU_R, handle1, STARPU_VALUE, &compare_ptr, sizeof(compare_ptr), 0);
+		starpu_task_insert(&starpu_my_data_display_codelet, STARPU_VALUE, "node0 received value", strlen("node0 received value")+1, STARPU_R, handle1, 0);
+		starpu_task_insert(&starpu_my_data_compare_codelet, STARPU_R, handle0, STARPU_R, handle1, STARPU_VALUE, &compare_ptr, sizeof(compare_ptr), 0);
 	}
 	else if (rank == 1)
 	{
-		starpu_task_insert(&starpu_my_interface_display_codelet, STARPU_VALUE, "node1 initial value", strlen("node1 initial value")+1, STARPU_R, handle0, 0);
+		starpu_task_insert(&starpu_my_data_display_codelet, STARPU_VALUE, "node1 initial value", strlen("node1 initial value")+1, STARPU_R, handle0, 0);
 		starpu_mpi_irecv_detached(handle0, 0, 10, MPI_COMM_WORLD, NULL, NULL);
-		starpu_task_insert(&starpu_my_interface_display_codelet, STARPU_VALUE, "node1 received value", strlen("node1 received value")+1, STARPU_R, handle0, 0);
+		starpu_task_insert(&starpu_my_data_display_codelet, STARPU_VALUE, "node1 received value", strlen("node1 received value")+1, STARPU_R, handle0, 0);
 		starpu_mpi_isend_detached(handle0, 0, 20, MPI_COMM_WORLD, NULL, NULL);
 	}