mpi-support.texi 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571
  1. @c -*-texinfo-*-
  2. @c This file is part of the StarPU Handbook.
  3. @c Copyright (C) 2009--2011 Universit@'e de Bordeaux 1
  4. @c Copyright (C) 2010, 2011, 2012 Centre National de la Recherche Scientifique
  5. @c Copyright (C) 2011 Institut National de Recherche en Informatique et Automatique
  6. @c See the file starpu.texi for copying conditions.
  7. The integration of MPI transfers within task parallelism is done in a
  8. very natural way by the means of asynchronous interactions between the
  9. application and StarPU. This is implemented in a separate libstarpumpi library
  10. which basically provides "StarPU" equivalents of @code{MPI_*} functions, where
  11. @code{void *} buffers are replaced with @code{starpu_data_handle_t}s, and all
  12. GPU-RAM-NIC transfers are handled efficiently by StarPU-MPI. The user has to
  13. use the usual @code{mpirun} command of the MPI implementation to start StarPU on
  14. the different MPI nodes.
  15. An MPI Insert Task function provides an even more seamless transition to a
  16. distributed application, by automatically issuing all required data transfers
  17. according to the task graph and an application-provided distribution.
  18. @menu
  19. * The API::
  20. * Simple Example::
  21. * Exchanging User Defined Data Interface::
  22. * MPI Insert Task Utility::
  23. * MPI Collective Operations::
  24. @end menu
  25. @node The API
  26. @section The API
  27. @subsection Compilation
  28. The flags required to compile or link against the MPI layer are then
  29. accessible with the following commands:
  30. @example
  31. % pkg-config --cflags starpumpi-1.0 # options for the compiler
  32. % pkg-config --libs starpumpi-1.0 # options for the linker
  33. @end example
  34. Also pass the @code{--static} option if the application is to be linked statically.
  35. @subsection Initialisation
  36. @deftypefun int starpu_mpi_init (int *@var{argc}, char ***@var{argv}, int initialize_mpi)
  37. Initializes the starpumpi library. @code{initialize_mpi} indicates if
  38. MPI should be initialized or not by StarPU. If the value is not @code{0},
  39. MPI will be initialized by calling @code{MPI_Init_Thread(argc, argv,
  40. MPI_THREAD_SERIALIZED, ...)}.
  41. @end deftypefun
  42. @deftypefun int starpu_mpi_initialize (void)
  43. This function has been made deprecated. One should use instead the
  44. function @code{starpu_mpi_init()} defined above.
  45. This function does not call @code{MPI_Init}, it should be called beforehand.
  46. @end deftypefun
  47. @deftypefun int starpu_mpi_initialize_extended (int *@var{rank}, int *@var{world_size})
  48. This function has been made deprecated. One should use instead the
  49. function @code{starpu_mpi_init()} defined above.
  50. MPI will be initialized by starpumpi by calling @code{MPI_Init_Thread(argc, argv,
  51. MPI_THREAD_SERIALIZED, ...)}.
  52. @end deftypefun
  53. @deftypefun int starpu_mpi_shutdown (void)
  54. Cleans the starpumpi library. This must be called between calling
  55. @code{starpu_mpi} functions and @code{starpu_shutdown()}.
  56. @code{MPI_Finalize()} will be called if StarPU-MPI has been initialized
  57. by @code{starpu_mpi_init()}.
  58. @end deftypefun
  59. @deftypefun void starpu_mpi_comm_amounts_retrieve (size_t *@var{comm_amounts})
  60. Retrieve the current amount of communications from the current node in
  61. the array @code{comm_amounts} which must have a size greater or equal
  62. to the world size. Communications statistics must be enabled
  63. (@pxref{STARPU_COMM_STATS}).
  64. @end deftypefun
  65. @subsection Communication
  66. The standard point to point communications of MPI have been
  67. implemented. The semantic is similar to the MPI one, but adapted to
  68. the DSM provided by StarPU. A MPI request will only be submitted when
  69. the data is available in the main memory of the node submitting the
  70. request.
  71. @deftypefun int starpu_mpi_send (starpu_data_handle_t @var{data_handle}, int @var{dest}, int @var{mpi_tag}, MPI_Comm @var{comm})
  72. Performs a standard-mode, blocking send of @var{data_handle} to the
  73. node @var{dest} using the message tag @code{mpi_tag} within the
  74. communicator @var{comm}.
  75. @end deftypefun
  76. @deftypefun int starpu_mpi_recv (starpu_data_handle_t @var{data_handle}, int @var{source}, int @var{mpi_tag}, MPI_Comm @var{comm}, MPI_Status *@var{status})
  77. Performs a standard-mode, blocking receive in @var{data_handle} from the
  78. node @var{source} using the message tag @code{mpi_tag} within the
  79. communicator @var{comm}.
  80. @end deftypefun
  81. @deftypefun int starpu_mpi_isend (starpu_data_handle_t @var{data_handle}, starpu_mpi_req *@var{req}, int @var{dest}, int @var{mpi_tag}, MPI_Comm @var{comm})
  82. Posts a standard-mode, non blocking send of @var{data_handle} to the
  83. node @var{dest} using the message tag @code{mpi_tag} within the
  84. communicator @var{comm}. After the call, the pointer to the request
  85. @var{req} can be used to test the completion of the communication.
  86. @end deftypefun
  87. @deftypefun int starpu_mpi_irecv (starpu_data_handle_t @var{data_handle}, starpu_mpi_req *@var{req}, int @var{source}, int @var{mpi_tag}, MPI_Comm @var{comm})
  88. Posts a nonblocking receive in @var{data_handle} from the
  89. node @var{source} using the message tag @code{mpi_tag} within the
  90. communicator @var{comm}. After the call, the pointer to the request
  91. @var{req} can be used to test the completion of the communication.
  92. @end deftypefun
  93. @deftypefun int starpu_mpi_isend_detached (starpu_data_handle_t @var{data_handle}, int @var{dest}, int @var{mpi_tag}, MPI_Comm @var{comm}, void (*@var{callback})(void *), void *@var{arg})
  94. Posts a standard-mode, non blocking send of @var{data_handle} to the
  95. node @var{dest} using the message tag @code{mpi_tag} within the
  96. communicator @var{comm}. On completion, the @var{callback} function is
  97. called with the argument @var{arg}.
  98. @end deftypefun
  99. @deftypefun int starpu_mpi_irecv_detached (starpu_data_handle_t @var{data_handle}, int @var{source}, int @var{mpi_tag}, MPI_Comm @var{comm}, void (*@var{callback})(void *), void *@var{arg})
  100. Posts a nonblocking receive in @var{data_handle} from the
  101. node @var{source} using the message tag @code{mpi_tag} within the
  102. communicator @var{comm}. On completion, the @var{callback} function is
  103. called with the argument @var{arg}.
  104. @end deftypefun
  105. @deftypefun int starpu_mpi_wait (starpu_mpi_req *@var{req}, MPI_Status *@var{status})
  106. Returns when the operation identified by request @var{req} is complete.
  107. @end deftypefun
  108. @deftypefun int starpu_mpi_test (starpu_mpi_req *@var{req}, int *@var{flag}, MPI_Status *@var{status})
  109. If the operation identified by @var{req} is complete, set @var{flag}
  110. to 1. The @var{status} object is set to contain information on the
  111. completed operation.
  112. @end deftypefun
  113. @deftypefun int starpu_mpi_barrier (MPI_Comm @var{comm})
  114. Blocks the caller until all group members of the communicator
  115. @var{comm} have called it.
  116. @end deftypefun
  117. @deftypefun int starpu_mpi_isend_detached_unlock_tag (starpu_data_handle_t @var{data_handle}, int @var{dest}, int @var{mpi_tag}, MPI_Comm @var{comm}, starpu_tag_t @var{tag})
  118. Posts a standard-mode, non blocking send of @var{data_handle} to the
  119. node @var{dest} using the message tag @code{mpi_tag} within the
  120. communicator @var{comm}. On completion, @var{tag} is unlocked.
  121. @end deftypefun
  122. @deftypefun int starpu_mpi_irecv_detached_unlock_tag (starpu_data_handle_t @var{data_handle}, int @var{source}, int @var{mpi_tag}, MPI_Comm @var{comm}, starpu_tag_t @var{tag})
  123. Posts a nonblocking receive in @var{data_handle} from the
  124. node @var{source} using the message tag @code{mpi_tag} within the
  125. communicator @var{comm}. On completion, @var{tag} is unlocked.
  126. @end deftypefun
  127. @deftypefun int starpu_mpi_isend_array_detached_unlock_tag (unsigned @var{array_size}, starpu_data_handle_t *@var{data_handle}, int *@var{dest}, int *@var{mpi_tag}, MPI_Comm *@var{comm}, starpu_tag_t @var{tag})
  128. Posts @var{array_size} standard-mode, non blocking send. Each post
  129. sends the n-th data of the array @var{data_handle} to the n-th node of
  130. the array @var{dest}
  131. using the n-th message tag of the array @code{mpi_tag} within the n-th
  132. communicator of the array
  133. @var{comm}. On completion of the all the requests, @var{tag} is unlocked.
  134. @end deftypefun
  135. @deftypefun int starpu_mpi_irecv_array_detached_unlock_tag (unsigned @var{array_size}, starpu_data_handle_t *@var{data_handle}, int *@var{source}, int *@var{mpi_tag}, MPI_Comm *@var{comm}, starpu_tag_t @var{tag})
  136. Posts @var{array_size} nonblocking receive. Each post receives in the
  137. n-th data of the array @var{data_handle} from the n-th
  138. node of the array @var{source} using the n-th message tag of the array
  139. @code{mpi_tag} within the n-th communicator of the array @var{comm}.
  140. On completion of the all the requests, @var{tag} is unlocked.
  141. @end deftypefun
  142. @page
  143. @node Simple Example
  144. @section Simple Example
  145. @cartouche
  146. @smallexample
  147. void increment_token(void)
  148. @{
  149. struct starpu_task *task = starpu_task_create();
  150. task->cl = &increment_cl;
  151. task->handles[0] = token_handle;
  152. starpu_task_submit(task);
  153. @}
  154. @end smallexample
  155. @end cartouche
  156. @cartouche
  157. @smallexample
  158. int main(int argc, char **argv)
  159. @{
  160. int rank, size;
  161. starpu_init(NULL);
  162. starpu_mpi_initialize_extended(&rank, &size);
  163. starpu_vector_data_register(&token_handle, 0, (uintptr_t)&token, 1, sizeof(unsigned));
  164. unsigned nloops = NITER;
  165. unsigned loop;
  166. unsigned last_loop = nloops - 1;
  167. unsigned last_rank = size - 1;
  168. @end smallexample
  169. @end cartouche
  170. @cartouche
  171. @smallexample
  172. for (loop = 0; loop < nloops; loop++) @{
  173. int tag = loop*size + rank;
  174. if (loop == 0 && rank == 0)
  175. @{
  176. token = 0;
  177. fprintf(stdout, "Start with token value %d\n", token);
  178. @}
  179. else
  180. @{
  181. starpu_mpi_irecv_detached(token_handle, (rank+size-1)%size, tag,
  182. MPI_COMM_WORLD, NULL, NULL);
  183. @}
  184. increment_token();
  185. if (loop == last_loop && rank == last_rank)
  186. @{
  187. starpu_data_acquire(token_handle, STARPU_R);
  188. fprintf(stdout, "Finished: token value %d\n", token);
  189. starpu_data_release(token_handle);
  190. @}
  191. else
  192. @{
  193. starpu_mpi_isend_detached(token_handle, (rank+1)%size, tag+1,
  194. MPI_COMM_WORLD, NULL, NULL);
  195. @}
  196. @}
  197. starpu_task_wait_for_all();
  198. @end smallexample
  199. @end cartouche
  200. @cartouche
  201. @smallexample
  202. starpu_mpi_shutdown();
  203. starpu_shutdown();
  204. if (rank == last_rank)
  205. @{
  206. fprintf(stderr, "[%d] token = %d == %d * %d ?\n", rank, token, nloops, size);
  207. STARPU_ASSERT(token == nloops*size);
  208. @}
  209. @end smallexample
  210. @end cartouche
  211. @page
  212. @node Exchanging User Defined Data Interface
  213. @section Exchanging User Defined Data Interface
  214. New data interfaces defined as explained in @ref{An example
  215. of data interface} can also be used within StarPU-MPI and exchanged
  216. between nodes. Two functions needs to be defined through
  217. the type @code{struct starpu_data_interface_ops} (@pxref{Data
  218. Interface API}). The pack function takes a handle and returns a
  219. contiguous memory buffer along with its size where data to be conveyed to another node
  220. should be copied. The reversed operation is implemented in the unpack
  221. function which takes a contiguous memory buffer and recreates the data
  222. handle.
  223. @cartouche
  224. @smallexample
  225. static int complex_pack_data(starpu_data_handle_t handle, uint32_t node, void **ptr, size_t *count)
  226. @{
  227. STARPU_ASSERT(starpu_data_test_if_allocated_on_node(handle, node));
  228. struct starpu_complex_interface *complex_interface =
  229. (struct starpu_complex_interface *) starpu_data_get_interface_on_node(handle, node);
  230. *count = complex_get_size(handle);
  231. *ptr = malloc(*count);
  232. memcpy(*ptr, complex_interface->real, complex_interface->nx*sizeof(double));
  233. memcpy(*ptr+complex_interface->nx*sizeof(double), complex_interface->imaginary,
  234. complex_interface->nx*sizeof(double));
  235. return 0;
  236. @}
  237. @end smallexample
  238. @end cartouche
  239. @cartouche
  240. @smallexample
  241. static int complex_unpack_data(starpu_data_handle_t handle, uint32_t node, void *ptr, size_t count)
  242. @{
  243. STARPU_ASSERT(starpu_data_test_if_allocated_on_node(handle, node));
  244. struct starpu_complex_interface *complex_interface =
  245. (struct starpu_complex_interface *) starpu_data_get_interface_on_node(handle, node);
  246. memcpy(complex_interface->real, ptr, complex_interface->nx*sizeof(double));
  247. memcpy(complex_interface->imaginary, ptr+complex_interface->nx*sizeof(double),
  248. complex_interface->nx*sizeof(double));
  249. return 0;
  250. @}
  251. @end smallexample
  252. @end cartouche
  253. @cartouche
  254. @smallexample
  255. static struct starpu_data_interface_ops interface_complex_ops =
  256. @{
  257. ...
  258. .pack_data = complex_pack_data,
  259. .unpack_data = complex_unpack_data
  260. @};
  261. @end smallexample
  262. @end cartouche
  263. @page
  264. @node MPI Insert Task Utility
  265. @section MPI Insert Task Utility
  266. To save the programmer from having to explicit all communications, StarPU
  267. provides an "MPI Insert Task Utility". The principe is that the application
  268. decides a distribution of the data over the MPI nodes by allocating it and
  269. notifying StarPU of that decision, i.e. tell StarPU which MPI node "owns"
  270. which data. It also decides, for each handle, an MPI tag which will be used to
  271. exchange the content of the handle. All MPI nodes then process the whole task
  272. graph, and StarPU automatically determines which node actually execute which
  273. task, and trigger the required MPI transfers.
  274. @deftypefun int starpu_data_set_tag (starpu_data_handle_t @var{handle}, int @var{tag})
  275. Tell StarPU-MPI which MPI tag to use when exchanging the data.
  276. @end deftypefun
  277. @deftypefun int starpu_data_get_tag (starpu_data_handle_t @var{handle})
  278. Returns the MPI tag to be used when exchanging the data.
  279. @end deftypefun
  280. @deftypefun int starpu_data_set_rank (starpu_data_handle_t @var{handle}, int @var{rank})
  281. Tell StarPU-MPI which MPI node "owns" a given data, that is, the node which will
  282. always keep an up-to-date value, and will by default execute tasks which write
  283. to it.
  284. @end deftypefun
  285. @deftypefun int starpu_data_get_rank (starpu_data_handle_t @var{handle})
  286. Returns the last value set by @code{starpu_data_set_rank}.
  287. @end deftypefun
  288. @defmac STARPU_EXECUTE_ON_NODE
  289. this macro is used when calling @code{starpu_mpi_insert_task}, and
  290. must be followed by a integer value which specified the node on which
  291. to execute the codelet.
  292. @end defmac
  293. @defmac STARPU_EXECUTE_ON_DATA
  294. this macro is used when calling @code{starpu_mpi_insert_task}, and
  295. must be followed by a data handle to specify that the node owning the
  296. given data will execute the codelet.
  297. @end defmac
  298. @deftypefun int starpu_mpi_insert_task (MPI_Comm @var{comm}, struct starpu_codelet *@var{codelet}, ...)
  299. Create and submit a task corresponding to @var{codelet} with the following
  300. arguments. The argument list must be zero-terminated.
  301. The arguments following the codelets are the same types as for the
  302. function @code{starpu_insert_task} defined in @ref{Insert Task
  303. Utility}. The extra argument @code{STARPU_EXECUTE_ON_NODE} followed by an
  304. integer allows to specify the MPI node to execute the codelet. It is also
  305. possible to specify that the node owning a specific data will execute
  306. the codelet, by using @code{STARPU_EXECUTE_ON_DATA} followed by a data
  307. handle.
  308. The internal algorithm is as follows:
  309. @enumerate
  310. @item Find out whether we (as an MPI node) are to execute the codelet
  311. because we own the data to be written to. If different nodes own data
  312. to be written to, the argument @code{STARPU_EXECUTE_ON_NODE} or
  313. @code{STARPU_EXECUTE_ON_DATA} has to be used to specify which MPI node will
  314. execute the task.
  315. @item Send and receive data as requested. Nodes owning data which need to be
  316. read by the task are sending them to the MPI node which will execute it. The
  317. latter receives them.
  318. @item Execute the codelet. This is done by the MPI node selected in the
  319. 1st step of the algorithm.
  320. @item In the case when different MPI nodes own data to be written to, send
  321. written data back to their owners.
  322. @end enumerate
  323. The algorithm also includes a communication cache mechanism that
  324. allows not to send data twice to the same MPI node, unless the data
  325. has been modified. The cache can be disabled
  326. (@pxref{STARPU_MPI_CACHE}).
  327. @end deftypefun
  328. @deftypefun void starpu_mpi_get_data_on_node (MPI_Comm @var{comm}, starpu_data_handle_t @var{data_handle}, int @var{node})
  329. Transfer data @var{data_handle} to MPI node @var{node}, sending it from its
  330. owner if needed. At least the target node and the owner have to call the
  331. function.
  332. @end deftypefun
  333. Here an stencil example showing how to use @code{starpu_mpi_insert_task}. One
  334. first needs to define a distribution function which specifies the
  335. locality of the data. Note that that distribution information needs to
  336. be given to StarPU by calling @code{starpu_data_set_rank}.
  337. @cartouche
  338. @smallexample
  339. /* Returns the MPI node number where data is */
  340. int my_distrib(int x, int y, int nb_nodes) @{
  341. /* Block distrib */
  342. return ((int)(x / sqrt(nb_nodes) + (y / sqrt(nb_nodes)) * sqrt(nb_nodes))) % nb_nodes;
  343. // /* Other examples useful for other kinds of computations */
  344. // /* / distrib */
  345. // return (x+y) % nb_nodes;
  346. // /* Block cyclic distrib */
  347. // unsigned side = sqrt(nb_nodes);
  348. // return x % side + (y % side) * size;
  349. @}
  350. @end smallexample
  351. @end cartouche
  352. Now the data can be registered within StarPU. Data which are not
  353. owned but will be needed for computations can be registered through
  354. the lazy allocation mechanism, i.e. with a @code{home_node} set to -1.
  355. StarPU will automatically allocate the memory when it is used for the
  356. first time.
  357. One can note an optimization here (the @code{else if} test): we only register
  358. data which will be needed by the tasks that we will execute.
  359. @cartouche
  360. @smallexample
  361. unsigned matrix[X][Y];
  362. starpu_data_handle_t data_handles[X][Y];
  363. for(x = 0; x < X; x++) @{
  364. for (y = 0; y < Y; y++) @{
  365. int mpi_rank = my_distrib(x, y, size);
  366. if (mpi_rank == my_rank)
  367. /* Owning data */
  368. starpu_variable_data_register(&data_handles[x][y], 0,
  369. (uintptr_t)&(matrix[x][y]), sizeof(unsigned));
  370. else if (my_rank == my_distrib(x+1, y, size) || my_rank == my_distrib(x-1, y, size)
  371. || my_rank == my_distrib(x, y+1, size) || my_rank == my_distrib(x, y-1, size))
  372. /* I don't own that index, but will need it for my computations */
  373. starpu_variable_data_register(&data_handles[x][y], -1,
  374. (uintptr_t)NULL, sizeof(unsigned));
  375. else
  376. /* I know it's useless to allocate anything for this */
  377. data_handles[x][y] = NULL;
  378. if (data_handles[x][y])
  379. starpu_data_set_rank(data_handles[x][y], mpi_rank);
  380. @}
  381. @}
  382. @end smallexample
  383. @end cartouche
  384. Now @code{starpu_mpi_insert_task()} can be called for the different
  385. steps of the application.
  386. @cartouche
  387. @smallexample
  388. for(loop=0 ; loop<niter; loop++)
  389. for (x = 1; x < X-1; x++)
  390. for (y = 1; y < Y-1; y++)
  391. starpu_mpi_insert_task(MPI_COMM_WORLD, &stencil5_cl,
  392. STARPU_RW, data_handles[x][y],
  393. STARPU_R, data_handles[x-1][y],
  394. STARPU_R, data_handles[x+1][y],
  395. STARPU_R, data_handles[x][y-1],
  396. STARPU_R, data_handles[x][y+1],
  397. 0);
  398. starpu_task_wait_for_all();
  399. @end smallexample
  400. @end cartouche
  401. I.e. all MPI nodes process the whole task graph, but as mentioned above, for
  402. each task, only the MPI node which owns the data being written to (here,
  403. @code{data_handles[x][y]}) will actually run the task. The other MPI nodes will
  404. automatically send the required data.
  405. This can be a concern with a growing number of nodes. To avoid this, the
  406. application can prune the task for loops according to the data distribution,
  407. so as to only submit tasks on nodes which have to care about them (either to
  408. execute them, or to send the required data).
  409. @node MPI Collective Operations
  410. @section MPI Collective Operations
  411. @deftypefun int starpu_mpi_scatter_detached (starpu_data_handle_t *@var{data_handles}, int @var{count}, int @var{root}, MPI_Comm @var{comm}, {void (*}@var{scallback})(void *), {void *}@var{sarg}, {void (*}@var{rcallback})(void *), {void *}@var{rarg})
  412. Scatter data among processes of the communicator based on the ownership of
  413. the data. For each data of the array @var{data_handles}, the
  414. process @var{root} sends the data to the process owning this data.
  415. Processes receiving data must have valid data handles to receive them.
  416. On completion of the collective communication, the @var{scallback} function is
  417. called with the argument @var{sarg} on the process @var{root}, the @var{rcallback} function is
  418. called with the argument @var{rarg} on any other process.
  419. @end deftypefun
  420. @deftypefun int starpu_mpi_gather_detached (starpu_data_handle_t *@var{data_handles}, int @var{count}, int @var{root}, MPI_Comm @var{comm}, {void (*}@var{scallback})(void *), {void *}@var{sarg}, {void (*}@var{rcallback})(void *), {void *}@var{rarg})
  421. Gather data from the different processes of the communicator onto the
  422. process @var{root}. Each process owning data handle in the array
  423. @var{data_handles} will send them to the process @var{root}. The
  424. process @var{root} must have valid data handles to receive the data.
  425. On completion of the collective communication, the @var{rcallback} function is
  426. called with the argument @var{rarg} on the process @var{root}, the @var{scallback} function is
  427. called with the argument @var{sarg} on any other process.
  428. @end deftypefun
  429. @page
  430. @cartouche
  431. @smallexample
  432. if (rank == root)
  433. @{
  434. /* Allocate the vector */
  435. vector = malloc(nblocks * sizeof(float *));
  436. for(x=0 ; x<nblocks ; x++)
  437. @{
  438. starpu_malloc((void **)&vector[x], block_size*sizeof(float));
  439. @}
  440. @}
  441. /* Allocate data handles and register data to StarPU */
  442. data_handles = malloc(nblocks*sizeof(starpu_data_handle_t *));
  443. for(x = 0; x < nblocks ; x++)
  444. @{
  445. int mpi_rank = my_distrib(x, nodes);
  446. if (rank == root) @{
  447. starpu_vector_data_register(&data_handles[x], 0, (uintptr_t)vector[x],
  448. blocks_size, sizeof(float));
  449. @}
  450. else if ((mpi_rank == rank) || ((rank == mpi_rank+1 || rank == mpi_rank-1))) @{
  451. /* I own that index, or i will need it for my computations */
  452. starpu_vector_data_register(&data_handles[x], -1, (uintptr_t)NULL,
  453. block_size, sizeof(float));
  454. @}
  455. else @{
  456. /* I know it's useless to allocate anything for this */
  457. data_handles[x] = NULL;
  458. @}
  459. if (data_handles[x]) @{
  460. starpu_data_set_rank(data_handles[x], mpi_rank);
  461. @}
  462. @}
  463. /* Scatter the matrix among the nodes */
  464. starpu_mpi_scatter_detached(data_handles, nblocks, root, MPI_COMM_WORLD);
  465. /* Calculation */
  466. for(x = 0; x < nblocks ; x++) @{
  467. if (data_handles[x]) @{
  468. int owner = starpu_data_get_rank(data_handles[x]);
  469. if (owner == rank) @{
  470. starpu_insert_task(&cl, STARPU_RW, data_handles[x], 0);
  471. @}
  472. @}
  473. @}
  474. /* Gather the matrix on main node */
  475. starpu_mpi_gather_detached(data_handles, nblocks, 0, MPI_COMM_WORLD);
  476. @end smallexample
  477. @end cartouche