/* * This file is part of the StarPU Handbook. * Copyright (C) 2009--2011 Universit@'e de Bordeaux * Copyright (C) 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017 CNRS * Copyright (C) 2011, 2012, 2017 INRIA * See the file version.doxy for copying conditions. */ /*! \defgroup API_MPI_Support MPI Support @name Initialisation \ingroup API_MPI_Support \def STARPU_USE_MPI \ingroup API_MPI_Support Defined when StarPU has been installed with MPI support. It should be used in your code to detect the availability of MPI. \fn int starpu_mpi_init_comm(int *argc, char ***argv, int initialize_mpi, MPI_Comm comm) \ingroup API_MPI_Support Initialize the starpumpi library with the given communicator \p comm. \p initialize_mpi indicates if MPI should be initialized or not by StarPU. If the value is not 0, MPI will be initialized by calling MPI_Init_Thread(argc, argv, MPI_THREAD_SERIALIZED, ...). starpu_init() must be called before starpu_mpi_init_comm(). \fn int starpu_mpi_init(int *argc, char ***argv, int initialize_mpi) \ingroup API_MPI_Support Call starpu_mpi_init_comm() with the MPI communicator \c MPI_COMM_WORLD. \fn int starpu_mpi_initialize(void) \deprecated \ingroup API_MPI_Support This function has been made deprecated. One should use instead the function starpu_mpi_init(). This function does not call \c MPI_Init(), it should be called beforehand. \fn int starpu_mpi_initialize_extended(int *rank, int *world_size) \deprecated \ingroup API_MPI_Support This function has been made deprecated. One should use instead the function starpu_mpi_init(). MPI will be initialized by starpumpi by calling MPI_Init_Thread(argc, argv, MPI_THREAD_SERIALIZED, ...). \fn int starpu_mpi_shutdown(void) \ingroup API_MPI_Support Clean the starpumpi library. This must be called between calling \c starpu_mpi functions and starpu_shutdown(). \c MPI_Finalize() will be called if StarPU-MPI has been initialized by starpu_mpi_init(). \fn void starpu_mpi_comm_amounts_retrieve(size_t *comm_amounts) \ingroup API_MPI_Support Retrieve the current amount of communications from the current node in the array \p comm_amounts which must have a size greater or equal to the world size. Communications statistics must be enabled (see \ref STARPU_COMM_STATS). \fn int starpu_mpi_comm_size(MPI_Comm comm, int *size) \ingroup API_MPI_Support Return in \p size the size of the communicator \p comm \fn int starpu_mpi_comm_rank(MPI_Comm comm, int *rank) \ingroup API_MPI_Support Return in \p rank the rank of the calling process in the communicator \p comm \fn int starpu_mpi_world_rank(void) \ingroup API_MPI_Support Return the rank of the calling process in the communicator \c MPI_COMM_WORLD \fn int starpu_mpi_world_size(void) \ingroup API_MPI_Support Return the size of the communicator \c MPI_COMM_WORLD @name Communication \anchor MPIPtpCommunication \ingroup API_MPI_Support \fn int starpu_mpi_send(starpu_data_handle_t data_handle, int dest, int64_t data_tag, MPI_Comm comm) \ingroup API_MPI_Support Perform a standard-mode, blocking send of \p data_handle to the node \p dest using the message tag \p data_tag within the communicator \p comm. \fn int starpu_mpi_send_prio(starpu_data_handle_t data_handle, int dest, int64_t data_tag, int prio, MPI_Comm comm) \ingroup API_MPI_Support Similar to starpu_mpi_send, but takes a priority \p prio. \fn int starpu_mpi_recv(starpu_data_handle_t data_handle, int source, int64_t data_tag, MPI_Comm comm, MPI_Status *status) \ingroup API_MPI_Support Perform a standard-mode, blocking receive in \p data_handle from the node \p source using the message tag \p data_tag within the communicator \p comm. \fn int starpu_mpi_isend(starpu_data_handle_t data_handle, starpu_mpi_req *req, int dest, int64_t data_tag, MPI_Comm comm) \ingroup API_MPI_Support Post a standard-mode, non blocking send of \p data_handle to the node \p dest using the message tag \p data_tag within the communicator \p comm. After the call, the pointer to the request \p req can be used to test or to wait for the completion of the communication. \fn int starpu_mpi_isend_prio(starpu_data_handle_t data_handle, starpu_mpi_req *req, int dest, int64_t data_tag, int prio, MPI_Comm comm) \ingroup API_MPI_Support Similar to starpu_mpi_isend, but takes a priority \p prio. \fn int starpu_mpi_irecv(starpu_data_handle_t data_handle, starpu_mpi_req *req, int source, int64_t data_tag, MPI_Comm comm) \ingroup API_MPI_Support Post a nonblocking receive in \p data_handle from the node \p source using the message tag \p data_tag within the communicator \p comm. After the call, the pointer to the request \p req can be used to test or to wait for the completion of the communication. \fn int starpu_mpi_isend_detached(starpu_data_handle_t data_handle, int dest, int64_t data_tag, MPI_Comm comm, void (*callback)(void *), void *arg) \ingroup API_MPI_Support Post a standard-mode, non blocking send of \p data_handle to the node \p dest using the message tag \p data_tag within the communicator \p comm. On completion, the \p callback function is called with the argument \p arg. Similarly to the pthread detached functionality, when a detached communication completes, its resources are automatically released back to the system, there is no need to test or to wait for the completion of the request. \fn int starpu_mpi_isend_detached_prio(starpu_data_handle_t data_handle, int dest, int64_t data_tag, int prio, MPI_Comm comm, void (*callback)(void *), void *arg) \ingroup API_MPI_Support Similar to starpu_mpi_isend_detached, but takes a priority \p prio. \fn int starpu_mpi_irecv_detached(starpu_data_handle_t data_handle, int source, int64_t data_tag, MPI_Comm comm, void (*callback)(void *), void *arg) \ingroup API_MPI_Support Post a nonblocking receive in \p data_handle from the node \p source using the message tag \p data_tag within the communicator \p comm. On completion, the \p callback function is called with the argument \p arg. Similarly to the pthread detached functionality, when a detached communication completes, its resources are automatically released back to the system, there is no need to test or to wait for the completion of the request. \fn int starpu_mpi_irecv_detached_sequential_consistency(starpu_data_handle_t data_handle, int source, int64_t data_tag, MPI_Comm comm, void (*callback)(void *), void *arg, int sequential_consistency) \ingroup API_MPI_Support Post a nonblocking receive in \p data_handle from the node \p source using the message tag \p data_tag within the communicator \p comm. On completion, the \p callback function is called with the argument \p arg. The parameter \p sequential_consistency allows to enable or disable the sequential consistency for \p data handle (sequential consistency will be enabled or disabled based on the value of the parameter \p sequential_consistency and the value of the sequential consistency defined for \p data_handle). Similarly to the pthread detached functionality, when a detached communication completes, its resources are automatically released back to the system, there is no need to test or to wait for the completion of the request. \fn int starpu_mpi_issend(starpu_data_handle_t data_handle, starpu_mpi_req *req, int dest, int64_t data_tag, MPI_Comm comm) \ingroup API_MPI_Support Perform a synchronous-mode, non-blocking send of \p data_handle to the node \p dest using the message tag \p data_tag within the communicator \p comm. \fn int starpu_mpi_issend_prio(starpu_data_handle_t data_handle, starpu_mpi_req *req, int dest, int64_t data_tag, int prio, MPI_Comm comm) \ingroup API_MPI_Support Similar to starpu_mpi_issend, but takes a priority \p prio. \fn int starpu_mpi_issend_detached(starpu_data_handle_t data_handle, int dest, int64_t data_tag, MPI_Comm comm, void (*callback)(void *), void *arg) \ingroup API_MPI_Support Perform a synchronous-mode, non-blocking send of \p data_handle to the node \p dest using the message tag \p data_tag within the communicator \p comm. On completion, the \p callback function is called with the argument \p arg. Similarly to the pthread detached functionality, when a detached communication completes, its resources are automatically released back to the system, there is no need to test or to wait for the completion of the request. \fn int starpu_mpi_wait(starpu_mpi_req *req, MPI_Status *status) \ingroup API_MPI_Support Return when the operation identified by request \p req is complete. \fn int starpu_mpi_test(starpu_mpi_req *req, int *flag, MPI_Status *status) \ingroup API_MPI_Support If the operation identified by \p req is complete, set \p flag to 1. The \p status object is set to contain information on the completed operation. \fn int starpu_mpi_barrier(MPI_Comm comm) \ingroup API_MPI_Support Block the caller until all group members of the communicator \p comm have called it. \fn int starpu_mpi_wait_for_all(MPI_Comm comm) \ingroup API_MPI_Support Wait until all StarPU tasks and communications for the given communicator are completed. \fn int starpu_mpi_isend_detached_unlock_tag(starpu_data_handle_t data_handle, int dest, int64_t data_tag, MPI_Comm comm, starpu_tag_t tag) \ingroup API_MPI_Support Post a standard-mode, non blocking send of \p data_handle to the node \p dest using the message tag \p data_tag within the communicator \p comm. On completion, \p tag is unlocked. \fn int starpu_mpi_isend_detached_unlock_tag_prio(starpu_data_handle_t data_handle, int dest, int64_t data_tag, int prio, MPI_Comm comm, starpu_tag_t tag) \ingroup API_MPI_Support Similar to starpu_mpi_isend_detached_unlock_tag(), but takes a priority \p prio. \fn int starpu_mpi_irecv_detached_unlock_tag(starpu_data_handle_t data_handle, int source, int64_t data_tag, MPI_Comm comm, starpu_tag_t tag) \ingroup API_MPI_Support Post a nonblocking receive in \p data_handle from the node \p source using the message tag \p data_tag within the communicator \p comm. On completion, \p tag is unlocked. \fn int starpu_mpi_isend_array_detached_unlock_tag(unsigned array_size, starpu_data_handle_t *data_handle, int *dest, int64_t *data_tag, MPI_Comm *comm, starpu_tag_t tag) \ingroup API_MPI_Support Post \p array_size standard-mode, non blocking send. Each post sends the n-th data of the array \p data_handle to the n-th node of the array \p dest using the n-th message tag of the array \p data_tag within the n-th communicator of the array \p comm. On completion of the all the requests, \p tag is unlocked. \fn int starpu_mpi_isend_array_detached_unlock_tag_prio(unsigned array_size, starpu_data_handle_t *data_handle, int *dest, int64_t *data_tag, int *prio, MPI_Comm *comm, starpu_tag_t tag) \ingroup API_MPI_Support Similar to starpu_mpi_isend_array_detached_unlock_tag(), but takes a priority \p prio. \fn int starpu_mpi_irecv_array_detached_unlock_tag(unsigned array_size, starpu_data_handle_t *data_handle, int *source, int64_t *data_tag, MPI_Comm *comm, starpu_tag_t tag) \ingroup API_MPI_Support Post \p array_size nonblocking receive. Each post receives in the n-th data of the array \p data_handle from the n-th node of the array \p source using the n-th message tag of the array \p data_tag within the n-th communicator of the array \p comm. On completion of the all the requests, \p tag is unlocked. \fn int starpu_mpi_get_communication_tag(void) \ingroup API_MPI_Support todo \fn void starpu_mpi_set_communication_tag(int tag) \ingroup API_MPI_Support todo \fn int starpu_mpi_datatype_register(starpu_data_handle_t handle, starpu_mpi_datatype_allocate_func_t allocate_datatype_func, starpu_mpi_datatype_free_func_t free_datatype_func) \ingroup API_MPI_Support Register functions to create and free a MPI datatype for the given handle. It is important that the function is called before any communication can take place for a data with the given handle. See \ref ExchangingUserDefinedDataInterface for an example. \fn int starpu_mpi_datatype_unregister(starpu_data_handle_t handle); \ingroup API_MPI_Support Unregister the MPI datatype functions stored for the interface of the given handle. \def STARPU_MPI_TAG_UB \ingroup API_MPI_Support When given to the function starpu_mpi_comm_get_attr(), retrieve the value for the upper bound for tag value. \fn int starpu_mpi_comm_get_attr(MPI_Comm comm, int keyval, void *attribute_val, int *flag); \ingroup API_MPI_Support Retrieve an attribute value by key, similarly to the MPI function \c MPI_comm_get_attr(). If an attribute is attached on \p comm to \p keyval, then the call returns \p flag equal to \c 1, and the attribute value in \p attribute_val. Otherwise, \p flag is set to \0. @name Communication Cache \ingroup API_MPI_Support \fn int starpu_mpi_cache_is_enabled() \ingroup API_MPI_Support Return 1 if the communication cache is enabled, 0 otherwise \fn int starpu_mpi_cache_set(int enabled) \ingroup API_MPI_Support If \p enabled is 1, enable the communication cache. Otherwise, clean the cache if it was enabled and disable it. \fn void starpu_mpi_cache_flush(MPI_Comm comm, starpu_data_handle_t data_handle) \ingroup API_MPI_Support Clear the send and receive communication cache for the data \p data_handle and invalidate the value. The function has to be called at the same point of task graph submission by all the MPI nodes on which the handle was registered. The function does nothing if the cache mechanism is disabled (see \ref STARPU_MPI_CACHE). \fn void starpu_mpi_cache_flush_all_data(MPI_Comm comm) \ingroup API_MPI_Support Clear the send and receive communication cache for all data and invalidate their values. The function has to be called at the same point of task graph submission by all the MPI nodes. The function does nothing if the cache mechanism is disabled (see \ref STARPU_MPI_CACHE). \fn int starpu_mpi_cached_receive(starpu_data_handle_t data_handle); \ingroup API_MPI_Support Test whether \p data_handle is cached for reception, i.e. the value was previously received from the owner node, and not flushed since then. \fn int starpu_mpi_cached_send(starpu_data_handle_t data_handle, int dest); \ingroup API_MPI_Support Test whether \p data_handle is cached for emission to node \p dest , i.e. the value was previously sent to \p dest, and not flushed since then. @name MPI Insert Task \anchor MPIInsertTask \ingroup API_MPI_Support \fn void starpu_mpi_data_register_comm(starpu_data_handle_t data_handle, int64_t data_tag, int rank, MPI_Comm comm) \ingroup API_MPI_Support Register to MPI a StarPU data handle with the given tag, rank and MPI communicator. It also automatically clears the MPI communication cache when unregistering the data. \def starpu_mpi_data_register(data_handle, data_tag, rank) \ingroup API_MPI_Support Register to MPI a StarPU data handle with the given tag, rank and the MPI communicator \c MPI_COMM_WORLD. It also automatically clears the MPI communication cache when unregistering the data. \fn void starpu_mpi_data_set_tag(starpu_data_handle_t handle, int64_t data_tag) \ingroup API_MPI_Support Register to MPI a StarPU data handle with the given tag. No rank will be defined. It also automatically clears the MPI communication cache when unregistering the data. \def starpu_data_set_tag \ingroup API_MPI_Support Symbol kept for backward compatibility. Calling function starpu_mpi_data_set_tag() \fn void starpu_mpi_data_set_rank_comm(starpu_data_handle_t handle, int rank, MPI_Comm comm) \ingroup API_MPI_Support Register to MPI a StarPU data handle with the given rank and given communicator. No tag will be defined. It also automatically clears the MPI communication cache when unregistering the data. \def starpu_mpi_data_set_rank \ingroup API_MPI_Support Register to MPI a StarPU data handle with the given rank and the MPI communicator \c MPI_COMM_WORLD. No tag will be defined. It also automatically clears the MPI communication cache when unregistering the data. Symbol kept for backward compatibility. Calling function starpu_mpi_data_set_rank() \def starpu_data_set_rank \ingroup API_MPI_Support Register to MPI a StarPU data handle with the given rank and the MPI communicator \c MPI_COMM_WORLD. No tag will be defined. It also automatically clears the MPI communication cache when unregistering the data. Symbol kept for backward compatibility. Calling function starpu_mpi_data_set_rank() \fn int starpu_mpi_data_get_rank(starpu_data_handle_t handle) \ingroup API_MPI_Support Return the rank of the given data. \def starpu_data_get_rank \ingroup API_MPI_Support Return the rank of the given data. Symbol kept for backward compatibility. Calling function starpu_mpi_data_get_rank() \fn int64_t starpu_mpi_data_get_tag(starpu_data_handle_t handle) \ingroup API_MPI_Support Return the tag of the given data. \def starpu_data_get_tag \ingroup API_MPI_Support Return the tag of the given data. Symbol kept for backward compatibility. Calling function starpu_mpi_data_get_tag() \def STARPU_MPI_PER_NODE \ingroup API_MPI_Support Can be used as rank when calling starpu_mpi_data_register() and alike, to specify that the data is per-node: each node will have its own value. Tasks writing to such data will be replicated on all nodes (and all parameters then have to be per-node). Tasks not writing to such data will just take the node-local value without any MPI communication. \fn void starpu_mpi_data_migrate(MPI_Comm comm, starpu_data_handle_t handle, int new_rank) \ingroup API_MPI_Support Migrate the data onto the \p new_rank MPI node. This means both transferring the data to node \p new_rank if it hasn't been transferred already, and setting the home node of the data to the new node. Further data transfers triggered by starpu_mpi_task_insert() will be done from that new node. This function thus needs to be called on all nodes which have registered the data. This also flushes the cache for this data to avoid incoherencies. \def STARPU_EXECUTE_ON_NODE \ingroup API_MPI_Support Used when calling starpu_mpi_task_insert(), must be followed by a integer value which specified the node on which to execute the codelet. \def STARPU_EXECUTE_ON_DATA \ingroup API_MPI_Support Used when calling starpu_mpi_task_insert(), must be followed by a data handle to specify that the node owning the given data will execute the codelet. \def STARPU_NODE_SELECTION_POLICY \ingroup API_MPI_Support Used when calling starpu_mpi_task_insert(), must be followed by a identifier to a node selection policy. This is needed when several nodes own data in ::STARPU_W mode. \fn int starpu_mpi_insert_task(MPI_Comm comm, struct starpu_codelet *codelet, ...) \ingroup API_MPI_Support Call starpu_mpi_task_insert(). Symbol kept for backward compatibility. \fn int starpu_mpi_task_insert(MPI_Comm comm, struct starpu_codelet *codelet, ...) \ingroup API_MPI_Support Create and submit a task corresponding to codelet with the following arguments. The argument list must be zero-terminated. The arguments following the codelet are the same types as for the function starpu_task_insert(). Access modes for data can also be set with ::STARPU_SSEND to specify the data has to be sent using a synchronous and non-blocking mode (see starpu_mpi_issend()). The extra argument ::STARPU_EXECUTE_ON_NODE followed by an integer allows to specify the MPI node to execute the codelet. It is also possible to specify that the node owning a specific data will execute the codelet, by using ::STARPU_EXECUTE_ON_DATA followed by a data handle. The internal algorithm is as follows:
  1. Find out which MPI node is going to execute the codelet.
  2. Send and receive data as requested. Nodes owning data which need to be read by the task are sending them to the MPI node which will execute it. The latter receives them.
  3. Execute the codelet. This is done by the MPI node selected in the 1st step of the algorithm.
  4. If several MPI nodes own data to be written to, send written data back to their owners.
The algorithm also includes a communication cache mechanism that allows not to send data twice to the same MPI node, unless the data has been modified. The cache can be disabled (see \ref STARPU_MPI_CACHE). \fn struct starpu_task *starpu_mpi_task_build(MPI_Comm comm, struct starpu_codelet *codelet, ...) \ingroup API_MPI_Support Create a task corresponding to \p codelet with the following given arguments. The argument list must be zero-terminated. The function performs the first two steps of the function starpu_mpi_task_insert(). Only the MPI node selected in the first step of the algorithm will return a valid task structure which can then be submitted, others will return NULL. The function starpu_mpi_task_post_build() MUST be called after that on all nodes, and after the submission of the task on the node which creates it, with the SAME list of arguments. \fn int starpu_mpi_task_post_build(MPI_Comm comm, struct starpu_codelet *codelet, ...) \ingroup API_MPI_Support MUST be called after a call to starpu_mpi_task_build(), with the SAME list of arguments. Perform the fourth -- last -- step of the algorithm described in starpu_mpi_task_insert(). \fn void starpu_mpi_get_data_on_node(MPI_Comm comm, starpu_data_handle_t data_handle, int node) \ingroup API_MPI_Support Transfer data \p data_handle to MPI node \p node, sending it from its owner if needed. At least the target node and the owner have to call the function. \fn void starpu_mpi_get_data_on_node_detached(MPI_Comm comm, starpu_data_handle_t data_handle, int node, void (*callback)(void*), void *arg) \ingroup API_MPI_Support Transfer data \p data_handle to MPI node \p node, sending it from its owner if needed. At least the target node and the owner have to call the function. On reception, the \p callback function is called with the argument \p arg. \fn void starpu_mpi_get_data_on_all_nodes_detached(MPI_Comm comm, starpu_data_handle_t data_handle) \ingroup API_MPI_Support Transfer data \p data_handle to all MPI nodes, sending it from its owner if needed. All nodes have to call the function. @name Node Selection Policy \anchor MPINodeSelectionPolicy \ingroup API_MPI_Support \def STARPU_MPI_NODE_SELECTION_CURRENT_POLICY \ingroup API_MPI_Support todo \def STARPU_MPI_NODE_SELECTION_MOST_R_DATA \ingroup API_MPI_Support todo \fn int starpu_mpi_node_selection_get_current_policy() \ingroup API_MPI_Support Return the current policy used to select the node which will execute the codelet \fn int starpu_mpi_node_selection_set_current_policy(int policy) \ingroup API_MPI_Support Set the current policy used to select the node which will execute the codelet. The policy ::STARPU_MPI_NODE_SELECTION_MOST_R_DATA selects the node having the most data in ::STARPU_R mode so as to minimize the amount of data to be transfered. \fn int starpu_mpi_node_selection_register_policy(starpu_mpi_select_node_policy_func_t policy_func) \ingroup API_MPI_Support Register a new policy which can then be used when there is several nodes owning data in ::STARPU_W mode. Here an example of function defining a node selection policy. The codelet will be executed on the node owing the first data with a size bigger than 1M, or on the node 0 if no data fits the given size. \code{.c} int my_node_selection_policy(int me, int nb_nodes, struct starpu_data_descr *descr, int nb_data) { // me is the current MPI rank // nb_nodes is the number of MPI nodes // descr is the description of the data specified when calling starpu_mpi_task_insert // nb_data is the number of data in descr int i; for(i= 0 ; i 1024*1024) return rank; } } return 0; } \endcode \fn int starpu_mpi_node_selection_unregister_policy(int policy) \ingroup API_MPI_Support Unregister a previously registered policy. @name Collective Operations \anchor MPICollectiveOperations \ingroup API_MPI_Support \fn void starpu_mpi_redux_data(MPI_Comm comm, starpu_data_handle_t data_handle) \ingroup API_MPI_Support Perform a reduction on the given data \p handle. All nodes send the data to its owner node which will perform a reduction. \fn void starpu_mpi_redux_data_prio(MPI_Comm comm, starpu_data_handle_t data_handle, int prio) \ingroup API_MPI_Support Similar to starpu_mpi_redux_data, but takes a priority \p prio. \fn int starpu_mpi_scatter_detached(starpu_data_handle_t *data_handles, int count, int root, MPI_Comm comm, void (*scallback)(void *), void *sarg, void (*rcallback)(void *), void *rarg) \ingroup API_MPI_Support Scatter data among processes of the communicator based on the ownership of the data. For each data of the array \p data_handles, the process \p root sends the data to the process owning this data. Processes receiving data must have valid data handles to receive them. On completion of the collective communication, the \p scallback function is called with the argument \p sarg on the process \p root, the \p rcallback function is called with the argument \p rarg on any other process. \fn int starpu_mpi_gather_detached(starpu_data_handle_t *data_handles, int count, int root, MPI_Comm comm, void (*scallback)(void *), void *sarg, void (*rcallback)(void *), void *rarg) \ingroup API_MPI_Support Gather data from the different processes of the communicator onto the process \p root. Each process owning data handle in the array \p data_handles will send them to the process \p root. The process \p root must have valid data handles to receive the data. On completion of the collective communication, the \p rcallback function is called with the argument \p rarg on the process root, the \p scallback function is called with the argument \p sarg on any other process. @name MPI Master Slave \anchor MPIMasterSlaveSupport \ingroup API_MPI_Support \def STARPU_USE_MPI_MASTER_SLAVE \ingroup API_MPI_Support Defined when StarPU has been installed with MPI Master Slave support. It should be used in your code to detect the availability of MPI Master Slave. */