Let applications call starpu_data_unregister_submit on data with a home buffer (starpu-mpi actually even does it), but put fat warnings about this being unsafe

include/starpu_data.h

@@ -163,9 +163,11 @@ void starpu_data_unregister_no_coherency(starpu_data_handle_t handle);
    Destroy the data \p handle once it is no longer needed by any
    submitted task. No coherency is provided.
 
-   This is not safe to use on a handle coming from the registration of a
-   non-NULL application home buffer, since the moment when the unregistration
-   will happen is unknown to the application.
+   This is not safe to call starpu_data_unregister_submit() on a handle that
+   comes from the registration of a non-NULL application home buffer, since the
+   moment when the unregistration will happen is unknown to the
+   application. Only calling starpu_shutdown() allows to be sure that the data
+   was really unregistered.
 */
 void starpu_data_unregister_submit(starpu_data_handle_t handle);
 

mpi/src/mpi/starpu_mpi_mpi.c

@@ -957,6 +957,9 @@ static void _starpu_mpi_early_data_cb(void* arg)
 	starpu_data_release(args->early_handle);
 
 	_STARPU_MPI_DEBUG(3, "Done, handling unregister of early_handle..\n");
+	/* XXX: note that we have already freed the registered buffer above. In
+	 * principle that's unsafe. As of now it is fine because StarPU has no
+	 reason to access it. */
 	starpu_data_unregister_submit(args->early_handle);
 
 	_STARPU_MPI_DEBUG(3, "Done, handling request %p termination of the already received request\n",args->req);

src/datawizard/interfaces/data_interface.c

@@ -982,7 +982,6 @@ void starpu_data_unregister_submit(starpu_data_handle_t handle)
 {
 	STARPU_ASSERT_MSG(handle->magic == 42, "data %p is invalid (was it already registered?)", handle);
 	STARPU_ASSERT_MSG(!handle->lazy_unregister, "data %p can not be unregistered twice", handle);
-	STARPU_ASSERT_MSG(handle->home_node < 0, "data %p with application-allocated home buffer can not be safely unregistered asynchronously", handle);
 
 	/* Wait for all task dependencies on this handle before putting it for free */
 	starpu_data_acquire_on_node_cb(handle, STARPU_ACQUIRE_NO_NODE_LOCK_ALL, handle->initialized?STARPU_RW:STARPU_W, _starpu_data_unregister_submit_cb, handle);

tests/datawizard/lazy_unregister.c

@@ -67,6 +67,8 @@ int main(void)
 	STARPU_CHECK_RETURN_VALUE(ret, "starpu_task_submit");
 
 	starpu_data_unregister_submit(handle);
+	/* Note: we have no way to know when this will happen. We have to wait
+	 * for starpu_shutdown before being able to free the registered buffer */
 
 	ret = starpu_task_submit(t1);
 	if (ret == -ENODEV)