Merge branch 'master' into starpurm

AUTHORS

@@ -6,6 +6,7 @@ William Braik <wbraik@gmail.com>
 Berenger Bramas <berenger.bramas@inria.fr>
 Alfredo Buttari <alfredo.buttari@enseeiht.fr>
 Adrien Cassagne <adrien.cassagne@inria.fr>
+Arthur Chevalier <arthur.chevalier@inria.fr>
 Jérôme Clet-Ortega <jerome.clet-ortega@inria.fr>
 Terry Cojean <terry.cojean@inria.fr>
 Nicolas Collin <nicolas.collin@inria.fr>
@@ -29,6 +30,7 @@ Benjamin Lorendeau <benjamin.lorendeau@inria.fr>
 Antoine Lucas <antoine.lucas.33@gmail.com>
 Brice Mortier <brice.mortier@etu.u-bordeaux1.fr>
 Stojce Nakov <stojce.nakov@inria.fr>
+Lucas Leandro Nesi <llnesi@inf.ufrgs.br>
 Joris Pablo <joris.pablo@orange.fr>
 Damien Pasqualinotto <dam.pasqualinotto@wanadoo.fr>
 Samuel Pitoiset <samuel.pitoiset@inria.fr>

ChangeLog

@@ -41,6 +41,23 @@ New features:
     more, StarPU will make the appropriate calls as needed.
   * Add starpu_task_notify_ready_soon_register to be notified when it is
     determined when a task will be ready an estimated amount of time from now.
+  * New StarPU-MPI initialization function (starpu_mpi_init_conf)
+    which allows StarPU-MPI to manage reserving a core for the MPI thread, or
+    merging it with CPU driver 0.
+  * Add possibility to delay the termination of a task with the
+    functions starpu_task_end_dep_add() which specifies the number of
+    calls to the function starpu_task_end_dep_release() needed to
+    trigger the task termination.
+  * Add possibility to define the sequential consistency at the task level
+    for each handle used by the task.
+  * Add STARPU_SPECIFIC_NODE_LOCAL, STARPU_SPECIFIC_NODE_CPU, and
+    STARPU_SPECIFIC_NODE_SLOW as generic values for codelet specific memory
+    nodes which can be used instead of exact node numbers.
+  * Add starpu_get_next_bindid and starpu_bind_thread_on to allow binding an
+    application-started thread on a free core. Use it in StarPU-MPI to
+    automatically bind the MPI thread on an available core.
+  * Add STARPU_RESERVE_NCPU environment variable and reserve_ncpus config
+    field to make StarPU use a few cores less.
 
 Small features:
   * Scheduling contexts may now be associated a user data pointer at creation
@@ -63,7 +80,7 @@ Small features:
   * Add SubmitOrder trace field.
   * Add workerids and workerids_len task fields.
   * Add priority management to StarPU-MPI. Can be disabled with
-  the STARPU_MPI_PRIORITIES environment variable.
+    the STARPU_MPI_PRIORITIES environment variable.
   * Add STARPU_MAIN_THREAD_CPUID and STARPU_MPI_THREAD_CPUID environment
     variables.
   * Add disk to disk copy functions and support asynchronous full read/write
@@ -76,11 +93,30 @@ Small features:
   * Add dependency backward information in debugging mode for gdb's
     starpu-print-task
   * Add sched_data field in starpu_task structure.
+  * New starpu_fxt_tool option -label-deps to label dependencies on
+    the output graph
+  * New environment variable STARPU_GENERATE_TRACE_OPTIONS to specify
+    fxt options (to be used with STARPU_GENERATE_TRACE)
+  * New function starpu_task_set() similar as starpu_task_build() but
+    with a task object given as the first parameter
+  * New functions
+    starpu_data_partition_submit_sequential_consistency() and
+    starpu_data_unpartition_submit_sequential_consistency()
+  * Add a new value STARPU_TASK_SYNCHRONOUS to be used in
+    starpu_task_insert() to define if the task is (or not) synchronous
+  * Add memory states events in the traces.
+  * Add starpu_sched_component_estimated_end_min_add() to fix termination
+    estimations in modular schedulers.
+  * New function starpu_data_partition_not_automatic() to disable the
+    automatic partitioning of a data handle for which a asynchronous
+    plan has previously been submitted
 
 Changes:
   * Vastly improve simgrid simulation time.
   * Switch default scheduler to lws.
   * Add "to" parameter to pull_task and can_push methods of components.
+  * Deprecate handle_to_pointer interface operation in favor of new to_pointer
+  operation.
 
 Small changes:
   * Use asynchronous transfers for task data fetches with were not prefetched.
@@ -88,6 +124,17 @@ Small changes:
     scheduler context
   * Fonction starpu_is_initialized() is moved to the public API.
 
+StarPU 1.2.5 (git revision xxx)
+==============================================
+
+Small features:
+  * Add a new value STARPU_TASK_COLOR to be used in
+    starpu_task_insert() to pick up the color of a task in dag.dot
+  * Add starpu_data_pointer_is_inside().
+
+Changes:
+  * Do not export -lcuda -lcudart -lOpenCL in *starpu*.pc.
+
 StarPU 1.2.4 (git revision 255cf98175ef462749780f30bfed21452b74b594)
 ==============================================
 
@@ -105,6 +152,7 @@ Small features:
      acquisitions too.
    * Add a way to choose the dag.dot colors.
 
+
 StarPU 1.2.3 (svn revision 22444)
 ==============================================
 

Makefile.am

@@ -2,7 +2,7 @@
 #
 # Copyright (C) 2011-2017                                Inria
 # Copyright (C) 2009-2017                                Université de Bordeaux
-# Copyright (C) 2010-2017                                CNRS
+# Copyright (C) 2010-2018                                CNRS
 #
 # StarPU is free software; you can redistribute it and/or modify
 # it under the terms of the GNU Lesser General Public License as published by
@@ -126,17 +126,17 @@ else
 all-local:
 endif
 if STARPU_DEVEL
-	@if grep -r sys/time.h $$( find $(srcdir)/examples $(srcdir)/tests $(srcdir)/src $(srcdir)/mpi/src $(srcdir)/include -name \*.[ch] -a \! -name starpu_util.h -a \! -name timer.h -a \! -name loader.c ) ; \
+	@if $(GREP) -r sys/time.h $$( find $(srcdir)/examples $(srcdir)/tests $(srcdir)/src $(srcdir)/mpi/src $(srcdir)/include -name \*.[ch] -a \! -name starpu_util.h -a \! -name timer.h -a \! -name loader.c ) ; \
 	then \
 		echo "Please do not include sys/time, it is not available on Windows, include starpu_util.h and use starpu_timing_now() instead" ; \
 		false ; \
 	fi
-	@if grep -re '\<ssize_t' $$( find $(srcdir)/examples $(srcdir)/tests $(srcdir)/src $(srcdir)/mpi/src $(srcdir)/include -name \*.[ch] -a \! -name starpu_config.h ) ; \
+	@if $(GREP) -re '\<ssize_t' $$( find $(srcdir)/examples $(srcdir)/tests $(srcdir)/src $(srcdir)/mpi/src $(srcdir)/include -name \*.[ch] -a \! -name starpu_config.h ) ; \
 	then \
 		echo "Please do not use ssize_t, it is not available on Windows, use starpu_ssize_t instead"; \
 		false ; \
 	fi
-	@if grep -re '\<getenv\>' $$( find $(srcdir)/src $(srcdir)/mpi/src $(srcdir)/include -name \*.[ch] -a \! -name starpu_util.h -a \! -name utils.c -a \! -name simgrid.h) ; \
+	@if $(GREP) -re '\<getenv\>' $$( find $(srcdir)/src $(srcdir)/mpi/src $(srcdir)/include -name \*.[ch] -a \! -name starpu_util.h -a \! -name utils.c -a \! -name simgrid.h) ; \
 	then \
 		echo "Please do not use getenv, use starpu_getenv instead, which catches unsafe uses"; \
 		false ; \
@@ -144,8 +144,8 @@ if STARPU_DEVEL
 # we count the number of files which include unistd.h
 # we count the number of files which properly include unistd.h i.e by first detecting if it's available
 # and then we check both numbers are the same ...a
-	@UNISTD_ALL_LINES=$(shell grep -B1 -rs "^#include <unistd.h>" $(srcdir)/src/ $(srcdir)/include/ $(srcdir)/mpi/src $(srcdir)/mpi/include  |grep -v dolib|grep -v -e "--" | tr '\012' '@' | sed 's/unistd.h>@/unistd.h>\n/g' | wc -l) ;\
-	UNISTD_CORRECT_LINES=$(shell grep -B1 -rs "^#include <unistd.h>" $(srcdir)/src/ $(srcdir)/include/ $(srcdir)/mpi/src $(srcdir)/mpi/include  |grep -v dolib|grep -v -e "--" | tr '\012' '@' | sed 's/unistd.h>@/unistd.h>\n/g' | grep '#ifdef .*HAVE_UNISTD_H.*:#include <unistd.h>' | wc -l) ;\
+	@UNISTD_ALL_LINES=$(shell $(GREP) -B1 -rs "^#include <unistd.h>" $(srcdir)/src/ $(srcdir)/include/ $(srcdir)/mpi/src $(srcdir)/mpi/include  |$(GREP) -v dolib|$(GREP) -v -e "--" | tr '\012' '@' | $(SED) 's/unistd.h>@/unistd.h>\n/g' | wc -l) ;\
+	UNISTD_CORRECT_LINES=$(shell $(GREP) -B1 -rs "^#include <unistd.h>" $(srcdir)/src/ $(srcdir)/include/ $(srcdir)/mpi/src $(srcdir)/mpi/include  |$(GREP) -v dolib|$(GREP) -v -e "--" | tr '\012' '@' | $(SED) 's/unistd.h>@/unistd.h>\n/g' | $(GREP) '#ifdef .*HAVE_UNISTD_H.*:#include <unistd.h>' | wc -l) ;\
 	if test $$UNISTD_ALL_LINES -ne $$UNISTD_CORRECT_LINES ; \
 	then \
 		echo "Please do not unconditionally include unistd.h, it is not available on Windows, include config.h and test for HAVE_UNISTD_H" ; \
@@ -205,7 +205,7 @@ showcheck:
 
 ctags-local:
 	cd $(top_srcdir) ; $(CTAGS) -R -I LIST_TYPE
-	sed -i $(top_srcdir)/tags -e '/^[^	]* [^	]*	/d' -e '/^[^	]*$$/d' 
+	$(SED) -i $(top_srcdir)/tags -e '/^[^	]* [^	]*	/d' -e '/^[^	]*$$/d' 
 
 
 # Cyclomatic complexity reports.

configure.ac

@@ -84,6 +84,8 @@ AC_PROG_F77
 AC_PROG_FC
 AC_PROG_GREP
 AC_PROG_EGREP
+AC_PROG_INSTALL
+AC_PROG_MKDIR_P
 AC_CHECK_PROGS(PROG_STAT,gstat stat)
 AC_CHECK_PROGS(PROG_DATE,gdate date)
 AC_OPENMP
@@ -173,7 +175,7 @@ if test x$enable_simgrid = xyes ; then
 
 	# Latest functions
 	AC_CHECK_FUNCS([MSG_process_attach MSG_zone_get_hosts MSG_process_self_name])
-	AC_CHECK_FUNCS([xbt_mutex_try_acquire smpi_process_set_user_data sg_zone_get_by_name sg_link_name sg_host_route sg_host_self sg_host_speed simcall_process_create])
+	AC_CHECK_FUNCS([xbt_mutex_try_acquire smpi_process_set_user_data sg_zone_get_by_name sg_link_name sg_host_route sg_host_self sg_host_speed simcall_process_create sg_config_continue_after_help])
 	AC_CHECK_FUNCS([xbt_barrier_init], [AC_DEFINE([STARPU_SIMGRID_HAVE_XBT_BARRIER_INIT], [1], [Define to 1 if you have the `xbt_barrier_init' function.])])
 	AC_CHECK_DECLS([smpi_process_set_user_data], [], [], [[#include <smpi/smpi.h>]])
 
@@ -246,10 +248,10 @@ if test x$enable_blocking = xno ; then
 fi
 
 if test x$enable_blocking = xyes ; then
-AC_MSG_CHECKING(whether worker callbacks should be enabled)
-AC_ARG_ENABLE(worker-callbacks, [AS_HELP_STRING([--enable-worker-callbacks], [enable worker callbacks])],
+	AC_MSG_CHECKING(whether worker callbacks should be enabled)
+	AC_ARG_ENABLE(worker-callbacks, [AS_HELP_STRING([--enable-worker-callbacks], [enable worker callbacks])],
 				enable_worker_cb=$enableval, enable_worker_cb=no)
-AC_MSG_RESULT($enable_worker_cb)
+	AC_MSG_RESULT($enable_worker_cb)
 else
 	# worker sleep/wake-up callbacks only make sense if blocking drivers are enabled
 	enable_worker_cb=no
@@ -394,7 +396,7 @@ AC_SUBST(CC_OR_MPICC, $cc_or_mpicc)
 AC_ARG_ENABLE(mpi-pedantic-isend, [AS_HELP_STRING([--enable-mpi-pedantic-isend],
 				   [Enable StarPU MPI pedantic isend])],
 				   enable_mpi_pedantic_isend=$enableval, enable_mpi_pedantic_isend=no)
-if  test x$enable_mpi_pedantic_isend = xyes; then
+if test x$enable_mpi_pedantic_isend = xyes; then
 	AC_DEFINE(STARPU_MPI_PEDANTIC_ISEND, [1], [enable StarPU MPI pedantic isend])
 fi
 
@@ -470,7 +472,7 @@ AC_SUBST(CC_OR_MPICC, $cc_or_mpicc)
 if test x$use_mpi = xyes -a x$enable_nmad = xyes ; then
     build_nmad_lib=yes
     enable_mpi=no
-    PKG_CHECK_MODULES([NMAD],[nmad pioman tbx])
+    PKG_CHECK_MODULES([NMAD],[nmad])
 else
     build_nmad_lib=no
 fi
@@ -502,10 +504,6 @@ fi
 LT_PREREQ([2.2])
 LT_INIT([win32-dll])
 
-AC_PROG_INSTALL
-AC_PROG_MKDIR_P
-AC_PROG_LN_S
-
 AC_HEADER_STDC
 
 AC_C_RESTRICT
@@ -561,8 +559,7 @@ if test x$enable_simgrid = xno ; then
         ],
         [
             # nothing was specified: look in the path
-	    if test x$mpicc_path = x
-	    then
+	    if test x$mpicc_path = x ; then
 		AC_PATH_PROG(mpiexec_path, mpiexec, [no], [$PATH])
 	    else
 		AC_PATH_PROG(mpiexec_path, mpiexec, [no], [$(dirname $mpicc_path):$PATH])
@@ -575,8 +572,7 @@ if test x$enable_simgrid = xno ; then
     # We test if MPIEXEC exists
     if test ! -x $mpiexec_path; then
         # if it's not valid, it could be the parameter given to configure.ac was not a full path, let's look for it
-	if test x$mpicc_path = x
-	then
+	if test x$mpicc_path = x ; then
             AC_PATH_PROG(mpiexec_path_bis, $mpiexec_path, [no], [$PATH])
 	else
             AC_PATH_PROG(mpiexec_path_bis, $mpiexec_path, [no], [$(dirname $mpicc_path):$PATH])
@@ -752,8 +748,7 @@ case "$target" in
     				   enable_native_winthreads=$enableval, enable_native_winthreads=no)
     ;;
 esac
-if test x"$enable_native_winthreads" != xyes
-then
+if test x"$enable_native_winthreads" != xyes ; then
     INCLUDE_PTHREAD_H='#include <pthread.h>'
 fi
 
@@ -771,8 +766,7 @@ AC_CHECK_TYPE([struct timespec],
 $INCLUDE_PTHREAD_H
 ])
 
-if test x"$enable_native_winthreads" = xyes
-then
+if test x"$enable_native_winthreads" = xyes ; then
     CPPFLAGS+=" -I$STARPU_SRC_DIR/include/pthread_win32 "
     AC_COMPILE_IFELSE(
           [AC_LANG_PROGRAM([[
@@ -900,8 +894,7 @@ AC_CHECK_FUNCS([pread pwrite])
 AC_ARG_ENABLE(hdf5, [AS_HELP_STRING([--disable-hdf5], [disable HDF5 support])],
                     enable_hdf5=$enableval, enable_hdf5=maybe)
 
-if test "x$enable_hdf5" != xno
-then
+if test "x$enable_hdf5" != xno ; then
 	AC_ARG_WITH(hdf5-include-dir,
 		[AS_HELP_STRING([--with-hdf5-include-dir=<path>],
 		[specify where HDF5 headers are installed])],
@@ -2153,7 +2146,7 @@ if test x$use_fxt = xyes; then
 		AC_DEFINE(STARPU_HAVE_POTI, [1], [Define to 1 if you have libpoti and it is meant to be used])
 		save_LIBS="$LIBS"
 		LIBS="$LIBS $POTI_LIBS"
-		AC_CHECK_FUNCS([poti_init_custom])
+		AC_CHECK_FUNCS([poti_init_custom poti_user_NewEvent])
 		LIBS="$save_LIBS"
 		FXT_CFLAGS="$FXT_CFLAGS $POTI_CFLAGS"
 		FXT_LIBS="$FXT_LIBS $POTI_LIBS"
@@ -2274,8 +2267,7 @@ if test x$maxnodes = x0 ; then
 	if test x$enable_simgrid = xyes ; then
 		# We need the room for the virtual CUDA/OpenCL devices
 		nodes=`expr 4 + $nmaxcudadev + $nmaxopencldev + $nmaxmicdev + 1 + $nmaxmpidev`
-		if test $nodes -gt 32
-		then
+		if test $nodes -gt 32 ; then
 			nodes=32
 		fi
 	else
@@ -2593,6 +2585,9 @@ if test "x$STARPU_DEVEL" != x; then
 	IS_SUPPORTED_CFLAG(-Wundef)
 	IS_SUPPORTED_CFLAG(-Wshadow)
 	IS_SUPPORTED_CFLAG(-Werror=pointer-arith)
+	if test x$enable_debug = xyes; then
+		IS_SUPPORTED_CFLAG(-fno-optimize-sibling-calls)
+	fi
 fi
 AM_CONDITIONAL([STARPU_DEVEL],[test "x$STARPU_DEVEL" != x])
 
@@ -3026,8 +3021,7 @@ fi
 
 if test x$blas_lib = xmaybe -o x$blas_lib = xmkl; then
     # Should we use MKL ?
-    if test -n "$MKLROOT"
-    then
+    if test -n "$MKLROOT" ; then
         CPPFLAGS="${CPPFLAGS} -I$MKLROOT/include"
 	case $host_vendor in
 	    *1om) mkl_plat=mic ;;
@@ -3473,8 +3467,7 @@ if test "$enable_build_doc" = "yes" ; then
    fi
 fi
 available_doc="no"
-if test -f "$srcdir/doc/doxygen/starpu.pdf"
-then
+if test -f "$srcdir/doc/doxygen/starpu.pdf" ; then
    enable_build_doc="no"
    available_doc="yes"
 fi
@@ -3493,7 +3486,7 @@ AM_CONDITIONAL(AVAILABLE_DOC, [test x$available_doc != xno])
 ###############################################################################
 
 # these are the flags needed for linking libstarpu (and thus also for static linking)
-LIBSTARPU_LDFLAGS="$HWLOC_LIBS $FXT_LIBS $STARPU_COI_LDFLAGS $STARPU_SCIF_LDFLAGS $STARPU_RCCE_LDFLAGS $STARPU_LEVELDB_LDFLAGS $STARPU_GLPK_LDFLAGS $STARPU_LEVELDB_LDFLAGS $SIMGRID_LIBS $STARPU_BLAS_LDFLAGS $STARPU_OMP_LDFLAGS $DGELS_LIBS"
+LIBSTARPU_LDFLAGS="$STARPU_OPENCL_LDFLAGS $STARPU_CUDA_LDFLAGS $HWLOC_LIBS $FXT_LIBS $STARPU_COI_LDFLAGS $STARPU_SCIF_LDFLAGS $STARPU_RCCE_LDFLAGS $STARPU_LEVELDB_LDFLAGS $STARPU_GLPK_LDFLAGS $STARPU_LEVELDB_LDFLAGS $SIMGRID_LIBS $STARPU_BLAS_LDFLAGS $STARPU_OMP_LDFLAGS $DGELS_LIBS"
 AC_SUBST([LIBSTARPU_LDFLAGS])
 
 LIBSTARPU_LINK=libstarpu-$STARPU_EFFECTIVE_VERSION.la
@@ -3547,6 +3540,7 @@ AC_CONFIG_COMMANDS([executable-scripts], [
   test -e examples/heat/heat.sh || ln -sf $ac_abs_top_srcdir/examples/heat/heat.sh examples/heat/
   mkdir -p examples/lu
   test -e examples/lu/lu.sh || ln -sf $ac_abs_top_srcdir/examples/lu/lu.sh examples/lu/
+  test -e examples/cholesky/cholesky.sh || ln -sf $ac_abs_top_srcdir/examples/cholesky/cholesky.sh examples/cholesky/
   test -e tools/starpu_paje_draw_histogram.R || ln -sf $ac_abs_top_srcdir/tools/starpu_paje_draw_histogram.R tools/starpu_paje_draw_histogram.R
   test -e tools/starpu_paje_state_stats.R || ln -sf $ac_abs_top_srcdir/tools/starpu_paje_state_stats.R tools/starpu_paje_state_stats.R
   test -e tools/starpu_trace_state_stats.py || ln -sf $ac_abs_top_srcdir/tools/starpu_trace_state_stats.py tools/starpu_trace_state_stats.py
@@ -3707,16 +3701,14 @@ To run the tests, you need to install the OCL implementation of ICD
 and set the variable SOCL_OCL_LIB_OPENCL to the location of the libOpenCL.so.])
 fi
 
-if test x"$have_valid_hwloc" = xno -a "$enable_simgrid" = "no"
-then
+if test x"$have_valid_hwloc" = xno -a "$enable_simgrid" = "no" ; then
   AC_MSG_NOTICE([
 WARNING: hwloc was not enabled.  If the target machine is hyperthreaded the
 performance may be impacted a lot.  It is strongly recommended to install
 hwloc])
 fi
 
-if test x"$starpu_windows" = xyes -a "x$STARPU_MS_LIB" = "x"
-then
+if test x"$starpu_windows" = xyes -a "x$STARPU_MS_LIB" = "x" ; then
   AC_MSG_NOTICE([
 WARNING: lib was not found, you will not be able to build StarPU applications
 with Microsoft Visual Studio. Add to your PATH the directories for MSVC, e.g

doc/doxygen/chapters/110_basic_examples.doxy

@@ -307,7 +307,7 @@ struct starpu_codelet cl =
     .where = STARPU_CPU,
     .cpu_funcs = { cpu_func },
     .cpu_funcs_name = { "cpu_func" },
-     .nbuffers = 0
+    .nbuffers = 0
 };
 \endcode
 
@@ -338,13 +338,10 @@ has to be defined:
 
 \code{.c}
 /* Declare the `vector_scal' task.  */
-static void vector_scal (unsigned size, float vector[size],
-                         float factor)
-  __attribute__ ((task));
+static void vector_scal (unsigned size, float vector[size], float factor) __attribute__ ((task));
 
 /* Define the standard CPU implementation.  */
-static void
-vector_scal (unsigned size, float vector[size], float factor)
+static void vector_scal (unsigned size, float vector[size], float factor)
 {
   unsigned i;
   for (i = 0; i < size; i++)
@@ -415,12 +412,10 @@ in our C file like this:
 /* The OpenCL programs, loaded from 'main' (see below). */
 static struct starpu_opencl_program cl_programs;
 
-static void vector_scal_opencl (unsigned size, float vector[size],
-                                float factor)
+static void vector_scal_opencl (unsigned size, float vector[size], float factor)
   __attribute__ ((task_implementation ("opencl", vector_scal)));
 
-static void
-vector_scal_opencl (unsigned size, float vector[size], float factor)
+static void vector_scal_opencl (unsigned size, float vector[size], float factor)
 {
   int id, devid, err;
   cl_kernel kernel;
@@ -434,22 +429,17 @@ vector_scal_opencl (unsigned size, float vector[size], float factor)
   devid = starpu_worker_get_devid (id);
 
   /* Prepare to invoke the kernel.  In the future, this will be largely automated.  */
-  err = starpu_opencl_load_kernel (&kernel, &queue, &cl_programs,
-                                   "vector_mult_opencl", devid);
-  if (err != CL_SUCCESS)
-    STARPU_OPENCL_REPORT_ERROR (err);
+  err = starpu_opencl_load_kernel (&kernel, &queue, &cl_programs, "vector_mult_opencl", devid);
+  if (err != CL_SUCCESS) STARPU_OPENCL_REPORT_ERROR (err);
 
   err = clSetKernelArg (kernel, 0, sizeof (size), &size);
   err |= clSetKernelArg (kernel, 1, sizeof (val), &val);
   err |= clSetKernelArg (kernel, 2, sizeof (factor), &factor);
-  if (err)
-    STARPU_OPENCL_REPORT_ERROR (err);
+  if (err) STARPU_OPENCL_REPORT_ERROR (err);
 
   size_t global = 1, local = 1;
-  err = clEnqueueNDRangeKernel (queue, kernel, 1, NULL, &global,
-                                &local, 0, NULL, &event);
-  if (err != CL_SUCCESS)
-    STARPU_OPENCL_REPORT_ERROR (err);
+  err = clEnqueueNDRangeKernel (queue, kernel, 1, NULL, &global, &local, 0, NULL, &event);
+  if (err != CL_SUCCESS) STARPU_OPENCL_REPORT_ERROR (err);
 
   clFinish (queue);
   starpu_opencl_collect_stats (event);
@@ -464,8 +454,7 @@ The OpenCL kernel itself must be loaded from <c>main</c>, sometime after
 the pragma <c>initialize</c>:
 
 \code{.c}
-starpu_opencl_load_opencl_from_file ("vector_scal_opencl_kernel.cl",
-                                       &cl_programs, "");
+starpu_opencl_load_opencl_from_file ("vector_scal_opencl_kernel.cl", &cl_programs, "");
 \endcode
 
 And that's it.  The task <c>vector_scal</c> now has an additional
@@ -482,8 +471,7 @@ with <c>nvcc</c>.  Thus, the C file only needs to contain an external
 declaration for the task implementation:
 
 \code{.c}
-extern void vector_scal_cuda (unsigned size, float vector[size],
-                              float factor)
+extern void vector_scal_cuda (unsigned size, float vector[size], float factor)
   __attribute__ ((task_implementation ("cuda", vector_scal)));
 \endcode
 
@@ -532,8 +520,7 @@ The following lines show how to declare an array of <c>NX</c> elements of type
 float vector[NX];
 
 starpu_data_handle_t vector_handle;
-starpu_vector_data_register(&vector_handle, STARPU_MAIN_RAM, (uintptr_t)vector, NX,
-                            sizeof(vector[0]));
+starpu_vector_data_register(&vector_handle, STARPU_MAIN_RAM, (uintptr_t)vector, NX, sizeof(vector[0]));
 \endcode
 
 The first argument, called the <b>data handle</b>, is an opaque pointer which

doc/doxygen/chapters/210_check_list_performance.doxy

@@ -79,6 +79,11 @@ link to \ref StaticScheduling
 
 \section CUDA-specificOptimizations CUDA-specific Optimizations
 
+For proper overlapping of asynchronous GPU data transfers, data has to be pinned
+by CUDA. Data allocated with starpu_malloc() is always properly pinned. If the
+application is registering to StarPU some data which has not been allocated with
+starpu_malloc(), it should use starpu_memory_pin() to pin it.
+
 Due to CUDA limitations, StarPU will have a hard time overlapping its own
 communications and the codelet computations if the application does not use a
 dedicated CUDA stream for its computations instead of the default stream,
@@ -408,9 +413,7 @@ void feed(void)
         starpu_data_handle_t handle;
 	starpu_vector_data_register(&handle, -1, 0, measure->size, sizeof(float));
 	task.handles[0] = handle;
-	starpu_perfmodel_update_history(&perf_model, &task,
-	                                STARPU_CUDA_DEFAULT + measure->cudadev, 0,
-	                                measure->implementation, measure->time);
+	starpu_perfmodel_update_history(&perf_model, &task, STARPU_CUDA_DEFAULT + measure->cudadev, 0, measure->implementation, measure->time);
 	starpu_task_clean(&task);
 	starpu_data_unregister(handle);
     }

doc/doxygen/chapters/301_tasks.doxy

@@ -55,6 +55,66 @@ other tasks and may thus be a bottleneck if not executed early
 enough), the field starpu_task::priority should be set to transmit the
 priority information to StarPU.
 
+\section TaskDependencies Task Dependencies
+
+\subsection SequentialConsistency Sequential Consistency
+
+By default, task dependencies are inferred from data dependency (sequential
+coherency) by StarPU. The application can however disable sequential coherency
+for some data, and dependencies can be specifically expressed.
+
+Setting (or unsetting) sequential consistency can be done at the data
+level by calling starpu_data_set_sequential_consistency_flag() for a
+specific data or starpu_data_set_default_sequential_consistency_flag()
+for all datas.
+
+Setting (or unsetting) sequential consistency can also be done at task
+level by setting the field starpu_task::sequential_consistency to 0.
+
+Sequential consistency can also be set (or unset) for each handle of a
+specific task, this is done by using the field
+starpu_task::handles_sequential_consistency. When set, its value
+should be a array with the number of elements being the number of
+handles for the task, each element of the array being the sequential
+consistency for the i-th handle of the task. The field can easily be
+set when calling starpu_task_insert() with the flag
+::STARPU_HANDLES_SEQUENTIAL_CONSISTENCY
+
+\code{.c}
+char *seq_consistency = malloc(cl.nbuffers * sizeof(char));
+seq_consistency[0] = 1;
+seq_consistency[1] = 1;
+seq_consistency[2] = 0;
+ret = starpu_task_insert(&cl,
+	STARPU_RW, handleA, STARPU_RW, handleB, STARPU_RW, handleC,
+	STARPU_HANDLES_SEQUENTIAL_CONSISTENCY, seq_consistency,
+	0);
+free(seq_consistency);
+\endcode
+
+The internal algorithm used by StarPU to set up implicit dependency is
+as follows:
+\code{.c}
+if (sequential_consistency(task) == 1)
+    for(i=0 ; i<STARPU_TASK_GET_NBUFFERS(task) ; i++)
+      if (sequential_consistency(i-th data, task) == 1)
+        if (sequential_consistency(i-th data) == 1)
+           create_implicit_dependency(...)
+\endcode
+
+\subsection TasksAndTagsDependencies Tasks And Tags Dependencies
+
+One can explicitely set dependencies between tasks using
+starpu_task_declare_deps_array(). Dependencies between tasks can be
+expressed through tags associated to a tag with the field
+starpu_task::tag_id and using the function starpu_tag_declare_deps()
+or starpu_tag_declare_deps_array().
+
+The termination of a task can be delayed through the function
+starpu_task_end_dep_add() which specifies the number of calls to the
+function starpu_task_end_dep_release() needed to trigger the task
+termination.
+
 \section SettingManyDataHandlesForATask Setting Many Data Handles For a Task
 
 The maximum number of data a task can manage is fixed by the environment variable
@@ -98,9 +158,9 @@ for(i=0 ; i<dummy_big_cl.nbuffers ; i++)
 	handles[i] = handle;
 }
 starpu_task_insert(&dummy_big_cl,
-        	 STARPU_VALUE, &dummy_big_cl.nbuffers, sizeof(dummy_big_cl.nbuffers),
-		 STARPU_DATA_ARRAY, handles, dummy_big_cl.nbuffers,
-		 0);
+         	  STARPU_VALUE, &dummy_big_cl.nbuffers, sizeof(dummy_big_cl.nbuffers),
+		  STARPU_DATA_ARRAY, handles, dummy_big_cl.nbuffers,
+		  0);
 \endcode
 
 The whole code for this complex data interface is available in the
@@ -355,9 +415,7 @@ starpu_task_insert(&which_index, STARPU_W, i_handle, 0);
 
 /* And submit the corresponding task */
 STARPU_DATA_ACQUIRE_CB(i_handle, STARPU_R,
-                       starpu_task_insert(&work,
-		                          STARPU_RW, A_handle[i],
-					  0));
+                       starpu_task_insert(&work, STARPU_RW, A_handle[i], 0));
 \endcode
 
 The macro ::STARPU_DATA_ACQUIRE_CB submits an asynchronous request for

doc/doxygen/chapters/310_data_management.doxy

@@ -261,8 +261,7 @@ int vector[NX];
 starpu_data_handle_t handle;
 
 /* Declare data to StarPU */
-starpu_vector_data_register(&handle, STARPU_MAIN_RAM, (uintptr_t)vector,
-                            NX, sizeof(vector[0]));
+starpu_vector_data_register(&handle, STARPU_MAIN_RAM, (uintptr_t)vector, NX, sizeof(vector[0]));
 
 /* Partition the vector in PARTS sub-vectors */
 struct starpu_data_filter f =
@@ -428,8 +427,8 @@ struct starpu_codelet cl_switch =
 };
 
 ret = starpu_task_insert(&cl_switch, STARPU_RW, handle,
-			STARPU_W, vert_handle[0], 
-			STARPU_W, vert_handle[1], 
+			STARPU_W, vert_handle[0],
+			STARPU_W, vert_handle[1],
 			0);
 \endcode
 
@@ -510,8 +509,7 @@ and attaches them as reduction methods for its handle <c>dtq</c>:
 
 \code{.c}
 starpu_variable_data_register(&dtq_handle, -1, NULL, sizeof(type));
-starpu_data_set_reduction_methods(dtq_handle,
-        &accumulate_variable_cl, &bzero_variable_cl);
+starpu_data_set_reduction_methods(dtq_handle, &accumulate_variable_cl, &bzero_variable_cl);
 \endcode
 
 and <c>dtq_handle</c> can now be used in mode ::STARPU_REDUX for the
@@ -554,8 +552,7 @@ with a new reduction:
 for (i = 0; i < 100; i++)
 {
     starpu_mpi_task_insert(MPI_COMM_WORLD, &init_res, STARPU_W, res, 0);
-    starpu_mpi_task_insert(MPI_COMM_WORLD, &work, STARPU_RW, A,
-               STARPU_R, B, STARPU_REDUX, res, 0);
+    starpu_mpi_task_insert(MPI_COMM_WORLD, &work, STARPU_RW, A, STARPU_R, B, STARPU_REDUX, res, 0);
     starpu_mpi_redux_data(MPI_COMM_WORLD, res);
     starpu_mpi_task_insert(MPI_COMM_WORLD, &work2, STARPU_RW, B, STARPU_R, res, 0);
 }
@@ -573,22 +570,10 @@ the ::STARPU_COMMUTE data access flag. Accesses without this flag will however
 properly be serialized against accesses with this flag. For instance:
 
 \code{.c}
-    starpu_task_insert(&cl1,
-        STARPU_R, h,
-        STARPU_RW, handle,
-        0);
-    starpu_task_insert(&cl2,
-        STARPU_R, handle1,
-        STARPU_RW|STARPU_COMMUTE, handle,
-        0);
-    starpu_task_insert(&cl2,
-        STARPU_R, handle2,
-        STARPU_RW|STARPU_COMMUTE, handle,
-        0);
-    starpu_task_insert(&cl3,
-        STARPU_R, g,
-        STARPU_RW, handle,
-        0);
+    starpu_task_insert(&cl1, STARPU_R, h, STARPU_RW, handle, 0);
+    starpu_task_insert(&cl2, STARPU_R, handle1, STARPU_RW|STARPU_COMMUTE, handle, 0);
+    starpu_task_insert(&cl2, STARPU_R, handle2, STARPU_RW|STARPU_COMMUTE, handle, 0);
+    starpu_task_insert(&cl3, STARPU_R, g, STARPU_RW, handle, 0);
 \endcode
 
 The two tasks running <c>cl2</c> will be able to commute: depending on whether the
@@ -680,8 +665,7 @@ memory for it, and StarPU will allocate it on demand at task execution.
 \code{.c}
 starpu_vector_data_register(&workspace, -1, 0, sizeof(float));
 for (i = 0; i < N; i++)
-    starpu_task_insert(&compute, STARPU_R, input[i],
-                       STARPU_SCRATCH, workspace, STARPU_W, output[i], 0);
+    starpu_task_insert(&compute, STARPU_R, input[i], STARPU_SCRATCH, workspace, STARPU_W, output[i], 0);
 \endcode
 
 StarPU will make sure that the buffer is allocated before executing the task,
@@ -856,8 +840,16 @@ main memory instead of copied in the GPU, a pivoting vector for instance.
 This can be achieved by setting the starpu_codelet::specific_nodes flag to
 <c>1</c>, and then fill the starpu_codelet::nodes array (or starpu_codelet::dyn_nodes when
 starpu_codelet::nbuffers is greater than \ref STARPU_NMAXBUFS) with the node numbers
-where data should be copied to, or <c>-1</c> to let StarPU copy it to the memory node
-where the task will be executed. For instance, with the following codelet:
+where data should be copied to, or <c>STARPU_SPECIFIC_NODE_LOCAL</c> to let
+StarPU copy it to the memory node where the task will be executed.
+
+<c>STARPU_SPECIFIC_NODE_CPU</c> can also be used to request data to be
+put in CPU-accessible memory (and let StarPU choose the NUMA node).
+<c>STARPU_SPECIFIC_NODE_FAST</c> and <c>STARPU_SPECIFIC_NODE_SLOW</c> can als be
+used
+
+For instance,
+with the following codelet:
 
 \code{.c}
 struct starpu_codelet cl =
@@ -866,12 +858,31 @@ struct starpu_codelet cl =
 	.nbuffers = 2,
 	.modes = {STARPU_RW, STARPU_RW},
 	.specific_nodes = 1,
-	.nodes = {STARPU_MAIN_RAM, -1},
+	.nodes = {STARPU_SPECIFIC_NODE_CPU, STARPU_SPECIFIC_NODE_LOCAL},
 };
 \endcode
 
-the first data of the task will be kept in the main memory, while the second
+the first data of the task will be kept in the CPU memory, while the second
 data will be copied to the CUDA GPU as usual. A working example is available in
 <c>tests/datawizard/specific_node.c</c>
 
+With the following codelet:
+
+\code{.c}
+struct starpu_codelet cl =
+{
+	.cuda_funcs = { kernel },
+	.nbuffers = 2,
+	.modes = {STARPU_RW, STARPU_RW},
+	.specific_nodes = 1,
+	.nodes = {STARPU_SPECIFIC_NODE_LOCAL, STARPU_SPECIFIC_NODE_SLOW},
+};
+\endcode
+
+The first data will be copied into fast (but probably size-limited) local memory
+while the second data will be left in slow (but large) memory. This makes sense
+when the kernel does not make so many accesses to the second data, and thus data
+being remote e.g. over a PCI bus is not a performance problem, and avoids
+filling the fast local memory with data which does not need the performance.
+
 */

doc/doxygen/chapters/320_scheduling.doxy

@@ -250,7 +250,7 @@ methods of the policy.
 
 Make sure to have a look at the \ref API_Scheduling_Policy section, which
 provides a list of the available functions for writing advanced schedulers, such
-as starpu_task_expected_length(), starpu_task_expected_data_transfer_time(),
+as starpu_task_expected_length(), starpu_task_expected_data_transfer_time_for(),
 starpu_task_expected_energy(), etc. Other
 useful functions include starpu_transfer_bandwidth(), starpu_transfer_latency(),
 starpu_transfer_predict(), ...

doc/doxygen/chapters/330_scheduling_contexts.doxy

@@ -1,7 +1,7 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
  * Copyright (C) 2011-2013,2016                           Inria
- * Copyright (C) 2010-2017                                CNRS
+ * Copyright (C) 2010-2018                                CNRS
  * Copyright (C) 2009-2011,2014                           Université de Bordeaux
  * Copyright (C) 2016                                     Uppsala University
  *
@@ -113,7 +113,7 @@ The contexts can also be used to group set of SMs of an NVIDIA GPU in order to i
 the parallel kernels and allow them to coexecution on a specified partiton of the GPU.
 
 Each context will be mapped to a stream and the user can indicate the number of SMs.
-The context can be added to a larger context already grouping CPU cores. 
+The context can be added to a larger context already grouping CPU cores.
 This larger context can use a scheduling policy that assigns tasks to both CPUs and contexts (partitions of the GPU)
 based on performance models adjusted to the number of SMs.
 
@@ -132,7 +132,7 @@ int workers[ncpus+nstreams];
 workers[ncpus+0] = stream_workerids[0];
 workers[ncpus+1] = stream_workerids[1];
 
-big_sched_ctx = starpu_sched_ctx_create(workers, ncpus+nstreams, "ctx1", STARPU_SCHED_CTX_SUB_CTXS, sched_ctxs, nstreams, STARPU_SCHED_CTX_POLICY_NAME, "dmdas", 0); 
+big_sched_ctx = starpu_sched_ctx_create(workers, ncpus+nstreams, "ctx1", STARPU_SCHED_CTX_SUB_CTXS, sched_ctxs, nstreams, STARPU_SCHED_CTX_POLICY_NAME, "dmdas", 0);
 
 starpu_task_submit_to_ctx(task, big_sched_ctx);
 
@@ -160,25 +160,25 @@ starpu_sched_ctx_remove_workers(workerids, 3, sched_ctx1);
 \endcode
 
 \section SubmittingTasksToAContext Submitting Tasks To A Context
-The application may submit tasks to several contexts either 
+The application may submit tasks to several contexts either
 simultaneously or sequnetially. If several threads of submission
 are used the function starpu_sched_ctx_set_context() may be called just
-before starpu_task_submit(). Thus StarPU considers that 
+before starpu_task_submit(). Thus StarPU considers that
 the current thread will submit tasks to the coresponding context.
- 
+
 When the application may not assign a thread of submission to each
 context, the id of the context must be indicated by using the
-function starpu_task_submit_to_ctx() or the field \ref STARPU_SCHED_CTX 
+function starpu_task_submit_to_ctx() or the field \ref STARPU_SCHED_CTX
 for starpu_task_insert().
 
 \section DeletingAContext Deleting A Context
 
 When a context is no longer needed it must be deleted. The application
 can indicate which context should keep the resources of a deleted one.
-All the tasks of the context should be executed before doing this. 
-Thus, the programmer may use either a barrier and then delete the context 
+All the tasks of the context should be executed before doing this.
+Thus, the programmer may use either a barrier and then delete the context
 directly, or just indicate
-that other tasks will not be submitted later on to the context (such that when 
+that other tasks will not be submitted later on to the context (such that when
 the last task is executed its workers will be moved to the inheritor)
 and delete the context at the end of the execution (when a barrier will
 be used eventually).
@@ -212,8 +212,8 @@ A context may have no resources at the begining or at a certain
 moment of the execution. Task can still be submitted to these contexts
 and they will be executed as soon as the contexts will have resources. A list
 of tasks pending to be executed is kept and when workers are added to
-the contexts these tasks start being submitted. However, if resources 
-are never allocated to the context the program will not terminate. 
+the contexts these tasks start being submitted. However, if resources
+are never allocated to the context the program will not terminate.
 If these tasks have low
 priority the programmer can forbid the application to submit them
 by calling the function starpu_sched_ctx_stop_task_submission().

doc/doxygen/chapters/350_modularized_scheduler.doxy

@@ -1,7 +1,7 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
  * Copyright (C) 2013                                     Inria
- * Copyright (C) 2014,2016-2017                           CNRS
+ * Copyright (C) 2014,2016-2018                           CNRS
  * Copyright (C) 2014,2017                                Université de Bordeaux
  * Copyright (C) 2013                                     Simon Archipoff
  *
@@ -204,23 +204,19 @@ static void initialize_eager_prefetching_center_policy(unsigned sched_ctx_id)
     (sched_ctx_id, STARPU_WORKER_LIST);
 
   /* Create the Scheduling Tree */
-  struct starpu_sched_tree * t =
-    starpu_sched_tree_create(sched_ctx_id);
+  struct starpu_sched_tree * t = starpu_sched_tree_create(sched_ctx_id);
 
   /* The Root Component is a Flow-control Fifo Component */
    t->root = starpu_sched_component_fifo_create(NULL);
 
   /* The Resource-mapping Component of the strategy is an Eager Component
    */
-  struct starpu_sched_component * eager_component =
-    starpu_sched_component_eager_create(NULL);
+  struct starpu_sched_component *eager_component = starpu_sched_component_eager_create(NULL);
 
   /* Create links between Components : the Eager Component is the child
    * of the Root Component */
-  t->root->add_child
-    (t->root, eager_component);
-  eager_component->add_father
-    (eager_component, t->root);
+  t->root->add_child(t->root, eager_component);
+  eager_component->add_father(eager_component, t->root);
 
   /* A task threshold is set for the Flow-control Components which will
    * be connected to Worker Components. By doing so, this Modularized
@@ -233,44 +229,32 @@ static void initialize_eager_prefetching_center_policy(unsigned sched_ctx_id)
   };
 
   unsigned i;
-  for(i = 0;
-    i < starpu_worker_get_count() +
-    starpu_combined_worker_get_count();
-    i++)
+  for(i = 0; i < starpu_worker_get_count() + starpu_combined_worker_get_count(); i++)
   {
     /* Each Worker Component has a Flow-control Fifo Component as
      * father */
-    struct starpu_sched_component * worker_component =
-	  starpu_sched_component_worker_new(i);
-    struct starpu_sched_component * fifo_component =
-	  starpu_sched_component_fifo_create(&fifo_data);
-    fifo_component->add_child
-      (fifo_component, worker_component);
-    worker_component->add_father
-      (worker_component, fifo_component);
+    struct starpu_sched_component * worker_component = starpu_sched_component_worker_new(i);
+    struct starpu_sched_component * fifo_component = starpu_sched_component_fifo_create(&fifo_data);
+    fifo_component->add_child(fifo_component, worker_component);
+    worker_component->add_father(worker_component, fifo_component);
 
     /* Each Flow-control Fifo Component associated to a Worker
      * Component is linked to the Eager Component as one of its
      * children */
-    eager_component->add_child
-      (eager_component, fifo_component);
-    fifo_component->add_father
-      (fifo_component, eager_component);
+    eager_component->add_child(eager_component, fifo_component);
+    fifo_component->add_father(fifo_component, eager_component);
   }
 
   starpu_sched_tree_update_workers(t);
-  starpu_sched_ctx_set_policy_data
-    (sched_ctx_id, (void*)t);
+  starpu_sched_ctx_set_policy_data(sched_ctx_id, (void*)t);
 }
 
 /* Properly destroy the Scheduling Tree and all its Components */
 static void deinitialize_eager_prefetching_center_policy(unsigned sched_ctx_id)
 {
-  struct starpu_sched_tree * tree =
-  	(struct starpu_sched_tree*)starpu_sched_ctx_get_policy_data(sched_ctx_id);
+  struct starpu_sched_tree * tree = (struct starpu_sched_tree*)starpu_sched_ctx_get_policy_data(sched_ctx_id);
   starpu_sched_tree_destroy(tree);
-  starpu_sched_ctx_delete_worker_collection
-    (sched_ctx_id);
+  starpu_sched_ctx_delete_worker_collection(sched_ctx_id);
 }
 
 /* Initializing the starpu_sched_policy struct associated to the Modularized

doc/doxygen/chapters/370_online_performance_tools.doxy

@@ -1,8 +1,8 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2010-2017                                CNRS
+ * Copyright (C) 2010-2018                                CNRS
  * Copyright (C) 2011-2012,2016                           Inria
- * Copyright (C) 2009-2011,2014,2016, 2018                      Université de Bordeaux
+ * Copyright (C) 2009-2011,2014,2016, 2018                Université de Bordeaux
  *
  * StarPU is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
@@ -313,12 +313,9 @@ for (worker = 0; worker < starpu_worker_get_count(); worker++)
         starpu_worker_get_name(worker, workername, 128);
         fprintf(stderr, "Worker %s:\n", workername);
         fprintf(stderr, "\ttotal time: %.2lf ms\n", total_time*1e-3);
-        fprintf(stderr, "\texec time: %.2lf ms (%.2f %%)\n",
-                executing_time*1e-3, executing_ratio);
-        fprintf(stderr, "\tblocked time: %.2lf ms (%.2f %%)\n",
-                sleeping_time*1e-3, sleeping_ratio);
-        fprintf(stderr, "\toverhead time: %.2lf ms (%.2f %%)\n",
-                overhead_time*1e-3, overhead_ratio);
+        fprintf(stderr, "\texec time: %.2lf ms (%.2f %%)\n", executing_time*1e-3, executing_ratio);
+        fprintf(stderr, "\tblocked time: %.2lf ms (%.2f %%)\n", sleeping_time*1e-3, sleeping_ratio);
+        fprintf(stderr, "\toverhead time: %.2lf ms (%.2f %%)\n", overhead_time*1e-3, overhead_ratio);
 }
 \endcode
 

doc/doxygen/chapters/380_offline_performance_tools.doxy

@@ -1,7 +1,7 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
  * Copyright (C) 2011-2012,2015-2017                      Inria
- * Copyright (C) 2010-2017                                CNRS
+ * Copyright (C) 2010-2018                                CNRS
  * Copyright (C) 2009-2011,2014-2017                      Université de Bordeaux
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -108,10 +108,8 @@ StarPU's internal locking.
 The environment variable \ref STARPU_FXT_TRACE can be set to 0 to disable the
 generation of the <c>prof_file_XXX_YYY</c> file.
 
-\subsection CreatingAGanttDiagram Creating a Gantt Diagram
-
-When the FxT trace file <c>prof_file_something</c> has been generated, it is possible to
-generate a trace in the Paje format by calling:
+When the FxT trace file <c>prof_file_something</c> has been generated,
+it is possible to generate different trace formats by calling:
 
 \verbatim
 $ starpu_fxt_tool -i /tmp/prof_file_something
@@ -121,46 +119,67 @@ Or alternatively, setting the environment variable \ref STARPU_GENERATE_TRACE
 to <c>1</c> before application execution will make StarPU do it automatically at
 application shutdown.
 
-This will create a file <c>paje.trace</c> in the current directory that
-can be inspected with the ViTE (http://vite.gforge.inria.fr/) trace
-visualizing open-source tool.  It is possible to open the
-file <c>paje.trace</c> with ViTE by using the following command:
+One can also set the environment variable \ref
+STARPU_GENERATE_TRACE_OPTIONS to specify options, see
+<c>starpu_fxt_tool --help</c>, for example:
 
 \verbatim
-$ vite paje.trace
+$ export STARPU_GENERATE_TRACE=1
+$ export STARPU_GENERATE_TRACE_OPTIONS="-no-acquire"
 \endverbatim
 
-To get names of tasks instead of "unknown", fill the optional
-starpu_codelet::name, or use a performance model for them.
-
-One can also introduce user-defined events in the diagram thanks to the
-starpu_fxt_trace_user_event_string() function.
-
-In the MPI execution case, \ref STARPU_GENERATE_TRACE will not work as expected
-(each node will try to generate paje.trace, thus mixing outputs...), you have to
-collect the trace files from the MPI nodes, and
-specify them all on the command <c>starpu_fxt_tool</c>, for instance:
+When running a MPI application, \ref STARPU_GENERATE_TRACE will not
+work as expected (each node will try to generate trace files, thus
+mixing outputs...), you have to collect the trace files from the MPI
+nodes, and specify them all on the command <c>starpu_fxt_tool</c>, for
+instance:
 
 \verbatim
 $ starpu_fxt_tool -i /tmp/prof_file_something*
 \endverbatim
 
-By default, all tasks are displayed using a green color. To display tasks with
-varying colors, pass option <c>-c</c> to <c>starpu_fxt_tool</c>.
+By default, the generated trace contains all informations. To reduce
+the trace size, various <c>-no-foo</c> options can be passed to
+<c>starpu_fxt_tool</c>, see <c>starpu_fxt_tool --help</c> .
+
+\subsubsection CreatingAGanttDiagram Creating a Gantt Diagram
+
+One of the generated files is a trace in the Paje format. The file,
+located in the current directory, is named <c>paje.trace</c>. It can
+be viewed with ViTE (http://vite.gforge.inria.fr/) a trace
+visualizing open-source tool.  To open the file <c>paje.trace</c> with
+ViTE, use the following command:
 
-By default, the trace contains all informations. To reduce the trace size,
-various <c>-no-foo</c> options can be passed to <c>starpu_fxt_tool</c>, see
-<c>starpu_fxt_tool --help</c> .
+\verbatim
+$ vite paje.trace
+\endverbatim
 
-To identify tasks precisely, the application can set the starpu_task::tag_id field of the
-task (or use \ref STARPU_TAG_ONLY when using starpu_task_insert()), and
-the value of the tag will show up in the trace.
+Tasks can be assigned a name (instead of the default 'unknown') by
+filling the optional starpu_codelet::name, or assigning them a
+performance model. The name can also be set with the field
+starpu_task::name or by using \ref STARPU_NAME when calling
+starpu_task_insert().
+
+Tasks are assigned default colors based on the worker which executed
+them (green for CPUs, yellow/orange/red for CUDAs, blue for OpenCLs,
+red for MICs, ...). To use a different color for every type of task,
+one can specify the option <c>-c</c> to <c>starpu_fxt_tool</c> or in
+\ref STARPU_GENERATE_TRACE_OPTIONS. Tasks can also be given a specific
+color by setting the field starpu_codelet::color or the
+starpu_task::color. Colors are expressed with the following format
+0xRRGGBB (e.g 0xFF0000 for red). See
+<c>basic_examples/task_insert_color</c> for examples on how to assign
+colors.
+
+To identify tasks precisely, the application can also set the field
+starpu_task::tag_id or setting \ref STARPU_TAG_ONLY when calling
+starpu_task_insert(). The value of the tag will then show up in the
+trace.
 
-It can also set the starpu_task::name field of the task (or use \ref STARPU_NAME)
-when using starpu_task_insert()), to replace in traces the name of the codelet
-with an arbitrarily chosen name.
+One can also introduce user-defined events in the diagram thanks to the
+starpu_fxt_trace_user_event_string() function.
 
-It can also set the iteration number, by just calling starpu_iteration_push()
+One can also set the iteration number, by just calling starpu_iteration_push()
 at the beginning of submission loops and starpu_iteration_pop() at the end of
 submission loops. These iteration numbers will show up in traces for all tasks
 submitted from there.
@@ -177,33 +196,23 @@ $ fxt_print -o -f /tmp/prof_file_something
 
 Timings are in nanoseconds (while timings as seen in ViTE are in milliseconds).
 
-\subsection CreatingADAGWithGraphviz Creating a DAG With Graphviz
+\subsubsection CreatingADAGWithGraphviz Creating a DAG With Graphviz
 
-When the FxT trace file <c>prof_file_something</c> has been generated, it is possible to
-generate a task graph in the DOT format by calling:
+Another generated trace file is a task graph described using the DOT
+language. The file, created in the current directory, is named
+<c>dag.dot</c> file in the current directory.
+It is possible to get a graphical output of the graph by using the
+<c>graphviz</c> library:
 
-\verbatim
-$ starpu_fxt_tool -i /tmp/prof_file_something
-\endverbatim
-
-This will create a <c>dag.dot</c> file in the current directory. This file is a
-task graph described using the DOT language. It is possible to get a
-graphical output of the graph by using the graphviz library:
 
 \verbatim
 $ dot -Tpdf dag.dot -o output.pdf
 \endverbatim
 
-\subsection TraceTaskDetails Getting Task Details
-
-When the FxT trace file <c>prof_file_something</c> has been generated, details on the
-executed tasks can be retrieved by calling:
+\subsubsection TraceTaskDetails Getting Task Details
 
-\verbatim
-$ starpu_fxt_tool -i /tmp/prof_file_something
-\endverbatim
-
-This will create a <c>tasks.rec</c> file in the current directory.  This file
+Another generated trace file gives details on the executed tasks. The
+file, created in the current directory, is named <c>tasks.rec</c>. This file
 is in the recutils format, i.e. <c>Field: value</c> lines, and empty lines to
 separate each task.  This can be used as a convenient input for various ad-hoc
 analysis tools. By default it only contains information about the actual
@@ -226,18 +235,12 @@ Another possibility is to obtain the performance models as an auxiliary <c>perfm
 $ starpu_perfmodel_recdump tasks.rec -o perfmodel.rec
 \endverbatim
 
-\subsection MonitoringActivity Monitoring Activity
+\subsubsection MonitoringActivity Monitoring Activity
 
-When the FxT trace file <c>prof_file_something</c> has been generated, it is possible to
-generate an activity trace by calling:
-
-\verbatim
-$ starpu_fxt_tool -i /tmp/prof_file_something
-\endverbatim
-
-This will create a file <c>activity.data</c> in the current
-directory. A profile of the application showing the activity of StarPU
-during the execution of the program can be generated:
+Another generated trace file is an activity trace. The file, created
+in the current directory, is named <c>activity.data</c>. A profile of
+the application showing the activity of StarPU during the execution of
+the program can be generated:
 
 \verbatim
 $ starpu_workers_activity activity.data
@@ -259,18 +262,13 @@ evolution of the number of tasks available in the system during the execution.
 Ready tasks are shown in black, and tasks that are submitted but not
 schedulable yet are shown in grey.
 
-\subsection Animation Getting Modular Schedular Animation
+\subsubsection Animation Getting Modular Schedular Animation
 
 When using modular schedulers (i.e. schedulers which use a modular architecture,
-and whose name start with "modular-"), the command
-
-\verbatim
-$ starpu_fxt_tool -i /tmp/prof_file_something
-\endverbatim
-
-will also produce a <c>trace.html</c> file which can be viewed in a
-javascript-enabled web browser. It shows the flow of tasks between the
-components of the modular scheduler.
+and whose name start with "modular-"), the call to
+<c>starpu_fxt_tool</c> will also produce a <c>trace.html</c> file
+which can be viewed in a javascript-enabled web browser. It shows the
+flow of tasks between the components of the modular scheduler.
 
 \subsection LimitingScopeTrace Limiting The Scope Of The Trace
 

doc/doxygen/chapters/390_faq.doxy

@@ -1,6 +1,6 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2010-2017                                CNRS
+ * Copyright (C) 2010-2018                                CNRS
  * Copyright (C) 2009-2011,2014,2016-2017                 Université de Bordeaux
  * Copyright (C) 2011-2012                                Inria
  *
@@ -159,8 +159,7 @@ for (workerid = 0; workerid < starpu_worker_get_count(); workerid++)
 cudaGraphicsResourceGetMappedPointer((void**)&output, &num_bytes, resource);
 
 /* And register it to StarPU */
-starpu_vector_data_register(&handle, starpu_worker_get_memory_node(workerid),
-                            output, num_bytes / sizeof(float4), sizeof(float4));
+starpu_vector_data_register(&handle, starpu_worker_get_memory_node(workerid), output, num_bytes / sizeof(float4), sizeof(float4));
 
 /* The handle can now be used as usual */
 starpu_task_insert(&cl, STARPU_RW, handle, 0);

doc/doxygen/chapters/401_out_of_core.doxy

@@ -1,8 +1,8 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2013-2014,2016-2017                      CNRS
+ * Copyright (C) 2013-2014,2016-2018                      CNRS
  * Copyright (C) 2013                                     Inria
- * Copyright (C) 2013-2014,2017-2018                           Université de Bordeaux
+ * Copyright (C) 2013-2014,2017-2018                      Université de Bordeaux
  * Copyright (C) 2013                                     Corentin Salingue
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -51,7 +51,7 @@ automatically read and write data as appropriate.
 To use a disk memory node, you have to register it with this function:
 
 \code{.c}
-	int new_dd = starpu_disk_register(&starpu_disk_unistd_ops, (void *) "/tmp/", 1024*1024*200);
+int new_dd = starpu_disk_register(&starpu_disk_unistd_ops, (void *) "/tmp/", 1024*1024*200);
 \endcode
 
 Here, we use the unistd library to realize the read/write operations, i.e.

doc/doxygen/chapters/410_mpi_support.doxy

@@ -1,7 +1,7 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
  * Copyright (C) 2010-2018                                CNRS
- * Copyright (C) 2011-2013,2017                           Inria
+ * Copyright (C) 2011-2013,2016,2017                      Inria
  * Copyright (C) 2009-2011,2013-2018                      Université de Bordeaux
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -38,31 +38,31 @@ initializes a token on node 0, and the token is passed from node to node,
 incremented by one on each step. The code is not using StarPU yet.
 
 \code{.c}
-    for (loop = 0; loop < nloops; loop++)
-    {
-        int tag = loop*size + rank;
+for (loop = 0; loop < nloops; loop++)
+{
+    int tag = loop*size + rank;
 
-        if (loop == 0 && rank == 0)
-        {
-            token = 0;
-            fprintf(stdout, "Start with token value %d\n", token);
-        }
-        else
-        {
-            MPI_Recv(&token, 1, MPI_INT, (rank+size-1)%size, tag, MPI_COMM_WORLD);
-        }
+    if (loop == 0 && rank == 0)
+    {
+        token = 0;
+        fprintf(stdout, "Start with token value %d\n", token);
+    }
+    else
+    {
+        MPI_Recv(&token, 1, MPI_INT, (rank+size-1)%size, tag, MPI_COMM_WORLD);
+    }
 
-        token++;
+    token++;
 
-        if (loop == last_loop && rank == last_rank)
-        {
-            fprintf(stdout, "Finished: token value %d\n", token);
-        }
-        else
-        {
-            MPI_Send(&token, 1, MPI_INT, (rank+1)%size, tag+1, MPI_COMM_WORLD);
-        }
+    if (loop == last_loop && rank == last_rank)
+    {
+        fprintf(stdout, "Finished: token value %d\n", token);
     }
+    else
+    {
+        MPI_Send(&token, 1, MPI_INT, (rank+1)%size, tag+1, MPI_COMM_WORLD);
+    }
+}
 \endcode
 
 \section NotUsingMPISupport About not using the MPI support
@@ -73,39 +73,39 @@ execution to StarPU.  This is possible by just using starpu_data_acquire(), for
 instance:
 
 \code{.c}
-    for (loop = 0; loop < nloops; loop++)
-    {
-        int tag = loop*size + rank;
+for (loop = 0; loop < nloops; loop++)
+{
+    int tag = loop*size + rank;
 
-	/* Acquire the data to be able to write to it */
-	starpu_data_acquire(token_handle, STARPU_W);
-        if (loop == 0 && rank == 0)
-        {
-            token = 0;
-            fprintf(stdout, "Start with token value %d\n", token);
-        }
-        else
-        {
-            MPI_Recv(&token, 1, MPI_INT, (rank+size-1)%size, tag, MPI_COMM_WORLD);
-        }
+    /* Acquire the data to be able to write to it */
+    starpu_data_acquire(token_handle, STARPU_W);
+    if (loop == 0 && rank == 0)
+    {
+        token = 0;
+        fprintf(stdout, "Start with token value %d\n", token);
+    }
+    else
+    {
+        MPI_Recv(&token, 1, MPI_INT, (rank+size-1)%size, tag, MPI_COMM_WORLD);
+    }
 	starpu_data_release(token_handle);
 
-        /* Task delegation to StarPU to increment the token. The execution might
-         * be performed on a CPU, a GPU, etc. */
-        increment_token();
+    /* Task delegation to StarPU to increment the token. The execution might
+     * be performed on a CPU, a GPU, etc. */
+    increment_token();
 
-	/* Acquire the update data to be able to read from it */
-	starpu_data_acquire(token_handle, STARPU_R);
-        if (loop == last_loop && rank == last_rank)
-        {
-            fprintf(stdout, "Finished: token value %d\n", token);
-        }
-        else
-        {
-            MPI_Send(&token, 1, MPI_INT, (rank+1)%size, tag+1, MPI_COMM_WORLD);
-        }
-	starpu_data_release(token_handle);
+    /* Acquire the update data to be able to read from it */
+    starpu_data_acquire(token_handle, STARPU_R);
+    if (loop == last_loop && rank == last_rank)
+    {
+        fprintf(stdout, "Finished: token value %d\n", token);
     }
+    else
+    {
+        MPI_Send(&token, 1, MPI_INT, (rank+1)%size, tag+1, MPI_COMM_WORLD);
+    }
+	starpu_data_release(token_handle);
+}
 \endcode
 
 In that case, <c>libstarpumpi</c> is not needed. One can also use <c>MPI_Isend()</c> and
@@ -141,8 +141,7 @@ int main(int argc, char **argv)
 {
     int rank, size;
 
-    starpu_init(NULL);
-    starpu_mpi_init(&argc, &argv, 1);
+    starpu_mpi_init_conf(&argc, &argv, 1, MPI_COMM_WORLD, NULL);
     starpu_mpi_comm_rank(MPI_COMM_WORLD, &rank);
     starpu_mpi_comm_size(MPI_COMM_WORLD, &size);
 
@@ -167,8 +166,7 @@ int main(int argc, char **argv)
         }
         else
         {
-            starpu_mpi_irecv_detached(token_handle, (rank+size-1)%size, tag,
-                    MPI_COMM_WORLD, NULL, NULL);
+            starpu_mpi_irecv_detached(token_handle, (rank+size-1)%size, tag, MPI_COMM_WORLD, NULL, NULL);
         }
 
         increment_token();
@@ -181,15 +179,13 @@ int main(int argc, char **argv)
         }
         else
         {
-            starpu_mpi_isend_detached(token_handle, (rank+1)%size, tag+1,
-                    MPI_COMM_WORLD, NULL, NULL);
+            starpu_mpi_isend_detached(token_handle, (rank+1)%size, tag+1, MPI_COMM_WORLD, NULL, NULL);
         }
     }
 
     starpu_task_wait_for_all();
 
     starpu_mpi_shutdown();
-    starpu_shutdown();
 
     if (rank == last_rank)
     {
@@ -207,10 +203,10 @@ the beginning and the end.
 
 \section MPIInitialization How to Initialize StarPU-MPI
 
-As seen in the previous example, one has to call starpu_mpi_init() to
+As seen in the previous example, one has to call starpu_mpi_init_conf() to
 initialize StarPU-MPI. The third parameter of the function indicates
-if MPI should be initialized by StarPU or if the application will do
-it itself. If the application initializes MPI itself, it must call
+if MPI should be initialized by StarPU or if the application did it
+itself. If the application initializes MPI itself, it must call
 <c>MPI_Init_thread()</c> with <c>MPI_THREAD_SERIALIZED</c> or
 <c>MPI_THREAD_MULTIPLE</c>, since StarPU-MPI uses a separate thread to
 perform the communications. <c>MPI_THREAD_MULTIPLE</c> is necessary if
@@ -316,14 +312,12 @@ static int complex_pack_data(starpu_data_handle_t handle, unsigned node, void **
 {
   STARPU_ASSERT(starpu_data_test_if_allocated_on_node(handle, node));
 
-  struct starpu_complex_interface *complex_interface =
-    (struct starpu_complex_interface *) starpu_data_get_interface_on_node(handle, node);
+  struct starpu_complex_interface *complex_interface = (struct starpu_complex_interface *) starpu_data_get_interface_on_node(handle, node);
 
   *count = complex_get_size(handle);
   starpu_malloc_flags(ptr, *count, 0);
   memcpy(*ptr, complex_interface->real, complex_interface->nx*sizeof(double));
-  memcpy(*ptr+complex_interface->nx*sizeof(double), complex_interface->imaginary,
-         complex_interface->nx*sizeof(double));
+  memcpy(*ptr+complex_interface->nx*sizeof(double), complex_interface->imaginary, complex_interface->nx*sizeof(double));
 
   return 0;
 }
@@ -332,12 +326,10 @@ static int complex_unpack_data(starpu_data_handle_t handle, unsigned node, void
 {
   STARPU_ASSERT(starpu_data_test_if_allocated_on_node(handle, node));
 
-  struct starpu_complex_interface *complex_interface =
-    (struct starpu_complex_interface *)	starpu_data_get_interface_on_node(handle, node);
+  struct starpu_complex_interface *complex_interface = (struct starpu_complex_interface *) starpu_data_get_interface_on_node(handle, node);
 
   memcpy(complex_interface->real, ptr, complex_interface->nx*sizeof(double));
-  memcpy(complex_interface->imaginary, ptr+complex_interface->nx*sizeof(double),
-         complex_interface->nx*sizeof(double));
+  memcpy(complex_interface->imaginary, ptr+complex_interface->nx*sizeof(double), complex_interface->nx*sizeof(double));
 
   return 0;
 }
@@ -369,8 +361,7 @@ void starpu_complex_interface_datatype_allocate(starpu_data_handle_t handle, MPI
 	MPI_Aint displacements[2];
 	MPI_Datatype types[2] = {MPI_DOUBLE, MPI_DOUBLE};
 
-	struct starpu_complex_interface *complex_interface =
-          (struct starpu_complex_interface *) starpu_data_get_interface_on_node(handle, STARPU_MAIN_RAM);
+	struct starpu_complex_interface *complex_interface = (struct starpu_complex_interface *) starpu_data_get_interface_on_node(handle, STARPU_MAIN_RAM);
 
 	MPI_Address(complex_interface, displacements);
 	MPI_Address(&complex_interface->imaginary, displacements+1);
@@ -461,13 +452,11 @@ data which will be needed by the tasks that we will execute.
             int mpi_rank = my_distrib(x, y, size);
             if (mpi_rank == my_rank)
                 /* Owning data */
-                starpu_variable_data_register(&data_handles[x][y], STARPU_MAIN_RAM,
-                                              (uintptr_t)&(matrix[x][y]), sizeof(unsigned));
+                starpu_variable_data_register(&data_handles[x][y], STARPU_MAIN_RAM, (uintptr_t)&(matrix[x][y]), sizeof(unsigned));
             else if (my_rank == my_distrib(x+1, y, size) || my_rank == my_distrib(x-1, y, size)
                   || my_rank == my_distrib(x, y+1, size) || my_rank == my_distrib(x, y-1, size))
                 /* I don't own this index, but will need it for my computations */
-                starpu_variable_data_register(&data_handles[x][y], -1,
-                                              (uintptr_t)NULL, sizeof(unsigned));
+                starpu_variable_data_register(&data_handles[x][y], -1, (uintptr_t)NULL, sizeof(unsigned));
             else
                 /* I know it's useless to allocate anything for this */
                 data_handles[x][y] = NULL;
@@ -600,7 +589,8 @@ can just pass NULL to starpu_mpi_task_insert():
 
 \code{.c}
 starpu_data_handle_t data0 = NULL;
-if (rank == 0) {
+if (rank == 0)
+{
 	starpu_variable_data_register(&data0, STARPU_MAIN_RAM, (uintptr_t) &val0, sizeof(val0));
 	starpu_mpi_data_register(data0, 0, rank);
 }
@@ -615,12 +605,15 @@ data1 to node 0, which owns data and thus will need the value of data1 to execut
 
 \code{.c}
 starpu_data_handle_t data0 = NULL, data1, data;
-if (rank == 0) {
+if (rank == 0)
+{
 	starpu_variable_data_register(&data0, STARPU_MAIN_RAM, (uintptr_t) &val0, sizeof(val0));
 	starpu_mpi_data_register(data0, -1, rank);
 	starpu_variable_data_register(&data1, -1, 0, sizeof(val1));
 	starpu_variable_data_register(&data, STARPU_MAIN_RAM, (uintptr_t) &val, sizeof(val));
-} else if (rank == 1) {
+}
+else if (rank == 1)
+{
 	starpu_variable_data_register(&data1, STARPU_MAIN_RAM, (uintptr_t) &val1, sizeof(val1));
 	starpu_variable_data_register(&data, -1, 0, sizeof(val));
 }
@@ -641,10 +634,13 @@ starpu_variable_data_register(&pernode, -1, 0, sizeof(val));
 starpu_mpi_data_register(pernode, -1, STARPU_MPI_PER_NODE);
 
 /* Normal data: one on node0, one on node1 */
-if (rank == 0) {
+if (rank == 0)
+{
 	starpu_variable_data_register(&data0, STARPU_MAIN_RAM, (uintptr_t) &val0, sizeof(val0));
 	starpu_variable_data_register(&data1, -1, 0, sizeof(val1));
-} else if (rank == 1) {
+}
+else if (rank == 1)
+{
 	starpu_variable_data_register(&data0, -1, 0, sizeof(val1));
 	starpu_variable_data_register(&data1, STARPU_MAIN_RAM, (uintptr_t) &val1, sizeof(val1));
 }
@@ -744,8 +740,7 @@ migrate the data, and register the new location.
                   || my_rank == my_distrib(x+1, y, size) || my_rank == my_distrib(x-1, y, size)
                   || my_rank == my_distrib(x, y+1, size) || my_rank == my_distrib(x, y-1, size)))
                 /* Register newly-needed data */
-                starpu_variable_data_register(&data_handles[x][y], -1,
-                                              (uintptr_t)NULL, sizeof(unsigned));
+                starpu_variable_data_register(&data_handles[x][y], -1, (uintptr_t)NULL, sizeof(unsigned));
             if (data_handles[x][y])
 	    {
                 /* Migrate the data */
@@ -808,14 +803,12 @@ for(x = 0; x < nblocks ;  x++)
     int mpi_rank = my_distrib(x, nodes);
     if (rank == root)
     {
-        starpu_vector_data_register(&data_handles[x], STARPU_MAIN_RAM, (uintptr_t)vector[x],
-                                    blocks_size, sizeof(float));
+        starpu_vector_data_register(&data_handles[x], STARPU_MAIN_RAM, (uintptr_t)vector[x], blocks_size, sizeof(float));
     }
     else if ((mpi_rank == rank) || ((rank == mpi_rank+1 || rank == mpi_rank-1)))
     {
         /* I own this index, or i will need it for my computations */
-        starpu_vector_data_register(&data_handles[x], -1, (uintptr_t)NULL,
-                                   block_size, sizeof(float));
+        starpu_vector_data_register(&data_handles[x], -1, (uintptr_t)NULL, block_size, sizeof(float));
     }
     else
     {
@@ -851,6 +844,46 @@ starpu_mpi_gather_detached(data_handles, nblocks, 0, MPI_COMM_WORLD);
 Other collective operations would be easy to define, just ask starpu-devel for
 them!
 
+\section MPIDriver Make StarPU-MPI progression thread execute tasks
+
+The default behaviour of StarPU-MPI is to spawn an MPI thread to take care only
+of MPI communications in an active fashion (i.e the StarPU-MPI thread sleeps
+only when there is no active request submitted by the application), with the
+goal of being as reactive as possible to communications. Knowing that, users
+usually leave one free core for the MPI thread when starting a distributed
+execution with StarPU-MPI.  However, this could result in a loss of performance
+for applications that does not require an extreme reactivity to MPI
+communications.
+
+The starpu_mpi_init_conf() routine allows the user to give the
+starpu_conf configuration structure of StarPU (usually given to the
+starpu_init() routine) to StarPU-MPI, so that StarPU-MPI reserves for its own
+use one of the CPU drivers of the current computing node, or one of the CPU
+cores, and then calls starpu_init() internally.
+
+This allows the MPI communication thread to call a StarPU CPU driver to run
+tasks when there is no active requests to take care of, and thus recover the
+computational power of the "lost" core. Since there is a trade-off between
+executing tasks and polling MPI requests, which is how much the application
+wants to lose in reactivity to MPI communications to get back the computing
+power of the core dedicated to the StarPU-MPI thread, there are two environment
+variables to pilot the behaviour of the MPI thread so that users can tune
+this trade-off depending of the behaviour of the application.
+
+The \ref STARPU_MPI_DRIVER_CALL_FREQUENCY environment variable sets how many times
+the MPI progression thread goes through the MPI_Test() loop on each active communication request
+(and thus try to make communications progress by going into the MPI layer)
+before executing tasks. The default value for this environment variable is 0,
+which means that the support for interleaving task execution and communication
+polling is deactivated, thus returning the MPI progression thread to its
+original behaviour.
+
+The \ref STARPU_MPI_DRIVER_TASK_FREQUENCY environment variable sets how many tasks
+are executed by the MPI communication thread before checking all active
+requests again. While this environment variable allows a better use of the core
+dedicated to StarPU-MPI for computations, it also decreases the reactivity of
+the MPI communication thread as much.
+
 \section MPIDebug Debugging MPI
 
 Communication trace will be enabled when the environment variable

doc/doxygen/chapters/440_c_extensions.doxy

@@ -1,6 +1,6 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2010-2017                                CNRS
+ * Copyright (C) 2010-2018                                CNRS
  * Copyright (C) 2009-2011,2014-2015                      Université de Bordeaux
  * Copyright (C) 2011-2012                                Inria
  *
@@ -126,20 +126,15 @@ Here is an example:
 \code{.c}
 #define __output  __attribute__ ((output))
 
-static void matmul (const float *A, const float *B,
-                    __output float *C,
-                    unsigned nx, unsigned ny, unsigned nz)
+static void matmul (const float *A, const float *B, __output float *C, unsigned nx, unsigned ny, unsigned nz)
   __attribute__ ((task));
 
-static void matmul_cpu (const float *A, const float *B,
-                        __output float *C,
-                        unsigned nx, unsigned ny, unsigned nz)
+static void matmul_cpu (const float *A, const float *B, __output float *C, unsigned nx, unsigned ny, unsigned nz)
   __attribute__ ((task_implementation ("cpu", matmul)));
 
 
 static void
-matmul_cpu (const float *A, const float *B, __output float *C,
-            unsigned nx, unsigned ny, unsigned nz)
+matmul_cpu (const float *A, const float *B, __output float *C, unsigned nx, unsigned ny, unsigned nz)
 {
   unsigned i, j, k;
 
@@ -165,16 +160,12 @@ implementation.  Thus, the above snippet can be simplified like this:
 \code{.c}
 #define __output  __attribute__ ((output))
 
-static void matmul (const float *A, const float *B,
-                    __output float *C,
-                    unsigned nx, unsigned ny, unsigned nz)
+static void matmul (const float *A, const float *B, __output float *C, unsigned nx, unsigned ny, unsigned nz)
   __attribute__ ((task));
 
 /* Implicit definition of the CPU implementation of the
    `matmul' task.  */
-static void
-matmul (const float *A, const float *B, __output float *C,
-        unsigned nx, unsigned ny, unsigned nz)
+static void matmul (const float *A, const float *B, __output float *C, unsigned nx, unsigned ny, unsigned nz)
 {
   unsigned i, j, k;
 
@@ -194,12 +185,10 @@ the code is valid sequential code when StarPU's GCC plug-in is not used
 CUDA and OpenCL implementations can be declared in a similar way:
 
 \code{.c}
-static void matmul_cuda (const float *A, const float *B, float *C,
-                         unsigned nx, unsigned ny, unsigned nz)
+static void matmul_cuda (const float *A, const float *B, float *C, unsigned nx, unsigned ny, unsigned nz)
   __attribute__ ((task_implementation ("cuda", matmul)));
 
-static void matmul_opencl (const float *A, const float *B, float *C,
-                           unsigned nx, unsigned ny, unsigned nz)
+static void matmul_opencl (const float *A, const float *B, float *C, unsigned nx, unsigned ny, unsigned nz)
   __attribute__ ((task_implementation ("opencl", matmul)));
 \endcode
 
@@ -209,13 +198,9 @@ written in CUDA or OpenCL (for similar code, \ref CUDAKernel, and
 OpenCL under the hood, such as CUBLAS functions:
 
 \code{.c}
-static void
-matmul_cuda (const float *A, const float *B, float *C,
-             unsigned nx, unsigned ny, unsigned nz)
+static void matmul_cuda (const float *A, const float *B, float *C, unsigned nx, unsigned ny, unsigned nz)
 {
-  cublasSgemm ('n', 'n', nx, ny, nz,
-               1.0f, A, 0, B, 0,
-               0.0f, C, 0);
+  cublasSgemm ('n', 'n', nx, ny, nz, 1.0f, A, 0, B, 0, 0.0f, C, 0);
   cudaStreamSynchronize (starpu_cuda_get_local_stream ());
 }
 \endcode

doc/doxygen/chapters/470_simgrid.doxy

@@ -19,7 +19,7 @@
 /*! \page SimGridSupport SimGrid Support
 
 StarPU can use Simgrid in order to simulate execution on an arbitrary
-platform. This was tested with simgrid from 3.11 to 3.16, and 3.18 to 3.19.
+platform. This was tested with simgrid from 3.11 to 3.16, and 3.18 to 3.20.
 Other versions may have compatibility issues. 3.17 notably does not build at
 all.
 
@@ -180,6 +180,14 @@ application running with simgrid, pass the <c>--cfg=contexts/factory:thread</c>
 option to the application, to make simgrid use system threads, which gdb will be
 able to manipulate as usual.
 
+It is also worth noting Simgrid 3.21's new parameter
+<c>--cfg=simix/breakpoint</c> which allows to put a breakpoint at a precise
+(deterministic!) timing of the execution. If for instance in an execution
+trace we see that something odd is happening at time 19000ms, we can use
+<c>--cfg=simix/breakpoint:19.000</c> and SIGTRAP will be raised at that point,
+which will thus interrupt execution within gdb, allowing to inspect e.g.
+scheduler state, etc.
+
 \section SimulationMemoryUsage Memory Usage
 
 Since kernels are not actually run and data transfers are not actually

doc/doxygen/chapters/490_clustering_a_machine.doxy

@@ -1,8 +1,8 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2015-2017                                CNRS
+ * Copyright (C) 2015-2018                                CNRS
  * Copyright (C) 2015-2016                                Inria
- * Copyright (C) 2015, 2018                                     Université de Bordeaux
+ * Copyright (C) 2015, 2018                               Université de Bordeaux
  *
  * StarPU is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
@@ -64,7 +64,7 @@ struct starpu_cluster_machine *clusters;
 clusters = starpu_cluster_machine(HWLOC_OBJ_SOCKET, 0);
 starpu_cluster_print(clusters);
 
-//... submit some tasks with OpenMP computations 
+//... submit some tasks with OpenMP computations
 
 starpu_uncluster_machine(clusters);
 //... we are back in the default starpu state
@@ -111,6 +111,7 @@ and create the aforementioned OpenMP threads constrained in the cluster's
 resources set:
 \code{.c}
 void starpu_openmp_prologue(void * sched_ctx_id)
+{
   int sched_ctx = *(int*)sched_ctx_id;
   int *cpuids = NULL;
   int ncpuids = 0;

doc/doxygen/chapters/501_environment_variables.doxy

@@ -2,7 +2,7 @@
  *
  * Copyright (C) 2011-2013,2015-2017                      Inria
  * Copyright (C) 2010-2018                                CNRS
- * Copyright (C) 2009-2011,2013-2017                      Université de Bordeaux
+ * Copyright (C) 2009-2011,2013-2018                      Université de Bordeaux
  * Copyright (C) 2016                                     Uppsala University
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -36,6 +36,17 @@ not allocate more CPU workers than there are physical CPUs, and that
 some CPUs are used to control the accelerators.
 </dd>
 
+<dt>STARPU_RESERVE_NCPU</dt>
+<dd>
+\anchor STARPU_RESERVE_NCPU
+\addindex __env__STARPU_RESERVE_NCPU
+Specify the number of CPU cores that should not be used by StarPU, so the
+application can use starpu_get_next_bindid() and starpu_bind_thread_on() to bind
+its own threads.
+
+This option is ignored if \ref STARPU_NCPU or starpu_config::ncpus is set.
+</dd>
+
 <dt>STARPU_NCPUS</dt>
 <dd>
 \anchor STARPU_NCPUS
@@ -66,19 +77,21 @@ which will be concurrently running on the devices. The default value is 1.
 <dd>
 \anchor STARPU_CUDA_THREAD_PER_WORKER
 \addindex __env__STARPU_CUDA_THREAD_PER_WORKER
-Specify if the cuda driver should provide a thread per stream or a single thread 
-dealing with all the streams. 0 if one thread per stream, 1 otherwise. The default 
-value is 0. Setting it to 1 is contradictory with setting STARPU_CUDA_THREAD_PER_DEV to 1.
+Specify whether the cuda driver should use one thread per stream (1) or to use
+a single thread to drive all the streams of the device or all devices (0), and
+STARPU_CUDA_THREAD_PER_DEV determines whether is it one thread per device or one
+thread for all devices. The default value is 0. Setting it to 1 is contradictory
+with setting STARPU_CUDA_THREAD_PER_DEV.
 </dd>
 
 <dt>STARPU_CUDA_THREAD_PER_DEV</dt>
 <dd>
 \anchor STARPU_CUDA_THREAD_PER_DEV
 \addindex __env__STARPU_CUDA_THREAD_PER_DEV
-Specify if the cuda driver should provide a thread per device or a single thread 
-dealing with all the devices. 0 if one thread per device, 1 otherwise. The default 
-value is 1, unless STARPU_CUDA_THREAD_PER_WORKER is set to 1. Setting it to 1 is
-contradictory with setting STARPU_CUDA_THREAD_PER_WORKER to 1.
+Specify whether the cuda driver should use one thread per device (1) or to use a
+single thread to drive all the devices (0). The default value is 1.  It does not
+make sense to set this variable if STARPU_CUDA_THREAD_PER_WORKER is set to to 1
+(since STARPU_CUDA_THREAD_PER_DEV is then meaningless).
 </dd>
 
 <dt>STARPU_CUDA_PIPELINE</dt>
@@ -369,7 +382,7 @@ Disable asynchronous copies between CPU and MIC devices.
 
 <dt>STARPU_DISABLE_ASYNCHRONOUS_MPI_MS_COPY</dt>
 <dd>
-\anchor STARPU_DISABLE_ASYNCHRONOUS_MPI_MS_COPY 
+\anchor STARPU_DISABLE_ASYNCHRONOUS_MPI_MS_COPY
 \addindex __env__STARPU_DISABLE_ASYNCHRONOUS_MPI_MS_COPY
 Disable asynchronous copies between CPU and MPI Slave devices.
 </dd>
@@ -613,6 +626,30 @@ of one of the nodes of a big cluster without actually running the rest.
 It of course does not provide computation results and timing.
 </dd>
 
+<dt>STARPU_MPI_DRIVER_CALL_FREQUENCY</dt>
+<dd>
+\anchor STARPU_MPI_DRIVER_CALL_FREQUENCY
+\addindex __env__STARPU_MPI_DRIVER_CALL_FREQUENCY
+When set to a positive value, activates the interleaving of the execution of
+tasks with the progression of MPI communications (\ref MPISupport). The
+starpu_mpi_init_conf() function must have been called by the application
+for that environment variable to be used. When set to 0, the MPI progression
+thread does not use at all the driver given by the user, and only focuses on
+making MPI communications progress.
+</dd>
+
+<dt>STARPU_MPI_DRIVER_TASK_FREQUENCY</dt>
+<dd>
+\anchor STARPU_MPI_DRIVER_TASK_FREQUENCY
+\addindex __env__STARPU_MPI_DRIVER_TASK_FREQUENCY
+When set to a positive value, the interleaving of the execution of tasks with
+the progression of MPI communications mechanism to execute several tasks before
+checking communication requests again (\ref MPISupport). The
+starpu_mpi_init_conf() function must have been called by the application
+for that environment variable to be used, and the
+STARPU_MPI_DRIVER_CALL_FREQUENCY environment variable set to a positive value.
+</dd>
+
 <dt>STARPU_SIMGRID_CUDA_MALLOC_COST</dt>
 <dd>
 \anchor STARPU_SIMGRID_CUDA_MALLOC_COST
@@ -946,7 +983,7 @@ full.
 This specifies then backend to be used by StarPU to push data when the main
 memory is getting full. The default is unistd (i.e. using read/write functions),
 other values are stdio (i.e. using fread/fwrite), unistd_o_direct (i.e. using
-read/write with O_DIRECT), leveldb (i.e. using a leveldb database), and hdf5 
+read/write with O_DIRECT), leveldb (i.e. using a leveldb database), and hdf5
 (i.e. using HDF5 library).
 </dd>
 
@@ -961,7 +998,7 @@ memory is getting full. The default is unlimited.
 <dt>STARPU_LIMIT_MAX_SUBMITTED_TASKS</dt>
 <dd>
 \anchor STARPU_LIMIT_MAX_SUBMITTED_TASKS
-\addindex __env__STARPU_LIMIT_MAX_SUBMITTED_TASKS    
+\addindex __env__STARPU_LIMIT_MAX_SUBMITTED_TASKS
 This variable allows the user to control the task submission flow by specifying
 to StarPU a maximum number of submitted tasks allowed at a given time, i.e. when
 this limit is reached task submission becomes blocking until enough tasks have
@@ -972,7 +1009,7 @@ Setting it enables allocation cache buffer reuse in main memory.
 <dt>STARPU_LIMIT_MIN_SUBMITTED_TASKS</dt>
 <dd>
 \anchor STARPU_LIMIT_MIN_SUBMITTED_TASKS
-\addindex __env__STARPU_LIMIT_MIN_SUBMITTED_TASKS    
+\addindex __env__STARPU_LIMIT_MIN_SUBMITTED_TASKS
 This variable allows the user to control the task submission flow by specifying
 to StarPU a submitted task threshold to wait before unblocking task submission. This
 variable has to be used in conjunction with \ref STARPU_LIMIT_MAX_SUBMITTED_TASKS
@@ -997,6 +1034,15 @@ When set to <c>1</c>, this variable indicates that StarPU should automatically
 generate a Paje trace when starpu_shutdown() is called.
 </dd>
 
+<dt>STARPU_GENERATE_TRACE_OPTIONS</dt>
+<dd>
+\anchor STARPU_GENERATE_TRACE_OPTIONS
+\addindex __env__STARPU_GENERATE_TRACE_OPTIONS
+When the variable \ref STARPU_GENERATE_TRACE is set to <c>1</c> to
+generate a Paje trace, this variable can be set to specify options (see
+<c>starpu_fxt_tool --help</c>).
+</dd>
+
 <dt>STARPU_ENABLE_STATS</dt>
 <dd>
 \anchor STARPU_ENABLE_STATS
@@ -1162,7 +1208,7 @@ accesses (see \ref ConcurrentDataAccess).
 
 <dt>STARPU_USE_NUMA</dt>
 <dd>
-\anchor STARPU_USE_NUMA 
+\anchor STARPU_USE_NUMA
 \addindex __env__STARPU_USE_NUMA
 When defined, NUMA nodes are taking into account by StarPU. Otherwise, memory
 is considered as only one node. This is experimental for now.
@@ -1191,7 +1237,7 @@ The file will contain the sum of the idle times of all the workers.
 <dd>
 \anchor SC_HYPERVISOR_POLICY
 \addindex __env__SC_HYPERVISOR_POLICY
-Choose between the different resizing policies proposed by StarPU for the hypervisor: 
+Choose between the different resizing policies proposed by StarPU for the hypervisor:
 idle, app_driven, feft_lp, teft_lp; ispeed_lp, throughput_lp etc.
 
 Use <c>SC_HYPERVISOR_POLICY=help</c> to get the list of available policies for the hypervisor
@@ -1221,7 +1267,7 @@ of the total time of execution of the application. The default value is the resi
 \addindex __env__SC_HYPERVISOR_MAX_SPEED_GAP
 Indicate the ratio of speed difference between contexts that should trigger the hypervisor.
 This situation may occur only when a theoretical speed could not be computed and the hypervisor
-has no value to compare the speed to. Otherwise the resizing of a context is not influenced by the 
+has no value to compare the speed to. Otherwise the resizing of a context is not influenced by the
 the speed of the other contexts, but only by the the value that a context should have.
 </dd>
 
@@ -1239,7 +1285,7 @@ is not done.
 \anchor SC_HYPERVISOR_LAZY_RESIZE
 \addindex __env__SC_HYPERVISOR_LAZY_RESIZE
 By default the hypervisor resizes the contexts in a lazy way, that is workers are firstly added to a new context
-before removing them from the previous one. Once this workers are clearly taken into account 
+before removing them from the previous one. Once this workers are clearly taken into account
 into the new context (a task was poped there) we remove them from the previous one. However if the application
 would like that the change in the distribution of workers should change right away this variable should be set to 0
 </dd>

doc/doxygen/chapters/510_configure_options.doxy

@@ -505,7 +505,8 @@ Enable performance model debugging.
 <dd>
 \anchor enable-fxt-lock
 \addindex __configure__--enable-fxt-lock
-Enable additional trace events which describes locks behaviour.
+Enable additional trace events which describes locks behaviour. This is however
+extremely heavy and should only be enabled when debugging insides of StarPU.
 </dd>
 
 <dt>--enable-maxbuffers</dt>

doc/doxygen/chapters/api/codelet_and_tasks.doxy

@@ -1,7 +1,7 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
  * Copyright (C) 2011-2013,2015,2017                      Inria
- * Copyright (C) 2010-2017                                CNRS
+ * Copyright (C) 2010-2018                                CNRS
  * Copyright (C) 2009-2011,2013-2018                      Université de Bordeaux
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -415,7 +415,7 @@ compiler implicitly do it in e.g. static storage case.
     Optional name of the codelet. This can be useful for debugging
     purposes.
 
-\var const char *starpu_codelet::color
+\var unsigned starpu_codelet::color
     Optional color of the codelet. This can be useful for debugging
     purposes.
 
@@ -520,6 +520,10 @@ the configuration of a task allocated with starpu_task_create().
     codelet, one should either define this field or the field
     starpu_task::modes defined above.
 
+\var char *starpu_task::handles_sequential_consistency
+    Optional pointer to an array of characters which allows to define
+    the sequential consistency for each handle for the current task.
+
 \var void *starpu_task::cl_arg
     Optional pointer which is passed to the codelet through the second
     argument of the codelet implementation (e.g.
@@ -760,7 +764,7 @@ the configuration of a task allocated with starpu_task_create().
     This field allows to specify the type of task, for filtering
     out tasks in profiling outputs, whether it is really internal
     to StarPU (::STARPU_TASK_TYPE_INTERNAL), a data acquisition
-    synchronization task (STARPU_TASK_TYPE_DATA_ACQUIRE), or a normal task
+    synchronization task (::STARPU_TASK_TYPE_DATA_ACQUIRE), or a normal task
     (::STARPU_TASK_TYPE_NORMAL)
 
 \var int starpu_task::color
@@ -860,6 +864,24 @@ starpu_task::modes or the \p i -th element of the field
 starpu_task::dyn_modes
 (see \ref SettingManyDataHandlesForATask)
 
+\def STARPU_CODELET_GET_NODE(codelet, i)
+\ingroup API_Codelet_And_Tasks
+Return the target node of the \p i -th data handle of \p codelet.
+If \p node is defined with a static or dynamic number of
+handles, will either return the \p i -th element of the field
+starpu_codelet::nodes or the \p i -th element of the field
+starpu_codelet::dyn_nodes
+(see \ref SettingManyDataHandlesForATask)
+
+\def STARPU_CODELET_SET_NODE(codelet, node, i)
+\ingroup API_Codelet_And_Tasks
+Set the target node of the \p i -th data handle of \p codelet.
+If \p codelet is defined with a static or dynamic number of
+handles, will either set the \p i -th element of the field
+starpu_codelet::nodes or the \p i -th element of the field
+starpu_codelet::dyn_nodes
+(see \ref SettingManyDataHandlesForATask)
+
 \fn struct starpu_task *starpu_task_create(void)
 \ingroup API_Codelet_And_Tasks
 Allocate a task structure and initialize it with default
@@ -977,6 +999,16 @@ Return the task currently executed by the
 worker, or <c>NULL</c> if it is called either from a thread that is not a
 task or simply because there is no task being executed at the moment.
 
+\fn int starpu_task_get_current_data_node(unsigned i)
+\ingroup API_Codelet_And_Tasks
+Return the memory node number of parameter \p i of the task currently executed,
+or -1 if it is called either from a thread that is not a task or simply because
+there is no task being executed at the moment.
+
+Usually, the returned memory node number is simply the memory node
+for the current worker. That may however be different when using e.g.
+starpu_codelet::specific_nodes .
+
 \fn const char *starpu_task_get_name(struct starpu_task *task)
 \ingroup API_Codelet_And_Tasks
 Return the name of \p task, i.e. either its starpu_task::name field, or

doc/doxygen/chapters/api/data_interfaces.doxy

@@ -1,8 +1,8 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2010-2017                                CNRS
+ * Copyright (C) 2010-2018                                CNRS
  * Copyright (C) 2011-2014,2017                           Inria
- * Copyright (C) 2009-2011,2014-2017                      Université de Bordeaux
+ * Copyright (C) 2009-2011,2014-2018                      Université de Bordeaux
  *
  * StarPU is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
@@ -43,8 +43,16 @@ Per-interface data transfer methods.
     This provides a series of methods for performing ram/cuda/opencl synchronous and asynchronous transfers.
 
 \var void *(*starpu_data_interface_ops::handle_to_pointer)(starpu_data_handle_t handle, unsigned node)
+\deprecated
+    Use starpu_data_interface_ops::to_pointer instead.
     Return the current pointer (if any) for the handle on the given node.
 
+\var void *(*starpu_data_interface_ops::to_pointer)(void *data_interface, unsigned node)
+    Return the current pointer (if any) for the given interface on the given node.
+
+\var int (*starpu_data_interface_ops::pointer_is_inside)(void *data_interface, unsigned node, void *ptr)
+    Return whether the given \p ptr is within the data for the given interface on the given node.
+
 \var size_t (*starpu_data_interface_ops::get_size)(starpu_data_handle_t handle)
     Return an estimation of the size of data, for performance models.
 
@@ -524,6 +532,12 @@ Return the pointer associated with \p handle on node \p node or <c>NULL</c>
 if handle’s interface does not support this operation or data for this
 \p handle is not allocated on that \p node.
 
+\fn int starpu_data_pointer_is_inside(starpu_data_handle_t handle, unsigned node, void *ptr)
+\ingroup API_Data_Interfaces
+Return whether the given \p ptr is within the data for \p handle on node \p
+node (1) or not (0). If the handle interface does not support this operation,
+and thus the result is unknown, -1 is returned.
+
 \fn void *starpu_data_get_local_ptr(starpu_data_handle_t handle)
 \ingroup API_Data_Interfaces
 Return the local pointer associated with \p handle or <c>NULL</c> if
@@ -1212,6 +1226,21 @@ the starpu_data_copy_methods::any_to_any copy method, which is provided with \p
 be passed to starpu_interface_copy(). this returns <c>-EAGAIN</c> if the
 transfer is still ongoing, or 0 if the transfer is already completed.
 
+\fn void starpu_interface_start_driver_copy_async(unsigned src_node, unsigned dst_node, double *start)
+\ingroup API_Data_Interfaces
+When an asynchonous implementation of the data transfer is implemented, the call
+to the underlying CUDA, OpenCL, etc. call should be surrounded
+by calls to starpu_interface_start_driver_copy_async() and
+starpu_interface_end_driver_copy_async(), so that it is recorded in offline
+execution traces, and the timing of the submission is checked. \p start must
+point to a variable whose value will be passed unchanged to
+starpu_interface_end_driver_copy_async().
+
+\fn void starpu_interface_end_driver_copy_async(unsigned src_node, unsigned dst_node, double start)
+\ingroup API_Data_Interfaces
+See starpu_interface_start_driver_copy_async().
+
+
 \fn uint32_t starpu_hash_crc32c_be_n(const void *input, size_t n, uint32_t inputcrc)
 \ingroup API_Data_Interfaces
 Compute the CRC of a byte buffer seeded by the \p inputcrc

doc/doxygen/chapters/api/data_management.doxy

@@ -1,6 +1,6 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2010-2017                                CNRS
+ * Copyright (C) 2010-2018                                CNRS
  * Copyright (C) 2011-2012,2017                           Inria
  * Copyright (C) 2009-2011,2014-2017                      Université de Bordeaux
  *
@@ -374,7 +374,7 @@ memory.
 ::STARPU_ACQUIRE_NO_NODE and ::STARPU_ACQUIRE_NO_NODE_LOCK_ALL can be used instead of an
 explicit node number.
 
-\fn int starpu_data_acquire_on_node_cb_sequential_consistency_sync_jobids(starpu_data_handle_t handle, int node, enum starpu_data_access_mode mode, void (*callback)(void *), void *arg, int sequential_consistency, long *pre_sync_jobid, long *post_sync_jobid)
+\fn int starpu_data_acquire_on_node_cb_sequential_consistency_sync_jobids(starpu_data_handle_t handle, int node, enum starpu_data_access_mode mode, void (*callback)(void *), void *arg, int sequential_consistency, int quick, long *pre_sync_jobid, long *post_sync_jobid)
 \ingroup API_Data_Management
 This is the same as starpu_data_acquire_on_node_cb_sequential_consistency(),
 except that the \e pre_sync_jobid and \e post_sync_jobid parameters can be used

doc/doxygen/chapters/api/data_partition.doxy

@@ -1,6 +1,6 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2010-2015,2017                           CNRS
+ * Copyright (C) 2010-2015,2017,2018                      CNRS
  * Copyright (C) 2011-2013                                Inria
  * Copyright (C) 2009-2011,2014-2015,2017-2018            Université de Bordeaux
  *
@@ -190,6 +190,16 @@ in readonly mode through starpu_data_partition_readonly_submit(), and will upgra
 that partitioning into read-write mode for the \p children, by invalidating \p
 initial_handle, and adding the necessary dependencies.
 
+\fn void starpu_data_partition_submit_sequential_consistency(starpu_data_handle_t initial_handle, unsigned nparts, starpu_data_handle_t *children, int sequential_consistency)
+\ingroup API_Data_Partition
+Similar to starpu_data_partition_submit() but also allows to
+specify the coherency to be used for the main data \p initial_handle
+through the parameter \p sequential_consistency.
+
+\fn void starpu_data_partition_not_automatic(starpu_data_handle_t handle)
+\ingroup API_Data_Partition
+Disable the automatic partitioning of the data \p handle for which a asynchronous plan has previously been submitted
+
 \fn void starpu_data_unpartition_submit(starpu_data_handle_t initial_handle, unsigned nparts, starpu_data_handle_t *children, int gathering_node)
 \ingroup API_Data_Partition
 This assumes that \p initial_handle is partitioned into \p children, and submits
@@ -215,6 +225,12 @@ state as obtained with starpu_data_partition_readonly_submit().
 \p gathering_node can be set to -1 to let the runtime decide which memory node
 should be used to gather the pieces.
 
+\fn void starpu_data_unpartition_submit_sequential_consistency(starpu_data_handle_t initial_handle, unsigned nparts, starpu_data_handle_t *children, int gathering_node, int sequential_consistency)
+\ingroup API_Data_Partition
+Similar to starpu_data_unpartition_submit() but also allows to
+specify the coherency to be used for the main data \p initial_handle
+through the parameter \p sequential_consistency.
+
 \fn void starpu_data_partition_clean(starpu_data_handle_t root_data, unsigned nparts, starpu_data_handle_t *children)
 \ingroup API_Data_Partition
 This should be used to clear the partition planning established between \p

doc/doxygen/chapters/api/explicit_dependencies.doxy

@@ -1,6 +1,6 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2010-2013,2015,2017                      CNRS
+ * Copyright (C) 2010-2013,2015,2017,2018                      CNRS
  * Copyright (C) 2009-2011,2014-2016                      Université de Bordeaux
  * Copyright (C) 2011-2012                                Inria
  *
@@ -135,4 +135,15 @@ to execute the tasks. When called several times on the same tag,
 notification will be done only on first call, thus implementing "OR"
 dependencies, until the tag is restarted using starpu_tag_restart().
 
+\fn void starpu_task_end_dep_add(struct starpu_task *t, int nb_deps)
+\ingroup API_Explicit_Dependencies
+Add \p nb_deps end dependencies to the task \p t. This means the task
+will not terminate until the required number of calls to the function
+starpu_task_end_dep_release() has been made.
+
+\fn void starpu_task_end_dep_release(struct starpu_task *t)
+\ingroup API_Explicit_Dependencies
+Unlock 1 end dependency to the task \p t. This function must be called
+after starpu_task_end_dep_add().
+
 */

doc/doxygen/chapters/api/initialization.doxy

@@ -1,8 +1,8 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2010-2017                                CNRS
+ * Copyright (C) 2010-2018                                CNRS
  * Copyright (C) 2011-2012,2014,2017                      Inria
- * Copyright (C) 2009-2011,2014                           Université de Bordeaux
+ * Copyright (C) 2009-2011,2014,2018                      Université de Bordeaux
  *
  * StarPU is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
@@ -281,12 +281,43 @@ the argument was <c>NULL</c>.
 \ingroup API_Initialization_and_Termination
 Return 1 if StarPU is already initialized.
 
+\fn void starpu_wait_initialized(void)
+\ingroup API_Initialization_and_Termination
+Wait for starpu_init() call to finish.
+
 \fn void starpu_shutdown(void)
 \ingroup API_Initialization_and_Termination
 This is StarPU termination method. It must be called at the end of the
 application: statistics and other post-mortem debugging information
 are not guaranteed to be available until this method has been called.
 
+\def STARPU_THREAD_ACTIVE
+\ingroup API_Initialization_and_Termination
+This flag should be passed to starpu_get_next_bindid() and
+starpu_bind_thread_on() when binding a thread which will significantly eat CPU
+time, and should thus have its own dedicated CPU.
+
+\fn unsigned starpu_get_next_bindid(unsigned flags, unsigned *preferred, unsigned npreferred)
+\ingroup API_Initialization_and_Termination
+This returns a PU binding ID which can be used to bind threads with
+starpu_bind_thread_on(). \p flags can be set to STARPU_THREAD_ACTIVE or 0.
+When \p npreferred is set to non-zero, \p preferred is an array of size \p
+npreferred in which a preference of PU binding IDs can be set. By default StarPU
+will return the first PU available for binding.
+
+\fn int starpu_bind_thread_on(int cpuid, unsigned flags, const char *name)
+\ingroup API_Initialization_and_Termination
+This binds the calling thread on the given \p cpuid (which should have been
+obtained with starpu_get_next_bindid()).
+
+This returns -1 if a thread was already bound to this PU (but binding will still
+have been done, and a warning will have been printed), so the caller can tell
+the user how to avoid the issue.
+
+\p name should be set to a unique string so that different calls with the same
+name for the same cpuid does not produce a warning.
+
+
 \fn void starpu_pause(void)
 \ingroup API_Initialization_and_Termination
 Suspend the processing of new tasks by

doc/doxygen/chapters/api/insert_task.doxy

@@ -42,7 +42,7 @@ starpu_task::execute_on_a_specific_worker)
 <li> the specific values ::STARPU_VALUE, ::STARPU_CALLBACK,
 ::STARPU_CALLBACK_ARG, ::STARPU_CALLBACK_WITH_ARG, ::STARPU_PRIORITY,
 ::STARPU_TAG, ::STARPU_TAG_ONLY, ::STARPU_FLOPS, ::STARPU_SCHED_CTX, ::STARPU_CL_ARGS, ::STARPU_CL_ARGS_NFREE,
-::STARPU_TASK_DEPS_ARRAY,
+::STARPU_TASK_DEPS_ARRAY, ::STARPU_TASK_COLOR, ::STARPU_HANDLES_SEQUENTIAL_CONSISTENCY, ::STARPU_TASK_SYNCHRONOUS
 followed by the appropriated objects as defined elsewhere.
 </ul>
 
@@ -156,6 +156,22 @@ be followed by a number of tasks, and an array containing these tasks.
 The function starpu_task_declare_deps_array() will be called with the
 given values.
 
+\def STARPU_TASK_COLOR
+\ingroup API_Insert_Task
+Used when calling starpu_task_insert(), must be followed by an integer
+representing a color
+
+\def STARPU_TASK_SYNCHRONOUS
+\ingroup API_Insert_Task
+Used when calling starpu_task_insert(), must be followed by an integer
+stating if the task is synchronous or not
+
+\def STARPU_HANDLES_SEQUENTIAL_CONSISTENCY
+\ingroup API_Insert_Task
+Used when calling starpu_task_insert(), must be followed by an array
+of characters representing the sequential consistency for each buffer
+of the task.
+
 \fn void starpu_task_insert_data_make_room(struct starpu_codelet *cl, struct starpu_task *task, int *allocated_buffers, int current_buffer, int room)
 \ingroup API_Insert_Task
 Assuming that there are already \p current_buffer data handles passed to
@@ -197,7 +213,7 @@ data, then starpu_codelet_pack_arg_fini().
 Initialize struct starpu_codelet_pack_arg before calling starpu_codelet_pack_arg() and
 starpu_codelet_pack_arg_fini(). This will simply initialize the content of the structure.
 
-\fn void starpu_codelet_pack_arg(struct starpu_codelet_pack_arg_data *state, void *ptr, size_t ptr_size)
+\fn void starpu_codelet_pack_arg(struct starpu_codelet_pack_arg_data *state, const void *ptr, size_t ptr_size)
 \ingroup API_Insert_Task
 Pack one argument into struct starpu_codelet_pack_arg \p state. That structure
 has to be initialized before with starpu_codelet_pack_arg_init(), and after all
@@ -230,4 +246,13 @@ starpu_task_insert().
 If some arguments of type ::STARPU_VALUE are given, the parameter
 starpu_task::cl_arg_free will be set to 1.
 
+\fn int starpu_task_set(struct starpu_task *task, struct starpu_codelet *cl, ...)
+\ingroup API_Insert_Task
+Set the given \p task corresponding to \p cl with the following arguments.
+The argument list must be zero-terminated. The arguments
+following the codelet are the same as the ones for the function
+starpu_task_insert().
+If some arguments of type ::STARPU_VALUE are given, the parameter
+starpu_task::cl_arg_free will be set to 1.
+
 */

doc/doxygen/chapters/api/modularized_scheduler.doxy

@@ -1,7 +1,7 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
  * Copyright (C) 2013-2014                                Inria
- * Copyright (C) 2014-2017                                CNRS
+ * Copyright (C) 2014-2018                                CNRS
  * Copyright (C) 2009-2011,2014-2015,2017-2018                 Université de Bordeaux
  * Copyright (C) 2013                                     Simon Archipoff
  *
@@ -251,7 +251,7 @@ The actual scheduler
 @name Flow-control Fifo Component API
 \ingroup API_Modularized_Scheduler
 
-\fn int starpu_sched_component_can_push(struct starpu_sched_component * component)
+\fn int starpu_sched_component_can_push(struct starpu_sched_component * component, struct starpu_sched_component * to)
 \ingroup API_Modularized_Scheduler
 default function for the can_push component method, just calls can_push of parents until one of them returns non-zero
 
@@ -268,6 +268,14 @@ of the children of the component.
 \ingroup API_Modularized_Scheduler
 function that can be used for the estimated_end component method, which just computes the minimum completion time of the children.
 
+\fn double starpu_sched_component_estimated_end_min_add(struct starpu_sched_component * component, double exp_len);
+\ingroup API_Modularized_Scheduler
+function that can be used for the estimated_end component method, which computes
+the minimum completion time of the children, and adds to it an estimation of how
+existing queued work, plus the exp_len work, can be completed. This is typically
+used instead of starpu_sched_component_estimated_end_min when the component
+contains a queue of tasks, which thus needs to be added to the estimations.
+
 \fn double starpu_sched_component_estimated_end_average(struct starpu_sched_component * component);
 \ingroup API_Modularized_Scheduler
 default function for the estimated_end component method, which just computes the average completion time of the children.

doc/doxygen/chapters/api/mpi.doxy

@@ -1,7 +1,7 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2010-2017                                CNRS
- * Copyright (C) 2011-2012,2017                           Inria
+ * Copyright (C) 2010-2018                                CNRS
+ * Copyright (C) 2011-2012,2016,2017                      Inria
  * Copyright (C) 2009-2011,2014-2018                      Université de Bordeaux
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -26,13 +26,18 @@
 Defined when StarPU has been installed with MPI support. It should be
 used in your code to detect the availability of MPI.
 
+\fn int starpu_mpi_init_conf(int *argc, char ***argv, int initialize_mpi, MPI_Comm comm, struct starpu_conf *conf)
+\ingroup API_MPI_Support
+Initialize the StarPU library with the given \p conf, and initialize the
+StarPU-MPI library with the given MPI communicator \p comm. \p initialize_mpi
+indicates if MPI should be initialized or not by StarPU.
+StarPU-MPI takes the opportunity to modify \p conf to either reserve a core for
+its MPI thread (by default), or execute MPI calls on the CPU driver 0 between tasks.
+
 \fn int starpu_mpi_init_comm(int *argc, char ***argv, int initialize_mpi, MPI_Comm comm)
 \ingroup API_MPI_Support
-Initialize the starpumpi library with the given communicator \p comm.
-\p initialize_mpi indicates if MPI should be initialized or not by StarPU.
-If the value is not 0, MPI will be initialized by calling
-<c>MPI_Init_Thread(argc, argv, MPI_THREAD_SERIALIZED, ...)</c>.
-starpu_init() must be called before starpu_mpi_init_comm().
+This is the same as starpu_mpi_init_conf(), except that this does not initialize
+the StarPU library. The caller thus has to call starpu_init() before this.
 
 \fn int starpu_mpi_init(int *argc, char ***argv, int initialize_mpi)
 \ingroup API_MPI_Support
@@ -55,9 +60,9 @@ calling <c>MPI_Init_Thread(argc, argv, MPI_THREAD_SERIALIZED,
 
 \fn int starpu_mpi_shutdown(void)
 \ingroup API_MPI_Support
-Clean the starpumpi library. This must be called between calling
-\c starpu_mpi functions and starpu_shutdown(). \c MPI_Finalize() will be
-called if StarPU-MPI has been initialized by starpu_mpi_init().
+Clean the starpumpi library. This must be called after calling any
+\c starpu_mpi functions and before the call to starpu_shutdown(), if any.
+\c MPI_Finalize() will be called if StarPU-MPI has been initialized by starpu_mpi_init().
 
 \fn void starpu_mpi_comm_amounts_retrieve(size_t *comm_amounts)
 \ingroup API_MPI_Support

doc/doxygen/chapters/api/opencl_extensions.doxy

@@ -1,6 +1,6 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2010-2017                                CNRS
+ * Copyright (C) 2010-2018                                CNRS
  * Copyright (C) 2009-2011,2014,2016                      Université de Bordeaux
  * Copyright (C) 2011-2012                                Inria
  *
@@ -82,10 +82,7 @@ Here an example:
 int n;
 cl_int err;
 cl_kernel kernel;
-n = starpu_opencl_set_kernel_args(&err, 2, &kernel,
-                                  sizeof(foo), &foo,
-                                  sizeof(bar), &bar,
-                                  0);
+n = starpu_opencl_set_kernel_args(&err, 2, &kernel, sizeof(foo), &foo, sizeof(bar), &bar, 0);
 if (n != 2)
    fprintf(stderr, "Error : %d\n", err);
 \endcode

doc/doxygen/chapters/api/scheduling_policy.doxy

@@ -1,7 +1,7 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
  * Copyright (C) 2011-2013                                Inria
- * Copyright (C) 2010-2017                                CNRS
+ * Copyright (C) 2010-2018                                CNRS
  * Copyright (C) 2009-2011,2014-2018                      Université de Bordeaux
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -229,7 +229,13 @@ Return an estimated speedup factor relative to CPU speed
 
 \fn double starpu_task_expected_data_transfer_time(unsigned memory_node, struct starpu_task *task)
 \ingroup API_Scheduling_Policy
-Return expected data transfer time in micro-seconds.
+Return expected data transfer time in micro-seconds for the given \p
+memory_node. Prefer using starpu_task_expected_data_transfer_time_for() which is
+more precise.
+
+\fn double starpu_task_expected_data_transfer_time_for(struct starpu_task *task, unsigned worker)
+\ingroup API_Scheduling_Policy
+Return expected data transfer time in micro-seconds for the given \p worker.
 
 \fn double starpu_data_expected_transfer_time(starpu_data_handle_t handle, unsigned memory_node, enum starpu_data_access_mode mode)
 \ingroup API_Scheduling_Policy
@@ -249,11 +255,35 @@ Whether \ref STARPU_PREFETCH was set
 
 \fn int starpu_prefetch_task_input_on_node(struct starpu_task *task, unsigned node)
 \ingroup API_Scheduling_Policy
-Prefetch data for a given task on a given node
+Prefetch data for a given p task on a given p node
+
+\fn int starpu_prefetch_task_input_on_node_prio(struct starpu_task *task, unsigned node, int prio)
+\ingroup API_Scheduling_Policy
+Prefetch data for a given p task on a given p node with a given priority
 
 \fn int starpu_idle_prefetch_task_input_on_node(struct starpu_task *task, unsigned node)
 \ingroup API_Scheduling_Policy
-Prefetch data for a given task on a given node when the bus is idle
+Prefetch data for a given p task on a given p node when the bus is idle
+
+\fn int starpu_idle_prefetch_task_input_on_node_prio(struct starpu_task *task, unsigned node, int prio)
+\ingroup API_Scheduling_Policy
+Prefetch data for a given p task on a given p node when the bus is idle with a given priority
+
+\fn int starpu_prefetch_task_input_for(struct starpu_task *task, unsigned worker)
+\ingroup API_Scheduling_Policy
+Prefetch data for a given p task on a given p worker
+
+\fn int starpu_prefetch_task_input_for_prio(struct starpu_task *task, unsigned worker, int prio)
+\ingroup API_Scheduling_Policy
+Prefetch data for a given p task on a given p worker with a given priority
+
+\fn int starpu_idle_prefetch_task_input_for(struct starpu_task *task, unsigned worker)
+\ingroup API_Scheduling_Policy
+Prefetch data for a given p task on a given p worker when the bus is idle
+
+\fn int starpu_idle_prefetch_task_input_for_prio(struct starpu_task *task, unsigned worker, int prio)
+\ingroup API_Scheduling_Policy
+Prefetch data for a given p task on a given p worker when the bus is idle with a given priority
 
 \fn void starpu_task_notify_ready_soon_register(starpu_notify_ready_soon_func f, void *data);
 \ingroup API_Scheduling_Policy
@@ -266,11 +296,4 @@ started and we know how long it will take.
 The scheduling policies indicates if the worker may pop tasks from the list of other workers
 or if there is a central list with task for all the workers
 
-\fn void starpu_sched_break(struct starpu_task *task)
-\ingroup API_Scheduling_Policy
-This should be called just after a scheduling decision for task \p task was
-made. It will raise SIGTRAP if the ::STARPU_TASK_BREAK_ON_SCHED environment
-variable was used, which thus breakpoints inside a debugger, thus allowing to
-inspect why that scheduling decision was made.
-
 */

doc/doxygen/chapters/code/complex.c

@@ -1,6 +1,6 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2010-2013,2015,2017                      CNRS
+ * Copyright (C) 2010-2013,2015,2017,2018                 CNRS
  * Copyright (C) 2010-2014                                Université de Bordeaux
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -16,10 +16,7 @@
  */
 
 //! [To be included. You should update doxygen if you see this text.]
-#define STARPU_COMPLEX_GET_REAL(interface)	\
-        (((struct starpu_complex_interface *)(interface))->real)
-#define STARPU_COMPLEX_GET_IMAGINARY(interface)	\
-        (((struct starpu_complex_interface *)(interface))->imaginary)
-#define STARPU_COMPLEX_GET_NX(interface)	\
-        (((struct starpu_complex_interface *)(interface))->nx)
+#define STARPU_COMPLEX_GET_REAL(interface)	(((struct starpu_complex_interface *)(interface))->real)
+#define STARPU_COMPLEX_GET_IMAGINARY(interface)	(((struct starpu_complex_interface *)(interface))->imaginary)
+#define STARPU_COMPLEX_GET_NX(interface)	(((struct starpu_complex_interface *)(interface))->nx)
 //! [To be included. You should update doxygen if you see this text.]

doc/doxygen/chapters/code/disk_compute.c

@@ -1,6 +1,6 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2013,2017                                CNRS
+ * Copyright (C) 2013,2017,2018                           CNRS
  * Copyright (C) 2013                                     Inria
  * Copyright (C) 2014                                     Université de Bordeaux
  * Copyright (C) 2013                                     Corentin Salingue
@@ -79,9 +79,6 @@ int main(int argc, char **argv)
 		C[j] = 0;
 	}
 
-
-
-
 	/* you create a file to store the vector ON the disk */
 	FILE * f = fopen(path_file_start, "wb+");
 	if (f == NULL)

doc/doxygen/chapters/code/matmul_pragma2.c

@@ -1,6 +1,6 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2010-2013,2015,2017                      CNRS
+ * Copyright (C) 2010-2013,2015,2017,2018                 CNRS
  * Copyright (C) 2010-2014                                Université de Bordeaux
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -24,6 +24,5 @@
 # define __task
 #endif
 
-static void matmul (const float *A, const float *B, float *C,
-                    unsigned nx, unsigned ny, unsigned nz) __task;
+static void matmul (const float *A, const float *B, float *C, unsigned nx, unsigned ny, unsigned nz) __task;
 //! [To be included. You should update doxygen if you see this text.]

doc/doxygen/chapters/code/scal_pragma.cu

@@ -1,6 +1,6 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2010-2013,2015,2017                      CNRS
+ * Copyright (C) 2010-2013,2015,2017,2018                 CNRS
  * Copyright (C) 2010-2014,2016                           Université de Bordeaux
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -21,8 +21,7 @@
 #include <starpu.h>
 #include <stdlib.h>
 
-static __global__ void
-vector_mult_cuda (unsigned n, float *val, float factor)
+static __global__ void vector_mult_cuda (unsigned n, float *val, float factor)
 {
   unsigned i = blockIdx.x * blockDim.x + threadIdx.x;
 
@@ -36,8 +35,7 @@ extern "C" void vector_scal_cuda (size_t size, float vector[], float factor)
   unsigned threads_per_block = 64;
   unsigned nblocks = (size + threads_per_block - 1) / threads_per_block;
 
-  vector_mult_cuda <<< nblocks, threads_per_block, 0,
-    starpu_cuda_get_local_stream () >>> (size, vector, factor);
+  vector_mult_cuda <<< nblocks, threads_per_block, 0, starpu_cuda_get_local_stream () >>> (size, vector, factor);
 
   cudaStreamSynchronize (starpu_cuda_get_local_stream ());
 }

doc/doxygen/chapters/code/vector_scal_c.c

@@ -1,6 +1,6 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2010-2013,2015,2017                      CNRS
+ * Copyright (C) 2010-2013,2015,2017,2018                 CNRS
  * Copyright (C) 2013                                     Inria
  * Copyright (C) 2010-2014                                Université de Bordeaux
  *
@@ -70,8 +70,7 @@ int main(int argc, char **argv)
     starpu_init(NULL);
 
 #ifdef STARPU_USE_OPENCL
-        starpu_opencl_load_opencl_from_file(
-               "examples/basic_examples/vector_scal_opencl_kernel.cl", &programs, NULL);
+    starpu_opencl_load_opencl_from_file("examples/basic_examples/vector_scal_opencl_kernel.cl", &programs, NULL);
 #endif
 
     /* Tell StaPU to associate the "vector" vector with the "vector_handle"
@@ -88,8 +87,7 @@ int main(int argc, char **argv)
      *  - the fifth argument is the size of each element.
      */
     starpu_data_handle_t vector_handle;
-    starpu_vector_data_register(&vector_handle, STARPU_MAIN_RAM, (uintptr_t)vector,
-                                NX, sizeof(vector[0]));
+    starpu_vector_data_register(&vector_handle, STARPU_MAIN_RAM, (uintptr_t)vector, NX, sizeof(vector[0]));
 
     float factor = 3.14;
 

doc/doxygen/chapters/code/vector_scal_cuda.cu

@@ -1,6 +1,6 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2010-2013,2015,2017                      CNRS
+ * Copyright (C) 2010-2013,2015,2017,2018                      CNRS
  * Copyright (C) 2010,2014                                Université de Bordeaux
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -17,8 +17,7 @@
 //! [To be included. You should update doxygen if you see this text.]
 #include <starpu.h>
 
-static __global__ void vector_mult_cuda(unsigned n, float *val,
-                                        float factor)
+static __global__ void vector_mult_cuda(unsigned n, float *val, float factor)
 {
         unsigned i =  blockIdx.x*blockDim.x + threadIdx.x;
         if (i < n)
@@ -36,8 +35,7 @@ extern "C" void scal_cuda_func(void *buffers[], void *_args)
         unsigned threads_per_block = 64;
         unsigned nblocks = (n + threads_per_block-1) / threads_per_block;
 
-        vector_mult_cuda<<<nblocks,threads_per_block, 0, starpu_cuda_get_local_stream()>>>
-	                (n, val, *factor);
+        vector_mult_cuda<<<nblocks,threads_per_block, 0, starpu_cuda_get_local_stream()>>>(n, val, *factor);
 
         cudaStreamSynchronize(starpu_cuda_get_local_stream());
 }

doc/doxygen/chapters/code/vector_scal_opencl.c

@@ -1,6 +1,6 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2010,2012-2013,2015,2017                 CNRS
+ * Copyright (C) 2010,2012-2013,2015,2017,2018            CNRS
  * Copyright (C) 2011,2014                                Université de Bordeaux
  * Copyright (C) 2010                                     Inria
  *
@@ -38,8 +38,7 @@ void scal_opencl_func(void *buffers[], void *_args)
 	 id = starpu_worker_get_id();
 	 devid = starpu_worker_get_devid(id);
 
-	 err = starpu_opencl_load_kernel(&kernel, &queue,
-					 &programs,
+	 err = starpu_opencl_load_kernel(&kernel, &queue, &programs,
 					 "vector_mult_opencl", /* Name of the codelet */
 					 devid);
 	 if (err != CL_SUCCESS) STARPU_OPENCL_REPORT_ERROR(err);
@@ -57,13 +56,11 @@ void scal_opencl_func(void *buffers[], void *_args)
         cl_device_id device;
 
         starpu_opencl_get_device(devid, &device);
-        err = clGetKernelWorkGroupInfo (kernel, device, CL_KERNEL_WORK_GROUP_SIZE,
-                                        sizeof(local), &local, &s);
+        err = clGetKernelWorkGroupInfo (kernel, device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(local), &local, &s);
         if (err != CL_SUCCESS) STARPU_OPENCL_REPORT_ERROR(err);
         if (local > global) local=global;
 
-        err = clEnqueueNDRangeKernel(queue, kernel, 1, NULL, &global, &local, 0,
-                                     NULL, &event);
+        err = clEnqueueNDRangeKernel(queue, kernel, 1, NULL, &global, &local, 0, NULL, &event);
         if (err != CL_SUCCESS) STARPU_OPENCL_REPORT_ERROR(err);
     }
 

doc/doxygen/refman.tex

@@ -1,7 +1,7 @@
 % StarPU --- Runtime system for heterogeneous multicore architectures.
 %
-% Copyright (C) 2013,2015-2016                           Inria
-% Copyright (C) 2013-2017                                CNRS
+% Copyright (C) 2013,2015-2016,2018                      Inria
+% Copyright (C) 2013-2018                                CNRS
 % Copyright (C) 2014,2018                                Université de Bordeaux
 %
 % StarPU is free software; you can redistribute it and/or modify
@@ -16,6 +16,7 @@
 % See the GNU Lesser General Public License in COPYING.LGPL for more details.
 %
 \input{./version.sty}
+\setlength{\parskip}{0pt}
 \begin{titlepage}
 \vspace*{4cm}
 {\Huge \textbf{StarPU Handbook}}\\
@@ -54,6 +55,7 @@ Documentation License”.
 \end{figure}
 
 \pagenumbering{roman}
+\setcounter{tocdepth}{2}
 \tableofcontents
 \pagenumbering{arabic}
 \hypersetup{pageanchor=true,citecolor=blue}
@@ -114,7 +116,7 @@ Documentation License”.
 \hypertarget{SchedulingContextHypervisor}{}
 \input{SchedulingContextHypervisor}
 
-\chapter{Modularized Scheduler}
+\chapter{Modularized Schedulers}
 \label{ModularizedScheduler}
 \hypertarget{ModularizedScheduler}{}
 \input{ModularizedScheduler}
@@ -156,7 +158,7 @@ Documentation License”.
 \hypertarget{FFTSupport}{}
 \input{FFTSupport}
 
-\chapter{MIC/SCC Support}
+\chapter{MIC Xeon Phi / SCC Support}
 \label{MICSCCSupport}
 \hypertarget{MICSCCSupport}{}
 \input{MICSCCSupport}
@@ -181,7 +183,7 @@ Documentation License”.
 \hypertarget{SimGridSupport}{}
 \input{SimGridSupport}
 
-\chapter{OpenMP Runtime Support}
+\chapter{The StarPU OpenMP Runtime Support (SORS)}
 \label{OpenMPRuntimeSupport}
 \hypertarget{OpenMPRuntimeSupport}{}
 \input{OpenMPRuntimeSupport}
@@ -206,6 +208,9 @@ Documentation License”.
 \chapter{Module Index}
 \input{modules}
 
+\chapter{Deprecated List}
+\input{deprecated}
+
 \chapter{Module Documentation a.k.a StarPU's API}
 \label{ModuleDocumentation}
 \hypertarget{ModuleDocumentation}{}
@@ -276,15 +281,19 @@ Documentation License”.
 \input{starpu__fxt_8h}
 \input{starpu__hash_8h}
 \input{starpu__mic_8h}
+\input{starpu__mod_8f90}
+\input{starpu__mpi_8h}
 \input{starpu__opencl_8h}
 \input{starpu__openmp_8h}
 \input{starpu__perfmodel_8h}
 \input{starpu__profiling_8h}
 \input{starpu__rand_8h}
 \input{starpu__scc_8h}
+\input{starpu__sched__component_8h}
 \input{starpu__sched__ctx_8h}
 \input{starpu__sched__ctx__hypervisor_8h}
 \input{starpu__scheduler_8h}
+\input{starpu__simgrid__wrap_8h}
 \input{starpu__sink_8h}
 \input{starpu__stdlib_8h}
 \input{starpu__task_8h}
@@ -297,7 +306,6 @@ Documentation License”.
 \input{starpu__tree_8h}
 \input{starpu__util_8h}
 \input{starpu__worker_8h}
-\input{starpu__mpi_8h}
 \input{starpufft_8h}
 \input{sc__hypervisor_8h}
 \input{sc__hypervisor__config_8h}
@@ -317,7 +325,7 @@ Documentation License”.
 \hypertarget{FullSourceCodeVectorScal}{}
 \input{FullSourceCodeVectorScal}
 
-\chapter{GNU Free Documentation License}
+\chapter{The GNU Free Documentation License}
 \label{GNUFreeDocumentationLicense}
 \hypertarget{GNUFreeDocumentationLicense}{}
 \input{GNUFreeDocumentationLicense}

examples/Makefile.am

@@ -80,6 +80,7 @@ EXTRA_DIST = 					\
 	sched_ctx/axpy_partition_gpu.h			\
 	sched_ctx/axpy_partition_gpu.cu			\
 	heat/heat.sh					\
+	cholesky/cholesky.sh				\
 	lu/lu.sh
 
 
@@ -216,6 +217,7 @@ STARPU_EXAMPLES +=				\
 	basic_examples/variable			\
 	basic_examples/multiformat              \
 	basic_examples/dynamic_handles          \
+	basic_examples/task_insert_color	\
 	mlr/mlr					\
 	cpp/incrementer_cpp			\
 	cpp/add_vectors				\
@@ -228,6 +230,7 @@ STARPU_EXAMPLES +=				\
 	filters/fmultiple_submit_readonly	\
 	filters/fmultiple_submit_implicit	\
 	filters/frecursive			\
+	filters/fplan_notautomatic		\
 	tag_example/tag_example			\
 	tag_example/tag_example2		\
 	tag_example/tag_example3		\
@@ -248,7 +251,9 @@ STARPU_EXAMPLES +=				\
 	sched_ctx/dummy_sched_with_ctx		\
 	worker_collections/worker_tree_example  \
 	reductions/dot_product			\
-	reductions/minmax_reduction
+	reductions/minmax_reduction		\
+	dependency/task_end_dep			\
+	dependency/sequential_consistency
 
 endif
 
@@ -312,6 +317,12 @@ TESTS += \
 	lu/lu.sh
 
 endif
+
+endif
+
+if STARPU_SIMGRID
+TESTS += \
+	 cholesky/cholesky.sh
 endif
 endif
 

examples/basic_examples/task_insert_color.c

@@ -0,0 +1,89 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2018                                      CNRS
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <starpu.h>
+
+#define FPRINTF(ofile, fmt, ...) do { if (!getenv("STARPU_SSILENT")) {fprintf(ofile, fmt, ## __VA_ARGS__); }} while(0)
+
+void func(void *descr[], void *_args)
+{
+	int *x = (int *)STARPU_VARIABLE_GET_PTR(descr[0]);
+	(void)_args;
+
+	*x *= 2;
+}
+
+struct starpu_codelet mycodelet =
+{
+	.modes = { STARPU_RW },
+	.cpu_funcs = {func},
+	.cpu_funcs_name = {"func"},
+        .nbuffers = 1
+};
+
+struct starpu_codelet mycodelet_color =
+{
+	.modes = { STARPU_RW },
+	.cpu_funcs = {func},
+	.cpu_funcs_name = {"func"},
+        .nbuffers = 1,
+	.color = 0x0000FF,
+};
+
+int main(void)
+{
+	unsigned i;
+	int value=42;
+	starpu_data_handle_t handle;
+	int ret;
+
+	ret = starpu_init(NULL);
+	if (ret == -ENODEV) goto enodev;
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_init");
+
+	starpu_variable_data_register(&handle, STARPU_MAIN_RAM, (uintptr_t)&value, sizeof(value));
+
+	// In the trace file, the following task should be green (executed on CPU)
+	ret = starpu_task_insert(&mycodelet, STARPU_RW, handle, STARPU_NAME, "mytask",
+				 0);
+	if (STARPU_UNLIKELY(ret == -ENODEV))
+	{
+		starpu_data_unregister(handle);
+		goto enodev;
+	}
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_task_insert");
+
+	// In the trace file, the following task will be red as specified by STARPU_TASK_COLOR
+	ret = starpu_task_insert(&mycodelet, STARPU_RW, handle, STARPU_NAME, "mytask",
+				 STARPU_TASK_COLOR, 0xFF0000,
+				 0);
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_task_insert");
+
+	// In the trace file, the following task will be blue as specified by the field color of mycodelet_color
+	ret = starpu_task_insert(&mycodelet_color, STARPU_RW, handle, STARPU_NAME, "mytask",
+				 0);
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_task_insert");
+
+	starpu_task_wait_for_all();
+	starpu_data_unregister(handle);
+
+	starpu_shutdown();
+
+	return 0;
+
+ enodev:
+	return 77;
+}

examples/basic_examples/vector_scal.c

@@ -1,7 +1,7 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
  * Copyright (C) 2011-2013                                Inria
- * Copyright (C) 2009-2016                                Université de Bordeaux
+ * Copyright (C) 2009-2016,2018                           Université de Bordeaux
  * Copyright (C) 2010-2013,2015,2017                      CNRS
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -151,6 +151,7 @@ int main(void)
 	 *  - the fifth argument is the size of each element.
 	 */
 	starpu_data_handle_t vector_handle;
+	starpu_memory_pin(vector, sizeof(vector));
 	starpu_vector_data_register(&vector_handle, STARPU_MAIN_RAM, (uintptr_t)vector, NX, sizeof(vector[0]));
 
 	float factor = 3.14;
@@ -178,6 +179,7 @@ int main(void)
 	/* StarPU does not need to manipulate the array anymore so we can stop
  	 * monitoring it */
 	starpu_data_unregister(vector_handle);
+	starpu_memory_unpin(vector, sizeof(vector));
 
 #ifdef STARPU_USE_OPENCL
         ret = starpu_opencl_unload_opencl(&opencl_program);

examples/cholesky/cholesky.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+# StarPU --- Runtime system for heterogeneous multicore architectures.
+#
+# Copyright (C) 2018                 Université de Bordeaux
+#
+# StarPU is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 2.1 of the License, or (at
+# your option) any later version.
+#
+# StarPU is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See the GNU Lesser General Public License in COPYING.LGPL for more details.
+#
+
+ROOT=${0%.sh}
+#[ -n "$STARPU_SCHEDS" ] || STARPU_SCHEDS=`$(dirname $0)/../../tools/starpu_sched_display`
+[ -n "$STARPU_SCHEDS" ] || STARPU_SCHEDS="dmdas modular-heft modular-heft-prio dmdar dmda dmdasd prio lws"
+[ -n "$STARPU_HOSTNAME" ] || export STARPU_HOSTNAME=mirage
+unset MALLOC_PERTURB_
+
+(
+echo -n "#"
+for STARPU_SCHED in $STARPU_SCHEDS ; do
+	echo -n "	$STARPU_SCHED"
+done
+echo
+
+for size in `seq 2 2 30` ; do
+	echo -n "$((size * 960))"
+	for STARPU_SCHED in $STARPU_SCHEDS
+	do
+		export STARPU_SCHED
+		GFLOPS=`${ROOT}_implicit -size $((size * 960)) -nblocks $size 2> /dev/null | grep -v GFlops | cut -d '	' -f 3`
+		[ -n "$GFLOPS" ] || GFLOPS='""'
+		echo -n "	$GFLOPS"
+	done
+	echo 
+done
+) | tee cholesky.output
+
+[ -n "$TERMINAL" ] || TERMINAL=eps
+[ -n "$OUTFILE" ] || OUTFILE=cholesky.eps
+cat > cholesky.gp << EOF
+set terminal $TERMINAL
+set output "$OUTFILE"
+set key top left
+set xlabel "size"
+set ylabel "GFlops"
+plot \\
+EOF
+
+N=2
+for STARPU_SCHED in $STARPU_SCHEDS
+do
+	echo "'cholesky.output' using 1:$N with lines title '$STARPU_SCHED', \\" >> cholesky.gp
+	N=$(($N + 1))
+done
+gnuplot cholesky.gp
+gv $OUTFILE
+true

examples/cpp/add_vectors_interface.cpp

@@ -2,7 +2,7 @@
  *
  * Copyright (C) 2010-2014,2016-2017                      CNRS
  * Copyright (C) 2012,2017                                Inria
- * Copyright (C) 2009-2011,2013-2015,2017                 Université de Bordeaux
+ * Copyright (C) 2009-2011,2013-2015,2017-2018                 Université de Bordeaux
  *
  * StarPU is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
@@ -259,7 +259,8 @@ static const struct starpu_data_copy_methods vector_cpp_copy_data_methods_s =
 
 static void register_vector_cpp_handle(starpu_data_handle_t handle, unsigned home_node, void *data_interface);
 static starpu_ssize_t allocate_vector_cpp_buffer_on_node(void *data_interface_, unsigned dst_node);
-static void *vector_cpp_handle_to_pointer(starpu_data_handle_t handle, unsigned node);
+static void *vector_cpp_to_pointer(void *data_interface, unsigned node);
+static void *vector_cpp_pointer_is_inside(void *data_interface, unsigned node, void *ptr);
 static void free_vector_cpp_buffer_on_node(void *data_interface, unsigned node);
 static void free_vector_cpp_buffer_on_node(void *data_interface, unsigned node);
 static size_t vector_cpp_interface_get_size(starpu_data_handle_t handle);
@@ -277,7 +278,8 @@ static struct starpu_data_interface_ops interface_vector_cpp_ops =
 	.allocate_data_on_node = allocate_vector_cpp_buffer_on_node,
 	.free_data_on_node = free_vector_cpp_buffer_on_node,
 	.copy_methods = &vector_cpp_copy_data_methods_s,
-	.handle_to_pointer = vector_cpp_handle_to_pointer,
+	.to_pointer = vector_cpp_to_pointer,
+	.pointer_is_inside = vector_cpp_pointer_is_inside,
 	.get_size = vector_cpp_interface_get_size,
 	.footprint = footprint_vector_cpp_interface_crc32,
 	.compare = vector_cpp_compare,
@@ -299,7 +301,8 @@ static struct starpu_data_interface_ops interface_vector_cpp_ops =
 	allocate_vector_cpp_buffer_on_node,
 	free_vector_cpp_buffer_on_node,
 	&vector_cpp_copy_data_methods_s,
-	vector_cpp_handle_to_pointer,
+	vector_cpp_to_pointer,
+	vector_cpp_pointer_is_inside,
 	vector_cpp_interface_get_size,
 	footprint_vector_cpp_interface_crc32,
 	vector_cpp_compare,
@@ -316,16 +319,23 @@ static struct starpu_data_interface_ops interface_vector_cpp_ops =
 };
 #endif
 
-static void *vector_cpp_handle_to_pointer(starpu_data_handle_t handle, unsigned node)
+static void *vector_cpp_to_pointer(void *data_interface, unsigned node)
 {
-	STARPU_ASSERT(starpu_data_test_if_allocated_on_node(handle, node));
-
-	struct vector_cpp_interface *vector_interface = (struct vector_cpp_interface *)
-		starpu_data_get_interface_on_node(handle, node);
+	(void) node;
+	struct vector_cpp_interface *vector_interface = (struct vector_cpp_interface *) data_interface;
 
 	return (void*) vector_interface->ptr;
 }
 
+static int vector_cpp_pointer_is_inside(void *data_interface, unsigned node, void *ptr)
+{
+	(void) node;
+	struct vector_cpp_interface *vector_interface = (struct vector_cpp_interface *) data_interface;
+
+	return (char*) ptr >= (char*) vector_interface->ptr &&
+		(char*) ptr < (char*) vector_interface->ptr + vector_interface->nx*vector_interface->elemsize;
+}
+
 static void register_vector_cpp_handle(starpu_data_handle_t handle, unsigned home_node, void *data_interface)
 {
 	struct vector_cpp_interface *vector_interface = (struct vector_cpp_interface *) data_interface;

examples/dependency/sequential_consistency.c

@@ -0,0 +1,181 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2018                                     CNRS
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <starpu.h>
+
+#define FPRINTF(ofile, fmt, ...) do { if (!getenv("STARPU_SSILENT")) {fprintf(ofile, fmt, ## __VA_ARGS__); }} while(0)
+
+void cpu_codeletA(void *descr[], void *args);
+void cpu_codeletB(void *descr[], void *args);
+void cpu_codeletC(void *descr[], void *args);
+
+struct starpu_codelet clA =
+{
+	.cpu_funcs = {cpu_codeletA},
+	.cpu_funcs_name = {"cpu_codeletA"},
+	.nbuffers = 1,
+	.modes = {STARPU_RW},
+	.name = "codeletA"
+};
+
+struct starpu_codelet clB =
+{
+	.cpu_funcs = {cpu_codeletB},
+	.cpu_funcs_name = {"cpu_codeletB"},
+	.nbuffers = 1,
+	.modes = {STARPU_RW},
+	.name = "codeletB"
+};
+
+struct starpu_codelet clC =
+{
+	.cpu_funcs = {cpu_codeletC},
+	.cpu_funcs_name = {"cpu_codeletC"},
+	.nbuffers = 1,
+	.modes = {STARPU_RW},
+	.name = "codeletC"
+};
+
+void cpu_codeletA(void *descr[], void *args)
+{
+	int *val = (int *)STARPU_VARIABLE_GET_PTR(descr[0]);
+	starpu_data_handle_t value_handle;
+	starpu_tag_t tagHoldC;
+	int ret;
+	unsigned char handle_sequential_consistency[] = {0};
+
+	FPRINTF(stderr, "[Task A] Value = %d\n", *val);
+
+	starpu_codelet_unpack_args(args, &value_handle, &tagHoldC);
+
+	// With several data, one would need to use a dynamically
+	// allocated array for the sequential consistency,
+	// the array could be freed immediately after calling
+	// starpu_task_insert()
+
+	ret = starpu_task_insert(&clB,
+				 STARPU_RW, value_handle,
+				 STARPU_CALLBACK_WITH_ARG, starpu_tag_notify_from_apps, tagHoldC,
+				 STARPU_HANDLES_SEQUENTIAL_CONSISTENCY, handle_sequential_consistency,
+				 STARPU_NAME, "taskB",
+				 0);
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_task_insert");
+
+	*val *= 2;
+}
+
+void cpu_codeletB(void *descr[], void *args)
+{
+	(void)args;
+	int *val = (int *)STARPU_VARIABLE_GET_PTR(descr[0]);
+
+	FPRINTF(stderr, "[Task B] Value = %d\n", *val);
+	STARPU_ASSERT_MSG(*val == 24, "Incorrect value %d (expected 24)\n", *val);
+	*val += 1;
+}
+
+void cpu_codeletC(void *descr[], void *args)
+{
+	(void)args;
+	int *val = (int *)STARPU_VARIABLE_GET_PTR(descr[0]);
+
+	FPRINTF(stderr, "[Task C] Value = %d\n", *val);
+	STARPU_ASSERT_MSG(*val == 25, "Incorrect value %d (expected 25)\n", *val);
+	*val *= 2;
+}
+
+/*
+ * Submit taskA and hold it
+ * Submit taskC and hold it
+ * Release taskA
+ * Execute taskA       --> submit taskB
+ * Execute taskB       --> callback: release taskC
+ *
+ * All three tasks use the same data in RW, taskB is submitted after
+ * taskC, so taskB should normally only execute after taskC but as the
+ * sequential consistency for (taskB, data) is unset, taskB can
+ * execute straightaway
+ */
+int main(void)
+{
+        int value=12;
+	int ret;
+	starpu_data_handle_t value_handle;
+	starpu_tag_t tagHoldA = 42;
+	starpu_tag_t tagHoldC = 84;
+	starpu_tag_t tagA = 421;
+	starpu_tag_t tagC = 842;
+
+	struct starpu_conf conf;
+
+	if (sizeof(starpu_tag_t) > sizeof(void*))
+	{
+		// Can't pass a tag_t through callback arg :/
+		return 77;
+	}
+
+	starpu_conf_init(&conf);
+	conf.nmic = 0;
+	conf.nscc = 0;
+	conf.nmpi_ms = 0;
+
+        ret = starpu_init(&conf);
+	if (STARPU_UNLIKELY(ret == -ENODEV))
+	{
+		return 77;
+	}
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_init");
+
+	if (starpu_cpu_worker_get_count() < 1)
+	{
+		FPRINTF(stderr, "This application requires at least 1 cpu worker\n");
+		starpu_shutdown();
+		return 77;
+	}
+
+	starpu_variable_data_register(&value_handle, STARPU_MAIN_RAM, (uintptr_t)&value, sizeof(value));
+
+	starpu_tag_declare_deps_array(tagA, 1, &tagHoldA);
+	starpu_tag_declare_deps_array(tagC, 1, &tagHoldC);
+
+	ret = starpu_task_insert(&clA,
+				 STARPU_TAG, tagA,
+				 STARPU_RW, value_handle,
+				 STARPU_VALUE, &value_handle, sizeof(starpu_data_handle_t),
+				 STARPU_VALUE, &tagHoldC, sizeof(starpu_tag_t),
+				 STARPU_NAME, "taskA",
+				 0);
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_task_insert");
+	ret = starpu_task_insert(&clC,
+				 STARPU_TAG, tagC,
+				 STARPU_RW, value_handle,
+				 STARPU_NAME, "taskC",
+				 0);
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_task_insert");
+
+	// Release taskA (we want to make sure it will execute after taskC has been submitted)
+	starpu_tag_notify_from_apps(tagHoldA);
+
+	starpu_data_unregister(value_handle);
+
+	STARPU_ASSERT_MSG(value == 50, "Incorrect value %d (expected 50)\n", value);
+
+	starpu_shutdown();
+
+	FPRINTF(stderr, "Value = %d\n", value);
+
+	return ret;
+}

examples/dependency/task_end_dep.c

@@ -0,0 +1,98 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2018                                     CNRS
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <starpu.h>
+
+#define FPRINTF(ofile, fmt, ...) do { if (!getenv("STARPU_SSILENT")) {fprintf(ofile, fmt, ## __VA_ARGS__); }} while(0)
+
+void cpu_codelet2(void *descr[], void *args)
+{
+	(void)descr;
+	(void)args;
+}
+
+struct starpu_codelet cl2 =
+{
+	.cpu_funcs = {cpu_codelet2},
+	.cpu_funcs_name = {"cpu_codelet2"},
+	.name = "codelet2"
+};
+
+void cpu_codelet(void *descr[], void *args)
+{
+	(void)args;
+	int *val = (int *)STARPU_VARIABLE_GET_PTR(descr[0]);
+	struct starpu_task *task;
+
+	task = starpu_task_get_current();
+	starpu_task_end_dep_add(task, 1);
+
+	starpu_task_insert(&cl2,
+			   STARPU_CALLBACK_WITH_ARG, starpu_task_end_dep_release, task,
+			   0);
+	*val *= 2;
+}
+
+struct starpu_codelet cl =
+{
+	.cpu_funcs = {cpu_codelet},
+	.cpu_funcs_name = {"cpu_codelet"},
+	.nbuffers = 1,
+	.modes = {STARPU_RW},
+	.name = "codelet"
+};
+
+int main(void)
+{
+        int value=12;
+	int ret;
+	starpu_data_handle_t value_handle;
+	struct starpu_conf conf;
+
+	starpu_conf_init(&conf);
+	conf.nmic = 0;
+	conf.nscc = 0;
+	conf.nmpi_ms = 0;
+
+        ret = starpu_init(&conf);
+	if (STARPU_UNLIKELY(ret == -ENODEV))
+	{
+		return 77;
+	}
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_init");
+
+	if (starpu_cpu_worker_get_count() < 1)
+	{
+		FPRINTF(stderr, "This application requires at least 1 cpu worker\n");
+		starpu_shutdown();
+		return 77;
+	}
+
+	starpu_variable_data_register(&value_handle, STARPU_MAIN_RAM, (uintptr_t)&value, sizeof(value));
+
+	ret = starpu_task_insert(&cl,
+				 STARPU_RW, value_handle,
+				 0);
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_task_insert");
+
+	starpu_data_unregister(value_handle);
+
+        starpu_shutdown();
+
+	FPRINTF(stderr, "Value = %d\n", value);
+
+	return ret;
+}

examples/filters/custom_mf/custom_interface.c

@@ -2,7 +2,7 @@
  *
  * Copyright (C) 2012-2013                                Inria
  * Copyright (C) 2012-2013,2015-2017                      CNRS
- * Copyright (C) 2012-2014                                Université de Bordeaux
+ * Copyright (C) 2012-2014, 2018                                Université de Bordeaux
  *
  * StarPU is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
@@ -73,8 +73,8 @@ static void     register_custom_handle(starpu_data_handle_t handle,
 				       void *data_interface);
 static starpu_ssize_t  allocate_custom_buffer_on_node(void *data_interface_,
 					       unsigned dst_node);
-static void*    custom_handle_to_pointer(starpu_data_handle_t data_handle,
-					 unsigned node);
+static void*    custom_to_pointer(void *data_interface, unsigned node);
+static int      custom_pointer_is_inside(void *data_interface, unsigned node, void *ptr);
 static void     free_custom_buffer_on_node(void *data_interface, unsigned node);
 static size_t   custom_interface_get_size(starpu_data_handle_t handle);
 static uint32_t footprint_custom_interface_crc32(starpu_data_handle_t handle);
@@ -94,7 +94,8 @@ static struct starpu_data_interface_ops interface_custom_ops =
 {
 	.register_data_handle  = register_custom_handle,
 	.allocate_data_on_node = allocate_custom_buffer_on_node,
-	.handle_to_pointer     = custom_handle_to_pointer,
+	.to_pointer            = custom_to_pointer,
+	.pointer_is_inside     = custom_pointer_is_inside,
 	.free_data_on_node     = free_custom_buffer_on_node,
 	.copy_methods          = &custom_copy_data_methods_s,
 	.get_size              = custom_interface_get_size,
@@ -203,10 +204,9 @@ static void free_custom_buffer_on_node(void *data_interface, unsigned node)
 }
 
 static void*
-custom_handle_to_pointer(starpu_data_handle_t handle, unsigned node)
+custom_to_pointer(void *data, unsigned node)
 {
-	struct custom_data_interface *data_interface =
-		(struct custom_data_interface *) starpu_data_get_interface_on_node(handle, node);
+	struct custom_data_interface *data_interface = data;
 
 
 	switch(starpu_node_get_kind(node))
@@ -226,6 +226,31 @@ custom_handle_to_pointer(starpu_data_handle_t handle, unsigned node)
 	}
 }
 
+static int
+custom_pointer_is_inside(void *data, unsigned node, void *ptr)
+{
+	struct custom_data_interface *data_interface = data;
+
+	switch(starpu_node_get_kind(node))
+	{
+		case STARPU_CPU_RAM:
+			return (char*) ptr >= (char*) data_interface->cpu_ptr &&
+				(char*) ptr < (char*) data_interface->cpu_ptr + data_interface->nx * data_interface->ops->cpu_elemsize;
+#ifdef STARPU_USE_CUDA
+		case STARPU_CUDA_RAM:
+			return (char*) ptr >= (char*) data_interface->cuda_ptr &&
+				(char*) ptr < (char*) data_interface->cuda_ptr + data_interface->nx * data_interface->ops->cuda_elemsize;
+#endif
+#ifdef STARPU_USE_OPENCL
+		case STARPU_OPENCL_RAM:
+			return (char*) ptr >= (char*) data_interface->opencl_ptr &&
+				(char*) ptr < (char*) data_interface->opencl_ptr + data_interface->nx * data_interface->ops->opencl_elemsize;
+#endif
+		default:
+			assert(0);
+	}
+}
+
 static size_t custom_interface_get_size(starpu_data_handle_t handle)
 {
 	size_t size;

examples/filters/fmultiple_manual.c

@@ -2,7 +2,7 @@
  *
  * Copyright (C) 2017                                     CNRS
  * Copyright (C) 2017                                     Inria
- * Copyright (C) 2015                                     Université de Bordeaux
+ * Copyright (C) 2015,2018                                Université de Bordeaux
  *
  * StarPU is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
@@ -107,6 +107,14 @@ void empty(void *buffers[], void *cl_arg)
 	 * data, thus getting back all data pieces there.  */
 	(void)buffers;
 	(void)cl_arg;
+
+	/* This check is just for testsuite */
+	int node = starpu_task_get_current_data_node(0);
+	unsigned i;
+	unsigned nbuffers = STARPU_TASK_GET_NBUFFERS(starpu_task_get_current());
+	STARPU_ASSERT(node >= 0);
+	for (i = 1; i < nbuffers; i++)
+		STARPU_ASSERT(starpu_task_get_current_data_node(i) == node);
 }
 
 struct starpu_codelet cl_switch =

examples/filters/fplan_notautomatic.c

@@ -0,0 +1,227 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2018                                     CNRS
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <starpu.h>
+
+#define FPRINTF(ofile, fmt, ...) do { if (!getenv("STARPU_SSILENT")) {fprintf(ofile, fmt, ## __VA_ARGS__); }} while(0)
+
+#define NX    9
+#define PARTS 3
+
+struct starpu_codelet task_codelet;
+
+// CPU implementations
+void task_cpu(void *descr[], void *args)
+{
+	int *values = (int*)STARPU_VECTOR_GET_PTR(descr[0]);
+	int nx = STARPU_VECTOR_GET_NX(descr[0]);
+	int i, add;
+	char message[10000];
+	int cur = 0;
+
+	starpu_codelet_unpack_args(args, &add);
+
+	cur += snprintf(&message[cur], 10000-cur, "Values ");
+	for(i=0 ; i<nx ; i++)
+	{
+		values[i] += add;
+		cur += snprintf(&message[cur], 10000-cur, "%d ", values[i]);
+	}
+	FPRINTF(stderr, "%s\n", message);
+}
+
+void split_cpu(void *descr[], void *args)
+{
+	(void)descr;
+	//	starpu_data_handle_t data_handle = starpu_data_lookup((void*)STARPU_VECTOR_GET_PTR(descr[0]));
+
+	starpu_data_handle_t value_handle, sub_handles[PARTS];
+	starpu_codelet_unpack_args(args, &value_handle, &sub_handles);
+
+	FPRINTF(stderr, "Partition for handle %p into handles %p %p and %p\n", value_handle, sub_handles[0], sub_handles[1], sub_handles[2]);
+
+	starpu_data_partition_submit_sequential_consistency(value_handle, PARTS, sub_handles, 0);
+}
+
+void supertask_cpu(void *descr[], void *args)
+{
+	(void)descr;
+	//	starpu_data_handle_t data_handle = starpu_data_lookup((void*)STARPU_VECTOR_GET_PTR(descr[0]));
+	starpu_data_handle_t sub_handles[PARTS];
+	int add;
+
+	starpu_codelet_unpack_args(args, &sub_handles, &add);
+
+	FPRINTF(stderr, "Submitting tasks on %d subdata (add %d)\n", PARTS, add);
+
+	int i;
+	for(i=0 ; i<PARTS ; i++)
+	{
+		int ret = starpu_task_insert(&task_codelet,
+					     STARPU_RW, sub_handles[i],
+					     STARPU_VALUE, &add, sizeof(add),
+					     0);
+		STARPU_CHECK_RETURN_VALUE(ret, "starpu_task_insert");
+	}
+}
+
+void merge_cpu(void *descr[], void *args)
+{
+	(void)descr;
+	//	starpu_data_handle_t value_handle = starpu_data_lookup((void*)STARPU_VECTOR_GET_PTR(descr[0]));
+
+	starpu_data_handle_t value_handle, sub_handles[PARTS];
+	starpu_codelet_unpack_args(args, &value_handle, &sub_handles);
+
+	FPRINTF(stderr, "Unpartition for handle %p from handles %p %p and %p\n", value_handle, sub_handles[0], sub_handles[1], sub_handles[2]);
+
+	starpu_data_unpartition_submit_sequential_consistency(value_handle, PARTS, sub_handles, STARPU_MAIN_RAM, 0);
+}
+
+
+// Codelets
+struct starpu_codelet task_codelet =
+{
+	.cpu_funcs = {task_cpu},
+	.nbuffers = 1,
+	.modes = {STARPU_RW},
+	.name = "task_codelet"
+};
+
+struct starpu_codelet supertask_codelet =
+{
+	.cpu_funcs = {supertask_cpu},
+	.nbuffers = 1,
+	.modes = {STARPU_RW},
+	.name = "supertask_codelet"
+};
+
+struct starpu_codelet split_codelet =
+{
+	.cpu_funcs = {split_cpu},
+	.nbuffers = 1,
+	.modes = {STARPU_RW},
+	.name = "split_codelet"
+};
+
+struct starpu_codelet merge_codelet =
+{
+	.cpu_funcs = {merge_cpu},
+	.nbuffers = 1,
+	.modes = {STARPU_RW},
+	.name = "merge_codelet"
+};
+
+int main(void)
+{
+	int ret, i;
+	int values[NX];
+	int check[NX];
+	int add=1;
+	starpu_data_handle_t value_handle;
+	starpu_data_handle_t sub_handles[PARTS];
+
+	ret = starpu_init(NULL);
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_init");
+
+	if (starpu_cpu_worker_get_count() == 0)
+	{
+		FPRINTF(stderr, "We need at least 1 CPU worker.\n");
+		starpu_shutdown();
+		return 77;
+	}
+
+	struct starpu_data_filter f =
+	{
+		.filter_func = starpu_vector_filter_block,
+		.nchildren = PARTS
+	};
+
+	values[NX-1] = 2;
+	for(i=NX-2 ; i>= 0 ; i--) values[i] = values[i+1] * 2;
+	for(i=0 ; i<NX ; i++) check[i] = values[i] + (4 * add);
+
+	starpu_vector_data_register(&value_handle, STARPU_MAIN_RAM, (uintptr_t)&values[0], NX, sizeof(values[0]));
+	starpu_data_partition_plan(value_handle, &f, sub_handles);
+
+	// tell StarPU not to partition data, the application will decide itself when to do it
+	starpu_data_partition_not_automatic(value_handle);
+	for(i=0 ; i<PARTS ; i++)
+		starpu_data_partition_not_automatic(sub_handles[i]);
+
+	// insert a task on the whole data
+	ret = starpu_task_insert(&task_codelet, STARPU_RW, value_handle,
+				 STARPU_VALUE, &add, sizeof(add),
+				 STARPU_NAME, "task_1", 0);
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_task_insert");
+
+	// insert a task to split the data
+	ret = starpu_task_insert(&split_codelet, STARPU_RW, value_handle,
+				 STARPU_VALUE, &value_handle, sizeof(starpu_data_handle_t),
+				 STARPU_VALUE, sub_handles, PARTS*sizeof(starpu_data_handle_t),
+				 STARPU_NAME, "split", 0);
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_task_insert");
+
+	// insert a task that will work on the subdata
+	ret = starpu_task_insert(&supertask_codelet, STARPU_RW, value_handle,
+				 STARPU_VALUE, sub_handles, PARTS*sizeof(starpu_data_handle_t),
+				 STARPU_VALUE, &add, sizeof(add),
+				 STARPU_NAME, "supertask_1", 0);
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_task_insert");
+
+	// insert another task that will work on the subdata
+	ret = starpu_task_insert(&supertask_codelet, STARPU_RW, value_handle,
+				 STARPU_VALUE, sub_handles, PARTS*sizeof(starpu_data_handle_t),
+				 STARPU_VALUE, &add, sizeof(add),
+				 STARPU_NAME, "supertask_2", 0);
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_task_insert");
+
+	// insert a task to merge the data
+	ret = starpu_task_insert(&merge_codelet, STARPU_RW, value_handle,
+				 STARPU_VALUE, &value_handle, sizeof(starpu_data_handle_t),
+				 STARPU_VALUE, sub_handles, PARTS*sizeof(starpu_data_handle_t),
+				 STARPU_NAME, "merge", 0);
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_task_insert");
+
+	// insert a task that will work on the whole data
+	ret = starpu_task_insert(&task_codelet, STARPU_RW, value_handle,
+				 STARPU_VALUE, &add, sizeof(add),
+				 STARPU_NAME, "task_2", 0);
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_task_insert");
+
+	starpu_task_wait_for_all();
+	starpu_data_partition_clean(value_handle, PARTS, sub_handles);
+	starpu_data_unregister(value_handle);
+
+	FPRINTF(stderr, "Values : ");
+	for(i=0 ; i<NX ; i++)
+	{
+		FPRINTF(stderr, "%d ", values[i]);
+	}
+	FPRINTF(stderr, "\n");
+	for(i=0 ; i<NX ; i++)
+	{
+		if (values[i] != check[i])
+		{
+			FPRINTF(stderr, "Incorrect value for %d. %d != %d\n", i, values[i], check[i]);
+			ret = 1;
+		}
+	}
+
+	starpu_shutdown();
+
+	return ret;
+}

examples/filters/frecursive.c

@@ -31,7 +31,7 @@ void cpu_codelet(void *buffers[], void *cl_arg)
         /* local copy of the matrix pointer */
         int *val = (int *)STARPU_MATRIX_GET_PTR(buffers[0]);
 
-	FPRINTF(stderr, "computing on matrix with nx=%d, ny=%d, ld=%d\n", nx, ny, ld);
+	FPRINTF(stderr, "computing on matrix with nx=%u, ny=%u, ld=%u\n", nx, ny, ld);
         for(j=0; j<ny ; j++)
 	{
                 for(i=0; i<nx ; i++)
@@ -58,7 +58,7 @@ int main(void)
 	starpu_data_handle_t subhandle_l1[PARTS];
 	starpu_data_handle_t subhandle_l2[PARTS][PARTS];
 	starpu_data_handle_t subhandle_l3[PARTS][PARTS][PARTS];
-	int ret, submit;
+	int ret;
 
 	int factor = 12;
 	int n=1;

examples/interface/complex_interface.c

@@ -1,8 +1,8 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
  * Copyright (C) 2012-2013                                Inria
- * Copyright (C) 2012-2015,2017                           CNRS
- * Copyright (C) 2013-2015                                Université de Bordeaux
+ * Copyright (C) 2012-2015,2017,2018                      CNRS
+ * Copyright (C) 2013-2015, 2018                          Université de Bordeaux
  *
  * StarPU is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
@@ -20,6 +20,17 @@
 
 #include "complex_interface.h"
 
+static int complex_pointer_is_inside(void *data_interface, unsigned node, void *ptr)
+{
+	(void)node;
+	struct starpu_complex_interface *complex_interface = data_interface;
+
+	return ((char*) ptr >= (char*) &complex_interface->real &&
+		(char*) ptr < (char*) (&complex_interface->real + 1))
+	    || ((char*) ptr >= (char*) &complex_interface->imaginary &&
+		(char*) ptr < (char*) (&complex_interface->imaginary + 1));
+}
+
 double *starpu_complex_get_real(starpu_data_handle_t handle)
 {
 	struct starpu_complex_interface *complex_interface =
@@ -195,7 +206,8 @@ static struct starpu_data_interface_ops interface_complex_ops =
 	.footprint = complex_footprint,
 	.interfaceid = STARPU_UNKNOWN_INTERFACE_ID,
 	.interface_size = sizeof(struct starpu_complex_interface),
-	.handle_to_pointer = NULL,
+	.to_pointer = NULL,
+	.pointer_is_inside = complex_pointer_is_inside,
 	.pack_data = complex_pack_data,
 	.unpack_data = complex_unpack_data,
 	.describe = complex_describe

examples/lu/xlu_pivot.c

@@ -129,7 +129,7 @@ static int create_task_12(starpu_data_handle_t *dataAp, unsigned nblocks, unsign
 	task->cl = &cl12;
 	task->color = 0x8080ff;
 
-	task->cl_arg = (void *)(task->tag_id);
+	task->cl_arg = (void *)(uintptr_t)(task->tag_id);
 
 	/* which sub-data is manipulated ? */
 	task->handles[0] = get_block(dataAp, nblocks, k, k);
@@ -177,7 +177,7 @@ static int create_task_21(starpu_data_handle_t *dataAp, unsigned nblocks, unsign
 		task->priority = STARPU_MAX_PRIO;
 	}
 
-	task->cl_arg = (void *)(task->tag_id);
+	task->cl_arg = (void *)(uintptr_t)(task->tag_id);
 
 	/* enforce dependencies ... */
 	starpu_tag_declare_deps(TAG21(k, i), 1, PIVOT(k, i));
@@ -199,7 +199,7 @@ static int create_task_22(starpu_data_handle_t *dataAp, unsigned nblocks, unsign
 	task->cl = &cl22;
 	task->color = 0x00ff00;
 
-	task->cl_arg = (void *)(task->tag_id);
+	task->cl_arg = (void *)(uintptr_t)(task->tag_id);
 
 	/* which sub-data is manipulated ? */
 	task->handles[0] = get_block(dataAp, nblocks, k, i); /* produced by TAG21(k, i) */

examples/spmv/matrix_market/mm_to_bcsr.h

@@ -1,6 +1,6 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2008-2009,2011                           Université de Bordeaux
+ * Copyright (C) 2008-2009,2011, 2018                           Université de Bordeaux
  * Copyright (C) 2010-2011,2015,2017                      CNRS
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -51,4 +51,4 @@ typedef struct
 bcsr_t *mm_file_to_bcsr(char *filename, unsigned c, unsigned r);
 
 /* read the matrix as a set of valuated coordinates */
-bcsr_t *mm_to_bcsr(unsigned nz, unsigned *I, unsigned *J, float *val, unsigned c, unsigned r);
+bcsr_t *mm_to_bcsr(unsigned nz, unsigned *I_, unsigned *J, float *val, unsigned c, unsigned r);

examples/spmv/matrix_market/mmio.h

@@ -1,6 +1,6 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2008-2009,2011,2016                      Université de Bordeaux
+ * Copyright (C) 2008-2009,2011,2016, 2018                      Université de Bordeaux
  * Copyright (C) 2010,2015,2017                           CNRS
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -135,11 +135,11 @@ int mm_is_valid(MM_typecode matcode);		/* too complex for a macro */
 
 /*  high level routines */
 
-int mm_write_mtx_crd(char fname[], int M, int N, int nz, int I[], int J[],
+int mm_write_mtx_crd(char fname[], int M, int N, int nz, int I_[], int J[],
 		 double val[], MM_typecode matcode);
-int mm_read_mtx_crd_data(FILE *f, int M, int N, int nz, int I[], int J[],
+int mm_read_mtx_crd_data(FILE *f, int M, int N, int nz, int I_[], int J[],
 		double val[], MM_typecode matcode);
-int mm_read_mtx_crd_entry(FILE *f, int *I, int *J, double *real, double *img,
+int mm_read_mtx_crd_entry(FILE *f, int *I_, int *J, double *real, double *img,
 			MM_typecode matcode);
 
 int mm_read_unsymmetric_sparse(const char *fname, int *M_, int *N_, int *nz_,

examples/stencil/Makefile.am

@@ -1,7 +1,7 @@
 # StarPU --- Runtime system for heterogeneous multicore architectures.
 #
 # Copyright (C) 2011,2016-2017                           Inria
-# Copyright (C) 2011-2017                                CNRS
+# Copyright (C) 2011-2018                                CNRS
 # Copyright (C) 2010-2017                                Université de Bordeaux
 #
 # StarPU is free software; you can redistribute it and/or modify
@@ -185,7 +185,7 @@ CLEANFILES = *.gcno *.gcda *.xpm starpu_idle_microsec.log
 
 
 .out.out2:
-	grep '^|' $< | tr -d ' ' > $@
+	$(GREP) '^|' $< | tr -d ' ' > $@
 
 .out2.xpm:
 	( width=$$(expr $$(head -n 1 < $< | wc -c) - 1) ; \
@@ -202,7 +202,7 @@ CLEANFILES = *.gcno *.gcda *.xpm starpu_idle_microsec.log
 	echo "\"5	c #00FFFF\"," ; \
 	echo "\"|	c #FFFFFF\"," ; \
 	echo "\"*	c #000000\"," ; \
-	< $< sed -e 's/^/"/' -e 's/$$/",/' | sed -e '$$s/",$$/"};/' ) > $@
+	< $< $(SED) -e 's/^/"/' -e 's/$$/",/' | $(SED) -e '$$s/",$$/"};/' ) > $@
 
 view:
 	feh --zoom 800 -F 0.xpm 0.5.xpm 1.xpm 2.xpm 3.xpm 4.xpm 6.xpm mpi.xpm

examples/stencil/life_opencl.c

@@ -2,7 +2,7 @@
  *
  * Copyright (C) 2011-2012                                Inria
  * Copyright (C) 2012-2013,2016-2017                      CNRS
- * Copyright (C) 2010-2011,2013-2014                      Université de Bordeaux
+ * Copyright (C) 2010-2011,2013-2014,2018                 Université de Bordeaux
  *
  * StarPU is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
@@ -21,6 +21,7 @@
 /* #define _externC extern "C" */
 
 #include <stencil.h>
+#define CL_TARGET_OPENCL_VERSION 100
 #ifdef __APPLE__
 #include <OpenCL/cl.h>
 #else

include/fstarpu_mod.f90

@@ -1,8 +1,8 @@
 ! StarPU --- Runtime system for heterogeneous multicore architectures.
 !
-! Copyright (C) 2017                                     CNRS
+! Copyright (C) 2017, 2018                                     CNRS
 ! Copyright (C) 2016-2017                                Inria
-! Copyright (C) 2016-2017                                Université de Bordeaux
+! Copyright (C) 2016-2018                                Université de Bordeaux
 !
 ! StarPU is free software; you can redistribute it and/or modify
 ! it under the terms of the GNU Lesser General Public License as published by
@@ -55,6 +55,9 @@ module fstarpu_mod
         type(c_ptr), bind(C) :: FSTARPU_TAG
         type(c_ptr), bind(C) :: FSTARPU_TAG_ONLY
         type(c_ptr), bind(C) :: FSTARPU_NAME
+        type(c_ptr), bind(C) :: FSTARPU_TASK_COLOR
+        type(c_ptr), bind(C) :: FSTARPU_TASK_SYNCHRONOUS
+        type(c_ptr), bind(C) :: FSTARPU_HANDLES_SEQUENTIAL_CONSISTENCY
         type(c_ptr), bind(C) :: FSTARPU_NODE_SELECTION_POLICY
 
         type(c_ptr), bind(C) :: FSTARPU_VALUE
@@ -781,6 +784,15 @@ module fstarpu_mod
                         integer(c_int), value, intent(in) :: node
                 end function fstarpu_data_handle_to_pointer
 
+                ! void *starpu_data_pointer_is_inside(starpu_data_handle_t handle, unsigned node, void *ptr);
+                function fstarpu_data_pointer_is_inside (dh,node,ptr) bind(C,name="starpu_data_pointer_is_inside")
+                        use iso_c_binding, only: c_ptr, c_int, c_ptr
+                        integer(c_int) :: fstarpu_data_pointer_is_inside
+                        type(c_ptr), value, intent(in) :: dh
+                        integer(c_int), value, intent(in) :: node
+                        type(c_ptr), value, intent(in) :: ptr
+                end function fstarpu_data_pointer_is_inside
+
                 ! void *starpu_data_get_local_ptr(starpu_data_handle_t handle);
                 function fstarpu_data_get_local_ptr (dh) bind(C,name="starpu_data_get_local_ptr")
                         use iso_c_binding, only: c_ptr, c_int

include/starpu.h

@@ -1,7 +1,7 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
  * Copyright (C) 2012-2017                                Inria
- * Copyright (C) 2009-2014,2016-2017                      Université de Bordeaux
+ * Copyright (C) 2009-2014,2016-2018                      Université de Bordeaux
  * Copyright (C) 2010-2015,2017                           CNRS
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -94,6 +94,7 @@ struct starpu_conf
 	void (*sched_policy_init)(unsigned);
 
 	int ncpus;
+	int reserve_ncpus;
 	int ncuda;
 	int nopencl;
 	int nmic;
@@ -152,6 +153,11 @@ int starpu_conf_init(struct starpu_conf *conf);
 int starpu_init(struct starpu_conf *conf) STARPU_WARN_UNUSED_RESULT;
 int starpu_initialize(struct starpu_conf *user_conf, int *argc, char ***argv);
 int starpu_is_initialized(void);
+void starpu_wait_initialized(void);
+
+#define STARPU_THREAD_ACTIVE (1 << 0)
+unsigned starpu_get_next_bindid(unsigned flags, unsigned *preferred, unsigned npreferred);
+int starpu_bind_thread_on(int cpuid, unsigned flags, const char *name);
 
 void starpu_pause(void);
 void starpu_resume(void);

include/starpu_data_filters.h

@@ -3,7 +3,7 @@
  * Copyright (C) 2011                                     Antoine Lucas
  * Copyright (C) 2009-2012,2014-2015,2017                 Université de Bordeaux
  * Copyright (C) 2010                                     Mehdi Juhoor
- * Copyright (C) 2010-2013,2015,2017                      CNRS
+ * Copyright (C) 2010-2013,2015,2017,2018                 CNRS
  * Copyright (C) 2011                                     Inria
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -53,6 +53,10 @@ void starpu_data_unpartition_submit_r(starpu_data_handle_t initial_handle, int g
 void starpu_data_unpartition_readonly_submit(starpu_data_handle_t initial_handle, unsigned nparts, starpu_data_handle_t *children, int gathering_node);
 void starpu_data_partition_clean(starpu_data_handle_t root_data, unsigned nparts, starpu_data_handle_t *children);
 
+void starpu_data_partition_submit_sequential_consistency(starpu_data_handle_t initial_handle, unsigned nparts, starpu_data_handle_t *children, int sequential_consistency);
+void starpu_data_unpartition_submit_sequential_consistency(starpu_data_handle_t initial_handle, unsigned nparts, starpu_data_handle_t *children, int gathering_node, int sequential_consistency);
+void starpu_data_partition_not_automatic(starpu_data_handle_t handle);
+
 int starpu_data_get_nb_children(starpu_data_handle_t handle);
 starpu_data_handle_t starpu_data_get_child(starpu_data_handle_t handle, unsigned i);
 

include/starpu_data_interfaces.h

@@ -1,7 +1,7 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
  * Copyright (C) 2011-2014,2016-2017                      Inria
- * Copyright (C) 2009-2016                                Université de Bordeaux
+ * Copyright (C) 2009-2016,2018                           Université de Bordeaux
  * Copyright (C) 2010-2015,2017                           CNRS
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -94,6 +94,8 @@ struct starpu_data_copy_methods
 };
 
 int starpu_interface_copy(uintptr_t src, size_t src_offset, unsigned src_node, uintptr_t dst, size_t dst_offset, unsigned dst_node, size_t size, void *async_data);
+void starpu_interface_start_driver_copy_async(unsigned src_node, unsigned dst_node, double *start);
+void starpu_interface_end_driver_copy_async(unsigned src_node, unsigned dst_node, double start);
 uintptr_t starpu_malloc_on_node_flags(unsigned dst_node, size_t size, int flags);
 uintptr_t starpu_malloc_on_node(unsigned dst_node, size_t size);
 void starpu_free_on_node_flags(unsigned dst_node, uintptr_t addr, size_t size, int flags);
@@ -122,7 +124,9 @@ struct starpu_data_interface_ops
 	starpu_ssize_t	 (*allocate_data_on_node)	(void *data_interface, unsigned node);
 	void 		 (*free_data_on_node)		(void *data_interface, unsigned node);
 	const struct starpu_data_copy_methods *copy_methods;
-	void * 		 (*handle_to_pointer)		(starpu_data_handle_t handle, unsigned node);
+	void * 		 (*handle_to_pointer)		(starpu_data_handle_t handle, unsigned node); /* deprecated */
+	void * 		 (*to_pointer)			(void *data_interface, unsigned node);
+	int 		 (*pointer_is_inside)		(void *data_interface, unsigned node, void *ptr);
 	size_t 		 (*get_size)			(starpu_data_handle_t handle);
 	uint32_t 	 (*footprint)			(starpu_data_handle_t handle);
 	int 		 (*compare)			(void *data_interface_a, void *data_interface_b);
@@ -148,6 +152,7 @@ void starpu_data_ptr_register(starpu_data_handle_t handle, unsigned node);
 void starpu_data_register_same(starpu_data_handle_t *handledst, starpu_data_handle_t handlesrc);
 
 void *starpu_data_handle_to_pointer(starpu_data_handle_t handle, unsigned node);
+int starpu_data_pointer_is_inside(starpu_data_handle_t handle, unsigned node, void *ptr);
 void *starpu_data_get_local_ptr(starpu_data_handle_t handle);
 
 void *starpu_data_get_interface_on_node(starpu_data_handle_t handle, unsigned memory_node);

include/starpu_fxt.h

@@ -2,7 +2,7 @@
  *
  * Copyright (C) 2012-2013,2016                           Inria
  * Copyright (C) 2013                                     Joris Pablo
- * Copyright (C) 2010-2015,2017                           CNRS
+ * Copyright (C) 2010-2015,2017,2018                      CNRS
  * Copyright (C) 2010-2011,2013-2018                      Université de Bordeaux
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -49,7 +49,9 @@ struct starpu_fxt_options
 	unsigned ninputfiles;
 	unsigned no_smooth;
 	unsigned no_acquire;
+	unsigned memory_states;
 	unsigned internal;
+	unsigned label_deps;
 	char *filenames[STARPU_FXT_MAX_FILES];
 	char *out_paje_path;
 	char *distrib_time_path;

include/starpu_opencl.h

@@ -1,7 +1,7 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
  * Copyright (C) 2011-2012                                Inria
- * Copyright (C) 2010-2014                                Université de Bordeaux
+ * Copyright (C) 2010-2014,2018                           Université de Bordeaux
  * Copyright (C) 2010-2013,2015-2017                      CNRS
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -21,6 +21,9 @@
 
 #include <starpu_config.h>
 #ifdef STARPU_USE_OPENCL
+#ifndef CL_TARGET_OPENCL_VERSION
+#define CL_TARGET_OPENCL_VERSION 100
+#endif
 #ifdef __APPLE__
 #include <OpenCL/cl.h>
 #else

include/starpu_sched_component.h

@@ -63,6 +63,8 @@ struct starpu_sched_component
 	int (*can_push)(struct starpu_sched_component *from, struct starpu_sched_component *to);
 	int (*can_pull)(struct starpu_sched_component *component);
 
+	int (*notify)(struct starpu_sched_component* component, int message_ID, void* arg);
+
 	double (*estimated_load)(struct starpu_sched_component *component);
 	double (*estimated_end)(struct starpu_sched_component *component);
 
@@ -126,6 +128,7 @@ int starpu_sched_component_can_push(struct starpu_sched_component * component, s
 int starpu_sched_component_can_pull(struct starpu_sched_component * component);
 double starpu_sched_component_estimated_load(struct starpu_sched_component * component);
 double starpu_sched_component_estimated_end_min(struct starpu_sched_component * component);
+double starpu_sched_component_estimated_end_min_add(struct starpu_sched_component * component, double exp_len);
 double starpu_sched_component_estimated_end_average(struct starpu_sched_component * component);
 
 struct starpu_sched_component_fifo_data
@@ -233,6 +236,8 @@ do \
 } \
 while(0)
 
+#define STARPU_COMPONENT_MUTEX_TRYLOCK(m) STARPU_PTHREAD_MUTEX_TRYLOCK((m))
+
 #define STARPU_COMPONENT_MUTEX_UNLOCK(m) STARPU_PTHREAD_MUTEX_UNLOCK((m))
 
 #ifdef __cplusplus

include/starpu_sched_ctx.h

@@ -111,6 +111,8 @@ int starpu_sched_ctx_max_priority_is_set(unsigned sched_ctx_id);
 
 void *starpu_sched_ctx_get_user_data(unsigned sched_ctx_id);
 
+void starpu_sched_ctx_set_user_data(unsigned sched_ctx_id, void* user_data);
+
 struct starpu_worker_collection *starpu_sched_ctx_create_worker_collection(unsigned sched_ctx_id, enum starpu_worker_collection_type type) STARPU_ATTRIBUTE_MALLOC;
 
 void starpu_sched_ctx_delete_worker_collection(unsigned sched_ctx_id);

include/starpu_scheduler.h

@@ -83,11 +83,17 @@ int starpu_prefetch_task_input_on_node(struct starpu_task *task, unsigned node);
 int starpu_idle_prefetch_task_input_on_node_prio(struct starpu_task *task, unsigned node, int prio);
 int starpu_idle_prefetch_task_input_on_node(struct starpu_task *task, unsigned node);
 
+int starpu_prefetch_task_input_for_prio(struct starpu_task *task, unsigned worker, int prio);
+int starpu_prefetch_task_input_for(struct starpu_task *task, unsigned worker);
+int starpu_idle_prefetch_task_input_for_prio(struct starpu_task *task, unsigned worker, int prio);
+int starpu_idle_prefetch_task_input_for(struct starpu_task *task, unsigned worker);
+
 uint32_t starpu_task_footprint(struct starpu_perfmodel *model, struct starpu_task *task, struct starpu_perfmodel_arch *arch, unsigned nimpl);
 uint32_t starpu_task_data_footprint(struct starpu_task *task);
 double starpu_task_expected_length(struct starpu_task *task, struct starpu_perfmodel_arch *arch, unsigned nimpl);
 double starpu_worker_get_relative_speedup(struct starpu_perfmodel_arch *perf_arch);
 double starpu_task_expected_data_transfer_time(unsigned memory_node, struct starpu_task *task);
+double starpu_task_expected_data_transfer_time_for(struct starpu_task *task, unsigned worker);
 double starpu_data_expected_transfer_time(starpu_data_handle_t handle, unsigned memory_node, enum starpu_data_access_mode mode);
 double starpu_task_expected_energy(struct starpu_task *task, struct starpu_perfmodel_arch *arch, unsigned nimpl);
 double starpu_task_expected_conversion_time(struct starpu_task *task, struct starpu_perfmodel_arch *arch, unsigned nimpl);

include/starpu_task.h

@@ -2,7 +2,7 @@
  *
  * Copyright (C) 2011-2017                                Inria
  * Copyright (C) 2009-2018                                Université de Bordeaux
- * Copyright (C) 2010-2015,2017                           CNRS
+ * Copyright (C) 2010-2015,2017,2018                           CNRS
  * Copyright (C) 2011                                     Télécom-SudParis
  * Copyright (C) 2016                                     Uppsala University
  *
@@ -91,6 +91,9 @@ typedef starpu_scc_kernel_t (*starpu_scc_func_t)(void);
 
 #define STARPU_VARIABLE_NBUFFERS (-1)
 
+#define STARPU_SPECIFIC_NODE_LOCAL (-1)
+#define STARPU_SPECIFIC_NODE_CPU (-2)
+#define STARPU_SPECIFIC_NODE_SLOW (-3)
 struct starpu_task;
 struct starpu_codelet
 {
@@ -153,6 +156,8 @@ struct starpu_task
 	void *interfaces[STARPU_NMAXBUFS];
 	enum starpu_data_access_mode modes[STARPU_NMAXBUFS];
 
+	unsigned char *handles_sequential_consistency;
+
 	void *cl_arg;
 	size_t cl_arg_size;
 
@@ -225,6 +230,7 @@ struct starpu_task
 #else
 	void *omp_task;
 #endif
+	unsigned nb_termination_call_required;
 	void *sched_data;
 };
 
@@ -299,6 +305,9 @@ void starpu_tag_declare_deps_array(starpu_tag_t id, unsigned ndeps, starpu_tag_t
 
 void starpu_task_declare_deps_array(struct starpu_task *task, unsigned ndeps, struct starpu_task *task_array[]);
 
+void starpu_task_end_dep_add(struct starpu_task *t, int nb_deps);
+void starpu_task_end_dep_release(struct starpu_task *t);
+
 int starpu_task_get_task_succs(struct starpu_task *task, unsigned ndeps, struct starpu_task *task_array[]);
 int starpu_task_get_task_scheduled_succs(struct starpu_task *task, unsigned ndeps, struct starpu_task *task_array[]);
 
@@ -348,6 +357,7 @@ void starpu_codelet_init(struct starpu_codelet *cl);
 void starpu_codelet_display_stats(struct starpu_codelet *cl);
 
 struct starpu_task *starpu_task_get_current(void);
+int starpu_task_get_current_data_node(unsigned i);
 
 const char *starpu_task_get_model_name(struct starpu_task *task);
 const char *starpu_task_get_name(struct starpu_task *task);

include/starpu_task_util.h

@@ -63,8 +63,12 @@ void starpu_create_sync_task(starpu_tag_t sync_tag, unsigned ndeps, starpu_tag_t
 #define STARPU_CL_ARGS		(25<<STARPU_MODE_SHIFT)
 #define STARPU_CL_ARGS_NFREE	(26<<STARPU_MODE_SHIFT)
 #define STARPU_TASK_DEPS_ARRAY	(27<<STARPU_MODE_SHIFT)
-#define STARPU_SHIFTED_MODE_MAX (28<<STARPU_MODE_SHIFT)
+#define STARPU_TASK_COLOR       (28<<STARPU_MODE_SHIFT)
+#define STARPU_HANDLES_SEQUENTIAL_CONSISTENCY (29<<STARPU_MODE_SHIFT)
+#define STARPU_TASK_SYNCHRONOUS (30<<STARPU_MODE_SHIFT)
+#define STARPU_SHIFTED_MODE_MAX (31<<STARPU_MODE_SHIFT)
 
+int starpu_task_set(struct starpu_task *task, struct starpu_codelet *cl, ...);
 struct starpu_task *starpu_task_build(struct starpu_codelet *cl, ...);
 int starpu_task_insert(struct starpu_codelet *cl, ...);
 /* the function starpu_insert_task has the same semantics as starpu_task_insert, it is kept to avoid breaking old codes */

libstarpu-mic.pc.in

@@ -1,7 +1,7 @@
 # StarPU --- Runtime system for heterogeneous multicore architectures.
 #
 # Copyright (C) 2010-2013,2015,2017                      CNRS
-# Copyright (C) 2009-2011,2013-2015                      Université de Bordeaux
+# Copyright (C) 2009-2011,2013-2015,2018                 Université de Bordeaux
 #
 # StarPU is free software; you can redistribute it and/or modify
 # it under the terms of the GNU Lesser General Public License as published by
@@ -23,6 +23,6 @@ Name: starpu
 Description: offers support for heterogeneous multicore architecture
 Version: @PACKAGE_VERSION@
 Cflags: -I${includedir}/starpu/@STARPU_EFFECTIVE_VERSION@ @STARPU_CUDA_CPPFLAGS@ @SIMGRID_CFLAGS@ -DSTARPU_USE_DEPRECATED_API
-Libs: @STARPU_EXPORT_DYNAMIC@ -L${libdir} -lstarpu-@STARPU_EFFECTIVE_VERSION@ @STARPU_CUDA_LDFLAGS@ @STARPU_OPENCL_LDFLAGS@ @STARPU_SC_HYPERVISOR@ @STARPU_EXPORTED_LIBS@
+Libs: @STARPU_EXPORT_DYNAMIC@ -L${libdir} -lstarpu-@STARPU_EFFECTIVE_VERSION@ @STARPU_SC_HYPERVISOR@ @STARPU_EXPORTED_LIBS@
 Libs.private: @LDFLAGS@ @LIBS@ @LIBSTARPU_LDFLAGS@
 Requires: @HWLOC_REQUIRES@

libstarpu.pc.in

@@ -1,7 +1,7 @@
 # StarPU --- Runtime system for heterogeneous multicore architectures.
 #
 # Copyright (C) 2011-2012                                Inria
-# Copyright (C) 2009-2015                                Université de Bordeaux
+# Copyright (C) 2009-2015,2018                           Université de Bordeaux
 # Copyright (C) 2010-2013,2015,2017                      CNRS
 #
 # StarPU is free software; you can redistribute it and/or modify
@@ -24,6 +24,6 @@ Name: starpu
 Description: offers support for heterogeneous multicore architecture
 Version: @PACKAGE_VERSION@
 Cflags: -I${includedir}/starpu/@STARPU_EFFECTIVE_VERSION@ @STARPU_CUDA_CPPFLAGS@ @SIMGRID_CFLAGS@ -DSTARPU_USE_DEPRECATED_API -DSTARPU_USE_DEPRECATED_ONE_ZERO_API
-Libs: @STARPU_EXPORT_DYNAMIC@ -L${libdir} -lstarpu-@STARPU_EFFECTIVE_VERSION@ @STARPU_CUDA_LDFLAGS@ @STARPU_OPENCL_LDFLAGS@ @STARPU_SC_HYPERVISOR@ @STARPU_EXPORTED_LIBS@
+Libs: @STARPU_EXPORT_DYNAMIC@ -L${libdir} -lstarpu-@STARPU_EFFECTIVE_VERSION@ @STARPU_SC_HYPERVISOR@ @STARPU_EXPORTED_LIBS@
 Libs.private: @LDFLAGS@ @LIBS@ @LIBSTARPU_LDFLAGS@
 Requires: @HWLOC_REQUIRES@

mpi/examples/comm/comm.c

@@ -1,7 +1,7 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
  * Copyright (C) 2015-2017                                CNRS
- * Copyright (C) 2015,2017                                Université de Bordeaux
+ * Copyright (C) 2015,2017-2018                                Université de Bordeaux
  *
  * StarPU is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
@@ -90,10 +90,8 @@ int main(int argc, char **argv)
 		FPRINTF(stderr, "[%d][%d] received %d\n", rank, newrank, x);
 	}
 
-        ret = starpu_init(NULL);
-        STARPU_CHECK_RETURN_VALUE(ret, "starpu_init");
-        ret = starpu_mpi_init_comm(NULL, NULL, 0, newcomm);
-        STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init");
+        ret = starpu_mpi_init_conf(NULL, NULL, 0, newcomm, NULL);
+        STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init_conf");
 
 	if (newrank == 0)
 	{
@@ -148,7 +146,6 @@ int main(int argc, char **argv)
 	}
 
 	starpu_mpi_shutdown();
-	starpu_shutdown();
 	MPI_Comm_free(&newcomm);
         MPI_Finalize();
 	return 0;

mpi/examples/comm/mix_comm.c

@@ -1,7 +1,7 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
  * Copyright (C) 2015,2017                                CNRS
- * Copyright (C) 2015,2017                                Université de Bordeaux
+ * Copyright (C) 2015,2017-2018                                Université de Bordeaux
  *
  * StarPU is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
@@ -87,10 +87,8 @@ int main(int argc, char **argv)
 		FPRINTF(stderr, "[%d][%d] received %d\n", rank, newrank, x);
 	}
 
-        ret = starpu_init(NULL);
-        STARPU_CHECK_RETURN_VALUE(ret, "starpu_init");
-        ret = starpu_mpi_init(NULL, NULL, 0);
-        STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init");
+        ret = starpu_mpi_init_conf(NULL, NULL, 0, MPI_COMM_WORLD, NULL);
+        STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init_conf");
 
 	if (rank == 0)
 	{
@@ -180,7 +178,6 @@ int main(int argc, char **argv)
 	starpu_data_unregister(data[2]);
 
 	starpu_mpi_shutdown();
-	starpu_shutdown();
 	MPI_Comm_free(&newcomm);
         MPI_Finalize();
 	return 0;

mpi/examples/complex/mpi_complex.c

@@ -2,7 +2,7 @@
  *
  * Copyright (C) 2012-2013,2015-2017                      CNRS
  * Copyright (C) 2013                                     Inria
- * Copyright (C) 2013-2015,2017                           Université de Bordeaux
+ * Copyright (C) 2013-2015,2017-2018                           Université de Bordeaux
  *
  * StarPU is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
@@ -43,10 +43,8 @@ int main(int argc, char **argv)
 	int ret;
 	int compare=0;
 
-	ret = starpu_init(NULL);
-	STARPU_CHECK_RETURN_VALUE(ret, "starpu_init");
-	ret = starpu_mpi_init(&argc, &argv, 1);
-	STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init");
+	ret = starpu_mpi_init_conf(&argc, &argv, 1, MPI_COMM_WORLD, NULL);
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init_conf");
 	starpu_mpi_comm_rank(MPI_COMM_WORLD, &rank);
 	starpu_mpi_comm_size(MPI_COMM_WORLD, &nodes);
 
@@ -60,7 +58,6 @@ int main(int argc, char **argv)
 				fprintf(stderr, "We need at least 1 CPU.\n");
 		}
 		starpu_mpi_shutdown();
-		starpu_shutdown();
 		return 77;
 	}
 
@@ -108,7 +105,6 @@ int main(int argc, char **argv)
 	starpu_data_unregister(handle2);
 
 	starpu_mpi_shutdown();
-	starpu_shutdown();
 
 	return (rank == 0) ? !compare : 0;
 }

mpi/examples/matrix_decomposition/mpi_cholesky.c

@@ -1,7 +1,7 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
  * Copyright (C) 2012                                     Inria
- * Copyright (C) 2009-2012,2014-2015                      Université de Bordeaux
+ * Copyright (C) 2009-2012,2014-2015, 2018                      Université de Bordeaux
  * Copyright (C) 2010-2013,2015-2017                      CNRS
  * Copyright (C) 2010                                     Mehdi Juhoor
  *
@@ -34,11 +34,8 @@ int main(int argc, char **argv)
 	int correctness;
 #endif
 
-	ret = starpu_init(NULL);
-	STARPU_CHECK_RETURN_VALUE(ret, "starpu_init");
-
-	ret = starpu_mpi_init(&argc, &argv, 1);
-	STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init");
+	ret = starpu_mpi_init_conf(&argc, &argv, 1, MPI_COMM_WORLD, NULL);
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init_conf");
 	starpu_mpi_comm_rank(MPI_COMM_WORLD, &rank);
 	starpu_mpi_comm_size(MPI_COMM_WORLD, &nodes);
 	starpu_cublas_init();
@@ -50,7 +47,6 @@ int main(int argc, char **argv)
 			FPRINTF(stderr, "We need at least 1 CPU or CUDA worker.\n");
 		}
 		starpu_mpi_shutdown();
-		starpu_shutdown();
 		return STARPU_TEST_SKIPPED;
 	}
 
@@ -61,6 +57,7 @@ int main(int argc, char **argv)
 
 	dw_cholesky(bmat, size/nblocks, rank, nodes, &timing, &flops);
 
+	starpu_cublas_shutdown();
 	starpu_mpi_shutdown();
 
 #ifndef STARPU_SIMGRID
@@ -70,8 +67,6 @@ int main(int argc, char **argv)
 #endif
 
 	matrix_free(&bmat, rank, nodes, 1);
-	starpu_cublas_shutdown();
-	starpu_shutdown();
 
 #ifndef STARPU_SIMGRID
 	assert(correctness);

mpi/examples/matrix_decomposition/mpi_cholesky_codelets.c

@@ -2,7 +2,7 @@
  *
  * Copyright (C) 2010-2015,2017,2018                           CNRS
  * Copyright (C) 2013                                     Inria
- * Copyright (C) 2009-2010,2014-2015,2017                 Université de Bordeaux
+ * Copyright (C) 2009-2010,2014-2015,2017-2018                 Université de Bordeaux
  *
  * StarPU is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
@@ -35,7 +35,8 @@ static struct starpu_codelet cl11 =
 #endif
 	.nbuffers = 1,
 	.modes = {STARPU_RW},
-	.model = &chol_model_11
+	.model = &chol_model_11,
+	.color = 0xffff00,
 };
 
 static struct starpu_codelet cl21 =
@@ -46,9 +47,11 @@ static struct starpu_codelet cl21 =
 #elif defined(STARPU_SIMGRID)
 	.cuda_funcs = {(void*)1},
 #endif
+	.cuda_flags = {STARPU_CUDA_ASYNC},
 	.nbuffers = 2,
 	.modes = {STARPU_R, STARPU_RW},
-	.model = &chol_model_21
+	.model = &chol_model_21,
+	.color = 0x8080ff,
 };
 
 static struct starpu_codelet cl22 =
@@ -59,9 +62,11 @@ static struct starpu_codelet cl22 =
 #elif defined(STARPU_SIMGRID)
 	.cuda_funcs = {(void*)1},
 #endif
+	.cuda_flags = {STARPU_CUDA_ASYNC},
 	.nbuffers = 3,
 	.modes = {STARPU_R, STARPU_R, STARPU_RW | STARPU_COMMUTE},
-	.model = &chol_model_22
+	.model = &chol_model_22,
+	.color = 0x00ff00,
 };
 
 /*
@@ -157,6 +162,9 @@ void dw_cholesky(float ***matA, unsigned ld, int rank, int nodes, double *timing
 
 	starpu_task_wait_for_all();
 
+	starpu_mpi_barrier(MPI_COMM_WORLD);
+	end = starpu_timing_now();
+
 	for(x = 0; x < nblocks ; x++)
 	{
 		for (y = 0; y < nblocks; y++)
@@ -168,9 +176,6 @@ void dw_cholesky(float ***matA, unsigned ld, int rank, int nodes, double *timing
 	}
 	free(data_handles);
 
-	starpu_mpi_barrier(MPI_COMM_WORLD);
-	end = starpu_timing_now();
-
 	if (rank == 0)
 	{
 		*timing = end - start;

mpi/examples/matrix_decomposition/mpi_cholesky_distributed.c

@@ -1,7 +1,7 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
  * Copyright (C) 2012                                     Inria
- * Copyright (C) 2009-2011,2014-2015,2017                 Université de Bordeaux
+ * Copyright (C) 2009-2011,2014-2015,2017-2018                 Université de Bordeaux
  * Copyright (C) 2010-2013,2015,2017                      CNRS
  * Copyright (C) 2010                                     Mehdi Juhoor
  *
@@ -34,11 +34,8 @@ int main(int argc, char **argv)
 	int rank, nodes, ret;
 	double timing, flops;
 
-	ret = starpu_init(NULL);
-	STARPU_CHECK_RETURN_VALUE(ret, "starpu_init");
-
-	ret = starpu_mpi_init(&argc, &argv, 1);
-	STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init");
+	ret = starpu_mpi_init_conf(&argc, &argv, 1, MPI_COMM_WORLD, NULL);
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init_conf");
 	starpu_mpi_comm_rank(MPI_COMM_WORLD, &rank);
 	starpu_mpi_comm_size(MPI_COMM_WORLD, &nodes);
 	starpu_cublas_init();
@@ -49,11 +46,10 @@ int main(int argc, char **argv)
 
 	dw_cholesky(bmat, size/nblocks, rank, nodes, &timing, &flops);
 
+	starpu_cublas_shutdown();
 	starpu_mpi_shutdown();
 
 	matrix_free(&bmat, rank, nodes, 0);
-	starpu_cublas_shutdown();
-	starpu_shutdown();
 
 	if (rank == 0)
 	{

mpi/examples/matrix_decomposition/mpi_cholesky_kernels.c

@@ -1,7 +1,7 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
  * Copyright (C) 2012                                     Inria
- * Copyright (C) 2009-2014                                Université de Bordeaux
+ * Copyright (C) 2009-2014, 2018                                Université de Bordeaux
  * Copyright (C) 2010-2013,2015,2017                      CNRS
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -23,6 +23,7 @@
 #include <cuda.h>
 #include <cuda_runtime.h>
 #include <cublas.h>
+#include <starpu_cublas_v2.h>
 #ifdef STARPU_HAVE_MAGMA
 #include "magma.h"
 #include "magma_lapack.h"
@@ -33,6 +34,11 @@
  * U22
  */
 
+#if defined(STARPU_USE_CUDA)
+static const float p1 =  1.0;
+static const float m1 = -1.0;
+#endif
+
 static inline void chol_common_cpu_codelet_update_u22(void *descr[], int s, void *_args)
 {
 	(void)_args;
@@ -49,10 +55,6 @@ static inline void chol_common_cpu_codelet_update_u22(void *descr[], int s, void
 	unsigned ld12 = STARPU_MATRIX_GET_LD(descr[1]);
 	unsigned ld22 = STARPU_MATRIX_GET_LD(descr[2]);
 
-#ifdef STARPU_USE_CUDA
-	cublasStatus st;
-#endif
-
 	switch (s)
 	{
 		case 0:
@@ -61,19 +63,16 @@ static inline void chol_common_cpu_codelet_update_u22(void *descr[], int s, void
 			break;
 #ifdef STARPU_USE_CUDA
 		case 1:
-#ifdef STARPU_HAVE_MAGMA
-			cublasSetKernelStream(starpu_cuda_get_local_stream());
-#endif
-			cublasSgemm('n', 't', dy, dx, dz,
-					-1.0f, left, ld21, right, ld12,
-					 1.0f, center, ld22);
-			st = cublasGetError();
-			if (STARPU_UNLIKELY(st != CUBLAS_STATUS_SUCCESS))
-				STARPU_CUBLAS_REPORT_ERROR(st);
-
-			cudaStreamSynchronize(starpu_cuda_get_local_stream());
+		{
+			cublasStatus_t status = cublasSgemm(starpu_cublas_get_local_handle(),
+					CUBLAS_OP_N, CUBLAS_OP_T, dy, dx, dz,
+					&m1, left, ld21, right, ld12,
+					&p1, center, ld22);
+			if (status != CUBLAS_STATUS_SUCCESS)
+				STARPU_CUBLAS_REPORT_ERROR(status);
 
 			break;
+		}
 #endif
 		default:
 			STARPU_ABORT();
@@ -113,6 +112,10 @@ static inline void chol_common_codelet_update_u21(void *descr[], int s, void *_a
 	unsigned nx21 = STARPU_MATRIX_GET_NY(descr[1]);
 	unsigned ny21 = STARPU_MATRIX_GET_NX(descr[1]);
 
+#ifdef STARPU_USE_CUDA
+	cublasStatus status;
+#endif
+
 	switch (s)
 	{
 		case 0:
@@ -120,11 +123,11 @@ static inline void chol_common_codelet_update_u21(void *descr[], int s, void *_a
 			break;
 #ifdef STARPU_USE_CUDA
 		case 1:
-#ifdef STARPU_HAVE_MAGMA
-			cublasSetKernelStream(starpu_cuda_get_local_stream());
-#endif
-			cublasStrsm('R', 'L', 'T', 'N', nx21, ny21, 1.0f, sub11, ld11, sub21, ld21);
-			cudaStreamSynchronize(starpu_cuda_get_local_stream());
+			status = cublasStrsm(starpu_cublas_get_local_handle(),
+					CUBLAS_SIDE_RIGHT, CUBLAS_FILL_MODE_LOWER, CUBLAS_OP_T, CUBLAS_DIAG_NON_UNIT,
+					nx21, ny21, &p1, sub11, ld11, sub21, ld21);
+			if (status != CUBLAS_STATUS_SUCCESS)
+				STARPU_CUBLAS_REPORT_ERROR(status);
 			break;
 #endif
 		default:
@@ -197,36 +200,65 @@ static inline void chol_common_codelet_update_u11(void *descr[], int s, void *_a
 			{
 				int ret;
 				int info;
+			cudaStream_t stream = starpu_cuda_get_local_stream();
+#if (MAGMA_VERSION_MAJOR > 1) || (MAGMA_VERSION_MAJOR == 1 && MAGMA_VERSION_MINOR >= 4)
+			cublasSetKernelStream(stream);
+			magmablasSetKernelStream(stream);
+#else
+			starpu_cublas_set_stream();
+#endif
 				ret = magma_spotrf_gpu(MagmaLower, nx, sub11, ld, &info);
 				if (ret != MAGMA_SUCCESS)
 				{
 					fprintf(stderr, "Error in Magma: %d\n", ret);
 					STARPU_ABORT();
 				}
+#if (MAGMA_VERSION_MAJOR > 1) || (MAGMA_VERSION_MAJOR == 1 && MAGMA_VERSION_MINOR >= 4)
+			cudaError_t cures = cudaStreamSynchronize(stream);
+#else
 				cudaError_t cures = cudaThreadSynchronize();
+#endif
 				STARPU_ASSERT(!cures);
 			}
 #else
+			{
+
+			float *lambda11;
+			cublasStatus_t status;
+			cudaStream_t stream = starpu_cuda_get_local_stream();
+			cublasHandle_t handle = starpu_cublas_get_local_handle();
+			cudaHostAlloc((void **)&lambda11, sizeof(float), 0);
+
 			for (z = 0; z < nx; z++)
 			{
-				float lambda11;
-				cudaMemcpyAsync(&lambda11, &sub11[z+z*ld], sizeof(float), cudaMemcpyDeviceToHost, starpu_cuda_get_local_stream());
-				cudaStreamSynchronize(starpu_cuda_get_local_stream());
+				cudaMemcpyAsync(lambda11, &sub11[z+z*ld], sizeof(float), cudaMemcpyDeviceToHost, stream);
+				cudaStreamSynchronize(stream);
 
-				STARPU_ASSERT(lambda11 != 0.0f);
+				STARPU_ASSERT(*lambda11 != 0.0f);
 
-				lambda11 = sqrt(lambda11);
+				*lambda11 = sqrt(*lambda11);
 
-				cublasSetVector(1, sizeof(float), &lambda11, sizeof(float), &sub11[z+z*ld], sizeof(float));
+/*				cublasSetVector(1, sizeof(float), lambda11, sizeof(float), &sub11[z+z*ld], sizeof(float)); */
+				cudaMemcpyAsync(&sub11[z+z*ld], lambda11, sizeof(float), cudaMemcpyHostToDevice, stream);
+				float scal = 1.0f/(*lambda11);
 
-				cublasSscal(nx - z - 1, 1.0f/lambda11, &sub11[(z+1)+z*ld], 1);
+				status = cublasSscal(handle,
+						     nx - z - 1, &scal, &sub11[(z+1)+z*ld], 1);
+				if (status != CUBLAS_STATUS_SUCCESS)
+					STARPU_CUBLAS_REPORT_ERROR(status);
 
-				cublasSsyr('U', nx - z - 1, -1.0f,
+				status = cublasSsyr(handle,
+						    CUBLAS_FILL_MODE_UPPER,
+						    nx - z - 1, &m1,
 							&sub11[(z+1)+z*ld], 1,
 							&sub11[(z+1)+(z+1)*ld], ld);
+				if (status != CUBLAS_STATUS_SUCCESS)
+					STARPU_CUBLAS_REPORT_ERROR(status);
 			}
 
-			cudaStreamSynchronize(starpu_cuda_get_local_stream());
+			cudaStreamSynchronize(stream);
+			cudaFreeHost(lambda11);
+			}
 #endif
 			break;
 #endif

mpi/examples/matrix_mult/mm.c

@@ -2,7 +2,7 @@
  *
  * Copyright (C) 2016-2017                                CNRS
  * Copyright (C) 2016                                     Inria
- * Copyright (C) 2016-2017                                Université de Bordeaux
+ * Copyright (C) 2016-2018                                Université de Bordeaux
  *
  * StarPU is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
@@ -296,19 +296,14 @@ static struct starpu_codelet gemm_cl =
 
 int main(int argc, char *argv[])
 {
-	/* Initializes the StarPU core */
-	int ret = starpu_init(NULL);
-	STARPU_CHECK_RETURN_VALUE(ret, "starpu_init");
-
-	/* Initializes the StarPU-MPI layer */
-	ret = starpu_mpi_init(&argc, &argv, 1);
-	STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init");
+	/* Initializes STarPU and the StarPU-MPI layer */
+	int ret = starpu_mpi_init_conf(&argc, &argv, 1, MPI_COMM_WORLD, NULL);
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_ini_conft");
 
 	if (starpu_cpu_worker_get_count() == 0)
 	{
 		FPRINTF(stderr, "We need at least 1 CPU worker.\n");
 		starpu_mpi_shutdown();
-		starpu_shutdown();
 		return STARPU_TEST_SKIPPED;
 	}
 
@@ -387,7 +382,6 @@ int main(int argc, char *argv[])
 	}
 
 	starpu_mpi_shutdown();
-	starpu_shutdown();
 	return 0;
 }
 

mpi/examples/mpi_lu/plu_example.c

@@ -1,7 +1,7 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
  * Copyright (C) 2012-2013                                Inria
- * Copyright (C) 2010-2011,2013-2015,2017                 Université de Bordeaux
+ * Copyright (C) 2010-2011,2013-2015,2017-2018                 Université de Bordeaux
  * Copyright (C) 2010-2013,2015-2018                      CNRS
  * Copyright (C) 2013                                     Thibaut Lambert
  *
@@ -425,6 +425,7 @@ int main(int argc, char **argv)
 {
 	int rank;
 	int world_size;
+	int ret;
 
 	/*
 	 *	Initialization
@@ -447,15 +448,12 @@ int main(int argc, char **argv)
 
 	parse_args(rank, argc, argv);
 
-	int ret = starpu_init(NULL);
-	STARPU_CHECK_RETURN_VALUE(ret, "starpu_init");
+	ret = starpu_mpi_init_conf(NULL, NULL, 0, MPI_COMM_WORLD, NULL);
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init_conf");
 
 	/* We disable sequential consistency in this example */
 	starpu_data_set_default_sequential_consistency_flag(0);
 
-	ret = starpu_mpi_init(NULL, NULL, 0);
-	STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init");
-
 	STARPU_ASSERT(p*q == world_size);
 
 	starpu_cublas_init();
@@ -594,7 +592,6 @@ int main(int argc, char **argv)
 
 	starpu_cublas_shutdown();
 	starpu_mpi_shutdown();
-	starpu_shutdown();
 
 #if 0
 	MPI_Finalize();

mpi/examples/mpi_lu/plu_implicit_example.c

@@ -1,7 +1,7 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
  * Copyright (C) 2012-2013                                Inria
- * Copyright (C) 2010-2011,2013-2015,2017                 Université de Bordeaux
+ * Copyright (C) 2010-2011,2013-2015,2017-2018                 Université de Bordeaux
  * Copyright (C) 2010-2018                                CNRS
  * Copyright (C) 2013                                     Thibaut Lambert
  *
@@ -240,16 +240,14 @@ int main(int argc, char **argv)
 {
 	int rank;
 	int world_size;
+	int ret;
 
 	starpu_srand48((long int)time(NULL));
 
 	parse_args(argc, argv);
 
-	int ret = starpu_init(NULL);
-	STARPU_CHECK_RETURN_VALUE(ret, "starpu_init");
-
-	ret = starpu_mpi_init(&argc, &argv, 1);
-	STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init");
+	ret = starpu_mpi_init_conf(&argc, &argv, 1, MPI_COMM_WORLD, NULL);
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init_conf");
 
 	starpu_mpi_comm_rank(MPI_COMM_WORLD, &rank);
 	starpu_mpi_comm_size(MPI_COMM_WORLD, &world_size);
@@ -366,7 +364,6 @@ int main(int argc, char **argv)
 
 	starpu_cublas_shutdown();
 	starpu_mpi_shutdown();
-	starpu_shutdown();
 
 	return 0;
 }

mpi/examples/mpi_lu/plu_outofcore_example.c

@@ -1,7 +1,7 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
  * Copyright (C) 2012-2014                                Inria
- * Copyright (C) 2010-2011,2013-2015,2017                 Université de Bordeaux
+ * Copyright (C) 2010-2011,2013-2015,2017-2018                 Université de Bordeaux
  * Copyright (C) 2010-2018                                CNRS
  * Copyright (C) 2013                                     Thibaut Lambert
  *
@@ -272,11 +272,8 @@ int main(int argc, char **argv)
 		exit(1);
 	}
 
-	ret = starpu_init(NULL);
-	STARPU_CHECK_RETURN_VALUE(ret, "starpu_init");
-
-	ret = starpu_mpi_init(&argc, &argv, 1);
-	STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init");
+	ret = starpu_mpi_init_conf(&argc, &argv, 1, MPI_COMM_WORLD, NULL);
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init_conf");
 
 	starpu_mpi_comm_rank(MPI_COMM_WORLD, &rank);
 	starpu_mpi_comm_size(MPI_COMM_WORLD, &world_size);
@@ -401,7 +398,6 @@ int main(int argc, char **argv)
 
 	starpu_cublas_shutdown();
 	starpu_mpi_shutdown();
-	starpu_shutdown();
 
 	return 0;
 }

mpi/examples/stencil/stencil5.c

@@ -2,7 +2,7 @@
  *
  * Copyright (C) 2012-2013                                Inria
  * Copyright (C) 2011-2018                                CNRS
- * Copyright (C) 2011-2017                                Université de Bordeaux
+ * Copyright (C) 2011-2018                                Université de Bordeaux
  *
  * StarPU is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
@@ -100,11 +100,10 @@ int main(int argc, char **argv)
 	float mean=0;
 	float matrix[X][Y];
 	starpu_data_handle_t data_handles[X][Y];
+	int ret;
 
-	int ret = starpu_init(NULL);
-	STARPU_CHECK_RETURN_VALUE(ret, "starpu_init");
-	ret = starpu_mpi_init(&argc, &argv, 1);
-	STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init");
+	ret = starpu_mpi_init_conf(&argc, &argv, 1, MPI_COMM_WORLD, NULL);
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init_conf");
 	starpu_mpi_comm_rank(MPI_COMM_WORLD, &my_rank);
 	starpu_mpi_comm_size(MPI_COMM_WORLD, &size);
 
@@ -112,7 +111,6 @@ int main(int argc, char **argv)
 	{
 		FPRINTF(stderr, "We need at least 1 CPU worker.\n");
 		starpu_mpi_shutdown();
-		starpu_shutdown();
 		return 77;
 	}
 
@@ -254,7 +252,6 @@ int main(int argc, char **argv)
 	}
 
 	starpu_mpi_shutdown();
-	starpu_shutdown();
 
 	if (display)
 	{

mpi/examples/stencil/stencil5_lb.c

@@ -1,7 +1,7 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
  * Copyright (C) 2011-2018                                CNRS
- * Copyright (C) 2011,2013,2015-2017                      Université de Bordeaux
+ * Copyright (C) 2011,2013,2015-2018                      Université de Bordeaux
  *
  * StarPU is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
@@ -147,14 +147,13 @@ int main(int argc, char **argv)
 	float mean=0;
 	float matrix[X][Y];
 	struct starpu_mpi_lb_conf itf;
+	int ret;
 
 	itf.get_neighbors = get_neighbors;
 	itf.get_data_unit_to_migrate = get_data_unit_to_migrate;
 
-	int ret = starpu_init(NULL);
-	STARPU_CHECK_RETURN_VALUE(ret, "starpu_init");
-	ret = starpu_mpi_init(&argc, &argv, 1);
-	STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init");
+	ret = starpu_mpi_init_conf(&argc, &argv, 1, MPI_COMM_WORLD, NULL);
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init_conf");
 	starpu_mpi_comm_rank(MPI_COMM_WORLD, &my_rank);
 	starpu_mpi_comm_size(MPI_COMM_WORLD, &size);
 
@@ -162,14 +161,12 @@ int main(int argc, char **argv)
 	{
 		FPRINTF(stderr, "Only works with 2 nodes\n");
 		starpu_mpi_shutdown();
-		starpu_shutdown();
 		return 77;
 	}
 	if (starpu_cpu_worker_get_count() == 0)
 	{
 		FPRINTF(stderr, "We need at least 1 CPU worker.\n");
 		starpu_mpi_shutdown();
-		starpu_shutdown();
 		return 77;
 	}
 
@@ -276,7 +273,6 @@ int main(int argc, char **argv)
 	}
 
 	starpu_mpi_shutdown();
-	starpu_shutdown();
 
 	if (display)
 	{

mpi/examples/user_datatype/my_interface.c

@@ -167,11 +167,10 @@ static starpu_ssize_t data_describe(void *data_interface, char *buf, size_t size
 	return snprintf(buf, size, "Data%d-%c", my_interface->d, my_interface->c);
 }
 
-static void *data_handle_to_pointer(starpu_data_handle_t handle, unsigned node)
+static void *data_to_pointer(void *data_interface, unsigned node)
 {
-	STARPU_ASSERT(starpu_data_test_if_allocated_on_node(handle, node));
-
-	struct starpu_my_interface *my_interface = (struct starpu_my_interface *) starpu_data_get_interface_on_node(handle, node);
+	(void) node;
+	struct starpu_my_interface *my_interface = data_interface;
 
 	return (void*) &my_interface->d;
 }
@@ -211,7 +210,7 @@ static struct starpu_data_interface_ops interface_data_ops =
 	.footprint = data_footprint,
 	.interfaceid = STARPU_UNKNOWN_INTERFACE_ID,
 	.interface_size = sizeof(struct starpu_my_interface),
-	.handle_to_pointer = data_handle_to_pointer,
+	.to_pointer = data_to_pointer,
 	.pack_data = data_pack_data,
 	.unpack_data = data_unpack_data,
 	.describe = data_describe

mpi/examples/user_datatype/user_datatype.c

@@ -31,10 +31,8 @@ int main(int argc, char **argv)
 	starpu_data_handle_t handle0;
 	starpu_data_handle_t handle1;
 
-	ret = starpu_init(NULL);
-	STARPU_CHECK_RETURN_VALUE(ret, "starpu_init");
-	ret = starpu_mpi_init(&argc, &argv, 1);
-	STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init");
+	ret = starpu_mpi_init_conf(&argc, &argv, 1, MPI_COMM_WORLD, NULL);
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init_conf");
 	starpu_mpi_comm_rank(MPI_COMM_WORLD, &rank);
 	starpu_mpi_comm_size(MPI_COMM_WORLD, &nodes);
 
@@ -48,7 +46,6 @@ int main(int argc, char **argv)
 				fprintf(stderr, "We need at least 1 CPU.\n");
 		}
 		starpu_mpi_shutdown();
-		starpu_shutdown();
 		return 77;
 	}
 
@@ -107,7 +104,6 @@ int main(int argc, char **argv)
 	starpu_data_unregister(handle1);
 
 	starpu_mpi_shutdown();
-	starpu_shutdown();
 
 	if (rank == 0)
 	{

mpi/include/starpu_mpi.h

@@ -1,6 +1,6 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2009-2012,2014-2017                      Université de Bordeaux
+ * Copyright (C) 2009-2012,2014-2018                      Université de Bordeaux
  * Copyright (C) 2010-2018                                CNRS
  * Copyright (C) 2016                                     Inria
  *
@@ -54,6 +54,7 @@ int starpu_mpi_barrier(MPI_Comm comm);
 
 int starpu_mpi_irecv_detached_sequential_consistency(starpu_data_handle_t data_handle, int source, starpu_mpi_tag_t data_tag, MPI_Comm comm, void (*callback)(void *), void *arg, int sequential_consistency);
 
+int starpu_mpi_init_conf(int *argc, char ***argv, int initialize_mpi, MPI_Comm comm, struct starpu_conf *conf);
 int starpu_mpi_init_comm(int *argc, char ***argv, int initialize_mpi, MPI_Comm comm);
 int starpu_mpi_init(int *argc, char ***argv, int initialize_mpi);
 int starpu_mpi_initialize(void) STARPU_DEPRECATED;

mpi/src/Makefile.am

@@ -1,7 +1,7 @@
 # StarPU --- Runtime system for heterogeneous multicore architectures.
 #
 # Copyright (C) 2012                                     Inria
-# Copyright (C) 2010-2017                                CNRS
+# Copyright (C) 2010-2018                                CNRS
 # Copyright (C) 2009-2014, 2018                                Université de Bordeaux
 #
 # StarPU is free software; you can redistribute it and/or modify
@@ -72,6 +72,7 @@ noinst_HEADERS =					\
 	mpi/starpu_mpi_sync_data.h			\
 	mpi/starpu_mpi_comm.h				\
 	mpi/starpu_mpi_tag.h				\
+	mpi/starpu_mpi_driver.h				\
 	load_balancer/policy/data_movements_interface.h	\
 	load_balancer/policy/load_data_interface.h	\
 	load_balancer/policy/load_balancer_policy.h

mpi/src/load_balancer/policy/data_movements_interface.c

@@ -260,7 +260,7 @@ static struct starpu_data_interface_ops interface_data_movements_ops =
 	.footprint = data_movements_footprint,
 	.interfaceid = STARPU_UNKNOWN_INTERFACE_ID,
 	.interface_size = sizeof(struct data_movements_interface),
-	.handle_to_pointer = NULL,
+	.to_pointer = NULL,
 	.pack_data = data_movements_pack_data,
 	.unpack_data = data_movements_unpack_data,
 	.describe = NULL

mpi/src/load_balancer/policy/load_data_interface.c

@@ -243,7 +243,7 @@ static struct starpu_data_interface_ops interface_load_data_ops =
 	.footprint = load_data_footprint,
 	.interfaceid = STARPU_UNKNOWN_INTERFACE_ID,
 	.interface_size = sizeof(struct load_data_interface),
-	.handle_to_pointer = NULL,
+	.to_pointer = NULL,
 	.pack_data = load_data_pack_data,
 	.unpack_data = load_data_unpack_data,
 	.describe = NULL

mpi/src/mpi/starpu_mpi_driver.h

@@ -0,0 +1,37 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2015-2018                                CNRS
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#ifndef __STARPU_MPI_DRIVER_H__
+#define __STARPU_MPI_DRIVER_H__
+
+#include <starpu.h>
+
+#ifdef STARPU_USE_MPI_MPI
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+void _starpu_mpi_driver_init(struct starpu_conf *conf);
+void _starpu_mpi_driver_shutdown();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // STARPU_USE_MPI_MPI
+#endif // __STARPU_MPI_DRIVER_H__

mpi/src/mpi/starpu_mpi_mpi.c

@@ -3,7 +3,7 @@
  * Copyright (C) 2012-2013,2016-2017                      Inria
  * Copyright (C) 2009-2018                                Université de Bordeaux
  * Copyright (C) 2017                                     Guillaume Beauchamp
- * Copyright (C) 2010-2017                                CNRS
+ * Copyright (C) 2010-2018                                CNRS
  *
  * StarPU is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
@@ -78,6 +78,12 @@ static starpu_pthread_t progress_thread;
 #endif
 static int running = 0;
 
+/* Driver taken by StarPU-MPI to process tasks when there is no requests to
+ * handle instead of polling endlessly */
+static struct starpu_driver *mpi_driver = NULL;
+static int mpi_driver_call_freq = 0;
+static int mpi_driver_task_freq = 0;
+
 #ifdef STARPU_SIMGRID
 static int wait_counter;
 static starpu_pthread_cond_t wait_counter_cond;
@@ -126,11 +132,13 @@ void _starpu_mpi_submit_ready_request_inc(struct _starpu_mpi_req *req)
 
 void _starpu_mpi_coop_sends_build_tree(struct _starpu_mpi_coop_sends *coop_sends)
 {
+	(void)coop_sends;
 	/* TODO: turn them into redirects & forwards */
 }
 
 void _starpu_mpi_submit_coop_sends(struct _starpu_mpi_coop_sends *coop_sends, int submit_control, int submit_data)
 {
+	(void)submit_control;
 	unsigned i, n = coop_sends->n;
 
 	/* Note: coop_sends might disappear very very soon after last request is submitted */
@@ -962,6 +970,7 @@ static void _starpu_mpi_test_detached_requests(void)
 		}
 		else
 		{
+			_STARPU_MPI_TRACE_POLLING_END();
 		     	struct _starpu_mpi_req *next_req;
 			next_req = _starpu_mpi_req_list_next(req);
 
@@ -993,6 +1002,7 @@ static void _starpu_mpi_test_detached_requests(void)
 			}
 
 			req = next_req;
+			_STARPU_MPI_TRACE_POLLING_BEGIN();
 		}
 
 		STARPU_PTHREAD_MUTEX_LOCK(&detached_requests_mutex);
@@ -1110,12 +1120,19 @@ static void *_starpu_mpi_progress_thread_func(void *arg)
 	starpu_pthread_setname("MPI");
 
 #ifndef STARPU_SIMGRID
-	if (_starpu_mpi_thread_cpuid >= 0)
-		_starpu_bind_thread_on_cpu(_starpu_mpi_thread_cpuid, STARPU_NOWORKERID);
+	if (_starpu_mpi_thread_cpuid < 0)
+	{
+		_starpu_mpi_thread_cpuid = starpu_get_next_bindid(STARPU_THREAD_ACTIVE, NULL, 0);
+	}
+
+	if (starpu_bind_thread_on(_starpu_mpi_thread_cpuid, STARPU_THREAD_ACTIVE, "MPI") < 0)
+	{
+		_STARPU_DISP("No core was available for the MPI thread. You should use STARPU_RESERVE_NCPU to leave one core available for MPI, or specify one core less in STARPU_NCPU\n");
+	}
 	_starpu_mpi_do_initialize(argc_argv);
 	if (_starpu_mpi_thread_cpuid >= 0)
 		/* In case MPI changed the binding */
-		_starpu_bind_thread_on_cpu(_starpu_mpi_thread_cpuid, STARPU_NOWORKERID);
+		starpu_bind_thread_on(_starpu_mpi_thread_cpuid, STARPU_THREAD_ACTIVE, "MPI");
 #endif
 
 	_starpu_mpi_env_init();
@@ -1136,6 +1153,9 @@ static void *_starpu_mpi_progress_thread_func(void *arg)
 		_STARPU_ERROR("Your version of simgrid does not provide smpi_process_set_user_data, we can not continue without it\n");
 	}
 	smpi_process_set_user_data(tsd);
+        /* And wait for StarPU to get initialized, to come back to the same
+         * situation as native execution where that's always the case. */
+	starpu_wait_initialized();
 #endif
 
 	_starpu_mpi_comm_amounts_init(argc_argv->comm);
@@ -1149,6 +1169,9 @@ static void *_starpu_mpi_progress_thread_func(void *arg)
 	_starpu_mpi_sync_data_init();
 	_starpu_mpi_datatype_init();
 
+	if (mpi_driver)
+		starpu_driver_init(mpi_driver);
+
 #ifdef STARPU_SIMGRID
 	starpu_pthread_wait_init(&wait);
 	starpu_pthread_queue_init(&dontsleep);
@@ -1172,6 +1195,9 @@ static void *_starpu_mpi_progress_thread_func(void *arg)
 	STARPU_PTHREAD_MUTEX_LOCK(&progress_mutex);
 
  	int envelope_request_submitted = 0;
+	int mpi_driver_loop_counter = 0;
+	int mpi_driver_task_counter = 0;
+	_STARPU_MPI_TRACE_POLLING_BEGIN();
 
 	while (running || posted_requests || !(_starpu_mpi_req_list_empty(&ready_recv_requests)) || !(_starpu_mpi_req_prio_list_empty(&ready_send_requests)) || !(_starpu_mpi_req_list_empty(&detached_requests)))// || !(_starpu_mpi_early_request_count()) || !(_starpu_mpi_sync_data_count()))
 	{
@@ -1198,6 +1224,7 @@ static void *_starpu_mpi_progress_thread_func(void *arg)
 		unsigned n = 0;
 		while (!_starpu_mpi_req_list_empty(&ready_recv_requests))
 		{
+			_STARPU_MPI_TRACE_POLLING_END();
 			struct _starpu_mpi_req *req;
 
 			if (n++ == nready_process)
@@ -1238,6 +1265,8 @@ static void *_starpu_mpi_progress_thread_func(void *arg)
 			STARPU_PTHREAD_MUTEX_LOCK(&progress_mutex);
 		}
 
+		_STARPU_MPI_TRACE_POLLING_BEGIN();
+
 		/* If there is no currently submitted envelope_request submitted to
                  * catch envelopes from senders, and there is some pending
                  * receive requests on our side, we resubmit a header request. */
@@ -1264,6 +1293,7 @@ static void *_starpu_mpi_progress_thread_func(void *arg)
 
 			if (flag)
 			{
+				_STARPU_MPI_TRACE_POLLING_END();
 				_STARPU_MPI_COMM_FROM_DEBUG(envelope, sizeof(struct _starpu_mpi_envelope), MPI_BYTE, envelope_status.MPI_SOURCE, _STARPU_MPI_TAG_ENVELOPE, envelope->data_tag, envelope_comm);
 				_STARPU_MPI_DEBUG(4, "Envelope received with mode %d\n", envelope->mode);
 				if (envelope->mode == _STARPU_MPI_ENVELOPE_SYNC_READY)
@@ -1355,9 +1385,34 @@ static void *_starpu_mpi_progress_thread_func(void *arg)
 					}
 				}
 				envelope_request_submitted = 0;
+				_STARPU_MPI_TRACE_POLLING_BEGIN();
 			}
 			else
 			{
+				/* A call is made to driver_run_once only when
+				 * the progression thread have gone through the
+				 * communication progression loop
+				 * mpi_driver_call_freq times. It is
+				 * interesting to tune the
+				 * STARPU_MPI_DRIVER_CALL_FREQUENCY
+				 * depending on whether the user wants
+				 * reactivity or computing power from the MPI
+				 * progression thread. */
+				if ( mpi_driver && ( ++mpi_driver_loop_counter == mpi_driver_call_freq ))
+				{
+					mpi_driver_loop_counter = 0;
+					mpi_driver_task_counter = 0;
+					while (mpi_driver_task_counter++ < mpi_driver_task_freq)
+					{
+						_STARPU_MPI_TRACE_DRIVER_RUN_BEGIN();
+						STARPU_PTHREAD_MUTEX_UNLOCK(&progress_mutex);
+						_STARPU_MPI_DEBUG(4, "running once mpi driver\n");
+						starpu_driver_run_once(mpi_driver);
+						STARPU_PTHREAD_MUTEX_LOCK(&progress_mutex);
+						_STARPU_MPI_TRACE_DRIVER_RUN_END();
+					}
+				}
+
 				//_STARPU_MPI_DEBUG(4, "Nothing received, continue ..\n");
 			}
 		}
@@ -1368,6 +1423,7 @@ static void *_starpu_mpi_progress_thread_func(void *arg)
 #endif
 	}
 
+	_STARPU_MPI_TRACE_POLLING_END();
 	if (envelope_request_submitted)
 	{
 		_starpu_mpi_comm_cancel_recv();
@@ -1539,4 +1595,42 @@ int starpu_mpi_comm_get_attr(MPI_Comm comm, int keyval, void *attribute_val, int
 	return 0;
 }
 
+void _starpu_mpi_driver_init(struct starpu_conf *conf)
+{
+	/* We only initialize the driver if the environment variable
+	 * STARPU_MPI_DRIVER_CALL_FREQUENCY is defined by the user. If this environment
+	 * variable is not defined or defined at a value lower than or equal to zero,
+	 * StarPU-MPI will not use a driver. */
+	int driver_env = starpu_get_env_number_default("STARPU_MPI_DRIVER_CALL_FREQUENCY", 0);
+	if (driver_env > 0)
+	{
+#ifdef STARPU_SIMGRID
+		_STARPU_DISP("Warning: MPI driver is not supported with simgrid, this will be disabled");
+		return;
+#endif
+		mpi_driver_call_freq = driver_env;
+
+		_STARPU_MALLOC(mpi_driver, sizeof(struct starpu_driver));
+		mpi_driver->type = STARPU_CPU_WORKER;
+		mpi_driver->id.cpu_id = 0;
+
+		conf->not_launched_drivers = mpi_driver;
+		conf->n_not_launched_drivers = 1;
+
+		int tasks_freq_env = starpu_get_env_number_default("STARPU_MPI_DRIVER_TASK_FREQUENCY", 0);
+		if (tasks_freq_env > 0)
+			mpi_driver_task_freq = tasks_freq_env;
+	}
+}
+
+void _starpu_mpi_driver_shutdown()
+{
+	if (mpi_driver)
+	{
+		starpu_driver_deinit(mpi_driver);
+		free(mpi_driver);
+		mpi_driver = NULL;
+	}
+}
+
 #endif /* STARPU_USE_MPI_MPI */

mpi/src/nmad/starpu_mpi_nmad.c

@@ -477,15 +477,20 @@ static void *_starpu_mpi_progress_thread_func(void *arg)
 {
 	struct _starpu_mpi_argc_argv *argc_argv = (struct _starpu_mpi_argc_argv *) arg;
 
-	starpu_pthread_setname("MPI");
-
 #ifndef STARPU_SIMGRID
-	if (_starpu_mpi_thread_cpuid >= 0)
-		_starpu_bind_thread_on_cpu(_starpu_mpi_thread_cpuid, STARPU_NOWORKERID);
+	if (_starpu_mpi_thread_cpuid < 0)
+	{
+		_starpu_mpi_thread_cpuid = starpu_get_next_bindid(STARPU_THREAD_ACTIVE, NULL, 0);
+	}
+
+	if (starpu_bind_thread_on(_starpu_mpi_thread_cpuid, STARPU_THREAD_ACTIVE, "MPI") < 0)
+	{
+		_STARPU_DISP("No core was available for the MPI thread. You should use STARPU_RESERVE_NCPU to leave one core available for MPI, or specify one core less in STARPU_NCPU\n");
+	}
 	_starpu_mpi_do_initialize(argc_argv);
 	if (_starpu_mpi_thread_cpuid >= 0)
 		/* In case MPI changed the binding */
-		_starpu_bind_thread_on_cpu(_starpu_mpi_thread_cpuid, STARPU_NOWORKERID);
+		starpu_bind_thread_on(_starpu_mpi_thread_cpuid, STARPU_THREAD_ACTIVE, "MPI");
 #endif
 
 	_starpu_mpi_env_init();

mpi/src/starpu_mpi_datatype.c

@@ -1,7 +1,7 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
  * Copyright (C) 2011-2012,2015                           Inria
- * Copyright (C) 2009-2011,2014-2015                      Université de Bordeaux
+ * Copyright (C) 2009-2011,2014-2015, 2018                      Université de Bordeaux
  * Copyright (C) 2010-2017                                CNRS
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -301,7 +301,7 @@ int starpu_mpi_datatype_register(starpu_data_handle_t handle, starpu_mpi_datatyp
 		table->free_datatype_func = free_datatype_func;
 		HASH_ADD_INT(_starpu_mpi_datatype_funcs_table, id, table);
 	}
-	STARPU_ASSERT_MSG(handle->ops->handle_to_pointer, "The data interface must define the operation 'handle_to_pointer'\n");
+	STARPU_ASSERT_MSG(handle->ops->handle_to_pointer || handle->ops->to_pointer, "The data interface must define the operation 'to_pointer'\n");
 	STARPU_PTHREAD_MUTEX_UNLOCK(&_starpu_mpi_datatype_funcs_table_mutex);
 	return 0;
 }

mpi/src/starpu_mpi_fxt.h

@@ -1,8 +1,8 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2012-2013                                Inria
+ * Copyright (C) 2012-2013,2016                           Inria
  * Copyright (C) 2010-2011,2014,2017                      Université de Bordeaux
- * Copyright (C) 2010,2012,2014-2017                      CNRS
+ * Copyright (C) 2010,2012,2014-2018                      CNRS
  *
  * StarPU is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
@@ -37,16 +37,6 @@ extern "C"
 #define _STARPU_MPI_FUT_IRECV_SUBMIT_END		0x5207
 #define _STARPU_MPI_FUT_ISEND_COMPLETE_BEGIN		0x5208
 #define _STARPU_MPI_FUT_ISEND_COMPLETE_END		0x5209
-#define _STARPU_MPI_FUT_IRECV_COMPLETE_BEGIN		0x5210
-#define _STARPU_MPI_FUT_IRECV_COMPLETE_END		0x5211
-#define _STARPU_MPI_FUT_SLEEP_BEGIN			0x5212
-#define _STARPU_MPI_FUT_SLEEP_END			0x5213
-#define _STARPU_MPI_FUT_DTESTING_BEGIN			0x5214
-#define _STARPU_MPI_FUT_DTESTING_END			0x5215
-#define _STARPU_MPI_FUT_UTESTING_BEGIN			0x5216
-#define _STARPU_MPI_FUT_UTESTING_END			0x5217
-#define _STARPU_MPI_FUT_UWAIT_BEGIN			0x5218
-#define _STARPU_MPI_FUT_UWAIT_END			0x5219
 #define _STARPU_MPI_FUT_DATA_SET_RANK			0x521a
 #define _STARPU_MPI_FUT_IRECV_TERMINATED		0x521b
 #define _STARPU_MPI_FUT_ISEND_TERMINATED		0x521c
@@ -54,8 +44,24 @@ extern "C"
 #define _STARPU_MPI_FUT_TESTING_DETACHED_END		0x521e
 #define _STARPU_MPI_FUT_TEST_BEGIN			0x521f
 #define _STARPU_MPI_FUT_TEST_END			0x5220
+#define _STARPU_MPI_FUT_IRECV_COMPLETE_BEGIN		0x520a
+#define _STARPU_MPI_FUT_IRECV_COMPLETE_END		0x520b
+#define _STARPU_MPI_FUT_SLEEP_BEGIN			0x520c
+#define _STARPU_MPI_FUT_SLEEP_END			0x520d
+#define _STARPU_MPI_FUT_DTESTING_BEGIN			0x520e
+#define _STARPU_MPI_FUT_DTESTING_END			0x520f
+#define _STARPU_MPI_FUT_UTESTING_BEGIN			0x5210
+#define _STARPU_MPI_FUT_UTESTING_END			0x5211
+#define _STARPU_MPI_FUT_UWAIT_BEGIN			0x5212
+#define _STARPU_MPI_FUT_UWAIT_END			0x5213
+#define _STARPU_MPI_FUT_POLLING_BEGIN			0x5214
+#define _STARPU_MPI_FUT_POLLING_END			0x5215
+#define _STARPU_MPI_FUT_DRIVER_RUN_BEGIN		0x5216
+#define _STARPU_MPI_FUT_DRIVER_RUN_END			0x5217
 
 #ifdef STARPU_USE_FXT
+static int trace_loop = 0;
+
 #define _STARPU_MPI_TRACE_START(rank, worldsize)	\
 	FUT_DO_PROBE3(_STARPU_MPI_FUT_START, (rank), (worldsize), _starpu_gettid());
 #define _STARPU_MPI_TRACE_STOP(rank, worldsize)	\
@@ -119,6 +125,24 @@ extern "C"
 #define _STARPU_MPI_TRACE_TEST_BEGIN(peer, data_tag)		do {} while(0)
 #define _STARPU_MPI_TRACE_TEST_END(peer, data_tag)		do {} while(0)
 #endif
+#define _STARPU_MPI_TRACE_POLLING_BEGIN()					\
+	if(!trace_loop) {						\
+		trace_loop = 1;							\
+		FUT_DO_PROBE1(_STARPU_MPI_FUT_POLLING_BEGIN, _starpu_gettid()); \
+	}
+#define _STARPU_MPI_TRACE_POLLING_END()	\
+	if(trace_loop) {							\
+		trace_loop = 0;							\
+		FUT_DO_PROBE1(_STARPU_MPI_FUT_POLLING_END, _starpu_gettid());	\
+	}
+#define _STARPU_MPI_TRACE_DRIVER_RUN_BEGIN()	\
+	FUT_DO_PROBE1(_STARPU_MPI_FUT_DRIVER_RUN_BEGIN,  _starpu_gettid());
+#define _STARPU_MPI_TRACE_DRIVER_RUN_END()	\
+	FUT_DO_PROBE1(_STARPU_MPI_FUT_DRIVER_RUN_END, _starpu_gettid());
+#define _STARPU_MPI_TRACE_DRIVER_RUN_BEGIN()	\
+	FUT_DO_PROBE1(_STARPU_MPI_FUT_DRIVER_RUN_BEGIN,  _starpu_gettid());
+#define _STARPU_MPI_TRACE_DRIVER_RUN_END()	\
+	FUT_DO_PROBE1(_STARPU_MPI_FUT_DRIVER_RUN_END, _starpu_gettid());
 #define TRACE
 #else
 #define _STARPU_MPI_TRACE_START(a, b)				do {} while(0);
@@ -148,6 +172,10 @@ extern "C"
 #define _STARPU_MPI_TRACE_TESTING_DETACHED_END()		do {} while(0)
 #define _STARPU_MPI_TRACE_TEST_BEGIN(peer, data_tag)		do {} while(0)
 #define _STARPU_MPI_TRACE_TEST_END(peer, data_tag)		do {} while(0)
+#define _STARPU_MPI_TRACE_POLLING_BEGIN()			do {} while(0);
+#define _STARPU_MPI_TRACE_POLLING_END()				do {} while(0);
+#define _STARPU_MPI_TRACE_DRIVER_RUN_BEGIN()			do {} while(0);
+#define _STARPU_MPI_TRACE_DRIVER_RUN_END()			do {} while(0);
 #endif
 
 #ifdef __cplusplus

mpi/src/starpu_mpi_init.c

@@ -1,6 +1,6 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2010-2017                                CNRS
+ * Copyright (C) 2010-2018                                CNRS
  * Copyright (C) 2009-2018                                Université de Bordeaux
  * Copyright (C) 2016                                     Inria
  *
@@ -35,12 +35,14 @@
 #if defined(STARPU_USE_MPI_MPI)
 #include <mpi/starpu_mpi_comm.h>
 #include <mpi/starpu_mpi_tag.h>
+#include <mpi/starpu_mpi_driver.h>
 #endif
 
 #ifdef STARPU_SIMGRID
 static int _mpi_world_size;
 static int _mpi_world_rank;
 #endif
+static int _mpi_initialized_starpu;
 
 static void _starpu_mpi_print_thread_level_support(int thread_level, char *msg)
 {
@@ -71,6 +73,7 @@ void _starpu_mpi_do_initialize(struct _starpu_mpi_argc_argv *argc_argv)
 {
 	if (argc_argv->initialize_mpi)
 	{
+		STARPU_ASSERT_MSG(argc_argv->comm == MPI_COMM_WORLD, "It does not make sense to ask StarPU-MPI to initialize MPI while a non-world communicator was given");
 		int thread_support;
 #ifdef STARPU_USE_MPI_NMAD
 		/* strat_prio is preferred for StarPU instead of default strat_aggreg */
@@ -177,6 +180,36 @@ int starpu_mpi_initialize_extended(int *rank, int *world_size)
 #endif
 }
 
+int starpu_mpi_init_conf(int *argc, char ***argv, int initialize_mpi, MPI_Comm comm, struct starpu_conf *conf)
+{
+	struct starpu_conf localconf;
+	if (!conf)
+	{
+		starpu_conf_init(&localconf);
+		conf = &localconf;
+	}
+
+#if defined(STARPU_USE_MPI_MPI)
+	_starpu_mpi_driver_init(conf);
+
+	if (starpu_get_env_number_default("STARPU_MPI_DRIVER_CALL_FREQUENCY", 0) <= 0)
+#endif
+	{
+		/* Reserve a core for our progression thread */
+		if (conf->reserve_ncpus == -1)
+			conf->reserve_ncpus = 1;
+		else
+			conf->reserve_ncpus++;
+	}
+
+	int ret = starpu_init(conf);
+	if (ret < 0)
+		return ret;
+	_mpi_initialized_starpu = 1;
+
+	return starpu_mpi_init_comm(argc, argv, initialize_mpi, comm);
+}
+
 int starpu_mpi_shutdown(void)
 {
 	void *value;
@@ -197,7 +230,10 @@ int starpu_mpi_shutdown(void)
 #if defined(STARPU_USE_MPI_MPI)
 	_starpu_mpi_tag_shutdown();
 	_starpu_mpi_comm_shutdown();
+	_starpu_mpi_driver_shutdown();
 #endif
+	if (_mpi_initialized_starpu)
+		starpu_shutdown();
 
 	return 0;
 }

mpi/src/starpu_mpi_task_insert.c

@@ -1,7 +1,7 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
  * Copyright (C) 2012,2014,2016-2017                      Inria
- * Copyright (C) 2011-2017                                CNRS
+ * Copyright (C) 2011-2018                                CNRS
  * Copyright (C) 2011-2018                                Université de Bordeaux
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -448,6 +448,18 @@ int _starpu_mpi_task_decode_v(struct starpu_codelet *codelet, int me, int nb_nod
 		{
 			select_node_policy = va_arg(varg_list_copy, int);
 		}
+		else if (arg_type==STARPU_TASK_COLOR)
+                {
+                        (void)va_arg(varg_list_copy, int);
+                }
+		else if (arg_type==STARPU_TASK_SYNCHRONOUS)
+                {
+                        (void)va_arg(varg_list_copy, int);
+                }
+		else if (arg_type==STARPU_HANDLES_SEQUENTIAL_CONSISTENCY)
+                {
+                        (void)va_arg(varg_list_copy, char *);
+                }
 		else
 		{
 			STARPU_ABORT_MSG("Unrecognized argument %d, did you perhaps forget to end arguments with 0?\n", arg_type);

mpi/src/starpu_mpi_task_insert_fortran.c

@@ -1,6 +1,6 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2016-2017                                CNRS
+ * Copyright (C) 2016-2018                                CNRS
  * Copyright (C) 2017-2018                                     Université de Bordeaux
  * Copyright (C) 2016                                     Inria
  *
@@ -293,6 +293,21 @@ int _fstarpu_mpi_task_decode_v(struct starpu_codelet *codelet, int me, int nb_no
 			arg_i++;
 			/* int* */
 		}
+		else if (arg_type==STARPU_TASK_COLOR)
+		{
+			arg_i++;
+			/* int* */
+		}
+		else if (arg_type==STARPU_TASK_SYNCHRONOUS)
+		{
+			arg_i++;
+			/* int* */
+		}
+		else if (arg_type==STARPU_HANDLES_SEQUENTIAL_CONSISTENCY)
+		{
+			arg_i++;
+			/* char* */
+		}
 		else
 		{
 			STARPU_ABORT_MSG("Unrecognized argument %d, did you perhaps forget to end arguments with 0?\n", arg_type);

mpi/tests/Makefile.am

@@ -1,6 +1,6 @@
 # StarPU --- Runtime system for heterogeneous multicore architectures.
 #
-# Copyright (C) 2010-2017                                CNRS
+# Copyright (C) 2010-2018                                CNRS
 # Copyright (C) 2009-2018                                Université de Bordeaux
 # Copyright (C) 2013                                     Thibaut Lambert
 #
@@ -158,7 +158,8 @@ starpu_mpi_TESTS +=				\
 	tags_checking				\
 	sync					\
 	gather					\
-	gather2
+	gather2					\
+	driver
 
 if STARPU_USE_MPI_MPI
 starpu_mpi_TESTS +=				\
@@ -228,7 +229,8 @@ noinst_PROGRAMS =				\
 	policy_selection2			\
 	early_request				\
 	starpu_redefine				\
-	load_balancer
+	load_balancer				\
+	driver
 
 XFAIL_TESTS=					\
 	policy_register_toomany			\

mpi/tests/block_interface.c

@@ -1,6 +1,6 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2009-2011,2014-2015,2017                 Université de Bordeaux
+ * Copyright (C) 2009-2011,2014-2015,2017-2018                 Université de Bordeaux
  * Copyright (C) 2010-2012,2014-2015,2017                 CNRS
  * Copyright (C) 2013                                     Inria
  *
@@ -32,10 +32,8 @@ int main(int argc, char **argv)
 
 	MPI_INIT_THREAD(&argc, &argv, MPI_THREAD_SERIALIZED, &mpi_init);
 
-	ret = starpu_init(NULL);
-	STARPU_CHECK_RETURN_VALUE(ret, "starpu_init");
-	ret = starpu_mpi_init(&argc, &argv, mpi_init);
-	STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init");
+	ret = starpu_mpi_init_conf(&argc, &argv, mpi_init, MPI_COMM_WORLD, NULL);
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init_conf");
 
 	starpu_mpi_comm_rank(MPI_COMM_WORLD, &rank);
 	starpu_mpi_comm_size(MPI_COMM_WORLD, &size);
@@ -46,7 +44,6 @@ int main(int argc, char **argv)
 			FPRINTF(stderr, "We need at least 2 processes.\n");
 
 		starpu_mpi_shutdown();
-		starpu_shutdown();
 		MPI_Finalize();
 		return STARPU_TEST_SKIPPED;
 	}
@@ -143,7 +140,6 @@ int main(int argc, char **argv)
 		free(block);
 	}
 	starpu_mpi_shutdown();
-	starpu_shutdown();
 
 	if (!mpi_init)
 		MPI_Finalize();