Add STARPU_RESERVE_NCPU environment variable and reserve_ncpus config field

to make StarPU use a few cores less

ChangeLog

@@ -58,6 +58,8 @@ New features:
   * Add starpu_get_next_bindid and starpu_bind_thread_on to allow binding an
     application-started thread on a free core. Use it in StarPU-MPI to
     automatically bind the MPI thread on an available core.
+  * Add STARPU_RESERVE_NCPU environment variable and reserve_ncpus config
+    field to make StarPU use a few cores less.
 
 Small features:
   * Scheduling contexts may now be associated a user data pointer at creation

doc/doxygen/chapters/501_environment_variables.doxy

@@ -36,6 +36,17 @@ not allocate more CPU workers than there are physical CPUs, and that
 some CPUs are used to control the accelerators.
 </dd>
 
+<dt>STARPU_RESERVE_NCPU</dt>
+<dd>
+\anchor STARPU_RESERVE_NCPU
+\addindex __env__STARPU_RESERVE_NCPU
+Specify the number of CPU cores that should not be used by StarPU, so the
+application can use starpu_get_next_bindid() and starpu_bind_thread_on() to bind
+its own threads.
+
+This option is ignored if \ref STARPU_NCPU or starpu_config::ncpus is set.
+</dd>
+
 <dt>STARPU_NCPUS</dt>
 <dd>
 \anchor STARPU_NCPUS

include/starpu.h

@@ -94,6 +94,7 @@ struct starpu_conf
 	void (*sched_policy_init)(unsigned);
 
 	int ncpus;
+	int reserve_ncpus;
 	int ncuda;
 	int nopencl;
 	int nmic;

mpi/src/mpi/starpu_mpi_mpi.c

@@ -1125,10 +1125,7 @@ static void *_starpu_mpi_progress_thread_func(void *arg)
 
 	if (starpu_bind_thread_on(_starpu_mpi_thread_cpuid, STARPU_THREAD_ACTIVE, "MPI") < 0)
 	{
-#ifdef STARPU_DEVEL
-#warning we should make this automatic by adding a CPU reservation field in starpu_config
-#endif
-		_STARPU_DISP("No core was available for the MPI thread. You should use STARPU_NCPU to leave one core available for MPI\n");
+		_STARPU_DISP("No core was available for the MPI thread. You should use STARPU_RESERVE_NCPU to leave one core available for MPI, or specify one core less in STARPU_NCPU\n");
 	}
 	_starpu_mpi_do_initialize(argc_argv);
 	if (_starpu_mpi_thread_cpuid >= 0)

mpi/src/nmad/starpu_mpi_nmad.c

@@ -485,10 +485,7 @@ static void *_starpu_mpi_progress_thread_func(void *arg)
 
 	if (starpu_bind_thread_on(_starpu_mpi_thread_cpuid, STARPU_THREAD_ACTIVE, "MPI") < 0)
 	{
-#ifdef STARPU_DEVEL
-#warning we should make this automatic by adding a CPU reservation field in starpu_config
-#endif
-		_STARPU_DISP("No core was available for the MPI thread. You should use STARPU_NCPU to leave one core available for MPI\n");
+		_STARPU_DISP("No core was available for the MPI thread. You should use STARPU_RESERVE_NCPU to leave one core available for MPI, or specify one core less in STARPU_NCPU\n");
 	}
 	_starpu_mpi_do_initialize(argc_argv);
 	if (_starpu_mpi_thread_cpuid >= 0)

src/core/topology.c

@@ -1856,19 +1856,37 @@ _starpu_init_machine_config(struct _starpu_machine_config *config, int no_mp_con
 				avail_cpus = 0;
 			int nth_per_core = starpu_get_env_number_default("STARPU_NTHREADS_PER_CORE", 1);
 			avail_cpus *= nth_per_core;
-			ncpu = STARPU_MIN(avail_cpus, STARPU_MAXCPUS);
+
+			if (config->conf.reserve_ncpus > 0)
+			{
+				if (avail_cpus < config->conf.reserve_ncpus)
+				{
+					_STARPU_DISP("Warning: %d CPU cores were requested to be reserved, but only %ld were available,\n", config->conf.reserve_ncpus, avail_cpus);
+					avail_cpus = 0;
+				}
+				else
+				{
+					avail_cpus -= config->conf.reserve_ncpus;
+				}
+			}
+
+			ncpu = avail_cpus;
+			if (ncpu > STARPU_MAXCPUS)
+			{
+				_STARPU_DISP("Warning: %d CPU cores detected. Only %d enabled. Use configure option --enable-maxcpus=xxx to update the maximum value of supported CPU devices.\n", ncpu, STARPU_MAXCPUS);
+				ncpu = STARPU_MAXCPUS;
+			}
 		}
 		else
 		{
 			if (ncpu > STARPU_MAXCPUS)
 			{
-				_STARPU_DISP("Warning: %d CPU devices requested. Only %d enabled. Use configure option --enable-maxcpus=xxx to update the maximum value of supported CPU devices.\n", ncpu, STARPU_MAXCPUS);
+				_STARPU_DISP("Warning: %d CPU cores requested. Only %d enabled. Use configure option --enable-maxcpus=xxx to update the maximum value of supported CPU devices.\n", ncpu, STARPU_MAXCPUS);
 				ncpu = STARPU_MAXCPUS;
 			}
 		}
 	}
 
-
 	topology->ncpus = ncpu;
 	STARPU_ASSERT(topology->ncpus + topology->nworkers <= STARPU_NMAXWORKERS);
 

src/core/workers.c

@@ -981,6 +981,7 @@ int starpu_conf_init(struct starpu_conf *conf)
 	conf->ncpus = starpu_get_env_number("STARPU_NCPU");
 	if (conf->ncpus == -1)
 		conf->ncpus = starpu_get_env_number("STARPU_NCPUS");
+	conf->reserve_ncpus = starpu_get_env_number("STARPU_RESERVE_NCPU");
 	conf->ncuda = starpu_get_env_number("STARPU_NCUDA");
 	conf->nopencl = starpu_get_env_number("STARPU_NOPENCL");
 	conf->nmic = starpu_get_env_number("STARPU_NMIC");
@@ -1072,6 +1073,7 @@ void _starpu_conf_check_environment(struct starpu_conf *conf)
 
 	_starpu_conf_set_value_against_environment("STARPU_NCPUS", &conf->ncpus);
 	_starpu_conf_set_value_against_environment("STARPU_NCPU", &conf->ncpus);
+	_starpu_conf_set_value_against_environment("STARPU_RESERVE_NCPU", &conf->reserve_ncpus);
 	_starpu_conf_set_value_against_environment("STARPU_NCUDA", &conf->ncuda);
 	_starpu_conf_set_value_against_environment("STARPU_NOPENCL", &conf->nopencl);
 	_starpu_conf_set_value_against_environment("STARPU_CALIBRATE", &conf->calibrate);