Add field starpu_conf::ignore_environment_variables to ignore environment variables when parameters are set directly in starpu_conf

ChangeLog

@@ -54,6 +54,8 @@ Small features:
   * Add STARPU_WORKERS_GETBIND environment variable.
   * Add STARPU_SCHED_SIMPLE_DECIDE_ALWAYS modular scheduler flag.
   * And STARPU_LIMIT_BANDWIDTH environment variable.
+  * Add field starpu_conf::ignore_environment_variables to ignore
+    environment variables when parameters are set directly in starpu_conf
 
 StarPU 1.3.3 (git revision 11afc5b007fe1ab1c729b55b47a5a98ef7f3cfad)
 ====================================================================

include/starpu.h

@@ -126,6 +126,17 @@ struct starpu_conf
 	void (*sched_policy_init)(unsigned);
 
 	/**
+	   For all parameters specified in this structure that can
+	   also be set with environment variables, by default,
+	   StarPU chooses the value of the environment variable
+	   against the value set in starpu_conf. Setting the parameter
+	   starpu_conf::ignore_environment_variables to 1 allows to give precedence
+	   to the value set in the structure over the environment
+	   variable.
+	 */
+	int ignore_environment_variables;
+
+	/**
 	   Number of CPU cores that StarPU can use. This can also be
 	   specified with the environment variable \ref STARPU_NCPU.
 	   (default = -1)

socl/src/init.c

@@ -57,8 +57,7 @@ int socl_init_starpu(void)
 	STARPU_PTHREAD_MUTEX_UNLOCK(&_socl_mutex);
 
 	starpu_conf_init(&conf);
-	unsetenv("STARPU_NCPU");
-	unsetenv("STARPU_NCUDA");
+	conf.ignore_environment_variables = 1;
 	conf.ncuda = 0;
 	conf.ncpus = 0;
 

src/core/workers.c

@@ -1148,13 +1148,16 @@ int starpu_conf_init(struct starpu_conf *conf)
 	return 0;
 }
 
-static void _starpu_conf_set_value_against_environment(char *name, int *value)
+static void _starpu_conf_set_value_against_environment(char *name, int *value, int ignore_env)
 {
-	int number;
-	number = starpu_get_env_number(name);
-	if (number != -1)
+	if (ignore_env == 0)
 	{
-		*value = number;
+		int number;
+		number = starpu_get_env_number(name);
+		if (number != -1)
+		{
+			*value = number;
+		}
 	}
 }
 
@@ -1166,16 +1169,16 @@ void _starpu_conf_check_environment(struct starpu_conf *conf)
 		conf->sched_policy_name = sched;
 	}
 
-	_starpu_conf_set_value_against_environment("STARPU_NCPUS", &conf->ncpus);
-	_starpu_conf_set_value_against_environment("STARPU_NCPU", &conf->ncpus);
-	_starpu_conf_set_value_against_environment("STARPU_RESERVE_NCPU", &conf->reserve_ncpus);
+	_starpu_conf_set_value_against_environment("STARPU_NCPUS", &conf->ncpus, conf->ignore_environment_variables);
+	_starpu_conf_set_value_against_environment("STARPU_NCPU", &conf->ncpus, conf->ignore_environment_variables);
+	_starpu_conf_set_value_against_environment("STARPU_RESERVE_NCPU", &conf->reserve_ncpus, conf->ignore_environment_variables);
 	int main_thread_bind = starpu_get_env_number_default("STARPU_MAIN_THREAD_BIND", 0);
 	if (main_thread_bind)
 		conf->reserve_ncpus++;
-	_starpu_conf_set_value_against_environment("STARPU_NCUDA", &conf->ncuda);
-	_starpu_conf_set_value_against_environment("STARPU_NOPENCL", &conf->nopencl);
-	_starpu_conf_set_value_against_environment("STARPU_CALIBRATE", &conf->calibrate);
-	_starpu_conf_set_value_against_environment("STARPU_BUS_CALIBRATE", &conf->bus_calibrate);
+	_starpu_conf_set_value_against_environment("STARPU_NCUDA", &conf->ncuda, conf->ignore_environment_variables);
+	_starpu_conf_set_value_against_environment("STARPU_NOPENCL", &conf->nopencl, conf->ignore_environment_variables);
+	_starpu_conf_set_value_against_environment("STARPU_CALIBRATE", &conf->calibrate, conf->ignore_environment_variables);
+	_starpu_conf_set_value_against_environment("STARPU_BUS_CALIBRATE", &conf->bus_calibrate, conf->ignore_environment_variables);
 #ifdef STARPU_SIMGRID
 	if (conf->calibrate == 2)
 	{
@@ -1186,12 +1189,12 @@ void _starpu_conf_check_environment(struct starpu_conf *conf)
 		_STARPU_DISP("Warning: Bus calibration will be cleared due to bus_calibrate or STARPU_BUS_CALIBRATE being set. This will prevent simgrid from having data transfer simulation times!");
 	}
 #endif
-	_starpu_conf_set_value_against_environment("STARPU_SINGLE_COMBINED_WORKER", &conf->single_combined_worker);
-	_starpu_conf_set_value_against_environment("STARPU_DISABLE_ASYNCHRONOUS_COPY", &conf->disable_asynchronous_copy);
-	_starpu_conf_set_value_against_environment("STARPU_DISABLE_ASYNCHRONOUS_CUDA_COPY", &conf->disable_asynchronous_cuda_copy);
-	_starpu_conf_set_value_against_environment("STARPU_DISABLE_ASYNCHRONOUS_OPENCL_COPY", &conf->disable_asynchronous_opencl_copy);
-	_starpu_conf_set_value_against_environment("STARPU_DISABLE_ASYNCHRONOUS_MIC_COPY", &conf->disable_asynchronous_mic_copy);
-	_starpu_conf_set_value_against_environment("STARPU_DISABLE_ASYNCHRONOUS_MPI_MS_COPY", &conf->disable_asynchronous_mpi_ms_copy);
+	_starpu_conf_set_value_against_environment("STARPU_SINGLE_COMBINED_WORKER", &conf->single_combined_worker, conf->ignore_environment_variables);
+	_starpu_conf_set_value_against_environment("STARPU_DISABLE_ASYNCHRONOUS_COPY", &conf->disable_asynchronous_copy, conf->ignore_environment_variables);
+	_starpu_conf_set_value_against_environment("STARPU_DISABLE_ASYNCHRONOUS_CUDA_COPY", &conf->disable_asynchronous_cuda_copy, conf->ignore_environment_variables);
+	_starpu_conf_set_value_against_environment("STARPU_DISABLE_ASYNCHRONOUS_OPENCL_COPY", &conf->disable_asynchronous_opencl_copy, conf->ignore_environment_variables);
+	_starpu_conf_set_value_against_environment("STARPU_DISABLE_ASYNCHRONOUS_MIC_COPY", &conf->disable_asynchronous_mic_copy, conf->ignore_environment_variables);
+	_starpu_conf_set_value_against_environment("STARPU_DISABLE_ASYNCHRONOUS_MPI_MS_COPY", &conf->disable_asynchronous_mpi_ms_copy, conf->ignore_environment_variables);
 }
 
 struct starpu_tree* starpu_workers_get_tree(void)

tests/datawizard/scratch_reuse.c

@@ -17,8 +17,8 @@
 #include <starpu.h>
 #include "../helper.h"
 
-#if !defined(STARPU_HAVE_UNSETENV) || !defined(STARPU_USE_CPU) || !defined(STARPU_HAVE_HWLOC)
-#warning unsetenv is not defined or no cpu are available. Skipping test
+#if !defined(STARPU_HAVE_SETENV) || !defined(STARPU_USE_CPU) || !defined(STARPU_HAVE_HWLOC)
+#warning setenv is not defined or no cpu are available. Skipping test
 int main(void)
 {
 	return STARPU_TEST_SKIPPED;

tests/disk/disk_copy.c

@@ -61,13 +61,11 @@ int dotest(struct starpu_disk_ops *ops, void *param)
 
 	/* Initialize StarPU without GPU devices to make sure the memory of the GPU devices will not be used */
 	// Ignore environment variables as we want to force the exact number of workers
-	unsetenv("STARPU_NCUDA");
-	unsetenv("STARPU_NOPENCL");
-	unsetenv("STARPU_NMIC");
 	struct starpu_conf conf;
 	ret = starpu_conf_init(&conf);
 	if (ret == -EINVAL)
 		return EXIT_FAILURE;
+	conf.ignore_environment_variables = 1;
 	conf.ncpus = 1;
 	conf.ncuda = 0;
 	conf.nopencl = 0;

tests/disk/disk_copy_unpack.c

@@ -49,13 +49,11 @@ int dotest(struct starpu_disk_ops *ops, void *param)
 
 	/* Initialize StarPU without GPU devices to make sure the memory of the GPU devices will not be used */
 	// Ignore environment variables as we want to force the exact number of workers
-	unsetenv("STARPU_NCUDA");
-	unsetenv("STARPU_NOPENCL");
-	unsetenv("STARPU_NMIC");
 	struct starpu_conf conf;
 	ret = starpu_conf_init(&conf);
 	if (ret == -EINVAL)
 		return EXIT_FAILURE;
+	conf.ignore_environment_variables = 1;
 	conf.ncuda = 0;
 	conf.nopencl = 0;
 	conf.nmic = 0;
@@ -85,7 +83,7 @@ int dotest(struct starpu_disk_ops *ops, void *param)
 	{
 		.any_to_any = NULL,
 	};
-	
+
 	starpu_interface_vector_ops.copy_methods = &my_vector_copy_data_methods_s;
 
 	/* register vector in starpu */

tests/disk/disk_pack.c

@@ -26,14 +26,6 @@
 #include <math.h>
 #include "../helper.h"
 
-#if !defined(STARPU_HAVE_UNSETENV)
-#warning unsetenv is not defined. Skipping test
-int main(void)
-{
-	return STARPU_TEST_SKIPPED;
-}
-#else
-
 /*
  * Try to write into disk memory
  * Use mechanism to push datas from main ram to disk ram
@@ -69,14 +61,11 @@ int dotest(struct starpu_disk_ops *ops, char *base)
 
 	/* Initialize StarPU without GPU devices to make sure the memory of the GPU devices will not be used */
 	// Ignore environment variables as we want to force the exact number of workers
-	unsetenv("STARPU_NCUDA");
-	unsetenv("STARPU_NOPENCL");
-	unsetenv("STARPU_NMIC");
-
 	struct starpu_conf conf;
 	int ret = starpu_conf_init(&conf);
 	if (ret == -EINVAL)
 		return EXIT_FAILURE;
+	conf.ignore_environment_variables = 1;
 	conf.ncuda = 0;
 	conf.nopencl = 0;
 	conf.nmic = 0;
@@ -288,4 +277,3 @@ int main(void)
 		STARPU_CHECK_RETURN_VALUE(-errno, "rmdir '%s'\n", s);
 	return ret;
 }
-#endif

tests/disk/mem_reclaim.c

@@ -154,15 +154,14 @@ int dotest(struct starpu_disk_ops *ops, char *base, void (*vector_data_register)
 	FPRINTF(stderr, "Testing <%s>\n", text);
 	/* Initialize StarPU without GPU devices to make sure the memory of the GPU devices will not be used */
 	// Ignore environment variables as we want to force the exact number of workers
-	unsetenv("STARPU_NCUDA");
-	unsetenv("STARPU_NOPENCL");
-	unsetenv("STARPU_NMIC");
 	struct starpu_conf conf;
 	int ret = starpu_conf_init(&conf);
 	if (ret == -EINVAL)
 		return EXIT_FAILURE;
+	conf.ignore_environment_variables = 1;
 	conf.ncuda = 0;
 	conf.nopencl = 0;
+	conf.nmic = 0;
 	ret = starpu_init(&conf);
 	if (ret == -ENODEV) goto enodev;
 

tests/errorcheck/invalid_tasks.c

@@ -21,8 +21,8 @@
  * Check that we detect that with only a CPU we can't submit a GPU-only task
  */
 
-#if !defined(STARPU_HAVE_UNSETENV) || !defined(STARPU_USE_CPU)
-#warning unsetenv is not defined or no cpu are available. Skipping test
+#if !defined(STARPU_USE_CPU)
+#warning no cpu are available. Skipping test
 int main(void)
 {
 	return STARPU_TEST_SKIPPED;
@@ -48,11 +48,9 @@ int main(void)
 	int ret;
 
 	/* We force StarPU to use 1 CPU only */
-	unsetenv("STARPU_NCUDA");
-	unsetenv("STARPU_NOPENCL");
-	unsetenv("STARPU_NCPUS");
 	struct starpu_conf conf;
 	starpu_conf_init(&conf);
+	conf.ignore_environment_variables = 1;
 	conf.ncpus = 1;
 	conf.nopencl = 0;
 	conf.ncuda = 0;

tests/errorcheck/starpu_init_noworker.c

@@ -25,31 +25,14 @@
  * Test that starpu_initialize returns ENODEV when no worker is available
  */
 
-#if !defined(STARPU_HAVE_UNSETENV)
-#warning unsetenv is not defined. Skipping test
-int main(void)
-{
-	return STARPU_TEST_SKIPPED;
-}
-#else
-static void unset_env_variables(void)
-{
-	(void) unsetenv("STARPU_NCPUS");
-	(void) unsetenv("STARPU_NCPU");
-	(void) unsetenv("STARPU_NCUDA");
-	(void) unsetenv("STARPU_NOPENCL");
-	(void) unsetenv("STARPU_NMIC");
-}
-
 int main(int argc, char **argv)
 {
 	int ret;
 
-	unset_env_variables();
-
 	/* We try to initialize StarPU without any worker */
 	struct starpu_conf conf;
 	starpu_conf_init(&conf);
+	conf.ignore_environment_variables = 1;
 	conf.ncpus = 0;
 	conf.ncuda = 0;
 	conf.nopencl = 0;
@@ -78,4 +61,3 @@ int main(int argc, char **argv)
 
 
 }
-#endif

tests/errorcheck/workers_cpuid.c

@@ -22,8 +22,8 @@
  * expected binding does happen
  */
 
-#if !defined(STARPU_HAVE_UNSETENV) || !defined(STARPU_USE_CPU) || !defined(STARPU_HAVE_HWLOC)
-#warning unsetenv is not defined or no cpu are available. Skipping test
+#if !defined(STARPU_USE_CPU) || !defined(STARPU_HAVE_HWLOC)
+#warning no cpu are available. Skipping test
 int main(void)
 {
 	return STARPU_TEST_SKIPPED;
@@ -44,8 +44,6 @@ int nhwpus;
 long workers_cpuid[STARPU_NMAXWORKERS];
 int workers_id[STARPU_NMAXWORKERS];
 
-
-
 static int check_workers_mapping(long *cpuid, int *workerids, int nb_workers)
 {
 	int i;
@@ -64,7 +62,6 @@ static int check_workers_mapping(long *cpuid, int *workerids, int nb_workers)
 	return 1;
 }
 
-
 static void copy_cpuid_array(long *dst, long *src, unsigned n)
 {
 	int i;
@@ -102,7 +99,15 @@ static int test_combination(long *combination, unsigned n)
 	setenv("STARPU_WORKERS_CPUID", str, 1);
 	free(str);
 
-	ret = starpu_init(NULL);
+	struct starpu_conf conf;
+	starpu_conf_init(&conf);
+	conf.ignore_environment_variables = 1;
+	conf.ncuda = 0;
+	conf.nopencl = 0;
+	conf.nmic = 0;
+	conf.nmpi_ms = 0;
+
+	ret = starpu_init(&conf);
 	if (ret == -ENODEV) return STARPU_TEST_SKIPPED;
 	STARPU_CHECK_RETURN_VALUE(ret, "starpu_init");
 
@@ -161,12 +166,6 @@ int main(void)
 	if (nhwpus > STARPU_NMAXWORKERS)
 		nhwpus = STARPU_NMAXWORKERS;
 
-	setenv("STARPU_NCUDA","0",1);
-	setenv("STARPU_NOPENCL","0",1);
-	setenv("STARPU_NMIC","0",1);
-	setenv("STARPU_NMPI_MS","0",1);
-	unsetenv("STARPU_NCPUS");
-
 	for (i=0; i<STARPU_NMAXWORKERS; i++)
 		workers_id[i] = -1;
 

tests/main/driver_api/init_run_deinit.c

@@ -18,14 +18,6 @@
 
 #include "../../helper.h"
 
-#if !defined(STARPU_HAVE_UNSETENV)
-#warning unsetenv is not defined. Skipping test
-int main(void)
-{
-	return STARPU_TEST_SKIPPED;
-}
-#else
-
 #define NTASKS 8
 
 #if defined(STARPU_USE_CPU) || defined(STARPU_USE_CUDA) || defined(STARPU_USE_OPENCL)
@@ -236,6 +228,7 @@ static int test_opencl(void)
 		.id.opencl_id = device_id
 	};
 
+	conf.ignore_environment_variables = 1;
 	conf.ncpus = 0;
 	conf.ncuda = 0;
 	conf.nopencl = 1;
@@ -284,11 +277,6 @@ int main(void)
 {
 	int ret = STARPU_TEST_SKIPPED;
 
-	// Ignore environment variables as we want to force the exact number of workers
-	unsetenv("STARPU_NCUDA");
-	unsetenv("STARPU_NOPENCL");
-	unsetenv("STARPU_NCPUS");
-
 #ifdef STARPU_USE_CPU
 	ret = test_cpu();
 	if (ret == 1)
@@ -307,4 +295,4 @@ int main(void)
 
 	return ret;
 }
-#endif
+

tests/main/driver_api/run_driver.c

@@ -19,14 +19,6 @@
 
 #include "../../helper.h"
 
-#if !defined(STARPU_HAVE_UNSETENV)
-#warning unsetenv is not defined. Skipping test
-int main(void)
-{
-	return STARPU_TEST_SKIPPED;
-}
-#else
-
 /*
  * Users can directly control drivers by using the starpu_driver* functions.
  *
@@ -228,6 +220,7 @@ static int test_opencl(void)
 	};
 
 	starpu_conf_init(&conf);
+	conf.ignore_environment_variables = 1;
 	conf.n_not_launched_drivers = 1;
 	conf.not_launched_drivers = &d;
 	conf.ncpus = 1;
@@ -276,11 +269,6 @@ int main(void)
 {
 	int ret = STARPU_TEST_SKIPPED;
 
-	// Ignore environment variables as we want to force the exact number of workers
-	unsetenv("STARPU_NCUDA");
-	unsetenv("STARPU_NOPENCL");
-	unsetenv("STARPU_NCPUS");
-
 #ifdef STARPU_USE_CPU
 	ret = test_cpu();
 	if (ret == 1)
@@ -298,4 +286,3 @@ int main(void)
 #endif
 	return ret;
 }
-#endif

tests/main/starpu_init.c

@@ -32,11 +32,11 @@ int main(void)
 #else
 
 
-static int check_cpu(int env_cpu, int conf_cpu, int expected_cpu, int *cpu)
+static int check_cpu(int env_cpu, int conf_cpu, int ignore_env, int expected_cpu, int *cpu)
 {
 	int ret;
 
-	FPRINTF(stderr, "Testing with env=%d - conf=%d\n", env_cpu, conf_cpu);
+	FPRINTF(stderr, "\nTesting with env=%d - conf=%d - expected %d (ignore env %d)\n", env_cpu, conf_cpu, expected_cpu, ignore_env);
 
 	if (env_cpu != -1)
 	{
@@ -47,6 +47,8 @@ static int check_cpu(int env_cpu, int conf_cpu, int expected_cpu, int *cpu)
 
 	struct starpu_conf user_conf;
 	starpu_conf_init(&user_conf);
+	user_conf.ignore_environment_variables = ignore_env;
+
 	if (conf_cpu != -1)
 	{
 		user_conf.ncpus = conf_cpu;
@@ -74,7 +76,7 @@ static int check_cpu(int env_cpu, int conf_cpu, int expected_cpu, int *cpu)
 	}
 	else
 	{
-		FPRINTF(stderr, "Number of CPUS: %3d -- Number of expected CPUs: %3d\n", *cpu, expected_cpu);
+		FPRINTF(stderr, "Number of CPUS: %3d -- Number of expected CPUs: %3d    --> %s\n", *cpu, expected_cpu, *cpu==expected_cpu?"SUCCESS":"FAILURE");
 		return *cpu != expected_cpu;
 	}
 }
@@ -88,7 +90,7 @@ int main(void)
 	unsetenv("STARPU_NCPUS");
 	unsetenv("STARPU_NCPU");
 
-	ret = check_cpu(-1, -1, -1, &cpu_init);
+	ret = check_cpu(-1, -1, 0, -1, &cpu_init);
 	if (ret) return ret;
 	if (cpu_init <= 1) return STARPU_TEST_SKIPPED;
 
@@ -105,10 +107,10 @@ int main(void)
 		cpu_test3 = cpu_init+3;
 	}
 
-	ret = check_cpu(cpu_test1, -1, cpu_test1, &cpu);
+	ret = check_cpu(cpu_test1, -1, 0, cpu_test1, &cpu);
 	if (ret) return ret;
 
-	ret = check_cpu(-1, -1, -1, &cpu);
+	ret = check_cpu(-1, -1, 0, -1, &cpu);
 	if (ret) return ret;
 	if (cpu != cpu_init)
 	{
@@ -116,10 +118,13 @@ int main(void)
 		return 1;
 	}
 
-	ret = check_cpu(-1, cpu_test2, cpu_test2, &cpu);
+	ret = check_cpu(-1, cpu_test2, 0, cpu_test2, &cpu);
+	if (ret) return ret;
+
+	ret = check_cpu(cpu_test3, cpu_test1, 0, cpu_test3, &cpu);
 	if (ret) return ret;
 
-	ret = check_cpu(cpu_test3, cpu_test1, cpu_test3, &cpu);
+	ret = check_cpu(cpu_test3, cpu_test1, 1, cpu_test1, &cpu);
 	if (ret) return ret;
 
 	return 0;