Merge branch 'master' of git+ssh://scm.gforge.inria.fr/gitroot/starpu/starpu into ft_checkpoint

ChangeLog

@@ -53,6 +53,7 @@ Small features:
   * Add STARPU_LIMIT_CPU_NUMA_MEM environment variable.
   * Add STARPU_WORKERS_GETBIND environment variable.
   * Add STARPU_SCHED_SIMPLE_DECIDE_ALWAYS modular scheduler flag.
+  * And STARPU_LIMIT_BANDWIDTH environment variable.
 
 StarPU 1.3.3 (git revision 11afc5b007fe1ab1c729b55b47a5a98ef7f3cfad)
 ====================================================================

configure.ac

@@ -191,7 +191,7 @@ if test x$enable_simgrid = xyes ; then
 
 	# Latest functions
 	AC_CHECK_FUNCS([MSG_process_attach sg_actor_attach sg_actor_init sg_actor_set_stacksize MSG_zone_get_hosts sg_zone_get_hosts MSG_process_self_name MSG_process_userdata_init sg_actor_data])
-	AC_CHECK_FUNCS([xbt_mutex_try_acquire smpi_process_set_user_data SMPI_thread_create sg_zone_get_by_name sg_link_name sg_host_route sg_host_self sg_host_list sg_host_speed simcall_process_create sg_config_continue_after_help])
+	AC_CHECK_FUNCS([xbt_mutex_try_acquire smpi_process_set_user_data SMPI_thread_create sg_zone_get_by_name sg_link_name sg_link_bandwidth_set sg_host_route sg_host_self sg_host_list sg_host_speed simcall_process_create sg_config_continue_after_help])
 	AC_CHECK_FUNCS([simgrid_init], [AC_DEFINE([STARPU_SIMGRID_HAVE_SIMGRID_INIT], [1], [Define to 1 if you have the `simgrid_init' function.])])
 	AC_CHECK_FUNCS([xbt_barrier_init], [AC_DEFINE([STARPU_SIMGRID_HAVE_XBT_BARRIER_INIT], [1], [Define to 1 if you have the `xbt_barrier_init' function.])])
 	AC_CHECK_FUNCS([sg_actor_sleep_for sg_actor_self sg_actor_ref sg_host_get_properties sg_host_send_to sg_host_sendto sg_cfg_set_int sg_actor_self_execute sg_actor_execute simgrid_get_clock])

doc/doxygen/chapters/501_environment_variables.doxy

@@ -986,6 +986,23 @@ NUMA nodes used by StarPU. Any \ref STARPU_LIMIT_CPU_NUMA_devid_MEM additionally
 specified will take over STARPU_LIMIT_CPU_NUMA_MEM.
 </dd>
 
+<dt>STARPU_LIMIT_BANDWIDTH</dt>
+<dd>
+\anchor STARPU_LIMIT_BANDWIDTH
+\addindex __env__STARPU_LIMIT_BANDWIDTH
+Specify the maximum available PCI bandwidth of the system in MB/s. This can only
+be effective with simgrid simulation. This allows to easily override the
+bandwidths stored in the platform file generated from measurements on the native
+system. This can be used e.g. for convenient
+
+Specify the maximum number of megabytes that should be available to the
+application on each NUMA node. This is the same as specifying that same amount
+with \ref STARPU_LIMIT_CPU_NUMA_devid_MEM for each NUMA node number. The total
+memory available to StarPU will thus be this amount multiplied by the number of
+NUMA nodes used by StarPU. Any \ref STARPU_LIMIT_CPU_NUMA_devid_MEM additionally
+specified will take over STARPU_LIMIT_BANDWIDTH.
+</dd>
+
 <dt>STARPU_MINIMUM_AVAILABLE_MEM</dt>
 <dd>
 \anchor STARPU_MINIMUM_AVAILABLE_MEM

julia/examples/mandelbrot/Makefile

@@ -26,7 +26,7 @@ LIBPATH=${PWD}/../StarPU.jl/lib
 all: ${EXTERNLIB}
 
 mandelbrot: mandelbrot.c cpu_mandelbrot.o #gpu_mandelbrot.o
-	$(CC) $(CPU_CFLAGS) $^ -o $@ $(LDFLAGS)
+	$(CC) $(CPU_CFLAGS) $^ -o $@ $(LDFLAGS) -lm
 
 %.o: %.c
 	$(CC) -c -fPIC $(CPU_CFLAGS) $^ -o $@

julia/examples/task_insert_color/Makefile

@@ -0,0 +1,51 @@
+CC=gcc
+NVCC=nvcc
+ENABLE_CUDA=no
+LD=$(CC)
+
+ifeq ($(ENABLE_CUDA),yes)
+        LD := ${NVCC}
+endif
+
+CFLAGS = -O3 -g $(shell pkg-config --cflags starpu-1.3)
+CPU_CFLAGS = ${CFLAGS} -Wall -mavx -fomit-frame-pointer -march=native -ffast-math
+CUDA_CFLAGS = ${CFLAGS}
+LDFLAGS +=$(shell pkg-config --libs starpu-1.3)
+
+EXTERNLIB=extern_tasks.so
+GENERATEDLIB=generated_tasks.so
+
+C_OBJECTS=$(patsubst %.c,%.o,$(wildcard gen*.c))
+CUDA_OBJECTS=$(patsubst %.cu,%.o,$(wildcard gen*.cu))
+ifneq ($(ENABLE_CUDA),yes)
+	CUDA_OBJECTS:=
+endif
+
+LIBPATH=${PWD}/../StarPU.jl/lib
+
+all: task_insert_color
+
+task_insert_color: task_insert_color.o
+	$(CC) $(CPU_CFLAGS) $^ -o $@ $(LDFLAGS)
+
+%.o: %.c
+	$(CC) -c -fPIC $(CPU_CFLAGS) $^ -o $@
+
+%.o: %.cu
+	$(NVCC) -dc $(CUDA_CFLAGS) $^ --shared --compiler-options '-fPIC' -o $@ $(LDFLAGS)
+
+${GENERATEDLIB}: $(C_OBJECTS) $(CUDA_OBJECTS)
+	$(LD) -shared $(LDFLAGS) $^ -o $@
+
+PHONY: clean
+
+clean:
+	rm -f vector_scal *.so *.o genc_*.c gencuda_*.cu *.dat
+
+# Performance Tests
+cstarpu.dat: task_insert_color
+	STARPU_NOPENCL=0 STARPU_SCHED=dmda STARPU_CALIBRATE=1 ./task_insert_color > $@
+julia_generatedc.dat:
+	LD_LIBRARY_PATH+=${LIBPATH} STARPU_NOPENCL=0 STARPU_SCHED=dmda STARPU_CALIBRATE=1 julia task_insert_colorl.jl
+
+test: cstarpu.dat julia_generatedc.dat

julia/examples/task_insert_color/task_insert_color.c

@@ -0,0 +1,89 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2018-2020  Université de Bordeaux, CNRS (LaBRI UMR 5800), Inria
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <starpu.h>
+
+#define FPRINTF(ofile, fmt, ...) do { if (!getenv("STARPU_SSILENT")) {fprintf(ofile, fmt, ## __VA_ARGS__); }} while(0)
+
+void func(void *descr[], void *_args)
+{
+	int *x = (int *)STARPU_VARIABLE_GET_PTR(descr[0]);
+	(void)_args;
+
+	*x *= 2;
+}
+
+struct starpu_codelet mycodelet =
+{
+	.modes = { STARPU_RW },
+	.cpu_funcs = {func},
+	.cpu_funcs_name = {"func"},
+        .nbuffers = 1
+};
+
+struct starpu_codelet mycodelet_color =
+{
+	.modes = { STARPU_RW },
+	.cpu_funcs = {func},
+	.cpu_funcs_name = {"func"},
+        .nbuffers = 1,
+	.color = 0x0000FF,
+};
+
+int main(void)
+{
+	unsigned i;
+	int value=42;
+	starpu_data_handle_t handle;
+	int ret;
+
+	ret = starpu_init(NULL);
+	if (ret == -ENODEV) goto enodev;
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_init");
+
+	starpu_variable_data_register(&handle, STARPU_MAIN_RAM, (uintptr_t)&value, sizeof(value));
+
+	// In the trace file, the following task should be green (executed on CPU)
+	ret = starpu_task_insert(&mycodelet, STARPU_RW, handle, STARPU_NAME, "mytask",
+				 0);
+	if (STARPU_UNLIKELY(ret == -ENODEV))
+	{
+		starpu_data_unregister(handle);
+		goto enodev;
+	}
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_task_insert");
+
+	// In the trace file, the following task will be red as specified by STARPU_TASK_COLOR
+	ret = starpu_task_insert(&mycodelet, STARPU_RW, handle, STARPU_NAME, "mytask",
+				 STARPU_TASK_COLOR, 0xFF0000,
+				 0);
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_task_insert");
+
+	// In the trace file, the following task will be blue as specified by the field color of mycodelet_color
+	ret = starpu_task_insert(&mycodelet_color, STARPU_RW, handle, STARPU_NAME, "mytask",
+				 0);
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_task_insert");
+
+	starpu_task_wait_for_all();
+	starpu_data_unregister(handle);
+
+	starpu_shutdown();
+
+	return 0;
+
+ enodev:
+	return 77;
+}

julia/examples/task_insert_color/task_insert_color.jl

@@ -0,0 +1,48 @@
+import Libdl
+using StarPU
+
+@target STARPU_CPU
+@codelet function task_insert_color(val ::Ref{Int32}) :: Nothing
+    val[] = val[] * 2
+
+    return
+end
+
+starpu_init()
+
+function task_insert_color_with_starpu(val ::Ref{Int32})
+    @starpu_block let
+	hVal = starpu_data_register(val)
+
+        cl1 = StarpuCodelet(
+            cpu_func = CPU_CODELETS["task_insert_color"],
+            modes = [STARPU_RW]
+        )
+
+        cl2 = StarpuCodelet(
+            cpu_func = CPU_CODELETS["task_insert_color"],
+            modes = [STARPU_RW],
+            color = 0x0000FF
+        )
+
+	@starpu_sync_tasks begin
+
+            # In the trace file, the following task should be green (executed on CPU)
+            starpu_task_submit(StarpuTask(cl = cl1, handles = [hVal]))
+
+            # In the trace file, the following task will be blue as specified by the field color of cl2
+            starpu_task_submit(StarpuTask(cl = cl2, handles = [hVal]))
+
+            # In the trace file, the following tasks will be red as specified in @starpu_async_cl
+            @starpu_async_cl task_insert_color(hVal) [STARPU_RW] [] 0xFF0000
+
+	end
+    end
+end
+
+
+foo = Ref(convert(Int32, 42))
+
+task_insert_color_with_starpu(foo)
+
+starpu_shutdown()

julia/examples/vector_scal/vector_scal.jl

@@ -55,11 +55,18 @@ end
 function compute_times(io,start_dim, step_dim, stop_dim)
     for size in (start_dim : step_dim : stop_dim)
         V = Array(rand(Cfloat, size))
+        starpu_memory_pin(V)
+
         m :: Int32 = 10
         k :: Float32 = 2.
         l :: Float32 = 3.
+
         println("INPUT ", V[1:10])
+
         mt =  vector_scal_with_starpu(V, m, k, l)
+
+        starpu_memory_unpin(V)
+
         println("OUTPUT ", V[1:10])
         println(io,"$size $mt")
         println("$size $mt")

julia/src/StarPU.jl

@@ -49,6 +49,8 @@ end
 
 export starpu_init
 export starpu_shutdown
+export starpu_memory_pin
+export starpu_memory_unpin
 export starpu_data_unregister
 export starpu_data_register
 export starpu_data_get_sub_data
@@ -185,6 +187,8 @@ end
 struct StarpuCodelet
     where_to_execute :: UInt32
 
+    color :: UInt32
+
     cpu_func :: String
     cuda_func :: String
     opencl_func :: String
@@ -201,7 +205,8 @@ struct StarpuCodelet
                            opencl_func :: String = "",
                            modes :: Vector{StarpuDataAccessMode} = StarpuDataAccessMode[],
                            perfmodel :: StarpuPerfmodel = StarpuPerfmodel(),
-                           where_to_execute :: Union{Cvoid, UInt32} = nothing
+                           where_to_execute :: Union{Cvoid, UInt32} = nothing,
+                           color :: UInt32 = 0x00000000
                            )
 
         if (length(modes) > STARPU_NMAXBUFS)
@@ -217,7 +222,7 @@ struct StarpuCodelet
             real_where = where_to_execute
         end
 
-        output = new(real_where, cpu_func, cuda_func, opencl_func,modes, perfmodel, real_c_codelet_ptr)
+        output = new(real_where, color, cpu_func, cuda_func, opencl_func,modes, perfmodel, real_c_codelet_ptr)
 
         starpu_c_codelet_update(output)
 
@@ -651,9 +656,23 @@ end
 
 STARPU_MAIN_RAM = 0 #TODO: ENUM
 
+function starpu_memory_pin(data) :: Nothing
+    data_pointer = pointer(data)
 
+    @starpucall(starpu_memory_pin,
+                Cvoid, (Ptr{Cvoid}, Csize_t),
+                data_pointer,
+                sizeof(data))
+end
 
+function starpu_memory_unpin(data) :: Nothing
+    data_pointer = pointer(data)
 
+    @starpucall(starpu_memory_unpin,
+                Cvoid, (Ptr{Cvoid}, Csize_t),
+                data_pointer,
+                sizeof(data))
+end
 
 function StarpuNewDataHandle(ptr :: StarpuDataHandlePointer, destr :: Function...) :: StarpuDataHandle
     return StarpuDestructible(ptr, destr...)
@@ -858,7 +877,7 @@ end
     Creates and submits an asynchronous task running cl Codelet function.
     Ex : @starpu_async_cl cl(handle1, handle2)
 """
-macro starpu_async_cl(expr,modes,cl_arg=[])
+macro starpu_async_cl(expr, modes, cl_arg=[], color ::UInt32=0x00000000)
 
     if (!isa(expr, Expr) || expr.head != :call)
         error("Invalid task submit syntax")
@@ -877,7 +896,8 @@ macro starpu_async_cl(expr,modes,cl_arg=[])
         #opencl_func="ocl_matrix_mult",
         ### TODO: CORRECT !
         modes = map((x -> starpu_modes(x)),modes.args),
-        perfmodel = perfmodel
+        perfmodel = perfmodel,
+        color = color
     )
     handles = Expr(:vect, expr.args[2:end]...)
     #dump(handles)
@@ -1214,6 +1234,8 @@ mutable struct StarpuCodeletTranslator
 
     where_to_execute :: UInt32
 
+    color :: UInt32
+
     cpu_func :: Ptr{Cvoid}
     cpu_func_name :: Cstring
 
@@ -1237,6 +1259,7 @@ mutable struct StarpuCodeletTranslator
         end
 
         output.where_to_execute = cl.where_to_execute
+        output.color = cl.color
 
         cpu_func_ptr = load_starpu_function_pointer(cl.cpu_func)
         cuda_func_ptr = load_starpu_function_pointer(cl.cuda_func)

julia/src/compiler/expressions.jl

@@ -422,6 +422,10 @@ end
 function print(io :: IO, x :: StarpuExprFor ; indent = 0,restrict=false)
 
     print_newline(io, indent)
+    print(io, "{")
+    indent += starpu_indent_size
+    print_newline(io, indent)
+
     print(io, StarpuExprBlock(x.set_declarations), indent = indent)
 
     id = x.set.id
@@ -449,12 +453,20 @@ function print(io :: IO, x :: StarpuExprFor ; indent = 0,restrict=false)
 
     print_newline(io, indent)
     print(io, "{")
-    print_newline(io, indent + starpu_indent_size)
-    print(io, x.body, indent = indent + starpu_indent_size)
+    indent += starpu_indent_size
+
+    print_newline(io, indent)
+    print(io, x.body, indent = indent)
+
+    indent -= starpu_indent_size
     print_newline(io, indent)
     print(io, "}")
+
+    indent -= starpu_indent_size
     print_newline(io, indent)
+    print(io, "}")
 
+    print_newline(io, indent)
 end
 
 

julia/src/jlstarpu_task.h

@@ -30,6 +30,8 @@ struct jlstarpu_codelet
 {
 	uint32_t where;
 
+  	uint32_t color;
+
 	starpu_cpu_func_t cpu_func;
 	char * cpu_func_name;
 

julia/src/jlstarpu_task_submit.c

@@ -63,6 +63,8 @@ void jlstarpu_codelet_update(const struct jlstarpu_codelet * const input, struct
 {
 	output->where = input->where;
 
+	output->color = input->color;
+
 	output->cpu_funcs[0] = input->cpu_func;
 	output->cpu_funcs_name[0] = input->cpu_func_name;
 

mpi/src/mpi/starpu_mpi_mpi.c

@@ -51,7 +51,10 @@ static unsigned nready_process;
 /* Number of send requests to submit to MPI at the same time */
 static unsigned ndetached_send;
 
+#ifdef STARPU_USE_FXT
 static void _starpu_mpi_add_sync_point_in_fxt(void);
+#endif
+
 static void _starpu_mpi_handle_ready_request(struct _starpu_mpi_req *req);
 static void _starpu_mpi_handle_request_termination(struct _starpu_mpi_req *req);
 #ifdef STARPU_MPI_VERBOSE
@@ -1173,6 +1176,8 @@ static void *_starpu_mpi_progress_thread_func(void *arg)
 
 	starpu_pthread_setname("MPI");
 
+	_starpu_mpi_env_init();
+
 #ifndef STARPU_SIMGRID
 	if (_starpu_mpi_thread_cpuid < 0)
 	{
@@ -1189,11 +1194,7 @@ static void *_starpu_mpi_progress_thread_func(void *arg)
 	if (_starpu_mpi_thread_cpuid >= 0)
 		/* In case MPI changed the binding */
 		starpu_bind_thread_on(_starpu_mpi_thread_cpuid, STARPU_THREAD_ACTIVE, "MPI");
-#endif
-
-	_starpu_mpi_env_init();
-
-#ifdef STARPU_SIMGRID
+#else
 	/* Now that MPI is set up, let the rest of simgrid get initialized */
 	char **argv_cpy;
 	_STARPU_MPI_MALLOC(argv_cpy, *(argc_argv->argc) * sizeof(char*));
@@ -1532,9 +1533,9 @@ static void *_starpu_mpi_progress_thread_func(void *arg)
 	return NULL;
 }
 
+#ifdef STARPU_USE_FXT
 static void _starpu_mpi_add_sync_point_in_fxt(void)
 {
-#ifdef STARPU_USE_FXT
 	int rank;
 	int worldsize;
 	int ret;
@@ -1563,8 +1564,8 @@ static void _starpu_mpi_add_sync_point_in_fxt(void)
 	_STARPU_MPI_TRACE_BARRIER(rank, worldsize, random_number);
 
 	_STARPU_MPI_DEBUG(3, "unique key %x\n", random_number);
-#endif
 }
+#endif
 
 int _starpu_mpi_progress_init(struct _starpu_mpi_argc_argv *argc_argv)
 {

mpi/tests/sendrecv_gemm_bench.c

@@ -287,14 +287,15 @@ static void parse_args(int argc, char **argv)
 
 		else if (strcmp(argv[i], "-help") == 0 || strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "-h") == 0)
 		{
-			fprintf(stderr,"Usage: %s [-nblocks n] [-size size] [-check] [-spmd] [-comm_thread_cpuid cpuid]\n", argv[0]);
+			fprintf(stderr,"Usage: %s [-nblocks n] [-size size] [-check] [-spmd] [-comm-thread-cpuid cpuid]\n", argv[0]);
 			fprintf(stderr,"Currently selected: matrix size: %u - %u blocks\n", matrix_dim, nslices);
-			fprintf(stderr, "Use -comm_thread_cpuid to specifiy where to bind the comm benchmarking thread\n");
+			fprintf(stderr, "Use -comm-thread-cpuid to specifiy where to bind the comm benchmarking thread\n");
 			exit(EXIT_SUCCESS);
 		}
+
 		else
 		{
-			fprintf(stderr,"Unrecognized option %s", argv[i]);
+			fprintf(stderr,"Unrecognized option %s\n", argv[i]);
 			exit(EXIT_FAILURE);
 		}
 	}

src/core/perfmodel/perfmodel_bus.c

@@ -80,8 +80,8 @@ struct dev_timing
 };
 
 /* TODO: measure latency */
-static double bandwidth_matrix[STARPU_MAXNODES][STARPU_MAXNODES];
-static double latency_matrix[STARPU_MAXNODES][STARPU_MAXNODES];
+static double bandwidth_matrix[STARPU_MAXNODES][STARPU_MAXNODES]; /* MB/s */
+static double latency_matrix[STARPU_MAXNODES][STARPU_MAXNODES]; /* µs */
 static unsigned was_benchmarked = 0;
 #ifndef STARPU_SIMGRID
 static unsigned ncpus = 0;
@@ -1546,6 +1546,20 @@ static int load_bus_bandwidth_file_content(void)
 				return 0;
 			}
 
+			int limit_bandwidth = starpu_get_env_number("STARPU_LIMIT_BANDWIDTH");
+			if (limit_bandwidth >= 0)
+			{
+#ifndef STARPU_SIMGRID
+				_STARPU_DISP("Warning: STARPU_LIMIT_BANDWIDTH set to %d but simgrid not enabled, thus ignored\n", limit_bandwidth);
+#else
+#ifdef HAVE_SG_LINK_BANDWIDTH_SET
+				bandwidth = limit_bandwidth;
+#else
+				_STARPU_DISP("Warning: STARPU_LIMIT_BANDWIDTH set to %d but this requires simgrid 3.26\n", limit_bandwidth);
+#endif
+#endif
+			}
+
 			bandwidth_matrix[src][dst] = bandwidth;
 
 			/* Look out for \t\n */

src/core/simgrid.c

@@ -337,6 +337,19 @@ void _starpu_start_simgrid(int *argc, char **argv)
 #else
 	MSG_create_environment(path);
 #endif
+	int limit_bandwidth = starpu_get_env_number("STARPU_LIMIT_BANDWIDTH");
+	if (limit_bandwidth >= 0)
+	{
+#ifdef HAVE_SG_LINK_BANDWIDTH_SET
+		sg_link_t *links = sg_link_list();
+		int count = sg_link_count(), i;
+		for (i = 0; i < count; i++) {
+			sg_link_bandwidth_set(links[i], limit_bandwidth * 1000000.);
+		}
+#else
+		_STARPU_DISP("Warning: STARPU_LIMIT_BANDWIDTH set to %d but this requires simgrid 3.26, thus ignored\n", limit_bandwidth);
+#endif
+	}
 
 	simgrid_transfer_cost = starpu_get_env_number_default("STARPU_SIMGRID_TRANSFER_COST", 1);
 }

tools/dev/lsan/suppressions

@@ -29,3 +29,4 @@ leak:hwloc_topology_load
 leak:hwloc_topology_set_xml
 leak:hwloc_components_init
 leak:hwloc_plugins_init
+leak:hwloc_plugins_exit

tools/dev/valgrind/valgrind.sh

@@ -17,6 +17,13 @@
 EXEC=$(basename $0 .sh)
 DIRNAME=$(dirname $0)
 
+CLIMIT=$(ulimit -c)
+if [ "$CLIMIT" = unlimited ]
+then
+	# valgrind cores are often *huge*, 100MB will already be quite big...
+	ulimit -c 100000
+fi
+
 if test "$EXEC" == "valgrind"
 then
     RUN="valgrind --track-origins=yes --show-reachable=yes --leak-check=full --errors-for-leak-kinds=all --show-leak-kinds=all --error-exitcode=42"