Merge from trunk

configure.ac

@@ -1947,6 +1947,10 @@ if test x$use_fxt = xyes; then
 	##########################################
 	PKG_CHECK_MODULES([POTI], [poti], [
 		AC_DEFINE(STARPU_HAVE_POTI, [1], [Define to 1 if you have libpoti])
+		save_LIBS="$LIBS"
+		LIBS="$LIBS $POTI_LIBS"
+		AC_CHECK_FUNCS([poti_init_custom])
+		LIBS="$save_LIBS"
 	], [:])
 	FXT_CFLAGS="$FXT_CFLAGS $POTI_CFLAGS"
 	FXT_LIBS="$FXT_LIBS $POTI_LIBS"

mpi/tests/load_balancer.c

@@ -29,7 +29,7 @@ int main(int argc, char **argv)
 
 void get_neighbors(int **neighbor_ids, int *nneighbors)
 {
-	int ret, rank, size;
+	int rank, size;
 	starpu_mpi_comm_rank(MPI_COMM_WORLD, &rank);
 	starpu_mpi_comm_size(MPI_COMM_WORLD, &size);
 	*nneighbors = 1;

src/datawizard/memalloc.c

@@ -820,10 +820,11 @@ static size_t flush_memchunk_cache(unsigned node, size_t reclaim)
 
 	size_t freed = 0;
 
+restart:
 	_starpu_spin_lock(&mc_lock[node]);
 	HASH_ITER(hh, mc_cache[node], entry, tmp)
 	{
-		while (!_starpu_mem_chunk_list_empty(&entry->list))
+		if (!_starpu_mem_chunk_list_empty(&entry->list))
 		{
 			mc = _starpu_mem_chunk_list_pop_front(&entry->list);
 			STARPU_ASSERT(!mc->data);
@@ -833,19 +834,23 @@ static size_t flush_memchunk_cache(unsigned node, size_t reclaim)
 			STARPU_ASSERT(mc_cache_nb[node] >= 0);
 			mc_cache_size[node] -= mc->size;
 			STARPU_ASSERT(mc_cache_size[node] >= 0);
+			_starpu_spin_unlock(&mc_lock[node]);
+
 			freed += free_memory_on_node(mc, node);
 
 			free(mc->chunk_interface);
 			_starpu_mem_chunk_delete(mc);
 
 			if (reclaim && freed >= reclaim)
-				break;
+				goto out;
+			goto restart;
 		}
 
 		if (reclaim && freed >= reclaim)
 			break;
 	}
 	_starpu_spin_unlock(&mc_lock[node]);
+out:
 	return freed;
 }
 

src/debug/traces/starpu_fxt.c

@@ -3645,8 +3645,18 @@ void _starpu_fxt_paje_file_init(struct starpu_fxt_options *options)
 		}
 
 #ifdef STARPU_HAVE_POTI
+#ifdef HAVE_POTI_INIT_CUSTOM
+		fclose(out_paje_file);
+		poti_init_custom (options->out_paje_path,
+				  0, //if false, allow extended events
+				  1, //if true, an old header (pj_dump -n)
+				  0, //if false, the trace has no comments
+				  1, //if true, events have aliases
+				  1);//if true, relative timestamps
+#else
 		poti_init (out_paje_file);
 #endif
+#endif
 		_starpu_fxt_write_paje_header(out_paje_file);
 	}
 	else

src/debug/traces/starpu_paje.c

@@ -26,7 +26,11 @@ void _starpu_fxt_write_paje_header(FILE *file STARPU_ATTRIBUTE_UNUSED)
 {
 	unsigned i;
 #ifdef STARPU_HAVE_POTI
-	poti_header(1, 1); /* 1 as parameter means basic, no extended events */
+#ifdef HAVE_POTI_INIT_CUSTOM
+	poti_header();     /* see poti_init_custom to customize the header */
+#else
+	poti_header(1,1);
+#endif
 #else
 	fprintf(file, "%%EventDef	PajeDefineContainerType	1\n");
 	fprintf(file, "%%	Alias	string\n");
@@ -135,8 +139,6 @@ void _starpu_fxt_write_paje_header(FILE *file STARPU_ATTRIBUTE_UNUSED)
 	fprintf(file, "%%	Container	string\n");
 	fprintf(file, "%%	Type	string\n");
 	fprintf(file, "%%	Value	string\n");
-#if 0
-	/* TODO: implement in worker_set_detailed_state() and handle_codelet_details() */
 	fprintf(file, "%%	Size	string\n");
 	fprintf(file, "%%	Params	string\n");
 	fprintf(file, "%%	Footprint	string\n");
@@ -148,7 +150,6 @@ void _starpu_fxt_write_paje_header(FILE *file STARPU_ATTRIBUTE_UNUSED)
 	fprintf(file, "%%	Z	string\n");
 	fprintf(file, "%%	Iteration	string\n");
 	fprintf(file, "%%	Subiteration	string\n");
-#endif
 	fprintf(file, "%%EndEventDef\n");
 #endif
 
@@ -282,11 +283,11 @@ void _starpu_fxt_write_paje_header(FILE *file STARPU_ATTRIBUTE_UNUSED)
 	poti_DefineVariableType("gft", "Sc", "Total GFlops", "0 0 0");
 
 	/* Link types */
-	poti_DefineLinkType("MPIL", "MPIP", "MPICt", "MPICt", "MPI communications");
+	poti_DefineLinkType("MPIL", "MPIP", "MPICt", "MPICt", "MPI communication");
 	poti_DefineLinkType("F", "P", "Mm", "Mm", "Intra-node data Fetch");
 	poti_DefineLinkType("PF", "P", "Mm", "Mm", "Intra-node data PreFetch");
 	poti_DefineLinkType("IF", "P", "Mm", "Mm", "Intra-node data IdleFetch");
-	poti_DefineLinkType("WSL", "P", "W", "W", "Work steals");
+	poti_DefineLinkType("WSL", "P", "W", "W", "Work steal");
 
 	/* Creating the MPI Program */
 	poti_CreateContainer(0, "MPIroot", "MPIP", "0", "root");
@@ -411,11 +412,11 @@ void _starpu_fxt_write_paje_header(FILE *file STARPU_ATTRIBUTE_UNUSED)
 6       Co       MS     DriverCopy         \".3 .5 .1\"		\n\
 6       CoA      MS     DriverCopyAsync         \".1 .3 .1\"		\n\
 6       No       MS     Nothing         \".0 .0 .0\"		\n\
-5       MPIL     MPIP	MPICt	MPICt   MPIL			\n\
-5       F       P	Mm	Mm      F\n\
-5       PF      P	Mm	Mm      PF\n\
-5       IF      P	Mm	Mm      IF\n\
-5       WSL     P	W	W       WSL\n");
+5       MPIL     MPIP	MPICt	MPICt   \"MPI communication\"\n\
+5       F       P	Mm	Mm      \"Intra-node data Fetch\"\n\
+5       PF      P	Mm	Mm      \"Intra-node data PreFetch\"\n\
+5       IF      P	Mm	Mm      \"Intra-node data IdleFetch\"\n\
+5       WSL     P	W	W       \"Work steal\"\n");
 
 	fprintf(file, "7      0.0 MPIroot      MPIP      0       root\n");
 #endif

src/drivers/scc/driver_scc_common.c

@@ -16,6 +16,7 @@
 
 #include <sys/mman.h>
 #include <fcntl.h>
+#include <stdint.h>
 
 #include <core/workers.h>
 
@@ -119,7 +120,7 @@ void _starpu_scc_common_unmap_shared_memory()
 /* To know if the pointer "ptr" points into the shared memory map */
 int _starpu_scc_common_is_in_shared_memory(void *ptr)
 {
-	return (void*)shm_addr <= ptr && ptr < (void*)shm_addr + SHMSIZE;
+	return (void*)shm_addr <= ptr && ptr < (void*)((uintptr_t)shm_addr + SHMSIZE);
 }
 
 int _starpu_scc_common_is_mp_initialized()

src/drivers/scc/driver_scc_sink.c

@@ -17,6 +17,7 @@
 
 #include <RCCE.h>
 #include <dlfcn.h>
+#include <stdint.h>
 
 #include <datawizard/interfaces/data_interface.h>
 #include <drivers/mp_common/sink_common.h>
@@ -97,10 +98,10 @@ void _starpu_scc_sink_execute(const struct _starpu_mp_node *node, void *arg, int
 	void *local_arg = arg;
 
 	/* point after the kernel */
-	local_arg += sizeof(void(*)(void**, void*));
+	local_arg = (void*) ((uintptr_t)local_arg + sizeof(void(*)(void**, void*)));
 
 	unsigned nb_interfaces = *(unsigned*)local_arg;
-	local_arg += sizeof(nb_interfaces);
+	local_arg = (void*) ((uintptr_t)local_arg + sizeof(nb_interfaces));
 
 	uintptr_t shm_addr = (uintptr_t)_starpu_scc_common_get_shared_memory_addr();
 
@@ -152,7 +153,7 @@ void _starpu_scc_sink_execute(const struct _starpu_mp_node *node, void *arg, int
 		}
 
 		/* point to the next interface */
-		local_arg += sizeof(union _starpu_interface);
+		local_arg = (void*)((uintptr_t)local_arg + sizeof(union _starpu_interface));
 	}
 
 	_starpu_sink_common_execute(node, arg, arg_size);

tests/datawizard/allocate.c

@@ -1,6 +1,6 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2013  CNRS
+ * Copyright (C) 2013, 2017  CNRS
  *
  * StarPU is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
@@ -36,7 +36,6 @@ int main(int argc, char **argv)
 static
 int test_prefetch(unsigned memnodes)
 {
-	int ret;
 	float *buffers[4];
 	starpu_data_handle_t handles[4];
 	unsigned i;
@@ -138,7 +137,6 @@ void test_malloc()
 	float *buffer;
 	float *buffer2;
 	float *buffer3;
-	size_t global_size;
 
 	/* Allocate one byte */
 	ret = starpu_malloc_flags((void **)&buffer, 1, STARPU_MALLOC_COUNT);

tests/datawizard/noreclaim.c

@@ -74,7 +74,7 @@ static void emit_empty_task(void)
 
 int main(int argc, char **argv)
 {
-	int i, ret;
+	int ret;
 	struct starpu_conf conf;
 	starpu_data_handle_t handle;
 	void *allocated;

tests/datawizard/reclaim.c

@@ -129,14 +129,15 @@ int main(int argc, char **argv)
 	/* Register mb buffers of 1MB */
 	for (j = 0; j < mb; j++)
 	{
-		host_ptr_array[j] = calloc(BLOCK_SIZE, 1);
+		size_t size = random()%BLOCK_SIZE + 1;
+		host_ptr_array[j] = calloc(size, 1);
 		if (host_ptr_array[j] == NULL)
 		{
 			mb = j;
 			FPRINTF(stderr, "Cannot allocate more than %u buffers\n", mb);
 			break;
 		}
-		starpu_variable_data_register(&handle_array[j], STARPU_MAIN_RAM, (uintptr_t)host_ptr_array[j], BLOCK_SIZE);
+		starpu_variable_data_register(&handle_array[j], STARPU_MAIN_RAM, (uintptr_t)host_ptr_array[j], size);
 		STARPU_ASSERT(handle_array[j]);
 	}
 
@@ -155,12 +156,19 @@ int main(int argc, char **argv)
 		STARPU_CHECK_RETURN_VALUE(ret, "starpu_task_submit");
 	}
 
+	for (j = 0; j < mb; j++)
+	{
+		if ( j%20 == 0 )
+			starpu_data_unregister_submit(handle_array[j]);
+	}
+
 	ret = starpu_task_wait_for_all();
 	STARPU_CHECK_RETURN_VALUE(ret, "starpu_task_wait_for_all");
 
 	for (j = 0; j < mb; j++)
 	{
-		starpu_data_unregister(handle_array[j]);
+		if ( j%20 != 0 )
+			starpu_data_unregister(handle_array[j]);
 		free(host_ptr_array[j]);
 	}
 

tools/Makefile.am

@@ -134,37 +134,38 @@ dist_pkgdata_perfmodels_sampling_codelets_DATA = \
 	perfmodels/sampling/codelets/44/starpu_dlu_lu_model_22.sirocco
 
 EXTRA_DIST =				\
-	dev/rename.sed			\
-	dev/rename.sh			\
+	dev/checker/rename.sed		\
+	dev/checker/rename.sh		\
+	dev/cppcheck/suppressions.txt	\
+	dev/valgrind/fxt.suppr		\
+	dev/valgrind/hwloc.suppr	\
+	dev/valgrind/libc.suppr		\
+	dev/valgrind/libgomp.suppr	\
+	dev/valgrind/libnuma.suppr	\
+	dev/valgrind/madmpi.suppr	\
+	dev/valgrind/opencl.suppr	\
+	dev/valgrind/openmpi.suppr	\
+	dev/valgrind/openmp.suppr	\
+	dev/valgrind/padico.suppr	\
+	dev/valgrind/pthread.suppr	\
+	dev/valgrind/starpu.suppr	\
+	dev/valgrind/valgrind.suppr	\
+	dev/valgrind/valgrind.sh	\
+	dev/valgrind/helgrind.sh	\
+	dev/tsan/starpu.suppr		\
+	dev/lsan/libc.suppr		\
+	dev/lsan/openmpi.suppr		\
+	dev/sonarqube/sonar-scanner.sh	\
 	perfmodels/README		\
 	perfmodels/cluster.xml		\
 	perfmodels/hostfile		\
 	perfmodels/sampling/codelets/tmp/mlr_init.out	 \
-	valgrind/fxt.suppr		\
-	valgrind/hwloc.suppr		\
-	valgrind/libc.suppr		\
-	valgrind/libgomp.suppr		\
-	valgrind/libnuma.suppr		\
-	valgrind/madmpi.suppr		\
-	valgrind/opencl.suppr		\
-	valgrind/openmpi.suppr		\
-	valgrind/openmp.suppr		\
-	valgrind/padico.suppr		\
-	valgrind/pthread.suppr		\
-	valgrind/starpu.suppr		\
-	valgrind/valgrind.suppr		\
-	valgrind/valgrind.sh		\
-	valgrind/helgrind.sh		\
-	tsan/starpu.suppr		\
-	lsan/libc.suppr			\
-	lsan/openmpi.suppr		\
 	msvc/starpu_clean.bat		\
 	msvc/starpu_open.bat		\
 	msvc/starpu_exec.bat		\
 	msvc/starpu_var.bat		\
 	msvc/starpu.sln			\
-	msvc/starpu/starpu.vcxproj	\
-	cppcheck/suppressions.txt
+	msvc/starpu/starpu.vcxproj
 
 CLEANFILES = *.gcno *.gcda *.linkinfo starpu_idle_microsec.log figure/* mlr_*
 

tools/dev/checker/starpu_check_copyright.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+#
+# StarPU --- Runtime system for heterogeneous multicore architectures.
+#
+# Copyright (C) 2013 Centre National de la Recherche Scientifique
+#
+# StarPU is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 2.1 of the License, or (at
+# your option) any later version.
+#
+# StarPU is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See the GNU Lesser General Public License in COPYING.LGPL for more details.
+
+REP=${1:-.}
+
+for f in $(find $REP -not -path "*svn*" -not -path "*build*" -not -path "*starpu-top*"  -not -path "*min-dgels*" -not -name ".gitignore"  -not -name "*.doxy"  -not -name "*.eps"  -not -name "*.pdf" -not -name "*.png" -type f)
+do
+#    copyright=$(grep "StarPU is free software" $f 2>/dev/null)
+#    if test -z "$copyright"
+#    then
+#	echo "File $f does not include a proper copyright"
+#    fi
+
+    svn log $f | grep '|' | awk -F'|' '{print $2}' | sort | uniq
+done

tools/dev/checker/starpu_check_macros.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+#
+# StarPU --- Runtime system for heterogeneous multicore architectures.
+#
+# Copyright (C) 2016, 2017 CNRS
+#
+# StarPU is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 2.1 of the License, or (at
+# your option) any later version.
+#
+# StarPU is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See the GNU Lesser General Public License in COPYING.LGPL for more details.
+
+dirname=$(dirname $0)
+
+INC_DIRS=$(find $dirname/../../ -name include -type d)
+STARPU_H_FILES=$(find $INC_DIRS -name '*.h')
+STARPU_CONFIG=$dirname/../../include/starpu_config.h.in
+STARPU_CONFIG_DISPLAY=$(python -c "import os.path; print os.path.relpath('$STARPU_CONFIG', '.')")
+macros1=$(grep 'ifdef' $STARPU_H_FILES|grep STARPU|awk '{print $NF}')
+macros2=$(grep defined $STARPU_H_FILES | tr ' (' '\012' | grep STARPU | sed 's/defined//' | tr -d '()!,')
+macros=$(echo $macros1 $macros2 | tr ' ' '\012' | sort | uniq)
+
+for m in $macros
+do
+    count=$(grep -c $m $STARPU_CONFIG)
+    if test $count -eq 0
+    then
+	echo $m missing in $STARPU_CONFIG_DISPLAY
+    fi
+done
+
+

tools/dev/sonarqube/sonar-scanner.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+export CPPCHECK_INCLUDES="-Iinclude -Impi/include"
+export SOURCES_TO_ANALYZE="include src mpi/src tools tests examples mpi/tests mpi/examples"
+export SONAR_SOURCES=$(echo $SOURCES_TO_ANALYZE | tr ' ' ',')
+export SOURCES_TO_EXCLUDE="-itools/dev"
+
+if test -z "$SONAR_LOGIN_FILE"
+then
+    echo "Error. Environment variable SONAR_LOGIN_FILE missing"
+    exit 1
+fi
+
+SONAR_PROJECT=storm:starpu_trunk
+SONAR_LOGIN=$(grep ${SONAR_PROJECT} ${SONAR_LOGIN_FILE} | awk '{print $2}')
+if test -z "$SONAR_LOGIN"
+then
+    echo "Error. Project ${SONAR_PROJECT} not available in file ${SONAR_LOGIN_FILE}"
+    exit 1
+fi
+
+if test -f build/config.log
+then
+    CONFIG_LOG=build/config.log
+elif test -f config.log
+then
+    CONFIG_LOG=config.log
+else
+    echo Error no config.log file found
+    exit 1
+fi
+
+export DEFINITIONS_LOG=$(grep "^#define" ${CONFIG_LOG} | sed -e "s#\#define #-D#g" | sed -e "s# #=#g" | xargs)
+export DEFINITIONS_SRC=$(grep "^#undef" src/common/config.h.in| sed -e "s#\#undef #-D#g" | sed -e "s#\$#=1#g" | xargs)
+export DEFINITIONS_LOCAL=$(grep -rs "#ifdef" src/ mpi/src |awk -F':' '{print $2}' | awk '{print "-D"$2"=1"}' |sort|uniq)
+export DEFINITIONS="${DEFINITIONS_LOG} ${DEFINITIONS_SRC} ${DEFINITIONS_LOCAL}"
+
+cppcheck --max-configs=1000 --language=c++ --platform=unix64 --force -v --enable=all --inline-suppr --xml --xml-version=2 ${DEFINITIONS} ${CPPCHECK_INCLUDES} ${SOURCES_TO_EXCLUDE} ${SOURCES_TO_ANALYZE} 2> cppcheck.xml
+#--suppressions-list=tools/cppcheck/suppressions.txt
+
+sonar-scanner -Dsonar.login=${SONAR_LOGIN} -Dsonar.projectKey=${SONAR_PROJECT} -Dsonar.projectName=StarPU_trunk -Dsonar.projectVersion=HEAD -Dsonar.sources=${SONAR_SOURCES} -Dsonar.cxx.cppcheck.reportPath=cppcheck.xml