Replace use of starpu_xchg/starpu_cmpxchg with proper macros

starpu_xchg/starpu_cmpxchg were only meant to contain assembly snippet, they
are not meant to be called. Recent gcc provides a generic __atomic_exchange
which can be used instead.

configure.ac

@@ -994,6 +994,21 @@ STARPU_CHECK_SYNC_FETCH_AND_OR
 # This defines HAVE_SYNC_LOCK_TEST_AND_SET
 STARPU_CHECK_SYNC_LOCK_TEST_AND_SET
 
+# This defines HAVE_ATOMIC_COMPARE_EXCHANGE_N
+STARPU_CHECK_ATOMIC_COMPARE_EXCHANGE_N
+
+# This defines HAVE_ATOMIC_EXCHANGE_N
+STARPU_CHECK_ATOMIC_EXCHANGE_N
+
+# This defines HAVE_ATOMIC_FETCH_ADD
+STARPU_CHECK_ATOMIC_FETCH_ADD
+
+# This defines HAVE_ATOMIC_FETCH_OR
+STARPU_CHECK_ATOMIC_FETCH_OR
+
+# This defines HAVE_ATOMIC_TEST_AND_SET
+STARPU_CHECK_ATOMIC_TEST_AND_SET
+
 # This defines HAVE_SYNC_SYNCHRONIZE
 STARPU_CHECK_SYNC_SYNCHRONIZE
 

include/starpu_util.h

@@ -322,20 +322,20 @@ extern "C"
 			STARPU_ABORT(); }}
 #endif
 
-/* Note: do not use starpu_cmpxchg / starpu_xchg / starpu_cmpxchgl /
- * starpu_xchgl / starpu_cmpxchg64 / starpu_xchg64, which only
+/* Note: do not use _starpu_cmpxchg / _starpu_xchg / _starpu_cmpxchgl /
+ * _starpu_xchgl / _starpu_cmpxchg64 / _starpu_xchg64, which only
  * assembly-hand-written fallbacks used when building with an old gcc.
- * Rather use STARPU_VAL_COMPARE_AND_SWAP available on all platforms with a
- * recent-enough gcc */
+ * Rather use STARPU_VAL_COMPARE_AND_SWAP and STARPU_VAL_EXCHANGE available on
+ * all platforms with a recent-enough gcc */
 
 #if defined(__i386__) || defined(__x86_64__)
-static __starpu_inline unsigned starpu_cmpxchg(unsigned *ptr, unsigned old, unsigned next)
+static __starpu_inline unsigned _starpu_cmpxchg(unsigned *ptr, unsigned old, unsigned next)
 {
 	__asm__ __volatile__("lock cmpxchgl %2,%1": "+a" (old), "+m" (*ptr) : "q" (next) : "memory");
 	return old;
 }
 #define STARPU_HAVE_CMPXCHG
-static __starpu_inline unsigned starpu_xchg(unsigned *ptr, unsigned next)
+static __starpu_inline unsigned _starpu_xchg(unsigned *ptr, unsigned next)
 {
 	/* Note: xchg is always locked already */
 	__asm__ __volatile__("xchgl %1,%0": "+m" (*ptr), "+q" (next) : : "memory");
@@ -343,13 +343,13 @@ static __starpu_inline unsigned starpu_xchg(unsigned *ptr, unsigned next)
 }
 #define STARPU_HAVE_XCHG
 
-static __starpu_inline uint32_t starpu_cmpxchg32(uint32_t *ptr, uint32_t old, uint32_t next)
+static __starpu_inline uint32_t _starpu_cmpxchg32(uint32_t *ptr, uint32_t old, uint32_t next)
 {
 	__asm__ __volatile__("lock cmpxchgl %2,%1": "+a" (old), "+m" (*ptr) : "q" (next) : "memory");
 	return old;
 }
 #define STARPU_HAVE_CMPXCHG32
-static __starpu_inline uint32_t starpu_xchg32(uint32_t *ptr, uint32_t next)
+static __starpu_inline uint32_t _starpu_xchg32(uint32_t *ptr, uint32_t next)
 {
 	/* Note: xchg is always locked already */
 	__asm__ __volatile__("xchgl %1,%0": "+m" (*ptr), "+q" (next) : : "memory");
@@ -358,13 +358,13 @@ static __starpu_inline uint32_t starpu_xchg32(uint32_t *ptr, uint32_t next)
 #define STARPU_HAVE_XCHG32
 
 #if defined(__i386__)
-static __starpu_inline unsigned long starpu_cmpxchgl(unsigned long *ptr, unsigned long old, unsigned long next)
+static __starpu_inline unsigned long _starpu_cmpxchgl(unsigned long *ptr, unsigned long old, unsigned long next)
 {
 	__asm__ __volatile__("lock cmpxchgl %2,%1": "+a" (old), "+m" (*ptr) : "q" (next) : "memory");
 	return old;
 }
 #define STARPU_HAVE_CMPXCHGL
-static __starpu_inline unsigned long starpu_xchgl(unsigned long *ptr, unsigned long next)
+static __starpu_inline unsigned long _starpu_xchgl(unsigned long *ptr, unsigned long next)
 {
 	/* Note: xchg is always locked already */
 	__asm__ __volatile__("xchgl %1,%0": "+m" (*ptr), "+q" (next) : : "memory");
@@ -374,13 +374,13 @@ static __starpu_inline unsigned long starpu_xchgl(unsigned long *ptr, unsigned l
 #endif
 
 #if defined(__x86_64__)
-static __starpu_inline unsigned long starpu_cmpxchgl(unsigned long *ptr, unsigned long old, unsigned long next)
+static __starpu_inline unsigned long _starpu_cmpxchgl(unsigned long *ptr, unsigned long old, unsigned long next)
 {
 	__asm__ __volatile__("lock cmpxchgq %2,%1": "+a" (old), "+m" (*ptr) : "q" (next) : "memory");
 	return old;
 }
 #define STARPU_HAVE_CMPXCHGL
-static __starpu_inline unsigned long starpu_xchgl(unsigned long *ptr, unsigned long next)
+static __starpu_inline unsigned long _starpu_xchgl(unsigned long *ptr, unsigned long next)
 {
 	/* Note: xchg is always locked already */
 	__asm__ __volatile__("xchgq %1,%0": "+m" (*ptr), "+q" (next) : : "memory");
@@ -390,7 +390,7 @@ static __starpu_inline unsigned long starpu_xchgl(unsigned long *ptr, unsigned l
 #endif
 
 #if defined(__i386__)
-static __starpu_inline uint64_t starpu_cmpxchg64(uint64_t *ptr, uint64_t old, uint64_t next)
+static __starpu_inline uint64_t _starpu_cmpxchg64(uint64_t *ptr, uint64_t old, uint64_t next)
 {
 	uint32_t next_hi = next >> 32;
 	uint32_t next_lo = next & 0xfffffffful;
@@ -401,13 +401,13 @@ static __starpu_inline uint64_t starpu_cmpxchg64(uint64_t *ptr, uint64_t old, ui
 #endif
 
 #if defined(__x86_64__)
-static __starpu_inline uint64_t starpu_cmpxchg64(uint64_t *ptr, uint64_t old, uint64_t next)
+static __starpu_inline uint64_t _starpu_cmpxchg64(uint64_t *ptr, uint64_t old, uint64_t next)
 {
 	__asm__ __volatile__("lock cmpxchgq %2,%1": "+a" (old), "+m" (*ptr) : "q" (next) : "memory");
 	return old;
 }
 #define STARPU_HAVE_CMPXCHG64
-static __starpu_inline uint64_t starpu_xchg64(uint64_t *ptr, uint64_t next)
+static __starpu_inline uint64_t _starpu_xchg64(uint64_t *ptr, uint64_t next)
 {
 	/* Note: xchg is always locked already */
 	__asm__ __volatile__("xchgq %1,%0": "+m" (*ptr), "+q" (next) : : "memory");
@@ -426,7 +426,7 @@ static __starpu_inline unsigned starpu_atomic_##name(unsigned *ptr, unsigned val
 	{ \
 		old = *ptr; \
 		next = expr; \
-		if (starpu_cmpxchg(ptr, old, next) == old) \
+		if (_starpu_cmpxchg(ptr, old, next) == old) \
 			break; \
 	}; \
 	return expr; \
@@ -439,7 +439,7 @@ static __starpu_inline unsigned long starpu_atomic_##name##l(unsigned long *ptr,
 	{ \
 		old = *ptr; \
 		next = expr; \
-		if (starpu_cmpxchgl(ptr, old, next) == old) \
+		if (_starpu_cmpxchgl(ptr, old, next) == old) \
 			break; \
 	}; \
 	return expr; \
@@ -452,7 +452,7 @@ static __starpu_inline uint64_t starpu_atomic_##name##64(uint64_t *ptr, uint64_t
 	{ \
 		old = *ptr; \
 		next = expr; \
-		if (starpu_cmpxchg64(ptr, old, next) == old) \
+		if (_starpu_cmpxchg64(ptr, old, next) == old) \
 			break; \
 	}; \
 	return expr; \
@@ -503,13 +503,13 @@ STARPU_ATOMIC_SOMETHING64(or, old | value)
 #define STARPU_BOOL_COMPARE_AND_SWAP64(ptr, old, value) STARPU_BOOL_COMPARE_AND_SWAP(ptr, old, value)
 #else
 #ifdef STARPU_HAVE_CMPXCHG
-#define STARPU_BOOL_COMPARE_AND_SWAP(ptr, old, value) (starpu_cmpxchg((ptr), (old), (value)) == (old))
+#define STARPU_BOOL_COMPARE_AND_SWAP(ptr, old, value) (_starpu_cmpxchg((ptr), (old), (value)) == (old))
 #endif
 #ifdef STARPU_HAVE_CMPXCHG32
-#define STARPU_BOOL_COMPARE_AND_SWAP32(ptr, old, value) (starpu_cmpxchg32((ptr), (old), (value)) == (old))
+#define STARPU_BOOL_COMPARE_AND_SWAP32(ptr, old, value) (_starpu_cmpxchg32((ptr), (old), (value)) == (old))
 #endif
 #ifdef STARPU_HAVE_CMPXCHG64
-#define STARPU_BOOL_COMPARE_AND_SWAP64(ptr, old, value) (starpu_cmpxchg64((ptr), (old), (value)) == (old))
+#define STARPU_BOOL_COMPARE_AND_SWAP64(ptr, old, value) (_starpu_cmpxchg64((ptr), (old), (value)) == (old))
 #endif
 #endif
 
@@ -519,13 +519,33 @@ STARPU_ATOMIC_SOMETHING64(or, old | value)
 #define STARPU_VAL_COMPARE_AND_SWAP64(ptr, old, value) STARPU_VAL_COMPARE_AND_SWAP(ptr, old, value)
 #else
 #ifdef STARPU_HAVE_CMPXCHG
-#define STARPU_VAL_COMPARE_AND_SWAP(ptr, old, value) (starpu_cmpxchg((ptr), (old), (value)))
+#define STARPU_VAL_COMPARE_AND_SWAP(ptr, old, value) (_starpu_cmpxchg((ptr), (old), (value)))
 #endif
 #ifdef STARPU_HAVE_CMPXCHG32
-#define STARPU_VAL_COMPARE_AND_SWAP32(ptr, old, value) (starpu_cmpxchg32((ptr), (old), (value)))
+#define STARPU_VAL_COMPARE_AND_SWAP32(ptr, old, value) (_starpu_cmpxchg32((ptr), (old), (value)))
 #endif
 #ifdef STARPU_HAVE_CMPXCHG64
-#define STARPU_VAL_COMPARE_AND_SWAP64(ptr, old, value) (starpu_cmpxchg64((ptr), (old), (value)))
+#define STARPU_VAL_COMPARE_AND_SWAP64(ptr, old, value) (_starpu_cmpxchg64((ptr), (old), (value)))
+#endif
+#endif
+
+#ifdef STARPU_HAVE_ATOMIC_EXCHANGE_N
+#define STARPU_VAL_EXCHANGE(ptr, value) (__atomic_exchange_n((ptr), (value), __ATOMIC_SEQ_CST))
+#define STARPU_VAL_EXCHANGEL(ptr, value) STARPU_VAL_EXCHANGE((ptr) (value))
+#define STARPU_VAL_EXCHANGE32(ptr, value) STARPU_VAL_EXCHANGE((ptr) (value))
+#define STARPU_VAL_EXCHANGE64(ptr, value) STARPU_VAL_EXCHANGE((ptr) (value))
+#else
+#ifdef STARPU_HAVE_XCHG
+#define STARPU_VAL_EXCHANGE(ptr, value) (_starpu_xchg((ptr), (value)))
+#endif
+#ifdef STARPU_HAVE_XCHGL
+#define STARPU_VAL_EXCHANGEL(ptr, value) (_starpu_xchgl((ptr), (value)))
+#endif
+#ifdef STARPU_HAVE_XCHG32
+#define STARPU_VAL_EXCHANGE32(ptr, value) (_starpu_xchg32((ptr), (value)))
+#endif
+#ifdef STARPU_HAVE_XCHG64
+#define STARPU_VAL_EXCHANGE64(ptr, value) (_starpu_xchg64((ptr), (value)))
 #endif
 #endif
 
@@ -534,8 +554,8 @@ STARPU_ATOMIC_SOMETHING64(or, old | value)
 #define STARPU_TEST_AND_SET(ptr, value) (__sync_lock_test_and_set ((ptr), (value)))
 #define STARPU_RELEASE(ptr) (__sync_lock_release ((ptr)))
 #elif defined(STARPU_HAVE_XCHG)
-#define STARPU_TEST_AND_SET(ptr, value) (starpu_xchg((ptr), (value)))
-#define STARPU_RELEASE(ptr) (starpu_xchg((ptr), 0))
+#define STARPU_TEST_AND_SET(ptr, value) (_starpu_xchg((ptr), (value)))
+#define STARPU_RELEASE(ptr) (_starpu_xchg((ptr), 0))
 #endif
 
 #ifdef STARPU_HAVE_SYNC_SYNCHRONIZE

m4/acinclude.m4

@@ -2,7 +2,7 @@
 #
 # Copyright (C) 2012                                     Inria
 # Copyright (C) 2012,2017                                CNRS
-# Copyright (C) 2014                                     Université de Bordeaux
+# Copyright (C) 2014,2019                                Université de Bordeaux
 #
 # StarPU is free software; you can redistribute it and/or modify
 # it under the terms of the GNU Lesser General Public License as published by
@@ -81,6 +81,71 @@ AC_DEFUN([STARPU_CHECK_SYNC_LOCK_TEST_AND_SET], [
 	      [Define to 1 if the target supports __sync_lock_test_and_set])
   fi])
 
+# Check whether the target supports __atomic_compare_exchange_n.
+AC_DEFUN([STARPU_CHECK_ATOMIC_COMPARE_EXCHANGE_N], [
+  AC_CACHE_CHECK([whether the target supports __atomic_compare_exchange_n],
+		 ac_cv_have_atomic_compare_exchange_n, [
+  AC_LINK_IFELSE([AC_LANG_PROGRAM([int foo, bar, baz;],
+			[baz = __atomic_compare_exchange_n(&foo, &bar, 1, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);])],
+			[ac_cv_have_atomic_compare_exchange_n=yes],
+			[ac_cv_have_atomic_compare_exchange_n=no])])
+  if test $ac_cv_have_atomic_compare_exchange_n = yes; then
+    AC_DEFINE(STARPU_HAVE_ATOMIC_COMPARE_EXCHANGE_N, 1,
+	      [Define to 1 if the target supports __atomic_compare_exchange_n])
+  fi])
+
+# Check whether the target supports __atomic_exchange_n.
+AC_DEFUN([STARPU_CHECK_ATOMIC_EXCHANGE_N], [
+  AC_CACHE_CHECK([whether the target supports __atomic_exchange_n],
+		 ac_cv_have_atomic_exchange_n, [
+  AC_LINK_IFELSE([AC_LANG_PROGRAM([int foo, bar;],
+			[bar = __atomic_exchange_n(&foo, 1, __ATOMIC_SEQ_CST);])],
+			[ac_cv_have_atomic_exchange_n=yes],
+			[ac_cv_have_atomic_exchange_n=no])])
+  if test $ac_cv_have_atomic_exchange_n = yes; then
+    AC_DEFINE(STARPU_HAVE_ATOMIC_EXCHANGE_N, 1,
+	      [Define to 1 if the target supports __atomic_exchange_n])
+  fi])
+
+# Check whether the target supports __atomic_fetch_add.
+AC_DEFUN([STARPU_CHECK_ATOMIC_FETCH_ADD], [
+  AC_CACHE_CHECK([whether the target supports __atomic_fetch_add],
+		 ac_cv_have_atomic_fetch_add, [
+  AC_LINK_IFELSE([AC_LANG_PROGRAM([int foo, bar;],
+			[bar = __atomic_fetch_add(&foo, 1, __ATOMIC_SEQ_CST);])],
+			[ac_cv_have_atomic_fetch_add=yes],
+			[ac_cv_have_atomic_fetch_add=no])])
+  if test $ac_cv_have_atomic_fetch_add = yes; then
+    AC_DEFINE(STARPU_HAVE_ATOMIC_FETCH_ADD, 1,
+	      [Define to 1 if the target supports __atomic_fetch_add])
+  fi])
+
+# Check whether the target supports __atomic_fetch_or.
+AC_DEFUN([STARPU_CHECK_ATOMIC_FETCH_OR], [
+  AC_CACHE_CHECK([whether the target supports __atomic_fetch_or],
+		 ac_cv_have_atomic_fetch_or, [
+  AC_LINK_IFELSE([AC_LANG_PROGRAM([int foo, bar;],
+			[bar = __atomic_fetch_or(&foo, 1, __ATOMIC_SEQ_CST);])],
+			[ac_cv_have_atomic_fetch_or=yes],
+			[ac_cv_have_atomic_fetch_or=no])])
+  if test $ac_cv_have_atomic_fetch_or = yes; then
+    AC_DEFINE(STARPU_HAVE_ATOMIC_FETCH_OR, 1,
+	      [Define to 1 if the target supports __atomic_fetch_or])
+  fi])
+
+# Check whether the target supports __atomic_test_and_set.
+AC_DEFUN([STARPU_CHECK_ATOMIC_TEST_AND_SET], [
+  AC_CACHE_CHECK([whether the target supports __atomic_test_and_set],
+		 ac_cv_have_atomic_test_and_set, [
+  AC_LINK_IFELSE([AC_LANG_PROGRAM([int foo, bar;],
+			[bar = __atomic_test_and_set(&foo, __ATOMIC_SEQ_CST);])],
+			[ac_cv_have_atomic_test_and_set=yes],
+			[ac_cv_have_atomic_test_and_set=no])])
+  if test $ac_cv_have_atomic_test_and_set = yes; then
+    AC_DEFINE(STARPU_HAVE_ATOMIC_TEST_AND_SET, 1,
+	      [Define to 1 if the target supports __atomic_test_and_set])
+  fi])
+
 # Check whether the target supports __sync_synchronize.
 AC_DEFUN([STARPU_CHECK_SYNC_SYNCHRONIZE], [
   AC_CACHE_CHECK([whether the target supports __sync_synchronize],

src/common/thread.c

@@ -970,7 +970,7 @@ int _starpu_pthread_spin_do_lock(starpu_pthread_spinlock_t *lock)
 	while (1)
 	{
 		/* Tell releaser to wake us */
-		unsigned prev = starpu_xchg(&lock->taken, 2);
+		unsigned prev = STARPU_VAL_EXCHANGE(&lock->taken, 2);
 		if (prev == 0)
 			/* Ah, it just got released and we actually acquired
 			 * it!

src/drivers/driver_common/driver_common.c

@@ -45,7 +45,7 @@ void _starpu_driver_start_job(struct _starpu_worker *worker, struct _starpu_job
 		typedef unsigned __attribute__((__may_alias__)) alias_unsigned;
 		typedef int __attribute__((__may_alias__)) alias_int;
 
-		unsigned raw_bindid_requested = starpu_xchg((alias_unsigned *)&worker->bindid_requested, -1);
+		unsigned raw_bindid_requested = STARPU_VAL_EXCHANGE((alias_unsigned *)&worker->bindid_requested, -1);
 		int bindid_requested = *(alias_int *)&raw_bindid_requested;
 
 		if (bindid_requested != -1)