inline frequent spinlock operations

src/common/fxt.h

@@ -795,43 +795,43 @@ do {										\
 
 #define STARPU_TRACE_SPINLOCK_CONDITITION (starpu_worker_get_type(starpu_worker_get_id()) == STARPU_CUDA_WORKER)
 
-#define _STARPU_TRACE_LOCKING_SPINLOCK()	do {\
+#define _STARPU_TRACE_LOCKING_SPINLOCK(file, line)	do {\
 	if (STARPU_TRACE_SPINLOCK_CONDITITION) { \
 		const char *file; \
-		file = strrchr(__FILE__,'/') + 1; \
-		_STARPU_FUT_DO_PROBE2STR(_STARPU_FUT_LOCKING_SPINLOCK,__LINE__,_starpu_gettid(),file); \
+		file = strrchr(file,'/') + 1; \
+		_STARPU_FUT_DO_PROBE2STR(_STARPU_FUT_LOCKING_SPINLOCK,line,_starpu_gettid(file, line),file); \
 	} \
 } while(0)
 
-#define _STARPU_TRACE_SPINLOCK_LOCKED()		do { \
+#define _STARPU_TRACE_SPINLOCK_LOCKED(file, line)		do { \
 	if (STARPU_TRACE_SPINLOCK_CONDITITION) { \
 		const char *file; \
-		file = strrchr(__FILE__,'/') + 1; \
-		_STARPU_FUT_DO_PROBE2STR(_STARPU_FUT_SPINLOCK_LOCKED,__LINE__,_starpu_gettid(),file); \
+		file = strrchr(file,'/') + 1; \
+		_STARPU_FUT_DO_PROBE2STR(_STARPU_FUT_SPINLOCK_LOCKED,line,_starpu_gettid(file, line),file); \
 	} \
 } while(0)
 
-#define _STARPU_TRACE_UNLOCKING_SPINLOCK()	do { \
+#define _STARPU_TRACE_UNLOCKING_SPINLOCK(file, line)	do { \
 	if (STARPU_TRACE_SPINLOCK_CONDITITION) { \
 		const char *file; \
-		file = strrchr(__FILE__,'/') + 1; \
-		_STARPU_FUT_DO_PROBE2STR(_STARPU_FUT_UNLOCKING_SPINLOCK,__LINE__,_starpu_gettid(),file); \
+		file = strrchr(file,'/') + 1; \
+		_STARPU_FUT_DO_PROBE2STR(_STARPU_FUT_UNLOCKING_SPINLOCK,line,_starpu_gettid(file, line),file); \
 	} \
 } while(0)
 
-#define _STARPU_TRACE_SPINLOCK_UNLOCKED()	do { \
+#define _STARPU_TRACE_SPINLOCK_UNLOCKED(file, line)	do { \
 	if (STARPU_TRACE_SPINLOCK_CONDITITION) { \
 		const char *file; \
-		file = strrchr(__FILE__,'/') + 1; \
-		_STARPU_FUT_DO_PROBE2STR(_STARPU_FUT_SPINLOCK_UNLOCKED,__LINE__,_starpu_gettid(),file); \
+		file = strrchr(file,'/') + 1; \
+		_STARPU_FUT_DO_PROBE2STR(_STARPU_FUT_SPINLOCK_UNLOCKED,line,_starpu_gettid(file, line),file); \
 	} \
 } while(0)
 
-#define _STARPU_TRACE_TRYLOCK_SPINLOCK()	do { \
+#define _STARPU_TRACE_TRYLOCK_SPINLOCK(file, line)	do { \
 	if (STARPU_TRACE_SPINLOCK_CONDITITION) { \
 		const char *file; \
-		file = strrchr(__FILE__,'/') + 1; \
-		_STARPU_FUT_DO_PROBE2STR(_STARPU_FUT_TRYLOCK_SPINLOCK,__LINE__,_starpu_gettid(),file); \
+		file = strrchr(file,'/') + 1; \
+		_STARPU_FUT_DO_PROBE2STR(_STARPU_FUT_TRYLOCK_SPINLOCK,line,_starpu_gettid(file, line),file); \
 	} \
 } while(0)
 
@@ -872,11 +872,11 @@ do {										\
 #define _STARPU_TRACE_RWLOCK_WRLOCKED()		do {} while(0)
 #define _STARPU_TRACE_UNLOCKING_RWLOCK()		do {} while(0)
 #define _STARPU_TRACE_RWLOCK_UNLOCKED()		do {} while(0)
-#define _STARPU_TRACE_LOCKING_SPINLOCK()		do {} while(0)
-#define _STARPU_TRACE_SPINLOCK_LOCKED()		do {} while(0)
-#define _STARPU_TRACE_UNLOCKING_SPINLOCK()	do {} while(0)
-#define _STARPU_TRACE_SPINLOCK_UNLOCKED()		do {} while(0)
-#define _STARPU_TRACE_TRYLOCK_SPINLOCK()		do {} while(0)
+#define _STARPU_TRACE_LOCKING_SPINLOCK(file, line)		do {} while(0)
+#define _STARPU_TRACE_SPINLOCK_LOCKED(file, line)		do {} while(0)
+#define _STARPU_TRACE_UNLOCKING_SPINLOCK(file, line)	do {} while(0)
+#define _STARPU_TRACE_SPINLOCK_UNLOCKED(file, line)		do {} while(0)
+#define _STARPU_TRACE_TRYLOCK_SPINLOCK(file, line)		do {} while(0)
 #define _STARPU_TRACE_COND_WAIT_BEGIN()		do {} while(0)
 #define _STARPU_TRACE_COND_WAIT_END()			do {} while(0)
 #define _STARPU_TRACE_BARRIER_WAIT_BEGIN()		do {} while(0)
@@ -1000,11 +1000,11 @@ do {										\
 #define _STARPU_TRACE_RWLOCK_WRLOCKED()		do {} while(0)
 #define _STARPU_TRACE_UNLOCKING_RWLOCK()		do {} while(0)
 #define _STARPU_TRACE_RWLOCK_UNLOCKED()		do {} while(0)
-#define _STARPU_TRACE_LOCKING_SPINLOCK()		do {} while(0)
-#define _STARPU_TRACE_SPINLOCK_LOCKED()		do {} while(0)
-#define _STARPU_TRACE_UNLOCKING_SPINLOCK()	do {} while(0)
-#define _STARPU_TRACE_SPINLOCK_UNLOCKED()		do {} while(0)
-#define _STARPU_TRACE_TRYLOCK_SPINLOCK()		do {} while(0)
+#define _STARPU_TRACE_LOCKING_SPINLOCK(file, line)		do {} while(0)
+#define _STARPU_TRACE_SPINLOCK_LOCKED(file, line)		do {} while(0)
+#define _STARPU_TRACE_UNLOCKING_SPINLOCK(file, line)	do {} while(0)
+#define _STARPU_TRACE_SPINLOCK_UNLOCKED(file, line)		do {} while(0)
+#define _STARPU_TRACE_TRYLOCK_SPINLOCK(file, line)		do {} while(0)
 #define _STARPU_TRACE_COND_WAIT_BEGIN()		do {} while(0)
 #define _STARPU_TRACE_COND_WAIT_END()			do {} while(0)
 #define _STARPU_TRACE_BARRIER_WAIT_BEGIN()		do {} while(0)

src/common/starpu_spinlock.c

@@ -1,6 +1,6 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2010, 2012-2014  Université de Bordeaux
+ * Copyright (C) 2010, 2012-2014, 2016  Université de Bordeaux
  * Copyright (C) 2010, 2011, 2013, 2014  CNRS
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -21,9 +21,9 @@
 #include <common/fxt.h>
 #include <common/thread.h>
 
+#if defined(STARPU_SPINLOCK_CHECK)
 int _starpu_spin_init(struct _starpu_spinlock *lock)
 {
-#if defined(STARPU_SPINLOCK_CHECK)
 	starpu_pthread_mutexattr_t errcheck_attr;
 //	memcpy(&lock->errcheck_lock, PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP, sizeof(PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP));
 	int ret;
@@ -36,71 +36,10 @@ int _starpu_spin_init(struct _starpu_spinlock *lock)
 	ret = starpu_pthread_mutex_init(&lock->errcheck_lock, &errcheck_attr);
 	starpu_pthread_mutexattr_destroy(&errcheck_attr);
 	return ret;
-#else
-	int ret = starpu_pthread_spin_init(&lock->lock, 0);
-	STARPU_ASSERT(!ret);
-	return ret;
-#endif
 }
 
-int _starpu_spin_destroy(struct _starpu_spinlock *lock STARPU_ATTRIBUTE_UNUSED)
+int _starpu_spin_destroy(struct _starpu_spinlock *lock)
 {
-#if defined(STARPU_SPINLOCK_CHECK)
 	return starpu_pthread_mutex_destroy(&lock->errcheck_lock);
-#else
-	return starpu_pthread_spin_destroy(&lock->lock);
-#endif
-}
-
-#undef _starpu_spin_lock
-int _starpu_spin_lock(struct _starpu_spinlock *lock)
-{
-#if defined(STARPU_SPINLOCK_CHECK)
-	int ret = starpu_pthread_mutex_lock(&lock->errcheck_lock);
-	STARPU_ASSERT(!ret);
-	return ret;
-#else
-	int ret = starpu_pthread_spin_lock(&lock->lock);
-	STARPU_ASSERT(!ret);
-	return ret;
-#endif
-}
-
-int _starpu_spin_checklocked(struct _starpu_spinlock *lock)
-{
-#if defined(STARPU_SPINLOCK_CHECK)
-	int ret = starpu_pthread_mutex_trylock(&lock->errcheck_lock);
-	STARPU_ASSERT(ret != 0);
-	return ret == 0;
-#else
-	return _starpu_pthread_spin_checklocked(&lock->lock);
-#endif
-}
-
-#undef _starpu_spin_trylock
-int _starpu_spin_trylock(struct _starpu_spinlock *lock)
-{
-#if defined(STARPU_SPINLOCK_CHECK)
-	int ret = starpu_pthread_mutex_trylock(&lock->errcheck_lock);
-	STARPU_ASSERT(!ret || (ret == EBUSY));
-	return ret;
-#else
-	int ret = starpu_pthread_spin_trylock(&lock->lock);
-	STARPU_ASSERT(!ret || (ret == EBUSY));
-	return ret;
-#endif
 }
-
-#undef _starpu_spin_unlock
-int _starpu_spin_unlock(struct _starpu_spinlock *lock STARPU_ATTRIBUTE_UNUSED)
-{
-#if defined(STARPU_SPINLOCK_CHECK)
-	int ret = starpu_pthread_mutex_unlock(&lock->errcheck_lock);
-	STARPU_ASSERT(!ret);
-	return ret;
-#else
-	int ret = starpu_pthread_spin_unlock(&lock->lock);
-	STARPU_ASSERT(!ret);
-	return ret;
 #endif
-}

src/common/starpu_spinlock.h

@@ -1,6 +1,6 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
- * Copyright (C) 2010-2014  Université de Bordeaux
+ * Copyright (C) 2010-2014, 2016  Université de Bordeaux
  * Copyright (C) 2010, 2011, 2013, 2014, 2015  CNRS
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -21,57 +21,113 @@
 #include <stdint.h>
 #include <common/config.h>
 #include <common/fxt.h>
+#include <common/thread.h>
 #include <starpu.h>
 
+#ifdef STARPU_SPINLOCK_CHECK
+
+/* We don't care about performance */
+
 struct _starpu_spinlock
 {
-#if defined(STARPU_SPINLOCK_CHECK)
 	starpu_pthread_mutex_t errcheck_lock;
 	const char *last_taker;
+};
+
+int _starpu_spin_init(struct _starpu_spinlock *lock);
+int _starpu_spin_destroy(struct _starpu_spinlock *lock);
+
+static inline int __starpu_spin_lock(struct _starpu_spinlock *lock, const char *file STARPU_ATTRIBUTE_UNUSED, int line STARPU_ATTRIBUTE_UNUSED, const char *func STARPU_ATTRIBUTE_UNUSED)
+{
+	_STARPU_TRACE_LOCKING_SPINLOCK(file, line);
+	int ret = starpu_pthread_mutex_lock(&lock->errcheck_lock);
+	STARPU_ASSERT(!ret);
+	lock->last_taker = func;
+	_STARPU_TRACE_SPINLOCK_LOCKED(file, line);
+	return ret;
+}
+
+static inline void _starpu_spin_checklocked(struct _starpu_spinlock *lock STARPU_ATTRIBUTE_UNUSED)
+{
+	STARPU_ASSERT(starpu_pthread_mutex_trylock(&lock->errcheck_lock) != 0);
+}
+
+static inline int __starpu_spin_trylock(struct _starpu_spinlock *lock, const char *file STARPU_ATTRIBUTE_UNUSED, int line STARPU_ATTRIBUTE_UNUSED, const char *func STARPU_ATTRIBUTE_UNUSED)
+{
+	_STARPU_TRACE_TRYLOCK_SPINLOCK(file, line);
+	int ret = starpu_pthread_mutex_trylock(&lock->errcheck_lock);
+	STARPU_ASSERT(!ret || (ret == EBUSY));
+	if (STARPU_LIKELY(!ret))
+	{
+		lock->last_taker = func;
+		_STARPU_TRACE_SPINLOCK_LOCKED(file, line);
+	}
+	return ret;
+}
+
+static inline int __starpu_spin_unlock(struct _starpu_spinlock *lock, const char *file STARPU_ATTRIBUTE_UNUSED, int line STARPU_ATTRIBUTE_UNUSED, const char *func STARPU_ATTRIBUTE_UNUSED)
+{
+	_STARPU_TRACE_UNLOCKING_SPINLOCK(file, line);
+	int ret = starpu_pthread_mutex_unlock(&lock->errcheck_lock);
+	STARPU_ASSERT(!ret);
+	_STARPU_TRACE_SPINLOCK_UNLOCKED(file, line);
+	return ret;
+}
 #else
+
+/* We do care about performance, inline as much as possible */
+
+struct _starpu_spinlock
+{
 	starpu_pthread_spinlock_t lock;
-#endif
 };
 
-#ifdef STARPU_SPINLOCK_CHECK 
-#define STARPU_RECORD_LOCK(lock) do { 	\
-	(lock)->last_taker = __starpu_func__; \
-} while(0) 
-#else // !STARPU_SPINLOCK_CHECK
-#define STARPU_RECORD_LOCK(lock) do {} while(0)
-#endif // STARPU_SPINLOCK_CHECK
+static inline int _starpu_spin_init(struct _starpu_spinlock *lock)
+{
+	int ret = starpu_pthread_spin_init(&lock->lock, 0);
+	STARPU_ASSERT(!ret);
+	return ret;
+}
 
-int _starpu_spin_init(struct _starpu_spinlock *lock);
-int _starpu_spin_destroy(struct _starpu_spinlock *lock);
+#define _starpu_spin_destroy(_lock) starpu_pthread_spin_destroy(&(_lock)->lock)
 
-int _starpu_spin_lock(struct _starpu_spinlock *lock);
-#define _starpu_spin_lock(lock) ({ \
-	_STARPU_TRACE_LOCKING_SPINLOCK(); \
-	_starpu_spin_lock(lock); \
-	_STARPU_TRACE_SPINLOCK_LOCKED(); \
-	STARPU_RECORD_LOCK(lock); \
-	0; \
-}) 
-
-int _starpu_spin_trylock(struct _starpu_spinlock *lock);
-#define _starpu_spin_trylock(lock) ({ \
-	_STARPU_TRACE_TRYLOCK_SPINLOCK(); \
-	int err = _starpu_spin_trylock(lock); \
-	if (!err) { \
-		STARPU_RECORD_LOCK(lock); \
-		_STARPU_TRACE_SPINLOCK_LOCKED(); \
-	} \
-	err; \
-})
-int _starpu_spin_checklocked(struct _starpu_spinlock *lock);
-int _starpu_spin_unlock(struct _starpu_spinlock *lock);
-#define _starpu_spin_unlock(lock) ({ \
-	_STARPU_TRACE_UNLOCKING_SPINLOCK(); \
-	_starpu_spin_unlock(lock); \
-	_STARPU_TRACE_SPINLOCK_UNLOCKED(); \
-	0; \
-}) 
+static inline int __starpu_spin_lock(struct _starpu_spinlock *lock, const char *file STARPU_ATTRIBUTE_UNUSED, int line STARPU_ATTRIBUTE_UNUSED, const char *func STARPU_ATTRIBUTE_UNUSED)
+{
+	_STARPU_TRACE_LOCKING_SPINLOCK(file, line);
+	int ret = starpu_pthread_spin_lock(&lock->lock);
+	STARPU_ASSERT(!ret);
+	_STARPU_TRACE_SPINLOCK_LOCKED(file, line);
+	return ret;
+}
+
+#define _starpu_spin_checklocked(_lock) _starpu_pthread_spin_checklocked(&(_lock)->lock)
+
+static inline int __starpu_spin_trylock(struct _starpu_spinlock *lock, const char *file STARPU_ATTRIBUTE_UNUSED, int line STARPU_ATTRIBUTE_UNUSED, const char *func STARPU_ATTRIBUTE_UNUSED)
+{
+	_STARPU_TRACE_TRYLOCK_SPINLOCK(file, line);
+	int ret = starpu_pthread_spin_trylock(&lock->lock);
+	STARPU_ASSERT(!ret || (ret == EBUSY));
+	if (STARPU_LIKELY(!ret))
+		_STARPU_TRACE_SPINLOCK_LOCKED(file, line);
+	return ret;
+}
+
+static inline int __starpu_spin_unlock(struct _starpu_spinlock *lock, const char *file STARPU_ATTRIBUTE_UNUSED, int line STARPU_ATTRIBUTE_UNUSED, const char *func STARPU_ATTRIBUTE_UNUSED)
+{
+	_STARPU_TRACE_UNLOCKING_SPINLOCK(file, line);
+	int ret = starpu_pthread_spin_unlock(&lock->lock);
+	STARPU_ASSERT(!ret);
+	_STARPU_TRACE_SPINLOCK_UNLOCKED(file, line);
+	return ret;
+}
+#endif
 
+#define _starpu_spin_lock(lock) \
+	__starpu_spin_lock(lock, __FILE__, __LINE__, __starpu_func__)
+#define _starpu_spin_trylock(lock) \
+	__starpu_spin_trylock(lock, __FILE__, __LINE__, __starpu_func__)
+#define _starpu_spin_unlock(lock) \
+	__starpu_spin_unlock(lock, __FILE__, __LINE__, __starpu_func__)
 
 #define STARPU_SPIN_MAXTRY 10