port r11394 from 1.1: Replace rwlock (always taken in write mode) with spinlock

src/datawizard/memalloc.c

@@ -24,7 +24,7 @@
 
 /* This per-node RW-locks protect mc_list and memchunk_cache entries */
 /* Note: handle header lock is always taken before this */
-static starpu_pthread_rwlock_t mc_rwlock[STARPU_MAXNODES];
+static struct _starpu_spinlock mc_lock[STARPU_MAXNODES];
 
 /* Potentially in use memory chunks */
 static struct _starpu_mem_chunk_list *mc_list[STARPU_MAXNODES];
@@ -49,7 +49,7 @@ void _starpu_init_mem_chunk_lists(void)
 	unsigned i;
 	for (i = 0; i < STARPU_MAXNODES; i++)
 	{
-		STARPU_PTHREAD_RWLOCK_INIT(&mc_rwlock[i], NULL);
+		_starpu_spin_init(&mc_lock[i]);
 		mc_list[i] = _starpu_mem_chunk_list_new();
 	}
 }
@@ -67,7 +67,7 @@ void _starpu_deinit_mem_chunk_lists(void)
 			_starpu_mem_chunk_list_delete(entry->list);
 			free(entry);
 		}
-		STARPU_PTHREAD_RWLOCK_DESTROY(&mc_rwlock[i]);
+		_starpu_spin_destroy(&mc_lock[i]);
 	}
 }
 
@@ -399,7 +399,7 @@ static size_t try_to_free_mem_chunk(struct _starpu_mem_chunk *mc, unsigned node)
 }
 
 #ifdef STARPU_USE_ALLOCATION_CACHE
-/* We assume that mc_rwlock[node] is taken. is_already_in_mc_list indicates
+/* We assume that mc_lock[node] is taken. is_already_in_mc_list indicates
  * that the mc is already in the list of buffers that are possibly used, and
  * therefore not in the cache. */
 static void reuse_mem_chunk(unsigned node, struct _starpu_data_replicate *new_replicate, struct _starpu_mem_chunk *mc, unsigned is_already_in_mc_list)
@@ -484,7 +484,7 @@ static int _starpu_data_interface_compare(void *data_interface_a, struct starpu_
 	return ret;
 }
 
-/* This function must be called with mc_rwlock[node] taken in write mode */
+/* This function must be called with mc_lock[node] taken */
 static struct _starpu_mem_chunk *_starpu_memchunk_cache_lookup_locked(unsigned node, starpu_data_handle_t handle, uint32_t footprint)
 {
 	/* go through all buffers in the cache */
@@ -517,7 +517,7 @@ static struct _starpu_mem_chunk *_starpu_memchunk_cache_lookup_locked(unsigned n
 
 /* this function looks for a memory chunk that matches a given footprint in the
  * list of mem chunk that need to be freed. This function must be called with
- * mc_rwlock[node] taken in write mode. */
+ * mc_lock[node] taken. */
 static unsigned try_to_find_reusable_mem_chunk(unsigned node, starpu_data_handle_t data, struct _starpu_data_replicate *replicate, uint32_t footprint)
 {
 	struct _starpu_mem_chunk *mc, *next_mc;
@@ -558,7 +558,7 @@ static unsigned try_to_find_reusable_mem_chunk(unsigned node, starpu_data_handle
 
 /*
  * Free the memory chuncks that are explicitely tagged to be freed. The
- * mc_rwlock[node] rw-lock should be taken prior to calling this function.
+ * mc_lock[node] rw-lock should be taken prior to calling this function.
  */
 static size_t flush_memchunk_cache(unsigned node, size_t reclaim)
 {
@@ -568,7 +568,7 @@ static size_t flush_memchunk_cache(unsigned node, size_t reclaim)
 
 	size_t freed = 0;
 
-	STARPU_PTHREAD_RWLOCK_WRLOCK(&mc_rwlock[node]);
+	_starpu_spin_lock(&mc_lock[node]);
 	HASH_ITER(hh, mc_cache[node], entry, tmp)
 	{
 		busy_mc_cache = _starpu_mem_chunk_list_new();
@@ -600,7 +600,7 @@ static size_t flush_memchunk_cache(unsigned node, size_t reclaim)
 		if (reclaim && freed >= reclaim)
 			break;
 	}
-	STARPU_PTHREAD_RWLOCK_UNLOCK(&mc_rwlock[node]);
+	_starpu_spin_unlock(&mc_lock[node]);
 	return freed;
 }
 
@@ -608,7 +608,7 @@ static size_t flush_memchunk_cache(unsigned node, size_t reclaim)
  * Try to free the buffers currently in use on the memory node. If the force
  * flag is set, the memory is freed regardless of coherency concerns (this
  * should only be used at the termination of StarPU for instance). The
- * mc_rwlock[node] rw-lock should be taken prior to calling this function.
+ * mc_lock[node] should be taken prior to calling this function.
  */
 static size_t free_potentially_in_use_mc(unsigned node, unsigned force, size_t reclaim)
 {
@@ -617,7 +617,7 @@ static size_t free_potentially_in_use_mc(unsigned node, unsigned force, size_t r
 	struct _starpu_mem_chunk *mc, *next_mc;
 
 	/*
-	 * We have to unlock mc_rwlock before locking header_lock, so we have
+	 * We have to unlock mc_lock before locking header_lock, so we have
 	 * to be careful with the list.  We try to do just one pass, by
 	 * remembering the next mc to be tried. If it gets dropped, we restart
 	 * from zero. So we continue until we go through the whole list without
@@ -625,7 +625,7 @@ static size_t free_potentially_in_use_mc(unsigned node, unsigned force, size_t r
 	 */
 
 restart:
-	STARPU_PTHREAD_RWLOCK_WRLOCK(&mc_rwlock[node]);
+	_starpu_spin_lock(&mc_lock[node]);
 
 	for (mc = _starpu_mem_chunk_list_begin(mc_list[node]);
 	     mc != _starpu_mem_chunk_list_end(mc_list[node]);
@@ -652,7 +652,7 @@ restart:
 				 * still locking the handle. That's not
 				 * supposed to happen, but better be safe by
 				 * letting it go through. */
-				STARPU_PTHREAD_RWLOCK_UNLOCK(&mc_rwlock[node]);
+				_starpu_spin_unlock(&mc_lock[node]);
 				goto restart;
 			}
 
@@ -664,7 +664,7 @@ restart:
 			_starpu_spin_unlock(&handle->header_lock);
 		}
 	}
-	STARPU_PTHREAD_RWLOCK_UNLOCK(&mc_rwlock[node]);
+	_starpu_spin_unlock(&mc_lock[node]);
 
 	return freed;
 }
@@ -739,11 +739,11 @@ static void register_mem_chunk(struct _starpu_data_replicate *replicate, unsigne
 	/* Put this memchunk in the list of memchunk in use */
 	mc = _starpu_memchunk_init(replicate, interface_size, automatically_allocated);
 
-	STARPU_PTHREAD_RWLOCK_WRLOCK(&mc_rwlock[dst_node]);
+	_starpu_spin_lock(&mc_lock[dst_node]);
 
 	_starpu_mem_chunk_list_push_back(mc_list[dst_node], mc);
 
-	STARPU_PTHREAD_RWLOCK_UNLOCK(&mc_rwlock[dst_node]);
+	_starpu_spin_unlock(&mc_lock[dst_node]);
 }
 
 /* This function is called when the handle is destroyed (eg. when calling
@@ -766,13 +766,13 @@ void _starpu_request_mem_chunk_removal(starpu_data_handle_t handle, struct _star
 	replicate->allocated = 0;
 	replicate->automatically_allocated = 0;
 
-	STARPU_PTHREAD_RWLOCK_WRLOCK(&mc_rwlock[node]);
+	_starpu_spin_lock(&mc_lock[node]);
 
 	mc->data = NULL;
 	/* remove it from the main list */
 	_starpu_mem_chunk_list_erase(mc_list[node], mc);
 
-	STARPU_PTHREAD_RWLOCK_UNLOCK(&mc_rwlock[node]);
+	_starpu_spin_unlock(&mc_lock[node]);
 
 	/* We would only flush the RAM nodes cache if memory gets tight, either
 	 * because StarPU automatically knows the total memory size of the
@@ -798,7 +798,7 @@ void _starpu_request_mem_chunk_removal(starpu_data_handle_t handle, struct _star
 		/* put it in the list of buffers to be removed */
 		uint32_t footprint = mc->footprint;
 		struct mc_cache_entry *entry;
-		STARPU_PTHREAD_RWLOCK_WRLOCK(&mc_rwlock[node]);
+		_starpu_spin_lock(&mc_lock[node]);
 		HASH_FIND(hh, mc_cache[node], &footprint, sizeof(footprint), entry);
 		if (!entry) {
 			entry = malloc(sizeof(*entry));
@@ -806,7 +806,7 @@ void _starpu_request_mem_chunk_removal(starpu_data_handle_t handle, struct _star
 			HASH_ADD_KEYPTR(hh, mc_cache[node], &footprint, sizeof(footprint), entry);
 		}
 		_starpu_mem_chunk_list_push_front(entry->list, mc);
-		STARPU_PTHREAD_RWLOCK_UNLOCK(&mc_rwlock[node]);
+		_starpu_spin_unlock(&mc_lock[node]);
 	}
 }
 
@@ -838,16 +838,16 @@ static starpu_ssize_t _starpu_allocate_interface(starpu_data_handle_t handle, st
 	uint32_t footprint = _starpu_compute_data_footprint(handle);
 
 	_STARPU_TRACE_START_ALLOC_REUSE(dst_node, data_size);
-	STARPU_PTHREAD_RWLOCK_WRLOCK(&mc_rwlock[dst_node]);
+	_starpu_spin_lock(&mc_lock[dst_node]);
 
 	if (try_to_find_reusable_mem_chunk(dst_node, handle, replicate, footprint))
 	{
-		STARPU_PTHREAD_RWLOCK_UNLOCK(&mc_rwlock[dst_node]);
+		_starpu_spin_unlock(&mc_lock[dst_node]);
 		_starpu_allocation_cache_hit(dst_node);
 		return data_size;
 	}
 
-	STARPU_PTHREAD_RWLOCK_UNLOCK(&mc_rwlock[dst_node]);
+	_starpu_spin_unlock(&mc_lock[dst_node]);
 	_STARPU_TRACE_END_ALLOC_REUSE(dst_node);
 #endif
 	STARPU_ASSERT(handle->ops);
@@ -977,16 +977,16 @@ void _starpu_memchunk_recently_used(struct _starpu_mem_chunk *mc, unsigned node)
 	if (!mc)
 		/* user-allocated memory */
 		return;
-	STARPU_PTHREAD_RWLOCK_WRLOCK(&mc_rwlock[node]);
+	_starpu_spin_lock(&mc_lock[node]);
 	_starpu_mem_chunk_list_erase(mc_list[node], mc);
 	_starpu_mem_chunk_list_push_back(mc_list[node], mc);
-	STARPU_PTHREAD_RWLOCK_UNLOCK(&mc_rwlock[node]);
+	_starpu_spin_unlock(&mc_lock[node]);
 }
 
 #ifdef STARPU_MEMORY_STATS
 void _starpu_memory_display_stats_by_node(int node)
 {
-	STARPU_PTHREAD_RWLOCK_WRLOCK(&mc_rwlock[node]);
+	_starpu_spin_lock(&mc_lock[node]);
 
 	if (!_starpu_mem_chunk_list_empty(mc_list[node]))
 	{
@@ -1005,7 +1005,7 @@ void _starpu_memory_display_stats_by_node(int node)
 
 	}
 
-	STARPU_PTHREAD_RWLOCK_UNLOCK(&mc_rwlock[node]);
+	_starpu_spin_unlock(&mc_lock[node]);
 }
 #endif