use atomic operations for locks

include/dmmlib/allocator.h

@@ -27,10 +27,6 @@
 #include <dmmlib/config.h>
 #include "dmmlib/raw_block.h"
 
-#ifdef HAVE_LOCKS
-#include <pthread.h>
-#endif /* HAVE_LOCKS */
-
 #ifdef WITH_KNOBS
 #include "dmmlib/knobs.h"
 #endif /* WITH_KNOBS */
@@ -50,7 +46,7 @@ typedef struct allocator_s {
     struct rb_head_s rb_head;
 #ifdef HAVE_LOCKS
 /** Mutex to allow the creation of new raw blocks. */
-    pthread_mutex_t creation_mutex;
+    locktype_t edit_lock;
 #endif /* HAVE_LOCKS */
 #ifdef WITH_KNOBS
     dmm_knobs_t dmm_knobs; /**< Global knobs. */

include/dmmlib/raw_block.h

@@ -28,16 +28,14 @@
 
 #include <stddef.h>
 
-#ifdef HAVE_LOCKS
-#include <pthread.h> /* FIXME To be removed once mutex is removed. */
-#endif /* HAVE_LOCKS */
-
 #include <dmmlib/lists.h>
 
 #ifdef WITH_RAWBLOCK_STATS
 #include <dmmlib/dmmstats.h>
 #endif /* WITH_RAWBLOCK_STATS */
 
+#include "locks.h"
+
 /** Enumeration of raw block's different types */
 typedef enum rb_type_en {
 #ifdef FL_RB_ONLY
@@ -59,7 +57,7 @@ typedef struct raw_block_header_s {
     SLIST_ENTRY(raw_block_header_s) pointers; /**< Pointer to the next raw
                                                 block. */
 #ifdef HAVE_LOCKS
-    pthread_mutex_t mutex;/**< Mutex. */
+    locktype_t lock; /**< Atomic lock. */
 #endif /* HAVE_LOCKS */
 } raw_block_header_t;
 

private-include/locks.h

@@ -29,39 +29,36 @@
 
 #ifdef HAVE_LOCKS
 
-#include <pthread.h>
+/** Enumeration of lock states */
+typedef enum locktype_en {STATE_LOCKED, STATE_UNLOCKED} locktype_t;
 
-/** Locks the systemallocator object */
-#define LOCK_GLOBAL() pthread_mutex_lock(&tls_allocator->creation_mutex)
-/** Unlocks the systemallocator object */
-#define UNLOCK_GLOBAL() pthread_mutex_unlock(&tls_allocator->creation_mutex)
-/** Locks a specific raw block */
-#define LOCK_RAW_BLOCK(rb) pthread_mutex_lock(&rb->mutex)
-/** Tries to lock a specific raw block */
-#define TRYLOCK_RAW_BLOCK(rb) pthread_mutex_trylock(&rb->mutex)
-/** Initializes a raw block lock */
-#define INIT_RAW_BLOCK_LOCK(rb) pthread_mutex_init(&rb->mutex, NULL);
-/** Destroys a raw block lock */
-#define DESTROY_RAW_BLOCK_LOCK(rb) pthread_mutex_destroy(&rb->mutex);
-/** Unlocks a specific raw block */
-#define UNLOCK_RAW_BLOCK(rb) pthread_mutex_unlock(&rb->mutex)
+/** Initializes a locktype_t variable */
+#define INIT_DEALY_LOCK(lock) lock = STATE_UNLOCKED
+/** Locks a locktype_t variable */
+#define DEALY_LOCK(lock) \
+    while(__sync_bool_compare_and_swap(&lock, \
+                STATE_UNLOCKED, STATE_LOCKED) != STATE_UNLOCKED){}
+/** Tries to lock a locktype_t variable */
+#define DEALY_TRYLOCK(lock) \
+    ((__sync_bool_compare_and_swap(&lock, STATE_UNLOCKED, STATE_LOCKED) != \
+      STATE_UNLOCKED))
+/** Unlocks a locktype_t variable */
+#define DEALY_UNLOCK(lock) lock = STATE_UNLOCKED
+/** Destroys a locktype_t */
+#define DEALY_DESTROY_LOCK(lock) lock = STATE_UNLOCKED
 
 #else /* HAVE_LOCKS */
 
 /** Does nothing. */
-#define LOCK_GLOBAL()
+#define INIT_DEALY_LOCK()
 /** Does nothing. */
-#define UNLOCK_GLOBAL()
+#define DEALY_LOCK()
 /** Does nothing. */
-#define LOCK_RAW_BLOCK(RB)
+#define DEALY_TRYLOCK(lock)
 /** Does nothing. */
-#define TRYLOCK_RAW_BLOCK(RB)
+#define DEALY_UNLOCK(lock)
 /** Does nothing. */
-#define INIT_RAW_BLOCK_LOCK(RB)
-/** Does nothing. */
-#define DESTROY_RAW_BLOCK_LOCK(RB)
-/** Does nothing. */
-#define UNLOCK_RAW_BLOCK(RB)
+#define DEALY_DESTROY_LOCK(lock)
 
 #endif /* HAVE_LOCKS */
 

private-include/statistics.h

@@ -37,15 +37,15 @@
 #ifdef REQUEST_SIZE_INFO
 /** Updates the global statistics for specific dmmlib events */
 #define UPDATE_GLOBAL_STATS(EVENT_TYPE, REQ_SIZE) \
-    LOCK_GLOBAL(); \
+    DEALY_LOCK(tls_allocator->edit_lock); \
     update_stats(&systemallocator->dmm_stats, EVENT_TYPE, REQ_SIZE); \
-    UNLOCK_GLOBAL();
+    DEALY_UNLOCK(tls_allocator->edit_lock);
 #else /* REQUEST_SIZE_INFO */
 /** Updates the global statistics for specific dmmlib events */
 #define UPDATE_GLOBAL_STATS(EVENT_TYPE) \
-    LOCK_GLOBAL(); \
+    DEALY_LOCK(tls_allocator->edit_lock); \
     update_stats(&systemallocator->dmm_stats, EVENT_TYPE); \
-    UNLOCK_GLOBAL();
+    DEALY_UNLOCK(tls_allocator->edit_lock);
 #endif /* REQUEST_SIZE_INFO */
 
 /** DMM event type enumeration */

private-include/tls_allocator.h

@@ -26,6 +26,7 @@
 #define DEALY_TLS_ALLOCATOR_H_
 
 #include <dmmlib/allocator.h>
+#include <pthread.h>
 
 /** Variable storing allocator's settings per thread. */
 extern __thread allocator_t *tls_allocator;
@@ -38,7 +39,7 @@ SLIST_HEAD(allocator_list_head_s, allocator_s);
 /** Singly-linked list for allocator_t nodes with a mutex lock */
 typedef struct locking_allocator_list_s {
     struct allocator_list_head_s head; /**< The head node. */
-    pthread_mutex_t lock; /**< The lock. */
+    locktype_t lock; /**< The lock. */
 } locking_allocator_list_t;
 
 /** List of destructed allocators that can be re-used */

src/free.c

@@ -58,9 +58,9 @@ void free(void *ptr) {
         DEFAULT_RB_T *encapsulated_rb = (DEFAULT_RB_T *)
             ((uintptr_t) owner_raw_block + sizeof(raw_block_header_t));
 
-        LOCK_RAW_BLOCK(owner_raw_block);
+        DEALY_LOCK(owner_raw_block->lock);
         dmmlib_free(encapsulated_rb, ptr);
-        UNLOCK_RAW_BLOCK(owner_raw_block);
+        DEALY_UNLOCK(owner_raw_block->lock);
 
     } else { // It has to be a BIGBLOCK, just munmap it
         owner_raw_block = (raw_block_header_t *)
@@ -71,7 +71,7 @@ void free(void *ptr) {
         !defined COALESCING_VARIABLE
         get_tls_allocator();
 #endif /* !PARSE_ENV || (PARSE_ENV && !MEM_TRACE) || !COALESCING_VARIABLE */
-        LOCK_GLOBAL();
+        DEALY_LOCK(tls_allocator->edit_lock);
         update_stats
             ( &tls_allocator->dmm_stats
             , FREE
@@ -80,7 +80,7 @@ void free(void *ptr) {
 #endif /* REQUEST_SIZE_INFO */
             );
         tls_allocator->dmm_stats.total_mem_allocated -= owner_raw_block->size;
-        UNLOCK_GLOBAL();
+        DEALY_UNLOCK(tls_allocator->edit_lock);
 #endif /* WITH_ALLOCATOR_STATS */
 
         release_memory(owner_raw_block);

src/freelist/realloc.c

@@ -88,9 +88,9 @@ void * freelist_realloc(freelist_rb_t *raw_block, void *ptr,
     /* TODO check if the next memory block is free and try to coalesce if
      * possible */
 
-    LOCK_RAW_BLOCK(rb);
+    DEALY_LOCK(rb->lock);
     ret = freelist_malloc(raw_block, req_size);
-    UNLOCK_RAW_BLOCK(rb);
+    DEALY_UNLOCK(rb->lock);
 
     if(ret != NULL) {
 #ifdef REQUEST_SIZE_INFO
@@ -110,7 +110,7 @@ void * freelist_realloc(freelist_rb_t *raw_block, void *ptr,
 
     memcpy(ret, ptr, get_size(block));
 
-    LOCK_RAW_BLOCK(rb);
+    DEALY_LOCK(rb->lock);
 
     /* start of copy from freelist_free() */
 #if defined (COALESCING_FIXED) || defined (COALESCING_VARIABLE)
@@ -121,7 +121,7 @@ void * freelist_realloc(freelist_rb_t *raw_block, void *ptr,
 #endif /* COALESCING_FIXED || COALESCING_VARIABLE */
     /* end of copy from freelist_free() */
 
-    UNLOCK_RAW_BLOCK(rb);
+    DEALY_UNLOCK(rb->lock);
 
     return ret;
 }

src/initialize.c

@@ -33,7 +33,7 @@
 __thread allocator_t * tls_allocator = NULL;
 
 locking_allocator_list_t destructed_list =
-{ SLIST_HEAD_INITIALIZER(head), PTHREAD_MUTEX_INITIALIZER};
+{ SLIST_HEAD_INITIALIZER(head), STATE_UNLOCKED };
 
 /** Initializes space to store the allocator state in heap. */
 allocator_t * initialize_allocator(void) {
@@ -48,7 +48,7 @@ allocator_t * initialize_allocator(void) {
 
     SLIST_INIT(&allocator->rb_head);
 #ifdef HAVE_LOCKS
-    pthread_mutex_init(&allocator->creation_mutex, NULL);
+    INIT_DEALY_LOCK(allocator->edit_lock);
 #endif /* HAVE_LOCKS */
 #ifdef WITH_KNOBS
     allocator->dmm_knobs.sys_alloc_size = SYS_ALLOC_SIZE;

src/knobs.c

@@ -30,7 +30,7 @@
 
 uint32_t dmm_set_knobs(dmm_knobs_t *conf) {
     get_tls_allocator();
-    LOCK_GLOBAL();
+    DEALY_LOCK(tls_allocator->edit_lock);
     tls_allocator->dmm_knobs.sys_alloc_size = conf->sys_alloc_size;
 #ifdef GOOD_FIT
     tls_allocator->dmm_knobs.fit_percentage = conf->fit_percentage;
@@ -41,7 +41,7 @@ uint32_t dmm_set_knobs(dmm_knobs_t *conf) {
 #ifdef SPLITTING_VARIABLE
     tls_allocator->dmm_knobs.min_split_size = conf->min_split_size;
 #endif /* SPLITTING_VARIABLE */
-    UNLOCK_GLOBAL();
+    DEALY_UNLOCK(tls_allocator->edit_lock);
     return 0;
 }
 

src/malloc.c

@@ -79,14 +79,14 @@ void * malloc(size_t size) {
         /* Try to find a raw block available for allocation */
         SLIST_FOREACH(raw_block, &tls_allocator->rb_head, pointers) {
 #ifdef TRYLOCK_ON_MALLOC
-            if(TRYLOCK_RAW_BLOCK(raw_block) == 0) {
+            if(DEALY_TRYLOCK(raw_block->lock) == 0) {
 #else /* TRYLOCK_ON_MALLOC */
-            LOCK_RAW_BLOCK(raw_block);
+            DEALY_LOCK(raw_block->lock);
 #endif /* TRYLOCK_ON_MALLOC */
             encapsulated_rb = (DEFAULT_RB_T *)
                 ((uintptr_t) raw_block + sizeof(raw_block_header_t));
             ptr = dmmlib_malloc(encapsulated_rb, size);
-            UNLOCK_RAW_BLOCK(raw_block);
+            DEALY_UNLOCK(raw_block->lock);
 
             if(ptr != NULL) {
                 /* Check that a valid pointer has been returned */

src/memalign.c

@@ -73,14 +73,14 @@ void *memalign(size_t alignment, size_t size) {
         /* Search the available freelist-organized raw blocks */
         SLIST_FOREACH(raw_block, &tls_allocator->rb_head, pointers) {
 #ifdef TRYLOCK_ON_MALLOC
-            if(TRYLOCK_RAW_BLOCK(raw_block) == 0) {
+            if(DEALY_TRYLOCK(raw_block->lock) == 0) {
 #else /* TRYLOCK_ON_MALLOC */
-            LOCK_RAW_BLOCK(raw_block);
+            DEALY_LOCK(raw_block->lock);
 #endif /* TRYLOCK_ON_MALLOC */
             encapsulated_rb = (freelist_rb_t *)
                 ((uintptr_t) raw_block + sizeof(raw_block_header_t));
             memptr = freelist_memalign(encapsulated_rb, alignment, size);
-            UNLOCK_RAW_BLOCK(raw_block);
+            DEALY_UNLOCK(raw_block->lock);
 
             if(memptr != NULL) {
                 break;

src/raw_block.c

@@ -70,8 +70,8 @@ raw_block_header_t *create_raw_block(size_t raw_block_size, rb_type type) {
         padding = 0;
     }
 
-    INIT_RAW_BLOCK_LOCK(ptr);
-    LOCK_RAW_BLOCK(ptr);
+    INIT_DEALY_LOCK(ptr->lock);
+    DEALY_LOCK(ptr->lock);
 
     ptr->size = raw_block_size + SYS_ALLOC_SIZE - 1 - padding;
 
@@ -137,7 +137,7 @@ raw_block_header_t *create_raw_block(size_t raw_block_size, rb_type type) {
             break;
     }
 
-    UNLOCK_RAW_BLOCK(ptr);
+    DEALY_UNLOCK(ptr->lock);
 
     return ptr;
 }

src/realloc.c

@@ -101,11 +101,11 @@ void * realloc(void *ptr, size_t size) {
             UPDATE_GLOBAL_STATS(REALLOC_LT);
 #endif /* REQUEST_SIZE_INFO */
 
-            LOCK_RAW_BLOCK(owner_raw_block);
+            DEALY_LOCK(owner_raw_block->lock);
 #ifdef REQUEST_SIZE_INFO
             owner_raw_block->requested_size = size;
 #endif /* REQUEST_SIZE_INFO */
-            UNLOCK_RAW_BLOCK(owner_raw_block);
+            DEALY_UNLOCK(owner_raw_block->lock);
 
             // FIXME This is pagesize and mmap-specific
 #ifdef PAGESIZE_ALIGN
@@ -113,15 +113,15 @@ void * realloc(void *ptr, size_t size) {
                 remaining_size = (remaining_size + SYS_PAGESIZE - 1) &
                     (size_t) ~(SYS_PAGESIZE - 1);
 
-                LOCK_RAW_BLOCK(owner_raw_block);
+                DEALY_LOCK(owner_raw_block->lock);
                 owner_raw_block->size = full_size;
-                UNLOCK_RAW_BLOCK(owner_raw_block);
+                DEALY_UNLOCK(owner_raw_block->lock);
 
 #ifdef WITH_ALLOCATOR_STATS
                 get_tls_allocator();
-                LOCK_GLOBAL();
+                DEALY_LOCK(tls_allocator->edit_lock);
                 tls_allocator->dmm_stats.total_mem_allocated -= remaining_size;
-                UNLOCK_GLOBAL();
+                DEALY_UNLOCK(tls_allocator->edit_lock);
 #endif /* WITH_ALLOCATOR_STATS */
 
                 munmap((void *)((uintptr_t) owner_raw_block +
@@ -157,10 +157,10 @@ void * realloc(void *ptr, size_t size) {
 #endif /* REQUEST_SIZE_INFO */
 
 #ifdef WITH_ALLOCATOR_STATS
-                LOCK_GLOBAL();
+                DEALY_LOCK(tls_allocator->edit_lock);
                 tls_allocator->dmm_stats.total_mem_allocated += new_block->size
                     - owner_raw_block->size;
-                UNLOCK_GLOBAL();
+                DEALY_UNLOCK(tls_allocator->edit_lock);
 #endif /* WITH_ALLOCATOR_STATS */
 
                 release_memory(owner_raw_block);

src/request_memory_mmap_linux.c

@@ -52,9 +52,9 @@ void *request_memory(size_t size) {
         return NULL;
     } else {
 #ifdef WITH_ALLOCATOR_STATS
-        LOCK_GLOBAL();
+        DEALY_LOCK(tls_allocator->edit_lock);
         systemallocator->dmm_stats.total_mem_allocated += size;
-        UNLOCK_GLOBAL();
+        DEALY_UNLOCK(tls_allocator->edit_lock);
 #endif /* WITH_ALLOCATOR_STATS */
         return zone;
     }

src/tls_allocator.c

@@ -1,24 +1,24 @@
-#include <pthread.h>
-
 #include <dmmlib/lists.h>
 #include "tls_allocator.h"
 
 void push_to_destructed_list(void) {
-    pthread_mutex_lock(&destructed_list.lock);
+    while(__sync_bool_compare_and_swap(&destructed_list.lock, STATE_UNLOCKED,
+                STATE_LOCKED) != STATE_UNLOCKED){};
     SLIST_INSERT_HEAD(&destructed_list.head, tls_allocator, pointer);
-    pthread_mutex_unlock(&destructed_list.lock);
+    destructed_list.lock = STATE_UNLOCKED;
 }
 
 void get_tls_allocator(void) {
     if(__builtin_expect(!!(tls_allocator == NULL), 0)) {
-        if(pthread_mutex_trylock(&destructed_list.lock) == 0) {
+        if(((__sync_bool_compare_and_swap(&destructed_list.lock, STATE_UNLOCKED,
+                            STATE_LOCKED) != STATE_UNLOCKED)) == 0) {
             tls_allocator = SLIST_FIRST(&destructed_list.head);
             if(tls_allocator == NULL) {
                 tls_allocator = initialize_allocator();
             } else {
                 SLIST_REMOVE_HEAD(&destructed_list.head, pointer);
             }
-            pthread_mutex_unlock(&destructed_list.lock);
+            destructed_list.lock = STATE_UNLOCKED;
         } else {
             tls_allocator = initialize_allocator();
         }