support for multiple allocator states, each one for each thread

include/dmmlib/allocator.h

@@ -67,6 +67,7 @@ typedef struct allocator_s {
     bool initialized; /**< Initialization value. */
 #endif /* WITH_MEM_TRACE || WITH_STATS_TRACE || WITH_DEBUG || PARSE_ENV */
 
+    SLIST_ENTRY(allocator_s) pointer; /**< Pointer to the next allocator node. */
 } allocator_t;
 
 #endif /* ALLOCATOR_H */

include/dmmlib/dmmlib.h

@@ -30,12 +30,9 @@
 #ifndef DMMLIB_H
 #define DMMLIB_H
 #include <dmmlib/config.h>
-#include <dmmlib/allocator.h>
-
-/** Global variable storing allocator's settings */
-allocator_t *systemallocator;
 
 #if !defined _STDLIB_H && !defined _STDLIB_H_
+#include <stddef.h>
 
 /**
  * Allocates size bytes by using the system's allocator. The memory is not

private-include/dmmlib_trace.h

@@ -25,6 +25,7 @@
 #ifndef _DMMLIB_TRACE_H_
 #define _DMMLIB_TRACE_H_
 #include "dmmlib/config.h"
+#include "tls_allocator.h"
 
 #ifdef PARSE_ENV
 #include "dmmlib/dmmlib.h" /* to check if systemallocator is initialized */
@@ -48,7 +49,8 @@ FILE* mem_fd;
 
 /** Function for memory trace messages */
 #define MEM_TRACE(...) \
-    if(__builtin_expect (systemallocator->initialized, true)) { \
+    get_tls_allocator(); \
+    if(__builtin_expect (tls_allocator->initialized, true)) { \
         fprintf(MEM_FD, __VA_ARGS__); \
     }
 
@@ -79,7 +81,8 @@ FILE* stats_fd;
 
 /** Function for statistics trace messages */
 #define STATS_TRACE(...) \
-    if(__builtin_expect (systemallocator->initialized, true)) { \
+    get_tls_allocator(); \
+    if(__builtin_expect (tls_allocator->initialized, true)) { \
         fprintf(STATS_FD, __VA_ARGS__); \
     }
 
@@ -110,7 +113,8 @@ FILE* dbg_fd;
 
 /** Function for debug trace messages */
 #define DBG_TRACE(...) \
-    if(__builtin_expect (systemallocator->initialized, true)) { \
+    get_tls_allocator(); \
+    if(__builtin_expect (tls_allocator->initialized, true)) { \
         fprintf(DBG_FD, __VA_ARGS__); \
     }
 

private-include/locks.h

@@ -32,9 +32,9 @@
 #include <pthread.h>
 
 /** Locks the systemallocator object */
-#define LOCK_GLOBAL() pthread_mutex_lock(&systemallocator->creation_mutex)
+#define LOCK_GLOBAL() pthread_mutex_lock(&tls_allocator->creation_mutex)
 /** Unlocks the systemallocator object */
-#define UNLOCK_GLOBAL() pthread_mutex_unlock(&systemallocator->creation_mutex)
+#define UNLOCK_GLOBAL() pthread_mutex_unlock(&tls_allocator->creation_mutex)
 /** Locks a specific raw block */
 #define LOCK_RAW_BLOCK(rb) pthread_mutex_lock(&rb->mutex)
 /** Tries to lock a specific raw block */

private-include/parse_env.h

@@ -22,5 +22,11 @@
  * @brief  Function to parse environment variables
  */
 
+#ifndef DEALY_PARSE_ENV_H_
+#define DEALY_PARSE_ENV_H_
+#include <dmmlib/allocator.h>
+
 /** Parses various allocator-related enviroment variables. */
-void parse_env(void);
+void parse_env(allocator_t *allocator);
+
+#endif /* DEALY_PARSE_ENV_H_ */

private-include/tls_allocator.h

@@ -0,0 +1,57 @@
+/*
+ *   Copyright RubberDuck Software P.C.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ *
+ */
+
+/**
+ * @file   tls_allocator.h
+ * @author Ioannis Koutras (joko@rdsoftware.eu)
+ * @date   February 2014
+ * @brief  Functions for TLS allocators
+ */
+
+#ifndef DEALY_TLS_ALLOCATOR_H_
+#define DEALY_TLS_ALLOCATOR_H_
+
+#include <dmmlib/allocator.h>
+
+/** Variable storing allocator's settings per thread. */
+extern __thread allocator_t *tls_allocator;
+/** Key to re-use the previous variable for other threads. */
+pthread_key_t destruction_key;
+
+/** Struct for head of a singly-linked list for allocator_t nodes */
+SLIST_HEAD(allocator_list_head_s, allocator_s);
+
+/** Singly-linked list for allocator_t nodes with a mutex lock */
+typedef struct locking_allocator_list_s {
+    struct allocator_list_head_s head; /**< The head node. */
+    pthread_mutex_t lock; /**< The lock. */
+} locking_allocator_list_t;
+
+/** List of destructed allocators that can be re-used */
+extern locking_allocator_list_t destructed_list;
+
+/** Pushes the soon-to-be-dead thread's allocator state to a list for reuse. */
+void push_to_destructed_list(void);
+
+/** Allocates in the heap space and initializes allocator state to be used by a
+ *  specific thread. */
+allocator_t * initialize_allocator(void);
+
+/** Gets the current thread's allocator. */
+void get_tls_allocator(void);
+
+#endif /* DEALY_TLS_ALLOCATOR_H_ */

src/CMakeLists.txt

@@ -27,6 +27,7 @@ endif (WITH_STATIC_LIB)
 set(dmmlib_SRCS
   padding.c
   raw_block.c
+  tls_allocator.c
   )
 
 if(NOT WITH_SYSTEM_CALLS STREQUAL "none")

src/free.c

@@ -27,6 +27,7 @@
 
 #include "dmmlib/dmmlib.h"
 
+#include "tls_allocator.h"
 #include "default_rb.h"
 #include "dmmlib_trace.h"
 #include "locks.h"
@@ -42,7 +43,8 @@ void free(void *ptr) {
     }
 
 #if defined PARSE_ENV && defined WITH_MEM_TRACE
-    if(__builtin_expect (systemallocator->initialized, true)) {
+    get_tls_allocator();
+    if(__builtin_expect (tls_allocator->initialized, true)) {
 #endif /* PARSE_ENV && WITH_MEM_TRACE */
     MEM_TRACE("dmmlib - f %p\n", ptr);
 #if defined PARSE_ENV && defined WITH_MEM_TRACE
@@ -50,7 +52,8 @@ void free(void *ptr) {
 #endif /* PARSE_ENV && WITH_MEM_TRACE */
 
     owner_raw_block = find_raw_block_owner(ptr);
-    if( (uintptr_t) owner_raw_block + sizeof(raw_block_header_t) != (uintptr_t) ptr) {
+    if( (uintptr_t) owner_raw_block + sizeof(raw_block_header_t) !=
+            (uintptr_t) ptr) {
         /* It is not a big block, so act accordingly. */
         DEFAULT_RB_T *encapsulated_rb = (DEFAULT_RB_T *)
             ((uintptr_t) owner_raw_block + sizeof(raw_block_header_t));
@@ -64,15 +67,19 @@ void free(void *ptr) {
             ((uintptr_t) ptr - sizeof(raw_block_header_t));
 
 #ifdef WITH_ALLOCATOR_STATS
+#if !defined PARSE_ENV || (defined PARSE_ENV && !defined MEM_TRACE) || \
+        !defined COALESCING_VARIABLE
+        get_tls_allocator();
+#endif /* !PARSE_ENV || (PARSE_ENV && !MEM_TRACE) || !COALESCING_VARIABLE */
         LOCK_GLOBAL();
         update_stats
-            ( &systemallocator->dmm_stats
+            ( &tls_allocator->dmm_stats
             , FREE
 #ifdef REQUEST_SIZE_INFO
             , owner_raw_block->requested_size
 #endif /* REQUEST_SIZE_INFO */
             );
-        systemallocator->dmm_stats.total_mem_allocated -= owner_raw_block->size;
+        tls_allocator->dmm_stats.total_mem_allocated -= owner_raw_block->size;
         UNLOCK_GLOBAL();
 #endif /* WITH_ALLOCATOR_STATS */
 

src/freelist/coalesce.c

@@ -17,9 +17,12 @@
 
 #include "dmmlib/config.h"
 
-#ifdef WITH_KNOBS
+#ifdef COALESCING_VARIABLE
 #include "dmmlib/dmmlib.h"
-#endif /* WITH_KNOBS */
+#if !defined PARSE_ENV || (defined PARSE_ENV && !defined MEM_TRACE)
+#include "tls_allocator.h"
+#endif /* !PARSE_ENV || (PARSE_ENV && !MEM_TRACE) */
+#endif /* COALESCING_VARIABLE */
 
 #include "freelist/coalesce.h"
 #include "freelist/block_header_funcs.h"
@@ -37,7 +40,10 @@ void coalesce(freelist_rb_t *raw_block, block_header_t *ptr) {
     max_coal_size = MAX_COAL_SIZE;
 #endif /* COALESCING_FIXED */
 #ifdef COALESCING_VARIABLE
-    max_coal_size = (size_t) systemallocator->dmm_knobs.max_coal_size;
+#if !defined PARSE_ENV || (defined PARSE_ENV && !defined MEM_TRACE)
+    get_tls_allocator();
+#endif /* !PARSE_ENV || (PARSE_ENV && !MEM_TRACE) */
+    max_coal_size = (size_t) tls_allocator->dmm_knobs.max_coal_size;
 #endif /* COALESCING_VARIABLE */
 
     size = get_size(ptr);

src/freelist/fitting/good.c

@@ -18,7 +18,7 @@
 #include <dmmlib/config.h>
 
 #ifdef WITH_KNOBS
-#include "dmmlib/dmmlib.h"
+#include "tls_allocator.h"
 #endif /* WITH_KNOBS */
 
 #include "freelist/fitting/good.h"
@@ -36,7 +36,7 @@ block_header_t * fl_good_fit(freelist_rb_t *raw_block, size_t requested_size) {
     good_size = (size_t) -1; /* SIZE_MAX */
 #ifdef WITH_KNOBS
     ideal_size = (size_t) ((double) requested_size /
-            systemallocator->dmm_knobs.fit_percentage);
+            tls_allocator->dmm_knobs.fit_percentage);
 #else /* WITH_KNOBS */
     ideal_size = (size_t) ((double) requested_size / GOOD_FIT_PERCENTAGE);
 #endif /* WITH_KNOBS */

src/freelist/split.c

@@ -20,6 +20,7 @@
 #include "dmmlib/config.h"
 #ifdef WITH_KNOBS
 #include "dmmlib/dmmlib.h"
+#include "tls_allocator.h"
 #endif /* WITH_KNOBS */
 
 #include "other.h"
@@ -58,7 +59,7 @@ void split(freelist_rb_t *raw_block, block_header_t *ptr, size_t req_size) {
     min_split_size = MIN_SPLIT_SIZE;
 #endif /* SPLITTING_FIXED */
 #ifdef SPLITTING_VARIABLE
-    min_split_size = (size_t) systemallocator->dmm_knobs.min_split_size;
+    min_split_size = (size_t) tls_allocator->dmm_knobs.min_split_size;
 #endif /* SPLITTING_VARIABLE */
 
     if(new_size < min_split_size) {
@@ -80,7 +81,7 @@ void split(freelist_rb_t *raw_block, block_header_t *ptr, size_t req_size) {
     max_coal_size = MAX_COAL_SIZE;
 #endif /* COALESCING_FIXED */
 #ifdef COALESCING_VARIABLE
-    max_coal_size = systemallocator->dmm_knobs.max_coal_size;
+    max_coal_size = tls_allocator->dmm_knobs.max_coal_size;
 #endif /* COALESCING_VARIABLE */
 
     if(next_block != NULL && is_free(next_block) == true) {

src/initialize.c

@@ -28,18 +28,22 @@
 #include <dmmlib/config.h>
 #include "dmmlib/dmmlib.h"
 #include "parse_env.h"
+#include "tls_allocator.h"
 
-__attribute__((constructor)) int initialize_allocator(void);
+__thread allocator_t * tls_allocator = NULL;
+
+locking_allocator_list_t destructed_list =
+{ SLIST_HEAD_INITIALIZER(head), PTHREAD_MUTEX_INITIALIZER};
 
 /** Initializes space to store the allocator state in heap. */
-int initialize_allocator(void) {
+allocator_t * initialize_allocator(void) {
 
     int fd = open("/dev/zero", O_RDWR);
     allocator_t *allocator = mmap(NULL, SYS_PAGESIZE,
             PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
 
     if(__builtin_expect(!!(allocator == MAP_FAILED), 0)) {
-        return -1;
+        return NULL;
     }
 
     SLIST_INIT(&allocator->rb_head);
@@ -77,14 +81,12 @@ int initialize_allocator(void) {
 #endif /* WITH_MEM_TRACE || WITH_STATS_TRACE || WITH_DEBUG */
 #endif /* PARSE_ENV */
 
-    systemallocator = allocator;
-
 #ifdef PARSE_ENV
-    parse_env();
+    parse_env(allocator);
 #if defined  WITH_MEM_TRACE || defined WITH_STATS_TRACE || defined WITH_DEBUG
     allocator->initialized = true;
 #endif /* WITH_MEM_TRACE || WITH_STATS_TRACE || WITH_DEBUG */
 #endif /* PARSE_ENV */
 
-    return 0;
+    return allocator;
 }

src/knobs.c

@@ -24,20 +24,22 @@
 
 #include <dmmlib/config.h>
 #include "dmmlib/dmmlib.h"
+#include "tls_allocator.h"
 
 #include "locks.h"
 
 uint32_t dmm_set_knobs(dmm_knobs_t *conf) {
+    get_tls_allocator();
     LOCK_GLOBAL();
-    systemallocator->dmm_knobs.sys_alloc_size = conf->sys_alloc_size;
+    tls_allocator->dmm_knobs.sys_alloc_size = conf->sys_alloc_size;
 #ifdef GOOD_FIT
-    systemallocator->dmm_knobs.fit_percentage = conf->fit_percentage;
+    tls_allocator->dmm_knobs.fit_percentage = conf->fit_percentage;
 #endif /* GOOD_FIT_PERCENTAGE */
 #ifdef COALESCING_VARIABLE
-    systemallocator->dmm_knobs.max_coal_size = conf->max_coal_size;
+    tls_allocator->dmm_knobs.max_coal_size = conf->max_coal_size;
 #endif /* COALESCING_VARIABLE */
 #ifdef SPLITTING_VARIABLE
-    systemallocator->dmm_knobs.min_split_size = conf->min_split_size;
+    tls_allocator->dmm_knobs.min_split_size = conf->min_split_size;
 #endif /* SPLITTING_VARIABLE */
     UNLOCK_GLOBAL();
     return 0;

src/malloc.c

@@ -28,6 +28,7 @@
 #include <inttypes.h>
 #include <assert.h>
 
+#include "tls_allocator.h"
 #include "dmmlib/debug.h"
 #include "dmmlib/lists.h"
 
@@ -47,8 +48,10 @@ void * malloc(size_t size) {
         return (void *) 0xD34D;
     }
 
+    get_tls_allocator();
+
 #ifdef WITH_KNOBS
-    sys_alloc_size = (size_t) systemallocator->dmm_knobs.sys_alloc_size;
+    sys_alloc_size = (size_t) tls_allocator->dmm_knobs.sys_alloc_size;
 #else /* WITH_KNOBS */
     sys_alloc_size = SYS_ALLOC_SIZE;
 #endif /* WITH_KNOBS */
@@ -74,7 +77,7 @@ void * malloc(size_t size) {
         ptr = NULL;
 
         /* Try to find a raw block available for allocation */
-        SLIST_FOREACH(raw_block, &systemallocator->rb_head, pointers) {
+        SLIST_FOREACH(raw_block, &tls_allocator->rb_head, pointers) {
 #ifdef TRYLOCK_ON_MALLOC
             if(TRYLOCK_RAW_BLOCK(raw_block) == 0) {
 #else /* TRYLOCK_ON_MALLOC */
@@ -105,7 +108,7 @@ void * malloc(size_t size) {
             if(new_raw_block != NULL) {
                 LOCK_GLOBAL();
                 LOCK_RAW_BLOCK(new_raw_block);
-                SLIST_INSERT_HEAD(&systemallocator->rb_head, new_raw_block,
+                SLIST_INSERT_HEAD(&tls_allocator->rb_head, new_raw_block,
                         pointers);
                 UNLOCK_GLOBAL();
 

src/memalign.c

@@ -33,6 +33,7 @@
 #include "dmmlib/config.h"
 #include "dmmlib/dmmlib.h"
 
+#include "tls_allocator.h"
 #include "memcpy.h"
 
 #include "default_rb.h"
@@ -67,8 +68,10 @@ void *memalign(size_t alignment, size_t size) {
     if(2 * (size + extra_size) < SYS_ALLOC_SIZE - sizeof(raw_block_header_t) -
             sizeof(freelist_rb_t)) {
 
+        get_tls_allocator();
+
         /* Search the available freelist-organized raw blocks */
-        SLIST_FOREACH(raw_block, &systemallocator->rb_head, pointers) {
+        SLIST_FOREACH(raw_block, &tls_allocator->rb_head, pointers) {
 #ifdef TRYLOCK_ON_MALLOC
             if(TRYLOCK_RAW_BLOCK(raw_block) == 0) {
 #else /* TRYLOCK_ON_MALLOC */
@@ -93,7 +96,7 @@ void *memalign(size_t alignment, size_t size) {
             if(raw_block != NULL) {
                 LOCK_GLOBAL();
                 LOCK_RAW_BLOCK(raw_block);
-                SLIST_INSERT_HEAD(&systemallocator->rb_head, raw_block, pointers);
+                SLIST_INSERT_HEAD(&tls_allocator->rb_head, raw_block, pointers);
                 UNLOCK_GLOBAL();
 
                 encapsulated_rb = (DEFAULT_RB_T *)

src/parse_env.c

@@ -32,7 +32,7 @@
 #include "dmmlib_trace.h"
 #include "parse_env.h"
 
-void parse_env(void) {
+void parse_env(allocator_t *allocator) {
     const char* env;
 
 #ifdef WITH_MEM_TRACE
@@ -72,7 +72,7 @@ void parse_env(void) {
         char *end;
         double good_fit_perc = strtod(env, &end);
         if(good_fit_perc != 0) {
-            systemallocator->dmm_knobs.fit_percentage = good_fit_perc;
+            allocator->dmm_knobs.fit_percentage = good_fit_perc;
         }
     }
 #endif /* WITH_KNOBS && GOOD_FIT */
@@ -84,7 +84,7 @@ void parse_env(void) {
         char *end;
         uintmax_t num = strtoumax(env, &end, 10);
         if(num != UINTMAX_MAX) {
-            systemallocator->dmm_knobs.sys_alloc_size = (double) num;
+            allocator->dmm_knobs.sys_alloc_size = (double) num;
         }
     }
 #endif /* WITH_KNOBS */

src/realloc.c

@@ -25,6 +25,7 @@
 
 #include "dmmlib/dmmlib.h"
 
+#include "tls_allocator.h"
 #include "dmmlib/debug.h"
 
 #include "locks.h"
@@ -117,8 +118,9 @@ void * realloc(void *ptr, size_t size) {
                 UNLOCK_RAW_BLOCK(owner_raw_block);
 
 #ifdef WITH_ALLOCATOR_STATS
+                get_tls_allocator();
                 LOCK_GLOBAL();
-                systemallocator->dmm_stats.total_mem_allocated -= remaining_size;
+                tls_allocator->dmm_stats.total_mem_allocated -= remaining_size;
                 UNLOCK_GLOBAL();
 #endif /* WITH_ALLOCATOR_STATS */
 
@@ -156,7 +158,7 @@ void * realloc(void *ptr, size_t size) {
 
 #ifdef WITH_ALLOCATOR_STATS
                 LOCK_GLOBAL();
-                systemallocator->dmm_stats.total_mem_allocated += new_block->size
+                tls_allocator->dmm_stats.total_mem_allocated += new_block->size
                     - owner_raw_block->size;
                 UNLOCK_GLOBAL();
 #endif /* WITH_ALLOCATOR_STATS */

src/tls_allocator.c

@@ -0,0 +1,29 @@
+#include <pthread.h>
+
+#include <dmmlib/lists.h>
+#include "tls_allocator.h"
+
+void push_to_destructed_list(void) {
+    pthread_mutex_lock(&destructed_list.lock);
+    SLIST_INSERT_HEAD(&destructed_list.head, tls_allocator, pointer);
+    pthread_mutex_unlock(&destructed_list.lock);
+}
+
+void get_tls_allocator(void) {
+    if(__builtin_expect(!!(tls_allocator == NULL), 0)) {
+        if(pthread_mutex_trylock(&destructed_list.lock) == 0) {
+            tls_allocator = SLIST_FIRST(&destructed_list.head);
+            if(tls_allocator == NULL) {
+                tls_allocator = initialize_allocator();
+            } else {
+                SLIST_REMOVE_HEAD(&destructed_list.head, pointer);
+            }
+            pthread_mutex_unlock(&destructed_list.lock);
+        } else {
+            tls_allocator = initialize_allocator();
+        }
+        pthread_key_create(&destruction_key,
+                (void (*)(void *)) push_to_destructed_list);
+        pthread_setspecific(destruction_key, NULL);
+    }
+}