introducing slab raw blocks

include/dmmlib/allocator.h

@@ -42,6 +42,8 @@
 
 /** The allocator structure of dmmlib. */
 typedef struct allocator_s {
+    /** The head of the slab raw blocks list. */
+    struct rb_head_s slab_rb_head[32];
     /** The head of the raw blocks list. */
     struct rb_head_s rb_head;
 #ifdef HAVE_LOCKS

include/dmmlib/raw_block.h

@@ -38,6 +38,7 @@
 
 /** Enumeration of raw block's different types */
 typedef enum rb_type_en {
+    SLAB,
 #ifdef FL_RB_ONLY
     FREELIST,
 #endif /* FL_RB_ONLY */

private-include/slab.h

@@ -0,0 +1,75 @@
+/*
+ *   Copyright RubberDuck Software P.C.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ *
+ */
+
+/**
+ * \file   slab.h
+ * \author Iraklis Anagnostopoulos (irasklis@rdsoftware.eu)
+ * \date   March 2014
+ * \brief  Slab-related prototypes
+ */
+
+#ifndef DEALY_SLAB_H
+#define DEALY_SLAB_H
+#include <stddef.h>
+
+#include "dmmlib/lists.h"
+
+/** The header structure of every memory block inside a heap. */
+typedef struct slab_block_header_s {
+    SLIST_ENTRY(slab_block_header_s) pointers;
+} slab_block_header_t;
+
+/** The head element of a singly-linked list of slab_block_header_t records */
+SLIST_HEAD(slab_head_s, slab_block_header_s);
+
+/** Data structure of the required elements for a free-list organized raw block.
+ */
+typedef struct slab_rb_s {
+    size_t remaining_size; /**< The remaining size for new memory blocks. */
+    size_t handling_size; /**< The size that the slab raw block handles. */
+    slab_block_header_t *border_ptr; /**< Pointer to the memory block
+                                       initialized last. */
+    struct slab_head_s fl_head; /**< Head of the free list of
+                                     memory blocks. */
+    slab_block_header_t *fl_tail; /**< The tail of the free list. */
+} slab_rb_t;
+
+/** Initializes meta-data of a slab raw block.
+ *
+ * @param address        The address of the slab metadata
+ * @param available_size The available size of the raw block for slab
+ *                       metadata and memory blocks
+ */
+void initialize_slab(void *address, size_t available_size);
+
+/** Tries to allocate memory from a specific slab raw block.
+ *
+ * @param raw_block  Pointer to the slab raw block.
+ * @param size       Requested allocation size in bytes.
+ *
+ * @return The pointer to the memory location.
+ */
+void * slab_malloc(slab_rb_t *raw_block, size_t size);
+
+/** Frees the memory block inside of a specific slab raw block.
+ *
+ * @param raw_block  Pointer to the slab raw block.
+ * @param ptr        Pointer to the memory block to be freed.
+ */
+void slab_free(slab_rb_t *raw_block, void *ptr);
+
+#endif /* DEALY_SLAB_H */

src/CMakeLists.txt

@@ -50,6 +50,9 @@ if(RAW_BLOCKS_TYPE STREQUAL "freelist")
     freelist/malloc.c
     freelist/free.c
     freelist/initialize.c
+    slab/initialize.c
+    slab/malloc.c
+    slab/free.c
     )
 
   if(FITTING_POLICY STREQUAL "best")

src/free.c

@@ -28,6 +28,7 @@
 #include "dmmlib/dmmlib.h"
 
 #include "tls_allocator.h"
+#include "slab.h"
 #include "default_rb.h"
 #include "dmmlib_trace.h"
 #include "locks.h"
@@ -52,17 +53,19 @@ void free(void *ptr) {
 #endif /* PARSE_ENV && WITH_MEM_TRACE */
 
     owner_raw_block = find_raw_block_owner(ptr);
-    if( (uintptr_t) owner_raw_block + sizeof(raw_block_header_t) !=
-            (uintptr_t) ptr) {
-        /* It is not a big block, so act accordingly. */
+    if(owner_raw_block->type == SLAB) {
+        slab_rb_t *encapsulated_slab_rb = (slab_rb_t *)
+            ((uintptr_t) owner_raw_block + sizeof(raw_block_header_t));
+        DEALY_LOCK(owner_raw_block->lock);
+        slab_free(encapsulated_slab_rb, ptr);
+        DEALY_UNLOCK(owner_raw_block->lock);
+    } else if(owner_raw_block->type == FREELIST) {
         DEFAULT_RB_T *encapsulated_rb = (DEFAULT_RB_T *)
             ((uintptr_t) owner_raw_block + sizeof(raw_block_header_t));
-
         DEALY_LOCK(owner_raw_block->lock);
         dmmlib_free(encapsulated_rb, ptr);
         DEALY_UNLOCK(owner_raw_block->lock);
-
-    } else { // It has to be a BIGBLOCK, just munmap it
+    } else if(owner_raw_block->type == BIGBLOCK) {
         owner_raw_block = (raw_block_header_t *)
             ((uintptr_t) ptr - sizeof(raw_block_header_t));
 

src/initialize.c

@@ -46,6 +46,9 @@ allocator_t * initialize_allocator(void) {
         return NULL;
     }
 
+    for(int i=0; i < 32; i++) {
+        SLIST_INIT(&allocator->slab_rb_head[i]);
+    }
     SLIST_INIT(&allocator->rb_head);
 #ifdef HAVE_LOCKS
     INIT_DEALY_LOCK(allocator->edit_lock);

src/malloc.c

@@ -34,6 +34,7 @@
 
 #include "locks.h"
 #include "default_rb.h"
+#include "slab.h"
 
 #include "dmmlib_trace.h"
 #include "statistics.h"
@@ -56,37 +57,21 @@ void * malloc(size_t size) {
     sys_alloc_size = SYS_ALLOC_SIZE;
 #endif /* WITH_KNOBS */
 
-    /* Check if the size is appropriate for the use of a freelist-organized or
-     * bitmap-organized raw block. */
+    ptr = NULL;
 
-#ifdef BITMAP_RB_ONLY
-    /* FIXME currently the raw block size and resolution are fixed */
-    size_t bm_vector_size = BMAP_EL_SIZE * (sys_alloc_size + BMAP_EL_SIZE -
-            sizeof(raw_block_header_t) - sizeof(bitmap_rb_t)) /
-        (BMAP_EL_SIZE + BMAP_EL_SIZE_BITS * BITMAP_RESOLUTION);
+    /* Check for slab allocation */
+    if(size <= 512) {
+        size = (size + 15) & ~(size_t) 15;
+        size_t index = (size >> 4) - 1;
 
-    if(2 * size < sys_alloc_size - sizeof(raw_block_header_t) -
-            sizeof(bitmap_rb_t) - bm_vector_size) {
-#endif /* BITMAP_RB_ONLY */
+        slab_rb_t *encapsulated_slab_rb;
 
-#ifdef FL_RB_ONLY
-    if(2 * size < sys_alloc_size - sizeof(raw_block_header_t) -
-            sizeof(freelist_rb_t)) {
-#endif /* FL_RB_ONLY */
-
-        ptr = NULL;
-
-        /* Try to find a raw block available for allocation */
-        SLIST_FOREACH(raw_block, &tls_allocator->rb_head, pointers) {
-#ifdef TRYLOCK_ON_MALLOC
-            if(DEALY_TRYLOCK(raw_block->lock) == 0) {
-#else /* TRYLOCK_ON_MALLOC */
-            DEALY_LOCK(raw_block->lock);
-#endif /* TRYLOCK_ON_MALLOC */
-            encapsulated_rb = (DEFAULT_RB_T *)
+        /* Try to find a slab raw block available for allocation */
+        SLIST_FOREACH(raw_block, &tls_allocator->slab_rb_head[index],
+                pointers) {
+            encapsulated_slab_rb = (slab_rb_t *)
                 ((uintptr_t) raw_block + sizeof(raw_block_header_t));
-            ptr = dmmlib_malloc(encapsulated_rb, size);
-            DEALY_UNLOCK(raw_block->lock);
+            ptr = slab_malloc(encapsulated_slab_rb, size);
 
             if(ptr != NULL) {
                 /* Check that a valid pointer has been returned */
@@ -95,36 +80,96 @@ void * malloc(size_t size) {
                          raw_block->size + sizeof(raw_block_header_t)));
                 break;
             }
-#ifdef TRYLOCK_ON_MALLOC
-            }
-#endif /* TRYLOCK_ON_MALLOC */
         }
 
         if(ptr == NULL) {
-            /* No raw block was found, try to create a new one and allocate a
-             * memory block from there. */
-
-            new_raw_block = create_raw_block(sys_alloc_size, DEFAULT_RB_TYPE);
+            /* No slab raw block was found, try to create a new one and allocate
+             * a memory block from there. */
+            new_raw_block = create_raw_block(sys_alloc_size, SLAB);
             if(new_raw_block != NULL) {
-                SLIST_INSERT_HEAD(&tls_allocator->rb_head, new_raw_block,
-                        pointers);
-
-                encapsulated_rb = (DEFAULT_RB_T *)
+                SLIST_INSERT_HEAD(&tls_allocator->slab_rb_head[index],
+                        new_raw_block, pointers);
+                encapsulated_slab_rb = (slab_rb_t *)
                     ((uintptr_t) new_raw_block + sizeof(raw_block_header_t));
-                ptr = dmmlib_malloc(encapsulated_rb, size);
+                encapsulated_slab_rb->handling_size = size;
+                ptr = slab_malloc(encapsulated_slab_rb, size);
             }
         }
     } else {
-        /* The memory request size is too large to serve it inside a
-         * freelist-organized or bitmap-organized raw block, so a big raw block
-         * shall be used. */
+        /* Check if the size is appropriate for the use of a freelist-organized
+         * or bitmap-organized raw block. */
 
-        ptr = create_raw_block(size + sizeof(raw_block_header_t), BIGBLOCK);
+#ifdef BITMAP_RB_ONLY
+        /* FIXME currently the raw block size and resolution are fixed */
+        size_t bm_vector_size = BMAP_EL_SIZE * (sys_alloc_size + BMAP_EL_SIZE -
+                sizeof(raw_block_header_t) - sizeof(bitmap_rb_t)) /
+            (BMAP_EL_SIZE + BMAP_EL_SIZE_BITS * BITMAP_RESOLUTION);
+
+        if(2 * size < sys_alloc_size - sizeof(raw_block_header_t) -
+                sizeof(bitmap_rb_t) - bm_vector_size) {
+#endif /* BITMAP_RB_ONLY */
 
-        if(ptr != NULL) {
-            ptr = (void *)((uintptr_t) ptr + sizeof(raw_block_header_t));
+#ifdef FL_RB_ONLY
+            if(2 * size < sys_alloc_size - sizeof(raw_block_header_t) -
+                    sizeof(freelist_rb_t)) {
+#endif /* FL_RB_ONLY */
+
+                /* Try to find a raw block available for allocation */
+                SLIST_FOREACH(raw_block, &tls_allocator->rb_head, pointers) {
+#ifdef TRYLOCK_ON_MALLOC
+                    if(DEALY_TRYLOCK(raw_block->lock) == 0) {
+#else /* TRYLOCK_ON_MALLOC */
+                        DEALY_LOCK(raw_block->lock);
+#endif /* TRYLOCK_ON_MALLOC */
+                        encapsulated_rb = (DEFAULT_RB_T *)
+                            ((uintptr_t) raw_block +
+                             sizeof(raw_block_header_t));
+                        ptr = dmmlib_malloc(encapsulated_rb, size);
+                        DEALY_UNLOCK(raw_block->lock);
+
+                        if(ptr != NULL) {
+                            /* Check that a valid pointer has been returned */
+                            assert(((uintptr_t) raw_block < (uintptr_t) ptr) &&
+                                    ((uintptr_t) ptr < (uintptr_t) raw_block +
+                                     raw_block->size +
+                                     sizeof(raw_block_header_t)));
+                            break;
+                        }
+#ifdef TRYLOCK_ON_MALLOC
+                    }
+#endif /* TRYLOCK_ON_MALLOC */
+                }
+
+                if(ptr == NULL) {
+                    /* No raw block was found, try to create a new one and
+                     * allocate a memory block from there. */
+
+                    new_raw_block = create_raw_block(sys_alloc_size,
+                            DEFAULT_RB_TYPE);
+                    if(new_raw_block != NULL) {
+                        SLIST_INSERT_HEAD(&tls_allocator->rb_head,
+                                new_raw_block, pointers);
+
+                        encapsulated_rb = (DEFAULT_RB_T *)
+                            ((uintptr_t) new_raw_block +
+                             sizeof(raw_block_header_t));
+                        ptr = dmmlib_malloc(encapsulated_rb, size);
+                    }
+                }
+            } else {
+                /* The memory request size is too large to serve it inside a
+                 * freelist-organized or bitmap-organized raw block, so a big
+                 * raw block shall be used. */
+
+                ptr = create_raw_block(size + sizeof(raw_block_header_t),
+                        BIGBLOCK);
+
+                if(ptr != NULL) {
+                    ptr = (void *)((uintptr_t) ptr +
+                            sizeof(raw_block_header_t));
+                }
+            }
         }
-    }
 
     MEM_TRACE("dmmlib - m %p %zu\n", ptr, size);
 

src/memalign.c

@@ -40,6 +40,7 @@
 #include "dmmlib_trace.h"
 #include "locks.h"
 #include "statistics.h"
+#include "padding.h"
 
 #ifdef BITMAP_RB_ONLY
 #error Current memory-aligned allocation implementation supports only \
@@ -58,7 +59,7 @@ void *memalign(size_t alignment, size_t size) {
 
     /* Verify that alignment is a power of two */
     assert((alignment && !(alignment & (alignment - 1))) != 0);
-
+    size = req_padding(size);
     memptr = NULL;
 
     /* extra_size denotes the worst-case scenario. Because memcpy() is used, at

src/raw_block.c

@@ -21,6 +21,8 @@
 #include <sys/mman.h>
 
 #include "request_memory.h"
+
+#include "slab.h"
 #ifdef FL_RB_ONLY
 #include "dmmlib/freelist/freelist_rb.h"
 #include "dmmlib/freelist/initialize.h"
@@ -73,10 +75,16 @@ raw_block_header_t *create_raw_block(size_t raw_block_size, rb_type type) {
     INIT_DEALY_LOCK(ptr->lock);
     DEALY_LOCK(ptr->lock);
 
+    ptr->type = type;
     ptr->size = raw_block_size + SYS_ALLOC_SIZE - 1 - padding;
 
     switch(type) {
 
+        case SLAB:
+            initialize_slab((char *) ptr + sizeof(raw_block_header_t),
+                    raw_block_size - sizeof(raw_block_header_t));
+	    break;
+
 #ifdef FL_RB_ONLY
         case FREELIST:
             initialize_freelist((char *) ptr + sizeof(raw_block_header_t),
@@ -135,6 +143,7 @@ raw_block_header_t *create_raw_block(size_t raw_block_size, rb_type type) {
             ptr->requested_size = requested_size;
 #endif /* REQUEST_SIZE_INFO */
             break;
+
     }
 
     DEALY_UNLOCK(ptr->lock);

src/realloc.c

@@ -29,6 +29,7 @@
 #include "dmmlib/debug.h"
 
 #include "locks.h"
+#include "slab.h"
 #include "default_rb.h"
 #include "dmmlib_trace.h"
 #include "other.h"
@@ -61,15 +62,23 @@ void * realloc(void *ptr, size_t size) {
 
     owner_raw_block = find_raw_block_owner(ptr);
 
-    if( (uintptr_t) owner_raw_block + sizeof(raw_block_header_t) != (uintptr_t) ptr) {
-        /* It is not a big block, so act accordingly. */
-
+    if(owner_raw_block->type == SLAB) {
+        slab_rb_t *encapsulated_slab_rb = (slab_rb_t *)
+            ((uintptr_t) owner_raw_block + sizeof(raw_block_header_t));
+        if(encapsulated_slab_rb->handling_size > size) {
+            return_ptr = ptr ;
+        } else {
+            return_ptr = malloc(size);
+            if(return_ptr != NULL) {
+                memcpy(return_ptr, ptr, encapsulated_slab_rb->handling_size);
+                free(ptr);
+            }
+        }
+    } else if(owner_raw_block->type == FREELIST) {
         DEFAULT_RB_T *encapsulated_rb = (DEFAULT_RB_T *)
             ((uintptr_t) owner_raw_block + sizeof(raw_block_header_t));
-        
         return_ptr = dmmlib_realloc(encapsulated_rb, ptr, size);
-
-    } else { /* This has to be a big block */
+    } else if(owner_raw_block->type == BIGBLOCK) {
 
         // The new size is checked and if it is smaller than the one from the
         // original request, the remaining memory is simply released. If it is

src/slab/free.c

@@ -0,0 +1,36 @@
+/*
+ *   Copyright RubberDuck Software P.C.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ *
+ */
+
+/**
+ * \file   slab/free.c
+ * \author Iraklis Anagnostopoulos (iraklis@rdsoftware.eu)
+ * \date   March 2014
+ * \brief  free() implementation for slab raw blocks
+ */
+
+#include "slab.h"
+
+void slab_free(slab_rb_t *slab_raw_block, void *ptr) {
+    slab_block_header_t *block = (slab_block_header_t *) ptr;
+    if(slab_raw_block->fl_tail != NULL) {
+        slab_raw_block->fl_tail->pointers.sle_next = block;
+        block->pointers.sle_next = NULL;
+    } else {
+        SLIST_INSERT_HEAD(&slab_raw_block->fl_head, block, pointers);
+    }
+    slab_raw_block->fl_tail = block;
+}

src/slab/initialize.c

@@ -0,0 +1,39 @@
+/*
+ *   Copyright RubberDuck Software P.C.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ *
+ */
+
+/**
+ * @file   slab/initialize.c
+ * @author Iraklis Anagnostopoulos (iraklis@rdsoftware.eu)
+ * @date   March 2014
+ *
+ * @brief  Implementation of a slab initialization.
+ */
+
+#include "dmmlib/lists.h"
+#include "slab.h"
+
+void initialize_slab(void *address, size_t available_size) {
+    slab_rb_t *fl_rb = (slab_rb_t *)address;
+
+    fl_rb->remaining_size = available_size - sizeof(slab_rb_t);
+    fl_rb->border_ptr = NULL;
+
+    SLIST_INIT(&fl_rb->fl_head);
+    fl_rb->fl_tail = NULL;
+
+    return;
+}

src/slab/malloc.c

@@ -0,0 +1,74 @@
+/*
+ *   Copyright Institute of Communication and Computer Systems (ICCS) 
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ *
+ */
+
+/**
+ * @file   slab/malloc.c
+ * @author Iraklis Anagnostopoulos
+ * @date   March 2013
+ * @brief  malloc() implementation for slab allocation
+ */
+
+
+#include <inttypes.h>
+
+#include "dmmlib/raw_block.h"
+#include "slab.h"
+
+/** Tries to allocate memory from a specific free-list organized raw block.
+ * @param slab_raw_block The pointer of the freelist-organised slab raw block.
+ * @param size      The requested size.
+ * @retval          The address of the returned memory space.
+ */
+void * slab_malloc(slab_rb_t *slab_raw_block, size_t size) {
+    slab_block_header_t *ptr = NULL;
+    raw_block_header_t *current_slab_raw_block;
+
+    if((&slab_raw_block->fl_head)->slh_first != NULL) {
+        if((&slab_raw_block->fl_head)->slh_first == slab_raw_block->fl_tail) {
+            current_slab_raw_block = (raw_block_header_t *)
+                ((uintptr_t) slab_raw_block - sizeof(raw_block_header_t));
+            DEALY_LOCK(current_slab_raw_block->lock);
+            ptr = (slab_block_header_t *) (&slab_raw_block->fl_head)->slh_first;
+            SLIST_REMOVE_HEAD(&slab_raw_block->fl_head, pointers);
+            slab_raw_block->fl_tail = (&slab_raw_block->fl_head)->slh_first;
+            DEALY_UNLOCK(current_slab_raw_block->lock);
+        } else {
+            ptr = (slab_block_header_t *) (&slab_raw_block->fl_head)->slh_first;
+            SLIST_REMOVE_HEAD(&slab_raw_block->fl_head, pointers);
+        }
+    } else {
+        if(size <= slab_raw_block->remaining_size) {
+            if(slab_raw_block->border_ptr == NULL) {
+                ptr = (slab_block_header_t *)((uintptr_t) slab_raw_block +
+                        sizeof(slab_rb_t));
+            } else {
+                ptr = (slab_block_header_t *)
+                    ((uintptr_t) slab_raw_block->border_ptr + size);
+            }
+
+            // Update raw block metadata
+            slab_raw_block->remaining_size -= size;
+            slab_raw_block->border_ptr = ptr;
+        }
+    }
+
+    if(ptr != NULL) {
+        return (void *) ptr;
+    } else {
+        return NULL;
+    }
+}