memalign() implementation for free-list organised raw blocks

CMakeLists.txt

@@ -193,4 +193,5 @@ message(STATUS "Support for debug functions: " ${WITH_DEBUG})
 message(STATUS "Trace level: " ${TRACE_LEVEL})
 message(STATUS "Support for realloc(): " ${WITH_REALLOC})
 message(STATUS "Support for calloc(): " ${WITH_CALLOC})
+message(STATUS "Support for memalign(): " ${WITH_MEMALIGN})
 message(STATUS "********************************************")

DefineOptions.cmake

@@ -25,8 +25,9 @@ set(BITMAP_RESOLUTION 256 CACHE INTEGER "Choose the size of cells in bitmap-orga
 
 # Function Implementation Settings
 
-option(WITH_REALLOC "Build with realloc" ON)
-option(WITH_CALLOC "Build with calloc" ON)
+option(WITH_REALLOC "Build with realloc() support" ON)
+option(WITH_CALLOC "Build with calloc() support" ON)
+option(WITH_MEMALIGN "Build with memalign() support" ON)
 
 # Knobs Settings
 
@@ -67,6 +68,7 @@ if(P2012)
   set(WITH_SPLITTING "never")
   set(WITH_REALLOC OFF)
   set(WITH_CALLOC OFF)
+  set(WITH_MEMALIGN OFF)
   set(WITH_KNOBS OFF)
   set(STATS "none")
   set(REQUEST_SIZE_INFO OFF)
@@ -87,6 +89,7 @@ if(LEON3)
   set(WITH_SPLITTING "never")
   set(WITH_REALLOC OFF)
   set(WITH_CALLOC OFF)
+  set(WITH_MEMALIGN OFF)
   set(WITH_KNOBS OFF)
   set(STATS "none")
   set(REQUEST_SIZE_INFO OFF)
@@ -105,20 +108,20 @@ if(LINUX)
   set(ORDERING_POLICY "lifo")
   set(FITTING_POLICY "good")
   set(GOOD_FIT_PERCENTAGE 0.8)
-  set(WITH_COALESCING "variable")
+  set(WITH_COALESCING "fixed")
   set(MAX_COALESCE_SIZE 60000)
-  set(WITH_SPLITTING "variable")
+  set(WITH_SPLITTING "fixed")
   set(MIN_SPLITTING_SIZE 64)
   set(FREELIST_COALESCE_AFTER_SPLIT ON)
   set(TRACE_LEVEL 3)
   set(WITH_REALLOC ON)
   set(WITH_CALLOC ON)
-  set(WITH_KNOBS ON)
+  set(WITH_MEMALIGN ON)
+  set(WITH_KNOBS OFF)
   set(STATS "global")
   set(REQUEST_SIZE_INFO ON)
   set(WITH_SHARED_LIB ON)
   set(WITH_STATIC_LIB ON)
-  set(WITH_REALLOC ON)
   set(WITH_EXAMPLES ON)
   set(WITH_DOC ON)
 endif(LINUX)

dmm_config.h.in

@@ -120,6 +120,9 @@
 /** Support for realloc() calls */
 #cmakedefine WITH_REALLOC
 
+/** Support for memalign() calls */
+#cmakedefine WITH_MEMALIGN
+
 /** Make raw block requests for sizes that are multiple of the pagesize */
 #cmakedefine PAGESIZE_ALIGN
 

include/dmmlib/memalign.h

@@ -0,0 +1,8 @@
+#ifndef MEMALIGN_H
+#define MEMALIGN_H
+
+#include <stddef.h>
+
+int posix_memalign(void **memptr, size_t alignment, size_t size);
+
+#endif /* MEMALIGN_H */

scripts/benchmark_template.c

@@ -1,4 +1,5 @@
 #include <dmmlib/dmmlib.h>
+#include <string.h>
 
 int main(void) {
 

scripts/generate_c_testbench.py

@@ -10,6 +10,7 @@ def parse_memory_trace(input_file, output_file):
 
     malloc_pattern = re.compile(r'dmmlib - m (0x\w+) (\d+)')
     realloc_pattern = re.compile(r'dmmlib - r (0x\w+) (0x\w+) (\d+)')
+    memalign_pattern = re.compile(r'dmmlib - ma (0x\w+) (\d+) (\d+)')
     free_pattern   = re.compile(r'dmmlib - f (0x\w+)')
 
     for line in input_file:
@@ -21,6 +22,8 @@ def parse_memory_trace(input_file, output_file):
                 output_file.write(variable_declaration)
             new_line = ''.join(['    var_', active_memory_requests[malloc_match.group(1)], ' = malloc(', malloc_match.group(2), ');\n'])
             output_file.write(new_line)
+            new_line = ''.join(['    var_', active_memory_requests[malloc_match.group(1)], ' = memset(var_', active_memory_requests[malloc_match.group(1)], ', 0, ', malloc_match.group(2), ');\n'])
+            output_file.write(new_line)
 
         realloc_match = realloc_pattern.search(line)
         if realloc_match:
@@ -30,6 +33,19 @@ def parse_memory_trace(input_file, output_file):
                 output_file.write(variable_declaration)
             new_line = ''.join(['    var_', active_memory_requests[realloc_match.group(2)], ' = realloc(', realloc_match.group(1), ', ', realloc_match.group(3), ');\n'])
             output_file.write(new_line)
+            new_line = ''.join(['    var_', active_memory_requests[realloc_match.group(2)], ' = memset(var_', active_memory_requests[realloc_match.group(2)], ', 0, ', realloc_match.group(3), ');\n'])
+            output_file.write(new_line)
+
+        memalign_match = memalign_pattern.search(line)
+        if memalign_match:
+            if memalign_match.group(1) not in active_memory_requests:
+                active_memory_requests[memalign_match.group(1)] = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(6))
+                variable_declaration = ''.join(['    void *var_', active_memory_requests[memalign_match.group(1)], ';\n'])
+                output_file.write(variable_declaration)
+            new_line = ''.join(['    posix_memalign(&var_', active_memory_requests[memalign_match.group(1)], ', ', memalign_match.group(2), ', ', memalign_match.group(3), ');\n'])
+            output_file.write(new_line)
+            new_line = ''.join(['    var_', active_memory_requests[memalign_match.group(1)], ' = memset(var_', active_memory_requests[memalign_match.group(1)], ', 0, ', memalign_match.group(3), ');\n'])
+            output_file.write(new_line)
 
         free_match = free_pattern.search(line)
         if free_match:

src/CMakeLists.txt

@@ -202,6 +202,13 @@ if(WITH_CALLOC)
   set(dmmlib_SRCS ${dmmlib_SRCS} calloc.c)
 endif(WITH_CALLOC)
 
+if(WITH_MEMALIGN)
+  set(dmmlib_SRCS
+    ${dmmlib_SRCS}
+    memalign.c
+  )
+endif(WITH_MEMALIGN)
+
 if (WITH_ADAPTIVITY)
   set(dmmlib_SRCS
     ${dmmlib_SRCS}

src/malloc.c

@@ -127,6 +127,8 @@ void * malloc(size_t size) {
             unlock_global();
 
         } else {
+            /* Try to create a new raw block and allocate a memory block from
+             * there */
 
             lock_global();
             new_raw_block = create_raw_block((size_t) sys_alloc_size,

src/memalign.c

@@ -0,0 +1,228 @@
+/*
+ *   Copyright Institute of Communication and Computer Systems (ICCS) 
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ *
+ */
+
+/**
+ * @file   src/memalign.c
+ * @author Ioannis Koutras (joko@microlab.ntua.gr)
+ * @date   November 2012
+ *
+ * @brief  Implementation of memory-aligned allocation calls for
+ * freelist-organized raw blocks.
+ */
+
+#include "dmmlib/memalign.h"
+
+#include <inttypes.h>
+#include <sys/mman.h>
+#include <assert.h>
+
+#include "dmm_config.h"
+#include "dmmlib/dmmlib.h"
+#include "memcpy.h"
+#include "locks.h"
+#include "default_rb.h"
+#include "freelist/block_header_funcs.h"
+
+#ifdef WITH_ALLOCATOR_STATS
+#include "statistics.h"
+#endif /* WITH_ALLOCATOR_STATS */
+
+#include "trace.h"
+
+#ifdef BITMAP_RB_ONLY
+#error Current memory-aligned allocation implementation support only \
+freelist-organized raw blocks.
+#endif /* BITMAP_RB_ONLY */
+
+/** The function posix_memalign() allocates size bytes and places the address of
+ * the allocated memory in *memptr. The address of the allocated memory will be
+ * a multiple of alignment, which must be a power of two and a multiple of
+ * sizeof(void *). If size is 0, then posix_memalign() returns either NULL.
+ */
+int posix_memalign(void **memptr, size_t alignment, size_t size) {
+    DEFAULT_RB_T *encapsulated_rb;
+
+    *memptr = NULL;
+
+    /* Input check */
+    if(size == 0 || (alignment & 1) != 0 || alignment % sizeof(void *) != 0) {
+        return 0;
+    }
+
+    raw_block_header_t *raw_block;
+
+    /* Search the available freelist-organized raw blocks for a block whose size
+     * is size + alignment - 1 */
+    SLIST_FOREACH(raw_block, &systemallocator.rb_head, pointers) {
+        lock_raw_block(raw_block);
+        encapsulated_rb = (freelist_rb_t *) 
+            ((uintptr_t) raw_block + sizeof(raw_block_header_t));
+        *memptr = dmmlib_malloc(encapsulated_rb, size + alignment - 1);
+        unlock_raw_block(raw_block);
+
+CheckAlignment:
+
+        if(*memptr != NULL) {
+
+            /* Check that a valid pointer has been returned */
+            assert(((uintptr_t) raw_block < (uintptr_t) *memptr) &&
+                    ((uintptr_t) *memptr < (uintptr_t) raw_block +
+                     raw_block->size));
+
+            if(((uintptr_t) *memptr) % alignment == 0) {
+                /* Memory address is already aligned, no need for
+                 * modifications */
+                break;
+            }
+
+            size_t padding = (- (size_t) *memptr) & (alignment - 1);
+
+            block_header_t *block_header = get_header(*memptr);
+
+            /* Check if the currently selected memory block is the first
+             * allocated in the raw block. If it is, then there is no previous
+             * block in the data layout, so we cannot increase its size as this
+             * would conclude to wasted space. Instead of this, we try to
+             * allocate another block which is guaranteed not to be the first
+             * memory block at this point. If we find one, then we free the
+             * first allocated block and repeat the process of alignment.
+             * Otherwise, we go to the next raw block. */
+
+            size_t previous_size_availability =
+                get_previous_size_availability(block_header);
+
+            if(previous_size_availability == 1) {
+                *memptr = dmmlib_malloc(encapsulated_rb, size + alignment - 1);
+
+                dmmlib_free(encapsulated_rb, *memptr);
+
+                if(*memptr != NULL) {
+                    goto CheckAlignment;
+                } else {
+                    /* Abandon current raw block */
+                }
+            } else {
+                size_t previous_size = previous_size_availability >> 1;
+                size_t previous_availability = previous_size_availability & 1;
+                block_header_t *previous_block = (block_header_t *)
+                    ((uintptr_t) block_header - previous_size);
+
+                block_header_t *next_block = get_dlnext(encapsulated_rb,
+                        block_header);
+
+                previous_size += padding;
+                previous_block->size = (previous_size << 1) |
+                    previous_availability;
+
+                block_header->previous_size = (previous_size << 1) |
+                    previous_availability;
+
+                size_t new_size = get_size(block_header) - padding;
+
+                block_header->size = (new_size << 1) | 1;
+
+                if(next_block != NULL) {
+                    next_block->previous_size = block_header->size;
+                } else { /* The aligned memory block is the border pointer */
+                    encapsulated_rb->border_ptr = (block_header_t *)
+                        ((uintptr_t) block_header + padding);
+                }
+
+                memcpy((void *) ((uintptr_t) block_header + padding),
+                        (void *) block_header, HEADER_SIZE);
+
+                *memptr = (void *)((uintptr_t) *memptr + padding);
+
+                break;
+            }
+        }
+    }
+
+    if(*memptr == NULL) {
+
+        if( 2 * size > SYS_ALLOC_SIZE - sizeof(raw_block_header_t) -
+                sizeof(freelist_rb_t)) { /* A big block has to be created */
+
+            *memptr = (void *)create_raw_block(size + alignment - 1 +
+                    sizeof(raw_block_header_t), BIGBLOCK);
+            if(*memptr != NULL) {
+
+#ifdef WITH_DEBUG
+                lock_global();
+                SLIST_INSERT_HEAD(&systemallocator.bb_head,
+                        (raw_block_header_t *) *memptr, pointers);
+                unlock_global();
+#endif /* WITH_DEBUG */
+
+#ifdef WITH_ALLOCATOR_STATS
+                update_stats(&systemallocator.dmm_stats,
+                        MALLOC,
+#ifdef REQUEST_SIZE_INFO
+                        size,
+#endif /* REQUEST_SIZE_INFO */
+                        size + sizeof(raw_block_header_t));
+#endif /* WITH_ALLOCATOR_STATS */
+
+                *memptr = (void *)((uintptr_t) *memptr + sizeof(raw_block_header_t));
+
+                if(((uintptr_t) *memptr) % alignment != 0) {
+                    size_t padding = (- (size_t) *memptr) & (alignment - 1);
+
+                    memcpy((void *)((uintptr_t) *memptr
+                                - sizeof(raw_block_header_t) + padding),
+                                (void *)((uintptr_t) *memptr
+                                - sizeof(raw_block_header_t)),
+                                sizeof(raw_block_header_t)
+                           );
+
+                    munmap((void *)((uintptr_t) *memptr
+                                - sizeof(raw_block_header_t)),
+                            (size_t) padding);
+
+                    *memptr = (void *)((uintptr_t) *memptr + padding);
+                }
+            }
+        } else { /* Create a new raw block and use it */
+            lock_global();
+            raw_block = create_raw_block((size_t) SYS_ALLOC_SIZE,
+                    DEFAULT_RB_TYPE);
+            if(raw_block != NULL) {
+                lock_raw_block(raw_block);
+                SLIST_INSERT_HEAD(&systemallocator.rb_head, raw_block, pointers);
+                unlock_global();
+
+                encapsulated_rb = (DEFAULT_RB_T *)
+                    ((uintptr_t) raw_block + sizeof(raw_block_header_t));
+                *memptr = dmmlib_malloc(encapsulated_rb, size + alignment - 1);
+                unlock_raw_block(raw_block);
+
+                if(*memptr != NULL) {
+                    goto CheckAlignment;
+                }
+            }
+        }
+    }
+
+    /* Assert that the returned address is a multiple of alignment */
+    if(*memptr != NULL) {
+        assert((uintptr_t) *memptr % alignment == 0);
+    }
+
+    TRACE_1("dmmlib - ma %p %zu %zu\n", *memptr, alignment, size);
+
+    return 0;
+}

src/realloc.c

@@ -46,13 +46,12 @@ void * realloc(void *ptr, size_t size) {
     void *return_ptr = NULL;
 
     if(ptr == NULL) {
-        return_ptr = malloc(size);
-        goto done;
+        return malloc(size);
     }
 
     if(size == 0) {
         free(ptr);
-        goto done;
+        return NULL;
     }
 
     owner_raw_block = find_raw_block_owner(systemallocator.rb_head, ptr);
@@ -122,8 +121,7 @@ void * realloc(void *ptr, size_t size) {
                 return_ptr = (void *)
                     ((uintptr_t) new_block + sizeof(raw_block_header_t));
 
-                memcpy(return_ptr,
-                        ptr,
+                memcpy(return_ptr, ptr,
                         owner_raw_block->size - sizeof(raw_block_header_t));
 
 #ifdef WITH_DEBUG