- add 'inline' variants for omp master/omp single/omp critical

- add test cases for inline variants

include/starpu_openmp.h

@@ -57,8 +57,12 @@ extern void starpu_omp_task_region(const struct starpu_codelet * const _task_reg
 		int if_clause, int final_clause, int untied_clause, int mergeable_clause) __STARPU_OMP_NOTHROW;
 extern void starpu_omp_barrier(void) __STARPU_OMP_NOTHROW;
 extern void starpu_omp_master(void (*f)(void *arg), void *arg) __STARPU_OMP_NOTHROW;
+extern int starpu_omp_master_inline(void) __STARPU_OMP_NOTHROW;
 extern void starpu_omp_single(void (*f)(void *arg), void *arg, int nowait) __STARPU_OMP_NOTHROW;
+extern int starpu_omp_single_inline(void) __STARPU_OMP_NOTHROW;
 extern void starpu_omp_critical(void (*f)(void *arg), void *arg, const char *name) __STARPU_OMP_NOTHROW;
+extern void starpu_omp_critical_inline_begin(const char *name) __STARPU_OMP_NOTHROW;
+extern void starpu_omp_critical_inline_end(const char *name) __STARPU_OMP_NOTHROW;
 extern void starpu_omp_taskwait(void) __STARPU_OMP_NOTHROW;
 extern void starpu_omp_taskgroup(void (*f)(void *arg), void *arg) __STARPU_OMP_NOTHROW;
 extern void starpu_omp_for(void (*f)(unsigned long _first_i, unsigned long _nb_i, void *arg), void *arg, unsigned long nb_iterations, unsigned long chunk, int schedule, int ordered, int nowait) __STARPU_OMP_NOTHROW;

src/util/openmp_runtime_support.c

@@ -840,7 +840,7 @@ void starpu_omp_master(void (*f)(void *arg), void *arg)
 {
 	struct starpu_omp_task *task = STARPU_PTHREAD_GETSPECIFIC(omp_task_key);
 	struct starpu_omp_thread *thread = STARPU_PTHREAD_GETSPECIFIC(omp_thread_key);
-	/* Assume singles are performed in by the implicit tasks of a region */
+	/* Assume master is performed in by the implicit tasks of a region */
 	STARPU_ASSERT(task->is_implicit);
 	struct starpu_omp_region *region = task->owner_region;
 
@@ -850,6 +850,20 @@ void starpu_omp_master(void (*f)(void *arg), void *arg)
 	}
 }
 
+/* variant of omp_master for inlined code
+ * return !0 for the task that should perform the master section
+ * return  0 for the tasks that should not perform the master section */
+int starpu_omp_master_inline(void)
+{
+	struct starpu_omp_task *task = STARPU_PTHREAD_GETSPECIFIC(omp_task_key);
+	struct starpu_omp_thread *thread = STARPU_PTHREAD_GETSPECIFIC(omp_thread_key);
+	/* Assume master is performed in by the implicit tasks of a region */
+	STARPU_ASSERT(task->is_implicit);
+	struct starpu_omp_region *region = task->owner_region;
+
+	return thread == region->master_thread;
+}
+
 void starpu_omp_single(void (*f)(void *arg), void *arg, int nowait)
 {
 	struct starpu_omp_task *task = STARPU_PTHREAD_GETSPECIFIC(omp_task_key);
@@ -870,6 +884,22 @@ void starpu_omp_single(void (*f)(void *arg), void *arg, int nowait)
 	}
 }
 
+/* variant of omp_single for inlined code
+ * return !0 for the task that should perform the single section
+ * return  0 for the tasks that should not perform the single section
+ * wait/nowait should be handled directly by the calling code using starpu_omp_barrier */
+int starpu_omp_single_inline(void)
+{
+	struct starpu_omp_task *task = STARPU_PTHREAD_GETSPECIFIC(omp_task_key);
+	/* Assume singles are performed in by the implicit tasks of a region */
+	STARPU_ASSERT(task->is_implicit);
+	struct starpu_omp_region *region = task->owner_region;
+	int first = STARPU_BOOL_COMPARE_AND_SWAP(&region->single_id, task->single_id, task->single_id+1);
+	task->single_id++;
+
+	return first;
+}
+
 static void critical__sleep_callback(void *_critical)
 {
 	struct starpu_omp_critical *critical = _critical;
@@ -935,6 +965,82 @@ void starpu_omp_critical(void (*f)(void *arg), void *arg, const char *name)
 	_starpu_spin_unlock(&critical->lock);
 }
 
+void starpu_omp_critical_inline_begin(const char *name)
+{
+	struct starpu_omp_task *task = STARPU_PTHREAD_GETSPECIFIC(omp_task_key);
+	struct starpu_omp_critical *critical = NULL;
+	struct starpu_omp_task_link link;
+
+	if (name)
+	{
+		_starpu_spin_lock(&_global_state.named_criticals_lock);
+		HASH_FIND_STR(_global_state.named_criticals, name, critical);
+		if (critical == NULL)
+		{
+			critical = create_omp_critical_struct();
+			critical->name = name;
+			HASH_ADD_STR(_global_state.named_criticals, name, critical);
+		}
+		_starpu_spin_unlock(&_global_state.named_criticals_lock);
+	}
+	else
+	{
+		critical = _global_state.default_critical;
+	}
+
+	_starpu_spin_lock(&critical->lock);
+	while (critical->state != 0)
+	{
+		_starpu_spin_lock(&task->lock);
+		task->wait_on |= starpu_omp_task_wait_on_critical;
+		_starpu_spin_unlock(&task->lock);
+		link.task = task;
+		link.next = critical->contention_list_head;
+		critical->contention_list_head = &link;
+		_starpu_task_prepare_for_continuation_ext(0, critical__sleep_callback, critical);
+		starpu_omp_task_preempt();
+
+		/* re-acquire the spin lock */
+		_starpu_spin_lock(&critical->lock);
+	}
+	critical->state = 1;
+	_starpu_spin_unlock(&critical->lock);
+}
+
+void starpu_omp_critical_inline_end(const char *name)
+{
+	struct starpu_omp_task *task = STARPU_PTHREAD_GETSPECIFIC(omp_task_key);
+	struct starpu_omp_critical *critical = NULL;
+	struct starpu_omp_task_link link;
+
+	if (name)
+	{
+		_starpu_spin_lock(&_global_state.named_criticals_lock);
+		HASH_FIND_STR(_global_state.named_criticals, name, critical);
+		STARPU_ASSERT(critical != NULL);
+		_starpu_spin_unlock(&_global_state.named_criticals_lock);
+	}
+	else
+	{
+		critical = _global_state.default_critical;
+	}
+
+	_starpu_spin_lock(&critical->lock);
+	STARPU_ASSERT(critical->state == 1);
+	critical->state = 0;
+	if (critical->contention_list_head != NULL)
+	{
+		struct starpu_omp_task *next_task = critical->contention_list_head->task;
+		critical->contention_list_head = critical->contention_list_head->next;
+		_starpu_spin_lock(&next_task->lock);
+		STARPU_ASSERT(next_task->wait_on & starpu_omp_task_wait_on_critical);
+		next_task->wait_on &= ~starpu_omp_task_wait_on_critical;
+		_wake_up_locked_task(next_task);
+		_starpu_spin_unlock(&next_task->lock);
+	}
+	_starpu_spin_unlock(&critical->lock);
+}
+
 static void explicit_task__destroy_callback(void *_task)
 {
 	struct starpu_omp_task *task = _task;

tests/Makefile.am

@@ -227,10 +227,14 @@ noinst_PROGRAMS =				\
 	openmp/parallel_03			\
 	openmp/parallel_barrier_01		\
 	openmp/parallel_master_01		\
+	openmp/parallel_master_inline_01	\
 	openmp/parallel_single_wait_01		\
 	openmp/parallel_single_nowait_01	\
+	openmp/parallel_single_inline_01	\
 	openmp/parallel_critical_01		\
+	openmp/parallel_critical_inline_01	\
 	openmp/parallel_critical_named_01	\
+	openmp/parallel_critical_named_inline_01\
 	openmp/parallel_for_01			\
 	openmp/parallel_for_02			\
 	openmp/task_01				\
@@ -470,18 +474,30 @@ openmp_parallel_barrier_01_SOURCES = 	\
 openmp_parallel_master_01_SOURCES = 	\
 	openmp/parallel_master_01.c
 
+openmp_parallel_master_inline_01_SOURCES = 	\
+	openmp/parallel_master_inline_01.c
+
 openmp_parallel_single_wait_01_SOURCES = 	\
 	openmp/parallel_single_wait_01.c
 
 openmp_parallel_single_nowait_01_SOURCES = 	\
 	openmp/parallel_single_nowait_01.c
 
+openmp_parallel_single_inline_01_SOURCES = 	\
+	openmp/parallel_single_inline_01.c
+
 openmp_parallel_critical_01_SOURCES = 	\
 	openmp/parallel_critical_01.c
 
+openmp_parallel_critical_inline_01_SOURCES = 	\
+	openmp/parallel_critical_inline_01.c
+
 openmp_parallel_critical_named_01_SOURCES = 	\
 	openmp/parallel_critical_named_01.c
 
+openmp_parallel_critical_named_inline_01_SOURCES = 	\
+	openmp/parallel_critical_named_inline_01.c
+
 openmp_parallel_for_01_SOURCES = 	\
 	openmp/parallel_for_01.c
 

tests/openmp/parallel_critical_inline_01.c

@@ -0,0 +1,89 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2014  Inria
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <pthread.h>
+#include <starpu.h>
+#include "../helper.h"
+#include <stdio.h>
+
+#if !defined(STARPU_OPENMP)
+int main(int argc, char **argv)
+{
+	return STARPU_TEST_SKIPPED;
+}
+#else
+__attribute__((constructor))
+static void omp_constructor(void)
+{
+	int ret = starpu_omp_init();
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_omp_init");
+}
+
+__attribute__((destructor))
+static void omp_destructor(void)
+{
+	starpu_omp_shutdown();
+}
+
+void parallel_region_f(void *buffers[], void *args)
+{
+	(void) buffers;
+	(void) args;
+	int worker_id;
+	pthread_t tid;
+	tid = pthread_self();
+	worker_id = starpu_worker_get_id();
+	printf("[tid %p] task thread = %d -- parallel -->\n", (void *)tid, worker_id);
+
+	starpu_omp_critical_inline_begin(NULL);
+	printf("[tid %p] task thread = %d -- critical\n", (void *)tid, worker_id);
+	starpu_omp_critical_inline_end(NULL);
+
+	starpu_omp_critical_inline_begin(NULL);
+	printf("[tid %p] task thread = %d -- critical\n", (void *)tid, worker_id);
+	starpu_omp_critical_inline_end(NULL);
+
+	starpu_omp_critical_inline_begin(NULL);
+	printf("[tid %p] task thread = %d -- critical\n", (void *)tid, worker_id);
+	starpu_omp_critical_inline_end(NULL);
+
+	starpu_omp_critical_inline_begin(NULL);
+	printf("[tid %p] task thread = %d -- critical\n", (void *)tid, worker_id);
+	starpu_omp_critical_inline_end(NULL);
+
+	printf("[tid %p] task thread = %d -- parallel <--\n", (void *)tid, worker_id);
+}
+
+static struct starpu_codelet parallel_region_cl =
+{
+	.cpu_funcs    = { parallel_region_f, NULL },
+	.where        = STARPU_CPU,
+	.nbuffers     = 0
+
+};
+
+int
+main (int argc, char *argv[]) {
+	pthread_t tid;
+	tid = pthread_self();
+	printf("<main>\n");
+	starpu_omp_parallel_region(&parallel_region_cl, NULL);
+	printf("<main>\n");
+	starpu_omp_parallel_region(&parallel_region_cl, NULL);
+	printf("<main>\n");
+	return 0;
+}
+#endif

tests/openmp/parallel_critical_named_inline_01.c

@@ -0,0 +1,89 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2014  Inria
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <pthread.h>
+#include <starpu.h>
+#include "../helper.h"
+#include <stdio.h>
+
+#if !defined(STARPU_OPENMP)
+int main(int argc, char **argv)
+{
+	return STARPU_TEST_SKIPPED;
+}
+#else
+__attribute__((constructor))
+static void omp_constructor(void)
+{
+	int ret = starpu_omp_init();
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_omp_init");
+}
+
+__attribute__((destructor))
+static void omp_destructor(void)
+{
+	starpu_omp_shutdown();
+}
+
+void parallel_region_f(void *buffers[], void *args)
+{
+	(void) buffers;
+	(void) args;
+	int worker_id;
+	pthread_t tid;
+	tid = pthread_self();
+	worker_id = starpu_worker_get_id();
+	printf("[tid %p] task thread = %d -- parallel -->\n", (void *)tid, worker_id);
+
+	starpu_omp_critical_inline_begin("g");
+	printf("[tid %p] task thread = %d -- critical \"g\"\n", (void *)tid, worker_id);
+	starpu_omp_critical_inline_end("g");
+
+	starpu_omp_critical_inline_begin("h");
+	printf("[tid %p] task thread = %d -- critical \"h\"\n", (void *)tid, worker_id);
+	starpu_omp_critical_inline_end("h");
+
+	starpu_omp_critical_inline_begin("g");
+	printf("[tid %p] task thread = %d -- critical \"g\"\n", (void *)tid, worker_id);
+	starpu_omp_critical_inline_end("g");
+
+	starpu_omp_critical_inline_begin("h");
+	printf("[tid %p] task thread = %d -- critical \"h\"\n", (void *)tid, worker_id);
+	starpu_omp_critical_inline_end("h");
+
+	printf("[tid %p] task thread = %d -- parallel <--\n", (void *)tid, worker_id);
+}
+
+static struct starpu_codelet parallel_region_cl =
+{
+	.cpu_funcs    = { parallel_region_f, NULL },
+	.where        = STARPU_CPU,
+	.nbuffers     = 0
+
+};
+
+int
+main (int argc, char *argv[]) {
+	pthread_t tid;
+	tid = pthread_self();
+	printf("<main>\n");
+	starpu_omp_parallel_region(&parallel_region_cl, NULL);
+	printf("<main>\n");
+	starpu_omp_parallel_region(&parallel_region_cl, NULL);
+	printf("<main>\n");
+	return 0;
+}
+#endif

tests/openmp/parallel_master_inline_01.c

@@ -0,0 +1,80 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2014  Inria
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <pthread.h>
+#include <starpu.h>
+#include "../helper.h"
+#include <stdio.h>
+
+#if !defined(STARPU_OPENMP)
+int main(int argc, char **argv)
+{
+	return STARPU_TEST_SKIPPED;
+}
+#else
+__attribute__((constructor))
+static void omp_constructor(void)
+{
+	int ret = starpu_omp_init();
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_omp_init");
+}
+
+__attribute__((destructor))
+static void omp_destructor(void)
+{
+	starpu_omp_shutdown();
+}
+
+void parallel_region_f(void *buffers[], void *args)
+{
+	(void) buffers;
+	(void) args;
+	int worker_id;
+	pthread_t tid;
+	tid = pthread_self();
+	worker_id = starpu_worker_get_id();
+	printf("[tid %p] task thread = %d -- parallel -->\n", (void *)tid, worker_id);
+	if (starpu_omp_master_inline())
+		printf("[tid %p] task thread = %d -- master\n", (void *)tid, worker_id);
+	if (starpu_omp_master_inline())
+		printf("[tid %p] task thread = %d -- master\n", (void *)tid, worker_id);
+	if (starpu_omp_master_inline())
+		printf("[tid %p] task thread = %d -- master\n", (void *)tid, worker_id);
+	if (starpu_omp_master_inline())
+		printf("[tid %p] task thread = %d -- master\n", (void *)tid, worker_id);
+	printf("[tid %p] task thread = %d -- parallel <--\n", (void *)tid, worker_id);
+}
+
+static struct starpu_codelet parallel_region_cl =
+{
+	.cpu_funcs    = { parallel_region_f, NULL },
+	.where        = STARPU_CPU,
+	.nbuffers     = 0
+
+};
+
+int
+main (int argc, char *argv[]) {
+	pthread_t tid;
+	tid = pthread_self();
+	printf("<main>\n");
+	starpu_omp_parallel_region(&parallel_region_cl, NULL);
+	printf("<main>\n");
+	starpu_omp_parallel_region(&parallel_region_cl, NULL);
+	printf("<main>\n");
+	return 0;
+}
+#endif

tests/openmp/parallel_single_inline_01.c

@@ -0,0 +1,97 @@
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
+ *
+ * Copyright (C) 2014  Inria
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <pthread.h>
+#include <starpu.h>
+#include "../helper.h"
+#include <stdio.h>
+
+#if !defined(STARPU_OPENMP)
+int main(int argc, char **argv)
+{
+	return STARPU_TEST_SKIPPED;
+}
+#else
+__attribute__((constructor))
+static void omp_constructor(void)
+{
+	int ret = starpu_omp_init();
+	STARPU_CHECK_RETURN_VALUE(ret, "starpu_omp_init");
+}
+
+__attribute__((destructor))
+static void omp_destructor(void)
+{
+	starpu_omp_shutdown();
+}
+
+void parallel_region_f(void *buffers[], void *args)
+{
+	(void) buffers;
+	(void) args;
+	int worker_id;
+	pthread_t tid;
+	tid = pthread_self();
+	worker_id = starpu_worker_get_id();
+	printf("[tid %p] task thread = %d -- parallel -->\n", (void *)tid, worker_id);
+	
+	/* nowait = 0 */
+	if (starpu_omp_single_inline())
+		printf("[tid %p] task thread = %d -- single nowait\n", (void *)tid, worker_id);
+	starpu_omp_barrier();
+	if (starpu_omp_single_inline())
+		printf("[tid %p] task thread = %d -- single nowait\n", (void *)tid, worker_id);
+	starpu_omp_barrier();
+	if (starpu_omp_single_inline())
+		printf("[tid %p] task thread = %d -- single nowait\n", (void *)tid, worker_id);
+	starpu_omp_barrier();
+	if (starpu_omp_single_inline())
+		printf("[tid %p] task thread = %d -- single nowait\n", (void *)tid, worker_id);
+	starpu_omp_barrier();
+
+	/* nowait = 1 */
+	if (starpu_omp_single_inline())
+		printf("[tid %p] task thread = %d -- single nowait\n", (void *)tid, worker_id);
+	if (starpu_omp_single_inline())
+		printf("[tid %p] task thread = %d -- single nowait\n", (void *)tid, worker_id);
+	if (starpu_omp_single_inline())
+		printf("[tid %p] task thread = %d -- single nowait\n", (void *)tid, worker_id);
+	if (starpu_omp_single_inline())
+		printf("[tid %p] task thread = %d -- single nowait\n", (void *)tid, worker_id);
+
+	printf("[tid %p] task thread = %d -- parallel <--\n", (void *)tid, worker_id);
+}
+
+static struct starpu_codelet parallel_region_cl =
+{
+	.cpu_funcs    = { parallel_region_f, NULL },
+	.where        = STARPU_CPU,
+	.nbuffers     = 0
+
+};
+
+int
+main (int argc, char *argv[]) {
+	pthread_t tid;
+	tid = pthread_self();
+	printf("<main>\n");
+	starpu_omp_parallel_region(&parallel_region_cl, NULL);
+	printf("<main>\n");
+	starpu_omp_parallel_region(&parallel_region_cl, NULL);
+	printf("<main>\n");
+	return 0;
+}
+#endif