add resource manager source files

Makefile.am

@@ -54,6 +54,10 @@ if BUILD_STARPUFFT
 SUBDIRS += starpufft
 endif
 
+if BUILD_STARPURM
+SUBDIRS += starpurm
+endif
+
 if STARPU_BUILD_SC_HYPERVISOR
 SUBDIRS += sc_hypervisor
 endif

STARPU-VERSION

@@ -1,6 +1,6 @@
 # StarPU --- Runtime system for heterogeneous multicore architectures.
 #
-# Copyright (C) 2012                                     Inria
+# Copyright (C) 2012,2018                                Inria
 # Copyright (C) 2012,2015,2017                           CNRS
 # Copyright (C) 2012-2014                                Université de Bordeaux
 #
@@ -58,3 +58,7 @@ LIBSTARPUMPI_INTERFACE_AGE=0		# set to CURRENT - PREVIOUS interface
 LIBSOCL_INTERFACE_CURRENT=0	# increment upon ABI change
 LIBSOCL_INTERFACE_REVISION=0	# increment upon implementation change
 LIBSOCL_INTERFACE_AGE=0		# set to CURRENT - PREVIOUS interface
+
+LIBSTARPURM_INTERFACE_CURRENT=0	# increment upon ABI change
+LIBSTARPURM_INTERFACE_REVISION=0	# increment upon implementation change
+LIBSTARPURM_INTERFACE_AGE=0	# set to CURRENT - PREVIOUS interface

configure.ac

@@ -1,6 +1,6 @@
 # StarPU --- Runtime system for heterogeneous multicore architectures.
 #
-# Copyright (C) 2011-2017                                Inria
+# Copyright (C) 2011-2018                                Inria
 # Copyright (C) 2009-2018                                Université de Bordeaux
 # Copyright (C) 2010-2018                                CNRS
 # Copyright (C) 2013                                     Thibaut Lambert
@@ -52,6 +52,9 @@ AC_SUBST([LIBSTARPUMPI_INTERFACE_AGE])
 AC_SUBST([LIBSTARPUFFT_INTERFACE_CURRENT])
 AC_SUBST([LIBSTARPUFFT_INTERFACE_REVISION])
 AC_SUBST([LIBSTARPUFFT_INTERFACE_AGE])
+AC_SUBST([LIBSTARPURM_INTERFACE_CURRENT])
+AC_SUBST([LIBSTARPURM_INTERFACE_REVISION])
+AC_SUBST([LIBSTARPURM_INTERFACE_AGE])
 AC_SUBST([LIBSOCL_INTERFACE_CURRENT])
 AC_SUBST([LIBSOCL_INTERFACE_REVISION])
 AC_SUBST([LIBSOCL_INTERFACE_AGE])
@@ -3314,6 +3317,130 @@ m4_ifdef([AM_SILENT_RULES],
   AM_CONDITIONAL([STARPU_HAVE_AM111], [false]))
 
 ##########################################
+# Resource Manager                       #
+##########################################
+
+starpurm_support=no
+
+AC_ARG_ENABLE(starpurm, [AS_HELP_STRING([--enable-starpurm], [enable resource management support])],
+                    enable_starpurm=$enableval, enable_starpurm=no)
+if test "x$enable_starpurm" != xno
+then
+	starpurm_support=yes
+
+	# DLB
+	DLB_CFLAGS=""
+	DLB_LIBS=""
+	AC_ARG_ENABLE(dlb, [AS_HELP_STRING([--enable-dlb], [enable DLB support])],
+	                    enable_dlb=$enableval, enable_dlb=no)
+	if test "x$enable_dlb" != xno
+	then
+		AC_ARG_WITH(dlb-include-dir,
+			[AS_HELP_STRING([--with-dlb-include-dir=<path>],
+			[specify where DLB headers are installed])],
+			[dlb_inc_dirs="$withval"], [dlb_inc_dirs=""])
+	
+		dlb_inc_dirs="${dlb_inc_dirs} /usr/include/dlb"
+	
+		dlb_incdir_found=no
+		for dlb_incdir in $dlb_inc_dirs
+		do
+			if test -n "$dlb_incdir"
+			then
+				SAVED_CPPFLAGS="${CPPFLAGS}"
+				CPPFLAGS=-I${dlb_incdir}
+				AC_CHECK_HEADERS([dlb.h])
+				if test "$ac_cv_header_dlb_h" = "yes"
+				then
+					CPPFLAGS="-I$dlb_incdir ${SAVED_CPPFLAGS}"
+					DLB_CFLAGS="-I${dlb_incdir}"
+					dlb_incdir_found=yes
+					break
+				else
+					CPPFLAGS=${SAVED_CPPFLAGS}
+				fi
+				unset ac_cv_header_dlb_h
+			fi
+		done
+	
+		AC_ARG_WITH(dlb-lib-dir,
+			[AS_HELP_STRING([--with-dlb-lib-dir=<path>],
+			[specify where DLB libraries are installed])],
+			[dlb_lib_dirs="$withval"], [dlb_lib_dirs=""])
+	
+		dlb_lib_dirs="${dlb_lib_dirs} /usr/lib/dlb"
+	
+		dlb_libdir_found=no
+		for dlb_libdir in $dlb_lib_dirs
+		do
+			if test -n "$dlb_libdir" 
+			then
+				SAVED_LDFLAGS="${LDFLAGS}"
+				LDFLAGS=-L${dlb_libdir}
+				AC_CHECK_LIB(dlb, [DLB_Init])
+				if test "$ac_cv_lib_dlb_DLB_Init" = "yes"
+				then
+					LDFLAGS="-L${dlb_libdir} ${SAVED_LDFLAGS} ${STARPU_DLB_LDFLAGS}"
+					DLB_LIBS="-L${dlb_libdir} -ldlb"
+					dlb_libdir_found=yes
+					break
+				else
+					LDFLAGS=${SAVED_LDFLAGS}
+				fi
+				unset ac_cv_lib_dlb_DLB_Init
+			fi
+		done
+	
+		SAVED_CPPFLAGS="${CPPFLAGS}"
+		SAVED_CFLAGS="${CFLAGS}"
+		SAVED_LDFLAGS="${LDFLAGS}"
+		CPPFLAGS="$HWLOC_CPPFLAGS -D_GNU_SOURCE $CPPFLAGS"
+		CFLAGS="$HWLOC_CFLAGS $CFLAGS"
+		LIBS="$HWLOC_LIBS $LIBS"
+		# check whether libhwloc has a dedicated glibc-sched.h include for conversion with glibc cpusets
+		AC_CHECK_HEADERS([hwloc/glibc-sched.h])
+		CPPFLAGS="$SAVED_CPPFLAGS"
+		CFLAGS="$SAVED_CFLAGS"
+		LIBS="$SAVED_LIBS"
+	
+		SAVED_CPPFLAGS="${CPPFLAGS}"
+		SAVED_CFLAGS="${CFLAGS}"
+		SAVED_LDFLAGS="${LDFLAGS}"
+		CPPFLAGS="$STARPU_CPPFLAGS $CPPFLAGS"
+		CFLAGS="$STARPU_CFLAGS $CFLAGS"
+		LIBS="$STARPU_LIBS $LIBS"
+		# check if StarPU implements starpu_worker_set_going_to_sleep_callback()
+		AC_CHECK_FUNC([starpu_worker_set_going_to_sleep_callback],AC_DEFINE([STARPURM_STARPU_HAVE_WORKER_CALLBACKS], [1], [Define to 1 if StarPU has support for worker callbacks.]))
+		CPPFLAGS="$SAVED_CPPFLAGS"
+		CFLAGS="$SAVED_CFLAGS"
+		LIBS="$SAVED_LIBS"
+	
+		if test "x$dlb_incdir_found" != "xyes" -o "x$dlb_libdir_found" != "xyes"
+		then
+			enable_dlb=no
+		fi
+	fi
+	
+	AC_MSG_CHECKING(whether DLB support should be enabled)
+	AC_MSG_RESULT($enable_dlb)
+	if test "x$enable_dlb" != "xno"
+	then
+		AC_DEFINE([STARPURM_HAVE_DLB], [1], [Define to 1 if dlb support is enabled.])
+	fi
+	AM_CONDITIONAL(STARPURM_HAVE_DLB, test "x$enable_dlb" != "xno")
+	AC_SUBST(DLB_CFLAGS)
+	AC_SUBST(DLB_LIBS)
+fi
+AM_CONDITIONAL(BUILD_STARPURM, [test x$starpurm_support = xyes])
+
+AC_ARG_ENABLE(starpurm-examples, [AS_HELP_STRING([--enable-starpurm-examples],
+			[enable build of StarPU Resource Manager examples])],
+			enable_starpurm_examples=$enableval, enable_starpurm_examples=no)
+AM_CONDITIONAL(BUILD_STARPURM_EXAMPLES, [test x$enable_starpurm_examples = xyes])
+
+
+
+##########################################
 # Documentation                          #
 ##########################################
 
@@ -3432,7 +3559,7 @@ done
 AC_SUBST(SOCL_VENDORS)
 
 AC_CONFIG_FILES(tests/regression/regression.sh tests/regression/profiles tests/regression/profiles.build.only)
-AC_CONFIG_HEADER(src/common/config.h include/starpu_config.h gcc-plugin/include/starpu-gcc/config.h starpu-top/config.h)
+AC_CONFIG_HEADER(src/common/config.h include/starpu_config.h gcc-plugin/include/starpu-gcc/config.h starpu-top/config.h starpurm/src/config.h starpurm/include/starpurm_config.h)
 
 SANITIZE=$(echo $CFLAGS | grep sanitize)
 AM_CONDITIONAL(STARPU_SANITIZE, test -n "$SANITIZE")
@@ -3473,6 +3600,10 @@ AC_OUTPUT([
 	starpufft/starpufft-1.1.pc
 	starpufft/starpufft-1.2.pc
 	starpufft/starpufft-1.3.pc
+	starpurm/Makefile
+	starpurm/src/Makefile
+	starpurm/examples/Makefile
+	starpurm/starpurm-1.3.pc
 	examples/Makefile
 	examples/stencil/Makefile
 	tests/Makefile
@@ -3545,6 +3676,7 @@ AC_MSG_NOTICE([
 	       MPI test suite:                                $running_mpi_check
 	       Master-Slave MPI enabled:                      $use_mpi_master_slave
 	       FFT Support:                                   $fft_support
+	       Resource Management enable:                    $starpurm_support
 	       GCC plug-in:                                   $build_gcc_plugin
 	       GCC plug-in test suite (requires GNU Guile):   $run_gcc_plugin_test_suite
 	       OpenMP runtime support enabled:                $enable_openmp

starpurm/Makefile.am

@@ -0,0 +1,44 @@
+# StarPU --- Resource Management Layer.
+#
+# Copyright (C) 2017, 2018                               Inria
+#
+# StarPU is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 2.1 of the License, or (at
+# your option) any later version.
+#
+# StarPU is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See the GNU Lesser General Public License in COPYING.LGPL for more details.
+
+SUBDIRS=src
+
+if BUILD_STARPURM_EXAMPLES
+SUBDIRS += examples
+endif
+
+pkgconfigdir = $(libdir)/pkgconfig
+pkgconfig_DATA = starpurm-1.3.pc
+
+versincludedir = $(includedir)/starpu/$(STARPU_EFFECTIVE_VERSION)
+versinclude_HEADERS = 					\
+	include/starpurm.h
+
+nodist_versinclude_HEADERS = \
+	include/starpurm_config.h
+
+recheck:
+	RET=0 ; \
+	for i in $(SUBDIRS) ; do \
+		make -C $$i recheck || RET=1 ; \
+	done ; \
+	exit $$RET
+
+showcheck:
+	RET=0 ; \
+	for i in $(SUBDIRS) ; do \
+		make -C $$i showcheck || RET=1 ; \
+	done ; \
+	exit $$RET

starpurm/dev/Session.vim

@@ -0,0 +1,435 @@
+let SessionLoad = 1
+if &cp | set nocp | endif
+let s:cpo_save=&cpo
+set cpo&vim
+nmap gx <Plug>NetrwBrowseX
+nnoremap <silent> <Plug>NetrwBrowseX :call netrw#NetrwBrowseX(expand("<cWORD>"),0)
+let &cpo=s:cpo_save
+unlet s:cpo_save
+set autoindent
+set backspace=2
+set backup
+set fileencodings=ucs-bom,utf-8,latin1
+set grepprg=grep\ -nH\ $*
+set guicursor=n-v-c:block,o:hor50,i-ci:hor15,r-cr:hor30,sm:block,a:blinkon0
+set helplang=fr
+set hidden
+set history=50
+set incsearch
+set nojoinspaces
+set laststatus=2
+set listchars=tab:>-,trail:_
+set modelines=2
+set mouse=a
+set ruler
+set scrolloff=2
+set showcmd
+set showfulltag
+set showmatch
+set showtabline=2
+set smartcase
+set spellfile=en.utf-8.add,fr.utf-8.add,cs.utf-8.add,sigles.utf-8.add,noms.utf-8.add
+set statusline=%n\ %t%w%h%q%m%r,\ %y,\ %p%%,\ %c:%l/%L,\ %o
+set suffixes=.bak,~,.swp,.o,.info,.aux,.log,.dvi,.bbl,.blg,.brf,.cb,.ind,.idx,.ilg,.inx,.out,.toc
+set tags=./tags,./TAGS,tags,TAGS,../../tags,~/SVN/StarPU/trunk/tags
+set viminfo='20,\"50
+set visualbell
+set whichwrap=b,s,<,>,[,]
+set wildmenu
+set wildmode=longest,list
+let s:so_save = &so | let s:siso_save = &siso | set so=0 siso=0
+let v:this_session=expand("<sfile>:p")
+silent only
+cd ~/local/src/INTW/INTERTWinE_RM/starpurm.gitlab/dev-examples/block_test
+if expand('%') == '' && !&modified && line('$') <= 1 && getline(1) == ''
+  let s:wipebuf = bufnr('%')
+endif
+set shortmess=aoO
+badd +87 block_test.c
+badd +405 ~/local/src/INTW/INTERTWinE_RM/starpurm.gitlab/src/starpurm.c
+badd +393 ~/SVN/StarPU/trunk/src/drivers/driver_common/driver_common.c
+badd +0 ~/SVN/StarPU/trunk/src/core/workers.c
+badd +0 ~/SVN/StarPU/trunk/src/core/sched_ctx.c
+args block_test.c
+edit ~/SVN/StarPU/trunk/src/core/sched_ctx.c
+set splitbelow splitright
+wincmd _ | wincmd |
+vsplit
+1wincmd h
+wincmd w
+wincmd _ | wincmd |
+split
+1wincmd k
+wincmd w
+set nosplitbelow
+set nosplitright
+wincmd t
+set winheight=1 winwidth=1
+exe 'vert 1resize ' . ((&columns * 90 + 90) / 180)
+exe '2resize ' . ((&lines * 27 + 28) / 57)
+exe 'vert 2resize ' . ((&columns * 89 + 90) / 180)
+exe '3resize ' . ((&lines * 26 + 28) / 57)
+exe 'vert 3resize ' . ((&columns * 89 + 90) / 180)
+argglobal
+setlocal keymap=
+setlocal noarabic
+setlocal autoindent
+setlocal nobinary
+setlocal bufhidden=
+setlocal buflisted
+setlocal buftype=
+setlocal cindent
+setlocal cinkeys=0{,0},0),:,0#,!^F,o,O,e
+setlocal cinoptions=
+setlocal cinwords=if,else,while,do,for,switch
+setlocal colorcolumn=
+setlocal comments=sO:*\ -,mO:*\ \ ,exO:*/,s1:/*,mb:*,ex:*/,://
+setlocal commentstring=/*%s*/
+setlocal complete=.,w,b,u,t,i
+setlocal concealcursor=
+setlocal conceallevel=0
+setlocal completefunc=
+setlocal nocopyindent
+setlocal cryptmethod=
+setlocal nocursorbind
+setlocal nocursorcolumn
+set cursorline
+setlocal cursorline
+setlocal define=
+setlocal dictionary=
+setlocal nodiff
+setlocal equalprg=
+setlocal errorformat=
+setlocal noexpandtab
+if &filetype != 'c'
+setlocal filetype=c
+endif
+set foldcolumn=1
+setlocal foldcolumn=1
+set nofoldenable
+setlocal nofoldenable
+setlocal foldexpr=0
+setlocal foldignore=#
+setlocal foldlevel=0
+setlocal foldmarker={{{,}}}
+setlocal foldmethod=manual
+setlocal foldminlines=1
+setlocal foldnestmax=20
+setlocal foldtext=foldtext()
+setlocal formatexpr=
+setlocal formatoptions=croql
+setlocal formatlistpat=^\\s*\\d\\+[\\]:.)}\\t\ ]\\s*
+setlocal grepprg=
+setlocal iminsert=0
+setlocal imsearch=0
+setlocal include=
+setlocal includeexpr=
+setlocal indentexpr=
+setlocal indentkeys=0{,0},:,0#,!^F,o,O,e
+setlocal noinfercase
+setlocal iskeyword=@,48-57,_,192-255
+setlocal keywordprg=
+setlocal nolinebreak
+setlocal nolisp
+set list
+setlocal list
+setlocal makeprg=
+setlocal matchpairs=(:),{:},[:]
+setlocal modeline
+setlocal modifiable
+setlocal nrformats=octal,hex
+set number
+setlocal number
+setlocal numberwidth=4
+setlocal omnifunc=ccomplete#Complete
+setlocal path=
+setlocal nopreserveindent
+setlocal nopreviewwindow
+setlocal quoteescape=\\
+setlocal noreadonly
+setlocal norelativenumber
+setlocal norightleft
+setlocal rightleftcmd=search
+setlocal noscrollbind
+setlocal shiftwidth=8
+setlocal noshortname
+setlocal nosmartindent
+setlocal softtabstop=0
+setlocal nospell
+setlocal spellcapcheck=[.?!]\\_[\\])'\"\	\ ]\\+
+setlocal spellfile=en.utf-8.add,fr.utf-8.add,cs.utf-8.add,sigles.utf-8.add,noms.utf-8.add
+setlocal spelllang=en
+setlocal statusline=
+setlocal suffixesadd=
+setlocal swapfile
+setlocal synmaxcol=3000
+if &syntax != 'c'
+setlocal syntax=c
+endif
+setlocal tabstop=8
+setlocal tags=
+setlocal textwidth=0
+setlocal thesaurus=
+setlocal noundofile
+setlocal undolevels=-123456
+setlocal nowinfixheight
+setlocal nowinfixwidth
+setlocal wrap
+setlocal wrapmargin=0
+silent! normal! zE
+let s:l = 2072 - ((2 * winheight(0) + 27) / 54)
+if s:l < 1 | let s:l = 1 | endif
+exe s:l
+normal! zt
+2072
+normal! 0
+wincmd w
+argglobal
+edit ~/SVN/StarPU/trunk/src/core/workers.c
+setlocal keymap=
+setlocal noarabic
+setlocal autoindent
+setlocal nobinary
+setlocal bufhidden=
+setlocal buflisted
+setlocal buftype=
+setlocal cindent
+setlocal cinkeys=0{,0},0),:,0#,!^F,o,O,e
+setlocal cinoptions=
+setlocal cinwords=if,else,while,do,for,switch
+setlocal colorcolumn=
+setlocal comments=sO:*\ -,mO:*\ \ ,exO:*/,s1:/*,mb:*,ex:*/,://
+setlocal commentstring=/*%s*/
+setlocal complete=.,w,b,u,t,i
+setlocal concealcursor=
+setlocal conceallevel=0
+setlocal completefunc=
+setlocal nocopyindent
+setlocal cryptmethod=
+setlocal nocursorbind
+setlocal nocursorcolumn
+set cursorline
+setlocal cursorline
+setlocal define=
+setlocal dictionary=
+setlocal nodiff
+setlocal equalprg=
+setlocal errorformat=
+setlocal noexpandtab
+if &filetype != 'c'
+setlocal filetype=c
+endif
+set foldcolumn=1
+setlocal foldcolumn=1
+set nofoldenable
+setlocal nofoldenable
+setlocal foldexpr=0
+setlocal foldignore=#
+setlocal foldlevel=0
+setlocal foldmarker={{{,}}}
+setlocal foldmethod=manual
+setlocal foldminlines=1
+setlocal foldnestmax=20
+setlocal foldtext=foldtext()
+setlocal formatexpr=
+setlocal formatoptions=croql
+setlocal formatlistpat=^\\s*\\d\\+[\\]:.)}\\t\ ]\\s*
+setlocal grepprg=
+setlocal iminsert=0
+setlocal imsearch=0
+setlocal include=
+setlocal includeexpr=
+setlocal indentexpr=
+setlocal indentkeys=0{,0},:,0#,!^F,o,O,e
+setlocal noinfercase
+setlocal iskeyword=@,48-57,_,192-255
+setlocal keywordprg=
+setlocal nolinebreak
+setlocal nolisp
+set list
+setlocal list
+setlocal makeprg=
+setlocal matchpairs=(:),{:},[:]
+setlocal modeline
+setlocal modifiable
+setlocal nrformats=octal,hex
+set number
+setlocal number
+setlocal numberwidth=4
+setlocal omnifunc=ccomplete#Complete
+setlocal path=
+setlocal nopreserveindent
+setlocal nopreviewwindow
+setlocal quoteescape=\\
+setlocal noreadonly
+setlocal norelativenumber
+setlocal norightleft
+setlocal rightleftcmd=search
+setlocal noscrollbind
+setlocal shiftwidth=8
+setlocal noshortname
+setlocal nosmartindent
+setlocal softtabstop=0
+setlocal nospell
+setlocal spellcapcheck=[.?!]\\_[\\])'\"\	\ ]\\+
+setlocal spellfile=en.utf-8.add,fr.utf-8.add,cs.utf-8.add,sigles.utf-8.add,noms.utf-8.add
+setlocal spelllang=en
+setlocal statusline=
+setlocal suffixesadd=
+setlocal swapfile
+setlocal synmaxcol=3000
+if &syntax != 'c'
+setlocal syntax=c
+endif
+setlocal tabstop=8
+setlocal tags=
+setlocal textwidth=0
+setlocal thesaurus=
+setlocal noundofile
+setlocal undolevels=-123456
+setlocal nowinfixheight
+setlocal nowinfixwidth
+setlocal wrap
+setlocal wrapmargin=0
+silent! normal! zE
+let s:l = 1585 - ((20 * winheight(0) + 13) / 27)
+if s:l < 1 | let s:l = 1 | endif
+exe s:l
+normal! zt
+1585
+normal! 0
+wincmd w
+argglobal
+edit ~/SVN/StarPU/trunk/src/drivers/driver_common/driver_common.c
+setlocal keymap=
+setlocal noarabic
+setlocal autoindent
+setlocal nobinary
+setlocal bufhidden=
+setlocal buflisted
+setlocal buftype=
+setlocal cindent
+setlocal cinkeys=0{,0},0),:,0#,!^F,o,O,e
+setlocal cinoptions=
+setlocal cinwords=if,else,while,do,for,switch
+setlocal colorcolumn=
+setlocal comments=sO:*\ -,mO:*\ \ ,exO:*/,s1:/*,mb:*,ex:*/,://
+setlocal commentstring=/*%s*/
+setlocal complete=.,w,b,u,t,i
+setlocal concealcursor=
+setlocal conceallevel=0
+setlocal completefunc=
+setlocal nocopyindent
+setlocal cryptmethod=
+setlocal nocursorbind
+setlocal nocursorcolumn
+set cursorline
+setlocal cursorline
+setlocal define=
+setlocal dictionary=
+setlocal nodiff
+setlocal equalprg=
+setlocal errorformat=
+setlocal noexpandtab
+if &filetype != 'c'
+setlocal filetype=c
+endif
+set foldcolumn=1
+setlocal foldcolumn=1
+set nofoldenable
+setlocal nofoldenable
+setlocal foldexpr=0
+setlocal foldignore=#
+setlocal foldlevel=0
+setlocal foldmarker={{{,}}}
+setlocal foldmethod=manual
+setlocal foldminlines=1
+setlocal foldnestmax=20
+setlocal foldtext=foldtext()
+setlocal formatexpr=
+setlocal formatoptions=croql
+setlocal formatlistpat=^\\s*\\d\\+[\\]:.)}\\t\ ]\\s*
+setlocal grepprg=
+setlocal iminsert=0
+setlocal imsearch=0
+setlocal include=
+setlocal includeexpr=
+setlocal indentexpr=
+setlocal indentkeys=0{,0},:,0#,!^F,o,O,e
+setlocal noinfercase
+setlocal iskeyword=@,48-57,_,192-255
+setlocal keywordprg=
+setlocal nolinebreak
+setlocal nolisp
+set list
+setlocal list
+setlocal makeprg=
+setlocal matchpairs=(:),{:},[:]
+setlocal modeline
+setlocal modifiable
+setlocal nrformats=octal,hex
+set number
+setlocal number
+setlocal numberwidth=4
+setlocal omnifunc=ccomplete#Complete
+setlocal path=
+setlocal nopreserveindent
+setlocal nopreviewwindow
+setlocal quoteescape=\\
+setlocal noreadonly
+setlocal norelativenumber
+setlocal norightleft
+setlocal rightleftcmd=search
+setlocal noscrollbind
+setlocal shiftwidth=8
+setlocal noshortname
+setlocal nosmartindent
+setlocal softtabstop=0
+setlocal nospell
+setlocal spellcapcheck=[.?!]\\_[\\])'\"\	\ ]\\+
+setlocal spellfile=en.utf-8.add,fr.utf-8.add,cs.utf-8.add,sigles.utf-8.add,noms.utf-8.add
+setlocal spelllang=en
+setlocal statusline=
+setlocal suffixesadd=
+setlocal swapfile
+setlocal synmaxcol=3000
+if &syntax != 'c'
+setlocal syntax=c
+endif
+setlocal tabstop=8
+setlocal tags=
+setlocal textwidth=0
+setlocal thesaurus=
+setlocal noundofile
+setlocal undolevels=-123456
+setlocal nowinfixheight
+setlocal nowinfixwidth
+setlocal wrap
+setlocal wrapmargin=0
+silent! normal! zE
+let s:l = 394 - ((15 * winheight(0) + 13) / 26)
+if s:l < 1 | let s:l = 1 | endif
+exe s:l
+normal! zt
+394
+normal! 033|
+wincmd w
+exe 'vert 1resize ' . ((&columns * 90 + 90) / 180)
+exe '2resize ' . ((&lines * 27 + 28) / 57)
+exe 'vert 2resize ' . ((&columns * 89 + 90) / 180)
+exe '3resize ' . ((&lines * 26 + 28) / 57)
+exe 'vert 3resize ' . ((&columns * 89 + 90) / 180)
+tabnext 1
+if exists('s:wipebuf')
+  silent exe 'bwipe ' . s:wipebuf
+endif
+unlet! s:wipebuf
+set winheight=1 winwidth=20 shortmess=filnxtToO
+let s:sx = expand("<sfile>:p:r")."x.vim"
+if file_readable(s:sx)
+  exe "source " . fnameescape(s:sx)
+endif
+let &so = s:so_save | let &siso = s:siso_save
+doautoall SessionLoadPost
+unlet SessionLoad
+" vim: set ft=vim :

starpurm/dev/block_test/block_test.c

@@ -0,0 +1,139 @@
+/* StarPURM --- StarPU Resource Management Layer.
+ *
+ * Copyright (C) 2017  Inria
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+/* This example shows a basic StarPU vector scale app on top of StarPURM with a nVidia CUDA kernel */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+#include <starpu.h>
+#include <starpurm.h>
+
+static int rm_cpu_type_id = -1;
+static int rm_nb_cpu_units = 0;
+
+static void test1();
+static void init_rm_infos(void);
+
+static int global_nb_tasks = 100;
+static const int nb_random_tests = 10;
+
+/* vector scale codelet */
+static void work_func(void *cl_buffers[], void *cl_arg)
+{
+	double timestamp = starpu_timing_now();
+	double timestamp2;
+	do
+	{
+		timestamp2 = starpu_timing_now();
+	}
+	while ((timestamp2 - timestamp) < 1e6);
+}
+
+static struct starpu_codelet work_cl =
+{
+	.cpu_funcs = {work_func},
+};
+
+/* main routines */
+static void test1()
+{
+	int i;
+	for (i=0; i<global_nb_tasks; i++)
+	{
+		int ret = starpu_task_insert(&work_cl, 
+				0);
+		assert(ret == 0);
+	}
+	starpu_task_wait_for_all();
+}
+
+static void init_rm_infos(void)
+{
+	int cpu_type = starpurm_get_device_type_id("cpu");
+	int nb_cpu_units = starpurm_get_nb_devices_by_type(cpu_type);
+	if (nb_cpu_units < 1)
+	{
+		/* No CPU unit available. */
+		exit(77);
+	}
+
+	rm_cpu_type_id = cpu_type;
+	rm_nb_cpu_units = nb_cpu_units;
+}
+
+static void disp_selected_cpuset(void)
+{
+	hwloc_cpuset_t selected_cpuset = starpurm_get_selected_cpuset();
+	int strl = hwloc_bitmap_snprintf(NULL, 0, selected_cpuset);
+	char str[strl+1];
+	hwloc_bitmap_snprintf(str, strl+1, selected_cpuset);
+	printf("selected cpuset = %s\n", str);
+}
+
+int main(int argc, char *argv[])
+{
+	srandom(time(NULL));
+	int drs_enabled;
+	if (argc > 1)
+	{
+		global_nb_tasks = atoi(argv[1]);
+	}
+	starpurm_initialize();
+	init_rm_infos();
+	printf("using default units\n");
+	disp_selected_cpuset();
+	test1();
+
+	if (rm_nb_cpu_units > 1)
+	{
+		const int nb_cpus = rm_nb_cpu_units;
+		const int half_nb_cpus = nb_cpus/2;
+		printf("nb_cpu_units = %d\n", nb_cpus);
+
+		starpurm_set_drs_enable(NULL);
+		drs_enabled = starpurm_drs_enabled_p();
+		assert(drs_enabled != 0);
+
+		printf("withdrawing %d cpus from StarPU\n", half_nb_cpus);
+		starpurm_withdraw_cpus_from_starpu(NULL, half_nb_cpus);
+		disp_selected_cpuset();
+		test1();
+
+		printf("assigning %d cpus to StarPU\n", half_nb_cpus);
+		starpurm_assign_cpus_to_starpu(NULL, half_nb_cpus);
+		disp_selected_cpuset();
+		test1();
+
+		int i;
+		for (i=0; i<nb_random_tests; i++)
+		{
+			int some_cpus = 1+ random()%nb_cpus;
+			printf("assigning exactly %d cpus to StarPU\n", some_cpus);
+			starpurm_withdraw_all_cpus_from_starpu(NULL);
+			starpurm_assign_cpus_to_starpu(NULL, some_cpus);
+			disp_selected_cpuset();
+			test1();
+		}
+
+		starpurm_set_drs_disable(NULL);
+		drs_enabled = starpurm_drs_enabled_p();
+		assert(drs_enabled == 0);
+	}
+
+	starpurm_shutdown();
+	return 0;
+}

starpurm/dev/chameleon_test/dgemm.c

@@ -0,0 +1,266 @@
+#define _GNU_SOURCE
+#include <sched.h>
+#include <stdio.h>
+#include <float.h>
+#include <mkl.h>
+#include <morse.h>
+#include <starpurm.h>
+#include <hwloc.h>
+
+static int rm_cpu_type_id = -1;
+static int rm_nb_cpu_units = 0;
+
+static void test1();
+static void init_rm_infos(void);
+
+static const int nb_random_tests = 10;
+
+static void test1()
+{
+	int i;
+}
+
+static void init_rm_infos(void)
+{
+	int cpu_type = starpurm_get_device_type_id("cpu");
+	int nb_cpu_units = starpurm_get_nb_devices_by_type(cpu_type);
+	if (nb_cpu_units < 1)
+	{
+		/* No CPU unit available. */
+		exit(77);
+	}
+
+	rm_cpu_type_id = cpu_type;
+	rm_nb_cpu_units = nb_cpu_units;
+}
+
+static void disp_selected_cpuset(void)
+{
+	hwloc_cpuset_t selected_cpuset = starpurm_get_selected_cpuset();
+	int strl = hwloc_bitmap_snprintf(NULL, 0, selected_cpuset);
+	char str[strl+1];
+	hwloc_bitmap_snprintf(str, strl+1, selected_cpuset);
+	printf("selected cpuset = %s\n", str);
+}
+
+int main( int argc, char const *argv[])
+{
+	starpurm_initialize();
+	init_rm_infos();
+	printf("using default units\n");
+	disp_selected_cpuset();
+	test1();
+	starpurm_shutdown();
+#if 0
+
+	if(argc < 6 || argc > 6)
+	{ 		
+		fprintf(stderr, "Usage: ./test_dgemm M N K TRANS_A TRANS_B\n" );
+		return 1;
+	}
+	
+	// Local variables
+	int i, j;
+	int m, n, k;
+	const char *transA_input = NULL;
+	const char *transB_input = NULL;
+	enum DDSS_TRANS transA = Trans;
+	enum DDSS_TRANS transB = Trans;
+	double alpha; 
+	double beta;
+	double error;
+	double max_error;
+	double count_error;	
+	double *A;
+	double *B;
+	double *C;
+	double *C_test;
+	struct timeval start, end;
+	double flops;
+	double flops_ddss; 
+	double flops_ref; 
+	int ret;
+	m = atoi( argv[1] );
+	n = atoi( argv[2] );
+	k = atoi( argv[3] );
+	
+	if ( strlen( argv[4] ) != 1 ) 
+	{
+		fprintf(stderr,"Illegal value of TRANS_A, TRANS_A can be T or N\n");
+		return 1;
+	}
+	transA_input = argv[4];	
+	
+	if ( strlen( argv[5] ) != 1 ) 
+	{
+		fprintf(stderr,"Illegal value of TRANS_B, TRANS_B can be T or N\n");
+		return 1;
+	}
+	transB_input = argv[5];	
+
+	// Set seed 
+	srand(time(NULL));
+
+	max_error = 1.0;
+	count_error = 0.0;
+
+	// Checking inputs
+	if ( m < 0 )
+	{
+		fprintf(stderr, "Illegal value of M, M must be >= 0\n");
+		return 1;
+	}
+	if ( n < 0 )
+	{
+		fprintf(stderr, "Illegal value of N, N must be >= 0\n");
+		return 1;
+	}
+	if ( k < 0 )
+	{
+		fprintf(stderr, "Illegal value of K, K must be >= 0\n");
+		return 1;
+	}
+
+	if ( transA_input[0] == 'T' )
+	{
+		transA = Trans;
+	}
+	else if ( transA_input[0] == 'N' )
+	{
+		transA = NoTrans;
+	}
+	else
+	{
+		fprintf(stderr, "Illegal value of TRANS_A, TRANS_A can be T or N\n");
+		return 1;
+	}
+	
+	if ( transB_input[0] == 'T' )
+	{
+		transB = Trans;
+	}
+	else if ( transB_input[0] == 'N' )
+	{
+		transB = NoTrans;
+	}
+	else
+	{
+		fprintf(stderr, "Illegal value of TRANS_B, TRANS_B can be T or N\n");
+		return 1;
+	}
+
+	// Matrices allocation
+	A = ( double * ) malloc( sizeof( double ) * m * k );
+	B = ( double * ) malloc( sizeof( double ) * k * n );
+	C = ( double * ) malloc( sizeof( double ) * m * n );
+	C_test = ( double * ) malloc( sizeof( double ) * m * n );
+
+	// Alpha and beta initialization
+	alpha = ( double ) rand() / (double) rand() + DBL_MIN;
+	beta  = ( double ) rand() / (double) rand() + DBL_MIN;
+ 
+	// Matrix A, B, C and C_test initialization
+	for ( i = 0; i < m; i++ )
+	{
+		for ( j = 0; j < n; j++ )
+		{
+			A[ i * n + j ] = ( double ) rand() / (double) rand() 
+							  + DBL_MIN;
+			B[ i * n + j ] = ( double ) rand() / (double) rand() 
+							  + DBL_MIN;
+			C[ i * n + j ] = 0.0;
+			C_test[ i * n + j ] = 0.0;
+		}
+	}
+
+	/* Test case */
+	{
+		/* pocl_starpu_init */
+		{
+			hwloc_topology_init(&topology);
+			hwloc_topology_load(topology);
+			starpurm_initialize();
+			starpurm_set_drs_enable(NULL);
+		}
+
+		/* pocl_starpu_submit_task */
+		{
+			/* GLIBC cpu_mask as supplied by POCL */
+			cpu_set_t cpu_mask;
+			CPU_ZERO(&cpu_mask);
+			CPU_SET (0, &cpu_mask);
+			CPU_SET (1, &cpu_mask);
+			CPU_SET (2, &cpu_mask);
+			CPU_SET (3, &cpu_mask);
+
+			/* Convert GLIBC cpu_mask into HWLOC cpuset */
+			hwloc_cpuset_t hwloc_cpuset = hwloc_bitmap_alloc();
+			int status = hwloc_cpuset_from_glibc_sched_affinity(topology, hwloc_cpuset, &cpu_mask, sizeof(cpu_set_t));
+			assert(status == 0);
+
+			/* Reset any unit previously allocated to StarPU */
+			starpurm_withdraw_all_cpus_from_starpu(NULL);
+			/* Enforce new cpu mask */
+			starpurm_assign_cpu_mask_to_starpu(NULL, hwloc_cpuset);
+
+			/* task function */
+			{
+				int TRANS_A = transA==NoTrans?MorseNoTrans:MorseTrans;
+				int TRANS_B = transB==NoTrans?MorseNoTrans:MorseTrans;
+				int M = m;
+				int N = n;
+				int K = k;
+				double ALPHA = alpha;
+				int LDA = k;
+				int LDB = n;
+				double BETA = beta;
+				int LDC = n;
+
+				MORSE_Init(4, 0);
+				int res = MORSE_dgemm(TRANS_A, TRANS_B, M, N, K,
+						ALPHA, A, LDA, B, LDB,
+						BETA, C, LDC);
+				MORSE_Finalize();
+			}
+
+			/* Withdraw all CPU units from StarPU */
+			starpurm_withdraw_all_cpus_from_starpu(NULL);
+
+			hwloc_bitmap_free(hwloc_cpuset);
+		}
+
+		/* pocl_starpu_shutdown() */
+		{
+			starpurm_shutdown();
+		}
+	}
+
+#if 0
+	/* Check */
+	cblas_dgemm( CblasColMajor, 
+				 ( CBLAS_TRANSPOSE ) transA,
+				 ( CBLAS_TRANSPOSE ) transB,
+									 m, n, k,
+							 		 alpha, A, k,
+							 			    B, n,
+							 		  beta, C_test, n );
+	// Error computation
+	for ( i = 0; i < m; i++ )
+	{
+		for ( j = 0; j < n; j++ )
+		{
+			error = abs( C[ i * n + j ] - C_test[ i * n + j ] );
+			if ( max_error > error )
+				max_error = error;
+			count_error += error;
+		}
+	}
+
+	fprintf(stdout, "Max. error = %1.2f\n", max_error );
+	fprintf(stdout, "Av. error = %1.2f\n", count_error / ( m * n ) );
+#endif
+#endif
+
+	return 0;
+
+}

starpurm/dev/cuda_vector_scale/vector_scale.c

@@ -0,0 +1,286 @@
+/* StarPURM --- StarPU Resource Management Layer.
+ *
+ * Copyright (C) 2017  Inria
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+/* This example shows a basic StarPU vector scale app on top of StarPURM with a nVidia CUDA kernel */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+#include <starpu.h>
+#include <starpurm.h>
+
+static int rm_cpu_type_id = -1;
+static int rm_cuda_type_id = -1;
+static int rm_nb_cpu_units = 0;
+static int rm_nb_cuda_units = 0;
+
+static void usage(void);
+static void test1(const int N);
+static void test2(const int N, const int task_mult);
+static void init_rm_infos(void);
+
+/* vector scale codelet */
+static void vector_scale_func(void *cl_buffers[], void *cl_arg)
+{
+	float scalar = -1.0;
+	int n = STARPU_VECTOR_GET_NX(cl_buffers[0]);
+	float *vector = (float *)STARPU_VECTOR_GET_PTR(cl_buffers[0]);
+	int i;
+	starpu_codelet_unpack_args(cl_arg, &scalar);
+
+	{
+		int workerid = starpu_worker_get_id();
+		hwloc_cpuset_t worker_cpuset = starpu_worker_get_hwloc_cpuset(workerid);
+		hwloc_cpuset_t check_cpuset = starpurm_get_selected_cpuset();
+#if 0
+		{
+			int strl1 = hwloc_bitmap_snprintf(NULL, 0, worker_cpuset);
+			char str1[strl1+1];
+			hwloc_bitmap_snprintf(str1, strl1+1, worker_cpuset);
+			int strl2 = hwloc_bitmap_snprintf(NULL, 0, check_cpuset);
+			char str2[strl2+1];
+			hwloc_bitmap_snprintf(str2, strl2+1, check_cpuset);
+			printf("worker[%03d] - task: vector=%p, n=%d, scalar=%lf, worker cpuset = %s, selected cpuset = %s\n", workerid, vector, n, scalar, str1, str2);
+		}
+#endif
+		hwloc_bitmap_and(check_cpuset, check_cpuset, worker_cpuset);
+		assert(!hwloc_bitmap_iszero(check_cpuset));
+		hwloc_bitmap_free(check_cpuset);
+		hwloc_bitmap_free(worker_cpuset);
+	}
+
+	for (i = 0; i < n; i++)
+	{
+		vector[i] *= scalar;
+	}
+}
+
+extern void vector_scale_cuda_func(void *cl_buffers[], void *cl_arg);
+
+static struct starpu_codelet vector_scale_cl =
+{
+	.cpu_funcs = {vector_scale_func},
+	.cuda_funcs = {vector_scale_cuda_func},
+	.cuda_flags = {STARPU_CUDA_ASYNC},
+	.nbuffers = 1
+};
+
+/* main routines */
+static void usage(void)
+{
+	fprintf(stderr, "usage: 05_vector_scale [VECTOR_SIZE]\n");
+	exit(1);
+}
+
+static void test1(const int N)
+{
+	float *vector = NULL;
+	const float scalar = 2.0;
+	starpu_data_handle_t vector_handle;
+	int ret;
+	
+	vector = malloc(N * sizeof(*vector));
+	{
+		int i;
+		for (i = 0; i < N; i++)
+		{
+			vector[i] = i;
+		}
+	}
+	starpu_vector_data_register(&vector_handle, STARPU_MAIN_RAM, (uintptr_t)vector, N, sizeof(*vector));
+
+	ret = starpu_task_insert(&vector_scale_cl, 
+			STARPU_RW, vector_handle,
+			STARPU_VALUE, &scalar, sizeof(scalar),
+			0);
+	assert(ret == 0);
+	starpu_task_wait_for_all();
+
+	starpu_data_unregister(vector_handle);
+	{
+		int i;
+		for (i = 0; i < N; i++)
+		{
+			float d_i = i;
+			if (vector[i] != d_i*scalar)
+			{
+				fprintf(stderr, "%s: check_failed, vector[%d]: %f != %f\n", __func__, i, vector[i], d_i*scalar);
+				exit(1);
+			}
+		}
+	}
+	free(vector);
+}
+
+static void test2(const int N, const int task_mult)
+{
+	float *vector = NULL;
+	const float scalar = 3.0;
+	starpu_data_handle_t vector_handle;
+	int ret;
+	
+	vector = malloc(N * sizeof(*vector));
+	{
+		int i;
+		for (i = 0; i < N; i++)
+		{
+			vector[i] = i;
+		}
+	}
+	const int nparts = (rm_nb_cpu_units+rm_nb_cuda_units) * task_mult;
+	starpu_vector_data_register(&vector_handle, STARPU_MAIN_RAM, (uintptr_t)vector, N, sizeof(*vector));
+	struct starpu_data_filter partition_filter =
+	{
+		.filter_func = starpu_vector_filter_block,
+		.nchildren = nparts
+	};
+
+	starpu_data_partition(vector_handle, &partition_filter);
+
+	{
+		int i;
+		for (i = 0; i < nparts; i++)
+		{
+			starpu_data_handle_t sub_vector_handle = starpu_data_get_sub_data(vector_handle, 1, i);
+			ret = starpu_task_insert(&vector_scale_cl, 
+					STARPU_RW, sub_vector_handle,
+					STARPU_VALUE, &scalar, sizeof(scalar),
+					0);
+			assert(ret == 0);
+		}
+	}
+	starpu_task_wait_for_all();
+	starpu_data_unpartition(vector_handle, STARPU_MAIN_RAM);
+
+	starpu_data_unregister(vector_handle);
+	{
+		int i;
+		for (i = 0; i < N; i++)
+		{
+			float d_i = i;
+			if (vector[i] != d_i*scalar)
+			{
+				fprintf(stderr, "%s: check_failed, vector[%d]: %f != %f\n", __func__, i, vector[i], d_i*scalar);
+				exit(1);
+			}
+		}
+	}
+	free(vector);
+}
+
+static void init_rm_infos(void)
+{
+	int cpu_type = starpurm_get_device_type_id("cpu");
+	int nb_cpu_units = starpurm_get_nb_devices_by_type(cpu_type);
+	if (nb_cpu_units < 1)
+	{
+		/* No CPU unit available. */
+		exit(77);
+	}
+
+	int cuda_type = starpurm_get_device_type_id("cuda");
+	int nb_cuda_units = starpurm_get_nb_devices_by_type(cuda_type);
+	if (nb_cuda_units < 1)
+	{
+		/* No CUDA unit available. */
+		exit(77);
+	}
+
+	rm_cpu_type_id = cpu_type;
+	rm_cuda_type_id = cuda_type;
+	rm_nb_cpu_units = nb_cpu_units;
+	rm_nb_cuda_units = nb_cuda_units;
+}
+
+static void disp_selected_cpuset(void)
+{
+	hwloc_cpuset_t selected_cpuset = starpurm_get_selected_cpuset();
+	int strl = hwloc_bitmap_snprintf(NULL, 0, selected_cpuset);
+	char str[strl+1];
+	hwloc_bitmap_snprintf(str, strl+1, selected_cpuset);
+	printf("selected cpuset = %s\n", str);
+}
+
+int main(int argc, char *argv[])
+{
+	int param_N = 1000000;
+	int drs_enabled;
+	if (argc > 1)
+	{
+		param_N = atoi(argv[1]);
+		if (param_N < 1)
+		{
+			usage();
+		}
+	}
+
+	starpurm_initialize();
+	init_rm_infos();
+	printf("using default units\n");
+	disp_selected_cpuset();
+	test1(param_N);
+	test2(param_N, 1);
+	test2(param_N, 10);
+	test2(param_N, 100);
+
+	if (rm_nb_cpu_units > 1 && rm_nb_cuda_units > 1)
+	{
+		const int nb_cpus = rm_nb_cpu_units;
+		const int nb_cudas = rm_nb_cuda_units;
+		const int cuda_type = rm_cuda_type_id;
+		printf("nb_cpu_units = %d\n", nb_cpus);
+		printf("nb_cuda_units = %d\n", nb_cudas);
+
+		starpurm_set_drs_enable(NULL);
+		drs_enabled = starpurm_drs_enabled_p();
+		assert(drs_enabled != 0);
+
+		printf("withdrawing %d cpus from StarPU\n", nb_cpus);
+		starpurm_withdraw_cpus_from_starpu(NULL, nb_cpus);
+		disp_selected_cpuset();
+		test2(param_N, 1);
+		test2(param_N, 10);
+		test2(param_N, 100);
+
+		printf("assigning %d cpus to StarPU\n", nb_cpus);
+		starpurm_assign_cpus_to_starpu(NULL, nb_cpus);
+		disp_selected_cpuset();
+		test2(param_N, 1);
+		test2(param_N, 10);
+		test2(param_N, 100);
+
+		printf("withdrawing %d cuda devices from StarPU\n", nb_cudas);
+		starpurm_withdraw_devices_from_starpu(NULL, cuda_type, nb_cudas);
+		disp_selected_cpuset();
+		test2(param_N, 1);
+		test2(param_N, 10);
+		test2(param_N, 100);
+
+		printf("lending %d cuda devices to StarPU\n", nb_cudas);
+		starpurm_assign_devices_to_starpu(NULL, cuda_type, nb_cudas);
+		disp_selected_cpuset();
+		test2(param_N, 1);
+		test2(param_N, 10);
+		test2(param_N, 100);
+
+		starpurm_set_drs_disable(NULL);
+		drs_enabled = starpurm_drs_enabled_p();
+		assert(drs_enabled == 0);
+	}
+
+	starpurm_shutdown();
+	return 0;
+}

starpurm/dev/cuda_vector_scale/vs_cuda_kernel.cu

@@ -0,0 +1,62 @@
+/* StarPURM --- StarPU Resource Management Layer.
+ *
+ * Copyright (C) 2017  Inria
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+/* This example shows a basic StarPU vector scale app on top of StarPURM with a nVidia CUDA kernel */
+
+#include <starpu.h>
+#include <starpurm.h>
+
+static __global__ void vector_scale_cuda_kernel(float *vector, unsigned n, float scalar)
+{
+	unsigned i = blockIdx.x*blockDim.x + threadIdx.x;
+	if (i < n)
+	{
+		vector[i] *= scalar;
+	}
+}
+
+extern "C" void vector_scale_cuda_func(void *cl_buffers[], void *cl_arg)
+{
+	float scalar = -1.0;
+	unsigned n = STARPU_VECTOR_GET_NX(cl_buffers[0]);
+	float *vector = (float *)STARPU_VECTOR_GET_PTR(cl_buffers[0]);
+	starpu_codelet_unpack_args(cl_arg, &scalar);
+
+	{
+		int workerid = starpu_worker_get_id();
+		hwloc_cpuset_t worker_cpuset = starpu_worker_get_hwloc_cpuset(workerid);
+		hwloc_cpuset_t check_cpuset = starpurm_get_selected_cpuset();
+#if 0
+		{
+			int strl1 = hwloc_bitmap_snprintf(NULL, 0, worker_cpuset);
+			char str1[strl1+1];
+			hwloc_bitmap_snprintf(str1, strl1+1, worker_cpuset);
+			int strl2 = hwloc_bitmap_snprintf(NULL, 0, check_cpuset);
+			char str2[strl2+1];
+			hwloc_bitmap_snprintf(str2, strl2+1, check_cpuset);
+			printf("worker[%03d] - task: vector=%p, n=%d, scalar=%lf, worker cpuset = %s, selected cpuset = %s\n", workerid, vector, n, scalar, str1, str2);
+		}
+#endif
+		hwloc_bitmap_and(check_cpuset, check_cpuset, worker_cpuset);
+		assert(!hwloc_bitmap_iszero(check_cpuset));
+		hwloc_bitmap_free(check_cpuset);
+		hwloc_bitmap_free(worker_cpuset);
+	}
+
+	unsigned nb_threads_per_block = 64;
+	unsigned nb_blocks = (n + nb_threads_per_block-1) / nb_threads_per_block;
+	vector_scale_cuda_kernel<<<nb_blocks,nb_threads_per_block,0,starpu_cuda_get_local_stream()>>>(vector, n, scalar);
+}

starpurm/examples/01_init_exit.c

@@ -0,0 +1,27 @@
+/* StarPURM --- StarPU Resource Management Layer.
+ *
+ * Copyright (C) 2017  Inria
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+/* This example tests the proper initialization and shutdown of StarPURM. */
+
+#include <stdio.h>
+#include <starpurm.h>
+
+int main(int argc, char *argv[])
+{
+	starpurm_initialize();
+	starpurm_shutdown();
+	return 0;
+}

starpurm/examples/02_list_units.c

@@ -0,0 +1,53 @@
+/* StarPU --- Resource Management Layer.
+ *
+ * Copyright (C) 2017, 2018                              Inria
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+/* This example lists the CPU and device units detected and managed by
+ * StarPURM. */
+
+#include <stdio.h>
+#include <starpurm.h>
+
+int main(int argc, char *argv[])
+{
+	int ids[4];
+	int i;
+	starpurm_initialize();
+	ids[0] = starpurm_get_device_type_id("cpu");
+	ids[1] = starpurm_get_device_type_id("opencl");
+	ids[2] = starpurm_get_device_type_id("cuda");
+	ids[3] = starpurm_get_device_type_id("mic");
+
+	for (i=0; i<4; i++)
+	{
+		const int id = ids[i];
+		if (id == -1)
+			continue;
+		const int nb_units = starpurm_get_nb_devices_by_type(id);
+		printf("%s: %d units\n", starpurm_get_device_type_name(id), nb_units);
+		int j;
+		for (j=0; j<nb_units; j++)
+		{
+			hwloc_cpuset_t cpuset = starpurm_get_device_worker_cpuset(id, j);
+			int strl = hwloc_bitmap_snprintf(NULL, 0, cpuset);
+			char str[strl+1];
+			hwloc_bitmap_snprintf(str, strl+1, cpuset);
+			printf(". %d: %s\n", j, str);
+			hwloc_bitmap_free(cpuset);
+		}
+	}
+	starpurm_shutdown();
+	return 0;
+}

starpurm/examples/03_cpusets.c

@@ -0,0 +1,63 @@
+/* StarPU --- Resource Management Layer.
+ *
+ * Copyright (C) 2017, 2018                              Inria
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#include <stdio.h>
+#include <starpu.h>
+#include <starpurm.h>
+
+static void disp_cpuset(const char * name, hwloc_cpuset_t cpuset)
+{
+	int strl = hwloc_bitmap_snprintf(NULL, 0, cpuset);
+	char str[strl+1];
+	hwloc_bitmap_snprintf(str, strl+1, cpuset);
+	printf(". %s: %s\n", name, str);
+}
+
+int main(int argc, char *argv[])
+{
+	starpurm_initialize();
+	int cpu_id = starpurm_get_device_type_id("cpu");
+	const int nb_cpu_units = starpurm_get_nb_devices_by_type(cpu_id);
+	if (nb_cpu_units < 1)
+	{
+		starpurm_shutdown();
+		return 77;
+	}
+	hwloc_cpuset_t cpuset;
+	cpuset = starpurm_get_device_worker_cpuset(cpu_id, 0);
+	disp_cpuset("worker cpuset", cpuset);
+	hwloc_bitmap_free(cpuset);
+
+	cpuset = starpurm_get_global_cpuset();
+	disp_cpuset("global cpuset", cpuset);
+	hwloc_bitmap_free(cpuset);
+
+	cpuset = starpurm_get_selected_cpuset();
+	disp_cpuset("selected cpuset", cpuset);
+	hwloc_bitmap_free(cpuset);
+
+	cpuset = starpurm_get_all_cpu_workers_cpuset();
+	disp_cpuset("all cpu workers cpuset", cpuset);
+	hwloc_bitmap_free(cpuset);
+
+	cpuset = starpurm_get_all_device_workers_cpuset();
+	disp_cpuset("all device workers cpuset", cpuset);
+	hwloc_bitmap_free(cpuset);
+
+	starpurm_shutdown();
+
+	return 0;
+}

starpurm/examples/04_drs_enable.c

@@ -0,0 +1,38 @@
+/* StarPU --- Resource Management Layer.
+ *
+ * Copyright (C) 2017, 2018                              Inria
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+/* This example tests the proper initialization and shutdown of StarPURM. */
+
+#include <stdio.h>
+#include <starpurm.h>
+
+int main(int argc, char *argv[])
+{
+	int drs_enabled;
+	starpurm_initialize();
+	drs_enabled = starpurm_drs_enabled_p();
+	printf("drs enabled at startup: %d\n", drs_enabled);
+
+	starpurm_set_drs_enable(NULL);
+	drs_enabled = starpurm_drs_enabled_p();
+	printf("drs state after explicit enable: %d\n", drs_enabled);
+
+	starpurm_set_drs_disable(NULL);
+	drs_enabled = starpurm_drs_enabled_p();
+	printf("drs state after explicit disable: %d\n", drs_enabled);
+	starpurm_shutdown();
+	return 0;
+}

starpurm/examples/05_vector_scale.c

@@ -0,0 +1,248 @@
+/* StarPU --- Resource Management Layer.
+ *
+ * Copyright (C) 2017, 2018                              Inria
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+/* This example shows a basic StarPU vector scale app on top of StarPURM */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+#include <starpu.h>
+#include <starpurm.h>
+
+static int rm_cpu_type_id = -1;
+static int rm_nb_cpu_units = 0;
+
+static void usage(void);
+static void test1(const int N);
+static void test2(const int N, const int task_mult);
+static void init_rm_infos(void);
+
+/* vector scale codelet */
+static void vector_scale_func(void *cl_buffers[], void *cl_arg)
+{
+	double scalar = -1.0;
+	int n = STARPU_VECTOR_GET_NX(cl_buffers[0]);
+	double *vector = (double *)STARPU_VECTOR_GET_PTR(cl_buffers[0]);
+	int i;
+	starpu_codelet_unpack_args(cl_arg, &scalar);
+
+	int workerid = starpu_worker_get_id();
+	hwloc_cpuset_t worker_cpuset = starpu_worker_get_hwloc_cpuset(workerid);
+	hwloc_cpuset_t check_cpuset = starpurm_get_selected_cpuset();
+	{
+		int strl1 = hwloc_bitmap_snprintf(NULL, 0, worker_cpuset);
+		char str1[strl1+1];
+		hwloc_bitmap_snprintf(str1, strl1+1, worker_cpuset);
+		int strl2 = hwloc_bitmap_snprintf(NULL, 0, check_cpuset);
+		char str2[strl2+1];
+		hwloc_bitmap_snprintf(str2, strl2+1, check_cpuset);
+		printf("worker[%03d] - task: vector=%p, n=%d, scalar=%lf, worker cpuset = %s, selected cpuset = %s\n", workerid, vector, n, scalar, str1, str2);
+	}
+	hwloc_bitmap_and(check_cpuset, check_cpuset, worker_cpuset);
+	assert(!hwloc_bitmap_iszero(check_cpuset));
+	hwloc_bitmap_free(check_cpuset);
+	hwloc_bitmap_free(worker_cpuset);
+
+	for (i = 0; i < n; i++)
+	{
+		vector[i] *= scalar;
+	}
+}
+
+static struct starpu_codelet vector_scale_cl =
+{
+	.cpu_funcs = {vector_scale_func},
+	.nbuffers = 1
+};
+
+/* main routines */
+static void usage(void)
+{
+	fprintf(stderr, "usage: 05_vector_scale [VECTOR_SIZE]\n");
+	exit(1);
+}
+
+static void test1(const int N)
+{
+	double *vector = NULL;
+	const double scalar = 2.0;
+	starpu_data_handle_t vector_handle;
+	int ret;
+	
+	vector = malloc(N * sizeof(*vector));
+	{
+		int i;
+		for (i = 0; i < N; i++)
+		{
+			vector[i] = i;
+		}
+	}
+	starpu_vector_data_register(&vector_handle, STARPU_MAIN_RAM, (uintptr_t)vector, N, sizeof(*vector));
+
+	ret = starpu_task_insert(&vector_scale_cl, 
+			STARPU_RW, vector_handle,
+			STARPU_VALUE, &scalar, sizeof(scalar),
+			0);
+	assert(ret == 0);
+	starpu_task_wait_for_all();
+
+	starpu_data_unregister(vector_handle);
+	{
+		int i;
+		for (i = 0; i < N; i++)
+		{
+			double d_i = i;
+			if (vector[i] != d_i*scalar)
+			{
+				fprintf(stderr, "%s: check_failed\n", __func__);
+				exit(1);
+			}
+		}
+	}
+	free(vector);
+}
+
+static void test2(const int N, const int task_mult)
+{
+	double *vector = NULL;
+	const double scalar = 3.0;
+	starpu_data_handle_t vector_handle;
+	int ret;
+	
+	vector = malloc(N * sizeof(*vector));
+	{
+		int i;
+		for (i = 0; i < N; i++)
+		{
+			vector[i] = i;
+		}
+	}
+	starpu_vector_data_register(&vector_handle, STARPU_MAIN_RAM, (uintptr_t)vector, N, sizeof(*vector));
+	struct starpu_data_filter partition_filter =
+	{
+		.filter_func = starpu_vector_filter_block,
+		.nchildren = rm_nb_cpu_units * task_mult
+	};
+
+	starpu_data_partition(vector_handle, &partition_filter);
+
+	{
+		int i;
+		for (i = 0; i < rm_nb_cpu_units*task_mult; i++)
+		{
+			starpu_data_handle_t sub_vector_handle = starpu_data_get_sub_data(vector_handle, 1, i);
+			ret = starpu_task_insert(&vector_scale_cl, 
+					STARPU_RW, sub_vector_handle,
+					STARPU_VALUE, &scalar, sizeof(scalar),
+					0);
+			assert(ret == 0);
+		}
+	}
+	starpu_task_wait_for_all();
+	starpu_data_unpartition(vector_handle, STARPU_MAIN_RAM);
+
+	starpu_data_unregister(vector_handle);
+	{
+		int i;
+		for (i = 0; i < N; i++)
+		{
+			double d_i = i;
+			if (vector[i] != d_i*scalar)
+			{
+				fprintf(stderr, "%s: check_failed\n", __func__);
+				exit(1);
+			}
+		}
+	}
+	free(vector);
+}
+
+static void init_rm_infos(void)
+{
+	int cpu_type = starpurm_get_device_type_id("cpu");
+	int nb_cpu_units = starpurm_get_nb_devices_by_type(cpu_type);
+	if (nb_cpu_units < 1)
+	{
+		/* No CPU unit available. */
+		exit(77);
+	}
+
+	rm_cpu_type_id = cpu_type;
+	rm_nb_cpu_units = nb_cpu_units;
+}
+
+int main(int argc, char *argv[])
+{
+	int param_N = 1000000;
+	int drs_enabled;
+	if (argc > 1)
+	{
+		param_N = atoi(argv[1]);
+		if (param_N < 1)
+		{
+			usage();
+		}
+	}
+
+	starpurm_initialize();
+	init_rm_infos();
+	test1(param_N);
+	test2(param_N, 1);
+	test2(param_N, 10);
+	test2(param_N, 100);
+
+	if (rm_nb_cpu_units > 1)
+	{
+		const int half_nb_cpus = rm_nb_cpu_units/2;
+		starpurm_set_drs_enable(NULL);
+		drs_enabled = starpurm_drs_enabled_p();
+		assert(drs_enabled != 0);
+
+		printf("withdrawing %d cpus from StarPU\n", half_nb_cpus);
+		starpurm_withdraw_cpus_from_starpu(NULL, half_nb_cpus);
+		test2(param_N, 1);
+		test2(param_N, 10);
+		test2(param_N, 100);
+
+		printf("assigning %d cpus to StarPU\n", half_nb_cpus);
+		starpurm_assign_cpus_to_starpu(NULL, half_nb_cpus);
+		test2(param_N, 1);
+		test2(param_N, 10);
+		test2(param_N, 100);
+
+		int i;
+		for (i = rm_nb_cpu_units-1; i > 0; i--)
+		{
+			starpurm_set_max_parallelism(NULL, i);
+			test2(param_N, 10);
+		}
+
+		printf("withdrawing all cpus from StarPU\n");
+		starpurm_withdraw_all_cpus_from_starpu(NULL);
+		printf("assigning %d cpus to StarPU\n", rm_nb_cpu_units);
+		starpurm_assign_cpus_to_starpu(NULL, rm_nb_cpu_units);
+		test2(param_N, 1);
+		test2(param_N, 10);
+		test2(param_N, 100);
+
+		starpurm_set_drs_disable(NULL);
+		drs_enabled = starpurm_drs_enabled_p();
+		assert(drs_enabled == 0);
+	}
+
+	starpurm_shutdown();
+	return 0;
+}

starpurm/examples/06_spawn.c

@@ -0,0 +1,270 @@
+/* StarPU --- Resource Management Layer.
+ *
+ * Copyright (C) 2017, 2018                              Inria
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+/* This example shows a basic StarPU vector scale app on top of StarPURM,
+ * making use of both the main RM API and the spawn_kernel_on_cpus API func */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+#include <starpu.h>
+#include <starpurm.h>
+
+static int rm_cpu_type_id = -1;
+static int rm_nb_cpu_units = 0;
+
+static void usage(void);
+static void test1(const int N);
+static void test2(const int N, const int task_mult);
+static void init_rm_infos(void);
+
+/* vector scale codelet */
+static void vector_scale_func(void *cl_buffers[], void *cl_arg)
+{
+	double scalar = -1.0;
+	int n = STARPU_VECTOR_GET_NX(cl_buffers[0]);
+	double *vector = (double *)STARPU_VECTOR_GET_PTR(cl_buffers[0]);
+	int i;
+	starpu_codelet_unpack_args(cl_arg, &scalar);
+
+	int workerid = starpu_worker_get_id();
+	hwloc_cpuset_t worker_cpuset = starpu_worker_get_hwloc_cpuset(workerid);
+	{
+		int strl1 = hwloc_bitmap_snprintf(NULL, 0, worker_cpuset);
+		char str1[strl1+1];
+		hwloc_bitmap_snprintf(str1, strl1+1, worker_cpuset);
+		printf("worker[%03d] - task: vector=%p, n=%d, scalar=%lf, worker cpuset = %s\n", workerid, vector, n, scalar, str1);
+	}
+	hwloc_bitmap_free(worker_cpuset);
+
+	for (i = 0; i < n; i++)
+	{
+		vector[i] *= scalar;
+	}
+}
+
+static struct starpu_codelet vector_scale_cl =
+{
+	.cpu_funcs = {vector_scale_func},
+	.nbuffers = 1
+};
+
+/* main routines */
+static void usage(void)
+{
+	fprintf(stderr, "usage: 05_vector_scale [VECTOR_SIZE]\n");
+	exit(1);
+}
+
+static void test1(const int N)
+{
+	double *vector = NULL;
+	const double scalar = 2.0;
+	starpu_data_handle_t vector_handle;
+	int ret;
+	
+	vector = malloc(N * sizeof(*vector));
+	{
+		int i;
+		for (i = 0; i < N; i++)
+		{
+			vector[i] = i;
+		}
+	}
+	starpu_vector_data_register(&vector_handle, STARPU_MAIN_RAM, (uintptr_t)vector, N, sizeof(*vector));
+
+	ret = starpu_task_insert(&vector_scale_cl, 
+			STARPU_RW, vector_handle,
+			STARPU_VALUE, &scalar, sizeof(scalar),
+			0);
+	assert(ret == 0);
+	starpu_task_wait_for_all();
+
+	starpu_data_unregister(vector_handle);
+	{
+		int i;
+		for (i = 0; i < N; i++)
+		{
+			double d_i = i;
+			if (vector[i] != d_i*scalar)
+			{
+				fprintf(stderr, "%s: check_failed\n", __func__);
+				exit(1);
+			}
+		}
+	}
+	free(vector);
+}
+
+static void test2(const int N, const int task_mult)
+{
+	double *vector = NULL;
+	const double scalar = 3.0;
+	starpu_data_handle_t vector_handle;
+	int ret;
+	
+	vector = malloc(N * sizeof(*vector));
+	{
+		int i;
+		for (i = 0; i < N; i++)
+		{
+			vector[i] = i;
+		}
+	}
+	starpu_vector_data_register(&vector_handle, STARPU_MAIN_RAM, (uintptr_t)vector, N, sizeof(*vector));
+	struct starpu_data_filter partition_filter =
+	{
+		.filter_func = starpu_vector_filter_block,
+		.nchildren = rm_nb_cpu_units * task_mult
+	};
+
+	starpu_data_partition(vector_handle, &partition_filter);
+
+	{
+		int i;
+		for (i = 0; i < rm_nb_cpu_units*task_mult; i++)
+		{
+			starpu_data_handle_t sub_vector_handle = starpu_data_get_sub_data(vector_handle, 1, i);
+			ret = starpu_task_insert(&vector_scale_cl, 
+					STARPU_RW, sub_vector_handle,
+					STARPU_VALUE, &scalar, sizeof(scalar),
+					0);
+			assert(ret == 0);
+		}
+	}
+	starpu_task_wait_for_all();
+	starpu_data_unpartition(vector_handle, STARPU_MAIN_RAM);
+
+	starpu_data_unregister(vector_handle);
+	{
+		int i;
+		for (i = 0; i < N; i++)
+		{
+			double d_i = i;
+			if (vector[i] != d_i*scalar)
+			{
+				fprintf(stderr, "%s: check_failed\n", __func__);
+				exit(1);
+			}
+		}
+	}
+	free(vector);
+}
+
+static void init_rm_infos(void)
+{
+	int cpu_type = starpurm_get_device_type_id("cpu");
+	int nb_cpu_units = starpurm_get_nb_devices_by_type(cpu_type);
+	if (nb_cpu_units < 1)
+	{
+		/* No CPU unit available. */
+		exit(77);
+	}
+
+	rm_cpu_type_id = cpu_type;
+	rm_nb_cpu_units = nb_cpu_units;
+}
+
+static void kernel_to_spawn(void *args)
+{
+	int param_N = *(int*)args;
+	test1(param_N);
+	test2(param_N, 1);
+	test2(param_N, 10);
+	test2(param_N, 100);
+}
+
+int main(int argc, char *argv[])
+{
+	int param_N = 1000000;
+	int drs_enabled;
+	if (argc > 1)
+	{
+		param_N = atoi(argv[1]);
+		if (param_N < 1)
+		{
+			usage();
+		}
+	}
+
+	starpurm_initialize();
+	init_rm_infos();
+	if (rm_nb_cpu_units > 1)
+	{
+		const int half_nb_cpus = rm_nb_cpu_units/2;
+		starpurm_set_drs_enable(NULL);
+		drs_enabled = starpurm_drs_enabled_p();
+		assert(drs_enabled != 0);
+
+		{
+			hwloc_cpuset_t cpu_cpuset = starpurm_get_all_cpu_workers_cpuset();
+			{
+				int strl1 = hwloc_bitmap_snprintf(NULL, 0, cpu_cpuset);
+				char str1[strl1+1];
+				hwloc_bitmap_snprintf(str1, strl1+1, cpu_cpuset);
+				printf("all cpus cpuset = %s\n", str1);
+			}
+			int first_idx = hwloc_bitmap_first(cpu_cpuset);
+			int last_idx = hwloc_bitmap_last(cpu_cpuset);
+			hwloc_cpuset_t sel_cpuset = hwloc_bitmap_alloc();
+			assert(sel_cpuset != NULL);
+			int count = 0;
+			int idx = first_idx;
+			while (idx != -1 && idx <= last_idx && count < half_nb_cpus)
+			{
+				if (hwloc_bitmap_isset(cpu_cpuset, idx))
+				{
+					hwloc_bitmap_set(sel_cpuset, idx);
+					count ++;
+				}
+				idx = hwloc_bitmap_next(cpu_cpuset, idx);
+			}
+			assert(count == half_nb_cpus);
+
+			{
+				int strl1 = hwloc_bitmap_snprintf(NULL, 0, sel_cpuset);
+				char str1[strl1+1];
+				hwloc_bitmap_snprintf(str1, strl1+1, sel_cpuset);
+				printf("spawning a kernel on cpuset = %s\n", str1);
+			}
+			starpurm_spawn_kernel_on_cpus(NULL, kernel_to_spawn, &param_N, sel_cpuset);
+
+			hwloc_bitmap_free(sel_cpuset);
+			hwloc_bitmap_free(cpu_cpuset);
+		}
+
+		printf("withdrawing %d cpus from StarPU\n", half_nb_cpus);
+		starpurm_withdraw_cpus_from_starpu(NULL, half_nb_cpus);
+		test1(param_N);
+		test2(param_N, 1);
+		test2(param_N, 10);
+		test2(param_N, 100);
+
+		printf("assigning %d cpus to StarPU\n", half_nb_cpus);
+		starpurm_assign_cpus_to_starpu(NULL, half_nb_cpus);
+		test1(param_N);
+		test2(param_N, 1);
+		test2(param_N, 10);
+		test2(param_N, 100);
+
+		starpurm_set_drs_disable(NULL);
+		drs_enabled = starpurm_drs_enabled_p();
+		assert(drs_enabled == 0);
+	}
+
+	starpurm_shutdown();
+	return 0;
+}

starpurm/examples/07_async_spawn.c

@@ -0,0 +1,314 @@
+/* StarPURM --- StarPU Resource Management Layer.
+ *
+ * Copyright (C) 2017  Inria
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+/* This example shows a basic StarPU vector scale app on top of StarPURM,
+ * making use of both the main RM API and the spawn_kernel_on_cpus API func */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+#include <starpu.h>
+#include <starpurm.h>
+#include <starpurm.h>
+#include <pthread.h>
+
+static int rm_cpu_type_id = -1;
+static int rm_nb_cpu_units = 0;
+
+static void usage(void);
+static void test1(const int N);
+static void test2(const int N, const int task_mult);
+static void init_rm_infos(void);
+
+static unsigned spawn_pending = 0;
+static pthread_mutex_t spawn_pending_mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t spawn_pending_cond;
+
+static void _inc_spawn_pending(void)
+{
+	pthread_mutex_lock(&spawn_pending_mutex);
+	assert(spawn_pending < UINT_MAX);
+	spawn_pending++;
+	pthread_mutex_unlock(&spawn_pending_mutex);
+}
+
+static void _dec_spawn_pending(void)
+{
+	pthread_mutex_lock(&spawn_pending_mutex);
+	assert(spawn_pending > 0);
+	spawn_pending--;
+	if (spawn_pending == 0)
+		pthread_cond_broadcast(&spawn_pending_cond);
+	pthread_mutex_unlock(&spawn_pending_mutex);
+}
+
+static void _wait_pending_spawns(void)
+{
+	pthread_mutex_lock(&spawn_pending_mutex);
+	while (spawn_pending > 0)
+		pthread_cond_wait(&spawn_pending_cond, &spawn_pending_mutex);
+	pthread_mutex_unlock(&spawn_pending_mutex);
+}
+
+static void spawn_callback(void *_arg)
+{
+	assert(42 == (uintptr_t)_arg);
+	_dec_spawn_pending();
+}
+
+/* vector scale codelet */
+static void vector_scale_func(void *cl_buffers[], void *cl_arg)
+{
+	double scalar = -1.0;
+	int n = STARPU_VECTOR_GET_NX(cl_buffers[0]);
+	double *vector = (double *)STARPU_VECTOR_GET_PTR(cl_buffers[0]);
+	int i;
+	starpu_codelet_unpack_args(cl_arg, &scalar);
+
+	int workerid = starpu_worker_get_id();
+	hwloc_cpuset_t worker_cpuset = starpu_worker_get_hwloc_cpuset(workerid);
+	{
+		int strl1 = hwloc_bitmap_snprintf(NULL, 0, worker_cpuset);
+		char str1[strl1+1];
+		hwloc_bitmap_snprintf(str1, strl1+1, worker_cpuset);
+		printf("worker[%03d] - task: vector=%p, n=%d, scalar=%lf, worker cpuset = %s\n", workerid, vector, n, scalar, str1);
+	}
+	hwloc_bitmap_free(worker_cpuset);
+
+	for (i = 0; i < n; i++)
+	{
+		vector[i] *= scalar;
+	}
+}
+
+static struct starpu_codelet vector_scale_cl =
+{
+	.cpu_funcs = {vector_scale_func},
+	.nbuffers = 1
+};
+
+/* main routines */
+static void usage(void)
+{
+	fprintf(stderr, "usage: 05_vector_scale [VECTOR_SIZE]\n");
+	exit(1);
+}
+
+static void test1(const int N)
+{
+	double *vector = NULL;
+	const double scalar = 2.0;
+	starpu_data_handle_t vector_handle;
+	int ret;
+	
+	vector = malloc(N * sizeof(*vector));
+	{
+		int i;
+		for (i = 0; i < N; i++)
+		{
+			vector[i] = i;
+		}
+	}
+	starpu_vector_data_register(&vector_handle, STARPU_MAIN_RAM, (uintptr_t)vector, N, sizeof(*vector));
+
+	ret = starpu_task_insert(&vector_scale_cl, 
+			STARPU_RW, vector_handle,
+			STARPU_VALUE, &scalar, sizeof(scalar),
+			0);
+	assert(ret == 0);
+	starpu_task_wait_for_all();
+
+	starpu_data_unregister(vector_handle);
+	{
+		int i;
+		for (i = 0; i < N; i++)
+		{
+			double d_i = i;
+			if (vector[i] != d_i*scalar)
+			{
+				fprintf(stderr, "%s: check_failed\n", __func__);
+				exit(1);
+			}
+		}
+	}
+	free(vector);
+}
+
+static void test2(const int N, const int task_mult)
+{
+	double *vector = NULL;
+	const double scalar = 3.0;
+	starpu_data_handle_t vector_handle;
+	int ret;
+	
+	vector = malloc(N * sizeof(*vector));
+	{
+		int i;
+		for (i = 0; i < N; i++)
+		{
+			vector[i] = i;
+		}
+	}
+	starpu_vector_data_register(&vector_handle, STARPU_MAIN_RAM, (uintptr_t)vector, N, sizeof(*vector));
+	struct starpu_data_filter partition_filter =
+	{
+		.filter_func = starpu_vector_filter_block,
+		.nchildren = rm_nb_cpu_units * task_mult
+	};
+
+	starpu_data_partition(vector_handle, &partition_filter);
+
+	{
+		int i;
+		for (i = 0; i < rm_nb_cpu_units*task_mult; i++)
+		{
+			starpu_data_handle_t sub_vector_handle = starpu_data_get_sub_data(vector_handle, 1, i);
+			ret = starpu_task_insert(&vector_scale_cl, 
+					STARPU_RW, sub_vector_handle,
+					STARPU_VALUE, &scalar, sizeof(scalar),
+					0);
+			assert(ret == 0);
+		}
+	}
+	starpu_task_wait_for_all();
+	starpu_data_unpartition(vector_handle, STARPU_MAIN_RAM);
+
+	starpu_data_unregister(vector_handle);
+	{
+		int i;
+		for (i = 0; i < N; i++)
+		{
+			double d_i = i;
+			if (vector[i] != d_i*scalar)
+			{
+				fprintf(stderr, "%s: check_failed\n", __func__);
+				exit(1);
+			}
+		}
+	}
+	free(vector);
+}
+
+static void init_rm_infos(void)
+{
+	int cpu_type = starpurm_get_device_type_id("cpu");
+	int nb_cpu_units = starpurm_get_nb_devices_by_type(cpu_type);
+	if (nb_cpu_units < 1)
+	{
+		/* No CPU unit available. */
+		exit(77);
+	}
+
+	rm_cpu_type_id = cpu_type;
+	rm_nb_cpu_units = nb_cpu_units;
+}
+
+static void kernel_to_spawn(void *args)
+{
+	int param_N = *(int*)args;
+	//test1(param_N);
+	test2(param_N, 1);
+	//test2(param_N, 10);
+	//test2(param_N, 100);
+}
+
+int main(int argc, char *argv[])
+{
+	pthread_cond_init(&spawn_pending_cond, NULL);
+	int param_N = 1000000;
+	int drs_enabled;
+	if (argc > 1)
+	{
+		param_N = atoi(argv[1]);
+		if (param_N < 1)
+		{
+			usage();
+		}
+	}
+
+	starpurm_initialize();
+	init_rm_infos();
+	if (rm_nb_cpu_units > 1)
+	{
+		const int half_nb_cpus = rm_nb_cpu_units/2;
+		starpurm_set_drs_enable(NULL);
+		drs_enabled = starpurm_drs_enabled_p();
+		assert(drs_enabled != 0);
+
+		int repeat;
+		for (repeat=0; repeat < 20; repeat++)
+		{
+			hwloc_cpuset_t cpu_cpuset = starpurm_get_all_cpu_workers_cpuset();
+			{
+				int strl1 = hwloc_bitmap_snprintf(NULL, 0, cpu_cpuset);
+				char str1[strl1+1];
+				hwloc_bitmap_snprintf(str1, strl1+1, cpu_cpuset);
+				printf("all cpus cpuset = %s\n", str1);
+			}
+			int first_idx = hwloc_bitmap_first(cpu_cpuset);
+			int last_idx = hwloc_bitmap_last(cpu_cpuset);
+			hwloc_cpuset_t sel_cpuset = hwloc_bitmap_alloc();
+			assert(sel_cpuset != NULL);
+			int count = 0;
+			int idx = first_idx;
+			while (idx != -1 && idx <= last_idx && count < half_nb_cpus)
+			{
+				if (hwloc_bitmap_isset(cpu_cpuset, idx))
+				{
+					hwloc_bitmap_set(sel_cpuset, idx);
+					count ++;
+				}
+				idx = hwloc_bitmap_next(cpu_cpuset, idx);
+			}
+			assert(count == half_nb_cpus);
+
+			{
+				int strl1 = hwloc_bitmap_snprintf(NULL, 0, sel_cpuset);
+				char str1[strl1+1];
+				hwloc_bitmap_snprintf(str1, strl1+1, sel_cpuset);
+				printf("spawning a kernel on cpuset = %s\n", str1);
+			}
+			_inc_spawn_pending();
+			starpurm_spawn_kernel_on_cpus_callback(NULL, kernel_to_spawn, &param_N, sel_cpuset, spawn_callback, (void*)(uintptr_t)42);
+
+			hwloc_bitmap_free(sel_cpuset);
+			hwloc_bitmap_free(cpu_cpuset);
+		}
+		_wait_pending_spawns();
+
+		printf("withdrawing %d cpus from StarPU\n", half_nb_cpus);
+		starpurm_withdraw_cpus_from_starpu(NULL, half_nb_cpus);
+		test1(param_N);
+		test2(param_N, 1);
+		test2(param_N, 10);
+		test2(param_N, 100);
+
+		printf("assigning %d cpus to StarPU\n", half_nb_cpus);
+		starpurm_assign_cpus_to_starpu(NULL, half_nb_cpus);
+		test1(param_N);
+		test2(param_N, 1);
+		test2(param_N, 10);
+		test2(param_N, 100);
+
+		starpurm_set_drs_disable(NULL);
+		drs_enabled = starpurm_drs_enabled_p();
+		assert(drs_enabled == 0);
+	}
+
+	starpurm_shutdown();
+	pthread_cond_destroy(&spawn_pending_cond);
+	return 0;
+}

starpurm/examples/Makefile.am

@@ -0,0 +1,48 @@
+# StarPURM --- Resource Management Layer.
+#
+# Copyright (C) 2017, 2018                               Inria
+#
+# StarPU is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 2.1 of the License, or (at
+# your option) any later version.
+#
+# StarPU is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See the GNU Lesser General Public License in COPYING.LGPL for more details.
+
+SUBDIRS =
+
+CLEANFILES = *.gcno *.gcda *.linkinfo
+
+AM_CPPFLAGS	= -I$(top_srcdir)/include -I$(top_srcdir)/src -I$(top_builddir)/src -I$(top_builddir)/include
+AM_CPPFLAGS	+= -I$(top_srcdir)/starpurm/include -I$(top_srcdir)/starpurm/src -I$(top_builddir)/starpurm/src -I$(top_builddir)/starpurm/include
+AM_CFLAGS	= -Wall -g $(HWLOC_CFLAGS) $(DLB_CFLAGS)
+LDADD	= $(top_builddir)/starpurm/src/libstarpurm-@STARPU_EFFECTIVE_VERSION@.la $(HWLOC_LIBS) $(DLB_LIBS)
+
+examplebindir = $(libdir)/starpu/examples/starpurm
+
+examplebin_PROGRAMS =
+examplebin_PROGRAMS += 01_init_exit
+examplebin_PROGRAMS += 02_list_units
+examplebin_PROGRAMS += 03_cpusets
+examplebin_PROGRAMS += 04_drs_enable
+examplebin_PROGRAMS += 05_vector_scale
+examplebin_PROGRAMS += 06_spawn
+examplebin_PROGRAMS += 07_async_spawn
+
+01_init_exit_SOURCES = 01_init_exit.c
+
+02_list_units_SOURCES = 02_list_units.c
+
+03_cpusets_SOURCES = 03_cpusets.c
+
+04_drs_enable_SOURCES = 04_drs_enable.c
+
+05_vector_scale_SOURCES = 05_vector_scale.c
+
+06_spawn_SOURCES = 06_spawn.c
+
+07_async_spawn_SOURCES = 07_async_spawn.c

starpurm/examples/Makefile.example

@@ -0,0 +1,31 @@
+# StarPURM --- StarPU Resource Management Layer.
+#
+# Copyright (C) 2017 Inria
+#
+# StarPU is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 2.1 of the License, or (at
+# your option) any later version.
+#
+# StarPU is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See the GNU Lesser General Public License in COPYING.LGPL for more details.
+
+# Example Makefile showing how to build examples from outside the StarPURM
+# tree.
+
+PRGS=01_init_exit 02_list_units
+
+CFLAGS+=$(shell pkg-config --cflags starpurm-1.3)
+LDFLAGS+=$(shell pkg-config --libs-only-L starpurm-1.3)
+
+LDLIBS += -Wl,--start-group 
+LDLIBS += $(shell pkg-config --libs-only-l starpurm-1.3)
+LDLIBS += -Wl,--end-group 
+
+.phony: all clean
+all: $(PRGS)
+clean:
+	rm -fv $(PRGS)

starpurm/include/starpurm.h

@@ -0,0 +1,145 @@
+/* StarPURM --- StarPU Resource Management Layer.
+ *
+ * Copyright (C) 2017  Inria
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#ifndef __STARPURM_H
+#define __STARPURM_H
+#include <hwloc.h>
+#include <starpurm_config.h>
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+/* type mapping */
+enum e_starpurm_drs_ret
+{
+	starpurm_DRS_SUCCESS,
+	starpurm_DRS_DISABLD,
+	starpurm_DRS_PERM,
+	starpurm_DRS_NOTED,
+	starpurm_DRS_REQST
+};
+typedef int starpurm_drs_ret_t;
+typedef void *starpurm_drs_desc_t;
+typedef void *starpurm_drs_cbs_t;
+typedef void (*starpurm_drs_cb_t)(void *);
+typedef void *starpurm_block_cond_t;
+typedef int (*starpurm_polling_t)(void *);
+
+/* Resource enforcement */
+void starpurm_initialize(void);
+
+void starpurm_shutdown(void);
+
+void starpurm_spawn_kernel_on_cpus(void *data, void(*f)(void *), void *args, hwloc_cpuset_t cpuset);
+void starpurm_spawn_kernel_on_cpus_callback(void *data, void(*f)(void *), void *args, hwloc_cpuset_t cpuset, void(*cb_f)(void *), void *cb_args);
+
+/* Dynamic resource sharing */
+starpurm_drs_ret_t starpurm_set_drs_enable(starpurm_drs_desc_t *spd);
+starpurm_drs_ret_t starpurm_set_drs_disable(starpurm_drs_desc_t *spd);
+int starpurm_drs_enabled_p(void);
+
+starpurm_drs_ret_t starpurm_set_max_parallelism(starpurm_drs_desc_t *spd, int max);
+
+starpurm_drs_ret_t starpurm_callback_set(starpurm_drs_desc_t *spd, starpurm_drs_cbs_t which, starpurm_drs_cb_t callback);
+starpurm_drs_ret_t starpurm_callback_get(starpurm_drs_desc_t *spd, starpurm_drs_cbs_t which, starpurm_drs_cb_t *callback);
+
+starpurm_drs_ret_t starpurm_assign_cpu_to_starpu(starpurm_drs_desc_t *spd, int cpuid);
+starpurm_drs_ret_t starpurm_assign_cpus_to_starpu(starpurm_drs_desc_t *spd, int ncpus);
+starpurm_drs_ret_t starpurm_assign_cpu_mask_to_starpu(starpurm_drs_desc_t *spd, const hwloc_cpuset_t mask);
+starpurm_drs_ret_t starpurm_assign_all_cpus_to_starpu(starpurm_drs_desc_t *spd);
+
+starpurm_drs_ret_t starpurm_withdraw_cpu_from_starpu(starpurm_drs_desc_t *spd, int cpuid);
+starpurm_drs_ret_t starpurm_withdraw_cpus_from_starpu(starpurm_drs_desc_t *spd, int ncpus);
+starpurm_drs_ret_t starpurm_withdraw_cpu_mask_from_starpu(starpurm_drs_desc_t *spd, const hwloc_cpuset_t mask);
+starpurm_drs_ret_t starpurm_withdraw_all_cpus_from_starpu(starpurm_drs_desc_t *spd);
+
+/* --- */
+
+starpurm_drs_ret_t starpurm_lend(starpurm_drs_desc_t *spd);
+starpurm_drs_ret_t starpurm_lend_cpu(starpurm_drs_desc_t *spd, int cpuid);
+starpurm_drs_ret_t starpurm_lend_cpus(starpurm_drs_desc_t *spd, int ncpus);
+starpurm_drs_ret_t starpurm_lend_cpu_mask(starpurm_drs_desc_t *spd, const hwloc_cpuset_t mask);
+
+starpurm_drs_ret_t starpurm_reclaim(starpurm_drs_desc_t *spd);
+starpurm_drs_ret_t starpurm_reclaim_cpu(starpurm_drs_desc_t *spd, int cpuid);
+starpurm_drs_ret_t starpurm_reclaim_cpus(starpurm_drs_desc_t *spd, int ncpus);
+starpurm_drs_ret_t starpurm_reclaim_cpu_mask(starpurm_drs_desc_t *spd, const hwloc_cpuset_t mask);
+
+starpurm_drs_ret_t starpurm_acquire(starpurm_drs_desc_t *spd);
+starpurm_drs_ret_t starpurm_acquire_cpu(starpurm_drs_desc_t *spd, int cpuid);
+starpurm_drs_ret_t starpurm_acquire_cpus(starpurm_drs_desc_t *spd, int ncpus);
+starpurm_drs_ret_t starpurm_acquire_cpu_mask(starpurm_drs_desc_t *spd, const hwloc_cpuset_t mask);
+
+starpurm_drs_ret_t starpurm_return_all(starpurm_drs_desc_t *spd);
+starpurm_drs_ret_t starpurm_return_cpu(starpurm_drs_desc_t *spd, int cpuid);
+
+/* Pause/resume */
+starpurm_drs_ret_t starpurm_create_block_condition(starpurm_block_cond_t *cond);
+void starpurm_block_current_task(starpurm_block_cond_t *cond);
+void starpurm_signal_block_condition(starpurm_block_cond_t *cond);
+ 
+void starpurm_register_polling_service(const char *service_name, starpurm_polling_t function, void *data);
+void starpurm_unregister_polling_service(const char *service_name, starpurm_polling_t function, void *data);
+
+/* Devices */
+int starpurm_get_device_type_id(const char *type_str);
+const char *starpurm_get_device_type_name(int type_id);
+int starpurm_get_nb_devices_by_type(int type_id);
+int starpurm_get_device_id(int type_id, int device_rank);
+
+starpurm_drs_ret_t starpurm_assign_device_to_starpu(starpurm_drs_desc_t *spd, int type_id, int unit_rank);
+starpurm_drs_ret_t starpurm_assign_devices_to_starpu(starpurm_drs_desc_t *spd, int type_id, int ndevices);
+starpurm_drs_ret_t starpurm_assign_device_mask_to_starpu(starpurm_drs_desc_t *spd, const hwloc_cpuset_t mask);
+starpurm_drs_ret_t starpurm_assign_all_devices_to_starpu(starpurm_drs_desc_t *spd, int type_id);
+
+starpurm_drs_ret_t starpurm_withdraw_device_from_starpu(starpurm_drs_desc_t *spd, int type_id, int unit_rank);
+starpurm_drs_ret_t starpurm_withdraw_devices_from_starpu(starpurm_drs_desc_t *spd, int type_id, int ndevices);
+starpurm_drs_ret_t starpurm_withdraw_device_mask_from_starpu(starpurm_drs_desc_t *spd, const hwloc_cpuset_t mask);
+starpurm_drs_ret_t starpurm_withdraw_all_devices_from_starpu(starpurm_drs_desc_t *spd, int type_id);
+
+/* --- */
+
+starpurm_drs_ret_t starpurm_lend_device(starpurm_drs_desc_t *spd, int type_id, int unit_rank);
+starpurm_drs_ret_t starpurm_lend_devices(starpurm_drs_desc_t *spd, int type_id, int ndevices);
+starpurm_drs_ret_t starpurm_lend_device_mask(starpurm_drs_desc_t *spd, const hwloc_cpuset_t mask);
+starpurm_drs_ret_t starpurm_lend_all_devices(starpurm_drs_desc_t *spd, int type_id);
+
+starpurm_drs_ret_t starpurm_reclaim_device(starpurm_drs_desc_t *spd, int type_id, int unit_rank);
+starpurm_drs_ret_t starpurm_reclaim_devices(starpurm_drs_desc_t *spd, int type_id, int ndevices);
+starpurm_drs_ret_t starpurm_reclaim_device_mask(starpurm_drs_desc_t *spd, const hwloc_cpuset_t mask);
+starpurm_drs_ret_t starpurm_reclaim_all_devices(starpurm_drs_desc_t *spd, int type_id);
+
+starpurm_drs_ret_t starpurm_acquire_device(starpurm_drs_desc_t *spd, int type_id, int unit_rank);
+starpurm_drs_ret_t starpurm_acquire_devices(starpurm_drs_desc_t *spd, int type_id, int ndevices);
+starpurm_drs_ret_t starpurm_acquire_device_mask(starpurm_drs_desc_t *spd, const hwloc_cpuset_t mask);
+starpurm_drs_ret_t starpurm_acquire_all_devices(starpurm_drs_desc_t *spd, int type_id);
+
+starpurm_drs_ret_t starpurm_return_all_devices(starpurm_drs_desc_t *spd, int type_id);
+starpurm_drs_ret_t starpurm_return_device(starpurm_drs_desc_t *spd, int type_id, int unit_rank);
+
+/* cpusets */
+hwloc_cpuset_t starpurm_get_device_worker_cpuset(int type_id, int unit_rank);
+hwloc_cpuset_t starpurm_get_global_cpuset(void);
+hwloc_cpuset_t starpurm_get_selected_cpuset(void);
+hwloc_cpuset_t starpurm_get_all_cpu_workers_cpuset(void);
+hwloc_cpuset_t starpurm_get_all_device_workers_cpuset(void);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* __STARPURM_H */

starpurm/include/starpurm_config.h.in

@@ -0,0 +1,29 @@
+/* StarPURM --- StarPU Resource Management Layer.
+ *
+ * Copyright (C) 2017 Inria
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#ifndef __STARPURM_CONFIG_H__
+#define __STARPURM_CONFIG_H__
+
+/* Major version number of StarPU RM. */
+#undef STARPURM_MAJOR_VERSION
+
+/* Minor version number of StarPU RM. */
+#undef STARPURM_MINOR_VERSION
+
+/* Release version number of StarPU RM. */
+#undef STARPURM_RELEASE_VERSION
+
+#endif

starpurm/src/Makefile.am

@@ -0,0 +1,43 @@
+# StarPU --- Resource Management Layer.
+#
+# Copyright (C) 2017, 2018                               Inria
+#
+# StarPU is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 2.1 of the License, or (at
+# your option) any later version.
+#
+# StarPU is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See the GNU Lesser General Public License in COPYING.LGPL for more details.
+
+SUBDIRS =
+
+CLEANFILES = *.gcno *.gcda *.linkinfo
+
+AM_CPPFLAGS	= -I$(top_srcdir)/include -I$(top_srcdir)/src -I$(top_builddir)/src -I$(top_builddir)/include
+AM_CPPFLAGS	+= -I$(top_srcdir)/starpurm/include -I$(top_srcdir)/starpurm/src -I$(top_builddir)/starpurm/src -I$(top_builddir)/starpurm/include
+AM_CFLAGS	= -Wall -g $(STARPU_CUDA_CPPFLAGS) $(STARPU_OPENCL_CPPFLAGS) $(FXT_CFLAGS) $(MAGMA_CFLAGS) $(HWLOC_CFLAGS) $(GLOBAL_AM_CFLAGS) $(NMAD_CFLAGS)
+AM_LDFLAGS	= $(STARPU_OPENCL_LDFLAGS) $(STARPU_CUDA_LDFLAGS) $(STARPU_COI_LDFLAGS) $(STARPU_SCIF_LDFLAGS) $(NMAD_LDFLAGS)
+AM_LIBADD	=
+
+libstarpurm_so_version = $(LIBSTARPURM_INTERFACE_CURRENT):$(LIBSTARPURM_INTERFACE_REVISION):$(LIBSTARPURM_INTERFACE_AGE)
+
+lib_LTLIBRARIES = libstarpurm-@STARPU_EFFECTIVE_VERSION@.la
+
+libstarpurm_@STARPU_EFFECTIVE_VERSION@_la_CPPFLAGS	= $(AM_CPPFLAGS)
+libstarpurm_@STARPU_EFFECTIVE_VERSION@_la_CFLAGS	= $(AM_CFLAGS)
+libstarpurm_@STARPU_EFFECTIVE_VERSION@_la_LDFLAGS	= $(AM_LDFLAGS) -no-undefined -version-info $(libstarpurm_so_version)
+libstarpurm_@STARPU_EFFECTIVE_VERSION@_la_LIBADD	= $(AM_LIBADD) $(top_builddir)/src/libstarpu-@STARPU_EFFECTIVE_VERSION@.la $(HWLOC_LIBS)
+libstarpurm_@STARPU_EFFECTIVE_VERSION@_la_SOURCES	= \
+	starpurm.c
+
+if STARPURM_HAVE_DLB
+libstarpurm_@STARPU_EFFECTIVE_VERSION@_la_SOURCES	+= \
+	starpurm_dlb.c
+endif
+
+showcheck:
+	-cat /dev/null

starpurm/src/config.h.in

@@ -0,0 +1,103 @@
+/* src/config.h.in.  Generated from configure.ac by autoheader.  */
+
+/* Define to 1 if you have the <dlb.h> header file. */
+#undef HAVE_DLB_H
+
+/* Define to 1 if you have the <dlfcn.h> header file. */
+#undef HAVE_DLFCN_H
+
+/* Define to 1 if you have the <hwloc/glibc-sched.h> header file. */
+#undef HAVE_HWLOC_GLIBC_SCHED_H
+
+/* Define to 1 if you have the <inttypes.h> header file. */
+#undef HAVE_INTTYPES_H
+
+/* Define to 1 if you have the `dlb' library (-ldlb). */
+#undef HAVE_LIBDLB
+
+/* Define to 1 if you have the <memory.h> header file. */
+#undef HAVE_MEMORY_H
+
+/* Define to 1 if you have the <stdint.h> header file. */
+#undef HAVE_STDINT_H
+
+/* Define to 1 if you have the <stdlib.h> header file. */
+#undef HAVE_STDLIB_H
+
+/* Define to 1 if you have the <strings.h> header file. */
+#undef HAVE_STRINGS_H
+
+/* Define to 1 if you have the <string.h> header file. */
+#undef HAVE_STRING_H
+
+/* Define to 1 if you have the <sys/stat.h> header file. */
+#undef HAVE_SYS_STAT_H
+
+/* Define to 1 if you have the <sys/types.h> header file. */
+#undef HAVE_SYS_TYPES_H
+
+/* Define to 1 if you have the <unistd.h> header file. */
+#undef HAVE_UNISTD_H
+
+/* Define to the sub-directory in which libtool stores uninstalled libraries.
+   */
+#undef LT_OBJDIR
+
+/* Define to 1 if your C compiler doesn't accept -c and -o together. */
+#undef NO_MINUS_C_MINUS_O
+
+/* Name of package */
+#undef PACKAGE
+
+/* Define to the address where bug reports for this package should be sent. */
+#undef PACKAGE_BUGREPORT
+
+/* Define to the full name of this package. */
+#undef PACKAGE_NAME
+
+/* Define to the full name and version of this package. */
+#undef PACKAGE_STRING
+
+/* Define to the one symbol short name of this package. */
+#undef PACKAGE_TARNAME
+
+/* Define to the home page for this package. */
+#undef PACKAGE_URL
+
+/* Define to the version of this package. */
+#undef PACKAGE_VERSION
+
+/* Define to 1 if dlb support is enabled. */
+#undef STARPURM_HAVE_DLB
+
+/* Major version number of StarPU RM. */
+#undef STARPURM_MAJOR_VERSION
+
+/* Minor version number of StarPU RM. */
+#undef STARPURM_MINOR_VERSION
+
+/* Release version number of StarPU RM. */
+#undef STARPURM_RELEASE_VERSION
+
+/* Define to 1 if StarPU has support for worker callbacks. */
+#undef STARPURM_STARPU_HAVE_WORKER_CALLBACKS
+
+/* Define to 1 if you have the ANSI C header files. */
+#undef STDC_HEADERS
+
+/* Version number of package */
+#undef VERSION
+
+/* Define to the equivalent of the C99 'restrict' keyword, or to
+   nothing if this is not supported.  Do not define if restrict is
+   supported directly.  */
+#undef restrict
+/* Work around a bug in Sun C++: it does not support _Restrict or
+   __restrict__, even though the corresponding Sun C compiler ends up with
+   "#define restrict _Restrict" or "#define restrict __restrict__" in the
+   previous line.  Perhaps some future version of Sun C++ will work with
+   restrict; if so, hopefully it defines __RESTRICT like Sun C does.  */
+#if defined __SUNPRO_CC && !defined __RESTRICT
+# define _Restrict
+# define __restrict__
+#endif

starpurm/src/starpurm_dlb.c

@@ -0,0 +1,149 @@
+/* StarPU --- Resource Management Layer.
+ *
+ * Copyright (C) 2017, 2018                              Inria
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+/* CPUSET routines */
+#define _GNU_SOURCE
+#include <sched.h>
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <assert.h>
+#include <hwloc.h>
+#include <hwloc/glibc-sched.h>
+#include <pthread.h>
+#include <starpu.h>
+#include <starpurm.h>
+#include <config.h>
+#include <starpurm_private.h>
+
+#ifndef STARPURM_HAVE_DLB
+#error "STARPU-RM DLB support not enabled"
+#endif
+
+#include <dlb_sp.h>
+
+/*
+ * DLB interfacing
+ */
+
+static dlb_handler_t      dlb_handle;
+static cpu_set_t          starpurm_process_mask;
+static struct s_starpurm *_starpurm = NULL;
+static pthread_mutex_t dlb_handle_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+#if 0
+/* unused for now */
+static void _glibc_cpuset_to_hwloc_cpuset(const cpu_set_t *glibc_cpuset, hwloc_cpuset_t *hwloc_cpuset)
+{
+	assert(_starpurm != NULL);
+	assert(_starpurm->state != state_uninitialized);
+	struct s_starpurm *rm = _starpurm;
+	int status = hwloc_cpuset_from_glibc_sched_affinity(rm->topology, *hwloc_cpuset, glibc_cpuset, sizeof(cpu_set_t));
+	assert(status == 0);
+}
+#endif
+
+static void _hwloc_cpuset_to_glibc_cpuset(const hwloc_cpuset_t hwloc_cpuset, cpu_set_t *glibc_cpuset)
+{
+	assert(_starpurm != NULL);
+	assert(_starpurm->state != state_uninitialized);
+	struct s_starpurm *rm = _starpurm;
+	int status = hwloc_cpuset_to_glibc_sched_affinity(rm->topology, hwloc_cpuset, glibc_cpuset, sizeof(cpu_set_t));
+	assert(status == 0);
+}
+
+int starpurm_dlb_notify_starpu_worker_mask_going_to_sleep(const hwloc_cpuset_t hwloc_workers_cpuset)
+{
+	int status = 0;
+	pthread_mutex_lock(&dlb_handle_mutex);
+	if (dlb_handle != NULL)
+	{
+		cpu_set_t glibc_workers_cpuset;
+		CPU_ZERO(&glibc_workers_cpuset);
+		_hwloc_cpuset_to_glibc_cpuset(hwloc_workers_cpuset, &glibc_workers_cpuset);
+		DLB_LendCpuMask_sp(dlb_handle, &glibc_workers_cpuset);
+		status = 1;
+	}
+	pthread_mutex_unlock(&dlb_handle_mutex);
+	return status;
+}
+
+int starpurm_dlb_notify_starpu_worker_mask_waking_up(const hwloc_cpuset_t hwloc_workers_cpuset)
+{
+	int status = 0;
+	pthread_mutex_lock(&dlb_handle_mutex);
+	if (dlb_handle != NULL)
+	{
+		cpu_set_t glibc_workers_cpuset;
+		CPU_ZERO(&glibc_workers_cpuset);
+		_hwloc_cpuset_to_glibc_cpuset(hwloc_workers_cpuset, &glibc_workers_cpuset);
+		DLB_ReclaimCpuMask_sp(dlb_handle, &glibc_workers_cpuset);
+		status = 1;
+	}
+	pthread_mutex_unlock(&dlb_handle_mutex);
+	return status;
+}
+
+#ifdef STARPURM_STARPU_HAVE_WORKER_CALLBACKS
+static void _dlb_callback_enable_cpu(int cpuid)
+{
+	starpurm_enqueue_event_cpu_unit_available(cpuid);
+}
+
+static void _dlb_callback_disable_cpu(int cpuid)
+{
+	/* nothing */
+}
+#endif
+
+void starpurm_dlb_init(struct s_starpurm *rm)
+{
+	_starpurm = rm;
+
+	CPU_ZERO(&starpurm_process_mask);
+	_hwloc_cpuset_to_glibc_cpuset(rm->selected_cpuset, &starpurm_process_mask);
+
+	pthread_mutex_lock(&dlb_handle_mutex);
+	dlb_handle = DLB_Init_sp(0, &starpurm_process_mask, "--policy=new --mode=async");
+
+	/* cpu-based callbacks are mutually exclusive with mask-based callbacks,
+	 * we only register cpu-based callbacks */
+#ifdef STARPURM_STARPU_HAVE_WORKER_CALLBACKS
+	assert(DLB_CallbackSet_sp(dlb_handle, dlb_callback_disable_cpu, (dlb_callback_t)_dlb_callback_disable_cpu) == DLB_SUCCESS);
+	assert(DLB_CallbackSet_sp(dlb_handle, dlb_callback_enable_cpu, (dlb_callback_t)_dlb_callback_enable_cpu) == DLB_SUCCESS);
+#endif
+
+	DLB_Enable_sp(dlb_handle);
+	pthread_mutex_unlock(&dlb_handle_mutex);
+
+}
+
+void starpurm_dlb_exit(void)
+{
+	pthread_mutex_lock(&dlb_handle_mutex);
+	dlb_handler_t dlb_handle_save = dlb_handle;
+	dlb_handle = 0;
+	pthread_mutex_unlock(&dlb_handle_mutex);
+
+	/* lend every resources that StarPU may still have */
+	DLB_Lend_sp(dlb_handle_save);
+
+	pthread_mutex_lock(&dlb_handle_mutex);
+	DLB_Disable_sp(dlb_handle_save);
+	DLB_Finalize_sp(dlb_handle_save);
+	pthread_mutex_unlock(&dlb_handle_mutex);
+}

starpurm/src/starpurm_private.h

@@ -0,0 +1,125 @@
+/* StarPURM --- StarPU Resource Management Layer.
+ *
+ * Copyright (C) 2017  Inria
+ *
+ * StarPU is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at
+ * your option) any later version.
+ *
+ * StarPU is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
+ */
+
+#ifndef __STARPURM_PRIVATE_H
+#define __STARPURM_PRIVATE_H
+
+enum e_state {
+	state_uninitialized = 0,
+	state_init
+};
+
+enum e_starpurm_unit_type
+{
+	starpurm_unit_cpu    = 0,
+	starpurm_unit_opencl = 1,
+	starpurm_unit_cuda   = 2,
+	starpurm_unit_mic    = 3,
+	starpurm_unit_ntypes = 4
+};
+
+struct s_starpurm
+{
+	/* Machine topology as detected by hwloc. */
+	hwloc_topology_t topology;
+
+	/* Current upper bound on the number of CPU cores selectable for computing with the runtime system. */
+	unsigned max_ncpus;
+
+	/* Number of currently selected CPU workers */
+	unsigned selected_ncpus;
+
+	/* Number of currently selected workers (CPU+devices) */
+	unsigned selected_nworkers;
+
+	/* Initialization state of the RM instance. */
+	int state;
+
+	/* Boolean indicating the state of the dynamic resource sharing layer.
+	 *
+	 * !0 indicates that dynamic resource sharing is enabled.
+	 * 0 indicates that dynamic resource sharing is disabled.
+	 */
+	int dynamic_resource_sharing;
+
+	/* Id of the StarPU's sched_ctx used by the RM instance. */
+	unsigned sched_ctx_id;
+
+	/* Number of unit types supported by this RM instance. */
+	int unit_ntypes;
+
+	/* Number of unitss available for each type. */
+	int *nunits_by_type;
+
+	/* Number of units. */
+	int nunits;
+
+	/* Offset of unit numbering for each type. */
+	int *unit_offsets_by_type;
+
+	/* Array of units. */
+	struct s_starpurm_unit *units;
+
+	/* Cpuset of all the StarPU's workers (CPU+devices. */
+	hwloc_cpuset_t global_cpuset;
+
+	/* Cpuset of all StarPU CPU workers. */
+	hwloc_cpuset_t all_cpu_workers_cpuset;
+
+	/* Cpuset of all StarPU device workers. */
+	hwloc_cpuset_t all_device_workers_cpuset;
+
+	/* Cpuset of all selected workers (CPU+devices). */
+	hwloc_cpuset_t selected_cpuset;
+
+	/* maximum value among worker ids */
+	int max_worker_id;
+
+	/* worker id to unit id table */
+	int *worker_unit_ids;
+
+	/* Temporary contexts accounting. */
+	unsigned int max_temporary_ctxs;
+	unsigned int avail_temporary_ctxs;
+	pthread_mutex_t temporary_ctxs_mutex;
+	pthread_cond_t temporary_ctxs_cond;
+
+	/* Global StarPU pause state */
+	int starpu_in_pause;
+
+	/* Event list. */
+	pthread_t event_thread;
+	pthread_mutex_t event_list_mutex;
+	pthread_cond_t event_list_cond;
+	pthread_cond_t event_processing_cond;
+	int event_processing_enabled;
+	int event_processing_ended;
+	struct s_starpurm_event *event_list_head;
+	struct s_starpurm_event *event_list_tail;
+};
+
+
+#ifdef STARPURM_HAVE_DLB
+void starpurm_dlb_init(struct s_starpurm *rm);
+void starpurm_dlb_exit(void);
+int starpurm_dlb_notify_starpu_worker_mask_going_to_sleep(const hwloc_cpuset_t hwloc_workers_cpuset);
+int starpurm_dlb_notify_starpu_worker_mask_waking_up(const hwloc_cpuset_t hwloc_workers_cpuset);
+#ifdef STARPURM_STARPU_HAVE_WORKER_CALLBACKS
+void starpurm_enqueue_event_cpu_unit_available(int cpuid);
+#endif
+#endif
+
+#endif /* __STARPURM_PRIVATE_H */

starpurm/starpurm-1.3.pc.in

@@ -0,0 +1,28 @@
+# StarPU --- Resource Management Layer.
+#
+# Copyright (C) 2017, 2018                               Inria
+#
+# StarPU is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 2.1 of the License, or (at
+# your option) any later version.
+#
+# StarPU is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See the GNU Lesser General Public License in COPYING.LGPL for more details.
+
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libdir=@libdir@
+pkglibdir=@pkglibdir@
+includedir=@includedir@
+
+Name: starpurm
+Description: resource management layer on top of StarPU
+Version: @PACKAGE_VERSION@
+Cflags: -I${includedir}/starpurm/@STARPU_EFFECTIVE_VERSION@ @DLB_CFLAGS@
+Libs: -L${libdir} -lstarpurm-@STARPU_EFFECTIVE_VERSION@ @DLB_LIBS@
+Libs.private: @LDFLAGS@ @LIBS@
+Requires: starpu-1.3 hwloc