|
@@ -19,6 +19,7 @@
|
|
|
#include <common/config.h>
|
|
|
#include <common/utils.h>
|
|
|
#include <datawizard/datawizard.h>
|
|
|
+#include <datawizard/memory_nodes.h>
|
|
|
#include <core/disk.h>
|
|
|
|
|
|
/* TODO: This should be tuned according to driver capabilities
|
|
@@ -30,13 +31,13 @@
|
|
|
#define MAX_PENDING_IDLE_REQUESTS_PER_NODE 1
|
|
|
|
|
|
/* requests that have not been treated at all */
|
|
|
-static struct _starpu_data_request_list *data_requests[STARPU_MAXNODES];
|
|
|
-static struct _starpu_data_request_list *prefetch_requests[STARPU_MAXNODES];
|
|
|
-static struct _starpu_data_request_list *idle_requests[STARPU_MAXNODES];
|
|
|
+static struct _starpu_data_request_list data_requests[STARPU_MAXNODES];
|
|
|
+static struct _starpu_data_request_list prefetch_requests[STARPU_MAXNODES];
|
|
|
+static struct _starpu_data_request_list idle_requests[STARPU_MAXNODES];
|
|
|
static starpu_pthread_mutex_t data_requests_list_mutex[STARPU_MAXNODES];
|
|
|
|
|
|
/* requests that are not terminated (eg. async transfers) */
|
|
|
-static struct _starpu_data_request_list *data_requests_pending[STARPU_MAXNODES];
|
|
|
+static struct _starpu_data_request_list data_requests_pending[STARPU_MAXNODES];
|
|
|
static unsigned data_requests_npending[STARPU_MAXNODES];
|
|
|
static starpu_pthread_mutex_t data_requests_pending_list_mutex[STARPU_MAXNODES];
|
|
|
|
|
@@ -45,23 +46,20 @@ void _starpu_init_data_request_lists(void)
|
|
|
unsigned i;
|
|
|
for (i = 0; i < STARPU_MAXNODES; i++)
|
|
|
{
|
|
|
- data_requests[i] = _starpu_data_request_list_new();
|
|
|
- prefetch_requests[i] = _starpu_data_request_list_new();
|
|
|
- idle_requests[i] = _starpu_data_request_list_new();
|
|
|
+ _starpu_data_request_list_init(&data_requests[i]);
|
|
|
+ _starpu_data_request_list_init(&prefetch_requests[i]);
|
|
|
+ _starpu_data_request_list_init(&idle_requests[i]);
|
|
|
|
|
|
/* Tell helgrind that we are fine with checking for list_empty
|
|
|
* in _starpu_handle_node_data_requests, we will call it
|
|
|
* periodically anyway */
|
|
|
- STARPU_HG_DISABLE_CHECKING(data_requests[i]);
|
|
|
- STARPU_HG_DISABLE_CHECKING(data_requests[i]->_head);
|
|
|
- STARPU_HG_DISABLE_CHECKING(prefetch_requests[i]);
|
|
|
- STARPU_HG_DISABLE_CHECKING(prefetch_requests[i]->_head);
|
|
|
- STARPU_HG_DISABLE_CHECKING(idle_requests[i]);
|
|
|
- STARPU_HG_DISABLE_CHECKING(idle_requests[i]->_head);
|
|
|
+ STARPU_HG_DISABLE_CHECKING(data_requests[i]._head);
|
|
|
+ STARPU_HG_DISABLE_CHECKING(prefetch_requests[i]._head);
|
|
|
+ STARPU_HG_DISABLE_CHECKING(idle_requests[i]._head);
|
|
|
|
|
|
STARPU_PTHREAD_MUTEX_INIT(&data_requests_list_mutex[i], NULL);
|
|
|
|
|
|
- data_requests_pending[i] = _starpu_data_request_list_new();
|
|
|
+ _starpu_data_request_list_init(&data_requests_pending[i]);
|
|
|
data_requests_npending[i] = 0;
|
|
|
STARPU_PTHREAD_MUTEX_INIT(&data_requests_pending_list_mutex[i], NULL);
|
|
|
}
|
|
@@ -74,12 +72,7 @@ void _starpu_deinit_data_request_lists(void)
|
|
|
for (i = 0; i < STARPU_MAXNODES; i++)
|
|
|
{
|
|
|
STARPU_PTHREAD_MUTEX_DESTROY(&data_requests_pending_list_mutex[i]);
|
|
|
- _starpu_data_request_list_delete(data_requests_pending[i]);
|
|
|
-
|
|
|
STARPU_PTHREAD_MUTEX_DESTROY(&data_requests_list_mutex[i]);
|
|
|
- _starpu_data_request_list_delete(data_requests[i]);
|
|
|
- _starpu_data_request_list_delete(prefetch_requests[i]);
|
|
|
- _starpu_data_request_list_delete(idle_requests[i]);
|
|
|
}
|
|
|
}
|
|
|
|
|
@@ -244,11 +237,11 @@ void _starpu_post_data_request(struct _starpu_data_request *r, unsigned handling
|
|
|
/* insert the request in the proper list */
|
|
|
STARPU_PTHREAD_MUTEX_LOCK(&data_requests_list_mutex[handling_node]);
|
|
|
if (r->prefetch == 2)
|
|
|
- _starpu_data_request_list_push_back(idle_requests[handling_node], r);
|
|
|
+ _starpu_data_request_list_push_back(&idle_requests[handling_node], r);
|
|
|
else if (r->prefetch)
|
|
|
- _starpu_data_request_list_push_back(prefetch_requests[handling_node], r);
|
|
|
+ _starpu_data_request_list_push_back(&prefetch_requests[handling_node], r);
|
|
|
else
|
|
|
- _starpu_data_request_list_push_back(data_requests[handling_node], r);
|
|
|
+ _starpu_data_request_list_push_back(&data_requests[handling_node], r);
|
|
|
STARPU_PTHREAD_MUTEX_UNLOCK(&data_requests_list_mutex[handling_node]);
|
|
|
|
|
|
#ifndef STARPU_NON_BLOCKING_DRIVERS
|
|
@@ -437,7 +430,7 @@ static int starpu_handle_data_request(struct _starpu_data_request *r, unsigned m
|
|
|
_starpu_spin_unlock(&handle->header_lock);
|
|
|
|
|
|
STARPU_PTHREAD_MUTEX_LOCK(&data_requests_pending_list_mutex[r->handling_node]);
|
|
|
- _starpu_data_request_list_push_back(data_requests_pending[r->handling_node], r);
|
|
|
+ _starpu_data_request_list_push_back(&data_requests_pending[r->handling_node], r);
|
|
|
data_requests_npending[r->handling_node]++;
|
|
|
STARPU_PTHREAD_MUTEX_UNLOCK(&data_requests_pending_list_mutex[r->handling_node]);
|
|
|
|
|
@@ -451,11 +444,10 @@ static int starpu_handle_data_request(struct _starpu_data_request *r, unsigned m
|
|
|
return 0;
|
|
|
}
|
|
|
|
|
|
-int __starpu_handle_node_data_requests(struct _starpu_data_request_list **reqlist, unsigned src_node, unsigned may_alloc, unsigned n, unsigned *pushed, unsigned prefetch)
|
|
|
+static int __starpu_handle_node_data_requests(struct _starpu_data_request_list *reqlist, unsigned src_node, unsigned may_alloc, unsigned n, unsigned *pushed, unsigned prefetch)
|
|
|
{
|
|
|
struct _starpu_data_request *r;
|
|
|
- struct _starpu_data_request_list *new_data_requests[prefetch + 1]; /* Indexed by prefetch level */
|
|
|
- struct _starpu_data_request_list *empty_list;
|
|
|
+ struct _starpu_data_request_list new_data_requests[prefetch + 1]; /* Indexed by prefetch level */
|
|
|
unsigned i;
|
|
|
int ret = 0;
|
|
|
|
|
@@ -465,47 +457,43 @@ int __starpu_handle_node_data_requests(struct _starpu_data_request_list **reqlis
|
|
|
/* This is racy, but not posing problems actually, since we know we
|
|
|
* will come back here to probe again regularly anyway.
|
|
|
* Thus, do not expose this optimization to helgrind */
|
|
|
- if (!RUNNING_ON_VALGRIND && _starpu_data_request_list_empty(reqlist[src_node]))
|
|
|
+ if (!RUNNING_ON_VALGRIND && _starpu_data_request_list_empty(&reqlist[src_node]))
|
|
|
return 0;
|
|
|
#endif
|
|
|
|
|
|
- empty_list = _starpu_data_request_list_new();
|
|
|
+ /* TODO optimize */
|
|
|
|
|
|
#ifdef STARPU_NON_BLOCKING_DRIVERS
|
|
|
/* take all the entries from the request list */
|
|
|
if (STARPU_PTHREAD_MUTEX_TRYLOCK(&data_requests_list_mutex[src_node]))
|
|
|
{
|
|
|
/* List is busy, do not bother with it */
|
|
|
- _starpu_data_request_list_delete(empty_list);
|
|
|
return -EBUSY;
|
|
|
}
|
|
|
#else
|
|
|
STARPU_PTHREAD_MUTEX_LOCK(&data_requests_list_mutex[src_node]);
|
|
|
#endif
|
|
|
|
|
|
- struct _starpu_data_request_list *local_list = reqlist[src_node];
|
|
|
-
|
|
|
- if (_starpu_data_request_list_empty(local_list))
|
|
|
+ if (_starpu_data_request_list_empty(&reqlist[src_node]))
|
|
|
{
|
|
|
/* there is no request */
|
|
|
STARPU_PTHREAD_MUTEX_UNLOCK(&data_requests_list_mutex[src_node]);
|
|
|
- _starpu_data_request_list_delete(empty_list);
|
|
|
return 0;
|
|
|
}
|
|
|
|
|
|
/* There is an entry: we create a new empty list to replace the list of
|
|
|
* requests, and we handle the request(s) one by one in the former
|
|
|
* list, without concurrency issues.*/
|
|
|
- reqlist[src_node] = empty_list;
|
|
|
- STARPU_HG_DISABLE_CHECKING(reqlist[src_node]->_head);
|
|
|
+ struct _starpu_data_request_list local_list = reqlist[src_node];
|
|
|
+ _starpu_data_request_list_init(&reqlist[src_node]);
|
|
|
|
|
|
STARPU_PTHREAD_MUTEX_UNLOCK(&data_requests_list_mutex[src_node]);
|
|
|
|
|
|
for (i = 0; i <= prefetch; i++)
|
|
|
- new_data_requests[i] = _starpu_data_request_list_new();
|
|
|
+ _starpu_data_request_list_init(&new_data_requests[i]);
|
|
|
|
|
|
/* for all entries of the list */
|
|
|
- while (!_starpu_data_request_list_empty(local_list))
|
|
|
+ while (!_starpu_data_request_list_empty(&local_list))
|
|
|
{
|
|
|
int res;
|
|
|
|
|
@@ -517,7 +505,7 @@ int __starpu_handle_node_data_requests(struct _starpu_data_request_list **reqlis
|
|
|
break;
|
|
|
}
|
|
|
|
|
|
- r = _starpu_data_request_list_pop_front(local_list);
|
|
|
+ r = _starpu_data_request_list_pop_front(&local_list);
|
|
|
|
|
|
res = starpu_handle_data_request(r, may_alloc, prefetch);
|
|
|
if (res != 0 && res != -EAGAIN)
|
|
@@ -525,7 +513,7 @@ int __starpu_handle_node_data_requests(struct _starpu_data_request_list **reqlis
|
|
|
/* handle is busy, or not enough memory, postpone for now */
|
|
|
ret = res;
|
|
|
/* Prefetch requests might have gotten promoted while in tmp list */
|
|
|
- _starpu_data_request_list_push_back(new_data_requests[r->prefetch], r);
|
|
|
+ _starpu_data_request_list_push_back(&new_data_requests[r->prefetch], r);
|
|
|
break;
|
|
|
}
|
|
|
|
|
@@ -533,26 +521,26 @@ int __starpu_handle_node_data_requests(struct _starpu_data_request_list **reqlis
|
|
|
}
|
|
|
|
|
|
/* Push back requests we didn't handle on the proper list */
|
|
|
- while (!_starpu_data_request_list_empty(local_list))
|
|
|
+ while (!_starpu_data_request_list_empty(&local_list))
|
|
|
{
|
|
|
- r = _starpu_data_request_list_pop_front(local_list);
|
|
|
+ r = _starpu_data_request_list_pop_front(&local_list);
|
|
|
/* Prefetch requests might have gotten promoted while in tmp list */
|
|
|
- _starpu_data_request_list_push_back(new_data_requests[r->prefetch], r);
|
|
|
+ _starpu_data_request_list_push_back(&new_data_requests[r->prefetch], r);
|
|
|
}
|
|
|
|
|
|
for (i = 0; i <= prefetch; i++)
|
|
|
- if (!_starpu_data_request_list_empty(new_data_requests[i]))
|
|
|
+ if (!_starpu_data_request_list_empty(&new_data_requests[i]))
|
|
|
break;
|
|
|
|
|
|
if (i <= prefetch)
|
|
|
{
|
|
|
STARPU_PTHREAD_MUTEX_LOCK(&data_requests_list_mutex[src_node]);
|
|
|
- if (!(_starpu_data_request_list_empty(new_data_requests[0])))
|
|
|
- _starpu_data_request_list_push_list_front(new_data_requests[0], data_requests[src_node]);
|
|
|
- if (prefetch >= 1 && !(_starpu_data_request_list_empty(new_data_requests[1])))
|
|
|
- _starpu_data_request_list_push_list_front(new_data_requests[1], prefetch_requests[src_node]);
|
|
|
- if (prefetch >= 2 && !(_starpu_data_request_list_empty(new_data_requests[2])))
|
|
|
- _starpu_data_request_list_push_list_front(new_data_requests[2], idle_requests[src_node]);
|
|
|
+ if (!(_starpu_data_request_list_empty(&new_data_requests[0])))
|
|
|
+ _starpu_data_request_list_push_list_front(&new_data_requests[0], &data_requests[src_node]);
|
|
|
+ if (prefetch >= 1 && !(_starpu_data_request_list_empty(&new_data_requests[1])))
|
|
|
+ _starpu_data_request_list_push_list_front(&new_data_requests[1], &prefetch_requests[src_node]);
|
|
|
+ if (prefetch >= 2 && !(_starpu_data_request_list_empty(&new_data_requests[2])))
|
|
|
+ _starpu_data_request_list_push_list_front(&new_data_requests[2], &idle_requests[src_node]);
|
|
|
STARPU_PTHREAD_MUTEX_UNLOCK(&data_requests_list_mutex[src_node]);
|
|
|
|
|
|
#ifndef STARPU_NON_BLOCKING_DRIVERS
|
|
@@ -560,11 +548,6 @@ int __starpu_handle_node_data_requests(struct _starpu_data_request_list **reqlis
|
|
|
#endif
|
|
|
}
|
|
|
|
|
|
- for (i = 0; i <= prefetch; i++)
|
|
|
- _starpu_data_request_list_delete(new_data_requests[i]);
|
|
|
- _starpu_data_request_list_delete(local_list);
|
|
|
-
|
|
|
-
|
|
|
return ret;
|
|
|
}
|
|
|
|
|
@@ -587,26 +570,25 @@ static int _handle_pending_node_data_requests(unsigned src_node, unsigned force)
|
|
|
{
|
|
|
// _STARPU_DEBUG("_starpu_handle_pending_node_data_requests ...\n");
|
|
|
//
|
|
|
- struct _starpu_data_request_list *new_data_requests_pending;
|
|
|
- struct _starpu_data_request_list *empty_list;
|
|
|
+ struct _starpu_data_request_list new_data_requests_pending;
|
|
|
+ struct _starpu_data_request_list empty_list;
|
|
|
unsigned taken, kept;
|
|
|
|
|
|
#ifdef STARPU_NON_BLOCKING_DRIVERS
|
|
|
/* Here helgrind would should that this is an un protected access.
|
|
|
* We however don't care about missing an entry, we will get called
|
|
|
* again sooner or later. */
|
|
|
- if (!RUNNING_ON_VALGRIND && _starpu_data_request_list_empty(data_requests_pending[src_node]))
|
|
|
+ if (!RUNNING_ON_VALGRIND && _starpu_data_request_list_empty(&data_requests_pending[src_node]))
|
|
|
return 0;
|
|
|
#endif
|
|
|
|
|
|
- empty_list = _starpu_data_request_list_new();
|
|
|
+ _starpu_data_request_list_init(&empty_list);
|
|
|
#ifdef STARPU_NON_BLOCKING_DRIVERS
|
|
|
if (!force)
|
|
|
{
|
|
|
if (STARPU_PTHREAD_MUTEX_TRYLOCK(&data_requests_pending_list_mutex[src_node]))
|
|
|
{
|
|
|
/* List is busy, do not bother with it */
|
|
|
- _starpu_data_request_list_delete(empty_list);
|
|
|
return 0;
|
|
|
}
|
|
|
}
|
|
@@ -615,27 +597,26 @@ static int _handle_pending_node_data_requests(unsigned src_node, unsigned force)
|
|
|
/* We really want to handle requests */
|
|
|
STARPU_PTHREAD_MUTEX_LOCK(&data_requests_pending_list_mutex[src_node]);
|
|
|
|
|
|
- /* for all entries of the list */
|
|
|
- struct _starpu_data_request_list *local_list = data_requests_pending[src_node];
|
|
|
- if (_starpu_data_request_list_empty(local_list))
|
|
|
+ if (_starpu_data_request_list_empty(&data_requests_pending[src_node]))
|
|
|
{
|
|
|
/* there is no request */
|
|
|
STARPU_PTHREAD_MUTEX_UNLOCK(&data_requests_pending_list_mutex[src_node]);
|
|
|
- _starpu_data_request_list_delete(empty_list);
|
|
|
return 0;
|
|
|
}
|
|
|
- data_requests_pending[src_node] = empty_list;
|
|
|
+ /* for all entries of the list */
|
|
|
+ struct _starpu_data_request_list local_list = data_requests_pending[src_node];
|
|
|
+ _starpu_data_request_list_init(&data_requests_pending[src_node]);
|
|
|
|
|
|
STARPU_PTHREAD_MUTEX_UNLOCK(&data_requests_pending_list_mutex[src_node]);
|
|
|
|
|
|
- new_data_requests_pending = _starpu_data_request_list_new();
|
|
|
+ _starpu_data_request_list_init(&new_data_requests_pending);
|
|
|
taken = 0;
|
|
|
kept = 0;
|
|
|
|
|
|
- while (!_starpu_data_request_list_empty(local_list))
|
|
|
+ while (!_starpu_data_request_list_empty(&local_list))
|
|
|
{
|
|
|
struct _starpu_data_request *r;
|
|
|
- r = _starpu_data_request_list_pop_front(local_list);
|
|
|
+ r = _starpu_data_request_list_pop_front(&local_list);
|
|
|
taken++;
|
|
|
|
|
|
starpu_data_handle_t handle = r->handle;
|
|
@@ -653,7 +634,7 @@ static int _handle_pending_node_data_requests(unsigned src_node, unsigned force)
|
|
|
if (_starpu_spin_trylock(&handle->header_lock))
|
|
|
{
|
|
|
/* Handle is busy, retry this later */
|
|
|
- _starpu_data_request_list_push_back(new_data_requests_pending, r);
|
|
|
+ _starpu_data_request_list_push_back(&new_data_requests_pending, r);
|
|
|
kept++;
|
|
|
continue;
|
|
|
}
|
|
@@ -683,7 +664,7 @@ static int _handle_pending_node_data_requests(unsigned src_node, unsigned force)
|
|
|
_starpu_spin_unlock(&r->lock);
|
|
|
_starpu_spin_unlock(&handle->header_lock);
|
|
|
|
|
|
- _starpu_data_request_list_push_back(new_data_requests_pending, r);
|
|
|
+ _starpu_data_request_list_push_back(&new_data_requests_pending, r);
|
|
|
kept++;
|
|
|
}
|
|
|
}
|
|
@@ -691,12 +672,9 @@ static int _handle_pending_node_data_requests(unsigned src_node, unsigned force)
|
|
|
STARPU_PTHREAD_MUTEX_LOCK(&data_requests_pending_list_mutex[src_node]);
|
|
|
data_requests_npending[src_node] -= taken - kept;
|
|
|
if (kept)
|
|
|
- _starpu_data_request_list_push_list_back(data_requests_pending[src_node], new_data_requests_pending);
|
|
|
+ _starpu_data_request_list_push_list_back(&data_requests_pending[src_node], &new_data_requests_pending);
|
|
|
STARPU_PTHREAD_MUTEX_UNLOCK(&data_requests_pending_list_mutex[src_node]);
|
|
|
|
|
|
- _starpu_data_request_list_delete(local_list);
|
|
|
- _starpu_data_request_list_delete(new_data_requests_pending);
|
|
|
-
|
|
|
return taken - kept;
|
|
|
}
|
|
|
|
|
@@ -718,9 +696,9 @@ int _starpu_check_that_no_data_request_exists(unsigned node)
|
|
|
int no_pending;
|
|
|
|
|
|
STARPU_PTHREAD_MUTEX_LOCK(&data_requests_list_mutex[node]);
|
|
|
- no_request = _starpu_data_request_list_empty(data_requests[node])
|
|
|
- && _starpu_data_request_list_empty(prefetch_requests[node])
|
|
|
- && _starpu_data_request_list_empty(idle_requests[node]);
|
|
|
+ no_request = _starpu_data_request_list_empty(&data_requests[node])
|
|
|
+ && _starpu_data_request_list_empty(&prefetch_requests[node])
|
|
|
+ && _starpu_data_request_list_empty(&idle_requests[node]);
|
|
|
STARPU_PTHREAD_MUTEX_UNLOCK(&data_requests_list_mutex[node]);
|
|
|
STARPU_PTHREAD_MUTEX_LOCK(&data_requests_pending_list_mutex[node]);
|
|
|
no_pending = !data_requests_npending[node];
|
|
@@ -756,33 +734,33 @@ void _starpu_update_prefetch_status(struct _starpu_data_request *r, unsigned pre
|
|
|
/* The request can be in a different list (handling request or the temp list)
|
|
|
* we have to check that it is really in the prefetch list. */
|
|
|
struct _starpu_data_request *r_iter;
|
|
|
- for (r_iter = _starpu_data_request_list_begin(prefetch_requests[r->handling_node]);
|
|
|
- r_iter != _starpu_data_request_list_end(prefetch_requests[r->handling_node]);
|
|
|
+ for (r_iter = _starpu_data_request_list_begin(&prefetch_requests[r->handling_node]);
|
|
|
+ r_iter != _starpu_data_request_list_end(&prefetch_requests[r->handling_node]);
|
|
|
r_iter = _starpu_data_request_list_next(r_iter))
|
|
|
{
|
|
|
|
|
|
if (r==r_iter)
|
|
|
{
|
|
|
- _starpu_data_request_list_erase(prefetch_requests[r->handling_node],r);
|
|
|
- _starpu_data_request_list_push_front(data_requests[r->handling_node],r);
|
|
|
+ _starpu_data_request_list_erase(&prefetch_requests[r->handling_node],r);
|
|
|
+ _starpu_data_request_list_push_front(&data_requests[r->handling_node],r);
|
|
|
goto found;
|
|
|
}
|
|
|
}
|
|
|
|
|
|
/* The request can be in a different list (handling request or the temp list)
|
|
|
* we have to check that it is really in the idle list. */
|
|
|
- for (r_iter = _starpu_data_request_list_begin(idle_requests[r->handling_node]);
|
|
|
- r_iter != _starpu_data_request_list_end(idle_requests[r->handling_node]);
|
|
|
+ for (r_iter = _starpu_data_request_list_begin(&idle_requests[r->handling_node]);
|
|
|
+ r_iter != _starpu_data_request_list_end(&idle_requests[r->handling_node]);
|
|
|
r_iter = _starpu_data_request_list_next(r_iter))
|
|
|
{
|
|
|
|
|
|
if (r==r_iter)
|
|
|
{
|
|
|
- _starpu_data_request_list_erase(idle_requests[r->handling_node],r);
|
|
|
+ _starpu_data_request_list_erase(&idle_requests[r->handling_node],r);
|
|
|
if (prefetch == 1)
|
|
|
- _starpu_data_request_list_push_front(prefetch_requests[r->handling_node],r);
|
|
|
+ _starpu_data_request_list_push_front(&prefetch_requests[r->handling_node],r);
|
|
|
else
|
|
|
- _starpu_data_request_list_push_front(data_requests[r->handling_node],r);
|
|
|
+ _starpu_data_request_list_push_front(&data_requests[r->handling_node],r);
|
|
|
goto found;
|
|
|
}
|
|
|
}
|