|
@@ -63,7 +63,7 @@ static struct starpu_codelet cl22 =
|
|
|
/* Returns the MPI node number where data indexes index is */
|
|
|
int my_distrib(int x, int y, int nb_nodes)
|
|
|
{
|
|
|
- return (x+y) % nb_nodes;
|
|
|
+ return (x+y) % nb_nodes;
|
|
|
}
|
|
|
|
|
|
/*
|
|
@@ -74,93 +74,93 @@ static void dw_cholesky(float ***matA, unsigned size, unsigned ld, unsigned nblo
|
|
|
{
|
|
|
struct timeval start;
|
|
|
struct timeval end;
|
|
|
- starpu_data_handle_t **data_handles;
|
|
|
- int x, y;
|
|
|
+ starpu_data_handle_t **data_handles;
|
|
|
+ int x, y;
|
|
|
|
|
|
/* create all the DAG nodes */
|
|
|
unsigned i,j,k;
|
|
|
|
|
|
- data_handles = malloc(nblocks*sizeof(starpu_data_handle_t *));
|
|
|
- for(x=0 ; x<nblocks ; x++) data_handles[x] = malloc(nblocks*sizeof(starpu_data_handle_t));
|
|
|
+ data_handles = malloc(nblocks*sizeof(starpu_data_handle_t *));
|
|
|
+ for(x=0 ; x<nblocks ; x++) data_handles[x] = malloc(nblocks*sizeof(starpu_data_handle_t));
|
|
|
|
|
|
starpu_mpi_barrier(MPI_COMM_WORLD);
|
|
|
gettimeofday(&start, NULL);
|
|
|
|
|
|
- for(x = 0; x < nblocks ; x++)
|
|
|
+ for(x = 0; x < nblocks ; x++)
|
|
|
{
|
|
|
- for (y = 0; y < nblocks; y++)
|
|
|
+ for (y = 0; y < nblocks; y++)
|
|
|
{
|
|
|
- int mpi_rank = my_distrib(x, y, nodes);
|
|
|
- if (mpi_rank == rank)
|
|
|
+ int mpi_rank = my_distrib(x, y, nodes);
|
|
|
+ if (mpi_rank == rank)
|
|
|
{
|
|
|
- //fprintf(stderr, "[%d] Owning data[%d][%d]\n", rank, x, y);
|
|
|
- starpu_matrix_data_register(&data_handles[x][y], 0, (uintptr_t)matA[x][y],
|
|
|
- ld, size/nblocks, size/nblocks, sizeof(float));
|
|
|
- }
|
|
|
+ //fprintf(stderr, "[%d] Owning data[%d][%d]\n", rank, x, y);
|
|
|
+ starpu_matrix_data_register(&data_handles[x][y], 0, (uintptr_t)matA[x][y],
|
|
|
+ ld, size/nblocks, size/nblocks, sizeof(float));
|
|
|
+ }
|
|
|
/* TODO: make better test to only registering what is needed */
|
|
|
- else
|
|
|
+ else
|
|
|
{
|
|
|
- /* I don't own that index, but will need it for my computations */
|
|
|
- //fprintf(stderr, "[%d] Neighbour of data[%d][%d]\n", rank, x, y);
|
|
|
- starpu_matrix_data_register(&data_handles[x][y], -1, (uintptr_t)NULL,
|
|
|
- ld, size/nblocks, size/nblocks, sizeof(float));
|
|
|
- }
|
|
|
- if (data_handles[x][y])
|
|
|
+ /* I don't own that index, but will need it for my computations */
|
|
|
+ //fprintf(stderr, "[%d] Neighbour of data[%d][%d]\n", rank, x, y);
|
|
|
+ starpu_matrix_data_register(&data_handles[x][y], -1, (uintptr_t)NULL,
|
|
|
+ ld, size/nblocks, size/nblocks, sizeof(float));
|
|
|
+ }
|
|
|
+ if (data_handles[x][y])
|
|
|
{
|
|
|
- starpu_data_set_rank(data_handles[x][y], mpi_rank);
|
|
|
- starpu_data_set_tag(data_handles[x][y], (y*nblocks)+x);
|
|
|
+ starpu_data_set_rank(data_handles[x][y], mpi_rank);
|
|
|
+ starpu_data_set_tag(data_handles[x][y], (y*nblocks)+x);
|
|
|
}
|
|
|
- }
|
|
|
- }
|
|
|
+ }
|
|
|
+ }
|
|
|
|
|
|
for (k = 0; k < nblocks; k++)
|
|
|
- {
|
|
|
- int prio = STARPU_DEFAULT_PRIO;
|
|
|
- if (!noprio) prio = STARPU_MAX_PRIO;
|
|
|
+ {
|
|
|
+ int prio = STARPU_DEFAULT_PRIO;
|
|
|
+ if (!noprio) prio = STARPU_MAX_PRIO;
|
|
|
|
|
|
- starpu_mpi_insert_task(MPI_COMM_WORLD, &cl11,
|
|
|
- STARPU_PRIORITY, prio,
|
|
|
- STARPU_RW, data_handles[k][k],
|
|
|
- 0);
|
|
|
+ starpu_mpi_insert_task(MPI_COMM_WORLD, &cl11,
|
|
|
+ STARPU_PRIORITY, prio,
|
|
|
+ STARPU_RW, data_handles[k][k],
|
|
|
+ 0);
|
|
|
|
|
|
for (j = k+1; j<nblocks; j++)
|
|
|
{
|
|
|
- prio = STARPU_DEFAULT_PRIO;
|
|
|
- if (!noprio&& (j == k+1)) prio = STARPU_MAX_PRIO;
|
|
|
- starpu_mpi_insert_task(MPI_COMM_WORLD, &cl21,
|
|
|
- STARPU_PRIORITY, prio,
|
|
|
- STARPU_R, data_handles[k][k],
|
|
|
- STARPU_RW, data_handles[k][j],
|
|
|
- 0);
|
|
|
+ prio = STARPU_DEFAULT_PRIO;
|
|
|
+ if (!noprio&& (j == k+1)) prio = STARPU_MAX_PRIO;
|
|
|
+ starpu_mpi_insert_task(MPI_COMM_WORLD, &cl21,
|
|
|
+ STARPU_PRIORITY, prio,
|
|
|
+ STARPU_R, data_handles[k][k],
|
|
|
+ STARPU_RW, data_handles[k][j],
|
|
|
+ 0);
|
|
|
|
|
|
for (i = k+1; i<nblocks; i++)
|
|
|
{
|
|
|
if (i <= j)
|
|
|
- {
|
|
|
- prio = STARPU_DEFAULT_PRIO;
|
|
|
- if (!noprio && (i == k + 1) && (j == k +1) ) prio = STARPU_MAX_PRIO;
|
|
|
- starpu_mpi_insert_task(MPI_COMM_WORLD, &cl22,
|
|
|
- STARPU_PRIORITY, prio,
|
|
|
- STARPU_R, data_handles[k][i],
|
|
|
- STARPU_R, data_handles[k][j],
|
|
|
- STARPU_RW, data_handles[i][j],
|
|
|
- 0);
|
|
|
- }
|
|
|
+ {
|
|
|
+ prio = STARPU_DEFAULT_PRIO;
|
|
|
+ if (!noprio && (i == k + 1) && (j == k +1) ) prio = STARPU_MAX_PRIO;
|
|
|
+ starpu_mpi_insert_task(MPI_COMM_WORLD, &cl22,
|
|
|
+ STARPU_PRIORITY, prio,
|
|
|
+ STARPU_R, data_handles[k][i],
|
|
|
+ STARPU_R, data_handles[k][j],
|
|
|
+ STARPU_RW, data_handles[i][j],
|
|
|
+ 0);
|
|
|
+ }
|
|
|
}
|
|
|
}
|
|
|
- }
|
|
|
+ }
|
|
|
|
|
|
- starpu_task_wait_for_all();
|
|
|
+ starpu_task_wait_for_all();
|
|
|
|
|
|
- for(x = 0; x < nblocks ; x++)
|
|
|
+ for(x = 0; x < nblocks ; x++)
|
|
|
{
|
|
|
- for (y = 0; y < nblocks; y++)
|
|
|
+ for (y = 0; y < nblocks; y++)
|
|
|
{
|
|
|
- if (data_handles[x][y])
|
|
|
- starpu_data_unregister(data_handles[x][y]);
|
|
|
- }
|
|
|
+ if (data_handles[x][y])
|
|
|
+ starpu_data_unregister(data_handles[x][y]);
|
|
|
+ }
|
|
|
free(data_handles[x]);
|
|
|
- }
|
|
|
+ }
|
|
|
free(data_handles);
|
|
|
|
|
|
starpu_mpi_barrier(MPI_COMM_WORLD);
|
|
@@ -185,7 +185,7 @@ int main(int argc, char **argv)
|
|
|
* */
|
|
|
|
|
|
float ***bmat;
|
|
|
- int rank, nodes;
|
|
|
+ int rank, nodes;
|
|
|
|
|
|
parse_args(argc, argv);
|
|
|
|
|
@@ -201,14 +201,14 @@ int main(int argc, char **argv)
|
|
|
starpu_helper_cublas_init();
|
|
|
|
|
|
unsigned i,j,x,y;
|
|
|
- bmat = malloc(nblocks * sizeof(float *));
|
|
|
- for(x=0 ; x<nblocks ; x++)
|
|
|
+ bmat = malloc(nblocks * sizeof(float *));
|
|
|
+ for(x=0 ; x<nblocks ; x++)
|
|
|
{
|
|
|
- bmat[x] = malloc(nblocks * sizeof(float *));
|
|
|
- for(y=0 ; y<nblocks ; y++)
|
|
|
+ bmat[x] = malloc(nblocks * sizeof(float *));
|
|
|
+ for(y=0 ; y<nblocks ; y++)
|
|
|
{
|
|
|
- int mpi_rank = my_distrib(x, y, nodes);
|
|
|
- if (mpi_rank == rank)
|
|
|
+ int mpi_rank = my_distrib(x, y, nodes);
|
|
|
+ if (mpi_rank == rank)
|
|
|
{
|
|
|
starpu_malloc((void **)&bmat[x][y], BLOCKSIZE*BLOCKSIZE*sizeof(float));
|
|
|
for (i = 0; i < BLOCKSIZE; i++)
|
|
@@ -227,12 +227,12 @@ int main(int argc, char **argv)
|
|
|
|
|
|
starpu_mpi_shutdown();
|
|
|
|
|
|
- for(x=0 ; x<nblocks ; x++)
|
|
|
+ for(x=0 ; x<nblocks ; x++)
|
|
|
{
|
|
|
- for(y=0 ; y<nblocks ; y++)
|
|
|
+ for(y=0 ; y<nblocks ; y++)
|
|
|
{
|
|
|
- int mpi_rank = my_distrib(x, y, nodes);
|
|
|
- if (mpi_rank == rank)
|
|
|
+ int mpi_rank = my_distrib(x, y, nodes);
|
|
|
+ if (mpi_rank == rank)
|
|
|
{
|
|
|
starpu_free((void *)bmat[x][y]);
|
|
|
}
|
|
@@ -241,7 +241,7 @@ int main(int argc, char **argv)
|
|
|
}
|
|
|
free(bmat);
|
|
|
|
|
|
- starpu_helper_cublas_shutdown();
|
|
|
+ starpu_helper_cublas_shutdown();
|
|
|
starpu_shutdown();
|
|
|
|
|
|
return 0;
|