|
|
@@ -18,32 +18,23 @@
|
|
|
#include "helper.h"
|
|
|
|
|
|
/* Returns the MPI node number where data indexes index is */
|
|
|
-int my_distrib(int x, int y, int nb_nodes)
|
|
|
+int my_distrib(int x, int nb_nodes)
|
|
|
{
|
|
|
- return (x+y) % nb_nodes;
|
|
|
+ return x % nb_nodes;
|
|
|
}
|
|
|
|
|
|
void cpu_codelet(void *descr[], void *_args)
|
|
|
{
|
|
|
- float *block;
|
|
|
- unsigned nx = STARPU_MATRIX_GET_NY(descr[0]);
|
|
|
- unsigned ld = STARPU_MATRIX_GET_LD(descr[0]);
|
|
|
- unsigned i,j;
|
|
|
+ int *vector = (int *)STARPU_VECTOR_GET_PTR(descr[0]);
|
|
|
+ unsigned nx = STARPU_VECTOR_GET_NX(descr[0]);
|
|
|
+ unsigned i;
|
|
|
int rank;
|
|
|
- float factor;
|
|
|
|
|
|
- block = (float *)STARPU_MATRIX_GET_PTR(descr[0]);
|
|
|
starpu_codelet_unpack_args(_args, &rank);
|
|
|
- factor = block[0];
|
|
|
-
|
|
|
- //FPRINTF_MPI(stderr, "rank %d factor %f\n", rank, factor);
|
|
|
- for (j = 0; j < nx; j++)
|
|
|
+ for (i = 0; i < nx; i++)
|
|
|
{
|
|
|
- for (i = 0; i < nx; i++)
|
|
|
- {
|
|
|
- //FPRINTF_MPI(stderr, "rank %d factor %f --> %f %f\n", rank, factor, block[j+i*ld], block[j+i*ld]*factor);
|
|
|
- block[j+i*ld] *= factor;
|
|
|
- }
|
|
|
+ //fprintf(stderr,"rank %d v[%d] = %d\n", rank, i, vector[i]);
|
|
|
+ vector[i] *= rank+2;
|
|
|
}
|
|
|
}
|
|
|
|
|
|
@@ -68,18 +59,12 @@ void rcallback(void *arg STARPU_ATTRIBUTE_UNUSED)
|
|
|
|
|
|
int main(int argc, char **argv)
|
|
|
{
|
|
|
- int rank, nodes;
|
|
|
- float ***bmat = NULL;
|
|
|
+ int rank, nodes, ret, x;
|
|
|
+ int *vector = NULL;
|
|
|
starpu_data_handle_t *data_handles;
|
|
|
+ int size=10;
|
|
|
|
|
|
- unsigned i,j,x,y;
|
|
|
-
|
|
|
- unsigned nblocks=4;
|
|
|
- unsigned block_size=2;
|
|
|
- unsigned size = nblocks*block_size;
|
|
|
- unsigned ld = size / nblocks;
|
|
|
-
|
|
|
- int ret = starpu_init(NULL);
|
|
|
+ ret = starpu_init(NULL);
|
|
|
STARPU_CHECK_RETURN_VALUE(ret, "starpu_init");
|
|
|
ret = starpu_mpi_init(&argc, &argv, 1);
|
|
|
STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init");
|
|
|
@@ -88,95 +73,65 @@ int main(int argc, char **argv)
|
|
|
|
|
|
if (rank == 0)
|
|
|
{
|
|
|
- /* Allocate the matrix */
|
|
|
- int block_number=10;
|
|
|
- bmat = malloc(nblocks * sizeof(float *));
|
|
|
- for(x=0 ; x<nblocks ; x++)
|
|
|
+ /* Allocate the vector */
|
|
|
+ vector = malloc(size * sizeof(int));
|
|
|
+ for(x=0 ; x<size ; x++)
|
|
|
+ vector[x] = x+10;
|
|
|
+
|
|
|
+ // Print vector
|
|
|
+ FPRINTF_MPI(stderr, " Input vector: ");
|
|
|
+ for(x=0 ; x<size ; x++)
|
|
|
{
|
|
|
- bmat[x] = malloc(nblocks * sizeof(float *));
|
|
|
- for(y=0 ; y<nblocks ; y++)
|
|
|
- {
|
|
|
- float value=0.0;
|
|
|
- starpu_malloc((void **)&bmat[x][y], block_size*block_size*sizeof(float));
|
|
|
- for (i = 0; i < block_size; i++)
|
|
|
- {
|
|
|
- for (j = 0; j < block_size; j++)
|
|
|
- {
|
|
|
- bmat[x][y][j +i*block_size] = block_number + value;
|
|
|
- value++;
|
|
|
- }
|
|
|
- }
|
|
|
- block_number += 10;
|
|
|
- }
|
|
|
+ FPRINTF(stderr, "%d\t", vector[x]);
|
|
|
}
|
|
|
+ FPRINTF(stderr,"\n");
|
|
|
}
|
|
|
|
|
|
-#if 0
|
|
|
- // Print matrix
|
|
|
- if (rank == 0)
|
|
|
+ /* Allocate data handles and register data to StarPU */
|
|
|
+ data_handles = (starpu_data_handle_t *) calloc(size, sizeof(starpu_data_handle_t));
|
|
|
+ for(x = 0; x < size ; x++)
|
|
|
{
|
|
|
- fprintf(stderr, "Input matrix\n");
|
|
|
- for(x=0 ; x<nblocks ; x++)
|
|
|
+ int mpi_rank = my_distrib(x, nodes);
|
|
|
+ if (rank == 0)
|
|
|
{
|
|
|
- for(y=0 ; y<nblocks ; y++)
|
|
|
- {
|
|
|
- for (j = 0; j < block_size; j++)
|
|
|
- {
|
|
|
- for (i = 0; i < block_size; i++)
|
|
|
- {
|
|
|
- fprintf(stderr, "%2.2f\t", bmat[x][y][j+i*block_size]);
|
|
|
- }
|
|
|
- fprintf(stderr,"\n");
|
|
|
- }
|
|
|
- fprintf(stderr,"\n");
|
|
|
- }
|
|
|
+ starpu_vector_data_register(&data_handles[x], 0, (uintptr_t)&vector[x], 1, sizeof(int));
|
|
|
+ }
|
|
|
+ else if ((mpi_rank == rank))
|
|
|
+ {
|
|
|
+ /* I do not own that index but i will need it for my computations */
|
|
|
+ starpu_vector_data_register(&data_handles[x], -1, (uintptr_t)NULL, 1, sizeof(int));
|
|
|
+ }
|
|
|
+ else
|
|
|
+ {
|
|
|
+ /* I know it's useless to allocate anything for this */
|
|
|
+ data_handles[x] = NULL;
|
|
|
+ }
|
|
|
+ if (data_handles[x])
|
|
|
+ {
|
|
|
+ starpu_mpi_data_register(data_handles[x], x, 0);
|
|
|
}
|
|
|
}
|
|
|
-#endif
|
|
|
|
|
|
- /* Allocate data handles and register data to StarPU */
|
|
|
- data_handles = malloc(nblocks*nblocks*sizeof(starpu_data_handle_t *));
|
|
|
- for(x = 0; x < nblocks ; x++)
|
|
|
+ /* Scatter the matrix among the nodes */
|
|
|
+ for(x = 0; x < size ; x++)
|
|
|
{
|
|
|
- for (y = 0; y < nblocks; y++)
|
|
|
+ if (data_handles[x])
|
|
|
{
|
|
|
- int mpi_rank = my_distrib(x, y, nodes);
|
|
|
- if (rank == 0)
|
|
|
- {
|
|
|
- starpu_matrix_data_register(&data_handles[x+y*nblocks], STARPU_MAIN_RAM, (uintptr_t)bmat[x][y],
|
|
|
- ld, size/nblocks, size/nblocks, sizeof(float));
|
|
|
- }
|
|
|
- else if ((mpi_rank == rank) || ((rank == mpi_rank+1 || rank == mpi_rank-1)))
|
|
|
- {
|
|
|
- /* I own that index, or i will need it for my computations */
|
|
|
- //fprintf(stderr, "[%d] Owning or neighbor of data[%d][%d]\n", rank, x, y);
|
|
|
- starpu_matrix_data_register(&data_handles[x+y*nblocks], -1, (uintptr_t)NULL,
|
|
|
- ld, size/nblocks, size/nblocks, sizeof(float));
|
|
|
- }
|
|
|
- else
|
|
|
- {
|
|
|
- /* I know it's useless to allocate anything for this */
|
|
|
- data_handles[x+y*nblocks] = NULL;
|
|
|
- }
|
|
|
- if (data_handles[x+y*nblocks])
|
|
|
- {
|
|
|
- starpu_mpi_data_register(data_handles[x+y*nblocks], (y*nblocks)+x, mpi_rank);
|
|
|
- }
|
|
|
+ int mpi_rank = my_distrib(x, nodes);
|
|
|
+ starpu_mpi_data_set_rank(data_handles[x], mpi_rank);
|
|
|
}
|
|
|
}
|
|
|
-
|
|
|
- /* Scatter the matrix among the nodes */
|
|
|
- starpu_mpi_scatter_detached(data_handles, nblocks*nblocks, 0, MPI_COMM_WORLD, scallback, "scatter", NULL, NULL);
|
|
|
+ starpu_mpi_scatter_detached(data_handles, size, 0, MPI_COMM_WORLD, scallback, "scatter", NULL, NULL);
|
|
|
|
|
|
/* Calculation */
|
|
|
- for(x = 0; x < nblocks*nblocks ; x++)
|
|
|
+ for(x = 0; x < size ; x++)
|
|
|
{
|
|
|
if (data_handles[x])
|
|
|
{
|
|
|
- int owner = starpu_data_get_rank(data_handles[x]);
|
|
|
+ int owner = starpu_mpi_data_get_rank(data_handles[x]);
|
|
|
if (owner == rank)
|
|
|
{
|
|
|
- //fprintf(stderr,"[%d] Computing on data[%d]\n", rank, x);
|
|
|
+ FPRINTF_MPI(stderr,"Computing on data[%d]\n", x);
|
|
|
starpu_task_insert(&cl,
|
|
|
STARPU_VALUE, &rank, sizeof(rank),
|
|
|
STARPU_RW, data_handles[x],
|
|
|
@@ -186,57 +141,49 @@ int main(int argc, char **argv)
|
|
|
}
|
|
|
|
|
|
/* Gather the matrix on main node */
|
|
|
- starpu_mpi_gather_detached(data_handles, nblocks*nblocks, 0, MPI_COMM_WORLD, scallback, "gather", rcallback, "gather");
|
|
|
-
|
|
|
- /* Unregister matrix from StarPU */
|
|
|
- for(x=0 ; x<nblocks*nblocks ; x++)
|
|
|
+ starpu_mpi_gather_detached(data_handles, size, 0, MPI_COMM_WORLD, scallback, "gather", rcallback, "gather");
|
|
|
+ for(x = 0; x < size ; x++)
|
|
|
{
|
|
|
if (data_handles[x])
|
|
|
{
|
|
|
- starpu_data_unregister(data_handles[x]);
|
|
|
+ starpu_mpi_data_set_rank(data_handles[x], 0);
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-#if 0
|
|
|
- // Print matrix
|
|
|
- if (rank == 0)
|
|
|
+ /* Unregister matrix from StarPU */
|
|
|
+ for(x=0 ; x<size ; x++)
|
|
|
{
|
|
|
- fprintf(stderr, "Output matrix\n");
|
|
|
- for(x=0 ; x<nblocks ; x++)
|
|
|
+ if (data_handles[x])
|
|
|
{
|
|
|
- for(y=0 ; y<nblocks ; y++)
|
|
|
- {
|
|
|
- for (j = 0; j < block_size; j++)
|
|
|
- {
|
|
|
- for (i = 0; i < block_size; i++)
|
|
|
- {
|
|
|
- fprintf(stderr, "%2.2f\t", bmat[x][y][j+i*block_size]);
|
|
|
- }
|
|
|
- fprintf(stderr,"\n");
|
|
|
- }
|
|
|
- fprintf(stderr,"\n");
|
|
|
- }
|
|
|
+ starpu_data_unregister(data_handles[x]);
|
|
|
}
|
|
|
}
|
|
|
-#endif
|
|
|
|
|
|
- // Free memory
|
|
|
- free(data_handles);
|
|
|
+ // Print vector
|
|
|
if (rank == 0)
|
|
|
{
|
|
|
- for(x=0 ; x<nblocks ; x++)
|
|
|
+ FPRINTF_MPI(stderr, "Output vector: ");
|
|
|
+ for(x=0 ; x<size ; x++)
|
|
|
+ {
|
|
|
+ FPRINTF(stderr, "%d\t", vector[x]);
|
|
|
+ }
|
|
|
+ FPRINTF(stderr,"\n");
|
|
|
+ for(x=0 ; x<size ; x++)
|
|
|
{
|
|
|
- for(y=0 ; y<nblocks ; y++)
|
|
|
+ int mpi_rank = my_distrib(x, nodes);
|
|
|
+ if (vector[x] != (x+10) * (mpi_rank+2))
|
|
|
{
|
|
|
- starpu_free((void *)bmat[x][y]);
|
|
|
+ FPRINTF_MPI(stderr, "Incorrect value for vector[%d]. computed %d != expected %d\n", x, vector[x], (x+10) * (mpi_rank+2));
|
|
|
+ ret = 1;
|
|
|
}
|
|
|
- free(bmat[x]);
|
|
|
}
|
|
|
- free(bmat);
|
|
|
+ free(vector);
|
|
|
}
|
|
|
|
|
|
+ // Free memory
|
|
|
+ free(data_handles);
|
|
|
|
|
|
starpu_mpi_shutdown();
|
|
|
starpu_shutdown();
|
|
|
- return 0;
|
|
|
+ return (rank == 0) ? ret : 0;
|
|
|
}
|
Nathalie Furmento