|
|
@@ -0,0 +1,220 @@
|
|
|
+/* StarPU --- Runtime system for heterogeneous multicore architectures.
|
|
|
+ *
|
|
|
+ * Copyright (C) 2009-2011 Université de Bordeaux 1
|
|
|
+ * Copyright (C) 2010 Mehdi Juhoor <mjuhoor@gmail.com>
|
|
|
+ * Copyright (C) 2010, 2011 Centre National de la Recherche Scientifique
|
|
|
+ *
|
|
|
+ * StarPU is free software; you can redistribute it and/or modify
|
|
|
+ * it under the terms of the GNU Lesser General Public License as published by
|
|
|
+ * the Free Software Foundation; either version 2.1 of the License, or (at
|
|
|
+ * your option) any later version.
|
|
|
+ *
|
|
|
+ * StarPU is distributed in the hope that it will be useful, but
|
|
|
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
|
|
|
+ *
|
|
|
+ * See the GNU Lesser General Public License in COPYING.LGPL for more details.
|
|
|
+ */
|
|
|
+
|
|
|
+#include <starpu_mpi.h>
|
|
|
+#include "mpi_cholesky.h"
|
|
|
+#include "mpi_cholesky_models.h"
|
|
|
+
|
|
|
+/*
|
|
|
+ * Create the codelets
|
|
|
+ */
|
|
|
+
|
|
|
+static starpu_codelet cl11 =
|
|
|
+{
|
|
|
+ .where = STARPU_CPU|STARPU_CUDA,
|
|
|
+ .cpu_func = chol_cpu_codelet_update_u11,
|
|
|
+#ifdef STARPU_USE_CUDA
|
|
|
+ .cuda_func = chol_cublas_codelet_update_u11,
|
|
|
+#endif
|
|
|
+ .nbuffers = 1,
|
|
|
+ .model = &chol_model_11
|
|
|
+};
|
|
|
+
|
|
|
+static starpu_codelet cl21 =
|
|
|
+{
|
|
|
+ .where = STARPU_CPU|STARPU_CUDA,
|
|
|
+ .cpu_func = chol_cpu_codelet_update_u21,
|
|
|
+#ifdef STARPU_USE_CUDA
|
|
|
+ .cuda_func = chol_cublas_codelet_update_u21,
|
|
|
+#endif
|
|
|
+ .nbuffers = 2,
|
|
|
+ .model = &chol_model_21
|
|
|
+};
|
|
|
+
|
|
|
+static starpu_codelet cl22 =
|
|
|
+{
|
|
|
+ .where = STARPU_CPU|STARPU_CUDA,
|
|
|
+ .cpu_func = chol_cpu_codelet_update_u22,
|
|
|
+#ifdef STARPU_USE_CUDA
|
|
|
+ .cuda_func = chol_cublas_codelet_update_u22,
|
|
|
+#endif
|
|
|
+ .nbuffers = 3,
|
|
|
+ .model = &chol_model_22
|
|
|
+};
|
|
|
+
|
|
|
+/* Returns the MPI node number where data indexes index is */
|
|
|
+int my_distrib(int x, int y, int nb_nodes) {
|
|
|
+ return (x+y) % nb_nodes;
|
|
|
+}
|
|
|
+
|
|
|
+/*
|
|
|
+ * code to bootstrap the factorization
|
|
|
+ * and construct the DAG
|
|
|
+ */
|
|
|
+static void dw_cholesky(float ***matA, unsigned size, unsigned ld, unsigned nblocks, int rank, int nodes)
|
|
|
+{
|
|
|
+ struct timeval start;
|
|
|
+ struct timeval end;
|
|
|
+ starpu_data_handle **data_handles;
|
|
|
+ int x, y;
|
|
|
+
|
|
|
+ /* create all the DAG nodes */
|
|
|
+ unsigned i,j,k;
|
|
|
+
|
|
|
+ data_handles = malloc(nblocks*sizeof(starpu_data_handle *));
|
|
|
+ for(x=0 ; x<nblocks ; x++) data_handles[x] = malloc(nblocks*sizeof(starpu_data_handle));
|
|
|
+
|
|
|
+ gettimeofday(&start, NULL);
|
|
|
+ for(x = 0; x < nblocks ; x++) {
|
|
|
+ for (y = 0; y < nblocks; y++) {
|
|
|
+ int mpi_rank = my_distrib(x, y, nodes);
|
|
|
+ if (mpi_rank == rank) {
|
|
|
+ //fprintf(stderr, "[%d] Owning data[%d][%d]\n", rank, x, y);
|
|
|
+ starpu_matrix_data_register(&data_handles[x][y], 0, (uintptr_t)matA[x][y],
|
|
|
+ ld, size/nblocks, size/nblocks, sizeof(float));
|
|
|
+ }
|
|
|
+ /* TODO: make better test to only registering what is needed */
|
|
|
+ else {
|
|
|
+ /* I don't own that index, but will need it for my computations */
|
|
|
+ //fprintf(stderr, "[%d] Neighbour of data[%d][%d]\n", rank, x, y);
|
|
|
+ starpu_matrix_data_register(&data_handles[x][y], -1, (uintptr_t)NULL,
|
|
|
+ ld, size/nblocks, size/nblocks, sizeof(float));
|
|
|
+ }
|
|
|
+ if (data_handles[x][y])
|
|
|
+ starpu_data_set_rank(data_handles[x][y], mpi_rank);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ for (k = 0; k < nblocks; k++)
|
|
|
+ {
|
|
|
+ int prio = STARPU_DEFAULT_PRIO;
|
|
|
+ if (!noprio) prio = STARPU_MAX_PRIO;
|
|
|
+
|
|
|
+ starpu_mpi_insert_task(MPI_COMM_WORLD, &cl11,
|
|
|
+ STARPU_PRIORITY, prio,
|
|
|
+ STARPU_RW, data_handles[k][k],
|
|
|
+ 0);
|
|
|
+
|
|
|
+ for (j = k+1; j<nblocks; j++)
|
|
|
+ {
|
|
|
+ prio = STARPU_DEFAULT_PRIO;
|
|
|
+ if (!noprio&& (j == k+1)) prio = STARPU_MAX_PRIO;
|
|
|
+ starpu_mpi_insert_task(MPI_COMM_WORLD, &cl21,
|
|
|
+ STARPU_PRIORITY, prio,
|
|
|
+ STARPU_R, data_handles[k][k],
|
|
|
+ STARPU_RW, data_handles[k][j],
|
|
|
+ 0);
|
|
|
+
|
|
|
+ for (i = k+1; i<nblocks; i++)
|
|
|
+ {
|
|
|
+ if (i <= j)
|
|
|
+ {
|
|
|
+ prio = STARPU_DEFAULT_PRIO;
|
|
|
+ if (!noprio && (i == k + 1) && (j == k +1) ) prio = STARPU_MAX_PRIO;
|
|
|
+ starpu_mpi_insert_task(MPI_COMM_WORLD, &cl22,
|
|
|
+ STARPU_PRIORITY, prio,
|
|
|
+ STARPU_R, data_handles[k][i],
|
|
|
+ STARPU_R, data_handles[k][j],
|
|
|
+ STARPU_RW, data_handles[i][j],
|
|
|
+ 0);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ starpu_task_wait_for_all();
|
|
|
+
|
|
|
+ for(x = 0; x < nblocks ; x++) {
|
|
|
+ for (y = 0; y < nblocks; y++) {
|
|
|
+ if (data_handles[x][y])
|
|
|
+ starpu_data_unregister(data_handles[x][y]);
|
|
|
+ }
|
|
|
+ free(data_handles[x]);
|
|
|
+ }
|
|
|
+ free(data_handles);
|
|
|
+
|
|
|
+ gettimeofday(&end, NULL);
|
|
|
+
|
|
|
+ double timing = (double)((end.tv_sec - start.tv_sec)*1000000 + (end.tv_usec - start.tv_usec));
|
|
|
+ fprintf(stderr, "Computation took (in ms)\n");
|
|
|
+ printf("%2.2f\n", timing/1000);
|
|
|
+
|
|
|
+ double flop = (1.0f*size*size*size)/3.0f;
|
|
|
+ fprintf(stderr, "Synthetic GFlops : %2.2f\n", (flop/timing/1000.0f));
|
|
|
+}
|
|
|
+
|
|
|
+int main(int argc, char **argv)
|
|
|
+{
|
|
|
+ /* create a simple definite positive symetric matrix example
|
|
|
+ *
|
|
|
+ * Hilbert matrix : h(i,j) = 1/(i+j+1)
|
|
|
+ * */
|
|
|
+
|
|
|
+ float ***bmat;
|
|
|
+ int rank, nodes;
|
|
|
+
|
|
|
+ parse_args(argc, argv);
|
|
|
+
|
|
|
+ starpu_init(NULL);
|
|
|
+ starpu_mpi_initialize_extended(&rank, &nodes);
|
|
|
+ starpu_helper_cublas_init();
|
|
|
+
|
|
|
+ unsigned i,j,x,y;
|
|
|
+ bmat = malloc(nblocks * sizeof(float *));
|
|
|
+ for(x=0 ; x<nblocks ; x++)
|
|
|
+ {
|
|
|
+ bmat[x] = malloc(nblocks * sizeof(float *));
|
|
|
+ for(y=0 ; y<nblocks ; y++)
|
|
|
+ {
|
|
|
+ int mpi_rank = my_distrib(x, y, nodes);
|
|
|
+ if (mpi_rank == rank) {
|
|
|
+ starpu_malloc((void **)&bmat[x][y], BLOCKSIZE*BLOCKSIZE*sizeof(float));
|
|
|
+ for (i = 0; i < BLOCKSIZE; i++)
|
|
|
+ {
|
|
|
+ for (j = 0; j < BLOCKSIZE; j++)
|
|
|
+ {
|
|
|
+ bmat[x][y][j +i*BLOCKSIZE] = (1.0f/(1.0f+(i+(x*BLOCKSIZE)+j+(y*BLOCKSIZE)))) + ((i+(x*BLOCKSIZE) == j+(y*BLOCKSIZE))?1.0f*size:0.0f);
|
|
|
+ //mat[j +i*size] = ((i == j)?1.0f*size:0.0f);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ dw_cholesky(bmat, size, size/nblocks, nblocks, rank, nodes);
|
|
|
+
|
|
|
+ starpu_mpi_shutdown();
|
|
|
+
|
|
|
+ for(x=0 ; x<nblocks ; x++)
|
|
|
+ {
|
|
|
+ for(y=0 ; y<nblocks ; y++)
|
|
|
+ {
|
|
|
+ int mpi_rank = my_distrib(x, y, nodes);
|
|
|
+ if (mpi_rank == rank) {
|
|
|
+ starpu_free((void *)bmat[x][y]);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ free(bmat[x]);
|
|
|
+ }
|
|
|
+ free(bmat);
|
|
|
+
|
|
|
+ starpu_helper_cublas_shutdown();
|
|
|
+ starpu_shutdown();
|
|
|
+
|
|
|
+ return 0;
|
|
|
+}
|
Nathalie Furmento