add numa buses and workers to manage data on them

src/core/perfmodel/perfmodel_bus.c

@@ -2,6 +2,7 @@
  *
  * Copyright (C) 2009-2016  Université de Bordeaux
  * Copyright (C) 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017  CNRS
+ * Copyright (C) 2017  Inria
  * Copyright (C) 2013 Corentin Salingue
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -2940,8 +2941,6 @@ double starpu_transfer_predict(unsigned src_node, unsigned dst_node, size_t size
 	return latency + (size/bandwidth)*2*ngpus;
 }
 
-/* TODO: NUMA nodes */
-
 /* calculate save bandwidth and latency */
 /* bandwidth in MB/s - latency in µs */
 void _starpu_save_bandwidth_and_latency_disk(double bandwidth_write, double bandwidth_read, double latency_write, double latency_read, unsigned node)

src/core/topology.c

@@ -66,6 +66,7 @@ static int nobind;
 static int cpu_worker[STARPU_MAXCPUS];
 static unsigned nb_numa_nodes = 0;
 static unsigned numa_memory_nodes[STARPU_MAXNUMANODES]; /* indexed by StarPU numa node to convert in hwloc logid */
+static unsigned numa_bus_id[STARPU_MAXNUMANODES*STARPU_MAXNUMANODES];
 static int _starpu_worker_numa_node(unsigned workerid);
 
 #if defined(STARPU_USE_CUDA) || defined(STARPU_USE_OPENCL) || defined(STARPU_USE_SCC) || defined(STARPU_SIMGRID) || defined(STARPU_USE_MPI_MASTER_SLAVE)
@@ -2020,6 +2021,15 @@ static void _starpu_init_numa_node(struct _starpu_machine_config *config)
 	}	
 }
 
+static void _starpu_init_numa_bus()
+{
+	unsigned i, j;
+	for (i = 0; i < nb_numa_nodes; i++)
+		for (j = 0; j < nb_numa_nodes; j++)
+			if (i != j)
+				numa_bus_id[i*nb_numa_nodes+j] = _starpu_register_bus(i, j);
+}
+
 static void
 _starpu_init_workers_binding_and_memory (struct _starpu_machine_config *config, int no_mp_config STARPU_ATTRIBUTE_UNUSED)
 {
@@ -2063,6 +2073,7 @@ _starpu_init_workers_binding_and_memory (struct _starpu_machine_config *config,
 
 	/* Initialize NUMA nodes */
 	_starpu_init_numa_node(config);
+	_starpu_init_numa_bus();
 
 	unsigned worker;
 	for (worker = 0; worker < config->topology.nworkers; worker++)
@@ -2096,6 +2107,8 @@ _starpu_init_workers_binding_and_memory (struct _starpu_machine_config *config,
 			}
 #if defined(STARPU_USE_CUDA) || defined(STARPU_SIMGRID)
 			case STARPU_CUDA_WORKER:
+			{
+				unsigned numa;
 #ifndef STARPU_SIMGRID
 				if (may_bind_automatically[STARPU_CUDA_WORKER])
 				{
@@ -2125,9 +2138,11 @@ _starpu_init_workers_binding_and_memory (struct _starpu_machine_config *config,
 						workerarg->bindid = cuda_bindid[devid] = _starpu_get_next_bindid(config, preferred_binding, npreferred);
 					memory_node = cuda_memory_nodes[devid] = _starpu_memory_node_register(STARPU_CUDA_RAM, devid);
 
-					/* TODO: NUMA nodes */
-					_starpu_cuda_bus_ids[0][devid+1] = _starpu_register_bus(STARPU_MAIN_RAM, memory_node);
-					_starpu_cuda_bus_ids[devid+1][0] = _starpu_register_bus(memory_node, STARPU_MAIN_RAM);
+					for (numa = 0; numa < nb_numa_nodes; numa++)
+					{
+						_starpu_cuda_bus_ids[numa][devid+STARPU_MAXNUMANODES] = _starpu_register_bus(numa, memory_node);
+						_starpu_cuda_bus_ids[devid+STARPU_MAXNUMANODES][numa] = _starpu_register_bus(memory_node, numa);
+					}
 #ifdef STARPU_SIMGRID
 					const char* cuda_memcpy_peer;
 					snprintf(name, sizeof(name), "CUDA%d", devid);
@@ -2154,8 +2169,8 @@ _starpu_init_workers_binding_and_memory (struct _starpu_machine_config *config,
 							if (workerarg2->arch == STARPU_CUDA_WORKER)
 							{
 								unsigned memory_node2 = starpu_worker_get_memory_node(worker2);
-								_starpu_cuda_bus_ids[devid2][devid] = _starpu_register_bus(memory_node2, memory_node);
-								_starpu_cuda_bus_ids[devid][devid2] = _starpu_register_bus(memory_node, memory_node2);
+								_starpu_cuda_bus_ids[devid2+STARPU_MAXNUMANODES][devid+STARPU_MAXNUMANODES] = _starpu_register_bus(memory_node2, memory_node);
+								_starpu_cuda_bus_ids[devid+STARPU_MAXNUMANODES][devid2+STARPU_MAXNUMANODES] = _starpu_register_bus(memory_node, memory_node2);
 #ifndef STARPU_SIMGRID
 #if defined(HAVE_DECL_HWLOC_CUDA_GET_DEVICE_OSDEV_BY_INDEX) && HAVE_DECL_HWLOC_CUDA_GET_DEVICE_OSDEV_BY_INDEX
 								{
@@ -2173,8 +2188,8 @@ _starpu_init_workers_binding_and_memory (struct _starpu_machine_config *config,
 											_STARPU_DEBUG("CUDA%u and CUDA%u are linked through %s, along %u GPUs\n", devid, devid2, name, data->ngpus);
 										}
 #endif
-										starpu_bus_set_ngpus(_starpu_cuda_bus_ids[devid2][devid], data->ngpus);
-										starpu_bus_set_ngpus(_starpu_cuda_bus_ids[devid][devid2], data->ngpus);
+										starpu_bus_set_ngpus(_starpu_cuda_bus_ids[devid2+STARPU_MAXNUMANODES][devid+STARPU_MAXNUMANODES], data->ngpus);
+										starpu_bus_set_ngpus(_starpu_cuda_bus_ids[devid+STARPU_MAXNUMANODES][devid2+STARPU_MAXNUMANODES], data->ngpus);
 									}
 								}
 #endif
@@ -2185,13 +2200,19 @@ _starpu_init_workers_binding_and_memory (struct _starpu_machine_config *config,
 				}
 				_starpu_memory_node_add_nworkers(memory_node);
 
-                                _starpu_worker_drives_memory_node(&workerarg->set->workers[0], STARPU_MAIN_RAM);
+				//This worker can manage transfers on NUMA nodes
+				for (numa = 0; numa < nb_numa_nodes; numa++)
+						_starpu_worker_drives_memory_node(&workerarg->set->workers[0], numa);
+
 				_starpu_worker_drives_memory_node(&workerarg->set->workers[0], memory_node);
 				break;
+			}
 #endif
 
 #if defined(STARPU_USE_OPENCL) || defined(STARPU_SIMGRID)
 		        case STARPU_OPENCL_WORKER:
+			{
+				unsigned numa;
 #ifndef STARPU_SIMGRID
 				if (may_bind_automatically[STARPU_OPENCL_WORKER])
 				{
@@ -2212,9 +2233,12 @@ _starpu_init_workers_binding_and_memory (struct _starpu_machine_config *config,
 					opencl_init[devid] = 1;
 					workerarg->bindid = opencl_bindid[devid] = _starpu_get_next_bindid(config, preferred_binding, npreferred);
 					memory_node = opencl_memory_nodes[devid] = _starpu_memory_node_register(STARPU_OPENCL_RAM, devid);
-					/* TODO: NUMA nodes */
-					_starpu_register_bus(STARPU_MAIN_RAM, memory_node);
-					_starpu_register_bus(memory_node, STARPU_MAIN_RAM);
+
+					for (numa = 0; numa < nb_numa_nodes; numa++)
+					{
+						_starpu_register_bus(numa, memory_node);
+						_starpu_register_bus(memory_node, numa);
+					}
 #ifdef STARPU_SIMGRID
 					snprintf(name, sizeof(name), "OpenCL%d", devid);
 					host = _starpu_simgrid_get_host_by_name(name);
@@ -2224,13 +2248,19 @@ _starpu_init_workers_binding_and_memory (struct _starpu_machine_config *config,
 				}
 				_starpu_memory_node_add_nworkers(memory_node);
 
-                                _starpu_worker_drives_memory_node(workerarg, STARPU_MAIN_RAM);
+				//This worker can manage transfers on NUMA nodes
+				for (numa = 0; numa < nb_numa_nodes; numa++)
+						_starpu_worker_drives_memory_node(workerarg, numa);
+
 				_starpu_worker_drives_memory_node(workerarg, memory_node);
 				break;
+			}
 #endif
 
 #ifdef STARPU_USE_MIC
 		        case STARPU_MIC_WORKER:
+			{
+				unsigned numa;
 				if (mic_init[devid])
 				{
 					memory_node = mic_memory_nodes[devid];
@@ -2248,22 +2278,29 @@ _starpu_init_workers_binding_and_memory (struct _starpu_machine_config *config,
 					mic_bindid[devid] = _starpu_get_next_bindid(config, preferred_binding, npreferred);
 					memory_node = mic_memory_nodes[devid] = _starpu_memory_node_register(STARPU_MIC_RAM, devid);
 
-					/* TODO: NUMA nodes */
-					_starpu_register_bus(STARPU_MAIN_RAM, memory_node);
-					_starpu_register_bus(memory_node, STARPU_MAIN_RAM);
+					for (numa = 0; numa < nb_numa_nodes; numa++)
+					{
+						_starpu_register_bus(numa, memory_node);
+						_starpu_register_bus(memory_node, numa);
+					}
 
 				}
 				workerarg->bindid = mic_bindid[devid];
 				_starpu_memory_node_add_nworkers(memory_node);
 
-                                _starpu_worker_drives_memory_node(&workerarg->set->workers[0], STARPU_MAIN_RAM);
+				//This worker can manage transfers on NUMA nodes
+				for (numa = 0; numa < nb_numa_nodes; numa++)
+						_starpu_worker_drives_memory_node(&workerarg->set->workers[0], numa);
+
 				_starpu_worker_drives_memory_node(&workerarg->set->workers[0], memory_node);
 				break;
+			}
 #endif /* STARPU_USE_MIC */
 
 #ifdef STARPU_USE_SCC
 			case STARPU_SCC_WORKER:
 			{
+				unsigned numa;
 				/* Node 0 represents the SCC shared memory when we're on SCC. */
 				struct _starpu_memory_node_descr *descr = _starpu_memory_node_get_description();
 				descr->nodes[ram_memory_node] = STARPU_SCC_SHM;
@@ -2271,7 +2308,10 @@ _starpu_init_workers_binding_and_memory (struct _starpu_machine_config *config,
 				memory_node = ram_memory_node;
 				_starpu_memory_node_add_nworkers(memory_node);
 
-                                _starpu_worker_drives_memory_node(workerarg, STARPU_MAIN_RAM);
+				//This worker can manage transfers on NUMA nodes
+				for (numa = 0; numa < nb_numa_nodes; numa++)
+						_starpu_worker_drives_memory_node(workerarg, numa);
+
 				_starpu_worker_drives_memory_node(workerarg, memory_node);
 			}
 				break;
@@ -2280,6 +2320,7 @@ _starpu_init_workers_binding_and_memory (struct _starpu_machine_config *config,
 #ifdef STARPU_USE_MPI_MASTER_SLAVE
 			case STARPU_MPI_MS_WORKER:
 			{
+				unsigned numa;
 				if (mpi_init[devid])
 				{
 					memory_node = mpi_memory_nodes[devid];
@@ -2289,11 +2330,18 @@ _starpu_init_workers_binding_and_memory (struct _starpu_machine_config *config,
 					mpi_init[devid] = 1;
 					mpi_bindid[devid] = _starpu_get_next_bindid(config, preferred_binding, npreferred);
 					memory_node = mpi_memory_nodes[devid] = _starpu_memory_node_register(STARPU_MPI_MS_RAM, devid);
-					_starpu_register_bus(STARPU_MAIN_RAM, memory_node);
-					_starpu_register_bus(memory_node, STARPU_MAIN_RAM);
+		
+					for (numa = 0; numa < nb_numa_nodes; numa++)
+					{	
+						_starpu_register_bus(numa, memory_node);
+						_starpu_register_bus(memory_node, numa);
+					}
 
 				}
-                                _starpu_worker_drives_memory_node(&workerarg->set->workers[0], STARPU_MAIN_RAM);
+				//This worker can manage transfers on NUMA nodes
+				for (numa = 0; numa < nb_numa_nodes; numa++)
+						_starpu_worker_drives_memory_node(&workerarg->set->workers[0], numa);
+
 				_starpu_worker_drives_memory_node(&workerarg->set->workers[0], memory_node);
 #ifndef STARPU_MPI_MASTER_SLAVE_MULTIPLE_THREAD
                                 /* MPI driver thread can manage all slave memories if we disable the MPI multiple thread */

src/datawizard/memory_nodes.c

@@ -2,6 +2,7 @@
  *
  * Copyright (C) 2009-2017  Université de Bordeaux
  * Copyright (C) 2010, 2011, 2012, 2013, 2014, 2015  CNRS
+ * Copyright (C) 2017  Inria
  *
  * StarPU is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
@@ -76,7 +77,7 @@ void _starpu_memory_node_get_name(unsigned node, char *name, int size)
 	switch (_starpu_descr.nodes[node])
 	{
 	case STARPU_CPU_RAM:
-		prefix = "RAM";
+		prefix = "NUMA";
 		break;
 	case STARPU_CUDA_RAM:
 		prefix = "CUDA";

src/drivers/cuda/driver_cuda.c

@@ -5,6 +5,7 @@
  * Copyright (C) 2010, 2011, 2012, 2013, 2014, 2016, 2017  CNRS
  * Copyright (C) 2011  Télécom-SudParis
  * Copyright (C) 2016  Uppsala University
+ * Copyright (C) 2017  Inria
  *
  * StarPU is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
@@ -52,7 +53,7 @@
 static int ncudagpus = -1;
 
 static size_t global_mem[STARPU_MAXCUDADEVS];
-int _starpu_cuda_bus_ids[STARPU_MAXCUDADEVS+1][STARPU_MAXCUDADEVS+1];
+int _starpu_cuda_bus_ids[STARPU_MAXCUDADEVS+STARPU_MAXNUMANODES][STARPU_MAXCUDADEVS+STARPU_MAXNUMANODES];
 #ifdef STARPU_USE_CUDA
 static cudaStream_t streams[STARPU_NMAXWORKERS];
 static cudaStream_t out_transfer_streams[STARPU_MAXCUDADEVS];
@@ -322,7 +323,7 @@ static void init_device_context(unsigned devid, unsigned memnode)
 					{
 						_STARPU_DEBUG("Enabled GPU-Direct %d -> %d\n", worker->devid, devid);
 						/* direct copies are made from the destination, see link_supports_direct_transfers */
-						starpu_bus_set_direct(_starpu_cuda_bus_ids[worker->devid][devid], 1);
+						starpu_bus_set_direct(_starpu_cuda_bus_ids[worker->devid+STARPU_MAXNUMANODES][devid+STARPU_MAXNUMANODES], 1);
 					}
 				}
 			}