Move NUMA size registration to NUMA node registration instead of worker startup, thus fixing size for NUMA nodes without a worker

src/core/topology.c

@@ -25,6 +25,7 @@
 #include <core/debug.h>
 #include <core/topology.h>
 #include <drivers/cuda/driver_cuda.h>
+#include <drivers/cpu/driver_cpu.h>
 #include <drivers/mic/driver_mic_source.h>
 #include <drivers/scc/driver_scc_source.h>
 #include <drivers/mpi/driver_mpi_source.h>
@@ -35,6 +36,7 @@
 #include <profiling/profiling.h>
 #include <datawizard/datastats.h>
 #include <datawizard/memory_nodes.h>
+#include <datawizard/memory_manager.h>
 #include <common/uthash.h>
 
 #ifdef STARPU_HAVE_HWLOC
@@ -2018,6 +2020,7 @@ static void _starpu_init_numa_node(struct _starpu_machine_config *config)
 				{
 					int devid = numa_logical_id == STARPU_NUMA_MAIN_RAM ? 0 : numa_logical_id;
 					int memnode = _starpu_memory_node_register(STARPU_CPU_RAM, devid);
+					_starpu_memory_manager_set_global_memory_size(memnode, _starpu_cpu_get_global_mem_size(devid, config));
 					STARPU_ASSERT_MSG(memnode < STARPU_MAXNUMANODES, "Wrong Memory Node : %d (only %d available)", memnode, STARPU_MAXNUMANODES);
 					numa_memory_nodes_to_hwloclogid[memnode] = numa_logical_id;
 					int numa_physical_id = _starpu_get_physical_numa_node_worker(worker);
@@ -2064,6 +2067,7 @@ static void _starpu_init_numa_node(struct _starpu_machine_config *config)
 			if (numa_starpu_id == -1)
 			{
 				int memnode = _starpu_memory_node_register(STARPU_CPU_RAM, obj->logical_index);
+				_starpu_memory_manager_set_global_memory_size(memnode, _starpu_cpu_get_global_mem_size(obj->logical_index, config));
 				STARPU_ASSERT_MSG(memnode < STARPU_MAXNUMANODES, "Wrong Memory Node : %d (only %d available)", memnode, STARPU_MAXNUMANODES);
 				numa_memory_nodes_to_hwloclogid[memnode] = obj->logical_index;
 				numa_memory_nodes_to_physicalid[memnode] = obj->os_index;
@@ -2123,6 +2127,7 @@ static void _starpu_init_numa_node(struct _starpu_machine_config *config)
 					if (numa_starpu_id == -1)
 					{
 						int memnode = _starpu_memory_node_register(STARPU_CPU_RAM, obj->logical_index);
+						_starpu_memory_manager_set_global_memory_size(memnode, _starpu_cpu_get_global_mem_size(obj->logical_index, config));
 						STARPU_ASSERT_MSG(memnode < STARPU_MAXNUMANODES, "Wrong Memory Node : %d (only %d available)", memnode, STARPU_MAXNUMANODES);
 						numa_memory_nodes_to_hwloclogid[memnode] = obj->logical_index;
 						numa_memory_nodes_to_physicalid[memnode] = obj->os_index;
@@ -2164,44 +2169,39 @@ static void _starpu_init_numa_node(struct _starpu_machine_config *config)
 	for (numa = 0; numa < nnuma; numa++)
 	{
 #if defined(STARPU_HAVE_HWLOC)
-		if (nnuma > 1)
-		{
-			hwloc_obj_t obj = hwloc_get_obj_by_type(config->topology.hwtopology, HWLOC_OBJ_NUMANODE, numa);
-			unsigned numa_logical_id = obj->logical_index;
-			unsigned numa_physical_id = obj->os_index;
+		hwloc_obj_t obj = hwloc_get_obj_by_type(config->topology.hwtopology, HWLOC_OBJ_NUMANODE, numa);
+		unsigned numa_logical_id = obj->logical_index;
+		unsigned numa_physical_id = obj->os_index;
+#else
+		unsigned numa_logical_id = 0;
+		unsigned numa_physical_id = 0;
+#endif
+		int memnode = _starpu_memory_node_register(STARPU_CPU_RAM, numa_logical_id);
+		_starpu_memory_manager_set_global_memory_size(memnode, _starpu_cpu_get_global_mem_size(numa_logical_id, config));
 
-			int memnode = _starpu_memory_node_register(STARPU_CPU_RAM, numa);
-			STARPU_ASSERT_MSG(memnode < STARPU_MAXNUMANODES, "Wrong Memory Node : %d (only %d available) \n", memnode, STARPU_MAXNUMANODES);
+		numa_memory_nodes_to_hwloclogid[memnode] = numa_logical_id;
+		numa_memory_nodes_to_physicalid[memnode] = numa_physical_id;
+		nb_numa_nodes++;
 
-			numa_memory_nodes_to_hwloclogid[memnode] = numa_logical_id;
-			numa_memory_nodes_to_physicalid[memnode] = numa_physical_id;
-			nb_numa_nodes++;
+		if (numa == 0)
+			STARPU_ASSERT_MSG(memnode == STARPU_MAIN_RAM, "Wrong Memory Node : %d (expected %d) \n", memnode, STARPU_MAIN_RAM);
+		STARPU_ASSERT_MSG(memnode < STARPU_MAXNUMANODES, "Wrong Memory Node : %d (only %d available) \n", memnode, STARPU_MAXNUMANODES);
 
 #ifdef STARPU_SIMGRID
+		if (nnuma > 1)
+		{
 			snprintf(name, sizeof(name), "RAM%d", memnode);
 			host = _starpu_simgrid_get_host_by_name(name);
-			STARPU_ASSERT(host);
-			_starpu_simgrid_memory_node_set_host(memnode, host);
-#endif
 		}
 		else
-#endif /* defined(STARPU_HAVE_HWLOC) */
 		{
-
 			/* In this case, nnuma has only one node */
-			int memnode = _starpu_memory_node_register(STARPU_CPU_RAM, numa);
-			STARPU_ASSERT_MSG(memnode == STARPU_MAIN_RAM, "Wrong Memory Node : %d (expected %d) \n", memnode, STARPU_MAIN_RAM);
-
-			numa_memory_nodes_to_hwloclogid[memnode] = STARPU_NUMA_MAIN_RAM;
-			numa_memory_nodes_to_physicalid[memnode] = STARPU_NUMA_MAIN_RAM;
-			nb_numa_nodes++;
-#ifdef STARPU_SIMGRID
 			host = _starpu_simgrid_get_host_by_name("RAM");
-			STARPU_ASSERT(host);
-			_starpu_simgrid_memory_node_set_host(STARPU_MAIN_RAM, host);
-#endif
 		}
 
+		STARPU_ASSERT(host);
+		_starpu_simgrid_memory_node_set_host(memnode, host);
+#endif
 	}
 
 	STARPU_ASSERT_MSG(nb_numa_nodes > 0, "No NUMA node found... We need at least one memory node !\n");

src/drivers/cpu/driver_cpu.c

@@ -153,7 +153,7 @@ static int execute_job_on_cpu(struct _starpu_job *j, struct starpu_task *worker_
 	return 0;
 }
 
-static size_t _starpu_cpu_get_global_mem_size(int nodeid STARPU_ATTRIBUTE_UNUSED, struct _starpu_machine_config *config STARPU_ATTRIBUTE_UNUSED)
+size_t _starpu_cpu_get_global_mem_size(int nodeid STARPU_ATTRIBUTE_UNUSED, struct _starpu_machine_config *config STARPU_ATTRIBUTE_UNUSED)
 {
 	size_t global_mem;
 	starpu_ssize_t limit = -1;
@@ -161,8 +161,7 @@ static size_t _starpu_cpu_get_global_mem_size(int nodeid STARPU_ATTRIBUTE_UNUSED
 #if defined(STARPU_HAVE_HWLOC)
 	struct _starpu_machine_topology *topology = &config->topology;
 
-	int nnumas = starpu_memory_nodes_get_numa_count();
-	if (nnumas > 1)
+	if (starpu_get_env_number_default("STARPU_USE_NUMA", 0))
 	{
 		int depth_node = hwloc_get_type_depth(topology->hwtopology, HWLOC_OBJ_NUMANODE);
 
@@ -223,7 +222,6 @@ int _starpu_cpu_driver_init(struct _starpu_worker *cpu_worker)
 	int devid = cpu_worker->devid;
 
 	_starpu_driver_start(cpu_worker, _STARPU_FUT_CPU_KEY, 1);
-	_starpu_memory_manager_set_global_memory_size(cpu_worker->memory_node, _starpu_cpu_get_global_mem_size(cpu_worker->numa_memory_node, cpu_worker->config));
 	snprintf(cpu_worker->name, sizeof(cpu_worker->name), "CPU %d", devid);
 	snprintf(cpu_worker->short_name, sizeof(cpu_worker->short_name), "CPU %d", devid);
 	starpu_pthread_setname(cpu_worker->short_name);

src/drivers/cpu/driver_cpu.h

@@ -1,7 +1,7 @@
 /* StarPU --- Runtime system for heterogeneous multicore architectures.
  *
  * Copyright (C) 2012,2014                                Inria
- * Copyright (C) 2008-2011,2014                           Université de Bordeaux
+ * Copyright (C) 2008-2011,2014,2018                      Université de Bordeaux
  * Copyright (C) 2010,2012-2013,2015,2017                 CNRS
  *
  * StarPU is free software; you can redistribute it and/or modify
@@ -24,6 +24,7 @@
 #ifdef STARPU_USE_CPU
 
 extern struct _starpu_driver_ops _starpu_driver_cpu_ops;
+size_t _starpu_cpu_get_global_mem_size(int nodeid STARPU_ATTRIBUTE_UNUSED, struct _starpu_machine_config *config STARPU_ATTRIBUTE_UNUSED);
 void *_starpu_cpu_worker(void *);
 
 #endif /* !STARPU_USE_CPU */