fix potential for confusion between physical and logical NUMA node numbers

doc/doxygen/chapters/api/data_management.doxy

@@ -108,6 +108,10 @@ data to StarPU, the specified memory node indicates where the piece of
 data initially resides (we also call this memory node the home node of
 a piece of data).
 
+In the case of NUMA systems, functions starpu_numaphysid_get_memory_node()
+and starpu_memory_node_get_numaphysid() can be used to convert from NUMA node
+numbers as seen by the Operating System and NUMA node numbers as seen by StarPU.
+
 \fn void starpu_data_register(starpu_data_handle_t *handleptr, int home_node, void *data_interface, struct starpu_data_interface_ops *ops)
 \ingroup API_Data_Management
 Register a piece of data into the handle located at the

doc/doxygen/chapters/api/workers.doxy

@@ -238,6 +238,16 @@ Returns the type of the given node as defined by
 this function should be used in the allocation function to determine
 on which device the memory needs to be allocated.
 
+\fn unsigned starpu_numaphysid_get_memory_node(unsigned numaphysid)
+\ingroup API_Workers_Properties
+This function returns the identifier of the memory node associated to the NUMA
+node identified by \p numaphysid by the Operating System.
+
+\fn unsigned starpu_memory_node_get_numaphysid(unsigned node)
+\ingroup API_Workers_Properties
+This function returns the Operating System identifier of the memory node
+whose StarPU identifier is \p node.
+
 \fn char *starpu_worker_get_type_as_string(enum starpu_worker_archtype type)
 \ingroup API_Workers_Properties
 Returns the given worker type as a string.

include/starpu_data.h

@@ -123,6 +123,8 @@ enum starpu_node_kind
 
 unsigned starpu_worker_get_memory_node(unsigned workerid);
 unsigned starpu_memory_nodes_get_count(void);
+unsigned starpu_numaphysid_get_memory_node(unsigned numaphysid);
+unsigned starpu_memory_node_get_numaphysid(unsigned node);
 enum starpu_node_kind starpu_node_get_kind(unsigned node);
 
 void starpu_data_set_wt_mask(starpu_data_handle_t handle, uint32_t wt_mask);

src/core/topology.c

@@ -58,13 +58,15 @@ static int nobind;
 /* For checking whether two workers share the same PU, indexed by PU number */
 static int cpu_worker[STARPU_MAXCPUS];
 static unsigned nb_numa_nodes = 0;
-static unsigned numa_memory_nodes[STARPU_MAXNUMANODES]; /* indexed by system logical numaid */
-static unsigned numa_numaid[STARPU_MAXNODES]; /* indexed by starpu memory node number */
+static unsigned numa_memory_nodes[STARPU_MAXNUMANODES]; /* indexed by system logical numalogid */
+static unsigned numa_numalogid[STARPU_MAXNODES]; /* indexed by starpu memory node number */
+static unsigned numa_numaphysid[STARPU_MAXNODES]; /* indexed by starpu memory node number */
 #ifdef STARPU_USE_NUMA
 static int _starpu_worker_numa_node(unsigned workerid);
 #else
 #define _starpu_worker_numa_node(workerid) 0
 #endif
+static int _starpu_numa_logid_get_physid(unsigned logid);
 
 
 #if defined(STARPU_USE_CUDA) || defined(STARPU_USE_OPENCL) || defined(STARPU_USE_SCC) || defined(STARPU_SIMGRID)
@@ -1562,7 +1564,8 @@ _starpu_init_workers_binding (struct _starpu_machine_config *config, int no_mp_c
 	unsigned numa_init[STARPU_MAXNUMANODES];
 	numa_init[0] = 1 ;
 	numa_memory_nodes[0] = ram_memory_node ;
-	numa_numaid[ram_memory_node] = 0;
+	numa_numalogid[ram_memory_node] = 0;
+	numa_numaphysid[ram_memory_node] = 0;
 	for (n=1; n<STARPU_MAXNUMANODES; n++)
 	{
 		numa_init[n] = 0;
@@ -1612,25 +1615,26 @@ _starpu_init_workers_binding (struct _starpu_machine_config *config, int no_mp_c
 			{
 				/* "dedicate" a cpu core to that worker */
 				workerarg->bindid = _starpu_get_next_bindid(config, NULL, 0);
-				int numaid = workerarg->numa_memory_node = _starpu_worker_numa_node(worker);
-				if (!numa_init[numaid] && nb_numa_nodes == STARPU_MAXNUMANODES)
+				int numalogid = workerarg->numa_memory_node = _starpu_worker_numa_node(worker);
+				if (!numa_init[numalogid] && nb_numa_nodes == STARPU_MAXNUMANODES)
 				{
 					_STARPU_MSG("Warning: %u NUMA nodes available. Only %u enabled. Use configure option --enable-maxnumanodes=xxx to update the maximum value of supported NUMA nodes.\n", _starpu_topology_get_nnumanodes(config), STARPU_MAXNUMANODES);
-					numaid = STARPU_MAIN_RAM;
+					numalogid = STARPU_MAIN_RAM;
 				}
 
-				if (numa_init[numaid])
+				if (numa_init[numalogid])
 				{
-					memory_node = numa_memory_nodes[numaid];
+					memory_node = numa_memory_nodes[numalogid];
 				}
 				else
 				{
-					numa_init[numaid] = 1;
+					numa_init[numalogid] = 1;
 					nb_numa_nodes++;
-					memory_node = numa_memory_nodes[numaid] = _starpu_memory_node_register(STARPU_CPU_RAM, numaid);
-					numa_numaid[memory_node] = numaid;
+					memory_node = numa_memory_nodes[numalogid] = _starpu_memory_node_register(STARPU_CPU_RAM, numalogid);
+					numa_numalogid[memory_node] = numalogid;
+					numa_numaphysid[memory_node] = _starpu_numa_logid_get_physid(numalogid);
 #ifdef STARPU_SIMGRID
-					snprintf(name, sizeof(name), "RAM%d", numaid);
+					snprintf(name, sizeof(name), "RAM%d", numalogid);
 					host = _starpu_simgrid_get_host_by_name(name);
 					STARPU_ASSERT(host);
 					_starpu_simgrid_memory_node_set_host(memory_node, host);
@@ -1981,6 +1985,7 @@ starpu_topology_print (FILE *output)
 	unsigned ncombinedworkers = topology->ncombinedworkers;
 	unsigned nthreads_per_core = topology->nhwpus / topology->nhwcpus;
 
+	/* TODO: show NUMA nodes */
 	for (pu = 0; pu < topology->nhwpus; pu++)
 	{
 		if ((pu % nthreads_per_core) == 0)
@@ -2023,16 +2028,37 @@ int _starpu_get_nb_numa_nodes(void)
 	return nb_numa_nodes;
 }
 
-int _starpu_numaid_to_memnode(unsigned numaid)
+int _starpu_numalogid_to_memnode(unsigned numalogid)
 {
-	if (numaid < nb_numa_nodes)
-		return numa_memory_nodes[numaid];
+	if (numalogid < nb_numa_nodes)
+		return numa_memory_nodes[numalogid];
 	return -1;
 }
 
-int _starpu_memnode_to_numaid(unsigned memnode)
+int _starpu_memnode_to_numalogid(unsigned memnode)
 {
-	return numa_numaid[memnode];
+	return numa_numalogid[memnode];
+}
+
+unsigned starpu_memory_node_get_numaphysid(unsigned memnode)
+{
+	return numa_numaphysid[memnode];
+}
+
+unsigned starpu_numaphysid_get_memory_node(unsigned physid)
+{
+#ifdef STARPU_USE_NUMA
+	unsigned logid;
+	for (logid = 0; logid < nb_numa_nodes; logid++)
+	{
+		unsigned node = numa_memory_nodes[logid];
+		if (numa_numaphysid[node] == physid)
+			return node;
+	}
+	STARPU_ASSERT_MSG(0, "unknown physid passed to starpu_numaphysid_get_memory_node\n");
+#else
+	return 0;
+#endif
 }
 
 #ifdef STARPU_USE_NUMA
@@ -2060,3 +2086,15 @@ static int _starpu_worker_numa_node(unsigned workerid)
 	#endif /* STARPU_HAVE_HWLOC */
 }
 #endif /* STARPU_USE_NUMA */
+
+static int _starpu_numa_logid_get_physid(unsigned logid)
+{
+#ifdef STARPU_HAVE_HWLOC
+	struct _starpu_machine_config *config = (struct _starpu_machine_config *)_starpu_get_machine_config() ;
+	struct _starpu_machine_topology *topology = &config->topology ;
+	hwloc_obj_t obj = hwloc_get_obj_by_type(topology->hwtopology, HWLOC_OBJ_NODE, logid);
+	return obj->os_index;
+#else /* STARPU_HAVE_HWLOC */
+	return 0;
+#endif /* STARPU_HAVE_HWLOC */
+}

src/core/topology.h

@@ -66,8 +66,8 @@ void _starpu_bind_thread_on_cpus(struct _starpu_machine_config *config STARPU_AT
 
 void *_starpu_get_worker_from_driver(struct starpu_driver *d);
 
-int _starpu_numaid_to_memnode(unsigned numaid);
-int _starpu_memnode_to_numaid(unsigned memnode);
+int _starpu_numalogid_to_memnode(unsigned numalogid);
+int _starpu_memnode_to_numalogid(unsigned memnode);
 int _starpu_get_nb_numa_nodes(void);
 	
 #endif // __TOPOLOGY_H__

src/core/workers.c

@@ -1520,7 +1520,7 @@ void starpu_shutdown(void)
 	unsigned nb_numa_nodes = _starpu_get_nb_numa_nodes();
 	for (i=0; i<nb_numa_nodes; i++)
 	{
-		unsigned id = _starpu_numaid_to_memnode(i);
+		unsigned id = _starpu_numalogid_to_memnode(i);
 		_starpu_free_all_automatically_allocated_buffers(id);
 	}
 

src/datawizard/datawizard.c

@@ -75,7 +75,7 @@ int __starpu_datawizard_progress_ram(unsigned may_alloc, unsigned push_requests)
 	unsigned nb_numa_nodes = _starpu_get_nb_numa_nodes();
 	for (i=0; i<nb_numa_nodes; i++)
 	{
-		unsigned id = _starpu_numaid_to_memnode(i);
+		unsigned id = _starpu_numalogid_to_memnode(i);
 		res |= __starpu_datawizard_progress(id, may_alloc, push_requests);
 	}
 	return res;

src/datawizard/malloc.c

@@ -295,10 +295,10 @@ int _starpu_malloc_flags_on_node(unsigned dst_node, void **A, size_t dim, int fl
 #ifdef STARPU_HAVE_HWLOC
 	if (_starpu_get_nb_numa_nodes() > 1) {
 		hwloc_topology_t hwtopology = config->topology.hwtopology;
-		hwloc_obj_t numa_node_obj = hwloc_get_obj_by_type(hwtopology, HWLOC_OBJ_NODE, _starpu_memnode_to_numaid(dst_node));
+		hwloc_obj_t numa_node_obj = hwloc_get_obj_by_type(hwtopology, HWLOC_OBJ_NODE, _starpu_memnode_to_numalogid(dst_node));
 		hwloc_bitmap_t nodeset = numa_node_obj->nodeset;
 		*A = hwloc_alloc_membind_nodeset(hwtopology, dim, nodeset, HWLOC_MEMBIND_BIND | HWLOC_MEMBIND_NOCPUBIND, flags);
-		//fprintf(stderr, "Allocation %lu bytes on NUMA node %d [%p]\n", (unsigned long) dim, _starpu_memnode_to_numaid(dst_node), *A);
+		//fprintf(stderr, "Allocation %lu bytes on NUMA node %d [%p]\n", (unsigned long) dim, starpu_memnode_get_numaphysid(dst_node), *A);
 		if (!*A)
 			ret = -ENOMEM;
 	}

src/datawizard/memalloc.c

@@ -1616,7 +1616,7 @@ choose_target(starpu_data_handle_t handle, unsigned node)
 			unsigned nb_numa_nodes = _starpu_get_nb_numa_nodes();
 			for (i=0; i<nb_numa_nodes; i++)
 			{
-				unsigned id = _starpu_numaid_to_memnode(i);
+				unsigned id = _starpu_numalogid_to_memnode(i);
 				if (handle->per_node[id].allocated || 
 				    _starpu_memory_manager_test_allocate_size(id, size_handle) == 1)
 				{
@@ -1649,7 +1649,7 @@ choose_target(starpu_data_handle_t handle, unsigned node)
 			unsigned nb_numa_nodes = _starpu_get_nb_numa_nodes();
 			for (i=0; i<nb_numa_nodes; i++)
 			{
-				unsigned id = _starpu_numaid_to_memnode(i);
+				unsigned id = _starpu_numalogid_to_memnode(i);
 				if (handle->per_node[id].allocated || 
 				    _starpu_memory_manager_test_allocate_size(id, size_handle) == 1)
 				{