libnuma: adding support for locality information and node distances
authorReto Achermann <reto.achermann@inf.ethz.ch>
Thu, 4 Jun 2015 12:18:34 +0000 (14:18 +0200)
committerReto Achermann <reto.achermann@inf.ethz.ch>
Thu, 4 Jun 2015 12:18:34 +0000 (14:18 +0200)
Signed-off-by: Reto Achermann <reto.achermann@inf.ethz.ch>

lib/numa/numa.c
lib/numa/numa_debug.h
lib/numa/numa_internal.h
lib/numa/utilities.c

index 460082d..5050113 100644 (file)
@@ -422,8 +422,11 @@ uint32_t numa_distance(nodeid_t from, nodeid_t to)
 {
     numa_check_init();
 
-    assert(!"NYI");
-    return 0;
+    if (from >= numa_topology.num_nodes || to >= numa_topology.num_nodes) {
+        return (uint32_t)NUMA_NODE_INVALID;
+    }
+
+    return numa_topology.distances[from * numa_topology.num_nodes + to];
 }
 
 
index 13dfc0f..490ebad 100644 (file)
 
 #define NUMA_DEBUG_ALLOC(x...) NUMA_DEBUG_PRINT("[numa alloc] " x);
 
-#define NUMA_ERROR(fmt, ...) debug_printf("[numa error] " fmt " in %s():", __VA_ARGS__, __FUNCTION__);
-#define NUMA_WARNING(fmt, ...) debug_printf("[numa  warn] " fmt " in %s():", __VA_ARGS__, __FUNCTION__);
+#define NUMA_ERROR(fmt, ...) \
+                debug_printf("[numa error] " fmt " in %s():", \
+                             __VA_ARGS__, __FUNCTION__);
+#define NUMA_WARNING(fmt, ...) \
+                debug_printf("[numa  warn] " fmt " in %s():", \
+                             __VA_ARGS__, __FUNCTION__);
 
 #endif /* NUMA_DEBUG_H_ */
index c77967c..e751464 100644 (file)
@@ -49,6 +49,7 @@ struct numa_topology {
     numa_policy_t strict;      ///< numa policy
     numa_policy_t bind;        ///< memory bind policy
     size_t pagesize;           ///< numa page size
+    uint32_t *distances;       ///< numa distances
     struct numa_node *nodes;   ///< nodes in the system
     struct numa_core **cores;  ///< cores in the system (sorted by core id)
 };
index c21f8f7..78be2c1 100644 (file)
@@ -61,6 +61,22 @@ void numa_dump_topology(struct numa_topology *topology)
                PRIuNODEID "]\n", coreid, core->apicid, core->node->id);
     }
 
+    printf("---------------------------------------\n");
+    printf("Locality map:\n");
+    printf("     ");
+    for (coreid_t node_from = 0; node_from < topology->num_nodes; ++node_from) {
+        printf("%02" PRIuNODEID " ", node_from);
+    }
+    printf("\n");
+    for (coreid_t node_from = 0; node_from < topology->num_nodes; ++node_from) {
+        printf("  %02" PRIuNODEID  " ", node_from);
+        for (coreid_t node_to = 0; node_to < topology->num_nodes; ++node_to) {
+            printf("%02" PRIu32 " ",
+                   topology->distances[node_from * topology->num_nodes + node_to]);
+        }
+        printf("\n");
+    }
+
     printf("\n<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n");
 }
 
@@ -88,9 +104,9 @@ errval_t numa_get_topology_from_skb(struct numa_topology *topology)
         return err_push(err, NUMA_ERR_SKB);
     }
 
-    err = skb_execute_query("get_system_topology(Nnodes,Ncores,Lnodes,Lcores),"
+    err = skb_execute_query("get_system_topology(Nnodes,Ncores,Lnodes,Lcores,Llocalities),"
                             "writeln(num(nodes(Nnodes),cores(Ncores))),"
-                            "writeln(Lnodes),writeln(Lcores).");
+                            "writeln(Lnodes),writeln(Lcores), writeln(Llocalities).");
     if (err_is_fail(err)) {
         DEBUG_ERR(err, "skb query failed");
         return err_push(err, NUMA_ERR_SKB);
@@ -125,16 +141,17 @@ errval_t numa_get_topology_from_skb(struct numa_topology *topology)
     topology->strict = NUMA_POLICY_DEFAULT;
 
     topology->nodes = malloc(node * sizeof(struct numa_node)
+                                + node * node * sizeof(uint32_t)
                                 + core * sizeof(struct numa_core)
                                 + core * sizeof(void *));
     if (topology->nodes == NULL) {
         return LIB_ERR_MALLOC_FAIL;
     }
 
-
     struct numa_core *cores_array = (struct numa_core *)(topology->nodes + node);
 
     topology->cores = (struct numa_core **) (cores_array + core);
+    topology->distances = (uint32_t*)(topology->cores + core);
 
     /* skip over the initial node and core information */
     char *output = strchr(skb_get_output(), '\n') + 1;
@@ -234,6 +251,35 @@ errval_t numa_get_topology_from_skb(struct numa_topology *topology)
         goto error_out;
     }
 
+    output = strchr(output, '\n') + 1;
+    skb_read_list_init_offset(&parser, output, 0);
+    parsed = 0;
+
+    uint32_t from, to;
+    uint32_t distance;
+    NUMA_DEBUG_INIT("parsing locality information...\n");
+    while (skb_read_list(&parser, "node_distance(%" PRIuNODEID ", %" PRIuNODEID ", %" PRIu32 ")",
+                          &from, &to, &distance)) {\
+        NUMA_DEBUG_INIT("  > [%" PRIuNODEID "] -> [%" PRIuNODEID "] = %" PRIu32"\n",
+                        from, to, distance);
+        topology->distances[from * topology->num_nodes + to] = distance;
+        parsed++;
+    }
+
+    if (parsed == 0) {
+        NUMA_DEBUG_INIT("locality list not existent setting all to 10...\n");
+        for (nodeid_t i = 0; i < topology->num_nodes; ++i) {
+            for (nodeid_t j = 0; j < topology->num_nodes; ++j) {
+                topology->distances[from * topology->num_nodes + to] = 10;
+            }
+        }
+    } else if (parsed != (uint32_t)topology->num_nodes * topology->num_nodes) {
+        NUMA_DEBUG_INIT("locality list incomplete: %" PRIu32" / %" PRIu32 "\n",
+                        parsed, (uint32_t)topology->num_nodes * topology->num_nodes);
+        err = NUMA_ERR_SKB_DATA;
+        goto error_out;
+    }
+
     return SYS_ERR_OK;
 
     error_out: