Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/benh/powerpc

[~andy/linux] / arch / powerpc / mm / numa.c

diff --git a/arch/powerpc/mm/numa.c b/arch/powerpc/mm/numa.c
index 5a944f25e94ff4618661cd6efaea6ca3140157c8..86a63de072c617c35dc65b138b456d0ba900001a 100644 (file)
--- a/arch/powerpc/mm/numa.c
+++ b/arch/powerpc/mm/numa.c
@@ -31,6 +31,8 @@
 #include <asm/sparsemem.h>
 #include <asm/prom.h>
 #include <asm/smp.h>
+#include <asm/cputhreads.h>
+#include <asm/topology.h>
 #include <asm/firmware.h>
 #include <asm/paca.h>
 #include <asm/hvcall.h>
@@ -152,9 +154,22 @@ static void __init get_node_active_region(unsigned long pfn,
        }
 }
 
-static void map_cpu_to_node(int cpu, int node)
+static void reset_numa_cpu_lookup_table(void)
+{
+       unsigned int cpu;
+
+       for_each_possible_cpu(cpu)
+               numa_cpu_lookup_table[cpu] = -1;
+}
+
+static void update_numa_cpu_lookup_table(unsigned int cpu, int node)
 {
        numa_cpu_lookup_table[cpu] = node;
+}
+
+static void map_cpu_to_node(int cpu, int node)
+{
+       update_numa_cpu_lookup_table(cpu, node);
 
        dbg("adding cpu %d to node %d\n", cpu, node);
 
@@ -522,11 +537,24 @@ static int of_drconf_to_nid_single(struct of_drconf_cell *drmem,
  */
 static int numa_setup_cpu(unsigned long lcpu)
 {
-       int nid = 0;
-       struct device_node *cpu = of_get_cpu_node(lcpu, NULL);
+       int nid;
+       struct device_node *cpu;
+
+       /*
+        * If a valid cpu-to-node mapping is already available, use it
+        * directly instead of querying the firmware, since it represents
+        * the most recent mapping notified to us by the platform (eg: VPHN).
+        */
+       if ((nid = numa_cpu_lookup_table[lcpu]) >= 0) {
+               map_cpu_to_node(lcpu, nid);
+               return nid;
+       }
+
+       cpu = of_get_cpu_node(lcpu, NULL);
 
        if (!cpu) {
                WARN_ON(1);
+               nid = 0;
                goto out;
        }
 
@@ -542,16 +570,38 @@ out:
        return nid;
 }
 
+static void verify_cpu_node_mapping(int cpu, int node)
+{
+       int base, sibling, i;
+
+       /* Verify that all the threads in the core belong to the same node */
+       base = cpu_first_thread_sibling(cpu);
+
+       for (i = 0; i < threads_per_core; i++) {
+               sibling = base + i;
+
+               if (sibling == cpu || cpu_is_offline(sibling))
+                       continue;
+
+               if (cpu_to_node(sibling) != node) {
+                       WARN(1, "CPU thread siblings %d and %d don't belong"
+                               " to the same node!\n", cpu, sibling);
+                       break;
+               }
+       }
+}
+
 static int cpu_numa_callback(struct notifier_block *nfb, unsigned long action,
                             void *hcpu)
 {
        unsigned long lcpu = (unsigned long)hcpu;
-       int ret = NOTIFY_DONE;
+       int ret = NOTIFY_DONE, nid;
 
        switch (action) {
        case CPU_UP_PREPARE:
        case CPU_UP_PREPARE_FROZEN:
-               numa_setup_cpu(lcpu);
+               nid = numa_setup_cpu(lcpu);
+               verify_cpu_node_mapping((int)lcpu, nid);
                ret = NOTIFY_OK;
                break;
 #ifdef CONFIG_HOTPLUG_CPU
@@ -1069,6 +1119,7 @@ void __init do_init_bootmem(void)
         */
        setup_node_to_cpumask_map();
 
+       reset_numa_cpu_lookup_table();
        register_cpu_notifier(&ppc64_numa_nb);
        cpu_numa_callback(&ppc64_numa_nb, CPU_UP_PREPARE,
                          (void *)(unsigned long)boot_cpuid);
@@ -1447,6 +1498,33 @@ static int update_cpu_topology(void *data)
        return 0;
 }
 
+static int update_lookup_table(void *data)
+{
+       struct topology_update_data *update;
+
+       if (!data)
+               return -EINVAL;
+
+       /*
+        * Upon topology update, the numa-cpu lookup table needs to be updated
+        * for all threads in the core, including offline CPUs, to ensure that
+        * future hotplug operations respect the cpu-to-node associativity
+        * properly.
+        */
+       for (update = data; update; update = update->next) {
+               int nid, base, j;
+
+               nid = update->new_nid;
+               base = cpu_first_thread_sibling(update->cpu);
+
+               for (j = 0; j < threads_per_core; j++) {
+                       update_numa_cpu_lookup_table(base + j, nid);
+               }
+       }
+
+       return 0;
+}
+
 /*
  * Update the node maps and sysfs entries for each cpu whose home node
  * has changed. Returns 1 when the topology has changed, and 0 otherwise.
@@ -1515,6 +1593,14 @@ int arch_update_cpu_topology(void)
 
        stop_machine(update_cpu_topology, &updates[0], &updated_cpus);
 
+       /*
+        * Update the numa-cpu lookup table with the new mappings, even for
+        * offline CPUs. It is best to perform this update from the stop-
+        * machine context.
+        */
+       stop_machine(update_lookup_table, &updates[0],
+                                       cpumask_of(raw_smp_processor_id()));
+
        for (ud = &updates[0]; ud; ud = ud->next) {
                unregister_cpu_under_node(ud->cpu, ud->old_nid);
                register_cpu_under_node(ud->cpu, ud->new_nid);