2 * ACPI 3.0 based NUMA setup
3 * Copyright 2004 Andi Kleen, SuSE Labs.
5 * Reads the ACPI SRAT table to figure out what memory belongs to which CPUs.
7 * Called from acpi_numa_init while reading the SRAT and SLIT tables.
8 * Assumes all memory regions belonging to a single proximity domain
9 * are in one chunk. Holes between them will be included in the node.
12 #include <linux/kernel.h>
13 #include <linux/acpi.h>
14 #include <linux/mmzone.h>
15 #include <linux/bitmap.h>
16 #include <linux/module.h>
17 #include <linux/topology.h>
18 #include <linux/bootmem.h>
19 #include <linux/memblock.h>
21 #include <asm/proto.h>
25 #include <asm/uv/uv.h>
27 int acpi_numa __initdata;
29 static struct acpi_table_slit *acpi_slit;
31 static nodemask_t nodes_parsed __initdata;
32 static nodemask_t cpu_nodes_parsed __initdata;
33 static struct bootnode nodes[MAX_NUMNODES] __initdata;
34 static struct bootnode nodes_add[MAX_NUMNODES];
36 static int num_node_memblks __initdata;
37 static struct bootnode node_memblk_range[NR_NODE_MEMBLKS] __initdata;
38 static int memblk_nodeid[NR_NODE_MEMBLKS] __initdata;
40 static __init int setup_node(int pxm)
42 return acpi_map_pxm_to_node(pxm);
45 static __init int conflicting_memblks(unsigned long start, unsigned long end)
48 for (i = 0; i < num_node_memblks; i++) {
49 struct bootnode *nd = &node_memblk_range[i];
50 if (nd->start == nd->end)
52 if (nd->end > start && nd->start < end)
53 return memblk_nodeid[i];
54 if (nd->end == end && nd->start == start)
55 return memblk_nodeid[i];
60 static __init void cutoff_node(int i, unsigned long start, unsigned long end)
62 struct bootnode *nd = &nodes[i];
64 if (nd->start < start) {
66 if (nd->end < nd->start)
71 if (nd->start > nd->end)
76 static __init void bad_srat(void)
79 printk(KERN_ERR "SRAT: SRAT not used.\n");
81 for (i = 0; i < MAX_LOCAL_APIC; i++)
82 apicid_to_node[i] = NUMA_NO_NODE;
83 for (i = 0; i < MAX_NUMNODES; i++) {
84 nodes[i].start = nodes[i].end = 0;
85 nodes_add[i].start = nodes_add[i].end = 0;
87 remove_all_active_ranges();
90 static __init inline int srat_disabled(void)
92 return numa_off || acpi_numa < 0;
95 /* Callback for SLIT parsing */
96 void __init acpi_numa_slit_init(struct acpi_table_slit *slit)
101 length = slit->header.length;
102 phys = memblock_find_in_range(0, max_pfn_mapped<<PAGE_SHIFT, length,
105 if (phys == MEMBLOCK_ERROR)
106 panic(" Can not save slit!\n");
108 acpi_slit = __va(phys);
109 memcpy(acpi_slit, slit, length);
110 memblock_x86_reserve_range(phys, phys + length, "ACPI SLIT");
113 /* Callback for Proximity Domain -> x2APIC mapping */
115 acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa)
122 if (pa->header.length < sizeof(struct acpi_srat_x2apic_cpu_affinity)) {
126 if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
128 pxm = pa->proximity_domain;
129 node = setup_node(pxm);
131 printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
136 apic_id = pa->apic_id;
137 if (apic_id >= MAX_LOCAL_APIC) {
138 printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%04x -> Node %u skipped apicid that is too big\n", pxm, apic_id, node);
141 apicid_to_node[apic_id] = node;
142 node_set(node, cpu_nodes_parsed);
144 printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%04x -> Node %u\n",
148 /* Callback for Proximity Domain -> LAPIC mapping */
150 acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa)
157 if (pa->header.length != sizeof(struct acpi_srat_cpu_affinity)) {
161 if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
163 pxm = pa->proximity_domain_lo;
164 node = setup_node(pxm);
166 printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
171 if (get_uv_system_type() >= UV_X2APIC)
172 apic_id = (pa->apic_id << 8) | pa->local_sapic_eid;
174 apic_id = pa->apic_id;
176 if (apic_id >= MAX_LOCAL_APIC) {
177 printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%02x -> Node %u skipped apicid that is too big\n", pxm, apic_id, node);
181 apicid_to_node[apic_id] = node;
182 node_set(node, cpu_nodes_parsed);
184 printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%02x -> Node %u\n",
188 #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
189 static inline int save_add_info(void) {return 1;}
191 static inline int save_add_info(void) {return 0;}
195 * This code supports one contiguous hot add area per node
198 update_nodes_add(int node, unsigned long start, unsigned long end)
200 unsigned long s_pfn = start >> PAGE_SHIFT;
201 unsigned long e_pfn = end >> PAGE_SHIFT;
203 struct bootnode *nd = &nodes_add[node];
205 /* I had some trouble with strange memory hotadd regions breaking
206 the boot. Be very strict here and reject anything unexpected.
207 If you want working memory hotadd write correct SRATs.
209 The node size check is a basic sanity check to guard against
211 if ((signed long)(end - start) < NODE_MIN_SIZE) {
212 printk(KERN_ERR "SRAT: Hotplug area too small\n");
216 /* This check might be a bit too strict, but I'm keeping it for now. */
217 if (absent_pages_in_range(s_pfn, e_pfn) != e_pfn - s_pfn) {
219 "SRAT: Hotplug area %lu -> %lu has existing memory\n",
226 if (nd->start == nd->end) {
231 if (nd->start == end) {
235 if (nd->end == start) {
240 printk(KERN_ERR "SRAT: Hotplug zone not continuous. Partly ignored\n");
244 node_set(node, cpu_nodes_parsed);
245 printk(KERN_INFO "SRAT: hot plug zone found %Lx - %Lx\n",
250 /* Callback for parsing of the Proximity Domain <-> Memory Area mappings */
252 acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma)
254 struct bootnode *nd, oldnode;
255 unsigned long start, end;
261 if (ma->header.length != sizeof(struct acpi_srat_mem_affinity)) {
265 if ((ma->flags & ACPI_SRAT_MEM_ENABLED) == 0)
268 if ((ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) && !save_add_info())
270 start = ma->base_address;
271 end = start + ma->length;
272 pxm = ma->proximity_domain;
273 node = setup_node(pxm);
275 printk(KERN_ERR "SRAT: Too many proximity domains.\n");
279 i = conflicting_memblks(start, end);
282 "SRAT: Warning: PXM %d (%lx-%lx) overlaps with itself (%Lx-%Lx)\n",
283 pxm, start, end, nodes[i].start, nodes[i].end);
286 "SRAT: PXM %d (%lx-%lx) overlaps with PXM %d (%Lx-%Lx)\n",
287 pxm, start, end, node_to_pxm(i),
288 nodes[i].start, nodes[i].end);
294 if (!node_test_and_set(node, nodes_parsed)) {
298 if (start < nd->start)
304 printk(KERN_INFO "SRAT: Node %u PXM %u %lx-%lx\n", node, pxm,
307 if (ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) {
308 update_nodes_add(node, start, end);
309 /* restore nodes[node] */
311 if ((nd->start | nd->end) == 0)
312 node_clear(node, nodes_parsed);
315 node_memblk_range[num_node_memblks].start = start;
316 node_memblk_range[num_node_memblks].end = end;
317 memblk_nodeid[num_node_memblks] = node;
321 /* Sanity check to catch more bad SRATs (they are amazingly common).
322 Make sure the PXMs cover all memory. */
323 static int __init nodes_cover_memory(const struct bootnode *nodes)
326 unsigned long pxmram, e820ram;
329 for_each_node_mask(i, nodes_parsed) {
330 unsigned long s = nodes[i].start >> PAGE_SHIFT;
331 unsigned long e = nodes[i].end >> PAGE_SHIFT;
333 pxmram -= __absent_pages_in_range(i, s, e);
334 if ((long)pxmram < 0)
338 e820ram = max_pfn - (memblock_x86_hole_size(0, max_pfn<<PAGE_SHIFT)>>PAGE_SHIFT);
339 /* We seem to lose 3 pages somewhere. Allow 1M of slack. */
340 if ((long)(e820ram - pxmram) >= (1<<(20 - PAGE_SHIFT))) {
342 "SRAT: PXMs only cover %luMB of your %luMB e820 RAM. Not used.\n",
343 (pxmram << PAGE_SHIFT) >> 20,
344 (e820ram << PAGE_SHIFT) >> 20);
350 void __init acpi_numa_arch_fixup(void) {}
352 #ifdef CONFIG_NUMA_EMU
353 void __init acpi_get_nodes(struct bootnode *physnodes, unsigned long start,
358 for_each_node_mask(i, nodes_parsed) {
359 cutoff_node(i, start, end);
360 physnodes[i].start = nodes[i].start;
361 physnodes[i].end = nodes[i].end;
364 #endif /* CONFIG_NUMA_EMU */
366 /* Use the information discovered above to actually set up the nodes. */
367 int __init acpi_scan_nodes(unsigned long start, unsigned long end)
374 /* First clean up the node list */
375 for (i = 0; i < MAX_NUMNODES; i++)
376 cutoff_node(i, start, end);
379 * Join together blocks on the same node, holes between
380 * which don't overlap with memory on other nodes.
382 for (i = 0; i < num_node_memblks; ++i) {
385 for (j = i + 1; j < num_node_memblks; ++j) {
386 unsigned long start, end;
388 if (memblk_nodeid[i] != memblk_nodeid[j])
390 start = min(node_memblk_range[i].end,
391 node_memblk_range[j].end);
392 end = max(node_memblk_range[i].start,
393 node_memblk_range[j].start);
394 for (k = 0; k < num_node_memblks; ++k) {
395 if (memblk_nodeid[i] == memblk_nodeid[k])
397 if (start < node_memblk_range[k].end &&
398 end > node_memblk_range[k].start)
401 if (k < num_node_memblks)
403 start = min(node_memblk_range[i].start,
404 node_memblk_range[j].start);
405 end = max(node_memblk_range[i].end,
406 node_memblk_range[j].end);
407 printk(KERN_INFO "SRAT: Node %d "
408 "[%Lx,%Lx) + [%Lx,%Lx) -> [%lx,%lx)\n",
410 node_memblk_range[i].start,
411 node_memblk_range[i].end,
412 node_memblk_range[j].start,
413 node_memblk_range[j].end,
415 node_memblk_range[i].start = start;
416 node_memblk_range[i].end = end;
417 k = --num_node_memblks - j;
418 memmove(memblk_nodeid + j, memblk_nodeid + j+1,
419 k * sizeof(*memblk_nodeid));
420 memmove(node_memblk_range + j, node_memblk_range + j+1,
421 k * sizeof(*node_memblk_range));
426 memnode_shift = compute_hash_shift(node_memblk_range, num_node_memblks,
428 if (memnode_shift < 0) {
430 "SRAT: No NUMA node hash function found. Contact maintainer\n");
435 for (i = 0; i < num_node_memblks; i++)
436 memblock_x86_register_active_regions(memblk_nodeid[i],
437 node_memblk_range[i].start >> PAGE_SHIFT,
438 node_memblk_range[i].end >> PAGE_SHIFT);
440 /* for out of order entries in SRAT */
442 if (!nodes_cover_memory(nodes)) {
447 /* Account for nodes with cpus and no memory */
448 nodes_or(node_possible_map, nodes_parsed, cpu_nodes_parsed);
450 /* Finally register nodes */
451 for_each_node_mask(i, node_possible_map)
452 setup_node_bootmem(i, nodes[i].start, nodes[i].end);
453 /* Try again in case setup_node_bootmem missed one due
454 to missing bootmem */
455 for_each_node_mask(i, node_possible_map)
457 setup_node_bootmem(i, nodes[i].start, nodes[i].end);
459 for (i = 0; i < nr_cpu_ids; i++) {
460 int node = early_cpu_to_node(i);
462 if (node == NUMA_NO_NODE)
464 if (!node_online(node))
471 #ifdef CONFIG_NUMA_EMU
472 static int fake_node_to_pxm_map[MAX_NUMNODES] __initdata = {
473 [0 ... MAX_NUMNODES-1] = PXM_INVAL
475 static s16 fake_apicid_to_node[MAX_LOCAL_APIC] __initdata = {
476 [0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE
478 static int __init find_node_by_addr(unsigned long addr)
480 int ret = NUMA_NO_NODE;
483 for_each_node_mask(i, nodes_parsed) {
485 * Find the real node that this emulated node appears on. For
486 * the sake of simplicity, we only use a real node's starting
487 * address to determine which emulated node it appears on.
489 if (addr >= nodes[i].start && addr < nodes[i].end) {
498 * In NUMA emulation, we need to setup proximity domain (_PXM) to node ID
499 * mappings that respect the real ACPI topology but reflect our emulated
500 * environment. For each emulated node, we find which real node it appears on
501 * and create PXM to NID mappings for those fake nodes which mirror that
502 * locality. SLIT will now represent the correct distances between emulated
503 * nodes as a result of the real topology.
505 void __init acpi_fake_nodes(const struct bootnode *fake_nodes, int num_nodes)
509 for (i = 0; i < num_nodes; i++) {
512 nid = find_node_by_addr(fake_nodes[i].start);
513 if (nid == NUMA_NO_NODE)
515 pxm = node_to_pxm(nid);
516 if (pxm == PXM_INVAL)
518 fake_node_to_pxm_map[i] = pxm;
520 * For each apicid_to_node mapping that exists for this real
521 * node, it must now point to the fake node ID.
523 for (j = 0; j < MAX_LOCAL_APIC; j++)
524 if (apicid_to_node[j] == nid &&
525 fake_apicid_to_node[j] == NUMA_NO_NODE)
526 fake_apicid_to_node[j] = i;
530 * If there are apicid-to-node mappings for physical nodes that do not
531 * have a corresponding emulated node, it should default to a guaranteed
534 for (i = 0; i < MAX_LOCAL_APIC; i++)
535 if (apicid_to_node[i] != NUMA_NO_NODE &&
536 fake_apicid_to_node[i] == NUMA_NO_NODE)
537 fake_apicid_to_node[i] = 0;
539 for (i = 0; i < num_nodes; i++)
540 __acpi_map_pxm_to_node(fake_node_to_pxm_map[i], i);
541 memcpy(apicid_to_node, fake_apicid_to_node, sizeof(apicid_to_node));
543 nodes_clear(nodes_parsed);
544 for (i = 0; i < num_nodes; i++)
545 if (fake_nodes[i].start != fake_nodes[i].end)
546 node_set(i, nodes_parsed);
549 static int null_slit_node_compare(int a, int b)
551 return node_to_pxm(a) == node_to_pxm(b);
554 static int null_slit_node_compare(int a, int b)
558 #endif /* CONFIG_NUMA_EMU */
560 int __node_distance(int a, int b)
565 return null_slit_node_compare(a, b) ? LOCAL_DISTANCE :
567 index = acpi_slit->locality_count * node_to_pxm(a);
568 return acpi_slit->entry[index + node_to_pxm(b)];
571 EXPORT_SYMBOL(__node_distance);
573 #if defined(CONFIG_MEMORY_HOTPLUG_SPARSE) || defined(CONFIG_ACPI_HOTPLUG_MEMORY)
574 int memory_add_physaddr_to_nid(u64 start)
579 if (nodes_add[i].start <= start && nodes_add[i].end > start)
584 EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid);