2 * Simple NUMA memory policy for the Linux kernel.
4 * Copyright 2003,2004 Andi Kleen, SuSE Labs.
5 * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc.
6 * Subject to the GNU Public License, version 2.
8 * NUMA policy allows the user to give hints in which node(s) memory should
11 * Support four policies per VMA and per process:
13 * The VMA policy has priority over the process policy for a page fault.
15 * interleave Allocate memory interleaved over a set of nodes,
16 * with normal fallback if it fails.
17 * For VMA based allocations this interleaves based on the
18 * offset into the backing object or offset into the mapping
19 * for anonymous memory. For process policy an process counter
22 * bind Only allocate memory on a specific set of nodes,
24 * FIXME: memory is allocated starting with the first node
25 * to the last. It would be better if bind would truly restrict
26 * the allocation to memory nodes instead
28 * preferred Try a specific node first before normal fallback.
29 * As a special case node -1 here means do the allocation
30 * on the local CPU. This is normally identical to default,
31 * but useful to set in a VMA when you have a non default
34 * default Allocate on the local node first, or when on a VMA
35 * use the process policy. This is what Linux always did
36 * in a NUMA aware kernel and still does by, ahem, default.
38 * The process policy is applied for most non interrupt memory allocations
39 * in that process' context. Interrupts ignore the policies and always
40 * try to allocate on the local CPU. The VMA policy is only applied for memory
41 * allocations for a VMA in the VM.
43 * Currently there are a few corner cases in swapping where the policy
44 * is not applied, but the majority should be handled. When process policy
45 * is used it is not remembered over swap outs/swap ins.
47 * Only the highest zone in the zone hierarchy gets policied. Allocations
48 * requesting a lower zone just use default policy. This implies that
49 * on systems with highmem kernel lowmem allocation don't get policied.
50 * Same with GFP_DMA allocations.
52 * For shmfs/tmpfs/hugetlbfs shared memory the policy is shared between
53 * all users and remembered even when nobody has memory mapped.
57 fix mmap readahead to honour policy and enable policy for any page cache
59 statistics for bigpages
60 global policy for page cache? currently it uses process policy. Requires
62 handle mremap for shared memory (currently ignored for the policy)
64 make bind policy root only? It can trigger oom much faster and the
65 kernel is not always grateful with that.
66 could replace all the switch()es with a mempolicy_ops structure.
69 #include <linux/mempolicy.h>
71 #include <linux/highmem.h>
72 #include <linux/hugetlb.h>
73 #include <linux/kernel.h>
74 #include <linux/sched.h>
76 #include <linux/nodemask.h>
77 #include <linux/cpuset.h>
78 #include <linux/gfp.h>
79 #include <linux/slab.h>
80 #include <linux/string.h>
81 #include <linux/module.h>
82 #include <linux/interrupt.h>
83 #include <linux/init.h>
84 #include <linux/compat.h>
85 #include <linux/mempolicy.h>
86 #include <linux/swap.h>
87 #include <linux/seq_file.h>
88 #include <linux/proc_fs.h>
90 #include <asm/tlbflush.h>
91 #include <asm/uaccess.h>
94 #define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0) /* Skip checks for continuous vmas */
95 #define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1) /* Invert check for nodemask */
96 #define MPOL_MF_STATS (MPOL_MF_INTERNAL << 2) /* Gather statistics */
98 static kmem_cache_t *policy_cache;
99 static kmem_cache_t *sn_cache;
101 #define PDprintk(fmt...)
103 /* Highest zone. An specific allocation for a zone below that is not
105 int policy_zone = ZONE_DMA;
107 struct mempolicy default_policy = {
108 .refcnt = ATOMIC_INIT(1), /* never free it */
109 .policy = MPOL_DEFAULT,
112 /* Do sanity checking on a policy */
113 static int mpol_check_policy(int mode, nodemask_t *nodes)
115 int empty = nodes_empty(*nodes);
123 case MPOL_INTERLEAVE:
124 /* Preferred will only use the first bit, but allow
130 return nodes_subset(*nodes, node_online_map) ? 0 : -EINVAL;
132 /* Generate a custom zonelist for the BIND policy. */
133 static struct zonelist *bind_zonelist(nodemask_t *nodes)
138 max = 1 + MAX_NR_ZONES * nodes_weight(*nodes);
139 zl = kmalloc(sizeof(void *) * max, GFP_KERNEL);
143 for_each_node_mask(nd, *nodes)
144 zl->zones[num++] = &NODE_DATA(nd)->node_zones[policy_zone];
145 zl->zones[num] = NULL;
149 /* Create a new policy */
150 static struct mempolicy *mpol_new(int mode, nodemask_t *nodes)
152 struct mempolicy *policy;
154 PDprintk("setting mode %d nodes[0] %lx\n", mode, nodes_addr(*nodes)[0]);
155 if (mode == MPOL_DEFAULT)
157 policy = kmem_cache_alloc(policy_cache, GFP_KERNEL);
159 return ERR_PTR(-ENOMEM);
160 atomic_set(&policy->refcnt, 1);
162 case MPOL_INTERLEAVE:
163 policy->v.nodes = *nodes;
164 if (nodes_weight(*nodes) == 0) {
165 kmem_cache_free(policy_cache, policy);
166 return ERR_PTR(-EINVAL);
170 policy->v.preferred_node = first_node(*nodes);
171 if (policy->v.preferred_node >= MAX_NUMNODES)
172 policy->v.preferred_node = -1;
175 policy->v.zonelist = bind_zonelist(nodes);
176 if (policy->v.zonelist == NULL) {
177 kmem_cache_free(policy_cache, policy);
178 return ERR_PTR(-ENOMEM);
182 policy->policy = mode;
186 static void gather_stats(struct page *, void *);
187 static void migrate_page_add(struct vm_area_struct *vma,
188 struct page *page, struct list_head *pagelist, unsigned long flags);
190 /* Scan through pages checking if pages follow certain conditions. */
191 static int check_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
192 unsigned long addr, unsigned long end,
193 const nodemask_t *nodes, unsigned long flags,
200 orig_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
205 if (!pte_present(*pte))
207 page = vm_normal_page(vma, addr, *pte);
210 nid = page_to_nid(page);
211 if (node_isset(nid, *nodes) == !!(flags & MPOL_MF_INVERT))
214 if (flags & MPOL_MF_STATS)
215 gather_stats(page, private);
216 else if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
218 migrate_page_add(vma, page, private, flags);
223 } while (pte++, addr += PAGE_SIZE, addr != end);
224 pte_unmap_unlock(orig_pte, ptl);
228 static inline int check_pmd_range(struct vm_area_struct *vma, pud_t *pud,
229 unsigned long addr, unsigned long end,
230 const nodemask_t *nodes, unsigned long flags,
236 pmd = pmd_offset(pud, addr);
238 next = pmd_addr_end(addr, end);
239 if (pmd_none_or_clear_bad(pmd))
241 if (check_pte_range(vma, pmd, addr, next, nodes,
244 } while (pmd++, addr = next, addr != end);
248 static inline int check_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
249 unsigned long addr, unsigned long end,
250 const nodemask_t *nodes, unsigned long flags,
256 pud = pud_offset(pgd, addr);
258 next = pud_addr_end(addr, end);
259 if (pud_none_or_clear_bad(pud))
261 if (check_pmd_range(vma, pud, addr, next, nodes,
264 } while (pud++, addr = next, addr != end);
268 static inline int check_pgd_range(struct vm_area_struct *vma,
269 unsigned long addr, unsigned long end,
270 const nodemask_t *nodes, unsigned long flags,
276 pgd = pgd_offset(vma->vm_mm, addr);
278 next = pgd_addr_end(addr, end);
279 if (pgd_none_or_clear_bad(pgd))
281 if (check_pud_range(vma, pgd, addr, next, nodes,
284 } while (pgd++, addr = next, addr != end);
288 /* Check if a vma is migratable */
289 static inline int vma_migratable(struct vm_area_struct *vma)
291 if (vma->vm_flags & (
292 VM_LOCKED|VM_IO|VM_HUGETLB|VM_PFNMAP))
298 * Check if all pages in a range are on a set of nodes.
299 * If pagelist != NULL then isolate pages from the LRU and
300 * put them on the pagelist.
302 static struct vm_area_struct *
303 check_range(struct mm_struct *mm, unsigned long start, unsigned long end,
304 const nodemask_t *nodes, unsigned long flags, void *private)
307 struct vm_area_struct *first, *vma, *prev;
309 first = find_vma(mm, start);
311 return ERR_PTR(-EFAULT);
313 for (vma = first; vma && vma->vm_start < end; vma = vma->vm_next) {
314 if (!(flags & MPOL_MF_DISCONTIG_OK)) {
315 if (!vma->vm_next && vma->vm_end < end)
316 return ERR_PTR(-EFAULT);
317 if (prev && prev->vm_end < vma->vm_start)
318 return ERR_PTR(-EFAULT);
320 if (!is_vm_hugetlb_page(vma) &&
321 ((flags & MPOL_MF_STRICT) ||
322 ((flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) &&
323 vma_migratable(vma)))) {
324 unsigned long endvma = vma->vm_end;
328 if (vma->vm_start > start)
329 start = vma->vm_start;
330 err = check_pgd_range(vma, start, endvma, nodes,
333 first = ERR_PTR(err);
342 /* Apply policy to a single VMA */
343 static int policy_vma(struct vm_area_struct *vma, struct mempolicy *new)
346 struct mempolicy *old = vma->vm_policy;
348 PDprintk("vma %lx-%lx/%lx vm_ops %p vm_file %p set_policy %p\n",
349 vma->vm_start, vma->vm_end, vma->vm_pgoff,
350 vma->vm_ops, vma->vm_file,
351 vma->vm_ops ? vma->vm_ops->set_policy : NULL);
353 if (vma->vm_ops && vma->vm_ops->set_policy)
354 err = vma->vm_ops->set_policy(vma, new);
357 vma->vm_policy = new;
363 /* Step 2: apply policy to a range and do splits. */
364 static int mbind_range(struct vm_area_struct *vma, unsigned long start,
365 unsigned long end, struct mempolicy *new)
367 struct vm_area_struct *next;
371 for (; vma && vma->vm_start < end; vma = next) {
373 if (vma->vm_start < start)
374 err = split_vma(vma->vm_mm, vma, start, 1);
375 if (!err && vma->vm_end > end)
376 err = split_vma(vma->vm_mm, vma, end, 0);
378 err = policy_vma(vma, new);
385 static int contextualize_policy(int mode, nodemask_t *nodes)
390 /* Update current mems_allowed */
391 cpuset_update_current_mems_allowed();
392 /* Ignore nodes not set in current->mems_allowed */
393 cpuset_restrict_to_mems_allowed(nodes->bits);
394 return mpol_check_policy(mode, nodes);
397 /* Set the process memory policy */
398 long do_set_mempolicy(int mode, nodemask_t *nodes)
400 struct mempolicy *new;
402 if (contextualize_policy(mode, nodes))
404 new = mpol_new(mode, nodes);
407 mpol_free(current->mempolicy);
408 current->mempolicy = new;
409 if (new && new->policy == MPOL_INTERLEAVE)
410 current->il_next = first_node(new->v.nodes);
414 /* Fill a zone bitmap for a policy */
415 static void get_zonemask(struct mempolicy *p, nodemask_t *nodes)
422 for (i = 0; p->v.zonelist->zones[i]; i++)
423 node_set(p->v.zonelist->zones[i]->zone_pgdat->node_id,
428 case MPOL_INTERLEAVE:
432 /* or use current node instead of online map? */
433 if (p->v.preferred_node < 0)
434 *nodes = node_online_map;
436 node_set(p->v.preferred_node, *nodes);
443 static int lookup_node(struct mm_struct *mm, unsigned long addr)
448 err = get_user_pages(current, mm, addr & PAGE_MASK, 1, 0, 0, &p, NULL);
450 err = page_to_nid(p);
456 /* Retrieve NUMA policy */
457 long do_get_mempolicy(int *policy, nodemask_t *nmask,
458 unsigned long addr, unsigned long flags)
461 struct mm_struct *mm = current->mm;
462 struct vm_area_struct *vma = NULL;
463 struct mempolicy *pol = current->mempolicy;
465 cpuset_update_current_mems_allowed();
466 if (flags & ~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR))
468 if (flags & MPOL_F_ADDR) {
469 down_read(&mm->mmap_sem);
470 vma = find_vma_intersection(mm, addr, addr+1);
472 up_read(&mm->mmap_sem);
475 if (vma->vm_ops && vma->vm_ops->get_policy)
476 pol = vma->vm_ops->get_policy(vma, addr);
478 pol = vma->vm_policy;
483 pol = &default_policy;
485 if (flags & MPOL_F_NODE) {
486 if (flags & MPOL_F_ADDR) {
487 err = lookup_node(mm, addr);
491 } else if (pol == current->mempolicy &&
492 pol->policy == MPOL_INTERLEAVE) {
493 *policy = current->il_next;
499 *policy = pol->policy;
502 up_read(¤t->mm->mmap_sem);
508 get_zonemask(pol, nmask);
512 up_read(¤t->mm->mmap_sem);
520 /* Check if we are the only process mapping the page in question */
521 static inline int single_mm_mapping(struct mm_struct *mm,
522 struct address_space *mapping)
524 struct vm_area_struct *vma;
525 struct prio_tree_iter iter;
528 spin_lock(&mapping->i_mmap_lock);
529 vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, 0, ULONG_MAX)
530 if (mm != vma->vm_mm) {
534 list_for_each_entry(vma, &mapping->i_mmap_nonlinear, shared.vm_set.list)
535 if (mm != vma->vm_mm) {
540 spin_unlock(&mapping->i_mmap_lock);
545 * Add a page to be migrated to the pagelist
547 static void migrate_page_add(struct vm_area_struct *vma,
548 struct page *page, struct list_head *pagelist, unsigned long flags)
551 * Avoid migrating a page that is shared by others and not writable.
553 if ((flags & MPOL_MF_MOVE_ALL) || !page->mapping || PageAnon(page) ||
554 mapping_writably_mapped(page->mapping) ||
555 single_mm_mapping(vma->vm_mm, page->mapping)) {
556 int rc = isolate_lru_page(page);
559 list_add(&page->lru, pagelist);
561 * If the isolate attempt was not successful then we just
562 * encountered an unswappable page. Something must be wrong.
568 static int swap_pages(struct list_head *pagelist)
574 n = migrate_pages(pagelist, NULL, &moved, &failed);
575 putback_lru_pages(&failed);
576 putback_lru_pages(&moved);
582 * For now migrate_pages simply swaps out the pages from nodes that are in
583 * the source set but not in the target set. In the future, we would
584 * want a function that moves pages between the two nodesets in such
585 * a way as to preserve the physical layout as much as possible.
587 * Returns the number of page that could not be moved.
589 int do_migrate_pages(struct mm_struct *mm,
590 const nodemask_t *from_nodes, const nodemask_t *to_nodes, int flags)
596 nodes_andnot(nodes, *from_nodes, *to_nodes);
598 down_read(&mm->mmap_sem);
599 check_range(mm, mm->mmap->vm_start, TASK_SIZE, &nodes,
600 flags | MPOL_MF_DISCONTIG_OK, &pagelist);
602 if (!list_empty(&pagelist)) {
603 count = swap_pages(&pagelist);
604 putback_lru_pages(&pagelist);
607 up_read(&mm->mmap_sem);
611 long do_mbind(unsigned long start, unsigned long len,
612 unsigned long mode, nodemask_t *nmask, unsigned long flags)
614 struct vm_area_struct *vma;
615 struct mm_struct *mm = current->mm;
616 struct mempolicy *new;
621 if ((flags & ~(unsigned long)(MPOL_MF_STRICT |
622 MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
625 if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_RESOURCE))
628 if (start & ~PAGE_MASK)
631 if (mode == MPOL_DEFAULT)
632 flags &= ~MPOL_MF_STRICT;
634 len = (len + PAGE_SIZE - 1) & PAGE_MASK;
642 if (mpol_check_policy(mode, nmask))
645 new = mpol_new(mode, nmask);
650 * If we are using the default policy then operation
651 * on discontinuous address spaces is okay after all
654 flags |= MPOL_MF_DISCONTIG_OK;
656 PDprintk("mbind %lx-%lx mode:%ld nodes:%lx\n",start,start+len,
657 mode,nodes_addr(nodes)[0]);
659 down_write(&mm->mmap_sem);
660 vma = check_range(mm, start, end, nmask,
661 flags | MPOL_MF_INVERT, &pagelist);
667 err = mbind_range(vma, start, end, new);
668 if (!list_empty(&pagelist))
669 nr_failed = swap_pages(&pagelist);
671 if (!err && nr_failed && (flags & MPOL_MF_STRICT))
674 if (!list_empty(&pagelist))
675 putback_lru_pages(&pagelist);
677 up_write(&mm->mmap_sem);
683 * User space interface with variable sized bitmaps for nodelists.
686 /* Copy a node mask from user space. */
687 static int get_nodes(nodemask_t *nodes, const unsigned long __user *nmask,
688 unsigned long maxnode)
691 unsigned long nlongs;
692 unsigned long endmask;
696 if (maxnode == 0 || !nmask)
699 nlongs = BITS_TO_LONGS(maxnode);
700 if ((maxnode % BITS_PER_LONG) == 0)
703 endmask = (1UL << (maxnode % BITS_PER_LONG)) - 1;
705 /* When the user specified more nodes than supported just check
706 if the non supported part is all zero. */
707 if (nlongs > BITS_TO_LONGS(MAX_NUMNODES)) {
708 if (nlongs > PAGE_SIZE/sizeof(long))
710 for (k = BITS_TO_LONGS(MAX_NUMNODES); k < nlongs; k++) {
712 if (get_user(t, nmask + k))
714 if (k == nlongs - 1) {
720 nlongs = BITS_TO_LONGS(MAX_NUMNODES);
724 if (copy_from_user(nodes_addr(*nodes), nmask, nlongs*sizeof(unsigned long)))
726 nodes_addr(*nodes)[nlongs-1] &= endmask;
730 /* Copy a kernel node mask to user space */
731 static int copy_nodes_to_user(unsigned long __user *mask, unsigned long maxnode,
734 unsigned long copy = ALIGN(maxnode-1, 64) / 8;
735 const int nbytes = BITS_TO_LONGS(MAX_NUMNODES) * sizeof(long);
738 if (copy > PAGE_SIZE)
740 if (clear_user((char __user *)mask + nbytes, copy - nbytes))
744 return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0;
747 asmlinkage long sys_mbind(unsigned long start, unsigned long len,
749 unsigned long __user *nmask, unsigned long maxnode,
755 err = get_nodes(&nodes, nmask, maxnode);
758 return do_mbind(start, len, mode, &nodes, flags);
761 /* Set the process memory policy */
762 asmlinkage long sys_set_mempolicy(int mode, unsigned long __user *nmask,
763 unsigned long maxnode)
768 if (mode < 0 || mode > MPOL_MAX)
770 err = get_nodes(&nodes, nmask, maxnode);
773 return do_set_mempolicy(mode, &nodes);
776 /* Macro needed until Paul implements this function in kernel/cpusets.c */
777 #define cpuset_mems_allowed(task) node_online_map
779 asmlinkage long sys_migrate_pages(pid_t pid, unsigned long maxnode,
780 const unsigned long __user *old_nodes,
781 const unsigned long __user *new_nodes)
783 struct mm_struct *mm;
784 struct task_struct *task;
787 nodemask_t task_nodes;
790 err = get_nodes(&old, old_nodes, maxnode);
794 err = get_nodes(&new, new_nodes, maxnode);
798 /* Find the mm_struct */
799 read_lock(&tasklist_lock);
800 task = pid ? find_task_by_pid(pid) : current;
802 read_unlock(&tasklist_lock);
805 mm = get_task_mm(task);
806 read_unlock(&tasklist_lock);
812 * Check if this process has the right to modify the specified
813 * process. The right exists if the process has administrative
814 * capabilities, superuser priviledges or the same
815 * userid as the target process.
817 if ((current->euid != task->suid) && (current->euid != task->uid) &&
818 (current->uid != task->suid) && (current->uid != task->uid) &&
819 !capable(CAP_SYS_ADMIN)) {
824 task_nodes = cpuset_mems_allowed(task);
825 /* Is the user allowed to access the target nodes? */
826 if (!nodes_subset(new, task_nodes) && !capable(CAP_SYS_ADMIN)) {
831 err = do_migrate_pages(mm, &old, &new, MPOL_MF_MOVE);
838 /* Retrieve NUMA policy */
839 asmlinkage long sys_get_mempolicy(int __user *policy,
840 unsigned long __user *nmask,
841 unsigned long maxnode,
842 unsigned long addr, unsigned long flags)
847 if (nmask != NULL && maxnode < MAX_NUMNODES)
850 err = do_get_mempolicy(&pval, &nodes, addr, flags);
855 if (policy && put_user(pval, policy))
859 err = copy_nodes_to_user(nmask, maxnode, &nodes);
866 asmlinkage long compat_sys_get_mempolicy(int __user *policy,
867 compat_ulong_t __user *nmask,
868 compat_ulong_t maxnode,
869 compat_ulong_t addr, compat_ulong_t flags)
872 unsigned long __user *nm = NULL;
873 unsigned long nr_bits, alloc_size;
874 DECLARE_BITMAP(bm, MAX_NUMNODES);
876 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
877 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
880 nm = compat_alloc_user_space(alloc_size);
882 err = sys_get_mempolicy(policy, nm, nr_bits+1, addr, flags);
885 err = copy_from_user(bm, nm, alloc_size);
886 /* ensure entire bitmap is zeroed */
887 err |= clear_user(nmask, ALIGN(maxnode-1, 8) / 8);
888 err |= compat_put_bitmap(nmask, bm, nr_bits);
894 asmlinkage long compat_sys_set_mempolicy(int mode, compat_ulong_t __user *nmask,
895 compat_ulong_t maxnode)
898 unsigned long __user *nm = NULL;
899 unsigned long nr_bits, alloc_size;
900 DECLARE_BITMAP(bm, MAX_NUMNODES);
902 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
903 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
906 err = compat_get_bitmap(bm, nmask, nr_bits);
907 nm = compat_alloc_user_space(alloc_size);
908 err |= copy_to_user(nm, bm, alloc_size);
914 return sys_set_mempolicy(mode, nm, nr_bits+1);
917 asmlinkage long compat_sys_mbind(compat_ulong_t start, compat_ulong_t len,
918 compat_ulong_t mode, compat_ulong_t __user *nmask,
919 compat_ulong_t maxnode, compat_ulong_t flags)
922 unsigned long __user *nm = NULL;
923 unsigned long nr_bits, alloc_size;
926 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
927 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
930 err = compat_get_bitmap(nodes_addr(bm), nmask, nr_bits);
931 nm = compat_alloc_user_space(alloc_size);
932 err |= copy_to_user(nm, nodes_addr(bm), alloc_size);
938 return sys_mbind(start, len, mode, nm, nr_bits+1, flags);
943 /* Return effective policy for a VMA */
944 static struct mempolicy * get_vma_policy(struct task_struct *task,
945 struct vm_area_struct *vma, unsigned long addr)
947 struct mempolicy *pol = task->mempolicy;
950 if (vma->vm_ops && vma->vm_ops->get_policy)
951 pol = vma->vm_ops->get_policy(vma, addr);
952 else if (vma->vm_policy &&
953 vma->vm_policy->policy != MPOL_DEFAULT)
954 pol = vma->vm_policy;
957 pol = &default_policy;
961 /* Return a zonelist representing a mempolicy */
962 static struct zonelist *zonelist_policy(gfp_t gfp, struct mempolicy *policy)
966 switch (policy->policy) {
968 nd = policy->v.preferred_node;
973 /* Lower zones don't get a policy applied */
974 /* Careful: current->mems_allowed might have moved */
975 if (gfp_zone(gfp) >= policy_zone)
976 if (cpuset_zonelist_valid_mems_allowed(policy->v.zonelist))
977 return policy->v.zonelist;
979 case MPOL_INTERLEAVE: /* should not happen */
987 return NODE_DATA(nd)->node_zonelists + gfp_zone(gfp);
990 /* Do dynamic interleaving for a process */
991 static unsigned interleave_nodes(struct mempolicy *policy)
994 struct task_struct *me = current;
997 next = next_node(nid, policy->v.nodes);
998 if (next >= MAX_NUMNODES)
999 next = first_node(policy->v.nodes);
1004 /* Do static interleaving for a VMA with known offset. */
1005 static unsigned offset_il_node(struct mempolicy *pol,
1006 struct vm_area_struct *vma, unsigned long off)
1008 unsigned nnodes = nodes_weight(pol->v.nodes);
1009 unsigned target = (unsigned)off % nnodes;
1015 nid = next_node(nid, pol->v.nodes);
1017 } while (c <= target);
1021 /* Determine a node number for interleave */
1022 static inline unsigned interleave_nid(struct mempolicy *pol,
1023 struct vm_area_struct *vma, unsigned long addr, int shift)
1028 off = vma->vm_pgoff;
1029 off += (addr - vma->vm_start) >> shift;
1030 return offset_il_node(pol, vma, off);
1032 return interleave_nodes(pol);
1035 /* Return a zonelist suitable for a huge page allocation. */
1036 struct zonelist *huge_zonelist(struct vm_area_struct *vma, unsigned long addr)
1038 struct mempolicy *pol = get_vma_policy(current, vma, addr);
1040 if (pol->policy == MPOL_INTERLEAVE) {
1043 nid = interleave_nid(pol, vma, addr, HPAGE_SHIFT);
1044 return NODE_DATA(nid)->node_zonelists + gfp_zone(GFP_HIGHUSER);
1046 return zonelist_policy(GFP_HIGHUSER, pol);
1049 /* Allocate a page in interleaved policy.
1050 Own path because it needs to do special accounting. */
1051 static struct page *alloc_page_interleave(gfp_t gfp, unsigned order,
1054 struct zonelist *zl;
1057 zl = NODE_DATA(nid)->node_zonelists + gfp_zone(gfp);
1058 page = __alloc_pages(gfp, order, zl);
1059 if (page && page_zone(page) == zl->zones[0]) {
1060 zone_pcp(zl->zones[0],get_cpu())->interleave_hit++;
1067 * alloc_page_vma - Allocate a page for a VMA.
1070 * %GFP_USER user allocation.
1071 * %GFP_KERNEL kernel allocations,
1072 * %GFP_HIGHMEM highmem/user allocations,
1073 * %GFP_FS allocation should not call back into a file system.
1074 * %GFP_ATOMIC don't sleep.
1076 * @vma: Pointer to VMA or NULL if not available.
1077 * @addr: Virtual Address of the allocation. Must be inside the VMA.
1079 * This function allocates a page from the kernel page pool and applies
1080 * a NUMA policy associated with the VMA or the current process.
1081 * When VMA is not NULL caller must hold down_read on the mmap_sem of the
1082 * mm_struct of the VMA to prevent it from going away. Should be used for
1083 * all allocations for pages that will be mapped into
1084 * user space. Returns NULL when no page can be allocated.
1086 * Should be called with the mm_sem of the vma hold.
1089 alloc_page_vma(gfp_t gfp, struct vm_area_struct *vma, unsigned long addr)
1091 struct mempolicy *pol = get_vma_policy(current, vma, addr);
1093 cpuset_update_current_mems_allowed();
1095 if (unlikely(pol->policy == MPOL_INTERLEAVE)) {
1098 nid = interleave_nid(pol, vma, addr, PAGE_SHIFT);
1099 return alloc_page_interleave(gfp, 0, nid);
1101 return __alloc_pages(gfp, 0, zonelist_policy(gfp, pol));
1105 * alloc_pages_current - Allocate pages.
1108 * %GFP_USER user allocation,
1109 * %GFP_KERNEL kernel allocation,
1110 * %GFP_HIGHMEM highmem allocation,
1111 * %GFP_FS don't call back into a file system.
1112 * %GFP_ATOMIC don't sleep.
1113 * @order: Power of two of allocation size in pages. 0 is a single page.
1115 * Allocate a page from the kernel page pool. When not in
1116 * interrupt context and apply the current process NUMA policy.
1117 * Returns NULL when no page can be allocated.
1119 * Don't call cpuset_update_current_mems_allowed() unless
1120 * 1) it's ok to take cpuset_sem (can WAIT), and
1121 * 2) allocating for current task (not interrupt).
1123 struct page *alloc_pages_current(gfp_t gfp, unsigned order)
1125 struct mempolicy *pol = current->mempolicy;
1127 if ((gfp & __GFP_WAIT) && !in_interrupt())
1128 cpuset_update_current_mems_allowed();
1129 if (!pol || in_interrupt())
1130 pol = &default_policy;
1131 if (pol->policy == MPOL_INTERLEAVE)
1132 return alloc_page_interleave(gfp, order, interleave_nodes(pol));
1133 return __alloc_pages(gfp, order, zonelist_policy(gfp, pol));
1135 EXPORT_SYMBOL(alloc_pages_current);
1137 /* Slow path of a mempolicy copy */
1138 struct mempolicy *__mpol_copy(struct mempolicy *old)
1140 struct mempolicy *new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
1143 return ERR_PTR(-ENOMEM);
1145 atomic_set(&new->refcnt, 1);
1146 if (new->policy == MPOL_BIND) {
1147 int sz = ksize(old->v.zonelist);
1148 new->v.zonelist = kmalloc(sz, SLAB_KERNEL);
1149 if (!new->v.zonelist) {
1150 kmem_cache_free(policy_cache, new);
1151 return ERR_PTR(-ENOMEM);
1153 memcpy(new->v.zonelist, old->v.zonelist, sz);
1158 /* Slow path of a mempolicy comparison */
1159 int __mpol_equal(struct mempolicy *a, struct mempolicy *b)
1163 if (a->policy != b->policy)
1165 switch (a->policy) {
1168 case MPOL_INTERLEAVE:
1169 return nodes_equal(a->v.nodes, b->v.nodes);
1170 case MPOL_PREFERRED:
1171 return a->v.preferred_node == b->v.preferred_node;
1174 for (i = 0; a->v.zonelist->zones[i]; i++)
1175 if (a->v.zonelist->zones[i] != b->v.zonelist->zones[i])
1177 return b->v.zonelist->zones[i] == NULL;
1185 /* Slow path of a mpol destructor. */
1186 void __mpol_free(struct mempolicy *p)
1188 if (!atomic_dec_and_test(&p->refcnt))
1190 if (p->policy == MPOL_BIND)
1191 kfree(p->v.zonelist);
1192 p->policy = MPOL_DEFAULT;
1193 kmem_cache_free(policy_cache, p);
1197 * Shared memory backing store policy support.
1199 * Remember policies even when nobody has shared memory mapped.
1200 * The policies are kept in Red-Black tree linked from the inode.
1201 * They are protected by the sp->lock spinlock, which should be held
1202 * for any accesses to the tree.
1205 /* lookup first element intersecting start-end */
1206 /* Caller holds sp->lock */
1207 static struct sp_node *
1208 sp_lookup(struct shared_policy *sp, unsigned long start, unsigned long end)
1210 struct rb_node *n = sp->root.rb_node;
1213 struct sp_node *p = rb_entry(n, struct sp_node, nd);
1215 if (start >= p->end)
1217 else if (end <= p->start)
1225 struct sp_node *w = NULL;
1226 struct rb_node *prev = rb_prev(n);
1229 w = rb_entry(prev, struct sp_node, nd);
1230 if (w->end <= start)
1234 return rb_entry(n, struct sp_node, nd);
1237 /* Insert a new shared policy into the list. */
1238 /* Caller holds sp->lock */
1239 static void sp_insert(struct shared_policy *sp, struct sp_node *new)
1241 struct rb_node **p = &sp->root.rb_node;
1242 struct rb_node *parent = NULL;
1247 nd = rb_entry(parent, struct sp_node, nd);
1248 if (new->start < nd->start)
1250 else if (new->end > nd->end)
1251 p = &(*p)->rb_right;
1255 rb_link_node(&new->nd, parent, p);
1256 rb_insert_color(&new->nd, &sp->root);
1257 PDprintk("inserting %lx-%lx: %d\n", new->start, new->end,
1258 new->policy ? new->policy->policy : 0);
1261 /* Find shared policy intersecting idx */
1263 mpol_shared_policy_lookup(struct shared_policy *sp, unsigned long idx)
1265 struct mempolicy *pol = NULL;
1268 if (!sp->root.rb_node)
1270 spin_lock(&sp->lock);
1271 sn = sp_lookup(sp, idx, idx+1);
1273 mpol_get(sn->policy);
1276 spin_unlock(&sp->lock);
1280 static void sp_delete(struct shared_policy *sp, struct sp_node *n)
1282 PDprintk("deleting %lx-l%x\n", n->start, n->end);
1283 rb_erase(&n->nd, &sp->root);
1284 mpol_free(n->policy);
1285 kmem_cache_free(sn_cache, n);
1289 sp_alloc(unsigned long start, unsigned long end, struct mempolicy *pol)
1291 struct sp_node *n = kmem_cache_alloc(sn_cache, GFP_KERNEL);
1302 /* Replace a policy range. */
1303 static int shared_policy_replace(struct shared_policy *sp, unsigned long start,
1304 unsigned long end, struct sp_node *new)
1306 struct sp_node *n, *new2 = NULL;
1309 spin_lock(&sp->lock);
1310 n = sp_lookup(sp, start, end);
1311 /* Take care of old policies in the same range. */
1312 while (n && n->start < end) {
1313 struct rb_node *next = rb_next(&n->nd);
1314 if (n->start >= start) {
1320 /* Old policy spanning whole new range. */
1323 spin_unlock(&sp->lock);
1324 new2 = sp_alloc(end, n->end, n->policy);
1330 sp_insert(sp, new2);
1338 n = rb_entry(next, struct sp_node, nd);
1342 spin_unlock(&sp->lock);
1344 mpol_free(new2->policy);
1345 kmem_cache_free(sn_cache, new2);
1350 int mpol_set_shared_policy(struct shared_policy *info,
1351 struct vm_area_struct *vma, struct mempolicy *npol)
1354 struct sp_node *new = NULL;
1355 unsigned long sz = vma_pages(vma);
1357 PDprintk("set_shared_policy %lx sz %lu %d %lx\n",
1359 sz, npol? npol->policy : -1,
1360 npol ? nodes_addr(npol->v.nodes)[0] : -1);
1363 new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, npol);
1367 err = shared_policy_replace(info, vma->vm_pgoff, vma->vm_pgoff+sz, new);
1369 kmem_cache_free(sn_cache, new);
1373 /* Free a backing policy store on inode delete. */
1374 void mpol_free_shared_policy(struct shared_policy *p)
1377 struct rb_node *next;
1379 if (!p->root.rb_node)
1381 spin_lock(&p->lock);
1382 next = rb_first(&p->root);
1384 n = rb_entry(next, struct sp_node, nd);
1385 next = rb_next(&n->nd);
1386 rb_erase(&n->nd, &p->root);
1387 mpol_free(n->policy);
1388 kmem_cache_free(sn_cache, n);
1390 spin_unlock(&p->lock);
1393 /* assumes fs == KERNEL_DS */
1394 void __init numa_policy_init(void)
1396 policy_cache = kmem_cache_create("numa_policy",
1397 sizeof(struct mempolicy),
1398 0, SLAB_PANIC, NULL, NULL);
1400 sn_cache = kmem_cache_create("shared_policy_node",
1401 sizeof(struct sp_node),
1402 0, SLAB_PANIC, NULL, NULL);
1404 /* Set interleaving policy for system init. This way not all
1405 the data structures allocated at system boot end up in node zero. */
1407 if (do_set_mempolicy(MPOL_INTERLEAVE, &node_online_map))
1408 printk("numa_policy_init: interleaving failed\n");
1411 /* Reset policy of current process to default */
1412 void numa_default_policy(void)
1414 do_set_mempolicy(MPOL_DEFAULT, NULL);
1417 /* Migrate a policy to a different set of nodes */
1418 static void rebind_policy(struct mempolicy *pol, const nodemask_t *old,
1419 const nodemask_t *new)
1426 switch (pol->policy) {
1429 case MPOL_INTERLEAVE:
1430 nodes_remap(tmp, pol->v.nodes, *old, *new);
1432 current->il_next = node_remap(current->il_next, *old, *new);
1434 case MPOL_PREFERRED:
1435 pol->v.preferred_node = node_remap(pol->v.preferred_node,
1441 struct zonelist *zonelist;
1444 for (z = pol->v.zonelist->zones; *z; z++)
1445 node_set((*z)->zone_pgdat->node_id, nodes);
1446 nodes_remap(tmp, nodes, *old, *new);
1449 zonelist = bind_zonelist(&nodes);
1451 /* If no mem, then zonelist is NULL and we keep old zonelist.
1452 * If that old zonelist has no remaining mems_allowed nodes,
1453 * then zonelist_policy() will "FALL THROUGH" to MPOL_DEFAULT.
1457 /* Good - got mem - substitute new zonelist */
1458 kfree(pol->v.zonelist);
1459 pol->v.zonelist = zonelist;
1470 * Someone moved this task to different nodes. Fixup mempolicies.
1472 * TODO - fixup current->mm->vma and shmfs/tmpfs/hugetlbfs policies as well,
1473 * once we have a cpuset mechanism to mark which cpuset subtree is migrating.
1475 void numa_policy_rebind(const nodemask_t *old, const nodemask_t *new)
1477 rebind_policy(current->mempolicy, old, new);
1481 * Display pages allocated per node and memory policy via /proc.
1484 static const char *policy_types[] = { "default", "prefer", "bind",
1488 * Convert a mempolicy into a string.
1489 * Returns the number of characters in buffer (if positive)
1490 * or an error (negative)
1492 static inline int mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol)
1497 int mode = pol ? pol->policy : MPOL_DEFAULT;
1504 case MPOL_PREFERRED:
1506 node_set(pol->v.preferred_node, nodes);
1510 get_zonemask(pol, &nodes);
1513 case MPOL_INTERLEAVE:
1514 nodes = pol->v.nodes;
1522 l = strlen(policy_types[mode]);
1523 if (buffer + maxlen < p + l + 1)
1526 strcpy(p, policy_types[mode]);
1529 if (!nodes_empty(nodes)) {
1530 if (buffer + maxlen < p + 2)
1533 p += nodelist_scnprintf(p, buffer + maxlen - p, nodes);
1539 unsigned long pages;
1541 unsigned long mapped;
1542 unsigned long mapcount_max;
1543 unsigned long node[MAX_NUMNODES];
1546 static void gather_stats(struct page *page, void *private)
1548 struct numa_maps *md = private;
1549 int count = page_mapcount(page);
1554 if (count > md->mapcount_max)
1555 md->mapcount_max = count;
1562 md->node[page_to_nid(page)]++;
1566 int show_numa_map(struct seq_file *m, void *v)
1568 struct task_struct *task = m->private;
1569 struct vm_area_struct *vma = v;
1570 struct numa_maps *md;
1577 md = kzalloc(sizeof(struct numa_maps), GFP_KERNEL);
1581 check_pgd_range(vma, vma->vm_start, vma->vm_end,
1582 &node_online_map, MPOL_MF_STATS, md);
1585 mpol_to_str(buffer, sizeof(buffer),
1586 get_vma_policy(task, vma, vma->vm_start));
1588 seq_printf(m, "%08lx %s pages=%lu mapped=%lu maxref=%lu",
1589 vma->vm_start, buffer, md->pages,
1590 md->mapped, md->mapcount_max);
1593 seq_printf(m," anon=%lu",md->anon);
1595 for_each_online_node(n)
1597 seq_printf(m, " N%d=%lu", n, md->node[n]);
1603 if (m->count < m->size)
1604 m->version = (vma != get_gate_vma(task)) ? vma->vm_start : 0;