sched, numa, mm/mpol: Make mempolicy home-node aware

Add another layer of fallback policy to make the home node concept
useful from a memory allocation PoV.

This changes the mpol order to:

 - vma->vm_ops->get_policy	[if applicable]
 - vma->vm_policy		[if applicable]
 - task->mempolicy
 - tsk_home_node() preferred	[NEW]
 - default_policy

Note that the tsk_home_node() policy has Migrate-on-Fault enabled to
facilitate efficient on-demand memory migration.

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Paul Turner <pjt@google.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: http://lkml.kernel.org/n/tip-ybn5oygsqXZa6jtwpwjywmdu@git.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 5ee326c9cef09b83abb4b03887d9bc23187ec108..97e6d9950670384abf6f013a0939019816e64549 100644 (file)
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -117,6 +117,22 @@ static struct mempolicy default_policy = {
        .flags = MPOL_F_LOCAL,
 };
 
+static struct mempolicy preferred_node_policy[MAX_NUMNODES];
+
+static struct mempolicy *get_task_policy(struct task_struct *p)
+{
+       struct mempolicy *pol = p->mempolicy;
+       int node;
+
+       if (!pol) {
+               node = tsk_home_node(p);
+               if (node != -1)
+                       pol = &preferred_node_policy[node];
+       }
+
+       return pol;
+}
+
 static const struct mempolicy_operations {
        int (*create)(struct mempolicy *pol, const nodemask_t *nodes);
        /*
@@ -1565,7 +1581,7 @@ asmlinkage long compat_sys_mbind(compat_ulong_t start, compat_ulong_t len,
 struct mempolicy *get_vma_policy(struct task_struct *task,
                struct vm_area_struct *vma, unsigned long addr)
 {
-       struct mempolicy *pol = task->mempolicy;
+       struct mempolicy *pol = get_task_policy(task);
 
        if (vma) {
                if (vma->vm_ops && vma->vm_ops->get_policy) {
@@ -1988,7 +2004,7 @@ retry_cpuset:
  */
 struct page *alloc_pages_current(gfp_t gfp, unsigned order)
 {
-       struct mempolicy *pol = current->mempolicy;
+       struct mempolicy *pol = get_task_policy(current);
        struct page *page;
        unsigned int cpuset_mems_cookie;
 
@@ -2450,6 +2466,15 @@ void __init numa_policy_init(void)
                                     sizeof(struct sp_node),
                                     0, SLAB_PANIC, NULL);
 
+       for_each_node(nid) {
+               preferred_node_policy[nid] = (struct mempolicy) {
+                       .refcnt = ATOMIC_INIT(1),
+                       .mode = MPOL_PREFERRED,
+                       .flags = MPOL_F_MOF,
+                       .v = { .preferred_node = nid, },
+               };
+       }
+
        /*
         * Set interleaving policy for system init. Interleaving is only
         * enabled across suitably sized nodes (default is >= 16MB), or