Merge branch 'for-linus' of git://brick.kernel.dk/data/git/linux-2.6-block

[karo-tx-linux.git] / mm / mempolicy.c

diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 34d566ac147f54dffd4cc86cd240fcec5779345d..3171f884d2459a30ad113d9008b82b04d833671c 100644 (file)
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -180,6 +180,7 @@ static struct mempolicy *mpol_new(int mode, nodemask_t *nodes)
                break;
        }
        policy->policy = mode;
+       policy->cpuset_mems_allowed = cpuset_mems_allowed(current);
        return policy;
 }
 
@@ -207,6 +208,8 @@ static int check_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
                page = vm_normal_page(vma, addr, *pte);
                if (!page)
                        continue;
+               if (PageReserved(page))
+                       continue;
                nid = page_to_nid(page);
                if (node_isset(nid, *nodes) == !!(flags & MPOL_MF_INVERT))
                        continue;
@@ -289,7 +292,7 @@ static inline int check_pgd_range(struct vm_area_struct *vma,
 static inline int vma_migratable(struct vm_area_struct *vma)
 {
        if (vma->vm_flags & (
-               VM_LOCKED|VM_IO|VM_HUGETLB|VM_PFNMAP))
+               VM_LOCKED|VM_IO|VM_HUGETLB|VM_PFNMAP|VM_RESERVED))
                return 0;
        return 1;
 }
@@ -1130,6 +1133,15 @@ struct page *alloc_pages_current(gfp_t gfp, unsigned order)
 }
 EXPORT_SYMBOL(alloc_pages_current);
 
+/*
+ * If mpol_copy() sees current->cpuset == cpuset_being_rebound, then it
+ * rebinds the mempolicy its copying by calling mpol_rebind_policy()
+ * with the mems_allowed returned by cpuset_mems_allowed().  This
+ * keeps mempolicies cpuset relative after its cpuset moves.  See
+ * further kernel/cpuset.c update_nodemask().
+ */
+void *cpuset_being_rebound;
+
 /* Slow path of a mempolicy copy */
 struct mempolicy *__mpol_copy(struct mempolicy *old)
 {
@@ -1137,6 +1149,10 @@ struct mempolicy *__mpol_copy(struct mempolicy *old)
 
        if (!new)
                return ERR_PTR(-ENOMEM);
+       if (current_cpuset_is_being_rebound()) {
+               nodemask_t mems = cpuset_mems_allowed(current);
+               mpol_rebind_policy(old, &mems);
+       }
        *new = *old;
        atomic_set(&new->refcnt, 1);
        if (new->policy == MPOL_BIND) {
@@ -1343,6 +1359,30 @@ restart:
        return 0;
 }
 
+void mpol_shared_policy_init(struct shared_policy *info, int policy,
+                               nodemask_t *policy_nodes)
+{
+       info->root = RB_ROOT;
+       spin_lock_init(&info->lock);
+
+       if (policy != MPOL_DEFAULT) {
+               struct mempolicy *newpol;
+
+               /* Falls back to MPOL_DEFAULT on any error */
+               newpol = mpol_new(policy, policy_nodes);
+               if (!IS_ERR(newpol)) {
+                       /* Create pseudo-vma that contains just the policy */
+                       struct vm_area_struct pvma;
+
+                       memset(&pvma, 0, sizeof(struct vm_area_struct));
+                       /* Policy covers entire file */
+                       pvma.vm_end = TASK_SIZE;
+                       mpol_set_shared_policy(info, &pvma, newpol);
+                       mpol_free(newpol);
+               }
+       }
+}
+
 int mpol_set_shared_policy(struct shared_policy *info,
                        struct vm_area_struct *vma, struct mempolicy *npol)
 {
@@ -1411,25 +1451,31 @@ void numa_default_policy(void)
 }
 
 /* Migrate a policy to a different set of nodes */
-static void rebind_policy(struct mempolicy *pol, const nodemask_t *old,
-                                                       const nodemask_t *new)
+void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask)
 {
+       nodemask_t *mpolmask;
        nodemask_t tmp;
 
        if (!pol)
                return;
+       mpolmask = &pol->cpuset_mems_allowed;
+       if (nodes_equal(*mpolmask, *newmask))
+               return;
 
        switch (pol->policy) {
        case MPOL_DEFAULT:
                break;
        case MPOL_INTERLEAVE:
-               nodes_remap(tmp, pol->v.nodes, *old, *new);
+               nodes_remap(tmp, pol->v.nodes, *mpolmask, *newmask);
                pol->v.nodes = tmp;
-               current->il_next = node_remap(current->il_next, *old, *new);
+               *mpolmask = *newmask;
+               current->il_next = node_remap(current->il_next,
+                                               *mpolmask, *newmask);
                break;
        case MPOL_PREFERRED:
                pol->v.preferred_node = node_remap(pol->v.preferred_node,
-                                                               *old, *new);
+                                               *mpolmask, *newmask);
+               *mpolmask = *newmask;
                break;
        case MPOL_BIND: {
                nodemask_t nodes;
@@ -1439,7 +1485,7 @@ static void rebind_policy(struct mempolicy *pol, const nodemask_t *old,
                nodes_clear(nodes);
                for (z = pol->v.zonelist->zones; *z; z++)
                        node_set((*z)->zone_pgdat->node_id, nodes);
-               nodes_remap(tmp, nodes, *old, *new);
+               nodes_remap(tmp, nodes, *mpolmask, *newmask);
                nodes = tmp;
 
                zonelist = bind_zonelist(&nodes);
@@ -1454,6 +1500,7 @@ static void rebind_policy(struct mempolicy *pol, const nodemask_t *old,
                        kfree(pol->v.zonelist);
                        pol->v.zonelist = zonelist;
                }
+               *mpolmask = *newmask;
                break;
        }
        default:
@@ -1463,14 +1510,29 @@ static void rebind_policy(struct mempolicy *pol, const nodemask_t *old,
 }
 
 /*
- * Someone moved this task to different nodes.  Fixup mempolicies.
+ * Wrapper for mpol_rebind_policy() that just requires task
+ * pointer, and updates task mempolicy.
+ */
+
+void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new)
+{
+       mpol_rebind_policy(tsk->mempolicy, new);
+}
+
+/*
+ * Rebind each vma in mm to new nodemask.
  *
- * TODO - fixup current->mm->vma and shmfs/tmpfs/hugetlbfs policies as well,
- * once we have a cpuset mechanism to mark which cpuset subtree is migrating.
+ * Call holding a reference to mm.  Takes mm->mmap_sem during call.
  */
-void numa_policy_rebind(const nodemask_t *old, const nodemask_t *new)
+
+void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
 {
-       rebind_policy(current->mempolicy, old, new);
+       struct vm_area_struct *vma;
+
+       down_write(&mm->mmap_sem);
+       for (vma = mm->mmap; vma; vma = vma->vm_next)
+               mpol_rebind_policy(vma->vm_policy, new);
+       up_write(&mm->mmap_sem);
 }
 
 /*