mm: move all mmu notifier invocations to be done outside the PT lock

[karo-tx-linux.git] / mm / memory.c

diff --git a/mm/memory.c b/mm/memory.c
index 060d36f6a048eb98d478e669e0aebce1e58c90b7..4c1a3f21199a69bd097d21db70beec9070e2234c 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -717,7 +717,7 @@ static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr,
        add_taint(TAINT_BAD_PAGE);
 }
 
-static inline int is_cow_mapping(vm_flags_t flags)
+static inline bool is_cow_mapping(vm_flags_t flags)
 {
        return (flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
 }
@@ -1044,6 +1044,9 @@ int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
        unsigned long next;
        unsigned long addr = vma->vm_start;
        unsigned long end = vma->vm_end;
+       unsigned long mmun_start;       /* For mmu_notifiers */
+       unsigned long mmun_end;         /* For mmu_notifiers */
+       bool is_cow;
        int ret;
 
        /*
@@ -1077,8 +1080,12 @@ int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
         * parent mm. And a permission downgrade will only happen if
         * is_cow_mapping() returns true.
         */
-       if (is_cow_mapping(vma->vm_flags))
-               mmu_notifier_invalidate_range_start(src_mm, addr, end);
+       is_cow = is_cow_mapping(vma->vm_flags);
+       mmun_start = addr;
+       mmun_end   = end;
+       if (is_cow)
+               mmu_notifier_invalidate_range_start(src_mm, mmun_start,
+                                                   mmun_end);
 
        ret = 0;
        dst_pgd = pgd_offset(dst_mm, addr);
@@ -1094,9 +1101,8 @@ int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
                }
        } while (dst_pgd++, src_pgd++, addr = next, addr != end);
 
-       if (is_cow_mapping(vma->vm_flags))
-               mmu_notifier_invalidate_range_end(src_mm,
-                                                 vma->vm_start, end);
+       if (is_cow)
+               mmu_notifier_invalidate_range_end(src_mm, mmun_start, mmun_end);
        return ret;
 }
 
@@ -2521,7 +2527,7 @@ static int do_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
                spinlock_t *ptl, pte_t orig_pte)
        __releases(ptl)
 {
-       struct page *old_page, *new_page;
+       struct page *old_page, *new_page = NULL;
        pte_t entry;
        int ret = 0;
        int page_mkwrite = 0;
@@ -2765,10 +2771,14 @@ gotten:
        } else
                mem_cgroup_uncharge_page(new_page);
 
-       if (new_page)
-               page_cache_release(new_page);
 unlock:
        pte_unmap_unlock(page_table, ptl);
+       if (new_page) {
+               if (new_page == old_page)
+                       /* cow happened, notify before releasing old_page */
+                       mmu_notifier_invalidate_page(mm, address);
+               page_cache_release(new_page);
+       }
        if (old_page) {
                /*
                 * Don't let another task, with possibly unlocked vma,