mm: introduce VM_POPULATE flag to better deal with racy userspace programs

The vm_populate() code populates user mappings without constantly
holding the mmap_sem. This makes it susceptible to racy userspace
programs: the user mappings may change while vm_populate() is running,
and in this case vm_populate() may end up populating the new mapping
instead of the old one.

In order to reduce the possibility of userspace getting surprised by
this behavior, this change introduces the VM_POPULATE vma flag which
gets set on vmas we want vm_populate() to work on. This way
vm_populate() may still end up populating the new mapping after such a
race, but only if the new mapping is also one that the user has
requested (using MAP_SHARED, MAP_LOCKED or mlock) to be populated.

Signed-off-by: Michel Lespinasse <walken@google.com>
Acked-by: Rik van Riel <riel@redhat.com>
Tested-by: Andy Lutomirski <luto@amacapital.net>
Cc: Greg Ungerer <gregungerer@westnet.com.au>
Cc: David Howells <dhowells@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/mm.h b/include/linux/mm.h
index a4f7769769fe3157e4872b70465b92e00b0699d7..e7fedf5b8c03d5f4a9168bbd09f6117aed713536 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -87,6 +87,7 @@ extern unsigned int kobjsize(const void *objp);
 #define VM_PFNMAP      0x00000400      /* Page-ranges managed without "struct page", just pure PFN */
 #define VM_DENYWRITE   0x00000800      /* ETXTBSY on write attempts.. */
 
+#define VM_POPULATE     0x00001000
 #define VM_LOCKED      0x00002000
 #define VM_IO           0x00004000     /* Memory mapped I/O or similar */
 

diff --git a/include/linux/mman.h b/include/linux/mman.h
index 9aa863da287fedf383f3c507c287aa220cbd6a86..61c7a87e5d2b358484dc16ab1b0db79253fc93b3 100644 (file)
--- a/include/linux/mman.h
+++ b/include/linux/mman.h
@@ -79,6 +79,8 @@ calc_vm_flag_bits(unsigned long flags)
 {
        return _calc_vm_trans(flags, MAP_GROWSDOWN,  VM_GROWSDOWN ) |
               _calc_vm_trans(flags, MAP_DENYWRITE,  VM_DENYWRITE ) |
-              _calc_vm_trans(flags, MAP_LOCKED,     VM_LOCKED    );
+              ((flags & MAP_LOCKED) ? (VM_LOCKED | VM_POPULATE) : 0) |
+              (((flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE) ?
+                                                       VM_POPULATE : 0);
 }
 #endif /* _LINUX_MMAN_H */

diff --git a/mm/fremap.c b/mm/fremap.c
index 503a72387087ec59c8b3073e1d09487fcc23c523..0cd4c11488edf8c8fd33346e9b94e28fb3272aa1 100644 (file)
--- a/mm/fremap.c
+++ b/mm/fremap.c
@@ -204,8 +204,10 @@ get_write_lock:
                        unsigned long addr;
                        struct file *file = get_file(vma->vm_file);
 
-                       addr = mmap_region(file, start, size,
-                                       vma->vm_flags, pgoff);
+                       vm_flags = vma->vm_flags;
+                       if (!(flags & MAP_NONBLOCK))
+                               vm_flags |= VM_POPULATE;
+                       addr = mmap_region(file, start, size, vm_flags, pgoff);
                        fput(file);
                        if (IS_ERR_VALUE(addr)) {
                                err = addr;
@@ -224,6 +226,12 @@ get_write_lock:
                mutex_unlock(&mapping->i_mmap_mutex);
        }
 
+       if (!(flags & MAP_NONBLOCK) && !(vma->vm_flags & VM_POPULATE)) {
+               if (!has_write_lock)
+                       goto get_write_lock;
+               vma->vm_flags |= VM_POPULATE;
+       }
+
        if (vma->vm_flags & VM_LOCKED) {
                /*
                 * drop PG_Mlocked flag for over-mapped range

diff --git a/mm/mlock.c b/mm/mlock.c
index 569400a5d07901d39d63ee1d225fe52837e8c386..d6378feb2950678f1799e7ffecac9630a2bb3996 100644 (file)
--- a/mm/mlock.c
+++ b/mm/mlock.c
@@ -340,9 +340,9 @@ static int do_mlock(unsigned long start, size_t len, int on)
 
                /* Here we know that  vma->vm_start <= nstart < vma->vm_end. */
 
-               newflags = vma->vm_flags | VM_LOCKED;
-               if (!on)
-                       newflags &= ~VM_LOCKED;
+               newflags = vma->vm_flags & ~VM_LOCKED;
+               if (on)
+                       newflags |= VM_LOCKED | VM_POPULATE;
 
                tmp = vma->vm_end;
                if (tmp > end)
@@ -402,7 +402,8 @@ int __mm_populate(unsigned long start, unsigned long len, int ignore_errors)
                 * range with the first VMA. Also, skip undesirable VMA types.
                 */
                nend = min(end, vma->vm_end);
-               if (vma->vm_flags & (VM_IO | VM_PFNMAP))
+               if ((vma->vm_flags & (VM_IO | VM_PFNMAP | VM_POPULATE)) !=
+                   VM_POPULATE)
                        continue;
                if (nstart < vma->vm_start)
                        nstart = vma->vm_start;
@@ -475,18 +476,18 @@ static int do_mlockall(int flags)
        struct vm_area_struct * vma, * prev = NULL;
 
        if (flags & MCL_FUTURE)
-               current->mm->def_flags |= VM_LOCKED;
+               current->mm->def_flags |= VM_LOCKED | VM_POPULATE;
        else
-               current->mm->def_flags &= ~VM_LOCKED;
+               current->mm->def_flags &= ~(VM_LOCKED | VM_POPULATE);
        if (flags == MCL_FUTURE)
                goto out;
 
        for (vma = current->mm->mmap; vma ; vma = prev->vm_next) {
                vm_flags_t newflags;
 
-               newflags = vma->vm_flags | VM_LOCKED;
-               if (!(flags & MCL_CURRENT))
-                       newflags &= ~VM_LOCKED;
+               newflags = vma->vm_flags & ~VM_LOCKED;
+               if (flags & MCL_CURRENT)
+                       newflags |= VM_LOCKED | VM_POPULATE;
 
                /* Ignore errors */
                mlock_fixup(vma, &prev, vma->vm_start, vma->vm_end, newflags);

diff --git a/mm/mmap.c b/mm/mmap.c
index 8826c77513a9dc172d4568799fb89e54f6969aca..39a3944e1658ad99519631833d1029c3b8e2247c 100644 (file)
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -1306,9 +1306,7 @@ unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
        }
 
        addr = mmap_region(file, addr, len, vm_flags, pgoff);
-       if (!IS_ERR_VALUE(addr) &&
-           ((vm_flags & VM_LOCKED) ||
-            (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE))
+       if (!IS_ERR_VALUE(addr) && (vm_flags & VM_POPULATE))
                *populate = true;
        return addr;
 }