Reinstate ZERO_PAGE optimization in 'get_user_pages()' and fix XIP

commit 89f5b7da2a6bad2e84670422ab8192382a5aeb9f upstream

KAMEZAWA Hiroyuki and Oleg Nesterov point out that since the commit
557ed1fa2620dc119adb86b34c614e152a629a80 ("remove ZERO_PAGE") removed
the ZERO_PAGE from the VM mappings, any users of get_user_pages() will
generally now populate the VM with real empty pages needlessly.

We used to get the ZERO_PAGE when we did the "handle_mm_fault()", but
since fault handling no longer uses ZERO_PAGE for new anonymous pages,
we now need to handle that special case in follow_page() instead.

In particular, the removal of ZERO_PAGE effectively removed the core
file writing optimization where we would skip writing pages that had not
been populated at all, and increased memory pressure a lot by allocating
all those useless newly zeroed pages.

This reinstates the optimization by making the unmapped PTE case the
same as for a non-existent page table, which already did this correctly.

While at it, this also fixes the XIP case for follow_page(), where the
caller could not differentiate between the case of a page that simply
could not be used (because it had no "struct page" associated with it)
and a page that just wasn't mapped.

We do that by simply returning an error pointer for pages that could not
be turned into a "struct page *".  The error is arbitrarily picked to be
EFAULT, since that was what get_user_pages() already used for the
equivalent IO-mapped page case.

[ Also removed an impossible test for pte_offset_map_lock() failing:
  that's not how that function works ]

Acked-by: Oleg Nesterov <oleg@tv-sign.ru>
Acked-by: Nick Piggin <npiggin@suse.de>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Hugh Dickins <hugh@veritas.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Roland McGrath <roland@redhat.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Greg Kroah-Hartman <gregkh@suse.de>

diff --git a/arch/powerpc/kernel/vdso.c b/arch/powerpc/kernel/vdso.c
index d3437c4c4a6f9b81f99020353dceaff248a54a06..e3b2185521f5cff1be1c9b06b6c098ad0d12541e 100644 (file)
--- a/arch/powerpc/kernel/vdso.c
+++ b/arch/powerpc/kernel/vdso.c
@@ -141,7 +141,7 @@ static void dump_one_vdso_page(struct page *pg, struct page *upg)
        printk("kpg: %p (c:%d,f:%08lx)", __va(page_to_pfn(pg) << PAGE_SHIFT),
               page_count(pg),
               pg->flags);
-       if (upg/* && pg != upg*/) {
+       if (upg && !IS_ERR(upg) /* && pg != upg*/) {
                printk(" upg: %p (c:%d,f:%08lx)", __va(page_to_pfn(upg)
                                                       << PAGE_SHIFT),
                       page_count(upg),

diff --git a/mm/memory.c b/mm/memory.c
index 0d14d1e58a5fa78e6b6a01369cc1e0869c77dfe9..db6590c8cc11adc70a62e678d14f887b8ad60fda 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -943,17 +943,15 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
        }
 
        ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
-       if (!ptep)
-               goto out;
 
        pte = *ptep;
        if (!pte_present(pte))
-               goto unlock;
+               goto no_page;
        if ((flags & FOLL_WRITE) && !pte_write(pte))
                goto unlock;
        page = vm_normal_page(vma, address, pte);
        if (unlikely(!page))
-               goto unlock;
+               goto bad_page;
 
        if (flags & FOLL_GET)
                get_page(page);
@@ -968,6 +966,15 @@ unlock:
 out:
        return page;
 
+bad_page:
+       pte_unmap_unlock(ptep, ptl);
+       return ERR_PTR(-EFAULT);
+
+no_page:
+       pte_unmap_unlock(ptep, ptl);
+       if (!pte_none(pte))
+               return page;
+       /* Fall through to ZERO_PAGE handling */
 no_page_table:
        /*
         * When core dumping an enormous anonymous area that nobody
@@ -1104,6 +1111,8 @@ int get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
 
                                cond_resched();
                        }
+                       if (IS_ERR(page))
+                               return i ? i : PTR_ERR(page);
                        if (pages) {
                                pages[i] = page;
 

diff --git a/mm/migrate.c b/mm/migrate.c
index 4e0eccca5e265ac19bc507a171a2f720d27f8c21..6acc81577ea6be189e7bed2fb78fe31ac241a2f5 100644 (file)
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -858,6 +858,11 @@ static int do_move_pages(struct mm_struct *mm, struct page_to_node *pm,
                        goto set_status;
 
                page = follow_page(vma, pp->addr, FOLL_GET);
+
+               err = PTR_ERR(page);
+               if (IS_ERR(page))
+                       goto set_status;
+
                err = -ENOENT;
                if (!page)
                        goto set_status;
@@ -921,6 +926,11 @@ static int do_pages_stat(struct mm_struct *mm, struct page_to_node *pm)
                        goto set_status;
 
                page = follow_page(vma, pm->addr, 0);
+
+               err = PTR_ERR(page);
+               if (IS_ERR(page))
+                       goto set_status;
+
                err = -ENOENT;
                /* Use PageReserved to check for zero page */
                if (!page || PageReserved(page))