numa_maps: redefine callback functions for page table walker

gather_pte_stats() connected to pmd_entry() does both of pmd loop and pte
loop.  So this patch moves pte part into pte_entry().

Signed-off-by: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Matt Mackall <mpm@selenic.com>
Cc: Cliff Wickman <cpw@sgi.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Michal Hocko <mhocko@suse.cz>
Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com>
Cc: Pavel Emelyanov <xemul@parallels.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index e9b21aef452412b89eedfeeb5a14ee76e810906e..933e2469733eac07a2b73c7fda6cf7c3758fd7ee 100644 (file)
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -1194,7 +1194,6 @@ const struct file_operations proc_pagemap_operations = {
 #ifdef CONFIG_NUMA
 
 struct numa_maps {
-       struct vm_area_struct *vma;
        unsigned long pages;
        unsigned long anon;
        unsigned long active;
@@ -1260,43 +1259,41 @@ static struct page *can_gather_numa_stats(pte_t pte, struct vm_area_struct *vma,
        return page;
 }
 
-static int gather_pte_stats(pmd_t *pmd, unsigned long addr,
+static int gather_pte_stats(pte_t *pte, unsigned long addr,
                unsigned long end, struct mm_walk *walk)
 {
-       struct numa_maps *md;
-       spinlock_t *ptl;
-       pte_t *orig_pte;
-       pte_t *pte;
+       struct numa_maps *md = walk->private;
 
-       md = walk->private;
+       struct page *page = can_gather_numa_stats(*pte, walk->vma, addr);
+       if (!page)
+               return 0;
+       gather_stats(page, md, pte_dirty(*pte), 1);
+       return 0;
+}
+
+static int gather_pmd_stats(pmd_t *pmd, unsigned long addr,
+               unsigned long end, struct mm_walk *walk)
+{
+       struct numa_maps *md = walk->private;
+       struct vm_area_struct *vma = walk->vma;
+       spinlock_t *ptl;
 
-       if (pmd_trans_huge_lock(pmd, md->vma, &ptl) == 1) {
+       if (pmd_trans_huge_lock(pmd, vma, &ptl) == 1) {
                pte_t huge_pte = *(pte_t *)pmd;
                struct page *page;
 
-               page = can_gather_numa_stats(huge_pte, md->vma, addr);
+               page = can_gather_numa_stats(huge_pte, vma, addr);
                if (page)
                        gather_stats(page, md, pte_dirty(huge_pte),
                                     HPAGE_PMD_SIZE/PAGE_SIZE);
                spin_unlock(ptl);
-               return 0;
+               /* don't call gather_pte_stats() */
+               walk->skip = 1;
        }
-
-       if (pmd_trans_unstable(pmd))
-               return 0;
-       orig_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
-       do {
-               struct page *page = can_gather_numa_stats(*pte, md->vma, addr);
-               if (!page)
-                       continue;
-               gather_stats(page, md, pte_dirty(*pte), 1);
-
-       } while (pte++, addr += PAGE_SIZE, addr != end);
-       pte_unmap_unlock(orig_pte, ptl);
        return 0;
 }
 #ifdef CONFIG_HUGETLB_PAGE
-static int gather_hugetbl_stats(pte_t *pte, unsigned long hmask,
+static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask,
                unsigned long addr, unsigned long end, struct mm_walk *walk)
 {
        struct numa_maps *md;
@@ -1315,7 +1312,7 @@ static int gather_hugetbl_stats(pte_t *pte, unsigned long hmask,
 }
 
 #else
-static int gather_hugetbl_stats(pte_t *pte, unsigned long hmask,
+static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask,
                unsigned long addr, unsigned long end, struct mm_walk *walk)
 {
        return 0;
@@ -1345,12 +1342,12 @@ static int show_numa_map(struct seq_file *m, void *v, int is_pid)
        /* Ensure we start with an empty set of numa_maps statistics. */
        memset(md, 0, sizeof(*md));
 
-       md->vma = vma;
-
-       walk.hugetlb_entry = gather_hugetbl_stats;
-       walk.pmd_entry = gather_pte_stats;
+       walk.hugetlb_entry = gather_hugetlb_stats;
+       walk.pmd_entry = gather_pmd_stats;
+       walk.pte_entry = gather_pte_stats;
        walk.private = md;
        walk.mm = mm;
+       walk.vma = vma;
 
        pol = get_vma_policy(task, vma, vma->vm_start);
        mpol_to_str(buffer, sizeof(buffer), pol);
@@ -1381,6 +1378,7 @@ static int show_numa_map(struct seq_file *m, void *v, int is_pid)
        if (is_vm_hugetlb_page(vma))
                seq_printf(m, " huge");
 
+       /* mmap_sem is held by m_start */
        walk_page_range(vma->vm_start, vma->vm_end, &walk);
 
        if (!md->pages)