thp: change split_huge_page_pmd() interface

Pass vma instead of mm and add address parameter.

In most cases we already have vma on the stack. We provides
split_huge_page_pmd_mm() for few cases when we have mm, but not vma.

This change is preparation to huge zero pmd splitting implementation.

Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: "H. Peter Anvin" <hpa@linux.intel.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/Documentation/vm/transhuge.txt b/Documentation/vm/transhuge.txt
index f734bb2a78dc797aa62a2d2b0e1e7cc3390fef87..677a599be43094e856ded2c4838fad52ef5d8159 100644 (file)
--- a/Documentation/vm/transhuge.txt
+++ b/Documentation/vm/transhuge.txt
@@ -276,7 +276,7 @@ unaffected. libhugetlbfs will also work fine as usual.
 == Graceful fallback ==
 
 Code walking pagetables but unware about huge pmds can simply call
-split_huge_page_pmd(mm, pmd) where the pmd is the one returned by
+split_huge_page_pmd(vma, pmd, addr) where the pmd is the one returned by
 pmd_offset. It's trivial to make the code transparent hugepage aware
 by just grepping for "pmd_offset" and adding split_huge_page_pmd where
 missing after pmd_offset returns the pmd. Thanks to the graceful
@@ -299,7 +299,7 @@ diff --git a/mm/mremap.c b/mm/mremap.c
                return NULL;
 
        pmd = pmd_offset(pud, addr);
-+      split_huge_page_pmd(mm, pmd);
++      split_huge_page_pmd(vma, pmd, addr);
        if (pmd_none_or_clear_bad(pmd))
                return NULL;
 

diff --git a/arch/x86/kernel/vm86_32.c b/arch/x86/kernel/vm86_32.c
index 5c9687b1bde6706293e199331e7c866f1b752095..1dfe69cc78a81b90cc7d1edefc6d8e53f51c38c8 100644 (file)
--- a/arch/x86/kernel/vm86_32.c
+++ b/arch/x86/kernel/vm86_32.c
@@ -182,7 +182,7 @@ static void mark_screen_rdonly(struct mm_struct *mm)
        if (pud_none_or_clear_bad(pud))
                goto out;
        pmd = pmd_offset(pud, 0xA0000);
-       split_huge_page_pmd(mm, pmd);
+       split_huge_page_pmd_mm(mm, 0xA0000, pmd);
        if (pmd_none_or_clear_bad(pmd))
                goto out;
        pte = pte_offset_map_lock(mm, pmd, 0xA0000, &ptl);

diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index 90c63f9392a595318c5d1e03e1d69305bf9413fa..291a0d15a0be48fdc4792885cde2d9211e9acbef 100644 (file)
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -643,7 +643,7 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
        spinlock_t *ptl;
        struct page *page;
 
-       split_huge_page_pmd(walk->mm, pmd);
+       split_huge_page_pmd(vma, addr, pmd);
        if (pmd_trans_unstable(pmd))
                return 0;
 

diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index 110b005a2586414c17b50c05f27abf7e434fa609..5d50f6583085748a00d6a206eb55074a045cf4a4 100644 (file)
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -95,12 +95,14 @@ extern int handle_pte_fault(struct mm_struct *mm,
                            struct vm_area_struct *vma, unsigned long address,
                            pte_t *pte, pmd_t *pmd, unsigned int flags);
 extern int split_huge_page(struct page *page);
-extern void __split_huge_page_pmd(struct mm_struct *mm, pmd_t *pmd);
-#define split_huge_page_pmd(__mm, __pmd)                               \
+extern void __split_huge_page_pmd(struct vm_area_struct *vma,
+               unsigned long address, pmd_t *pmd);
+#define split_huge_page_pmd(__vma, __address, __pmd)                   \
        do {                                                            \
                pmd_t *____pmd = (__pmd);                               \
                if (unlikely(pmd_trans_huge(*____pmd)))                 \
-                       __split_huge_page_pmd(__mm, ____pmd);           \
+                       __split_huge_page_pmd(__vma, __address,         \
+                                       ____pmd);                       \
        }  while (0)
 #define wait_split_huge_page(__anon_vma, __pmd)                                \
        do {                                                            \
@@ -110,6 +112,8 @@ extern void __split_huge_page_pmd(struct mm_struct *mm, pmd_t *pmd);
                BUG_ON(pmd_trans_splitting(*____pmd) ||                 \
                       pmd_trans_huge(*____pmd));                       \
        } while (0)
+extern void split_huge_page_pmd_mm(struct mm_struct *mm, unsigned long address,
+               pmd_t *pmd);
 #if HPAGE_PMD_ORDER > MAX_ORDER
 #error "hugepages can't be allocated by the buddy allocator"
 #endif
@@ -177,10 +181,12 @@ static inline int split_huge_page(struct page *page)
 {
        return 0;
 }
-#define split_huge_page_pmd(__mm, __pmd)       \
+#define split_huge_page_pmd(__vma, __address, __pmd)   \
        do { } while (0)
 #define wait_split_huge_page(__anon_vma, __pmd)        \
        do { } while (0)
+#define split_huge_page_pmd_mm(__mm, __address, __pmd) \
+       do { } while (0)
 #define compound_trans_head(page) compound_head(page)
 static inline int hugepage_madvise(struct vm_area_struct *vma,
                                   unsigned long *vm_flags, int advice)

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 12dc04046b9f6f2360893d0da3faacfdb257a335..53292fb1334f4ddbf32de5b20d14b2a979f0810f 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -2639,19 +2639,23 @@ static int khugepaged(void *none)
        return 0;
 }
 
-void __split_huge_page_pmd(struct mm_struct *mm, pmd_t *pmd)
+void __split_huge_page_pmd(struct vm_area_struct *vma, unsigned long address,
+               pmd_t *pmd)
 {
        struct page *page;
+       unsigned long haddr = address & HPAGE_PMD_MASK;
 
-       spin_lock(&mm->page_table_lock);
+       BUG_ON(vma->vm_start > haddr || vma->vm_end < haddr + HPAGE_PMD_SIZE);
+
+       spin_lock(&vma->vm_mm->page_table_lock);
        if (unlikely(!pmd_trans_huge(*pmd))) {
-               spin_unlock(&mm->page_table_lock);
+               spin_unlock(&vma->vm_mm->page_table_lock);
                return;
        }
        page = pmd_page(*pmd);
        VM_BUG_ON(!page_count(page));
        get_page(page);
-       spin_unlock(&mm->page_table_lock);
+       spin_unlock(&vma->vm_mm->page_table_lock);
 
        split_huge_page(page);
 
@@ -2659,6 +2663,16 @@ void __split_huge_page_pmd(struct mm_struct *mm, pmd_t *pmd)
        BUG_ON(pmd_trans_huge(*pmd));
 }
 
+void split_huge_page_pmd_mm(struct mm_struct *mm, unsigned long address,
+               pmd_t *pmd)
+{
+       struct vm_area_struct *vma;
+
+       vma = find_vma(mm, address);
+       BUG_ON(vma == NULL);
+       split_huge_page_pmd(vma, address, pmd);
+}
+
 static void split_huge_page_address(struct mm_struct *mm,
                                    unsigned long address)
 {
@@ -2673,7 +2687,7 @@ static void split_huge_page_address(struct mm_struct *mm,
         * Caller holds the mmap_sem write mode, so a huge pmd cannot
         * materialize from under us.
         */
-       split_huge_page_pmd(mm, pmd);
+       split_huge_page_pmd_mm(mm, address, pmd);
 }
 
 void __vma_adjust_trans_huge(struct vm_area_struct *vma,

diff --git a/mm/memory.c b/mm/memory.c
index 6e28a89ab5f3328c6539549e85672a7c05e72898..528ae530d2fd02656553a8961cb698dd6cf7bfc7 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -1250,7 +1250,7 @@ static inline unsigned long zap_pmd_range(struct mmu_gather *tlb,
                                        BUG();
                                }
 #endif
-                               split_huge_page_pmd(vma->vm_mm, pmd);
+                               split_huge_page_pmd(vma, addr, pmd);
                        } else if (zap_huge_pmd(tlb, vma, pmd, addr))
                                goto next;
                        /* fall through */
@@ -1521,7 +1521,7 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
                goto no_page_table;
        if (pmd_trans_huge(*pmd)) {
                if (flags & FOLL_SPLIT) {
-                       split_huge_page_pmd(mm, pmd);
+                       split_huge_page_pmd(vma, address, pmd);
                        goto split_fallthrough;
                }
                spin_lock(&mm->page_table_lock);

diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 5ad2bd9495109c37a1c9e66aca4df737edcbd4ea..54a698dcbf2b7b80e9a39dd04f835b0257e49a8d 100644 (file)
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -529,7 +529,7 @@ static inline int check_pmd_range(struct vm_area_struct *vma, pud_t *pud,
        pmd = pmd_offset(pud, addr);
        do {
                next = pmd_addr_end(addr, end);
-               split_huge_page_pmd(vma->vm_mm, pmd);
+               split_huge_page_pmd(vma, addr, pmd);
                if (pmd_none_or_trans_huge_or_clear_bad(pmd))
                        continue;
                if (check_pte_range(vma, pmd, addr, next, nodes,

diff --git a/mm/mprotect.c b/mm/mprotect.c
index 6ff2d5e2ce3b6964092b11b27f80ccf024411559..d74a6fbc4b5eed2badf3554e5c52eb769f2d5bdb 100644 (file)
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -89,7 +89,7 @@ static inline unsigned long change_pmd_range(struct vm_area_struct *vma, pud_t *
                next = pmd_addr_end(addr, end);
                if (pmd_trans_huge(*pmd)) {
                        if (next - addr != HPAGE_PMD_SIZE)
-                               split_huge_page_pmd(vma->vm_mm, pmd);
+                               split_huge_page_pmd(vma, addr, pmd);
                        else if (change_huge_pmd(vma, pmd, addr, newprot)) {
                                pages += HPAGE_PMD_NR;
                                continue;

diff --git a/mm/mremap.c b/mm/mremap.c
index 1b61c2d3307a800c9c958e5f949d81fb76515899..eabb24da6c9e1bf00ee64075626630331ca94039 100644 (file)
--- a/mm/mremap.c
+++ b/mm/mremap.c
@@ -182,7 +182,7 @@ unsigned long move_page_tables(struct vm_area_struct *vma,
                                need_flush = true;
                                continue;
                        } else if (!err) {
-                               split_huge_page_pmd(vma->vm_mm, old_pmd);
+                               split_huge_page_pmd(vma, old_addr, old_pmd);
                        }
                        VM_BUG_ON(pmd_trans_huge(*old_pmd));
                }

diff --git a/mm/pagewalk.c b/mm/pagewalk.c
index 6c118d012bb5a27be54ab331a4491e3ec421ac31..35aa294656cd812d2773fede0d51179a8b7ee09a 100644 (file)
--- a/mm/pagewalk.c
+++ b/mm/pagewalk.c
@@ -58,7 +58,7 @@ again:
                if (!walk->pte_entry)
                        continue;
 
-               split_huge_page_pmd(walk->mm, pmd);
+               split_huge_page_pmd_mm(walk->mm, addr, pmd);
                if (pmd_none_or_trans_huge_or_clear_bad(pmd))
                        goto again;
                err = walk_pte_range(pmd, addr, next, walk);