Merge tag 'imx-fixes-rc' of git://git.pengutronix.de/git/imx/linux-2.6 into fixes

[karo-tx-linux.git] / arch / s390 / mm / pgtable.c

diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c
index b402991e43d71614cc111918a6b03a4af8525b2e..c8188a18af05b565e57304b02e153ed4629940d5 100644 (file)
--- a/arch/s390/mm/pgtable.c
+++ b/arch/s390/mm/pgtable.c
@@ -787,6 +787,30 @@ void tlb_remove_table(struct mmu_gather *tlb, void *table)
                tlb_table_flush(tlb);
 }
 
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+void thp_split_vma(struct vm_area_struct *vma)
+{
+       unsigned long addr;
+       struct page *page;
+
+       for (addr = vma->vm_start; addr < vma->vm_end; addr += PAGE_SIZE) {
+               page = follow_page(vma, addr, FOLL_SPLIT);
+       }
+}
+
+void thp_split_mm(struct mm_struct *mm)
+{
+       struct vm_area_struct *vma = mm->mmap;
+
+       while (vma != NULL) {
+               thp_split_vma(vma);
+               vma->vm_flags &= ~VM_HUGEPAGE;
+               vma->vm_flags |= VM_NOHUGEPAGE;
+               vma = vma->vm_next;
+       }
+}
+#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
+
 /*
  * switch on pgstes for its userspace process (for kvm)
  */
@@ -824,6 +848,12 @@ int s390_enable_sie(void)
        if (!mm)
                return -ENOMEM;
 
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+       /* split thp mappings and disable thp for future mappings */
+       thp_split_mm(mm);
+       mm->def_flags |= VM_NOHUGEPAGE;
+#endif
+
        /* Now lets check again if something happened */
        task_lock(tsk);
        if (!tsk->mm || atomic_read(&tsk->mm->mm_users) > 1 ||
@@ -866,3 +896,81 @@ bool kernel_page_present(struct page *page)
        return cc == 0;
 }
 #endif /* CONFIG_HIBERNATION && CONFIG_DEBUG_PAGEALLOC */
+
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+int pmdp_clear_flush_young(struct vm_area_struct *vma, unsigned long address,
+                          pmd_t *pmdp)
+{
+       VM_BUG_ON(address & ~HPAGE_PMD_MASK);
+       /* No need to flush TLB
+        * On s390 reference bits are in storage key and never in TLB */
+       return pmdp_test_and_clear_young(vma, address, pmdp);
+}
+
+int pmdp_set_access_flags(struct vm_area_struct *vma,
+                         unsigned long address, pmd_t *pmdp,
+                         pmd_t entry, int dirty)
+{
+       VM_BUG_ON(address & ~HPAGE_PMD_MASK);
+
+       if (pmd_same(*pmdp, entry))
+               return 0;
+       pmdp_invalidate(vma, address, pmdp);
+       set_pmd_at(vma->vm_mm, address, pmdp, entry);
+       return 1;
+}
+
+static void pmdp_splitting_flush_sync(void *arg)
+{
+       /* Simply deliver the interrupt */
+}
+
+void pmdp_splitting_flush(struct vm_area_struct *vma, unsigned long address,
+                         pmd_t *pmdp)
+{
+       VM_BUG_ON(address & ~HPAGE_PMD_MASK);
+       if (!test_and_set_bit(_SEGMENT_ENTRY_SPLIT_BIT,
+                             (unsigned long *) pmdp)) {
+               /* need to serialize against gup-fast (IRQ disabled) */
+               smp_call_function(pmdp_splitting_flush_sync, NULL, 1);
+       }
+}
+
+void pgtable_trans_huge_deposit(struct mm_struct *mm, pgtable_t pgtable)
+{
+       struct list_head *lh = (struct list_head *) pgtable;
+
+       assert_spin_locked(&mm->page_table_lock);
+
+       /* FIFO */
+       if (!mm->pmd_huge_pte)
+               INIT_LIST_HEAD(lh);
+       else
+               list_add(lh, (struct list_head *) mm->pmd_huge_pte);
+       mm->pmd_huge_pte = pgtable;
+}
+
+pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm)
+{
+       struct list_head *lh;
+       pgtable_t pgtable;
+       pte_t *ptep;
+
+       assert_spin_locked(&mm->page_table_lock);
+
+       /* FIFO */
+       pgtable = mm->pmd_huge_pte;
+       lh = (struct list_head *) pgtable;
+       if (list_empty(lh))
+               mm->pmd_huge_pte = NULL;
+       else {
+               mm->pmd_huge_pte = (pgtable_t) lh->next;
+               list_del(lh);
+       }
+       ptep = (pte_t *) pgtable;
+       pte_val(*ptep) = _PAGE_TYPE_EMPTY;
+       ptep++;
+       pte_val(*ptep) = _PAGE_TYPE_EMPTY;
+       return pgtable;
+}
+#endif /* CONFIG_TRANSPARENT_HUGEPAGE */