sparc64: Add 64K page size support

[karo-tx-linux.git] / fs / userfaultfd.c

diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c
index 5139d05f80e6dd99a15d9299d45ffa01736f21cd..18406158e13fbf5e9b4e7041489f2d20e404c075 100644 (file)
--- a/fs/userfaultfd.c
+++ b/fs/userfaultfd.c
@@ -202,6 +202,49 @@ static inline struct uffd_msg userfault_msg(unsigned long address,
        return msg;
 }
 
+#ifdef CONFIG_HUGETLB_PAGE
+/*
+ * Same functionality as userfaultfd_must_wait below with modifications for
+ * hugepmd ranges.
+ */
+static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx,
+                                        unsigned long address,
+                                        unsigned long flags,
+                                        unsigned long reason)
+{
+       struct mm_struct *mm = ctx->mm;
+       pte_t *pte;
+       bool ret = true;
+
+       VM_BUG_ON(!rwsem_is_locked(&mm->mmap_sem));
+
+       pte = huge_pte_offset(mm, address);
+       if (!pte)
+               goto out;
+
+       ret = false;
+
+       /*
+        * Lockless access: we're in a wait_event so it's ok if it
+        * changes under us.
+        */
+       if (huge_pte_none(*pte))
+               ret = true;
+       if (!huge_pte_write(*pte) && (reason & VM_UFFD_WP))
+               ret = true;
+out:
+       return ret;
+}
+#else
+static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx,
+                                        unsigned long address,
+                                        unsigned long flags,
+                                        unsigned long reason)
+{
+       return false;   /* should never get here */
+}
+#endif /* CONFIG_HUGETLB_PAGE */
+
 /*
  * Verify the pagetables are still not ok after having reigstered into
  * the fault_pending_wqh to avoid userland having to UFFDIO_WAKE any
@@ -378,8 +421,12 @@ int handle_userfault(struct vm_fault *vmf, unsigned long reason)
        set_current_state(blocking_state);
        spin_unlock(&ctx->fault_pending_wqh.lock);
 
-       must_wait = userfaultfd_must_wait(ctx, vmf->address, vmf->flags,
-                                         reason);
+       if (!is_vm_hugetlb_page(vmf->vma))
+               must_wait = userfaultfd_must_wait(ctx, vmf->address, vmf->flags,
+                                                 reason);
+       else
+               must_wait = userfaultfd_huge_must_wait(ctx, vmf->address,
+                                                      vmf->flags, reason);
        up_read(&mm->mmap_sem);
 
        if (likely(must_wait && !ACCESS_ONCE(ctx->released) &&
@@ -1049,6 +1096,12 @@ static __always_inline int validate_range(struct mm_struct *mm,
        return 0;
 }
 
+static inline bool vma_can_userfault(struct vm_area_struct *vma)
+{
+       return vma_is_anonymous(vma) || is_vm_hugetlb_page(vma) ||
+               vma_is_shmem(vma);
+}
+
 static int userfaultfd_register(struct userfaultfd_ctx *ctx,
                                unsigned long arg)
 {
@@ -1059,7 +1112,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
        struct uffdio_register __user *user_uffdio_register;
        unsigned long vm_flags, new_flags;
        bool found;
-       bool huge_pages;
+       bool non_anon_pages;
        unsigned long start, end, vma_end;
 
        user_uffdio_register = (struct uffdio_register __user *) arg;
@@ -1123,13 +1176,9 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
 
        /*
         * Search for not compatible vmas.
-        *
-        * FIXME: this shall be relaxed later so that it doesn't fail
-        * on tmpfs backed vmas (in addition to the current allowance
-        * on anonymous vmas).
         */
        found = false;
-       huge_pages = false;
+       non_anon_pages = false;
        for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) {
                cond_resched();
 
@@ -1138,7 +1187,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
 
                /* check not compatible vmas */
                ret = -EINVAL;
-               if (!vma_is_anonymous(cur) && !is_vm_hugetlb_page(cur))
+               if (!vma_can_userfault(cur))
                        goto out_unlock;
                /*
                 * If this vma contains ending address, and huge pages
@@ -1168,8 +1217,8 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
                /*
                 * Note vmas containing huge pages
                 */
-               if (is_vm_hugetlb_page(cur))
-                       huge_pages = true;
+               if (is_vm_hugetlb_page(cur) || vma_is_shmem(cur))
+                       non_anon_pages = true;
 
                found = true;
        }
@@ -1182,7 +1231,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
        do {
                cond_resched();
 
-               BUG_ON(!vma_is_anonymous(vma) && !is_vm_hugetlb_page(vma));
+               BUG_ON(!vma_can_userfault(vma));
                BUG_ON(vma->vm_userfaultfd_ctx.ctx &&
                       vma->vm_userfaultfd_ctx.ctx != ctx);
 
@@ -1240,7 +1289,7 @@ out_unlock:
                 * userland which ioctls methods are guaranteed to
                 * succeed on this range.
                 */
-               if (put_user(huge_pages ? UFFD_API_RANGE_IOCTLS_HPAGE :
+               if (put_user(non_anon_pages ? UFFD_API_RANGE_IOCTLS_BASIC :
                             UFFD_API_RANGE_IOCTLS,
                             &user_uffdio_register->ioctls))
                        ret = -EFAULT;
@@ -1300,10 +1349,6 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
 
        /*
         * Search for not compatible vmas.
-        *
-        * FIXME: this shall be relaxed later so that it doesn't fail
-        * on tmpfs backed vmas (in addition to the current allowance
-        * on anonymous vmas).
         */
        found = false;
        ret = -EINVAL;
@@ -1320,7 +1365,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
                 * provides for more strict behavior to notice
                 * unregistration errors.
                 */
-               if (!vma_is_anonymous(cur) && !is_vm_hugetlb_page(cur))
+               if (!vma_can_userfault(cur))
                        goto out_unlock;
 
                found = true;
@@ -1334,7 +1379,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
        do {
                cond_resched();
 
-               BUG_ON(!vma_is_anonymous(vma) && !is_vm_hugetlb_page(vma));
+               BUG_ON(!vma_can_userfault(vma));
 
                /*
                 * Nothing to do: this vma is already registered into this