userfaultfd: hugetlbfs: gup: support VM_FAULT_RETRY

[karo-tx-linux.git] / mm / hugetlb.c

diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index a4b29054cc3f683d0b0329bf8c66ea48da0e22c3..f6c7ff316daf5a7451b2bd3daef648d6f0b8491b 100644 (file)
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -4065,7 +4065,7 @@ out_release_unlock:
 long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
                         struct page **pages, struct vm_area_struct **vmas,
                         unsigned long *position, unsigned long *nr_pages,
-                        long i, unsigned int flags)
+                        long i, unsigned int flags, int *nonblocking)
 {
        unsigned long pfn_offset;
        unsigned long vaddr = *position;
@@ -4128,16 +4128,43 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
                    ((flags & FOLL_WRITE) &&
                      !huge_pte_write(huge_ptep_get(pte)))) {
                        int ret;
+                       unsigned int fault_flags = 0;
 
                        if (pte)
                                spin_unlock(ptl);
-                       ret = hugetlb_fault(mm, vma, vaddr,
-                               (flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0);
-                       if (!(ret & VM_FAULT_ERROR))
-                               continue;
-
-                       remainder = 0;
-                       break;
+                       if (flags & FOLL_WRITE)
+                               fault_flags |= FAULT_FLAG_WRITE;
+                       if (nonblocking)
+                               fault_flags |= FAULT_FLAG_ALLOW_RETRY;
+                       if (flags & FOLL_NOWAIT)
+                               fault_flags |= FAULT_FLAG_ALLOW_RETRY |
+                                       FAULT_FLAG_RETRY_NOWAIT;
+                       if (flags & FOLL_TRIED) {
+                               VM_WARN_ON_ONCE(fault_flags &
+                                               FAULT_FLAG_ALLOW_RETRY);
+                               fault_flags |= FAULT_FLAG_TRIED;
+                       }
+                       ret = hugetlb_fault(mm, vma, vaddr, fault_flags);
+                       if (ret & VM_FAULT_ERROR) {
+                               remainder = 0;
+                               break;
+                       }
+                       if (ret & VM_FAULT_RETRY) {
+                               if (nonblocking)
+                                       *nonblocking = 0;
+                               *nr_pages = 0;
+                               /*
+                                * VM_FAULT_RETRY must not return an
+                                * error, it will return zero
+                                * instead.
+                                *
+                                * No need to update "position" as the
+                                * caller will not check it after
+                                * *nr_pages is set to 0.
+                                */
+                               return i;
+                       }
+                       continue;
                }
 
                pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
@@ -4166,6 +4193,11 @@ same_page:
                spin_unlock(ptl);
        }
        *nr_pages = remainder;
+       /*
+        * setting position is actually required only if remainder is
+        * not zero but it's faster not to add a "if (remainder)"
+        * branch.
+        */
        *position = vaddr;
 
        return i ? i : -EFAULT;