arm64: mm: Update perf accounting to handle poison faults

[karo-tx-linux.git] / arch / arm64 / mm / fault.c

diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c
index d73e7f1fe184173c4067d524099f196cf1712725..ea2ea68d1bd76223ba4afc5d71c2c746831895c0 100644 (file)
--- a/arch/arm64/mm/fault.c
+++ b/arch/arm64/mm/fault.c
@@ -359,7 +359,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
 {
        struct task_struct *tsk;
        struct mm_struct *mm;
-       int fault, sig, code;
+       int fault, sig, code, major = 0;
        unsigned long vm_flags = VM_READ | VM_WRITE;
        unsigned int mm_flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
 
@@ -398,6 +398,8 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
                        die("Accessing user space memory outside uaccess.h routines", regs, esr);
        }
 
+       perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);
+
        /*
         * As per x86, we may deadlock here. However, since the kernel only
         * validly references user space from well defined areas of the code,
@@ -421,24 +423,42 @@ retry:
        }
 
        fault = __do_page_fault(mm, addr, mm_flags, vm_flags, tsk);
+       major |= fault & VM_FAULT_MAJOR;
 
-       /*
-        * If we need to retry but a fatal signal is pending, handle the
-        * signal first. We do not need to release the mmap_sem because it
-        * would already be released in __lock_page_or_retry in mm/filemap.c.
-        */
-       if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
-               return 0;
+       if (fault & VM_FAULT_RETRY) {
+               /*
+                * If we need to retry but a fatal signal is pending,
+                * handle the signal first. We do not need to release
+                * the mmap_sem because it would already be released
+                * in __lock_page_or_retry in mm/filemap.c.
+                */
+               if (fatal_signal_pending(current))
+                       return 0;
+
+               /*
+                * Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk of
+                * starvation.
+                */
+               if (mm_flags & FAULT_FLAG_ALLOW_RETRY) {
+                       mm_flags &= ~FAULT_FLAG_ALLOW_RETRY;
+                       mm_flags |= FAULT_FLAG_TRIED;
+                       goto retry;
+               }
+       }
+       up_read(&mm->mmap_sem);
 
        /*
-        * Major/minor page fault accounting is only done on the initial
-        * attempt. If we go through a retry, it is extremely likely that the
-        * page will be found in page cache at that point.
+        * Handle the "normal" (no error) case first.
         */
-
-       perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);
-       if (mm_flags & FAULT_FLAG_ALLOW_RETRY) {
-               if (fault & VM_FAULT_MAJOR) {
+       if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP |
+                             VM_FAULT_BADACCESS)))) {
+               /*
+                * Major/minor page fault accounting is only done
+                * once. If we go through a retry, it is extremely
+                * likely that the page will be found in page cache at
+                * that point.
+                */
+               if (major) {
                        tsk->maj_flt++;
                        perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs,
                                      addr);
@@ -447,25 +467,9 @@ retry:
                        perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs,
                                      addr);
                }
-               if (fault & VM_FAULT_RETRY) {
-                       /*
-                        * Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk of
-                        * starvation.
-                        */
-                       mm_flags &= ~FAULT_FLAG_ALLOW_RETRY;
-                       mm_flags |= FAULT_FLAG_TRIED;
-                       goto retry;
-               }
-       }
-
-       up_read(&mm->mmap_sem);
 
-       /*
-        * Handle the "normal" case first - VM_FAULT_MAJOR
-        */
-       if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP |
-                             VM_FAULT_BADACCESS))))
                return 0;
+       }
 
        /*
         * If we are in kernel mode at this point, we have no context to