sched, numa, mm: Count WS scanning against present PTEs, not virtual memory ranges

By accounting against the present PTEs, scanning speed reflects the
actual present (mapped) memory.

Suggested-by: Ingo Molnar <mingo@kernel.org>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 151a3cda64750a7bed387274a3637cac1a14c53c..da28315777b7441872b4345ad8c644563d092699 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -914,8 +914,8 @@ void task_numa_work(struct callback_head *work)
        struct task_struct *p = current;
        struct mm_struct *mm = p->mm;
        struct vm_area_struct *vma;
-       unsigned long offset, end;
-       long length;
+       unsigned long start, end;
+       long pages;
 
        WARN_ON_ONCE(p != container_of(work, struct task_struct, numa_work));
 
@@ -942,30 +942,35 @@ void task_numa_work(struct callback_head *work)
        if (cmpxchg(&mm->numa_next_scan, migrate, next_scan) != migrate)
                return;
 
-       offset = mm->numa_scan_offset;
-       length = sysctl_sched_numa_scan_size;
-       length <<= 20;
+       start = mm->numa_scan_offset;
+       pages = sysctl_sched_numa_scan_size;
+       pages <<= 20 - PAGE_SHIFT; /* MB in pages */
+       if (!pages)
+               return;
 
        down_write(&mm->mmap_sem);
-       vma = find_vma(mm, offset);
+       vma = find_vma(mm, start);
        if (!vma) {
                ACCESS_ONCE(mm->numa_scan_seq)++;
-               offset = 0;
+               start = 0;
                vma = mm->mmap;
        }
-       for (; vma && length > 0; vma = vma->vm_next) {
+       for (; vma; vma = vma->vm_next) {
                if (!vma_migratable(vma))
                        continue;
 
-               offset = max(offset, vma->vm_start);
-               end = min(ALIGN(offset + length, HPAGE_SIZE), vma->vm_end);
-               length -= end - offset;
-
-               change_prot_numa(vma, offset, end);
-
-               offset = end;
+               do {
+                       start = max(start, vma->vm_start);
+                       end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE);
+                       end = min(end, vma->vm_end);
+                       pages -= change_prot_numa(vma, start, end);
+                       start = end;
+                       if (pages <= 0)
+                               goto out;
+               } while (end != vma->vm_end);
        }
-       mm->numa_scan_offset = offset;
+out:
+       mm->numa_scan_offset = start;
        up_write(&mm->mmap_sem);
 }