mm: correctly synchronize rss-counters at exit/exec

mm->rss_stat counters have per-task delta: task->rss_stat.  Before
changing task->mm pointer the kernel must flush this delta with
sync_mm_rss().

do_exit() already calls sync_mm_rss() to flush the rss-counters before
committing the rss statistics into task->signal->maxrss, taskstats, audit
and other stuff.  Unfortunately the kernel does this before calling
mm_release(), which can call put_user() for processing
task->clear_child_tid.  So at this point we can trigger page-faults and
task->rss_stat becomes non-zero again.  As a result mm->rss_stat becomes
inconsistent and check_mm() will print something like this:

| BUG: Bad rss-counter state mm:ffff88020813c380 idx:1 val:-1
| BUG: Bad rss-counter state mm:ffff88020813c380 idx:2 val:1

This patch moves sync_mm_rss() into mm_release(), and moves mm_release()
out of do_exit() and calls it earlier.  After mm_release() there should be
no pagefaults.

[akpm@linux-foundation.org: tweak comment]
Signed-off-by: Konstantin Khlebnikov <khlebnikov@openvz.org>
Reported-by: Markus Trippelsdorf <markus@trippelsdorf.de>
Cc: Hugh Dickins <hughd@google.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/fs/exec.c b/fs/exec.c
index d038968b54b420a3844c6e07f6bf25bee34d7e5c..39d839078a153fcbd3dac586c8927b8ede7769a5 100644 (file)
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -823,7 +823,6 @@ static int exec_mmap(struct mm_struct *mm)
        /* Notify parent that we're no longer interested in the old VM */
        tsk = current;
        old_mm = current->mm;
-       sync_mm_rss(old_mm);
        mm_release(tsk, old_mm);
 
        if (old_mm) {

diff --git a/kernel/exit.c b/kernel/exit.c
index d8bd3b425fa78413f94c9278f481672faebb999c..70875a6de8467273e84edfdd47a2cfe9be9890cd 100644 (file)
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -423,6 +423,7 @@ void daemonize(const char *name, ...)
         * user space pages.  We don't need them, and if we didn't close them
         * they would be locked into memory.
         */
+       mm_release(current, current->mm);
        exit_mm(current);
        /*
         * We don't want to get frozen, in case system-wide hibernation
@@ -640,7 +641,6 @@ static void exit_mm(struct task_struct * tsk)
        struct mm_struct *mm = tsk->mm;
        struct core_state *core_state;
 
-       mm_release(tsk, mm);
        if (!mm)
                return;
        /*
@@ -959,9 +959,13 @@ void do_exit(long code)
                                preempt_count());
 
        acct_update_integrals(tsk);
-       /* sync mm's RSS info before statistics gathering */
-       if (tsk->mm)
-               sync_mm_rss(tsk->mm);
+
+       /* Set exit_code before complete_vfork_done() in mm_release() */
+       tsk->exit_code = code;
+
+       /* Release mm and sync mm's RSS info before statistics gathering */
+       mm_release(tsk, tsk->mm);
+
        group_dead = atomic_dec_and_test(&tsk->signal->live);
        if (group_dead) {
                hrtimer_cancel(&tsk->signal->real_timer);
@@ -974,7 +978,6 @@ void do_exit(long code)
                tty_audit_exit();
        audit_free(tsk);
 
-       tsk->exit_code = code;
        taskstats_exit(tsk, group_dead);
 
        exit_mm(tsk);

diff --git a/kernel/fork.c b/kernel/fork.c
index 40a0903a877e9325f4ff1b0326cd6236066f19cc..2877339f06d5945ba14f89f96cb8b61777ecb80b 100644 (file)
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -783,6 +783,14 @@ void mm_release(struct task_struct *tsk, struct mm_struct *mm)
                }
                tsk->clear_child_tid = NULL;
        }
+
+       /*
+        * Final rss-counter synchronization. After this point there must be
+        * no pagefaults into this mm from the current context.  Otherwise
+        * mm->rss_stat will be inconsistent.
+        */
+       if (mm)
+               sync_mm_rss(mm);
 }
 
 /*