Merge branch 'numa/core'

diff --git a/include/linux/sched.h b/include/linux/sched.h
index c86db44109ab61161bf943f936e82a0c8a64eb4f..788268619552ad212161e0a9689ca70a3d94ef80 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1529,6 +1529,7 @@ struct task_struct {
        u64 node_stamp;                 /* migration stamp  */
        unsigned long numa_contrib;
        unsigned long *numa_faults;
+       struct callback_head numa_work;
 #endif /* CONFIG_SCHED_NUMA */
 
        struct rcu_head rcu;

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 23ad8b94aeb2fef96493f91b9a8225c8f0bb206c..08661fe39a4f9373e5ea4e2217ca4cee20dd0842 100644 (file)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -1546,6 +1546,7 @@ static void __sched_fork(struct task_struct *p)
        p->numa_migrate_seq = p->mm ? p->mm->numa_scan_seq - 1 : 0;
        p->numa_faults = NULL;
        p->numa_task_period = sysctl_sched_numa_task_period_min;
+       p->numa_work.next = &p->numa_work;
 #endif /* CONFIG_SCHED_NUMA */
 }
 

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 00fd74cdcf8f7cf998d2ad06fd85f1e24ca26266..2679bcfec7310410cd09d1b04bcb531399d753ec 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -900,8 +900,9 @@ void task_numa_work(struct callback_head *work)
        struct task_struct *p = current;
        struct mm_struct *mm = p->mm;
 
-       WARN_ON_ONCE(p != container_of(work, struct task_struct, rcu));
+       WARN_ON_ONCE(p != container_of(work, struct task_struct, numa_work));
 
+       work->next = work; /* protect against double add */
        /*
         * Who cares about NUMA placement when they're dying.
         *
@@ -933,12 +934,13 @@ void task_numa_work(struct callback_head *work)
  */
 void task_tick_numa(struct rq *rq, struct task_struct *curr)
 {
+       struct callback_head *work = &curr->numa_work;
        u64 period, now;
 
        /*
         * We don't care about NUMA placement if we don't have memory.
         */
-       if (!curr->mm)
+       if (!curr->mm || (curr->flags & PF_EXITING) || work->next != work)
                return;
 
        /*
@@ -954,14 +956,8 @@ void task_tick_numa(struct rq *rq, struct task_struct *curr)
                curr->node_stamp = now;
 
                if (!time_before(jiffies, curr->mm->numa_next_scan)) {
-                       /*
-                        * We can re-use curr->rcu because we checked curr->mm
-                        * != NULL so release_task()->call_rcu() was not called
-                        * yet and exit_task_work() is called before
-                        * exit_notify().
-                        */
-                       init_task_work(&curr->rcu, task_numa_work);
-                       task_work_add(curr, &curr->rcu, true);
+                       init_task_work(work, task_numa_work); /* TODO: move this into sched_fork() */
+                       task_work_add(curr, work, true);
                }
        }
 }