rcu: Add RCU context switching to schedule_tail()

The new rcu_read_lock_nesting and rcu_read_unlock_special per-CPU
variables must be saved and restored at every context switch, including
those involving schedule_tail().  This commit therefore adds the saving
and restoring to schedul_tail().

Reported-by: Sasha Levin <levinsasha928@gmail.com>
Signed-off-by: Paul E. McKenney <paul.mckenney@linaro.org>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Tested-by: Sasha Levin <levinsasha928@gmail.com>

diff --git a/arch/um/drivers/mconsole_kern.c b/arch/um/drivers/mconsole_kern.c
index a173735046ee7c72e565f629ac1520f080ae07ce..18310b298339752a104438fbd4d70de351d38fff 100644 (file)
--- a/arch/um/drivers/mconsole_kern.c
+++ b/arch/um/drivers/mconsole_kern.c
@@ -705,7 +705,7 @@ static void stack_proc(void *arg)
        struct task_struct *from = current, *to = arg;
 
        to->thread.saved_task = from;
-       rcu_switch_from();
+       rcu_switch_from(from);
        switch_to(from, to, from);
        rcu_switch_to();
 }

diff --git a/include/linux/sched.h b/include/linux/sched.h
index f2468cb59e2bfd0f6d7d259d1e06e8db98047901..0d4860990ead957d79e5c2fa8fea3d93be7d57a1 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1937,10 +1937,8 @@ static inline void rcu_copy_process(struct task_struct *p)
  *
  * The caller must have disabled preemption.
  */
-static inline void rcu_switch_from(void)
+static inline void rcu_switch_from(struct task_struct *t)
 {
-       struct task_struct *t = current;
-
        if (__this_cpu_read(rcu_read_lock_nesting) != 0)
                rcu_preempt_note_context_switch();
        t->rcu_read_lock_nesting_save = __this_cpu_read(rcu_read_lock_nesting);
@@ -1991,7 +1989,7 @@ static inline void rcu_copy_process(struct task_struct *p)
 {
 }
 
-static inline void rcu_switch_from(void)
+static inline void rcu_switch_from(struct task_struct *t)
 {
 }
 

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 51ce5376dcd070c086916f8d92e534e5e184205c..17ae2671dba77ab9394f84f613b09e2b100ac706 100644 (file)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2024,6 +2024,8 @@ asmlinkage void schedule_tail(struct task_struct *prev)
 {
        struct rq *rq = this_rq();
 
+       rcu_switch_from(prev);
+       rcu_switch_to();
        finish_task_switch(rq, prev);
 
        /*
@@ -2083,7 +2085,7 @@ context_switch(struct rq *rq, struct task_struct *prev,
 #endif
 
        /* Here we just switch the register state and the stack. */
-       rcu_switch_from();
+       rcu_switch_from(current);
        switch_to(prev, next, prev);
        rcu_switch_to();