rcu: Move early-boot callbacks to no-CBs lists for no-CBs CPUs

When a CPU is first determined to be a no-CBs CPUs, this commit causes
any early boot callbacks to be moved to the no-CBs callback list,
allowing them to be invoked.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 92fd3eab5823df8dfd160401ec710471a89c5fec..0317bf7d997f4b3065aa397fa7a03889cb09dc96 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -2851,6 +2851,7 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),
                 * and then drop through to queue the callback.
                 */
                BUG_ON(cpu != -1);
+               WARN_ON_ONCE(!rcu_is_watching());
                if (!likely(rdp->nxtlist))
                        init_default_callback_list(rdp);
        }

diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 75d5f096bcb00add431fc886f5153ef6db76d69d..afddd5641bea7212e24fdd206b161ed04f56c2df 100644 (file)
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -2393,18 +2393,8 @@ void __init rcu_init_nohz(void)
                pr_info("\tPoll for callbacks from no-CBs CPUs.\n");
 
        for_each_rcu_flavor(rsp) {
-               for_each_cpu(cpu, rcu_nocb_mask) {
-                       struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
-
-                       /*
-                        * If there are early callbacks, they will need
-                        * to be moved to the nocb lists.
-                        */
-                       WARN_ON_ONCE(rdp->nxttail[RCU_NEXT_TAIL] !=
-                                    &rdp->nxtlist &&
-                                    rdp->nxttail[RCU_NEXT_TAIL] != NULL);
-                       init_nocb_callback_list(rdp);
-               }
+               for_each_cpu(cpu, rcu_nocb_mask)
+                       init_nocb_callback_list(per_cpu_ptr(rsp->rda, cpu));
                rcu_organize_nocb_kthreads(rsp);
        }
 }
@@ -2541,6 +2531,16 @@ static bool init_nocb_callback_list(struct rcu_data *rdp)
        if (!rcu_is_nocb_cpu(rdp->cpu))
                return false;
 
+       /* If there are early-boot callbacks, move them to nocb lists. */
+       if (rdp->nxtlist) {
+               rdp->nocb_head = rdp->nxtlist;
+               rdp->nocb_tail = rdp->nxttail[RCU_NEXT_TAIL];
+               atomic_long_set(&rdp->nocb_q_count, rdp->qlen);
+               atomic_long_set(&rdp->nocb_q_count_lazy, rdp->qlen_lazy);
+               rdp->nxtlist = NULL;
+               rdp->qlen = 0;
+               rdp->qlen_lazy = 0;
+       }
        rdp->nxttail[RCU_NEXT_TAIL] = NULL;
        return true;
 }