rcu: Move code to apply callback-numbering simplifications

The addition of callback numbering allows combining the detection of the
ends of old grace periods and the beginnings of new grace periods.  This
commit moves code to set the stage for this combining.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>

diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index 1009c0ccd4b186ebaf66740ac0f864b29e246b0f..c36e52dc091d432dfb0325b157f4c125a05aa739 100644 (file)
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -984,65 +984,6 @@ void rcu_cpu_stall_reset(void)
                rsp->jiffies_stall = jiffies + ULONG_MAX / 2;
 }
 
-/*
- * Update CPU-local rcu_data state to record the newly noticed grace period.
- * This is used both when we started the grace period and when we notice
- * that someone else started the grace period.  The caller must hold the
- * ->lock of the leaf rcu_node structure corresponding to the current CPU,
- *  and must have irqs disabled.
- */
-static void __note_new_gpnum(struct rcu_state *rsp, struct rcu_node *rnp, struct rcu_data *rdp)
-{
-       if (rdp->gpnum != rnp->gpnum) {
-               /*
-                * If the current grace period is waiting for this CPU,
-                * set up to detect a quiescent state, otherwise don't
-                * go looking for one.
-                */
-               rdp->gpnum = rnp->gpnum;
-               trace_rcu_grace_period(rsp->name, rdp->gpnum, "cpustart");
-               rdp->passed_quiesce = 0;
-               rdp->qs_pending = !!(rnp->qsmask & rdp->grpmask);
-               zero_cpu_stall_ticks(rdp);
-       }
-}
-
-static void note_new_gpnum(struct rcu_state *rsp, struct rcu_data *rdp)
-{
-       unsigned long flags;
-       struct rcu_node *rnp;
-
-       local_irq_save(flags);
-       rnp = rdp->mynode;
-       if (rdp->gpnum == ACCESS_ONCE(rnp->gpnum) || /* outside lock. */
-           !raw_spin_trylock(&rnp->lock)) { /* irqs already off, so later. */
-               local_irq_restore(flags);
-               return;
-       }
-       __note_new_gpnum(rsp, rnp, rdp);
-       raw_spin_unlock_irqrestore(&rnp->lock, flags);
-}
-
-/*
- * Did someone else start a new RCU grace period start since we last
- * checked?  Update local state appropriately if so.  Must be called
- * on the CPU corresponding to rdp.
- */
-static int
-check_for_new_grace_period(struct rcu_state *rsp, struct rcu_data *rdp)
-{
-       unsigned long flags;
-       int ret = 0;
-
-       local_irq_save(flags);
-       if (rdp->gpnum != rsp->gpnum) {
-               note_new_gpnum(rsp, rdp);
-               ret = 1;
-       }
-       local_irq_restore(flags);
-       return ret;
-}
-
 /*
  * Initialize the specified rcu_data structure's callback list to empty.
  */
@@ -1359,6 +1300,45 @@ __rcu_process_gp_end(struct rcu_state *rsp, struct rcu_node *rnp, struct rcu_dat
        }
 }
 
+/*
+ * Update CPU-local rcu_data state to record the newly noticed grace period.
+ * This is used both when we started the grace period and when we notice
+ * that someone else started the grace period.  The caller must hold the
+ * ->lock of the leaf rcu_node structure corresponding to the current CPU,
+ *  and must have irqs disabled.
+ */
+static void __note_new_gpnum(struct rcu_state *rsp, struct rcu_node *rnp, struct rcu_data *rdp)
+{
+       if (rdp->gpnum != rnp->gpnum) {
+               /*
+                * If the current grace period is waiting for this CPU,
+                * set up to detect a quiescent state, otherwise don't
+                * go looking for one.
+                */
+               rdp->gpnum = rnp->gpnum;
+               trace_rcu_grace_period(rsp->name, rdp->gpnum, "cpustart");
+               rdp->passed_quiesce = 0;
+               rdp->qs_pending = !!(rnp->qsmask & rdp->grpmask);
+               zero_cpu_stall_ticks(rdp);
+       }
+}
+
+static void note_new_gpnum(struct rcu_state *rsp, struct rcu_data *rdp)
+{
+       unsigned long flags;
+       struct rcu_node *rnp;
+
+       local_irq_save(flags);
+       rnp = rdp->mynode;
+       if (rdp->gpnum == ACCESS_ONCE(rnp->gpnum) || /* outside lock. */
+           !raw_spin_trylock(&rnp->lock)) { /* irqs already off, so later. */
+               local_irq_restore(flags);
+               return;
+       }
+       __note_new_gpnum(rsp, rnp, rdp);
+       raw_spin_unlock_irqrestore(&rnp->lock, flags);
+}
+
 /*
  * Advance this CPU's callbacks, but only if the current grace period
  * has ended.  This may be called only from the CPU to whom the rdp
@@ -1381,6 +1361,26 @@ rcu_process_gp_end(struct rcu_state *rsp, struct rcu_data *rdp)
        raw_spin_unlock_irqrestore(&rnp->lock, flags);
 }
 
+/*
+ * Did someone else start a new RCU grace period start since we last
+ * checked?  Update local state appropriately if so.  Must be called
+ * on the CPU corresponding to rdp.
+ */
+static int
+check_for_new_grace_period(struct rcu_state *rsp, struct rcu_data *rdp)
+{
+       unsigned long flags;
+       int ret = 0;
+
+       local_irq_save(flags);
+       if (rdp->gpnum != rsp->gpnum) {
+               note_new_gpnum(rsp, rdp);
+               ret = 1;
+       }
+       local_irq_restore(flags);
+       return ret;
+}
+
 /*
  * Do per-CPU grace-period initialization for running CPU.  The caller
  * must hold the lock of the leaf rcu_node structure corresponding to