rcu: Abstract the dynticks momentary-idle operation

This commit is the first step towards full abstraction of all accesses to
the ->dynticks counter, implementing the previously open-coded atomic add
of two in a new rcu_dynticks_momentary_idle() function.  This abstraction
will ease changes to the ->dynticks counter operation.

Note that this commit gets rid of the smp_mb__before_atomic() and the
smp_mb__after_atomic() calls that were previously present.  The reason
that this is OK from a memory-ordering perspective is that the atomic
operation is now atomic_add_return(), which, as a value-returning atomic,
guarantees full ordering.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index cb4e2056ccf3cf799bb7c045aca346fedb2ed698..14e283c351f6be238d4d3504304f75980f6421c9 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -281,6 +281,19 @@ static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
 #endif /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
 };
 
+/*
+ * Do a double-increment of the ->dynticks counter to emulate a
+ * momentary idle-CPU quiescent state.
+ */
+static void rcu_dynticks_momentary_idle(void)
+{
+       struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+       int special = atomic_add_return(2, &rdtp->dynticks);
+
+       /* It is illegal to call this from idle state. */
+       WARN_ON_ONCE(!(special & 0x1));
+}
+
 DEFINE_PER_CPU_SHARED_ALIGNED(unsigned long, rcu_qs_ctr);
 EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr);
 
@@ -300,7 +313,6 @@ EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr);
 static void rcu_momentary_dyntick_idle(void)
 {
        struct rcu_data *rdp;
-       struct rcu_dynticks *rdtp;
        int resched_mask;
        struct rcu_state *rsp;
 
@@ -327,10 +339,7 @@ static void rcu_momentary_dyntick_idle(void)
                 * quiescent state, with no need for this CPU to do anything
                 * further.
                 */
-               rdtp = this_cpu_ptr(&rcu_dynticks);
-               smp_mb__before_atomic(); /* Earlier stuff before QS. */
-               atomic_add(2, &rdtp->dynticks);  /* QS. */
-               smp_mb__after_atomic(); /* Later stuff after QS. */
+               rcu_dynticks_momentary_idle();
                break;
        }
 }