rcu: Eliminate read-modify-write ACCESS_ONCE() calls

RCU contains code of the following forms:

	ACCESS_ONCE(x)++;
	ACCESS_ONCE(x) += y;
	ACCESS_ONCE(x) -= y;

Now these constructs do operate correctly, but they really result in a
pair of volatile accesses, one to do the load and another to do the store.
This can be confusing, as the casual reader might well assume that (for
example) gcc might generate a memory-to-memory add instruction for each
of these three cases.  In fact, gcc will do no such thing.  Also, there
is a good chance that the kernel will move to separate load and store
variants of ACCESS_ONCE(), and constructs like the above could easily
confuse both people and scripts attempting to make that sort of change.
Finally, most of RCU's read-modify-write uses of ACCESS_ONCE() really
only need the store to be volatile, so that the read-modify-write form
might be misleading.

This commit therefore changes the above forms in RCU so that each instance
of ACCESS_ONCE() either does a load or a store, but not both.  In a few
cases, ACCESS_ONCE() was not critical, for example, for maintaining
statisitics.  In these cases, ACCESS_ONCE() has been dispensed with
entirely.

Suggested-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

diff --git a/kernel/rcu/srcu.c b/kernel/rcu/srcu.c
index c639556f3fa06234dcc34517826fbf0420641e56..e037f3eb2f7bf2f44a6de08cdcfe89b05d4d5bca 100644 (file)
--- a/kernel/rcu/srcu.c
+++ b/kernel/rcu/srcu.c
@@ -298,9 +298,9 @@ int __srcu_read_lock(struct srcu_struct *sp)
 
        idx = ACCESS_ONCE(sp->completed) & 0x1;
        preempt_disable();
-       ACCESS_ONCE(this_cpu_ptr(sp->per_cpu_ref)->c[idx]) += 1;
+       __this_cpu_inc(sp->per_cpu_ref->c[idx]);
        smp_mb(); /* B */  /* Avoid leaking the critical section. */
-       ACCESS_ONCE(this_cpu_ptr(sp->per_cpu_ref)->seq[idx]) += 1;
+       __this_cpu_inc(sp->per_cpu_ref->seq[idx]);
        preempt_enable();
        return idx;
 }

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 987fd64f70dc1ecd3a95180f7b5b7611498c5d34..c5f8284169a8cbccf09ad3653050e8ecbc75004d 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -2356,7 +2356,7 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp)
        }
        smp_mb(); /* List handling before counting for rcu_barrier(). */
        rdp->qlen_lazy -= count_lazy;
-       ACCESS_ONCE(rdp->qlen) -= count;
+       ACCESS_ONCE(rdp->qlen) = rdp->qlen - count;
        rdp->n_cbs_invoked += count;
 
        /* Reinstate batch limit if we have worked down the excess. */
@@ -2501,7 +2501,7 @@ static void force_quiescent_state(struct rcu_state *rsp)
                if (rnp_old != NULL)
                        raw_spin_unlock(&rnp_old->fqslock);
                if (ret) {
-                       ACCESS_ONCE(rsp->n_force_qs_lh)++;
+                       rsp->n_force_qs_lh++;
                        return;
                }
                rnp_old = rnp;
@@ -2513,7 +2513,7 @@ static void force_quiescent_state(struct rcu_state *rsp)
        smp_mb__after_unlock_lock();
        raw_spin_unlock(&rnp_old->fqslock);
        if (ACCESS_ONCE(rsp->gp_flags) & RCU_GP_FLAG_FQS) {
-               ACCESS_ONCE(rsp->n_force_qs_lh)++;
+               rsp->n_force_qs_lh++;
                raw_spin_unlock_irqrestore(&rnp_old->lock, flags);
                return;  /* Someone beat us to it. */
        }
@@ -2702,7 +2702,7 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),
                local_irq_restore(flags);
                return;
        }
-       ACCESS_ONCE(rdp->qlen)++;
+       ACCESS_ONCE(rdp->qlen) = rdp->qlen + 1;
        if (lazy)
                rdp->qlen_lazy++;
        else
@@ -3266,7 +3266,7 @@ static void _rcu_barrier(struct rcu_state *rsp)
         * ACCESS_ONCE() to prevent the compiler from speculating
         * the increment to precede the early-exit check.
         */
-       ACCESS_ONCE(rsp->n_barrier_done)++;
+       ACCESS_ONCE(rsp->n_barrier_done) = rsp->n_barrier_done + 1;
        WARN_ON_ONCE((rsp->n_barrier_done & 0x1) != 1);
        _rcu_barrier_trace(rsp, "Inc1", -1, rsp->n_barrier_done);
        smp_mb(); /* Order ->n_barrier_done increment with below mechanism. */
@@ -3316,7 +3316,7 @@ static void _rcu_barrier(struct rcu_state *rsp)
 
        /* Increment ->n_barrier_done to prevent duplicate work. */
        smp_mb(); /* Keep increment after above mechanism. */
-       ACCESS_ONCE(rsp->n_barrier_done)++;
+       ACCESS_ONCE(rsp->n_barrier_done) = rsp->n_barrier_done + 1;
        WARN_ON_ONCE((rsp->n_barrier_done & 0x1) != 0);
        _rcu_barrier_trace(rsp, "Inc2", -1, rsp->n_barrier_done);
        smp_mb(); /* Keep increment before caller's subsequent code. */

diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 55bbf0fd4f7eaadc4891ea226c1fb0bc7aa079f4..bc6895996a779afc596f1c4b726a9ea856748e17 100644 (file)
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -2274,8 +2274,8 @@ static int rcu_nocb_kthread(void *arg)
                tail = xchg(&rdp->nocb_tail, &rdp->nocb_head);
                c = atomic_long_xchg(&rdp->nocb_q_count, 0);
                cl = atomic_long_xchg(&rdp->nocb_q_count_lazy, 0);
-               ACCESS_ONCE(rdp->nocb_p_count) += c;
-               ACCESS_ONCE(rdp->nocb_p_count_lazy) += cl;
+               rdp->nocb_p_count += c;
+               rdp->nocb_p_count_lazy += cl;
                rcu_nocb_wait_gp(rdp);
 
                /* Each pass through the following loop invokes a callback. */