From caf5b32a9fe73165cb9e1db8a27f34d2ad9d62e6 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 4 Aug 2011 01:22:27 -0700 Subject: [PATCH] rcu: Add memory barriers for busted atomic implementations Some arches fail to provide full memory barrier on atomics that return values. Long term, these need to be fixed, but in the meantime allow testing to proceed. Signed-off-by: Paul E. McKenney --- kernel/rcutree.c | 4 ++++ kernel/rcutree_plugin.h | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/kernel/rcutree.c b/kernel/rcutree.c index a07bf553e02a..710b1ab5a7b1 100644 --- a/kernel/rcutree.c +++ b/kernel/rcutree.c @@ -476,7 +476,9 @@ void rcu_irq_exit(void) */ static int dyntick_save_progress_counter(struct rcu_data *rdp) { + smp_mb(); /* Work around some architectures weak impls. */ rdp->dynticks_snap = atomic_add_return(0, &rdp->dynticks->dynticks); + smp_mb(); /* Work around some architectures weak impls. */ return 0; } @@ -491,7 +493,9 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) unsigned int curr; unsigned int snap; + smp_mb(); /* Work around some architectures weak impls. */ curr = (unsigned int)atomic_add_return(0, &rdp->dynticks->dynticks); + smp_mb(); /* Work around some architectures weak impls. */ snap = (unsigned int)rdp->dynticks_snap; /* diff --git a/kernel/rcutree_plugin.h b/kernel/rcutree_plugin.h index ed70f6bf4c31..ba42ce887a1e 100644 --- a/kernel/rcutree_plugin.h +++ b/kernel/rcutree_plugin.h @@ -1615,7 +1615,9 @@ static int rcu_node_kthread(void *arg) rcu_wait(atomic_read(&rnp->wakemask) != 0); rnp->node_kthread_status = RCU_KTHREAD_RUNNING; raw_spin_lock_irqsave(&rnp->lock, flags); + smp_mb(); /* Work around some architectures weak impls. */ mask = atomic_xchg(&rnp->wakemask, 0); + smp_mb(); /* Work around some architectures weak impls. */ rcu_initiate_boost(rnp, flags); /* releases rnp->lock. */ for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++, mask >>= 1) { if ((mask & 0x1) == 0) @@ -1851,7 +1853,9 @@ void synchronize_sched_expedited(void) int firstsnap, s, snap, trycount = 0; /* Note that atomic_inc_return() implies full memory barrier. */ + smp_mb(); /* Work around some architectures weak impls. */ firstsnap = snap = atomic_inc_return(&sync_sched_expedited_started); + smp_mb(); /* Work around some architectures weak impls. */ get_online_cpus(); /* @@ -1904,6 +1908,7 @@ void synchronize_sched_expedited(void) break; } } while (atomic_cmpxchg(&sync_sched_expedited_done, s, snap) != s); + smp_mb(); put_online_cpus(); } @@ -1974,9 +1979,10 @@ int rcu_needs_cpu(int cpu) for_each_online_cpu(thatcpu) { if (thatcpu == cpu) continue; + smp_mb(); /* Work around some architectures weak impls. */ snap = atomic_add_return(0, &per_cpu(rcu_dynticks, thatcpu).dynticks); - smp_mb(); /* Order sampling of snap with end of grace period. */ + smp_mb(); /* Work around some architectures weak impls. */ if ((snap & 0x1) != 0) { per_cpu(rcu_dyntick_drain, cpu) = 0; per_cpu(rcu_dyntick_holdoff, cpu) = jiffies - 1; -- 2.39.5