kernel/rcu/tiny.c

   1 /*
   2  * Read-Copy Update mechanism for mutual exclusion, the Bloatwatch edition.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, you can access it online at
  16  * http://www.gnu.org/licenses/gpl-2.0.html.
  17  *
  18  * Copyright IBM Corporation, 2008
  19  *
  20  * Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
  21  *
  22  * For detailed explanation of Read-Copy Update mechanism see -
  23  *              Documentation/RCU
  24  */
  25 #include <linux/completion.h>
  26 #include <linux/interrupt.h>
  27 #include <linux/notifier.h>
  28 #include <linux/rcupdate_wait.h>
  29 #include <linux/kernel.h>
  30 #include <linux/export.h>
  31 #include <linux/mutex.h>
  32 #include <linux/sched.h>
  33 #include <linux/types.h>
  34 #include <linux/init.h>
  35 #include <linux/time.h>
  36 #include <linux/cpu.h>
  37 #include <linux/prefetch.h>
  38 #include <linux/trace_events.h>
  39
  40 #include "rcu.h"
  41
  42 /* Forward declarations for tiny_plugin.h. */
  43 struct rcu_ctrlblk;
  44 static void __call_rcu(struct rcu_head *head,
  45                        rcu_callback_t func,
  46                        struct rcu_ctrlblk *rcp);
  47
  48 #include "tiny_plugin.h"
  49
  50 void rcu_barrier_bh(void)
  51 {
  52         wait_rcu_gp(call_rcu_bh);
  53 }
  54 EXPORT_SYMBOL(rcu_barrier_bh);
  55
  56 void rcu_barrier_sched(void)
  57 {
  58         wait_rcu_gp(call_rcu_sched);
  59 }
  60 EXPORT_SYMBOL(rcu_barrier_sched);
  61
  62 #if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE)
  63
  64 /*
  65  * Test whether RCU thinks that the current CPU is idle.
  66  */
  67 bool notrace __rcu_is_watching(void)
  68 {
  69         return true;
  70 }
  71 EXPORT_SYMBOL(__rcu_is_watching);
  72
  73 #endif /* defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) */
  74
  75 /*
  76  * Helper function for rcu_sched_qs() and rcu_bh_qs().
  77  * Also irqs are disabled to avoid confusion due to interrupt handlers
  78  * invoking call_rcu().
  79  */
  80 static int rcu_qsctr_help(struct rcu_ctrlblk *rcp)
  81 {
  82         RCU_TRACE(reset_cpu_stall_ticks(rcp);)
  83         if (rcp->donetail != rcp->curtail) {
  84                 rcp->donetail = rcp->curtail;
  85                 return 1;
  86         }
  87
  88         return 0;
  89 }
  90
  91 /*
  92  * Record an rcu quiescent state.  And an rcu_bh quiescent state while we
  93  * are at it, given that any rcu quiescent state is also an rcu_bh
  94  * quiescent state.  Use "+" instead of "||" to defeat short circuiting.
  95  */
  96 void rcu_sched_qs(void)
  97 {
  98         unsigned long flags;
  99
 100         local_irq_save(flags);
 101         if (rcu_qsctr_help(&rcu_sched_ctrlblk) +
 102             rcu_qsctr_help(&rcu_bh_ctrlblk))
 103                 raise_softirq(RCU_SOFTIRQ);
 104         local_irq_restore(flags);
 105 }
 106
 107 /*
 108  * Record an rcu_bh quiescent state.
 109  */
 110 void rcu_bh_qs(void)
 111 {
 112         unsigned long flags;
 113
 114         local_irq_save(flags);
 115         if (rcu_qsctr_help(&rcu_bh_ctrlblk))
 116                 raise_softirq(RCU_SOFTIRQ);
 117         local_irq_restore(flags);
 118 }
 119
 120 /*
 121  * Check to see if the scheduling-clock interrupt came from an extended
 122  * quiescent state, and, if so, tell RCU about it.  This function must
 123  * be called from hardirq context.  It is normally called from the
 124  * scheduling-clock interrupt.
 125  */
 126 void rcu_check_callbacks(int user)
 127 {
 128         RCU_TRACE(check_cpu_stalls();)
 129         if (user)
 130                 rcu_sched_qs();
 131         else if (!in_softirq())
 132                 rcu_bh_qs();
 133         if (user)
 134                 rcu_note_voluntary_context_switch(current);
 135 }
 136
 137 /*
 138  * Invoke the RCU callbacks on the specified rcu_ctrlkblk structure
 139  * whose grace period has elapsed.
 140  */
 141 static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp)
 142 {
 143         const char *rn = NULL;
 144         struct rcu_head *next, *list;
 145         unsigned long flags;
 146         RCU_TRACE(int cb_count = 0;)
 147
 148         /* Move the ready-to-invoke callbacks to a local list. */
 149         local_irq_save(flags);
 150         if (rcp->donetail == &rcp->rcucblist) {
 151                 /* No callbacks ready, so just leave. */
 152                 local_irq_restore(flags);
 153                 return;
 154         }
 155         RCU_TRACE(trace_rcu_batch_start(rcp->name, 0, rcp->qlen, -1);)
 156         list = rcp->rcucblist;
 157         rcp->rcucblist = *rcp->donetail;
 158         *rcp->donetail = NULL;
 159         if (rcp->curtail == rcp->donetail)
 160                 rcp->curtail = &rcp->rcucblist;
 161         rcp->donetail = &rcp->rcucblist;
 162         local_irq_restore(flags);
 163
 164         /* Invoke the callbacks on the local list. */
 165         RCU_TRACE(rn = rcp->name;)
 166         while (list) {
 167                 next = list->next;
 168                 prefetch(next);
 169                 debug_rcu_head_unqueue(list);
 170                 local_bh_disable();
 171                 __rcu_reclaim(rn, list);
 172                 local_bh_enable();
 173                 list = next;
 174                 RCU_TRACE(cb_count++;)
 175         }
 176         RCU_TRACE(rcu_trace_sub_qlen(rcp, cb_count);)
 177         RCU_TRACE(trace_rcu_batch_end(rcp->name,
 178                                       cb_count, 0, need_resched(),
 179                                       is_idle_task(current),
 180                                       false));
 181 }
 182
 183 static __latent_entropy void rcu_process_callbacks(struct softirq_action *unused)
 184 {
 185         __rcu_process_callbacks(&rcu_sched_ctrlblk);
 186         __rcu_process_callbacks(&rcu_bh_ctrlblk);
 187 }
 188
 189 /*
 190  * Wait for a grace period to elapse.  But it is illegal to invoke
 191  * synchronize_sched() from within an RCU read-side critical section.
 192  * Therefore, any legal call to synchronize_sched() is a quiescent
 193  * state, and so on a UP system, synchronize_sched() need do nothing.
 194  * Ditto for synchronize_rcu_bh().  (But Lai Jiangshan points out the
 195  * benefits of doing might_sleep() to reduce latency.)
 196  *
 197  * Cool, huh?  (Due to Josh Triplett.)
 198  */
 199 void synchronize_sched(void)
 200 {
 201         RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) ||
 202                          lock_is_held(&rcu_lock_map) ||
 203                          lock_is_held(&rcu_sched_lock_map),
 204                          "Illegal synchronize_sched() in RCU read-side critical section");
 205 }
 206 EXPORT_SYMBOL_GPL(synchronize_sched);
 207
 208 /*
 209  * Helper function for call_rcu() and call_rcu_bh().
 210  */
 211 static void __call_rcu(struct rcu_head *head,
 212                        rcu_callback_t func,
 213                        struct rcu_ctrlblk *rcp)
 214 {
 215         unsigned long flags;
 216
 217         debug_rcu_head_queue(head);
 218         head->func = func;
 219         head->next = NULL;
 220
 221         local_irq_save(flags);
 222         *rcp->curtail = head;
 223         rcp->curtail = &head->next;
 224         RCU_TRACE(rcp->qlen++;)
 225         local_irq_restore(flags);
 226
 227         if (unlikely(is_idle_task(current))) {
 228                 /* force scheduling for rcu_sched_qs() */
 229                 resched_cpu(0);
 230         }
 231 }
 232
 233 /*
 234  * Post an RCU callback to be invoked after the end of an RCU-sched grace
 235  * period.  But since we have but one CPU, that would be after any
 236  * quiescent state.
 237  */
 238 void call_rcu_sched(struct rcu_head *head, rcu_callback_t func)
 239 {
 240         __call_rcu(head, func, &rcu_sched_ctrlblk);
 241 }
 242 EXPORT_SYMBOL_GPL(call_rcu_sched);
 243
 244 /*
 245  * Post an RCU bottom-half callback to be invoked after any subsequent
 246  * quiescent state.
 247  */
 248 void call_rcu_bh(struct rcu_head *head, rcu_callback_t func)
 249 {
 250         __call_rcu(head, func, &rcu_bh_ctrlblk);
 251 }
 252 EXPORT_SYMBOL_GPL(call_rcu_bh);
 253
 254 void __init rcu_init(void)
 255 {
 256         open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
 257         RCU_TRACE(reset_cpu_stall_ticks(&rcu_sched_ctrlblk);)
 258         RCU_TRACE(reset_cpu_stall_ticks(&rcu_bh_ctrlblk);)
 259
 260         rcu_early_boot_tests();
 261 }