struct task_struct *from = current, *to = arg;
to->thread.saved_task = from;
- rcu_switch_from(from);
+ rcu_switch_from();
switch_to(from, to, from);
+ rcu_switch_to();
}
/*
#endif
#ifdef CONFIG_PREEMPT_RCU
#define INIT_TASK_RCU_PREEMPT(tsk) \
- .rcu_read_lock_nesting = 0, \
- .rcu_read_unlock_special = 0, \
+ .rcu_read_lock_nesting_save = 0, \
+ .rcu_read_unlock_special_save = 0, \
.rcu_node_entry = LIST_HEAD_INIT(tsk.rcu_node_entry), \
INIT_TASK_RCU_TREE_PREEMPT() \
INIT_TASK_RCU_BOOST()
DECLARE_PER_CPU(int, rcu_read_lock_nesting);
DECLARE_PER_CPU(int, rcu_read_unlock_special);
+#ifdef CONFIG_PROVE_RCU
+DECLARE_PER_CPU(struct task_struct *, rcu_current_task);
+#endif /* #ifdef CONFIG_PROVE_RCU */
extern void __rcu_read_lock(void);
extern void __rcu_read_unlock(void);
* nesting depth, but makes sense only if CONFIG_PREEMPT_RCU -- in other
* types of kernel builds, the rcu_read_lock() nesting depth is unknowable.
*/
-#define rcu_preempt_depth() (current->rcu_read_lock_nesting)
+#define rcu_preempt_depth() (__this_cpu_read(rcu_read_lock_nesting))
+
+/*
+ * Check for a running RCU reader on the current CPU. If used from
+ * TINY_PREEMPT_RCU, works globally, as there can be but one running
+ * RCU reader at a time in that case. ;-)
+ *
+ * Returns zero if there are no running readers. Returns a positive
+ * number if there is at least one reader within its RCU read-side
+ * critical section. Returns a negative number if an outermost reader
+ * is in the midst of exiting from its RCU read-side critical section
+ *
+ * This differs from rcu_preempt_depth() in throwing a build error
+ * if used from under !CONFIG_PREEMPT_RCU.
+ */
+#define rcu_preempt_running_reader() (__this_cpu_read(rcu_read_lock_nesting))
#else /* #ifdef CONFIG_PREEMPT_RCU */
cpumask_t cpus_allowed;
#ifdef CONFIG_PREEMPT_RCU
- int rcu_read_lock_nesting;
- char rcu_read_unlock_special;
+ int rcu_read_lock_nesting_save;
+ char rcu_read_unlock_special_save;
struct list_head rcu_node_entry;
#endif /* #ifdef CONFIG_PREEMPT_RCU */
#ifdef CONFIG_TREE_PREEMPT_RCU
static inline void rcu_copy_process(struct task_struct *p)
{
- p->rcu_read_lock_nesting = 0;
- p->rcu_read_unlock_special = 0;
+ p->rcu_read_lock_nesting_save = 0;
+ p->rcu_read_unlock_special_save = 0;
#ifdef CONFIG_TREE_PREEMPT_RCU
p->rcu_blocked_node = NULL;
#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */
INIT_LIST_HEAD(&p->rcu_node_entry);
}
-static inline void rcu_switch_from(struct task_struct *prev)
+/*
+ * Let preemptible RCU know about a switch away from a task.
+ *
+ * First, if there is an ongoing RCU read-side critical section, invoke
+ * rcu_preempt_note_context_switch() to enqueue the task.
+ *
+ * We need to save both of the __rcu_read_lock() / __rcu_read_unlock()
+ * per-CPU variables in either case so that they can be correctly restored
+ * when this task resumes.
+ *
+ * Subsequent RCU read-side critical sections can occur either in the
+ * architecture switch_to() function or in interrupt handlers. Both
+ * cases rely on the fact that if rcu_read_unlock_special is zero, then
+ * rcu_read_lock_nesting must also be zero. Thus, the only way that such
+ * a critical section can enter rcu_read_unlock_special() is if:
+ *
+ * 1. The per-CPU rcu_read_lock_nesting variable was zero, and:
+ * 2. The critical section took a scheduling-clock interrupt,
+ * setting the RCU_READ_UNLOCK_NEED_QS bit.
+ *
+ * This is harmless. It will cause the CPU to register a quiescent state,
+ * which is OK because the CPU really is in a quiescent state and if the
+ * outgoing task was not in a quiescent state, it has already been queued.
+ * In particular, RCU_READ_UNLOCK_BLOCKED cannot be set because preemption
+ * is disabled.
+ *
+ * If rcu_read_lock_nesting is non-zero, then any subsequent RCU read-side
+ * critical section will act as if nested, thus refusing to enter the
+ * rcu_read_unlock_special() function in the first place.
+ *
+ * The caller must have disabled preemption.
+ */
+static inline void rcu_switch_from(void)
{
- if (prev->rcu_read_lock_nesting != 0)
+ struct task_struct *t = current;
+
+ if (__this_cpu_read(rcu_read_lock_nesting) != 0)
rcu_preempt_note_context_switch();
+ t->rcu_read_lock_nesting_save = __this_cpu_read(rcu_read_lock_nesting);
+ t->rcu_read_unlock_special_save =
+ __this_cpu_read(rcu_read_unlock_special);
+#ifdef CONFIG_PROVE_RCU
+ barrier();
+ /* Idle tasks can have NULL rcu_current_task at boot time. */
+ BUG_ON(__this_cpu_read(rcu_current_task) != t &&
+ __this_cpu_read(rcu_current_task) != NULL);
+ __this_cpu_write(rcu_current_task, NULL);
+#endif /* #ifdef CONFIG_PROVE_RCU */
+}
+
+/*
+ * Let preemptible RCU know about a switch to a task.
+ *
+ * This requires some care in order to preserve the invariant
+ * mentioned above. First zero rcu_read_unlock_special, then
+ * restore the value of rcu_read_lock_nesting, and only then restore
+ * rcu_read_unlock_special_save. (Yes, there is a check in
+ * rcu_read_unlock_special() that is supposed to prevent interrupt
+ * handlers from getting to far into that function, but this check
+ * are unavoidably heuristic in nature.)
+ *
+ * The caller must have disabled preemption.
+ */
+static inline void rcu_switch_to(void)
+{
+ struct task_struct *t = current;
+
+ __this_cpu_write(rcu_read_unlock_special, 0);
+ barrier(); /* Ensure ordering to maintain invariant. */
+ __this_cpu_write(rcu_read_lock_nesting, t->rcu_read_lock_nesting_save);
+ barrier(); /* Ensure ordering to maintain invariant. */
+ __this_cpu_write(rcu_read_unlock_special,
+ t->rcu_read_unlock_special_save);
+#ifdef CONFIG_PROVE_RCU
+ barrier();
+ BUG_ON(__this_cpu_read(rcu_current_task) != NULL);
+ __this_cpu_write(rcu_current_task, t);
+#endif /* #ifdef CONFIG_PROVE_RCU */
}
#else
{
}
-static inline void rcu_switch_from(struct task_struct *prev)
+static inline void rcu_switch_from(void)
+{
+}
+
+static inline void rcu_switch_to(void)
{
}
}
}
+#ifdef CONFIG_PREEMPT_RCU
+extern void rcu_read_unlock_do_special(void);
+#endif /* #ifdef CONFIG_PREEMPT_RCU */
+
#endif /* __LINUX_RCU_H */
#ifdef CONFIG_PREEMPT_RCU
DEFINE_PER_CPU(int, rcu_read_lock_nesting);
DEFINE_PER_CPU(int, rcu_read_unlock_special);
+#ifdef CONFIG_PROVE_RCU
+DEFINE_PER_CPU(struct task_struct *, rcu_current_task);
+#endif /* #ifdef CONFIG_PROVE_RCU */
+
+/*
+ * Preemptible-RCU implementation for rcu_read_lock(). Just increment
+ * the per-CPU rcu_read_lock_nesting: Shared state and per-task state will
+ * be updated if we block.
+ */
+void __rcu_read_lock(void)
+{
+ __this_cpu_inc(rcu_read_lock_nesting);
+ barrier(); /* Keep code within RCU read-side critical section. */
+}
+EXPORT_SYMBOL_GPL(__rcu_read_lock);
+
+/*
+ * Tree-preemptible RCU implementation for rcu_read_unlock().
+ * Decrement rcu_read_lock_nesting. If the result is zero (outermost
+ * rcu_read_unlock()) and rcu_read_unlock_special is non-zero, then
+ * invoke rcu_read_unlock_do_special() to clean up after a context switch
+ * in an RCU read-side critical section and other special cases.
+ * Set rcu_read_lock_nesting to a large negative value during cleanup
+ * in order to ensure that if rcu_read_unlock_special is non-zero, then
+ * rcu_read_lock_nesting is also non-zero.
+ */
+void __rcu_read_unlock(void)
+{
+ if (__this_cpu_read(rcu_read_lock_nesting) != 1)
+ __this_cpu_dec(rcu_read_lock_nesting);
+ else {
+ barrier(); /* critical section before exit code. */
+ __this_cpu_write(rcu_read_lock_nesting, INT_MIN);
+ barrier(); /* assign before ->rcu_read_unlock_special load */
+ if (unlikely(__this_cpu_read(rcu_read_unlock_special)))
+ rcu_read_unlock_do_special();
+ barrier(); /* ->rcu_read_unlock_special load before assign */
+ __this_cpu_write(rcu_read_lock_nesting, 0);
+ }
+#ifdef CONFIG_PROVE_LOCKING
+ {
+ int rln = __this_cpu_read(rcu_read_lock_nesting);
+
+ WARN_ON_ONCE(rln < 0 && rln > INT_MIN / 2);
+ }
+#endif /* #ifdef CONFIG_PROVE_LOCKING */
+}
+EXPORT_SYMBOL_GPL(__rcu_read_unlock);
/*
* Check for a task exiting while in a preemptible-RCU read-side
*/
void exit_rcu(void)
{
- struct task_struct *t = current;
-
if (likely(list_empty(¤t->rcu_node_entry)))
return;
- t->rcu_read_lock_nesting = 1;
+ __this_cpu_write(rcu_read_lock_nesting, 1);
barrier();
- t->rcu_read_unlock_special = RCU_READ_UNLOCK_BLOCKED;
+ __this_cpu_write(rcu_read_unlock_special, RCU_READ_UNLOCK_BLOCKED);
__rcu_read_unlock();
}
RCU_TRACE(.rcb.name = "rcu_preempt")
};
-static void rcu_read_unlock_do_special(struct task_struct *t);
static int rcu_preempted_readers_exp(void);
static void rcu_report_exp_done(void);
return rcu_preempt_ctrlblk.gpcpu != rcu_preempt_ctrlblk.gpnum;
}
-/*
- * Check for a running RCU reader. Because there is only one CPU,
- * there can be but one running RCU reader at a time. ;-)
- *
- * Returns zero if there are no running readers. Returns a positive
- * number if there is at least one reader within its RCU read-side
- * critical section. Returns a negative number if an outermost reader
- * is in the midst of exiting from its RCU read-side critical section
- *
- * Returns zero if there are no running readers. Returns a positive
- * number if there is at least one reader within its RCU read-side
- * critical section. Returns a negative number if an outermost reader
- * is in the midst of exiting from its RCU read-side critical section.
- */
-static int rcu_preempt_running_reader(void)
-{
- return current->rcu_read_lock_nesting;
-}
-
/*
* Check for preempted RCU readers blocking any grace period.
* If the caller needs a reliable answer, it must disable hard irqs.
*
* Unlike the other rcu_*_qs() functions, callers to this function
* must disable irqs in order to protect the assignment to
- * ->rcu_read_unlock_special.
+ * rcu_read_unlock_special.
*
* Because this is a single-CPU implementation, the only way a grace
* period can end is if the CPU is in a quiescent state. The reason is
{
/* Record both CPU and task as having responded to current GP. */
rcu_preempt_ctrlblk.gpcpu = rcu_preempt_ctrlblk.gpnum;
- current->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_NEED_QS;
+ __get_cpu_var(rcu_read_unlock_special) &= ~RCU_READ_UNLOCK_NEED_QS;
/* If there is no GP then there is nothing more to do. */
if (!rcu_preempt_gp_in_progress())
local_irq_save(flags); /* must exclude scheduler_tick(). */
if (rcu_preempt_running_reader() > 0 &&
- (t->rcu_read_unlock_special & RCU_READ_UNLOCK_BLOCKED) == 0) {
+ (__this_cpu_read(rcu_read_unlock_special) &
+ RCU_READ_UNLOCK_BLOCKED) == 0) {
/* Possibly blocking in an RCU read-side critical section. */
- t->rcu_read_unlock_special |= RCU_READ_UNLOCK_BLOCKED;
+ __get_cpu_var(rcu_read_unlock_special) |=
+ RCU_READ_UNLOCK_BLOCKED;
/*
* If this CPU has already checked in, then this task
if (rcu_cpu_blocking_cur_gp())
rcu_preempt_ctrlblk.gp_tasks = &t->rcu_node_entry;
} else if (rcu_preempt_running_reader() < 0 &&
- t->rcu_read_unlock_special) {
+ __this_cpu_read(rcu_read_unlock_special)) {
/*
* Complete exit from RCU read-side critical section on
* behalf of preempted instance of __rcu_read_unlock().
*/
- rcu_read_unlock_do_special(t);
+ rcu_read_unlock_do_special();
}
/*
local_irq_restore(flags);
}
-/*
- * Tiny-preemptible RCU implementation for rcu_read_lock().
- * Just increment ->rcu_read_lock_nesting, shared state will be updated
- * if we block.
- */
-void __rcu_read_lock(void)
-{
- current->rcu_read_lock_nesting++;
- barrier(); /* needed if we ever invoke rcu_read_lock in rcutiny.c */
-}
-EXPORT_SYMBOL_GPL(__rcu_read_lock);
-
/*
* Handle special cases during rcu_read_unlock(), such as needing to
* notify RCU core processing or task having blocked during the RCU
* read-side critical section.
*/
-static noinline void rcu_read_unlock_do_special(struct task_struct *t)
+void rcu_read_unlock_do_special(void)
{
int empty;
int empty_exp;
* If RCU core is waiting for this CPU to exit critical section,
* let it know that we have done so.
*/
- special = t->rcu_read_unlock_special;
+ special = __this_cpu_read(rcu_read_unlock_special);
if (special & RCU_READ_UNLOCK_NEED_QS)
rcu_preempt_cpu_qs();
/* Clean up if blocked during RCU read-side critical section. */
if (special & RCU_READ_UNLOCK_BLOCKED) {
- t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_BLOCKED;
+ struct task_struct *t = current;
+
+ __get_cpu_var(rcu_read_unlock_special) &=
+ ~RCU_READ_UNLOCK_BLOCKED;
/*
* Remove this task from the ->blkd_tasks list and adjust
*/
if (!empty_exp && rcu_preempt_ctrlblk.exp_tasks == NULL)
rcu_report_exp_done();
- }
#ifdef CONFIG_RCU_BOOST
- /* Unboost self if was boosted. */
- if (t->rcu_boost_mutex != NULL) {
- rbmp = t->rcu_boost_mutex;
- t->rcu_boost_mutex = NULL;
- rt_mutex_unlock(rbmp);
- }
+ /* Unboost self if was boosted. */
+ if (t->rcu_boost_mutex != NULL) {
+ rbmp = t->rcu_boost_mutex;
+ t->rcu_boost_mutex = NULL;
+ rt_mutex_unlock(rbmp);
+ }
#endif /* #ifdef CONFIG_RCU_BOOST */
- local_irq_restore(flags);
-}
-
-/*
- * Tiny-preemptible RCU implementation for rcu_read_unlock().
- * Decrement ->rcu_read_lock_nesting. If the result is zero (outermost
- * rcu_read_unlock()) and ->rcu_read_unlock_special is non-zero, then
- * invoke rcu_read_unlock_do_special() to clean up after a context switch
- * in an RCU read-side critical section and other special cases.
- */
-void __rcu_read_unlock(void)
-{
- struct task_struct *t = current;
-
- barrier(); /* needed if we ever invoke rcu_read_unlock in rcutiny.c */
- if (t->rcu_read_lock_nesting != 1)
- --t->rcu_read_lock_nesting;
- else {
- t->rcu_read_lock_nesting = INT_MIN;
- barrier(); /* assign before ->rcu_read_unlock_special load */
- if (unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))
- rcu_read_unlock_do_special(t);
- barrier(); /* ->rcu_read_unlock_special load before assign */
- t->rcu_read_lock_nesting = 0;
- }
-#ifdef CONFIG_PROVE_LOCKING
- {
- int rrln = ACCESS_ONCE(t->rcu_read_lock_nesting);
-
- WARN_ON_ONCE(rrln < 0 && rrln > INT_MIN / 2);
}
-#endif /* #ifdef CONFIG_PROVE_LOCKING */
+ local_irq_restore(flags);
}
-EXPORT_SYMBOL_GPL(__rcu_read_unlock);
/*
* Check for a quiescent state from the current CPU. When a task blocks,
*/
static void rcu_preempt_check_callbacks(void)
{
- struct task_struct *t = current;
-
if (rcu_preempt_gp_in_progress() &&
(!rcu_preempt_running_reader() ||
!rcu_cpu_blocking_cur_gp()))
if (rcu_preempt_gp_in_progress() &&
rcu_cpu_blocking_cur_gp() &&
rcu_preempt_running_reader() > 0)
- t->rcu_read_unlock_special |= RCU_READ_UNLOCK_NEED_QS;
+ __get_cpu_var(rcu_read_unlock_special) |=
+ RCU_READ_UNLOCK_NEED_QS;
}
/*
DEFINE_PER_CPU(struct rcu_data, rcu_preempt_data);
static struct rcu_state *rcu_state = &rcu_preempt_state;
-static void rcu_read_unlock_do_special(struct task_struct *t);
static int rcu_preempted_readers_exp(struct rcu_node *rnp);
/*
*
* Unlike the other rcu_*_qs() functions, callers to this function
* must disable irqs in order to protect the assignment to
- * ->rcu_read_unlock_special.
+ * rcu_read_unlock_special.
*/
static void rcu_preempt_qs(int cpu)
{
if (rdp->passed_quiesce == 0)
trace_rcu_grace_period("rcu_preempt", rdp->gpnum, "cpuqs");
rdp->passed_quiesce = 1;
- current->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_NEED_QS;
+ __get_cpu_var(rcu_read_unlock_special) &= ~RCU_READ_UNLOCK_NEED_QS;
}
/*
struct rcu_data *rdp;
struct rcu_node *rnp;
- if (t->rcu_read_lock_nesting > 0 &&
- (t->rcu_read_unlock_special & RCU_READ_UNLOCK_BLOCKED) == 0) {
+ if (__this_cpu_read(rcu_read_lock_nesting) > 0 &&
+ (__this_cpu_read(rcu_read_unlock_special) &
+ RCU_READ_UNLOCK_BLOCKED) == 0) {
/* Possibly blocking in an RCU read-side critical section. */
rdp = __this_cpu_ptr(rcu_preempt_state.rda);
rnp = rdp->mynode;
raw_spin_lock_irqsave(&rnp->lock, flags);
- t->rcu_read_unlock_special |= RCU_READ_UNLOCK_BLOCKED;
+ __get_cpu_var(rcu_read_unlock_special) |=
+ RCU_READ_UNLOCK_BLOCKED;
t->rcu_blocked_node = rnp;
/*
? rnp->gpnum
: rnp->gpnum + 1);
raw_spin_unlock_irqrestore(&rnp->lock, flags);
- } else if (t->rcu_read_lock_nesting < 0 &&
- t->rcu_read_unlock_special) {
+ } else if (__this_cpu_read(rcu_read_lock_nesting) < 0 &&
+ __this_cpu_read(rcu_read_unlock_special)) {
/*
- * Complete exit from RCU read-side critical section on
+ * Finish the exit from RCU read-side critical section on
* behalf of preempted instance of __rcu_read_unlock().
*/
- rcu_read_unlock_do_special(t);
+ rcu_read_unlock_do_special();
}
/*
local_irq_restore(flags);
}
-/*
- * Tree-preemptible RCU implementation for rcu_read_lock().
- * Just increment ->rcu_read_lock_nesting, shared state will be updated
- * if we block.
- */
-void __rcu_read_lock(void)
-{
- current->rcu_read_lock_nesting++;
- barrier(); /* needed if we ever invoke rcu_read_lock in rcutree.c */
-}
-EXPORT_SYMBOL_GPL(__rcu_read_lock);
-
/*
* Check for preempted RCU readers blocking the current grace period
* for the specified rcu_node structure. If the caller needs a reliable
* notify RCU core processing or task having blocked during the RCU
* read-side critical section.
*/
-static noinline void rcu_read_unlock_do_special(struct task_struct *t)
+void rcu_read_unlock_do_special(void)
{
int empty;
int empty_exp;
* If RCU core is waiting for this CPU to exit critical section,
* let it know that we have done so.
*/
- special = t->rcu_read_unlock_special;
+ special = __this_cpu_read(rcu_read_unlock_special);
if (special & RCU_READ_UNLOCK_NEED_QS) {
rcu_preempt_qs(smp_processor_id());
}
/* Clean up if blocked during RCU read-side critical section. */
if (special & RCU_READ_UNLOCK_BLOCKED) {
- t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_BLOCKED;
+ struct task_struct *t = current;
+
+ __get_cpu_var(rcu_read_unlock_special) &=
+ ~RCU_READ_UNLOCK_BLOCKED;
/*
* Remove this task from the list it blocked on. The
}
}
-/*
- * Tree-preemptible RCU implementation for rcu_read_unlock().
- * Decrement ->rcu_read_lock_nesting. If the result is zero (outermost
- * rcu_read_unlock()) and ->rcu_read_unlock_special is non-zero, then
- * invoke rcu_read_unlock_do_special() to clean up after a context switch
- * in an RCU read-side critical section and other special cases.
- */
-void __rcu_read_unlock(void)
-{
- struct task_struct *t = current;
-
- if (t->rcu_read_lock_nesting != 1)
- --t->rcu_read_lock_nesting;
- else {
- barrier(); /* critical section before exit code. */
- t->rcu_read_lock_nesting = INT_MIN;
- barrier(); /* assign before ->rcu_read_unlock_special load */
- if (unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))
- rcu_read_unlock_do_special(t);
- barrier(); /* ->rcu_read_unlock_special load before assign */
- t->rcu_read_lock_nesting = 0;
- }
-#ifdef CONFIG_PROVE_LOCKING
- {
- int rrln = ACCESS_ONCE(t->rcu_read_lock_nesting);
-
- WARN_ON_ONCE(rrln < 0 && rrln > INT_MIN / 2);
- }
-#endif /* #ifdef CONFIG_PROVE_LOCKING */
-}
-EXPORT_SYMBOL_GPL(__rcu_read_unlock);
-
#ifdef CONFIG_RCU_CPU_STALL_VERBOSE
/*
*/
static void rcu_preempt_check_callbacks(int cpu)
{
- struct task_struct *t = current;
-
- if (t->rcu_read_lock_nesting == 0) {
+ if (__this_cpu_read(rcu_read_lock_nesting) == 0) {
rcu_preempt_qs(cpu);
return;
}
- if (t->rcu_read_lock_nesting > 0 &&
- per_cpu(rcu_preempt_data, cpu).qs_pending)
- t->rcu_read_unlock_special |= RCU_READ_UNLOCK_NEED_QS;
+ if (__this_cpu_read(rcu_read_lock_nesting) > 0 &&
+ __get_cpu_var(rcu_preempt_data).qs_pending)
+ __get_cpu_var(rcu_read_unlock_special) |=
+ RCU_READ_UNLOCK_NEED_QS;
}
/*
#endif
/* Here we just switch the register state and the stack. */
- rcu_switch_from(prev);
+ rcu_switch_from();
switch_to(prev, next, prev);
+ rcu_switch_to();
barrier();
/*
projects / karo-tx-linux.git / commitdiff