rcu: New rcu_user_enter_irq() and rcu_user_exit_irq() APIs

In some cases, it is necessary to enter or exit userspace-RCU-idle mode
from an interrupt handler, for example, if some other CPU sends this
CPU a resched IPI.  In this case, the current CPU would enter the IPI
handler in userspace-RCU-idle mode, but would need to exit the IPI handler
after having exited that mode.

To allow this to work, this commit adds two new APIs to TREE_RCU:

- rcu_user_enter_irq(). This must be called from an interrupt between
rcu_irq_enter() and rcu_irq_exit().  After the irq calls rcu_irq_exit(),
the irq handler will return into an RCU extended quiescent state.
In theory, this interrupt is never a nested interrupt, but in practice
it might interrupt softirq, which looks to RCU like a nested interrupt.

- rcu_user_exit_irq(). This must be called from a non-nesting
interrupt, interrupting an RCU extended quiescent state, also
between rcu_irq_enter() and rcu_irq_exit(). After the irq calls
rcu_irq_exit(), the irq handler will return in an RCU non-quiescent
state.

[ Combined with "Allow calls to rcu_exit_user_irq from nesting irqs." ]

Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index 2a7549cf98a2a6f277c0d9a13ddba5f11fbf8835..81d3d5c53c43a6d5e274d4119631f49989fba271 100644 (file)
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -193,6 +193,8 @@ extern void rcu_irq_enter(void);
 extern void rcu_irq_exit(void);
 extern void rcu_user_enter(void);
 extern void rcu_user_exit(void);
+extern void rcu_user_enter_irq(void);
+extern void rcu_user_exit_irq(void);
 extern void exit_rcu(void);
 
 /**

diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index c0507b773007f589e2492e0c13445681faaa16f2..8fdea17af81d5bfdb158a1b9e10a07dbf9a969d1 100644 (file)
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -439,6 +439,27 @@ void rcu_user_enter(void)
 EXPORT_SYMBOL_GPL(rcu_user_enter);
 
 
+/**
+ * rcu_user_enter_irq - inform RCU that we are going to resume userspace
+ * after the current irq returns.
+ *
+ * This is similar to rcu_user_enter() but in the context of a non-nesting
+ * irq. After this call, RCU enters into idle mode when the interrupt
+ * returns.
+ */
+void rcu_user_enter_irq(void)
+{
+       unsigned long flags;
+       struct rcu_dynticks *rdtp;
+
+       local_irq_save(flags);
+       rdtp = &__get_cpu_var(rcu_dynticks);
+       /* Ensure this irq is interrupting a non-idle RCU state.  */
+       WARN_ON_ONCE(!(rdtp->dynticks_nesting & DYNTICK_TASK_MASK));
+       rdtp->dynticks_nesting = 1;
+       local_irq_restore(flags);
+}
+
 /**
  * rcu_irq_exit - inform RCU that current CPU is exiting irq towards idle
  *
@@ -553,6 +574,28 @@ void rcu_user_exit(void)
 }
 EXPORT_SYMBOL_GPL(rcu_user_exit);
 
+/**
+ * rcu_user_exit_irq - inform RCU that we won't resume to userspace
+ * idle mode after the current non-nesting irq returns.
+ *
+ * This is similar to rcu_user_exit() but in the context of an irq.
+ * This is called when the irq has interrupted a userspace RCU idle mode
+ * context. When the current non-nesting interrupt returns after this call,
+ * the CPU won't restore the RCU idle mode.
+ */
+void rcu_user_exit_irq(void)
+{
+       unsigned long flags;
+       struct rcu_dynticks *rdtp;
+
+       local_irq_save(flags);
+       rdtp = &__get_cpu_var(rcu_dynticks);
+       /* Ensure we are interrupting an RCU idle mode. */
+       WARN_ON_ONCE(rdtp->dynticks_nesting & DYNTICK_TASK_NEST_MASK);
+       rdtp->dynticks_nesting += DYNTICK_TASK_EXIT_IDLE;
+       local_irq_restore(flags);
+}
+
 /**
  * rcu_irq_enter - inform RCU that current CPU is entering irq away from idle
  *