rcu: Restore checks for blocking in RCU read-side critical sections

Long ago, using TREE_RCU with PREEMPT would result in "scheduling
while atomic" diagnostics if you blocked in an RCU read-side critical
section.  However, PREEMPT now implies TREE_PREEMPT_RCU, which defeats
this diagnostic.  This commit therefore adds a replacement diagnostic
based on PROVE_RCU.

Because rcu_lockdep_assert() and lockdep_rcu_dereference() are now being
used for things that have nothing to do with rcu_dereference(), rename
lockdep_rcu_dereference() to lockdep_rcu_suspicious() and add a third
argument that is a string indicating what is suspicious.  This third
argument is passed in from a new third argument to rcu_lockdep_assert().
Update all calls to rcu_lockdep_assert() to add an informative third
argument.

Also, add a pair of rcu_lockdep_assert() calls from within
rcu_note_context_switch(), one complaining if a context switch occurs
in an RCU-bh read-side critical section and another complaining if a
context switch occurs in an RCU-sched read-side critical section.
These are present only if the PROVE_RCU kernel parameter is enabled.

Finally, fix some checkpatch whitespace complaints in lockdep.c.

Again, you must enable PROVE_RCU to see these new diagnostics.  But you
are enabling PROVE_RCU to check out new RCU uses in any case, aren't you?

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h
index ef820a3c378bb421a51fb0920f289571f8f96382..b6a56e37284c41e0954a7a794ddab4b617ebf547 100644 (file)
--- a/include/linux/lockdep.h
+++ b/include/linux/lockdep.h
@@ -548,7 +548,7 @@ do {                                                                        \
 #endif
 
 #ifdef CONFIG_PROVE_RCU
-extern void lockdep_rcu_dereference(const char *file, const int line);
+void lockdep_rcu_suspicious(const char *file, const int line, const char *s);
 #endif
 
 #endif /* __LINUX_LOCKDEP_H */

diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index 99f9aa7c2804e2775ff552a87aeb2f27de7bac97..8be0433221d576a9290a17d7e97cdf4c5eb40d21 100644 (file)
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -297,19 +297,31 @@ extern int rcu_my_thread_group_empty(void);
 /**
  * rcu_lockdep_assert - emit lockdep splat if specified condition not met
  * @c: condition to check
+ * @s: informative message
  */
-#define rcu_lockdep_assert(c)                                          \
+#define rcu_lockdep_assert(c, s)                                       \
        do {                                                            \
                static bool __warned;                                   \
                if (debug_lockdep_rcu_enabled() && !__warned && !(c)) { \
                        __warned = true;                                \
-                       lockdep_rcu_dereference(__FILE__, __LINE__);    \
+                       lockdep_rcu_suspicious(__FILE__, __LINE__, s);  \
                }                                                       \
        } while (0)
 
+#define rcu_sleep_check()                                              \
+       do {                                                            \
+               rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map),     \
+                                  "Illegal context switch in RCU-bh"   \
+                                  " read-side critical section");      \
+               rcu_lockdep_assert(!lock_is_held(&rcu_sched_lock_map),  \
+                                  "Illegal context switch in RCU-sched"\
+                                  " read-side critical section");      \
+       } while (0)
+
 #else /* #ifdef CONFIG_PROVE_RCU */
 
-#define rcu_lockdep_assert(c) do { } while (0)
+#define rcu_lockdep_assert(c, s) do { } while (0)
+#define rcu_sleep_check() do { } while (0)
 
 #endif /* #else #ifdef CONFIG_PROVE_RCU */
 
@@ -338,14 +350,16 @@ extern int rcu_my_thread_group_empty(void);
 #define __rcu_dereference_check(p, c, space) \
        ({ \
                typeof(*p) *_________p1 = (typeof(*p)*__force )ACCESS_ONCE(p); \
-               rcu_lockdep_assert(c); \
+               rcu_lockdep_assert(c, "suspicious rcu_dereference_check()" \
+                                     " usage"); \
                rcu_dereference_sparse(p, space); \
                smp_read_barrier_depends(); \
                ((typeof(*p) __force __kernel *)(_________p1)); \
        })
 #define __rcu_dereference_protected(p, c, space) \
        ({ \
-               rcu_lockdep_assert(c); \
+               rcu_lockdep_assert(c, "suspicious rcu_dereference_protected()" \
+                                     " usage"); \
                rcu_dereference_sparse(p, space); \
                ((typeof(*p) __force __kernel *)(p)); \
        })
@@ -359,7 +373,9 @@ extern int rcu_my_thread_group_empty(void);
 #define __rcu_dereference_index_check(p, c) \
        ({ \
                typeof(p) _________p1 = ACCESS_ONCE(p); \
-               rcu_lockdep_assert(c); \
+               rcu_lockdep_assert(c, \
+                                  "suspicious rcu_dereference_index_check()" \
+                                  " usage"); \
                smp_read_barrier_depends(); \
                (_________p1); \
        })

diff --git a/kernel/lockdep.c b/kernel/lockdep.c
index 298c9276dfdb378877249384db860c3658485ef2..df2ad3727a8a8c8e0ab43da4ea780b1d6c7f7d60 100644 (file)
--- a/kernel/lockdep.c
+++ b/kernel/lockdep.c
@@ -1129,10 +1129,11 @@ print_circular_bug_header(struct lock_list *entry, unsigned int depth,
        if (debug_locks_silent)
                return 0;
 
-       printk("\n=======================================================\n");
-       printk(  "[ INFO: possible circular locking dependency detected ]\n");
+       printk("\n");
+       printk("======================================================\n");
+       printk("[ INFO: possible circular locking dependency detected ]\n");
        print_kernel_version();
-       printk(  "-------------------------------------------------------\n");
+       printk("-------------------------------------------------------\n");
        printk("%s/%d is trying to acquire lock:\n",
                curr->comm, task_pid_nr(curr));
        print_lock(check_src);
@@ -1463,11 +1464,12 @@ print_bad_irq_dependency(struct task_struct *curr,
        if (!debug_locks_off_graph_unlock() || debug_locks_silent)
                return 0;
 
-       printk("\n======================================================\n");
-       printk(  "[ INFO: %s-safe -> %s-unsafe lock order detected ]\n",
+       printk("\n");
+       printk("======================================================\n");
+       printk("[ INFO: %s-safe -> %s-unsafe lock order detected ]\n",
                irqclass, irqclass);
        print_kernel_version();
-       printk(  "------------------------------------------------------\n");
+       printk("------------------------------------------------------\n");
        printk("%s/%d [HC%u[%lu]:SC%u[%lu]:HE%u:SE%u] is trying to acquire:\n",
                curr->comm, task_pid_nr(curr),
                curr->hardirq_context, hardirq_count() >> HARDIRQ_SHIFT,
@@ -1692,10 +1694,11 @@ print_deadlock_bug(struct task_struct *curr, struct held_lock *prev,
        if (!debug_locks_off_graph_unlock() || debug_locks_silent)
                return 0;
 
-       printk("\n=============================================\n");
-       printk(  "[ INFO: possible recursive locking detected ]\n");
+       printk("\n");
+       printk("=============================================\n");
+       printk("[ INFO: possible recursive locking detected ]\n");
        print_kernel_version();
-       printk(  "---------------------------------------------\n");
+       printk("---------------------------------------------\n");
        printk("%s/%d is trying to acquire lock:\n",
                curr->comm, task_pid_nr(curr));
        print_lock(next);
@@ -2177,10 +2180,11 @@ print_usage_bug(struct task_struct *curr, struct held_lock *this,
        if (!debug_locks_off_graph_unlock() || debug_locks_silent)
                return 0;
 
-       printk("\n=================================\n");
-       printk(  "[ INFO: inconsistent lock state ]\n");
+       printk("\n");
+       printk("=================================\n");
+       printk("[ INFO: inconsistent lock state ]\n");
        print_kernel_version();
-       printk(  "---------------------------------\n");
+       printk("---------------------------------\n");
 
        printk("inconsistent {%s} -> {%s} usage.\n",
                usage_str[prev_bit], usage_str[new_bit]);
@@ -2241,10 +2245,11 @@ print_irq_inversion_bug(struct task_struct *curr,
        if (!debug_locks_off_graph_unlock() || debug_locks_silent)
                return 0;
 
-       printk("\n=========================================================\n");
-       printk(  "[ INFO: possible irq lock inversion dependency detected ]\n");
+       printk("\n");
+       printk("=========================================================\n");
+       printk("[ INFO: possible irq lock inversion dependency detected ]\n");
        print_kernel_version();
-       printk(  "---------------------------------------------------------\n");
+       printk("---------------------------------------------------------\n");
        printk("%s/%d just changed the state of lock:\n",
                curr->comm, task_pid_nr(curr));
        print_lock(this);
@@ -3053,9 +3058,10 @@ print_unlock_inbalance_bug(struct task_struct *curr, struct lockdep_map *lock,
        if (debug_locks_silent)
                return 0;
 
-       printk("\n=====================================\n");
-       printk(  "[ BUG: bad unlock balance detected! ]\n");
-       printk(  "-------------------------------------\n");
+       printk("\n");
+       printk("=====================================\n");
+       printk("[ BUG: bad unlock balance detected! ]\n");
+       printk("-------------------------------------\n");
        printk("%s/%d is trying to release lock (",
                curr->comm, task_pid_nr(curr));
        print_lockdep_cache(lock);
@@ -3460,9 +3466,10 @@ print_lock_contention_bug(struct task_struct *curr, struct lockdep_map *lock,
        if (debug_locks_silent)
                return 0;
 
-       printk("\n=================================\n");
-       printk(  "[ BUG: bad contention detected! ]\n");
-       printk(  "---------------------------------\n");
+       printk("\n");
+       printk("=================================\n");
+       printk("[ BUG: bad contention detected! ]\n");
+       printk("---------------------------------\n");
        printk("%s/%d is trying to contend lock (",
                curr->comm, task_pid_nr(curr));
        print_lockdep_cache(lock);
@@ -3821,9 +3828,10 @@ print_freed_lock_bug(struct task_struct *curr, const void *mem_from,
        if (debug_locks_silent)
                return;
 
-       printk("\n=========================\n");
-       printk(  "[ BUG: held lock freed! ]\n");
-       printk(  "-------------------------\n");
+       printk("\n");
+       printk("=========================\n");
+       printk("[ BUG: held lock freed! ]\n");
+       printk("-------------------------\n");
        printk("%s/%d is freeing memory %p-%p, with a lock still held there!\n",
                curr->comm, task_pid_nr(curr), mem_from, mem_to-1);
        print_lock(hlock);
@@ -3877,9 +3885,10 @@ static void print_held_locks_bug(struct task_struct *curr)
        if (debug_locks_silent)
                return;
 
-       printk("\n=====================================\n");
-       printk(  "[ BUG: lock held at task exit time! ]\n");
-       printk(  "-------------------------------------\n");
+       printk("\n");
+       printk("=====================================\n");
+       printk("[ BUG: lock held at task exit time! ]\n");
+       printk("-------------------------------------\n");
        printk("%s/%d is exiting with locks still held!\n",
                curr->comm, task_pid_nr(curr));
        lockdep_print_held_locks(curr);
@@ -3973,16 +3982,17 @@ void lockdep_sys_exit(void)
        if (unlikely(curr->lockdep_depth)) {
                if (!debug_locks_off())
                        return;
-               printk("\n================================================\n");
-               printk(  "[ BUG: lock held when returning to user space! ]\n");
-               printk(  "------------------------------------------------\n");
+               printk("\n");
+               printk("================================================\n");
+               printk("[ BUG: lock held when returning to user space! ]\n");
+               printk("------------------------------------------------\n");
                printk("%s/%d is leaving the kernel with locks still held!\n",
                                curr->comm, curr->pid);
                lockdep_print_held_locks(curr);
        }
 }
 
-void lockdep_rcu_dereference(const char *file, const int line)
+void lockdep_rcu_suspicious(const char *file, const int line, const char *s)
 {
        struct task_struct *curr = current;
 
@@ -3991,15 +4001,15 @@ void lockdep_rcu_dereference(const char *file, const int line)
                return;
 #endif /* #ifdef CONFIG_PROVE_RCU_REPEATEDLY */
        /* Note: the following can be executed concurrently, so be careful. */
-       printk("\n===================================================\n");
-       printk(  "[ INFO: suspicious rcu_dereference_check() usage. ]\n");
-       printk(  "---------------------------------------------------\n");
-       printk("%s:%d invoked rcu_dereference_check() without protection!\n",
-                       file, line);
+       printk("\n");
+       printk("===============================\n");
+       printk("[ INFO: suspicious RCU usage. ]\n");
+       printk("-------------------------------\n");
+       printk("%s:%d %s!\n", file, line, s);
        printk("\nother info that might help us debug this:\n\n");
        printk("\nrcu_scheduler_active = %d, debug_locks = %d\n", rcu_scheduler_active, debug_locks);
        lockdep_print_held_locks(curr);
        printk("\nstack backtrace:\n");
        dump_stack();
 }
-EXPORT_SYMBOL_GPL(lockdep_rcu_dereference);
+EXPORT_SYMBOL_GPL(lockdep_rcu_suspicious);

diff --git a/kernel/pid.c b/kernel/pid.c
index 57a8346a270e07702e21d7bab15303427bf2fce0..a7577b3887c6bdd21fdeaf4cfb81db8057a74205 100644 (file)
--- a/kernel/pid.c
+++ b/kernel/pid.c
@@ -419,7 +419,9 @@ EXPORT_SYMBOL(pid_task);
  */
 struct task_struct *find_task_by_pid_ns(pid_t nr, struct pid_namespace *ns)
 {
-       rcu_lockdep_assert(rcu_read_lock_held());
+       rcu_lockdep_assert(rcu_read_lock_held(),
+                          "find_task_by_pid_ns() needs rcu_read_lock()"
+                          " protection");
        return pid_task(find_pid_ns(nr, ns), PIDTYPE_PID);
 }
 

diff --git a/kernel/sched.c b/kernel/sched.c
index fde6ff90352583d65ff890a407200f2fb0c3073e..1c8791762e4546f6d15c5cb05bd98599747382b5 100644 (file)
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -4200,6 +4200,7 @@ static inline void schedule_debug(struct task_struct *prev)
         */
        if (unlikely(in_atomic_preempt_off() && !prev->exit_state))
                __schedule_bug(prev);
+       rcu_sleep_check();
 
        profile_hit(SCHED_PROFILING, __builtin_return_address(0));
 
@@ -8198,6 +8199,7 @@ void __might_sleep(const char *file, int line, int preempt_offset)
 #ifdef in_atomic
        static unsigned long prev_jiffy;        /* ratelimiting */
 
+       rcu_sleep_check(); /* WARN_ON_ONCE() by default, no rate limit reqd. */
        if ((preempt_count_equals(preempt_offset) && !irqs_disabled()) ||
            system_state != SYSTEM_RUNNING || oops_in_progress)
                return;