rcu: Provide diagnostic option to slow down grace-period scans

Grace-period scans of the rcu_node combining tree normally
proceed quite quickly, so that it is very difficult to reproduce
races against them.  This commit therefore allows grace-period
pre-initialization and cleanup to be artificially slowed down,
increasing race-reproduction probability.  A pair of pairs of new
Kconfig parameters are provided, RCU_TORTURE_TEST_SLOW_PREINIT to
enable the slowing down of propagating CPU-hotplug changes up the
combining tree along with RCU_TORTURE_TEST_SLOW_PREINIT_DELAY to
specify the delay in jiffies, and RCU_TORTURE_TEST_SLOW_CLEANUP
to enable the slowing down of the end-of-grace-period cleanup scan
along with RCU_TORTURE_TEST_SLOW_CLEANUP_DELAY to specify the delay
in jiffies.  Boot-time parameters named rcutree.gp_preinit_delay and
rcutree.gp_cleanup_delay allow these delays to be specified at boot time.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

diff --git a/Documentation/kernel-parameters.txt b/Documentation/kernel-parameters.txt
index 61ab1628a057cc2c4d8b11d892d834f7e5f7773a..10a4fb80c0333b2006bfb0e5ac6121a5f405a2c9 100644 (file)
--- a/Documentation/kernel-parameters.txt
+++ b/Documentation/kernel-parameters.txt
@@ -2992,11 +2992,23 @@ bytes respectively. Such letter suffixes can also be entirely omitted.
                        Set maximum number of finished RCU callbacks to
                        process in one batch.
 
+       rcutree.gp_cleanup_delay=       [KNL]
+                       Set the number of jiffies to delay each step of
+                       RCU grace-period cleanup.  This only has effect
+                       when CONFIG_RCU_TORTURE_TEST_SLOW_CLEANUP is set.
+
        rcutree.gp_init_delay=  [KNL]
                        Set the number of jiffies to delay each step of
                        RCU grace-period initialization.  This only has
-                       effect when CONFIG_RCU_TORTURE_TEST_SLOW_INIT is
-                       set.
+                       effect when CONFIG_RCU_TORTURE_TEST_SLOW_INIT
+                       is set.
+
+       rcutree.gp_preinit_delay=       [KNL]
+                       Set the number of jiffies to delay each step of
+                       RCU grace-period pre-initialization, that is,
+                       the propagation of recent CPU-hotplug changes up
+                       the rcu_node combining tree.  This only has effect
+                       when CONFIG_RCU_TORTURE_TEST_SLOW_PREINIT is set.
 
        rcutree.rcu_fanout_leaf= [KNL]
                        Increase the number of CPUs assigned to each

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 9b076b2846954720baa6fc5722b86b7ca7d8a2c3..2f3cb5513ca3bbb26e9293264b5d095bb8841784 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -163,6 +163,14 @@ static int kthread_prio = CONFIG_RCU_KTHREAD_PRIO;
 module_param(kthread_prio, int, 0644);
 
 /* Delay in jiffies for grace-period initialization delays, debug only. */
+
+#ifdef CONFIG_RCU_TORTURE_TEST_SLOW_PREINIT
+static int gp_preinit_delay = CONFIG_RCU_TORTURE_TEST_SLOW_PREINIT_DELAY;
+module_param(gp_preinit_delay, int, 0644);
+#else /* #ifdef CONFIG_RCU_TORTURE_TEST_SLOW_PREINIT */
+static const int gp_preinit_delay;
+#endif /* #else #ifdef CONFIG_RCU_TORTURE_TEST_SLOW_PREINIT */
+
 #ifdef CONFIG_RCU_TORTURE_TEST_SLOW_INIT
 static int gp_init_delay = CONFIG_RCU_TORTURE_TEST_SLOW_INIT_DELAY;
 module_param(gp_init_delay, int, 0644);
@@ -170,6 +178,13 @@ module_param(gp_init_delay, int, 0644);
 static const int gp_init_delay;
 #endif /* #else #ifdef CONFIG_RCU_TORTURE_TEST_SLOW_INIT */
 
+#ifdef CONFIG_RCU_TORTURE_TEST_SLOW_CLEANUP
+static int gp_cleanup_delay = CONFIG_RCU_TORTURE_TEST_SLOW_CLEANUP_DELAY;
+module_param(gp_cleanup_delay, int, 0644);
+#else /* #ifdef CONFIG_RCU_TORTURE_TEST_SLOW_CLEANUP */
+static const int gp_cleanup_delay;
+#endif /* #else #ifdef CONFIG_RCU_TORTURE_TEST_SLOW_CLEANUP */
+
 /*
  * Number of grace periods between delays, normalized by the duration of
  * the delay.  The longer the the delay, the more the grace periods between
@@ -1742,6 +1757,13 @@ static void note_gp_changes(struct rcu_state *rsp, struct rcu_data *rdp)
                rcu_gp_kthread_wake(rsp);
 }
 
+static void rcu_gp_slow(struct rcu_state *rsp, int delay)
+{
+       if (delay > 0 &&
+           !(rsp->gpnum % (rcu_num_nodes * PER_RCU_NODE_PERIOD * delay)))
+               schedule_timeout_uninterruptible(delay);
+}
+
 /*
  * Initialize a new grace period.  Return 0 if no grace period required.
  */
@@ -1784,6 +1806,7 @@ static int rcu_gp_init(struct rcu_state *rsp)
         * will handle subsequent offline CPUs.
         */
        rcu_for_each_leaf_node(rsp, rnp) {
+               rcu_gp_slow(rsp, gp_preinit_delay);
                raw_spin_lock_irq(&rnp->lock);
                smp_mb__after_unlock_lock();
                if (rnp->qsmaskinit == rnp->qsmaskinitnext &&
@@ -1840,6 +1863,7 @@ static int rcu_gp_init(struct rcu_state *rsp)
         * process finishes, because this kthread handles both.
         */
        rcu_for_each_node_breadth_first(rsp, rnp) {
+               rcu_gp_slow(rsp, gp_init_delay);
                raw_spin_lock_irq(&rnp->lock);
                smp_mb__after_unlock_lock();
                rdp = this_cpu_ptr(rsp->rda);
@@ -1857,10 +1881,6 @@ static int rcu_gp_init(struct rcu_state *rsp)
                raw_spin_unlock_irq(&rnp->lock);
                cond_resched_rcu_qs();
                WRITE_ONCE(rsp->gp_activity, jiffies);
-               if (gp_init_delay > 0 &&
-                   !(rsp->gpnum %
-                     (rcu_num_nodes * PER_RCU_NODE_PERIOD * gp_init_delay)))
-                       schedule_timeout_uninterruptible(gp_init_delay);
        }
 
        return 1;
@@ -1955,6 +1975,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
                raw_spin_unlock_irq(&rnp->lock);
                cond_resched_rcu_qs();
                WRITE_ONCE(rsp->gp_activity, jiffies);
+               rcu_gp_slow(rsp, gp_cleanup_delay);
        }
        rnp = rcu_get_root(rsp);
        raw_spin_lock_irq(&rnp->lock);

diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index ba2b0c87e65b196c7c1f016798e0b59ea8dba97c..e1af93ae246b93cf4ba9398c46a73439c363ccef 100644 (file)
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -1261,12 +1261,38 @@ config RCU_TORTURE_TEST_RUNNABLE
          Say N here if you want the RCU torture tests to start only
          after being manually enabled via /proc.
 
+config RCU_TORTURE_TEST_SLOW_PREINIT
+       bool "Slow down RCU grace-period pre-initialization to expose races"
+       depends on RCU_TORTURE_TEST
+       help
+         This option delays grace-period pre-initialization (the
+         propagation of CPU-hotplug changes up the rcu_node combining
+         tree) for a few jiffies between initializing each pair of
+         consecutive rcu_node structures.  This helps to expose races
+         involving grace-period pre-initialization, in other words, it
+         makes your kernel less stable.  It can also greatly increase
+         grace-period latency, especially on systems with large numbers
+         of CPUs.  This is useful when torture-testing RCU, but in
+         almost no other circumstance.
+
+         Say Y here if you want your system to crash and hang more often.
+         Say N if you want a sane system.
+
+config RCU_TORTURE_TEST_SLOW_PREINIT_DELAY
+       int "How much to slow down RCU grace-period pre-initialization"
+       range 0 5
+       default 3
+       depends on RCU_TORTURE_TEST_SLOW_PREINIT
+       help
+         This option specifies the number of jiffies to wait between
+         each rcu_node structure pre-initialization step.
+
 config RCU_TORTURE_TEST_SLOW_INIT
        bool "Slow down RCU grace-period initialization to expose races"
        depends on RCU_TORTURE_TEST
        help
-         This option makes grace-period initialization block for a
-         few jiffies between initializing each pair of consecutive
+         This option delays grace-period initialization for a few
+         jiffies between initializing each pair of consecutive
          rcu_node structures.  This helps to expose races involving
          grace-period initialization, in other words, it makes your
          kernel less stable.  It can also greatly increase grace-period
@@ -1286,6 +1312,30 @@ config RCU_TORTURE_TEST_SLOW_INIT_DELAY
          This option specifies the number of jiffies to wait between
          each rcu_node structure initialization.
 
+config RCU_TORTURE_TEST_SLOW_CLEANUP
+       bool "Slow down RCU grace-period cleanup to expose races"
+       depends on RCU_TORTURE_TEST
+       help
+         This option delays grace-period cleanup for a few jiffies
+         between cleaning up each pair of consecutive rcu_node
+         structures.  This helps to expose races involving grace-period
+         cleanup, in other words, it makes your kernel less stable.
+         It can also greatly increase grace-period latency, especially
+         on systems with large numbers of CPUs.  This is useful when
+         torture-testing RCU, but in almost no other circumstance.
+
+         Say Y here if you want your system to crash and hang more often.
+         Say N if you want a sane system.
+
+config RCU_TORTURE_TEST_SLOW_CLEANUP_DELAY
+       int "How much to slow down RCU grace-period cleanup"
+       range 0 5
+       default 3
+       depends on RCU_TORTURE_TEST_SLOW_CLEANUP
+       help
+         This option specifies the number of jiffies to wait between
+         each rcu_node structure cleanup operation.
+
 config RCU_CPU_STALL_TIMEOUT
        int "RCU CPU stall timeout in seconds"
        depends on RCU_STALL_COMMON

diff --git a/tools/testing/selftests/rcutorture/configs/rcu/CFcommon b/tools/testing/selftests/rcutorture/configs/rcu/CFcommon
index 49701218dc620481ff32344863411413e208da66..f824b4c9d9d9132d9681c5fe51c26caa76d4d8de 100644 (file)
--- a/tools/testing/selftests/rcutorture/configs/rcu/CFcommon
+++ b/tools/testing/selftests/rcutorture/configs/rcu/CFcommon
@@ -1,3 +1,5 @@
 CONFIG_RCU_TORTURE_TEST=y
 CONFIG_PRINTK_TIME=y
+CONFIG_RCU_TORTURE_TEST_SLOW_CLEANUP=y
 CONFIG_RCU_TORTURE_TEST_SLOW_INIT=y
+CONFIG_RCU_TORTURE_TEST_SLOW_PREINIT=y