2 * Module-based torture test facility for locking
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, you can access it online at
16 * http://www.gnu.org/licenses/gpl-2.0.html.
18 * Copyright (C) IBM Corporation, 2014
20 * Author: Paul E. McKenney <paulmck@us.ibm.com>
21 * Based on kernel/rcu/torture.c.
23 #include <linux/kernel.h>
24 #include <linux/module.h>
25 #include <linux/kthread.h>
26 #include <linux/spinlock.h>
27 #include <linux/rwlock.h>
28 #include <linux/mutex.h>
29 #include <linux/rwsem.h>
30 #include <linux/smp.h>
31 #include <linux/interrupt.h>
32 #include <linux/sched.h>
33 #include <linux/atomic.h>
34 #include <linux/moduleparam.h>
35 #include <linux/delay.h>
36 #include <linux/slab.h>
37 #include <linux/torture.h>
39 MODULE_LICENSE("GPL");
40 MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com>");
42 torture_param(int, nwriters_stress, -1,
43 "Number of write-locking stress-test threads");
44 torture_param(int, nreaders_stress, -1,
45 "Number of read-locking stress-test threads");
46 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)");
47 torture_param(int, onoff_interval, 0,
48 "Time between CPU hotplugs (s), 0=disable");
49 torture_param(int, shuffle_interval, 3,
50 "Number of jiffies between shuffles, 0=disable");
51 torture_param(int, shutdown_secs, 0, "Shutdown time (j), <= zero to disable.");
52 torture_param(int, stat_interval, 60,
53 "Number of seconds between stats printk()s");
54 torture_param(int, stutter, 5, "Number of jiffies to run/halt test, 0=disable");
55 torture_param(bool, verbose, true,
56 "Enable verbose debugging printk()s");
58 static char *torture_type = "spin_lock";
59 module_param(torture_type, charp, 0444);
60 MODULE_PARM_DESC(torture_type,
61 "Type of lock to torture (spin_lock, spin_lock_irq, mutex_lock, ...)");
63 static struct task_struct *stats_task;
64 static struct task_struct **writer_tasks;
65 static struct task_struct **reader_tasks;
67 static bool lock_is_write_held;
68 static bool lock_is_read_held;
70 struct lock_stress_stats {
76 #define LOCKTORTURE_RUNNABLE_INIT 1
78 #define LOCKTORTURE_RUNNABLE_INIT 0
80 int torture_runnable = LOCKTORTURE_RUNNABLE_INIT;
81 module_param(torture_runnable, int, 0444);
82 MODULE_PARM_DESC(torture_runnable, "Start locktorture at module init");
84 /* Forward reference. */
85 static void lock_torture_cleanup(void);
88 * Operations vector for selecting different types of tests.
90 struct lock_torture_ops {
92 int (*writelock)(void);
93 void (*write_delay)(struct torture_random_state *trsp);
94 void (*writeunlock)(void);
95 int (*readlock)(void);
96 void (*read_delay)(struct torture_random_state *trsp);
97 void (*readunlock)(void);
102 struct lock_torture_cxt {
103 int nrealwriters_stress;
104 int nrealreaders_stress;
106 atomic_t n_lock_torture_errors;
107 struct lock_torture_ops *cur_ops;
108 struct lock_stress_stats *lwsa; /* writer statistics */
109 struct lock_stress_stats *lrsa; /* reader statistics */
111 static struct lock_torture_cxt cxt = { 0, 0, false,
115 * Definitions for lock torture testing.
118 static int torture_lock_busted_write_lock(void)
120 return 0; /* BUGGY, do not use in real life!!! */
123 static void torture_lock_busted_write_delay(struct torture_random_state *trsp)
125 const unsigned long longdelay_us = 100;
127 /* We want a long delay occasionally to force massive contention. */
128 if (!(torture_random(trsp) %
129 (cxt.nrealwriters_stress * 2000 * longdelay_us)))
130 mdelay(longdelay_us);
131 #ifdef CONFIG_PREEMPT
132 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
133 preempt_schedule(); /* Allow test to be preempted. */
137 static void torture_lock_busted_write_unlock(void)
139 /* BUGGY, do not use in real life!!! */
142 static struct lock_torture_ops lock_busted_ops = {
143 .writelock = torture_lock_busted_write_lock,
144 .write_delay = torture_lock_busted_write_delay,
145 .writeunlock = torture_lock_busted_write_unlock,
149 .name = "lock_busted"
152 static DEFINE_SPINLOCK(torture_spinlock);
154 static int torture_spin_lock_write_lock(void) __acquires(torture_spinlock)
156 spin_lock(&torture_spinlock);
160 static void torture_spin_lock_write_delay(struct torture_random_state *trsp)
162 const unsigned long shortdelay_us = 2;
163 const unsigned long longdelay_us = 100;
165 /* We want a short delay mostly to emulate likely code, and
166 * we want a long delay occasionally to force massive contention.
168 if (!(torture_random(trsp) %
169 (cxt.nrealwriters_stress * 2000 * longdelay_us)))
170 mdelay(longdelay_us);
171 if (!(torture_random(trsp) %
172 (cxt.nrealwriters_stress * 2 * shortdelay_us)))
173 udelay(shortdelay_us);
174 #ifdef CONFIG_PREEMPT
175 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
176 preempt_schedule(); /* Allow test to be preempted. */
180 static void torture_spin_lock_write_unlock(void) __releases(torture_spinlock)
182 spin_unlock(&torture_spinlock);
185 static struct lock_torture_ops spin_lock_ops = {
186 .writelock = torture_spin_lock_write_lock,
187 .write_delay = torture_spin_lock_write_delay,
188 .writeunlock = torture_spin_lock_write_unlock,
195 static int torture_spin_lock_write_lock_irq(void)
196 __acquires(torture_spinlock)
200 spin_lock_irqsave(&torture_spinlock, flags);
201 cxt.cur_ops->flags = flags;
205 static void torture_lock_spin_write_unlock_irq(void)
206 __releases(torture_spinlock)
208 spin_unlock_irqrestore(&torture_spinlock, cxt.cur_ops->flags);
211 static struct lock_torture_ops spin_lock_irq_ops = {
212 .writelock = torture_spin_lock_write_lock_irq,
213 .write_delay = torture_spin_lock_write_delay,
214 .writeunlock = torture_lock_spin_write_unlock_irq,
218 .name = "spin_lock_irq"
221 static DEFINE_RWLOCK(torture_rwlock);
223 static int torture_rwlock_write_lock(void) __acquires(torture_rwlock)
225 write_lock(&torture_rwlock);
229 static void torture_rwlock_write_delay(struct torture_random_state *trsp)
231 const unsigned long shortdelay_us = 2;
232 const unsigned long longdelay_ms = 100;
234 /* We want a short delay mostly to emulate likely code, and
235 * we want a long delay occasionally to force massive contention.
237 if (!(torture_random(trsp) %
238 (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
239 mdelay(longdelay_ms);
241 udelay(shortdelay_us);
244 static void torture_rwlock_write_unlock(void) __releases(torture_rwlock)
246 write_unlock(&torture_rwlock);
249 static int torture_rwlock_read_lock(void) __acquires(torture_rwlock)
251 read_lock(&torture_rwlock);
255 static void torture_rwlock_read_delay(struct torture_random_state *trsp)
257 const unsigned long shortdelay_us = 10;
258 const unsigned long longdelay_ms = 100;
260 /* We want a short delay mostly to emulate likely code, and
261 * we want a long delay occasionally to force massive contention.
263 if (!(torture_random(trsp) %
264 (cxt.nrealreaders_stress * 2000 * longdelay_ms)))
265 mdelay(longdelay_ms);
267 udelay(shortdelay_us);
270 static void torture_rwlock_read_unlock(void) __releases(torture_rwlock)
272 read_unlock(&torture_rwlock);
275 static struct lock_torture_ops rw_lock_ops = {
276 .writelock = torture_rwlock_write_lock,
277 .write_delay = torture_rwlock_write_delay,
278 .writeunlock = torture_rwlock_write_unlock,
279 .readlock = torture_rwlock_read_lock,
280 .read_delay = torture_rwlock_read_delay,
281 .readunlock = torture_rwlock_read_unlock,
285 static int torture_rwlock_write_lock_irq(void) __acquires(torture_rwlock)
289 write_lock_irqsave(&torture_rwlock, flags);
290 cxt.cur_ops->flags = flags;
294 static void torture_rwlock_write_unlock_irq(void)
295 __releases(torture_rwlock)
297 write_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags);
300 static int torture_rwlock_read_lock_irq(void) __acquires(torture_rwlock)
304 read_lock_irqsave(&torture_rwlock, flags);
305 cxt.cur_ops->flags = flags;
309 static void torture_rwlock_read_unlock_irq(void)
310 __releases(torture_rwlock)
312 write_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags);
315 static struct lock_torture_ops rw_lock_irq_ops = {
316 .writelock = torture_rwlock_write_lock_irq,
317 .write_delay = torture_rwlock_write_delay,
318 .writeunlock = torture_rwlock_write_unlock_irq,
319 .readlock = torture_rwlock_read_lock_irq,
320 .read_delay = torture_rwlock_read_delay,
321 .readunlock = torture_rwlock_read_unlock_irq,
322 .name = "rw_lock_irq"
325 static DEFINE_MUTEX(torture_mutex);
327 static int torture_mutex_lock(void) __acquires(torture_mutex)
329 mutex_lock(&torture_mutex);
333 static void torture_mutex_delay(struct torture_random_state *trsp)
335 const unsigned long longdelay_ms = 100;
337 /* We want a long delay occasionally to force massive contention. */
338 if (!(torture_random(trsp) %
339 (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
340 mdelay(longdelay_ms * 5);
342 mdelay(longdelay_ms / 5);
343 #ifdef CONFIG_PREEMPT
344 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
345 preempt_schedule(); /* Allow test to be preempted. */
349 static void torture_mutex_unlock(void) __releases(torture_mutex)
351 mutex_unlock(&torture_mutex);
354 static struct lock_torture_ops mutex_lock_ops = {
355 .writelock = torture_mutex_lock,
356 .write_delay = torture_mutex_delay,
357 .writeunlock = torture_mutex_unlock,
364 static DECLARE_RWSEM(torture_rwsem);
365 static int torture_rwsem_down_write(void) __acquires(torture_rwsem)
367 down_write(&torture_rwsem);
371 static void torture_rwsem_write_delay(struct torture_random_state *trsp)
373 const unsigned long longdelay_ms = 100;
375 /* We want a long delay occasionally to force massive contention. */
376 if (!(torture_random(trsp) %
377 (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
378 mdelay(longdelay_ms * 10);
380 mdelay(longdelay_ms / 10);
381 #ifdef CONFIG_PREEMPT
382 if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
383 preempt_schedule(); /* Allow test to be preempted. */
387 static void torture_rwsem_up_write(void) __releases(torture_rwsem)
389 up_write(&torture_rwsem);
392 static int torture_rwsem_down_read(void) __acquires(torture_rwsem)
394 down_read(&torture_rwsem);
398 static void torture_rwsem_read_delay(struct torture_random_state *trsp)
400 const unsigned long longdelay_ms = 100;
402 /* We want a long delay occasionally to force massive contention. */
403 if (!(torture_random(trsp) %
404 (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
405 mdelay(longdelay_ms * 2);
407 mdelay(longdelay_ms / 2);
408 #ifdef CONFIG_PREEMPT
409 if (!(torture_random(trsp) % (cxt.nrealreaders_stress * 20000)))
410 preempt_schedule(); /* Allow test to be preempted. */
414 static void torture_rwsem_up_read(void) __releases(torture_rwsem)
416 up_read(&torture_rwsem);
419 static struct lock_torture_ops rwsem_lock_ops = {
420 .writelock = torture_rwsem_down_write,
421 .write_delay = torture_rwsem_write_delay,
422 .writeunlock = torture_rwsem_up_write,
423 .readlock = torture_rwsem_down_read,
424 .read_delay = torture_rwsem_read_delay,
425 .readunlock = torture_rwsem_up_read,
430 * Lock torture writer kthread. Repeatedly acquires and releases
431 * the lock, checking for duplicate acquisitions.
433 static int lock_torture_writer(void *arg)
435 struct lock_stress_stats *lwsp = arg;
436 static DEFINE_TORTURE_RANDOM(rand);
438 VERBOSE_TOROUT_STRING("lock_torture_writer task started");
439 set_user_nice(current, MAX_NICE);
442 if ((torture_random(&rand) & 0xfffff) == 0)
443 schedule_timeout_uninterruptible(1);
445 cxt.cur_ops->writelock();
446 if (WARN_ON_ONCE(lock_is_write_held))
448 lock_is_write_held = 1;
449 if (WARN_ON_ONCE(lock_is_read_held))
450 lwsp->n_lock_fail++; /* rare, but... */
452 lwsp->n_lock_acquired++;
453 cxt.cur_ops->write_delay(&rand);
454 lock_is_write_held = 0;
455 cxt.cur_ops->writeunlock();
457 stutter_wait("lock_torture_writer");
458 } while (!torture_must_stop());
459 torture_kthread_stopping("lock_torture_writer");
464 * Lock torture reader kthread. Repeatedly acquires and releases
467 static int lock_torture_reader(void *arg)
469 struct lock_stress_stats *lrsp = arg;
470 static DEFINE_TORTURE_RANDOM(rand);
472 VERBOSE_TOROUT_STRING("lock_torture_reader task started");
473 set_user_nice(current, MAX_NICE);
476 if ((torture_random(&rand) & 0xfffff) == 0)
477 schedule_timeout_uninterruptible(1);
479 cxt.cur_ops->readlock();
480 lock_is_read_held = 1;
481 if (WARN_ON_ONCE(lock_is_write_held))
482 lrsp->n_lock_fail++; /* rare, but... */
484 lrsp->n_lock_acquired++;
485 cxt.cur_ops->read_delay(&rand);
486 lock_is_read_held = 0;
487 cxt.cur_ops->readunlock();
489 stutter_wait("lock_torture_reader");
490 } while (!torture_must_stop());
491 torture_kthread_stopping("lock_torture_reader");
496 * Create an lock-torture-statistics message in the specified buffer.
498 static void __torture_print_stats(char *page,
499 struct lock_stress_stats *statp, bool write)
504 long min = statp[0].n_lock_acquired;
507 n_stress = write ? cxt.nrealwriters_stress : cxt.nrealreaders_stress;
508 for (i = 0; i < n_stress; i++) {
509 if (statp[i].n_lock_fail)
511 sum += statp[i].n_lock_acquired;
512 if (max < statp[i].n_lock_fail)
513 max = statp[i].n_lock_fail;
514 if (min > statp[i].n_lock_fail)
515 min = statp[i].n_lock_fail;
517 page += sprintf(page,
518 "%s: Total: %lld Max/Min: %ld/%ld %s Fail: %d %s\n",
519 write ? "Writes" : "Reads ",
520 sum, max, min, max / 2 > min ? "???" : "",
521 fail, fail ? "!!!" : "");
523 atomic_inc(&cxt.n_lock_torture_errors);
527 * Print torture statistics. Caller must ensure that there is only one
528 * call to this function at a given time!!! This is normally accomplished
529 * by relying on the module system to only have one copy of the module
530 * loaded, and then by giving the lock_torture_stats kthread full control
531 * (or the init/cleanup functions when lock_torture_stats thread is not
534 static void lock_torture_stats_print(void)
536 int size = cxt.nrealwriters_stress * 200 + 8192;
539 if (cxt.cur_ops->readlock)
540 size += cxt.nrealreaders_stress * 200 + 8192;
542 buf = kmalloc(size, GFP_KERNEL);
544 pr_err("lock_torture_stats_print: Out of memory, need: %d",
549 __torture_print_stats(buf, cxt.lwsa, true);
553 if (cxt.cur_ops->readlock) {
554 buf = kmalloc(size, GFP_KERNEL);
556 pr_err("lock_torture_stats_print: Out of memory, need: %d",
561 __torture_print_stats(buf, cxt.lrsa, false);
568 * Periodically prints torture statistics, if periodic statistics printing
569 * was specified via the stat_interval module parameter.
571 * No need to worry about fullstop here, since this one doesn't reference
572 * volatile state or register callbacks.
574 static int lock_torture_stats(void *arg)
576 VERBOSE_TOROUT_STRING("lock_torture_stats task started");
578 schedule_timeout_interruptible(stat_interval * HZ);
579 lock_torture_stats_print();
580 torture_shutdown_absorb("lock_torture_stats");
581 } while (!torture_must_stop());
582 torture_kthread_stopping("lock_torture_stats");
587 lock_torture_print_module_parms(struct lock_torture_ops *cur_ops,
590 pr_alert("%s" TORTURE_FLAG
591 "--- %s%s: nwriters_stress=%d nreaders_stress=%d stat_interval=%d verbose=%d shuffle_interval=%d stutter=%d shutdown_secs=%d onoff_interval=%d onoff_holdoff=%d\n",
592 torture_type, tag, cxt.debug_lock ? " [debug]": "",
593 cxt.nrealwriters_stress, cxt.nrealreaders_stress, stat_interval,
594 verbose, shuffle_interval, stutter, shutdown_secs,
595 onoff_interval, onoff_holdoff);
598 static void lock_torture_cleanup(void)
602 if (torture_cleanup_begin())
606 for (i = 0; i < cxt.nrealwriters_stress; i++)
607 torture_stop_kthread(lock_torture_writer,
614 for (i = 0; i < cxt.nrealreaders_stress; i++)
615 torture_stop_kthread(lock_torture_reader,
621 torture_stop_kthread(lock_torture_stats, stats_task);
622 lock_torture_stats_print(); /* -After- the stats thread is stopped! */
624 if (atomic_read(&cxt.n_lock_torture_errors))
625 lock_torture_print_module_parms(cxt.cur_ops,
626 "End of test: FAILURE");
627 else if (torture_onoff_failures())
628 lock_torture_print_module_parms(cxt.cur_ops,
629 "End of test: LOCK_HOTPLUG");
631 lock_torture_print_module_parms(cxt.cur_ops,
632 "End of test: SUCCESS");
633 torture_cleanup_end();
636 static int __init lock_torture_init(void)
640 static struct lock_torture_ops *torture_ops[] = {
642 &spin_lock_ops, &spin_lock_irq_ops,
643 &rw_lock_ops, &rw_lock_irq_ops,
648 if (!torture_init_begin(torture_type, verbose, &torture_runnable))
651 /* Process args and tell the world that the torturer is on the job. */
652 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
653 cxt.cur_ops = torture_ops[i];
654 if (strcmp(torture_type, cxt.cur_ops->name) == 0)
657 if (i == ARRAY_SIZE(torture_ops)) {
658 pr_alert("lock-torture: invalid torture type: \"%s\"\n",
660 pr_alert("lock-torture types:");
661 for (i = 0; i < ARRAY_SIZE(torture_ops); i++)
662 pr_alert(" %s", torture_ops[i]->name);
667 if (cxt.cur_ops->init)
668 cxt.cur_ops->init(); /* no "goto unwind" prior to this point!!! */
670 if (nwriters_stress >= 0)
671 cxt.nrealwriters_stress = nwriters_stress;
673 cxt.nrealwriters_stress = 2 * num_online_cpus();
675 #ifdef CONFIG_DEBUG_MUTEXES
676 if (strncmp(torture_type, "mutex", 5) == 0)
677 cxt.debug_lock = true;
679 #ifdef CONFIG_DEBUG_SPINLOCK
680 if ((strncmp(torture_type, "spin", 4) == 0) ||
681 (strncmp(torture_type, "rw_lock", 7) == 0))
682 cxt.debug_lock = true;
685 /* Initialize the statistics so that each run gets its own numbers. */
687 lock_is_write_held = 0;
688 cxt.lwsa = kmalloc(sizeof(*cxt.lwsa) * cxt.nrealwriters_stress, GFP_KERNEL);
689 if (cxt.lwsa == NULL) {
690 VERBOSE_TOROUT_STRING("cxt.lwsa: Out of memory");
694 for (i = 0; i < cxt.nrealwriters_stress; i++) {
695 cxt.lwsa[i].n_lock_fail = 0;
696 cxt.lwsa[i].n_lock_acquired = 0;
699 if (cxt.cur_ops->readlock) {
700 if (nreaders_stress >= 0)
701 cxt.nrealreaders_stress = nreaders_stress;
704 * By default distribute evenly the number of
705 * readers and writers. We still run the same number
706 * of threads as the writer-only locks default.
708 if (nwriters_stress < 0) /* user doesn't care */
709 cxt.nrealwriters_stress = num_online_cpus();
710 cxt.nrealreaders_stress = cxt.nrealwriters_stress;
713 lock_is_read_held = 0;
714 cxt.lrsa = kmalloc(sizeof(*cxt.lrsa) * cxt.nrealreaders_stress, GFP_KERNEL);
715 if (cxt.lrsa == NULL) {
716 VERBOSE_TOROUT_STRING("cxt.lrsa: Out of memory");
722 for (i = 0; i < cxt.nrealreaders_stress; i++) {
723 cxt.lrsa[i].n_lock_fail = 0;
724 cxt.lrsa[i].n_lock_acquired = 0;
727 lock_torture_print_module_parms(cxt.cur_ops, "Start of test");
729 /* Prepare torture context. */
730 if (onoff_interval > 0) {
731 firsterr = torture_onoff_init(onoff_holdoff * HZ,
732 onoff_interval * HZ);
736 if (shuffle_interval > 0) {
737 firsterr = torture_shuffle_init(shuffle_interval);
741 if (shutdown_secs > 0) {
742 firsterr = torture_shutdown_init(shutdown_secs,
743 lock_torture_cleanup);
748 firsterr = torture_stutter_init(stutter);
753 writer_tasks = kzalloc(cxt.nrealwriters_stress * sizeof(writer_tasks[0]),
755 if (writer_tasks == NULL) {
756 VERBOSE_TOROUT_ERRSTRING("writer_tasks: Out of memory");
761 if (cxt.cur_ops->readlock) {
762 reader_tasks = kzalloc(cxt.nrealreaders_stress * sizeof(reader_tasks[0]),
764 if (reader_tasks == NULL) {
765 VERBOSE_TOROUT_ERRSTRING("reader_tasks: Out of memory");
772 * Create the kthreads and start torturing (oh, those poor little locks).
774 * TODO: Note that we interleave writers with readers, giving writers a
775 * slight advantage, by creating its kthread first. This can be modified
776 * for very specific needs, or even let the user choose the policy, if
779 for (i = 0, j = 0; i < cxt.nrealwriters_stress ||
780 j < cxt.nrealreaders_stress; i++, j++) {
781 if (i >= cxt.nrealwriters_stress)
785 firsterr = torture_create_kthread(lock_torture_writer, &cxt.lwsa[i],
791 if (cxt.cur_ops->readlock == NULL || (j >= cxt.nrealreaders_stress))
794 firsterr = torture_create_kthread(lock_torture_reader, &cxt.lrsa[j],
799 if (stat_interval > 0) {
800 firsterr = torture_create_kthread(lock_torture_stats, NULL,
810 lock_torture_cleanup();
814 module_init(lock_torture_init);
815 module_exit(lock_torture_cleanup);