]> git.karo-electronics.de Git - karo-tx-linux.git/blob - kernel/locking/locktorture.c
locktorture: Cleanup header usage
[karo-tx-linux.git] / kernel / locking / locktorture.c
1 /*
2  * Module-based torture test facility for locking
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License as published by
6  * the Free Software Foundation; either version 2 of the License, or
7  * (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, you can access it online at
16  * http://www.gnu.org/licenses/gpl-2.0.html.
17  *
18  * Copyright (C) IBM Corporation, 2014
19  *
20  * Author: Paul E. McKenney <paulmck@us.ibm.com>
21  *      Based on kernel/rcu/torture.c.
22  */
23 #include <linux/kernel.h>
24 #include <linux/module.h>
25 #include <linux/kthread.h>
26 #include <linux/spinlock.h>
27 #include <linux/rwlock.h>
28 #include <linux/mutex.h>
29 #include <linux/rwsem.h>
30 #include <linux/smp.h>
31 #include <linux/interrupt.h>
32 #include <linux/sched.h>
33 #include <linux/atomic.h>
34 #include <linux/moduleparam.h>
35 #include <linux/delay.h>
36 #include <linux/slab.h>
37 #include <linux/torture.h>
38
39 MODULE_LICENSE("GPL");
40 MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com>");
41
42 torture_param(int, nwriters_stress, -1,
43              "Number of write-locking stress-test threads");
44 torture_param(int, nreaders_stress, -1,
45              "Number of read-locking stress-test threads");
46 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)");
47 torture_param(int, onoff_interval, 0,
48              "Time between CPU hotplugs (s), 0=disable");
49 torture_param(int, shuffle_interval, 3,
50              "Number of jiffies between shuffles, 0=disable");
51 torture_param(int, shutdown_secs, 0, "Shutdown time (j), <= zero to disable.");
52 torture_param(int, stat_interval, 60,
53              "Number of seconds between stats printk()s");
54 torture_param(int, stutter, 5, "Number of jiffies to run/halt test, 0=disable");
55 torture_param(bool, verbose, true,
56              "Enable verbose debugging printk()s");
57
58 static char *torture_type = "spin_lock";
59 module_param(torture_type, charp, 0444);
60 MODULE_PARM_DESC(torture_type,
61                  "Type of lock to torture (spin_lock, spin_lock_irq, mutex_lock, ...)");
62
63 static struct task_struct *stats_task;
64 static struct task_struct **writer_tasks;
65 static struct task_struct **reader_tasks;
66
67 static bool lock_is_write_held;
68 static bool lock_is_read_held;
69
70 struct lock_stress_stats {
71         long n_lock_fail;
72         long n_lock_acquired;
73 };
74
75 #if defined(MODULE)
76 #define LOCKTORTURE_RUNNABLE_INIT 1
77 #else
78 #define LOCKTORTURE_RUNNABLE_INIT 0
79 #endif
80 int torture_runnable = LOCKTORTURE_RUNNABLE_INIT;
81 module_param(torture_runnable, int, 0444);
82 MODULE_PARM_DESC(torture_runnable, "Start locktorture at module init");
83
84 /* Forward reference. */
85 static void lock_torture_cleanup(void);
86
87 /*
88  * Operations vector for selecting different types of tests.
89  */
90 struct lock_torture_ops {
91         void (*init)(void);
92         int (*writelock)(void);
93         void (*write_delay)(struct torture_random_state *trsp);
94         void (*writeunlock)(void);
95         int (*readlock)(void);
96         void (*read_delay)(struct torture_random_state *trsp);
97         void (*readunlock)(void);
98         unsigned long flags;
99         const char *name;
100 };
101
102 struct lock_torture_cxt {
103         int nrealwriters_stress;
104         int nrealreaders_stress;
105         bool debug_lock;
106         atomic_t n_lock_torture_errors;
107         struct lock_torture_ops *cur_ops;
108         struct lock_stress_stats *lwsa; /* writer statistics */
109         struct lock_stress_stats *lrsa; /* reader statistics */
110 };
111 static struct lock_torture_cxt cxt = { 0, 0, false,
112                                        ATOMIC_INIT(0),
113                                        NULL, NULL};
114 /*
115  * Definitions for lock torture testing.
116  */
117
118 static int torture_lock_busted_write_lock(void)
119 {
120         return 0;  /* BUGGY, do not use in real life!!! */
121 }
122
123 static void torture_lock_busted_write_delay(struct torture_random_state *trsp)
124 {
125         const unsigned long longdelay_us = 100;
126
127         /* We want a long delay occasionally to force massive contention.  */
128         if (!(torture_random(trsp) %
129               (cxt.nrealwriters_stress * 2000 * longdelay_us)))
130                 mdelay(longdelay_us);
131 #ifdef CONFIG_PREEMPT
132         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
133                 preempt_schedule();  /* Allow test to be preempted. */
134 #endif
135 }
136
137 static void torture_lock_busted_write_unlock(void)
138 {
139           /* BUGGY, do not use in real life!!! */
140 }
141
142 static struct lock_torture_ops lock_busted_ops = {
143         .writelock      = torture_lock_busted_write_lock,
144         .write_delay    = torture_lock_busted_write_delay,
145         .writeunlock    = torture_lock_busted_write_unlock,
146         .readlock       = NULL,
147         .read_delay     = NULL,
148         .readunlock     = NULL,
149         .name           = "lock_busted"
150 };
151
152 static DEFINE_SPINLOCK(torture_spinlock);
153
154 static int torture_spin_lock_write_lock(void) __acquires(torture_spinlock)
155 {
156         spin_lock(&torture_spinlock);
157         return 0;
158 }
159
160 static void torture_spin_lock_write_delay(struct torture_random_state *trsp)
161 {
162         const unsigned long shortdelay_us = 2;
163         const unsigned long longdelay_us = 100;
164
165         /* We want a short delay mostly to emulate likely code, and
166          * we want a long delay occasionally to force massive contention.
167          */
168         if (!(torture_random(trsp) %
169               (cxt.nrealwriters_stress * 2000 * longdelay_us)))
170                 mdelay(longdelay_us);
171         if (!(torture_random(trsp) %
172               (cxt.nrealwriters_stress * 2 * shortdelay_us)))
173                 udelay(shortdelay_us);
174 #ifdef CONFIG_PREEMPT
175         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
176                 preempt_schedule();  /* Allow test to be preempted. */
177 #endif
178 }
179
180 static void torture_spin_lock_write_unlock(void) __releases(torture_spinlock)
181 {
182         spin_unlock(&torture_spinlock);
183 }
184
185 static struct lock_torture_ops spin_lock_ops = {
186         .writelock      = torture_spin_lock_write_lock,
187         .write_delay    = torture_spin_lock_write_delay,
188         .writeunlock    = torture_spin_lock_write_unlock,
189         .readlock       = NULL,
190         .read_delay     = NULL,
191         .readunlock     = NULL,
192         .name           = "spin_lock"
193 };
194
195 static int torture_spin_lock_write_lock_irq(void)
196 __acquires(torture_spinlock)
197 {
198         unsigned long flags;
199
200         spin_lock_irqsave(&torture_spinlock, flags);
201         cxt.cur_ops->flags = flags;
202         return 0;
203 }
204
205 static void torture_lock_spin_write_unlock_irq(void)
206 __releases(torture_spinlock)
207 {
208         spin_unlock_irqrestore(&torture_spinlock, cxt.cur_ops->flags);
209 }
210
211 static struct lock_torture_ops spin_lock_irq_ops = {
212         .writelock      = torture_spin_lock_write_lock_irq,
213         .write_delay    = torture_spin_lock_write_delay,
214         .writeunlock    = torture_lock_spin_write_unlock_irq,
215         .readlock       = NULL,
216         .read_delay     = NULL,
217         .readunlock     = NULL,
218         .name           = "spin_lock_irq"
219 };
220
221 static DEFINE_RWLOCK(torture_rwlock);
222
223 static int torture_rwlock_write_lock(void) __acquires(torture_rwlock)
224 {
225         write_lock(&torture_rwlock);
226         return 0;
227 }
228
229 static void torture_rwlock_write_delay(struct torture_random_state *trsp)
230 {
231         const unsigned long shortdelay_us = 2;
232         const unsigned long longdelay_ms = 100;
233
234         /* We want a short delay mostly to emulate likely code, and
235          * we want a long delay occasionally to force massive contention.
236          */
237         if (!(torture_random(trsp) %
238               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
239                 mdelay(longdelay_ms);
240         else
241                 udelay(shortdelay_us);
242 }
243
244 static void torture_rwlock_write_unlock(void) __releases(torture_rwlock)
245 {
246         write_unlock(&torture_rwlock);
247 }
248
249 static int torture_rwlock_read_lock(void) __acquires(torture_rwlock)
250 {
251         read_lock(&torture_rwlock);
252         return 0;
253 }
254
255 static void torture_rwlock_read_delay(struct torture_random_state *trsp)
256 {
257         const unsigned long shortdelay_us = 10;
258         const unsigned long longdelay_ms = 100;
259
260         /* We want a short delay mostly to emulate likely code, and
261          * we want a long delay occasionally to force massive contention.
262          */
263         if (!(torture_random(trsp) %
264               (cxt.nrealreaders_stress * 2000 * longdelay_ms)))
265                 mdelay(longdelay_ms);
266         else
267                 udelay(shortdelay_us);
268 }
269
270 static void torture_rwlock_read_unlock(void) __releases(torture_rwlock)
271 {
272         read_unlock(&torture_rwlock);
273 }
274
275 static struct lock_torture_ops rw_lock_ops = {
276         .writelock      = torture_rwlock_write_lock,
277         .write_delay    = torture_rwlock_write_delay,
278         .writeunlock    = torture_rwlock_write_unlock,
279         .readlock       = torture_rwlock_read_lock,
280         .read_delay     = torture_rwlock_read_delay,
281         .readunlock     = torture_rwlock_read_unlock,
282         .name           = "rw_lock"
283 };
284
285 static int torture_rwlock_write_lock_irq(void) __acquires(torture_rwlock)
286 {
287         unsigned long flags;
288
289         write_lock_irqsave(&torture_rwlock, flags);
290         cxt.cur_ops->flags = flags;
291         return 0;
292 }
293
294 static void torture_rwlock_write_unlock_irq(void)
295 __releases(torture_rwlock)
296 {
297         write_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags);
298 }
299
300 static int torture_rwlock_read_lock_irq(void) __acquires(torture_rwlock)
301 {
302         unsigned long flags;
303
304         read_lock_irqsave(&torture_rwlock, flags);
305         cxt.cur_ops->flags = flags;
306         return 0;
307 }
308
309 static void torture_rwlock_read_unlock_irq(void)
310 __releases(torture_rwlock)
311 {
312         write_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags);
313 }
314
315 static struct lock_torture_ops rw_lock_irq_ops = {
316         .writelock      = torture_rwlock_write_lock_irq,
317         .write_delay    = torture_rwlock_write_delay,
318         .writeunlock    = torture_rwlock_write_unlock_irq,
319         .readlock       = torture_rwlock_read_lock_irq,
320         .read_delay     = torture_rwlock_read_delay,
321         .readunlock     = torture_rwlock_read_unlock_irq,
322         .name           = "rw_lock_irq"
323 };
324
325 static DEFINE_MUTEX(torture_mutex);
326
327 static int torture_mutex_lock(void) __acquires(torture_mutex)
328 {
329         mutex_lock(&torture_mutex);
330         return 0;
331 }
332
333 static void torture_mutex_delay(struct torture_random_state *trsp)
334 {
335         const unsigned long longdelay_ms = 100;
336
337         /* We want a long delay occasionally to force massive contention.  */
338         if (!(torture_random(trsp) %
339               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
340                 mdelay(longdelay_ms * 5);
341         else
342                 mdelay(longdelay_ms / 5);
343 #ifdef CONFIG_PREEMPT
344         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
345                 preempt_schedule();  /* Allow test to be preempted. */
346 #endif
347 }
348
349 static void torture_mutex_unlock(void) __releases(torture_mutex)
350 {
351         mutex_unlock(&torture_mutex);
352 }
353
354 static struct lock_torture_ops mutex_lock_ops = {
355         .writelock      = torture_mutex_lock,
356         .write_delay    = torture_mutex_delay,
357         .writeunlock    = torture_mutex_unlock,
358         .readlock       = NULL,
359         .read_delay     = NULL,
360         .readunlock     = NULL,
361         .name           = "mutex_lock"
362 };
363
364 static DECLARE_RWSEM(torture_rwsem);
365 static int torture_rwsem_down_write(void) __acquires(torture_rwsem)
366 {
367         down_write(&torture_rwsem);
368         return 0;
369 }
370
371 static void torture_rwsem_write_delay(struct torture_random_state *trsp)
372 {
373         const unsigned long longdelay_ms = 100;
374
375         /* We want a long delay occasionally to force massive contention.  */
376         if (!(torture_random(trsp) %
377               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
378                 mdelay(longdelay_ms * 10);
379         else
380                 mdelay(longdelay_ms / 10);
381 #ifdef CONFIG_PREEMPT
382         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
383                 preempt_schedule();  /* Allow test to be preempted. */
384 #endif
385 }
386
387 static void torture_rwsem_up_write(void) __releases(torture_rwsem)
388 {
389         up_write(&torture_rwsem);
390 }
391
392 static int torture_rwsem_down_read(void) __acquires(torture_rwsem)
393 {
394         down_read(&torture_rwsem);
395         return 0;
396 }
397
398 static void torture_rwsem_read_delay(struct torture_random_state *trsp)
399 {
400         const unsigned long longdelay_ms = 100;
401
402         /* We want a long delay occasionally to force massive contention.  */
403         if (!(torture_random(trsp) %
404               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
405                 mdelay(longdelay_ms * 2);
406         else
407                 mdelay(longdelay_ms / 2);
408 #ifdef CONFIG_PREEMPT
409         if (!(torture_random(trsp) % (cxt.nrealreaders_stress * 20000)))
410                 preempt_schedule();  /* Allow test to be preempted. */
411 #endif
412 }
413
414 static void torture_rwsem_up_read(void) __releases(torture_rwsem)
415 {
416         up_read(&torture_rwsem);
417 }
418
419 static struct lock_torture_ops rwsem_lock_ops = {
420         .writelock      = torture_rwsem_down_write,
421         .write_delay    = torture_rwsem_write_delay,
422         .writeunlock    = torture_rwsem_up_write,
423         .readlock       = torture_rwsem_down_read,
424         .read_delay     = torture_rwsem_read_delay,
425         .readunlock     = torture_rwsem_up_read,
426         .name           = "rwsem_lock"
427 };
428
429 /*
430  * Lock torture writer kthread.  Repeatedly acquires and releases
431  * the lock, checking for duplicate acquisitions.
432  */
433 static int lock_torture_writer(void *arg)
434 {
435         struct lock_stress_stats *lwsp = arg;
436         static DEFINE_TORTURE_RANDOM(rand);
437
438         VERBOSE_TOROUT_STRING("lock_torture_writer task started");
439         set_user_nice(current, MAX_NICE);
440
441         do {
442                 if ((torture_random(&rand) & 0xfffff) == 0)
443                         schedule_timeout_uninterruptible(1);
444
445                 cxt.cur_ops->writelock();
446                 if (WARN_ON_ONCE(lock_is_write_held))
447                         lwsp->n_lock_fail++;
448                 lock_is_write_held = 1;
449                 if (WARN_ON_ONCE(lock_is_read_held))
450                         lwsp->n_lock_fail++; /* rare, but... */
451
452                 lwsp->n_lock_acquired++;
453                 cxt.cur_ops->write_delay(&rand);
454                 lock_is_write_held = 0;
455                 cxt.cur_ops->writeunlock();
456
457                 stutter_wait("lock_torture_writer");
458         } while (!torture_must_stop());
459         torture_kthread_stopping("lock_torture_writer");
460         return 0;
461 }
462
463 /*
464  * Lock torture reader kthread.  Repeatedly acquires and releases
465  * the reader lock.
466  */
467 static int lock_torture_reader(void *arg)
468 {
469         struct lock_stress_stats *lrsp = arg;
470         static DEFINE_TORTURE_RANDOM(rand);
471
472         VERBOSE_TOROUT_STRING("lock_torture_reader task started");
473         set_user_nice(current, MAX_NICE);
474
475         do {
476                 if ((torture_random(&rand) & 0xfffff) == 0)
477                         schedule_timeout_uninterruptible(1);
478
479                 cxt.cur_ops->readlock();
480                 lock_is_read_held = 1;
481                 if (WARN_ON_ONCE(lock_is_write_held))
482                         lrsp->n_lock_fail++; /* rare, but... */
483
484                 lrsp->n_lock_acquired++;
485                 cxt.cur_ops->read_delay(&rand);
486                 lock_is_read_held = 0;
487                 cxt.cur_ops->readunlock();
488
489                 stutter_wait("lock_torture_reader");
490         } while (!torture_must_stop());
491         torture_kthread_stopping("lock_torture_reader");
492         return 0;
493 }
494
495 /*
496  * Create an lock-torture-statistics message in the specified buffer.
497  */
498 static void __torture_print_stats(char *page,
499                                   struct lock_stress_stats *statp, bool write)
500 {
501         bool fail = 0;
502         int i, n_stress;
503         long max = 0;
504         long min = statp[0].n_lock_acquired;
505         long long sum = 0;
506
507         n_stress = write ? cxt.nrealwriters_stress : cxt.nrealreaders_stress;
508         for (i = 0; i < n_stress; i++) {
509                 if (statp[i].n_lock_fail)
510                         fail = true;
511                 sum += statp[i].n_lock_acquired;
512                 if (max < statp[i].n_lock_fail)
513                         max = statp[i].n_lock_fail;
514                 if (min > statp[i].n_lock_fail)
515                         min = statp[i].n_lock_fail;
516         }
517         page += sprintf(page,
518                         "%s:  Total: %lld  Max/Min: %ld/%ld %s  Fail: %d %s\n",
519                         write ? "Writes" : "Reads ",
520                         sum, max, min, max / 2 > min ? "???" : "",
521                         fail, fail ? "!!!" : "");
522         if (fail)
523                 atomic_inc(&cxt.n_lock_torture_errors);
524 }
525
526 /*
527  * Print torture statistics.  Caller must ensure that there is only one
528  * call to this function at a given time!!!  This is normally accomplished
529  * by relying on the module system to only have one copy of the module
530  * loaded, and then by giving the lock_torture_stats kthread full control
531  * (or the init/cleanup functions when lock_torture_stats thread is not
532  * running).
533  */
534 static void lock_torture_stats_print(void)
535 {
536         int size = cxt.nrealwriters_stress * 200 + 8192;
537         char *buf;
538
539         if (cxt.cur_ops->readlock)
540                 size += cxt.nrealreaders_stress * 200 + 8192;
541
542         buf = kmalloc(size, GFP_KERNEL);
543         if (!buf) {
544                 pr_err("lock_torture_stats_print: Out of memory, need: %d",
545                        size);
546                 return;
547         }
548
549         __torture_print_stats(buf, cxt.lwsa, true);
550         pr_alert("%s", buf);
551         kfree(buf);
552
553         if (cxt.cur_ops->readlock) {
554                 buf = kmalloc(size, GFP_KERNEL);
555                 if (!buf) {
556                         pr_err("lock_torture_stats_print: Out of memory, need: %d",
557                                size);
558                         return;
559                 }
560
561                 __torture_print_stats(buf, cxt.lrsa, false);
562                 pr_alert("%s", buf);
563                 kfree(buf);
564         }
565 }
566
567 /*
568  * Periodically prints torture statistics, if periodic statistics printing
569  * was specified via the stat_interval module parameter.
570  *
571  * No need to worry about fullstop here, since this one doesn't reference
572  * volatile state or register callbacks.
573  */
574 static int lock_torture_stats(void *arg)
575 {
576         VERBOSE_TOROUT_STRING("lock_torture_stats task started");
577         do {
578                 schedule_timeout_interruptible(stat_interval * HZ);
579                 lock_torture_stats_print();
580                 torture_shutdown_absorb("lock_torture_stats");
581         } while (!torture_must_stop());
582         torture_kthread_stopping("lock_torture_stats");
583         return 0;
584 }
585
586 static inline void
587 lock_torture_print_module_parms(struct lock_torture_ops *cur_ops,
588                                 const char *tag)
589 {
590         pr_alert("%s" TORTURE_FLAG
591                  "--- %s%s: nwriters_stress=%d nreaders_stress=%d stat_interval=%d verbose=%d shuffle_interval=%d stutter=%d shutdown_secs=%d onoff_interval=%d onoff_holdoff=%d\n",
592                  torture_type, tag, cxt.debug_lock ? " [debug]": "",
593                  cxt.nrealwriters_stress, cxt.nrealreaders_stress, stat_interval,
594                  verbose, shuffle_interval, stutter, shutdown_secs,
595                  onoff_interval, onoff_holdoff);
596 }
597
598 static void lock_torture_cleanup(void)
599 {
600         int i;
601
602         if (torture_cleanup_begin())
603                 return;
604
605         if (writer_tasks) {
606                 for (i = 0; i < cxt.nrealwriters_stress; i++)
607                         torture_stop_kthread(lock_torture_writer,
608                                              writer_tasks[i]);
609                 kfree(writer_tasks);
610                 writer_tasks = NULL;
611         }
612
613         if (reader_tasks) {
614                 for (i = 0; i < cxt.nrealreaders_stress; i++)
615                         torture_stop_kthread(lock_torture_reader,
616                                              reader_tasks[i]);
617                 kfree(reader_tasks);
618                 reader_tasks = NULL;
619         }
620
621         torture_stop_kthread(lock_torture_stats, stats_task);
622         lock_torture_stats_print();  /* -After- the stats thread is stopped! */
623
624         if (atomic_read(&cxt.n_lock_torture_errors))
625                 lock_torture_print_module_parms(cxt.cur_ops,
626                                                 "End of test: FAILURE");
627         else if (torture_onoff_failures())
628                 lock_torture_print_module_parms(cxt.cur_ops,
629                                                 "End of test: LOCK_HOTPLUG");
630         else
631                 lock_torture_print_module_parms(cxt.cur_ops,
632                                                 "End of test: SUCCESS");
633         torture_cleanup_end();
634 }
635
636 static int __init lock_torture_init(void)
637 {
638         int i, j;
639         int firsterr = 0;
640         static struct lock_torture_ops *torture_ops[] = {
641                 &lock_busted_ops,
642                 &spin_lock_ops, &spin_lock_irq_ops,
643                 &rw_lock_ops, &rw_lock_irq_ops,
644                 &mutex_lock_ops,
645                 &rwsem_lock_ops,
646         };
647
648         if (!torture_init_begin(torture_type, verbose, &torture_runnable))
649                 return -EBUSY;
650
651         /* Process args and tell the world that the torturer is on the job. */
652         for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
653                 cxt.cur_ops = torture_ops[i];
654                 if (strcmp(torture_type, cxt.cur_ops->name) == 0)
655                         break;
656         }
657         if (i == ARRAY_SIZE(torture_ops)) {
658                 pr_alert("lock-torture: invalid torture type: \"%s\"\n",
659                          torture_type);
660                 pr_alert("lock-torture types:");
661                 for (i = 0; i < ARRAY_SIZE(torture_ops); i++)
662                         pr_alert(" %s", torture_ops[i]->name);
663                 pr_alert("\n");
664                 torture_init_end();
665                 return -EINVAL;
666         }
667         if (cxt.cur_ops->init)
668                 cxt.cur_ops->init(); /* no "goto unwind" prior to this point!!! */
669
670         if (nwriters_stress >= 0)
671                 cxt.nrealwriters_stress = nwriters_stress;
672         else
673                 cxt.nrealwriters_stress = 2 * num_online_cpus();
674
675 #ifdef CONFIG_DEBUG_MUTEXES
676         if (strncmp(torture_type, "mutex", 5) == 0)
677                 cxt.debug_lock = true;
678 #endif
679 #ifdef CONFIG_DEBUG_SPINLOCK
680         if ((strncmp(torture_type, "spin", 4) == 0) ||
681             (strncmp(torture_type, "rw_lock", 7) == 0))
682                 cxt.debug_lock = true;
683 #endif
684
685         /* Initialize the statistics so that each run gets its own numbers. */
686
687         lock_is_write_held = 0;
688         cxt.lwsa = kmalloc(sizeof(*cxt.lwsa) * cxt.nrealwriters_stress, GFP_KERNEL);
689         if (cxt.lwsa == NULL) {
690                 VERBOSE_TOROUT_STRING("cxt.lwsa: Out of memory");
691                 firsterr = -ENOMEM;
692                 goto unwind;
693         }
694         for (i = 0; i < cxt.nrealwriters_stress; i++) {
695                 cxt.lwsa[i].n_lock_fail = 0;
696                 cxt.lwsa[i].n_lock_acquired = 0;
697         }
698
699         if (cxt.cur_ops->readlock) {
700                 if (nreaders_stress >= 0)
701                         cxt.nrealreaders_stress = nreaders_stress;
702                 else {
703                         /*
704                          * By default distribute evenly the number of
705                          * readers and writers. We still run the same number
706                          * of threads as the writer-only locks default.
707                          */
708                         if (nwriters_stress < 0) /* user doesn't care */
709                                 cxt.nrealwriters_stress = num_online_cpus();
710                         cxt.nrealreaders_stress = cxt.nrealwriters_stress;
711                 }
712
713                 lock_is_read_held = 0;
714                 cxt.lrsa = kmalloc(sizeof(*cxt.lrsa) * cxt.nrealreaders_stress, GFP_KERNEL);
715                 if (cxt.lrsa == NULL) {
716                         VERBOSE_TOROUT_STRING("cxt.lrsa: Out of memory");
717                         firsterr = -ENOMEM;
718                         kfree(cxt.lwsa);
719                         goto unwind;
720                 }
721
722                 for (i = 0; i < cxt.nrealreaders_stress; i++) {
723                         cxt.lrsa[i].n_lock_fail = 0;
724                         cxt.lrsa[i].n_lock_acquired = 0;
725                 }
726         }
727         lock_torture_print_module_parms(cxt.cur_ops, "Start of test");
728
729         /* Prepare torture context. */
730         if (onoff_interval > 0) {
731                 firsterr = torture_onoff_init(onoff_holdoff * HZ,
732                                               onoff_interval * HZ);
733                 if (firsterr)
734                         goto unwind;
735         }
736         if (shuffle_interval > 0) {
737                 firsterr = torture_shuffle_init(shuffle_interval);
738                 if (firsterr)
739                         goto unwind;
740         }
741         if (shutdown_secs > 0) {
742                 firsterr = torture_shutdown_init(shutdown_secs,
743                                                  lock_torture_cleanup);
744                 if (firsterr)
745                         goto unwind;
746         }
747         if (stutter > 0) {
748                 firsterr = torture_stutter_init(stutter);
749                 if (firsterr)
750                         goto unwind;
751         }
752
753         writer_tasks = kzalloc(cxt.nrealwriters_stress * sizeof(writer_tasks[0]),
754                                GFP_KERNEL);
755         if (writer_tasks == NULL) {
756                 VERBOSE_TOROUT_ERRSTRING("writer_tasks: Out of memory");
757                 firsterr = -ENOMEM;
758                 goto unwind;
759         }
760
761         if (cxt.cur_ops->readlock) {
762                 reader_tasks = kzalloc(cxt.nrealreaders_stress * sizeof(reader_tasks[0]),
763                                        GFP_KERNEL);
764                 if (reader_tasks == NULL) {
765                         VERBOSE_TOROUT_ERRSTRING("reader_tasks: Out of memory");
766                         firsterr = -ENOMEM;
767                         goto unwind;
768                 }
769         }
770
771         /*
772          * Create the kthreads and start torturing (oh, those poor little locks).
773          *
774          * TODO: Note that we interleave writers with readers, giving writers a
775          * slight advantage, by creating its kthread first. This can be modified
776          * for very specific needs, or even let the user choose the policy, if
777          * ever wanted.
778          */
779         for (i = 0, j = 0; i < cxt.nrealwriters_stress ||
780                     j < cxt.nrealreaders_stress; i++, j++) {
781                 if (i >= cxt.nrealwriters_stress)
782                         goto create_reader;
783
784                 /* Create writer. */
785                 firsterr = torture_create_kthread(lock_torture_writer, &cxt.lwsa[i],
786                                                   writer_tasks[i]);
787                 if (firsterr)
788                         goto unwind;
789
790         create_reader:
791                 if (cxt.cur_ops->readlock == NULL || (j >= cxt.nrealreaders_stress))
792                         continue;
793                 /* Create reader. */
794                 firsterr = torture_create_kthread(lock_torture_reader, &cxt.lrsa[j],
795                                                   reader_tasks[j]);
796                 if (firsterr)
797                         goto unwind;
798         }
799         if (stat_interval > 0) {
800                 firsterr = torture_create_kthread(lock_torture_stats, NULL,
801                                                   stats_task);
802                 if (firsterr)
803                         goto unwind;
804         }
805         torture_init_end();
806         return 0;
807
808 unwind:
809         torture_init_end();
810         lock_torture_cleanup();
811         return firsterr;
812 }
813
814 module_init(lock_torture_init);
815 module_exit(lock_torture_cleanup);