| 1 |
// SPDX-License-Identifier: GPL-2.0+ |
| 2 |
/* |
| 3 |
* Module-based torture test facility for locking |
| 4 |
* |
| 5 |
* Copyright (C) IBM Corporation, 2014 |
| 6 |
* |
| 7 |
* Authors: Paul E. McKenney <[email protected]> |
| 8 |
* Davidlohr Bueso <[email protected]> |
| 9 |
* Based on kernel/rcu/torture.c. |
| 10 |
*/ |
| 11 |
|
| 12 |
#define pr_fmt(fmt) fmt |
| 13 |
|
| 14 |
#include <linux/kernel.h> |
| 15 |
#include <linux/module.h> |
| 16 |
#include <linux/kthread.h> |
| 17 |
#include <linux/sched/rt.h> |
| 18 |
#include <linux/spinlock.h> |
| 19 |
#include <linux/mutex.h> |
| 20 |
#include <linux/rwsem.h> |
| 21 |
#include <linux/smp.h> |
| 22 |
#include <linux/interrupt.h> |
| 23 |
#include <linux/sched.h> |
| 24 |
#include <uapi/linux/sched/types.h> |
| 25 |
#include <linux/rtmutex.h> |
| 26 |
#include <linux/atomic.h> |
| 27 |
#include <linux/moduleparam.h> |
| 28 |
#include <linux/delay.h> |
| 29 |
#include <linux/slab.h> |
| 30 |
#include <linux/torture.h> |
| 31 |
#include <linux/reboot.h> |
| 32 |
|
| 33 |
MODULE_DESCRIPTION("torture test facility for locking"); |
| 34 |
MODULE_LICENSE("GPL"); |
| 35 |
MODULE_AUTHOR("Paul E. McKenney <[email protected]>"); |
| 36 |
|
| 37 |
torture_param(int, acq_writer_lim, 0, "Write_acquisition time limit (jiffies)."); |
| 38 |
torture_param(int, call_rcu_chains, 0, "Self-propagate call_rcu() chains during test (0=disable)."); |
| 39 |
torture_param(int, long_hold, 100, "Do occasional long hold of lock (ms), 0=disable"); |
| 40 |
torture_param(int, nested_locks, 0, "Number of nested locks (max = 8)"); |
| 41 |
torture_param(int, nreaders_stress, -1, "Number of read-locking stress-test threads"); |
| 42 |
torture_param(int, nwriters_stress, -1, "Number of write-locking stress-test threads"); |
| 43 |
torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); |
| 44 |
torture_param(int, onoff_interval, 0, "Time between CPU hotplugs (s), 0=disable"); |
| 45 |
torture_param(int, rt_boost, 2, |
| 46 |
"Do periodic rt-boost. 0=Disable, 1=Only for rt_mutex, 2=For all lock types."); |
| 47 |
torture_param(int, rt_boost_factor, 50, "A factor determining how often rt-boost happens."); |
| 48 |
torture_param(int, shuffle_interval, 3, "Number of jiffies between shuffles, 0=disable"); |
| 49 |
torture_param(int, shutdown_secs, 0, "Shutdown time (j), <= zero to disable."); |
| 50 |
torture_param(int, stat_interval, 60, "Number of seconds between stats printk()s"); |
| 51 |
torture_param(int, stutter, 5, "Number of jiffies to run/halt test, 0=disable"); |
| 52 |
torture_param(int, verbose, 1, "Enable verbose debugging printk()s"); |
| 53 |
torture_param(int, writer_fifo, 0, "Run writers at sched_set_fifo() priority"); |
| 54 |
/* Going much higher trips "BUG: MAX_LOCKDEP_CHAIN_HLOCKS too low!" errors */ |
| 55 |
#define MAX_NESTED_LOCKS 8 |
| 56 |
|
| 57 |
static char *torture_type = IS_ENABLED(CONFIG_PREEMPT_RT) ? "raw_spin_lock" : "spin_lock"; |
| 58 |
module_param(torture_type, charp, 0444); |
| 59 |
MODULE_PARM_DESC(torture_type, |
| 60 |
"Type of lock to torture (spin_lock, spin_lock_irq, mutex_lock, ...)"); |
| 61 |
|
| 62 |
static cpumask_var_t bind_readers; // Bind the readers to the specified set of CPUs. |
| 63 |
static cpumask_var_t bind_writers; // Bind the writers to the specified set of CPUs. |
| 64 |
|
| 65 |
// Parse a cpumask kernel parameter. If there are more users later on, |
| 66 |
// this might need to got to a more central location. |
| 67 |
static int param_set_cpumask(const char *val, const struct kernel_param *kp) |
| 68 |
{ |
| 69 |
cpumask_var_t *cm_bind = kp->arg; |
| 70 |
int ret; |
| 71 |
char *s; |
| 72 |
|
| 73 |
if (!alloc_cpumask_var(cm_bind, GFP_KERNEL)) { |
| 74 |
s = "Out of memory"; |
| 75 |
ret = -ENOMEM; |
| 76 |
goto out_err; |
| 77 |
} |
| 78 |
ret = cpulist_parse(val, *cm_bind); |
| 79 |
if (!ret) |
| 80 |
return ret; |
| 81 |
s = "Bad CPU range"; |
| 82 |
out_err: |
| 83 |
pr_warn("%s: %s, all CPUs set\n", kp->name, s); |
| 84 |
cpumask_setall(*cm_bind); |
| 85 |
return ret; |
| 86 |
} |
| 87 |
|
| 88 |
// Output a cpumask kernel parameter. |
| 89 |
static int param_get_cpumask(char *buffer, const struct kernel_param *kp) |
| 90 |
{ |
| 91 |
cpumask_var_t *cm_bind = kp->arg; |
| 92 |
|
| 93 |
return sprintf(buffer, "%*pbl", cpumask_pr_args(*cm_bind)); |
| 94 |
} |
| 95 |
|
| 96 |
static bool cpumask_nonempty(cpumask_var_t mask) |
| 97 |
{ |
| 98 |
return cpumask_available(mask) && !cpumask_empty(mask); |
| 99 |
} |
| 100 |
|
| 101 |
static const struct kernel_param_ops lt_bind_ops = { |
| 102 |
.set = param_set_cpumask, |
| 103 |
.get = param_get_cpumask, |
| 104 |
}; |
| 105 |
|
| 106 |
module_param_cb(bind_readers, <_bind_ops, &bind_readers, 0444); |
| 107 |
module_param_cb(bind_writers, <_bind_ops, &bind_writers, 0444); |
| 108 |
|
| 109 |
long torture_sched_setaffinity(pid_t pid, const struct cpumask *in_mask, bool dowarn); |
| 110 |
|
| 111 |
static struct task_struct *stats_task; |
| 112 |
static struct task_struct **writer_tasks; |
| 113 |
static struct task_struct **reader_tasks; |
| 114 |
|
| 115 |
static bool lock_is_write_held; |
| 116 |
static atomic_t lock_is_read_held; |
| 117 |
static unsigned long last_lock_release; |
| 118 |
|
| 119 |
struct lock_stress_stats { |
| 120 |
long n_lock_fail; |
| 121 |
long n_lock_acquired; |
| 122 |
}; |
| 123 |
|
| 124 |
struct call_rcu_chain { |
| 125 |
struct rcu_head crc_rh; |
| 126 |
bool crc_stop; |
| 127 |
}; |
| 128 |
struct call_rcu_chain *call_rcu_chain_list; |
| 129 |
|
| 130 |
/* Forward reference. */ |
| 131 |
static void lock_torture_cleanup(void); |
| 132 |
|
| 133 |
/* |
| 134 |
* Operations vector for selecting different types of tests. |
| 135 |
*/ |
| 136 |
struct lock_torture_ops { |
| 137 |
void (*init)(void); |
| 138 |
void (*exit)(void); |
| 139 |
int (*nested_lock)(int tid, u32 lockset); |
| 140 |
int (*writelock)(int tid); |
| 141 |
void (*write_delay)(struct torture_random_state *trsp); |
| 142 |
void (*task_boost)(struct torture_random_state *trsp); |
| 143 |
void (*writeunlock)(int tid); |
| 144 |
void (*nested_unlock)(int tid, u32 lockset); |
| 145 |
int (*readlock)(int tid); |
| 146 |
void (*read_delay)(struct torture_random_state *trsp); |
| 147 |
void (*readunlock)(int tid); |
| 148 |
|
| 149 |
unsigned long flags; /* for irq spinlocks */ |
| 150 |
const char *name; |
| 151 |
}; |
| 152 |
|
| 153 |
struct lock_torture_cxt { |
| 154 |
int nrealwriters_stress; |
| 155 |
int nrealreaders_stress; |
| 156 |
bool debug_lock; |
| 157 |
bool init_called; |
| 158 |
atomic_t n_lock_torture_errors; |
| 159 |
struct lock_torture_ops *cur_ops; |
| 160 |
struct lock_stress_stats *lwsa; /* writer statistics */ |
| 161 |
struct lock_stress_stats *lrsa; /* reader statistics */ |
| 162 |
}; |
| 163 |
static struct lock_torture_cxt cxt = { 0, 0, false, false, |
| 164 |
ATOMIC_INIT(0), |
| 165 |
NULL, NULL}; |
| 166 |
/* |
| 167 |
* Definitions for lock torture testing. |
| 168 |
*/ |
| 169 |
|
| 170 |
static int torture_lock_busted_write_lock(int tid __maybe_unused) |
| 171 |
{ |
| 172 |
return 0; /* BUGGY, do not use in real life!!! */ |
| 173 |
} |
| 174 |
|
| 175 |
static void torture_lock_busted_write_delay(struct torture_random_state *trsp) |
| 176 |
{ |
| 177 |
/* We want a long delay occasionally to force massive contention. */ |
| 178 |
if (long_hold && !(torture_random(trsp) % (cxt.nrealwriters_stress * 2000 * long_hold))) |
| 179 |
mdelay(long_hold); |
| 180 |
if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000))) |
| 181 |
torture_preempt_schedule(); /* Allow test to be preempted. */ |
| 182 |
} |
| 183 |
|
| 184 |
static void torture_lock_busted_write_unlock(int tid __maybe_unused) |
| 185 |
{ |
| 186 |
/* BUGGY, do not use in real life!!! */ |
| 187 |
} |
| 188 |
|
| 189 |
static void __torture_rt_boost(struct torture_random_state *trsp) |
| 190 |
{ |
| 191 |
const unsigned int factor = rt_boost_factor; |
| 192 |
|
| 193 |
if (!rt_task(current)) { |
| 194 |
/* |
| 195 |
* Boost priority once every rt_boost_factor operations. When |
| 196 |
* the task tries to take the lock, the rtmutex it will account |
| 197 |
* for the new priority, and do any corresponding pi-dance. |
| 198 |
*/ |
| 199 |
if (trsp && !(torture_random(trsp) % |
| 200 |
(cxt.nrealwriters_stress * factor))) { |
| 201 |
sched_set_fifo(current); |
| 202 |
} else /* common case, do nothing */ |
| 203 |
return; |
| 204 |
} else { |
| 205 |
/* |
| 206 |
* The task will remain boosted for another 10 * rt_boost_factor |
| 207 |
* operations, then restored back to its original prio, and so |
| 208 |
* forth. |
| 209 |
* |
| 210 |
* When @trsp is nil, we want to force-reset the task for |
| 211 |
* stopping the kthread. |
| 212 |
*/ |
| 213 |
if (!trsp || !(torture_random(trsp) % |
| 214 |
(cxt.nrealwriters_stress * factor * 2))) { |
| 215 |
sched_set_normal(current, 0); |
| 216 |
} else /* common case, do nothing */ |
| 217 |
return; |
| 218 |
} |
| 219 |
} |
| 220 |
|
| 221 |
static void torture_rt_boost(struct torture_random_state *trsp) |
| 222 |
{ |
| 223 |
if (rt_boost != 2) |
| 224 |
return; |
| 225 |
|
| 226 |
__torture_rt_boost(trsp); |
| 227 |
} |
| 228 |
|
| 229 |
static struct lock_torture_ops lock_busted_ops = { |
| 230 |
.writelock = torture_lock_busted_write_lock, |
| 231 |
.write_delay = torture_lock_busted_write_delay, |
| 232 |
.task_boost = torture_rt_boost, |
| 233 |
.writeunlock = torture_lock_busted_write_unlock, |
| 234 |
.readlock = NULL, |
| 235 |
.read_delay = NULL, |
| 236 |
.readunlock = NULL, |
| 237 |
.name = "lock_busted" |
| 238 |
}; |
| 239 |
|
| 240 |
static DEFINE_SPINLOCK(torture_spinlock); |
| 241 |
|
| 242 |
static int torture_spin_lock_write_lock(int tid __maybe_unused) |
| 243 |
__acquires(torture_spinlock) |
| 244 |
{ |
| 245 |
spin_lock(&torture_spinlock); |
| 246 |
return 0; |
| 247 |
} |
| 248 |
|
| 249 |
static void torture_spin_lock_write_delay(struct torture_random_state *trsp) |
| 250 |
{ |
| 251 |
const unsigned long shortdelay_us = 2; |
| 252 |
unsigned long j; |
| 253 |
|
| 254 |
/* We want a short delay mostly to emulate likely code, and |
| 255 |
* we want a long delay occasionally to force massive contention. |
| 256 |
*/ |
| 257 |
if (long_hold && !(torture_random(trsp) % (cxt.nrealwriters_stress * 2000 * long_hold))) { |
| 258 |
j = jiffies; |
| 259 |
mdelay(long_hold); |
| 260 |
pr_alert("%s: delay = %lu jiffies.\n", __func__, jiffies - j); |
| 261 |
} |
| 262 |
if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 200 * shortdelay_us))) |
| 263 |
udelay(shortdelay_us); |
| 264 |
if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000))) |
| 265 |
torture_preempt_schedule(); /* Allow test to be preempted. */ |
| 266 |
} |
| 267 |
|
| 268 |
static void torture_spin_lock_write_unlock(int tid __maybe_unused) |
| 269 |
__releases(torture_spinlock) |
| 270 |
{ |
| 271 |
spin_unlock(&torture_spinlock); |
| 272 |
} |
| 273 |
|
| 274 |
static struct lock_torture_ops spin_lock_ops = { |
| 275 |
.writelock = torture_spin_lock_write_lock, |
| 276 |
.write_delay = torture_spin_lock_write_delay, |
| 277 |
.task_boost = torture_rt_boost, |
| 278 |
.writeunlock = torture_spin_lock_write_unlock, |
| 279 |
.readlock = NULL, |
| 280 |
.read_delay = NULL, |
| 281 |
.readunlock = NULL, |
| 282 |
.name = "spin_lock" |
| 283 |
}; |
| 284 |
|
| 285 |
static int torture_spin_lock_write_lock_irq(int tid __maybe_unused) |
| 286 |
__acquires(torture_spinlock) |
| 287 |
{ |
| 288 |
unsigned long flags; |
| 289 |
|
| 290 |
spin_lock_irqsave(&torture_spinlock, flags); |
| 291 |
cxt.cur_ops->flags = flags; |
| 292 |
return 0; |
| 293 |
} |
| 294 |
|
| 295 |
static void torture_lock_spin_write_unlock_irq(int tid __maybe_unused) |
| 296 |
__releases(torture_spinlock) |
| 297 |
{ |
| 298 |
spin_unlock_irqrestore(&torture_spinlock, cxt.cur_ops->flags); |
| 299 |
} |
| 300 |
|
| 301 |
static struct lock_torture_ops spin_lock_irq_ops = { |
| 302 |
.writelock = torture_spin_lock_write_lock_irq, |
| 303 |
.write_delay = torture_spin_lock_write_delay, |
| 304 |
.task_boost = torture_rt_boost, |
| 305 |
.writeunlock = torture_lock_spin_write_unlock_irq, |
| 306 |
.readlock = NULL, |
| 307 |
.read_delay = NULL, |
| 308 |
.readunlock = NULL, |
| 309 |
.name = "spin_lock_irq" |
| 310 |
}; |
| 311 |
|
| 312 |
static DEFINE_RAW_SPINLOCK(torture_raw_spinlock); |
| 313 |
|
| 314 |
static int torture_raw_spin_lock_write_lock(int tid __maybe_unused) |
| 315 |
__acquires(torture_raw_spinlock) |
| 316 |
{ |
| 317 |
raw_spin_lock(&torture_raw_spinlock); |
| 318 |
return 0; |
| 319 |
} |
| 320 |
|
| 321 |
static void torture_raw_spin_lock_write_unlock(int tid __maybe_unused) |
| 322 |
__releases(torture_raw_spinlock) |
| 323 |
{ |
| 324 |
raw_spin_unlock(&torture_raw_spinlock); |
| 325 |
} |
| 326 |
|
| 327 |
static struct lock_torture_ops raw_spin_lock_ops = { |
| 328 |
.writelock = torture_raw_spin_lock_write_lock, |
| 329 |
.write_delay = torture_spin_lock_write_delay, |
| 330 |
.task_boost = torture_rt_boost, |
| 331 |
.writeunlock = torture_raw_spin_lock_write_unlock, |
| 332 |
.readlock = NULL, |
| 333 |
.read_delay = NULL, |
| 334 |
.readunlock = NULL, |
| 335 |
.name = "raw_spin_lock" |
| 336 |
}; |
| 337 |
|
| 338 |
static int torture_raw_spin_lock_write_lock_irq(int tid __maybe_unused) |
| 339 |
__acquires(torture_raw_spinlock) |
| 340 |
{ |
| 341 |
unsigned long flags; |
| 342 |
|
| 343 |
raw_spin_lock_irqsave(&torture_raw_spinlock, flags); |
| 344 |
cxt.cur_ops->flags = flags; |
| 345 |
return 0; |
| 346 |
} |
| 347 |
|
| 348 |
static void torture_raw_spin_lock_write_unlock_irq(int tid __maybe_unused) |
| 349 |
__releases(torture_raw_spinlock) |
| 350 |
{ |
| 351 |
raw_spin_unlock_irqrestore(&torture_raw_spinlock, cxt.cur_ops->flags); |
| 352 |
} |
| 353 |
|
| 354 |
static struct lock_torture_ops raw_spin_lock_irq_ops = { |
| 355 |
.writelock = torture_raw_spin_lock_write_lock_irq, |
| 356 |
.write_delay = torture_spin_lock_write_delay, |
| 357 |
.task_boost = torture_rt_boost, |
| 358 |
.writeunlock = torture_raw_spin_lock_write_unlock_irq, |
| 359 |
.readlock = NULL, |
| 360 |
.read_delay = NULL, |
| 361 |
.readunlock = NULL, |
| 362 |
.name = "raw_spin_lock_irq" |
| 363 |
}; |
| 364 |
|
| 365 |
#ifdef CONFIG_BPF_SYSCALL |
| 366 |
|
| 367 |
#include <asm/rqspinlock.h> |
| 368 |
static rqspinlock_t rqspinlock; |
| 369 |
|
| 370 |
static int torture_raw_res_spin_write_lock(int tid __maybe_unused) |
| 371 |
{ |
| 372 |
raw_res_spin_lock(&rqspinlock); |
| 373 |
return 0; |
| 374 |
} |
| 375 |
|
| 376 |
static void torture_raw_res_spin_write_unlock(int tid __maybe_unused) |
| 377 |
{ |
| 378 |
raw_res_spin_unlock(&rqspinlock); |
| 379 |
} |
| 380 |
|
| 381 |
static struct lock_torture_ops raw_res_spin_lock_ops = { |
| 382 |
.writelock = torture_raw_res_spin_write_lock, |
| 383 |
.write_delay = torture_spin_lock_write_delay, |
| 384 |
.task_boost = torture_rt_boost, |
| 385 |
.writeunlock = torture_raw_res_spin_write_unlock, |
| 386 |
.readlock = NULL, |
| 387 |
.read_delay = NULL, |
| 388 |
.readunlock = NULL, |
| 389 |
.name = "raw_res_spin_lock" |
| 390 |
}; |
| 391 |
|
| 392 |
static int torture_raw_res_spin_write_lock_irq(int tid __maybe_unused) |
| 393 |
{ |
| 394 |
unsigned long flags; |
| 395 |
|
| 396 |
raw_res_spin_lock_irqsave(&rqspinlock, flags); |
| 397 |
cxt.cur_ops->flags = flags; |
| 398 |
return 0; |
| 399 |
} |
| 400 |
|
| 401 |
static void torture_raw_res_spin_write_unlock_irq(int tid __maybe_unused) |
| 402 |
{ |
| 403 |
raw_res_spin_unlock_irqrestore(&rqspinlock, cxt.cur_ops->flags); |
| 404 |
} |
| 405 |
|
| 406 |
static struct lock_torture_ops raw_res_spin_lock_irq_ops = { |
| 407 |
.writelock = torture_raw_res_spin_write_lock_irq, |
| 408 |
.write_delay = torture_spin_lock_write_delay, |
| 409 |
.task_boost = torture_rt_boost, |
| 410 |
.writeunlock = torture_raw_res_spin_write_unlock_irq, |
| 411 |
.readlock = NULL, |
| 412 |
.read_delay = NULL, |
| 413 |
.readunlock = NULL, |
| 414 |
.name = "raw_res_spin_lock_irq" |
| 415 |
}; |
| 416 |
|
| 417 |
#endif |
| 418 |
|
| 419 |
static DEFINE_RWLOCK(torture_rwlock); |
| 420 |
|
| 421 |
static int torture_rwlock_write_lock(int tid __maybe_unused) |
| 422 |
__acquires(torture_rwlock) |
| 423 |
{ |
| 424 |
write_lock(&torture_rwlock); |
| 425 |
return 0; |
| 426 |
} |
| 427 |
|
| 428 |
static void torture_rwlock_write_delay(struct torture_random_state *trsp) |
| 429 |
{ |
| 430 |
const unsigned long shortdelay_us = 2; |
| 431 |
|
| 432 |
/* We want a short delay mostly to emulate likely code, and |
| 433 |
* we want a long delay occasionally to force massive contention. |
| 434 |
*/ |
| 435 |
if (long_hold && !(torture_random(trsp) % (cxt.nrealwriters_stress * 2000 * long_hold))) |
| 436 |
mdelay(long_hold); |
| 437 |
else |
| 438 |
udelay(shortdelay_us); |
| 439 |
} |
| 440 |
|
| 441 |
static void torture_rwlock_write_unlock(int tid __maybe_unused) |
| 442 |
__releases(torture_rwlock) |
| 443 |
{ |
| 444 |
write_unlock(&torture_rwlock); |
| 445 |
} |
| 446 |
|
| 447 |
static int torture_rwlock_read_lock(int tid __maybe_unused) |
| 448 |
__acquires(torture_rwlock) |
| 449 |
{ |
| 450 |
read_lock(&torture_rwlock); |
| 451 |
return 0; |
| 452 |
} |
| 453 |
|
| 454 |
static void torture_rwlock_read_delay(struct torture_random_state *trsp) |
| 455 |
{ |
| 456 |
const unsigned long shortdelay_us = 10; |
| 457 |
|
| 458 |
/* We want a short delay mostly to emulate likely code, and |
| 459 |
* we want a long delay occasionally to force massive contention. |
| 460 |
*/ |
| 461 |
if (long_hold && !(torture_random(trsp) % (cxt.nrealreaders_stress * 2000 * long_hold))) |
| 462 |
mdelay(long_hold); |
| 463 |
else |
| 464 |
udelay(shortdelay_us); |
| 465 |
} |
| 466 |
|
| 467 |
static void torture_rwlock_read_unlock(int tid __maybe_unused) |
| 468 |
__releases(torture_rwlock) |
| 469 |
{ |
| 470 |
read_unlock(&torture_rwlock); |
| 471 |
} |
| 472 |
|
| 473 |
static struct lock_torture_ops rw_lock_ops = { |
| 474 |
.writelock = torture_rwlock_write_lock, |
| 475 |
.write_delay = torture_rwlock_write_delay, |
| 476 |
.task_boost = torture_rt_boost, |
| 477 |
.writeunlock = torture_rwlock_write_unlock, |
| 478 |
.readlock = torture_rwlock_read_lock, |
| 479 |
.read_delay = torture_rwlock_read_delay, |
| 480 |
.readunlock = torture_rwlock_read_unlock, |
| 481 |
.name = "rw_lock" |
| 482 |
}; |
| 483 |
|
| 484 |
static int torture_rwlock_write_lock_irq(int tid __maybe_unused) |
| 485 |
__acquires(torture_rwlock) |
| 486 |
{ |
| 487 |
unsigned long flags; |
| 488 |
|
| 489 |
write_lock_irqsave(&torture_rwlock, flags); |
| 490 |
cxt.cur_ops->flags = flags; |
| 491 |
return 0; |
| 492 |
} |
| 493 |
|
| 494 |
static void torture_rwlock_write_unlock_irq(int tid __maybe_unused) |
| 495 |
__releases(torture_rwlock) |
| 496 |
{ |
| 497 |
write_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags); |
| 498 |
} |
| 499 |
|
| 500 |
static int torture_rwlock_read_lock_irq(int tid __maybe_unused) |
| 501 |
__acquires(torture_rwlock) |
| 502 |
{ |
| 503 |
unsigned long flags; |
| 504 |
|
| 505 |
read_lock_irqsave(&torture_rwlock, flags); |
| 506 |
cxt.cur_ops->flags = flags; |
| 507 |
return 0; |
| 508 |
} |
| 509 |
|
| 510 |
static void torture_rwlock_read_unlock_irq(int tid __maybe_unused) |
| 511 |
__releases(torture_rwlock) |
| 512 |
{ |
| 513 |
read_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags); |
| 514 |
} |
| 515 |
|
| 516 |
static struct lock_torture_ops rw_lock_irq_ops = { |
| 517 |
.writelock = torture_rwlock_write_lock_irq, |
| 518 |
.write_delay = torture_rwlock_write_delay, |
| 519 |
.task_boost = torture_rt_boost, |
| 520 |
.writeunlock = torture_rwlock_write_unlock_irq, |
| 521 |
.readlock = torture_rwlock_read_lock_irq, |
| 522 |
.read_delay = torture_rwlock_read_delay, |
| 523 |
.readunlock = torture_rwlock_read_unlock_irq, |
| 524 |
.name = "rw_lock_irq" |
| 525 |
}; |
| 526 |
|
| 527 |
static DEFINE_MUTEX(torture_mutex); |
| 528 |
static struct mutex torture_nested_mutexes[MAX_NESTED_LOCKS]; |
| 529 |
static struct lock_class_key nested_mutex_keys[MAX_NESTED_LOCKS]; |
| 530 |
|
| 531 |
static void torture_mutex_init(void) |
| 532 |
{ |
| 533 |
int i; |
| 534 |
|
| 535 |
for (i = 0; i < MAX_NESTED_LOCKS; i++) |
| 536 |
__mutex_init(&torture_nested_mutexes[i], __func__, |
| 537 |
&nested_mutex_keys[i]); |
| 538 |
} |
| 539 |
|
| 540 |
static int torture_mutex_nested_lock(int tid __maybe_unused, |
| 541 |
u32 lockset) |
| 542 |
{ |
| 543 |
int i; |
| 544 |
|
| 545 |
for (i = 0; i < nested_locks; i++) |
| 546 |
if (lockset & (1 << i)) |
| 547 |
mutex_lock(&torture_nested_mutexes[i]); |
| 548 |
return 0; |
| 549 |
} |
| 550 |
|
| 551 |
static int torture_mutex_lock(int tid __maybe_unused) |
| 552 |
__acquires(torture_mutex) |
| 553 |
{ |
| 554 |
mutex_lock(&torture_mutex); |
| 555 |
return 0; |
| 556 |
} |
| 557 |
|
| 558 |
static void torture_mutex_delay(struct torture_random_state *trsp) |
| 559 |
{ |
| 560 |
/* We want a long delay occasionally to force massive contention. */ |
| 561 |
if (long_hold && !(torture_random(trsp) % (cxt.nrealwriters_stress * 2000 * long_hold))) |
| 562 |
mdelay(long_hold * 5); |
| 563 |
if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000))) |
| 564 |
torture_preempt_schedule(); /* Allow test to be preempted. */ |
| 565 |
} |
| 566 |
|
| 567 |
static void torture_mutex_unlock(int tid __maybe_unused) |
| 568 |
__releases(torture_mutex) |
| 569 |
{ |
| 570 |
mutex_unlock(&torture_mutex); |
| 571 |
} |
| 572 |
|
| 573 |
static void torture_mutex_nested_unlock(int tid __maybe_unused, |
| 574 |
u32 lockset) |
| 575 |
{ |
| 576 |
int i; |
| 577 |
|
| 578 |
for (i = nested_locks - 1; i >= 0; i--) |
| 579 |
if (lockset & (1 << i)) |
| 580 |
mutex_unlock(&torture_nested_mutexes[i]); |
| 581 |
} |
| 582 |
|
| 583 |
static struct lock_torture_ops mutex_lock_ops = { |
| 584 |
.init = torture_mutex_init, |
| 585 |
.nested_lock = torture_mutex_nested_lock, |
| 586 |
.writelock = torture_mutex_lock, |
| 587 |
.write_delay = torture_mutex_delay, |
| 588 |
.task_boost = torture_rt_boost, |
| 589 |
.writeunlock = torture_mutex_unlock, |
| 590 |
.nested_unlock = torture_mutex_nested_unlock, |
| 591 |
.readlock = NULL, |
| 592 |
.read_delay = NULL, |
| 593 |
.readunlock = NULL, |
| 594 |
.name = "mutex_lock" |
| 595 |
}; |
| 596 |
|
| 597 |
#include <linux/ww_mutex.h> |
| 598 |
/* |
| 599 |
* The torture ww_mutexes should belong to the same lock class as |
| 600 |
* torture_ww_class to avoid lockdep problem. The ww_mutex_init() |
| 601 |
* function is called for initialization to ensure that. |
| 602 |
*/ |
| 603 |
static DEFINE_WD_CLASS(torture_ww_class); |
| 604 |
static struct ww_mutex torture_ww_mutex_0, torture_ww_mutex_1, torture_ww_mutex_2; |
| 605 |
static struct ww_acquire_ctx *ww_acquire_ctxs; |
| 606 |
|
| 607 |
static void torture_ww_mutex_init(void) |
| 608 |
{ |
| 609 |
ww_mutex_init(&torture_ww_mutex_0, &torture_ww_class); |
| 610 |
ww_mutex_init(&torture_ww_mutex_1, &torture_ww_class); |
| 611 |
ww_mutex_init(&torture_ww_mutex_2, &torture_ww_class); |
| 612 |
|
| 613 |
ww_acquire_ctxs = kmalloc_objs(*ww_acquire_ctxs, |
| 614 |
cxt.nrealwriters_stress); |
| 615 |
if (!ww_acquire_ctxs) |
| 616 |
VERBOSE_TOROUT_STRING("ww_acquire_ctx: Out of memory"); |
| 617 |
} |
| 618 |
|
| 619 |
static void torture_ww_mutex_exit(void) |
| 620 |
{ |
| 621 |
kfree(ww_acquire_ctxs); |
| 622 |
} |
| 623 |
|
| 624 |
static int torture_ww_mutex_lock(int tid) |
| 625 |
__acquires(torture_ww_mutex_0) |
| 626 |
__acquires(torture_ww_mutex_1) |
| 627 |
__acquires(torture_ww_mutex_2) |
| 628 |
{ |
| 629 |
LIST_HEAD(list); |
| 630 |
struct reorder_lock { |
| 631 |
struct list_head link; |
| 632 |
struct ww_mutex *lock; |
| 633 |
} locks[3], *ll, *ln; |
| 634 |
struct ww_acquire_ctx *ctx = &ww_acquire_ctxs[tid]; |
| 635 |
|
| 636 |
locks[0].lock = &torture_ww_mutex_0; |
| 637 |
list_add(&locks[0].link, &list); |
| 638 |
|
| 639 |
locks[1].lock = &torture_ww_mutex_1; |
| 640 |
list_add(&locks[1].link, &list); |
| 641 |
|
| 642 |
locks[2].lock = &torture_ww_mutex_2; |
| 643 |
list_add(&locks[2].link, &list); |
| 644 |
|
| 645 |
ww_acquire_init(ctx, &torture_ww_class); |
| 646 |
|
| 647 |
list_for_each_entry(ll, &list, link) { |
| 648 |
int err; |
| 649 |
|
| 650 |
err = ww_mutex_lock(ll->lock, ctx); |
| 651 |
if (!err) |
| 652 |
continue; |
| 653 |
|
| 654 |
ln = ll; |
| 655 |
list_for_each_entry_continue_reverse(ln, &list, link) |
| 656 |
ww_mutex_unlock(ln->lock); |
| 657 |
|
| 658 |
if (err != -EDEADLK) |
| 659 |
return err; |
| 660 |
|
| 661 |
ww_mutex_lock_slow(ll->lock, ctx); |
| 662 |
list_move(&ll->link, &list); |
| 663 |
} |
| 664 |
|
| 665 |
return 0; |
| 666 |
} |
| 667 |
|
| 668 |
static void torture_ww_mutex_unlock(int tid) |
| 669 |
__releases(torture_ww_mutex_0) |
| 670 |
__releases(torture_ww_mutex_1) |
| 671 |
__releases(torture_ww_mutex_2) |
| 672 |
{ |
| 673 |
struct ww_acquire_ctx *ctx = &ww_acquire_ctxs[tid]; |
| 674 |
|
| 675 |
ww_mutex_unlock(&torture_ww_mutex_0); |
| 676 |
ww_mutex_unlock(&torture_ww_mutex_1); |
| 677 |
ww_mutex_unlock(&torture_ww_mutex_2); |
| 678 |
ww_acquire_fini(ctx); |
| 679 |
} |
| 680 |
|
| 681 |
static struct lock_torture_ops ww_mutex_lock_ops = { |
| 682 |
.init = torture_ww_mutex_init, |
| 683 |
.exit = torture_ww_mutex_exit, |
| 684 |
.writelock = torture_ww_mutex_lock, |
| 685 |
.write_delay = torture_mutex_delay, |
| 686 |
.task_boost = torture_rt_boost, |
| 687 |
.writeunlock = torture_ww_mutex_unlock, |
| 688 |
.readlock = NULL, |
| 689 |
.read_delay = NULL, |
| 690 |
.readunlock = NULL, |
| 691 |
.name = "ww_mutex_lock" |
| 692 |
}; |
| 693 |
|
| 694 |
#ifdef CONFIG_RT_MUTEXES |
| 695 |
static DEFINE_RT_MUTEX(torture_rtmutex); |
| 696 |
static struct rt_mutex torture_nested_rtmutexes[MAX_NESTED_LOCKS]; |
| 697 |
static struct lock_class_key nested_rtmutex_keys[MAX_NESTED_LOCKS]; |
| 698 |
|
| 699 |
static void torture_rtmutex_init(void) |
| 700 |
{ |
| 701 |
int i; |
| 702 |
|
| 703 |
for (i = 0; i < MAX_NESTED_LOCKS; i++) |
| 704 |
__rt_mutex_init(&torture_nested_rtmutexes[i], __func__, |
| 705 |
&nested_rtmutex_keys[i]); |
| 706 |
} |
| 707 |
|
| 708 |
static int torture_rtmutex_nested_lock(int tid __maybe_unused, |
| 709 |
u32 lockset) |
| 710 |
{ |
| 711 |
int i; |
| 712 |
|
| 713 |
for (i = 0; i < nested_locks; i++) |
| 714 |
if (lockset & (1 << i)) |
| 715 |
rt_mutex_lock(&torture_nested_rtmutexes[i]); |
| 716 |
return 0; |
| 717 |
} |
| 718 |
|
| 719 |
static int torture_rtmutex_lock(int tid __maybe_unused) |
| 720 |
__acquires(torture_rtmutex) |
| 721 |
{ |
| 722 |
rt_mutex_lock(&torture_rtmutex); |
| 723 |
return 0; |
| 724 |
} |
| 725 |
|
| 726 |
static void torture_rtmutex_delay(struct torture_random_state *trsp) |
| 727 |
{ |
| 728 |
const unsigned long shortdelay_us = 2; |
| 729 |
|
| 730 |
/* |
| 731 |
* We want a short delay mostly to emulate likely code, and |
| 732 |
* we want a long delay occasionally to force massive contention. |
| 733 |
*/ |
| 734 |
if (long_hold && !(torture_random(trsp) % (cxt.nrealwriters_stress * 2000 * long_hold))) |
| 735 |
mdelay(long_hold); |
| 736 |
if (!(torture_random(trsp) % |
| 737 |
(cxt.nrealwriters_stress * 200 * shortdelay_us))) |
| 738 |
udelay(shortdelay_us); |
| 739 |
if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000))) |
| 740 |
torture_preempt_schedule(); /* Allow test to be preempted. */ |
| 741 |
} |
| 742 |
|
| 743 |
static void torture_rtmutex_unlock(int tid __maybe_unused) |
| 744 |
__releases(torture_rtmutex) |
| 745 |
{ |
| 746 |
rt_mutex_unlock(&torture_rtmutex); |
| 747 |
} |
| 748 |
|
| 749 |
static void torture_rt_boost_rtmutex(struct torture_random_state *trsp) |
| 750 |
{ |
| 751 |
if (!rt_boost) |
| 752 |
return; |
| 753 |
|
| 754 |
__torture_rt_boost(trsp); |
| 755 |
} |
| 756 |
|
| 757 |
static void torture_rtmutex_nested_unlock(int tid __maybe_unused, |
| 758 |
u32 lockset) |
| 759 |
{ |
| 760 |
int i; |
| 761 |
|
| 762 |
for (i = nested_locks - 1; i >= 0; i--) |
| 763 |
if (lockset & (1 << i)) |
| 764 |
rt_mutex_unlock(&torture_nested_rtmutexes[i]); |
| 765 |
} |
| 766 |
|
| 767 |
static struct lock_torture_ops rtmutex_lock_ops = { |
| 768 |
.init = torture_rtmutex_init, |
| 769 |
.nested_lock = torture_rtmutex_nested_lock, |
| 770 |
.writelock = torture_rtmutex_lock, |
| 771 |
.write_delay = torture_rtmutex_delay, |
| 772 |
.task_boost = torture_rt_boost_rtmutex, |
| 773 |
.writeunlock = torture_rtmutex_unlock, |
| 774 |
.nested_unlock = torture_rtmutex_nested_unlock, |
| 775 |
.readlock = NULL, |
| 776 |
.read_delay = NULL, |
| 777 |
.readunlock = NULL, |
| 778 |
.name = "rtmutex_lock" |
| 779 |
}; |
| 780 |
#endif |
| 781 |
|
| 782 |
static DECLARE_RWSEM(torture_rwsem); |
| 783 |
static int torture_rwsem_down_write(int tid __maybe_unused) |
| 784 |
__acquires(torture_rwsem) |
| 785 |
{ |
| 786 |
down_write(&torture_rwsem); |
| 787 |
return 0; |
| 788 |
} |
| 789 |
|
| 790 |
static void torture_rwsem_write_delay(struct torture_random_state *trsp) |
| 791 |
{ |
| 792 |
/* We want a long delay occasionally to force massive contention. */ |
| 793 |
if (long_hold && !(torture_random(trsp) % (cxt.nrealwriters_stress * 2000 * long_hold))) |
| 794 |
mdelay(long_hold * 10); |
| 795 |
if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000))) |
| 796 |
torture_preempt_schedule(); /* Allow test to be preempted. */ |
| 797 |
} |
| 798 |
|
| 799 |
static void torture_rwsem_up_write(int tid __maybe_unused) |
| 800 |
__releases(torture_rwsem) |
| 801 |
{ |
| 802 |
up_write(&torture_rwsem); |
| 803 |
} |
| 804 |
|
| 805 |
static int torture_rwsem_down_read(int tid __maybe_unused) |
| 806 |
__acquires(torture_rwsem) |
| 807 |
{ |
| 808 |
down_read(&torture_rwsem); |
| 809 |
return 0; |
| 810 |
} |
| 811 |
|
| 812 |
static void torture_rwsem_read_delay(struct torture_random_state *trsp) |
| 813 |
{ |
| 814 |
/* We want a long delay occasionally to force massive contention. */ |
| 815 |
if (long_hold && !(torture_random(trsp) % (cxt.nrealreaders_stress * 2000 * long_hold))) |
| 816 |
mdelay(long_hold * 2); |
| 817 |
else |
| 818 |
mdelay(long_hold / 2); |
| 819 |
if (!(torture_random(trsp) % (cxt.nrealreaders_stress * 20000))) |
| 820 |
torture_preempt_schedule(); /* Allow test to be preempted. */ |
| 821 |
} |
| 822 |
|
| 823 |
static void torture_rwsem_up_read(int tid __maybe_unused) |
| 824 |
__releases(torture_rwsem) |
| 825 |
{ |
| 826 |
up_read(&torture_rwsem); |
| 827 |
} |
| 828 |
|
| 829 |
static struct lock_torture_ops rwsem_lock_ops = { |
| 830 |
.writelock = torture_rwsem_down_write, |
| 831 |
.write_delay = torture_rwsem_write_delay, |
| 832 |
.task_boost = torture_rt_boost, |
| 833 |
.writeunlock = torture_rwsem_up_write, |
| 834 |
.readlock = torture_rwsem_down_read, |
| 835 |
.read_delay = torture_rwsem_read_delay, |
| 836 |
.readunlock = torture_rwsem_up_read, |
| 837 |
.name = "rwsem_lock" |
| 838 |
}; |
| 839 |
|
| 840 |
#include <linux/percpu-rwsem.h> |
| 841 |
static struct percpu_rw_semaphore pcpu_rwsem; |
| 842 |
|
| 843 |
static void torture_percpu_rwsem_init(void) |
| 844 |
{ |
| 845 |
BUG_ON(percpu_init_rwsem(&pcpu_rwsem)); |
| 846 |
} |
| 847 |
|
| 848 |
static void torture_percpu_rwsem_exit(void) |
| 849 |
{ |
| 850 |
percpu_free_rwsem(&pcpu_rwsem); |
| 851 |
} |
| 852 |
|
| 853 |
static int torture_percpu_rwsem_down_write(int tid __maybe_unused) |
| 854 |
__acquires(pcpu_rwsem) |
| 855 |
{ |
| 856 |
percpu_down_write(&pcpu_rwsem); |
| 857 |
return 0; |
| 858 |
} |
| 859 |
|
| 860 |
static void torture_percpu_rwsem_up_write(int tid __maybe_unused) |
| 861 |
__releases(pcpu_rwsem) |
| 862 |
{ |
| 863 |
percpu_up_write(&pcpu_rwsem); |
| 864 |
} |
| 865 |
|
| 866 |
static int torture_percpu_rwsem_down_read(int tid __maybe_unused) |
| 867 |
__acquires(pcpu_rwsem) |
| 868 |
{ |
| 869 |
percpu_down_read(&pcpu_rwsem); |
| 870 |
return 0; |
| 871 |
} |
| 872 |
|
| 873 |
static void torture_percpu_rwsem_up_read(int tid __maybe_unused) |
| 874 |
__releases(pcpu_rwsem) |
| 875 |
{ |
| 876 |
percpu_up_read(&pcpu_rwsem); |
| 877 |
} |
| 878 |
|
| 879 |
static struct lock_torture_ops percpu_rwsem_lock_ops = { |
| 880 |
.init = torture_percpu_rwsem_init, |
| 881 |
.exit = torture_percpu_rwsem_exit, |
| 882 |
.writelock = torture_percpu_rwsem_down_write, |
| 883 |
.write_delay = torture_rwsem_write_delay, |
| 884 |
.task_boost = torture_rt_boost, |
| 885 |
.writeunlock = torture_percpu_rwsem_up_write, |
| 886 |
.readlock = torture_percpu_rwsem_down_read, |
| 887 |
.read_delay = torture_rwsem_read_delay, |
| 888 |
.readunlock = torture_percpu_rwsem_up_read, |
| 889 |
.name = "percpu_rwsem_lock" |
| 890 |
}; |
| 891 |
|
| 892 |
/* |
| 893 |
* Lock torture writer kthread. Repeatedly acquires and releases |
| 894 |
* the lock, checking for duplicate acquisitions. |
| 895 |
*/ |
| 896 |
static int lock_torture_writer(void *arg) |
| 897 |
{ |
| 898 |
unsigned long j; |
| 899 |
unsigned long j1; |
| 900 |
u32 lockset_mask; |
| 901 |
struct lock_stress_stats *lwsp = arg; |
| 902 |
DEFINE_TORTURE_RANDOM(rand); |
| 903 |
bool skip_main_lock; |
| 904 |
int tid = lwsp - cxt.lwsa; |
| 905 |
|
| 906 |
VERBOSE_TOROUT_STRING("lock_torture_writer task started"); |
| 907 |
if (!rt_task(current)) |
| 908 |
set_user_nice(current, MAX_NICE); |
| 909 |
|
| 910 |
do { |
| 911 |
if ((torture_random(&rand) & 0xfffff) == 0) |
| 912 |
schedule_timeout_uninterruptible(1); |
| 913 |
|
| 914 |
lockset_mask = torture_random(&rand); |
| 915 |
/* |
| 916 |
* When using nested_locks, we want to occasionally |
| 917 |
* skip the main lock so we can avoid always serializing |
| 918 |
* the lock chains on that central lock. By skipping the |
| 919 |
* main lock occasionally, we can create different |
| 920 |
* contention patterns (allowing for multiple disjoint |
| 921 |
* blocked trees) |
| 922 |
*/ |
| 923 |
skip_main_lock = (nested_locks && |
| 924 |
!(torture_random(&rand) % 100)); |
| 925 |
|
| 926 |
cxt.cur_ops->task_boost(&rand); |
| 927 |
if (cxt.cur_ops->nested_lock) |
| 928 |
cxt.cur_ops->nested_lock(tid, lockset_mask); |
| 929 |
|
| 930 |
if (!skip_main_lock) { |
| 931 |
if (acq_writer_lim > 0) |
| 932 |
j = jiffies; |
| 933 |
cxt.cur_ops->writelock(tid); |
| 934 |
if (WARN_ON_ONCE(lock_is_write_held)) |
| 935 |
lwsp->n_lock_fail++; |
| 936 |
lock_is_write_held = true; |
| 937 |
if (WARN_ON_ONCE(atomic_read(&lock_is_read_held))) |
| 938 |
lwsp->n_lock_fail++; /* rare, but... */ |
| 939 |
if (acq_writer_lim > 0) { |
| 940 |
j1 = jiffies; |
| 941 |
WARN_ONCE(time_after(j1, j + acq_writer_lim), |
| 942 |
"%s: Lock acquisition took %lu jiffies.\n", |
| 943 |
__func__, j1 - j); |
| 944 |
} |
| 945 |
lwsp->n_lock_acquired++; |
| 946 |
|
| 947 |
cxt.cur_ops->write_delay(&rand); |
| 948 |
|
| 949 |
lock_is_write_held = false; |
| 950 |
WRITE_ONCE(last_lock_release, jiffies); |
| 951 |
cxt.cur_ops->writeunlock(tid); |
| 952 |
} |
| 953 |
if (cxt.cur_ops->nested_unlock) |
| 954 |
cxt.cur_ops->nested_unlock(tid, lockset_mask); |
| 955 |
|
| 956 |
stutter_wait("lock_torture_writer"); |
| 957 |
} while (!torture_must_stop()); |
| 958 |
|
| 959 |
cxt.cur_ops->task_boost(NULL); /* reset prio */ |
| 960 |
torture_kthread_stopping("lock_torture_writer"); |
| 961 |
return 0; |
| 962 |
} |
| 963 |
|
| 964 |
/* |
| 965 |
* Lock torture reader kthread. Repeatedly acquires and releases |
| 966 |
* the reader lock. |
| 967 |
*/ |
| 968 |
static int lock_torture_reader(void *arg) |
| 969 |
{ |
| 970 |
struct lock_stress_stats *lrsp = arg; |
| 971 |
int tid = lrsp - cxt.lrsa; |
| 972 |
DEFINE_TORTURE_RANDOM(rand); |
| 973 |
|
| 974 |
VERBOSE_TOROUT_STRING("lock_torture_reader task started"); |
| 975 |
set_user_nice(current, MAX_NICE); |
| 976 |
|
| 977 |
do { |
| 978 |
if ((torture_random(&rand) & 0xfffff) == 0) |
| 979 |
schedule_timeout_uninterruptible(1); |
| 980 |
|
| 981 |
cxt.cur_ops->readlock(tid); |
| 982 |
atomic_inc(&lock_is_read_held); |
| 983 |
if (WARN_ON_ONCE(lock_is_write_held)) |
| 984 |
lrsp->n_lock_fail++; /* rare, but... */ |
| 985 |
|
| 986 |
lrsp->n_lock_acquired++; |
| 987 |
cxt.cur_ops->read_delay(&rand); |
| 988 |
atomic_dec(&lock_is_read_held); |
| 989 |
cxt.cur_ops->readunlock(tid); |
| 990 |
|
| 991 |
stutter_wait("lock_torture_reader"); |
| 992 |
} while (!torture_must_stop()); |
| 993 |
torture_kthread_stopping("lock_torture_reader"); |
| 994 |
return 0; |
| 995 |
} |
| 996 |
|
| 997 |
/* |
| 998 |
* Create an lock-torture-statistics message in the specified buffer. |
| 999 |
*/ |
| 1000 |
static void __torture_print_stats(char *page, |
| 1001 |
struct lock_stress_stats *statp, bool write) |
| 1002 |
{ |
| 1003 |
long cur; |
| 1004 |
bool fail = false; |
| 1005 |
int i, n_stress; |
| 1006 |
long max = 0, min = statp ? data_race(statp[0].n_lock_acquired) : 0; |
| 1007 |
long long sum = 0; |
| 1008 |
|
| 1009 |
n_stress = write ? cxt.nrealwriters_stress : cxt.nrealreaders_stress; |
| 1010 |
for (i = 0; i < n_stress; i++) { |
| 1011 |
if (data_race(statp[i].n_lock_fail)) |
| 1012 |
fail = true; |
| 1013 |
cur = data_race(statp[i].n_lock_acquired); |
| 1014 |
sum += cur; |
| 1015 |
if (max < cur) |
| 1016 |
max = cur; |
| 1017 |
if (min > cur) |
| 1018 |
min = cur; |
| 1019 |
} |
| 1020 |
page += sprintf(page, |
| 1021 |
"%s: Total: %lld Max/Min: %ld/%ld %s Fail: %d %s\n", |
| 1022 |
write ? "Writes" : "Reads ", |
| 1023 |
sum, max, min, |
| 1024 |
!onoff_interval && max / 2 > min ? "???" : "", |
| 1025 |
fail, fail ? "!!!" : ""); |
| 1026 |
if (fail) |
| 1027 |
atomic_inc(&cxt.n_lock_torture_errors); |
| 1028 |
} |
| 1029 |
|
| 1030 |
/* |
| 1031 |
* Print torture statistics. Caller must ensure that there is only one |
| 1032 |
* call to this function at a given time!!! This is normally accomplished |
| 1033 |
* by relying on the module system to only have one copy of the module |
| 1034 |
* loaded, and then by giving the lock_torture_stats kthread full control |
| 1035 |
* (or the init/cleanup functions when lock_torture_stats thread is not |
| 1036 |
* running). |
| 1037 |
*/ |
| 1038 |
static void lock_torture_stats_print(void) |
| 1039 |
{ |
| 1040 |
int size = cxt.nrealwriters_stress * 200 + 8192; |
| 1041 |
char *buf; |
| 1042 |
|
| 1043 |
if (cxt.cur_ops->readlock) |
| 1044 |
size += cxt.nrealreaders_stress * 200 + 8192; |
| 1045 |
|
| 1046 |
buf = kmalloc(size, GFP_KERNEL); |
| 1047 |
if (!buf) { |
| 1048 |
pr_err("lock_torture_stats_print: Out of memory, need: %d", |
| 1049 |
size); |
| 1050 |
return; |
| 1051 |
} |
| 1052 |
|
| 1053 |
__torture_print_stats(buf, cxt.lwsa, true); |
| 1054 |
pr_alert("%s", buf); |
| 1055 |
kfree(buf); |
| 1056 |
|
| 1057 |
if (cxt.cur_ops->readlock) { |
| 1058 |
buf = kmalloc(size, GFP_KERNEL); |
| 1059 |
if (!buf) { |
| 1060 |
pr_err("lock_torture_stats_print: Out of memory, need: %d", |
| 1061 |
size); |
| 1062 |
return; |
| 1063 |
} |
| 1064 |
|
| 1065 |
__torture_print_stats(buf, cxt.lrsa, false); |
| 1066 |
pr_alert("%s", buf); |
| 1067 |
kfree(buf); |
| 1068 |
} |
| 1069 |
} |
| 1070 |
|
| 1071 |
/* |
| 1072 |
* Periodically prints torture statistics, if periodic statistics printing |
| 1073 |
* was specified via the stat_interval module parameter. |
| 1074 |
* |
| 1075 |
* No need to worry about fullstop here, since this one doesn't reference |
| 1076 |
* volatile state or register callbacks. |
| 1077 |
*/ |
| 1078 |
static int lock_torture_stats(void *arg) |
| 1079 |
{ |
| 1080 |
VERBOSE_TOROUT_STRING("lock_torture_stats task started"); |
| 1081 |
do { |
| 1082 |
schedule_timeout_interruptible(stat_interval * HZ); |
| 1083 |
lock_torture_stats_print(); |
| 1084 |
torture_shutdown_absorb("lock_torture_stats"); |
| 1085 |
} while (!torture_must_stop()); |
| 1086 |
torture_kthread_stopping("lock_torture_stats"); |
| 1087 |
return 0; |
| 1088 |
} |
| 1089 |
|
| 1090 |
|
| 1091 |
static inline void |
| 1092 |
lock_torture_print_module_parms(struct lock_torture_ops *cur_ops, |
| 1093 |
const char *tag) |
| 1094 |
{ |
| 1095 |
static cpumask_t cpumask_all; |
| 1096 |
cpumask_t *rcmp = cpumask_nonempty(bind_readers) ? bind_readers : &cpumask_all; |
| 1097 |
cpumask_t *wcmp = cpumask_nonempty(bind_writers) ? bind_writers : &cpumask_all; |
| 1098 |
|
| 1099 |
cpumask_setall(&cpumask_all); |
| 1100 |
pr_alert("%s" TORTURE_FLAG |
| 1101 |
"--- %s%s: acq_writer_lim=%d bind_readers=%*pbl bind_writers=%*pbl call_rcu_chains=%d long_hold=%d nested_locks=%d nreaders_stress=%d nwriters_stress=%d onoff_holdoff=%d onoff_interval=%d rt_boost=%d rt_boost_factor=%d shuffle_interval=%d shutdown_secs=%d stat_interval=%d stutter=%d verbose=%d writer_fifo=%d\n", |
| 1102 |
torture_type, tag, cxt.debug_lock ? " [debug]": "", |
| 1103 |
acq_writer_lim, cpumask_pr_args(rcmp), cpumask_pr_args(wcmp), |
| 1104 |
call_rcu_chains, long_hold, nested_locks, cxt.nrealreaders_stress, |
| 1105 |
cxt.nrealwriters_stress, onoff_holdoff, onoff_interval, rt_boost, |
| 1106 |
rt_boost_factor, shuffle_interval, shutdown_secs, stat_interval, stutter, |
| 1107 |
verbose, writer_fifo); |
| 1108 |
} |
| 1109 |
|
| 1110 |
// If requested, maintain call_rcu() chains to keep a grace period always |
| 1111 |
// in flight. These increase the probability of getting an RCU CPU stall |
| 1112 |
// warning and associated diagnostics when a locking primitive stalls. |
| 1113 |
|
| 1114 |
static void call_rcu_chain_cb(struct rcu_head *rhp) |
| 1115 |
{ |
| 1116 |
struct call_rcu_chain *crcp = container_of(rhp, struct call_rcu_chain, crc_rh); |
| 1117 |
|
| 1118 |
if (!smp_load_acquire(&crcp->crc_stop)) { |
| 1119 |
(void)start_poll_synchronize_rcu(); // Start one grace period... |
| 1120 |
call_rcu(&crcp->crc_rh, call_rcu_chain_cb); // ... and later start another. |
| 1121 |
} |
| 1122 |
} |
| 1123 |
|
| 1124 |
// Start the requested number of call_rcu() chains. |
| 1125 |
static int call_rcu_chain_init(void) |
| 1126 |
{ |
| 1127 |
int i; |
| 1128 |
|
| 1129 |
if (call_rcu_chains <= 0) |
| 1130 |
return 0; |
| 1131 |
call_rcu_chain_list = kzalloc_objs(*call_rcu_chain_list, |
| 1132 |
call_rcu_chains); |
| 1133 |
if (!call_rcu_chain_list) |
| 1134 |
return -ENOMEM; |
| 1135 |
for (i = 0; i < call_rcu_chains; i++) { |
| 1136 |
call_rcu_chain_list[i].crc_stop = false; |
| 1137 |
call_rcu(&call_rcu_chain_list[i].crc_rh, call_rcu_chain_cb); |
| 1138 |
} |
| 1139 |
return 0; |
| 1140 |
} |
| 1141 |
|
| 1142 |
// Stop all of the call_rcu() chains. |
| 1143 |
static void call_rcu_chain_cleanup(void) |
| 1144 |
{ |
| 1145 |
int i; |
| 1146 |
|
| 1147 |
if (!call_rcu_chain_list) |
| 1148 |
return; |
| 1149 |
for (i = 0; i < call_rcu_chains; i++) |
| 1150 |
smp_store_release(&call_rcu_chain_list[i].crc_stop, true); |
| 1151 |
rcu_barrier(); |
| 1152 |
kfree(call_rcu_chain_list); |
| 1153 |
call_rcu_chain_list = NULL; |
| 1154 |
} |
| 1155 |
|
| 1156 |
static void lock_torture_cleanup(void) |
| 1157 |
{ |
| 1158 |
int i; |
| 1159 |
|
| 1160 |
if (torture_cleanup_begin()) |
| 1161 |
return; |
| 1162 |
|
| 1163 |
/* |
| 1164 |
* Indicates early cleanup, meaning that the test has not run, |
| 1165 |
* such as when passing bogus args when loading the module. |
| 1166 |
* However cxt->cur_ops.init() may have been invoked, so beside |
| 1167 |
* perform the underlying torture-specific cleanups, cur_ops.exit() |
| 1168 |
* will be invoked if needed. |
| 1169 |
*/ |
| 1170 |
if (!cxt.lwsa && !cxt.lrsa) |
| 1171 |
goto end; |
| 1172 |
|
| 1173 |
if (writer_tasks) { |
| 1174 |
for (i = 0; i < cxt.nrealwriters_stress; i++) |
| 1175 |
torture_stop_kthread(lock_torture_writer, writer_tasks[i]); |
| 1176 |
kfree(writer_tasks); |
| 1177 |
writer_tasks = NULL; |
| 1178 |
} |
| 1179 |
|
| 1180 |
if (reader_tasks) { |
| 1181 |
for (i = 0; i < cxt.nrealreaders_stress; i++) |
| 1182 |
torture_stop_kthread(lock_torture_reader, |
| 1183 |
reader_tasks[i]); |
| 1184 |
kfree(reader_tasks); |
| 1185 |
reader_tasks = NULL; |
| 1186 |
} |
| 1187 |
|
| 1188 |
torture_stop_kthread(lock_torture_stats, stats_task); |
| 1189 |
lock_torture_stats_print(); /* -After- the stats thread is stopped! */ |
| 1190 |
|
| 1191 |
if (atomic_read(&cxt.n_lock_torture_errors)) |
| 1192 |
lock_torture_print_module_parms(cxt.cur_ops, |
| 1193 |
"End of test: FAILURE"); |
| 1194 |
else if (torture_onoff_failures()) |
| 1195 |
lock_torture_print_module_parms(cxt.cur_ops, |
| 1196 |
"End of test: LOCK_HOTPLUG"); |
| 1197 |
else |
| 1198 |
lock_torture_print_module_parms(cxt.cur_ops, |
| 1199 |
"End of test: SUCCESS"); |
| 1200 |
|
| 1201 |
kfree(cxt.lwsa); |
| 1202 |
cxt.lwsa = NULL; |
| 1203 |
kfree(cxt.lrsa); |
| 1204 |
cxt.lrsa = NULL; |
| 1205 |
|
| 1206 |
call_rcu_chain_cleanup(); |
| 1207 |
|
| 1208 |
end: |
| 1209 |
if (cxt.init_called) { |
| 1210 |
if (cxt.cur_ops->exit) |
| 1211 |
cxt.cur_ops->exit(); |
| 1212 |
cxt.init_called = false; |
| 1213 |
} |
| 1214 |
|
| 1215 |
free_cpumask_var(bind_readers); |
| 1216 |
free_cpumask_var(bind_writers); |
| 1217 |
|
| 1218 |
torture_cleanup_end(); |
| 1219 |
} |
| 1220 |
|
| 1221 |
static int __init lock_torture_init(void) |
| 1222 |
{ |
| 1223 |
int i, j; |
| 1224 |
int firsterr = 0; |
| 1225 |
static struct lock_torture_ops *torture_ops[] = { |
| 1226 |
&lock_busted_ops, |
| 1227 |
&spin_lock_ops, &spin_lock_irq_ops, |
| 1228 |
&raw_spin_lock_ops, &raw_spin_lock_irq_ops, |
| 1229 |
#ifdef CONFIG_BPF_SYSCALL |
| 1230 |
&raw_res_spin_lock_ops, &raw_res_spin_lock_irq_ops, |
| 1231 |
#endif |
| 1232 |
&rw_lock_ops, &rw_lock_irq_ops, |
| 1233 |
&mutex_lock_ops, |
| 1234 |
&ww_mutex_lock_ops, |
| 1235 |
#ifdef CONFIG_RT_MUTEXES |
| 1236 |
&rtmutex_lock_ops, |
| 1237 |
#endif |
| 1238 |
&rwsem_lock_ops, |
| 1239 |
&percpu_rwsem_lock_ops, |
| 1240 |
}; |
| 1241 |
|
| 1242 |
if (!torture_init_begin(torture_type, verbose)) |
| 1243 |
return -EBUSY; |
| 1244 |
|
| 1245 |
/* Process args and tell the world that the torturer is on the job. */ |
| 1246 |
for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { |
| 1247 |
cxt.cur_ops = torture_ops[i]; |
| 1248 |
if (strcmp(torture_type, cxt.cur_ops->name) == 0) |
| 1249 |
break; |
| 1250 |
} |
| 1251 |
if (i == ARRAY_SIZE(torture_ops)) { |
| 1252 |
pr_alert("lock-torture: invalid torture type: \"%s\"\n", |
| 1253 |
torture_type); |
| 1254 |
pr_alert("lock-torture types:"); |
| 1255 |
for (i = 0; i < ARRAY_SIZE(torture_ops); i++) |
| 1256 |
pr_alert(" %s", torture_ops[i]->name); |
| 1257 |
pr_alert("\n"); |
| 1258 |
firsterr = -EINVAL; |
| 1259 |
goto unwind; |
| 1260 |
} |
| 1261 |
|
| 1262 |
if (nwriters_stress == 0 && |
| 1263 |
(!cxt.cur_ops->readlock || nreaders_stress == 0)) { |
| 1264 |
pr_alert("lock-torture: must run at least one locking thread\n"); |
| 1265 |
firsterr = -EINVAL; |
| 1266 |
goto unwind; |
| 1267 |
} |
| 1268 |
|
| 1269 |
if (nwriters_stress >= 0) |
| 1270 |
cxt.nrealwriters_stress = nwriters_stress; |
| 1271 |
else |
| 1272 |
cxt.nrealwriters_stress = 2 * num_online_cpus(); |
| 1273 |
|
| 1274 |
if (cxt.cur_ops->init) { |
| 1275 |
cxt.cur_ops->init(); |
| 1276 |
cxt.init_called = true; |
| 1277 |
} |
| 1278 |
|
| 1279 |
#ifdef CONFIG_DEBUG_MUTEXES |
| 1280 |
if (str_has_prefix(torture_type, "mutex")) |
| 1281 |
cxt.debug_lock = true; |
| 1282 |
#endif |
| 1283 |
#ifdef CONFIG_DEBUG_RT_MUTEXES |
| 1284 |
if (str_has_prefix(torture_type, "rtmutex")) |
| 1285 |
cxt.debug_lock = true; |
| 1286 |
#endif |
| 1287 |
#ifdef CONFIG_DEBUG_SPINLOCK |
| 1288 |
if ((str_has_prefix(torture_type, "spin")) || |
| 1289 |
(str_has_prefix(torture_type, "rw_lock"))) |
| 1290 |
cxt.debug_lock = true; |
| 1291 |
#endif |
| 1292 |
|
| 1293 |
/* Initialize the statistics so that each run gets its own numbers. */ |
| 1294 |
if (nwriters_stress) { |
| 1295 |
lock_is_write_held = false; |
| 1296 |
cxt.lwsa = kmalloc_objs(*cxt.lwsa, cxt.nrealwriters_stress); |
| 1297 |
if (cxt.lwsa == NULL) { |
| 1298 |
VERBOSE_TOROUT_STRING("cxt.lwsa: Out of memory"); |
| 1299 |
firsterr = -ENOMEM; |
| 1300 |
goto unwind; |
| 1301 |
} |
| 1302 |
|
| 1303 |
for (i = 0; i < cxt.nrealwriters_stress; i++) { |
| 1304 |
cxt.lwsa[i].n_lock_fail = 0; |
| 1305 |
cxt.lwsa[i].n_lock_acquired = 0; |
| 1306 |
} |
| 1307 |
} |
| 1308 |
|
| 1309 |
if (cxt.cur_ops->readlock) { |
| 1310 |
if (nreaders_stress >= 0) |
| 1311 |
cxt.nrealreaders_stress = nreaders_stress; |
| 1312 |
else { |
| 1313 |
/* |
| 1314 |
* By default distribute evenly the number of |
| 1315 |
* readers and writers. We still run the same number |
| 1316 |
* of threads as the writer-only locks default. |
| 1317 |
*/ |
| 1318 |
if (nwriters_stress < 0) /* user doesn't care */ |
| 1319 |
cxt.nrealwriters_stress = num_online_cpus(); |
| 1320 |
cxt.nrealreaders_stress = cxt.nrealwriters_stress; |
| 1321 |
} |
| 1322 |
|
| 1323 |
if (nreaders_stress) { |
| 1324 |
cxt.lrsa = kmalloc_objs(*cxt.lrsa, |
| 1325 |
cxt.nrealreaders_stress); |
| 1326 |
if (cxt.lrsa == NULL) { |
| 1327 |
VERBOSE_TOROUT_STRING("cxt.lrsa: Out of memory"); |
| 1328 |
firsterr = -ENOMEM; |
| 1329 |
kfree(cxt.lwsa); |
| 1330 |
cxt.lwsa = NULL; |
| 1331 |
goto unwind; |
| 1332 |
} |
| 1333 |
|
| 1334 |
for (i = 0; i < cxt.nrealreaders_stress; i++) { |
| 1335 |
cxt.lrsa[i].n_lock_fail = 0; |
| 1336 |
cxt.lrsa[i].n_lock_acquired = 0; |
| 1337 |
} |
| 1338 |
} |
| 1339 |
} |
| 1340 |
|
| 1341 |
firsterr = call_rcu_chain_init(); |
| 1342 |
if (torture_init_error(firsterr)) |
| 1343 |
goto unwind; |
| 1344 |
|
| 1345 |
lock_torture_print_module_parms(cxt.cur_ops, "Start of test"); |
| 1346 |
|
| 1347 |
/* Prepare torture context. */ |
| 1348 |
if (onoff_interval > 0) { |
| 1349 |
firsterr = torture_onoff_init(onoff_holdoff * HZ, |
| 1350 |
onoff_interval * HZ, NULL); |
| 1351 |
if (torture_init_error(firsterr)) |
| 1352 |
goto unwind; |
| 1353 |
} |
| 1354 |
if (shuffle_interval > 0) { |
| 1355 |
firsterr = torture_shuffle_init(shuffle_interval); |
| 1356 |
if (torture_init_error(firsterr)) |
| 1357 |
goto unwind; |
| 1358 |
} |
| 1359 |
if (shutdown_secs > 0) { |
| 1360 |
firsterr = torture_shutdown_init(shutdown_secs, |
| 1361 |
lock_torture_cleanup); |
| 1362 |
if (torture_init_error(firsterr)) |
| 1363 |
goto unwind; |
| 1364 |
} |
| 1365 |
if (stutter > 0) { |
| 1366 |
firsterr = torture_stutter_init(stutter, stutter); |
| 1367 |
if (torture_init_error(firsterr)) |
| 1368 |
goto unwind; |
| 1369 |
} |
| 1370 |
|
| 1371 |
if (nwriters_stress) { |
| 1372 |
writer_tasks = kzalloc_objs(writer_tasks[0], |
| 1373 |
cxt.nrealwriters_stress); |
| 1374 |
if (writer_tasks == NULL) { |
| 1375 |
TOROUT_ERRSTRING("writer_tasks: Out of memory"); |
| 1376 |
firsterr = -ENOMEM; |
| 1377 |
goto unwind; |
| 1378 |
} |
| 1379 |
} |
| 1380 |
|
| 1381 |
/* cap nested_locks to MAX_NESTED_LOCKS */ |
| 1382 |
if (nested_locks > MAX_NESTED_LOCKS) |
| 1383 |
nested_locks = MAX_NESTED_LOCKS; |
| 1384 |
|
| 1385 |
if (cxt.cur_ops->readlock) { |
| 1386 |
reader_tasks = kzalloc_objs(reader_tasks[0], |
| 1387 |
cxt.nrealreaders_stress); |
| 1388 |
if (reader_tasks == NULL) { |
| 1389 |
TOROUT_ERRSTRING("reader_tasks: Out of memory"); |
| 1390 |
kfree(writer_tasks); |
| 1391 |
writer_tasks = NULL; |
| 1392 |
firsterr = -ENOMEM; |
| 1393 |
goto unwind; |
| 1394 |
} |
| 1395 |
} |
| 1396 |
|
| 1397 |
/* |
| 1398 |
* Create the kthreads and start torturing (oh, those poor little locks). |
| 1399 |
* |
| 1400 |
* TODO: Note that we interleave writers with readers, giving writers a |
| 1401 |
* slight advantage, by creating its kthread first. This can be modified |
| 1402 |
* for very specific needs, or even let the user choose the policy, if |
| 1403 |
* ever wanted. |
| 1404 |
*/ |
| 1405 |
for (i = 0, j = 0; i < cxt.nrealwriters_stress || |
| 1406 |
j < cxt.nrealreaders_stress; i++, j++) { |
| 1407 |
if (i >= cxt.nrealwriters_stress) |
| 1408 |
goto create_reader; |
| 1409 |
|
| 1410 |
/* Create writer. */ |
| 1411 |
firsterr = torture_create_kthread_cb(lock_torture_writer, &cxt.lwsa[i], |
| 1412 |
writer_tasks[i], |
| 1413 |
writer_fifo ? sched_set_fifo : NULL); |
| 1414 |
if (torture_init_error(firsterr)) |
| 1415 |
goto unwind; |
| 1416 |
if (cpumask_nonempty(bind_writers)) |
| 1417 |
torture_sched_setaffinity(writer_tasks[i]->pid, bind_writers, true); |
| 1418 |
|
| 1419 |
create_reader: |
| 1420 |
if (cxt.cur_ops->readlock == NULL || (j >= cxt.nrealreaders_stress)) |
| 1421 |
continue; |
| 1422 |
/* Create reader. */ |
| 1423 |
firsterr = torture_create_kthread(lock_torture_reader, &cxt.lrsa[j], |
| 1424 |
reader_tasks[j]); |
| 1425 |
if (torture_init_error(firsterr)) |
| 1426 |
goto unwind; |
| 1427 |
if (cpumask_nonempty(bind_readers)) |
| 1428 |
torture_sched_setaffinity(reader_tasks[j]->pid, bind_readers, true); |
| 1429 |
} |
| 1430 |
if (stat_interval > 0) { |
| 1431 |
firsterr = torture_create_kthread(lock_torture_stats, NULL, |
| 1432 |
stats_task); |
| 1433 |
if (torture_init_error(firsterr)) |
| 1434 |
goto unwind; |
| 1435 |
} |
| 1436 |
torture_init_end(); |
| 1437 |
return 0; |
| 1438 |
|
| 1439 |
unwind: |
| 1440 |
torture_init_end(); |
| 1441 |
lock_torture_cleanup(); |
| 1442 |
if (shutdown_secs) { |
| 1443 |
WARN_ON(!IS_MODULE(CONFIG_LOCK_TORTURE_TEST)); |
| 1444 |
kernel_power_off(); |
| 1445 |
} |
| 1446 |
return firsterr; |
| 1447 |
} |
| 1448 |
|
| 1449 |
module_init(lock_torture_init); |
| 1450 |
module_exit(lock_torture_cleanup); |