Commit | Line | Data |
---|---|---|
653ed64b JFG |
1 | // SPDX-License-Identifier: GPL-2.0+ |
2 | // | |
8e4ec3d0 | 3 | // Scalability test comparing RCU vs other mechanisms |
653ed64b JFG |
4 | // for acquiring references on objects. |
5 | // | |
6 | // Copyright (C) Google, 2020. | |
7 | // | |
8 | // Author: Joel Fernandes <joel@joelfernandes.org> | |
9 | ||
10 | #define pr_fmt(fmt) fmt | |
11 | ||
12 | #include <linux/atomic.h> | |
13 | #include <linux/bitops.h> | |
14 | #include <linux/completion.h> | |
15 | #include <linux/cpu.h> | |
16 | #include <linux/delay.h> | |
17 | #include <linux/err.h> | |
18 | #include <linux/init.h> | |
19 | #include <linux/interrupt.h> | |
20 | #include <linux/kthread.h> | |
21 | #include <linux/kernel.h> | |
22 | #include <linux/mm.h> | |
23 | #include <linux/module.h> | |
24 | #include <linux/moduleparam.h> | |
25 | #include <linux/notifier.h> | |
26 | #include <linux/percpu.h> | |
27 | #include <linux/rcupdate.h> | |
72bb749e | 28 | #include <linux/rcupdate_trace.h> |
653ed64b JFG |
29 | #include <linux/reboot.h> |
30 | #include <linux/sched.h> | |
31 | #include <linux/spinlock.h> | |
32 | #include <linux/smp.h> | |
33 | #include <linux/stat.h> | |
34 | #include <linux/srcu.h> | |
35 | #include <linux/slab.h> | |
36 | #include <linux/torture.h> | |
37 | #include <linux/types.h> | |
38 | ||
39 | #include "rcu.h" | |
40 | ||
1fbeb3a8 | 41 | #define SCALE_FLAG "-ref-scale: " |
653ed64b | 42 | |
1fbeb3a8 PM |
43 | #define SCALEOUT(s, x...) \ |
44 | pr_alert("%s" SCALE_FLAG s, scale_type, ## x) | |
653ed64b | 45 | |
1fbeb3a8 PM |
46 | #define VERBOSE_SCALEOUT(s, x...) \ |
47 | do { if (verbose) pr_alert("%s" SCALE_FLAG s, scale_type, ## x); } while (0) | |
653ed64b | 48 | |
e76506f0 PM |
49 | static atomic_t verbose_batch_ctr; |
50 | ||
51 | #define VERBOSE_SCALEOUT_BATCH(s, x...) \ | |
52 | do { \ | |
53 | if (verbose && \ | |
54 | (verbose_batched <= 0 || \ | |
414c116e PM |
55 | !(atomic_inc_return(&verbose_batch_ctr) % verbose_batched))) { \ |
56 | schedule_timeout_uninterruptible(1); \ | |
e76506f0 | 57 | pr_alert("%s" SCALE_FLAG s, scale_type, ## x); \ |
414c116e | 58 | } \ |
e76506f0 PM |
59 | } while (0) |
60 | ||
1fbeb3a8 PM |
61 | #define VERBOSE_SCALEOUT_ERRSTRING(s, x...) \ |
62 | do { if (verbose) pr_alert("%s" SCALE_FLAG "!!! " s, scale_type, ## x); } while (0) | |
653ed64b JFG |
63 | |
64 | MODULE_LICENSE("GPL"); | |
65 | MODULE_AUTHOR("Joel Fernandes (Google) <joel@joelfernandes.org>"); | |
66 | ||
1fbeb3a8 PM |
67 | static char *scale_type = "rcu"; |
68 | module_param(scale_type, charp, 0444); | |
69 | MODULE_PARM_DESC(scale_type, "Type of test (rcu, srcu, refcnt, rwsem, rwlock."); | |
653ed64b JFG |
70 | |
71 | torture_param(int, verbose, 0, "Enable verbose debugging printk()s"); | |
e76506f0 | 72 | torture_param(int, verbose_batched, 0, "Batch verbose debugging printk()s"); |
653ed64b | 73 | |
777a54c9 | 74 | // Wait until there are multiple CPUs before starting test. |
8e4ec3d0 | 75 | torture_param(int, holdoff, IS_BUILTIN(CONFIG_RCU_REF_SCALE_TEST) ? 10 : 0, |
777a54c9 PM |
76 | "Holdoff time before test start (s)"); |
77 | // Number of loops per experiment, all readers execute operations concurrently. | |
4dd72a33 | 78 | torture_param(long, loops, 10000, "Number of loops per experiment."); |
8fc28783 PM |
79 | // Number of readers, with -1 defaulting to about 75% of the CPUs. |
80 | torture_param(int, nreaders, -1, "Number of readers, -1 for 75% of CPUs."); | |
81 | // Number of runs. | |
82 | torture_param(int, nruns, 30, "Number of experiments to run."); | |
918b351d PM |
83 | // Reader delay in nanoseconds, 0 for no delay. |
84 | torture_param(int, readdelay, 0, "Read-side delay in nanoseconds."); | |
653ed64b JFG |
85 | |
86 | #ifdef MODULE | |
1fbeb3a8 | 87 | # define REFSCALE_SHUTDOWN 0 |
653ed64b | 88 | #else |
1fbeb3a8 | 89 | # define REFSCALE_SHUTDOWN 1 |
653ed64b JFG |
90 | #endif |
91 | ||
1fbeb3a8 PM |
92 | torture_param(bool, shutdown, REFSCALE_SHUTDOWN, |
93 | "Shutdown at end of scalability tests."); | |
653ed64b JFG |
94 | |
95 | struct reader_task { | |
96 | struct task_struct *task; | |
af2789db | 97 | int start_reader; |
653ed64b JFG |
98 | wait_queue_head_t wq; |
99 | u64 last_duration_ns; | |
653ed64b JFG |
100 | }; |
101 | ||
102 | static struct task_struct *shutdown_task; | |
103 | static wait_queue_head_t shutdown_wq; | |
104 | ||
105 | static struct task_struct *main_task; | |
106 | static wait_queue_head_t main_wq; | |
107 | static int shutdown_start; | |
108 | ||
109 | static struct reader_task *reader_tasks; | |
653ed64b JFG |
110 | |
111 | // Number of readers that are part of the current experiment. | |
112 | static atomic_t nreaders_exp; | |
113 | ||
114 | // Use to wait for all threads to start. | |
115 | static atomic_t n_init; | |
86e0da2b | 116 | static atomic_t n_started; |
2db0bda3 PM |
117 | static atomic_t n_warmedup; |
118 | static atomic_t n_cooleddown; | |
653ed64b JFG |
119 | |
120 | // Track which experiment is currently running. | |
121 | static int exp_idx; | |
122 | ||
123 | // Operations vector for selecting different types of tests. | |
1fbeb3a8 | 124 | struct ref_scale_ops { |
653ed64b JFG |
125 | void (*init)(void); |
126 | void (*cleanup)(void); | |
75dd8efe | 127 | void (*readsection)(const int nloops); |
918b351d | 128 | void (*delaysection)(const int nloops, const int udl, const int ndl); |
653ed64b JFG |
129 | const char *name; |
130 | }; | |
131 | ||
1fbeb3a8 | 132 | static struct ref_scale_ops *cur_ops; |
653ed64b | 133 | |
918b351d PM |
134 | static void un_delay(const int udl, const int ndl) |
135 | { | |
136 | if (udl) | |
137 | udelay(udl); | |
138 | if (ndl) | |
139 | ndelay(ndl); | |
140 | } | |
141 | ||
75dd8efe | 142 | static void ref_rcu_read_section(const int nloops) |
653ed64b | 143 | { |
75dd8efe | 144 | int i; |
653ed64b | 145 | |
75dd8efe PM |
146 | for (i = nloops; i >= 0; i--) { |
147 | rcu_read_lock(); | |
148 | rcu_read_unlock(); | |
149 | } | |
653ed64b JFG |
150 | } |
151 | ||
918b351d | 152 | static void ref_rcu_delay_section(const int nloops, const int udl, const int ndl) |
b4d1e34f PM |
153 | { |
154 | int i; | |
155 | ||
156 | for (i = nloops; i >= 0; i--) { | |
157 | rcu_read_lock(); | |
918b351d | 158 | un_delay(udl, ndl); |
b4d1e34f PM |
159 | rcu_read_unlock(); |
160 | } | |
161 | } | |
162 | ||
1fbeb3a8 | 163 | static void rcu_sync_scale_init(void) |
653ed64b JFG |
164 | { |
165 | } | |
166 | ||
1fbeb3a8 PM |
167 | static struct ref_scale_ops rcu_ops = { |
168 | .init = rcu_sync_scale_init, | |
75dd8efe | 169 | .readsection = ref_rcu_read_section, |
b4d1e34f | 170 | .delaysection = ref_rcu_delay_section, |
653ed64b JFG |
171 | .name = "rcu" |
172 | }; | |
173 | ||
1fbeb3a8 PM |
174 | // Definitions for SRCU ref scale testing. |
175 | DEFINE_STATIC_SRCU(srcu_refctl_scale); | |
176 | static struct srcu_struct *srcu_ctlp = &srcu_refctl_scale; | |
653ed64b | 177 | |
1fbeb3a8 | 178 | static void srcu_ref_scale_read_section(const int nloops) |
653ed64b | 179 | { |
75dd8efe PM |
180 | int i; |
181 | int idx; | |
653ed64b | 182 | |
75dd8efe PM |
183 | for (i = nloops; i >= 0; i--) { |
184 | idx = srcu_read_lock(srcu_ctlp); | |
185 | srcu_read_unlock(srcu_ctlp, idx); | |
186 | } | |
653ed64b JFG |
187 | } |
188 | ||
1fbeb3a8 | 189 | static void srcu_ref_scale_delay_section(const int nloops, const int udl, const int ndl) |
b4d1e34f PM |
190 | { |
191 | int i; | |
192 | int idx; | |
193 | ||
194 | for (i = nloops; i >= 0; i--) { | |
195 | idx = srcu_read_lock(srcu_ctlp); | |
918b351d | 196 | un_delay(udl, ndl); |
b4d1e34f PM |
197 | srcu_read_unlock(srcu_ctlp, idx); |
198 | } | |
199 | } | |
200 | ||
1fbeb3a8 PM |
201 | static struct ref_scale_ops srcu_ops = { |
202 | .init = rcu_sync_scale_init, | |
203 | .readsection = srcu_ref_scale_read_section, | |
204 | .delaysection = srcu_ref_scale_delay_section, | |
653ed64b JFG |
205 | .name = "srcu" |
206 | }; | |
207 | ||
1fbeb3a8 | 208 | // Definitions for RCU Tasks ref scale testing: Empty read markers. |
e13ef442 | 209 | // These definitions also work for RCU Rude readers. |
1fbeb3a8 | 210 | static void rcu_tasks_ref_scale_read_section(const int nloops) |
e13ef442 PM |
211 | { |
212 | int i; | |
213 | ||
214 | for (i = nloops; i >= 0; i--) | |
215 | continue; | |
216 | } | |
217 | ||
1fbeb3a8 | 218 | static void rcu_tasks_ref_scale_delay_section(const int nloops, const int udl, const int ndl) |
e13ef442 PM |
219 | { |
220 | int i; | |
221 | ||
222 | for (i = nloops; i >= 0; i--) | |
223 | un_delay(udl, ndl); | |
224 | } | |
225 | ||
1fbeb3a8 PM |
226 | static struct ref_scale_ops rcu_tasks_ops = { |
227 | .init = rcu_sync_scale_init, | |
228 | .readsection = rcu_tasks_ref_scale_read_section, | |
229 | .delaysection = rcu_tasks_ref_scale_delay_section, | |
e13ef442 PM |
230 | .name = "rcu-tasks" |
231 | }; | |
232 | ||
1fbeb3a8 PM |
233 | // Definitions for RCU Tasks Trace ref scale testing. |
234 | static void rcu_trace_ref_scale_read_section(const int nloops) | |
72bb749e PM |
235 | { |
236 | int i; | |
237 | ||
238 | for (i = nloops; i >= 0; i--) { | |
239 | rcu_read_lock_trace(); | |
240 | rcu_read_unlock_trace(); | |
241 | } | |
242 | } | |
243 | ||
1fbeb3a8 | 244 | static void rcu_trace_ref_scale_delay_section(const int nloops, const int udl, const int ndl) |
72bb749e PM |
245 | { |
246 | int i; | |
247 | ||
248 | for (i = nloops; i >= 0; i--) { | |
249 | rcu_read_lock_trace(); | |
250 | un_delay(udl, ndl); | |
251 | rcu_read_unlock_trace(); | |
252 | } | |
253 | } | |
254 | ||
1fbeb3a8 PM |
255 | static struct ref_scale_ops rcu_trace_ops = { |
256 | .init = rcu_sync_scale_init, | |
257 | .readsection = rcu_trace_ref_scale_read_section, | |
258 | .delaysection = rcu_trace_ref_scale_delay_section, | |
72bb749e PM |
259 | .name = "rcu-trace" |
260 | }; | |
261 | ||
653ed64b JFG |
262 | // Definitions for reference count |
263 | static atomic_t refcnt; | |
264 | ||
b4d1e34f | 265 | static void ref_refcnt_section(const int nloops) |
653ed64b | 266 | { |
75dd8efe | 267 | int i; |
653ed64b | 268 | |
75dd8efe PM |
269 | for (i = nloops; i >= 0; i--) { |
270 | atomic_inc(&refcnt); | |
271 | atomic_dec(&refcnt); | |
272 | } | |
653ed64b JFG |
273 | } |
274 | ||
918b351d | 275 | static void ref_refcnt_delay_section(const int nloops, const int udl, const int ndl) |
b4d1e34f PM |
276 | { |
277 | int i; | |
278 | ||
279 | for (i = nloops; i >= 0; i--) { | |
280 | atomic_inc(&refcnt); | |
918b351d | 281 | un_delay(udl, ndl); |
b4d1e34f PM |
282 | atomic_dec(&refcnt); |
283 | } | |
284 | } | |
285 | ||
1fbeb3a8 PM |
286 | static struct ref_scale_ops refcnt_ops = { |
287 | .init = rcu_sync_scale_init, | |
b4d1e34f PM |
288 | .readsection = ref_refcnt_section, |
289 | .delaysection = ref_refcnt_delay_section, | |
653ed64b JFG |
290 | .name = "refcnt" |
291 | }; | |
292 | ||
293 | // Definitions for rwlock | |
294 | static rwlock_t test_rwlock; | |
295 | ||
b4d1e34f | 296 | static void ref_rwlock_init(void) |
653ed64b JFG |
297 | { |
298 | rwlock_init(&test_rwlock); | |
299 | } | |
300 | ||
b4d1e34f PM |
301 | static void ref_rwlock_section(const int nloops) |
302 | { | |
303 | int i; | |
304 | ||
305 | for (i = nloops; i >= 0; i--) { | |
306 | read_lock(&test_rwlock); | |
307 | read_unlock(&test_rwlock); | |
308 | } | |
309 | } | |
310 | ||
918b351d | 311 | static void ref_rwlock_delay_section(const int nloops, const int udl, const int ndl) |
653ed64b | 312 | { |
75dd8efe | 313 | int i; |
653ed64b | 314 | |
75dd8efe PM |
315 | for (i = nloops; i >= 0; i--) { |
316 | read_lock(&test_rwlock); | |
918b351d | 317 | un_delay(udl, ndl); |
75dd8efe PM |
318 | read_unlock(&test_rwlock); |
319 | } | |
653ed64b JFG |
320 | } |
321 | ||
1fbeb3a8 | 322 | static struct ref_scale_ops rwlock_ops = { |
b4d1e34f PM |
323 | .init = ref_rwlock_init, |
324 | .readsection = ref_rwlock_section, | |
325 | .delaysection = ref_rwlock_delay_section, | |
653ed64b JFG |
326 | .name = "rwlock" |
327 | }; | |
328 | ||
329 | // Definitions for rwsem | |
330 | static struct rw_semaphore test_rwsem; | |
331 | ||
b4d1e34f | 332 | static void ref_rwsem_init(void) |
653ed64b JFG |
333 | { |
334 | init_rwsem(&test_rwsem); | |
335 | } | |
336 | ||
b4d1e34f | 337 | static void ref_rwsem_section(const int nloops) |
653ed64b | 338 | { |
75dd8efe | 339 | int i; |
653ed64b | 340 | |
75dd8efe PM |
341 | for (i = nloops; i >= 0; i--) { |
342 | down_read(&test_rwsem); | |
343 | up_read(&test_rwsem); | |
344 | } | |
653ed64b JFG |
345 | } |
346 | ||
918b351d | 347 | static void ref_rwsem_delay_section(const int nloops, const int udl, const int ndl) |
b4d1e34f PM |
348 | { |
349 | int i; | |
350 | ||
351 | for (i = nloops; i >= 0; i--) { | |
352 | down_read(&test_rwsem); | |
918b351d | 353 | un_delay(udl, ndl); |
b4d1e34f PM |
354 | up_read(&test_rwsem); |
355 | } | |
356 | } | |
357 | ||
1fbeb3a8 | 358 | static struct ref_scale_ops rwsem_ops = { |
b4d1e34f PM |
359 | .init = ref_rwsem_init, |
360 | .readsection = ref_rwsem_section, | |
361 | .delaysection = ref_rwsem_delay_section, | |
653ed64b JFG |
362 | .name = "rwsem" |
363 | }; | |
364 | ||
e9b800db PM |
365 | // Definitions for global spinlock |
366 | static DEFINE_SPINLOCK(test_lock); | |
367 | ||
368 | static void ref_lock_section(const int nloops) | |
369 | { | |
370 | int i; | |
371 | ||
372 | preempt_disable(); | |
373 | for (i = nloops; i >= 0; i--) { | |
374 | spin_lock(&test_lock); | |
375 | spin_unlock(&test_lock); | |
376 | } | |
377 | preempt_enable(); | |
378 | } | |
379 | ||
380 | static void ref_lock_delay_section(const int nloops, const int udl, const int ndl) | |
381 | { | |
382 | int i; | |
383 | ||
384 | preempt_disable(); | |
385 | for (i = nloops; i >= 0; i--) { | |
386 | spin_lock(&test_lock); | |
387 | un_delay(udl, ndl); | |
388 | spin_unlock(&test_lock); | |
389 | } | |
390 | preempt_enable(); | |
391 | } | |
392 | ||
393 | static struct ref_scale_ops lock_ops = { | |
394 | .readsection = ref_lock_section, | |
395 | .delaysection = ref_lock_delay_section, | |
396 | .name = "lock" | |
397 | }; | |
398 | ||
399 | // Definitions for global irq-save spinlock | |
400 | ||
401 | static void ref_lock_irq_section(const int nloops) | |
402 | { | |
403 | unsigned long flags; | |
404 | int i; | |
405 | ||
406 | preempt_disable(); | |
407 | for (i = nloops; i >= 0; i--) { | |
408 | spin_lock_irqsave(&test_lock, flags); | |
409 | spin_unlock_irqrestore(&test_lock, flags); | |
410 | } | |
411 | preempt_enable(); | |
412 | } | |
413 | ||
414 | static void ref_lock_irq_delay_section(const int nloops, const int udl, const int ndl) | |
415 | { | |
416 | unsigned long flags; | |
417 | int i; | |
418 | ||
419 | preempt_disable(); | |
420 | for (i = nloops; i >= 0; i--) { | |
421 | spin_lock_irqsave(&test_lock, flags); | |
422 | un_delay(udl, ndl); | |
423 | spin_unlock_irqrestore(&test_lock, flags); | |
424 | } | |
425 | preempt_enable(); | |
426 | } | |
427 | ||
428 | static struct ref_scale_ops lock_irq_ops = { | |
429 | .readsection = ref_lock_irq_section, | |
430 | .delaysection = ref_lock_irq_delay_section, | |
431 | .name = "lock-irq" | |
432 | }; | |
433 | ||
434 | // Definitions acquire-release. | |
435 | static DEFINE_PER_CPU(unsigned long, test_acqrel); | |
436 | ||
437 | static void ref_acqrel_section(const int nloops) | |
438 | { | |
439 | unsigned long x; | |
440 | int i; | |
441 | ||
442 | preempt_disable(); | |
443 | for (i = nloops; i >= 0; i--) { | |
444 | x = smp_load_acquire(this_cpu_ptr(&test_acqrel)); | |
445 | smp_store_release(this_cpu_ptr(&test_acqrel), x + 1); | |
446 | } | |
447 | preempt_enable(); | |
448 | } | |
449 | ||
450 | static void ref_acqrel_delay_section(const int nloops, const int udl, const int ndl) | |
451 | { | |
452 | unsigned long x; | |
453 | int i; | |
454 | ||
455 | preempt_disable(); | |
456 | for (i = nloops; i >= 0; i--) { | |
457 | x = smp_load_acquire(this_cpu_ptr(&test_acqrel)); | |
458 | un_delay(udl, ndl); | |
459 | smp_store_release(this_cpu_ptr(&test_acqrel), x + 1); | |
460 | } | |
461 | preempt_enable(); | |
462 | } | |
463 | ||
464 | static struct ref_scale_ops acqrel_ops = { | |
465 | .readsection = ref_acqrel_section, | |
466 | .delaysection = ref_acqrel_delay_section, | |
467 | .name = "acqrel" | |
468 | }; | |
469 | ||
25f6fa53 PM |
470 | static volatile u64 stopopts; |
471 | ||
472 | static void ref_clock_section(const int nloops) | |
473 | { | |
474 | u64 x = 0; | |
475 | int i; | |
476 | ||
477 | preempt_disable(); | |
478 | for (i = nloops; i >= 0; i--) | |
479 | x += ktime_get_real_fast_ns(); | |
480 | preempt_enable(); | |
481 | stopopts = x; | |
482 | } | |
483 | ||
484 | static void ref_clock_delay_section(const int nloops, const int udl, const int ndl) | |
485 | { | |
486 | u64 x = 0; | |
487 | int i; | |
488 | ||
489 | preempt_disable(); | |
490 | for (i = nloops; i >= 0; i--) { | |
491 | x += ktime_get_real_fast_ns(); | |
492 | un_delay(udl, ndl); | |
493 | } | |
494 | preempt_enable(); | |
495 | stopopts = x; | |
496 | } | |
497 | ||
498 | static struct ref_scale_ops clock_ops = { | |
499 | .readsection = ref_clock_section, | |
500 | .delaysection = ref_clock_delay_section, | |
501 | .name = "clock" | |
502 | }; | |
503 | ||
1fbeb3a8 | 504 | static void rcu_scale_one_reader(void) |
b4d1e34f PM |
505 | { |
506 | if (readdelay <= 0) | |
507 | cur_ops->readsection(loops); | |
508 | else | |
918b351d | 509 | cur_ops->delaysection(loops, readdelay / 1000, readdelay % 1000); |
b4d1e34f PM |
510 | } |
511 | ||
653ed64b JFG |
512 | // Reader kthread. Repeatedly does empty RCU read-side |
513 | // critical section, minimizing update-side interference. | |
514 | static int | |
1fbeb3a8 | 515 | ref_scale_reader(void *arg) |
653ed64b JFG |
516 | { |
517 | unsigned long flags; | |
518 | long me = (long)arg; | |
519 | struct reader_task *rt = &(reader_tasks[me]); | |
653ed64b JFG |
520 | u64 start; |
521 | s64 duration; | |
522 | ||
e76506f0 | 523 | VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: task started", me); |
05bc276c | 524 | WARN_ON_ONCE(set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids))); |
653ed64b JFG |
525 | set_user_nice(current, MAX_NICE); |
526 | atomic_inc(&n_init); | |
777a54c9 PM |
527 | if (holdoff) |
528 | schedule_timeout_interruptible(holdoff * HZ); | |
653ed64b | 529 | repeat: |
05bc276c | 530 | VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: waiting to start next experiment on cpu %d", me, raw_smp_processor_id()); |
653ed64b JFG |
531 | |
532 | // Wait for signal that this reader can start. | |
af2789db | 533 | wait_event(rt->wq, (atomic_read(&nreaders_exp) && smp_load_acquire(&rt->start_reader)) || |
653ed64b JFG |
534 | torture_must_stop()); |
535 | ||
536 | if (torture_must_stop()) | |
537 | goto end; | |
538 | ||
539 | // Make sure that the CPU is affinitized appropriately during testing. | |
05bc276c | 540 | WARN_ON_ONCE(raw_smp_processor_id() != me); |
653ed64b | 541 | |
af2789db | 542 | WRITE_ONCE(rt->start_reader, 0); |
86e0da2b PM |
543 | if (!atomic_dec_return(&n_started)) |
544 | while (atomic_read_acquire(&n_started)) | |
545 | cpu_relax(); | |
653ed64b | 546 | |
e76506f0 | 547 | VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d started", me, exp_idx); |
b864f89f | 548 | |
2db0bda3 PM |
549 | |
550 | // To reduce noise, do an initial cache-warming invocation, check | |
551 | // in, and then keep warming until everyone has checked in. | |
1fbeb3a8 | 552 | rcu_scale_one_reader(); |
2db0bda3 PM |
553 | if (!atomic_dec_return(&n_warmedup)) |
554 | while (atomic_read_acquire(&n_warmedup)) | |
1fbeb3a8 | 555 | rcu_scale_one_reader(); |
2db0bda3 PM |
556 | // Also keep interrupts disabled. This also has the effect |
557 | // of preventing entries into slow path for rcu_read_unlock(). | |
653ed64b JFG |
558 | local_irq_save(flags); |
559 | start = ktime_get_mono_fast_ns(); | |
560 | ||
1fbeb3a8 | 561 | rcu_scale_one_reader(); |
653ed64b JFG |
562 | |
563 | duration = ktime_get_mono_fast_ns() - start; | |
564 | local_irq_restore(flags); | |
565 | ||
566 | rt->last_duration_ns = WARN_ON_ONCE(duration < 0) ? 0 : duration; | |
2db0bda3 PM |
567 | // To reduce runtime-skew noise, do maintain-load invocations until |
568 | // everyone is done. | |
569 | if (!atomic_dec_return(&n_cooleddown)) | |
570 | while (atomic_read_acquire(&n_cooleddown)) | |
1fbeb3a8 | 571 | rcu_scale_one_reader(); |
653ed64b | 572 | |
b864f89f PM |
573 | if (atomic_dec_and_test(&nreaders_exp)) |
574 | wake_up(&main_wq); | |
653ed64b | 575 | |
e76506f0 PM |
576 | VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d ended, (readers remaining=%d)", |
577 | me, exp_idx, atomic_read(&nreaders_exp)); | |
653ed64b | 578 | |
653ed64b JFG |
579 | if (!torture_must_stop()) |
580 | goto repeat; | |
581 | end: | |
1fbeb3a8 | 582 | torture_kthread_stopping("ref_scale_reader"); |
653ed64b JFG |
583 | return 0; |
584 | } | |
585 | ||
2990750b | 586 | static void reset_readers(void) |
653ed64b JFG |
587 | { |
588 | int i; | |
589 | struct reader_task *rt; | |
590 | ||
dbf28efd | 591 | for (i = 0; i < nreaders; i++) { |
653ed64b JFG |
592 | rt = &(reader_tasks[i]); |
593 | ||
594 | rt->last_duration_ns = 0; | |
595 | } | |
596 | } | |
597 | ||
598 | // Print the results of each reader and return the sum of all their durations. | |
2990750b | 599 | static u64 process_durations(int n) |
653ed64b JFG |
600 | { |
601 | int i; | |
602 | struct reader_task *rt; | |
603 | char buf1[64]; | |
2e90de76 | 604 | char *buf; |
653ed64b JFG |
605 | u64 sum = 0; |
606 | ||
2e90de76 PM |
607 | buf = kmalloc(128 + nreaders * 32, GFP_KERNEL); |
608 | if (!buf) | |
609 | return 0; | |
653ed64b JFG |
610 | buf[0] = 0; |
611 | sprintf(buf, "Experiment #%d (Format: <THREAD-NUM>:<Total loop time in ns>)", | |
612 | exp_idx); | |
613 | ||
dbf28efd | 614 | for (i = 0; i < n && !torture_must_stop(); i++) { |
653ed64b JFG |
615 | rt = &(reader_tasks[i]); |
616 | sprintf(buf1, "%d: %llu\t", i, rt->last_duration_ns); | |
617 | ||
618 | if (i % 5 == 0) | |
619 | strcat(buf, "\n"); | |
620 | strcat(buf, buf1); | |
621 | ||
622 | sum += rt->last_duration_ns; | |
623 | } | |
624 | strcat(buf, "\n"); | |
625 | ||
1fbeb3a8 | 626 | SCALEOUT("%s\n", buf); |
653ed64b | 627 | |
2e90de76 | 628 | kfree(buf); |
653ed64b JFG |
629 | return sum; |
630 | } | |
631 | ||
632 | // The main_func is the main orchestrator, it performs a bunch of | |
633 | // experiments. For every experiment, it orders all the readers | |
634 | // involved to start and waits for them to finish the experiment. It | |
635 | // then reads their timestamps and starts the next experiment. Each | |
636 | // experiment progresses from 1 concurrent reader to N of them at which | |
637 | // point all the timestamps are printed. | |
638 | static int main_func(void *arg) | |
639 | { | |
f518f154 | 640 | bool errexit = false; |
653ed64b JFG |
641 | int exp, r; |
642 | char buf1[64]; | |
f518f154 | 643 | char *buf; |
dbf28efd | 644 | u64 *result_avg; |
653ed64b JFG |
645 | |
646 | set_cpus_allowed_ptr(current, cpumask_of(nreaders % nr_cpu_ids)); | |
647 | set_user_nice(current, MAX_NICE); | |
648 | ||
1fbeb3a8 | 649 | VERBOSE_SCALEOUT("main_func task started"); |
dbf28efd | 650 | result_avg = kzalloc(nruns * sizeof(*result_avg), GFP_KERNEL); |
f518f154 PM |
651 | buf = kzalloc(64 + nruns * 32, GFP_KERNEL); |
652 | if (!result_avg || !buf) { | |
1fbeb3a8 | 653 | VERBOSE_SCALEOUT_ERRSTRING("out of memory"); |
f518f154 PM |
654 | errexit = true; |
655 | } | |
777a54c9 PM |
656 | if (holdoff) |
657 | schedule_timeout_interruptible(holdoff * HZ); | |
653ed64b | 658 | |
96af8669 PM |
659 | // Wait for all threads to start. |
660 | atomic_inc(&n_init); | |
661 | while (atomic_read(&n_init) < nreaders + 1) | |
662 | schedule_timeout_uninterruptible(1); | |
663 | ||
653ed64b | 664 | // Start exp readers up per experiment |
dbf28efd | 665 | for (exp = 0; exp < nruns && !torture_must_stop(); exp++) { |
f518f154 | 666 | if (errexit) |
dbf28efd | 667 | break; |
653ed64b JFG |
668 | if (torture_must_stop()) |
669 | goto end; | |
670 | ||
dbf28efd PM |
671 | reset_readers(); |
672 | atomic_set(&nreaders_exp, nreaders); | |
86e0da2b | 673 | atomic_set(&n_started, nreaders); |
2db0bda3 PM |
674 | atomic_set(&n_warmedup, nreaders); |
675 | atomic_set(&n_cooleddown, nreaders); | |
653ed64b JFG |
676 | |
677 | exp_idx = exp; | |
678 | ||
dbf28efd | 679 | for (r = 0; r < nreaders; r++) { |
af2789db | 680 | smp_store_release(&reader_tasks[r].start_reader, 1); |
653ed64b JFG |
681 | wake_up(&reader_tasks[r].wq); |
682 | } | |
683 | ||
1fbeb3a8 | 684 | VERBOSE_SCALEOUT("main_func: experiment started, waiting for %d readers", |
dbf28efd | 685 | nreaders); |
653ed64b JFG |
686 | |
687 | wait_event(main_wq, | |
688 | !atomic_read(&nreaders_exp) || torture_must_stop()); | |
689 | ||
1fbeb3a8 | 690 | VERBOSE_SCALEOUT("main_func: experiment ended"); |
653ed64b JFG |
691 | |
692 | if (torture_must_stop()) | |
693 | goto end; | |
694 | ||
7c944d7c | 695 | result_avg[exp] = div_u64(1000 * process_durations(nreaders), nreaders * loops); |
653ed64b JFG |
696 | } |
697 | ||
698 | // Print the average of all experiments | |
1fbeb3a8 | 699 | SCALEOUT("END OF TEST. Calculating average duration per loop (nanoseconds)...\n"); |
653ed64b | 700 | |
58db5785 CIK |
701 | if (!errexit) { |
702 | buf[0] = 0; | |
703 | strcat(buf, "\n"); | |
704 | strcat(buf, "Runs\tTime(ns)\n"); | |
705 | } | |
653ed64b | 706 | |
dbf28efd | 707 | for (exp = 0; exp < nruns; exp++) { |
7c944d7c AB |
708 | u64 avg; |
709 | u32 rem; | |
710 | ||
f518f154 | 711 | if (errexit) |
dbf28efd | 712 | break; |
7c944d7c AB |
713 | avg = div_u64_rem(result_avg[exp], 1000, &rem); |
714 | sprintf(buf1, "%d\t%llu.%03u\n", exp + 1, avg, rem); | |
653ed64b JFG |
715 | strcat(buf, buf1); |
716 | } | |
717 | ||
f518f154 | 718 | if (!errexit) |
1fbeb3a8 | 719 | SCALEOUT("%s", buf); |
653ed64b JFG |
720 | |
721 | // This will shutdown everything including us. | |
722 | if (shutdown) { | |
723 | shutdown_start = 1; | |
724 | wake_up(&shutdown_wq); | |
725 | } | |
726 | ||
727 | // Wait for torture to stop us | |
728 | while (!torture_must_stop()) | |
729 | schedule_timeout_uninterruptible(1); | |
730 | ||
731 | end: | |
732 | torture_kthread_stopping("main_func"); | |
f518f154 PM |
733 | kfree(result_avg); |
734 | kfree(buf); | |
653ed64b JFG |
735 | return 0; |
736 | } | |
737 | ||
738 | static void | |
1fbeb3a8 | 739 | ref_scale_print_module_parms(struct ref_scale_ops *cur_ops, const char *tag) |
653ed64b | 740 | { |
1fbeb3a8 PM |
741 | pr_alert("%s" SCALE_FLAG |
742 | "--- %s: verbose=%d shutdown=%d holdoff=%d loops=%ld nreaders=%d nruns=%d readdelay=%d\n", scale_type, tag, | |
b4d1e34f | 743 | verbose, shutdown, holdoff, loops, nreaders, nruns, readdelay); |
653ed64b JFG |
744 | } |
745 | ||
746 | static void | |
1fbeb3a8 | 747 | ref_scale_cleanup(void) |
653ed64b JFG |
748 | { |
749 | int i; | |
750 | ||
751 | if (torture_cleanup_begin()) | |
752 | return; | |
753 | ||
754 | if (!cur_ops) { | |
755 | torture_cleanup_end(); | |
756 | return; | |
757 | } | |
758 | ||
759 | if (reader_tasks) { | |
760 | for (i = 0; i < nreaders; i++) | |
1fbeb3a8 | 761 | torture_stop_kthread("ref_scale_reader", |
653ed64b JFG |
762 | reader_tasks[i].task); |
763 | } | |
764 | kfree(reader_tasks); | |
765 | ||
766 | torture_stop_kthread("main_task", main_task); | |
767 | kfree(main_task); | |
768 | ||
1fbeb3a8 | 769 | // Do scale-type-specific cleanup operations. |
653ed64b JFG |
770 | if (cur_ops->cleanup != NULL) |
771 | cur_ops->cleanup(); | |
772 | ||
773 | torture_cleanup_end(); | |
774 | } | |
775 | ||
776 | // Shutdown kthread. Just waits to be awakened, then shuts down system. | |
777 | static int | |
1fbeb3a8 | 778 | ref_scale_shutdown(void *arg) |
653ed64b JFG |
779 | { |
780 | wait_event(shutdown_wq, shutdown_start); | |
781 | ||
782 | smp_mb(); // Wake before output. | |
1fbeb3a8 | 783 | ref_scale_cleanup(); |
653ed64b JFG |
784 | kernel_power_off(); |
785 | ||
786 | return -EINVAL; | |
787 | } | |
788 | ||
789 | static int __init | |
1fbeb3a8 | 790 | ref_scale_init(void) |
653ed64b JFG |
791 | { |
792 | long i; | |
793 | int firsterr = 0; | |
1fbeb3a8 | 794 | static struct ref_scale_ops *scale_ops[] = { |
e9b800db | 795 | &rcu_ops, &srcu_ops, &rcu_trace_ops, &rcu_tasks_ops, &refcnt_ops, &rwlock_ops, |
25f6fa53 | 796 | &rwsem_ops, &lock_ops, &lock_irq_ops, &acqrel_ops, &clock_ops, |
653ed64b JFG |
797 | }; |
798 | ||
1fbeb3a8 | 799 | if (!torture_init_begin(scale_type, verbose)) |
653ed64b JFG |
800 | return -EBUSY; |
801 | ||
1fbeb3a8 PM |
802 | for (i = 0; i < ARRAY_SIZE(scale_ops); i++) { |
803 | cur_ops = scale_ops[i]; | |
804 | if (strcmp(scale_type, cur_ops->name) == 0) | |
653ed64b JFG |
805 | break; |
806 | } | |
1fbeb3a8 PM |
807 | if (i == ARRAY_SIZE(scale_ops)) { |
808 | pr_alert("rcu-scale: invalid scale type: \"%s\"\n", scale_type); | |
809 | pr_alert("rcu-scale types:"); | |
810 | for (i = 0; i < ARRAY_SIZE(scale_ops); i++) | |
811 | pr_cont(" %s", scale_ops[i]->name); | |
653ed64b | 812 | pr_cont("\n"); |
653ed64b JFG |
813 | firsterr = -EINVAL; |
814 | cur_ops = NULL; | |
815 | goto unwind; | |
816 | } | |
817 | if (cur_ops->init) | |
818 | cur_ops->init(); | |
819 | ||
1fbeb3a8 | 820 | ref_scale_print_module_parms(cur_ops, "Start of test"); |
653ed64b JFG |
821 | |
822 | // Shutdown task | |
823 | if (shutdown) { | |
824 | init_waitqueue_head(&shutdown_wq); | |
1fbeb3a8 | 825 | firsterr = torture_create_kthread(ref_scale_shutdown, NULL, |
653ed64b | 826 | shutdown_task); |
ed60ad73 | 827 | if (torture_init_error(firsterr)) |
653ed64b JFG |
828 | goto unwind; |
829 | schedule_timeout_uninterruptible(1); | |
830 | } | |
831 | ||
8fc28783 PM |
832 | // Reader tasks (default to ~75% of online CPUs). |
833 | if (nreaders < 0) | |
834 | nreaders = (num_online_cpus() >> 1) + (num_online_cpus() >> 2); | |
0c6d18d8 PM |
835 | if (WARN_ONCE(loops <= 0, "%s: loops = %ld, adjusted to 1\n", __func__, loops)) |
836 | loops = 1; | |
837 | if (WARN_ONCE(nreaders <= 0, "%s: nreaders = %d, adjusted to 1\n", __func__, nreaders)) | |
838 | nreaders = 1; | |
839 | if (WARN_ONCE(nruns <= 0, "%s: nruns = %d, adjusted to 1\n", __func__, nruns)) | |
840 | nruns = 1; | |
653ed64b JFG |
841 | reader_tasks = kcalloc(nreaders, sizeof(reader_tasks[0]), |
842 | GFP_KERNEL); | |
843 | if (!reader_tasks) { | |
1fbeb3a8 | 844 | VERBOSE_SCALEOUT_ERRSTRING("out of memory"); |
653ed64b JFG |
845 | firsterr = -ENOMEM; |
846 | goto unwind; | |
847 | } | |
848 | ||
1fbeb3a8 | 849 | VERBOSE_SCALEOUT("Starting %d reader threads\n", nreaders); |
653ed64b JFG |
850 | |
851 | for (i = 0; i < nreaders; i++) { | |
1fbeb3a8 | 852 | firsterr = torture_create_kthread(ref_scale_reader, (void *)i, |
653ed64b | 853 | reader_tasks[i].task); |
ed60ad73 | 854 | if (torture_init_error(firsterr)) |
653ed64b JFG |
855 | goto unwind; |
856 | ||
857 | init_waitqueue_head(&(reader_tasks[i].wq)); | |
858 | } | |
859 | ||
860 | // Main Task | |
861 | init_waitqueue_head(&main_wq); | |
862 | firsterr = torture_create_kthread(main_func, NULL, main_task); | |
ed60ad73 | 863 | if (torture_init_error(firsterr)) |
653ed64b | 864 | goto unwind; |
653ed64b JFG |
865 | |
866 | torture_init_end(); | |
867 | return 0; | |
868 | ||
869 | unwind: | |
870 | torture_init_end(); | |
1fbeb3a8 | 871 | ref_scale_cleanup(); |
bc80d353 PM |
872 | if (shutdown) { |
873 | WARN_ON(!IS_MODULE(CONFIG_RCU_REF_SCALE_TEST)); | |
874 | kernel_power_off(); | |
875 | } | |
653ed64b JFG |
876 | return firsterr; |
877 | } | |
878 | ||
1fbeb3a8 PM |
879 | module_init(ref_scale_init); |
880 | module_exit(ref_scale_cleanup); |