Commit | Line | Data |
---|---|---|
653ed64b JFG |
1 | // SPDX-License-Identifier: GPL-2.0+ |
2 | // | |
8e4ec3d0 | 3 | // Scalability test comparing RCU vs other mechanisms |
653ed64b JFG |
4 | // for acquiring references on objects. |
5 | // | |
6 | // Copyright (C) Google, 2020. | |
7 | // | |
8 | // Author: Joel Fernandes <joel@joelfernandes.org> | |
9 | ||
10 | #define pr_fmt(fmt) fmt | |
11 | ||
12 | #include <linux/atomic.h> | |
13 | #include <linux/bitops.h> | |
14 | #include <linux/completion.h> | |
15 | #include <linux/cpu.h> | |
16 | #include <linux/delay.h> | |
17 | #include <linux/err.h> | |
18 | #include <linux/init.h> | |
19 | #include <linux/interrupt.h> | |
20 | #include <linux/kthread.h> | |
21 | #include <linux/kernel.h> | |
22 | #include <linux/mm.h> | |
23 | #include <linux/module.h> | |
24 | #include <linux/moduleparam.h> | |
25 | #include <linux/notifier.h> | |
26 | #include <linux/percpu.h> | |
27 | #include <linux/rcupdate.h> | |
72bb749e | 28 | #include <linux/rcupdate_trace.h> |
653ed64b JFG |
29 | #include <linux/reboot.h> |
30 | #include <linux/sched.h> | |
31 | #include <linux/spinlock.h> | |
32 | #include <linux/smp.h> | |
33 | #include <linux/stat.h> | |
34 | #include <linux/srcu.h> | |
35 | #include <linux/slab.h> | |
36 | #include <linux/torture.h> | |
37 | #include <linux/types.h> | |
38 | ||
39 | #include "rcu.h" | |
40 | ||
1fbeb3a8 | 41 | #define SCALE_FLAG "-ref-scale: " |
653ed64b | 42 | |
1fbeb3a8 PM |
43 | #define SCALEOUT(s, x...) \ |
44 | pr_alert("%s" SCALE_FLAG s, scale_type, ## x) | |
653ed64b | 45 | |
1fbeb3a8 PM |
46 | #define VERBOSE_SCALEOUT(s, x...) \ |
47 | do { if (verbose) pr_alert("%s" SCALE_FLAG s, scale_type, ## x); } while (0) | |
653ed64b | 48 | |
e76506f0 PM |
49 | static atomic_t verbose_batch_ctr; |
50 | ||
51 | #define VERBOSE_SCALEOUT_BATCH(s, x...) \ | |
52 | do { \ | |
53 | if (verbose && \ | |
54 | (verbose_batched <= 0 || \ | |
414c116e PM |
55 | !(atomic_inc_return(&verbose_batch_ctr) % verbose_batched))) { \ |
56 | schedule_timeout_uninterruptible(1); \ | |
e76506f0 | 57 | pr_alert("%s" SCALE_FLAG s, scale_type, ## x); \ |
414c116e | 58 | } \ |
e76506f0 PM |
59 | } while (0) |
60 | ||
1fbeb3a8 PM |
61 | #define VERBOSE_SCALEOUT_ERRSTRING(s, x...) \ |
62 | do { if (verbose) pr_alert("%s" SCALE_FLAG "!!! " s, scale_type, ## x); } while (0) | |
653ed64b JFG |
63 | |
64 | MODULE_LICENSE("GPL"); | |
65 | MODULE_AUTHOR("Joel Fernandes (Google) <joel@joelfernandes.org>"); | |
66 | ||
1fbeb3a8 PM |
67 | static char *scale_type = "rcu"; |
68 | module_param(scale_type, charp, 0444); | |
69 | MODULE_PARM_DESC(scale_type, "Type of test (rcu, srcu, refcnt, rwsem, rwlock."); | |
653ed64b JFG |
70 | |
71 | torture_param(int, verbose, 0, "Enable verbose debugging printk()s"); | |
e76506f0 | 72 | torture_param(int, verbose_batched, 0, "Batch verbose debugging printk()s"); |
653ed64b | 73 | |
777a54c9 | 74 | // Wait until there are multiple CPUs before starting test. |
8e4ec3d0 | 75 | torture_param(int, holdoff, IS_BUILTIN(CONFIG_RCU_REF_SCALE_TEST) ? 10 : 0, |
777a54c9 PM |
76 | "Holdoff time before test start (s)"); |
77 | // Number of loops per experiment, all readers execute operations concurrently. | |
4dd72a33 | 78 | torture_param(long, loops, 10000, "Number of loops per experiment."); |
8fc28783 PM |
79 | // Number of readers, with -1 defaulting to about 75% of the CPUs. |
80 | torture_param(int, nreaders, -1, "Number of readers, -1 for 75% of CPUs."); | |
81 | // Number of runs. | |
82 | torture_param(int, nruns, 30, "Number of experiments to run."); | |
918b351d PM |
83 | // Reader delay in nanoseconds, 0 for no delay. |
84 | torture_param(int, readdelay, 0, "Read-side delay in nanoseconds."); | |
653ed64b JFG |
85 | |
86 | #ifdef MODULE | |
1fbeb3a8 | 87 | # define REFSCALE_SHUTDOWN 0 |
653ed64b | 88 | #else |
1fbeb3a8 | 89 | # define REFSCALE_SHUTDOWN 1 |
653ed64b JFG |
90 | #endif |
91 | ||
1fbeb3a8 PM |
92 | torture_param(bool, shutdown, REFSCALE_SHUTDOWN, |
93 | "Shutdown at end of scalability tests."); | |
653ed64b JFG |
94 | |
95 | struct reader_task { | |
96 | struct task_struct *task; | |
af2789db | 97 | int start_reader; |
653ed64b JFG |
98 | wait_queue_head_t wq; |
99 | u64 last_duration_ns; | |
653ed64b JFG |
100 | }; |
101 | ||
102 | static struct task_struct *shutdown_task; | |
103 | static wait_queue_head_t shutdown_wq; | |
104 | ||
105 | static struct task_struct *main_task; | |
106 | static wait_queue_head_t main_wq; | |
107 | static int shutdown_start; | |
108 | ||
109 | static struct reader_task *reader_tasks; | |
653ed64b JFG |
110 | |
111 | // Number of readers that are part of the current experiment. | |
112 | static atomic_t nreaders_exp; | |
113 | ||
114 | // Use to wait for all threads to start. | |
115 | static atomic_t n_init; | |
86e0da2b | 116 | static atomic_t n_started; |
2db0bda3 PM |
117 | static atomic_t n_warmedup; |
118 | static atomic_t n_cooleddown; | |
653ed64b JFG |
119 | |
120 | // Track which experiment is currently running. | |
121 | static int exp_idx; | |
122 | ||
123 | // Operations vector for selecting different types of tests. | |
1fbeb3a8 | 124 | struct ref_scale_ops { |
653ed64b JFG |
125 | void (*init)(void); |
126 | void (*cleanup)(void); | |
75dd8efe | 127 | void (*readsection)(const int nloops); |
918b351d | 128 | void (*delaysection)(const int nloops, const int udl, const int ndl); |
653ed64b JFG |
129 | const char *name; |
130 | }; | |
131 | ||
1fbeb3a8 | 132 | static struct ref_scale_ops *cur_ops; |
653ed64b | 133 | |
918b351d PM |
134 | static void un_delay(const int udl, const int ndl) |
135 | { | |
136 | if (udl) | |
137 | udelay(udl); | |
138 | if (ndl) | |
139 | ndelay(ndl); | |
140 | } | |
141 | ||
75dd8efe | 142 | static void ref_rcu_read_section(const int nloops) |
653ed64b | 143 | { |
75dd8efe | 144 | int i; |
653ed64b | 145 | |
75dd8efe PM |
146 | for (i = nloops; i >= 0; i--) { |
147 | rcu_read_lock(); | |
148 | rcu_read_unlock(); | |
149 | } | |
653ed64b JFG |
150 | } |
151 | ||
918b351d | 152 | static void ref_rcu_delay_section(const int nloops, const int udl, const int ndl) |
b4d1e34f PM |
153 | { |
154 | int i; | |
155 | ||
156 | for (i = nloops; i >= 0; i--) { | |
157 | rcu_read_lock(); | |
918b351d | 158 | un_delay(udl, ndl); |
b4d1e34f PM |
159 | rcu_read_unlock(); |
160 | } | |
161 | } | |
162 | ||
1fbeb3a8 | 163 | static void rcu_sync_scale_init(void) |
653ed64b JFG |
164 | { |
165 | } | |
166 | ||
1fbeb3a8 PM |
167 | static struct ref_scale_ops rcu_ops = { |
168 | .init = rcu_sync_scale_init, | |
75dd8efe | 169 | .readsection = ref_rcu_read_section, |
b4d1e34f | 170 | .delaysection = ref_rcu_delay_section, |
653ed64b JFG |
171 | .name = "rcu" |
172 | }; | |
173 | ||
1fbeb3a8 PM |
174 | // Definitions for SRCU ref scale testing. |
175 | DEFINE_STATIC_SRCU(srcu_refctl_scale); | |
176 | static struct srcu_struct *srcu_ctlp = &srcu_refctl_scale; | |
653ed64b | 177 | |
1fbeb3a8 | 178 | static void srcu_ref_scale_read_section(const int nloops) |
653ed64b | 179 | { |
75dd8efe PM |
180 | int i; |
181 | int idx; | |
653ed64b | 182 | |
75dd8efe PM |
183 | for (i = nloops; i >= 0; i--) { |
184 | idx = srcu_read_lock(srcu_ctlp); | |
185 | srcu_read_unlock(srcu_ctlp, idx); | |
186 | } | |
653ed64b JFG |
187 | } |
188 | ||
1fbeb3a8 | 189 | static void srcu_ref_scale_delay_section(const int nloops, const int udl, const int ndl) |
b4d1e34f PM |
190 | { |
191 | int i; | |
192 | int idx; | |
193 | ||
194 | for (i = nloops; i >= 0; i--) { | |
195 | idx = srcu_read_lock(srcu_ctlp); | |
918b351d | 196 | un_delay(udl, ndl); |
b4d1e34f PM |
197 | srcu_read_unlock(srcu_ctlp, idx); |
198 | } | |
199 | } | |
200 | ||
1fbeb3a8 PM |
201 | static struct ref_scale_ops srcu_ops = { |
202 | .init = rcu_sync_scale_init, | |
203 | .readsection = srcu_ref_scale_read_section, | |
204 | .delaysection = srcu_ref_scale_delay_section, | |
653ed64b JFG |
205 | .name = "srcu" |
206 | }; | |
207 | ||
1fbeb3a8 | 208 | // Definitions for RCU Tasks ref scale testing: Empty read markers. |
e13ef442 | 209 | // These definitions also work for RCU Rude readers. |
1fbeb3a8 | 210 | static void rcu_tasks_ref_scale_read_section(const int nloops) |
e13ef442 PM |
211 | { |
212 | int i; | |
213 | ||
214 | for (i = nloops; i >= 0; i--) | |
215 | continue; | |
216 | } | |
217 | ||
1fbeb3a8 | 218 | static void rcu_tasks_ref_scale_delay_section(const int nloops, const int udl, const int ndl) |
e13ef442 PM |
219 | { |
220 | int i; | |
221 | ||
222 | for (i = nloops; i >= 0; i--) | |
223 | un_delay(udl, ndl); | |
224 | } | |
225 | ||
1fbeb3a8 PM |
226 | static struct ref_scale_ops rcu_tasks_ops = { |
227 | .init = rcu_sync_scale_init, | |
228 | .readsection = rcu_tasks_ref_scale_read_section, | |
229 | .delaysection = rcu_tasks_ref_scale_delay_section, | |
e13ef442 PM |
230 | .name = "rcu-tasks" |
231 | }; | |
232 | ||
1fbeb3a8 PM |
233 | // Definitions for RCU Tasks Trace ref scale testing. |
234 | static void rcu_trace_ref_scale_read_section(const int nloops) | |
72bb749e PM |
235 | { |
236 | int i; | |
237 | ||
238 | for (i = nloops; i >= 0; i--) { | |
239 | rcu_read_lock_trace(); | |
240 | rcu_read_unlock_trace(); | |
241 | } | |
242 | } | |
243 | ||
1fbeb3a8 | 244 | static void rcu_trace_ref_scale_delay_section(const int nloops, const int udl, const int ndl) |
72bb749e PM |
245 | { |
246 | int i; | |
247 | ||
248 | for (i = nloops; i >= 0; i--) { | |
249 | rcu_read_lock_trace(); | |
250 | un_delay(udl, ndl); | |
251 | rcu_read_unlock_trace(); | |
252 | } | |
253 | } | |
254 | ||
1fbeb3a8 PM |
255 | static struct ref_scale_ops rcu_trace_ops = { |
256 | .init = rcu_sync_scale_init, | |
257 | .readsection = rcu_trace_ref_scale_read_section, | |
258 | .delaysection = rcu_trace_ref_scale_delay_section, | |
72bb749e PM |
259 | .name = "rcu-trace" |
260 | }; | |
261 | ||
653ed64b JFG |
262 | // Definitions for reference count |
263 | static atomic_t refcnt; | |
264 | ||
b4d1e34f | 265 | static void ref_refcnt_section(const int nloops) |
653ed64b | 266 | { |
75dd8efe | 267 | int i; |
653ed64b | 268 | |
75dd8efe PM |
269 | for (i = nloops; i >= 0; i--) { |
270 | atomic_inc(&refcnt); | |
271 | atomic_dec(&refcnt); | |
272 | } | |
653ed64b JFG |
273 | } |
274 | ||
918b351d | 275 | static void ref_refcnt_delay_section(const int nloops, const int udl, const int ndl) |
b4d1e34f PM |
276 | { |
277 | int i; | |
278 | ||
279 | for (i = nloops; i >= 0; i--) { | |
280 | atomic_inc(&refcnt); | |
918b351d | 281 | un_delay(udl, ndl); |
b4d1e34f PM |
282 | atomic_dec(&refcnt); |
283 | } | |
284 | } | |
285 | ||
1fbeb3a8 PM |
286 | static struct ref_scale_ops refcnt_ops = { |
287 | .init = rcu_sync_scale_init, | |
b4d1e34f PM |
288 | .readsection = ref_refcnt_section, |
289 | .delaysection = ref_refcnt_delay_section, | |
653ed64b JFG |
290 | .name = "refcnt" |
291 | }; | |
292 | ||
293 | // Definitions for rwlock | |
294 | static rwlock_t test_rwlock; | |
295 | ||
b4d1e34f | 296 | static void ref_rwlock_init(void) |
653ed64b JFG |
297 | { |
298 | rwlock_init(&test_rwlock); | |
299 | } | |
300 | ||
b4d1e34f PM |
301 | static void ref_rwlock_section(const int nloops) |
302 | { | |
303 | int i; | |
304 | ||
305 | for (i = nloops; i >= 0; i--) { | |
306 | read_lock(&test_rwlock); | |
307 | read_unlock(&test_rwlock); | |
308 | } | |
309 | } | |
310 | ||
918b351d | 311 | static void ref_rwlock_delay_section(const int nloops, const int udl, const int ndl) |
653ed64b | 312 | { |
75dd8efe | 313 | int i; |
653ed64b | 314 | |
75dd8efe PM |
315 | for (i = nloops; i >= 0; i--) { |
316 | read_lock(&test_rwlock); | |
918b351d | 317 | un_delay(udl, ndl); |
75dd8efe PM |
318 | read_unlock(&test_rwlock); |
319 | } | |
653ed64b JFG |
320 | } |
321 | ||
1fbeb3a8 | 322 | static struct ref_scale_ops rwlock_ops = { |
b4d1e34f PM |
323 | .init = ref_rwlock_init, |
324 | .readsection = ref_rwlock_section, | |
325 | .delaysection = ref_rwlock_delay_section, | |
653ed64b JFG |
326 | .name = "rwlock" |
327 | }; | |
328 | ||
329 | // Definitions for rwsem | |
330 | static struct rw_semaphore test_rwsem; | |
331 | ||
b4d1e34f | 332 | static void ref_rwsem_init(void) |
653ed64b JFG |
333 | { |
334 | init_rwsem(&test_rwsem); | |
335 | } | |
336 | ||
b4d1e34f | 337 | static void ref_rwsem_section(const int nloops) |
653ed64b | 338 | { |
75dd8efe | 339 | int i; |
653ed64b | 340 | |
75dd8efe PM |
341 | for (i = nloops; i >= 0; i--) { |
342 | down_read(&test_rwsem); | |
343 | up_read(&test_rwsem); | |
344 | } | |
653ed64b JFG |
345 | } |
346 | ||
918b351d | 347 | static void ref_rwsem_delay_section(const int nloops, const int udl, const int ndl) |
b4d1e34f PM |
348 | { |
349 | int i; | |
350 | ||
351 | for (i = nloops; i >= 0; i--) { | |
352 | down_read(&test_rwsem); | |
918b351d | 353 | un_delay(udl, ndl); |
b4d1e34f PM |
354 | up_read(&test_rwsem); |
355 | } | |
356 | } | |
357 | ||
1fbeb3a8 | 358 | static struct ref_scale_ops rwsem_ops = { |
b4d1e34f PM |
359 | .init = ref_rwsem_init, |
360 | .readsection = ref_rwsem_section, | |
361 | .delaysection = ref_rwsem_delay_section, | |
653ed64b JFG |
362 | .name = "rwsem" |
363 | }; | |
364 | ||
e9b800db PM |
365 | // Definitions for global spinlock |
366 | static DEFINE_SPINLOCK(test_lock); | |
367 | ||
368 | static void ref_lock_section(const int nloops) | |
369 | { | |
370 | int i; | |
371 | ||
372 | preempt_disable(); | |
373 | for (i = nloops; i >= 0; i--) { | |
374 | spin_lock(&test_lock); | |
375 | spin_unlock(&test_lock); | |
376 | } | |
377 | preempt_enable(); | |
378 | } | |
379 | ||
380 | static void ref_lock_delay_section(const int nloops, const int udl, const int ndl) | |
381 | { | |
382 | int i; | |
383 | ||
384 | preempt_disable(); | |
385 | for (i = nloops; i >= 0; i--) { | |
386 | spin_lock(&test_lock); | |
387 | un_delay(udl, ndl); | |
388 | spin_unlock(&test_lock); | |
389 | } | |
390 | preempt_enable(); | |
391 | } | |
392 | ||
393 | static struct ref_scale_ops lock_ops = { | |
394 | .readsection = ref_lock_section, | |
395 | .delaysection = ref_lock_delay_section, | |
396 | .name = "lock" | |
397 | }; | |
398 | ||
399 | // Definitions for global irq-save spinlock | |
400 | ||
401 | static void ref_lock_irq_section(const int nloops) | |
402 | { | |
403 | unsigned long flags; | |
404 | int i; | |
405 | ||
406 | preempt_disable(); | |
407 | for (i = nloops; i >= 0; i--) { | |
408 | spin_lock_irqsave(&test_lock, flags); | |
409 | spin_unlock_irqrestore(&test_lock, flags); | |
410 | } | |
411 | preempt_enable(); | |
412 | } | |
413 | ||
414 | static void ref_lock_irq_delay_section(const int nloops, const int udl, const int ndl) | |
415 | { | |
416 | unsigned long flags; | |
417 | int i; | |
418 | ||
419 | preempt_disable(); | |
420 | for (i = nloops; i >= 0; i--) { | |
421 | spin_lock_irqsave(&test_lock, flags); | |
422 | un_delay(udl, ndl); | |
423 | spin_unlock_irqrestore(&test_lock, flags); | |
424 | } | |
425 | preempt_enable(); | |
426 | } | |
427 | ||
428 | static struct ref_scale_ops lock_irq_ops = { | |
429 | .readsection = ref_lock_irq_section, | |
430 | .delaysection = ref_lock_irq_delay_section, | |
431 | .name = "lock-irq" | |
432 | }; | |
433 | ||
434 | // Definitions acquire-release. | |
435 | static DEFINE_PER_CPU(unsigned long, test_acqrel); | |
436 | ||
437 | static void ref_acqrel_section(const int nloops) | |
438 | { | |
439 | unsigned long x; | |
440 | int i; | |
441 | ||
442 | preempt_disable(); | |
443 | for (i = nloops; i >= 0; i--) { | |
444 | x = smp_load_acquire(this_cpu_ptr(&test_acqrel)); | |
445 | smp_store_release(this_cpu_ptr(&test_acqrel), x + 1); | |
446 | } | |
447 | preempt_enable(); | |
448 | } | |
449 | ||
450 | static void ref_acqrel_delay_section(const int nloops, const int udl, const int ndl) | |
451 | { | |
452 | unsigned long x; | |
453 | int i; | |
454 | ||
455 | preempt_disable(); | |
456 | for (i = nloops; i >= 0; i--) { | |
457 | x = smp_load_acquire(this_cpu_ptr(&test_acqrel)); | |
458 | un_delay(udl, ndl); | |
459 | smp_store_release(this_cpu_ptr(&test_acqrel), x + 1); | |
460 | } | |
461 | preempt_enable(); | |
462 | } | |
463 | ||
464 | static struct ref_scale_ops acqrel_ops = { | |
465 | .readsection = ref_acqrel_section, | |
466 | .delaysection = ref_acqrel_delay_section, | |
467 | .name = "acqrel" | |
468 | }; | |
469 | ||
1fbeb3a8 | 470 | static void rcu_scale_one_reader(void) |
b4d1e34f PM |
471 | { |
472 | if (readdelay <= 0) | |
473 | cur_ops->readsection(loops); | |
474 | else | |
918b351d | 475 | cur_ops->delaysection(loops, readdelay / 1000, readdelay % 1000); |
b4d1e34f PM |
476 | } |
477 | ||
653ed64b JFG |
478 | // Reader kthread. Repeatedly does empty RCU read-side |
479 | // critical section, minimizing update-side interference. | |
480 | static int | |
1fbeb3a8 | 481 | ref_scale_reader(void *arg) |
653ed64b JFG |
482 | { |
483 | unsigned long flags; | |
484 | long me = (long)arg; | |
485 | struct reader_task *rt = &(reader_tasks[me]); | |
653ed64b JFG |
486 | u64 start; |
487 | s64 duration; | |
488 | ||
e76506f0 | 489 | VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: task started", me); |
653ed64b JFG |
490 | set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids)); |
491 | set_user_nice(current, MAX_NICE); | |
492 | atomic_inc(&n_init); | |
777a54c9 PM |
493 | if (holdoff) |
494 | schedule_timeout_interruptible(holdoff * HZ); | |
653ed64b | 495 | repeat: |
e76506f0 | 496 | VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: waiting to start next experiment on cpu %d", me, smp_processor_id()); |
653ed64b JFG |
497 | |
498 | // Wait for signal that this reader can start. | |
af2789db | 499 | wait_event(rt->wq, (atomic_read(&nreaders_exp) && smp_load_acquire(&rt->start_reader)) || |
653ed64b JFG |
500 | torture_must_stop()); |
501 | ||
502 | if (torture_must_stop()) | |
503 | goto end; | |
504 | ||
505 | // Make sure that the CPU is affinitized appropriately during testing. | |
506 | WARN_ON_ONCE(smp_processor_id() != me); | |
507 | ||
af2789db | 508 | WRITE_ONCE(rt->start_reader, 0); |
86e0da2b PM |
509 | if (!atomic_dec_return(&n_started)) |
510 | while (atomic_read_acquire(&n_started)) | |
511 | cpu_relax(); | |
653ed64b | 512 | |
e76506f0 | 513 | VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d started", me, exp_idx); |
b864f89f | 514 | |
2db0bda3 PM |
515 | |
516 | // To reduce noise, do an initial cache-warming invocation, check | |
517 | // in, and then keep warming until everyone has checked in. | |
1fbeb3a8 | 518 | rcu_scale_one_reader(); |
2db0bda3 PM |
519 | if (!atomic_dec_return(&n_warmedup)) |
520 | while (atomic_read_acquire(&n_warmedup)) | |
1fbeb3a8 | 521 | rcu_scale_one_reader(); |
2db0bda3 PM |
522 | // Also keep interrupts disabled. This also has the effect |
523 | // of preventing entries into slow path for rcu_read_unlock(). | |
653ed64b JFG |
524 | local_irq_save(flags); |
525 | start = ktime_get_mono_fast_ns(); | |
526 | ||
1fbeb3a8 | 527 | rcu_scale_one_reader(); |
653ed64b JFG |
528 | |
529 | duration = ktime_get_mono_fast_ns() - start; | |
530 | local_irq_restore(flags); | |
531 | ||
532 | rt->last_duration_ns = WARN_ON_ONCE(duration < 0) ? 0 : duration; | |
2db0bda3 PM |
533 | // To reduce runtime-skew noise, do maintain-load invocations until |
534 | // everyone is done. | |
535 | if (!atomic_dec_return(&n_cooleddown)) | |
536 | while (atomic_read_acquire(&n_cooleddown)) | |
1fbeb3a8 | 537 | rcu_scale_one_reader(); |
653ed64b | 538 | |
b864f89f PM |
539 | if (atomic_dec_and_test(&nreaders_exp)) |
540 | wake_up(&main_wq); | |
653ed64b | 541 | |
e76506f0 PM |
542 | VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d ended, (readers remaining=%d)", |
543 | me, exp_idx, atomic_read(&nreaders_exp)); | |
653ed64b | 544 | |
653ed64b JFG |
545 | if (!torture_must_stop()) |
546 | goto repeat; | |
547 | end: | |
1fbeb3a8 | 548 | torture_kthread_stopping("ref_scale_reader"); |
653ed64b JFG |
549 | return 0; |
550 | } | |
551 | ||
2990750b | 552 | static void reset_readers(void) |
653ed64b JFG |
553 | { |
554 | int i; | |
555 | struct reader_task *rt; | |
556 | ||
dbf28efd | 557 | for (i = 0; i < nreaders; i++) { |
653ed64b JFG |
558 | rt = &(reader_tasks[i]); |
559 | ||
560 | rt->last_duration_ns = 0; | |
561 | } | |
562 | } | |
563 | ||
564 | // Print the results of each reader and return the sum of all their durations. | |
2990750b | 565 | static u64 process_durations(int n) |
653ed64b JFG |
566 | { |
567 | int i; | |
568 | struct reader_task *rt; | |
569 | char buf1[64]; | |
2e90de76 | 570 | char *buf; |
653ed64b JFG |
571 | u64 sum = 0; |
572 | ||
2e90de76 PM |
573 | buf = kmalloc(128 + nreaders * 32, GFP_KERNEL); |
574 | if (!buf) | |
575 | return 0; | |
653ed64b JFG |
576 | buf[0] = 0; |
577 | sprintf(buf, "Experiment #%d (Format: <THREAD-NUM>:<Total loop time in ns>)", | |
578 | exp_idx); | |
579 | ||
dbf28efd | 580 | for (i = 0; i < n && !torture_must_stop(); i++) { |
653ed64b JFG |
581 | rt = &(reader_tasks[i]); |
582 | sprintf(buf1, "%d: %llu\t", i, rt->last_duration_ns); | |
583 | ||
584 | if (i % 5 == 0) | |
585 | strcat(buf, "\n"); | |
586 | strcat(buf, buf1); | |
587 | ||
588 | sum += rt->last_duration_ns; | |
589 | } | |
590 | strcat(buf, "\n"); | |
591 | ||
1fbeb3a8 | 592 | SCALEOUT("%s\n", buf); |
653ed64b | 593 | |
2e90de76 | 594 | kfree(buf); |
653ed64b JFG |
595 | return sum; |
596 | } | |
597 | ||
598 | // The main_func is the main orchestrator, it performs a bunch of | |
599 | // experiments. For every experiment, it orders all the readers | |
600 | // involved to start and waits for them to finish the experiment. It | |
601 | // then reads their timestamps and starts the next experiment. Each | |
602 | // experiment progresses from 1 concurrent reader to N of them at which | |
603 | // point all the timestamps are printed. | |
604 | static int main_func(void *arg) | |
605 | { | |
f518f154 | 606 | bool errexit = false; |
653ed64b JFG |
607 | int exp, r; |
608 | char buf1[64]; | |
f518f154 | 609 | char *buf; |
dbf28efd | 610 | u64 *result_avg; |
653ed64b JFG |
611 | |
612 | set_cpus_allowed_ptr(current, cpumask_of(nreaders % nr_cpu_ids)); | |
613 | set_user_nice(current, MAX_NICE); | |
614 | ||
1fbeb3a8 | 615 | VERBOSE_SCALEOUT("main_func task started"); |
dbf28efd | 616 | result_avg = kzalloc(nruns * sizeof(*result_avg), GFP_KERNEL); |
f518f154 PM |
617 | buf = kzalloc(64 + nruns * 32, GFP_KERNEL); |
618 | if (!result_avg || !buf) { | |
1fbeb3a8 | 619 | VERBOSE_SCALEOUT_ERRSTRING("out of memory"); |
f518f154 PM |
620 | errexit = true; |
621 | } | |
777a54c9 PM |
622 | if (holdoff) |
623 | schedule_timeout_interruptible(holdoff * HZ); | |
653ed64b | 624 | |
96af8669 PM |
625 | // Wait for all threads to start. |
626 | atomic_inc(&n_init); | |
627 | while (atomic_read(&n_init) < nreaders + 1) | |
628 | schedule_timeout_uninterruptible(1); | |
629 | ||
653ed64b | 630 | // Start exp readers up per experiment |
dbf28efd | 631 | for (exp = 0; exp < nruns && !torture_must_stop(); exp++) { |
f518f154 | 632 | if (errexit) |
dbf28efd | 633 | break; |
653ed64b JFG |
634 | if (torture_must_stop()) |
635 | goto end; | |
636 | ||
dbf28efd PM |
637 | reset_readers(); |
638 | atomic_set(&nreaders_exp, nreaders); | |
86e0da2b | 639 | atomic_set(&n_started, nreaders); |
2db0bda3 PM |
640 | atomic_set(&n_warmedup, nreaders); |
641 | atomic_set(&n_cooleddown, nreaders); | |
653ed64b JFG |
642 | |
643 | exp_idx = exp; | |
644 | ||
dbf28efd | 645 | for (r = 0; r < nreaders; r++) { |
af2789db | 646 | smp_store_release(&reader_tasks[r].start_reader, 1); |
653ed64b JFG |
647 | wake_up(&reader_tasks[r].wq); |
648 | } | |
649 | ||
1fbeb3a8 | 650 | VERBOSE_SCALEOUT("main_func: experiment started, waiting for %d readers", |
dbf28efd | 651 | nreaders); |
653ed64b JFG |
652 | |
653 | wait_event(main_wq, | |
654 | !atomic_read(&nreaders_exp) || torture_must_stop()); | |
655 | ||
1fbeb3a8 | 656 | VERBOSE_SCALEOUT("main_func: experiment ended"); |
653ed64b JFG |
657 | |
658 | if (torture_must_stop()) | |
659 | goto end; | |
660 | ||
7c944d7c | 661 | result_avg[exp] = div_u64(1000 * process_durations(nreaders), nreaders * loops); |
653ed64b JFG |
662 | } |
663 | ||
664 | // Print the average of all experiments | |
1fbeb3a8 | 665 | SCALEOUT("END OF TEST. Calculating average duration per loop (nanoseconds)...\n"); |
653ed64b | 666 | |
58db5785 CIK |
667 | if (!errexit) { |
668 | buf[0] = 0; | |
669 | strcat(buf, "\n"); | |
670 | strcat(buf, "Runs\tTime(ns)\n"); | |
671 | } | |
653ed64b | 672 | |
dbf28efd | 673 | for (exp = 0; exp < nruns; exp++) { |
7c944d7c AB |
674 | u64 avg; |
675 | u32 rem; | |
676 | ||
f518f154 | 677 | if (errexit) |
dbf28efd | 678 | break; |
7c944d7c AB |
679 | avg = div_u64_rem(result_avg[exp], 1000, &rem); |
680 | sprintf(buf1, "%d\t%llu.%03u\n", exp + 1, avg, rem); | |
653ed64b JFG |
681 | strcat(buf, buf1); |
682 | } | |
683 | ||
f518f154 | 684 | if (!errexit) |
1fbeb3a8 | 685 | SCALEOUT("%s", buf); |
653ed64b JFG |
686 | |
687 | // This will shutdown everything including us. | |
688 | if (shutdown) { | |
689 | shutdown_start = 1; | |
690 | wake_up(&shutdown_wq); | |
691 | } | |
692 | ||
693 | // Wait for torture to stop us | |
694 | while (!torture_must_stop()) | |
695 | schedule_timeout_uninterruptible(1); | |
696 | ||
697 | end: | |
698 | torture_kthread_stopping("main_func"); | |
f518f154 PM |
699 | kfree(result_avg); |
700 | kfree(buf); | |
653ed64b JFG |
701 | return 0; |
702 | } | |
703 | ||
704 | static void | |
1fbeb3a8 | 705 | ref_scale_print_module_parms(struct ref_scale_ops *cur_ops, const char *tag) |
653ed64b | 706 | { |
1fbeb3a8 PM |
707 | pr_alert("%s" SCALE_FLAG |
708 | "--- %s: verbose=%d shutdown=%d holdoff=%d loops=%ld nreaders=%d nruns=%d readdelay=%d\n", scale_type, tag, | |
b4d1e34f | 709 | verbose, shutdown, holdoff, loops, nreaders, nruns, readdelay); |
653ed64b JFG |
710 | } |
711 | ||
712 | static void | |
1fbeb3a8 | 713 | ref_scale_cleanup(void) |
653ed64b JFG |
714 | { |
715 | int i; | |
716 | ||
717 | if (torture_cleanup_begin()) | |
718 | return; | |
719 | ||
720 | if (!cur_ops) { | |
721 | torture_cleanup_end(); | |
722 | return; | |
723 | } | |
724 | ||
725 | if (reader_tasks) { | |
726 | for (i = 0; i < nreaders; i++) | |
1fbeb3a8 | 727 | torture_stop_kthread("ref_scale_reader", |
653ed64b JFG |
728 | reader_tasks[i].task); |
729 | } | |
730 | kfree(reader_tasks); | |
731 | ||
732 | torture_stop_kthread("main_task", main_task); | |
733 | kfree(main_task); | |
734 | ||
1fbeb3a8 | 735 | // Do scale-type-specific cleanup operations. |
653ed64b JFG |
736 | if (cur_ops->cleanup != NULL) |
737 | cur_ops->cleanup(); | |
738 | ||
739 | torture_cleanup_end(); | |
740 | } | |
741 | ||
742 | // Shutdown kthread. Just waits to be awakened, then shuts down system. | |
743 | static int | |
1fbeb3a8 | 744 | ref_scale_shutdown(void *arg) |
653ed64b JFG |
745 | { |
746 | wait_event(shutdown_wq, shutdown_start); | |
747 | ||
748 | smp_mb(); // Wake before output. | |
1fbeb3a8 | 749 | ref_scale_cleanup(); |
653ed64b JFG |
750 | kernel_power_off(); |
751 | ||
752 | return -EINVAL; | |
753 | } | |
754 | ||
755 | static int __init | |
1fbeb3a8 | 756 | ref_scale_init(void) |
653ed64b JFG |
757 | { |
758 | long i; | |
759 | int firsterr = 0; | |
1fbeb3a8 | 760 | static struct ref_scale_ops *scale_ops[] = { |
e9b800db PM |
761 | &rcu_ops, &srcu_ops, &rcu_trace_ops, &rcu_tasks_ops, &refcnt_ops, &rwlock_ops, |
762 | &rwsem_ops, &lock_ops, &lock_irq_ops, &acqrel_ops, | |
653ed64b JFG |
763 | }; |
764 | ||
1fbeb3a8 | 765 | if (!torture_init_begin(scale_type, verbose)) |
653ed64b JFG |
766 | return -EBUSY; |
767 | ||
1fbeb3a8 PM |
768 | for (i = 0; i < ARRAY_SIZE(scale_ops); i++) { |
769 | cur_ops = scale_ops[i]; | |
770 | if (strcmp(scale_type, cur_ops->name) == 0) | |
653ed64b JFG |
771 | break; |
772 | } | |
1fbeb3a8 PM |
773 | if (i == ARRAY_SIZE(scale_ops)) { |
774 | pr_alert("rcu-scale: invalid scale type: \"%s\"\n", scale_type); | |
775 | pr_alert("rcu-scale types:"); | |
776 | for (i = 0; i < ARRAY_SIZE(scale_ops); i++) | |
777 | pr_cont(" %s", scale_ops[i]->name); | |
653ed64b | 778 | pr_cont("\n"); |
653ed64b JFG |
779 | firsterr = -EINVAL; |
780 | cur_ops = NULL; | |
781 | goto unwind; | |
782 | } | |
783 | if (cur_ops->init) | |
784 | cur_ops->init(); | |
785 | ||
1fbeb3a8 | 786 | ref_scale_print_module_parms(cur_ops, "Start of test"); |
653ed64b JFG |
787 | |
788 | // Shutdown task | |
789 | if (shutdown) { | |
790 | init_waitqueue_head(&shutdown_wq); | |
1fbeb3a8 | 791 | firsterr = torture_create_kthread(ref_scale_shutdown, NULL, |
653ed64b JFG |
792 | shutdown_task); |
793 | if (firsterr) | |
794 | goto unwind; | |
795 | schedule_timeout_uninterruptible(1); | |
796 | } | |
797 | ||
8fc28783 PM |
798 | // Reader tasks (default to ~75% of online CPUs). |
799 | if (nreaders < 0) | |
800 | nreaders = (num_online_cpus() >> 1) + (num_online_cpus() >> 2); | |
0c6d18d8 PM |
801 | if (WARN_ONCE(loops <= 0, "%s: loops = %ld, adjusted to 1\n", __func__, loops)) |
802 | loops = 1; | |
803 | if (WARN_ONCE(nreaders <= 0, "%s: nreaders = %d, adjusted to 1\n", __func__, nreaders)) | |
804 | nreaders = 1; | |
805 | if (WARN_ONCE(nruns <= 0, "%s: nruns = %d, adjusted to 1\n", __func__, nruns)) | |
806 | nruns = 1; | |
653ed64b JFG |
807 | reader_tasks = kcalloc(nreaders, sizeof(reader_tasks[0]), |
808 | GFP_KERNEL); | |
809 | if (!reader_tasks) { | |
1fbeb3a8 | 810 | VERBOSE_SCALEOUT_ERRSTRING("out of memory"); |
653ed64b JFG |
811 | firsterr = -ENOMEM; |
812 | goto unwind; | |
813 | } | |
814 | ||
1fbeb3a8 | 815 | VERBOSE_SCALEOUT("Starting %d reader threads\n", nreaders); |
653ed64b JFG |
816 | |
817 | for (i = 0; i < nreaders; i++) { | |
1fbeb3a8 | 818 | firsterr = torture_create_kthread(ref_scale_reader, (void *)i, |
653ed64b JFG |
819 | reader_tasks[i].task); |
820 | if (firsterr) | |
821 | goto unwind; | |
822 | ||
823 | init_waitqueue_head(&(reader_tasks[i].wq)); | |
824 | } | |
825 | ||
826 | // Main Task | |
827 | init_waitqueue_head(&main_wq); | |
828 | firsterr = torture_create_kthread(main_func, NULL, main_task); | |
829 | if (firsterr) | |
830 | goto unwind; | |
653ed64b JFG |
831 | |
832 | torture_init_end(); | |
833 | return 0; | |
834 | ||
835 | unwind: | |
836 | torture_init_end(); | |
1fbeb3a8 | 837 | ref_scale_cleanup(); |
bc80d353 PM |
838 | if (shutdown) { |
839 | WARN_ON(!IS_MODULE(CONFIG_RCU_REF_SCALE_TEST)); | |
840 | kernel_power_off(); | |
841 | } | |
653ed64b JFG |
842 | return firsterr; |
843 | } | |
844 | ||
1fbeb3a8 PM |
845 | module_init(ref_scale_init); |
846 | module_exit(ref_scale_cleanup); |