Commit | Line | Data |
---|---|---|
1fe84fd4 ME |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * KCSAN test with various race scenarious to test runtime behaviour. Since the | |
4 | * interface with which KCSAN's reports are obtained is via the console, this is | |
5 | * the output we should verify. For each test case checks the presence (or | |
6 | * absence) of generated reports. Relies on 'console' tracepoint to capture | |
7 | * reports as they appear in the kernel log. | |
8 | * | |
9 | * Makes use of KUnit for test organization, and the Torture framework for test | |
10 | * thread control. | |
11 | * | |
12 | * Copyright (C) 2020, Google LLC. | |
13 | * Author: Marco Elver <elver@google.com> | |
14 | */ | |
15 | ||
f6a14914 ME |
16 | #define pr_fmt(fmt) "kcsan_test: " fmt |
17 | ||
1fe84fd4 | 18 | #include <kunit/test.h> |
8bc32b34 ME |
19 | #include <linux/atomic.h> |
20 | #include <linux/bitops.h> | |
1fe84fd4 ME |
21 | #include <linux/jiffies.h> |
22 | #include <linux/kcsan-checks.h> | |
23 | #include <linux/kernel.h> | |
8bc32b34 | 24 | #include <linux/mutex.h> |
1fe84fd4 ME |
25 | #include <linux/sched.h> |
26 | #include <linux/seqlock.h> | |
27 | #include <linux/spinlock.h> | |
28 | #include <linux/string.h> | |
29 | #include <linux/timer.h> | |
30 | #include <linux/torture.h> | |
31 | #include <linux/tracepoint.h> | |
32 | #include <linux/types.h> | |
33 | #include <trace/events/printk.h> | |
34 | ||
80804284 ME |
35 | #define KCSAN_TEST_REQUIRES(test, cond) do { \ |
36 | if (!(cond)) \ | |
37 | kunit_skip((test), "Test requires: " #cond); \ | |
38 | } while (0) | |
39 | ||
bec4a247 ME |
40 | #ifdef CONFIG_CC_HAS_TSAN_COMPOUND_READ_BEFORE_WRITE |
41 | #define __KCSAN_ACCESS_RW(alt) (KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE) | |
42 | #else | |
43 | #define __KCSAN_ACCESS_RW(alt) (alt) | |
44 | #endif | |
45 | ||
1fe84fd4 ME |
46 | /* Points to current test-case memory access "kernels". */ |
47 | static void (*access_kernels[2])(void); | |
48 | ||
49 | static struct task_struct **threads; /* Lists of threads. */ | |
50 | static unsigned long end_time; /* End time of test. */ | |
51 | ||
52 | /* Report as observed from console. */ | |
53 | static struct { | |
54 | spinlock_t lock; | |
55 | int nlines; | |
56 | char lines[3][512]; | |
57 | } observed = { | |
58 | .lock = __SPIN_LOCK_UNLOCKED(observed.lock), | |
59 | }; | |
60 | ||
61 | /* Setup test checking loop. */ | |
2888557f | 62 | static __no_kcsan inline void |
1fe84fd4 ME |
63 | begin_test_checks(void (*func1)(void), void (*func2)(void)) |
64 | { | |
65 | kcsan_disable_current(); | |
66 | ||
67 | /* | |
68 | * Require at least as long as KCSAN_REPORT_ONCE_IN_MS, to ensure at | |
69 | * least one race is reported. | |
70 | */ | |
71 | end_time = jiffies + msecs_to_jiffies(CONFIG_KCSAN_REPORT_ONCE_IN_MS + 500); | |
72 | ||
73 | /* Signal start; release potential initialization of shared data. */ | |
74 | smp_store_release(&access_kernels[0], func1); | |
75 | smp_store_release(&access_kernels[1], func2); | |
76 | } | |
77 | ||
78 | /* End test checking loop. */ | |
2888557f | 79 | static __no_kcsan inline bool |
1fe84fd4 ME |
80 | end_test_checks(bool stop) |
81 | { | |
82 | if (!stop && time_before(jiffies, end_time)) { | |
83 | /* Continue checking */ | |
84 | might_sleep(); | |
85 | return false; | |
86 | } | |
87 | ||
88 | kcsan_enable_current(); | |
89 | return true; | |
90 | } | |
91 | ||
92 | /* | |
93 | * Probe for console output: checks if a race was reported, and obtains observed | |
94 | * lines of interest. | |
95 | */ | |
96 | __no_kcsan | |
97 | static void probe_console(void *ignore, const char *buf, size_t len) | |
98 | { | |
99 | unsigned long flags; | |
100 | int nlines; | |
101 | ||
102 | /* | |
103 | * Note that KCSAN reports under a global lock, so we do not risk the | |
104 | * possibility of having multiple reports interleaved. If that were the | |
105 | * case, we'd expect tests to fail. | |
106 | */ | |
107 | ||
108 | spin_lock_irqsave(&observed.lock, flags); | |
109 | nlines = observed.nlines; | |
110 | ||
111 | if (strnstr(buf, "BUG: KCSAN: ", len) && strnstr(buf, "test_", len)) { | |
112 | /* | |
113 | * KCSAN report and related to the test. | |
114 | * | |
115 | * The provided @buf is not NUL-terminated; copy no more than | |
116 | * @len bytes and let strscpy() add the missing NUL-terminator. | |
117 | */ | |
118 | strscpy(observed.lines[0], buf, min(len + 1, sizeof(observed.lines[0]))); | |
119 | nlines = 1; | |
120 | } else if ((nlines == 1 || nlines == 2) && strnstr(buf, "bytes by", len)) { | |
121 | strscpy(observed.lines[nlines++], buf, min(len + 1, sizeof(observed.lines[0]))); | |
122 | ||
123 | if (strnstr(buf, "race at unknown origin", len)) { | |
124 | if (WARN_ON(nlines != 2)) | |
125 | goto out; | |
126 | ||
127 | /* No second line of interest. */ | |
128 | strcpy(observed.lines[nlines++], "<none>"); | |
129 | } | |
130 | } | |
131 | ||
132 | out: | |
133 | WRITE_ONCE(observed.nlines, nlines); /* Publish new nlines. */ | |
134 | spin_unlock_irqrestore(&observed.lock, flags); | |
135 | } | |
136 | ||
137 | /* Check if a report related to the test exists. */ | |
138 | __no_kcsan | |
139 | static bool report_available(void) | |
140 | { | |
141 | return READ_ONCE(observed.nlines) == ARRAY_SIZE(observed.lines); | |
142 | } | |
143 | ||
144 | /* Report information we expect in a report. */ | |
145 | struct expect_report { | |
146 | /* Access information of both accesses. */ | |
147 | struct { | |
148 | void *fn; /* Function pointer to expected function of top frame. */ | |
149 | void *addr; /* Address of access; unchecked if NULL. */ | |
150 | size_t size; /* Size of access; unchecked if @addr is NULL. */ | |
151 | int type; /* Access type, see KCSAN_ACCESS definitions. */ | |
152 | } access[2]; | |
153 | }; | |
154 | ||
155 | /* Check observed report matches information in @r. */ | |
156 | __no_kcsan | |
7310bd1f | 157 | static bool __report_matches(const struct expect_report *r) |
1fe84fd4 ME |
158 | { |
159 | const bool is_assert = (r->access[0].type | r->access[1].type) & KCSAN_ACCESS_ASSERT; | |
160 | bool ret = false; | |
161 | unsigned long flags; | |
5b24ac2d | 162 | typeof(*observed.lines) *expect; |
1fe84fd4 ME |
163 | const char *end; |
164 | char *cur; | |
165 | int i; | |
166 | ||
167 | /* Doubled-checked locking. */ | |
168 | if (!report_available()) | |
169 | return false; | |
170 | ||
5b24ac2d MF |
171 | expect = kmalloc(sizeof(observed.lines), GFP_KERNEL); |
172 | if (WARN_ON(!expect)) | |
173 | return false; | |
174 | ||
1fe84fd4 ME |
175 | /* Generate expected report contents. */ |
176 | ||
177 | /* Title */ | |
178 | cur = expect[0]; | |
179 | end = &expect[0][sizeof(expect[0]) - 1]; | |
180 | cur += scnprintf(cur, end - cur, "BUG: KCSAN: %s in ", | |
181 | is_assert ? "assert: race" : "data-race"); | |
182 | if (r->access[1].fn) { | |
183 | char tmp[2][64]; | |
184 | int cmp; | |
185 | ||
186 | /* Expect lexographically sorted function names in title. */ | |
187 | scnprintf(tmp[0], sizeof(tmp[0]), "%pS", r->access[0].fn); | |
188 | scnprintf(tmp[1], sizeof(tmp[1]), "%pS", r->access[1].fn); | |
189 | cmp = strcmp(tmp[0], tmp[1]); | |
190 | cur += scnprintf(cur, end - cur, "%ps / %ps", | |
191 | cmp < 0 ? r->access[0].fn : r->access[1].fn, | |
192 | cmp < 0 ? r->access[1].fn : r->access[0].fn); | |
193 | } else { | |
194 | scnprintf(cur, end - cur, "%pS", r->access[0].fn); | |
195 | /* The exact offset won't match, remove it. */ | |
196 | cur = strchr(expect[0], '+'); | |
197 | if (cur) | |
198 | *cur = '\0'; | |
199 | } | |
200 | ||
201 | /* Access 1 */ | |
202 | cur = expect[1]; | |
203 | end = &expect[1][sizeof(expect[1]) - 1]; | |
204 | if (!r->access[1].fn) | |
205 | cur += scnprintf(cur, end - cur, "race at unknown origin, with "); | |
206 | ||
207 | /* Access 1 & 2 */ | |
208 | for (i = 0; i < 2; ++i) { | |
bec4a247 | 209 | const int ty = r->access[i].type; |
1fe84fd4 | 210 | const char *const access_type = |
bec4a247 ME |
211 | (ty & KCSAN_ACCESS_ASSERT) ? |
212 | ((ty & KCSAN_ACCESS_WRITE) ? | |
213 | "assert no accesses" : | |
214 | "assert no writes") : | |
215 | ((ty & KCSAN_ACCESS_WRITE) ? | |
216 | ((ty & KCSAN_ACCESS_COMPOUND) ? | |
217 | "read-write" : | |
218 | "write") : | |
219 | "read"); | |
d627c537 ME |
220 | const bool is_atomic = (ty & KCSAN_ACCESS_ATOMIC); |
221 | const bool is_scoped = (ty & KCSAN_ACCESS_SCOPED); | |
1fe84fd4 | 222 | const char *const access_type_aux = |
3cc21a53 | 223 | (is_atomic && is_scoped) ? " (marked, reordered)" |
d627c537 | 224 | : (is_atomic ? " (marked)" |
3cc21a53 | 225 | : (is_scoped ? " (reordered)" : "")); |
1fe84fd4 ME |
226 | |
227 | if (i == 1) { | |
228 | /* Access 2 */ | |
229 | cur = expect[2]; | |
230 | end = &expect[2][sizeof(expect[2]) - 1]; | |
231 | ||
232 | if (!r->access[1].fn) { | |
233 | /* Dummy string if no second access is available. */ | |
234 | strcpy(cur, "<none>"); | |
235 | break; | |
236 | } | |
237 | } | |
238 | ||
239 | cur += scnprintf(cur, end - cur, "%s%s to ", access_type, | |
240 | access_type_aux); | |
241 | ||
242 | if (r->access[i].addr) /* Address is optional. */ | |
243 | cur += scnprintf(cur, end - cur, "0x%px of %zu bytes", | |
244 | r->access[i].addr, r->access[i].size); | |
245 | } | |
246 | ||
247 | spin_lock_irqsave(&observed.lock, flags); | |
248 | if (!report_available()) | |
249 | goto out; /* A new report is being captured. */ | |
250 | ||
251 | /* Finally match expected output to what we actually observed. */ | |
252 | ret = strstr(observed.lines[0], expect[0]) && | |
253 | /* Access info may appear in any order. */ | |
254 | ((strstr(observed.lines[1], expect[1]) && | |
255 | strstr(observed.lines[2], expect[2])) || | |
256 | (strstr(observed.lines[1], expect[2]) && | |
257 | strstr(observed.lines[2], expect[1]))); | |
258 | out: | |
259 | spin_unlock_irqrestore(&observed.lock, flags); | |
5b24ac2d | 260 | kfree(expect); |
1fe84fd4 ME |
261 | return ret; |
262 | } | |
263 | ||
7310bd1f ME |
264 | static __always_inline const struct expect_report * |
265 | __report_set_scoped(struct expect_report *r, int accesses) | |
266 | { | |
267 | BUILD_BUG_ON(accesses > 3); | |
268 | ||
269 | if (accesses & 1) | |
270 | r->access[0].type |= KCSAN_ACCESS_SCOPED; | |
271 | else | |
272 | r->access[0].type &= ~KCSAN_ACCESS_SCOPED; | |
273 | ||
274 | if (accesses & 2) | |
275 | r->access[1].type |= KCSAN_ACCESS_SCOPED; | |
276 | else | |
277 | r->access[1].type &= ~KCSAN_ACCESS_SCOPED; | |
278 | ||
279 | return r; | |
280 | } | |
281 | ||
282 | __no_kcsan | |
283 | static bool report_matches_any_reordered(struct expect_report *r) | |
284 | { | |
285 | return __report_matches(__report_set_scoped(r, 0)) || | |
286 | __report_matches(__report_set_scoped(r, 1)) || | |
287 | __report_matches(__report_set_scoped(r, 2)) || | |
288 | __report_matches(__report_set_scoped(r, 3)); | |
289 | } | |
290 | ||
291 | #ifdef CONFIG_KCSAN_WEAK_MEMORY | |
292 | /* Due to reordering accesses, any access may appear as "(reordered)". */ | |
293 | #define report_matches report_matches_any_reordered | |
294 | #else | |
295 | #define report_matches __report_matches | |
296 | #endif | |
297 | ||
1fe84fd4 ME |
298 | /* ===== Test kernels ===== */ |
299 | ||
300 | static long test_sink; | |
301 | static long test_var; | |
302 | /* @test_array should be large enough to fall into multiple watchpoint slots. */ | |
303 | static long test_array[3 * PAGE_SIZE / sizeof(long)]; | |
304 | static struct { | |
305 | long val[8]; | |
306 | } test_struct; | |
31f605a3 | 307 | static long __data_racy test_data_racy; |
1fe84fd4 | 308 | static DEFINE_SEQLOCK(test_seqlock); |
a70d36e6 ME |
309 | static DEFINE_SPINLOCK(test_spinlock); |
310 | static DEFINE_MUTEX(test_mutex); | |
1fe84fd4 ME |
311 | |
312 | /* | |
313 | * Helper to avoid compiler optimizing out reads, and to generate source values | |
314 | * for writes. | |
315 | */ | |
316 | __no_kcsan | |
317 | static noinline void sink_value(long v) { WRITE_ONCE(test_sink, v); } | |
318 | ||
8bc32b34 ME |
319 | /* |
320 | * Generates a delay and some accesses that enter the runtime but do not produce | |
321 | * data races. | |
322 | */ | |
323 | static noinline void test_delay(int iter) | |
324 | { | |
325 | while (iter--) | |
326 | sink_value(READ_ONCE(test_sink)); | |
327 | } | |
328 | ||
1fe84fd4 ME |
329 | static noinline void test_kernel_read(void) { sink_value(test_var); } |
330 | ||
331 | static noinline void test_kernel_write(void) | |
332 | { | |
333 | test_var = READ_ONCE_NOCHECK(test_sink) + 1; | |
334 | } | |
335 | ||
336 | static noinline void test_kernel_write_nochange(void) { test_var = 42; } | |
337 | ||
338 | /* Suffixed by value-change exception filter. */ | |
339 | static noinline void test_kernel_write_nochange_rcu(void) { test_var = 42; } | |
340 | ||
341 | static noinline void test_kernel_read_atomic(void) | |
342 | { | |
343 | sink_value(READ_ONCE(test_var)); | |
344 | } | |
345 | ||
346 | static noinline void test_kernel_write_atomic(void) | |
347 | { | |
348 | WRITE_ONCE(test_var, READ_ONCE_NOCHECK(test_sink) + 1); | |
349 | } | |
350 | ||
bec4a247 ME |
351 | static noinline void test_kernel_atomic_rmw(void) |
352 | { | |
353 | /* Use builtin, so we can set up the "bad" atomic/non-atomic scenario. */ | |
354 | __atomic_fetch_add(&test_var, 1, __ATOMIC_RELAXED); | |
355 | } | |
356 | ||
1fe84fd4 ME |
357 | __no_kcsan |
358 | static noinline void test_kernel_write_uninstrumented(void) { test_var++; } | |
359 | ||
360 | static noinline void test_kernel_data_race(void) { data_race(test_var++); } | |
361 | ||
31f605a3 ME |
362 | static noinline void test_kernel_data_racy_qualifier(void) { test_data_racy++; } |
363 | ||
1fe84fd4 ME |
364 | static noinline void test_kernel_assert_writer(void) |
365 | { | |
366 | ASSERT_EXCLUSIVE_WRITER(test_var); | |
367 | } | |
368 | ||
369 | static noinline void test_kernel_assert_access(void) | |
370 | { | |
371 | ASSERT_EXCLUSIVE_ACCESS(test_var); | |
372 | } | |
373 | ||
374 | #define TEST_CHANGE_BITS 0xff00ff00 | |
375 | ||
376 | static noinline void test_kernel_change_bits(void) | |
377 | { | |
378 | if (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) { | |
379 | /* | |
380 | * Avoid race of unknown origin for this test, just pretend they | |
381 | * are atomic. | |
382 | */ | |
383 | kcsan_nestable_atomic_begin(); | |
384 | test_var ^= TEST_CHANGE_BITS; | |
385 | kcsan_nestable_atomic_end(); | |
386 | } else | |
387 | WRITE_ONCE(test_var, READ_ONCE(test_var) ^ TEST_CHANGE_BITS); | |
388 | } | |
389 | ||
390 | static noinline void test_kernel_assert_bits_change(void) | |
391 | { | |
392 | ASSERT_EXCLUSIVE_BITS(test_var, TEST_CHANGE_BITS); | |
393 | } | |
394 | ||
395 | static noinline void test_kernel_assert_bits_nochange(void) | |
396 | { | |
397 | ASSERT_EXCLUSIVE_BITS(test_var, ~TEST_CHANGE_BITS); | |
398 | } | |
399 | ||
6c65eb75 ME |
400 | /* |
401 | * Scoped assertions do trigger anywhere in scope. However, the report should | |
402 | * still only point at the start of the scope. | |
403 | */ | |
1fe84fd4 ME |
404 | static noinline void test_enter_scope(void) |
405 | { | |
406 | int x = 0; | |
407 | ||
408 | /* Unrelated accesses to scoped assert. */ | |
409 | READ_ONCE(test_sink); | |
410 | kcsan_check_read(&x, sizeof(x)); | |
411 | } | |
412 | ||
413 | static noinline void test_kernel_assert_writer_scoped(void) | |
414 | { | |
415 | ASSERT_EXCLUSIVE_WRITER_SCOPED(test_var); | |
416 | test_enter_scope(); | |
417 | } | |
418 | ||
419 | static noinline void test_kernel_assert_access_scoped(void) | |
420 | { | |
421 | ASSERT_EXCLUSIVE_ACCESS_SCOPED(test_var); | |
422 | test_enter_scope(); | |
423 | } | |
424 | ||
425 | static noinline void test_kernel_rmw_array(void) | |
426 | { | |
427 | int i; | |
428 | ||
429 | for (i = 0; i < ARRAY_SIZE(test_array); ++i) | |
430 | test_array[i]++; | |
431 | } | |
432 | ||
433 | static noinline void test_kernel_write_struct(void) | |
434 | { | |
435 | kcsan_check_write(&test_struct, sizeof(test_struct)); | |
436 | kcsan_disable_current(); | |
437 | test_struct.val[3]++; /* induce value change */ | |
438 | kcsan_enable_current(); | |
439 | } | |
440 | ||
441 | static noinline void test_kernel_write_struct_part(void) | |
442 | { | |
443 | test_struct.val[3] = 42; | |
444 | } | |
445 | ||
446 | static noinline void test_kernel_read_struct_zero_size(void) | |
447 | { | |
448 | kcsan_check_read(&test_struct.val[3], 0); | |
449 | } | |
450 | ||
56b031f0 ME |
451 | static noinline void test_kernel_jiffies_reader(void) |
452 | { | |
453 | sink_value((long)jiffies); | |
454 | } | |
455 | ||
1fe84fd4 ME |
456 | static noinline void test_kernel_seqlock_reader(void) |
457 | { | |
458 | unsigned int seq; | |
459 | ||
460 | do { | |
461 | seq = read_seqbegin(&test_seqlock); | |
462 | sink_value(test_var); | |
463 | } while (read_seqretry(&test_seqlock, seq)); | |
464 | } | |
465 | ||
466 | static noinline void test_kernel_seqlock_writer(void) | |
467 | { | |
468 | unsigned long flags; | |
469 | ||
470 | write_seqlock_irqsave(&test_seqlock, flags); | |
471 | test_var++; | |
472 | write_sequnlock_irqrestore(&test_seqlock, flags); | |
473 | } | |
474 | ||
f9ea6319 ME |
475 | static noinline void test_kernel_atomic_builtins(void) |
476 | { | |
477 | /* | |
478 | * Generate concurrent accesses, expecting no reports, ensuring KCSAN | |
479 | * treats builtin atomics as actually atomic. | |
480 | */ | |
481 | __atomic_load_n(&test_var, __ATOMIC_RELAXED); | |
482 | } | |
483 | ||
d8fd74d3 ME |
484 | static noinline void test_kernel_xor_1bit(void) |
485 | { | |
486 | /* Do not report data races between the read-writes. */ | |
487 | kcsan_nestable_atomic_begin(); | |
488 | test_var ^= 0x10000; | |
489 | kcsan_nestable_atomic_end(); | |
490 | } | |
491 | ||
8bc32b34 ME |
492 | #define TEST_KERNEL_LOCKED(name, acquire, release) \ |
493 | static noinline void test_kernel_##name(void) \ | |
494 | { \ | |
495 | long *flag = &test_struct.val[0]; \ | |
496 | long v = 0; \ | |
497 | if (!(acquire)) \ | |
498 | return; \ | |
499 | while (v++ < 100) { \ | |
500 | test_var++; \ | |
501 | barrier(); \ | |
502 | } \ | |
503 | release; \ | |
504 | test_delay(10); \ | |
505 | } | |
506 | ||
507 | TEST_KERNEL_LOCKED(with_memorder, | |
508 | cmpxchg_acquire(flag, 0, 1) == 0, | |
509 | smp_store_release(flag, 0)); | |
510 | TEST_KERNEL_LOCKED(wrong_memorder, | |
511 | cmpxchg_relaxed(flag, 0, 1) == 0, | |
512 | WRITE_ONCE(*flag, 0)); | |
513 | TEST_KERNEL_LOCKED(atomic_builtin_with_memorder, | |
514 | __atomic_compare_exchange_n(flag, &v, 1, 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED), | |
515 | __atomic_store_n(flag, 0, __ATOMIC_RELEASE)); | |
516 | TEST_KERNEL_LOCKED(atomic_builtin_wrong_memorder, | |
517 | __atomic_compare_exchange_n(flag, &v, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED), | |
518 | __atomic_store_n(flag, 0, __ATOMIC_RELAXED)); | |
519 | ||
1fe84fd4 ME |
520 | /* ===== Test cases ===== */ |
521 | ||
8bc32b34 ME |
522 | /* |
523 | * Tests that various barriers have the expected effect on internal state. Not | |
524 | * exhaustive on atomic_t operations. Unlike the selftest, also checks for | |
525 | * too-strict barrier instrumentation; these can be tolerated, because it does | |
526 | * not cause false positives, but at least we should be aware of such cases. | |
527 | */ | |
528 | static void test_barrier_nothreads(struct kunit *test) | |
529 | { | |
530 | #ifdef CONFIG_KCSAN_WEAK_MEMORY | |
531 | struct kcsan_scoped_access *reorder_access = ¤t->kcsan_ctx.reorder_access; | |
532 | #else | |
533 | struct kcsan_scoped_access *reorder_access = NULL; | |
534 | #endif | |
535 | arch_spinlock_t arch_spinlock = __ARCH_SPIN_LOCK_UNLOCKED; | |
8bc32b34 ME |
536 | atomic_t dummy; |
537 | ||
538 | KCSAN_TEST_REQUIRES(test, reorder_access != NULL); | |
539 | KCSAN_TEST_REQUIRES(test, IS_ENABLED(CONFIG_SMP)); | |
540 | ||
541 | #define __KCSAN_EXPECT_BARRIER(access_type, barrier, order_before, name) \ | |
542 | do { \ | |
543 | reorder_access->type = (access_type) | KCSAN_ACCESS_SCOPED; \ | |
544 | reorder_access->size = sizeof(test_var); \ | |
545 | barrier; \ | |
546 | KUNIT_EXPECT_EQ_MSG(test, reorder_access->size, \ | |
547 | order_before ? 0 : sizeof(test_var), \ | |
548 | "improperly instrumented type=(" #access_type "): " name); \ | |
549 | } while (0) | |
550 | #define KCSAN_EXPECT_READ_BARRIER(b, o) __KCSAN_EXPECT_BARRIER(0, b, o, #b) | |
551 | #define KCSAN_EXPECT_WRITE_BARRIER(b, o) __KCSAN_EXPECT_BARRIER(KCSAN_ACCESS_WRITE, b, o, #b) | |
552 | #define KCSAN_EXPECT_RW_BARRIER(b, o) __KCSAN_EXPECT_BARRIER(KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE, b, o, #b) | |
553 | ||
a70d36e6 ME |
554 | /* |
555 | * Lockdep initialization can strengthen certain locking operations due | |
556 | * to calling into instrumented files; "warm up" our locks. | |
557 | */ | |
558 | spin_lock(&test_spinlock); | |
559 | spin_unlock(&test_spinlock); | |
560 | mutex_lock(&test_mutex); | |
561 | mutex_unlock(&test_mutex); | |
562 | ||
8bc32b34 ME |
563 | /* Force creating a valid entry in reorder_access first. */ |
564 | test_var = 0; | |
565 | while (test_var++ < 1000000 && reorder_access->size != sizeof(test_var)) | |
566 | __kcsan_check_read(&test_var, sizeof(test_var)); | |
567 | KUNIT_ASSERT_EQ(test, reorder_access->size, sizeof(test_var)); | |
568 | ||
569 | kcsan_nestable_atomic_begin(); /* No watchpoints in called functions. */ | |
570 | ||
571 | KCSAN_EXPECT_READ_BARRIER(mb(), true); | |
572 | KCSAN_EXPECT_READ_BARRIER(wmb(), false); | |
573 | KCSAN_EXPECT_READ_BARRIER(rmb(), true); | |
574 | KCSAN_EXPECT_READ_BARRIER(smp_mb(), true); | |
575 | KCSAN_EXPECT_READ_BARRIER(smp_wmb(), false); | |
576 | KCSAN_EXPECT_READ_BARRIER(smp_rmb(), true); | |
577 | KCSAN_EXPECT_READ_BARRIER(dma_wmb(), false); | |
578 | KCSAN_EXPECT_READ_BARRIER(dma_rmb(), true); | |
579 | KCSAN_EXPECT_READ_BARRIER(smp_mb__before_atomic(), true); | |
580 | KCSAN_EXPECT_READ_BARRIER(smp_mb__after_atomic(), true); | |
581 | KCSAN_EXPECT_READ_BARRIER(smp_mb__after_spinlock(), true); | |
582 | KCSAN_EXPECT_READ_BARRIER(smp_store_mb(test_var, 0), true); | |
583 | KCSAN_EXPECT_READ_BARRIER(smp_load_acquire(&test_var), false); | |
584 | KCSAN_EXPECT_READ_BARRIER(smp_store_release(&test_var, 0), true); | |
585 | KCSAN_EXPECT_READ_BARRIER(xchg(&test_var, 0), true); | |
586 | KCSAN_EXPECT_READ_BARRIER(xchg_release(&test_var, 0), true); | |
587 | KCSAN_EXPECT_READ_BARRIER(xchg_relaxed(&test_var, 0), false); | |
588 | KCSAN_EXPECT_READ_BARRIER(cmpxchg(&test_var, 0, 0), true); | |
589 | KCSAN_EXPECT_READ_BARRIER(cmpxchg_release(&test_var, 0, 0), true); | |
590 | KCSAN_EXPECT_READ_BARRIER(cmpxchg_relaxed(&test_var, 0, 0), false); | |
591 | KCSAN_EXPECT_READ_BARRIER(atomic_read(&dummy), false); | |
592 | KCSAN_EXPECT_READ_BARRIER(atomic_read_acquire(&dummy), false); | |
593 | KCSAN_EXPECT_READ_BARRIER(atomic_set(&dummy, 0), false); | |
594 | KCSAN_EXPECT_READ_BARRIER(atomic_set_release(&dummy, 0), true); | |
595 | KCSAN_EXPECT_READ_BARRIER(atomic_add(1, &dummy), false); | |
596 | KCSAN_EXPECT_READ_BARRIER(atomic_add_return(1, &dummy), true); | |
597 | KCSAN_EXPECT_READ_BARRIER(atomic_add_return_acquire(1, &dummy), false); | |
598 | KCSAN_EXPECT_READ_BARRIER(atomic_add_return_release(1, &dummy), true); | |
599 | KCSAN_EXPECT_READ_BARRIER(atomic_add_return_relaxed(1, &dummy), false); | |
600 | KCSAN_EXPECT_READ_BARRIER(atomic_fetch_add(1, &dummy), true); | |
601 | KCSAN_EXPECT_READ_BARRIER(atomic_fetch_add_acquire(1, &dummy), false); | |
602 | KCSAN_EXPECT_READ_BARRIER(atomic_fetch_add_release(1, &dummy), true); | |
603 | KCSAN_EXPECT_READ_BARRIER(atomic_fetch_add_relaxed(1, &dummy), false); | |
604 | KCSAN_EXPECT_READ_BARRIER(test_and_set_bit(0, &test_var), true); | |
605 | KCSAN_EXPECT_READ_BARRIER(test_and_clear_bit(0, &test_var), true); | |
606 | KCSAN_EXPECT_READ_BARRIER(test_and_change_bit(0, &test_var), true); | |
607 | KCSAN_EXPECT_READ_BARRIER(clear_bit_unlock(0, &test_var), true); | |
608 | KCSAN_EXPECT_READ_BARRIER(__clear_bit_unlock(0, &test_var), true); | |
8bc32b34 ME |
609 | KCSAN_EXPECT_READ_BARRIER(arch_spin_lock(&arch_spinlock), false); |
610 | KCSAN_EXPECT_READ_BARRIER(arch_spin_unlock(&arch_spinlock), true); | |
a70d36e6 ME |
611 | KCSAN_EXPECT_READ_BARRIER(spin_lock(&test_spinlock), false); |
612 | KCSAN_EXPECT_READ_BARRIER(spin_unlock(&test_spinlock), true); | |
613 | KCSAN_EXPECT_READ_BARRIER(mutex_lock(&test_mutex), false); | |
614 | KCSAN_EXPECT_READ_BARRIER(mutex_unlock(&test_mutex), true); | |
8bc32b34 ME |
615 | |
616 | KCSAN_EXPECT_WRITE_BARRIER(mb(), true); | |
617 | KCSAN_EXPECT_WRITE_BARRIER(wmb(), true); | |
618 | KCSAN_EXPECT_WRITE_BARRIER(rmb(), false); | |
619 | KCSAN_EXPECT_WRITE_BARRIER(smp_mb(), true); | |
620 | KCSAN_EXPECT_WRITE_BARRIER(smp_wmb(), true); | |
621 | KCSAN_EXPECT_WRITE_BARRIER(smp_rmb(), false); | |
622 | KCSAN_EXPECT_WRITE_BARRIER(dma_wmb(), true); | |
623 | KCSAN_EXPECT_WRITE_BARRIER(dma_rmb(), false); | |
624 | KCSAN_EXPECT_WRITE_BARRIER(smp_mb__before_atomic(), true); | |
625 | KCSAN_EXPECT_WRITE_BARRIER(smp_mb__after_atomic(), true); | |
626 | KCSAN_EXPECT_WRITE_BARRIER(smp_mb__after_spinlock(), true); | |
627 | KCSAN_EXPECT_WRITE_BARRIER(smp_store_mb(test_var, 0), true); | |
628 | KCSAN_EXPECT_WRITE_BARRIER(smp_load_acquire(&test_var), false); | |
629 | KCSAN_EXPECT_WRITE_BARRIER(smp_store_release(&test_var, 0), true); | |
630 | KCSAN_EXPECT_WRITE_BARRIER(xchg(&test_var, 0), true); | |
631 | KCSAN_EXPECT_WRITE_BARRIER(xchg_release(&test_var, 0), true); | |
632 | KCSAN_EXPECT_WRITE_BARRIER(xchg_relaxed(&test_var, 0), false); | |
633 | KCSAN_EXPECT_WRITE_BARRIER(cmpxchg(&test_var, 0, 0), true); | |
634 | KCSAN_EXPECT_WRITE_BARRIER(cmpxchg_release(&test_var, 0, 0), true); | |
635 | KCSAN_EXPECT_WRITE_BARRIER(cmpxchg_relaxed(&test_var, 0, 0), false); | |
636 | KCSAN_EXPECT_WRITE_BARRIER(atomic_read(&dummy), false); | |
637 | KCSAN_EXPECT_WRITE_BARRIER(atomic_read_acquire(&dummy), false); | |
638 | KCSAN_EXPECT_WRITE_BARRIER(atomic_set(&dummy, 0), false); | |
639 | KCSAN_EXPECT_WRITE_BARRIER(atomic_set_release(&dummy, 0), true); | |
640 | KCSAN_EXPECT_WRITE_BARRIER(atomic_add(1, &dummy), false); | |
641 | KCSAN_EXPECT_WRITE_BARRIER(atomic_add_return(1, &dummy), true); | |
642 | KCSAN_EXPECT_WRITE_BARRIER(atomic_add_return_acquire(1, &dummy), false); | |
643 | KCSAN_EXPECT_WRITE_BARRIER(atomic_add_return_release(1, &dummy), true); | |
644 | KCSAN_EXPECT_WRITE_BARRIER(atomic_add_return_relaxed(1, &dummy), false); | |
645 | KCSAN_EXPECT_WRITE_BARRIER(atomic_fetch_add(1, &dummy), true); | |
646 | KCSAN_EXPECT_WRITE_BARRIER(atomic_fetch_add_acquire(1, &dummy), false); | |
647 | KCSAN_EXPECT_WRITE_BARRIER(atomic_fetch_add_release(1, &dummy), true); | |
648 | KCSAN_EXPECT_WRITE_BARRIER(atomic_fetch_add_relaxed(1, &dummy), false); | |
649 | KCSAN_EXPECT_WRITE_BARRIER(test_and_set_bit(0, &test_var), true); | |
650 | KCSAN_EXPECT_WRITE_BARRIER(test_and_clear_bit(0, &test_var), true); | |
651 | KCSAN_EXPECT_WRITE_BARRIER(test_and_change_bit(0, &test_var), true); | |
652 | KCSAN_EXPECT_WRITE_BARRIER(clear_bit_unlock(0, &test_var), true); | |
653 | KCSAN_EXPECT_WRITE_BARRIER(__clear_bit_unlock(0, &test_var), true); | |
8bc32b34 ME |
654 | KCSAN_EXPECT_WRITE_BARRIER(arch_spin_lock(&arch_spinlock), false); |
655 | KCSAN_EXPECT_WRITE_BARRIER(arch_spin_unlock(&arch_spinlock), true); | |
a70d36e6 ME |
656 | KCSAN_EXPECT_WRITE_BARRIER(spin_lock(&test_spinlock), false); |
657 | KCSAN_EXPECT_WRITE_BARRIER(spin_unlock(&test_spinlock), true); | |
658 | KCSAN_EXPECT_WRITE_BARRIER(mutex_lock(&test_mutex), false); | |
659 | KCSAN_EXPECT_WRITE_BARRIER(mutex_unlock(&test_mutex), true); | |
8bc32b34 ME |
660 | |
661 | KCSAN_EXPECT_RW_BARRIER(mb(), true); | |
662 | KCSAN_EXPECT_RW_BARRIER(wmb(), true); | |
663 | KCSAN_EXPECT_RW_BARRIER(rmb(), true); | |
664 | KCSAN_EXPECT_RW_BARRIER(smp_mb(), true); | |
665 | KCSAN_EXPECT_RW_BARRIER(smp_wmb(), true); | |
666 | KCSAN_EXPECT_RW_BARRIER(smp_rmb(), true); | |
667 | KCSAN_EXPECT_RW_BARRIER(dma_wmb(), true); | |
668 | KCSAN_EXPECT_RW_BARRIER(dma_rmb(), true); | |
669 | KCSAN_EXPECT_RW_BARRIER(smp_mb__before_atomic(), true); | |
670 | KCSAN_EXPECT_RW_BARRIER(smp_mb__after_atomic(), true); | |
671 | KCSAN_EXPECT_RW_BARRIER(smp_mb__after_spinlock(), true); | |
672 | KCSAN_EXPECT_RW_BARRIER(smp_store_mb(test_var, 0), true); | |
673 | KCSAN_EXPECT_RW_BARRIER(smp_load_acquire(&test_var), false); | |
674 | KCSAN_EXPECT_RW_BARRIER(smp_store_release(&test_var, 0), true); | |
675 | KCSAN_EXPECT_RW_BARRIER(xchg(&test_var, 0), true); | |
676 | KCSAN_EXPECT_RW_BARRIER(xchg_release(&test_var, 0), true); | |
677 | KCSAN_EXPECT_RW_BARRIER(xchg_relaxed(&test_var, 0), false); | |
678 | KCSAN_EXPECT_RW_BARRIER(cmpxchg(&test_var, 0, 0), true); | |
679 | KCSAN_EXPECT_RW_BARRIER(cmpxchg_release(&test_var, 0, 0), true); | |
680 | KCSAN_EXPECT_RW_BARRIER(cmpxchg_relaxed(&test_var, 0, 0), false); | |
681 | KCSAN_EXPECT_RW_BARRIER(atomic_read(&dummy), false); | |
682 | KCSAN_EXPECT_RW_BARRIER(atomic_read_acquire(&dummy), false); | |
683 | KCSAN_EXPECT_RW_BARRIER(atomic_set(&dummy, 0), false); | |
684 | KCSAN_EXPECT_RW_BARRIER(atomic_set_release(&dummy, 0), true); | |
685 | KCSAN_EXPECT_RW_BARRIER(atomic_add(1, &dummy), false); | |
686 | KCSAN_EXPECT_RW_BARRIER(atomic_add_return(1, &dummy), true); | |
687 | KCSAN_EXPECT_RW_BARRIER(atomic_add_return_acquire(1, &dummy), false); | |
688 | KCSAN_EXPECT_RW_BARRIER(atomic_add_return_release(1, &dummy), true); | |
689 | KCSAN_EXPECT_RW_BARRIER(atomic_add_return_relaxed(1, &dummy), false); | |
690 | KCSAN_EXPECT_RW_BARRIER(atomic_fetch_add(1, &dummy), true); | |
691 | KCSAN_EXPECT_RW_BARRIER(atomic_fetch_add_acquire(1, &dummy), false); | |
692 | KCSAN_EXPECT_RW_BARRIER(atomic_fetch_add_release(1, &dummy), true); | |
693 | KCSAN_EXPECT_RW_BARRIER(atomic_fetch_add_relaxed(1, &dummy), false); | |
694 | KCSAN_EXPECT_RW_BARRIER(test_and_set_bit(0, &test_var), true); | |
695 | KCSAN_EXPECT_RW_BARRIER(test_and_clear_bit(0, &test_var), true); | |
696 | KCSAN_EXPECT_RW_BARRIER(test_and_change_bit(0, &test_var), true); | |
697 | KCSAN_EXPECT_RW_BARRIER(clear_bit_unlock(0, &test_var), true); | |
698 | KCSAN_EXPECT_RW_BARRIER(__clear_bit_unlock(0, &test_var), true); | |
8bc32b34 ME |
699 | KCSAN_EXPECT_RW_BARRIER(arch_spin_lock(&arch_spinlock), false); |
700 | KCSAN_EXPECT_RW_BARRIER(arch_spin_unlock(&arch_spinlock), true); | |
a70d36e6 ME |
701 | KCSAN_EXPECT_RW_BARRIER(spin_lock(&test_spinlock), false); |
702 | KCSAN_EXPECT_RW_BARRIER(spin_unlock(&test_spinlock), true); | |
703 | KCSAN_EXPECT_RW_BARRIER(mutex_lock(&test_mutex), false); | |
704 | KCSAN_EXPECT_RW_BARRIER(mutex_unlock(&test_mutex), true); | |
247dbcdb MWO |
705 | KCSAN_EXPECT_READ_BARRIER(xor_unlock_is_negative_byte(1, &test_var), true); |
706 | KCSAN_EXPECT_WRITE_BARRIER(xor_unlock_is_negative_byte(1, &test_var), true); | |
707 | KCSAN_EXPECT_RW_BARRIER(xor_unlock_is_negative_byte(1, &test_var), true); | |
8bc32b34 ME |
708 | kcsan_nestable_atomic_end(); |
709 | } | |
710 | ||
1fe84fd4 ME |
711 | /* Simple test with normal data race. */ |
712 | __no_kcsan | |
713 | static void test_basic(struct kunit *test) | |
714 | { | |
7310bd1f | 715 | struct expect_report expect = { |
1fe84fd4 ME |
716 | .access = { |
717 | { test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, | |
718 | { test_kernel_read, &test_var, sizeof(test_var), 0 }, | |
719 | }, | |
720 | }; | |
7310bd1f | 721 | struct expect_report never = { |
1fe84fd4 ME |
722 | .access = { |
723 | { test_kernel_read, &test_var, sizeof(test_var), 0 }, | |
724 | { test_kernel_read, &test_var, sizeof(test_var), 0 }, | |
725 | }, | |
726 | }; | |
727 | bool match_expect = false; | |
728 | bool match_never = false; | |
729 | ||
730 | begin_test_checks(test_kernel_write, test_kernel_read); | |
731 | do { | |
732 | match_expect |= report_matches(&expect); | |
733 | match_never = report_matches(&never); | |
734 | } while (!end_test_checks(match_never)); | |
735 | KUNIT_EXPECT_TRUE(test, match_expect); | |
736 | KUNIT_EXPECT_FALSE(test, match_never); | |
737 | } | |
738 | ||
739 | /* | |
740 | * Stress KCSAN with lots of concurrent races on different addresses until | |
741 | * timeout. | |
742 | */ | |
743 | __no_kcsan | |
744 | static void test_concurrent_races(struct kunit *test) | |
745 | { | |
7310bd1f | 746 | struct expect_report expect = { |
1fe84fd4 ME |
747 | .access = { |
748 | /* NULL will match any address. */ | |
bec4a247 ME |
749 | { test_kernel_rmw_array, NULL, 0, __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) }, |
750 | { test_kernel_rmw_array, NULL, 0, __KCSAN_ACCESS_RW(0) }, | |
1fe84fd4 ME |
751 | }, |
752 | }; | |
7310bd1f | 753 | struct expect_report never = { |
1fe84fd4 ME |
754 | .access = { |
755 | { test_kernel_rmw_array, NULL, 0, 0 }, | |
756 | { test_kernel_rmw_array, NULL, 0, 0 }, | |
757 | }, | |
758 | }; | |
759 | bool match_expect = false; | |
760 | bool match_never = false; | |
761 | ||
762 | begin_test_checks(test_kernel_rmw_array, test_kernel_rmw_array); | |
763 | do { | |
764 | match_expect |= report_matches(&expect); | |
765 | match_never |= report_matches(&never); | |
766 | } while (!end_test_checks(false)); | |
767 | KUNIT_EXPECT_TRUE(test, match_expect); /* Sanity check matches exist. */ | |
768 | KUNIT_EXPECT_FALSE(test, match_never); | |
769 | } | |
770 | ||
771 | /* Test the KCSAN_REPORT_VALUE_CHANGE_ONLY option. */ | |
772 | __no_kcsan | |
773 | static void test_novalue_change(struct kunit *test) | |
774 | { | |
7310bd1f | 775 | struct expect_report expect_rw = { |
1fe84fd4 ME |
776 | .access = { |
777 | { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, | |
778 | { test_kernel_read, &test_var, sizeof(test_var), 0 }, | |
779 | }, | |
780 | }; | |
7310bd1f | 781 | struct expect_report expect_ww = { |
ade3a58b ME |
782 | .access = { |
783 | { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, | |
784 | { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, | |
785 | }, | |
786 | }; | |
1fe84fd4 ME |
787 | bool match_expect = false; |
788 | ||
ade3a58b | 789 | test_kernel_write_nochange(); /* Reset value. */ |
1fe84fd4 ME |
790 | begin_test_checks(test_kernel_write_nochange, test_kernel_read); |
791 | do { | |
ade3a58b | 792 | match_expect = report_matches(&expect_rw) || report_matches(&expect_ww); |
1fe84fd4 ME |
793 | } while (!end_test_checks(match_expect)); |
794 | if (IS_ENABLED(CONFIG_KCSAN_REPORT_VALUE_CHANGE_ONLY)) | |
795 | KUNIT_EXPECT_FALSE(test, match_expect); | |
796 | else | |
797 | KUNIT_EXPECT_TRUE(test, match_expect); | |
798 | } | |
799 | ||
800 | /* | |
801 | * Test that the rules where the KCSAN_REPORT_VALUE_CHANGE_ONLY option should | |
802 | * never apply work. | |
803 | */ | |
804 | __no_kcsan | |
805 | static void test_novalue_change_exception(struct kunit *test) | |
806 | { | |
7310bd1f | 807 | struct expect_report expect_rw = { |
1fe84fd4 ME |
808 | .access = { |
809 | { test_kernel_write_nochange_rcu, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, | |
810 | { test_kernel_read, &test_var, sizeof(test_var), 0 }, | |
811 | }, | |
812 | }; | |
7310bd1f | 813 | struct expect_report expect_ww = { |
ade3a58b ME |
814 | .access = { |
815 | { test_kernel_write_nochange_rcu, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, | |
816 | { test_kernel_write_nochange_rcu, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, | |
817 | }, | |
818 | }; | |
1fe84fd4 ME |
819 | bool match_expect = false; |
820 | ||
ade3a58b | 821 | test_kernel_write_nochange_rcu(); /* Reset value. */ |
1fe84fd4 ME |
822 | begin_test_checks(test_kernel_write_nochange_rcu, test_kernel_read); |
823 | do { | |
ade3a58b | 824 | match_expect = report_matches(&expect_rw) || report_matches(&expect_ww); |
1fe84fd4 ME |
825 | } while (!end_test_checks(match_expect)); |
826 | KUNIT_EXPECT_TRUE(test, match_expect); | |
827 | } | |
828 | ||
829 | /* Test that data races of unknown origin are reported. */ | |
830 | __no_kcsan | |
831 | static void test_unknown_origin(struct kunit *test) | |
832 | { | |
7310bd1f | 833 | struct expect_report expect = { |
1fe84fd4 ME |
834 | .access = { |
835 | { test_kernel_read, &test_var, sizeof(test_var), 0 }, | |
836 | { NULL }, | |
837 | }, | |
838 | }; | |
839 | bool match_expect = false; | |
840 | ||
841 | begin_test_checks(test_kernel_write_uninstrumented, test_kernel_read); | |
842 | do { | |
843 | match_expect = report_matches(&expect); | |
844 | } while (!end_test_checks(match_expect)); | |
845 | if (IS_ENABLED(CONFIG_KCSAN_REPORT_RACE_UNKNOWN_ORIGIN)) | |
846 | KUNIT_EXPECT_TRUE(test, match_expect); | |
847 | else | |
848 | KUNIT_EXPECT_FALSE(test, match_expect); | |
849 | } | |
850 | ||
851 | /* Test KCSAN_ASSUME_PLAIN_WRITES_ATOMIC if it is selected. */ | |
852 | __no_kcsan | |
853 | static void test_write_write_assume_atomic(struct kunit *test) | |
854 | { | |
7310bd1f | 855 | struct expect_report expect = { |
1fe84fd4 ME |
856 | .access = { |
857 | { test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, | |
858 | { test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, | |
859 | }, | |
860 | }; | |
861 | bool match_expect = false; | |
862 | ||
863 | begin_test_checks(test_kernel_write, test_kernel_write); | |
864 | do { | |
865 | sink_value(READ_ONCE(test_var)); /* induce value-change */ | |
866 | match_expect = report_matches(&expect); | |
867 | } while (!end_test_checks(match_expect)); | |
868 | if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC)) | |
869 | KUNIT_EXPECT_FALSE(test, match_expect); | |
870 | else | |
871 | KUNIT_EXPECT_TRUE(test, match_expect); | |
872 | } | |
873 | ||
874 | /* | |
875 | * Test that data races with writes larger than word-size are always reported, | |
876 | * even if KCSAN_ASSUME_PLAIN_WRITES_ATOMIC is selected. | |
877 | */ | |
878 | __no_kcsan | |
879 | static void test_write_write_struct(struct kunit *test) | |
880 | { | |
7310bd1f | 881 | struct expect_report expect = { |
1fe84fd4 ME |
882 | .access = { |
883 | { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE }, | |
884 | { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE }, | |
885 | }, | |
886 | }; | |
887 | bool match_expect = false; | |
888 | ||
889 | begin_test_checks(test_kernel_write_struct, test_kernel_write_struct); | |
890 | do { | |
891 | match_expect = report_matches(&expect); | |
892 | } while (!end_test_checks(match_expect)); | |
893 | KUNIT_EXPECT_TRUE(test, match_expect); | |
894 | } | |
895 | ||
896 | /* | |
897 | * Test that data races where only one write is larger than word-size are always | |
898 | * reported, even if KCSAN_ASSUME_PLAIN_WRITES_ATOMIC is selected. | |
899 | */ | |
900 | __no_kcsan | |
901 | static void test_write_write_struct_part(struct kunit *test) | |
902 | { | |
7310bd1f | 903 | struct expect_report expect = { |
1fe84fd4 ME |
904 | .access = { |
905 | { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE }, | |
906 | { test_kernel_write_struct_part, &test_struct.val[3], sizeof(test_struct.val[3]), KCSAN_ACCESS_WRITE }, | |
907 | }, | |
908 | }; | |
909 | bool match_expect = false; | |
910 | ||
911 | begin_test_checks(test_kernel_write_struct, test_kernel_write_struct_part); | |
912 | do { | |
913 | match_expect = report_matches(&expect); | |
914 | } while (!end_test_checks(match_expect)); | |
915 | KUNIT_EXPECT_TRUE(test, match_expect); | |
916 | } | |
917 | ||
918 | /* Test that races with atomic accesses never result in reports. */ | |
919 | __no_kcsan | |
920 | static void test_read_atomic_write_atomic(struct kunit *test) | |
921 | { | |
922 | bool match_never = false; | |
923 | ||
924 | begin_test_checks(test_kernel_read_atomic, test_kernel_write_atomic); | |
925 | do { | |
926 | match_never = report_available(); | |
927 | } while (!end_test_checks(match_never)); | |
928 | KUNIT_EXPECT_FALSE(test, match_never); | |
929 | } | |
930 | ||
931 | /* Test that a race with an atomic and plain access result in reports. */ | |
932 | __no_kcsan | |
933 | static void test_read_plain_atomic_write(struct kunit *test) | |
934 | { | |
7310bd1f | 935 | struct expect_report expect = { |
1fe84fd4 ME |
936 | .access = { |
937 | { test_kernel_read, &test_var, sizeof(test_var), 0 }, | |
938 | { test_kernel_write_atomic, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC }, | |
939 | }, | |
940 | }; | |
941 | bool match_expect = false; | |
942 | ||
80804284 | 943 | KCSAN_TEST_REQUIRES(test, !IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)); |
1fe84fd4 ME |
944 | |
945 | begin_test_checks(test_kernel_read, test_kernel_write_atomic); | |
946 | do { | |
947 | match_expect = report_matches(&expect); | |
948 | } while (!end_test_checks(match_expect)); | |
949 | KUNIT_EXPECT_TRUE(test, match_expect); | |
950 | } | |
951 | ||
bec4a247 ME |
952 | /* Test that atomic RMWs generate correct report. */ |
953 | __no_kcsan | |
954 | static void test_read_plain_atomic_rmw(struct kunit *test) | |
955 | { | |
7310bd1f | 956 | struct expect_report expect = { |
bec4a247 ME |
957 | .access = { |
958 | { test_kernel_read, &test_var, sizeof(test_var), 0 }, | |
959 | { test_kernel_atomic_rmw, &test_var, sizeof(test_var), | |
960 | KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC }, | |
961 | }, | |
962 | }; | |
963 | bool match_expect = false; | |
964 | ||
80804284 | 965 | KCSAN_TEST_REQUIRES(test, !IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)); |
bec4a247 ME |
966 | |
967 | begin_test_checks(test_kernel_read, test_kernel_atomic_rmw); | |
968 | do { | |
969 | match_expect = report_matches(&expect); | |
970 | } while (!end_test_checks(match_expect)); | |
971 | KUNIT_EXPECT_TRUE(test, match_expect); | |
972 | } | |
973 | ||
1fe84fd4 ME |
974 | /* Zero-sized accesses should never cause data race reports. */ |
975 | __no_kcsan | |
976 | static void test_zero_size_access(struct kunit *test) | |
977 | { | |
7310bd1f | 978 | struct expect_report expect = { |
1fe84fd4 ME |
979 | .access = { |
980 | { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE }, | |
981 | { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE }, | |
982 | }, | |
983 | }; | |
7310bd1f | 984 | struct expect_report never = { |
1fe84fd4 ME |
985 | .access = { |
986 | { test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE }, | |
987 | { test_kernel_read_struct_zero_size, &test_struct.val[3], 0, 0 }, | |
988 | }, | |
989 | }; | |
990 | bool match_expect = false; | |
991 | bool match_never = false; | |
992 | ||
993 | begin_test_checks(test_kernel_write_struct, test_kernel_read_struct_zero_size); | |
994 | do { | |
995 | match_expect |= report_matches(&expect); | |
996 | match_never = report_matches(&never); | |
997 | } while (!end_test_checks(match_never)); | |
998 | KUNIT_EXPECT_TRUE(test, match_expect); /* Sanity check. */ | |
999 | KUNIT_EXPECT_FALSE(test, match_never); | |
1000 | } | |
1001 | ||
1002 | /* Test the data_race() macro. */ | |
1003 | __no_kcsan | |
1004 | static void test_data_race(struct kunit *test) | |
1005 | { | |
1006 | bool match_never = false; | |
1007 | ||
1008 | begin_test_checks(test_kernel_data_race, test_kernel_data_race); | |
1009 | do { | |
1010 | match_never = report_available(); | |
1011 | } while (!end_test_checks(match_never)); | |
1012 | KUNIT_EXPECT_FALSE(test, match_never); | |
1013 | } | |
1014 | ||
31f605a3 ME |
1015 | /* Test the __data_racy type qualifier. */ |
1016 | __no_kcsan | |
1017 | static void test_data_racy_qualifier(struct kunit *test) | |
1018 | { | |
1019 | bool match_never = false; | |
1020 | ||
1021 | begin_test_checks(test_kernel_data_racy_qualifier, test_kernel_data_racy_qualifier); | |
1022 | do { | |
1023 | match_never = report_available(); | |
1024 | } while (!end_test_checks(match_never)); | |
1025 | KUNIT_EXPECT_FALSE(test, match_never); | |
1026 | } | |
1027 | ||
1fe84fd4 ME |
1028 | __no_kcsan |
1029 | static void test_assert_exclusive_writer(struct kunit *test) | |
1030 | { | |
7310bd1f | 1031 | struct expect_report expect = { |
1fe84fd4 ME |
1032 | .access = { |
1033 | { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT }, | |
1034 | { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, | |
1035 | }, | |
1036 | }; | |
1037 | bool match_expect = false; | |
1038 | ||
1039 | begin_test_checks(test_kernel_assert_writer, test_kernel_write_nochange); | |
1040 | do { | |
1041 | match_expect = report_matches(&expect); | |
1042 | } while (!end_test_checks(match_expect)); | |
1043 | KUNIT_EXPECT_TRUE(test, match_expect); | |
1044 | } | |
1045 | ||
1046 | __no_kcsan | |
1047 | static void test_assert_exclusive_access(struct kunit *test) | |
1048 | { | |
7310bd1f | 1049 | struct expect_report expect = { |
1fe84fd4 ME |
1050 | .access = { |
1051 | { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE }, | |
1052 | { test_kernel_read, &test_var, sizeof(test_var), 0 }, | |
1053 | }, | |
1054 | }; | |
1055 | bool match_expect = false; | |
1056 | ||
1057 | begin_test_checks(test_kernel_assert_access, test_kernel_read); | |
1058 | do { | |
1059 | match_expect = report_matches(&expect); | |
1060 | } while (!end_test_checks(match_expect)); | |
1061 | KUNIT_EXPECT_TRUE(test, match_expect); | |
1062 | } | |
1063 | ||
1064 | __no_kcsan | |
1065 | static void test_assert_exclusive_access_writer(struct kunit *test) | |
1066 | { | |
7310bd1f | 1067 | struct expect_report expect_access_writer = { |
1fe84fd4 ME |
1068 | .access = { |
1069 | { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE }, | |
1070 | { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT }, | |
1071 | }, | |
1072 | }; | |
7310bd1f | 1073 | struct expect_report expect_access_access = { |
1fe84fd4 ME |
1074 | .access = { |
1075 | { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE }, | |
1076 | { test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE }, | |
1077 | }, | |
1078 | }; | |
7310bd1f | 1079 | struct expect_report never = { |
1fe84fd4 ME |
1080 | .access = { |
1081 | { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT }, | |
1082 | { test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT }, | |
1083 | }, | |
1084 | }; | |
1085 | bool match_expect_access_writer = false; | |
1086 | bool match_expect_access_access = false; | |
1087 | bool match_never = false; | |
1088 | ||
1089 | begin_test_checks(test_kernel_assert_access, test_kernel_assert_writer); | |
1090 | do { | |
1091 | match_expect_access_writer |= report_matches(&expect_access_writer); | |
1092 | match_expect_access_access |= report_matches(&expect_access_access); | |
1093 | match_never |= report_matches(&never); | |
1094 | } while (!end_test_checks(match_never)); | |
1095 | KUNIT_EXPECT_TRUE(test, match_expect_access_writer); | |
1096 | KUNIT_EXPECT_TRUE(test, match_expect_access_access); | |
1097 | KUNIT_EXPECT_FALSE(test, match_never); | |
1098 | } | |
1099 | ||
1100 | __no_kcsan | |
1101 | static void test_assert_exclusive_bits_change(struct kunit *test) | |
1102 | { | |
7310bd1f | 1103 | struct expect_report expect = { |
1fe84fd4 ME |
1104 | .access = { |
1105 | { test_kernel_assert_bits_change, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT }, | |
1106 | { test_kernel_change_bits, &test_var, sizeof(test_var), | |
1107 | KCSAN_ACCESS_WRITE | (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS) ? 0 : KCSAN_ACCESS_ATOMIC) }, | |
1108 | }, | |
1109 | }; | |
1110 | bool match_expect = false; | |
1111 | ||
1112 | begin_test_checks(test_kernel_assert_bits_change, test_kernel_change_bits); | |
1113 | do { | |
1114 | match_expect = report_matches(&expect); | |
1115 | } while (!end_test_checks(match_expect)); | |
1116 | KUNIT_EXPECT_TRUE(test, match_expect); | |
1117 | } | |
1118 | ||
1119 | __no_kcsan | |
1120 | static void test_assert_exclusive_bits_nochange(struct kunit *test) | |
1121 | { | |
1122 | bool match_never = false; | |
1123 | ||
1124 | begin_test_checks(test_kernel_assert_bits_nochange, test_kernel_change_bits); | |
1125 | do { | |
1126 | match_never = report_available(); | |
1127 | } while (!end_test_checks(match_never)); | |
1128 | KUNIT_EXPECT_FALSE(test, match_never); | |
1129 | } | |
1130 | ||
1131 | __no_kcsan | |
1132 | static void test_assert_exclusive_writer_scoped(struct kunit *test) | |
1133 | { | |
7310bd1f | 1134 | struct expect_report expect_start = { |
1fe84fd4 ME |
1135 | .access = { |
1136 | { test_kernel_assert_writer_scoped, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_SCOPED }, | |
1137 | { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, | |
1138 | }, | |
1139 | }; | |
7310bd1f | 1140 | struct expect_report expect_inscope = { |
1fe84fd4 ME |
1141 | .access = { |
1142 | { test_enter_scope, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_SCOPED }, | |
1143 | { test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE }, | |
1144 | }, | |
1145 | }; | |
1146 | bool match_expect_start = false; | |
6c65eb75 | 1147 | bool match_expect_inscope = false; |
1fe84fd4 ME |
1148 | |
1149 | begin_test_checks(test_kernel_assert_writer_scoped, test_kernel_write_nochange); | |
1150 | do { | |
1151 | match_expect_start |= report_matches(&expect_start); | |
6c65eb75 ME |
1152 | match_expect_inscope |= report_matches(&expect_inscope); |
1153 | } while (!end_test_checks(match_expect_inscope)); | |
1fe84fd4 | 1154 | KUNIT_EXPECT_TRUE(test, match_expect_start); |
6c65eb75 | 1155 | KUNIT_EXPECT_FALSE(test, match_expect_inscope); |
1fe84fd4 ME |
1156 | } |
1157 | ||
1158 | __no_kcsan | |
1159 | static void test_assert_exclusive_access_scoped(struct kunit *test) | |
1160 | { | |
7310bd1f | 1161 | struct expect_report expect_start1 = { |
1fe84fd4 ME |
1162 | .access = { |
1163 | { test_kernel_assert_access_scoped, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_SCOPED }, | |
1164 | { test_kernel_read, &test_var, sizeof(test_var), 0 }, | |
1165 | }, | |
1166 | }; | |
7310bd1f | 1167 | struct expect_report expect_start2 = { |
1fe84fd4 ME |
1168 | .access = { expect_start1.access[0], expect_start1.access[0] }, |
1169 | }; | |
7310bd1f | 1170 | struct expect_report expect_inscope = { |
1fe84fd4 ME |
1171 | .access = { |
1172 | { test_enter_scope, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_SCOPED }, | |
1173 | { test_kernel_read, &test_var, sizeof(test_var), 0 }, | |
1174 | }, | |
1175 | }; | |
1176 | bool match_expect_start = false; | |
1177 | bool match_expect_inscope = false; | |
1178 | ||
1179 | begin_test_checks(test_kernel_assert_access_scoped, test_kernel_read); | |
1180 | end_time += msecs_to_jiffies(1000); /* This test requires a bit more time. */ | |
1181 | do { | |
1182 | match_expect_start |= report_matches(&expect_start1) || report_matches(&expect_start2); | |
1183 | match_expect_inscope |= report_matches(&expect_inscope); | |
6c65eb75 | 1184 | } while (!end_test_checks(match_expect_inscope)); |
1fe84fd4 | 1185 | KUNIT_EXPECT_TRUE(test, match_expect_start); |
6c65eb75 | 1186 | KUNIT_EXPECT_FALSE(test, match_expect_inscope); |
1fe84fd4 ME |
1187 | } |
1188 | ||
56b031f0 ME |
1189 | /* |
1190 | * jiffies is special (declared to be volatile) and its accesses are typically | |
1191 | * not marked; this test ensures that the compiler nor KCSAN gets confused about | |
1192 | * jiffies's declaration on different architectures. | |
1193 | */ | |
1194 | __no_kcsan | |
1195 | static void test_jiffies_noreport(struct kunit *test) | |
1196 | { | |
1197 | bool match_never = false; | |
1198 | ||
1199 | begin_test_checks(test_kernel_jiffies_reader, test_kernel_jiffies_reader); | |
1200 | do { | |
1201 | match_never = report_available(); | |
1202 | } while (!end_test_checks(match_never)); | |
1203 | KUNIT_EXPECT_FALSE(test, match_never); | |
1204 | } | |
1205 | ||
1fe84fd4 ME |
1206 | /* Test that racing accesses in seqlock critical sections are not reported. */ |
1207 | __no_kcsan | |
1208 | static void test_seqlock_noreport(struct kunit *test) | |
1209 | { | |
1210 | bool match_never = false; | |
1211 | ||
1212 | begin_test_checks(test_kernel_seqlock_reader, test_kernel_seqlock_writer); | |
1213 | do { | |
1214 | match_never = report_available(); | |
1215 | } while (!end_test_checks(match_never)); | |
1216 | KUNIT_EXPECT_FALSE(test, match_never); | |
1217 | } | |
1218 | ||
f9ea6319 ME |
1219 | /* |
1220 | * Test atomic builtins work and required instrumentation functions exist. We | |
1221 | * also test that KCSAN understands they're atomic by racing with them via | |
1222 | * test_kernel_atomic_builtins(), and expect no reports. | |
1223 | * | |
1224 | * The atomic builtins _SHOULD NOT_ be used in normal kernel code! | |
1225 | */ | |
1226 | static void test_atomic_builtins(struct kunit *test) | |
1227 | { | |
1228 | bool match_never = false; | |
1229 | ||
1230 | begin_test_checks(test_kernel_atomic_builtins, test_kernel_atomic_builtins); | |
1231 | do { | |
1232 | long tmp; | |
1233 | ||
1234 | kcsan_enable_current(); | |
1235 | ||
1236 | __atomic_store_n(&test_var, 42L, __ATOMIC_RELAXED); | |
1237 | KUNIT_EXPECT_EQ(test, 42L, __atomic_load_n(&test_var, __ATOMIC_RELAXED)); | |
1238 | ||
1239 | KUNIT_EXPECT_EQ(test, 42L, __atomic_exchange_n(&test_var, 20, __ATOMIC_RELAXED)); | |
1240 | KUNIT_EXPECT_EQ(test, 20L, test_var); | |
1241 | ||
1242 | tmp = 20L; | |
1243 | KUNIT_EXPECT_TRUE(test, __atomic_compare_exchange_n(&test_var, &tmp, 30L, | |
1244 | 0, __ATOMIC_RELAXED, | |
1245 | __ATOMIC_RELAXED)); | |
1246 | KUNIT_EXPECT_EQ(test, tmp, 20L); | |
1247 | KUNIT_EXPECT_EQ(test, test_var, 30L); | |
1248 | KUNIT_EXPECT_FALSE(test, __atomic_compare_exchange_n(&test_var, &tmp, 40L, | |
1249 | 1, __ATOMIC_RELAXED, | |
1250 | __ATOMIC_RELAXED)); | |
1251 | KUNIT_EXPECT_EQ(test, tmp, 30L); | |
1252 | KUNIT_EXPECT_EQ(test, test_var, 30L); | |
1253 | ||
1254 | KUNIT_EXPECT_EQ(test, 30L, __atomic_fetch_add(&test_var, 1, __ATOMIC_RELAXED)); | |
1255 | KUNIT_EXPECT_EQ(test, 31L, __atomic_fetch_sub(&test_var, 1, __ATOMIC_RELAXED)); | |
1256 | KUNIT_EXPECT_EQ(test, 30L, __atomic_fetch_and(&test_var, 0xf, __ATOMIC_RELAXED)); | |
1257 | KUNIT_EXPECT_EQ(test, 14L, __atomic_fetch_xor(&test_var, 0xf, __ATOMIC_RELAXED)); | |
1258 | KUNIT_EXPECT_EQ(test, 1L, __atomic_fetch_or(&test_var, 0xf0, __ATOMIC_RELAXED)); | |
1259 | KUNIT_EXPECT_EQ(test, 241L, __atomic_fetch_nand(&test_var, 0xf, __ATOMIC_RELAXED)); | |
1260 | KUNIT_EXPECT_EQ(test, -2L, test_var); | |
1261 | ||
1262 | __atomic_thread_fence(__ATOMIC_SEQ_CST); | |
1263 | __atomic_signal_fence(__ATOMIC_SEQ_CST); | |
1264 | ||
1265 | kcsan_disable_current(); | |
1266 | ||
1267 | match_never = report_available(); | |
1268 | } while (!end_test_checks(match_never)); | |
1269 | KUNIT_EXPECT_FALSE(test, match_never); | |
1270 | } | |
1271 | ||
d8fd74d3 ME |
1272 | __no_kcsan |
1273 | static void test_1bit_value_change(struct kunit *test) | |
1274 | { | |
7310bd1f | 1275 | struct expect_report expect = { |
d8fd74d3 ME |
1276 | .access = { |
1277 | { test_kernel_read, &test_var, sizeof(test_var), 0 }, | |
1278 | { test_kernel_xor_1bit, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) }, | |
1279 | }, | |
1280 | }; | |
1281 | bool match = false; | |
1282 | ||
1283 | begin_test_checks(test_kernel_read, test_kernel_xor_1bit); | |
1284 | do { | |
1285 | match = IS_ENABLED(CONFIG_KCSAN_PERMISSIVE) | |
1286 | ? report_available() | |
1287 | : report_matches(&expect); | |
1288 | } while (!end_test_checks(match)); | |
1289 | if (IS_ENABLED(CONFIG_KCSAN_PERMISSIVE)) | |
1290 | KUNIT_EXPECT_FALSE(test, match); | |
1291 | else | |
1292 | KUNIT_EXPECT_TRUE(test, match); | |
1293 | } | |
1294 | ||
8bc32b34 ME |
1295 | __no_kcsan |
1296 | static void test_correct_barrier(struct kunit *test) | |
1297 | { | |
1298 | struct expect_report expect = { | |
1299 | .access = { | |
1300 | { test_kernel_with_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) }, | |
1301 | { test_kernel_with_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(0) }, | |
1302 | }, | |
1303 | }; | |
1304 | bool match_expect = false; | |
1305 | ||
1306 | test_struct.val[0] = 0; /* init unlocked */ | |
1307 | begin_test_checks(test_kernel_with_memorder, test_kernel_with_memorder); | |
1308 | do { | |
1309 | match_expect = report_matches_any_reordered(&expect); | |
1310 | } while (!end_test_checks(match_expect)); | |
1311 | KUNIT_EXPECT_FALSE(test, match_expect); | |
1312 | } | |
1313 | ||
1314 | __no_kcsan | |
1315 | static void test_missing_barrier(struct kunit *test) | |
1316 | { | |
1317 | struct expect_report expect = { | |
1318 | .access = { | |
1319 | { test_kernel_wrong_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) }, | |
1320 | { test_kernel_wrong_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(0) }, | |
1321 | }, | |
1322 | }; | |
1323 | bool match_expect = false; | |
1324 | ||
1325 | test_struct.val[0] = 0; /* init unlocked */ | |
1326 | begin_test_checks(test_kernel_wrong_memorder, test_kernel_wrong_memorder); | |
1327 | do { | |
1328 | match_expect = report_matches_any_reordered(&expect); | |
1329 | } while (!end_test_checks(match_expect)); | |
1330 | if (IS_ENABLED(CONFIG_KCSAN_WEAK_MEMORY)) | |
1331 | KUNIT_EXPECT_TRUE(test, match_expect); | |
1332 | else | |
1333 | KUNIT_EXPECT_FALSE(test, match_expect); | |
1334 | } | |
1335 | ||
1336 | __no_kcsan | |
1337 | static void test_atomic_builtins_correct_barrier(struct kunit *test) | |
1338 | { | |
1339 | struct expect_report expect = { | |
1340 | .access = { | |
1341 | { test_kernel_atomic_builtin_with_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) }, | |
1342 | { test_kernel_atomic_builtin_with_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(0) }, | |
1343 | }, | |
1344 | }; | |
1345 | bool match_expect = false; | |
1346 | ||
1347 | test_struct.val[0] = 0; /* init unlocked */ | |
1348 | begin_test_checks(test_kernel_atomic_builtin_with_memorder, | |
1349 | test_kernel_atomic_builtin_with_memorder); | |
1350 | do { | |
1351 | match_expect = report_matches_any_reordered(&expect); | |
1352 | } while (!end_test_checks(match_expect)); | |
1353 | KUNIT_EXPECT_FALSE(test, match_expect); | |
1354 | } | |
1355 | ||
1356 | __no_kcsan | |
1357 | static void test_atomic_builtins_missing_barrier(struct kunit *test) | |
1358 | { | |
1359 | struct expect_report expect = { | |
1360 | .access = { | |
1361 | { test_kernel_atomic_builtin_wrong_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) }, | |
1362 | { test_kernel_atomic_builtin_wrong_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(0) }, | |
1363 | }, | |
1364 | }; | |
1365 | bool match_expect = false; | |
1366 | ||
1367 | test_struct.val[0] = 0; /* init unlocked */ | |
1368 | begin_test_checks(test_kernel_atomic_builtin_wrong_memorder, | |
1369 | test_kernel_atomic_builtin_wrong_memorder); | |
1370 | do { | |
1371 | match_expect = report_matches_any_reordered(&expect); | |
1372 | } while (!end_test_checks(match_expect)); | |
1373 | if (IS_ENABLED(CONFIG_KCSAN_WEAK_MEMORY)) | |
1374 | KUNIT_EXPECT_TRUE(test, match_expect); | |
1375 | else | |
1376 | KUNIT_EXPECT_FALSE(test, match_expect); | |
1377 | } | |
1378 | ||
1fe84fd4 | 1379 | /* |
f6a14914 ME |
1380 | * Generate thread counts for all test cases. Values generated are in interval |
1381 | * [2, 5] followed by exponentially increasing thread counts from 8 to 32. | |
1fe84fd4 ME |
1382 | * |
1383 | * The thread counts are chosen to cover potentially interesting boundaries and | |
f6a14914 | 1384 | * corner cases (2 to 5), and then stress the system with larger counts. |
1fe84fd4 | 1385 | */ |
f6a14914 ME |
1386 | static const void *nthreads_gen_params(const void *prev, char *desc) |
1387 | { | |
1388 | long nthreads = (long)prev; | |
1389 | ||
1390 | if (nthreads < 0 || nthreads >= 32) | |
1391 | nthreads = 0; /* stop */ | |
1392 | else if (!nthreads) | |
1393 | nthreads = 2; /* initial value */ | |
1394 | else if (nthreads < 5) | |
1395 | nthreads++; | |
1396 | else if (nthreads == 5) | |
1397 | nthreads = 8; | |
1398 | else | |
1399 | nthreads *= 2; | |
1fe84fd4 | 1400 | |
5693fa74 VS |
1401 | if (!preempt_model_preemptible() || |
1402 | !IS_ENABLED(CONFIG_KCSAN_INTERRUPT_WATCHER)) { | |
f6a14914 ME |
1403 | /* |
1404 | * Without any preemption, keep 2 CPUs free for other tasks, one | |
1405 | * of which is the main test case function checking for | |
1406 | * completion or failure. | |
1407 | */ | |
5693fa74 | 1408 | const long min_unused_cpus = preempt_model_none() ? 2 : 0; |
f6a14914 ME |
1409 | const long min_required_cpus = 2 + min_unused_cpus; |
1410 | ||
1411 | if (num_online_cpus() < min_required_cpus) { | |
f4abe996 | 1412 | pr_err_once("Too few online CPUs (%u < %ld) for test\n", |
f6a14914 ME |
1413 | num_online_cpus(), min_required_cpus); |
1414 | nthreads = 0; | |
1415 | } else if (nthreads >= num_online_cpus() - min_unused_cpus) { | |
1416 | /* Use negative value to indicate last param. */ | |
1417 | nthreads = -(num_online_cpus() - min_unused_cpus); | |
1418 | pr_warn_once("Limiting number of threads to %ld (only %d online CPUs)\n", | |
1419 | -nthreads, num_online_cpus()); | |
1420 | } | |
1421 | } | |
1422 | ||
1423 | snprintf(desc, KUNIT_PARAM_DESC_SIZE, "threads=%ld", abs(nthreads)); | |
1424 | return (void *)nthreads; | |
1425 | } | |
1426 | ||
1427 | #define KCSAN_KUNIT_CASE(test_name) KUNIT_CASE_PARAM(test_name, nthreads_gen_params) | |
1fe84fd4 | 1428 | static struct kunit_case kcsan_test_cases[] = { |
8bc32b34 | 1429 | KUNIT_CASE(test_barrier_nothreads), |
1fe84fd4 ME |
1430 | KCSAN_KUNIT_CASE(test_basic), |
1431 | KCSAN_KUNIT_CASE(test_concurrent_races), | |
1432 | KCSAN_KUNIT_CASE(test_novalue_change), | |
1433 | KCSAN_KUNIT_CASE(test_novalue_change_exception), | |
1434 | KCSAN_KUNIT_CASE(test_unknown_origin), | |
1435 | KCSAN_KUNIT_CASE(test_write_write_assume_atomic), | |
1436 | KCSAN_KUNIT_CASE(test_write_write_struct), | |
1437 | KCSAN_KUNIT_CASE(test_write_write_struct_part), | |
1438 | KCSAN_KUNIT_CASE(test_read_atomic_write_atomic), | |
1439 | KCSAN_KUNIT_CASE(test_read_plain_atomic_write), | |
bec4a247 | 1440 | KCSAN_KUNIT_CASE(test_read_plain_atomic_rmw), |
1fe84fd4 ME |
1441 | KCSAN_KUNIT_CASE(test_zero_size_access), |
1442 | KCSAN_KUNIT_CASE(test_data_race), | |
31f605a3 | 1443 | KCSAN_KUNIT_CASE(test_data_racy_qualifier), |
1fe84fd4 ME |
1444 | KCSAN_KUNIT_CASE(test_assert_exclusive_writer), |
1445 | KCSAN_KUNIT_CASE(test_assert_exclusive_access), | |
1446 | KCSAN_KUNIT_CASE(test_assert_exclusive_access_writer), | |
1447 | KCSAN_KUNIT_CASE(test_assert_exclusive_bits_change), | |
1448 | KCSAN_KUNIT_CASE(test_assert_exclusive_bits_nochange), | |
1449 | KCSAN_KUNIT_CASE(test_assert_exclusive_writer_scoped), | |
1450 | KCSAN_KUNIT_CASE(test_assert_exclusive_access_scoped), | |
56b031f0 | 1451 | KCSAN_KUNIT_CASE(test_jiffies_noreport), |
1fe84fd4 | 1452 | KCSAN_KUNIT_CASE(test_seqlock_noreport), |
f9ea6319 | 1453 | KCSAN_KUNIT_CASE(test_atomic_builtins), |
d8fd74d3 | 1454 | KCSAN_KUNIT_CASE(test_1bit_value_change), |
8bc32b34 ME |
1455 | KCSAN_KUNIT_CASE(test_correct_barrier), |
1456 | KCSAN_KUNIT_CASE(test_missing_barrier), | |
1457 | KCSAN_KUNIT_CASE(test_atomic_builtins_correct_barrier), | |
1458 | KCSAN_KUNIT_CASE(test_atomic_builtins_missing_barrier), | |
1fe84fd4 ME |
1459 | {}, |
1460 | }; | |
1461 | ||
1462 | /* ===== End test cases ===== */ | |
1463 | ||
1fe84fd4 ME |
1464 | /* Concurrent accesses from interrupts. */ |
1465 | __no_kcsan | |
1466 | static void access_thread_timer(struct timer_list *timer) | |
1467 | { | |
1468 | static atomic_t cnt = ATOMIC_INIT(0); | |
1469 | unsigned int idx; | |
1470 | void (*func)(void); | |
1471 | ||
1472 | idx = (unsigned int)atomic_inc_return(&cnt) % ARRAY_SIZE(access_kernels); | |
1473 | /* Acquire potential initialization. */ | |
1474 | func = smp_load_acquire(&access_kernels[idx]); | |
1475 | if (func) | |
1476 | func(); | |
1477 | } | |
1478 | ||
1479 | /* The main loop for each thread. */ | |
1480 | __no_kcsan | |
1481 | static int access_thread(void *arg) | |
1482 | { | |
1483 | struct timer_list timer; | |
1484 | unsigned int cnt = 0; | |
1485 | unsigned int idx; | |
1486 | void (*func)(void); | |
1487 | ||
1488 | timer_setup_on_stack(&timer, access_thread_timer, 0); | |
1489 | do { | |
1490 | might_sleep(); | |
1491 | ||
1492 | if (!timer_pending(&timer)) | |
1493 | mod_timer(&timer, jiffies + 1); | |
1494 | else { | |
1495 | /* Iterate through all kernels. */ | |
1496 | idx = cnt++ % ARRAY_SIZE(access_kernels); | |
1497 | /* Acquire potential initialization. */ | |
1498 | func = smp_load_acquire(&access_kernels[idx]); | |
1499 | if (func) | |
1500 | func(); | |
1501 | } | |
1502 | } while (!torture_must_stop()); | |
1503 | del_timer_sync(&timer); | |
1504 | destroy_timer_on_stack(&timer); | |
1505 | ||
1506 | torture_kthread_stopping("access_thread"); | |
1507 | return 0; | |
1508 | } | |
1509 | ||
1510 | __no_kcsan | |
1511 | static int test_init(struct kunit *test) | |
1512 | { | |
1513 | unsigned long flags; | |
1514 | int nthreads; | |
1515 | int i; | |
1516 | ||
1517 | spin_lock_irqsave(&observed.lock, flags); | |
1518 | for (i = 0; i < ARRAY_SIZE(observed.lines); ++i) | |
1519 | observed.lines[i][0] = '\0'; | |
1520 | observed.nlines = 0; | |
1521 | spin_unlock_irqrestore(&observed.lock, flags); | |
1522 | ||
8bc32b34 ME |
1523 | if (strstr(test->name, "nothreads")) |
1524 | return 0; | |
1525 | ||
1fe84fd4 ME |
1526 | if (!torture_init_begin((char *)test->name, 1)) |
1527 | return -EBUSY; | |
1528 | ||
1fe84fd4 ME |
1529 | if (WARN_ON(threads)) |
1530 | goto err; | |
1531 | ||
1532 | for (i = 0; i < ARRAY_SIZE(access_kernels); ++i) { | |
1533 | if (WARN_ON(access_kernels[i])) | |
1534 | goto err; | |
1535 | } | |
1536 | ||
f6a14914 ME |
1537 | nthreads = abs((long)test->param_value); |
1538 | if (WARN_ON(!nthreads)) | |
1539 | goto err; | |
1fe84fd4 | 1540 | |
f6a14914 ME |
1541 | threads = kcalloc(nthreads + 1, sizeof(struct task_struct *), GFP_KERNEL); |
1542 | if (WARN_ON(!threads)) | |
1543 | goto err; | |
1fe84fd4 | 1544 | |
f6a14914 ME |
1545 | threads[nthreads] = NULL; |
1546 | for (i = 0; i < nthreads; ++i) { | |
1547 | if (torture_create_kthread(access_thread, NULL, threads[i])) | |
1fe84fd4 | 1548 | goto err; |
1fe84fd4 ME |
1549 | } |
1550 | ||
1551 | torture_init_end(); | |
1552 | ||
1553 | return 0; | |
1554 | ||
1555 | err: | |
1556 | kfree(threads); | |
1557 | threads = NULL; | |
1558 | torture_init_end(); | |
1559 | return -EINVAL; | |
1560 | } | |
1561 | ||
1562 | __no_kcsan | |
1563 | static void test_exit(struct kunit *test) | |
1564 | { | |
1565 | struct task_struct **stop_thread; | |
1566 | int i; | |
1567 | ||
8bc32b34 ME |
1568 | if (strstr(test->name, "nothreads")) |
1569 | return; | |
1570 | ||
1fe84fd4 ME |
1571 | if (torture_cleanup_begin()) |
1572 | return; | |
1573 | ||
1574 | for (i = 0; i < ARRAY_SIZE(access_kernels); ++i) | |
1575 | WRITE_ONCE(access_kernels[i], NULL); | |
1576 | ||
1577 | if (threads) { | |
1578 | for (stop_thread = threads; *stop_thread; stop_thread++) | |
1579 | torture_stop_kthread(reader_thread, *stop_thread); | |
1580 | ||
1581 | kfree(threads); | |
1582 | threads = NULL; | |
1583 | } | |
1584 | ||
1585 | torture_cleanup_end(); | |
1586 | } | |
1587 | ||
1fe84fd4 | 1588 | __no_kcsan |
1f6ab566 | 1589 | static void register_tracepoints(void) |
1fe84fd4 | 1590 | { |
1f6ab566 | 1591 | register_trace_console(probe_console, NULL); |
1fe84fd4 ME |
1592 | } |
1593 | ||
1594 | __no_kcsan | |
1f6ab566 | 1595 | static void unregister_tracepoints(void) |
1fe84fd4 | 1596 | { |
1f6ab566 | 1597 | unregister_trace_console(probe_console, NULL); |
1fe84fd4 ME |
1598 | } |
1599 | ||
2434031c | 1600 | static int kcsan_suite_init(struct kunit_suite *suite) |
1fe84fd4 | 1601 | { |
1f6ab566 | 1602 | register_tracepoints(); |
2434031c | 1603 | return 0; |
1fe84fd4 ME |
1604 | } |
1605 | ||
2434031c | 1606 | static void kcsan_suite_exit(struct kunit_suite *suite) |
1fe84fd4 | 1607 | { |
1f6ab566 | 1608 | unregister_tracepoints(); |
1fe84fd4 ME |
1609 | tracepoint_synchronize_unregister(); |
1610 | } | |
1611 | ||
2434031c ME |
1612 | static struct kunit_suite kcsan_test_suite = { |
1613 | .name = "kcsan", | |
1614 | .test_cases = kcsan_test_cases, | |
1615 | .init = test_init, | |
1616 | .exit = test_exit, | |
1617 | .suite_init = kcsan_suite_init, | |
1618 | .suite_exit = kcsan_suite_exit, | |
1619 | }; | |
1620 | ||
1621 | kunit_test_suites(&kcsan_test_suite); | |
1fe84fd4 | 1622 | |
ddd7432d | 1623 | MODULE_DESCRIPTION("KCSAN test suite"); |
1fe84fd4 ME |
1624 | MODULE_LICENSE("GPL v2"); |
1625 | MODULE_AUTHOR("Marco Elver <elver@google.com>"); |