Commit | Line | Data |
---|---|---|
5015a300 AP |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * Test cases for SL[AOU]B/page initialization at alloc/free time. | |
4 | */ | |
5 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | |
6 | ||
7 | #include <linux/init.h> | |
8 | #include <linux/kernel.h> | |
9 | #include <linux/mm.h> | |
10 | #include <linux/module.h> | |
11 | #include <linux/slab.h> | |
12 | #include <linux/string.h> | |
13 | #include <linux/vmalloc.h> | |
14 | ||
15 | #define GARBAGE_INT (0x09A7BA9E) | |
16 | #define GARBAGE_BYTE (0x9E) | |
17 | ||
18 | #define REPORT_FAILURES_IN_FN() \ | |
19 | do { \ | |
20 | if (failures) \ | |
21 | pr_info("%s failed %d out of %d times\n", \ | |
22 | __func__, failures, num_tests); \ | |
23 | else \ | |
24 | pr_info("all %d tests in %s passed\n", \ | |
25 | num_tests, __func__); \ | |
26 | } while (0) | |
27 | ||
28 | /* Calculate the number of uninitialized bytes in the buffer. */ | |
29 | static int __init count_nonzero_bytes(void *ptr, size_t size) | |
30 | { | |
31 | int i, ret = 0; | |
32 | unsigned char *p = (unsigned char *)ptr; | |
33 | ||
34 | for (i = 0; i < size; i++) | |
35 | if (p[i]) | |
36 | ret++; | |
37 | return ret; | |
38 | } | |
39 | ||
40 | /* Fill a buffer with garbage, skipping |skip| first bytes. */ | |
41 | static void __init fill_with_garbage_skip(void *ptr, size_t size, size_t skip) | |
42 | { | |
43 | unsigned int *p = (unsigned int *)ptr; | |
44 | int i = 0; | |
45 | ||
46 | if (skip) { | |
47 | WARN_ON(skip > size); | |
48 | p += skip; | |
49 | } | |
50 | while (size >= sizeof(*p)) { | |
51 | p[i] = GARBAGE_INT; | |
52 | i++; | |
53 | size -= sizeof(*p); | |
54 | } | |
55 | if (size) | |
56 | memset(&p[i], GARBAGE_BYTE, size); | |
57 | } | |
58 | ||
59 | static void __init fill_with_garbage(void *ptr, size_t size) | |
60 | { | |
61 | fill_with_garbage_skip(ptr, size, 0); | |
62 | } | |
63 | ||
64 | static int __init do_alloc_pages_order(int order, int *total_failures) | |
65 | { | |
66 | struct page *page; | |
67 | void *buf; | |
68 | size_t size = PAGE_SIZE << order; | |
69 | ||
70 | page = alloc_pages(GFP_KERNEL, order); | |
71 | buf = page_address(page); | |
72 | fill_with_garbage(buf, size); | |
73 | __free_pages(page, order); | |
74 | ||
75 | page = alloc_pages(GFP_KERNEL, order); | |
76 | buf = page_address(page); | |
77 | if (count_nonzero_bytes(buf, size)) | |
78 | (*total_failures)++; | |
79 | fill_with_garbage(buf, size); | |
80 | __free_pages(page, order); | |
81 | return 1; | |
82 | } | |
83 | ||
84 | /* Test the page allocator by calling alloc_pages with different orders. */ | |
85 | static int __init test_pages(int *total_failures) | |
86 | { | |
87 | int failures = 0, num_tests = 0; | |
88 | int i; | |
89 | ||
90 | for (i = 0; i < 10; i++) | |
91 | num_tests += do_alloc_pages_order(i, &failures); | |
92 | ||
93 | REPORT_FAILURES_IN_FN(); | |
94 | *total_failures += failures; | |
95 | return num_tests; | |
96 | } | |
97 | ||
98 | /* Test kmalloc() with given parameters. */ | |
99 | static int __init do_kmalloc_size(size_t size, int *total_failures) | |
100 | { | |
101 | void *buf; | |
102 | ||
103 | buf = kmalloc(size, GFP_KERNEL); | |
104 | fill_with_garbage(buf, size); | |
105 | kfree(buf); | |
106 | ||
107 | buf = kmalloc(size, GFP_KERNEL); | |
108 | if (count_nonzero_bytes(buf, size)) | |
109 | (*total_failures)++; | |
110 | fill_with_garbage(buf, size); | |
111 | kfree(buf); | |
112 | return 1; | |
113 | } | |
114 | ||
115 | /* Test vmalloc() with given parameters. */ | |
116 | static int __init do_vmalloc_size(size_t size, int *total_failures) | |
117 | { | |
118 | void *buf; | |
119 | ||
120 | buf = vmalloc(size); | |
121 | fill_with_garbage(buf, size); | |
122 | vfree(buf); | |
123 | ||
124 | buf = vmalloc(size); | |
125 | if (count_nonzero_bytes(buf, size)) | |
126 | (*total_failures)++; | |
127 | fill_with_garbage(buf, size); | |
128 | vfree(buf); | |
129 | return 1; | |
130 | } | |
131 | ||
132 | /* Test kmalloc()/vmalloc() by allocating objects of different sizes. */ | |
133 | static int __init test_kvmalloc(int *total_failures) | |
134 | { | |
135 | int failures = 0, num_tests = 0; | |
136 | int i, size; | |
137 | ||
138 | for (i = 0; i < 20; i++) { | |
139 | size = 1 << i; | |
140 | num_tests += do_kmalloc_size(size, &failures); | |
141 | num_tests += do_vmalloc_size(size, &failures); | |
142 | } | |
143 | ||
144 | REPORT_FAILURES_IN_FN(); | |
145 | *total_failures += failures; | |
146 | return num_tests; | |
147 | } | |
148 | ||
149 | #define CTOR_BYTES (sizeof(unsigned int)) | |
150 | #define CTOR_PATTERN (0x41414141) | |
151 | /* Initialize the first 4 bytes of the object. */ | |
152 | static void test_ctor(void *obj) | |
153 | { | |
154 | *(unsigned int *)obj = CTOR_PATTERN; | |
155 | } | |
156 | ||
157 | /* | |
158 | * Check the invariants for the buffer allocated from a slab cache. | |
159 | * If the cache has a test constructor, the first 4 bytes of the object must | |
160 | * always remain equal to CTOR_PATTERN. | |
161 | * If the cache isn't an RCU-typesafe one, or if the allocation is done with | |
162 | * __GFP_ZERO, then the object contents must be zeroed after allocation. | |
163 | * If the cache is an RCU-typesafe one, the object contents must never be | |
164 | * zeroed after the first use. This is checked by memcmp() in | |
165 | * do_kmem_cache_size(). | |
166 | */ | |
167 | static bool __init check_buf(void *buf, int size, bool want_ctor, | |
168 | bool want_rcu, bool want_zero) | |
169 | { | |
170 | int bytes; | |
171 | bool fail = false; | |
172 | ||
173 | bytes = count_nonzero_bytes(buf, size); | |
174 | WARN_ON(want_ctor && want_zero); | |
175 | if (want_zero) | |
176 | return bytes; | |
177 | if (want_ctor) { | |
178 | if (*(unsigned int *)buf != CTOR_PATTERN) | |
179 | fail = 1; | |
180 | } else { | |
181 | if (bytes) | |
182 | fail = !want_rcu; | |
183 | } | |
184 | return fail; | |
185 | } | |
186 | ||
187 | /* | |
188 | * Test kmem_cache with given parameters: | |
189 | * want_ctor - use a constructor; | |
190 | * want_rcu - use SLAB_TYPESAFE_BY_RCU; | |
191 | * want_zero - use __GFP_ZERO. | |
192 | */ | |
193 | static int __init do_kmem_cache_size(size_t size, bool want_ctor, | |
194 | bool want_rcu, bool want_zero, | |
195 | int *total_failures) | |
196 | { | |
197 | struct kmem_cache *c; | |
198 | int iter; | |
199 | bool fail = false; | |
200 | gfp_t alloc_mask = GFP_KERNEL | (want_zero ? __GFP_ZERO : 0); | |
201 | void *buf, *buf_copy; | |
202 | ||
203 | c = kmem_cache_create("test_cache", size, 1, | |
204 | want_rcu ? SLAB_TYPESAFE_BY_RCU : 0, | |
205 | want_ctor ? test_ctor : NULL); | |
206 | for (iter = 0; iter < 10; iter++) { | |
207 | buf = kmem_cache_alloc(c, alloc_mask); | |
208 | /* Check that buf is zeroed, if it must be. */ | |
209 | fail = check_buf(buf, size, want_ctor, want_rcu, want_zero); | |
210 | fill_with_garbage_skip(buf, size, want_ctor ? CTOR_BYTES : 0); | |
211 | /* | |
212 | * If this is an RCU cache, use a critical section to ensure we | |
213 | * can touch objects after they're freed. | |
214 | */ | |
215 | if (want_rcu) { | |
216 | rcu_read_lock(); | |
217 | /* | |
218 | * Copy the buffer to check that it's not wiped on | |
219 | * free(). | |
220 | */ | |
221 | buf_copy = kmalloc(size, GFP_KERNEL); | |
222 | if (buf_copy) | |
223 | memcpy(buf_copy, buf, size); | |
224 | } | |
225 | kmem_cache_free(c, buf); | |
226 | if (want_rcu) { | |
227 | /* | |
228 | * Check that |buf| is intact after kmem_cache_free(). | |
229 | * |want_zero| is false, because we wrote garbage to | |
230 | * the buffer already. | |
231 | */ | |
232 | fail |= check_buf(buf, size, want_ctor, want_rcu, | |
233 | false); | |
234 | if (buf_copy) { | |
235 | fail |= (bool)memcmp(buf, buf_copy, size); | |
236 | kfree(buf_copy); | |
237 | } | |
238 | rcu_read_unlock(); | |
239 | } | |
240 | } | |
241 | kmem_cache_destroy(c); | |
242 | ||
243 | *total_failures += fail; | |
244 | return 1; | |
245 | } | |
246 | ||
247 | /* | |
248 | * Check that the data written to an RCU-allocated object survives | |
249 | * reallocation. | |
250 | */ | |
251 | static int __init do_kmem_cache_rcu_persistent(int size, int *total_failures) | |
252 | { | |
253 | struct kmem_cache *c; | |
254 | void *buf, *buf_contents, *saved_ptr; | |
255 | void **used_objects; | |
256 | int i, iter, maxiter = 1024; | |
257 | bool fail = false; | |
258 | ||
259 | c = kmem_cache_create("test_cache", size, size, SLAB_TYPESAFE_BY_RCU, | |
260 | NULL); | |
261 | buf = kmem_cache_alloc(c, GFP_KERNEL); | |
262 | saved_ptr = buf; | |
263 | fill_with_garbage(buf, size); | |
264 | buf_contents = kmalloc(size, GFP_KERNEL); | |
265 | if (!buf_contents) | |
266 | goto out; | |
267 | used_objects = kmalloc_array(maxiter, sizeof(void *), GFP_KERNEL); | |
268 | if (!used_objects) { | |
269 | kfree(buf_contents); | |
270 | goto out; | |
271 | } | |
272 | memcpy(buf_contents, buf, size); | |
273 | kmem_cache_free(c, buf); | |
274 | /* | |
275 | * Run for a fixed number of iterations. If we never hit saved_ptr, | |
276 | * assume the test passes. | |
277 | */ | |
278 | for (iter = 0; iter < maxiter; iter++) { | |
279 | buf = kmem_cache_alloc(c, GFP_KERNEL); | |
280 | used_objects[iter] = buf; | |
281 | if (buf == saved_ptr) { | |
282 | fail = memcmp(buf_contents, buf, size); | |
283 | for (i = 0; i <= iter; i++) | |
284 | kmem_cache_free(c, used_objects[i]); | |
285 | goto free_out; | |
286 | } | |
287 | } | |
288 | ||
289 | free_out: | |
290 | kmem_cache_destroy(c); | |
291 | kfree(buf_contents); | |
292 | kfree(used_objects); | |
293 | out: | |
294 | *total_failures += fail; | |
295 | return 1; | |
296 | } | |
297 | ||
298 | /* | |
299 | * Test kmem_cache allocation by creating caches of different sizes, with and | |
300 | * without constructors, with and without SLAB_TYPESAFE_BY_RCU. | |
301 | */ | |
302 | static int __init test_kmemcache(int *total_failures) | |
303 | { | |
304 | int failures = 0, num_tests = 0; | |
305 | int i, flags, size; | |
306 | bool ctor, rcu, zero; | |
307 | ||
308 | for (i = 0; i < 10; i++) { | |
309 | size = 8 << i; | |
310 | for (flags = 0; flags < 8; flags++) { | |
311 | ctor = flags & 1; | |
312 | rcu = flags & 2; | |
313 | zero = flags & 4; | |
314 | if (ctor & zero) | |
315 | continue; | |
316 | num_tests += do_kmem_cache_size(size, ctor, rcu, zero, | |
317 | &failures); | |
318 | } | |
319 | } | |
320 | REPORT_FAILURES_IN_FN(); | |
321 | *total_failures += failures; | |
322 | return num_tests; | |
323 | } | |
324 | ||
325 | /* Test the behavior of SLAB_TYPESAFE_BY_RCU caches of different sizes. */ | |
326 | static int __init test_rcu_persistent(int *total_failures) | |
327 | { | |
328 | int failures = 0, num_tests = 0; | |
329 | int i, size; | |
330 | ||
331 | for (i = 0; i < 10; i++) { | |
332 | size = 8 << i; | |
333 | num_tests += do_kmem_cache_rcu_persistent(size, &failures); | |
334 | } | |
335 | REPORT_FAILURES_IN_FN(); | |
336 | *total_failures += failures; | |
337 | return num_tests; | |
338 | } | |
339 | ||
340 | /* | |
341 | * Run the tests. Each test function returns the number of executed tests and | |
342 | * updates |failures| with the number of failed tests. | |
343 | */ | |
344 | static int __init test_meminit_init(void) | |
345 | { | |
346 | int failures = 0, num_tests = 0; | |
347 | ||
348 | num_tests += test_pages(&failures); | |
349 | num_tests += test_kvmalloc(&failures); | |
350 | num_tests += test_kmemcache(&failures); | |
351 | num_tests += test_rcu_persistent(&failures); | |
352 | ||
353 | if (failures == 0) | |
354 | pr_info("all %d tests passed!\n", num_tests); | |
355 | else | |
356 | pr_info("failures: %d out of %d\n", failures, num_tests); | |
357 | ||
358 | return failures ? -EINVAL : 0; | |
359 | } | |
360 | module_init(test_meminit_init); | |
361 | ||
362 | MODULE_LICENSE("GPL"); |