Commit | Line | Data |
---|---|---|
b073d7f8 AP |
1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | /* | |
3 | * KMSAN API for subsystems. | |
4 | * | |
5 | * Copyright (C) 2017-2022 Google LLC | |
6 | * Author: Alexander Potapenko <glider@google.com> | |
7 | * | |
8 | */ | |
9 | #ifndef _LINUX_KMSAN_H | |
10 | #define _LINUX_KMSAN_H | |
11 | ||
7ade4f10 | 12 | #include <linux/dma-direction.h> |
b073d7f8 AP |
13 | #include <linux/gfp.h> |
14 | #include <linux/kmsan-checks.h> | |
15 | #include <linux/types.h> | |
16 | ||
17 | struct page; | |
68ef169a | 18 | struct kmem_cache; |
50b5e49c | 19 | struct task_struct; |
7ade4f10 | 20 | struct scatterlist; |
553a8018 | 21 | struct urb; |
b073d7f8 AP |
22 | |
23 | #ifdef CONFIG_KMSAN | |
24 | ||
50b5e49c AP |
25 | /** |
26 | * kmsan_task_create() - Initialize KMSAN state for the task. | |
27 | * @task: task to initialize. | |
28 | */ | |
29 | void kmsan_task_create(struct task_struct *task); | |
30 | ||
31 | /** | |
32 | * kmsan_task_exit() - Notify KMSAN that a task has exited. | |
33 | * @task: task about to finish. | |
34 | */ | |
35 | void kmsan_task_exit(struct task_struct *task); | |
36 | ||
3c206509 AP |
37 | /** |
38 | * kmsan_init_shadow() - Initialize KMSAN shadow at boot time. | |
39 | * | |
40 | * Allocate and initialize KMSAN metadata for early allocations. | |
41 | */ | |
42 | void __init kmsan_init_shadow(void); | |
43 | ||
44 | /** | |
45 | * kmsan_init_runtime() - Initialize KMSAN state and enable KMSAN. | |
46 | */ | |
47 | void __init kmsan_init_runtime(void); | |
48 | ||
49 | /** | |
50 | * kmsan_memblock_free_pages() - handle freeing of memblock pages. | |
51 | * @page: struct page to free. | |
52 | * @order: order of @page. | |
53 | * | |
54 | * Freed pages are either returned to buddy allocator or held back to be used | |
55 | * as metadata pages. | |
56 | */ | |
bb1508c2 AP |
57 | bool __init __must_check kmsan_memblock_free_pages(struct page *page, |
58 | unsigned int order); | |
3c206509 | 59 | |
b073d7f8 AP |
60 | /** |
61 | * kmsan_alloc_page() - Notify KMSAN about an alloc_pages() call. | |
62 | * @page: struct page pointer returned by alloc_pages(). | |
63 | * @order: order of allocated struct page. | |
64 | * @flags: GFP flags used by alloc_pages() | |
65 | * | |
66 | * KMSAN marks 1<<@order pages starting at @page as uninitialized, unless | |
67 | * @flags contain __GFP_ZERO. | |
68 | */ | |
69 | void kmsan_alloc_page(struct page *page, unsigned int order, gfp_t flags); | |
70 | ||
71 | /** | |
72 | * kmsan_free_page() - Notify KMSAN about a free_pages() call. | |
73 | * @page: struct page pointer passed to free_pages(). | |
74 | * @order: order of deallocated struct page. | |
75 | * | |
76 | * KMSAN marks freed memory as uninitialized. | |
77 | */ | |
78 | void kmsan_free_page(struct page *page, unsigned int order); | |
79 | ||
80 | /** | |
81 | * kmsan_copy_page_meta() - Copy KMSAN metadata between two pages. | |
82 | * @dst: destination page. | |
83 | * @src: source page. | |
84 | * | |
85 | * KMSAN copies the contents of metadata pages for @src into the metadata pages | |
86 | * for @dst. If @dst has no associated metadata pages, nothing happens. | |
87 | * If @src has no associated metadata pages, @dst metadata pages are unpoisoned. | |
88 | */ | |
89 | void kmsan_copy_page_meta(struct page *dst, struct page *src); | |
90 | ||
68ef169a AP |
91 | /** |
92 | * kmsan_slab_alloc() - Notify KMSAN about a slab allocation. | |
93 | * @s: slab cache the object belongs to. | |
94 | * @object: object pointer. | |
95 | * @flags: GFP flags passed to the allocator. | |
96 | * | |
97 | * Depending on cache flags and GFP flags, KMSAN sets up the metadata of the | |
98 | * newly created object, marking it as initialized or uninitialized. | |
99 | */ | |
100 | void kmsan_slab_alloc(struct kmem_cache *s, void *object, gfp_t flags); | |
101 | ||
102 | /** | |
103 | * kmsan_slab_free() - Notify KMSAN about a slab deallocation. | |
104 | * @s: slab cache the object belongs to. | |
105 | * @object: object pointer. | |
106 | * | |
107 | * KMSAN marks the freed object as uninitialized. | |
108 | */ | |
109 | void kmsan_slab_free(struct kmem_cache *s, void *object); | |
110 | ||
111 | /** | |
112 | * kmsan_kmalloc_large() - Notify KMSAN about a large slab allocation. | |
113 | * @ptr: object pointer. | |
114 | * @size: object size. | |
115 | * @flags: GFP flags passed to the allocator. | |
116 | * | |
117 | * Similar to kmsan_slab_alloc(), but for large allocations. | |
118 | */ | |
119 | void kmsan_kmalloc_large(const void *ptr, size_t size, gfp_t flags); | |
120 | ||
121 | /** | |
122 | * kmsan_kfree_large() - Notify KMSAN about a large slab deallocation. | |
123 | * @ptr: object pointer. | |
124 | * | |
125 | * Similar to kmsan_slab_free(), but for large allocations. | |
126 | */ | |
127 | void kmsan_kfree_large(const void *ptr); | |
128 | ||
b073d7f8 AP |
129 | /** |
130 | * kmsan_map_kernel_range_noflush() - Notify KMSAN about a vmap. | |
131 | * @start: start of vmapped range. | |
132 | * @end: end of vmapped range. | |
133 | * @prot: page protection flags used for vmap. | |
134 | * @pages: array of pages. | |
135 | * @page_shift: page_shift passed to vmap_range_noflush(). | |
136 | * | |
137 | * KMSAN maps shadow and origin pages of @pages into contiguous ranges in | |
47ebd031 AP |
138 | * vmalloc metadata address range. Returns 0 on success, callers must check |
139 | * for non-zero return value. | |
b073d7f8 | 140 | */ |
bb1508c2 AP |
141 | int __must_check kmsan_vmap_pages_range_noflush(unsigned long start, |
142 | unsigned long end, | |
143 | pgprot_t prot, | |
144 | struct page **pages, | |
145 | unsigned int page_shift); | |
b073d7f8 AP |
146 | |
147 | /** | |
148 | * kmsan_vunmap_kernel_range_noflush() - Notify KMSAN about a vunmap. | |
149 | * @start: start of vunmapped range. | |
150 | * @end: end of vunmapped range. | |
151 | * | |
152 | * KMSAN unmaps the contiguous metadata ranges created by | |
153 | * kmsan_map_kernel_range_noflush(). | |
154 | */ | |
155 | void kmsan_vunmap_range_noflush(unsigned long start, unsigned long end); | |
156 | ||
157 | /** | |
158 | * kmsan_ioremap_page_range() - Notify KMSAN about a ioremap_page_range() call. | |
159 | * @addr: range start. | |
160 | * @end: range end. | |
161 | * @phys_addr: physical range start. | |
162 | * @prot: page protection flags used for ioremap_page_range(). | |
163 | * @page_shift: page_shift argument passed to vmap_range_noflush(). | |
164 | * | |
165 | * KMSAN creates new metadata pages for the physical pages mapped into the | |
fdea03e1 AP |
166 | * virtual memory. Returns 0 on success, callers must check for non-zero return |
167 | * value. | |
b073d7f8 | 168 | */ |
bb1508c2 AP |
169 | int __must_check kmsan_ioremap_page_range(unsigned long addr, unsigned long end, |
170 | phys_addr_t phys_addr, pgprot_t prot, | |
171 | unsigned int page_shift); | |
b073d7f8 AP |
172 | |
173 | /** | |
174 | * kmsan_iounmap_page_range() - Notify KMSAN about a iounmap_page_range() call. | |
175 | * @start: range start. | |
176 | * @end: range end. | |
177 | * | |
178 | * KMSAN unmaps the metadata pages for the given range and, unlike for | |
179 | * vunmap_page_range(), also deallocates them. | |
180 | */ | |
181 | void kmsan_iounmap_page_range(unsigned long start, unsigned long end); | |
182 | ||
7ade4f10 AP |
183 | /** |
184 | * kmsan_handle_dma() - Handle a DMA data transfer. | |
185 | * @page: first page of the buffer. | |
186 | * @offset: offset of the buffer within the first page. | |
187 | * @size: buffer size. | |
188 | * @dir: one of possible dma_data_direction values. | |
189 | * | |
190 | * Depending on @direction, KMSAN: | |
191 | * * checks the buffer, if it is copied to device; | |
192 | * * initializes the buffer, if it is copied from device; | |
193 | * * does both, if this is a DMA_BIDIRECTIONAL transfer. | |
194 | */ | |
195 | void kmsan_handle_dma(struct page *page, size_t offset, size_t size, | |
196 | enum dma_data_direction dir); | |
197 | ||
198 | /** | |
199 | * kmsan_handle_dma_sg() - Handle a DMA transfer using scatterlist. | |
200 | * @sg: scatterlist holding DMA buffers. | |
201 | * @nents: number of scatterlist entries. | |
202 | * @dir: one of possible dma_data_direction values. | |
203 | * | |
204 | * Depending on @direction, KMSAN: | |
205 | * * checks the buffers in the scatterlist, if they are copied to device; | |
206 | * * initializes the buffers, if they are copied from device; | |
207 | * * does both, if this is a DMA_BIDIRECTIONAL transfer. | |
208 | */ | |
209 | void kmsan_handle_dma_sg(struct scatterlist *sg, int nents, | |
210 | enum dma_data_direction dir); | |
211 | ||
553a8018 AP |
212 | /** |
213 | * kmsan_handle_urb() - Handle a USB data transfer. | |
214 | * @urb: struct urb pointer. | |
215 | * @is_out: data transfer direction (true means output to hardware). | |
216 | * | |
217 | * If @is_out is true, KMSAN checks the transfer buffer of @urb. Otherwise, | |
218 | * KMSAN initializes the transfer buffer. | |
219 | */ | |
220 | void kmsan_handle_urb(const struct urb *urb, bool is_out); | |
221 | ||
6cae637f AP |
222 | /** |
223 | * kmsan_unpoison_entry_regs() - Handle pt_regs in low-level entry code. | |
224 | * @regs: struct pt_regs pointer received from assembly code. | |
225 | * | |
226 | * KMSAN unpoisons the contents of the passed pt_regs, preventing potential | |
227 | * false positive reports. Unlike kmsan_unpoison_memory(), | |
228 | * kmsan_unpoison_entry_regs() can be called from the regions where | |
229 | * kmsan_in_runtime() returns true, which is the case in early entry code. | |
230 | */ | |
231 | void kmsan_unpoison_entry_regs(const struct pt_regs *regs); | |
232 | ||
6b1709d4 IL |
233 | /** |
234 | * kmsan_get_metadata() - Return a pointer to KMSAN shadow or origins. | |
235 | * @addr: kernel address. | |
236 | * @is_origin: whether to return origins or shadow. | |
237 | * | |
238 | * Return NULL if metadata cannot be found. | |
239 | */ | |
240 | void *kmsan_get_metadata(void *addr, bool is_origin); | |
241 | ||
ec3e837d IL |
242 | /** |
243 | * kmsan_enable_current(): Enable KMSAN for the current task. | |
244 | * | |
245 | * Each kmsan_enable_current() current call must be preceded by a | |
246 | * kmsan_disable_current() call. These call pairs may be nested. | |
247 | */ | |
248 | void kmsan_enable_current(void); | |
249 | ||
250 | /** | |
251 | * kmsan_disable_current(): Disable KMSAN for the current task. | |
252 | * | |
253 | * Each kmsan_disable_current() current call must be followed by a | |
254 | * kmsan_enable_current() call. These call pairs may be nested. | |
255 | */ | |
256 | void kmsan_disable_current(void); | |
257 | ||
1fdb3c70 IL |
258 | /** |
259 | * memset_no_sanitize_memory(): Fill memory without KMSAN instrumentation. | |
260 | * @s: address of kernel memory to fill. | |
261 | * @c: constant byte to fill the memory with. | |
262 | * @n: number of bytes to fill. | |
263 | * | |
264 | * This is like memset(), but without KMSAN instrumentation. | |
265 | */ | |
266 | static inline void *memset_no_sanitize_memory(void *s, int c, size_t n) | |
267 | { | |
268 | return __memset(s, c, n); | |
269 | } | |
270 | ||
e6553e2f IL |
271 | extern bool kmsan_enabled; |
272 | extern int panic_on_kmsan; | |
273 | ||
274 | /* | |
275 | * KMSAN performs a lot of consistency checks that are currently enabled by | |
276 | * default. BUG_ON is normally discouraged in the kernel, unless used for | |
277 | * debugging, but KMSAN itself is a debugging tool, so it makes little sense to | |
278 | * recover if something goes wrong. | |
279 | */ | |
280 | #define KMSAN_WARN_ON(cond) \ | |
281 | ({ \ | |
282 | const bool __cond = WARN_ON(cond); \ | |
283 | if (unlikely(__cond)) { \ | |
284 | WRITE_ONCE(kmsan_enabled, false); \ | |
285 | if (panic_on_kmsan) { \ | |
286 | /* Can't call panic() here because */ \ | |
287 | /* of uaccess checks. */ \ | |
288 | BUG(); \ | |
289 | } \ | |
290 | } \ | |
291 | __cond; \ | |
292 | }) | |
293 | ||
b073d7f8 AP |
294 | #else |
295 | ||
3c206509 AP |
296 | static inline void kmsan_init_shadow(void) |
297 | { | |
298 | } | |
299 | ||
300 | static inline void kmsan_init_runtime(void) | |
301 | { | |
302 | } | |
303 | ||
bb1508c2 AP |
304 | static inline bool __must_check kmsan_memblock_free_pages(struct page *page, |
305 | unsigned int order) | |
3c206509 AP |
306 | { |
307 | return true; | |
308 | } | |
309 | ||
50b5e49c AP |
310 | static inline void kmsan_task_create(struct task_struct *task) |
311 | { | |
312 | } | |
313 | ||
314 | static inline void kmsan_task_exit(struct task_struct *task) | |
315 | { | |
316 | } | |
317 | ||
bb1508c2 AP |
318 | static inline void kmsan_alloc_page(struct page *page, unsigned int order, |
319 | gfp_t flags) | |
b073d7f8 | 320 | { |
b073d7f8 AP |
321 | } |
322 | ||
323 | static inline void kmsan_free_page(struct page *page, unsigned int order) | |
324 | { | |
325 | } | |
326 | ||
327 | static inline void kmsan_copy_page_meta(struct page *dst, struct page *src) | |
328 | { | |
329 | } | |
330 | ||
68ef169a AP |
331 | static inline void kmsan_slab_alloc(struct kmem_cache *s, void *object, |
332 | gfp_t flags) | |
333 | { | |
334 | } | |
335 | ||
336 | static inline void kmsan_slab_free(struct kmem_cache *s, void *object) | |
337 | { | |
338 | } | |
339 | ||
340 | static inline void kmsan_kmalloc_large(const void *ptr, size_t size, | |
341 | gfp_t flags) | |
342 | { | |
343 | } | |
344 | ||
345 | static inline void kmsan_kfree_large(const void *ptr) | |
346 | { | |
347 | } | |
348 | ||
bb1508c2 AP |
349 | static inline int __must_check kmsan_vmap_pages_range_noflush( |
350 | unsigned long start, unsigned long end, pgprot_t prot, | |
351 | struct page **pages, unsigned int page_shift) | |
b073d7f8 | 352 | { |
47ebd031 | 353 | return 0; |
b073d7f8 AP |
354 | } |
355 | ||
356 | static inline void kmsan_vunmap_range_noflush(unsigned long start, | |
357 | unsigned long end) | |
358 | { | |
359 | } | |
360 | ||
bb1508c2 AP |
361 | static inline int __must_check kmsan_ioremap_page_range(unsigned long start, |
362 | unsigned long end, | |
363 | phys_addr_t phys_addr, | |
364 | pgprot_t prot, | |
365 | unsigned int page_shift) | |
b073d7f8 | 366 | { |
fdea03e1 | 367 | return 0; |
b073d7f8 AP |
368 | } |
369 | ||
370 | static inline void kmsan_iounmap_page_range(unsigned long start, | |
371 | unsigned long end) | |
372 | { | |
373 | } | |
374 | ||
7ade4f10 AP |
375 | static inline void kmsan_handle_dma(struct page *page, size_t offset, |
376 | size_t size, enum dma_data_direction dir) | |
377 | { | |
378 | } | |
379 | ||
380 | static inline void kmsan_handle_dma_sg(struct scatterlist *sg, int nents, | |
381 | enum dma_data_direction dir) | |
382 | { | |
383 | } | |
384 | ||
553a8018 AP |
385 | static inline void kmsan_handle_urb(const struct urb *urb, bool is_out) |
386 | { | |
387 | } | |
388 | ||
6cae637f AP |
389 | static inline void kmsan_unpoison_entry_regs(const struct pt_regs *regs) |
390 | { | |
391 | } | |
392 | ||
ec3e837d IL |
393 | static inline void kmsan_enable_current(void) |
394 | { | |
395 | } | |
396 | ||
397 | static inline void kmsan_disable_current(void) | |
398 | { | |
399 | } | |
400 | ||
1fdb3c70 IL |
401 | static inline void *memset_no_sanitize_memory(void *s, int c, size_t n) |
402 | { | |
403 | return memset(s, c, n); | |
404 | } | |
405 | ||
e6553e2f IL |
406 | #define KMSAN_WARN_ON WARN_ON |
407 | ||
b073d7f8 AP |
408 | #endif |
409 | ||
410 | #endif /* _LINUX_KMSAN_H */ |