Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
40cbf09f AS |
2 | /* |
3 | * NOTE: | |
4 | * | |
5 | * This header has combined a lot of unrelated to each other stuff. | |
6 | * The process of splitting its content is in progress while keeping | |
7 | * backward compatibility. That's why it's highly recommended NOT to | |
8 | * include this header inside another header file, especially under | |
9 | * generic or architectural include/ directory. | |
10 | */ | |
1da177e4 LT |
11 | #ifndef _LINUX_KERNEL_H |
12 | #define _LINUX_KERNEL_H | |
13 | ||
c0891ac1 | 14 | #include <linux/stdarg.h> |
08c5188e | 15 | #include <linux/align.h> |
54d50897 | 16 | #include <linux/limits.h> |
1da177e4 LT |
17 | #include <linux/linkage.h> |
18 | #include <linux/stddef.h> | |
19 | #include <linux/types.h> | |
20 | #include <linux/compiler.h> | |
d2a8ebbf | 21 | #include <linux/container_of.h> |
1da177e4 | 22 | #include <linux/bitops.h> |
890a3ee3 | 23 | #include <linux/hex.h> |
4c527293 | 24 | #include <linux/kstrtox.h> |
f0d1b0b3 | 25 | #include <linux/log2.h> |
aa6159ab | 26 | #include <linux/math.h> |
b296a6d5 | 27 | #include <linux/minmax.h> |
e0deaff4 | 28 | #include <linux/typecheck.h> |
f39650de | 29 | #include <linux/panic.h> |
968ab183 | 30 | #include <linux/printk.h> |
c7acec71 | 31 | #include <linux/build_bug.h> |
39ced19b | 32 | #include <linux/sprintf.h> |
b965f1dd | 33 | #include <linux/static_call_types.h> |
e52340de | 34 | #include <linux/instruction_pointer.h> |
1da177e4 | 35 | #include <asm/byteorder.h> |
aa6159ab | 36 | |
607ca46e | 37 | #include <uapi/linux/kernel.h> |
1da177e4 | 38 | |
1da177e4 LT |
39 | #define STACK_MAGIC 0xdeadbeef |
40 | ||
e8c97af0 RD |
41 | /** |
42 | * REPEAT_BYTE - repeat the value @x multiple times as an unsigned long value | |
43 | * @x: value to repeat | |
44 | * | |
45 | * NOTE: @x is not checked for > 0xff; larger values produce odd results. | |
46 | */ | |
44696908 DM |
47 | #define REPEAT_BYTE(x) ((~0ul / 0xff) * (x)) |
48 | ||
d3849953 CH |
49 | /* generic data direction definitions */ |
50 | #define READ 0 | |
51 | #define WRITE 1 | |
52 | ||
e8c97af0 RD |
53 | /** |
54 | * ARRAY_SIZE - get the number of elements in array @arr | |
55 | * @arr: array to be sized | |
56 | */ | |
c5e631cf RR |
57 | #define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]) + __must_be_array(arr)) |
58 | ||
0ab1438b MY |
59 | #define PTR_IF(cond, ptr) ((cond) ? (ptr) : NULL) |
60 | ||
3ed605bc GP |
61 | #define u64_to_user_ptr(x) ( \ |
62 | { \ | |
a0fe2c64 JH |
63 | typecheck(u64, (x)); \ |
64 | (void __user *)(uintptr_t)(x); \ | |
3ed605bc GP |
65 | } \ |
66 | ) | |
67 | ||
218e180e AM |
68 | /** |
69 | * upper_32_bits - return bits 32-63 of a number | |
70 | * @n: the number we're accessing | |
71 | * | |
72 | * A basic shift-right of a 64- or 32-bit quantity. Use this to suppress | |
73 | * the "right shift count >= width of type" warning when that quantity is | |
74 | * 32-bits. | |
75 | */ | |
76 | #define upper_32_bits(n) ((u32)(((n) >> 16) >> 16)) | |
77 | ||
204b885e JR |
78 | /** |
79 | * lower_32_bits - return bits 0-31 of a number | |
80 | * @n: the number we're accessing | |
81 | */ | |
ef91bb19 | 82 | #define lower_32_bits(n) ((u32)((n) & 0xffffffff)) |
204b885e | 83 | |
03cb4473 JK |
84 | /** |
85 | * upper_16_bits - return bits 16-31 of a number | |
86 | * @n: the number we're accessing | |
87 | */ | |
88 | #define upper_16_bits(n) ((u16)((n) >> 16)) | |
89 | ||
90 | /** | |
91 | * lower_16_bits - return bits 0-15 of a number | |
92 | * @n: the number we're accessing | |
93 | */ | |
94 | #define lower_16_bits(n) ((u16)((n) & 0xffff)) | |
95 | ||
1da177e4 | 96 | struct completion; |
df2e71fb | 97 | struct user; |
1da177e4 | 98 | |
a8b76910 | 99 | #ifdef CONFIG_PREEMPT_VOLUNTARY_BUILD |
b965f1dd PZI |
100 | |
101 | extern int __cond_resched(void); | |
102 | # define might_resched() __cond_resched() | |
103 | ||
99cf983c | 104 | #elif defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) |
b965f1dd PZI |
105 | |
106 | extern int __cond_resched(void); | |
107 | ||
108 | DECLARE_STATIC_CALL(might_resched, __cond_resched); | |
109 | ||
110 | static __always_inline void might_resched(void) | |
111 | { | |
ef72661e | 112 | static_call_mod(might_resched)(); |
b965f1dd PZI |
113 | } |
114 | ||
99cf983c MR |
115 | #elif defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) |
116 | ||
117 | extern int dynamic_might_resched(void); | |
118 | # define might_resched() dynamic_might_resched() | |
119 | ||
070cb065 | 120 | #else |
b965f1dd | 121 | |
070cb065 | 122 | # define might_resched() do { } while (0) |
b965f1dd PZI |
123 | |
124 | #endif /* CONFIG_PREEMPT_* */ | |
070cb065 | 125 | |
d902db1e | 126 | #ifdef CONFIG_DEBUG_ATOMIC_SLEEP |
50e081b9 | 127 | extern void __might_resched(const char *file, int line, unsigned int offsets); |
42a38756 | 128 | extern void __might_sleep(const char *file, int line); |
568f1967 | 129 | extern void __cant_sleep(const char *file, int line, int preempt_offset); |
74d862b6 | 130 | extern void __cant_migrate(const char *file, int line); |
568f1967 | 131 | |
1da177e4 LT |
132 | /** |
133 | * might_sleep - annotation for functions that can sleep | |
134 | * | |
135 | * this macro will print a stack trace if it is executed in an atomic | |
312364f3 DV |
136 | * context (spinlock, irq-handler, ...). Additional sections where blocking is |
137 | * not allowed can be annotated with non_block_start() and non_block_end() | |
138 | * pairs. | |
1da177e4 LT |
139 | * |
140 | * This is a useful debugging help to be able to catch problems early and not | |
e20ec991 | 141 | * be bitten later when the calling function happens to sleep when it is not |
1da177e4 LT |
142 | * supposed to. |
143 | */ | |
f8cbd99b | 144 | # define might_sleep() \ |
42a38756 | 145 | do { __might_sleep(__FILE__, __LINE__); might_resched(); } while (0) |
568f1967 PZ |
146 | /** |
147 | * cant_sleep - annotation for functions that cannot sleep | |
148 | * | |
149 | * this macro will print a stack trace if it is executed with preemption enabled | |
150 | */ | |
151 | # define cant_sleep() \ | |
152 | do { __cant_sleep(__FILE__, __LINE__, 0); } while (0) | |
00845eb9 | 153 | # define sched_annotate_sleep() (current->task_state_change = 0) |
74d862b6 TG |
154 | |
155 | /** | |
156 | * cant_migrate - annotation for functions that cannot migrate | |
157 | * | |
158 | * Will print a stack trace if executed in code which is migratable | |
159 | */ | |
160 | # define cant_migrate() \ | |
161 | do { \ | |
162 | if (IS_ENABLED(CONFIG_SMP)) \ | |
163 | __cant_migrate(__FILE__, __LINE__); \ | |
164 | } while (0) | |
165 | ||
312364f3 DV |
166 | /** |
167 | * non_block_start - annotate the start of section where sleeping is prohibited | |
168 | * | |
169 | * This is on behalf of the oom reaper, specifically when it is calling the mmu | |
170 | * notifiers. The problem is that if the notifier were to block on, for example, | |
171 | * mutex_lock() and if the process which holds that mutex were to perform a | |
172 | * sleeping memory allocation, the oom reaper is now blocked on completion of | |
173 | * that memory allocation. Other blocking calls like wait_event() pose similar | |
174 | * issues. | |
175 | */ | |
176 | # define non_block_start() (current->non_block_count++) | |
177 | /** | |
178 | * non_block_end - annotate the end of section where sleeping is prohibited | |
179 | * | |
180 | * Closes a section opened by non_block_start(). | |
181 | */ | |
182 | # define non_block_end() WARN_ON(current->non_block_count-- == 0) | |
1da177e4 | 183 | #else |
874f670e | 184 | static inline void __might_resched(const char *file, int line, |
50e081b9 | 185 | unsigned int offsets) { } |
42a38756 | 186 | static inline void __might_sleep(const char *file, int line) { } |
f8cbd99b | 187 | # define might_sleep() do { might_resched(); } while (0) |
568f1967 | 188 | # define cant_sleep() do { } while (0) |
74d862b6 | 189 | # define cant_migrate() do { } while (0) |
1029a2b5 | 190 | # define sched_annotate_sleep() do { } while (0) |
312364f3 DV |
191 | # define non_block_start() do { } while (0) |
192 | # define non_block_end() do { } while (0) | |
1da177e4 LT |
193 | #endif |
194 | ||
368a5fa1 | 195 | #define might_sleep_if(cond) do { if (cond) might_sleep(); } while (0) |
f8cbd99b | 196 | |
386e7906 AL |
197 | #if defined(CONFIG_MMU) && \ |
198 | (defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP)) | |
9ec23531 DH |
199 | #define might_fault() __might_fault(__FILE__, __LINE__) |
200 | void __might_fault(const char *file, int line); | |
3ee1afa3 | 201 | #else |
662bbcb2 | 202 | static inline void might_fault(void) { } |
3ee1afa3 NP |
203 | #endif |
204 | ||
9af6528e | 205 | void do_exit(long error_code) __noreturn; |
33ee3b2e | 206 | |
1da177e4 LT |
207 | extern int get_option(char **str, int *pint); |
208 | extern char *get_options(const char *str, int nints, int *ints); | |
d974ae37 | 209 | extern unsigned long long memparse(const char *ptr, char **retptr); |
6ccc72b8 | 210 | extern bool parse_option_str(const char *str, const char *option); |
f51b17c8 | 211 | extern char *next_arg(char *args, char **param, char **val); |
1da177e4 | 212 | |
5e376613 | 213 | extern int core_kernel_text(unsigned long addr); |
1da177e4 LT |
214 | extern int __kernel_text_address(unsigned long addr); |
215 | extern int kernel_text_address(unsigned long addr); | |
ab7476cf | 216 | extern int func_ptr_is_kernel_text(void *ptr); |
ab7476cf | 217 | |
1da177e4 | 218 | extern void bust_spinlocks(int yes); |
5375b708 | 219 | |
b920de1b | 220 | extern int root_mountflags; |
1da177e4 | 221 | |
2ce802f6 TH |
222 | extern bool early_boot_irqs_disabled; |
223 | ||
69a78ff2 TG |
224 | /* |
225 | * Values used for system_state. Ordering of the states must not be changed | |
226 | * as code checks for <, <=, >, >= STATE. | |
227 | */ | |
1da177e4 LT |
228 | extern enum system_states { |
229 | SYSTEM_BOOTING, | |
69a78ff2 | 230 | SYSTEM_SCHEDULING, |
d2635f20 | 231 | SYSTEM_FREEING_INITMEM, |
1da177e4 LT |
232 | SYSTEM_RUNNING, |
233 | SYSTEM_HALT, | |
234 | SYSTEM_POWER_OFF, | |
235 | SYSTEM_RESTART, | |
c1a957d1 | 236 | SYSTEM_SUSPEND, |
1da177e4 LT |
237 | } system_state; |
238 | ||
526211bc IM |
239 | /* |
240 | * General tracing related utility functions - trace_printk(), | |
2002c258 SR |
241 | * tracing_on/tracing_off and tracing_start()/tracing_stop |
242 | * | |
243 | * Use tracing_on/tracing_off when you want to quickly turn on or off | |
244 | * tracing. It simply enables or disables the recording of the trace events. | |
2455f0e1 | 245 | * This also corresponds to the user space /sys/kernel/tracing/tracing_on |
2002c258 SR |
246 | * file, which gives a means for the kernel and userspace to interact. |
247 | * Place a tracing_off() in the kernel where you want tracing to end. | |
248 | * From user space, examine the trace, and then echo 1 > tracing_on | |
249 | * to continue tracing. | |
250 | * | |
251 | * tracing_stop/tracing_start has slightly more overhead. It is used | |
252 | * by things like suspend to ram where disabling the recording of the | |
253 | * trace is not enough, but tracing must actually stop because things | |
254 | * like calling smp_processor_id() may crash the system. | |
255 | * | |
256 | * Most likely, you want to use tracing_on/tracing_off. | |
526211bc | 257 | */ |
cecbca96 FW |
258 | |
259 | enum ftrace_dump_mode { | |
260 | DUMP_NONE, | |
261 | DUMP_ALL, | |
262 | DUMP_ORIG, | |
263 | }; | |
264 | ||
526211bc | 265 | #ifdef CONFIG_TRACING |
93d68e52 SR |
266 | void tracing_on(void); |
267 | void tracing_off(void); | |
268 | int tracing_is_on(void); | |
ad909e21 SRRH |
269 | void tracing_snapshot(void); |
270 | void tracing_snapshot_alloc(void); | |
93d68e52 | 271 | |
526211bc IM |
272 | extern void tracing_start(void); |
273 | extern void tracing_stop(void); | |
526211bc | 274 | |
b9075fa9 JP |
275 | static inline __printf(1, 2) |
276 | void ____trace_printk_check_format(const char *fmt, ...) | |
769b0441 FW |
277 | { |
278 | } | |
279 | #define __trace_printk_check_format(fmt, args...) \ | |
280 | do { \ | |
281 | if (0) \ | |
282 | ____trace_printk_check_format(fmt, ##args); \ | |
283 | } while (0) | |
284 | ||
526211bc IM |
285 | /** |
286 | * trace_printk - printf formatting in the ftrace buffer | |
287 | * @fmt: the printf format for printing | |
288 | * | |
e8c97af0 RD |
289 | * Note: __trace_printk is an internal function for trace_printk() and |
290 | * the @ip is passed in via the trace_printk() macro. | |
526211bc IM |
291 | * |
292 | * This function allows a kernel developer to debug fast path sections | |
293 | * that printk is not appropriate for. By scattering in various | |
294 | * printk like tracing in the code, a developer can quickly see | |
295 | * where problems are occurring. | |
296 | * | |
297 | * This is intended as a debugging tool for the developer only. | |
298 | * Please refrain from leaving trace_printks scattered around in | |
09ae7234 | 299 | * your code. (Extra memory is used for special buffers that are |
e8c97af0 | 300 | * allocated when trace_printk() is used.) |
9d3c752c | 301 | * |
8730662d | 302 | * A little optimization trick is done here. If there's only one |
9d3c752c SRRH |
303 | * argument, there's no need to scan the string for printf formats. |
304 | * The trace_puts() will suffice. But how can we take advantage of | |
305 | * using trace_puts() when trace_printk() has only one argument? | |
306 | * By stringifying the args and checking the size we can tell | |
307 | * whether or not there are args. __stringify((__VA_ARGS__)) will | |
308 | * turn into "()\0" with a size of 3 when there are no args, anything | |
309 | * else will be bigger. All we need to do is define a string to this, | |
310 | * and then take its size and compare to 3. If it's bigger, use | |
311 | * do_trace_printk() otherwise, optimize it to trace_puts(). Then just | |
312 | * let gcc optimize the rest. | |
526211bc | 313 | */ |
769b0441 | 314 | |
9d3c752c SRRH |
315 | #define trace_printk(fmt, ...) \ |
316 | do { \ | |
317 | char _______STR[] = __stringify((__VA_ARGS__)); \ | |
318 | if (sizeof(_______STR) > 3) \ | |
319 | do_trace_printk(fmt, ##__VA_ARGS__); \ | |
320 | else \ | |
321 | trace_puts(fmt); \ | |
322 | } while (0) | |
323 | ||
324 | #define do_trace_printk(fmt, args...) \ | |
769b0441 | 325 | do { \ |
3debb0a9 | 326 | static const char *trace_printk_fmt __used \ |
33def849 | 327 | __section("__trace_printk_fmt") = \ |
07d777fe SR |
328 | __builtin_constant_p(fmt) ? fmt : NULL; \ |
329 | \ | |
769b0441 | 330 | __trace_printk_check_format(fmt, ##args); \ |
48ead020 | 331 | \ |
07d777fe | 332 | if (__builtin_constant_p(fmt)) \ |
48ead020 | 333 | __trace_bprintk(_THIS_IP_, trace_printk_fmt, ##args); \ |
07d777fe SR |
334 | else \ |
335 | __trace_printk(_THIS_IP_, fmt, ##args); \ | |
769b0441 FW |
336 | } while (0) |
337 | ||
b9075fa9 JP |
338 | extern __printf(2, 3) |
339 | int __trace_bprintk(unsigned long ip, const char *fmt, ...); | |
48ead020 | 340 | |
b9075fa9 JP |
341 | extern __printf(2, 3) |
342 | int __trace_printk(unsigned long ip, const char *fmt, ...); | |
769b0441 | 343 | |
09ae7234 SRRH |
344 | /** |
345 | * trace_puts - write a string into the ftrace buffer | |
346 | * @str: the string to record | |
347 | * | |
348 | * Note: __trace_bputs is an internal function for trace_puts and | |
349 | * the @ip is passed in via the trace_puts macro. | |
350 | * | |
351 | * This is similar to trace_printk() but is made for those really fast | |
e8c97af0 | 352 | * paths that a developer wants the least amount of "Heisenbug" effects, |
09ae7234 SRRH |
353 | * where the processing of the print format is still too much. |
354 | * | |
355 | * This function allows a kernel developer to debug fast path sections | |
356 | * that printk is not appropriate for. By scattering in various | |
357 | * printk like tracing in the code, a developer can quickly see | |
358 | * where problems are occurring. | |
359 | * | |
360 | * This is intended as a debugging tool for the developer only. | |
361 | * Please refrain from leaving trace_puts scattered around in | |
362 | * your code. (Extra memory is used for special buffers that are | |
e8c97af0 | 363 | * allocated when trace_puts() is used.) |
09ae7234 SRRH |
364 | * |
365 | * Returns: 0 if nothing was written, positive # if string was. | |
366 | * (1 when __trace_bputs is used, strlen(str) when __trace_puts is used) | |
367 | */ | |
368 | ||
09ae7234 | 369 | #define trace_puts(str) ({ \ |
3debb0a9 | 370 | static const char *trace_printk_fmt __used \ |
33def849 | 371 | __section("__trace_printk_fmt") = \ |
09ae7234 SRRH |
372 | __builtin_constant_p(str) ? str : NULL; \ |
373 | \ | |
374 | if (__builtin_constant_p(str)) \ | |
375 | __trace_bputs(_THIS_IP_, trace_printk_fmt); \ | |
376 | else \ | |
377 | __trace_puts(_THIS_IP_, str, strlen(str)); \ | |
378 | }) | |
bcf312cf SR |
379 | extern int __trace_bputs(unsigned long ip, const char *str); |
380 | extern int __trace_puts(unsigned long ip, const char *str, int size); | |
09ae7234 | 381 | |
c142be8e | 382 | extern void trace_dump_stack(int skip); |
03889384 | 383 | |
48ead020 FW |
384 | /* |
385 | * The double __builtin_constant_p is because gcc will give us an error | |
386 | * if we try to allocate the static variable to fmt if it is not a | |
387 | * constant. Even with the outer if statement. | |
388 | */ | |
769b0441 FW |
389 | #define ftrace_vprintk(fmt, vargs) \ |
390 | do { \ | |
48ead020 | 391 | if (__builtin_constant_p(fmt)) { \ |
3debb0a9 | 392 | static const char *trace_printk_fmt __used \ |
33def849 | 393 | __section("__trace_printk_fmt") = \ |
48ead020 | 394 | __builtin_constant_p(fmt) ? fmt : NULL; \ |
7bffc23e | 395 | \ |
48ead020 FW |
396 | __ftrace_vbprintk(_THIS_IP_, trace_printk_fmt, vargs); \ |
397 | } else \ | |
398 | __ftrace_vprintk(_THIS_IP_, fmt, vargs); \ | |
769b0441 FW |
399 | } while (0) |
400 | ||
8db14860 | 401 | extern __printf(2, 0) int |
48ead020 FW |
402 | __ftrace_vbprintk(unsigned long ip, const char *fmt, va_list ap); |
403 | ||
8db14860 | 404 | extern __printf(2, 0) int |
526211bc | 405 | __ftrace_vprintk(unsigned long ip, const char *fmt, va_list ap); |
769b0441 | 406 | |
cecbca96 | 407 | extern void ftrace_dump(enum ftrace_dump_mode oops_dump_mode); |
526211bc | 408 | #else |
526211bc IM |
409 | static inline void tracing_start(void) { } |
410 | static inline void tracing_stop(void) { } | |
e67bc51e | 411 | static inline void trace_dump_stack(int skip) { } |
93d68e52 SR |
412 | |
413 | static inline void tracing_on(void) { } | |
414 | static inline void tracing_off(void) { } | |
415 | static inline int tracing_is_on(void) { return 0; } | |
ad909e21 SRRH |
416 | static inline void tracing_snapshot(void) { } |
417 | static inline void tracing_snapshot_alloc(void) { } | |
93d68e52 | 418 | |
60efc15a MH |
419 | static inline __printf(1, 2) |
420 | int trace_printk(const char *fmt, ...) | |
526211bc IM |
421 | { |
422 | return 0; | |
423 | } | |
8db14860 | 424 | static __printf(1, 0) inline int |
526211bc IM |
425 | ftrace_vprintk(const char *fmt, va_list ap) |
426 | { | |
427 | return 0; | |
428 | } | |
cecbca96 | 429 | static inline void ftrace_dump(enum ftrace_dump_mode oops_dump_mode) { } |
769b0441 | 430 | #endif /* CONFIG_TRACING */ |
526211bc | 431 | |
b9d4f426 AL |
432 | /* Rebuild everything on CONFIG_FTRACE_MCOUNT_RECORD */ |
433 | #ifdef CONFIG_FTRACE_MCOUNT_RECORD | |
434 | # define REBUILD_DUE_TO_FTRACE_MCOUNT_RECORD | |
435 | #endif | |
9d00f92f | 436 | |
58f86cc8 | 437 | /* Permissions on a sysfs file: you didn't miss the 0 prefix did you? */ |
28b8d0c8 GCM |
438 | #define VERIFY_OCTAL_PERMISSIONS(perms) \ |
439 | (BUILD_BUG_ON_ZERO((perms) < 0) + \ | |
440 | BUILD_BUG_ON_ZERO((perms) > 0777) + \ | |
441 | /* USER_READABLE >= GROUP_READABLE >= OTHER_READABLE */ \ | |
442 | BUILD_BUG_ON_ZERO((((perms) >> 6) & 4) < (((perms) >> 3) & 4)) + \ | |
443 | BUILD_BUG_ON_ZERO((((perms) >> 3) & 4) < ((perms) & 4)) + \ | |
444 | /* USER_WRITABLE >= GROUP_WRITABLE */ \ | |
445 | BUILD_BUG_ON_ZERO((((perms) >> 6) & 2) < (((perms) >> 3) & 2)) + \ | |
446 | /* OTHER_WRITABLE? Generally considered a bad idea. */ \ | |
447 | BUILD_BUG_ON_ZERO((perms) & 2) + \ | |
58f86cc8 | 448 | (perms)) |
1da177e4 | 449 | #endif |