Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
1da177e4 LT |
2 | #ifndef LINUX_HARDIRQ_H |
3 | #define LINUX_HARDIRQ_H | |
4 | ||
aaf2bc50 | 5 | #include <linux/context_tracking_state.h> |
92cf2118 | 6 | #include <linux/preempt.h> |
fbb9ce95 | 7 | #include <linux/lockdep.h> |
6a60dd12 | 8 | #include <linux/ftrace_irq.h> |
728b478d | 9 | #include <linux/sched.h> |
dcbf832e | 10 | #include <linux/vtime.h> |
0bd3a173 | 11 | #include <asm/hardirq.h> |
1da177e4 | 12 | |
1da177e4 | 13 | extern void synchronize_irq(unsigned int irq); |
02cea395 | 14 | extern bool synchronize_hardirq(unsigned int irq); |
1da177e4 | 15 | |
aaf2bc50 PM |
16 | #ifdef CONFIG_NO_HZ_FULL |
17 | void __rcu_irq_enter_check_tick(void); | |
18 | #else | |
19 | static inline void __rcu_irq_enter_check_tick(void) { } | |
20 | #endif | |
9b1d82fa | 21 | |
aaf2bc50 | 22 | static __always_inline void rcu_irq_enter_check_tick(void) |
9b1d82fa | 23 | { |
aaf2bc50 PM |
24 | if (context_tracking_enabled()) |
25 | __rcu_irq_enter_check_tick(); | |
9b1d82fa PM |
26 | } |
27 | ||
de30a2b3 IM |
28 | /* |
29 | * It is safe to do non-atomic ops on ->hardirq_context, | |
30 | * because NMI handlers may not preempt and the ops are | |
31 | * always balanced, so the interrupted value of ->hardirq_context | |
32 | * will always be restored. | |
33 | */ | |
79bf2bb3 TG |
34 | #define __irq_enter() \ |
35 | do { \ | |
bdb43806 | 36 | preempt_count_add(HARDIRQ_OFFSET); \ |
2502ec37 | 37 | lockdep_hardirq_enter(); \ |
d3759e71 | 38 | account_hardirq_enter(current); \ |
79bf2bb3 TG |
39 | } while (0) |
40 | ||
98a3bf19 TG |
41 | /* |
42 | * Like __irq_enter() without time accounting for fast | |
43 | * interrupts, e.g. reschedule IPI where time accounting | |
44 | * is more expensive than the actual interrupt. | |
45 | */ | |
46 | #define __irq_enter_raw() \ | |
47 | do { \ | |
48 | preempt_count_add(HARDIRQ_OFFSET); \ | |
49 | lockdep_hardirq_enter(); \ | |
50 | } while (0) | |
51 | ||
79bf2bb3 TG |
52 | /* |
53 | * Enter irq context (on NO_HZ, update jiffies): | |
54 | */ | |
8a6bc478 TG |
55 | void irq_enter(void); |
56 | /* | |
57 | * Like irq_enter(), but RCU is already watching. | |
58 | */ | |
59 | void irq_enter_rcu(void); | |
de30a2b3 IM |
60 | |
61 | /* | |
62 | * Exit irq context without processing softirqs: | |
63 | */ | |
64 | #define __irq_exit() \ | |
65 | do { \ | |
d3759e71 | 66 | account_hardirq_exit(current); \ |
2502ec37 | 67 | lockdep_hardirq_exit(); \ |
bdb43806 | 68 | preempt_count_sub(HARDIRQ_OFFSET); \ |
1da177e4 LT |
69 | } while (0) |
70 | ||
98a3bf19 TG |
71 | /* |
72 | * Like __irq_exit() without time accounting | |
73 | */ | |
74 | #define __irq_exit_raw() \ | |
75 | do { \ | |
76 | lockdep_hardirq_exit(); \ | |
77 | preempt_count_sub(HARDIRQ_OFFSET); \ | |
78 | } while (0) | |
79 | ||
de30a2b3 IM |
80 | /* |
81 | * Exit irq context and process softirqs if needed: | |
82 | */ | |
8a6bc478 TG |
83 | void irq_exit(void); |
84 | ||
85 | /* | |
86 | * Like irq_exit(), but return with RCU watching. | |
87 | */ | |
88 | void irq_exit_rcu(void); | |
1da177e4 | 89 | |
5870970b JT |
90 | #ifndef arch_nmi_enter |
91 | #define arch_nmi_enter() do { } while (0) | |
92 | #define arch_nmi_exit() do { } while (0) | |
93 | #endif | |
94 | ||
f93524eb PZ |
95 | /* |
96 | * NMI vs Tracing | |
97 | * -------------- | |
98 | * | |
99 | * We must not land in a tracer until (or after) we've changed preempt_count | |
100 | * such that in_nmi() becomes true. To that effect all NMI C entry points must | |
101 | * be marked 'notrace' and call nmi_enter() as soon as possible. | |
102 | */ | |
103 | ||
69ea03b5 PZ |
104 | /* |
105 | * nmi_enter() can nest up to 15 times; see NMI_BITS. | |
106 | */ | |
ba1f2b2e | 107 | #define __nmi_enter() \ |
2a7b8df0 | 108 | do { \ |
ba1f2b2e | 109 | lockdep_off(); \ |
5870970b | 110 | arch_nmi_enter(); \ |
69ea03b5 | 111 | BUG_ON(in_nmi() == NMI_MASK); \ |
f93524eb | 112 | __preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET); \ |
ba1f2b2e PZ |
113 | } while (0) |
114 | ||
115 | #define nmi_enter() \ | |
116 | do { \ | |
117 | __nmi_enter(); \ | |
2502ec37 | 118 | lockdep_hardirq_enter(); \ |
493c1822 | 119 | ct_nmi_enter(); \ |
2ab70319 TG |
120 | instrumentation_begin(); \ |
121 | ftrace_nmi_enter(); \ | |
122 | instrumentation_end(); \ | |
17666f02 | 123 | } while (0) |
5f34fe1c | 124 | |
ba1f2b2e PZ |
125 | #define __nmi_exit() \ |
126 | do { \ | |
127 | BUG_ON(!in_nmi()); \ | |
128 | __preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); \ | |
ba1f2b2e PZ |
129 | arch_nmi_exit(); \ |
130 | lockdep_on(); \ | |
131 | } while (0) | |
132 | ||
2a7b8df0 SR |
133 | #define nmi_exit() \ |
134 | do { \ | |
2ab70319 TG |
135 | instrumentation_begin(); \ |
136 | ftrace_nmi_exit(); \ | |
137 | instrumentation_end(); \ | |
493c1822 | 138 | ct_nmi_exit(); \ |
ba1f2b2e PZ |
139 | lockdep_hardirq_exit(); \ |
140 | __nmi_exit(); \ | |
17666f02 | 141 | } while (0) |
de30a2b3 | 142 | |
1da177e4 | 143 | #endif /* LINUX_HARDIRQ_H */ |