Commit | Line | Data |
---|---|---|
c942fddf | 1 | /* SPDX-License-Identifier: GPL-2.0-or-later */ |
ca74e92b SN |
2 | /* delayacct.h - per-task delay accounting |
3 | * | |
4 | * Copyright (C) Shailabh Nagar, IBM Corp. 2006 | |
ca74e92b SN |
5 | */ |
6 | ||
7 | #ifndef _LINUX_DELAYACCT_H | |
8 | #define _LINUX_DELAYACCT_H | |
9 | ||
4025819d | 10 | #include <uapi/linux/taskstats.h> |
ca74e92b SN |
11 | |
12 | #ifdef CONFIG_TASK_DELAY_ACCT | |
47913d4e | 13 | struct task_delay_info { |
02acc80d | 14 | raw_spinlock_t lock; |
ca74e92b | 15 | |
47913d4e IM |
16 | /* For each stat XXX, add following, aligned appropriately |
17 | * | |
18 | * struct timespec XXX_start, XXX_end; | |
19 | * u64 XXX_delay; | |
20 | * u32 XXX_count; | |
21 | * | |
22 | * Atomicity of updates to XXX_delay, XXX_count protected by | |
23 | * single lock above (split into XXX_lock if contention is an issue). | |
24 | */ | |
25 | ||
26 | /* | |
27 | * XXX_count is incremented on every XXX operation, the delay | |
28 | * associated with the operation is added to XXX_delay. | |
29 | * XXX_delay contains the accumulated delay time in nanoseconds. | |
30 | */ | |
a3d5dc90 | 31 | u64 blkio_start; |
47913d4e | 32 | u64 blkio_delay; /* wait for sync block io completion */ |
a3d5dc90 YY |
33 | u64 swapin_start; |
34 | u64 swapin_delay; /* wait for swapin */ | |
47913d4e IM |
35 | u32 blkio_count; /* total count of the number of sync block */ |
36 | /* io operations performed */ | |
a3d5dc90 | 37 | u32 swapin_count; /* total count of swapin */ |
47913d4e IM |
38 | |
39 | u64 freepages_start; | |
40 | u64 freepages_delay; /* wait for memory reclaim */ | |
b1d29ba8 JW |
41 | |
42 | u64 thrashing_start; | |
43 | u64 thrashing_delay; /* wait for thrashing page */ | |
44 | ||
5bf18281 | 45 | u64 compact_start; |
46 | u64 compact_delay; /* wait for memory compact */ | |
47 | ||
662ce1dc YY |
48 | u64 wpcopy_start; |
49 | u64 wpcopy_delay; /* wait for write-protect copy */ | |
50 | ||
a3b2aeac YY |
51 | u64 irq_delay; /* wait for IRQ/SOFTIRQ */ |
52 | ||
47913d4e | 53 | u32 freepages_count; /* total count of memory reclaim */ |
b1d29ba8 | 54 | u32 thrashing_count; /* total count of thrash waits */ |
5bf18281 | 55 | u32 compact_count; /* total count of memory compact */ |
662ce1dc | 56 | u32 wpcopy_count; /* total count of write-protect copy */ |
a3b2aeac | 57 | u32 irq_count; /* total count of IRQ/SOFTIRQ */ |
47913d4e IM |
58 | }; |
59 | #endif | |
60 | ||
61 | #include <linux/sched.h> | |
62 | #include <linux/slab.h> | |
eee4d9fe | 63 | #include <linux/jump_label.h> |
47913d4e IM |
64 | |
65 | #ifdef CONFIG_TASK_DELAY_ACCT | |
e4042ad4 | 66 | DECLARE_STATIC_KEY_FALSE(delayacct_key); |
ca74e92b | 67 | extern int delayacct_on; /* Delay accounting turned on/off */ |
e18b890b | 68 | extern struct kmem_cache *delayacct_cache; |
ca74e92b | 69 | extern void delayacct_init(void); |
0cd7c741 | 70 | |
ca74e92b SN |
71 | extern void __delayacct_tsk_init(struct task_struct *); |
72 | extern void __delayacct_tsk_exit(struct task_struct *); | |
0ff92245 | 73 | extern void __delayacct_blkio_start(void); |
c96f5471 | 74 | extern void __delayacct_blkio_end(struct task_struct *); |
e4042ad4 | 75 | extern int delayacct_add_tsk(struct taskstats *, struct task_struct *); |
25890454 | 76 | extern __u64 __delayacct_blkio_ticks(struct task_struct *); |
873b4771 KK |
77 | extern void __delayacct_freepages_start(void); |
78 | extern void __delayacct_freepages_end(void); | |
aa1cf99b YY |
79 | extern void __delayacct_thrashing_start(bool *in_thrashing); |
80 | extern void __delayacct_thrashing_end(bool *in_thrashing); | |
a3d5dc90 YY |
81 | extern void __delayacct_swapin_start(void); |
82 | extern void __delayacct_swapin_end(void); | |
5bf18281 | 83 | extern void __delayacct_compact_start(void); |
84 | extern void __delayacct_compact_end(void); | |
662ce1dc YY |
85 | extern void __delayacct_wpcopy_start(void); |
86 | extern void __delayacct_wpcopy_end(void); | |
a3b2aeac | 87 | extern void __delayacct_irq(struct task_struct *task, u32 delta); |
846c7bb0 | 88 | |
ca74e92b SN |
89 | static inline void delayacct_tsk_init(struct task_struct *tsk) |
90 | { | |
91 | /* reinitialize in case parent's non-null pointer was dup'ed*/ | |
92 | tsk->delays = NULL; | |
163ecdff | 93 | if (delayacct_on) |
ca74e92b SN |
94 | __delayacct_tsk_init(tsk); |
95 | } | |
96 | ||
35df17c5 SN |
97 | /* Free tsk->delays. Called from bad fork and __put_task_struct |
98 | * where there's no risk of tsk->delays being accessed elsewhere | |
99 | */ | |
100 | static inline void delayacct_tsk_free(struct task_struct *tsk) | |
ca74e92b SN |
101 | { |
102 | if (tsk->delays) | |
35df17c5 SN |
103 | kmem_cache_free(delayacct_cache, tsk->delays); |
104 | tsk->delays = NULL; | |
ca74e92b SN |
105 | } |
106 | ||
0ff92245 SN |
107 | static inline void delayacct_blkio_start(void) |
108 | { | |
e4042ad4 | 109 | if (!static_branch_unlikely(&delayacct_key)) |
eee4d9fe PZ |
110 | return; |
111 | ||
0ff92245 SN |
112 | if (current->delays) |
113 | __delayacct_blkio_start(); | |
114 | } | |
115 | ||
c96f5471 | 116 | static inline void delayacct_blkio_end(struct task_struct *p) |
0ff92245 | 117 | { |
e4042ad4 | 118 | if (!static_branch_unlikely(&delayacct_key)) |
eee4d9fe PZ |
119 | return; |
120 | ||
b512719f | 121 | if (p->delays) |
c96f5471 | 122 | __delayacct_blkio_end(p); |
0ff92245 SN |
123 | } |
124 | ||
25890454 SN |
125 | static inline __u64 delayacct_blkio_ticks(struct task_struct *tsk) |
126 | { | |
127 | if (tsk->delays) | |
128 | return __delayacct_blkio_ticks(tsk); | |
129 | return 0; | |
130 | } | |
131 | ||
873b4771 KK |
132 | static inline void delayacct_freepages_start(void) |
133 | { | |
82065b72 YY |
134 | if (!static_branch_unlikely(&delayacct_key)) |
135 | return; | |
136 | ||
873b4771 KK |
137 | if (current->delays) |
138 | __delayacct_freepages_start(); | |
139 | } | |
140 | ||
141 | static inline void delayacct_freepages_end(void) | |
142 | { | |
82065b72 YY |
143 | if (!static_branch_unlikely(&delayacct_key)) |
144 | return; | |
145 | ||
873b4771 KK |
146 | if (current->delays) |
147 | __delayacct_freepages_end(); | |
148 | } | |
149 | ||
aa1cf99b | 150 | static inline void delayacct_thrashing_start(bool *in_thrashing) |
b1d29ba8 | 151 | { |
82065b72 YY |
152 | if (!static_branch_unlikely(&delayacct_key)) |
153 | return; | |
154 | ||
b1d29ba8 | 155 | if (current->delays) |
aa1cf99b | 156 | __delayacct_thrashing_start(in_thrashing); |
b1d29ba8 JW |
157 | } |
158 | ||
aa1cf99b | 159 | static inline void delayacct_thrashing_end(bool *in_thrashing) |
b1d29ba8 | 160 | { |
82065b72 YY |
161 | if (!static_branch_unlikely(&delayacct_key)) |
162 | return; | |
163 | ||
b1d29ba8 | 164 | if (current->delays) |
aa1cf99b | 165 | __delayacct_thrashing_end(in_thrashing); |
b1d29ba8 JW |
166 | } |
167 | ||
a3d5dc90 YY |
168 | static inline void delayacct_swapin_start(void) |
169 | { | |
82065b72 YY |
170 | if (!static_branch_unlikely(&delayacct_key)) |
171 | return; | |
172 | ||
a3d5dc90 YY |
173 | if (current->delays) |
174 | __delayacct_swapin_start(); | |
175 | } | |
176 | ||
177 | static inline void delayacct_swapin_end(void) | |
178 | { | |
82065b72 YY |
179 | if (!static_branch_unlikely(&delayacct_key)) |
180 | return; | |
181 | ||
a3d5dc90 YY |
182 | if (current->delays) |
183 | __delayacct_swapin_end(); | |
184 | } | |
185 | ||
5bf18281 | 186 | static inline void delayacct_compact_start(void) |
187 | { | |
188 | if (!static_branch_unlikely(&delayacct_key)) | |
189 | return; | |
190 | ||
191 | if (current->delays) | |
192 | __delayacct_compact_start(); | |
193 | } | |
194 | ||
195 | static inline void delayacct_compact_end(void) | |
196 | { | |
197 | if (!static_branch_unlikely(&delayacct_key)) | |
198 | return; | |
199 | ||
200 | if (current->delays) | |
201 | __delayacct_compact_end(); | |
202 | } | |
203 | ||
662ce1dc YY |
204 | static inline void delayacct_wpcopy_start(void) |
205 | { | |
206 | if (!static_branch_unlikely(&delayacct_key)) | |
207 | return; | |
208 | ||
209 | if (current->delays) | |
210 | __delayacct_wpcopy_start(); | |
211 | } | |
212 | ||
213 | static inline void delayacct_wpcopy_end(void) | |
214 | { | |
215 | if (!static_branch_unlikely(&delayacct_key)) | |
216 | return; | |
217 | ||
218 | if (current->delays) | |
219 | __delayacct_wpcopy_end(); | |
220 | } | |
221 | ||
a3b2aeac YY |
222 | static inline void delayacct_irq(struct task_struct *task, u32 delta) |
223 | { | |
224 | if (!static_branch_unlikely(&delayacct_key)) | |
225 | return; | |
226 | ||
227 | if (task->delays) | |
228 | __delayacct_irq(task, delta); | |
229 | } | |
230 | ||
ca74e92b | 231 | #else |
ca74e92b SN |
232 | static inline void delayacct_init(void) |
233 | {} | |
234 | static inline void delayacct_tsk_init(struct task_struct *tsk) | |
235 | {} | |
35df17c5 | 236 | static inline void delayacct_tsk_free(struct task_struct *tsk) |
ca74e92b | 237 | {} |
0ff92245 SN |
238 | static inline void delayacct_blkio_start(void) |
239 | {} | |
c96f5471 | 240 | static inline void delayacct_blkio_end(struct task_struct *p) |
0ff92245 | 241 | {} |
6f44993f SN |
242 | static inline int delayacct_add_tsk(struct taskstats *d, |
243 | struct task_struct *tsk) | |
244 | { return 0; } | |
25890454 SN |
245 | static inline __u64 delayacct_blkio_ticks(struct task_struct *tsk) |
246 | { return 0; } | |
846c7bb0 BS |
247 | static inline int delayacct_is_task_waiting_on_io(struct task_struct *p) |
248 | { return 0; } | |
873b4771 KK |
249 | static inline void delayacct_freepages_start(void) |
250 | {} | |
251 | static inline void delayacct_freepages_end(void) | |
252 | {} | |
aa1cf99b | 253 | static inline void delayacct_thrashing_start(bool *in_thrashing) |
b1d29ba8 | 254 | {} |
aa1cf99b | 255 | static inline void delayacct_thrashing_end(bool *in_thrashing) |
b1d29ba8 | 256 | {} |
a3d5dc90 YY |
257 | static inline void delayacct_swapin_start(void) |
258 | {} | |
259 | static inline void delayacct_swapin_end(void) | |
260 | {} | |
5bf18281 | 261 | static inline void delayacct_compact_start(void) |
262 | {} | |
263 | static inline void delayacct_compact_end(void) | |
264 | {} | |
662ce1dc YY |
265 | static inline void delayacct_wpcopy_start(void) |
266 | {} | |
267 | static inline void delayacct_wpcopy_end(void) | |
268 | {} | |
a3b2aeac YY |
269 | static inline void delayacct_irq(struct task_struct *task, u32 delta) |
270 | {} | |
873b4771 | 271 | |
ca74e92b SN |
272 | #endif /* CONFIG_TASK_DELAY_ACCT */ |
273 | ||
274 | #endif |