Commit | Line | Data |
---|---|---|
457c8996 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
041cd640 TH |
2 | #include "cgroup-internal.h" |
3 | ||
4 | #include <linux/sched/cputime.h> | |
5 | ||
0fa294fb | 6 | static DEFINE_SPINLOCK(cgroup_rstat_lock); |
c58632b3 | 7 | static DEFINE_PER_CPU(raw_spinlock_t, cgroup_rstat_cpu_lock); |
041cd640 | 8 | |
a17556f8 TH |
9 | static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu); |
10 | ||
c58632b3 | 11 | static struct cgroup_rstat_cpu *cgroup_rstat_cpu(struct cgroup *cgrp, int cpu) |
041cd640 | 12 | { |
c58632b3 | 13 | return per_cpu_ptr(cgrp->rstat_cpu, cpu); |
041cd640 TH |
14 | } |
15 | ||
16 | /** | |
6162cef0 | 17 | * cgroup_rstat_updated - keep track of updated rstat_cpu |
041cd640 | 18 | * @cgrp: target cgroup |
c58632b3 | 19 | * @cpu: cpu on which rstat_cpu was updated |
041cd640 | 20 | * |
c58632b3 TH |
21 | * @cgrp's rstat_cpu on @cpu was updated. Put it on the parent's matching |
22 | * rstat_cpu->updated_children list. See the comment on top of | |
23 | * cgroup_rstat_cpu definition for details. | |
041cd640 | 24 | */ |
6162cef0 | 25 | void cgroup_rstat_updated(struct cgroup *cgrp, int cpu) |
041cd640 | 26 | { |
c58632b3 | 27 | raw_spinlock_t *cpu_lock = per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu); |
041cd640 TH |
28 | unsigned long flags; |
29 | ||
30 | /* | |
d8ef4b38 TH |
31 | * Speculative already-on-list test. This may race leading to |
32 | * temporary inaccuracies, which is fine. | |
33 | * | |
041cd640 TH |
34 | * Because @parent's updated_children is terminated with @parent |
35 | * instead of NULL, we can tell whether @cgrp is on the list by | |
36 | * testing the next pointer for NULL. | |
37 | */ | |
eda09706 | 38 | if (data_race(cgroup_rstat_cpu(cgrp, cpu)->updated_next)) |
041cd640 TH |
39 | return; |
40 | ||
41 | raw_spin_lock_irqsave(cpu_lock, flags); | |
42 | ||
43 | /* put @cgrp and all ancestors on the corresponding updated lists */ | |
dc26532a | 44 | while (true) { |
c58632b3 | 45 | struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu); |
dc26532a JW |
46 | struct cgroup *parent = cgroup_parent(cgrp); |
47 | struct cgroup_rstat_cpu *prstatc; | |
041cd640 TH |
48 | |
49 | /* | |
50 | * Both additions and removals are bottom-up. If a cgroup | |
51 | * is already in the tree, all ancestors are. | |
52 | */ | |
c58632b3 | 53 | if (rstatc->updated_next) |
041cd640 TH |
54 | break; |
55 | ||
dc26532a JW |
56 | /* Root has no parent to link it to, but mark it busy */ |
57 | if (!parent) { | |
58 | rstatc->updated_next = cgrp; | |
59 | break; | |
60 | } | |
61 | ||
62 | prstatc = cgroup_rstat_cpu(parent, cpu); | |
c58632b3 TH |
63 | rstatc->updated_next = prstatc->updated_children; |
64 | prstatc->updated_children = cgrp; | |
dc26532a JW |
65 | |
66 | cgrp = parent; | |
041cd640 TH |
67 | } |
68 | ||
69 | raw_spin_unlock_irqrestore(cpu_lock, flags); | |
70 | } | |
71 | ||
72 | /** | |
c58632b3 | 73 | * cgroup_rstat_cpu_pop_updated - iterate and dismantle rstat_cpu updated tree |
041cd640 TH |
74 | * @pos: current position |
75 | * @root: root of the tree to traversal | |
76 | * @cpu: target cpu | |
77 | * | |
08b2b6fd | 78 | * Walks the updated rstat_cpu tree on @cpu from @root. %NULL @pos starts |
041cd640 TH |
79 | * the traversal and %NULL return indicates the end. During traversal, |
80 | * each returned cgroup is unlinked from the tree. Must be called with the | |
c58632b3 | 81 | * matching cgroup_rstat_cpu_lock held. |
041cd640 TH |
82 | * |
83 | * The only ordering guarantee is that, for a parent and a child pair | |
84 | * covered by a given traversal, if a child is visited, its parent is | |
85 | * guaranteed to be visited afterwards. | |
86 | */ | |
c58632b3 TH |
87 | static struct cgroup *cgroup_rstat_cpu_pop_updated(struct cgroup *pos, |
88 | struct cgroup *root, int cpu) | |
041cd640 | 89 | { |
c58632b3 | 90 | struct cgroup_rstat_cpu *rstatc; |
f5f60d23 | 91 | struct cgroup *parent; |
041cd640 TH |
92 | |
93 | if (pos == root) | |
94 | return NULL; | |
95 | ||
96 | /* | |
97 | * We're gonna walk down to the first leaf and visit/remove it. We | |
98 | * can pick whatever unvisited node as the starting point. | |
99 | */ | |
f5f60d23 | 100 | if (!pos) { |
041cd640 | 101 | pos = root; |
f5f60d23 WY |
102 | /* return NULL if this subtree is not on-list */ |
103 | if (!cgroup_rstat_cpu(pos, cpu)->updated_next) | |
104 | return NULL; | |
105 | } else { | |
041cd640 | 106 | pos = cgroup_parent(pos); |
f5f60d23 | 107 | } |
041cd640 TH |
108 | |
109 | /* walk down to the first leaf */ | |
110 | while (true) { | |
c58632b3 TH |
111 | rstatc = cgroup_rstat_cpu(pos, cpu); |
112 | if (rstatc->updated_children == pos) | |
041cd640 | 113 | break; |
c58632b3 | 114 | pos = rstatc->updated_children; |
041cd640 TH |
115 | } |
116 | ||
117 | /* | |
118 | * Unlink @pos from the tree. As the updated_children list is | |
119 | * singly linked, we have to walk it to find the removal point. | |
120 | * However, due to the way we traverse, @pos will be the first | |
121 | * child in most cases. The only exception is @root. | |
122 | */ | |
f5f60d23 WY |
123 | parent = cgroup_parent(pos); |
124 | if (parent) { | |
125 | struct cgroup_rstat_cpu *prstatc; | |
126 | struct cgroup **nextp; | |
041cd640 | 127 | |
f5f60d23 WY |
128 | prstatc = cgroup_rstat_cpu(parent, cpu); |
129 | nextp = &prstatc->updated_children; | |
130 | while (*nextp != pos) { | |
131 | struct cgroup_rstat_cpu *nrstatc; | |
132 | ||
133 | nrstatc = cgroup_rstat_cpu(*nextp, cpu); | |
134 | WARN_ON_ONCE(*nextp == parent); | |
135 | nextp = &nrstatc->updated_next; | |
136 | } | |
137 | *nextp = rstatc->updated_next; | |
041cd640 TH |
138 | } |
139 | ||
f5f60d23 WY |
140 | rstatc->updated_next = NULL; |
141 | return pos; | |
041cd640 TH |
142 | } |
143 | ||
a17556f8 | 144 | /* see cgroup_rstat_flush() */ |
0fa294fb TH |
145 | static void cgroup_rstat_flush_locked(struct cgroup *cgrp, bool may_sleep) |
146 | __releases(&cgroup_rstat_lock) __acquires(&cgroup_rstat_lock) | |
a17556f8 TH |
147 | { |
148 | int cpu; | |
149 | ||
0fa294fb | 150 | lockdep_assert_held(&cgroup_rstat_lock); |
a17556f8 TH |
151 | |
152 | for_each_possible_cpu(cpu) { | |
153 | raw_spinlock_t *cpu_lock = per_cpu_ptr(&cgroup_rstat_cpu_lock, | |
154 | cpu); | |
155 | struct cgroup *pos = NULL; | |
156 | ||
0fa294fb | 157 | raw_spin_lock(cpu_lock); |
8f53470b TH |
158 | while ((pos = cgroup_rstat_cpu_pop_updated(pos, cgrp, cpu))) { |
159 | struct cgroup_subsys_state *css; | |
160 | ||
a17556f8 | 161 | cgroup_base_stat_flush(pos, cpu); |
8f53470b TH |
162 | |
163 | rcu_read_lock(); | |
164 | list_for_each_entry_rcu(css, &pos->rstat_css_list, | |
165 | rstat_css_node) | |
166 | css->ss->css_rstat_flush(css, cpu); | |
167 | rcu_read_unlock(); | |
168 | } | |
0fa294fb TH |
169 | raw_spin_unlock(cpu_lock); |
170 | ||
171 | /* if @may_sleep, play nice and yield if necessary */ | |
172 | if (may_sleep && (need_resched() || | |
173 | spin_needbreak(&cgroup_rstat_lock))) { | |
174 | spin_unlock_irq(&cgroup_rstat_lock); | |
175 | if (!cond_resched()) | |
176 | cpu_relax(); | |
177 | spin_lock_irq(&cgroup_rstat_lock); | |
178 | } | |
a17556f8 TH |
179 | } |
180 | } | |
181 | ||
182 | /** | |
183 | * cgroup_rstat_flush - flush stats in @cgrp's subtree | |
184 | * @cgrp: target cgroup | |
185 | * | |
186 | * Collect all per-cpu stats in @cgrp's subtree into the global counters | |
187 | * and propagate them upwards. After this function returns, all cgroups in | |
188 | * the subtree have up-to-date ->stat. | |
189 | * | |
190 | * This also gets all cgroups in the subtree including @cgrp off the | |
191 | * ->updated_children lists. | |
0fa294fb TH |
192 | * |
193 | * This function may block. | |
a17556f8 TH |
194 | */ |
195 | void cgroup_rstat_flush(struct cgroup *cgrp) | |
196 | { | |
0fa294fb TH |
197 | might_sleep(); |
198 | ||
199 | spin_lock_irq(&cgroup_rstat_lock); | |
200 | cgroup_rstat_flush_locked(cgrp, true); | |
201 | spin_unlock_irq(&cgroup_rstat_lock); | |
202 | } | |
203 | ||
204 | /** | |
205 | * cgroup_rstat_flush_irqsafe - irqsafe version of cgroup_rstat_flush() | |
206 | * @cgrp: target cgroup | |
207 | * | |
208 | * This function can be called from any context. | |
209 | */ | |
210 | void cgroup_rstat_flush_irqsafe(struct cgroup *cgrp) | |
211 | { | |
212 | unsigned long flags; | |
213 | ||
214 | spin_lock_irqsave(&cgroup_rstat_lock, flags); | |
215 | cgroup_rstat_flush_locked(cgrp, false); | |
216 | spin_unlock_irqrestore(&cgroup_rstat_lock, flags); | |
a17556f8 TH |
217 | } |
218 | ||
6162cef0 | 219 | /** |
2ca11b0e | 220 | * cgroup_rstat_flush_hold - flush stats in @cgrp's subtree and hold |
6162cef0 TH |
221 | * @cgrp: target cgroup |
222 | * | |
223 | * Flush stats in @cgrp's subtree and prevent further flushes. Must be | |
224 | * paired with cgroup_rstat_flush_release(). | |
0fa294fb TH |
225 | * |
226 | * This function may block. | |
6162cef0 TH |
227 | */ |
228 | void cgroup_rstat_flush_hold(struct cgroup *cgrp) | |
0fa294fb | 229 | __acquires(&cgroup_rstat_lock) |
6162cef0 | 230 | { |
0fa294fb TH |
231 | might_sleep(); |
232 | spin_lock_irq(&cgroup_rstat_lock); | |
233 | cgroup_rstat_flush_locked(cgrp, true); | |
6162cef0 TH |
234 | } |
235 | ||
236 | /** | |
237 | * cgroup_rstat_flush_release - release cgroup_rstat_flush_hold() | |
238 | */ | |
239 | void cgroup_rstat_flush_release(void) | |
0fa294fb | 240 | __releases(&cgroup_rstat_lock) |
6162cef0 | 241 | { |
0fa294fb | 242 | spin_unlock_irq(&cgroup_rstat_lock); |
6162cef0 TH |
243 | } |
244 | ||
a17556f8 TH |
245 | int cgroup_rstat_init(struct cgroup *cgrp) |
246 | { | |
247 | int cpu; | |
248 | ||
249 | /* the root cgrp has rstat_cpu preallocated */ | |
250 | if (!cgrp->rstat_cpu) { | |
251 | cgrp->rstat_cpu = alloc_percpu(struct cgroup_rstat_cpu); | |
252 | if (!cgrp->rstat_cpu) | |
253 | return -ENOMEM; | |
254 | } | |
255 | ||
256 | /* ->updated_children list is self terminated */ | |
257 | for_each_possible_cpu(cpu) { | |
258 | struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu); | |
259 | ||
260 | rstatc->updated_children = cgrp; | |
261 | u64_stats_init(&rstatc->bsync); | |
262 | } | |
263 | ||
264 | return 0; | |
265 | } | |
266 | ||
267 | void cgroup_rstat_exit(struct cgroup *cgrp) | |
268 | { | |
269 | int cpu; | |
270 | ||
271 | cgroup_rstat_flush(cgrp); | |
272 | ||
273 | /* sanity check */ | |
274 | for_each_possible_cpu(cpu) { | |
275 | struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu); | |
276 | ||
277 | if (WARN_ON_ONCE(rstatc->updated_children != cgrp) || | |
278 | WARN_ON_ONCE(rstatc->updated_next)) | |
279 | return; | |
280 | } | |
281 | ||
282 | free_percpu(cgrp->rstat_cpu); | |
283 | cgrp->rstat_cpu = NULL; | |
284 | } | |
285 | ||
286 | void __init cgroup_rstat_boot(void) | |
287 | { | |
288 | int cpu; | |
289 | ||
290 | for_each_possible_cpu(cpu) | |
291 | raw_spin_lock_init(per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu)); | |
a17556f8 TH |
292 | } |
293 | ||
294 | /* | |
295 | * Functions for cgroup basic resource statistics implemented on top of | |
296 | * rstat. | |
297 | */ | |
1bb5ec2e TH |
298 | static void cgroup_base_stat_add(struct cgroup_base_stat *dst_bstat, |
299 | struct cgroup_base_stat *src_bstat) | |
041cd640 | 300 | { |
d4ff749b TH |
301 | dst_bstat->cputime.utime += src_bstat->cputime.utime; |
302 | dst_bstat->cputime.stime += src_bstat->cputime.stime; | |
303 | dst_bstat->cputime.sum_exec_runtime += src_bstat->cputime.sum_exec_runtime; | |
041cd640 TH |
304 | } |
305 | ||
1bb5ec2e TH |
306 | static void cgroup_base_stat_sub(struct cgroup_base_stat *dst_bstat, |
307 | struct cgroup_base_stat *src_bstat) | |
308 | { | |
309 | dst_bstat->cputime.utime -= src_bstat->cputime.utime; | |
310 | dst_bstat->cputime.stime -= src_bstat->cputime.stime; | |
311 | dst_bstat->cputime.sum_exec_runtime -= src_bstat->cputime.sum_exec_runtime; | |
312 | } | |
313 | ||
d4ff749b | 314 | static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu) |
041cd640 | 315 | { |
c58632b3 | 316 | struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu); |
dc26532a | 317 | struct cgroup *parent = cgroup_parent(cgrp); |
1bb5ec2e | 318 | struct cgroup_base_stat cur, delta; |
041cd640 TH |
319 | unsigned seq; |
320 | ||
dc26532a JW |
321 | /* Root-level stats are sourced from system-wide CPU stats */ |
322 | if (!parent) | |
323 | return; | |
324 | ||
041cd640 TH |
325 | /* fetch the current per-cpu values */ |
326 | do { | |
d4ff749b | 327 | seq = __u64_stats_fetch_begin(&rstatc->bsync); |
1bb5ec2e | 328 | cur.cputime = rstatc->bstat.cputime; |
d4ff749b | 329 | } while (__u64_stats_fetch_retry(&rstatc->bsync, seq)); |
041cd640 | 330 | |
1bb5ec2e TH |
331 | /* propagate percpu delta to global */ |
332 | delta = cur; | |
333 | cgroup_base_stat_sub(&delta, &rstatc->last_bstat); | |
334 | cgroup_base_stat_add(&cgrp->bstat, &delta); | |
335 | cgroup_base_stat_add(&rstatc->last_bstat, &delta); | |
336 | ||
dc26532a JW |
337 | /* propagate global delta to parent (unless that's root) */ |
338 | if (cgroup_parent(parent)) { | |
1bb5ec2e TH |
339 | delta = cgrp->bstat; |
340 | cgroup_base_stat_sub(&delta, &cgrp->last_bstat); | |
341 | cgroup_base_stat_add(&parent->bstat, &delta); | |
342 | cgroup_base_stat_add(&cgrp->last_bstat, &delta); | |
343 | } | |
041cd640 TH |
344 | } |
345 | ||
c58632b3 | 346 | static struct cgroup_rstat_cpu * |
c3df5fb5 | 347 | cgroup_base_stat_cputime_account_begin(struct cgroup *cgrp, unsigned long *flags) |
041cd640 | 348 | { |
c58632b3 | 349 | struct cgroup_rstat_cpu *rstatc; |
041cd640 | 350 | |
c58632b3 | 351 | rstatc = get_cpu_ptr(cgrp->rstat_cpu); |
c3df5fb5 | 352 | *flags = u64_stats_update_begin_irqsave(&rstatc->bsync); |
c58632b3 | 353 | return rstatc; |
041cd640 TH |
354 | } |
355 | ||
d4ff749b | 356 | static void cgroup_base_stat_cputime_account_end(struct cgroup *cgrp, |
c3df5fb5 TH |
357 | struct cgroup_rstat_cpu *rstatc, |
358 | unsigned long flags) | |
041cd640 | 359 | { |
c3df5fb5 | 360 | u64_stats_update_end_irqrestore(&rstatc->bsync, flags); |
6162cef0 | 361 | cgroup_rstat_updated(cgrp, smp_processor_id()); |
c58632b3 | 362 | put_cpu_ptr(rstatc); |
041cd640 TH |
363 | } |
364 | ||
365 | void __cgroup_account_cputime(struct cgroup *cgrp, u64 delta_exec) | |
366 | { | |
c58632b3 | 367 | struct cgroup_rstat_cpu *rstatc; |
c3df5fb5 | 368 | unsigned long flags; |
041cd640 | 369 | |
c3df5fb5 | 370 | rstatc = cgroup_base_stat_cputime_account_begin(cgrp, &flags); |
d4ff749b | 371 | rstatc->bstat.cputime.sum_exec_runtime += delta_exec; |
c3df5fb5 | 372 | cgroup_base_stat_cputime_account_end(cgrp, rstatc, flags); |
041cd640 TH |
373 | } |
374 | ||
375 | void __cgroup_account_cputime_field(struct cgroup *cgrp, | |
376 | enum cpu_usage_stat index, u64 delta_exec) | |
377 | { | |
c58632b3 | 378 | struct cgroup_rstat_cpu *rstatc; |
c3df5fb5 | 379 | unsigned long flags; |
041cd640 | 380 | |
c3df5fb5 | 381 | rstatc = cgroup_base_stat_cputime_account_begin(cgrp, &flags); |
041cd640 TH |
382 | |
383 | switch (index) { | |
384 | case CPUTIME_USER: | |
385 | case CPUTIME_NICE: | |
d4ff749b | 386 | rstatc->bstat.cputime.utime += delta_exec; |
041cd640 TH |
387 | break; |
388 | case CPUTIME_SYSTEM: | |
389 | case CPUTIME_IRQ: | |
390 | case CPUTIME_SOFTIRQ: | |
d4ff749b | 391 | rstatc->bstat.cputime.stime += delta_exec; |
041cd640 TH |
392 | break; |
393 | default: | |
394 | break; | |
395 | } | |
396 | ||
c3df5fb5 | 397 | cgroup_base_stat_cputime_account_end(cgrp, rstatc, flags); |
041cd640 TH |
398 | } |
399 | ||
936f2a70 BB |
400 | /* |
401 | * compute the cputime for the root cgroup by getting the per cpu data | |
402 | * at a global level, then categorizing the fields in a manner consistent | |
403 | * with how it is done by __cgroup_account_cputime_field for each bit of | |
404 | * cpu time attributed to a cgroup. | |
405 | */ | |
406 | static void root_cgroup_cputime(struct task_cputime *cputime) | |
407 | { | |
408 | int i; | |
409 | ||
410 | cputime->stime = 0; | |
411 | cputime->utime = 0; | |
412 | cputime->sum_exec_runtime = 0; | |
413 | for_each_possible_cpu(i) { | |
414 | struct kernel_cpustat kcpustat; | |
415 | u64 *cpustat = kcpustat.cpustat; | |
416 | u64 user = 0; | |
417 | u64 sys = 0; | |
418 | ||
419 | kcpustat_cpu_fetch(&kcpustat, i); | |
420 | ||
421 | user += cpustat[CPUTIME_USER]; | |
422 | user += cpustat[CPUTIME_NICE]; | |
423 | cputime->utime += user; | |
424 | ||
425 | sys += cpustat[CPUTIME_SYSTEM]; | |
426 | sys += cpustat[CPUTIME_IRQ]; | |
427 | sys += cpustat[CPUTIME_SOFTIRQ]; | |
428 | cputime->stime += sys; | |
429 | ||
430 | cputime->sum_exec_runtime += user; | |
431 | cputime->sum_exec_runtime += sys; | |
432 | cputime->sum_exec_runtime += cpustat[CPUTIME_STEAL]; | |
936f2a70 BB |
433 | } |
434 | } | |
435 | ||
d4ff749b | 436 | void cgroup_base_stat_cputime_show(struct seq_file *seq) |
041cd640 TH |
437 | { |
438 | struct cgroup *cgrp = seq_css(seq)->cgroup; | |
439 | u64 usage, utime, stime; | |
936f2a70 BB |
440 | struct task_cputime cputime; |
441 | ||
442 | if (cgroup_parent(cgrp)) { | |
443 | cgroup_rstat_flush_hold(cgrp); | |
444 | usage = cgrp->bstat.cputime.sum_exec_runtime; | |
445 | cputime_adjust(&cgrp->bstat.cputime, &cgrp->prev_cputime, | |
446 | &utime, &stime); | |
447 | cgroup_rstat_flush_release(); | |
448 | } else { | |
449 | root_cgroup_cputime(&cputime); | |
450 | usage = cputime.sum_exec_runtime; | |
451 | utime = cputime.utime; | |
452 | stime = cputime.stime; | |
453 | } | |
041cd640 TH |
454 | |
455 | do_div(usage, NSEC_PER_USEC); | |
456 | do_div(utime, NSEC_PER_USEC); | |
457 | do_div(stime, NSEC_PER_USEC); | |
458 | ||
d41bf8c9 TH |
459 | seq_printf(seq, "usage_usec %llu\n" |
460 | "user_usec %llu\n" | |
461 | "system_usec %llu\n", | |
462 | usage, utime, stime); | |
041cd640 | 463 | } |