Commit | Line | Data |
---|---|---|
d4ccaf58 HL |
1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* Copyright (c) 2022 Google */ | |
3 | #include <linux/bpf.h> | |
4 | #include <linux/btf_ids.h> | |
5 | #include <linux/cgroup.h> | |
6 | #include <linux/kernel.h> | |
7 | #include <linux/seq_file.h> | |
8 | ||
9 | #include "../cgroup/cgroup-internal.h" /* cgroup_mutex and cgroup_is_dead */ | |
10 | ||
11 | /* cgroup_iter provides four modes of traversal to the cgroup hierarchy. | |
12 | * | |
13 | * 1. Walk the descendants of a cgroup in pre-order. | |
14 | * 2. Walk the descendants of a cgroup in post-order. | |
15 | * 3. Walk the ancestors of a cgroup. | |
16 | * 4. Show the given cgroup only. | |
17 | * | |
18 | * For walking descendants, cgroup_iter can walk in either pre-order or | |
19 | * post-order. For walking ancestors, the iter walks up from a cgroup to | |
20 | * the root. | |
21 | * | |
22 | * The iter program can terminate the walk early by returning 1. Walk | |
23 | * continues if prog returns 0. | |
24 | * | |
25 | * The prog can check (seq->num == 0) to determine whether this is | |
26 | * the first element. The prog may also be passed a NULL cgroup, | |
27 | * which means the walk has completed and the prog has a chance to | |
28 | * do post-processing, such as outputting an epilogue. | |
29 | * | |
30 | * Note: the iter_prog is called with cgroup_mutex held. | |
31 | * | |
32 | * Currently only one session is supported, which means, depending on the | |
33 | * volume of data bpf program intends to send to user space, the number | |
34 | * of cgroups that can be walked is limited. For example, given the current | |
35 | * buffer size is 8 * PAGE_SIZE, if the program sends 64B data for each | |
36 | * cgroup, assuming PAGE_SIZE is 4kb, the total number of cgroups that can | |
37 | * be walked is 512. This is a limitation of cgroup_iter. If the output data | |
38 | * is larger than the kernel buffer size, after all data in the kernel buffer | |
39 | * is consumed by user space, the subsequent read() syscall will signal | |
40 | * EOPNOTSUPP. In order to work around, the user may have to update their | |
41 | * program to reduce the volume of data sent to output. For example, skip | |
42 | * some uninteresting cgroups. | |
43 | */ | |
44 | ||
45 | struct bpf_iter__cgroup { | |
46 | __bpf_md_ptr(struct bpf_iter_meta *, meta); | |
47 | __bpf_md_ptr(struct cgroup *, cgroup); | |
48 | }; | |
49 | ||
50 | struct cgroup_iter_priv { | |
51 | struct cgroup_subsys_state *start_css; | |
52 | bool visited_all; | |
53 | bool terminate; | |
54 | int order; | |
55 | }; | |
56 | ||
57 | static void *cgroup_iter_seq_start(struct seq_file *seq, loff_t *pos) | |
58 | { | |
59 | struct cgroup_iter_priv *p = seq->private; | |
60 | ||
4cdb91b0 | 61 | cgroup_lock(); |
d4ccaf58 HL |
62 | |
63 | /* cgroup_iter doesn't support read across multiple sessions. */ | |
64 | if (*pos > 0) { | |
65 | if (p->visited_all) | |
66 | return NULL; | |
67 | ||
68 | /* Haven't visited all, but because cgroup_mutex has dropped, | |
69 | * return -EOPNOTSUPP to indicate incomplete iteration. | |
70 | */ | |
71 | return ERR_PTR(-EOPNOTSUPP); | |
72 | } | |
73 | ||
74 | ++*pos; | |
75 | p->terminate = false; | |
76 | p->visited_all = false; | |
d4ffb6f3 | 77 | if (p->order == BPF_CGROUP_ITER_DESCENDANTS_PRE) |
d4ccaf58 | 78 | return css_next_descendant_pre(NULL, p->start_css); |
d4ffb6f3 | 79 | else if (p->order == BPF_CGROUP_ITER_DESCENDANTS_POST) |
d4ccaf58 | 80 | return css_next_descendant_post(NULL, p->start_css); |
ccf365ea | 81 | else /* BPF_CGROUP_ITER_SELF_ONLY and BPF_CGROUP_ITER_ANCESTORS_UP */ |
d4ccaf58 HL |
82 | return p->start_css; |
83 | } | |
84 | ||
85 | static int __cgroup_iter_seq_show(struct seq_file *seq, | |
86 | struct cgroup_subsys_state *css, int in_stop); | |
87 | ||
88 | static void cgroup_iter_seq_stop(struct seq_file *seq, void *v) | |
89 | { | |
90 | struct cgroup_iter_priv *p = seq->private; | |
91 | ||
4cdb91b0 | 92 | cgroup_unlock(); |
d4ccaf58 HL |
93 | |
94 | /* pass NULL to the prog for post-processing */ | |
95 | if (!v) { | |
96 | __cgroup_iter_seq_show(seq, NULL, true); | |
97 | p->visited_all = true; | |
98 | } | |
99 | } | |
100 | ||
101 | static void *cgroup_iter_seq_next(struct seq_file *seq, void *v, loff_t *pos) | |
102 | { | |
103 | struct cgroup_subsys_state *curr = (struct cgroup_subsys_state *)v; | |
104 | struct cgroup_iter_priv *p = seq->private; | |
105 | ||
106 | ++*pos; | |
107 | if (p->terminate) | |
108 | return NULL; | |
109 | ||
d4ffb6f3 | 110 | if (p->order == BPF_CGROUP_ITER_DESCENDANTS_PRE) |
d4ccaf58 | 111 | return css_next_descendant_pre(curr, p->start_css); |
d4ffb6f3 | 112 | else if (p->order == BPF_CGROUP_ITER_DESCENDANTS_POST) |
d4ccaf58 | 113 | return css_next_descendant_post(curr, p->start_css); |
d4ffb6f3 | 114 | else if (p->order == BPF_CGROUP_ITER_ANCESTORS_UP) |
d4ccaf58 | 115 | return curr->parent; |
d4ffb6f3 | 116 | else /* BPF_CGROUP_ITER_SELF_ONLY */ |
d4ccaf58 HL |
117 | return NULL; |
118 | } | |
119 | ||
120 | static int __cgroup_iter_seq_show(struct seq_file *seq, | |
121 | struct cgroup_subsys_state *css, int in_stop) | |
122 | { | |
123 | struct cgroup_iter_priv *p = seq->private; | |
124 | struct bpf_iter__cgroup ctx; | |
125 | struct bpf_iter_meta meta; | |
126 | struct bpf_prog *prog; | |
127 | int ret = 0; | |
128 | ||
129 | /* cgroup is dead, skip this element */ | |
130 | if (css && cgroup_is_dead(css->cgroup)) | |
131 | return 0; | |
132 | ||
133 | ctx.meta = &meta; | |
134 | ctx.cgroup = css ? css->cgroup : NULL; | |
135 | meta.seq = seq; | |
136 | prog = bpf_iter_get_info(&meta, in_stop); | |
137 | if (prog) | |
138 | ret = bpf_iter_run_prog(prog, &ctx); | |
139 | ||
140 | /* if prog returns > 0, terminate after this element. */ | |
141 | if (ret != 0) | |
142 | p->terminate = true; | |
143 | ||
144 | return 0; | |
145 | } | |
146 | ||
147 | static int cgroup_iter_seq_show(struct seq_file *seq, void *v) | |
148 | { | |
149 | return __cgroup_iter_seq_show(seq, (struct cgroup_subsys_state *)v, | |
150 | false); | |
151 | } | |
152 | ||
153 | static const struct seq_operations cgroup_iter_seq_ops = { | |
154 | .start = cgroup_iter_seq_start, | |
155 | .next = cgroup_iter_seq_next, | |
156 | .stop = cgroup_iter_seq_stop, | |
157 | .show = cgroup_iter_seq_show, | |
158 | }; | |
159 | ||
5e67b8ef | 160 | BTF_ID_LIST_GLOBAL_SINGLE(bpf_cgroup_btf_id, struct, cgroup) |
d4ccaf58 HL |
161 | |
162 | static int cgroup_iter_seq_init(void *priv, struct bpf_iter_aux_info *aux) | |
163 | { | |
164 | struct cgroup_iter_priv *p = (struct cgroup_iter_priv *)priv; | |
165 | struct cgroup *cgrp = aux->cgroup.start; | |
166 | ||
1a5160d4 HT |
167 | /* bpf_iter_attach_cgroup() has already acquired an extra reference |
168 | * for the start cgroup, but the reference may be released after | |
169 | * cgroup_iter_seq_init(), so acquire another reference for the | |
170 | * start cgroup. | |
171 | */ | |
d4ccaf58 | 172 | p->start_css = &cgrp->self; |
1a5160d4 | 173 | css_get(p->start_css); |
d4ccaf58 HL |
174 | p->terminate = false; |
175 | p->visited_all = false; | |
176 | p->order = aux->cgroup.order; | |
177 | return 0; | |
178 | } | |
179 | ||
1a5160d4 HT |
180 | static void cgroup_iter_seq_fini(void *priv) |
181 | { | |
182 | struct cgroup_iter_priv *p = (struct cgroup_iter_priv *)priv; | |
183 | ||
184 | css_put(p->start_css); | |
185 | } | |
186 | ||
d4ccaf58 HL |
187 | static const struct bpf_iter_seq_info cgroup_iter_seq_info = { |
188 | .seq_ops = &cgroup_iter_seq_ops, | |
189 | .init_seq_private = cgroup_iter_seq_init, | |
1a5160d4 | 190 | .fini_seq_private = cgroup_iter_seq_fini, |
d4ccaf58 HL |
191 | .seq_priv_size = sizeof(struct cgroup_iter_priv), |
192 | }; | |
193 | ||
194 | static int bpf_iter_attach_cgroup(struct bpf_prog *prog, | |
195 | union bpf_iter_link_info *linfo, | |
196 | struct bpf_iter_aux_info *aux) | |
197 | { | |
198 | int fd = linfo->cgroup.cgroup_fd; | |
199 | u64 id = linfo->cgroup.cgroup_id; | |
200 | int order = linfo->cgroup.order; | |
201 | struct cgroup *cgrp; | |
202 | ||
d4ffb6f3 HL |
203 | if (order != BPF_CGROUP_ITER_DESCENDANTS_PRE && |
204 | order != BPF_CGROUP_ITER_DESCENDANTS_POST && | |
205 | order != BPF_CGROUP_ITER_ANCESTORS_UP && | |
206 | order != BPF_CGROUP_ITER_SELF_ONLY) | |
d4ccaf58 HL |
207 | return -EINVAL; |
208 | ||
209 | if (fd && id) | |
210 | return -EINVAL; | |
211 | ||
212 | if (fd) | |
35256d67 | 213 | cgrp = cgroup_v1v2_get_from_fd(fd); |
d4ccaf58 HL |
214 | else if (id) |
215 | cgrp = cgroup_get_from_id(id); | |
216 | else /* walk the entire hierarchy by default. */ | |
217 | cgrp = cgroup_get_from_path("/"); | |
218 | ||
219 | if (IS_ERR(cgrp)) | |
220 | return PTR_ERR(cgrp); | |
221 | ||
222 | aux->cgroup.start = cgrp; | |
223 | aux->cgroup.order = order; | |
224 | return 0; | |
225 | } | |
226 | ||
227 | static void bpf_iter_detach_cgroup(struct bpf_iter_aux_info *aux) | |
228 | { | |
229 | cgroup_put(aux->cgroup.start); | |
230 | } | |
231 | ||
232 | static void bpf_iter_cgroup_show_fdinfo(const struct bpf_iter_aux_info *aux, | |
233 | struct seq_file *seq) | |
234 | { | |
235 | char *buf; | |
236 | ||
237 | buf = kzalloc(PATH_MAX, GFP_KERNEL); | |
238 | if (!buf) { | |
239 | seq_puts(seq, "cgroup_path:\t<unknown>\n"); | |
240 | goto show_order; | |
241 | } | |
242 | ||
243 | /* If cgroup_path_ns() fails, buf will be an empty string, cgroup_path | |
244 | * will print nothing. | |
245 | * | |
246 | * Path is in the calling process's cgroup namespace. | |
247 | */ | |
248 | cgroup_path_ns(aux->cgroup.start, buf, PATH_MAX, | |
249 | current->nsproxy->cgroup_ns); | |
250 | seq_printf(seq, "cgroup_path:\t%s\n", buf); | |
251 | kfree(buf); | |
252 | ||
253 | show_order: | |
d4ffb6f3 | 254 | if (aux->cgroup.order == BPF_CGROUP_ITER_DESCENDANTS_PRE) |
d4ccaf58 | 255 | seq_puts(seq, "order: descendants_pre\n"); |
d4ffb6f3 | 256 | else if (aux->cgroup.order == BPF_CGROUP_ITER_DESCENDANTS_POST) |
d4ccaf58 | 257 | seq_puts(seq, "order: descendants_post\n"); |
d4ffb6f3 | 258 | else if (aux->cgroup.order == BPF_CGROUP_ITER_ANCESTORS_UP) |
d4ccaf58 | 259 | seq_puts(seq, "order: ancestors_up\n"); |
d4ffb6f3 | 260 | else /* BPF_CGROUP_ITER_SELF_ONLY */ |
d4ccaf58 HL |
261 | seq_puts(seq, "order: self_only\n"); |
262 | } | |
263 | ||
264 | static int bpf_iter_cgroup_fill_link_info(const struct bpf_iter_aux_info *aux, | |
265 | struct bpf_link_info *info) | |
266 | { | |
267 | info->iter.cgroup.order = aux->cgroup.order; | |
268 | info->iter.cgroup.cgroup_id = cgroup_id(aux->cgroup.start); | |
269 | return 0; | |
270 | } | |
271 | ||
272 | DEFINE_BPF_ITER_FUNC(cgroup, struct bpf_iter_meta *meta, | |
273 | struct cgroup *cgroup) | |
274 | ||
275 | static struct bpf_iter_reg bpf_cgroup_reg_info = { | |
276 | .target = "cgroup", | |
277 | .feature = BPF_ITER_RESCHED, | |
278 | .attach_target = bpf_iter_attach_cgroup, | |
279 | .detach_target = bpf_iter_detach_cgroup, | |
280 | .show_fdinfo = bpf_iter_cgroup_show_fdinfo, | |
281 | .fill_link_info = bpf_iter_cgroup_fill_link_info, | |
282 | .ctx_arg_info_size = 1, | |
283 | .ctx_arg_info = { | |
284 | { offsetof(struct bpf_iter__cgroup, cgroup), | |
0de4f50d | 285 | PTR_TO_BTF_ID_OR_NULL | PTR_TRUSTED }, |
d4ccaf58 HL |
286 | }, |
287 | .seq_info = &cgroup_iter_seq_info, | |
288 | }; | |
289 | ||
290 | static int __init bpf_cgroup_iter_init(void) | |
291 | { | |
292 | bpf_cgroup_reg_info.ctx_arg_info[0].btf_id = bpf_cgroup_btf_id[0]; | |
293 | return bpf_iter_reg_target(&bpf_cgroup_reg_info); | |
294 | } | |
295 | ||
296 | late_initcall(bpf_cgroup_iter_init); | |
7251d090 CZ |
297 | |
298 | struct bpf_iter_css { | |
299 | __u64 __opaque[3]; | |
300 | } __attribute__((aligned(8))); | |
301 | ||
302 | struct bpf_iter_css_kern { | |
303 | struct cgroup_subsys_state *start; | |
304 | struct cgroup_subsys_state *pos; | |
305 | unsigned int flags; | |
306 | } __attribute__((aligned(8))); | |
307 | ||
391145ba | 308 | __bpf_kfunc_start_defs(); |
7251d090 CZ |
309 | |
310 | __bpf_kfunc int bpf_iter_css_new(struct bpf_iter_css *it, | |
311 | struct cgroup_subsys_state *start, unsigned int flags) | |
312 | { | |
313 | struct bpf_iter_css_kern *kit = (void *)it; | |
314 | ||
315 | BUILD_BUG_ON(sizeof(struct bpf_iter_css_kern) > sizeof(struct bpf_iter_css)); | |
316 | BUILD_BUG_ON(__alignof__(struct bpf_iter_css_kern) != __alignof__(struct bpf_iter_css)); | |
317 | ||
318 | kit->start = NULL; | |
319 | switch (flags) { | |
320 | case BPF_CGROUP_ITER_DESCENDANTS_PRE: | |
321 | case BPF_CGROUP_ITER_DESCENDANTS_POST: | |
322 | case BPF_CGROUP_ITER_ANCESTORS_UP: | |
323 | break; | |
324 | default: | |
325 | return -EINVAL; | |
326 | } | |
327 | ||
328 | kit->start = start; | |
329 | kit->pos = NULL; | |
330 | kit->flags = flags; | |
331 | return 0; | |
332 | } | |
333 | ||
334 | __bpf_kfunc struct cgroup_subsys_state *bpf_iter_css_next(struct bpf_iter_css *it) | |
335 | { | |
336 | struct bpf_iter_css_kern *kit = (void *)it; | |
337 | ||
338 | if (!kit->start) | |
339 | return NULL; | |
340 | ||
341 | switch (kit->flags) { | |
342 | case BPF_CGROUP_ITER_DESCENDANTS_PRE: | |
343 | kit->pos = css_next_descendant_pre(kit->pos, kit->start); | |
344 | break; | |
345 | case BPF_CGROUP_ITER_DESCENDANTS_POST: | |
346 | kit->pos = css_next_descendant_post(kit->pos, kit->start); | |
347 | break; | |
348 | case BPF_CGROUP_ITER_ANCESTORS_UP: | |
349 | kit->pos = kit->pos ? kit->pos->parent : kit->start; | |
350 | } | |
351 | ||
352 | return kit->pos; | |
353 | } | |
354 | ||
355 | __bpf_kfunc void bpf_iter_css_destroy(struct bpf_iter_css *it) | |
356 | { | |
357 | } | |
358 | ||
391145ba | 359 | __bpf_kfunc_end_defs(); |