Commit | Line | Data |
---|---|---|
516f4d33 DV |
1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* Copyright (c) 2023 Meta, Inc */ | |
3 | #include <linux/bpf.h> | |
4 | #include <linux/bpf_mem_alloc.h> | |
5 | #include <linux/btf.h> | |
6 | #include <linux/btf_ids.h> | |
7 | #include <linux/cpumask.h> | |
8 | ||
9 | /** | |
10 | * struct bpf_cpumask - refcounted BPF cpumask wrapper structure | |
11 | * @cpumask: The actual cpumask embedded in the struct. | |
77473d1a | 12 | * @rcu: The RCU head used to free the cpumask with RCU safety. |
516f4d33 DV |
13 | * @usage: Object reference counter. When the refcount goes to 0, the |
14 | * memory is released back to the BPF allocator, which provides | |
15 | * RCU safety. | |
16 | * | |
17 | * Note that we explicitly embed a cpumask_t rather than a cpumask_var_t. This | |
18 | * is done to avoid confusing the verifier due to the typedef of cpumask_var_t | |
19 | * changing depending on whether CONFIG_CPUMASK_OFFSTACK is defined or not. See | |
20 | * the details in <linux/cpumask.h>. The consequence is that this structure is | |
21 | * likely a bit larger than it needs to be when CONFIG_CPUMASK_OFFSTACK is | |
22 | * defined due to embedding the whole NR_CPUS-size bitmap, but the extra memory | |
23 | * overhead is minimal. For the more typical case of CONFIG_CPUMASK_OFFSTACK | |
24 | * not being defined, the structure is the same size regardless. | |
25 | */ | |
26 | struct bpf_cpumask { | |
27 | cpumask_t cpumask; | |
77473d1a | 28 | struct rcu_head rcu; |
516f4d33 DV |
29 | refcount_t usage; |
30 | }; | |
31 | ||
32 | static struct bpf_mem_alloc bpf_cpumask_ma; | |
33 | ||
34 | static bool cpu_valid(u32 cpu) | |
35 | { | |
36 | return cpu < nr_cpu_ids; | |
37 | } | |
38 | ||
39 | __diag_push(); | |
40 | __diag_ignore_all("-Wmissing-prototypes", | |
41 | "Global kfuncs as their definitions will be in BTF"); | |
42 | ||
bdbda395 DV |
43 | /** |
44 | * bpf_cpumask_create() - Create a mutable BPF cpumask. | |
45 | * | |
46 | * Allocates a cpumask that can be queried, mutated, acquired, and released by | |
47 | * a BPF program. The cpumask returned by this function must either be embedded | |
48 | * in a map as a kptr, or freed with bpf_cpumask_release(). | |
49 | * | |
50 | * bpf_cpumask_create() allocates memory using the BPF memory allocator, and | |
51 | * will not block. It may return NULL if no memory is available. | |
52 | */ | |
400031e0 | 53 | __bpf_kfunc struct bpf_cpumask *bpf_cpumask_create(void) |
516f4d33 DV |
54 | { |
55 | struct bpf_cpumask *cpumask; | |
56 | ||
cb4a21ea DV |
57 | /* cpumask must be the first element so struct bpf_cpumask be cast to struct cpumask. */ |
58 | BUILD_BUG_ON(offsetof(struct bpf_cpumask, cpumask) != 0); | |
59 | ||
5d5de3a4 | 60 | cpumask = bpf_mem_cache_alloc(&bpf_cpumask_ma); |
516f4d33 DV |
61 | if (!cpumask) |
62 | return NULL; | |
63 | ||
64 | memset(cpumask, 0, sizeof(*cpumask)); | |
65 | refcount_set(&cpumask->usage, 1); | |
66 | ||
67 | return cpumask; | |
68 | } | |
69 | ||
bdbda395 DV |
70 | /** |
71 | * bpf_cpumask_acquire() - Acquire a reference to a BPF cpumask. | |
72 | * @cpumask: The BPF cpumask being acquired. The cpumask must be a trusted | |
73 | * pointer. | |
74 | * | |
75 | * Acquires a reference to a BPF cpumask. The cpumask returned by this function | |
76 | * must either be embedded in a map as a kptr, or freed with | |
77 | * bpf_cpumask_release(). | |
78 | */ | |
400031e0 | 79 | __bpf_kfunc struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask) |
516f4d33 DV |
80 | { |
81 | refcount_inc(&cpumask->usage); | |
82 | return cpumask; | |
83 | } | |
84 | ||
77473d1a DV |
85 | static void cpumask_free_cb(struct rcu_head *head) |
86 | { | |
87 | struct bpf_cpumask *cpumask; | |
88 | ||
89 | cpumask = container_of(head, struct bpf_cpumask, rcu); | |
90 | migrate_disable(); | |
91 | bpf_mem_cache_free(&bpf_cpumask_ma, cpumask); | |
92 | migrate_enable(); | |
93 | } | |
94 | ||
bdbda395 DV |
95 | /** |
96 | * bpf_cpumask_release() - Release a previously acquired BPF cpumask. | |
97 | * @cpumask: The cpumask being released. | |
98 | * | |
99 | * Releases a previously acquired reference to a BPF cpumask. When the final | |
100 | * reference of the BPF cpumask has been released, it is subsequently freed in | |
101 | * an RCU callback in the BPF memory allocator. | |
102 | */ | |
400031e0 | 103 | __bpf_kfunc void bpf_cpumask_release(struct bpf_cpumask *cpumask) |
516f4d33 | 104 | { |
77473d1a DV |
105 | if (refcount_dec_and_test(&cpumask->usage)) |
106 | call_rcu(&cpumask->rcu, cpumask_free_cb); | |
516f4d33 DV |
107 | } |
108 | ||
bdbda395 DV |
109 | /** |
110 | * bpf_cpumask_first() - Get the index of the first nonzero bit in the cpumask. | |
111 | * @cpumask: The cpumask being queried. | |
112 | * | |
113 | * Find the index of the first nonzero bit of the cpumask. A struct bpf_cpumask | |
114 | * pointer may be safely passed to this function. | |
115 | */ | |
400031e0 | 116 | __bpf_kfunc u32 bpf_cpumask_first(const struct cpumask *cpumask) |
516f4d33 DV |
117 | { |
118 | return cpumask_first(cpumask); | |
119 | } | |
120 | ||
bdbda395 DV |
121 | /** |
122 | * bpf_cpumask_first_zero() - Get the index of the first unset bit in the | |
123 | * cpumask. | |
124 | * @cpumask: The cpumask being queried. | |
125 | * | |
126 | * Find the index of the first unset bit of the cpumask. A struct bpf_cpumask | |
127 | * pointer may be safely passed to this function. | |
128 | */ | |
400031e0 | 129 | __bpf_kfunc u32 bpf_cpumask_first_zero(const struct cpumask *cpumask) |
516f4d33 DV |
130 | { |
131 | return cpumask_first_zero(cpumask); | |
132 | } | |
133 | ||
bdbda395 DV |
134 | /** |
135 | * bpf_cpumask_set_cpu() - Set a bit for a CPU in a BPF cpumask. | |
136 | * @cpu: The CPU to be set in the cpumask. | |
137 | * @cpumask: The BPF cpumask in which a bit is being set. | |
138 | */ | |
400031e0 | 139 | __bpf_kfunc void bpf_cpumask_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) |
516f4d33 DV |
140 | { |
141 | if (!cpu_valid(cpu)) | |
142 | return; | |
143 | ||
144 | cpumask_set_cpu(cpu, (struct cpumask *)cpumask); | |
145 | } | |
146 | ||
bdbda395 DV |
147 | /** |
148 | * bpf_cpumask_clear_cpu() - Clear a bit for a CPU in a BPF cpumask. | |
149 | * @cpu: The CPU to be cleared from the cpumask. | |
150 | * @cpumask: The BPF cpumask in which a bit is being cleared. | |
151 | */ | |
400031e0 | 152 | __bpf_kfunc void bpf_cpumask_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) |
516f4d33 DV |
153 | { |
154 | if (!cpu_valid(cpu)) | |
155 | return; | |
156 | ||
157 | cpumask_clear_cpu(cpu, (struct cpumask *)cpumask); | |
158 | } | |
159 | ||
bdbda395 DV |
160 | /** |
161 | * bpf_cpumask_test_cpu() - Test whether a CPU is set in a cpumask. | |
162 | * @cpu: The CPU being queried for. | |
163 | * @cpumask: The cpumask being queried for containing a CPU. | |
164 | * | |
165 | * Return: | |
166 | * * true - @cpu is set in the cpumask | |
167 | * * false - @cpu was not set in the cpumask, or @cpu is an invalid cpu. | |
168 | */ | |
400031e0 | 169 | __bpf_kfunc bool bpf_cpumask_test_cpu(u32 cpu, const struct cpumask *cpumask) |
516f4d33 DV |
170 | { |
171 | if (!cpu_valid(cpu)) | |
172 | return false; | |
173 | ||
174 | return cpumask_test_cpu(cpu, (struct cpumask *)cpumask); | |
175 | } | |
176 | ||
bdbda395 DV |
177 | /** |
178 | * bpf_cpumask_test_and_set_cpu() - Atomically test and set a CPU in a BPF cpumask. | |
179 | * @cpu: The CPU being set and queried for. | |
180 | * @cpumask: The BPF cpumask being set and queried for containing a CPU. | |
181 | * | |
182 | * Return: | |
183 | * * true - @cpu is set in the cpumask | |
184 | * * false - @cpu was not set in the cpumask, or @cpu is invalid. | |
185 | */ | |
400031e0 | 186 | __bpf_kfunc bool bpf_cpumask_test_and_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) |
516f4d33 DV |
187 | { |
188 | if (!cpu_valid(cpu)) | |
189 | return false; | |
190 | ||
191 | return cpumask_test_and_set_cpu(cpu, (struct cpumask *)cpumask); | |
192 | } | |
193 | ||
bdbda395 DV |
194 | /** |
195 | * bpf_cpumask_test_and_clear_cpu() - Atomically test and clear a CPU in a BPF | |
196 | * cpumask. | |
197 | * @cpu: The CPU being cleared and queried for. | |
198 | * @cpumask: The BPF cpumask being cleared and queried for containing a CPU. | |
199 | * | |
200 | * Return: | |
201 | * * true - @cpu is set in the cpumask | |
202 | * * false - @cpu was not set in the cpumask, or @cpu is invalid. | |
203 | */ | |
400031e0 | 204 | __bpf_kfunc bool bpf_cpumask_test_and_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) |
516f4d33 DV |
205 | { |
206 | if (!cpu_valid(cpu)) | |
207 | return false; | |
208 | ||
209 | return cpumask_test_and_clear_cpu(cpu, (struct cpumask *)cpumask); | |
210 | } | |
211 | ||
bdbda395 DV |
212 | /** |
213 | * bpf_cpumask_setall() - Set all of the bits in a BPF cpumask. | |
214 | * @cpumask: The BPF cpumask having all of its bits set. | |
215 | */ | |
400031e0 | 216 | __bpf_kfunc void bpf_cpumask_setall(struct bpf_cpumask *cpumask) |
516f4d33 DV |
217 | { |
218 | cpumask_setall((struct cpumask *)cpumask); | |
219 | } | |
220 | ||
bdbda395 DV |
221 | /** |
222 | * bpf_cpumask_clear() - Clear all of the bits in a BPF cpumask. | |
223 | * @cpumask: The BPF cpumask being cleared. | |
224 | */ | |
400031e0 | 225 | __bpf_kfunc void bpf_cpumask_clear(struct bpf_cpumask *cpumask) |
516f4d33 DV |
226 | { |
227 | cpumask_clear((struct cpumask *)cpumask); | |
228 | } | |
229 | ||
bdbda395 DV |
230 | /** |
231 | * bpf_cpumask_and() - AND two cpumasks and store the result. | |
232 | * @dst: The BPF cpumask where the result is being stored. | |
233 | * @src1: The first input. | |
234 | * @src2: The second input. | |
235 | * | |
236 | * Return: | |
237 | * * true - @dst has at least one bit set following the operation | |
238 | * * false - @dst is empty following the operation | |
239 | * | |
240 | * struct bpf_cpumask pointers may be safely passed to @src1 and @src2. | |
241 | */ | |
400031e0 DV |
242 | __bpf_kfunc bool bpf_cpumask_and(struct bpf_cpumask *dst, |
243 | const struct cpumask *src1, | |
244 | const struct cpumask *src2) | |
516f4d33 DV |
245 | { |
246 | return cpumask_and((struct cpumask *)dst, src1, src2); | |
247 | } | |
248 | ||
bdbda395 DV |
249 | /** |
250 | * bpf_cpumask_or() - OR two cpumasks and store the result. | |
251 | * @dst: The BPF cpumask where the result is being stored. | |
252 | * @src1: The first input. | |
253 | * @src2: The second input. | |
254 | * | |
255 | * struct bpf_cpumask pointers may be safely passed to @src1 and @src2. | |
256 | */ | |
400031e0 DV |
257 | __bpf_kfunc void bpf_cpumask_or(struct bpf_cpumask *dst, |
258 | const struct cpumask *src1, | |
259 | const struct cpumask *src2) | |
516f4d33 DV |
260 | { |
261 | cpumask_or((struct cpumask *)dst, src1, src2); | |
262 | } | |
263 | ||
bdbda395 DV |
264 | /** |
265 | * bpf_cpumask_xor() - XOR two cpumasks and store the result. | |
266 | * @dst: The BPF cpumask where the result is being stored. | |
267 | * @src1: The first input. | |
268 | * @src2: The second input. | |
269 | * | |
270 | * struct bpf_cpumask pointers may be safely passed to @src1 and @src2. | |
271 | */ | |
400031e0 DV |
272 | __bpf_kfunc void bpf_cpumask_xor(struct bpf_cpumask *dst, |
273 | const struct cpumask *src1, | |
274 | const struct cpumask *src2) | |
516f4d33 DV |
275 | { |
276 | cpumask_xor((struct cpumask *)dst, src1, src2); | |
277 | } | |
278 | ||
bdbda395 DV |
279 | /** |
280 | * bpf_cpumask_equal() - Check two cpumasks for equality. | |
281 | * @src1: The first input. | |
282 | * @src2: The second input. | |
283 | * | |
284 | * Return: | |
285 | * * true - @src1 and @src2 have the same bits set. | |
286 | * * false - @src1 and @src2 differ in at least one bit. | |
287 | * | |
288 | * struct bpf_cpumask pointers may be safely passed to @src1 and @src2. | |
289 | */ | |
400031e0 | 290 | __bpf_kfunc bool bpf_cpumask_equal(const struct cpumask *src1, const struct cpumask *src2) |
516f4d33 DV |
291 | { |
292 | return cpumask_equal(src1, src2); | |
293 | } | |
294 | ||
bdbda395 DV |
295 | /** |
296 | * bpf_cpumask_intersects() - Check two cpumasks for overlap. | |
297 | * @src1: The first input. | |
298 | * @src2: The second input. | |
299 | * | |
300 | * Return: | |
301 | * * true - @src1 and @src2 have at least one of the same bits set. | |
302 | * * false - @src1 and @src2 don't have any of the same bits set. | |
303 | * | |
304 | * struct bpf_cpumask pointers may be safely passed to @src1 and @src2. | |
305 | */ | |
400031e0 | 306 | __bpf_kfunc bool bpf_cpumask_intersects(const struct cpumask *src1, const struct cpumask *src2) |
516f4d33 DV |
307 | { |
308 | return cpumask_intersects(src1, src2); | |
309 | } | |
310 | ||
bdbda395 DV |
311 | /** |
312 | * bpf_cpumask_subset() - Check if a cpumask is a subset of another. | |
313 | * @src1: The first cpumask being checked as a subset. | |
314 | * @src2: The second cpumask being checked as a superset. | |
315 | * | |
316 | * Return: | |
317 | * * true - All of the bits of @src1 are set in @src2. | |
318 | * * false - At least one bit in @src1 is not set in @src2. | |
319 | * | |
320 | * struct bpf_cpumask pointers may be safely passed to @src1 and @src2. | |
321 | */ | |
400031e0 | 322 | __bpf_kfunc bool bpf_cpumask_subset(const struct cpumask *src1, const struct cpumask *src2) |
516f4d33 DV |
323 | { |
324 | return cpumask_subset(src1, src2); | |
325 | } | |
326 | ||
bdbda395 DV |
327 | /** |
328 | * bpf_cpumask_empty() - Check if a cpumask is empty. | |
329 | * @cpumask: The cpumask being checked. | |
330 | * | |
331 | * Return: | |
332 | * * true - None of the bits in @cpumask are set. | |
333 | * * false - At least one bit in @cpumask is set. | |
334 | * | |
335 | * A struct bpf_cpumask pointer may be safely passed to @cpumask. | |
336 | */ | |
400031e0 | 337 | __bpf_kfunc bool bpf_cpumask_empty(const struct cpumask *cpumask) |
516f4d33 DV |
338 | { |
339 | return cpumask_empty(cpumask); | |
340 | } | |
341 | ||
bdbda395 DV |
342 | /** |
343 | * bpf_cpumask_full() - Check if a cpumask has all bits set. | |
344 | * @cpumask: The cpumask being checked. | |
345 | * | |
346 | * Return: | |
347 | * * true - All of the bits in @cpumask are set. | |
348 | * * false - At least one bit in @cpumask is cleared. | |
349 | * | |
350 | * A struct bpf_cpumask pointer may be safely passed to @cpumask. | |
351 | */ | |
400031e0 | 352 | __bpf_kfunc bool bpf_cpumask_full(const struct cpumask *cpumask) |
516f4d33 DV |
353 | { |
354 | return cpumask_full(cpumask); | |
355 | } | |
356 | ||
bdbda395 DV |
357 | /** |
358 | * bpf_cpumask_copy() - Copy the contents of a cpumask into a BPF cpumask. | |
359 | * @dst: The BPF cpumask being copied into. | |
360 | * @src: The cpumask being copied. | |
361 | * | |
362 | * A struct bpf_cpumask pointer may be safely passed to @src. | |
363 | */ | |
400031e0 | 364 | __bpf_kfunc void bpf_cpumask_copy(struct bpf_cpumask *dst, const struct cpumask *src) |
516f4d33 DV |
365 | { |
366 | cpumask_copy((struct cpumask *)dst, src); | |
367 | } | |
368 | ||
bdbda395 DV |
369 | /** |
370 | * bpf_cpumask_any() - Return a random set CPU from a cpumask. | |
371 | * @cpumask: The cpumask being queried. | |
372 | * | |
373 | * Return: | |
374 | * * A random set bit within [0, num_cpus) if at least one bit is set. | |
375 | * * >= num_cpus if no bit is set. | |
376 | * | |
377 | * A struct bpf_cpumask pointer may be safely passed to @src. | |
378 | */ | |
400031e0 | 379 | __bpf_kfunc u32 bpf_cpumask_any(const struct cpumask *cpumask) |
516f4d33 DV |
380 | { |
381 | return cpumask_any(cpumask); | |
382 | } | |
383 | ||
bdbda395 DV |
384 | /** |
385 | * bpf_cpumask_any_and() - Return a random set CPU from the AND of two | |
386 | * cpumasks. | |
387 | * @src1: The first cpumask. | |
388 | * @src2: The second cpumask. | |
389 | * | |
390 | * Return: | |
391 | * * A random set bit within [0, num_cpus) if at least one bit is set. | |
392 | * * >= num_cpus if no bit is set. | |
393 | * | |
394 | * struct bpf_cpumask pointers may be safely passed to @src1 and @src2. | |
395 | */ | |
400031e0 | 396 | __bpf_kfunc u32 bpf_cpumask_any_and(const struct cpumask *src1, const struct cpumask *src2) |
516f4d33 DV |
397 | { |
398 | return cpumask_any_and(src1, src2); | |
399 | } | |
400 | ||
401 | __diag_pop(); | |
402 | ||
403 | BTF_SET8_START(cpumask_kfunc_btf_ids) | |
404 | BTF_ID_FLAGS(func, bpf_cpumask_create, KF_ACQUIRE | KF_RET_NULL) | |
6c831c46 | 405 | BTF_ID_FLAGS(func, bpf_cpumask_release, KF_RELEASE) |
516f4d33 | 406 | BTF_ID_FLAGS(func, bpf_cpumask_acquire, KF_ACQUIRE | KF_TRUSTED_ARGS) |
6fcd486b AS |
407 | BTF_ID_FLAGS(func, bpf_cpumask_first, KF_RCU) |
408 | BTF_ID_FLAGS(func, bpf_cpumask_first_zero, KF_RCU) | |
409 | BTF_ID_FLAGS(func, bpf_cpumask_set_cpu, KF_RCU) | |
410 | BTF_ID_FLAGS(func, bpf_cpumask_clear_cpu, KF_RCU) | |
411 | BTF_ID_FLAGS(func, bpf_cpumask_test_cpu, KF_RCU) | |
412 | BTF_ID_FLAGS(func, bpf_cpumask_test_and_set_cpu, KF_RCU) | |
413 | BTF_ID_FLAGS(func, bpf_cpumask_test_and_clear_cpu, KF_RCU) | |
414 | BTF_ID_FLAGS(func, bpf_cpumask_setall, KF_RCU) | |
415 | BTF_ID_FLAGS(func, bpf_cpumask_clear, KF_RCU) | |
416 | BTF_ID_FLAGS(func, bpf_cpumask_and, KF_RCU) | |
417 | BTF_ID_FLAGS(func, bpf_cpumask_or, KF_RCU) | |
418 | BTF_ID_FLAGS(func, bpf_cpumask_xor, KF_RCU) | |
419 | BTF_ID_FLAGS(func, bpf_cpumask_equal, KF_RCU) | |
420 | BTF_ID_FLAGS(func, bpf_cpumask_intersects, KF_RCU) | |
421 | BTF_ID_FLAGS(func, bpf_cpumask_subset, KF_RCU) | |
422 | BTF_ID_FLAGS(func, bpf_cpumask_empty, KF_RCU) | |
423 | BTF_ID_FLAGS(func, bpf_cpumask_full, KF_RCU) | |
424 | BTF_ID_FLAGS(func, bpf_cpumask_copy, KF_RCU) | |
425 | BTF_ID_FLAGS(func, bpf_cpumask_any, KF_RCU) | |
426 | BTF_ID_FLAGS(func, bpf_cpumask_any_and, KF_RCU) | |
516f4d33 DV |
427 | BTF_SET8_END(cpumask_kfunc_btf_ids) |
428 | ||
429 | static const struct btf_kfunc_id_set cpumask_kfunc_set = { | |
430 | .owner = THIS_MODULE, | |
431 | .set = &cpumask_kfunc_btf_ids, | |
432 | }; | |
433 | ||
434 | BTF_ID_LIST(cpumask_dtor_ids) | |
435 | BTF_ID(struct, bpf_cpumask) | |
436 | BTF_ID(func, bpf_cpumask_release) | |
437 | ||
438 | static int __init cpumask_kfunc_init(void) | |
439 | { | |
440 | int ret; | |
441 | const struct btf_id_dtor_kfunc cpumask_dtors[] = { | |
442 | { | |
443 | .btf_id = cpumask_dtor_ids[0], | |
444 | .kfunc_btf_id = cpumask_dtor_ids[1] | |
445 | }, | |
446 | }; | |
447 | ||
5d5de3a4 | 448 | ret = bpf_mem_alloc_init(&bpf_cpumask_ma, sizeof(struct bpf_cpumask), false); |
516f4d33 DV |
449 | ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &cpumask_kfunc_set); |
450 | ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &cpumask_kfunc_set); | |
451 | return ret ?: register_btf_id_dtor_kfuncs(cpumask_dtors, | |
452 | ARRAY_SIZE(cpumask_dtors), | |
453 | THIS_MODULE); | |
454 | } | |
455 | ||
456 | late_initcall(cpumask_kfunc_init); |