projects / linux-block.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
Merge tag 'mm-hotfixes-stable-2025-07-11-16-16' of git://git.kernel.org/pub/scm/linux...
[linux-block.git] / include / linux / cpumask.h
... / ...
CommitLineData
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef __LINUX_CPUMASK_H
3#define __LINUX_CPUMASK_H
4
5/*
6 * Cpumasks provide a bitmap suitable for representing the
7 * set of CPUs in a system, one bit position per CPU number. In general,
8 * only nr_cpu_ids (<= NR_CPUS) bits are valid.
9 */
10#include <linux/cleanup.h>
11#include <linux/kernel.h>
12#include <linux/bitmap.h>
13#include <linux/cpumask_types.h>
14#include <linux/atomic.h>
15#include <linux/bug.h>
16#include <linux/gfp_types.h>
17#include <linux/numa.h>
18
19/**
20 * cpumask_pr_args - printf args to output a cpumask
21 * @maskp: cpumask to be printed
22 *
23 * Can be used to provide arguments for '%*pb[l]' when printing a cpumask.
24 */
25#define cpumask_pr_args(maskp) nr_cpu_ids, cpumask_bits(maskp)
26
27#if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS)
28#define nr_cpu_ids ((unsigned int)NR_CPUS)
29#else
30extern unsigned int nr_cpu_ids;
31#endif
32
33static __always_inline void set_nr_cpu_ids(unsigned int nr)
34{
35#if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS)
36 WARN_ON(nr != nr_cpu_ids);
37#else
38 nr_cpu_ids = nr;
39#endif
40}
41
42/*
43 * We have several different "preferred sizes" for the cpumask
44 * operations, depending on operation.
45 *
46 * For example, the bitmap scanning and operating operations have
47 * optimized routines that work for the single-word case, but only when
48 * the size is constant. So if NR_CPUS fits in one single word, we are
49 * better off using that small constant, in order to trigger the
50 * optimized bit finding. That is 'small_cpumask_size'.
51 *
52 * The clearing and copying operations will similarly perform better
53 * with a constant size, but we limit that size arbitrarily to four
54 * words. We call this 'large_cpumask_size'.
55 *
56 * Finally, some operations just want the exact limit, either because
57 * they set bits or just don't have any faster fixed-sized versions. We
58 * call this just 'nr_cpumask_bits'.
59 *
60 * Note that these optional constants are always guaranteed to be at
61 * least as big as 'nr_cpu_ids' itself is, and all our cpumask
62 * allocations are at least that size (see cpumask_size()). The
63 * optimization comes from being able to potentially use a compile-time
64 * constant instead of a run-time generated exact number of CPUs.
65 */
66#if NR_CPUS <= BITS_PER_LONG
67 #define small_cpumask_bits ((unsigned int)NR_CPUS)
68 #define large_cpumask_bits ((unsigned int)NR_CPUS)
69#elif NR_CPUS <= 4*BITS_PER_LONG
70 #define small_cpumask_bits nr_cpu_ids
71 #define large_cpumask_bits ((unsigned int)NR_CPUS)
72#else
73 #define small_cpumask_bits nr_cpu_ids
74 #define large_cpumask_bits nr_cpu_ids
75#endif
76#define nr_cpumask_bits nr_cpu_ids
77
78/*
79 * The following particular system cpumasks and operations manage
80 * possible, present, active and online cpus.
81 *
82 * cpu_possible_mask- has bit 'cpu' set iff cpu is populatable
83 * cpu_present_mask - has bit 'cpu' set iff cpu is populated
84 * cpu_enabled_mask - has bit 'cpu' set iff cpu can be brought online
85 * cpu_online_mask - has bit 'cpu' set iff cpu available to scheduler
86 * cpu_active_mask - has bit 'cpu' set iff cpu available to migration
87 *
88 * If !CONFIG_HOTPLUG_CPU, present == possible, and active == online.
89 *
90 * The cpu_possible_mask is fixed at boot time, as the set of CPU IDs
91 * that it is possible might ever be plugged in at anytime during the
92 * life of that system boot. The cpu_present_mask is dynamic(*),
93 * representing which CPUs are currently plugged in. And
94 * cpu_online_mask is the dynamic subset of cpu_present_mask,
95 * indicating those CPUs available for scheduling.
96 *
97 * If HOTPLUG is enabled, then cpu_present_mask varies dynamically,
98 * depending on what ACPI reports as currently plugged in, otherwise
99 * cpu_present_mask is just a copy of cpu_possible_mask.
100 *
101 * (*) Well, cpu_present_mask is dynamic in the hotplug case. If not
102 * hotplug, it's a copy of cpu_possible_mask, hence fixed at boot.
103 *
104 * Subtleties:
105 * 1) UP ARCHes (NR_CPUS == 1, CONFIG_SMP not defined) hardcode
106 * assumption that their single CPU is online. The UP
107 * cpu_{online,possible,present}_masks are placebos. Changing them
108 * will have no useful affect on the following num_*_cpus()
109 * and cpu_*() macros in the UP case. This ugliness is a UP
110 * optimization - don't waste any instructions or memory references
111 * asking if you're online or how many CPUs there are if there is
112 * only one CPU.
113 */
114
115extern struct cpumask __cpu_possible_mask;
116extern struct cpumask __cpu_online_mask;
117extern struct cpumask __cpu_enabled_mask;
118extern struct cpumask __cpu_present_mask;
119extern struct cpumask __cpu_active_mask;
120extern struct cpumask __cpu_dying_mask;
121#define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask)
122#define cpu_online_mask ((const struct cpumask *)&__cpu_online_mask)
123#define cpu_enabled_mask ((const struct cpumask *)&__cpu_enabled_mask)
124#define cpu_present_mask ((const struct cpumask *)&__cpu_present_mask)
125#define cpu_active_mask ((const struct cpumask *)&__cpu_active_mask)
126#define cpu_dying_mask ((const struct cpumask *)&__cpu_dying_mask)
127
128extern atomic_t __num_online_cpus;
129
130extern cpumask_t cpus_booted_once_mask;
131
132static __always_inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits)
133{
134#ifdef CONFIG_DEBUG_PER_CPU_MAPS
135 WARN_ON_ONCE(cpu >= bits);
136#endif /* CONFIG_DEBUG_PER_CPU_MAPS */
137}
138
139/* verify cpu argument to cpumask_* operators */
140static __always_inline unsigned int cpumask_check(unsigned int cpu)
141{
142 cpu_max_bits_warn(cpu, small_cpumask_bits);
143 return cpu;
144}
145
146/**
147 * cpumask_first - get the first cpu in a cpumask
148 * @srcp: the cpumask pointer
149 *
150 * Return: >= nr_cpu_ids if no cpus set.
151 */
152static __always_inline unsigned int cpumask_first(const struct cpumask *srcp)
153{
154 return find_first_bit(cpumask_bits(srcp), small_cpumask_bits);
155}
156
157/**
158 * cpumask_first_zero - get the first unset cpu in a cpumask
159 * @srcp: the cpumask pointer
160 *
161 * Return: >= nr_cpu_ids if all cpus are set.
162 */
163static __always_inline unsigned int cpumask_first_zero(const struct cpumask *srcp)
164{
165 return find_first_zero_bit(cpumask_bits(srcp), small_cpumask_bits);
166}
167
168/**
169 * cpumask_first_and - return the first cpu from *srcp1 & *srcp2
170 * @srcp1: the first input
171 * @srcp2: the second input
172 *
173 * Return: >= nr_cpu_ids if no cpus set in both. See also cpumask_next_and().
174 */
175static __always_inline
176unsigned int cpumask_first_and(const struct cpumask *srcp1, const struct cpumask *srcp2)
177{
178 return find_first_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits);
179}
180
181/**
182 * cpumask_first_andnot - return the first cpu from *srcp1 & ~*srcp2
183 * @srcp1: the first input
184 * @srcp2: the second input
185 *
186 * Return: >= nr_cpu_ids if no such cpu found.
187 */
188static __always_inline
189unsigned int cpumask_first_andnot(const struct cpumask *srcp1, const struct cpumask *srcp2)
190{
191 return find_first_andnot_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits);
192}
193
194/**
195 * cpumask_first_and_and - return the first cpu from *srcp1 & *srcp2 & *srcp3
196 * @srcp1: the first input
197 * @srcp2: the second input
198 * @srcp3: the third input
199 *
200 * Return: >= nr_cpu_ids if no cpus set in all.
201 */
202static __always_inline
203unsigned int cpumask_first_and_and(const struct cpumask *srcp1,
204 const struct cpumask *srcp2,
205 const struct cpumask *srcp3)
206{
207 return find_first_and_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2),
208 cpumask_bits(srcp3), small_cpumask_bits);
209}
210
211/**
212 * cpumask_last - get the last CPU in a cpumask
213 * @srcp: - the cpumask pointer
214 *
215 * Return: >= nr_cpumask_bits if no CPUs set.
216 */
217static __always_inline unsigned int cpumask_last(const struct cpumask *srcp)
218{
219 return find_last_bit(cpumask_bits(srcp), small_cpumask_bits);
220}
221
222/**
223 * cpumask_next - get the next cpu in a cpumask
224 * @n: the cpu prior to the place to search (i.e. return will be > @n)
225 * @srcp: the cpumask pointer
226 *
227 * Return: >= nr_cpu_ids if no further cpus set.
228 */
229static __always_inline
230unsigned int cpumask_next(int n, const struct cpumask *srcp)
231{
232 /* -1 is a legal arg here. */
233 if (n != -1)
234 cpumask_check(n);
235 return find_next_bit(cpumask_bits(srcp), small_cpumask_bits, n + 1);
236}
237
238/**
239 * cpumask_next_zero - get the next unset cpu in a cpumask
240 * @n: the cpu prior to the place to search (i.e. return will be > @n)
241 * @srcp: the cpumask pointer
242 *
243 * Return: >= nr_cpu_ids if no further cpus unset.
244 */
245static __always_inline
246unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
247{
248 /* -1 is a legal arg here. */
249 if (n != -1)
250 cpumask_check(n);
251 return find_next_zero_bit(cpumask_bits(srcp), small_cpumask_bits, n+1);
252}
253
254#if NR_CPUS == 1
255/* Uniprocessor: there is only one valid CPU */
256static __always_inline
257unsigned int cpumask_local_spread(unsigned int i, int node)
258{
259 return 0;
260}
261
262static __always_inline
263unsigned int cpumask_any_and_distribute(const struct cpumask *src1p,
264 const struct cpumask *src2p)
265{
266 return cpumask_first_and(src1p, src2p);
267}
268
269static __always_inline
270unsigned int cpumask_any_distribute(const struct cpumask *srcp)
271{
272 return cpumask_first(srcp);
273}
274#else
275unsigned int cpumask_local_spread(unsigned int i, int node);
276unsigned int cpumask_any_and_distribute(const struct cpumask *src1p,
277 const struct cpumask *src2p);
278unsigned int cpumask_any_distribute(const struct cpumask *srcp);
279#endif /* NR_CPUS */
280
281/**
282 * cpumask_next_and - get the next cpu in *src1p & *src2p
283 * @n: the cpu prior to the place to search (i.e. return will be > @n)
284 * @src1p: the first cpumask pointer
285 * @src2p: the second cpumask pointer
286 *
287 * Return: >= nr_cpu_ids if no further cpus set in both.
288 */
289static __always_inline
290unsigned int cpumask_next_and(int n, const struct cpumask *src1p,
291 const struct cpumask *src2p)
292{
293 /* -1 is a legal arg here. */
294 if (n != -1)
295 cpumask_check(n);
296 return find_next_and_bit(cpumask_bits(src1p), cpumask_bits(src2p),
297 small_cpumask_bits, n + 1);
298}
299
300/**
301 * cpumask_next_andnot - get the next cpu in *src1p & ~*src2p
302 * @n: the cpu prior to the place to search (i.e. return will be > @n)
303 * @src1p: the first cpumask pointer
304 * @src2p: the second cpumask pointer
305 *
306 * Return: >= nr_cpu_ids if no further cpus set in both.
307 */
308static __always_inline
309unsigned int cpumask_next_andnot(int n, const struct cpumask *src1p,
310 const struct cpumask *src2p)
311{
312 /* -1 is a legal arg here. */
313 if (n != -1)
314 cpumask_check(n);
315 return find_next_andnot_bit(cpumask_bits(src1p), cpumask_bits(src2p),
316 small_cpumask_bits, n + 1);
317}
318
319/**
320 * cpumask_next_and_wrap - get the next cpu in *src1p & *src2p, starting from
321 * @n+1. If nothing found, wrap around and start from
322 * the beginning
323 * @n: the cpu prior to the place to search (i.e. search starts from @n+1)
324 * @src1p: the first cpumask pointer
325 * @src2p: the second cpumask pointer
326 *
327 * Return: next set bit, wrapped if needed, or >= nr_cpu_ids if @src1p & @src2p is empty.
328 */
329static __always_inline
330unsigned int cpumask_next_and_wrap(int n, const struct cpumask *src1p,
331 const struct cpumask *src2p)
332{
333 /* -1 is a legal arg here. */
334 if (n != -1)
335 cpumask_check(n);
336 return find_next_and_bit_wrap(cpumask_bits(src1p), cpumask_bits(src2p),
337 small_cpumask_bits, n + 1);
338}
339
340/**
341 * cpumask_next_wrap - get the next cpu in *src, starting from @n+1. If nothing
342 * found, wrap around and start from the beginning
343 * @n: the cpu prior to the place to search (i.e. search starts from @n+1)
344 * @src: cpumask pointer
345 *
346 * Return: next set bit, wrapped if needed, or >= nr_cpu_ids if @src is empty.
347 */
348static __always_inline
349unsigned int cpumask_next_wrap(int n, const struct cpumask *src)
350{
351 /* -1 is a legal arg here. */
352 if (n != -1)
353 cpumask_check(n);
354 return find_next_bit_wrap(cpumask_bits(src), small_cpumask_bits, n + 1);
355}
356
357/**
358 * for_each_cpu - iterate over every cpu in a mask
359 * @cpu: the (optionally unsigned) integer iterator
360 * @mask: the cpumask pointer
361 *
362 * After the loop, cpu is >= nr_cpu_ids.
363 */
364#define for_each_cpu(cpu, mask) \
365 for_each_set_bit(cpu, cpumask_bits(mask), small_cpumask_bits)
366
367/**
368 * for_each_cpu_wrap - iterate over every cpu in a mask, starting at a specified location
369 * @cpu: the (optionally unsigned) integer iterator
370 * @mask: the cpumask pointer
371 * @start: the start location
372 *
373 * The implementation does not assume any bit in @mask is set (including @start).
374 *
375 * After the loop, cpu is >= nr_cpu_ids.
376 */
377#define for_each_cpu_wrap(cpu, mask, start) \
378 for_each_set_bit_wrap(cpu, cpumask_bits(mask), small_cpumask_bits, start)
379
380/**
381 * for_each_cpu_and - iterate over every cpu in both masks
382 * @cpu: the (optionally unsigned) integer iterator
383 * @mask1: the first cpumask pointer
384 * @mask2: the second cpumask pointer
385 *
386 * This saves a temporary CPU mask in many places. It is equivalent to:
387 * struct cpumask tmp;
388 * cpumask_and(&tmp, &mask1, &mask2);
389 * for_each_cpu(cpu, &tmp)
390 * ...
391 *
392 * After the loop, cpu is >= nr_cpu_ids.
393 */
394#define for_each_cpu_and(cpu, mask1, mask2) \
395 for_each_and_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
396
397/**
398 * for_each_cpu_andnot - iterate over every cpu present in one mask, excluding
399 * those present in another.
400 * @cpu: the (optionally unsigned) integer iterator
401 * @mask1: the first cpumask pointer
402 * @mask2: the second cpumask pointer
403 *
404 * This saves a temporary CPU mask in many places. It is equivalent to:
405 * struct cpumask tmp;
406 * cpumask_andnot(&tmp, &mask1, &mask2);
407 * for_each_cpu(cpu, &tmp)
408 * ...
409 *
410 * After the loop, cpu is >= nr_cpu_ids.
411 */
412#define for_each_cpu_andnot(cpu, mask1, mask2) \
413 for_each_andnot_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
414
415/**
416 * for_each_cpu_or - iterate over every cpu present in either mask
417 * @cpu: the (optionally unsigned) integer iterator
418 * @mask1: the first cpumask pointer
419 * @mask2: the second cpumask pointer
420 *
421 * This saves a temporary CPU mask in many places. It is equivalent to:
422 * struct cpumask tmp;
423 * cpumask_or(&tmp, &mask1, &mask2);
424 * for_each_cpu(cpu, &tmp)
425 * ...
426 *
427 * After the loop, cpu is >= nr_cpu_ids.
428 */
429#define for_each_cpu_or(cpu, mask1, mask2) \
430 for_each_or_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
431
432/**
433 * for_each_cpu_from - iterate over CPUs present in @mask, from @cpu to the end of @mask.
434 * @cpu: the (optionally unsigned) integer iterator
435 * @mask: the cpumask pointer
436 *
437 * After the loop, cpu is >= nr_cpu_ids.
438 */
439#define for_each_cpu_from(cpu, mask) \
440 for_each_set_bit_from(cpu, cpumask_bits(mask), small_cpumask_bits)
441
442/**
443 * cpumask_any_but - return an arbitrary cpu in a cpumask, but not this one.
444 * @mask: the cpumask to search
445 * @cpu: the cpu to ignore.
446 *
447 * Often used to find any cpu but smp_processor_id() in a mask.
448 * If @cpu == -1, the function is equivalent to cpumask_any().
449 * Return: >= nr_cpu_ids if no cpus set.
450 */
451static __always_inline
452unsigned int cpumask_any_but(const struct cpumask *mask, int cpu)
453{
454 unsigned int i;
455
456 /* -1 is a legal arg here. */
457 if (cpu != -1)
458 cpumask_check(cpu);
459
460 for_each_cpu(i, mask)
461 if (i != cpu)
462 break;
463 return i;
464}
465
466/**
467 * cpumask_any_and_but - pick an arbitrary cpu from *mask1 & *mask2, but not this one.
468 * @mask1: the first input cpumask
469 * @mask2: the second input cpumask
470 * @cpu: the cpu to ignore
471 *
472 * If @cpu == -1, the function is equivalent to cpumask_any_and().
473 * Returns >= nr_cpu_ids if no cpus set.
474 */
475static __always_inline
476unsigned int cpumask_any_and_but(const struct cpumask *mask1,
477 const struct cpumask *mask2,
478 int cpu)
479{
480 unsigned int i;
481
482 /* -1 is a legal arg here. */
483 if (cpu != -1)
484 cpumask_check(cpu);
485
486 i = cpumask_first_and(mask1, mask2);
487 if (i != cpu)
488 return i;
489
490 return cpumask_next_and(cpu, mask1, mask2);
491}
492
493/**
494 * cpumask_any_andnot_but - pick an arbitrary cpu from *mask1 & ~*mask2, but not this one.
495 * @mask1: the first input cpumask
496 * @mask2: the second input cpumask
497 * @cpu: the cpu to ignore
498 *
499 * If @cpu == -1, the function returns the first matching cpu.
500 * Returns >= nr_cpu_ids if no cpus set.
501 */
502static __always_inline
503unsigned int cpumask_any_andnot_but(const struct cpumask *mask1,
504 const struct cpumask *mask2,
505 int cpu)
506{
507 unsigned int i;
508
509 /* -1 is a legal arg here. */
510 if (cpu != -1)
511 cpumask_check(cpu);
512
513 i = cpumask_first_andnot(mask1, mask2);
514 if (i != cpu)
515 return i;
516
517 return cpumask_next_andnot(cpu, mask1, mask2);
518}
519
520/**
521 * cpumask_nth - get the Nth cpu in a cpumask
522 * @srcp: the cpumask pointer
523 * @cpu: the Nth cpu to find, starting from 0
524 *
525 * Return: >= nr_cpu_ids if such cpu doesn't exist.
526 */
527static __always_inline
528unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *srcp)
529{
530 return find_nth_bit(cpumask_bits(srcp), small_cpumask_bits, cpumask_check(cpu));
531}
532
533/**
534 * cpumask_nth_and - get the Nth cpu in 2 cpumasks
535 * @srcp1: the cpumask pointer
536 * @srcp2: the cpumask pointer
537 * @cpu: the Nth cpu to find, starting from 0
538 *
539 * Return: >= nr_cpu_ids if such cpu doesn't exist.
540 */
541static __always_inline
542unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1,
543 const struct cpumask *srcp2)
544{
545 return find_nth_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2),
546 small_cpumask_bits, cpumask_check(cpu));
547}
548
549/**
550 * cpumask_nth_andnot - get the Nth cpu set in 1st cpumask, and clear in 2nd.
551 * @srcp1: the cpumask pointer
552 * @srcp2: the cpumask pointer
553 * @cpu: the Nth cpu to find, starting from 0
554 *
555 * Return: >= nr_cpu_ids if such cpu doesn't exist.
556 */
557static __always_inline
558unsigned int cpumask_nth_andnot(unsigned int cpu, const struct cpumask *srcp1,
559 const struct cpumask *srcp2)
560{
561 return find_nth_andnot_bit(cpumask_bits(srcp1), cpumask_bits(srcp2),
562 small_cpumask_bits, cpumask_check(cpu));
563}
564
565/**
566 * cpumask_nth_and_andnot - get the Nth cpu set in 1st and 2nd cpumask, and clear in 3rd.
567 * @srcp1: the cpumask pointer
568 * @srcp2: the cpumask pointer
569 * @srcp3: the cpumask pointer
570 * @cpu: the Nth cpu to find, starting from 0
571 *
572 * Return: >= nr_cpu_ids if such cpu doesn't exist.
573 */
574static __always_inline
575unsigned int cpumask_nth_and_andnot(unsigned int cpu, const struct cpumask *srcp1,
576 const struct cpumask *srcp2,
577 const struct cpumask *srcp3)
578{
579 return find_nth_and_andnot_bit(cpumask_bits(srcp1),
580 cpumask_bits(srcp2),
581 cpumask_bits(srcp3),
582 small_cpumask_bits, cpumask_check(cpu));
583}
584
585#define CPU_BITS_NONE \
586{ \
587 [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
588}
589
590#define CPU_BITS_CPU0 \
591{ \
592 [0] = 1UL \
593}
594
595/**
596 * cpumask_set_cpu - set a cpu in a cpumask
597 * @cpu: cpu number (< nr_cpu_ids)
598 * @dstp: the cpumask pointer
599 */
600static __always_inline
601void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
602{
603 set_bit(cpumask_check(cpu), cpumask_bits(dstp));
604}
605
606static __always_inline
607void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
608{
609 __set_bit(cpumask_check(cpu), cpumask_bits(dstp));
610}
611
612
613/**
614 * cpumask_clear_cpu - clear a cpu in a cpumask
615 * @cpu: cpu number (< nr_cpu_ids)
616 * @dstp: the cpumask pointer
617 */
618static __always_inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp)
619{
620 clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
621}
622
623static __always_inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp)
624{
625 __clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
626}
627
628/**
629 * cpumask_test_cpu - test for a cpu in a cpumask
630 * @cpu: cpu number (< nr_cpu_ids)
631 * @cpumask: the cpumask pointer
632 *
633 * Return: true if @cpu is set in @cpumask, else returns false
634 */
635static __always_inline
636bool cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
637{
638 return test_bit(cpumask_check(cpu), cpumask_bits((cpumask)));
639}
640
641/**
642 * cpumask_test_and_set_cpu - atomically test and set a cpu in a cpumask
643 * @cpu: cpu number (< nr_cpu_ids)
644 * @cpumask: the cpumask pointer
645 *
646 * test_and_set_bit wrapper for cpumasks.
647 *
648 * Return: true if @cpu is set in old bitmap of @cpumask, else returns false
649 */
650static __always_inline
651bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
652{
653 return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));
654}
655
656/**
657 * cpumask_test_and_clear_cpu - atomically test and clear a cpu in a cpumask
658 * @cpu: cpu number (< nr_cpu_ids)
659 * @cpumask: the cpumask pointer
660 *
661 * test_and_clear_bit wrapper for cpumasks.
662 *
663 * Return: true if @cpu is set in old bitmap of @cpumask, else returns false
664 */
665static __always_inline
666bool cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
667{
668 return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));
669}
670
671/**
672 * cpumask_setall - set all cpus (< nr_cpu_ids) in a cpumask
673 * @dstp: the cpumask pointer
674 */
675static __always_inline void cpumask_setall(struct cpumask *dstp)
676{
677 if (small_const_nbits(small_cpumask_bits)) {
678 cpumask_bits(dstp)[0] = BITMAP_LAST_WORD_MASK(nr_cpumask_bits);
679 return;
680 }
681 bitmap_fill(cpumask_bits(dstp), nr_cpumask_bits);
682}
683
684/**
685 * cpumask_clear - clear all cpus (< nr_cpu_ids) in a cpumask
686 * @dstp: the cpumask pointer
687 */
688static __always_inline void cpumask_clear(struct cpumask *dstp)
689{
690 bitmap_zero(cpumask_bits(dstp), large_cpumask_bits);
691}
692
693/**
694 * cpumask_and - *dstp = *src1p & *src2p
695 * @dstp: the cpumask result
696 * @src1p: the first input
697 * @src2p: the second input
698 *
699 * Return: false if *@dstp is empty, else returns true
700 */
701static __always_inline
702bool cpumask_and(struct cpumask *dstp, const struct cpumask *src1p,
703 const struct cpumask *src2p)
704{
705 return bitmap_and(cpumask_bits(dstp), cpumask_bits(src1p),
706 cpumask_bits(src2p), small_cpumask_bits);
707}
708
709/**
710 * cpumask_or - *dstp = *src1p | *src2p
711 * @dstp: the cpumask result
712 * @src1p: the first input
713 * @src2p: the second input
714 */
715static __always_inline
716void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p,
717 const struct cpumask *src2p)
718{
719 bitmap_or(cpumask_bits(dstp), cpumask_bits(src1p),
720 cpumask_bits(src2p), small_cpumask_bits);
721}
722
723/**
724 * cpumask_xor - *dstp = *src1p ^ *src2p
725 * @dstp: the cpumask result
726 * @src1p: the first input
727 * @src2p: the second input
728 */
729static __always_inline
730void cpumask_xor(struct cpumask *dstp, const struct cpumask *src1p,
731 const struct cpumask *src2p)
732{
733 bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p),
734 cpumask_bits(src2p), small_cpumask_bits);
735}
736
737/**
738 * cpumask_andnot - *dstp = *src1p & ~*src2p
739 * @dstp: the cpumask result
740 * @src1p: the first input
741 * @src2p: the second input
742 *
743 * Return: false if *@dstp is empty, else returns true
744 */
745static __always_inline
746bool cpumask_andnot(struct cpumask *dstp, const struct cpumask *src1p,
747 const struct cpumask *src2p)
748{
749 return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p),
750 cpumask_bits(src2p), small_cpumask_bits);
751}
752
753/**
754 * cpumask_equal - *src1p == *src2p
755 * @src1p: the first input
756 * @src2p: the second input
757 *
758 * Return: true if the cpumasks are equal, false if not
759 */
760static __always_inline
761bool cpumask_equal(const struct cpumask *src1p, const struct cpumask *src2p)
762{
763 return bitmap_equal(cpumask_bits(src1p), cpumask_bits(src2p),
764 small_cpumask_bits);
765}
766
767/**
768 * cpumask_or_equal - *src1p | *src2p == *src3p
769 * @src1p: the first input
770 * @src2p: the second input
771 * @src3p: the third input
772 *
773 * Return: true if first cpumask ORed with second cpumask == third cpumask,
774 * otherwise false
775 */
776static __always_inline
777bool cpumask_or_equal(const struct cpumask *src1p, const struct cpumask *src2p,
778 const struct cpumask *src3p)
779{
780 return bitmap_or_equal(cpumask_bits(src1p), cpumask_bits(src2p),
781 cpumask_bits(src3p), small_cpumask_bits);
782}
783
784/**
785 * cpumask_intersects - (*src1p & *src2p) != 0
786 * @src1p: the first input
787 * @src2p: the second input
788 *
789 * Return: true if first cpumask ANDed with second cpumask is non-empty,
790 * otherwise false
791 */
792static __always_inline
793bool cpumask_intersects(const struct cpumask *src1p, const struct cpumask *src2p)
794{
795 return bitmap_intersects(cpumask_bits(src1p), cpumask_bits(src2p),
796 small_cpumask_bits);
797}
798
799/**
800 * cpumask_subset - (*src1p & ~*src2p) == 0
801 * @src1p: the first input
802 * @src2p: the second input
803 *
804 * Return: true if *@src1p is a subset of *@src2p, else returns false
805 */
806static __always_inline
807bool cpumask_subset(const struct cpumask *src1p, const struct cpumask *src2p)
808{
809 return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p),
810 small_cpumask_bits);
811}
812
813/**
814 * cpumask_empty - *srcp == 0
815 * @srcp: the cpumask to that all cpus < nr_cpu_ids are clear.
816 *
817 * Return: true if srcp is empty (has no bits set), else false
818 */
819static __always_inline bool cpumask_empty(const struct cpumask *srcp)
820{
821 return bitmap_empty(cpumask_bits(srcp), small_cpumask_bits);
822}
823
824/**
825 * cpumask_full - *srcp == 0xFFFFFFFF...
826 * @srcp: the cpumask to that all cpus < nr_cpu_ids are set.
827 *
828 * Return: true if srcp is full (has all bits set), else false
829 */
830static __always_inline bool cpumask_full(const struct cpumask *srcp)
831{
832 return bitmap_full(cpumask_bits(srcp), nr_cpumask_bits);
833}
834
835/**
836 * cpumask_weight - Count of bits in *srcp
837 * @srcp: the cpumask to count bits (< nr_cpu_ids) in.
838 *
839 * Return: count of bits set in *srcp
840 */
841static __always_inline unsigned int cpumask_weight(const struct cpumask *srcp)
842{
843 return bitmap_weight(cpumask_bits(srcp), small_cpumask_bits);
844}
845
846/**
847 * cpumask_weight_and - Count of bits in (*srcp1 & *srcp2)
848 * @srcp1: the cpumask to count bits (< nr_cpu_ids) in.
849 * @srcp2: the cpumask to count bits (< nr_cpu_ids) in.
850 *
851 * Return: count of bits set in both *srcp1 and *srcp2
852 */
853static __always_inline
854unsigned int cpumask_weight_and(const struct cpumask *srcp1, const struct cpumask *srcp2)
855{
856 return bitmap_weight_and(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits);
857}
858
859/**
860 * cpumask_weight_andnot - Count of bits in (*srcp1 & ~*srcp2)
861 * @srcp1: the cpumask to count bits (< nr_cpu_ids) in.
862 * @srcp2: the cpumask to count bits (< nr_cpu_ids) in.
863 *
864 * Return: count of bits set in both *srcp1 and *srcp2
865 */
866static __always_inline
867unsigned int cpumask_weight_andnot(const struct cpumask *srcp1,
868 const struct cpumask *srcp2)
869{
870 return bitmap_weight_andnot(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits);
871}
872
873/**
874 * cpumask_shift_right - *dstp = *srcp >> n
875 * @dstp: the cpumask result
876 * @srcp: the input to shift
877 * @n: the number of bits to shift by
878 */
879static __always_inline
880void cpumask_shift_right(struct cpumask *dstp, const struct cpumask *srcp, int n)
881{
882 bitmap_shift_right(cpumask_bits(dstp), cpumask_bits(srcp), n,
883 small_cpumask_bits);
884}
885
886/**
887 * cpumask_shift_left - *dstp = *srcp << n
888 * @dstp: the cpumask result
889 * @srcp: the input to shift
890 * @n: the number of bits to shift by
891 */
892static __always_inline
893void cpumask_shift_left(struct cpumask *dstp, const struct cpumask *srcp, int n)
894{
895 bitmap_shift_left(cpumask_bits(dstp), cpumask_bits(srcp), n,
896 nr_cpumask_bits);
897}
898
899/**
900 * cpumask_copy - *dstp = *srcp
901 * @dstp: the result
902 * @srcp: the input cpumask
903 */
904static __always_inline
905void cpumask_copy(struct cpumask *dstp, const struct cpumask *srcp)
906{
907 bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), large_cpumask_bits);
908}
909
910/**
911 * cpumask_any - pick an arbitrary cpu from *srcp
912 * @srcp: the input cpumask
913 *
914 * Return: >= nr_cpu_ids if no cpus set.
915 */
916#define cpumask_any(srcp) cpumask_first(srcp)
917
918/**
919 * cpumask_any_and - pick an arbitrary cpu from *mask1 & *mask2
920 * @mask1: the first input cpumask
921 * @mask2: the second input cpumask
922 *
923 * Return: >= nr_cpu_ids if no cpus set.
924 */
925#define cpumask_any_and(mask1, mask2) cpumask_first_and((mask1), (mask2))
926
927/**
928 * cpumask_of - the cpumask containing just a given cpu
929 * @cpu: the cpu (<= nr_cpu_ids)
930 */
931#define cpumask_of(cpu) (get_cpu_mask(cpu))
932
933/**
934 * cpumask_parse_user - extract a cpumask from a user string
935 * @buf: the buffer to extract from
936 * @len: the length of the buffer
937 * @dstp: the cpumask to set.
938 *
939 * Return: -errno, or 0 for success.
940 */
941static __always_inline
942int cpumask_parse_user(const char __user *buf, int len, struct cpumask *dstp)
943{
944 return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits);
945}
946
947/**
948 * cpumask_parselist_user - extract a cpumask from a user string
949 * @buf: the buffer to extract from
950 * @len: the length of the buffer
951 * @dstp: the cpumask to set.
952 *
953 * Return: -errno, or 0 for success.
954 */
955static __always_inline
956int cpumask_parselist_user(const char __user *buf, int len, struct cpumask *dstp)
957{
958 return bitmap_parselist_user(buf, len, cpumask_bits(dstp),
959 nr_cpumask_bits);
960}
961
962/**
963 * cpumask_parse - extract a cpumask from a string
964 * @buf: the buffer to extract from
965 * @dstp: the cpumask to set.
966 *
967 * Return: -errno, or 0 for success.
968 */
969static __always_inline int cpumask_parse(const char *buf, struct cpumask *dstp)
970{
971 return bitmap_parse(buf, UINT_MAX, cpumask_bits(dstp), nr_cpumask_bits);
972}
973
974/**
975 * cpulist_parse - extract a cpumask from a user string of ranges
976 * @buf: the buffer to extract from
977 * @dstp: the cpumask to set.
978 *
979 * Return: -errno, or 0 for success.
980 */
981static __always_inline int cpulist_parse(const char *buf, struct cpumask *dstp)
982{
983 return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpumask_bits);
984}
985
986/**
987 * cpumask_size - calculate size to allocate for a 'struct cpumask' in bytes
988 *
989 * Return: size to allocate for a &struct cpumask in bytes
990 */
991static __always_inline unsigned int cpumask_size(void)
992{
993 return bitmap_size(large_cpumask_bits);
994}
995
996#ifdef CONFIG_CPUMASK_OFFSTACK
997
998#define this_cpu_cpumask_var_ptr(x) this_cpu_read(x)
999#define __cpumask_var_read_mostly __read_mostly
1000
1001bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
1002
1003static __always_inline
1004bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node)
1005{
1006 return alloc_cpumask_var_node(mask, flags | __GFP_ZERO, node);
1007}
1008
1009/**
1010 * alloc_cpumask_var - allocate a struct cpumask
1011 * @mask: pointer to cpumask_var_t where the cpumask is returned
1012 * @flags: GFP_ flags
1013 *
1014 * Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is
1015 * a nop returning a constant 1 (in <linux/cpumask.h>).
1016 *
1017 * See alloc_cpumask_var_node.
1018 *
1019 * Return: %true if allocation succeeded, %false if not
1020 */
1021static __always_inline
1022bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
1023{
1024 return alloc_cpumask_var_node(mask, flags, NUMA_NO_NODE);
1025}
1026
1027static __always_inline
1028bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
1029{
1030 return alloc_cpumask_var(mask, flags | __GFP_ZERO);
1031}
1032
1033void alloc_bootmem_cpumask_var(cpumask_var_t *mask);
1034void free_cpumask_var(cpumask_var_t mask);
1035void free_bootmem_cpumask_var(cpumask_var_t mask);
1036
1037static __always_inline bool cpumask_available(cpumask_var_t mask)
1038{
1039 return mask != NULL;
1040}
1041
1042#else
1043
1044#define this_cpu_cpumask_var_ptr(x) this_cpu_ptr(x)
1045#define __cpumask_var_read_mostly
1046
1047static __always_inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
1048{
1049 return true;
1050}
1051
1052static __always_inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
1053 int node)
1054{
1055 return true;
1056}
1057
1058static __always_inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
1059{
1060 cpumask_clear(*mask);
1061 return true;
1062}
1063
1064static __always_inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
1065 int node)
1066{
1067 cpumask_clear(*mask);
1068 return true;
1069}
1070
1071static __always_inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask)
1072{
1073}
1074
1075static __always_inline void free_cpumask_var(cpumask_var_t mask)
1076{
1077}
1078
1079static __always_inline void free_bootmem_cpumask_var(cpumask_var_t mask)
1080{
1081}
1082
1083static __always_inline bool cpumask_available(cpumask_var_t mask)
1084{
1085 return true;
1086}
1087#endif /* CONFIG_CPUMASK_OFFSTACK */
1088
1089DEFINE_FREE(free_cpumask_var, struct cpumask *, if (_T) free_cpumask_var(_T));
1090
1091/* It's common to want to use cpu_all_mask in struct member initializers,
1092 * so it has to refer to an address rather than a pointer. */
1093extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS);
1094#define cpu_all_mask to_cpumask(cpu_all_bits)
1095
1096/* First bits of cpu_bit_bitmap are in fact unset. */
1097#define cpu_none_mask to_cpumask(cpu_bit_bitmap[0])
1098
1099#if NR_CPUS == 1
1100/* Uniprocessor: the possible/online/present masks are always "1" */
1101#define for_each_possible_cpu(cpu) for ((cpu) = 0; (cpu) < 1; (cpu)++)
1102#define for_each_online_cpu(cpu) for ((cpu) = 0; (cpu) < 1; (cpu)++)
1103#define for_each_present_cpu(cpu) for ((cpu) = 0; (cpu) < 1; (cpu)++)
1104
1105#define for_each_possible_cpu_wrap(cpu, start) \
1106 for ((void)(start), (cpu) = 0; (cpu) < 1; (cpu)++)
1107#define for_each_online_cpu_wrap(cpu, start) \
1108 for ((void)(start), (cpu) = 0; (cpu) < 1; (cpu)++)
1109#else
1110#define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask)
1111#define for_each_online_cpu(cpu) for_each_cpu((cpu), cpu_online_mask)
1112#define for_each_enabled_cpu(cpu) for_each_cpu((cpu), cpu_enabled_mask)
1113#define for_each_present_cpu(cpu) for_each_cpu((cpu), cpu_present_mask)
1114
1115#define for_each_possible_cpu_wrap(cpu, start) \
1116 for_each_cpu_wrap((cpu), cpu_possible_mask, (start))
1117#define for_each_online_cpu_wrap(cpu, start) \
1118 for_each_cpu_wrap((cpu), cpu_online_mask, (start))
1119#endif
1120
1121/* Wrappers for arch boot code to manipulate normally-constant masks */
1122void init_cpu_present(const struct cpumask *src);
1123void init_cpu_possible(const struct cpumask *src);
1124
1125#define assign_cpu(cpu, mask, val) \
1126 assign_bit(cpumask_check(cpu), cpumask_bits(mask), (val))
1127
1128#define __assign_cpu(cpu, mask, val) \
1129 __assign_bit(cpumask_check(cpu), cpumask_bits(mask), (val))
1130
1131#define set_cpu_possible(cpu, possible) assign_cpu((cpu), &__cpu_possible_mask, (possible))
1132#define set_cpu_enabled(cpu, enabled) assign_cpu((cpu), &__cpu_enabled_mask, (enabled))
1133#define set_cpu_present(cpu, present) assign_cpu((cpu), &__cpu_present_mask, (present))
1134#define set_cpu_active(cpu, active) assign_cpu((cpu), &__cpu_active_mask, (active))
1135#define set_cpu_dying(cpu, dying) assign_cpu((cpu), &__cpu_dying_mask, (dying))
1136
1137void set_cpu_online(unsigned int cpu, bool online);
1138
1139/**
1140 * to_cpumask - convert a NR_CPUS bitmap to a struct cpumask *
1141 * @bitmap: the bitmap
1142 *
1143 * There are a few places where cpumask_var_t isn't appropriate and
1144 * static cpumasks must be used (eg. very early boot), yet we don't
1145 * expose the definition of 'struct cpumask'.
1146 *
1147 * This does the conversion, and can be used as a constant initializer.
1148 */
1149#define to_cpumask(bitmap) \
1150 ((struct cpumask *)(1 ? (bitmap) \
1151 : (void *)sizeof(__check_is_bitmap(bitmap))))
1152
1153static __always_inline int __check_is_bitmap(const unsigned long *bitmap)
1154{
1155 return 1;
1156}
1157
1158/*
1159 * Special-case data structure for "single bit set only" constant CPU masks.
1160 *
1161 * We pre-generate all the 64 (or 32) possible bit positions, with enough
1162 * padding to the left and the right, and return the constant pointer
1163 * appropriately offset.
1164 */
1165extern const unsigned long
1166 cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)];
1167
1168static __always_inline const struct cpumask *get_cpu_mask(unsigned int cpu)
1169{
1170 const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG];
1171 p -= cpu / BITS_PER_LONG;
1172 return to_cpumask(p);
1173}
1174
1175#if NR_CPUS > 1
1176/**
1177 * num_online_cpus() - Read the number of online CPUs
1178 *
1179 * Despite the fact that __num_online_cpus is of type atomic_t, this
1180 * interface gives only a momentary snapshot and is not protected against
1181 * concurrent CPU hotplug operations unless invoked from a cpuhp_lock held
1182 * region.
1183 *
1184 * Return: momentary snapshot of the number of online CPUs
1185 */
1186static __always_inline unsigned int num_online_cpus(void)
1187{
1188 return raw_atomic_read(&__num_online_cpus);
1189}
1190#define num_possible_cpus() cpumask_weight(cpu_possible_mask)
1191#define num_enabled_cpus() cpumask_weight(cpu_enabled_mask)
1192#define num_present_cpus() cpumask_weight(cpu_present_mask)
1193#define num_active_cpus() cpumask_weight(cpu_active_mask)
1194
1195static __always_inline bool cpu_online(unsigned int cpu)
1196{
1197 return cpumask_test_cpu(cpu, cpu_online_mask);
1198}
1199
1200static __always_inline bool cpu_enabled(unsigned int cpu)
1201{
1202 return cpumask_test_cpu(cpu, cpu_enabled_mask);
1203}
1204
1205static __always_inline bool cpu_possible(unsigned int cpu)
1206{
1207 return cpumask_test_cpu(cpu, cpu_possible_mask);
1208}
1209
1210static __always_inline bool cpu_present(unsigned int cpu)
1211{
1212 return cpumask_test_cpu(cpu, cpu_present_mask);
1213}
1214
1215static __always_inline bool cpu_active(unsigned int cpu)
1216{
1217 return cpumask_test_cpu(cpu, cpu_active_mask);
1218}
1219
1220static __always_inline bool cpu_dying(unsigned int cpu)
1221{
1222 return cpumask_test_cpu(cpu, cpu_dying_mask);
1223}
1224
1225#else
1226
1227#define num_online_cpus() 1U
1228#define num_possible_cpus() 1U
1229#define num_enabled_cpus() 1U
1230#define num_present_cpus() 1U
1231#define num_active_cpus() 1U
1232
1233static __always_inline bool cpu_online(unsigned int cpu)
1234{
1235 return cpu == 0;
1236}
1237
1238static __always_inline bool cpu_possible(unsigned int cpu)
1239{
1240 return cpu == 0;
1241}
1242
1243static __always_inline bool cpu_enabled(unsigned int cpu)
1244{
1245 return cpu == 0;
1246}
1247
1248static __always_inline bool cpu_present(unsigned int cpu)
1249{
1250 return cpu == 0;
1251}
1252
1253static __always_inline bool cpu_active(unsigned int cpu)
1254{
1255 return cpu == 0;
1256}
1257
1258static __always_inline bool cpu_dying(unsigned int cpu)
1259{
1260 return false;
1261}
1262
1263#endif /* NR_CPUS > 1 */
1264
1265#define cpu_is_offline(cpu) unlikely(!cpu_online(cpu))
1266
1267#if NR_CPUS <= BITS_PER_LONG
1268#define CPU_BITS_ALL \
1269{ \
1270 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
1271}
1272
1273#else /* NR_CPUS > BITS_PER_LONG */
1274
1275#define CPU_BITS_ALL \
1276{ \
1277 [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
1278 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
1279}
1280#endif /* NR_CPUS > BITS_PER_LONG */
1281
1282/**
1283 * cpumap_print_to_pagebuf - copies the cpumask into the buffer either
1284 * as comma-separated list of cpus or hex values of cpumask
1285 * @list: indicates whether the cpumap must be list
1286 * @mask: the cpumask to copy
1287 * @buf: the buffer to copy into
1288 *
1289 * Return: the length of the (null-terminated) @buf string, zero if
1290 * nothing is copied.
1291 */
1292static __always_inline ssize_t
1293cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask)
1294{
1295 return bitmap_print_to_pagebuf(list, buf, cpumask_bits(mask),
1296 nr_cpu_ids);
1297}
1298
1299/**
1300 * cpumap_print_bitmask_to_buf - copies the cpumask into the buffer as
1301 * hex values of cpumask
1302 *
1303 * @buf: the buffer to copy into
1304 * @mask: the cpumask to copy
1305 * @off: in the string from which we are copying, we copy to @buf
1306 * @count: the maximum number of bytes to print
1307 *
1308 * The function prints the cpumask into the buffer as hex values of
1309 * cpumask; Typically used by bin_attribute to export cpumask bitmask
1310 * ABI.
1311 *
1312 * Return: the length of how many bytes have been copied, excluding
1313 * terminating '\0'.
1314 */
1315static __always_inline
1316ssize_t cpumap_print_bitmask_to_buf(char *buf, const struct cpumask *mask,
1317 loff_t off, size_t count)
1318{
1319 return bitmap_print_bitmask_to_buf(buf, cpumask_bits(mask),
1320 nr_cpu_ids, off, count) - 1;
1321}
1322
1323/**
1324 * cpumap_print_list_to_buf - copies the cpumask into the buffer as
1325 * comma-separated list of cpus
1326 * @buf: the buffer to copy into
1327 * @mask: the cpumask to copy
1328 * @off: in the string from which we are copying, we copy to @buf
1329 * @count: the maximum number of bytes to print
1330 *
1331 * Everything is same with the above cpumap_print_bitmask_to_buf()
1332 * except the print format.
1333 *
1334 * Return: the length of how many bytes have been copied, excluding
1335 * terminating '\0'.
1336 */
1337static __always_inline
1338ssize_t cpumap_print_list_to_buf(char *buf, const struct cpumask *mask,
1339 loff_t off, size_t count)
1340{
1341 return bitmap_print_list_to_buf(buf, cpumask_bits(mask),
1342 nr_cpu_ids, off, count) - 1;
1343}
1344
1345#if NR_CPUS <= BITS_PER_LONG
1346#define CPU_MASK_ALL \
1347(cpumask_t) { { \
1348 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
1349} }
1350#else
1351#define CPU_MASK_ALL \
1352(cpumask_t) { { \
1353 [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
1354 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
1355} }
1356#endif /* NR_CPUS > BITS_PER_LONG */
1357
1358#define CPU_MASK_NONE \
1359(cpumask_t) { { \
1360 [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
1361} }
1362
1363#define CPU_MASK_CPU0 \
1364(cpumask_t) { { \
1365 [0] = 1UL \
1366} }
1367
1368/*
1369 * Provide a valid theoretical max size for cpumap and cpulist sysfs files
1370 * to avoid breaking userspace which may allocate a buffer based on the size
1371 * reported by e.g. fstat.
1372 *
1373 * for cpumap NR_CPUS * 9/32 - 1 should be an exact length.
1374 *
1375 * For cpulist 7 is (ceil(log10(NR_CPUS)) + 1) allowing for NR_CPUS to be up
1376 * to 2 orders of magnitude larger than 8192. And then we divide by 2 to
1377 * cover a worst-case of every other cpu being on one of two nodes for a
1378 * very large NR_CPUS.
1379 *
1380 * Use PAGE_SIZE as a minimum for smaller configurations while avoiding
1381 * unsigned comparison to -1.
1382 */
1383#define CPUMAP_FILE_MAX_BYTES (((NR_CPUS * 9)/32 > PAGE_SIZE) \
1384 ? (NR_CPUS * 9)/32 - 1 : PAGE_SIZE)
1385#define CPULIST_FILE_MAX_BYTES (((NR_CPUS * 7)/2 > PAGE_SIZE) ? (NR_CPUS * 7)/2 : PAGE_SIZE)
1386
1387#endif /* __LINUX_CPUMASK_H */
Linux 6.x block layer and io_uring tree(s)