Line data Source code
1 : /* SPDX-License-Identifier: GPL-2.0 */
2 : #ifndef __LINUX_CPUMASK_H
3 : #define __LINUX_CPUMASK_H
4 :
5 : /*
6 : * Cpumasks provide a bitmap suitable for representing the
7 : * set of CPU's in a system, one bit position per CPU number. In general,
8 : * only nr_cpu_ids (<= NR_CPUS) bits are valid.
9 : */
10 : #include <linux/kernel.h>
11 : #include <linux/threads.h>
12 : #include <linux/bitmap.h>
13 : #include <linux/atomic.h>
14 : #include <linux/bug.h>
15 :
16 : /* Don't assign or return these: may not be this big! */
17 : typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t;
18 :
19 : /**
20 : * cpumask_bits - get the bits in a cpumask
21 : * @maskp: the struct cpumask *
22 : *
23 : * You should only assume nr_cpu_ids bits of this mask are valid. This is
24 : * a macro so it's const-correct.
25 : */
26 : #define cpumask_bits(maskp) ((maskp)->bits)
27 :
28 : /**
29 : * cpumask_pr_args - printf args to output a cpumask
30 : * @maskp: cpumask to be printed
31 : *
32 : * Can be used to provide arguments for '%*pb[l]' when printing a cpumask.
33 : */
34 : #define cpumask_pr_args(maskp) nr_cpu_ids, cpumask_bits(maskp)
35 :
36 : #if NR_CPUS == 1
37 : #define nr_cpu_ids 1U
38 : #else
39 : extern unsigned int nr_cpu_ids;
40 : #endif
41 :
42 : #ifdef CONFIG_CPUMASK_OFFSTACK
43 : /* Assuming NR_CPUS is huge, a runtime limit is more efficient. Also,
44 : * not all bits may be allocated. */
45 : #define nr_cpumask_bits nr_cpu_ids
46 : #else
47 : #define nr_cpumask_bits ((unsigned int)NR_CPUS)
48 : #endif
49 :
50 : /*
51 : * The following particular system cpumasks and operations manage
52 : * possible, present, active and online cpus.
53 : *
54 : * cpu_possible_mask- has bit 'cpu' set iff cpu is populatable
55 : * cpu_present_mask - has bit 'cpu' set iff cpu is populated
56 : * cpu_online_mask - has bit 'cpu' set iff cpu available to scheduler
57 : * cpu_active_mask - has bit 'cpu' set iff cpu available to migration
58 : *
59 : * If !CONFIG_HOTPLUG_CPU, present == possible, and active == online.
60 : *
61 : * The cpu_possible_mask is fixed at boot time, as the set of CPU id's
62 : * that it is possible might ever be plugged in at anytime during the
63 : * life of that system boot. The cpu_present_mask is dynamic(*),
64 : * representing which CPUs are currently plugged in. And
65 : * cpu_online_mask is the dynamic subset of cpu_present_mask,
66 : * indicating those CPUs available for scheduling.
67 : *
68 : * If HOTPLUG is enabled, then cpu_possible_mask is forced to have
69 : * all NR_CPUS bits set, otherwise it is just the set of CPUs that
70 : * ACPI reports present at boot.
71 : *
72 : * If HOTPLUG is enabled, then cpu_present_mask varies dynamically,
73 : * depending on what ACPI reports as currently plugged in, otherwise
74 : * cpu_present_mask is just a copy of cpu_possible_mask.
75 : *
76 : * (*) Well, cpu_present_mask is dynamic in the hotplug case. If not
77 : * hotplug, it's a copy of cpu_possible_mask, hence fixed at boot.
78 : *
79 : * Subtleties:
80 : * 1) UP arch's (NR_CPUS == 1, CONFIG_SMP not defined) hardcode
81 : * assumption that their single CPU is online. The UP
82 : * cpu_{online,possible,present}_masks are placebos. Changing them
83 : * will have no useful affect on the following num_*_cpus()
84 : * and cpu_*() macros in the UP case. This ugliness is a UP
85 : * optimization - don't waste any instructions or memory references
86 : * asking if you're online or how many CPUs there are if there is
87 : * only one CPU.
88 : */
89 :
90 : extern struct cpumask __cpu_possible_mask;
91 : extern struct cpumask __cpu_online_mask;
92 : extern struct cpumask __cpu_present_mask;
93 : extern struct cpumask __cpu_active_mask;
94 : extern struct cpumask __cpu_dying_mask;
95 : #define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask)
96 : #define cpu_online_mask ((const struct cpumask *)&__cpu_online_mask)
97 : #define cpu_present_mask ((const struct cpumask *)&__cpu_present_mask)
98 : #define cpu_active_mask ((const struct cpumask *)&__cpu_active_mask)
99 : #define cpu_dying_mask ((const struct cpumask *)&__cpu_dying_mask)
100 :
101 : extern atomic_t __num_online_cpus;
102 :
103 : extern cpumask_t cpus_booted_once_mask;
104 :
105 : static __always_inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits)
106 : {
107 : #ifdef CONFIG_DEBUG_PER_CPU_MAPS
108 : WARN_ON_ONCE(cpu >= bits);
109 : #endif /* CONFIG_DEBUG_PER_CPU_MAPS */
110 : }
111 :
112 : /* verify cpu argument to cpumask_* operators */
113 : static __always_inline unsigned int cpumask_check(unsigned int cpu)
114 : {
115 117 : cpu_max_bits_warn(cpu, nr_cpumask_bits);
116 : return cpu;
117 : }
118 :
119 : #if NR_CPUS == 1
120 : /* Uniprocessor. Assume all masks are "1". */
121 : static inline unsigned int cpumask_first(const struct cpumask *srcp)
122 : {
123 : return 0;
124 : }
125 :
126 : static inline unsigned int cpumask_first_zero(const struct cpumask *srcp)
127 : {
128 : return 0;
129 : }
130 :
131 : static inline unsigned int cpumask_first_and(const struct cpumask *srcp1,
132 : const struct cpumask *srcp2)
133 : {
134 : return 0;
135 : }
136 :
137 : static inline unsigned int cpumask_last(const struct cpumask *srcp)
138 : {
139 : return 0;
140 : }
141 :
142 : /* Valid inputs for n are -1 and 0. */
143 : static inline unsigned int cpumask_next(int n, const struct cpumask *srcp)
144 : {
145 0 : return n+1;
146 : }
147 :
148 : static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
149 : {
150 : return n+1;
151 : }
152 :
153 : static inline unsigned int cpumask_next_and(int n,
154 : const struct cpumask *srcp,
155 : const struct cpumask *andp)
156 : {
157 0 : return n+1;
158 : }
159 :
160 : static inline unsigned int cpumask_next_wrap(int n, const struct cpumask *mask,
161 : int start, bool wrap)
162 : {
163 : /* cpu0 unless stop condition, wrap and at cpu0, then nr_cpumask_bits */
164 : return (wrap && n == 0);
165 : }
166 :
167 : /* cpu must be a valid cpu, ie 0, so there's no other choice. */
168 : static inline unsigned int cpumask_any_but(const struct cpumask *mask,
169 : unsigned int cpu)
170 : {
171 : return 1;
172 : }
173 :
174 : static inline unsigned int cpumask_local_spread(unsigned int i, int node)
175 : {
176 : return 0;
177 : }
178 :
179 : static inline int cpumask_any_and_distribute(const struct cpumask *src1p,
180 : const struct cpumask *src2p) {
181 : return cpumask_first_and(src1p, src2p);
182 : }
183 :
184 : static inline int cpumask_any_distribute(const struct cpumask *srcp)
185 : {
186 : return cpumask_first(srcp);
187 : }
188 :
189 : #define for_each_cpu(cpu, mask) \
190 : for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
191 : #define for_each_cpu_not(cpu, mask) \
192 : for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
193 : #define for_each_cpu_wrap(cpu, mask, start) \
194 : for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask, (void)(start))
195 : #define for_each_cpu_and(cpu, mask1, mask2) \
196 : for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask1, (void)mask2)
197 : #else
198 : /**
199 : * cpumask_first - get the first cpu in a cpumask
200 : * @srcp: the cpumask pointer
201 : *
202 : * Returns >= nr_cpu_ids if no cpus set.
203 : */
204 : static inline unsigned int cpumask_first(const struct cpumask *srcp)
205 : {
206 : return find_first_bit(cpumask_bits(srcp), nr_cpumask_bits);
207 : }
208 :
209 : /**
210 : * cpumask_first_zero - get the first unset cpu in a cpumask
211 : * @srcp: the cpumask pointer
212 : *
213 : * Returns >= nr_cpu_ids if all cpus are set.
214 : */
215 : static inline unsigned int cpumask_first_zero(const struct cpumask *srcp)
216 : {
217 : return find_first_zero_bit(cpumask_bits(srcp), nr_cpumask_bits);
218 : }
219 :
220 : /**
221 : * cpumask_first_and - return the first cpu from *srcp1 & *srcp2
222 : * @src1p: the first input
223 : * @src2p: the second input
224 : *
225 : * Returns >= nr_cpu_ids if no cpus set in both. See also cpumask_next_and().
226 : */
227 : static inline
228 : unsigned int cpumask_first_and(const struct cpumask *srcp1, const struct cpumask *srcp2)
229 : {
230 : return find_first_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), nr_cpumask_bits);
231 : }
232 :
233 : /**
234 : * cpumask_last - get the last CPU in a cpumask
235 : * @srcp: - the cpumask pointer
236 : *
237 : * Returns >= nr_cpumask_bits if no CPUs set.
238 : */
239 : static inline unsigned int cpumask_last(const struct cpumask *srcp)
240 : {
241 : return find_last_bit(cpumask_bits(srcp), nr_cpumask_bits);
242 : }
243 :
244 : unsigned int __pure cpumask_next(int n, const struct cpumask *srcp);
245 :
246 : /**
247 : * cpumask_next_zero - get the next unset cpu in a cpumask
248 : * @n: the cpu prior to the place to search (ie. return will be > @n)
249 : * @srcp: the cpumask pointer
250 : *
251 : * Returns >= nr_cpu_ids if no further cpus unset.
252 : */
253 : static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
254 : {
255 : /* -1 is a legal arg here. */
256 : if (n != -1)
257 : cpumask_check(n);
258 : return find_next_zero_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1);
259 : }
260 :
261 : int __pure cpumask_next_and(int n, const struct cpumask *, const struct cpumask *);
262 : int __pure cpumask_any_but(const struct cpumask *mask, unsigned int cpu);
263 : unsigned int cpumask_local_spread(unsigned int i, int node);
264 : int cpumask_any_and_distribute(const struct cpumask *src1p,
265 : const struct cpumask *src2p);
266 : int cpumask_any_distribute(const struct cpumask *srcp);
267 :
268 : /**
269 : * for_each_cpu - iterate over every cpu in a mask
270 : * @cpu: the (optionally unsigned) integer iterator
271 : * @mask: the cpumask pointer
272 : *
273 : * After the loop, cpu is >= nr_cpu_ids.
274 : */
275 : #define for_each_cpu(cpu, mask) \
276 : for ((cpu) = -1; \
277 : (cpu) = cpumask_next((cpu), (mask)), \
278 : (cpu) < nr_cpu_ids;)
279 :
280 : /**
281 : * for_each_cpu_not - iterate over every cpu in a complemented mask
282 : * @cpu: the (optionally unsigned) integer iterator
283 : * @mask: the cpumask pointer
284 : *
285 : * After the loop, cpu is >= nr_cpu_ids.
286 : */
287 : #define for_each_cpu_not(cpu, mask) \
288 : for ((cpu) = -1; \
289 : (cpu) = cpumask_next_zero((cpu), (mask)), \
290 : (cpu) < nr_cpu_ids;)
291 :
292 : extern int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap);
293 :
294 : /**
295 : * for_each_cpu_wrap - iterate over every cpu in a mask, starting at a specified location
296 : * @cpu: the (optionally unsigned) integer iterator
297 : * @mask: the cpumask pointer
298 : * @start: the start location
299 : *
300 : * The implementation does not assume any bit in @mask is set (including @start).
301 : *
302 : * After the loop, cpu is >= nr_cpu_ids.
303 : */
304 : #define for_each_cpu_wrap(cpu, mask, start) \
305 : for ((cpu) = cpumask_next_wrap((start)-1, (mask), (start), false); \
306 : (cpu) < nr_cpumask_bits; \
307 : (cpu) = cpumask_next_wrap((cpu), (mask), (start), true))
308 :
309 : /**
310 : * for_each_cpu_and - iterate over every cpu in both masks
311 : * @cpu: the (optionally unsigned) integer iterator
312 : * @mask1: the first cpumask pointer
313 : * @mask2: the second cpumask pointer
314 : *
315 : * This saves a temporary CPU mask in many places. It is equivalent to:
316 : * struct cpumask tmp;
317 : * cpumask_and(&tmp, &mask1, &mask2);
318 : * for_each_cpu(cpu, &tmp)
319 : * ...
320 : *
321 : * After the loop, cpu is >= nr_cpu_ids.
322 : */
323 : #define for_each_cpu_and(cpu, mask1, mask2) \
324 : for ((cpu) = -1; \
325 : (cpu) = cpumask_next_and((cpu), (mask1), (mask2)), \
326 : (cpu) < nr_cpu_ids;)
327 : #endif /* SMP */
328 :
329 : #define CPU_BITS_NONE \
330 : { \
331 : [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
332 : }
333 :
334 : #define CPU_BITS_CPU0 \
335 : { \
336 : [0] = 1UL \
337 : }
338 :
339 : /**
340 : * cpumask_set_cpu - set a cpu in a cpumask
341 : * @cpu: cpu number (< nr_cpu_ids)
342 : * @dstp: the cpumask pointer
343 : */
344 : static __always_inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
345 : {
346 6 : set_bit(cpumask_check(cpu), cpumask_bits(dstp));
347 : }
348 :
349 : static __always_inline void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
350 : {
351 : __set_bit(cpumask_check(cpu), cpumask_bits(dstp));
352 : }
353 :
354 :
355 : /**
356 : * cpumask_clear_cpu - clear a cpu in a cpumask
357 : * @cpu: cpu number (< nr_cpu_ids)
358 : * @dstp: the cpumask pointer
359 : */
360 : static __always_inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp)
361 : {
362 0 : clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
363 : }
364 :
365 : static __always_inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp)
366 : {
367 : __clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
368 : }
369 :
370 : /**
371 : * cpumask_test_cpu - test for a cpu in a cpumask
372 : * @cpu: cpu number (< nr_cpu_ids)
373 : * @cpumask: the cpumask pointer
374 : *
375 : * Returns 1 if @cpu is set in @cpumask, else returns 0
376 : */
377 : static __always_inline int cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
378 : {
379 339 : return test_bit(cpumask_check(cpu), cpumask_bits((cpumask)));
380 : }
381 :
382 : /**
383 : * cpumask_test_and_set_cpu - atomically test and set a cpu in a cpumask
384 : * @cpu: cpu number (< nr_cpu_ids)
385 : * @cpumask: the cpumask pointer
386 : *
387 : * Returns 1 if @cpu is set in old bitmap of @cpumask, else returns 0
388 : *
389 : * test_and_set_bit wrapper for cpumasks.
390 : */
391 : static __always_inline int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
392 : {
393 3 : return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));
394 : }
395 :
396 : /**
397 : * cpumask_test_and_clear_cpu - atomically test and clear a cpu in a cpumask
398 : * @cpu: cpu number (< nr_cpu_ids)
399 : * @cpumask: the cpumask pointer
400 : *
401 : * Returns 1 if @cpu is set in old bitmap of @cpumask, else returns 0
402 : *
403 : * test_and_clear_bit wrapper for cpumasks.
404 : */
405 : static __always_inline int cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
406 : {
407 0 : return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));
408 : }
409 :
410 : /**
411 : * cpumask_setall - set all cpus (< nr_cpu_ids) in a cpumask
412 : * @dstp: the cpumask pointer
413 : */
414 : static inline void cpumask_setall(struct cpumask *dstp)
415 : {
416 : bitmap_fill(cpumask_bits(dstp), nr_cpumask_bits);
417 : }
418 :
419 : /**
420 : * cpumask_clear - clear all cpus (< nr_cpu_ids) in a cpumask
421 : * @dstp: the cpumask pointer
422 : */
423 : static inline void cpumask_clear(struct cpumask *dstp)
424 : {
425 0 : bitmap_zero(cpumask_bits(dstp), nr_cpumask_bits);
426 : }
427 :
428 : /**
429 : * cpumask_and - *dstp = *src1p & *src2p
430 : * @dstp: the cpumask result
431 : * @src1p: the first input
432 : * @src2p: the second input
433 : *
434 : * If *@dstp is empty, returns 0, else returns 1
435 : */
436 : static inline int cpumask_and(struct cpumask *dstp,
437 : const struct cpumask *src1p,
438 : const struct cpumask *src2p)
439 : {
440 14 : return bitmap_and(cpumask_bits(dstp), cpumask_bits(src1p),
441 6 : cpumask_bits(src2p), nr_cpumask_bits);
442 : }
443 :
444 : /**
445 : * cpumask_or - *dstp = *src1p | *src2p
446 : * @dstp: the cpumask result
447 : * @src1p: the first input
448 : * @src2p: the second input
449 : */
450 : static inline void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p,
451 : const struct cpumask *src2p)
452 : {
453 : bitmap_or(cpumask_bits(dstp), cpumask_bits(src1p),
454 : cpumask_bits(src2p), nr_cpumask_bits);
455 : }
456 :
457 : /**
458 : * cpumask_xor - *dstp = *src1p ^ *src2p
459 : * @dstp: the cpumask result
460 : * @src1p: the first input
461 : * @src2p: the second input
462 : */
463 : static inline void cpumask_xor(struct cpumask *dstp,
464 : const struct cpumask *src1p,
465 : const struct cpumask *src2p)
466 : {
467 : bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p),
468 : cpumask_bits(src2p), nr_cpumask_bits);
469 : }
470 :
471 : /**
472 : * cpumask_andnot - *dstp = *src1p & ~*src2p
473 : * @dstp: the cpumask result
474 : * @src1p: the first input
475 : * @src2p: the second input
476 : *
477 : * If *@dstp is empty, returns 0, else returns 1
478 : */
479 : static inline int cpumask_andnot(struct cpumask *dstp,
480 : const struct cpumask *src1p,
481 : const struct cpumask *src2p)
482 : {
483 0 : return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p),
484 0 : cpumask_bits(src2p), nr_cpumask_bits);
485 : }
486 :
487 : /**
488 : * cpumask_complement - *dstp = ~*srcp
489 : * @dstp: the cpumask result
490 : * @srcp: the input to invert
491 : */
492 : static inline void cpumask_complement(struct cpumask *dstp,
493 : const struct cpumask *srcp)
494 : {
495 : bitmap_complement(cpumask_bits(dstp), cpumask_bits(srcp),
496 : nr_cpumask_bits);
497 : }
498 :
499 : /**
500 : * cpumask_equal - *src1p == *src2p
501 : * @src1p: the first input
502 : * @src2p: the second input
503 : */
504 : static inline bool cpumask_equal(const struct cpumask *src1p,
505 : const struct cpumask *src2p)
506 : {
507 4 : return bitmap_equal(cpumask_bits(src1p), cpumask_bits(src2p),
508 : nr_cpumask_bits);
509 : }
510 :
511 : /**
512 : * cpumask_or_equal - *src1p | *src2p == *src3p
513 : * @src1p: the first input
514 : * @src2p: the second input
515 : * @src3p: the third input
516 : */
517 : static inline bool cpumask_or_equal(const struct cpumask *src1p,
518 : const struct cpumask *src2p,
519 : const struct cpumask *src3p)
520 : {
521 : return bitmap_or_equal(cpumask_bits(src1p), cpumask_bits(src2p),
522 : cpumask_bits(src3p), nr_cpumask_bits);
523 : }
524 :
525 : /**
526 : * cpumask_intersects - (*src1p & *src2p) != 0
527 : * @src1p: the first input
528 : * @src2p: the second input
529 : */
530 : static inline bool cpumask_intersects(const struct cpumask *src1p,
531 : const struct cpumask *src2p)
532 : {
533 : return bitmap_intersects(cpumask_bits(src1p), cpumask_bits(src2p),
534 : nr_cpumask_bits);
535 : }
536 :
537 : /**
538 : * cpumask_subset - (*src1p & ~*src2p) == 0
539 : * @src1p: the first input
540 : * @src2p: the second input
541 : *
542 : * Returns 1 if *@src1p is a subset of *@src2p, else returns 0
543 : */
544 : static inline int cpumask_subset(const struct cpumask *src1p,
545 : const struct cpumask *src2p)
546 : {
547 0 : return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p),
548 : nr_cpumask_bits);
549 : }
550 :
551 : /**
552 : * cpumask_empty - *srcp == 0
553 : * @srcp: the cpumask to that all cpus < nr_cpu_ids are clear.
554 : */
555 : static inline bool cpumask_empty(const struct cpumask *srcp)
556 : {
557 10 : return bitmap_empty(cpumask_bits(srcp), nr_cpumask_bits);
558 : }
559 :
560 : /**
561 : * cpumask_full - *srcp == 0xFFFFFFFF...
562 : * @srcp: the cpumask to that all cpus < nr_cpu_ids are set.
563 : */
564 : static inline bool cpumask_full(const struct cpumask *srcp)
565 : {
566 : return bitmap_full(cpumask_bits(srcp), nr_cpumask_bits);
567 : }
568 :
569 : /**
570 : * cpumask_weight - Count of bits in *srcp
571 : * @srcp: the cpumask to count bits (< nr_cpu_ids) in.
572 : */
573 : static inline unsigned int cpumask_weight(const struct cpumask *srcp)
574 : {
575 6 : return bitmap_weight(cpumask_bits(srcp), nr_cpumask_bits);
576 : }
577 :
578 : /**
579 : * cpumask_shift_right - *dstp = *srcp >> n
580 : * @dstp: the cpumask result
581 : * @srcp: the input to shift
582 : * @n: the number of bits to shift by
583 : */
584 : static inline void cpumask_shift_right(struct cpumask *dstp,
585 : const struct cpumask *srcp, int n)
586 : {
587 : bitmap_shift_right(cpumask_bits(dstp), cpumask_bits(srcp), n,
588 : nr_cpumask_bits);
589 : }
590 :
591 : /**
592 : * cpumask_shift_left - *dstp = *srcp << n
593 : * @dstp: the cpumask result
594 : * @srcp: the input to shift
595 : * @n: the number of bits to shift by
596 : */
597 : static inline void cpumask_shift_left(struct cpumask *dstp,
598 : const struct cpumask *srcp, int n)
599 : {
600 : bitmap_shift_left(cpumask_bits(dstp), cpumask_bits(srcp), n,
601 : nr_cpumask_bits);
602 : }
603 :
604 : /**
605 : * cpumask_copy - *dstp = *srcp
606 : * @dstp: the result
607 : * @srcp: the input cpumask
608 : */
609 : static inline void cpumask_copy(struct cpumask *dstp,
610 : const struct cpumask *srcp)
611 : {
612 70 : bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), nr_cpumask_bits);
613 : }
614 :
615 : /**
616 : * cpumask_any - pick a "random" cpu from *srcp
617 : * @srcp: the input cpumask
618 : *
619 : * Returns >= nr_cpu_ids if no cpus set.
620 : */
621 : #define cpumask_any(srcp) cpumask_first(srcp)
622 :
623 : /**
624 : * cpumask_any_and - pick a "random" cpu from *mask1 & *mask2
625 : * @mask1: the first input cpumask
626 : * @mask2: the second input cpumask
627 : *
628 : * Returns >= nr_cpu_ids if no cpus set.
629 : */
630 : #define cpumask_any_and(mask1, mask2) cpumask_first_and((mask1), (mask2))
631 :
632 : /**
633 : * cpumask_of - the cpumask containing just a given cpu
634 : * @cpu: the cpu (<= nr_cpu_ids)
635 : */
636 : #define cpumask_of(cpu) (get_cpu_mask(cpu))
637 :
638 : /**
639 : * cpumask_parse_user - extract a cpumask from a user string
640 : * @buf: the buffer to extract from
641 : * @len: the length of the buffer
642 : * @dstp: the cpumask to set.
643 : *
644 : * Returns -errno, or 0 for success.
645 : */
646 : static inline int cpumask_parse_user(const char __user *buf, int len,
647 : struct cpumask *dstp)
648 : {
649 : return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits);
650 : }
651 :
652 : /**
653 : * cpumask_parselist_user - extract a cpumask from a user string
654 : * @buf: the buffer to extract from
655 : * @len: the length of the buffer
656 : * @dstp: the cpumask to set.
657 : *
658 : * Returns -errno, or 0 for success.
659 : */
660 : static inline int cpumask_parselist_user(const char __user *buf, int len,
661 : struct cpumask *dstp)
662 : {
663 : return bitmap_parselist_user(buf, len, cpumask_bits(dstp),
664 : nr_cpumask_bits);
665 : }
666 :
667 : /**
668 : * cpumask_parse - extract a cpumask from a string
669 : * @buf: the buffer to extract from
670 : * @dstp: the cpumask to set.
671 : *
672 : * Returns -errno, or 0 for success.
673 : */
674 : static inline int cpumask_parse(const char *buf, struct cpumask *dstp)
675 : {
676 0 : return bitmap_parse(buf, UINT_MAX, cpumask_bits(dstp), nr_cpumask_bits);
677 : }
678 :
679 : /**
680 : * cpulist_parse - extract a cpumask from a user string of ranges
681 : * @buf: the buffer to extract from
682 : * @dstp: the cpumask to set.
683 : *
684 : * Returns -errno, or 0 for success.
685 : */
686 : static inline int cpulist_parse(const char *buf, struct cpumask *dstp)
687 : {
688 : return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpumask_bits);
689 : }
690 :
691 : /**
692 : * cpumask_size - size to allocate for a 'struct cpumask' in bytes
693 : */
694 : static inline unsigned int cpumask_size(void)
695 : {
696 : return BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long);
697 : }
698 :
699 : /*
700 : * cpumask_var_t: struct cpumask for stack usage.
701 : *
702 : * Oh, the wicked games we play! In order to make kernel coding a
703 : * little more difficult, we typedef cpumask_var_t to an array or a
704 : * pointer: doing &mask on an array is a noop, so it still works.
705 : *
706 : * ie.
707 : * cpumask_var_t tmpmask;
708 : * if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
709 : * return -ENOMEM;
710 : *
711 : * ... use 'tmpmask' like a normal struct cpumask * ...
712 : *
713 : * free_cpumask_var(tmpmask);
714 : *
715 : *
716 : * However, one notable exception is there. alloc_cpumask_var() allocates
717 : * only nr_cpumask_bits bits (in the other hand, real cpumask_t always has
718 : * NR_CPUS bits). Therefore you don't have to dereference cpumask_var_t.
719 : *
720 : * cpumask_var_t tmpmask;
721 : * if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
722 : * return -ENOMEM;
723 : *
724 : * var = *tmpmask;
725 : *
726 : * This code makes NR_CPUS length memcopy and brings to a memory corruption.
727 : * cpumask_copy() provide safe copy functionality.
728 : *
729 : * Note that there is another evil here: If you define a cpumask_var_t
730 : * as a percpu variable then the way to obtain the address of the cpumask
731 : * structure differently influences what this_cpu_* operation needs to be
732 : * used. Please use this_cpu_cpumask_var_t in those cases. The direct use
733 : * of this_cpu_ptr() or this_cpu_read() will lead to failures when the
734 : * other type of cpumask_var_t implementation is configured.
735 : *
736 : * Please also note that __cpumask_var_read_mostly can be used to declare
737 : * a cpumask_var_t variable itself (not its content) as read mostly.
738 : */
739 : #ifdef CONFIG_CPUMASK_OFFSTACK
740 : typedef struct cpumask *cpumask_var_t;
741 :
742 : #define this_cpu_cpumask_var_ptr(x) this_cpu_read(x)
743 : #define __cpumask_var_read_mostly __read_mostly
744 :
745 : bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
746 : bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);
747 : bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
748 : bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);
749 : void alloc_bootmem_cpumask_var(cpumask_var_t *mask);
750 : void free_cpumask_var(cpumask_var_t mask);
751 : void free_bootmem_cpumask_var(cpumask_var_t mask);
752 :
753 : static inline bool cpumask_available(cpumask_var_t mask)
754 : {
755 : return mask != NULL;
756 : }
757 :
758 : #else
759 : typedef struct cpumask cpumask_var_t[1];
760 :
761 : #define this_cpu_cpumask_var_ptr(x) this_cpu_ptr(x)
762 : #define __cpumask_var_read_mostly
763 :
764 : static inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
765 : {
766 : return true;
767 : }
768 :
769 : static inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
770 : int node)
771 : {
772 : return true;
773 : }
774 :
775 : static inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
776 : {
777 0 : cpumask_clear(*mask);
778 : return true;
779 : }
780 :
781 : static inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
782 : int node)
783 : {
784 0 : cpumask_clear(*mask);
785 : return true;
786 : }
787 :
788 : static inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask)
789 : {
790 : }
791 :
792 : static inline void free_cpumask_var(cpumask_var_t mask)
793 : {
794 : }
795 :
796 : static inline void free_bootmem_cpumask_var(cpumask_var_t mask)
797 : {
798 : }
799 :
800 : static inline bool cpumask_available(cpumask_var_t mask)
801 : {
802 : return true;
803 : }
804 : #endif /* CONFIG_CPUMASK_OFFSTACK */
805 :
806 : /* It's common to want to use cpu_all_mask in struct member initializers,
807 : * so it has to refer to an address rather than a pointer. */
808 : extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS);
809 : #define cpu_all_mask to_cpumask(cpu_all_bits)
810 :
811 : /* First bits of cpu_bit_bitmap are in fact unset. */
812 : #define cpu_none_mask to_cpumask(cpu_bit_bitmap[0])
813 :
814 : #define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask)
815 : #define for_each_online_cpu(cpu) for_each_cpu((cpu), cpu_online_mask)
816 : #define for_each_present_cpu(cpu) for_each_cpu((cpu), cpu_present_mask)
817 :
818 : /* Wrappers for arch boot code to manipulate normally-constant masks */
819 : void init_cpu_present(const struct cpumask *src);
820 : void init_cpu_possible(const struct cpumask *src);
821 : void init_cpu_online(const struct cpumask *src);
822 :
823 : static inline void reset_cpu_possible_mask(void)
824 : {
825 : bitmap_zero(cpumask_bits(&__cpu_possible_mask), NR_CPUS);
826 : }
827 :
828 : static inline void
829 : set_cpu_possible(unsigned int cpu, bool possible)
830 : {
831 : if (possible)
832 : cpumask_set_cpu(cpu, &__cpu_possible_mask);
833 : else
834 : cpumask_clear_cpu(cpu, &__cpu_possible_mask);
835 : }
836 :
837 : static inline void
838 : set_cpu_present(unsigned int cpu, bool present)
839 : {
840 : if (present)
841 : cpumask_set_cpu(cpu, &__cpu_present_mask);
842 : else
843 : cpumask_clear_cpu(cpu, &__cpu_present_mask);
844 : }
845 :
846 : void set_cpu_online(unsigned int cpu, bool online);
847 :
848 : static inline void
849 : set_cpu_active(unsigned int cpu, bool active)
850 : {
851 : if (active)
852 : cpumask_set_cpu(cpu, &__cpu_active_mask);
853 : else
854 : cpumask_clear_cpu(cpu, &__cpu_active_mask);
855 : }
856 :
857 : static inline void
858 : set_cpu_dying(unsigned int cpu, bool dying)
859 : {
860 : if (dying)
861 : cpumask_set_cpu(cpu, &__cpu_dying_mask);
862 : else
863 : cpumask_clear_cpu(cpu, &__cpu_dying_mask);
864 : }
865 :
866 : /**
867 : * to_cpumask - convert an NR_CPUS bitmap to a struct cpumask *
868 : * @bitmap: the bitmap
869 : *
870 : * There are a few places where cpumask_var_t isn't appropriate and
871 : * static cpumasks must be used (eg. very early boot), yet we don't
872 : * expose the definition of 'struct cpumask'.
873 : *
874 : * This does the conversion, and can be used as a constant initializer.
875 : */
876 : #define to_cpumask(bitmap) \
877 : ((struct cpumask *)(1 ? (bitmap) \
878 : : (void *)sizeof(__check_is_bitmap(bitmap))))
879 :
880 : static inline int __check_is_bitmap(const unsigned long *bitmap)
881 : {
882 : return 1;
883 : }
884 :
885 : /*
886 : * Special-case data structure for "single bit set only" constant CPU masks.
887 : *
888 : * We pre-generate all the 64 (or 32) possible bit positions, with enough
889 : * padding to the left and the right, and return the constant pointer
890 : * appropriately offset.
891 : */
892 : extern const unsigned long
893 : cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)];
894 :
895 : static inline const struct cpumask *get_cpu_mask(unsigned int cpu)
896 : {
897 8 : const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG];
898 8 : p -= cpu / BITS_PER_LONG;
899 : return to_cpumask(p);
900 : }
901 :
902 : #if NR_CPUS > 1
903 : /**
904 : * num_online_cpus() - Read the number of online CPUs
905 : *
906 : * Despite the fact that __num_online_cpus is of type atomic_t, this
907 : * interface gives only a momentary snapshot and is not protected against
908 : * concurrent CPU hotplug operations unless invoked from a cpuhp_lock held
909 : * region.
910 : */
911 : static inline unsigned int num_online_cpus(void)
912 : {
913 : return atomic_read(&__num_online_cpus);
914 : }
915 : #define num_possible_cpus() cpumask_weight(cpu_possible_mask)
916 : #define num_present_cpus() cpumask_weight(cpu_present_mask)
917 : #define num_active_cpus() cpumask_weight(cpu_active_mask)
918 :
919 : static inline bool cpu_online(unsigned int cpu)
920 : {
921 : return cpumask_test_cpu(cpu, cpu_online_mask);
922 : }
923 :
924 : static inline bool cpu_possible(unsigned int cpu)
925 : {
926 : return cpumask_test_cpu(cpu, cpu_possible_mask);
927 : }
928 :
929 : static inline bool cpu_present(unsigned int cpu)
930 : {
931 : return cpumask_test_cpu(cpu, cpu_present_mask);
932 : }
933 :
934 : static inline bool cpu_active(unsigned int cpu)
935 : {
936 : return cpumask_test_cpu(cpu, cpu_active_mask);
937 : }
938 :
939 : static inline bool cpu_dying(unsigned int cpu)
940 : {
941 : return cpumask_test_cpu(cpu, cpu_dying_mask);
942 : }
943 :
944 : #else
945 :
946 : #define num_online_cpus() 1U
947 : #define num_possible_cpus() 1U
948 : #define num_present_cpus() 1U
949 : #define num_active_cpus() 1U
950 :
951 : static inline bool cpu_online(unsigned int cpu)
952 : {
953 0 : return cpu == 0;
954 : }
955 :
956 : static inline bool cpu_possible(unsigned int cpu)
957 : {
958 : return cpu == 0;
959 : }
960 :
961 : static inline bool cpu_present(unsigned int cpu)
962 : {
963 : return cpu == 0;
964 : }
965 :
966 : static inline bool cpu_active(unsigned int cpu)
967 : {
968 : return cpu == 0;
969 : }
970 :
971 : static inline bool cpu_dying(unsigned int cpu)
972 : {
973 : return false;
974 : }
975 :
976 : #endif /* NR_CPUS > 1 */
977 :
978 : #define cpu_is_offline(cpu) unlikely(!cpu_online(cpu))
979 :
980 : #if NR_CPUS <= BITS_PER_LONG
981 : #define CPU_BITS_ALL \
982 : { \
983 : [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
984 : }
985 :
986 : #else /* NR_CPUS > BITS_PER_LONG */
987 :
988 : #define CPU_BITS_ALL \
989 : { \
990 : [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
991 : [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
992 : }
993 : #endif /* NR_CPUS > BITS_PER_LONG */
994 :
995 : /**
996 : * cpumap_print_to_pagebuf - copies the cpumask into the buffer either
997 : * as comma-separated list of cpus or hex values of cpumask
998 : * @list: indicates whether the cpumap must be list
999 : * @mask: the cpumask to copy
1000 : * @buf: the buffer to copy into
1001 : *
1002 : * Returns the length of the (null-terminated) @buf string, zero if
1003 : * nothing is copied.
1004 : */
1005 : static inline ssize_t
1006 : cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask)
1007 : {
1008 0 : return bitmap_print_to_pagebuf(list, buf, cpumask_bits(mask),
1009 : nr_cpu_ids);
1010 : }
1011 :
1012 : /**
1013 : * cpumap_print_bitmask_to_buf - copies the cpumask into the buffer as
1014 : * hex values of cpumask
1015 : *
1016 : * @buf: the buffer to copy into
1017 : * @mask: the cpumask to copy
1018 : * @off: in the string from which we are copying, we copy to @buf
1019 : * @count: the maximum number of bytes to print
1020 : *
1021 : * The function prints the cpumask into the buffer as hex values of
1022 : * cpumask; Typically used by bin_attribute to export cpumask bitmask
1023 : * ABI.
1024 : *
1025 : * Returns the length of how many bytes have been copied, excluding
1026 : * terminating '\0'.
1027 : */
1028 : static inline ssize_t
1029 : cpumap_print_bitmask_to_buf(char *buf, const struct cpumask *mask,
1030 : loff_t off, size_t count)
1031 : {
1032 0 : return bitmap_print_bitmask_to_buf(buf, cpumask_bits(mask),
1033 0 : nr_cpu_ids, off, count) - 1;
1034 : }
1035 :
1036 : /**
1037 : * cpumap_print_list_to_buf - copies the cpumask into the buffer as
1038 : * comma-separated list of cpus
1039 : *
1040 : * Everything is same with the above cpumap_print_bitmask_to_buf()
1041 : * except the print format.
1042 : */
1043 : static inline ssize_t
1044 : cpumap_print_list_to_buf(char *buf, const struct cpumask *mask,
1045 : loff_t off, size_t count)
1046 : {
1047 0 : return bitmap_print_list_to_buf(buf, cpumask_bits(mask),
1048 0 : nr_cpu_ids, off, count) - 1;
1049 : }
1050 :
1051 : #if NR_CPUS <= BITS_PER_LONG
1052 : #define CPU_MASK_ALL \
1053 : (cpumask_t) { { \
1054 : [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
1055 : } }
1056 : #else
1057 : #define CPU_MASK_ALL \
1058 : (cpumask_t) { { \
1059 : [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
1060 : [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
1061 : } }
1062 : #endif /* NR_CPUS > BITS_PER_LONG */
1063 :
1064 : #define CPU_MASK_NONE \
1065 : (cpumask_t) { { \
1066 : [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
1067 : } }
1068 :
1069 : #define CPU_MASK_CPU0 \
1070 : (cpumask_t) { { \
1071 : [0] = 1UL \
1072 : } }
1073 :
1074 : #endif /* __LINUX_CPUMASK_H */
|
