cpumask.h 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. #ifndef __LINUX_CPUMASK_H
  3. #define __LINUX_CPUMASK_H
  4. /*
  5. * Cpumasks provide a bitmap suitable for representing the
  6. * set of CPU's in a system, one bit position per CPU number. In general,
  7. * only nr_cpu_ids (<= NR_CPUS) bits are valid.
  8. */
  9. #include <linux/kernel.h>
  10. #include <linux/threads.h>
  11. #include <linux/bitmap.h>
  12. #include <linux/atomic.h>
  13. #include <linux/bug.h>
  14. /* Don't assign or return these: may not be this big! */
  15. typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t;
  16. /**
  17. * cpumask_bits - get the bits in a cpumask
  18. * @maskp: the struct cpumask *
  19. *
  20. * You should only assume nr_cpu_ids bits of this mask are valid. This is
  21. * a macro so it's const-correct.
  22. */
  23. #define cpumask_bits(maskp) ((maskp)->bits)
  24. /**
  25. * cpumask_pr_args - printf args to output a cpumask
  26. * @maskp: cpumask to be printed
  27. *
  28. * Can be used to provide arguments for '%*pb[l]' when printing a cpumask.
  29. */
  30. #define cpumask_pr_args(maskp) nr_cpu_ids, cpumask_bits(maskp)
  31. #if NR_CPUS == 1
  32. #define nr_cpu_ids 1U
  33. #else
  34. extern unsigned int nr_cpu_ids;
  35. #endif
  36. #ifdef CONFIG_CPUMASK_OFFSTACK
  37. /* Assuming NR_CPUS is huge, a runtime limit is more efficient. Also,
  38. * not all bits may be allocated. */
  39. #define nr_cpumask_bits nr_cpu_ids
  40. #else
  41. #define nr_cpumask_bits ((unsigned int)NR_CPUS)
  42. #endif
  43. /*
  44. * The following particular system cpumasks and operations manage
  45. * possible, present, active and online cpus.
  46. *
  47. * cpu_possible_mask- has bit 'cpu' set iff cpu is populatable
  48. * cpu_present_mask - has bit 'cpu' set iff cpu is populated
  49. * cpu_online_mask - has bit 'cpu' set iff cpu available to scheduler
  50. * cpu_active_mask - has bit 'cpu' set iff cpu available to migration
  51. *
  52. * If !CONFIG_HOTPLUG_CPU, present == possible, and active == online.
  53. *
  54. * The cpu_possible_mask is fixed at boot time, as the set of CPU id's
  55. * that it is possible might ever be plugged in at anytime during the
  56. * life of that system boot. The cpu_present_mask is dynamic(*),
  57. * representing which CPUs are currently plugged in. And
  58. * cpu_online_mask is the dynamic subset of cpu_present_mask,
  59. * indicating those CPUs available for scheduling.
  60. *
  61. * If HOTPLUG is enabled, then cpu_possible_mask is forced to have
  62. * all NR_CPUS bits set, otherwise it is just the set of CPUs that
  63. * ACPI reports present at boot.
  64. *
  65. * If HOTPLUG is enabled, then cpu_present_mask varies dynamically,
  66. * depending on what ACPI reports as currently plugged in, otherwise
  67. * cpu_present_mask is just a copy of cpu_possible_mask.
  68. *
  69. * (*) Well, cpu_present_mask is dynamic in the hotplug case. If not
  70. * hotplug, it's a copy of cpu_possible_mask, hence fixed at boot.
  71. *
  72. * Subtleties:
  73. * 1) UP arch's (NR_CPUS == 1, CONFIG_SMP not defined) hardcode
  74. * assumption that their single CPU is online. The UP
  75. * cpu_{online,possible,present}_masks are placebos. Changing them
  76. * will have no useful affect on the following num_*_cpus()
  77. * and cpu_*() macros in the UP case. This ugliness is a UP
  78. * optimization - don't waste any instructions or memory references
  79. * asking if you're online or how many CPUs there are if there is
  80. * only one CPU.
  81. */
  82. extern struct cpumask __cpu_possible_mask;
  83. extern struct cpumask __cpu_online_mask;
  84. extern struct cpumask __cpu_present_mask;
  85. extern struct cpumask __cpu_active_mask;
  86. #define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask)
  87. #define cpu_online_mask ((const struct cpumask *)&__cpu_online_mask)
  88. #define cpu_present_mask ((const struct cpumask *)&__cpu_present_mask)
  89. #define cpu_active_mask ((const struct cpumask *)&__cpu_active_mask)
  90. extern atomic_t __num_online_cpus;
  91. #if NR_CPUS > 1
  92. /**
  93. * num_online_cpus() - Read the number of online CPUs
  94. *
  95. * Despite the fact that __num_online_cpus is of type atomic_t, this
  96. * interface gives only a momentary snapshot and is not protected against
  97. * concurrent CPU hotplug operations unless invoked from a cpuhp_lock held
  98. * region.
  99. */
  100. static inline unsigned int num_online_cpus(void)
  101. {
  102. return atomic_read(&__num_online_cpus);
  103. }
  104. #define num_possible_cpus() cpumask_weight(cpu_possible_mask)
  105. #define num_present_cpus() cpumask_weight(cpu_present_mask)
  106. #define num_active_cpus() cpumask_weight(cpu_active_mask)
  107. #define cpu_online(cpu) cpumask_test_cpu((cpu), cpu_online_mask)
  108. #define cpu_possible(cpu) cpumask_test_cpu((cpu), cpu_possible_mask)
  109. #define cpu_present(cpu) cpumask_test_cpu((cpu), cpu_present_mask)
  110. #define cpu_active(cpu) cpumask_test_cpu((cpu), cpu_active_mask)
  111. #else
  112. #define num_online_cpus() 1U
  113. #define num_possible_cpus() 1U
  114. #define num_present_cpus() 1U
  115. #define num_active_cpus() 1U
  116. #define cpu_online(cpu) ((cpu) == 0)
  117. #define cpu_possible(cpu) ((cpu) == 0)
  118. #define cpu_present(cpu) ((cpu) == 0)
  119. #define cpu_active(cpu) ((cpu) == 0)
  120. #endif
  121. extern cpumask_t cpus_booted_once_mask;
  122. static inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits)
  123. {
  124. #ifdef CONFIG_DEBUG_PER_CPU_MAPS
  125. WARN_ON_ONCE(cpu >= bits);
  126. #endif /* CONFIG_DEBUG_PER_CPU_MAPS */
  127. }
  128. /* verify cpu argument to cpumask_* operators */
  129. static inline unsigned int cpumask_check(unsigned int cpu)
  130. {
  131. cpu_max_bits_warn(cpu, nr_cpumask_bits);
  132. return cpu;
  133. }
  134. #if NR_CPUS == 1
  135. /* Uniprocessor. Assume all masks are "1". */
  136. static inline unsigned int cpumask_first(const struct cpumask *srcp)
  137. {
  138. return 0;
  139. }
  140. static inline unsigned int cpumask_last(const struct cpumask *srcp)
  141. {
  142. return 0;
  143. }
  144. /* Valid inputs for n are -1 and 0. */
  145. static inline unsigned int cpumask_next(int n, const struct cpumask *srcp)
  146. {
  147. return n+1;
  148. }
  149. static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
  150. {
  151. return n+1;
  152. }
  153. static inline unsigned int cpumask_next_and(int n,
  154. const struct cpumask *srcp,
  155. const struct cpumask *andp)
  156. {
  157. return n+1;
  158. }
  159. static inline unsigned int cpumask_next_wrap(int n, const struct cpumask *mask,
  160. int start, bool wrap)
  161. {
  162. /* cpu0 unless stop condition, wrap and at cpu0, then nr_cpumask_bits */
  163. return (wrap && n == 0);
  164. }
  165. /* cpu must be a valid cpu, ie 0, so there's no other choice. */
  166. static inline unsigned int cpumask_any_but(const struct cpumask *mask,
  167. unsigned int cpu)
  168. {
  169. return 1;
  170. }
  171. static inline unsigned int cpumask_local_spread(unsigned int i, int node)
  172. {
  173. return 0;
  174. }
  175. static inline int cpumask_any_and_distribute(const struct cpumask *src1p,
  176. const struct cpumask *src2p) {
  177. return cpumask_next_and(-1, src1p, src2p);
  178. }
  179. #define for_each_cpu(cpu, mask) \
  180. for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
  181. #define for_each_cpu_not(cpu, mask) \
  182. for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
  183. #define for_each_cpu_wrap(cpu, mask, start) \
  184. for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask, (void)(start))
  185. #define for_each_cpu_and(cpu, mask1, mask2) \
  186. for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask1, (void)mask2)
  187. #else
  188. /**
  189. * cpumask_first - get the first cpu in a cpumask
  190. * @srcp: the cpumask pointer
  191. *
  192. * Returns >= nr_cpu_ids if no cpus set.
  193. */
  194. static inline unsigned int cpumask_first(const struct cpumask *srcp)
  195. {
  196. return find_first_bit(cpumask_bits(srcp), nr_cpumask_bits);
  197. }
  198. /**
  199. * cpumask_last - get the last CPU in a cpumask
  200. * @srcp: - the cpumask pointer
  201. *
  202. * Returns >= nr_cpumask_bits if no CPUs set.
  203. */
  204. static inline unsigned int cpumask_last(const struct cpumask *srcp)
  205. {
  206. return find_last_bit(cpumask_bits(srcp), nr_cpumask_bits);
  207. }
  208. unsigned int cpumask_next(int n, const struct cpumask *srcp);
  209. /**
  210. * cpumask_next_zero - get the next unset cpu in a cpumask
  211. * @n: the cpu prior to the place to search (ie. return will be > @n)
  212. * @srcp: the cpumask pointer
  213. *
  214. * Returns >= nr_cpu_ids if no further cpus unset.
  215. */
  216. static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
  217. {
  218. /* -1 is a legal arg here. */
  219. if (n != -1)
  220. cpumask_check(n);
  221. return find_next_zero_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1);
  222. }
  223. int cpumask_next_and(int n, const struct cpumask *, const struct cpumask *);
  224. int cpumask_any_but(const struct cpumask *mask, unsigned int cpu);
  225. unsigned int cpumask_local_spread(unsigned int i, int node);
  226. int cpumask_any_and_distribute(const struct cpumask *src1p,
  227. const struct cpumask *src2p);
  228. /**
  229. * for_each_cpu - iterate over every cpu in a mask
  230. * @cpu: the (optionally unsigned) integer iterator
  231. * @mask: the cpumask pointer
  232. *
  233. * After the loop, cpu is >= nr_cpu_ids.
  234. */
  235. #define for_each_cpu(cpu, mask) \
  236. for ((cpu) = -1; \
  237. (cpu) = cpumask_next((cpu), (mask)), \
  238. (cpu) < nr_cpu_ids;)
  239. /**
  240. * for_each_cpu_not - iterate over every cpu in a complemented mask
  241. * @cpu: the (optionally unsigned) integer iterator
  242. * @mask: the cpumask pointer
  243. *
  244. * After the loop, cpu is >= nr_cpu_ids.
  245. */
  246. #define for_each_cpu_not(cpu, mask) \
  247. for ((cpu) = -1; \
  248. (cpu) = cpumask_next_zero((cpu), (mask)), \
  249. (cpu) < nr_cpu_ids;)
  250. extern int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap);
  251. /**
  252. * for_each_cpu_wrap - iterate over every cpu in a mask, starting at a specified location
  253. * @cpu: the (optionally unsigned) integer iterator
  254. * @mask: the cpumask poiter
  255. * @start: the start location
  256. *
  257. * The implementation does not assume any bit in @mask is set (including @start).
  258. *
  259. * After the loop, cpu is >= nr_cpu_ids.
  260. */
  261. #define for_each_cpu_wrap(cpu, mask, start) \
  262. for ((cpu) = cpumask_next_wrap((start)-1, (mask), (start), false); \
  263. (cpu) < nr_cpumask_bits; \
  264. (cpu) = cpumask_next_wrap((cpu), (mask), (start), true))
  265. /**
  266. * for_each_cpu_and - iterate over every cpu in both masks
  267. * @cpu: the (optionally unsigned) integer iterator
  268. * @mask1: the first cpumask pointer
  269. * @mask2: the second cpumask pointer
  270. *
  271. * This saves a temporary CPU mask in many places. It is equivalent to:
  272. * struct cpumask tmp;
  273. * cpumask_and(&tmp, &mask1, &mask2);
  274. * for_each_cpu(cpu, &tmp)
  275. * ...
  276. *
  277. * After the loop, cpu is >= nr_cpu_ids.
  278. */
  279. #define for_each_cpu_and(cpu, mask1, mask2) \
  280. for ((cpu) = -1; \
  281. (cpu) = cpumask_next_and((cpu), (mask1), (mask2)), \
  282. (cpu) < nr_cpu_ids;)
  283. #endif /* SMP */
  284. #define CPU_BITS_NONE \
  285. { \
  286. [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
  287. }
  288. #define CPU_BITS_CPU0 \
  289. { \
  290. [0] = 1UL \
  291. }
  292. /**
  293. * cpumask_set_cpu - set a cpu in a cpumask
  294. * @cpu: cpu number (< nr_cpu_ids)
  295. * @dstp: the cpumask pointer
  296. */
  297. static inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
  298. {
  299. set_bit(cpumask_check(cpu), cpumask_bits(dstp));
  300. }
  301. static inline void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
  302. {
  303. __set_bit(cpumask_check(cpu), cpumask_bits(dstp));
  304. }
  305. /**
  306. * cpumask_clear_cpu - clear a cpu in a cpumask
  307. * @cpu: cpu number (< nr_cpu_ids)
  308. * @dstp: the cpumask pointer
  309. */
  310. static inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp)
  311. {
  312. clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
  313. }
  314. static inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp)
  315. {
  316. __clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
  317. }
  318. /**
  319. * cpumask_test_cpu - test for a cpu in a cpumask
  320. * @cpu: cpu number (< nr_cpu_ids)
  321. * @cpumask: the cpumask pointer
  322. *
  323. * Returns 1 if @cpu is set in @cpumask, else returns 0
  324. */
  325. static inline int cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
  326. {
  327. return test_bit(cpumask_check(cpu), cpumask_bits((cpumask)));
  328. }
  329. /**
  330. * cpumask_test_and_set_cpu - atomically test and set a cpu in a cpumask
  331. * @cpu: cpu number (< nr_cpu_ids)
  332. * @cpumask: the cpumask pointer
  333. *
  334. * Returns 1 if @cpu is set in old bitmap of @cpumask, else returns 0
  335. *
  336. * test_and_set_bit wrapper for cpumasks.
  337. */
  338. static inline int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
  339. {
  340. return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));
  341. }
  342. /**
  343. * cpumask_test_and_clear_cpu - atomically test and clear a cpu in a cpumask
  344. * @cpu: cpu number (< nr_cpu_ids)
  345. * @cpumask: the cpumask pointer
  346. *
  347. * Returns 1 if @cpu is set in old bitmap of @cpumask, else returns 0
  348. *
  349. * test_and_clear_bit wrapper for cpumasks.
  350. */
  351. static inline int cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
  352. {
  353. return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));
  354. }
  355. /**
  356. * cpumask_setall - set all cpus (< nr_cpu_ids) in a cpumask
  357. * @dstp: the cpumask pointer
  358. */
  359. static inline void cpumask_setall(struct cpumask *dstp)
  360. {
  361. bitmap_fill(cpumask_bits(dstp), nr_cpumask_bits);
  362. }
  363. /**
  364. * cpumask_clear - clear all cpus (< nr_cpu_ids) in a cpumask
  365. * @dstp: the cpumask pointer
  366. */
  367. static inline void cpumask_clear(struct cpumask *dstp)
  368. {
  369. bitmap_zero(cpumask_bits(dstp), nr_cpumask_bits);
  370. }
  371. /**
  372. * cpumask_and - *dstp = *src1p & *src2p
  373. * @dstp: the cpumask result
  374. * @src1p: the first input
  375. * @src2p: the second input
  376. *
  377. * If *@dstp is empty, returns 0, else returns 1
  378. */
  379. static inline int cpumask_and(struct cpumask *dstp,
  380. const struct cpumask *src1p,
  381. const struct cpumask *src2p)
  382. {
  383. return bitmap_and(cpumask_bits(dstp), cpumask_bits(src1p),
  384. cpumask_bits(src2p), nr_cpumask_bits);
  385. }
  386. /**
  387. * cpumask_or - *dstp = *src1p | *src2p
  388. * @dstp: the cpumask result
  389. * @src1p: the first input
  390. * @src2p: the second input
  391. */
  392. static inline void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p,
  393. const struct cpumask *src2p)
  394. {
  395. bitmap_or(cpumask_bits(dstp), cpumask_bits(src1p),
  396. cpumask_bits(src2p), nr_cpumask_bits);
  397. }
  398. /**
  399. * cpumask_xor - *dstp = *src1p ^ *src2p
  400. * @dstp: the cpumask result
  401. * @src1p: the first input
  402. * @src2p: the second input
  403. */
  404. static inline void cpumask_xor(struct cpumask *dstp,
  405. const struct cpumask *src1p,
  406. const struct cpumask *src2p)
  407. {
  408. bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p),
  409. cpumask_bits(src2p), nr_cpumask_bits);
  410. }
  411. /**
  412. * cpumask_andnot - *dstp = *src1p & ~*src2p
  413. * @dstp: the cpumask result
  414. * @src1p: the first input
  415. * @src2p: the second input
  416. *
  417. * If *@dstp is empty, returns 0, else returns 1
  418. */
  419. static inline int cpumask_andnot(struct cpumask *dstp,
  420. const struct cpumask *src1p,
  421. const struct cpumask *src2p)
  422. {
  423. return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p),
  424. cpumask_bits(src2p), nr_cpumask_bits);
  425. }
  426. /**
  427. * cpumask_complement - *dstp = ~*srcp
  428. * @dstp: the cpumask result
  429. * @srcp: the input to invert
  430. */
  431. static inline void cpumask_complement(struct cpumask *dstp,
  432. const struct cpumask *srcp)
  433. {
  434. bitmap_complement(cpumask_bits(dstp), cpumask_bits(srcp),
  435. nr_cpumask_bits);
  436. }
  437. /**
  438. * cpumask_equal - *src1p == *src2p
  439. * @src1p: the first input
  440. * @src2p: the second input
  441. */
  442. static inline bool cpumask_equal(const struct cpumask *src1p,
  443. const struct cpumask *src2p)
  444. {
  445. return bitmap_equal(cpumask_bits(src1p), cpumask_bits(src2p),
  446. nr_cpumask_bits);
  447. }
  448. /**
  449. * cpumask_or_equal - *src1p | *src2p == *src3p
  450. * @src1p: the first input
  451. * @src2p: the second input
  452. * @src3p: the third input
  453. */
  454. static inline bool cpumask_or_equal(const struct cpumask *src1p,
  455. const struct cpumask *src2p,
  456. const struct cpumask *src3p)
  457. {
  458. return bitmap_or_equal(cpumask_bits(src1p), cpumask_bits(src2p),
  459. cpumask_bits(src3p), nr_cpumask_bits);
  460. }
  461. /**
  462. * cpumask_intersects - (*src1p & *src2p) != 0
  463. * @src1p: the first input
  464. * @src2p: the second input
  465. */
  466. static inline bool cpumask_intersects(const struct cpumask *src1p,
  467. const struct cpumask *src2p)
  468. {
  469. return bitmap_intersects(cpumask_bits(src1p), cpumask_bits(src2p),
  470. nr_cpumask_bits);
  471. }
  472. /**
  473. * cpumask_subset - (*src1p & ~*src2p) == 0
  474. * @src1p: the first input
  475. * @src2p: the second input
  476. *
  477. * Returns 1 if *@src1p is a subset of *@src2p, else returns 0
  478. */
  479. static inline int cpumask_subset(const struct cpumask *src1p,
  480. const struct cpumask *src2p)
  481. {
  482. return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p),
  483. nr_cpumask_bits);
  484. }
  485. /**
  486. * cpumask_empty - *srcp == 0
  487. * @srcp: the cpumask to that all cpus < nr_cpu_ids are clear.
  488. */
  489. static inline bool cpumask_empty(const struct cpumask *srcp)
  490. {
  491. return bitmap_empty(cpumask_bits(srcp), nr_cpumask_bits);
  492. }
  493. /**
  494. * cpumask_full - *srcp == 0xFFFFFFFF...
  495. * @srcp: the cpumask to that all cpus < nr_cpu_ids are set.
  496. */
  497. static inline bool cpumask_full(const struct cpumask *srcp)
  498. {
  499. return bitmap_full(cpumask_bits(srcp), nr_cpumask_bits);
  500. }
  501. /**
  502. * cpumask_weight - Count of bits in *srcp
  503. * @srcp: the cpumask to count bits (< nr_cpu_ids) in.
  504. */
  505. static inline unsigned int cpumask_weight(const struct cpumask *srcp)
  506. {
  507. return bitmap_weight(cpumask_bits(srcp), nr_cpumask_bits);
  508. }
  509. /**
  510. * cpumask_shift_right - *dstp = *srcp >> n
  511. * @dstp: the cpumask result
  512. * @srcp: the input to shift
  513. * @n: the number of bits to shift by
  514. */
  515. static inline void cpumask_shift_right(struct cpumask *dstp,
  516. const struct cpumask *srcp, int n)
  517. {
  518. bitmap_shift_right(cpumask_bits(dstp), cpumask_bits(srcp), n,
  519. nr_cpumask_bits);
  520. }
  521. /**
  522. * cpumask_shift_left - *dstp = *srcp << n
  523. * @dstp: the cpumask result
  524. * @srcp: the input to shift
  525. * @n: the number of bits to shift by
  526. */
  527. static inline void cpumask_shift_left(struct cpumask *dstp,
  528. const struct cpumask *srcp, int n)
  529. {
  530. bitmap_shift_left(cpumask_bits(dstp), cpumask_bits(srcp), n,
  531. nr_cpumask_bits);
  532. }
  533. /**
  534. * cpumask_copy - *dstp = *srcp
  535. * @dstp: the result
  536. * @srcp: the input cpumask
  537. */
  538. static inline void cpumask_copy(struct cpumask *dstp,
  539. const struct cpumask *srcp)
  540. {
  541. bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), nr_cpumask_bits);
  542. }
  543. /**
  544. * cpumask_any - pick a "random" cpu from *srcp
  545. * @srcp: the input cpumask
  546. *
  547. * Returns >= nr_cpu_ids if no cpus set.
  548. */
  549. #define cpumask_any(srcp) cpumask_first(srcp)
  550. /**
  551. * cpumask_first_and - return the first cpu from *srcp1 & *srcp2
  552. * @src1p: the first input
  553. * @src2p: the second input
  554. *
  555. * Returns >= nr_cpu_ids if no cpus set in both. See also cpumask_next_and().
  556. */
  557. #define cpumask_first_and(src1p, src2p) cpumask_next_and(-1, (src1p), (src2p))
  558. /**
  559. * cpumask_any_and - pick a "random" cpu from *mask1 & *mask2
  560. * @mask1: the first input cpumask
  561. * @mask2: the second input cpumask
  562. *
  563. * Returns >= nr_cpu_ids if no cpus set.
  564. */
  565. #define cpumask_any_and(mask1, mask2) cpumask_first_and((mask1), (mask2))
  566. /**
  567. * cpumask_of - the cpumask containing just a given cpu
  568. * @cpu: the cpu (<= nr_cpu_ids)
  569. */
  570. #define cpumask_of(cpu) (get_cpu_mask(cpu))
  571. /**
  572. * cpumask_parse_user - extract a cpumask from a user string
  573. * @buf: the buffer to extract from
  574. * @len: the length of the buffer
  575. * @dstp: the cpumask to set.
  576. *
  577. * Returns -errno, or 0 for success.
  578. */
  579. static inline int cpumask_parse_user(const char __user *buf, int len,
  580. struct cpumask *dstp)
  581. {
  582. return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits);
  583. }
  584. /**
  585. * cpumask_parselist_user - extract a cpumask from a user string
  586. * @buf: the buffer to extract from
  587. * @len: the length of the buffer
  588. * @dstp: the cpumask to set.
  589. *
  590. * Returns -errno, or 0 for success.
  591. */
  592. static inline int cpumask_parselist_user(const char __user *buf, int len,
  593. struct cpumask *dstp)
  594. {
  595. return bitmap_parselist_user(buf, len, cpumask_bits(dstp),
  596. nr_cpumask_bits);
  597. }
  598. /**
  599. * cpumask_parse - extract a cpumask from a string
  600. * @buf: the buffer to extract from
  601. * @dstp: the cpumask to set.
  602. *
  603. * Returns -errno, or 0 for success.
  604. */
  605. static inline int cpumask_parse(const char *buf, struct cpumask *dstp)
  606. {
  607. return bitmap_parse(buf, UINT_MAX, cpumask_bits(dstp), nr_cpumask_bits);
  608. }
  609. /**
  610. * cpulist_parse - extract a cpumask from a user string of ranges
  611. * @buf: the buffer to extract from
  612. * @dstp: the cpumask to set.
  613. *
  614. * Returns -errno, or 0 for success.
  615. */
  616. static inline int cpulist_parse(const char *buf, struct cpumask *dstp)
  617. {
  618. return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpumask_bits);
  619. }
  620. /**
  621. * cpumask_size - size to allocate for a 'struct cpumask' in bytes
  622. */
  623. static inline unsigned int cpumask_size(void)
  624. {
  625. return BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long);
  626. }
  627. /*
  628. * cpumask_var_t: struct cpumask for stack usage.
  629. *
  630. * Oh, the wicked games we play! In order to make kernel coding a
  631. * little more difficult, we typedef cpumask_var_t to an array or a
  632. * pointer: doing &mask on an array is a noop, so it still works.
  633. *
  634. * ie.
  635. * cpumask_var_t tmpmask;
  636. * if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
  637. * return -ENOMEM;
  638. *
  639. * ... use 'tmpmask' like a normal struct cpumask * ...
  640. *
  641. * free_cpumask_var(tmpmask);
  642. *
  643. *
  644. * However, one notable exception is there. alloc_cpumask_var() allocates
  645. * only nr_cpumask_bits bits (in the other hand, real cpumask_t always has
  646. * NR_CPUS bits). Therefore you don't have to dereference cpumask_var_t.
  647. *
  648. * cpumask_var_t tmpmask;
  649. * if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
  650. * return -ENOMEM;
  651. *
  652. * var = *tmpmask;
  653. *
  654. * This code makes NR_CPUS length memcopy and brings to a memory corruption.
  655. * cpumask_copy() provide safe copy functionality.
  656. *
  657. * Note that there is another evil here: If you define a cpumask_var_t
  658. * as a percpu variable then the way to obtain the address of the cpumask
  659. * structure differently influences what this_cpu_* operation needs to be
  660. * used. Please use this_cpu_cpumask_var_t in those cases. The direct use
  661. * of this_cpu_ptr() or this_cpu_read() will lead to failures when the
  662. * other type of cpumask_var_t implementation is configured.
  663. *
  664. * Please also note that __cpumask_var_read_mostly can be used to declare
  665. * a cpumask_var_t variable itself (not its content) as read mostly.
  666. */
  667. #ifdef CONFIG_CPUMASK_OFFSTACK
  668. typedef struct cpumask *cpumask_var_t;
  669. #define this_cpu_cpumask_var_ptr(x) this_cpu_read(x)
  670. #define __cpumask_var_read_mostly __read_mostly
  671. bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
  672. bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);
  673. bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
  674. bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);
  675. void alloc_bootmem_cpumask_var(cpumask_var_t *mask);
  676. void free_cpumask_var(cpumask_var_t mask);
  677. void free_bootmem_cpumask_var(cpumask_var_t mask);
  678. static inline bool cpumask_available(cpumask_var_t mask)
  679. {
  680. return mask != NULL;
  681. }
  682. #else
  683. typedef struct cpumask cpumask_var_t[1];
  684. #define this_cpu_cpumask_var_ptr(x) this_cpu_ptr(x)
  685. #define __cpumask_var_read_mostly
  686. static inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
  687. {
  688. return true;
  689. }
  690. static inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
  691. int node)
  692. {
  693. return true;
  694. }
  695. static inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
  696. {
  697. cpumask_clear(*mask);
  698. return true;
  699. }
  700. static inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
  701. int node)
  702. {
  703. cpumask_clear(*mask);
  704. return true;
  705. }
  706. static inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask)
  707. {
  708. }
  709. static inline void free_cpumask_var(cpumask_var_t mask)
  710. {
  711. }
  712. static inline void free_bootmem_cpumask_var(cpumask_var_t mask)
  713. {
  714. }
  715. static inline bool cpumask_available(cpumask_var_t mask)
  716. {
  717. return true;
  718. }
  719. #endif /* CONFIG_CPUMASK_OFFSTACK */
  720. /* It's common to want to use cpu_all_mask in struct member initializers,
  721. * so it has to refer to an address rather than a pointer. */
  722. extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS);
  723. #define cpu_all_mask to_cpumask(cpu_all_bits)
  724. /* First bits of cpu_bit_bitmap are in fact unset. */
  725. #define cpu_none_mask to_cpumask(cpu_bit_bitmap[0])
  726. #define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask)
  727. #define for_each_online_cpu(cpu) for_each_cpu((cpu), cpu_online_mask)
  728. #define for_each_present_cpu(cpu) for_each_cpu((cpu), cpu_present_mask)
  729. /* Wrappers for arch boot code to manipulate normally-constant masks */
  730. void init_cpu_present(const struct cpumask *src);
  731. void init_cpu_possible(const struct cpumask *src);
  732. void init_cpu_online(const struct cpumask *src);
  733. static inline void reset_cpu_possible_mask(void)
  734. {
  735. bitmap_zero(cpumask_bits(&__cpu_possible_mask), NR_CPUS);
  736. }
  737. static inline void
  738. set_cpu_possible(unsigned int cpu, bool possible)
  739. {
  740. if (possible)
  741. cpumask_set_cpu(cpu, &__cpu_possible_mask);
  742. else
  743. cpumask_clear_cpu(cpu, &__cpu_possible_mask);
  744. }
  745. static inline void
  746. set_cpu_present(unsigned int cpu, bool present)
  747. {
  748. if (present)
  749. cpumask_set_cpu(cpu, &__cpu_present_mask);
  750. else
  751. cpumask_clear_cpu(cpu, &__cpu_present_mask);
  752. }
  753. void set_cpu_online(unsigned int cpu, bool online);
  754. static inline void
  755. set_cpu_active(unsigned int cpu, bool active)
  756. {
  757. if (active)
  758. cpumask_set_cpu(cpu, &__cpu_active_mask);
  759. else
  760. cpumask_clear_cpu(cpu, &__cpu_active_mask);
  761. }
  762. /**
  763. * to_cpumask - convert an NR_CPUS bitmap to a struct cpumask *
  764. * @bitmap: the bitmap
  765. *
  766. * There are a few places where cpumask_var_t isn't appropriate and
  767. * static cpumasks must be used (eg. very early boot), yet we don't
  768. * expose the definition of 'struct cpumask'.
  769. *
  770. * This does the conversion, and can be used as a constant initializer.
  771. */
  772. #define to_cpumask(bitmap) \
  773. ((struct cpumask *)(1 ? (bitmap) \
  774. : (void *)sizeof(__check_is_bitmap(bitmap))))
  775. static inline int __check_is_bitmap(const unsigned long *bitmap)
  776. {
  777. return 1;
  778. }
  779. /*
  780. * Special-case data structure for "single bit set only" constant CPU masks.
  781. *
  782. * We pre-generate all the 64 (or 32) possible bit positions, with enough
  783. * padding to the left and the right, and return the constant pointer
  784. * appropriately offset.
  785. */
  786. extern const unsigned long
  787. cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)];
  788. static inline const struct cpumask *get_cpu_mask(unsigned int cpu)
  789. {
  790. const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG];
  791. p -= cpu / BITS_PER_LONG;
  792. return to_cpumask(p);
  793. }
  794. #define cpu_is_offline(cpu) unlikely(!cpu_online(cpu))
  795. #if NR_CPUS <= BITS_PER_LONG
  796. #define CPU_BITS_ALL \
  797. { \
  798. [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
  799. }
  800. #else /* NR_CPUS > BITS_PER_LONG */
  801. #define CPU_BITS_ALL \
  802. { \
  803. [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
  804. [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
  805. }
  806. #endif /* NR_CPUS > BITS_PER_LONG */
  807. /**
  808. * cpumap_print_to_pagebuf - copies the cpumask into the buffer either
  809. * as comma-separated list of cpus or hex values of cpumask
  810. * @list: indicates whether the cpumap must be list
  811. * @mask: the cpumask to copy
  812. * @buf: the buffer to copy into
  813. *
  814. * Returns the length of the (null-terminated) @buf string, zero if
  815. * nothing is copied.
  816. */
  817. static inline ssize_t
  818. cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask)
  819. {
  820. return bitmap_print_to_pagebuf(list, buf, cpumask_bits(mask),
  821. nr_cpu_ids);
  822. }
  823. #if NR_CPUS <= BITS_PER_LONG
  824. #define CPU_MASK_ALL \
  825. (cpumask_t) { { \
  826. [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
  827. } }
  828. #else
  829. #define CPU_MASK_ALL \
  830. (cpumask_t) { { \
  831. [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
  832. [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
  833. } }
  834. #endif /* NR_CPUS > BITS_PER_LONG */
  835. #define CPU_MASK_NONE \
  836. (cpumask_t) { { \
  837. [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
  838. } }
  839. #define CPU_MASK_CPU0 \
  840. (cpumask_t) { { \
  841. [0] = 1UL \
  842. } }
  843. #endif /* __LINUX_CPUMASK_H */