cpumask.c 6.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263
  1. // SPDX-License-Identifier: GPL-2.0
  2. #include <linux/slab.h>
  3. #include <linux/kernel.h>
  4. #include <linux/bitops.h>
  5. #include <linux/cpumask.h>
  6. #include <linux/export.h>
  7. #include <linux/memblock.h>
  8. #include <linux/numa.h>
  9. /**
  10. * cpumask_next - get the next cpu in a cpumask
  11. * @n: the cpu prior to the place to search (ie. return will be > @n)
  12. * @srcp: the cpumask pointer
  13. *
  14. * Returns >= nr_cpu_ids if no further cpus set.
  15. */
  16. unsigned int cpumask_next(int n, const struct cpumask *srcp)
  17. {
  18. /* -1 is a legal arg here. */
  19. if (n != -1)
  20. cpumask_check(n);
  21. return find_next_bit(cpumask_bits(srcp), nr_cpumask_bits, n + 1);
  22. }
  23. EXPORT_SYMBOL(cpumask_next);
  24. /**
  25. * cpumask_next_and - get the next cpu in *src1p & *src2p
  26. * @n: the cpu prior to the place to search (ie. return will be > @n)
  27. * @src1p: the first cpumask pointer
  28. * @src2p: the second cpumask pointer
  29. *
  30. * Returns >= nr_cpu_ids if no further cpus set in both.
  31. */
  32. int cpumask_next_and(int n, const struct cpumask *src1p,
  33. const struct cpumask *src2p)
  34. {
  35. /* -1 is a legal arg here. */
  36. if (n != -1)
  37. cpumask_check(n);
  38. return find_next_and_bit(cpumask_bits(src1p), cpumask_bits(src2p),
  39. nr_cpumask_bits, n + 1);
  40. }
  41. EXPORT_SYMBOL(cpumask_next_and);
  42. /**
  43. * cpumask_any_but - return a "random" in a cpumask, but not this one.
  44. * @mask: the cpumask to search
  45. * @cpu: the cpu to ignore.
  46. *
  47. * Often used to find any cpu but smp_processor_id() in a mask.
  48. * Returns >= nr_cpu_ids if no cpus set.
  49. */
  50. int cpumask_any_but(const struct cpumask *mask, unsigned int cpu)
  51. {
  52. unsigned int i;
  53. cpumask_check(cpu);
  54. for_each_cpu(i, mask)
  55. if (i != cpu)
  56. break;
  57. return i;
  58. }
  59. EXPORT_SYMBOL(cpumask_any_but);
  60. /**
  61. * cpumask_next_wrap - helper to implement for_each_cpu_wrap
  62. * @n: the cpu prior to the place to search
  63. * @mask: the cpumask pointer
  64. * @start: the start point of the iteration
  65. * @wrap: assume @n crossing @start terminates the iteration
  66. *
  67. * Returns >= nr_cpu_ids on completion
  68. *
  69. * Note: the @wrap argument is required for the start condition when
  70. * we cannot assume @start is set in @mask.
  71. */
  72. int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap)
  73. {
  74. int next;
  75. again:
  76. next = cpumask_next(n, mask);
  77. if (wrap && n < start && next >= start) {
  78. return nr_cpumask_bits;
  79. } else if (next >= nr_cpumask_bits) {
  80. wrap = true;
  81. n = -1;
  82. goto again;
  83. }
  84. return next;
  85. }
  86. EXPORT_SYMBOL(cpumask_next_wrap);
  87. /* These are not inline because of header tangles. */
  88. #ifdef CONFIG_CPUMASK_OFFSTACK
  89. /**
  90. * alloc_cpumask_var_node - allocate a struct cpumask on a given node
  91. * @mask: pointer to cpumask_var_t where the cpumask is returned
  92. * @flags: GFP_ flags
  93. *
  94. * Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is
  95. * a nop returning a constant 1 (in <linux/cpumask.h>)
  96. * Returns TRUE if memory allocation succeeded, FALSE otherwise.
  97. *
  98. * In addition, mask will be NULL if this fails. Note that gcc is
  99. * usually smart enough to know that mask can never be NULL if
  100. * CONFIG_CPUMASK_OFFSTACK=n, so does code elimination in that case
  101. * too.
  102. */
  103. bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node)
  104. {
  105. *mask = kmalloc_node(cpumask_size(), flags, node);
  106. #ifdef CONFIG_DEBUG_PER_CPU_MAPS
  107. if (!*mask) {
  108. printk(KERN_ERR "=> alloc_cpumask_var: failed!\n");
  109. dump_stack();
  110. }
  111. #endif
  112. return *mask != NULL;
  113. }
  114. EXPORT_SYMBOL(alloc_cpumask_var_node);
  115. bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node)
  116. {
  117. return alloc_cpumask_var_node(mask, flags | __GFP_ZERO, node);
  118. }
  119. EXPORT_SYMBOL(zalloc_cpumask_var_node);
  120. /**
  121. * alloc_cpumask_var - allocate a struct cpumask
  122. * @mask: pointer to cpumask_var_t where the cpumask is returned
  123. * @flags: GFP_ flags
  124. *
  125. * Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is
  126. * a nop returning a constant 1 (in <linux/cpumask.h>).
  127. *
  128. * See alloc_cpumask_var_node.
  129. */
  130. bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
  131. {
  132. return alloc_cpumask_var_node(mask, flags, NUMA_NO_NODE);
  133. }
  134. EXPORT_SYMBOL(alloc_cpumask_var);
  135. bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
  136. {
  137. return alloc_cpumask_var(mask, flags | __GFP_ZERO);
  138. }
  139. EXPORT_SYMBOL(zalloc_cpumask_var);
  140. /**
  141. * alloc_bootmem_cpumask_var - allocate a struct cpumask from the bootmem arena.
  142. * @mask: pointer to cpumask_var_t where the cpumask is returned
  143. *
  144. * Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is
  145. * a nop (in <linux/cpumask.h>).
  146. * Either returns an allocated (zero-filled) cpumask, or causes the
  147. * system to panic.
  148. */
  149. void __init alloc_bootmem_cpumask_var(cpumask_var_t *mask)
  150. {
  151. *mask = memblock_alloc(cpumask_size(), SMP_CACHE_BYTES);
  152. if (!*mask)
  153. panic("%s: Failed to allocate %u bytes\n", __func__,
  154. cpumask_size());
  155. }
  156. /**
  157. * free_cpumask_var - frees memory allocated for a struct cpumask.
  158. * @mask: cpumask to free
  159. *
  160. * This is safe on a NULL mask.
  161. */
  162. void free_cpumask_var(cpumask_var_t mask)
  163. {
  164. kfree(mask);
  165. }
  166. EXPORT_SYMBOL(free_cpumask_var);
  167. /**
  168. * free_bootmem_cpumask_var - frees result of alloc_bootmem_cpumask_var
  169. * @mask: cpumask to free
  170. */
  171. void __init free_bootmem_cpumask_var(cpumask_var_t mask)
  172. {
  173. memblock_free_early(__pa(mask), cpumask_size());
  174. }
  175. #endif
  176. /**
  177. * cpumask_local_spread - select the i'th cpu with local numa cpu's first
  178. * @i: index number
  179. * @node: local numa_node
  180. *
  181. * This function selects an online CPU according to a numa aware policy;
  182. * local cpus are returned first, followed by non-local ones, then it
  183. * wraps around.
  184. *
  185. * It's not very efficient, but useful for setup.
  186. */
  187. unsigned int cpumask_local_spread(unsigned int i, int node)
  188. {
  189. int cpu;
  190. /* Wrap: we always want a cpu. */
  191. i %= num_online_cpus();
  192. if (node == NUMA_NO_NODE) {
  193. for_each_cpu(cpu, cpu_online_mask)
  194. if (i-- == 0)
  195. return cpu;
  196. } else {
  197. /* NUMA first. */
  198. for_each_cpu_and(cpu, cpumask_of_node(node), cpu_online_mask)
  199. if (i-- == 0)
  200. return cpu;
  201. for_each_cpu(cpu, cpu_online_mask) {
  202. /* Skip NUMA nodes, done above. */
  203. if (cpumask_test_cpu(cpu, cpumask_of_node(node)))
  204. continue;
  205. if (i-- == 0)
  206. return cpu;
  207. }
  208. }
  209. BUG();
  210. }
  211. EXPORT_SYMBOL(cpumask_local_spread);
  212. static DEFINE_PER_CPU(int, distribute_cpu_mask_prev);
  213. /**
  214. * Returns an arbitrary cpu within srcp1 & srcp2.
  215. *
  216. * Iterated calls using the same srcp1 and srcp2 will be distributed within
  217. * their intersection.
  218. *
  219. * Returns >= nr_cpu_ids if the intersection is empty.
  220. */
  221. int cpumask_any_and_distribute(const struct cpumask *src1p,
  222. const struct cpumask *src2p)
  223. {
  224. int next, prev;
  225. /* NOTE: our first selection will skip 0. */
  226. prev = __this_cpu_read(distribute_cpu_mask_prev);
  227. next = cpumask_next_and(prev, src1p, src2p);
  228. if (next >= nr_cpu_ids)
  229. next = cpumask_first_and(src1p, src2p);
  230. if (next < nr_cpu_ids)
  231. __this_cpu_write(distribute_cpu_mask_prev, next);
  232. return next;
  233. }
  234. EXPORT_SYMBOL(cpumask_any_and_distribute);