nodemask.h 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. #ifndef __LINUX_NODEMASK_H
  3. #define __LINUX_NODEMASK_H
  4. /*
  5. * Nodemasks provide a bitmap suitable for representing the
  6. * set of Node's in a system, one bit position per Node number.
  7. *
  8. * See detailed comments in the file linux/bitmap.h describing the
  9. * data type on which these nodemasks are based.
  10. *
  11. * For details of nodemask_parse_user(), see bitmap_parse_user() in
  12. * lib/bitmap.c. For details of nodelist_parse(), see bitmap_parselist(),
  13. * also in bitmap.c. For details of node_remap(), see bitmap_bitremap in
  14. * lib/bitmap.c. For details of nodes_remap(), see bitmap_remap in
  15. * lib/bitmap.c. For details of nodes_onto(), see bitmap_onto in
  16. * lib/bitmap.c. For details of nodes_fold(), see bitmap_fold in
  17. * lib/bitmap.c.
  18. *
  19. * The available nodemask operations are:
  20. *
  21. * void node_set(node, mask) turn on bit 'node' in mask
  22. * void node_clear(node, mask) turn off bit 'node' in mask
  23. * void nodes_setall(mask) set all bits
  24. * void nodes_clear(mask) clear all bits
  25. * int node_isset(node, mask) true iff bit 'node' set in mask
  26. * int node_test_and_set(node, mask) test and set bit 'node' in mask
  27. *
  28. * void nodes_and(dst, src1, src2) dst = src1 & src2 [intersection]
  29. * void nodes_or(dst, src1, src2) dst = src1 | src2 [union]
  30. * void nodes_xor(dst, src1, src2) dst = src1 ^ src2
  31. * void nodes_andnot(dst, src1, src2) dst = src1 & ~src2
  32. * void nodes_complement(dst, src) dst = ~src
  33. *
  34. * int nodes_equal(mask1, mask2) Does mask1 == mask2?
  35. * int nodes_intersects(mask1, mask2) Do mask1 and mask2 intersect?
  36. * int nodes_subset(mask1, mask2) Is mask1 a subset of mask2?
  37. * int nodes_empty(mask) Is mask empty (no bits sets)?
  38. * int nodes_full(mask) Is mask full (all bits sets)?
  39. * int nodes_weight(mask) Hamming weight - number of set bits
  40. *
  41. * void nodes_shift_right(dst, src, n) Shift right
  42. * void nodes_shift_left(dst, src, n) Shift left
  43. *
  44. * int first_node(mask) Number lowest set bit, or MAX_NUMNODES
  45. * int next_node(node, mask) Next node past 'node', or MAX_NUMNODES
  46. * int next_node_in(node, mask) Next node past 'node', or wrap to first,
  47. * or MAX_NUMNODES
  48. * int first_unset_node(mask) First node not set in mask, or
  49. * MAX_NUMNODES
  50. *
  51. * nodemask_t nodemask_of_node(node) Return nodemask with bit 'node' set
  52. * NODE_MASK_ALL Initializer - all bits set
  53. * NODE_MASK_NONE Initializer - no bits set
  54. * unsigned long *nodes_addr(mask) Array of unsigned long's in mask
  55. *
  56. * int nodemask_parse_user(ubuf, ulen, mask) Parse ascii string as nodemask
  57. * int nodelist_parse(buf, map) Parse ascii string as nodelist
  58. * int node_remap(oldbit, old, new) newbit = map(old, new)(oldbit)
  59. * void nodes_remap(dst, src, old, new) *dst = map(old, new)(src)
  60. * void nodes_onto(dst, orig, relmap) *dst = orig relative to relmap
  61. * void nodes_fold(dst, orig, sz) dst bits = orig bits mod sz
  62. *
  63. * for_each_node_mask(node, mask) for-loop node over mask
  64. *
  65. * int num_online_nodes() Number of online Nodes
  66. * int num_possible_nodes() Number of all possible Nodes
  67. *
  68. * int node_random(mask) Random node with set bit in mask
  69. *
  70. * int node_online(node) Is some node online?
  71. * int node_possible(node) Is some node possible?
  72. *
  73. * node_set_online(node) set bit 'node' in node_online_map
  74. * node_set_offline(node) clear bit 'node' in node_online_map
  75. *
  76. * for_each_node(node) for-loop node over node_possible_map
  77. * for_each_online_node(node) for-loop node over node_online_map
  78. *
  79. * Subtlety:
  80. * 1) The 'type-checked' form of node_isset() causes gcc (3.3.2, anyway)
  81. * to generate slightly worse code. So use a simple one-line #define
  82. * for node_isset(), instead of wrapping an inline inside a macro, the
  83. * way we do the other calls.
  84. *
  85. * NODEMASK_SCRATCH
  86. * When doing above logical AND, OR, XOR, Remap operations the callers tend to
  87. * need temporary nodemask_t's on the stack. But if NODES_SHIFT is large,
  88. * nodemask_t's consume too much stack space. NODEMASK_SCRATCH is a helper
  89. * for such situations. See below and CPUMASK_ALLOC also.
  90. */
  91. #include <linux/threads.h>
  92. #include <linux/bitmap.h>
  93. #include <linux/minmax.h>
  94. #include <linux/numa.h>
  95. typedef struct { DECLARE_BITMAP(bits, MAX_NUMNODES); } nodemask_t;
  96. extern nodemask_t _unused_nodemask_arg_;
  97. /**
  98. * nodemask_pr_args - printf args to output a nodemask
  99. * @maskp: nodemask to be printed
  100. *
  101. * Can be used to provide arguments for '%*pb[l]' when printing a nodemask.
  102. */
  103. #define nodemask_pr_args(maskp) __nodemask_pr_numnodes(maskp), \
  104. __nodemask_pr_bits(maskp)
  105. static inline unsigned int __nodemask_pr_numnodes(const nodemask_t *m)
  106. {
  107. return m ? MAX_NUMNODES : 0;
  108. }
  109. static inline const unsigned long *__nodemask_pr_bits(const nodemask_t *m)
  110. {
  111. return m ? m->bits : NULL;
  112. }
  113. /*
  114. * The inline keyword gives the compiler room to decide to inline, or
  115. * not inline a function as it sees best. However, as these functions
  116. * are called in both __init and non-__init functions, if they are not
  117. * inlined we will end up with a section mis-match error (of the type of
  118. * freeable items not being freed). So we must use __always_inline here
  119. * to fix the problem. If other functions in the future also end up in
  120. * this situation they will also need to be annotated as __always_inline
  121. */
  122. #define node_set(node, dst) __node_set((node), &(dst))
  123. static __always_inline void __node_set(int node, volatile nodemask_t *dstp)
  124. {
  125. set_bit(node, dstp->bits);
  126. }
  127. #define node_clear(node, dst) __node_clear((node), &(dst))
  128. static inline void __node_clear(int node, volatile nodemask_t *dstp)
  129. {
  130. clear_bit(node, dstp->bits);
  131. }
  132. #define nodes_setall(dst) __nodes_setall(&(dst), MAX_NUMNODES)
  133. static inline void __nodes_setall(nodemask_t *dstp, unsigned int nbits)
  134. {
  135. bitmap_fill(dstp->bits, nbits);
  136. }
  137. #define nodes_clear(dst) __nodes_clear(&(dst), MAX_NUMNODES)
  138. static inline void __nodes_clear(nodemask_t *dstp, unsigned int nbits)
  139. {
  140. bitmap_zero(dstp->bits, nbits);
  141. }
  142. /* No static inline type checking - see Subtlety (1) above. */
  143. #define node_isset(node, nodemask) test_bit((node), (nodemask).bits)
  144. #define node_test_and_set(node, nodemask) \
  145. __node_test_and_set((node), &(nodemask))
  146. static inline int __node_test_and_set(int node, nodemask_t *addr)
  147. {
  148. return test_and_set_bit(node, addr->bits);
  149. }
  150. #define nodes_and(dst, src1, src2) \
  151. __nodes_and(&(dst), &(src1), &(src2), MAX_NUMNODES)
  152. static inline void __nodes_and(nodemask_t *dstp, const nodemask_t *src1p,
  153. const nodemask_t *src2p, unsigned int nbits)
  154. {
  155. bitmap_and(dstp->bits, src1p->bits, src2p->bits, nbits);
  156. }
  157. #define nodes_or(dst, src1, src2) \
  158. __nodes_or(&(dst), &(src1), &(src2), MAX_NUMNODES)
  159. static inline void __nodes_or(nodemask_t *dstp, const nodemask_t *src1p,
  160. const nodemask_t *src2p, unsigned int nbits)
  161. {
  162. bitmap_or(dstp->bits, src1p->bits, src2p->bits, nbits);
  163. }
  164. #define nodes_xor(dst, src1, src2) \
  165. __nodes_xor(&(dst), &(src1), &(src2), MAX_NUMNODES)
  166. static inline void __nodes_xor(nodemask_t *dstp, const nodemask_t *src1p,
  167. const nodemask_t *src2p, unsigned int nbits)
  168. {
  169. bitmap_xor(dstp->bits, src1p->bits, src2p->bits, nbits);
  170. }
  171. #define nodes_andnot(dst, src1, src2) \
  172. __nodes_andnot(&(dst), &(src1), &(src2), MAX_NUMNODES)
  173. static inline void __nodes_andnot(nodemask_t *dstp, const nodemask_t *src1p,
  174. const nodemask_t *src2p, unsigned int nbits)
  175. {
  176. bitmap_andnot(dstp->bits, src1p->bits, src2p->bits, nbits);
  177. }
  178. #define nodes_complement(dst, src) \
  179. __nodes_complement(&(dst), &(src), MAX_NUMNODES)
  180. static inline void __nodes_complement(nodemask_t *dstp,
  181. const nodemask_t *srcp, unsigned int nbits)
  182. {
  183. bitmap_complement(dstp->bits, srcp->bits, nbits);
  184. }
  185. #define nodes_equal(src1, src2) \
  186. __nodes_equal(&(src1), &(src2), MAX_NUMNODES)
  187. static inline int __nodes_equal(const nodemask_t *src1p,
  188. const nodemask_t *src2p, unsigned int nbits)
  189. {
  190. return bitmap_equal(src1p->bits, src2p->bits, nbits);
  191. }
  192. #define nodes_intersects(src1, src2) \
  193. __nodes_intersects(&(src1), &(src2), MAX_NUMNODES)
  194. static inline int __nodes_intersects(const nodemask_t *src1p,
  195. const nodemask_t *src2p, unsigned int nbits)
  196. {
  197. return bitmap_intersects(src1p->bits, src2p->bits, nbits);
  198. }
  199. #define nodes_subset(src1, src2) \
  200. __nodes_subset(&(src1), &(src2), MAX_NUMNODES)
  201. static inline int __nodes_subset(const nodemask_t *src1p,
  202. const nodemask_t *src2p, unsigned int nbits)
  203. {
  204. return bitmap_subset(src1p->bits, src2p->bits, nbits);
  205. }
  206. #define nodes_empty(src) __nodes_empty(&(src), MAX_NUMNODES)
  207. static inline int __nodes_empty(const nodemask_t *srcp, unsigned int nbits)
  208. {
  209. return bitmap_empty(srcp->bits, nbits);
  210. }
  211. #define nodes_full(nodemask) __nodes_full(&(nodemask), MAX_NUMNODES)
  212. static inline int __nodes_full(const nodemask_t *srcp, unsigned int nbits)
  213. {
  214. return bitmap_full(srcp->bits, nbits);
  215. }
  216. #define nodes_weight(nodemask) __nodes_weight(&(nodemask), MAX_NUMNODES)
  217. static inline int __nodes_weight(const nodemask_t *srcp, unsigned int nbits)
  218. {
  219. return bitmap_weight(srcp->bits, nbits);
  220. }
  221. #define nodes_shift_right(dst, src, n) \
  222. __nodes_shift_right(&(dst), &(src), (n), MAX_NUMNODES)
  223. static inline void __nodes_shift_right(nodemask_t *dstp,
  224. const nodemask_t *srcp, int n, int nbits)
  225. {
  226. bitmap_shift_right(dstp->bits, srcp->bits, n, nbits);
  227. }
  228. #define nodes_shift_left(dst, src, n) \
  229. __nodes_shift_left(&(dst), &(src), (n), MAX_NUMNODES)
  230. static inline void __nodes_shift_left(nodemask_t *dstp,
  231. const nodemask_t *srcp, int n, int nbits)
  232. {
  233. bitmap_shift_left(dstp->bits, srcp->bits, n, nbits);
  234. }
  235. /* FIXME: better would be to fix all architectures to never return
  236. > MAX_NUMNODES, then the silly min_ts could be dropped. */
  237. #define first_node(src) __first_node(&(src))
  238. static inline int __first_node(const nodemask_t *srcp)
  239. {
  240. return min_t(int, MAX_NUMNODES, find_first_bit(srcp->bits, MAX_NUMNODES));
  241. }
  242. #define next_node(n, src) __next_node((n), &(src))
  243. static inline int __next_node(int n, const nodemask_t *srcp)
  244. {
  245. return min_t(int,MAX_NUMNODES,find_next_bit(srcp->bits, MAX_NUMNODES, n+1));
  246. }
  247. /*
  248. * Find the next present node in src, starting after node n, wrapping around to
  249. * the first node in src if needed. Returns MAX_NUMNODES if src is empty.
  250. */
  251. #define next_node_in(n, src) __next_node_in((n), &(src))
  252. int __next_node_in(int node, const nodemask_t *srcp);
  253. static inline void init_nodemask_of_node(nodemask_t *mask, int node)
  254. {
  255. nodes_clear(*mask);
  256. node_set(node, *mask);
  257. }
  258. #define nodemask_of_node(node) \
  259. ({ \
  260. typeof(_unused_nodemask_arg_) m; \
  261. if (sizeof(m) == sizeof(unsigned long)) { \
  262. m.bits[0] = 1UL << (node); \
  263. } else { \
  264. init_nodemask_of_node(&m, (node)); \
  265. } \
  266. m; \
  267. })
  268. #define first_unset_node(mask) __first_unset_node(&(mask))
  269. static inline int __first_unset_node(const nodemask_t *maskp)
  270. {
  271. return min_t(int,MAX_NUMNODES,
  272. find_first_zero_bit(maskp->bits, MAX_NUMNODES));
  273. }
  274. #define NODE_MASK_LAST_WORD BITMAP_LAST_WORD_MASK(MAX_NUMNODES)
  275. #if MAX_NUMNODES <= BITS_PER_LONG
  276. #define NODE_MASK_ALL \
  277. ((nodemask_t) { { \
  278. [BITS_TO_LONGS(MAX_NUMNODES)-1] = NODE_MASK_LAST_WORD \
  279. } })
  280. #else
  281. #define NODE_MASK_ALL \
  282. ((nodemask_t) { { \
  283. [0 ... BITS_TO_LONGS(MAX_NUMNODES)-2] = ~0UL, \
  284. [BITS_TO_LONGS(MAX_NUMNODES)-1] = NODE_MASK_LAST_WORD \
  285. } })
  286. #endif
  287. #define NODE_MASK_NONE \
  288. ((nodemask_t) { { \
  289. [0 ... BITS_TO_LONGS(MAX_NUMNODES)-1] = 0UL \
  290. } })
  291. #define nodes_addr(src) ((src).bits)
  292. #define nodemask_parse_user(ubuf, ulen, dst) \
  293. __nodemask_parse_user((ubuf), (ulen), &(dst), MAX_NUMNODES)
  294. static inline int __nodemask_parse_user(const char __user *buf, int len,
  295. nodemask_t *dstp, int nbits)
  296. {
  297. return bitmap_parse_user(buf, len, dstp->bits, nbits);
  298. }
  299. #define nodelist_parse(buf, dst) __nodelist_parse((buf), &(dst), MAX_NUMNODES)
  300. static inline int __nodelist_parse(const char *buf, nodemask_t *dstp, int nbits)
  301. {
  302. return bitmap_parselist(buf, dstp->bits, nbits);
  303. }
  304. #define node_remap(oldbit, old, new) \
  305. __node_remap((oldbit), &(old), &(new), MAX_NUMNODES)
  306. static inline int __node_remap(int oldbit,
  307. const nodemask_t *oldp, const nodemask_t *newp, int nbits)
  308. {
  309. return bitmap_bitremap(oldbit, oldp->bits, newp->bits, nbits);
  310. }
  311. #define nodes_remap(dst, src, old, new) \
  312. __nodes_remap(&(dst), &(src), &(old), &(new), MAX_NUMNODES)
  313. static inline void __nodes_remap(nodemask_t *dstp, const nodemask_t *srcp,
  314. const nodemask_t *oldp, const nodemask_t *newp, int nbits)
  315. {
  316. bitmap_remap(dstp->bits, srcp->bits, oldp->bits, newp->bits, nbits);
  317. }
  318. #define nodes_onto(dst, orig, relmap) \
  319. __nodes_onto(&(dst), &(orig), &(relmap), MAX_NUMNODES)
  320. static inline void __nodes_onto(nodemask_t *dstp, const nodemask_t *origp,
  321. const nodemask_t *relmapp, int nbits)
  322. {
  323. bitmap_onto(dstp->bits, origp->bits, relmapp->bits, nbits);
  324. }
  325. #define nodes_fold(dst, orig, sz) \
  326. __nodes_fold(&(dst), &(orig), sz, MAX_NUMNODES)
  327. static inline void __nodes_fold(nodemask_t *dstp, const nodemask_t *origp,
  328. int sz, int nbits)
  329. {
  330. bitmap_fold(dstp->bits, origp->bits, sz, nbits);
  331. }
  332. #if MAX_NUMNODES > 1
  333. #define for_each_node_mask(node, mask) \
  334. for ((node) = first_node(mask); \
  335. (node) < MAX_NUMNODES; \
  336. (node) = next_node((node), (mask)))
  337. #else /* MAX_NUMNODES == 1 */
  338. #define for_each_node_mask(node, mask) \
  339. if (!nodes_empty(mask)) \
  340. for ((node) = 0; (node) < 1; (node)++)
  341. #endif /* MAX_NUMNODES */
  342. /*
  343. * Bitmasks that are kept for all the nodes.
  344. */
  345. enum node_states {
  346. N_POSSIBLE, /* The node could become online at some point */
  347. N_ONLINE, /* The node is online */
  348. N_NORMAL_MEMORY, /* The node has regular memory */
  349. #ifdef CONFIG_HIGHMEM
  350. N_HIGH_MEMORY, /* The node has regular or high memory */
  351. #else
  352. N_HIGH_MEMORY = N_NORMAL_MEMORY,
  353. #endif
  354. N_MEMORY, /* The node has memory(regular, high, movable) */
  355. N_CPU, /* The node has one or more cpus */
  356. N_GENERIC_INITIATOR, /* The node has one or more Generic Initiators */
  357. NR_NODE_STATES
  358. };
  359. /*
  360. * The following particular system nodemasks and operations
  361. * on them manage all possible and online nodes.
  362. */
  363. extern nodemask_t node_states[NR_NODE_STATES];
  364. #if MAX_NUMNODES > 1
  365. static inline int node_state(int node, enum node_states state)
  366. {
  367. return node_isset(node, node_states[state]);
  368. }
  369. static inline void node_set_state(int node, enum node_states state)
  370. {
  371. __node_set(node, &node_states[state]);
  372. }
  373. static inline void node_clear_state(int node, enum node_states state)
  374. {
  375. __node_clear(node, &node_states[state]);
  376. }
  377. static inline int num_node_state(enum node_states state)
  378. {
  379. return nodes_weight(node_states[state]);
  380. }
  381. #define for_each_node_state(__node, __state) \
  382. for_each_node_mask((__node), node_states[__state])
  383. #define first_online_node first_node(node_states[N_ONLINE])
  384. #define first_memory_node first_node(node_states[N_MEMORY])
  385. static inline int next_online_node(int nid)
  386. {
  387. return next_node(nid, node_states[N_ONLINE]);
  388. }
  389. static inline int next_memory_node(int nid)
  390. {
  391. return next_node(nid, node_states[N_MEMORY]);
  392. }
  393. extern unsigned int nr_node_ids;
  394. extern unsigned int nr_online_nodes;
  395. static inline void node_set_online(int nid)
  396. {
  397. node_set_state(nid, N_ONLINE);
  398. nr_online_nodes = num_node_state(N_ONLINE);
  399. }
  400. static inline void node_set_offline(int nid)
  401. {
  402. node_clear_state(nid, N_ONLINE);
  403. nr_online_nodes = num_node_state(N_ONLINE);
  404. }
  405. #else
  406. static inline int node_state(int node, enum node_states state)
  407. {
  408. return node == 0;
  409. }
  410. static inline void node_set_state(int node, enum node_states state)
  411. {
  412. }
  413. static inline void node_clear_state(int node, enum node_states state)
  414. {
  415. }
  416. static inline int num_node_state(enum node_states state)
  417. {
  418. return 1;
  419. }
  420. #define for_each_node_state(node, __state) \
  421. for ( (node) = 0; (node) == 0; (node) = 1)
  422. #define first_online_node 0
  423. #define first_memory_node 0
  424. #define next_online_node(nid) (MAX_NUMNODES)
  425. #define nr_node_ids 1U
  426. #define nr_online_nodes 1U
  427. #define node_set_online(node) node_set_state((node), N_ONLINE)
  428. #define node_set_offline(node) node_clear_state((node), N_ONLINE)
  429. #endif
  430. #if defined(CONFIG_NUMA) && (MAX_NUMNODES > 1)
  431. extern int node_random(const nodemask_t *maskp);
  432. #else
  433. static inline int node_random(const nodemask_t *mask)
  434. {
  435. return 0;
  436. }
  437. #endif
  438. #define node_online_map node_states[N_ONLINE]
  439. #define node_possible_map node_states[N_POSSIBLE]
  440. #define num_online_nodes() num_node_state(N_ONLINE)
  441. #define num_possible_nodes() num_node_state(N_POSSIBLE)
  442. #define node_online(node) node_state((node), N_ONLINE)
  443. #define node_possible(node) node_state((node), N_POSSIBLE)
  444. #define for_each_node(node) for_each_node_state(node, N_POSSIBLE)
  445. #define for_each_online_node(node) for_each_node_state(node, N_ONLINE)
  446. /*
  447. * For nodemask scrach area.
  448. * NODEMASK_ALLOC(type, name) allocates an object with a specified type and
  449. * name.
  450. */
  451. #if NODES_SHIFT > 8 /* nodemask_t > 32 bytes */
  452. #define NODEMASK_ALLOC(type, name, gfp_flags) \
  453. type *name = kmalloc(sizeof(*name), gfp_flags)
  454. #define NODEMASK_FREE(m) kfree(m)
  455. #else
  456. #define NODEMASK_ALLOC(type, name, gfp_flags) type _##name, *name = &_##name
  457. #define NODEMASK_FREE(m) do {} while (0)
  458. #endif
  459. /* A example struture for using NODEMASK_ALLOC, used in mempolicy. */
  460. struct nodemask_scratch {
  461. nodemask_t mask1;
  462. nodemask_t mask2;
  463. };
  464. #define NODEMASK_SCRATCH(x) \
  465. NODEMASK_ALLOC(struct nodemask_scratch, x, \
  466. GFP_KERNEL | __GFP_NORETRY)
  467. #define NODEMASK_SCRATCH_FREE(x) NODEMASK_FREE(x)
  468. #endif /* __LINUX_NODEMASK_H */