percpu-internal.h 7.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. #ifndef _MM_PERCPU_INTERNAL_H
  3. #define _MM_PERCPU_INTERNAL_H
  4. #include <linux/types.h>
  5. #include <linux/percpu.h>
  6. /*
  7. * There are two chunk types: root and memcg-aware.
  8. * Chunks of each type have separate slots list.
  9. *
  10. * Memcg-aware chunks have an attached vector of obj_cgroup pointers, which is
  11. * used to store memcg membership data of a percpu object. Obj_cgroups are
  12. * ref-counted pointers to a memory cgroup with an ability to switch dynamically
  13. * to the parent memory cgroup. This allows to reclaim a deleted memory cgroup
  14. * without reclaiming of all outstanding objects, which hold a reference at it.
  15. */
  16. enum pcpu_chunk_type {
  17. PCPU_CHUNK_ROOT,
  18. #ifdef CONFIG_MEMCG_KMEM
  19. PCPU_CHUNK_MEMCG,
  20. #endif
  21. PCPU_NR_CHUNK_TYPES,
  22. PCPU_FAIL_ALLOC = PCPU_NR_CHUNK_TYPES
  23. };
  24. /*
  25. * pcpu_block_md is the metadata block struct.
  26. * Each chunk's bitmap is split into a number of full blocks.
  27. * All units are in terms of bits.
  28. *
  29. * The scan hint is the largest known contiguous area before the contig hint.
  30. * It is not necessarily the actual largest contig hint though. There is an
  31. * invariant that the scan_hint_start > contig_hint_start iff
  32. * scan_hint == contig_hint. This is necessary because when scanning forward,
  33. * we don't know if a new contig hint would be better than the current one.
  34. */
  35. struct pcpu_block_md {
  36. int scan_hint; /* scan hint for block */
  37. int scan_hint_start; /* block relative starting
  38. position of the scan hint */
  39. int contig_hint; /* contig hint for block */
  40. int contig_hint_start; /* block relative starting
  41. position of the contig hint */
  42. int left_free; /* size of free space along
  43. the left side of the block */
  44. int right_free; /* size of free space along
  45. the right side of the block */
  46. int first_free; /* block position of first free */
  47. int nr_bits; /* total bits responsible for */
  48. };
  49. struct pcpu_chunk {
  50. #ifdef CONFIG_PERCPU_STATS
  51. int nr_alloc; /* # of allocations */
  52. size_t max_alloc_size; /* largest allocation size */
  53. #endif
  54. struct list_head list; /* linked to pcpu_slot lists */
  55. int free_bytes; /* free bytes in the chunk */
  56. struct pcpu_block_md chunk_md;
  57. void *base_addr; /* base address of this chunk */
  58. unsigned long *alloc_map; /* allocation map */
  59. unsigned long *bound_map; /* boundary map */
  60. struct pcpu_block_md *md_blocks; /* metadata blocks */
  61. void *data; /* chunk data */
  62. bool immutable; /* no [de]population allowed */
  63. int start_offset; /* the overlap with the previous
  64. region to have a page aligned
  65. base_addr */
  66. int end_offset; /* additional area required to
  67. have the region end page
  68. aligned */
  69. #ifdef CONFIG_MEMCG_KMEM
  70. struct obj_cgroup **obj_cgroups; /* vector of object cgroups */
  71. #endif
  72. int nr_pages; /* # of pages served by this chunk */
  73. int nr_populated; /* # of populated pages */
  74. int nr_empty_pop_pages; /* # of empty populated pages */
  75. unsigned long populated[]; /* populated bitmap */
  76. };
  77. extern spinlock_t pcpu_lock;
  78. extern struct list_head *pcpu_chunk_lists;
  79. extern int pcpu_nr_slots;
  80. extern int pcpu_nr_empty_pop_pages[];
  81. extern struct pcpu_chunk *pcpu_first_chunk;
  82. extern struct pcpu_chunk *pcpu_reserved_chunk;
  83. /**
  84. * pcpu_chunk_nr_blocks - converts nr_pages to # of md_blocks
  85. * @chunk: chunk of interest
  86. *
  87. * This conversion is from the number of physical pages that the chunk
  88. * serves to the number of bitmap blocks used.
  89. */
  90. static inline int pcpu_chunk_nr_blocks(struct pcpu_chunk *chunk)
  91. {
  92. return chunk->nr_pages * PAGE_SIZE / PCPU_BITMAP_BLOCK_SIZE;
  93. }
  94. /**
  95. * pcpu_nr_pages_to_map_bits - converts the pages to size of bitmap
  96. * @pages: number of physical pages
  97. *
  98. * This conversion is from physical pages to the number of bits
  99. * required in the bitmap.
  100. */
  101. static inline int pcpu_nr_pages_to_map_bits(int pages)
  102. {
  103. return pages * PAGE_SIZE / PCPU_MIN_ALLOC_SIZE;
  104. }
  105. /**
  106. * pcpu_chunk_map_bits - helper to convert nr_pages to size of bitmap
  107. * @chunk: chunk of interest
  108. *
  109. * This conversion is from the number of physical pages that the chunk
  110. * serves to the number of bits in the bitmap.
  111. */
  112. static inline int pcpu_chunk_map_bits(struct pcpu_chunk *chunk)
  113. {
  114. return pcpu_nr_pages_to_map_bits(chunk->nr_pages);
  115. }
  116. #ifdef CONFIG_MEMCG_KMEM
  117. static inline enum pcpu_chunk_type pcpu_chunk_type(struct pcpu_chunk *chunk)
  118. {
  119. if (chunk->obj_cgroups)
  120. return PCPU_CHUNK_MEMCG;
  121. return PCPU_CHUNK_ROOT;
  122. }
  123. static inline bool pcpu_is_memcg_chunk(enum pcpu_chunk_type chunk_type)
  124. {
  125. return chunk_type == PCPU_CHUNK_MEMCG;
  126. }
  127. #else
  128. static inline enum pcpu_chunk_type pcpu_chunk_type(struct pcpu_chunk *chunk)
  129. {
  130. return PCPU_CHUNK_ROOT;
  131. }
  132. static inline bool pcpu_is_memcg_chunk(enum pcpu_chunk_type chunk_type)
  133. {
  134. return false;
  135. }
  136. #endif
  137. static inline struct list_head *pcpu_chunk_list(enum pcpu_chunk_type chunk_type)
  138. {
  139. return &pcpu_chunk_lists[pcpu_nr_slots *
  140. pcpu_is_memcg_chunk(chunk_type)];
  141. }
  142. #ifdef CONFIG_PERCPU_STATS
  143. #include <linux/spinlock.h>
  144. struct percpu_stats {
  145. u64 nr_alloc; /* lifetime # of allocations */
  146. u64 nr_dealloc; /* lifetime # of deallocations */
  147. u64 nr_cur_alloc; /* current # of allocations */
  148. u64 nr_max_alloc; /* max # of live allocations */
  149. u32 nr_chunks; /* current # of live chunks */
  150. u32 nr_max_chunks; /* max # of live chunks */
  151. size_t min_alloc_size; /* min allocaiton size */
  152. size_t max_alloc_size; /* max allocation size */
  153. };
  154. extern struct percpu_stats pcpu_stats;
  155. extern struct pcpu_alloc_info pcpu_stats_ai;
  156. /*
  157. * For debug purposes. We don't care about the flexible array.
  158. */
  159. static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai)
  160. {
  161. memcpy(&pcpu_stats_ai, ai, sizeof(struct pcpu_alloc_info));
  162. /* initialize min_alloc_size to unit_size */
  163. pcpu_stats.min_alloc_size = pcpu_stats_ai.unit_size;
  164. }
  165. /*
  166. * pcpu_stats_area_alloc - increment area allocation stats
  167. * @chunk: the location of the area being allocated
  168. * @size: size of area to allocate in bytes
  169. *
  170. * CONTEXT:
  171. * pcpu_lock.
  172. */
  173. static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size)
  174. {
  175. lockdep_assert_held(&pcpu_lock);
  176. pcpu_stats.nr_alloc++;
  177. pcpu_stats.nr_cur_alloc++;
  178. pcpu_stats.nr_max_alloc =
  179. max(pcpu_stats.nr_max_alloc, pcpu_stats.nr_cur_alloc);
  180. pcpu_stats.min_alloc_size =
  181. min(pcpu_stats.min_alloc_size, size);
  182. pcpu_stats.max_alloc_size =
  183. max(pcpu_stats.max_alloc_size, size);
  184. chunk->nr_alloc++;
  185. chunk->max_alloc_size = max(chunk->max_alloc_size, size);
  186. }
  187. /*
  188. * pcpu_stats_area_dealloc - decrement allocation stats
  189. * @chunk: the location of the area being deallocated
  190. *
  191. * CONTEXT:
  192. * pcpu_lock.
  193. */
  194. static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk)
  195. {
  196. lockdep_assert_held(&pcpu_lock);
  197. pcpu_stats.nr_dealloc++;
  198. pcpu_stats.nr_cur_alloc--;
  199. chunk->nr_alloc--;
  200. }
  201. /*
  202. * pcpu_stats_chunk_alloc - increment chunk stats
  203. */
  204. static inline void pcpu_stats_chunk_alloc(void)
  205. {
  206. unsigned long flags;
  207. spin_lock_irqsave(&pcpu_lock, flags);
  208. pcpu_stats.nr_chunks++;
  209. pcpu_stats.nr_max_chunks =
  210. max(pcpu_stats.nr_max_chunks, pcpu_stats.nr_chunks);
  211. spin_unlock_irqrestore(&pcpu_lock, flags);
  212. }
  213. /*
  214. * pcpu_stats_chunk_dealloc - decrement chunk stats
  215. */
  216. static inline void pcpu_stats_chunk_dealloc(void)
  217. {
  218. unsigned long flags;
  219. spin_lock_irqsave(&pcpu_lock, flags);
  220. pcpu_stats.nr_chunks--;
  221. spin_unlock_irqrestore(&pcpu_lock, flags);
  222. }
  223. #else
  224. static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai)
  225. {
  226. }
  227. static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size)
  228. {
  229. }
  230. static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk)
  231. {
  232. }
  233. static inline void pcpu_stats_chunk_alloc(void)
  234. {
  235. }
  236. static inline void pcpu_stats_chunk_dealloc(void)
  237. {
  238. }
  239. #endif /* !CONFIG_PERCPU_STATS */
  240. #endif