compaction.h 7.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. #ifndef _LINUX_COMPACTION_H
  3. #define _LINUX_COMPACTION_H
  4. /*
  5. * Determines how hard direct compaction should try to succeed.
  6. * Lower value means higher priority, analogically to reclaim priority.
  7. */
  8. enum compact_priority {
  9. COMPACT_PRIO_SYNC_FULL,
  10. MIN_COMPACT_PRIORITY = COMPACT_PRIO_SYNC_FULL,
  11. COMPACT_PRIO_SYNC_LIGHT,
  12. MIN_COMPACT_COSTLY_PRIORITY = COMPACT_PRIO_SYNC_LIGHT,
  13. DEF_COMPACT_PRIORITY = COMPACT_PRIO_SYNC_LIGHT,
  14. COMPACT_PRIO_ASYNC,
  15. INIT_COMPACT_PRIORITY = COMPACT_PRIO_ASYNC
  16. };
  17. /* Return values for compact_zone() and try_to_compact_pages() */
  18. /* When adding new states, please adjust include/trace/events/compaction.h */
  19. enum compact_result {
  20. /* For more detailed tracepoint output - internal to compaction */
  21. COMPACT_NOT_SUITABLE_ZONE,
  22. /*
  23. * compaction didn't start as it was not possible or direct reclaim
  24. * was more suitable
  25. */
  26. COMPACT_SKIPPED,
  27. /* compaction didn't start as it was deferred due to past failures */
  28. COMPACT_DEFERRED,
  29. /* For more detailed tracepoint output - internal to compaction */
  30. COMPACT_NO_SUITABLE_PAGE,
  31. /* compaction should continue to another pageblock */
  32. COMPACT_CONTINUE,
  33. /*
  34. * The full zone was compacted scanned but wasn't successfull to compact
  35. * suitable pages.
  36. */
  37. COMPACT_COMPLETE,
  38. /*
  39. * direct compaction has scanned part of the zone but wasn't successfull
  40. * to compact suitable pages.
  41. */
  42. COMPACT_PARTIAL_SKIPPED,
  43. /* compaction terminated prematurely due to lock contentions */
  44. COMPACT_CONTENDED,
  45. /*
  46. * direct compaction terminated after concluding that the allocation
  47. * should now succeed
  48. */
  49. COMPACT_SUCCESS,
  50. };
  51. struct alloc_context; /* in mm/internal.h */
  52. /*
  53. * Number of free order-0 pages that should be available above given watermark
  54. * to make sure compaction has reasonable chance of not running out of free
  55. * pages that it needs to isolate as migration target during its work.
  56. */
  57. static inline unsigned long compact_gap(unsigned int order)
  58. {
  59. /*
  60. * Although all the isolations for migration are temporary, compaction
  61. * free scanner may have up to 1 << order pages on its list and then
  62. * try to split an (order - 1) free page. At that point, a gap of
  63. * 1 << order might not be enough, so it's safer to require twice that
  64. * amount. Note that the number of pages on the list is also
  65. * effectively limited by COMPACT_CLUSTER_MAX, as that's the maximum
  66. * that the migrate scanner can have isolated on migrate list, and free
  67. * scanner is only invoked when the number of isolated free pages is
  68. * lower than that. But it's not worth to complicate the formula here
  69. * as a bigger gap for higher orders than strictly necessary can also
  70. * improve chances of compaction success.
  71. */
  72. return 2UL << order;
  73. }
  74. #ifdef CONFIG_COMPACTION
  75. extern int sysctl_compact_memory;
  76. extern unsigned int sysctl_compaction_proactiveness;
  77. extern int sysctl_compaction_handler(struct ctl_table *table, int write,
  78. void *buffer, size_t *length, loff_t *ppos);
  79. extern int compaction_proactiveness_sysctl_handler(struct ctl_table *table,
  80. int write, void *buffer, size_t *length, loff_t *ppos);
  81. extern int sysctl_extfrag_threshold;
  82. extern int sysctl_compact_unevictable_allowed;
  83. extern unsigned int extfrag_for_order(struct zone *zone, unsigned int order);
  84. extern int fragmentation_index(struct zone *zone, unsigned int order);
  85. extern enum compact_result try_to_compact_pages(gfp_t gfp_mask,
  86. unsigned int order, unsigned int alloc_flags,
  87. const struct alloc_context *ac, enum compact_priority prio,
  88. struct page **page);
  89. extern void reset_isolation_suitable(pg_data_t *pgdat);
  90. extern enum compact_result compaction_suitable(struct zone *zone, int order,
  91. unsigned int alloc_flags, int highest_zoneidx);
  92. extern void defer_compaction(struct zone *zone, int order);
  93. extern bool compaction_deferred(struct zone *zone, int order);
  94. extern void compaction_defer_reset(struct zone *zone, int order,
  95. bool alloc_success);
  96. extern bool compaction_restarting(struct zone *zone, int order);
  97. /* Compaction has made some progress and retrying makes sense */
  98. static inline bool compaction_made_progress(enum compact_result result)
  99. {
  100. /*
  101. * Even though this might sound confusing this in fact tells us
  102. * that the compaction successfully isolated and migrated some
  103. * pageblocks.
  104. */
  105. if (result == COMPACT_SUCCESS)
  106. return true;
  107. return false;
  108. }
  109. /* Compaction has failed and it doesn't make much sense to keep retrying. */
  110. static inline bool compaction_failed(enum compact_result result)
  111. {
  112. /* All zones were scanned completely and still not result. */
  113. if (result == COMPACT_COMPLETE)
  114. return true;
  115. return false;
  116. }
  117. /* Compaction needs reclaim to be performed first, so it can continue. */
  118. static inline bool compaction_needs_reclaim(enum compact_result result)
  119. {
  120. /*
  121. * Compaction backed off due to watermark checks for order-0
  122. * so the regular reclaim has to try harder and reclaim something.
  123. */
  124. if (result == COMPACT_SKIPPED)
  125. return true;
  126. return false;
  127. }
  128. /*
  129. * Compaction has backed off for some reason after doing some work or none
  130. * at all. It might be throttling or lock contention. Retrying might be still
  131. * worthwhile, but with a higher priority if allowed.
  132. */
  133. static inline bool compaction_withdrawn(enum compact_result result)
  134. {
  135. /*
  136. * If compaction is deferred for high-order allocations, it is
  137. * because sync compaction recently failed. If this is the case
  138. * and the caller requested a THP allocation, we do not want
  139. * to heavily disrupt the system, so we fail the allocation
  140. * instead of entering direct reclaim.
  141. */
  142. if (result == COMPACT_DEFERRED)
  143. return true;
  144. /*
  145. * If compaction in async mode encounters contention or blocks higher
  146. * priority task we back off early rather than cause stalls.
  147. */
  148. if (result == COMPACT_CONTENDED)
  149. return true;
  150. /*
  151. * Page scanners have met but we haven't scanned full zones so this
  152. * is a back off in fact.
  153. */
  154. if (result == COMPACT_PARTIAL_SKIPPED)
  155. return true;
  156. return false;
  157. }
  158. bool compaction_zonelist_suitable(struct alloc_context *ac, int order,
  159. int alloc_flags);
  160. extern int kcompactd_run(int nid);
  161. extern void kcompactd_stop(int nid);
  162. extern void wakeup_kcompactd(pg_data_t *pgdat, int order, int highest_zoneidx);
  163. extern unsigned long isolate_and_split_free_page(struct page *page,
  164. struct list_head *list);
  165. #else
  166. static inline void reset_isolation_suitable(pg_data_t *pgdat)
  167. {
  168. }
  169. static inline enum compact_result compaction_suitable(struct zone *zone, int order,
  170. int alloc_flags, int highest_zoneidx)
  171. {
  172. return COMPACT_SKIPPED;
  173. }
  174. static inline void defer_compaction(struct zone *zone, int order)
  175. {
  176. }
  177. static inline bool compaction_deferred(struct zone *zone, int order)
  178. {
  179. return true;
  180. }
  181. static inline bool compaction_made_progress(enum compact_result result)
  182. {
  183. return false;
  184. }
  185. static inline bool compaction_failed(enum compact_result result)
  186. {
  187. return false;
  188. }
  189. static inline bool compaction_needs_reclaim(enum compact_result result)
  190. {
  191. return false;
  192. }
  193. static inline bool compaction_withdrawn(enum compact_result result)
  194. {
  195. return true;
  196. }
  197. static inline int kcompactd_run(int nid)
  198. {
  199. return 0;
  200. }
  201. static inline void kcompactd_stop(int nid)
  202. {
  203. }
  204. static inline void wakeup_kcompactd(pg_data_t *pgdat,
  205. int order, int highest_zoneidx)
  206. {
  207. }
  208. static inline unsigned long isolate_and_split_free_page(struct page *page,
  209. struct list_head *list)
  210. {
  211. return 0;
  212. }
  213. #endif /* CONFIG_COMPACTION */
  214. struct node;
  215. #if defined(CONFIG_COMPACTION) && defined(CONFIG_SYSFS) && defined(CONFIG_NUMA)
  216. extern int compaction_register_node(struct node *node);
  217. extern void compaction_unregister_node(struct node *node);
  218. #else
  219. static inline int compaction_register_node(struct node *node)
  220. {
  221. return 0;
  222. }
  223. static inline void compaction_unregister_node(struct node *node)
  224. {
  225. }
  226. #endif /* CONFIG_COMPACTION && CONFIG_SYSFS && CONFIG_NUMA */
  227. #endif /* _LINUX_COMPACTION_H */