memory_hotplug.h 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. #ifndef __LINUX_MEMORY_HOTPLUG_H
  3. #define __LINUX_MEMORY_HOTPLUG_H
  4. #include <linux/mmzone.h>
  5. #include <linux/spinlock.h>
  6. #include <linux/notifier.h>
  7. #include <linux/bug.h>
  8. struct page;
  9. struct zone;
  10. struct pglist_data;
  11. struct mem_section;
  12. struct memory_block;
  13. struct resource;
  14. struct vmem_altmap;
  15. #ifdef CONFIG_MEMORY_HOTPLUG
  16. /*
  17. * Return page for the valid pfn only if the page is online. All pfn
  18. * walkers which rely on the fully initialized page->flags and others
  19. * should use this rather than pfn_valid && pfn_to_page
  20. */
  21. #define pfn_to_online_page(pfn) \
  22. ({ \
  23. struct page *___page = NULL; \
  24. unsigned long ___pfn = pfn; \
  25. unsigned long ___nr = pfn_to_section_nr(___pfn); \
  26. \
  27. if (___nr < NR_MEM_SECTIONS && online_section_nr(___nr) && \
  28. pfn_valid_within(___pfn)) \
  29. ___page = pfn_to_page(___pfn); \
  30. ___page; \
  31. })
  32. /*
  33. * Types for free bootmem stored in page->lru.next. These have to be in
  34. * some random range in unsigned long space for debugging purposes.
  35. */
  36. enum {
  37. MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE = 12,
  38. SECTION_INFO = MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE,
  39. MIX_SECTION_INFO,
  40. NODE_INFO,
  41. MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE = NODE_INFO,
  42. };
  43. /* Types for control the zone type of onlined and offlined memory */
  44. enum {
  45. /* Offline the memory. */
  46. MMOP_OFFLINE = 0,
  47. /* Online the memory. Zone depends, see default_zone_for_pfn(). */
  48. MMOP_ONLINE,
  49. /* Online the memory to ZONE_NORMAL. */
  50. MMOP_ONLINE_KERNEL,
  51. /* Online the memory to ZONE_MOVABLE. */
  52. MMOP_ONLINE_MOVABLE,
  53. };
  54. /* Flags for add_memory() and friends to specify memory hotplug details. */
  55. typedef int __bitwise mhp_t;
  56. /* No special request */
  57. #define MHP_NONE ((__force mhp_t)0)
  58. /*
  59. * Allow merging of the added System RAM resource with adjacent,
  60. * mergeable resources. After a successful call to add_memory_resource()
  61. * with this flag set, the resource pointer must no longer be used as it
  62. * might be stale, or the resource might have changed.
  63. */
  64. #define MEMHP_MERGE_RESOURCE ((__force mhp_t)BIT(0))
  65. /*
  66. * Extended parameters for memory hotplug:
  67. * altmap: alternative allocator for memmap array (optional)
  68. * pgprot: page protection flags to apply to newly created page tables
  69. * (required)
  70. */
  71. struct mhp_params {
  72. struct vmem_altmap *altmap;
  73. pgprot_t pgprot;
  74. };
  75. /*
  76. * Zone resizing functions
  77. *
  78. * Note: any attempt to resize a zone should has pgdat_resize_lock()
  79. * zone_span_writelock() both held. This ensure the size of a zone
  80. * can't be changed while pgdat_resize_lock() held.
  81. */
  82. static inline unsigned zone_span_seqbegin(struct zone *zone)
  83. {
  84. return read_seqbegin(&zone->span_seqlock);
  85. }
  86. static inline int zone_span_seqretry(struct zone *zone, unsigned iv)
  87. {
  88. return read_seqretry(&zone->span_seqlock, iv);
  89. }
  90. static inline void zone_span_writelock(struct zone *zone)
  91. {
  92. write_seqlock(&zone->span_seqlock);
  93. }
  94. static inline void zone_span_writeunlock(struct zone *zone)
  95. {
  96. write_sequnlock(&zone->span_seqlock);
  97. }
  98. static inline void zone_seqlock_init(struct zone *zone)
  99. {
  100. seqlock_init(&zone->span_seqlock);
  101. }
  102. extern int zone_grow_free_lists(struct zone *zone, unsigned long new_nr_pages);
  103. extern int zone_grow_waitqueues(struct zone *zone, unsigned long nr_pages);
  104. extern int add_one_highpage(struct page *page, int pfn, int bad_ppro);
  105. /* VM interface that may be used by firmware interface */
  106. extern int online_pages(unsigned long pfn, unsigned long nr_pages,
  107. int online_type, int nid);
  108. extern struct zone *test_pages_in_a_zone(unsigned long start_pfn,
  109. unsigned long end_pfn);
  110. extern void __offline_isolated_pages(unsigned long start_pfn,
  111. unsigned long end_pfn);
  112. typedef void (*online_page_callback_t)(struct page *page, unsigned int order);
  113. extern void generic_online_page(struct page *page, unsigned int order);
  114. extern int set_online_page_callback(online_page_callback_t callback);
  115. extern int restore_online_page_callback(online_page_callback_t callback);
  116. extern int try_online_node(int nid);
  117. extern int arch_add_memory(int nid, u64 start, u64 size,
  118. struct mhp_params *params);
  119. extern u64 max_mem_size;
  120. extern int memhp_online_type_from_str(const char *str);
  121. /* Default online_type (MMOP_*) when new memory blocks are added. */
  122. extern int memhp_default_online_type;
  123. /* If movable_node boot option specified */
  124. extern bool movable_node_enabled;
  125. static inline bool movable_node_is_enabled(void)
  126. {
  127. return movable_node_enabled;
  128. }
  129. extern void arch_remove_memory(int nid, u64 start, u64 size,
  130. struct vmem_altmap *altmap);
  131. extern void __remove_pages(unsigned long start_pfn, unsigned long nr_pages,
  132. struct vmem_altmap *altmap);
  133. /* reasonably generic interface to expand the physical pages */
  134. extern int __add_pages(int nid, unsigned long start_pfn, unsigned long nr_pages,
  135. struct mhp_params *params);
  136. #ifndef CONFIG_ARCH_HAS_ADD_PAGES
  137. static inline int add_pages(int nid, unsigned long start_pfn,
  138. unsigned long nr_pages, struct mhp_params *params)
  139. {
  140. return __add_pages(nid, start_pfn, nr_pages, params);
  141. }
  142. #else /* ARCH_HAS_ADD_PAGES */
  143. int add_pages(int nid, unsigned long start_pfn, unsigned long nr_pages,
  144. struct mhp_params *params);
  145. #endif /* ARCH_HAS_ADD_PAGES */
  146. #ifdef CONFIG_HAVE_ARCH_NODEDATA_EXTENSION
  147. /*
  148. * For supporting node-hotadd, we have to allocate a new pgdat.
  149. *
  150. * If an arch has generic style NODE_DATA(),
  151. * node_data[nid] = kzalloc() works well. But it depends on the architecture.
  152. *
  153. * In general, generic_alloc_nodedata() is used.
  154. * Now, arch_free_nodedata() is just defined for error path of node_hot_add.
  155. *
  156. */
  157. extern pg_data_t *arch_alloc_nodedata(int nid);
  158. extern void arch_free_nodedata(pg_data_t *pgdat);
  159. extern void arch_refresh_nodedata(int nid, pg_data_t *pgdat);
  160. #else /* CONFIG_HAVE_ARCH_NODEDATA_EXTENSION */
  161. #define arch_alloc_nodedata(nid) generic_alloc_nodedata(nid)
  162. #define arch_free_nodedata(pgdat) generic_free_nodedata(pgdat)
  163. #ifdef CONFIG_NUMA
  164. /*
  165. * If ARCH_HAS_NODEDATA_EXTENSION=n, this func is used to allocate pgdat.
  166. * XXX: kmalloc_node() can't work well to get new node's memory at this time.
  167. * Because, pgdat for the new node is not allocated/initialized yet itself.
  168. * To use new node's memory, more consideration will be necessary.
  169. */
  170. #define generic_alloc_nodedata(nid) \
  171. ({ \
  172. kzalloc(sizeof(pg_data_t), GFP_KERNEL); \
  173. })
  174. /*
  175. * This definition is just for error path in node hotadd.
  176. * For node hotremove, we have to replace this.
  177. */
  178. #define generic_free_nodedata(pgdat) kfree(pgdat)
  179. extern pg_data_t *node_data[];
  180. static inline void arch_refresh_nodedata(int nid, pg_data_t *pgdat)
  181. {
  182. node_data[nid] = pgdat;
  183. }
  184. #else /* !CONFIG_NUMA */
  185. /* never called */
  186. static inline pg_data_t *generic_alloc_nodedata(int nid)
  187. {
  188. BUG();
  189. return NULL;
  190. }
  191. static inline void generic_free_nodedata(pg_data_t *pgdat)
  192. {
  193. }
  194. static inline void arch_refresh_nodedata(int nid, pg_data_t *pgdat)
  195. {
  196. }
  197. #endif /* CONFIG_NUMA */
  198. #endif /* CONFIG_HAVE_ARCH_NODEDATA_EXTENSION */
  199. #ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE
  200. extern void __init register_page_bootmem_info_node(struct pglist_data *pgdat);
  201. #else
  202. static inline void register_page_bootmem_info_node(struct pglist_data *pgdat)
  203. {
  204. }
  205. #endif
  206. extern void put_page_bootmem(struct page *page);
  207. extern void get_page_bootmem(unsigned long ingo, struct page *page,
  208. unsigned long type);
  209. void get_online_mems(void);
  210. void put_online_mems(void);
  211. void mem_hotplug_begin(void);
  212. void mem_hotplug_done(void);
  213. #else /* ! CONFIG_MEMORY_HOTPLUG */
  214. #define pfn_to_online_page(pfn) \
  215. ({ \
  216. struct page *___page = NULL; \
  217. if (pfn_valid(pfn)) \
  218. ___page = pfn_to_page(pfn); \
  219. ___page; \
  220. })
  221. static inline unsigned zone_span_seqbegin(struct zone *zone)
  222. {
  223. return 0;
  224. }
  225. static inline int zone_span_seqretry(struct zone *zone, unsigned iv)
  226. {
  227. return 0;
  228. }
  229. static inline void zone_span_writelock(struct zone *zone) {}
  230. static inline void zone_span_writeunlock(struct zone *zone) {}
  231. static inline void zone_seqlock_init(struct zone *zone) {}
  232. static inline void register_page_bootmem_info_node(struct pglist_data *pgdat)
  233. {
  234. }
  235. static inline int try_online_node(int nid)
  236. {
  237. return 0;
  238. }
  239. static inline void get_online_mems(void) {}
  240. static inline void put_online_mems(void) {}
  241. static inline void mem_hotplug_begin(void) {}
  242. static inline void mem_hotplug_done(void) {}
  243. static inline bool movable_node_is_enabled(void)
  244. {
  245. return false;
  246. }
  247. #endif /* ! CONFIG_MEMORY_HOTPLUG */
  248. #if defined(CONFIG_MEMORY_HOTPLUG) || defined(CONFIG_DEFERRED_STRUCT_PAGE_INIT)
  249. /*
  250. * pgdat resizing functions
  251. */
  252. static inline
  253. void pgdat_resize_lock(struct pglist_data *pgdat, unsigned long *flags)
  254. {
  255. spin_lock_irqsave(&pgdat->node_size_lock, *flags);
  256. }
  257. static inline
  258. void pgdat_resize_unlock(struct pglist_data *pgdat, unsigned long *flags)
  259. {
  260. spin_unlock_irqrestore(&pgdat->node_size_lock, *flags);
  261. }
  262. static inline
  263. void pgdat_resize_init(struct pglist_data *pgdat)
  264. {
  265. spin_lock_init(&pgdat->node_size_lock);
  266. }
  267. #else /* !(CONFIG_MEMORY_HOTPLUG || CONFIG_DEFERRED_STRUCT_PAGE_INIT) */
  268. /*
  269. * Stub functions for when hotplug is off
  270. */
  271. static inline void pgdat_resize_lock(struct pglist_data *p, unsigned long *f) {}
  272. static inline void pgdat_resize_unlock(struct pglist_data *p, unsigned long *f) {}
  273. static inline void pgdat_resize_init(struct pglist_data *pgdat) {}
  274. #endif /* !(CONFIG_MEMORY_HOTPLUG || CONFIG_DEFERRED_STRUCT_PAGE_INIT) */
  275. #ifdef CONFIG_MEMORY_HOTREMOVE
  276. extern void try_offline_node(int nid);
  277. extern int offline_pages(unsigned long start_pfn, unsigned long nr_pages);
  278. extern int remove_memory(int nid, u64 start, u64 size);
  279. extern int remove_memory_subsection(int nid, u64 start, u64 size);
  280. extern void __remove_memory(int nid, u64 start, u64 size);
  281. extern int offline_and_remove_memory(int nid, u64 start, u64 size);
  282. #else
  283. static inline void try_offline_node(int nid) {}
  284. static inline int offline_pages(unsigned long start_pfn, unsigned long nr_pages)
  285. {
  286. return -EINVAL;
  287. }
  288. static inline int remove_memory(int nid, u64 start, u64 size)
  289. {
  290. return -EBUSY;
  291. }
  292. static inline void __remove_memory(int nid, u64 start, u64 size) {}
  293. #endif /* CONFIG_MEMORY_HOTREMOVE */
  294. extern void set_zone_contiguous(struct zone *zone);
  295. extern void clear_zone_contiguous(struct zone *zone);
  296. #ifdef CONFIG_MEMORY_HOTPLUG
  297. extern void __ref free_area_init_core_hotplug(int nid);
  298. extern int __add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags);
  299. extern int add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags);
  300. extern int add_memory_subsection(int nid, u64 start, u64 size);
  301. extern int add_memory_resource(int nid, struct resource *resource,
  302. mhp_t mhp_flags);
  303. extern int add_memory_driver_managed(int nid, u64 start, u64 size,
  304. const char *resource_name,
  305. mhp_t mhp_flags);
  306. extern void move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn,
  307. unsigned long nr_pages,
  308. struct vmem_altmap *altmap, int migratetype);
  309. extern void remove_pfn_range_from_zone(struct zone *zone,
  310. unsigned long start_pfn,
  311. unsigned long nr_pages);
  312. extern bool is_memblock_offlined(struct memory_block *mem);
  313. extern int sparse_add_section(int nid, unsigned long pfn,
  314. unsigned long nr_pages, struct vmem_altmap *altmap);
  315. extern void sparse_remove_section(struct mem_section *ms,
  316. unsigned long pfn, unsigned long nr_pages,
  317. unsigned long map_offset, struct vmem_altmap *altmap);
  318. extern struct page *sparse_decode_mem_map(unsigned long coded_mem_map,
  319. unsigned long pnum);
  320. extern struct zone *zone_for_pfn_range(int online_type, int nid,
  321. unsigned long start_pfn, unsigned long nr_pages);
  322. #endif /* CONFIG_MEMORY_HOTPLUG */
  323. #endif /* __LINUX_MEMORY_HOTPLUG_H */