memblock.c 57 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * Procedures for maintaining information about logical memory blocks.
  4. *
  5. * Peter Bergner, IBM Corp. June 2001.
  6. * Copyright (C) 2001 Peter Bergner.
  7. */
  8. #include <linux/kernel.h>
  9. #include <linux/slab.h>
  10. #include <linux/init.h>
  11. #include <linux/bitops.h>
  12. #include <linux/poison.h>
  13. #include <linux/pfn.h>
  14. #include <linux/debugfs.h>
  15. #include <linux/kmemleak.h>
  16. #include <linux/seq_file.h>
  17. #include <linux/memblock.h>
  18. #include <asm/sections.h>
  19. #include <linux/io.h>
  20. #include "internal.h"
  21. #define INIT_MEMBLOCK_REGIONS 128
  22. #define INIT_PHYSMEM_REGIONS 4
  23. #ifndef INIT_MEMBLOCK_RESERVED_REGIONS
  24. # define INIT_MEMBLOCK_RESERVED_REGIONS INIT_MEMBLOCK_REGIONS
  25. #endif
  26. /**
  27. * DOC: memblock overview
  28. *
  29. * Memblock is a method of managing memory regions during the early
  30. * boot period when the usual kernel memory allocators are not up and
  31. * running.
  32. *
  33. * Memblock views the system memory as collections of contiguous
  34. * regions. There are several types of these collections:
  35. *
  36. * * ``memory`` - describes the physical memory available to the
  37. * kernel; this may differ from the actual physical memory installed
  38. * in the system, for instance when the memory is restricted with
  39. * ``mem=`` command line parameter
  40. * * ``reserved`` - describes the regions that were allocated
  41. * * ``physmem`` - describes the actual physical memory available during
  42. * boot regardless of the possible restrictions and memory hot(un)plug;
  43. * the ``physmem`` type is only available on some architectures.
  44. *
  45. * Each region is represented by struct memblock_region that
  46. * defines the region extents, its attributes and NUMA node id on NUMA
  47. * systems. Every memory type is described by the struct memblock_type
  48. * which contains an array of memory regions along with
  49. * the allocator metadata. The "memory" and "reserved" types are nicely
  50. * wrapped with struct memblock. This structure is statically
  51. * initialized at build time. The region arrays are initially sized to
  52. * %INIT_MEMBLOCK_REGIONS for "memory" and %INIT_MEMBLOCK_RESERVED_REGIONS
  53. * for "reserved". The region array for "physmem" is initially sized to
  54. * %INIT_PHYSMEM_REGIONS.
  55. * The memblock_allow_resize() enables automatic resizing of the region
  56. * arrays during addition of new regions. This feature should be used
  57. * with care so that memory allocated for the region array will not
  58. * overlap with areas that should be reserved, for example initrd.
  59. *
  60. * The early architecture setup should tell memblock what the physical
  61. * memory layout is by using memblock_add() or memblock_add_node()
  62. * functions. The first function does not assign the region to a NUMA
  63. * node and it is appropriate for UMA systems. Yet, it is possible to
  64. * use it on NUMA systems as well and assign the region to a NUMA node
  65. * later in the setup process using memblock_set_node(). The
  66. * memblock_add_node() performs such an assignment directly.
  67. *
  68. * Once memblock is setup the memory can be allocated using one of the
  69. * API variants:
  70. *
  71. * * memblock_phys_alloc*() - these functions return the **physical**
  72. * address of the allocated memory
  73. * * memblock_alloc*() - these functions return the **virtual** address
  74. * of the allocated memory.
  75. *
  76. * Note, that both API variants use implicit assumptions about allowed
  77. * memory ranges and the fallback methods. Consult the documentation
  78. * of memblock_alloc_internal() and memblock_alloc_range_nid()
  79. * functions for more elaborate description.
  80. *
  81. * As the system boot progresses, the architecture specific mem_init()
  82. * function frees all the memory to the buddy page allocator.
  83. *
  84. * Unless an architecture enables %CONFIG_ARCH_KEEP_MEMBLOCK, the
  85. * memblock data structures (except "physmem") will be discarded after the
  86. * system initialization completes.
  87. */
  88. #ifndef CONFIG_NEED_MULTIPLE_NODES
  89. struct pglist_data __refdata contig_page_data;
  90. EXPORT_SYMBOL(contig_page_data);
  91. #endif
  92. unsigned long max_low_pfn;
  93. unsigned long min_low_pfn;
  94. unsigned long max_pfn;
  95. unsigned long long max_possible_pfn;
  96. static struct memblock_region memblock_memory_init_regions[INIT_MEMBLOCK_REGIONS] __initdata_memblock;
  97. static struct memblock_region memblock_reserved_init_regions[INIT_MEMBLOCK_RESERVED_REGIONS] __initdata_memblock;
  98. #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
  99. static struct memblock_region memblock_physmem_init_regions[INIT_PHYSMEM_REGIONS];
  100. #endif
  101. struct memblock memblock __initdata_memblock = {
  102. .memory.regions = memblock_memory_init_regions,
  103. .memory.cnt = 1, /* empty dummy entry */
  104. .memory.max = INIT_MEMBLOCK_REGIONS,
  105. .memory.name = "memory",
  106. .reserved.regions = memblock_reserved_init_regions,
  107. .reserved.cnt = 1, /* empty dummy entry */
  108. .reserved.max = INIT_MEMBLOCK_RESERVED_REGIONS,
  109. .reserved.name = "reserved",
  110. .bottom_up = false,
  111. .current_limit = MEMBLOCK_ALLOC_ANYWHERE,
  112. };
  113. #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
  114. struct memblock_type physmem = {
  115. .regions = memblock_physmem_init_regions,
  116. .cnt = 1, /* empty dummy entry */
  117. .max = INIT_PHYSMEM_REGIONS,
  118. .name = "physmem",
  119. };
  120. #endif
  121. /*
  122. * keep a pointer to &memblock.memory in the text section to use it in
  123. * __next_mem_range() and its helpers.
  124. * For architectures that do not keep memblock data after init, this
  125. * pointer will be reset to NULL at memblock_discard()
  126. */
  127. static __refdata struct memblock_type *memblock_memory = &memblock.memory;
  128. #define for_each_memblock_type(i, memblock_type, rgn) \
  129. for (i = 0, rgn = &memblock_type->regions[0]; \
  130. i < memblock_type->cnt; \
  131. i++, rgn = &memblock_type->regions[i])
  132. #define memblock_dbg(fmt, ...) \
  133. do { \
  134. if (memblock_debug) \
  135. pr_info(fmt, ##__VA_ARGS__); \
  136. } while (0)
  137. static int memblock_debug __initdata_memblock;
  138. static bool system_has_some_mirror __initdata_memblock = false;
  139. static int memblock_can_resize __initdata_memblock;
  140. static int memblock_memory_in_slab __initdata_memblock = 0;
  141. static int memblock_reserved_in_slab __initdata_memblock = 0;
  142. static enum memblock_flags __init_memblock choose_memblock_flags(void)
  143. {
  144. return system_has_some_mirror ? MEMBLOCK_MIRROR : MEMBLOCK_NONE;
  145. }
  146. /* adjust *@size so that (@base + *@size) doesn't overflow, return new size */
  147. static inline phys_addr_t memblock_cap_size(phys_addr_t base, phys_addr_t *size)
  148. {
  149. return *size = min(*size, PHYS_ADDR_MAX - base);
  150. }
  151. /*
  152. * Address comparison utilities
  153. */
  154. static unsigned long __init_memblock memblock_addrs_overlap(phys_addr_t base1, phys_addr_t size1,
  155. phys_addr_t base2, phys_addr_t size2)
  156. {
  157. return ((base1 < (base2 + size2)) && (base2 < (base1 + size1)));
  158. }
  159. bool __init_memblock memblock_overlaps_region(struct memblock_type *type,
  160. phys_addr_t base, phys_addr_t size)
  161. {
  162. unsigned long i;
  163. memblock_cap_size(base, &size);
  164. for (i = 0; i < type->cnt; i++)
  165. if (memblock_addrs_overlap(base, size, type->regions[i].base,
  166. type->regions[i].size))
  167. break;
  168. return i < type->cnt;
  169. }
  170. /**
  171. * __memblock_find_range_bottom_up - find free area utility in bottom-up
  172. * @start: start of candidate range
  173. * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
  174. * %MEMBLOCK_ALLOC_ACCESSIBLE
  175. * @size: size of free area to find
  176. * @align: alignment of free area to find
  177. * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
  178. * @flags: pick from blocks based on memory attributes
  179. *
  180. * Utility called from memblock_find_in_range_node(), find free area bottom-up.
  181. *
  182. * Return:
  183. * Found address on success, 0 on failure.
  184. */
  185. static phys_addr_t __init_memblock
  186. __memblock_find_range_bottom_up(phys_addr_t start, phys_addr_t end,
  187. phys_addr_t size, phys_addr_t align, int nid,
  188. enum memblock_flags flags)
  189. {
  190. phys_addr_t this_start, this_end, cand;
  191. u64 i;
  192. for_each_free_mem_range(i, nid, flags, &this_start, &this_end, NULL) {
  193. this_start = clamp(this_start, start, end);
  194. this_end = clamp(this_end, start, end);
  195. cand = round_up(this_start, align);
  196. if (cand < this_end && this_end - cand >= size)
  197. return cand;
  198. }
  199. return 0;
  200. }
  201. /**
  202. * __memblock_find_range_top_down - find free area utility, in top-down
  203. * @start: start of candidate range
  204. * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
  205. * %MEMBLOCK_ALLOC_ACCESSIBLE
  206. * @size: size of free area to find
  207. * @align: alignment of free area to find
  208. * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
  209. * @flags: pick from blocks based on memory attributes
  210. *
  211. * Utility called from memblock_find_in_range_node(), find free area top-down.
  212. *
  213. * Return:
  214. * Found address on success, 0 on failure.
  215. */
  216. static phys_addr_t __init_memblock
  217. __memblock_find_range_top_down(phys_addr_t start, phys_addr_t end,
  218. phys_addr_t size, phys_addr_t align, int nid,
  219. enum memblock_flags flags)
  220. {
  221. phys_addr_t this_start, this_end, cand;
  222. u64 i;
  223. for_each_free_mem_range_reverse(i, nid, flags, &this_start, &this_end,
  224. NULL) {
  225. this_start = clamp(this_start, start, end);
  226. this_end = clamp(this_end, start, end);
  227. if (this_end < size)
  228. continue;
  229. cand = round_down(this_end - size, align);
  230. if (cand >= this_start)
  231. return cand;
  232. }
  233. return 0;
  234. }
  235. /**
  236. * memblock_find_in_range_node - find free area in given range and node
  237. * @size: size of free area to find
  238. * @align: alignment of free area to find
  239. * @start: start of candidate range
  240. * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
  241. * %MEMBLOCK_ALLOC_ACCESSIBLE
  242. * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
  243. * @flags: pick from blocks based on memory attributes
  244. *
  245. * Find @size free area aligned to @align in the specified range and node.
  246. *
  247. * Return:
  248. * Found address on success, 0 on failure.
  249. */
  250. static phys_addr_t __init_memblock memblock_find_in_range_node(phys_addr_t size,
  251. phys_addr_t align, phys_addr_t start,
  252. phys_addr_t end, int nid,
  253. enum memblock_flags flags)
  254. {
  255. /* pump up @end */
  256. if (end == MEMBLOCK_ALLOC_ACCESSIBLE ||
  257. end == MEMBLOCK_ALLOC_KASAN)
  258. end = memblock.current_limit;
  259. /* avoid allocating the first page */
  260. start = max_t(phys_addr_t, start, PAGE_SIZE);
  261. end = max(start, end);
  262. if (memblock_bottom_up())
  263. return __memblock_find_range_bottom_up(start, end, size, align,
  264. nid, flags);
  265. else
  266. return __memblock_find_range_top_down(start, end, size, align,
  267. nid, flags);
  268. }
  269. /**
  270. * memblock_find_in_range - find free area in given range
  271. * @start: start of candidate range
  272. * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
  273. * %MEMBLOCK_ALLOC_ACCESSIBLE
  274. * @size: size of free area to find
  275. * @align: alignment of free area to find
  276. *
  277. * Find @size free area aligned to @align in the specified range.
  278. *
  279. * Return:
  280. * Found address on success, 0 on failure.
  281. */
  282. phys_addr_t __init_memblock memblock_find_in_range(phys_addr_t start,
  283. phys_addr_t end, phys_addr_t size,
  284. phys_addr_t align)
  285. {
  286. phys_addr_t ret;
  287. enum memblock_flags flags = choose_memblock_flags();
  288. again:
  289. ret = memblock_find_in_range_node(size, align, start, end,
  290. NUMA_NO_NODE, flags);
  291. if (!ret && (flags & MEMBLOCK_MIRROR)) {
  292. pr_warn("Could not allocate %pap bytes of mirrored memory\n",
  293. &size);
  294. flags &= ~MEMBLOCK_MIRROR;
  295. goto again;
  296. }
  297. return ret;
  298. }
  299. static void __init_memblock memblock_remove_region(struct memblock_type *type, unsigned long r)
  300. {
  301. type->total_size -= type->regions[r].size;
  302. memmove(&type->regions[r], &type->regions[r + 1],
  303. (type->cnt - (r + 1)) * sizeof(type->regions[r]));
  304. type->cnt--;
  305. /* Special case for empty arrays */
  306. if (type->cnt == 0) {
  307. WARN_ON(type->total_size != 0);
  308. type->cnt = 1;
  309. type->regions[0].base = 0;
  310. type->regions[0].size = 0;
  311. type->regions[0].flags = 0;
  312. memblock_set_region_node(&type->regions[0], MAX_NUMNODES);
  313. }
  314. }
  315. #ifndef CONFIG_ARCH_KEEP_MEMBLOCK
  316. /**
  317. * memblock_discard - discard memory and reserved arrays if they were allocated
  318. */
  319. void __init memblock_discard(void)
  320. {
  321. phys_addr_t addr, size;
  322. if (memblock.reserved.regions != memblock_reserved_init_regions) {
  323. addr = __pa(memblock.reserved.regions);
  324. size = PAGE_ALIGN(sizeof(struct memblock_region) *
  325. memblock.reserved.max);
  326. if (memblock_reserved_in_slab)
  327. kfree(memblock.reserved.regions);
  328. else
  329. __memblock_free_late(addr, size);
  330. }
  331. if (memblock.memory.regions != memblock_memory_init_regions) {
  332. addr = __pa(memblock.memory.regions);
  333. size = PAGE_ALIGN(sizeof(struct memblock_region) *
  334. memblock.memory.max);
  335. if (memblock_memory_in_slab)
  336. kfree(memblock.memory.regions);
  337. else
  338. __memblock_free_late(addr, size);
  339. }
  340. memblock_memory = NULL;
  341. }
  342. #endif
  343. /**
  344. * memblock_double_array - double the size of the memblock regions array
  345. * @type: memblock type of the regions array being doubled
  346. * @new_area_start: starting address of memory range to avoid overlap with
  347. * @new_area_size: size of memory range to avoid overlap with
  348. *
  349. * Double the size of the @type regions array. If memblock is being used to
  350. * allocate memory for a new reserved regions array and there is a previously
  351. * allocated memory range [@new_area_start, @new_area_start + @new_area_size]
  352. * waiting to be reserved, ensure the memory used by the new array does
  353. * not overlap.
  354. *
  355. * Return:
  356. * 0 on success, -1 on failure.
  357. */
  358. static int __init_memblock memblock_double_array(struct memblock_type *type,
  359. phys_addr_t new_area_start,
  360. phys_addr_t new_area_size)
  361. {
  362. struct memblock_region *new_array, *old_array;
  363. phys_addr_t old_alloc_size, new_alloc_size;
  364. phys_addr_t old_size, new_size, addr, new_end;
  365. int use_slab = slab_is_available();
  366. int *in_slab;
  367. /* We don't allow resizing until we know about the reserved regions
  368. * of memory that aren't suitable for allocation
  369. */
  370. if (!memblock_can_resize)
  371. return -1;
  372. /* Calculate new doubled size */
  373. old_size = type->max * sizeof(struct memblock_region);
  374. new_size = old_size << 1;
  375. /*
  376. * We need to allocated new one align to PAGE_SIZE,
  377. * so we can free them completely later.
  378. */
  379. old_alloc_size = PAGE_ALIGN(old_size);
  380. new_alloc_size = PAGE_ALIGN(new_size);
  381. /* Retrieve the slab flag */
  382. if (type == &memblock.memory)
  383. in_slab = &memblock_memory_in_slab;
  384. else
  385. in_slab = &memblock_reserved_in_slab;
  386. /* Try to find some space for it */
  387. if (use_slab) {
  388. new_array = kmalloc(new_size, GFP_KERNEL);
  389. addr = new_array ? __pa(new_array) : 0;
  390. } else {
  391. /* only exclude range when trying to double reserved.regions */
  392. if (type != &memblock.reserved)
  393. new_area_start = new_area_size = 0;
  394. addr = memblock_find_in_range(new_area_start + new_area_size,
  395. memblock.current_limit,
  396. new_alloc_size, PAGE_SIZE);
  397. if (!addr && new_area_size)
  398. addr = memblock_find_in_range(0,
  399. min(new_area_start, memblock.current_limit),
  400. new_alloc_size, PAGE_SIZE);
  401. new_array = addr ? __va(addr) : NULL;
  402. }
  403. if (!addr) {
  404. pr_err("memblock: Failed to double %s array from %ld to %ld entries !\n",
  405. type->name, type->max, type->max * 2);
  406. return -1;
  407. }
  408. new_end = addr + new_size - 1;
  409. memblock_dbg("memblock: %s is doubled to %ld at [%pa-%pa]",
  410. type->name, type->max * 2, &addr, &new_end);
  411. /*
  412. * Found space, we now need to move the array over before we add the
  413. * reserved region since it may be our reserved array itself that is
  414. * full.
  415. */
  416. memcpy(new_array, type->regions, old_size);
  417. memset(new_array + type->max, 0, old_size);
  418. old_array = type->regions;
  419. type->regions = new_array;
  420. type->max <<= 1;
  421. /* Free old array. We needn't free it if the array is the static one */
  422. if (*in_slab)
  423. kfree(old_array);
  424. else if (old_array != memblock_memory_init_regions &&
  425. old_array != memblock_reserved_init_regions)
  426. memblock_free(__pa(old_array), old_alloc_size);
  427. /*
  428. * Reserve the new array if that comes from the memblock. Otherwise, we
  429. * needn't do it
  430. */
  431. if (!use_slab)
  432. BUG_ON(memblock_reserve(addr, new_alloc_size));
  433. /* Update slab flag */
  434. *in_slab = use_slab;
  435. return 0;
  436. }
  437. /**
  438. * memblock_merge_regions - merge neighboring compatible regions
  439. * @type: memblock type to scan
  440. *
  441. * Scan @type and merge neighboring compatible regions.
  442. */
  443. static void __init_memblock memblock_merge_regions(struct memblock_type *type)
  444. {
  445. int i = 0;
  446. /* cnt never goes below 1 */
  447. while (i < type->cnt - 1) {
  448. struct memblock_region *this = &type->regions[i];
  449. struct memblock_region *next = &type->regions[i + 1];
  450. if (this->base + this->size != next->base ||
  451. memblock_get_region_node(this) !=
  452. memblock_get_region_node(next) ||
  453. this->flags != next->flags) {
  454. BUG_ON(this->base + this->size > next->base);
  455. i++;
  456. continue;
  457. }
  458. this->size += next->size;
  459. /* move forward from next + 1, index of which is i + 2 */
  460. memmove(next, next + 1, (type->cnt - (i + 2)) * sizeof(*next));
  461. type->cnt--;
  462. }
  463. }
  464. /**
  465. * memblock_insert_region - insert new memblock region
  466. * @type: memblock type to insert into
  467. * @idx: index for the insertion point
  468. * @base: base address of the new region
  469. * @size: size of the new region
  470. * @nid: node id of the new region
  471. * @flags: flags of the new region
  472. *
  473. * Insert new memblock region [@base, @base + @size) into @type at @idx.
  474. * @type must already have extra room to accommodate the new region.
  475. */
  476. static void __init_memblock memblock_insert_region(struct memblock_type *type,
  477. int idx, phys_addr_t base,
  478. phys_addr_t size,
  479. int nid,
  480. enum memblock_flags flags)
  481. {
  482. struct memblock_region *rgn = &type->regions[idx];
  483. BUG_ON(type->cnt >= type->max);
  484. memmove(rgn + 1, rgn, (type->cnt - idx) * sizeof(*rgn));
  485. rgn->base = base;
  486. rgn->size = size;
  487. rgn->flags = flags;
  488. memblock_set_region_node(rgn, nid);
  489. type->cnt++;
  490. type->total_size += size;
  491. }
  492. /**
  493. * memblock_add_range - add new memblock region
  494. * @type: memblock type to add new region into
  495. * @base: base address of the new region
  496. * @size: size of the new region
  497. * @nid: nid of the new region
  498. * @flags: flags of the new region
  499. *
  500. * Add new memblock region [@base, @base + @size) into @type. The new region
  501. * is allowed to overlap with existing ones - overlaps don't affect already
  502. * existing regions. @type is guaranteed to be minimal (all neighbouring
  503. * compatible regions are merged) after the addition.
  504. *
  505. * Return:
  506. * 0 on success, -errno on failure.
  507. */
  508. static int __init_memblock memblock_add_range(struct memblock_type *type,
  509. phys_addr_t base, phys_addr_t size,
  510. int nid, enum memblock_flags flags)
  511. {
  512. bool insert = false;
  513. phys_addr_t obase = base;
  514. phys_addr_t end = base + memblock_cap_size(base, &size);
  515. int idx, nr_new;
  516. struct memblock_region *rgn;
  517. if (!size)
  518. return 0;
  519. /* special case for empty array */
  520. if (type->regions[0].size == 0) {
  521. WARN_ON(type->cnt != 1 || type->total_size);
  522. type->regions[0].base = base;
  523. type->regions[0].size = size;
  524. type->regions[0].flags = flags;
  525. memblock_set_region_node(&type->regions[0], nid);
  526. type->total_size = size;
  527. return 0;
  528. }
  529. repeat:
  530. /*
  531. * The following is executed twice. Once with %false @insert and
  532. * then with %true. The first counts the number of regions needed
  533. * to accommodate the new area. The second actually inserts them.
  534. */
  535. base = obase;
  536. nr_new = 0;
  537. for_each_memblock_type(idx, type, rgn) {
  538. phys_addr_t rbase = rgn->base;
  539. phys_addr_t rend = rbase + rgn->size;
  540. if (rbase >= end)
  541. break;
  542. if (rend <= base)
  543. continue;
  544. /*
  545. * @rgn overlaps. If it separates the lower part of new
  546. * area, insert that portion.
  547. */
  548. if (rbase > base) {
  549. #ifdef CONFIG_NEED_MULTIPLE_NODES
  550. WARN_ON(nid != memblock_get_region_node(rgn));
  551. #endif
  552. WARN_ON(flags != rgn->flags);
  553. nr_new++;
  554. if (insert)
  555. memblock_insert_region(type, idx++, base,
  556. rbase - base, nid,
  557. flags);
  558. }
  559. /* area below @rend is dealt with, forget about it */
  560. base = min(rend, end);
  561. }
  562. /* insert the remaining portion */
  563. if (base < end) {
  564. nr_new++;
  565. if (insert)
  566. memblock_insert_region(type, idx, base, end - base,
  567. nid, flags);
  568. }
  569. if (!nr_new)
  570. return 0;
  571. /*
  572. * If this was the first round, resize array and repeat for actual
  573. * insertions; otherwise, merge and return.
  574. */
  575. if (!insert) {
  576. while (type->cnt + nr_new > type->max)
  577. if (memblock_double_array(type, obase, size) < 0)
  578. return -ENOMEM;
  579. insert = true;
  580. goto repeat;
  581. } else {
  582. memblock_merge_regions(type);
  583. return 0;
  584. }
  585. }
  586. /**
  587. * memblock_add_node - add new memblock region within a NUMA node
  588. * @base: base address of the new region
  589. * @size: size of the new region
  590. * @nid: nid of the new region
  591. *
  592. * Add new memblock region [@base, @base + @size) to the "memory"
  593. * type. See memblock_add_range() description for mode details
  594. *
  595. * Return:
  596. * 0 on success, -errno on failure.
  597. */
  598. int __init_memblock memblock_add_node(phys_addr_t base, phys_addr_t size,
  599. int nid)
  600. {
  601. return memblock_add_range(&memblock.memory, base, size, nid, 0);
  602. }
  603. /**
  604. * memblock_add - add new memblock region
  605. * @base: base address of the new region
  606. * @size: size of the new region
  607. *
  608. * Add new memblock region [@base, @base + @size) to the "memory"
  609. * type. See memblock_add_range() description for mode details
  610. *
  611. * Return:
  612. * 0 on success, -errno on failure.
  613. */
  614. int __init_memblock memblock_add(phys_addr_t base, phys_addr_t size)
  615. {
  616. phys_addr_t end = base + size - 1;
  617. memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
  618. &base, &end, (void *)_RET_IP_);
  619. return memblock_add_range(&memblock.memory, base, size, MAX_NUMNODES, 0);
  620. }
  621. /**
  622. * memblock_isolate_range - isolate given range into disjoint memblocks
  623. * @type: memblock type to isolate range for
  624. * @base: base of range to isolate
  625. * @size: size of range to isolate
  626. * @start_rgn: out parameter for the start of isolated region
  627. * @end_rgn: out parameter for the end of isolated region
  628. *
  629. * Walk @type and ensure that regions don't cross the boundaries defined by
  630. * [@base, @base + @size). Crossing regions are split at the boundaries,
  631. * which may create at most two more regions. The index of the first
  632. * region inside the range is returned in *@start_rgn and end in *@end_rgn.
  633. *
  634. * Return:
  635. * 0 on success, -errno on failure.
  636. */
  637. static int __init_memblock memblock_isolate_range(struct memblock_type *type,
  638. phys_addr_t base, phys_addr_t size,
  639. int *start_rgn, int *end_rgn)
  640. {
  641. phys_addr_t end = base + memblock_cap_size(base, &size);
  642. int idx;
  643. struct memblock_region *rgn;
  644. *start_rgn = *end_rgn = 0;
  645. if (!size)
  646. return 0;
  647. /* we'll create at most two more regions */
  648. while (type->cnt + 2 > type->max)
  649. if (memblock_double_array(type, base, size) < 0)
  650. return -ENOMEM;
  651. for_each_memblock_type(idx, type, rgn) {
  652. phys_addr_t rbase = rgn->base;
  653. phys_addr_t rend = rbase + rgn->size;
  654. if (rbase >= end)
  655. break;
  656. if (rend <= base)
  657. continue;
  658. if (rbase < base) {
  659. /*
  660. * @rgn intersects from below. Split and continue
  661. * to process the next region - the new top half.
  662. */
  663. rgn->base = base;
  664. rgn->size -= base - rbase;
  665. type->total_size -= base - rbase;
  666. memblock_insert_region(type, idx, rbase, base - rbase,
  667. memblock_get_region_node(rgn),
  668. rgn->flags);
  669. } else if (rend > end) {
  670. /*
  671. * @rgn intersects from above. Split and redo the
  672. * current region - the new bottom half.
  673. */
  674. rgn->base = end;
  675. rgn->size -= end - rbase;
  676. type->total_size -= end - rbase;
  677. memblock_insert_region(type, idx--, rbase, end - rbase,
  678. memblock_get_region_node(rgn),
  679. rgn->flags);
  680. } else {
  681. /* @rgn is fully contained, record it */
  682. if (!*end_rgn)
  683. *start_rgn = idx;
  684. *end_rgn = idx + 1;
  685. }
  686. }
  687. return 0;
  688. }
  689. static int __init_memblock memblock_remove_range(struct memblock_type *type,
  690. phys_addr_t base, phys_addr_t size)
  691. {
  692. int start_rgn, end_rgn;
  693. int i, ret;
  694. ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn);
  695. if (ret)
  696. return ret;
  697. for (i = end_rgn - 1; i >= start_rgn; i--)
  698. memblock_remove_region(type, i);
  699. return 0;
  700. }
  701. int __init_memblock memblock_remove(phys_addr_t base, phys_addr_t size)
  702. {
  703. phys_addr_t end = base + size - 1;
  704. memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
  705. &base, &end, (void *)_RET_IP_);
  706. return memblock_remove_range(&memblock.memory, base, size);
  707. }
  708. /**
  709. * memblock_free - free boot memory block
  710. * @base: phys starting address of the boot memory block
  711. * @size: size of the boot memory block in bytes
  712. *
  713. * Free boot memory block previously allocated by memblock_alloc_xx() API.
  714. * The freeing memory will not be released to the buddy allocator.
  715. */
  716. int __init_memblock memblock_free(phys_addr_t base, phys_addr_t size)
  717. {
  718. phys_addr_t end = base + size - 1;
  719. memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
  720. &base, &end, (void *)_RET_IP_);
  721. kmemleak_free_part_phys(base, size);
  722. return memblock_remove_range(&memblock.reserved, base, size);
  723. }
  724. #ifdef CONFIG_ARCH_KEEP_MEMBLOCK
  725. EXPORT_SYMBOL_GPL(memblock_free);
  726. #endif
  727. int __init_memblock memblock_reserve(phys_addr_t base, phys_addr_t size)
  728. {
  729. phys_addr_t end = base + size - 1;
  730. memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
  731. &base, &end, (void *)_RET_IP_);
  732. return memblock_add_range(&memblock.reserved, base, size, MAX_NUMNODES, 0);
  733. }
  734. #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
  735. int __init_memblock memblock_physmem_add(phys_addr_t base, phys_addr_t size)
  736. {
  737. phys_addr_t end = base + size - 1;
  738. memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
  739. &base, &end, (void *)_RET_IP_);
  740. return memblock_add_range(&physmem, base, size, MAX_NUMNODES, 0);
  741. }
  742. #endif
  743. /**
  744. * memblock_setclr_flag - set or clear flag for a memory region
  745. * @base: base address of the region
  746. * @size: size of the region
  747. * @set: set or clear the flag
  748. * @flag: the flag to udpate
  749. *
  750. * This function isolates region [@base, @base + @size), and sets/clears flag
  751. *
  752. * Return: 0 on success, -errno on failure.
  753. */
  754. static int __init_memblock memblock_setclr_flag(phys_addr_t base,
  755. phys_addr_t size, int set, int flag)
  756. {
  757. struct memblock_type *type = &memblock.memory;
  758. int i, ret, start_rgn, end_rgn;
  759. ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn);
  760. if (ret)
  761. return ret;
  762. for (i = start_rgn; i < end_rgn; i++) {
  763. struct memblock_region *r = &type->regions[i];
  764. if (set)
  765. r->flags |= flag;
  766. else
  767. r->flags &= ~flag;
  768. }
  769. memblock_merge_regions(type);
  770. return 0;
  771. }
  772. /**
  773. * memblock_mark_hotplug - Mark hotpluggable memory with flag MEMBLOCK_HOTPLUG.
  774. * @base: the base phys addr of the region
  775. * @size: the size of the region
  776. *
  777. * Return: 0 on success, -errno on failure.
  778. */
  779. int __init_memblock memblock_mark_hotplug(phys_addr_t base, phys_addr_t size)
  780. {
  781. return memblock_setclr_flag(base, size, 1, MEMBLOCK_HOTPLUG);
  782. }
  783. /**
  784. * memblock_clear_hotplug - Clear flag MEMBLOCK_HOTPLUG for a specified region.
  785. * @base: the base phys addr of the region
  786. * @size: the size of the region
  787. *
  788. * Return: 0 on success, -errno on failure.
  789. */
  790. int __init_memblock memblock_clear_hotplug(phys_addr_t base, phys_addr_t size)
  791. {
  792. return memblock_setclr_flag(base, size, 0, MEMBLOCK_HOTPLUG);
  793. }
  794. /**
  795. * memblock_mark_mirror - Mark mirrored memory with flag MEMBLOCK_MIRROR.
  796. * @base: the base phys addr of the region
  797. * @size: the size of the region
  798. *
  799. * Return: 0 on success, -errno on failure.
  800. */
  801. int __init_memblock memblock_mark_mirror(phys_addr_t base, phys_addr_t size)
  802. {
  803. system_has_some_mirror = true;
  804. return memblock_setclr_flag(base, size, 1, MEMBLOCK_MIRROR);
  805. }
  806. /**
  807. * memblock_mark_nomap - Mark a memory region with flag MEMBLOCK_NOMAP.
  808. * @base: the base phys addr of the region
  809. * @size: the size of the region
  810. *
  811. * Return: 0 on success, -errno on failure.
  812. */
  813. int __init_memblock memblock_mark_nomap(phys_addr_t base, phys_addr_t size)
  814. {
  815. return memblock_setclr_flag(base, size, 1, MEMBLOCK_NOMAP);
  816. }
  817. /**
  818. * memblock_clear_nomap - Clear flag MEMBLOCK_NOMAP for a specified region.
  819. * @base: the base phys addr of the region
  820. * @size: the size of the region
  821. *
  822. * Return: 0 on success, -errno on failure.
  823. */
  824. int __init_memblock memblock_clear_nomap(phys_addr_t base, phys_addr_t size)
  825. {
  826. return memblock_setclr_flag(base, size, 0, MEMBLOCK_NOMAP);
  827. }
  828. static bool should_skip_region(struct memblock_type *type,
  829. struct memblock_region *m,
  830. int nid, int flags)
  831. {
  832. int m_nid = memblock_get_region_node(m);
  833. /* we never skip regions when iterating memblock.reserved or physmem */
  834. if (type != memblock_memory)
  835. return false;
  836. /* only memory regions are associated with nodes, check it */
  837. if (nid != NUMA_NO_NODE && nid != m_nid)
  838. return true;
  839. /* skip hotpluggable memory regions if needed */
  840. if (movable_node_is_enabled() && memblock_is_hotpluggable(m) &&
  841. !(flags & MEMBLOCK_HOTPLUG))
  842. return true;
  843. /* if we want mirror memory skip non-mirror memory regions */
  844. if ((flags & MEMBLOCK_MIRROR) && !memblock_is_mirror(m))
  845. return true;
  846. /* skip nomap memory unless we were asked for it explicitly */
  847. if (!(flags & MEMBLOCK_NOMAP) && memblock_is_nomap(m))
  848. return true;
  849. return false;
  850. }
  851. /**
  852. * __next_mem_range - next function for for_each_free_mem_range() etc.
  853. * @idx: pointer to u64 loop variable
  854. * @nid: node selector, %NUMA_NO_NODE for all nodes
  855. * @flags: pick from blocks based on memory attributes
  856. * @type_a: pointer to memblock_type from where the range is taken
  857. * @type_b: pointer to memblock_type which excludes memory from being taken
  858. * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL
  859. * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL
  860. * @out_nid: ptr to int for nid of the range, can be %NULL
  861. *
  862. * Find the first area from *@idx which matches @nid, fill the out
  863. * parameters, and update *@idx for the next iteration. The lower 32bit of
  864. * *@idx contains index into type_a and the upper 32bit indexes the
  865. * areas before each region in type_b. For example, if type_b regions
  866. * look like the following,
  867. *
  868. * 0:[0-16), 1:[32-48), 2:[128-130)
  869. *
  870. * The upper 32bit indexes the following regions.
  871. *
  872. * 0:[0-0), 1:[16-32), 2:[48-128), 3:[130-MAX)
  873. *
  874. * As both region arrays are sorted, the function advances the two indices
  875. * in lockstep and returns each intersection.
  876. */
  877. void __next_mem_range(u64 *idx, int nid, enum memblock_flags flags,
  878. struct memblock_type *type_a,
  879. struct memblock_type *type_b, phys_addr_t *out_start,
  880. phys_addr_t *out_end, int *out_nid)
  881. {
  882. int idx_a = *idx & 0xffffffff;
  883. int idx_b = *idx >> 32;
  884. if (WARN_ONCE(nid == MAX_NUMNODES,
  885. "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n"))
  886. nid = NUMA_NO_NODE;
  887. for (; idx_a < type_a->cnt; idx_a++) {
  888. struct memblock_region *m = &type_a->regions[idx_a];
  889. phys_addr_t m_start = m->base;
  890. phys_addr_t m_end = m->base + m->size;
  891. int m_nid = memblock_get_region_node(m);
  892. if (should_skip_region(type_a, m, nid, flags))
  893. continue;
  894. if (!type_b) {
  895. if (out_start)
  896. *out_start = m_start;
  897. if (out_end)
  898. *out_end = m_end;
  899. if (out_nid)
  900. *out_nid = m_nid;
  901. idx_a++;
  902. *idx = (u32)idx_a | (u64)idx_b << 32;
  903. return;
  904. }
  905. /* scan areas before each reservation */
  906. for (; idx_b < type_b->cnt + 1; idx_b++) {
  907. struct memblock_region *r;
  908. phys_addr_t r_start;
  909. phys_addr_t r_end;
  910. r = &type_b->regions[idx_b];
  911. r_start = idx_b ? r[-1].base + r[-1].size : 0;
  912. r_end = idx_b < type_b->cnt ?
  913. r->base : PHYS_ADDR_MAX;
  914. /*
  915. * if idx_b advanced past idx_a,
  916. * break out to advance idx_a
  917. */
  918. if (r_start >= m_end)
  919. break;
  920. /* if the two regions intersect, we're done */
  921. if (m_start < r_end) {
  922. if (out_start)
  923. *out_start =
  924. max(m_start, r_start);
  925. if (out_end)
  926. *out_end = min(m_end, r_end);
  927. if (out_nid)
  928. *out_nid = m_nid;
  929. /*
  930. * The region which ends first is
  931. * advanced for the next iteration.
  932. */
  933. if (m_end <= r_end)
  934. idx_a++;
  935. else
  936. idx_b++;
  937. *idx = (u32)idx_a | (u64)idx_b << 32;
  938. return;
  939. }
  940. }
  941. }
  942. /* signal end of iteration */
  943. *idx = ULLONG_MAX;
  944. }
  945. /**
  946. * __next_mem_range_rev - generic next function for for_each_*_range_rev()
  947. *
  948. * @idx: pointer to u64 loop variable
  949. * @nid: node selector, %NUMA_NO_NODE for all nodes
  950. * @flags: pick from blocks based on memory attributes
  951. * @type_a: pointer to memblock_type from where the range is taken
  952. * @type_b: pointer to memblock_type which excludes memory from being taken
  953. * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL
  954. * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL
  955. * @out_nid: ptr to int for nid of the range, can be %NULL
  956. *
  957. * Finds the next range from type_a which is not marked as unsuitable
  958. * in type_b.
  959. *
  960. * Reverse of __next_mem_range().
  961. */
  962. void __init_memblock __next_mem_range_rev(u64 *idx, int nid,
  963. enum memblock_flags flags,
  964. struct memblock_type *type_a,
  965. struct memblock_type *type_b,
  966. phys_addr_t *out_start,
  967. phys_addr_t *out_end, int *out_nid)
  968. {
  969. int idx_a = *idx & 0xffffffff;
  970. int idx_b = *idx >> 32;
  971. if (WARN_ONCE(nid == MAX_NUMNODES, "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n"))
  972. nid = NUMA_NO_NODE;
  973. if (*idx == (u64)ULLONG_MAX) {
  974. idx_a = type_a->cnt - 1;
  975. if (type_b != NULL)
  976. idx_b = type_b->cnt;
  977. else
  978. idx_b = 0;
  979. }
  980. for (; idx_a >= 0; idx_a--) {
  981. struct memblock_region *m = &type_a->regions[idx_a];
  982. phys_addr_t m_start = m->base;
  983. phys_addr_t m_end = m->base + m->size;
  984. int m_nid = memblock_get_region_node(m);
  985. if (should_skip_region(type_a, m, nid, flags))
  986. continue;
  987. if (!type_b) {
  988. if (out_start)
  989. *out_start = m_start;
  990. if (out_end)
  991. *out_end = m_end;
  992. if (out_nid)
  993. *out_nid = m_nid;
  994. idx_a--;
  995. *idx = (u32)idx_a | (u64)idx_b << 32;
  996. return;
  997. }
  998. /* scan areas before each reservation */
  999. for (; idx_b >= 0; idx_b--) {
  1000. struct memblock_region *r;
  1001. phys_addr_t r_start;
  1002. phys_addr_t r_end;
  1003. r = &type_b->regions[idx_b];
  1004. r_start = idx_b ? r[-1].base + r[-1].size : 0;
  1005. r_end = idx_b < type_b->cnt ?
  1006. r->base : PHYS_ADDR_MAX;
  1007. /*
  1008. * if idx_b advanced past idx_a,
  1009. * break out to advance idx_a
  1010. */
  1011. if (r_end <= m_start)
  1012. break;
  1013. /* if the two regions intersect, we're done */
  1014. if (m_end > r_start) {
  1015. if (out_start)
  1016. *out_start = max(m_start, r_start);
  1017. if (out_end)
  1018. *out_end = min(m_end, r_end);
  1019. if (out_nid)
  1020. *out_nid = m_nid;
  1021. if (m_start >= r_start)
  1022. idx_a--;
  1023. else
  1024. idx_b--;
  1025. *idx = (u32)idx_a | (u64)idx_b << 32;
  1026. return;
  1027. }
  1028. }
  1029. }
  1030. /* signal end of iteration */
  1031. *idx = ULLONG_MAX;
  1032. }
  1033. /*
  1034. * Common iterator interface used to define for_each_mem_pfn_range().
  1035. */
  1036. void __init_memblock __next_mem_pfn_range(int *idx, int nid,
  1037. unsigned long *out_start_pfn,
  1038. unsigned long *out_end_pfn, int *out_nid)
  1039. {
  1040. struct memblock_type *type = &memblock.memory;
  1041. struct memblock_region *r;
  1042. int r_nid;
  1043. while (++*idx < type->cnt) {
  1044. r = &type->regions[*idx];
  1045. r_nid = memblock_get_region_node(r);
  1046. if (PFN_UP(r->base) >= PFN_DOWN(r->base + r->size))
  1047. continue;
  1048. if (nid == MAX_NUMNODES || nid == r_nid)
  1049. break;
  1050. }
  1051. if (*idx >= type->cnt) {
  1052. *idx = -1;
  1053. return;
  1054. }
  1055. if (out_start_pfn)
  1056. *out_start_pfn = PFN_UP(r->base);
  1057. if (out_end_pfn)
  1058. *out_end_pfn = PFN_DOWN(r->base + r->size);
  1059. if (out_nid)
  1060. *out_nid = r_nid;
  1061. }
  1062. /**
  1063. * memblock_set_node - set node ID on memblock regions
  1064. * @base: base of area to set node ID for
  1065. * @size: size of area to set node ID for
  1066. * @type: memblock type to set node ID for
  1067. * @nid: node ID to set
  1068. *
  1069. * Set the nid of memblock @type regions in [@base, @base + @size) to @nid.
  1070. * Regions which cross the area boundaries are split as necessary.
  1071. *
  1072. * Return:
  1073. * 0 on success, -errno on failure.
  1074. */
  1075. int __init_memblock memblock_set_node(phys_addr_t base, phys_addr_t size,
  1076. struct memblock_type *type, int nid)
  1077. {
  1078. #ifdef CONFIG_NEED_MULTIPLE_NODES
  1079. int start_rgn, end_rgn;
  1080. int i, ret;
  1081. ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn);
  1082. if (ret)
  1083. return ret;
  1084. for (i = start_rgn; i < end_rgn; i++)
  1085. memblock_set_region_node(&type->regions[i], nid);
  1086. memblock_merge_regions(type);
  1087. #endif
  1088. return 0;
  1089. }
  1090. #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
  1091. /**
  1092. * __next_mem_pfn_range_in_zone - iterator for for_each_*_range_in_zone()
  1093. *
  1094. * @idx: pointer to u64 loop variable
  1095. * @zone: zone in which all of the memory blocks reside
  1096. * @out_spfn: ptr to ulong for start pfn of the range, can be %NULL
  1097. * @out_epfn: ptr to ulong for end pfn of the range, can be %NULL
  1098. *
  1099. * This function is meant to be a zone/pfn specific wrapper for the
  1100. * for_each_mem_range type iterators. Specifically they are used in the
  1101. * deferred memory init routines and as such we were duplicating much of
  1102. * this logic throughout the code. So instead of having it in multiple
  1103. * locations it seemed like it would make more sense to centralize this to
  1104. * one new iterator that does everything they need.
  1105. */
  1106. void __init_memblock
  1107. __next_mem_pfn_range_in_zone(u64 *idx, struct zone *zone,
  1108. unsigned long *out_spfn, unsigned long *out_epfn)
  1109. {
  1110. int zone_nid = zone_to_nid(zone);
  1111. phys_addr_t spa, epa;
  1112. int nid;
  1113. __next_mem_range(idx, zone_nid, MEMBLOCK_NONE,
  1114. &memblock.memory, &memblock.reserved,
  1115. &spa, &epa, &nid);
  1116. while (*idx != U64_MAX) {
  1117. unsigned long epfn = PFN_DOWN(epa);
  1118. unsigned long spfn = PFN_UP(spa);
  1119. /*
  1120. * Verify the end is at least past the start of the zone and
  1121. * that we have at least one PFN to initialize.
  1122. */
  1123. if (zone->zone_start_pfn < epfn && spfn < epfn) {
  1124. /* if we went too far just stop searching */
  1125. if (zone_end_pfn(zone) <= spfn) {
  1126. *idx = U64_MAX;
  1127. break;
  1128. }
  1129. if (out_spfn)
  1130. *out_spfn = max(zone->zone_start_pfn, spfn);
  1131. if (out_epfn)
  1132. *out_epfn = min(zone_end_pfn(zone), epfn);
  1133. return;
  1134. }
  1135. __next_mem_range(idx, zone_nid, MEMBLOCK_NONE,
  1136. &memblock.memory, &memblock.reserved,
  1137. &spa, &epa, &nid);
  1138. }
  1139. /* signal end of iteration */
  1140. if (out_spfn)
  1141. *out_spfn = ULONG_MAX;
  1142. if (out_epfn)
  1143. *out_epfn = 0;
  1144. }
  1145. #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
  1146. /**
  1147. * memblock_alloc_range_nid - allocate boot memory block
  1148. * @size: size of memory block to be allocated in bytes
  1149. * @align: alignment of the region and block's size
  1150. * @start: the lower bound of the memory region to allocate (phys address)
  1151. * @end: the upper bound of the memory region to allocate (phys address)
  1152. * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
  1153. * @exact_nid: control the allocation fall back to other nodes
  1154. *
  1155. * The allocation is performed from memory region limited by
  1156. * memblock.current_limit if @end == %MEMBLOCK_ALLOC_ACCESSIBLE.
  1157. *
  1158. * If the specified node can not hold the requested memory and @exact_nid
  1159. * is false, the allocation falls back to any node in the system.
  1160. *
  1161. * For systems with memory mirroring, the allocation is attempted first
  1162. * from the regions with mirroring enabled and then retried from any
  1163. * memory region.
  1164. *
  1165. * In addition, function sets the min_count to 0 using kmemleak_alloc_phys for
  1166. * allocated boot memory block, so that it is never reported as leaks.
  1167. *
  1168. * Return:
  1169. * Physical address of allocated memory block on success, %0 on failure.
  1170. */
  1171. phys_addr_t __init memblock_alloc_range_nid(phys_addr_t size,
  1172. phys_addr_t align, phys_addr_t start,
  1173. phys_addr_t end, int nid,
  1174. bool exact_nid)
  1175. {
  1176. enum memblock_flags flags = choose_memblock_flags();
  1177. phys_addr_t found;
  1178. if (WARN_ONCE(nid == MAX_NUMNODES, "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n"))
  1179. nid = NUMA_NO_NODE;
  1180. if (!align) {
  1181. /* Can't use WARNs this early in boot on powerpc */
  1182. dump_stack();
  1183. align = SMP_CACHE_BYTES;
  1184. }
  1185. again:
  1186. found = memblock_find_in_range_node(size, align, start, end, nid,
  1187. flags);
  1188. if (found && !memblock_reserve(found, size))
  1189. goto done;
  1190. if (nid != NUMA_NO_NODE && !exact_nid) {
  1191. found = memblock_find_in_range_node(size, align, start,
  1192. end, NUMA_NO_NODE,
  1193. flags);
  1194. if (found && !memblock_reserve(found, size))
  1195. goto done;
  1196. }
  1197. if (flags & MEMBLOCK_MIRROR) {
  1198. flags &= ~MEMBLOCK_MIRROR;
  1199. pr_warn("Could not allocate %pap bytes of mirrored memory\n",
  1200. &size);
  1201. goto again;
  1202. }
  1203. return 0;
  1204. done:
  1205. /* Skip kmemleak for kasan_init() due to high volume. */
  1206. if (end != MEMBLOCK_ALLOC_KASAN)
  1207. /*
  1208. * The min_count is set to 0 so that memblock allocated
  1209. * blocks are never reported as leaks. This is because many
  1210. * of these blocks are only referred via the physical
  1211. * address which is not looked up by kmemleak.
  1212. */
  1213. kmemleak_alloc_phys(found, size, 0, 0);
  1214. return found;
  1215. }
  1216. /**
  1217. * memblock_phys_alloc_range - allocate a memory block inside specified range
  1218. * @size: size of memory block to be allocated in bytes
  1219. * @align: alignment of the region and block's size
  1220. * @start: the lower bound of the memory region to allocate (physical address)
  1221. * @end: the upper bound of the memory region to allocate (physical address)
  1222. *
  1223. * Allocate @size bytes in the between @start and @end.
  1224. *
  1225. * Return: physical address of the allocated memory block on success,
  1226. * %0 on failure.
  1227. */
  1228. phys_addr_t __init memblock_phys_alloc_range(phys_addr_t size,
  1229. phys_addr_t align,
  1230. phys_addr_t start,
  1231. phys_addr_t end)
  1232. {
  1233. memblock_dbg("%s: %llu bytes align=0x%llx from=%pa max_addr=%pa %pS\n",
  1234. __func__, (u64)size, (u64)align, &start, &end,
  1235. (void *)_RET_IP_);
  1236. return memblock_alloc_range_nid(size, align, start, end, NUMA_NO_NODE,
  1237. false);
  1238. }
  1239. /**
  1240. * memblock_phys_alloc_try_nid - allocate a memory block from specified MUMA node
  1241. * @size: size of memory block to be allocated in bytes
  1242. * @align: alignment of the region and block's size
  1243. * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
  1244. *
  1245. * Allocates memory block from the specified NUMA node. If the node
  1246. * has no available memory, attempts to allocated from any node in the
  1247. * system.
  1248. *
  1249. * Return: physical address of the allocated memory block on success,
  1250. * %0 on failure.
  1251. */
  1252. phys_addr_t __init memblock_phys_alloc_try_nid(phys_addr_t size, phys_addr_t align, int nid)
  1253. {
  1254. return memblock_alloc_range_nid(size, align, 0,
  1255. MEMBLOCK_ALLOC_ACCESSIBLE, nid, false);
  1256. }
  1257. /**
  1258. * memblock_alloc_internal - allocate boot memory block
  1259. * @size: size of memory block to be allocated in bytes
  1260. * @align: alignment of the region and block's size
  1261. * @min_addr: the lower bound of the memory region to allocate (phys address)
  1262. * @max_addr: the upper bound of the memory region to allocate (phys address)
  1263. * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
  1264. * @exact_nid: control the allocation fall back to other nodes
  1265. *
  1266. * Allocates memory block using memblock_alloc_range_nid() and
  1267. * converts the returned physical address to virtual.
  1268. *
  1269. * The @min_addr limit is dropped if it can not be satisfied and the allocation
  1270. * will fall back to memory below @min_addr. Other constraints, such
  1271. * as node and mirrored memory will be handled again in
  1272. * memblock_alloc_range_nid().
  1273. *
  1274. * Return:
  1275. * Virtual address of allocated memory block on success, NULL on failure.
  1276. */
  1277. static void * __init memblock_alloc_internal(
  1278. phys_addr_t size, phys_addr_t align,
  1279. phys_addr_t min_addr, phys_addr_t max_addr,
  1280. int nid, bool exact_nid)
  1281. {
  1282. phys_addr_t alloc;
  1283. /*
  1284. * Detect any accidental use of these APIs after slab is ready, as at
  1285. * this moment memblock may be deinitialized already and its
  1286. * internal data may be destroyed (after execution of memblock_free_all)
  1287. */
  1288. if (WARN_ON_ONCE(slab_is_available()))
  1289. return kzalloc_node(size, GFP_NOWAIT, nid);
  1290. if (max_addr > memblock.current_limit)
  1291. max_addr = memblock.current_limit;
  1292. alloc = memblock_alloc_range_nid(size, align, min_addr, max_addr, nid,
  1293. exact_nid);
  1294. /* retry allocation without lower limit */
  1295. if (!alloc && min_addr)
  1296. alloc = memblock_alloc_range_nid(size, align, 0, max_addr, nid,
  1297. exact_nid);
  1298. if (!alloc)
  1299. return NULL;
  1300. return phys_to_virt(alloc);
  1301. }
  1302. /**
  1303. * memblock_alloc_exact_nid_raw - allocate boot memory block on the exact node
  1304. * without zeroing memory
  1305. * @size: size of memory block to be allocated in bytes
  1306. * @align: alignment of the region and block's size
  1307. * @min_addr: the lower bound of the memory region from where the allocation
  1308. * is preferred (phys address)
  1309. * @max_addr: the upper bound of the memory region from where the allocation
  1310. * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to
  1311. * allocate only from memory limited by memblock.current_limit value
  1312. * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
  1313. *
  1314. * Public function, provides additional debug information (including caller
  1315. * info), if enabled. Does not zero allocated memory.
  1316. *
  1317. * Return:
  1318. * Virtual address of allocated memory block on success, NULL on failure.
  1319. */
  1320. void * __init memblock_alloc_exact_nid_raw(
  1321. phys_addr_t size, phys_addr_t align,
  1322. phys_addr_t min_addr, phys_addr_t max_addr,
  1323. int nid)
  1324. {
  1325. void *ptr;
  1326. memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n",
  1327. __func__, (u64)size, (u64)align, nid, &min_addr,
  1328. &max_addr, (void *)_RET_IP_);
  1329. ptr = memblock_alloc_internal(size, align,
  1330. min_addr, max_addr, nid, true);
  1331. if (ptr && size > 0)
  1332. page_init_poison(ptr, size);
  1333. return ptr;
  1334. }
  1335. /**
  1336. * memblock_alloc_try_nid_raw - allocate boot memory block without zeroing
  1337. * memory and without panicking
  1338. * @size: size of memory block to be allocated in bytes
  1339. * @align: alignment of the region and block's size
  1340. * @min_addr: the lower bound of the memory region from where the allocation
  1341. * is preferred (phys address)
  1342. * @max_addr: the upper bound of the memory region from where the allocation
  1343. * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to
  1344. * allocate only from memory limited by memblock.current_limit value
  1345. * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
  1346. *
  1347. * Public function, provides additional debug information (including caller
  1348. * info), if enabled. Does not zero allocated memory, does not panic if request
  1349. * cannot be satisfied.
  1350. *
  1351. * Return:
  1352. * Virtual address of allocated memory block on success, NULL on failure.
  1353. */
  1354. void * __init memblock_alloc_try_nid_raw(
  1355. phys_addr_t size, phys_addr_t align,
  1356. phys_addr_t min_addr, phys_addr_t max_addr,
  1357. int nid)
  1358. {
  1359. void *ptr;
  1360. memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n",
  1361. __func__, (u64)size, (u64)align, nid, &min_addr,
  1362. &max_addr, (void *)_RET_IP_);
  1363. ptr = memblock_alloc_internal(size, align,
  1364. min_addr, max_addr, nid, false);
  1365. if (ptr && size > 0)
  1366. page_init_poison(ptr, size);
  1367. return ptr;
  1368. }
  1369. /**
  1370. * memblock_alloc_try_nid - allocate boot memory block
  1371. * @size: size of memory block to be allocated in bytes
  1372. * @align: alignment of the region and block's size
  1373. * @min_addr: the lower bound of the memory region from where the allocation
  1374. * is preferred (phys address)
  1375. * @max_addr: the upper bound of the memory region from where the allocation
  1376. * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to
  1377. * allocate only from memory limited by memblock.current_limit value
  1378. * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
  1379. *
  1380. * Public function, provides additional debug information (including caller
  1381. * info), if enabled. This function zeroes the allocated memory.
  1382. *
  1383. * Return:
  1384. * Virtual address of allocated memory block on success, NULL on failure.
  1385. */
  1386. void * __init memblock_alloc_try_nid(
  1387. phys_addr_t size, phys_addr_t align,
  1388. phys_addr_t min_addr, phys_addr_t max_addr,
  1389. int nid)
  1390. {
  1391. void *ptr;
  1392. memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n",
  1393. __func__, (u64)size, (u64)align, nid, &min_addr,
  1394. &max_addr, (void *)_RET_IP_);
  1395. ptr = memblock_alloc_internal(size, align,
  1396. min_addr, max_addr, nid, false);
  1397. if (ptr)
  1398. memset(ptr, 0, size);
  1399. return ptr;
  1400. }
  1401. /**
  1402. * __memblock_free_late - free pages directly to buddy allocator
  1403. * @base: phys starting address of the boot memory block
  1404. * @size: size of the boot memory block in bytes
  1405. *
  1406. * This is only useful when the memblock allocator has already been torn
  1407. * down, but we are still initializing the system. Pages are released directly
  1408. * to the buddy allocator.
  1409. */
  1410. void __init __memblock_free_late(phys_addr_t base, phys_addr_t size)
  1411. {
  1412. phys_addr_t cursor, end;
  1413. end = base + size - 1;
  1414. memblock_dbg("%s: [%pa-%pa] %pS\n",
  1415. __func__, &base, &end, (void *)_RET_IP_);
  1416. kmemleak_free_part_phys(base, size);
  1417. cursor = PFN_UP(base);
  1418. end = PFN_DOWN(base + size);
  1419. for (; cursor < end; cursor++) {
  1420. memblock_free_pages(pfn_to_page(cursor), cursor, 0);
  1421. totalram_pages_inc();
  1422. }
  1423. }
  1424. /*
  1425. * Remaining API functions
  1426. */
  1427. phys_addr_t __init_memblock memblock_phys_mem_size(void)
  1428. {
  1429. return memblock.memory.total_size;
  1430. }
  1431. phys_addr_t __init_memblock memblock_reserved_size(void)
  1432. {
  1433. return memblock.reserved.total_size;
  1434. }
  1435. /* lowest address */
  1436. phys_addr_t __init_memblock memblock_start_of_DRAM(void)
  1437. {
  1438. return memblock.memory.regions[0].base;
  1439. }
  1440. phys_addr_t __init_memblock memblock_end_of_DRAM(void)
  1441. {
  1442. int idx = memblock.memory.cnt - 1;
  1443. return (memblock.memory.regions[idx].base + memblock.memory.regions[idx].size);
  1444. }
  1445. EXPORT_SYMBOL_GPL(memblock_end_of_DRAM);
  1446. static phys_addr_t __init_memblock __find_max_addr(phys_addr_t limit)
  1447. {
  1448. phys_addr_t max_addr = PHYS_ADDR_MAX;
  1449. struct memblock_region *r;
  1450. /*
  1451. * translate the memory @limit size into the max address within one of
  1452. * the memory memblock regions, if the @limit exceeds the total size
  1453. * of those regions, max_addr will keep original value PHYS_ADDR_MAX
  1454. */
  1455. for_each_mem_region(r) {
  1456. if (limit <= r->size) {
  1457. max_addr = r->base + limit;
  1458. break;
  1459. }
  1460. limit -= r->size;
  1461. }
  1462. return max_addr;
  1463. }
  1464. void __init memblock_enforce_memory_limit(phys_addr_t limit)
  1465. {
  1466. phys_addr_t max_addr;
  1467. if (!limit)
  1468. return;
  1469. max_addr = __find_max_addr(limit);
  1470. /* @limit exceeds the total size of the memory, do nothing */
  1471. if (max_addr == PHYS_ADDR_MAX)
  1472. return;
  1473. /* truncate both memory and reserved regions */
  1474. memblock_remove_range(&memblock.memory, max_addr,
  1475. PHYS_ADDR_MAX);
  1476. memblock_remove_range(&memblock.reserved, max_addr,
  1477. PHYS_ADDR_MAX);
  1478. }
  1479. void __init memblock_cap_memory_range(phys_addr_t base, phys_addr_t size)
  1480. {
  1481. int start_rgn, end_rgn;
  1482. int i, ret;
  1483. if (!size)
  1484. return;
  1485. ret = memblock_isolate_range(&memblock.memory, base, size,
  1486. &start_rgn, &end_rgn);
  1487. if (ret)
  1488. return;
  1489. /* remove all the MAP regions */
  1490. for (i = memblock.memory.cnt - 1; i >= end_rgn; i--)
  1491. if (!memblock_is_nomap(&memblock.memory.regions[i]))
  1492. memblock_remove_region(&memblock.memory, i);
  1493. for (i = start_rgn - 1; i >= 0; i--)
  1494. if (!memblock_is_nomap(&memblock.memory.regions[i]))
  1495. memblock_remove_region(&memblock.memory, i);
  1496. /* truncate the reserved regions */
  1497. memblock_remove_range(&memblock.reserved, 0, base);
  1498. memblock_remove_range(&memblock.reserved,
  1499. base + size, PHYS_ADDR_MAX);
  1500. }
  1501. void __init memblock_mem_limit_remove_map(phys_addr_t limit)
  1502. {
  1503. phys_addr_t max_addr;
  1504. if (!limit)
  1505. return;
  1506. max_addr = __find_max_addr(limit);
  1507. /* @limit exceeds the total size of the memory, do nothing */
  1508. if (max_addr == PHYS_ADDR_MAX)
  1509. return;
  1510. memblock_cap_memory_range(0, max_addr);
  1511. }
  1512. static int __init_memblock memblock_search(struct memblock_type *type, phys_addr_t addr)
  1513. {
  1514. unsigned int left = 0, right = type->cnt;
  1515. do {
  1516. unsigned int mid = (right + left) / 2;
  1517. if (addr < type->regions[mid].base)
  1518. right = mid;
  1519. else if (addr >= (type->regions[mid].base +
  1520. type->regions[mid].size))
  1521. left = mid + 1;
  1522. else
  1523. return mid;
  1524. } while (left < right);
  1525. return -1;
  1526. }
  1527. bool __init_memblock memblock_is_reserved(phys_addr_t addr)
  1528. {
  1529. return memblock_search(&memblock.reserved, addr) != -1;
  1530. }
  1531. bool __init_memblock memblock_is_memory(phys_addr_t addr)
  1532. {
  1533. return memblock_search(&memblock.memory, addr) != -1;
  1534. }
  1535. bool __init_memblock memblock_is_map_memory(phys_addr_t addr)
  1536. {
  1537. int i = memblock_search(&memblock.memory, addr);
  1538. if (i == -1)
  1539. return false;
  1540. return !memblock_is_nomap(&memblock.memory.regions[i]);
  1541. }
  1542. int __init_memblock memblock_search_pfn_nid(unsigned long pfn,
  1543. unsigned long *start_pfn, unsigned long *end_pfn)
  1544. {
  1545. struct memblock_type *type = &memblock.memory;
  1546. int mid = memblock_search(type, PFN_PHYS(pfn));
  1547. if (mid == -1)
  1548. return -1;
  1549. *start_pfn = PFN_DOWN(type->regions[mid].base);
  1550. *end_pfn = PFN_DOWN(type->regions[mid].base + type->regions[mid].size);
  1551. return memblock_get_region_node(&type->regions[mid]);
  1552. }
  1553. /**
  1554. * memblock_is_region_memory - check if a region is a subset of memory
  1555. * @base: base of region to check
  1556. * @size: size of region to check
  1557. *
  1558. * Check if the region [@base, @base + @size) is a subset of a memory block.
  1559. *
  1560. * Return:
  1561. * 0 if false, non-zero if true
  1562. */
  1563. bool __init_memblock memblock_is_region_memory(phys_addr_t base, phys_addr_t size)
  1564. {
  1565. int idx = memblock_search(&memblock.memory, base);
  1566. phys_addr_t end = base + memblock_cap_size(base, &size);
  1567. if (idx == -1)
  1568. return false;
  1569. return (memblock.memory.regions[idx].base +
  1570. memblock.memory.regions[idx].size) >= end;
  1571. }
  1572. /**
  1573. * memblock_is_region_reserved - check if a region intersects reserved memory
  1574. * @base: base of region to check
  1575. * @size: size of region to check
  1576. *
  1577. * Check if the region [@base, @base + @size) intersects a reserved
  1578. * memory block.
  1579. *
  1580. * Return:
  1581. * True if they intersect, false if not.
  1582. */
  1583. bool __init_memblock memblock_is_region_reserved(phys_addr_t base, phys_addr_t size)
  1584. {
  1585. return memblock_overlaps_region(&memblock.reserved, base, size);
  1586. }
  1587. void __init_memblock memblock_trim_memory(phys_addr_t align)
  1588. {
  1589. phys_addr_t start, end, orig_start, orig_end;
  1590. struct memblock_region *r;
  1591. for_each_mem_region(r) {
  1592. orig_start = r->base;
  1593. orig_end = r->base + r->size;
  1594. start = round_up(orig_start, align);
  1595. end = round_down(orig_end, align);
  1596. if (start == orig_start && end == orig_end)
  1597. continue;
  1598. if (start < end) {
  1599. r->base = start;
  1600. r->size = end - start;
  1601. } else {
  1602. memblock_remove_region(&memblock.memory,
  1603. r - memblock.memory.regions);
  1604. r--;
  1605. }
  1606. }
  1607. }
  1608. void __init_memblock memblock_set_current_limit(phys_addr_t limit)
  1609. {
  1610. memblock.current_limit = limit;
  1611. }
  1612. phys_addr_t __init_memblock memblock_get_current_limit(void)
  1613. {
  1614. return memblock.current_limit;
  1615. }
  1616. static void __init_memblock memblock_dump(struct memblock_type *type)
  1617. {
  1618. phys_addr_t base, end, size;
  1619. enum memblock_flags flags;
  1620. int idx;
  1621. struct memblock_region *rgn;
  1622. pr_info(" %s.cnt = 0x%lx\n", type->name, type->cnt);
  1623. for_each_memblock_type(idx, type, rgn) {
  1624. char nid_buf[32] = "";
  1625. base = rgn->base;
  1626. size = rgn->size;
  1627. end = base + size - 1;
  1628. flags = rgn->flags;
  1629. #ifdef CONFIG_NEED_MULTIPLE_NODES
  1630. if (memblock_get_region_node(rgn) != MAX_NUMNODES)
  1631. snprintf(nid_buf, sizeof(nid_buf), " on node %d",
  1632. memblock_get_region_node(rgn));
  1633. #endif
  1634. pr_info(" %s[%#x]\t[%pa-%pa], %pa bytes%s flags: %#x\n",
  1635. type->name, idx, &base, &end, &size, nid_buf, flags);
  1636. }
  1637. }
  1638. static void __init_memblock __memblock_dump_all(void)
  1639. {
  1640. pr_info("MEMBLOCK configuration:\n");
  1641. pr_info(" memory size = %pa reserved size = %pa\n",
  1642. &memblock.memory.total_size,
  1643. &memblock.reserved.total_size);
  1644. memblock_dump(&memblock.memory);
  1645. memblock_dump(&memblock.reserved);
  1646. #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
  1647. memblock_dump(&physmem);
  1648. #endif
  1649. }
  1650. void __init_memblock memblock_dump_all(void)
  1651. {
  1652. if (memblock_debug)
  1653. __memblock_dump_all();
  1654. }
  1655. void __init memblock_allow_resize(void)
  1656. {
  1657. memblock_can_resize = 1;
  1658. }
  1659. static int __init early_memblock(char *p)
  1660. {
  1661. if (p && strstr(p, "debug"))
  1662. memblock_debug = 1;
  1663. return 0;
  1664. }
  1665. early_param("memblock", early_memblock);
  1666. static void __init __free_pages_memory(unsigned long start, unsigned long end)
  1667. {
  1668. int order;
  1669. while (start < end) {
  1670. order = min(MAX_ORDER - 1UL, __ffs(start));
  1671. while (start + (1UL << order) > end)
  1672. order--;
  1673. memblock_free_pages(pfn_to_page(start), start, order);
  1674. start += (1UL << order);
  1675. }
  1676. }
  1677. static unsigned long __init __free_memory_core(phys_addr_t start,
  1678. phys_addr_t end)
  1679. {
  1680. unsigned long start_pfn = PFN_UP(start);
  1681. unsigned long end_pfn = min_t(unsigned long,
  1682. PFN_DOWN(end), max_low_pfn);
  1683. if (start_pfn >= end_pfn)
  1684. return 0;
  1685. __free_pages_memory(start_pfn, end_pfn);
  1686. return end_pfn - start_pfn;
  1687. }
  1688. static unsigned long __init free_low_memory_core_early(void)
  1689. {
  1690. unsigned long count = 0;
  1691. phys_addr_t start, end;
  1692. u64 i;
  1693. memblock_clear_hotplug(0, -1);
  1694. for_each_reserved_mem_range(i, &start, &end)
  1695. reserve_bootmem_region(start, end);
  1696. /*
  1697. * We need to use NUMA_NO_NODE instead of NODE_DATA(0)->node_id
  1698. * because in some case like Node0 doesn't have RAM installed
  1699. * low ram will be on Node1
  1700. */
  1701. for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE, &start, &end,
  1702. NULL)
  1703. count += __free_memory_core(start, end);
  1704. return count;
  1705. }
  1706. static int reset_managed_pages_done __initdata;
  1707. void reset_node_managed_pages(pg_data_t *pgdat)
  1708. {
  1709. struct zone *z;
  1710. for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++)
  1711. atomic_long_set(&z->managed_pages, 0);
  1712. }
  1713. void __init reset_all_zones_managed_pages(void)
  1714. {
  1715. struct pglist_data *pgdat;
  1716. if (reset_managed_pages_done)
  1717. return;
  1718. for_each_online_pgdat(pgdat)
  1719. reset_node_managed_pages(pgdat);
  1720. reset_managed_pages_done = 1;
  1721. }
  1722. /**
  1723. * memblock_free_all - release free pages to the buddy allocator
  1724. *
  1725. * Return: the number of pages actually released.
  1726. */
  1727. unsigned long __init memblock_free_all(void)
  1728. {
  1729. unsigned long pages;
  1730. reset_all_zones_managed_pages();
  1731. pages = free_low_memory_core_early();
  1732. totalram_pages_add(pages);
  1733. return pages;
  1734. }
  1735. #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_ARCH_KEEP_MEMBLOCK)
  1736. static int memblock_debug_show(struct seq_file *m, void *private)
  1737. {
  1738. struct memblock_type *type = m->private;
  1739. struct memblock_region *reg;
  1740. int i;
  1741. phys_addr_t end;
  1742. for (i = 0; i < type->cnt; i++) {
  1743. reg = &type->regions[i];
  1744. end = reg->base + reg->size - 1;
  1745. seq_printf(m, "%4d: ", i);
  1746. seq_printf(m, "%pa..%pa\n", &reg->base, &end);
  1747. }
  1748. return 0;
  1749. }
  1750. DEFINE_SHOW_ATTRIBUTE(memblock_debug);
  1751. static int __init memblock_init_debugfs(void)
  1752. {
  1753. struct dentry *root = debugfs_create_dir("memblock", NULL);
  1754. debugfs_create_file("memory", 0444, root,
  1755. &memblock.memory, &memblock_debug_fops);
  1756. debugfs_create_file("reserved", 0444, root,
  1757. &memblock.reserved, &memblock_debug_fops);
  1758. #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
  1759. debugfs_create_file("physmem", 0444, root, &physmem,
  1760. &memblock_debug_fops);
  1761. #endif
  1762. return 0;
  1763. }
  1764. __initcall(memblock_init_debugfs);
  1765. #endif /* CONFIG_DEBUG_FS */