sparse-vmemmap.c 6.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Virtual Memory Map support
  4. *
  5. * (C) 2007 sgi. Christoph Lameter.
  6. *
  7. * Virtual memory maps allow VM primitives pfn_to_page, page_to_pfn,
  8. * virt_to_page, page_address() to be implemented as a base offset
  9. * calculation without memory access.
  10. *
  11. * However, virtual mappings need a page table and TLBs. Many Linux
  12. * architectures already map their physical space using 1-1 mappings
  13. * via TLBs. For those arches the virtual memory map is essentially
  14. * for free if we use the same page size as the 1-1 mappings. In that
  15. * case the overhead consists of a few additional pages that are
  16. * allocated to create a view of memory for vmemmap.
  17. *
  18. * The architecture is expected to provide a vmemmap_populate() function
  19. * to instantiate the mapping.
  20. */
  21. #include <linux/mm.h>
  22. #include <linux/mmzone.h>
  23. #include <linux/memblock.h>
  24. #include <linux/memremap.h>
  25. #include <linux/highmem.h>
  26. #include <linux/slab.h>
  27. #include <linux/spinlock.h>
  28. #include <linux/vmalloc.h>
  29. #include <linux/sched.h>
  30. #include <asm/dma.h>
  31. #include <asm/pgalloc.h>
  32. /*
  33. * Allocate a block of memory to be used to back the virtual memory map
  34. * or to back the page tables that are used to create the mapping.
  35. * Uses the main allocators if they are available, else bootmem.
  36. */
  37. static void * __ref __earlyonly_bootmem_alloc(int node,
  38. unsigned long size,
  39. unsigned long align,
  40. unsigned long goal)
  41. {
  42. return memblock_alloc_try_nid_raw(size, align, goal,
  43. MEMBLOCK_ALLOC_ACCESSIBLE, node);
  44. }
  45. void * __meminit vmemmap_alloc_block(unsigned long size, int node)
  46. {
  47. /* If the main allocator is up use that, fallback to bootmem. */
  48. if (slab_is_available()) {
  49. gfp_t gfp_mask = GFP_KERNEL|__GFP_RETRY_MAYFAIL|__GFP_NOWARN;
  50. int order = get_order(size);
  51. static bool warned;
  52. struct page *page;
  53. page = alloc_pages_node(node, gfp_mask, order);
  54. if (page)
  55. return page_address(page);
  56. if (!warned) {
  57. warn_alloc(gfp_mask & ~__GFP_NOWARN, NULL,
  58. "vmemmap alloc failure: order:%u", order);
  59. warned = true;
  60. }
  61. return NULL;
  62. } else
  63. return __earlyonly_bootmem_alloc(node, size, size,
  64. __pa(MAX_DMA_ADDRESS));
  65. }
  66. static void * __meminit altmap_alloc_block_buf(unsigned long size,
  67. struct vmem_altmap *altmap);
  68. /* need to make sure size is all the same during early stage */
  69. void * __meminit vmemmap_alloc_block_buf(unsigned long size, int node,
  70. struct vmem_altmap *altmap)
  71. {
  72. void *ptr;
  73. if (altmap)
  74. return altmap_alloc_block_buf(size, altmap);
  75. ptr = sparse_buffer_alloc(size);
  76. if (!ptr)
  77. ptr = vmemmap_alloc_block(size, node);
  78. return ptr;
  79. }
  80. static unsigned long __meminit vmem_altmap_next_pfn(struct vmem_altmap *altmap)
  81. {
  82. return altmap->base_pfn + altmap->reserve + altmap->alloc
  83. + altmap->align;
  84. }
  85. static unsigned long __meminit vmem_altmap_nr_free(struct vmem_altmap *altmap)
  86. {
  87. unsigned long allocated = altmap->alloc + altmap->align;
  88. if (altmap->free > allocated)
  89. return altmap->free - allocated;
  90. return 0;
  91. }
  92. static void * __meminit altmap_alloc_block_buf(unsigned long size,
  93. struct vmem_altmap *altmap)
  94. {
  95. unsigned long pfn, nr_pfns, nr_align;
  96. if (size & ~PAGE_MASK) {
  97. pr_warn_once("%s: allocations must be multiple of PAGE_SIZE (%ld)\n",
  98. __func__, size);
  99. return NULL;
  100. }
  101. pfn = vmem_altmap_next_pfn(altmap);
  102. nr_pfns = size >> PAGE_SHIFT;
  103. nr_align = 1UL << find_first_bit(&nr_pfns, BITS_PER_LONG);
  104. nr_align = ALIGN(pfn, nr_align) - pfn;
  105. if (nr_pfns + nr_align > vmem_altmap_nr_free(altmap))
  106. return NULL;
  107. altmap->alloc += nr_pfns;
  108. altmap->align += nr_align;
  109. pfn += nr_align;
  110. pr_debug("%s: pfn: %#lx alloc: %ld align: %ld nr: %#lx\n",
  111. __func__, pfn, altmap->alloc, altmap->align, nr_pfns);
  112. return __va(__pfn_to_phys(pfn));
  113. }
  114. void __meminit vmemmap_verify(pte_t *pte, int node,
  115. unsigned long start, unsigned long end)
  116. {
  117. unsigned long pfn = pte_pfn(*pte);
  118. int actual_node = early_pfn_to_nid(pfn);
  119. if (node_distance(actual_node, node) > LOCAL_DISTANCE)
  120. pr_warn("[%lx-%lx] potential offnode page_structs\n",
  121. start, end - 1);
  122. }
  123. pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node,
  124. struct vmem_altmap *altmap)
  125. {
  126. pte_t *pte = pte_offset_kernel(pmd, addr);
  127. if (pte_none(*pte)) {
  128. pte_t entry;
  129. void *p;
  130. p = vmemmap_alloc_block_buf(PAGE_SIZE, node, altmap);
  131. if (!p)
  132. return NULL;
  133. entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL);
  134. set_pte_at(&init_mm, addr, pte, entry);
  135. }
  136. return pte;
  137. }
  138. static void * __meminit vmemmap_alloc_block_zero(unsigned long size, int node)
  139. {
  140. void *p = vmemmap_alloc_block(size, node);
  141. if (!p)
  142. return NULL;
  143. memset(p, 0, size);
  144. return p;
  145. }
  146. pmd_t * __meminit vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node)
  147. {
  148. pmd_t *pmd = pmd_offset(pud, addr);
  149. if (pmd_none(*pmd)) {
  150. void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node);
  151. if (!p)
  152. return NULL;
  153. pmd_populate_kernel(&init_mm, pmd, p);
  154. }
  155. return pmd;
  156. }
  157. pud_t * __meminit vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node)
  158. {
  159. pud_t *pud = pud_offset(p4d, addr);
  160. if (pud_none(*pud)) {
  161. void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node);
  162. if (!p)
  163. return NULL;
  164. pud_populate(&init_mm, pud, p);
  165. }
  166. return pud;
  167. }
  168. p4d_t * __meminit vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node)
  169. {
  170. p4d_t *p4d = p4d_offset(pgd, addr);
  171. if (p4d_none(*p4d)) {
  172. void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node);
  173. if (!p)
  174. return NULL;
  175. p4d_populate(&init_mm, p4d, p);
  176. }
  177. return p4d;
  178. }
  179. pgd_t * __meminit vmemmap_pgd_populate(unsigned long addr, int node)
  180. {
  181. pgd_t *pgd = pgd_offset_k(addr);
  182. if (pgd_none(*pgd)) {
  183. void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node);
  184. if (!p)
  185. return NULL;
  186. pgd_populate(&init_mm, pgd, p);
  187. }
  188. return pgd;
  189. }
  190. int __meminit vmemmap_populate_basepages(unsigned long start, unsigned long end,
  191. int node, struct vmem_altmap *altmap)
  192. {
  193. unsigned long addr = start;
  194. pgd_t *pgd;
  195. p4d_t *p4d;
  196. pud_t *pud;
  197. pmd_t *pmd;
  198. pte_t *pte;
  199. for (; addr < end; addr += PAGE_SIZE) {
  200. pgd = vmemmap_pgd_populate(addr, node);
  201. if (!pgd)
  202. return -ENOMEM;
  203. p4d = vmemmap_p4d_populate(pgd, addr, node);
  204. if (!p4d)
  205. return -ENOMEM;
  206. pud = vmemmap_pud_populate(p4d, addr, node);
  207. if (!pud)
  208. return -ENOMEM;
  209. pmd = vmemmap_pmd_populate(pud, addr, node);
  210. if (!pmd)
  211. return -ENOMEM;
  212. pte = vmemmap_pte_populate(pmd, addr, node, altmap);
  213. if (!pte)
  214. return -ENOMEM;
  215. vmemmap_verify(pte, node, addr, addr + PAGE_SIZE);
  216. }
  217. return 0;
  218. }
  219. struct page * __meminit __populate_section_memmap(unsigned long pfn,
  220. unsigned long nr_pages, int nid, struct vmem_altmap *altmap)
  221. {
  222. unsigned long start = (unsigned long) pfn_to_page(pfn);
  223. unsigned long end = start + nr_pages * sizeof(struct page);
  224. if (WARN_ON_ONCE(!IS_ALIGNED(pfn, PAGES_PER_SUBSECTION) ||
  225. !IS_ALIGNED(nr_pages, PAGES_PER_SUBSECTION)))
  226. return NULL;
  227. if (vmemmap_populate(start, end, nid, altmap))
  228. return NULL;
  229. return pfn_to_page(pfn);
  230. }