sparse.c 8.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340
  1. /*
  2. * sparse memory mappings.
  3. */
  4. #include <linux/mm.h>
  5. #include <linux/mmzone.h>
  6. #include <linux/bootmem.h>
  7. #include <linux/highmem.h>
  8. #include <linux/module.h>
  9. #include <linux/spinlock.h>
  10. #include <linux/vmalloc.h>
  11. #include <asm/dma.h>
  12. /*
  13. * Permanent SPARSEMEM data:
  14. *
  15. * 1) mem_section - memory sections, mem_map's for valid memory
  16. */
  17. #ifdef CONFIG_SPARSEMEM_EXTREME
  18. struct mem_section *mem_section[NR_SECTION_ROOTS]
  19. ____cacheline_internodealigned_in_smp;
  20. #else
  21. struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT]
  22. ____cacheline_internodealigned_in_smp;
  23. #endif
  24. EXPORT_SYMBOL(mem_section);
  25. #ifdef NODE_NOT_IN_PAGE_FLAGS
  26. /*
  27. * If we did not store the node number in the page then we have to
  28. * do a lookup in the section_to_node_table in order to find which
  29. * node the page belongs to.
  30. */
  31. #if MAX_NUMNODES <= 256
  32. static u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned;
  33. #else
  34. static u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned;
  35. #endif
  36. int page_to_nid(struct page *page)
  37. {
  38. return section_to_node_table[page_to_section(page)];
  39. }
  40. EXPORT_SYMBOL(page_to_nid);
  41. #endif
  42. #ifdef CONFIG_SPARSEMEM_EXTREME
  43. static struct mem_section *sparse_index_alloc(int nid)
  44. {
  45. struct mem_section *section = NULL;
  46. unsigned long array_size = SECTIONS_PER_ROOT *
  47. sizeof(struct mem_section);
  48. if (slab_is_available())
  49. section = kmalloc_node(array_size, GFP_KERNEL, nid);
  50. else
  51. section = alloc_bootmem_node(NODE_DATA(nid), array_size);
  52. if (section)
  53. memset(section, 0, array_size);
  54. return section;
  55. }
  56. static int sparse_index_init(unsigned long section_nr, int nid)
  57. {
  58. static DEFINE_SPINLOCK(index_init_lock);
  59. unsigned long root = SECTION_NR_TO_ROOT(section_nr);
  60. struct mem_section *section;
  61. int ret = 0;
  62. #ifdef NODE_NOT_IN_PAGE_FLAGS
  63. section_to_node_table[section_nr] = nid;
  64. #endif
  65. if (mem_section[root])
  66. return -EEXIST;
  67. section = sparse_index_alloc(nid);
  68. /*
  69. * This lock keeps two different sections from
  70. * reallocating for the same index
  71. */
  72. spin_lock(&index_init_lock);
  73. if (mem_section[root]) {
  74. ret = -EEXIST;
  75. goto out;
  76. }
  77. mem_section[root] = section;
  78. out:
  79. spin_unlock(&index_init_lock);
  80. return ret;
  81. }
  82. #else /* !SPARSEMEM_EXTREME */
  83. static inline int sparse_index_init(unsigned long section_nr, int nid)
  84. {
  85. return 0;
  86. }
  87. #endif
  88. /*
  89. * Although written for the SPARSEMEM_EXTREME case, this happens
  90. * to also work for the flat array case becase
  91. * NR_SECTION_ROOTS==NR_MEM_SECTIONS.
  92. */
  93. int __section_nr(struct mem_section* ms)
  94. {
  95. unsigned long root_nr;
  96. struct mem_section* root;
  97. for (root_nr = 0; root_nr < NR_SECTION_ROOTS; root_nr++) {
  98. root = __nr_to_section(root_nr * SECTIONS_PER_ROOT);
  99. if (!root)
  100. continue;
  101. if ((ms >= root) && (ms < (root + SECTIONS_PER_ROOT)))
  102. break;
  103. }
  104. return (root_nr * SECTIONS_PER_ROOT) + (ms - root);
  105. }
  106. /*
  107. * During early boot, before section_mem_map is used for an actual
  108. * mem_map, we use section_mem_map to store the section's NUMA
  109. * node. This keeps us from having to use another data structure. The
  110. * node information is cleared just before we store the real mem_map.
  111. */
  112. static inline unsigned long sparse_encode_early_nid(int nid)
  113. {
  114. return (nid << SECTION_NID_SHIFT);
  115. }
  116. static inline int sparse_early_nid(struct mem_section *section)
  117. {
  118. return (section->section_mem_map >> SECTION_NID_SHIFT);
  119. }
  120. /* Record a memory area against a node. */
  121. void memory_present(int nid, unsigned long start, unsigned long end)
  122. {
  123. unsigned long pfn;
  124. start &= PAGE_SECTION_MASK;
  125. for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) {
  126. unsigned long section = pfn_to_section_nr(pfn);
  127. struct mem_section *ms;
  128. sparse_index_init(section, nid);
  129. ms = __nr_to_section(section);
  130. if (!ms->section_mem_map)
  131. ms->section_mem_map = sparse_encode_early_nid(nid) |
  132. SECTION_MARKED_PRESENT;
  133. }
  134. }
  135. /*
  136. * Only used by the i386 NUMA architecures, but relatively
  137. * generic code.
  138. */
  139. unsigned long __init node_memmap_size_bytes(int nid, unsigned long start_pfn,
  140. unsigned long end_pfn)
  141. {
  142. unsigned long pfn;
  143. unsigned long nr_pages = 0;
  144. for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
  145. if (nid != early_pfn_to_nid(pfn))
  146. continue;
  147. if (pfn_valid(pfn))
  148. nr_pages += PAGES_PER_SECTION;
  149. }
  150. return nr_pages * sizeof(struct page);
  151. }
  152. /*
  153. * Subtle, we encode the real pfn into the mem_map such that
  154. * the identity pfn - section_mem_map will return the actual
  155. * physical page frame number.
  156. */
  157. static unsigned long sparse_encode_mem_map(struct page *mem_map, unsigned long pnum)
  158. {
  159. return (unsigned long)(mem_map - (section_nr_to_pfn(pnum)));
  160. }
  161. /*
  162. * We need this if we ever free the mem_maps. While not implemented yet,
  163. * this function is included for parity with its sibling.
  164. */
  165. static __attribute((unused))
  166. struct page *sparse_decode_mem_map(unsigned long coded_mem_map, unsigned long pnum)
  167. {
  168. return ((struct page *)coded_mem_map) + section_nr_to_pfn(pnum);
  169. }
  170. static int sparse_init_one_section(struct mem_section *ms,
  171. unsigned long pnum, struct page *mem_map)
  172. {
  173. if (!valid_section(ms))
  174. return -EINVAL;
  175. ms->section_mem_map &= ~SECTION_MAP_MASK;
  176. ms->section_mem_map |= sparse_encode_mem_map(mem_map, pnum);
  177. return 1;
  178. }
  179. __attribute__((weak))
  180. void *alloc_bootmem_high_node(pg_data_t *pgdat, unsigned long size)
  181. {
  182. return NULL;
  183. }
  184. static struct page *sparse_early_mem_map_alloc(unsigned long pnum)
  185. {
  186. struct page *map;
  187. struct mem_section *ms = __nr_to_section(pnum);
  188. int nid = sparse_early_nid(ms);
  189. map = alloc_remap(nid, sizeof(struct page) * PAGES_PER_SECTION);
  190. if (map)
  191. return map;
  192. map = alloc_bootmem_high_node(NODE_DATA(nid),
  193. sizeof(struct page) * PAGES_PER_SECTION);
  194. if (map)
  195. return map;
  196. map = alloc_bootmem_node(NODE_DATA(nid),
  197. sizeof(struct page) * PAGES_PER_SECTION);
  198. if (map)
  199. return map;
  200. printk(KERN_WARNING "%s: allocation failed\n", __FUNCTION__);
  201. ms->section_mem_map = 0;
  202. return NULL;
  203. }
  204. static struct page *__kmalloc_section_memmap(unsigned long nr_pages)
  205. {
  206. struct page *page, *ret;
  207. unsigned long memmap_size = sizeof(struct page) * nr_pages;
  208. page = alloc_pages(GFP_KERNEL|__GFP_NOWARN, get_order(memmap_size));
  209. if (page)
  210. goto got_map_page;
  211. ret = vmalloc(memmap_size);
  212. if (ret)
  213. goto got_map_ptr;
  214. return NULL;
  215. got_map_page:
  216. ret = (struct page *)pfn_to_kaddr(page_to_pfn(page));
  217. got_map_ptr:
  218. memset(ret, 0, memmap_size);
  219. return ret;
  220. }
  221. static int vaddr_in_vmalloc_area(void *addr)
  222. {
  223. if (addr >= (void *)VMALLOC_START &&
  224. addr < (void *)VMALLOC_END)
  225. return 1;
  226. return 0;
  227. }
  228. static void __kfree_section_memmap(struct page *memmap, unsigned long nr_pages)
  229. {
  230. if (vaddr_in_vmalloc_area(memmap))
  231. vfree(memmap);
  232. else
  233. free_pages((unsigned long)memmap,
  234. get_order(sizeof(struct page) * nr_pages));
  235. }
  236. /*
  237. * Allocate the accumulated non-linear sections, allocate a mem_map
  238. * for each and record the physical to section mapping.
  239. */
  240. void sparse_init(void)
  241. {
  242. unsigned long pnum;
  243. struct page *map;
  244. for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) {
  245. if (!valid_section_nr(pnum))
  246. continue;
  247. map = sparse_early_mem_map_alloc(pnum);
  248. if (!map)
  249. continue;
  250. sparse_init_one_section(__nr_to_section(pnum), pnum, map);
  251. }
  252. }
  253. /*
  254. * returns the number of sections whose mem_maps were properly
  255. * set. If this is <=0, then that means that the passed-in
  256. * map was not consumed and must be freed.
  257. */
  258. int sparse_add_one_section(struct zone *zone, unsigned long start_pfn,
  259. int nr_pages)
  260. {
  261. unsigned long section_nr = pfn_to_section_nr(start_pfn);
  262. struct pglist_data *pgdat = zone->zone_pgdat;
  263. struct mem_section *ms;
  264. struct page *memmap;
  265. unsigned long flags;
  266. int ret;
  267. /*
  268. * no locking for this, because it does its own
  269. * plus, it does a kmalloc
  270. */
  271. sparse_index_init(section_nr, pgdat->node_id);
  272. memmap = __kmalloc_section_memmap(nr_pages);
  273. pgdat_resize_lock(pgdat, &flags);
  274. ms = __pfn_to_section(start_pfn);
  275. if (ms->section_mem_map & SECTION_MARKED_PRESENT) {
  276. ret = -EEXIST;
  277. goto out;
  278. }
  279. ms->section_mem_map |= SECTION_MARKED_PRESENT;
  280. ret = sparse_init_one_section(ms, section_nr, memmap);
  281. out:
  282. pgdat_resize_unlock(pgdat, &flags);
  283. if (ret <= 0)
  284. __kfree_section_memmap(memmap, nr_pages);
  285. return ret;
  286. }