highmem.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * High memory handling common code and variables.
  4. *
  5. * (C) 1999 Andrea Arcangeli, SuSE GmbH, andrea@suse.de
  6. * Gerhard Wichert, Siemens AG, Gerhard.Wichert@pdb.siemens.de
  7. *
  8. *
  9. * Redesigned the x86 32-bit VM architecture to deal with
  10. * 64-bit physical space. With current x86 CPUs this
  11. * means up to 64 Gigabytes physical RAM.
  12. *
  13. * Rewrote high memory support to move the page cache into
  14. * high memory. Implemented permanent (schedulable) kmaps
  15. * based on Linus' idea.
  16. *
  17. * Copyright (C) 1999 Ingo Molnar <mingo@redhat.com>
  18. */
  19. #include <linux/mm.h>
  20. #include <linux/export.h>
  21. #include <linux/swap.h>
  22. #include <linux/bio.h>
  23. #include <linux/pagemap.h>
  24. #include <linux/mempool.h>
  25. #include <linux/blkdev.h>
  26. #include <linux/init.h>
  27. #include <linux/hash.h>
  28. #include <linux/highmem.h>
  29. #include <linux/kgdb.h>
  30. #include <asm/tlbflush.h>
  31. #include <linux/vmalloc.h>
  32. #if defined(CONFIG_HIGHMEM) || defined(CONFIG_X86_32)
  33. DEFINE_PER_CPU(int, __kmap_atomic_idx);
  34. #endif
  35. /*
  36. * Virtual_count is not a pure "count".
  37. * 0 means that it is not mapped, and has not been mapped
  38. * since a TLB flush - it is usable.
  39. * 1 means that there are no users, but it has been mapped
  40. * since the last TLB flush - so we can't use it.
  41. * n means that there are (n-1) current users of it.
  42. */
  43. #ifdef CONFIG_HIGHMEM
  44. /*
  45. * Architecture with aliasing data cache may define the following family of
  46. * helper functions in its asm/highmem.h to control cache color of virtual
  47. * addresses where physical memory pages are mapped by kmap.
  48. */
  49. #ifndef get_pkmap_color
  50. /*
  51. * Determine color of virtual address where the page should be mapped.
  52. */
  53. static inline unsigned int get_pkmap_color(struct page *page)
  54. {
  55. return 0;
  56. }
  57. #define get_pkmap_color get_pkmap_color
  58. /*
  59. * Get next index for mapping inside PKMAP region for page with given color.
  60. */
  61. static inline unsigned int get_next_pkmap_nr(unsigned int color)
  62. {
  63. static unsigned int last_pkmap_nr;
  64. last_pkmap_nr = (last_pkmap_nr + 1) & LAST_PKMAP_MASK;
  65. return last_pkmap_nr;
  66. }
  67. /*
  68. * Determine if page index inside PKMAP region (pkmap_nr) of given color
  69. * has wrapped around PKMAP region end. When this happens an attempt to
  70. * flush all unused PKMAP slots is made.
  71. */
  72. static inline int no_more_pkmaps(unsigned int pkmap_nr, unsigned int color)
  73. {
  74. return pkmap_nr == 0;
  75. }
  76. /*
  77. * Get the number of PKMAP entries of the given color. If no free slot is
  78. * found after checking that many entries, kmap will sleep waiting for
  79. * someone to call kunmap and free PKMAP slot.
  80. */
  81. static inline int get_pkmap_entries_count(unsigned int color)
  82. {
  83. return LAST_PKMAP;
  84. }
  85. /*
  86. * Get head of a wait queue for PKMAP entries of the given color.
  87. * Wait queues for different mapping colors should be independent to avoid
  88. * unnecessary wakeups caused by freeing of slots of other colors.
  89. */
  90. static inline wait_queue_head_t *get_pkmap_wait_queue_head(unsigned int color)
  91. {
  92. static DECLARE_WAIT_QUEUE_HEAD(pkmap_map_wait);
  93. return &pkmap_map_wait;
  94. }
  95. #endif
  96. atomic_long_t _totalhigh_pages __read_mostly;
  97. EXPORT_SYMBOL(_totalhigh_pages);
  98. EXPORT_PER_CPU_SYMBOL(__kmap_atomic_idx);
  99. unsigned int nr_free_highpages (void)
  100. {
  101. struct zone *zone;
  102. unsigned int pages = 0;
  103. for_each_populated_zone(zone) {
  104. if (is_highmem(zone))
  105. pages += zone_page_state(zone, NR_FREE_PAGES);
  106. }
  107. return pages;
  108. }
  109. static int pkmap_count[LAST_PKMAP];
  110. static __cacheline_aligned_in_smp DEFINE_SPINLOCK(kmap_lock);
  111. pte_t * pkmap_page_table;
  112. /*
  113. * Most architectures have no use for kmap_high_get(), so let's abstract
  114. * the disabling of IRQ out of the locking in that case to save on a
  115. * potential useless overhead.
  116. */
  117. #ifdef ARCH_NEEDS_KMAP_HIGH_GET
  118. #define lock_kmap() spin_lock_irq(&kmap_lock)
  119. #define unlock_kmap() spin_unlock_irq(&kmap_lock)
  120. #define lock_kmap_any(flags) spin_lock_irqsave(&kmap_lock, flags)
  121. #define unlock_kmap_any(flags) spin_unlock_irqrestore(&kmap_lock, flags)
  122. #else
  123. #define lock_kmap() spin_lock(&kmap_lock)
  124. #define unlock_kmap() spin_unlock(&kmap_lock)
  125. #define lock_kmap_any(flags) \
  126. do { spin_lock(&kmap_lock); (void)(flags); } while (0)
  127. #define unlock_kmap_any(flags) \
  128. do { spin_unlock(&kmap_lock); (void)(flags); } while (0)
  129. #endif
  130. struct page *kmap_to_page(void *vaddr)
  131. {
  132. unsigned long addr = (unsigned long)vaddr;
  133. if (addr >= PKMAP_ADDR(0) && addr < PKMAP_ADDR(LAST_PKMAP)) {
  134. int i = PKMAP_NR(addr);
  135. return pte_page(pkmap_page_table[i]);
  136. }
  137. return virt_to_page(addr);
  138. }
  139. EXPORT_SYMBOL(kmap_to_page);
  140. static void flush_all_zero_pkmaps(void)
  141. {
  142. int i;
  143. int need_flush = 0;
  144. flush_cache_kmaps();
  145. for (i = 0; i < LAST_PKMAP; i++) {
  146. struct page *page;
  147. /*
  148. * zero means we don't have anything to do,
  149. * >1 means that it is still in use. Only
  150. * a count of 1 means that it is free but
  151. * needs to be unmapped
  152. */
  153. if (pkmap_count[i] != 1)
  154. continue;
  155. pkmap_count[i] = 0;
  156. /* sanity check */
  157. BUG_ON(pte_none(pkmap_page_table[i]));
  158. /*
  159. * Don't need an atomic fetch-and-clear op here;
  160. * no-one has the page mapped, and cannot get at
  161. * its virtual address (and hence PTE) without first
  162. * getting the kmap_lock (which is held here).
  163. * So no dangers, even with speculative execution.
  164. */
  165. page = pte_page(pkmap_page_table[i]);
  166. pte_clear(&init_mm, PKMAP_ADDR(i), &pkmap_page_table[i]);
  167. set_page_address(page, NULL);
  168. need_flush = 1;
  169. }
  170. if (need_flush)
  171. flush_tlb_kernel_range(PKMAP_ADDR(0), PKMAP_ADDR(LAST_PKMAP));
  172. }
  173. /**
  174. * kmap_flush_unused - flush all unused kmap mappings in order to remove stray mappings
  175. */
  176. void kmap_flush_unused(void)
  177. {
  178. lock_kmap();
  179. flush_all_zero_pkmaps();
  180. unlock_kmap();
  181. }
  182. static inline unsigned long map_new_virtual(struct page *page)
  183. {
  184. unsigned long vaddr;
  185. int count;
  186. unsigned int last_pkmap_nr;
  187. unsigned int color = get_pkmap_color(page);
  188. start:
  189. count = get_pkmap_entries_count(color);
  190. /* Find an empty entry */
  191. for (;;) {
  192. last_pkmap_nr = get_next_pkmap_nr(color);
  193. if (no_more_pkmaps(last_pkmap_nr, color)) {
  194. flush_all_zero_pkmaps();
  195. count = get_pkmap_entries_count(color);
  196. }
  197. if (!pkmap_count[last_pkmap_nr])
  198. break; /* Found a usable entry */
  199. if (--count)
  200. continue;
  201. /*
  202. * Sleep for somebody else to unmap their entries
  203. */
  204. {
  205. DECLARE_WAITQUEUE(wait, current);
  206. wait_queue_head_t *pkmap_map_wait =
  207. get_pkmap_wait_queue_head(color);
  208. __set_current_state(TASK_UNINTERRUPTIBLE);
  209. add_wait_queue(pkmap_map_wait, &wait);
  210. unlock_kmap();
  211. schedule();
  212. remove_wait_queue(pkmap_map_wait, &wait);
  213. lock_kmap();
  214. /* Somebody else might have mapped it while we slept */
  215. if (page_address(page))
  216. return (unsigned long)page_address(page);
  217. /* Re-start */
  218. goto start;
  219. }
  220. }
  221. vaddr = PKMAP_ADDR(last_pkmap_nr);
  222. set_pte_at(&init_mm, vaddr,
  223. &(pkmap_page_table[last_pkmap_nr]), mk_pte(page, kmap_prot));
  224. pkmap_count[last_pkmap_nr] = 1;
  225. set_page_address(page, (void *)vaddr);
  226. return vaddr;
  227. }
  228. /**
  229. * kmap_high - map a highmem page into memory
  230. * @page: &struct page to map
  231. *
  232. * Returns the page's virtual memory address.
  233. *
  234. * We cannot call this from interrupts, as it may block.
  235. */
  236. void *kmap_high(struct page *page)
  237. {
  238. unsigned long vaddr;
  239. /*
  240. * For highmem pages, we can't trust "virtual" until
  241. * after we have the lock.
  242. */
  243. lock_kmap();
  244. vaddr = (unsigned long)page_address(page);
  245. if (!vaddr)
  246. vaddr = map_new_virtual(page);
  247. pkmap_count[PKMAP_NR(vaddr)]++;
  248. BUG_ON(pkmap_count[PKMAP_NR(vaddr)] < 2);
  249. unlock_kmap();
  250. return (void*) vaddr;
  251. }
  252. EXPORT_SYMBOL(kmap_high);
  253. #ifdef ARCH_NEEDS_KMAP_HIGH_GET
  254. /**
  255. * kmap_high_get - pin a highmem page into memory
  256. * @page: &struct page to pin
  257. *
  258. * Returns the page's current virtual memory address, or NULL if no mapping
  259. * exists. If and only if a non null address is returned then a
  260. * matching call to kunmap_high() is necessary.
  261. *
  262. * This can be called from any context.
  263. */
  264. void *kmap_high_get(struct page *page)
  265. {
  266. unsigned long vaddr, flags;
  267. lock_kmap_any(flags);
  268. vaddr = (unsigned long)page_address(page);
  269. if (vaddr) {
  270. BUG_ON(pkmap_count[PKMAP_NR(vaddr)] < 1);
  271. pkmap_count[PKMAP_NR(vaddr)]++;
  272. }
  273. unlock_kmap_any(flags);
  274. return (void*) vaddr;
  275. }
  276. #endif
  277. /**
  278. * kunmap_high - unmap a highmem page into memory
  279. * @page: &struct page to unmap
  280. *
  281. * If ARCH_NEEDS_KMAP_HIGH_GET is not defined then this may be called
  282. * only from user context.
  283. */
  284. void kunmap_high(struct page *page)
  285. {
  286. unsigned long vaddr;
  287. unsigned long nr;
  288. unsigned long flags;
  289. int need_wakeup;
  290. unsigned int color = get_pkmap_color(page);
  291. wait_queue_head_t *pkmap_map_wait;
  292. lock_kmap_any(flags);
  293. vaddr = (unsigned long)page_address(page);
  294. BUG_ON(!vaddr);
  295. nr = PKMAP_NR(vaddr);
  296. /*
  297. * A count must never go down to zero
  298. * without a TLB flush!
  299. */
  300. need_wakeup = 0;
  301. switch (--pkmap_count[nr]) {
  302. case 0:
  303. BUG();
  304. case 1:
  305. /*
  306. * Avoid an unnecessary wake_up() function call.
  307. * The common case is pkmap_count[] == 1, but
  308. * no waiters.
  309. * The tasks queued in the wait-queue are guarded
  310. * by both the lock in the wait-queue-head and by
  311. * the kmap_lock. As the kmap_lock is held here,
  312. * no need for the wait-queue-head's lock. Simply
  313. * test if the queue is empty.
  314. */
  315. pkmap_map_wait = get_pkmap_wait_queue_head(color);
  316. need_wakeup = waitqueue_active(pkmap_map_wait);
  317. }
  318. unlock_kmap_any(flags);
  319. /* do wake-up, if needed, race-free outside of the spin lock */
  320. if (need_wakeup)
  321. wake_up(pkmap_map_wait);
  322. }
  323. EXPORT_SYMBOL(kunmap_high);
  324. #endif /* CONFIG_HIGHMEM */
  325. #if defined(HASHED_PAGE_VIRTUAL)
  326. #define PA_HASH_ORDER 7
  327. /*
  328. * Describes one page->virtual association
  329. */
  330. struct page_address_map {
  331. struct page *page;
  332. void *virtual;
  333. struct list_head list;
  334. };
  335. static struct page_address_map page_address_maps[LAST_PKMAP];
  336. /*
  337. * Hash table bucket
  338. */
  339. static struct page_address_slot {
  340. struct list_head lh; /* List of page_address_maps */
  341. spinlock_t lock; /* Protect this bucket's list */
  342. } ____cacheline_aligned_in_smp page_address_htable[1<<PA_HASH_ORDER];
  343. static struct page_address_slot *page_slot(const struct page *page)
  344. {
  345. return &page_address_htable[hash_ptr(page, PA_HASH_ORDER)];
  346. }
  347. /**
  348. * page_address - get the mapped virtual address of a page
  349. * @page: &struct page to get the virtual address of
  350. *
  351. * Returns the page's virtual address.
  352. */
  353. void *page_address(const struct page *page)
  354. {
  355. unsigned long flags;
  356. void *ret;
  357. struct page_address_slot *pas;
  358. if (!PageHighMem(page))
  359. return lowmem_page_address(page);
  360. pas = page_slot(page);
  361. ret = NULL;
  362. spin_lock_irqsave(&pas->lock, flags);
  363. if (!list_empty(&pas->lh)) {
  364. struct page_address_map *pam;
  365. list_for_each_entry(pam, &pas->lh, list) {
  366. if (pam->page == page) {
  367. ret = pam->virtual;
  368. goto done;
  369. }
  370. }
  371. }
  372. done:
  373. spin_unlock_irqrestore(&pas->lock, flags);
  374. return ret;
  375. }
  376. EXPORT_SYMBOL(page_address);
  377. /**
  378. * set_page_address - set a page's virtual address
  379. * @page: &struct page to set
  380. * @virtual: virtual address to use
  381. */
  382. void set_page_address(struct page *page, void *virtual)
  383. {
  384. unsigned long flags;
  385. struct page_address_slot *pas;
  386. struct page_address_map *pam;
  387. BUG_ON(!PageHighMem(page));
  388. pas = page_slot(page);
  389. if (virtual) { /* Add */
  390. pam = &page_address_maps[PKMAP_NR((unsigned long)virtual)];
  391. pam->page = page;
  392. pam->virtual = virtual;
  393. spin_lock_irqsave(&pas->lock, flags);
  394. list_add_tail(&pam->list, &pas->lh);
  395. spin_unlock_irqrestore(&pas->lock, flags);
  396. } else { /* Remove */
  397. spin_lock_irqsave(&pas->lock, flags);
  398. list_for_each_entry(pam, &pas->lh, list) {
  399. if (pam->page == page) {
  400. list_del(&pam->list);
  401. spin_unlock_irqrestore(&pas->lock, flags);
  402. goto done;
  403. }
  404. }
  405. spin_unlock_irqrestore(&pas->lock, flags);
  406. }
  407. done:
  408. return;
  409. }
  410. void __init page_address_init(void)
  411. {
  412. int i;
  413. for (i = 0; i < ARRAY_SIZE(page_address_htable); i++) {
  414. INIT_LIST_HEAD(&page_address_htable[i].lh);
  415. spin_lock_init(&page_address_htable[i].lock);
  416. }
  417. }
  418. #endif /* defined(HASHED_PAGE_VIRTUAL) */