highmem.c 7.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335
  1. /*
  2. * High memory handling common code and variables.
  3. *
  4. * (C) 1999 Andrea Arcangeli, SuSE GmbH, andrea@suse.de
  5. * Gerhard Wichert, Siemens AG, Gerhard.Wichert@pdb.siemens.de
  6. *
  7. *
  8. * Redesigned the x86 32-bit VM architecture to deal with
  9. * 64-bit physical space. With current x86 CPUs this
  10. * means up to 64 Gigabytes physical RAM.
  11. *
  12. * Rewrote high memory support to move the page cache into
  13. * high memory. Implemented permanent (schedulable) kmaps
  14. * based on Linus' idea.
  15. *
  16. * Copyright (C) 1999 Ingo Molnar <mingo@redhat.com>
  17. */
  18. #include <linux/mm.h>
  19. #include <linux/module.h>
  20. #include <linux/swap.h>
  21. #include <linux/bio.h>
  22. #include <linux/pagemap.h>
  23. #include <linux/mempool.h>
  24. #include <linux/blkdev.h>
  25. #include <linux/init.h>
  26. #include <linux/hash.h>
  27. #include <linux/highmem.h>
  28. #include <linux/blktrace_api.h>
  29. #include <asm/tlbflush.h>
  30. /*
  31. * Virtual_count is not a pure "count".
  32. * 0 means that it is not mapped, and has not been mapped
  33. * since a TLB flush - it is usable.
  34. * 1 means that there are no users, but it has been mapped
  35. * since the last TLB flush - so we can't use it.
  36. * n means that there are (n-1) current users of it.
  37. */
  38. #ifdef CONFIG_HIGHMEM
  39. unsigned long totalhigh_pages __read_mostly;
  40. unsigned int nr_free_highpages (void)
  41. {
  42. pg_data_t *pgdat;
  43. unsigned int pages = 0;
  44. for_each_online_pgdat(pgdat)
  45. pages += zone_page_state(&pgdat->node_zones[ZONE_HIGHMEM],
  46. NR_FREE_PAGES);
  47. return pages;
  48. }
  49. static int pkmap_count[LAST_PKMAP];
  50. static unsigned int last_pkmap_nr;
  51. static __cacheline_aligned_in_smp DEFINE_SPINLOCK(kmap_lock);
  52. pte_t * pkmap_page_table;
  53. static DECLARE_WAIT_QUEUE_HEAD(pkmap_map_wait);
  54. static void flush_all_zero_pkmaps(void)
  55. {
  56. int i;
  57. flush_cache_kmaps();
  58. for (i = 0; i < LAST_PKMAP; i++) {
  59. struct page *page;
  60. /*
  61. * zero means we don't have anything to do,
  62. * >1 means that it is still in use. Only
  63. * a count of 1 means that it is free but
  64. * needs to be unmapped
  65. */
  66. if (pkmap_count[i] != 1)
  67. continue;
  68. pkmap_count[i] = 0;
  69. /* sanity check */
  70. BUG_ON(pte_none(pkmap_page_table[i]));
  71. /*
  72. * Don't need an atomic fetch-and-clear op here;
  73. * no-one has the page mapped, and cannot get at
  74. * its virtual address (and hence PTE) without first
  75. * getting the kmap_lock (which is held here).
  76. * So no dangers, even with speculative execution.
  77. */
  78. page = pte_page(pkmap_page_table[i]);
  79. pte_clear(&init_mm, (unsigned long)page_address(page),
  80. &pkmap_page_table[i]);
  81. set_page_address(page, NULL);
  82. }
  83. flush_tlb_kernel_range(PKMAP_ADDR(0), PKMAP_ADDR(LAST_PKMAP));
  84. }
  85. static inline unsigned long map_new_virtual(struct page *page)
  86. {
  87. unsigned long vaddr;
  88. int count;
  89. start:
  90. count = LAST_PKMAP;
  91. /* Find an empty entry */
  92. for (;;) {
  93. last_pkmap_nr = (last_pkmap_nr + 1) & LAST_PKMAP_MASK;
  94. if (!last_pkmap_nr) {
  95. flush_all_zero_pkmaps();
  96. count = LAST_PKMAP;
  97. }
  98. if (!pkmap_count[last_pkmap_nr])
  99. break; /* Found a usable entry */
  100. if (--count)
  101. continue;
  102. /*
  103. * Sleep for somebody else to unmap their entries
  104. */
  105. {
  106. DECLARE_WAITQUEUE(wait, current);
  107. __set_current_state(TASK_UNINTERRUPTIBLE);
  108. add_wait_queue(&pkmap_map_wait, &wait);
  109. spin_unlock(&kmap_lock);
  110. schedule();
  111. remove_wait_queue(&pkmap_map_wait, &wait);
  112. spin_lock(&kmap_lock);
  113. /* Somebody else might have mapped it while we slept */
  114. if (page_address(page))
  115. return (unsigned long)page_address(page);
  116. /* Re-start */
  117. goto start;
  118. }
  119. }
  120. vaddr = PKMAP_ADDR(last_pkmap_nr);
  121. set_pte_at(&init_mm, vaddr,
  122. &(pkmap_page_table[last_pkmap_nr]), mk_pte(page, kmap_prot));
  123. pkmap_count[last_pkmap_nr] = 1;
  124. set_page_address(page, (void *)vaddr);
  125. return vaddr;
  126. }
  127. void fastcall *kmap_high(struct page *page)
  128. {
  129. unsigned long vaddr;
  130. /*
  131. * For highmem pages, we can't trust "virtual" until
  132. * after we have the lock.
  133. *
  134. * We cannot call this from interrupts, as it may block
  135. */
  136. spin_lock(&kmap_lock);
  137. vaddr = (unsigned long)page_address(page);
  138. if (!vaddr)
  139. vaddr = map_new_virtual(page);
  140. pkmap_count[PKMAP_NR(vaddr)]++;
  141. BUG_ON(pkmap_count[PKMAP_NR(vaddr)] < 2);
  142. spin_unlock(&kmap_lock);
  143. return (void*) vaddr;
  144. }
  145. EXPORT_SYMBOL(kmap_high);
  146. void fastcall kunmap_high(struct page *page)
  147. {
  148. unsigned long vaddr;
  149. unsigned long nr;
  150. int need_wakeup;
  151. spin_lock(&kmap_lock);
  152. vaddr = (unsigned long)page_address(page);
  153. BUG_ON(!vaddr);
  154. nr = PKMAP_NR(vaddr);
  155. /*
  156. * A count must never go down to zero
  157. * without a TLB flush!
  158. */
  159. need_wakeup = 0;
  160. switch (--pkmap_count[nr]) {
  161. case 0:
  162. BUG();
  163. case 1:
  164. /*
  165. * Avoid an unnecessary wake_up() function call.
  166. * The common case is pkmap_count[] == 1, but
  167. * no waiters.
  168. * The tasks queued in the wait-queue are guarded
  169. * by both the lock in the wait-queue-head and by
  170. * the kmap_lock. As the kmap_lock is held here,
  171. * no need for the wait-queue-head's lock. Simply
  172. * test if the queue is empty.
  173. */
  174. need_wakeup = waitqueue_active(&pkmap_map_wait);
  175. }
  176. spin_unlock(&kmap_lock);
  177. /* do wake-up, if needed, race-free outside of the spin lock */
  178. if (need_wakeup)
  179. wake_up(&pkmap_map_wait);
  180. }
  181. EXPORT_SYMBOL(kunmap_high);
  182. #endif
  183. #if defined(HASHED_PAGE_VIRTUAL)
  184. #define PA_HASH_ORDER 7
  185. /*
  186. * Describes one page->virtual association
  187. */
  188. struct page_address_map {
  189. struct page *page;
  190. void *virtual;
  191. struct list_head list;
  192. };
  193. /*
  194. * page_address_map freelist, allocated from page_address_maps.
  195. */
  196. static struct list_head page_address_pool; /* freelist */
  197. static spinlock_t pool_lock; /* protects page_address_pool */
  198. /*
  199. * Hash table bucket
  200. */
  201. static struct page_address_slot {
  202. struct list_head lh; /* List of page_address_maps */
  203. spinlock_t lock; /* Protect this bucket's list */
  204. } ____cacheline_aligned_in_smp page_address_htable[1<<PA_HASH_ORDER];
  205. static struct page_address_slot *page_slot(struct page *page)
  206. {
  207. return &page_address_htable[hash_ptr(page, PA_HASH_ORDER)];
  208. }
  209. void *page_address(struct page *page)
  210. {
  211. unsigned long flags;
  212. void *ret;
  213. struct page_address_slot *pas;
  214. if (!PageHighMem(page))
  215. return lowmem_page_address(page);
  216. pas = page_slot(page);
  217. ret = NULL;
  218. spin_lock_irqsave(&pas->lock, flags);
  219. if (!list_empty(&pas->lh)) {
  220. struct page_address_map *pam;
  221. list_for_each_entry(pam, &pas->lh, list) {
  222. if (pam->page == page) {
  223. ret = pam->virtual;
  224. goto done;
  225. }
  226. }
  227. }
  228. done:
  229. spin_unlock_irqrestore(&pas->lock, flags);
  230. return ret;
  231. }
  232. EXPORT_SYMBOL(page_address);
  233. void set_page_address(struct page *page, void *virtual)
  234. {
  235. unsigned long flags;
  236. struct page_address_slot *pas;
  237. struct page_address_map *pam;
  238. BUG_ON(!PageHighMem(page));
  239. pas = page_slot(page);
  240. if (virtual) { /* Add */
  241. BUG_ON(list_empty(&page_address_pool));
  242. spin_lock_irqsave(&pool_lock, flags);
  243. pam = list_entry(page_address_pool.next,
  244. struct page_address_map, list);
  245. list_del(&pam->list);
  246. spin_unlock_irqrestore(&pool_lock, flags);
  247. pam->page = page;
  248. pam->virtual = virtual;
  249. spin_lock_irqsave(&pas->lock, flags);
  250. list_add_tail(&pam->list, &pas->lh);
  251. spin_unlock_irqrestore(&pas->lock, flags);
  252. } else { /* Remove */
  253. spin_lock_irqsave(&pas->lock, flags);
  254. list_for_each_entry(pam, &pas->lh, list) {
  255. if (pam->page == page) {
  256. list_del(&pam->list);
  257. spin_unlock_irqrestore(&pas->lock, flags);
  258. spin_lock_irqsave(&pool_lock, flags);
  259. list_add_tail(&pam->list, &page_address_pool);
  260. spin_unlock_irqrestore(&pool_lock, flags);
  261. goto done;
  262. }
  263. }
  264. spin_unlock_irqrestore(&pas->lock, flags);
  265. }
  266. done:
  267. return;
  268. }
  269. static struct page_address_map page_address_maps[LAST_PKMAP];
  270. void __init page_address_init(void)
  271. {
  272. int i;
  273. INIT_LIST_HEAD(&page_address_pool);
  274. for (i = 0; i < ARRAY_SIZE(page_address_maps); i++)
  275. list_add(&page_address_maps[i].list, &page_address_pool);
  276. for (i = 0; i < ARRAY_SIZE(page_address_htable); i++) {
  277. INIT_LIST_HEAD(&page_address_htable[i].lh);
  278. spin_lock_init(&page_address_htable[i].lock);
  279. }
  280. spin_lock_init(&pool_lock);
  281. }
  282. #endif /* defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL) */