swap_state.c 9.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366
  1. /*
  2. * linux/mm/swap_state.c
  3. *
  4. * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
  5. * Swap reorganised 29.12.95, Stephen Tweedie
  6. *
  7. * Rewritten to use page cache, (C) 1998 Stephen Tweedie
  8. */
  9. #include <linux/module.h>
  10. #include <linux/mm.h>
  11. #include <linux/kernel_stat.h>
  12. #include <linux/swap.h>
  13. #include <linux/init.h>
  14. #include <linux/pagemap.h>
  15. #include <linux/buffer_head.h>
  16. #include <linux/backing-dev.h>
  17. #include <linux/pagevec.h>
  18. #include <linux/migrate.h>
  19. #include <asm/pgtable.h>
  20. /*
  21. * swapper_space is a fiction, retained to simplify the path through
  22. * vmscan's shrink_list, to make sync_page look nicer, and to allow
  23. * future use of radix_tree tags in the swap cache.
  24. */
  25. static const struct address_space_operations swap_aops = {
  26. .writepage = swap_writepage,
  27. .sync_page = block_sync_page,
  28. .set_page_dirty = __set_page_dirty_nobuffers,
  29. .migratepage = migrate_page,
  30. };
  31. static struct backing_dev_info swap_backing_dev_info = {
  32. .capabilities = BDI_CAP_NO_ACCT_DIRTY | BDI_CAP_NO_WRITEBACK,
  33. .unplug_io_fn = swap_unplug_io_fn,
  34. };
  35. struct address_space swapper_space = {
  36. .page_tree = RADIX_TREE_INIT(GFP_ATOMIC|__GFP_NOWARN),
  37. .tree_lock = __RW_LOCK_UNLOCKED(swapper_space.tree_lock),
  38. .a_ops = &swap_aops,
  39. .i_mmap_nonlinear = LIST_HEAD_INIT(swapper_space.i_mmap_nonlinear),
  40. .backing_dev_info = &swap_backing_dev_info,
  41. };
  42. #define INC_CACHE_INFO(x) do { swap_cache_info.x++; } while (0)
  43. static struct {
  44. unsigned long add_total;
  45. unsigned long del_total;
  46. unsigned long find_success;
  47. unsigned long find_total;
  48. unsigned long noent_race;
  49. unsigned long exist_race;
  50. } swap_cache_info;
  51. void show_swap_cache_info(void)
  52. {
  53. printk("Swap cache: add %lu, delete %lu, find %lu/%lu, race %lu+%lu\n",
  54. swap_cache_info.add_total, swap_cache_info.del_total,
  55. swap_cache_info.find_success, swap_cache_info.find_total,
  56. swap_cache_info.noent_race, swap_cache_info.exist_race);
  57. printk("Free swap = %lukB\n", nr_swap_pages << (PAGE_SHIFT - 10));
  58. printk("Total swap = %lukB\n", total_swap_pages << (PAGE_SHIFT - 10));
  59. }
  60. /*
  61. * __add_to_swap_cache resembles add_to_page_cache on swapper_space,
  62. * but sets SwapCache flag and private instead of mapping and index.
  63. */
  64. static int __add_to_swap_cache(struct page *page, swp_entry_t entry,
  65. gfp_t gfp_mask)
  66. {
  67. int error;
  68. BUG_ON(PageSwapCache(page));
  69. BUG_ON(PagePrivate(page));
  70. error = radix_tree_preload(gfp_mask);
  71. if (!error) {
  72. write_lock_irq(&swapper_space.tree_lock);
  73. error = radix_tree_insert(&swapper_space.page_tree,
  74. entry.val, page);
  75. if (!error) {
  76. page_cache_get(page);
  77. SetPageLocked(page);
  78. SetPageSwapCache(page);
  79. set_page_private(page, entry.val);
  80. total_swapcache_pages++;
  81. __inc_zone_page_state(page, NR_FILE_PAGES);
  82. }
  83. write_unlock_irq(&swapper_space.tree_lock);
  84. radix_tree_preload_end();
  85. }
  86. return error;
  87. }
  88. static int add_to_swap_cache(struct page *page, swp_entry_t entry)
  89. {
  90. int error;
  91. if (!swap_duplicate(entry)) {
  92. INC_CACHE_INFO(noent_race);
  93. return -ENOENT;
  94. }
  95. error = __add_to_swap_cache(page, entry, GFP_KERNEL);
  96. /*
  97. * Anon pages are already on the LRU, we don't run lru_cache_add here.
  98. */
  99. if (error) {
  100. swap_free(entry);
  101. if (error == -EEXIST)
  102. INC_CACHE_INFO(exist_race);
  103. return error;
  104. }
  105. INC_CACHE_INFO(add_total);
  106. return 0;
  107. }
  108. /*
  109. * This must be called only on pages that have
  110. * been verified to be in the swap cache.
  111. */
  112. void __delete_from_swap_cache(struct page *page)
  113. {
  114. BUG_ON(!PageLocked(page));
  115. BUG_ON(!PageSwapCache(page));
  116. BUG_ON(PageWriteback(page));
  117. BUG_ON(PagePrivate(page));
  118. radix_tree_delete(&swapper_space.page_tree, page_private(page));
  119. set_page_private(page, 0);
  120. ClearPageSwapCache(page);
  121. total_swapcache_pages--;
  122. __dec_zone_page_state(page, NR_FILE_PAGES);
  123. INC_CACHE_INFO(del_total);
  124. }
  125. /**
  126. * add_to_swap - allocate swap space for a page
  127. * @page: page we want to move to swap
  128. *
  129. * Allocate swap space for the page and add the page to the
  130. * swap cache. Caller needs to hold the page lock.
  131. */
  132. int add_to_swap(struct page * page, gfp_t gfp_mask)
  133. {
  134. swp_entry_t entry;
  135. int err;
  136. BUG_ON(!PageLocked(page));
  137. for (;;) {
  138. entry = get_swap_page();
  139. if (!entry.val)
  140. return 0;
  141. /*
  142. * Radix-tree node allocations from PF_MEMALLOC contexts could
  143. * completely exhaust the page allocator. __GFP_NOMEMALLOC
  144. * stops emergency reserves from being allocated.
  145. *
  146. * TODO: this could cause a theoretical memory reclaim
  147. * deadlock in the swap out path.
  148. */
  149. /*
  150. * Add it to the swap cache and mark it dirty
  151. */
  152. err = __add_to_swap_cache(page, entry,
  153. gfp_mask|__GFP_NOMEMALLOC|__GFP_NOWARN);
  154. switch (err) {
  155. case 0: /* Success */
  156. SetPageUptodate(page);
  157. SetPageDirty(page);
  158. INC_CACHE_INFO(add_total);
  159. return 1;
  160. case -EEXIST:
  161. /* Raced with "speculative" read_swap_cache_async */
  162. INC_CACHE_INFO(exist_race);
  163. swap_free(entry);
  164. continue;
  165. default:
  166. /* -ENOMEM radix-tree allocation failure */
  167. swap_free(entry);
  168. return 0;
  169. }
  170. }
  171. }
  172. /*
  173. * This must be called only on pages that have
  174. * been verified to be in the swap cache and locked.
  175. * It will never put the page into the free list,
  176. * the caller has a reference on the page.
  177. */
  178. void delete_from_swap_cache(struct page *page)
  179. {
  180. swp_entry_t entry;
  181. entry.val = page_private(page);
  182. write_lock_irq(&swapper_space.tree_lock);
  183. __delete_from_swap_cache(page);
  184. write_unlock_irq(&swapper_space.tree_lock);
  185. swap_free(entry);
  186. page_cache_release(page);
  187. }
  188. /*
  189. * Strange swizzling function only for use by shmem_writepage
  190. */
  191. int move_to_swap_cache(struct page *page, swp_entry_t entry)
  192. {
  193. int err = __add_to_swap_cache(page, entry, GFP_ATOMIC);
  194. if (!err) {
  195. remove_from_page_cache(page);
  196. page_cache_release(page); /* pagecache ref */
  197. if (!swap_duplicate(entry))
  198. BUG();
  199. SetPageDirty(page);
  200. INC_CACHE_INFO(add_total);
  201. } else if (err == -EEXIST)
  202. INC_CACHE_INFO(exist_race);
  203. return err;
  204. }
  205. /*
  206. * Strange swizzling function for shmem_getpage (and shmem_unuse)
  207. */
  208. int move_from_swap_cache(struct page *page, unsigned long index,
  209. struct address_space *mapping)
  210. {
  211. int err = add_to_page_cache(page, mapping, index, GFP_ATOMIC);
  212. if (!err) {
  213. delete_from_swap_cache(page);
  214. /* shift page from clean_pages to dirty_pages list */
  215. ClearPageDirty(page);
  216. set_page_dirty(page);
  217. }
  218. return err;
  219. }
  220. /*
  221. * If we are the only user, then try to free up the swap cache.
  222. *
  223. * Its ok to check for PageSwapCache without the page lock
  224. * here because we are going to recheck again inside
  225. * exclusive_swap_page() _with_ the lock.
  226. * - Marcelo
  227. */
  228. static inline void free_swap_cache(struct page *page)
  229. {
  230. if (PageSwapCache(page) && !TestSetPageLocked(page)) {
  231. remove_exclusive_swap_page(page);
  232. unlock_page(page);
  233. }
  234. }
  235. /*
  236. * Perform a free_page(), also freeing any swap cache associated with
  237. * this page if it is the last user of the page.
  238. */
  239. void free_page_and_swap_cache(struct page *page)
  240. {
  241. free_swap_cache(page);
  242. page_cache_release(page);
  243. }
  244. /*
  245. * Passed an array of pages, drop them all from swapcache and then release
  246. * them. They are removed from the LRU and freed if this is their last use.
  247. */
  248. void free_pages_and_swap_cache(struct page **pages, int nr)
  249. {
  250. struct page **pagep = pages;
  251. lru_add_drain();
  252. while (nr) {
  253. int todo = min(nr, PAGEVEC_SIZE);
  254. int i;
  255. for (i = 0; i < todo; i++)
  256. free_swap_cache(pagep[i]);
  257. release_pages(pagep, todo, 0);
  258. pagep += todo;
  259. nr -= todo;
  260. }
  261. }
  262. /*
  263. * Lookup a swap entry in the swap cache. A found page will be returned
  264. * unlocked and with its refcount incremented - we rely on the kernel
  265. * lock getting page table operations atomic even if we drop the page
  266. * lock before returning.
  267. */
  268. struct page * lookup_swap_cache(swp_entry_t entry)
  269. {
  270. struct page *page;
  271. page = find_get_page(&swapper_space, entry.val);
  272. if (page)
  273. INC_CACHE_INFO(find_success);
  274. INC_CACHE_INFO(find_total);
  275. return page;
  276. }
  277. /*
  278. * Locate a page of swap in physical memory, reserving swap cache space
  279. * and reading the disk if it is not already cached.
  280. * A failure return means that either the page allocation failed or that
  281. * the swap entry is no longer in use.
  282. */
  283. struct page *read_swap_cache_async(swp_entry_t entry,
  284. struct vm_area_struct *vma, unsigned long addr)
  285. {
  286. struct page *found_page, *new_page = NULL;
  287. int err;
  288. do {
  289. /*
  290. * First check the swap cache. Since this is normally
  291. * called after lookup_swap_cache() failed, re-calling
  292. * that would confuse statistics.
  293. */
  294. found_page = find_get_page(&swapper_space, entry.val);
  295. if (found_page)
  296. break;
  297. /*
  298. * Get a new page to read into from swap.
  299. */
  300. if (!new_page) {
  301. new_page = alloc_page_vma(GFP_HIGHUSER, vma, addr);
  302. if (!new_page)
  303. break; /* Out of memory */
  304. }
  305. /*
  306. * Associate the page with swap entry in the swap cache.
  307. * May fail (-ENOENT) if swap entry has been freed since
  308. * our caller observed it. May fail (-EEXIST) if there
  309. * is already a page associated with this entry in the
  310. * swap cache: added by a racing read_swap_cache_async,
  311. * or by try_to_swap_out (or shmem_writepage) re-using
  312. * the just freed swap entry for an existing page.
  313. * May fail (-ENOMEM) if radix-tree node allocation failed.
  314. */
  315. err = add_to_swap_cache(new_page, entry);
  316. if (!err) {
  317. /*
  318. * Initiate read into locked page and return.
  319. */
  320. lru_cache_add_active(new_page);
  321. swap_readpage(NULL, new_page);
  322. return new_page;
  323. }
  324. } while (err != -ENOENT && err != -ENOMEM);
  325. if (new_page)
  326. page_cache_release(new_page);
  327. return found_page;
  328. }