swap_state.c 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * linux/mm/swap_state.c
  4. *
  5. * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
  6. * Swap reorganised 29.12.95, Stephen Tweedie
  7. *
  8. * Rewritten to use page cache, (C) 1998 Stephen Tweedie
  9. */
  10. #include <linux/mm.h>
  11. #include <linux/gfp.h>
  12. #include <linux/kernel_stat.h>
  13. #include <linux/swap.h>
  14. #include <linux/swapops.h>
  15. #include <linux/init.h>
  16. #include <linux/pagemap.h>
  17. #include <linux/backing-dev.h>
  18. #include <linux/blkdev.h>
  19. #include <linux/pagevec.h>
  20. #include <linux/migrate.h>
  21. #include <linux/vmalloc.h>
  22. #include <linux/swap_slots.h>
  23. #include <linux/huge_mm.h>
  24. #include <linux/shmem_fs.h>
  25. #include "internal.h"
  26. /*
  27. * swapper_space is a fiction, retained to simplify the path through
  28. * vmscan's shrink_page_list.
  29. */
  30. static const struct address_space_operations swap_aops = {
  31. .writepage = swap_writepage,
  32. .set_page_dirty = swap_set_page_dirty,
  33. #ifdef CONFIG_MIGRATION
  34. .migratepage = migrate_page,
  35. #endif
  36. };
  37. struct address_space *swapper_spaces[MAX_SWAPFILES] __read_mostly;
  38. static unsigned int nr_swapper_spaces[MAX_SWAPFILES] __read_mostly;
  39. static bool enable_vma_readahead __read_mostly = true;
  40. #define SWAP_RA_WIN_SHIFT (PAGE_SHIFT / 2)
  41. #define SWAP_RA_HITS_MASK ((1UL << SWAP_RA_WIN_SHIFT) - 1)
  42. #define SWAP_RA_HITS_MAX SWAP_RA_HITS_MASK
  43. #define SWAP_RA_WIN_MASK (~PAGE_MASK & ~SWAP_RA_HITS_MASK)
  44. #define SWAP_RA_HITS(v) ((v) & SWAP_RA_HITS_MASK)
  45. #define SWAP_RA_WIN(v) (((v) & SWAP_RA_WIN_MASK) >> SWAP_RA_WIN_SHIFT)
  46. #define SWAP_RA_ADDR(v) ((v) & PAGE_MASK)
  47. #define SWAP_RA_VAL(addr, win, hits) \
  48. (((addr) & PAGE_MASK) | \
  49. (((win) << SWAP_RA_WIN_SHIFT) & SWAP_RA_WIN_MASK) | \
  50. ((hits) & SWAP_RA_HITS_MASK))
  51. /* Initial readahead hits is 4 to start up with a small window */
  52. #define GET_SWAP_RA_VAL(vma) \
  53. (atomic_long_read(&(vma)->swap_readahead_info) ? : 4)
  54. #define INC_CACHE_INFO(x) data_race(swap_cache_info.x++)
  55. #define ADD_CACHE_INFO(x, nr) data_race(swap_cache_info.x += (nr))
  56. static struct {
  57. unsigned long add_total;
  58. unsigned long del_total;
  59. unsigned long find_success;
  60. unsigned long find_total;
  61. } swap_cache_info;
  62. unsigned long total_swapcache_pages(void)
  63. {
  64. unsigned int i, j, nr;
  65. unsigned long ret = 0;
  66. struct address_space *spaces;
  67. struct swap_info_struct *si;
  68. for (i = 0; i < MAX_SWAPFILES; i++) {
  69. swp_entry_t entry = swp_entry(i, 1);
  70. /* Avoid get_swap_device() to warn for bad swap entry */
  71. if (!swp_swap_info(entry))
  72. continue;
  73. /* Prevent swapoff to free swapper_spaces */
  74. si = get_swap_device(entry);
  75. if (!si)
  76. continue;
  77. nr = nr_swapper_spaces[i];
  78. spaces = swapper_spaces[i];
  79. for (j = 0; j < nr; j++)
  80. ret += spaces[j].nrpages;
  81. put_swap_device(si);
  82. }
  83. return ret;
  84. }
  85. EXPORT_SYMBOL_GPL(total_swapcache_pages);
  86. static atomic_t swapin_readahead_hits = ATOMIC_INIT(4);
  87. void show_swap_cache_info(void)
  88. {
  89. printk("%lu pages in swap cache\n", total_swapcache_pages());
  90. printk("Swap cache stats: add %lu, delete %lu, find %lu/%lu\n",
  91. swap_cache_info.add_total, swap_cache_info.del_total,
  92. swap_cache_info.find_success, swap_cache_info.find_total);
  93. printk("Free swap = %ldkB\n",
  94. get_nr_swap_pages() << (PAGE_SHIFT - 10));
  95. printk("Total swap = %lukB\n", total_swap_pages << (PAGE_SHIFT - 10));
  96. }
  97. void *get_shadow_from_swap_cache(swp_entry_t entry)
  98. {
  99. struct address_space *address_space = swap_address_space(entry);
  100. pgoff_t idx = swp_offset(entry);
  101. struct page *page;
  102. page = find_get_entry(address_space, idx);
  103. if (xa_is_value(page))
  104. return page;
  105. if (page)
  106. put_page(page);
  107. return NULL;
  108. }
  109. /*
  110. * add_to_swap_cache resembles add_to_page_cache_locked on swapper_space,
  111. * but sets SwapCache flag and private instead of mapping and index.
  112. */
  113. int add_to_swap_cache(struct page *page, swp_entry_t entry,
  114. gfp_t gfp, void **shadowp)
  115. {
  116. struct address_space *address_space = swap_address_space(entry);
  117. pgoff_t idx = swp_offset(entry);
  118. XA_STATE_ORDER(xas, &address_space->i_pages, idx, compound_order(page));
  119. unsigned long i, nr = thp_nr_pages(page);
  120. void *old;
  121. VM_BUG_ON_PAGE(!PageLocked(page), page);
  122. VM_BUG_ON_PAGE(PageSwapCache(page), page);
  123. VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
  124. page_ref_add(page, nr);
  125. SetPageSwapCache(page);
  126. do {
  127. unsigned long nr_shadows = 0;
  128. xas_lock_irq(&xas);
  129. xas_create_range(&xas);
  130. if (xas_error(&xas))
  131. goto unlock;
  132. for (i = 0; i < nr; i++) {
  133. VM_BUG_ON_PAGE(xas.xa_index != idx + i, page);
  134. old = xas_load(&xas);
  135. if (xa_is_value(old)) {
  136. nr_shadows++;
  137. if (shadowp)
  138. *shadowp = old;
  139. }
  140. set_page_private(page + i, entry.val + i);
  141. xas_store(&xas, page);
  142. xas_next(&xas);
  143. }
  144. address_space->nrexceptional -= nr_shadows;
  145. address_space->nrpages += nr;
  146. __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr);
  147. ADD_CACHE_INFO(add_total, nr);
  148. unlock:
  149. xas_unlock_irq(&xas);
  150. } while (xas_nomem(&xas, gfp));
  151. if (!xas_error(&xas))
  152. return 0;
  153. ClearPageSwapCache(page);
  154. page_ref_sub(page, nr);
  155. return xas_error(&xas);
  156. }
  157. /*
  158. * This must be called only on pages that have
  159. * been verified to be in the swap cache.
  160. */
  161. void __delete_from_swap_cache(struct page *page,
  162. swp_entry_t entry, void *shadow)
  163. {
  164. struct address_space *address_space = swap_address_space(entry);
  165. int i, nr = thp_nr_pages(page);
  166. pgoff_t idx = swp_offset(entry);
  167. XA_STATE(xas, &address_space->i_pages, idx);
  168. VM_BUG_ON_PAGE(!PageLocked(page), page);
  169. VM_BUG_ON_PAGE(!PageSwapCache(page), page);
  170. VM_BUG_ON_PAGE(PageWriteback(page), page);
  171. for (i = 0; i < nr; i++) {
  172. void *entry = xas_store(&xas, shadow);
  173. VM_BUG_ON_PAGE(entry != page, entry);
  174. set_page_private(page + i, 0);
  175. xas_next(&xas);
  176. }
  177. ClearPageSwapCache(page);
  178. if (shadow)
  179. address_space->nrexceptional += nr;
  180. address_space->nrpages -= nr;
  181. __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, -nr);
  182. ADD_CACHE_INFO(del_total, nr);
  183. }
  184. /**
  185. * add_to_swap - allocate swap space for a page
  186. * @page: page we want to move to swap
  187. *
  188. * Allocate swap space for the page and add the page to the
  189. * swap cache. Caller needs to hold the page lock.
  190. */
  191. int add_to_swap(struct page *page)
  192. {
  193. swp_entry_t entry;
  194. int err;
  195. VM_BUG_ON_PAGE(!PageLocked(page), page);
  196. VM_BUG_ON_PAGE(!PageUptodate(page), page);
  197. entry = get_swap_page(page);
  198. if (!entry.val)
  199. return 0;
  200. /*
  201. * XArray node allocations from PF_MEMALLOC contexts could
  202. * completely exhaust the page allocator. __GFP_NOMEMALLOC
  203. * stops emergency reserves from being allocated.
  204. *
  205. * TODO: this could cause a theoretical memory reclaim
  206. * deadlock in the swap out path.
  207. */
  208. /*
  209. * Add it to the swap cache.
  210. */
  211. err = add_to_swap_cache(page, entry,
  212. __GFP_HIGH|__GFP_NOMEMALLOC|__GFP_NOWARN, NULL);
  213. if (err)
  214. /*
  215. * add_to_swap_cache() doesn't return -EEXIST, so we can safely
  216. * clear SWAP_HAS_CACHE flag.
  217. */
  218. goto fail;
  219. /*
  220. * Normally the page will be dirtied in unmap because its pte should be
  221. * dirty. A special case is MADV_FREE page. The page's pte could have
  222. * dirty bit cleared but the page's SwapBacked bit is still set because
  223. * clearing the dirty bit and SwapBacked bit has no lock protected. For
  224. * such page, unmap will not set dirty bit for it, so page reclaim will
  225. * not write the page out. This can cause data corruption when the page
  226. * is swap in later. Always setting the dirty bit for the page solves
  227. * the problem.
  228. */
  229. set_page_dirty(page);
  230. return 1;
  231. fail:
  232. put_swap_page(page, entry);
  233. return 0;
  234. }
  235. /*
  236. * This must be called only on pages that have
  237. * been verified to be in the swap cache and locked.
  238. * It will never put the page into the free list,
  239. * the caller has a reference on the page.
  240. */
  241. void delete_from_swap_cache(struct page *page)
  242. {
  243. swp_entry_t entry = { .val = page_private(page) };
  244. struct address_space *address_space = swap_address_space(entry);
  245. xa_lock_irq(&address_space->i_pages);
  246. __delete_from_swap_cache(page, entry, NULL);
  247. xa_unlock_irq(&address_space->i_pages);
  248. put_swap_page(page, entry);
  249. page_ref_sub(page, thp_nr_pages(page));
  250. }
  251. void clear_shadow_from_swap_cache(int type, unsigned long begin,
  252. unsigned long end)
  253. {
  254. unsigned long curr = begin;
  255. void *old;
  256. for (;;) {
  257. unsigned long nr_shadows = 0;
  258. swp_entry_t entry = swp_entry(type, curr);
  259. struct address_space *address_space = swap_address_space(entry);
  260. XA_STATE(xas, &address_space->i_pages, curr);
  261. xa_lock_irq(&address_space->i_pages);
  262. xas_for_each(&xas, old, end) {
  263. if (!xa_is_value(old))
  264. continue;
  265. xas_store(&xas, NULL);
  266. nr_shadows++;
  267. }
  268. address_space->nrexceptional -= nr_shadows;
  269. xa_unlock_irq(&address_space->i_pages);
  270. /* search the next swapcache until we meet end */
  271. curr >>= SWAP_ADDRESS_SPACE_SHIFT;
  272. curr++;
  273. curr <<= SWAP_ADDRESS_SPACE_SHIFT;
  274. if (curr > end)
  275. break;
  276. }
  277. }
  278. /*
  279. * If we are the only user, then try to free up the swap cache.
  280. *
  281. * Its ok to check for PageSwapCache without the page lock
  282. * here because we are going to recheck again inside
  283. * try_to_free_swap() _with_ the lock.
  284. * - Marcelo
  285. */
  286. static inline void free_swap_cache(struct page *page)
  287. {
  288. if (PageSwapCache(page) && !page_mapped(page) && trylock_page(page)) {
  289. try_to_free_swap(page);
  290. unlock_page(page);
  291. }
  292. }
  293. /*
  294. * Perform a free_page(), also freeing any swap cache associated with
  295. * this page if it is the last user of the page.
  296. */
  297. void free_page_and_swap_cache(struct page *page)
  298. {
  299. free_swap_cache(page);
  300. if (!is_huge_zero_page(page))
  301. put_page(page);
  302. }
  303. /*
  304. * Passed an array of pages, drop them all from swapcache and then release
  305. * them. They are removed from the LRU and freed if this is their last use.
  306. */
  307. void free_pages_and_swap_cache(struct page **pages, int nr)
  308. {
  309. struct page **pagep = pages;
  310. int i;
  311. lru_add_drain();
  312. for (i = 0; i < nr; i++)
  313. free_swap_cache(pagep[i]);
  314. release_pages(pagep, nr);
  315. }
  316. static inline bool swap_use_vma_readahead(void)
  317. {
  318. return READ_ONCE(enable_vma_readahead) && !atomic_read(&nr_rotate_swap);
  319. }
  320. /*
  321. * Lookup a swap entry in the swap cache. A found page will be returned
  322. * unlocked and with its refcount incremented - we rely on the kernel
  323. * lock getting page table operations atomic even if we drop the page
  324. * lock before returning.
  325. */
  326. struct page *lookup_swap_cache(swp_entry_t entry, struct vm_area_struct *vma,
  327. unsigned long addr)
  328. {
  329. struct page *page;
  330. struct swap_info_struct *si;
  331. si = get_swap_device(entry);
  332. if (!si)
  333. return NULL;
  334. page = find_get_page(swap_address_space(entry), swp_offset(entry));
  335. put_swap_device(si);
  336. INC_CACHE_INFO(find_total);
  337. if (page) {
  338. bool vma_ra = swap_use_vma_readahead();
  339. bool readahead;
  340. INC_CACHE_INFO(find_success);
  341. /*
  342. * At the moment, we don't support PG_readahead for anon THP
  343. * so let's bail out rather than confusing the readahead stat.
  344. */
  345. if (unlikely(PageTransCompound(page)))
  346. return page;
  347. readahead = TestClearPageReadahead(page);
  348. if (vma && vma_ra) {
  349. unsigned long ra_val;
  350. int win, hits;
  351. ra_val = GET_SWAP_RA_VAL(vma);
  352. win = SWAP_RA_WIN(ra_val);
  353. hits = SWAP_RA_HITS(ra_val);
  354. if (readahead)
  355. hits = min_t(int, hits + 1, SWAP_RA_HITS_MAX);
  356. atomic_long_set(&vma->swap_readahead_info,
  357. SWAP_RA_VAL(addr, win, hits));
  358. }
  359. if (readahead) {
  360. count_vm_event(SWAP_RA_HIT);
  361. if (!vma || !vma_ra)
  362. atomic_inc(&swapin_readahead_hits);
  363. }
  364. }
  365. return page;
  366. }
  367. /**
  368. * find_get_incore_page - Find and get a page from the page or swap caches.
  369. * @mapping: The address_space to search.
  370. * @index: The page cache index.
  371. *
  372. * This differs from find_get_page() in that it will also look for the
  373. * page in the swap cache.
  374. *
  375. * Return: The found page or %NULL.
  376. */
  377. struct page *find_get_incore_page(struct address_space *mapping, pgoff_t index)
  378. {
  379. swp_entry_t swp;
  380. struct swap_info_struct *si;
  381. struct page *page = find_get_entry(mapping, index);
  382. if (!page)
  383. return page;
  384. if (!xa_is_value(page))
  385. return find_subpage(page, index);
  386. if (!shmem_mapping(mapping))
  387. return NULL;
  388. swp = radix_to_swp_entry(page);
  389. /* Prevent swapoff from happening to us */
  390. si = get_swap_device(swp);
  391. if (!si)
  392. return NULL;
  393. page = find_get_page(swap_address_space(swp), swp_offset(swp));
  394. put_swap_device(si);
  395. return page;
  396. }
  397. struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
  398. struct vm_area_struct *vma, unsigned long addr,
  399. bool *new_page_allocated)
  400. {
  401. struct swap_info_struct *si;
  402. struct page *page;
  403. void *shadow = NULL;
  404. *new_page_allocated = false;
  405. for (;;) {
  406. int err;
  407. /*
  408. * First check the swap cache. Since this is normally
  409. * called after lookup_swap_cache() failed, re-calling
  410. * that would confuse statistics.
  411. */
  412. si = get_swap_device(entry);
  413. if (!si)
  414. return NULL;
  415. page = find_get_page(swap_address_space(entry),
  416. swp_offset(entry));
  417. put_swap_device(si);
  418. if (page)
  419. return page;
  420. /*
  421. * Just skip read ahead for unused swap slot.
  422. * During swap_off when swap_slot_cache is disabled,
  423. * we have to handle the race between putting
  424. * swap entry in swap cache and marking swap slot
  425. * as SWAP_HAS_CACHE. That's done in later part of code or
  426. * else swap_off will be aborted if we return NULL.
  427. */
  428. if (!__swp_swapcount(entry) && swap_slot_cache_enabled)
  429. return NULL;
  430. /*
  431. * Get a new page to read into from swap. Allocate it now,
  432. * before marking swap_map SWAP_HAS_CACHE, when -EEXIST will
  433. * cause any racers to loop around until we add it to cache.
  434. */
  435. page = alloc_page_vma(gfp_mask, vma, addr);
  436. if (!page)
  437. return NULL;
  438. /*
  439. * Swap entry may have been freed since our caller observed it.
  440. */
  441. err = swapcache_prepare(entry);
  442. if (!err)
  443. break;
  444. put_page(page);
  445. if (err != -EEXIST)
  446. return NULL;
  447. /*
  448. * We might race against __delete_from_swap_cache(), and
  449. * stumble across a swap_map entry whose SWAP_HAS_CACHE
  450. * has not yet been cleared. Or race against another
  451. * __read_swap_cache_async(), which has set SWAP_HAS_CACHE
  452. * in swap_map, but not yet added its page to swap cache.
  453. */
  454. schedule_timeout_uninterruptible(1);
  455. }
  456. /*
  457. * The swap entry is ours to swap in. Prepare the new page.
  458. */
  459. __SetPageLocked(page);
  460. __SetPageSwapBacked(page);
  461. /* May fail (-ENOMEM) if XArray node allocation failed. */
  462. if (add_to_swap_cache(page, entry, gfp_mask & GFP_RECLAIM_MASK, &shadow)) {
  463. put_swap_page(page, entry);
  464. goto fail_unlock;
  465. }
  466. if (mem_cgroup_charge(page, NULL, gfp_mask)) {
  467. delete_from_swap_cache(page);
  468. goto fail_unlock;
  469. }
  470. if (shadow)
  471. workingset_refault(page, shadow);
  472. /* Caller will initiate read into locked page */
  473. SetPageWorkingset(page);
  474. lru_cache_add(page);
  475. *new_page_allocated = true;
  476. return page;
  477. fail_unlock:
  478. unlock_page(page);
  479. put_page(page);
  480. return NULL;
  481. }
  482. /*
  483. * Locate a page of swap in physical memory, reserving swap cache space
  484. * and reading the disk if it is not already cached.
  485. * A failure return means that either the page allocation failed or that
  486. * the swap entry is no longer in use.
  487. */
  488. struct page *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
  489. struct vm_area_struct *vma, unsigned long addr, bool do_poll)
  490. {
  491. bool page_was_allocated;
  492. struct page *retpage = __read_swap_cache_async(entry, gfp_mask,
  493. vma, addr, &page_was_allocated);
  494. if (page_was_allocated)
  495. swap_readpage(retpage, do_poll);
  496. return retpage;
  497. }
  498. static unsigned int __swapin_nr_pages(unsigned long prev_offset,
  499. unsigned long offset,
  500. int hits,
  501. int max_pages,
  502. int prev_win)
  503. {
  504. unsigned int pages, last_ra;
  505. /*
  506. * This heuristic has been found to work well on both sequential and
  507. * random loads, swapping to hard disk or to SSD: please don't ask
  508. * what the "+ 2" means, it just happens to work well, that's all.
  509. */
  510. pages = hits + 2;
  511. if (pages == 2) {
  512. /*
  513. * We can have no readahead hits to judge by: but must not get
  514. * stuck here forever, so check for an adjacent offset instead
  515. * (and don't even bother to check whether swap type is same).
  516. */
  517. if (offset != prev_offset + 1 && offset != prev_offset - 1)
  518. pages = 1;
  519. } else {
  520. unsigned int roundup = 4;
  521. while (roundup < pages)
  522. roundup <<= 1;
  523. pages = roundup;
  524. }
  525. if (pages > max_pages)
  526. pages = max_pages;
  527. /* Don't shrink readahead too fast */
  528. last_ra = prev_win / 2;
  529. if (pages < last_ra)
  530. pages = last_ra;
  531. return pages;
  532. }
  533. static unsigned long swapin_nr_pages(unsigned long offset)
  534. {
  535. static unsigned long prev_offset;
  536. unsigned int hits, pages, max_pages;
  537. static atomic_t last_readahead_pages;
  538. max_pages = 1 << READ_ONCE(page_cluster);
  539. if (max_pages <= 1)
  540. return 1;
  541. hits = atomic_xchg(&swapin_readahead_hits, 0);
  542. pages = __swapin_nr_pages(READ_ONCE(prev_offset), offset, hits,
  543. max_pages,
  544. atomic_read(&last_readahead_pages));
  545. if (!hits)
  546. WRITE_ONCE(prev_offset, offset);
  547. atomic_set(&last_readahead_pages, pages);
  548. return pages;
  549. }
  550. /**
  551. * swap_cluster_readahead - swap in pages in hope we need them soon
  552. * @entry: swap entry of this memory
  553. * @gfp_mask: memory allocation flags
  554. * @vmf: fault information
  555. *
  556. * Returns the struct page for entry and addr, after queueing swapin.
  557. *
  558. * Primitive swap readahead code. We simply read an aligned block of
  559. * (1 << page_cluster) entries in the swap area. This method is chosen
  560. * because it doesn't cost us any seek time. We also make sure to queue
  561. * the 'original' request together with the readahead ones...
  562. *
  563. * This has been extended to use the NUMA policies from the mm triggering
  564. * the readahead.
  565. *
  566. * Caller must hold down_read on the vma->vm_mm if vmf->vma is not NULL.
  567. * This is needed to ensure the VMA will not be freed in our back. In the case
  568. * of the speculative page fault handler, this cannot happen, even if we don't
  569. * hold the mmap_sem. Callees are assumed to take care of reading VMA's fields
  570. * using READ_ONCE() to read consistent values.
  571. */
  572. struct page *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask,
  573. struct vm_fault *vmf)
  574. {
  575. struct page *page;
  576. unsigned long entry_offset = swp_offset(entry);
  577. unsigned long offset = entry_offset;
  578. unsigned long start_offset, end_offset;
  579. unsigned long mask;
  580. struct swap_info_struct *si = swp_swap_info(entry);
  581. struct blk_plug plug;
  582. bool do_poll = true, page_allocated;
  583. struct vm_area_struct *vma = vmf->vma;
  584. unsigned long addr = vmf->address;
  585. mask = swapin_nr_pages(offset) - 1;
  586. if (!mask)
  587. goto skip;
  588. /* Test swap type to make sure the dereference is safe */
  589. if (likely(si->flags & (SWP_BLKDEV | SWP_FS_OPS))) {
  590. struct inode *inode = si->swap_file->f_mapping->host;
  591. if (inode_read_congested(inode))
  592. goto skip;
  593. }
  594. do_poll = false;
  595. /* Read a page_cluster sized and aligned cluster around offset. */
  596. start_offset = offset & ~mask;
  597. end_offset = offset | mask;
  598. if (!start_offset) /* First page is swap header. */
  599. start_offset++;
  600. if (end_offset >= si->max)
  601. end_offset = si->max - 1;
  602. blk_start_plug(&plug);
  603. for (offset = start_offset; offset <= end_offset ; offset++) {
  604. /* Ok, do the async read-ahead now */
  605. page = __read_swap_cache_async(
  606. swp_entry(swp_type(entry), offset),
  607. gfp_mask, vma, addr, &page_allocated);
  608. if (!page)
  609. continue;
  610. if (page_allocated) {
  611. swap_readpage(page, false);
  612. if (offset != entry_offset) {
  613. SetPageReadahead(page);
  614. count_vm_event(SWAP_RA);
  615. }
  616. }
  617. put_page(page);
  618. }
  619. blk_finish_plug(&plug);
  620. lru_add_drain(); /* Push any new pages onto the LRU now */
  621. skip:
  622. return read_swap_cache_async(entry, gfp_mask, vma, addr, do_poll);
  623. }
  624. int init_swap_address_space(unsigned int type, unsigned long nr_pages)
  625. {
  626. struct address_space *spaces, *space;
  627. unsigned int i, nr;
  628. nr = DIV_ROUND_UP(nr_pages, SWAP_ADDRESS_SPACE_PAGES);
  629. spaces = kvcalloc(nr, sizeof(struct address_space), GFP_KERNEL);
  630. if (!spaces)
  631. return -ENOMEM;
  632. for (i = 0; i < nr; i++) {
  633. space = spaces + i;
  634. xa_init_flags(&space->i_pages, XA_FLAGS_LOCK_IRQ);
  635. atomic_set(&space->i_mmap_writable, 0);
  636. space->a_ops = &swap_aops;
  637. /* swap cache doesn't use writeback related tags */
  638. mapping_set_no_writeback_tags(space);
  639. }
  640. nr_swapper_spaces[type] = nr;
  641. swapper_spaces[type] = spaces;
  642. return 0;
  643. }
  644. void exit_swap_address_space(unsigned int type)
  645. {
  646. kvfree(swapper_spaces[type]);
  647. nr_swapper_spaces[type] = 0;
  648. swapper_spaces[type] = NULL;
  649. }
  650. static inline void swap_ra_clamp_pfn(struct vm_area_struct *vma,
  651. unsigned long faddr,
  652. unsigned long lpfn,
  653. unsigned long rpfn,
  654. unsigned long *start,
  655. unsigned long *end)
  656. {
  657. *start = max3(lpfn, PFN_DOWN(READ_ONCE(vma->vm_start)),
  658. PFN_DOWN(faddr & PMD_MASK));
  659. *end = min3(rpfn, PFN_DOWN(READ_ONCE(vma->vm_end)),
  660. PFN_DOWN((faddr & PMD_MASK) + PMD_SIZE));
  661. }
  662. static void swap_ra_info(struct vm_fault *vmf,
  663. struct vma_swap_readahead *ra_info)
  664. {
  665. struct vm_area_struct *vma = vmf->vma;
  666. unsigned long ra_val;
  667. swp_entry_t entry;
  668. unsigned long faddr, pfn, fpfn;
  669. unsigned long start, end;
  670. pte_t *pte, *orig_pte;
  671. unsigned int max_win, hits, prev_win, win, left;
  672. #ifndef CONFIG_64BIT
  673. pte_t *tpte;
  674. #endif
  675. max_win = 1 << min_t(unsigned int, READ_ONCE(page_cluster),
  676. SWAP_RA_ORDER_CEILING);
  677. if (max_win == 1) {
  678. ra_info->win = 1;
  679. return;
  680. }
  681. faddr = vmf->address;
  682. orig_pte = pte = pte_offset_map(vmf->pmd, faddr);
  683. entry = pte_to_swp_entry(*pte);
  684. if ((unlikely(non_swap_entry(entry)))) {
  685. pte_unmap(orig_pte);
  686. return;
  687. }
  688. fpfn = PFN_DOWN(faddr);
  689. ra_val = GET_SWAP_RA_VAL(vma);
  690. pfn = PFN_DOWN(SWAP_RA_ADDR(ra_val));
  691. prev_win = SWAP_RA_WIN(ra_val);
  692. hits = SWAP_RA_HITS(ra_val);
  693. ra_info->win = win = __swapin_nr_pages(pfn, fpfn, hits,
  694. max_win, prev_win);
  695. atomic_long_set(&vma->swap_readahead_info,
  696. SWAP_RA_VAL(faddr, win, 0));
  697. if (win == 1) {
  698. pte_unmap(orig_pte);
  699. return;
  700. }
  701. /* Copy the PTEs because the page table may be unmapped */
  702. if (fpfn == pfn + 1)
  703. swap_ra_clamp_pfn(vma, faddr, fpfn, fpfn + win, &start, &end);
  704. else if (pfn == fpfn + 1)
  705. swap_ra_clamp_pfn(vma, faddr, fpfn - win + 1, fpfn + 1,
  706. &start, &end);
  707. else {
  708. left = (win - 1) / 2;
  709. swap_ra_clamp_pfn(vma, faddr, fpfn - left, fpfn + win - left,
  710. &start, &end);
  711. }
  712. ra_info->nr_pte = end - start;
  713. ra_info->offset = fpfn - start;
  714. pte -= ra_info->offset;
  715. #ifdef CONFIG_64BIT
  716. ra_info->ptes = pte;
  717. #else
  718. tpte = ra_info->ptes;
  719. for (pfn = start; pfn != end; pfn++)
  720. *tpte++ = *pte++;
  721. #endif
  722. pte_unmap(orig_pte);
  723. }
  724. /**
  725. * swap_vma_readahead - swap in pages in hope we need them soon
  726. * @fentry: swap entry of this memory
  727. * @gfp_mask: memory allocation flags
  728. * @vmf: fault information
  729. *
  730. * Returns the struct page for entry and addr, after queueing swapin.
  731. *
  732. * Primitive swap readahead code. We simply read in a few pages whoes
  733. * virtual addresses are around the fault address in the same vma.
  734. *
  735. * Caller must hold read mmap_lock if vmf->vma is not NULL.
  736. *
  737. */
  738. static struct page *swap_vma_readahead(swp_entry_t fentry, gfp_t gfp_mask,
  739. struct vm_fault *vmf)
  740. {
  741. struct blk_plug plug;
  742. struct vm_area_struct *vma = vmf->vma;
  743. struct page *page;
  744. pte_t *pte, pentry;
  745. swp_entry_t entry;
  746. unsigned int i;
  747. bool page_allocated;
  748. struct vma_swap_readahead ra_info = {0,};
  749. swap_ra_info(vmf, &ra_info);
  750. if (ra_info.win == 1)
  751. goto skip;
  752. blk_start_plug(&plug);
  753. for (i = 0, pte = ra_info.ptes; i < ra_info.nr_pte;
  754. i++, pte++) {
  755. pentry = *pte;
  756. if (pte_none(pentry))
  757. continue;
  758. if (pte_present(pentry))
  759. continue;
  760. entry = pte_to_swp_entry(pentry);
  761. if (unlikely(non_swap_entry(entry)))
  762. continue;
  763. page = __read_swap_cache_async(entry, gfp_mask, vma,
  764. vmf->address, &page_allocated);
  765. if (!page)
  766. continue;
  767. if (page_allocated) {
  768. swap_readpage(page, false);
  769. if (i != ra_info.offset) {
  770. SetPageReadahead(page);
  771. count_vm_event(SWAP_RA);
  772. }
  773. }
  774. put_page(page);
  775. }
  776. blk_finish_plug(&plug);
  777. lru_add_drain();
  778. skip:
  779. return read_swap_cache_async(fentry, gfp_mask, vma, vmf->address,
  780. ra_info.win == 1);
  781. }
  782. /**
  783. * swapin_readahead - swap in pages in hope we need them soon
  784. * @entry: swap entry of this memory
  785. * @gfp_mask: memory allocation flags
  786. * @vmf: fault information
  787. *
  788. * Returns the struct page for entry and addr, after queueing swapin.
  789. *
  790. * It's a main entry function for swap readahead. By the configuration,
  791. * it will read ahead blocks by cluster-based(ie, physical disk based)
  792. * or vma-based(ie, virtual address based on faulty address) readahead.
  793. */
  794. struct page *swapin_readahead(swp_entry_t entry, gfp_t gfp_mask,
  795. struct vm_fault *vmf)
  796. {
  797. return swap_use_vma_readahead() ?
  798. swap_vma_readahead(entry, gfp_mask, vmf) :
  799. swap_cluster_readahead(entry, gfp_mask, vmf);
  800. }
  801. #ifdef CONFIG_SYSFS
  802. static ssize_t vma_ra_enabled_show(struct kobject *kobj,
  803. struct kobj_attribute *attr, char *buf)
  804. {
  805. return sprintf(buf, "%s\n", enable_vma_readahead ? "true" : "false");
  806. }
  807. static ssize_t vma_ra_enabled_store(struct kobject *kobj,
  808. struct kobj_attribute *attr,
  809. const char *buf, size_t count)
  810. {
  811. if (!strncmp(buf, "true", 4) || !strncmp(buf, "1", 1))
  812. enable_vma_readahead = true;
  813. else if (!strncmp(buf, "false", 5) || !strncmp(buf, "0", 1))
  814. enable_vma_readahead = false;
  815. else
  816. return -EINVAL;
  817. return count;
  818. }
  819. static struct kobj_attribute vma_ra_enabled_attr =
  820. __ATTR(vma_ra_enabled, 0644, vma_ra_enabled_show,
  821. vma_ra_enabled_store);
  822. static struct attribute *swap_attrs[] = {
  823. &vma_ra_enabled_attr.attr,
  824. NULL,
  825. };
  826. static struct attribute_group swap_attr_group = {
  827. .attrs = swap_attrs,
  828. };
  829. static int __init swap_init_sysfs(void)
  830. {
  831. int err;
  832. struct kobject *swap_kobj;
  833. swap_kobj = kobject_create_and_add("swap", mm_kobj);
  834. if (!swap_kobj) {
  835. pr_err("failed to create swap kobject\n");
  836. return -ENOMEM;
  837. }
  838. err = sysfs_create_group(swap_kobj, &swap_attr_group);
  839. if (err) {
  840. pr_err("failed to register swap group\n");
  841. goto delete_obj;
  842. }
  843. return 0;
  844. delete_obj:
  845. kobject_put(swap_kobj);
  846. return err;
  847. }
  848. subsys_initcall(swap_init_sysfs);
  849. #endif