frontswap.c 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Frontswap frontend
  4. *
  5. * This code provides the generic "frontend" layer to call a matching
  6. * "backend" driver implementation of frontswap. See
  7. * Documentation/vm/frontswap.rst for more information.
  8. *
  9. * Copyright (C) 2009-2012 Oracle Corp. All rights reserved.
  10. * Author: Dan Magenheimer
  11. */
  12. #include <linux/mman.h>
  13. #include <linux/swap.h>
  14. #include <linux/swapops.h>
  15. #include <linux/security.h>
  16. #include <linux/module.h>
  17. #include <linux/debugfs.h>
  18. #include <linux/frontswap.h>
  19. #include <linux/swapfile.h>
  20. DEFINE_STATIC_KEY_FALSE(frontswap_enabled_key);
  21. /*
  22. * frontswap_ops are added by frontswap_register_ops, and provide the
  23. * frontswap "backend" implementation functions. Multiple implementations
  24. * may be registered, but implementations can never deregister. This
  25. * is a simple singly-linked list of all registered implementations.
  26. */
  27. static struct frontswap_ops *frontswap_ops __read_mostly;
  28. #define for_each_frontswap_ops(ops) \
  29. for ((ops) = frontswap_ops; (ops); (ops) = (ops)->next)
  30. /*
  31. * If enabled, frontswap_store will return failure even on success. As
  32. * a result, the swap subsystem will always write the page to swap, in
  33. * effect converting frontswap into a writethrough cache. In this mode,
  34. * there is no direct reduction in swap writes, but a frontswap backend
  35. * can unilaterally "reclaim" any pages in use with no data loss, thus
  36. * providing increases control over maximum memory usage due to frontswap.
  37. */
  38. static bool frontswap_writethrough_enabled __read_mostly;
  39. /*
  40. * If enabled, the underlying tmem implementation is capable of doing
  41. * exclusive gets, so frontswap_load, on a successful tmem_get must
  42. * mark the page as no longer in frontswap AND mark it dirty.
  43. */
  44. static bool frontswap_tmem_exclusive_gets_enabled __read_mostly;
  45. #ifdef CONFIG_DEBUG_FS
  46. /*
  47. * Counters available via /sys/kernel/debug/frontswap (if debugfs is
  48. * properly configured). These are for information only so are not protected
  49. * against increment races.
  50. */
  51. static u64 frontswap_loads;
  52. static u64 frontswap_succ_stores;
  53. static u64 frontswap_failed_stores;
  54. static u64 frontswap_invalidates;
  55. static inline void inc_frontswap_loads(void) {
  56. data_race(frontswap_loads++);
  57. }
  58. static inline void inc_frontswap_succ_stores(void) {
  59. data_race(frontswap_succ_stores++);
  60. }
  61. static inline void inc_frontswap_failed_stores(void) {
  62. data_race(frontswap_failed_stores++);
  63. }
  64. static inline void inc_frontswap_invalidates(void) {
  65. data_race(frontswap_invalidates++);
  66. }
  67. #else
  68. static inline void inc_frontswap_loads(void) { }
  69. static inline void inc_frontswap_succ_stores(void) { }
  70. static inline void inc_frontswap_failed_stores(void) { }
  71. static inline void inc_frontswap_invalidates(void) { }
  72. #endif
  73. /*
  74. * Due to the asynchronous nature of the backends loading potentially
  75. * _after_ the swap system has been activated, we have chokepoints
  76. * on all frontswap functions to not call the backend until the backend
  77. * has registered.
  78. *
  79. * This would not guards us against the user deciding to call swapoff right as
  80. * we are calling the backend to initialize (so swapon is in action).
  81. * Fortunately for us, the swapon_mutex has been taken by the callee so we are
  82. * OK. The other scenario where calls to frontswap_store (called via
  83. * swap_writepage) is racing with frontswap_invalidate_area (called via
  84. * swapoff) is again guarded by the swap subsystem.
  85. *
  86. * While no backend is registered all calls to frontswap_[store|load|
  87. * invalidate_area|invalidate_page] are ignored or fail.
  88. *
  89. * The time between the backend being registered and the swap file system
  90. * calling the backend (via the frontswap_* functions) is indeterminate as
  91. * frontswap_ops is not atomic_t (or a value guarded by a spinlock).
  92. * That is OK as we are comfortable missing some of these calls to the newly
  93. * registered backend.
  94. *
  95. * Obviously the opposite (unloading the backend) must be done after all
  96. * the frontswap_[store|load|invalidate_area|invalidate_page] start
  97. * ignoring or failing the requests. However, there is currently no way
  98. * to unload a backend once it is registered.
  99. */
  100. /*
  101. * Register operations for frontswap
  102. */
  103. void frontswap_register_ops(struct frontswap_ops *ops)
  104. {
  105. DECLARE_BITMAP(a, MAX_SWAPFILES);
  106. DECLARE_BITMAP(b, MAX_SWAPFILES);
  107. struct swap_info_struct *si;
  108. unsigned int i;
  109. bitmap_zero(a, MAX_SWAPFILES);
  110. bitmap_zero(b, MAX_SWAPFILES);
  111. spin_lock(&swap_lock);
  112. plist_for_each_entry(si, &swap_active_head, list) {
  113. if (!WARN_ON(!si->frontswap_map))
  114. set_bit(si->type, a);
  115. }
  116. spin_unlock(&swap_lock);
  117. /* the new ops needs to know the currently active swap devices */
  118. for_each_set_bit(i, a, MAX_SWAPFILES)
  119. ops->init(i);
  120. /*
  121. * Setting frontswap_ops must happen after the ops->init() calls
  122. * above; cmpxchg implies smp_mb() which will ensure the init is
  123. * complete at this point.
  124. */
  125. do {
  126. ops->next = frontswap_ops;
  127. } while (cmpxchg(&frontswap_ops, ops->next, ops) != ops->next);
  128. static_branch_inc(&frontswap_enabled_key);
  129. spin_lock(&swap_lock);
  130. plist_for_each_entry(si, &swap_active_head, list) {
  131. if (si->frontswap_map)
  132. set_bit(si->type, b);
  133. }
  134. spin_unlock(&swap_lock);
  135. /*
  136. * On the very unlikely chance that a swap device was added or
  137. * removed between setting the "a" list bits and the ops init
  138. * calls, we re-check and do init or invalidate for any changed
  139. * bits.
  140. */
  141. if (unlikely(!bitmap_equal(a, b, MAX_SWAPFILES))) {
  142. for (i = 0; i < MAX_SWAPFILES; i++) {
  143. if (!test_bit(i, a) && test_bit(i, b))
  144. ops->init(i);
  145. else if (test_bit(i, a) && !test_bit(i, b))
  146. ops->invalidate_area(i);
  147. }
  148. }
  149. }
  150. EXPORT_SYMBOL(frontswap_register_ops);
  151. /*
  152. * Enable/disable frontswap writethrough (see above).
  153. */
  154. void frontswap_writethrough(bool enable)
  155. {
  156. frontswap_writethrough_enabled = enable;
  157. }
  158. EXPORT_SYMBOL(frontswap_writethrough);
  159. /*
  160. * Enable/disable frontswap exclusive gets (see above).
  161. */
  162. void frontswap_tmem_exclusive_gets(bool enable)
  163. {
  164. frontswap_tmem_exclusive_gets_enabled = enable;
  165. }
  166. EXPORT_SYMBOL(frontswap_tmem_exclusive_gets);
  167. /*
  168. * Called when a swap device is swapon'd.
  169. */
  170. void __frontswap_init(unsigned type, unsigned long *map)
  171. {
  172. struct swap_info_struct *sis = swap_info[type];
  173. struct frontswap_ops *ops;
  174. VM_BUG_ON(sis == NULL);
  175. /*
  176. * p->frontswap is a bitmap that we MUST have to figure out which page
  177. * has gone in frontswap. Without it there is no point of continuing.
  178. */
  179. if (WARN_ON(!map))
  180. return;
  181. /*
  182. * Irregardless of whether the frontswap backend has been loaded
  183. * before this function or it will be later, we _MUST_ have the
  184. * p->frontswap set to something valid to work properly.
  185. */
  186. frontswap_map_set(sis, map);
  187. for_each_frontswap_ops(ops)
  188. ops->init(type);
  189. }
  190. EXPORT_SYMBOL(__frontswap_init);
  191. bool __frontswap_test(struct swap_info_struct *sis,
  192. pgoff_t offset)
  193. {
  194. if (sis->frontswap_map)
  195. return test_bit(offset, sis->frontswap_map);
  196. return false;
  197. }
  198. EXPORT_SYMBOL(__frontswap_test);
  199. static inline void __frontswap_set(struct swap_info_struct *sis,
  200. pgoff_t offset)
  201. {
  202. set_bit(offset, sis->frontswap_map);
  203. atomic_inc(&sis->frontswap_pages);
  204. }
  205. static inline void __frontswap_clear(struct swap_info_struct *sis,
  206. pgoff_t offset)
  207. {
  208. clear_bit(offset, sis->frontswap_map);
  209. atomic_dec(&sis->frontswap_pages);
  210. }
  211. /*
  212. * "Store" data from a page to frontswap and associate it with the page's
  213. * swaptype and offset. Page must be locked and in the swap cache.
  214. * If frontswap already contains a page with matching swaptype and
  215. * offset, the frontswap implementation may either overwrite the data and
  216. * return success or invalidate the page from frontswap and return failure.
  217. */
  218. int __frontswap_store(struct page *page)
  219. {
  220. int ret = -1;
  221. swp_entry_t entry = { .val = page_private(page), };
  222. int type = swp_type(entry);
  223. struct swap_info_struct *sis = swap_info[type];
  224. pgoff_t offset = swp_offset(entry);
  225. struct frontswap_ops *ops;
  226. VM_BUG_ON(!frontswap_ops);
  227. VM_BUG_ON(!PageLocked(page));
  228. VM_BUG_ON(sis == NULL);
  229. /*
  230. * If a dup, we must remove the old page first; we can't leave the
  231. * old page no matter if the store of the new page succeeds or fails,
  232. * and we can't rely on the new page replacing the old page as we may
  233. * not store to the same implementation that contains the old page.
  234. */
  235. if (__frontswap_test(sis, offset)) {
  236. __frontswap_clear(sis, offset);
  237. for_each_frontswap_ops(ops)
  238. ops->invalidate_page(type, offset);
  239. }
  240. /* Try to store in each implementation, until one succeeds. */
  241. for_each_frontswap_ops(ops) {
  242. ret = ops->store(type, offset, page);
  243. if (!ret) /* successful store */
  244. break;
  245. }
  246. if (ret == 0) {
  247. __frontswap_set(sis, offset);
  248. inc_frontswap_succ_stores();
  249. } else {
  250. inc_frontswap_failed_stores();
  251. }
  252. if (frontswap_writethrough_enabled)
  253. /* report failure so swap also writes to swap device */
  254. ret = -1;
  255. return ret;
  256. }
  257. EXPORT_SYMBOL(__frontswap_store);
  258. /*
  259. * "Get" data from frontswap associated with swaptype and offset that were
  260. * specified when the data was put to frontswap and use it to fill the
  261. * specified page with data. Page must be locked and in the swap cache.
  262. */
  263. int __frontswap_load(struct page *page)
  264. {
  265. int ret = -1;
  266. swp_entry_t entry = { .val = page_private(page), };
  267. int type = swp_type(entry);
  268. struct swap_info_struct *sis = swap_info[type];
  269. pgoff_t offset = swp_offset(entry);
  270. struct frontswap_ops *ops;
  271. VM_BUG_ON(!frontswap_ops);
  272. VM_BUG_ON(!PageLocked(page));
  273. VM_BUG_ON(sis == NULL);
  274. if (!__frontswap_test(sis, offset))
  275. return -1;
  276. /* Try loading from each implementation, until one succeeds. */
  277. for_each_frontswap_ops(ops) {
  278. ret = ops->load(type, offset, page);
  279. if (!ret) /* successful load */
  280. break;
  281. }
  282. if (ret == 0) {
  283. inc_frontswap_loads();
  284. if (frontswap_tmem_exclusive_gets_enabled) {
  285. SetPageDirty(page);
  286. __frontswap_clear(sis, offset);
  287. }
  288. }
  289. return ret;
  290. }
  291. EXPORT_SYMBOL(__frontswap_load);
  292. /*
  293. * Invalidate any data from frontswap associated with the specified swaptype
  294. * and offset so that a subsequent "get" will fail.
  295. */
  296. void __frontswap_invalidate_page(unsigned type, pgoff_t offset)
  297. {
  298. struct swap_info_struct *sis = swap_info[type];
  299. struct frontswap_ops *ops;
  300. VM_BUG_ON(!frontswap_ops);
  301. VM_BUG_ON(sis == NULL);
  302. if (!__frontswap_test(sis, offset))
  303. return;
  304. for_each_frontswap_ops(ops)
  305. ops->invalidate_page(type, offset);
  306. __frontswap_clear(sis, offset);
  307. inc_frontswap_invalidates();
  308. }
  309. EXPORT_SYMBOL(__frontswap_invalidate_page);
  310. /*
  311. * Invalidate all data from frontswap associated with all offsets for the
  312. * specified swaptype.
  313. */
  314. void __frontswap_invalidate_area(unsigned type)
  315. {
  316. struct swap_info_struct *sis = swap_info[type];
  317. struct frontswap_ops *ops;
  318. VM_BUG_ON(!frontswap_ops);
  319. VM_BUG_ON(sis == NULL);
  320. if (sis->frontswap_map == NULL)
  321. return;
  322. for_each_frontswap_ops(ops)
  323. ops->invalidate_area(type);
  324. atomic_set(&sis->frontswap_pages, 0);
  325. bitmap_zero(sis->frontswap_map, sis->max);
  326. }
  327. EXPORT_SYMBOL(__frontswap_invalidate_area);
  328. static unsigned long __frontswap_curr_pages(void)
  329. {
  330. unsigned long totalpages = 0;
  331. struct swap_info_struct *si = NULL;
  332. assert_spin_locked(&swap_lock);
  333. plist_for_each_entry(si, &swap_active_head, list)
  334. totalpages += atomic_read(&si->frontswap_pages);
  335. return totalpages;
  336. }
  337. static int __frontswap_unuse_pages(unsigned long total, unsigned long *unused,
  338. int *swapid)
  339. {
  340. int ret = -EINVAL;
  341. struct swap_info_struct *si = NULL;
  342. int si_frontswap_pages;
  343. unsigned long total_pages_to_unuse = total;
  344. unsigned long pages = 0, pages_to_unuse = 0;
  345. assert_spin_locked(&swap_lock);
  346. plist_for_each_entry(si, &swap_active_head, list) {
  347. si_frontswap_pages = atomic_read(&si->frontswap_pages);
  348. if (total_pages_to_unuse < si_frontswap_pages) {
  349. pages = pages_to_unuse = total_pages_to_unuse;
  350. } else {
  351. pages = si_frontswap_pages;
  352. pages_to_unuse = 0; /* unuse all */
  353. }
  354. /* ensure there is enough RAM to fetch pages from frontswap */
  355. if (security_vm_enough_memory_mm(current->mm, pages)) {
  356. ret = -ENOMEM;
  357. continue;
  358. }
  359. vm_unacct_memory(pages);
  360. *unused = pages_to_unuse;
  361. *swapid = si->type;
  362. ret = 0;
  363. break;
  364. }
  365. return ret;
  366. }
  367. /*
  368. * Used to check if it's necessary and feasible to unuse pages.
  369. * Return 1 when nothing to do, 0 when need to shrink pages,
  370. * error code when there is an error.
  371. */
  372. static int __frontswap_shrink(unsigned long target_pages,
  373. unsigned long *pages_to_unuse,
  374. int *type)
  375. {
  376. unsigned long total_pages = 0, total_pages_to_unuse;
  377. assert_spin_locked(&swap_lock);
  378. total_pages = __frontswap_curr_pages();
  379. if (total_pages <= target_pages) {
  380. /* Nothing to do */
  381. *pages_to_unuse = 0;
  382. return 1;
  383. }
  384. total_pages_to_unuse = total_pages - target_pages;
  385. return __frontswap_unuse_pages(total_pages_to_unuse, pages_to_unuse, type);
  386. }
  387. /*
  388. * Frontswap, like a true swap device, may unnecessarily retain pages
  389. * under certain circumstances; "shrink" frontswap is essentially a
  390. * "partial swapoff" and works by calling try_to_unuse to attempt to
  391. * unuse enough frontswap pages to attempt to -- subject to memory
  392. * constraints -- reduce the number of pages in frontswap to the
  393. * number given in the parameter target_pages.
  394. */
  395. void frontswap_shrink(unsigned long target_pages)
  396. {
  397. unsigned long pages_to_unuse = 0;
  398. int type, ret;
  399. /*
  400. * we don't want to hold swap_lock while doing a very
  401. * lengthy try_to_unuse, but swap_list may change
  402. * so restart scan from swap_active_head each time
  403. */
  404. spin_lock(&swap_lock);
  405. ret = __frontswap_shrink(target_pages, &pages_to_unuse, &type);
  406. spin_unlock(&swap_lock);
  407. if (ret == 0)
  408. try_to_unuse(type, true, pages_to_unuse);
  409. return;
  410. }
  411. EXPORT_SYMBOL(frontswap_shrink);
  412. /*
  413. * Count and return the number of frontswap pages across all
  414. * swap devices. This is exported so that backend drivers can
  415. * determine current usage without reading debugfs.
  416. */
  417. unsigned long frontswap_curr_pages(void)
  418. {
  419. unsigned long totalpages = 0;
  420. spin_lock(&swap_lock);
  421. totalpages = __frontswap_curr_pages();
  422. spin_unlock(&swap_lock);
  423. return totalpages;
  424. }
  425. EXPORT_SYMBOL(frontswap_curr_pages);
  426. static int __init init_frontswap(void)
  427. {
  428. #ifdef CONFIG_DEBUG_FS
  429. struct dentry *root = debugfs_create_dir("frontswap", NULL);
  430. if (root == NULL)
  431. return -ENXIO;
  432. debugfs_create_u64("loads", 0444, root, &frontswap_loads);
  433. debugfs_create_u64("succ_stores", 0444, root, &frontswap_succ_stores);
  434. debugfs_create_u64("failed_stores", 0444, root,
  435. &frontswap_failed_stores);
  436. debugfs_create_u64("invalidates", 0444, root, &frontswap_invalidates);
  437. #endif
  438. return 0;
  439. }
  440. module_init(init_frontswap);