zswap.c 35 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * zswap.c - zswap driver file
  4. *
  5. * zswap is a backend for frontswap that takes pages that are in the process
  6. * of being swapped out and attempts to compress and store them in a
  7. * RAM-based memory pool. This can result in a significant I/O reduction on
  8. * the swap device and, in the case where decompressing from RAM is faster
  9. * than reading from the swap device, can also improve workload performance.
  10. *
  11. * Copyright (C) 2012 Seth Jennings <sjenning@linux.vnet.ibm.com>
  12. */
  13. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  14. #include <linux/module.h>
  15. #include <linux/cpu.h>
  16. #include <linux/highmem.h>
  17. #include <linux/slab.h>
  18. #include <linux/spinlock.h>
  19. #include <linux/types.h>
  20. #include <linux/atomic.h>
  21. #include <linux/frontswap.h>
  22. #include <linux/rbtree.h>
  23. #include <linux/swap.h>
  24. #include <linux/crypto.h>
  25. #include <linux/mempool.h>
  26. #include <linux/zpool.h>
  27. #include <linux/mm_types.h>
  28. #include <linux/page-flags.h>
  29. #include <linux/swapops.h>
  30. #include <linux/writeback.h>
  31. #include <linux/pagemap.h>
  32. #include <linux/workqueue.h>
  33. /*********************************
  34. * statistics
  35. **********************************/
  36. /* Total bytes used by the compressed storage */
  37. static u64 zswap_pool_total_size;
  38. /* The number of compressed pages currently stored in zswap */
  39. static atomic_t zswap_stored_pages = ATOMIC_INIT(0);
  40. /* The number of same-value filled pages currently stored in zswap */
  41. static atomic_t zswap_same_filled_pages = ATOMIC_INIT(0);
  42. /*
  43. * The statistics below are not protected from concurrent access for
  44. * performance reasons so they may not be a 100% accurate. However,
  45. * they do provide useful information on roughly how many times a
  46. * certain event is occurring.
  47. */
  48. /* Pool limit was hit (see zswap_max_pool_percent) */
  49. static u64 zswap_pool_limit_hit;
  50. /* Pages written back when pool limit was reached */
  51. static u64 zswap_written_back_pages;
  52. /* Store failed due to a reclaim failure after pool limit was reached */
  53. static u64 zswap_reject_reclaim_fail;
  54. /* Compressed page was too big for the allocator to (optimally) store */
  55. static u64 zswap_reject_compress_poor;
  56. /* Store failed because underlying allocator could not get memory */
  57. static u64 zswap_reject_alloc_fail;
  58. /* Store failed because the entry metadata could not be allocated (rare) */
  59. static u64 zswap_reject_kmemcache_fail;
  60. /* Duplicate store was encountered (rare) */
  61. static u64 zswap_duplicate_entry;
  62. /* Shrinker work queue */
  63. static struct workqueue_struct *shrink_wq;
  64. /* Pool limit was hit, we need to calm down */
  65. static bool zswap_pool_reached_full;
  66. /*********************************
  67. * tunables
  68. **********************************/
  69. #define ZSWAP_PARAM_UNSET ""
  70. /* Enable/disable zswap */
  71. static bool zswap_enabled = IS_ENABLED(CONFIG_ZSWAP_DEFAULT_ON);
  72. static int zswap_enabled_param_set(const char *,
  73. const struct kernel_param *);
  74. static struct kernel_param_ops zswap_enabled_param_ops = {
  75. .set = zswap_enabled_param_set,
  76. .get = param_get_bool,
  77. };
  78. module_param_cb(enabled, &zswap_enabled_param_ops, &zswap_enabled, 0644);
  79. /* Crypto compressor to use */
  80. static char *zswap_compressor = CONFIG_ZSWAP_COMPRESSOR_DEFAULT;
  81. static int zswap_compressor_param_set(const char *,
  82. const struct kernel_param *);
  83. static struct kernel_param_ops zswap_compressor_param_ops = {
  84. .set = zswap_compressor_param_set,
  85. .get = param_get_charp,
  86. .free = param_free_charp,
  87. };
  88. module_param_cb(compressor, &zswap_compressor_param_ops,
  89. &zswap_compressor, 0644);
  90. /* Compressed storage zpool to use */
  91. static char *zswap_zpool_type = CONFIG_ZSWAP_ZPOOL_DEFAULT;
  92. static int zswap_zpool_param_set(const char *, const struct kernel_param *);
  93. static struct kernel_param_ops zswap_zpool_param_ops = {
  94. .set = zswap_zpool_param_set,
  95. .get = param_get_charp,
  96. .free = param_free_charp,
  97. };
  98. module_param_cb(zpool, &zswap_zpool_param_ops, &zswap_zpool_type, 0644);
  99. /* The maximum percentage of memory that the compressed pool can occupy */
  100. static unsigned int zswap_max_pool_percent = 20;
  101. module_param_named(max_pool_percent, zswap_max_pool_percent, uint, 0644);
  102. /* The threshold for accepting new pages after the max_pool_percent was hit */
  103. static unsigned int zswap_accept_thr_percent = 90; /* of max pool size */
  104. module_param_named(accept_threshold_percent, zswap_accept_thr_percent,
  105. uint, 0644);
  106. /* Enable/disable handling same-value filled pages (enabled by default) */
  107. static bool zswap_same_filled_pages_enabled = true;
  108. module_param_named(same_filled_pages_enabled, zswap_same_filled_pages_enabled,
  109. bool, 0644);
  110. /*********************************
  111. * data structures
  112. **********************************/
  113. struct zswap_pool {
  114. struct zpool *zpool;
  115. struct crypto_comp * __percpu *tfm;
  116. struct kref kref;
  117. struct list_head list;
  118. struct work_struct release_work;
  119. struct work_struct shrink_work;
  120. struct hlist_node node;
  121. char tfm_name[CRYPTO_MAX_ALG_NAME];
  122. };
  123. /*
  124. * struct zswap_entry
  125. *
  126. * This structure contains the metadata for tracking a single compressed
  127. * page within zswap.
  128. *
  129. * rbnode - links the entry into red-black tree for the appropriate swap type
  130. * offset - the swap offset for the entry. Index into the red-black tree.
  131. * refcount - the number of outstanding reference to the entry. This is needed
  132. * to protect against premature freeing of the entry by code
  133. * concurrent calls to load, invalidate, and writeback. The lock
  134. * for the zswap_tree structure that contains the entry must
  135. * be held while changing the refcount. Since the lock must
  136. * be held, there is no reason to also make refcount atomic.
  137. * length - the length in bytes of the compressed page data. Needed during
  138. * decompression. For a same value filled page length is 0.
  139. * pool - the zswap_pool the entry's data is in
  140. * handle - zpool allocation handle that stores the compressed page data
  141. * value - value of the same-value filled pages which have same content
  142. */
  143. struct zswap_entry {
  144. struct rb_node rbnode;
  145. pgoff_t offset;
  146. int refcount;
  147. unsigned int length;
  148. struct zswap_pool *pool;
  149. union {
  150. unsigned long handle;
  151. unsigned long value;
  152. };
  153. };
  154. struct zswap_header {
  155. swp_entry_t swpentry;
  156. };
  157. /*
  158. * The tree lock in the zswap_tree struct protects a few things:
  159. * - the rbtree
  160. * - the refcount field of each entry in the tree
  161. */
  162. struct zswap_tree {
  163. struct rb_root rbroot;
  164. spinlock_t lock;
  165. };
  166. static struct zswap_tree *zswap_trees[MAX_SWAPFILES];
  167. /* RCU-protected iteration */
  168. static LIST_HEAD(zswap_pools);
  169. /* protects zswap_pools list modification */
  170. static DEFINE_SPINLOCK(zswap_pools_lock);
  171. /* pool counter to provide unique names to zpool */
  172. static atomic_t zswap_pools_count = ATOMIC_INIT(0);
  173. /* used by param callback function */
  174. static bool zswap_init_started;
  175. /* fatal error during init */
  176. static bool zswap_init_failed;
  177. /* init completed, but couldn't create the initial pool */
  178. static bool zswap_has_pool;
  179. /*********************************
  180. * helpers and fwd declarations
  181. **********************************/
  182. #define zswap_pool_debug(msg, p) \
  183. pr_debug("%s pool %s/%s\n", msg, (p)->tfm_name, \
  184. zpool_get_type((p)->zpool))
  185. static int zswap_writeback_entry(struct zpool *pool, unsigned long handle);
  186. static int zswap_pool_get(struct zswap_pool *pool);
  187. static void zswap_pool_put(struct zswap_pool *pool);
  188. static const struct zpool_ops zswap_zpool_ops = {
  189. .evict = zswap_writeback_entry
  190. };
  191. static bool zswap_is_full(void)
  192. {
  193. return totalram_pages() * zswap_max_pool_percent / 100 <
  194. DIV_ROUND_UP(zswap_pool_total_size, PAGE_SIZE);
  195. }
  196. static bool zswap_can_accept(void)
  197. {
  198. return totalram_pages() * zswap_accept_thr_percent / 100 *
  199. zswap_max_pool_percent / 100 >
  200. DIV_ROUND_UP(zswap_pool_total_size, PAGE_SIZE);
  201. }
  202. static void zswap_update_total_size(void)
  203. {
  204. struct zswap_pool *pool;
  205. u64 total = 0;
  206. rcu_read_lock();
  207. list_for_each_entry_rcu(pool, &zswap_pools, list)
  208. total += zpool_get_total_size(pool->zpool);
  209. rcu_read_unlock();
  210. zswap_pool_total_size = total;
  211. }
  212. /*********************************
  213. * zswap entry functions
  214. **********************************/
  215. static struct kmem_cache *zswap_entry_cache;
  216. static int __init zswap_entry_cache_create(void)
  217. {
  218. zswap_entry_cache = KMEM_CACHE(zswap_entry, 0);
  219. return zswap_entry_cache == NULL;
  220. }
  221. static void __init zswap_entry_cache_destroy(void)
  222. {
  223. kmem_cache_destroy(zswap_entry_cache);
  224. }
  225. static struct zswap_entry *zswap_entry_cache_alloc(gfp_t gfp)
  226. {
  227. struct zswap_entry *entry;
  228. entry = kmem_cache_alloc(zswap_entry_cache, gfp);
  229. if (!entry)
  230. return NULL;
  231. entry->refcount = 1;
  232. RB_CLEAR_NODE(&entry->rbnode);
  233. return entry;
  234. }
  235. static void zswap_entry_cache_free(struct zswap_entry *entry)
  236. {
  237. kmem_cache_free(zswap_entry_cache, entry);
  238. }
  239. /*********************************
  240. * rbtree functions
  241. **********************************/
  242. static struct zswap_entry *zswap_rb_search(struct rb_root *root, pgoff_t offset)
  243. {
  244. struct rb_node *node = root->rb_node;
  245. struct zswap_entry *entry;
  246. while (node) {
  247. entry = rb_entry(node, struct zswap_entry, rbnode);
  248. if (entry->offset > offset)
  249. node = node->rb_left;
  250. else if (entry->offset < offset)
  251. node = node->rb_right;
  252. else
  253. return entry;
  254. }
  255. return NULL;
  256. }
  257. /*
  258. * In the case that a entry with the same offset is found, a pointer to
  259. * the existing entry is stored in dupentry and the function returns -EEXIST
  260. */
  261. static int zswap_rb_insert(struct rb_root *root, struct zswap_entry *entry,
  262. struct zswap_entry **dupentry)
  263. {
  264. struct rb_node **link = &root->rb_node, *parent = NULL;
  265. struct zswap_entry *myentry;
  266. while (*link) {
  267. parent = *link;
  268. myentry = rb_entry(parent, struct zswap_entry, rbnode);
  269. if (myentry->offset > entry->offset)
  270. link = &(*link)->rb_left;
  271. else if (myentry->offset < entry->offset)
  272. link = &(*link)->rb_right;
  273. else {
  274. *dupentry = myentry;
  275. return -EEXIST;
  276. }
  277. }
  278. rb_link_node(&entry->rbnode, parent, link);
  279. rb_insert_color(&entry->rbnode, root);
  280. return 0;
  281. }
  282. static void zswap_rb_erase(struct rb_root *root, struct zswap_entry *entry)
  283. {
  284. if (!RB_EMPTY_NODE(&entry->rbnode)) {
  285. rb_erase(&entry->rbnode, root);
  286. RB_CLEAR_NODE(&entry->rbnode);
  287. }
  288. }
  289. /*
  290. * Carries out the common pattern of freeing and entry's zpool allocation,
  291. * freeing the entry itself, and decrementing the number of stored pages.
  292. */
  293. static void zswap_free_entry(struct zswap_entry *entry)
  294. {
  295. if (!entry->length)
  296. atomic_dec(&zswap_same_filled_pages);
  297. else {
  298. zpool_free(entry->pool->zpool, entry->handle);
  299. zswap_pool_put(entry->pool);
  300. }
  301. zswap_entry_cache_free(entry);
  302. atomic_dec(&zswap_stored_pages);
  303. zswap_update_total_size();
  304. }
  305. /* caller must hold the tree lock */
  306. static void zswap_entry_get(struct zswap_entry *entry)
  307. {
  308. entry->refcount++;
  309. }
  310. /* caller must hold the tree lock
  311. * remove from the tree and free it, if nobody reference the entry
  312. */
  313. static void zswap_entry_put(struct zswap_tree *tree,
  314. struct zswap_entry *entry)
  315. {
  316. int refcount = --entry->refcount;
  317. BUG_ON(refcount < 0);
  318. if (refcount == 0) {
  319. zswap_rb_erase(&tree->rbroot, entry);
  320. zswap_free_entry(entry);
  321. }
  322. }
  323. /* caller must hold the tree lock */
  324. static struct zswap_entry *zswap_entry_find_get(struct rb_root *root,
  325. pgoff_t offset)
  326. {
  327. struct zswap_entry *entry;
  328. entry = zswap_rb_search(root, offset);
  329. if (entry)
  330. zswap_entry_get(entry);
  331. return entry;
  332. }
  333. /*********************************
  334. * per-cpu code
  335. **********************************/
  336. static DEFINE_PER_CPU(u8 *, zswap_dstmem);
  337. static int zswap_dstmem_prepare(unsigned int cpu)
  338. {
  339. u8 *dst;
  340. dst = kmalloc_node(PAGE_SIZE * 2, GFP_KERNEL, cpu_to_node(cpu));
  341. if (!dst)
  342. return -ENOMEM;
  343. per_cpu(zswap_dstmem, cpu) = dst;
  344. return 0;
  345. }
  346. static int zswap_dstmem_dead(unsigned int cpu)
  347. {
  348. u8 *dst;
  349. dst = per_cpu(zswap_dstmem, cpu);
  350. kfree(dst);
  351. per_cpu(zswap_dstmem, cpu) = NULL;
  352. return 0;
  353. }
  354. static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node)
  355. {
  356. struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node);
  357. struct crypto_comp *tfm;
  358. if (WARN_ON(*per_cpu_ptr(pool->tfm, cpu)))
  359. return 0;
  360. tfm = crypto_alloc_comp(pool->tfm_name, 0, 0);
  361. if (IS_ERR_OR_NULL(tfm)) {
  362. pr_err("could not alloc crypto comp %s : %ld\n",
  363. pool->tfm_name, PTR_ERR(tfm));
  364. return -ENOMEM;
  365. }
  366. *per_cpu_ptr(pool->tfm, cpu) = tfm;
  367. return 0;
  368. }
  369. static int zswap_cpu_comp_dead(unsigned int cpu, struct hlist_node *node)
  370. {
  371. struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node);
  372. struct crypto_comp *tfm;
  373. tfm = *per_cpu_ptr(pool->tfm, cpu);
  374. if (!IS_ERR_OR_NULL(tfm))
  375. crypto_free_comp(tfm);
  376. *per_cpu_ptr(pool->tfm, cpu) = NULL;
  377. return 0;
  378. }
  379. /*********************************
  380. * pool functions
  381. **********************************/
  382. static struct zswap_pool *__zswap_pool_current(void)
  383. {
  384. struct zswap_pool *pool;
  385. pool = list_first_or_null_rcu(&zswap_pools, typeof(*pool), list);
  386. WARN_ONCE(!pool && zswap_has_pool,
  387. "%s: no page storage pool!\n", __func__);
  388. return pool;
  389. }
  390. static struct zswap_pool *zswap_pool_current(void)
  391. {
  392. assert_spin_locked(&zswap_pools_lock);
  393. return __zswap_pool_current();
  394. }
  395. static struct zswap_pool *zswap_pool_current_get(void)
  396. {
  397. struct zswap_pool *pool;
  398. rcu_read_lock();
  399. pool = __zswap_pool_current();
  400. if (!zswap_pool_get(pool))
  401. pool = NULL;
  402. rcu_read_unlock();
  403. return pool;
  404. }
  405. static struct zswap_pool *zswap_pool_last_get(void)
  406. {
  407. struct zswap_pool *pool, *last = NULL;
  408. rcu_read_lock();
  409. list_for_each_entry_rcu(pool, &zswap_pools, list)
  410. last = pool;
  411. WARN_ONCE(!last && zswap_has_pool,
  412. "%s: no page storage pool!\n", __func__);
  413. if (!zswap_pool_get(last))
  414. last = NULL;
  415. rcu_read_unlock();
  416. return last;
  417. }
  418. /* type and compressor must be null-terminated */
  419. static struct zswap_pool *zswap_pool_find_get(char *type, char *compressor)
  420. {
  421. struct zswap_pool *pool;
  422. assert_spin_locked(&zswap_pools_lock);
  423. list_for_each_entry_rcu(pool, &zswap_pools, list) {
  424. if (strcmp(pool->tfm_name, compressor))
  425. continue;
  426. if (strcmp(zpool_get_type(pool->zpool), type))
  427. continue;
  428. /* if we can't get it, it's about to be destroyed */
  429. if (!zswap_pool_get(pool))
  430. continue;
  431. return pool;
  432. }
  433. return NULL;
  434. }
  435. static void shrink_worker(struct work_struct *w)
  436. {
  437. struct zswap_pool *pool = container_of(w, typeof(*pool),
  438. shrink_work);
  439. if (zpool_shrink(pool->zpool, 1, NULL))
  440. zswap_reject_reclaim_fail++;
  441. zswap_pool_put(pool);
  442. }
  443. static struct zswap_pool *zswap_pool_create(char *type, char *compressor)
  444. {
  445. struct zswap_pool *pool;
  446. char name[38]; /* 'zswap' + 32 char (max) num + \0 */
  447. gfp_t gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM;
  448. int ret;
  449. if (!zswap_has_pool) {
  450. /* if either are unset, pool initialization failed, and we
  451. * need both params to be set correctly before trying to
  452. * create a pool.
  453. */
  454. if (!strcmp(type, ZSWAP_PARAM_UNSET))
  455. return NULL;
  456. if (!strcmp(compressor, ZSWAP_PARAM_UNSET))
  457. return NULL;
  458. }
  459. pool = kzalloc(sizeof(*pool), GFP_KERNEL);
  460. if (!pool)
  461. return NULL;
  462. /* unique name for each pool specifically required by zsmalloc */
  463. snprintf(name, 38, "zswap%x", atomic_inc_return(&zswap_pools_count));
  464. pool->zpool = zpool_create_pool(type, name, gfp, &zswap_zpool_ops);
  465. if (!pool->zpool) {
  466. pr_err("%s zpool not available\n", type);
  467. goto error;
  468. }
  469. pr_debug("using %s zpool\n", zpool_get_type(pool->zpool));
  470. strlcpy(pool->tfm_name, compressor, sizeof(pool->tfm_name));
  471. pool->tfm = alloc_percpu(struct crypto_comp *);
  472. if (!pool->tfm) {
  473. pr_err("percpu alloc failed\n");
  474. goto error;
  475. }
  476. ret = cpuhp_state_add_instance(CPUHP_MM_ZSWP_POOL_PREPARE,
  477. &pool->node);
  478. if (ret)
  479. goto error;
  480. pr_debug("using %s compressor\n", pool->tfm_name);
  481. /* being the current pool takes 1 ref; this func expects the
  482. * caller to always add the new pool as the current pool
  483. */
  484. kref_init(&pool->kref);
  485. INIT_LIST_HEAD(&pool->list);
  486. INIT_WORK(&pool->shrink_work, shrink_worker);
  487. zswap_pool_debug("created", pool);
  488. return pool;
  489. error:
  490. free_percpu(pool->tfm);
  491. if (pool->zpool)
  492. zpool_destroy_pool(pool->zpool);
  493. kfree(pool);
  494. return NULL;
  495. }
  496. static __init struct zswap_pool *__zswap_pool_create_fallback(void)
  497. {
  498. bool has_comp, has_zpool;
  499. has_comp = crypto_has_comp(zswap_compressor, 0, 0);
  500. if (!has_comp && strcmp(zswap_compressor,
  501. CONFIG_ZSWAP_COMPRESSOR_DEFAULT)) {
  502. pr_err("compressor %s not available, using default %s\n",
  503. zswap_compressor, CONFIG_ZSWAP_COMPRESSOR_DEFAULT);
  504. param_free_charp(&zswap_compressor);
  505. zswap_compressor = CONFIG_ZSWAP_COMPRESSOR_DEFAULT;
  506. has_comp = crypto_has_comp(zswap_compressor, 0, 0);
  507. }
  508. if (!has_comp) {
  509. pr_err("default compressor %s not available\n",
  510. zswap_compressor);
  511. param_free_charp(&zswap_compressor);
  512. zswap_compressor = ZSWAP_PARAM_UNSET;
  513. }
  514. has_zpool = zpool_has_pool(zswap_zpool_type);
  515. if (!has_zpool && strcmp(zswap_zpool_type,
  516. CONFIG_ZSWAP_ZPOOL_DEFAULT)) {
  517. pr_err("zpool %s not available, using default %s\n",
  518. zswap_zpool_type, CONFIG_ZSWAP_ZPOOL_DEFAULT);
  519. param_free_charp(&zswap_zpool_type);
  520. zswap_zpool_type = CONFIG_ZSWAP_ZPOOL_DEFAULT;
  521. has_zpool = zpool_has_pool(zswap_zpool_type);
  522. }
  523. if (!has_zpool) {
  524. pr_err("default zpool %s not available\n",
  525. zswap_zpool_type);
  526. param_free_charp(&zswap_zpool_type);
  527. zswap_zpool_type = ZSWAP_PARAM_UNSET;
  528. }
  529. if (!has_comp || !has_zpool)
  530. return NULL;
  531. return zswap_pool_create(zswap_zpool_type, zswap_compressor);
  532. }
  533. static void zswap_pool_destroy(struct zswap_pool *pool)
  534. {
  535. zswap_pool_debug("destroying", pool);
  536. cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node);
  537. free_percpu(pool->tfm);
  538. zpool_destroy_pool(pool->zpool);
  539. kfree(pool);
  540. }
  541. static int __must_check zswap_pool_get(struct zswap_pool *pool)
  542. {
  543. if (!pool)
  544. return 0;
  545. return kref_get_unless_zero(&pool->kref);
  546. }
  547. static void __zswap_pool_release(struct work_struct *work)
  548. {
  549. struct zswap_pool *pool = container_of(work, typeof(*pool),
  550. release_work);
  551. synchronize_rcu();
  552. /* nobody should have been able to get a kref... */
  553. WARN_ON(kref_get_unless_zero(&pool->kref));
  554. /* pool is now off zswap_pools list and has no references. */
  555. zswap_pool_destroy(pool);
  556. }
  557. static void __zswap_pool_empty(struct kref *kref)
  558. {
  559. struct zswap_pool *pool;
  560. pool = container_of(kref, typeof(*pool), kref);
  561. spin_lock(&zswap_pools_lock);
  562. WARN_ON(pool == zswap_pool_current());
  563. list_del_rcu(&pool->list);
  564. INIT_WORK(&pool->release_work, __zswap_pool_release);
  565. schedule_work(&pool->release_work);
  566. spin_unlock(&zswap_pools_lock);
  567. }
  568. static void zswap_pool_put(struct zswap_pool *pool)
  569. {
  570. kref_put(&pool->kref, __zswap_pool_empty);
  571. }
  572. /*********************************
  573. * param callbacks
  574. **********************************/
  575. /* val must be a null-terminated string */
  576. static int __zswap_param_set(const char *val, const struct kernel_param *kp,
  577. char *type, char *compressor)
  578. {
  579. struct zswap_pool *pool, *put_pool = NULL;
  580. char *s = strstrip((char *)val);
  581. int ret;
  582. if (zswap_init_failed) {
  583. pr_err("can't set param, initialization failed\n");
  584. return -ENODEV;
  585. }
  586. /* no change required */
  587. if (!strcmp(s, *(char **)kp->arg) && zswap_has_pool)
  588. return 0;
  589. /* if this is load-time (pre-init) param setting,
  590. * don't create a pool; that's done during init.
  591. */
  592. if (!zswap_init_started)
  593. return param_set_charp(s, kp);
  594. if (!type) {
  595. if (!zpool_has_pool(s)) {
  596. pr_err("zpool %s not available\n", s);
  597. return -ENOENT;
  598. }
  599. type = s;
  600. } else if (!compressor) {
  601. if (!crypto_has_comp(s, 0, 0)) {
  602. pr_err("compressor %s not available\n", s);
  603. return -ENOENT;
  604. }
  605. compressor = s;
  606. } else {
  607. WARN_ON(1);
  608. return -EINVAL;
  609. }
  610. spin_lock(&zswap_pools_lock);
  611. pool = zswap_pool_find_get(type, compressor);
  612. if (pool) {
  613. zswap_pool_debug("using existing", pool);
  614. WARN_ON(pool == zswap_pool_current());
  615. list_del_rcu(&pool->list);
  616. }
  617. spin_unlock(&zswap_pools_lock);
  618. if (!pool)
  619. pool = zswap_pool_create(type, compressor);
  620. if (pool)
  621. ret = param_set_charp(s, kp);
  622. else
  623. ret = -EINVAL;
  624. spin_lock(&zswap_pools_lock);
  625. if (!ret) {
  626. put_pool = zswap_pool_current();
  627. list_add_rcu(&pool->list, &zswap_pools);
  628. zswap_has_pool = true;
  629. } else if (pool) {
  630. /* add the possibly pre-existing pool to the end of the pools
  631. * list; if it's new (and empty) then it'll be removed and
  632. * destroyed by the put after we drop the lock
  633. */
  634. list_add_tail_rcu(&pool->list, &zswap_pools);
  635. put_pool = pool;
  636. }
  637. spin_unlock(&zswap_pools_lock);
  638. if (!zswap_has_pool && !pool) {
  639. /* if initial pool creation failed, and this pool creation also
  640. * failed, maybe both compressor and zpool params were bad.
  641. * Allow changing this param, so pool creation will succeed
  642. * when the other param is changed. We already verified this
  643. * param is ok in the zpool_has_pool() or crypto_has_comp()
  644. * checks above.
  645. */
  646. ret = param_set_charp(s, kp);
  647. }
  648. /* drop the ref from either the old current pool,
  649. * or the new pool we failed to add
  650. */
  651. if (put_pool)
  652. zswap_pool_put(put_pool);
  653. return ret;
  654. }
  655. static int zswap_compressor_param_set(const char *val,
  656. const struct kernel_param *kp)
  657. {
  658. return __zswap_param_set(val, kp, zswap_zpool_type, NULL);
  659. }
  660. static int zswap_zpool_param_set(const char *val,
  661. const struct kernel_param *kp)
  662. {
  663. return __zswap_param_set(val, kp, NULL, zswap_compressor);
  664. }
  665. static int zswap_enabled_param_set(const char *val,
  666. const struct kernel_param *kp)
  667. {
  668. if (zswap_init_failed) {
  669. pr_err("can't enable, initialization failed\n");
  670. return -ENODEV;
  671. }
  672. if (!zswap_has_pool && zswap_init_started) {
  673. pr_err("can't enable, no pool configured\n");
  674. return -ENODEV;
  675. }
  676. return param_set_bool(val, kp);
  677. }
  678. /*********************************
  679. * writeback code
  680. **********************************/
  681. /* return enum for zswap_get_swap_cache_page */
  682. enum zswap_get_swap_ret {
  683. ZSWAP_SWAPCACHE_NEW,
  684. ZSWAP_SWAPCACHE_EXIST,
  685. ZSWAP_SWAPCACHE_FAIL,
  686. };
  687. /*
  688. * zswap_get_swap_cache_page
  689. *
  690. * This is an adaption of read_swap_cache_async()
  691. *
  692. * This function tries to find a page with the given swap entry
  693. * in the swapper_space address space (the swap cache). If the page
  694. * is found, it is returned in retpage. Otherwise, a page is allocated,
  695. * added to the swap cache, and returned in retpage.
  696. *
  697. * If success, the swap cache page is returned in retpage
  698. * Returns ZSWAP_SWAPCACHE_EXIST if page was already in the swap cache
  699. * Returns ZSWAP_SWAPCACHE_NEW if the new page needs to be populated,
  700. * the new page is added to swapcache and locked
  701. * Returns ZSWAP_SWAPCACHE_FAIL on error
  702. */
  703. static int zswap_get_swap_cache_page(swp_entry_t entry,
  704. struct page **retpage)
  705. {
  706. bool page_was_allocated;
  707. *retpage = __read_swap_cache_async(entry, GFP_KERNEL,
  708. NULL, 0, &page_was_allocated);
  709. if (page_was_allocated)
  710. return ZSWAP_SWAPCACHE_NEW;
  711. if (!*retpage)
  712. return ZSWAP_SWAPCACHE_FAIL;
  713. return ZSWAP_SWAPCACHE_EXIST;
  714. }
  715. /*
  716. * Attempts to free an entry by adding a page to the swap cache,
  717. * decompressing the entry data into the page, and issuing a
  718. * bio write to write the page back to the swap device.
  719. *
  720. * This can be thought of as a "resumed writeback" of the page
  721. * to the swap device. We are basically resuming the same swap
  722. * writeback path that was intercepted with the frontswap_store()
  723. * in the first place. After the page has been decompressed into
  724. * the swap cache, the compressed version stored by zswap can be
  725. * freed.
  726. */
  727. static int zswap_writeback_entry(struct zpool *pool, unsigned long handle)
  728. {
  729. struct zswap_header *zhdr;
  730. swp_entry_t swpentry;
  731. struct zswap_tree *tree;
  732. pgoff_t offset;
  733. struct zswap_entry *entry;
  734. struct page *page;
  735. struct crypto_comp *tfm;
  736. u8 *src, *dst;
  737. unsigned int dlen;
  738. int ret;
  739. struct writeback_control wbc = {
  740. .sync_mode = WB_SYNC_NONE,
  741. };
  742. /* extract swpentry from data */
  743. zhdr = zpool_map_handle(pool, handle, ZPOOL_MM_RO);
  744. swpentry = zhdr->swpentry; /* here */
  745. tree = zswap_trees[swp_type(swpentry)];
  746. offset = swp_offset(swpentry);
  747. /* find and ref zswap entry */
  748. spin_lock(&tree->lock);
  749. entry = zswap_entry_find_get(&tree->rbroot, offset);
  750. if (!entry) {
  751. /* entry was invalidated */
  752. spin_unlock(&tree->lock);
  753. zpool_unmap_handle(pool, handle);
  754. return 0;
  755. }
  756. spin_unlock(&tree->lock);
  757. BUG_ON(offset != entry->offset);
  758. /* try to allocate swap cache page */
  759. switch (zswap_get_swap_cache_page(swpentry, &page)) {
  760. case ZSWAP_SWAPCACHE_FAIL: /* no memory or invalidate happened */
  761. ret = -ENOMEM;
  762. goto fail;
  763. case ZSWAP_SWAPCACHE_EXIST:
  764. /* page is already in the swap cache, ignore for now */
  765. put_page(page);
  766. ret = -EEXIST;
  767. goto fail;
  768. case ZSWAP_SWAPCACHE_NEW: /* page is locked */
  769. /* decompress */
  770. dlen = PAGE_SIZE;
  771. src = (u8 *)zhdr + sizeof(struct zswap_header);
  772. dst = kmap_atomic(page);
  773. tfm = *get_cpu_ptr(entry->pool->tfm);
  774. ret = crypto_comp_decompress(tfm, src, entry->length,
  775. dst, &dlen);
  776. put_cpu_ptr(entry->pool->tfm);
  777. kunmap_atomic(dst);
  778. BUG_ON(ret);
  779. BUG_ON(dlen != PAGE_SIZE);
  780. /* page is up to date */
  781. SetPageUptodate(page);
  782. }
  783. /* move it to the tail of the inactive list after end_writeback */
  784. SetPageReclaim(page);
  785. /* start writeback */
  786. __swap_writepage(page, &wbc, end_swap_bio_write);
  787. put_page(page);
  788. zswap_written_back_pages++;
  789. spin_lock(&tree->lock);
  790. /* drop local reference */
  791. zswap_entry_put(tree, entry);
  792. /*
  793. * There are two possible situations for entry here:
  794. * (1) refcount is 1(normal case), entry is valid and on the tree
  795. * (2) refcount is 0, entry is freed and not on the tree
  796. * because invalidate happened during writeback
  797. * search the tree and free the entry if find entry
  798. */
  799. if (entry == zswap_rb_search(&tree->rbroot, offset))
  800. zswap_entry_put(tree, entry);
  801. spin_unlock(&tree->lock);
  802. goto end;
  803. /*
  804. * if we get here due to ZSWAP_SWAPCACHE_EXIST
  805. * a load may happening concurrently
  806. * it is safe and okay to not free the entry
  807. * if we free the entry in the following put
  808. * it it either okay to return !0
  809. */
  810. fail:
  811. spin_lock(&tree->lock);
  812. zswap_entry_put(tree, entry);
  813. spin_unlock(&tree->lock);
  814. end:
  815. zpool_unmap_handle(pool, handle);
  816. return ret;
  817. }
  818. static int zswap_is_page_same_filled(void *ptr, unsigned long *value)
  819. {
  820. unsigned int pos;
  821. unsigned long *page;
  822. page = (unsigned long *)ptr;
  823. for (pos = 1; pos < PAGE_SIZE / sizeof(*page); pos++) {
  824. if (page[pos] != page[0])
  825. return 0;
  826. }
  827. *value = page[0];
  828. return 1;
  829. }
  830. static void zswap_fill_page(void *ptr, unsigned long value)
  831. {
  832. unsigned long *page;
  833. page = (unsigned long *)ptr;
  834. memset_l(page, value, PAGE_SIZE / sizeof(unsigned long));
  835. }
  836. /*********************************
  837. * frontswap hooks
  838. **********************************/
  839. /* attempts to compress and store an single page */
  840. static int zswap_frontswap_store(unsigned type, pgoff_t offset,
  841. struct page *page)
  842. {
  843. struct zswap_tree *tree = zswap_trees[type];
  844. struct zswap_entry *entry, *dupentry;
  845. struct crypto_comp *tfm;
  846. int ret;
  847. unsigned int hlen, dlen = PAGE_SIZE;
  848. unsigned long handle, value;
  849. char *buf;
  850. u8 *src, *dst;
  851. struct zswap_header zhdr = { .swpentry = swp_entry(type, offset) };
  852. gfp_t gfp;
  853. /* THP isn't supported */
  854. if (PageTransHuge(page)) {
  855. ret = -EINVAL;
  856. goto reject;
  857. }
  858. if (!zswap_enabled || !tree) {
  859. ret = -ENODEV;
  860. goto reject;
  861. }
  862. /* reclaim space if needed */
  863. if (zswap_is_full()) {
  864. struct zswap_pool *pool;
  865. zswap_pool_limit_hit++;
  866. zswap_pool_reached_full = true;
  867. pool = zswap_pool_last_get();
  868. if (pool)
  869. queue_work(shrink_wq, &pool->shrink_work);
  870. ret = -ENOMEM;
  871. goto reject;
  872. }
  873. if (zswap_pool_reached_full) {
  874. if (!zswap_can_accept()) {
  875. ret = -ENOMEM;
  876. goto reject;
  877. } else
  878. zswap_pool_reached_full = false;
  879. }
  880. /* allocate entry */
  881. entry = zswap_entry_cache_alloc(GFP_KERNEL);
  882. if (!entry) {
  883. zswap_reject_kmemcache_fail++;
  884. ret = -ENOMEM;
  885. goto reject;
  886. }
  887. if (zswap_same_filled_pages_enabled) {
  888. src = kmap_atomic(page);
  889. if (zswap_is_page_same_filled(src, &value)) {
  890. kunmap_atomic(src);
  891. entry->offset = offset;
  892. entry->length = 0;
  893. entry->value = value;
  894. atomic_inc(&zswap_same_filled_pages);
  895. goto insert_entry;
  896. }
  897. kunmap_atomic(src);
  898. }
  899. /* if entry is successfully added, it keeps the reference */
  900. entry->pool = zswap_pool_current_get();
  901. if (!entry->pool) {
  902. ret = -EINVAL;
  903. goto freepage;
  904. }
  905. /* compress */
  906. dst = get_cpu_var(zswap_dstmem);
  907. tfm = *get_cpu_ptr(entry->pool->tfm);
  908. src = kmap_atomic(page);
  909. ret = crypto_comp_compress(tfm, src, PAGE_SIZE, dst, &dlen);
  910. kunmap_atomic(src);
  911. put_cpu_ptr(entry->pool->tfm);
  912. if (ret) {
  913. ret = -EINVAL;
  914. goto put_dstmem;
  915. }
  916. /* store */
  917. hlen = zpool_evictable(entry->pool->zpool) ? sizeof(zhdr) : 0;
  918. gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM;
  919. if (zpool_malloc_support_movable(entry->pool->zpool))
  920. gfp |= __GFP_HIGHMEM | __GFP_MOVABLE;
  921. ret = zpool_malloc(entry->pool->zpool, hlen + dlen, gfp, &handle);
  922. if (ret == -ENOSPC) {
  923. zswap_reject_compress_poor++;
  924. goto put_dstmem;
  925. }
  926. if (ret) {
  927. zswap_reject_alloc_fail++;
  928. goto put_dstmem;
  929. }
  930. buf = zpool_map_handle(entry->pool->zpool, handle, ZPOOL_MM_RW);
  931. memcpy(buf, &zhdr, hlen);
  932. memcpy(buf + hlen, dst, dlen);
  933. zpool_unmap_handle(entry->pool->zpool, handle);
  934. put_cpu_var(zswap_dstmem);
  935. /* populate entry */
  936. entry->offset = offset;
  937. entry->handle = handle;
  938. entry->length = dlen;
  939. insert_entry:
  940. /* map */
  941. spin_lock(&tree->lock);
  942. do {
  943. ret = zswap_rb_insert(&tree->rbroot, entry, &dupentry);
  944. if (ret == -EEXIST) {
  945. zswap_duplicate_entry++;
  946. /* remove from rbtree */
  947. zswap_rb_erase(&tree->rbroot, dupentry);
  948. zswap_entry_put(tree, dupentry);
  949. }
  950. } while (ret == -EEXIST);
  951. spin_unlock(&tree->lock);
  952. /* update stats */
  953. atomic_inc(&zswap_stored_pages);
  954. zswap_update_total_size();
  955. return 0;
  956. put_dstmem:
  957. put_cpu_var(zswap_dstmem);
  958. zswap_pool_put(entry->pool);
  959. freepage:
  960. zswap_entry_cache_free(entry);
  961. reject:
  962. return ret;
  963. }
  964. /*
  965. * returns 0 if the page was successfully decompressed
  966. * return -1 on entry not found or error
  967. */
  968. static int zswap_frontswap_load(unsigned type, pgoff_t offset,
  969. struct page *page)
  970. {
  971. struct zswap_tree *tree = zswap_trees[type];
  972. struct zswap_entry *entry;
  973. struct crypto_comp *tfm;
  974. u8 *src, *dst;
  975. unsigned int dlen;
  976. int ret;
  977. /* find */
  978. spin_lock(&tree->lock);
  979. entry = zswap_entry_find_get(&tree->rbroot, offset);
  980. if (!entry) {
  981. /* entry was written back */
  982. spin_unlock(&tree->lock);
  983. return -1;
  984. }
  985. spin_unlock(&tree->lock);
  986. if (!entry->length) {
  987. dst = kmap_atomic(page);
  988. zswap_fill_page(dst, entry->value);
  989. kunmap_atomic(dst);
  990. goto freeentry;
  991. }
  992. /* decompress */
  993. dlen = PAGE_SIZE;
  994. src = zpool_map_handle(entry->pool->zpool, entry->handle, ZPOOL_MM_RO);
  995. if (zpool_evictable(entry->pool->zpool))
  996. src += sizeof(struct zswap_header);
  997. dst = kmap_atomic(page);
  998. tfm = *get_cpu_ptr(entry->pool->tfm);
  999. ret = crypto_comp_decompress(tfm, src, entry->length, dst, &dlen);
  1000. put_cpu_ptr(entry->pool->tfm);
  1001. kunmap_atomic(dst);
  1002. zpool_unmap_handle(entry->pool->zpool, entry->handle);
  1003. BUG_ON(ret);
  1004. freeentry:
  1005. spin_lock(&tree->lock);
  1006. zswap_entry_put(tree, entry);
  1007. spin_unlock(&tree->lock);
  1008. return 0;
  1009. }
  1010. /* frees an entry in zswap */
  1011. static void zswap_frontswap_invalidate_page(unsigned type, pgoff_t offset)
  1012. {
  1013. struct zswap_tree *tree = zswap_trees[type];
  1014. struct zswap_entry *entry;
  1015. /* find */
  1016. spin_lock(&tree->lock);
  1017. entry = zswap_rb_search(&tree->rbroot, offset);
  1018. if (!entry) {
  1019. /* entry was written back */
  1020. spin_unlock(&tree->lock);
  1021. return;
  1022. }
  1023. /* remove from rbtree */
  1024. zswap_rb_erase(&tree->rbroot, entry);
  1025. /* drop the initial reference from entry creation */
  1026. zswap_entry_put(tree, entry);
  1027. spin_unlock(&tree->lock);
  1028. }
  1029. /* frees all zswap entries for the given swap type */
  1030. static void zswap_frontswap_invalidate_area(unsigned type)
  1031. {
  1032. struct zswap_tree *tree = zswap_trees[type];
  1033. struct zswap_entry *entry, *n;
  1034. if (!tree)
  1035. return;
  1036. /* walk the tree and free everything */
  1037. spin_lock(&tree->lock);
  1038. rbtree_postorder_for_each_entry_safe(entry, n, &tree->rbroot, rbnode)
  1039. zswap_free_entry(entry);
  1040. tree->rbroot = RB_ROOT;
  1041. spin_unlock(&tree->lock);
  1042. kfree(tree);
  1043. zswap_trees[type] = NULL;
  1044. }
  1045. static void zswap_frontswap_init(unsigned type)
  1046. {
  1047. struct zswap_tree *tree;
  1048. tree = kzalloc(sizeof(*tree), GFP_KERNEL);
  1049. if (!tree) {
  1050. pr_err("alloc failed, zswap disabled for swap type %d\n", type);
  1051. return;
  1052. }
  1053. tree->rbroot = RB_ROOT;
  1054. spin_lock_init(&tree->lock);
  1055. zswap_trees[type] = tree;
  1056. }
  1057. static struct frontswap_ops zswap_frontswap_ops = {
  1058. .store = zswap_frontswap_store,
  1059. .load = zswap_frontswap_load,
  1060. .invalidate_page = zswap_frontswap_invalidate_page,
  1061. .invalidate_area = zswap_frontswap_invalidate_area,
  1062. .init = zswap_frontswap_init
  1063. };
  1064. /*********************************
  1065. * debugfs functions
  1066. **********************************/
  1067. #ifdef CONFIG_DEBUG_FS
  1068. #include <linux/debugfs.h>
  1069. static struct dentry *zswap_debugfs_root;
  1070. static int __init zswap_debugfs_init(void)
  1071. {
  1072. if (!debugfs_initialized())
  1073. return -ENODEV;
  1074. zswap_debugfs_root = debugfs_create_dir("zswap", NULL);
  1075. debugfs_create_u64("pool_limit_hit", 0444,
  1076. zswap_debugfs_root, &zswap_pool_limit_hit);
  1077. debugfs_create_u64("reject_reclaim_fail", 0444,
  1078. zswap_debugfs_root, &zswap_reject_reclaim_fail);
  1079. debugfs_create_u64("reject_alloc_fail", 0444,
  1080. zswap_debugfs_root, &zswap_reject_alloc_fail);
  1081. debugfs_create_u64("reject_kmemcache_fail", 0444,
  1082. zswap_debugfs_root, &zswap_reject_kmemcache_fail);
  1083. debugfs_create_u64("reject_compress_poor", 0444,
  1084. zswap_debugfs_root, &zswap_reject_compress_poor);
  1085. debugfs_create_u64("written_back_pages", 0444,
  1086. zswap_debugfs_root, &zswap_written_back_pages);
  1087. debugfs_create_u64("duplicate_entry", 0444,
  1088. zswap_debugfs_root, &zswap_duplicate_entry);
  1089. debugfs_create_u64("pool_total_size", 0444,
  1090. zswap_debugfs_root, &zswap_pool_total_size);
  1091. debugfs_create_atomic_t("stored_pages", 0444,
  1092. zswap_debugfs_root, &zswap_stored_pages);
  1093. debugfs_create_atomic_t("same_filled_pages", 0444,
  1094. zswap_debugfs_root, &zswap_same_filled_pages);
  1095. return 0;
  1096. }
  1097. static void __exit zswap_debugfs_exit(void)
  1098. {
  1099. debugfs_remove_recursive(zswap_debugfs_root);
  1100. }
  1101. #else
  1102. static int __init zswap_debugfs_init(void)
  1103. {
  1104. return 0;
  1105. }
  1106. static void __exit zswap_debugfs_exit(void) { }
  1107. #endif
  1108. /*********************************
  1109. * module init and exit
  1110. **********************************/
  1111. static int __init init_zswap(void)
  1112. {
  1113. struct zswap_pool *pool;
  1114. int ret;
  1115. zswap_init_started = true;
  1116. if (zswap_entry_cache_create()) {
  1117. pr_err("entry cache creation failed\n");
  1118. goto cache_fail;
  1119. }
  1120. ret = cpuhp_setup_state(CPUHP_MM_ZSWP_MEM_PREPARE, "mm/zswap:prepare",
  1121. zswap_dstmem_prepare, zswap_dstmem_dead);
  1122. if (ret) {
  1123. pr_err("dstmem alloc failed\n");
  1124. goto dstmem_fail;
  1125. }
  1126. ret = cpuhp_setup_state_multi(CPUHP_MM_ZSWP_POOL_PREPARE,
  1127. "mm/zswap_pool:prepare",
  1128. zswap_cpu_comp_prepare,
  1129. zswap_cpu_comp_dead);
  1130. if (ret)
  1131. goto hp_fail;
  1132. pool = __zswap_pool_create_fallback();
  1133. if (pool) {
  1134. pr_info("loaded using pool %s/%s\n", pool->tfm_name,
  1135. zpool_get_type(pool->zpool));
  1136. list_add(&pool->list, &zswap_pools);
  1137. zswap_has_pool = true;
  1138. } else {
  1139. pr_err("pool creation failed\n");
  1140. zswap_enabled = false;
  1141. }
  1142. shrink_wq = create_workqueue("zswap-shrink");
  1143. if (!shrink_wq)
  1144. goto fallback_fail;
  1145. frontswap_register_ops(&zswap_frontswap_ops);
  1146. if (zswap_debugfs_init())
  1147. pr_warn("debugfs initialization failed\n");
  1148. return 0;
  1149. fallback_fail:
  1150. if (pool)
  1151. zswap_pool_destroy(pool);
  1152. hp_fail:
  1153. cpuhp_remove_state(CPUHP_MM_ZSWP_MEM_PREPARE);
  1154. dstmem_fail:
  1155. zswap_entry_cache_destroy();
  1156. cache_fail:
  1157. /* if built-in, we aren't unloaded on failure; don't allow use */
  1158. zswap_init_failed = true;
  1159. zswap_enabled = false;
  1160. return -ENOMEM;
  1161. }
  1162. /* must be late so crypto has time to come up */
  1163. late_initcall(init_zswap);
  1164. MODULE_LICENSE("GPL");
  1165. MODULE_AUTHOR("Seth Jennings <sjennings@variantweb.net>");
  1166. MODULE_DESCRIPTION("Compressed cache for swap pages");