mempool.c 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * linux/mm/mempool.c
  4. *
  5. * memory buffer pool support. Such pools are mostly used
  6. * for guaranteed, deadlock-free memory allocations during
  7. * extreme VM load.
  8. *
  9. * started by Ingo Molnar, Copyright (C) 2001
  10. * debugging by David Rientjes, Copyright (C) 2015
  11. */
  12. #include <linux/mm.h>
  13. #include <linux/slab.h>
  14. #include <linux/highmem.h>
  15. #include <linux/kasan.h>
  16. #include <linux/kmemleak.h>
  17. #include <linux/export.h>
  18. #include <linux/mempool.h>
  19. #include <linux/blkdev.h>
  20. #include <linux/writeback.h>
  21. #include "slab.h"
  22. #if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_SLUB_DEBUG_ON)
  23. static void poison_error(mempool_t *pool, void *element, size_t size,
  24. size_t byte)
  25. {
  26. const int nr = pool->curr_nr;
  27. const int start = max_t(int, byte - (BITS_PER_LONG / 8), 0);
  28. const int end = min_t(int, byte + (BITS_PER_LONG / 8), size);
  29. int i;
  30. pr_err("BUG: mempool element poison mismatch\n");
  31. pr_err("Mempool %p size %zu\n", pool, size);
  32. pr_err(" nr=%d @ %p: %s0x", nr, element, start > 0 ? "... " : "");
  33. for (i = start; i < end; i++)
  34. pr_cont("%x ", *(u8 *)(element + i));
  35. pr_cont("%s\n", end < size ? "..." : "");
  36. dump_stack();
  37. }
  38. static void __check_element(mempool_t *pool, void *element, size_t size)
  39. {
  40. u8 *obj = element;
  41. size_t i;
  42. for (i = 0; i < size; i++) {
  43. u8 exp = (i < size - 1) ? POISON_FREE : POISON_END;
  44. if (obj[i] != exp) {
  45. poison_error(pool, element, size, i);
  46. return;
  47. }
  48. }
  49. memset(obj, POISON_INUSE, size);
  50. }
  51. static void check_element(mempool_t *pool, void *element)
  52. {
  53. /* Mempools backed by slab allocator */
  54. if (pool->free == mempool_free_slab || pool->free == mempool_kfree) {
  55. __check_element(pool, element, ksize(element));
  56. } else if (pool->free == mempool_free_pages) {
  57. /* Mempools backed by page allocator */
  58. int order = (int)(long)pool->pool_data;
  59. void *addr = kmap_atomic((struct page *)element);
  60. __check_element(pool, addr, 1UL << (PAGE_SHIFT + order));
  61. kunmap_atomic(addr);
  62. }
  63. }
  64. static void __poison_element(void *element, size_t size)
  65. {
  66. u8 *obj = element;
  67. memset(obj, POISON_FREE, size - 1);
  68. obj[size - 1] = POISON_END;
  69. }
  70. static void poison_element(mempool_t *pool, void *element)
  71. {
  72. /* Mempools backed by slab allocator */
  73. if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc) {
  74. __poison_element(element, ksize(element));
  75. } else if (pool->alloc == mempool_alloc_pages) {
  76. /* Mempools backed by page allocator */
  77. int order = (int)(long)pool->pool_data;
  78. void *addr = kmap_atomic((struct page *)element);
  79. __poison_element(addr, 1UL << (PAGE_SHIFT + order));
  80. kunmap_atomic(addr);
  81. }
  82. }
  83. #else /* CONFIG_DEBUG_SLAB || CONFIG_SLUB_DEBUG_ON */
  84. static inline void check_element(mempool_t *pool, void *element)
  85. {
  86. }
  87. static inline void poison_element(mempool_t *pool, void *element)
  88. {
  89. }
  90. #endif /* CONFIG_DEBUG_SLAB || CONFIG_SLUB_DEBUG_ON */
  91. static __always_inline void kasan_poison_element(mempool_t *pool, void *element)
  92. {
  93. if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc)
  94. kasan_slab_free_mempool(element);
  95. else if (pool->alloc == mempool_alloc_pages)
  96. kasan_poison_pages(element, (unsigned long)pool->pool_data,
  97. false);
  98. }
  99. static void kasan_unpoison_element(mempool_t *pool, void *element)
  100. {
  101. if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc)
  102. kasan_unpoison_range(element, __ksize(element));
  103. else if (pool->alloc == mempool_alloc_pages)
  104. kasan_unpoison_pages(element, (unsigned long)pool->pool_data,
  105. false);
  106. }
  107. static __always_inline void add_element(mempool_t *pool, void *element)
  108. {
  109. BUG_ON(pool->curr_nr >= pool->min_nr);
  110. poison_element(pool, element);
  111. kasan_poison_element(pool, element);
  112. pool->elements[pool->curr_nr++] = element;
  113. }
  114. static void *remove_element(mempool_t *pool)
  115. {
  116. void *element = pool->elements[--pool->curr_nr];
  117. BUG_ON(pool->curr_nr < 0);
  118. kasan_unpoison_element(pool, element);
  119. check_element(pool, element);
  120. return element;
  121. }
  122. /**
  123. * mempool_exit - exit a mempool initialized with mempool_init()
  124. * @pool: pointer to the memory pool which was initialized with
  125. * mempool_init().
  126. *
  127. * Free all reserved elements in @pool and @pool itself. This function
  128. * only sleeps if the free_fn() function sleeps.
  129. *
  130. * May be called on a zeroed but uninitialized mempool (i.e. allocated with
  131. * kzalloc()).
  132. */
  133. void mempool_exit(mempool_t *pool)
  134. {
  135. while (pool->curr_nr) {
  136. void *element = remove_element(pool);
  137. pool->free(element, pool->pool_data);
  138. }
  139. kfree(pool->elements);
  140. pool->elements = NULL;
  141. }
  142. EXPORT_SYMBOL(mempool_exit);
  143. /**
  144. * mempool_destroy - deallocate a memory pool
  145. * @pool: pointer to the memory pool which was allocated via
  146. * mempool_create().
  147. *
  148. * Free all reserved elements in @pool and @pool itself. This function
  149. * only sleeps if the free_fn() function sleeps.
  150. */
  151. void mempool_destroy(mempool_t *pool)
  152. {
  153. if (unlikely(!pool))
  154. return;
  155. mempool_exit(pool);
  156. kfree(pool);
  157. }
  158. EXPORT_SYMBOL(mempool_destroy);
  159. int mempool_init_node(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,
  160. mempool_free_t *free_fn, void *pool_data,
  161. gfp_t gfp_mask, int node_id)
  162. {
  163. spin_lock_init(&pool->lock);
  164. pool->min_nr = min_nr;
  165. pool->pool_data = pool_data;
  166. pool->alloc = alloc_fn;
  167. pool->free = free_fn;
  168. init_waitqueue_head(&pool->wait);
  169. pool->elements = kmalloc_array_node(min_nr, sizeof(void *),
  170. gfp_mask, node_id);
  171. if (!pool->elements)
  172. return -ENOMEM;
  173. /*
  174. * First pre-allocate the guaranteed number of buffers.
  175. */
  176. while (pool->curr_nr < pool->min_nr) {
  177. void *element;
  178. element = pool->alloc(gfp_mask, pool->pool_data);
  179. if (unlikely(!element)) {
  180. mempool_exit(pool);
  181. return -ENOMEM;
  182. }
  183. add_element(pool, element);
  184. }
  185. return 0;
  186. }
  187. EXPORT_SYMBOL(mempool_init_node);
  188. /**
  189. * mempool_init - initialize a memory pool
  190. * @pool: pointer to the memory pool that should be initialized
  191. * @min_nr: the minimum number of elements guaranteed to be
  192. * allocated for this pool.
  193. * @alloc_fn: user-defined element-allocation function.
  194. * @free_fn: user-defined element-freeing function.
  195. * @pool_data: optional private data available to the user-defined functions.
  196. *
  197. * Like mempool_create(), but initializes the pool in (i.e. embedded in another
  198. * structure).
  199. *
  200. * Return: %0 on success, negative error code otherwise.
  201. */
  202. int mempool_init(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,
  203. mempool_free_t *free_fn, void *pool_data)
  204. {
  205. return mempool_init_node(pool, min_nr, alloc_fn, free_fn,
  206. pool_data, GFP_KERNEL, NUMA_NO_NODE);
  207. }
  208. EXPORT_SYMBOL(mempool_init);
  209. /**
  210. * mempool_create - create a memory pool
  211. * @min_nr: the minimum number of elements guaranteed to be
  212. * allocated for this pool.
  213. * @alloc_fn: user-defined element-allocation function.
  214. * @free_fn: user-defined element-freeing function.
  215. * @pool_data: optional private data available to the user-defined functions.
  216. *
  217. * this function creates and allocates a guaranteed size, preallocated
  218. * memory pool. The pool can be used from the mempool_alloc() and mempool_free()
  219. * functions. This function might sleep. Both the alloc_fn() and the free_fn()
  220. * functions might sleep - as long as the mempool_alloc() function is not called
  221. * from IRQ contexts.
  222. *
  223. * Return: pointer to the created memory pool object or %NULL on error.
  224. */
  225. mempool_t *mempool_create(int min_nr, mempool_alloc_t *alloc_fn,
  226. mempool_free_t *free_fn, void *pool_data)
  227. {
  228. return mempool_create_node(min_nr,alloc_fn,free_fn, pool_data,
  229. GFP_KERNEL, NUMA_NO_NODE);
  230. }
  231. EXPORT_SYMBOL(mempool_create);
  232. mempool_t *mempool_create_node(int min_nr, mempool_alloc_t *alloc_fn,
  233. mempool_free_t *free_fn, void *pool_data,
  234. gfp_t gfp_mask, int node_id)
  235. {
  236. mempool_t *pool;
  237. pool = kzalloc_node(sizeof(*pool), gfp_mask, node_id);
  238. if (!pool)
  239. return NULL;
  240. if (mempool_init_node(pool, min_nr, alloc_fn, free_fn, pool_data,
  241. gfp_mask, node_id)) {
  242. kfree(pool);
  243. return NULL;
  244. }
  245. return pool;
  246. }
  247. EXPORT_SYMBOL(mempool_create_node);
  248. /**
  249. * mempool_resize - resize an existing memory pool
  250. * @pool: pointer to the memory pool which was allocated via
  251. * mempool_create().
  252. * @new_min_nr: the new minimum number of elements guaranteed to be
  253. * allocated for this pool.
  254. *
  255. * This function shrinks/grows the pool. In the case of growing,
  256. * it cannot be guaranteed that the pool will be grown to the new
  257. * size immediately, but new mempool_free() calls will refill it.
  258. * This function may sleep.
  259. *
  260. * Note, the caller must guarantee that no mempool_destroy is called
  261. * while this function is running. mempool_alloc() & mempool_free()
  262. * might be called (eg. from IRQ contexts) while this function executes.
  263. *
  264. * Return: %0 on success, negative error code otherwise.
  265. */
  266. int mempool_resize(mempool_t *pool, int new_min_nr)
  267. {
  268. void *element;
  269. void **new_elements;
  270. unsigned long flags;
  271. BUG_ON(new_min_nr <= 0);
  272. might_sleep();
  273. spin_lock_irqsave(&pool->lock, flags);
  274. if (new_min_nr <= pool->min_nr) {
  275. while (new_min_nr < pool->curr_nr) {
  276. element = remove_element(pool);
  277. spin_unlock_irqrestore(&pool->lock, flags);
  278. pool->free(element, pool->pool_data);
  279. spin_lock_irqsave(&pool->lock, flags);
  280. }
  281. pool->min_nr = new_min_nr;
  282. goto out_unlock;
  283. }
  284. spin_unlock_irqrestore(&pool->lock, flags);
  285. /* Grow the pool */
  286. new_elements = kmalloc_array(new_min_nr, sizeof(*new_elements),
  287. GFP_KERNEL);
  288. if (!new_elements)
  289. return -ENOMEM;
  290. spin_lock_irqsave(&pool->lock, flags);
  291. if (unlikely(new_min_nr <= pool->min_nr)) {
  292. /* Raced, other resize will do our work */
  293. spin_unlock_irqrestore(&pool->lock, flags);
  294. kfree(new_elements);
  295. goto out;
  296. }
  297. memcpy(new_elements, pool->elements,
  298. pool->curr_nr * sizeof(*new_elements));
  299. kfree(pool->elements);
  300. pool->elements = new_elements;
  301. pool->min_nr = new_min_nr;
  302. while (pool->curr_nr < pool->min_nr) {
  303. spin_unlock_irqrestore(&pool->lock, flags);
  304. element = pool->alloc(GFP_KERNEL, pool->pool_data);
  305. if (!element)
  306. goto out;
  307. spin_lock_irqsave(&pool->lock, flags);
  308. if (pool->curr_nr < pool->min_nr) {
  309. add_element(pool, element);
  310. } else {
  311. spin_unlock_irqrestore(&pool->lock, flags);
  312. pool->free(element, pool->pool_data); /* Raced */
  313. goto out;
  314. }
  315. }
  316. out_unlock:
  317. spin_unlock_irqrestore(&pool->lock, flags);
  318. out:
  319. return 0;
  320. }
  321. EXPORT_SYMBOL(mempool_resize);
  322. /**
  323. * mempool_alloc - allocate an element from a specific memory pool
  324. * @pool: pointer to the memory pool which was allocated via
  325. * mempool_create().
  326. * @gfp_mask: the usual allocation bitmask.
  327. *
  328. * this function only sleeps if the alloc_fn() function sleeps or
  329. * returns NULL. Note that due to preallocation, this function
  330. * *never* fails when called from process contexts. (it might
  331. * fail if called from an IRQ context.)
  332. * Note: using __GFP_ZERO is not supported.
  333. *
  334. * Return: pointer to the allocated element or %NULL on error.
  335. */
  336. void *mempool_alloc(mempool_t *pool, gfp_t gfp_mask)
  337. {
  338. void *element;
  339. unsigned long flags;
  340. wait_queue_entry_t wait;
  341. gfp_t gfp_temp;
  342. VM_WARN_ON_ONCE(gfp_mask & __GFP_ZERO);
  343. might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM);
  344. gfp_mask |= __GFP_NOMEMALLOC; /* don't allocate emergency reserves */
  345. gfp_mask |= __GFP_NORETRY; /* don't loop in __alloc_pages */
  346. gfp_mask |= __GFP_NOWARN; /* failures are OK */
  347. gfp_temp = gfp_mask & ~(__GFP_DIRECT_RECLAIM|__GFP_IO);
  348. repeat_alloc:
  349. element = pool->alloc(gfp_temp, pool->pool_data);
  350. if (likely(element != NULL))
  351. return element;
  352. spin_lock_irqsave(&pool->lock, flags);
  353. if (likely(pool->curr_nr)) {
  354. element = remove_element(pool);
  355. spin_unlock_irqrestore(&pool->lock, flags);
  356. /* paired with rmb in mempool_free(), read comment there */
  357. smp_wmb();
  358. /*
  359. * Update the allocation stack trace as this is more useful
  360. * for debugging.
  361. */
  362. kmemleak_update_trace(element);
  363. return element;
  364. }
  365. /*
  366. * We use gfp mask w/o direct reclaim or IO for the first round. If
  367. * alloc failed with that and @pool was empty, retry immediately.
  368. */
  369. if (gfp_temp != gfp_mask) {
  370. spin_unlock_irqrestore(&pool->lock, flags);
  371. gfp_temp = gfp_mask;
  372. goto repeat_alloc;
  373. }
  374. /* We must not sleep if !__GFP_DIRECT_RECLAIM */
  375. if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) {
  376. spin_unlock_irqrestore(&pool->lock, flags);
  377. return NULL;
  378. }
  379. /* Let's wait for someone else to return an element to @pool */
  380. init_wait(&wait);
  381. prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);
  382. spin_unlock_irqrestore(&pool->lock, flags);
  383. /*
  384. * FIXME: this should be io_schedule(). The timeout is there as a
  385. * workaround for some DM problems in 2.6.18.
  386. */
  387. io_schedule_timeout(5*HZ);
  388. finish_wait(&pool->wait, &wait);
  389. goto repeat_alloc;
  390. }
  391. EXPORT_SYMBOL(mempool_alloc);
  392. /**
  393. * mempool_free - return an element to the pool.
  394. * @element: pool element pointer.
  395. * @pool: pointer to the memory pool which was allocated via
  396. * mempool_create().
  397. *
  398. * this function only sleeps if the free_fn() function sleeps.
  399. */
  400. void mempool_free(void *element, mempool_t *pool)
  401. {
  402. unsigned long flags;
  403. if (unlikely(element == NULL))
  404. return;
  405. /*
  406. * Paired with the wmb in mempool_alloc(). The preceding read is
  407. * for @element and the following @pool->curr_nr. This ensures
  408. * that the visible value of @pool->curr_nr is from after the
  409. * allocation of @element. This is necessary for fringe cases
  410. * where @element was passed to this task without going through
  411. * barriers.
  412. *
  413. * For example, assume @p is %NULL at the beginning and one task
  414. * performs "p = mempool_alloc(...);" while another task is doing
  415. * "while (!p) cpu_relax(); mempool_free(p, ...);". This function
  416. * may end up using curr_nr value which is from before allocation
  417. * of @p without the following rmb.
  418. */
  419. smp_rmb();
  420. /*
  421. * For correctness, we need a test which is guaranteed to trigger
  422. * if curr_nr + #allocated == min_nr. Testing curr_nr < min_nr
  423. * without locking achieves that and refilling as soon as possible
  424. * is desirable.
  425. *
  426. * Because curr_nr visible here is always a value after the
  427. * allocation of @element, any task which decremented curr_nr below
  428. * min_nr is guaranteed to see curr_nr < min_nr unless curr_nr gets
  429. * incremented to min_nr afterwards. If curr_nr gets incremented
  430. * to min_nr after the allocation of @element, the elements
  431. * allocated after that are subject to the same guarantee.
  432. *
  433. * Waiters happen iff curr_nr is 0 and the above guarantee also
  434. * ensures that there will be frees which return elements to the
  435. * pool waking up the waiters.
  436. */
  437. if (unlikely(READ_ONCE(pool->curr_nr) < pool->min_nr)) {
  438. spin_lock_irqsave(&pool->lock, flags);
  439. if (likely(pool->curr_nr < pool->min_nr)) {
  440. add_element(pool, element);
  441. spin_unlock_irqrestore(&pool->lock, flags);
  442. wake_up(&pool->wait);
  443. return;
  444. }
  445. spin_unlock_irqrestore(&pool->lock, flags);
  446. }
  447. pool->free(element, pool->pool_data);
  448. }
  449. EXPORT_SYMBOL(mempool_free);
  450. /*
  451. * A commonly used alloc and free fn.
  452. */
  453. void *mempool_alloc_slab(gfp_t gfp_mask, void *pool_data)
  454. {
  455. struct kmem_cache *mem = pool_data;
  456. VM_BUG_ON(mem->ctor);
  457. return kmem_cache_alloc(mem, gfp_mask);
  458. }
  459. EXPORT_SYMBOL(mempool_alloc_slab);
  460. void mempool_free_slab(void *element, void *pool_data)
  461. {
  462. struct kmem_cache *mem = pool_data;
  463. kmem_cache_free(mem, element);
  464. }
  465. EXPORT_SYMBOL(mempool_free_slab);
  466. /*
  467. * A commonly used alloc and free fn that kmalloc/kfrees the amount of memory
  468. * specified by pool_data
  469. */
  470. void *mempool_kmalloc(gfp_t gfp_mask, void *pool_data)
  471. {
  472. size_t size = (size_t)pool_data;
  473. return kmalloc(size, gfp_mask);
  474. }
  475. EXPORT_SYMBOL(mempool_kmalloc);
  476. void mempool_kfree(void *element, void *pool_data)
  477. {
  478. kfree(element);
  479. }
  480. EXPORT_SYMBOL(mempool_kfree);
  481. /*
  482. * A simple mempool-backed page allocator that allocates pages
  483. * of the order specified by pool_data.
  484. */
  485. void *mempool_alloc_pages(gfp_t gfp_mask, void *pool_data)
  486. {
  487. int order = (int)(long)pool_data;
  488. return alloc_pages(gfp_mask, order);
  489. }
  490. EXPORT_SYMBOL(mempool_alloc_pages);
  491. void mempool_free_pages(void *element, void *pool_data)
  492. {
  493. int order = (int)(long)pool_data;
  494. __free_pages(element, order);
  495. }
  496. EXPORT_SYMBOL(mempool_free_pages);