mempool.c 9.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338
  1. /*
  2. * linux/mm/mempool.c
  3. *
  4. * memory buffer pool support. Such pools are mostly used
  5. * for guaranteed, deadlock-free memory allocations during
  6. * extreme VM load.
  7. *
  8. * started by Ingo Molnar, Copyright (C) 2001
  9. */
  10. #include <linux/mm.h>
  11. #include <linux/slab.h>
  12. #include <linux/module.h>
  13. #include <linux/mempool.h>
  14. #include <linux/blkdev.h>
  15. #include <linux/writeback.h>
  16. static void add_element(mempool_t *pool, void *element)
  17. {
  18. BUG_ON(pool->curr_nr >= pool->min_nr);
  19. pool->elements[pool->curr_nr++] = element;
  20. }
  21. static void *remove_element(mempool_t *pool)
  22. {
  23. BUG_ON(pool->curr_nr <= 0);
  24. return pool->elements[--pool->curr_nr];
  25. }
  26. static void free_pool(mempool_t *pool)
  27. {
  28. while (pool->curr_nr) {
  29. void *element = remove_element(pool);
  30. pool->free(element, pool->pool_data);
  31. }
  32. kfree(pool->elements);
  33. kfree(pool);
  34. }
  35. /**
  36. * mempool_create - create a memory pool
  37. * @min_nr: the minimum number of elements guaranteed to be
  38. * allocated for this pool.
  39. * @alloc_fn: user-defined element-allocation function.
  40. * @free_fn: user-defined element-freeing function.
  41. * @pool_data: optional private data available to the user-defined functions.
  42. *
  43. * this function creates and allocates a guaranteed size, preallocated
  44. * memory pool. The pool can be used from the mempool_alloc() and mempool_free()
  45. * functions. This function might sleep. Both the alloc_fn() and the free_fn()
  46. * functions might sleep - as long as the mempool_alloc() function is not called
  47. * from IRQ contexts.
  48. */
  49. mempool_t *mempool_create(int min_nr, mempool_alloc_t *alloc_fn,
  50. mempool_free_t *free_fn, void *pool_data)
  51. {
  52. return mempool_create_node(min_nr,alloc_fn,free_fn, pool_data,-1);
  53. }
  54. EXPORT_SYMBOL(mempool_create);
  55. mempool_t *mempool_create_node(int min_nr, mempool_alloc_t *alloc_fn,
  56. mempool_free_t *free_fn, void *pool_data, int node_id)
  57. {
  58. mempool_t *pool;
  59. pool = kmalloc_node(sizeof(*pool), GFP_KERNEL, node_id);
  60. if (!pool)
  61. return NULL;
  62. memset(pool, 0, sizeof(*pool));
  63. pool->elements = kmalloc_node(min_nr * sizeof(void *),
  64. GFP_KERNEL, node_id);
  65. if (!pool->elements) {
  66. kfree(pool);
  67. return NULL;
  68. }
  69. spin_lock_init(&pool->lock);
  70. pool->min_nr = min_nr;
  71. pool->pool_data = pool_data;
  72. init_waitqueue_head(&pool->wait);
  73. pool->alloc = alloc_fn;
  74. pool->free = free_fn;
  75. /*
  76. * First pre-allocate the guaranteed number of buffers.
  77. */
  78. while (pool->curr_nr < pool->min_nr) {
  79. void *element;
  80. element = pool->alloc(GFP_KERNEL, pool->pool_data);
  81. if (unlikely(!element)) {
  82. free_pool(pool);
  83. return NULL;
  84. }
  85. add_element(pool, element);
  86. }
  87. return pool;
  88. }
  89. EXPORT_SYMBOL(mempool_create_node);
  90. /**
  91. * mempool_resize - resize an existing memory pool
  92. * @pool: pointer to the memory pool which was allocated via
  93. * mempool_create().
  94. * @new_min_nr: the new minimum number of elements guaranteed to be
  95. * allocated for this pool.
  96. * @gfp_mask: the usual allocation bitmask.
  97. *
  98. * This function shrinks/grows the pool. In the case of growing,
  99. * it cannot be guaranteed that the pool will be grown to the new
  100. * size immediately, but new mempool_free() calls will refill it.
  101. *
  102. * Note, the caller must guarantee that no mempool_destroy is called
  103. * while this function is running. mempool_alloc() & mempool_free()
  104. * might be called (eg. from IRQ contexts) while this function executes.
  105. */
  106. int mempool_resize(mempool_t *pool, int new_min_nr, gfp_t gfp_mask)
  107. {
  108. void *element;
  109. void **new_elements;
  110. unsigned long flags;
  111. BUG_ON(new_min_nr <= 0);
  112. spin_lock_irqsave(&pool->lock, flags);
  113. if (new_min_nr <= pool->min_nr) {
  114. while (new_min_nr < pool->curr_nr) {
  115. element = remove_element(pool);
  116. spin_unlock_irqrestore(&pool->lock, flags);
  117. pool->free(element, pool->pool_data);
  118. spin_lock_irqsave(&pool->lock, flags);
  119. }
  120. pool->min_nr = new_min_nr;
  121. goto out_unlock;
  122. }
  123. spin_unlock_irqrestore(&pool->lock, flags);
  124. /* Grow the pool */
  125. new_elements = kmalloc(new_min_nr * sizeof(*new_elements), gfp_mask);
  126. if (!new_elements)
  127. return -ENOMEM;
  128. spin_lock_irqsave(&pool->lock, flags);
  129. if (unlikely(new_min_nr <= pool->min_nr)) {
  130. /* Raced, other resize will do our work */
  131. spin_unlock_irqrestore(&pool->lock, flags);
  132. kfree(new_elements);
  133. goto out;
  134. }
  135. memcpy(new_elements, pool->elements,
  136. pool->curr_nr * sizeof(*new_elements));
  137. kfree(pool->elements);
  138. pool->elements = new_elements;
  139. pool->min_nr = new_min_nr;
  140. while (pool->curr_nr < pool->min_nr) {
  141. spin_unlock_irqrestore(&pool->lock, flags);
  142. element = pool->alloc(gfp_mask, pool->pool_data);
  143. if (!element)
  144. goto out;
  145. spin_lock_irqsave(&pool->lock, flags);
  146. if (pool->curr_nr < pool->min_nr) {
  147. add_element(pool, element);
  148. } else {
  149. spin_unlock_irqrestore(&pool->lock, flags);
  150. pool->free(element, pool->pool_data); /* Raced */
  151. goto out;
  152. }
  153. }
  154. out_unlock:
  155. spin_unlock_irqrestore(&pool->lock, flags);
  156. out:
  157. return 0;
  158. }
  159. EXPORT_SYMBOL(mempool_resize);
  160. /**
  161. * mempool_destroy - deallocate a memory pool
  162. * @pool: pointer to the memory pool which was allocated via
  163. * mempool_create().
  164. *
  165. * this function only sleeps if the free_fn() function sleeps. The caller
  166. * has to guarantee that all elements have been returned to the pool (ie:
  167. * freed) prior to calling mempool_destroy().
  168. */
  169. void mempool_destroy(mempool_t *pool)
  170. {
  171. /* Check for outstanding elements */
  172. BUG_ON(pool->curr_nr != pool->min_nr);
  173. free_pool(pool);
  174. }
  175. EXPORT_SYMBOL(mempool_destroy);
  176. /**
  177. * mempool_alloc - allocate an element from a specific memory pool
  178. * @pool: pointer to the memory pool which was allocated via
  179. * mempool_create().
  180. * @gfp_mask: the usual allocation bitmask.
  181. *
  182. * this function only sleeps if the alloc_fn() function sleeps or
  183. * returns NULL. Note that due to preallocation, this function
  184. * *never* fails when called from process contexts. (it might
  185. * fail if called from an IRQ context.)
  186. */
  187. void * mempool_alloc(mempool_t *pool, gfp_t gfp_mask)
  188. {
  189. void *element;
  190. unsigned long flags;
  191. wait_queue_t wait;
  192. gfp_t gfp_temp;
  193. might_sleep_if(gfp_mask & __GFP_WAIT);
  194. gfp_mask |= __GFP_NOMEMALLOC; /* don't allocate emergency reserves */
  195. gfp_mask |= __GFP_NORETRY; /* don't loop in __alloc_pages */
  196. gfp_mask |= __GFP_NOWARN; /* failures are OK */
  197. gfp_temp = gfp_mask & ~(__GFP_WAIT|__GFP_IO);
  198. repeat_alloc:
  199. element = pool->alloc(gfp_temp, pool->pool_data);
  200. if (likely(element != NULL))
  201. return element;
  202. spin_lock_irqsave(&pool->lock, flags);
  203. if (likely(pool->curr_nr)) {
  204. element = remove_element(pool);
  205. spin_unlock_irqrestore(&pool->lock, flags);
  206. return element;
  207. }
  208. spin_unlock_irqrestore(&pool->lock, flags);
  209. /* We must not sleep in the GFP_ATOMIC case */
  210. if (!(gfp_mask & __GFP_WAIT))
  211. return NULL;
  212. /* Now start performing page reclaim */
  213. gfp_temp = gfp_mask;
  214. init_wait(&wait);
  215. prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);
  216. smp_mb();
  217. if (!pool->curr_nr) {
  218. /*
  219. * FIXME: this should be io_schedule(). The timeout is there
  220. * as a workaround for some DM problems in 2.6.18.
  221. */
  222. io_schedule_timeout(5*HZ);
  223. }
  224. finish_wait(&pool->wait, &wait);
  225. goto repeat_alloc;
  226. }
  227. EXPORT_SYMBOL(mempool_alloc);
  228. /**
  229. * mempool_free - return an element to the pool.
  230. * @element: pool element pointer.
  231. * @pool: pointer to the memory pool which was allocated via
  232. * mempool_create().
  233. *
  234. * this function only sleeps if the free_fn() function sleeps.
  235. */
  236. void mempool_free(void *element, mempool_t *pool)
  237. {
  238. unsigned long flags;
  239. smp_mb();
  240. if (pool->curr_nr < pool->min_nr) {
  241. spin_lock_irqsave(&pool->lock, flags);
  242. if (pool->curr_nr < pool->min_nr) {
  243. add_element(pool, element);
  244. spin_unlock_irqrestore(&pool->lock, flags);
  245. wake_up(&pool->wait);
  246. return;
  247. }
  248. spin_unlock_irqrestore(&pool->lock, flags);
  249. }
  250. pool->free(element, pool->pool_data);
  251. }
  252. EXPORT_SYMBOL(mempool_free);
  253. /*
  254. * A commonly used alloc and free fn.
  255. */
  256. void *mempool_alloc_slab(gfp_t gfp_mask, void *pool_data)
  257. {
  258. struct kmem_cache *mem = pool_data;
  259. return kmem_cache_alloc(mem, gfp_mask);
  260. }
  261. EXPORT_SYMBOL(mempool_alloc_slab);
  262. void mempool_free_slab(void *element, void *pool_data)
  263. {
  264. struct kmem_cache *mem = pool_data;
  265. kmem_cache_free(mem, element);
  266. }
  267. EXPORT_SYMBOL(mempool_free_slab);
  268. /*
  269. * A commonly used alloc and free fn that kmalloc/kfrees the amount of memory
  270. * specfied by pool_data
  271. */
  272. void *mempool_kmalloc(gfp_t gfp_mask, void *pool_data)
  273. {
  274. size_t size = (size_t)(long)pool_data;
  275. return kmalloc(size, gfp_mask);
  276. }
  277. EXPORT_SYMBOL(mempool_kmalloc);
  278. void *mempool_kzalloc(gfp_t gfp_mask, void *pool_data)
  279. {
  280. size_t size = (size_t) pool_data;
  281. return kzalloc(size, gfp_mask);
  282. }
  283. EXPORT_SYMBOL(mempool_kzalloc);
  284. void mempool_kfree(void *element, void *pool_data)
  285. {
  286. kfree(element);
  287. }
  288. EXPORT_SYMBOL(mempool_kfree);
  289. /*
  290. * A simple mempool-backed page allocator that allocates pages
  291. * of the order specified by pool_data.
  292. */
  293. void *mempool_alloc_pages(gfp_t gfp_mask, void *pool_data)
  294. {
  295. int order = (int)(long)pool_data;
  296. return alloc_pages(gfp_mask, order);
  297. }
  298. EXPORT_SYMBOL(mempool_alloc_pages);
  299. void mempool_free_pages(void *element, void *pool_data)
  300. {
  301. int order = (int)(long)pool_data;
  302. __free_pages(element, order);
  303. }
  304. EXPORT_SYMBOL(mempool_free_pages);