sg_pool.c 4.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. #include <linux/module.h>
  3. #include <linux/scatterlist.h>
  4. #include <linux/mempool.h>
  5. #include <linux/slab.h>
  6. #define SG_MEMPOOL_NR ARRAY_SIZE(sg_pools)
  7. #define SG_MEMPOOL_SIZE 2
  8. struct sg_pool {
  9. size_t size;
  10. char *name;
  11. struct kmem_cache *slab;
  12. mempool_t *pool;
  13. };
  14. #define SP(x) { .size = x, "sgpool-" __stringify(x) }
  15. #if (SG_CHUNK_SIZE < 32)
  16. #error SG_CHUNK_SIZE is too small (must be 32 or greater)
  17. #endif
  18. static struct sg_pool sg_pools[] = {
  19. SP(8),
  20. SP(16),
  21. #if (SG_CHUNK_SIZE > 32)
  22. SP(32),
  23. #if (SG_CHUNK_SIZE > 64)
  24. SP(64),
  25. #if (SG_CHUNK_SIZE > 128)
  26. SP(128),
  27. #if (SG_CHUNK_SIZE > 256)
  28. #error SG_CHUNK_SIZE is too large (256 MAX)
  29. #endif
  30. #endif
  31. #endif
  32. #endif
  33. SP(SG_CHUNK_SIZE)
  34. };
  35. #undef SP
  36. static inline unsigned int sg_pool_index(unsigned short nents)
  37. {
  38. unsigned int index;
  39. BUG_ON(nents > SG_CHUNK_SIZE);
  40. if (nents <= 8)
  41. index = 0;
  42. else
  43. index = get_count_order(nents) - 3;
  44. return index;
  45. }
  46. static void sg_pool_free(struct scatterlist *sgl, unsigned int nents)
  47. {
  48. struct sg_pool *sgp;
  49. sgp = sg_pools + sg_pool_index(nents);
  50. mempool_free(sgl, sgp->pool);
  51. }
  52. static struct scatterlist *sg_pool_alloc(unsigned int nents, gfp_t gfp_mask)
  53. {
  54. struct sg_pool *sgp;
  55. sgp = sg_pools + sg_pool_index(nents);
  56. return mempool_alloc(sgp->pool, gfp_mask);
  57. }
  58. /**
  59. * sg_free_table_chained - Free a previously mapped sg table
  60. * @table: The sg table header to use
  61. * @nents_first_chunk: size of the first_chunk SGL passed to
  62. * sg_alloc_table_chained
  63. *
  64. * Description:
  65. * Free an sg table previously allocated and setup with
  66. * sg_alloc_table_chained().
  67. *
  68. * @nents_first_chunk has to be same with that same parameter passed
  69. * to sg_alloc_table_chained().
  70. *
  71. **/
  72. void sg_free_table_chained(struct sg_table *table,
  73. unsigned nents_first_chunk)
  74. {
  75. if (table->orig_nents <= nents_first_chunk)
  76. return;
  77. if (nents_first_chunk == 1)
  78. nents_first_chunk = 0;
  79. __sg_free_table(table, SG_CHUNK_SIZE, nents_first_chunk, sg_pool_free);
  80. }
  81. EXPORT_SYMBOL_GPL(sg_free_table_chained);
  82. /**
  83. * sg_alloc_table_chained - Allocate and chain SGLs in an sg table
  84. * @table: The sg table header to use
  85. * @nents: Number of entries in sg list
  86. * @first_chunk: first SGL
  87. * @nents_first_chunk: number of the SGL of @first_chunk
  88. *
  89. * Description:
  90. * Allocate and chain SGLs in an sg table. If @nents@ is larger than
  91. * @nents_first_chunk a chained sg table will be setup. @first_chunk is
  92. * ignored if nents_first_chunk <= 1 because user expects the SGL points
  93. * non-chain SGL.
  94. *
  95. **/
  96. int sg_alloc_table_chained(struct sg_table *table, int nents,
  97. struct scatterlist *first_chunk, unsigned nents_first_chunk)
  98. {
  99. int ret;
  100. BUG_ON(!nents);
  101. if (first_chunk && nents_first_chunk) {
  102. if (nents <= nents_first_chunk) {
  103. table->nents = table->orig_nents = nents;
  104. sg_init_table(table->sgl, nents);
  105. return 0;
  106. }
  107. }
  108. /* User supposes that the 1st SGL includes real entry */
  109. if (nents_first_chunk <= 1) {
  110. first_chunk = NULL;
  111. nents_first_chunk = 0;
  112. }
  113. ret = __sg_alloc_table(table, nents, SG_CHUNK_SIZE,
  114. first_chunk, nents_first_chunk,
  115. GFP_ATOMIC, sg_pool_alloc);
  116. if (unlikely(ret))
  117. sg_free_table_chained(table, nents_first_chunk);
  118. return ret;
  119. }
  120. EXPORT_SYMBOL_GPL(sg_alloc_table_chained);
  121. static __init int sg_pool_init(void)
  122. {
  123. int i;
  124. for (i = 0; i < SG_MEMPOOL_NR; i++) {
  125. struct sg_pool *sgp = sg_pools + i;
  126. int size = sgp->size * sizeof(struct scatterlist);
  127. sgp->slab = kmem_cache_create(sgp->name, size, 0,
  128. SLAB_HWCACHE_ALIGN, NULL);
  129. if (!sgp->slab) {
  130. printk(KERN_ERR "SG_POOL: can't init sg slab %s\n",
  131. sgp->name);
  132. goto cleanup_sdb;
  133. }
  134. sgp->pool = mempool_create_slab_pool(SG_MEMPOOL_SIZE,
  135. sgp->slab);
  136. if (!sgp->pool) {
  137. printk(KERN_ERR "SG_POOL: can't init sg mempool %s\n",
  138. sgp->name);
  139. goto cleanup_sdb;
  140. }
  141. }
  142. return 0;
  143. cleanup_sdb:
  144. for (i = 0; i < SG_MEMPOOL_NR; i++) {
  145. struct sg_pool *sgp = sg_pools + i;
  146. mempool_destroy(sgp->pool);
  147. kmem_cache_destroy(sgp->slab);
  148. }
  149. return -ENOMEM;
  150. }
  151. static __exit void sg_pool_exit(void)
  152. {
  153. int i;
  154. for (i = 0; i < SG_MEMPOOL_NR; i++) {
  155. struct sg_pool *sgp = sg_pools + i;
  156. mempool_destroy(sgp->pool);
  157. kmem_cache_destroy(sgp->slab);
  158. }
  159. }
  160. module_init(sg_pool_init);
  161. module_exit(sg_pool_exit);