dmapool.c 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * DMA Pool allocator
  4. *
  5. * Copyright 2001 David Brownell
  6. * Copyright 2007 Intel Corporation
  7. * Author: Matthew Wilcox <willy@linux.intel.com>
  8. *
  9. * This allocator returns small blocks of a given size which are DMA-able by
  10. * the given device. It uses the dma_alloc_coherent page allocator to get
  11. * new pages, then splits them up into blocks of the required size.
  12. * Many older drivers still have their own code to do this.
  13. *
  14. * The current design of this allocator is fairly simple. The pool is
  15. * represented by the 'struct dma_pool' which keeps a doubly-linked list of
  16. * allocated pages. Each page in the page_list is split into blocks of at
  17. * least 'size' bytes. Free blocks are tracked in an unsorted singly-linked
  18. * list of free blocks within the page. Used blocks aren't tracked, but we
  19. * keep a count of how many are currently allocated from each page.
  20. */
  21. #include <linux/device.h>
  22. #include <linux/dma-mapping.h>
  23. #include <linux/dmapool.h>
  24. #include <linux/kernel.h>
  25. #include <linux/list.h>
  26. #include <linux/export.h>
  27. #include <linux/mutex.h>
  28. #include <linux/poison.h>
  29. #include <linux/sched.h>
  30. #include <linux/slab.h>
  31. #include <linux/stat.h>
  32. #include <linux/spinlock.h>
  33. #include <linux/string.h>
  34. #include <linux/types.h>
  35. #include <linux/wait.h>
  36. #if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_SLUB_DEBUG_ON)
  37. #define DMAPOOL_DEBUG 1
  38. #endif
  39. struct dma_pool { /* the pool */
  40. struct list_head page_list;
  41. spinlock_t lock;
  42. size_t size;
  43. struct device *dev;
  44. size_t allocation;
  45. size_t boundary;
  46. char name[32];
  47. struct list_head pools;
  48. };
  49. struct dma_page { /* cacheable header for 'allocation' bytes */
  50. struct list_head page_list;
  51. void *vaddr;
  52. dma_addr_t dma;
  53. unsigned int in_use;
  54. unsigned int offset;
  55. };
  56. static DEFINE_MUTEX(pools_lock);
  57. static DEFINE_MUTEX(pools_reg_lock);
  58. static ssize_t
  59. show_pools(struct device *dev, struct device_attribute *attr, char *buf)
  60. {
  61. unsigned temp;
  62. unsigned size;
  63. char *next;
  64. struct dma_page *page;
  65. struct dma_pool *pool;
  66. next = buf;
  67. size = PAGE_SIZE;
  68. temp = scnprintf(next, size, "poolinfo - 0.1\n");
  69. size -= temp;
  70. next += temp;
  71. mutex_lock(&pools_lock);
  72. list_for_each_entry(pool, &dev->dma_pools, pools) {
  73. unsigned pages = 0;
  74. unsigned blocks = 0;
  75. spin_lock_irq(&pool->lock);
  76. list_for_each_entry(page, &pool->page_list, page_list) {
  77. pages++;
  78. blocks += page->in_use;
  79. }
  80. spin_unlock_irq(&pool->lock);
  81. /* per-pool info, no real statistics yet */
  82. temp = scnprintf(next, size, "%-16s %4u %4zu %4zu %2u\n",
  83. pool->name, blocks,
  84. pages * (pool->allocation / pool->size),
  85. pool->size, pages);
  86. size -= temp;
  87. next += temp;
  88. }
  89. mutex_unlock(&pools_lock);
  90. return PAGE_SIZE - size;
  91. }
  92. static DEVICE_ATTR(pools, 0444, show_pools, NULL);
  93. /**
  94. * dma_pool_create - Creates a pool of consistent memory blocks, for dma.
  95. * @name: name of pool, for diagnostics
  96. * @dev: device that will be doing the DMA
  97. * @size: size of the blocks in this pool.
  98. * @align: alignment requirement for blocks; must be a power of two
  99. * @boundary: returned blocks won't cross this power of two boundary
  100. * Context: not in_interrupt()
  101. *
  102. * Given one of these pools, dma_pool_alloc()
  103. * may be used to allocate memory. Such memory will all have "consistent"
  104. * DMA mappings, accessible by the device and its driver without using
  105. * cache flushing primitives. The actual size of blocks allocated may be
  106. * larger than requested because of alignment.
  107. *
  108. * If @boundary is nonzero, objects returned from dma_pool_alloc() won't
  109. * cross that size boundary. This is useful for devices which have
  110. * addressing restrictions on individual DMA transfers, such as not crossing
  111. * boundaries of 4KBytes.
  112. *
  113. * Return: a dma allocation pool with the requested characteristics, or
  114. * %NULL if one can't be created.
  115. */
  116. struct dma_pool *dma_pool_create(const char *name, struct device *dev,
  117. size_t size, size_t align, size_t boundary)
  118. {
  119. struct dma_pool *retval;
  120. size_t allocation;
  121. bool empty = false;
  122. if (align == 0)
  123. align = 1;
  124. else if (align & (align - 1))
  125. return NULL;
  126. if (size == 0)
  127. return NULL;
  128. else if (size < 4)
  129. size = 4;
  130. size = ALIGN(size, align);
  131. allocation = max_t(size_t, size, PAGE_SIZE);
  132. if (!boundary)
  133. boundary = allocation;
  134. else if ((boundary < size) || (boundary & (boundary - 1)))
  135. return NULL;
  136. retval = kmalloc_node(sizeof(*retval), GFP_KERNEL, dev_to_node(dev));
  137. if (!retval)
  138. return retval;
  139. strlcpy(retval->name, name, sizeof(retval->name));
  140. retval->dev = dev;
  141. INIT_LIST_HEAD(&retval->page_list);
  142. spin_lock_init(&retval->lock);
  143. retval->size = size;
  144. retval->boundary = boundary;
  145. retval->allocation = allocation;
  146. INIT_LIST_HEAD(&retval->pools);
  147. /*
  148. * pools_lock ensures that the ->dma_pools list does not get corrupted.
  149. * pools_reg_lock ensures that there is not a race between
  150. * dma_pool_create() and dma_pool_destroy() or within dma_pool_create()
  151. * when the first invocation of dma_pool_create() failed on
  152. * device_create_file() and the second assumes that it has been done (I
  153. * know it is a short window).
  154. */
  155. mutex_lock(&pools_reg_lock);
  156. mutex_lock(&pools_lock);
  157. if (list_empty(&dev->dma_pools))
  158. empty = true;
  159. list_add(&retval->pools, &dev->dma_pools);
  160. mutex_unlock(&pools_lock);
  161. if (empty) {
  162. int err;
  163. err = device_create_file(dev, &dev_attr_pools);
  164. if (err) {
  165. mutex_lock(&pools_lock);
  166. list_del(&retval->pools);
  167. mutex_unlock(&pools_lock);
  168. mutex_unlock(&pools_reg_lock);
  169. kfree(retval);
  170. return NULL;
  171. }
  172. }
  173. mutex_unlock(&pools_reg_lock);
  174. return retval;
  175. }
  176. EXPORT_SYMBOL(dma_pool_create);
  177. static void pool_initialise_page(struct dma_pool *pool, struct dma_page *page)
  178. {
  179. unsigned int offset = 0;
  180. unsigned int next_boundary = pool->boundary;
  181. do {
  182. unsigned int next = offset + pool->size;
  183. if (unlikely((next + pool->size) >= next_boundary)) {
  184. next = next_boundary;
  185. next_boundary += pool->boundary;
  186. }
  187. *(int *)(page->vaddr + offset) = next;
  188. offset = next;
  189. } while (offset < pool->allocation);
  190. }
  191. static struct dma_page *pool_alloc_page(struct dma_pool *pool, gfp_t mem_flags)
  192. {
  193. struct dma_page *page;
  194. page = kmalloc(sizeof(*page), mem_flags);
  195. if (!page)
  196. return NULL;
  197. page->vaddr = dma_alloc_coherent(pool->dev, pool->allocation,
  198. &page->dma, mem_flags);
  199. if (page->vaddr) {
  200. #ifdef DMAPOOL_DEBUG
  201. memset(page->vaddr, POOL_POISON_FREED, pool->allocation);
  202. #endif
  203. pool_initialise_page(pool, page);
  204. page->in_use = 0;
  205. page->offset = 0;
  206. } else {
  207. kfree(page);
  208. page = NULL;
  209. }
  210. return page;
  211. }
  212. static inline bool is_page_busy(struct dma_page *page)
  213. {
  214. return page->in_use != 0;
  215. }
  216. static void pool_free_page(struct dma_pool *pool, struct dma_page *page)
  217. {
  218. dma_addr_t dma = page->dma;
  219. #ifdef DMAPOOL_DEBUG
  220. memset(page->vaddr, POOL_POISON_FREED, pool->allocation);
  221. #endif
  222. dma_free_coherent(pool->dev, pool->allocation, page->vaddr, dma);
  223. list_del(&page->page_list);
  224. kfree(page);
  225. }
  226. /**
  227. * dma_pool_destroy - destroys a pool of dma memory blocks.
  228. * @pool: dma pool that will be destroyed
  229. * Context: !in_interrupt()
  230. *
  231. * Caller guarantees that no more memory from the pool is in use,
  232. * and that nothing will try to use the pool after this call.
  233. */
  234. void dma_pool_destroy(struct dma_pool *pool)
  235. {
  236. struct dma_page *page, *tmp;
  237. bool empty = false;
  238. if (unlikely(!pool))
  239. return;
  240. mutex_lock(&pools_reg_lock);
  241. mutex_lock(&pools_lock);
  242. list_del(&pool->pools);
  243. if (pool->dev && list_empty(&pool->dev->dma_pools))
  244. empty = true;
  245. mutex_unlock(&pools_lock);
  246. if (empty)
  247. device_remove_file(pool->dev, &dev_attr_pools);
  248. mutex_unlock(&pools_reg_lock);
  249. list_for_each_entry_safe(page, tmp, &pool->page_list, page_list) {
  250. if (is_page_busy(page)) {
  251. if (pool->dev)
  252. dev_err(pool->dev, "%s %s, %p busy\n", __func__,
  253. pool->name, page->vaddr);
  254. else
  255. pr_err("%s %s, %p busy\n", __func__,
  256. pool->name, page->vaddr);
  257. /* leak the still-in-use consistent memory */
  258. list_del(&page->page_list);
  259. kfree(page);
  260. } else
  261. pool_free_page(pool, page);
  262. }
  263. kfree(pool);
  264. }
  265. EXPORT_SYMBOL(dma_pool_destroy);
  266. /**
  267. * dma_pool_alloc - get a block of consistent memory
  268. * @pool: dma pool that will produce the block
  269. * @mem_flags: GFP_* bitmask
  270. * @handle: pointer to dma address of block
  271. *
  272. * Return: the kernel virtual address of a currently unused block,
  273. * and reports its dma address through the handle.
  274. * If such a memory block can't be allocated, %NULL is returned.
  275. */
  276. void *dma_pool_alloc(struct dma_pool *pool, gfp_t mem_flags,
  277. dma_addr_t *handle)
  278. {
  279. unsigned long flags;
  280. struct dma_page *page;
  281. size_t offset;
  282. void *retval;
  283. might_sleep_if(gfpflags_allow_blocking(mem_flags));
  284. spin_lock_irqsave(&pool->lock, flags);
  285. list_for_each_entry(page, &pool->page_list, page_list) {
  286. if (page->offset < pool->allocation)
  287. goto ready;
  288. }
  289. /* pool_alloc_page() might sleep, so temporarily drop &pool->lock */
  290. spin_unlock_irqrestore(&pool->lock, flags);
  291. page = pool_alloc_page(pool, mem_flags & (~__GFP_ZERO));
  292. if (!page)
  293. return NULL;
  294. spin_lock_irqsave(&pool->lock, flags);
  295. list_add(&page->page_list, &pool->page_list);
  296. ready:
  297. page->in_use++;
  298. offset = page->offset;
  299. page->offset = *(int *)(page->vaddr + offset);
  300. retval = offset + page->vaddr;
  301. *handle = offset + page->dma;
  302. #ifdef DMAPOOL_DEBUG
  303. {
  304. int i;
  305. u8 *data = retval;
  306. /* page->offset is stored in first 4 bytes */
  307. for (i = sizeof(page->offset); i < pool->size; i++) {
  308. if (data[i] == POOL_POISON_FREED)
  309. continue;
  310. if (pool->dev)
  311. dev_err(pool->dev, "%s %s, %p (corrupted)\n",
  312. __func__, pool->name, retval);
  313. else
  314. pr_err("%s %s, %p (corrupted)\n",
  315. __func__, pool->name, retval);
  316. /*
  317. * Dump the first 4 bytes even if they are not
  318. * POOL_POISON_FREED
  319. */
  320. print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 16, 1,
  321. data, pool->size, 1);
  322. break;
  323. }
  324. }
  325. if (!(mem_flags & __GFP_ZERO))
  326. memset(retval, POOL_POISON_ALLOCATED, pool->size);
  327. #endif
  328. spin_unlock_irqrestore(&pool->lock, flags);
  329. if (want_init_on_alloc(mem_flags))
  330. memset(retval, 0, pool->size);
  331. return retval;
  332. }
  333. EXPORT_SYMBOL(dma_pool_alloc);
  334. static struct dma_page *pool_find_page(struct dma_pool *pool, dma_addr_t dma)
  335. {
  336. struct dma_page *page;
  337. list_for_each_entry(page, &pool->page_list, page_list) {
  338. if (dma < page->dma)
  339. continue;
  340. if ((dma - page->dma) < pool->allocation)
  341. return page;
  342. }
  343. return NULL;
  344. }
  345. /**
  346. * dma_pool_free - put block back into dma pool
  347. * @pool: the dma pool holding the block
  348. * @vaddr: virtual address of block
  349. * @dma: dma address of block
  350. *
  351. * Caller promises neither device nor driver will again touch this block
  352. * unless it is first re-allocated.
  353. */
  354. void dma_pool_free(struct dma_pool *pool, void *vaddr, dma_addr_t dma)
  355. {
  356. struct dma_page *page;
  357. unsigned long flags;
  358. unsigned int offset;
  359. spin_lock_irqsave(&pool->lock, flags);
  360. page = pool_find_page(pool, dma);
  361. if (!page) {
  362. spin_unlock_irqrestore(&pool->lock, flags);
  363. if (pool->dev)
  364. dev_err(pool->dev, "%s %s, %p/%pad (bad dma)\n",
  365. __func__, pool->name, vaddr, &dma);
  366. else
  367. pr_err("%s %s, %p/%pad (bad dma)\n",
  368. __func__, pool->name, vaddr, &dma);
  369. return;
  370. }
  371. offset = vaddr - page->vaddr;
  372. if (want_init_on_free())
  373. memset(vaddr, 0, pool->size);
  374. #ifdef DMAPOOL_DEBUG
  375. if ((dma - page->dma) != offset) {
  376. spin_unlock_irqrestore(&pool->lock, flags);
  377. if (pool->dev)
  378. dev_err(pool->dev, "%s %s, %p (bad vaddr)/%pad\n",
  379. __func__, pool->name, vaddr, &dma);
  380. else
  381. pr_err("%s %s, %p (bad vaddr)/%pad\n",
  382. __func__, pool->name, vaddr, &dma);
  383. return;
  384. }
  385. {
  386. unsigned int chain = page->offset;
  387. while (chain < pool->allocation) {
  388. if (chain != offset) {
  389. chain = *(int *)(page->vaddr + chain);
  390. continue;
  391. }
  392. spin_unlock_irqrestore(&pool->lock, flags);
  393. if (pool->dev)
  394. dev_err(pool->dev, "%s %s, dma %pad already free\n",
  395. __func__, pool->name, &dma);
  396. else
  397. pr_err("%s %s, dma %pad already free\n",
  398. __func__, pool->name, &dma);
  399. return;
  400. }
  401. }
  402. memset(vaddr, POOL_POISON_FREED, pool->size);
  403. #endif
  404. page->in_use--;
  405. *(int *)vaddr = page->offset;
  406. page->offset = offset;
  407. /*
  408. * Resist a temptation to do
  409. * if (!is_page_busy(page)) pool_free_page(pool, page);
  410. * Better have a few empty pages hang around.
  411. */
  412. spin_unlock_irqrestore(&pool->lock, flags);
  413. }
  414. EXPORT_SYMBOL(dma_pool_free);
  415. /*
  416. * Managed DMA pool
  417. */
  418. static void dmam_pool_release(struct device *dev, void *res)
  419. {
  420. struct dma_pool *pool = *(struct dma_pool **)res;
  421. dma_pool_destroy(pool);
  422. }
  423. static int dmam_pool_match(struct device *dev, void *res, void *match_data)
  424. {
  425. return *(struct dma_pool **)res == match_data;
  426. }
  427. /**
  428. * dmam_pool_create - Managed dma_pool_create()
  429. * @name: name of pool, for diagnostics
  430. * @dev: device that will be doing the DMA
  431. * @size: size of the blocks in this pool.
  432. * @align: alignment requirement for blocks; must be a power of two
  433. * @allocation: returned blocks won't cross this boundary (or zero)
  434. *
  435. * Managed dma_pool_create(). DMA pool created with this function is
  436. * automatically destroyed on driver detach.
  437. *
  438. * Return: a managed dma allocation pool with the requested
  439. * characteristics, or %NULL if one can't be created.
  440. */
  441. struct dma_pool *dmam_pool_create(const char *name, struct device *dev,
  442. size_t size, size_t align, size_t allocation)
  443. {
  444. struct dma_pool **ptr, *pool;
  445. ptr = devres_alloc(dmam_pool_release, sizeof(*ptr), GFP_KERNEL);
  446. if (!ptr)
  447. return NULL;
  448. pool = *ptr = dma_pool_create(name, dev, size, align, allocation);
  449. if (pool)
  450. devres_add(dev, ptr);
  451. else
  452. devres_free(ptr);
  453. return pool;
  454. }
  455. EXPORT_SYMBOL(dmam_pool_create);
  456. /**
  457. * dmam_pool_destroy - Managed dma_pool_destroy()
  458. * @pool: dma pool that will be destroyed
  459. *
  460. * Managed dma_pool_destroy().
  461. */
  462. void dmam_pool_destroy(struct dma_pool *pool)
  463. {
  464. struct device *dev = pool->dev;
  465. WARN_ON(devres_release(dev, dmam_pool_release, dmam_pool_match, pool));
  466. }
  467. EXPORT_SYMBOL(dmam_pool_destroy);