dmabounce.c 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658
  1. /*
  2. * arch/arm/common/dmabounce.c
  3. *
  4. * Special dma_{map/unmap/dma_sync}_* routines for systems that have
  5. * limited DMA windows. These functions utilize bounce buffers to
  6. * copy data to/from buffers located outside the DMA region. This
  7. * only works for systems in which DMA memory is at the bottom of
  8. * RAM, the remainder of memory is at the top and the DMA memory
  9. * can be marked as ZONE_DMA. Anything beyond that such as discontigous
  10. * DMA windows will require custom implementations that reserve memory
  11. * areas at early bootup.
  12. *
  13. * Original version by Brad Parker (brad@heeltoe.com)
  14. * Re-written by Christopher Hoover <ch@murgatroid.com>
  15. * Made generic by Deepak Saxena <dsaxena@plexity.net>
  16. *
  17. * Copyright (C) 2002 Hewlett Packard Company.
  18. * Copyright (C) 2004 MontaVista Software, Inc.
  19. *
  20. * This program is free software; you can redistribute it and/or
  21. * modify it under the terms of the GNU General Public License
  22. * version 2 as published by the Free Software Foundation.
  23. */
  24. #include <linux/module.h>
  25. #include <linux/init.h>
  26. #include <linux/slab.h>
  27. #include <linux/device.h>
  28. #include <linux/dma-mapping.h>
  29. #include <linux/dmapool.h>
  30. #include <linux/list.h>
  31. #include <asm/cacheflush.h>
  32. #undef STATS
  33. #ifdef STATS
  34. #define DO_STATS(X) do { X ; } while (0)
  35. #else
  36. #define DO_STATS(X) do { } while (0)
  37. #endif
  38. /* ************************************************** */
  39. struct safe_buffer {
  40. struct list_head node;
  41. /* original request */
  42. void *ptr;
  43. size_t size;
  44. int direction;
  45. /* safe buffer info */
  46. struct dmabounce_pool *pool;
  47. void *safe;
  48. dma_addr_t safe_dma_addr;
  49. };
  50. struct dmabounce_pool {
  51. unsigned long size;
  52. struct dma_pool *pool;
  53. #ifdef STATS
  54. unsigned long allocs;
  55. #endif
  56. };
  57. struct dmabounce_device_info {
  58. struct device *dev;
  59. struct list_head safe_buffers;
  60. #ifdef STATS
  61. unsigned long total_allocs;
  62. unsigned long map_op_count;
  63. unsigned long bounce_count;
  64. int attr_res;
  65. #endif
  66. struct dmabounce_pool small;
  67. struct dmabounce_pool large;
  68. rwlock_t lock;
  69. };
  70. #ifdef STATS
  71. static ssize_t dmabounce_show(struct device *dev, struct device_attribute *attr,
  72. char *buf)
  73. {
  74. struct dmabounce_device_info *device_info = dev->archdata.dmabounce;
  75. return sprintf(buf, "%lu %lu %lu %lu %lu %lu\n",
  76. device_info->small.allocs,
  77. device_info->large.allocs,
  78. device_info->total_allocs - device_info->small.allocs -
  79. device_info->large.allocs,
  80. device_info->total_allocs,
  81. device_info->map_op_count,
  82. device_info->bounce_count);
  83. }
  84. static DEVICE_ATTR(dmabounce_stats, 0400, dmabounce_show, NULL);
  85. #endif
  86. /* allocate a 'safe' buffer and keep track of it */
  87. static inline struct safe_buffer *
  88. alloc_safe_buffer(struct dmabounce_device_info *device_info, void *ptr,
  89. size_t size, enum dma_data_direction dir)
  90. {
  91. struct safe_buffer *buf;
  92. struct dmabounce_pool *pool;
  93. struct device *dev = device_info->dev;
  94. unsigned long flags;
  95. dev_dbg(dev, "%s(ptr=%p, size=%d, dir=%d)\n",
  96. __func__, ptr, size, dir);
  97. if (size <= device_info->small.size) {
  98. pool = &device_info->small;
  99. } else if (size <= device_info->large.size) {
  100. pool = &device_info->large;
  101. } else {
  102. pool = NULL;
  103. }
  104. buf = kmalloc(sizeof(struct safe_buffer), GFP_ATOMIC);
  105. if (buf == NULL) {
  106. dev_warn(dev, "%s: kmalloc failed\n", __func__);
  107. return NULL;
  108. }
  109. buf->ptr = ptr;
  110. buf->size = size;
  111. buf->direction = dir;
  112. buf->pool = pool;
  113. if (pool) {
  114. buf->safe = dma_pool_alloc(pool->pool, GFP_ATOMIC,
  115. &buf->safe_dma_addr);
  116. } else {
  117. buf->safe = dma_alloc_coherent(dev, size, &buf->safe_dma_addr,
  118. GFP_ATOMIC);
  119. }
  120. if (buf->safe == NULL) {
  121. dev_warn(dev,
  122. "%s: could not alloc dma memory (size=%d)\n",
  123. __func__, size);
  124. kfree(buf);
  125. return NULL;
  126. }
  127. #ifdef STATS
  128. if (pool)
  129. pool->allocs++;
  130. device_info->total_allocs++;
  131. #endif
  132. write_lock_irqsave(&device_info->lock, flags);
  133. list_add(&buf->node, &device_info->safe_buffers);
  134. write_unlock_irqrestore(&device_info->lock, flags);
  135. return buf;
  136. }
  137. /* determine if a buffer is from our "safe" pool */
  138. static inline struct safe_buffer *
  139. find_safe_buffer(struct dmabounce_device_info *device_info, dma_addr_t safe_dma_addr)
  140. {
  141. struct safe_buffer *b, *rb = NULL;
  142. unsigned long flags;
  143. read_lock_irqsave(&device_info->lock, flags);
  144. list_for_each_entry(b, &device_info->safe_buffers, node)
  145. if (b->safe_dma_addr == safe_dma_addr) {
  146. rb = b;
  147. break;
  148. }
  149. read_unlock_irqrestore(&device_info->lock, flags);
  150. return rb;
  151. }
  152. static inline void
  153. free_safe_buffer(struct dmabounce_device_info *device_info, struct safe_buffer *buf)
  154. {
  155. unsigned long flags;
  156. dev_dbg(device_info->dev, "%s(buf=%p)\n", __func__, buf);
  157. write_lock_irqsave(&device_info->lock, flags);
  158. list_del(&buf->node);
  159. write_unlock_irqrestore(&device_info->lock, flags);
  160. if (buf->pool)
  161. dma_pool_free(buf->pool->pool, buf->safe, buf->safe_dma_addr);
  162. else
  163. dma_free_coherent(device_info->dev, buf->size, buf->safe,
  164. buf->safe_dma_addr);
  165. kfree(buf);
  166. }
  167. /* ************************************************** */
  168. static inline dma_addr_t
  169. map_single(struct device *dev, void *ptr, size_t size,
  170. enum dma_data_direction dir)
  171. {
  172. struct dmabounce_device_info *device_info = dev->archdata.dmabounce;
  173. dma_addr_t dma_addr;
  174. int needs_bounce = 0;
  175. if (device_info)
  176. DO_STATS ( device_info->map_op_count++ );
  177. dma_addr = virt_to_dma(dev, ptr);
  178. if (dev->dma_mask) {
  179. unsigned long mask = *dev->dma_mask;
  180. unsigned long limit;
  181. limit = (mask + 1) & ~mask;
  182. if (limit && size > limit) {
  183. dev_err(dev, "DMA mapping too big (requested %#x "
  184. "mask %#Lx)\n", size, *dev->dma_mask);
  185. return ~0;
  186. }
  187. /*
  188. * Figure out if we need to bounce from the DMA mask.
  189. */
  190. needs_bounce = (dma_addr | (dma_addr + size - 1)) & ~mask;
  191. }
  192. if (device_info && (needs_bounce || dma_needs_bounce(dev, dma_addr, size))) {
  193. struct safe_buffer *buf;
  194. buf = alloc_safe_buffer(device_info, ptr, size, dir);
  195. if (buf == 0) {
  196. dev_err(dev, "%s: unable to map unsafe buffer %p!\n",
  197. __func__, ptr);
  198. return 0;
  199. }
  200. dev_dbg(dev,
  201. "%s: unsafe buffer %p (phy=%p) mapped to %p (phy=%p)\n",
  202. __func__, buf->ptr, (void *) virt_to_dma(dev, buf->ptr),
  203. buf->safe, (void *) buf->safe_dma_addr);
  204. if ((dir == DMA_TO_DEVICE) ||
  205. (dir == DMA_BIDIRECTIONAL)) {
  206. dev_dbg(dev, "%s: copy unsafe %p to safe %p, size %d\n",
  207. __func__, ptr, buf->safe, size);
  208. memcpy(buf->safe, ptr, size);
  209. }
  210. ptr = buf->safe;
  211. dma_addr = buf->safe_dma_addr;
  212. } else {
  213. /*
  214. * We don't need to sync the DMA buffer since
  215. * it was allocated via the coherent allocators.
  216. */
  217. consistent_sync(ptr, size, dir);
  218. }
  219. return dma_addr;
  220. }
  221. static inline void
  222. unmap_single(struct device *dev, dma_addr_t dma_addr, size_t size,
  223. enum dma_data_direction dir)
  224. {
  225. struct dmabounce_device_info *device_info = dev->archdata.dmabounce;
  226. struct safe_buffer *buf = NULL;
  227. /*
  228. * Trying to unmap an invalid mapping
  229. */
  230. if (dma_mapping_error(dma_addr)) {
  231. dev_err(dev, "Trying to unmap invalid mapping\n");
  232. return;
  233. }
  234. if (device_info)
  235. buf = find_safe_buffer(device_info, dma_addr);
  236. if (buf) {
  237. BUG_ON(buf->size != size);
  238. dev_dbg(dev,
  239. "%s: unsafe buffer %p (phy=%p) mapped to %p (phy=%p)\n",
  240. __func__, buf->ptr, (void *) virt_to_dma(dev, buf->ptr),
  241. buf->safe, (void *) buf->safe_dma_addr);
  242. DO_STATS ( device_info->bounce_count++ );
  243. if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) {
  244. void *ptr = buf->ptr;
  245. dev_dbg(dev,
  246. "%s: copy back safe %p to unsafe %p size %d\n",
  247. __func__, buf->safe, ptr, size);
  248. memcpy(ptr, buf->safe, size);
  249. /*
  250. * DMA buffers must have the same cache properties
  251. * as if they were really used for DMA - which means
  252. * data must be written back to RAM. Note that
  253. * we don't use dmac_flush_range() here for the
  254. * bidirectional case because we know the cache
  255. * lines will be coherent with the data written.
  256. */
  257. dmac_clean_range(ptr, ptr + size);
  258. outer_clean_range(__pa(ptr), __pa(ptr) + size);
  259. }
  260. free_safe_buffer(device_info, buf);
  261. }
  262. }
  263. static inline void
  264. sync_single(struct device *dev, dma_addr_t dma_addr, size_t size,
  265. enum dma_data_direction dir)
  266. {
  267. struct dmabounce_device_info *device_info = dev->archdata.dmabounce;
  268. struct safe_buffer *buf = NULL;
  269. if (device_info)
  270. buf = find_safe_buffer(device_info, dma_addr);
  271. if (buf) {
  272. /*
  273. * Both of these checks from original code need to be
  274. * commented out b/c some drivers rely on the following:
  275. *
  276. * 1) Drivers may map a large chunk of memory into DMA space
  277. * but only sync a small portion of it. Good example is
  278. * allocating a large buffer, mapping it, and then
  279. * breaking it up into small descriptors. No point
  280. * in syncing the whole buffer if you only have to
  281. * touch one descriptor.
  282. *
  283. * 2) Buffers that are mapped as DMA_BIDIRECTIONAL are
  284. * usually only synced in one dir at a time.
  285. *
  286. * See drivers/net/eepro100.c for examples of both cases.
  287. *
  288. * -ds
  289. *
  290. * BUG_ON(buf->size != size);
  291. * BUG_ON(buf->direction != dir);
  292. */
  293. dev_dbg(dev,
  294. "%s: unsafe buffer %p (phy=%p) mapped to %p (phy=%p)\n",
  295. __func__, buf->ptr, (void *) virt_to_dma(dev, buf->ptr),
  296. buf->safe, (void *) buf->safe_dma_addr);
  297. DO_STATS ( device_info->bounce_count++ );
  298. switch (dir) {
  299. case DMA_FROM_DEVICE:
  300. dev_dbg(dev,
  301. "%s: copy back safe %p to unsafe %p size %d\n",
  302. __func__, buf->safe, buf->ptr, size);
  303. memcpy(buf->ptr, buf->safe, size);
  304. break;
  305. case DMA_TO_DEVICE:
  306. dev_dbg(dev,
  307. "%s: copy out unsafe %p to safe %p, size %d\n",
  308. __func__,buf->ptr, buf->safe, size);
  309. memcpy(buf->safe, buf->ptr, size);
  310. break;
  311. case DMA_BIDIRECTIONAL:
  312. BUG(); /* is this allowed? what does it mean? */
  313. default:
  314. BUG();
  315. }
  316. /*
  317. * No need to sync the safe buffer - it was allocated
  318. * via the coherent allocators.
  319. */
  320. } else {
  321. consistent_sync(dma_to_virt(dev, dma_addr), size, dir);
  322. }
  323. }
  324. /* ************************************************** */
  325. /*
  326. * see if a buffer address is in an 'unsafe' range. if it is
  327. * allocate a 'safe' buffer and copy the unsafe buffer into it.
  328. * substitute the safe buffer for the unsafe one.
  329. * (basically move the buffer from an unsafe area to a safe one)
  330. */
  331. dma_addr_t
  332. dma_map_single(struct device *dev, void *ptr, size_t size,
  333. enum dma_data_direction dir)
  334. {
  335. dma_addr_t dma_addr;
  336. dev_dbg(dev, "%s(ptr=%p,size=%d,dir=%x)\n",
  337. __func__, ptr, size, dir);
  338. BUG_ON(dir == DMA_NONE);
  339. dma_addr = map_single(dev, ptr, size, dir);
  340. return dma_addr;
  341. }
  342. /*
  343. * see if a mapped address was really a "safe" buffer and if so, copy
  344. * the data from the safe buffer back to the unsafe buffer and free up
  345. * the safe buffer. (basically return things back to the way they
  346. * should be)
  347. */
  348. void
  349. dma_unmap_single(struct device *dev, dma_addr_t dma_addr, size_t size,
  350. enum dma_data_direction dir)
  351. {
  352. dev_dbg(dev, "%s(ptr=%p,size=%d,dir=%x)\n",
  353. __func__, (void *) dma_addr, size, dir);
  354. BUG_ON(dir == DMA_NONE);
  355. unmap_single(dev, dma_addr, size, dir);
  356. }
  357. int
  358. dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
  359. enum dma_data_direction dir)
  360. {
  361. int i;
  362. dev_dbg(dev, "%s(sg=%p,nents=%d,dir=%x)\n",
  363. __func__, sg, nents, dir);
  364. BUG_ON(dir == DMA_NONE);
  365. for (i = 0; i < nents; i++, sg++) {
  366. struct page *page = sg->page;
  367. unsigned int offset = sg->offset;
  368. unsigned int length = sg->length;
  369. void *ptr = page_address(page) + offset;
  370. sg->dma_address =
  371. map_single(dev, ptr, length, dir);
  372. }
  373. return nents;
  374. }
  375. void
  376. dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
  377. enum dma_data_direction dir)
  378. {
  379. int i;
  380. dev_dbg(dev, "%s(sg=%p,nents=%d,dir=%x)\n",
  381. __func__, sg, nents, dir);
  382. BUG_ON(dir == DMA_NONE);
  383. for (i = 0; i < nents; i++, sg++) {
  384. dma_addr_t dma_addr = sg->dma_address;
  385. unsigned int length = sg->length;
  386. unmap_single(dev, dma_addr, length, dir);
  387. }
  388. }
  389. void
  390. dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_addr, size_t size,
  391. enum dma_data_direction dir)
  392. {
  393. dev_dbg(dev, "%s(ptr=%p,size=%d,dir=%x)\n",
  394. __func__, (void *) dma_addr, size, dir);
  395. sync_single(dev, dma_addr, size, dir);
  396. }
  397. void
  398. dma_sync_single_for_device(struct device *dev, dma_addr_t dma_addr, size_t size,
  399. enum dma_data_direction dir)
  400. {
  401. dev_dbg(dev, "%s(ptr=%p,size=%d,dir=%x)\n",
  402. __func__, (void *) dma_addr, size, dir);
  403. sync_single(dev, dma_addr, size, dir);
  404. }
  405. void
  406. dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, int nents,
  407. enum dma_data_direction dir)
  408. {
  409. int i;
  410. dev_dbg(dev, "%s(sg=%p,nents=%d,dir=%x)\n",
  411. __func__, sg, nents, dir);
  412. BUG_ON(dir == DMA_NONE);
  413. for (i = 0; i < nents; i++, sg++) {
  414. dma_addr_t dma_addr = sg->dma_address;
  415. unsigned int length = sg->length;
  416. sync_single(dev, dma_addr, length, dir);
  417. }
  418. }
  419. void
  420. dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, int nents,
  421. enum dma_data_direction dir)
  422. {
  423. int i;
  424. dev_dbg(dev, "%s(sg=%p,nents=%d,dir=%x)\n",
  425. __func__, sg, nents, dir);
  426. BUG_ON(dir == DMA_NONE);
  427. for (i = 0; i < nents; i++, sg++) {
  428. dma_addr_t dma_addr = sg->dma_address;
  429. unsigned int length = sg->length;
  430. sync_single(dev, dma_addr, length, dir);
  431. }
  432. }
  433. static int
  434. dmabounce_init_pool(struct dmabounce_pool *pool, struct device *dev, const char *name,
  435. unsigned long size)
  436. {
  437. pool->size = size;
  438. DO_STATS(pool->allocs = 0);
  439. pool->pool = dma_pool_create(name, dev, size,
  440. 0 /* byte alignment */,
  441. 0 /* no page-crossing issues */);
  442. return pool->pool ? 0 : -ENOMEM;
  443. }
  444. int
  445. dmabounce_register_dev(struct device *dev, unsigned long small_buffer_size,
  446. unsigned long large_buffer_size)
  447. {
  448. struct dmabounce_device_info *device_info;
  449. int ret;
  450. device_info = kmalloc(sizeof(struct dmabounce_device_info), GFP_ATOMIC);
  451. if (!device_info) {
  452. printk(KERN_ERR
  453. "Could not allocated dmabounce_device_info for %s",
  454. dev->bus_id);
  455. return -ENOMEM;
  456. }
  457. ret = dmabounce_init_pool(&device_info->small, dev,
  458. "small_dmabounce_pool", small_buffer_size);
  459. if (ret) {
  460. dev_err(dev,
  461. "dmabounce: could not allocate DMA pool for %ld byte objects\n",
  462. small_buffer_size);
  463. goto err_free;
  464. }
  465. if (large_buffer_size) {
  466. ret = dmabounce_init_pool(&device_info->large, dev,
  467. "large_dmabounce_pool",
  468. large_buffer_size);
  469. if (ret) {
  470. dev_err(dev,
  471. "dmabounce: could not allocate DMA pool for %ld byte objects\n",
  472. large_buffer_size);
  473. goto err_destroy;
  474. }
  475. }
  476. device_info->dev = dev;
  477. INIT_LIST_HEAD(&device_info->safe_buffers);
  478. rwlock_init(&device_info->lock);
  479. #ifdef STATS
  480. device_info->total_allocs = 0;
  481. device_info->map_op_count = 0;
  482. device_info->bounce_count = 0;
  483. device_info->attr_res = device_create_file(dev, &dev_attr_dmabounce_stats);
  484. #endif
  485. dev->archdata.dmabounce = device_info;
  486. printk(KERN_INFO "dmabounce: registered device %s on %s bus\n",
  487. dev->bus_id, dev->bus->name);
  488. return 0;
  489. err_destroy:
  490. dma_pool_destroy(device_info->small.pool);
  491. err_free:
  492. kfree(device_info);
  493. return ret;
  494. }
  495. void
  496. dmabounce_unregister_dev(struct device *dev)
  497. {
  498. struct dmabounce_device_info *device_info = dev->archdata.dmabounce;
  499. dev->archdata.dmabounce = NULL;
  500. if (!device_info) {
  501. printk(KERN_WARNING
  502. "%s: Never registered with dmabounce but attempting" \
  503. "to unregister!\n", dev->bus_id);
  504. return;
  505. }
  506. if (!list_empty(&device_info->safe_buffers)) {
  507. printk(KERN_ERR
  508. "%s: Removing from dmabounce with pending buffers!\n",
  509. dev->bus_id);
  510. BUG();
  511. }
  512. if (device_info->small.pool)
  513. dma_pool_destroy(device_info->small.pool);
  514. if (device_info->large.pool)
  515. dma_pool_destroy(device_info->large.pool);
  516. #ifdef STATS
  517. if (device_info->attr_res == 0)
  518. device_remove_file(dev, &dev_attr_dmabounce_stats);
  519. #endif
  520. kfree(device_info);
  521. printk(KERN_INFO "dmabounce: device %s on %s bus unregistered\n",
  522. dev->bus_id, dev->bus->name);
  523. }
  524. EXPORT_SYMBOL(dma_map_single);
  525. EXPORT_SYMBOL(dma_unmap_single);
  526. EXPORT_SYMBOL(dma_map_sg);
  527. EXPORT_SYMBOL(dma_unmap_sg);
  528. EXPORT_SYMBOL(dma_sync_single_for_cpu);
  529. EXPORT_SYMBOL(dma_sync_single_for_device);
  530. EXPORT_SYMBOL(dma_sync_sg);
  531. EXPORT_SYMBOL(dmabounce_register_dev);
  532. EXPORT_SYMBOL(dmabounce_unregister_dev);
  533. MODULE_AUTHOR("Christopher Hoover <ch@hpl.hp.com>, Deepak Saxena <dsaxena@plexity.net>");
  534. MODULE_DESCRIPTION("Special dma_{map/unmap/dma_sync}_* routines for systems with limited DMA windows");
  535. MODULE_LICENSE("GPL");