swiotlb.c 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795
  1. /*
  2. * Dynamic DMA mapping support.
  3. *
  4. * This implementation is a fallback for platforms that do not support
  5. * I/O TLBs (aka DMA address translation hardware).
  6. * Copyright (C) 2000 Asit Mallick <Asit.K.Mallick@intel.com>
  7. * Copyright (C) 2000 Goutham Rao <goutham.rao@intel.com>
  8. * Copyright (C) 2000, 2003 Hewlett-Packard Co
  9. * David Mosberger-Tang <davidm@hpl.hp.com>
  10. *
  11. * 03/05/07 davidm Switch from PCI-DMA to generic device DMA API.
  12. * 00/12/13 davidm Rename to swiotlb.c and add mark_clean() to avoid
  13. * unnecessary i-cache flushing.
  14. * 04/07/.. ak Better overflow handling. Assorted fixes.
  15. * 05/09/10 linville Add support for syncing ranges, support syncing for
  16. * DMA_BIDIRECTIONAL mappings, miscellaneous cleanup.
  17. */
  18. #include <linux/cache.h>
  19. #include <linux/dma-mapping.h>
  20. #include <linux/mm.h>
  21. #include <linux/module.h>
  22. #include <linux/spinlock.h>
  23. #include <linux/string.h>
  24. #include <linux/types.h>
  25. #include <linux/ctype.h>
  26. #include <asm/io.h>
  27. #include <asm/dma.h>
  28. #include <asm/scatterlist.h>
  29. #include <linux/init.h>
  30. #include <linux/bootmem.h>
  31. #define OFFSET(val,align) ((unsigned long) \
  32. ( (val) & ( (align) - 1)))
  33. #define SG_ENT_VIRT_ADDRESS(sg) (page_address((sg)->page) + (sg)->offset)
  34. #define SG_ENT_PHYS_ADDRESS(sg) virt_to_bus(SG_ENT_VIRT_ADDRESS(sg))
  35. /*
  36. * Maximum allowable number of contiguous slabs to map,
  37. * must be a power of 2. What is the appropriate value ?
  38. * The complexity of {map,unmap}_single is linearly dependent on this value.
  39. */
  40. #define IO_TLB_SEGSIZE 128
  41. /*
  42. * log of the size of each IO TLB slab. The number of slabs is command line
  43. * controllable.
  44. */
  45. #define IO_TLB_SHIFT 11
  46. #define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT))
  47. /*
  48. * Minimum IO TLB size to bother booting with. Systems with mainly
  49. * 64bit capable cards will only lightly use the swiotlb. If we can't
  50. * allocate a contiguous 1MB, we're probably in trouble anyway.
  51. */
  52. #define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT)
  53. /*
  54. * Enumeration for sync targets
  55. */
  56. enum dma_sync_target {
  57. SYNC_FOR_CPU = 0,
  58. SYNC_FOR_DEVICE = 1,
  59. };
  60. int swiotlb_force;
  61. /*
  62. * Used to do a quick range check in swiotlb_unmap_single and
  63. * swiotlb_sync_single_*, to see if the memory was in fact allocated by this
  64. * API.
  65. */
  66. static char *io_tlb_start, *io_tlb_end;
  67. /*
  68. * The number of IO TLB blocks (in groups of 64) betweeen io_tlb_start and
  69. * io_tlb_end. This is command line adjustable via setup_io_tlb_npages.
  70. */
  71. static unsigned long io_tlb_nslabs;
  72. /*
  73. * When the IOMMU overflows we return a fallback buffer. This sets the size.
  74. */
  75. static unsigned long io_tlb_overflow = 32*1024;
  76. void *io_tlb_overflow_buffer;
  77. /*
  78. * This is a free list describing the number of free entries available from
  79. * each index
  80. */
  81. static unsigned int *io_tlb_list;
  82. static unsigned int io_tlb_index;
  83. /*
  84. * We need to save away the original address corresponding to a mapped entry
  85. * for the sync operations.
  86. */
  87. static unsigned char **io_tlb_orig_addr;
  88. /*
  89. * Protect the above data structures in the map and unmap calls
  90. */
  91. static DEFINE_SPINLOCK(io_tlb_lock);
  92. static int __init
  93. setup_io_tlb_npages(char *str)
  94. {
  95. if (isdigit(*str)) {
  96. io_tlb_nslabs = simple_strtoul(str, &str, 0);
  97. /* avoid tail segment of size < IO_TLB_SEGSIZE */
  98. io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
  99. }
  100. if (*str == ',')
  101. ++str;
  102. if (!strcmp(str, "force"))
  103. swiotlb_force = 1;
  104. return 1;
  105. }
  106. __setup("swiotlb=", setup_io_tlb_npages);
  107. /* make io_tlb_overflow tunable too? */
  108. /*
  109. * Statically reserve bounce buffer space and initialize bounce buffer data
  110. * structures for the software IO TLB used to implement the DMA API.
  111. */
  112. void __init
  113. swiotlb_init_with_default_size(size_t default_size)
  114. {
  115. unsigned long i, bytes;
  116. if (!io_tlb_nslabs) {
  117. io_tlb_nslabs = (default_size >> IO_TLB_SHIFT);
  118. io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
  119. }
  120. bytes = io_tlb_nslabs << IO_TLB_SHIFT;
  121. /*
  122. * Get IO TLB memory from the low pages
  123. */
  124. io_tlb_start = alloc_bootmem_low_pages(bytes);
  125. if (!io_tlb_start)
  126. panic("Cannot allocate SWIOTLB buffer");
  127. io_tlb_end = io_tlb_start + bytes;
  128. /*
  129. * Allocate and initialize the free list array. This array is used
  130. * to find contiguous free memory regions of size up to IO_TLB_SEGSIZE
  131. * between io_tlb_start and io_tlb_end.
  132. */
  133. io_tlb_list = alloc_bootmem(io_tlb_nslabs * sizeof(int));
  134. for (i = 0; i < io_tlb_nslabs; i++)
  135. io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE);
  136. io_tlb_index = 0;
  137. io_tlb_orig_addr = alloc_bootmem(io_tlb_nslabs * sizeof(char *));
  138. /*
  139. * Get the overflow emergency buffer
  140. */
  141. io_tlb_overflow_buffer = alloc_bootmem_low(io_tlb_overflow);
  142. if (!io_tlb_overflow_buffer)
  143. panic("Cannot allocate SWIOTLB overflow buffer!\n");
  144. printk(KERN_INFO "Placing software IO TLB between 0x%lx - 0x%lx\n",
  145. virt_to_bus(io_tlb_start), virt_to_bus(io_tlb_end));
  146. }
  147. void __init
  148. swiotlb_init(void)
  149. {
  150. swiotlb_init_with_default_size(64 * (1<<20)); /* default to 64MB */
  151. }
  152. /*
  153. * Systems with larger DMA zones (those that don't support ISA) can
  154. * initialize the swiotlb later using the slab allocator if needed.
  155. * This should be just like above, but with some error catching.
  156. */
  157. int
  158. swiotlb_late_init_with_default_size(size_t default_size)
  159. {
  160. unsigned long i, bytes, req_nslabs = io_tlb_nslabs;
  161. unsigned int order;
  162. if (!io_tlb_nslabs) {
  163. io_tlb_nslabs = (default_size >> IO_TLB_SHIFT);
  164. io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
  165. }
  166. /*
  167. * Get IO TLB memory from the low pages
  168. */
  169. order = get_order(io_tlb_nslabs << IO_TLB_SHIFT);
  170. io_tlb_nslabs = SLABS_PER_PAGE << order;
  171. bytes = io_tlb_nslabs << IO_TLB_SHIFT;
  172. while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) {
  173. io_tlb_start = (char *)__get_free_pages(GFP_DMA | __GFP_NOWARN,
  174. order);
  175. if (io_tlb_start)
  176. break;
  177. order--;
  178. }
  179. if (!io_tlb_start)
  180. goto cleanup1;
  181. if (order != get_order(bytes)) {
  182. printk(KERN_WARNING "Warning: only able to allocate %ld MB "
  183. "for software IO TLB\n", (PAGE_SIZE << order) >> 20);
  184. io_tlb_nslabs = SLABS_PER_PAGE << order;
  185. bytes = io_tlb_nslabs << IO_TLB_SHIFT;
  186. }
  187. io_tlb_end = io_tlb_start + bytes;
  188. memset(io_tlb_start, 0, bytes);
  189. /*
  190. * Allocate and initialize the free list array. This array is used
  191. * to find contiguous free memory regions of size up to IO_TLB_SEGSIZE
  192. * between io_tlb_start and io_tlb_end.
  193. */
  194. io_tlb_list = (unsigned int *)__get_free_pages(GFP_KERNEL,
  195. get_order(io_tlb_nslabs * sizeof(int)));
  196. if (!io_tlb_list)
  197. goto cleanup2;
  198. for (i = 0; i < io_tlb_nslabs; i++)
  199. io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE);
  200. io_tlb_index = 0;
  201. io_tlb_orig_addr = (unsigned char **)__get_free_pages(GFP_KERNEL,
  202. get_order(io_tlb_nslabs * sizeof(char *)));
  203. if (!io_tlb_orig_addr)
  204. goto cleanup3;
  205. memset(io_tlb_orig_addr, 0, io_tlb_nslabs * sizeof(char *));
  206. /*
  207. * Get the overflow emergency buffer
  208. */
  209. io_tlb_overflow_buffer = (void *)__get_free_pages(GFP_DMA,
  210. get_order(io_tlb_overflow));
  211. if (!io_tlb_overflow_buffer)
  212. goto cleanup4;
  213. printk(KERN_INFO "Placing %luMB software IO TLB between 0x%lx - "
  214. "0x%lx\n", bytes >> 20,
  215. virt_to_bus(io_tlb_start), virt_to_bus(io_tlb_end));
  216. return 0;
  217. cleanup4:
  218. free_pages((unsigned long)io_tlb_orig_addr, get_order(io_tlb_nslabs *
  219. sizeof(char *)));
  220. io_tlb_orig_addr = NULL;
  221. cleanup3:
  222. free_pages((unsigned long)io_tlb_list, get_order(io_tlb_nslabs *
  223. sizeof(int)));
  224. io_tlb_list = NULL;
  225. cleanup2:
  226. io_tlb_end = NULL;
  227. free_pages((unsigned long)io_tlb_start, order);
  228. io_tlb_start = NULL;
  229. cleanup1:
  230. io_tlb_nslabs = req_nslabs;
  231. return -ENOMEM;
  232. }
  233. static int
  234. address_needs_mapping(struct device *hwdev, dma_addr_t addr)
  235. {
  236. dma_addr_t mask = 0xffffffff;
  237. /* If the device has a mask, use it, otherwise default to 32 bits */
  238. if (hwdev && hwdev->dma_mask)
  239. mask = *hwdev->dma_mask;
  240. return (addr & ~mask) != 0;
  241. }
  242. /*
  243. * Allocates bounce buffer and returns its kernel virtual address.
  244. */
  245. static void *
  246. map_single(struct device *hwdev, char *buffer, size_t size, int dir)
  247. {
  248. unsigned long flags;
  249. char *dma_addr;
  250. unsigned int nslots, stride, index, wrap;
  251. int i;
  252. /*
  253. * For mappings greater than a page, we limit the stride (and
  254. * hence alignment) to a page size.
  255. */
  256. nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
  257. if (size > PAGE_SIZE)
  258. stride = (1 << (PAGE_SHIFT - IO_TLB_SHIFT));
  259. else
  260. stride = 1;
  261. BUG_ON(!nslots);
  262. /*
  263. * Find suitable number of IO TLB entries size that will fit this
  264. * request and allocate a buffer from that IO TLB pool.
  265. */
  266. spin_lock_irqsave(&io_tlb_lock, flags);
  267. {
  268. wrap = index = ALIGN(io_tlb_index, stride);
  269. if (index >= io_tlb_nslabs)
  270. wrap = index = 0;
  271. do {
  272. /*
  273. * If we find a slot that indicates we have 'nslots'
  274. * number of contiguous buffers, we allocate the
  275. * buffers from that slot and mark the entries as '0'
  276. * indicating unavailable.
  277. */
  278. if (io_tlb_list[index] >= nslots) {
  279. int count = 0;
  280. for (i = index; i < (int) (index + nslots); i++)
  281. io_tlb_list[i] = 0;
  282. for (i = index - 1; (OFFSET(i, IO_TLB_SEGSIZE) != IO_TLB_SEGSIZE -1) && io_tlb_list[i]; i--)
  283. io_tlb_list[i] = ++count;
  284. dma_addr = io_tlb_start + (index << IO_TLB_SHIFT);
  285. /*
  286. * Update the indices to avoid searching in
  287. * the next round.
  288. */
  289. io_tlb_index = ((index + nslots) < io_tlb_nslabs
  290. ? (index + nslots) : 0);
  291. goto found;
  292. }
  293. index += stride;
  294. if (index >= io_tlb_nslabs)
  295. index = 0;
  296. } while (index != wrap);
  297. spin_unlock_irqrestore(&io_tlb_lock, flags);
  298. return NULL;
  299. }
  300. found:
  301. spin_unlock_irqrestore(&io_tlb_lock, flags);
  302. /*
  303. * Save away the mapping from the original address to the DMA address.
  304. * This is needed when we sync the memory. Then we sync the buffer if
  305. * needed.
  306. */
  307. io_tlb_orig_addr[index] = buffer;
  308. if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
  309. memcpy(dma_addr, buffer, size);
  310. return dma_addr;
  311. }
  312. /*
  313. * dma_addr is the kernel virtual address of the bounce buffer to unmap.
  314. */
  315. static void
  316. unmap_single(struct device *hwdev, char *dma_addr, size_t size, int dir)
  317. {
  318. unsigned long flags;
  319. int i, count, nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
  320. int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT;
  321. char *buffer = io_tlb_orig_addr[index];
  322. /*
  323. * First, sync the memory before unmapping the entry
  324. */
  325. if (buffer && ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL)))
  326. /*
  327. * bounce... copy the data back into the original buffer * and
  328. * delete the bounce buffer.
  329. */
  330. memcpy(buffer, dma_addr, size);
  331. /*
  332. * Return the buffer to the free list by setting the corresponding
  333. * entries to indicate the number of contigous entries available.
  334. * While returning the entries to the free list, we merge the entries
  335. * with slots below and above the pool being returned.
  336. */
  337. spin_lock_irqsave(&io_tlb_lock, flags);
  338. {
  339. count = ((index + nslots) < ALIGN(index + 1, IO_TLB_SEGSIZE) ?
  340. io_tlb_list[index + nslots] : 0);
  341. /*
  342. * Step 1: return the slots to the free list, merging the
  343. * slots with superceeding slots
  344. */
  345. for (i = index + nslots - 1; i >= index; i--)
  346. io_tlb_list[i] = ++count;
  347. /*
  348. * Step 2: merge the returned slots with the preceding slots,
  349. * if available (non zero)
  350. */
  351. for (i = index - 1; (OFFSET(i, IO_TLB_SEGSIZE) != IO_TLB_SEGSIZE -1) && io_tlb_list[i]; i--)
  352. io_tlb_list[i] = ++count;
  353. }
  354. spin_unlock_irqrestore(&io_tlb_lock, flags);
  355. }
  356. static void
  357. sync_single(struct device *hwdev, char *dma_addr, size_t size,
  358. int dir, int target)
  359. {
  360. int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT;
  361. char *buffer = io_tlb_orig_addr[index];
  362. switch (target) {
  363. case SYNC_FOR_CPU:
  364. if (likely(dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL))
  365. memcpy(buffer, dma_addr, size);
  366. else
  367. BUG_ON(dir != DMA_TO_DEVICE);
  368. break;
  369. case SYNC_FOR_DEVICE:
  370. if (likely(dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL))
  371. memcpy(dma_addr, buffer, size);
  372. else
  373. BUG_ON(dir != DMA_FROM_DEVICE);
  374. break;
  375. default:
  376. BUG();
  377. }
  378. }
  379. void *
  380. swiotlb_alloc_coherent(struct device *hwdev, size_t size,
  381. dma_addr_t *dma_handle, gfp_t flags)
  382. {
  383. dma_addr_t dev_addr;
  384. void *ret;
  385. int order = get_order(size);
  386. /*
  387. * XXX fix me: the DMA API should pass us an explicit DMA mask
  388. * instead, or use ZONE_DMA32 (ia64 overloads ZONE_DMA to be a ~32
  389. * bit range instead of a 16MB one).
  390. */
  391. flags |= GFP_DMA;
  392. ret = (void *)__get_free_pages(flags, order);
  393. if (ret && address_needs_mapping(hwdev, virt_to_bus(ret))) {
  394. /*
  395. * The allocated memory isn't reachable by the device.
  396. * Fall back on swiotlb_map_single().
  397. */
  398. free_pages((unsigned long) ret, order);
  399. ret = NULL;
  400. }
  401. if (!ret) {
  402. /*
  403. * We are either out of memory or the device can't DMA
  404. * to GFP_DMA memory; fall back on
  405. * swiotlb_map_single(), which will grab memory from
  406. * the lowest available address range.
  407. */
  408. dma_addr_t handle;
  409. handle = swiotlb_map_single(NULL, NULL, size, DMA_FROM_DEVICE);
  410. if (swiotlb_dma_mapping_error(handle))
  411. return NULL;
  412. ret = bus_to_virt(handle);
  413. }
  414. memset(ret, 0, size);
  415. dev_addr = virt_to_bus(ret);
  416. /* Confirm address can be DMA'd by device */
  417. if (address_needs_mapping(hwdev, dev_addr)) {
  418. printk("hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n",
  419. (unsigned long long)*hwdev->dma_mask,
  420. (unsigned long long)dev_addr);
  421. panic("swiotlb_alloc_coherent: allocated memory is out of "
  422. "range for device");
  423. }
  424. *dma_handle = dev_addr;
  425. return ret;
  426. }
  427. void
  428. swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr,
  429. dma_addr_t dma_handle)
  430. {
  431. if (!(vaddr >= (void *)io_tlb_start
  432. && vaddr < (void *)io_tlb_end))
  433. free_pages((unsigned long) vaddr, get_order(size));
  434. else
  435. /* DMA_TO_DEVICE to avoid memcpy in unmap_single */
  436. swiotlb_unmap_single (hwdev, dma_handle, size, DMA_TO_DEVICE);
  437. }
  438. static void
  439. swiotlb_full(struct device *dev, size_t size, int dir, int do_panic)
  440. {
  441. /*
  442. * Ran out of IOMMU space for this operation. This is very bad.
  443. * Unfortunately the drivers cannot handle this operation properly.
  444. * unless they check for dma_mapping_error (most don't)
  445. * When the mapping is small enough return a static buffer to limit
  446. * the damage, or panic when the transfer is too big.
  447. */
  448. printk(KERN_ERR "DMA: Out of SW-IOMMU space for %zu bytes at "
  449. "device %s\n", size, dev ? dev->bus_id : "?");
  450. if (size > io_tlb_overflow && do_panic) {
  451. if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
  452. panic("DMA: Memory would be corrupted\n");
  453. if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
  454. panic("DMA: Random memory would be DMAed\n");
  455. }
  456. }
  457. /*
  458. * Map a single buffer of the indicated size for DMA in streaming mode. The
  459. * physical address to use is returned.
  460. *
  461. * Once the device is given the dma address, the device owns this memory until
  462. * either swiotlb_unmap_single or swiotlb_dma_sync_single is performed.
  463. */
  464. dma_addr_t
  465. swiotlb_map_single(struct device *hwdev, void *ptr, size_t size, int dir)
  466. {
  467. dma_addr_t dev_addr = virt_to_bus(ptr);
  468. void *map;
  469. BUG_ON(dir == DMA_NONE);
  470. /*
  471. * If the pointer passed in happens to be in the device's DMA window,
  472. * we can safely return the device addr and not worry about bounce
  473. * buffering it.
  474. */
  475. if (!address_needs_mapping(hwdev, dev_addr) && !swiotlb_force)
  476. return dev_addr;
  477. /*
  478. * Oh well, have to allocate and map a bounce buffer.
  479. */
  480. map = map_single(hwdev, ptr, size, dir);
  481. if (!map) {
  482. swiotlb_full(hwdev, size, dir, 1);
  483. map = io_tlb_overflow_buffer;
  484. }
  485. dev_addr = virt_to_bus(map);
  486. /*
  487. * Ensure that the address returned is DMA'ble
  488. */
  489. if (address_needs_mapping(hwdev, dev_addr))
  490. panic("map_single: bounce buffer is not DMA'ble");
  491. return dev_addr;
  492. }
  493. /*
  494. * Unmap a single streaming mode DMA translation. The dma_addr and size must
  495. * match what was provided for in a previous swiotlb_map_single call. All
  496. * other usages are undefined.
  497. *
  498. * After this call, reads by the cpu to the buffer are guaranteed to see
  499. * whatever the device wrote there.
  500. */
  501. void
  502. swiotlb_unmap_single(struct device *hwdev, dma_addr_t dev_addr, size_t size,
  503. int dir)
  504. {
  505. char *dma_addr = bus_to_virt(dev_addr);
  506. BUG_ON(dir == DMA_NONE);
  507. if (dma_addr >= io_tlb_start && dma_addr < io_tlb_end)
  508. unmap_single(hwdev, dma_addr, size, dir);
  509. else if (dir == DMA_FROM_DEVICE)
  510. dma_mark_clean(dma_addr, size);
  511. }
  512. /*
  513. * Make physical memory consistent for a single streaming mode DMA translation
  514. * after a transfer.
  515. *
  516. * If you perform a swiotlb_map_single() but wish to interrogate the buffer
  517. * using the cpu, yet do not wish to teardown the dma mapping, you must
  518. * call this function before doing so. At the next point you give the dma
  519. * address back to the card, you must first perform a
  520. * swiotlb_dma_sync_for_device, and then the device again owns the buffer
  521. */
  522. static void
  523. swiotlb_sync_single(struct device *hwdev, dma_addr_t dev_addr,
  524. size_t size, int dir, int target)
  525. {
  526. char *dma_addr = bus_to_virt(dev_addr);
  527. BUG_ON(dir == DMA_NONE);
  528. if (dma_addr >= io_tlb_start && dma_addr < io_tlb_end)
  529. sync_single(hwdev, dma_addr, size, dir, target);
  530. else if (dir == DMA_FROM_DEVICE)
  531. dma_mark_clean(dma_addr, size);
  532. }
  533. void
  534. swiotlb_sync_single_for_cpu(struct device *hwdev, dma_addr_t dev_addr,
  535. size_t size, int dir)
  536. {
  537. swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_CPU);
  538. }
  539. void
  540. swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr,
  541. size_t size, int dir)
  542. {
  543. swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_DEVICE);
  544. }
  545. /*
  546. * Same as above, but for a sub-range of the mapping.
  547. */
  548. static void
  549. swiotlb_sync_single_range(struct device *hwdev, dma_addr_t dev_addr,
  550. unsigned long offset, size_t size,
  551. int dir, int target)
  552. {
  553. char *dma_addr = bus_to_virt(dev_addr) + offset;
  554. BUG_ON(dir == DMA_NONE);
  555. if (dma_addr >= io_tlb_start && dma_addr < io_tlb_end)
  556. sync_single(hwdev, dma_addr, size, dir, target);
  557. else if (dir == DMA_FROM_DEVICE)
  558. dma_mark_clean(dma_addr, size);
  559. }
  560. void
  561. swiotlb_sync_single_range_for_cpu(struct device *hwdev, dma_addr_t dev_addr,
  562. unsigned long offset, size_t size, int dir)
  563. {
  564. swiotlb_sync_single_range(hwdev, dev_addr, offset, size, dir,
  565. SYNC_FOR_CPU);
  566. }
  567. void
  568. swiotlb_sync_single_range_for_device(struct device *hwdev, dma_addr_t dev_addr,
  569. unsigned long offset, size_t size, int dir)
  570. {
  571. swiotlb_sync_single_range(hwdev, dev_addr, offset, size, dir,
  572. SYNC_FOR_DEVICE);
  573. }
  574. /*
  575. * Map a set of buffers described by scatterlist in streaming mode for DMA.
  576. * This is the scatter-gather version of the above swiotlb_map_single
  577. * interface. Here the scatter gather list elements are each tagged with the
  578. * appropriate dma address and length. They are obtained via
  579. * sg_dma_{address,length}(SG).
  580. *
  581. * NOTE: An implementation may be able to use a smaller number of
  582. * DMA address/length pairs than there are SG table elements.
  583. * (for example via virtual mapping capabilities)
  584. * The routine returns the number of addr/length pairs actually
  585. * used, at most nents.
  586. *
  587. * Device ownership issues as mentioned above for swiotlb_map_single are the
  588. * same here.
  589. */
  590. int
  591. swiotlb_map_sg(struct device *hwdev, struct scatterlist *sg, int nelems,
  592. int dir)
  593. {
  594. void *addr;
  595. dma_addr_t dev_addr;
  596. int i;
  597. BUG_ON(dir == DMA_NONE);
  598. for (i = 0; i < nelems; i++, sg++) {
  599. addr = SG_ENT_VIRT_ADDRESS(sg);
  600. dev_addr = virt_to_bus(addr);
  601. if (swiotlb_force || address_needs_mapping(hwdev, dev_addr)) {
  602. void *map = map_single(hwdev, addr, sg->length, dir);
  603. if (!map) {
  604. /* Don't panic here, we expect map_sg users
  605. to do proper error handling. */
  606. swiotlb_full(hwdev, sg->length, dir, 0);
  607. swiotlb_unmap_sg(hwdev, sg - i, i, dir);
  608. sg[0].dma_length = 0;
  609. return 0;
  610. }
  611. sg->dma_address = virt_to_bus(map);
  612. } else
  613. sg->dma_address = dev_addr;
  614. sg->dma_length = sg->length;
  615. }
  616. return nelems;
  617. }
  618. /*
  619. * Unmap a set of streaming mode DMA translations. Again, cpu read rules
  620. * concerning calls here are the same as for swiotlb_unmap_single() above.
  621. */
  622. void
  623. swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sg, int nelems,
  624. int dir)
  625. {
  626. int i;
  627. BUG_ON(dir == DMA_NONE);
  628. for (i = 0; i < nelems; i++, sg++)
  629. if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg))
  630. unmap_single(hwdev, bus_to_virt(sg->dma_address),
  631. sg->dma_length, dir);
  632. else if (dir == DMA_FROM_DEVICE)
  633. dma_mark_clean(SG_ENT_VIRT_ADDRESS(sg), sg->dma_length);
  634. }
  635. /*
  636. * Make physical memory consistent for a set of streaming mode DMA translations
  637. * after a transfer.
  638. *
  639. * The same as swiotlb_sync_single_* but for a scatter-gather list, same rules
  640. * and usage.
  641. */
  642. static void
  643. swiotlb_sync_sg(struct device *hwdev, struct scatterlist *sg,
  644. int nelems, int dir, int target)
  645. {
  646. int i;
  647. BUG_ON(dir == DMA_NONE);
  648. for (i = 0; i < nelems; i++, sg++)
  649. if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg))
  650. sync_single(hwdev, bus_to_virt(sg->dma_address),
  651. sg->dma_length, dir, target);
  652. else if (dir == DMA_FROM_DEVICE)
  653. dma_mark_clean(SG_ENT_VIRT_ADDRESS(sg), sg->dma_length);
  654. }
  655. void
  656. swiotlb_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg,
  657. int nelems, int dir)
  658. {
  659. swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_CPU);
  660. }
  661. void
  662. swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,
  663. int nelems, int dir)
  664. {
  665. swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_DEVICE);
  666. }
  667. int
  668. swiotlb_dma_mapping_error(dma_addr_t dma_addr)
  669. {
  670. return (dma_addr == virt_to_bus(io_tlb_overflow_buffer));
  671. }
  672. /*
  673. * Return whether the given device DMA address mask can be supported
  674. * properly. For example, if your device can only drive the low 24-bits
  675. * during bus mastering, then you would pass 0x00ffffff as the mask to
  676. * this function.
  677. */
  678. int
  679. swiotlb_dma_supported(struct device *hwdev, u64 mask)
  680. {
  681. return virt_to_bus(io_tlb_end - 1) <= mask;
  682. }
  683. EXPORT_SYMBOL(swiotlb_init);
  684. EXPORT_SYMBOL(swiotlb_map_single);
  685. EXPORT_SYMBOL(swiotlb_unmap_single);
  686. EXPORT_SYMBOL(swiotlb_map_sg);
  687. EXPORT_SYMBOL(swiotlb_unmap_sg);
  688. EXPORT_SYMBOL(swiotlb_sync_single_for_cpu);
  689. EXPORT_SYMBOL(swiotlb_sync_single_for_device);
  690. EXPORT_SYMBOL_GPL(swiotlb_sync_single_range_for_cpu);
  691. EXPORT_SYMBOL_GPL(swiotlb_sync_single_range_for_device);
  692. EXPORT_SYMBOL(swiotlb_sync_sg_for_cpu);
  693. EXPORT_SYMBOL(swiotlb_sync_sg_for_device);
  694. EXPORT_SYMBOL(swiotlb_dma_mapping_error);
  695. EXPORT_SYMBOL(swiotlb_alloc_coherent);
  696. EXPORT_SYMBOL(swiotlb_free_coherent);
  697. EXPORT_SYMBOL(swiotlb_dma_supported);