dmaengine.h 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366
  1. /*
  2. * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
  3. *
  4. * This program is free software; you can redistribute it and/or modify it
  5. * under the terms of the GNU General Public License as published by the Free
  6. * Software Foundation; either version 2 of the License, or (at your option)
  7. * any later version.
  8. *
  9. * This program is distributed in the hope that it will be useful, but WITHOUT
  10. * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11. * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  12. * more details.
  13. *
  14. * You should have received a copy of the GNU General Public License along with
  15. * this program; if not, write to the Free Software Foundation, Inc., 59
  16. * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  17. *
  18. * The full GNU General Public License is included in this distribution in the
  19. * file called COPYING.
  20. */
  21. #ifndef DMAENGINE_H
  22. #define DMAENGINE_H
  23. #ifdef CONFIG_DMA_ENGINE
  24. #include <linux/device.h>
  25. #include <linux/uio.h>
  26. #include <linux/kref.h>
  27. #include <linux/completion.h>
  28. #include <linux/rcupdate.h>
  29. /**
  30. * enum dma_event - resource PNP/power managment events
  31. * @DMA_RESOURCE_SUSPEND: DMA device going into low power state
  32. * @DMA_RESOURCE_RESUME: DMA device returning to full power
  33. * @DMA_RESOURCE_ADDED: DMA device added to the system
  34. * @DMA_RESOURCE_REMOVED: DMA device removed from the system
  35. */
  36. enum dma_event {
  37. DMA_RESOURCE_SUSPEND,
  38. DMA_RESOURCE_RESUME,
  39. DMA_RESOURCE_ADDED,
  40. DMA_RESOURCE_REMOVED,
  41. };
  42. /**
  43. * typedef dma_cookie_t - an opaque DMA cookie
  44. *
  45. * if dma_cookie_t is >0 it's a DMA request cookie, <0 it's an error code
  46. */
  47. typedef s32 dma_cookie_t;
  48. #define dma_submit_error(cookie) ((cookie) < 0 ? 1 : 0)
  49. /**
  50. * enum dma_status - DMA transaction status
  51. * @DMA_SUCCESS: transaction completed successfully
  52. * @DMA_IN_PROGRESS: transaction not yet processed
  53. * @DMA_ERROR: transaction failed
  54. */
  55. enum dma_status {
  56. DMA_SUCCESS,
  57. DMA_IN_PROGRESS,
  58. DMA_ERROR,
  59. };
  60. /**
  61. * struct dma_chan_percpu - the per-CPU part of struct dma_chan
  62. * @refcount: local_t used for open-coded "bigref" counting
  63. * @memcpy_count: transaction counter
  64. * @bytes_transferred: byte counter
  65. */
  66. struct dma_chan_percpu {
  67. local_t refcount;
  68. /* stats */
  69. unsigned long memcpy_count;
  70. unsigned long bytes_transferred;
  71. };
  72. /**
  73. * struct dma_chan - devices supply DMA channels, clients use them
  74. * @client: ptr to the client user of this chan, will be %NULL when unused
  75. * @device: ptr to the dma device who supplies this channel, always !%NULL
  76. * @cookie: last cookie value returned to client
  77. * @chan_id: channel ID for sysfs
  78. * @class_dev: class device for sysfs
  79. * @refcount: kref, used in "bigref" slow-mode
  80. * @slow_ref: indicates that the DMA channel is free
  81. * @rcu: the DMA channel's RCU head
  82. * @client_node: used to add this to the client chan list
  83. * @device_node: used to add this to the device chan list
  84. * @local: per-cpu pointer to a struct dma_chan_percpu
  85. */
  86. struct dma_chan {
  87. struct dma_client *client;
  88. struct dma_device *device;
  89. dma_cookie_t cookie;
  90. /* sysfs */
  91. int chan_id;
  92. struct class_device class_dev;
  93. struct kref refcount;
  94. int slow_ref;
  95. struct rcu_head rcu;
  96. struct list_head client_node;
  97. struct list_head device_node;
  98. struct dma_chan_percpu *local;
  99. };
  100. void dma_chan_cleanup(struct kref *kref);
  101. static inline void dma_chan_get(struct dma_chan *chan)
  102. {
  103. if (unlikely(chan->slow_ref))
  104. kref_get(&chan->refcount);
  105. else {
  106. local_inc(&(per_cpu_ptr(chan->local, get_cpu())->refcount));
  107. put_cpu();
  108. }
  109. }
  110. static inline void dma_chan_put(struct dma_chan *chan)
  111. {
  112. if (unlikely(chan->slow_ref))
  113. kref_put(&chan->refcount, dma_chan_cleanup);
  114. else {
  115. local_dec(&(per_cpu_ptr(chan->local, get_cpu())->refcount));
  116. put_cpu();
  117. }
  118. }
  119. /*
  120. * typedef dma_event_callback - function pointer to a DMA event callback
  121. */
  122. typedef void (*dma_event_callback) (struct dma_client *client,
  123. struct dma_chan *chan, enum dma_event event);
  124. /**
  125. * struct dma_client - info on the entity making use of DMA services
  126. * @event_callback: func ptr to call when something happens
  127. * @chan_count: number of chans allocated
  128. * @chans_desired: number of chans requested. Can be +/- chan_count
  129. * @lock: protects access to the channels list
  130. * @channels: the list of DMA channels allocated
  131. * @global_node: list_head for global dma_client_list
  132. */
  133. struct dma_client {
  134. dma_event_callback event_callback;
  135. unsigned int chan_count;
  136. unsigned int chans_desired;
  137. spinlock_t lock;
  138. struct list_head channels;
  139. struct list_head global_node;
  140. };
  141. /**
  142. * struct dma_device - info on the entity supplying DMA services
  143. * @chancnt: how many DMA channels are supported
  144. * @channels: the list of struct dma_chan
  145. * @global_node: list_head for global dma_device_list
  146. * @refcount: reference count
  147. * @done: IO completion struct
  148. * @dev_id: unique device ID
  149. * @device_alloc_chan_resources: allocate resources and return the
  150. * number of allocated descriptors
  151. * @device_free_chan_resources: release DMA channel's resources
  152. * @device_memcpy_buf_to_buf: memcpy buf pointer to buf pointer
  153. * @device_memcpy_buf_to_pg: memcpy buf pointer to struct page
  154. * @device_memcpy_pg_to_pg: memcpy struct page/offset to struct page/offset
  155. * @device_memcpy_complete: poll the status of an IOAT DMA transaction
  156. * @device_memcpy_issue_pending: push appended descriptors to hardware
  157. */
  158. struct dma_device {
  159. unsigned int chancnt;
  160. struct list_head channels;
  161. struct list_head global_node;
  162. struct kref refcount;
  163. struct completion done;
  164. int dev_id;
  165. int (*device_alloc_chan_resources)(struct dma_chan *chan);
  166. void (*device_free_chan_resources)(struct dma_chan *chan);
  167. dma_cookie_t (*device_memcpy_buf_to_buf)(struct dma_chan *chan,
  168. void *dest, void *src, size_t len);
  169. dma_cookie_t (*device_memcpy_buf_to_pg)(struct dma_chan *chan,
  170. struct page *page, unsigned int offset, void *kdata,
  171. size_t len);
  172. dma_cookie_t (*device_memcpy_pg_to_pg)(struct dma_chan *chan,
  173. struct page *dest_pg, unsigned int dest_off,
  174. struct page *src_pg, unsigned int src_off, size_t len);
  175. enum dma_status (*device_memcpy_complete)(struct dma_chan *chan,
  176. dma_cookie_t cookie, dma_cookie_t *last,
  177. dma_cookie_t *used);
  178. void (*device_memcpy_issue_pending)(struct dma_chan *chan);
  179. };
  180. /* --- public DMA engine API --- */
  181. struct dma_client *dma_async_client_register(dma_event_callback event_callback);
  182. void dma_async_client_unregister(struct dma_client *client);
  183. void dma_async_client_chan_request(struct dma_client *client,
  184. unsigned int number);
  185. /**
  186. * dma_async_memcpy_buf_to_buf - offloaded copy between virtual addresses
  187. * @chan: DMA channel to offload copy to
  188. * @dest: destination address (virtual)
  189. * @src: source address (virtual)
  190. * @len: length
  191. *
  192. * Both @dest and @src must be mappable to a bus address according to the
  193. * DMA mapping API rules for streaming mappings.
  194. * Both @dest and @src must stay memory resident (kernel memory or locked
  195. * user space pages).
  196. */
  197. static inline dma_cookie_t dma_async_memcpy_buf_to_buf(struct dma_chan *chan,
  198. void *dest, void *src, size_t len)
  199. {
  200. int cpu = get_cpu();
  201. per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;
  202. per_cpu_ptr(chan->local, cpu)->memcpy_count++;
  203. put_cpu();
  204. return chan->device->device_memcpy_buf_to_buf(chan, dest, src, len);
  205. }
  206. /**
  207. * dma_async_memcpy_buf_to_pg - offloaded copy from address to page
  208. * @chan: DMA channel to offload copy to
  209. * @page: destination page
  210. * @offset: offset in page to copy to
  211. * @kdata: source address (virtual)
  212. * @len: length
  213. *
  214. * Both @page/@offset and @kdata must be mappable to a bus address according
  215. * to the DMA mapping API rules for streaming mappings.
  216. * Both @page/@offset and @kdata must stay memory resident (kernel memory or
  217. * locked user space pages)
  218. */
  219. static inline dma_cookie_t dma_async_memcpy_buf_to_pg(struct dma_chan *chan,
  220. struct page *page, unsigned int offset, void *kdata, size_t len)
  221. {
  222. int cpu = get_cpu();
  223. per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;
  224. per_cpu_ptr(chan->local, cpu)->memcpy_count++;
  225. put_cpu();
  226. return chan->device->device_memcpy_buf_to_pg(chan, page, offset,
  227. kdata, len);
  228. }
  229. /**
  230. * dma_async_memcpy_pg_to_pg - offloaded copy from page to page
  231. * @chan: DMA channel to offload copy to
  232. * @dest_pg: destination page
  233. * @dest_off: offset in page to copy to
  234. * @src_pg: source page
  235. * @src_off: offset in page to copy from
  236. * @len: length
  237. *
  238. * Both @dest_page/@dest_off and @src_page/@src_off must be mappable to a bus
  239. * address according to the DMA mapping API rules for streaming mappings.
  240. * Both @dest_page/@dest_off and @src_page/@src_off must stay memory resident
  241. * (kernel memory or locked user space pages).
  242. */
  243. static inline dma_cookie_t dma_async_memcpy_pg_to_pg(struct dma_chan *chan,
  244. struct page *dest_pg, unsigned int dest_off, struct page *src_pg,
  245. unsigned int src_off, size_t len)
  246. {
  247. int cpu = get_cpu();
  248. per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;
  249. per_cpu_ptr(chan->local, cpu)->memcpy_count++;
  250. put_cpu();
  251. return chan->device->device_memcpy_pg_to_pg(chan, dest_pg, dest_off,
  252. src_pg, src_off, len);
  253. }
  254. /**
  255. * dma_async_memcpy_issue_pending - flush pending copies to HW
  256. * @chan: target DMA channel
  257. *
  258. * This allows drivers to push copies to HW in batches,
  259. * reducing MMIO writes where possible.
  260. */
  261. static inline void dma_async_memcpy_issue_pending(struct dma_chan *chan)
  262. {
  263. return chan->device->device_memcpy_issue_pending(chan);
  264. }
  265. /**
  266. * dma_async_memcpy_complete - poll for transaction completion
  267. * @chan: DMA channel
  268. * @cookie: transaction identifier to check status of
  269. * @last: returns last completed cookie, can be NULL
  270. * @used: returns last issued cookie, can be NULL
  271. *
  272. * If @last and @used are passed in, upon return they reflect the driver
  273. * internal state and can be used with dma_async_is_complete() to check
  274. * the status of multiple cookies without re-checking hardware state.
  275. */
  276. static inline enum dma_status dma_async_memcpy_complete(struct dma_chan *chan,
  277. dma_cookie_t cookie, dma_cookie_t *last, dma_cookie_t *used)
  278. {
  279. return chan->device->device_memcpy_complete(chan, cookie, last, used);
  280. }
  281. /**
  282. * dma_async_is_complete - test a cookie against chan state
  283. * @cookie: transaction identifier to test status of
  284. * @last_complete: last know completed transaction
  285. * @last_used: last cookie value handed out
  286. *
  287. * dma_async_is_complete() is used in dma_async_memcpy_complete()
  288. * the test logic is seperated for lightweight testing of multiple cookies
  289. */
  290. static inline enum dma_status dma_async_is_complete(dma_cookie_t cookie,
  291. dma_cookie_t last_complete, dma_cookie_t last_used)
  292. {
  293. if (last_complete <= last_used) {
  294. if ((cookie <= last_complete) || (cookie > last_used))
  295. return DMA_SUCCESS;
  296. } else {
  297. if ((cookie <= last_complete) && (cookie > last_used))
  298. return DMA_SUCCESS;
  299. }
  300. return DMA_IN_PROGRESS;
  301. }
  302. /* --- DMA device --- */
  303. int dma_async_device_register(struct dma_device *device);
  304. void dma_async_device_unregister(struct dma_device *device);
  305. /* --- Helper iov-locking functions --- */
  306. struct dma_page_list {
  307. char *base_address;
  308. int nr_pages;
  309. struct page **pages;
  310. };
  311. struct dma_pinned_list {
  312. int nr_iovecs;
  313. struct dma_page_list page_list[0];
  314. };
  315. struct dma_pinned_list *dma_pin_iovec_pages(struct iovec *iov, size_t len);
  316. void dma_unpin_iovec_pages(struct dma_pinned_list* pinned_list);
  317. dma_cookie_t dma_memcpy_to_iovec(struct dma_chan *chan, struct iovec *iov,
  318. struct dma_pinned_list *pinned_list, unsigned char *kdata, size_t len);
  319. dma_cookie_t dma_memcpy_pg_to_iovec(struct dma_chan *chan, struct iovec *iov,
  320. struct dma_pinned_list *pinned_list, struct page *page,
  321. unsigned int offset, size_t len);
  322. #endif /* CONFIG_DMA_ENGINE */
  323. #endif /* DMAENGINE_H */