dmaengine.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406
  1. /*
  2. * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
  3. *
  4. * This program is free software; you can redistribute it and/or modify it
  5. * under the terms of the GNU General Public License as published by the Free
  6. * Software Foundation; either version 2 of the License, or (at your option)
  7. * any later version.
  8. *
  9. * This program is distributed in the hope that it will be useful, but WITHOUT
  10. * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11. * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  12. * more details.
  13. *
  14. * You should have received a copy of the GNU General Public License along with
  15. * this program; if not, write to the Free Software Foundation, Inc., 59
  16. * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  17. *
  18. * The full GNU General Public License is included in this distribution in the
  19. * file called COPYING.
  20. */
  21. /*
  22. * This code implements the DMA subsystem. It provides a HW-neutral interface
  23. * for other kernel code to use asynchronous memory copy capabilities,
  24. * if present, and allows different HW DMA drivers to register as providing
  25. * this capability.
  26. *
  27. * Due to the fact we are accelerating what is already a relatively fast
  28. * operation, the code goes to great lengths to avoid additional overhead,
  29. * such as locking.
  30. *
  31. * LOCKING:
  32. *
  33. * The subsystem keeps two global lists, dma_device_list and dma_client_list.
  34. * Both of these are protected by a mutex, dma_list_mutex.
  35. *
  36. * Each device has a channels list, which runs unlocked but is never modified
  37. * once the device is registered, it's just setup by the driver.
  38. *
  39. * Each client has a channels list, it's only modified under the client->lock
  40. * and in an RCU callback, so it's safe to read under rcu_read_lock().
  41. *
  42. * Each device has a kref, which is initialized to 1 when the device is
  43. * registered. A kref_put is done for each class_device registered. When the
  44. * class_device is released, the coresponding kref_put is done in the release
  45. * method. Every time one of the device's channels is allocated to a client,
  46. * a kref_get occurs. When the channel is freed, the coresponding kref_put
  47. * happens. The device's release function does a completion, so
  48. * unregister_device does a remove event, class_device_unregister, a kref_put
  49. * for the first reference, then waits on the completion for all other
  50. * references to finish.
  51. *
  52. * Each channel has an open-coded implementation of Rusty Russell's "bigref,"
  53. * with a kref and a per_cpu local_t. A single reference is set when on an
  54. * ADDED event, and removed with a REMOVE event. Net DMA client takes an
  55. * extra reference per outstanding transaction. The relase function does a
  56. * kref_put on the device. -ChrisL
  57. */
  58. #include <linux/init.h>
  59. #include <linux/module.h>
  60. #include <linux/device.h>
  61. #include <linux/dmaengine.h>
  62. #include <linux/hardirq.h>
  63. #include <linux/spinlock.h>
  64. #include <linux/percpu.h>
  65. #include <linux/rcupdate.h>
  66. #include <linux/mutex.h>
  67. static DEFINE_MUTEX(dma_list_mutex);
  68. static LIST_HEAD(dma_device_list);
  69. static LIST_HEAD(dma_client_list);
  70. /* --- sysfs implementation --- */
  71. static ssize_t show_memcpy_count(struct class_device *cd, char *buf)
  72. {
  73. struct dma_chan *chan = container_of(cd, struct dma_chan, class_dev);
  74. unsigned long count = 0;
  75. int i;
  76. for_each_possible_cpu(i)
  77. count += per_cpu_ptr(chan->local, i)->memcpy_count;
  78. return sprintf(buf, "%lu\n", count);
  79. }
  80. static ssize_t show_bytes_transferred(struct class_device *cd, char *buf)
  81. {
  82. struct dma_chan *chan = container_of(cd, struct dma_chan, class_dev);
  83. unsigned long count = 0;
  84. int i;
  85. for_each_possible_cpu(i)
  86. count += per_cpu_ptr(chan->local, i)->bytes_transferred;
  87. return sprintf(buf, "%lu\n", count);
  88. }
  89. static ssize_t show_in_use(struct class_device *cd, char *buf)
  90. {
  91. struct dma_chan *chan = container_of(cd, struct dma_chan, class_dev);
  92. return sprintf(buf, "%d\n", (chan->client ? 1 : 0));
  93. }
  94. static struct class_device_attribute dma_class_attrs[] = {
  95. __ATTR(memcpy_count, S_IRUGO, show_memcpy_count, NULL),
  96. __ATTR(bytes_transferred, S_IRUGO, show_bytes_transferred, NULL),
  97. __ATTR(in_use, S_IRUGO, show_in_use, NULL),
  98. __ATTR_NULL
  99. };
  100. static void dma_async_device_cleanup(struct kref *kref);
  101. static void dma_class_dev_release(struct class_device *cd)
  102. {
  103. struct dma_chan *chan = container_of(cd, struct dma_chan, class_dev);
  104. kref_put(&chan->device->refcount, dma_async_device_cleanup);
  105. }
  106. static struct class dma_devclass = {
  107. .name = "dma",
  108. .class_dev_attrs = dma_class_attrs,
  109. .release = dma_class_dev_release,
  110. };
  111. /* --- client and device registration --- */
  112. /**
  113. * dma_client_chan_alloc - try to allocate a channel to a client
  114. * @client: &dma_client
  115. *
  116. * Called with dma_list_mutex held.
  117. */
  118. static struct dma_chan *dma_client_chan_alloc(struct dma_client *client)
  119. {
  120. struct dma_device *device;
  121. struct dma_chan *chan;
  122. unsigned long flags;
  123. int desc; /* allocated descriptor count */
  124. /* Find a channel, any DMA engine will do */
  125. list_for_each_entry(device, &dma_device_list, global_node) {
  126. list_for_each_entry(chan, &device->channels, device_node) {
  127. if (chan->client)
  128. continue;
  129. desc = chan->device->device_alloc_chan_resources(chan);
  130. if (desc >= 0) {
  131. kref_get(&device->refcount);
  132. kref_init(&chan->refcount);
  133. chan->slow_ref = 0;
  134. INIT_RCU_HEAD(&chan->rcu);
  135. chan->client = client;
  136. spin_lock_irqsave(&client->lock, flags);
  137. list_add_tail_rcu(&chan->client_node,
  138. &client->channels);
  139. spin_unlock_irqrestore(&client->lock, flags);
  140. return chan;
  141. }
  142. }
  143. }
  144. return NULL;
  145. }
  146. /**
  147. * dma_chan_cleanup - release a DMA channel's resources
  148. * @kref: kernel reference structure that contains the DMA channel device
  149. */
  150. void dma_chan_cleanup(struct kref *kref)
  151. {
  152. struct dma_chan *chan = container_of(kref, struct dma_chan, refcount);
  153. chan->device->device_free_chan_resources(chan);
  154. chan->client = NULL;
  155. kref_put(&chan->device->refcount, dma_async_device_cleanup);
  156. }
  157. EXPORT_SYMBOL(dma_chan_cleanup);
  158. static void dma_chan_free_rcu(struct rcu_head *rcu)
  159. {
  160. struct dma_chan *chan = container_of(rcu, struct dma_chan, rcu);
  161. int bias = 0x7FFFFFFF;
  162. int i;
  163. for_each_possible_cpu(i)
  164. bias -= local_read(&per_cpu_ptr(chan->local, i)->refcount);
  165. atomic_sub(bias, &chan->refcount.refcount);
  166. kref_put(&chan->refcount, dma_chan_cleanup);
  167. }
  168. static void dma_client_chan_free(struct dma_chan *chan)
  169. {
  170. atomic_add(0x7FFFFFFF, &chan->refcount.refcount);
  171. chan->slow_ref = 1;
  172. call_rcu(&chan->rcu, dma_chan_free_rcu);
  173. }
  174. /**
  175. * dma_chans_rebalance - reallocate channels to clients
  176. *
  177. * When the number of DMA channel in the system changes,
  178. * channels need to be rebalanced among clients.
  179. */
  180. static void dma_chans_rebalance(void)
  181. {
  182. struct dma_client *client;
  183. struct dma_chan *chan;
  184. unsigned long flags;
  185. mutex_lock(&dma_list_mutex);
  186. list_for_each_entry(client, &dma_client_list, global_node) {
  187. while (client->chans_desired > client->chan_count) {
  188. chan = dma_client_chan_alloc(client);
  189. if (!chan)
  190. break;
  191. client->chan_count++;
  192. client->event_callback(client,
  193. chan,
  194. DMA_RESOURCE_ADDED);
  195. }
  196. while (client->chans_desired < client->chan_count) {
  197. spin_lock_irqsave(&client->lock, flags);
  198. chan = list_entry(client->channels.next,
  199. struct dma_chan,
  200. client_node);
  201. list_del_rcu(&chan->client_node);
  202. spin_unlock_irqrestore(&client->lock, flags);
  203. client->chan_count--;
  204. client->event_callback(client,
  205. chan,
  206. DMA_RESOURCE_REMOVED);
  207. dma_client_chan_free(chan);
  208. }
  209. }
  210. mutex_unlock(&dma_list_mutex);
  211. }
  212. /**
  213. * dma_async_client_register - allocate and register a &dma_client
  214. * @event_callback: callback for notification of channel addition/removal
  215. */
  216. struct dma_client *dma_async_client_register(dma_event_callback event_callback)
  217. {
  218. struct dma_client *client;
  219. client = kzalloc(sizeof(*client), GFP_KERNEL);
  220. if (!client)
  221. return NULL;
  222. INIT_LIST_HEAD(&client->channels);
  223. spin_lock_init(&client->lock);
  224. client->chans_desired = 0;
  225. client->chan_count = 0;
  226. client->event_callback = event_callback;
  227. mutex_lock(&dma_list_mutex);
  228. list_add_tail(&client->global_node, &dma_client_list);
  229. mutex_unlock(&dma_list_mutex);
  230. return client;
  231. }
  232. EXPORT_SYMBOL(dma_async_client_register);
  233. /**
  234. * dma_async_client_unregister - unregister a client and free the &dma_client
  235. * @client: &dma_client to free
  236. *
  237. * Force frees any allocated DMA channels, frees the &dma_client memory
  238. */
  239. void dma_async_client_unregister(struct dma_client *client)
  240. {
  241. struct dma_chan *chan;
  242. if (!client)
  243. return;
  244. rcu_read_lock();
  245. list_for_each_entry_rcu(chan, &client->channels, client_node)
  246. dma_client_chan_free(chan);
  247. rcu_read_unlock();
  248. mutex_lock(&dma_list_mutex);
  249. list_del(&client->global_node);
  250. mutex_unlock(&dma_list_mutex);
  251. kfree(client);
  252. dma_chans_rebalance();
  253. }
  254. EXPORT_SYMBOL(dma_async_client_unregister);
  255. /**
  256. * dma_async_client_chan_request - request DMA channels
  257. * @client: &dma_client
  258. * @number: count of DMA channels requested
  259. *
  260. * Clients call dma_async_client_chan_request() to specify how many
  261. * DMA channels they need, 0 to free all currently allocated.
  262. * The resulting allocations/frees are indicated to the client via the
  263. * event callback.
  264. */
  265. void dma_async_client_chan_request(struct dma_client *client,
  266. unsigned int number)
  267. {
  268. client->chans_desired = number;
  269. dma_chans_rebalance();
  270. }
  271. EXPORT_SYMBOL(dma_async_client_chan_request);
  272. /**
  273. * dma_async_device_register - registers DMA devices found
  274. * @device: &dma_device
  275. */
  276. int dma_async_device_register(struct dma_device *device)
  277. {
  278. static int id;
  279. int chancnt = 0;
  280. struct dma_chan* chan;
  281. if (!device)
  282. return -ENODEV;
  283. init_completion(&device->done);
  284. kref_init(&device->refcount);
  285. device->dev_id = id++;
  286. /* represent channels in sysfs. Probably want devs too */
  287. list_for_each_entry(chan, &device->channels, device_node) {
  288. chan->local = alloc_percpu(typeof(*chan->local));
  289. if (chan->local == NULL)
  290. continue;
  291. chan->chan_id = chancnt++;
  292. chan->class_dev.class = &dma_devclass;
  293. chan->class_dev.dev = NULL;
  294. snprintf(chan->class_dev.class_id, BUS_ID_SIZE, "dma%dchan%d",
  295. device->dev_id, chan->chan_id);
  296. kref_get(&device->refcount);
  297. class_device_register(&chan->class_dev);
  298. }
  299. mutex_lock(&dma_list_mutex);
  300. list_add_tail(&device->global_node, &dma_device_list);
  301. mutex_unlock(&dma_list_mutex);
  302. dma_chans_rebalance();
  303. return 0;
  304. }
  305. EXPORT_SYMBOL(dma_async_device_register);
  306. /**
  307. * dma_async_device_cleanup - function called when all references are released
  308. * @kref: kernel reference object
  309. */
  310. static void dma_async_device_cleanup(struct kref *kref)
  311. {
  312. struct dma_device *device;
  313. device = container_of(kref, struct dma_device, refcount);
  314. complete(&device->done);
  315. }
  316. /**
  317. * dma_async_device_unregister - unregisters DMA devices
  318. * @device: &dma_device
  319. */
  320. void dma_async_device_unregister(struct dma_device *device)
  321. {
  322. struct dma_chan *chan;
  323. unsigned long flags;
  324. mutex_lock(&dma_list_mutex);
  325. list_del(&device->global_node);
  326. mutex_unlock(&dma_list_mutex);
  327. list_for_each_entry(chan, &device->channels, device_node) {
  328. if (chan->client) {
  329. spin_lock_irqsave(&chan->client->lock, flags);
  330. list_del(&chan->client_node);
  331. chan->client->chan_count--;
  332. spin_unlock_irqrestore(&chan->client->lock, flags);
  333. chan->client->event_callback(chan->client,
  334. chan,
  335. DMA_RESOURCE_REMOVED);
  336. dma_client_chan_free(chan);
  337. }
  338. class_device_unregister(&chan->class_dev);
  339. }
  340. dma_chans_rebalance();
  341. kref_put(&device->refcount, dma_async_device_cleanup);
  342. wait_for_completion(&device->done);
  343. }
  344. EXPORT_SYMBOL(dma_async_device_unregister);
  345. static int __init dma_bus_init(void)
  346. {
  347. mutex_init(&dma_list_mutex);
  348. return class_register(&dma_devclass);
  349. }
  350. subsys_initcall(dma_bus_init);