mpc512x_dma.c 30 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * Copyright (C) Freescale Semicondutor, Inc. 2007, 2008.
  4. * Copyright (C) Semihalf 2009
  5. * Copyright (C) Ilya Yanok, Emcraft Systems 2010
  6. * Copyright (C) Alexander Popov, Promcontroller 2014
  7. * Copyright (C) Mario Six, Guntermann & Drunck GmbH, 2016
  8. *
  9. * Written by Piotr Ziecik <kosmo@semihalf.com>. Hardware description
  10. * (defines, structures and comments) was taken from MPC5121 DMA driver
  11. * written by Hongjun Chen <hong-jun.chen@freescale.com>.
  12. *
  13. * Approved as OSADL project by a majority of OSADL members and funded
  14. * by OSADL membership fees in 2009; for details see www.osadl.org.
  15. */
  16. /*
  17. * MPC512x and MPC8308 DMA driver. It supports memory to memory data transfers
  18. * (tested using dmatest module) and data transfers between memory and
  19. * peripheral I/O memory by means of slave scatter/gather with these
  20. * limitations:
  21. * - chunked transfers (described by s/g lists with more than one item) are
  22. * refused as long as proper support for scatter/gather is missing
  23. * - transfers on MPC8308 always start from software as this SoC does not have
  24. * external request lines for peripheral flow control
  25. * - memory <-> I/O memory transfer chunks of sizes of 1, 2, 4, 16 (for
  26. * MPC512x), and 32 bytes are supported, and, consequently, source
  27. * addresses and destination addresses must be aligned accordingly;
  28. * furthermore, for MPC512x SoCs, the transfer size must be aligned on
  29. * (chunk size * maxburst)
  30. */
  31. #include <linux/module.h>
  32. #include <linux/dmaengine.h>
  33. #include <linux/dma-mapping.h>
  34. #include <linux/interrupt.h>
  35. #include <linux/io.h>
  36. #include <linux/slab.h>
  37. #include <linux/of_address.h>
  38. #include <linux/of_device.h>
  39. #include <linux/of_irq.h>
  40. #include <linux/of_dma.h>
  41. #include <linux/of_platform.h>
  42. #include <linux/random.h>
  43. #include "dmaengine.h"
  44. /* Number of DMA Transfer descriptors allocated per channel */
  45. #define MPC_DMA_DESCRIPTORS 64
  46. /* Macro definitions */
  47. #define MPC_DMA_TCD_OFFSET 0x1000
  48. /*
  49. * Maximum channel counts for individual hardware variants
  50. * and the maximum channel count over all supported controllers,
  51. * used for data structure size
  52. */
  53. #define MPC8308_DMACHAN_MAX 16
  54. #define MPC512x_DMACHAN_MAX 64
  55. #define MPC_DMA_CHANNELS 64
  56. /* Arbitration mode of group and channel */
  57. #define MPC_DMA_DMACR_EDCG (1 << 31)
  58. #define MPC_DMA_DMACR_ERGA (1 << 3)
  59. #define MPC_DMA_DMACR_ERCA (1 << 2)
  60. /* Error codes */
  61. #define MPC_DMA_DMAES_VLD (1 << 31)
  62. #define MPC_DMA_DMAES_GPE (1 << 15)
  63. #define MPC_DMA_DMAES_CPE (1 << 14)
  64. #define MPC_DMA_DMAES_ERRCHN(err) \
  65. (((err) >> 8) & 0x3f)
  66. #define MPC_DMA_DMAES_SAE (1 << 7)
  67. #define MPC_DMA_DMAES_SOE (1 << 6)
  68. #define MPC_DMA_DMAES_DAE (1 << 5)
  69. #define MPC_DMA_DMAES_DOE (1 << 4)
  70. #define MPC_DMA_DMAES_NCE (1 << 3)
  71. #define MPC_DMA_DMAES_SGE (1 << 2)
  72. #define MPC_DMA_DMAES_SBE (1 << 1)
  73. #define MPC_DMA_DMAES_DBE (1 << 0)
  74. #define MPC_DMA_DMAGPOR_SNOOP_ENABLE (1 << 6)
  75. #define MPC_DMA_TSIZE_1 0x00
  76. #define MPC_DMA_TSIZE_2 0x01
  77. #define MPC_DMA_TSIZE_4 0x02
  78. #define MPC_DMA_TSIZE_16 0x04
  79. #define MPC_DMA_TSIZE_32 0x05
  80. /* MPC5121 DMA engine registers */
  81. struct __attribute__ ((__packed__)) mpc_dma_regs {
  82. /* 0x00 */
  83. u32 dmacr; /* DMA control register */
  84. u32 dmaes; /* DMA error status */
  85. /* 0x08 */
  86. u32 dmaerqh; /* DMA enable request high(channels 63~32) */
  87. u32 dmaerql; /* DMA enable request low(channels 31~0) */
  88. u32 dmaeeih; /* DMA enable error interrupt high(ch63~32) */
  89. u32 dmaeeil; /* DMA enable error interrupt low(ch31~0) */
  90. /* 0x18 */
  91. u8 dmaserq; /* DMA set enable request */
  92. u8 dmacerq; /* DMA clear enable request */
  93. u8 dmaseei; /* DMA set enable error interrupt */
  94. u8 dmaceei; /* DMA clear enable error interrupt */
  95. /* 0x1c */
  96. u8 dmacint; /* DMA clear interrupt request */
  97. u8 dmacerr; /* DMA clear error */
  98. u8 dmassrt; /* DMA set start bit */
  99. u8 dmacdne; /* DMA clear DONE status bit */
  100. /* 0x20 */
  101. u32 dmainth; /* DMA interrupt request high(ch63~32) */
  102. u32 dmaintl; /* DMA interrupt request low(ch31~0) */
  103. u32 dmaerrh; /* DMA error high(ch63~32) */
  104. u32 dmaerrl; /* DMA error low(ch31~0) */
  105. /* 0x30 */
  106. u32 dmahrsh; /* DMA hw request status high(ch63~32) */
  107. u32 dmahrsl; /* DMA hardware request status low(ch31~0) */
  108. union {
  109. u32 dmaihsa; /* DMA interrupt high select AXE(ch63~32) */
  110. u32 dmagpor; /* (General purpose register on MPC8308) */
  111. };
  112. u32 dmailsa; /* DMA interrupt low select AXE(ch31~0) */
  113. /* 0x40 ~ 0xff */
  114. u32 reserve0[48]; /* Reserved */
  115. /* 0x100 */
  116. u8 dchpri[MPC_DMA_CHANNELS];
  117. /* DMA channels(0~63) priority */
  118. };
  119. struct __attribute__ ((__packed__)) mpc_dma_tcd {
  120. /* 0x00 */
  121. u32 saddr; /* Source address */
  122. u32 smod:5; /* Source address modulo */
  123. u32 ssize:3; /* Source data transfer size */
  124. u32 dmod:5; /* Destination address modulo */
  125. u32 dsize:3; /* Destination data transfer size */
  126. u32 soff:16; /* Signed source address offset */
  127. /* 0x08 */
  128. u32 nbytes; /* Inner "minor" byte count */
  129. u32 slast; /* Last source address adjustment */
  130. u32 daddr; /* Destination address */
  131. /* 0x14 */
  132. u32 citer_elink:1; /* Enable channel-to-channel linking on
  133. * minor loop complete
  134. */
  135. u32 citer_linkch:6; /* Link channel for minor loop complete */
  136. u32 citer:9; /* Current "major" iteration count */
  137. u32 doff:16; /* Signed destination address offset */
  138. /* 0x18 */
  139. u32 dlast_sga; /* Last Destination address adjustment/scatter
  140. * gather address
  141. */
  142. /* 0x1c */
  143. u32 biter_elink:1; /* Enable channel-to-channel linking on major
  144. * loop complete
  145. */
  146. u32 biter_linkch:6;
  147. u32 biter:9; /* Beginning "major" iteration count */
  148. u32 bwc:2; /* Bandwidth control */
  149. u32 major_linkch:6; /* Link channel number */
  150. u32 done:1; /* Channel done */
  151. u32 active:1; /* Channel active */
  152. u32 major_elink:1; /* Enable channel-to-channel linking on major
  153. * loop complete
  154. */
  155. u32 e_sg:1; /* Enable scatter/gather processing */
  156. u32 d_req:1; /* Disable request */
  157. u32 int_half:1; /* Enable an interrupt when major counter is
  158. * half complete
  159. */
  160. u32 int_maj:1; /* Enable an interrupt when major iteration
  161. * count completes
  162. */
  163. u32 start:1; /* Channel start */
  164. };
  165. struct mpc_dma_desc {
  166. struct dma_async_tx_descriptor desc;
  167. struct mpc_dma_tcd *tcd;
  168. dma_addr_t tcd_paddr;
  169. int error;
  170. struct list_head node;
  171. int will_access_peripheral;
  172. };
  173. struct mpc_dma_chan {
  174. struct dma_chan chan;
  175. struct list_head free;
  176. struct list_head prepared;
  177. struct list_head queued;
  178. struct list_head active;
  179. struct list_head completed;
  180. struct mpc_dma_tcd *tcd;
  181. dma_addr_t tcd_paddr;
  182. /* Settings for access to peripheral FIFO */
  183. dma_addr_t src_per_paddr;
  184. u32 src_tcd_nunits;
  185. u8 swidth;
  186. dma_addr_t dst_per_paddr;
  187. u32 dst_tcd_nunits;
  188. u8 dwidth;
  189. /* Lock for this structure */
  190. spinlock_t lock;
  191. };
  192. struct mpc_dma {
  193. struct dma_device dma;
  194. struct tasklet_struct tasklet;
  195. struct mpc_dma_chan channels[MPC_DMA_CHANNELS];
  196. struct mpc_dma_regs __iomem *regs;
  197. struct mpc_dma_tcd __iomem *tcd;
  198. int irq;
  199. int irq2;
  200. uint error_status;
  201. int is_mpc8308;
  202. /* Lock for error_status field in this structure */
  203. spinlock_t error_status_lock;
  204. };
  205. #define DRV_NAME "mpc512x_dma"
  206. /* Convert struct dma_chan to struct mpc_dma_chan */
  207. static inline struct mpc_dma_chan *dma_chan_to_mpc_dma_chan(struct dma_chan *c)
  208. {
  209. return container_of(c, struct mpc_dma_chan, chan);
  210. }
  211. /* Convert struct dma_chan to struct mpc_dma */
  212. static inline struct mpc_dma *dma_chan_to_mpc_dma(struct dma_chan *c)
  213. {
  214. struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(c);
  215. return container_of(mchan, struct mpc_dma, channels[c->chan_id]);
  216. }
  217. /*
  218. * Execute all queued DMA descriptors.
  219. *
  220. * Following requirements must be met while calling mpc_dma_execute():
  221. * a) mchan->lock is acquired,
  222. * b) mchan->active list is empty,
  223. * c) mchan->queued list contains at least one entry.
  224. */
  225. static void mpc_dma_execute(struct mpc_dma_chan *mchan)
  226. {
  227. struct mpc_dma *mdma = dma_chan_to_mpc_dma(&mchan->chan);
  228. struct mpc_dma_desc *first = NULL;
  229. struct mpc_dma_desc *prev = NULL;
  230. struct mpc_dma_desc *mdesc;
  231. int cid = mchan->chan.chan_id;
  232. while (!list_empty(&mchan->queued)) {
  233. mdesc = list_first_entry(&mchan->queued,
  234. struct mpc_dma_desc, node);
  235. /*
  236. * Grab either several mem-to-mem transfer descriptors
  237. * or one peripheral transfer descriptor,
  238. * don't mix mem-to-mem and peripheral transfer descriptors
  239. * within the same 'active' list.
  240. */
  241. if (mdesc->will_access_peripheral) {
  242. if (list_empty(&mchan->active))
  243. list_move_tail(&mdesc->node, &mchan->active);
  244. break;
  245. } else {
  246. list_move_tail(&mdesc->node, &mchan->active);
  247. }
  248. }
  249. /* Chain descriptors into one transaction */
  250. list_for_each_entry(mdesc, &mchan->active, node) {
  251. if (!first)
  252. first = mdesc;
  253. if (!prev) {
  254. prev = mdesc;
  255. continue;
  256. }
  257. prev->tcd->dlast_sga = mdesc->tcd_paddr;
  258. prev->tcd->e_sg = 1;
  259. mdesc->tcd->start = 1;
  260. prev = mdesc;
  261. }
  262. prev->tcd->int_maj = 1;
  263. /* Send first descriptor in chain into hardware */
  264. memcpy_toio(&mdma->tcd[cid], first->tcd, sizeof(struct mpc_dma_tcd));
  265. if (first != prev)
  266. mdma->tcd[cid].e_sg = 1;
  267. if (mdma->is_mpc8308) {
  268. /* MPC8308, no request lines, software initiated start */
  269. out_8(&mdma->regs->dmassrt, cid);
  270. } else if (first->will_access_peripheral) {
  271. /* Peripherals involved, start by external request signal */
  272. out_8(&mdma->regs->dmaserq, cid);
  273. } else {
  274. /* Memory to memory transfer, software initiated start */
  275. out_8(&mdma->regs->dmassrt, cid);
  276. }
  277. }
  278. /* Handle interrupt on one half of DMA controller (32 channels) */
  279. static void mpc_dma_irq_process(struct mpc_dma *mdma, u32 is, u32 es, int off)
  280. {
  281. struct mpc_dma_chan *mchan;
  282. struct mpc_dma_desc *mdesc;
  283. u32 status = is | es;
  284. int ch;
  285. while ((ch = fls(status) - 1) >= 0) {
  286. status &= ~(1 << ch);
  287. mchan = &mdma->channels[ch + off];
  288. spin_lock(&mchan->lock);
  289. out_8(&mdma->regs->dmacint, ch + off);
  290. out_8(&mdma->regs->dmacerr, ch + off);
  291. /* Check error status */
  292. if (es & (1 << ch))
  293. list_for_each_entry(mdesc, &mchan->active, node)
  294. mdesc->error = -EIO;
  295. /* Execute queued descriptors */
  296. list_splice_tail_init(&mchan->active, &mchan->completed);
  297. if (!list_empty(&mchan->queued))
  298. mpc_dma_execute(mchan);
  299. spin_unlock(&mchan->lock);
  300. }
  301. }
  302. /* Interrupt handler */
  303. static irqreturn_t mpc_dma_irq(int irq, void *data)
  304. {
  305. struct mpc_dma *mdma = data;
  306. uint es;
  307. /* Save error status register */
  308. es = in_be32(&mdma->regs->dmaes);
  309. spin_lock(&mdma->error_status_lock);
  310. if ((es & MPC_DMA_DMAES_VLD) && mdma->error_status == 0)
  311. mdma->error_status = es;
  312. spin_unlock(&mdma->error_status_lock);
  313. /* Handle interrupt on each channel */
  314. if (mdma->dma.chancnt > 32) {
  315. mpc_dma_irq_process(mdma, in_be32(&mdma->regs->dmainth),
  316. in_be32(&mdma->regs->dmaerrh), 32);
  317. }
  318. mpc_dma_irq_process(mdma, in_be32(&mdma->regs->dmaintl),
  319. in_be32(&mdma->regs->dmaerrl), 0);
  320. /* Schedule tasklet */
  321. tasklet_schedule(&mdma->tasklet);
  322. return IRQ_HANDLED;
  323. }
  324. /* process completed descriptors */
  325. static void mpc_dma_process_completed(struct mpc_dma *mdma)
  326. {
  327. dma_cookie_t last_cookie = 0;
  328. struct mpc_dma_chan *mchan;
  329. struct mpc_dma_desc *mdesc;
  330. struct dma_async_tx_descriptor *desc;
  331. unsigned long flags;
  332. LIST_HEAD(list);
  333. int i;
  334. for (i = 0; i < mdma->dma.chancnt; i++) {
  335. mchan = &mdma->channels[i];
  336. /* Get all completed descriptors */
  337. spin_lock_irqsave(&mchan->lock, flags);
  338. if (!list_empty(&mchan->completed))
  339. list_splice_tail_init(&mchan->completed, &list);
  340. spin_unlock_irqrestore(&mchan->lock, flags);
  341. if (list_empty(&list))
  342. continue;
  343. /* Execute callbacks and run dependencies */
  344. list_for_each_entry(mdesc, &list, node) {
  345. desc = &mdesc->desc;
  346. dmaengine_desc_get_callback_invoke(desc, NULL);
  347. last_cookie = desc->cookie;
  348. dma_run_dependencies(desc);
  349. }
  350. /* Free descriptors */
  351. spin_lock_irqsave(&mchan->lock, flags);
  352. list_splice_tail_init(&list, &mchan->free);
  353. mchan->chan.completed_cookie = last_cookie;
  354. spin_unlock_irqrestore(&mchan->lock, flags);
  355. }
  356. }
  357. /* DMA Tasklet */
  358. static void mpc_dma_tasklet(struct tasklet_struct *t)
  359. {
  360. struct mpc_dma *mdma = from_tasklet(mdma, t, tasklet);
  361. unsigned long flags;
  362. uint es;
  363. spin_lock_irqsave(&mdma->error_status_lock, flags);
  364. es = mdma->error_status;
  365. mdma->error_status = 0;
  366. spin_unlock_irqrestore(&mdma->error_status_lock, flags);
  367. /* Print nice error report */
  368. if (es) {
  369. dev_err(mdma->dma.dev,
  370. "Hardware reported following error(s) on channel %u:\n",
  371. MPC_DMA_DMAES_ERRCHN(es));
  372. if (es & MPC_DMA_DMAES_GPE)
  373. dev_err(mdma->dma.dev, "- Group Priority Error\n");
  374. if (es & MPC_DMA_DMAES_CPE)
  375. dev_err(mdma->dma.dev, "- Channel Priority Error\n");
  376. if (es & MPC_DMA_DMAES_SAE)
  377. dev_err(mdma->dma.dev, "- Source Address Error\n");
  378. if (es & MPC_DMA_DMAES_SOE)
  379. dev_err(mdma->dma.dev, "- Source Offset Configuration Error\n");
  380. if (es & MPC_DMA_DMAES_DAE)
  381. dev_err(mdma->dma.dev, "- Destination Address Error\n");
  382. if (es & MPC_DMA_DMAES_DOE)
  383. dev_err(mdma->dma.dev, "- Destination Offset Configuration Error\n");
  384. if (es & MPC_DMA_DMAES_NCE)
  385. dev_err(mdma->dma.dev, "- NBytes/Citter Configuration Error\n");
  386. if (es & MPC_DMA_DMAES_SGE)
  387. dev_err(mdma->dma.dev, "- Scatter/Gather Configuration Error\n");
  388. if (es & MPC_DMA_DMAES_SBE)
  389. dev_err(mdma->dma.dev, "- Source Bus Error\n");
  390. if (es & MPC_DMA_DMAES_DBE)
  391. dev_err(mdma->dma.dev, "- Destination Bus Error\n");
  392. }
  393. mpc_dma_process_completed(mdma);
  394. }
  395. /* Submit descriptor to hardware */
  396. static dma_cookie_t mpc_dma_tx_submit(struct dma_async_tx_descriptor *txd)
  397. {
  398. struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(txd->chan);
  399. struct mpc_dma_desc *mdesc;
  400. unsigned long flags;
  401. dma_cookie_t cookie;
  402. mdesc = container_of(txd, struct mpc_dma_desc, desc);
  403. spin_lock_irqsave(&mchan->lock, flags);
  404. /* Move descriptor to queue */
  405. list_move_tail(&mdesc->node, &mchan->queued);
  406. /* If channel is idle, execute all queued descriptors */
  407. if (list_empty(&mchan->active))
  408. mpc_dma_execute(mchan);
  409. /* Update cookie */
  410. cookie = dma_cookie_assign(txd);
  411. spin_unlock_irqrestore(&mchan->lock, flags);
  412. return cookie;
  413. }
  414. /* Alloc channel resources */
  415. static int mpc_dma_alloc_chan_resources(struct dma_chan *chan)
  416. {
  417. struct mpc_dma *mdma = dma_chan_to_mpc_dma(chan);
  418. struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);
  419. struct mpc_dma_desc *mdesc;
  420. struct mpc_dma_tcd *tcd;
  421. dma_addr_t tcd_paddr;
  422. unsigned long flags;
  423. LIST_HEAD(descs);
  424. int i;
  425. /* Alloc DMA memory for Transfer Control Descriptors */
  426. tcd = dma_alloc_coherent(mdma->dma.dev,
  427. MPC_DMA_DESCRIPTORS * sizeof(struct mpc_dma_tcd),
  428. &tcd_paddr, GFP_KERNEL);
  429. if (!tcd)
  430. return -ENOMEM;
  431. /* Alloc descriptors for this channel */
  432. for (i = 0; i < MPC_DMA_DESCRIPTORS; i++) {
  433. mdesc = kzalloc(sizeof(struct mpc_dma_desc), GFP_KERNEL);
  434. if (!mdesc) {
  435. dev_notice(mdma->dma.dev,
  436. "Memory allocation error. Allocated only %u descriptors\n", i);
  437. break;
  438. }
  439. dma_async_tx_descriptor_init(&mdesc->desc, chan);
  440. mdesc->desc.flags = DMA_CTRL_ACK;
  441. mdesc->desc.tx_submit = mpc_dma_tx_submit;
  442. mdesc->tcd = &tcd[i];
  443. mdesc->tcd_paddr = tcd_paddr + (i * sizeof(struct mpc_dma_tcd));
  444. list_add_tail(&mdesc->node, &descs);
  445. }
  446. /* Return error only if no descriptors were allocated */
  447. if (i == 0) {
  448. dma_free_coherent(mdma->dma.dev,
  449. MPC_DMA_DESCRIPTORS * sizeof(struct mpc_dma_tcd),
  450. tcd, tcd_paddr);
  451. return -ENOMEM;
  452. }
  453. spin_lock_irqsave(&mchan->lock, flags);
  454. mchan->tcd = tcd;
  455. mchan->tcd_paddr = tcd_paddr;
  456. list_splice_tail_init(&descs, &mchan->free);
  457. spin_unlock_irqrestore(&mchan->lock, flags);
  458. /* Enable Error Interrupt */
  459. out_8(&mdma->regs->dmaseei, chan->chan_id);
  460. return 0;
  461. }
  462. /* Free channel resources */
  463. static void mpc_dma_free_chan_resources(struct dma_chan *chan)
  464. {
  465. struct mpc_dma *mdma = dma_chan_to_mpc_dma(chan);
  466. struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);
  467. struct mpc_dma_desc *mdesc, *tmp;
  468. struct mpc_dma_tcd *tcd;
  469. dma_addr_t tcd_paddr;
  470. unsigned long flags;
  471. LIST_HEAD(descs);
  472. spin_lock_irqsave(&mchan->lock, flags);
  473. /* Channel must be idle */
  474. BUG_ON(!list_empty(&mchan->prepared));
  475. BUG_ON(!list_empty(&mchan->queued));
  476. BUG_ON(!list_empty(&mchan->active));
  477. BUG_ON(!list_empty(&mchan->completed));
  478. /* Move data */
  479. list_splice_tail_init(&mchan->free, &descs);
  480. tcd = mchan->tcd;
  481. tcd_paddr = mchan->tcd_paddr;
  482. spin_unlock_irqrestore(&mchan->lock, flags);
  483. /* Free DMA memory used by descriptors */
  484. dma_free_coherent(mdma->dma.dev,
  485. MPC_DMA_DESCRIPTORS * sizeof(struct mpc_dma_tcd),
  486. tcd, tcd_paddr);
  487. /* Free descriptors */
  488. list_for_each_entry_safe(mdesc, tmp, &descs, node)
  489. kfree(mdesc);
  490. /* Disable Error Interrupt */
  491. out_8(&mdma->regs->dmaceei, chan->chan_id);
  492. }
  493. /* Send all pending descriptor to hardware */
  494. static void mpc_dma_issue_pending(struct dma_chan *chan)
  495. {
  496. /*
  497. * We are posting descriptors to the hardware as soon as
  498. * they are ready, so this function does nothing.
  499. */
  500. }
  501. /* Check request completion status */
  502. static enum dma_status
  503. mpc_dma_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
  504. struct dma_tx_state *txstate)
  505. {
  506. return dma_cookie_status(chan, cookie, txstate);
  507. }
  508. /* Prepare descriptor for memory to memory copy */
  509. static struct dma_async_tx_descriptor *
  510. mpc_dma_prep_memcpy(struct dma_chan *chan, dma_addr_t dst, dma_addr_t src,
  511. size_t len, unsigned long flags)
  512. {
  513. struct mpc_dma *mdma = dma_chan_to_mpc_dma(chan);
  514. struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);
  515. struct mpc_dma_desc *mdesc = NULL;
  516. struct mpc_dma_tcd *tcd;
  517. unsigned long iflags;
  518. /* Get free descriptor */
  519. spin_lock_irqsave(&mchan->lock, iflags);
  520. if (!list_empty(&mchan->free)) {
  521. mdesc = list_first_entry(&mchan->free, struct mpc_dma_desc,
  522. node);
  523. list_del(&mdesc->node);
  524. }
  525. spin_unlock_irqrestore(&mchan->lock, iflags);
  526. if (!mdesc) {
  527. /* try to free completed descriptors */
  528. mpc_dma_process_completed(mdma);
  529. return NULL;
  530. }
  531. mdesc->error = 0;
  532. mdesc->will_access_peripheral = 0;
  533. tcd = mdesc->tcd;
  534. /* Prepare Transfer Control Descriptor for this transaction */
  535. memset(tcd, 0, sizeof(struct mpc_dma_tcd));
  536. if (IS_ALIGNED(src | dst | len, 32)) {
  537. tcd->ssize = MPC_DMA_TSIZE_32;
  538. tcd->dsize = MPC_DMA_TSIZE_32;
  539. tcd->soff = 32;
  540. tcd->doff = 32;
  541. } else if (!mdma->is_mpc8308 && IS_ALIGNED(src | dst | len, 16)) {
  542. /* MPC8308 doesn't support 16 byte transfers */
  543. tcd->ssize = MPC_DMA_TSIZE_16;
  544. tcd->dsize = MPC_DMA_TSIZE_16;
  545. tcd->soff = 16;
  546. tcd->doff = 16;
  547. } else if (IS_ALIGNED(src | dst | len, 4)) {
  548. tcd->ssize = MPC_DMA_TSIZE_4;
  549. tcd->dsize = MPC_DMA_TSIZE_4;
  550. tcd->soff = 4;
  551. tcd->doff = 4;
  552. } else if (IS_ALIGNED(src | dst | len, 2)) {
  553. tcd->ssize = MPC_DMA_TSIZE_2;
  554. tcd->dsize = MPC_DMA_TSIZE_2;
  555. tcd->soff = 2;
  556. tcd->doff = 2;
  557. } else {
  558. tcd->ssize = MPC_DMA_TSIZE_1;
  559. tcd->dsize = MPC_DMA_TSIZE_1;
  560. tcd->soff = 1;
  561. tcd->doff = 1;
  562. }
  563. tcd->saddr = src;
  564. tcd->daddr = dst;
  565. tcd->nbytes = len;
  566. tcd->biter = 1;
  567. tcd->citer = 1;
  568. /* Place descriptor in prepared list */
  569. spin_lock_irqsave(&mchan->lock, iflags);
  570. list_add_tail(&mdesc->node, &mchan->prepared);
  571. spin_unlock_irqrestore(&mchan->lock, iflags);
  572. return &mdesc->desc;
  573. }
  574. inline u8 buswidth_to_dmatsize(u8 buswidth)
  575. {
  576. u8 res;
  577. for (res = 0; buswidth > 1; buswidth /= 2)
  578. res++;
  579. return res;
  580. }
  581. static struct dma_async_tx_descriptor *
  582. mpc_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
  583. unsigned int sg_len, enum dma_transfer_direction direction,
  584. unsigned long flags, void *context)
  585. {
  586. struct mpc_dma *mdma = dma_chan_to_mpc_dma(chan);
  587. struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);
  588. struct mpc_dma_desc *mdesc = NULL;
  589. dma_addr_t per_paddr;
  590. u32 tcd_nunits;
  591. struct mpc_dma_tcd *tcd;
  592. unsigned long iflags;
  593. struct scatterlist *sg;
  594. size_t len;
  595. int iter, i;
  596. /* Currently there is no proper support for scatter/gather */
  597. if (sg_len != 1)
  598. return NULL;
  599. if (!is_slave_direction(direction))
  600. return NULL;
  601. for_each_sg(sgl, sg, sg_len, i) {
  602. spin_lock_irqsave(&mchan->lock, iflags);
  603. mdesc = list_first_entry(&mchan->free,
  604. struct mpc_dma_desc, node);
  605. if (!mdesc) {
  606. spin_unlock_irqrestore(&mchan->lock, iflags);
  607. /* Try to free completed descriptors */
  608. mpc_dma_process_completed(mdma);
  609. return NULL;
  610. }
  611. list_del(&mdesc->node);
  612. if (direction == DMA_DEV_TO_MEM) {
  613. per_paddr = mchan->src_per_paddr;
  614. tcd_nunits = mchan->src_tcd_nunits;
  615. } else {
  616. per_paddr = mchan->dst_per_paddr;
  617. tcd_nunits = mchan->dst_tcd_nunits;
  618. }
  619. spin_unlock_irqrestore(&mchan->lock, iflags);
  620. if (per_paddr == 0 || tcd_nunits == 0)
  621. goto err_prep;
  622. mdesc->error = 0;
  623. mdesc->will_access_peripheral = 1;
  624. /* Prepare Transfer Control Descriptor for this transaction */
  625. tcd = mdesc->tcd;
  626. memset(tcd, 0, sizeof(struct mpc_dma_tcd));
  627. if (direction == DMA_DEV_TO_MEM) {
  628. tcd->saddr = per_paddr;
  629. tcd->daddr = sg_dma_address(sg);
  630. if (!IS_ALIGNED(sg_dma_address(sg), mchan->dwidth))
  631. goto err_prep;
  632. tcd->soff = 0;
  633. tcd->doff = mchan->dwidth;
  634. } else {
  635. tcd->saddr = sg_dma_address(sg);
  636. tcd->daddr = per_paddr;
  637. if (!IS_ALIGNED(sg_dma_address(sg), mchan->swidth))
  638. goto err_prep;
  639. tcd->soff = mchan->swidth;
  640. tcd->doff = 0;
  641. }
  642. tcd->ssize = buswidth_to_dmatsize(mchan->swidth);
  643. tcd->dsize = buswidth_to_dmatsize(mchan->dwidth);
  644. if (mdma->is_mpc8308) {
  645. tcd->nbytes = sg_dma_len(sg);
  646. if (!IS_ALIGNED(tcd->nbytes, mchan->swidth))
  647. goto err_prep;
  648. /* No major loops for MPC8303 */
  649. tcd->biter = 1;
  650. tcd->citer = 1;
  651. } else {
  652. len = sg_dma_len(sg);
  653. tcd->nbytes = tcd_nunits * tcd->ssize;
  654. if (!IS_ALIGNED(len, tcd->nbytes))
  655. goto err_prep;
  656. iter = len / tcd->nbytes;
  657. if (iter >= 1 << 15) {
  658. /* len is too big */
  659. goto err_prep;
  660. }
  661. /* citer_linkch contains the high bits of iter */
  662. tcd->biter = iter & 0x1ff;
  663. tcd->biter_linkch = iter >> 9;
  664. tcd->citer = tcd->biter;
  665. tcd->citer_linkch = tcd->biter_linkch;
  666. }
  667. tcd->e_sg = 0;
  668. tcd->d_req = 1;
  669. /* Place descriptor in prepared list */
  670. spin_lock_irqsave(&mchan->lock, iflags);
  671. list_add_tail(&mdesc->node, &mchan->prepared);
  672. spin_unlock_irqrestore(&mchan->lock, iflags);
  673. }
  674. return &mdesc->desc;
  675. err_prep:
  676. /* Put the descriptor back */
  677. spin_lock_irqsave(&mchan->lock, iflags);
  678. list_add_tail(&mdesc->node, &mchan->free);
  679. spin_unlock_irqrestore(&mchan->lock, iflags);
  680. return NULL;
  681. }
  682. inline bool is_buswidth_valid(u8 buswidth, bool is_mpc8308)
  683. {
  684. switch (buswidth) {
  685. case 16:
  686. if (is_mpc8308)
  687. return false;
  688. case 1:
  689. case 2:
  690. case 4:
  691. case 32:
  692. break;
  693. default:
  694. return false;
  695. }
  696. return true;
  697. }
  698. static int mpc_dma_device_config(struct dma_chan *chan,
  699. struct dma_slave_config *cfg)
  700. {
  701. struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);
  702. struct mpc_dma *mdma = dma_chan_to_mpc_dma(&mchan->chan);
  703. unsigned long flags;
  704. /*
  705. * Software constraints:
  706. * - only transfers between a peripheral device and memory are
  707. * supported
  708. * - transfer chunk sizes of 1, 2, 4, 16 (for MPC512x), and 32 bytes
  709. * are supported, and, consequently, source addresses and
  710. * destination addresses; must be aligned accordingly; furthermore,
  711. * for MPC512x SoCs, the transfer size must be aligned on (chunk
  712. * size * maxburst)
  713. * - during the transfer, the RAM address is incremented by the size
  714. * of transfer chunk
  715. * - the peripheral port's address is constant during the transfer.
  716. */
  717. if (!IS_ALIGNED(cfg->src_addr, cfg->src_addr_width) ||
  718. !IS_ALIGNED(cfg->dst_addr, cfg->dst_addr_width)) {
  719. return -EINVAL;
  720. }
  721. if (!is_buswidth_valid(cfg->src_addr_width, mdma->is_mpc8308) ||
  722. !is_buswidth_valid(cfg->dst_addr_width, mdma->is_mpc8308))
  723. return -EINVAL;
  724. spin_lock_irqsave(&mchan->lock, flags);
  725. mchan->src_per_paddr = cfg->src_addr;
  726. mchan->src_tcd_nunits = cfg->src_maxburst;
  727. mchan->swidth = cfg->src_addr_width;
  728. mchan->dst_per_paddr = cfg->dst_addr;
  729. mchan->dst_tcd_nunits = cfg->dst_maxburst;
  730. mchan->dwidth = cfg->dst_addr_width;
  731. /* Apply defaults */
  732. if (mchan->src_tcd_nunits == 0)
  733. mchan->src_tcd_nunits = 1;
  734. if (mchan->dst_tcd_nunits == 0)
  735. mchan->dst_tcd_nunits = 1;
  736. spin_unlock_irqrestore(&mchan->lock, flags);
  737. return 0;
  738. }
  739. static int mpc_dma_device_terminate_all(struct dma_chan *chan)
  740. {
  741. struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);
  742. struct mpc_dma *mdma = dma_chan_to_mpc_dma(chan);
  743. unsigned long flags;
  744. /* Disable channel requests */
  745. spin_lock_irqsave(&mchan->lock, flags);
  746. out_8(&mdma->regs->dmacerq, chan->chan_id);
  747. list_splice_tail_init(&mchan->prepared, &mchan->free);
  748. list_splice_tail_init(&mchan->queued, &mchan->free);
  749. list_splice_tail_init(&mchan->active, &mchan->free);
  750. spin_unlock_irqrestore(&mchan->lock, flags);
  751. return 0;
  752. }
  753. static int mpc_dma_probe(struct platform_device *op)
  754. {
  755. struct device_node *dn = op->dev.of_node;
  756. struct device *dev = &op->dev;
  757. struct dma_device *dma;
  758. struct mpc_dma *mdma;
  759. struct mpc_dma_chan *mchan;
  760. struct resource res;
  761. ulong regs_start, regs_size;
  762. int retval, i;
  763. u8 chancnt;
  764. mdma = devm_kzalloc(dev, sizeof(struct mpc_dma), GFP_KERNEL);
  765. if (!mdma) {
  766. retval = -ENOMEM;
  767. goto err;
  768. }
  769. mdma->irq = irq_of_parse_and_map(dn, 0);
  770. if (!mdma->irq) {
  771. dev_err(dev, "Error mapping IRQ!\n");
  772. retval = -EINVAL;
  773. goto err;
  774. }
  775. if (of_device_is_compatible(dn, "fsl,mpc8308-dma")) {
  776. mdma->is_mpc8308 = 1;
  777. mdma->irq2 = irq_of_parse_and_map(dn, 1);
  778. if (!mdma->irq2) {
  779. dev_err(dev, "Error mapping IRQ!\n");
  780. retval = -EINVAL;
  781. goto err_dispose1;
  782. }
  783. }
  784. retval = of_address_to_resource(dn, 0, &res);
  785. if (retval) {
  786. dev_err(dev, "Error parsing memory region!\n");
  787. goto err_dispose2;
  788. }
  789. regs_start = res.start;
  790. regs_size = resource_size(&res);
  791. if (!devm_request_mem_region(dev, regs_start, regs_size, DRV_NAME)) {
  792. dev_err(dev, "Error requesting memory region!\n");
  793. retval = -EBUSY;
  794. goto err_dispose2;
  795. }
  796. mdma->regs = devm_ioremap(dev, regs_start, regs_size);
  797. if (!mdma->regs) {
  798. dev_err(dev, "Error mapping memory region!\n");
  799. retval = -ENOMEM;
  800. goto err_dispose2;
  801. }
  802. mdma->tcd = (struct mpc_dma_tcd *)((u8 *)(mdma->regs)
  803. + MPC_DMA_TCD_OFFSET);
  804. retval = request_irq(mdma->irq, &mpc_dma_irq, 0, DRV_NAME, mdma);
  805. if (retval) {
  806. dev_err(dev, "Error requesting IRQ!\n");
  807. retval = -EINVAL;
  808. goto err_dispose2;
  809. }
  810. if (mdma->is_mpc8308) {
  811. retval = request_irq(mdma->irq2, &mpc_dma_irq, 0,
  812. DRV_NAME, mdma);
  813. if (retval) {
  814. dev_err(dev, "Error requesting IRQ2!\n");
  815. retval = -EINVAL;
  816. goto err_free1;
  817. }
  818. }
  819. spin_lock_init(&mdma->error_status_lock);
  820. dma = &mdma->dma;
  821. dma->dev = dev;
  822. dma->device_alloc_chan_resources = mpc_dma_alloc_chan_resources;
  823. dma->device_free_chan_resources = mpc_dma_free_chan_resources;
  824. dma->device_issue_pending = mpc_dma_issue_pending;
  825. dma->device_tx_status = mpc_dma_tx_status;
  826. dma->device_prep_dma_memcpy = mpc_dma_prep_memcpy;
  827. dma->device_prep_slave_sg = mpc_dma_prep_slave_sg;
  828. dma->device_config = mpc_dma_device_config;
  829. dma->device_terminate_all = mpc_dma_device_terminate_all;
  830. INIT_LIST_HEAD(&dma->channels);
  831. dma_cap_set(DMA_MEMCPY, dma->cap_mask);
  832. dma_cap_set(DMA_SLAVE, dma->cap_mask);
  833. if (mdma->is_mpc8308)
  834. chancnt = MPC8308_DMACHAN_MAX;
  835. else
  836. chancnt = MPC512x_DMACHAN_MAX;
  837. for (i = 0; i < chancnt; i++) {
  838. mchan = &mdma->channels[i];
  839. mchan->chan.device = dma;
  840. dma_cookie_init(&mchan->chan);
  841. INIT_LIST_HEAD(&mchan->free);
  842. INIT_LIST_HEAD(&mchan->prepared);
  843. INIT_LIST_HEAD(&mchan->queued);
  844. INIT_LIST_HEAD(&mchan->active);
  845. INIT_LIST_HEAD(&mchan->completed);
  846. spin_lock_init(&mchan->lock);
  847. list_add_tail(&mchan->chan.device_node, &dma->channels);
  848. }
  849. tasklet_setup(&mdma->tasklet, mpc_dma_tasklet);
  850. /*
  851. * Configure DMA Engine:
  852. * - Dynamic clock,
  853. * - Round-robin group arbitration,
  854. * - Round-robin channel arbitration.
  855. */
  856. if (mdma->is_mpc8308) {
  857. /* MPC8308 has 16 channels and lacks some registers */
  858. out_be32(&mdma->regs->dmacr, MPC_DMA_DMACR_ERCA);
  859. /* enable snooping */
  860. out_be32(&mdma->regs->dmagpor, MPC_DMA_DMAGPOR_SNOOP_ENABLE);
  861. /* Disable error interrupts */
  862. out_be32(&mdma->regs->dmaeeil, 0);
  863. /* Clear interrupts status */
  864. out_be32(&mdma->regs->dmaintl, 0xFFFF);
  865. out_be32(&mdma->regs->dmaerrl, 0xFFFF);
  866. } else {
  867. out_be32(&mdma->regs->dmacr, MPC_DMA_DMACR_EDCG |
  868. MPC_DMA_DMACR_ERGA |
  869. MPC_DMA_DMACR_ERCA);
  870. /* Disable hardware DMA requests */
  871. out_be32(&mdma->regs->dmaerqh, 0);
  872. out_be32(&mdma->regs->dmaerql, 0);
  873. /* Disable error interrupts */
  874. out_be32(&mdma->regs->dmaeeih, 0);
  875. out_be32(&mdma->regs->dmaeeil, 0);
  876. /* Clear interrupts status */
  877. out_be32(&mdma->regs->dmainth, 0xFFFFFFFF);
  878. out_be32(&mdma->regs->dmaintl, 0xFFFFFFFF);
  879. out_be32(&mdma->regs->dmaerrh, 0xFFFFFFFF);
  880. out_be32(&mdma->regs->dmaerrl, 0xFFFFFFFF);
  881. /* Route interrupts to IPIC */
  882. out_be32(&mdma->regs->dmaihsa, 0);
  883. out_be32(&mdma->regs->dmailsa, 0);
  884. }
  885. /* Register DMA engine */
  886. dev_set_drvdata(dev, mdma);
  887. retval = dma_async_device_register(dma);
  888. if (retval)
  889. goto err_free2;
  890. /* Register with OF helpers for DMA lookups (nonfatal) */
  891. if (dev->of_node) {
  892. retval = of_dma_controller_register(dev->of_node,
  893. of_dma_xlate_by_chan_id, mdma);
  894. if (retval)
  895. dev_warn(dev, "Could not register for OF lookup\n");
  896. }
  897. return 0;
  898. err_free2:
  899. if (mdma->is_mpc8308)
  900. free_irq(mdma->irq2, mdma);
  901. err_free1:
  902. free_irq(mdma->irq, mdma);
  903. err_dispose2:
  904. if (mdma->is_mpc8308)
  905. irq_dispose_mapping(mdma->irq2);
  906. err_dispose1:
  907. irq_dispose_mapping(mdma->irq);
  908. err:
  909. return retval;
  910. }
  911. static int mpc_dma_remove(struct platform_device *op)
  912. {
  913. struct device *dev = &op->dev;
  914. struct mpc_dma *mdma = dev_get_drvdata(dev);
  915. if (dev->of_node)
  916. of_dma_controller_free(dev->of_node);
  917. dma_async_device_unregister(&mdma->dma);
  918. if (mdma->is_mpc8308) {
  919. free_irq(mdma->irq2, mdma);
  920. irq_dispose_mapping(mdma->irq2);
  921. }
  922. free_irq(mdma->irq, mdma);
  923. irq_dispose_mapping(mdma->irq);
  924. tasklet_kill(&mdma->tasklet);
  925. return 0;
  926. }
  927. static const struct of_device_id mpc_dma_match[] = {
  928. { .compatible = "fsl,mpc5121-dma", },
  929. { .compatible = "fsl,mpc8308-dma", },
  930. {},
  931. };
  932. MODULE_DEVICE_TABLE(of, mpc_dma_match);
  933. static struct platform_driver mpc_dma_driver = {
  934. .probe = mpc_dma_probe,
  935. .remove = mpc_dma_remove,
  936. .driver = {
  937. .name = DRV_NAME,
  938. .of_match_table = mpc_dma_match,
  939. },
  940. };
  941. module_platform_driver(mpc_dma_driver);
  942. MODULE_LICENSE("GPL");
  943. MODULE_AUTHOR("Piotr Ziecik <kosmo@semihalf.com>");