mmp_pdma.c 29 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright 2012 Marvell International Ltd.
  4. */
  5. #include <linux/err.h>
  6. #include <linux/module.h>
  7. #include <linux/init.h>
  8. #include <linux/types.h>
  9. #include <linux/interrupt.h>
  10. #include <linux/dma-mapping.h>
  11. #include <linux/slab.h>
  12. #include <linux/dmaengine.h>
  13. #include <linux/platform_device.h>
  14. #include <linux/device.h>
  15. #include <linux/platform_data/mmp_dma.h>
  16. #include <linux/dmapool.h>
  17. #include <linux/of_device.h>
  18. #include <linux/of_dma.h>
  19. #include <linux/of.h>
  20. #include <linux/dma/mmp-pdma.h>
  21. #include "dmaengine.h"
  22. #define DCSR 0x0000
  23. #define DALGN 0x00a0
  24. #define DINT 0x00f0
  25. #define DDADR 0x0200
  26. #define DSADR(n) (0x0204 + ((n) << 4))
  27. #define DTADR(n) (0x0208 + ((n) << 4))
  28. #define DCMD 0x020c
  29. #define DCSR_RUN BIT(31) /* Run Bit (read / write) */
  30. #define DCSR_NODESC BIT(30) /* No-Descriptor Fetch (read / write) */
  31. #define DCSR_STOPIRQEN BIT(29) /* Stop Interrupt Enable (read / write) */
  32. #define DCSR_REQPEND BIT(8) /* Request Pending (read-only) */
  33. #define DCSR_STOPSTATE BIT(3) /* Stop State (read-only) */
  34. #define DCSR_ENDINTR BIT(2) /* End Interrupt (read / write) */
  35. #define DCSR_STARTINTR BIT(1) /* Start Interrupt (read / write) */
  36. #define DCSR_BUSERR BIT(0) /* Bus Error Interrupt (read / write) */
  37. #define DCSR_EORIRQEN BIT(28) /* End of Receive Interrupt Enable (R/W) */
  38. #define DCSR_EORJMPEN BIT(27) /* Jump to next descriptor on EOR */
  39. #define DCSR_EORSTOPEN BIT(26) /* STOP on an EOR */
  40. #define DCSR_SETCMPST BIT(25) /* Set Descriptor Compare Status */
  41. #define DCSR_CLRCMPST BIT(24) /* Clear Descriptor Compare Status */
  42. #define DCSR_CMPST BIT(10) /* The Descriptor Compare Status */
  43. #define DCSR_EORINTR BIT(9) /* The end of Receive */
  44. #define DRCMR(n) ((((n) < 64) ? 0x0100 : 0x1100) + (((n) & 0x3f) << 2))
  45. #define DRCMR_MAPVLD BIT(7) /* Map Valid (read / write) */
  46. #define DRCMR_CHLNUM 0x1f /* mask for Channel Number (read / write) */
  47. #define DDADR_DESCADDR 0xfffffff0 /* Address of next descriptor (mask) */
  48. #define DDADR_STOP BIT(0) /* Stop (read / write) */
  49. #define DCMD_INCSRCADDR BIT(31) /* Source Address Increment Setting. */
  50. #define DCMD_INCTRGADDR BIT(30) /* Target Address Increment Setting. */
  51. #define DCMD_FLOWSRC BIT(29) /* Flow Control by the source. */
  52. #define DCMD_FLOWTRG BIT(28) /* Flow Control by the target. */
  53. #define DCMD_STARTIRQEN BIT(22) /* Start Interrupt Enable */
  54. #define DCMD_ENDIRQEN BIT(21) /* End Interrupt Enable */
  55. #define DCMD_ENDIAN BIT(18) /* Device Endian-ness. */
  56. #define DCMD_BURST8 (1 << 16) /* 8 byte burst */
  57. #define DCMD_BURST16 (2 << 16) /* 16 byte burst */
  58. #define DCMD_BURST32 (3 << 16) /* 32 byte burst */
  59. #define DCMD_WIDTH1 (1 << 14) /* 1 byte width */
  60. #define DCMD_WIDTH2 (2 << 14) /* 2 byte width (HalfWord) */
  61. #define DCMD_WIDTH4 (3 << 14) /* 4 byte width (Word) */
  62. #define DCMD_LENGTH 0x01fff /* length mask (max = 8K - 1) */
  63. #define PDMA_MAX_DESC_BYTES DCMD_LENGTH
  64. struct mmp_pdma_desc_hw {
  65. u32 ddadr; /* Points to the next descriptor + flags */
  66. u32 dsadr; /* DSADR value for the current transfer */
  67. u32 dtadr; /* DTADR value for the current transfer */
  68. u32 dcmd; /* DCMD value for the current transfer */
  69. } __aligned(32);
  70. struct mmp_pdma_desc_sw {
  71. struct mmp_pdma_desc_hw desc;
  72. struct list_head node;
  73. struct list_head tx_list;
  74. struct dma_async_tx_descriptor async_tx;
  75. };
  76. struct mmp_pdma_phy;
  77. struct mmp_pdma_chan {
  78. struct device *dev;
  79. struct dma_chan chan;
  80. struct dma_async_tx_descriptor desc;
  81. struct mmp_pdma_phy *phy;
  82. enum dma_transfer_direction dir;
  83. struct dma_slave_config slave_config;
  84. struct mmp_pdma_desc_sw *cyclic_first; /* first desc_sw if channel
  85. * is in cyclic mode */
  86. /* channel's basic info */
  87. struct tasklet_struct tasklet;
  88. u32 dcmd;
  89. u32 drcmr;
  90. u32 dev_addr;
  91. /* list for desc */
  92. spinlock_t desc_lock; /* Descriptor list lock */
  93. struct list_head chain_pending; /* Link descriptors queue for pending */
  94. struct list_head chain_running; /* Link descriptors queue for running */
  95. bool idle; /* channel statue machine */
  96. bool byte_align;
  97. struct dma_pool *desc_pool; /* Descriptors pool */
  98. };
  99. struct mmp_pdma_phy {
  100. int idx;
  101. void __iomem *base;
  102. struct mmp_pdma_chan *vchan;
  103. };
  104. struct mmp_pdma_device {
  105. int dma_channels;
  106. void __iomem *base;
  107. struct device *dev;
  108. struct dma_device device;
  109. struct mmp_pdma_phy *phy;
  110. spinlock_t phy_lock; /* protect alloc/free phy channels */
  111. };
  112. #define tx_to_mmp_pdma_desc(tx) \
  113. container_of(tx, struct mmp_pdma_desc_sw, async_tx)
  114. #define to_mmp_pdma_desc(lh) \
  115. container_of(lh, struct mmp_pdma_desc_sw, node)
  116. #define to_mmp_pdma_chan(dchan) \
  117. container_of(dchan, struct mmp_pdma_chan, chan)
  118. #define to_mmp_pdma_dev(dmadev) \
  119. container_of(dmadev, struct mmp_pdma_device, device)
  120. static int mmp_pdma_config_write(struct dma_chan *dchan,
  121. struct dma_slave_config *cfg,
  122. enum dma_transfer_direction direction);
  123. static void set_desc(struct mmp_pdma_phy *phy, dma_addr_t addr)
  124. {
  125. u32 reg = (phy->idx << 4) + DDADR;
  126. writel(addr, phy->base + reg);
  127. }
  128. static void enable_chan(struct mmp_pdma_phy *phy)
  129. {
  130. u32 reg, dalgn;
  131. if (!phy->vchan)
  132. return;
  133. reg = DRCMR(phy->vchan->drcmr);
  134. writel(DRCMR_MAPVLD | phy->idx, phy->base + reg);
  135. dalgn = readl(phy->base + DALGN);
  136. if (phy->vchan->byte_align)
  137. dalgn |= 1 << phy->idx;
  138. else
  139. dalgn &= ~(1 << phy->idx);
  140. writel(dalgn, phy->base + DALGN);
  141. reg = (phy->idx << 2) + DCSR;
  142. writel(readl(phy->base + reg) | DCSR_RUN, phy->base + reg);
  143. }
  144. static void disable_chan(struct mmp_pdma_phy *phy)
  145. {
  146. u32 reg;
  147. if (!phy)
  148. return;
  149. reg = (phy->idx << 2) + DCSR;
  150. writel(readl(phy->base + reg) & ~DCSR_RUN, phy->base + reg);
  151. }
  152. static int clear_chan_irq(struct mmp_pdma_phy *phy)
  153. {
  154. u32 dcsr;
  155. u32 dint = readl(phy->base + DINT);
  156. u32 reg = (phy->idx << 2) + DCSR;
  157. if (!(dint & BIT(phy->idx)))
  158. return -EAGAIN;
  159. /* clear irq */
  160. dcsr = readl(phy->base + reg);
  161. writel(dcsr, phy->base + reg);
  162. if ((dcsr & DCSR_BUSERR) && (phy->vchan))
  163. dev_warn(phy->vchan->dev, "DCSR_BUSERR\n");
  164. return 0;
  165. }
  166. static irqreturn_t mmp_pdma_chan_handler(int irq, void *dev_id)
  167. {
  168. struct mmp_pdma_phy *phy = dev_id;
  169. if (clear_chan_irq(phy) != 0)
  170. return IRQ_NONE;
  171. tasklet_schedule(&phy->vchan->tasklet);
  172. return IRQ_HANDLED;
  173. }
  174. static irqreturn_t mmp_pdma_int_handler(int irq, void *dev_id)
  175. {
  176. struct mmp_pdma_device *pdev = dev_id;
  177. struct mmp_pdma_phy *phy;
  178. u32 dint = readl(pdev->base + DINT);
  179. int i, ret;
  180. int irq_num = 0;
  181. while (dint) {
  182. i = __ffs(dint);
  183. /* only handle interrupts belonging to pdma driver*/
  184. if (i >= pdev->dma_channels)
  185. break;
  186. dint &= (dint - 1);
  187. phy = &pdev->phy[i];
  188. ret = mmp_pdma_chan_handler(irq, phy);
  189. if (ret == IRQ_HANDLED)
  190. irq_num++;
  191. }
  192. if (irq_num)
  193. return IRQ_HANDLED;
  194. return IRQ_NONE;
  195. }
  196. /* lookup free phy channel as descending priority */
  197. static struct mmp_pdma_phy *lookup_phy(struct mmp_pdma_chan *pchan)
  198. {
  199. int prio, i;
  200. struct mmp_pdma_device *pdev = to_mmp_pdma_dev(pchan->chan.device);
  201. struct mmp_pdma_phy *phy, *found = NULL;
  202. unsigned long flags;
  203. /*
  204. * dma channel priorities
  205. * ch 0 - 3, 16 - 19 <--> (0)
  206. * ch 4 - 7, 20 - 23 <--> (1)
  207. * ch 8 - 11, 24 - 27 <--> (2)
  208. * ch 12 - 15, 28 - 31 <--> (3)
  209. */
  210. spin_lock_irqsave(&pdev->phy_lock, flags);
  211. for (prio = 0; prio <= ((pdev->dma_channels - 1) & 0xf) >> 2; prio++) {
  212. for (i = 0; i < pdev->dma_channels; i++) {
  213. if (prio != (i & 0xf) >> 2)
  214. continue;
  215. phy = &pdev->phy[i];
  216. if (!phy->vchan) {
  217. phy->vchan = pchan;
  218. found = phy;
  219. goto out_unlock;
  220. }
  221. }
  222. }
  223. out_unlock:
  224. spin_unlock_irqrestore(&pdev->phy_lock, flags);
  225. return found;
  226. }
  227. static void mmp_pdma_free_phy(struct mmp_pdma_chan *pchan)
  228. {
  229. struct mmp_pdma_device *pdev = to_mmp_pdma_dev(pchan->chan.device);
  230. unsigned long flags;
  231. u32 reg;
  232. if (!pchan->phy)
  233. return;
  234. /* clear the channel mapping in DRCMR */
  235. reg = DRCMR(pchan->drcmr);
  236. writel(0, pchan->phy->base + reg);
  237. spin_lock_irqsave(&pdev->phy_lock, flags);
  238. pchan->phy->vchan = NULL;
  239. pchan->phy = NULL;
  240. spin_unlock_irqrestore(&pdev->phy_lock, flags);
  241. }
  242. /*
  243. * start_pending_queue - transfer any pending transactions
  244. * pending list ==> running list
  245. */
  246. static void start_pending_queue(struct mmp_pdma_chan *chan)
  247. {
  248. struct mmp_pdma_desc_sw *desc;
  249. /* still in running, irq will start the pending list */
  250. if (!chan->idle) {
  251. dev_dbg(chan->dev, "DMA controller still busy\n");
  252. return;
  253. }
  254. if (list_empty(&chan->chain_pending)) {
  255. /* chance to re-fetch phy channel with higher prio */
  256. mmp_pdma_free_phy(chan);
  257. dev_dbg(chan->dev, "no pending list\n");
  258. return;
  259. }
  260. if (!chan->phy) {
  261. chan->phy = lookup_phy(chan);
  262. if (!chan->phy) {
  263. dev_dbg(chan->dev, "no free dma channel\n");
  264. return;
  265. }
  266. }
  267. /*
  268. * pending -> running
  269. * reintilize pending list
  270. */
  271. desc = list_first_entry(&chan->chain_pending,
  272. struct mmp_pdma_desc_sw, node);
  273. list_splice_tail_init(&chan->chain_pending, &chan->chain_running);
  274. /*
  275. * Program the descriptor's address into the DMA controller,
  276. * then start the DMA transaction
  277. */
  278. set_desc(chan->phy, desc->async_tx.phys);
  279. enable_chan(chan->phy);
  280. chan->idle = false;
  281. }
  282. /* desc->tx_list ==> pending list */
  283. static dma_cookie_t mmp_pdma_tx_submit(struct dma_async_tx_descriptor *tx)
  284. {
  285. struct mmp_pdma_chan *chan = to_mmp_pdma_chan(tx->chan);
  286. struct mmp_pdma_desc_sw *desc = tx_to_mmp_pdma_desc(tx);
  287. struct mmp_pdma_desc_sw *child;
  288. unsigned long flags;
  289. dma_cookie_t cookie = -EBUSY;
  290. spin_lock_irqsave(&chan->desc_lock, flags);
  291. list_for_each_entry(child, &desc->tx_list, node) {
  292. cookie = dma_cookie_assign(&child->async_tx);
  293. }
  294. /* softly link to pending list - desc->tx_list ==> pending list */
  295. list_splice_tail_init(&desc->tx_list, &chan->chain_pending);
  296. spin_unlock_irqrestore(&chan->desc_lock, flags);
  297. return cookie;
  298. }
  299. static struct mmp_pdma_desc_sw *
  300. mmp_pdma_alloc_descriptor(struct mmp_pdma_chan *chan)
  301. {
  302. struct mmp_pdma_desc_sw *desc;
  303. dma_addr_t pdesc;
  304. desc = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &pdesc);
  305. if (!desc) {
  306. dev_err(chan->dev, "out of memory for link descriptor\n");
  307. return NULL;
  308. }
  309. INIT_LIST_HEAD(&desc->tx_list);
  310. dma_async_tx_descriptor_init(&desc->async_tx, &chan->chan);
  311. /* each desc has submit */
  312. desc->async_tx.tx_submit = mmp_pdma_tx_submit;
  313. desc->async_tx.phys = pdesc;
  314. return desc;
  315. }
  316. /*
  317. * mmp_pdma_alloc_chan_resources - Allocate resources for DMA channel.
  318. *
  319. * This function will create a dma pool for descriptor allocation.
  320. * Request irq only when channel is requested
  321. * Return - The number of allocated descriptors.
  322. */
  323. static int mmp_pdma_alloc_chan_resources(struct dma_chan *dchan)
  324. {
  325. struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
  326. if (chan->desc_pool)
  327. return 1;
  328. chan->desc_pool = dma_pool_create(dev_name(&dchan->dev->device),
  329. chan->dev,
  330. sizeof(struct mmp_pdma_desc_sw),
  331. __alignof__(struct mmp_pdma_desc_sw),
  332. 0);
  333. if (!chan->desc_pool) {
  334. dev_err(chan->dev, "unable to allocate descriptor pool\n");
  335. return -ENOMEM;
  336. }
  337. mmp_pdma_free_phy(chan);
  338. chan->idle = true;
  339. chan->dev_addr = 0;
  340. return 1;
  341. }
  342. static void mmp_pdma_free_desc_list(struct mmp_pdma_chan *chan,
  343. struct list_head *list)
  344. {
  345. struct mmp_pdma_desc_sw *desc, *_desc;
  346. list_for_each_entry_safe(desc, _desc, list, node) {
  347. list_del(&desc->node);
  348. dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys);
  349. }
  350. }
  351. static void mmp_pdma_free_chan_resources(struct dma_chan *dchan)
  352. {
  353. struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
  354. unsigned long flags;
  355. spin_lock_irqsave(&chan->desc_lock, flags);
  356. mmp_pdma_free_desc_list(chan, &chan->chain_pending);
  357. mmp_pdma_free_desc_list(chan, &chan->chain_running);
  358. spin_unlock_irqrestore(&chan->desc_lock, flags);
  359. dma_pool_destroy(chan->desc_pool);
  360. chan->desc_pool = NULL;
  361. chan->idle = true;
  362. chan->dev_addr = 0;
  363. mmp_pdma_free_phy(chan);
  364. return;
  365. }
  366. static struct dma_async_tx_descriptor *
  367. mmp_pdma_prep_memcpy(struct dma_chan *dchan,
  368. dma_addr_t dma_dst, dma_addr_t dma_src,
  369. size_t len, unsigned long flags)
  370. {
  371. struct mmp_pdma_chan *chan;
  372. struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new;
  373. size_t copy = 0;
  374. if (!dchan)
  375. return NULL;
  376. if (!len)
  377. return NULL;
  378. chan = to_mmp_pdma_chan(dchan);
  379. chan->byte_align = false;
  380. if (!chan->dir) {
  381. chan->dir = DMA_MEM_TO_MEM;
  382. chan->dcmd = DCMD_INCTRGADDR | DCMD_INCSRCADDR;
  383. chan->dcmd |= DCMD_BURST32;
  384. }
  385. do {
  386. /* Allocate the link descriptor from DMA pool */
  387. new = mmp_pdma_alloc_descriptor(chan);
  388. if (!new) {
  389. dev_err(chan->dev, "no memory for desc\n");
  390. goto fail;
  391. }
  392. copy = min_t(size_t, len, PDMA_MAX_DESC_BYTES);
  393. if (dma_src & 0x7 || dma_dst & 0x7)
  394. chan->byte_align = true;
  395. new->desc.dcmd = chan->dcmd | (DCMD_LENGTH & copy);
  396. new->desc.dsadr = dma_src;
  397. new->desc.dtadr = dma_dst;
  398. if (!first)
  399. first = new;
  400. else
  401. prev->desc.ddadr = new->async_tx.phys;
  402. new->async_tx.cookie = 0;
  403. async_tx_ack(&new->async_tx);
  404. prev = new;
  405. len -= copy;
  406. if (chan->dir == DMA_MEM_TO_DEV) {
  407. dma_src += copy;
  408. } else if (chan->dir == DMA_DEV_TO_MEM) {
  409. dma_dst += copy;
  410. } else if (chan->dir == DMA_MEM_TO_MEM) {
  411. dma_src += copy;
  412. dma_dst += copy;
  413. }
  414. /* Insert the link descriptor to the LD ring */
  415. list_add_tail(&new->node, &first->tx_list);
  416. } while (len);
  417. first->async_tx.flags = flags; /* client is in control of this ack */
  418. first->async_tx.cookie = -EBUSY;
  419. /* last desc and fire IRQ */
  420. new->desc.ddadr = DDADR_STOP;
  421. new->desc.dcmd |= DCMD_ENDIRQEN;
  422. chan->cyclic_first = NULL;
  423. return &first->async_tx;
  424. fail:
  425. if (first)
  426. mmp_pdma_free_desc_list(chan, &first->tx_list);
  427. return NULL;
  428. }
  429. static struct dma_async_tx_descriptor *
  430. mmp_pdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
  431. unsigned int sg_len, enum dma_transfer_direction dir,
  432. unsigned long flags, void *context)
  433. {
  434. struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
  435. struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new = NULL;
  436. size_t len, avail;
  437. struct scatterlist *sg;
  438. dma_addr_t addr;
  439. int i;
  440. if ((sgl == NULL) || (sg_len == 0))
  441. return NULL;
  442. chan->byte_align = false;
  443. mmp_pdma_config_write(dchan, &chan->slave_config, dir);
  444. for_each_sg(sgl, sg, sg_len, i) {
  445. addr = sg_dma_address(sg);
  446. avail = sg_dma_len(sgl);
  447. do {
  448. len = min_t(size_t, avail, PDMA_MAX_DESC_BYTES);
  449. if (addr & 0x7)
  450. chan->byte_align = true;
  451. /* allocate and populate the descriptor */
  452. new = mmp_pdma_alloc_descriptor(chan);
  453. if (!new) {
  454. dev_err(chan->dev, "no memory for desc\n");
  455. goto fail;
  456. }
  457. new->desc.dcmd = chan->dcmd | (DCMD_LENGTH & len);
  458. if (dir == DMA_MEM_TO_DEV) {
  459. new->desc.dsadr = addr;
  460. new->desc.dtadr = chan->dev_addr;
  461. } else {
  462. new->desc.dsadr = chan->dev_addr;
  463. new->desc.dtadr = addr;
  464. }
  465. if (!first)
  466. first = new;
  467. else
  468. prev->desc.ddadr = new->async_tx.phys;
  469. new->async_tx.cookie = 0;
  470. async_tx_ack(&new->async_tx);
  471. prev = new;
  472. /* Insert the link descriptor to the LD ring */
  473. list_add_tail(&new->node, &first->tx_list);
  474. /* update metadata */
  475. addr += len;
  476. avail -= len;
  477. } while (avail);
  478. }
  479. first->async_tx.cookie = -EBUSY;
  480. first->async_tx.flags = flags;
  481. /* last desc and fire IRQ */
  482. new->desc.ddadr = DDADR_STOP;
  483. new->desc.dcmd |= DCMD_ENDIRQEN;
  484. chan->dir = dir;
  485. chan->cyclic_first = NULL;
  486. return &first->async_tx;
  487. fail:
  488. if (first)
  489. mmp_pdma_free_desc_list(chan, &first->tx_list);
  490. return NULL;
  491. }
  492. static struct dma_async_tx_descriptor *
  493. mmp_pdma_prep_dma_cyclic(struct dma_chan *dchan,
  494. dma_addr_t buf_addr, size_t len, size_t period_len,
  495. enum dma_transfer_direction direction,
  496. unsigned long flags)
  497. {
  498. struct mmp_pdma_chan *chan;
  499. struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new;
  500. dma_addr_t dma_src, dma_dst;
  501. if (!dchan || !len || !period_len)
  502. return NULL;
  503. /* the buffer length must be a multiple of period_len */
  504. if (len % period_len != 0)
  505. return NULL;
  506. if (period_len > PDMA_MAX_DESC_BYTES)
  507. return NULL;
  508. chan = to_mmp_pdma_chan(dchan);
  509. mmp_pdma_config_write(dchan, &chan->slave_config, direction);
  510. switch (direction) {
  511. case DMA_MEM_TO_DEV:
  512. dma_src = buf_addr;
  513. dma_dst = chan->dev_addr;
  514. break;
  515. case DMA_DEV_TO_MEM:
  516. dma_dst = buf_addr;
  517. dma_src = chan->dev_addr;
  518. break;
  519. default:
  520. dev_err(chan->dev, "Unsupported direction for cyclic DMA\n");
  521. return NULL;
  522. }
  523. chan->dir = direction;
  524. do {
  525. /* Allocate the link descriptor from DMA pool */
  526. new = mmp_pdma_alloc_descriptor(chan);
  527. if (!new) {
  528. dev_err(chan->dev, "no memory for desc\n");
  529. goto fail;
  530. }
  531. new->desc.dcmd = (chan->dcmd | DCMD_ENDIRQEN |
  532. (DCMD_LENGTH & period_len));
  533. new->desc.dsadr = dma_src;
  534. new->desc.dtadr = dma_dst;
  535. if (!first)
  536. first = new;
  537. else
  538. prev->desc.ddadr = new->async_tx.phys;
  539. new->async_tx.cookie = 0;
  540. async_tx_ack(&new->async_tx);
  541. prev = new;
  542. len -= period_len;
  543. if (chan->dir == DMA_MEM_TO_DEV)
  544. dma_src += period_len;
  545. else
  546. dma_dst += period_len;
  547. /* Insert the link descriptor to the LD ring */
  548. list_add_tail(&new->node, &first->tx_list);
  549. } while (len);
  550. first->async_tx.flags = flags; /* client is in control of this ack */
  551. first->async_tx.cookie = -EBUSY;
  552. /* make the cyclic link */
  553. new->desc.ddadr = first->async_tx.phys;
  554. chan->cyclic_first = first;
  555. return &first->async_tx;
  556. fail:
  557. if (first)
  558. mmp_pdma_free_desc_list(chan, &first->tx_list);
  559. return NULL;
  560. }
  561. static int mmp_pdma_config_write(struct dma_chan *dchan,
  562. struct dma_slave_config *cfg,
  563. enum dma_transfer_direction direction)
  564. {
  565. struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
  566. u32 maxburst = 0, addr = 0;
  567. enum dma_slave_buswidth width = DMA_SLAVE_BUSWIDTH_UNDEFINED;
  568. if (!dchan)
  569. return -EINVAL;
  570. if (direction == DMA_DEV_TO_MEM) {
  571. chan->dcmd = DCMD_INCTRGADDR | DCMD_FLOWSRC;
  572. maxburst = cfg->src_maxburst;
  573. width = cfg->src_addr_width;
  574. addr = cfg->src_addr;
  575. } else if (direction == DMA_MEM_TO_DEV) {
  576. chan->dcmd = DCMD_INCSRCADDR | DCMD_FLOWTRG;
  577. maxburst = cfg->dst_maxburst;
  578. width = cfg->dst_addr_width;
  579. addr = cfg->dst_addr;
  580. }
  581. if (width == DMA_SLAVE_BUSWIDTH_1_BYTE)
  582. chan->dcmd |= DCMD_WIDTH1;
  583. else if (width == DMA_SLAVE_BUSWIDTH_2_BYTES)
  584. chan->dcmd |= DCMD_WIDTH2;
  585. else if (width == DMA_SLAVE_BUSWIDTH_4_BYTES)
  586. chan->dcmd |= DCMD_WIDTH4;
  587. if (maxburst == 8)
  588. chan->dcmd |= DCMD_BURST8;
  589. else if (maxburst == 16)
  590. chan->dcmd |= DCMD_BURST16;
  591. else if (maxburst == 32)
  592. chan->dcmd |= DCMD_BURST32;
  593. chan->dir = direction;
  594. chan->dev_addr = addr;
  595. return 0;
  596. }
  597. static int mmp_pdma_config(struct dma_chan *dchan,
  598. struct dma_slave_config *cfg)
  599. {
  600. struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
  601. memcpy(&chan->slave_config, cfg, sizeof(*cfg));
  602. return 0;
  603. }
  604. static int mmp_pdma_terminate_all(struct dma_chan *dchan)
  605. {
  606. struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
  607. unsigned long flags;
  608. if (!dchan)
  609. return -EINVAL;
  610. disable_chan(chan->phy);
  611. mmp_pdma_free_phy(chan);
  612. spin_lock_irqsave(&chan->desc_lock, flags);
  613. mmp_pdma_free_desc_list(chan, &chan->chain_pending);
  614. mmp_pdma_free_desc_list(chan, &chan->chain_running);
  615. spin_unlock_irqrestore(&chan->desc_lock, flags);
  616. chan->idle = true;
  617. return 0;
  618. }
  619. static unsigned int mmp_pdma_residue(struct mmp_pdma_chan *chan,
  620. dma_cookie_t cookie)
  621. {
  622. struct mmp_pdma_desc_sw *sw;
  623. u32 curr, residue = 0;
  624. bool passed = false;
  625. bool cyclic = chan->cyclic_first != NULL;
  626. /*
  627. * If the channel does not have a phy pointer anymore, it has already
  628. * been completed. Therefore, its residue is 0.
  629. */
  630. if (!chan->phy)
  631. return 0;
  632. if (chan->dir == DMA_DEV_TO_MEM)
  633. curr = readl(chan->phy->base + DTADR(chan->phy->idx));
  634. else
  635. curr = readl(chan->phy->base + DSADR(chan->phy->idx));
  636. list_for_each_entry(sw, &chan->chain_running, node) {
  637. u32 start, end, len;
  638. if (chan->dir == DMA_DEV_TO_MEM)
  639. start = sw->desc.dtadr;
  640. else
  641. start = sw->desc.dsadr;
  642. len = sw->desc.dcmd & DCMD_LENGTH;
  643. end = start + len;
  644. /*
  645. * 'passed' will be latched once we found the descriptor which
  646. * lies inside the boundaries of the curr pointer. All
  647. * descriptors that occur in the list _after_ we found that
  648. * partially handled descriptor are still to be processed and
  649. * are hence added to the residual bytes counter.
  650. */
  651. if (passed) {
  652. residue += len;
  653. } else if (curr >= start && curr <= end) {
  654. residue += end - curr;
  655. passed = true;
  656. }
  657. /*
  658. * Descriptors that have the ENDIRQEN bit set mark the end of a
  659. * transaction chain, and the cookie assigned with it has been
  660. * returned previously from mmp_pdma_tx_submit().
  661. *
  662. * In case we have multiple transactions in the running chain,
  663. * and the cookie does not match the one the user asked us
  664. * about, reset the state variables and start over.
  665. *
  666. * This logic does not apply to cyclic transactions, where all
  667. * descriptors have the ENDIRQEN bit set, and for which we
  668. * can't have multiple transactions on one channel anyway.
  669. */
  670. if (cyclic || !(sw->desc.dcmd & DCMD_ENDIRQEN))
  671. continue;
  672. if (sw->async_tx.cookie == cookie) {
  673. return residue;
  674. } else {
  675. residue = 0;
  676. passed = false;
  677. }
  678. }
  679. /* We should only get here in case of cyclic transactions */
  680. return residue;
  681. }
  682. static enum dma_status mmp_pdma_tx_status(struct dma_chan *dchan,
  683. dma_cookie_t cookie,
  684. struct dma_tx_state *txstate)
  685. {
  686. struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
  687. enum dma_status ret;
  688. ret = dma_cookie_status(dchan, cookie, txstate);
  689. if (likely(ret != DMA_ERROR))
  690. dma_set_residue(txstate, mmp_pdma_residue(chan, cookie));
  691. return ret;
  692. }
  693. /*
  694. * mmp_pdma_issue_pending - Issue the DMA start command
  695. * pending list ==> running list
  696. */
  697. static void mmp_pdma_issue_pending(struct dma_chan *dchan)
  698. {
  699. struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
  700. unsigned long flags;
  701. spin_lock_irqsave(&chan->desc_lock, flags);
  702. start_pending_queue(chan);
  703. spin_unlock_irqrestore(&chan->desc_lock, flags);
  704. }
  705. /*
  706. * dma_do_tasklet
  707. * Do call back
  708. * Start pending list
  709. */
  710. static void dma_do_tasklet(struct tasklet_struct *t)
  711. {
  712. struct mmp_pdma_chan *chan = from_tasklet(chan, t, tasklet);
  713. struct mmp_pdma_desc_sw *desc, *_desc;
  714. LIST_HEAD(chain_cleanup);
  715. unsigned long flags;
  716. struct dmaengine_desc_callback cb;
  717. if (chan->cyclic_first) {
  718. spin_lock_irqsave(&chan->desc_lock, flags);
  719. desc = chan->cyclic_first;
  720. dmaengine_desc_get_callback(&desc->async_tx, &cb);
  721. spin_unlock_irqrestore(&chan->desc_lock, flags);
  722. dmaengine_desc_callback_invoke(&cb, NULL);
  723. return;
  724. }
  725. /* submit pending list; callback for each desc; free desc */
  726. spin_lock_irqsave(&chan->desc_lock, flags);
  727. list_for_each_entry_safe(desc, _desc, &chan->chain_running, node) {
  728. /*
  729. * move the descriptors to a temporary list so we can drop
  730. * the lock during the entire cleanup operation
  731. */
  732. list_move(&desc->node, &chain_cleanup);
  733. /*
  734. * Look for the first list entry which has the ENDIRQEN flag
  735. * set. That is the descriptor we got an interrupt for, so
  736. * complete that transaction and its cookie.
  737. */
  738. if (desc->desc.dcmd & DCMD_ENDIRQEN) {
  739. dma_cookie_t cookie = desc->async_tx.cookie;
  740. dma_cookie_complete(&desc->async_tx);
  741. dev_dbg(chan->dev, "completed_cookie=%d\n", cookie);
  742. break;
  743. }
  744. }
  745. /*
  746. * The hardware is idle and ready for more when the
  747. * chain_running list is empty.
  748. */
  749. chan->idle = list_empty(&chan->chain_running);
  750. /* Start any pending transactions automatically */
  751. start_pending_queue(chan);
  752. spin_unlock_irqrestore(&chan->desc_lock, flags);
  753. /* Run the callback for each descriptor, in order */
  754. list_for_each_entry_safe(desc, _desc, &chain_cleanup, node) {
  755. struct dma_async_tx_descriptor *txd = &desc->async_tx;
  756. /* Remove from the list of transactions */
  757. list_del(&desc->node);
  758. /* Run the link descriptor callback function */
  759. dmaengine_desc_get_callback(txd, &cb);
  760. dmaengine_desc_callback_invoke(&cb, NULL);
  761. dma_pool_free(chan->desc_pool, desc, txd->phys);
  762. }
  763. }
  764. static int mmp_pdma_remove(struct platform_device *op)
  765. {
  766. struct mmp_pdma_device *pdev = platform_get_drvdata(op);
  767. struct mmp_pdma_phy *phy;
  768. int i, irq = 0, irq_num = 0;
  769. if (op->dev.of_node)
  770. of_dma_controller_free(op->dev.of_node);
  771. for (i = 0; i < pdev->dma_channels; i++) {
  772. if (platform_get_irq(op, i) > 0)
  773. irq_num++;
  774. }
  775. if (irq_num != pdev->dma_channels) {
  776. irq = platform_get_irq(op, 0);
  777. devm_free_irq(&op->dev, irq, pdev);
  778. } else {
  779. for (i = 0; i < pdev->dma_channels; i++) {
  780. phy = &pdev->phy[i];
  781. irq = platform_get_irq(op, i);
  782. devm_free_irq(&op->dev, irq, phy);
  783. }
  784. }
  785. dma_async_device_unregister(&pdev->device);
  786. return 0;
  787. }
  788. static int mmp_pdma_chan_init(struct mmp_pdma_device *pdev, int idx, int irq)
  789. {
  790. struct mmp_pdma_phy *phy = &pdev->phy[idx];
  791. struct mmp_pdma_chan *chan;
  792. int ret;
  793. chan = devm_kzalloc(pdev->dev, sizeof(*chan), GFP_KERNEL);
  794. if (chan == NULL)
  795. return -ENOMEM;
  796. phy->idx = idx;
  797. phy->base = pdev->base;
  798. if (irq) {
  799. ret = devm_request_irq(pdev->dev, irq, mmp_pdma_chan_handler,
  800. IRQF_SHARED, "pdma", phy);
  801. if (ret) {
  802. dev_err(pdev->dev, "channel request irq fail!\n");
  803. return ret;
  804. }
  805. }
  806. spin_lock_init(&chan->desc_lock);
  807. chan->dev = pdev->dev;
  808. chan->chan.device = &pdev->device;
  809. tasklet_setup(&chan->tasklet, dma_do_tasklet);
  810. INIT_LIST_HEAD(&chan->chain_pending);
  811. INIT_LIST_HEAD(&chan->chain_running);
  812. /* register virt channel to dma engine */
  813. list_add_tail(&chan->chan.device_node, &pdev->device.channels);
  814. return 0;
  815. }
  816. static const struct of_device_id mmp_pdma_dt_ids[] = {
  817. { .compatible = "marvell,pdma-1.0", },
  818. {}
  819. };
  820. MODULE_DEVICE_TABLE(of, mmp_pdma_dt_ids);
  821. static struct dma_chan *mmp_pdma_dma_xlate(struct of_phandle_args *dma_spec,
  822. struct of_dma *ofdma)
  823. {
  824. struct mmp_pdma_device *d = ofdma->of_dma_data;
  825. struct dma_chan *chan;
  826. chan = dma_get_any_slave_channel(&d->device);
  827. if (!chan)
  828. return NULL;
  829. to_mmp_pdma_chan(chan)->drcmr = dma_spec->args[0];
  830. return chan;
  831. }
  832. static int mmp_pdma_probe(struct platform_device *op)
  833. {
  834. struct mmp_pdma_device *pdev;
  835. const struct of_device_id *of_id;
  836. struct mmp_dma_platdata *pdata = dev_get_platdata(&op->dev);
  837. struct resource *iores;
  838. int i, ret, irq = 0;
  839. int dma_channels = 0, irq_num = 0;
  840. const enum dma_slave_buswidth widths =
  841. DMA_SLAVE_BUSWIDTH_1_BYTE | DMA_SLAVE_BUSWIDTH_2_BYTES |
  842. DMA_SLAVE_BUSWIDTH_4_BYTES;
  843. pdev = devm_kzalloc(&op->dev, sizeof(*pdev), GFP_KERNEL);
  844. if (!pdev)
  845. return -ENOMEM;
  846. pdev->dev = &op->dev;
  847. spin_lock_init(&pdev->phy_lock);
  848. iores = platform_get_resource(op, IORESOURCE_MEM, 0);
  849. pdev->base = devm_ioremap_resource(pdev->dev, iores);
  850. if (IS_ERR(pdev->base))
  851. return PTR_ERR(pdev->base);
  852. of_id = of_match_device(mmp_pdma_dt_ids, pdev->dev);
  853. if (of_id)
  854. of_property_read_u32(pdev->dev->of_node, "#dma-channels",
  855. &dma_channels);
  856. else if (pdata && pdata->dma_channels)
  857. dma_channels = pdata->dma_channels;
  858. else
  859. dma_channels = 32; /* default 32 channel */
  860. pdev->dma_channels = dma_channels;
  861. for (i = 0; i < dma_channels; i++) {
  862. if (platform_get_irq_optional(op, i) > 0)
  863. irq_num++;
  864. }
  865. pdev->phy = devm_kcalloc(pdev->dev, dma_channels, sizeof(*pdev->phy),
  866. GFP_KERNEL);
  867. if (pdev->phy == NULL)
  868. return -ENOMEM;
  869. INIT_LIST_HEAD(&pdev->device.channels);
  870. if (irq_num != dma_channels) {
  871. /* all chan share one irq, demux inside */
  872. irq = platform_get_irq(op, 0);
  873. ret = devm_request_irq(pdev->dev, irq, mmp_pdma_int_handler,
  874. IRQF_SHARED, "pdma", pdev);
  875. if (ret)
  876. return ret;
  877. }
  878. for (i = 0; i < dma_channels; i++) {
  879. irq = (irq_num != dma_channels) ? 0 : platform_get_irq(op, i);
  880. ret = mmp_pdma_chan_init(pdev, i, irq);
  881. if (ret)
  882. return ret;
  883. }
  884. dma_cap_set(DMA_SLAVE, pdev->device.cap_mask);
  885. dma_cap_set(DMA_MEMCPY, pdev->device.cap_mask);
  886. dma_cap_set(DMA_CYCLIC, pdev->device.cap_mask);
  887. dma_cap_set(DMA_PRIVATE, pdev->device.cap_mask);
  888. pdev->device.dev = &op->dev;
  889. pdev->device.device_alloc_chan_resources = mmp_pdma_alloc_chan_resources;
  890. pdev->device.device_free_chan_resources = mmp_pdma_free_chan_resources;
  891. pdev->device.device_tx_status = mmp_pdma_tx_status;
  892. pdev->device.device_prep_dma_memcpy = mmp_pdma_prep_memcpy;
  893. pdev->device.device_prep_slave_sg = mmp_pdma_prep_slave_sg;
  894. pdev->device.device_prep_dma_cyclic = mmp_pdma_prep_dma_cyclic;
  895. pdev->device.device_issue_pending = mmp_pdma_issue_pending;
  896. pdev->device.device_config = mmp_pdma_config;
  897. pdev->device.device_terminate_all = mmp_pdma_terminate_all;
  898. pdev->device.copy_align = DMAENGINE_ALIGN_8_BYTES;
  899. pdev->device.src_addr_widths = widths;
  900. pdev->device.dst_addr_widths = widths;
  901. pdev->device.directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM);
  902. pdev->device.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
  903. if (pdev->dev->coherent_dma_mask)
  904. dma_set_mask(pdev->dev, pdev->dev->coherent_dma_mask);
  905. else
  906. dma_set_mask(pdev->dev, DMA_BIT_MASK(64));
  907. ret = dma_async_device_register(&pdev->device);
  908. if (ret) {
  909. dev_err(pdev->device.dev, "unable to register\n");
  910. return ret;
  911. }
  912. if (op->dev.of_node) {
  913. /* Device-tree DMA controller registration */
  914. ret = of_dma_controller_register(op->dev.of_node,
  915. mmp_pdma_dma_xlate, pdev);
  916. if (ret < 0) {
  917. dev_err(&op->dev, "of_dma_controller_register failed\n");
  918. return ret;
  919. }
  920. }
  921. platform_set_drvdata(op, pdev);
  922. dev_info(pdev->device.dev, "initialized %d channels\n", dma_channels);
  923. return 0;
  924. }
  925. static const struct platform_device_id mmp_pdma_id_table[] = {
  926. { "mmp-pdma", },
  927. { },
  928. };
  929. static struct platform_driver mmp_pdma_driver = {
  930. .driver = {
  931. .name = "mmp-pdma",
  932. .of_match_table = mmp_pdma_dt_ids,
  933. },
  934. .id_table = mmp_pdma_id_table,
  935. .probe = mmp_pdma_probe,
  936. .remove = mmp_pdma_remove,
  937. };
  938. bool mmp_pdma_filter_fn(struct dma_chan *chan, void *param)
  939. {
  940. struct mmp_pdma_chan *c = to_mmp_pdma_chan(chan);
  941. if (chan->device->dev->driver != &mmp_pdma_driver.driver)
  942. return false;
  943. c->drcmr = *(unsigned int *)param;
  944. return true;
  945. }
  946. EXPORT_SYMBOL_GPL(mmp_pdma_filter_fn);
  947. module_platform_driver(mmp_pdma_driver);
  948. MODULE_DESCRIPTION("MARVELL MMP Peripheral DMA Driver");
  949. MODULE_AUTHOR("Marvell International Ltd.");
  950. MODULE_LICENSE("GPL v2");