pxa_dma.c 39 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright 2015 Robert Jarzmik <robert.jarzmik@free.fr>
  4. */
  5. #include <linux/err.h>
  6. #include <linux/module.h>
  7. #include <linux/init.h>
  8. #include <linux/types.h>
  9. #include <linux/interrupt.h>
  10. #include <linux/dma-mapping.h>
  11. #include <linux/slab.h>
  12. #include <linux/dmaengine.h>
  13. #include <linux/platform_device.h>
  14. #include <linux/device.h>
  15. #include <linux/platform_data/mmp_dma.h>
  16. #include <linux/dmapool.h>
  17. #include <linux/of_device.h>
  18. #include <linux/of_dma.h>
  19. #include <linux/of.h>
  20. #include <linux/wait.h>
  21. #include <linux/dma/pxa-dma.h>
  22. #include "dmaengine.h"
  23. #include "virt-dma.h"
  24. #define DCSR(n) (0x0000 + ((n) << 2))
  25. #define DALGN(n) 0x00a0
  26. #define DINT 0x00f0
  27. #define DDADR(n) (0x0200 + ((n) << 4))
  28. #define DSADR(n) (0x0204 + ((n) << 4))
  29. #define DTADR(n) (0x0208 + ((n) << 4))
  30. #define DCMD(n) (0x020c + ((n) << 4))
  31. #define PXA_DCSR_RUN BIT(31) /* Run Bit (read / write) */
  32. #define PXA_DCSR_NODESC BIT(30) /* No-Descriptor Fetch (read / write) */
  33. #define PXA_DCSR_STOPIRQEN BIT(29) /* Stop Interrupt Enable (R/W) */
  34. #define PXA_DCSR_REQPEND BIT(8) /* Request Pending (read-only) */
  35. #define PXA_DCSR_STOPSTATE BIT(3) /* Stop State (read-only) */
  36. #define PXA_DCSR_ENDINTR BIT(2) /* End Interrupt (read / write) */
  37. #define PXA_DCSR_STARTINTR BIT(1) /* Start Interrupt (read / write) */
  38. #define PXA_DCSR_BUSERR BIT(0) /* Bus Error Interrupt (read / write) */
  39. #define PXA_DCSR_EORIRQEN BIT(28) /* End of Receive IRQ Enable (R/W) */
  40. #define PXA_DCSR_EORJMPEN BIT(27) /* Jump to next descriptor on EOR */
  41. #define PXA_DCSR_EORSTOPEN BIT(26) /* STOP on an EOR */
  42. #define PXA_DCSR_SETCMPST BIT(25) /* Set Descriptor Compare Status */
  43. #define PXA_DCSR_CLRCMPST BIT(24) /* Clear Descriptor Compare Status */
  44. #define PXA_DCSR_CMPST BIT(10) /* The Descriptor Compare Status */
  45. #define PXA_DCSR_EORINTR BIT(9) /* The end of Receive */
  46. #define DRCMR_MAPVLD BIT(7) /* Map Valid (read / write) */
  47. #define DRCMR_CHLNUM 0x1f /* mask for Channel Number (read / write) */
  48. #define DDADR_DESCADDR 0xfffffff0 /* Address of next descriptor (mask) */
  49. #define DDADR_STOP BIT(0) /* Stop (read / write) */
  50. #define PXA_DCMD_INCSRCADDR BIT(31) /* Source Address Increment Setting. */
  51. #define PXA_DCMD_INCTRGADDR BIT(30) /* Target Address Increment Setting. */
  52. #define PXA_DCMD_FLOWSRC BIT(29) /* Flow Control by the source. */
  53. #define PXA_DCMD_FLOWTRG BIT(28) /* Flow Control by the target. */
  54. #define PXA_DCMD_STARTIRQEN BIT(22) /* Start Interrupt Enable */
  55. #define PXA_DCMD_ENDIRQEN BIT(21) /* End Interrupt Enable */
  56. #define PXA_DCMD_ENDIAN BIT(18) /* Device Endian-ness. */
  57. #define PXA_DCMD_BURST8 (1 << 16) /* 8 byte burst */
  58. #define PXA_DCMD_BURST16 (2 << 16) /* 16 byte burst */
  59. #define PXA_DCMD_BURST32 (3 << 16) /* 32 byte burst */
  60. #define PXA_DCMD_WIDTH1 (1 << 14) /* 1 byte width */
  61. #define PXA_DCMD_WIDTH2 (2 << 14) /* 2 byte width (HalfWord) */
  62. #define PXA_DCMD_WIDTH4 (3 << 14) /* 4 byte width (Word) */
  63. #define PXA_DCMD_LENGTH 0x01fff /* length mask (max = 8K - 1) */
  64. #define PDMA_ALIGNMENT 3
  65. #define PDMA_MAX_DESC_BYTES (PXA_DCMD_LENGTH & ~((1 << PDMA_ALIGNMENT) - 1))
  66. struct pxad_desc_hw {
  67. u32 ddadr; /* Points to the next descriptor + flags */
  68. u32 dsadr; /* DSADR value for the current transfer */
  69. u32 dtadr; /* DTADR value for the current transfer */
  70. u32 dcmd; /* DCMD value for the current transfer */
  71. } __aligned(16);
  72. struct pxad_desc_sw {
  73. struct virt_dma_desc vd; /* Virtual descriptor */
  74. int nb_desc; /* Number of hw. descriptors */
  75. size_t len; /* Number of bytes xfered */
  76. dma_addr_t first; /* First descriptor's addr */
  77. /* At least one descriptor has an src/dst address not multiple of 8 */
  78. bool misaligned;
  79. bool cyclic;
  80. struct dma_pool *desc_pool; /* Channel's used allocator */
  81. struct pxad_desc_hw *hw_desc[]; /* DMA coherent descriptors */
  82. };
  83. struct pxad_phy {
  84. int idx;
  85. void __iomem *base;
  86. struct pxad_chan *vchan;
  87. };
  88. struct pxad_chan {
  89. struct virt_dma_chan vc; /* Virtual channel */
  90. u32 drcmr; /* Requestor of the channel */
  91. enum pxad_chan_prio prio; /* Required priority of phy */
  92. /*
  93. * At least one desc_sw in submitted or issued transfers on this channel
  94. * has one address such as: addr % 8 != 0. This implies the DALGN
  95. * setting on the phy.
  96. */
  97. bool misaligned;
  98. struct dma_slave_config cfg; /* Runtime config */
  99. /* protected by vc->lock */
  100. struct pxad_phy *phy;
  101. struct dma_pool *desc_pool; /* Descriptors pool */
  102. dma_cookie_t bus_error;
  103. wait_queue_head_t wq_state;
  104. };
  105. struct pxad_device {
  106. struct dma_device slave;
  107. int nr_chans;
  108. int nr_requestors;
  109. void __iomem *base;
  110. struct pxad_phy *phys;
  111. spinlock_t phy_lock; /* Phy association */
  112. #ifdef CONFIG_DEBUG_FS
  113. struct dentry *dbgfs_root;
  114. struct dentry **dbgfs_chan;
  115. #endif
  116. };
  117. #define tx_to_pxad_desc(tx) \
  118. container_of(tx, struct pxad_desc_sw, async_tx)
  119. #define to_pxad_chan(dchan) \
  120. container_of(dchan, struct pxad_chan, vc.chan)
  121. #define to_pxad_dev(dmadev) \
  122. container_of(dmadev, struct pxad_device, slave)
  123. #define to_pxad_sw_desc(_vd) \
  124. container_of((_vd), struct pxad_desc_sw, vd)
  125. #define _phy_readl_relaxed(phy, _reg) \
  126. readl_relaxed((phy)->base + _reg((phy)->idx))
  127. #define phy_readl_relaxed(phy, _reg) \
  128. ({ \
  129. u32 _v; \
  130. _v = readl_relaxed((phy)->base + _reg((phy)->idx)); \
  131. dev_vdbg(&phy->vchan->vc.chan.dev->device, \
  132. "%s(): readl(%s): 0x%08x\n", __func__, #_reg, \
  133. _v); \
  134. _v; \
  135. })
  136. #define phy_writel(phy, val, _reg) \
  137. do { \
  138. writel((val), (phy)->base + _reg((phy)->idx)); \
  139. dev_vdbg(&phy->vchan->vc.chan.dev->device, \
  140. "%s(): writel(0x%08x, %s)\n", \
  141. __func__, (u32)(val), #_reg); \
  142. } while (0)
  143. #define phy_writel_relaxed(phy, val, _reg) \
  144. do { \
  145. writel_relaxed((val), (phy)->base + _reg((phy)->idx)); \
  146. dev_vdbg(&phy->vchan->vc.chan.dev->device, \
  147. "%s(): writel_relaxed(0x%08x, %s)\n", \
  148. __func__, (u32)(val), #_reg); \
  149. } while (0)
  150. static unsigned int pxad_drcmr(unsigned int line)
  151. {
  152. if (line < 64)
  153. return 0x100 + line * 4;
  154. return 0x1000 + line * 4;
  155. }
  156. static bool pxad_filter_fn(struct dma_chan *chan, void *param);
  157. /*
  158. * Debug fs
  159. */
  160. #ifdef CONFIG_DEBUG_FS
  161. #include <linux/debugfs.h>
  162. #include <linux/uaccess.h>
  163. #include <linux/seq_file.h>
  164. static int requester_chan_show(struct seq_file *s, void *p)
  165. {
  166. struct pxad_phy *phy = s->private;
  167. int i;
  168. u32 drcmr;
  169. seq_printf(s, "DMA channel %d requester :\n", phy->idx);
  170. for (i = 0; i < 70; i++) {
  171. drcmr = readl_relaxed(phy->base + pxad_drcmr(i));
  172. if ((drcmr & DRCMR_CHLNUM) == phy->idx)
  173. seq_printf(s, "\tRequester %d (MAPVLD=%d)\n", i,
  174. !!(drcmr & DRCMR_MAPVLD));
  175. }
  176. return 0;
  177. }
  178. static inline int dbg_burst_from_dcmd(u32 dcmd)
  179. {
  180. int burst = (dcmd >> 16) & 0x3;
  181. return burst ? 4 << burst : 0;
  182. }
  183. static int is_phys_valid(unsigned long addr)
  184. {
  185. return pfn_valid(__phys_to_pfn(addr));
  186. }
  187. #define PXA_DCSR_STR(flag) (dcsr & PXA_DCSR_##flag ? #flag" " : "")
  188. #define PXA_DCMD_STR(flag) (dcmd & PXA_DCMD_##flag ? #flag" " : "")
  189. static int descriptors_show(struct seq_file *s, void *p)
  190. {
  191. struct pxad_phy *phy = s->private;
  192. int i, max_show = 20, burst, width;
  193. u32 dcmd;
  194. unsigned long phys_desc, ddadr;
  195. struct pxad_desc_hw *desc;
  196. phys_desc = ddadr = _phy_readl_relaxed(phy, DDADR);
  197. seq_printf(s, "DMA channel %d descriptors :\n", phy->idx);
  198. seq_printf(s, "[%03d] First descriptor unknown\n", 0);
  199. for (i = 1; i < max_show && is_phys_valid(phys_desc); i++) {
  200. desc = phys_to_virt(phys_desc);
  201. dcmd = desc->dcmd;
  202. burst = dbg_burst_from_dcmd(dcmd);
  203. width = (1 << ((dcmd >> 14) & 0x3)) >> 1;
  204. seq_printf(s, "[%03d] Desc at %08lx(virt %p)\n",
  205. i, phys_desc, desc);
  206. seq_printf(s, "\tDDADR = %08x\n", desc->ddadr);
  207. seq_printf(s, "\tDSADR = %08x\n", desc->dsadr);
  208. seq_printf(s, "\tDTADR = %08x\n", desc->dtadr);
  209. seq_printf(s, "\tDCMD = %08x (%s%s%s%s%s%s%sburst=%d width=%d len=%d)\n",
  210. dcmd,
  211. PXA_DCMD_STR(INCSRCADDR), PXA_DCMD_STR(INCTRGADDR),
  212. PXA_DCMD_STR(FLOWSRC), PXA_DCMD_STR(FLOWTRG),
  213. PXA_DCMD_STR(STARTIRQEN), PXA_DCMD_STR(ENDIRQEN),
  214. PXA_DCMD_STR(ENDIAN), burst, width,
  215. dcmd & PXA_DCMD_LENGTH);
  216. phys_desc = desc->ddadr;
  217. }
  218. if (i == max_show)
  219. seq_printf(s, "[%03d] Desc at %08lx ... max display reached\n",
  220. i, phys_desc);
  221. else
  222. seq_printf(s, "[%03d] Desc at %08lx is %s\n",
  223. i, phys_desc, phys_desc == DDADR_STOP ?
  224. "DDADR_STOP" : "invalid");
  225. return 0;
  226. }
  227. static int chan_state_show(struct seq_file *s, void *p)
  228. {
  229. struct pxad_phy *phy = s->private;
  230. u32 dcsr, dcmd;
  231. int burst, width;
  232. static const char * const str_prio[] = {
  233. "high", "normal", "low", "invalid"
  234. };
  235. dcsr = _phy_readl_relaxed(phy, DCSR);
  236. dcmd = _phy_readl_relaxed(phy, DCMD);
  237. burst = dbg_burst_from_dcmd(dcmd);
  238. width = (1 << ((dcmd >> 14) & 0x3)) >> 1;
  239. seq_printf(s, "DMA channel %d\n", phy->idx);
  240. seq_printf(s, "\tPriority : %s\n",
  241. str_prio[(phy->idx & 0xf) / 4]);
  242. seq_printf(s, "\tUnaligned transfer bit: %s\n",
  243. _phy_readl_relaxed(phy, DALGN) & BIT(phy->idx) ?
  244. "yes" : "no");
  245. seq_printf(s, "\tDCSR = %08x (%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s)\n",
  246. dcsr, PXA_DCSR_STR(RUN), PXA_DCSR_STR(NODESC),
  247. PXA_DCSR_STR(STOPIRQEN), PXA_DCSR_STR(EORIRQEN),
  248. PXA_DCSR_STR(EORJMPEN), PXA_DCSR_STR(EORSTOPEN),
  249. PXA_DCSR_STR(SETCMPST), PXA_DCSR_STR(CLRCMPST),
  250. PXA_DCSR_STR(CMPST), PXA_DCSR_STR(EORINTR),
  251. PXA_DCSR_STR(REQPEND), PXA_DCSR_STR(STOPSTATE),
  252. PXA_DCSR_STR(ENDINTR), PXA_DCSR_STR(STARTINTR),
  253. PXA_DCSR_STR(BUSERR));
  254. seq_printf(s, "\tDCMD = %08x (%s%s%s%s%s%s%sburst=%d width=%d len=%d)\n",
  255. dcmd,
  256. PXA_DCMD_STR(INCSRCADDR), PXA_DCMD_STR(INCTRGADDR),
  257. PXA_DCMD_STR(FLOWSRC), PXA_DCMD_STR(FLOWTRG),
  258. PXA_DCMD_STR(STARTIRQEN), PXA_DCMD_STR(ENDIRQEN),
  259. PXA_DCMD_STR(ENDIAN), burst, width, dcmd & PXA_DCMD_LENGTH);
  260. seq_printf(s, "\tDSADR = %08x\n", _phy_readl_relaxed(phy, DSADR));
  261. seq_printf(s, "\tDTADR = %08x\n", _phy_readl_relaxed(phy, DTADR));
  262. seq_printf(s, "\tDDADR = %08x\n", _phy_readl_relaxed(phy, DDADR));
  263. return 0;
  264. }
  265. static int state_show(struct seq_file *s, void *p)
  266. {
  267. struct pxad_device *pdev = s->private;
  268. /* basic device status */
  269. seq_puts(s, "DMA engine status\n");
  270. seq_printf(s, "\tChannel number: %d\n", pdev->nr_chans);
  271. return 0;
  272. }
  273. DEFINE_SHOW_ATTRIBUTE(state);
  274. DEFINE_SHOW_ATTRIBUTE(chan_state);
  275. DEFINE_SHOW_ATTRIBUTE(descriptors);
  276. DEFINE_SHOW_ATTRIBUTE(requester_chan);
  277. static struct dentry *pxad_dbg_alloc_chan(struct pxad_device *pdev,
  278. int ch, struct dentry *chandir)
  279. {
  280. char chan_name[11];
  281. struct dentry *chan;
  282. void *dt;
  283. scnprintf(chan_name, sizeof(chan_name), "%d", ch);
  284. chan = debugfs_create_dir(chan_name, chandir);
  285. dt = (void *)&pdev->phys[ch];
  286. debugfs_create_file("state", 0400, chan, dt, &chan_state_fops);
  287. debugfs_create_file("descriptors", 0400, chan, dt, &descriptors_fops);
  288. debugfs_create_file("requesters", 0400, chan, dt, &requester_chan_fops);
  289. return chan;
  290. }
  291. static void pxad_init_debugfs(struct pxad_device *pdev)
  292. {
  293. int i;
  294. struct dentry *chandir;
  295. pdev->dbgfs_chan =
  296. kmalloc_array(pdev->nr_chans, sizeof(struct dentry *),
  297. GFP_KERNEL);
  298. if (!pdev->dbgfs_chan)
  299. return;
  300. pdev->dbgfs_root = debugfs_create_dir(dev_name(pdev->slave.dev), NULL);
  301. debugfs_create_file("state", 0400, pdev->dbgfs_root, pdev, &state_fops);
  302. chandir = debugfs_create_dir("channels", pdev->dbgfs_root);
  303. for (i = 0; i < pdev->nr_chans; i++)
  304. pdev->dbgfs_chan[i] = pxad_dbg_alloc_chan(pdev, i, chandir);
  305. }
  306. static void pxad_cleanup_debugfs(struct pxad_device *pdev)
  307. {
  308. debugfs_remove_recursive(pdev->dbgfs_root);
  309. }
  310. #else
  311. static inline void pxad_init_debugfs(struct pxad_device *pdev) {}
  312. static inline void pxad_cleanup_debugfs(struct pxad_device *pdev) {}
  313. #endif
  314. static struct pxad_phy *lookup_phy(struct pxad_chan *pchan)
  315. {
  316. int prio, i;
  317. struct pxad_device *pdev = to_pxad_dev(pchan->vc.chan.device);
  318. struct pxad_phy *phy, *found = NULL;
  319. unsigned long flags;
  320. /*
  321. * dma channel priorities
  322. * ch 0 - 3, 16 - 19 <--> (0)
  323. * ch 4 - 7, 20 - 23 <--> (1)
  324. * ch 8 - 11, 24 - 27 <--> (2)
  325. * ch 12 - 15, 28 - 31 <--> (3)
  326. */
  327. spin_lock_irqsave(&pdev->phy_lock, flags);
  328. for (prio = pchan->prio; prio >= PXAD_PRIO_HIGHEST; prio--) {
  329. for (i = 0; i < pdev->nr_chans; i++) {
  330. if (prio != (i & 0xf) >> 2)
  331. continue;
  332. phy = &pdev->phys[i];
  333. if (!phy->vchan) {
  334. phy->vchan = pchan;
  335. found = phy;
  336. goto out_unlock;
  337. }
  338. }
  339. }
  340. out_unlock:
  341. spin_unlock_irqrestore(&pdev->phy_lock, flags);
  342. dev_dbg(&pchan->vc.chan.dev->device,
  343. "%s(): phy=%p(%d)\n", __func__, found,
  344. found ? found->idx : -1);
  345. return found;
  346. }
  347. static void pxad_free_phy(struct pxad_chan *chan)
  348. {
  349. struct pxad_device *pdev = to_pxad_dev(chan->vc.chan.device);
  350. unsigned long flags;
  351. u32 reg;
  352. dev_dbg(&chan->vc.chan.dev->device,
  353. "%s(): freeing\n", __func__);
  354. if (!chan->phy)
  355. return;
  356. /* clear the channel mapping in DRCMR */
  357. if (chan->drcmr <= pdev->nr_requestors) {
  358. reg = pxad_drcmr(chan->drcmr);
  359. writel_relaxed(0, chan->phy->base + reg);
  360. }
  361. spin_lock_irqsave(&pdev->phy_lock, flags);
  362. chan->phy->vchan = NULL;
  363. chan->phy = NULL;
  364. spin_unlock_irqrestore(&pdev->phy_lock, flags);
  365. }
  366. static bool is_chan_running(struct pxad_chan *chan)
  367. {
  368. u32 dcsr;
  369. struct pxad_phy *phy = chan->phy;
  370. if (!phy)
  371. return false;
  372. dcsr = phy_readl_relaxed(phy, DCSR);
  373. return dcsr & PXA_DCSR_RUN;
  374. }
  375. static bool is_running_chan_misaligned(struct pxad_chan *chan)
  376. {
  377. u32 dalgn;
  378. BUG_ON(!chan->phy);
  379. dalgn = phy_readl_relaxed(chan->phy, DALGN);
  380. return dalgn & (BIT(chan->phy->idx));
  381. }
  382. static void phy_enable(struct pxad_phy *phy, bool misaligned)
  383. {
  384. struct pxad_device *pdev;
  385. u32 reg, dalgn;
  386. if (!phy->vchan)
  387. return;
  388. dev_dbg(&phy->vchan->vc.chan.dev->device,
  389. "%s(); phy=%p(%d) misaligned=%d\n", __func__,
  390. phy, phy->idx, misaligned);
  391. pdev = to_pxad_dev(phy->vchan->vc.chan.device);
  392. if (phy->vchan->drcmr <= pdev->nr_requestors) {
  393. reg = pxad_drcmr(phy->vchan->drcmr);
  394. writel_relaxed(DRCMR_MAPVLD | phy->idx, phy->base + reg);
  395. }
  396. dalgn = phy_readl_relaxed(phy, DALGN);
  397. if (misaligned)
  398. dalgn |= BIT(phy->idx);
  399. else
  400. dalgn &= ~BIT(phy->idx);
  401. phy_writel_relaxed(phy, dalgn, DALGN);
  402. phy_writel(phy, PXA_DCSR_STOPIRQEN | PXA_DCSR_ENDINTR |
  403. PXA_DCSR_BUSERR | PXA_DCSR_RUN, DCSR);
  404. }
  405. static void phy_disable(struct pxad_phy *phy)
  406. {
  407. u32 dcsr;
  408. if (!phy)
  409. return;
  410. dcsr = phy_readl_relaxed(phy, DCSR);
  411. dev_dbg(&phy->vchan->vc.chan.dev->device,
  412. "%s(): phy=%p(%d)\n", __func__, phy, phy->idx);
  413. phy_writel(phy, dcsr & ~PXA_DCSR_RUN & ~PXA_DCSR_STOPIRQEN, DCSR);
  414. }
  415. static void pxad_launch_chan(struct pxad_chan *chan,
  416. struct pxad_desc_sw *desc)
  417. {
  418. dev_dbg(&chan->vc.chan.dev->device,
  419. "%s(): desc=%p\n", __func__, desc);
  420. if (!chan->phy) {
  421. chan->phy = lookup_phy(chan);
  422. if (!chan->phy) {
  423. dev_dbg(&chan->vc.chan.dev->device,
  424. "%s(): no free dma channel\n", __func__);
  425. return;
  426. }
  427. }
  428. chan->bus_error = 0;
  429. /*
  430. * Program the descriptor's address into the DMA controller,
  431. * then start the DMA transaction
  432. */
  433. phy_writel(chan->phy, desc->first, DDADR);
  434. phy_enable(chan->phy, chan->misaligned);
  435. wake_up(&chan->wq_state);
  436. }
  437. static void set_updater_desc(struct pxad_desc_sw *sw_desc,
  438. unsigned long flags)
  439. {
  440. struct pxad_desc_hw *updater =
  441. sw_desc->hw_desc[sw_desc->nb_desc - 1];
  442. dma_addr_t dma = sw_desc->hw_desc[sw_desc->nb_desc - 2]->ddadr;
  443. updater->ddadr = DDADR_STOP;
  444. updater->dsadr = dma;
  445. updater->dtadr = dma + 8;
  446. updater->dcmd = PXA_DCMD_WIDTH4 | PXA_DCMD_BURST32 |
  447. (PXA_DCMD_LENGTH & sizeof(u32));
  448. if (flags & DMA_PREP_INTERRUPT)
  449. updater->dcmd |= PXA_DCMD_ENDIRQEN;
  450. if (sw_desc->cyclic)
  451. sw_desc->hw_desc[sw_desc->nb_desc - 2]->ddadr = sw_desc->first;
  452. }
  453. static bool is_desc_completed(struct virt_dma_desc *vd)
  454. {
  455. struct pxad_desc_sw *sw_desc = to_pxad_sw_desc(vd);
  456. struct pxad_desc_hw *updater =
  457. sw_desc->hw_desc[sw_desc->nb_desc - 1];
  458. return updater->dtadr != (updater->dsadr + 8);
  459. }
  460. static void pxad_desc_chain(struct virt_dma_desc *vd1,
  461. struct virt_dma_desc *vd2)
  462. {
  463. struct pxad_desc_sw *desc1 = to_pxad_sw_desc(vd1);
  464. struct pxad_desc_sw *desc2 = to_pxad_sw_desc(vd2);
  465. dma_addr_t dma_to_chain;
  466. dma_to_chain = desc2->first;
  467. desc1->hw_desc[desc1->nb_desc - 1]->ddadr = dma_to_chain;
  468. }
  469. static bool pxad_try_hotchain(struct virt_dma_chan *vc,
  470. struct virt_dma_desc *vd)
  471. {
  472. struct virt_dma_desc *vd_last_issued = NULL;
  473. struct pxad_chan *chan = to_pxad_chan(&vc->chan);
  474. /*
  475. * Attempt to hot chain the tx if the phy is still running. This is
  476. * considered successful only if either the channel is still running
  477. * after the chaining, or if the chained transfer is completed after
  478. * having been hot chained.
  479. * A change of alignment is not allowed, and forbids hotchaining.
  480. */
  481. if (is_chan_running(chan)) {
  482. BUG_ON(list_empty(&vc->desc_issued));
  483. if (!is_running_chan_misaligned(chan) &&
  484. to_pxad_sw_desc(vd)->misaligned)
  485. return false;
  486. vd_last_issued = list_entry(vc->desc_issued.prev,
  487. struct virt_dma_desc, node);
  488. pxad_desc_chain(vd_last_issued, vd);
  489. if (is_chan_running(chan) || is_desc_completed(vd))
  490. return true;
  491. }
  492. return false;
  493. }
  494. static unsigned int clear_chan_irq(struct pxad_phy *phy)
  495. {
  496. u32 dcsr;
  497. u32 dint = readl(phy->base + DINT);
  498. if (!(dint & BIT(phy->idx)))
  499. return PXA_DCSR_RUN;
  500. /* clear irq */
  501. dcsr = phy_readl_relaxed(phy, DCSR);
  502. phy_writel(phy, dcsr, DCSR);
  503. if ((dcsr & PXA_DCSR_BUSERR) && (phy->vchan))
  504. dev_warn(&phy->vchan->vc.chan.dev->device,
  505. "%s(chan=%p): PXA_DCSR_BUSERR\n",
  506. __func__, &phy->vchan);
  507. return dcsr & ~PXA_DCSR_RUN;
  508. }
  509. static irqreturn_t pxad_chan_handler(int irq, void *dev_id)
  510. {
  511. struct pxad_phy *phy = dev_id;
  512. struct pxad_chan *chan = phy->vchan;
  513. struct virt_dma_desc *vd, *tmp;
  514. unsigned int dcsr;
  515. unsigned long flags;
  516. bool vd_completed;
  517. dma_cookie_t last_started = 0;
  518. BUG_ON(!chan);
  519. dcsr = clear_chan_irq(phy);
  520. if (dcsr & PXA_DCSR_RUN)
  521. return IRQ_NONE;
  522. spin_lock_irqsave(&chan->vc.lock, flags);
  523. list_for_each_entry_safe(vd, tmp, &chan->vc.desc_issued, node) {
  524. vd_completed = is_desc_completed(vd);
  525. dev_dbg(&chan->vc.chan.dev->device,
  526. "%s(): checking txd %p[%x]: completed=%d dcsr=0x%x\n",
  527. __func__, vd, vd->tx.cookie, vd_completed,
  528. dcsr);
  529. last_started = vd->tx.cookie;
  530. if (to_pxad_sw_desc(vd)->cyclic) {
  531. vchan_cyclic_callback(vd);
  532. break;
  533. }
  534. if (vd_completed) {
  535. list_del(&vd->node);
  536. vchan_cookie_complete(vd);
  537. } else {
  538. break;
  539. }
  540. }
  541. if (dcsr & PXA_DCSR_BUSERR) {
  542. chan->bus_error = last_started;
  543. phy_disable(phy);
  544. }
  545. if (!chan->bus_error && dcsr & PXA_DCSR_STOPSTATE) {
  546. dev_dbg(&chan->vc.chan.dev->device,
  547. "%s(): channel stopped, submitted_empty=%d issued_empty=%d",
  548. __func__,
  549. list_empty(&chan->vc.desc_submitted),
  550. list_empty(&chan->vc.desc_issued));
  551. phy_writel_relaxed(phy, dcsr & ~PXA_DCSR_STOPIRQEN, DCSR);
  552. if (list_empty(&chan->vc.desc_issued)) {
  553. chan->misaligned =
  554. !list_empty(&chan->vc.desc_submitted);
  555. } else {
  556. vd = list_first_entry(&chan->vc.desc_issued,
  557. struct virt_dma_desc, node);
  558. pxad_launch_chan(chan, to_pxad_sw_desc(vd));
  559. }
  560. }
  561. spin_unlock_irqrestore(&chan->vc.lock, flags);
  562. wake_up(&chan->wq_state);
  563. return IRQ_HANDLED;
  564. }
  565. static irqreturn_t pxad_int_handler(int irq, void *dev_id)
  566. {
  567. struct pxad_device *pdev = dev_id;
  568. struct pxad_phy *phy;
  569. u32 dint = readl(pdev->base + DINT);
  570. int i, ret = IRQ_NONE;
  571. while (dint) {
  572. i = __ffs(dint);
  573. dint &= (dint - 1);
  574. phy = &pdev->phys[i];
  575. if (pxad_chan_handler(irq, phy) == IRQ_HANDLED)
  576. ret = IRQ_HANDLED;
  577. }
  578. return ret;
  579. }
  580. static int pxad_alloc_chan_resources(struct dma_chan *dchan)
  581. {
  582. struct pxad_chan *chan = to_pxad_chan(dchan);
  583. struct pxad_device *pdev = to_pxad_dev(chan->vc.chan.device);
  584. if (chan->desc_pool)
  585. return 1;
  586. chan->desc_pool = dma_pool_create(dma_chan_name(dchan),
  587. pdev->slave.dev,
  588. sizeof(struct pxad_desc_hw),
  589. __alignof__(struct pxad_desc_hw),
  590. 0);
  591. if (!chan->desc_pool) {
  592. dev_err(&chan->vc.chan.dev->device,
  593. "%s(): unable to allocate descriptor pool\n",
  594. __func__);
  595. return -ENOMEM;
  596. }
  597. return 1;
  598. }
  599. static void pxad_free_chan_resources(struct dma_chan *dchan)
  600. {
  601. struct pxad_chan *chan = to_pxad_chan(dchan);
  602. vchan_free_chan_resources(&chan->vc);
  603. dma_pool_destroy(chan->desc_pool);
  604. chan->desc_pool = NULL;
  605. chan->drcmr = U32_MAX;
  606. chan->prio = PXAD_PRIO_LOWEST;
  607. }
  608. static void pxad_free_desc(struct virt_dma_desc *vd)
  609. {
  610. int i;
  611. dma_addr_t dma;
  612. struct pxad_desc_sw *sw_desc = to_pxad_sw_desc(vd);
  613. BUG_ON(sw_desc->nb_desc == 0);
  614. for (i = sw_desc->nb_desc - 1; i >= 0; i--) {
  615. if (i > 0)
  616. dma = sw_desc->hw_desc[i - 1]->ddadr;
  617. else
  618. dma = sw_desc->first;
  619. dma_pool_free(sw_desc->desc_pool,
  620. sw_desc->hw_desc[i], dma);
  621. }
  622. sw_desc->nb_desc = 0;
  623. kfree(sw_desc);
  624. }
  625. static struct pxad_desc_sw *
  626. pxad_alloc_desc(struct pxad_chan *chan, unsigned int nb_hw_desc)
  627. {
  628. struct pxad_desc_sw *sw_desc;
  629. dma_addr_t dma;
  630. int i;
  631. sw_desc = kzalloc(sizeof(*sw_desc) +
  632. nb_hw_desc * sizeof(struct pxad_desc_hw *),
  633. GFP_NOWAIT);
  634. if (!sw_desc)
  635. return NULL;
  636. sw_desc->desc_pool = chan->desc_pool;
  637. for (i = 0; i < nb_hw_desc; i++) {
  638. sw_desc->hw_desc[i] = dma_pool_alloc(sw_desc->desc_pool,
  639. GFP_NOWAIT, &dma);
  640. if (!sw_desc->hw_desc[i]) {
  641. dev_err(&chan->vc.chan.dev->device,
  642. "%s(): Couldn't allocate the %dth hw_desc from dma_pool %p\n",
  643. __func__, i, sw_desc->desc_pool);
  644. goto err;
  645. }
  646. if (i == 0)
  647. sw_desc->first = dma;
  648. else
  649. sw_desc->hw_desc[i - 1]->ddadr = dma;
  650. sw_desc->nb_desc++;
  651. }
  652. return sw_desc;
  653. err:
  654. pxad_free_desc(&sw_desc->vd);
  655. return NULL;
  656. }
  657. static dma_cookie_t pxad_tx_submit(struct dma_async_tx_descriptor *tx)
  658. {
  659. struct virt_dma_chan *vc = to_virt_chan(tx->chan);
  660. struct pxad_chan *chan = to_pxad_chan(&vc->chan);
  661. struct virt_dma_desc *vd_chained = NULL,
  662. *vd = container_of(tx, struct virt_dma_desc, tx);
  663. dma_cookie_t cookie;
  664. unsigned long flags;
  665. set_updater_desc(to_pxad_sw_desc(vd), tx->flags);
  666. spin_lock_irqsave(&vc->lock, flags);
  667. cookie = dma_cookie_assign(tx);
  668. if (list_empty(&vc->desc_submitted) && pxad_try_hotchain(vc, vd)) {
  669. list_move_tail(&vd->node, &vc->desc_issued);
  670. dev_dbg(&chan->vc.chan.dev->device,
  671. "%s(): txd %p[%x]: submitted (hot linked)\n",
  672. __func__, vd, cookie);
  673. goto out;
  674. }
  675. /*
  676. * Fallback to placing the tx in the submitted queue
  677. */
  678. if (!list_empty(&vc->desc_submitted)) {
  679. vd_chained = list_entry(vc->desc_submitted.prev,
  680. struct virt_dma_desc, node);
  681. /*
  682. * Only chain the descriptors if no new misalignment is
  683. * introduced. If a new misalignment is chained, let the channel
  684. * stop, and be relaunched in misalign mode from the irq
  685. * handler.
  686. */
  687. if (chan->misaligned || !to_pxad_sw_desc(vd)->misaligned)
  688. pxad_desc_chain(vd_chained, vd);
  689. else
  690. vd_chained = NULL;
  691. }
  692. dev_dbg(&chan->vc.chan.dev->device,
  693. "%s(): txd %p[%x]: submitted (%s linked)\n",
  694. __func__, vd, cookie, vd_chained ? "cold" : "not");
  695. list_move_tail(&vd->node, &vc->desc_submitted);
  696. chan->misaligned |= to_pxad_sw_desc(vd)->misaligned;
  697. out:
  698. spin_unlock_irqrestore(&vc->lock, flags);
  699. return cookie;
  700. }
  701. static void pxad_issue_pending(struct dma_chan *dchan)
  702. {
  703. struct pxad_chan *chan = to_pxad_chan(dchan);
  704. struct virt_dma_desc *vd_first;
  705. unsigned long flags;
  706. spin_lock_irqsave(&chan->vc.lock, flags);
  707. if (list_empty(&chan->vc.desc_submitted))
  708. goto out;
  709. vd_first = list_first_entry(&chan->vc.desc_submitted,
  710. struct virt_dma_desc, node);
  711. dev_dbg(&chan->vc.chan.dev->device,
  712. "%s(): txd %p[%x]", __func__, vd_first, vd_first->tx.cookie);
  713. vchan_issue_pending(&chan->vc);
  714. if (!pxad_try_hotchain(&chan->vc, vd_first))
  715. pxad_launch_chan(chan, to_pxad_sw_desc(vd_first));
  716. out:
  717. spin_unlock_irqrestore(&chan->vc.lock, flags);
  718. }
  719. static inline struct dma_async_tx_descriptor *
  720. pxad_tx_prep(struct virt_dma_chan *vc, struct virt_dma_desc *vd,
  721. unsigned long tx_flags)
  722. {
  723. struct dma_async_tx_descriptor *tx;
  724. struct pxad_chan *chan = container_of(vc, struct pxad_chan, vc);
  725. INIT_LIST_HEAD(&vd->node);
  726. tx = vchan_tx_prep(vc, vd, tx_flags);
  727. tx->tx_submit = pxad_tx_submit;
  728. dev_dbg(&chan->vc.chan.dev->device,
  729. "%s(): vc=%p txd=%p[%x] flags=0x%lx\n", __func__,
  730. vc, vd, vd->tx.cookie,
  731. tx_flags);
  732. return tx;
  733. }
  734. static void pxad_get_config(struct pxad_chan *chan,
  735. enum dma_transfer_direction dir,
  736. u32 *dcmd, u32 *dev_src, u32 *dev_dst)
  737. {
  738. u32 maxburst = 0, dev_addr = 0;
  739. enum dma_slave_buswidth width = DMA_SLAVE_BUSWIDTH_UNDEFINED;
  740. struct pxad_device *pdev = to_pxad_dev(chan->vc.chan.device);
  741. *dcmd = 0;
  742. if (dir == DMA_DEV_TO_MEM) {
  743. maxburst = chan->cfg.src_maxburst;
  744. width = chan->cfg.src_addr_width;
  745. dev_addr = chan->cfg.src_addr;
  746. *dev_src = dev_addr;
  747. *dcmd |= PXA_DCMD_INCTRGADDR;
  748. if (chan->drcmr <= pdev->nr_requestors)
  749. *dcmd |= PXA_DCMD_FLOWSRC;
  750. }
  751. if (dir == DMA_MEM_TO_DEV) {
  752. maxburst = chan->cfg.dst_maxburst;
  753. width = chan->cfg.dst_addr_width;
  754. dev_addr = chan->cfg.dst_addr;
  755. *dev_dst = dev_addr;
  756. *dcmd |= PXA_DCMD_INCSRCADDR;
  757. if (chan->drcmr <= pdev->nr_requestors)
  758. *dcmd |= PXA_DCMD_FLOWTRG;
  759. }
  760. if (dir == DMA_MEM_TO_MEM)
  761. *dcmd |= PXA_DCMD_BURST32 | PXA_DCMD_INCTRGADDR |
  762. PXA_DCMD_INCSRCADDR;
  763. dev_dbg(&chan->vc.chan.dev->device,
  764. "%s(): dev_addr=0x%x maxburst=%d width=%d dir=%d\n",
  765. __func__, dev_addr, maxburst, width, dir);
  766. if (width == DMA_SLAVE_BUSWIDTH_1_BYTE)
  767. *dcmd |= PXA_DCMD_WIDTH1;
  768. else if (width == DMA_SLAVE_BUSWIDTH_2_BYTES)
  769. *dcmd |= PXA_DCMD_WIDTH2;
  770. else if (width == DMA_SLAVE_BUSWIDTH_4_BYTES)
  771. *dcmd |= PXA_DCMD_WIDTH4;
  772. if (maxburst == 8)
  773. *dcmd |= PXA_DCMD_BURST8;
  774. else if (maxburst == 16)
  775. *dcmd |= PXA_DCMD_BURST16;
  776. else if (maxburst == 32)
  777. *dcmd |= PXA_DCMD_BURST32;
  778. }
  779. static struct dma_async_tx_descriptor *
  780. pxad_prep_memcpy(struct dma_chan *dchan,
  781. dma_addr_t dma_dst, dma_addr_t dma_src,
  782. size_t len, unsigned long flags)
  783. {
  784. struct pxad_chan *chan = to_pxad_chan(dchan);
  785. struct pxad_desc_sw *sw_desc;
  786. struct pxad_desc_hw *hw_desc;
  787. u32 dcmd;
  788. unsigned int i, nb_desc = 0;
  789. size_t copy;
  790. if (!dchan || !len)
  791. return NULL;
  792. dev_dbg(&chan->vc.chan.dev->device,
  793. "%s(): dma_dst=0x%lx dma_src=0x%lx len=%zu flags=%lx\n",
  794. __func__, (unsigned long)dma_dst, (unsigned long)dma_src,
  795. len, flags);
  796. pxad_get_config(chan, DMA_MEM_TO_MEM, &dcmd, NULL, NULL);
  797. nb_desc = DIV_ROUND_UP(len, PDMA_MAX_DESC_BYTES);
  798. sw_desc = pxad_alloc_desc(chan, nb_desc + 1);
  799. if (!sw_desc)
  800. return NULL;
  801. sw_desc->len = len;
  802. if (!IS_ALIGNED(dma_src, 1 << PDMA_ALIGNMENT) ||
  803. !IS_ALIGNED(dma_dst, 1 << PDMA_ALIGNMENT))
  804. sw_desc->misaligned = true;
  805. i = 0;
  806. do {
  807. hw_desc = sw_desc->hw_desc[i++];
  808. copy = min_t(size_t, len, PDMA_MAX_DESC_BYTES);
  809. hw_desc->dcmd = dcmd | (PXA_DCMD_LENGTH & copy);
  810. hw_desc->dsadr = dma_src;
  811. hw_desc->dtadr = dma_dst;
  812. len -= copy;
  813. dma_src += copy;
  814. dma_dst += copy;
  815. } while (len);
  816. set_updater_desc(sw_desc, flags);
  817. return pxad_tx_prep(&chan->vc, &sw_desc->vd, flags);
  818. }
  819. static struct dma_async_tx_descriptor *
  820. pxad_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
  821. unsigned int sg_len, enum dma_transfer_direction dir,
  822. unsigned long flags, void *context)
  823. {
  824. struct pxad_chan *chan = to_pxad_chan(dchan);
  825. struct pxad_desc_sw *sw_desc;
  826. size_t len, avail;
  827. struct scatterlist *sg;
  828. dma_addr_t dma;
  829. u32 dcmd, dsadr = 0, dtadr = 0;
  830. unsigned int nb_desc = 0, i, j = 0;
  831. if ((sgl == NULL) || (sg_len == 0))
  832. return NULL;
  833. pxad_get_config(chan, dir, &dcmd, &dsadr, &dtadr);
  834. dev_dbg(&chan->vc.chan.dev->device,
  835. "%s(): dir=%d flags=%lx\n", __func__, dir, flags);
  836. for_each_sg(sgl, sg, sg_len, i)
  837. nb_desc += DIV_ROUND_UP(sg_dma_len(sg), PDMA_MAX_DESC_BYTES);
  838. sw_desc = pxad_alloc_desc(chan, nb_desc + 1);
  839. if (!sw_desc)
  840. return NULL;
  841. for_each_sg(sgl, sg, sg_len, i) {
  842. dma = sg_dma_address(sg);
  843. avail = sg_dma_len(sg);
  844. sw_desc->len += avail;
  845. do {
  846. len = min_t(size_t, avail, PDMA_MAX_DESC_BYTES);
  847. if (dma & 0x7)
  848. sw_desc->misaligned = true;
  849. sw_desc->hw_desc[j]->dcmd =
  850. dcmd | (PXA_DCMD_LENGTH & len);
  851. sw_desc->hw_desc[j]->dsadr = dsadr ? dsadr : dma;
  852. sw_desc->hw_desc[j++]->dtadr = dtadr ? dtadr : dma;
  853. dma += len;
  854. avail -= len;
  855. } while (avail);
  856. }
  857. set_updater_desc(sw_desc, flags);
  858. return pxad_tx_prep(&chan->vc, &sw_desc->vd, flags);
  859. }
  860. static struct dma_async_tx_descriptor *
  861. pxad_prep_dma_cyclic(struct dma_chan *dchan,
  862. dma_addr_t buf_addr, size_t len, size_t period_len,
  863. enum dma_transfer_direction dir, unsigned long flags)
  864. {
  865. struct pxad_chan *chan = to_pxad_chan(dchan);
  866. struct pxad_desc_sw *sw_desc;
  867. struct pxad_desc_hw **phw_desc;
  868. dma_addr_t dma;
  869. u32 dcmd, dsadr = 0, dtadr = 0;
  870. unsigned int nb_desc = 0;
  871. if (!dchan || !len || !period_len)
  872. return NULL;
  873. if ((dir != DMA_DEV_TO_MEM) && (dir != DMA_MEM_TO_DEV)) {
  874. dev_err(&chan->vc.chan.dev->device,
  875. "Unsupported direction for cyclic DMA\n");
  876. return NULL;
  877. }
  878. /* the buffer length must be a multiple of period_len */
  879. if (len % period_len != 0 || period_len > PDMA_MAX_DESC_BYTES ||
  880. !IS_ALIGNED(period_len, 1 << PDMA_ALIGNMENT))
  881. return NULL;
  882. pxad_get_config(chan, dir, &dcmd, &dsadr, &dtadr);
  883. dcmd |= PXA_DCMD_ENDIRQEN | (PXA_DCMD_LENGTH & period_len);
  884. dev_dbg(&chan->vc.chan.dev->device,
  885. "%s(): buf_addr=0x%lx len=%zu period=%zu dir=%d flags=%lx\n",
  886. __func__, (unsigned long)buf_addr, len, period_len, dir, flags);
  887. nb_desc = DIV_ROUND_UP(period_len, PDMA_MAX_DESC_BYTES);
  888. nb_desc *= DIV_ROUND_UP(len, period_len);
  889. sw_desc = pxad_alloc_desc(chan, nb_desc + 1);
  890. if (!sw_desc)
  891. return NULL;
  892. sw_desc->cyclic = true;
  893. sw_desc->len = len;
  894. phw_desc = sw_desc->hw_desc;
  895. dma = buf_addr;
  896. do {
  897. phw_desc[0]->dsadr = dsadr ? dsadr : dma;
  898. phw_desc[0]->dtadr = dtadr ? dtadr : dma;
  899. phw_desc[0]->dcmd = dcmd;
  900. phw_desc++;
  901. dma += period_len;
  902. len -= period_len;
  903. } while (len);
  904. set_updater_desc(sw_desc, flags);
  905. return pxad_tx_prep(&chan->vc, &sw_desc->vd, flags);
  906. }
  907. static int pxad_config(struct dma_chan *dchan,
  908. struct dma_slave_config *cfg)
  909. {
  910. struct pxad_chan *chan = to_pxad_chan(dchan);
  911. if (!dchan)
  912. return -EINVAL;
  913. chan->cfg = *cfg;
  914. return 0;
  915. }
  916. static int pxad_terminate_all(struct dma_chan *dchan)
  917. {
  918. struct pxad_chan *chan = to_pxad_chan(dchan);
  919. struct pxad_device *pdev = to_pxad_dev(chan->vc.chan.device);
  920. struct virt_dma_desc *vd = NULL;
  921. unsigned long flags;
  922. struct pxad_phy *phy;
  923. LIST_HEAD(head);
  924. dev_dbg(&chan->vc.chan.dev->device,
  925. "%s(): vchan %p: terminate all\n", __func__, &chan->vc);
  926. spin_lock_irqsave(&chan->vc.lock, flags);
  927. vchan_get_all_descriptors(&chan->vc, &head);
  928. list_for_each_entry(vd, &head, node) {
  929. dev_dbg(&chan->vc.chan.dev->device,
  930. "%s(): cancelling txd %p[%x] (completed=%d)", __func__,
  931. vd, vd->tx.cookie, is_desc_completed(vd));
  932. }
  933. phy = chan->phy;
  934. if (phy) {
  935. phy_disable(chan->phy);
  936. pxad_free_phy(chan);
  937. chan->phy = NULL;
  938. spin_lock(&pdev->phy_lock);
  939. phy->vchan = NULL;
  940. spin_unlock(&pdev->phy_lock);
  941. }
  942. spin_unlock_irqrestore(&chan->vc.lock, flags);
  943. vchan_dma_desc_free_list(&chan->vc, &head);
  944. return 0;
  945. }
  946. static unsigned int pxad_residue(struct pxad_chan *chan,
  947. dma_cookie_t cookie)
  948. {
  949. struct virt_dma_desc *vd = NULL;
  950. struct pxad_desc_sw *sw_desc = NULL;
  951. struct pxad_desc_hw *hw_desc = NULL;
  952. u32 curr, start, len, end, residue = 0;
  953. unsigned long flags;
  954. bool passed = false;
  955. int i;
  956. /*
  957. * If the channel does not have a phy pointer anymore, it has already
  958. * been completed. Therefore, its residue is 0.
  959. */
  960. if (!chan->phy)
  961. return 0;
  962. spin_lock_irqsave(&chan->vc.lock, flags);
  963. vd = vchan_find_desc(&chan->vc, cookie);
  964. if (!vd)
  965. goto out;
  966. sw_desc = to_pxad_sw_desc(vd);
  967. if (sw_desc->hw_desc[0]->dcmd & PXA_DCMD_INCSRCADDR)
  968. curr = phy_readl_relaxed(chan->phy, DSADR);
  969. else
  970. curr = phy_readl_relaxed(chan->phy, DTADR);
  971. /*
  972. * curr has to be actually read before checking descriptor
  973. * completion, so that a curr inside a status updater
  974. * descriptor implies the following test returns true, and
  975. * preventing reordering of curr load and the test.
  976. */
  977. rmb();
  978. if (is_desc_completed(vd))
  979. goto out;
  980. for (i = 0; i < sw_desc->nb_desc - 1; i++) {
  981. hw_desc = sw_desc->hw_desc[i];
  982. if (sw_desc->hw_desc[0]->dcmd & PXA_DCMD_INCSRCADDR)
  983. start = hw_desc->dsadr;
  984. else
  985. start = hw_desc->dtadr;
  986. len = hw_desc->dcmd & PXA_DCMD_LENGTH;
  987. end = start + len;
  988. /*
  989. * 'passed' will be latched once we found the descriptor
  990. * which lies inside the boundaries of the curr
  991. * pointer. All descriptors that occur in the list
  992. * _after_ we found that partially handled descriptor
  993. * are still to be processed and are hence added to the
  994. * residual bytes counter.
  995. */
  996. if (passed) {
  997. residue += len;
  998. } else if (curr >= start && curr <= end) {
  999. residue += end - curr;
  1000. passed = true;
  1001. }
  1002. }
  1003. if (!passed)
  1004. residue = sw_desc->len;
  1005. out:
  1006. spin_unlock_irqrestore(&chan->vc.lock, flags);
  1007. dev_dbg(&chan->vc.chan.dev->device,
  1008. "%s(): txd %p[%x] sw_desc=%p: %d\n",
  1009. __func__, vd, cookie, sw_desc, residue);
  1010. return residue;
  1011. }
  1012. static enum dma_status pxad_tx_status(struct dma_chan *dchan,
  1013. dma_cookie_t cookie,
  1014. struct dma_tx_state *txstate)
  1015. {
  1016. struct pxad_chan *chan = to_pxad_chan(dchan);
  1017. enum dma_status ret;
  1018. if (cookie == chan->bus_error)
  1019. return DMA_ERROR;
  1020. ret = dma_cookie_status(dchan, cookie, txstate);
  1021. if (likely(txstate && (ret != DMA_ERROR)))
  1022. dma_set_residue(txstate, pxad_residue(chan, cookie));
  1023. return ret;
  1024. }
  1025. static void pxad_synchronize(struct dma_chan *dchan)
  1026. {
  1027. struct pxad_chan *chan = to_pxad_chan(dchan);
  1028. wait_event(chan->wq_state, !is_chan_running(chan));
  1029. vchan_synchronize(&chan->vc);
  1030. }
  1031. static void pxad_free_channels(struct dma_device *dmadev)
  1032. {
  1033. struct pxad_chan *c, *cn;
  1034. list_for_each_entry_safe(c, cn, &dmadev->channels,
  1035. vc.chan.device_node) {
  1036. list_del(&c->vc.chan.device_node);
  1037. tasklet_kill(&c->vc.task);
  1038. }
  1039. }
  1040. static int pxad_remove(struct platform_device *op)
  1041. {
  1042. struct pxad_device *pdev = platform_get_drvdata(op);
  1043. pxad_cleanup_debugfs(pdev);
  1044. pxad_free_channels(&pdev->slave);
  1045. return 0;
  1046. }
  1047. static int pxad_init_phys(struct platform_device *op,
  1048. struct pxad_device *pdev,
  1049. unsigned int nb_phy_chans)
  1050. {
  1051. int irq0, irq, nr_irq = 0, i, ret;
  1052. struct pxad_phy *phy;
  1053. irq0 = platform_get_irq(op, 0);
  1054. if (irq0 < 0)
  1055. return irq0;
  1056. pdev->phys = devm_kcalloc(&op->dev, nb_phy_chans,
  1057. sizeof(pdev->phys[0]), GFP_KERNEL);
  1058. if (!pdev->phys)
  1059. return -ENOMEM;
  1060. for (i = 0; i < nb_phy_chans; i++)
  1061. if (platform_get_irq(op, i) > 0)
  1062. nr_irq++;
  1063. for (i = 0; i < nb_phy_chans; i++) {
  1064. phy = &pdev->phys[i];
  1065. phy->base = pdev->base;
  1066. phy->idx = i;
  1067. irq = platform_get_irq(op, i);
  1068. if ((nr_irq > 1) && (irq > 0))
  1069. ret = devm_request_irq(&op->dev, irq,
  1070. pxad_chan_handler,
  1071. IRQF_SHARED, "pxa-dma", phy);
  1072. if ((nr_irq == 1) && (i == 0))
  1073. ret = devm_request_irq(&op->dev, irq0,
  1074. pxad_int_handler,
  1075. IRQF_SHARED, "pxa-dma", pdev);
  1076. if (ret) {
  1077. dev_err(pdev->slave.dev,
  1078. "%s(): can't request irq %d:%d\n", __func__,
  1079. irq, ret);
  1080. return ret;
  1081. }
  1082. }
  1083. return 0;
  1084. }
  1085. static const struct of_device_id pxad_dt_ids[] = {
  1086. { .compatible = "marvell,pdma-1.0", },
  1087. {}
  1088. };
  1089. MODULE_DEVICE_TABLE(of, pxad_dt_ids);
  1090. static struct dma_chan *pxad_dma_xlate(struct of_phandle_args *dma_spec,
  1091. struct of_dma *ofdma)
  1092. {
  1093. struct pxad_device *d = ofdma->of_dma_data;
  1094. struct dma_chan *chan;
  1095. chan = dma_get_any_slave_channel(&d->slave);
  1096. if (!chan)
  1097. return NULL;
  1098. to_pxad_chan(chan)->drcmr = dma_spec->args[0];
  1099. to_pxad_chan(chan)->prio = dma_spec->args[1];
  1100. return chan;
  1101. }
  1102. static int pxad_init_dmadev(struct platform_device *op,
  1103. struct pxad_device *pdev,
  1104. unsigned int nr_phy_chans,
  1105. unsigned int nr_requestors)
  1106. {
  1107. int ret;
  1108. unsigned int i;
  1109. struct pxad_chan *c;
  1110. pdev->nr_chans = nr_phy_chans;
  1111. pdev->nr_requestors = nr_requestors;
  1112. INIT_LIST_HEAD(&pdev->slave.channels);
  1113. pdev->slave.device_alloc_chan_resources = pxad_alloc_chan_resources;
  1114. pdev->slave.device_free_chan_resources = pxad_free_chan_resources;
  1115. pdev->slave.device_tx_status = pxad_tx_status;
  1116. pdev->slave.device_issue_pending = pxad_issue_pending;
  1117. pdev->slave.device_config = pxad_config;
  1118. pdev->slave.device_synchronize = pxad_synchronize;
  1119. pdev->slave.device_terminate_all = pxad_terminate_all;
  1120. if (op->dev.coherent_dma_mask)
  1121. dma_set_mask(&op->dev, op->dev.coherent_dma_mask);
  1122. else
  1123. dma_set_mask(&op->dev, DMA_BIT_MASK(32));
  1124. ret = pxad_init_phys(op, pdev, nr_phy_chans);
  1125. if (ret)
  1126. return ret;
  1127. for (i = 0; i < nr_phy_chans; i++) {
  1128. c = devm_kzalloc(&op->dev, sizeof(*c), GFP_KERNEL);
  1129. if (!c)
  1130. return -ENOMEM;
  1131. c->drcmr = U32_MAX;
  1132. c->prio = PXAD_PRIO_LOWEST;
  1133. c->vc.desc_free = pxad_free_desc;
  1134. vchan_init(&c->vc, &pdev->slave);
  1135. init_waitqueue_head(&c->wq_state);
  1136. }
  1137. return dmaenginem_async_device_register(&pdev->slave);
  1138. }
  1139. static int pxad_probe(struct platform_device *op)
  1140. {
  1141. struct pxad_device *pdev;
  1142. const struct of_device_id *of_id;
  1143. const struct dma_slave_map *slave_map = NULL;
  1144. struct mmp_dma_platdata *pdata = dev_get_platdata(&op->dev);
  1145. struct resource *iores;
  1146. int ret, dma_channels = 0, nb_requestors = 0, slave_map_cnt = 0;
  1147. const enum dma_slave_buswidth widths =
  1148. DMA_SLAVE_BUSWIDTH_1_BYTE | DMA_SLAVE_BUSWIDTH_2_BYTES |
  1149. DMA_SLAVE_BUSWIDTH_4_BYTES;
  1150. pdev = devm_kzalloc(&op->dev, sizeof(*pdev), GFP_KERNEL);
  1151. if (!pdev)
  1152. return -ENOMEM;
  1153. spin_lock_init(&pdev->phy_lock);
  1154. iores = platform_get_resource(op, IORESOURCE_MEM, 0);
  1155. pdev->base = devm_ioremap_resource(&op->dev, iores);
  1156. if (IS_ERR(pdev->base))
  1157. return PTR_ERR(pdev->base);
  1158. of_id = of_match_device(pxad_dt_ids, &op->dev);
  1159. if (of_id) {
  1160. of_property_read_u32(op->dev.of_node, "#dma-channels",
  1161. &dma_channels);
  1162. ret = of_property_read_u32(op->dev.of_node, "#dma-requests",
  1163. &nb_requestors);
  1164. if (ret) {
  1165. dev_warn(pdev->slave.dev,
  1166. "#dma-requests set to default 32 as missing in OF: %d",
  1167. ret);
  1168. nb_requestors = 32;
  1169. }
  1170. } else if (pdata && pdata->dma_channels) {
  1171. dma_channels = pdata->dma_channels;
  1172. nb_requestors = pdata->nb_requestors;
  1173. slave_map = pdata->slave_map;
  1174. slave_map_cnt = pdata->slave_map_cnt;
  1175. } else {
  1176. dma_channels = 32; /* default 32 channel */
  1177. }
  1178. dma_cap_set(DMA_SLAVE, pdev->slave.cap_mask);
  1179. dma_cap_set(DMA_MEMCPY, pdev->slave.cap_mask);
  1180. dma_cap_set(DMA_CYCLIC, pdev->slave.cap_mask);
  1181. dma_cap_set(DMA_PRIVATE, pdev->slave.cap_mask);
  1182. pdev->slave.device_prep_dma_memcpy = pxad_prep_memcpy;
  1183. pdev->slave.device_prep_slave_sg = pxad_prep_slave_sg;
  1184. pdev->slave.device_prep_dma_cyclic = pxad_prep_dma_cyclic;
  1185. pdev->slave.filter.map = slave_map;
  1186. pdev->slave.filter.mapcnt = slave_map_cnt;
  1187. pdev->slave.filter.fn = pxad_filter_fn;
  1188. pdev->slave.copy_align = PDMA_ALIGNMENT;
  1189. pdev->slave.src_addr_widths = widths;
  1190. pdev->slave.dst_addr_widths = widths;
  1191. pdev->slave.directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM);
  1192. pdev->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
  1193. pdev->slave.descriptor_reuse = true;
  1194. pdev->slave.dev = &op->dev;
  1195. ret = pxad_init_dmadev(op, pdev, dma_channels, nb_requestors);
  1196. if (ret) {
  1197. dev_err(pdev->slave.dev, "unable to register\n");
  1198. return ret;
  1199. }
  1200. if (op->dev.of_node) {
  1201. /* Device-tree DMA controller registration */
  1202. ret = of_dma_controller_register(op->dev.of_node,
  1203. pxad_dma_xlate, pdev);
  1204. if (ret < 0) {
  1205. dev_err(pdev->slave.dev,
  1206. "of_dma_controller_register failed\n");
  1207. return ret;
  1208. }
  1209. }
  1210. platform_set_drvdata(op, pdev);
  1211. pxad_init_debugfs(pdev);
  1212. dev_info(pdev->slave.dev, "initialized %d channels on %d requestors\n",
  1213. dma_channels, nb_requestors);
  1214. return 0;
  1215. }
  1216. static const struct platform_device_id pxad_id_table[] = {
  1217. { "pxa-dma", },
  1218. { },
  1219. };
  1220. static struct platform_driver pxad_driver = {
  1221. .driver = {
  1222. .name = "pxa-dma",
  1223. .of_match_table = pxad_dt_ids,
  1224. },
  1225. .id_table = pxad_id_table,
  1226. .probe = pxad_probe,
  1227. .remove = pxad_remove,
  1228. };
  1229. static bool pxad_filter_fn(struct dma_chan *chan, void *param)
  1230. {
  1231. struct pxad_chan *c = to_pxad_chan(chan);
  1232. struct pxad_param *p = param;
  1233. if (chan->device->dev->driver != &pxad_driver.driver)
  1234. return false;
  1235. c->drcmr = p->drcmr;
  1236. c->prio = p->prio;
  1237. return true;
  1238. }
  1239. module_platform_driver(pxad_driver);
  1240. MODULE_DESCRIPTION("Marvell PXA Peripheral DMA Driver");
  1241. MODULE_AUTHOR("Robert Jarzmik <robert.jarzmik@free.fr>");
  1242. MODULE_LICENSE("GPL v2");