at_xdmac.c 66 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Driver for the Atmel Extensible DMA Controller (aka XDMAC on AT91 systems)
  4. *
  5. * Copyright (C) 2014 Atmel Corporation
  6. *
  7. * Author: Ludovic Desroches <ludovic.desroches@atmel.com>
  8. */
  9. #include <asm/barrier.h>
  10. #include <dt-bindings/dma/at91.h>
  11. #include <linux/clk.h>
  12. #include <linux/dmaengine.h>
  13. #include <linux/dmapool.h>
  14. #include <linux/interrupt.h>
  15. #include <linux/irq.h>
  16. #include <linux/kernel.h>
  17. #include <linux/list.h>
  18. #include <linux/module.h>
  19. #include <linux/of_dma.h>
  20. #include <linux/of_platform.h>
  21. #include <linux/platform_device.h>
  22. #include <linux/pm.h>
  23. #include "dmaengine.h"
  24. /* Global registers */
  25. #define AT_XDMAC_GTYPE 0x00 /* Global Type Register */
  26. #define AT_XDMAC_NB_CH(i) (((i) & 0x1F) + 1) /* Number of Channels Minus One */
  27. #define AT_XDMAC_FIFO_SZ(i) (((i) >> 5) & 0x7FF) /* Number of Bytes */
  28. #define AT_XDMAC_NB_REQ(i) ((((i) >> 16) & 0x3F) + 1) /* Number of Peripheral Requests Minus One */
  29. #define AT_XDMAC_GCFG 0x04 /* Global Configuration Register */
  30. #define AT_XDMAC_GWAC 0x08 /* Global Weighted Arbiter Configuration Register */
  31. #define AT_XDMAC_GIE 0x0C /* Global Interrupt Enable Register */
  32. #define AT_XDMAC_GID 0x10 /* Global Interrupt Disable Register */
  33. #define AT_XDMAC_GIM 0x14 /* Global Interrupt Mask Register */
  34. #define AT_XDMAC_GIS 0x18 /* Global Interrupt Status Register */
  35. #define AT_XDMAC_GE 0x1C /* Global Channel Enable Register */
  36. #define AT_XDMAC_GD 0x20 /* Global Channel Disable Register */
  37. #define AT_XDMAC_GS 0x24 /* Global Channel Status Register */
  38. #define AT_XDMAC_GRS 0x28 /* Global Channel Read Suspend Register */
  39. #define AT_XDMAC_GWS 0x2C /* Global Write Suspend Register */
  40. #define AT_XDMAC_GRWS 0x30 /* Global Channel Read Write Suspend Register */
  41. #define AT_XDMAC_GRWR 0x34 /* Global Channel Read Write Resume Register */
  42. #define AT_XDMAC_GSWR 0x38 /* Global Channel Software Request Register */
  43. #define AT_XDMAC_GSWS 0x3C /* Global channel Software Request Status Register */
  44. #define AT_XDMAC_GSWF 0x40 /* Global Channel Software Flush Request Register */
  45. #define AT_XDMAC_VERSION 0xFFC /* XDMAC Version Register */
  46. /* Channel relative registers offsets */
  47. #define AT_XDMAC_CIE 0x00 /* Channel Interrupt Enable Register */
  48. #define AT_XDMAC_CIE_BIE BIT(0) /* End of Block Interrupt Enable Bit */
  49. #define AT_XDMAC_CIE_LIE BIT(1) /* End of Linked List Interrupt Enable Bit */
  50. #define AT_XDMAC_CIE_DIE BIT(2) /* End of Disable Interrupt Enable Bit */
  51. #define AT_XDMAC_CIE_FIE BIT(3) /* End of Flush Interrupt Enable Bit */
  52. #define AT_XDMAC_CIE_RBEIE BIT(4) /* Read Bus Error Interrupt Enable Bit */
  53. #define AT_XDMAC_CIE_WBEIE BIT(5) /* Write Bus Error Interrupt Enable Bit */
  54. #define AT_XDMAC_CIE_ROIE BIT(6) /* Request Overflow Interrupt Enable Bit */
  55. #define AT_XDMAC_CID 0x04 /* Channel Interrupt Disable Register */
  56. #define AT_XDMAC_CID_BID BIT(0) /* End of Block Interrupt Disable Bit */
  57. #define AT_XDMAC_CID_LID BIT(1) /* End of Linked List Interrupt Disable Bit */
  58. #define AT_XDMAC_CID_DID BIT(2) /* End of Disable Interrupt Disable Bit */
  59. #define AT_XDMAC_CID_FID BIT(3) /* End of Flush Interrupt Disable Bit */
  60. #define AT_XDMAC_CID_RBEID BIT(4) /* Read Bus Error Interrupt Disable Bit */
  61. #define AT_XDMAC_CID_WBEID BIT(5) /* Write Bus Error Interrupt Disable Bit */
  62. #define AT_XDMAC_CID_ROID BIT(6) /* Request Overflow Interrupt Disable Bit */
  63. #define AT_XDMAC_CIM 0x08 /* Channel Interrupt Mask Register */
  64. #define AT_XDMAC_CIM_BIM BIT(0) /* End of Block Interrupt Mask Bit */
  65. #define AT_XDMAC_CIM_LIM BIT(1) /* End of Linked List Interrupt Mask Bit */
  66. #define AT_XDMAC_CIM_DIM BIT(2) /* End of Disable Interrupt Mask Bit */
  67. #define AT_XDMAC_CIM_FIM BIT(3) /* End of Flush Interrupt Mask Bit */
  68. #define AT_XDMAC_CIM_RBEIM BIT(4) /* Read Bus Error Interrupt Mask Bit */
  69. #define AT_XDMAC_CIM_WBEIM BIT(5) /* Write Bus Error Interrupt Mask Bit */
  70. #define AT_XDMAC_CIM_ROIM BIT(6) /* Request Overflow Interrupt Mask Bit */
  71. #define AT_XDMAC_CIS 0x0C /* Channel Interrupt Status Register */
  72. #define AT_XDMAC_CIS_BIS BIT(0) /* End of Block Interrupt Status Bit */
  73. #define AT_XDMAC_CIS_LIS BIT(1) /* End of Linked List Interrupt Status Bit */
  74. #define AT_XDMAC_CIS_DIS BIT(2) /* End of Disable Interrupt Status Bit */
  75. #define AT_XDMAC_CIS_FIS BIT(3) /* End of Flush Interrupt Status Bit */
  76. #define AT_XDMAC_CIS_RBEIS BIT(4) /* Read Bus Error Interrupt Status Bit */
  77. #define AT_XDMAC_CIS_WBEIS BIT(5) /* Write Bus Error Interrupt Status Bit */
  78. #define AT_XDMAC_CIS_ROIS BIT(6) /* Request Overflow Interrupt Status Bit */
  79. #define AT_XDMAC_CSA 0x10 /* Channel Source Address Register */
  80. #define AT_XDMAC_CDA 0x14 /* Channel Destination Address Register */
  81. #define AT_XDMAC_CNDA 0x18 /* Channel Next Descriptor Address Register */
  82. #define AT_XDMAC_CNDA_NDAIF(i) ((i) & 0x1) /* Channel x Next Descriptor Interface */
  83. #define AT_XDMAC_CNDA_NDA(i) ((i) & 0xfffffffc) /* Channel x Next Descriptor Address */
  84. #define AT_XDMAC_CNDC 0x1C /* Channel Next Descriptor Control Register */
  85. #define AT_XDMAC_CNDC_NDE (0x1 << 0) /* Channel x Next Descriptor Enable */
  86. #define AT_XDMAC_CNDC_NDSUP (0x1 << 1) /* Channel x Next Descriptor Source Update */
  87. #define AT_XDMAC_CNDC_NDDUP (0x1 << 2) /* Channel x Next Descriptor Destination Update */
  88. #define AT_XDMAC_CNDC_NDVIEW_MASK GENMASK(28, 27)
  89. #define AT_XDMAC_CNDC_NDVIEW_NDV0 (0x0 << 3) /* Channel x Next Descriptor View 0 */
  90. #define AT_XDMAC_CNDC_NDVIEW_NDV1 (0x1 << 3) /* Channel x Next Descriptor View 1 */
  91. #define AT_XDMAC_CNDC_NDVIEW_NDV2 (0x2 << 3) /* Channel x Next Descriptor View 2 */
  92. #define AT_XDMAC_CNDC_NDVIEW_NDV3 (0x3 << 3) /* Channel x Next Descriptor View 3 */
  93. #define AT_XDMAC_CUBC 0x20 /* Channel Microblock Control Register */
  94. #define AT_XDMAC_CBC 0x24 /* Channel Block Control Register */
  95. #define AT_XDMAC_CC 0x28 /* Channel Configuration Register */
  96. #define AT_XDMAC_CC_TYPE (0x1 << 0) /* Channel Transfer Type */
  97. #define AT_XDMAC_CC_TYPE_MEM_TRAN (0x0 << 0) /* Memory to Memory Transfer */
  98. #define AT_XDMAC_CC_TYPE_PER_TRAN (0x1 << 0) /* Peripheral to Memory or Memory to Peripheral Transfer */
  99. #define AT_XDMAC_CC_MBSIZE_MASK (0x3 << 1)
  100. #define AT_XDMAC_CC_MBSIZE_SINGLE (0x0 << 1)
  101. #define AT_XDMAC_CC_MBSIZE_FOUR (0x1 << 1)
  102. #define AT_XDMAC_CC_MBSIZE_EIGHT (0x2 << 1)
  103. #define AT_XDMAC_CC_MBSIZE_SIXTEEN (0x3 << 1)
  104. #define AT_XDMAC_CC_DSYNC (0x1 << 4) /* Channel Synchronization */
  105. #define AT_XDMAC_CC_DSYNC_PER2MEM (0x0 << 4)
  106. #define AT_XDMAC_CC_DSYNC_MEM2PER (0x1 << 4)
  107. #define AT_XDMAC_CC_PROT (0x1 << 5) /* Channel Protection */
  108. #define AT_XDMAC_CC_PROT_SEC (0x0 << 5)
  109. #define AT_XDMAC_CC_PROT_UNSEC (0x1 << 5)
  110. #define AT_XDMAC_CC_SWREQ (0x1 << 6) /* Channel Software Request Trigger */
  111. #define AT_XDMAC_CC_SWREQ_HWR_CONNECTED (0x0 << 6)
  112. #define AT_XDMAC_CC_SWREQ_SWR_CONNECTED (0x1 << 6)
  113. #define AT_XDMAC_CC_MEMSET (0x1 << 7) /* Channel Fill Block of memory */
  114. #define AT_XDMAC_CC_MEMSET_NORMAL_MODE (0x0 << 7)
  115. #define AT_XDMAC_CC_MEMSET_HW_MODE (0x1 << 7)
  116. #define AT_XDMAC_CC_CSIZE(i) ((0x7 & (i)) << 8) /* Channel Chunk Size */
  117. #define AT_XDMAC_CC_DWIDTH_OFFSET 11
  118. #define AT_XDMAC_CC_DWIDTH_MASK (0x3 << AT_XDMAC_CC_DWIDTH_OFFSET)
  119. #define AT_XDMAC_CC_DWIDTH(i) ((0x3 & (i)) << AT_XDMAC_CC_DWIDTH_OFFSET) /* Channel Data Width */
  120. #define AT_XDMAC_CC_DWIDTH_BYTE 0x0
  121. #define AT_XDMAC_CC_DWIDTH_HALFWORD 0x1
  122. #define AT_XDMAC_CC_DWIDTH_WORD 0x2
  123. #define AT_XDMAC_CC_DWIDTH_DWORD 0x3
  124. #define AT_XDMAC_CC_SIF(i) ((0x1 & (i)) << 13) /* Channel Source Interface Identifier */
  125. #define AT_XDMAC_CC_DIF(i) ((0x1 & (i)) << 14) /* Channel Destination Interface Identifier */
  126. #define AT_XDMAC_CC_SAM_MASK (0x3 << 16) /* Channel Source Addressing Mode */
  127. #define AT_XDMAC_CC_SAM_FIXED_AM (0x0 << 16)
  128. #define AT_XDMAC_CC_SAM_INCREMENTED_AM (0x1 << 16)
  129. #define AT_XDMAC_CC_SAM_UBS_AM (0x2 << 16)
  130. #define AT_XDMAC_CC_SAM_UBS_DS_AM (0x3 << 16)
  131. #define AT_XDMAC_CC_DAM_MASK (0x3 << 18) /* Channel Source Addressing Mode */
  132. #define AT_XDMAC_CC_DAM_FIXED_AM (0x0 << 18)
  133. #define AT_XDMAC_CC_DAM_INCREMENTED_AM (0x1 << 18)
  134. #define AT_XDMAC_CC_DAM_UBS_AM (0x2 << 18)
  135. #define AT_XDMAC_CC_DAM_UBS_DS_AM (0x3 << 18)
  136. #define AT_XDMAC_CC_INITD (0x1 << 21) /* Channel Initialization Terminated (read only) */
  137. #define AT_XDMAC_CC_INITD_TERMINATED (0x0 << 21)
  138. #define AT_XDMAC_CC_INITD_IN_PROGRESS (0x1 << 21)
  139. #define AT_XDMAC_CC_RDIP (0x1 << 22) /* Read in Progress (read only) */
  140. #define AT_XDMAC_CC_RDIP_DONE (0x0 << 22)
  141. #define AT_XDMAC_CC_RDIP_IN_PROGRESS (0x1 << 22)
  142. #define AT_XDMAC_CC_WRIP (0x1 << 23) /* Write in Progress (read only) */
  143. #define AT_XDMAC_CC_WRIP_DONE (0x0 << 23)
  144. #define AT_XDMAC_CC_WRIP_IN_PROGRESS (0x1 << 23)
  145. #define AT_XDMAC_CC_PERID(i) ((0x7f & (i)) << 24) /* Channel Peripheral Identifier */
  146. #define AT_XDMAC_CDS_MSP 0x2C /* Channel Data Stride Memory Set Pattern */
  147. #define AT_XDMAC_CSUS 0x30 /* Channel Source Microblock Stride */
  148. #define AT_XDMAC_CDUS 0x34 /* Channel Destination Microblock Stride */
  149. #define AT_XDMAC_CHAN_REG_BASE 0x50 /* Channel registers base address */
  150. /* Microblock control members */
  151. #define AT_XDMAC_MBR_UBC_UBLEN_MAX 0xFFFFFFUL /* Maximum Microblock Length */
  152. #define AT_XDMAC_MBR_UBC_NDE (0x1 << 24) /* Next Descriptor Enable */
  153. #define AT_XDMAC_MBR_UBC_NSEN (0x1 << 25) /* Next Descriptor Source Update */
  154. #define AT_XDMAC_MBR_UBC_NDEN (0x1 << 26) /* Next Descriptor Destination Update */
  155. #define AT_XDMAC_MBR_UBC_NDV0 (0x0 << 27) /* Next Descriptor View 0 */
  156. #define AT_XDMAC_MBR_UBC_NDV1 (0x1 << 27) /* Next Descriptor View 1 */
  157. #define AT_XDMAC_MBR_UBC_NDV2 (0x2 << 27) /* Next Descriptor View 2 */
  158. #define AT_XDMAC_MBR_UBC_NDV3 (0x3 << 27) /* Next Descriptor View 3 */
  159. #define AT_XDMAC_MAX_CHAN 0x20
  160. #define AT_XDMAC_MAX_CSIZE 16 /* 16 data */
  161. #define AT_XDMAC_MAX_DWIDTH 8 /* 64 bits */
  162. #define AT_XDMAC_RESIDUE_MAX_RETRIES 5
  163. #define AT_XDMAC_DMA_BUSWIDTHS\
  164. (BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) |\
  165. BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |\
  166. BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |\
  167. BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |\
  168. BIT(DMA_SLAVE_BUSWIDTH_8_BYTES))
  169. enum atc_status {
  170. AT_XDMAC_CHAN_IS_CYCLIC = 0,
  171. AT_XDMAC_CHAN_IS_PAUSED,
  172. };
  173. /* ----- Channels ----- */
  174. struct at_xdmac_chan {
  175. struct dma_chan chan;
  176. void __iomem *ch_regs;
  177. u32 mask; /* Channel Mask */
  178. u32 cfg; /* Channel Configuration Register */
  179. u8 perid; /* Peripheral ID */
  180. u8 perif; /* Peripheral Interface */
  181. u8 memif; /* Memory Interface */
  182. u32 save_cc;
  183. u32 save_cim;
  184. u32 save_cnda;
  185. u32 save_cndc;
  186. u32 irq_status;
  187. unsigned long status;
  188. struct tasklet_struct tasklet;
  189. struct dma_slave_config sconfig;
  190. spinlock_t lock;
  191. struct list_head xfers_list;
  192. struct list_head free_descs_list;
  193. };
  194. /* ----- Controller ----- */
  195. struct at_xdmac {
  196. struct dma_device dma;
  197. void __iomem *regs;
  198. int irq;
  199. struct clk *clk;
  200. u32 save_gim;
  201. struct dma_pool *at_xdmac_desc_pool;
  202. struct at_xdmac_chan chan[];
  203. };
  204. /* ----- Descriptors ----- */
  205. /* Linked List Descriptor */
  206. struct at_xdmac_lld {
  207. u32 mbr_nda; /* Next Descriptor Member */
  208. u32 mbr_ubc; /* Microblock Control Member */
  209. u32 mbr_sa; /* Source Address Member */
  210. u32 mbr_da; /* Destination Address Member */
  211. u32 mbr_cfg; /* Configuration Register */
  212. u32 mbr_bc; /* Block Control Register */
  213. u32 mbr_ds; /* Data Stride Register */
  214. u32 mbr_sus; /* Source Microblock Stride Register */
  215. u32 mbr_dus; /* Destination Microblock Stride Register */
  216. };
  217. /* 64-bit alignment needed to update CNDA and CUBC registers in an atomic way. */
  218. struct at_xdmac_desc {
  219. struct at_xdmac_lld lld;
  220. enum dma_transfer_direction direction;
  221. struct dma_async_tx_descriptor tx_dma_desc;
  222. struct list_head desc_node;
  223. /* Following members are only used by the first descriptor */
  224. bool active_xfer;
  225. unsigned int xfer_size;
  226. struct list_head descs_list;
  227. struct list_head xfer_node;
  228. } __aligned(sizeof(u64));
  229. static inline void __iomem *at_xdmac_chan_reg_base(struct at_xdmac *atxdmac, unsigned int chan_nb)
  230. {
  231. return atxdmac->regs + (AT_XDMAC_CHAN_REG_BASE + chan_nb * 0x40);
  232. }
  233. #define at_xdmac_read(atxdmac, reg) readl_relaxed((atxdmac)->regs + (reg))
  234. #define at_xdmac_write(atxdmac, reg, value) \
  235. writel_relaxed((value), (atxdmac)->regs + (reg))
  236. #define at_xdmac_chan_read(atchan, reg) readl_relaxed((atchan)->ch_regs + (reg))
  237. #define at_xdmac_chan_write(atchan, reg, value) writel_relaxed((value), (atchan)->ch_regs + (reg))
  238. static inline struct at_xdmac_chan *to_at_xdmac_chan(struct dma_chan *dchan)
  239. {
  240. return container_of(dchan, struct at_xdmac_chan, chan);
  241. }
  242. static struct device *chan2dev(struct dma_chan *chan)
  243. {
  244. return &chan->dev->device;
  245. }
  246. static inline struct at_xdmac *to_at_xdmac(struct dma_device *ddev)
  247. {
  248. return container_of(ddev, struct at_xdmac, dma);
  249. }
  250. static inline struct at_xdmac_desc *txd_to_at_desc(struct dma_async_tx_descriptor *txd)
  251. {
  252. return container_of(txd, struct at_xdmac_desc, tx_dma_desc);
  253. }
  254. static inline int at_xdmac_chan_is_cyclic(struct at_xdmac_chan *atchan)
  255. {
  256. return test_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status);
  257. }
  258. static inline int at_xdmac_chan_is_paused(struct at_xdmac_chan *atchan)
  259. {
  260. return test_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status);
  261. }
  262. static inline int at_xdmac_csize(u32 maxburst)
  263. {
  264. int csize;
  265. csize = ffs(maxburst) - 1;
  266. if (csize > 4)
  267. csize = -EINVAL;
  268. return csize;
  269. };
  270. static inline bool at_xdmac_chan_is_peripheral_xfer(u32 cfg)
  271. {
  272. return cfg & AT_XDMAC_CC_TYPE_PER_TRAN;
  273. }
  274. static inline u8 at_xdmac_get_dwidth(u32 cfg)
  275. {
  276. return (cfg & AT_XDMAC_CC_DWIDTH_MASK) >> AT_XDMAC_CC_DWIDTH_OFFSET;
  277. };
  278. static unsigned int init_nr_desc_per_channel = 64;
  279. module_param(init_nr_desc_per_channel, uint, 0644);
  280. MODULE_PARM_DESC(init_nr_desc_per_channel,
  281. "initial descriptors per channel (default: 64)");
  282. static bool at_xdmac_chan_is_enabled(struct at_xdmac_chan *atchan)
  283. {
  284. return at_xdmac_chan_read(atchan, AT_XDMAC_GS) & atchan->mask;
  285. }
  286. static void at_xdmac_off(struct at_xdmac *atxdmac)
  287. {
  288. at_xdmac_write(atxdmac, AT_XDMAC_GD, -1L);
  289. /* Wait that all chans are disabled. */
  290. while (at_xdmac_read(atxdmac, AT_XDMAC_GS))
  291. cpu_relax();
  292. at_xdmac_write(atxdmac, AT_XDMAC_GID, -1L);
  293. }
  294. /* Call with lock hold. */
  295. static void at_xdmac_start_xfer(struct at_xdmac_chan *atchan,
  296. struct at_xdmac_desc *first)
  297. {
  298. struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device);
  299. u32 reg;
  300. dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, first);
  301. /* Set transfer as active to not try to start it again. */
  302. first->active_xfer = true;
  303. /* Tell xdmac where to get the first descriptor. */
  304. reg = AT_XDMAC_CNDA_NDA(first->tx_dma_desc.phys)
  305. | AT_XDMAC_CNDA_NDAIF(atchan->memif);
  306. at_xdmac_chan_write(atchan, AT_XDMAC_CNDA, reg);
  307. /*
  308. * When doing non cyclic transfer we need to use the next
  309. * descriptor view 2 since some fields of the configuration register
  310. * depend on transfer size and src/dest addresses.
  311. */
  312. if (at_xdmac_chan_is_cyclic(atchan))
  313. reg = AT_XDMAC_CNDC_NDVIEW_NDV1;
  314. else if ((first->lld.mbr_ubc &
  315. AT_XDMAC_CNDC_NDVIEW_MASK) == AT_XDMAC_MBR_UBC_NDV3)
  316. reg = AT_XDMAC_CNDC_NDVIEW_NDV3;
  317. else
  318. reg = AT_XDMAC_CNDC_NDVIEW_NDV2;
  319. /*
  320. * Even if the register will be updated from the configuration in the
  321. * descriptor when using view 2 or higher, the PROT bit won't be set
  322. * properly. This bit can be modified only by using the channel
  323. * configuration register.
  324. */
  325. at_xdmac_chan_write(atchan, AT_XDMAC_CC, first->lld.mbr_cfg);
  326. reg |= AT_XDMAC_CNDC_NDDUP
  327. | AT_XDMAC_CNDC_NDSUP
  328. | AT_XDMAC_CNDC_NDE;
  329. at_xdmac_chan_write(atchan, AT_XDMAC_CNDC, reg);
  330. dev_vdbg(chan2dev(&atchan->chan),
  331. "%s: CC=0x%08x CNDA=0x%08x, CNDC=0x%08x, CSA=0x%08x, CDA=0x%08x, CUBC=0x%08x\n",
  332. __func__, at_xdmac_chan_read(atchan, AT_XDMAC_CC),
  333. at_xdmac_chan_read(atchan, AT_XDMAC_CNDA),
  334. at_xdmac_chan_read(atchan, AT_XDMAC_CNDC),
  335. at_xdmac_chan_read(atchan, AT_XDMAC_CSA),
  336. at_xdmac_chan_read(atchan, AT_XDMAC_CDA),
  337. at_xdmac_chan_read(atchan, AT_XDMAC_CUBC));
  338. at_xdmac_chan_write(atchan, AT_XDMAC_CID, 0xffffffff);
  339. reg = AT_XDMAC_CIE_RBEIE | AT_XDMAC_CIE_WBEIE;
  340. /*
  341. * Request Overflow Error is only for peripheral synchronized transfers
  342. */
  343. if (at_xdmac_chan_is_peripheral_xfer(first->lld.mbr_cfg))
  344. reg |= AT_XDMAC_CIE_ROIE;
  345. /*
  346. * There is no end of list when doing cyclic dma, we need to get
  347. * an interrupt after each periods.
  348. */
  349. if (at_xdmac_chan_is_cyclic(atchan))
  350. at_xdmac_chan_write(atchan, AT_XDMAC_CIE,
  351. reg | AT_XDMAC_CIE_BIE);
  352. else
  353. at_xdmac_chan_write(atchan, AT_XDMAC_CIE,
  354. reg | AT_XDMAC_CIE_LIE);
  355. at_xdmac_write(atxdmac, AT_XDMAC_GIE, atchan->mask);
  356. dev_vdbg(chan2dev(&atchan->chan),
  357. "%s: enable channel (0x%08x)\n", __func__, atchan->mask);
  358. wmb();
  359. at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask);
  360. dev_vdbg(chan2dev(&atchan->chan),
  361. "%s: CC=0x%08x CNDA=0x%08x, CNDC=0x%08x, CSA=0x%08x, CDA=0x%08x, CUBC=0x%08x\n",
  362. __func__, at_xdmac_chan_read(atchan, AT_XDMAC_CC),
  363. at_xdmac_chan_read(atchan, AT_XDMAC_CNDA),
  364. at_xdmac_chan_read(atchan, AT_XDMAC_CNDC),
  365. at_xdmac_chan_read(atchan, AT_XDMAC_CSA),
  366. at_xdmac_chan_read(atchan, AT_XDMAC_CDA),
  367. at_xdmac_chan_read(atchan, AT_XDMAC_CUBC));
  368. }
  369. static dma_cookie_t at_xdmac_tx_submit(struct dma_async_tx_descriptor *tx)
  370. {
  371. struct at_xdmac_desc *desc = txd_to_at_desc(tx);
  372. struct at_xdmac_chan *atchan = to_at_xdmac_chan(tx->chan);
  373. dma_cookie_t cookie;
  374. unsigned long irqflags;
  375. spin_lock_irqsave(&atchan->lock, irqflags);
  376. cookie = dma_cookie_assign(tx);
  377. list_add_tail(&desc->xfer_node, &atchan->xfers_list);
  378. spin_unlock_irqrestore(&atchan->lock, irqflags);
  379. dev_vdbg(chan2dev(tx->chan), "%s: atchan 0x%p, add desc 0x%p to xfers_list\n",
  380. __func__, atchan, desc);
  381. return cookie;
  382. }
  383. static struct at_xdmac_desc *at_xdmac_alloc_desc(struct dma_chan *chan,
  384. gfp_t gfp_flags)
  385. {
  386. struct at_xdmac_desc *desc;
  387. struct at_xdmac *atxdmac = to_at_xdmac(chan->device);
  388. dma_addr_t phys;
  389. desc = dma_pool_zalloc(atxdmac->at_xdmac_desc_pool, gfp_flags, &phys);
  390. if (desc) {
  391. INIT_LIST_HEAD(&desc->descs_list);
  392. dma_async_tx_descriptor_init(&desc->tx_dma_desc, chan);
  393. desc->tx_dma_desc.tx_submit = at_xdmac_tx_submit;
  394. desc->tx_dma_desc.phys = phys;
  395. }
  396. return desc;
  397. }
  398. static void at_xdmac_init_used_desc(struct at_xdmac_desc *desc)
  399. {
  400. memset(&desc->lld, 0, sizeof(desc->lld));
  401. INIT_LIST_HEAD(&desc->descs_list);
  402. desc->direction = DMA_TRANS_NONE;
  403. desc->xfer_size = 0;
  404. desc->active_xfer = false;
  405. }
  406. /* Call must be protected by lock. */
  407. static struct at_xdmac_desc *at_xdmac_get_desc(struct at_xdmac_chan *atchan)
  408. {
  409. struct at_xdmac_desc *desc;
  410. if (list_empty(&atchan->free_descs_list)) {
  411. desc = at_xdmac_alloc_desc(&atchan->chan, GFP_NOWAIT);
  412. } else {
  413. desc = list_first_entry(&atchan->free_descs_list,
  414. struct at_xdmac_desc, desc_node);
  415. list_del(&desc->desc_node);
  416. at_xdmac_init_used_desc(desc);
  417. }
  418. return desc;
  419. }
  420. static void at_xdmac_queue_desc(struct dma_chan *chan,
  421. struct at_xdmac_desc *prev,
  422. struct at_xdmac_desc *desc)
  423. {
  424. if (!prev || !desc)
  425. return;
  426. prev->lld.mbr_nda = desc->tx_dma_desc.phys;
  427. prev->lld.mbr_ubc |= AT_XDMAC_MBR_UBC_NDE;
  428. dev_dbg(chan2dev(chan), "%s: chain lld: prev=0x%p, mbr_nda=%pad\n",
  429. __func__, prev, &prev->lld.mbr_nda);
  430. }
  431. static inline void at_xdmac_increment_block_count(struct dma_chan *chan,
  432. struct at_xdmac_desc *desc)
  433. {
  434. if (!desc)
  435. return;
  436. desc->lld.mbr_bc++;
  437. dev_dbg(chan2dev(chan),
  438. "%s: incrementing the block count of the desc 0x%p\n",
  439. __func__, desc);
  440. }
  441. static struct dma_chan *at_xdmac_xlate(struct of_phandle_args *dma_spec,
  442. struct of_dma *of_dma)
  443. {
  444. struct at_xdmac *atxdmac = of_dma->of_dma_data;
  445. struct at_xdmac_chan *atchan;
  446. struct dma_chan *chan;
  447. struct device *dev = atxdmac->dma.dev;
  448. if (dma_spec->args_count != 1) {
  449. dev_err(dev, "dma phandler args: bad number of args\n");
  450. return NULL;
  451. }
  452. chan = dma_get_any_slave_channel(&atxdmac->dma);
  453. if (!chan) {
  454. dev_err(dev, "can't get a dma channel\n");
  455. return NULL;
  456. }
  457. atchan = to_at_xdmac_chan(chan);
  458. atchan->memif = AT91_XDMAC_DT_GET_MEM_IF(dma_spec->args[0]);
  459. atchan->perif = AT91_XDMAC_DT_GET_PER_IF(dma_spec->args[0]);
  460. atchan->perid = AT91_XDMAC_DT_GET_PERID(dma_spec->args[0]);
  461. dev_dbg(dev, "chan dt cfg: memif=%u perif=%u perid=%u\n",
  462. atchan->memif, atchan->perif, atchan->perid);
  463. return chan;
  464. }
  465. static int at_xdmac_compute_chan_conf(struct dma_chan *chan,
  466. enum dma_transfer_direction direction)
  467. {
  468. struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
  469. int csize, dwidth;
  470. if (direction == DMA_DEV_TO_MEM) {
  471. atchan->cfg =
  472. AT91_XDMAC_DT_PERID(atchan->perid)
  473. | AT_XDMAC_CC_DAM_INCREMENTED_AM
  474. | AT_XDMAC_CC_SAM_FIXED_AM
  475. | AT_XDMAC_CC_DIF(atchan->memif)
  476. | AT_XDMAC_CC_SIF(atchan->perif)
  477. | AT_XDMAC_CC_SWREQ_HWR_CONNECTED
  478. | AT_XDMAC_CC_DSYNC_PER2MEM
  479. | AT_XDMAC_CC_MBSIZE_SIXTEEN
  480. | AT_XDMAC_CC_TYPE_PER_TRAN;
  481. csize = ffs(atchan->sconfig.src_maxburst) - 1;
  482. if (csize < 0) {
  483. dev_err(chan2dev(chan), "invalid src maxburst value\n");
  484. return -EINVAL;
  485. }
  486. atchan->cfg |= AT_XDMAC_CC_CSIZE(csize);
  487. dwidth = ffs(atchan->sconfig.src_addr_width) - 1;
  488. if (dwidth < 0) {
  489. dev_err(chan2dev(chan), "invalid src addr width value\n");
  490. return -EINVAL;
  491. }
  492. atchan->cfg |= AT_XDMAC_CC_DWIDTH(dwidth);
  493. } else if (direction == DMA_MEM_TO_DEV) {
  494. atchan->cfg =
  495. AT91_XDMAC_DT_PERID(atchan->perid)
  496. | AT_XDMAC_CC_DAM_FIXED_AM
  497. | AT_XDMAC_CC_SAM_INCREMENTED_AM
  498. | AT_XDMAC_CC_DIF(atchan->perif)
  499. | AT_XDMAC_CC_SIF(atchan->memif)
  500. | AT_XDMAC_CC_SWREQ_HWR_CONNECTED
  501. | AT_XDMAC_CC_DSYNC_MEM2PER
  502. | AT_XDMAC_CC_MBSIZE_SIXTEEN
  503. | AT_XDMAC_CC_TYPE_PER_TRAN;
  504. csize = ffs(atchan->sconfig.dst_maxburst) - 1;
  505. if (csize < 0) {
  506. dev_err(chan2dev(chan), "invalid src maxburst value\n");
  507. return -EINVAL;
  508. }
  509. atchan->cfg |= AT_XDMAC_CC_CSIZE(csize);
  510. dwidth = ffs(atchan->sconfig.dst_addr_width) - 1;
  511. if (dwidth < 0) {
  512. dev_err(chan2dev(chan), "invalid dst addr width value\n");
  513. return -EINVAL;
  514. }
  515. atchan->cfg |= AT_XDMAC_CC_DWIDTH(dwidth);
  516. }
  517. dev_dbg(chan2dev(chan), "%s: cfg=0x%08x\n", __func__, atchan->cfg);
  518. return 0;
  519. }
  520. /*
  521. * Only check that maxburst and addr width values are supported by the
  522. * the controller but not that the configuration is good to perform the
  523. * transfer since we don't know the direction at this stage.
  524. */
  525. static int at_xdmac_check_slave_config(struct dma_slave_config *sconfig)
  526. {
  527. if ((sconfig->src_maxburst > AT_XDMAC_MAX_CSIZE)
  528. || (sconfig->dst_maxburst > AT_XDMAC_MAX_CSIZE))
  529. return -EINVAL;
  530. if ((sconfig->src_addr_width > AT_XDMAC_MAX_DWIDTH)
  531. || (sconfig->dst_addr_width > AT_XDMAC_MAX_DWIDTH))
  532. return -EINVAL;
  533. return 0;
  534. }
  535. static int at_xdmac_set_slave_config(struct dma_chan *chan,
  536. struct dma_slave_config *sconfig)
  537. {
  538. struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
  539. if (at_xdmac_check_slave_config(sconfig)) {
  540. dev_err(chan2dev(chan), "invalid slave configuration\n");
  541. return -EINVAL;
  542. }
  543. memcpy(&atchan->sconfig, sconfig, sizeof(atchan->sconfig));
  544. return 0;
  545. }
  546. static struct dma_async_tx_descriptor *
  547. at_xdmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
  548. unsigned int sg_len, enum dma_transfer_direction direction,
  549. unsigned long flags, void *context)
  550. {
  551. struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
  552. struct at_xdmac_desc *first = NULL, *prev = NULL;
  553. struct scatterlist *sg;
  554. int i;
  555. unsigned int xfer_size = 0;
  556. unsigned long irqflags;
  557. struct dma_async_tx_descriptor *ret = NULL;
  558. if (!sgl)
  559. return NULL;
  560. if (!is_slave_direction(direction)) {
  561. dev_err(chan2dev(chan), "invalid DMA direction\n");
  562. return NULL;
  563. }
  564. dev_dbg(chan2dev(chan), "%s: sg_len=%d, dir=%s, flags=0x%lx\n",
  565. __func__, sg_len,
  566. direction == DMA_MEM_TO_DEV ? "to device" : "from device",
  567. flags);
  568. /* Protect dma_sconfig field that can be modified by set_slave_conf. */
  569. spin_lock_irqsave(&atchan->lock, irqflags);
  570. if (at_xdmac_compute_chan_conf(chan, direction))
  571. goto spin_unlock;
  572. /* Prepare descriptors. */
  573. for_each_sg(sgl, sg, sg_len, i) {
  574. struct at_xdmac_desc *desc = NULL;
  575. u32 len, mem, dwidth, fixed_dwidth;
  576. len = sg_dma_len(sg);
  577. mem = sg_dma_address(sg);
  578. if (unlikely(!len)) {
  579. dev_err(chan2dev(chan), "sg data length is zero\n");
  580. goto spin_unlock;
  581. }
  582. dev_dbg(chan2dev(chan), "%s: * sg%d len=%u, mem=0x%08x\n",
  583. __func__, i, len, mem);
  584. desc = at_xdmac_get_desc(atchan);
  585. if (!desc) {
  586. dev_err(chan2dev(chan), "can't get descriptor\n");
  587. if (first)
  588. list_splice_init(&first->descs_list, &atchan->free_descs_list);
  589. goto spin_unlock;
  590. }
  591. /* Linked list descriptor setup. */
  592. if (direction == DMA_DEV_TO_MEM) {
  593. desc->lld.mbr_sa = atchan->sconfig.src_addr;
  594. desc->lld.mbr_da = mem;
  595. } else {
  596. desc->lld.mbr_sa = mem;
  597. desc->lld.mbr_da = atchan->sconfig.dst_addr;
  598. }
  599. dwidth = at_xdmac_get_dwidth(atchan->cfg);
  600. fixed_dwidth = IS_ALIGNED(len, 1 << dwidth)
  601. ? dwidth
  602. : AT_XDMAC_CC_DWIDTH_BYTE;
  603. desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV2 /* next descriptor view */
  604. | AT_XDMAC_MBR_UBC_NDEN /* next descriptor dst parameter update */
  605. | AT_XDMAC_MBR_UBC_NSEN /* next descriptor src parameter update */
  606. | (len >> fixed_dwidth); /* microblock length */
  607. desc->lld.mbr_cfg = (atchan->cfg & ~AT_XDMAC_CC_DWIDTH_MASK) |
  608. AT_XDMAC_CC_DWIDTH(fixed_dwidth);
  609. dev_dbg(chan2dev(chan),
  610. "%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x\n",
  611. __func__, &desc->lld.mbr_sa, &desc->lld.mbr_da, desc->lld.mbr_ubc);
  612. /* Chain lld. */
  613. if (prev)
  614. at_xdmac_queue_desc(chan, prev, desc);
  615. prev = desc;
  616. if (!first)
  617. first = desc;
  618. dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",
  619. __func__, desc, first);
  620. list_add_tail(&desc->desc_node, &first->descs_list);
  621. xfer_size += len;
  622. }
  623. first->tx_dma_desc.flags = flags;
  624. first->xfer_size = xfer_size;
  625. first->direction = direction;
  626. ret = &first->tx_dma_desc;
  627. spin_unlock:
  628. spin_unlock_irqrestore(&atchan->lock, irqflags);
  629. return ret;
  630. }
  631. static struct dma_async_tx_descriptor *
  632. at_xdmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr,
  633. size_t buf_len, size_t period_len,
  634. enum dma_transfer_direction direction,
  635. unsigned long flags)
  636. {
  637. struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
  638. struct at_xdmac_desc *first = NULL, *prev = NULL;
  639. unsigned int periods = buf_len / period_len;
  640. int i;
  641. unsigned long irqflags;
  642. dev_dbg(chan2dev(chan), "%s: buf_addr=%pad, buf_len=%zd, period_len=%zd, dir=%s, flags=0x%lx\n",
  643. __func__, &buf_addr, buf_len, period_len,
  644. direction == DMA_MEM_TO_DEV ? "mem2per" : "per2mem", flags);
  645. if (!is_slave_direction(direction)) {
  646. dev_err(chan2dev(chan), "invalid DMA direction\n");
  647. return NULL;
  648. }
  649. if (test_and_set_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status)) {
  650. dev_err(chan2dev(chan), "channel currently used\n");
  651. return NULL;
  652. }
  653. if (at_xdmac_compute_chan_conf(chan, direction))
  654. return NULL;
  655. for (i = 0; i < periods; i++) {
  656. struct at_xdmac_desc *desc = NULL;
  657. spin_lock_irqsave(&atchan->lock, irqflags);
  658. desc = at_xdmac_get_desc(atchan);
  659. if (!desc) {
  660. dev_err(chan2dev(chan), "can't get descriptor\n");
  661. if (first)
  662. list_splice_init(&first->descs_list, &atchan->free_descs_list);
  663. spin_unlock_irqrestore(&atchan->lock, irqflags);
  664. return NULL;
  665. }
  666. spin_unlock_irqrestore(&atchan->lock, irqflags);
  667. dev_dbg(chan2dev(chan),
  668. "%s: desc=0x%p, tx_dma_desc.phys=%pad\n",
  669. __func__, desc, &desc->tx_dma_desc.phys);
  670. if (direction == DMA_DEV_TO_MEM) {
  671. desc->lld.mbr_sa = atchan->sconfig.src_addr;
  672. desc->lld.mbr_da = buf_addr + i * period_len;
  673. } else {
  674. desc->lld.mbr_sa = buf_addr + i * period_len;
  675. desc->lld.mbr_da = atchan->sconfig.dst_addr;
  676. }
  677. desc->lld.mbr_cfg = atchan->cfg;
  678. desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV1
  679. | AT_XDMAC_MBR_UBC_NDEN
  680. | AT_XDMAC_MBR_UBC_NSEN
  681. | period_len >> at_xdmac_get_dwidth(desc->lld.mbr_cfg);
  682. dev_dbg(chan2dev(chan),
  683. "%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x\n",
  684. __func__, &desc->lld.mbr_sa, &desc->lld.mbr_da, desc->lld.mbr_ubc);
  685. /* Chain lld. */
  686. if (prev)
  687. at_xdmac_queue_desc(chan, prev, desc);
  688. prev = desc;
  689. if (!first)
  690. first = desc;
  691. dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",
  692. __func__, desc, first);
  693. list_add_tail(&desc->desc_node, &first->descs_list);
  694. }
  695. at_xdmac_queue_desc(chan, prev, first);
  696. first->tx_dma_desc.flags = flags;
  697. first->xfer_size = buf_len;
  698. first->direction = direction;
  699. return &first->tx_dma_desc;
  700. }
  701. static inline u32 at_xdmac_align_width(struct dma_chan *chan, dma_addr_t addr)
  702. {
  703. u32 width;
  704. /*
  705. * Check address alignment to select the greater data width we
  706. * can use.
  707. *
  708. * Some XDMAC implementations don't provide dword transfer, in
  709. * this case selecting dword has the same behavior as
  710. * selecting word transfers.
  711. */
  712. if (!(addr & 7)) {
  713. width = AT_XDMAC_CC_DWIDTH_DWORD;
  714. dev_dbg(chan2dev(chan), "%s: dwidth: double word\n", __func__);
  715. } else if (!(addr & 3)) {
  716. width = AT_XDMAC_CC_DWIDTH_WORD;
  717. dev_dbg(chan2dev(chan), "%s: dwidth: word\n", __func__);
  718. } else if (!(addr & 1)) {
  719. width = AT_XDMAC_CC_DWIDTH_HALFWORD;
  720. dev_dbg(chan2dev(chan), "%s: dwidth: half word\n", __func__);
  721. } else {
  722. width = AT_XDMAC_CC_DWIDTH_BYTE;
  723. dev_dbg(chan2dev(chan), "%s: dwidth: byte\n", __func__);
  724. }
  725. return width;
  726. }
  727. static struct at_xdmac_desc *
  728. at_xdmac_interleaved_queue_desc(struct dma_chan *chan,
  729. struct at_xdmac_chan *atchan,
  730. struct at_xdmac_desc *prev,
  731. dma_addr_t src, dma_addr_t dst,
  732. struct dma_interleaved_template *xt,
  733. struct data_chunk *chunk)
  734. {
  735. struct at_xdmac_desc *desc;
  736. u32 dwidth;
  737. unsigned long flags;
  738. size_t ublen;
  739. /*
  740. * WARNING: The channel configuration is set here since there is no
  741. * dmaengine_slave_config call in this case. Moreover we don't know the
  742. * direction, it involves we can't dynamically set the source and dest
  743. * interface so we have to use the same one. Only interface 0 allows EBI
  744. * access. Hopefully we can access DDR through both ports (at least on
  745. * SAMA5D4x), so we can use the same interface for source and dest,
  746. * that solves the fact we don't know the direction.
  747. * ERRATA: Even if useless for memory transfers, the PERID has to not
  748. * match the one of another channel. If not, it could lead to spurious
  749. * flag status.
  750. */
  751. u32 chan_cc = AT_XDMAC_CC_PERID(0x3f)
  752. | AT_XDMAC_CC_DIF(0)
  753. | AT_XDMAC_CC_SIF(0)
  754. | AT_XDMAC_CC_MBSIZE_SIXTEEN
  755. | AT_XDMAC_CC_TYPE_MEM_TRAN;
  756. dwidth = at_xdmac_align_width(chan, src | dst | chunk->size);
  757. if (chunk->size >= (AT_XDMAC_MBR_UBC_UBLEN_MAX << dwidth)) {
  758. dev_dbg(chan2dev(chan),
  759. "%s: chunk too big (%zu, max size %lu)...\n",
  760. __func__, chunk->size,
  761. AT_XDMAC_MBR_UBC_UBLEN_MAX << dwidth);
  762. return NULL;
  763. }
  764. if (prev)
  765. dev_dbg(chan2dev(chan),
  766. "Adding items at the end of desc 0x%p\n", prev);
  767. if (xt->src_inc) {
  768. if (xt->src_sgl)
  769. chan_cc |= AT_XDMAC_CC_SAM_UBS_AM;
  770. else
  771. chan_cc |= AT_XDMAC_CC_SAM_INCREMENTED_AM;
  772. }
  773. if (xt->dst_inc) {
  774. if (xt->dst_sgl)
  775. chan_cc |= AT_XDMAC_CC_DAM_UBS_AM;
  776. else
  777. chan_cc |= AT_XDMAC_CC_DAM_INCREMENTED_AM;
  778. }
  779. spin_lock_irqsave(&atchan->lock, flags);
  780. desc = at_xdmac_get_desc(atchan);
  781. spin_unlock_irqrestore(&atchan->lock, flags);
  782. if (!desc) {
  783. dev_err(chan2dev(chan), "can't get descriptor\n");
  784. return NULL;
  785. }
  786. chan_cc |= AT_XDMAC_CC_DWIDTH(dwidth);
  787. ublen = chunk->size >> dwidth;
  788. desc->lld.mbr_sa = src;
  789. desc->lld.mbr_da = dst;
  790. desc->lld.mbr_sus = dmaengine_get_src_icg(xt, chunk);
  791. desc->lld.mbr_dus = dmaengine_get_dst_icg(xt, chunk);
  792. desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV3
  793. | AT_XDMAC_MBR_UBC_NDEN
  794. | AT_XDMAC_MBR_UBC_NSEN
  795. | ublen;
  796. desc->lld.mbr_cfg = chan_cc;
  797. dev_dbg(chan2dev(chan),
  798. "%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x, mbr_cfg=0x%08x\n",
  799. __func__, &desc->lld.mbr_sa, &desc->lld.mbr_da,
  800. desc->lld.mbr_ubc, desc->lld.mbr_cfg);
  801. /* Chain lld. */
  802. if (prev)
  803. at_xdmac_queue_desc(chan, prev, desc);
  804. return desc;
  805. }
  806. static struct dma_async_tx_descriptor *
  807. at_xdmac_prep_interleaved(struct dma_chan *chan,
  808. struct dma_interleaved_template *xt,
  809. unsigned long flags)
  810. {
  811. struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
  812. struct at_xdmac_desc *prev = NULL, *first = NULL;
  813. dma_addr_t dst_addr, src_addr;
  814. size_t src_skip = 0, dst_skip = 0, len = 0;
  815. struct data_chunk *chunk;
  816. int i;
  817. if (!xt || !xt->numf || (xt->dir != DMA_MEM_TO_MEM))
  818. return NULL;
  819. /*
  820. * TODO: Handle the case where we have to repeat a chain of
  821. * descriptors...
  822. */
  823. if ((xt->numf > 1) && (xt->frame_size > 1))
  824. return NULL;
  825. dev_dbg(chan2dev(chan), "%s: src=%pad, dest=%pad, numf=%zu, frame_size=%zu, flags=0x%lx\n",
  826. __func__, &xt->src_start, &xt->dst_start, xt->numf,
  827. xt->frame_size, flags);
  828. src_addr = xt->src_start;
  829. dst_addr = xt->dst_start;
  830. if (xt->numf > 1) {
  831. first = at_xdmac_interleaved_queue_desc(chan, atchan,
  832. NULL,
  833. src_addr, dst_addr,
  834. xt, xt->sgl);
  835. /* Length of the block is (BLEN+1) microblocks. */
  836. for (i = 0; i < xt->numf - 1; i++)
  837. at_xdmac_increment_block_count(chan, first);
  838. dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",
  839. __func__, first, first);
  840. list_add_tail(&first->desc_node, &first->descs_list);
  841. } else {
  842. for (i = 0; i < xt->frame_size; i++) {
  843. size_t src_icg = 0, dst_icg = 0;
  844. struct at_xdmac_desc *desc;
  845. chunk = xt->sgl + i;
  846. dst_icg = dmaengine_get_dst_icg(xt, chunk);
  847. src_icg = dmaengine_get_src_icg(xt, chunk);
  848. src_skip = chunk->size + src_icg;
  849. dst_skip = chunk->size + dst_icg;
  850. dev_dbg(chan2dev(chan),
  851. "%s: chunk size=%zu, src icg=%zu, dst icg=%zu\n",
  852. __func__, chunk->size, src_icg, dst_icg);
  853. desc = at_xdmac_interleaved_queue_desc(chan, atchan,
  854. prev,
  855. src_addr, dst_addr,
  856. xt, chunk);
  857. if (!desc) {
  858. list_splice_init(&first->descs_list,
  859. &atchan->free_descs_list);
  860. return NULL;
  861. }
  862. if (!first)
  863. first = desc;
  864. dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",
  865. __func__, desc, first);
  866. list_add_tail(&desc->desc_node, &first->descs_list);
  867. if (xt->src_sgl)
  868. src_addr += src_skip;
  869. if (xt->dst_sgl)
  870. dst_addr += dst_skip;
  871. len += chunk->size;
  872. prev = desc;
  873. }
  874. }
  875. first->tx_dma_desc.cookie = -EBUSY;
  876. first->tx_dma_desc.flags = flags;
  877. first->xfer_size = len;
  878. return &first->tx_dma_desc;
  879. }
  880. static struct dma_async_tx_descriptor *
  881. at_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
  882. size_t len, unsigned long flags)
  883. {
  884. struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
  885. struct at_xdmac_desc *first = NULL, *prev = NULL;
  886. size_t remaining_size = len, xfer_size = 0, ublen;
  887. dma_addr_t src_addr = src, dst_addr = dest;
  888. u32 dwidth;
  889. /*
  890. * WARNING: We don't know the direction, it involves we can't
  891. * dynamically set the source and dest interface so we have to use the
  892. * same one. Only interface 0 allows EBI access. Hopefully we can
  893. * access DDR through both ports (at least on SAMA5D4x), so we can use
  894. * the same interface for source and dest, that solves the fact we
  895. * don't know the direction.
  896. * ERRATA: Even if useless for memory transfers, the PERID has to not
  897. * match the one of another channel. If not, it could lead to spurious
  898. * flag status.
  899. */
  900. u32 chan_cc = AT_XDMAC_CC_PERID(0x3f)
  901. | AT_XDMAC_CC_DAM_INCREMENTED_AM
  902. | AT_XDMAC_CC_SAM_INCREMENTED_AM
  903. | AT_XDMAC_CC_DIF(0)
  904. | AT_XDMAC_CC_SIF(0)
  905. | AT_XDMAC_CC_MBSIZE_SIXTEEN
  906. | AT_XDMAC_CC_TYPE_MEM_TRAN;
  907. unsigned long irqflags;
  908. dev_dbg(chan2dev(chan), "%s: src=%pad, dest=%pad, len=%zd, flags=0x%lx\n",
  909. __func__, &src, &dest, len, flags);
  910. if (unlikely(!len))
  911. return NULL;
  912. dwidth = at_xdmac_align_width(chan, src_addr | dst_addr);
  913. /* Prepare descriptors. */
  914. while (remaining_size) {
  915. struct at_xdmac_desc *desc = NULL;
  916. dev_dbg(chan2dev(chan), "%s: remaining_size=%zu\n", __func__, remaining_size);
  917. spin_lock_irqsave(&atchan->lock, irqflags);
  918. desc = at_xdmac_get_desc(atchan);
  919. spin_unlock_irqrestore(&atchan->lock, irqflags);
  920. if (!desc) {
  921. dev_err(chan2dev(chan), "can't get descriptor\n");
  922. if (first)
  923. list_splice_init(&first->descs_list, &atchan->free_descs_list);
  924. return NULL;
  925. }
  926. /* Update src and dest addresses. */
  927. src_addr += xfer_size;
  928. dst_addr += xfer_size;
  929. if (remaining_size >= AT_XDMAC_MBR_UBC_UBLEN_MAX << dwidth)
  930. xfer_size = AT_XDMAC_MBR_UBC_UBLEN_MAX << dwidth;
  931. else
  932. xfer_size = remaining_size;
  933. dev_dbg(chan2dev(chan), "%s: xfer_size=%zu\n", __func__, xfer_size);
  934. /* Check remaining length and change data width if needed. */
  935. dwidth = at_xdmac_align_width(chan,
  936. src_addr | dst_addr | xfer_size);
  937. chan_cc &= ~AT_XDMAC_CC_DWIDTH_MASK;
  938. chan_cc |= AT_XDMAC_CC_DWIDTH(dwidth);
  939. ublen = xfer_size >> dwidth;
  940. remaining_size -= xfer_size;
  941. desc->lld.mbr_sa = src_addr;
  942. desc->lld.mbr_da = dst_addr;
  943. desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV2
  944. | AT_XDMAC_MBR_UBC_NDEN
  945. | AT_XDMAC_MBR_UBC_NSEN
  946. | ublen;
  947. desc->lld.mbr_cfg = chan_cc;
  948. dev_dbg(chan2dev(chan),
  949. "%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x, mbr_cfg=0x%08x\n",
  950. __func__, &desc->lld.mbr_sa, &desc->lld.mbr_da, desc->lld.mbr_ubc, desc->lld.mbr_cfg);
  951. /* Chain lld. */
  952. if (prev)
  953. at_xdmac_queue_desc(chan, prev, desc);
  954. prev = desc;
  955. if (!first)
  956. first = desc;
  957. dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",
  958. __func__, desc, first);
  959. list_add_tail(&desc->desc_node, &first->descs_list);
  960. }
  961. first->tx_dma_desc.flags = flags;
  962. first->xfer_size = len;
  963. return &first->tx_dma_desc;
  964. }
  965. static struct at_xdmac_desc *at_xdmac_memset_create_desc(struct dma_chan *chan,
  966. struct at_xdmac_chan *atchan,
  967. dma_addr_t dst_addr,
  968. size_t len,
  969. int value)
  970. {
  971. struct at_xdmac_desc *desc;
  972. unsigned long flags;
  973. size_t ublen;
  974. u32 dwidth;
  975. /*
  976. * WARNING: The channel configuration is set here since there is no
  977. * dmaengine_slave_config call in this case. Moreover we don't know the
  978. * direction, it involves we can't dynamically set the source and dest
  979. * interface so we have to use the same one. Only interface 0 allows EBI
  980. * access. Hopefully we can access DDR through both ports (at least on
  981. * SAMA5D4x), so we can use the same interface for source and dest,
  982. * that solves the fact we don't know the direction.
  983. * ERRATA: Even if useless for memory transfers, the PERID has to not
  984. * match the one of another channel. If not, it could lead to spurious
  985. * flag status.
  986. */
  987. u32 chan_cc = AT_XDMAC_CC_PERID(0x3f)
  988. | AT_XDMAC_CC_DAM_UBS_AM
  989. | AT_XDMAC_CC_SAM_INCREMENTED_AM
  990. | AT_XDMAC_CC_DIF(0)
  991. | AT_XDMAC_CC_SIF(0)
  992. | AT_XDMAC_CC_MBSIZE_SIXTEEN
  993. | AT_XDMAC_CC_MEMSET_HW_MODE
  994. | AT_XDMAC_CC_TYPE_MEM_TRAN;
  995. dwidth = at_xdmac_align_width(chan, dst_addr);
  996. if (len >= (AT_XDMAC_MBR_UBC_UBLEN_MAX << dwidth)) {
  997. dev_err(chan2dev(chan),
  998. "%s: Transfer too large, aborting...\n",
  999. __func__);
  1000. return NULL;
  1001. }
  1002. spin_lock_irqsave(&atchan->lock, flags);
  1003. desc = at_xdmac_get_desc(atchan);
  1004. spin_unlock_irqrestore(&atchan->lock, flags);
  1005. if (!desc) {
  1006. dev_err(chan2dev(chan), "can't get descriptor\n");
  1007. return NULL;
  1008. }
  1009. chan_cc |= AT_XDMAC_CC_DWIDTH(dwidth);
  1010. ublen = len >> dwidth;
  1011. desc->lld.mbr_da = dst_addr;
  1012. desc->lld.mbr_ds = value;
  1013. desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV3
  1014. | AT_XDMAC_MBR_UBC_NDEN
  1015. | AT_XDMAC_MBR_UBC_NSEN
  1016. | ublen;
  1017. desc->lld.mbr_cfg = chan_cc;
  1018. dev_dbg(chan2dev(chan),
  1019. "%s: lld: mbr_da=%pad, mbr_ds=0x%08x, mbr_ubc=0x%08x, mbr_cfg=0x%08x\n",
  1020. __func__, &desc->lld.mbr_da, desc->lld.mbr_ds, desc->lld.mbr_ubc,
  1021. desc->lld.mbr_cfg);
  1022. return desc;
  1023. }
  1024. static struct dma_async_tx_descriptor *
  1025. at_xdmac_prep_dma_memset(struct dma_chan *chan, dma_addr_t dest, int value,
  1026. size_t len, unsigned long flags)
  1027. {
  1028. struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
  1029. struct at_xdmac_desc *desc;
  1030. dev_dbg(chan2dev(chan), "%s: dest=%pad, len=%zu, pattern=0x%x, flags=0x%lx\n",
  1031. __func__, &dest, len, value, flags);
  1032. if (unlikely(!len))
  1033. return NULL;
  1034. desc = at_xdmac_memset_create_desc(chan, atchan, dest, len, value);
  1035. list_add_tail(&desc->desc_node, &desc->descs_list);
  1036. desc->tx_dma_desc.cookie = -EBUSY;
  1037. desc->tx_dma_desc.flags = flags;
  1038. desc->xfer_size = len;
  1039. return &desc->tx_dma_desc;
  1040. }
  1041. static struct dma_async_tx_descriptor *
  1042. at_xdmac_prep_dma_memset_sg(struct dma_chan *chan, struct scatterlist *sgl,
  1043. unsigned int sg_len, int value,
  1044. unsigned long flags)
  1045. {
  1046. struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
  1047. struct at_xdmac_desc *desc, *pdesc = NULL,
  1048. *ppdesc = NULL, *first = NULL;
  1049. struct scatterlist *sg, *psg = NULL, *ppsg = NULL;
  1050. size_t stride = 0, pstride = 0, len = 0;
  1051. int i;
  1052. if (!sgl)
  1053. return NULL;
  1054. dev_dbg(chan2dev(chan), "%s: sg_len=%d, value=0x%x, flags=0x%lx\n",
  1055. __func__, sg_len, value, flags);
  1056. /* Prepare descriptors. */
  1057. for_each_sg(sgl, sg, sg_len, i) {
  1058. dev_dbg(chan2dev(chan), "%s: dest=%pad, len=%d, pattern=0x%x, flags=0x%lx\n",
  1059. __func__, &sg_dma_address(sg), sg_dma_len(sg),
  1060. value, flags);
  1061. desc = at_xdmac_memset_create_desc(chan, atchan,
  1062. sg_dma_address(sg),
  1063. sg_dma_len(sg),
  1064. value);
  1065. if (!desc && first)
  1066. list_splice_init(&first->descs_list,
  1067. &atchan->free_descs_list);
  1068. if (!first)
  1069. first = desc;
  1070. /* Update our strides */
  1071. pstride = stride;
  1072. if (psg)
  1073. stride = sg_dma_address(sg) -
  1074. (sg_dma_address(psg) + sg_dma_len(psg));
  1075. /*
  1076. * The scatterlist API gives us only the address and
  1077. * length of each elements.
  1078. *
  1079. * Unfortunately, we don't have the stride, which we
  1080. * will need to compute.
  1081. *
  1082. * That make us end up in a situation like this one:
  1083. * len stride len stride len
  1084. * +-------+ +-------+ +-------+
  1085. * | N-2 | | N-1 | | N |
  1086. * +-------+ +-------+ +-------+
  1087. *
  1088. * We need all these three elements (N-2, N-1 and N)
  1089. * to actually take the decision on whether we need to
  1090. * queue N-1 or reuse N-2.
  1091. *
  1092. * We will only consider N if it is the last element.
  1093. */
  1094. if (ppdesc && pdesc) {
  1095. if ((stride == pstride) &&
  1096. (sg_dma_len(ppsg) == sg_dma_len(psg))) {
  1097. dev_dbg(chan2dev(chan),
  1098. "%s: desc 0x%p can be merged with desc 0x%p\n",
  1099. __func__, pdesc, ppdesc);
  1100. /*
  1101. * Increment the block count of the
  1102. * N-2 descriptor
  1103. */
  1104. at_xdmac_increment_block_count(chan, ppdesc);
  1105. ppdesc->lld.mbr_dus = stride;
  1106. /*
  1107. * Put back the N-1 descriptor in the
  1108. * free descriptor list
  1109. */
  1110. list_add_tail(&pdesc->desc_node,
  1111. &atchan->free_descs_list);
  1112. /*
  1113. * Make our N-1 descriptor pointer
  1114. * point to the N-2 since they were
  1115. * actually merged.
  1116. */
  1117. pdesc = ppdesc;
  1118. /*
  1119. * Rule out the case where we don't have
  1120. * pstride computed yet (our second sg
  1121. * element)
  1122. *
  1123. * We also want to catch the case where there
  1124. * would be a negative stride,
  1125. */
  1126. } else if (pstride ||
  1127. sg_dma_address(sg) < sg_dma_address(psg)) {
  1128. /*
  1129. * Queue the N-1 descriptor after the
  1130. * N-2
  1131. */
  1132. at_xdmac_queue_desc(chan, ppdesc, pdesc);
  1133. /*
  1134. * Add the N-1 descriptor to the list
  1135. * of the descriptors used for this
  1136. * transfer
  1137. */
  1138. list_add_tail(&desc->desc_node,
  1139. &first->descs_list);
  1140. dev_dbg(chan2dev(chan),
  1141. "%s: add desc 0x%p to descs_list 0x%p\n",
  1142. __func__, desc, first);
  1143. }
  1144. }
  1145. /*
  1146. * If we are the last element, just see if we have the
  1147. * same size than the previous element.
  1148. *
  1149. * If so, we can merge it with the previous descriptor
  1150. * since we don't care about the stride anymore.
  1151. */
  1152. if ((i == (sg_len - 1)) &&
  1153. sg_dma_len(psg) == sg_dma_len(sg)) {
  1154. dev_dbg(chan2dev(chan),
  1155. "%s: desc 0x%p can be merged with desc 0x%p\n",
  1156. __func__, desc, pdesc);
  1157. /*
  1158. * Increment the block count of the N-1
  1159. * descriptor
  1160. */
  1161. at_xdmac_increment_block_count(chan, pdesc);
  1162. pdesc->lld.mbr_dus = stride;
  1163. /*
  1164. * Put back the N descriptor in the free
  1165. * descriptor list
  1166. */
  1167. list_add_tail(&desc->desc_node,
  1168. &atchan->free_descs_list);
  1169. }
  1170. /* Update our descriptors */
  1171. ppdesc = pdesc;
  1172. pdesc = desc;
  1173. /* Update our scatter pointers */
  1174. ppsg = psg;
  1175. psg = sg;
  1176. len += sg_dma_len(sg);
  1177. }
  1178. first->tx_dma_desc.cookie = -EBUSY;
  1179. first->tx_dma_desc.flags = flags;
  1180. first->xfer_size = len;
  1181. return &first->tx_dma_desc;
  1182. }
  1183. static enum dma_status
  1184. at_xdmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
  1185. struct dma_tx_state *txstate)
  1186. {
  1187. struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
  1188. struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device);
  1189. struct at_xdmac_desc *desc, *_desc, *iter;
  1190. struct list_head *descs_list;
  1191. enum dma_status ret;
  1192. int residue, retry;
  1193. u32 cur_nda, check_nda, cur_ubc, mask, value;
  1194. u8 dwidth = 0;
  1195. unsigned long flags;
  1196. bool initd;
  1197. ret = dma_cookie_status(chan, cookie, txstate);
  1198. if (ret == DMA_COMPLETE)
  1199. return ret;
  1200. if (!txstate)
  1201. return ret;
  1202. spin_lock_irqsave(&atchan->lock, flags);
  1203. desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc, xfer_node);
  1204. /*
  1205. * If the transfer has not been started yet, don't need to compute the
  1206. * residue, it's the transfer length.
  1207. */
  1208. if (!desc->active_xfer) {
  1209. dma_set_residue(txstate, desc->xfer_size);
  1210. goto spin_unlock;
  1211. }
  1212. residue = desc->xfer_size;
  1213. /*
  1214. * Flush FIFO: only relevant when the transfer is source peripheral
  1215. * synchronized. Flush is needed before reading CUBC because data in
  1216. * the FIFO are not reported by CUBC. Reporting a residue of the
  1217. * transfer length while we have data in FIFO can cause issue.
  1218. * Usecase: atmel USART has a timeout which means I have received
  1219. * characters but there is no more character received for a while. On
  1220. * timeout, it requests the residue. If the data are in the DMA FIFO,
  1221. * we will return a residue of the transfer length. It means no data
  1222. * received. If an application is waiting for these data, it will hang
  1223. * since we won't have another USART timeout without receiving new
  1224. * data.
  1225. */
  1226. mask = AT_XDMAC_CC_TYPE | AT_XDMAC_CC_DSYNC;
  1227. value = AT_XDMAC_CC_TYPE_PER_TRAN | AT_XDMAC_CC_DSYNC_PER2MEM;
  1228. if ((desc->lld.mbr_cfg & mask) == value) {
  1229. at_xdmac_write(atxdmac, AT_XDMAC_GSWF, atchan->mask);
  1230. while (!(at_xdmac_chan_read(atchan, AT_XDMAC_CIS) & AT_XDMAC_CIS_FIS))
  1231. cpu_relax();
  1232. }
  1233. /*
  1234. * The easiest way to compute the residue should be to pause the DMA
  1235. * but doing this can lead to miss some data as some devices don't
  1236. * have FIFO.
  1237. * We need to read several registers because:
  1238. * - DMA is running therefore a descriptor change is possible while
  1239. * reading these registers
  1240. * - When the block transfer is done, the value of the CUBC register
  1241. * is set to its initial value until the fetch of the next descriptor.
  1242. * This value will corrupt the residue calculation so we have to skip
  1243. * it.
  1244. *
  1245. * INITD -------- ------------
  1246. * |____________________|
  1247. * _______________________ _______________
  1248. * NDA @desc2 \/ @desc3
  1249. * _______________________/\_______________
  1250. * __________ ___________ _______________
  1251. * CUBC 0 \/ MAX desc1 \/ MAX desc2
  1252. * __________/\___________/\_______________
  1253. *
  1254. * Since descriptors are aligned on 64 bits, we can assume that
  1255. * the update of NDA and CUBC is atomic.
  1256. * Memory barriers are used to ensure the read order of the registers.
  1257. * A max number of retries is set because unlikely it could never ends.
  1258. */
  1259. for (retry = 0; retry < AT_XDMAC_RESIDUE_MAX_RETRIES; retry++) {
  1260. check_nda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA) & 0xfffffffc;
  1261. rmb();
  1262. cur_ubc = at_xdmac_chan_read(atchan, AT_XDMAC_CUBC);
  1263. rmb();
  1264. initd = !!(at_xdmac_chan_read(atchan, AT_XDMAC_CC) & AT_XDMAC_CC_INITD);
  1265. rmb();
  1266. cur_nda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA) & 0xfffffffc;
  1267. rmb();
  1268. if ((check_nda == cur_nda) && initd)
  1269. break;
  1270. }
  1271. if (unlikely(retry >= AT_XDMAC_RESIDUE_MAX_RETRIES)) {
  1272. ret = DMA_ERROR;
  1273. goto spin_unlock;
  1274. }
  1275. /*
  1276. * Flush FIFO: only relevant when the transfer is source peripheral
  1277. * synchronized. Another flush is needed here because CUBC is updated
  1278. * when the controller sends the data write command. It can lead to
  1279. * report data that are not written in the memory or the device. The
  1280. * FIFO flush ensures that data are really written.
  1281. */
  1282. if ((desc->lld.mbr_cfg & mask) == value) {
  1283. at_xdmac_write(atxdmac, AT_XDMAC_GSWF, atchan->mask);
  1284. while (!(at_xdmac_chan_read(atchan, AT_XDMAC_CIS) & AT_XDMAC_CIS_FIS))
  1285. cpu_relax();
  1286. }
  1287. /*
  1288. * Remove size of all microblocks already transferred and the current
  1289. * one. Then add the remaining size to transfer of the current
  1290. * microblock.
  1291. */
  1292. descs_list = &desc->descs_list;
  1293. list_for_each_entry_safe(iter, _desc, descs_list, desc_node) {
  1294. dwidth = at_xdmac_get_dwidth(iter->lld.mbr_cfg);
  1295. residue -= (iter->lld.mbr_ubc & 0xffffff) << dwidth;
  1296. if ((iter->lld.mbr_nda & 0xfffffffc) == cur_nda) {
  1297. desc = iter;
  1298. break;
  1299. }
  1300. }
  1301. residue += cur_ubc << dwidth;
  1302. dma_set_residue(txstate, residue);
  1303. dev_dbg(chan2dev(chan),
  1304. "%s: desc=0x%p, tx_dma_desc.phys=%pad, tx_status=%d, cookie=%d, residue=%d\n",
  1305. __func__, desc, &desc->tx_dma_desc.phys, ret, cookie, residue);
  1306. spin_unlock:
  1307. spin_unlock_irqrestore(&atchan->lock, flags);
  1308. return ret;
  1309. }
  1310. /* Call must be protected by lock. */
  1311. static void at_xdmac_remove_xfer(struct at_xdmac_chan *atchan,
  1312. struct at_xdmac_desc *desc)
  1313. {
  1314. dev_dbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc);
  1315. /*
  1316. * Remove the transfer from the transfer list then move the transfer
  1317. * descriptors into the free descriptors list.
  1318. */
  1319. list_del(&desc->xfer_node);
  1320. list_splice_init(&desc->descs_list, &atchan->free_descs_list);
  1321. }
  1322. static void at_xdmac_advance_work(struct at_xdmac_chan *atchan)
  1323. {
  1324. struct at_xdmac_desc *desc;
  1325. /*
  1326. * If channel is enabled, do nothing, advance_work will be triggered
  1327. * after the interruption.
  1328. */
  1329. if (!at_xdmac_chan_is_enabled(atchan) && !list_empty(&atchan->xfers_list)) {
  1330. desc = list_first_entry(&atchan->xfers_list,
  1331. struct at_xdmac_desc,
  1332. xfer_node);
  1333. dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc);
  1334. if (!desc->active_xfer)
  1335. at_xdmac_start_xfer(atchan, desc);
  1336. }
  1337. }
  1338. static void at_xdmac_handle_cyclic(struct at_xdmac_chan *atchan)
  1339. {
  1340. struct at_xdmac_desc *desc;
  1341. struct dma_async_tx_descriptor *txd;
  1342. spin_lock_irq(&atchan->lock);
  1343. if (list_empty(&atchan->xfers_list)) {
  1344. spin_unlock_irq(&atchan->lock);
  1345. return;
  1346. }
  1347. desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc,
  1348. xfer_node);
  1349. spin_unlock_irq(&atchan->lock);
  1350. txd = &desc->tx_dma_desc;
  1351. if (txd->flags & DMA_PREP_INTERRUPT)
  1352. dmaengine_desc_get_callback_invoke(txd, NULL);
  1353. }
  1354. static void at_xdmac_handle_error(struct at_xdmac_chan *atchan)
  1355. {
  1356. struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device);
  1357. struct at_xdmac_desc *bad_desc;
  1358. /*
  1359. * The descriptor currently at the head of the active list is
  1360. * broken. Since we don't have any way to report errors, we'll
  1361. * just have to scream loudly and try to continue with other
  1362. * descriptors queued (if any).
  1363. */
  1364. if (atchan->irq_status & AT_XDMAC_CIS_RBEIS)
  1365. dev_err(chan2dev(&atchan->chan), "read bus error!!!");
  1366. if (atchan->irq_status & AT_XDMAC_CIS_WBEIS)
  1367. dev_err(chan2dev(&atchan->chan), "write bus error!!!");
  1368. if (atchan->irq_status & AT_XDMAC_CIS_ROIS)
  1369. dev_err(chan2dev(&atchan->chan), "request overflow error!!!");
  1370. spin_lock_irq(&atchan->lock);
  1371. /* Channel must be disabled first as it's not done automatically */
  1372. at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);
  1373. while (at_xdmac_read(atxdmac, AT_XDMAC_GS) & atchan->mask)
  1374. cpu_relax();
  1375. bad_desc = list_first_entry(&atchan->xfers_list,
  1376. struct at_xdmac_desc,
  1377. xfer_node);
  1378. spin_unlock_irq(&atchan->lock);
  1379. /* Print bad descriptor's details if needed */
  1380. dev_dbg(chan2dev(&atchan->chan),
  1381. "%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x\n",
  1382. __func__, &bad_desc->lld.mbr_sa, &bad_desc->lld.mbr_da,
  1383. bad_desc->lld.mbr_ubc);
  1384. /* Then continue with usual descriptor management */
  1385. }
  1386. static void at_xdmac_tasklet(struct tasklet_struct *t)
  1387. {
  1388. struct at_xdmac_chan *atchan = from_tasklet(atchan, t, tasklet);
  1389. struct at_xdmac_desc *desc;
  1390. u32 error_mask;
  1391. dev_dbg(chan2dev(&atchan->chan), "%s: status=0x%08x\n",
  1392. __func__, atchan->irq_status);
  1393. error_mask = AT_XDMAC_CIS_RBEIS
  1394. | AT_XDMAC_CIS_WBEIS
  1395. | AT_XDMAC_CIS_ROIS;
  1396. if (at_xdmac_chan_is_cyclic(atchan)) {
  1397. at_xdmac_handle_cyclic(atchan);
  1398. } else if ((atchan->irq_status & AT_XDMAC_CIS_LIS)
  1399. || (atchan->irq_status & error_mask)) {
  1400. struct dma_async_tx_descriptor *txd;
  1401. if (atchan->irq_status & error_mask)
  1402. at_xdmac_handle_error(atchan);
  1403. spin_lock_irq(&atchan->lock);
  1404. desc = list_first_entry(&atchan->xfers_list,
  1405. struct at_xdmac_desc,
  1406. xfer_node);
  1407. dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc);
  1408. if (!desc->active_xfer) {
  1409. dev_err(chan2dev(&atchan->chan), "Xfer not active: exiting");
  1410. spin_unlock_irq(&atchan->lock);
  1411. return;
  1412. }
  1413. txd = &desc->tx_dma_desc;
  1414. at_xdmac_remove_xfer(atchan, desc);
  1415. spin_unlock_irq(&atchan->lock);
  1416. dma_cookie_complete(txd);
  1417. if (txd->flags & DMA_PREP_INTERRUPT)
  1418. dmaengine_desc_get_callback_invoke(txd, NULL);
  1419. dma_run_dependencies(txd);
  1420. spin_lock_irq(&atchan->lock);
  1421. at_xdmac_advance_work(atchan);
  1422. spin_unlock_irq(&atchan->lock);
  1423. }
  1424. }
  1425. static irqreturn_t at_xdmac_interrupt(int irq, void *dev_id)
  1426. {
  1427. struct at_xdmac *atxdmac = (struct at_xdmac *)dev_id;
  1428. struct at_xdmac_chan *atchan;
  1429. u32 imr, status, pending;
  1430. u32 chan_imr, chan_status;
  1431. int i, ret = IRQ_NONE;
  1432. do {
  1433. imr = at_xdmac_read(atxdmac, AT_XDMAC_GIM);
  1434. status = at_xdmac_read(atxdmac, AT_XDMAC_GIS);
  1435. pending = status & imr;
  1436. dev_vdbg(atxdmac->dma.dev,
  1437. "%s: status=0x%08x, imr=0x%08x, pending=0x%08x\n",
  1438. __func__, status, imr, pending);
  1439. if (!pending)
  1440. break;
  1441. /* We have to find which channel has generated the interrupt. */
  1442. for (i = 0; i < atxdmac->dma.chancnt; i++) {
  1443. if (!((1 << i) & pending))
  1444. continue;
  1445. atchan = &atxdmac->chan[i];
  1446. chan_imr = at_xdmac_chan_read(atchan, AT_XDMAC_CIM);
  1447. chan_status = at_xdmac_chan_read(atchan, AT_XDMAC_CIS);
  1448. atchan->irq_status = chan_status & chan_imr;
  1449. dev_vdbg(atxdmac->dma.dev,
  1450. "%s: chan%d: imr=0x%x, status=0x%x\n",
  1451. __func__, i, chan_imr, chan_status);
  1452. dev_vdbg(chan2dev(&atchan->chan),
  1453. "%s: CC=0x%08x CNDA=0x%08x, CNDC=0x%08x, CSA=0x%08x, CDA=0x%08x, CUBC=0x%08x\n",
  1454. __func__,
  1455. at_xdmac_chan_read(atchan, AT_XDMAC_CC),
  1456. at_xdmac_chan_read(atchan, AT_XDMAC_CNDA),
  1457. at_xdmac_chan_read(atchan, AT_XDMAC_CNDC),
  1458. at_xdmac_chan_read(atchan, AT_XDMAC_CSA),
  1459. at_xdmac_chan_read(atchan, AT_XDMAC_CDA),
  1460. at_xdmac_chan_read(atchan, AT_XDMAC_CUBC));
  1461. if (atchan->irq_status & (AT_XDMAC_CIS_RBEIS | AT_XDMAC_CIS_WBEIS))
  1462. at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);
  1463. tasklet_schedule(&atchan->tasklet);
  1464. ret = IRQ_HANDLED;
  1465. }
  1466. } while (pending);
  1467. return ret;
  1468. }
  1469. static void at_xdmac_issue_pending(struct dma_chan *chan)
  1470. {
  1471. struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
  1472. unsigned long flags;
  1473. dev_dbg(chan2dev(&atchan->chan), "%s\n", __func__);
  1474. spin_lock_irqsave(&atchan->lock, flags);
  1475. at_xdmac_advance_work(atchan);
  1476. spin_unlock_irqrestore(&atchan->lock, flags);
  1477. return;
  1478. }
  1479. static int at_xdmac_device_config(struct dma_chan *chan,
  1480. struct dma_slave_config *config)
  1481. {
  1482. struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
  1483. int ret;
  1484. unsigned long flags;
  1485. dev_dbg(chan2dev(chan), "%s\n", __func__);
  1486. spin_lock_irqsave(&atchan->lock, flags);
  1487. ret = at_xdmac_set_slave_config(chan, config);
  1488. spin_unlock_irqrestore(&atchan->lock, flags);
  1489. return ret;
  1490. }
  1491. static int at_xdmac_device_pause(struct dma_chan *chan)
  1492. {
  1493. struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
  1494. struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device);
  1495. unsigned long flags;
  1496. dev_dbg(chan2dev(chan), "%s\n", __func__);
  1497. if (test_and_set_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status))
  1498. return 0;
  1499. spin_lock_irqsave(&atchan->lock, flags);
  1500. at_xdmac_write(atxdmac, AT_XDMAC_GRWS, atchan->mask);
  1501. while (at_xdmac_chan_read(atchan, AT_XDMAC_CC)
  1502. & (AT_XDMAC_CC_WRIP | AT_XDMAC_CC_RDIP))
  1503. cpu_relax();
  1504. spin_unlock_irqrestore(&atchan->lock, flags);
  1505. return 0;
  1506. }
  1507. static int at_xdmac_device_resume(struct dma_chan *chan)
  1508. {
  1509. struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
  1510. struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device);
  1511. unsigned long flags;
  1512. dev_dbg(chan2dev(chan), "%s\n", __func__);
  1513. spin_lock_irqsave(&atchan->lock, flags);
  1514. if (!at_xdmac_chan_is_paused(atchan)) {
  1515. spin_unlock_irqrestore(&atchan->lock, flags);
  1516. return 0;
  1517. }
  1518. at_xdmac_write(atxdmac, AT_XDMAC_GRWR, atchan->mask);
  1519. clear_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status);
  1520. spin_unlock_irqrestore(&atchan->lock, flags);
  1521. return 0;
  1522. }
  1523. static int at_xdmac_device_terminate_all(struct dma_chan *chan)
  1524. {
  1525. struct at_xdmac_desc *desc, *_desc;
  1526. struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
  1527. struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device);
  1528. unsigned long flags;
  1529. dev_dbg(chan2dev(chan), "%s\n", __func__);
  1530. spin_lock_irqsave(&atchan->lock, flags);
  1531. at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);
  1532. while (at_xdmac_read(atxdmac, AT_XDMAC_GS) & atchan->mask)
  1533. cpu_relax();
  1534. /* Cancel all pending transfers. */
  1535. list_for_each_entry_safe(desc, _desc, &atchan->xfers_list, xfer_node)
  1536. at_xdmac_remove_xfer(atchan, desc);
  1537. clear_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status);
  1538. clear_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status);
  1539. spin_unlock_irqrestore(&atchan->lock, flags);
  1540. return 0;
  1541. }
  1542. static int at_xdmac_alloc_chan_resources(struct dma_chan *chan)
  1543. {
  1544. struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
  1545. struct at_xdmac_desc *desc;
  1546. int i;
  1547. if (at_xdmac_chan_is_enabled(atchan)) {
  1548. dev_err(chan2dev(chan),
  1549. "can't allocate channel resources (channel enabled)\n");
  1550. return -EIO;
  1551. }
  1552. if (!list_empty(&atchan->free_descs_list)) {
  1553. dev_err(chan2dev(chan),
  1554. "can't allocate channel resources (channel not free from a previous use)\n");
  1555. return -EIO;
  1556. }
  1557. for (i = 0; i < init_nr_desc_per_channel; i++) {
  1558. desc = at_xdmac_alloc_desc(chan, GFP_KERNEL);
  1559. if (!desc) {
  1560. dev_warn(chan2dev(chan),
  1561. "only %d descriptors have been allocated\n", i);
  1562. break;
  1563. }
  1564. list_add_tail(&desc->desc_node, &atchan->free_descs_list);
  1565. }
  1566. dma_cookie_init(chan);
  1567. dev_dbg(chan2dev(chan), "%s: allocated %d descriptors\n", __func__, i);
  1568. return i;
  1569. }
  1570. static void at_xdmac_free_chan_resources(struct dma_chan *chan)
  1571. {
  1572. struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
  1573. struct at_xdmac *atxdmac = to_at_xdmac(chan->device);
  1574. struct at_xdmac_desc *desc, *_desc;
  1575. list_for_each_entry_safe(desc, _desc, &atchan->free_descs_list, desc_node) {
  1576. dev_dbg(chan2dev(chan), "%s: freeing descriptor %p\n", __func__, desc);
  1577. list_del(&desc->desc_node);
  1578. dma_pool_free(atxdmac->at_xdmac_desc_pool, desc, desc->tx_dma_desc.phys);
  1579. }
  1580. return;
  1581. }
  1582. #ifdef CONFIG_PM
  1583. static int atmel_xdmac_prepare(struct device *dev)
  1584. {
  1585. struct at_xdmac *atxdmac = dev_get_drvdata(dev);
  1586. struct dma_chan *chan, *_chan;
  1587. list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) {
  1588. struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
  1589. /* Wait for transfer completion, except in cyclic case. */
  1590. if (at_xdmac_chan_is_enabled(atchan) && !at_xdmac_chan_is_cyclic(atchan))
  1591. return -EAGAIN;
  1592. }
  1593. return 0;
  1594. }
  1595. #else
  1596. # define atmel_xdmac_prepare NULL
  1597. #endif
  1598. #ifdef CONFIG_PM_SLEEP
  1599. static int atmel_xdmac_suspend(struct device *dev)
  1600. {
  1601. struct at_xdmac *atxdmac = dev_get_drvdata(dev);
  1602. struct dma_chan *chan, *_chan;
  1603. list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) {
  1604. struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
  1605. atchan->save_cc = at_xdmac_chan_read(atchan, AT_XDMAC_CC);
  1606. if (at_xdmac_chan_is_cyclic(atchan)) {
  1607. if (!at_xdmac_chan_is_paused(atchan))
  1608. at_xdmac_device_pause(chan);
  1609. atchan->save_cim = at_xdmac_chan_read(atchan, AT_XDMAC_CIM);
  1610. atchan->save_cnda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA);
  1611. atchan->save_cndc = at_xdmac_chan_read(atchan, AT_XDMAC_CNDC);
  1612. }
  1613. }
  1614. atxdmac->save_gim = at_xdmac_read(atxdmac, AT_XDMAC_GIM);
  1615. at_xdmac_off(atxdmac);
  1616. clk_disable_unprepare(atxdmac->clk);
  1617. return 0;
  1618. }
  1619. static int atmel_xdmac_resume(struct device *dev)
  1620. {
  1621. struct at_xdmac *atxdmac = dev_get_drvdata(dev);
  1622. struct at_xdmac_chan *atchan;
  1623. struct dma_chan *chan, *_chan;
  1624. int i;
  1625. int ret;
  1626. ret = clk_prepare_enable(atxdmac->clk);
  1627. if (ret)
  1628. return ret;
  1629. /* Clear pending interrupts. */
  1630. for (i = 0; i < atxdmac->dma.chancnt; i++) {
  1631. atchan = &atxdmac->chan[i];
  1632. while (at_xdmac_chan_read(atchan, AT_XDMAC_CIS))
  1633. cpu_relax();
  1634. }
  1635. at_xdmac_write(atxdmac, AT_XDMAC_GIE, atxdmac->save_gim);
  1636. list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) {
  1637. atchan = to_at_xdmac_chan(chan);
  1638. at_xdmac_chan_write(atchan, AT_XDMAC_CC, atchan->save_cc);
  1639. if (at_xdmac_chan_is_cyclic(atchan)) {
  1640. if (at_xdmac_chan_is_paused(atchan))
  1641. at_xdmac_device_resume(chan);
  1642. at_xdmac_chan_write(atchan, AT_XDMAC_CNDA, atchan->save_cnda);
  1643. at_xdmac_chan_write(atchan, AT_XDMAC_CNDC, atchan->save_cndc);
  1644. at_xdmac_chan_write(atchan, AT_XDMAC_CIE, atchan->save_cim);
  1645. wmb();
  1646. at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask);
  1647. }
  1648. }
  1649. return 0;
  1650. }
  1651. #endif /* CONFIG_PM_SLEEP */
  1652. static int at_xdmac_probe(struct platform_device *pdev)
  1653. {
  1654. struct at_xdmac *atxdmac;
  1655. int irq, size, nr_channels, i, ret;
  1656. void __iomem *base;
  1657. u32 reg;
  1658. irq = platform_get_irq(pdev, 0);
  1659. if (irq < 0)
  1660. return irq;
  1661. base = devm_platform_ioremap_resource(pdev, 0);
  1662. if (IS_ERR(base))
  1663. return PTR_ERR(base);
  1664. /*
  1665. * Read number of xdmac channels, read helper function can't be used
  1666. * since atxdmac is not yet allocated and we need to know the number
  1667. * of channels to do the allocation.
  1668. */
  1669. reg = readl_relaxed(base + AT_XDMAC_GTYPE);
  1670. nr_channels = AT_XDMAC_NB_CH(reg);
  1671. if (nr_channels > AT_XDMAC_MAX_CHAN) {
  1672. dev_err(&pdev->dev, "invalid number of channels (%u)\n",
  1673. nr_channels);
  1674. return -EINVAL;
  1675. }
  1676. size = sizeof(*atxdmac);
  1677. size += nr_channels * sizeof(struct at_xdmac_chan);
  1678. atxdmac = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
  1679. if (!atxdmac) {
  1680. dev_err(&pdev->dev, "can't allocate at_xdmac structure\n");
  1681. return -ENOMEM;
  1682. }
  1683. atxdmac->regs = base;
  1684. atxdmac->irq = irq;
  1685. atxdmac->clk = devm_clk_get(&pdev->dev, "dma_clk");
  1686. if (IS_ERR(atxdmac->clk)) {
  1687. dev_err(&pdev->dev, "can't get dma_clk\n");
  1688. return PTR_ERR(atxdmac->clk);
  1689. }
  1690. /* Do not use dev res to prevent races with tasklet */
  1691. ret = request_irq(atxdmac->irq, at_xdmac_interrupt, 0, "at_xdmac", atxdmac);
  1692. if (ret) {
  1693. dev_err(&pdev->dev, "can't request irq\n");
  1694. return ret;
  1695. }
  1696. ret = clk_prepare_enable(atxdmac->clk);
  1697. if (ret) {
  1698. dev_err(&pdev->dev, "can't prepare or enable clock\n");
  1699. goto err_free_irq;
  1700. }
  1701. atxdmac->at_xdmac_desc_pool =
  1702. dmam_pool_create(dev_name(&pdev->dev), &pdev->dev,
  1703. sizeof(struct at_xdmac_desc), 4, 0);
  1704. if (!atxdmac->at_xdmac_desc_pool) {
  1705. dev_err(&pdev->dev, "no memory for descriptors dma pool\n");
  1706. ret = -ENOMEM;
  1707. goto err_clk_disable;
  1708. }
  1709. dma_cap_set(DMA_CYCLIC, atxdmac->dma.cap_mask);
  1710. dma_cap_set(DMA_INTERLEAVE, atxdmac->dma.cap_mask);
  1711. dma_cap_set(DMA_MEMCPY, atxdmac->dma.cap_mask);
  1712. dma_cap_set(DMA_MEMSET, atxdmac->dma.cap_mask);
  1713. dma_cap_set(DMA_MEMSET_SG, atxdmac->dma.cap_mask);
  1714. dma_cap_set(DMA_SLAVE, atxdmac->dma.cap_mask);
  1715. /*
  1716. * Without DMA_PRIVATE the driver is not able to allocate more than
  1717. * one channel, second allocation fails in private_candidate.
  1718. */
  1719. dma_cap_set(DMA_PRIVATE, atxdmac->dma.cap_mask);
  1720. atxdmac->dma.dev = &pdev->dev;
  1721. atxdmac->dma.device_alloc_chan_resources = at_xdmac_alloc_chan_resources;
  1722. atxdmac->dma.device_free_chan_resources = at_xdmac_free_chan_resources;
  1723. atxdmac->dma.device_tx_status = at_xdmac_tx_status;
  1724. atxdmac->dma.device_issue_pending = at_xdmac_issue_pending;
  1725. atxdmac->dma.device_prep_dma_cyclic = at_xdmac_prep_dma_cyclic;
  1726. atxdmac->dma.device_prep_interleaved_dma = at_xdmac_prep_interleaved;
  1727. atxdmac->dma.device_prep_dma_memcpy = at_xdmac_prep_dma_memcpy;
  1728. atxdmac->dma.device_prep_dma_memset = at_xdmac_prep_dma_memset;
  1729. atxdmac->dma.device_prep_dma_memset_sg = at_xdmac_prep_dma_memset_sg;
  1730. atxdmac->dma.device_prep_slave_sg = at_xdmac_prep_slave_sg;
  1731. atxdmac->dma.device_config = at_xdmac_device_config;
  1732. atxdmac->dma.device_pause = at_xdmac_device_pause;
  1733. atxdmac->dma.device_resume = at_xdmac_device_resume;
  1734. atxdmac->dma.device_terminate_all = at_xdmac_device_terminate_all;
  1735. atxdmac->dma.src_addr_widths = AT_XDMAC_DMA_BUSWIDTHS;
  1736. atxdmac->dma.dst_addr_widths = AT_XDMAC_DMA_BUSWIDTHS;
  1737. atxdmac->dma.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
  1738. atxdmac->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
  1739. /* Disable all chans and interrupts. */
  1740. at_xdmac_off(atxdmac);
  1741. /* Init channels. */
  1742. INIT_LIST_HEAD(&atxdmac->dma.channels);
  1743. for (i = 0; i < nr_channels; i++) {
  1744. struct at_xdmac_chan *atchan = &atxdmac->chan[i];
  1745. atchan->chan.device = &atxdmac->dma;
  1746. list_add_tail(&atchan->chan.device_node,
  1747. &atxdmac->dma.channels);
  1748. atchan->ch_regs = at_xdmac_chan_reg_base(atxdmac, i);
  1749. atchan->mask = 1 << i;
  1750. spin_lock_init(&atchan->lock);
  1751. INIT_LIST_HEAD(&atchan->xfers_list);
  1752. INIT_LIST_HEAD(&atchan->free_descs_list);
  1753. tasklet_setup(&atchan->tasklet, at_xdmac_tasklet);
  1754. /* Clear pending interrupts. */
  1755. while (at_xdmac_chan_read(atchan, AT_XDMAC_CIS))
  1756. cpu_relax();
  1757. }
  1758. platform_set_drvdata(pdev, atxdmac);
  1759. ret = dma_async_device_register(&atxdmac->dma);
  1760. if (ret) {
  1761. dev_err(&pdev->dev, "fail to register DMA engine device\n");
  1762. goto err_clk_disable;
  1763. }
  1764. ret = of_dma_controller_register(pdev->dev.of_node,
  1765. at_xdmac_xlate, atxdmac);
  1766. if (ret) {
  1767. dev_err(&pdev->dev, "could not register of dma controller\n");
  1768. goto err_dma_unregister;
  1769. }
  1770. dev_info(&pdev->dev, "%d channels, mapped at 0x%p\n",
  1771. nr_channels, atxdmac->regs);
  1772. return 0;
  1773. err_dma_unregister:
  1774. dma_async_device_unregister(&atxdmac->dma);
  1775. err_clk_disable:
  1776. clk_disable_unprepare(atxdmac->clk);
  1777. err_free_irq:
  1778. free_irq(atxdmac->irq, atxdmac);
  1779. return ret;
  1780. }
  1781. static int at_xdmac_remove(struct platform_device *pdev)
  1782. {
  1783. struct at_xdmac *atxdmac = (struct at_xdmac *)platform_get_drvdata(pdev);
  1784. int i;
  1785. at_xdmac_off(atxdmac);
  1786. of_dma_controller_free(pdev->dev.of_node);
  1787. dma_async_device_unregister(&atxdmac->dma);
  1788. clk_disable_unprepare(atxdmac->clk);
  1789. free_irq(atxdmac->irq, atxdmac);
  1790. for (i = 0; i < atxdmac->dma.chancnt; i++) {
  1791. struct at_xdmac_chan *atchan = &atxdmac->chan[i];
  1792. tasklet_kill(&atchan->tasklet);
  1793. at_xdmac_free_chan_resources(&atchan->chan);
  1794. }
  1795. return 0;
  1796. }
  1797. static const struct dev_pm_ops atmel_xdmac_dev_pm_ops = {
  1798. .prepare = atmel_xdmac_prepare,
  1799. SET_LATE_SYSTEM_SLEEP_PM_OPS(atmel_xdmac_suspend, atmel_xdmac_resume)
  1800. };
  1801. static const struct of_device_id atmel_xdmac_dt_ids[] = {
  1802. {
  1803. .compatible = "atmel,sama5d4-dma",
  1804. }, {
  1805. /* sentinel */
  1806. }
  1807. };
  1808. MODULE_DEVICE_TABLE(of, atmel_xdmac_dt_ids);
  1809. static struct platform_driver at_xdmac_driver = {
  1810. .probe = at_xdmac_probe,
  1811. .remove = at_xdmac_remove,
  1812. .driver = {
  1813. .name = "at_xdmac",
  1814. .of_match_table = of_match_ptr(atmel_xdmac_dt_ids),
  1815. .pm = &atmel_xdmac_dev_pm_ops,
  1816. }
  1817. };
  1818. static int __init at_xdmac_init(void)
  1819. {
  1820. return platform_driver_probe(&at_xdmac_driver, at_xdmac_probe);
  1821. }
  1822. subsys_initcall(at_xdmac_init);
  1823. MODULE_DESCRIPTION("Atmel Extended DMA Controller driver");
  1824. MODULE_AUTHOR("Ludovic Desroches <ludovic.desroches@atmel.com>");
  1825. MODULE_LICENSE("GPL");