rcar-dmac.c 54 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Renesas R-Car Gen2/Gen3 DMA Controller Driver
  4. *
  5. * Copyright (C) 2014-2019 Renesas Electronics Inc.
  6. *
  7. * Author: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
  8. */
  9. #include <linux/delay.h>
  10. #include <linux/dma-mapping.h>
  11. #include <linux/dmaengine.h>
  12. #include <linux/interrupt.h>
  13. #include <linux/list.h>
  14. #include <linux/module.h>
  15. #include <linux/mutex.h>
  16. #include <linux/of.h>
  17. #include <linux/of_dma.h>
  18. #include <linux/of_platform.h>
  19. #include <linux/platform_device.h>
  20. #include <linux/pm_runtime.h>
  21. #include <linux/slab.h>
  22. #include <linux/spinlock.h>
  23. #include "../dmaengine.h"
  24. /*
  25. * struct rcar_dmac_xfer_chunk - Descriptor for a hardware transfer
  26. * @node: entry in the parent's chunks list
  27. * @src_addr: device source address
  28. * @dst_addr: device destination address
  29. * @size: transfer size in bytes
  30. */
  31. struct rcar_dmac_xfer_chunk {
  32. struct list_head node;
  33. dma_addr_t src_addr;
  34. dma_addr_t dst_addr;
  35. u32 size;
  36. };
  37. /*
  38. * struct rcar_dmac_hw_desc - Hardware descriptor for a transfer chunk
  39. * @sar: value of the SAR register (source address)
  40. * @dar: value of the DAR register (destination address)
  41. * @tcr: value of the TCR register (transfer count)
  42. */
  43. struct rcar_dmac_hw_desc {
  44. u32 sar;
  45. u32 dar;
  46. u32 tcr;
  47. u32 reserved;
  48. } __attribute__((__packed__));
  49. /*
  50. * struct rcar_dmac_desc - R-Car Gen2 DMA Transfer Descriptor
  51. * @async_tx: base DMA asynchronous transaction descriptor
  52. * @direction: direction of the DMA transfer
  53. * @xfer_shift: log2 of the transfer size
  54. * @chcr: value of the channel configuration register for this transfer
  55. * @node: entry in the channel's descriptors lists
  56. * @chunks: list of transfer chunks for this transfer
  57. * @running: the transfer chunk being currently processed
  58. * @nchunks: number of transfer chunks for this transfer
  59. * @hwdescs.use: whether the transfer descriptor uses hardware descriptors
  60. * @hwdescs.mem: hardware descriptors memory for the transfer
  61. * @hwdescs.dma: device address of the hardware descriptors memory
  62. * @hwdescs.size: size of the hardware descriptors in bytes
  63. * @size: transfer size in bytes
  64. * @cyclic: when set indicates that the DMA transfer is cyclic
  65. */
  66. struct rcar_dmac_desc {
  67. struct dma_async_tx_descriptor async_tx;
  68. enum dma_transfer_direction direction;
  69. unsigned int xfer_shift;
  70. u32 chcr;
  71. struct list_head node;
  72. struct list_head chunks;
  73. struct rcar_dmac_xfer_chunk *running;
  74. unsigned int nchunks;
  75. struct {
  76. bool use;
  77. struct rcar_dmac_hw_desc *mem;
  78. dma_addr_t dma;
  79. size_t size;
  80. } hwdescs;
  81. unsigned int size;
  82. bool cyclic;
  83. };
  84. #define to_rcar_dmac_desc(d) container_of(d, struct rcar_dmac_desc, async_tx)
  85. /*
  86. * struct rcar_dmac_desc_page - One page worth of descriptors
  87. * @node: entry in the channel's pages list
  88. * @descs: array of DMA descriptors
  89. * @chunks: array of transfer chunk descriptors
  90. */
  91. struct rcar_dmac_desc_page {
  92. struct list_head node;
  93. union {
  94. struct rcar_dmac_desc descs[0];
  95. struct rcar_dmac_xfer_chunk chunks[0];
  96. };
  97. };
  98. #define RCAR_DMAC_DESCS_PER_PAGE \
  99. ((PAGE_SIZE - offsetof(struct rcar_dmac_desc_page, descs)) / \
  100. sizeof(struct rcar_dmac_desc))
  101. #define RCAR_DMAC_XFER_CHUNKS_PER_PAGE \
  102. ((PAGE_SIZE - offsetof(struct rcar_dmac_desc_page, chunks)) / \
  103. sizeof(struct rcar_dmac_xfer_chunk))
  104. /*
  105. * struct rcar_dmac_chan_slave - Slave configuration
  106. * @slave_addr: slave memory address
  107. * @xfer_size: size (in bytes) of hardware transfers
  108. */
  109. struct rcar_dmac_chan_slave {
  110. phys_addr_t slave_addr;
  111. unsigned int xfer_size;
  112. };
  113. /*
  114. * struct rcar_dmac_chan_map - Map of slave device phys to dma address
  115. * @addr: slave dma address
  116. * @dir: direction of mapping
  117. * @slave: slave configuration that is mapped
  118. */
  119. struct rcar_dmac_chan_map {
  120. dma_addr_t addr;
  121. enum dma_data_direction dir;
  122. struct rcar_dmac_chan_slave slave;
  123. };
  124. /*
  125. * struct rcar_dmac_chan - R-Car Gen2 DMA Controller Channel
  126. * @chan: base DMA channel object
  127. * @iomem: channel I/O memory base
  128. * @index: index of this channel in the controller
  129. * @irq: channel IRQ
  130. * @src: slave memory address and size on the source side
  131. * @dst: slave memory address and size on the destination side
  132. * @mid_rid: hardware MID/RID for the DMA client using this channel
  133. * @lock: protects the channel CHCR register and the desc members
  134. * @desc.free: list of free descriptors
  135. * @desc.pending: list of pending descriptors (submitted with tx_submit)
  136. * @desc.active: list of active descriptors (activated with issue_pending)
  137. * @desc.done: list of completed descriptors
  138. * @desc.wait: list of descriptors waiting for an ack
  139. * @desc.running: the descriptor being processed (a member of the active list)
  140. * @desc.chunks_free: list of free transfer chunk descriptors
  141. * @desc.pages: list of pages used by allocated descriptors
  142. */
  143. struct rcar_dmac_chan {
  144. struct dma_chan chan;
  145. void __iomem *iomem;
  146. unsigned int index;
  147. int irq;
  148. struct rcar_dmac_chan_slave src;
  149. struct rcar_dmac_chan_slave dst;
  150. struct rcar_dmac_chan_map map;
  151. int mid_rid;
  152. spinlock_t lock;
  153. struct {
  154. struct list_head free;
  155. struct list_head pending;
  156. struct list_head active;
  157. struct list_head done;
  158. struct list_head wait;
  159. struct rcar_dmac_desc *running;
  160. struct list_head chunks_free;
  161. struct list_head pages;
  162. } desc;
  163. };
  164. #define to_rcar_dmac_chan(c) container_of(c, struct rcar_dmac_chan, chan)
  165. /*
  166. * struct rcar_dmac - R-Car Gen2 DMA Controller
  167. * @engine: base DMA engine object
  168. * @dev: the hardware device
  169. * @iomem: remapped I/O memory base
  170. * @n_channels: number of available channels
  171. * @channels: array of DMAC channels
  172. * @channels_mask: bitfield of which DMA channels are managed by this driver
  173. * @modules: bitmask of client modules in use
  174. */
  175. struct rcar_dmac {
  176. struct dma_device engine;
  177. struct device *dev;
  178. void __iomem *iomem;
  179. unsigned int n_channels;
  180. struct rcar_dmac_chan *channels;
  181. u32 channels_mask;
  182. DECLARE_BITMAP(modules, 256);
  183. };
  184. #define to_rcar_dmac(d) container_of(d, struct rcar_dmac, engine)
  185. /*
  186. * struct rcar_dmac_of_data - This driver's OF data
  187. * @chan_offset_base: DMAC channels base offset
  188. * @chan_offset_stride: DMAC channels offset stride
  189. */
  190. struct rcar_dmac_of_data {
  191. u32 chan_offset_base;
  192. u32 chan_offset_stride;
  193. };
  194. /* -----------------------------------------------------------------------------
  195. * Registers
  196. */
  197. #define RCAR_DMAISTA 0x0020
  198. #define RCAR_DMASEC 0x0030
  199. #define RCAR_DMAOR 0x0060
  200. #define RCAR_DMAOR_PRI_FIXED (0 << 8)
  201. #define RCAR_DMAOR_PRI_ROUND_ROBIN (3 << 8)
  202. #define RCAR_DMAOR_AE (1 << 2)
  203. #define RCAR_DMAOR_DME (1 << 0)
  204. #define RCAR_DMACHCLR 0x0080
  205. #define RCAR_DMADPSEC 0x00a0
  206. #define RCAR_DMASAR 0x0000
  207. #define RCAR_DMADAR 0x0004
  208. #define RCAR_DMATCR 0x0008
  209. #define RCAR_DMATCR_MASK 0x00ffffff
  210. #define RCAR_DMATSR 0x0028
  211. #define RCAR_DMACHCR 0x000c
  212. #define RCAR_DMACHCR_CAE (1 << 31)
  213. #define RCAR_DMACHCR_CAIE (1 << 30)
  214. #define RCAR_DMACHCR_DPM_DISABLED (0 << 28)
  215. #define RCAR_DMACHCR_DPM_ENABLED (1 << 28)
  216. #define RCAR_DMACHCR_DPM_REPEAT (2 << 28)
  217. #define RCAR_DMACHCR_DPM_INFINITE (3 << 28)
  218. #define RCAR_DMACHCR_RPT_SAR (1 << 27)
  219. #define RCAR_DMACHCR_RPT_DAR (1 << 26)
  220. #define RCAR_DMACHCR_RPT_TCR (1 << 25)
  221. #define RCAR_DMACHCR_DPB (1 << 22)
  222. #define RCAR_DMACHCR_DSE (1 << 19)
  223. #define RCAR_DMACHCR_DSIE (1 << 18)
  224. #define RCAR_DMACHCR_TS_1B ((0 << 20) | (0 << 3))
  225. #define RCAR_DMACHCR_TS_2B ((0 << 20) | (1 << 3))
  226. #define RCAR_DMACHCR_TS_4B ((0 << 20) | (2 << 3))
  227. #define RCAR_DMACHCR_TS_16B ((0 << 20) | (3 << 3))
  228. #define RCAR_DMACHCR_TS_32B ((1 << 20) | (0 << 3))
  229. #define RCAR_DMACHCR_TS_64B ((1 << 20) | (1 << 3))
  230. #define RCAR_DMACHCR_TS_8B ((1 << 20) | (3 << 3))
  231. #define RCAR_DMACHCR_DM_FIXED (0 << 14)
  232. #define RCAR_DMACHCR_DM_INC (1 << 14)
  233. #define RCAR_DMACHCR_DM_DEC (2 << 14)
  234. #define RCAR_DMACHCR_SM_FIXED (0 << 12)
  235. #define RCAR_DMACHCR_SM_INC (1 << 12)
  236. #define RCAR_DMACHCR_SM_DEC (2 << 12)
  237. #define RCAR_DMACHCR_RS_AUTO (4 << 8)
  238. #define RCAR_DMACHCR_RS_DMARS (8 << 8)
  239. #define RCAR_DMACHCR_IE (1 << 2)
  240. #define RCAR_DMACHCR_TE (1 << 1)
  241. #define RCAR_DMACHCR_DE (1 << 0)
  242. #define RCAR_DMATCRB 0x0018
  243. #define RCAR_DMATSRB 0x0038
  244. #define RCAR_DMACHCRB 0x001c
  245. #define RCAR_DMACHCRB_DCNT(n) ((n) << 24)
  246. #define RCAR_DMACHCRB_DPTR_MASK (0xff << 16)
  247. #define RCAR_DMACHCRB_DPTR_SHIFT 16
  248. #define RCAR_DMACHCRB_DRST (1 << 15)
  249. #define RCAR_DMACHCRB_DTS (1 << 8)
  250. #define RCAR_DMACHCRB_SLM_NORMAL (0 << 4)
  251. #define RCAR_DMACHCRB_SLM_CLK(n) ((8 | (n)) << 4)
  252. #define RCAR_DMACHCRB_PRI(n) ((n) << 0)
  253. #define RCAR_DMARS 0x0040
  254. #define RCAR_DMABUFCR 0x0048
  255. #define RCAR_DMABUFCR_MBU(n) ((n) << 16)
  256. #define RCAR_DMABUFCR_ULB(n) ((n) << 0)
  257. #define RCAR_DMADPBASE 0x0050
  258. #define RCAR_DMADPBASE_MASK 0xfffffff0
  259. #define RCAR_DMADPBASE_SEL (1 << 0)
  260. #define RCAR_DMADPCR 0x0054
  261. #define RCAR_DMADPCR_DIPT(n) ((n) << 24)
  262. #define RCAR_DMAFIXSAR 0x0010
  263. #define RCAR_DMAFIXDAR 0x0014
  264. #define RCAR_DMAFIXDPBASE 0x0060
  265. /* Hardcode the MEMCPY transfer size to 4 bytes. */
  266. #define RCAR_DMAC_MEMCPY_XFER_SIZE 4
  267. /* -----------------------------------------------------------------------------
  268. * Device access
  269. */
  270. static void rcar_dmac_write(struct rcar_dmac *dmac, u32 reg, u32 data)
  271. {
  272. if (reg == RCAR_DMAOR)
  273. writew(data, dmac->iomem + reg);
  274. else
  275. writel(data, dmac->iomem + reg);
  276. }
  277. static u32 rcar_dmac_read(struct rcar_dmac *dmac, u32 reg)
  278. {
  279. if (reg == RCAR_DMAOR)
  280. return readw(dmac->iomem + reg);
  281. else
  282. return readl(dmac->iomem + reg);
  283. }
  284. static u32 rcar_dmac_chan_read(struct rcar_dmac_chan *chan, u32 reg)
  285. {
  286. if (reg == RCAR_DMARS)
  287. return readw(chan->iomem + reg);
  288. else
  289. return readl(chan->iomem + reg);
  290. }
  291. static void rcar_dmac_chan_write(struct rcar_dmac_chan *chan, u32 reg, u32 data)
  292. {
  293. if (reg == RCAR_DMARS)
  294. writew(data, chan->iomem + reg);
  295. else
  296. writel(data, chan->iomem + reg);
  297. }
  298. /* -----------------------------------------------------------------------------
  299. * Initialization and configuration
  300. */
  301. static bool rcar_dmac_chan_is_busy(struct rcar_dmac_chan *chan)
  302. {
  303. u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR);
  304. return !!(chcr & (RCAR_DMACHCR_DE | RCAR_DMACHCR_TE));
  305. }
  306. static void rcar_dmac_chan_start_xfer(struct rcar_dmac_chan *chan)
  307. {
  308. struct rcar_dmac_desc *desc = chan->desc.running;
  309. u32 chcr = desc->chcr;
  310. WARN_ON_ONCE(rcar_dmac_chan_is_busy(chan));
  311. if (chan->mid_rid >= 0)
  312. rcar_dmac_chan_write(chan, RCAR_DMARS, chan->mid_rid);
  313. if (desc->hwdescs.use) {
  314. struct rcar_dmac_xfer_chunk *chunk =
  315. list_first_entry(&desc->chunks,
  316. struct rcar_dmac_xfer_chunk, node);
  317. dev_dbg(chan->chan.device->dev,
  318. "chan%u: queue desc %p: %u@%pad\n",
  319. chan->index, desc, desc->nchunks, &desc->hwdescs.dma);
  320. #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
  321. rcar_dmac_chan_write(chan, RCAR_DMAFIXSAR,
  322. chunk->src_addr >> 32);
  323. rcar_dmac_chan_write(chan, RCAR_DMAFIXDAR,
  324. chunk->dst_addr >> 32);
  325. rcar_dmac_chan_write(chan, RCAR_DMAFIXDPBASE,
  326. desc->hwdescs.dma >> 32);
  327. #endif
  328. rcar_dmac_chan_write(chan, RCAR_DMADPBASE,
  329. (desc->hwdescs.dma & 0xfffffff0) |
  330. RCAR_DMADPBASE_SEL);
  331. rcar_dmac_chan_write(chan, RCAR_DMACHCRB,
  332. RCAR_DMACHCRB_DCNT(desc->nchunks - 1) |
  333. RCAR_DMACHCRB_DRST);
  334. /*
  335. * Errata: When descriptor memory is accessed through an IOMMU
  336. * the DMADAR register isn't initialized automatically from the
  337. * first descriptor at beginning of transfer by the DMAC like it
  338. * should. Initialize it manually with the destination address
  339. * of the first chunk.
  340. */
  341. rcar_dmac_chan_write(chan, RCAR_DMADAR,
  342. chunk->dst_addr & 0xffffffff);
  343. /*
  344. * Program the descriptor stage interrupt to occur after the end
  345. * of the first stage.
  346. */
  347. rcar_dmac_chan_write(chan, RCAR_DMADPCR, RCAR_DMADPCR_DIPT(1));
  348. chcr |= RCAR_DMACHCR_RPT_SAR | RCAR_DMACHCR_RPT_DAR
  349. | RCAR_DMACHCR_RPT_TCR | RCAR_DMACHCR_DPB;
  350. /*
  351. * If the descriptor isn't cyclic enable normal descriptor mode
  352. * and the transfer completion interrupt.
  353. */
  354. if (!desc->cyclic)
  355. chcr |= RCAR_DMACHCR_DPM_ENABLED | RCAR_DMACHCR_IE;
  356. /*
  357. * If the descriptor is cyclic and has a callback enable the
  358. * descriptor stage interrupt in infinite repeat mode.
  359. */
  360. else if (desc->async_tx.callback)
  361. chcr |= RCAR_DMACHCR_DPM_INFINITE | RCAR_DMACHCR_DSIE;
  362. /*
  363. * Otherwise just select infinite repeat mode without any
  364. * interrupt.
  365. */
  366. else
  367. chcr |= RCAR_DMACHCR_DPM_INFINITE;
  368. } else {
  369. struct rcar_dmac_xfer_chunk *chunk = desc->running;
  370. dev_dbg(chan->chan.device->dev,
  371. "chan%u: queue chunk %p: %u@%pad -> %pad\n",
  372. chan->index, chunk, chunk->size, &chunk->src_addr,
  373. &chunk->dst_addr);
  374. #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
  375. rcar_dmac_chan_write(chan, RCAR_DMAFIXSAR,
  376. chunk->src_addr >> 32);
  377. rcar_dmac_chan_write(chan, RCAR_DMAFIXDAR,
  378. chunk->dst_addr >> 32);
  379. #endif
  380. rcar_dmac_chan_write(chan, RCAR_DMASAR,
  381. chunk->src_addr & 0xffffffff);
  382. rcar_dmac_chan_write(chan, RCAR_DMADAR,
  383. chunk->dst_addr & 0xffffffff);
  384. rcar_dmac_chan_write(chan, RCAR_DMATCR,
  385. chunk->size >> desc->xfer_shift);
  386. chcr |= RCAR_DMACHCR_DPM_DISABLED | RCAR_DMACHCR_IE;
  387. }
  388. rcar_dmac_chan_write(chan, RCAR_DMACHCR,
  389. chcr | RCAR_DMACHCR_DE | RCAR_DMACHCR_CAIE);
  390. }
  391. static int rcar_dmac_init(struct rcar_dmac *dmac)
  392. {
  393. u16 dmaor;
  394. /* Clear all channels and enable the DMAC globally. */
  395. rcar_dmac_write(dmac, RCAR_DMACHCLR, dmac->channels_mask);
  396. rcar_dmac_write(dmac, RCAR_DMAOR,
  397. RCAR_DMAOR_PRI_FIXED | RCAR_DMAOR_DME);
  398. dmaor = rcar_dmac_read(dmac, RCAR_DMAOR);
  399. if ((dmaor & (RCAR_DMAOR_AE | RCAR_DMAOR_DME)) != RCAR_DMAOR_DME) {
  400. dev_warn(dmac->dev, "DMAOR initialization failed.\n");
  401. return -EIO;
  402. }
  403. return 0;
  404. }
  405. /* -----------------------------------------------------------------------------
  406. * Descriptors submission
  407. */
  408. static dma_cookie_t rcar_dmac_tx_submit(struct dma_async_tx_descriptor *tx)
  409. {
  410. struct rcar_dmac_chan *chan = to_rcar_dmac_chan(tx->chan);
  411. struct rcar_dmac_desc *desc = to_rcar_dmac_desc(tx);
  412. unsigned long flags;
  413. dma_cookie_t cookie;
  414. spin_lock_irqsave(&chan->lock, flags);
  415. cookie = dma_cookie_assign(tx);
  416. dev_dbg(chan->chan.device->dev, "chan%u: submit #%d@%p\n",
  417. chan->index, tx->cookie, desc);
  418. list_add_tail(&desc->node, &chan->desc.pending);
  419. desc->running = list_first_entry(&desc->chunks,
  420. struct rcar_dmac_xfer_chunk, node);
  421. spin_unlock_irqrestore(&chan->lock, flags);
  422. return cookie;
  423. }
  424. /* -----------------------------------------------------------------------------
  425. * Descriptors allocation and free
  426. */
  427. /*
  428. * rcar_dmac_desc_alloc - Allocate a page worth of DMA descriptors
  429. * @chan: the DMA channel
  430. * @gfp: allocation flags
  431. */
  432. static int rcar_dmac_desc_alloc(struct rcar_dmac_chan *chan, gfp_t gfp)
  433. {
  434. struct rcar_dmac_desc_page *page;
  435. unsigned long flags;
  436. LIST_HEAD(list);
  437. unsigned int i;
  438. page = (void *)get_zeroed_page(gfp);
  439. if (!page)
  440. return -ENOMEM;
  441. for (i = 0; i < RCAR_DMAC_DESCS_PER_PAGE; ++i) {
  442. struct rcar_dmac_desc *desc = &page->descs[i];
  443. dma_async_tx_descriptor_init(&desc->async_tx, &chan->chan);
  444. desc->async_tx.tx_submit = rcar_dmac_tx_submit;
  445. INIT_LIST_HEAD(&desc->chunks);
  446. list_add_tail(&desc->node, &list);
  447. }
  448. spin_lock_irqsave(&chan->lock, flags);
  449. list_splice_tail(&list, &chan->desc.free);
  450. list_add_tail(&page->node, &chan->desc.pages);
  451. spin_unlock_irqrestore(&chan->lock, flags);
  452. return 0;
  453. }
  454. /*
  455. * rcar_dmac_desc_put - Release a DMA transfer descriptor
  456. * @chan: the DMA channel
  457. * @desc: the descriptor
  458. *
  459. * Put the descriptor and its transfer chunk descriptors back in the channel's
  460. * free descriptors lists. The descriptor's chunks list will be reinitialized to
  461. * an empty list as a result.
  462. *
  463. * The descriptor must have been removed from the channel's lists before calling
  464. * this function.
  465. */
  466. static void rcar_dmac_desc_put(struct rcar_dmac_chan *chan,
  467. struct rcar_dmac_desc *desc)
  468. {
  469. unsigned long flags;
  470. spin_lock_irqsave(&chan->lock, flags);
  471. list_splice_tail_init(&desc->chunks, &chan->desc.chunks_free);
  472. list_add(&desc->node, &chan->desc.free);
  473. spin_unlock_irqrestore(&chan->lock, flags);
  474. }
  475. static void rcar_dmac_desc_recycle_acked(struct rcar_dmac_chan *chan)
  476. {
  477. struct rcar_dmac_desc *desc, *_desc;
  478. unsigned long flags;
  479. LIST_HEAD(list);
  480. /*
  481. * We have to temporarily move all descriptors from the wait list to a
  482. * local list as iterating over the wait list, even with
  483. * list_for_each_entry_safe, isn't safe if we release the channel lock
  484. * around the rcar_dmac_desc_put() call.
  485. */
  486. spin_lock_irqsave(&chan->lock, flags);
  487. list_splice_init(&chan->desc.wait, &list);
  488. spin_unlock_irqrestore(&chan->lock, flags);
  489. list_for_each_entry_safe(desc, _desc, &list, node) {
  490. if (async_tx_test_ack(&desc->async_tx)) {
  491. list_del(&desc->node);
  492. rcar_dmac_desc_put(chan, desc);
  493. }
  494. }
  495. if (list_empty(&list))
  496. return;
  497. /* Put the remaining descriptors back in the wait list. */
  498. spin_lock_irqsave(&chan->lock, flags);
  499. list_splice(&list, &chan->desc.wait);
  500. spin_unlock_irqrestore(&chan->lock, flags);
  501. }
  502. /*
  503. * rcar_dmac_desc_get - Allocate a descriptor for a DMA transfer
  504. * @chan: the DMA channel
  505. *
  506. * Locking: This function must be called in a non-atomic context.
  507. *
  508. * Return: A pointer to the allocated descriptor or NULL if no descriptor can
  509. * be allocated.
  510. */
  511. static struct rcar_dmac_desc *rcar_dmac_desc_get(struct rcar_dmac_chan *chan)
  512. {
  513. struct rcar_dmac_desc *desc;
  514. unsigned long flags;
  515. int ret;
  516. /* Recycle acked descriptors before attempting allocation. */
  517. rcar_dmac_desc_recycle_acked(chan);
  518. spin_lock_irqsave(&chan->lock, flags);
  519. while (list_empty(&chan->desc.free)) {
  520. /*
  521. * No free descriptors, allocate a page worth of them and try
  522. * again, as someone else could race us to get the newly
  523. * allocated descriptors. If the allocation fails return an
  524. * error.
  525. */
  526. spin_unlock_irqrestore(&chan->lock, flags);
  527. ret = rcar_dmac_desc_alloc(chan, GFP_NOWAIT);
  528. if (ret < 0)
  529. return NULL;
  530. spin_lock_irqsave(&chan->lock, flags);
  531. }
  532. desc = list_first_entry(&chan->desc.free, struct rcar_dmac_desc, node);
  533. list_del(&desc->node);
  534. spin_unlock_irqrestore(&chan->lock, flags);
  535. return desc;
  536. }
  537. /*
  538. * rcar_dmac_xfer_chunk_alloc - Allocate a page worth of transfer chunks
  539. * @chan: the DMA channel
  540. * @gfp: allocation flags
  541. */
  542. static int rcar_dmac_xfer_chunk_alloc(struct rcar_dmac_chan *chan, gfp_t gfp)
  543. {
  544. struct rcar_dmac_desc_page *page;
  545. unsigned long flags;
  546. LIST_HEAD(list);
  547. unsigned int i;
  548. page = (void *)get_zeroed_page(gfp);
  549. if (!page)
  550. return -ENOMEM;
  551. for (i = 0; i < RCAR_DMAC_XFER_CHUNKS_PER_PAGE; ++i) {
  552. struct rcar_dmac_xfer_chunk *chunk = &page->chunks[i];
  553. list_add_tail(&chunk->node, &list);
  554. }
  555. spin_lock_irqsave(&chan->lock, flags);
  556. list_splice_tail(&list, &chan->desc.chunks_free);
  557. list_add_tail(&page->node, &chan->desc.pages);
  558. spin_unlock_irqrestore(&chan->lock, flags);
  559. return 0;
  560. }
  561. /*
  562. * rcar_dmac_xfer_chunk_get - Allocate a transfer chunk for a DMA transfer
  563. * @chan: the DMA channel
  564. *
  565. * Locking: This function must be called in a non-atomic context.
  566. *
  567. * Return: A pointer to the allocated transfer chunk descriptor or NULL if no
  568. * descriptor can be allocated.
  569. */
  570. static struct rcar_dmac_xfer_chunk *
  571. rcar_dmac_xfer_chunk_get(struct rcar_dmac_chan *chan)
  572. {
  573. struct rcar_dmac_xfer_chunk *chunk;
  574. unsigned long flags;
  575. int ret;
  576. spin_lock_irqsave(&chan->lock, flags);
  577. while (list_empty(&chan->desc.chunks_free)) {
  578. /*
  579. * No free descriptors, allocate a page worth of them and try
  580. * again, as someone else could race us to get the newly
  581. * allocated descriptors. If the allocation fails return an
  582. * error.
  583. */
  584. spin_unlock_irqrestore(&chan->lock, flags);
  585. ret = rcar_dmac_xfer_chunk_alloc(chan, GFP_NOWAIT);
  586. if (ret < 0)
  587. return NULL;
  588. spin_lock_irqsave(&chan->lock, flags);
  589. }
  590. chunk = list_first_entry(&chan->desc.chunks_free,
  591. struct rcar_dmac_xfer_chunk, node);
  592. list_del(&chunk->node);
  593. spin_unlock_irqrestore(&chan->lock, flags);
  594. return chunk;
  595. }
  596. static void rcar_dmac_realloc_hwdesc(struct rcar_dmac_chan *chan,
  597. struct rcar_dmac_desc *desc, size_t size)
  598. {
  599. /*
  600. * dma_alloc_coherent() allocates memory in page size increments. To
  601. * avoid reallocating the hardware descriptors when the allocated size
  602. * wouldn't change align the requested size to a multiple of the page
  603. * size.
  604. */
  605. size = PAGE_ALIGN(size);
  606. if (desc->hwdescs.size == size)
  607. return;
  608. if (desc->hwdescs.mem) {
  609. dma_free_coherent(chan->chan.device->dev, desc->hwdescs.size,
  610. desc->hwdescs.mem, desc->hwdescs.dma);
  611. desc->hwdescs.mem = NULL;
  612. desc->hwdescs.size = 0;
  613. }
  614. if (!size)
  615. return;
  616. desc->hwdescs.mem = dma_alloc_coherent(chan->chan.device->dev, size,
  617. &desc->hwdescs.dma, GFP_NOWAIT);
  618. if (!desc->hwdescs.mem)
  619. return;
  620. desc->hwdescs.size = size;
  621. }
  622. static int rcar_dmac_fill_hwdesc(struct rcar_dmac_chan *chan,
  623. struct rcar_dmac_desc *desc)
  624. {
  625. struct rcar_dmac_xfer_chunk *chunk;
  626. struct rcar_dmac_hw_desc *hwdesc;
  627. rcar_dmac_realloc_hwdesc(chan, desc, desc->nchunks * sizeof(*hwdesc));
  628. hwdesc = desc->hwdescs.mem;
  629. if (!hwdesc)
  630. return -ENOMEM;
  631. list_for_each_entry(chunk, &desc->chunks, node) {
  632. hwdesc->sar = chunk->src_addr;
  633. hwdesc->dar = chunk->dst_addr;
  634. hwdesc->tcr = chunk->size >> desc->xfer_shift;
  635. hwdesc++;
  636. }
  637. return 0;
  638. }
  639. /* -----------------------------------------------------------------------------
  640. * Stop and reset
  641. */
  642. static void rcar_dmac_chcr_de_barrier(struct rcar_dmac_chan *chan)
  643. {
  644. u32 chcr;
  645. unsigned int i;
  646. /*
  647. * Ensure that the setting of the DE bit is actually 0 after
  648. * clearing it.
  649. */
  650. for (i = 0; i < 1024; i++) {
  651. chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR);
  652. if (!(chcr & RCAR_DMACHCR_DE))
  653. return;
  654. udelay(1);
  655. }
  656. dev_err(chan->chan.device->dev, "CHCR DE check error\n");
  657. }
  658. static void rcar_dmac_clear_chcr_de(struct rcar_dmac_chan *chan)
  659. {
  660. u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR);
  661. /* set DE=0 and flush remaining data */
  662. rcar_dmac_chan_write(chan, RCAR_DMACHCR, (chcr & ~RCAR_DMACHCR_DE));
  663. /* make sure all remaining data was flushed */
  664. rcar_dmac_chcr_de_barrier(chan);
  665. }
  666. static void rcar_dmac_chan_halt(struct rcar_dmac_chan *chan)
  667. {
  668. u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR);
  669. chcr &= ~(RCAR_DMACHCR_DSE | RCAR_DMACHCR_DSIE | RCAR_DMACHCR_IE |
  670. RCAR_DMACHCR_TE | RCAR_DMACHCR_DE |
  671. RCAR_DMACHCR_CAE | RCAR_DMACHCR_CAIE);
  672. rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr);
  673. rcar_dmac_chcr_de_barrier(chan);
  674. }
  675. static void rcar_dmac_chan_reinit(struct rcar_dmac_chan *chan)
  676. {
  677. struct rcar_dmac_desc *desc, *_desc;
  678. unsigned long flags;
  679. LIST_HEAD(descs);
  680. spin_lock_irqsave(&chan->lock, flags);
  681. /* Move all non-free descriptors to the local lists. */
  682. list_splice_init(&chan->desc.pending, &descs);
  683. list_splice_init(&chan->desc.active, &descs);
  684. list_splice_init(&chan->desc.done, &descs);
  685. list_splice_init(&chan->desc.wait, &descs);
  686. chan->desc.running = NULL;
  687. spin_unlock_irqrestore(&chan->lock, flags);
  688. list_for_each_entry_safe(desc, _desc, &descs, node) {
  689. list_del(&desc->node);
  690. rcar_dmac_desc_put(chan, desc);
  691. }
  692. }
  693. static void rcar_dmac_stop_all_chan(struct rcar_dmac *dmac)
  694. {
  695. unsigned int i;
  696. /* Stop all channels. */
  697. for (i = 0; i < dmac->n_channels; ++i) {
  698. struct rcar_dmac_chan *chan = &dmac->channels[i];
  699. if (!(dmac->channels_mask & BIT(i)))
  700. continue;
  701. /* Stop and reinitialize the channel. */
  702. spin_lock_irq(&chan->lock);
  703. rcar_dmac_chan_halt(chan);
  704. spin_unlock_irq(&chan->lock);
  705. }
  706. }
  707. static int rcar_dmac_chan_pause(struct dma_chan *chan)
  708. {
  709. unsigned long flags;
  710. struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
  711. spin_lock_irqsave(&rchan->lock, flags);
  712. rcar_dmac_clear_chcr_de(rchan);
  713. spin_unlock_irqrestore(&rchan->lock, flags);
  714. return 0;
  715. }
  716. /* -----------------------------------------------------------------------------
  717. * Descriptors preparation
  718. */
  719. static void rcar_dmac_chan_configure_desc(struct rcar_dmac_chan *chan,
  720. struct rcar_dmac_desc *desc)
  721. {
  722. static const u32 chcr_ts[] = {
  723. RCAR_DMACHCR_TS_1B, RCAR_DMACHCR_TS_2B,
  724. RCAR_DMACHCR_TS_4B, RCAR_DMACHCR_TS_8B,
  725. RCAR_DMACHCR_TS_16B, RCAR_DMACHCR_TS_32B,
  726. RCAR_DMACHCR_TS_64B,
  727. };
  728. unsigned int xfer_size;
  729. u32 chcr;
  730. switch (desc->direction) {
  731. case DMA_DEV_TO_MEM:
  732. chcr = RCAR_DMACHCR_DM_INC | RCAR_DMACHCR_SM_FIXED
  733. | RCAR_DMACHCR_RS_DMARS;
  734. xfer_size = chan->src.xfer_size;
  735. break;
  736. case DMA_MEM_TO_DEV:
  737. chcr = RCAR_DMACHCR_DM_FIXED | RCAR_DMACHCR_SM_INC
  738. | RCAR_DMACHCR_RS_DMARS;
  739. xfer_size = chan->dst.xfer_size;
  740. break;
  741. case DMA_MEM_TO_MEM:
  742. default:
  743. chcr = RCAR_DMACHCR_DM_INC | RCAR_DMACHCR_SM_INC
  744. | RCAR_DMACHCR_RS_AUTO;
  745. xfer_size = RCAR_DMAC_MEMCPY_XFER_SIZE;
  746. break;
  747. }
  748. desc->xfer_shift = ilog2(xfer_size);
  749. desc->chcr = chcr | chcr_ts[desc->xfer_shift];
  750. }
  751. /*
  752. * rcar_dmac_chan_prep_sg - prepare transfer descriptors from an SG list
  753. *
  754. * Common routine for public (MEMCPY) and slave DMA. The MEMCPY case is also
  755. * converted to scatter-gather to guarantee consistent locking and a correct
  756. * list manipulation. For slave DMA direction carries the usual meaning, and,
  757. * logically, the SG list is RAM and the addr variable contains slave address,
  758. * e.g., the FIFO I/O register. For MEMCPY direction equals DMA_MEM_TO_MEM
  759. * and the SG list contains only one element and points at the source buffer.
  760. */
  761. static struct dma_async_tx_descriptor *
  762. rcar_dmac_chan_prep_sg(struct rcar_dmac_chan *chan, struct scatterlist *sgl,
  763. unsigned int sg_len, dma_addr_t dev_addr,
  764. enum dma_transfer_direction dir, unsigned long dma_flags,
  765. bool cyclic)
  766. {
  767. struct rcar_dmac_xfer_chunk *chunk;
  768. struct rcar_dmac_desc *desc;
  769. struct scatterlist *sg;
  770. unsigned int nchunks = 0;
  771. unsigned int max_chunk_size;
  772. unsigned int full_size = 0;
  773. bool cross_boundary = false;
  774. unsigned int i;
  775. #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
  776. u32 high_dev_addr;
  777. u32 high_mem_addr;
  778. #endif
  779. desc = rcar_dmac_desc_get(chan);
  780. if (!desc)
  781. return NULL;
  782. desc->async_tx.flags = dma_flags;
  783. desc->async_tx.cookie = -EBUSY;
  784. desc->cyclic = cyclic;
  785. desc->direction = dir;
  786. rcar_dmac_chan_configure_desc(chan, desc);
  787. max_chunk_size = RCAR_DMATCR_MASK << desc->xfer_shift;
  788. /*
  789. * Allocate and fill the transfer chunk descriptors. We own the only
  790. * reference to the DMA descriptor, there's no need for locking.
  791. */
  792. for_each_sg(sgl, sg, sg_len, i) {
  793. dma_addr_t mem_addr = sg_dma_address(sg);
  794. unsigned int len = sg_dma_len(sg);
  795. full_size += len;
  796. #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
  797. if (i == 0) {
  798. high_dev_addr = dev_addr >> 32;
  799. high_mem_addr = mem_addr >> 32;
  800. }
  801. if ((dev_addr >> 32 != high_dev_addr) ||
  802. (mem_addr >> 32 != high_mem_addr))
  803. cross_boundary = true;
  804. #endif
  805. while (len) {
  806. unsigned int size = min(len, max_chunk_size);
  807. #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
  808. /*
  809. * Prevent individual transfers from crossing 4GB
  810. * boundaries.
  811. */
  812. if (dev_addr >> 32 != (dev_addr + size - 1) >> 32) {
  813. size = ALIGN(dev_addr, 1ULL << 32) - dev_addr;
  814. cross_boundary = true;
  815. }
  816. if (mem_addr >> 32 != (mem_addr + size - 1) >> 32) {
  817. size = ALIGN(mem_addr, 1ULL << 32) - mem_addr;
  818. cross_boundary = true;
  819. }
  820. #endif
  821. chunk = rcar_dmac_xfer_chunk_get(chan);
  822. if (!chunk) {
  823. rcar_dmac_desc_put(chan, desc);
  824. return NULL;
  825. }
  826. if (dir == DMA_DEV_TO_MEM) {
  827. chunk->src_addr = dev_addr;
  828. chunk->dst_addr = mem_addr;
  829. } else {
  830. chunk->src_addr = mem_addr;
  831. chunk->dst_addr = dev_addr;
  832. }
  833. chunk->size = size;
  834. dev_dbg(chan->chan.device->dev,
  835. "chan%u: chunk %p/%p sgl %u@%p, %u/%u %pad -> %pad\n",
  836. chan->index, chunk, desc, i, sg, size, len,
  837. &chunk->src_addr, &chunk->dst_addr);
  838. mem_addr += size;
  839. if (dir == DMA_MEM_TO_MEM)
  840. dev_addr += size;
  841. len -= size;
  842. list_add_tail(&chunk->node, &desc->chunks);
  843. nchunks++;
  844. }
  845. }
  846. desc->nchunks = nchunks;
  847. desc->size = full_size;
  848. /*
  849. * Use hardware descriptor lists if possible when more than one chunk
  850. * needs to be transferred (otherwise they don't make much sense).
  851. *
  852. * Source/Destination address should be located in same 4GiB region
  853. * in the 40bit address space when it uses Hardware descriptor,
  854. * and cross_boundary is checking it.
  855. */
  856. desc->hwdescs.use = !cross_boundary && nchunks > 1;
  857. if (desc->hwdescs.use) {
  858. if (rcar_dmac_fill_hwdesc(chan, desc) < 0)
  859. desc->hwdescs.use = false;
  860. }
  861. return &desc->async_tx;
  862. }
  863. /* -----------------------------------------------------------------------------
  864. * DMA engine operations
  865. */
  866. static int rcar_dmac_alloc_chan_resources(struct dma_chan *chan)
  867. {
  868. struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
  869. int ret;
  870. INIT_LIST_HEAD(&rchan->desc.chunks_free);
  871. INIT_LIST_HEAD(&rchan->desc.pages);
  872. /* Preallocate descriptors. */
  873. ret = rcar_dmac_xfer_chunk_alloc(rchan, GFP_KERNEL);
  874. if (ret < 0)
  875. return -ENOMEM;
  876. ret = rcar_dmac_desc_alloc(rchan, GFP_KERNEL);
  877. if (ret < 0)
  878. return -ENOMEM;
  879. return pm_runtime_get_sync(chan->device->dev);
  880. }
  881. static void rcar_dmac_free_chan_resources(struct dma_chan *chan)
  882. {
  883. struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
  884. struct rcar_dmac *dmac = to_rcar_dmac(chan->device);
  885. struct rcar_dmac_chan_map *map = &rchan->map;
  886. struct rcar_dmac_desc_page *page, *_page;
  887. struct rcar_dmac_desc *desc;
  888. LIST_HEAD(list);
  889. /* Protect against ISR */
  890. spin_lock_irq(&rchan->lock);
  891. rcar_dmac_chan_halt(rchan);
  892. spin_unlock_irq(&rchan->lock);
  893. /*
  894. * Now no new interrupts will occur, but one might already be
  895. * running. Wait for it to finish before freeing resources.
  896. */
  897. synchronize_irq(rchan->irq);
  898. if (rchan->mid_rid >= 0) {
  899. /* The caller is holding dma_list_mutex */
  900. clear_bit(rchan->mid_rid, dmac->modules);
  901. rchan->mid_rid = -EINVAL;
  902. }
  903. list_splice_init(&rchan->desc.free, &list);
  904. list_splice_init(&rchan->desc.pending, &list);
  905. list_splice_init(&rchan->desc.active, &list);
  906. list_splice_init(&rchan->desc.done, &list);
  907. list_splice_init(&rchan->desc.wait, &list);
  908. rchan->desc.running = NULL;
  909. list_for_each_entry(desc, &list, node)
  910. rcar_dmac_realloc_hwdesc(rchan, desc, 0);
  911. list_for_each_entry_safe(page, _page, &rchan->desc.pages, node) {
  912. list_del(&page->node);
  913. free_page((unsigned long)page);
  914. }
  915. /* Remove slave mapping if present. */
  916. if (map->slave.xfer_size) {
  917. dma_unmap_resource(chan->device->dev, map->addr,
  918. map->slave.xfer_size, map->dir, 0);
  919. map->slave.xfer_size = 0;
  920. }
  921. pm_runtime_put(chan->device->dev);
  922. }
  923. static struct dma_async_tx_descriptor *
  924. rcar_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest,
  925. dma_addr_t dma_src, size_t len, unsigned long flags)
  926. {
  927. struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
  928. struct scatterlist sgl;
  929. if (!len)
  930. return NULL;
  931. sg_init_table(&sgl, 1);
  932. sg_set_page(&sgl, pfn_to_page(PFN_DOWN(dma_src)), len,
  933. offset_in_page(dma_src));
  934. sg_dma_address(&sgl) = dma_src;
  935. sg_dma_len(&sgl) = len;
  936. return rcar_dmac_chan_prep_sg(rchan, &sgl, 1, dma_dest,
  937. DMA_MEM_TO_MEM, flags, false);
  938. }
  939. static int rcar_dmac_map_slave_addr(struct dma_chan *chan,
  940. enum dma_transfer_direction dir)
  941. {
  942. struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
  943. struct rcar_dmac_chan_map *map = &rchan->map;
  944. phys_addr_t dev_addr;
  945. size_t dev_size;
  946. enum dma_data_direction dev_dir;
  947. if (dir == DMA_DEV_TO_MEM) {
  948. dev_addr = rchan->src.slave_addr;
  949. dev_size = rchan->src.xfer_size;
  950. dev_dir = DMA_TO_DEVICE;
  951. } else {
  952. dev_addr = rchan->dst.slave_addr;
  953. dev_size = rchan->dst.xfer_size;
  954. dev_dir = DMA_FROM_DEVICE;
  955. }
  956. /* Reuse current map if possible. */
  957. if (dev_addr == map->slave.slave_addr &&
  958. dev_size == map->slave.xfer_size &&
  959. dev_dir == map->dir)
  960. return 0;
  961. /* Remove old mapping if present. */
  962. if (map->slave.xfer_size)
  963. dma_unmap_resource(chan->device->dev, map->addr,
  964. map->slave.xfer_size, map->dir, 0);
  965. map->slave.xfer_size = 0;
  966. /* Create new slave address map. */
  967. map->addr = dma_map_resource(chan->device->dev, dev_addr, dev_size,
  968. dev_dir, 0);
  969. if (dma_mapping_error(chan->device->dev, map->addr)) {
  970. dev_err(chan->device->dev,
  971. "chan%u: failed to map %zx@%pap", rchan->index,
  972. dev_size, &dev_addr);
  973. return -EIO;
  974. }
  975. dev_dbg(chan->device->dev, "chan%u: map %zx@%pap to %pad dir: %s\n",
  976. rchan->index, dev_size, &dev_addr, &map->addr,
  977. dev_dir == DMA_TO_DEVICE ? "DMA_TO_DEVICE" : "DMA_FROM_DEVICE");
  978. map->slave.slave_addr = dev_addr;
  979. map->slave.xfer_size = dev_size;
  980. map->dir = dev_dir;
  981. return 0;
  982. }
  983. static struct dma_async_tx_descriptor *
  984. rcar_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
  985. unsigned int sg_len, enum dma_transfer_direction dir,
  986. unsigned long flags, void *context)
  987. {
  988. struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
  989. /* Someone calling slave DMA on a generic channel? */
  990. if (rchan->mid_rid < 0 || !sg_len || !sg_dma_len(sgl)) {
  991. dev_warn(chan->device->dev,
  992. "%s: bad parameter: len=%d, id=%d\n",
  993. __func__, sg_len, rchan->mid_rid);
  994. return NULL;
  995. }
  996. if (rcar_dmac_map_slave_addr(chan, dir))
  997. return NULL;
  998. return rcar_dmac_chan_prep_sg(rchan, sgl, sg_len, rchan->map.addr,
  999. dir, flags, false);
  1000. }
  1001. #define RCAR_DMAC_MAX_SG_LEN 32
  1002. static struct dma_async_tx_descriptor *
  1003. rcar_dmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr,
  1004. size_t buf_len, size_t period_len,
  1005. enum dma_transfer_direction dir, unsigned long flags)
  1006. {
  1007. struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
  1008. struct dma_async_tx_descriptor *desc;
  1009. struct scatterlist *sgl;
  1010. unsigned int sg_len;
  1011. unsigned int i;
  1012. /* Someone calling slave DMA on a generic channel? */
  1013. if (rchan->mid_rid < 0 || buf_len < period_len) {
  1014. dev_warn(chan->device->dev,
  1015. "%s: bad parameter: buf_len=%zu, period_len=%zu, id=%d\n",
  1016. __func__, buf_len, period_len, rchan->mid_rid);
  1017. return NULL;
  1018. }
  1019. if (rcar_dmac_map_slave_addr(chan, dir))
  1020. return NULL;
  1021. sg_len = buf_len / period_len;
  1022. if (sg_len > RCAR_DMAC_MAX_SG_LEN) {
  1023. dev_err(chan->device->dev,
  1024. "chan%u: sg length %d exceeds limit %d",
  1025. rchan->index, sg_len, RCAR_DMAC_MAX_SG_LEN);
  1026. return NULL;
  1027. }
  1028. /*
  1029. * Allocate the sg list dynamically as it would consume too much stack
  1030. * space.
  1031. */
  1032. sgl = kmalloc_array(sg_len, sizeof(*sgl), GFP_NOWAIT);
  1033. if (!sgl)
  1034. return NULL;
  1035. sg_init_table(sgl, sg_len);
  1036. for (i = 0; i < sg_len; ++i) {
  1037. dma_addr_t src = buf_addr + (period_len * i);
  1038. sg_set_page(&sgl[i], pfn_to_page(PFN_DOWN(src)), period_len,
  1039. offset_in_page(src));
  1040. sg_dma_address(&sgl[i]) = src;
  1041. sg_dma_len(&sgl[i]) = period_len;
  1042. }
  1043. desc = rcar_dmac_chan_prep_sg(rchan, sgl, sg_len, rchan->map.addr,
  1044. dir, flags, true);
  1045. kfree(sgl);
  1046. return desc;
  1047. }
  1048. static int rcar_dmac_device_config(struct dma_chan *chan,
  1049. struct dma_slave_config *cfg)
  1050. {
  1051. struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
  1052. /*
  1053. * We could lock this, but you shouldn't be configuring the
  1054. * channel, while using it...
  1055. */
  1056. rchan->src.slave_addr = cfg->src_addr;
  1057. rchan->dst.slave_addr = cfg->dst_addr;
  1058. rchan->src.xfer_size = cfg->src_addr_width;
  1059. rchan->dst.xfer_size = cfg->dst_addr_width;
  1060. return 0;
  1061. }
  1062. static int rcar_dmac_chan_terminate_all(struct dma_chan *chan)
  1063. {
  1064. struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
  1065. unsigned long flags;
  1066. spin_lock_irqsave(&rchan->lock, flags);
  1067. rcar_dmac_chan_halt(rchan);
  1068. spin_unlock_irqrestore(&rchan->lock, flags);
  1069. /*
  1070. * FIXME: No new interrupt can occur now, but the IRQ thread might still
  1071. * be running.
  1072. */
  1073. rcar_dmac_chan_reinit(rchan);
  1074. return 0;
  1075. }
  1076. static unsigned int rcar_dmac_chan_get_residue(struct rcar_dmac_chan *chan,
  1077. dma_cookie_t cookie)
  1078. {
  1079. struct rcar_dmac_desc *desc = chan->desc.running;
  1080. struct rcar_dmac_xfer_chunk *running = NULL;
  1081. struct rcar_dmac_xfer_chunk *chunk;
  1082. enum dma_status status;
  1083. unsigned int residue = 0;
  1084. unsigned int dptr = 0;
  1085. unsigned int chcrb;
  1086. unsigned int tcrb;
  1087. unsigned int i;
  1088. if (!desc)
  1089. return 0;
  1090. /*
  1091. * If the cookie corresponds to a descriptor that has been completed
  1092. * there is no residue. The same check has already been performed by the
  1093. * caller but without holding the channel lock, so the descriptor could
  1094. * now be complete.
  1095. */
  1096. status = dma_cookie_status(&chan->chan, cookie, NULL);
  1097. if (status == DMA_COMPLETE)
  1098. return 0;
  1099. /*
  1100. * If the cookie doesn't correspond to the currently running transfer
  1101. * then the descriptor hasn't been processed yet, and the residue is
  1102. * equal to the full descriptor size.
  1103. * Also, a client driver is possible to call this function before
  1104. * rcar_dmac_isr_channel_thread() runs. In this case, the "desc.running"
  1105. * will be the next descriptor, and the done list will appear. So, if
  1106. * the argument cookie matches the done list's cookie, we can assume
  1107. * the residue is zero.
  1108. */
  1109. if (cookie != desc->async_tx.cookie) {
  1110. list_for_each_entry(desc, &chan->desc.done, node) {
  1111. if (cookie == desc->async_tx.cookie)
  1112. return 0;
  1113. }
  1114. list_for_each_entry(desc, &chan->desc.pending, node) {
  1115. if (cookie == desc->async_tx.cookie)
  1116. return desc->size;
  1117. }
  1118. list_for_each_entry(desc, &chan->desc.active, node) {
  1119. if (cookie == desc->async_tx.cookie)
  1120. return desc->size;
  1121. }
  1122. /*
  1123. * No descriptor found for the cookie, there's thus no residue.
  1124. * This shouldn't happen if the calling driver passes a correct
  1125. * cookie value.
  1126. */
  1127. WARN(1, "No descriptor for cookie!");
  1128. return 0;
  1129. }
  1130. /*
  1131. * We need to read two registers.
  1132. * Make sure the control register does not skip to next chunk
  1133. * while reading the counter.
  1134. * Trying it 3 times should be enough: Initial read, retry, retry
  1135. * for the paranoid.
  1136. */
  1137. for (i = 0; i < 3; i++) {
  1138. chcrb = rcar_dmac_chan_read(chan, RCAR_DMACHCRB) &
  1139. RCAR_DMACHCRB_DPTR_MASK;
  1140. tcrb = rcar_dmac_chan_read(chan, RCAR_DMATCRB);
  1141. /* Still the same? */
  1142. if (chcrb == (rcar_dmac_chan_read(chan, RCAR_DMACHCRB) &
  1143. RCAR_DMACHCRB_DPTR_MASK))
  1144. break;
  1145. }
  1146. WARN_ONCE(i >= 3, "residue might be not continuous!");
  1147. /*
  1148. * In descriptor mode the descriptor running pointer is not maintained
  1149. * by the interrupt handler, find the running descriptor from the
  1150. * descriptor pointer field in the CHCRB register. In non-descriptor
  1151. * mode just use the running descriptor pointer.
  1152. */
  1153. if (desc->hwdescs.use) {
  1154. dptr = chcrb >> RCAR_DMACHCRB_DPTR_SHIFT;
  1155. if (dptr == 0)
  1156. dptr = desc->nchunks;
  1157. dptr--;
  1158. WARN_ON(dptr >= desc->nchunks);
  1159. } else {
  1160. running = desc->running;
  1161. }
  1162. /* Compute the size of all chunks still to be transferred. */
  1163. list_for_each_entry_reverse(chunk, &desc->chunks, node) {
  1164. if (chunk == running || ++dptr == desc->nchunks)
  1165. break;
  1166. residue += chunk->size;
  1167. }
  1168. /* Add the residue for the current chunk. */
  1169. residue += tcrb << desc->xfer_shift;
  1170. return residue;
  1171. }
  1172. static enum dma_status rcar_dmac_tx_status(struct dma_chan *chan,
  1173. dma_cookie_t cookie,
  1174. struct dma_tx_state *txstate)
  1175. {
  1176. struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
  1177. enum dma_status status;
  1178. unsigned long flags;
  1179. unsigned int residue;
  1180. bool cyclic;
  1181. status = dma_cookie_status(chan, cookie, txstate);
  1182. if (status == DMA_COMPLETE || !txstate)
  1183. return status;
  1184. spin_lock_irqsave(&rchan->lock, flags);
  1185. residue = rcar_dmac_chan_get_residue(rchan, cookie);
  1186. cyclic = rchan->desc.running ? rchan->desc.running->cyclic : false;
  1187. spin_unlock_irqrestore(&rchan->lock, flags);
  1188. /* if there's no residue, the cookie is complete */
  1189. if (!residue && !cyclic)
  1190. return DMA_COMPLETE;
  1191. dma_set_residue(txstate, residue);
  1192. return status;
  1193. }
  1194. static void rcar_dmac_issue_pending(struct dma_chan *chan)
  1195. {
  1196. struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
  1197. unsigned long flags;
  1198. spin_lock_irqsave(&rchan->lock, flags);
  1199. if (list_empty(&rchan->desc.pending))
  1200. goto done;
  1201. /* Append the pending list to the active list. */
  1202. list_splice_tail_init(&rchan->desc.pending, &rchan->desc.active);
  1203. /*
  1204. * If no transfer is running pick the first descriptor from the active
  1205. * list and start the transfer.
  1206. */
  1207. if (!rchan->desc.running) {
  1208. struct rcar_dmac_desc *desc;
  1209. desc = list_first_entry(&rchan->desc.active,
  1210. struct rcar_dmac_desc, node);
  1211. rchan->desc.running = desc;
  1212. rcar_dmac_chan_start_xfer(rchan);
  1213. }
  1214. done:
  1215. spin_unlock_irqrestore(&rchan->lock, flags);
  1216. }
  1217. static void rcar_dmac_device_synchronize(struct dma_chan *chan)
  1218. {
  1219. struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
  1220. synchronize_irq(rchan->irq);
  1221. }
  1222. /* -----------------------------------------------------------------------------
  1223. * IRQ handling
  1224. */
  1225. static irqreturn_t rcar_dmac_isr_desc_stage_end(struct rcar_dmac_chan *chan)
  1226. {
  1227. struct rcar_dmac_desc *desc = chan->desc.running;
  1228. unsigned int stage;
  1229. if (WARN_ON(!desc || !desc->cyclic)) {
  1230. /*
  1231. * This should never happen, there should always be a running
  1232. * cyclic descriptor when a descriptor stage end interrupt is
  1233. * triggered. Warn and return.
  1234. */
  1235. return IRQ_NONE;
  1236. }
  1237. /* Program the interrupt pointer to the next stage. */
  1238. stage = (rcar_dmac_chan_read(chan, RCAR_DMACHCRB) &
  1239. RCAR_DMACHCRB_DPTR_MASK) >> RCAR_DMACHCRB_DPTR_SHIFT;
  1240. rcar_dmac_chan_write(chan, RCAR_DMADPCR, RCAR_DMADPCR_DIPT(stage));
  1241. return IRQ_WAKE_THREAD;
  1242. }
  1243. static irqreturn_t rcar_dmac_isr_transfer_end(struct rcar_dmac_chan *chan)
  1244. {
  1245. struct rcar_dmac_desc *desc = chan->desc.running;
  1246. irqreturn_t ret = IRQ_WAKE_THREAD;
  1247. if (WARN_ON_ONCE(!desc)) {
  1248. /*
  1249. * This should never happen, there should always be a running
  1250. * descriptor when a transfer end interrupt is triggered. Warn
  1251. * and return.
  1252. */
  1253. return IRQ_NONE;
  1254. }
  1255. /*
  1256. * The transfer end interrupt isn't generated for each chunk when using
  1257. * descriptor mode. Only update the running chunk pointer in
  1258. * non-descriptor mode.
  1259. */
  1260. if (!desc->hwdescs.use) {
  1261. /*
  1262. * If we haven't completed the last transfer chunk simply move
  1263. * to the next one. Only wake the IRQ thread if the transfer is
  1264. * cyclic.
  1265. */
  1266. if (!list_is_last(&desc->running->node, &desc->chunks)) {
  1267. desc->running = list_next_entry(desc->running, node);
  1268. if (!desc->cyclic)
  1269. ret = IRQ_HANDLED;
  1270. goto done;
  1271. }
  1272. /*
  1273. * We've completed the last transfer chunk. If the transfer is
  1274. * cyclic, move back to the first one.
  1275. */
  1276. if (desc->cyclic) {
  1277. desc->running =
  1278. list_first_entry(&desc->chunks,
  1279. struct rcar_dmac_xfer_chunk,
  1280. node);
  1281. goto done;
  1282. }
  1283. }
  1284. /* The descriptor is complete, move it to the done list. */
  1285. list_move_tail(&desc->node, &chan->desc.done);
  1286. /* Queue the next descriptor, if any. */
  1287. if (!list_empty(&chan->desc.active))
  1288. chan->desc.running = list_first_entry(&chan->desc.active,
  1289. struct rcar_dmac_desc,
  1290. node);
  1291. else
  1292. chan->desc.running = NULL;
  1293. done:
  1294. if (chan->desc.running)
  1295. rcar_dmac_chan_start_xfer(chan);
  1296. return ret;
  1297. }
  1298. static irqreturn_t rcar_dmac_isr_channel(int irq, void *dev)
  1299. {
  1300. u32 mask = RCAR_DMACHCR_DSE | RCAR_DMACHCR_TE;
  1301. struct rcar_dmac_chan *chan = dev;
  1302. irqreturn_t ret = IRQ_NONE;
  1303. bool reinit = false;
  1304. u32 chcr;
  1305. spin_lock(&chan->lock);
  1306. chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR);
  1307. if (chcr & RCAR_DMACHCR_CAE) {
  1308. struct rcar_dmac *dmac = to_rcar_dmac(chan->chan.device);
  1309. /*
  1310. * We don't need to call rcar_dmac_chan_halt()
  1311. * because channel is already stopped in error case.
  1312. * We need to clear register and check DE bit as recovery.
  1313. */
  1314. rcar_dmac_write(dmac, RCAR_DMACHCLR, 1 << chan->index);
  1315. rcar_dmac_chcr_de_barrier(chan);
  1316. reinit = true;
  1317. goto spin_lock_end;
  1318. }
  1319. if (chcr & RCAR_DMACHCR_TE)
  1320. mask |= RCAR_DMACHCR_DE;
  1321. rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr & ~mask);
  1322. if (mask & RCAR_DMACHCR_DE)
  1323. rcar_dmac_chcr_de_barrier(chan);
  1324. if (chcr & RCAR_DMACHCR_DSE)
  1325. ret |= rcar_dmac_isr_desc_stage_end(chan);
  1326. if (chcr & RCAR_DMACHCR_TE)
  1327. ret |= rcar_dmac_isr_transfer_end(chan);
  1328. spin_lock_end:
  1329. spin_unlock(&chan->lock);
  1330. if (reinit) {
  1331. dev_err(chan->chan.device->dev, "Channel Address Error\n");
  1332. rcar_dmac_chan_reinit(chan);
  1333. ret = IRQ_HANDLED;
  1334. }
  1335. return ret;
  1336. }
  1337. static irqreturn_t rcar_dmac_isr_channel_thread(int irq, void *dev)
  1338. {
  1339. struct rcar_dmac_chan *chan = dev;
  1340. struct rcar_dmac_desc *desc;
  1341. struct dmaengine_desc_callback cb;
  1342. spin_lock_irq(&chan->lock);
  1343. /* For cyclic transfers notify the user after every chunk. */
  1344. if (chan->desc.running && chan->desc.running->cyclic) {
  1345. desc = chan->desc.running;
  1346. dmaengine_desc_get_callback(&desc->async_tx, &cb);
  1347. if (dmaengine_desc_callback_valid(&cb)) {
  1348. spin_unlock_irq(&chan->lock);
  1349. dmaengine_desc_callback_invoke(&cb, NULL);
  1350. spin_lock_irq(&chan->lock);
  1351. }
  1352. }
  1353. /*
  1354. * Call the callback function for all descriptors on the done list and
  1355. * move them to the ack wait list.
  1356. */
  1357. while (!list_empty(&chan->desc.done)) {
  1358. desc = list_first_entry(&chan->desc.done, struct rcar_dmac_desc,
  1359. node);
  1360. dma_cookie_complete(&desc->async_tx);
  1361. list_del(&desc->node);
  1362. dmaengine_desc_get_callback(&desc->async_tx, &cb);
  1363. if (dmaengine_desc_callback_valid(&cb)) {
  1364. spin_unlock_irq(&chan->lock);
  1365. /*
  1366. * We own the only reference to this descriptor, we can
  1367. * safely dereference it without holding the channel
  1368. * lock.
  1369. */
  1370. dmaengine_desc_callback_invoke(&cb, NULL);
  1371. spin_lock_irq(&chan->lock);
  1372. }
  1373. list_add_tail(&desc->node, &chan->desc.wait);
  1374. }
  1375. spin_unlock_irq(&chan->lock);
  1376. /* Recycle all acked descriptors. */
  1377. rcar_dmac_desc_recycle_acked(chan);
  1378. return IRQ_HANDLED;
  1379. }
  1380. /* -----------------------------------------------------------------------------
  1381. * OF xlate and channel filter
  1382. */
  1383. static bool rcar_dmac_chan_filter(struct dma_chan *chan, void *arg)
  1384. {
  1385. struct rcar_dmac *dmac = to_rcar_dmac(chan->device);
  1386. struct of_phandle_args *dma_spec = arg;
  1387. /*
  1388. * FIXME: Using a filter on OF platforms is a nonsense. The OF xlate
  1389. * function knows from which device it wants to allocate a channel from,
  1390. * and would be perfectly capable of selecting the channel it wants.
  1391. * Forcing it to call dma_request_channel() and iterate through all
  1392. * channels from all controllers is just pointless.
  1393. */
  1394. if (chan->device->device_config != rcar_dmac_device_config)
  1395. return false;
  1396. return !test_and_set_bit(dma_spec->args[0], dmac->modules);
  1397. }
  1398. static struct dma_chan *rcar_dmac_of_xlate(struct of_phandle_args *dma_spec,
  1399. struct of_dma *ofdma)
  1400. {
  1401. struct rcar_dmac_chan *rchan;
  1402. struct dma_chan *chan;
  1403. dma_cap_mask_t mask;
  1404. if (dma_spec->args_count != 1)
  1405. return NULL;
  1406. /* Only slave DMA channels can be allocated via DT */
  1407. dma_cap_zero(mask);
  1408. dma_cap_set(DMA_SLAVE, mask);
  1409. chan = __dma_request_channel(&mask, rcar_dmac_chan_filter, dma_spec,
  1410. ofdma->of_node);
  1411. if (!chan)
  1412. return NULL;
  1413. rchan = to_rcar_dmac_chan(chan);
  1414. rchan->mid_rid = dma_spec->args[0];
  1415. return chan;
  1416. }
  1417. /* -----------------------------------------------------------------------------
  1418. * Power management
  1419. */
  1420. #ifdef CONFIG_PM
  1421. static int rcar_dmac_runtime_suspend(struct device *dev)
  1422. {
  1423. return 0;
  1424. }
  1425. static int rcar_dmac_runtime_resume(struct device *dev)
  1426. {
  1427. struct rcar_dmac *dmac = dev_get_drvdata(dev);
  1428. return rcar_dmac_init(dmac);
  1429. }
  1430. #endif
  1431. static const struct dev_pm_ops rcar_dmac_pm = {
  1432. /*
  1433. * TODO for system sleep/resume:
  1434. * - Wait for the current transfer to complete and stop the device,
  1435. * - Resume transfers, if any.
  1436. */
  1437. SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
  1438. pm_runtime_force_resume)
  1439. SET_RUNTIME_PM_OPS(rcar_dmac_runtime_suspend, rcar_dmac_runtime_resume,
  1440. NULL)
  1441. };
  1442. /* -----------------------------------------------------------------------------
  1443. * Probe and remove
  1444. */
  1445. static int rcar_dmac_chan_probe(struct rcar_dmac *dmac,
  1446. struct rcar_dmac_chan *rchan,
  1447. const struct rcar_dmac_of_data *data,
  1448. unsigned int index)
  1449. {
  1450. struct platform_device *pdev = to_platform_device(dmac->dev);
  1451. struct dma_chan *chan = &rchan->chan;
  1452. char pdev_irqname[5];
  1453. char *irqname;
  1454. int ret;
  1455. rchan->index = index;
  1456. rchan->iomem = dmac->iomem + data->chan_offset_base +
  1457. data->chan_offset_stride * index;
  1458. rchan->mid_rid = -EINVAL;
  1459. spin_lock_init(&rchan->lock);
  1460. INIT_LIST_HEAD(&rchan->desc.free);
  1461. INIT_LIST_HEAD(&rchan->desc.pending);
  1462. INIT_LIST_HEAD(&rchan->desc.active);
  1463. INIT_LIST_HEAD(&rchan->desc.done);
  1464. INIT_LIST_HEAD(&rchan->desc.wait);
  1465. /* Request the channel interrupt. */
  1466. sprintf(pdev_irqname, "ch%u", index);
  1467. rchan->irq = platform_get_irq_byname(pdev, pdev_irqname);
  1468. if (rchan->irq < 0)
  1469. return -ENODEV;
  1470. irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u",
  1471. dev_name(dmac->dev), index);
  1472. if (!irqname)
  1473. return -ENOMEM;
  1474. /*
  1475. * Initialize the DMA engine channel and add it to the DMA engine
  1476. * channels list.
  1477. */
  1478. chan->device = &dmac->engine;
  1479. dma_cookie_init(chan);
  1480. list_add_tail(&chan->device_node, &dmac->engine.channels);
  1481. ret = devm_request_threaded_irq(dmac->dev, rchan->irq,
  1482. rcar_dmac_isr_channel,
  1483. rcar_dmac_isr_channel_thread, 0,
  1484. irqname, rchan);
  1485. if (ret) {
  1486. dev_err(dmac->dev, "failed to request IRQ %u (%d)\n",
  1487. rchan->irq, ret);
  1488. return ret;
  1489. }
  1490. return 0;
  1491. }
  1492. #define RCAR_DMAC_MAX_CHANNELS 32
  1493. static int rcar_dmac_parse_of(struct device *dev, struct rcar_dmac *dmac)
  1494. {
  1495. struct device_node *np = dev->of_node;
  1496. int ret;
  1497. ret = of_property_read_u32(np, "dma-channels", &dmac->n_channels);
  1498. if (ret < 0) {
  1499. dev_err(dev, "unable to read dma-channels property\n");
  1500. return ret;
  1501. }
  1502. /* The hardware and driver don't support more than 32 bits in CHCLR */
  1503. if (dmac->n_channels <= 0 ||
  1504. dmac->n_channels >= RCAR_DMAC_MAX_CHANNELS) {
  1505. dev_err(dev, "invalid number of channels %u\n",
  1506. dmac->n_channels);
  1507. return -EINVAL;
  1508. }
  1509. /*
  1510. * If the driver is unable to read dma-channel-mask property,
  1511. * the driver assumes that it can use all channels.
  1512. */
  1513. dmac->channels_mask = GENMASK(dmac->n_channels - 1, 0);
  1514. of_property_read_u32(np, "dma-channel-mask", &dmac->channels_mask);
  1515. /* If the property has out-of-channel mask, this driver clears it */
  1516. dmac->channels_mask &= GENMASK(dmac->n_channels - 1, 0);
  1517. return 0;
  1518. }
  1519. static int rcar_dmac_probe(struct platform_device *pdev)
  1520. {
  1521. const enum dma_slave_buswidth widths = DMA_SLAVE_BUSWIDTH_1_BYTE |
  1522. DMA_SLAVE_BUSWIDTH_2_BYTES | DMA_SLAVE_BUSWIDTH_4_BYTES |
  1523. DMA_SLAVE_BUSWIDTH_8_BYTES | DMA_SLAVE_BUSWIDTH_16_BYTES |
  1524. DMA_SLAVE_BUSWIDTH_32_BYTES | DMA_SLAVE_BUSWIDTH_64_BYTES;
  1525. struct dma_device *engine;
  1526. struct rcar_dmac *dmac;
  1527. const struct rcar_dmac_of_data *data;
  1528. unsigned int i;
  1529. int ret;
  1530. data = of_device_get_match_data(&pdev->dev);
  1531. if (!data)
  1532. return -EINVAL;
  1533. dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL);
  1534. if (!dmac)
  1535. return -ENOMEM;
  1536. dmac->dev = &pdev->dev;
  1537. platform_set_drvdata(pdev, dmac);
  1538. ret = dma_set_max_seg_size(dmac->dev, RCAR_DMATCR_MASK);
  1539. if (ret)
  1540. return ret;
  1541. ret = dma_set_mask_and_coherent(dmac->dev, DMA_BIT_MASK(40));
  1542. if (ret)
  1543. return ret;
  1544. ret = rcar_dmac_parse_of(&pdev->dev, dmac);
  1545. if (ret < 0)
  1546. return ret;
  1547. /*
  1548. * A still unconfirmed hardware bug prevents the IPMMU microTLB 0 to be
  1549. * flushed correctly, resulting in memory corruption. DMAC 0 channel 0
  1550. * is connected to microTLB 0 on currently supported platforms, so we
  1551. * can't use it with the IPMMU. As the IOMMU API operates at the device
  1552. * level we can't disable it selectively, so ignore channel 0 for now if
  1553. * the device is part of an IOMMU group.
  1554. */
  1555. if (device_iommu_mapped(&pdev->dev))
  1556. dmac->channels_mask &= ~BIT(0);
  1557. dmac->channels = devm_kcalloc(&pdev->dev, dmac->n_channels,
  1558. sizeof(*dmac->channels), GFP_KERNEL);
  1559. if (!dmac->channels)
  1560. return -ENOMEM;
  1561. /* Request resources. */
  1562. dmac->iomem = devm_platform_ioremap_resource(pdev, 0);
  1563. if (IS_ERR(dmac->iomem))
  1564. return PTR_ERR(dmac->iomem);
  1565. /* Enable runtime PM and initialize the device. */
  1566. pm_runtime_enable(&pdev->dev);
  1567. ret = pm_runtime_resume_and_get(&pdev->dev);
  1568. if (ret < 0) {
  1569. dev_err(&pdev->dev, "runtime PM get sync failed (%d)\n", ret);
  1570. return ret;
  1571. }
  1572. ret = rcar_dmac_init(dmac);
  1573. pm_runtime_put(&pdev->dev);
  1574. if (ret) {
  1575. dev_err(&pdev->dev, "failed to reset device\n");
  1576. goto error;
  1577. }
  1578. /* Initialize engine */
  1579. engine = &dmac->engine;
  1580. dma_cap_set(DMA_MEMCPY, engine->cap_mask);
  1581. dma_cap_set(DMA_SLAVE, engine->cap_mask);
  1582. engine->dev = &pdev->dev;
  1583. engine->copy_align = ilog2(RCAR_DMAC_MEMCPY_XFER_SIZE);
  1584. engine->src_addr_widths = widths;
  1585. engine->dst_addr_widths = widths;
  1586. engine->directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM);
  1587. engine->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
  1588. engine->device_alloc_chan_resources = rcar_dmac_alloc_chan_resources;
  1589. engine->device_free_chan_resources = rcar_dmac_free_chan_resources;
  1590. engine->device_prep_dma_memcpy = rcar_dmac_prep_dma_memcpy;
  1591. engine->device_prep_slave_sg = rcar_dmac_prep_slave_sg;
  1592. engine->device_prep_dma_cyclic = rcar_dmac_prep_dma_cyclic;
  1593. engine->device_config = rcar_dmac_device_config;
  1594. engine->device_pause = rcar_dmac_chan_pause;
  1595. engine->device_terminate_all = rcar_dmac_chan_terminate_all;
  1596. engine->device_tx_status = rcar_dmac_tx_status;
  1597. engine->device_issue_pending = rcar_dmac_issue_pending;
  1598. engine->device_synchronize = rcar_dmac_device_synchronize;
  1599. INIT_LIST_HEAD(&engine->channels);
  1600. for (i = 0; i < dmac->n_channels; ++i) {
  1601. if (!(dmac->channels_mask & BIT(i)))
  1602. continue;
  1603. ret = rcar_dmac_chan_probe(dmac, &dmac->channels[i], data, i);
  1604. if (ret < 0)
  1605. goto error;
  1606. }
  1607. /* Register the DMAC as a DMA provider for DT. */
  1608. ret = of_dma_controller_register(pdev->dev.of_node, rcar_dmac_of_xlate,
  1609. NULL);
  1610. if (ret < 0)
  1611. goto error;
  1612. /*
  1613. * Register the DMA engine device.
  1614. *
  1615. * Default transfer size of 32 bytes requires 32-byte alignment.
  1616. */
  1617. ret = dma_async_device_register(engine);
  1618. if (ret < 0)
  1619. goto error;
  1620. return 0;
  1621. error:
  1622. of_dma_controller_free(pdev->dev.of_node);
  1623. pm_runtime_disable(&pdev->dev);
  1624. return ret;
  1625. }
  1626. static int rcar_dmac_remove(struct platform_device *pdev)
  1627. {
  1628. struct rcar_dmac *dmac = platform_get_drvdata(pdev);
  1629. of_dma_controller_free(pdev->dev.of_node);
  1630. dma_async_device_unregister(&dmac->engine);
  1631. pm_runtime_disable(&pdev->dev);
  1632. return 0;
  1633. }
  1634. static void rcar_dmac_shutdown(struct platform_device *pdev)
  1635. {
  1636. struct rcar_dmac *dmac = platform_get_drvdata(pdev);
  1637. rcar_dmac_stop_all_chan(dmac);
  1638. }
  1639. static const struct rcar_dmac_of_data rcar_dmac_data = {
  1640. .chan_offset_base = 0x8000,
  1641. .chan_offset_stride = 0x80,
  1642. };
  1643. static const struct of_device_id rcar_dmac_of_ids[] = {
  1644. {
  1645. .compatible = "renesas,rcar-dmac",
  1646. .data = &rcar_dmac_data,
  1647. },
  1648. { /* Sentinel */ }
  1649. };
  1650. MODULE_DEVICE_TABLE(of, rcar_dmac_of_ids);
  1651. static struct platform_driver rcar_dmac_driver = {
  1652. .driver = {
  1653. .pm = &rcar_dmac_pm,
  1654. .name = "rcar-dmac",
  1655. .of_match_table = rcar_dmac_of_ids,
  1656. },
  1657. .probe = rcar_dmac_probe,
  1658. .remove = rcar_dmac_remove,
  1659. .shutdown = rcar_dmac_shutdown,
  1660. };
  1661. module_platform_driver(rcar_dmac_driver);
  1662. MODULE_DESCRIPTION("R-Car Gen2 DMA Controller Driver");
  1663. MODULE_AUTHOR("Laurent Pinchart <laurent.pinchart@ideasonboard.com>");
  1664. MODULE_LICENSE("GPL v2");