mxs-dma.c 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875
  1. // SPDX-License-Identifier: GPL-2.0
  2. //
  3. // Copyright 2011 Freescale Semiconductor, Inc. All Rights Reserved.
  4. //
  5. // Refer to drivers/dma/imx-sdma.c
  6. #include <linux/init.h>
  7. #include <linux/types.h>
  8. #include <linux/mm.h>
  9. #include <linux/interrupt.h>
  10. #include <linux/clk.h>
  11. #include <linux/wait.h>
  12. #include <linux/sched.h>
  13. #include <linux/semaphore.h>
  14. #include <linux/device.h>
  15. #include <linux/dma-mapping.h>
  16. #include <linux/slab.h>
  17. #include <linux/platform_device.h>
  18. #include <linux/dmaengine.h>
  19. #include <linux/delay.h>
  20. #include <linux/module.h>
  21. #include <linux/stmp_device.h>
  22. #include <linux/of.h>
  23. #include <linux/of_device.h>
  24. #include <linux/of_dma.h>
  25. #include <linux/list.h>
  26. #include <linux/dma/mxs-dma.h>
  27. #include <asm/irq.h>
  28. #include "dmaengine.h"
  29. /*
  30. * NOTE: The term "PIO" throughout the mxs-dma implementation means
  31. * PIO mode of mxs apbh-dma and apbx-dma. With this working mode,
  32. * dma can program the controller registers of peripheral devices.
  33. */
  34. #define dma_is_apbh(mxs_dma) ((mxs_dma)->type == MXS_DMA_APBH)
  35. #define apbh_is_old(mxs_dma) ((mxs_dma)->dev_id == IMX23_DMA)
  36. #define HW_APBHX_CTRL0 0x000
  37. #define BM_APBH_CTRL0_APB_BURST8_EN (1 << 29)
  38. #define BM_APBH_CTRL0_APB_BURST_EN (1 << 28)
  39. #define BP_APBH_CTRL0_RESET_CHANNEL 16
  40. #define HW_APBHX_CTRL1 0x010
  41. #define HW_APBHX_CTRL2 0x020
  42. #define HW_APBHX_CHANNEL_CTRL 0x030
  43. #define BP_APBHX_CHANNEL_CTRL_RESET_CHANNEL 16
  44. /*
  45. * The offset of NXTCMDAR register is different per both dma type and version,
  46. * while stride for each channel is all the same 0x70.
  47. */
  48. #define HW_APBHX_CHn_NXTCMDAR(d, n) \
  49. (((dma_is_apbh(d) && apbh_is_old(d)) ? 0x050 : 0x110) + (n) * 0x70)
  50. #define HW_APBHX_CHn_SEMA(d, n) \
  51. (((dma_is_apbh(d) && apbh_is_old(d)) ? 0x080 : 0x140) + (n) * 0x70)
  52. #define HW_APBHX_CHn_BAR(d, n) \
  53. (((dma_is_apbh(d) && apbh_is_old(d)) ? 0x070 : 0x130) + (n) * 0x70)
  54. #define HW_APBX_CHn_DEBUG1(d, n) (0x150 + (n) * 0x70)
  55. /*
  56. * ccw bits definitions
  57. *
  58. * COMMAND: 0..1 (2)
  59. * CHAIN: 2 (1)
  60. * IRQ: 3 (1)
  61. * NAND_LOCK: 4 (1) - not implemented
  62. * NAND_WAIT4READY: 5 (1) - not implemented
  63. * DEC_SEM: 6 (1)
  64. * WAIT4END: 7 (1)
  65. * HALT_ON_TERMINATE: 8 (1)
  66. * TERMINATE_FLUSH: 9 (1)
  67. * RESERVED: 10..11 (2)
  68. * PIO_NUM: 12..15 (4)
  69. */
  70. #define BP_CCW_COMMAND 0
  71. #define BM_CCW_COMMAND (3 << 0)
  72. #define CCW_CHAIN (1 << 2)
  73. #define CCW_IRQ (1 << 3)
  74. #define CCW_WAIT4RDY (1 << 5)
  75. #define CCW_DEC_SEM (1 << 6)
  76. #define CCW_WAIT4END (1 << 7)
  77. #define CCW_HALT_ON_TERM (1 << 8)
  78. #define CCW_TERM_FLUSH (1 << 9)
  79. #define BP_CCW_PIO_NUM 12
  80. #define BM_CCW_PIO_NUM (0xf << 12)
  81. #define BF_CCW(value, field) (((value) << BP_CCW_##field) & BM_CCW_##field)
  82. #define MXS_DMA_CMD_NO_XFER 0
  83. #define MXS_DMA_CMD_WRITE 1
  84. #define MXS_DMA_CMD_READ 2
  85. #define MXS_DMA_CMD_DMA_SENSE 3 /* not implemented */
  86. struct mxs_dma_ccw {
  87. u32 next;
  88. u16 bits;
  89. u16 xfer_bytes;
  90. #define MAX_XFER_BYTES 0xff00
  91. u32 bufaddr;
  92. #define MXS_PIO_WORDS 16
  93. u32 pio_words[MXS_PIO_WORDS];
  94. };
  95. #define CCW_BLOCK_SIZE (4 * PAGE_SIZE)
  96. #define NUM_CCW (int)(CCW_BLOCK_SIZE / sizeof(struct mxs_dma_ccw))
  97. struct mxs_dma_chan {
  98. struct mxs_dma_engine *mxs_dma;
  99. struct dma_chan chan;
  100. struct dma_async_tx_descriptor desc;
  101. struct tasklet_struct tasklet;
  102. unsigned int chan_irq;
  103. struct mxs_dma_ccw *ccw;
  104. dma_addr_t ccw_phys;
  105. int desc_count;
  106. enum dma_status status;
  107. unsigned int flags;
  108. bool reset;
  109. #define MXS_DMA_SG_LOOP (1 << 0)
  110. #define MXS_DMA_USE_SEMAPHORE (1 << 1)
  111. };
  112. #define MXS_DMA_CHANNELS 16
  113. #define MXS_DMA_CHANNELS_MASK 0xffff
  114. enum mxs_dma_devtype {
  115. MXS_DMA_APBH,
  116. MXS_DMA_APBX,
  117. };
  118. enum mxs_dma_id {
  119. IMX23_DMA,
  120. IMX28_DMA,
  121. };
  122. struct mxs_dma_engine {
  123. enum mxs_dma_id dev_id;
  124. enum mxs_dma_devtype type;
  125. void __iomem *base;
  126. struct clk *clk;
  127. struct dma_device dma_device;
  128. struct mxs_dma_chan mxs_chans[MXS_DMA_CHANNELS];
  129. struct platform_device *pdev;
  130. unsigned int nr_channels;
  131. };
  132. struct mxs_dma_type {
  133. enum mxs_dma_id id;
  134. enum mxs_dma_devtype type;
  135. };
  136. static struct mxs_dma_type mxs_dma_types[] = {
  137. {
  138. .id = IMX23_DMA,
  139. .type = MXS_DMA_APBH,
  140. }, {
  141. .id = IMX23_DMA,
  142. .type = MXS_DMA_APBX,
  143. }, {
  144. .id = IMX28_DMA,
  145. .type = MXS_DMA_APBH,
  146. }, {
  147. .id = IMX28_DMA,
  148. .type = MXS_DMA_APBX,
  149. }
  150. };
  151. static const struct platform_device_id mxs_dma_ids[] = {
  152. {
  153. .name = "imx23-dma-apbh",
  154. .driver_data = (kernel_ulong_t) &mxs_dma_types[0],
  155. }, {
  156. .name = "imx23-dma-apbx",
  157. .driver_data = (kernel_ulong_t) &mxs_dma_types[1],
  158. }, {
  159. .name = "imx28-dma-apbh",
  160. .driver_data = (kernel_ulong_t) &mxs_dma_types[2],
  161. }, {
  162. .name = "imx28-dma-apbx",
  163. .driver_data = (kernel_ulong_t) &mxs_dma_types[3],
  164. }, {
  165. /* end of list */
  166. }
  167. };
  168. static const struct of_device_id mxs_dma_dt_ids[] = {
  169. { .compatible = "fsl,imx23-dma-apbh", .data = &mxs_dma_ids[0], },
  170. { .compatible = "fsl,imx23-dma-apbx", .data = &mxs_dma_ids[1], },
  171. { .compatible = "fsl,imx28-dma-apbh", .data = &mxs_dma_ids[2], },
  172. { .compatible = "fsl,imx28-dma-apbx", .data = &mxs_dma_ids[3], },
  173. { /* sentinel */ }
  174. };
  175. MODULE_DEVICE_TABLE(of, mxs_dma_dt_ids);
  176. static struct mxs_dma_chan *to_mxs_dma_chan(struct dma_chan *chan)
  177. {
  178. return container_of(chan, struct mxs_dma_chan, chan);
  179. }
  180. static void mxs_dma_reset_chan(struct dma_chan *chan)
  181. {
  182. struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);
  183. struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;
  184. int chan_id = mxs_chan->chan.chan_id;
  185. /*
  186. * mxs dma channel resets can cause a channel stall. To recover from a
  187. * channel stall, we have to reset the whole DMA engine. To avoid this,
  188. * we use cyclic DMA with semaphores, that are enhanced in
  189. * mxs_dma_int_handler. To reset the channel, we can simply stop writing
  190. * into the semaphore counter.
  191. */
  192. if (mxs_chan->flags & MXS_DMA_USE_SEMAPHORE &&
  193. mxs_chan->flags & MXS_DMA_SG_LOOP) {
  194. mxs_chan->reset = true;
  195. } else if (dma_is_apbh(mxs_dma) && apbh_is_old(mxs_dma)) {
  196. writel(1 << (chan_id + BP_APBH_CTRL0_RESET_CHANNEL),
  197. mxs_dma->base + HW_APBHX_CTRL0 + STMP_OFFSET_REG_SET);
  198. } else {
  199. unsigned long elapsed = 0;
  200. const unsigned long max_wait = 50000; /* 50ms */
  201. void __iomem *reg_dbg1 = mxs_dma->base +
  202. HW_APBX_CHn_DEBUG1(mxs_dma, chan_id);
  203. /*
  204. * On i.MX28 APBX, the DMA channel can stop working if we reset
  205. * the channel while it is in READ_FLUSH (0x08) state.
  206. * We wait here until we leave the state. Then we trigger the
  207. * reset. Waiting a maximum of 50ms, the kernel shouldn't crash
  208. * because of this.
  209. */
  210. while ((readl(reg_dbg1) & 0xf) == 0x8 && elapsed < max_wait) {
  211. udelay(100);
  212. elapsed += 100;
  213. }
  214. if (elapsed >= max_wait)
  215. dev_err(&mxs_chan->mxs_dma->pdev->dev,
  216. "Failed waiting for the DMA channel %d to leave state READ_FLUSH, trying to reset channel in READ_FLUSH state now\n",
  217. chan_id);
  218. writel(1 << (chan_id + BP_APBHX_CHANNEL_CTRL_RESET_CHANNEL),
  219. mxs_dma->base + HW_APBHX_CHANNEL_CTRL + STMP_OFFSET_REG_SET);
  220. }
  221. mxs_chan->status = DMA_COMPLETE;
  222. }
  223. static void mxs_dma_enable_chan(struct dma_chan *chan)
  224. {
  225. struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);
  226. struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;
  227. int chan_id = mxs_chan->chan.chan_id;
  228. /* set cmd_addr up */
  229. writel(mxs_chan->ccw_phys,
  230. mxs_dma->base + HW_APBHX_CHn_NXTCMDAR(mxs_dma, chan_id));
  231. /* write 1 to SEMA to kick off the channel */
  232. if (mxs_chan->flags & MXS_DMA_USE_SEMAPHORE &&
  233. mxs_chan->flags & MXS_DMA_SG_LOOP) {
  234. /* A cyclic DMA consists of at least 2 segments, so initialize
  235. * the semaphore with 2 so we have enough time to add 1 to the
  236. * semaphore if we need to */
  237. writel(2, mxs_dma->base + HW_APBHX_CHn_SEMA(mxs_dma, chan_id));
  238. } else {
  239. writel(1, mxs_dma->base + HW_APBHX_CHn_SEMA(mxs_dma, chan_id));
  240. }
  241. mxs_chan->reset = false;
  242. }
  243. static void mxs_dma_disable_chan(struct dma_chan *chan)
  244. {
  245. struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);
  246. mxs_chan->status = DMA_COMPLETE;
  247. }
  248. static int mxs_dma_pause_chan(struct dma_chan *chan)
  249. {
  250. struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);
  251. struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;
  252. int chan_id = mxs_chan->chan.chan_id;
  253. /* freeze the channel */
  254. if (dma_is_apbh(mxs_dma) && apbh_is_old(mxs_dma))
  255. writel(1 << chan_id,
  256. mxs_dma->base + HW_APBHX_CTRL0 + STMP_OFFSET_REG_SET);
  257. else
  258. writel(1 << chan_id,
  259. mxs_dma->base + HW_APBHX_CHANNEL_CTRL + STMP_OFFSET_REG_SET);
  260. mxs_chan->status = DMA_PAUSED;
  261. return 0;
  262. }
  263. static int mxs_dma_resume_chan(struct dma_chan *chan)
  264. {
  265. struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);
  266. struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;
  267. int chan_id = mxs_chan->chan.chan_id;
  268. /* unfreeze the channel */
  269. if (dma_is_apbh(mxs_dma) && apbh_is_old(mxs_dma))
  270. writel(1 << chan_id,
  271. mxs_dma->base + HW_APBHX_CTRL0 + STMP_OFFSET_REG_CLR);
  272. else
  273. writel(1 << chan_id,
  274. mxs_dma->base + HW_APBHX_CHANNEL_CTRL + STMP_OFFSET_REG_CLR);
  275. mxs_chan->status = DMA_IN_PROGRESS;
  276. return 0;
  277. }
  278. static dma_cookie_t mxs_dma_tx_submit(struct dma_async_tx_descriptor *tx)
  279. {
  280. return dma_cookie_assign(tx);
  281. }
  282. static void mxs_dma_tasklet(struct tasklet_struct *t)
  283. {
  284. struct mxs_dma_chan *mxs_chan = from_tasklet(mxs_chan, t, tasklet);
  285. dmaengine_desc_get_callback_invoke(&mxs_chan->desc, NULL);
  286. }
  287. static int mxs_dma_irq_to_chan(struct mxs_dma_engine *mxs_dma, int irq)
  288. {
  289. int i;
  290. for (i = 0; i != mxs_dma->nr_channels; ++i)
  291. if (mxs_dma->mxs_chans[i].chan_irq == irq)
  292. return i;
  293. return -EINVAL;
  294. }
  295. static irqreturn_t mxs_dma_int_handler(int irq, void *dev_id)
  296. {
  297. struct mxs_dma_engine *mxs_dma = dev_id;
  298. struct mxs_dma_chan *mxs_chan;
  299. u32 completed;
  300. u32 err;
  301. int chan = mxs_dma_irq_to_chan(mxs_dma, irq);
  302. if (chan < 0)
  303. return IRQ_NONE;
  304. /* completion status */
  305. completed = readl(mxs_dma->base + HW_APBHX_CTRL1);
  306. completed = (completed >> chan) & 0x1;
  307. /* Clear interrupt */
  308. writel((1 << chan),
  309. mxs_dma->base + HW_APBHX_CTRL1 + STMP_OFFSET_REG_CLR);
  310. /* error status */
  311. err = readl(mxs_dma->base + HW_APBHX_CTRL2);
  312. err &= (1 << (MXS_DMA_CHANNELS + chan)) | (1 << chan);
  313. /*
  314. * error status bit is in the upper 16 bits, error irq bit in the lower
  315. * 16 bits. We transform it into a simpler error code:
  316. * err: 0x00 = no error, 0x01 = TERMINATION, 0x02 = BUS_ERROR
  317. */
  318. err = (err >> (MXS_DMA_CHANNELS + chan)) + (err >> chan);
  319. /* Clear error irq */
  320. writel((1 << chan),
  321. mxs_dma->base + HW_APBHX_CTRL2 + STMP_OFFSET_REG_CLR);
  322. /*
  323. * When both completion and error of termination bits set at the
  324. * same time, we do not take it as an error. IOW, it only becomes
  325. * an error we need to handle here in case of either it's a bus
  326. * error or a termination error with no completion. 0x01 is termination
  327. * error, so we can subtract err & completed to get the real error case.
  328. */
  329. err -= err & completed;
  330. mxs_chan = &mxs_dma->mxs_chans[chan];
  331. if (err) {
  332. dev_dbg(mxs_dma->dma_device.dev,
  333. "%s: error in channel %d\n", __func__,
  334. chan);
  335. mxs_chan->status = DMA_ERROR;
  336. mxs_dma_reset_chan(&mxs_chan->chan);
  337. } else if (mxs_chan->status != DMA_COMPLETE) {
  338. if (mxs_chan->flags & MXS_DMA_SG_LOOP) {
  339. mxs_chan->status = DMA_IN_PROGRESS;
  340. if (mxs_chan->flags & MXS_DMA_USE_SEMAPHORE)
  341. writel(1, mxs_dma->base +
  342. HW_APBHX_CHn_SEMA(mxs_dma, chan));
  343. } else {
  344. mxs_chan->status = DMA_COMPLETE;
  345. }
  346. }
  347. if (mxs_chan->status == DMA_COMPLETE) {
  348. if (mxs_chan->reset)
  349. return IRQ_HANDLED;
  350. dma_cookie_complete(&mxs_chan->desc);
  351. }
  352. /* schedule tasklet on this channel */
  353. tasklet_schedule(&mxs_chan->tasklet);
  354. return IRQ_HANDLED;
  355. }
  356. static int mxs_dma_alloc_chan_resources(struct dma_chan *chan)
  357. {
  358. struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);
  359. struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;
  360. int ret;
  361. mxs_chan->ccw = dma_alloc_coherent(mxs_dma->dma_device.dev,
  362. CCW_BLOCK_SIZE,
  363. &mxs_chan->ccw_phys, GFP_KERNEL);
  364. if (!mxs_chan->ccw) {
  365. ret = -ENOMEM;
  366. goto err_alloc;
  367. }
  368. ret = request_irq(mxs_chan->chan_irq, mxs_dma_int_handler,
  369. 0, "mxs-dma", mxs_dma);
  370. if (ret)
  371. goto err_irq;
  372. ret = clk_prepare_enable(mxs_dma->clk);
  373. if (ret)
  374. goto err_clk;
  375. mxs_dma_reset_chan(chan);
  376. dma_async_tx_descriptor_init(&mxs_chan->desc, chan);
  377. mxs_chan->desc.tx_submit = mxs_dma_tx_submit;
  378. /* the descriptor is ready */
  379. async_tx_ack(&mxs_chan->desc);
  380. return 0;
  381. err_clk:
  382. free_irq(mxs_chan->chan_irq, mxs_dma);
  383. err_irq:
  384. dma_free_coherent(mxs_dma->dma_device.dev, CCW_BLOCK_SIZE,
  385. mxs_chan->ccw, mxs_chan->ccw_phys);
  386. err_alloc:
  387. return ret;
  388. }
  389. static void mxs_dma_free_chan_resources(struct dma_chan *chan)
  390. {
  391. struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);
  392. struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;
  393. mxs_dma_disable_chan(chan);
  394. free_irq(mxs_chan->chan_irq, mxs_dma);
  395. dma_free_coherent(mxs_dma->dma_device.dev, CCW_BLOCK_SIZE,
  396. mxs_chan->ccw, mxs_chan->ccw_phys);
  397. clk_disable_unprepare(mxs_dma->clk);
  398. }
  399. /*
  400. * How to use the flags for ->device_prep_slave_sg() :
  401. * [1] If there is only one DMA command in the DMA chain, the code should be:
  402. * ......
  403. * ->device_prep_slave_sg(DMA_CTRL_ACK);
  404. * ......
  405. * [2] If there are two DMA commands in the DMA chain, the code should be
  406. * ......
  407. * ->device_prep_slave_sg(0);
  408. * ......
  409. * ->device_prep_slave_sg(DMA_CTRL_ACK);
  410. * ......
  411. * [3] If there are more than two DMA commands in the DMA chain, the code
  412. * should be:
  413. * ......
  414. * ->device_prep_slave_sg(0); // First
  415. * ......
  416. * ->device_prep_slave_sg(DMA_CTRL_ACK]);
  417. * ......
  418. * ->device_prep_slave_sg(DMA_CTRL_ACK); // Last
  419. * ......
  420. */
  421. static struct dma_async_tx_descriptor *mxs_dma_prep_slave_sg(
  422. struct dma_chan *chan, struct scatterlist *sgl,
  423. unsigned int sg_len, enum dma_transfer_direction direction,
  424. unsigned long flags, void *context)
  425. {
  426. struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);
  427. struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;
  428. struct mxs_dma_ccw *ccw;
  429. struct scatterlist *sg;
  430. u32 i, j;
  431. u32 *pio;
  432. int idx = 0;
  433. if (mxs_chan->status == DMA_IN_PROGRESS)
  434. idx = mxs_chan->desc_count;
  435. if (sg_len + idx > NUM_CCW) {
  436. dev_err(mxs_dma->dma_device.dev,
  437. "maximum number of sg exceeded: %d > %d\n",
  438. sg_len, NUM_CCW);
  439. goto err_out;
  440. }
  441. mxs_chan->status = DMA_IN_PROGRESS;
  442. mxs_chan->flags = 0;
  443. /*
  444. * If the sg is prepared with append flag set, the sg
  445. * will be appended to the last prepared sg.
  446. */
  447. if (idx) {
  448. BUG_ON(idx < 1);
  449. ccw = &mxs_chan->ccw[idx - 1];
  450. ccw->next = mxs_chan->ccw_phys + sizeof(*ccw) * idx;
  451. ccw->bits |= CCW_CHAIN;
  452. ccw->bits &= ~CCW_IRQ;
  453. ccw->bits &= ~CCW_DEC_SEM;
  454. } else {
  455. idx = 0;
  456. }
  457. if (direction == DMA_TRANS_NONE) {
  458. ccw = &mxs_chan->ccw[idx++];
  459. pio = (u32 *) sgl;
  460. for (j = 0; j < sg_len;)
  461. ccw->pio_words[j++] = *pio++;
  462. ccw->bits = 0;
  463. ccw->bits |= CCW_IRQ;
  464. ccw->bits |= CCW_DEC_SEM;
  465. if (flags & MXS_DMA_CTRL_WAIT4END)
  466. ccw->bits |= CCW_WAIT4END;
  467. ccw->bits |= CCW_HALT_ON_TERM;
  468. ccw->bits |= CCW_TERM_FLUSH;
  469. ccw->bits |= BF_CCW(sg_len, PIO_NUM);
  470. ccw->bits |= BF_CCW(MXS_DMA_CMD_NO_XFER, COMMAND);
  471. if (flags & MXS_DMA_CTRL_WAIT4RDY)
  472. ccw->bits |= CCW_WAIT4RDY;
  473. } else {
  474. for_each_sg(sgl, sg, sg_len, i) {
  475. if (sg_dma_len(sg) > MAX_XFER_BYTES) {
  476. dev_err(mxs_dma->dma_device.dev, "maximum bytes for sg entry exceeded: %d > %d\n",
  477. sg_dma_len(sg), MAX_XFER_BYTES);
  478. goto err_out;
  479. }
  480. ccw = &mxs_chan->ccw[idx++];
  481. ccw->next = mxs_chan->ccw_phys + sizeof(*ccw) * idx;
  482. ccw->bufaddr = sg->dma_address;
  483. ccw->xfer_bytes = sg_dma_len(sg);
  484. ccw->bits = 0;
  485. ccw->bits |= CCW_CHAIN;
  486. ccw->bits |= CCW_HALT_ON_TERM;
  487. ccw->bits |= CCW_TERM_FLUSH;
  488. ccw->bits |= BF_CCW(direction == DMA_DEV_TO_MEM ?
  489. MXS_DMA_CMD_WRITE : MXS_DMA_CMD_READ,
  490. COMMAND);
  491. if (i + 1 == sg_len) {
  492. ccw->bits &= ~CCW_CHAIN;
  493. ccw->bits |= CCW_IRQ;
  494. ccw->bits |= CCW_DEC_SEM;
  495. if (flags & MXS_DMA_CTRL_WAIT4END)
  496. ccw->bits |= CCW_WAIT4END;
  497. }
  498. }
  499. }
  500. mxs_chan->desc_count = idx;
  501. return &mxs_chan->desc;
  502. err_out:
  503. mxs_chan->status = DMA_ERROR;
  504. return NULL;
  505. }
  506. static struct dma_async_tx_descriptor *mxs_dma_prep_dma_cyclic(
  507. struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len,
  508. size_t period_len, enum dma_transfer_direction direction,
  509. unsigned long flags)
  510. {
  511. struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);
  512. struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;
  513. u32 num_periods = buf_len / period_len;
  514. u32 i = 0, buf = 0;
  515. if (mxs_chan->status == DMA_IN_PROGRESS)
  516. return NULL;
  517. mxs_chan->status = DMA_IN_PROGRESS;
  518. mxs_chan->flags |= MXS_DMA_SG_LOOP;
  519. mxs_chan->flags |= MXS_DMA_USE_SEMAPHORE;
  520. if (num_periods > NUM_CCW) {
  521. dev_err(mxs_dma->dma_device.dev,
  522. "maximum number of sg exceeded: %d > %d\n",
  523. num_periods, NUM_CCW);
  524. goto err_out;
  525. }
  526. if (period_len > MAX_XFER_BYTES) {
  527. dev_err(mxs_dma->dma_device.dev,
  528. "maximum period size exceeded: %zu > %d\n",
  529. period_len, MAX_XFER_BYTES);
  530. goto err_out;
  531. }
  532. while (buf < buf_len) {
  533. struct mxs_dma_ccw *ccw = &mxs_chan->ccw[i];
  534. if (i + 1 == num_periods)
  535. ccw->next = mxs_chan->ccw_phys;
  536. else
  537. ccw->next = mxs_chan->ccw_phys + sizeof(*ccw) * (i + 1);
  538. ccw->bufaddr = dma_addr;
  539. ccw->xfer_bytes = period_len;
  540. ccw->bits = 0;
  541. ccw->bits |= CCW_CHAIN;
  542. ccw->bits |= CCW_IRQ;
  543. ccw->bits |= CCW_HALT_ON_TERM;
  544. ccw->bits |= CCW_TERM_FLUSH;
  545. ccw->bits |= CCW_DEC_SEM;
  546. ccw->bits |= BF_CCW(direction == DMA_DEV_TO_MEM ?
  547. MXS_DMA_CMD_WRITE : MXS_DMA_CMD_READ, COMMAND);
  548. dma_addr += period_len;
  549. buf += period_len;
  550. i++;
  551. }
  552. mxs_chan->desc_count = i;
  553. return &mxs_chan->desc;
  554. err_out:
  555. mxs_chan->status = DMA_ERROR;
  556. return NULL;
  557. }
  558. static int mxs_dma_terminate_all(struct dma_chan *chan)
  559. {
  560. mxs_dma_reset_chan(chan);
  561. mxs_dma_disable_chan(chan);
  562. return 0;
  563. }
  564. static enum dma_status mxs_dma_tx_status(struct dma_chan *chan,
  565. dma_cookie_t cookie, struct dma_tx_state *txstate)
  566. {
  567. struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);
  568. struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;
  569. u32 residue = 0;
  570. if (mxs_chan->status == DMA_IN_PROGRESS &&
  571. mxs_chan->flags & MXS_DMA_SG_LOOP) {
  572. struct mxs_dma_ccw *last_ccw;
  573. u32 bar;
  574. last_ccw = &mxs_chan->ccw[mxs_chan->desc_count - 1];
  575. residue = last_ccw->xfer_bytes + last_ccw->bufaddr;
  576. bar = readl(mxs_dma->base +
  577. HW_APBHX_CHn_BAR(mxs_dma, chan->chan_id));
  578. residue -= bar;
  579. }
  580. dma_set_tx_state(txstate, chan->completed_cookie, chan->cookie,
  581. residue);
  582. return mxs_chan->status;
  583. }
  584. static int __init mxs_dma_init(struct mxs_dma_engine *mxs_dma)
  585. {
  586. int ret;
  587. ret = clk_prepare_enable(mxs_dma->clk);
  588. if (ret)
  589. return ret;
  590. ret = stmp_reset_block(mxs_dma->base);
  591. if (ret)
  592. goto err_out;
  593. /* enable apbh burst */
  594. if (dma_is_apbh(mxs_dma)) {
  595. writel(BM_APBH_CTRL0_APB_BURST_EN,
  596. mxs_dma->base + HW_APBHX_CTRL0 + STMP_OFFSET_REG_SET);
  597. writel(BM_APBH_CTRL0_APB_BURST8_EN,
  598. mxs_dma->base + HW_APBHX_CTRL0 + STMP_OFFSET_REG_SET);
  599. }
  600. /* enable irq for all the channels */
  601. writel(MXS_DMA_CHANNELS_MASK << MXS_DMA_CHANNELS,
  602. mxs_dma->base + HW_APBHX_CTRL1 + STMP_OFFSET_REG_SET);
  603. err_out:
  604. clk_disable_unprepare(mxs_dma->clk);
  605. return ret;
  606. }
  607. struct mxs_dma_filter_param {
  608. unsigned int chan_id;
  609. };
  610. static bool mxs_dma_filter_fn(struct dma_chan *chan, void *fn_param)
  611. {
  612. struct mxs_dma_filter_param *param = fn_param;
  613. struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);
  614. struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;
  615. int chan_irq;
  616. if (chan->chan_id != param->chan_id)
  617. return false;
  618. chan_irq = platform_get_irq(mxs_dma->pdev, param->chan_id);
  619. if (chan_irq < 0)
  620. return false;
  621. mxs_chan->chan_irq = chan_irq;
  622. return true;
  623. }
  624. static struct dma_chan *mxs_dma_xlate(struct of_phandle_args *dma_spec,
  625. struct of_dma *ofdma)
  626. {
  627. struct mxs_dma_engine *mxs_dma = ofdma->of_dma_data;
  628. dma_cap_mask_t mask = mxs_dma->dma_device.cap_mask;
  629. struct mxs_dma_filter_param param;
  630. if (dma_spec->args_count != 1)
  631. return NULL;
  632. param.chan_id = dma_spec->args[0];
  633. if (param.chan_id >= mxs_dma->nr_channels)
  634. return NULL;
  635. return __dma_request_channel(&mask, mxs_dma_filter_fn, &param,
  636. ofdma->of_node);
  637. }
  638. static int __init mxs_dma_probe(struct platform_device *pdev)
  639. {
  640. struct device_node *np = pdev->dev.of_node;
  641. const struct platform_device_id *id_entry;
  642. const struct of_device_id *of_id;
  643. const struct mxs_dma_type *dma_type;
  644. struct mxs_dma_engine *mxs_dma;
  645. struct resource *iores;
  646. int ret, i;
  647. mxs_dma = devm_kzalloc(&pdev->dev, sizeof(*mxs_dma), GFP_KERNEL);
  648. if (!mxs_dma)
  649. return -ENOMEM;
  650. ret = of_property_read_u32(np, "dma-channels", &mxs_dma->nr_channels);
  651. if (ret) {
  652. dev_err(&pdev->dev, "failed to read dma-channels\n");
  653. return ret;
  654. }
  655. of_id = of_match_device(mxs_dma_dt_ids, &pdev->dev);
  656. if (of_id)
  657. id_entry = of_id->data;
  658. else
  659. id_entry = platform_get_device_id(pdev);
  660. dma_type = (struct mxs_dma_type *)id_entry->driver_data;
  661. mxs_dma->type = dma_type->type;
  662. mxs_dma->dev_id = dma_type->id;
  663. iores = platform_get_resource(pdev, IORESOURCE_MEM, 0);
  664. mxs_dma->base = devm_ioremap_resource(&pdev->dev, iores);
  665. if (IS_ERR(mxs_dma->base))
  666. return PTR_ERR(mxs_dma->base);
  667. mxs_dma->clk = devm_clk_get(&pdev->dev, NULL);
  668. if (IS_ERR(mxs_dma->clk))
  669. return PTR_ERR(mxs_dma->clk);
  670. dma_cap_set(DMA_SLAVE, mxs_dma->dma_device.cap_mask);
  671. dma_cap_set(DMA_CYCLIC, mxs_dma->dma_device.cap_mask);
  672. INIT_LIST_HEAD(&mxs_dma->dma_device.channels);
  673. /* Initialize channel parameters */
  674. for (i = 0; i < MXS_DMA_CHANNELS; i++) {
  675. struct mxs_dma_chan *mxs_chan = &mxs_dma->mxs_chans[i];
  676. mxs_chan->mxs_dma = mxs_dma;
  677. mxs_chan->chan.device = &mxs_dma->dma_device;
  678. dma_cookie_init(&mxs_chan->chan);
  679. tasklet_setup(&mxs_chan->tasklet, mxs_dma_tasklet);
  680. /* Add the channel to mxs_chan list */
  681. list_add_tail(&mxs_chan->chan.device_node,
  682. &mxs_dma->dma_device.channels);
  683. }
  684. ret = mxs_dma_init(mxs_dma);
  685. if (ret)
  686. return ret;
  687. mxs_dma->pdev = pdev;
  688. mxs_dma->dma_device.dev = &pdev->dev;
  689. /* mxs_dma gets 65535 bytes maximum sg size */
  690. dma_set_max_seg_size(mxs_dma->dma_device.dev, MAX_XFER_BYTES);
  691. mxs_dma->dma_device.device_alloc_chan_resources = mxs_dma_alloc_chan_resources;
  692. mxs_dma->dma_device.device_free_chan_resources = mxs_dma_free_chan_resources;
  693. mxs_dma->dma_device.device_tx_status = mxs_dma_tx_status;
  694. mxs_dma->dma_device.device_prep_slave_sg = mxs_dma_prep_slave_sg;
  695. mxs_dma->dma_device.device_prep_dma_cyclic = mxs_dma_prep_dma_cyclic;
  696. mxs_dma->dma_device.device_pause = mxs_dma_pause_chan;
  697. mxs_dma->dma_device.device_resume = mxs_dma_resume_chan;
  698. mxs_dma->dma_device.device_terminate_all = mxs_dma_terminate_all;
  699. mxs_dma->dma_device.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
  700. mxs_dma->dma_device.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
  701. mxs_dma->dma_device.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
  702. mxs_dma->dma_device.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
  703. mxs_dma->dma_device.device_issue_pending = mxs_dma_enable_chan;
  704. ret = dmaenginem_async_device_register(&mxs_dma->dma_device);
  705. if (ret) {
  706. dev_err(mxs_dma->dma_device.dev, "unable to register\n");
  707. return ret;
  708. }
  709. ret = of_dma_controller_register(np, mxs_dma_xlate, mxs_dma);
  710. if (ret) {
  711. dev_err(mxs_dma->dma_device.dev,
  712. "failed to register controller\n");
  713. }
  714. dev_info(mxs_dma->dma_device.dev, "initialized\n");
  715. return 0;
  716. }
  717. static struct platform_driver mxs_dma_driver = {
  718. .driver = {
  719. .name = "mxs-dma",
  720. .of_match_table = mxs_dma_dt_ids,
  721. },
  722. .id_table = mxs_dma_ids,
  723. };
  724. static int __init mxs_dma_module_init(void)
  725. {
  726. return platform_driver_probe(&mxs_dma_driver, mxs_dma_probe);
  727. }
  728. subsys_initcall(mxs_dma_module_init);