altera-msgdma.c 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * DMA driver for Altera mSGDMA IP core
  4. *
  5. * Copyright (C) 2017 Stefan Roese <sr@denx.de>
  6. *
  7. * Based on drivers/dma/xilinx/zynqmp_dma.c, which is:
  8. * Copyright (C) 2016 Xilinx, Inc. All rights reserved.
  9. */
  10. #include <linux/bitops.h>
  11. #include <linux/delay.h>
  12. #include <linux/dma-mapping.h>
  13. #include <linux/dmapool.h>
  14. #include <linux/init.h>
  15. #include <linux/interrupt.h>
  16. #include <linux/io.h>
  17. #include <linux/iopoll.h>
  18. #include <linux/module.h>
  19. #include <linux/platform_device.h>
  20. #include <linux/slab.h>
  21. #include "dmaengine.h"
  22. #define MSGDMA_MAX_TRANS_LEN U32_MAX
  23. #define MSGDMA_DESC_NUM 1024
  24. /**
  25. * struct msgdma_extended_desc - implements an extended descriptor
  26. * @read_addr_lo: data buffer source address low bits
  27. * @write_addr_lo: data buffer destination address low bits
  28. * @len: the number of bytes to transfer per descriptor
  29. * @burst_seq_num: bit 31:24 write burst
  30. * bit 23:16 read burst
  31. * bit 15:00 sequence number
  32. * @stride: bit 31:16 write stride
  33. * bit 15:00 read stride
  34. * @read_addr_hi: data buffer source address high bits
  35. * @write_addr_hi: data buffer destination address high bits
  36. * @control: characteristics of the transfer
  37. */
  38. struct msgdma_extended_desc {
  39. u32 read_addr_lo;
  40. u32 write_addr_lo;
  41. u32 len;
  42. u32 burst_seq_num;
  43. u32 stride;
  44. u32 read_addr_hi;
  45. u32 write_addr_hi;
  46. u32 control;
  47. };
  48. /* mSGDMA descriptor control field bit definitions */
  49. #define MSGDMA_DESC_CTL_SET_CH(x) ((x) & 0xff)
  50. #define MSGDMA_DESC_CTL_GEN_SOP BIT(8)
  51. #define MSGDMA_DESC_CTL_GEN_EOP BIT(9)
  52. #define MSGDMA_DESC_CTL_PARK_READS BIT(10)
  53. #define MSGDMA_DESC_CTL_PARK_WRITES BIT(11)
  54. #define MSGDMA_DESC_CTL_END_ON_EOP BIT(12)
  55. #define MSGDMA_DESC_CTL_END_ON_LEN BIT(13)
  56. #define MSGDMA_DESC_CTL_TR_COMP_IRQ BIT(14)
  57. #define MSGDMA_DESC_CTL_EARLY_IRQ BIT(15)
  58. #define MSGDMA_DESC_CTL_TR_ERR_IRQ GENMASK(23, 16)
  59. #define MSGDMA_DESC_CTL_EARLY_DONE BIT(24)
  60. /*
  61. * Writing "1" the "go" bit commits the entire descriptor into the
  62. * descriptor FIFO(s)
  63. */
  64. #define MSGDMA_DESC_CTL_GO BIT(31)
  65. /* Tx buffer control flags */
  66. #define MSGDMA_DESC_CTL_TX_FIRST (MSGDMA_DESC_CTL_GEN_SOP | \
  67. MSGDMA_DESC_CTL_TR_ERR_IRQ | \
  68. MSGDMA_DESC_CTL_GO)
  69. #define MSGDMA_DESC_CTL_TX_MIDDLE (MSGDMA_DESC_CTL_TR_ERR_IRQ | \
  70. MSGDMA_DESC_CTL_GO)
  71. #define MSGDMA_DESC_CTL_TX_LAST (MSGDMA_DESC_CTL_GEN_EOP | \
  72. MSGDMA_DESC_CTL_TR_COMP_IRQ | \
  73. MSGDMA_DESC_CTL_TR_ERR_IRQ | \
  74. MSGDMA_DESC_CTL_GO)
  75. #define MSGDMA_DESC_CTL_TX_SINGLE (MSGDMA_DESC_CTL_GEN_SOP | \
  76. MSGDMA_DESC_CTL_GEN_EOP | \
  77. MSGDMA_DESC_CTL_TR_COMP_IRQ | \
  78. MSGDMA_DESC_CTL_TR_ERR_IRQ | \
  79. MSGDMA_DESC_CTL_GO)
  80. #define MSGDMA_DESC_CTL_RX_SINGLE (MSGDMA_DESC_CTL_END_ON_EOP | \
  81. MSGDMA_DESC_CTL_END_ON_LEN | \
  82. MSGDMA_DESC_CTL_TR_COMP_IRQ | \
  83. MSGDMA_DESC_CTL_EARLY_IRQ | \
  84. MSGDMA_DESC_CTL_TR_ERR_IRQ | \
  85. MSGDMA_DESC_CTL_GO)
  86. /* mSGDMA extended descriptor stride definitions */
  87. #define MSGDMA_DESC_STRIDE_RD 0x00000001
  88. #define MSGDMA_DESC_STRIDE_WR 0x00010000
  89. #define MSGDMA_DESC_STRIDE_RW 0x00010001
  90. /* mSGDMA dispatcher control and status register map */
  91. #define MSGDMA_CSR_STATUS 0x00 /* Read / Clear */
  92. #define MSGDMA_CSR_CONTROL 0x04 /* Read / Write */
  93. #define MSGDMA_CSR_RW_FILL_LEVEL 0x08 /* 31:16 - write fill level */
  94. /* 15:00 - read fill level */
  95. #define MSGDMA_CSR_RESP_FILL_LEVEL 0x0c /* response FIFO fill level */
  96. #define MSGDMA_CSR_RW_SEQ_NUM 0x10 /* 31:16 - write seq number */
  97. /* 15:00 - read seq number */
  98. /* mSGDMA CSR status register bit definitions */
  99. #define MSGDMA_CSR_STAT_BUSY BIT(0)
  100. #define MSGDMA_CSR_STAT_DESC_BUF_EMPTY BIT(1)
  101. #define MSGDMA_CSR_STAT_DESC_BUF_FULL BIT(2)
  102. #define MSGDMA_CSR_STAT_RESP_BUF_EMPTY BIT(3)
  103. #define MSGDMA_CSR_STAT_RESP_BUF_FULL BIT(4)
  104. #define MSGDMA_CSR_STAT_STOPPED BIT(5)
  105. #define MSGDMA_CSR_STAT_RESETTING BIT(6)
  106. #define MSGDMA_CSR_STAT_STOPPED_ON_ERR BIT(7)
  107. #define MSGDMA_CSR_STAT_STOPPED_ON_EARLY BIT(8)
  108. #define MSGDMA_CSR_STAT_IRQ BIT(9)
  109. #define MSGDMA_CSR_STAT_MASK GENMASK(9, 0)
  110. #define MSGDMA_CSR_STAT_MASK_WITHOUT_IRQ GENMASK(8, 0)
  111. #define DESC_EMPTY (MSGDMA_CSR_STAT_DESC_BUF_EMPTY | \
  112. MSGDMA_CSR_STAT_RESP_BUF_EMPTY)
  113. /* mSGDMA CSR control register bit definitions */
  114. #define MSGDMA_CSR_CTL_STOP BIT(0)
  115. #define MSGDMA_CSR_CTL_RESET BIT(1)
  116. #define MSGDMA_CSR_CTL_STOP_ON_ERR BIT(2)
  117. #define MSGDMA_CSR_CTL_STOP_ON_EARLY BIT(3)
  118. #define MSGDMA_CSR_CTL_GLOBAL_INTR BIT(4)
  119. #define MSGDMA_CSR_CTL_STOP_DESCS BIT(5)
  120. /* mSGDMA CSR fill level bits */
  121. #define MSGDMA_CSR_WR_FILL_LEVEL_GET(v) (((v) & 0xffff0000) >> 16)
  122. #define MSGDMA_CSR_RD_FILL_LEVEL_GET(v) ((v) & 0x0000ffff)
  123. #define MSGDMA_CSR_RESP_FILL_LEVEL_GET(v) ((v) & 0x0000ffff)
  124. #define MSGDMA_CSR_SEQ_NUM_GET(v) (((v) & 0xffff0000) >> 16)
  125. /* mSGDMA response register map */
  126. #define MSGDMA_RESP_BYTES_TRANSFERRED 0x00
  127. #define MSGDMA_RESP_STATUS 0x04
  128. /* mSGDMA response register bit definitions */
  129. #define MSGDMA_RESP_EARLY_TERM BIT(8)
  130. #define MSGDMA_RESP_ERR_MASK 0xff
  131. /**
  132. * struct msgdma_sw_desc - implements a sw descriptor
  133. * @async_tx: support for the async_tx api
  134. * @hw_desc: assosiated HW descriptor
  135. * @node: node to move from the free list to the tx list
  136. * @tx_list: transmit list node
  137. */
  138. struct msgdma_sw_desc {
  139. struct dma_async_tx_descriptor async_tx;
  140. struct msgdma_extended_desc hw_desc;
  141. struct list_head node;
  142. struct list_head tx_list;
  143. };
  144. /*
  145. * struct msgdma_device - DMA device structure
  146. */
  147. struct msgdma_device {
  148. spinlock_t lock;
  149. struct device *dev;
  150. struct tasklet_struct irq_tasklet;
  151. struct list_head pending_list;
  152. struct list_head free_list;
  153. struct list_head active_list;
  154. struct list_head done_list;
  155. u32 desc_free_cnt;
  156. bool idle;
  157. struct dma_device dmadev;
  158. struct dma_chan dmachan;
  159. dma_addr_t hw_desq;
  160. struct msgdma_sw_desc *sw_desq;
  161. unsigned int npendings;
  162. struct dma_slave_config slave_cfg;
  163. int irq;
  164. /* mSGDMA controller */
  165. void __iomem *csr;
  166. /* mSGDMA descriptors */
  167. void __iomem *desc;
  168. /* mSGDMA response */
  169. void __iomem *resp;
  170. };
  171. #define to_mdev(chan) container_of(chan, struct msgdma_device, dmachan)
  172. #define tx_to_desc(tx) container_of(tx, struct msgdma_sw_desc, async_tx)
  173. /**
  174. * msgdma_get_descriptor - Get the sw descriptor from the pool
  175. * @mdev: Pointer to the Altera mSGDMA device structure
  176. *
  177. * Return: The sw descriptor
  178. */
  179. static struct msgdma_sw_desc *msgdma_get_descriptor(struct msgdma_device *mdev)
  180. {
  181. struct msgdma_sw_desc *desc;
  182. unsigned long flags;
  183. spin_lock_irqsave(&mdev->lock, flags);
  184. desc = list_first_entry(&mdev->free_list, struct msgdma_sw_desc, node);
  185. list_del(&desc->node);
  186. spin_unlock_irqrestore(&mdev->lock, flags);
  187. INIT_LIST_HEAD(&desc->tx_list);
  188. return desc;
  189. }
  190. /**
  191. * msgdma_free_descriptor - Issue pending transactions
  192. * @mdev: Pointer to the Altera mSGDMA device structure
  193. * @desc: Transaction descriptor pointer
  194. */
  195. static void msgdma_free_descriptor(struct msgdma_device *mdev,
  196. struct msgdma_sw_desc *desc)
  197. {
  198. struct msgdma_sw_desc *child, *next;
  199. mdev->desc_free_cnt++;
  200. list_add_tail(&desc->node, &mdev->free_list);
  201. list_for_each_entry_safe(child, next, &desc->tx_list, node) {
  202. mdev->desc_free_cnt++;
  203. list_move_tail(&child->node, &mdev->free_list);
  204. }
  205. }
  206. /**
  207. * msgdma_free_desc_list - Free descriptors list
  208. * @mdev: Pointer to the Altera mSGDMA device structure
  209. * @list: List to parse and delete the descriptor
  210. */
  211. static void msgdma_free_desc_list(struct msgdma_device *mdev,
  212. struct list_head *list)
  213. {
  214. struct msgdma_sw_desc *desc, *next;
  215. list_for_each_entry_safe(desc, next, list, node)
  216. msgdma_free_descriptor(mdev, desc);
  217. }
  218. /**
  219. * msgdma_desc_config - Configure the descriptor
  220. * @desc: Hw descriptor pointer
  221. * @dst: Destination buffer address
  222. * @src: Source buffer address
  223. * @len: Transfer length
  224. * @stride: Read/write stride value to set
  225. */
  226. static void msgdma_desc_config(struct msgdma_extended_desc *desc,
  227. dma_addr_t dst, dma_addr_t src, size_t len,
  228. u32 stride)
  229. {
  230. /* Set lower 32bits of src & dst addresses in the descriptor */
  231. desc->read_addr_lo = lower_32_bits(src);
  232. desc->write_addr_lo = lower_32_bits(dst);
  233. /* Set upper 32bits of src & dst addresses in the descriptor */
  234. desc->read_addr_hi = upper_32_bits(src);
  235. desc->write_addr_hi = upper_32_bits(dst);
  236. desc->len = len;
  237. desc->stride = stride;
  238. desc->burst_seq_num = 0; /* 0 will result in max burst length */
  239. /*
  240. * Don't set interrupt on xfer end yet, this will be done later
  241. * for the "last" descriptor
  242. */
  243. desc->control = MSGDMA_DESC_CTL_TR_ERR_IRQ | MSGDMA_DESC_CTL_GO |
  244. MSGDMA_DESC_CTL_END_ON_LEN;
  245. }
  246. /**
  247. * msgdma_desc_config_eod - Mark the descriptor as end descriptor
  248. * @desc: Hw descriptor pointer
  249. */
  250. static void msgdma_desc_config_eod(struct msgdma_extended_desc *desc)
  251. {
  252. desc->control |= MSGDMA_DESC_CTL_TR_COMP_IRQ;
  253. }
  254. /**
  255. * msgdma_tx_submit - Submit DMA transaction
  256. * @tx: Async transaction descriptor pointer
  257. *
  258. * Return: cookie value
  259. */
  260. static dma_cookie_t msgdma_tx_submit(struct dma_async_tx_descriptor *tx)
  261. {
  262. struct msgdma_device *mdev = to_mdev(tx->chan);
  263. struct msgdma_sw_desc *new;
  264. dma_cookie_t cookie;
  265. unsigned long flags;
  266. new = tx_to_desc(tx);
  267. spin_lock_irqsave(&mdev->lock, flags);
  268. cookie = dma_cookie_assign(tx);
  269. list_add_tail(&new->node, &mdev->pending_list);
  270. spin_unlock_irqrestore(&mdev->lock, flags);
  271. return cookie;
  272. }
  273. /**
  274. * msgdma_prep_memcpy - prepare descriptors for memcpy transaction
  275. * @dchan: DMA channel
  276. * @dma_dst: Destination buffer address
  277. * @dma_src: Source buffer address
  278. * @len: Transfer length
  279. * @flags: transfer ack flags
  280. *
  281. * Return: Async transaction descriptor on success and NULL on failure
  282. */
  283. static struct dma_async_tx_descriptor *
  284. msgdma_prep_memcpy(struct dma_chan *dchan, dma_addr_t dma_dst,
  285. dma_addr_t dma_src, size_t len, ulong flags)
  286. {
  287. struct msgdma_device *mdev = to_mdev(dchan);
  288. struct msgdma_sw_desc *new, *first = NULL;
  289. struct msgdma_extended_desc *desc;
  290. size_t copy;
  291. u32 desc_cnt;
  292. unsigned long irqflags;
  293. desc_cnt = DIV_ROUND_UP(len, MSGDMA_MAX_TRANS_LEN);
  294. spin_lock_irqsave(&mdev->lock, irqflags);
  295. if (desc_cnt > mdev->desc_free_cnt) {
  296. spin_unlock_irqrestore(&mdev->lock, irqflags);
  297. dev_dbg(mdev->dev, "mdev %p descs are not available\n", mdev);
  298. return NULL;
  299. }
  300. mdev->desc_free_cnt -= desc_cnt;
  301. spin_unlock_irqrestore(&mdev->lock, irqflags);
  302. do {
  303. /* Allocate and populate the descriptor */
  304. new = msgdma_get_descriptor(mdev);
  305. copy = min_t(size_t, len, MSGDMA_MAX_TRANS_LEN);
  306. desc = &new->hw_desc;
  307. msgdma_desc_config(desc, dma_dst, dma_src, copy,
  308. MSGDMA_DESC_STRIDE_RW);
  309. len -= copy;
  310. dma_src += copy;
  311. dma_dst += copy;
  312. if (!first)
  313. first = new;
  314. else
  315. list_add_tail(&new->node, &first->tx_list);
  316. } while (len);
  317. msgdma_desc_config_eod(desc);
  318. async_tx_ack(&first->async_tx);
  319. first->async_tx.flags = flags;
  320. return &first->async_tx;
  321. }
  322. /**
  323. * msgdma_prep_slave_sg - prepare descriptors for a slave sg transaction
  324. *
  325. * @dchan: DMA channel
  326. * @sgl: Destination scatter list
  327. * @sg_len: Number of entries in destination scatter list
  328. * @dir: DMA transfer direction
  329. * @flags: transfer ack flags
  330. * @context: transfer context (unused)
  331. */
  332. static struct dma_async_tx_descriptor *
  333. msgdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
  334. unsigned int sg_len, enum dma_transfer_direction dir,
  335. unsigned long flags, void *context)
  336. {
  337. struct msgdma_device *mdev = to_mdev(dchan);
  338. struct dma_slave_config *cfg = &mdev->slave_cfg;
  339. struct msgdma_sw_desc *new, *first = NULL;
  340. void *desc = NULL;
  341. size_t len, avail;
  342. dma_addr_t dma_dst, dma_src;
  343. u32 desc_cnt = 0, i;
  344. struct scatterlist *sg;
  345. u32 stride;
  346. unsigned long irqflags;
  347. for_each_sg(sgl, sg, sg_len, i)
  348. desc_cnt += DIV_ROUND_UP(sg_dma_len(sg), MSGDMA_MAX_TRANS_LEN);
  349. spin_lock_irqsave(&mdev->lock, irqflags);
  350. if (desc_cnt > mdev->desc_free_cnt) {
  351. spin_unlock_irqrestore(&mdev->lock, irqflags);
  352. dev_dbg(mdev->dev, "mdev %p descs are not available\n", mdev);
  353. return NULL;
  354. }
  355. mdev->desc_free_cnt -= desc_cnt;
  356. spin_unlock_irqrestore(&mdev->lock, irqflags);
  357. avail = sg_dma_len(sgl);
  358. /* Run until we are out of scatterlist entries */
  359. while (true) {
  360. /* Allocate and populate the descriptor */
  361. new = msgdma_get_descriptor(mdev);
  362. desc = &new->hw_desc;
  363. len = min_t(size_t, avail, MSGDMA_MAX_TRANS_LEN);
  364. if (dir == DMA_MEM_TO_DEV) {
  365. dma_src = sg_dma_address(sgl) + sg_dma_len(sgl) - avail;
  366. dma_dst = cfg->dst_addr;
  367. stride = MSGDMA_DESC_STRIDE_RD;
  368. } else {
  369. dma_src = cfg->src_addr;
  370. dma_dst = sg_dma_address(sgl) + sg_dma_len(sgl) - avail;
  371. stride = MSGDMA_DESC_STRIDE_WR;
  372. }
  373. msgdma_desc_config(desc, dma_dst, dma_src, len, stride);
  374. avail -= len;
  375. if (!first)
  376. first = new;
  377. else
  378. list_add_tail(&new->node, &first->tx_list);
  379. /* Fetch the next scatterlist entry */
  380. if (avail == 0) {
  381. if (sg_len == 0)
  382. break;
  383. sgl = sg_next(sgl);
  384. if (sgl == NULL)
  385. break;
  386. sg_len--;
  387. avail = sg_dma_len(sgl);
  388. }
  389. }
  390. msgdma_desc_config_eod(desc);
  391. first->async_tx.flags = flags;
  392. return &first->async_tx;
  393. }
  394. static int msgdma_dma_config(struct dma_chan *dchan,
  395. struct dma_slave_config *config)
  396. {
  397. struct msgdma_device *mdev = to_mdev(dchan);
  398. memcpy(&mdev->slave_cfg, config, sizeof(*config));
  399. return 0;
  400. }
  401. static void msgdma_reset(struct msgdma_device *mdev)
  402. {
  403. u32 val;
  404. int ret;
  405. /* Reset mSGDMA */
  406. iowrite32(MSGDMA_CSR_STAT_MASK, mdev->csr + MSGDMA_CSR_STATUS);
  407. iowrite32(MSGDMA_CSR_CTL_RESET, mdev->csr + MSGDMA_CSR_CONTROL);
  408. ret = readl_poll_timeout(mdev->csr + MSGDMA_CSR_STATUS, val,
  409. (val & MSGDMA_CSR_STAT_RESETTING) == 0,
  410. 1, 10000);
  411. if (ret)
  412. dev_err(mdev->dev, "DMA channel did not reset\n");
  413. /* Clear all status bits */
  414. iowrite32(MSGDMA_CSR_STAT_MASK, mdev->csr + MSGDMA_CSR_STATUS);
  415. /* Enable the DMA controller including interrupts */
  416. iowrite32(MSGDMA_CSR_CTL_STOP_ON_ERR | MSGDMA_CSR_CTL_STOP_ON_EARLY |
  417. MSGDMA_CSR_CTL_GLOBAL_INTR, mdev->csr + MSGDMA_CSR_CONTROL);
  418. mdev->idle = true;
  419. };
  420. static void msgdma_copy_one(struct msgdma_device *mdev,
  421. struct msgdma_sw_desc *desc)
  422. {
  423. void __iomem *hw_desc = mdev->desc;
  424. /*
  425. * Check if the DESC FIFO it not full. If its full, we need to wait
  426. * for at least one entry to become free again
  427. */
  428. while (ioread32(mdev->csr + MSGDMA_CSR_STATUS) &
  429. MSGDMA_CSR_STAT_DESC_BUF_FULL)
  430. mdelay(1);
  431. /*
  432. * The descriptor needs to get copied into the descriptor FIFO
  433. * of the DMA controller. The descriptor will get flushed to the
  434. * FIFO, once the last word (control word) is written. Since we
  435. * are not 100% sure that memcpy() writes all word in the "correct"
  436. * oder (address from low to high) on all architectures, we make
  437. * sure this control word is written last by single coding it and
  438. * adding some write-barriers here.
  439. */
  440. memcpy((void __force *)hw_desc, &desc->hw_desc,
  441. sizeof(desc->hw_desc) - sizeof(u32));
  442. /* Write control word last to flush this descriptor into the FIFO */
  443. mdev->idle = false;
  444. wmb();
  445. iowrite32(desc->hw_desc.control, hw_desc +
  446. offsetof(struct msgdma_extended_desc, control));
  447. wmb();
  448. }
  449. /**
  450. * msgdma_copy_desc_to_fifo - copy descriptor(s) into controller FIFO
  451. * @mdev: Pointer to the Altera mSGDMA device structure
  452. * @desc: Transaction descriptor pointer
  453. */
  454. static void msgdma_copy_desc_to_fifo(struct msgdma_device *mdev,
  455. struct msgdma_sw_desc *desc)
  456. {
  457. struct msgdma_sw_desc *sdesc, *next;
  458. msgdma_copy_one(mdev, desc);
  459. list_for_each_entry_safe(sdesc, next, &desc->tx_list, node)
  460. msgdma_copy_one(mdev, sdesc);
  461. }
  462. /**
  463. * msgdma_start_transfer - Initiate the new transfer
  464. * @mdev: Pointer to the Altera mSGDMA device structure
  465. */
  466. static void msgdma_start_transfer(struct msgdma_device *mdev)
  467. {
  468. struct msgdma_sw_desc *desc;
  469. if (!mdev->idle)
  470. return;
  471. desc = list_first_entry_or_null(&mdev->pending_list,
  472. struct msgdma_sw_desc, node);
  473. if (!desc)
  474. return;
  475. list_splice_tail_init(&mdev->pending_list, &mdev->active_list);
  476. msgdma_copy_desc_to_fifo(mdev, desc);
  477. }
  478. /**
  479. * msgdma_issue_pending - Issue pending transactions
  480. * @chan: DMA channel pointer
  481. */
  482. static void msgdma_issue_pending(struct dma_chan *chan)
  483. {
  484. struct msgdma_device *mdev = to_mdev(chan);
  485. unsigned long flags;
  486. spin_lock_irqsave(&mdev->lock, flags);
  487. msgdma_start_transfer(mdev);
  488. spin_unlock_irqrestore(&mdev->lock, flags);
  489. }
  490. /**
  491. * msgdma_chan_desc_cleanup - Cleanup the completed descriptors
  492. * @mdev: Pointer to the Altera mSGDMA device structure
  493. */
  494. static void msgdma_chan_desc_cleanup(struct msgdma_device *mdev)
  495. {
  496. struct msgdma_sw_desc *desc, *next;
  497. list_for_each_entry_safe(desc, next, &mdev->done_list, node) {
  498. dma_async_tx_callback callback;
  499. void *callback_param;
  500. list_del(&desc->node);
  501. callback = desc->async_tx.callback;
  502. callback_param = desc->async_tx.callback_param;
  503. if (callback) {
  504. spin_unlock(&mdev->lock);
  505. callback(callback_param);
  506. spin_lock(&mdev->lock);
  507. }
  508. /* Run any dependencies, then free the descriptor */
  509. msgdma_free_descriptor(mdev, desc);
  510. }
  511. }
  512. /**
  513. * msgdma_complete_descriptor - Mark the active descriptor as complete
  514. * @mdev: Pointer to the Altera mSGDMA device structure
  515. */
  516. static void msgdma_complete_descriptor(struct msgdma_device *mdev)
  517. {
  518. struct msgdma_sw_desc *desc;
  519. desc = list_first_entry_or_null(&mdev->active_list,
  520. struct msgdma_sw_desc, node);
  521. if (!desc)
  522. return;
  523. list_del(&desc->node);
  524. dma_cookie_complete(&desc->async_tx);
  525. list_add_tail(&desc->node, &mdev->done_list);
  526. }
  527. /**
  528. * msgdma_free_descriptors - Free channel descriptors
  529. * @mdev: Pointer to the Altera mSGDMA device structure
  530. */
  531. static void msgdma_free_descriptors(struct msgdma_device *mdev)
  532. {
  533. msgdma_free_desc_list(mdev, &mdev->active_list);
  534. msgdma_free_desc_list(mdev, &mdev->pending_list);
  535. msgdma_free_desc_list(mdev, &mdev->done_list);
  536. }
  537. /**
  538. * msgdma_free_chan_resources - Free channel resources
  539. * @dchan: DMA channel pointer
  540. */
  541. static void msgdma_free_chan_resources(struct dma_chan *dchan)
  542. {
  543. struct msgdma_device *mdev = to_mdev(dchan);
  544. unsigned long flags;
  545. spin_lock_irqsave(&mdev->lock, flags);
  546. msgdma_free_descriptors(mdev);
  547. spin_unlock_irqrestore(&mdev->lock, flags);
  548. kfree(mdev->sw_desq);
  549. }
  550. /**
  551. * msgdma_alloc_chan_resources - Allocate channel resources
  552. * @dchan: DMA channel
  553. *
  554. * Return: Number of descriptors on success and failure value on error
  555. */
  556. static int msgdma_alloc_chan_resources(struct dma_chan *dchan)
  557. {
  558. struct msgdma_device *mdev = to_mdev(dchan);
  559. struct msgdma_sw_desc *desc;
  560. int i;
  561. mdev->sw_desq = kcalloc(MSGDMA_DESC_NUM, sizeof(*desc), GFP_NOWAIT);
  562. if (!mdev->sw_desq)
  563. return -ENOMEM;
  564. mdev->idle = true;
  565. mdev->desc_free_cnt = MSGDMA_DESC_NUM;
  566. INIT_LIST_HEAD(&mdev->free_list);
  567. for (i = 0; i < MSGDMA_DESC_NUM; i++) {
  568. desc = mdev->sw_desq + i;
  569. dma_async_tx_descriptor_init(&desc->async_tx, &mdev->dmachan);
  570. desc->async_tx.tx_submit = msgdma_tx_submit;
  571. list_add_tail(&desc->node, &mdev->free_list);
  572. }
  573. return MSGDMA_DESC_NUM;
  574. }
  575. /**
  576. * msgdma_tasklet - Schedule completion tasklet
  577. * @t: Pointer to the Altera sSGDMA channel structure
  578. */
  579. static void msgdma_tasklet(struct tasklet_struct *t)
  580. {
  581. struct msgdma_device *mdev = from_tasklet(mdev, t, irq_tasklet);
  582. u32 count;
  583. u32 __maybe_unused size;
  584. u32 __maybe_unused status;
  585. unsigned long flags;
  586. spin_lock_irqsave(&mdev->lock, flags);
  587. /* Read number of responses that are available */
  588. count = ioread32(mdev->csr + MSGDMA_CSR_RESP_FILL_LEVEL);
  589. dev_dbg(mdev->dev, "%s (%d): response count=%d\n",
  590. __func__, __LINE__, count);
  591. while (count--) {
  592. /*
  593. * Read both longwords to purge this response from the FIFO
  594. * On Avalon-MM implementations, size and status do not
  595. * have any real values, like transferred bytes or error
  596. * bits. So we need to just drop these values.
  597. */
  598. size = ioread32(mdev->resp + MSGDMA_RESP_BYTES_TRANSFERRED);
  599. status = ioread32(mdev->resp + MSGDMA_RESP_STATUS);
  600. msgdma_complete_descriptor(mdev);
  601. msgdma_chan_desc_cleanup(mdev);
  602. }
  603. spin_unlock_irqrestore(&mdev->lock, flags);
  604. }
  605. /**
  606. * msgdma_irq_handler - Altera mSGDMA Interrupt handler
  607. * @irq: IRQ number
  608. * @data: Pointer to the Altera mSGDMA device structure
  609. *
  610. * Return: IRQ_HANDLED/IRQ_NONE
  611. */
  612. static irqreturn_t msgdma_irq_handler(int irq, void *data)
  613. {
  614. struct msgdma_device *mdev = data;
  615. u32 status;
  616. status = ioread32(mdev->csr + MSGDMA_CSR_STATUS);
  617. if ((status & MSGDMA_CSR_STAT_BUSY) == 0) {
  618. /* Start next transfer if the DMA controller is idle */
  619. spin_lock(&mdev->lock);
  620. mdev->idle = true;
  621. msgdma_start_transfer(mdev);
  622. spin_unlock(&mdev->lock);
  623. }
  624. tasklet_schedule(&mdev->irq_tasklet);
  625. /* Clear interrupt in mSGDMA controller */
  626. iowrite32(MSGDMA_CSR_STAT_IRQ, mdev->csr + MSGDMA_CSR_STATUS);
  627. return IRQ_HANDLED;
  628. }
  629. /**
  630. * msgdma_chan_remove - Channel remove function
  631. * @mdev: Pointer to the Altera mSGDMA device structure
  632. */
  633. static void msgdma_dev_remove(struct msgdma_device *mdev)
  634. {
  635. if (!mdev)
  636. return;
  637. devm_free_irq(mdev->dev, mdev->irq, mdev);
  638. tasklet_kill(&mdev->irq_tasklet);
  639. list_del(&mdev->dmachan.device_node);
  640. }
  641. static int request_and_map(struct platform_device *pdev, const char *name,
  642. struct resource **res, void __iomem **ptr)
  643. {
  644. struct resource *region;
  645. struct device *device = &pdev->dev;
  646. *res = platform_get_resource_byname(pdev, IORESOURCE_MEM, name);
  647. if (*res == NULL) {
  648. dev_err(device, "resource %s not defined\n", name);
  649. return -ENODEV;
  650. }
  651. region = devm_request_mem_region(device, (*res)->start,
  652. resource_size(*res), dev_name(device));
  653. if (region == NULL) {
  654. dev_err(device, "unable to request %s\n", name);
  655. return -EBUSY;
  656. }
  657. *ptr = devm_ioremap(device, region->start,
  658. resource_size(region));
  659. if (*ptr == NULL) {
  660. dev_err(device, "ioremap of %s failed!", name);
  661. return -ENOMEM;
  662. }
  663. return 0;
  664. }
  665. /**
  666. * msgdma_probe - Driver probe function
  667. * @pdev: Pointer to the platform_device structure
  668. *
  669. * Return: '0' on success and failure value on error
  670. */
  671. static int msgdma_probe(struct platform_device *pdev)
  672. {
  673. struct msgdma_device *mdev;
  674. struct dma_device *dma_dev;
  675. struct resource *dma_res;
  676. int ret;
  677. mdev = devm_kzalloc(&pdev->dev, sizeof(*mdev), GFP_NOWAIT);
  678. if (!mdev)
  679. return -ENOMEM;
  680. mdev->dev = &pdev->dev;
  681. /* Map CSR space */
  682. ret = request_and_map(pdev, "csr", &dma_res, &mdev->csr);
  683. if (ret)
  684. return ret;
  685. /* Map (extended) descriptor space */
  686. ret = request_and_map(pdev, "desc", &dma_res, &mdev->desc);
  687. if (ret)
  688. return ret;
  689. /* Map response space */
  690. ret = request_and_map(pdev, "resp", &dma_res, &mdev->resp);
  691. if (ret)
  692. return ret;
  693. platform_set_drvdata(pdev, mdev);
  694. /* Get interrupt nr from platform data */
  695. mdev->irq = platform_get_irq(pdev, 0);
  696. if (mdev->irq < 0)
  697. return -ENXIO;
  698. ret = devm_request_irq(&pdev->dev, mdev->irq, msgdma_irq_handler,
  699. 0, dev_name(&pdev->dev), mdev);
  700. if (ret)
  701. return ret;
  702. tasklet_setup(&mdev->irq_tasklet, msgdma_tasklet);
  703. dma_cookie_init(&mdev->dmachan);
  704. spin_lock_init(&mdev->lock);
  705. INIT_LIST_HEAD(&mdev->active_list);
  706. INIT_LIST_HEAD(&mdev->pending_list);
  707. INIT_LIST_HEAD(&mdev->done_list);
  708. INIT_LIST_HEAD(&mdev->free_list);
  709. dma_dev = &mdev->dmadev;
  710. /* Set DMA capabilities */
  711. dma_cap_zero(dma_dev->cap_mask);
  712. dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
  713. dma_cap_set(DMA_SLAVE, dma_dev->cap_mask);
  714. dma_dev->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
  715. dma_dev->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
  716. dma_dev->directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM) |
  717. BIT(DMA_MEM_TO_MEM);
  718. dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
  719. /* Init DMA link list */
  720. INIT_LIST_HEAD(&dma_dev->channels);
  721. /* Set base routines */
  722. dma_dev->device_tx_status = dma_cookie_status;
  723. dma_dev->device_issue_pending = msgdma_issue_pending;
  724. dma_dev->dev = &pdev->dev;
  725. dma_dev->copy_align = DMAENGINE_ALIGN_4_BYTES;
  726. dma_dev->device_prep_dma_memcpy = msgdma_prep_memcpy;
  727. dma_dev->device_prep_slave_sg = msgdma_prep_slave_sg;
  728. dma_dev->device_config = msgdma_dma_config;
  729. dma_dev->device_alloc_chan_resources = msgdma_alloc_chan_resources;
  730. dma_dev->device_free_chan_resources = msgdma_free_chan_resources;
  731. mdev->dmachan.device = dma_dev;
  732. list_add_tail(&mdev->dmachan.device_node, &dma_dev->channels);
  733. /* Set DMA mask to 64 bits */
  734. ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
  735. if (ret) {
  736. dev_warn(&pdev->dev, "unable to set coherent mask to 64");
  737. ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
  738. if (ret)
  739. goto fail;
  740. }
  741. msgdma_reset(mdev);
  742. ret = dma_async_device_register(dma_dev);
  743. if (ret)
  744. goto fail;
  745. dev_notice(&pdev->dev, "Altera mSGDMA driver probe success\n");
  746. return 0;
  747. fail:
  748. msgdma_dev_remove(mdev);
  749. return ret;
  750. }
  751. /**
  752. * msgdma_dma_remove - Driver remove function
  753. * @pdev: Pointer to the platform_device structure
  754. *
  755. * Return: Always '0'
  756. */
  757. static int msgdma_remove(struct platform_device *pdev)
  758. {
  759. struct msgdma_device *mdev = platform_get_drvdata(pdev);
  760. dma_async_device_unregister(&mdev->dmadev);
  761. msgdma_dev_remove(mdev);
  762. dev_notice(&pdev->dev, "Altera mSGDMA driver removed\n");
  763. return 0;
  764. }
  765. static struct platform_driver msgdma_driver = {
  766. .driver = {
  767. .name = "altera-msgdma",
  768. },
  769. .probe = msgdma_probe,
  770. .remove = msgdma_remove,
  771. };
  772. module_platform_driver(msgdma_driver);
  773. MODULE_ALIAS("platform:altera-msgdma");
  774. MODULE_DESCRIPTION("Altera mSGDMA driver");
  775. MODULE_AUTHOR("Stefan Roese <sr@denx.de>");
  776. MODULE_LICENSE("GPL");