zx_dma.c 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright 2015 Linaro.
  4. */
  5. #include <linux/sched.h>
  6. #include <linux/device.h>
  7. #include <linux/dmaengine.h>
  8. #include <linux/dma-mapping.h>
  9. #include <linux/dmapool.h>
  10. #include <linux/init.h>
  11. #include <linux/interrupt.h>
  12. #include <linux/kernel.h>
  13. #include <linux/module.h>
  14. #include <linux/platform_device.h>
  15. #include <linux/slab.h>
  16. #include <linux/spinlock.h>
  17. #include <linux/of_device.h>
  18. #include <linux/of.h>
  19. #include <linux/clk.h>
  20. #include <linux/of_dma.h>
  21. #include "virt-dma.h"
  22. #define DRIVER_NAME "zx-dma"
  23. #define DMA_ALIGN 4
  24. #define DMA_MAX_SIZE (0x10000 - 512)
  25. #define LLI_BLOCK_SIZE (4 * PAGE_SIZE)
  26. #define REG_ZX_SRC_ADDR 0x00
  27. #define REG_ZX_DST_ADDR 0x04
  28. #define REG_ZX_TX_X_COUNT 0x08
  29. #define REG_ZX_TX_ZY_COUNT 0x0c
  30. #define REG_ZX_SRC_ZY_STEP 0x10
  31. #define REG_ZX_DST_ZY_STEP 0x14
  32. #define REG_ZX_LLI_ADDR 0x1c
  33. #define REG_ZX_CTRL 0x20
  34. #define REG_ZX_TC_IRQ 0x800
  35. #define REG_ZX_SRC_ERR_IRQ 0x804
  36. #define REG_ZX_DST_ERR_IRQ 0x808
  37. #define REG_ZX_CFG_ERR_IRQ 0x80c
  38. #define REG_ZX_TC_IRQ_RAW 0x810
  39. #define REG_ZX_SRC_ERR_IRQ_RAW 0x814
  40. #define REG_ZX_DST_ERR_IRQ_RAW 0x818
  41. #define REG_ZX_CFG_ERR_IRQ_RAW 0x81c
  42. #define REG_ZX_STATUS 0x820
  43. #define REG_ZX_DMA_GRP_PRIO 0x824
  44. #define REG_ZX_DMA_ARB 0x828
  45. #define ZX_FORCE_CLOSE BIT(31)
  46. #define ZX_DST_BURST_WIDTH(x) (((x) & 0x7) << 13)
  47. #define ZX_MAX_BURST_LEN 16
  48. #define ZX_SRC_BURST_LEN(x) (((x) & 0xf) << 9)
  49. #define ZX_SRC_BURST_WIDTH(x) (((x) & 0x7) << 6)
  50. #define ZX_IRQ_ENABLE_ALL (3 << 4)
  51. #define ZX_DST_FIFO_MODE BIT(3)
  52. #define ZX_SRC_FIFO_MODE BIT(2)
  53. #define ZX_SOFT_REQ BIT(1)
  54. #define ZX_CH_ENABLE BIT(0)
  55. #define ZX_DMA_BUSWIDTHS \
  56. (BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \
  57. BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
  58. BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
  59. BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \
  60. BIT(DMA_SLAVE_BUSWIDTH_8_BYTES))
  61. enum zx_dma_burst_width {
  62. ZX_DMA_WIDTH_8BIT = 0,
  63. ZX_DMA_WIDTH_16BIT = 1,
  64. ZX_DMA_WIDTH_32BIT = 2,
  65. ZX_DMA_WIDTH_64BIT = 3,
  66. };
  67. struct zx_desc_hw {
  68. u32 saddr;
  69. u32 daddr;
  70. u32 src_x;
  71. u32 src_zy;
  72. u32 src_zy_step;
  73. u32 dst_zy_step;
  74. u32 reserved1;
  75. u32 lli;
  76. u32 ctr;
  77. u32 reserved[7]; /* pack as hardware registers region size */
  78. } __aligned(32);
  79. struct zx_dma_desc_sw {
  80. struct virt_dma_desc vd;
  81. dma_addr_t desc_hw_lli;
  82. size_t desc_num;
  83. size_t size;
  84. struct zx_desc_hw *desc_hw;
  85. };
  86. struct zx_dma_phy;
  87. struct zx_dma_chan {
  88. struct dma_slave_config slave_cfg;
  89. int id; /* Request phy chan id */
  90. u32 ccfg;
  91. u32 cyclic;
  92. struct virt_dma_chan vc;
  93. struct zx_dma_phy *phy;
  94. struct list_head node;
  95. dma_addr_t dev_addr;
  96. enum dma_status status;
  97. };
  98. struct zx_dma_phy {
  99. u32 idx;
  100. void __iomem *base;
  101. struct zx_dma_chan *vchan;
  102. struct zx_dma_desc_sw *ds_run;
  103. struct zx_dma_desc_sw *ds_done;
  104. };
  105. struct zx_dma_dev {
  106. struct dma_device slave;
  107. void __iomem *base;
  108. spinlock_t lock; /* lock for ch and phy */
  109. struct list_head chan_pending;
  110. struct zx_dma_phy *phy;
  111. struct zx_dma_chan *chans;
  112. struct clk *clk;
  113. struct dma_pool *pool;
  114. u32 dma_channels;
  115. u32 dma_requests;
  116. int irq;
  117. };
  118. #define to_zx_dma(dmadev) container_of(dmadev, struct zx_dma_dev, slave)
  119. static struct zx_dma_chan *to_zx_chan(struct dma_chan *chan)
  120. {
  121. return container_of(chan, struct zx_dma_chan, vc.chan);
  122. }
  123. static void zx_dma_terminate_chan(struct zx_dma_phy *phy, struct zx_dma_dev *d)
  124. {
  125. u32 val = 0;
  126. val = readl_relaxed(phy->base + REG_ZX_CTRL);
  127. val &= ~ZX_CH_ENABLE;
  128. val |= ZX_FORCE_CLOSE;
  129. writel_relaxed(val, phy->base + REG_ZX_CTRL);
  130. val = 0x1 << phy->idx;
  131. writel_relaxed(val, d->base + REG_ZX_TC_IRQ_RAW);
  132. writel_relaxed(val, d->base + REG_ZX_SRC_ERR_IRQ_RAW);
  133. writel_relaxed(val, d->base + REG_ZX_DST_ERR_IRQ_RAW);
  134. writel_relaxed(val, d->base + REG_ZX_CFG_ERR_IRQ_RAW);
  135. }
  136. static void zx_dma_set_desc(struct zx_dma_phy *phy, struct zx_desc_hw *hw)
  137. {
  138. writel_relaxed(hw->saddr, phy->base + REG_ZX_SRC_ADDR);
  139. writel_relaxed(hw->daddr, phy->base + REG_ZX_DST_ADDR);
  140. writel_relaxed(hw->src_x, phy->base + REG_ZX_TX_X_COUNT);
  141. writel_relaxed(0, phy->base + REG_ZX_TX_ZY_COUNT);
  142. writel_relaxed(0, phy->base + REG_ZX_SRC_ZY_STEP);
  143. writel_relaxed(0, phy->base + REG_ZX_DST_ZY_STEP);
  144. writel_relaxed(hw->lli, phy->base + REG_ZX_LLI_ADDR);
  145. writel_relaxed(hw->ctr, phy->base + REG_ZX_CTRL);
  146. }
  147. static u32 zx_dma_get_curr_lli(struct zx_dma_phy *phy)
  148. {
  149. return readl_relaxed(phy->base + REG_ZX_LLI_ADDR);
  150. }
  151. static u32 zx_dma_get_chan_stat(struct zx_dma_dev *d)
  152. {
  153. return readl_relaxed(d->base + REG_ZX_STATUS);
  154. }
  155. static void zx_dma_init_state(struct zx_dma_dev *d)
  156. {
  157. /* set same priority */
  158. writel_relaxed(0x0, d->base + REG_ZX_DMA_ARB);
  159. /* clear all irq */
  160. writel_relaxed(0xffffffff, d->base + REG_ZX_TC_IRQ_RAW);
  161. writel_relaxed(0xffffffff, d->base + REG_ZX_SRC_ERR_IRQ_RAW);
  162. writel_relaxed(0xffffffff, d->base + REG_ZX_DST_ERR_IRQ_RAW);
  163. writel_relaxed(0xffffffff, d->base + REG_ZX_CFG_ERR_IRQ_RAW);
  164. }
  165. static int zx_dma_start_txd(struct zx_dma_chan *c)
  166. {
  167. struct zx_dma_dev *d = to_zx_dma(c->vc.chan.device);
  168. struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
  169. if (!c->phy)
  170. return -EAGAIN;
  171. if (BIT(c->phy->idx) & zx_dma_get_chan_stat(d))
  172. return -EAGAIN;
  173. if (vd) {
  174. struct zx_dma_desc_sw *ds =
  175. container_of(vd, struct zx_dma_desc_sw, vd);
  176. /*
  177. * fetch and remove request from vc->desc_issued
  178. * so vc->desc_issued only contains desc pending
  179. */
  180. list_del(&ds->vd.node);
  181. c->phy->ds_run = ds;
  182. c->phy->ds_done = NULL;
  183. /* start dma */
  184. zx_dma_set_desc(c->phy, ds->desc_hw);
  185. return 0;
  186. }
  187. c->phy->ds_done = NULL;
  188. c->phy->ds_run = NULL;
  189. return -EAGAIN;
  190. }
  191. static void zx_dma_task(struct zx_dma_dev *d)
  192. {
  193. struct zx_dma_phy *p;
  194. struct zx_dma_chan *c, *cn;
  195. unsigned pch, pch_alloc = 0;
  196. unsigned long flags;
  197. /* check new dma request of running channel in vc->desc_issued */
  198. list_for_each_entry_safe(c, cn, &d->slave.channels,
  199. vc.chan.device_node) {
  200. spin_lock_irqsave(&c->vc.lock, flags);
  201. p = c->phy;
  202. if (p && p->ds_done && zx_dma_start_txd(c)) {
  203. /* No current txd associated with this channel */
  204. dev_dbg(d->slave.dev, "pchan %u: free\n", p->idx);
  205. /* Mark this channel free */
  206. c->phy = NULL;
  207. p->vchan = NULL;
  208. }
  209. spin_unlock_irqrestore(&c->vc.lock, flags);
  210. }
  211. /* check new channel request in d->chan_pending */
  212. spin_lock_irqsave(&d->lock, flags);
  213. while (!list_empty(&d->chan_pending)) {
  214. c = list_first_entry(&d->chan_pending,
  215. struct zx_dma_chan, node);
  216. p = &d->phy[c->id];
  217. if (!p->vchan) {
  218. /* remove from d->chan_pending */
  219. list_del_init(&c->node);
  220. pch_alloc |= 1 << c->id;
  221. /* Mark this channel allocated */
  222. p->vchan = c;
  223. c->phy = p;
  224. } else {
  225. dev_dbg(d->slave.dev, "pchan %u: busy!\n", c->id);
  226. }
  227. }
  228. spin_unlock_irqrestore(&d->lock, flags);
  229. for (pch = 0; pch < d->dma_channels; pch++) {
  230. if (pch_alloc & (1 << pch)) {
  231. p = &d->phy[pch];
  232. c = p->vchan;
  233. if (c) {
  234. spin_lock_irqsave(&c->vc.lock, flags);
  235. zx_dma_start_txd(c);
  236. spin_unlock_irqrestore(&c->vc.lock, flags);
  237. }
  238. }
  239. }
  240. }
  241. static irqreturn_t zx_dma_int_handler(int irq, void *dev_id)
  242. {
  243. struct zx_dma_dev *d = (struct zx_dma_dev *)dev_id;
  244. struct zx_dma_phy *p;
  245. struct zx_dma_chan *c;
  246. u32 tc = readl_relaxed(d->base + REG_ZX_TC_IRQ);
  247. u32 serr = readl_relaxed(d->base + REG_ZX_SRC_ERR_IRQ);
  248. u32 derr = readl_relaxed(d->base + REG_ZX_DST_ERR_IRQ);
  249. u32 cfg = readl_relaxed(d->base + REG_ZX_CFG_ERR_IRQ);
  250. u32 i, irq_chan = 0, task = 0;
  251. while (tc) {
  252. i = __ffs(tc);
  253. tc &= ~BIT(i);
  254. p = &d->phy[i];
  255. c = p->vchan;
  256. if (c) {
  257. spin_lock(&c->vc.lock);
  258. if (c->cyclic) {
  259. vchan_cyclic_callback(&p->ds_run->vd);
  260. } else {
  261. vchan_cookie_complete(&p->ds_run->vd);
  262. p->ds_done = p->ds_run;
  263. task = 1;
  264. }
  265. spin_unlock(&c->vc.lock);
  266. irq_chan |= BIT(i);
  267. }
  268. }
  269. if (serr || derr || cfg)
  270. dev_warn(d->slave.dev, "DMA ERR src 0x%x, dst 0x%x, cfg 0x%x\n",
  271. serr, derr, cfg);
  272. writel_relaxed(irq_chan, d->base + REG_ZX_TC_IRQ_RAW);
  273. writel_relaxed(serr, d->base + REG_ZX_SRC_ERR_IRQ_RAW);
  274. writel_relaxed(derr, d->base + REG_ZX_DST_ERR_IRQ_RAW);
  275. writel_relaxed(cfg, d->base + REG_ZX_CFG_ERR_IRQ_RAW);
  276. if (task)
  277. zx_dma_task(d);
  278. return IRQ_HANDLED;
  279. }
  280. static void zx_dma_free_chan_resources(struct dma_chan *chan)
  281. {
  282. struct zx_dma_chan *c = to_zx_chan(chan);
  283. struct zx_dma_dev *d = to_zx_dma(chan->device);
  284. unsigned long flags;
  285. spin_lock_irqsave(&d->lock, flags);
  286. list_del_init(&c->node);
  287. spin_unlock_irqrestore(&d->lock, flags);
  288. vchan_free_chan_resources(&c->vc);
  289. c->ccfg = 0;
  290. }
  291. static enum dma_status zx_dma_tx_status(struct dma_chan *chan,
  292. dma_cookie_t cookie,
  293. struct dma_tx_state *state)
  294. {
  295. struct zx_dma_chan *c = to_zx_chan(chan);
  296. struct zx_dma_phy *p;
  297. struct virt_dma_desc *vd;
  298. unsigned long flags;
  299. enum dma_status ret;
  300. size_t bytes = 0;
  301. ret = dma_cookie_status(&c->vc.chan, cookie, state);
  302. if (ret == DMA_COMPLETE || !state)
  303. return ret;
  304. spin_lock_irqsave(&c->vc.lock, flags);
  305. p = c->phy;
  306. ret = c->status;
  307. /*
  308. * If the cookie is on our issue queue, then the residue is
  309. * its total size.
  310. */
  311. vd = vchan_find_desc(&c->vc, cookie);
  312. if (vd) {
  313. bytes = container_of(vd, struct zx_dma_desc_sw, vd)->size;
  314. } else if ((!p) || (!p->ds_run)) {
  315. bytes = 0;
  316. } else {
  317. struct zx_dma_desc_sw *ds = p->ds_run;
  318. u32 clli = 0, index = 0;
  319. bytes = 0;
  320. clli = zx_dma_get_curr_lli(p);
  321. index = (clli - ds->desc_hw_lli) /
  322. sizeof(struct zx_desc_hw) + 1;
  323. for (; index < ds->desc_num; index++) {
  324. bytes += ds->desc_hw[index].src_x;
  325. /* end of lli */
  326. if (!ds->desc_hw[index].lli)
  327. break;
  328. }
  329. }
  330. spin_unlock_irqrestore(&c->vc.lock, flags);
  331. dma_set_residue(state, bytes);
  332. return ret;
  333. }
  334. static void zx_dma_issue_pending(struct dma_chan *chan)
  335. {
  336. struct zx_dma_chan *c = to_zx_chan(chan);
  337. struct zx_dma_dev *d = to_zx_dma(chan->device);
  338. unsigned long flags;
  339. int issue = 0;
  340. spin_lock_irqsave(&c->vc.lock, flags);
  341. /* add request to vc->desc_issued */
  342. if (vchan_issue_pending(&c->vc)) {
  343. spin_lock(&d->lock);
  344. if (!c->phy && list_empty(&c->node)) {
  345. /* if new channel, add chan_pending */
  346. list_add_tail(&c->node, &d->chan_pending);
  347. issue = 1;
  348. dev_dbg(d->slave.dev, "vchan %p: issued\n", &c->vc);
  349. }
  350. spin_unlock(&d->lock);
  351. } else {
  352. dev_dbg(d->slave.dev, "vchan %p: nothing to issue\n", &c->vc);
  353. }
  354. spin_unlock_irqrestore(&c->vc.lock, flags);
  355. if (issue)
  356. zx_dma_task(d);
  357. }
  358. static void zx_dma_fill_desc(struct zx_dma_desc_sw *ds, dma_addr_t dst,
  359. dma_addr_t src, size_t len, u32 num, u32 ccfg)
  360. {
  361. if ((num + 1) < ds->desc_num)
  362. ds->desc_hw[num].lli = ds->desc_hw_lli + (num + 1) *
  363. sizeof(struct zx_desc_hw);
  364. ds->desc_hw[num].saddr = src;
  365. ds->desc_hw[num].daddr = dst;
  366. ds->desc_hw[num].src_x = len;
  367. ds->desc_hw[num].ctr = ccfg;
  368. }
  369. static struct zx_dma_desc_sw *zx_alloc_desc_resource(int num,
  370. struct dma_chan *chan)
  371. {
  372. struct zx_dma_chan *c = to_zx_chan(chan);
  373. struct zx_dma_desc_sw *ds;
  374. struct zx_dma_dev *d = to_zx_dma(chan->device);
  375. int lli_limit = LLI_BLOCK_SIZE / sizeof(struct zx_desc_hw);
  376. if (num > lli_limit) {
  377. dev_dbg(chan->device->dev, "vch %p: sg num %d exceed max %d\n",
  378. &c->vc, num, lli_limit);
  379. return NULL;
  380. }
  381. ds = kzalloc(sizeof(*ds), GFP_ATOMIC);
  382. if (!ds)
  383. return NULL;
  384. ds->desc_hw = dma_pool_zalloc(d->pool, GFP_NOWAIT, &ds->desc_hw_lli);
  385. if (!ds->desc_hw) {
  386. dev_dbg(chan->device->dev, "vch %p: dma alloc fail\n", &c->vc);
  387. kfree(ds);
  388. return NULL;
  389. }
  390. ds->desc_num = num;
  391. return ds;
  392. }
  393. static enum zx_dma_burst_width zx_dma_burst_width(enum dma_slave_buswidth width)
  394. {
  395. switch (width) {
  396. case DMA_SLAVE_BUSWIDTH_1_BYTE:
  397. case DMA_SLAVE_BUSWIDTH_2_BYTES:
  398. case DMA_SLAVE_BUSWIDTH_4_BYTES:
  399. case DMA_SLAVE_BUSWIDTH_8_BYTES:
  400. return ffs(width) - 1;
  401. default:
  402. return ZX_DMA_WIDTH_32BIT;
  403. }
  404. }
  405. static int zx_pre_config(struct zx_dma_chan *c, enum dma_transfer_direction dir)
  406. {
  407. struct dma_slave_config *cfg = &c->slave_cfg;
  408. enum zx_dma_burst_width src_width;
  409. enum zx_dma_burst_width dst_width;
  410. u32 maxburst = 0;
  411. switch (dir) {
  412. case DMA_MEM_TO_MEM:
  413. c->ccfg = ZX_CH_ENABLE | ZX_SOFT_REQ
  414. | ZX_SRC_BURST_LEN(ZX_MAX_BURST_LEN - 1)
  415. | ZX_SRC_BURST_WIDTH(ZX_DMA_WIDTH_32BIT)
  416. | ZX_DST_BURST_WIDTH(ZX_DMA_WIDTH_32BIT);
  417. break;
  418. case DMA_MEM_TO_DEV:
  419. c->dev_addr = cfg->dst_addr;
  420. /* dst len is calculated from src width, len and dst width.
  421. * We need make sure dst len not exceed MAX LEN.
  422. * Trailing single transaction that does not fill a full
  423. * burst also require identical src/dst data width.
  424. */
  425. dst_width = zx_dma_burst_width(cfg->dst_addr_width);
  426. maxburst = cfg->dst_maxburst;
  427. maxburst = maxburst < ZX_MAX_BURST_LEN ?
  428. maxburst : ZX_MAX_BURST_LEN;
  429. c->ccfg = ZX_DST_FIFO_MODE | ZX_CH_ENABLE
  430. | ZX_SRC_BURST_LEN(maxburst - 1)
  431. | ZX_SRC_BURST_WIDTH(dst_width)
  432. | ZX_DST_BURST_WIDTH(dst_width);
  433. break;
  434. case DMA_DEV_TO_MEM:
  435. c->dev_addr = cfg->src_addr;
  436. src_width = zx_dma_burst_width(cfg->src_addr_width);
  437. maxburst = cfg->src_maxburst;
  438. maxburst = maxburst < ZX_MAX_BURST_LEN ?
  439. maxburst : ZX_MAX_BURST_LEN;
  440. c->ccfg = ZX_SRC_FIFO_MODE | ZX_CH_ENABLE
  441. | ZX_SRC_BURST_LEN(maxburst - 1)
  442. | ZX_SRC_BURST_WIDTH(src_width)
  443. | ZX_DST_BURST_WIDTH(src_width);
  444. break;
  445. default:
  446. return -EINVAL;
  447. }
  448. return 0;
  449. }
  450. static struct dma_async_tx_descriptor *zx_dma_prep_memcpy(
  451. struct dma_chan *chan, dma_addr_t dst, dma_addr_t src,
  452. size_t len, unsigned long flags)
  453. {
  454. struct zx_dma_chan *c = to_zx_chan(chan);
  455. struct zx_dma_desc_sw *ds;
  456. size_t copy = 0;
  457. int num = 0;
  458. if (!len)
  459. return NULL;
  460. if (zx_pre_config(c, DMA_MEM_TO_MEM))
  461. return NULL;
  462. num = DIV_ROUND_UP(len, DMA_MAX_SIZE);
  463. ds = zx_alloc_desc_resource(num, chan);
  464. if (!ds)
  465. return NULL;
  466. ds->size = len;
  467. num = 0;
  468. do {
  469. copy = min_t(size_t, len, DMA_MAX_SIZE);
  470. zx_dma_fill_desc(ds, dst, src, copy, num++, c->ccfg);
  471. src += copy;
  472. dst += copy;
  473. len -= copy;
  474. } while (len);
  475. c->cyclic = 0;
  476. ds->desc_hw[num - 1].lli = 0; /* end of link */
  477. ds->desc_hw[num - 1].ctr |= ZX_IRQ_ENABLE_ALL;
  478. return vchan_tx_prep(&c->vc, &ds->vd, flags);
  479. }
  480. static struct dma_async_tx_descriptor *zx_dma_prep_slave_sg(
  481. struct dma_chan *chan, struct scatterlist *sgl, unsigned int sglen,
  482. enum dma_transfer_direction dir, unsigned long flags, void *context)
  483. {
  484. struct zx_dma_chan *c = to_zx_chan(chan);
  485. struct zx_dma_desc_sw *ds;
  486. size_t len, avail, total = 0;
  487. struct scatterlist *sg;
  488. dma_addr_t addr, src = 0, dst = 0;
  489. int num = sglen, i;
  490. if (!sgl)
  491. return NULL;
  492. if (zx_pre_config(c, dir))
  493. return NULL;
  494. for_each_sg(sgl, sg, sglen, i) {
  495. avail = sg_dma_len(sg);
  496. if (avail > DMA_MAX_SIZE)
  497. num += DIV_ROUND_UP(avail, DMA_MAX_SIZE) - 1;
  498. }
  499. ds = zx_alloc_desc_resource(num, chan);
  500. if (!ds)
  501. return NULL;
  502. c->cyclic = 0;
  503. num = 0;
  504. for_each_sg(sgl, sg, sglen, i) {
  505. addr = sg_dma_address(sg);
  506. avail = sg_dma_len(sg);
  507. total += avail;
  508. do {
  509. len = min_t(size_t, avail, DMA_MAX_SIZE);
  510. if (dir == DMA_MEM_TO_DEV) {
  511. src = addr;
  512. dst = c->dev_addr;
  513. } else if (dir == DMA_DEV_TO_MEM) {
  514. src = c->dev_addr;
  515. dst = addr;
  516. }
  517. zx_dma_fill_desc(ds, dst, src, len, num++, c->ccfg);
  518. addr += len;
  519. avail -= len;
  520. } while (avail);
  521. }
  522. ds->desc_hw[num - 1].lli = 0; /* end of link */
  523. ds->desc_hw[num - 1].ctr |= ZX_IRQ_ENABLE_ALL;
  524. ds->size = total;
  525. return vchan_tx_prep(&c->vc, &ds->vd, flags);
  526. }
  527. static struct dma_async_tx_descriptor *zx_dma_prep_dma_cyclic(
  528. struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len,
  529. size_t period_len, enum dma_transfer_direction dir,
  530. unsigned long flags)
  531. {
  532. struct zx_dma_chan *c = to_zx_chan(chan);
  533. struct zx_dma_desc_sw *ds;
  534. dma_addr_t src = 0, dst = 0;
  535. int num_periods = buf_len / period_len;
  536. int buf = 0, num = 0;
  537. if (period_len > DMA_MAX_SIZE) {
  538. dev_err(chan->device->dev, "maximum period size exceeded\n");
  539. return NULL;
  540. }
  541. if (zx_pre_config(c, dir))
  542. return NULL;
  543. ds = zx_alloc_desc_resource(num_periods, chan);
  544. if (!ds)
  545. return NULL;
  546. c->cyclic = 1;
  547. while (buf < buf_len) {
  548. if (dir == DMA_MEM_TO_DEV) {
  549. src = dma_addr;
  550. dst = c->dev_addr;
  551. } else if (dir == DMA_DEV_TO_MEM) {
  552. src = c->dev_addr;
  553. dst = dma_addr;
  554. }
  555. zx_dma_fill_desc(ds, dst, src, period_len, num++,
  556. c->ccfg | ZX_IRQ_ENABLE_ALL);
  557. dma_addr += period_len;
  558. buf += period_len;
  559. }
  560. ds->desc_hw[num - 1].lli = ds->desc_hw_lli;
  561. ds->size = buf_len;
  562. return vchan_tx_prep(&c->vc, &ds->vd, flags);
  563. }
  564. static int zx_dma_config(struct dma_chan *chan,
  565. struct dma_slave_config *cfg)
  566. {
  567. struct zx_dma_chan *c = to_zx_chan(chan);
  568. if (!cfg)
  569. return -EINVAL;
  570. memcpy(&c->slave_cfg, cfg, sizeof(*cfg));
  571. return 0;
  572. }
  573. static int zx_dma_terminate_all(struct dma_chan *chan)
  574. {
  575. struct zx_dma_chan *c = to_zx_chan(chan);
  576. struct zx_dma_dev *d = to_zx_dma(chan->device);
  577. struct zx_dma_phy *p = c->phy;
  578. unsigned long flags;
  579. LIST_HEAD(head);
  580. dev_dbg(d->slave.dev, "vchan %p: terminate all\n", &c->vc);
  581. /* Prevent this channel being scheduled */
  582. spin_lock(&d->lock);
  583. list_del_init(&c->node);
  584. spin_unlock(&d->lock);
  585. /* Clear the tx descriptor lists */
  586. spin_lock_irqsave(&c->vc.lock, flags);
  587. vchan_get_all_descriptors(&c->vc, &head);
  588. if (p) {
  589. /* vchan is assigned to a pchan - stop the channel */
  590. zx_dma_terminate_chan(p, d);
  591. c->phy = NULL;
  592. p->vchan = NULL;
  593. p->ds_run = NULL;
  594. p->ds_done = NULL;
  595. }
  596. spin_unlock_irqrestore(&c->vc.lock, flags);
  597. vchan_dma_desc_free_list(&c->vc, &head);
  598. return 0;
  599. }
  600. static int zx_dma_transfer_pause(struct dma_chan *chan)
  601. {
  602. struct zx_dma_chan *c = to_zx_chan(chan);
  603. u32 val = 0;
  604. val = readl_relaxed(c->phy->base + REG_ZX_CTRL);
  605. val &= ~ZX_CH_ENABLE;
  606. writel_relaxed(val, c->phy->base + REG_ZX_CTRL);
  607. return 0;
  608. }
  609. static int zx_dma_transfer_resume(struct dma_chan *chan)
  610. {
  611. struct zx_dma_chan *c = to_zx_chan(chan);
  612. u32 val = 0;
  613. val = readl_relaxed(c->phy->base + REG_ZX_CTRL);
  614. val |= ZX_CH_ENABLE;
  615. writel_relaxed(val, c->phy->base + REG_ZX_CTRL);
  616. return 0;
  617. }
  618. static void zx_dma_free_desc(struct virt_dma_desc *vd)
  619. {
  620. struct zx_dma_desc_sw *ds =
  621. container_of(vd, struct zx_dma_desc_sw, vd);
  622. struct zx_dma_dev *d = to_zx_dma(vd->tx.chan->device);
  623. dma_pool_free(d->pool, ds->desc_hw, ds->desc_hw_lli);
  624. kfree(ds);
  625. }
  626. static const struct of_device_id zx6702_dma_dt_ids[] = {
  627. { .compatible = "zte,zx296702-dma", },
  628. {}
  629. };
  630. MODULE_DEVICE_TABLE(of, zx6702_dma_dt_ids);
  631. static struct dma_chan *zx_of_dma_simple_xlate(struct of_phandle_args *dma_spec,
  632. struct of_dma *ofdma)
  633. {
  634. struct zx_dma_dev *d = ofdma->of_dma_data;
  635. unsigned int request = dma_spec->args[0];
  636. struct dma_chan *chan;
  637. struct zx_dma_chan *c;
  638. if (request >= d->dma_requests)
  639. return NULL;
  640. chan = dma_get_any_slave_channel(&d->slave);
  641. if (!chan) {
  642. dev_err(d->slave.dev, "get channel fail in %s.\n", __func__);
  643. return NULL;
  644. }
  645. c = to_zx_chan(chan);
  646. c->id = request;
  647. dev_info(d->slave.dev, "zx_dma: pchan %u: alloc vchan %p\n",
  648. c->id, &c->vc);
  649. return chan;
  650. }
  651. static int zx_dma_probe(struct platform_device *op)
  652. {
  653. struct zx_dma_dev *d;
  654. int i, ret = 0;
  655. d = devm_kzalloc(&op->dev, sizeof(*d), GFP_KERNEL);
  656. if (!d)
  657. return -ENOMEM;
  658. d->base = devm_platform_ioremap_resource(op, 0);
  659. if (IS_ERR(d->base))
  660. return PTR_ERR(d->base);
  661. of_property_read_u32((&op->dev)->of_node,
  662. "dma-channels", &d->dma_channels);
  663. of_property_read_u32((&op->dev)->of_node,
  664. "dma-requests", &d->dma_requests);
  665. if (!d->dma_requests || !d->dma_channels)
  666. return -EINVAL;
  667. d->clk = devm_clk_get(&op->dev, NULL);
  668. if (IS_ERR(d->clk)) {
  669. dev_err(&op->dev, "no dma clk\n");
  670. return PTR_ERR(d->clk);
  671. }
  672. d->irq = platform_get_irq(op, 0);
  673. ret = devm_request_irq(&op->dev, d->irq, zx_dma_int_handler,
  674. 0, DRIVER_NAME, d);
  675. if (ret)
  676. return ret;
  677. /* A DMA memory pool for LLIs, align on 32-byte boundary */
  678. d->pool = dmam_pool_create(DRIVER_NAME, &op->dev,
  679. LLI_BLOCK_SIZE, 32, 0);
  680. if (!d->pool)
  681. return -ENOMEM;
  682. /* init phy channel */
  683. d->phy = devm_kcalloc(&op->dev,
  684. d->dma_channels, sizeof(struct zx_dma_phy), GFP_KERNEL);
  685. if (!d->phy)
  686. return -ENOMEM;
  687. for (i = 0; i < d->dma_channels; i++) {
  688. struct zx_dma_phy *p = &d->phy[i];
  689. p->idx = i;
  690. p->base = d->base + i * 0x40;
  691. }
  692. INIT_LIST_HEAD(&d->slave.channels);
  693. dma_cap_set(DMA_SLAVE, d->slave.cap_mask);
  694. dma_cap_set(DMA_MEMCPY, d->slave.cap_mask);
  695. dma_cap_set(DMA_CYCLIC, d->slave.cap_mask);
  696. dma_cap_set(DMA_PRIVATE, d->slave.cap_mask);
  697. d->slave.dev = &op->dev;
  698. d->slave.device_free_chan_resources = zx_dma_free_chan_resources;
  699. d->slave.device_tx_status = zx_dma_tx_status;
  700. d->slave.device_prep_dma_memcpy = zx_dma_prep_memcpy;
  701. d->slave.device_prep_slave_sg = zx_dma_prep_slave_sg;
  702. d->slave.device_prep_dma_cyclic = zx_dma_prep_dma_cyclic;
  703. d->slave.device_issue_pending = zx_dma_issue_pending;
  704. d->slave.device_config = zx_dma_config;
  705. d->slave.device_terminate_all = zx_dma_terminate_all;
  706. d->slave.device_pause = zx_dma_transfer_pause;
  707. d->slave.device_resume = zx_dma_transfer_resume;
  708. d->slave.copy_align = DMA_ALIGN;
  709. d->slave.src_addr_widths = ZX_DMA_BUSWIDTHS;
  710. d->slave.dst_addr_widths = ZX_DMA_BUSWIDTHS;
  711. d->slave.directions = BIT(DMA_MEM_TO_MEM) | BIT(DMA_MEM_TO_DEV)
  712. | BIT(DMA_DEV_TO_MEM);
  713. d->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
  714. /* init virtual channel */
  715. d->chans = devm_kcalloc(&op->dev,
  716. d->dma_requests, sizeof(struct zx_dma_chan), GFP_KERNEL);
  717. if (!d->chans)
  718. return -ENOMEM;
  719. for (i = 0; i < d->dma_requests; i++) {
  720. struct zx_dma_chan *c = &d->chans[i];
  721. c->status = DMA_IN_PROGRESS;
  722. INIT_LIST_HEAD(&c->node);
  723. c->vc.desc_free = zx_dma_free_desc;
  724. vchan_init(&c->vc, &d->slave);
  725. }
  726. /* Enable clock before accessing registers */
  727. ret = clk_prepare_enable(d->clk);
  728. if (ret < 0) {
  729. dev_err(&op->dev, "clk_prepare_enable failed: %d\n", ret);
  730. goto zx_dma_out;
  731. }
  732. zx_dma_init_state(d);
  733. spin_lock_init(&d->lock);
  734. INIT_LIST_HEAD(&d->chan_pending);
  735. platform_set_drvdata(op, d);
  736. ret = dma_async_device_register(&d->slave);
  737. if (ret)
  738. goto clk_dis;
  739. ret = of_dma_controller_register((&op->dev)->of_node,
  740. zx_of_dma_simple_xlate, d);
  741. if (ret)
  742. goto of_dma_register_fail;
  743. dev_info(&op->dev, "initialized\n");
  744. return 0;
  745. of_dma_register_fail:
  746. dma_async_device_unregister(&d->slave);
  747. clk_dis:
  748. clk_disable_unprepare(d->clk);
  749. zx_dma_out:
  750. return ret;
  751. }
  752. static int zx_dma_remove(struct platform_device *op)
  753. {
  754. struct zx_dma_chan *c, *cn;
  755. struct zx_dma_dev *d = platform_get_drvdata(op);
  756. /* explictly free the irq */
  757. devm_free_irq(&op->dev, d->irq, d);
  758. dma_async_device_unregister(&d->slave);
  759. of_dma_controller_free((&op->dev)->of_node);
  760. list_for_each_entry_safe(c, cn, &d->slave.channels,
  761. vc.chan.device_node) {
  762. list_del(&c->vc.chan.device_node);
  763. }
  764. clk_disable_unprepare(d->clk);
  765. return 0;
  766. }
  767. #ifdef CONFIG_PM_SLEEP
  768. static int zx_dma_suspend_dev(struct device *dev)
  769. {
  770. struct zx_dma_dev *d = dev_get_drvdata(dev);
  771. u32 stat = 0;
  772. stat = zx_dma_get_chan_stat(d);
  773. if (stat) {
  774. dev_warn(d->slave.dev,
  775. "chan %d is running fail to suspend\n", stat);
  776. return -1;
  777. }
  778. clk_disable_unprepare(d->clk);
  779. return 0;
  780. }
  781. static int zx_dma_resume_dev(struct device *dev)
  782. {
  783. struct zx_dma_dev *d = dev_get_drvdata(dev);
  784. int ret = 0;
  785. ret = clk_prepare_enable(d->clk);
  786. if (ret < 0) {
  787. dev_err(d->slave.dev, "clk_prepare_enable failed: %d\n", ret);
  788. return ret;
  789. }
  790. zx_dma_init_state(d);
  791. return 0;
  792. }
  793. #endif
  794. static SIMPLE_DEV_PM_OPS(zx_dma_pmops, zx_dma_suspend_dev, zx_dma_resume_dev);
  795. static struct platform_driver zx_pdma_driver = {
  796. .driver = {
  797. .name = DRIVER_NAME,
  798. .pm = &zx_dma_pmops,
  799. .of_match_table = zx6702_dma_dt_ids,
  800. },
  801. .probe = zx_dma_probe,
  802. .remove = zx_dma_remove,
  803. };
  804. module_platform_driver(zx_pdma_driver);
  805. MODULE_DESCRIPTION("ZTE ZX296702 DMA Driver");
  806. MODULE_AUTHOR("Jun Nie jun.nie@linaro.org");
  807. MODULE_LICENSE("GPL v2");