img-mdc-dma.c 28 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * IMG Multi-threaded DMA Controller (MDC)
  4. *
  5. * Copyright (C) 2009,2012,2013 Imagination Technologies Ltd.
  6. * Copyright (C) 2014 Google, Inc.
  7. */
  8. #include <linux/clk.h>
  9. #include <linux/dma-mapping.h>
  10. #include <linux/dmaengine.h>
  11. #include <linux/dmapool.h>
  12. #include <linux/interrupt.h>
  13. #include <linux/io.h>
  14. #include <linux/irq.h>
  15. #include <linux/kernel.h>
  16. #include <linux/mfd/syscon.h>
  17. #include <linux/module.h>
  18. #include <linux/of.h>
  19. #include <linux/of_device.h>
  20. #include <linux/of_dma.h>
  21. #include <linux/platform_device.h>
  22. #include <linux/pm_runtime.h>
  23. #include <linux/regmap.h>
  24. #include <linux/slab.h>
  25. #include <linux/spinlock.h>
  26. #include "dmaengine.h"
  27. #include "virt-dma.h"
  28. #define MDC_MAX_DMA_CHANNELS 32
  29. #define MDC_GENERAL_CONFIG 0x000
  30. #define MDC_GENERAL_CONFIG_LIST_IEN BIT(31)
  31. #define MDC_GENERAL_CONFIG_IEN BIT(29)
  32. #define MDC_GENERAL_CONFIG_LEVEL_INT BIT(28)
  33. #define MDC_GENERAL_CONFIG_INC_W BIT(12)
  34. #define MDC_GENERAL_CONFIG_INC_R BIT(8)
  35. #define MDC_GENERAL_CONFIG_PHYSICAL_W BIT(7)
  36. #define MDC_GENERAL_CONFIG_WIDTH_W_SHIFT 4
  37. #define MDC_GENERAL_CONFIG_WIDTH_W_MASK 0x7
  38. #define MDC_GENERAL_CONFIG_PHYSICAL_R BIT(3)
  39. #define MDC_GENERAL_CONFIG_WIDTH_R_SHIFT 0
  40. #define MDC_GENERAL_CONFIG_WIDTH_R_MASK 0x7
  41. #define MDC_READ_PORT_CONFIG 0x004
  42. #define MDC_READ_PORT_CONFIG_STHREAD_SHIFT 28
  43. #define MDC_READ_PORT_CONFIG_STHREAD_MASK 0xf
  44. #define MDC_READ_PORT_CONFIG_RTHREAD_SHIFT 24
  45. #define MDC_READ_PORT_CONFIG_RTHREAD_MASK 0xf
  46. #define MDC_READ_PORT_CONFIG_WTHREAD_SHIFT 16
  47. #define MDC_READ_PORT_CONFIG_WTHREAD_MASK 0xf
  48. #define MDC_READ_PORT_CONFIG_BURST_SIZE_SHIFT 4
  49. #define MDC_READ_PORT_CONFIG_BURST_SIZE_MASK 0xff
  50. #define MDC_READ_PORT_CONFIG_DREQ_ENABLE BIT(1)
  51. #define MDC_READ_ADDRESS 0x008
  52. #define MDC_WRITE_ADDRESS 0x00c
  53. #define MDC_TRANSFER_SIZE 0x010
  54. #define MDC_TRANSFER_SIZE_MASK 0xffffff
  55. #define MDC_LIST_NODE_ADDRESS 0x014
  56. #define MDC_CMDS_PROCESSED 0x018
  57. #define MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT 16
  58. #define MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK 0x3f
  59. #define MDC_CMDS_PROCESSED_INT_ACTIVE BIT(8)
  60. #define MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT 0
  61. #define MDC_CMDS_PROCESSED_CMDS_DONE_MASK 0x3f
  62. #define MDC_CONTROL_AND_STATUS 0x01c
  63. #define MDC_CONTROL_AND_STATUS_CANCEL BIT(20)
  64. #define MDC_CONTROL_AND_STATUS_LIST_EN BIT(4)
  65. #define MDC_CONTROL_AND_STATUS_EN BIT(0)
  66. #define MDC_ACTIVE_TRANSFER_SIZE 0x030
  67. #define MDC_GLOBAL_CONFIG_A 0x900
  68. #define MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_SHIFT 16
  69. #define MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_MASK 0xff
  70. #define MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_SHIFT 8
  71. #define MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_MASK 0xff
  72. #define MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_SHIFT 0
  73. #define MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_MASK 0xff
  74. struct mdc_hw_list_desc {
  75. u32 gen_conf;
  76. u32 readport_conf;
  77. u32 read_addr;
  78. u32 write_addr;
  79. u32 xfer_size;
  80. u32 node_addr;
  81. u32 cmds_done;
  82. u32 ctrl_status;
  83. /*
  84. * Not part of the list descriptor, but instead used by the CPU to
  85. * traverse the list.
  86. */
  87. struct mdc_hw_list_desc *next_desc;
  88. };
  89. struct mdc_tx_desc {
  90. struct mdc_chan *chan;
  91. struct virt_dma_desc vd;
  92. dma_addr_t list_phys;
  93. struct mdc_hw_list_desc *list;
  94. bool cyclic;
  95. bool cmd_loaded;
  96. unsigned int list_len;
  97. unsigned int list_period_len;
  98. size_t list_xfer_size;
  99. unsigned int list_cmds_done;
  100. };
  101. struct mdc_chan {
  102. struct mdc_dma *mdma;
  103. struct virt_dma_chan vc;
  104. struct dma_slave_config config;
  105. struct mdc_tx_desc *desc;
  106. int irq;
  107. unsigned int periph;
  108. unsigned int thread;
  109. unsigned int chan_nr;
  110. };
  111. struct mdc_dma_soc_data {
  112. void (*enable_chan)(struct mdc_chan *mchan);
  113. void (*disable_chan)(struct mdc_chan *mchan);
  114. };
  115. struct mdc_dma {
  116. struct dma_device dma_dev;
  117. void __iomem *regs;
  118. struct clk *clk;
  119. struct dma_pool *desc_pool;
  120. struct regmap *periph_regs;
  121. spinlock_t lock;
  122. unsigned int nr_threads;
  123. unsigned int nr_channels;
  124. unsigned int bus_width;
  125. unsigned int max_burst_mult;
  126. unsigned int max_xfer_size;
  127. const struct mdc_dma_soc_data *soc;
  128. struct mdc_chan channels[MDC_MAX_DMA_CHANNELS];
  129. };
  130. static inline u32 mdc_readl(struct mdc_dma *mdma, u32 reg)
  131. {
  132. return readl(mdma->regs + reg);
  133. }
  134. static inline void mdc_writel(struct mdc_dma *mdma, u32 val, u32 reg)
  135. {
  136. writel(val, mdma->regs + reg);
  137. }
  138. static inline u32 mdc_chan_readl(struct mdc_chan *mchan, u32 reg)
  139. {
  140. return mdc_readl(mchan->mdma, mchan->chan_nr * 0x040 + reg);
  141. }
  142. static inline void mdc_chan_writel(struct mdc_chan *mchan, u32 val, u32 reg)
  143. {
  144. mdc_writel(mchan->mdma, val, mchan->chan_nr * 0x040 + reg);
  145. }
  146. static inline struct mdc_chan *to_mdc_chan(struct dma_chan *c)
  147. {
  148. return container_of(to_virt_chan(c), struct mdc_chan, vc);
  149. }
  150. static inline struct mdc_tx_desc *to_mdc_desc(struct dma_async_tx_descriptor *t)
  151. {
  152. struct virt_dma_desc *vdesc = container_of(t, struct virt_dma_desc, tx);
  153. return container_of(vdesc, struct mdc_tx_desc, vd);
  154. }
  155. static inline struct device *mdma2dev(struct mdc_dma *mdma)
  156. {
  157. return mdma->dma_dev.dev;
  158. }
  159. static inline unsigned int to_mdc_width(unsigned int bytes)
  160. {
  161. return ffs(bytes) - 1;
  162. }
  163. static inline void mdc_set_read_width(struct mdc_hw_list_desc *ldesc,
  164. unsigned int bytes)
  165. {
  166. ldesc->gen_conf |= to_mdc_width(bytes) <<
  167. MDC_GENERAL_CONFIG_WIDTH_R_SHIFT;
  168. }
  169. static inline void mdc_set_write_width(struct mdc_hw_list_desc *ldesc,
  170. unsigned int bytes)
  171. {
  172. ldesc->gen_conf |= to_mdc_width(bytes) <<
  173. MDC_GENERAL_CONFIG_WIDTH_W_SHIFT;
  174. }
  175. static void mdc_list_desc_config(struct mdc_chan *mchan,
  176. struct mdc_hw_list_desc *ldesc,
  177. enum dma_transfer_direction dir,
  178. dma_addr_t src, dma_addr_t dst, size_t len)
  179. {
  180. struct mdc_dma *mdma = mchan->mdma;
  181. unsigned int max_burst, burst_size;
  182. ldesc->gen_conf = MDC_GENERAL_CONFIG_IEN | MDC_GENERAL_CONFIG_LIST_IEN |
  183. MDC_GENERAL_CONFIG_LEVEL_INT | MDC_GENERAL_CONFIG_PHYSICAL_W |
  184. MDC_GENERAL_CONFIG_PHYSICAL_R;
  185. ldesc->readport_conf =
  186. (mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) |
  187. (mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) |
  188. (mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT);
  189. ldesc->read_addr = src;
  190. ldesc->write_addr = dst;
  191. ldesc->xfer_size = len - 1;
  192. ldesc->node_addr = 0;
  193. ldesc->cmds_done = 0;
  194. ldesc->ctrl_status = MDC_CONTROL_AND_STATUS_LIST_EN |
  195. MDC_CONTROL_AND_STATUS_EN;
  196. ldesc->next_desc = NULL;
  197. if (IS_ALIGNED(dst, mdma->bus_width) &&
  198. IS_ALIGNED(src, mdma->bus_width))
  199. max_burst = mdma->bus_width * mdma->max_burst_mult;
  200. else
  201. max_burst = mdma->bus_width * (mdma->max_burst_mult - 1);
  202. if (dir == DMA_MEM_TO_DEV) {
  203. ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_R;
  204. ldesc->readport_conf |= MDC_READ_PORT_CONFIG_DREQ_ENABLE;
  205. mdc_set_read_width(ldesc, mdma->bus_width);
  206. mdc_set_write_width(ldesc, mchan->config.dst_addr_width);
  207. burst_size = min(max_burst, mchan->config.dst_maxburst *
  208. mchan->config.dst_addr_width);
  209. } else if (dir == DMA_DEV_TO_MEM) {
  210. ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_W;
  211. ldesc->readport_conf |= MDC_READ_PORT_CONFIG_DREQ_ENABLE;
  212. mdc_set_read_width(ldesc, mchan->config.src_addr_width);
  213. mdc_set_write_width(ldesc, mdma->bus_width);
  214. burst_size = min(max_burst, mchan->config.src_maxburst *
  215. mchan->config.src_addr_width);
  216. } else {
  217. ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_R |
  218. MDC_GENERAL_CONFIG_INC_W;
  219. mdc_set_read_width(ldesc, mdma->bus_width);
  220. mdc_set_write_width(ldesc, mdma->bus_width);
  221. burst_size = max_burst;
  222. }
  223. ldesc->readport_conf |= (burst_size - 1) <<
  224. MDC_READ_PORT_CONFIG_BURST_SIZE_SHIFT;
  225. }
  226. static void mdc_list_desc_free(struct mdc_tx_desc *mdesc)
  227. {
  228. struct mdc_dma *mdma = mdesc->chan->mdma;
  229. struct mdc_hw_list_desc *curr, *next;
  230. dma_addr_t curr_phys, next_phys;
  231. curr = mdesc->list;
  232. curr_phys = mdesc->list_phys;
  233. while (curr) {
  234. next = curr->next_desc;
  235. next_phys = curr->node_addr;
  236. dma_pool_free(mdma->desc_pool, curr, curr_phys);
  237. curr = next;
  238. curr_phys = next_phys;
  239. }
  240. }
  241. static void mdc_desc_free(struct virt_dma_desc *vd)
  242. {
  243. struct mdc_tx_desc *mdesc = to_mdc_desc(&vd->tx);
  244. mdc_list_desc_free(mdesc);
  245. kfree(mdesc);
  246. }
  247. static struct dma_async_tx_descriptor *mdc_prep_dma_memcpy(
  248. struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, size_t len,
  249. unsigned long flags)
  250. {
  251. struct mdc_chan *mchan = to_mdc_chan(chan);
  252. struct mdc_dma *mdma = mchan->mdma;
  253. struct mdc_tx_desc *mdesc;
  254. struct mdc_hw_list_desc *curr, *prev = NULL;
  255. dma_addr_t curr_phys;
  256. if (!len)
  257. return NULL;
  258. mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
  259. if (!mdesc)
  260. return NULL;
  261. mdesc->chan = mchan;
  262. mdesc->list_xfer_size = len;
  263. while (len > 0) {
  264. size_t xfer_size;
  265. curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT, &curr_phys);
  266. if (!curr)
  267. goto free_desc;
  268. if (prev) {
  269. prev->node_addr = curr_phys;
  270. prev->next_desc = curr;
  271. } else {
  272. mdesc->list_phys = curr_phys;
  273. mdesc->list = curr;
  274. }
  275. xfer_size = min_t(size_t, mdma->max_xfer_size, len);
  276. mdc_list_desc_config(mchan, curr, DMA_MEM_TO_MEM, src, dest,
  277. xfer_size);
  278. prev = curr;
  279. mdesc->list_len++;
  280. src += xfer_size;
  281. dest += xfer_size;
  282. len -= xfer_size;
  283. }
  284. return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);
  285. free_desc:
  286. mdc_desc_free(&mdesc->vd);
  287. return NULL;
  288. }
  289. static int mdc_check_slave_width(struct mdc_chan *mchan,
  290. enum dma_transfer_direction dir)
  291. {
  292. enum dma_slave_buswidth width;
  293. if (dir == DMA_MEM_TO_DEV)
  294. width = mchan->config.dst_addr_width;
  295. else
  296. width = mchan->config.src_addr_width;
  297. switch (width) {
  298. case DMA_SLAVE_BUSWIDTH_1_BYTE:
  299. case DMA_SLAVE_BUSWIDTH_2_BYTES:
  300. case DMA_SLAVE_BUSWIDTH_4_BYTES:
  301. case DMA_SLAVE_BUSWIDTH_8_BYTES:
  302. break;
  303. default:
  304. return -EINVAL;
  305. }
  306. if (width > mchan->mdma->bus_width)
  307. return -EINVAL;
  308. return 0;
  309. }
  310. static struct dma_async_tx_descriptor *mdc_prep_dma_cyclic(
  311. struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
  312. size_t period_len, enum dma_transfer_direction dir,
  313. unsigned long flags)
  314. {
  315. struct mdc_chan *mchan = to_mdc_chan(chan);
  316. struct mdc_dma *mdma = mchan->mdma;
  317. struct mdc_tx_desc *mdesc;
  318. struct mdc_hw_list_desc *curr, *prev = NULL;
  319. dma_addr_t curr_phys;
  320. if (!buf_len && !period_len)
  321. return NULL;
  322. if (!is_slave_direction(dir))
  323. return NULL;
  324. if (mdc_check_slave_width(mchan, dir) < 0)
  325. return NULL;
  326. mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
  327. if (!mdesc)
  328. return NULL;
  329. mdesc->chan = mchan;
  330. mdesc->cyclic = true;
  331. mdesc->list_xfer_size = buf_len;
  332. mdesc->list_period_len = DIV_ROUND_UP(period_len,
  333. mdma->max_xfer_size);
  334. while (buf_len > 0) {
  335. size_t remainder = min(period_len, buf_len);
  336. while (remainder > 0) {
  337. size_t xfer_size;
  338. curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT,
  339. &curr_phys);
  340. if (!curr)
  341. goto free_desc;
  342. if (!prev) {
  343. mdesc->list_phys = curr_phys;
  344. mdesc->list = curr;
  345. } else {
  346. prev->node_addr = curr_phys;
  347. prev->next_desc = curr;
  348. }
  349. xfer_size = min_t(size_t, mdma->max_xfer_size,
  350. remainder);
  351. if (dir == DMA_MEM_TO_DEV) {
  352. mdc_list_desc_config(mchan, curr, dir,
  353. buf_addr,
  354. mchan->config.dst_addr,
  355. xfer_size);
  356. } else {
  357. mdc_list_desc_config(mchan, curr, dir,
  358. mchan->config.src_addr,
  359. buf_addr,
  360. xfer_size);
  361. }
  362. prev = curr;
  363. mdesc->list_len++;
  364. buf_addr += xfer_size;
  365. buf_len -= xfer_size;
  366. remainder -= xfer_size;
  367. }
  368. }
  369. prev->node_addr = mdesc->list_phys;
  370. return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);
  371. free_desc:
  372. mdc_desc_free(&mdesc->vd);
  373. return NULL;
  374. }
  375. static struct dma_async_tx_descriptor *mdc_prep_slave_sg(
  376. struct dma_chan *chan, struct scatterlist *sgl,
  377. unsigned int sg_len, enum dma_transfer_direction dir,
  378. unsigned long flags, void *context)
  379. {
  380. struct mdc_chan *mchan = to_mdc_chan(chan);
  381. struct mdc_dma *mdma = mchan->mdma;
  382. struct mdc_tx_desc *mdesc;
  383. struct scatterlist *sg;
  384. struct mdc_hw_list_desc *curr, *prev = NULL;
  385. dma_addr_t curr_phys;
  386. unsigned int i;
  387. if (!sgl)
  388. return NULL;
  389. if (!is_slave_direction(dir))
  390. return NULL;
  391. if (mdc_check_slave_width(mchan, dir) < 0)
  392. return NULL;
  393. mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
  394. if (!mdesc)
  395. return NULL;
  396. mdesc->chan = mchan;
  397. for_each_sg(sgl, sg, sg_len, i) {
  398. dma_addr_t buf = sg_dma_address(sg);
  399. size_t buf_len = sg_dma_len(sg);
  400. while (buf_len > 0) {
  401. size_t xfer_size;
  402. curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT,
  403. &curr_phys);
  404. if (!curr)
  405. goto free_desc;
  406. if (!prev) {
  407. mdesc->list_phys = curr_phys;
  408. mdesc->list = curr;
  409. } else {
  410. prev->node_addr = curr_phys;
  411. prev->next_desc = curr;
  412. }
  413. xfer_size = min_t(size_t, mdma->max_xfer_size,
  414. buf_len);
  415. if (dir == DMA_MEM_TO_DEV) {
  416. mdc_list_desc_config(mchan, curr, dir, buf,
  417. mchan->config.dst_addr,
  418. xfer_size);
  419. } else {
  420. mdc_list_desc_config(mchan, curr, dir,
  421. mchan->config.src_addr,
  422. buf, xfer_size);
  423. }
  424. prev = curr;
  425. mdesc->list_len++;
  426. mdesc->list_xfer_size += xfer_size;
  427. buf += xfer_size;
  428. buf_len -= xfer_size;
  429. }
  430. }
  431. return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);
  432. free_desc:
  433. mdc_desc_free(&mdesc->vd);
  434. return NULL;
  435. }
  436. static void mdc_issue_desc(struct mdc_chan *mchan)
  437. {
  438. struct mdc_dma *mdma = mchan->mdma;
  439. struct virt_dma_desc *vd;
  440. struct mdc_tx_desc *mdesc;
  441. u32 val;
  442. vd = vchan_next_desc(&mchan->vc);
  443. if (!vd)
  444. return;
  445. list_del(&vd->node);
  446. mdesc = to_mdc_desc(&vd->tx);
  447. mchan->desc = mdesc;
  448. dev_dbg(mdma2dev(mdma), "Issuing descriptor on channel %d\n",
  449. mchan->chan_nr);
  450. mdma->soc->enable_chan(mchan);
  451. val = mdc_chan_readl(mchan, MDC_GENERAL_CONFIG);
  452. val |= MDC_GENERAL_CONFIG_LIST_IEN | MDC_GENERAL_CONFIG_IEN |
  453. MDC_GENERAL_CONFIG_LEVEL_INT | MDC_GENERAL_CONFIG_PHYSICAL_W |
  454. MDC_GENERAL_CONFIG_PHYSICAL_R;
  455. mdc_chan_writel(mchan, val, MDC_GENERAL_CONFIG);
  456. val = (mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) |
  457. (mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) |
  458. (mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT);
  459. mdc_chan_writel(mchan, val, MDC_READ_PORT_CONFIG);
  460. mdc_chan_writel(mchan, mdesc->list_phys, MDC_LIST_NODE_ADDRESS);
  461. val = mdc_chan_readl(mchan, MDC_CONTROL_AND_STATUS);
  462. val |= MDC_CONTROL_AND_STATUS_LIST_EN;
  463. mdc_chan_writel(mchan, val, MDC_CONTROL_AND_STATUS);
  464. }
  465. static void mdc_issue_pending(struct dma_chan *chan)
  466. {
  467. struct mdc_chan *mchan = to_mdc_chan(chan);
  468. unsigned long flags;
  469. spin_lock_irqsave(&mchan->vc.lock, flags);
  470. if (vchan_issue_pending(&mchan->vc) && !mchan->desc)
  471. mdc_issue_desc(mchan);
  472. spin_unlock_irqrestore(&mchan->vc.lock, flags);
  473. }
  474. static enum dma_status mdc_tx_status(struct dma_chan *chan,
  475. dma_cookie_t cookie, struct dma_tx_state *txstate)
  476. {
  477. struct mdc_chan *mchan = to_mdc_chan(chan);
  478. struct mdc_tx_desc *mdesc;
  479. struct virt_dma_desc *vd;
  480. unsigned long flags;
  481. size_t bytes = 0;
  482. int ret;
  483. ret = dma_cookie_status(chan, cookie, txstate);
  484. if (ret == DMA_COMPLETE)
  485. return ret;
  486. if (!txstate)
  487. return ret;
  488. spin_lock_irqsave(&mchan->vc.lock, flags);
  489. vd = vchan_find_desc(&mchan->vc, cookie);
  490. if (vd) {
  491. mdesc = to_mdc_desc(&vd->tx);
  492. bytes = mdesc->list_xfer_size;
  493. } else if (mchan->desc && mchan->desc->vd.tx.cookie == cookie) {
  494. struct mdc_hw_list_desc *ldesc;
  495. u32 val1, val2, done, processed, residue;
  496. int i, cmds;
  497. mdesc = mchan->desc;
  498. /*
  499. * Determine the number of commands that haven't been
  500. * processed (handled by the IRQ handler) yet.
  501. */
  502. do {
  503. val1 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) &
  504. ~MDC_CMDS_PROCESSED_INT_ACTIVE;
  505. residue = mdc_chan_readl(mchan,
  506. MDC_ACTIVE_TRANSFER_SIZE);
  507. val2 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) &
  508. ~MDC_CMDS_PROCESSED_INT_ACTIVE;
  509. } while (val1 != val2);
  510. done = (val1 >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &
  511. MDC_CMDS_PROCESSED_CMDS_DONE_MASK;
  512. processed = (val1 >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) &
  513. MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK;
  514. cmds = (done - processed) %
  515. (MDC_CMDS_PROCESSED_CMDS_DONE_MASK + 1);
  516. /*
  517. * If the command loaded event hasn't been processed yet, then
  518. * the difference above includes an extra command.
  519. */
  520. if (!mdesc->cmd_loaded)
  521. cmds--;
  522. else
  523. cmds += mdesc->list_cmds_done;
  524. bytes = mdesc->list_xfer_size;
  525. ldesc = mdesc->list;
  526. for (i = 0; i < cmds; i++) {
  527. bytes -= ldesc->xfer_size + 1;
  528. ldesc = ldesc->next_desc;
  529. }
  530. if (ldesc) {
  531. if (residue != MDC_TRANSFER_SIZE_MASK)
  532. bytes -= ldesc->xfer_size - residue;
  533. else
  534. bytes -= ldesc->xfer_size + 1;
  535. }
  536. }
  537. spin_unlock_irqrestore(&mchan->vc.lock, flags);
  538. dma_set_residue(txstate, bytes);
  539. return ret;
  540. }
  541. static unsigned int mdc_get_new_events(struct mdc_chan *mchan)
  542. {
  543. u32 val, processed, done1, done2;
  544. unsigned int ret;
  545. val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);
  546. processed = (val >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) &
  547. MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK;
  548. /*
  549. * CMDS_DONE may have incremented between reading CMDS_PROCESSED
  550. * and clearing INT_ACTIVE. Re-read CMDS_PROCESSED to ensure we
  551. * didn't miss a command completion.
  552. */
  553. do {
  554. val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);
  555. done1 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &
  556. MDC_CMDS_PROCESSED_CMDS_DONE_MASK;
  557. val &= ~((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK <<
  558. MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) |
  559. MDC_CMDS_PROCESSED_INT_ACTIVE);
  560. val |= done1 << MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT;
  561. mdc_chan_writel(mchan, val, MDC_CMDS_PROCESSED);
  562. val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);
  563. done2 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &
  564. MDC_CMDS_PROCESSED_CMDS_DONE_MASK;
  565. } while (done1 != done2);
  566. if (done1 >= processed)
  567. ret = done1 - processed;
  568. else
  569. ret = ((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK + 1) -
  570. processed) + done1;
  571. return ret;
  572. }
  573. static int mdc_terminate_all(struct dma_chan *chan)
  574. {
  575. struct mdc_chan *mchan = to_mdc_chan(chan);
  576. unsigned long flags;
  577. LIST_HEAD(head);
  578. spin_lock_irqsave(&mchan->vc.lock, flags);
  579. mdc_chan_writel(mchan, MDC_CONTROL_AND_STATUS_CANCEL,
  580. MDC_CONTROL_AND_STATUS);
  581. if (mchan->desc) {
  582. vchan_terminate_vdesc(&mchan->desc->vd);
  583. mchan->desc = NULL;
  584. }
  585. vchan_get_all_descriptors(&mchan->vc, &head);
  586. mdc_get_new_events(mchan);
  587. spin_unlock_irqrestore(&mchan->vc.lock, flags);
  588. vchan_dma_desc_free_list(&mchan->vc, &head);
  589. return 0;
  590. }
  591. static void mdc_synchronize(struct dma_chan *chan)
  592. {
  593. struct mdc_chan *mchan = to_mdc_chan(chan);
  594. vchan_synchronize(&mchan->vc);
  595. }
  596. static int mdc_slave_config(struct dma_chan *chan,
  597. struct dma_slave_config *config)
  598. {
  599. struct mdc_chan *mchan = to_mdc_chan(chan);
  600. unsigned long flags;
  601. spin_lock_irqsave(&mchan->vc.lock, flags);
  602. mchan->config = *config;
  603. spin_unlock_irqrestore(&mchan->vc.lock, flags);
  604. return 0;
  605. }
  606. static int mdc_alloc_chan_resources(struct dma_chan *chan)
  607. {
  608. struct mdc_chan *mchan = to_mdc_chan(chan);
  609. struct device *dev = mdma2dev(mchan->mdma);
  610. return pm_runtime_get_sync(dev);
  611. }
  612. static void mdc_free_chan_resources(struct dma_chan *chan)
  613. {
  614. struct mdc_chan *mchan = to_mdc_chan(chan);
  615. struct mdc_dma *mdma = mchan->mdma;
  616. struct device *dev = mdma2dev(mdma);
  617. mdc_terminate_all(chan);
  618. mdma->soc->disable_chan(mchan);
  619. pm_runtime_put(dev);
  620. }
  621. static irqreturn_t mdc_chan_irq(int irq, void *dev_id)
  622. {
  623. struct mdc_chan *mchan = (struct mdc_chan *)dev_id;
  624. struct mdc_tx_desc *mdesc;
  625. unsigned int i, new_events;
  626. spin_lock(&mchan->vc.lock);
  627. dev_dbg(mdma2dev(mchan->mdma), "IRQ on channel %d\n", mchan->chan_nr);
  628. new_events = mdc_get_new_events(mchan);
  629. if (!new_events)
  630. goto out;
  631. mdesc = mchan->desc;
  632. if (!mdesc) {
  633. dev_warn(mdma2dev(mchan->mdma),
  634. "IRQ with no active descriptor on channel %d\n",
  635. mchan->chan_nr);
  636. goto out;
  637. }
  638. for (i = 0; i < new_events; i++) {
  639. /*
  640. * The first interrupt in a transfer indicates that the
  641. * command list has been loaded, not that a command has
  642. * been completed.
  643. */
  644. if (!mdesc->cmd_loaded) {
  645. mdesc->cmd_loaded = true;
  646. continue;
  647. }
  648. mdesc->list_cmds_done++;
  649. if (mdesc->cyclic) {
  650. mdesc->list_cmds_done %= mdesc->list_len;
  651. if (mdesc->list_cmds_done % mdesc->list_period_len == 0)
  652. vchan_cyclic_callback(&mdesc->vd);
  653. } else if (mdesc->list_cmds_done == mdesc->list_len) {
  654. mchan->desc = NULL;
  655. vchan_cookie_complete(&mdesc->vd);
  656. mdc_issue_desc(mchan);
  657. break;
  658. }
  659. }
  660. out:
  661. spin_unlock(&mchan->vc.lock);
  662. return IRQ_HANDLED;
  663. }
  664. static struct dma_chan *mdc_of_xlate(struct of_phandle_args *dma_spec,
  665. struct of_dma *ofdma)
  666. {
  667. struct mdc_dma *mdma = ofdma->of_dma_data;
  668. struct dma_chan *chan;
  669. if (dma_spec->args_count != 3)
  670. return NULL;
  671. list_for_each_entry(chan, &mdma->dma_dev.channels, device_node) {
  672. struct mdc_chan *mchan = to_mdc_chan(chan);
  673. if (!(dma_spec->args[1] & BIT(mchan->chan_nr)))
  674. continue;
  675. if (dma_get_slave_channel(chan)) {
  676. mchan->periph = dma_spec->args[0];
  677. mchan->thread = dma_spec->args[2];
  678. return chan;
  679. }
  680. }
  681. return NULL;
  682. }
  683. #define PISTACHIO_CR_PERIPH_DMA_ROUTE(ch) (0x120 + 0x4 * ((ch) / 4))
  684. #define PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(ch) (8 * ((ch) % 4))
  685. #define PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK 0x3f
  686. static void pistachio_mdc_enable_chan(struct mdc_chan *mchan)
  687. {
  688. struct mdc_dma *mdma = mchan->mdma;
  689. regmap_update_bits(mdma->periph_regs,
  690. PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr),
  691. PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK <<
  692. PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr),
  693. mchan->periph <<
  694. PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr));
  695. }
  696. static void pistachio_mdc_disable_chan(struct mdc_chan *mchan)
  697. {
  698. struct mdc_dma *mdma = mchan->mdma;
  699. regmap_update_bits(mdma->periph_regs,
  700. PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr),
  701. PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK <<
  702. PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr),
  703. 0);
  704. }
  705. static const struct mdc_dma_soc_data pistachio_mdc_data = {
  706. .enable_chan = pistachio_mdc_enable_chan,
  707. .disable_chan = pistachio_mdc_disable_chan,
  708. };
  709. static const struct of_device_id mdc_dma_of_match[] = {
  710. { .compatible = "img,pistachio-mdc-dma", .data = &pistachio_mdc_data, },
  711. { },
  712. };
  713. MODULE_DEVICE_TABLE(of, mdc_dma_of_match);
  714. static int img_mdc_runtime_suspend(struct device *dev)
  715. {
  716. struct mdc_dma *mdma = dev_get_drvdata(dev);
  717. clk_disable_unprepare(mdma->clk);
  718. return 0;
  719. }
  720. static int img_mdc_runtime_resume(struct device *dev)
  721. {
  722. struct mdc_dma *mdma = dev_get_drvdata(dev);
  723. return clk_prepare_enable(mdma->clk);
  724. }
  725. static int mdc_dma_probe(struct platform_device *pdev)
  726. {
  727. struct mdc_dma *mdma;
  728. struct resource *res;
  729. unsigned int i;
  730. u32 val;
  731. int ret;
  732. mdma = devm_kzalloc(&pdev->dev, sizeof(*mdma), GFP_KERNEL);
  733. if (!mdma)
  734. return -ENOMEM;
  735. platform_set_drvdata(pdev, mdma);
  736. mdma->soc = of_device_get_match_data(&pdev->dev);
  737. res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
  738. mdma->regs = devm_ioremap_resource(&pdev->dev, res);
  739. if (IS_ERR(mdma->regs))
  740. return PTR_ERR(mdma->regs);
  741. mdma->periph_regs = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
  742. "img,cr-periph");
  743. if (IS_ERR(mdma->periph_regs))
  744. return PTR_ERR(mdma->periph_regs);
  745. mdma->clk = devm_clk_get(&pdev->dev, "sys");
  746. if (IS_ERR(mdma->clk))
  747. return PTR_ERR(mdma->clk);
  748. dma_cap_zero(mdma->dma_dev.cap_mask);
  749. dma_cap_set(DMA_SLAVE, mdma->dma_dev.cap_mask);
  750. dma_cap_set(DMA_PRIVATE, mdma->dma_dev.cap_mask);
  751. dma_cap_set(DMA_CYCLIC, mdma->dma_dev.cap_mask);
  752. dma_cap_set(DMA_MEMCPY, mdma->dma_dev.cap_mask);
  753. val = mdc_readl(mdma, MDC_GLOBAL_CONFIG_A);
  754. mdma->nr_channels = (val >> MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_SHIFT) &
  755. MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_MASK;
  756. mdma->nr_threads =
  757. 1 << ((val >> MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_SHIFT) &
  758. MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_MASK);
  759. mdma->bus_width =
  760. (1 << ((val >> MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_SHIFT) &
  761. MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_MASK)) / 8;
  762. /*
  763. * Although transfer sizes of up to MDC_TRANSFER_SIZE_MASK + 1 bytes
  764. * are supported, this makes it possible for the value reported in
  765. * MDC_ACTIVE_TRANSFER_SIZE to be ambiguous - an active transfer size
  766. * of MDC_TRANSFER_SIZE_MASK may indicate either that 0 bytes or
  767. * MDC_TRANSFER_SIZE_MASK + 1 bytes are remaining. To eliminate this
  768. * ambiguity, restrict transfer sizes to one bus-width less than the
  769. * actual maximum.
  770. */
  771. mdma->max_xfer_size = MDC_TRANSFER_SIZE_MASK + 1 - mdma->bus_width;
  772. of_property_read_u32(pdev->dev.of_node, "dma-channels",
  773. &mdma->nr_channels);
  774. ret = of_property_read_u32(pdev->dev.of_node,
  775. "img,max-burst-multiplier",
  776. &mdma->max_burst_mult);
  777. if (ret)
  778. return ret;
  779. mdma->dma_dev.dev = &pdev->dev;
  780. mdma->dma_dev.device_prep_slave_sg = mdc_prep_slave_sg;
  781. mdma->dma_dev.device_prep_dma_cyclic = mdc_prep_dma_cyclic;
  782. mdma->dma_dev.device_prep_dma_memcpy = mdc_prep_dma_memcpy;
  783. mdma->dma_dev.device_alloc_chan_resources = mdc_alloc_chan_resources;
  784. mdma->dma_dev.device_free_chan_resources = mdc_free_chan_resources;
  785. mdma->dma_dev.device_tx_status = mdc_tx_status;
  786. mdma->dma_dev.device_issue_pending = mdc_issue_pending;
  787. mdma->dma_dev.device_terminate_all = mdc_terminate_all;
  788. mdma->dma_dev.device_synchronize = mdc_synchronize;
  789. mdma->dma_dev.device_config = mdc_slave_config;
  790. mdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
  791. mdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
  792. for (i = 1; i <= mdma->bus_width; i <<= 1) {
  793. mdma->dma_dev.src_addr_widths |= BIT(i);
  794. mdma->dma_dev.dst_addr_widths |= BIT(i);
  795. }
  796. INIT_LIST_HEAD(&mdma->dma_dev.channels);
  797. for (i = 0; i < mdma->nr_channels; i++) {
  798. struct mdc_chan *mchan = &mdma->channels[i];
  799. mchan->mdma = mdma;
  800. mchan->chan_nr = i;
  801. mchan->irq = platform_get_irq(pdev, i);
  802. if (mchan->irq < 0)
  803. return mchan->irq;
  804. ret = devm_request_irq(&pdev->dev, mchan->irq, mdc_chan_irq,
  805. IRQ_TYPE_LEVEL_HIGH,
  806. dev_name(&pdev->dev), mchan);
  807. if (ret < 0)
  808. return ret;
  809. mchan->vc.desc_free = mdc_desc_free;
  810. vchan_init(&mchan->vc, &mdma->dma_dev);
  811. }
  812. mdma->desc_pool = dmam_pool_create(dev_name(&pdev->dev), &pdev->dev,
  813. sizeof(struct mdc_hw_list_desc),
  814. 4, 0);
  815. if (!mdma->desc_pool)
  816. return -ENOMEM;
  817. pm_runtime_enable(&pdev->dev);
  818. if (!pm_runtime_enabled(&pdev->dev)) {
  819. ret = img_mdc_runtime_resume(&pdev->dev);
  820. if (ret)
  821. return ret;
  822. }
  823. ret = dma_async_device_register(&mdma->dma_dev);
  824. if (ret)
  825. goto suspend;
  826. ret = of_dma_controller_register(pdev->dev.of_node, mdc_of_xlate, mdma);
  827. if (ret)
  828. goto unregister;
  829. dev_info(&pdev->dev, "MDC with %u channels and %u threads\n",
  830. mdma->nr_channels, mdma->nr_threads);
  831. return 0;
  832. unregister:
  833. dma_async_device_unregister(&mdma->dma_dev);
  834. suspend:
  835. if (!pm_runtime_enabled(&pdev->dev))
  836. img_mdc_runtime_suspend(&pdev->dev);
  837. pm_runtime_disable(&pdev->dev);
  838. return ret;
  839. }
  840. static int mdc_dma_remove(struct platform_device *pdev)
  841. {
  842. struct mdc_dma *mdma = platform_get_drvdata(pdev);
  843. struct mdc_chan *mchan, *next;
  844. of_dma_controller_free(pdev->dev.of_node);
  845. dma_async_device_unregister(&mdma->dma_dev);
  846. list_for_each_entry_safe(mchan, next, &mdma->dma_dev.channels,
  847. vc.chan.device_node) {
  848. list_del(&mchan->vc.chan.device_node);
  849. devm_free_irq(&pdev->dev, mchan->irq, mchan);
  850. tasklet_kill(&mchan->vc.task);
  851. }
  852. pm_runtime_disable(&pdev->dev);
  853. if (!pm_runtime_status_suspended(&pdev->dev))
  854. img_mdc_runtime_suspend(&pdev->dev);
  855. return 0;
  856. }
  857. #ifdef CONFIG_PM_SLEEP
  858. static int img_mdc_suspend_late(struct device *dev)
  859. {
  860. struct mdc_dma *mdma = dev_get_drvdata(dev);
  861. int i;
  862. /* Check that all channels are idle */
  863. for (i = 0; i < mdma->nr_channels; i++) {
  864. struct mdc_chan *mchan = &mdma->channels[i];
  865. if (unlikely(mchan->desc))
  866. return -EBUSY;
  867. }
  868. return pm_runtime_force_suspend(dev);
  869. }
  870. static int img_mdc_resume_early(struct device *dev)
  871. {
  872. return pm_runtime_force_resume(dev);
  873. }
  874. #endif /* CONFIG_PM_SLEEP */
  875. static const struct dev_pm_ops img_mdc_pm_ops = {
  876. SET_RUNTIME_PM_OPS(img_mdc_runtime_suspend,
  877. img_mdc_runtime_resume, NULL)
  878. SET_LATE_SYSTEM_SLEEP_PM_OPS(img_mdc_suspend_late,
  879. img_mdc_resume_early)
  880. };
  881. static struct platform_driver mdc_dma_driver = {
  882. .driver = {
  883. .name = "img-mdc-dma",
  884. .pm = &img_mdc_pm_ops,
  885. .of_match_table = of_match_ptr(mdc_dma_of_match),
  886. },
  887. .probe = mdc_dma_probe,
  888. .remove = mdc_dma_remove,
  889. };
  890. module_platform_driver(mdc_dma_driver);
  891. MODULE_DESCRIPTION("IMG Multi-threaded DMA Controller (MDC) driver");
  892. MODULE_AUTHOR("Andrew Bresticker <abrestic@chromium.org>");
  893. MODULE_LICENSE("GPL v2");