dma-jz4780.c 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * Ingenic JZ4780 DMA controller
  4. *
  5. * Copyright (c) 2015 Imagination Technologies
  6. * Author: Alex Smith <alex@alex-smith.me.uk>
  7. */
  8. #include <linux/clk.h>
  9. #include <linux/dmapool.h>
  10. #include <linux/init.h>
  11. #include <linux/interrupt.h>
  12. #include <linux/module.h>
  13. #include <linux/of.h>
  14. #include <linux/of_device.h>
  15. #include <linux/of_dma.h>
  16. #include <linux/platform_device.h>
  17. #include <linux/slab.h>
  18. #include "dmaengine.h"
  19. #include "virt-dma.h"
  20. /* Global registers. */
  21. #define JZ_DMA_REG_DMAC 0x00
  22. #define JZ_DMA_REG_DIRQP 0x04
  23. #define JZ_DMA_REG_DDR 0x08
  24. #define JZ_DMA_REG_DDRS 0x0c
  25. #define JZ_DMA_REG_DCKE 0x10
  26. #define JZ_DMA_REG_DCKES 0x14
  27. #define JZ_DMA_REG_DCKEC 0x18
  28. #define JZ_DMA_REG_DMACP 0x1c
  29. #define JZ_DMA_REG_DSIRQP 0x20
  30. #define JZ_DMA_REG_DSIRQM 0x24
  31. #define JZ_DMA_REG_DCIRQP 0x28
  32. #define JZ_DMA_REG_DCIRQM 0x2c
  33. /* Per-channel registers. */
  34. #define JZ_DMA_REG_CHAN(n) (n * 0x20)
  35. #define JZ_DMA_REG_DSA 0x00
  36. #define JZ_DMA_REG_DTA 0x04
  37. #define JZ_DMA_REG_DTC 0x08
  38. #define JZ_DMA_REG_DRT 0x0c
  39. #define JZ_DMA_REG_DCS 0x10
  40. #define JZ_DMA_REG_DCM 0x14
  41. #define JZ_DMA_REG_DDA 0x18
  42. #define JZ_DMA_REG_DSD 0x1c
  43. #define JZ_DMA_DMAC_DMAE BIT(0)
  44. #define JZ_DMA_DMAC_AR BIT(2)
  45. #define JZ_DMA_DMAC_HLT BIT(3)
  46. #define JZ_DMA_DMAC_FAIC BIT(27)
  47. #define JZ_DMA_DMAC_FMSC BIT(31)
  48. #define JZ_DMA_DRT_AUTO 0x8
  49. #define JZ_DMA_DCS_CTE BIT(0)
  50. #define JZ_DMA_DCS_HLT BIT(2)
  51. #define JZ_DMA_DCS_TT BIT(3)
  52. #define JZ_DMA_DCS_AR BIT(4)
  53. #define JZ_DMA_DCS_DES8 BIT(30)
  54. #define JZ_DMA_DCM_LINK BIT(0)
  55. #define JZ_DMA_DCM_TIE BIT(1)
  56. #define JZ_DMA_DCM_STDE BIT(2)
  57. #define JZ_DMA_DCM_TSZ_SHIFT 8
  58. #define JZ_DMA_DCM_TSZ_MASK (0x7 << JZ_DMA_DCM_TSZ_SHIFT)
  59. #define JZ_DMA_DCM_DP_SHIFT 12
  60. #define JZ_DMA_DCM_SP_SHIFT 14
  61. #define JZ_DMA_DCM_DAI BIT(22)
  62. #define JZ_DMA_DCM_SAI BIT(23)
  63. #define JZ_DMA_SIZE_4_BYTE 0x0
  64. #define JZ_DMA_SIZE_1_BYTE 0x1
  65. #define JZ_DMA_SIZE_2_BYTE 0x2
  66. #define JZ_DMA_SIZE_16_BYTE 0x3
  67. #define JZ_DMA_SIZE_32_BYTE 0x4
  68. #define JZ_DMA_SIZE_64_BYTE 0x5
  69. #define JZ_DMA_SIZE_128_BYTE 0x6
  70. #define JZ_DMA_WIDTH_32_BIT 0x0
  71. #define JZ_DMA_WIDTH_8_BIT 0x1
  72. #define JZ_DMA_WIDTH_16_BIT 0x2
  73. #define JZ_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
  74. BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
  75. BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
  76. #define JZ4780_DMA_CTRL_OFFSET 0x1000
  77. /* macros for use with jz4780_dma_soc_data.flags */
  78. #define JZ_SOC_DATA_ALLOW_LEGACY_DT BIT(0)
  79. #define JZ_SOC_DATA_PROGRAMMABLE_DMA BIT(1)
  80. #define JZ_SOC_DATA_PER_CHAN_PM BIT(2)
  81. #define JZ_SOC_DATA_NO_DCKES_DCKEC BIT(3)
  82. #define JZ_SOC_DATA_BREAK_LINKS BIT(4)
  83. /**
  84. * struct jz4780_dma_hwdesc - descriptor structure read by the DMA controller.
  85. * @dcm: value for the DCM (channel command) register
  86. * @dsa: source address
  87. * @dta: target address
  88. * @dtc: transfer count (number of blocks of the transfer size specified in DCM
  89. * to transfer) in the low 24 bits, offset of the next descriptor from the
  90. * descriptor base address in the upper 8 bits.
  91. */
  92. struct jz4780_dma_hwdesc {
  93. uint32_t dcm;
  94. uint32_t dsa;
  95. uint32_t dta;
  96. uint32_t dtc;
  97. };
  98. /* Size of allocations for hardware descriptor blocks. */
  99. #define JZ_DMA_DESC_BLOCK_SIZE PAGE_SIZE
  100. #define JZ_DMA_MAX_DESC \
  101. (JZ_DMA_DESC_BLOCK_SIZE / sizeof(struct jz4780_dma_hwdesc))
  102. struct jz4780_dma_desc {
  103. struct virt_dma_desc vdesc;
  104. struct jz4780_dma_hwdesc *desc;
  105. dma_addr_t desc_phys;
  106. unsigned int count;
  107. enum dma_transaction_type type;
  108. uint32_t status;
  109. };
  110. struct jz4780_dma_chan {
  111. struct virt_dma_chan vchan;
  112. unsigned int id;
  113. struct dma_pool *desc_pool;
  114. uint32_t transfer_type;
  115. uint32_t transfer_shift;
  116. struct dma_slave_config config;
  117. struct jz4780_dma_desc *desc;
  118. unsigned int curr_hwdesc;
  119. };
  120. struct jz4780_dma_soc_data {
  121. unsigned int nb_channels;
  122. unsigned int transfer_ord_max;
  123. unsigned long flags;
  124. };
  125. struct jz4780_dma_dev {
  126. struct dma_device dma_device;
  127. void __iomem *chn_base;
  128. void __iomem *ctrl_base;
  129. struct clk *clk;
  130. unsigned int irq;
  131. const struct jz4780_dma_soc_data *soc_data;
  132. uint32_t chan_reserved;
  133. struct jz4780_dma_chan chan[];
  134. };
  135. struct jz4780_dma_filter_data {
  136. uint32_t transfer_type;
  137. int channel;
  138. };
  139. static inline struct jz4780_dma_chan *to_jz4780_dma_chan(struct dma_chan *chan)
  140. {
  141. return container_of(chan, struct jz4780_dma_chan, vchan.chan);
  142. }
  143. static inline struct jz4780_dma_desc *to_jz4780_dma_desc(
  144. struct virt_dma_desc *vdesc)
  145. {
  146. return container_of(vdesc, struct jz4780_dma_desc, vdesc);
  147. }
  148. static inline struct jz4780_dma_dev *jz4780_dma_chan_parent(
  149. struct jz4780_dma_chan *jzchan)
  150. {
  151. return container_of(jzchan->vchan.chan.device, struct jz4780_dma_dev,
  152. dma_device);
  153. }
  154. static inline uint32_t jz4780_dma_chn_readl(struct jz4780_dma_dev *jzdma,
  155. unsigned int chn, unsigned int reg)
  156. {
  157. return readl(jzdma->chn_base + reg + JZ_DMA_REG_CHAN(chn));
  158. }
  159. static inline void jz4780_dma_chn_writel(struct jz4780_dma_dev *jzdma,
  160. unsigned int chn, unsigned int reg, uint32_t val)
  161. {
  162. writel(val, jzdma->chn_base + reg + JZ_DMA_REG_CHAN(chn));
  163. }
  164. static inline uint32_t jz4780_dma_ctrl_readl(struct jz4780_dma_dev *jzdma,
  165. unsigned int reg)
  166. {
  167. return readl(jzdma->ctrl_base + reg);
  168. }
  169. static inline void jz4780_dma_ctrl_writel(struct jz4780_dma_dev *jzdma,
  170. unsigned int reg, uint32_t val)
  171. {
  172. writel(val, jzdma->ctrl_base + reg);
  173. }
  174. static inline void jz4780_dma_chan_enable(struct jz4780_dma_dev *jzdma,
  175. unsigned int chn)
  176. {
  177. if (jzdma->soc_data->flags & JZ_SOC_DATA_PER_CHAN_PM) {
  178. unsigned int reg;
  179. if (jzdma->soc_data->flags & JZ_SOC_DATA_NO_DCKES_DCKEC)
  180. reg = JZ_DMA_REG_DCKE;
  181. else
  182. reg = JZ_DMA_REG_DCKES;
  183. jz4780_dma_ctrl_writel(jzdma, reg, BIT(chn));
  184. }
  185. }
  186. static inline void jz4780_dma_chan_disable(struct jz4780_dma_dev *jzdma,
  187. unsigned int chn)
  188. {
  189. if ((jzdma->soc_data->flags & JZ_SOC_DATA_PER_CHAN_PM) &&
  190. !(jzdma->soc_data->flags & JZ_SOC_DATA_NO_DCKES_DCKEC))
  191. jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DCKEC, BIT(chn));
  192. }
  193. static struct jz4780_dma_desc *jz4780_dma_desc_alloc(
  194. struct jz4780_dma_chan *jzchan, unsigned int count,
  195. enum dma_transaction_type type)
  196. {
  197. struct jz4780_dma_desc *desc;
  198. if (count > JZ_DMA_MAX_DESC)
  199. return NULL;
  200. desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
  201. if (!desc)
  202. return NULL;
  203. desc->desc = dma_pool_alloc(jzchan->desc_pool, GFP_NOWAIT,
  204. &desc->desc_phys);
  205. if (!desc->desc) {
  206. kfree(desc);
  207. return NULL;
  208. }
  209. desc->count = count;
  210. desc->type = type;
  211. return desc;
  212. }
  213. static void jz4780_dma_desc_free(struct virt_dma_desc *vdesc)
  214. {
  215. struct jz4780_dma_desc *desc = to_jz4780_dma_desc(vdesc);
  216. struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(vdesc->tx.chan);
  217. dma_pool_free(jzchan->desc_pool, desc->desc, desc->desc_phys);
  218. kfree(desc);
  219. }
  220. static uint32_t jz4780_dma_transfer_size(struct jz4780_dma_chan *jzchan,
  221. unsigned long val, uint32_t *shift)
  222. {
  223. struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan);
  224. int ord = ffs(val) - 1;
  225. /*
  226. * 8 byte transfer sizes unsupported so fall back on 4. If it's larger
  227. * than the maximum, just limit it. It is perfectly safe to fall back
  228. * in this way since we won't exceed the maximum burst size supported
  229. * by the device, the only effect is reduced efficiency. This is better
  230. * than refusing to perform the request at all.
  231. */
  232. if (ord == 3)
  233. ord = 2;
  234. else if (ord > jzdma->soc_data->transfer_ord_max)
  235. ord = jzdma->soc_data->transfer_ord_max;
  236. *shift = ord;
  237. switch (ord) {
  238. case 0:
  239. return JZ_DMA_SIZE_1_BYTE;
  240. case 1:
  241. return JZ_DMA_SIZE_2_BYTE;
  242. case 2:
  243. return JZ_DMA_SIZE_4_BYTE;
  244. case 4:
  245. return JZ_DMA_SIZE_16_BYTE;
  246. case 5:
  247. return JZ_DMA_SIZE_32_BYTE;
  248. case 6:
  249. return JZ_DMA_SIZE_64_BYTE;
  250. default:
  251. return JZ_DMA_SIZE_128_BYTE;
  252. }
  253. }
  254. static int jz4780_dma_setup_hwdesc(struct jz4780_dma_chan *jzchan,
  255. struct jz4780_dma_hwdesc *desc, dma_addr_t addr, size_t len,
  256. enum dma_transfer_direction direction)
  257. {
  258. struct dma_slave_config *config = &jzchan->config;
  259. uint32_t width, maxburst, tsz;
  260. if (direction == DMA_MEM_TO_DEV) {
  261. desc->dcm = JZ_DMA_DCM_SAI;
  262. desc->dsa = addr;
  263. desc->dta = config->dst_addr;
  264. width = config->dst_addr_width;
  265. maxburst = config->dst_maxburst;
  266. } else {
  267. desc->dcm = JZ_DMA_DCM_DAI;
  268. desc->dsa = config->src_addr;
  269. desc->dta = addr;
  270. width = config->src_addr_width;
  271. maxburst = config->src_maxburst;
  272. }
  273. /*
  274. * This calculates the maximum transfer size that can be used with the
  275. * given address, length, width and maximum burst size. The address
  276. * must be aligned to the transfer size, the total length must be
  277. * divisible by the transfer size, and we must not use more than the
  278. * maximum burst specified by the user.
  279. */
  280. tsz = jz4780_dma_transfer_size(jzchan, addr | len | (width * maxburst),
  281. &jzchan->transfer_shift);
  282. switch (width) {
  283. case DMA_SLAVE_BUSWIDTH_1_BYTE:
  284. case DMA_SLAVE_BUSWIDTH_2_BYTES:
  285. break;
  286. case DMA_SLAVE_BUSWIDTH_4_BYTES:
  287. width = JZ_DMA_WIDTH_32_BIT;
  288. break;
  289. default:
  290. return -EINVAL;
  291. }
  292. desc->dcm |= tsz << JZ_DMA_DCM_TSZ_SHIFT;
  293. desc->dcm |= width << JZ_DMA_DCM_SP_SHIFT;
  294. desc->dcm |= width << JZ_DMA_DCM_DP_SHIFT;
  295. desc->dtc = len >> jzchan->transfer_shift;
  296. return 0;
  297. }
  298. static struct dma_async_tx_descriptor *jz4780_dma_prep_slave_sg(
  299. struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len,
  300. enum dma_transfer_direction direction, unsigned long flags,
  301. void *context)
  302. {
  303. struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);
  304. struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan);
  305. struct jz4780_dma_desc *desc;
  306. unsigned int i;
  307. int err;
  308. desc = jz4780_dma_desc_alloc(jzchan, sg_len, DMA_SLAVE);
  309. if (!desc)
  310. return NULL;
  311. for (i = 0; i < sg_len; i++) {
  312. err = jz4780_dma_setup_hwdesc(jzchan, &desc->desc[i],
  313. sg_dma_address(&sgl[i]),
  314. sg_dma_len(&sgl[i]),
  315. direction);
  316. if (err < 0) {
  317. jz4780_dma_desc_free(&jzchan->desc->vdesc);
  318. return NULL;
  319. }
  320. desc->desc[i].dcm |= JZ_DMA_DCM_TIE;
  321. if (i != (sg_len - 1) &&
  322. !(jzdma->soc_data->flags & JZ_SOC_DATA_BREAK_LINKS)) {
  323. /* Automatically proceeed to the next descriptor. */
  324. desc->desc[i].dcm |= JZ_DMA_DCM_LINK;
  325. /*
  326. * The upper 8 bits of the DTC field in the descriptor
  327. * must be set to (offset from descriptor base of next
  328. * descriptor >> 4).
  329. */
  330. desc->desc[i].dtc |=
  331. (((i + 1) * sizeof(*desc->desc)) >> 4) << 24;
  332. }
  333. }
  334. return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags);
  335. }
  336. static struct dma_async_tx_descriptor *jz4780_dma_prep_dma_cyclic(
  337. struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
  338. size_t period_len, enum dma_transfer_direction direction,
  339. unsigned long flags)
  340. {
  341. struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);
  342. struct jz4780_dma_desc *desc;
  343. unsigned int periods, i;
  344. int err;
  345. if (buf_len % period_len)
  346. return NULL;
  347. periods = buf_len / period_len;
  348. desc = jz4780_dma_desc_alloc(jzchan, periods, DMA_CYCLIC);
  349. if (!desc)
  350. return NULL;
  351. for (i = 0; i < periods; i++) {
  352. err = jz4780_dma_setup_hwdesc(jzchan, &desc->desc[i], buf_addr,
  353. period_len, direction);
  354. if (err < 0) {
  355. jz4780_dma_desc_free(&jzchan->desc->vdesc);
  356. return NULL;
  357. }
  358. buf_addr += period_len;
  359. /*
  360. * Set the link bit to indicate that the controller should
  361. * automatically proceed to the next descriptor. In
  362. * jz4780_dma_begin(), this will be cleared if we need to issue
  363. * an interrupt after each period.
  364. */
  365. desc->desc[i].dcm |= JZ_DMA_DCM_TIE | JZ_DMA_DCM_LINK;
  366. /*
  367. * The upper 8 bits of the DTC field in the descriptor must be
  368. * set to (offset from descriptor base of next descriptor >> 4).
  369. * If this is the last descriptor, link it back to the first,
  370. * i.e. leave offset set to 0, otherwise point to the next one.
  371. */
  372. if (i != (periods - 1)) {
  373. desc->desc[i].dtc |=
  374. (((i + 1) * sizeof(*desc->desc)) >> 4) << 24;
  375. }
  376. }
  377. return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags);
  378. }
  379. static struct dma_async_tx_descriptor *jz4780_dma_prep_dma_memcpy(
  380. struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
  381. size_t len, unsigned long flags)
  382. {
  383. struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);
  384. struct jz4780_dma_desc *desc;
  385. uint32_t tsz;
  386. desc = jz4780_dma_desc_alloc(jzchan, 1, DMA_MEMCPY);
  387. if (!desc)
  388. return NULL;
  389. tsz = jz4780_dma_transfer_size(jzchan, dest | src | len,
  390. &jzchan->transfer_shift);
  391. jzchan->transfer_type = JZ_DMA_DRT_AUTO;
  392. desc->desc[0].dsa = src;
  393. desc->desc[0].dta = dest;
  394. desc->desc[0].dcm = JZ_DMA_DCM_TIE | JZ_DMA_DCM_SAI | JZ_DMA_DCM_DAI |
  395. tsz << JZ_DMA_DCM_TSZ_SHIFT |
  396. JZ_DMA_WIDTH_32_BIT << JZ_DMA_DCM_SP_SHIFT |
  397. JZ_DMA_WIDTH_32_BIT << JZ_DMA_DCM_DP_SHIFT;
  398. desc->desc[0].dtc = len >> jzchan->transfer_shift;
  399. return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags);
  400. }
  401. static void jz4780_dma_begin(struct jz4780_dma_chan *jzchan)
  402. {
  403. struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan);
  404. struct virt_dma_desc *vdesc;
  405. unsigned int i;
  406. dma_addr_t desc_phys;
  407. if (!jzchan->desc) {
  408. vdesc = vchan_next_desc(&jzchan->vchan);
  409. if (!vdesc)
  410. return;
  411. list_del(&vdesc->node);
  412. jzchan->desc = to_jz4780_dma_desc(vdesc);
  413. jzchan->curr_hwdesc = 0;
  414. if (jzchan->desc->type == DMA_CYCLIC && vdesc->tx.callback) {
  415. /*
  416. * The DMA controller doesn't support triggering an
  417. * interrupt after processing each descriptor, only
  418. * after processing an entire terminated list of
  419. * descriptors. For a cyclic DMA setup the list of
  420. * descriptors is not terminated so we can never get an
  421. * interrupt.
  422. *
  423. * If the user requested a callback for a cyclic DMA
  424. * setup then we workaround this hardware limitation
  425. * here by degrading to a set of unlinked descriptors
  426. * which we will submit in sequence in response to the
  427. * completion of processing the previous descriptor.
  428. */
  429. for (i = 0; i < jzchan->desc->count; i++)
  430. jzchan->desc->desc[i].dcm &= ~JZ_DMA_DCM_LINK;
  431. }
  432. } else {
  433. /*
  434. * There is an existing transfer, therefore this must be one
  435. * for which we unlinked the descriptors above. Advance to the
  436. * next one in the list.
  437. */
  438. jzchan->curr_hwdesc =
  439. (jzchan->curr_hwdesc + 1) % jzchan->desc->count;
  440. }
  441. /* Enable the channel's clock. */
  442. jz4780_dma_chan_enable(jzdma, jzchan->id);
  443. /* Use 4-word descriptors. */
  444. jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DCS, 0);
  445. /* Set transfer type. */
  446. jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DRT,
  447. jzchan->transfer_type);
  448. /*
  449. * Set the transfer count. This is redundant for a descriptor-driven
  450. * transfer. However, there can be a delay between the transfer start
  451. * time and when DTCn reg contains the new transfer count. Setting
  452. * it explicitly ensures residue is computed correctly at all times.
  453. */
  454. jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DTC,
  455. jzchan->desc->desc[jzchan->curr_hwdesc].dtc);
  456. /* Write descriptor address and initiate descriptor fetch. */
  457. desc_phys = jzchan->desc->desc_phys +
  458. (jzchan->curr_hwdesc * sizeof(*jzchan->desc->desc));
  459. jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DDA, desc_phys);
  460. jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DDRS, BIT(jzchan->id));
  461. /* Enable the channel. */
  462. jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DCS,
  463. JZ_DMA_DCS_CTE);
  464. }
  465. static void jz4780_dma_issue_pending(struct dma_chan *chan)
  466. {
  467. struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);
  468. unsigned long flags;
  469. spin_lock_irqsave(&jzchan->vchan.lock, flags);
  470. if (vchan_issue_pending(&jzchan->vchan) && !jzchan->desc)
  471. jz4780_dma_begin(jzchan);
  472. spin_unlock_irqrestore(&jzchan->vchan.lock, flags);
  473. }
  474. static int jz4780_dma_terminate_all(struct dma_chan *chan)
  475. {
  476. struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);
  477. struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan);
  478. unsigned long flags;
  479. LIST_HEAD(head);
  480. spin_lock_irqsave(&jzchan->vchan.lock, flags);
  481. /* Clear the DMA status and stop the transfer. */
  482. jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DCS, 0);
  483. if (jzchan->desc) {
  484. vchan_terminate_vdesc(&jzchan->desc->vdesc);
  485. jzchan->desc = NULL;
  486. }
  487. jz4780_dma_chan_disable(jzdma, jzchan->id);
  488. vchan_get_all_descriptors(&jzchan->vchan, &head);
  489. spin_unlock_irqrestore(&jzchan->vchan.lock, flags);
  490. vchan_dma_desc_free_list(&jzchan->vchan, &head);
  491. return 0;
  492. }
  493. static void jz4780_dma_synchronize(struct dma_chan *chan)
  494. {
  495. struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);
  496. struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan);
  497. vchan_synchronize(&jzchan->vchan);
  498. jz4780_dma_chan_disable(jzdma, jzchan->id);
  499. }
  500. static int jz4780_dma_config(struct dma_chan *chan,
  501. struct dma_slave_config *config)
  502. {
  503. struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);
  504. if ((config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
  505. || (config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES))
  506. return -EINVAL;
  507. /* Copy the reset of the slave configuration, it is used later. */
  508. memcpy(&jzchan->config, config, sizeof(jzchan->config));
  509. return 0;
  510. }
  511. static size_t jz4780_dma_desc_residue(struct jz4780_dma_chan *jzchan,
  512. struct jz4780_dma_desc *desc, unsigned int next_sg)
  513. {
  514. struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan);
  515. unsigned int count = 0;
  516. unsigned int i;
  517. for (i = next_sg; i < desc->count; i++)
  518. count += desc->desc[i].dtc & GENMASK(23, 0);
  519. if (next_sg != 0)
  520. count += jz4780_dma_chn_readl(jzdma, jzchan->id,
  521. JZ_DMA_REG_DTC);
  522. return count << jzchan->transfer_shift;
  523. }
  524. static enum dma_status jz4780_dma_tx_status(struct dma_chan *chan,
  525. dma_cookie_t cookie, struct dma_tx_state *txstate)
  526. {
  527. struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);
  528. struct virt_dma_desc *vdesc;
  529. enum dma_status status;
  530. unsigned long flags;
  531. unsigned long residue = 0;
  532. spin_lock_irqsave(&jzchan->vchan.lock, flags);
  533. status = dma_cookie_status(chan, cookie, txstate);
  534. if ((status == DMA_COMPLETE) || (txstate == NULL))
  535. goto out_unlock_irqrestore;
  536. vdesc = vchan_find_desc(&jzchan->vchan, cookie);
  537. if (vdesc) {
  538. /* On the issued list, so hasn't been processed yet */
  539. residue = jz4780_dma_desc_residue(jzchan,
  540. to_jz4780_dma_desc(vdesc), 0);
  541. } else if (cookie == jzchan->desc->vdesc.tx.cookie) {
  542. residue = jz4780_dma_desc_residue(jzchan, jzchan->desc,
  543. jzchan->curr_hwdesc + 1);
  544. }
  545. dma_set_residue(txstate, residue);
  546. if (vdesc && jzchan->desc && vdesc == &jzchan->desc->vdesc
  547. && jzchan->desc->status & (JZ_DMA_DCS_AR | JZ_DMA_DCS_HLT))
  548. status = DMA_ERROR;
  549. out_unlock_irqrestore:
  550. spin_unlock_irqrestore(&jzchan->vchan.lock, flags);
  551. return status;
  552. }
  553. static bool jz4780_dma_chan_irq(struct jz4780_dma_dev *jzdma,
  554. struct jz4780_dma_chan *jzchan)
  555. {
  556. const unsigned int soc_flags = jzdma->soc_data->flags;
  557. struct jz4780_dma_desc *desc = jzchan->desc;
  558. uint32_t dcs;
  559. bool ack = true;
  560. spin_lock(&jzchan->vchan.lock);
  561. dcs = jz4780_dma_chn_readl(jzdma, jzchan->id, JZ_DMA_REG_DCS);
  562. jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DCS, 0);
  563. if (dcs & JZ_DMA_DCS_AR) {
  564. dev_warn(&jzchan->vchan.chan.dev->device,
  565. "address error (DCS=0x%x)\n", dcs);
  566. }
  567. if (dcs & JZ_DMA_DCS_HLT) {
  568. dev_warn(&jzchan->vchan.chan.dev->device,
  569. "channel halt (DCS=0x%x)\n", dcs);
  570. }
  571. if (jzchan->desc) {
  572. jzchan->desc->status = dcs;
  573. if ((dcs & (JZ_DMA_DCS_AR | JZ_DMA_DCS_HLT)) == 0) {
  574. if (jzchan->desc->type == DMA_CYCLIC) {
  575. vchan_cyclic_callback(&jzchan->desc->vdesc);
  576. jz4780_dma_begin(jzchan);
  577. } else if (dcs & JZ_DMA_DCS_TT) {
  578. if (!(soc_flags & JZ_SOC_DATA_BREAK_LINKS) ||
  579. (jzchan->curr_hwdesc + 1 == desc->count)) {
  580. vchan_cookie_complete(&desc->vdesc);
  581. jzchan->desc = NULL;
  582. }
  583. jz4780_dma_begin(jzchan);
  584. } else {
  585. /* False positive - continue the transfer */
  586. ack = false;
  587. jz4780_dma_chn_writel(jzdma, jzchan->id,
  588. JZ_DMA_REG_DCS,
  589. JZ_DMA_DCS_CTE);
  590. }
  591. }
  592. } else {
  593. dev_err(&jzchan->vchan.chan.dev->device,
  594. "channel IRQ with no active transfer\n");
  595. }
  596. spin_unlock(&jzchan->vchan.lock);
  597. return ack;
  598. }
  599. static irqreturn_t jz4780_dma_irq_handler(int irq, void *data)
  600. {
  601. struct jz4780_dma_dev *jzdma = data;
  602. unsigned int nb_channels = jzdma->soc_data->nb_channels;
  603. unsigned long pending;
  604. uint32_t dmac;
  605. int i;
  606. pending = jz4780_dma_ctrl_readl(jzdma, JZ_DMA_REG_DIRQP);
  607. for_each_set_bit(i, &pending, nb_channels) {
  608. if (jz4780_dma_chan_irq(jzdma, &jzdma->chan[i]))
  609. pending &= ~BIT(i);
  610. }
  611. /* Clear halt and address error status of all channels. */
  612. dmac = jz4780_dma_ctrl_readl(jzdma, JZ_DMA_REG_DMAC);
  613. dmac &= ~(JZ_DMA_DMAC_HLT | JZ_DMA_DMAC_AR);
  614. jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DMAC, dmac);
  615. /* Clear interrupt pending status. */
  616. jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DIRQP, pending);
  617. return IRQ_HANDLED;
  618. }
  619. static int jz4780_dma_alloc_chan_resources(struct dma_chan *chan)
  620. {
  621. struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);
  622. jzchan->desc_pool = dma_pool_create(dev_name(&chan->dev->device),
  623. chan->device->dev,
  624. JZ_DMA_DESC_BLOCK_SIZE,
  625. PAGE_SIZE, 0);
  626. if (!jzchan->desc_pool) {
  627. dev_err(&chan->dev->device,
  628. "failed to allocate descriptor pool\n");
  629. return -ENOMEM;
  630. }
  631. return 0;
  632. }
  633. static void jz4780_dma_free_chan_resources(struct dma_chan *chan)
  634. {
  635. struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);
  636. vchan_free_chan_resources(&jzchan->vchan);
  637. dma_pool_destroy(jzchan->desc_pool);
  638. jzchan->desc_pool = NULL;
  639. }
  640. static bool jz4780_dma_filter_fn(struct dma_chan *chan, void *param)
  641. {
  642. struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);
  643. struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan);
  644. struct jz4780_dma_filter_data *data = param;
  645. if (data->channel > -1) {
  646. if (data->channel != jzchan->id)
  647. return false;
  648. } else if (jzdma->chan_reserved & BIT(jzchan->id)) {
  649. return false;
  650. }
  651. jzchan->transfer_type = data->transfer_type;
  652. return true;
  653. }
  654. static struct dma_chan *jz4780_of_dma_xlate(struct of_phandle_args *dma_spec,
  655. struct of_dma *ofdma)
  656. {
  657. struct jz4780_dma_dev *jzdma = ofdma->of_dma_data;
  658. dma_cap_mask_t mask = jzdma->dma_device.cap_mask;
  659. struct jz4780_dma_filter_data data;
  660. if (dma_spec->args_count != 2)
  661. return NULL;
  662. data.transfer_type = dma_spec->args[0];
  663. data.channel = dma_spec->args[1];
  664. if (data.channel > -1) {
  665. if (data.channel >= jzdma->soc_data->nb_channels) {
  666. dev_err(jzdma->dma_device.dev,
  667. "device requested non-existent channel %u\n",
  668. data.channel);
  669. return NULL;
  670. }
  671. /* Can only select a channel marked as reserved. */
  672. if (!(jzdma->chan_reserved & BIT(data.channel))) {
  673. dev_err(jzdma->dma_device.dev,
  674. "device requested unreserved channel %u\n",
  675. data.channel);
  676. return NULL;
  677. }
  678. jzdma->chan[data.channel].transfer_type = data.transfer_type;
  679. return dma_get_slave_channel(
  680. &jzdma->chan[data.channel].vchan.chan);
  681. } else {
  682. return __dma_request_channel(&mask, jz4780_dma_filter_fn, &data,
  683. ofdma->of_node);
  684. }
  685. }
  686. static int jz4780_dma_probe(struct platform_device *pdev)
  687. {
  688. struct device *dev = &pdev->dev;
  689. const struct jz4780_dma_soc_data *soc_data;
  690. struct jz4780_dma_dev *jzdma;
  691. struct jz4780_dma_chan *jzchan;
  692. struct dma_device *dd;
  693. struct resource *res;
  694. int i, ret;
  695. if (!dev->of_node) {
  696. dev_err(dev, "This driver must be probed from devicetree\n");
  697. return -EINVAL;
  698. }
  699. soc_data = device_get_match_data(dev);
  700. if (!soc_data)
  701. return -EINVAL;
  702. jzdma = devm_kzalloc(dev, struct_size(jzdma, chan,
  703. soc_data->nb_channels), GFP_KERNEL);
  704. if (!jzdma)
  705. return -ENOMEM;
  706. jzdma->soc_data = soc_data;
  707. platform_set_drvdata(pdev, jzdma);
  708. jzdma->chn_base = devm_platform_ioremap_resource(pdev, 0);
  709. if (IS_ERR(jzdma->chn_base))
  710. return PTR_ERR(jzdma->chn_base);
  711. res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
  712. if (res) {
  713. jzdma->ctrl_base = devm_ioremap_resource(dev, res);
  714. if (IS_ERR(jzdma->ctrl_base))
  715. return PTR_ERR(jzdma->ctrl_base);
  716. } else if (soc_data->flags & JZ_SOC_DATA_ALLOW_LEGACY_DT) {
  717. /*
  718. * On JZ4780, if the second memory resource was not supplied,
  719. * assume we're using an old devicetree, and calculate the
  720. * offset to the control registers.
  721. */
  722. jzdma->ctrl_base = jzdma->chn_base + JZ4780_DMA_CTRL_OFFSET;
  723. } else {
  724. dev_err(dev, "failed to get I/O memory\n");
  725. return -EINVAL;
  726. }
  727. jzdma->clk = devm_clk_get(dev, NULL);
  728. if (IS_ERR(jzdma->clk)) {
  729. dev_err(dev, "failed to get clock\n");
  730. ret = PTR_ERR(jzdma->clk);
  731. return ret;
  732. }
  733. clk_prepare_enable(jzdma->clk);
  734. /* Property is optional, if it doesn't exist the value will remain 0. */
  735. of_property_read_u32_index(dev->of_node, "ingenic,reserved-channels",
  736. 0, &jzdma->chan_reserved);
  737. dd = &jzdma->dma_device;
  738. dma_cap_set(DMA_MEMCPY, dd->cap_mask);
  739. dma_cap_set(DMA_SLAVE, dd->cap_mask);
  740. dma_cap_set(DMA_CYCLIC, dd->cap_mask);
  741. dd->dev = dev;
  742. dd->copy_align = DMAENGINE_ALIGN_4_BYTES;
  743. dd->device_alloc_chan_resources = jz4780_dma_alloc_chan_resources;
  744. dd->device_free_chan_resources = jz4780_dma_free_chan_resources;
  745. dd->device_prep_slave_sg = jz4780_dma_prep_slave_sg;
  746. dd->device_prep_dma_cyclic = jz4780_dma_prep_dma_cyclic;
  747. dd->device_prep_dma_memcpy = jz4780_dma_prep_dma_memcpy;
  748. dd->device_config = jz4780_dma_config;
  749. dd->device_terminate_all = jz4780_dma_terminate_all;
  750. dd->device_synchronize = jz4780_dma_synchronize;
  751. dd->device_tx_status = jz4780_dma_tx_status;
  752. dd->device_issue_pending = jz4780_dma_issue_pending;
  753. dd->src_addr_widths = JZ_DMA_BUSWIDTHS;
  754. dd->dst_addr_widths = JZ_DMA_BUSWIDTHS;
  755. dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
  756. dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
  757. /*
  758. * Enable DMA controller, mark all channels as not programmable.
  759. * Also set the FMSC bit - it increases MSC performance, so it makes
  760. * little sense not to enable it.
  761. */
  762. jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DMAC, JZ_DMA_DMAC_DMAE |
  763. JZ_DMA_DMAC_FAIC | JZ_DMA_DMAC_FMSC);
  764. if (soc_data->flags & JZ_SOC_DATA_PROGRAMMABLE_DMA)
  765. jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DMACP, 0);
  766. INIT_LIST_HEAD(&dd->channels);
  767. for (i = 0; i < soc_data->nb_channels; i++) {
  768. jzchan = &jzdma->chan[i];
  769. jzchan->id = i;
  770. vchan_init(&jzchan->vchan, dd);
  771. jzchan->vchan.desc_free = jz4780_dma_desc_free;
  772. }
  773. ret = platform_get_irq(pdev, 0);
  774. if (ret < 0)
  775. goto err_disable_clk;
  776. jzdma->irq = ret;
  777. ret = request_irq(jzdma->irq, jz4780_dma_irq_handler, 0, dev_name(dev),
  778. jzdma);
  779. if (ret) {
  780. dev_err(dev, "failed to request IRQ %u!\n", jzdma->irq);
  781. goto err_disable_clk;
  782. }
  783. ret = dmaenginem_async_device_register(dd);
  784. if (ret) {
  785. dev_err(dev, "failed to register device\n");
  786. goto err_free_irq;
  787. }
  788. /* Register with OF DMA helpers. */
  789. ret = of_dma_controller_register(dev->of_node, jz4780_of_dma_xlate,
  790. jzdma);
  791. if (ret) {
  792. dev_err(dev, "failed to register OF DMA controller\n");
  793. goto err_free_irq;
  794. }
  795. dev_info(dev, "JZ4780 DMA controller initialised\n");
  796. return 0;
  797. err_free_irq:
  798. free_irq(jzdma->irq, jzdma);
  799. err_disable_clk:
  800. clk_disable_unprepare(jzdma->clk);
  801. return ret;
  802. }
  803. static int jz4780_dma_remove(struct platform_device *pdev)
  804. {
  805. struct jz4780_dma_dev *jzdma = platform_get_drvdata(pdev);
  806. int i;
  807. of_dma_controller_free(pdev->dev.of_node);
  808. clk_disable_unprepare(jzdma->clk);
  809. free_irq(jzdma->irq, jzdma);
  810. for (i = 0; i < jzdma->soc_data->nb_channels; i++)
  811. tasklet_kill(&jzdma->chan[i].vchan.task);
  812. return 0;
  813. }
  814. static const struct jz4780_dma_soc_data jz4740_dma_soc_data = {
  815. .nb_channels = 6,
  816. .transfer_ord_max = 5,
  817. .flags = JZ_SOC_DATA_BREAK_LINKS,
  818. };
  819. static const struct jz4780_dma_soc_data jz4725b_dma_soc_data = {
  820. .nb_channels = 6,
  821. .transfer_ord_max = 5,
  822. .flags = JZ_SOC_DATA_PER_CHAN_PM | JZ_SOC_DATA_NO_DCKES_DCKEC |
  823. JZ_SOC_DATA_BREAK_LINKS,
  824. };
  825. static const struct jz4780_dma_soc_data jz4770_dma_soc_data = {
  826. .nb_channels = 6,
  827. .transfer_ord_max = 6,
  828. .flags = JZ_SOC_DATA_PER_CHAN_PM,
  829. };
  830. static const struct jz4780_dma_soc_data jz4780_dma_soc_data = {
  831. .nb_channels = 32,
  832. .transfer_ord_max = 7,
  833. .flags = JZ_SOC_DATA_ALLOW_LEGACY_DT | JZ_SOC_DATA_PROGRAMMABLE_DMA,
  834. };
  835. static const struct jz4780_dma_soc_data x1000_dma_soc_data = {
  836. .nb_channels = 8,
  837. .transfer_ord_max = 7,
  838. .flags = JZ_SOC_DATA_PROGRAMMABLE_DMA,
  839. };
  840. static const struct jz4780_dma_soc_data x1830_dma_soc_data = {
  841. .nb_channels = 32,
  842. .transfer_ord_max = 7,
  843. .flags = JZ_SOC_DATA_PROGRAMMABLE_DMA,
  844. };
  845. static const struct of_device_id jz4780_dma_dt_match[] = {
  846. { .compatible = "ingenic,jz4740-dma", .data = &jz4740_dma_soc_data },
  847. { .compatible = "ingenic,jz4725b-dma", .data = &jz4725b_dma_soc_data },
  848. { .compatible = "ingenic,jz4770-dma", .data = &jz4770_dma_soc_data },
  849. { .compatible = "ingenic,jz4780-dma", .data = &jz4780_dma_soc_data },
  850. { .compatible = "ingenic,x1000-dma", .data = &x1000_dma_soc_data },
  851. { .compatible = "ingenic,x1830-dma", .data = &x1830_dma_soc_data },
  852. {},
  853. };
  854. MODULE_DEVICE_TABLE(of, jz4780_dma_dt_match);
  855. static struct platform_driver jz4780_dma_driver = {
  856. .probe = jz4780_dma_probe,
  857. .remove = jz4780_dma_remove,
  858. .driver = {
  859. .name = "jz4780-dma",
  860. .of_match_table = of_match_ptr(jz4780_dma_dt_match),
  861. },
  862. };
  863. static int __init jz4780_dma_init(void)
  864. {
  865. return platform_driver_register(&jz4780_dma_driver);
  866. }
  867. subsys_initcall(jz4780_dma_init);
  868. static void __exit jz4780_dma_exit(void)
  869. {
  870. platform_driver_unregister(&jz4780_dma_driver);
  871. }
  872. module_exit(jz4780_dma_exit);
  873. MODULE_AUTHOR("Alex Smith <alex@alex-smith.me.uk>");
  874. MODULE_DESCRIPTION("Ingenic JZ4780 DMA controller driver");
  875. MODULE_LICENSE("GPL");