mv_xor_v2.c 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * Copyright (C) 2015-2016 Marvell International Ltd.
  4. */
  5. #include <linux/clk.h>
  6. #include <linux/dma-mapping.h>
  7. #include <linux/interrupt.h>
  8. #include <linux/io.h>
  9. #include <linux/module.h>
  10. #include <linux/msi.h>
  11. #include <linux/of.h>
  12. #include <linux/of_irq.h>
  13. #include <linux/platform_device.h>
  14. #include <linux/spinlock.h>
  15. #include "dmaengine.h"
  16. /* DMA Engine Registers */
  17. #define MV_XOR_V2_DMA_DESQ_BALR_OFF 0x000
  18. #define MV_XOR_V2_DMA_DESQ_BAHR_OFF 0x004
  19. #define MV_XOR_V2_DMA_DESQ_SIZE_OFF 0x008
  20. #define MV_XOR_V2_DMA_DESQ_DONE_OFF 0x00C
  21. #define MV_XOR_V2_DMA_DESQ_DONE_PENDING_MASK 0x7FFF
  22. #define MV_XOR_V2_DMA_DESQ_DONE_PENDING_SHIFT 0
  23. #define MV_XOR_V2_DMA_DESQ_DONE_READ_PTR_MASK 0x1FFF
  24. #define MV_XOR_V2_DMA_DESQ_DONE_READ_PTR_SHIFT 16
  25. #define MV_XOR_V2_DMA_DESQ_ARATTR_OFF 0x010
  26. #define MV_XOR_V2_DMA_DESQ_ATTR_CACHE_MASK 0x3F3F
  27. #define MV_XOR_V2_DMA_DESQ_ATTR_OUTER_SHAREABLE 0x202
  28. #define MV_XOR_V2_DMA_DESQ_ATTR_CACHEABLE 0x3C3C
  29. #define MV_XOR_V2_DMA_IMSG_CDAT_OFF 0x014
  30. #define MV_XOR_V2_DMA_IMSG_THRD_OFF 0x018
  31. #define MV_XOR_V2_DMA_IMSG_THRD_MASK 0x7FFF
  32. #define MV_XOR_V2_DMA_IMSG_TIMER_EN BIT(18)
  33. #define MV_XOR_V2_DMA_DESQ_AWATTR_OFF 0x01C
  34. /* Same flags as MV_XOR_V2_DMA_DESQ_ARATTR_OFF */
  35. #define MV_XOR_V2_DMA_DESQ_ALLOC_OFF 0x04C
  36. #define MV_XOR_V2_DMA_DESQ_ALLOC_WRPTR_MASK 0xFFFF
  37. #define MV_XOR_V2_DMA_DESQ_ALLOC_WRPTR_SHIFT 16
  38. #define MV_XOR_V2_DMA_IMSG_BALR_OFF 0x050
  39. #define MV_XOR_V2_DMA_IMSG_BAHR_OFF 0x054
  40. #define MV_XOR_V2_DMA_DESQ_CTRL_OFF 0x100
  41. #define MV_XOR_V2_DMA_DESQ_CTRL_32B 1
  42. #define MV_XOR_V2_DMA_DESQ_CTRL_128B 7
  43. #define MV_XOR_V2_DMA_DESQ_STOP_OFF 0x800
  44. #define MV_XOR_V2_DMA_DESQ_DEALLOC_OFF 0x804
  45. #define MV_XOR_V2_DMA_DESQ_ADD_OFF 0x808
  46. #define MV_XOR_V2_DMA_IMSG_TMOT 0x810
  47. #define MV_XOR_V2_DMA_IMSG_TIMER_THRD_MASK 0x1FFF
  48. /* XOR Global registers */
  49. #define MV_XOR_V2_GLOB_BW_CTRL 0x4
  50. #define MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_RD_SHIFT 0
  51. #define MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_RD_VAL 64
  52. #define MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_WR_SHIFT 8
  53. #define MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_WR_VAL 8
  54. #define MV_XOR_V2_GLOB_BW_CTRL_RD_BURST_LEN_SHIFT 12
  55. #define MV_XOR_V2_GLOB_BW_CTRL_RD_BURST_LEN_VAL 4
  56. #define MV_XOR_V2_GLOB_BW_CTRL_WR_BURST_LEN_SHIFT 16
  57. #define MV_XOR_V2_GLOB_BW_CTRL_WR_BURST_LEN_VAL 4
  58. #define MV_XOR_V2_GLOB_PAUSE 0x014
  59. #define MV_XOR_V2_GLOB_PAUSE_AXI_TIME_DIS_VAL 0x8
  60. #define MV_XOR_V2_GLOB_SYS_INT_CAUSE 0x200
  61. #define MV_XOR_V2_GLOB_SYS_INT_MASK 0x204
  62. #define MV_XOR_V2_GLOB_MEM_INT_CAUSE 0x220
  63. #define MV_XOR_V2_GLOB_MEM_INT_MASK 0x224
  64. #define MV_XOR_V2_MIN_DESC_SIZE 32
  65. #define MV_XOR_V2_EXT_DESC_SIZE 128
  66. #define MV_XOR_V2_DESC_RESERVED_SIZE 12
  67. #define MV_XOR_V2_DESC_BUFF_D_ADDR_SIZE 12
  68. #define MV_XOR_V2_CMD_LINE_NUM_MAX_D_BUF 8
  69. /*
  70. * Descriptors queue size. With 32 bytes descriptors, up to 2^14
  71. * descriptors are allowed, with 128 bytes descriptors, up to 2^12
  72. * descriptors are allowed. This driver uses 128 bytes descriptors,
  73. * but experimentation has shown that a set of 1024 descriptors is
  74. * sufficient to reach a good level of performance.
  75. */
  76. #define MV_XOR_V2_DESC_NUM 1024
  77. /*
  78. * Threshold values for descriptors and timeout, determined by
  79. * experimentation as giving a good level of performance.
  80. */
  81. #define MV_XOR_V2_DONE_IMSG_THRD 0x14
  82. #define MV_XOR_V2_TIMER_THRD 0xB0
  83. /**
  84. * struct mv_xor_v2_descriptor - DMA HW descriptor
  85. * @desc_id: used by S/W and is not affected by H/W.
  86. * @flags: error and status flags
  87. * @crc32_result: CRC32 calculation result
  88. * @desc_ctrl: operation mode and control flags
  89. * @buff_size: amount of bytes to be processed
  90. * @fill_pattern_src_addr: Fill-Pattern or Source-Address and
  91. * AW-Attributes
  92. * @data_buff_addr: Source (and might be RAID6 destination)
  93. * addresses of data buffers in RAID5 and RAID6
  94. * @reserved: reserved
  95. */
  96. struct mv_xor_v2_descriptor {
  97. u16 desc_id;
  98. u16 flags;
  99. u32 crc32_result;
  100. u32 desc_ctrl;
  101. /* Definitions for desc_ctrl */
  102. #define DESC_NUM_ACTIVE_D_BUF_SHIFT 22
  103. #define DESC_OP_MODE_SHIFT 28
  104. #define DESC_OP_MODE_NOP 0 /* Idle operation */
  105. #define DESC_OP_MODE_MEMCPY 1 /* Pure-DMA operation */
  106. #define DESC_OP_MODE_MEMSET 2 /* Mem-Fill operation */
  107. #define DESC_OP_MODE_MEMINIT 3 /* Mem-Init operation */
  108. #define DESC_OP_MODE_MEM_COMPARE 4 /* Mem-Compare operation */
  109. #define DESC_OP_MODE_CRC32 5 /* CRC32 calculation */
  110. #define DESC_OP_MODE_XOR 6 /* RAID5 (XOR) operation */
  111. #define DESC_OP_MODE_RAID6 7 /* RAID6 P&Q-generation */
  112. #define DESC_OP_MODE_RAID6_REC 8 /* RAID6 Recovery */
  113. #define DESC_Q_BUFFER_ENABLE BIT(16)
  114. #define DESC_P_BUFFER_ENABLE BIT(17)
  115. #define DESC_IOD BIT(27)
  116. u32 buff_size;
  117. u32 fill_pattern_src_addr[4];
  118. u32 data_buff_addr[MV_XOR_V2_DESC_BUFF_D_ADDR_SIZE];
  119. u32 reserved[MV_XOR_V2_DESC_RESERVED_SIZE];
  120. };
  121. /**
  122. * struct mv_xor_v2_device - implements a xor device
  123. * @lock: lock for the engine
  124. * @clk: reference to the 'core' clock
  125. * @reg_clk: reference to the 'reg' clock
  126. * @dma_base: memory mapped DMA register base
  127. * @glob_base: memory mapped global register base
  128. * @irq_tasklet: tasklet used for IRQ handling call-backs
  129. * @free_sw_desc: linked list of free SW descriptors
  130. * @dmadev: dma device
  131. * @dmachan: dma channel
  132. * @hw_desq: HW descriptors queue
  133. * @hw_desq_virt: virtual address of DESCQ
  134. * @sw_desq: SW descriptors queue
  135. * @desc_size: HW descriptor size
  136. * @npendings: number of pending descriptors (for which tx_submit has
  137. * @hw_queue_idx: HW queue index
  138. * @msi_desc: local interrupt descriptor information
  139. * been called, but not yet issue_pending)
  140. */
  141. struct mv_xor_v2_device {
  142. spinlock_t lock;
  143. void __iomem *dma_base;
  144. void __iomem *glob_base;
  145. struct clk *clk;
  146. struct clk *reg_clk;
  147. struct tasklet_struct irq_tasklet;
  148. struct list_head free_sw_desc;
  149. struct dma_device dmadev;
  150. struct dma_chan dmachan;
  151. dma_addr_t hw_desq;
  152. struct mv_xor_v2_descriptor *hw_desq_virt;
  153. struct mv_xor_v2_sw_desc *sw_desq;
  154. int desc_size;
  155. unsigned int npendings;
  156. unsigned int hw_queue_idx;
  157. struct msi_desc *msi_desc;
  158. };
  159. /**
  160. * struct mv_xor_v2_sw_desc - implements a xor SW descriptor
  161. * @idx: descriptor index
  162. * @async_tx: support for the async_tx api
  163. * @hw_desc: assosiated HW descriptor
  164. * @free_list: node of the free SW descriprots list
  165. */
  166. struct mv_xor_v2_sw_desc {
  167. int idx;
  168. struct dma_async_tx_descriptor async_tx;
  169. struct mv_xor_v2_descriptor hw_desc;
  170. struct list_head free_list;
  171. };
  172. /*
  173. * Fill the data buffers to a HW descriptor
  174. */
  175. static void mv_xor_v2_set_data_buffers(struct mv_xor_v2_device *xor_dev,
  176. struct mv_xor_v2_descriptor *desc,
  177. dma_addr_t src, int index)
  178. {
  179. int arr_index = ((index >> 1) * 3);
  180. /*
  181. * Fill the buffer's addresses to the descriptor.
  182. *
  183. * The format of the buffers address for 2 sequential buffers
  184. * X and X + 1:
  185. *
  186. * First word: Buffer-DX-Address-Low[31:0]
  187. * Second word: Buffer-DX+1-Address-Low[31:0]
  188. * Third word: DX+1-Buffer-Address-High[47:32] [31:16]
  189. * DX-Buffer-Address-High[47:32] [15:0]
  190. */
  191. if ((index & 0x1) == 0) {
  192. desc->data_buff_addr[arr_index] = lower_32_bits(src);
  193. desc->data_buff_addr[arr_index + 2] &= ~0xFFFF;
  194. desc->data_buff_addr[arr_index + 2] |=
  195. upper_32_bits(src) & 0xFFFF;
  196. } else {
  197. desc->data_buff_addr[arr_index + 1] =
  198. lower_32_bits(src);
  199. desc->data_buff_addr[arr_index + 2] &= ~0xFFFF0000;
  200. desc->data_buff_addr[arr_index + 2] |=
  201. (upper_32_bits(src) & 0xFFFF) << 16;
  202. }
  203. }
  204. /*
  205. * notify the engine of new descriptors, and update the available index.
  206. */
  207. static void mv_xor_v2_add_desc_to_desq(struct mv_xor_v2_device *xor_dev,
  208. int num_of_desc)
  209. {
  210. /* write the number of new descriptors in the DESQ. */
  211. writel(num_of_desc, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_ADD_OFF);
  212. }
  213. /*
  214. * free HW descriptors
  215. */
  216. static void mv_xor_v2_free_desc_from_desq(struct mv_xor_v2_device *xor_dev,
  217. int num_of_desc)
  218. {
  219. /* write the number of new descriptors in the DESQ. */
  220. writel(num_of_desc, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_DEALLOC_OFF);
  221. }
  222. /*
  223. * Set descriptor size
  224. * Return the HW descriptor size in bytes
  225. */
  226. static int mv_xor_v2_set_desc_size(struct mv_xor_v2_device *xor_dev)
  227. {
  228. writel(MV_XOR_V2_DMA_DESQ_CTRL_128B,
  229. xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_CTRL_OFF);
  230. return MV_XOR_V2_EXT_DESC_SIZE;
  231. }
  232. /*
  233. * Set the IMSG threshold
  234. */
  235. static inline
  236. void mv_xor_v2_enable_imsg_thrd(struct mv_xor_v2_device *xor_dev)
  237. {
  238. u32 reg;
  239. /* Configure threshold of number of descriptors, and enable timer */
  240. reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_THRD_OFF);
  241. reg &= ~MV_XOR_V2_DMA_IMSG_THRD_MASK;
  242. reg |= MV_XOR_V2_DONE_IMSG_THRD;
  243. reg |= MV_XOR_V2_DMA_IMSG_TIMER_EN;
  244. writel(reg, xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_THRD_OFF);
  245. /* Configure Timer Threshold */
  246. reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_TMOT);
  247. reg &= ~MV_XOR_V2_DMA_IMSG_TIMER_THRD_MASK;
  248. reg |= MV_XOR_V2_TIMER_THRD;
  249. writel(reg, xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_TMOT);
  250. }
  251. static irqreturn_t mv_xor_v2_interrupt_handler(int irq, void *data)
  252. {
  253. struct mv_xor_v2_device *xor_dev = data;
  254. unsigned int ndescs;
  255. u32 reg;
  256. reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_DONE_OFF);
  257. ndescs = ((reg >> MV_XOR_V2_DMA_DESQ_DONE_PENDING_SHIFT) &
  258. MV_XOR_V2_DMA_DESQ_DONE_PENDING_MASK);
  259. /* No descriptors to process */
  260. if (!ndescs)
  261. return IRQ_NONE;
  262. /* schedule a tasklet to handle descriptors callbacks */
  263. tasklet_schedule(&xor_dev->irq_tasklet);
  264. return IRQ_HANDLED;
  265. }
  266. /*
  267. * submit a descriptor to the DMA engine
  268. */
  269. static dma_cookie_t
  270. mv_xor_v2_tx_submit(struct dma_async_tx_descriptor *tx)
  271. {
  272. void *dest_hw_desc;
  273. dma_cookie_t cookie;
  274. struct mv_xor_v2_sw_desc *sw_desc =
  275. container_of(tx, struct mv_xor_v2_sw_desc, async_tx);
  276. struct mv_xor_v2_device *xor_dev =
  277. container_of(tx->chan, struct mv_xor_v2_device, dmachan);
  278. dev_dbg(xor_dev->dmadev.dev,
  279. "%s sw_desc %p: async_tx %p\n",
  280. __func__, sw_desc, &sw_desc->async_tx);
  281. /* assign coookie */
  282. spin_lock_bh(&xor_dev->lock);
  283. cookie = dma_cookie_assign(tx);
  284. /* copy the HW descriptor from the SW descriptor to the DESQ */
  285. dest_hw_desc = xor_dev->hw_desq_virt + xor_dev->hw_queue_idx;
  286. memcpy(dest_hw_desc, &sw_desc->hw_desc, xor_dev->desc_size);
  287. xor_dev->npendings++;
  288. xor_dev->hw_queue_idx++;
  289. if (xor_dev->hw_queue_idx >= MV_XOR_V2_DESC_NUM)
  290. xor_dev->hw_queue_idx = 0;
  291. spin_unlock_bh(&xor_dev->lock);
  292. return cookie;
  293. }
  294. /*
  295. * Prepare a SW descriptor
  296. */
  297. static struct mv_xor_v2_sw_desc *
  298. mv_xor_v2_prep_sw_desc(struct mv_xor_v2_device *xor_dev)
  299. {
  300. struct mv_xor_v2_sw_desc *sw_desc;
  301. bool found = false;
  302. /* Lock the channel */
  303. spin_lock_bh(&xor_dev->lock);
  304. if (list_empty(&xor_dev->free_sw_desc)) {
  305. spin_unlock_bh(&xor_dev->lock);
  306. /* schedule tasklet to free some descriptors */
  307. tasklet_schedule(&xor_dev->irq_tasklet);
  308. return NULL;
  309. }
  310. list_for_each_entry(sw_desc, &xor_dev->free_sw_desc, free_list) {
  311. if (async_tx_test_ack(&sw_desc->async_tx)) {
  312. found = true;
  313. break;
  314. }
  315. }
  316. if (!found) {
  317. spin_unlock_bh(&xor_dev->lock);
  318. return NULL;
  319. }
  320. list_del(&sw_desc->free_list);
  321. /* Release the channel */
  322. spin_unlock_bh(&xor_dev->lock);
  323. return sw_desc;
  324. }
  325. /*
  326. * Prepare a HW descriptor for a memcpy operation
  327. */
  328. static struct dma_async_tx_descriptor *
  329. mv_xor_v2_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest,
  330. dma_addr_t src, size_t len, unsigned long flags)
  331. {
  332. struct mv_xor_v2_sw_desc *sw_desc;
  333. struct mv_xor_v2_descriptor *hw_descriptor;
  334. struct mv_xor_v2_device *xor_dev;
  335. xor_dev = container_of(chan, struct mv_xor_v2_device, dmachan);
  336. dev_dbg(xor_dev->dmadev.dev,
  337. "%s len: %zu src %pad dest %pad flags: %ld\n",
  338. __func__, len, &src, &dest, flags);
  339. sw_desc = mv_xor_v2_prep_sw_desc(xor_dev);
  340. if (!sw_desc)
  341. return NULL;
  342. sw_desc->async_tx.flags = flags;
  343. /* set the HW descriptor */
  344. hw_descriptor = &sw_desc->hw_desc;
  345. /* save the SW descriptor ID to restore when operation is done */
  346. hw_descriptor->desc_id = sw_desc->idx;
  347. /* Set the MEMCPY control word */
  348. hw_descriptor->desc_ctrl =
  349. DESC_OP_MODE_MEMCPY << DESC_OP_MODE_SHIFT;
  350. if (flags & DMA_PREP_INTERRUPT)
  351. hw_descriptor->desc_ctrl |= DESC_IOD;
  352. /* Set source address */
  353. hw_descriptor->fill_pattern_src_addr[0] = lower_32_bits(src);
  354. hw_descriptor->fill_pattern_src_addr[1] =
  355. upper_32_bits(src) & 0xFFFF;
  356. /* Set Destination address */
  357. hw_descriptor->fill_pattern_src_addr[2] = lower_32_bits(dest);
  358. hw_descriptor->fill_pattern_src_addr[3] =
  359. upper_32_bits(dest) & 0xFFFF;
  360. /* Set buffers size */
  361. hw_descriptor->buff_size = len;
  362. /* return the async tx descriptor */
  363. return &sw_desc->async_tx;
  364. }
  365. /*
  366. * Prepare a HW descriptor for a XOR operation
  367. */
  368. static struct dma_async_tx_descriptor *
  369. mv_xor_v2_prep_dma_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
  370. unsigned int src_cnt, size_t len, unsigned long flags)
  371. {
  372. struct mv_xor_v2_sw_desc *sw_desc;
  373. struct mv_xor_v2_descriptor *hw_descriptor;
  374. struct mv_xor_v2_device *xor_dev =
  375. container_of(chan, struct mv_xor_v2_device, dmachan);
  376. int i;
  377. if (src_cnt > MV_XOR_V2_CMD_LINE_NUM_MAX_D_BUF || src_cnt < 1)
  378. return NULL;
  379. dev_dbg(xor_dev->dmadev.dev,
  380. "%s src_cnt: %d len: %zu dest %pad flags: %ld\n",
  381. __func__, src_cnt, len, &dest, flags);
  382. sw_desc = mv_xor_v2_prep_sw_desc(xor_dev);
  383. if (!sw_desc)
  384. return NULL;
  385. sw_desc->async_tx.flags = flags;
  386. /* set the HW descriptor */
  387. hw_descriptor = &sw_desc->hw_desc;
  388. /* save the SW descriptor ID to restore when operation is done */
  389. hw_descriptor->desc_id = sw_desc->idx;
  390. /* Set the XOR control word */
  391. hw_descriptor->desc_ctrl =
  392. DESC_OP_MODE_XOR << DESC_OP_MODE_SHIFT;
  393. hw_descriptor->desc_ctrl |= DESC_P_BUFFER_ENABLE;
  394. if (flags & DMA_PREP_INTERRUPT)
  395. hw_descriptor->desc_ctrl |= DESC_IOD;
  396. /* Set the data buffers */
  397. for (i = 0; i < src_cnt; i++)
  398. mv_xor_v2_set_data_buffers(xor_dev, hw_descriptor, src[i], i);
  399. hw_descriptor->desc_ctrl |=
  400. src_cnt << DESC_NUM_ACTIVE_D_BUF_SHIFT;
  401. /* Set Destination address */
  402. hw_descriptor->fill_pattern_src_addr[2] = lower_32_bits(dest);
  403. hw_descriptor->fill_pattern_src_addr[3] =
  404. upper_32_bits(dest) & 0xFFFF;
  405. /* Set buffers size */
  406. hw_descriptor->buff_size = len;
  407. /* return the async tx descriptor */
  408. return &sw_desc->async_tx;
  409. }
  410. /*
  411. * Prepare a HW descriptor for interrupt operation.
  412. */
  413. static struct dma_async_tx_descriptor *
  414. mv_xor_v2_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags)
  415. {
  416. struct mv_xor_v2_sw_desc *sw_desc;
  417. struct mv_xor_v2_descriptor *hw_descriptor;
  418. struct mv_xor_v2_device *xor_dev =
  419. container_of(chan, struct mv_xor_v2_device, dmachan);
  420. sw_desc = mv_xor_v2_prep_sw_desc(xor_dev);
  421. if (!sw_desc)
  422. return NULL;
  423. /* set the HW descriptor */
  424. hw_descriptor = &sw_desc->hw_desc;
  425. /* save the SW descriptor ID to restore when operation is done */
  426. hw_descriptor->desc_id = sw_desc->idx;
  427. /* Set the INTERRUPT control word */
  428. hw_descriptor->desc_ctrl =
  429. DESC_OP_MODE_NOP << DESC_OP_MODE_SHIFT;
  430. hw_descriptor->desc_ctrl |= DESC_IOD;
  431. /* return the async tx descriptor */
  432. return &sw_desc->async_tx;
  433. }
  434. /*
  435. * push pending transactions to hardware
  436. */
  437. static void mv_xor_v2_issue_pending(struct dma_chan *chan)
  438. {
  439. struct mv_xor_v2_device *xor_dev =
  440. container_of(chan, struct mv_xor_v2_device, dmachan);
  441. spin_lock_bh(&xor_dev->lock);
  442. /*
  443. * update the engine with the number of descriptors to
  444. * process
  445. */
  446. mv_xor_v2_add_desc_to_desq(xor_dev, xor_dev->npendings);
  447. xor_dev->npendings = 0;
  448. spin_unlock_bh(&xor_dev->lock);
  449. }
  450. static inline
  451. int mv_xor_v2_get_pending_params(struct mv_xor_v2_device *xor_dev,
  452. int *pending_ptr)
  453. {
  454. u32 reg;
  455. reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_DONE_OFF);
  456. /* get the next pending descriptor index */
  457. *pending_ptr = ((reg >> MV_XOR_V2_DMA_DESQ_DONE_READ_PTR_SHIFT) &
  458. MV_XOR_V2_DMA_DESQ_DONE_READ_PTR_MASK);
  459. /* get the number of descriptors pending handle */
  460. return ((reg >> MV_XOR_V2_DMA_DESQ_DONE_PENDING_SHIFT) &
  461. MV_XOR_V2_DMA_DESQ_DONE_PENDING_MASK);
  462. }
  463. /*
  464. * handle the descriptors after HW process
  465. */
  466. static void mv_xor_v2_tasklet(struct tasklet_struct *t)
  467. {
  468. struct mv_xor_v2_device *xor_dev = from_tasklet(xor_dev, t,
  469. irq_tasklet);
  470. int pending_ptr, num_of_pending, i;
  471. struct mv_xor_v2_sw_desc *next_pending_sw_desc = NULL;
  472. dev_dbg(xor_dev->dmadev.dev, "%s %d\n", __func__, __LINE__);
  473. /* get the pending descriptors parameters */
  474. num_of_pending = mv_xor_v2_get_pending_params(xor_dev, &pending_ptr);
  475. /* loop over free descriptors */
  476. for (i = 0; i < num_of_pending; i++) {
  477. struct mv_xor_v2_descriptor *next_pending_hw_desc =
  478. xor_dev->hw_desq_virt + pending_ptr;
  479. /* get the SW descriptor related to the HW descriptor */
  480. next_pending_sw_desc =
  481. &xor_dev->sw_desq[next_pending_hw_desc->desc_id];
  482. /* call the callback */
  483. if (next_pending_sw_desc->async_tx.cookie > 0) {
  484. /*
  485. * update the channel's completed cookie - no
  486. * lock is required the IMSG threshold provide
  487. * the locking
  488. */
  489. dma_cookie_complete(&next_pending_sw_desc->async_tx);
  490. dma_descriptor_unmap(&next_pending_sw_desc->async_tx);
  491. dmaengine_desc_get_callback_invoke(
  492. &next_pending_sw_desc->async_tx, NULL);
  493. }
  494. dma_run_dependencies(&next_pending_sw_desc->async_tx);
  495. /* Lock the channel */
  496. spin_lock_bh(&xor_dev->lock);
  497. /* add the SW descriptor to the free descriptors list */
  498. list_add(&next_pending_sw_desc->free_list,
  499. &xor_dev->free_sw_desc);
  500. /* Release the channel */
  501. spin_unlock_bh(&xor_dev->lock);
  502. /* increment the next descriptor */
  503. pending_ptr++;
  504. if (pending_ptr >= MV_XOR_V2_DESC_NUM)
  505. pending_ptr = 0;
  506. }
  507. if (num_of_pending != 0) {
  508. /* free the descriptores */
  509. mv_xor_v2_free_desc_from_desq(xor_dev, num_of_pending);
  510. }
  511. }
  512. /*
  513. * Set DMA Interrupt-message (IMSG) parameters
  514. */
  515. static void mv_xor_v2_set_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
  516. {
  517. struct mv_xor_v2_device *xor_dev = dev_get_drvdata(desc->dev);
  518. writel(msg->address_lo,
  519. xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_BALR_OFF);
  520. writel(msg->address_hi & 0xFFFF,
  521. xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_BAHR_OFF);
  522. writel(msg->data,
  523. xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_CDAT_OFF);
  524. }
  525. static int mv_xor_v2_descq_init(struct mv_xor_v2_device *xor_dev)
  526. {
  527. u32 reg;
  528. /* write the DESQ size to the DMA engine */
  529. writel(MV_XOR_V2_DESC_NUM,
  530. xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_SIZE_OFF);
  531. /* write the DESQ address to the DMA enngine*/
  532. writel(lower_32_bits(xor_dev->hw_desq),
  533. xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_BALR_OFF);
  534. writel(upper_32_bits(xor_dev->hw_desq),
  535. xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_BAHR_OFF);
  536. /*
  537. * This is a temporary solution, until we activate the
  538. * SMMU. Set the attributes for reading & writing data buffers
  539. * & descriptors to:
  540. *
  541. * - OuterShareable - Snoops will be performed on CPU caches
  542. * - Enable cacheable - Bufferable, Modifiable, Other Allocate
  543. * and Allocate
  544. */
  545. reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_ARATTR_OFF);
  546. reg &= ~MV_XOR_V2_DMA_DESQ_ATTR_CACHE_MASK;
  547. reg |= MV_XOR_V2_DMA_DESQ_ATTR_OUTER_SHAREABLE |
  548. MV_XOR_V2_DMA_DESQ_ATTR_CACHEABLE;
  549. writel(reg, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_ARATTR_OFF);
  550. reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_AWATTR_OFF);
  551. reg &= ~MV_XOR_V2_DMA_DESQ_ATTR_CACHE_MASK;
  552. reg |= MV_XOR_V2_DMA_DESQ_ATTR_OUTER_SHAREABLE |
  553. MV_XOR_V2_DMA_DESQ_ATTR_CACHEABLE;
  554. writel(reg, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_AWATTR_OFF);
  555. /* BW CTRL - set values to optimize the XOR performance:
  556. *
  557. * - Set WrBurstLen & RdBurstLen - the unit will issue
  558. * maximum of 256B write/read transactions.
  559. * - Limit the number of outstanding write & read data
  560. * (OBB/IBB) requests to the maximal value.
  561. */
  562. reg = ((MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_RD_VAL <<
  563. MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_RD_SHIFT) |
  564. (MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_WR_VAL <<
  565. MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_WR_SHIFT) |
  566. (MV_XOR_V2_GLOB_BW_CTRL_RD_BURST_LEN_VAL <<
  567. MV_XOR_V2_GLOB_BW_CTRL_RD_BURST_LEN_SHIFT) |
  568. (MV_XOR_V2_GLOB_BW_CTRL_WR_BURST_LEN_VAL <<
  569. MV_XOR_V2_GLOB_BW_CTRL_WR_BURST_LEN_SHIFT));
  570. writel(reg, xor_dev->glob_base + MV_XOR_V2_GLOB_BW_CTRL);
  571. /* Disable the AXI timer feature */
  572. reg = readl(xor_dev->glob_base + MV_XOR_V2_GLOB_PAUSE);
  573. reg |= MV_XOR_V2_GLOB_PAUSE_AXI_TIME_DIS_VAL;
  574. writel(reg, xor_dev->glob_base + MV_XOR_V2_GLOB_PAUSE);
  575. /* enable the DMA engine */
  576. writel(0, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_STOP_OFF);
  577. return 0;
  578. }
  579. static int mv_xor_v2_suspend(struct platform_device *dev, pm_message_t state)
  580. {
  581. struct mv_xor_v2_device *xor_dev = platform_get_drvdata(dev);
  582. /* Set this bit to disable to stop the XOR unit. */
  583. writel(0x1, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_STOP_OFF);
  584. return 0;
  585. }
  586. static int mv_xor_v2_resume(struct platform_device *dev)
  587. {
  588. struct mv_xor_v2_device *xor_dev = platform_get_drvdata(dev);
  589. mv_xor_v2_set_desc_size(xor_dev);
  590. mv_xor_v2_enable_imsg_thrd(xor_dev);
  591. mv_xor_v2_descq_init(xor_dev);
  592. return 0;
  593. }
  594. static int mv_xor_v2_probe(struct platform_device *pdev)
  595. {
  596. struct mv_xor_v2_device *xor_dev;
  597. struct resource *res;
  598. int i, ret = 0;
  599. struct dma_device *dma_dev;
  600. struct mv_xor_v2_sw_desc *sw_desc;
  601. struct msi_desc *msi_desc;
  602. BUILD_BUG_ON(sizeof(struct mv_xor_v2_descriptor) !=
  603. MV_XOR_V2_EXT_DESC_SIZE);
  604. xor_dev = devm_kzalloc(&pdev->dev, sizeof(*xor_dev), GFP_KERNEL);
  605. if (!xor_dev)
  606. return -ENOMEM;
  607. res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
  608. xor_dev->dma_base = devm_ioremap_resource(&pdev->dev, res);
  609. if (IS_ERR(xor_dev->dma_base))
  610. return PTR_ERR(xor_dev->dma_base);
  611. res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
  612. xor_dev->glob_base = devm_ioremap_resource(&pdev->dev, res);
  613. if (IS_ERR(xor_dev->glob_base))
  614. return PTR_ERR(xor_dev->glob_base);
  615. platform_set_drvdata(pdev, xor_dev);
  616. ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40));
  617. if (ret)
  618. return ret;
  619. xor_dev->reg_clk = devm_clk_get(&pdev->dev, "reg");
  620. if (PTR_ERR(xor_dev->reg_clk) != -ENOENT) {
  621. if (!IS_ERR(xor_dev->reg_clk)) {
  622. ret = clk_prepare_enable(xor_dev->reg_clk);
  623. if (ret)
  624. return ret;
  625. } else {
  626. return PTR_ERR(xor_dev->reg_clk);
  627. }
  628. }
  629. xor_dev->clk = devm_clk_get(&pdev->dev, NULL);
  630. if (PTR_ERR(xor_dev->clk) == -EPROBE_DEFER) {
  631. ret = EPROBE_DEFER;
  632. goto disable_reg_clk;
  633. }
  634. if (!IS_ERR(xor_dev->clk)) {
  635. ret = clk_prepare_enable(xor_dev->clk);
  636. if (ret)
  637. goto disable_reg_clk;
  638. }
  639. ret = platform_msi_domain_alloc_irqs(&pdev->dev, 1,
  640. mv_xor_v2_set_msi_msg);
  641. if (ret)
  642. goto disable_clk;
  643. msi_desc = first_msi_entry(&pdev->dev);
  644. if (!msi_desc) {
  645. ret = -ENODEV;
  646. goto free_msi_irqs;
  647. }
  648. xor_dev->msi_desc = msi_desc;
  649. ret = devm_request_irq(&pdev->dev, msi_desc->irq,
  650. mv_xor_v2_interrupt_handler, 0,
  651. dev_name(&pdev->dev), xor_dev);
  652. if (ret)
  653. goto free_msi_irqs;
  654. tasklet_setup(&xor_dev->irq_tasklet, mv_xor_v2_tasklet);
  655. xor_dev->desc_size = mv_xor_v2_set_desc_size(xor_dev);
  656. dma_cookie_init(&xor_dev->dmachan);
  657. /*
  658. * allocate coherent memory for hardware descriptors
  659. * note: writecombine gives slightly better performance, but
  660. * requires that we explicitly flush the writes
  661. */
  662. xor_dev->hw_desq_virt =
  663. dma_alloc_coherent(&pdev->dev,
  664. xor_dev->desc_size * MV_XOR_V2_DESC_NUM,
  665. &xor_dev->hw_desq, GFP_KERNEL);
  666. if (!xor_dev->hw_desq_virt) {
  667. ret = -ENOMEM;
  668. goto free_msi_irqs;
  669. }
  670. /* alloc memory for the SW descriptors */
  671. xor_dev->sw_desq = devm_kcalloc(&pdev->dev,
  672. MV_XOR_V2_DESC_NUM, sizeof(*sw_desc),
  673. GFP_KERNEL);
  674. if (!xor_dev->sw_desq) {
  675. ret = -ENOMEM;
  676. goto free_hw_desq;
  677. }
  678. spin_lock_init(&xor_dev->lock);
  679. /* init the free SW descriptors list */
  680. INIT_LIST_HEAD(&xor_dev->free_sw_desc);
  681. /* add all SW descriptors to the free list */
  682. for (i = 0; i < MV_XOR_V2_DESC_NUM; i++) {
  683. struct mv_xor_v2_sw_desc *sw_desc =
  684. xor_dev->sw_desq + i;
  685. sw_desc->idx = i;
  686. dma_async_tx_descriptor_init(&sw_desc->async_tx,
  687. &xor_dev->dmachan);
  688. sw_desc->async_tx.tx_submit = mv_xor_v2_tx_submit;
  689. async_tx_ack(&sw_desc->async_tx);
  690. list_add(&sw_desc->free_list,
  691. &xor_dev->free_sw_desc);
  692. }
  693. dma_dev = &xor_dev->dmadev;
  694. /* set DMA capabilities */
  695. dma_cap_zero(dma_dev->cap_mask);
  696. dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
  697. dma_cap_set(DMA_XOR, dma_dev->cap_mask);
  698. dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask);
  699. /* init dma link list */
  700. INIT_LIST_HEAD(&dma_dev->channels);
  701. /* set base routines */
  702. dma_dev->device_tx_status = dma_cookie_status;
  703. dma_dev->device_issue_pending = mv_xor_v2_issue_pending;
  704. dma_dev->dev = &pdev->dev;
  705. dma_dev->device_prep_dma_memcpy = mv_xor_v2_prep_dma_memcpy;
  706. dma_dev->device_prep_dma_interrupt = mv_xor_v2_prep_dma_interrupt;
  707. dma_dev->max_xor = 8;
  708. dma_dev->device_prep_dma_xor = mv_xor_v2_prep_dma_xor;
  709. xor_dev->dmachan.device = dma_dev;
  710. list_add_tail(&xor_dev->dmachan.device_node,
  711. &dma_dev->channels);
  712. mv_xor_v2_enable_imsg_thrd(xor_dev);
  713. mv_xor_v2_descq_init(xor_dev);
  714. ret = dma_async_device_register(dma_dev);
  715. if (ret)
  716. goto free_hw_desq;
  717. dev_notice(&pdev->dev, "Marvell Version 2 XOR driver\n");
  718. return 0;
  719. free_hw_desq:
  720. dma_free_coherent(&pdev->dev,
  721. xor_dev->desc_size * MV_XOR_V2_DESC_NUM,
  722. xor_dev->hw_desq_virt, xor_dev->hw_desq);
  723. free_msi_irqs:
  724. platform_msi_domain_free_irqs(&pdev->dev);
  725. disable_clk:
  726. clk_disable_unprepare(xor_dev->clk);
  727. disable_reg_clk:
  728. clk_disable_unprepare(xor_dev->reg_clk);
  729. return ret;
  730. }
  731. static int mv_xor_v2_remove(struct platform_device *pdev)
  732. {
  733. struct mv_xor_v2_device *xor_dev = platform_get_drvdata(pdev);
  734. dma_async_device_unregister(&xor_dev->dmadev);
  735. dma_free_coherent(&pdev->dev,
  736. xor_dev->desc_size * MV_XOR_V2_DESC_NUM,
  737. xor_dev->hw_desq_virt, xor_dev->hw_desq);
  738. devm_free_irq(&pdev->dev, xor_dev->msi_desc->irq, xor_dev);
  739. platform_msi_domain_free_irqs(&pdev->dev);
  740. tasklet_kill(&xor_dev->irq_tasklet);
  741. clk_disable_unprepare(xor_dev->clk);
  742. return 0;
  743. }
  744. #ifdef CONFIG_OF
  745. static const struct of_device_id mv_xor_v2_dt_ids[] = {
  746. { .compatible = "marvell,xor-v2", },
  747. {},
  748. };
  749. MODULE_DEVICE_TABLE(of, mv_xor_v2_dt_ids);
  750. #endif
  751. static struct platform_driver mv_xor_v2_driver = {
  752. .probe = mv_xor_v2_probe,
  753. .suspend = mv_xor_v2_suspend,
  754. .resume = mv_xor_v2_resume,
  755. .remove = mv_xor_v2_remove,
  756. .driver = {
  757. .name = "mv_xor_v2",
  758. .of_match_table = of_match_ptr(mv_xor_v2_dt_ids),
  759. },
  760. };
  761. module_platform_driver(mv_xor_v2_driver);
  762. MODULE_DESCRIPTION("DMA engine driver for Marvell's Version 2 of XOR engine");
  763. MODULE_LICENSE("GPL");