hidma_ll.c 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Qualcomm Technologies HIDMA DMA engine low level code
  4. *
  5. * Copyright (c) 2015-2016, The Linux Foundation. All rights reserved.
  6. */
  7. #include <linux/dmaengine.h>
  8. #include <linux/slab.h>
  9. #include <linux/interrupt.h>
  10. #include <linux/mm.h>
  11. #include <linux/highmem.h>
  12. #include <linux/dma-mapping.h>
  13. #include <linux/delay.h>
  14. #include <linux/atomic.h>
  15. #include <linux/iopoll.h>
  16. #include <linux/kfifo.h>
  17. #include <linux/bitops.h>
  18. #include "hidma.h"
  19. #define HIDMA_EVRE_SIZE 16 /* each EVRE is 16 bytes */
  20. #define HIDMA_TRCA_CTRLSTS_REG 0x000
  21. #define HIDMA_TRCA_RING_LOW_REG 0x008
  22. #define HIDMA_TRCA_RING_HIGH_REG 0x00C
  23. #define HIDMA_TRCA_RING_LEN_REG 0x010
  24. #define HIDMA_TRCA_DOORBELL_REG 0x400
  25. #define HIDMA_EVCA_CTRLSTS_REG 0x000
  26. #define HIDMA_EVCA_INTCTRL_REG 0x004
  27. #define HIDMA_EVCA_RING_LOW_REG 0x008
  28. #define HIDMA_EVCA_RING_HIGH_REG 0x00C
  29. #define HIDMA_EVCA_RING_LEN_REG 0x010
  30. #define HIDMA_EVCA_WRITE_PTR_REG 0x020
  31. #define HIDMA_EVCA_DOORBELL_REG 0x400
  32. #define HIDMA_EVCA_IRQ_STAT_REG 0x100
  33. #define HIDMA_EVCA_IRQ_CLR_REG 0x108
  34. #define HIDMA_EVCA_IRQ_EN_REG 0x110
  35. #define HIDMA_EVRE_CFG_IDX 0
  36. #define HIDMA_EVRE_ERRINFO_BIT_POS 24
  37. #define HIDMA_EVRE_CODE_BIT_POS 28
  38. #define HIDMA_EVRE_ERRINFO_MASK GENMASK(3, 0)
  39. #define HIDMA_EVRE_CODE_MASK GENMASK(3, 0)
  40. #define HIDMA_CH_CONTROL_MASK GENMASK(7, 0)
  41. #define HIDMA_CH_STATE_MASK GENMASK(7, 0)
  42. #define HIDMA_CH_STATE_BIT_POS 0x8
  43. #define HIDMA_IRQ_EV_CH_EOB_IRQ_BIT_POS 0
  44. #define HIDMA_IRQ_EV_CH_WR_RESP_BIT_POS 1
  45. #define HIDMA_IRQ_TR_CH_TRE_RD_RSP_ER_BIT_POS 9
  46. #define HIDMA_IRQ_TR_CH_DATA_RD_ER_BIT_POS 10
  47. #define HIDMA_IRQ_TR_CH_DATA_WR_ER_BIT_POS 11
  48. #define HIDMA_IRQ_TR_CH_INVALID_TRE_BIT_POS 14
  49. #define ENABLE_IRQS (BIT(HIDMA_IRQ_EV_CH_EOB_IRQ_BIT_POS) | \
  50. BIT(HIDMA_IRQ_EV_CH_WR_RESP_BIT_POS) | \
  51. BIT(HIDMA_IRQ_TR_CH_TRE_RD_RSP_ER_BIT_POS) | \
  52. BIT(HIDMA_IRQ_TR_CH_DATA_RD_ER_BIT_POS) | \
  53. BIT(HIDMA_IRQ_TR_CH_DATA_WR_ER_BIT_POS) | \
  54. BIT(HIDMA_IRQ_TR_CH_INVALID_TRE_BIT_POS))
  55. #define HIDMA_INCREMENT_ITERATOR(iter, size, ring_size) \
  56. do { \
  57. iter += size; \
  58. if (iter >= ring_size) \
  59. iter -= ring_size; \
  60. } while (0)
  61. #define HIDMA_CH_STATE(val) \
  62. ((val >> HIDMA_CH_STATE_BIT_POS) & HIDMA_CH_STATE_MASK)
  63. #define HIDMA_ERR_INT_MASK \
  64. (BIT(HIDMA_IRQ_TR_CH_INVALID_TRE_BIT_POS) | \
  65. BIT(HIDMA_IRQ_TR_CH_TRE_RD_RSP_ER_BIT_POS) | \
  66. BIT(HIDMA_IRQ_EV_CH_WR_RESP_BIT_POS) | \
  67. BIT(HIDMA_IRQ_TR_CH_DATA_RD_ER_BIT_POS) | \
  68. BIT(HIDMA_IRQ_TR_CH_DATA_WR_ER_BIT_POS))
  69. enum ch_command {
  70. HIDMA_CH_DISABLE = 0,
  71. HIDMA_CH_ENABLE = 1,
  72. HIDMA_CH_SUSPEND = 2,
  73. HIDMA_CH_RESET = 9,
  74. };
  75. enum ch_state {
  76. HIDMA_CH_DISABLED = 0,
  77. HIDMA_CH_ENABLED = 1,
  78. HIDMA_CH_RUNNING = 2,
  79. HIDMA_CH_SUSPENDED = 3,
  80. HIDMA_CH_STOPPED = 4,
  81. };
  82. enum err_code {
  83. HIDMA_EVRE_STATUS_COMPLETE = 1,
  84. HIDMA_EVRE_STATUS_ERROR = 4,
  85. };
  86. static int hidma_is_chan_enabled(int state)
  87. {
  88. switch (state) {
  89. case HIDMA_CH_ENABLED:
  90. case HIDMA_CH_RUNNING:
  91. return true;
  92. default:
  93. return false;
  94. }
  95. }
  96. void hidma_ll_free(struct hidma_lldev *lldev, u32 tre_ch)
  97. {
  98. struct hidma_tre *tre;
  99. if (tre_ch >= lldev->nr_tres) {
  100. dev_err(lldev->dev, "invalid TRE number in free:%d", tre_ch);
  101. return;
  102. }
  103. tre = &lldev->trepool[tre_ch];
  104. if (atomic_read(&tre->allocated) != true) {
  105. dev_err(lldev->dev, "trying to free an unused TRE:%d", tre_ch);
  106. return;
  107. }
  108. atomic_set(&tre->allocated, 0);
  109. }
  110. int hidma_ll_request(struct hidma_lldev *lldev, u32 sig, const char *dev_name,
  111. void (*callback)(void *data), void *data, u32 *tre_ch)
  112. {
  113. unsigned int i;
  114. struct hidma_tre *tre;
  115. u32 *tre_local;
  116. if (!tre_ch || !lldev)
  117. return -EINVAL;
  118. /* need to have at least one empty spot in the queue */
  119. for (i = 0; i < lldev->nr_tres - 1; i++) {
  120. if (atomic_add_unless(&lldev->trepool[i].allocated, 1, 1))
  121. break;
  122. }
  123. if (i == (lldev->nr_tres - 1))
  124. return -ENOMEM;
  125. tre = &lldev->trepool[i];
  126. tre->dma_sig = sig;
  127. tre->dev_name = dev_name;
  128. tre->callback = callback;
  129. tre->data = data;
  130. tre->idx = i;
  131. tre->status = 0;
  132. tre->queued = 0;
  133. tre->err_code = 0;
  134. tre->err_info = 0;
  135. tre->lldev = lldev;
  136. tre_local = &tre->tre_local[0];
  137. tre_local[HIDMA_TRE_CFG_IDX] = (lldev->chidx & 0xFF) << 8;
  138. tre_local[HIDMA_TRE_CFG_IDX] |= BIT(16); /* set IEOB */
  139. *tre_ch = i;
  140. if (callback)
  141. callback(data);
  142. return 0;
  143. }
  144. /*
  145. * Multiple TREs may be queued and waiting in the pending queue.
  146. */
  147. static void hidma_ll_tre_complete(struct tasklet_struct *t)
  148. {
  149. struct hidma_lldev *lldev = from_tasklet(lldev, t, task);
  150. struct hidma_tre *tre;
  151. while (kfifo_out(&lldev->handoff_fifo, &tre, 1)) {
  152. /* call the user if it has been read by the hardware */
  153. if (tre->callback)
  154. tre->callback(tre->data);
  155. }
  156. }
  157. static int hidma_post_completed(struct hidma_lldev *lldev, u8 err_info,
  158. u8 err_code)
  159. {
  160. struct hidma_tre *tre;
  161. unsigned long flags;
  162. u32 tre_iterator;
  163. spin_lock_irqsave(&lldev->lock, flags);
  164. tre_iterator = lldev->tre_processed_off;
  165. tre = lldev->pending_tre_list[tre_iterator / HIDMA_TRE_SIZE];
  166. if (!tre) {
  167. spin_unlock_irqrestore(&lldev->lock, flags);
  168. dev_warn(lldev->dev, "tre_index [%d] and tre out of sync\n",
  169. tre_iterator / HIDMA_TRE_SIZE);
  170. return -EINVAL;
  171. }
  172. lldev->pending_tre_list[tre->tre_index] = NULL;
  173. /*
  174. * Keep track of pending TREs that SW is expecting to receive
  175. * from HW. We got one now. Decrement our counter.
  176. */
  177. if (atomic_dec_return(&lldev->pending_tre_count) < 0) {
  178. dev_warn(lldev->dev, "tre count mismatch on completion");
  179. atomic_set(&lldev->pending_tre_count, 0);
  180. }
  181. HIDMA_INCREMENT_ITERATOR(tre_iterator, HIDMA_TRE_SIZE,
  182. lldev->tre_ring_size);
  183. lldev->tre_processed_off = tre_iterator;
  184. spin_unlock_irqrestore(&lldev->lock, flags);
  185. tre->err_info = err_info;
  186. tre->err_code = err_code;
  187. tre->queued = 0;
  188. kfifo_put(&lldev->handoff_fifo, tre);
  189. tasklet_schedule(&lldev->task);
  190. return 0;
  191. }
  192. /*
  193. * Called to handle the interrupt for the channel.
  194. * Return a positive number if TRE or EVRE were consumed on this run.
  195. * Return a positive number if there are pending TREs or EVREs.
  196. * Return 0 if there is nothing to consume or no pending TREs/EVREs found.
  197. */
  198. static int hidma_handle_tre_completion(struct hidma_lldev *lldev)
  199. {
  200. u32 evre_ring_size = lldev->evre_ring_size;
  201. u32 err_info, err_code, evre_write_off;
  202. u32 evre_iterator;
  203. u32 num_completed = 0;
  204. evre_write_off = readl_relaxed(lldev->evca + HIDMA_EVCA_WRITE_PTR_REG);
  205. evre_iterator = lldev->evre_processed_off;
  206. if ((evre_write_off > evre_ring_size) ||
  207. (evre_write_off % HIDMA_EVRE_SIZE)) {
  208. dev_err(lldev->dev, "HW reports invalid EVRE write offset\n");
  209. return 0;
  210. }
  211. /*
  212. * By the time control reaches here the number of EVREs and TREs
  213. * may not match. Only consume the ones that hardware told us.
  214. */
  215. while ((evre_iterator != evre_write_off)) {
  216. u32 *current_evre = lldev->evre_ring + evre_iterator;
  217. u32 cfg;
  218. cfg = current_evre[HIDMA_EVRE_CFG_IDX];
  219. err_info = cfg >> HIDMA_EVRE_ERRINFO_BIT_POS;
  220. err_info &= HIDMA_EVRE_ERRINFO_MASK;
  221. err_code =
  222. (cfg >> HIDMA_EVRE_CODE_BIT_POS) & HIDMA_EVRE_CODE_MASK;
  223. if (hidma_post_completed(lldev, err_info, err_code))
  224. break;
  225. HIDMA_INCREMENT_ITERATOR(evre_iterator, HIDMA_EVRE_SIZE,
  226. evre_ring_size);
  227. /*
  228. * Read the new event descriptor written by the HW.
  229. * As we are processing the delivered events, other events
  230. * get queued to the SW for processing.
  231. */
  232. evre_write_off =
  233. readl_relaxed(lldev->evca + HIDMA_EVCA_WRITE_PTR_REG);
  234. num_completed++;
  235. /*
  236. * An error interrupt might have arrived while we are processing
  237. * the completed interrupt.
  238. */
  239. if (!hidma_ll_isenabled(lldev))
  240. break;
  241. }
  242. if (num_completed) {
  243. u32 evre_read_off = (lldev->evre_processed_off +
  244. HIDMA_EVRE_SIZE * num_completed);
  245. evre_read_off = evre_read_off % evre_ring_size;
  246. writel(evre_read_off, lldev->evca + HIDMA_EVCA_DOORBELL_REG);
  247. /* record the last processed tre offset */
  248. lldev->evre_processed_off = evre_read_off;
  249. }
  250. return num_completed;
  251. }
  252. void hidma_cleanup_pending_tre(struct hidma_lldev *lldev, u8 err_info,
  253. u8 err_code)
  254. {
  255. while (atomic_read(&lldev->pending_tre_count)) {
  256. if (hidma_post_completed(lldev, err_info, err_code))
  257. break;
  258. }
  259. }
  260. static int hidma_ll_reset(struct hidma_lldev *lldev)
  261. {
  262. u32 val;
  263. int ret;
  264. val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG);
  265. val &= ~(HIDMA_CH_CONTROL_MASK << 16);
  266. val |= HIDMA_CH_RESET << 16;
  267. writel(val, lldev->trca + HIDMA_TRCA_CTRLSTS_REG);
  268. /*
  269. * Delay 10ms after reset to allow DMA logic to quiesce.
  270. * Do a polled read up to 1ms and 10ms maximum.
  271. */
  272. ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val,
  273. HIDMA_CH_STATE(val) == HIDMA_CH_DISABLED,
  274. 1000, 10000);
  275. if (ret) {
  276. dev_err(lldev->dev, "transfer channel did not reset\n");
  277. return ret;
  278. }
  279. val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG);
  280. val &= ~(HIDMA_CH_CONTROL_MASK << 16);
  281. val |= HIDMA_CH_RESET << 16;
  282. writel(val, lldev->evca + HIDMA_EVCA_CTRLSTS_REG);
  283. /*
  284. * Delay 10ms after reset to allow DMA logic to quiesce.
  285. * Do a polled read up to 1ms and 10ms maximum.
  286. */
  287. ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val,
  288. HIDMA_CH_STATE(val) == HIDMA_CH_DISABLED,
  289. 1000, 10000);
  290. if (ret)
  291. return ret;
  292. lldev->trch_state = HIDMA_CH_DISABLED;
  293. lldev->evch_state = HIDMA_CH_DISABLED;
  294. return 0;
  295. }
  296. /*
  297. * The interrupt handler for HIDMA will try to consume as many pending
  298. * EVRE from the event queue as possible. Each EVRE has an associated
  299. * TRE that holds the user interface parameters. EVRE reports the
  300. * result of the transaction. Hardware guarantees ordering between EVREs
  301. * and TREs. We use last processed offset to figure out which TRE is
  302. * associated with which EVRE. If two TREs are consumed by HW, the EVREs
  303. * are in order in the event ring.
  304. *
  305. * This handler will do a one pass for consuming EVREs. Other EVREs may
  306. * be delivered while we are working. It will try to consume incoming
  307. * EVREs one more time and return.
  308. *
  309. * For unprocessed EVREs, hardware will trigger another interrupt until
  310. * all the interrupt bits are cleared.
  311. *
  312. * Hardware guarantees that by the time interrupt is observed, all data
  313. * transactions in flight are delivered to their respective places and
  314. * are visible to the CPU.
  315. *
  316. * On demand paging for IOMMU is only supported for PCIe via PRI
  317. * (Page Request Interface) not for HIDMA. All other hardware instances
  318. * including HIDMA work on pinned DMA addresses.
  319. *
  320. * HIDMA is not aware of IOMMU presence since it follows the DMA API. All
  321. * IOMMU latency will be built into the data movement time. By the time
  322. * interrupt happens, IOMMU lookups + data movement has already taken place.
  323. *
  324. * While the first read in a typical PCI endpoint ISR flushes all outstanding
  325. * requests traditionally to the destination, this concept does not apply
  326. * here for this HW.
  327. */
  328. static void hidma_ll_int_handler_internal(struct hidma_lldev *lldev, int cause)
  329. {
  330. unsigned long irqflags;
  331. if (cause & HIDMA_ERR_INT_MASK) {
  332. dev_err(lldev->dev, "error 0x%x, disabling...\n",
  333. cause);
  334. /* Clear out pending interrupts */
  335. writel(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
  336. /* No further submissions. */
  337. hidma_ll_disable(lldev);
  338. /* Driver completes the txn and intimates the client.*/
  339. hidma_cleanup_pending_tre(lldev, 0xFF,
  340. HIDMA_EVRE_STATUS_ERROR);
  341. return;
  342. }
  343. spin_lock_irqsave(&lldev->lock, irqflags);
  344. writel_relaxed(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
  345. spin_unlock_irqrestore(&lldev->lock, irqflags);
  346. /*
  347. * Fine tuned for this HW...
  348. *
  349. * This ISR has been designed for this particular hardware. Relaxed
  350. * read and write accessors are used for performance reasons due to
  351. * interrupt delivery guarantees. Do not copy this code blindly and
  352. * expect that to work.
  353. *
  354. * Try to consume as many EVREs as possible.
  355. */
  356. hidma_handle_tre_completion(lldev);
  357. }
  358. irqreturn_t hidma_ll_inthandler(int chirq, void *arg)
  359. {
  360. struct hidma_lldev *lldev = arg;
  361. u32 status;
  362. u32 enable;
  363. u32 cause;
  364. status = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG);
  365. enable = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
  366. cause = status & enable;
  367. while (cause) {
  368. hidma_ll_int_handler_internal(lldev, cause);
  369. /*
  370. * Another interrupt might have arrived while we are
  371. * processing this one. Read the new cause.
  372. */
  373. status = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG);
  374. enable = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
  375. cause = status & enable;
  376. }
  377. return IRQ_HANDLED;
  378. }
  379. irqreturn_t hidma_ll_inthandler_msi(int chirq, void *arg, int cause)
  380. {
  381. struct hidma_lldev *lldev = arg;
  382. hidma_ll_int_handler_internal(lldev, cause);
  383. return IRQ_HANDLED;
  384. }
  385. int hidma_ll_enable(struct hidma_lldev *lldev)
  386. {
  387. u32 val;
  388. int ret;
  389. val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG);
  390. val &= ~(HIDMA_CH_CONTROL_MASK << 16);
  391. val |= HIDMA_CH_ENABLE << 16;
  392. writel(val, lldev->evca + HIDMA_EVCA_CTRLSTS_REG);
  393. ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val,
  394. hidma_is_chan_enabled(HIDMA_CH_STATE(val)),
  395. 1000, 10000);
  396. if (ret) {
  397. dev_err(lldev->dev, "event channel did not get enabled\n");
  398. return ret;
  399. }
  400. val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG);
  401. val &= ~(HIDMA_CH_CONTROL_MASK << 16);
  402. val |= HIDMA_CH_ENABLE << 16;
  403. writel(val, lldev->trca + HIDMA_TRCA_CTRLSTS_REG);
  404. ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val,
  405. hidma_is_chan_enabled(HIDMA_CH_STATE(val)),
  406. 1000, 10000);
  407. if (ret) {
  408. dev_err(lldev->dev, "transfer channel did not get enabled\n");
  409. return ret;
  410. }
  411. lldev->trch_state = HIDMA_CH_ENABLED;
  412. lldev->evch_state = HIDMA_CH_ENABLED;
  413. /* enable irqs */
  414. writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
  415. return 0;
  416. }
  417. void hidma_ll_start(struct hidma_lldev *lldev)
  418. {
  419. unsigned long irqflags;
  420. spin_lock_irqsave(&lldev->lock, irqflags);
  421. writel(lldev->tre_write_offset, lldev->trca + HIDMA_TRCA_DOORBELL_REG);
  422. spin_unlock_irqrestore(&lldev->lock, irqflags);
  423. }
  424. bool hidma_ll_isenabled(struct hidma_lldev *lldev)
  425. {
  426. u32 val;
  427. val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG);
  428. lldev->trch_state = HIDMA_CH_STATE(val);
  429. val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG);
  430. lldev->evch_state = HIDMA_CH_STATE(val);
  431. /* both channels have to be enabled before calling this function */
  432. if (hidma_is_chan_enabled(lldev->trch_state) &&
  433. hidma_is_chan_enabled(lldev->evch_state))
  434. return true;
  435. return false;
  436. }
  437. void hidma_ll_queue_request(struct hidma_lldev *lldev, u32 tre_ch)
  438. {
  439. struct hidma_tre *tre;
  440. unsigned long flags;
  441. tre = &lldev->trepool[tre_ch];
  442. /* copy the TRE into its location in the TRE ring */
  443. spin_lock_irqsave(&lldev->lock, flags);
  444. tre->tre_index = lldev->tre_write_offset / HIDMA_TRE_SIZE;
  445. lldev->pending_tre_list[tre->tre_index] = tre;
  446. memcpy(lldev->tre_ring + lldev->tre_write_offset,
  447. &tre->tre_local[0], HIDMA_TRE_SIZE);
  448. tre->err_code = 0;
  449. tre->err_info = 0;
  450. tre->queued = 1;
  451. atomic_inc(&lldev->pending_tre_count);
  452. lldev->tre_write_offset = (lldev->tre_write_offset + HIDMA_TRE_SIZE)
  453. % lldev->tre_ring_size;
  454. spin_unlock_irqrestore(&lldev->lock, flags);
  455. }
  456. /*
  457. * Note that even though we stop this channel if there is a pending transaction
  458. * in flight it will complete and follow the callback. This request will
  459. * prevent further requests to be made.
  460. */
  461. int hidma_ll_disable(struct hidma_lldev *lldev)
  462. {
  463. u32 val;
  464. int ret;
  465. /* The channel needs to be in working state */
  466. if (!hidma_ll_isenabled(lldev))
  467. return 0;
  468. val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG);
  469. val &= ~(HIDMA_CH_CONTROL_MASK << 16);
  470. val |= HIDMA_CH_SUSPEND << 16;
  471. writel(val, lldev->trca + HIDMA_TRCA_CTRLSTS_REG);
  472. /*
  473. * Start the wait right after the suspend is confirmed.
  474. * Do a polled read up to 1ms and 10ms maximum.
  475. */
  476. ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val,
  477. HIDMA_CH_STATE(val) == HIDMA_CH_SUSPENDED,
  478. 1000, 10000);
  479. if (ret)
  480. return ret;
  481. val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG);
  482. val &= ~(HIDMA_CH_CONTROL_MASK << 16);
  483. val |= HIDMA_CH_SUSPEND << 16;
  484. writel(val, lldev->evca + HIDMA_EVCA_CTRLSTS_REG);
  485. /*
  486. * Start the wait right after the suspend is confirmed
  487. * Delay up to 10ms after reset to allow DMA logic to quiesce.
  488. */
  489. ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val,
  490. HIDMA_CH_STATE(val) == HIDMA_CH_SUSPENDED,
  491. 1000, 10000);
  492. if (ret)
  493. return ret;
  494. lldev->trch_state = HIDMA_CH_SUSPENDED;
  495. lldev->evch_state = HIDMA_CH_SUSPENDED;
  496. /* disable interrupts */
  497. writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
  498. return 0;
  499. }
  500. void hidma_ll_set_transfer_params(struct hidma_lldev *lldev, u32 tre_ch,
  501. dma_addr_t src, dma_addr_t dest, u32 len,
  502. u32 flags, u32 txntype)
  503. {
  504. struct hidma_tre *tre;
  505. u32 *tre_local;
  506. if (tre_ch >= lldev->nr_tres) {
  507. dev_err(lldev->dev, "invalid TRE number in transfer params:%d",
  508. tre_ch);
  509. return;
  510. }
  511. tre = &lldev->trepool[tre_ch];
  512. if (atomic_read(&tre->allocated) != true) {
  513. dev_err(lldev->dev, "trying to set params on an unused TRE:%d",
  514. tre_ch);
  515. return;
  516. }
  517. tre_local = &tre->tre_local[0];
  518. tre_local[HIDMA_TRE_CFG_IDX] &= ~GENMASK(7, 0);
  519. tre_local[HIDMA_TRE_CFG_IDX] |= txntype;
  520. tre_local[HIDMA_TRE_LEN_IDX] = len;
  521. tre_local[HIDMA_TRE_SRC_LOW_IDX] = lower_32_bits(src);
  522. tre_local[HIDMA_TRE_SRC_HI_IDX] = upper_32_bits(src);
  523. tre_local[HIDMA_TRE_DEST_LOW_IDX] = lower_32_bits(dest);
  524. tre_local[HIDMA_TRE_DEST_HI_IDX] = upper_32_bits(dest);
  525. tre->int_flags = flags;
  526. }
  527. /*
  528. * Called during initialization and after an error condition
  529. * to restore hardware state.
  530. */
  531. int hidma_ll_setup(struct hidma_lldev *lldev)
  532. {
  533. int rc;
  534. u64 addr;
  535. u32 val;
  536. u32 nr_tres = lldev->nr_tres;
  537. atomic_set(&lldev->pending_tre_count, 0);
  538. lldev->tre_processed_off = 0;
  539. lldev->evre_processed_off = 0;
  540. lldev->tre_write_offset = 0;
  541. /* disable interrupts */
  542. writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
  543. /* clear all pending interrupts */
  544. val = readl(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG);
  545. writel(val, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
  546. rc = hidma_ll_reset(lldev);
  547. if (rc)
  548. return rc;
  549. /*
  550. * Clear all pending interrupts again.
  551. * Otherwise, we observe reset complete interrupts.
  552. */
  553. val = readl(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG);
  554. writel(val, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
  555. /* disable interrupts again after reset */
  556. writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
  557. addr = lldev->tre_dma;
  558. writel(lower_32_bits(addr), lldev->trca + HIDMA_TRCA_RING_LOW_REG);
  559. writel(upper_32_bits(addr), lldev->trca + HIDMA_TRCA_RING_HIGH_REG);
  560. writel(lldev->tre_ring_size, lldev->trca + HIDMA_TRCA_RING_LEN_REG);
  561. addr = lldev->evre_dma;
  562. writel(lower_32_bits(addr), lldev->evca + HIDMA_EVCA_RING_LOW_REG);
  563. writel(upper_32_bits(addr), lldev->evca + HIDMA_EVCA_RING_HIGH_REG);
  564. writel(HIDMA_EVRE_SIZE * nr_tres,
  565. lldev->evca + HIDMA_EVCA_RING_LEN_REG);
  566. /* configure interrupts */
  567. hidma_ll_setup_irq(lldev, lldev->msi_support);
  568. rc = hidma_ll_enable(lldev);
  569. if (rc)
  570. return rc;
  571. return rc;
  572. }
  573. void hidma_ll_setup_irq(struct hidma_lldev *lldev, bool msi)
  574. {
  575. u32 val;
  576. lldev->msi_support = msi;
  577. /* disable interrupts again after reset */
  578. writel(0, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
  579. writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
  580. /* support IRQ by default */
  581. val = readl(lldev->evca + HIDMA_EVCA_INTCTRL_REG);
  582. val &= ~0xF;
  583. if (!lldev->msi_support)
  584. val = val | 0x1;
  585. writel(val, lldev->evca + HIDMA_EVCA_INTCTRL_REG);
  586. /* clear all pending interrupts and enable them */
  587. writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
  588. writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
  589. }
  590. struct hidma_lldev *hidma_ll_init(struct device *dev, u32 nr_tres,
  591. void __iomem *trca, void __iomem *evca,
  592. u8 chidx)
  593. {
  594. u32 required_bytes;
  595. struct hidma_lldev *lldev;
  596. int rc;
  597. size_t sz;
  598. if (!trca || !evca || !dev || !nr_tres)
  599. return NULL;
  600. /* need at least four TREs */
  601. if (nr_tres < 4)
  602. return NULL;
  603. /* need an extra space */
  604. nr_tres += 1;
  605. lldev = devm_kzalloc(dev, sizeof(struct hidma_lldev), GFP_KERNEL);
  606. if (!lldev)
  607. return NULL;
  608. lldev->evca = evca;
  609. lldev->trca = trca;
  610. lldev->dev = dev;
  611. sz = sizeof(struct hidma_tre);
  612. lldev->trepool = devm_kcalloc(lldev->dev, nr_tres, sz, GFP_KERNEL);
  613. if (!lldev->trepool)
  614. return NULL;
  615. required_bytes = sizeof(lldev->pending_tre_list[0]);
  616. lldev->pending_tre_list = devm_kcalloc(dev, nr_tres, required_bytes,
  617. GFP_KERNEL);
  618. if (!lldev->pending_tre_list)
  619. return NULL;
  620. sz = (HIDMA_TRE_SIZE + 1) * nr_tres;
  621. lldev->tre_ring = dmam_alloc_coherent(dev, sz, &lldev->tre_dma,
  622. GFP_KERNEL);
  623. if (!lldev->tre_ring)
  624. return NULL;
  625. lldev->tre_ring_size = HIDMA_TRE_SIZE * nr_tres;
  626. lldev->nr_tres = nr_tres;
  627. /* the TRE ring has to be TRE_SIZE aligned */
  628. if (!IS_ALIGNED(lldev->tre_dma, HIDMA_TRE_SIZE)) {
  629. u8 tre_ring_shift;
  630. tre_ring_shift = lldev->tre_dma % HIDMA_TRE_SIZE;
  631. tre_ring_shift = HIDMA_TRE_SIZE - tre_ring_shift;
  632. lldev->tre_dma += tre_ring_shift;
  633. lldev->tre_ring += tre_ring_shift;
  634. }
  635. sz = (HIDMA_EVRE_SIZE + 1) * nr_tres;
  636. lldev->evre_ring = dmam_alloc_coherent(dev, sz, &lldev->evre_dma,
  637. GFP_KERNEL);
  638. if (!lldev->evre_ring)
  639. return NULL;
  640. lldev->evre_ring_size = HIDMA_EVRE_SIZE * nr_tres;
  641. /* the EVRE ring has to be EVRE_SIZE aligned */
  642. if (!IS_ALIGNED(lldev->evre_dma, HIDMA_EVRE_SIZE)) {
  643. u8 evre_ring_shift;
  644. evre_ring_shift = lldev->evre_dma % HIDMA_EVRE_SIZE;
  645. evre_ring_shift = HIDMA_EVRE_SIZE - evre_ring_shift;
  646. lldev->evre_dma += evre_ring_shift;
  647. lldev->evre_ring += evre_ring_shift;
  648. }
  649. lldev->nr_tres = nr_tres;
  650. lldev->chidx = chidx;
  651. sz = nr_tres * sizeof(struct hidma_tre *);
  652. rc = kfifo_alloc(&lldev->handoff_fifo, sz, GFP_KERNEL);
  653. if (rc)
  654. return NULL;
  655. rc = hidma_ll_setup(lldev);
  656. if (rc)
  657. return NULL;
  658. spin_lock_init(&lldev->lock);
  659. tasklet_setup(&lldev->task, hidma_ll_tre_complete);
  660. lldev->initialized = 1;
  661. writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
  662. return lldev;
  663. }
  664. int hidma_ll_uninit(struct hidma_lldev *lldev)
  665. {
  666. u32 required_bytes;
  667. int rc = 0;
  668. u32 val;
  669. if (!lldev)
  670. return -ENODEV;
  671. if (!lldev->initialized)
  672. return 0;
  673. lldev->initialized = 0;
  674. required_bytes = sizeof(struct hidma_tre) * lldev->nr_tres;
  675. tasklet_kill(&lldev->task);
  676. memset(lldev->trepool, 0, required_bytes);
  677. lldev->trepool = NULL;
  678. atomic_set(&lldev->pending_tre_count, 0);
  679. lldev->tre_write_offset = 0;
  680. rc = hidma_ll_reset(lldev);
  681. /*
  682. * Clear all pending interrupts again.
  683. * Otherwise, we observe reset complete interrupts.
  684. */
  685. val = readl(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG);
  686. writel(val, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
  687. writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
  688. return rc;
  689. }
  690. enum dma_status hidma_ll_status(struct hidma_lldev *lldev, u32 tre_ch)
  691. {
  692. enum dma_status ret = DMA_ERROR;
  693. struct hidma_tre *tre;
  694. unsigned long flags;
  695. u8 err_code;
  696. spin_lock_irqsave(&lldev->lock, flags);
  697. tre = &lldev->trepool[tre_ch];
  698. err_code = tre->err_code;
  699. if (err_code & HIDMA_EVRE_STATUS_COMPLETE)
  700. ret = DMA_COMPLETE;
  701. else if (err_code & HIDMA_EVRE_STATUS_ERROR)
  702. ret = DMA_ERROR;
  703. else
  704. ret = DMA_IN_PROGRESS;
  705. spin_unlock_irqrestore(&lldev->lock, flags);
  706. return ret;
  707. }