cppi41.c 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. #include <linux/delay.h>
  3. #include <linux/dmaengine.h>
  4. #include <linux/dma-mapping.h>
  5. #include <linux/platform_device.h>
  6. #include <linux/module.h>
  7. #include <linux/of.h>
  8. #include <linux/slab.h>
  9. #include <linux/of_dma.h>
  10. #include <linux/of_irq.h>
  11. #include <linux/dmapool.h>
  12. #include <linux/interrupt.h>
  13. #include <linux/of_address.h>
  14. #include <linux/pm_runtime.h>
  15. #include "../dmaengine.h"
  16. #define DESC_TYPE 27
  17. #define DESC_TYPE_HOST 0x10
  18. #define DESC_TYPE_TEARD 0x13
  19. #define TD_DESC_IS_RX (1 << 16)
  20. #define TD_DESC_DMA_NUM 10
  21. #define DESC_LENGTH_BITS_NUM 21
  22. #define DESC_TYPE_USB (5 << 26)
  23. #define DESC_PD_COMPLETE (1 << 31)
  24. /* DMA engine */
  25. #define DMA_TDFDQ 4
  26. #define DMA_TXGCR(x) (0x800 + (x) * 0x20)
  27. #define DMA_RXGCR(x) (0x808 + (x) * 0x20)
  28. #define RXHPCRA0 4
  29. #define GCR_CHAN_ENABLE (1 << 31)
  30. #define GCR_TEARDOWN (1 << 30)
  31. #define GCR_STARV_RETRY (1 << 24)
  32. #define GCR_DESC_TYPE_HOST (1 << 14)
  33. /* DMA scheduler */
  34. #define DMA_SCHED_CTRL 0
  35. #define DMA_SCHED_CTRL_EN (1 << 31)
  36. #define DMA_SCHED_WORD(x) ((x) * 4 + 0x800)
  37. #define SCHED_ENTRY0_CHAN(x) ((x) << 0)
  38. #define SCHED_ENTRY0_IS_RX (1 << 7)
  39. #define SCHED_ENTRY1_CHAN(x) ((x) << 8)
  40. #define SCHED_ENTRY1_IS_RX (1 << 15)
  41. #define SCHED_ENTRY2_CHAN(x) ((x) << 16)
  42. #define SCHED_ENTRY2_IS_RX (1 << 23)
  43. #define SCHED_ENTRY3_CHAN(x) ((x) << 24)
  44. #define SCHED_ENTRY3_IS_RX (1 << 31)
  45. /* Queue manager */
  46. /* 4 KiB of memory for descriptors, 2 for each endpoint */
  47. #define ALLOC_DECS_NUM 128
  48. #define DESCS_AREAS 1
  49. #define TOTAL_DESCS_NUM (ALLOC_DECS_NUM * DESCS_AREAS)
  50. #define QMGR_SCRATCH_SIZE (TOTAL_DESCS_NUM * 4)
  51. #define QMGR_LRAM0_BASE 0x80
  52. #define QMGR_LRAM_SIZE 0x84
  53. #define QMGR_LRAM1_BASE 0x88
  54. #define QMGR_MEMBASE(x) (0x1000 + (x) * 0x10)
  55. #define QMGR_MEMCTRL(x) (0x1004 + (x) * 0x10)
  56. #define QMGR_MEMCTRL_IDX_SH 16
  57. #define QMGR_MEMCTRL_DESC_SH 8
  58. #define QMGR_PEND(x) (0x90 + (x) * 4)
  59. #define QMGR_PENDING_SLOT_Q(x) (x / 32)
  60. #define QMGR_PENDING_BIT_Q(x) (x % 32)
  61. #define QMGR_QUEUE_A(n) (0x2000 + (n) * 0x10)
  62. #define QMGR_QUEUE_B(n) (0x2004 + (n) * 0x10)
  63. #define QMGR_QUEUE_C(n) (0x2008 + (n) * 0x10)
  64. #define QMGR_QUEUE_D(n) (0x200c + (n) * 0x10)
  65. /* Packet Descriptor */
  66. #define PD2_ZERO_LENGTH (1 << 19)
  67. struct cppi41_channel {
  68. struct dma_chan chan;
  69. struct dma_async_tx_descriptor txd;
  70. struct cppi41_dd *cdd;
  71. struct cppi41_desc *desc;
  72. dma_addr_t desc_phys;
  73. void __iomem *gcr_reg;
  74. int is_tx;
  75. u32 residue;
  76. unsigned int q_num;
  77. unsigned int q_comp_num;
  78. unsigned int port_num;
  79. unsigned td_retry;
  80. unsigned td_queued:1;
  81. unsigned td_seen:1;
  82. unsigned td_desc_seen:1;
  83. struct list_head node; /* Node for pending list */
  84. };
  85. struct cppi41_desc {
  86. u32 pd0;
  87. u32 pd1;
  88. u32 pd2;
  89. u32 pd3;
  90. u32 pd4;
  91. u32 pd5;
  92. u32 pd6;
  93. u32 pd7;
  94. } __aligned(32);
  95. struct chan_queues {
  96. u16 submit;
  97. u16 complete;
  98. };
  99. struct cppi41_dd {
  100. struct dma_device ddev;
  101. void *qmgr_scratch;
  102. dma_addr_t scratch_phys;
  103. struct cppi41_desc *cd;
  104. dma_addr_t descs_phys;
  105. u32 first_td_desc;
  106. struct cppi41_channel *chan_busy[ALLOC_DECS_NUM];
  107. void __iomem *ctrl_mem;
  108. void __iomem *sched_mem;
  109. void __iomem *qmgr_mem;
  110. unsigned int irq;
  111. const struct chan_queues *queues_rx;
  112. const struct chan_queues *queues_tx;
  113. struct chan_queues td_queue;
  114. u16 first_completion_queue;
  115. u16 qmgr_num_pend;
  116. u32 n_chans;
  117. u8 platform;
  118. struct list_head pending; /* Pending queued transfers */
  119. spinlock_t lock; /* Lock for pending list */
  120. /* context for suspend/resume */
  121. unsigned int dma_tdfdq;
  122. bool is_suspended;
  123. };
  124. static struct chan_queues am335x_usb_queues_tx[] = {
  125. /* USB0 ENDP 1 */
  126. [ 0] = { .submit = 32, .complete = 93},
  127. [ 1] = { .submit = 34, .complete = 94},
  128. [ 2] = { .submit = 36, .complete = 95},
  129. [ 3] = { .submit = 38, .complete = 96},
  130. [ 4] = { .submit = 40, .complete = 97},
  131. [ 5] = { .submit = 42, .complete = 98},
  132. [ 6] = { .submit = 44, .complete = 99},
  133. [ 7] = { .submit = 46, .complete = 100},
  134. [ 8] = { .submit = 48, .complete = 101},
  135. [ 9] = { .submit = 50, .complete = 102},
  136. [10] = { .submit = 52, .complete = 103},
  137. [11] = { .submit = 54, .complete = 104},
  138. [12] = { .submit = 56, .complete = 105},
  139. [13] = { .submit = 58, .complete = 106},
  140. [14] = { .submit = 60, .complete = 107},
  141. /* USB1 ENDP1 */
  142. [15] = { .submit = 62, .complete = 125},
  143. [16] = { .submit = 64, .complete = 126},
  144. [17] = { .submit = 66, .complete = 127},
  145. [18] = { .submit = 68, .complete = 128},
  146. [19] = { .submit = 70, .complete = 129},
  147. [20] = { .submit = 72, .complete = 130},
  148. [21] = { .submit = 74, .complete = 131},
  149. [22] = { .submit = 76, .complete = 132},
  150. [23] = { .submit = 78, .complete = 133},
  151. [24] = { .submit = 80, .complete = 134},
  152. [25] = { .submit = 82, .complete = 135},
  153. [26] = { .submit = 84, .complete = 136},
  154. [27] = { .submit = 86, .complete = 137},
  155. [28] = { .submit = 88, .complete = 138},
  156. [29] = { .submit = 90, .complete = 139},
  157. };
  158. static const struct chan_queues am335x_usb_queues_rx[] = {
  159. /* USB0 ENDP 1 */
  160. [ 0] = { .submit = 1, .complete = 109},
  161. [ 1] = { .submit = 2, .complete = 110},
  162. [ 2] = { .submit = 3, .complete = 111},
  163. [ 3] = { .submit = 4, .complete = 112},
  164. [ 4] = { .submit = 5, .complete = 113},
  165. [ 5] = { .submit = 6, .complete = 114},
  166. [ 6] = { .submit = 7, .complete = 115},
  167. [ 7] = { .submit = 8, .complete = 116},
  168. [ 8] = { .submit = 9, .complete = 117},
  169. [ 9] = { .submit = 10, .complete = 118},
  170. [10] = { .submit = 11, .complete = 119},
  171. [11] = { .submit = 12, .complete = 120},
  172. [12] = { .submit = 13, .complete = 121},
  173. [13] = { .submit = 14, .complete = 122},
  174. [14] = { .submit = 15, .complete = 123},
  175. /* USB1 ENDP 1 */
  176. [15] = { .submit = 16, .complete = 141},
  177. [16] = { .submit = 17, .complete = 142},
  178. [17] = { .submit = 18, .complete = 143},
  179. [18] = { .submit = 19, .complete = 144},
  180. [19] = { .submit = 20, .complete = 145},
  181. [20] = { .submit = 21, .complete = 146},
  182. [21] = { .submit = 22, .complete = 147},
  183. [22] = { .submit = 23, .complete = 148},
  184. [23] = { .submit = 24, .complete = 149},
  185. [24] = { .submit = 25, .complete = 150},
  186. [25] = { .submit = 26, .complete = 151},
  187. [26] = { .submit = 27, .complete = 152},
  188. [27] = { .submit = 28, .complete = 153},
  189. [28] = { .submit = 29, .complete = 154},
  190. [29] = { .submit = 30, .complete = 155},
  191. };
  192. static const struct chan_queues da8xx_usb_queues_tx[] = {
  193. [0] = { .submit = 16, .complete = 24},
  194. [1] = { .submit = 18, .complete = 24},
  195. [2] = { .submit = 20, .complete = 24},
  196. [3] = { .submit = 22, .complete = 24},
  197. };
  198. static const struct chan_queues da8xx_usb_queues_rx[] = {
  199. [0] = { .submit = 1, .complete = 26},
  200. [1] = { .submit = 3, .complete = 26},
  201. [2] = { .submit = 5, .complete = 26},
  202. [3] = { .submit = 7, .complete = 26},
  203. };
  204. struct cppi_glue_infos {
  205. const struct chan_queues *queues_rx;
  206. const struct chan_queues *queues_tx;
  207. struct chan_queues td_queue;
  208. u16 first_completion_queue;
  209. u16 qmgr_num_pend;
  210. };
  211. static struct cppi41_channel *to_cpp41_chan(struct dma_chan *c)
  212. {
  213. return container_of(c, struct cppi41_channel, chan);
  214. }
  215. static struct cppi41_channel *desc_to_chan(struct cppi41_dd *cdd, u32 desc)
  216. {
  217. struct cppi41_channel *c;
  218. u32 descs_size;
  219. u32 desc_num;
  220. descs_size = sizeof(struct cppi41_desc) * ALLOC_DECS_NUM;
  221. if (!((desc >= cdd->descs_phys) &&
  222. (desc < (cdd->descs_phys + descs_size)))) {
  223. return NULL;
  224. }
  225. desc_num = (desc - cdd->descs_phys) / sizeof(struct cppi41_desc);
  226. BUG_ON(desc_num >= ALLOC_DECS_NUM);
  227. c = cdd->chan_busy[desc_num];
  228. cdd->chan_busy[desc_num] = NULL;
  229. /* Usecount for chan_busy[], paired with push_desc_queue() */
  230. pm_runtime_put(cdd->ddev.dev);
  231. return c;
  232. }
  233. static void cppi_writel(u32 val, void *__iomem *mem)
  234. {
  235. __raw_writel(val, mem);
  236. }
  237. static u32 cppi_readl(void *__iomem *mem)
  238. {
  239. return __raw_readl(mem);
  240. }
  241. static u32 pd_trans_len(u32 val)
  242. {
  243. return val & ((1 << (DESC_LENGTH_BITS_NUM + 1)) - 1);
  244. }
  245. static u32 cppi41_pop_desc(struct cppi41_dd *cdd, unsigned queue_num)
  246. {
  247. u32 desc;
  248. desc = cppi_readl(cdd->qmgr_mem + QMGR_QUEUE_D(queue_num));
  249. desc &= ~0x1f;
  250. return desc;
  251. }
  252. static irqreturn_t cppi41_irq(int irq, void *data)
  253. {
  254. struct cppi41_dd *cdd = data;
  255. u16 first_completion_queue = cdd->first_completion_queue;
  256. u16 qmgr_num_pend = cdd->qmgr_num_pend;
  257. struct cppi41_channel *c;
  258. int i;
  259. for (i = QMGR_PENDING_SLOT_Q(first_completion_queue); i < qmgr_num_pend;
  260. i++) {
  261. u32 val;
  262. u32 q_num;
  263. val = cppi_readl(cdd->qmgr_mem + QMGR_PEND(i));
  264. if (i == QMGR_PENDING_SLOT_Q(first_completion_queue) && val) {
  265. u32 mask;
  266. /* set corresponding bit for completetion Q 93 */
  267. mask = 1 << QMGR_PENDING_BIT_Q(first_completion_queue);
  268. /* not set all bits for queues less than Q 93 */
  269. mask--;
  270. /* now invert and keep only Q 93+ set */
  271. val &= ~mask;
  272. }
  273. if (val)
  274. __iormb();
  275. while (val) {
  276. u32 desc, len;
  277. /*
  278. * This should never trigger, see the comments in
  279. * push_desc_queue()
  280. */
  281. WARN_ON(cdd->is_suspended);
  282. q_num = __fls(val);
  283. val &= ~(1 << q_num);
  284. q_num += 32 * i;
  285. desc = cppi41_pop_desc(cdd, q_num);
  286. c = desc_to_chan(cdd, desc);
  287. if (WARN_ON(!c)) {
  288. pr_err("%s() q %d desc %08x\n", __func__,
  289. q_num, desc);
  290. continue;
  291. }
  292. if (c->desc->pd2 & PD2_ZERO_LENGTH)
  293. len = 0;
  294. else
  295. len = pd_trans_len(c->desc->pd0);
  296. c->residue = pd_trans_len(c->desc->pd6) - len;
  297. dma_cookie_complete(&c->txd);
  298. dmaengine_desc_get_callback_invoke(&c->txd, NULL);
  299. }
  300. }
  301. return IRQ_HANDLED;
  302. }
  303. static dma_cookie_t cppi41_tx_submit(struct dma_async_tx_descriptor *tx)
  304. {
  305. dma_cookie_t cookie;
  306. cookie = dma_cookie_assign(tx);
  307. return cookie;
  308. }
  309. static int cppi41_dma_alloc_chan_resources(struct dma_chan *chan)
  310. {
  311. struct cppi41_channel *c = to_cpp41_chan(chan);
  312. struct cppi41_dd *cdd = c->cdd;
  313. int error;
  314. error = pm_runtime_get_sync(cdd->ddev.dev);
  315. if (error < 0) {
  316. dev_err(cdd->ddev.dev, "%s pm runtime get: %i\n",
  317. __func__, error);
  318. pm_runtime_put_noidle(cdd->ddev.dev);
  319. return error;
  320. }
  321. dma_cookie_init(chan);
  322. dma_async_tx_descriptor_init(&c->txd, chan);
  323. c->txd.tx_submit = cppi41_tx_submit;
  324. if (!c->is_tx)
  325. cppi_writel(c->q_num, c->gcr_reg + RXHPCRA0);
  326. pm_runtime_mark_last_busy(cdd->ddev.dev);
  327. pm_runtime_put_autosuspend(cdd->ddev.dev);
  328. return 0;
  329. }
  330. static void cppi41_dma_free_chan_resources(struct dma_chan *chan)
  331. {
  332. struct cppi41_channel *c = to_cpp41_chan(chan);
  333. struct cppi41_dd *cdd = c->cdd;
  334. int error;
  335. error = pm_runtime_get_sync(cdd->ddev.dev);
  336. if (error < 0) {
  337. pm_runtime_put_noidle(cdd->ddev.dev);
  338. return;
  339. }
  340. WARN_ON(!list_empty(&cdd->pending));
  341. pm_runtime_mark_last_busy(cdd->ddev.dev);
  342. pm_runtime_put_autosuspend(cdd->ddev.dev);
  343. }
  344. static enum dma_status cppi41_dma_tx_status(struct dma_chan *chan,
  345. dma_cookie_t cookie, struct dma_tx_state *txstate)
  346. {
  347. struct cppi41_channel *c = to_cpp41_chan(chan);
  348. enum dma_status ret;
  349. ret = dma_cookie_status(chan, cookie, txstate);
  350. dma_set_residue(txstate, c->residue);
  351. return ret;
  352. }
  353. static void push_desc_queue(struct cppi41_channel *c)
  354. {
  355. struct cppi41_dd *cdd = c->cdd;
  356. u32 desc_num;
  357. u32 desc_phys;
  358. u32 reg;
  359. c->residue = 0;
  360. reg = GCR_CHAN_ENABLE;
  361. if (!c->is_tx) {
  362. reg |= GCR_STARV_RETRY;
  363. reg |= GCR_DESC_TYPE_HOST;
  364. reg |= c->q_comp_num;
  365. }
  366. cppi_writel(reg, c->gcr_reg);
  367. /*
  368. * We don't use writel() but __raw_writel() so we have to make sure
  369. * that the DMA descriptor in coherent memory made to the main memory
  370. * before starting the dma engine.
  371. */
  372. __iowmb();
  373. /*
  374. * DMA transfers can take at least 200ms to complete with USB mass
  375. * storage connected. To prevent autosuspend timeouts, we must use
  376. * pm_runtime_get/put() when chan_busy[] is modified. This will get
  377. * cleared in desc_to_chan() or cppi41_stop_chan() depending on the
  378. * outcome of the transfer.
  379. */
  380. pm_runtime_get(cdd->ddev.dev);
  381. desc_phys = lower_32_bits(c->desc_phys);
  382. desc_num = (desc_phys - cdd->descs_phys) / sizeof(struct cppi41_desc);
  383. WARN_ON(cdd->chan_busy[desc_num]);
  384. cdd->chan_busy[desc_num] = c;
  385. reg = (sizeof(struct cppi41_desc) - 24) / 4;
  386. reg |= desc_phys;
  387. cppi_writel(reg, cdd->qmgr_mem + QMGR_QUEUE_D(c->q_num));
  388. }
  389. /*
  390. * Caller must hold cdd->lock to prevent push_desc_queue()
  391. * getting called out of order. We have both cppi41_dma_issue_pending()
  392. * and cppi41_runtime_resume() call this function.
  393. */
  394. static void cppi41_run_queue(struct cppi41_dd *cdd)
  395. {
  396. struct cppi41_channel *c, *_c;
  397. list_for_each_entry_safe(c, _c, &cdd->pending, node) {
  398. push_desc_queue(c);
  399. list_del(&c->node);
  400. }
  401. }
  402. static void cppi41_dma_issue_pending(struct dma_chan *chan)
  403. {
  404. struct cppi41_channel *c = to_cpp41_chan(chan);
  405. struct cppi41_dd *cdd = c->cdd;
  406. unsigned long flags;
  407. int error;
  408. error = pm_runtime_get(cdd->ddev.dev);
  409. if ((error != -EINPROGRESS) && error < 0) {
  410. pm_runtime_put_noidle(cdd->ddev.dev);
  411. dev_err(cdd->ddev.dev, "Failed to pm_runtime_get: %i\n",
  412. error);
  413. return;
  414. }
  415. spin_lock_irqsave(&cdd->lock, flags);
  416. list_add_tail(&c->node, &cdd->pending);
  417. if (!cdd->is_suspended)
  418. cppi41_run_queue(cdd);
  419. spin_unlock_irqrestore(&cdd->lock, flags);
  420. pm_runtime_mark_last_busy(cdd->ddev.dev);
  421. pm_runtime_put_autosuspend(cdd->ddev.dev);
  422. }
  423. static u32 get_host_pd0(u32 length)
  424. {
  425. u32 reg;
  426. reg = DESC_TYPE_HOST << DESC_TYPE;
  427. reg |= length;
  428. return reg;
  429. }
  430. static u32 get_host_pd1(struct cppi41_channel *c)
  431. {
  432. u32 reg;
  433. reg = 0;
  434. return reg;
  435. }
  436. static u32 get_host_pd2(struct cppi41_channel *c)
  437. {
  438. u32 reg;
  439. reg = DESC_TYPE_USB;
  440. reg |= c->q_comp_num;
  441. return reg;
  442. }
  443. static u32 get_host_pd3(u32 length)
  444. {
  445. u32 reg;
  446. /* PD3 = packet size */
  447. reg = length;
  448. return reg;
  449. }
  450. static u32 get_host_pd6(u32 length)
  451. {
  452. u32 reg;
  453. /* PD6 buffer size */
  454. reg = DESC_PD_COMPLETE;
  455. reg |= length;
  456. return reg;
  457. }
  458. static u32 get_host_pd4_or_7(u32 addr)
  459. {
  460. u32 reg;
  461. reg = addr;
  462. return reg;
  463. }
  464. static u32 get_host_pd5(void)
  465. {
  466. u32 reg;
  467. reg = 0;
  468. return reg;
  469. }
  470. static struct dma_async_tx_descriptor *cppi41_dma_prep_slave_sg(
  471. struct dma_chan *chan, struct scatterlist *sgl, unsigned sg_len,
  472. enum dma_transfer_direction dir, unsigned long tx_flags, void *context)
  473. {
  474. struct cppi41_channel *c = to_cpp41_chan(chan);
  475. struct dma_async_tx_descriptor *txd = NULL;
  476. struct cppi41_dd *cdd = c->cdd;
  477. struct cppi41_desc *d;
  478. struct scatterlist *sg;
  479. unsigned int i;
  480. int error;
  481. error = pm_runtime_get(cdd->ddev.dev);
  482. if (error < 0) {
  483. pm_runtime_put_noidle(cdd->ddev.dev);
  484. return NULL;
  485. }
  486. if (cdd->is_suspended)
  487. goto err_out_not_ready;
  488. d = c->desc;
  489. for_each_sg(sgl, sg, sg_len, i) {
  490. u32 addr;
  491. u32 len;
  492. /* We need to use more than one desc once musb supports sg */
  493. addr = lower_32_bits(sg_dma_address(sg));
  494. len = sg_dma_len(sg);
  495. d->pd0 = get_host_pd0(len);
  496. d->pd1 = get_host_pd1(c);
  497. d->pd2 = get_host_pd2(c);
  498. d->pd3 = get_host_pd3(len);
  499. d->pd4 = get_host_pd4_or_7(addr);
  500. d->pd5 = get_host_pd5();
  501. d->pd6 = get_host_pd6(len);
  502. d->pd7 = get_host_pd4_or_7(addr);
  503. d++;
  504. }
  505. txd = &c->txd;
  506. err_out_not_ready:
  507. pm_runtime_mark_last_busy(cdd->ddev.dev);
  508. pm_runtime_put_autosuspend(cdd->ddev.dev);
  509. return txd;
  510. }
  511. static void cppi41_compute_td_desc(struct cppi41_desc *d)
  512. {
  513. d->pd0 = DESC_TYPE_TEARD << DESC_TYPE;
  514. }
  515. static int cppi41_tear_down_chan(struct cppi41_channel *c)
  516. {
  517. struct dmaengine_result abort_result;
  518. struct cppi41_dd *cdd = c->cdd;
  519. struct cppi41_desc *td;
  520. u32 reg;
  521. u32 desc_phys;
  522. u32 td_desc_phys;
  523. td = cdd->cd;
  524. td += cdd->first_td_desc;
  525. td_desc_phys = cdd->descs_phys;
  526. td_desc_phys += cdd->first_td_desc * sizeof(struct cppi41_desc);
  527. if (!c->td_queued) {
  528. cppi41_compute_td_desc(td);
  529. __iowmb();
  530. reg = (sizeof(struct cppi41_desc) - 24) / 4;
  531. reg |= td_desc_phys;
  532. cppi_writel(reg, cdd->qmgr_mem +
  533. QMGR_QUEUE_D(cdd->td_queue.submit));
  534. reg = GCR_CHAN_ENABLE;
  535. if (!c->is_tx) {
  536. reg |= GCR_STARV_RETRY;
  537. reg |= GCR_DESC_TYPE_HOST;
  538. reg |= cdd->td_queue.complete;
  539. }
  540. reg |= GCR_TEARDOWN;
  541. cppi_writel(reg, c->gcr_reg);
  542. c->td_queued = 1;
  543. c->td_retry = 500;
  544. }
  545. if (!c->td_seen || !c->td_desc_seen) {
  546. desc_phys = cppi41_pop_desc(cdd, cdd->td_queue.complete);
  547. if (!desc_phys && c->is_tx)
  548. desc_phys = cppi41_pop_desc(cdd, c->q_comp_num);
  549. if (desc_phys == c->desc_phys) {
  550. c->td_desc_seen = 1;
  551. } else if (desc_phys == td_desc_phys) {
  552. u32 pd0;
  553. __iormb();
  554. pd0 = td->pd0;
  555. WARN_ON((pd0 >> DESC_TYPE) != DESC_TYPE_TEARD);
  556. WARN_ON(!c->is_tx && !(pd0 & TD_DESC_IS_RX));
  557. WARN_ON((pd0 & 0x1f) != c->port_num);
  558. c->td_seen = 1;
  559. } else if (desc_phys) {
  560. WARN_ON_ONCE(1);
  561. }
  562. }
  563. c->td_retry--;
  564. /*
  565. * If the TX descriptor / channel is in use, the caller needs to poke
  566. * his TD bit multiple times. After that he hardware releases the
  567. * transfer descriptor followed by TD descriptor. Waiting seems not to
  568. * cause any difference.
  569. * RX seems to be thrown out right away. However once the TearDown
  570. * descriptor gets through we are done. If we have seens the transfer
  571. * descriptor before the TD we fetch it from enqueue, it has to be
  572. * there waiting for us.
  573. */
  574. if (!c->td_seen && c->td_retry) {
  575. udelay(1);
  576. return -EAGAIN;
  577. }
  578. WARN_ON(!c->td_retry);
  579. if (!c->td_desc_seen) {
  580. desc_phys = cppi41_pop_desc(cdd, c->q_num);
  581. if (!desc_phys)
  582. desc_phys = cppi41_pop_desc(cdd, c->q_comp_num);
  583. WARN_ON(!desc_phys);
  584. }
  585. c->td_queued = 0;
  586. c->td_seen = 0;
  587. c->td_desc_seen = 0;
  588. cppi_writel(0, c->gcr_reg);
  589. /* Invoke the callback to do the necessary clean-up */
  590. abort_result.result = DMA_TRANS_ABORTED;
  591. dma_cookie_complete(&c->txd);
  592. dmaengine_desc_get_callback_invoke(&c->txd, &abort_result);
  593. return 0;
  594. }
  595. static int cppi41_stop_chan(struct dma_chan *chan)
  596. {
  597. struct cppi41_channel *c = to_cpp41_chan(chan);
  598. struct cppi41_dd *cdd = c->cdd;
  599. u32 desc_num;
  600. u32 desc_phys;
  601. int ret;
  602. desc_phys = lower_32_bits(c->desc_phys);
  603. desc_num = (desc_phys - cdd->descs_phys) / sizeof(struct cppi41_desc);
  604. if (!cdd->chan_busy[desc_num]) {
  605. struct cppi41_channel *cc, *_ct;
  606. /*
  607. * channels might still be in the pendling list if
  608. * cppi41_dma_issue_pending() is called after
  609. * cppi41_runtime_suspend() is called
  610. */
  611. list_for_each_entry_safe(cc, _ct, &cdd->pending, node) {
  612. if (cc != c)
  613. continue;
  614. list_del(&cc->node);
  615. break;
  616. }
  617. return 0;
  618. }
  619. ret = cppi41_tear_down_chan(c);
  620. if (ret)
  621. return ret;
  622. WARN_ON(!cdd->chan_busy[desc_num]);
  623. cdd->chan_busy[desc_num] = NULL;
  624. /* Usecount for chan_busy[], paired with push_desc_queue() */
  625. pm_runtime_put(cdd->ddev.dev);
  626. return 0;
  627. }
  628. static int cppi41_add_chans(struct device *dev, struct cppi41_dd *cdd)
  629. {
  630. struct cppi41_channel *cchan, *chans;
  631. int i;
  632. u32 n_chans = cdd->n_chans;
  633. /*
  634. * The channels can only be used as TX or as RX. So we add twice
  635. * that much dma channels because USB can only do RX or TX.
  636. */
  637. n_chans *= 2;
  638. chans = devm_kcalloc(dev, n_chans, sizeof(*chans), GFP_KERNEL);
  639. if (!chans)
  640. return -ENOMEM;
  641. for (i = 0; i < n_chans; i++) {
  642. cchan = &chans[i];
  643. cchan->cdd = cdd;
  644. if (i & 1) {
  645. cchan->gcr_reg = cdd->ctrl_mem + DMA_TXGCR(i >> 1);
  646. cchan->is_tx = 1;
  647. } else {
  648. cchan->gcr_reg = cdd->ctrl_mem + DMA_RXGCR(i >> 1);
  649. cchan->is_tx = 0;
  650. }
  651. cchan->port_num = i >> 1;
  652. cchan->desc = &cdd->cd[i];
  653. cchan->desc_phys = cdd->descs_phys;
  654. cchan->desc_phys += i * sizeof(struct cppi41_desc);
  655. cchan->chan.device = &cdd->ddev;
  656. list_add_tail(&cchan->chan.device_node, &cdd->ddev.channels);
  657. }
  658. cdd->first_td_desc = n_chans;
  659. return 0;
  660. }
  661. static void purge_descs(struct device *dev, struct cppi41_dd *cdd)
  662. {
  663. unsigned int mem_decs;
  664. int i;
  665. mem_decs = ALLOC_DECS_NUM * sizeof(struct cppi41_desc);
  666. for (i = 0; i < DESCS_AREAS; i++) {
  667. cppi_writel(0, cdd->qmgr_mem + QMGR_MEMBASE(i));
  668. cppi_writel(0, cdd->qmgr_mem + QMGR_MEMCTRL(i));
  669. dma_free_coherent(dev, mem_decs, cdd->cd,
  670. cdd->descs_phys);
  671. }
  672. }
  673. static void disable_sched(struct cppi41_dd *cdd)
  674. {
  675. cppi_writel(0, cdd->sched_mem + DMA_SCHED_CTRL);
  676. }
  677. static void deinit_cppi41(struct device *dev, struct cppi41_dd *cdd)
  678. {
  679. disable_sched(cdd);
  680. purge_descs(dev, cdd);
  681. cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM0_BASE);
  682. cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM0_BASE);
  683. dma_free_coherent(dev, QMGR_SCRATCH_SIZE, cdd->qmgr_scratch,
  684. cdd->scratch_phys);
  685. }
  686. static int init_descs(struct device *dev, struct cppi41_dd *cdd)
  687. {
  688. unsigned int desc_size;
  689. unsigned int mem_decs;
  690. int i;
  691. u32 reg;
  692. u32 idx;
  693. BUILD_BUG_ON(sizeof(struct cppi41_desc) &
  694. (sizeof(struct cppi41_desc) - 1));
  695. BUILD_BUG_ON(sizeof(struct cppi41_desc) < 32);
  696. BUILD_BUG_ON(ALLOC_DECS_NUM < 32);
  697. desc_size = sizeof(struct cppi41_desc);
  698. mem_decs = ALLOC_DECS_NUM * desc_size;
  699. idx = 0;
  700. for (i = 0; i < DESCS_AREAS; i++) {
  701. reg = idx << QMGR_MEMCTRL_IDX_SH;
  702. reg |= (ilog2(desc_size) - 5) << QMGR_MEMCTRL_DESC_SH;
  703. reg |= ilog2(ALLOC_DECS_NUM) - 5;
  704. BUILD_BUG_ON(DESCS_AREAS != 1);
  705. cdd->cd = dma_alloc_coherent(dev, mem_decs,
  706. &cdd->descs_phys, GFP_KERNEL);
  707. if (!cdd->cd)
  708. return -ENOMEM;
  709. cppi_writel(cdd->descs_phys, cdd->qmgr_mem + QMGR_MEMBASE(i));
  710. cppi_writel(reg, cdd->qmgr_mem + QMGR_MEMCTRL(i));
  711. idx += ALLOC_DECS_NUM;
  712. }
  713. return 0;
  714. }
  715. static void init_sched(struct cppi41_dd *cdd)
  716. {
  717. unsigned ch;
  718. unsigned word;
  719. u32 reg;
  720. word = 0;
  721. cppi_writel(0, cdd->sched_mem + DMA_SCHED_CTRL);
  722. for (ch = 0; ch < cdd->n_chans; ch += 2) {
  723. reg = SCHED_ENTRY0_CHAN(ch);
  724. reg |= SCHED_ENTRY1_CHAN(ch) | SCHED_ENTRY1_IS_RX;
  725. reg |= SCHED_ENTRY2_CHAN(ch + 1);
  726. reg |= SCHED_ENTRY3_CHAN(ch + 1) | SCHED_ENTRY3_IS_RX;
  727. cppi_writel(reg, cdd->sched_mem + DMA_SCHED_WORD(word));
  728. word++;
  729. }
  730. reg = cdd->n_chans * 2 - 1;
  731. reg |= DMA_SCHED_CTRL_EN;
  732. cppi_writel(reg, cdd->sched_mem + DMA_SCHED_CTRL);
  733. }
  734. static int init_cppi41(struct device *dev, struct cppi41_dd *cdd)
  735. {
  736. int ret;
  737. BUILD_BUG_ON(QMGR_SCRATCH_SIZE > ((1 << 14) - 1));
  738. cdd->qmgr_scratch = dma_alloc_coherent(dev, QMGR_SCRATCH_SIZE,
  739. &cdd->scratch_phys, GFP_KERNEL);
  740. if (!cdd->qmgr_scratch)
  741. return -ENOMEM;
  742. cppi_writel(cdd->scratch_phys, cdd->qmgr_mem + QMGR_LRAM0_BASE);
  743. cppi_writel(TOTAL_DESCS_NUM, cdd->qmgr_mem + QMGR_LRAM_SIZE);
  744. cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM1_BASE);
  745. ret = init_descs(dev, cdd);
  746. if (ret)
  747. goto err_td;
  748. cppi_writel(cdd->td_queue.submit, cdd->ctrl_mem + DMA_TDFDQ);
  749. init_sched(cdd);
  750. return 0;
  751. err_td:
  752. deinit_cppi41(dev, cdd);
  753. return ret;
  754. }
  755. static struct platform_driver cpp41_dma_driver;
  756. /*
  757. * The param format is:
  758. * X Y
  759. * X: Port
  760. * Y: 0 = RX else TX
  761. */
  762. #define INFO_PORT 0
  763. #define INFO_IS_TX 1
  764. static bool cpp41_dma_filter_fn(struct dma_chan *chan, void *param)
  765. {
  766. struct cppi41_channel *cchan;
  767. struct cppi41_dd *cdd;
  768. const struct chan_queues *queues;
  769. u32 *num = param;
  770. if (chan->device->dev->driver != &cpp41_dma_driver.driver)
  771. return false;
  772. cchan = to_cpp41_chan(chan);
  773. if (cchan->port_num != num[INFO_PORT])
  774. return false;
  775. if (cchan->is_tx && !num[INFO_IS_TX])
  776. return false;
  777. cdd = cchan->cdd;
  778. if (cchan->is_tx)
  779. queues = cdd->queues_tx;
  780. else
  781. queues = cdd->queues_rx;
  782. BUILD_BUG_ON(ARRAY_SIZE(am335x_usb_queues_rx) !=
  783. ARRAY_SIZE(am335x_usb_queues_tx));
  784. if (WARN_ON(cchan->port_num >= ARRAY_SIZE(am335x_usb_queues_rx)))
  785. return false;
  786. cchan->q_num = queues[cchan->port_num].submit;
  787. cchan->q_comp_num = queues[cchan->port_num].complete;
  788. return true;
  789. }
  790. static struct of_dma_filter_info cpp41_dma_info = {
  791. .filter_fn = cpp41_dma_filter_fn,
  792. };
  793. static struct dma_chan *cppi41_dma_xlate(struct of_phandle_args *dma_spec,
  794. struct of_dma *ofdma)
  795. {
  796. int count = dma_spec->args_count;
  797. struct of_dma_filter_info *info = ofdma->of_dma_data;
  798. if (!info || !info->filter_fn)
  799. return NULL;
  800. if (count != 2)
  801. return NULL;
  802. return dma_request_channel(info->dma_cap, info->filter_fn,
  803. &dma_spec->args[0]);
  804. }
  805. static const struct cppi_glue_infos am335x_usb_infos = {
  806. .queues_rx = am335x_usb_queues_rx,
  807. .queues_tx = am335x_usb_queues_tx,
  808. .td_queue = { .submit = 31, .complete = 0 },
  809. .first_completion_queue = 93,
  810. .qmgr_num_pend = 5,
  811. };
  812. static const struct cppi_glue_infos da8xx_usb_infos = {
  813. .queues_rx = da8xx_usb_queues_rx,
  814. .queues_tx = da8xx_usb_queues_tx,
  815. .td_queue = { .submit = 31, .complete = 0 },
  816. .first_completion_queue = 24,
  817. .qmgr_num_pend = 2,
  818. };
  819. static const struct of_device_id cppi41_dma_ids[] = {
  820. { .compatible = "ti,am3359-cppi41", .data = &am335x_usb_infos},
  821. { .compatible = "ti,da830-cppi41", .data = &da8xx_usb_infos},
  822. {},
  823. };
  824. MODULE_DEVICE_TABLE(of, cppi41_dma_ids);
  825. static const struct cppi_glue_infos *get_glue_info(struct device *dev)
  826. {
  827. const struct of_device_id *of_id;
  828. of_id = of_match_node(cppi41_dma_ids, dev->of_node);
  829. if (!of_id)
  830. return NULL;
  831. return of_id->data;
  832. }
  833. #define CPPI41_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
  834. BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
  835. BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
  836. BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
  837. static int cppi41_dma_probe(struct platform_device *pdev)
  838. {
  839. struct cppi41_dd *cdd;
  840. struct device *dev = &pdev->dev;
  841. const struct cppi_glue_infos *glue_info;
  842. struct resource *mem;
  843. int index;
  844. int irq;
  845. int ret;
  846. glue_info = get_glue_info(dev);
  847. if (!glue_info)
  848. return -EINVAL;
  849. cdd = devm_kzalloc(&pdev->dev, sizeof(*cdd), GFP_KERNEL);
  850. if (!cdd)
  851. return -ENOMEM;
  852. dma_cap_set(DMA_SLAVE, cdd->ddev.cap_mask);
  853. cdd->ddev.device_alloc_chan_resources = cppi41_dma_alloc_chan_resources;
  854. cdd->ddev.device_free_chan_resources = cppi41_dma_free_chan_resources;
  855. cdd->ddev.device_tx_status = cppi41_dma_tx_status;
  856. cdd->ddev.device_issue_pending = cppi41_dma_issue_pending;
  857. cdd->ddev.device_prep_slave_sg = cppi41_dma_prep_slave_sg;
  858. cdd->ddev.device_terminate_all = cppi41_stop_chan;
  859. cdd->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
  860. cdd->ddev.src_addr_widths = CPPI41_DMA_BUSWIDTHS;
  861. cdd->ddev.dst_addr_widths = CPPI41_DMA_BUSWIDTHS;
  862. cdd->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
  863. cdd->ddev.dev = dev;
  864. INIT_LIST_HEAD(&cdd->ddev.channels);
  865. cpp41_dma_info.dma_cap = cdd->ddev.cap_mask;
  866. index = of_property_match_string(dev->of_node,
  867. "reg-names", "controller");
  868. if (index < 0)
  869. return index;
  870. mem = platform_get_resource(pdev, IORESOURCE_MEM, index);
  871. cdd->ctrl_mem = devm_ioremap_resource(dev, mem);
  872. if (IS_ERR(cdd->ctrl_mem))
  873. return PTR_ERR(cdd->ctrl_mem);
  874. mem = platform_get_resource(pdev, IORESOURCE_MEM, index + 1);
  875. cdd->sched_mem = devm_ioremap_resource(dev, mem);
  876. if (IS_ERR(cdd->sched_mem))
  877. return PTR_ERR(cdd->sched_mem);
  878. mem = platform_get_resource(pdev, IORESOURCE_MEM, index + 2);
  879. cdd->qmgr_mem = devm_ioremap_resource(dev, mem);
  880. if (IS_ERR(cdd->qmgr_mem))
  881. return PTR_ERR(cdd->qmgr_mem);
  882. spin_lock_init(&cdd->lock);
  883. INIT_LIST_HEAD(&cdd->pending);
  884. platform_set_drvdata(pdev, cdd);
  885. pm_runtime_enable(dev);
  886. pm_runtime_set_autosuspend_delay(dev, 100);
  887. pm_runtime_use_autosuspend(dev);
  888. ret = pm_runtime_get_sync(dev);
  889. if (ret < 0)
  890. goto err_get_sync;
  891. cdd->queues_rx = glue_info->queues_rx;
  892. cdd->queues_tx = glue_info->queues_tx;
  893. cdd->td_queue = glue_info->td_queue;
  894. cdd->qmgr_num_pend = glue_info->qmgr_num_pend;
  895. cdd->first_completion_queue = glue_info->first_completion_queue;
  896. ret = of_property_read_u32(dev->of_node,
  897. "#dma-channels", &cdd->n_chans);
  898. if (ret)
  899. goto err_get_n_chans;
  900. ret = init_cppi41(dev, cdd);
  901. if (ret)
  902. goto err_init_cppi;
  903. ret = cppi41_add_chans(dev, cdd);
  904. if (ret)
  905. goto err_chans;
  906. irq = irq_of_parse_and_map(dev->of_node, 0);
  907. if (!irq) {
  908. ret = -EINVAL;
  909. goto err_chans;
  910. }
  911. ret = devm_request_irq(&pdev->dev, irq, cppi41_irq, IRQF_SHARED,
  912. dev_name(dev), cdd);
  913. if (ret)
  914. goto err_chans;
  915. cdd->irq = irq;
  916. ret = dma_async_device_register(&cdd->ddev);
  917. if (ret)
  918. goto err_chans;
  919. ret = of_dma_controller_register(dev->of_node,
  920. cppi41_dma_xlate, &cpp41_dma_info);
  921. if (ret)
  922. goto err_of;
  923. pm_runtime_mark_last_busy(dev);
  924. pm_runtime_put_autosuspend(dev);
  925. return 0;
  926. err_of:
  927. dma_async_device_unregister(&cdd->ddev);
  928. err_chans:
  929. deinit_cppi41(dev, cdd);
  930. err_init_cppi:
  931. pm_runtime_dont_use_autosuspend(dev);
  932. err_get_n_chans:
  933. err_get_sync:
  934. pm_runtime_put_sync(dev);
  935. pm_runtime_disable(dev);
  936. return ret;
  937. }
  938. static int cppi41_dma_remove(struct platform_device *pdev)
  939. {
  940. struct cppi41_dd *cdd = platform_get_drvdata(pdev);
  941. int error;
  942. error = pm_runtime_get_sync(&pdev->dev);
  943. if (error < 0)
  944. dev_err(&pdev->dev, "%s could not pm_runtime_get: %i\n",
  945. __func__, error);
  946. of_dma_controller_free(pdev->dev.of_node);
  947. dma_async_device_unregister(&cdd->ddev);
  948. devm_free_irq(&pdev->dev, cdd->irq, cdd);
  949. deinit_cppi41(&pdev->dev, cdd);
  950. pm_runtime_dont_use_autosuspend(&pdev->dev);
  951. pm_runtime_put_sync(&pdev->dev);
  952. pm_runtime_disable(&pdev->dev);
  953. return 0;
  954. }
  955. static int __maybe_unused cppi41_suspend(struct device *dev)
  956. {
  957. struct cppi41_dd *cdd = dev_get_drvdata(dev);
  958. cdd->dma_tdfdq = cppi_readl(cdd->ctrl_mem + DMA_TDFDQ);
  959. disable_sched(cdd);
  960. return 0;
  961. }
  962. static int __maybe_unused cppi41_resume(struct device *dev)
  963. {
  964. struct cppi41_dd *cdd = dev_get_drvdata(dev);
  965. struct cppi41_channel *c;
  966. int i;
  967. for (i = 0; i < DESCS_AREAS; i++)
  968. cppi_writel(cdd->descs_phys, cdd->qmgr_mem + QMGR_MEMBASE(i));
  969. list_for_each_entry(c, &cdd->ddev.channels, chan.device_node)
  970. if (!c->is_tx)
  971. cppi_writel(c->q_num, c->gcr_reg + RXHPCRA0);
  972. init_sched(cdd);
  973. cppi_writel(cdd->dma_tdfdq, cdd->ctrl_mem + DMA_TDFDQ);
  974. cppi_writel(cdd->scratch_phys, cdd->qmgr_mem + QMGR_LRAM0_BASE);
  975. cppi_writel(QMGR_SCRATCH_SIZE, cdd->qmgr_mem + QMGR_LRAM_SIZE);
  976. cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM1_BASE);
  977. return 0;
  978. }
  979. static int __maybe_unused cppi41_runtime_suspend(struct device *dev)
  980. {
  981. struct cppi41_dd *cdd = dev_get_drvdata(dev);
  982. unsigned long flags;
  983. spin_lock_irqsave(&cdd->lock, flags);
  984. cdd->is_suspended = true;
  985. WARN_ON(!list_empty(&cdd->pending));
  986. spin_unlock_irqrestore(&cdd->lock, flags);
  987. return 0;
  988. }
  989. static int __maybe_unused cppi41_runtime_resume(struct device *dev)
  990. {
  991. struct cppi41_dd *cdd = dev_get_drvdata(dev);
  992. unsigned long flags;
  993. spin_lock_irqsave(&cdd->lock, flags);
  994. cdd->is_suspended = false;
  995. cppi41_run_queue(cdd);
  996. spin_unlock_irqrestore(&cdd->lock, flags);
  997. return 0;
  998. }
  999. static const struct dev_pm_ops cppi41_pm_ops = {
  1000. SET_LATE_SYSTEM_SLEEP_PM_OPS(cppi41_suspend, cppi41_resume)
  1001. SET_RUNTIME_PM_OPS(cppi41_runtime_suspend,
  1002. cppi41_runtime_resume,
  1003. NULL)
  1004. };
  1005. static struct platform_driver cpp41_dma_driver = {
  1006. .probe = cppi41_dma_probe,
  1007. .remove = cppi41_dma_remove,
  1008. .driver = {
  1009. .name = "cppi41-dma-engine",
  1010. .pm = &cppi41_pm_ops,
  1011. .of_match_table = of_match_ptr(cppi41_dma_ids),
  1012. },
  1013. };
  1014. module_platform_driver(cpp41_dma_driver);
  1015. MODULE_LICENSE("GPL");
  1016. MODULE_AUTHOR("Sebastian Andrzej Siewior <bigeasy@linutronix.de>");