musb_cppi41.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813
  1. // SPDX-License-Identifier: GPL-2.0
  2. #include <linux/device.h>
  3. #include <linux/dma-mapping.h>
  4. #include <linux/dmaengine.h>
  5. #include <linux/sizes.h>
  6. #include <linux/platform_device.h>
  7. #include <linux/of.h>
  8. #include "cppi_dma.h"
  9. #include "musb_core.h"
  10. #include "musb_trace.h"
  11. #define RNDIS_REG(x) (0x80 + ((x - 1) * 4))
  12. #define EP_MODE_AUTOREQ_NONE 0
  13. #define EP_MODE_AUTOREQ_ALL_NEOP 1
  14. #define EP_MODE_AUTOREQ_ALWAYS 3
  15. #define EP_MODE_DMA_TRANSPARENT 0
  16. #define EP_MODE_DMA_RNDIS 1
  17. #define EP_MODE_DMA_GEN_RNDIS 3
  18. #define USB_CTRL_TX_MODE 0x70
  19. #define USB_CTRL_RX_MODE 0x74
  20. #define USB_CTRL_AUTOREQ 0xd0
  21. #define USB_TDOWN 0xd8
  22. #define MUSB_DMA_NUM_CHANNELS 15
  23. #define DA8XX_USB_MODE 0x10
  24. #define DA8XX_USB_AUTOREQ 0x14
  25. #define DA8XX_USB_TEARDOWN 0x1c
  26. #define DA8XX_DMA_NUM_CHANNELS 4
  27. struct cppi41_dma_controller {
  28. struct dma_controller controller;
  29. struct cppi41_dma_channel *rx_channel;
  30. struct cppi41_dma_channel *tx_channel;
  31. struct hrtimer early_tx;
  32. struct list_head early_tx_list;
  33. u32 rx_mode;
  34. u32 tx_mode;
  35. u32 auto_req;
  36. u32 tdown_reg;
  37. u32 autoreq_reg;
  38. void (*set_dma_mode)(struct cppi41_dma_channel *cppi41_channel,
  39. unsigned int mode);
  40. u8 num_channels;
  41. };
  42. static void save_rx_toggle(struct cppi41_dma_channel *cppi41_channel)
  43. {
  44. u16 csr;
  45. u8 toggle;
  46. if (cppi41_channel->is_tx)
  47. return;
  48. if (!is_host_active(cppi41_channel->controller->controller.musb))
  49. return;
  50. csr = musb_readw(cppi41_channel->hw_ep->regs, MUSB_RXCSR);
  51. toggle = csr & MUSB_RXCSR_H_DATATOGGLE ? 1 : 0;
  52. cppi41_channel->usb_toggle = toggle;
  53. }
  54. static void update_rx_toggle(struct cppi41_dma_channel *cppi41_channel)
  55. {
  56. struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep;
  57. struct musb *musb = hw_ep->musb;
  58. u16 csr;
  59. u8 toggle;
  60. if (cppi41_channel->is_tx)
  61. return;
  62. if (!is_host_active(musb))
  63. return;
  64. musb_ep_select(musb->mregs, hw_ep->epnum);
  65. csr = musb_readw(hw_ep->regs, MUSB_RXCSR);
  66. toggle = csr & MUSB_RXCSR_H_DATATOGGLE ? 1 : 0;
  67. /*
  68. * AM335x Advisory 1.0.13: Due to internal synchronisation error the
  69. * data toggle may reset from DATA1 to DATA0 during receiving data from
  70. * more than one endpoint.
  71. */
  72. if (!toggle && toggle == cppi41_channel->usb_toggle) {
  73. csr |= MUSB_RXCSR_H_DATATOGGLE | MUSB_RXCSR_H_WR_DATATOGGLE;
  74. musb_writew(cppi41_channel->hw_ep->regs, MUSB_RXCSR, csr);
  75. musb_dbg(musb, "Restoring DATA1 toggle.");
  76. }
  77. cppi41_channel->usb_toggle = toggle;
  78. }
  79. static bool musb_is_tx_fifo_empty(struct musb_hw_ep *hw_ep)
  80. {
  81. u8 epnum = hw_ep->epnum;
  82. struct musb *musb = hw_ep->musb;
  83. void __iomem *epio = musb->endpoints[epnum].regs;
  84. u16 csr;
  85. musb_ep_select(musb->mregs, hw_ep->epnum);
  86. csr = musb_readw(epio, MUSB_TXCSR);
  87. if (csr & MUSB_TXCSR_TXPKTRDY)
  88. return false;
  89. return true;
  90. }
  91. static void cppi41_dma_callback(void *private_data,
  92. const struct dmaengine_result *result);
  93. static void cppi41_trans_done(struct cppi41_dma_channel *cppi41_channel)
  94. {
  95. struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep;
  96. struct musb *musb = hw_ep->musb;
  97. void __iomem *epio = hw_ep->regs;
  98. u16 csr;
  99. if (!cppi41_channel->prog_len ||
  100. (cppi41_channel->channel.status == MUSB_DMA_STATUS_FREE)) {
  101. /* done, complete */
  102. cppi41_channel->channel.actual_len =
  103. cppi41_channel->transferred;
  104. cppi41_channel->channel.status = MUSB_DMA_STATUS_FREE;
  105. cppi41_channel->channel.rx_packet_done = true;
  106. /*
  107. * transmit ZLP using PIO mode for transfers which size is
  108. * multiple of EP packet size.
  109. */
  110. if (cppi41_channel->tx_zlp && (cppi41_channel->transferred %
  111. cppi41_channel->packet_sz) == 0) {
  112. musb_ep_select(musb->mregs, hw_ep->epnum);
  113. csr = MUSB_TXCSR_MODE | MUSB_TXCSR_TXPKTRDY;
  114. musb_writew(epio, MUSB_TXCSR, csr);
  115. }
  116. trace_musb_cppi41_done(cppi41_channel);
  117. musb_dma_completion(musb, hw_ep->epnum, cppi41_channel->is_tx);
  118. } else {
  119. /* next iteration, reload */
  120. struct dma_chan *dc = cppi41_channel->dc;
  121. struct dma_async_tx_descriptor *dma_desc;
  122. enum dma_transfer_direction direction;
  123. u32 remain_bytes;
  124. cppi41_channel->buf_addr += cppi41_channel->packet_sz;
  125. remain_bytes = cppi41_channel->total_len;
  126. remain_bytes -= cppi41_channel->transferred;
  127. remain_bytes = min(remain_bytes, cppi41_channel->packet_sz);
  128. cppi41_channel->prog_len = remain_bytes;
  129. direction = cppi41_channel->is_tx ? DMA_MEM_TO_DEV
  130. : DMA_DEV_TO_MEM;
  131. dma_desc = dmaengine_prep_slave_single(dc,
  132. cppi41_channel->buf_addr,
  133. remain_bytes,
  134. direction,
  135. DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
  136. if (WARN_ON(!dma_desc))
  137. return;
  138. dma_desc->callback_result = cppi41_dma_callback;
  139. dma_desc->callback_param = &cppi41_channel->channel;
  140. cppi41_channel->cookie = dma_desc->tx_submit(dma_desc);
  141. trace_musb_cppi41_cont(cppi41_channel);
  142. dma_async_issue_pending(dc);
  143. if (!cppi41_channel->is_tx) {
  144. musb_ep_select(musb->mregs, hw_ep->epnum);
  145. csr = musb_readw(epio, MUSB_RXCSR);
  146. csr |= MUSB_RXCSR_H_REQPKT;
  147. musb_writew(epio, MUSB_RXCSR, csr);
  148. }
  149. }
  150. }
  151. static enum hrtimer_restart cppi41_recheck_tx_req(struct hrtimer *timer)
  152. {
  153. struct cppi41_dma_controller *controller;
  154. struct cppi41_dma_channel *cppi41_channel, *n;
  155. struct musb *musb;
  156. unsigned long flags;
  157. enum hrtimer_restart ret = HRTIMER_NORESTART;
  158. controller = container_of(timer, struct cppi41_dma_controller,
  159. early_tx);
  160. musb = controller->controller.musb;
  161. spin_lock_irqsave(&musb->lock, flags);
  162. list_for_each_entry_safe(cppi41_channel, n, &controller->early_tx_list,
  163. tx_check) {
  164. bool empty;
  165. struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep;
  166. empty = musb_is_tx_fifo_empty(hw_ep);
  167. if (empty) {
  168. list_del_init(&cppi41_channel->tx_check);
  169. cppi41_trans_done(cppi41_channel);
  170. }
  171. }
  172. if (!list_empty(&controller->early_tx_list) &&
  173. !hrtimer_is_queued(&controller->early_tx)) {
  174. ret = HRTIMER_RESTART;
  175. hrtimer_forward_now(&controller->early_tx, 20 * NSEC_PER_USEC);
  176. }
  177. spin_unlock_irqrestore(&musb->lock, flags);
  178. return ret;
  179. }
  180. static void cppi41_dma_callback(void *private_data,
  181. const struct dmaengine_result *result)
  182. {
  183. struct dma_channel *channel = private_data;
  184. struct cppi41_dma_channel *cppi41_channel = channel->private_data;
  185. struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep;
  186. struct cppi41_dma_controller *controller;
  187. struct musb *musb = hw_ep->musb;
  188. unsigned long flags;
  189. struct dma_tx_state txstate;
  190. u32 transferred;
  191. int is_hs = 0;
  192. bool empty;
  193. controller = cppi41_channel->controller;
  194. if (controller->controller.dma_callback)
  195. controller->controller.dma_callback(&controller->controller);
  196. if (result->result == DMA_TRANS_ABORTED)
  197. return;
  198. spin_lock_irqsave(&musb->lock, flags);
  199. dmaengine_tx_status(cppi41_channel->dc, cppi41_channel->cookie,
  200. &txstate);
  201. transferred = cppi41_channel->prog_len - txstate.residue;
  202. cppi41_channel->transferred += transferred;
  203. trace_musb_cppi41_gb(cppi41_channel);
  204. update_rx_toggle(cppi41_channel);
  205. if (cppi41_channel->transferred == cppi41_channel->total_len ||
  206. transferred < cppi41_channel->packet_sz)
  207. cppi41_channel->prog_len = 0;
  208. if (cppi41_channel->is_tx) {
  209. u8 type;
  210. if (is_host_active(musb))
  211. type = hw_ep->out_qh->type;
  212. else
  213. type = hw_ep->ep_in.type;
  214. if (type == USB_ENDPOINT_XFER_ISOC)
  215. /*
  216. * Don't use the early-TX-interrupt workaround below
  217. * for Isoch transfter. Since Isoch are periodic
  218. * transfer, by the time the next transfer is
  219. * scheduled, the current one should be done already.
  220. *
  221. * This avoids audio playback underrun issue.
  222. */
  223. empty = true;
  224. else
  225. empty = musb_is_tx_fifo_empty(hw_ep);
  226. }
  227. if (!cppi41_channel->is_tx || empty) {
  228. cppi41_trans_done(cppi41_channel);
  229. goto out;
  230. }
  231. /*
  232. * On AM335x it has been observed that the TX interrupt fires
  233. * too early that means the TXFIFO is not yet empty but the DMA
  234. * engine says that it is done with the transfer. We don't
  235. * receive a FIFO empty interrupt so the only thing we can do is
  236. * to poll for the bit. On HS it usually takes 2us, on FS around
  237. * 110us - 150us depending on the transfer size.
  238. * We spin on HS (no longer than than 25us and setup a timer on
  239. * FS to check for the bit and complete the transfer.
  240. */
  241. if (is_host_active(musb)) {
  242. if (musb->port1_status & USB_PORT_STAT_HIGH_SPEED)
  243. is_hs = 1;
  244. } else {
  245. if (musb->g.speed == USB_SPEED_HIGH)
  246. is_hs = 1;
  247. }
  248. if (is_hs) {
  249. unsigned wait = 25;
  250. do {
  251. empty = musb_is_tx_fifo_empty(hw_ep);
  252. if (empty) {
  253. cppi41_trans_done(cppi41_channel);
  254. goto out;
  255. }
  256. wait--;
  257. if (!wait)
  258. break;
  259. cpu_relax();
  260. } while (1);
  261. }
  262. list_add_tail(&cppi41_channel->tx_check,
  263. &controller->early_tx_list);
  264. if (!hrtimer_is_queued(&controller->early_tx)) {
  265. unsigned long usecs = cppi41_channel->total_len / 10;
  266. hrtimer_start_range_ns(&controller->early_tx,
  267. usecs * NSEC_PER_USEC,
  268. 20 * NSEC_PER_USEC,
  269. HRTIMER_MODE_REL);
  270. }
  271. out:
  272. spin_unlock_irqrestore(&musb->lock, flags);
  273. }
  274. static u32 update_ep_mode(unsigned ep, unsigned mode, u32 old)
  275. {
  276. unsigned shift;
  277. shift = (ep - 1) * 2;
  278. old &= ~(3 << shift);
  279. old |= mode << shift;
  280. return old;
  281. }
  282. static void cppi41_set_dma_mode(struct cppi41_dma_channel *cppi41_channel,
  283. unsigned mode)
  284. {
  285. struct cppi41_dma_controller *controller = cppi41_channel->controller;
  286. struct musb *musb = controller->controller.musb;
  287. u32 port;
  288. u32 new_mode;
  289. u32 old_mode;
  290. if (cppi41_channel->is_tx)
  291. old_mode = controller->tx_mode;
  292. else
  293. old_mode = controller->rx_mode;
  294. port = cppi41_channel->port_num;
  295. new_mode = update_ep_mode(port, mode, old_mode);
  296. if (new_mode == old_mode)
  297. return;
  298. if (cppi41_channel->is_tx) {
  299. controller->tx_mode = new_mode;
  300. musb_writel(musb->ctrl_base, USB_CTRL_TX_MODE, new_mode);
  301. } else {
  302. controller->rx_mode = new_mode;
  303. musb_writel(musb->ctrl_base, USB_CTRL_RX_MODE, new_mode);
  304. }
  305. }
  306. static void da8xx_set_dma_mode(struct cppi41_dma_channel *cppi41_channel,
  307. unsigned int mode)
  308. {
  309. struct cppi41_dma_controller *controller = cppi41_channel->controller;
  310. struct musb *musb = controller->controller.musb;
  311. unsigned int shift;
  312. u32 port;
  313. u32 new_mode;
  314. u32 old_mode;
  315. old_mode = controller->tx_mode;
  316. port = cppi41_channel->port_num;
  317. shift = (port - 1) * 4;
  318. if (!cppi41_channel->is_tx)
  319. shift += 16;
  320. new_mode = old_mode & ~(3 << shift);
  321. new_mode |= mode << shift;
  322. if (new_mode == old_mode)
  323. return;
  324. controller->tx_mode = new_mode;
  325. musb_writel(musb->ctrl_base, DA8XX_USB_MODE, new_mode);
  326. }
  327. static void cppi41_set_autoreq_mode(struct cppi41_dma_channel *cppi41_channel,
  328. unsigned mode)
  329. {
  330. struct cppi41_dma_controller *controller = cppi41_channel->controller;
  331. u32 port;
  332. u32 new_mode;
  333. u32 old_mode;
  334. old_mode = controller->auto_req;
  335. port = cppi41_channel->port_num;
  336. new_mode = update_ep_mode(port, mode, old_mode);
  337. if (new_mode == old_mode)
  338. return;
  339. controller->auto_req = new_mode;
  340. musb_writel(controller->controller.musb->ctrl_base,
  341. controller->autoreq_reg, new_mode);
  342. }
  343. static bool cppi41_configure_channel(struct dma_channel *channel,
  344. u16 packet_sz, u8 mode,
  345. dma_addr_t dma_addr, u32 len)
  346. {
  347. struct cppi41_dma_channel *cppi41_channel = channel->private_data;
  348. struct cppi41_dma_controller *controller = cppi41_channel->controller;
  349. struct dma_chan *dc = cppi41_channel->dc;
  350. struct dma_async_tx_descriptor *dma_desc;
  351. enum dma_transfer_direction direction;
  352. struct musb *musb = cppi41_channel->controller->controller.musb;
  353. unsigned use_gen_rndis = 0;
  354. cppi41_channel->buf_addr = dma_addr;
  355. cppi41_channel->total_len = len;
  356. cppi41_channel->transferred = 0;
  357. cppi41_channel->packet_sz = packet_sz;
  358. cppi41_channel->tx_zlp = (cppi41_channel->is_tx && mode) ? 1 : 0;
  359. /*
  360. * Due to AM335x' Advisory 1.0.13 we are not allowed to transfer more
  361. * than max packet size at a time.
  362. */
  363. if (cppi41_channel->is_tx)
  364. use_gen_rndis = 1;
  365. if (use_gen_rndis) {
  366. /* RNDIS mode */
  367. if (len > packet_sz) {
  368. musb_writel(musb->ctrl_base,
  369. RNDIS_REG(cppi41_channel->port_num), len);
  370. /* gen rndis */
  371. controller->set_dma_mode(cppi41_channel,
  372. EP_MODE_DMA_GEN_RNDIS);
  373. /* auto req */
  374. cppi41_set_autoreq_mode(cppi41_channel,
  375. EP_MODE_AUTOREQ_ALL_NEOP);
  376. } else {
  377. musb_writel(musb->ctrl_base,
  378. RNDIS_REG(cppi41_channel->port_num), 0);
  379. controller->set_dma_mode(cppi41_channel,
  380. EP_MODE_DMA_TRANSPARENT);
  381. cppi41_set_autoreq_mode(cppi41_channel,
  382. EP_MODE_AUTOREQ_NONE);
  383. }
  384. } else {
  385. /* fallback mode */
  386. controller->set_dma_mode(cppi41_channel,
  387. EP_MODE_DMA_TRANSPARENT);
  388. cppi41_set_autoreq_mode(cppi41_channel, EP_MODE_AUTOREQ_NONE);
  389. len = min_t(u32, packet_sz, len);
  390. }
  391. cppi41_channel->prog_len = len;
  392. direction = cppi41_channel->is_tx ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM;
  393. dma_desc = dmaengine_prep_slave_single(dc, dma_addr, len, direction,
  394. DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
  395. if (!dma_desc)
  396. return false;
  397. dma_desc->callback_result = cppi41_dma_callback;
  398. dma_desc->callback_param = channel;
  399. cppi41_channel->cookie = dma_desc->tx_submit(dma_desc);
  400. cppi41_channel->channel.rx_packet_done = false;
  401. trace_musb_cppi41_config(cppi41_channel);
  402. save_rx_toggle(cppi41_channel);
  403. dma_async_issue_pending(dc);
  404. return true;
  405. }
  406. static struct dma_channel *cppi41_dma_channel_allocate(struct dma_controller *c,
  407. struct musb_hw_ep *hw_ep, u8 is_tx)
  408. {
  409. struct cppi41_dma_controller *controller = container_of(c,
  410. struct cppi41_dma_controller, controller);
  411. struct cppi41_dma_channel *cppi41_channel = NULL;
  412. u8 ch_num = hw_ep->epnum - 1;
  413. if (ch_num >= controller->num_channels)
  414. return NULL;
  415. if (is_tx)
  416. cppi41_channel = &controller->tx_channel[ch_num];
  417. else
  418. cppi41_channel = &controller->rx_channel[ch_num];
  419. if (!cppi41_channel->dc)
  420. return NULL;
  421. if (cppi41_channel->is_allocated)
  422. return NULL;
  423. cppi41_channel->hw_ep = hw_ep;
  424. cppi41_channel->is_allocated = 1;
  425. trace_musb_cppi41_alloc(cppi41_channel);
  426. return &cppi41_channel->channel;
  427. }
  428. static void cppi41_dma_channel_release(struct dma_channel *channel)
  429. {
  430. struct cppi41_dma_channel *cppi41_channel = channel->private_data;
  431. trace_musb_cppi41_free(cppi41_channel);
  432. if (cppi41_channel->is_allocated) {
  433. cppi41_channel->is_allocated = 0;
  434. channel->status = MUSB_DMA_STATUS_FREE;
  435. channel->actual_len = 0;
  436. }
  437. }
  438. static int cppi41_dma_channel_program(struct dma_channel *channel,
  439. u16 packet_sz, u8 mode,
  440. dma_addr_t dma_addr, u32 len)
  441. {
  442. int ret;
  443. struct cppi41_dma_channel *cppi41_channel = channel->private_data;
  444. int hb_mult = 0;
  445. BUG_ON(channel->status == MUSB_DMA_STATUS_UNKNOWN ||
  446. channel->status == MUSB_DMA_STATUS_BUSY);
  447. if (is_host_active(cppi41_channel->controller->controller.musb)) {
  448. if (cppi41_channel->is_tx)
  449. hb_mult = cppi41_channel->hw_ep->out_qh->hb_mult;
  450. else
  451. hb_mult = cppi41_channel->hw_ep->in_qh->hb_mult;
  452. }
  453. channel->status = MUSB_DMA_STATUS_BUSY;
  454. channel->actual_len = 0;
  455. if (hb_mult)
  456. packet_sz = hb_mult * (packet_sz & 0x7FF);
  457. ret = cppi41_configure_channel(channel, packet_sz, mode, dma_addr, len);
  458. if (!ret)
  459. channel->status = MUSB_DMA_STATUS_FREE;
  460. return ret;
  461. }
  462. static int cppi41_is_compatible(struct dma_channel *channel, u16 maxpacket,
  463. void *buf, u32 length)
  464. {
  465. struct cppi41_dma_channel *cppi41_channel = channel->private_data;
  466. struct cppi41_dma_controller *controller = cppi41_channel->controller;
  467. struct musb *musb = controller->controller.musb;
  468. if (is_host_active(musb)) {
  469. WARN_ON(1);
  470. return 1;
  471. }
  472. if (cppi41_channel->hw_ep->ep_in.type != USB_ENDPOINT_XFER_BULK)
  473. return 0;
  474. if (cppi41_channel->is_tx)
  475. return 1;
  476. /* AM335x Advisory 1.0.13. No workaround for device RX mode */
  477. return 0;
  478. }
  479. static int cppi41_dma_channel_abort(struct dma_channel *channel)
  480. {
  481. struct cppi41_dma_channel *cppi41_channel = channel->private_data;
  482. struct cppi41_dma_controller *controller = cppi41_channel->controller;
  483. struct musb *musb = controller->controller.musb;
  484. void __iomem *epio = cppi41_channel->hw_ep->regs;
  485. int tdbit;
  486. int ret;
  487. unsigned is_tx;
  488. u16 csr;
  489. is_tx = cppi41_channel->is_tx;
  490. trace_musb_cppi41_abort(cppi41_channel);
  491. if (cppi41_channel->channel.status == MUSB_DMA_STATUS_FREE)
  492. return 0;
  493. list_del_init(&cppi41_channel->tx_check);
  494. if (is_tx) {
  495. csr = musb_readw(epio, MUSB_TXCSR);
  496. csr &= ~MUSB_TXCSR_DMAENAB;
  497. musb_writew(epio, MUSB_TXCSR, csr);
  498. } else {
  499. cppi41_set_autoreq_mode(cppi41_channel, EP_MODE_AUTOREQ_NONE);
  500. /* delay to drain to cppi dma pipeline for isoch */
  501. udelay(250);
  502. csr = musb_readw(epio, MUSB_RXCSR);
  503. csr &= ~(MUSB_RXCSR_H_REQPKT | MUSB_RXCSR_DMAENAB);
  504. musb_writew(epio, MUSB_RXCSR, csr);
  505. /* wait to drain cppi dma pipe line */
  506. udelay(50);
  507. csr = musb_readw(epio, MUSB_RXCSR);
  508. if (csr & MUSB_RXCSR_RXPKTRDY) {
  509. csr |= MUSB_RXCSR_FLUSHFIFO;
  510. musb_writew(epio, MUSB_RXCSR, csr);
  511. musb_writew(epio, MUSB_RXCSR, csr);
  512. }
  513. }
  514. /* DA8xx Advisory 2.3.27: wait 250 ms before to start the teardown */
  515. if (musb->ops->quirks & MUSB_DA8XX)
  516. mdelay(250);
  517. tdbit = 1 << cppi41_channel->port_num;
  518. if (is_tx)
  519. tdbit <<= 16;
  520. do {
  521. if (is_tx)
  522. musb_writel(musb->ctrl_base, controller->tdown_reg,
  523. tdbit);
  524. ret = dmaengine_terminate_all(cppi41_channel->dc);
  525. } while (ret == -EAGAIN);
  526. if (is_tx) {
  527. musb_writel(musb->ctrl_base, controller->tdown_reg, tdbit);
  528. csr = musb_readw(epio, MUSB_TXCSR);
  529. if (csr & MUSB_TXCSR_TXPKTRDY) {
  530. csr |= MUSB_TXCSR_FLUSHFIFO;
  531. musb_writew(epio, MUSB_TXCSR, csr);
  532. }
  533. }
  534. cppi41_channel->channel.status = MUSB_DMA_STATUS_FREE;
  535. return 0;
  536. }
  537. static void cppi41_release_all_dma_chans(struct cppi41_dma_controller *ctrl)
  538. {
  539. struct dma_chan *dc;
  540. int i;
  541. for (i = 0; i < ctrl->num_channels; i++) {
  542. dc = ctrl->tx_channel[i].dc;
  543. if (dc)
  544. dma_release_channel(dc);
  545. dc = ctrl->rx_channel[i].dc;
  546. if (dc)
  547. dma_release_channel(dc);
  548. }
  549. }
  550. static void cppi41_dma_controller_stop(struct cppi41_dma_controller *controller)
  551. {
  552. cppi41_release_all_dma_chans(controller);
  553. }
  554. static int cppi41_dma_controller_start(struct cppi41_dma_controller *controller)
  555. {
  556. struct musb *musb = controller->controller.musb;
  557. struct device *dev = musb->controller;
  558. struct device_node *np = dev->parent->of_node;
  559. struct cppi41_dma_channel *cppi41_channel;
  560. int count;
  561. int i;
  562. int ret;
  563. count = of_property_count_strings(np, "dma-names");
  564. if (count < 0)
  565. return count;
  566. for (i = 0; i < count; i++) {
  567. struct dma_chan *dc;
  568. struct dma_channel *musb_dma;
  569. const char *str;
  570. unsigned is_tx;
  571. unsigned int port;
  572. ret = of_property_read_string_index(np, "dma-names", i, &str);
  573. if (ret)
  574. goto err;
  575. if (strstarts(str, "tx"))
  576. is_tx = 1;
  577. else if (strstarts(str, "rx"))
  578. is_tx = 0;
  579. else {
  580. dev_err(dev, "Wrong dmatype %s\n", str);
  581. goto err;
  582. }
  583. ret = kstrtouint(str + 2, 0, &port);
  584. if (ret)
  585. goto err;
  586. ret = -EINVAL;
  587. if (port > controller->num_channels || !port)
  588. goto err;
  589. if (is_tx)
  590. cppi41_channel = &controller->tx_channel[port - 1];
  591. else
  592. cppi41_channel = &controller->rx_channel[port - 1];
  593. cppi41_channel->controller = controller;
  594. cppi41_channel->port_num = port;
  595. cppi41_channel->is_tx = is_tx;
  596. INIT_LIST_HEAD(&cppi41_channel->tx_check);
  597. musb_dma = &cppi41_channel->channel;
  598. musb_dma->private_data = cppi41_channel;
  599. musb_dma->status = MUSB_DMA_STATUS_FREE;
  600. musb_dma->max_len = SZ_4M;
  601. dc = dma_request_chan(dev->parent, str);
  602. if (IS_ERR(dc)) {
  603. ret = PTR_ERR(dc);
  604. if (ret != -EPROBE_DEFER)
  605. dev_err(dev, "Failed to request %s: %d.\n",
  606. str, ret);
  607. goto err;
  608. }
  609. cppi41_channel->dc = dc;
  610. }
  611. return 0;
  612. err:
  613. cppi41_release_all_dma_chans(controller);
  614. return ret;
  615. }
  616. void cppi41_dma_controller_destroy(struct dma_controller *c)
  617. {
  618. struct cppi41_dma_controller *controller = container_of(c,
  619. struct cppi41_dma_controller, controller);
  620. hrtimer_cancel(&controller->early_tx);
  621. cppi41_dma_controller_stop(controller);
  622. kfree(controller->rx_channel);
  623. kfree(controller->tx_channel);
  624. kfree(controller);
  625. }
  626. EXPORT_SYMBOL_GPL(cppi41_dma_controller_destroy);
  627. struct dma_controller *
  628. cppi41_dma_controller_create(struct musb *musb, void __iomem *base)
  629. {
  630. struct cppi41_dma_controller *controller;
  631. int channel_size;
  632. int ret = 0;
  633. if (!musb->controller->parent->of_node) {
  634. dev_err(musb->controller, "Need DT for the DMA engine.\n");
  635. return NULL;
  636. }
  637. controller = kzalloc(sizeof(*controller), GFP_KERNEL);
  638. if (!controller)
  639. goto kzalloc_fail;
  640. hrtimer_init(&controller->early_tx, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  641. controller->early_tx.function = cppi41_recheck_tx_req;
  642. INIT_LIST_HEAD(&controller->early_tx_list);
  643. controller->controller.channel_alloc = cppi41_dma_channel_allocate;
  644. controller->controller.channel_release = cppi41_dma_channel_release;
  645. controller->controller.channel_program = cppi41_dma_channel_program;
  646. controller->controller.channel_abort = cppi41_dma_channel_abort;
  647. controller->controller.is_compatible = cppi41_is_compatible;
  648. controller->controller.musb = musb;
  649. if (musb->ops->quirks & MUSB_DA8XX) {
  650. controller->tdown_reg = DA8XX_USB_TEARDOWN;
  651. controller->autoreq_reg = DA8XX_USB_AUTOREQ;
  652. controller->set_dma_mode = da8xx_set_dma_mode;
  653. controller->num_channels = DA8XX_DMA_NUM_CHANNELS;
  654. } else {
  655. controller->tdown_reg = USB_TDOWN;
  656. controller->autoreq_reg = USB_CTRL_AUTOREQ;
  657. controller->set_dma_mode = cppi41_set_dma_mode;
  658. controller->num_channels = MUSB_DMA_NUM_CHANNELS;
  659. }
  660. channel_size = controller->num_channels *
  661. sizeof(struct cppi41_dma_channel);
  662. controller->rx_channel = kzalloc(channel_size, GFP_KERNEL);
  663. if (!controller->rx_channel)
  664. goto rx_channel_alloc_fail;
  665. controller->tx_channel = kzalloc(channel_size, GFP_KERNEL);
  666. if (!controller->tx_channel)
  667. goto tx_channel_alloc_fail;
  668. ret = cppi41_dma_controller_start(controller);
  669. if (ret)
  670. goto plat_get_fail;
  671. return &controller->controller;
  672. plat_get_fail:
  673. kfree(controller->tx_channel);
  674. tx_channel_alloc_fail:
  675. kfree(controller->rx_channel);
  676. rx_channel_alloc_fail:
  677. kfree(controller);
  678. kzalloc_fail:
  679. if (ret == -EPROBE_DEFER)
  680. return ERR_PTR(ret);
  681. return NULL;
  682. }
  683. EXPORT_SYMBOL_GPL(cppi41_dma_controller_create);