cppi_dma.c 44 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (C) 2005-2006 by Texas Instruments
  4. *
  5. * This file implements a DMA interface using TI's CPPI DMA.
  6. * For now it's DaVinci-only, but CPPI isn't specific to DaVinci or USB.
  7. * The TUSB6020, using VLYNQ, has CPPI that looks much like DaVinci.
  8. */
  9. #include <linux/module.h>
  10. #include <linux/platform_device.h>
  11. #include <linux/slab.h>
  12. #include <linux/usb.h>
  13. #include "musb_core.h"
  14. #include "musb_debug.h"
  15. #include "cppi_dma.h"
  16. #include "davinci.h"
  17. /* CPPI DMA status 7-mar-2006:
  18. *
  19. * - See musb_{host,gadget}.c for more info
  20. *
  21. * - Correct RX DMA generally forces the engine into irq-per-packet mode,
  22. * which can easily saturate the CPU under non-mass-storage loads.
  23. *
  24. * NOTES 24-aug-2006 (2.6.18-rc4):
  25. *
  26. * - peripheral RXDMA wedged in a test with packets of length 512/512/1.
  27. * evidently after the 1 byte packet was received and acked, the queue
  28. * of BDs got garbaged so it wouldn't empty the fifo. (rxcsr 0x2003,
  29. * and RX DMA0: 4 left, 80000000 8feff880, 8feff860 8feff860; 8f321401
  30. * 004001ff 00000001 .. 8feff860) Host was just getting NAKed on tx
  31. * of its next (512 byte) packet. IRQ issues?
  32. *
  33. * REVISIT: the "transfer DMA" glue between CPPI and USB fifos will
  34. * evidently also directly update the RX and TX CSRs ... so audit all
  35. * host and peripheral side DMA code to avoid CSR access after DMA has
  36. * been started.
  37. */
  38. /* REVISIT now we can avoid preallocating these descriptors; or
  39. * more simply, switch to a global freelist not per-channel ones.
  40. * Note: at full speed, 64 descriptors == 4K bulk data.
  41. */
  42. #define NUM_TXCHAN_BD 64
  43. #define NUM_RXCHAN_BD 64
  44. static inline void cpu_drain_writebuffer(void)
  45. {
  46. wmb();
  47. #ifdef CONFIG_CPU_ARM926T
  48. /* REVISIT this "should not be needed",
  49. * but lack of it sure seemed to hurt ...
  50. */
  51. asm("mcr p15, 0, r0, c7, c10, 4 @ drain write buffer\n");
  52. #endif
  53. }
  54. static inline struct cppi_descriptor *cppi_bd_alloc(struct cppi_channel *c)
  55. {
  56. struct cppi_descriptor *bd = c->freelist;
  57. if (bd)
  58. c->freelist = bd->next;
  59. return bd;
  60. }
  61. static inline void
  62. cppi_bd_free(struct cppi_channel *c, struct cppi_descriptor *bd)
  63. {
  64. if (!bd)
  65. return;
  66. bd->next = c->freelist;
  67. c->freelist = bd;
  68. }
  69. /*
  70. * Start DMA controller
  71. *
  72. * Initialize the DMA controller as necessary.
  73. */
  74. /* zero out entire rx state RAM entry for the channel */
  75. static void cppi_reset_rx(struct cppi_rx_stateram __iomem *rx)
  76. {
  77. musb_writel(&rx->rx_skipbytes, 0, 0);
  78. musb_writel(&rx->rx_head, 0, 0);
  79. musb_writel(&rx->rx_sop, 0, 0);
  80. musb_writel(&rx->rx_current, 0, 0);
  81. musb_writel(&rx->rx_buf_current, 0, 0);
  82. musb_writel(&rx->rx_len_len, 0, 0);
  83. musb_writel(&rx->rx_cnt_cnt, 0, 0);
  84. }
  85. /* zero out entire tx state RAM entry for the channel */
  86. static void cppi_reset_tx(struct cppi_tx_stateram __iomem *tx, u32 ptr)
  87. {
  88. musb_writel(&tx->tx_head, 0, 0);
  89. musb_writel(&tx->tx_buf, 0, 0);
  90. musb_writel(&tx->tx_current, 0, 0);
  91. musb_writel(&tx->tx_buf_current, 0, 0);
  92. musb_writel(&tx->tx_info, 0, 0);
  93. musb_writel(&tx->tx_rem_len, 0, 0);
  94. /* musb_writel(&tx->tx_dummy, 0, 0); */
  95. musb_writel(&tx->tx_complete, 0, ptr);
  96. }
  97. static void cppi_pool_init(struct cppi *cppi, struct cppi_channel *c)
  98. {
  99. int j;
  100. /* initialize channel fields */
  101. c->head = NULL;
  102. c->tail = NULL;
  103. c->last_processed = NULL;
  104. c->channel.status = MUSB_DMA_STATUS_UNKNOWN;
  105. c->controller = cppi;
  106. c->is_rndis = 0;
  107. c->freelist = NULL;
  108. /* build the BD Free list for the channel */
  109. for (j = 0; j < NUM_TXCHAN_BD + 1; j++) {
  110. struct cppi_descriptor *bd;
  111. dma_addr_t dma;
  112. bd = dma_pool_alloc(cppi->pool, GFP_KERNEL, &dma);
  113. bd->dma = dma;
  114. cppi_bd_free(c, bd);
  115. }
  116. }
  117. static int cppi_channel_abort(struct dma_channel *);
  118. static void cppi_pool_free(struct cppi_channel *c)
  119. {
  120. struct cppi *cppi = c->controller;
  121. struct cppi_descriptor *bd;
  122. (void) cppi_channel_abort(&c->channel);
  123. c->channel.status = MUSB_DMA_STATUS_UNKNOWN;
  124. c->controller = NULL;
  125. /* free all its bds */
  126. bd = c->last_processed;
  127. do {
  128. if (bd)
  129. dma_pool_free(cppi->pool, bd, bd->dma);
  130. bd = cppi_bd_alloc(c);
  131. } while (bd);
  132. c->last_processed = NULL;
  133. }
  134. static void cppi_controller_start(struct cppi *controller)
  135. {
  136. void __iomem *tibase;
  137. int i;
  138. /* do whatever is necessary to start controller */
  139. for (i = 0; i < ARRAY_SIZE(controller->tx); i++) {
  140. controller->tx[i].transmit = true;
  141. controller->tx[i].index = i;
  142. }
  143. for (i = 0; i < ARRAY_SIZE(controller->rx); i++) {
  144. controller->rx[i].transmit = false;
  145. controller->rx[i].index = i;
  146. }
  147. /* setup BD list on a per channel basis */
  148. for (i = 0; i < ARRAY_SIZE(controller->tx); i++)
  149. cppi_pool_init(controller, controller->tx + i);
  150. for (i = 0; i < ARRAY_SIZE(controller->rx); i++)
  151. cppi_pool_init(controller, controller->rx + i);
  152. tibase = controller->tibase;
  153. INIT_LIST_HEAD(&controller->tx_complete);
  154. /* initialise tx/rx channel head pointers to zero */
  155. for (i = 0; i < ARRAY_SIZE(controller->tx); i++) {
  156. struct cppi_channel *tx_ch = controller->tx + i;
  157. struct cppi_tx_stateram __iomem *tx;
  158. INIT_LIST_HEAD(&tx_ch->tx_complete);
  159. tx = tibase + DAVINCI_TXCPPI_STATERAM_OFFSET(i);
  160. tx_ch->state_ram = tx;
  161. cppi_reset_tx(tx, 0);
  162. }
  163. for (i = 0; i < ARRAY_SIZE(controller->rx); i++) {
  164. struct cppi_channel *rx_ch = controller->rx + i;
  165. struct cppi_rx_stateram __iomem *rx;
  166. INIT_LIST_HEAD(&rx_ch->tx_complete);
  167. rx = tibase + DAVINCI_RXCPPI_STATERAM_OFFSET(i);
  168. rx_ch->state_ram = rx;
  169. cppi_reset_rx(rx);
  170. }
  171. /* enable individual cppi channels */
  172. musb_writel(tibase, DAVINCI_TXCPPI_INTENAB_REG,
  173. DAVINCI_DMA_ALL_CHANNELS_ENABLE);
  174. musb_writel(tibase, DAVINCI_RXCPPI_INTENAB_REG,
  175. DAVINCI_DMA_ALL_CHANNELS_ENABLE);
  176. /* enable tx/rx CPPI control */
  177. musb_writel(tibase, DAVINCI_TXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_ENABLE);
  178. musb_writel(tibase, DAVINCI_RXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_ENABLE);
  179. /* disable RNDIS mode, also host rx RNDIS autorequest */
  180. musb_writel(tibase, DAVINCI_RNDIS_REG, 0);
  181. musb_writel(tibase, DAVINCI_AUTOREQ_REG, 0);
  182. }
  183. /*
  184. * Stop DMA controller
  185. *
  186. * De-Init the DMA controller as necessary.
  187. */
  188. static void cppi_controller_stop(struct cppi *controller)
  189. {
  190. void __iomem *tibase;
  191. int i;
  192. struct musb *musb;
  193. musb = controller->controller.musb;
  194. tibase = controller->tibase;
  195. /* DISABLE INDIVIDUAL CHANNEL Interrupts */
  196. musb_writel(tibase, DAVINCI_TXCPPI_INTCLR_REG,
  197. DAVINCI_DMA_ALL_CHANNELS_ENABLE);
  198. musb_writel(tibase, DAVINCI_RXCPPI_INTCLR_REG,
  199. DAVINCI_DMA_ALL_CHANNELS_ENABLE);
  200. musb_dbg(musb, "Tearing down RX and TX Channels");
  201. for (i = 0; i < ARRAY_SIZE(controller->tx); i++) {
  202. /* FIXME restructure of txdma to use bds like rxdma */
  203. controller->tx[i].last_processed = NULL;
  204. cppi_pool_free(controller->tx + i);
  205. }
  206. for (i = 0; i < ARRAY_SIZE(controller->rx); i++)
  207. cppi_pool_free(controller->rx + i);
  208. /* in Tx Case proper teardown is supported. We resort to disabling
  209. * Tx/Rx CPPI after cleanup of Tx channels. Before TX teardown is
  210. * complete TX CPPI cannot be disabled.
  211. */
  212. /*disable tx/rx cppi */
  213. musb_writel(tibase, DAVINCI_TXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_DISABLE);
  214. musb_writel(tibase, DAVINCI_RXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_DISABLE);
  215. }
  216. /* While dma channel is allocated, we only want the core irqs active
  217. * for fault reports, otherwise we'd get irqs that we don't care about.
  218. * Except for TX irqs, where dma done != fifo empty and reusable ...
  219. *
  220. * NOTE: docs don't say either way, but irq masking **enables** irqs.
  221. *
  222. * REVISIT same issue applies to pure PIO usage too, and non-cppi dma...
  223. */
  224. static inline void core_rxirq_disable(void __iomem *tibase, unsigned epnum)
  225. {
  226. musb_writel(tibase, DAVINCI_USB_INT_MASK_CLR_REG, 1 << (epnum + 8));
  227. }
  228. static inline void core_rxirq_enable(void __iomem *tibase, unsigned epnum)
  229. {
  230. musb_writel(tibase, DAVINCI_USB_INT_MASK_SET_REG, 1 << (epnum + 8));
  231. }
  232. /*
  233. * Allocate a CPPI Channel for DMA. With CPPI, channels are bound to
  234. * each transfer direction of a non-control endpoint, so allocating
  235. * (and deallocating) is mostly a way to notice bad housekeeping on
  236. * the software side. We assume the irqs are always active.
  237. */
  238. static struct dma_channel *
  239. cppi_channel_allocate(struct dma_controller *c,
  240. struct musb_hw_ep *ep, u8 transmit)
  241. {
  242. struct cppi *controller;
  243. u8 index;
  244. struct cppi_channel *cppi_ch;
  245. void __iomem *tibase;
  246. struct musb *musb;
  247. controller = container_of(c, struct cppi, controller);
  248. tibase = controller->tibase;
  249. musb = c->musb;
  250. /* ep0 doesn't use DMA; remember cppi indices are 0..N-1 */
  251. index = ep->epnum - 1;
  252. /* return the corresponding CPPI Channel Handle, and
  253. * probably disable the non-CPPI irq until we need it.
  254. */
  255. if (transmit) {
  256. if (index >= ARRAY_SIZE(controller->tx)) {
  257. musb_dbg(musb, "no %cX%d CPPI channel", 'T', index);
  258. return NULL;
  259. }
  260. cppi_ch = controller->tx + index;
  261. } else {
  262. if (index >= ARRAY_SIZE(controller->rx)) {
  263. musb_dbg(musb, "no %cX%d CPPI channel", 'R', index);
  264. return NULL;
  265. }
  266. cppi_ch = controller->rx + index;
  267. core_rxirq_disable(tibase, ep->epnum);
  268. }
  269. /* REVISIT make this an error later once the same driver code works
  270. * with the other DMA engine too
  271. */
  272. if (cppi_ch->hw_ep)
  273. musb_dbg(musb, "re-allocating DMA%d %cX channel %p",
  274. index, transmit ? 'T' : 'R', cppi_ch);
  275. cppi_ch->hw_ep = ep;
  276. cppi_ch->channel.status = MUSB_DMA_STATUS_FREE;
  277. cppi_ch->channel.max_len = 0x7fffffff;
  278. musb_dbg(musb, "Allocate CPPI%d %cX", index, transmit ? 'T' : 'R');
  279. return &cppi_ch->channel;
  280. }
  281. /* Release a CPPI Channel. */
  282. static void cppi_channel_release(struct dma_channel *channel)
  283. {
  284. struct cppi_channel *c;
  285. void __iomem *tibase;
  286. /* REVISIT: for paranoia, check state and abort if needed... */
  287. c = container_of(channel, struct cppi_channel, channel);
  288. tibase = c->controller->tibase;
  289. if (!c->hw_ep)
  290. musb_dbg(c->controller->controller.musb,
  291. "releasing idle DMA channel %p", c);
  292. else if (!c->transmit)
  293. core_rxirq_enable(tibase, c->index + 1);
  294. /* for now, leave its cppi IRQ enabled (we won't trigger it) */
  295. c->hw_ep = NULL;
  296. channel->status = MUSB_DMA_STATUS_UNKNOWN;
  297. }
  298. /* Context: controller irqlocked */
  299. static void
  300. cppi_dump_rx(int level, struct cppi_channel *c, const char *tag)
  301. {
  302. void __iomem *base = c->controller->mregs;
  303. struct cppi_rx_stateram __iomem *rx = c->state_ram;
  304. musb_ep_select(base, c->index + 1);
  305. musb_dbg(c->controller->controller.musb,
  306. "RX DMA%d%s: %d left, csr %04x, "
  307. "%08x H%08x S%08x C%08x, "
  308. "B%08x L%08x %08x .. %08x",
  309. c->index, tag,
  310. musb_readl(c->controller->tibase,
  311. DAVINCI_RXCPPI_BUFCNT0_REG + 4 * c->index),
  312. musb_readw(c->hw_ep->regs, MUSB_RXCSR),
  313. musb_readl(&rx->rx_skipbytes, 0),
  314. musb_readl(&rx->rx_head, 0),
  315. musb_readl(&rx->rx_sop, 0),
  316. musb_readl(&rx->rx_current, 0),
  317. musb_readl(&rx->rx_buf_current, 0),
  318. musb_readl(&rx->rx_len_len, 0),
  319. musb_readl(&rx->rx_cnt_cnt, 0),
  320. musb_readl(&rx->rx_complete, 0)
  321. );
  322. }
  323. /* Context: controller irqlocked */
  324. static void
  325. cppi_dump_tx(int level, struct cppi_channel *c, const char *tag)
  326. {
  327. void __iomem *base = c->controller->mregs;
  328. struct cppi_tx_stateram __iomem *tx = c->state_ram;
  329. musb_ep_select(base, c->index + 1);
  330. musb_dbg(c->controller->controller.musb,
  331. "TX DMA%d%s: csr %04x, "
  332. "H%08x S%08x C%08x %08x, "
  333. "F%08x L%08x .. %08x",
  334. c->index, tag,
  335. musb_readw(c->hw_ep->regs, MUSB_TXCSR),
  336. musb_readl(&tx->tx_head, 0),
  337. musb_readl(&tx->tx_buf, 0),
  338. musb_readl(&tx->tx_current, 0),
  339. musb_readl(&tx->tx_buf_current, 0),
  340. musb_readl(&tx->tx_info, 0),
  341. musb_readl(&tx->tx_rem_len, 0),
  342. /* dummy/unused word 6 */
  343. musb_readl(&tx->tx_complete, 0)
  344. );
  345. }
  346. /* Context: controller irqlocked */
  347. static inline void
  348. cppi_rndis_update(struct cppi_channel *c, int is_rx,
  349. void __iomem *tibase, int is_rndis)
  350. {
  351. /* we may need to change the rndis flag for this cppi channel */
  352. if (c->is_rndis != is_rndis) {
  353. u32 value = musb_readl(tibase, DAVINCI_RNDIS_REG);
  354. u32 temp = 1 << (c->index);
  355. if (is_rx)
  356. temp <<= 16;
  357. if (is_rndis)
  358. value |= temp;
  359. else
  360. value &= ~temp;
  361. musb_writel(tibase, DAVINCI_RNDIS_REG, value);
  362. c->is_rndis = is_rndis;
  363. }
  364. }
  365. static void cppi_dump_rxbd(const char *tag, struct cppi_descriptor *bd)
  366. {
  367. pr_debug("RXBD/%s %08x: "
  368. "nxt %08x buf %08x off.blen %08x opt.plen %08x\n",
  369. tag, bd->dma,
  370. bd->hw_next, bd->hw_bufp, bd->hw_off_len,
  371. bd->hw_options);
  372. }
  373. static void cppi_dump_rxq(int level, const char *tag, struct cppi_channel *rx)
  374. {
  375. struct cppi_descriptor *bd;
  376. cppi_dump_rx(level, rx, tag);
  377. if (rx->last_processed)
  378. cppi_dump_rxbd("last", rx->last_processed);
  379. for (bd = rx->head; bd; bd = bd->next)
  380. cppi_dump_rxbd("active", bd);
  381. }
  382. /* NOTE: DaVinci autoreq is ignored except for host side "RNDIS" mode RX;
  383. * so we won't ever use it (see "CPPI RX Woes" below).
  384. */
  385. static inline int cppi_autoreq_update(struct cppi_channel *rx,
  386. void __iomem *tibase, int onepacket, unsigned n_bds)
  387. {
  388. u32 val;
  389. #ifdef RNDIS_RX_IS_USABLE
  390. u32 tmp;
  391. /* assert(is_host_active(musb)) */
  392. /* start from "AutoReq never" */
  393. tmp = musb_readl(tibase, DAVINCI_AUTOREQ_REG);
  394. val = tmp & ~((0x3) << (rx->index * 2));
  395. /* HCD arranged reqpkt for packet #1. we arrange int
  396. * for all but the last one, maybe in two segments.
  397. */
  398. if (!onepacket) {
  399. #if 0
  400. /* use two segments, autoreq "all" then the last "never" */
  401. val |= ((0x3) << (rx->index * 2));
  402. n_bds--;
  403. #else
  404. /* one segment, autoreq "all-but-last" */
  405. val |= ((0x1) << (rx->index * 2));
  406. #endif
  407. }
  408. if (val != tmp) {
  409. int n = 100;
  410. /* make sure that autoreq is updated before continuing */
  411. musb_writel(tibase, DAVINCI_AUTOREQ_REG, val);
  412. do {
  413. tmp = musb_readl(tibase, DAVINCI_AUTOREQ_REG);
  414. if (tmp == val)
  415. break;
  416. cpu_relax();
  417. } while (n-- > 0);
  418. }
  419. #endif
  420. /* REQPKT is turned off after each segment */
  421. if (n_bds && rx->channel.actual_len) {
  422. void __iomem *regs = rx->hw_ep->regs;
  423. val = musb_readw(regs, MUSB_RXCSR);
  424. if (!(val & MUSB_RXCSR_H_REQPKT)) {
  425. val |= MUSB_RXCSR_H_REQPKT | MUSB_RXCSR_H_WZC_BITS;
  426. musb_writew(regs, MUSB_RXCSR, val);
  427. /* flush writebuffer */
  428. val = musb_readw(regs, MUSB_RXCSR);
  429. }
  430. }
  431. return n_bds;
  432. }
  433. /* Buffer enqueuing Logic:
  434. *
  435. * - RX builds new queues each time, to help handle routine "early
  436. * termination" cases (faults, including errors and short reads)
  437. * more correctly.
  438. *
  439. * - for now, TX reuses the same queue of BDs every time
  440. *
  441. * REVISIT long term, we want a normal dynamic model.
  442. * ... the goal will be to append to the
  443. * existing queue, processing completed "dma buffers" (segments) on the fly.
  444. *
  445. * Otherwise we force an IRQ latency between requests, which slows us a lot
  446. * (especially in "transparent" dma). Unfortunately that model seems to be
  447. * inherent in the DMA model from the Mentor code, except in the rare case
  448. * of transfers big enough (~128+ KB) that we could append "middle" segments
  449. * in the TX paths. (RX can't do this, see below.)
  450. *
  451. * That's true even in the CPPI- friendly iso case, where most urbs have
  452. * several small segments provided in a group and where the "packet at a time"
  453. * "transparent" DMA model is always correct, even on the RX side.
  454. */
  455. /*
  456. * CPPI TX:
  457. * ========
  458. * TX is a lot more reasonable than RX; it doesn't need to run in
  459. * irq-per-packet mode very often. RNDIS mode seems to behave too
  460. * (except how it handles the exactly-N-packets case). Building a
  461. * txdma queue with multiple requests (urb or usb_request) looks
  462. * like it would work ... but fault handling would need much testing.
  463. *
  464. * The main issue with TX mode RNDIS relates to transfer lengths that
  465. * are an exact multiple of the packet length. It appears that there's
  466. * a hiccup in that case (maybe the DMA completes before the ZLP gets
  467. * written?) boiling down to not being able to rely on CPPI writing any
  468. * terminating zero length packet before the next transfer is written.
  469. * So that's punted to PIO; better yet, gadget drivers can avoid it.
  470. *
  471. * Plus, there's allegedly an undocumented constraint that rndis transfer
  472. * length be a multiple of 64 bytes ... but the chip doesn't act that
  473. * way, and we really don't _want_ that behavior anyway.
  474. *
  475. * On TX, "transparent" mode works ... although experiments have shown
  476. * problems trying to use the SOP/EOP bits in different USB packets.
  477. *
  478. * REVISIT try to handle terminating zero length packets using CPPI
  479. * instead of doing it by PIO after an IRQ. (Meanwhile, make Ethernet
  480. * links avoid that issue by forcing them to avoid zlps.)
  481. */
  482. static void
  483. cppi_next_tx_segment(struct musb *musb, struct cppi_channel *tx)
  484. {
  485. unsigned maxpacket = tx->maxpacket;
  486. dma_addr_t addr = tx->buf_dma + tx->offset;
  487. size_t length = tx->buf_len - tx->offset;
  488. struct cppi_descriptor *bd;
  489. unsigned n_bds;
  490. unsigned i;
  491. struct cppi_tx_stateram __iomem *tx_ram = tx->state_ram;
  492. int rndis;
  493. /* TX can use the CPPI "rndis" mode, where we can probably fit this
  494. * transfer in one BD and one IRQ. The only time we would NOT want
  495. * to use it is when hardware constraints prevent it, or if we'd
  496. * trigger the "send a ZLP?" confusion.
  497. */
  498. rndis = (maxpacket & 0x3f) == 0
  499. && length > maxpacket
  500. && length < 0xffff
  501. && (length % maxpacket) != 0;
  502. if (rndis) {
  503. maxpacket = length;
  504. n_bds = 1;
  505. } else {
  506. if (length)
  507. n_bds = DIV_ROUND_UP(length, maxpacket);
  508. else
  509. n_bds = 1;
  510. n_bds = min(n_bds, (unsigned) NUM_TXCHAN_BD);
  511. length = min(n_bds * maxpacket, length);
  512. }
  513. musb_dbg(musb, "TX DMA%d, pktSz %d %s bds %d dma 0x%llx len %u",
  514. tx->index,
  515. maxpacket,
  516. rndis ? "rndis" : "transparent",
  517. n_bds,
  518. (unsigned long long)addr, length);
  519. cppi_rndis_update(tx, 0, musb->ctrl_base, rndis);
  520. /* assuming here that channel_program is called during
  521. * transfer initiation ... current code maintains state
  522. * for one outstanding request only (no queues, not even
  523. * the implicit ones of an iso urb).
  524. */
  525. bd = tx->freelist;
  526. tx->head = bd;
  527. tx->last_processed = NULL;
  528. /* FIXME use BD pool like RX side does, and just queue
  529. * the minimum number for this request.
  530. */
  531. /* Prepare queue of BDs first, then hand it to hardware.
  532. * All BDs except maybe the last should be of full packet
  533. * size; for RNDIS there _is_ only that last packet.
  534. */
  535. for (i = 0; i < n_bds; ) {
  536. if (++i < n_bds && bd->next)
  537. bd->hw_next = bd->next->dma;
  538. else
  539. bd->hw_next = 0;
  540. bd->hw_bufp = tx->buf_dma + tx->offset;
  541. /* FIXME set EOP only on the last packet,
  542. * SOP only on the first ... avoid IRQs
  543. */
  544. if ((tx->offset + maxpacket) <= tx->buf_len) {
  545. tx->offset += maxpacket;
  546. bd->hw_off_len = maxpacket;
  547. bd->hw_options = CPPI_SOP_SET | CPPI_EOP_SET
  548. | CPPI_OWN_SET | maxpacket;
  549. } else {
  550. /* only this one may be a partial USB Packet */
  551. u32 partial_len;
  552. partial_len = tx->buf_len - tx->offset;
  553. tx->offset = tx->buf_len;
  554. bd->hw_off_len = partial_len;
  555. bd->hw_options = CPPI_SOP_SET | CPPI_EOP_SET
  556. | CPPI_OWN_SET | partial_len;
  557. if (partial_len == 0)
  558. bd->hw_options |= CPPI_ZERO_SET;
  559. }
  560. musb_dbg(musb, "TXBD %p: nxt %08x buf %08x len %04x opt %08x",
  561. bd, bd->hw_next, bd->hw_bufp,
  562. bd->hw_off_len, bd->hw_options);
  563. /* update the last BD enqueued to the list */
  564. tx->tail = bd;
  565. bd = bd->next;
  566. }
  567. /* BDs live in DMA-coherent memory, but writes might be pending */
  568. cpu_drain_writebuffer();
  569. /* Write to the HeadPtr in state RAM to trigger */
  570. musb_writel(&tx_ram->tx_head, 0, (u32)tx->freelist->dma);
  571. cppi_dump_tx(5, tx, "/S");
  572. }
  573. /*
  574. * CPPI RX Woes:
  575. * =============
  576. * Consider a 1KB bulk RX buffer in two scenarios: (a) it's fed two 300 byte
  577. * packets back-to-back, and (b) it's fed two 512 byte packets back-to-back.
  578. * (Full speed transfers have similar scenarios.)
  579. *
  580. * The correct behavior for Linux is that (a) fills the buffer with 300 bytes,
  581. * and the next packet goes into a buffer that's queued later; while (b) fills
  582. * the buffer with 1024 bytes. How to do that with CPPI?
  583. *
  584. * - RX queues in "rndis" mode -- one single BD -- handle (a) correctly, but
  585. * (b) loses **BADLY** because nothing (!) happens when that second packet
  586. * fills the buffer, much less when a third one arrives. (Which makes this
  587. * not a "true" RNDIS mode. In the RNDIS protocol short-packet termination
  588. * is optional, and it's fine if peripherals -- not hosts! -- pad messages
  589. * out to end-of-buffer. Standard PCI host controller DMA descriptors
  590. * implement that mode by default ... which is no accident.)
  591. *
  592. * - RX queues in "transparent" mode -- two BDs with 512 bytes each -- have
  593. * converse problems: (b) is handled right, but (a) loses badly. CPPI RX
  594. * ignores SOP/EOP markings and processes both of those BDs; so both packets
  595. * are loaded into the buffer (with a 212 byte gap between them), and the next
  596. * buffer queued will NOT get its 300 bytes of data. (It seems like SOP/EOP
  597. * are intended as outputs for RX queues, not inputs...)
  598. *
  599. * - A variant of "transparent" mode -- one BD at a time -- is the only way to
  600. * reliably make both cases work, with software handling both cases correctly
  601. * and at the significant penalty of needing an IRQ per packet. (The lack of
  602. * I/O overlap can be slightly ameliorated by enabling double buffering.)
  603. *
  604. * So how to get rid of IRQ-per-packet? The transparent multi-BD case could
  605. * be used in special cases like mass storage, which sets URB_SHORT_NOT_OK
  606. * (or maybe its peripheral side counterpart) to flag (a) scenarios as errors
  607. * with guaranteed driver level fault recovery and scrubbing out what's left
  608. * of that garbaged datastream.
  609. *
  610. * But there seems to be no way to identify the cases where CPPI RNDIS mode
  611. * is appropriate -- which do NOT include RNDIS host drivers, but do include
  612. * the CDC Ethernet driver! -- and the documentation is incomplete/wrong.
  613. * So we can't _ever_ use RX RNDIS mode ... except by using a heuristic
  614. * that applies best on the peripheral side (and which could fail rudely).
  615. *
  616. * Leaving only "transparent" mode; we avoid multi-bd modes in almost all
  617. * cases other than mass storage class. Otherwise we're correct but slow,
  618. * since CPPI penalizes our need for a "true RNDIS" default mode.
  619. */
  620. /* Heuristic, intended to kick in for ethernet/rndis peripheral ONLY
  621. *
  622. * IFF
  623. * (a) peripheral mode ... since rndis peripherals could pad their
  624. * writes to hosts, causing i/o failure; or we'd have to cope with
  625. * a largely unknowable variety of host side protocol variants
  626. * (b) and short reads are NOT errors ... since full reads would
  627. * cause those same i/o failures
  628. * (c) and read length is
  629. * - less than 64KB (max per cppi descriptor)
  630. * - not a multiple of 4096 (g_zero default, full reads typical)
  631. * - N (>1) packets long, ditto (full reads not EXPECTED)
  632. * THEN
  633. * try rx rndis mode
  634. *
  635. * Cost of heuristic failing: RXDMA wedges at the end of transfers that
  636. * fill out the whole buffer. Buggy host side usb network drivers could
  637. * trigger that, but "in the field" such bugs seem to be all but unknown.
  638. *
  639. * So this module parameter lets the heuristic be disabled. When using
  640. * gadgetfs, the heuristic will probably need to be disabled.
  641. */
  642. static bool cppi_rx_rndis = 1;
  643. module_param(cppi_rx_rndis, bool, 0);
  644. MODULE_PARM_DESC(cppi_rx_rndis, "enable/disable RX RNDIS heuristic");
  645. /**
  646. * cppi_next_rx_segment - dma read for the next chunk of a buffer
  647. * @musb: the controller
  648. * @rx: dma channel
  649. * @onepacket: true unless caller treats short reads as errors, and
  650. * performs fault recovery above usbcore.
  651. * Context: controller irqlocked
  652. *
  653. * See above notes about why we can't use multi-BD RX queues except in
  654. * rare cases (mass storage class), and can never use the hardware "rndis"
  655. * mode (since it's not a "true" RNDIS mode) with complete safety..
  656. *
  657. * It's ESSENTIAL that callers specify "onepacket" mode unless they kick in
  658. * code to recover from corrupted datastreams after each short transfer.
  659. */
  660. static void
  661. cppi_next_rx_segment(struct musb *musb, struct cppi_channel *rx, int onepacket)
  662. {
  663. unsigned maxpacket = rx->maxpacket;
  664. dma_addr_t addr = rx->buf_dma + rx->offset;
  665. size_t length = rx->buf_len - rx->offset;
  666. struct cppi_descriptor *bd, *tail;
  667. unsigned n_bds;
  668. unsigned i;
  669. void __iomem *tibase = musb->ctrl_base;
  670. int is_rndis = 0;
  671. struct cppi_rx_stateram __iomem *rx_ram = rx->state_ram;
  672. struct cppi_descriptor *d;
  673. if (onepacket) {
  674. /* almost every USB driver, host or peripheral side */
  675. n_bds = 1;
  676. /* maybe apply the heuristic above */
  677. if (cppi_rx_rndis
  678. && is_peripheral_active(musb)
  679. && length > maxpacket
  680. && (length & ~0xffff) == 0
  681. && (length & 0x0fff) != 0
  682. && (length & (maxpacket - 1)) == 0) {
  683. maxpacket = length;
  684. is_rndis = 1;
  685. }
  686. } else {
  687. /* virtually nothing except mass storage class */
  688. if (length > 0xffff) {
  689. n_bds = 0xffff / maxpacket;
  690. length = n_bds * maxpacket;
  691. } else {
  692. n_bds = DIV_ROUND_UP(length, maxpacket);
  693. }
  694. if (n_bds == 1)
  695. onepacket = 1;
  696. else
  697. n_bds = min(n_bds, (unsigned) NUM_RXCHAN_BD);
  698. }
  699. /* In host mode, autorequest logic can generate some IN tokens; it's
  700. * tricky since we can't leave REQPKT set in RXCSR after the transfer
  701. * finishes. So: multipacket transfers involve two or more segments.
  702. * And always at least two IRQs ... RNDIS mode is not an option.
  703. */
  704. if (is_host_active(musb))
  705. n_bds = cppi_autoreq_update(rx, tibase, onepacket, n_bds);
  706. cppi_rndis_update(rx, 1, musb->ctrl_base, is_rndis);
  707. length = min(n_bds * maxpacket, length);
  708. musb_dbg(musb, "RX DMA%d seg, maxp %d %s bds %d (cnt %d) "
  709. "dma 0x%llx len %u %u/%u",
  710. rx->index, maxpacket,
  711. onepacket
  712. ? (is_rndis ? "rndis" : "onepacket")
  713. : "multipacket",
  714. n_bds,
  715. musb_readl(tibase,
  716. DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4))
  717. & 0xffff,
  718. (unsigned long long)addr, length,
  719. rx->channel.actual_len, rx->buf_len);
  720. /* only queue one segment at a time, since the hardware prevents
  721. * correct queue shutdown after unexpected short packets
  722. */
  723. bd = cppi_bd_alloc(rx);
  724. rx->head = bd;
  725. /* Build BDs for all packets in this segment */
  726. for (i = 0, tail = NULL; bd && i < n_bds; i++, tail = bd) {
  727. u32 bd_len;
  728. if (i) {
  729. bd = cppi_bd_alloc(rx);
  730. if (!bd)
  731. break;
  732. tail->next = bd;
  733. tail->hw_next = bd->dma;
  734. }
  735. bd->hw_next = 0;
  736. /* all but the last packet will be maxpacket size */
  737. if (maxpacket < length)
  738. bd_len = maxpacket;
  739. else
  740. bd_len = length;
  741. bd->hw_bufp = addr;
  742. addr += bd_len;
  743. rx->offset += bd_len;
  744. bd->hw_off_len = (0 /*offset*/ << 16) + bd_len;
  745. bd->buflen = bd_len;
  746. bd->hw_options = CPPI_OWN_SET | (i == 0 ? length : 0);
  747. length -= bd_len;
  748. }
  749. /* we always expect at least one reusable BD! */
  750. if (!tail) {
  751. WARNING("rx dma%d -- no BDs? need %d\n", rx->index, n_bds);
  752. return;
  753. } else if (i < n_bds)
  754. WARNING("rx dma%d -- only %d of %d BDs\n", rx->index, i, n_bds);
  755. tail->next = NULL;
  756. tail->hw_next = 0;
  757. bd = rx->head;
  758. rx->tail = tail;
  759. /* short reads and other faults should terminate this entire
  760. * dma segment. we want one "dma packet" per dma segment, not
  761. * one per USB packet, terminating the whole queue at once...
  762. * NOTE that current hardware seems to ignore SOP and EOP.
  763. */
  764. bd->hw_options |= CPPI_SOP_SET;
  765. tail->hw_options |= CPPI_EOP_SET;
  766. for (d = rx->head; d; d = d->next)
  767. cppi_dump_rxbd("S", d);
  768. /* in case the preceding transfer left some state... */
  769. tail = rx->last_processed;
  770. if (tail) {
  771. tail->next = bd;
  772. tail->hw_next = bd->dma;
  773. }
  774. core_rxirq_enable(tibase, rx->index + 1);
  775. /* BDs live in DMA-coherent memory, but writes might be pending */
  776. cpu_drain_writebuffer();
  777. /* REVISIT specs say to write this AFTER the BUFCNT register
  778. * below ... but that loses badly.
  779. */
  780. musb_writel(&rx_ram->rx_head, 0, bd->dma);
  781. /* bufferCount must be at least 3, and zeroes on completion
  782. * unless it underflows below zero, or stops at two, or keeps
  783. * growing ... grr.
  784. */
  785. i = musb_readl(tibase,
  786. DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4))
  787. & 0xffff;
  788. if (!i)
  789. musb_writel(tibase,
  790. DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4),
  791. n_bds + 2);
  792. else if (n_bds > (i - 3))
  793. musb_writel(tibase,
  794. DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4),
  795. n_bds - (i - 3));
  796. i = musb_readl(tibase,
  797. DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4))
  798. & 0xffff;
  799. if (i < (2 + n_bds)) {
  800. musb_dbg(musb, "bufcnt%d underrun - %d (for %d)",
  801. rx->index, i, n_bds);
  802. musb_writel(tibase,
  803. DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4),
  804. n_bds + 2);
  805. }
  806. cppi_dump_rx(4, rx, "/S");
  807. }
  808. /**
  809. * cppi_channel_program - program channel for data transfer
  810. * @ch: the channel
  811. * @maxpacket: max packet size
  812. * @mode: For RX, 1 unless the usb protocol driver promised to treat
  813. * all short reads as errors and kick in high level fault recovery.
  814. * For TX, ignored because of RNDIS mode races/glitches.
  815. * @dma_addr: dma address of buffer
  816. * @len: length of buffer
  817. * Context: controller irqlocked
  818. */
  819. static int cppi_channel_program(struct dma_channel *ch,
  820. u16 maxpacket, u8 mode,
  821. dma_addr_t dma_addr, u32 len)
  822. {
  823. struct cppi_channel *cppi_ch;
  824. struct cppi *controller;
  825. struct musb *musb;
  826. cppi_ch = container_of(ch, struct cppi_channel, channel);
  827. controller = cppi_ch->controller;
  828. musb = controller->controller.musb;
  829. switch (ch->status) {
  830. case MUSB_DMA_STATUS_BUS_ABORT:
  831. case MUSB_DMA_STATUS_CORE_ABORT:
  832. /* fault irq handler should have handled cleanup */
  833. WARNING("%cX DMA%d not cleaned up after abort!\n",
  834. cppi_ch->transmit ? 'T' : 'R',
  835. cppi_ch->index);
  836. /* WARN_ON(1); */
  837. break;
  838. case MUSB_DMA_STATUS_BUSY:
  839. WARNING("program active channel? %cX DMA%d\n",
  840. cppi_ch->transmit ? 'T' : 'R',
  841. cppi_ch->index);
  842. /* WARN_ON(1); */
  843. break;
  844. case MUSB_DMA_STATUS_UNKNOWN:
  845. musb_dbg(musb, "%cX DMA%d not allocated!",
  846. cppi_ch->transmit ? 'T' : 'R',
  847. cppi_ch->index);
  848. fallthrough;
  849. case MUSB_DMA_STATUS_FREE:
  850. break;
  851. }
  852. ch->status = MUSB_DMA_STATUS_BUSY;
  853. /* set transfer parameters, then queue up its first segment */
  854. cppi_ch->buf_dma = dma_addr;
  855. cppi_ch->offset = 0;
  856. cppi_ch->maxpacket = maxpacket;
  857. cppi_ch->buf_len = len;
  858. cppi_ch->channel.actual_len = 0;
  859. /* TX channel? or RX? */
  860. if (cppi_ch->transmit)
  861. cppi_next_tx_segment(musb, cppi_ch);
  862. else
  863. cppi_next_rx_segment(musb, cppi_ch, mode);
  864. return true;
  865. }
  866. static bool cppi_rx_scan(struct cppi *cppi, unsigned ch)
  867. {
  868. struct cppi_channel *rx = &cppi->rx[ch];
  869. struct cppi_rx_stateram __iomem *state = rx->state_ram;
  870. struct cppi_descriptor *bd;
  871. struct cppi_descriptor *last = rx->last_processed;
  872. bool completed = false;
  873. bool acked = false;
  874. int i;
  875. dma_addr_t safe2ack;
  876. void __iomem *regs = rx->hw_ep->regs;
  877. struct musb *musb = cppi->controller.musb;
  878. cppi_dump_rx(6, rx, "/K");
  879. bd = last ? last->next : rx->head;
  880. if (!bd)
  881. return false;
  882. /* run through all completed BDs */
  883. for (i = 0, safe2ack = musb_readl(&state->rx_complete, 0);
  884. (safe2ack || completed) && bd && i < NUM_RXCHAN_BD;
  885. i++, bd = bd->next) {
  886. u16 len;
  887. /* catch latest BD writes from CPPI */
  888. rmb();
  889. if (!completed && (bd->hw_options & CPPI_OWN_SET))
  890. break;
  891. musb_dbg(musb, "C/RXBD %llx: nxt %08x buf %08x "
  892. "off.len %08x opt.len %08x (%d)",
  893. (unsigned long long)bd->dma, bd->hw_next, bd->hw_bufp,
  894. bd->hw_off_len, bd->hw_options,
  895. rx->channel.actual_len);
  896. /* actual packet received length */
  897. if ((bd->hw_options & CPPI_SOP_SET) && !completed)
  898. len = bd->hw_off_len & CPPI_RECV_PKTLEN_MASK;
  899. else
  900. len = 0;
  901. if (bd->hw_options & CPPI_EOQ_MASK)
  902. completed = true;
  903. if (!completed && len < bd->buflen) {
  904. /* NOTE: when we get a short packet, RXCSR_H_REQPKT
  905. * must have been cleared, and no more DMA packets may
  906. * active be in the queue... TI docs didn't say, but
  907. * CPPI ignores those BDs even though OWN is still set.
  908. */
  909. completed = true;
  910. musb_dbg(musb, "rx short %d/%d (%d)",
  911. len, bd->buflen,
  912. rx->channel.actual_len);
  913. }
  914. /* If we got here, we expect to ack at least one BD; meanwhile
  915. * CPPI may completing other BDs while we scan this list...
  916. *
  917. * RACE: we can notice OWN cleared before CPPI raises the
  918. * matching irq by writing that BD as the completion pointer.
  919. * In such cases, stop scanning and wait for the irq, avoiding
  920. * lost acks and states where BD ownership is unclear.
  921. */
  922. if (bd->dma == safe2ack) {
  923. musb_writel(&state->rx_complete, 0, safe2ack);
  924. safe2ack = musb_readl(&state->rx_complete, 0);
  925. acked = true;
  926. if (bd->dma == safe2ack)
  927. safe2ack = 0;
  928. }
  929. rx->channel.actual_len += len;
  930. cppi_bd_free(rx, last);
  931. last = bd;
  932. /* stop scanning on end-of-segment */
  933. if (bd->hw_next == 0)
  934. completed = true;
  935. }
  936. rx->last_processed = last;
  937. /* dma abort, lost ack, or ... */
  938. if (!acked && last) {
  939. int csr;
  940. if (safe2ack == 0 || safe2ack == rx->last_processed->dma)
  941. musb_writel(&state->rx_complete, 0, safe2ack);
  942. if (safe2ack == 0) {
  943. cppi_bd_free(rx, last);
  944. rx->last_processed = NULL;
  945. /* if we land here on the host side, H_REQPKT will
  946. * be clear and we need to restart the queue...
  947. */
  948. WARN_ON(rx->head);
  949. }
  950. musb_ep_select(cppi->mregs, rx->index + 1);
  951. csr = musb_readw(regs, MUSB_RXCSR);
  952. if (csr & MUSB_RXCSR_DMAENAB) {
  953. musb_dbg(musb, "list%d %p/%p, last %llx%s, csr %04x",
  954. rx->index,
  955. rx->head, rx->tail,
  956. rx->last_processed
  957. ? (unsigned long long)
  958. rx->last_processed->dma
  959. : 0,
  960. completed ? ", completed" : "",
  961. csr);
  962. cppi_dump_rxq(4, "/what?", rx);
  963. }
  964. }
  965. if (!completed) {
  966. int csr;
  967. rx->head = bd;
  968. /* REVISIT seems like "autoreq all but EOP" doesn't...
  969. * setting it here "should" be racey, but seems to work
  970. */
  971. csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR);
  972. if (is_host_active(cppi->controller.musb)
  973. && bd
  974. && !(csr & MUSB_RXCSR_H_REQPKT)) {
  975. csr |= MUSB_RXCSR_H_REQPKT;
  976. musb_writew(regs, MUSB_RXCSR,
  977. MUSB_RXCSR_H_WZC_BITS | csr);
  978. csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR);
  979. }
  980. } else {
  981. rx->head = NULL;
  982. rx->tail = NULL;
  983. }
  984. cppi_dump_rx(6, rx, completed ? "/completed" : "/cleaned");
  985. return completed;
  986. }
  987. irqreturn_t cppi_interrupt(int irq, void *dev_id)
  988. {
  989. struct musb *musb = dev_id;
  990. struct cppi *cppi;
  991. void __iomem *tibase;
  992. struct musb_hw_ep *hw_ep = NULL;
  993. u32 rx, tx;
  994. int i, index;
  995. unsigned long flags;
  996. cppi = container_of(musb->dma_controller, struct cppi, controller);
  997. if (cppi->irq)
  998. spin_lock_irqsave(&musb->lock, flags);
  999. tibase = musb->ctrl_base;
  1000. tx = musb_readl(tibase, DAVINCI_TXCPPI_MASKED_REG);
  1001. rx = musb_readl(tibase, DAVINCI_RXCPPI_MASKED_REG);
  1002. if (!tx && !rx) {
  1003. if (cppi->irq)
  1004. spin_unlock_irqrestore(&musb->lock, flags);
  1005. return IRQ_NONE;
  1006. }
  1007. musb_dbg(musb, "CPPI IRQ Tx%x Rx%x", tx, rx);
  1008. /* process TX channels */
  1009. for (index = 0; tx; tx = tx >> 1, index++) {
  1010. struct cppi_channel *tx_ch;
  1011. struct cppi_tx_stateram __iomem *tx_ram;
  1012. bool completed = false;
  1013. struct cppi_descriptor *bd;
  1014. if (!(tx & 1))
  1015. continue;
  1016. tx_ch = cppi->tx + index;
  1017. tx_ram = tx_ch->state_ram;
  1018. /* FIXME need a cppi_tx_scan() routine, which
  1019. * can also be called from abort code
  1020. */
  1021. cppi_dump_tx(5, tx_ch, "/E");
  1022. bd = tx_ch->head;
  1023. /*
  1024. * If Head is null then this could mean that a abort interrupt
  1025. * that needs to be acknowledged.
  1026. */
  1027. if (NULL == bd) {
  1028. musb_dbg(musb, "null BD");
  1029. musb_writel(&tx_ram->tx_complete, 0, 0);
  1030. continue;
  1031. }
  1032. /* run through all completed BDs */
  1033. for (i = 0; !completed && bd && i < NUM_TXCHAN_BD;
  1034. i++, bd = bd->next) {
  1035. u16 len;
  1036. /* catch latest BD writes from CPPI */
  1037. rmb();
  1038. if (bd->hw_options & CPPI_OWN_SET)
  1039. break;
  1040. musb_dbg(musb, "C/TXBD %p n %x b %x off %x opt %x",
  1041. bd, bd->hw_next, bd->hw_bufp,
  1042. bd->hw_off_len, bd->hw_options);
  1043. len = bd->hw_off_len & CPPI_BUFFER_LEN_MASK;
  1044. tx_ch->channel.actual_len += len;
  1045. tx_ch->last_processed = bd;
  1046. /* write completion register to acknowledge
  1047. * processing of completed BDs, and possibly
  1048. * release the IRQ; EOQ might not be set ...
  1049. *
  1050. * REVISIT use the same ack strategy as rx
  1051. *
  1052. * REVISIT have observed bit 18 set; huh??
  1053. */
  1054. /* if ((bd->hw_options & CPPI_EOQ_MASK)) */
  1055. musb_writel(&tx_ram->tx_complete, 0, bd->dma);
  1056. /* stop scanning on end-of-segment */
  1057. if (bd->hw_next == 0)
  1058. completed = true;
  1059. }
  1060. /* on end of segment, maybe go to next one */
  1061. if (completed) {
  1062. /* cppi_dump_tx(4, tx_ch, "/complete"); */
  1063. /* transfer more, or report completion */
  1064. if (tx_ch->offset >= tx_ch->buf_len) {
  1065. tx_ch->head = NULL;
  1066. tx_ch->tail = NULL;
  1067. tx_ch->channel.status = MUSB_DMA_STATUS_FREE;
  1068. hw_ep = tx_ch->hw_ep;
  1069. musb_dma_completion(musb, index + 1, 1);
  1070. } else {
  1071. /* Bigger transfer than we could fit in
  1072. * that first batch of descriptors...
  1073. */
  1074. cppi_next_tx_segment(musb, tx_ch);
  1075. }
  1076. } else
  1077. tx_ch->head = bd;
  1078. }
  1079. /* Start processing the RX block */
  1080. for (index = 0; rx; rx = rx >> 1, index++) {
  1081. if (rx & 1) {
  1082. struct cppi_channel *rx_ch;
  1083. rx_ch = cppi->rx + index;
  1084. /* let incomplete dma segments finish */
  1085. if (!cppi_rx_scan(cppi, index))
  1086. continue;
  1087. /* start another dma segment if needed */
  1088. if (rx_ch->channel.actual_len != rx_ch->buf_len
  1089. && rx_ch->channel.actual_len
  1090. == rx_ch->offset) {
  1091. cppi_next_rx_segment(musb, rx_ch, 1);
  1092. continue;
  1093. }
  1094. /* all segments completed! */
  1095. rx_ch->channel.status = MUSB_DMA_STATUS_FREE;
  1096. hw_ep = rx_ch->hw_ep;
  1097. core_rxirq_disable(tibase, index + 1);
  1098. musb_dma_completion(musb, index + 1, 0);
  1099. }
  1100. }
  1101. /* write to CPPI EOI register to re-enable interrupts */
  1102. musb_writel(tibase, DAVINCI_CPPI_EOI_REG, 0);
  1103. if (cppi->irq)
  1104. spin_unlock_irqrestore(&musb->lock, flags);
  1105. return IRQ_HANDLED;
  1106. }
  1107. EXPORT_SYMBOL_GPL(cppi_interrupt);
  1108. /* Instantiate a software object representing a DMA controller. */
  1109. struct dma_controller *
  1110. cppi_dma_controller_create(struct musb *musb, void __iomem *mregs)
  1111. {
  1112. struct cppi *controller;
  1113. struct device *dev = musb->controller;
  1114. struct platform_device *pdev = to_platform_device(dev);
  1115. int irq = platform_get_irq_byname(pdev, "dma");
  1116. controller = kzalloc(sizeof *controller, GFP_KERNEL);
  1117. if (!controller)
  1118. return NULL;
  1119. controller->mregs = mregs;
  1120. controller->tibase = mregs - DAVINCI_BASE_OFFSET;
  1121. controller->controller.musb = musb;
  1122. controller->controller.channel_alloc = cppi_channel_allocate;
  1123. controller->controller.channel_release = cppi_channel_release;
  1124. controller->controller.channel_program = cppi_channel_program;
  1125. controller->controller.channel_abort = cppi_channel_abort;
  1126. /* NOTE: allocating from on-chip SRAM would give the least
  1127. * contention for memory access, if that ever matters here.
  1128. */
  1129. /* setup BufferPool */
  1130. controller->pool = dma_pool_create("cppi",
  1131. controller->controller.musb->controller,
  1132. sizeof(struct cppi_descriptor),
  1133. CPPI_DESCRIPTOR_ALIGN, 0);
  1134. if (!controller->pool) {
  1135. kfree(controller);
  1136. return NULL;
  1137. }
  1138. if (irq > 0) {
  1139. if (request_irq(irq, cppi_interrupt, 0, "cppi-dma", musb)) {
  1140. dev_err(dev, "request_irq %d failed!\n", irq);
  1141. musb_dma_controller_destroy(&controller->controller);
  1142. return NULL;
  1143. }
  1144. controller->irq = irq;
  1145. }
  1146. cppi_controller_start(controller);
  1147. return &controller->controller;
  1148. }
  1149. EXPORT_SYMBOL_GPL(cppi_dma_controller_create);
  1150. /*
  1151. * Destroy a previously-instantiated DMA controller.
  1152. */
  1153. void cppi_dma_controller_destroy(struct dma_controller *c)
  1154. {
  1155. struct cppi *cppi;
  1156. cppi = container_of(c, struct cppi, controller);
  1157. cppi_controller_stop(cppi);
  1158. if (cppi->irq)
  1159. free_irq(cppi->irq, cppi->controller.musb);
  1160. /* assert: caller stopped the controller first */
  1161. dma_pool_destroy(cppi->pool);
  1162. kfree(cppi);
  1163. }
  1164. EXPORT_SYMBOL_GPL(cppi_dma_controller_destroy);
  1165. /*
  1166. * Context: controller irqlocked, endpoint selected
  1167. */
  1168. static int cppi_channel_abort(struct dma_channel *channel)
  1169. {
  1170. struct cppi_channel *cppi_ch;
  1171. struct cppi *controller;
  1172. void __iomem *mbase;
  1173. void __iomem *tibase;
  1174. void __iomem *regs;
  1175. u32 value;
  1176. struct cppi_descriptor *queue;
  1177. cppi_ch = container_of(channel, struct cppi_channel, channel);
  1178. controller = cppi_ch->controller;
  1179. switch (channel->status) {
  1180. case MUSB_DMA_STATUS_BUS_ABORT:
  1181. case MUSB_DMA_STATUS_CORE_ABORT:
  1182. /* from RX or TX fault irq handler */
  1183. case MUSB_DMA_STATUS_BUSY:
  1184. /* the hardware needs shutting down */
  1185. regs = cppi_ch->hw_ep->regs;
  1186. break;
  1187. case MUSB_DMA_STATUS_UNKNOWN:
  1188. case MUSB_DMA_STATUS_FREE:
  1189. return 0;
  1190. default:
  1191. return -EINVAL;
  1192. }
  1193. if (!cppi_ch->transmit && cppi_ch->head)
  1194. cppi_dump_rxq(3, "/abort", cppi_ch);
  1195. mbase = controller->mregs;
  1196. tibase = controller->tibase;
  1197. queue = cppi_ch->head;
  1198. cppi_ch->head = NULL;
  1199. cppi_ch->tail = NULL;
  1200. /* REVISIT should rely on caller having done this,
  1201. * and caller should rely on us not changing it.
  1202. * peripheral code is safe ... check host too.
  1203. */
  1204. musb_ep_select(mbase, cppi_ch->index + 1);
  1205. if (cppi_ch->transmit) {
  1206. struct cppi_tx_stateram __iomem *tx_ram;
  1207. /* REVISIT put timeouts on these controller handshakes */
  1208. cppi_dump_tx(6, cppi_ch, " (teardown)");
  1209. /* teardown DMA engine then usb core */
  1210. do {
  1211. value = musb_readl(tibase, DAVINCI_TXCPPI_TEAR_REG);
  1212. } while (!(value & CPPI_TEAR_READY));
  1213. musb_writel(tibase, DAVINCI_TXCPPI_TEAR_REG, cppi_ch->index);
  1214. tx_ram = cppi_ch->state_ram;
  1215. do {
  1216. value = musb_readl(&tx_ram->tx_complete, 0);
  1217. } while (0xFFFFFFFC != value);
  1218. /* FIXME clean up the transfer state ... here?
  1219. * the completion routine should get called with
  1220. * an appropriate status code.
  1221. */
  1222. value = musb_readw(regs, MUSB_TXCSR);
  1223. value &= ~MUSB_TXCSR_DMAENAB;
  1224. value |= MUSB_TXCSR_FLUSHFIFO;
  1225. musb_writew(regs, MUSB_TXCSR, value);
  1226. musb_writew(regs, MUSB_TXCSR, value);
  1227. /*
  1228. * 1. Write to completion Ptr value 0x1(bit 0 set)
  1229. * (write back mode)
  1230. * 2. Wait for abort interrupt and then put the channel in
  1231. * compare mode by writing 1 to the tx_complete register.
  1232. */
  1233. cppi_reset_tx(tx_ram, 1);
  1234. cppi_ch->head = NULL;
  1235. musb_writel(&tx_ram->tx_complete, 0, 1);
  1236. cppi_dump_tx(5, cppi_ch, " (done teardown)");
  1237. /* REVISIT tx side _should_ clean up the same way
  1238. * as the RX side ... this does no cleanup at all!
  1239. */
  1240. } else /* RX */ {
  1241. u16 csr;
  1242. /* NOTE: docs don't guarantee any of this works ... we
  1243. * expect that if the usb core stops telling the cppi core
  1244. * to pull more data from it, then it'll be safe to flush
  1245. * current RX DMA state iff any pending fifo transfer is done.
  1246. */
  1247. core_rxirq_disable(tibase, cppi_ch->index + 1);
  1248. /* for host, ensure ReqPkt is never set again */
  1249. if (is_host_active(cppi_ch->controller->controller.musb)) {
  1250. value = musb_readl(tibase, DAVINCI_AUTOREQ_REG);
  1251. value &= ~((0x3) << (cppi_ch->index * 2));
  1252. musb_writel(tibase, DAVINCI_AUTOREQ_REG, value);
  1253. }
  1254. csr = musb_readw(regs, MUSB_RXCSR);
  1255. /* for host, clear (just) ReqPkt at end of current packet(s) */
  1256. if (is_host_active(cppi_ch->controller->controller.musb)) {
  1257. csr |= MUSB_RXCSR_H_WZC_BITS;
  1258. csr &= ~MUSB_RXCSR_H_REQPKT;
  1259. } else
  1260. csr |= MUSB_RXCSR_P_WZC_BITS;
  1261. /* clear dma enable */
  1262. csr &= ~(MUSB_RXCSR_DMAENAB);
  1263. musb_writew(regs, MUSB_RXCSR, csr);
  1264. csr = musb_readw(regs, MUSB_RXCSR);
  1265. /* Quiesce: wait for current dma to finish (if not cleanup).
  1266. * We can't use bit zero of stateram->rx_sop, since that
  1267. * refers to an entire "DMA packet" not just emptying the
  1268. * current fifo. Most segments need multiple usb packets.
  1269. */
  1270. if (channel->status == MUSB_DMA_STATUS_BUSY)
  1271. udelay(50);
  1272. /* scan the current list, reporting any data that was
  1273. * transferred and acking any IRQ
  1274. */
  1275. cppi_rx_scan(controller, cppi_ch->index);
  1276. /* clobber the existing state once it's idle
  1277. *
  1278. * NOTE: arguably, we should also wait for all the other
  1279. * RX channels to quiesce (how??) and then temporarily
  1280. * disable RXCPPI_CTRL_REG ... but it seems that we can
  1281. * rely on the controller restarting from state ram, with
  1282. * only RXCPPI_BUFCNT state being bogus. BUFCNT will
  1283. * correct itself after the next DMA transfer though.
  1284. *
  1285. * REVISIT does using rndis mode change that?
  1286. */
  1287. cppi_reset_rx(cppi_ch->state_ram);
  1288. /* next DMA request _should_ load cppi head ptr */
  1289. /* ... we don't "free" that list, only mutate it in place. */
  1290. cppi_dump_rx(5, cppi_ch, " (done abort)");
  1291. /* clean up previously pending bds */
  1292. cppi_bd_free(cppi_ch, cppi_ch->last_processed);
  1293. cppi_ch->last_processed = NULL;
  1294. while (queue) {
  1295. struct cppi_descriptor *tmp = queue->next;
  1296. cppi_bd_free(cppi_ch, queue);
  1297. queue = tmp;
  1298. }
  1299. }
  1300. channel->status = MUSB_DMA_STATUS_FREE;
  1301. cppi_ch->buf_dma = 0;
  1302. cppi_ch->offset = 0;
  1303. cppi_ch->buf_len = 0;
  1304. cppi_ch->maxpacket = 0;
  1305. return 0;
  1306. }
  1307. /* TBD Queries:
  1308. *
  1309. * Power Management ... probably turn off cppi during suspend, restart;
  1310. * check state ram? Clocking is presumably shared with usb core.
  1311. */