musb_host.c 74 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * MUSB OTG driver host support
  4. *
  5. * Copyright 2005 Mentor Graphics Corporation
  6. * Copyright (C) 2005-2006 by Texas Instruments
  7. * Copyright (C) 2006-2007 Nokia Corporation
  8. * Copyright (C) 2008-2009 MontaVista Software, Inc. <source@mvista.com>
  9. */
  10. #include <linux/module.h>
  11. #include <linux/kernel.h>
  12. #include <linux/delay.h>
  13. #include <linux/sched.h>
  14. #include <linux/slab.h>
  15. #include <linux/errno.h>
  16. #include <linux/list.h>
  17. #include <linux/dma-mapping.h>
  18. #include "musb_core.h"
  19. #include "musb_host.h"
  20. #include "musb_trace.h"
  21. /* MUSB HOST status 22-mar-2006
  22. *
  23. * - There's still lots of partial code duplication for fault paths, so
  24. * they aren't handled as consistently as they need to be.
  25. *
  26. * - PIO mostly behaved when last tested.
  27. * + including ep0, with all usbtest cases 9, 10
  28. * + usbtest 14 (ep0out) doesn't seem to run at all
  29. * + double buffered OUT/TX endpoints saw stalls(!) with certain usbtest
  30. * configurations, but otherwise double buffering passes basic tests.
  31. * + for 2.6.N, for N > ~10, needs API changes for hcd framework.
  32. *
  33. * - DMA (CPPI) ... partially behaves, not currently recommended
  34. * + about 1/15 the speed of typical EHCI implementations (PCI)
  35. * + RX, all too often reqpkt seems to misbehave after tx
  36. * + TX, no known issues (other than evident silicon issue)
  37. *
  38. * - DMA (Mentor/OMAP) ...has at least toggle update problems
  39. *
  40. * - [23-feb-2009] minimal traffic scheduling to avoid bulk RX packet
  41. * starvation ... nothing yet for TX, interrupt, or bulk.
  42. *
  43. * - Not tested with HNP, but some SRP paths seem to behave.
  44. *
  45. * NOTE 24-August-2006:
  46. *
  47. * - Bulk traffic finally uses both sides of hardware ep1, freeing up an
  48. * extra endpoint for periodic use enabling hub + keybd + mouse. That
  49. * mostly works, except that with "usbnet" it's easy to trigger cases
  50. * with "ping" where RX loses. (a) ping to davinci, even "ping -f",
  51. * fine; but (b) ping _from_ davinci, even "ping -c 1", ICMP RX loses
  52. * although ARP RX wins. (That test was done with a full speed link.)
  53. */
  54. /*
  55. * NOTE on endpoint usage:
  56. *
  57. * CONTROL transfers all go through ep0. BULK ones go through dedicated IN
  58. * and OUT endpoints ... hardware is dedicated for those "async" queue(s).
  59. * (Yes, bulk _could_ use more of the endpoints than that, and would even
  60. * benefit from it.)
  61. *
  62. * INTERUPPT and ISOCHRONOUS transfers are scheduled to the other endpoints.
  63. * So far that scheduling is both dumb and optimistic: the endpoint will be
  64. * "claimed" until its software queue is no longer refilled. No multiplexing
  65. * of transfers between endpoints, or anything clever.
  66. */
  67. struct musb *hcd_to_musb(struct usb_hcd *hcd)
  68. {
  69. return *(struct musb **) hcd->hcd_priv;
  70. }
  71. static void musb_ep_program(struct musb *musb, u8 epnum,
  72. struct urb *urb, int is_out,
  73. u8 *buf, u32 offset, u32 len);
  74. /*
  75. * Clear TX fifo. Needed to avoid BABBLE errors.
  76. */
  77. static void musb_h_tx_flush_fifo(struct musb_hw_ep *ep)
  78. {
  79. struct musb *musb = ep->musb;
  80. void __iomem *epio = ep->regs;
  81. u16 csr;
  82. int retries = 1000;
  83. csr = musb_readw(epio, MUSB_TXCSR);
  84. while (csr & MUSB_TXCSR_FIFONOTEMPTY) {
  85. csr |= MUSB_TXCSR_FLUSHFIFO | MUSB_TXCSR_TXPKTRDY;
  86. musb_writew(epio, MUSB_TXCSR, csr);
  87. csr = musb_readw(epio, MUSB_TXCSR);
  88. /*
  89. * FIXME: sometimes the tx fifo flush failed, it has been
  90. * observed during device disconnect on AM335x.
  91. *
  92. * To reproduce the issue, ensure tx urb(s) are queued when
  93. * unplug the usb device which is connected to AM335x usb
  94. * host port.
  95. *
  96. * I found using a usb-ethernet device and running iperf
  97. * (client on AM335x) has very high chance to trigger it.
  98. *
  99. * Better to turn on musb_dbg() in musb_cleanup_urb() with
  100. * CPPI enabled to see the issue when aborting the tx channel.
  101. */
  102. if (dev_WARN_ONCE(musb->controller, retries-- < 1,
  103. "Could not flush host TX%d fifo: csr: %04x\n",
  104. ep->epnum, csr))
  105. return;
  106. mdelay(1);
  107. }
  108. }
  109. static void musb_h_ep0_flush_fifo(struct musb_hw_ep *ep)
  110. {
  111. void __iomem *epio = ep->regs;
  112. u16 csr;
  113. int retries = 5;
  114. /* scrub any data left in the fifo */
  115. do {
  116. csr = musb_readw(epio, MUSB_TXCSR);
  117. if (!(csr & (MUSB_CSR0_TXPKTRDY | MUSB_CSR0_RXPKTRDY)))
  118. break;
  119. musb_writew(epio, MUSB_TXCSR, MUSB_CSR0_FLUSHFIFO);
  120. csr = musb_readw(epio, MUSB_TXCSR);
  121. udelay(10);
  122. } while (--retries);
  123. WARN(!retries, "Could not flush host TX%d fifo: csr: %04x\n",
  124. ep->epnum, csr);
  125. /* and reset for the next transfer */
  126. musb_writew(epio, MUSB_TXCSR, 0);
  127. }
  128. /*
  129. * Start transmit. Caller is responsible for locking shared resources.
  130. * musb must be locked.
  131. */
  132. static inline void musb_h_tx_start(struct musb_hw_ep *ep)
  133. {
  134. u16 txcsr;
  135. /* NOTE: no locks here; caller should lock and select EP */
  136. if (ep->epnum) {
  137. txcsr = musb_readw(ep->regs, MUSB_TXCSR);
  138. txcsr |= MUSB_TXCSR_TXPKTRDY | MUSB_TXCSR_H_WZC_BITS;
  139. musb_writew(ep->regs, MUSB_TXCSR, txcsr);
  140. } else {
  141. txcsr = MUSB_CSR0_H_SETUPPKT | MUSB_CSR0_TXPKTRDY;
  142. musb_writew(ep->regs, MUSB_CSR0, txcsr);
  143. }
  144. }
  145. static inline void musb_h_tx_dma_start(struct musb_hw_ep *ep)
  146. {
  147. u16 txcsr;
  148. /* NOTE: no locks here; caller should lock and select EP */
  149. txcsr = musb_readw(ep->regs, MUSB_TXCSR);
  150. txcsr |= MUSB_TXCSR_DMAENAB | MUSB_TXCSR_H_WZC_BITS;
  151. if (is_cppi_enabled(ep->musb))
  152. txcsr |= MUSB_TXCSR_DMAMODE;
  153. musb_writew(ep->regs, MUSB_TXCSR, txcsr);
  154. }
  155. static void musb_ep_set_qh(struct musb_hw_ep *ep, int is_in, struct musb_qh *qh)
  156. {
  157. if (is_in != 0 || ep->is_shared_fifo)
  158. ep->in_qh = qh;
  159. if (is_in == 0 || ep->is_shared_fifo)
  160. ep->out_qh = qh;
  161. }
  162. static struct musb_qh *musb_ep_get_qh(struct musb_hw_ep *ep, int is_in)
  163. {
  164. return is_in ? ep->in_qh : ep->out_qh;
  165. }
  166. /*
  167. * Start the URB at the front of an endpoint's queue
  168. * end must be claimed from the caller.
  169. *
  170. * Context: controller locked, irqs blocked
  171. */
  172. static void
  173. musb_start_urb(struct musb *musb, int is_in, struct musb_qh *qh)
  174. {
  175. u32 len;
  176. void __iomem *mbase = musb->mregs;
  177. struct urb *urb = next_urb(qh);
  178. void *buf = urb->transfer_buffer;
  179. u32 offset = 0;
  180. struct musb_hw_ep *hw_ep = qh->hw_ep;
  181. int epnum = hw_ep->epnum;
  182. /* initialize software qh state */
  183. qh->offset = 0;
  184. qh->segsize = 0;
  185. /* gather right source of data */
  186. switch (qh->type) {
  187. case USB_ENDPOINT_XFER_CONTROL:
  188. /* control transfers always start with SETUP */
  189. is_in = 0;
  190. musb->ep0_stage = MUSB_EP0_START;
  191. buf = urb->setup_packet;
  192. len = 8;
  193. break;
  194. case USB_ENDPOINT_XFER_ISOC:
  195. qh->iso_idx = 0;
  196. qh->frame = 0;
  197. offset = urb->iso_frame_desc[0].offset;
  198. len = urb->iso_frame_desc[0].length;
  199. break;
  200. default: /* bulk, interrupt */
  201. /* actual_length may be nonzero on retry paths */
  202. buf = urb->transfer_buffer + urb->actual_length;
  203. len = urb->transfer_buffer_length - urb->actual_length;
  204. }
  205. trace_musb_urb_start(musb, urb);
  206. /* Configure endpoint */
  207. musb_ep_set_qh(hw_ep, is_in, qh);
  208. musb_ep_program(musb, epnum, urb, !is_in, buf, offset, len);
  209. /* transmit may have more work: start it when it is time */
  210. if (is_in)
  211. return;
  212. /* determine if the time is right for a periodic transfer */
  213. switch (qh->type) {
  214. case USB_ENDPOINT_XFER_ISOC:
  215. case USB_ENDPOINT_XFER_INT:
  216. musb_dbg(musb, "check whether there's still time for periodic Tx");
  217. /* FIXME this doesn't implement that scheduling policy ...
  218. * or handle framecounter wrapping
  219. */
  220. if (1) { /* Always assume URB_ISO_ASAP */
  221. /* REVISIT the SOF irq handler shouldn't duplicate
  222. * this code; and we don't init urb->start_frame...
  223. */
  224. qh->frame = 0;
  225. goto start;
  226. } else {
  227. qh->frame = urb->start_frame;
  228. /* enable SOF interrupt so we can count down */
  229. musb_dbg(musb, "SOF for %d", epnum);
  230. #if 1 /* ifndef CONFIG_ARCH_DAVINCI */
  231. musb_writeb(mbase, MUSB_INTRUSBE, 0xff);
  232. #endif
  233. }
  234. break;
  235. default:
  236. start:
  237. musb_dbg(musb, "Start TX%d %s", epnum,
  238. hw_ep->tx_channel ? "dma" : "pio");
  239. if (!hw_ep->tx_channel)
  240. musb_h_tx_start(hw_ep);
  241. else if (is_cppi_enabled(musb) || tusb_dma_omap(musb))
  242. musb_h_tx_dma_start(hw_ep);
  243. }
  244. }
  245. /* Context: caller owns controller lock, IRQs are blocked */
  246. static void musb_giveback(struct musb *musb, struct urb *urb, int status)
  247. __releases(musb->lock)
  248. __acquires(musb->lock)
  249. {
  250. trace_musb_urb_gb(musb, urb);
  251. usb_hcd_unlink_urb_from_ep(musb->hcd, urb);
  252. spin_unlock(&musb->lock);
  253. usb_hcd_giveback_urb(musb->hcd, urb, status);
  254. spin_lock(&musb->lock);
  255. }
  256. /*
  257. * Advance this hardware endpoint's queue, completing the specified URB and
  258. * advancing to either the next URB queued to that qh, or else invalidating
  259. * that qh and advancing to the next qh scheduled after the current one.
  260. *
  261. * Context: caller owns controller lock, IRQs are blocked
  262. */
  263. static void musb_advance_schedule(struct musb *musb, struct urb *urb,
  264. struct musb_hw_ep *hw_ep, int is_in)
  265. {
  266. struct musb_qh *qh = musb_ep_get_qh(hw_ep, is_in);
  267. struct musb_hw_ep *ep = qh->hw_ep;
  268. int ready = qh->is_ready;
  269. int status;
  270. u16 toggle;
  271. status = (urb->status == -EINPROGRESS) ? 0 : urb->status;
  272. /* save toggle eagerly, for paranoia */
  273. switch (qh->type) {
  274. case USB_ENDPOINT_XFER_BULK:
  275. case USB_ENDPOINT_XFER_INT:
  276. toggle = musb->io.get_toggle(qh, !is_in);
  277. usb_settoggle(urb->dev, qh->epnum, !is_in, toggle ? 1 : 0);
  278. break;
  279. case USB_ENDPOINT_XFER_ISOC:
  280. if (status == 0 && urb->error_count)
  281. status = -EXDEV;
  282. break;
  283. }
  284. qh->is_ready = 0;
  285. musb_giveback(musb, urb, status);
  286. qh->is_ready = ready;
  287. /* reclaim resources (and bandwidth) ASAP; deschedule it, and
  288. * invalidate qh as soon as list_empty(&hep->urb_list)
  289. */
  290. if (list_empty(&qh->hep->urb_list)) {
  291. struct list_head *head;
  292. struct dma_controller *dma = musb->dma_controller;
  293. if (is_in) {
  294. ep->rx_reinit = 1;
  295. if (ep->rx_channel) {
  296. dma->channel_release(ep->rx_channel);
  297. ep->rx_channel = NULL;
  298. }
  299. } else {
  300. ep->tx_reinit = 1;
  301. if (ep->tx_channel) {
  302. dma->channel_release(ep->tx_channel);
  303. ep->tx_channel = NULL;
  304. }
  305. }
  306. /* Clobber old pointers to this qh */
  307. musb_ep_set_qh(ep, is_in, NULL);
  308. qh->hep->hcpriv = NULL;
  309. switch (qh->type) {
  310. case USB_ENDPOINT_XFER_CONTROL:
  311. case USB_ENDPOINT_XFER_BULK:
  312. /* fifo policy for these lists, except that NAKing
  313. * should rotate a qh to the end (for fairness).
  314. */
  315. if (qh->mux == 1) {
  316. head = qh->ring.prev;
  317. list_del(&qh->ring);
  318. kfree(qh);
  319. qh = first_qh(head);
  320. break;
  321. }
  322. fallthrough;
  323. case USB_ENDPOINT_XFER_ISOC:
  324. case USB_ENDPOINT_XFER_INT:
  325. /* this is where periodic bandwidth should be
  326. * de-allocated if it's tracked and allocated;
  327. * and where we'd update the schedule tree...
  328. */
  329. kfree(qh);
  330. qh = NULL;
  331. break;
  332. }
  333. }
  334. if (qh != NULL && qh->is_ready) {
  335. musb_dbg(musb, "... next ep%d %cX urb %p",
  336. hw_ep->epnum, is_in ? 'R' : 'T', next_urb(qh));
  337. musb_start_urb(musb, is_in, qh);
  338. }
  339. }
  340. static u16 musb_h_flush_rxfifo(struct musb_hw_ep *hw_ep, u16 csr)
  341. {
  342. /* we don't want fifo to fill itself again;
  343. * ignore dma (various models),
  344. * leave toggle alone (may not have been saved yet)
  345. */
  346. csr |= MUSB_RXCSR_FLUSHFIFO | MUSB_RXCSR_RXPKTRDY;
  347. csr &= ~(MUSB_RXCSR_H_REQPKT
  348. | MUSB_RXCSR_H_AUTOREQ
  349. | MUSB_RXCSR_AUTOCLEAR);
  350. /* write 2x to allow double buffering */
  351. musb_writew(hw_ep->regs, MUSB_RXCSR, csr);
  352. musb_writew(hw_ep->regs, MUSB_RXCSR, csr);
  353. /* flush writebuffer */
  354. return musb_readw(hw_ep->regs, MUSB_RXCSR);
  355. }
  356. /*
  357. * PIO RX for a packet (or part of it).
  358. */
  359. static bool
  360. musb_host_packet_rx(struct musb *musb, struct urb *urb, u8 epnum, u8 iso_err)
  361. {
  362. u16 rx_count;
  363. u8 *buf;
  364. u16 csr;
  365. bool done = false;
  366. u32 length;
  367. int do_flush = 0;
  368. struct musb_hw_ep *hw_ep = musb->endpoints + epnum;
  369. void __iomem *epio = hw_ep->regs;
  370. struct musb_qh *qh = hw_ep->in_qh;
  371. int pipe = urb->pipe;
  372. void *buffer = urb->transfer_buffer;
  373. /* musb_ep_select(mbase, epnum); */
  374. rx_count = musb_readw(epio, MUSB_RXCOUNT);
  375. musb_dbg(musb, "RX%d count %d, buffer %p len %d/%d", epnum, rx_count,
  376. urb->transfer_buffer, qh->offset,
  377. urb->transfer_buffer_length);
  378. /* unload FIFO */
  379. if (usb_pipeisoc(pipe)) {
  380. int status = 0;
  381. struct usb_iso_packet_descriptor *d;
  382. if (iso_err) {
  383. status = -EILSEQ;
  384. urb->error_count++;
  385. }
  386. d = urb->iso_frame_desc + qh->iso_idx;
  387. buf = buffer + d->offset;
  388. length = d->length;
  389. if (rx_count > length) {
  390. if (status == 0) {
  391. status = -EOVERFLOW;
  392. urb->error_count++;
  393. }
  394. musb_dbg(musb, "OVERFLOW %d into %d", rx_count, length);
  395. do_flush = 1;
  396. } else
  397. length = rx_count;
  398. urb->actual_length += length;
  399. d->actual_length = length;
  400. d->status = status;
  401. /* see if we are done */
  402. done = (++qh->iso_idx >= urb->number_of_packets);
  403. } else {
  404. /* non-isoch */
  405. buf = buffer + qh->offset;
  406. length = urb->transfer_buffer_length - qh->offset;
  407. if (rx_count > length) {
  408. if (urb->status == -EINPROGRESS)
  409. urb->status = -EOVERFLOW;
  410. musb_dbg(musb, "OVERFLOW %d into %d", rx_count, length);
  411. do_flush = 1;
  412. } else
  413. length = rx_count;
  414. urb->actual_length += length;
  415. qh->offset += length;
  416. /* see if we are done */
  417. done = (urb->actual_length == urb->transfer_buffer_length)
  418. || (rx_count < qh->maxpacket)
  419. || (urb->status != -EINPROGRESS);
  420. if (done
  421. && (urb->status == -EINPROGRESS)
  422. && (urb->transfer_flags & URB_SHORT_NOT_OK)
  423. && (urb->actual_length
  424. < urb->transfer_buffer_length))
  425. urb->status = -EREMOTEIO;
  426. }
  427. musb_read_fifo(hw_ep, length, buf);
  428. csr = musb_readw(epio, MUSB_RXCSR);
  429. csr |= MUSB_RXCSR_H_WZC_BITS;
  430. if (unlikely(do_flush))
  431. musb_h_flush_rxfifo(hw_ep, csr);
  432. else {
  433. /* REVISIT this assumes AUTOCLEAR is never set */
  434. csr &= ~(MUSB_RXCSR_RXPKTRDY | MUSB_RXCSR_H_REQPKT);
  435. if (!done)
  436. csr |= MUSB_RXCSR_H_REQPKT;
  437. musb_writew(epio, MUSB_RXCSR, csr);
  438. }
  439. return done;
  440. }
  441. /* we don't always need to reinit a given side of an endpoint...
  442. * when we do, use tx/rx reinit routine and then construct a new CSR
  443. * to address data toggle, NYET, and DMA or PIO.
  444. *
  445. * it's possible that driver bugs (especially for DMA) or aborting a
  446. * transfer might have left the endpoint busier than it should be.
  447. * the busy/not-empty tests are basically paranoia.
  448. */
  449. static void
  450. musb_rx_reinit(struct musb *musb, struct musb_qh *qh, u8 epnum)
  451. {
  452. struct musb_hw_ep *ep = musb->endpoints + epnum;
  453. u16 csr;
  454. /* NOTE: we know the "rx" fifo reinit never triggers for ep0.
  455. * That always uses tx_reinit since ep0 repurposes TX register
  456. * offsets; the initial SETUP packet is also a kind of OUT.
  457. */
  458. /* if programmed for Tx, put it in RX mode */
  459. if (ep->is_shared_fifo) {
  460. csr = musb_readw(ep->regs, MUSB_TXCSR);
  461. if (csr & MUSB_TXCSR_MODE) {
  462. musb_h_tx_flush_fifo(ep);
  463. csr = musb_readw(ep->regs, MUSB_TXCSR);
  464. musb_writew(ep->regs, MUSB_TXCSR,
  465. csr | MUSB_TXCSR_FRCDATATOG);
  466. }
  467. /*
  468. * Clear the MODE bit (and everything else) to enable Rx.
  469. * NOTE: we mustn't clear the DMAMODE bit before DMAENAB.
  470. */
  471. if (csr & MUSB_TXCSR_DMAMODE)
  472. musb_writew(ep->regs, MUSB_TXCSR, MUSB_TXCSR_DMAMODE);
  473. musb_writew(ep->regs, MUSB_TXCSR, 0);
  474. /* scrub all previous state, clearing toggle */
  475. }
  476. csr = musb_readw(ep->regs, MUSB_RXCSR);
  477. if (csr & MUSB_RXCSR_RXPKTRDY)
  478. WARNING("rx%d, packet/%d ready?\n", ep->epnum,
  479. musb_readw(ep->regs, MUSB_RXCOUNT));
  480. musb_h_flush_rxfifo(ep, MUSB_RXCSR_CLRDATATOG);
  481. /* target addr and (for multipoint) hub addr/port */
  482. if (musb->is_multipoint) {
  483. musb_write_rxfunaddr(musb, epnum, qh->addr_reg);
  484. musb_write_rxhubaddr(musb, epnum, qh->h_addr_reg);
  485. musb_write_rxhubport(musb, epnum, qh->h_port_reg);
  486. } else
  487. musb_writeb(musb->mregs, MUSB_FADDR, qh->addr_reg);
  488. /* protocol/endpoint, interval/NAKlimit, i/o size */
  489. musb_writeb(ep->regs, MUSB_RXTYPE, qh->type_reg);
  490. musb_writeb(ep->regs, MUSB_RXINTERVAL, qh->intv_reg);
  491. /* NOTE: bulk combining rewrites high bits of maxpacket */
  492. /* Set RXMAXP with the FIFO size of the endpoint
  493. * to disable double buffer mode.
  494. */
  495. musb_writew(ep->regs, MUSB_RXMAXP,
  496. qh->maxpacket | ((qh->hb_mult - 1) << 11));
  497. ep->rx_reinit = 0;
  498. }
  499. static void musb_tx_dma_set_mode_mentor(struct dma_controller *dma,
  500. struct musb_hw_ep *hw_ep, struct musb_qh *qh,
  501. struct urb *urb, u32 offset,
  502. u32 *length, u8 *mode)
  503. {
  504. struct dma_channel *channel = hw_ep->tx_channel;
  505. void __iomem *epio = hw_ep->regs;
  506. u16 pkt_size = qh->maxpacket;
  507. u16 csr;
  508. if (*length > channel->max_len)
  509. *length = channel->max_len;
  510. csr = musb_readw(epio, MUSB_TXCSR);
  511. if (*length > pkt_size) {
  512. *mode = 1;
  513. csr |= MUSB_TXCSR_DMAMODE | MUSB_TXCSR_DMAENAB;
  514. /* autoset shouldn't be set in high bandwidth */
  515. /*
  516. * Enable Autoset according to table
  517. * below
  518. * bulk_split hb_mult Autoset_Enable
  519. * 0 1 Yes(Normal)
  520. * 0 >1 No(High BW ISO)
  521. * 1 1 Yes(HS bulk)
  522. * 1 >1 Yes(FS bulk)
  523. */
  524. if (qh->hb_mult == 1 || (qh->hb_mult > 1 &&
  525. can_bulk_split(hw_ep->musb, qh->type)))
  526. csr |= MUSB_TXCSR_AUTOSET;
  527. } else {
  528. *mode = 0;
  529. csr &= ~(MUSB_TXCSR_AUTOSET | MUSB_TXCSR_DMAMODE);
  530. csr |= MUSB_TXCSR_DMAENAB; /* against programmer's guide */
  531. }
  532. channel->desired_mode = *mode;
  533. musb_writew(epio, MUSB_TXCSR, csr);
  534. }
  535. static void musb_tx_dma_set_mode_cppi_tusb(struct dma_controller *dma,
  536. struct musb_hw_ep *hw_ep,
  537. struct musb_qh *qh,
  538. struct urb *urb,
  539. u32 offset,
  540. u32 *length,
  541. u8 *mode)
  542. {
  543. struct dma_channel *channel = hw_ep->tx_channel;
  544. channel->actual_len = 0;
  545. /*
  546. * TX uses "RNDIS" mode automatically but needs help
  547. * to identify the zero-length-final-packet case.
  548. */
  549. *mode = (urb->transfer_flags & URB_ZERO_PACKET) ? 1 : 0;
  550. }
  551. static bool musb_tx_dma_program(struct dma_controller *dma,
  552. struct musb_hw_ep *hw_ep, struct musb_qh *qh,
  553. struct urb *urb, u32 offset, u32 length)
  554. {
  555. struct dma_channel *channel = hw_ep->tx_channel;
  556. u16 pkt_size = qh->maxpacket;
  557. u8 mode;
  558. if (musb_dma_inventra(hw_ep->musb) || musb_dma_ux500(hw_ep->musb))
  559. musb_tx_dma_set_mode_mentor(dma, hw_ep, qh, urb, offset,
  560. &length, &mode);
  561. else if (is_cppi_enabled(hw_ep->musb) || tusb_dma_omap(hw_ep->musb))
  562. musb_tx_dma_set_mode_cppi_tusb(dma, hw_ep, qh, urb, offset,
  563. &length, &mode);
  564. else
  565. return false;
  566. qh->segsize = length;
  567. /*
  568. * Ensure the data reaches to main memory before starting
  569. * DMA transfer
  570. */
  571. wmb();
  572. if (!dma->channel_program(channel, pkt_size, mode,
  573. urb->transfer_dma + offset, length)) {
  574. void __iomem *epio = hw_ep->regs;
  575. u16 csr;
  576. dma->channel_release(channel);
  577. hw_ep->tx_channel = NULL;
  578. csr = musb_readw(epio, MUSB_TXCSR);
  579. csr &= ~(MUSB_TXCSR_AUTOSET | MUSB_TXCSR_DMAENAB);
  580. musb_writew(epio, MUSB_TXCSR, csr | MUSB_TXCSR_H_WZC_BITS);
  581. return false;
  582. }
  583. return true;
  584. }
  585. /*
  586. * Program an HDRC endpoint as per the given URB
  587. * Context: irqs blocked, controller lock held
  588. */
  589. static void musb_ep_program(struct musb *musb, u8 epnum,
  590. struct urb *urb, int is_out,
  591. u8 *buf, u32 offset, u32 len)
  592. {
  593. struct dma_controller *dma_controller;
  594. struct dma_channel *dma_channel;
  595. u8 dma_ok;
  596. void __iomem *mbase = musb->mregs;
  597. struct musb_hw_ep *hw_ep = musb->endpoints + epnum;
  598. void __iomem *epio = hw_ep->regs;
  599. struct musb_qh *qh = musb_ep_get_qh(hw_ep, !is_out);
  600. u16 packet_sz = qh->maxpacket;
  601. u8 use_dma = 1;
  602. u16 csr;
  603. musb_dbg(musb, "%s hw%d urb %p spd%d dev%d ep%d%s "
  604. "h_addr%02x h_port%02x bytes %d",
  605. is_out ? "-->" : "<--",
  606. epnum, urb, urb->dev->speed,
  607. qh->addr_reg, qh->epnum, is_out ? "out" : "in",
  608. qh->h_addr_reg, qh->h_port_reg,
  609. len);
  610. musb_ep_select(mbase, epnum);
  611. if (is_out && !len) {
  612. use_dma = 0;
  613. csr = musb_readw(epio, MUSB_TXCSR);
  614. csr &= ~MUSB_TXCSR_DMAENAB;
  615. musb_writew(epio, MUSB_TXCSR, csr);
  616. hw_ep->tx_channel = NULL;
  617. }
  618. /* candidate for DMA? */
  619. dma_controller = musb->dma_controller;
  620. if (use_dma && is_dma_capable() && epnum && dma_controller) {
  621. dma_channel = is_out ? hw_ep->tx_channel : hw_ep->rx_channel;
  622. if (!dma_channel) {
  623. dma_channel = dma_controller->channel_alloc(
  624. dma_controller, hw_ep, is_out);
  625. if (is_out)
  626. hw_ep->tx_channel = dma_channel;
  627. else
  628. hw_ep->rx_channel = dma_channel;
  629. }
  630. } else
  631. dma_channel = NULL;
  632. /* make sure we clear DMAEnab, autoSet bits from previous run */
  633. /* OUT/transmit/EP0 or IN/receive? */
  634. if (is_out) {
  635. u16 csr;
  636. u16 int_txe;
  637. u16 load_count;
  638. csr = musb_readw(epio, MUSB_TXCSR);
  639. /* disable interrupt in case we flush */
  640. int_txe = musb->intrtxe;
  641. musb_writew(mbase, MUSB_INTRTXE, int_txe & ~(1 << epnum));
  642. /* general endpoint setup */
  643. if (epnum) {
  644. /* flush all old state, set default */
  645. /*
  646. * We could be flushing valid
  647. * packets in double buffering
  648. * case
  649. */
  650. if (!hw_ep->tx_double_buffered)
  651. musb_h_tx_flush_fifo(hw_ep);
  652. /*
  653. * We must not clear the DMAMODE bit before or in
  654. * the same cycle with the DMAENAB bit, so we clear
  655. * the latter first...
  656. */
  657. csr &= ~(MUSB_TXCSR_H_NAKTIMEOUT
  658. | MUSB_TXCSR_AUTOSET
  659. | MUSB_TXCSR_DMAENAB
  660. | MUSB_TXCSR_FRCDATATOG
  661. | MUSB_TXCSR_H_RXSTALL
  662. | MUSB_TXCSR_H_ERROR
  663. | MUSB_TXCSR_TXPKTRDY
  664. );
  665. csr |= MUSB_TXCSR_MODE;
  666. if (!hw_ep->tx_double_buffered)
  667. csr |= musb->io.set_toggle(qh, is_out, urb);
  668. musb_writew(epio, MUSB_TXCSR, csr);
  669. /* REVISIT may need to clear FLUSHFIFO ... */
  670. csr &= ~MUSB_TXCSR_DMAMODE;
  671. musb_writew(epio, MUSB_TXCSR, csr);
  672. csr = musb_readw(epio, MUSB_TXCSR);
  673. } else {
  674. /* endpoint 0: just flush */
  675. musb_h_ep0_flush_fifo(hw_ep);
  676. }
  677. /* target addr and (for multipoint) hub addr/port */
  678. if (musb->is_multipoint) {
  679. musb_write_txfunaddr(musb, epnum, qh->addr_reg);
  680. musb_write_txhubaddr(musb, epnum, qh->h_addr_reg);
  681. musb_write_txhubport(musb, epnum, qh->h_port_reg);
  682. /* FIXME if !epnum, do the same for RX ... */
  683. } else
  684. musb_writeb(mbase, MUSB_FADDR, qh->addr_reg);
  685. /* protocol/endpoint/interval/NAKlimit */
  686. if (epnum) {
  687. musb_writeb(epio, MUSB_TXTYPE, qh->type_reg);
  688. if (can_bulk_split(musb, qh->type)) {
  689. qh->hb_mult = hw_ep->max_packet_sz_tx
  690. / packet_sz;
  691. musb_writew(epio, MUSB_TXMAXP, packet_sz
  692. | ((qh->hb_mult) - 1) << 11);
  693. } else {
  694. musb_writew(epio, MUSB_TXMAXP,
  695. qh->maxpacket |
  696. ((qh->hb_mult - 1) << 11));
  697. }
  698. musb_writeb(epio, MUSB_TXINTERVAL, qh->intv_reg);
  699. } else {
  700. musb_writeb(epio, MUSB_NAKLIMIT0, qh->intv_reg);
  701. if (musb->is_multipoint)
  702. musb_writeb(epio, MUSB_TYPE0,
  703. qh->type_reg);
  704. }
  705. if (can_bulk_split(musb, qh->type))
  706. load_count = min((u32) hw_ep->max_packet_sz_tx,
  707. len);
  708. else
  709. load_count = min((u32) packet_sz, len);
  710. if (dma_channel && musb_tx_dma_program(dma_controller,
  711. hw_ep, qh, urb, offset, len))
  712. load_count = 0;
  713. if (load_count) {
  714. /* PIO to load FIFO */
  715. qh->segsize = load_count;
  716. if (!buf) {
  717. sg_miter_start(&qh->sg_miter, urb->sg, 1,
  718. SG_MITER_ATOMIC
  719. | SG_MITER_FROM_SG);
  720. if (!sg_miter_next(&qh->sg_miter)) {
  721. dev_err(musb->controller,
  722. "error: sg"
  723. "list empty\n");
  724. sg_miter_stop(&qh->sg_miter);
  725. goto finish;
  726. }
  727. buf = qh->sg_miter.addr + urb->sg->offset +
  728. urb->actual_length;
  729. load_count = min_t(u32, load_count,
  730. qh->sg_miter.length);
  731. musb_write_fifo(hw_ep, load_count, buf);
  732. qh->sg_miter.consumed = load_count;
  733. sg_miter_stop(&qh->sg_miter);
  734. } else
  735. musb_write_fifo(hw_ep, load_count, buf);
  736. }
  737. finish:
  738. /* re-enable interrupt */
  739. musb_writew(mbase, MUSB_INTRTXE, int_txe);
  740. /* IN/receive */
  741. } else {
  742. u16 csr = 0;
  743. if (hw_ep->rx_reinit) {
  744. musb_rx_reinit(musb, qh, epnum);
  745. csr |= musb->io.set_toggle(qh, is_out, urb);
  746. if (qh->type == USB_ENDPOINT_XFER_INT)
  747. csr |= MUSB_RXCSR_DISNYET;
  748. } else {
  749. csr = musb_readw(hw_ep->regs, MUSB_RXCSR);
  750. if (csr & (MUSB_RXCSR_RXPKTRDY
  751. | MUSB_RXCSR_DMAENAB
  752. | MUSB_RXCSR_H_REQPKT))
  753. ERR("broken !rx_reinit, ep%d csr %04x\n",
  754. hw_ep->epnum, csr);
  755. /* scrub any stale state, leaving toggle alone */
  756. csr &= MUSB_RXCSR_DISNYET;
  757. }
  758. /* kick things off */
  759. if ((is_cppi_enabled(musb) || tusb_dma_omap(musb)) && dma_channel) {
  760. /* Candidate for DMA */
  761. dma_channel->actual_len = 0L;
  762. qh->segsize = len;
  763. /* AUTOREQ is in a DMA register */
  764. musb_writew(hw_ep->regs, MUSB_RXCSR, csr);
  765. csr = musb_readw(hw_ep->regs, MUSB_RXCSR);
  766. /*
  767. * Unless caller treats short RX transfers as
  768. * errors, we dare not queue multiple transfers.
  769. */
  770. dma_ok = dma_controller->channel_program(dma_channel,
  771. packet_sz, !(urb->transfer_flags &
  772. URB_SHORT_NOT_OK),
  773. urb->transfer_dma + offset,
  774. qh->segsize);
  775. if (!dma_ok) {
  776. dma_controller->channel_release(dma_channel);
  777. hw_ep->rx_channel = dma_channel = NULL;
  778. } else
  779. csr |= MUSB_RXCSR_DMAENAB;
  780. }
  781. csr |= MUSB_RXCSR_H_REQPKT;
  782. musb_dbg(musb, "RXCSR%d := %04x", epnum, csr);
  783. musb_writew(hw_ep->regs, MUSB_RXCSR, csr);
  784. csr = musb_readw(hw_ep->regs, MUSB_RXCSR);
  785. }
  786. }
  787. /* Schedule next QH from musb->in_bulk/out_bulk and move the current qh to
  788. * the end; avoids starvation for other endpoints.
  789. */
  790. static void musb_bulk_nak_timeout(struct musb *musb, struct musb_hw_ep *ep,
  791. int is_in)
  792. {
  793. struct dma_channel *dma;
  794. struct urb *urb;
  795. void __iomem *mbase = musb->mregs;
  796. void __iomem *epio = ep->regs;
  797. struct musb_qh *cur_qh, *next_qh;
  798. u16 rx_csr, tx_csr;
  799. u16 toggle;
  800. musb_ep_select(mbase, ep->epnum);
  801. if (is_in) {
  802. dma = is_dma_capable() ? ep->rx_channel : NULL;
  803. /*
  804. * Need to stop the transaction by clearing REQPKT first
  805. * then the NAK Timeout bit ref MUSBMHDRC USB 2.0 HIGH-SPEED
  806. * DUAL-ROLE CONTROLLER Programmer's Guide, section 9.2.2
  807. */
  808. rx_csr = musb_readw(epio, MUSB_RXCSR);
  809. rx_csr |= MUSB_RXCSR_H_WZC_BITS;
  810. rx_csr &= ~MUSB_RXCSR_H_REQPKT;
  811. musb_writew(epio, MUSB_RXCSR, rx_csr);
  812. rx_csr &= ~MUSB_RXCSR_DATAERROR;
  813. musb_writew(epio, MUSB_RXCSR, rx_csr);
  814. cur_qh = first_qh(&musb->in_bulk);
  815. } else {
  816. dma = is_dma_capable() ? ep->tx_channel : NULL;
  817. /* clear nak timeout bit */
  818. tx_csr = musb_readw(epio, MUSB_TXCSR);
  819. tx_csr |= MUSB_TXCSR_H_WZC_BITS;
  820. tx_csr &= ~MUSB_TXCSR_H_NAKTIMEOUT;
  821. musb_writew(epio, MUSB_TXCSR, tx_csr);
  822. cur_qh = first_qh(&musb->out_bulk);
  823. }
  824. if (cur_qh) {
  825. urb = next_urb(cur_qh);
  826. if (dma_channel_status(dma) == MUSB_DMA_STATUS_BUSY) {
  827. dma->status = MUSB_DMA_STATUS_CORE_ABORT;
  828. musb->dma_controller->channel_abort(dma);
  829. urb->actual_length += dma->actual_len;
  830. dma->actual_len = 0L;
  831. }
  832. toggle = musb->io.get_toggle(cur_qh, !is_in);
  833. usb_settoggle(urb->dev, cur_qh->epnum, !is_in, toggle ? 1 : 0);
  834. if (is_in) {
  835. /* move cur_qh to end of queue */
  836. list_move_tail(&cur_qh->ring, &musb->in_bulk);
  837. /* get the next qh from musb->in_bulk */
  838. next_qh = first_qh(&musb->in_bulk);
  839. /* set rx_reinit and schedule the next qh */
  840. ep->rx_reinit = 1;
  841. } else {
  842. /* move cur_qh to end of queue */
  843. list_move_tail(&cur_qh->ring, &musb->out_bulk);
  844. /* get the next qh from musb->out_bulk */
  845. next_qh = first_qh(&musb->out_bulk);
  846. /* set tx_reinit and schedule the next qh */
  847. ep->tx_reinit = 1;
  848. }
  849. if (next_qh)
  850. musb_start_urb(musb, is_in, next_qh);
  851. }
  852. }
  853. /*
  854. * Service the default endpoint (ep0) as host.
  855. * Return true until it's time to start the status stage.
  856. */
  857. static bool musb_h_ep0_continue(struct musb *musb, u16 len, struct urb *urb)
  858. {
  859. bool more = false;
  860. u8 *fifo_dest = NULL;
  861. u16 fifo_count = 0;
  862. struct musb_hw_ep *hw_ep = musb->control_ep;
  863. struct musb_qh *qh = hw_ep->in_qh;
  864. struct usb_ctrlrequest *request;
  865. switch (musb->ep0_stage) {
  866. case MUSB_EP0_IN:
  867. fifo_dest = urb->transfer_buffer + urb->actual_length;
  868. fifo_count = min_t(size_t, len, urb->transfer_buffer_length -
  869. urb->actual_length);
  870. if (fifo_count < len)
  871. urb->status = -EOVERFLOW;
  872. musb_read_fifo(hw_ep, fifo_count, fifo_dest);
  873. urb->actual_length += fifo_count;
  874. if (len < qh->maxpacket) {
  875. /* always terminate on short read; it's
  876. * rarely reported as an error.
  877. */
  878. } else if (urb->actual_length <
  879. urb->transfer_buffer_length)
  880. more = true;
  881. break;
  882. case MUSB_EP0_START:
  883. request = (struct usb_ctrlrequest *) urb->setup_packet;
  884. if (!request->wLength) {
  885. musb_dbg(musb, "start no-DATA");
  886. break;
  887. } else if (request->bRequestType & USB_DIR_IN) {
  888. musb_dbg(musb, "start IN-DATA");
  889. musb->ep0_stage = MUSB_EP0_IN;
  890. more = true;
  891. break;
  892. } else {
  893. musb_dbg(musb, "start OUT-DATA");
  894. musb->ep0_stage = MUSB_EP0_OUT;
  895. more = true;
  896. }
  897. fallthrough;
  898. case MUSB_EP0_OUT:
  899. fifo_count = min_t(size_t, qh->maxpacket,
  900. urb->transfer_buffer_length -
  901. urb->actual_length);
  902. if (fifo_count) {
  903. fifo_dest = (u8 *) (urb->transfer_buffer
  904. + urb->actual_length);
  905. musb_dbg(musb, "Sending %d byte%s to ep0 fifo %p",
  906. fifo_count,
  907. (fifo_count == 1) ? "" : "s",
  908. fifo_dest);
  909. musb_write_fifo(hw_ep, fifo_count, fifo_dest);
  910. urb->actual_length += fifo_count;
  911. more = true;
  912. }
  913. break;
  914. default:
  915. ERR("bogus ep0 stage %d\n", musb->ep0_stage);
  916. break;
  917. }
  918. return more;
  919. }
  920. /*
  921. * Handle default endpoint interrupt as host. Only called in IRQ time
  922. * from musb_interrupt().
  923. *
  924. * called with controller irqlocked
  925. */
  926. irqreturn_t musb_h_ep0_irq(struct musb *musb)
  927. {
  928. struct urb *urb;
  929. u16 csr, len;
  930. int status = 0;
  931. void __iomem *mbase = musb->mregs;
  932. struct musb_hw_ep *hw_ep = musb->control_ep;
  933. void __iomem *epio = hw_ep->regs;
  934. struct musb_qh *qh = hw_ep->in_qh;
  935. bool complete = false;
  936. irqreturn_t retval = IRQ_NONE;
  937. /* ep0 only has one queue, "in" */
  938. urb = next_urb(qh);
  939. musb_ep_select(mbase, 0);
  940. csr = musb_readw(epio, MUSB_CSR0);
  941. len = (csr & MUSB_CSR0_RXPKTRDY)
  942. ? musb_readb(epio, MUSB_COUNT0)
  943. : 0;
  944. musb_dbg(musb, "<== csr0 %04x, qh %p, count %d, urb %p, stage %d",
  945. csr, qh, len, urb, musb->ep0_stage);
  946. /* if we just did status stage, we are done */
  947. if (MUSB_EP0_STATUS == musb->ep0_stage) {
  948. retval = IRQ_HANDLED;
  949. complete = true;
  950. }
  951. /* prepare status */
  952. if (csr & MUSB_CSR0_H_RXSTALL) {
  953. musb_dbg(musb, "STALLING ENDPOINT");
  954. status = -EPIPE;
  955. } else if (csr & MUSB_CSR0_H_ERROR) {
  956. musb_dbg(musb, "no response, csr0 %04x", csr);
  957. status = -EPROTO;
  958. } else if (csr & MUSB_CSR0_H_NAKTIMEOUT) {
  959. musb_dbg(musb, "control NAK timeout");
  960. /* NOTE: this code path would be a good place to PAUSE a
  961. * control transfer, if another one is queued, so that
  962. * ep0 is more likely to stay busy. That's already done
  963. * for bulk RX transfers.
  964. *
  965. * if (qh->ring.next != &musb->control), then
  966. * we have a candidate... NAKing is *NOT* an error
  967. */
  968. musb_writew(epio, MUSB_CSR0, 0);
  969. retval = IRQ_HANDLED;
  970. }
  971. if (status) {
  972. musb_dbg(musb, "aborting");
  973. retval = IRQ_HANDLED;
  974. if (urb)
  975. urb->status = status;
  976. complete = true;
  977. /* use the proper sequence to abort the transfer */
  978. if (csr & MUSB_CSR0_H_REQPKT) {
  979. csr &= ~MUSB_CSR0_H_REQPKT;
  980. musb_writew(epio, MUSB_CSR0, csr);
  981. csr &= ~MUSB_CSR0_H_NAKTIMEOUT;
  982. musb_writew(epio, MUSB_CSR0, csr);
  983. } else {
  984. musb_h_ep0_flush_fifo(hw_ep);
  985. }
  986. musb_writeb(epio, MUSB_NAKLIMIT0, 0);
  987. /* clear it */
  988. musb_writew(epio, MUSB_CSR0, 0);
  989. }
  990. if (unlikely(!urb)) {
  991. /* stop endpoint since we have no place for its data, this
  992. * SHOULD NEVER HAPPEN! */
  993. ERR("no URB for end 0\n");
  994. musb_h_ep0_flush_fifo(hw_ep);
  995. goto done;
  996. }
  997. if (!complete) {
  998. /* call common logic and prepare response */
  999. if (musb_h_ep0_continue(musb, len, urb)) {
  1000. /* more packets required */
  1001. csr = (MUSB_EP0_IN == musb->ep0_stage)
  1002. ? MUSB_CSR0_H_REQPKT : MUSB_CSR0_TXPKTRDY;
  1003. } else {
  1004. /* data transfer complete; perform status phase */
  1005. if (usb_pipeout(urb->pipe)
  1006. || !urb->transfer_buffer_length)
  1007. csr = MUSB_CSR0_H_STATUSPKT
  1008. | MUSB_CSR0_H_REQPKT;
  1009. else
  1010. csr = MUSB_CSR0_H_STATUSPKT
  1011. | MUSB_CSR0_TXPKTRDY;
  1012. /* disable ping token in status phase */
  1013. csr |= MUSB_CSR0_H_DIS_PING;
  1014. /* flag status stage */
  1015. musb->ep0_stage = MUSB_EP0_STATUS;
  1016. musb_dbg(musb, "ep0 STATUS, csr %04x", csr);
  1017. }
  1018. musb_writew(epio, MUSB_CSR0, csr);
  1019. retval = IRQ_HANDLED;
  1020. } else
  1021. musb->ep0_stage = MUSB_EP0_IDLE;
  1022. /* call completion handler if done */
  1023. if (complete)
  1024. musb_advance_schedule(musb, urb, hw_ep, 1);
  1025. done:
  1026. return retval;
  1027. }
  1028. #ifdef CONFIG_USB_INVENTRA_DMA
  1029. /* Host side TX (OUT) using Mentor DMA works as follows:
  1030. submit_urb ->
  1031. - if queue was empty, Program Endpoint
  1032. - ... which starts DMA to fifo in mode 1 or 0
  1033. DMA Isr (transfer complete) -> TxAvail()
  1034. - Stop DMA (~DmaEnab) (<--- Alert ... currently happens
  1035. only in musb_cleanup_urb)
  1036. - TxPktRdy has to be set in mode 0 or for
  1037. short packets in mode 1.
  1038. */
  1039. #endif
  1040. /* Service a Tx-Available or dma completion irq for the endpoint */
  1041. void musb_host_tx(struct musb *musb, u8 epnum)
  1042. {
  1043. int pipe;
  1044. bool done = false;
  1045. u16 tx_csr;
  1046. size_t length = 0;
  1047. size_t offset = 0;
  1048. struct musb_hw_ep *hw_ep = musb->endpoints + epnum;
  1049. void __iomem *epio = hw_ep->regs;
  1050. struct musb_qh *qh = hw_ep->out_qh;
  1051. struct urb *urb = next_urb(qh);
  1052. u32 status = 0;
  1053. void __iomem *mbase = musb->mregs;
  1054. struct dma_channel *dma;
  1055. bool transfer_pending = false;
  1056. musb_ep_select(mbase, epnum);
  1057. tx_csr = musb_readw(epio, MUSB_TXCSR);
  1058. /* with CPPI, DMA sometimes triggers "extra" irqs */
  1059. if (!urb) {
  1060. musb_dbg(musb, "extra TX%d ready, csr %04x", epnum, tx_csr);
  1061. return;
  1062. }
  1063. pipe = urb->pipe;
  1064. dma = is_dma_capable() ? hw_ep->tx_channel : NULL;
  1065. trace_musb_urb_tx(musb, urb);
  1066. musb_dbg(musb, "OUT/TX%d end, csr %04x%s", epnum, tx_csr,
  1067. dma ? ", dma" : "");
  1068. /* check for errors */
  1069. if (tx_csr & MUSB_TXCSR_H_RXSTALL) {
  1070. /* dma was disabled, fifo flushed */
  1071. musb_dbg(musb, "TX end %d stall", epnum);
  1072. /* stall; record URB status */
  1073. status = -EPIPE;
  1074. } else if (tx_csr & MUSB_TXCSR_H_ERROR) {
  1075. /* (NON-ISO) dma was disabled, fifo flushed */
  1076. musb_dbg(musb, "TX 3strikes on ep=%d", epnum);
  1077. status = -ETIMEDOUT;
  1078. } else if (tx_csr & MUSB_TXCSR_H_NAKTIMEOUT) {
  1079. if (USB_ENDPOINT_XFER_BULK == qh->type && qh->mux == 1
  1080. && !list_is_singular(&musb->out_bulk)) {
  1081. musb_dbg(musb, "NAK timeout on TX%d ep", epnum);
  1082. musb_bulk_nak_timeout(musb, hw_ep, 0);
  1083. } else {
  1084. musb_dbg(musb, "TX ep%d device not responding", epnum);
  1085. /* NOTE: this code path would be a good place to PAUSE a
  1086. * transfer, if there's some other (nonperiodic) tx urb
  1087. * that could use this fifo. (dma complicates it...)
  1088. * That's already done for bulk RX transfers.
  1089. *
  1090. * if (bulk && qh->ring.next != &musb->out_bulk), then
  1091. * we have a candidate... NAKing is *NOT* an error
  1092. */
  1093. musb_ep_select(mbase, epnum);
  1094. musb_writew(epio, MUSB_TXCSR,
  1095. MUSB_TXCSR_H_WZC_BITS
  1096. | MUSB_TXCSR_TXPKTRDY);
  1097. }
  1098. return;
  1099. }
  1100. done:
  1101. if (status) {
  1102. if (dma_channel_status(dma) == MUSB_DMA_STATUS_BUSY) {
  1103. dma->status = MUSB_DMA_STATUS_CORE_ABORT;
  1104. musb->dma_controller->channel_abort(dma);
  1105. }
  1106. /* do the proper sequence to abort the transfer in the
  1107. * usb core; the dma engine should already be stopped.
  1108. */
  1109. musb_h_tx_flush_fifo(hw_ep);
  1110. tx_csr &= ~(MUSB_TXCSR_AUTOSET
  1111. | MUSB_TXCSR_DMAENAB
  1112. | MUSB_TXCSR_H_ERROR
  1113. | MUSB_TXCSR_H_RXSTALL
  1114. | MUSB_TXCSR_H_NAKTIMEOUT
  1115. );
  1116. musb_ep_select(mbase, epnum);
  1117. musb_writew(epio, MUSB_TXCSR, tx_csr);
  1118. /* REVISIT may need to clear FLUSHFIFO ... */
  1119. musb_writew(epio, MUSB_TXCSR, tx_csr);
  1120. musb_writeb(epio, MUSB_TXINTERVAL, 0);
  1121. done = true;
  1122. }
  1123. /* second cppi case */
  1124. if (dma_channel_status(dma) == MUSB_DMA_STATUS_BUSY) {
  1125. musb_dbg(musb, "extra TX%d ready, csr %04x", epnum, tx_csr);
  1126. return;
  1127. }
  1128. if (is_dma_capable() && dma && !status) {
  1129. /*
  1130. * DMA has completed. But if we're using DMA mode 1 (multi
  1131. * packet DMA), we need a terminal TXPKTRDY interrupt before
  1132. * we can consider this transfer completed, lest we trash
  1133. * its last packet when writing the next URB's data. So we
  1134. * switch back to mode 0 to get that interrupt; we'll come
  1135. * back here once it happens.
  1136. */
  1137. if (tx_csr & MUSB_TXCSR_DMAMODE) {
  1138. /*
  1139. * We shouldn't clear DMAMODE with DMAENAB set; so
  1140. * clear them in a safe order. That should be OK
  1141. * once TXPKTRDY has been set (and I've never seen
  1142. * it being 0 at this moment -- DMA interrupt latency
  1143. * is significant) but if it hasn't been then we have
  1144. * no choice but to stop being polite and ignore the
  1145. * programmer's guide... :-)
  1146. *
  1147. * Note that we must write TXCSR with TXPKTRDY cleared
  1148. * in order not to re-trigger the packet send (this bit
  1149. * can't be cleared by CPU), and there's another caveat:
  1150. * TXPKTRDY may be set shortly and then cleared in the
  1151. * double-buffered FIFO mode, so we do an extra TXCSR
  1152. * read for debouncing...
  1153. */
  1154. tx_csr &= musb_readw(epio, MUSB_TXCSR);
  1155. if (tx_csr & MUSB_TXCSR_TXPKTRDY) {
  1156. tx_csr &= ~(MUSB_TXCSR_DMAENAB |
  1157. MUSB_TXCSR_TXPKTRDY);
  1158. musb_writew(epio, MUSB_TXCSR,
  1159. tx_csr | MUSB_TXCSR_H_WZC_BITS);
  1160. }
  1161. tx_csr &= ~(MUSB_TXCSR_DMAMODE |
  1162. MUSB_TXCSR_TXPKTRDY);
  1163. musb_writew(epio, MUSB_TXCSR,
  1164. tx_csr | MUSB_TXCSR_H_WZC_BITS);
  1165. /*
  1166. * There is no guarantee that we'll get an interrupt
  1167. * after clearing DMAMODE as we might have done this
  1168. * too late (after TXPKTRDY was cleared by controller).
  1169. * Re-read TXCSR as we have spoiled its previous value.
  1170. */
  1171. tx_csr = musb_readw(epio, MUSB_TXCSR);
  1172. }
  1173. /*
  1174. * We may get here from a DMA completion or TXPKTRDY interrupt.
  1175. * In any case, we must check the FIFO status here and bail out
  1176. * only if the FIFO still has data -- that should prevent the
  1177. * "missed" TXPKTRDY interrupts and deal with double-buffered
  1178. * FIFO mode too...
  1179. */
  1180. if (tx_csr & (MUSB_TXCSR_FIFONOTEMPTY | MUSB_TXCSR_TXPKTRDY)) {
  1181. musb_dbg(musb,
  1182. "DMA complete but FIFO not empty, CSR %04x",
  1183. tx_csr);
  1184. return;
  1185. }
  1186. }
  1187. if (!status || dma || usb_pipeisoc(pipe)) {
  1188. if (dma)
  1189. length = dma->actual_len;
  1190. else
  1191. length = qh->segsize;
  1192. qh->offset += length;
  1193. if (usb_pipeisoc(pipe)) {
  1194. struct usb_iso_packet_descriptor *d;
  1195. d = urb->iso_frame_desc + qh->iso_idx;
  1196. d->actual_length = length;
  1197. d->status = status;
  1198. if (++qh->iso_idx >= urb->number_of_packets) {
  1199. done = true;
  1200. } else {
  1201. d++;
  1202. offset = d->offset;
  1203. length = d->length;
  1204. }
  1205. } else if (dma && urb->transfer_buffer_length == qh->offset) {
  1206. done = true;
  1207. } else {
  1208. /* see if we need to send more data, or ZLP */
  1209. if (qh->segsize < qh->maxpacket)
  1210. done = true;
  1211. else if (qh->offset == urb->transfer_buffer_length
  1212. && !(urb->transfer_flags
  1213. & URB_ZERO_PACKET))
  1214. done = true;
  1215. if (!done) {
  1216. offset = qh->offset;
  1217. length = urb->transfer_buffer_length - offset;
  1218. transfer_pending = true;
  1219. }
  1220. }
  1221. }
  1222. /* urb->status != -EINPROGRESS means request has been faulted,
  1223. * so we must abort this transfer after cleanup
  1224. */
  1225. if (urb->status != -EINPROGRESS) {
  1226. done = true;
  1227. if (status == 0)
  1228. status = urb->status;
  1229. }
  1230. if (done) {
  1231. /* set status */
  1232. urb->status = status;
  1233. urb->actual_length = qh->offset;
  1234. musb_advance_schedule(musb, urb, hw_ep, USB_DIR_OUT);
  1235. return;
  1236. } else if ((usb_pipeisoc(pipe) || transfer_pending) && dma) {
  1237. if (musb_tx_dma_program(musb->dma_controller, hw_ep, qh, urb,
  1238. offset, length)) {
  1239. if (is_cppi_enabled(musb) || tusb_dma_omap(musb))
  1240. musb_h_tx_dma_start(hw_ep);
  1241. return;
  1242. }
  1243. } else if (tx_csr & MUSB_TXCSR_DMAENAB) {
  1244. musb_dbg(musb, "not complete, but DMA enabled?");
  1245. return;
  1246. }
  1247. /*
  1248. * PIO: start next packet in this URB.
  1249. *
  1250. * REVISIT: some docs say that when hw_ep->tx_double_buffered,
  1251. * (and presumably, FIFO is not half-full) we should write *two*
  1252. * packets before updating TXCSR; other docs disagree...
  1253. */
  1254. if (length > qh->maxpacket)
  1255. length = qh->maxpacket;
  1256. /* Unmap the buffer so that CPU can use it */
  1257. usb_hcd_unmap_urb_for_dma(musb->hcd, urb);
  1258. /*
  1259. * We need to map sg if the transfer_buffer is
  1260. * NULL.
  1261. */
  1262. if (!urb->transfer_buffer) {
  1263. /* sg_miter_start is already done in musb_ep_program */
  1264. if (!sg_miter_next(&qh->sg_miter)) {
  1265. dev_err(musb->controller, "error: sg list empty\n");
  1266. sg_miter_stop(&qh->sg_miter);
  1267. status = -EINVAL;
  1268. goto done;
  1269. }
  1270. length = min_t(u32, length, qh->sg_miter.length);
  1271. musb_write_fifo(hw_ep, length, qh->sg_miter.addr);
  1272. qh->sg_miter.consumed = length;
  1273. sg_miter_stop(&qh->sg_miter);
  1274. } else {
  1275. musb_write_fifo(hw_ep, length, urb->transfer_buffer + offset);
  1276. }
  1277. qh->segsize = length;
  1278. musb_ep_select(mbase, epnum);
  1279. musb_writew(epio, MUSB_TXCSR,
  1280. MUSB_TXCSR_H_WZC_BITS | MUSB_TXCSR_TXPKTRDY);
  1281. }
  1282. #ifdef CONFIG_USB_TI_CPPI41_DMA
  1283. /* Seems to set up ISO for cppi41 and not advance len. See commit c57c41d */
  1284. static int musb_rx_dma_iso_cppi41(struct dma_controller *dma,
  1285. struct musb_hw_ep *hw_ep,
  1286. struct musb_qh *qh,
  1287. struct urb *urb,
  1288. size_t len)
  1289. {
  1290. struct dma_channel *channel = hw_ep->rx_channel;
  1291. void __iomem *epio = hw_ep->regs;
  1292. dma_addr_t *buf;
  1293. u32 length;
  1294. u16 val;
  1295. buf = (void *)urb->iso_frame_desc[qh->iso_idx].offset +
  1296. (u32)urb->transfer_dma;
  1297. length = urb->iso_frame_desc[qh->iso_idx].length;
  1298. val = musb_readw(epio, MUSB_RXCSR);
  1299. val |= MUSB_RXCSR_DMAENAB;
  1300. musb_writew(hw_ep->regs, MUSB_RXCSR, val);
  1301. return dma->channel_program(channel, qh->maxpacket, 0,
  1302. (u32)buf, length);
  1303. }
  1304. #else
  1305. static inline int musb_rx_dma_iso_cppi41(struct dma_controller *dma,
  1306. struct musb_hw_ep *hw_ep,
  1307. struct musb_qh *qh,
  1308. struct urb *urb,
  1309. size_t len)
  1310. {
  1311. return false;
  1312. }
  1313. #endif
  1314. #if defined(CONFIG_USB_INVENTRA_DMA) || defined(CONFIG_USB_UX500_DMA) || \
  1315. defined(CONFIG_USB_TI_CPPI41_DMA)
  1316. /* Host side RX (IN) using Mentor DMA works as follows:
  1317. submit_urb ->
  1318. - if queue was empty, ProgramEndpoint
  1319. - first IN token is sent out (by setting ReqPkt)
  1320. LinuxIsr -> RxReady()
  1321. /\ => first packet is received
  1322. | - Set in mode 0 (DmaEnab, ~ReqPkt)
  1323. | -> DMA Isr (transfer complete) -> RxReady()
  1324. | - Ack receive (~RxPktRdy), turn off DMA (~DmaEnab)
  1325. | - if urb not complete, send next IN token (ReqPkt)
  1326. | | else complete urb.
  1327. | |
  1328. ---------------------------
  1329. *
  1330. * Nuances of mode 1:
  1331. * For short packets, no ack (+RxPktRdy) is sent automatically
  1332. * (even if AutoClear is ON)
  1333. * For full packets, ack (~RxPktRdy) and next IN token (+ReqPkt) is sent
  1334. * automatically => major problem, as collecting the next packet becomes
  1335. * difficult. Hence mode 1 is not used.
  1336. *
  1337. * REVISIT
  1338. * All we care about at this driver level is that
  1339. * (a) all URBs terminate with REQPKT cleared and fifo(s) empty;
  1340. * (b) termination conditions are: short RX, or buffer full;
  1341. * (c) fault modes include
  1342. * - iff URB_SHORT_NOT_OK, short RX status is -EREMOTEIO.
  1343. * (and that endpoint's dma queue stops immediately)
  1344. * - overflow (full, PLUS more bytes in the terminal packet)
  1345. *
  1346. * So for example, usb-storage sets URB_SHORT_NOT_OK, and would
  1347. * thus be a great candidate for using mode 1 ... for all but the
  1348. * last packet of one URB's transfer.
  1349. */
  1350. static int musb_rx_dma_inventra_cppi41(struct dma_controller *dma,
  1351. struct musb_hw_ep *hw_ep,
  1352. struct musb_qh *qh,
  1353. struct urb *urb,
  1354. size_t len)
  1355. {
  1356. struct dma_channel *channel = hw_ep->rx_channel;
  1357. void __iomem *epio = hw_ep->regs;
  1358. u16 val;
  1359. int pipe;
  1360. bool done;
  1361. pipe = urb->pipe;
  1362. if (usb_pipeisoc(pipe)) {
  1363. struct usb_iso_packet_descriptor *d;
  1364. d = urb->iso_frame_desc + qh->iso_idx;
  1365. d->actual_length = len;
  1366. /* even if there was an error, we did the dma
  1367. * for iso_frame_desc->length
  1368. */
  1369. if (d->status != -EILSEQ && d->status != -EOVERFLOW)
  1370. d->status = 0;
  1371. if (++qh->iso_idx >= urb->number_of_packets) {
  1372. done = true;
  1373. } else {
  1374. /* REVISIT: Why ignore return value here? */
  1375. if (musb_dma_cppi41(hw_ep->musb))
  1376. done = musb_rx_dma_iso_cppi41(dma, hw_ep, qh,
  1377. urb, len);
  1378. done = false;
  1379. }
  1380. } else {
  1381. /* done if urb buffer is full or short packet is recd */
  1382. done = (urb->actual_length + len >=
  1383. urb->transfer_buffer_length
  1384. || channel->actual_len < qh->maxpacket
  1385. || channel->rx_packet_done);
  1386. }
  1387. /* send IN token for next packet, without AUTOREQ */
  1388. if (!done) {
  1389. val = musb_readw(epio, MUSB_RXCSR);
  1390. val |= MUSB_RXCSR_H_REQPKT;
  1391. musb_writew(epio, MUSB_RXCSR, MUSB_RXCSR_H_WZC_BITS | val);
  1392. }
  1393. return done;
  1394. }
  1395. /* Disadvantage of using mode 1:
  1396. * It's basically usable only for mass storage class; essentially all
  1397. * other protocols also terminate transfers on short packets.
  1398. *
  1399. * Details:
  1400. * An extra IN token is sent at the end of the transfer (due to AUTOREQ)
  1401. * If you try to use mode 1 for (transfer_buffer_length - 512), and try
  1402. * to use the extra IN token to grab the last packet using mode 0, then
  1403. * the problem is that you cannot be sure when the device will send the
  1404. * last packet and RxPktRdy set. Sometimes the packet is recd too soon
  1405. * such that it gets lost when RxCSR is re-set at the end of the mode 1
  1406. * transfer, while sometimes it is recd just a little late so that if you
  1407. * try to configure for mode 0 soon after the mode 1 transfer is
  1408. * completed, you will find rxcount 0. Okay, so you might think why not
  1409. * wait for an interrupt when the pkt is recd. Well, you won't get any!
  1410. */
  1411. static int musb_rx_dma_in_inventra_cppi41(struct dma_controller *dma,
  1412. struct musb_hw_ep *hw_ep,
  1413. struct musb_qh *qh,
  1414. struct urb *urb,
  1415. size_t len,
  1416. u8 iso_err)
  1417. {
  1418. struct musb *musb = hw_ep->musb;
  1419. void __iomem *epio = hw_ep->regs;
  1420. struct dma_channel *channel = hw_ep->rx_channel;
  1421. u16 rx_count, val;
  1422. int length, pipe, done;
  1423. dma_addr_t buf;
  1424. rx_count = musb_readw(epio, MUSB_RXCOUNT);
  1425. pipe = urb->pipe;
  1426. if (usb_pipeisoc(pipe)) {
  1427. int d_status = 0;
  1428. struct usb_iso_packet_descriptor *d;
  1429. d = urb->iso_frame_desc + qh->iso_idx;
  1430. if (iso_err) {
  1431. d_status = -EILSEQ;
  1432. urb->error_count++;
  1433. }
  1434. if (rx_count > d->length) {
  1435. if (d_status == 0) {
  1436. d_status = -EOVERFLOW;
  1437. urb->error_count++;
  1438. }
  1439. musb_dbg(musb, "** OVERFLOW %d into %d",
  1440. rx_count, d->length);
  1441. length = d->length;
  1442. } else
  1443. length = rx_count;
  1444. d->status = d_status;
  1445. buf = urb->transfer_dma + d->offset;
  1446. } else {
  1447. length = rx_count;
  1448. buf = urb->transfer_dma + urb->actual_length;
  1449. }
  1450. channel->desired_mode = 0;
  1451. #ifdef USE_MODE1
  1452. /* because of the issue below, mode 1 will
  1453. * only rarely behave with correct semantics.
  1454. */
  1455. if ((urb->transfer_flags & URB_SHORT_NOT_OK)
  1456. && (urb->transfer_buffer_length - urb->actual_length)
  1457. > qh->maxpacket)
  1458. channel->desired_mode = 1;
  1459. if (rx_count < hw_ep->max_packet_sz_rx) {
  1460. length = rx_count;
  1461. channel->desired_mode = 0;
  1462. } else {
  1463. length = urb->transfer_buffer_length;
  1464. }
  1465. #endif
  1466. /* See comments above on disadvantages of using mode 1 */
  1467. val = musb_readw(epio, MUSB_RXCSR);
  1468. val &= ~MUSB_RXCSR_H_REQPKT;
  1469. if (channel->desired_mode == 0)
  1470. val &= ~MUSB_RXCSR_H_AUTOREQ;
  1471. else
  1472. val |= MUSB_RXCSR_H_AUTOREQ;
  1473. val |= MUSB_RXCSR_DMAENAB;
  1474. /* autoclear shouldn't be set in high bandwidth */
  1475. if (qh->hb_mult == 1)
  1476. val |= MUSB_RXCSR_AUTOCLEAR;
  1477. musb_writew(epio, MUSB_RXCSR, MUSB_RXCSR_H_WZC_BITS | val);
  1478. /* REVISIT if when actual_length != 0,
  1479. * transfer_buffer_length needs to be
  1480. * adjusted first...
  1481. */
  1482. done = dma->channel_program(channel, qh->maxpacket,
  1483. channel->desired_mode,
  1484. buf, length);
  1485. if (!done) {
  1486. dma->channel_release(channel);
  1487. hw_ep->rx_channel = NULL;
  1488. channel = NULL;
  1489. val = musb_readw(epio, MUSB_RXCSR);
  1490. val &= ~(MUSB_RXCSR_DMAENAB
  1491. | MUSB_RXCSR_H_AUTOREQ
  1492. | MUSB_RXCSR_AUTOCLEAR);
  1493. musb_writew(epio, MUSB_RXCSR, val);
  1494. }
  1495. return done;
  1496. }
  1497. #else
  1498. static inline int musb_rx_dma_inventra_cppi41(struct dma_controller *dma,
  1499. struct musb_hw_ep *hw_ep,
  1500. struct musb_qh *qh,
  1501. struct urb *urb,
  1502. size_t len)
  1503. {
  1504. return false;
  1505. }
  1506. static inline int musb_rx_dma_in_inventra_cppi41(struct dma_controller *dma,
  1507. struct musb_hw_ep *hw_ep,
  1508. struct musb_qh *qh,
  1509. struct urb *urb,
  1510. size_t len,
  1511. u8 iso_err)
  1512. {
  1513. return false;
  1514. }
  1515. #endif
  1516. /*
  1517. * Service an RX interrupt for the given IN endpoint; docs cover bulk, iso,
  1518. * and high-bandwidth IN transfer cases.
  1519. */
  1520. void musb_host_rx(struct musb *musb, u8 epnum)
  1521. {
  1522. struct urb *urb;
  1523. struct musb_hw_ep *hw_ep = musb->endpoints + epnum;
  1524. struct dma_controller *c = musb->dma_controller;
  1525. void __iomem *epio = hw_ep->regs;
  1526. struct musb_qh *qh = hw_ep->in_qh;
  1527. size_t xfer_len;
  1528. void __iomem *mbase = musb->mregs;
  1529. u16 rx_csr, val;
  1530. bool iso_err = false;
  1531. bool done = false;
  1532. u32 status;
  1533. struct dma_channel *dma;
  1534. unsigned int sg_flags = SG_MITER_ATOMIC | SG_MITER_TO_SG;
  1535. musb_ep_select(mbase, epnum);
  1536. urb = next_urb(qh);
  1537. dma = is_dma_capable() ? hw_ep->rx_channel : NULL;
  1538. status = 0;
  1539. xfer_len = 0;
  1540. rx_csr = musb_readw(epio, MUSB_RXCSR);
  1541. val = rx_csr;
  1542. if (unlikely(!urb)) {
  1543. /* REVISIT -- THIS SHOULD NEVER HAPPEN ... but, at least
  1544. * usbtest #11 (unlinks) triggers it regularly, sometimes
  1545. * with fifo full. (Only with DMA??)
  1546. */
  1547. musb_dbg(musb, "BOGUS RX%d ready, csr %04x, count %d",
  1548. epnum, val, musb_readw(epio, MUSB_RXCOUNT));
  1549. musb_h_flush_rxfifo(hw_ep, MUSB_RXCSR_CLRDATATOG);
  1550. return;
  1551. }
  1552. trace_musb_urb_rx(musb, urb);
  1553. /* check for errors, concurrent stall & unlink is not really
  1554. * handled yet! */
  1555. if (rx_csr & MUSB_RXCSR_H_RXSTALL) {
  1556. musb_dbg(musb, "RX end %d STALL", epnum);
  1557. /* stall; record URB status */
  1558. status = -EPIPE;
  1559. } else if (rx_csr & MUSB_RXCSR_H_ERROR) {
  1560. dev_err(musb->controller, "ep%d RX three-strikes error", epnum);
  1561. /*
  1562. * The three-strikes error could only happen when the USB
  1563. * device is not accessible, for example detached or powered
  1564. * off. So return the fatal error -ESHUTDOWN so hopefully the
  1565. * USB device drivers won't immediately resubmit the same URB.
  1566. */
  1567. status = -ESHUTDOWN;
  1568. musb_writeb(epio, MUSB_RXINTERVAL, 0);
  1569. rx_csr &= ~MUSB_RXCSR_H_ERROR;
  1570. musb_writew(epio, MUSB_RXCSR, rx_csr);
  1571. } else if (rx_csr & MUSB_RXCSR_DATAERROR) {
  1572. if (USB_ENDPOINT_XFER_ISOC != qh->type) {
  1573. musb_dbg(musb, "RX end %d NAK timeout", epnum);
  1574. /* NOTE: NAKing is *NOT* an error, so we want to
  1575. * continue. Except ... if there's a request for
  1576. * another QH, use that instead of starving it.
  1577. *
  1578. * Devices like Ethernet and serial adapters keep
  1579. * reads posted at all times, which will starve
  1580. * other devices without this logic.
  1581. */
  1582. if (usb_pipebulk(urb->pipe)
  1583. && qh->mux == 1
  1584. && !list_is_singular(&musb->in_bulk)) {
  1585. musb_bulk_nak_timeout(musb, hw_ep, 1);
  1586. return;
  1587. }
  1588. musb_ep_select(mbase, epnum);
  1589. rx_csr |= MUSB_RXCSR_H_WZC_BITS;
  1590. rx_csr &= ~MUSB_RXCSR_DATAERROR;
  1591. musb_writew(epio, MUSB_RXCSR, rx_csr);
  1592. goto finish;
  1593. } else {
  1594. musb_dbg(musb, "RX end %d ISO data error", epnum);
  1595. /* packet error reported later */
  1596. iso_err = true;
  1597. }
  1598. } else if (rx_csr & MUSB_RXCSR_INCOMPRX) {
  1599. musb_dbg(musb, "end %d high bandwidth incomplete ISO packet RX",
  1600. epnum);
  1601. status = -EPROTO;
  1602. }
  1603. /* faults abort the transfer */
  1604. if (status) {
  1605. /* clean up dma and collect transfer count */
  1606. if (dma_channel_status(dma) == MUSB_DMA_STATUS_BUSY) {
  1607. dma->status = MUSB_DMA_STATUS_CORE_ABORT;
  1608. musb->dma_controller->channel_abort(dma);
  1609. xfer_len = dma->actual_len;
  1610. }
  1611. musb_h_flush_rxfifo(hw_ep, MUSB_RXCSR_CLRDATATOG);
  1612. musb_writeb(epio, MUSB_RXINTERVAL, 0);
  1613. done = true;
  1614. goto finish;
  1615. }
  1616. if (unlikely(dma_channel_status(dma) == MUSB_DMA_STATUS_BUSY)) {
  1617. /* SHOULD NEVER HAPPEN ... but at least DaVinci has done it */
  1618. ERR("RX%d dma busy, csr %04x\n", epnum, rx_csr);
  1619. goto finish;
  1620. }
  1621. /* thorough shutdown for now ... given more precise fault handling
  1622. * and better queueing support, we might keep a DMA pipeline going
  1623. * while processing this irq for earlier completions.
  1624. */
  1625. /* FIXME this is _way_ too much in-line logic for Mentor DMA */
  1626. if (!musb_dma_inventra(musb) && !musb_dma_ux500(musb) &&
  1627. (rx_csr & MUSB_RXCSR_H_REQPKT)) {
  1628. /* REVISIT this happened for a while on some short reads...
  1629. * the cleanup still needs investigation... looks bad...
  1630. * and also duplicates dma cleanup code above ... plus,
  1631. * shouldn't this be the "half full" double buffer case?
  1632. */
  1633. if (dma_channel_status(dma) == MUSB_DMA_STATUS_BUSY) {
  1634. dma->status = MUSB_DMA_STATUS_CORE_ABORT;
  1635. musb->dma_controller->channel_abort(dma);
  1636. xfer_len = dma->actual_len;
  1637. done = true;
  1638. }
  1639. musb_dbg(musb, "RXCSR%d %04x, reqpkt, len %zu%s", epnum, rx_csr,
  1640. xfer_len, dma ? ", dma" : "");
  1641. rx_csr &= ~MUSB_RXCSR_H_REQPKT;
  1642. musb_ep_select(mbase, epnum);
  1643. musb_writew(epio, MUSB_RXCSR,
  1644. MUSB_RXCSR_H_WZC_BITS | rx_csr);
  1645. }
  1646. if (dma && (rx_csr & MUSB_RXCSR_DMAENAB)) {
  1647. xfer_len = dma->actual_len;
  1648. val &= ~(MUSB_RXCSR_DMAENAB
  1649. | MUSB_RXCSR_H_AUTOREQ
  1650. | MUSB_RXCSR_AUTOCLEAR
  1651. | MUSB_RXCSR_RXPKTRDY);
  1652. musb_writew(hw_ep->regs, MUSB_RXCSR, val);
  1653. if (musb_dma_inventra(musb) || musb_dma_ux500(musb) ||
  1654. musb_dma_cppi41(musb)) {
  1655. done = musb_rx_dma_inventra_cppi41(c, hw_ep, qh, urb, xfer_len);
  1656. musb_dbg(hw_ep->musb,
  1657. "ep %d dma %s, rxcsr %04x, rxcount %d",
  1658. epnum, done ? "off" : "reset",
  1659. musb_readw(epio, MUSB_RXCSR),
  1660. musb_readw(epio, MUSB_RXCOUNT));
  1661. } else {
  1662. done = true;
  1663. }
  1664. } else if (urb->status == -EINPROGRESS) {
  1665. /* if no errors, be sure a packet is ready for unloading */
  1666. if (unlikely(!(rx_csr & MUSB_RXCSR_RXPKTRDY))) {
  1667. status = -EPROTO;
  1668. ERR("Rx interrupt with no errors or packet!\n");
  1669. /* FIXME this is another "SHOULD NEVER HAPPEN" */
  1670. /* SCRUB (RX) */
  1671. /* do the proper sequence to abort the transfer */
  1672. musb_ep_select(mbase, epnum);
  1673. val &= ~MUSB_RXCSR_H_REQPKT;
  1674. musb_writew(epio, MUSB_RXCSR, val);
  1675. goto finish;
  1676. }
  1677. /* we are expecting IN packets */
  1678. if ((musb_dma_inventra(musb) || musb_dma_ux500(musb) ||
  1679. musb_dma_cppi41(musb)) && dma) {
  1680. musb_dbg(hw_ep->musb,
  1681. "RX%d count %d, buffer 0x%llx len %d/%d",
  1682. epnum, musb_readw(epio, MUSB_RXCOUNT),
  1683. (unsigned long long) urb->transfer_dma
  1684. + urb->actual_length,
  1685. qh->offset,
  1686. urb->transfer_buffer_length);
  1687. if (musb_rx_dma_in_inventra_cppi41(c, hw_ep, qh, urb,
  1688. xfer_len, iso_err))
  1689. goto finish;
  1690. else
  1691. dev_err(musb->controller, "error: rx_dma failed\n");
  1692. }
  1693. if (!dma) {
  1694. unsigned int received_len;
  1695. /* Unmap the buffer so that CPU can use it */
  1696. usb_hcd_unmap_urb_for_dma(musb->hcd, urb);
  1697. /*
  1698. * We need to map sg if the transfer_buffer is
  1699. * NULL.
  1700. */
  1701. if (!urb->transfer_buffer) {
  1702. qh->use_sg = true;
  1703. sg_miter_start(&qh->sg_miter, urb->sg, 1,
  1704. sg_flags);
  1705. }
  1706. if (qh->use_sg) {
  1707. if (!sg_miter_next(&qh->sg_miter)) {
  1708. dev_err(musb->controller, "error: sg list empty\n");
  1709. sg_miter_stop(&qh->sg_miter);
  1710. status = -EINVAL;
  1711. done = true;
  1712. goto finish;
  1713. }
  1714. urb->transfer_buffer = qh->sg_miter.addr;
  1715. received_len = urb->actual_length;
  1716. qh->offset = 0x0;
  1717. done = musb_host_packet_rx(musb, urb, epnum,
  1718. iso_err);
  1719. /* Calculate the number of bytes received */
  1720. received_len = urb->actual_length -
  1721. received_len;
  1722. qh->sg_miter.consumed = received_len;
  1723. sg_miter_stop(&qh->sg_miter);
  1724. } else {
  1725. done = musb_host_packet_rx(musb, urb,
  1726. epnum, iso_err);
  1727. }
  1728. musb_dbg(musb, "read %spacket", done ? "last " : "");
  1729. }
  1730. }
  1731. finish:
  1732. urb->actual_length += xfer_len;
  1733. qh->offset += xfer_len;
  1734. if (done) {
  1735. if (qh->use_sg) {
  1736. qh->use_sg = false;
  1737. urb->transfer_buffer = NULL;
  1738. }
  1739. if (urb->status == -EINPROGRESS)
  1740. urb->status = status;
  1741. musb_advance_schedule(musb, urb, hw_ep, USB_DIR_IN);
  1742. }
  1743. }
  1744. /* schedule nodes correspond to peripheral endpoints, like an OHCI QH.
  1745. * the software schedule associates multiple such nodes with a given
  1746. * host side hardware endpoint + direction; scheduling may activate
  1747. * that hardware endpoint.
  1748. */
  1749. static int musb_schedule(
  1750. struct musb *musb,
  1751. struct musb_qh *qh,
  1752. int is_in)
  1753. {
  1754. int idle = 0;
  1755. int best_diff;
  1756. int best_end, epnum;
  1757. struct musb_hw_ep *hw_ep = NULL;
  1758. struct list_head *head = NULL;
  1759. u8 toggle;
  1760. u8 txtype;
  1761. struct urb *urb = next_urb(qh);
  1762. /* use fixed hardware for control and bulk */
  1763. if (qh->type == USB_ENDPOINT_XFER_CONTROL) {
  1764. head = &musb->control;
  1765. hw_ep = musb->control_ep;
  1766. goto success;
  1767. }
  1768. /* else, periodic transfers get muxed to other endpoints */
  1769. /*
  1770. * We know this qh hasn't been scheduled, so all we need to do
  1771. * is choose which hardware endpoint to put it on ...
  1772. *
  1773. * REVISIT what we really want here is a regular schedule tree
  1774. * like e.g. OHCI uses.
  1775. */
  1776. best_diff = 4096;
  1777. best_end = -1;
  1778. for (epnum = 1, hw_ep = musb->endpoints + 1;
  1779. epnum < musb->nr_endpoints;
  1780. epnum++, hw_ep++) {
  1781. int diff;
  1782. if (musb_ep_get_qh(hw_ep, is_in) != NULL)
  1783. continue;
  1784. if (hw_ep == musb->bulk_ep)
  1785. continue;
  1786. if (is_in)
  1787. diff = hw_ep->max_packet_sz_rx;
  1788. else
  1789. diff = hw_ep->max_packet_sz_tx;
  1790. diff -= (qh->maxpacket * qh->hb_mult);
  1791. if (diff >= 0 && best_diff > diff) {
  1792. /*
  1793. * Mentor controller has a bug in that if we schedule
  1794. * a BULK Tx transfer on an endpoint that had earlier
  1795. * handled ISOC then the BULK transfer has to start on
  1796. * a zero toggle. If the BULK transfer starts on a 1
  1797. * toggle then this transfer will fail as the mentor
  1798. * controller starts the Bulk transfer on a 0 toggle
  1799. * irrespective of the programming of the toggle bits
  1800. * in the TXCSR register. Check for this condition
  1801. * while allocating the EP for a Tx Bulk transfer. If
  1802. * so skip this EP.
  1803. */
  1804. hw_ep = musb->endpoints + epnum;
  1805. toggle = usb_gettoggle(urb->dev, qh->epnum, !is_in);
  1806. txtype = (musb_readb(hw_ep->regs, MUSB_TXTYPE)
  1807. >> 4) & 0x3;
  1808. if (!is_in && (qh->type == USB_ENDPOINT_XFER_BULK) &&
  1809. toggle && (txtype == USB_ENDPOINT_XFER_ISOC))
  1810. continue;
  1811. best_diff = diff;
  1812. best_end = epnum;
  1813. }
  1814. }
  1815. /* use bulk reserved ep1 if no other ep is free */
  1816. if (best_end < 0 && qh->type == USB_ENDPOINT_XFER_BULK) {
  1817. hw_ep = musb->bulk_ep;
  1818. if (is_in)
  1819. head = &musb->in_bulk;
  1820. else
  1821. head = &musb->out_bulk;
  1822. /* Enable bulk RX/TX NAK timeout scheme when bulk requests are
  1823. * multiplexed. This scheme does not work in high speed to full
  1824. * speed scenario as NAK interrupts are not coming from a
  1825. * full speed device connected to a high speed device.
  1826. * NAK timeout interval is 8 (128 uframe or 16ms) for HS and
  1827. * 4 (8 frame or 8ms) for FS device.
  1828. */
  1829. if (qh->dev)
  1830. qh->intv_reg =
  1831. (USB_SPEED_HIGH == qh->dev->speed) ? 8 : 4;
  1832. goto success;
  1833. } else if (best_end < 0) {
  1834. dev_err(musb->controller,
  1835. "%s hwep alloc failed for %dx%d\n",
  1836. musb_ep_xfertype_string(qh->type),
  1837. qh->hb_mult, qh->maxpacket);
  1838. return -ENOSPC;
  1839. }
  1840. idle = 1;
  1841. qh->mux = 0;
  1842. hw_ep = musb->endpoints + best_end;
  1843. musb_dbg(musb, "qh %p periodic slot %d", qh, best_end);
  1844. success:
  1845. if (head) {
  1846. idle = list_empty(head);
  1847. list_add_tail(&qh->ring, head);
  1848. qh->mux = 1;
  1849. }
  1850. qh->hw_ep = hw_ep;
  1851. qh->hep->hcpriv = qh;
  1852. if (idle)
  1853. musb_start_urb(musb, is_in, qh);
  1854. return 0;
  1855. }
  1856. static int musb_urb_enqueue(
  1857. struct usb_hcd *hcd,
  1858. struct urb *urb,
  1859. gfp_t mem_flags)
  1860. {
  1861. unsigned long flags;
  1862. struct musb *musb = hcd_to_musb(hcd);
  1863. struct usb_host_endpoint *hep = urb->ep;
  1864. struct musb_qh *qh;
  1865. struct usb_endpoint_descriptor *epd = &hep->desc;
  1866. int ret;
  1867. unsigned type_reg;
  1868. unsigned interval;
  1869. /* host role must be active */
  1870. if (!is_host_active(musb) || !musb->is_active)
  1871. return -ENODEV;
  1872. trace_musb_urb_enq(musb, urb);
  1873. spin_lock_irqsave(&musb->lock, flags);
  1874. ret = usb_hcd_link_urb_to_ep(hcd, urb);
  1875. qh = ret ? NULL : hep->hcpriv;
  1876. if (qh)
  1877. urb->hcpriv = qh;
  1878. spin_unlock_irqrestore(&musb->lock, flags);
  1879. /* DMA mapping was already done, if needed, and this urb is on
  1880. * hep->urb_list now ... so we're done, unless hep wasn't yet
  1881. * scheduled onto a live qh.
  1882. *
  1883. * REVISIT best to keep hep->hcpriv valid until the endpoint gets
  1884. * disabled, testing for empty qh->ring and avoiding qh setup costs
  1885. * except for the first urb queued after a config change.
  1886. */
  1887. if (qh || ret)
  1888. return ret;
  1889. /* Allocate and initialize qh, minimizing the work done each time
  1890. * hw_ep gets reprogrammed, or with irqs blocked. Then schedule it.
  1891. *
  1892. * REVISIT consider a dedicated qh kmem_cache, so it's harder
  1893. * for bugs in other kernel code to break this driver...
  1894. */
  1895. qh = kzalloc(sizeof *qh, mem_flags);
  1896. if (!qh) {
  1897. spin_lock_irqsave(&musb->lock, flags);
  1898. usb_hcd_unlink_urb_from_ep(hcd, urb);
  1899. spin_unlock_irqrestore(&musb->lock, flags);
  1900. return -ENOMEM;
  1901. }
  1902. qh->hep = hep;
  1903. qh->dev = urb->dev;
  1904. INIT_LIST_HEAD(&qh->ring);
  1905. qh->is_ready = 1;
  1906. qh->maxpacket = usb_endpoint_maxp(epd);
  1907. qh->type = usb_endpoint_type(epd);
  1908. /* Bits 11 & 12 of wMaxPacketSize encode high bandwidth multiplier.
  1909. * Some musb cores don't support high bandwidth ISO transfers; and
  1910. * we don't (yet!) support high bandwidth interrupt transfers.
  1911. */
  1912. qh->hb_mult = usb_endpoint_maxp_mult(epd);
  1913. if (qh->hb_mult > 1) {
  1914. int ok = (qh->type == USB_ENDPOINT_XFER_ISOC);
  1915. if (ok)
  1916. ok = (usb_pipein(urb->pipe) && musb->hb_iso_rx)
  1917. || (usb_pipeout(urb->pipe) && musb->hb_iso_tx);
  1918. if (!ok) {
  1919. dev_err(musb->controller,
  1920. "high bandwidth %s (%dx%d) not supported\n",
  1921. musb_ep_xfertype_string(qh->type),
  1922. qh->hb_mult, qh->maxpacket & 0x7ff);
  1923. ret = -EMSGSIZE;
  1924. goto done;
  1925. }
  1926. qh->maxpacket &= 0x7ff;
  1927. }
  1928. qh->epnum = usb_endpoint_num(epd);
  1929. /* NOTE: urb->dev->devnum is wrong during SET_ADDRESS */
  1930. qh->addr_reg = (u8) usb_pipedevice(urb->pipe);
  1931. /* precompute rxtype/txtype/type0 register */
  1932. type_reg = (qh->type << 4) | qh->epnum;
  1933. switch (urb->dev->speed) {
  1934. case USB_SPEED_LOW:
  1935. type_reg |= 0xc0;
  1936. break;
  1937. case USB_SPEED_FULL:
  1938. type_reg |= 0x80;
  1939. break;
  1940. default:
  1941. type_reg |= 0x40;
  1942. }
  1943. qh->type_reg = type_reg;
  1944. /* Precompute RXINTERVAL/TXINTERVAL register */
  1945. switch (qh->type) {
  1946. case USB_ENDPOINT_XFER_INT:
  1947. /*
  1948. * Full/low speeds use the linear encoding,
  1949. * high speed uses the logarithmic encoding.
  1950. */
  1951. if (urb->dev->speed <= USB_SPEED_FULL) {
  1952. interval = max_t(u8, epd->bInterval, 1);
  1953. break;
  1954. }
  1955. fallthrough;
  1956. case USB_ENDPOINT_XFER_ISOC:
  1957. /* ISO always uses logarithmic encoding */
  1958. interval = min_t(u8, epd->bInterval, 16);
  1959. break;
  1960. default:
  1961. /* REVISIT we actually want to use NAK limits, hinting to the
  1962. * transfer scheduling logic to try some other qh, e.g. try
  1963. * for 2 msec first:
  1964. *
  1965. * interval = (USB_SPEED_HIGH == urb->dev->speed) ? 16 : 2;
  1966. *
  1967. * The downside of disabling this is that transfer scheduling
  1968. * gets VERY unfair for nonperiodic transfers; a misbehaving
  1969. * peripheral could make that hurt. That's perfectly normal
  1970. * for reads from network or serial adapters ... so we have
  1971. * partial NAKlimit support for bulk RX.
  1972. *
  1973. * The upside of disabling it is simpler transfer scheduling.
  1974. */
  1975. interval = 0;
  1976. }
  1977. qh->intv_reg = interval;
  1978. /* precompute addressing for external hub/tt ports */
  1979. if (musb->is_multipoint) {
  1980. struct usb_device *parent = urb->dev->parent;
  1981. if (parent != hcd->self.root_hub) {
  1982. qh->h_addr_reg = (u8) parent->devnum;
  1983. /* set up tt info if needed */
  1984. if (urb->dev->tt) {
  1985. qh->h_port_reg = (u8) urb->dev->ttport;
  1986. if (urb->dev->tt->hub)
  1987. qh->h_addr_reg =
  1988. (u8) urb->dev->tt->hub->devnum;
  1989. if (urb->dev->tt->multi)
  1990. qh->h_addr_reg |= 0x80;
  1991. }
  1992. }
  1993. }
  1994. /* invariant: hep->hcpriv is null OR the qh that's already scheduled.
  1995. * until we get real dma queues (with an entry for each urb/buffer),
  1996. * we only have work to do in the former case.
  1997. */
  1998. spin_lock_irqsave(&musb->lock, flags);
  1999. if (hep->hcpriv || !next_urb(qh)) {
  2000. /* some concurrent activity submitted another urb to hep...
  2001. * odd, rare, error prone, but legal.
  2002. */
  2003. kfree(qh);
  2004. qh = NULL;
  2005. ret = 0;
  2006. } else
  2007. ret = musb_schedule(musb, qh,
  2008. epd->bEndpointAddress & USB_ENDPOINT_DIR_MASK);
  2009. if (ret == 0) {
  2010. urb->hcpriv = qh;
  2011. /* FIXME set urb->start_frame for iso/intr, it's tested in
  2012. * musb_start_urb(), but otherwise only konicawc cares ...
  2013. */
  2014. }
  2015. spin_unlock_irqrestore(&musb->lock, flags);
  2016. done:
  2017. if (ret != 0) {
  2018. spin_lock_irqsave(&musb->lock, flags);
  2019. usb_hcd_unlink_urb_from_ep(hcd, urb);
  2020. spin_unlock_irqrestore(&musb->lock, flags);
  2021. kfree(qh);
  2022. }
  2023. return ret;
  2024. }
  2025. /*
  2026. * abort a transfer that's at the head of a hardware queue.
  2027. * called with controller locked, irqs blocked
  2028. * that hardware queue advances to the next transfer, unless prevented
  2029. */
  2030. static int musb_cleanup_urb(struct urb *urb, struct musb_qh *qh)
  2031. {
  2032. struct musb_hw_ep *ep = qh->hw_ep;
  2033. struct musb *musb = ep->musb;
  2034. void __iomem *epio = ep->regs;
  2035. unsigned hw_end = ep->epnum;
  2036. void __iomem *regs = ep->musb->mregs;
  2037. int is_in = usb_pipein(urb->pipe);
  2038. int status = 0;
  2039. u16 csr;
  2040. struct dma_channel *dma = NULL;
  2041. musb_ep_select(regs, hw_end);
  2042. if (is_dma_capable()) {
  2043. dma = is_in ? ep->rx_channel : ep->tx_channel;
  2044. if (dma) {
  2045. status = ep->musb->dma_controller->channel_abort(dma);
  2046. musb_dbg(musb, "abort %cX%d DMA for urb %p --> %d",
  2047. is_in ? 'R' : 'T', ep->epnum,
  2048. urb, status);
  2049. urb->actual_length += dma->actual_len;
  2050. }
  2051. }
  2052. /* turn off DMA requests, discard state, stop polling ... */
  2053. if (ep->epnum && is_in) {
  2054. /* giveback saves bulk toggle */
  2055. csr = musb_h_flush_rxfifo(ep, 0);
  2056. /* clear the endpoint's irq status here to avoid bogus irqs */
  2057. if (is_dma_capable() && dma)
  2058. musb_platform_clear_ep_rxintr(musb, ep->epnum);
  2059. } else if (ep->epnum) {
  2060. musb_h_tx_flush_fifo(ep);
  2061. csr = musb_readw(epio, MUSB_TXCSR);
  2062. csr &= ~(MUSB_TXCSR_AUTOSET
  2063. | MUSB_TXCSR_DMAENAB
  2064. | MUSB_TXCSR_H_RXSTALL
  2065. | MUSB_TXCSR_H_NAKTIMEOUT
  2066. | MUSB_TXCSR_H_ERROR
  2067. | MUSB_TXCSR_TXPKTRDY);
  2068. musb_writew(epio, MUSB_TXCSR, csr);
  2069. /* REVISIT may need to clear FLUSHFIFO ... */
  2070. musb_writew(epio, MUSB_TXCSR, csr);
  2071. /* flush cpu writebuffer */
  2072. csr = musb_readw(epio, MUSB_TXCSR);
  2073. } else {
  2074. musb_h_ep0_flush_fifo(ep);
  2075. }
  2076. if (status == 0)
  2077. musb_advance_schedule(ep->musb, urb, ep, is_in);
  2078. return status;
  2079. }
  2080. static int musb_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
  2081. {
  2082. struct musb *musb = hcd_to_musb(hcd);
  2083. struct musb_qh *qh;
  2084. unsigned long flags;
  2085. int is_in = usb_pipein(urb->pipe);
  2086. int ret;
  2087. trace_musb_urb_deq(musb, urb);
  2088. spin_lock_irqsave(&musb->lock, flags);
  2089. ret = usb_hcd_check_unlink_urb(hcd, urb, status);
  2090. if (ret)
  2091. goto done;
  2092. qh = urb->hcpriv;
  2093. if (!qh)
  2094. goto done;
  2095. /*
  2096. * Any URB not actively programmed into endpoint hardware can be
  2097. * immediately given back; that's any URB not at the head of an
  2098. * endpoint queue, unless someday we get real DMA queues. And even
  2099. * if it's at the head, it might not be known to the hardware...
  2100. *
  2101. * Otherwise abort current transfer, pending DMA, etc.; urb->status
  2102. * has already been updated. This is a synchronous abort; it'd be
  2103. * OK to hold off until after some IRQ, though.
  2104. *
  2105. * NOTE: qh is invalid unless !list_empty(&hep->urb_list)
  2106. */
  2107. if (!qh->is_ready
  2108. || urb->urb_list.prev != &qh->hep->urb_list
  2109. || musb_ep_get_qh(qh->hw_ep, is_in) != qh) {
  2110. int ready = qh->is_ready;
  2111. qh->is_ready = 0;
  2112. musb_giveback(musb, urb, 0);
  2113. qh->is_ready = ready;
  2114. /* If nothing else (usually musb_giveback) is using it
  2115. * and its URB list has emptied, recycle this qh.
  2116. */
  2117. if (ready && list_empty(&qh->hep->urb_list)) {
  2118. qh->hep->hcpriv = NULL;
  2119. list_del(&qh->ring);
  2120. kfree(qh);
  2121. }
  2122. } else
  2123. ret = musb_cleanup_urb(urb, qh);
  2124. done:
  2125. spin_unlock_irqrestore(&musb->lock, flags);
  2126. return ret;
  2127. }
  2128. /* disable an endpoint */
  2129. static void
  2130. musb_h_disable(struct usb_hcd *hcd, struct usb_host_endpoint *hep)
  2131. {
  2132. u8 is_in = hep->desc.bEndpointAddress & USB_DIR_IN;
  2133. unsigned long flags;
  2134. struct musb *musb = hcd_to_musb(hcd);
  2135. struct musb_qh *qh;
  2136. struct urb *urb;
  2137. spin_lock_irqsave(&musb->lock, flags);
  2138. qh = hep->hcpriv;
  2139. if (qh == NULL)
  2140. goto exit;
  2141. /* NOTE: qh is invalid unless !list_empty(&hep->urb_list) */
  2142. /* Kick the first URB off the hardware, if needed */
  2143. qh->is_ready = 0;
  2144. if (musb_ep_get_qh(qh->hw_ep, is_in) == qh) {
  2145. urb = next_urb(qh);
  2146. /* make software (then hardware) stop ASAP */
  2147. if (!urb->unlinked)
  2148. urb->status = -ESHUTDOWN;
  2149. /* cleanup */
  2150. musb_cleanup_urb(urb, qh);
  2151. /* Then nuke all the others ... and advance the
  2152. * queue on hw_ep (e.g. bulk ring) when we're done.
  2153. */
  2154. while (!list_empty(&hep->urb_list)) {
  2155. urb = next_urb(qh);
  2156. urb->status = -ESHUTDOWN;
  2157. musb_advance_schedule(musb, urb, qh->hw_ep, is_in);
  2158. }
  2159. } else {
  2160. /* Just empty the queue; the hardware is busy with
  2161. * other transfers, and since !qh->is_ready nothing
  2162. * will activate any of these as it advances.
  2163. */
  2164. while (!list_empty(&hep->urb_list))
  2165. musb_giveback(musb, next_urb(qh), -ESHUTDOWN);
  2166. hep->hcpriv = NULL;
  2167. list_del(&qh->ring);
  2168. kfree(qh);
  2169. }
  2170. exit:
  2171. spin_unlock_irqrestore(&musb->lock, flags);
  2172. }
  2173. static int musb_h_get_frame_number(struct usb_hcd *hcd)
  2174. {
  2175. struct musb *musb = hcd_to_musb(hcd);
  2176. return musb_readw(musb->mregs, MUSB_FRAME);
  2177. }
  2178. static int musb_h_start(struct usb_hcd *hcd)
  2179. {
  2180. struct musb *musb = hcd_to_musb(hcd);
  2181. /* NOTE: musb_start() is called when the hub driver turns
  2182. * on port power, or when (OTG) peripheral starts.
  2183. */
  2184. hcd->state = HC_STATE_RUNNING;
  2185. musb->port1_status = 0;
  2186. return 0;
  2187. }
  2188. static void musb_h_stop(struct usb_hcd *hcd)
  2189. {
  2190. musb_stop(hcd_to_musb(hcd));
  2191. hcd->state = HC_STATE_HALT;
  2192. }
  2193. static int musb_bus_suspend(struct usb_hcd *hcd)
  2194. {
  2195. struct musb *musb = hcd_to_musb(hcd);
  2196. u8 devctl;
  2197. int ret;
  2198. ret = musb_port_suspend(musb, true);
  2199. if (ret)
  2200. return ret;
  2201. if (!is_host_active(musb))
  2202. return 0;
  2203. switch (musb->xceiv->otg->state) {
  2204. case OTG_STATE_A_SUSPEND:
  2205. return 0;
  2206. case OTG_STATE_A_WAIT_VRISE:
  2207. /* ID could be grounded even if there's no device
  2208. * on the other end of the cable. NOTE that the
  2209. * A_WAIT_VRISE timers are messy with MUSB...
  2210. */
  2211. devctl = musb_readb(musb->mregs, MUSB_DEVCTL);
  2212. if ((devctl & MUSB_DEVCTL_VBUS) == MUSB_DEVCTL_VBUS)
  2213. musb->xceiv->otg->state = OTG_STATE_A_WAIT_BCON;
  2214. break;
  2215. default:
  2216. break;
  2217. }
  2218. if (musb->is_active) {
  2219. WARNING("trying to suspend as %s while active\n",
  2220. usb_otg_state_string(musb->xceiv->otg->state));
  2221. return -EBUSY;
  2222. } else
  2223. return 0;
  2224. }
  2225. static int musb_bus_resume(struct usb_hcd *hcd)
  2226. {
  2227. struct musb *musb = hcd_to_musb(hcd);
  2228. if (musb->config &&
  2229. musb->config->host_port_deassert_reset_at_resume)
  2230. musb_port_reset(musb, false);
  2231. return 0;
  2232. }
  2233. #ifndef CONFIG_MUSB_PIO_ONLY
  2234. #define MUSB_USB_DMA_ALIGN 4
  2235. struct musb_temp_buffer {
  2236. void *kmalloc_ptr;
  2237. void *old_xfer_buffer;
  2238. u8 data[];
  2239. };
  2240. static void musb_free_temp_buffer(struct urb *urb)
  2241. {
  2242. enum dma_data_direction dir;
  2243. struct musb_temp_buffer *temp;
  2244. size_t length;
  2245. if (!(urb->transfer_flags & URB_ALIGNED_TEMP_BUFFER))
  2246. return;
  2247. dir = usb_urb_dir_in(urb) ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
  2248. temp = container_of(urb->transfer_buffer, struct musb_temp_buffer,
  2249. data);
  2250. if (dir == DMA_FROM_DEVICE) {
  2251. if (usb_pipeisoc(urb->pipe))
  2252. length = urb->transfer_buffer_length;
  2253. else
  2254. length = urb->actual_length;
  2255. memcpy(temp->old_xfer_buffer, temp->data, length);
  2256. }
  2257. urb->transfer_buffer = temp->old_xfer_buffer;
  2258. kfree(temp->kmalloc_ptr);
  2259. urb->transfer_flags &= ~URB_ALIGNED_TEMP_BUFFER;
  2260. }
  2261. static int musb_alloc_temp_buffer(struct urb *urb, gfp_t mem_flags)
  2262. {
  2263. enum dma_data_direction dir;
  2264. struct musb_temp_buffer *temp;
  2265. void *kmalloc_ptr;
  2266. size_t kmalloc_size;
  2267. if (urb->num_sgs || urb->sg ||
  2268. urb->transfer_buffer_length == 0 ||
  2269. !((uintptr_t)urb->transfer_buffer & (MUSB_USB_DMA_ALIGN - 1)))
  2270. return 0;
  2271. dir = usb_urb_dir_in(urb) ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
  2272. /* Allocate a buffer with enough padding for alignment */
  2273. kmalloc_size = urb->transfer_buffer_length +
  2274. sizeof(struct musb_temp_buffer) + MUSB_USB_DMA_ALIGN - 1;
  2275. kmalloc_ptr = kmalloc(kmalloc_size, mem_flags);
  2276. if (!kmalloc_ptr)
  2277. return -ENOMEM;
  2278. /* Position our struct temp_buffer such that data is aligned */
  2279. temp = PTR_ALIGN(kmalloc_ptr, MUSB_USB_DMA_ALIGN);
  2280. temp->kmalloc_ptr = kmalloc_ptr;
  2281. temp->old_xfer_buffer = urb->transfer_buffer;
  2282. if (dir == DMA_TO_DEVICE)
  2283. memcpy(temp->data, urb->transfer_buffer,
  2284. urb->transfer_buffer_length);
  2285. urb->transfer_buffer = temp->data;
  2286. urb->transfer_flags |= URB_ALIGNED_TEMP_BUFFER;
  2287. return 0;
  2288. }
  2289. static int musb_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb,
  2290. gfp_t mem_flags)
  2291. {
  2292. struct musb *musb = hcd_to_musb(hcd);
  2293. int ret;
  2294. /*
  2295. * The DMA engine in RTL1.8 and above cannot handle
  2296. * DMA addresses that are not aligned to a 4 byte boundary.
  2297. * For such engine implemented (un)map_urb_for_dma hooks.
  2298. * Do not use these hooks for RTL<1.8
  2299. */
  2300. if (musb->hwvers < MUSB_HWVERS_1800)
  2301. return usb_hcd_map_urb_for_dma(hcd, urb, mem_flags);
  2302. ret = musb_alloc_temp_buffer(urb, mem_flags);
  2303. if (ret)
  2304. return ret;
  2305. ret = usb_hcd_map_urb_for_dma(hcd, urb, mem_flags);
  2306. if (ret)
  2307. musb_free_temp_buffer(urb);
  2308. return ret;
  2309. }
  2310. static void musb_unmap_urb_for_dma(struct usb_hcd *hcd, struct urb *urb)
  2311. {
  2312. struct musb *musb = hcd_to_musb(hcd);
  2313. usb_hcd_unmap_urb_for_dma(hcd, urb);
  2314. /* Do not use this hook for RTL<1.8 (see description above) */
  2315. if (musb->hwvers < MUSB_HWVERS_1800)
  2316. return;
  2317. musb_free_temp_buffer(urb);
  2318. }
  2319. #endif /* !CONFIG_MUSB_PIO_ONLY */
  2320. static const struct hc_driver musb_hc_driver = {
  2321. .description = "musb-hcd",
  2322. .product_desc = "MUSB HDRC host driver",
  2323. .hcd_priv_size = sizeof(struct musb *),
  2324. .flags = HCD_USB2 | HCD_DMA | HCD_MEMORY,
  2325. /* not using irq handler or reset hooks from usbcore, since
  2326. * those must be shared with peripheral code for OTG configs
  2327. */
  2328. .start = musb_h_start,
  2329. .stop = musb_h_stop,
  2330. .get_frame_number = musb_h_get_frame_number,
  2331. .urb_enqueue = musb_urb_enqueue,
  2332. .urb_dequeue = musb_urb_dequeue,
  2333. .endpoint_disable = musb_h_disable,
  2334. #ifndef CONFIG_MUSB_PIO_ONLY
  2335. .map_urb_for_dma = musb_map_urb_for_dma,
  2336. .unmap_urb_for_dma = musb_unmap_urb_for_dma,
  2337. #endif
  2338. .hub_status_data = musb_hub_status_data,
  2339. .hub_control = musb_hub_control,
  2340. .bus_suspend = musb_bus_suspend,
  2341. .bus_resume = musb_bus_resume,
  2342. /* .start_port_reset = NULL, */
  2343. /* .hub_irq_enable = NULL, */
  2344. };
  2345. int musb_host_alloc(struct musb *musb)
  2346. {
  2347. struct device *dev = musb->controller;
  2348. /* usbcore sets dev->driver_data to hcd, and sometimes uses that... */
  2349. musb->hcd = usb_create_hcd(&musb_hc_driver, dev, dev_name(dev));
  2350. if (!musb->hcd)
  2351. return -EINVAL;
  2352. *musb->hcd->hcd_priv = (unsigned long) musb;
  2353. musb->hcd->self.uses_pio_for_control = 1;
  2354. musb->hcd->uses_new_polling = 1;
  2355. musb->hcd->has_tt = 1;
  2356. return 0;
  2357. }
  2358. void musb_host_cleanup(struct musb *musb)
  2359. {
  2360. if (musb->port_mode == MUSB_PERIPHERAL)
  2361. return;
  2362. usb_remove_hcd(musb->hcd);
  2363. }
  2364. void musb_host_free(struct musb *musb)
  2365. {
  2366. usb_put_hcd(musb->hcd);
  2367. }
  2368. int musb_host_setup(struct musb *musb, int power_budget)
  2369. {
  2370. int ret;
  2371. struct usb_hcd *hcd = musb->hcd;
  2372. if (musb->port_mode == MUSB_HOST) {
  2373. MUSB_HST_MODE(musb);
  2374. musb->xceiv->otg->state = OTG_STATE_A_IDLE;
  2375. }
  2376. otg_set_host(musb->xceiv->otg, &hcd->self);
  2377. /* don't support otg protocols */
  2378. hcd->self.otg_port = 0;
  2379. musb->xceiv->otg->host = &hcd->self;
  2380. hcd->power_budget = 2 * (power_budget ? : 250);
  2381. hcd->skip_phy_initialization = 1;
  2382. ret = usb_add_hcd(hcd, 0, 0);
  2383. if (ret < 0)
  2384. return ret;
  2385. device_wakeup_enable(hcd->self.controller);
  2386. return 0;
  2387. }
  2388. void musb_host_resume_root_hub(struct musb *musb)
  2389. {
  2390. usb_hcd_resume_root_hub(musb->hcd);
  2391. }
  2392. void musb_host_poke_root_hub(struct musb *musb)
  2393. {
  2394. MUSB_HST_MODE(musb);
  2395. if (musb->hcd->status_urb)
  2396. usb_hcd_poll_rh_status(musb->hcd);
  2397. else
  2398. usb_hcd_resume_root_hub(musb->hcd);
  2399. }