gadget.c 72 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Cadence USBSS DRD Driver - gadget side.
  4. *
  5. * Copyright (C) 2018-2019 Cadence Design Systems.
  6. * Copyright (C) 2017-2018 NXP
  7. *
  8. * Authors: Pawel Jez <pjez@cadence.com>,
  9. * Pawel Laszczak <pawell@cadence.com>
  10. * Peter Chen <peter.chen@nxp.com>
  11. */
  12. /*
  13. * Work around 1:
  14. * At some situations, the controller may get stale data address in TRB
  15. * at below sequences:
  16. * 1. Controller read TRB includes data address
  17. * 2. Software updates TRBs includes data address and Cycle bit
  18. * 3. Controller read TRB which includes Cycle bit
  19. * 4. DMA run with stale data address
  20. *
  21. * To fix this problem, driver needs to make the first TRB in TD as invalid.
  22. * After preparing all TRBs driver needs to check the position of DMA and
  23. * if the DMA point to the first just added TRB and doorbell is 1,
  24. * then driver must defer making this TRB as valid. This TRB will be make
  25. * as valid during adding next TRB only if DMA is stopped or at TRBERR
  26. * interrupt.
  27. *
  28. * Issue has been fixed in DEV_VER_V3 version of controller.
  29. *
  30. * Work around 2:
  31. * Controller for OUT endpoints has shared on-chip buffers for all incoming
  32. * packets, including ep0out. It's FIFO buffer, so packets must be handle by DMA
  33. * in correct order. If the first packet in the buffer will not be handled,
  34. * then the following packets directed for other endpoints and functions
  35. * will be blocked.
  36. * Additionally the packets directed to one endpoint can block entire on-chip
  37. * buffers. In this case transfer to other endpoints also will blocked.
  38. *
  39. * To resolve this issue after raising the descriptor missing interrupt
  40. * driver prepares internal usb_request object and use it to arm DMA transfer.
  41. *
  42. * The problematic situation was observed in case when endpoint has been enabled
  43. * but no usb_request were queued. Driver try detects such endpoints and will
  44. * use this workaround only for these endpoint.
  45. *
  46. * Driver use limited number of buffer. This number can be set by macro
  47. * CDNS3_WA2_NUM_BUFFERS.
  48. *
  49. * Such blocking situation was observed on ACM gadget. For this function
  50. * host send OUT data packet but ACM function is not prepared for this packet.
  51. * It's cause that buffer placed in on chip memory block transfer to other
  52. * endpoints.
  53. *
  54. * Issue has been fixed in DEV_VER_V2 version of controller.
  55. *
  56. */
  57. #include <dm.h>
  58. #include <dm/device_compat.h>
  59. #include <dm/devres.h>
  60. #include <linux/bitops.h>
  61. #include <linux/delay.h>
  62. #include <linux/err.h>
  63. #include <linux/usb/gadget.h>
  64. #include <linux/compat.h>
  65. #include <linux/iopoll.h>
  66. #include <linux/dma-mapping.h>
  67. #include <linux/bitmap.h>
  68. #include <linux/bug.h>
  69. #include "core.h"
  70. #include "gadget-export.h"
  71. #include "gadget.h"
  72. #include "trace.h"
  73. #include "drd.h"
  74. #define readl_poll_timeout_atomic readl_poll_timeout
  75. #define usleep_range(a, b) udelay((b))
  76. static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
  77. struct usb_request *request,
  78. gfp_t gfp_flags);
  79. /**
  80. * cdns3_set_register_bit - set bit in given register.
  81. * @ptr: address of device controller register to be read and changed
  82. * @mask: bits requested to set
  83. */
  84. void cdns3_set_register_bit(void __iomem *ptr, u32 mask)
  85. {
  86. mask = readl(ptr) | mask;
  87. writel(mask, ptr);
  88. }
  89. /**
  90. * cdns3_ep_addr_to_index - Macro converts endpoint address to
  91. * index of endpoint object in cdns3_device.eps[] container
  92. * @ep_addr: endpoint address for which endpoint object is required
  93. *
  94. */
  95. u8 cdns3_ep_addr_to_index(u8 ep_addr)
  96. {
  97. return (((ep_addr & 0x7F)) + ((ep_addr & USB_DIR_IN) ? 16 : 0));
  98. }
  99. static int cdns3_get_dma_pos(struct cdns3_device *priv_dev,
  100. struct cdns3_endpoint *priv_ep)
  101. {
  102. int dma_index;
  103. dma_index = readl(&priv_dev->regs->ep_traddr) - priv_ep->trb_pool_dma;
  104. return dma_index / TRB_SIZE;
  105. }
  106. /**
  107. * cdns3_next_request - returns next request from list
  108. * @list: list containing requests
  109. *
  110. * Returns request or NULL if no requests in list
  111. */
  112. struct usb_request *cdns3_next_request(struct list_head *list)
  113. {
  114. return list_first_entry_or_null(list, struct usb_request, list);
  115. }
  116. /**
  117. * cdns3_next_align_buf - returns next buffer from list
  118. * @list: list containing buffers
  119. *
  120. * Returns buffer or NULL if no buffers in list
  121. */
  122. struct cdns3_aligned_buf *cdns3_next_align_buf(struct list_head *list)
  123. {
  124. return list_first_entry_or_null(list, struct cdns3_aligned_buf, list);
  125. }
  126. /**
  127. * cdns3_next_priv_request - returns next request from list
  128. * @list: list containing requests
  129. *
  130. * Returns request or NULL if no requests in list
  131. */
  132. struct cdns3_request *cdns3_next_priv_request(struct list_head *list)
  133. {
  134. return list_first_entry_or_null(list, struct cdns3_request, list);
  135. }
  136. /**
  137. * select_ep - selects endpoint
  138. * @priv_dev: extended gadget object
  139. * @ep: endpoint address
  140. */
  141. void cdns3_select_ep(struct cdns3_device *priv_dev, u32 ep)
  142. {
  143. if (priv_dev->selected_ep == ep)
  144. return;
  145. priv_dev->selected_ep = ep;
  146. writel(ep, &priv_dev->regs->ep_sel);
  147. }
  148. dma_addr_t cdns3_trb_virt_to_dma(struct cdns3_endpoint *priv_ep,
  149. struct cdns3_trb *trb)
  150. {
  151. u32 offset = (char *)trb - (char *)priv_ep->trb_pool;
  152. return priv_ep->trb_pool_dma + offset;
  153. }
  154. int cdns3_ring_size(struct cdns3_endpoint *priv_ep)
  155. {
  156. switch (priv_ep->type) {
  157. case USB_ENDPOINT_XFER_ISOC:
  158. return TRB_ISO_RING_SIZE;
  159. case USB_ENDPOINT_XFER_CONTROL:
  160. return TRB_CTRL_RING_SIZE;
  161. default:
  162. return TRB_RING_SIZE;
  163. }
  164. }
  165. /**
  166. * cdns3_allocate_trb_pool - Allocates TRB's pool for selected endpoint
  167. * @priv_ep: endpoint object
  168. *
  169. * Function will return 0 on success or -ENOMEM on allocation error
  170. */
  171. int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep)
  172. {
  173. int ring_size = cdns3_ring_size(priv_ep);
  174. struct cdns3_trb *link_trb;
  175. if (!priv_ep->trb_pool) {
  176. priv_ep->trb_pool =
  177. dma_alloc_coherent(ring_size,
  178. (unsigned long *)&priv_ep->trb_pool_dma);
  179. if (!priv_ep->trb_pool)
  180. return -ENOMEM;
  181. } else {
  182. memset(priv_ep->trb_pool, 0, ring_size);
  183. }
  184. if (!priv_ep->num)
  185. return 0;
  186. priv_ep->num_trbs = ring_size / TRB_SIZE;
  187. /* Initialize the last TRB as Link TRB. */
  188. link_trb = (priv_ep->trb_pool + (priv_ep->num_trbs - 1));
  189. link_trb->buffer = TRB_BUFFER(priv_ep->trb_pool_dma);
  190. link_trb->control = TRB_CYCLE | TRB_TYPE(TRB_LINK) | TRB_TOGGLE;
  191. return 0;
  192. }
  193. static void cdns3_free_trb_pool(struct cdns3_endpoint *priv_ep)
  194. {
  195. if (priv_ep->trb_pool) {
  196. dma_free_coherent(priv_ep->trb_pool);
  197. priv_ep->trb_pool = NULL;
  198. }
  199. }
  200. /**
  201. * cdns3_ep_stall_flush - Stalls and flushes selected endpoint
  202. * @priv_ep: endpoint object
  203. *
  204. * Endpoint must be selected before call to this function
  205. */
  206. static void cdns3_ep_stall_flush(struct cdns3_endpoint *priv_ep)
  207. {
  208. struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
  209. int val;
  210. trace_cdns3_halt(priv_ep, 1, 1);
  211. writel(EP_CMD_DFLUSH | EP_CMD_ERDY | EP_CMD_SSTALL,
  212. &priv_dev->regs->ep_cmd);
  213. /* wait for DFLUSH cleared */
  214. readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
  215. !(val & EP_CMD_DFLUSH), 1000);
  216. priv_ep->flags |= EP_STALLED;
  217. priv_ep->flags &= ~EP_STALL_PENDING;
  218. }
  219. /**
  220. * cdns3_hw_reset_eps_config - reset endpoints configuration kept by controller.
  221. * @priv_dev: extended gadget object
  222. */
  223. void cdns3_hw_reset_eps_config(struct cdns3_device *priv_dev)
  224. {
  225. writel(USB_CONF_CFGRST, &priv_dev->regs->usb_conf);
  226. cdns3_allow_enable_l1(priv_dev, 0);
  227. priv_dev->hw_configured_flag = 0;
  228. priv_dev->onchip_used_size = 0;
  229. priv_dev->out_mem_is_allocated = 0;
  230. priv_dev->wait_for_setup = 0;
  231. }
  232. /**
  233. * cdns3_ep_inc_trb - increment a trb index.
  234. * @index: Pointer to the TRB index to increment.
  235. * @cs: Cycle state
  236. * @trb_in_seg: number of TRBs in segment
  237. *
  238. * The index should never point to the link TRB. After incrementing,
  239. * if it is point to the link TRB, wrap around to the beginning and revert
  240. * cycle state bit The
  241. * link TRB is always at the last TRB entry.
  242. */
  243. static void cdns3_ep_inc_trb(int *index, u8 *cs, int trb_in_seg)
  244. {
  245. (*index)++;
  246. if (*index == (trb_in_seg - 1)) {
  247. *index = 0;
  248. *cs ^= 1;
  249. }
  250. }
  251. /**
  252. * cdns3_ep_inc_enq - increment endpoint's enqueue pointer
  253. * @priv_ep: The endpoint whose enqueue pointer we're incrementing
  254. */
  255. static void cdns3_ep_inc_enq(struct cdns3_endpoint *priv_ep)
  256. {
  257. priv_ep->free_trbs--;
  258. cdns3_ep_inc_trb(&priv_ep->enqueue, &priv_ep->pcs, priv_ep->num_trbs);
  259. }
  260. /**
  261. * cdns3_ep_inc_deq - increment endpoint's dequeue pointer
  262. * @priv_ep: The endpoint whose dequeue pointer we're incrementing
  263. */
  264. static void cdns3_ep_inc_deq(struct cdns3_endpoint *priv_ep)
  265. {
  266. priv_ep->free_trbs++;
  267. cdns3_ep_inc_trb(&priv_ep->dequeue, &priv_ep->ccs, priv_ep->num_trbs);
  268. }
  269. void cdns3_move_deq_to_next_trb(struct cdns3_request *priv_req)
  270. {
  271. struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
  272. int current_trb = priv_req->start_trb;
  273. while (current_trb != priv_req->end_trb) {
  274. cdns3_ep_inc_deq(priv_ep);
  275. current_trb = priv_ep->dequeue;
  276. }
  277. cdns3_ep_inc_deq(priv_ep);
  278. }
  279. /**
  280. * cdns3_allow_enable_l1 - enable/disable permits to transition to L1.
  281. * @priv_dev: Extended gadget object
  282. * @enable: Enable/disable permit to transition to L1.
  283. *
  284. * If bit USB_CONF_L1EN is set and device receive Extended Token packet,
  285. * then controller answer with ACK handshake.
  286. * If bit USB_CONF_L1DS is set and device receive Extended Token packet,
  287. * then controller answer with NYET handshake.
  288. */
  289. void cdns3_allow_enable_l1(struct cdns3_device *priv_dev, int enable)
  290. {
  291. if (enable)
  292. writel(USB_CONF_L1EN, &priv_dev->regs->usb_conf);
  293. else
  294. writel(USB_CONF_L1DS, &priv_dev->regs->usb_conf);
  295. }
  296. enum usb_device_speed cdns3_get_speed(struct cdns3_device *priv_dev)
  297. {
  298. u32 reg;
  299. reg = readl(&priv_dev->regs->usb_sts);
  300. if (DEV_SUPERSPEED(reg))
  301. return USB_SPEED_SUPER;
  302. else if (DEV_HIGHSPEED(reg))
  303. return USB_SPEED_HIGH;
  304. else if (DEV_FULLSPEED(reg))
  305. return USB_SPEED_FULL;
  306. else if (DEV_LOWSPEED(reg))
  307. return USB_SPEED_LOW;
  308. return USB_SPEED_UNKNOWN;
  309. }
  310. /**
  311. * cdns3_start_all_request - add to ring all request not started
  312. * @priv_dev: Extended gadget object
  313. * @priv_ep: The endpoint for whom request will be started.
  314. *
  315. * Returns return ENOMEM if transfer ring i not enough TRBs to start
  316. * all requests.
  317. */
  318. static int cdns3_start_all_request(struct cdns3_device *priv_dev,
  319. struct cdns3_endpoint *priv_ep)
  320. {
  321. struct usb_request *request;
  322. int ret = 0;
  323. while (!list_empty(&priv_ep->deferred_req_list)) {
  324. request = cdns3_next_request(&priv_ep->deferred_req_list);
  325. ret = cdns3_ep_run_transfer(priv_ep, request);
  326. if (ret)
  327. return ret;
  328. list_del(&request->list);
  329. list_add_tail(&request->list,
  330. &priv_ep->pending_req_list);
  331. }
  332. priv_ep->flags &= ~EP_RING_FULL;
  333. return ret;
  334. }
  335. /*
  336. * WA2: Set flag for all not ISOC OUT endpoints. If this flag is set
  337. * driver try to detect whether endpoint need additional internal
  338. * buffer for unblocking on-chip FIFO buffer. This flag will be cleared
  339. * if before first DESCMISS interrupt the DMA will be armed.
  340. */
  341. #define cdns3_wa2_enable_detection(priv_dev, ep_priv, reg) do { \
  342. if (!priv_ep->dir && priv_ep->type != USB_ENDPOINT_XFER_ISOC) { \
  343. priv_ep->flags |= EP_QUIRK_EXTRA_BUF_DET; \
  344. (reg) |= EP_STS_EN_DESCMISEN; \
  345. } } while (0)
  346. /**
  347. * cdns3_wa2_descmiss_copy_data copy data from internal requests to
  348. * request queued by class driver.
  349. * @priv_ep: extended endpoint object
  350. * @request: request object
  351. */
  352. static void cdns3_wa2_descmiss_copy_data(struct cdns3_endpoint *priv_ep,
  353. struct usb_request *request)
  354. {
  355. struct usb_request *descmiss_req;
  356. struct cdns3_request *descmiss_priv_req;
  357. while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
  358. int chunk_end;
  359. int length;
  360. descmiss_priv_req =
  361. cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
  362. descmiss_req = &descmiss_priv_req->request;
  363. /* driver can't touch pending request */
  364. if (descmiss_priv_req->flags & REQUEST_PENDING)
  365. break;
  366. chunk_end = descmiss_priv_req->flags & REQUEST_INTERNAL_CH;
  367. length = request->actual + descmiss_req->actual;
  368. request->status = descmiss_req->status;
  369. if (length <= request->length) {
  370. memcpy(&((u8 *)request->buf)[request->actual],
  371. descmiss_req->buf,
  372. descmiss_req->actual);
  373. request->actual = length;
  374. } else {
  375. /* It should never occur */
  376. request->status = -ENOMEM;
  377. }
  378. list_del_init(&descmiss_priv_req->list);
  379. kfree(descmiss_req->buf);
  380. cdns3_gadget_ep_free_request(&priv_ep->endpoint, descmiss_req);
  381. --priv_ep->wa2_counter;
  382. if (!chunk_end)
  383. break;
  384. }
  385. }
  386. struct usb_request *cdns3_wa2_gadget_giveback(struct cdns3_device *priv_dev,
  387. struct cdns3_endpoint *priv_ep,
  388. struct cdns3_request *priv_req)
  389. {
  390. if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN &&
  391. priv_req->flags & REQUEST_INTERNAL) {
  392. struct usb_request *req;
  393. req = cdns3_next_request(&priv_ep->deferred_req_list);
  394. priv_ep->descmis_req = NULL;
  395. if (!req)
  396. return NULL;
  397. cdns3_wa2_descmiss_copy_data(priv_ep, req);
  398. if (!(priv_ep->flags & EP_QUIRK_END_TRANSFER) &&
  399. req->length != req->actual) {
  400. /* wait for next part of transfer */
  401. return NULL;
  402. }
  403. if (req->status == -EINPROGRESS)
  404. req->status = 0;
  405. list_del_init(&req->list);
  406. cdns3_start_all_request(priv_dev, priv_ep);
  407. return req;
  408. }
  409. return &priv_req->request;
  410. }
  411. int cdns3_wa2_gadget_ep_queue(struct cdns3_device *priv_dev,
  412. struct cdns3_endpoint *priv_ep,
  413. struct cdns3_request *priv_req)
  414. {
  415. int deferred = 0;
  416. /*
  417. * If transfer was queued before DESCMISS appear than we
  418. * can disable handling of DESCMISS interrupt. Driver assumes that it
  419. * can disable special treatment for this endpoint.
  420. */
  421. if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
  422. u32 reg;
  423. cdns3_select_ep(priv_dev, priv_ep->num | priv_ep->dir);
  424. priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
  425. reg = readl(&priv_dev->regs->ep_sts_en);
  426. reg &= ~EP_STS_EN_DESCMISEN;
  427. trace_cdns3_wa2(priv_ep, "workaround disabled\n");
  428. writel(reg, &priv_dev->regs->ep_sts_en);
  429. }
  430. if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
  431. u8 pending_empty = list_empty(&priv_ep->pending_req_list);
  432. u8 descmiss_empty = list_empty(&priv_ep->wa2_descmiss_req_list);
  433. /*
  434. * DESCMISS transfer has been finished, so data will be
  435. * directly copied from internal allocated usb_request
  436. * objects.
  437. */
  438. if (pending_empty && !descmiss_empty &&
  439. !(priv_req->flags & REQUEST_INTERNAL)) {
  440. cdns3_wa2_descmiss_copy_data(priv_ep,
  441. &priv_req->request);
  442. trace_cdns3_wa2(priv_ep, "get internal stored data");
  443. list_add_tail(&priv_req->request.list,
  444. &priv_ep->pending_req_list);
  445. cdns3_gadget_giveback(priv_ep, priv_req,
  446. priv_req->request.status);
  447. /*
  448. * Intentionally driver returns positive value as
  449. * correct value. It informs that transfer has
  450. * been finished.
  451. */
  452. return EINPROGRESS;
  453. }
  454. /*
  455. * Driver will wait for completion DESCMISS transfer,
  456. * before starts new, not DESCMISS transfer.
  457. */
  458. if (!pending_empty && !descmiss_empty) {
  459. trace_cdns3_wa2(priv_ep, "wait for pending transfer\n");
  460. deferred = 1;
  461. }
  462. if (priv_req->flags & REQUEST_INTERNAL)
  463. list_add_tail(&priv_req->list,
  464. &priv_ep->wa2_descmiss_req_list);
  465. }
  466. return deferred;
  467. }
  468. static void cdns3_wa2_remove_old_request(struct cdns3_endpoint *priv_ep)
  469. {
  470. struct cdns3_request *priv_req;
  471. while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
  472. u8 chain;
  473. priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
  474. chain = !!(priv_req->flags & REQUEST_INTERNAL_CH);
  475. trace_cdns3_wa2(priv_ep, "removes eldest request");
  476. kfree(priv_req->request.buf);
  477. cdns3_gadget_ep_free_request(&priv_ep->endpoint,
  478. &priv_req->request);
  479. list_del_init(&priv_req->list);
  480. --priv_ep->wa2_counter;
  481. if (!chain)
  482. break;
  483. }
  484. }
  485. /**
  486. * cdns3_wa2_descmissing_packet - handles descriptor missing event.
  487. * @priv_dev: extended gadget object
  488. *
  489. * This function is used only for WA2. For more information see Work around 2
  490. * description.
  491. */
  492. static void cdns3_wa2_descmissing_packet(struct cdns3_endpoint *priv_ep)
  493. {
  494. struct cdns3_request *priv_req;
  495. struct usb_request *request;
  496. if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
  497. priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
  498. priv_ep->flags |= EP_QUIRK_EXTRA_BUF_EN;
  499. }
  500. trace_cdns3_wa2(priv_ep, "Description Missing detected\n");
  501. if (priv_ep->wa2_counter >= CDNS3_WA2_NUM_BUFFERS)
  502. cdns3_wa2_remove_old_request(priv_ep);
  503. request = cdns3_gadget_ep_alloc_request(&priv_ep->endpoint,
  504. GFP_ATOMIC);
  505. if (!request)
  506. goto err;
  507. priv_req = to_cdns3_request(request);
  508. priv_req->flags |= REQUEST_INTERNAL;
  509. /* if this field is still assigned it indicate that transfer related
  510. * with this request has not been finished yet. Driver in this
  511. * case simply allocate next request and assign flag REQUEST_INTERNAL_CH
  512. * flag to previous one. It will indicate that current request is
  513. * part of the previous one.
  514. */
  515. if (priv_ep->descmis_req)
  516. priv_ep->descmis_req->flags |= REQUEST_INTERNAL_CH;
  517. priv_req->request.buf = kzalloc(CDNS3_DESCMIS_BUF_SIZE,
  518. GFP_ATOMIC);
  519. priv_ep->wa2_counter++;
  520. if (!priv_req->request.buf) {
  521. cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
  522. goto err;
  523. }
  524. priv_req->request.length = CDNS3_DESCMIS_BUF_SIZE;
  525. priv_ep->descmis_req = priv_req;
  526. __cdns3_gadget_ep_queue(&priv_ep->endpoint,
  527. &priv_ep->descmis_req->request,
  528. GFP_ATOMIC);
  529. return;
  530. err:
  531. dev_err(priv_ep->cdns3_dev->dev,
  532. "Failed: No sufficient memory for DESCMIS\n");
  533. }
  534. /**
  535. * cdns3_gadget_giveback - call struct usb_request's ->complete callback
  536. * @priv_ep: The endpoint to whom the request belongs to
  537. * @priv_req: The request we're giving back
  538. * @status: completion code for the request
  539. *
  540. * Must be called with controller's lock held and interrupts disabled. This
  541. * function will unmap @req and call its ->complete() callback to notify upper
  542. * layers that it has completed.
  543. */
  544. void cdns3_gadget_giveback(struct cdns3_endpoint *priv_ep,
  545. struct cdns3_request *priv_req,
  546. int status)
  547. {
  548. struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
  549. struct usb_request *request = &priv_req->request;
  550. list_del_init(&request->list);
  551. if (request->status == -EINPROGRESS)
  552. request->status = status;
  553. usb_gadget_unmap_request(&priv_dev->gadget, request,
  554. priv_ep->dir);
  555. if ((priv_req->flags & REQUEST_UNALIGNED) &&
  556. priv_ep->dir == USB_DIR_OUT && !request->status)
  557. memcpy(request->buf, priv_req->aligned_buf->buf,
  558. request->length);
  559. priv_req->flags &= ~(REQUEST_PENDING | REQUEST_UNALIGNED);
  560. trace_cdns3_gadget_giveback(priv_req);
  561. if (priv_dev->dev_ver < DEV_VER_V2) {
  562. request = cdns3_wa2_gadget_giveback(priv_dev, priv_ep,
  563. priv_req);
  564. if (!request)
  565. return;
  566. }
  567. if (request->complete) {
  568. spin_unlock(&priv_dev->lock);
  569. usb_gadget_giveback_request(&priv_ep->endpoint,
  570. request);
  571. spin_lock(&priv_dev->lock);
  572. }
  573. if (request->buf == priv_dev->zlp_buf)
  574. cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
  575. }
  576. void cdns3_wa1_restore_cycle_bit(struct cdns3_endpoint *priv_ep)
  577. {
  578. /* Work around for stale data address in TRB*/
  579. if (priv_ep->wa1_set) {
  580. trace_cdns3_wa1(priv_ep, "restore cycle bit");
  581. priv_ep->wa1_set = 0;
  582. priv_ep->wa1_trb_index = 0xFFFF;
  583. if (priv_ep->wa1_cycle_bit) {
  584. priv_ep->wa1_trb->control =
  585. priv_ep->wa1_trb->control | 0x1;
  586. } else {
  587. priv_ep->wa1_trb->control =
  588. priv_ep->wa1_trb->control & ~0x1;
  589. }
  590. }
  591. }
  592. static void cdns3_free_aligned_request_buf(struct cdns3_device *priv_dev)
  593. {
  594. struct cdns3_aligned_buf *buf, *tmp;
  595. unsigned long flags;
  596. spin_lock_irqsave(&priv_dev->lock, flags);
  597. list_for_each_entry_safe(buf, tmp, &priv_dev->aligned_buf_list, list) {
  598. if (!buf->in_use) {
  599. list_del(&buf->list);
  600. /*
  601. * Re-enable interrupts to free DMA capable memory.
  602. * Driver can't free this memory with disabled
  603. * interrupts.
  604. */
  605. spin_unlock_irqrestore(&priv_dev->lock, flags);
  606. dma_free_coherent(buf->buf);
  607. kfree(buf);
  608. spin_lock_irqsave(&priv_dev->lock, flags);
  609. }
  610. }
  611. spin_unlock_irqrestore(&priv_dev->lock, flags);
  612. }
  613. static int cdns3_prepare_aligned_request_buf(struct cdns3_request *priv_req)
  614. {
  615. struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
  616. struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
  617. struct cdns3_aligned_buf *buf;
  618. /* check if buffer is aligned to 8. */
  619. if (!((uintptr_t)priv_req->request.buf & 0x7))
  620. return 0;
  621. buf = priv_req->aligned_buf;
  622. if (!buf || priv_req->request.length > buf->size) {
  623. buf = kzalloc(sizeof(*buf), GFP_ATOMIC);
  624. if (!buf)
  625. return -ENOMEM;
  626. buf->size = priv_req->request.length;
  627. buf->buf = dma_alloc_coherent(buf->size,
  628. (unsigned long *)&buf->dma);
  629. if (!buf->buf) {
  630. kfree(buf);
  631. return -ENOMEM;
  632. }
  633. if (priv_req->aligned_buf) {
  634. trace_cdns3_free_aligned_request(priv_req);
  635. priv_req->aligned_buf->in_use = 0;
  636. #ifndef __UBOOT__
  637. queue_work(system_freezable_wq,
  638. &priv_dev->aligned_buf_wq);
  639. #else
  640. cdns3_free_aligned_request_buf(priv_dev);
  641. #endif
  642. }
  643. buf->in_use = 1;
  644. priv_req->aligned_buf = buf;
  645. list_add_tail(&buf->list,
  646. &priv_dev->aligned_buf_list);
  647. }
  648. if (priv_ep->dir == USB_DIR_IN) {
  649. memcpy(buf->buf, priv_req->request.buf,
  650. priv_req->request.length);
  651. }
  652. priv_req->flags |= REQUEST_UNALIGNED;
  653. trace_cdns3_prepare_aligned_request(priv_req);
  654. return 0;
  655. }
  656. static int cdns3_wa1_update_guard(struct cdns3_endpoint *priv_ep,
  657. struct cdns3_trb *trb)
  658. {
  659. struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
  660. if (!priv_ep->wa1_set) {
  661. u32 doorbell;
  662. doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
  663. if (doorbell) {
  664. priv_ep->wa1_cycle_bit = priv_ep->pcs ? TRB_CYCLE : 0;
  665. priv_ep->wa1_set = 1;
  666. priv_ep->wa1_trb = trb;
  667. priv_ep->wa1_trb_index = priv_ep->enqueue;
  668. trace_cdns3_wa1(priv_ep, "set guard");
  669. return 0;
  670. }
  671. }
  672. return 1;
  673. }
  674. static void cdns3_wa1_tray_restore_cycle_bit(struct cdns3_device *priv_dev,
  675. struct cdns3_endpoint *priv_ep)
  676. {
  677. int dma_index;
  678. u32 doorbell;
  679. doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
  680. dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
  681. if (!doorbell || dma_index != priv_ep->wa1_trb_index)
  682. cdns3_wa1_restore_cycle_bit(priv_ep);
  683. }
  684. /**
  685. * cdns3_ep_run_transfer - start transfer on no-default endpoint hardware
  686. * @priv_ep: endpoint object
  687. *
  688. * Returns zero on success or negative value on failure
  689. */
  690. int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
  691. struct usb_request *request)
  692. {
  693. struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
  694. struct cdns3_request *priv_req;
  695. struct cdns3_trb *trb;
  696. dma_addr_t trb_dma;
  697. u32 togle_pcs = 1;
  698. int sg_iter = 0;
  699. int num_trb = 1;
  700. int address;
  701. u32 control;
  702. int pcs;
  703. if (num_trb > priv_ep->free_trbs) {
  704. priv_ep->flags |= EP_RING_FULL;
  705. return -ENOBUFS;
  706. }
  707. priv_req = to_cdns3_request(request);
  708. address = priv_ep->endpoint.desc->bEndpointAddress;
  709. priv_ep->flags |= EP_PENDING_REQUEST;
  710. /* must allocate buffer aligned to 8 */
  711. if (priv_req->flags & REQUEST_UNALIGNED)
  712. trb_dma = priv_req->aligned_buf->dma;
  713. else
  714. trb_dma = request->dma;
  715. trb = priv_ep->trb_pool + priv_ep->enqueue;
  716. priv_req->start_trb = priv_ep->enqueue;
  717. priv_req->trb = trb;
  718. cdns3_select_ep(priv_ep->cdns3_dev, address);
  719. /* prepare ring */
  720. if ((priv_ep->enqueue + num_trb) >= (priv_ep->num_trbs - 1)) {
  721. struct cdns3_trb *link_trb;
  722. int doorbell, dma_index;
  723. u32 ch_bit = 0;
  724. doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
  725. dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
  726. /* Driver can't update LINK TRB if it is current processed. */
  727. if (doorbell && dma_index == priv_ep->num_trbs - 1) {
  728. priv_ep->flags |= EP_DEFERRED_DRDY;
  729. return -ENOBUFS;
  730. }
  731. /*updating C bt in Link TRB before starting DMA*/
  732. link_trb = priv_ep->trb_pool + (priv_ep->num_trbs - 1);
  733. /*
  734. * For TRs size equal 2 enabling TRB_CHAIN for epXin causes
  735. * that DMA stuck at the LINK TRB.
  736. * On the other hand, removing TRB_CHAIN for longer TRs for
  737. * epXout cause that DMA stuck after handling LINK TRB.
  738. * To eliminate this strange behavioral driver set TRB_CHAIN
  739. * bit only for TR size > 2.
  740. */
  741. if (priv_ep->type == USB_ENDPOINT_XFER_ISOC ||
  742. TRBS_PER_SEGMENT > 2)
  743. ch_bit = TRB_CHAIN;
  744. link_trb->control = ((priv_ep->pcs) ? TRB_CYCLE : 0) |
  745. TRB_TYPE(TRB_LINK) | TRB_TOGGLE | ch_bit;
  746. }
  747. if (priv_dev->dev_ver <= DEV_VER_V2)
  748. togle_pcs = cdns3_wa1_update_guard(priv_ep, trb);
  749. /* set incorrect Cycle Bit for first trb*/
  750. control = priv_ep->pcs ? 0 : TRB_CYCLE;
  751. do {
  752. u32 length;
  753. u16 td_size = 0;
  754. /* fill TRB */
  755. control |= TRB_TYPE(TRB_NORMAL);
  756. trb->buffer = TRB_BUFFER(trb_dma);
  757. length = request->length;
  758. if (likely(priv_dev->dev_ver >= DEV_VER_V2))
  759. td_size = DIV_ROUND_UP(length,
  760. priv_ep->endpoint.maxpacket);
  761. trb->length = TRB_BURST_LEN(priv_ep->trb_burst_size) |
  762. TRB_LEN(length);
  763. if (priv_dev->gadget.speed == USB_SPEED_SUPER)
  764. trb->length |= TRB_TDL_SS_SIZE(td_size);
  765. else
  766. control |= TRB_TDL_HS_SIZE(td_size);
  767. pcs = priv_ep->pcs ? TRB_CYCLE : 0;
  768. /*
  769. * first trb should be prepared as last to avoid processing
  770. * transfer to early
  771. */
  772. if (sg_iter != 0)
  773. control |= pcs;
  774. if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) {
  775. control |= TRB_IOC | TRB_ISP;
  776. } else {
  777. /* for last element in TD or in SG list */
  778. if (sg_iter == (num_trb - 1) && sg_iter != 0)
  779. control |= pcs | TRB_IOC | TRB_ISP;
  780. }
  781. if (sg_iter)
  782. trb->control = control;
  783. else
  784. priv_req->trb->control = control;
  785. control = 0;
  786. ++sg_iter;
  787. priv_req->end_trb = priv_ep->enqueue;
  788. cdns3_ep_inc_enq(priv_ep);
  789. trb = priv_ep->trb_pool + priv_ep->enqueue;
  790. } while (sg_iter < num_trb);
  791. trb = priv_req->trb;
  792. priv_req->flags |= REQUEST_PENDING;
  793. if (sg_iter == 1)
  794. trb->control |= TRB_IOC | TRB_ISP;
  795. /*
  796. * Memory barrier - cycle bit must be set before other filds in trb.
  797. */
  798. dmb();
  799. /* give the TD to the consumer*/
  800. if (togle_pcs)
  801. trb->control = trb->control ^ 1;
  802. if (priv_dev->dev_ver <= DEV_VER_V2)
  803. cdns3_wa1_tray_restore_cycle_bit(priv_dev, priv_ep);
  804. trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
  805. /*
  806. * Memory barrier - Cycle Bit must be set before trb->length and
  807. * trb->buffer fields.
  808. */
  809. dmb();
  810. /*
  811. * For DMULT mode we can set address to transfer ring only once after
  812. * enabling endpoint.
  813. */
  814. if (priv_ep->flags & EP_UPDATE_EP_TRBADDR) {
  815. /*
  816. * Until SW is not ready to handle the OUT transfer the ISO OUT
  817. * Endpoint should be disabled (EP_CFG.ENABLE = 0).
  818. * EP_CFG_ENABLE must be set before updating ep_traddr.
  819. */
  820. if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir &&
  821. !(priv_ep->flags & EP_QUIRK_ISO_OUT_EN)) {
  822. priv_ep->flags |= EP_QUIRK_ISO_OUT_EN;
  823. cdns3_set_register_bit(&priv_dev->regs->ep_cfg,
  824. EP_CFG_ENABLE);
  825. }
  826. writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma +
  827. priv_req->start_trb * TRB_SIZE),
  828. &priv_dev->regs->ep_traddr);
  829. priv_ep->flags &= ~EP_UPDATE_EP_TRBADDR;
  830. }
  831. if (!priv_ep->wa1_set && !(priv_ep->flags & EP_STALLED)) {
  832. trace_cdns3_ring(priv_ep);
  833. /*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/
  834. writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
  835. writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
  836. trace_cdns3_doorbell_epx(priv_ep->name,
  837. readl(&priv_dev->regs->ep_traddr));
  838. }
  839. /* WORKAROUND for transition to L0 */
  840. __cdns3_gadget_wakeup(priv_dev);
  841. return 0;
  842. }
  843. void cdns3_set_hw_configuration(struct cdns3_device *priv_dev)
  844. {
  845. struct cdns3_endpoint *priv_ep;
  846. struct usb_ep *ep;
  847. int val;
  848. if (priv_dev->hw_configured_flag)
  849. return;
  850. writel(USB_CONF_CFGSET, &priv_dev->regs->usb_conf);
  851. writel(EP_CMD_ERDY | EP_CMD_REQ_CMPL, &priv_dev->regs->ep_cmd);
  852. cdns3_set_register_bit(&priv_dev->regs->usb_conf,
  853. USB_CONF_U1EN | USB_CONF_U2EN);
  854. /* wait until configuration set */
  855. readl_poll_timeout_atomic(&priv_dev->regs->usb_sts, val,
  856. val & USB_STS_CFGSTS_MASK, 100);
  857. priv_dev->hw_configured_flag = 1;
  858. list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
  859. priv_ep = ep_to_cdns3_ep(ep);
  860. if (priv_ep->flags & EP_ENABLED)
  861. cdns3_start_all_request(priv_dev, priv_ep);
  862. }
  863. }
  864. /**
  865. * cdns3_request_handled - check whether request has been handled by DMA
  866. *
  867. * @priv_ep: extended endpoint object.
  868. * @priv_req: request object for checking
  869. *
  870. * Endpoint must be selected before invoking this function.
  871. *
  872. * Returns false if request has not been handled by DMA, else returns true.
  873. *
  874. * SR - start ring
  875. * ER - end ring
  876. * DQ = priv_ep->dequeue - dequeue position
  877. * EQ = priv_ep->enqueue - enqueue position
  878. * ST = priv_req->start_trb - index of first TRB in transfer ring
  879. * ET = priv_req->end_trb - index of last TRB in transfer ring
  880. * CI = current_index - index of processed TRB by DMA.
  881. *
  882. * As first step, function checks if cycle bit for priv_req->start_trb is
  883. * correct.
  884. *
  885. * some rules:
  886. * 1. priv_ep->dequeue never exceed current_index.
  887. * 2 priv_ep->enqueue never exceed priv_ep->dequeue
  888. * 3. exception: priv_ep->enqueue == priv_ep->dequeue
  889. * and priv_ep->free_trbs is zero.
  890. * This case indicate that TR is full.
  891. *
  892. * Then We can split recognition into two parts:
  893. * Case 1 - priv_ep->dequeue < current_index
  894. * SR ... EQ ... DQ ... CI ... ER
  895. * SR ... DQ ... CI ... EQ ... ER
  896. *
  897. * Request has been handled by DMA if ST and ET is between DQ and CI.
  898. *
  899. * Case 2 - priv_ep->dequeue > current_index
  900. * This situation take place when CI go through the LINK TRB at the end of
  901. * transfer ring.
  902. * SR ... CI ... EQ ... DQ ... ER
  903. *
  904. * Request has been handled by DMA if ET is less then CI or
  905. * ET is greater or equal DQ.
  906. */
  907. static bool cdns3_request_handled(struct cdns3_endpoint *priv_ep,
  908. struct cdns3_request *priv_req)
  909. {
  910. struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
  911. struct cdns3_trb *trb = priv_req->trb;
  912. int current_index = 0;
  913. int handled = 0;
  914. int doorbell;
  915. current_index = cdns3_get_dma_pos(priv_dev, priv_ep);
  916. doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
  917. trb = &priv_ep->trb_pool[priv_req->start_trb];
  918. if ((trb->control & TRB_CYCLE) != priv_ep->ccs)
  919. goto finish;
  920. if (doorbell == 1 && current_index == priv_ep->dequeue)
  921. goto finish;
  922. /* The corner case for TRBS_PER_SEGMENT equal 2). */
  923. if (TRBS_PER_SEGMENT == 2 && priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
  924. handled = 1;
  925. goto finish;
  926. }
  927. if (priv_ep->enqueue == priv_ep->dequeue &&
  928. priv_ep->free_trbs == 0) {
  929. handled = 1;
  930. } else if (priv_ep->dequeue < current_index) {
  931. if ((current_index == (priv_ep->num_trbs - 1)) &&
  932. !priv_ep->dequeue)
  933. goto finish;
  934. if (priv_req->end_trb >= priv_ep->dequeue &&
  935. priv_req->end_trb < current_index)
  936. handled = 1;
  937. } else if (priv_ep->dequeue > current_index) {
  938. if (priv_req->end_trb < current_index ||
  939. priv_req->end_trb >= priv_ep->dequeue)
  940. handled = 1;
  941. }
  942. finish:
  943. trace_cdns3_request_handled(priv_req, current_index, handled);
  944. return handled;
  945. }
  946. static void cdns3_transfer_completed(struct cdns3_device *priv_dev,
  947. struct cdns3_endpoint *priv_ep)
  948. {
  949. struct cdns3_request *priv_req;
  950. struct usb_request *request;
  951. struct cdns3_trb *trb;
  952. while (!list_empty(&priv_ep->pending_req_list)) {
  953. request = cdns3_next_request(&priv_ep->pending_req_list);
  954. priv_req = to_cdns3_request(request);
  955. /* Re-select endpoint. It could be changed by other CPU during
  956. * handling usb_gadget_giveback_request.
  957. */
  958. #ifndef __UBOOT__
  959. cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
  960. #else
  961. cdns3_select_ep(priv_dev,
  962. priv_ep->endpoint.desc->bEndpointAddress);
  963. #endif
  964. if (!cdns3_request_handled(priv_ep, priv_req))
  965. goto prepare_next_td;
  966. trb = priv_ep->trb_pool + priv_ep->dequeue;
  967. trace_cdns3_complete_trb(priv_ep, trb);
  968. if (trb != priv_req->trb)
  969. dev_warn(priv_dev->dev,
  970. "request_trb=0x%p, queue_trb=0x%p\n",
  971. priv_req->trb, trb);
  972. request->actual = TRB_LEN(le32_to_cpu(trb->length));
  973. cdns3_move_deq_to_next_trb(priv_req);
  974. cdns3_gadget_giveback(priv_ep, priv_req, 0);
  975. if (priv_ep->type != USB_ENDPOINT_XFER_ISOC &&
  976. TRBS_PER_SEGMENT == 2)
  977. break;
  978. }
  979. priv_ep->flags &= ~EP_PENDING_REQUEST;
  980. prepare_next_td:
  981. if (!(priv_ep->flags & EP_STALLED) &&
  982. !(priv_ep->flags & EP_STALL_PENDING))
  983. cdns3_start_all_request(priv_dev, priv_ep);
  984. }
  985. void cdns3_rearm_transfer(struct cdns3_endpoint *priv_ep, u8 rearm)
  986. {
  987. struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
  988. cdns3_wa1_restore_cycle_bit(priv_ep);
  989. if (rearm) {
  990. trace_cdns3_ring(priv_ep);
  991. /* Cycle Bit must be updated before arming DMA. */
  992. dmb();
  993. writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
  994. __cdns3_gadget_wakeup(priv_dev);
  995. trace_cdns3_doorbell_epx(priv_ep->name,
  996. readl(&priv_dev->regs->ep_traddr));
  997. }
  998. }
  999. /**
  1000. * cdns3_check_ep_interrupt_proceed - Processes interrupt related to endpoint
  1001. * @priv_ep: endpoint object
  1002. *
  1003. * Returns 0
  1004. */
  1005. static int cdns3_check_ep_interrupt_proceed(struct cdns3_endpoint *priv_ep)
  1006. {
  1007. struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
  1008. u32 ep_sts_reg;
  1009. #ifndef __UBOOT__
  1010. cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
  1011. #else
  1012. cdns3_select_ep(priv_dev, priv_ep->endpoint.desc->bEndpointAddress);
  1013. #endif
  1014. trace_cdns3_epx_irq(priv_dev, priv_ep);
  1015. ep_sts_reg = readl(&priv_dev->regs->ep_sts);
  1016. writel(ep_sts_reg, &priv_dev->regs->ep_sts);
  1017. if (ep_sts_reg & EP_STS_TRBERR) {
  1018. if (priv_ep->flags & EP_STALL_PENDING &&
  1019. !(ep_sts_reg & EP_STS_DESCMIS &&
  1020. priv_dev->dev_ver < DEV_VER_V2)) {
  1021. cdns3_ep_stall_flush(priv_ep);
  1022. }
  1023. /*
  1024. * For isochronous transfer driver completes request on
  1025. * IOC or on TRBERR. IOC appears only when device receive
  1026. * OUT data packet. If host disable stream or lost some packet
  1027. * then the only way to finish all queued transfer is to do it
  1028. * on TRBERR event.
  1029. */
  1030. if (priv_ep->type == USB_ENDPOINT_XFER_ISOC &&
  1031. !priv_ep->wa1_set) {
  1032. if (!priv_ep->dir) {
  1033. u32 ep_cfg = readl(&priv_dev->regs->ep_cfg);
  1034. ep_cfg &= ~EP_CFG_ENABLE;
  1035. writel(ep_cfg, &priv_dev->regs->ep_cfg);
  1036. priv_ep->flags &= ~EP_QUIRK_ISO_OUT_EN;
  1037. }
  1038. cdns3_transfer_completed(priv_dev, priv_ep);
  1039. } else if (!(priv_ep->flags & EP_STALLED) &&
  1040. !(priv_ep->flags & EP_STALL_PENDING)) {
  1041. if (priv_ep->flags & EP_DEFERRED_DRDY) {
  1042. priv_ep->flags &= ~EP_DEFERRED_DRDY;
  1043. cdns3_start_all_request(priv_dev, priv_ep);
  1044. } else {
  1045. cdns3_rearm_transfer(priv_ep,
  1046. priv_ep->wa1_set);
  1047. }
  1048. }
  1049. }
  1050. if ((ep_sts_reg & EP_STS_IOC) || (ep_sts_reg & EP_STS_ISP)) {
  1051. if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
  1052. if (ep_sts_reg & EP_STS_ISP)
  1053. priv_ep->flags |= EP_QUIRK_END_TRANSFER;
  1054. else
  1055. priv_ep->flags &= ~EP_QUIRK_END_TRANSFER;
  1056. }
  1057. cdns3_transfer_completed(priv_dev, priv_ep);
  1058. }
  1059. /*
  1060. * WA2: this condition should only be meet when
  1061. * priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET or
  1062. * priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN.
  1063. * In other cases this interrupt will be disabled/
  1064. */
  1065. if (ep_sts_reg & EP_STS_DESCMIS && priv_dev->dev_ver < DEV_VER_V2 &&
  1066. !(priv_ep->flags & EP_STALLED))
  1067. cdns3_wa2_descmissing_packet(priv_ep);
  1068. return 0;
  1069. }
  1070. static void cdns3_disconnect_gadget(struct cdns3_device *priv_dev)
  1071. {
  1072. if (priv_dev->gadget_driver && priv_dev->gadget_driver->disconnect) {
  1073. spin_unlock(&priv_dev->lock);
  1074. priv_dev->gadget_driver->disconnect(&priv_dev->gadget);
  1075. spin_lock(&priv_dev->lock);
  1076. }
  1077. }
  1078. /**
  1079. * cdns3_check_usb_interrupt_proceed - Processes interrupt related to device
  1080. * @priv_dev: extended gadget object
  1081. * @usb_ists: bitmap representation of device's reported interrupts
  1082. * (usb_ists register value)
  1083. */
  1084. static void cdns3_check_usb_interrupt_proceed(struct cdns3_device *priv_dev,
  1085. u32 usb_ists)
  1086. {
  1087. int speed = 0;
  1088. trace_cdns3_usb_irq(priv_dev, usb_ists);
  1089. if (usb_ists & USB_ISTS_L1ENTI) {
  1090. /*
  1091. * WORKAROUND: CDNS3 controller has issue with hardware resuming
  1092. * from L1. To fix it, if any DMA transfer is pending driver
  1093. * must starts driving resume signal immediately.
  1094. */
  1095. if (readl(&priv_dev->regs->drbl))
  1096. __cdns3_gadget_wakeup(priv_dev);
  1097. }
  1098. /* Connection detected */
  1099. if (usb_ists & (USB_ISTS_CON2I | USB_ISTS_CONI)) {
  1100. speed = cdns3_get_speed(priv_dev);
  1101. priv_dev->gadget.speed = speed;
  1102. usb_gadget_set_state(&priv_dev->gadget, USB_STATE_POWERED);
  1103. cdns3_ep0_config(priv_dev);
  1104. }
  1105. /* Disconnection detected */
  1106. if (usb_ists & (USB_ISTS_DIS2I | USB_ISTS_DISI)) {
  1107. cdns3_disconnect_gadget(priv_dev);
  1108. priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
  1109. usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
  1110. cdns3_hw_reset_eps_config(priv_dev);
  1111. }
  1112. if (usb_ists & (USB_ISTS_L2ENTI | USB_ISTS_U3ENTI)) {
  1113. if (priv_dev->gadget_driver &&
  1114. priv_dev->gadget_driver->suspend) {
  1115. spin_unlock(&priv_dev->lock);
  1116. priv_dev->gadget_driver->suspend(&priv_dev->gadget);
  1117. spin_lock(&priv_dev->lock);
  1118. }
  1119. }
  1120. if (usb_ists & (USB_ISTS_L2EXTI | USB_ISTS_U3EXTI)) {
  1121. if (priv_dev->gadget_driver &&
  1122. priv_dev->gadget_driver->resume) {
  1123. spin_unlock(&priv_dev->lock);
  1124. priv_dev->gadget_driver->resume(&priv_dev->gadget);
  1125. spin_lock(&priv_dev->lock);
  1126. }
  1127. }
  1128. /* reset*/
  1129. if (usb_ists & (USB_ISTS_UWRESI | USB_ISTS_UHRESI | USB_ISTS_U2RESI)) {
  1130. if (priv_dev->gadget_driver) {
  1131. spin_unlock(&priv_dev->lock);
  1132. usb_gadget_udc_reset(&priv_dev->gadget,
  1133. priv_dev->gadget_driver);
  1134. spin_lock(&priv_dev->lock);
  1135. /*read again to check the actual speed*/
  1136. speed = cdns3_get_speed(priv_dev);
  1137. priv_dev->gadget.speed = speed;
  1138. cdns3_hw_reset_eps_config(priv_dev);
  1139. cdns3_ep0_config(priv_dev);
  1140. }
  1141. }
  1142. }
  1143. /**
  1144. * cdns3_device_irq_handler- interrupt handler for device part of controller
  1145. *
  1146. * @irq: irq number for cdns3 core device
  1147. * @data: structure of cdns3
  1148. *
  1149. * Returns IRQ_HANDLED or IRQ_NONE
  1150. */
  1151. static irqreturn_t cdns3_device_irq_handler(int irq, void *data)
  1152. {
  1153. struct cdns3_device *priv_dev;
  1154. struct cdns3 *cdns = data;
  1155. irqreturn_t ret = IRQ_NONE;
  1156. u32 reg;
  1157. priv_dev = cdns->gadget_dev;
  1158. /* check USB device interrupt */
  1159. reg = readl(&priv_dev->regs->usb_ists);
  1160. if (reg) {
  1161. /* After masking interrupts the new interrupts won't be
  1162. * reported in usb_ists/ep_ists. In order to not lose some
  1163. * of them driver disables only detected interrupts.
  1164. * They will be enabled ASAP after clearing source of
  1165. * interrupt. This an unusual behavior only applies to
  1166. * usb_ists register.
  1167. */
  1168. reg = ~reg & readl(&priv_dev->regs->usb_ien);
  1169. /* mask deferred interrupt. */
  1170. writel(reg, &priv_dev->regs->usb_ien);
  1171. ret = IRQ_WAKE_THREAD;
  1172. }
  1173. /* check endpoint interrupt */
  1174. reg = readl(&priv_dev->regs->ep_ists);
  1175. if (reg) {
  1176. writel(0, &priv_dev->regs->ep_ien);
  1177. ret = IRQ_WAKE_THREAD;
  1178. }
  1179. return ret;
  1180. }
  1181. /**
  1182. * cdns3_device_thread_irq_handler- interrupt handler for device part
  1183. * of controller
  1184. *
  1185. * @irq: irq number for cdns3 core device
  1186. * @data: structure of cdns3
  1187. *
  1188. * Returns IRQ_HANDLED or IRQ_NONE
  1189. */
  1190. static irqreturn_t cdns3_device_thread_irq_handler(int irq, void *data)
  1191. {
  1192. struct cdns3_device *priv_dev;
  1193. struct cdns3 *cdns = data;
  1194. irqreturn_t ret = IRQ_NONE;
  1195. unsigned long flags;
  1196. int bit;
  1197. u32 reg;
  1198. priv_dev = cdns->gadget_dev;
  1199. spin_lock_irqsave(&priv_dev->lock, flags);
  1200. reg = readl(&priv_dev->regs->usb_ists);
  1201. if (reg) {
  1202. writel(reg, &priv_dev->regs->usb_ists);
  1203. writel(USB_IEN_INIT, &priv_dev->regs->usb_ien);
  1204. cdns3_check_usb_interrupt_proceed(priv_dev, reg);
  1205. ret = IRQ_HANDLED;
  1206. }
  1207. reg = readl(&priv_dev->regs->ep_ists);
  1208. /* handle default endpoint OUT */
  1209. if (reg & EP_ISTS_EP_OUT0) {
  1210. cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_OUT);
  1211. ret = IRQ_HANDLED;
  1212. }
  1213. /* handle default endpoint IN */
  1214. if (reg & EP_ISTS_EP_IN0) {
  1215. cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_IN);
  1216. ret = IRQ_HANDLED;
  1217. }
  1218. /* check if interrupt from non default endpoint, if no exit */
  1219. reg &= ~(EP_ISTS_EP_OUT0 | EP_ISTS_EP_IN0);
  1220. if (!reg)
  1221. goto irqend;
  1222. for_each_set_bit(bit, (unsigned long *)&reg,
  1223. sizeof(u32) * BITS_PER_BYTE) {
  1224. cdns3_check_ep_interrupt_proceed(priv_dev->eps[bit]);
  1225. ret = IRQ_HANDLED;
  1226. }
  1227. irqend:
  1228. writel(~0, &priv_dev->regs->ep_ien);
  1229. spin_unlock_irqrestore(&priv_dev->lock, flags);
  1230. return ret;
  1231. }
  1232. /**
  1233. * cdns3_ep_onchip_buffer_reserve - Try to reserve onchip buf for EP
  1234. *
  1235. * The real reservation will occur during write to EP_CFG register,
  1236. * this function is used to check if the 'size' reservation is allowed.
  1237. *
  1238. * @priv_dev: extended gadget object
  1239. * @size: the size (KB) for EP would like to allocate
  1240. * @is_in: endpoint direction
  1241. *
  1242. * Return 0 if the required size can met or negative value on failure
  1243. */
  1244. static int cdns3_ep_onchip_buffer_reserve(struct cdns3_device *priv_dev,
  1245. int size, int is_in)
  1246. {
  1247. int remained;
  1248. /* 2KB are reserved for EP0*/
  1249. remained = priv_dev->onchip_buffers - priv_dev->onchip_used_size - 2;
  1250. if (is_in) {
  1251. if (remained < size)
  1252. return -EPERM;
  1253. priv_dev->onchip_used_size += size;
  1254. } else {
  1255. int required;
  1256. /**
  1257. * ALL OUT EPs are shared the same chunk onchip memory, so
  1258. * driver checks if it already has assigned enough buffers
  1259. */
  1260. if (priv_dev->out_mem_is_allocated >= size)
  1261. return 0;
  1262. required = size - priv_dev->out_mem_is_allocated;
  1263. if (required > remained)
  1264. return -EPERM;
  1265. priv_dev->out_mem_is_allocated += required;
  1266. priv_dev->onchip_used_size += required;
  1267. }
  1268. return 0;
  1269. }
  1270. void cdns3_configure_dmult(struct cdns3_device *priv_dev,
  1271. struct cdns3_endpoint *priv_ep)
  1272. {
  1273. struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
  1274. /* For dev_ver > DEV_VER_V2 DMULT is configured per endpoint */
  1275. if (priv_dev->dev_ver <= DEV_VER_V2)
  1276. writel(USB_CONF_DMULT, &regs->usb_conf);
  1277. if (priv_dev->dev_ver == DEV_VER_V2)
  1278. writel(USB_CONF2_EN_TDL_TRB, &regs->usb_conf2);
  1279. if (priv_dev->dev_ver >= DEV_VER_V3 && priv_ep) {
  1280. u32 mask;
  1281. if (priv_ep->dir)
  1282. mask = BIT(priv_ep->num + 16);
  1283. else
  1284. mask = BIT(priv_ep->num);
  1285. if (priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
  1286. cdns3_set_register_bit(&regs->tdl_from_trb, mask);
  1287. cdns3_set_register_bit(&regs->tdl_beh, mask);
  1288. cdns3_set_register_bit(&regs->tdl_beh2, mask);
  1289. cdns3_set_register_bit(&regs->dma_adv_td, mask);
  1290. }
  1291. if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
  1292. cdns3_set_register_bit(&regs->tdl_from_trb, mask);
  1293. cdns3_set_register_bit(&regs->dtrans, mask);
  1294. }
  1295. }
  1296. /**
  1297. * cdns3_ep_config Configure hardware endpoint
  1298. * @priv_ep: extended endpoint object
  1299. */
  1300. void cdns3_ep_config(struct cdns3_endpoint *priv_ep)
  1301. {
  1302. bool is_iso_ep = (priv_ep->type == USB_ENDPOINT_XFER_ISOC);
  1303. struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
  1304. u32 bEndpointAddress = priv_ep->num | priv_ep->dir;
  1305. u32 max_packet_size = 0;
  1306. u8 maxburst = 0;
  1307. u32 ep_cfg = 0;
  1308. u8 buffering;
  1309. u8 mult = 0;
  1310. int ret;
  1311. buffering = CDNS3_EP_BUF_SIZE - 1;
  1312. cdns3_configure_dmult(priv_dev, priv_ep);
  1313. switch (priv_ep->type) {
  1314. case USB_ENDPOINT_XFER_INT:
  1315. ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_INT);
  1316. if ((priv_dev->dev_ver == DEV_VER_V2 && !priv_ep->dir) ||
  1317. priv_dev->dev_ver > DEV_VER_V2)
  1318. ep_cfg |= EP_CFG_TDL_CHK;
  1319. break;
  1320. case USB_ENDPOINT_XFER_BULK:
  1321. ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_BULK);
  1322. if ((priv_dev->dev_ver == DEV_VER_V2 && !priv_ep->dir) ||
  1323. priv_dev->dev_ver > DEV_VER_V2)
  1324. ep_cfg |= EP_CFG_TDL_CHK;
  1325. break;
  1326. default:
  1327. ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_ISOC);
  1328. mult = CDNS3_EP_ISO_HS_MULT - 1;
  1329. buffering = mult + 1;
  1330. }
  1331. switch (priv_dev->gadget.speed) {
  1332. case USB_SPEED_FULL:
  1333. max_packet_size = is_iso_ep ? 1023 : 64;
  1334. break;
  1335. case USB_SPEED_HIGH:
  1336. max_packet_size = is_iso_ep ? 1024 : 512;
  1337. break;
  1338. case USB_SPEED_SUPER:
  1339. /* It's limitation that driver assumes in driver. */
  1340. mult = 0;
  1341. max_packet_size = 1024;
  1342. if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
  1343. maxburst = CDNS3_EP_ISO_SS_BURST - 1;
  1344. buffering = (mult + 1) *
  1345. (maxburst + 1);
  1346. if (priv_ep->interval > 1)
  1347. buffering++;
  1348. } else {
  1349. maxburst = CDNS3_EP_BUF_SIZE - 1;
  1350. }
  1351. break;
  1352. default:
  1353. /* all other speed are not supported */
  1354. return;
  1355. }
  1356. if (max_packet_size == 1024)
  1357. priv_ep->trb_burst_size = 128;
  1358. else if (max_packet_size >= 512)
  1359. priv_ep->trb_burst_size = 64;
  1360. else
  1361. priv_ep->trb_burst_size = 16;
  1362. ret = cdns3_ep_onchip_buffer_reserve(priv_dev, buffering + 1,
  1363. !!priv_ep->dir);
  1364. if (ret) {
  1365. dev_err(priv_dev->dev, "onchip mem is full, ep is invalid\n");
  1366. return;
  1367. }
  1368. ep_cfg |= EP_CFG_MAXPKTSIZE(max_packet_size) |
  1369. EP_CFG_MULT(mult) |
  1370. EP_CFG_BUFFERING(buffering) |
  1371. EP_CFG_MAXBURST(maxburst);
  1372. cdns3_select_ep(priv_dev, bEndpointAddress);
  1373. writel(ep_cfg, &priv_dev->regs->ep_cfg);
  1374. dev_dbg(priv_dev->dev, "Configure %s: with val %08x\n",
  1375. priv_ep->name, ep_cfg);
  1376. }
  1377. /* Find correct direction for HW endpoint according to description */
  1378. static int cdns3_ep_dir_is_correct(struct usb_endpoint_descriptor *desc,
  1379. struct cdns3_endpoint *priv_ep)
  1380. {
  1381. return (priv_ep->endpoint.caps.dir_in && usb_endpoint_dir_in(desc)) ||
  1382. (priv_ep->endpoint.caps.dir_out && usb_endpoint_dir_out(desc));
  1383. }
  1384. static struct
  1385. cdns3_endpoint *cdns3_find_available_ep(struct cdns3_device *priv_dev,
  1386. struct usb_endpoint_descriptor *desc)
  1387. {
  1388. struct usb_ep *ep;
  1389. struct cdns3_endpoint *priv_ep;
  1390. list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
  1391. unsigned long num;
  1392. /* ep name pattern likes epXin or epXout */
  1393. char c[2] = {ep->name[2], '\0'};
  1394. num = dectoul(c, NULL);
  1395. priv_ep = ep_to_cdns3_ep(ep);
  1396. if (cdns3_ep_dir_is_correct(desc, priv_ep)) {
  1397. if (!(priv_ep->flags & EP_CLAIMED)) {
  1398. priv_ep->num = num;
  1399. return priv_ep;
  1400. }
  1401. }
  1402. }
  1403. return ERR_PTR(-ENOENT);
  1404. }
  1405. /*
  1406. * Cadence IP has one limitation that all endpoints must be configured
  1407. * (Type & MaxPacketSize) before setting configuration through hardware
  1408. * register, it means we can't change endpoints configuration after
  1409. * set_configuration.
  1410. *
  1411. * This function set EP_CLAIMED flag which is added when the gadget driver
  1412. * uses usb_ep_autoconfig to configure specific endpoint;
  1413. * When the udc driver receives set_configurion request,
  1414. * it goes through all claimed endpoints, and configure all endpoints
  1415. * accordingly.
  1416. *
  1417. * At usb_ep_ops.enable/disable, we only enable and disable endpoint through
  1418. * ep_cfg register which can be changed after set_configuration, and do
  1419. * some software operation accordingly.
  1420. */
  1421. static struct
  1422. usb_ep *cdns3_gadget_match_ep(struct usb_gadget *gadget,
  1423. struct usb_endpoint_descriptor *desc,
  1424. struct usb_ss_ep_comp_descriptor *comp_desc)
  1425. {
  1426. struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
  1427. struct cdns3_endpoint *priv_ep;
  1428. unsigned long flags;
  1429. priv_ep = cdns3_find_available_ep(priv_dev, desc);
  1430. if (IS_ERR(priv_ep)) {
  1431. dev_err(priv_dev->dev, "no available ep\n");
  1432. return NULL;
  1433. }
  1434. dev_dbg(priv_dev->dev, "match endpoint: %s\n", priv_ep->name);
  1435. spin_lock_irqsave(&priv_dev->lock, flags);
  1436. priv_ep->endpoint.desc = desc;
  1437. priv_ep->dir = usb_endpoint_dir_in(desc) ? USB_DIR_IN : USB_DIR_OUT;
  1438. priv_ep->type = usb_endpoint_type(desc);
  1439. priv_ep->flags |= EP_CLAIMED;
  1440. priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
  1441. spin_unlock_irqrestore(&priv_dev->lock, flags);
  1442. return &priv_ep->endpoint;
  1443. }
  1444. /**
  1445. * cdns3_gadget_ep_alloc_request Allocates request
  1446. * @ep: endpoint object associated with request
  1447. * @gfp_flags: gfp flags
  1448. *
  1449. * Returns allocated request address, NULL on allocation error
  1450. */
  1451. struct usb_request *cdns3_gadget_ep_alloc_request(struct usb_ep *ep,
  1452. gfp_t gfp_flags)
  1453. {
  1454. struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
  1455. struct cdns3_request *priv_req;
  1456. priv_req = kzalloc(sizeof(*priv_req), gfp_flags);
  1457. if (!priv_req)
  1458. return NULL;
  1459. priv_req->priv_ep = priv_ep;
  1460. trace_cdns3_alloc_request(priv_req);
  1461. return &priv_req->request;
  1462. }
  1463. /**
  1464. * cdns3_gadget_ep_free_request Free memory occupied by request
  1465. * @ep: endpoint object associated with request
  1466. * @request: request to free memory
  1467. */
  1468. void cdns3_gadget_ep_free_request(struct usb_ep *ep,
  1469. struct usb_request *request)
  1470. {
  1471. struct cdns3_request *priv_req = to_cdns3_request(request);
  1472. if (priv_req->aligned_buf)
  1473. priv_req->aligned_buf->in_use = 0;
  1474. trace_cdns3_free_request(priv_req);
  1475. kfree(priv_req);
  1476. }
  1477. /**
  1478. * cdns3_gadget_ep_enable Enable endpoint
  1479. * @ep: endpoint object
  1480. * @desc: endpoint descriptor
  1481. *
  1482. * Returns 0 on success, error code elsewhere
  1483. */
  1484. static int cdns3_gadget_ep_enable(struct usb_ep *ep,
  1485. const struct usb_endpoint_descriptor *desc)
  1486. {
  1487. struct cdns3_endpoint *priv_ep;
  1488. struct cdns3_device *priv_dev;
  1489. u32 reg = EP_STS_EN_TRBERREN;
  1490. u32 bEndpointAddress;
  1491. unsigned long flags;
  1492. int enable = 1;
  1493. int ret;
  1494. int val;
  1495. priv_ep = ep_to_cdns3_ep(ep);
  1496. priv_dev = priv_ep->cdns3_dev;
  1497. if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
  1498. dev_dbg(priv_dev->dev, "usbss: invalid parameters\n");
  1499. return -EINVAL;
  1500. }
  1501. if (!desc->wMaxPacketSize) {
  1502. dev_err(priv_dev->dev, "usbss: missing wMaxPacketSize\n");
  1503. return -EINVAL;
  1504. }
  1505. if (WARN_ON(priv_ep->flags & EP_ENABLED))
  1506. return 0;
  1507. spin_lock_irqsave(&priv_dev->lock, flags);
  1508. priv_ep->endpoint.desc = desc;
  1509. priv_ep->type = usb_endpoint_type(desc);
  1510. priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
  1511. if (priv_ep->interval > ISO_MAX_INTERVAL &&
  1512. priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
  1513. dev_err(priv_dev->dev, "Driver is limited to %d period\n",
  1514. ISO_MAX_INTERVAL);
  1515. ret = -EINVAL;
  1516. goto exit;
  1517. }
  1518. ret = cdns3_allocate_trb_pool(priv_ep);
  1519. if (ret)
  1520. goto exit;
  1521. bEndpointAddress = priv_ep->num | priv_ep->dir;
  1522. cdns3_select_ep(priv_dev, bEndpointAddress);
  1523. trace_cdns3_gadget_ep_enable(priv_ep);
  1524. writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
  1525. ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
  1526. !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
  1527. 1000);
  1528. if (unlikely(ret)) {
  1529. cdns3_free_trb_pool(priv_ep);
  1530. ret = -EINVAL;
  1531. goto exit;
  1532. }
  1533. /* enable interrupt for selected endpoint */
  1534. cdns3_set_register_bit(&priv_dev->regs->ep_ien,
  1535. BIT(cdns3_ep_addr_to_index(bEndpointAddress)));
  1536. if (priv_dev->dev_ver < DEV_VER_V2)
  1537. cdns3_wa2_enable_detection(priv_dev, priv_ep, reg);
  1538. writel(reg, &priv_dev->regs->ep_sts_en);
  1539. /*
  1540. * For some versions of controller at some point during ISO OUT traffic
  1541. * DMA reads Transfer Ring for the EP which has never got doorbell.
  1542. * This issue was detected only on simulation, but to avoid this issue
  1543. * driver add protection against it. To fix it driver enable ISO OUT
  1544. * endpoint before setting DRBL. This special treatment of ISO OUT
  1545. * endpoints are recommended by controller specification.
  1546. */
  1547. if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
  1548. enable = 0;
  1549. if (enable)
  1550. cdns3_set_register_bit(&priv_dev->regs->ep_cfg, EP_CFG_ENABLE);
  1551. ep->desc = desc;
  1552. priv_ep->flags &= ~(EP_PENDING_REQUEST | EP_STALLED | EP_STALL_PENDING |
  1553. EP_QUIRK_ISO_OUT_EN | EP_QUIRK_EXTRA_BUF_EN);
  1554. priv_ep->flags |= EP_ENABLED | EP_UPDATE_EP_TRBADDR;
  1555. priv_ep->wa1_set = 0;
  1556. priv_ep->enqueue = 0;
  1557. priv_ep->dequeue = 0;
  1558. reg = readl(&priv_dev->regs->ep_sts);
  1559. priv_ep->pcs = !!EP_STS_CCS(reg);
  1560. priv_ep->ccs = !!EP_STS_CCS(reg);
  1561. /* one TRB is reserved for link TRB used in DMULT mode*/
  1562. priv_ep->free_trbs = priv_ep->num_trbs - 1;
  1563. exit:
  1564. spin_unlock_irqrestore(&priv_dev->lock, flags);
  1565. return ret;
  1566. }
  1567. /**
  1568. * cdns3_gadget_ep_disable Disable endpoint
  1569. * @ep: endpoint object
  1570. *
  1571. * Returns 0 on success, error code elsewhere
  1572. */
  1573. static int cdns3_gadget_ep_disable(struct usb_ep *ep)
  1574. {
  1575. struct cdns3_endpoint *priv_ep;
  1576. struct cdns3_request *priv_req;
  1577. struct cdns3_device *priv_dev;
  1578. struct usb_request *request;
  1579. unsigned long flags;
  1580. int ret = 0;
  1581. u32 ep_cfg;
  1582. int val;
  1583. if (!ep) {
  1584. pr_err("usbss: invalid parameters\n");
  1585. return -EINVAL;
  1586. }
  1587. priv_ep = ep_to_cdns3_ep(ep);
  1588. priv_dev = priv_ep->cdns3_dev;
  1589. if (WARN_ON(!(priv_ep->flags & EP_ENABLED)))
  1590. return 0;
  1591. spin_lock_irqsave(&priv_dev->lock, flags);
  1592. trace_cdns3_gadget_ep_disable(priv_ep);
  1593. cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
  1594. ep_cfg = readl(&priv_dev->regs->ep_cfg);
  1595. ep_cfg &= ~EP_CFG_ENABLE;
  1596. writel(ep_cfg, &priv_dev->regs->ep_cfg);
  1597. /**
  1598. * Driver needs some time before resetting endpoint.
  1599. * It need waits for clearing DBUSY bit or for timeout expired.
  1600. * 10us is enough time for controller to stop transfer.
  1601. */
  1602. readl_poll_timeout_atomic(&priv_dev->regs->ep_sts, val,
  1603. !(val & EP_STS_DBUSY), 10);
  1604. writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
  1605. readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
  1606. !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
  1607. 1000);
  1608. if (unlikely(ret))
  1609. dev_err(priv_dev->dev, "Timeout: %s resetting failed.\n",
  1610. priv_ep->name);
  1611. while (!list_empty(&priv_ep->pending_req_list)) {
  1612. request = cdns3_next_request(&priv_ep->pending_req_list);
  1613. cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
  1614. -ESHUTDOWN);
  1615. }
  1616. while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
  1617. priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
  1618. kfree(priv_req->request.buf);
  1619. cdns3_gadget_ep_free_request(&priv_ep->endpoint,
  1620. &priv_req->request);
  1621. list_del_init(&priv_req->list);
  1622. --priv_ep->wa2_counter;
  1623. }
  1624. while (!list_empty(&priv_ep->deferred_req_list)) {
  1625. request = cdns3_next_request(&priv_ep->deferred_req_list);
  1626. cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
  1627. -ESHUTDOWN);
  1628. }
  1629. priv_ep->descmis_req = NULL;
  1630. ep->desc = NULL;
  1631. priv_ep->flags &= ~EP_ENABLED;
  1632. spin_unlock_irqrestore(&priv_dev->lock, flags);
  1633. return ret;
  1634. }
  1635. /**
  1636. * cdns3_gadget_ep_queue Transfer data on endpoint
  1637. * @ep: endpoint object
  1638. * @request: request object
  1639. * @gfp_flags: gfp flags
  1640. *
  1641. * Returns 0 on success, error code elsewhere
  1642. */
  1643. static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
  1644. struct usb_request *request,
  1645. gfp_t gfp_flags)
  1646. {
  1647. struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
  1648. struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
  1649. struct cdns3_request *priv_req;
  1650. int ret = 0;
  1651. request->actual = 0;
  1652. request->status = -EINPROGRESS;
  1653. priv_req = to_cdns3_request(request);
  1654. trace_cdns3_ep_queue(priv_req);
  1655. if (priv_dev->dev_ver < DEV_VER_V2) {
  1656. ret = cdns3_wa2_gadget_ep_queue(priv_dev, priv_ep,
  1657. priv_req);
  1658. if (ret == EINPROGRESS)
  1659. return 0;
  1660. }
  1661. ret = cdns3_prepare_aligned_request_buf(priv_req);
  1662. if (ret < 0)
  1663. return ret;
  1664. ret = usb_gadget_map_request(&priv_dev->gadget, request,
  1665. usb_endpoint_dir_in(ep->desc));
  1666. if (ret)
  1667. return ret;
  1668. list_add_tail(&request->list, &priv_ep->deferred_req_list);
  1669. /*
  1670. * If hardware endpoint configuration has not been set yet then
  1671. * just queue request in deferred list. Transfer will be started in
  1672. * cdns3_set_hw_configuration.
  1673. */
  1674. if (priv_dev->hw_configured_flag && !(priv_ep->flags & EP_STALLED) &&
  1675. !(priv_ep->flags & EP_STALL_PENDING))
  1676. cdns3_start_all_request(priv_dev, priv_ep);
  1677. return 0;
  1678. }
  1679. static int cdns3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
  1680. gfp_t gfp_flags)
  1681. {
  1682. struct usb_request *zlp_request;
  1683. struct cdns3_endpoint *priv_ep;
  1684. struct cdns3_device *priv_dev;
  1685. unsigned long flags;
  1686. int ret;
  1687. if (!request || !ep)
  1688. return -EINVAL;
  1689. priv_ep = ep_to_cdns3_ep(ep);
  1690. priv_dev = priv_ep->cdns3_dev;
  1691. spin_lock_irqsave(&priv_dev->lock, flags);
  1692. ret = __cdns3_gadget_ep_queue(ep, request, gfp_flags);
  1693. if (ret == 0 && request->zero && request->length &&
  1694. (request->length % ep->maxpacket == 0)) {
  1695. struct cdns3_request *priv_req;
  1696. zlp_request = cdns3_gadget_ep_alloc_request(ep, GFP_ATOMIC);
  1697. zlp_request->buf = priv_dev->zlp_buf;
  1698. zlp_request->length = 0;
  1699. priv_req = to_cdns3_request(zlp_request);
  1700. priv_req->flags |= REQUEST_ZLP;
  1701. dev_dbg(priv_dev->dev, "Queuing ZLP for endpoint: %s\n",
  1702. priv_ep->name);
  1703. ret = __cdns3_gadget_ep_queue(ep, zlp_request, gfp_flags);
  1704. }
  1705. spin_unlock_irqrestore(&priv_dev->lock, flags);
  1706. return ret;
  1707. }
  1708. /**
  1709. * cdns3_gadget_ep_dequeue Remove request from transfer queue
  1710. * @ep: endpoint object associated with request
  1711. * @request: request object
  1712. *
  1713. * Returns 0 on success, error code elsewhere
  1714. */
  1715. int cdns3_gadget_ep_dequeue(struct usb_ep *ep,
  1716. struct usb_request *request)
  1717. {
  1718. struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
  1719. struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
  1720. struct usb_request *req, *req_temp;
  1721. struct cdns3_request *priv_req;
  1722. struct cdns3_trb *link_trb;
  1723. unsigned long flags;
  1724. int ret = 0;
  1725. if (!ep || !request || !ep->desc)
  1726. return -EINVAL;
  1727. spin_lock_irqsave(&priv_dev->lock, flags);
  1728. priv_req = to_cdns3_request(request);
  1729. trace_cdns3_ep_dequeue(priv_req);
  1730. cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
  1731. list_for_each_entry_safe(req, req_temp, &priv_ep->pending_req_list,
  1732. list) {
  1733. if (request == req)
  1734. goto found;
  1735. }
  1736. list_for_each_entry_safe(req, req_temp, &priv_ep->deferred_req_list,
  1737. list) {
  1738. if (request == req)
  1739. goto found;
  1740. }
  1741. goto not_found;
  1742. found:
  1743. if (priv_ep->wa1_trb == priv_req->trb)
  1744. cdns3_wa1_restore_cycle_bit(priv_ep);
  1745. link_trb = priv_req->trb;
  1746. cdns3_move_deq_to_next_trb(priv_req);
  1747. cdns3_gadget_giveback(priv_ep, priv_req, -ECONNRESET);
  1748. /* Update ring */
  1749. request = cdns3_next_request(&priv_ep->deferred_req_list);
  1750. if (request) {
  1751. priv_req = to_cdns3_request(request);
  1752. link_trb->buffer = TRB_BUFFER(priv_ep->trb_pool_dma +
  1753. (priv_req->start_trb * TRB_SIZE));
  1754. link_trb->control = (link_trb->control & TRB_CYCLE) |
  1755. TRB_TYPE(TRB_LINK) | TRB_CHAIN | TRB_TOGGLE;
  1756. } else {
  1757. priv_ep->flags |= EP_UPDATE_EP_TRBADDR;
  1758. }
  1759. not_found:
  1760. spin_unlock_irqrestore(&priv_dev->lock, flags);
  1761. return ret;
  1762. }
  1763. /**
  1764. * __cdns3_gadget_ep_set_halt Sets stall on selected endpoint
  1765. * Should be called after acquiring spin_lock and selecting ep
  1766. * @ep: endpoint object to set stall on.
  1767. */
  1768. void __cdns3_gadget_ep_set_halt(struct cdns3_endpoint *priv_ep)
  1769. {
  1770. struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
  1771. trace_cdns3_halt(priv_ep, 1, 0);
  1772. if (!(priv_ep->flags & EP_STALLED)) {
  1773. u32 ep_sts_reg = readl(&priv_dev->regs->ep_sts);
  1774. if (!(ep_sts_reg & EP_STS_DBUSY))
  1775. cdns3_ep_stall_flush(priv_ep);
  1776. else
  1777. priv_ep->flags |= EP_STALL_PENDING;
  1778. }
  1779. }
  1780. /**
  1781. * __cdns3_gadget_ep_clear_halt Clears stall on selected endpoint
  1782. * Should be called after acquiring spin_lock and selecting ep
  1783. * @ep: endpoint object to clear stall on
  1784. */
  1785. int __cdns3_gadget_ep_clear_halt(struct cdns3_endpoint *priv_ep)
  1786. {
  1787. struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
  1788. struct usb_request *request;
  1789. int ret = 0;
  1790. int val;
  1791. trace_cdns3_halt(priv_ep, 0, 0);
  1792. writel(EP_CMD_CSTALL | EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
  1793. /* wait for EPRST cleared */
  1794. readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
  1795. !(val & EP_CMD_EPRST), 100);
  1796. if (ret)
  1797. return -EINVAL;
  1798. priv_ep->flags &= ~(EP_STALLED | EP_STALL_PENDING);
  1799. request = cdns3_next_request(&priv_ep->pending_req_list);
  1800. if (request)
  1801. cdns3_rearm_transfer(priv_ep, 1);
  1802. cdns3_start_all_request(priv_dev, priv_ep);
  1803. return ret;
  1804. }
  1805. /**
  1806. * cdns3_gadget_ep_set_halt Sets/clears stall on selected endpoint
  1807. * @ep: endpoint object to set/clear stall on
  1808. * @value: 1 for set stall, 0 for clear stall
  1809. *
  1810. * Returns 0 on success, error code elsewhere
  1811. */
  1812. int cdns3_gadget_ep_set_halt(struct usb_ep *ep, int value)
  1813. {
  1814. struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
  1815. struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
  1816. unsigned long flags;
  1817. int ret = 0;
  1818. if (!(priv_ep->flags & EP_ENABLED))
  1819. return -EPERM;
  1820. spin_lock_irqsave(&priv_dev->lock, flags);
  1821. cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
  1822. if (!value) {
  1823. priv_ep->flags &= ~EP_WEDGE;
  1824. ret = __cdns3_gadget_ep_clear_halt(priv_ep);
  1825. } else {
  1826. __cdns3_gadget_ep_set_halt(priv_ep);
  1827. }
  1828. spin_unlock_irqrestore(&priv_dev->lock, flags);
  1829. return ret;
  1830. }
  1831. extern const struct usb_ep_ops cdns3_gadget_ep0_ops;
  1832. static const struct usb_ep_ops cdns3_gadget_ep_ops = {
  1833. .enable = cdns3_gadget_ep_enable,
  1834. .disable = cdns3_gadget_ep_disable,
  1835. .alloc_request = cdns3_gadget_ep_alloc_request,
  1836. .free_request = cdns3_gadget_ep_free_request,
  1837. .queue = cdns3_gadget_ep_queue,
  1838. .dequeue = cdns3_gadget_ep_dequeue,
  1839. .set_halt = cdns3_gadget_ep_set_halt,
  1840. .set_wedge = cdns3_gadget_ep_set_wedge,
  1841. };
  1842. /**
  1843. * cdns3_gadget_get_frame Returns number of actual ITP frame
  1844. * @gadget: gadget object
  1845. *
  1846. * Returns number of actual ITP frame
  1847. */
  1848. static int cdns3_gadget_get_frame(struct usb_gadget *gadget)
  1849. {
  1850. struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
  1851. return readl(&priv_dev->regs->usb_itpn);
  1852. }
  1853. int __cdns3_gadget_wakeup(struct cdns3_device *priv_dev)
  1854. {
  1855. enum usb_device_speed speed;
  1856. speed = cdns3_get_speed(priv_dev);
  1857. if (speed >= USB_SPEED_SUPER)
  1858. return 0;
  1859. /* Start driving resume signaling to indicate remote wakeup. */
  1860. writel(USB_CONF_LGO_L0, &priv_dev->regs->usb_conf);
  1861. return 0;
  1862. }
  1863. static int cdns3_gadget_wakeup(struct usb_gadget *gadget)
  1864. {
  1865. struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
  1866. unsigned long flags;
  1867. int ret = 0;
  1868. spin_lock_irqsave(&priv_dev->lock, flags);
  1869. ret = __cdns3_gadget_wakeup(priv_dev);
  1870. spin_unlock_irqrestore(&priv_dev->lock, flags);
  1871. return ret;
  1872. }
  1873. static int cdns3_gadget_set_selfpowered(struct usb_gadget *gadget,
  1874. int is_selfpowered)
  1875. {
  1876. struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
  1877. unsigned long flags;
  1878. spin_lock_irqsave(&priv_dev->lock, flags);
  1879. priv_dev->is_selfpowered = !!is_selfpowered;
  1880. spin_unlock_irqrestore(&priv_dev->lock, flags);
  1881. return 0;
  1882. }
  1883. static int cdns3_gadget_pullup(struct usb_gadget *gadget, int is_on)
  1884. {
  1885. struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
  1886. if (is_on)
  1887. writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf);
  1888. else
  1889. writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
  1890. return 0;
  1891. }
  1892. static void cdns3_gadget_config(struct cdns3_device *priv_dev)
  1893. {
  1894. struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
  1895. u32 reg;
  1896. cdns3_ep0_config(priv_dev);
  1897. /* enable interrupts for endpoint 0 (in and out) */
  1898. writel(EP_IEN_EP_OUT0 | EP_IEN_EP_IN0, &regs->ep_ien);
  1899. /*
  1900. * Driver needs to modify LFPS minimal U1 Exit time for DEV_VER_TI_V1
  1901. * revision of controller.
  1902. */
  1903. if (priv_dev->dev_ver == DEV_VER_TI_V1) {
  1904. reg = readl(&regs->dbg_link1);
  1905. reg &= ~DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_MASK;
  1906. reg |= DBG_LINK1_LFPS_MIN_GEN_U1_EXIT(0x55) |
  1907. DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_SET;
  1908. writel(reg, &regs->dbg_link1);
  1909. }
  1910. /*
  1911. * By default some platforms has set protected access to memory.
  1912. * This cause problem with cache, so driver restore non-secure
  1913. * access to memory.
  1914. */
  1915. reg = readl(&regs->dma_axi_ctrl);
  1916. reg |= DMA_AXI_CTRL_MARPROT(DMA_AXI_CTRL_NON_SECURE) |
  1917. DMA_AXI_CTRL_MAWPROT(DMA_AXI_CTRL_NON_SECURE);
  1918. writel(reg, &regs->dma_axi_ctrl);
  1919. /* enable generic interrupt*/
  1920. writel(USB_IEN_INIT, &regs->usb_ien);
  1921. writel(USB_CONF_CLK2OFFDS | USB_CONF_L1DS, &regs->usb_conf);
  1922. cdns3_configure_dmult(priv_dev, NULL);
  1923. cdns3_gadget_pullup(&priv_dev->gadget, 1);
  1924. }
  1925. /**
  1926. * cdns3_gadget_udc_start Gadget start
  1927. * @gadget: gadget object
  1928. * @driver: driver which operates on this gadget
  1929. *
  1930. * Returns 0 on success, error code elsewhere
  1931. */
  1932. static int cdns3_gadget_udc_start(struct usb_gadget *gadget,
  1933. struct usb_gadget_driver *driver)
  1934. {
  1935. struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
  1936. unsigned long flags;
  1937. spin_lock_irqsave(&priv_dev->lock, flags);
  1938. priv_dev->gadget_driver = driver;
  1939. cdns3_gadget_config(priv_dev);
  1940. spin_unlock_irqrestore(&priv_dev->lock, flags);
  1941. return 0;
  1942. }
  1943. /**
  1944. * cdns3_gadget_udc_stop Stops gadget
  1945. * @gadget: gadget object
  1946. *
  1947. * Returns 0
  1948. */
  1949. static int cdns3_gadget_udc_stop(struct usb_gadget *gadget)
  1950. {
  1951. struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
  1952. struct cdns3_endpoint *priv_ep;
  1953. u32 bEndpointAddress;
  1954. struct usb_ep *ep;
  1955. int ret = 0;
  1956. int val;
  1957. priv_dev->gadget_driver = NULL;
  1958. priv_dev->onchip_used_size = 0;
  1959. priv_dev->out_mem_is_allocated = 0;
  1960. priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
  1961. list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
  1962. priv_ep = ep_to_cdns3_ep(ep);
  1963. bEndpointAddress = priv_ep->num | priv_ep->dir;
  1964. cdns3_select_ep(priv_dev, bEndpointAddress);
  1965. writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
  1966. readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
  1967. !(val & EP_CMD_EPRST), 100);
  1968. }
  1969. /* disable interrupt for device */
  1970. writel(0, &priv_dev->regs->usb_ien);
  1971. writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
  1972. return ret;
  1973. }
  1974. static void cdns3_gadget_udc_set_speed(struct usb_gadget *gadget,
  1975. enum usb_device_speed speed)
  1976. {
  1977. struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
  1978. switch (speed) {
  1979. case USB_SPEED_FULL:
  1980. writel(USB_CONF_SFORCE_FS, &priv_dev->regs->usb_conf);
  1981. writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
  1982. break;
  1983. case USB_SPEED_HIGH:
  1984. writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
  1985. break;
  1986. case USB_SPEED_SUPER:
  1987. break;
  1988. default:
  1989. dev_err(priv_dev->dev, "invalid speed parameter %d\n", speed);
  1990. }
  1991. priv_dev->gadget.speed = speed;
  1992. }
  1993. static const struct usb_gadget_ops cdns3_gadget_ops = {
  1994. .get_frame = cdns3_gadget_get_frame,
  1995. .wakeup = cdns3_gadget_wakeup,
  1996. .set_selfpowered = cdns3_gadget_set_selfpowered,
  1997. .pullup = cdns3_gadget_pullup,
  1998. .udc_start = cdns3_gadget_udc_start,
  1999. .udc_stop = cdns3_gadget_udc_stop,
  2000. .match_ep = cdns3_gadget_match_ep,
  2001. .udc_set_speed = cdns3_gadget_udc_set_speed,
  2002. };
  2003. static void cdns3_free_all_eps(struct cdns3_device *priv_dev)
  2004. {
  2005. int i;
  2006. /* ep0 OUT point to ep0 IN. */
  2007. priv_dev->eps[16] = NULL;
  2008. for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++)
  2009. if (priv_dev->eps[i]) {
  2010. cdns3_free_trb_pool(priv_dev->eps[i]);
  2011. devm_kfree(priv_dev->dev, priv_dev->eps[i]);
  2012. }
  2013. }
  2014. /**
  2015. * cdns3_init_eps Initializes software endpoints of gadget
  2016. * @cdns3: extended gadget object
  2017. *
  2018. * Returns 0 on success, error code elsewhere
  2019. */
  2020. static int cdns3_init_eps(struct cdns3_device *priv_dev)
  2021. {
  2022. u32 ep_enabled_reg, iso_ep_reg;
  2023. struct cdns3_endpoint *priv_ep;
  2024. int ep_dir, ep_number;
  2025. u32 ep_mask;
  2026. int ret = 0;
  2027. int i;
  2028. /* Read it from USB_CAP3 to USB_CAP5 */
  2029. ep_enabled_reg = readl(&priv_dev->regs->usb_cap3);
  2030. iso_ep_reg = readl(&priv_dev->regs->usb_cap4);
  2031. dev_dbg(priv_dev->dev, "Initializing non-zero endpoints\n");
  2032. for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++) {
  2033. ep_dir = i >> 4; /* i div 16 */
  2034. ep_number = i & 0xF; /* i % 16 */
  2035. ep_mask = BIT(i);
  2036. if (!(ep_enabled_reg & ep_mask))
  2037. continue;
  2038. if (ep_dir && !ep_number) {
  2039. priv_dev->eps[i] = priv_dev->eps[0];
  2040. continue;
  2041. }
  2042. priv_ep = devm_kzalloc(priv_dev->dev, sizeof(*priv_ep),
  2043. GFP_KERNEL);
  2044. if (!priv_ep) {
  2045. ret = -ENOMEM;
  2046. goto err;
  2047. }
  2048. /* set parent of endpoint object */
  2049. priv_ep->cdns3_dev = priv_dev;
  2050. priv_dev->eps[i] = priv_ep;
  2051. priv_ep->num = ep_number;
  2052. priv_ep->dir = ep_dir ? USB_DIR_IN : USB_DIR_OUT;
  2053. if (!ep_number) {
  2054. ret = cdns3_init_ep0(priv_dev, priv_ep);
  2055. if (ret) {
  2056. dev_err(priv_dev->dev, "Failed to init ep0\n");
  2057. goto err;
  2058. }
  2059. } else {
  2060. snprintf(priv_ep->name, sizeof(priv_ep->name), "ep%d%s",
  2061. ep_number, !!ep_dir ? "in" : "out");
  2062. priv_ep->endpoint.name = priv_ep->name;
  2063. usb_ep_set_maxpacket_limit(&priv_ep->endpoint,
  2064. CDNS3_EP_MAX_PACKET_LIMIT);
  2065. priv_ep->endpoint.max_streams = CDNS3_EP_MAX_STREAMS;
  2066. priv_ep->endpoint.ops = &cdns3_gadget_ep_ops;
  2067. if (ep_dir)
  2068. priv_ep->endpoint.caps.dir_in = 1;
  2069. else
  2070. priv_ep->endpoint.caps.dir_out = 1;
  2071. if (iso_ep_reg & ep_mask)
  2072. priv_ep->endpoint.caps.type_iso = 1;
  2073. priv_ep->endpoint.caps.type_bulk = 1;
  2074. priv_ep->endpoint.caps.type_int = 1;
  2075. list_add_tail(&priv_ep->endpoint.ep_list,
  2076. &priv_dev->gadget.ep_list);
  2077. }
  2078. priv_ep->flags = 0;
  2079. dev_info(priv_dev->dev, "Initialized %s support: %s %s\n",
  2080. priv_ep->name,
  2081. priv_ep->endpoint.caps.type_bulk ? "BULK, INT" : "",
  2082. priv_ep->endpoint.caps.type_iso ? "ISO" : "");
  2083. INIT_LIST_HEAD(&priv_ep->pending_req_list);
  2084. INIT_LIST_HEAD(&priv_ep->deferred_req_list);
  2085. INIT_LIST_HEAD(&priv_ep->wa2_descmiss_req_list);
  2086. }
  2087. return 0;
  2088. err:
  2089. cdns3_free_all_eps(priv_dev);
  2090. return -ENOMEM;
  2091. }
  2092. void cdns3_gadget_exit(struct cdns3 *cdns)
  2093. {
  2094. struct cdns3_device *priv_dev;
  2095. priv_dev = cdns->gadget_dev;
  2096. usb_del_gadget_udc(&priv_dev->gadget);
  2097. cdns3_free_all_eps(priv_dev);
  2098. while (!list_empty(&priv_dev->aligned_buf_list)) {
  2099. struct cdns3_aligned_buf *buf;
  2100. buf = cdns3_next_align_buf(&priv_dev->aligned_buf_list);
  2101. dma_free_coherent(buf->buf);
  2102. list_del(&buf->list);
  2103. kfree(buf);
  2104. }
  2105. dma_free_coherent(priv_dev->setup_buf);
  2106. kfree(priv_dev->zlp_buf);
  2107. kfree(priv_dev);
  2108. cdns->gadget_dev = NULL;
  2109. cdns3_drd_switch_gadget(cdns, 0);
  2110. }
  2111. static int cdns3_gadget_start(struct cdns3 *cdns)
  2112. {
  2113. struct cdns3_device *priv_dev;
  2114. u32 max_speed;
  2115. int ret;
  2116. priv_dev = kzalloc(sizeof(*priv_dev), GFP_KERNEL);
  2117. if (!priv_dev)
  2118. return -ENOMEM;
  2119. cdns->gadget_dev = priv_dev;
  2120. priv_dev->sysdev = cdns->dev;
  2121. priv_dev->dev = cdns->dev;
  2122. priv_dev->regs = cdns->dev_regs;
  2123. dev_read_u32(priv_dev->dev, "cdns,on-chip-buff-size",
  2124. &priv_dev->onchip_buffers);
  2125. if (priv_dev->onchip_buffers <= 0) {
  2126. u32 reg = readl(&priv_dev->regs->usb_cap2);
  2127. priv_dev->onchip_buffers = USB_CAP2_ACTUAL_MEM_SIZE(reg);
  2128. }
  2129. if (!priv_dev->onchip_buffers)
  2130. priv_dev->onchip_buffers = 256;
  2131. max_speed = usb_get_maximum_speed(dev_ofnode(cdns->dev));
  2132. /* Check the maximum_speed parameter */
  2133. switch (max_speed) {
  2134. case USB_SPEED_FULL:
  2135. /* fall through */
  2136. case USB_SPEED_HIGH:
  2137. /* fall through */
  2138. case USB_SPEED_SUPER:
  2139. break;
  2140. default:
  2141. dev_err(cdns->dev, "invalid maximum_speed parameter %d\n",
  2142. max_speed);
  2143. /* fall through */
  2144. case USB_SPEED_UNKNOWN:
  2145. /* default to superspeed */
  2146. max_speed = USB_SPEED_SUPER;
  2147. break;
  2148. }
  2149. /* fill gadget fields */
  2150. priv_dev->gadget.max_speed = max_speed;
  2151. priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
  2152. priv_dev->gadget.ops = &cdns3_gadget_ops;
  2153. priv_dev->gadget.name = "cdns3-gadget";
  2154. #ifndef __UBOOT__
  2155. priv_dev->gadget.name = "usb-ss-gadget";
  2156. priv_dev->gadget.sg_supported = 1;
  2157. priv_dev->gadget.quirk_avoids_skb_reserve = 1;
  2158. #endif
  2159. spin_lock_init(&priv_dev->lock);
  2160. INIT_WORK(&priv_dev->pending_status_wq,
  2161. cdns3_pending_setup_status_handler);
  2162. /* initialize endpoint container */
  2163. INIT_LIST_HEAD(&priv_dev->gadget.ep_list);
  2164. INIT_LIST_HEAD(&priv_dev->aligned_buf_list);
  2165. ret = cdns3_init_eps(priv_dev);
  2166. if (ret) {
  2167. dev_err(priv_dev->dev, "Failed to create endpoints\n");
  2168. goto err1;
  2169. }
  2170. /* allocate memory for setup packet buffer */
  2171. priv_dev->setup_buf =
  2172. dma_alloc_coherent(8, (unsigned long *)&priv_dev->setup_dma);
  2173. if (!priv_dev->setup_buf) {
  2174. ret = -ENOMEM;
  2175. goto err2;
  2176. }
  2177. priv_dev->dev_ver = readl(&priv_dev->regs->usb_cap6);
  2178. dev_dbg(priv_dev->dev, "Device Controller version: %08x\n",
  2179. readl(&priv_dev->regs->usb_cap6));
  2180. dev_dbg(priv_dev->dev, "USB Capabilities:: %08x\n",
  2181. readl(&priv_dev->regs->usb_cap1));
  2182. dev_dbg(priv_dev->dev, "On-Chip memory cnfiguration: %08x\n",
  2183. readl(&priv_dev->regs->usb_cap2));
  2184. priv_dev->dev_ver = GET_DEV_BASE_VERSION(priv_dev->dev_ver);
  2185. priv_dev->zlp_buf = kzalloc(CDNS3_EP_ZLP_BUF_SIZE, GFP_KERNEL);
  2186. if (!priv_dev->zlp_buf) {
  2187. ret = -ENOMEM;
  2188. goto err3;
  2189. }
  2190. /* add USB gadget device */
  2191. ret = usb_add_gadget_udc((struct device *)priv_dev->dev,
  2192. &priv_dev->gadget);
  2193. if (ret < 0) {
  2194. dev_err(priv_dev->dev,
  2195. "Failed to register USB device controller\n");
  2196. goto err4;
  2197. }
  2198. return 0;
  2199. err4:
  2200. kfree(priv_dev->zlp_buf);
  2201. err3:
  2202. dma_free_coherent(priv_dev->setup_buf);
  2203. err2:
  2204. cdns3_free_all_eps(priv_dev);
  2205. err1:
  2206. cdns->gadget_dev = NULL;
  2207. return ret;
  2208. }
  2209. static int __cdns3_gadget_init(struct cdns3 *cdns)
  2210. {
  2211. int ret = 0;
  2212. cdns3_drd_switch_gadget(cdns, 1);
  2213. ret = cdns3_gadget_start(cdns);
  2214. if (ret)
  2215. return ret;
  2216. return 0;
  2217. }
  2218. static int cdns3_gadget_suspend(struct cdns3 *cdns, bool do_wakeup)
  2219. {
  2220. struct cdns3_device *priv_dev = cdns->gadget_dev;
  2221. cdns3_disconnect_gadget(priv_dev);
  2222. priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
  2223. usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
  2224. cdns3_hw_reset_eps_config(priv_dev);
  2225. /* disable interrupt for device */
  2226. writel(0, &priv_dev->regs->usb_ien);
  2227. cdns3_gadget_pullup(&priv_dev->gadget, 0);
  2228. return 0;
  2229. }
  2230. static int cdns3_gadget_resume(struct cdns3 *cdns, bool hibernated)
  2231. {
  2232. struct cdns3_device *priv_dev = cdns->gadget_dev;
  2233. if (!priv_dev->gadget_driver)
  2234. return 0;
  2235. cdns3_gadget_config(priv_dev);
  2236. return 0;
  2237. }
  2238. /**
  2239. * cdns3_gadget_init - initialize device structure
  2240. *
  2241. * cdns: cdns3 instance
  2242. *
  2243. * This function initializes the gadget.
  2244. */
  2245. int cdns3_gadget_init(struct cdns3 *cdns)
  2246. {
  2247. struct cdns3_role_driver *rdrv;
  2248. rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
  2249. if (!rdrv)
  2250. return -ENOMEM;
  2251. rdrv->start = __cdns3_gadget_init;
  2252. rdrv->stop = cdns3_gadget_exit;
  2253. rdrv->suspend = cdns3_gadget_suspend;
  2254. rdrv->resume = cdns3_gadget_resume;
  2255. rdrv->state = CDNS3_ROLE_STATE_INACTIVE;
  2256. rdrv->name = "gadget";
  2257. cdns->roles[USB_ROLE_DEVICE] = rdrv;
  2258. return 0;
  2259. }
  2260. /**
  2261. * cdns3_gadget_uboot_handle_interrupt - handle cdns3 gadget interrupt
  2262. * @cdns: pointer to struct cdns3
  2263. *
  2264. * Handles ep0 and gadget interrupt
  2265. */
  2266. static void cdns3_gadget_uboot_handle_interrupt(struct cdns3 *cdns)
  2267. {
  2268. int ret = cdns3_device_irq_handler(0, cdns);
  2269. if (ret == IRQ_WAKE_THREAD)
  2270. cdns3_device_thread_irq_handler(0, cdns);
  2271. }
  2272. int dm_usb_gadget_handle_interrupts(struct udevice *dev)
  2273. {
  2274. struct cdns3 *cdns = dev_get_priv(dev);
  2275. cdns3_gadget_uboot_handle_interrupt(cdns);
  2276. return 0;
  2277. }