gadget.c 140 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (c) 2011 Samsung Electronics Co., Ltd.
  4. * http://www.samsung.com
  5. *
  6. * Copyright 2008 Openmoko, Inc.
  7. * Copyright 2008 Simtec Electronics
  8. * Ben Dooks <ben@simtec.co.uk>
  9. * http://armlinux.simtec.co.uk/
  10. *
  11. * S3C USB2.0 High-speed / OtG driver
  12. */
  13. #include <linux/kernel.h>
  14. #include <linux/module.h>
  15. #include <linux/spinlock.h>
  16. #include <linux/interrupt.h>
  17. #include <linux/platform_device.h>
  18. #include <linux/dma-mapping.h>
  19. #include <linux/mutex.h>
  20. #include <linux/seq_file.h>
  21. #include <linux/delay.h>
  22. #include <linux/io.h>
  23. #include <linux/slab.h>
  24. #include <linux/of_platform.h>
  25. #include <linux/usb/ch9.h>
  26. #include <linux/usb/gadget.h>
  27. #include <linux/usb/phy.h>
  28. #include <linux/usb/composite.h>
  29. #include "core.h"
  30. #include "hw.h"
  31. /* conversion functions */
  32. static inline struct dwc2_hsotg_req *our_req(struct usb_request *req)
  33. {
  34. return container_of(req, struct dwc2_hsotg_req, req);
  35. }
  36. static inline struct dwc2_hsotg_ep *our_ep(struct usb_ep *ep)
  37. {
  38. return container_of(ep, struct dwc2_hsotg_ep, ep);
  39. }
  40. static inline struct dwc2_hsotg *to_hsotg(struct usb_gadget *gadget)
  41. {
  42. return container_of(gadget, struct dwc2_hsotg, gadget);
  43. }
  44. static inline void dwc2_set_bit(struct dwc2_hsotg *hsotg, u32 offset, u32 val)
  45. {
  46. dwc2_writel(hsotg, dwc2_readl(hsotg, offset) | val, offset);
  47. }
  48. static inline void dwc2_clear_bit(struct dwc2_hsotg *hsotg, u32 offset, u32 val)
  49. {
  50. dwc2_writel(hsotg, dwc2_readl(hsotg, offset) & ~val, offset);
  51. }
  52. static inline struct dwc2_hsotg_ep *index_to_ep(struct dwc2_hsotg *hsotg,
  53. u32 ep_index, u32 dir_in)
  54. {
  55. if (dir_in)
  56. return hsotg->eps_in[ep_index];
  57. else
  58. return hsotg->eps_out[ep_index];
  59. }
  60. /* forward declaration of functions */
  61. static void dwc2_hsotg_dump(struct dwc2_hsotg *hsotg);
  62. /**
  63. * using_dma - return the DMA status of the driver.
  64. * @hsotg: The driver state.
  65. *
  66. * Return true if we're using DMA.
  67. *
  68. * Currently, we have the DMA support code worked into everywhere
  69. * that needs it, but the AMBA DMA implementation in the hardware can
  70. * only DMA from 32bit aligned addresses. This means that gadgets such
  71. * as the CDC Ethernet cannot work as they often pass packets which are
  72. * not 32bit aligned.
  73. *
  74. * Unfortunately the choice to use DMA or not is global to the controller
  75. * and seems to be only settable when the controller is being put through
  76. * a core reset. This means we either need to fix the gadgets to take
  77. * account of DMA alignment, or add bounce buffers (yuerk).
  78. *
  79. * g_using_dma is set depending on dts flag.
  80. */
  81. static inline bool using_dma(struct dwc2_hsotg *hsotg)
  82. {
  83. return hsotg->params.g_dma;
  84. }
  85. /*
  86. * using_desc_dma - return the descriptor DMA status of the driver.
  87. * @hsotg: The driver state.
  88. *
  89. * Return true if we're using descriptor DMA.
  90. */
  91. static inline bool using_desc_dma(struct dwc2_hsotg *hsotg)
  92. {
  93. return hsotg->params.g_dma_desc;
  94. }
  95. /**
  96. * dwc2_gadget_incr_frame_num - Increments the targeted frame number.
  97. * @hs_ep: The endpoint
  98. *
  99. * This function will also check if the frame number overruns DSTS_SOFFN_LIMIT.
  100. * If an overrun occurs it will wrap the value and set the frame_overrun flag.
  101. */
  102. static inline void dwc2_gadget_incr_frame_num(struct dwc2_hsotg_ep *hs_ep)
  103. {
  104. struct dwc2_hsotg *hsotg = hs_ep->parent;
  105. u16 limit = DSTS_SOFFN_LIMIT;
  106. if (hsotg->gadget.speed != USB_SPEED_HIGH)
  107. limit >>= 3;
  108. hs_ep->target_frame += hs_ep->interval;
  109. if (hs_ep->target_frame > limit) {
  110. hs_ep->frame_overrun = true;
  111. hs_ep->target_frame &= limit;
  112. } else {
  113. hs_ep->frame_overrun = false;
  114. }
  115. }
  116. /**
  117. * dwc2_gadget_dec_frame_num_by_one - Decrements the targeted frame number
  118. * by one.
  119. * @hs_ep: The endpoint.
  120. *
  121. * This function used in service interval based scheduling flow to calculate
  122. * descriptor frame number filed value. For service interval mode frame
  123. * number in descriptor should point to last (u)frame in the interval.
  124. *
  125. */
  126. static inline void dwc2_gadget_dec_frame_num_by_one(struct dwc2_hsotg_ep *hs_ep)
  127. {
  128. struct dwc2_hsotg *hsotg = hs_ep->parent;
  129. u16 limit = DSTS_SOFFN_LIMIT;
  130. if (hsotg->gadget.speed != USB_SPEED_HIGH)
  131. limit >>= 3;
  132. if (hs_ep->target_frame)
  133. hs_ep->target_frame -= 1;
  134. else
  135. hs_ep->target_frame = limit;
  136. }
  137. /**
  138. * dwc2_hsotg_en_gsint - enable one or more of the general interrupt
  139. * @hsotg: The device state
  140. * @ints: A bitmask of the interrupts to enable
  141. */
  142. static void dwc2_hsotg_en_gsint(struct dwc2_hsotg *hsotg, u32 ints)
  143. {
  144. u32 gsintmsk = dwc2_readl(hsotg, GINTMSK);
  145. u32 new_gsintmsk;
  146. new_gsintmsk = gsintmsk | ints;
  147. if (new_gsintmsk != gsintmsk) {
  148. dev_dbg(hsotg->dev, "gsintmsk now 0x%08x\n", new_gsintmsk);
  149. dwc2_writel(hsotg, new_gsintmsk, GINTMSK);
  150. }
  151. }
  152. /**
  153. * dwc2_hsotg_disable_gsint - disable one or more of the general interrupt
  154. * @hsotg: The device state
  155. * @ints: A bitmask of the interrupts to enable
  156. */
  157. static void dwc2_hsotg_disable_gsint(struct dwc2_hsotg *hsotg, u32 ints)
  158. {
  159. u32 gsintmsk = dwc2_readl(hsotg, GINTMSK);
  160. u32 new_gsintmsk;
  161. new_gsintmsk = gsintmsk & ~ints;
  162. if (new_gsintmsk != gsintmsk)
  163. dwc2_writel(hsotg, new_gsintmsk, GINTMSK);
  164. }
  165. /**
  166. * dwc2_hsotg_ctrl_epint - enable/disable an endpoint irq
  167. * @hsotg: The device state
  168. * @ep: The endpoint index
  169. * @dir_in: True if direction is in.
  170. * @en: The enable value, true to enable
  171. *
  172. * Set or clear the mask for an individual endpoint's interrupt
  173. * request.
  174. */
  175. static void dwc2_hsotg_ctrl_epint(struct dwc2_hsotg *hsotg,
  176. unsigned int ep, unsigned int dir_in,
  177. unsigned int en)
  178. {
  179. unsigned long flags;
  180. u32 bit = 1 << ep;
  181. u32 daint;
  182. if (!dir_in)
  183. bit <<= 16;
  184. local_irq_save(flags);
  185. daint = dwc2_readl(hsotg, DAINTMSK);
  186. if (en)
  187. daint |= bit;
  188. else
  189. daint &= ~bit;
  190. dwc2_writel(hsotg, daint, DAINTMSK);
  191. local_irq_restore(flags);
  192. }
  193. /**
  194. * dwc2_hsotg_tx_fifo_count - return count of TX FIFOs in device mode
  195. *
  196. * @hsotg: Programming view of the DWC_otg controller
  197. */
  198. int dwc2_hsotg_tx_fifo_count(struct dwc2_hsotg *hsotg)
  199. {
  200. if (hsotg->hw_params.en_multiple_tx_fifo)
  201. /* In dedicated FIFO mode we need count of IN EPs */
  202. return hsotg->hw_params.num_dev_in_eps;
  203. else
  204. /* In shared FIFO mode we need count of Periodic IN EPs */
  205. return hsotg->hw_params.num_dev_perio_in_ep;
  206. }
  207. /**
  208. * dwc2_hsotg_tx_fifo_total_depth - return total FIFO depth available for
  209. * device mode TX FIFOs
  210. *
  211. * @hsotg: Programming view of the DWC_otg controller
  212. */
  213. int dwc2_hsotg_tx_fifo_total_depth(struct dwc2_hsotg *hsotg)
  214. {
  215. int addr;
  216. int tx_addr_max;
  217. u32 np_tx_fifo_size;
  218. np_tx_fifo_size = min_t(u32, hsotg->hw_params.dev_nperio_tx_fifo_size,
  219. hsotg->params.g_np_tx_fifo_size);
  220. /* Get Endpoint Info Control block size in DWORDs. */
  221. tx_addr_max = hsotg->hw_params.total_fifo_size;
  222. addr = hsotg->params.g_rx_fifo_size + np_tx_fifo_size;
  223. if (tx_addr_max <= addr)
  224. return 0;
  225. return tx_addr_max - addr;
  226. }
  227. /**
  228. * dwc2_gadget_wkup_alert_handler - Handler for WKUP_ALERT interrupt
  229. *
  230. * @hsotg: Programming view of the DWC_otg controller
  231. *
  232. */
  233. static void dwc2_gadget_wkup_alert_handler(struct dwc2_hsotg *hsotg)
  234. {
  235. u32 gintsts2;
  236. u32 gintmsk2;
  237. gintsts2 = dwc2_readl(hsotg, GINTSTS2);
  238. gintmsk2 = dwc2_readl(hsotg, GINTMSK2);
  239. gintsts2 &= gintmsk2;
  240. if (gintsts2 & GINTSTS2_WKUP_ALERT_INT) {
  241. dev_dbg(hsotg->dev, "%s: Wkup_Alert_Int\n", __func__);
  242. dwc2_set_bit(hsotg, GINTSTS2, GINTSTS2_WKUP_ALERT_INT);
  243. dwc2_set_bit(hsotg, DCTL, DCTL_RMTWKUPSIG);
  244. }
  245. }
  246. /**
  247. * dwc2_hsotg_tx_fifo_average_depth - returns average depth of device mode
  248. * TX FIFOs
  249. *
  250. * @hsotg: Programming view of the DWC_otg controller
  251. */
  252. int dwc2_hsotg_tx_fifo_average_depth(struct dwc2_hsotg *hsotg)
  253. {
  254. int tx_fifo_count;
  255. int tx_fifo_depth;
  256. tx_fifo_depth = dwc2_hsotg_tx_fifo_total_depth(hsotg);
  257. tx_fifo_count = dwc2_hsotg_tx_fifo_count(hsotg);
  258. if (!tx_fifo_count)
  259. return tx_fifo_depth;
  260. else
  261. return tx_fifo_depth / tx_fifo_count;
  262. }
  263. /**
  264. * dwc2_hsotg_init_fifo - initialise non-periodic FIFOs
  265. * @hsotg: The device instance.
  266. */
  267. static void dwc2_hsotg_init_fifo(struct dwc2_hsotg *hsotg)
  268. {
  269. unsigned int ep;
  270. unsigned int addr;
  271. int timeout;
  272. u32 val;
  273. u32 *txfsz = hsotg->params.g_tx_fifo_size;
  274. /* Reset fifo map if not correctly cleared during previous session */
  275. WARN_ON(hsotg->fifo_map);
  276. hsotg->fifo_map = 0;
  277. /* set RX/NPTX FIFO sizes */
  278. dwc2_writel(hsotg, hsotg->params.g_rx_fifo_size, GRXFSIZ);
  279. dwc2_writel(hsotg, (hsotg->params.g_rx_fifo_size <<
  280. FIFOSIZE_STARTADDR_SHIFT) |
  281. (hsotg->params.g_np_tx_fifo_size << FIFOSIZE_DEPTH_SHIFT),
  282. GNPTXFSIZ);
  283. /*
  284. * arange all the rest of the TX FIFOs, as some versions of this
  285. * block have overlapping default addresses. This also ensures
  286. * that if the settings have been changed, then they are set to
  287. * known values.
  288. */
  289. /* start at the end of the GNPTXFSIZ, rounded up */
  290. addr = hsotg->params.g_rx_fifo_size + hsotg->params.g_np_tx_fifo_size;
  291. /*
  292. * Configure fifos sizes from provided configuration and assign
  293. * them to endpoints dynamically according to maxpacket size value of
  294. * given endpoint.
  295. */
  296. for (ep = 1; ep < MAX_EPS_CHANNELS; ep++) {
  297. if (!txfsz[ep])
  298. continue;
  299. val = addr;
  300. val |= txfsz[ep] << FIFOSIZE_DEPTH_SHIFT;
  301. WARN_ONCE(addr + txfsz[ep] > hsotg->fifo_mem,
  302. "insufficient fifo memory");
  303. addr += txfsz[ep];
  304. dwc2_writel(hsotg, val, DPTXFSIZN(ep));
  305. val = dwc2_readl(hsotg, DPTXFSIZN(ep));
  306. }
  307. dwc2_writel(hsotg, hsotg->hw_params.total_fifo_size |
  308. addr << GDFIFOCFG_EPINFOBASE_SHIFT,
  309. GDFIFOCFG);
  310. /*
  311. * according to p428 of the design guide, we need to ensure that
  312. * all fifos are flushed before continuing
  313. */
  314. dwc2_writel(hsotg, GRSTCTL_TXFNUM(0x10) | GRSTCTL_TXFFLSH |
  315. GRSTCTL_RXFFLSH, GRSTCTL);
  316. /* wait until the fifos are both flushed */
  317. timeout = 100;
  318. while (1) {
  319. val = dwc2_readl(hsotg, GRSTCTL);
  320. if ((val & (GRSTCTL_TXFFLSH | GRSTCTL_RXFFLSH)) == 0)
  321. break;
  322. if (--timeout == 0) {
  323. dev_err(hsotg->dev,
  324. "%s: timeout flushing fifos (GRSTCTL=%08x)\n",
  325. __func__, val);
  326. break;
  327. }
  328. udelay(1);
  329. }
  330. dev_dbg(hsotg->dev, "FIFOs reset, timeout at %d\n", timeout);
  331. }
  332. /**
  333. * dwc2_hsotg_ep_alloc_request - allocate USB rerequest structure
  334. * @ep: USB endpoint to allocate request for.
  335. * @flags: Allocation flags
  336. *
  337. * Allocate a new USB request structure appropriate for the specified endpoint
  338. */
  339. static struct usb_request *dwc2_hsotg_ep_alloc_request(struct usb_ep *ep,
  340. gfp_t flags)
  341. {
  342. struct dwc2_hsotg_req *req;
  343. req = kzalloc(sizeof(*req), flags);
  344. if (!req)
  345. return NULL;
  346. INIT_LIST_HEAD(&req->queue);
  347. return &req->req;
  348. }
  349. /**
  350. * is_ep_periodic - return true if the endpoint is in periodic mode.
  351. * @hs_ep: The endpoint to query.
  352. *
  353. * Returns true if the endpoint is in periodic mode, meaning it is being
  354. * used for an Interrupt or ISO transfer.
  355. */
  356. static inline int is_ep_periodic(struct dwc2_hsotg_ep *hs_ep)
  357. {
  358. return hs_ep->periodic;
  359. }
  360. /**
  361. * dwc2_hsotg_unmap_dma - unmap the DMA memory being used for the request
  362. * @hsotg: The device state.
  363. * @hs_ep: The endpoint for the request
  364. * @hs_req: The request being processed.
  365. *
  366. * This is the reverse of dwc2_hsotg_map_dma(), called for the completion
  367. * of a request to ensure the buffer is ready for access by the caller.
  368. */
  369. static void dwc2_hsotg_unmap_dma(struct dwc2_hsotg *hsotg,
  370. struct dwc2_hsotg_ep *hs_ep,
  371. struct dwc2_hsotg_req *hs_req)
  372. {
  373. struct usb_request *req = &hs_req->req;
  374. usb_gadget_unmap_request(&hsotg->gadget, req, hs_ep->map_dir);
  375. }
  376. /*
  377. * dwc2_gadget_alloc_ctrl_desc_chains - allocate DMA descriptor chains
  378. * for Control endpoint
  379. * @hsotg: The device state.
  380. *
  381. * This function will allocate 4 descriptor chains for EP 0: 2 for
  382. * Setup stage, per one for IN and OUT data/status transactions.
  383. */
  384. static int dwc2_gadget_alloc_ctrl_desc_chains(struct dwc2_hsotg *hsotg)
  385. {
  386. hsotg->setup_desc[0] =
  387. dmam_alloc_coherent(hsotg->dev,
  388. sizeof(struct dwc2_dma_desc),
  389. &hsotg->setup_desc_dma[0],
  390. GFP_KERNEL);
  391. if (!hsotg->setup_desc[0])
  392. goto fail;
  393. hsotg->setup_desc[1] =
  394. dmam_alloc_coherent(hsotg->dev,
  395. sizeof(struct dwc2_dma_desc),
  396. &hsotg->setup_desc_dma[1],
  397. GFP_KERNEL);
  398. if (!hsotg->setup_desc[1])
  399. goto fail;
  400. hsotg->ctrl_in_desc =
  401. dmam_alloc_coherent(hsotg->dev,
  402. sizeof(struct dwc2_dma_desc),
  403. &hsotg->ctrl_in_desc_dma,
  404. GFP_KERNEL);
  405. if (!hsotg->ctrl_in_desc)
  406. goto fail;
  407. hsotg->ctrl_out_desc =
  408. dmam_alloc_coherent(hsotg->dev,
  409. sizeof(struct dwc2_dma_desc),
  410. &hsotg->ctrl_out_desc_dma,
  411. GFP_KERNEL);
  412. if (!hsotg->ctrl_out_desc)
  413. goto fail;
  414. return 0;
  415. fail:
  416. return -ENOMEM;
  417. }
  418. /**
  419. * dwc2_hsotg_write_fifo - write packet Data to the TxFIFO
  420. * @hsotg: The controller state.
  421. * @hs_ep: The endpoint we're going to write for.
  422. * @hs_req: The request to write data for.
  423. *
  424. * This is called when the TxFIFO has some space in it to hold a new
  425. * transmission and we have something to give it. The actual setup of
  426. * the data size is done elsewhere, so all we have to do is to actually
  427. * write the data.
  428. *
  429. * The return value is zero if there is more space (or nothing was done)
  430. * otherwise -ENOSPC is returned if the FIFO space was used up.
  431. *
  432. * This routine is only needed for PIO
  433. */
  434. static int dwc2_hsotg_write_fifo(struct dwc2_hsotg *hsotg,
  435. struct dwc2_hsotg_ep *hs_ep,
  436. struct dwc2_hsotg_req *hs_req)
  437. {
  438. bool periodic = is_ep_periodic(hs_ep);
  439. u32 gnptxsts = dwc2_readl(hsotg, GNPTXSTS);
  440. int buf_pos = hs_req->req.actual;
  441. int to_write = hs_ep->size_loaded;
  442. void *data;
  443. int can_write;
  444. int pkt_round;
  445. int max_transfer;
  446. to_write -= (buf_pos - hs_ep->last_load);
  447. /* if there's nothing to write, get out early */
  448. if (to_write == 0)
  449. return 0;
  450. if (periodic && !hsotg->dedicated_fifos) {
  451. u32 epsize = dwc2_readl(hsotg, DIEPTSIZ(hs_ep->index));
  452. int size_left;
  453. int size_done;
  454. /*
  455. * work out how much data was loaded so we can calculate
  456. * how much data is left in the fifo.
  457. */
  458. size_left = DXEPTSIZ_XFERSIZE_GET(epsize);
  459. /*
  460. * if shared fifo, we cannot write anything until the
  461. * previous data has been completely sent.
  462. */
  463. if (hs_ep->fifo_load != 0) {
  464. dwc2_hsotg_en_gsint(hsotg, GINTSTS_PTXFEMP);
  465. return -ENOSPC;
  466. }
  467. dev_dbg(hsotg->dev, "%s: left=%d, load=%d, fifo=%d, size %d\n",
  468. __func__, size_left,
  469. hs_ep->size_loaded, hs_ep->fifo_load, hs_ep->fifo_size);
  470. /* how much of the data has moved */
  471. size_done = hs_ep->size_loaded - size_left;
  472. /* how much data is left in the fifo */
  473. can_write = hs_ep->fifo_load - size_done;
  474. dev_dbg(hsotg->dev, "%s: => can_write1=%d\n",
  475. __func__, can_write);
  476. can_write = hs_ep->fifo_size - can_write;
  477. dev_dbg(hsotg->dev, "%s: => can_write2=%d\n",
  478. __func__, can_write);
  479. if (can_write <= 0) {
  480. dwc2_hsotg_en_gsint(hsotg, GINTSTS_PTXFEMP);
  481. return -ENOSPC;
  482. }
  483. } else if (hsotg->dedicated_fifos && hs_ep->index != 0) {
  484. can_write = dwc2_readl(hsotg,
  485. DTXFSTS(hs_ep->fifo_index));
  486. can_write &= 0xffff;
  487. can_write *= 4;
  488. } else {
  489. if (GNPTXSTS_NP_TXQ_SPC_AVAIL_GET(gnptxsts) == 0) {
  490. dev_dbg(hsotg->dev,
  491. "%s: no queue slots available (0x%08x)\n",
  492. __func__, gnptxsts);
  493. dwc2_hsotg_en_gsint(hsotg, GINTSTS_NPTXFEMP);
  494. return -ENOSPC;
  495. }
  496. can_write = GNPTXSTS_NP_TXF_SPC_AVAIL_GET(gnptxsts);
  497. can_write *= 4; /* fifo size is in 32bit quantities. */
  498. }
  499. max_transfer = hs_ep->ep.maxpacket * hs_ep->mc;
  500. dev_dbg(hsotg->dev, "%s: GNPTXSTS=%08x, can=%d, to=%d, max_transfer %d\n",
  501. __func__, gnptxsts, can_write, to_write, max_transfer);
  502. /*
  503. * limit to 512 bytes of data, it seems at least on the non-periodic
  504. * FIFO, requests of >512 cause the endpoint to get stuck with a
  505. * fragment of the end of the transfer in it.
  506. */
  507. if (can_write > 512 && !periodic)
  508. can_write = 512;
  509. /*
  510. * limit the write to one max-packet size worth of data, but allow
  511. * the transfer to return that it did not run out of fifo space
  512. * doing it.
  513. */
  514. if (to_write > max_transfer) {
  515. to_write = max_transfer;
  516. /* it's needed only when we do not use dedicated fifos */
  517. if (!hsotg->dedicated_fifos)
  518. dwc2_hsotg_en_gsint(hsotg,
  519. periodic ? GINTSTS_PTXFEMP :
  520. GINTSTS_NPTXFEMP);
  521. }
  522. /* see if we can write data */
  523. if (to_write > can_write) {
  524. to_write = can_write;
  525. pkt_round = to_write % max_transfer;
  526. /*
  527. * Round the write down to an
  528. * exact number of packets.
  529. *
  530. * Note, we do not currently check to see if we can ever
  531. * write a full packet or not to the FIFO.
  532. */
  533. if (pkt_round)
  534. to_write -= pkt_round;
  535. /*
  536. * enable correct FIFO interrupt to alert us when there
  537. * is more room left.
  538. */
  539. /* it's needed only when we do not use dedicated fifos */
  540. if (!hsotg->dedicated_fifos)
  541. dwc2_hsotg_en_gsint(hsotg,
  542. periodic ? GINTSTS_PTXFEMP :
  543. GINTSTS_NPTXFEMP);
  544. }
  545. dev_dbg(hsotg->dev, "write %d/%d, can_write %d, done %d\n",
  546. to_write, hs_req->req.length, can_write, buf_pos);
  547. if (to_write <= 0)
  548. return -ENOSPC;
  549. hs_req->req.actual = buf_pos + to_write;
  550. hs_ep->total_data += to_write;
  551. if (periodic)
  552. hs_ep->fifo_load += to_write;
  553. to_write = DIV_ROUND_UP(to_write, 4);
  554. data = hs_req->req.buf + buf_pos;
  555. dwc2_writel_rep(hsotg, EPFIFO(hs_ep->index), data, to_write);
  556. return (to_write >= can_write) ? -ENOSPC : 0;
  557. }
  558. /**
  559. * get_ep_limit - get the maximum data legnth for this endpoint
  560. * @hs_ep: The endpoint
  561. *
  562. * Return the maximum data that can be queued in one go on a given endpoint
  563. * so that transfers that are too long can be split.
  564. */
  565. static unsigned int get_ep_limit(struct dwc2_hsotg_ep *hs_ep)
  566. {
  567. int index = hs_ep->index;
  568. unsigned int maxsize;
  569. unsigned int maxpkt;
  570. if (index != 0) {
  571. maxsize = DXEPTSIZ_XFERSIZE_LIMIT + 1;
  572. maxpkt = DXEPTSIZ_PKTCNT_LIMIT + 1;
  573. } else {
  574. maxsize = 64 + 64;
  575. if (hs_ep->dir_in)
  576. maxpkt = DIEPTSIZ0_PKTCNT_LIMIT + 1;
  577. else
  578. maxpkt = 2;
  579. }
  580. /* we made the constant loading easier above by using +1 */
  581. maxpkt--;
  582. maxsize--;
  583. /*
  584. * constrain by packet count if maxpkts*pktsize is greater
  585. * than the length register size.
  586. */
  587. if ((maxpkt * hs_ep->ep.maxpacket) < maxsize)
  588. maxsize = maxpkt * hs_ep->ep.maxpacket;
  589. return maxsize;
  590. }
  591. /**
  592. * dwc2_hsotg_read_frameno - read current frame number
  593. * @hsotg: The device instance
  594. *
  595. * Return the current frame number
  596. */
  597. static u32 dwc2_hsotg_read_frameno(struct dwc2_hsotg *hsotg)
  598. {
  599. u32 dsts;
  600. dsts = dwc2_readl(hsotg, DSTS);
  601. dsts &= DSTS_SOFFN_MASK;
  602. dsts >>= DSTS_SOFFN_SHIFT;
  603. return dsts;
  604. }
  605. /**
  606. * dwc2_gadget_get_chain_limit - get the maximum data payload value of the
  607. * DMA descriptor chain prepared for specific endpoint
  608. * @hs_ep: The endpoint
  609. *
  610. * Return the maximum data that can be queued in one go on a given endpoint
  611. * depending on its descriptor chain capacity so that transfers that
  612. * are too long can be split.
  613. */
  614. static unsigned int dwc2_gadget_get_chain_limit(struct dwc2_hsotg_ep *hs_ep)
  615. {
  616. const struct usb_endpoint_descriptor *ep_desc = hs_ep->ep.desc;
  617. int is_isoc = hs_ep->isochronous;
  618. unsigned int maxsize;
  619. u32 mps = hs_ep->ep.maxpacket;
  620. int dir_in = hs_ep->dir_in;
  621. if (is_isoc)
  622. maxsize = (hs_ep->dir_in ? DEV_DMA_ISOC_TX_NBYTES_LIMIT :
  623. DEV_DMA_ISOC_RX_NBYTES_LIMIT) *
  624. MAX_DMA_DESC_NUM_HS_ISOC;
  625. else
  626. maxsize = DEV_DMA_NBYTES_LIMIT * MAX_DMA_DESC_NUM_GENERIC;
  627. /* Interrupt OUT EP with mps not multiple of 4 */
  628. if (hs_ep->index)
  629. if (usb_endpoint_xfer_int(ep_desc) && !dir_in && (mps % 4))
  630. maxsize = mps * MAX_DMA_DESC_NUM_GENERIC;
  631. return maxsize;
  632. }
  633. /*
  634. * dwc2_gadget_get_desc_params - get DMA descriptor parameters.
  635. * @hs_ep: The endpoint
  636. * @mask: RX/TX bytes mask to be defined
  637. *
  638. * Returns maximum data payload for one descriptor after analyzing endpoint
  639. * characteristics.
  640. * DMA descriptor transfer bytes limit depends on EP type:
  641. * Control out - MPS,
  642. * Isochronous - descriptor rx/tx bytes bitfield limit,
  643. * Control In/Bulk/Interrupt - multiple of mps. This will allow to not
  644. * have concatenations from various descriptors within one packet.
  645. * Interrupt OUT - if mps not multiple of 4 then a single packet corresponds
  646. * to a single descriptor.
  647. *
  648. * Selects corresponding mask for RX/TX bytes as well.
  649. */
  650. static u32 dwc2_gadget_get_desc_params(struct dwc2_hsotg_ep *hs_ep, u32 *mask)
  651. {
  652. const struct usb_endpoint_descriptor *ep_desc = hs_ep->ep.desc;
  653. u32 mps = hs_ep->ep.maxpacket;
  654. int dir_in = hs_ep->dir_in;
  655. u32 desc_size = 0;
  656. if (!hs_ep->index && !dir_in) {
  657. desc_size = mps;
  658. *mask = DEV_DMA_NBYTES_MASK;
  659. } else if (hs_ep->isochronous) {
  660. if (dir_in) {
  661. desc_size = DEV_DMA_ISOC_TX_NBYTES_LIMIT;
  662. *mask = DEV_DMA_ISOC_TX_NBYTES_MASK;
  663. } else {
  664. desc_size = DEV_DMA_ISOC_RX_NBYTES_LIMIT;
  665. *mask = DEV_DMA_ISOC_RX_NBYTES_MASK;
  666. }
  667. } else {
  668. desc_size = DEV_DMA_NBYTES_LIMIT;
  669. *mask = DEV_DMA_NBYTES_MASK;
  670. /* Round down desc_size to be mps multiple */
  671. desc_size -= desc_size % mps;
  672. }
  673. /* Interrupt OUT EP with mps not multiple of 4 */
  674. if (hs_ep->index)
  675. if (usb_endpoint_xfer_int(ep_desc) && !dir_in && (mps % 4)) {
  676. desc_size = mps;
  677. *mask = DEV_DMA_NBYTES_MASK;
  678. }
  679. return desc_size;
  680. }
  681. static void dwc2_gadget_fill_nonisoc_xfer_ddma_one(struct dwc2_hsotg_ep *hs_ep,
  682. struct dwc2_dma_desc **desc,
  683. dma_addr_t dma_buff,
  684. unsigned int len,
  685. bool true_last)
  686. {
  687. int dir_in = hs_ep->dir_in;
  688. u32 mps = hs_ep->ep.maxpacket;
  689. u32 maxsize = 0;
  690. u32 offset = 0;
  691. u32 mask = 0;
  692. int i;
  693. maxsize = dwc2_gadget_get_desc_params(hs_ep, &mask);
  694. hs_ep->desc_count = (len / maxsize) +
  695. ((len % maxsize) ? 1 : 0);
  696. if (len == 0)
  697. hs_ep->desc_count = 1;
  698. for (i = 0; i < hs_ep->desc_count; ++i) {
  699. (*desc)->status = 0;
  700. (*desc)->status |= (DEV_DMA_BUFF_STS_HBUSY
  701. << DEV_DMA_BUFF_STS_SHIFT);
  702. if (len > maxsize) {
  703. if (!hs_ep->index && !dir_in)
  704. (*desc)->status |= (DEV_DMA_L | DEV_DMA_IOC);
  705. (*desc)->status |=
  706. maxsize << DEV_DMA_NBYTES_SHIFT & mask;
  707. (*desc)->buf = dma_buff + offset;
  708. len -= maxsize;
  709. offset += maxsize;
  710. } else {
  711. if (true_last)
  712. (*desc)->status |= (DEV_DMA_L | DEV_DMA_IOC);
  713. if (dir_in)
  714. (*desc)->status |= (len % mps) ? DEV_DMA_SHORT :
  715. ((hs_ep->send_zlp && true_last) ?
  716. DEV_DMA_SHORT : 0);
  717. (*desc)->status |=
  718. len << DEV_DMA_NBYTES_SHIFT & mask;
  719. (*desc)->buf = dma_buff + offset;
  720. }
  721. (*desc)->status &= ~DEV_DMA_BUFF_STS_MASK;
  722. (*desc)->status |= (DEV_DMA_BUFF_STS_HREADY
  723. << DEV_DMA_BUFF_STS_SHIFT);
  724. (*desc)++;
  725. }
  726. }
  727. /*
  728. * dwc2_gadget_config_nonisoc_xfer_ddma - prepare non ISOC DMA desc chain.
  729. * @hs_ep: The endpoint
  730. * @ureq: Request to transfer
  731. * @offset: offset in bytes
  732. * @len: Length of the transfer
  733. *
  734. * This function will iterate over descriptor chain and fill its entries
  735. * with corresponding information based on transfer data.
  736. */
  737. static void dwc2_gadget_config_nonisoc_xfer_ddma(struct dwc2_hsotg_ep *hs_ep,
  738. dma_addr_t dma_buff,
  739. unsigned int len)
  740. {
  741. struct usb_request *ureq = NULL;
  742. struct dwc2_dma_desc *desc = hs_ep->desc_list;
  743. struct scatterlist *sg;
  744. int i;
  745. u8 desc_count = 0;
  746. if (hs_ep->req)
  747. ureq = &hs_ep->req->req;
  748. /* non-DMA sg buffer */
  749. if (!ureq || !ureq->num_sgs) {
  750. dwc2_gadget_fill_nonisoc_xfer_ddma_one(hs_ep, &desc,
  751. dma_buff, len, true);
  752. return;
  753. }
  754. /* DMA sg buffer */
  755. for_each_sg(ureq->sg, sg, ureq->num_sgs, i) {
  756. dwc2_gadget_fill_nonisoc_xfer_ddma_one(hs_ep, &desc,
  757. sg_dma_address(sg) + sg->offset, sg_dma_len(sg),
  758. sg_is_last(sg));
  759. desc_count += hs_ep->desc_count;
  760. }
  761. hs_ep->desc_count = desc_count;
  762. }
  763. /*
  764. * dwc2_gadget_fill_isoc_desc - fills next isochronous descriptor in chain.
  765. * @hs_ep: The isochronous endpoint.
  766. * @dma_buff: usb requests dma buffer.
  767. * @len: usb request transfer length.
  768. *
  769. * Fills next free descriptor with the data of the arrived usb request,
  770. * frame info, sets Last and IOC bits increments next_desc. If filled
  771. * descriptor is not the first one, removes L bit from the previous descriptor
  772. * status.
  773. */
  774. static int dwc2_gadget_fill_isoc_desc(struct dwc2_hsotg_ep *hs_ep,
  775. dma_addr_t dma_buff, unsigned int len)
  776. {
  777. struct dwc2_dma_desc *desc;
  778. struct dwc2_hsotg *hsotg = hs_ep->parent;
  779. u32 index;
  780. u32 mask = 0;
  781. u8 pid = 0;
  782. dwc2_gadget_get_desc_params(hs_ep, &mask);
  783. index = hs_ep->next_desc;
  784. desc = &hs_ep->desc_list[index];
  785. /* Check if descriptor chain full */
  786. if ((desc->status >> DEV_DMA_BUFF_STS_SHIFT) ==
  787. DEV_DMA_BUFF_STS_HREADY) {
  788. dev_dbg(hsotg->dev, "%s: desc chain full\n", __func__);
  789. return 1;
  790. }
  791. /* Clear L bit of previous desc if more than one entries in the chain */
  792. if (hs_ep->next_desc)
  793. hs_ep->desc_list[index - 1].status &= ~DEV_DMA_L;
  794. dev_dbg(hsotg->dev, "%s: Filling ep %d, dir %s isoc desc # %d\n",
  795. __func__, hs_ep->index, hs_ep->dir_in ? "in" : "out", index);
  796. desc->status = 0;
  797. desc->status |= (DEV_DMA_BUFF_STS_HBUSY << DEV_DMA_BUFF_STS_SHIFT);
  798. desc->buf = dma_buff;
  799. desc->status |= (DEV_DMA_L | DEV_DMA_IOC |
  800. ((len << DEV_DMA_NBYTES_SHIFT) & mask));
  801. if (hs_ep->dir_in) {
  802. if (len)
  803. pid = DIV_ROUND_UP(len, hs_ep->ep.maxpacket);
  804. else
  805. pid = 1;
  806. desc->status |= ((pid << DEV_DMA_ISOC_PID_SHIFT) &
  807. DEV_DMA_ISOC_PID_MASK) |
  808. ((len % hs_ep->ep.maxpacket) ?
  809. DEV_DMA_SHORT : 0) |
  810. ((hs_ep->target_frame <<
  811. DEV_DMA_ISOC_FRNUM_SHIFT) &
  812. DEV_DMA_ISOC_FRNUM_MASK);
  813. }
  814. desc->status &= ~DEV_DMA_BUFF_STS_MASK;
  815. desc->status |= (DEV_DMA_BUFF_STS_HREADY << DEV_DMA_BUFF_STS_SHIFT);
  816. /* Increment frame number by interval for IN */
  817. if (hs_ep->dir_in)
  818. dwc2_gadget_incr_frame_num(hs_ep);
  819. /* Update index of last configured entry in the chain */
  820. hs_ep->next_desc++;
  821. if (hs_ep->next_desc >= MAX_DMA_DESC_NUM_HS_ISOC)
  822. hs_ep->next_desc = 0;
  823. return 0;
  824. }
  825. /*
  826. * dwc2_gadget_start_isoc_ddma - start isochronous transfer in DDMA
  827. * @hs_ep: The isochronous endpoint.
  828. *
  829. * Prepare descriptor chain for isochronous endpoints. Afterwards
  830. * write DMA address to HW and enable the endpoint.
  831. */
  832. static void dwc2_gadget_start_isoc_ddma(struct dwc2_hsotg_ep *hs_ep)
  833. {
  834. struct dwc2_hsotg *hsotg = hs_ep->parent;
  835. struct dwc2_hsotg_req *hs_req, *treq;
  836. int index = hs_ep->index;
  837. int ret;
  838. int i;
  839. u32 dma_reg;
  840. u32 depctl;
  841. u32 ctrl;
  842. struct dwc2_dma_desc *desc;
  843. if (list_empty(&hs_ep->queue)) {
  844. hs_ep->target_frame = TARGET_FRAME_INITIAL;
  845. dev_dbg(hsotg->dev, "%s: No requests in queue\n", __func__);
  846. return;
  847. }
  848. /* Initialize descriptor chain by Host Busy status */
  849. for (i = 0; i < MAX_DMA_DESC_NUM_HS_ISOC; i++) {
  850. desc = &hs_ep->desc_list[i];
  851. desc->status = 0;
  852. desc->status |= (DEV_DMA_BUFF_STS_HBUSY
  853. << DEV_DMA_BUFF_STS_SHIFT);
  854. }
  855. hs_ep->next_desc = 0;
  856. list_for_each_entry_safe(hs_req, treq, &hs_ep->queue, queue) {
  857. dma_addr_t dma_addr = hs_req->req.dma;
  858. if (hs_req->req.num_sgs) {
  859. WARN_ON(hs_req->req.num_sgs > 1);
  860. dma_addr = sg_dma_address(hs_req->req.sg);
  861. }
  862. ret = dwc2_gadget_fill_isoc_desc(hs_ep, dma_addr,
  863. hs_req->req.length);
  864. if (ret)
  865. break;
  866. }
  867. hs_ep->compl_desc = 0;
  868. depctl = hs_ep->dir_in ? DIEPCTL(index) : DOEPCTL(index);
  869. dma_reg = hs_ep->dir_in ? DIEPDMA(index) : DOEPDMA(index);
  870. /* write descriptor chain address to control register */
  871. dwc2_writel(hsotg, hs_ep->desc_list_dma, dma_reg);
  872. ctrl = dwc2_readl(hsotg, depctl);
  873. ctrl |= DXEPCTL_EPENA | DXEPCTL_CNAK;
  874. dwc2_writel(hsotg, ctrl, depctl);
  875. }
  876. static bool dwc2_gadget_target_frame_elapsed(struct dwc2_hsotg_ep *hs_ep);
  877. static void dwc2_hsotg_complete_request(struct dwc2_hsotg *hsotg,
  878. struct dwc2_hsotg_ep *hs_ep,
  879. struct dwc2_hsotg_req *hs_req,
  880. int result);
  881. /**
  882. * dwc2_hsotg_start_req - start a USB request from an endpoint's queue
  883. * @hsotg: The controller state.
  884. * @hs_ep: The endpoint to process a request for
  885. * @hs_req: The request to start.
  886. * @continuing: True if we are doing more for the current request.
  887. *
  888. * Start the given request running by setting the endpoint registers
  889. * appropriately, and writing any data to the FIFOs.
  890. */
  891. static void dwc2_hsotg_start_req(struct dwc2_hsotg *hsotg,
  892. struct dwc2_hsotg_ep *hs_ep,
  893. struct dwc2_hsotg_req *hs_req,
  894. bool continuing)
  895. {
  896. struct usb_request *ureq = &hs_req->req;
  897. int index = hs_ep->index;
  898. int dir_in = hs_ep->dir_in;
  899. u32 epctrl_reg;
  900. u32 epsize_reg;
  901. u32 epsize;
  902. u32 ctrl;
  903. unsigned int length;
  904. unsigned int packets;
  905. unsigned int maxreq;
  906. unsigned int dma_reg;
  907. if (index != 0) {
  908. if (hs_ep->req && !continuing) {
  909. dev_err(hsotg->dev, "%s: active request\n", __func__);
  910. WARN_ON(1);
  911. return;
  912. } else if (hs_ep->req != hs_req && continuing) {
  913. dev_err(hsotg->dev,
  914. "%s: continue different req\n", __func__);
  915. WARN_ON(1);
  916. return;
  917. }
  918. }
  919. dma_reg = dir_in ? DIEPDMA(index) : DOEPDMA(index);
  920. epctrl_reg = dir_in ? DIEPCTL(index) : DOEPCTL(index);
  921. epsize_reg = dir_in ? DIEPTSIZ(index) : DOEPTSIZ(index);
  922. dev_dbg(hsotg->dev, "%s: DxEPCTL=0x%08x, ep %d, dir %s\n",
  923. __func__, dwc2_readl(hsotg, epctrl_reg), index,
  924. hs_ep->dir_in ? "in" : "out");
  925. /* If endpoint is stalled, we will restart request later */
  926. ctrl = dwc2_readl(hsotg, epctrl_reg);
  927. if (index && ctrl & DXEPCTL_STALL) {
  928. dev_warn(hsotg->dev, "%s: ep%d is stalled\n", __func__, index);
  929. return;
  930. }
  931. length = ureq->length - ureq->actual;
  932. dev_dbg(hsotg->dev, "ureq->length:%d ureq->actual:%d\n",
  933. ureq->length, ureq->actual);
  934. if (!using_desc_dma(hsotg))
  935. maxreq = get_ep_limit(hs_ep);
  936. else
  937. maxreq = dwc2_gadget_get_chain_limit(hs_ep);
  938. if (length > maxreq) {
  939. int round = maxreq % hs_ep->ep.maxpacket;
  940. dev_dbg(hsotg->dev, "%s: length %d, max-req %d, r %d\n",
  941. __func__, length, maxreq, round);
  942. /* round down to multiple of packets */
  943. if (round)
  944. maxreq -= round;
  945. length = maxreq;
  946. }
  947. if (length)
  948. packets = DIV_ROUND_UP(length, hs_ep->ep.maxpacket);
  949. else
  950. packets = 1; /* send one packet if length is zero. */
  951. if (dir_in && index != 0)
  952. if (hs_ep->isochronous)
  953. epsize = DXEPTSIZ_MC(packets);
  954. else
  955. epsize = DXEPTSIZ_MC(1);
  956. else
  957. epsize = 0;
  958. /*
  959. * zero length packet should be programmed on its own and should not
  960. * be counted in DIEPTSIZ.PktCnt with other packets.
  961. */
  962. if (dir_in && ureq->zero && !continuing) {
  963. /* Test if zlp is actually required. */
  964. if ((ureq->length >= hs_ep->ep.maxpacket) &&
  965. !(ureq->length % hs_ep->ep.maxpacket))
  966. hs_ep->send_zlp = 1;
  967. }
  968. epsize |= DXEPTSIZ_PKTCNT(packets);
  969. epsize |= DXEPTSIZ_XFERSIZE(length);
  970. dev_dbg(hsotg->dev, "%s: %d@%d/%d, 0x%08x => 0x%08x\n",
  971. __func__, packets, length, ureq->length, epsize, epsize_reg);
  972. /* store the request as the current one we're doing */
  973. hs_ep->req = hs_req;
  974. if (using_desc_dma(hsotg)) {
  975. u32 offset = 0;
  976. u32 mps = hs_ep->ep.maxpacket;
  977. /* Adjust length: EP0 - MPS, other OUT EPs - multiple of MPS */
  978. if (!dir_in) {
  979. if (!index)
  980. length = mps;
  981. else if (length % mps)
  982. length += (mps - (length % mps));
  983. }
  984. if (continuing)
  985. offset = ureq->actual;
  986. /* Fill DDMA chain entries */
  987. dwc2_gadget_config_nonisoc_xfer_ddma(hs_ep, ureq->dma + offset,
  988. length);
  989. /* write descriptor chain address to control register */
  990. dwc2_writel(hsotg, hs_ep->desc_list_dma, dma_reg);
  991. dev_dbg(hsotg->dev, "%s: %08x pad => 0x%08x\n",
  992. __func__, (u32)hs_ep->desc_list_dma, dma_reg);
  993. } else {
  994. /* write size / packets */
  995. dwc2_writel(hsotg, epsize, epsize_reg);
  996. if (using_dma(hsotg) && !continuing && (length != 0)) {
  997. /*
  998. * write DMA address to control register, buffer
  999. * already synced by dwc2_hsotg_ep_queue().
  1000. */
  1001. dwc2_writel(hsotg, ureq->dma, dma_reg);
  1002. dev_dbg(hsotg->dev, "%s: %pad => 0x%08x\n",
  1003. __func__, &ureq->dma, dma_reg);
  1004. }
  1005. }
  1006. if (hs_ep->isochronous) {
  1007. if (!dwc2_gadget_target_frame_elapsed(hs_ep)) {
  1008. if (hs_ep->interval == 1) {
  1009. if (hs_ep->target_frame & 0x1)
  1010. ctrl |= DXEPCTL_SETODDFR;
  1011. else
  1012. ctrl |= DXEPCTL_SETEVENFR;
  1013. }
  1014. ctrl |= DXEPCTL_CNAK;
  1015. } else {
  1016. hs_req->req.frame_number = hs_ep->target_frame;
  1017. hs_req->req.actual = 0;
  1018. dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, -ENODATA);
  1019. return;
  1020. }
  1021. }
  1022. ctrl |= DXEPCTL_EPENA; /* ensure ep enabled */
  1023. dev_dbg(hsotg->dev, "ep0 state:%d\n", hsotg->ep0_state);
  1024. /* For Setup request do not clear NAK */
  1025. if (!(index == 0 && hsotg->ep0_state == DWC2_EP0_SETUP))
  1026. ctrl |= DXEPCTL_CNAK; /* clear NAK set by core */
  1027. dev_dbg(hsotg->dev, "%s: DxEPCTL=0x%08x\n", __func__, ctrl);
  1028. dwc2_writel(hsotg, ctrl, epctrl_reg);
  1029. /*
  1030. * set these, it seems that DMA support increments past the end
  1031. * of the packet buffer so we need to calculate the length from
  1032. * this information.
  1033. */
  1034. hs_ep->size_loaded = length;
  1035. hs_ep->last_load = ureq->actual;
  1036. if (dir_in && !using_dma(hsotg)) {
  1037. /* set these anyway, we may need them for non-periodic in */
  1038. hs_ep->fifo_load = 0;
  1039. dwc2_hsotg_write_fifo(hsotg, hs_ep, hs_req);
  1040. }
  1041. /*
  1042. * Note, trying to clear the NAK here causes problems with transmit
  1043. * on the S3C6400 ending up with the TXFIFO becoming full.
  1044. */
  1045. /* check ep is enabled */
  1046. if (!(dwc2_readl(hsotg, epctrl_reg) & DXEPCTL_EPENA))
  1047. dev_dbg(hsotg->dev,
  1048. "ep%d: failed to become enabled (DXEPCTL=0x%08x)?\n",
  1049. index, dwc2_readl(hsotg, epctrl_reg));
  1050. dev_dbg(hsotg->dev, "%s: DXEPCTL=0x%08x\n",
  1051. __func__, dwc2_readl(hsotg, epctrl_reg));
  1052. /* enable ep interrupts */
  1053. dwc2_hsotg_ctrl_epint(hsotg, hs_ep->index, hs_ep->dir_in, 1);
  1054. }
  1055. /**
  1056. * dwc2_hsotg_map_dma - map the DMA memory being used for the request
  1057. * @hsotg: The device state.
  1058. * @hs_ep: The endpoint the request is on.
  1059. * @req: The request being processed.
  1060. *
  1061. * We've been asked to queue a request, so ensure that the memory buffer
  1062. * is correctly setup for DMA. If we've been passed an extant DMA address
  1063. * then ensure the buffer has been synced to memory. If our buffer has no
  1064. * DMA memory, then we map the memory and mark our request to allow us to
  1065. * cleanup on completion.
  1066. */
  1067. static int dwc2_hsotg_map_dma(struct dwc2_hsotg *hsotg,
  1068. struct dwc2_hsotg_ep *hs_ep,
  1069. struct usb_request *req)
  1070. {
  1071. int ret;
  1072. hs_ep->map_dir = hs_ep->dir_in;
  1073. ret = usb_gadget_map_request(&hsotg->gadget, req, hs_ep->dir_in);
  1074. if (ret)
  1075. goto dma_error;
  1076. return 0;
  1077. dma_error:
  1078. dev_err(hsotg->dev, "%s: failed to map buffer %p, %d bytes\n",
  1079. __func__, req->buf, req->length);
  1080. return -EIO;
  1081. }
  1082. static int dwc2_hsotg_handle_unaligned_buf_start(struct dwc2_hsotg *hsotg,
  1083. struct dwc2_hsotg_ep *hs_ep,
  1084. struct dwc2_hsotg_req *hs_req)
  1085. {
  1086. void *req_buf = hs_req->req.buf;
  1087. /* If dma is not being used or buffer is aligned */
  1088. if (!using_dma(hsotg) || !((long)req_buf & 3))
  1089. return 0;
  1090. WARN_ON(hs_req->saved_req_buf);
  1091. dev_dbg(hsotg->dev, "%s: %s: buf=%p length=%d\n", __func__,
  1092. hs_ep->ep.name, req_buf, hs_req->req.length);
  1093. hs_req->req.buf = kmalloc(hs_req->req.length, GFP_ATOMIC);
  1094. if (!hs_req->req.buf) {
  1095. hs_req->req.buf = req_buf;
  1096. dev_err(hsotg->dev,
  1097. "%s: unable to allocate memory for bounce buffer\n",
  1098. __func__);
  1099. return -ENOMEM;
  1100. }
  1101. /* Save actual buffer */
  1102. hs_req->saved_req_buf = req_buf;
  1103. if (hs_ep->dir_in)
  1104. memcpy(hs_req->req.buf, req_buf, hs_req->req.length);
  1105. return 0;
  1106. }
  1107. static void
  1108. dwc2_hsotg_handle_unaligned_buf_complete(struct dwc2_hsotg *hsotg,
  1109. struct dwc2_hsotg_ep *hs_ep,
  1110. struct dwc2_hsotg_req *hs_req)
  1111. {
  1112. /* If dma is not being used or buffer was aligned */
  1113. if (!using_dma(hsotg) || !hs_req->saved_req_buf)
  1114. return;
  1115. dev_dbg(hsotg->dev, "%s: %s: status=%d actual-length=%d\n", __func__,
  1116. hs_ep->ep.name, hs_req->req.status, hs_req->req.actual);
  1117. /* Copy data from bounce buffer on successful out transfer */
  1118. if (!hs_ep->dir_in && !hs_req->req.status)
  1119. memcpy(hs_req->saved_req_buf, hs_req->req.buf,
  1120. hs_req->req.actual);
  1121. /* Free bounce buffer */
  1122. kfree(hs_req->req.buf);
  1123. hs_req->req.buf = hs_req->saved_req_buf;
  1124. hs_req->saved_req_buf = NULL;
  1125. }
  1126. /**
  1127. * dwc2_gadget_target_frame_elapsed - Checks target frame
  1128. * @hs_ep: The driver endpoint to check
  1129. *
  1130. * Returns 1 if targeted frame elapsed. If returned 1 then we need to drop
  1131. * corresponding transfer.
  1132. */
  1133. static bool dwc2_gadget_target_frame_elapsed(struct dwc2_hsotg_ep *hs_ep)
  1134. {
  1135. struct dwc2_hsotg *hsotg = hs_ep->parent;
  1136. u32 target_frame = hs_ep->target_frame;
  1137. u32 current_frame = hsotg->frame_number;
  1138. bool frame_overrun = hs_ep->frame_overrun;
  1139. u16 limit = DSTS_SOFFN_LIMIT;
  1140. if (hsotg->gadget.speed != USB_SPEED_HIGH)
  1141. limit >>= 3;
  1142. if (!frame_overrun && current_frame >= target_frame)
  1143. return true;
  1144. if (frame_overrun && current_frame >= target_frame &&
  1145. ((current_frame - target_frame) < limit / 2))
  1146. return true;
  1147. return false;
  1148. }
  1149. /*
  1150. * dwc2_gadget_set_ep0_desc_chain - Set EP's desc chain pointers
  1151. * @hsotg: The driver state
  1152. * @hs_ep: the ep descriptor chain is for
  1153. *
  1154. * Called to update EP0 structure's pointers depend on stage of
  1155. * control transfer.
  1156. */
  1157. static int dwc2_gadget_set_ep0_desc_chain(struct dwc2_hsotg *hsotg,
  1158. struct dwc2_hsotg_ep *hs_ep)
  1159. {
  1160. switch (hsotg->ep0_state) {
  1161. case DWC2_EP0_SETUP:
  1162. case DWC2_EP0_STATUS_OUT:
  1163. hs_ep->desc_list = hsotg->setup_desc[0];
  1164. hs_ep->desc_list_dma = hsotg->setup_desc_dma[0];
  1165. break;
  1166. case DWC2_EP0_DATA_IN:
  1167. case DWC2_EP0_STATUS_IN:
  1168. hs_ep->desc_list = hsotg->ctrl_in_desc;
  1169. hs_ep->desc_list_dma = hsotg->ctrl_in_desc_dma;
  1170. break;
  1171. case DWC2_EP0_DATA_OUT:
  1172. hs_ep->desc_list = hsotg->ctrl_out_desc;
  1173. hs_ep->desc_list_dma = hsotg->ctrl_out_desc_dma;
  1174. break;
  1175. default:
  1176. dev_err(hsotg->dev, "invalid EP 0 state in queue %d\n",
  1177. hsotg->ep0_state);
  1178. return -EINVAL;
  1179. }
  1180. return 0;
  1181. }
  1182. static int dwc2_hsotg_ep_queue(struct usb_ep *ep, struct usb_request *req,
  1183. gfp_t gfp_flags)
  1184. {
  1185. struct dwc2_hsotg_req *hs_req = our_req(req);
  1186. struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
  1187. struct dwc2_hsotg *hs = hs_ep->parent;
  1188. bool first;
  1189. int ret;
  1190. u32 maxsize = 0;
  1191. u32 mask = 0;
  1192. dev_dbg(hs->dev, "%s: req %p: %d@%p, noi=%d, zero=%d, snok=%d\n",
  1193. ep->name, req, req->length, req->buf, req->no_interrupt,
  1194. req->zero, req->short_not_ok);
  1195. /* Prevent new request submission when controller is suspended */
  1196. if (hs->lx_state != DWC2_L0) {
  1197. dev_dbg(hs->dev, "%s: submit request only in active state\n",
  1198. __func__);
  1199. return -EAGAIN;
  1200. }
  1201. /* initialise status of the request */
  1202. INIT_LIST_HEAD(&hs_req->queue);
  1203. req->actual = 0;
  1204. req->status = -EINPROGRESS;
  1205. /* Don't queue ISOC request if length greater than mps*mc */
  1206. if (hs_ep->isochronous &&
  1207. req->length > (hs_ep->mc * hs_ep->ep.maxpacket)) {
  1208. dev_err(hs->dev, "req length > maxpacket*mc\n");
  1209. return -EINVAL;
  1210. }
  1211. /* In DDMA mode for ISOC's don't queue request if length greater
  1212. * than descriptor limits.
  1213. */
  1214. if (using_desc_dma(hs) && hs_ep->isochronous) {
  1215. maxsize = dwc2_gadget_get_desc_params(hs_ep, &mask);
  1216. if (hs_ep->dir_in && req->length > maxsize) {
  1217. dev_err(hs->dev, "wrong length %d (maxsize=%d)\n",
  1218. req->length, maxsize);
  1219. return -EINVAL;
  1220. }
  1221. if (!hs_ep->dir_in && req->length > hs_ep->ep.maxpacket) {
  1222. dev_err(hs->dev, "ISOC OUT: wrong length %d (mps=%d)\n",
  1223. req->length, hs_ep->ep.maxpacket);
  1224. return -EINVAL;
  1225. }
  1226. }
  1227. ret = dwc2_hsotg_handle_unaligned_buf_start(hs, hs_ep, hs_req);
  1228. if (ret)
  1229. return ret;
  1230. /* if we're using DMA, sync the buffers as necessary */
  1231. if (using_dma(hs)) {
  1232. ret = dwc2_hsotg_map_dma(hs, hs_ep, req);
  1233. if (ret)
  1234. return ret;
  1235. }
  1236. /* If using descriptor DMA configure EP0 descriptor chain pointers */
  1237. if (using_desc_dma(hs) && !hs_ep->index) {
  1238. ret = dwc2_gadget_set_ep0_desc_chain(hs, hs_ep);
  1239. if (ret)
  1240. return ret;
  1241. }
  1242. first = list_empty(&hs_ep->queue);
  1243. list_add_tail(&hs_req->queue, &hs_ep->queue);
  1244. /*
  1245. * Handle DDMA isochronous transfers separately - just add new entry
  1246. * to the descriptor chain.
  1247. * Transfer will be started once SW gets either one of NAK or
  1248. * OutTknEpDis interrupts.
  1249. */
  1250. if (using_desc_dma(hs) && hs_ep->isochronous) {
  1251. if (hs_ep->target_frame != TARGET_FRAME_INITIAL) {
  1252. dma_addr_t dma_addr = hs_req->req.dma;
  1253. if (hs_req->req.num_sgs) {
  1254. WARN_ON(hs_req->req.num_sgs > 1);
  1255. dma_addr = sg_dma_address(hs_req->req.sg);
  1256. }
  1257. dwc2_gadget_fill_isoc_desc(hs_ep, dma_addr,
  1258. hs_req->req.length);
  1259. }
  1260. return 0;
  1261. }
  1262. /* Change EP direction if status phase request is after data out */
  1263. if (!hs_ep->index && !req->length && !hs_ep->dir_in &&
  1264. hs->ep0_state == DWC2_EP0_DATA_OUT)
  1265. hs_ep->dir_in = 1;
  1266. if (first) {
  1267. if (!hs_ep->isochronous) {
  1268. dwc2_hsotg_start_req(hs, hs_ep, hs_req, false);
  1269. return 0;
  1270. }
  1271. /* Update current frame number value. */
  1272. hs->frame_number = dwc2_hsotg_read_frameno(hs);
  1273. while (dwc2_gadget_target_frame_elapsed(hs_ep)) {
  1274. dwc2_gadget_incr_frame_num(hs_ep);
  1275. /* Update current frame number value once more as it
  1276. * changes here.
  1277. */
  1278. hs->frame_number = dwc2_hsotg_read_frameno(hs);
  1279. }
  1280. if (hs_ep->target_frame != TARGET_FRAME_INITIAL)
  1281. dwc2_hsotg_start_req(hs, hs_ep, hs_req, false);
  1282. }
  1283. return 0;
  1284. }
  1285. static int dwc2_hsotg_ep_queue_lock(struct usb_ep *ep, struct usb_request *req,
  1286. gfp_t gfp_flags)
  1287. {
  1288. struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
  1289. struct dwc2_hsotg *hs = hs_ep->parent;
  1290. unsigned long flags = 0;
  1291. int ret = 0;
  1292. spin_lock_irqsave(&hs->lock, flags);
  1293. ret = dwc2_hsotg_ep_queue(ep, req, gfp_flags);
  1294. spin_unlock_irqrestore(&hs->lock, flags);
  1295. return ret;
  1296. }
  1297. static void dwc2_hsotg_ep_free_request(struct usb_ep *ep,
  1298. struct usb_request *req)
  1299. {
  1300. struct dwc2_hsotg_req *hs_req = our_req(req);
  1301. kfree(hs_req);
  1302. }
  1303. /**
  1304. * dwc2_hsotg_complete_oursetup - setup completion callback
  1305. * @ep: The endpoint the request was on.
  1306. * @req: The request completed.
  1307. *
  1308. * Called on completion of any requests the driver itself
  1309. * submitted that need cleaning up.
  1310. */
  1311. static void dwc2_hsotg_complete_oursetup(struct usb_ep *ep,
  1312. struct usb_request *req)
  1313. {
  1314. struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
  1315. struct dwc2_hsotg *hsotg = hs_ep->parent;
  1316. dev_dbg(hsotg->dev, "%s: ep %p, req %p\n", __func__, ep, req);
  1317. dwc2_hsotg_ep_free_request(ep, req);
  1318. }
  1319. /**
  1320. * ep_from_windex - convert control wIndex value to endpoint
  1321. * @hsotg: The driver state.
  1322. * @windex: The control request wIndex field (in host order).
  1323. *
  1324. * Convert the given wIndex into a pointer to an driver endpoint
  1325. * structure, or return NULL if it is not a valid endpoint.
  1326. */
  1327. static struct dwc2_hsotg_ep *ep_from_windex(struct dwc2_hsotg *hsotg,
  1328. u32 windex)
  1329. {
  1330. int dir = (windex & USB_DIR_IN) ? 1 : 0;
  1331. int idx = windex & 0x7F;
  1332. if (windex >= 0x100)
  1333. return NULL;
  1334. if (idx > hsotg->num_of_eps)
  1335. return NULL;
  1336. return index_to_ep(hsotg, idx, dir);
  1337. }
  1338. /**
  1339. * dwc2_hsotg_set_test_mode - Enable usb Test Modes
  1340. * @hsotg: The driver state.
  1341. * @testmode: requested usb test mode
  1342. * Enable usb Test Mode requested by the Host.
  1343. */
  1344. int dwc2_hsotg_set_test_mode(struct dwc2_hsotg *hsotg, int testmode)
  1345. {
  1346. int dctl = dwc2_readl(hsotg, DCTL);
  1347. dctl &= ~DCTL_TSTCTL_MASK;
  1348. switch (testmode) {
  1349. case USB_TEST_J:
  1350. case USB_TEST_K:
  1351. case USB_TEST_SE0_NAK:
  1352. case USB_TEST_PACKET:
  1353. case USB_TEST_FORCE_ENABLE:
  1354. dctl |= testmode << DCTL_TSTCTL_SHIFT;
  1355. break;
  1356. default:
  1357. return -EINVAL;
  1358. }
  1359. dwc2_writel(hsotg, dctl, DCTL);
  1360. return 0;
  1361. }
  1362. /**
  1363. * dwc2_hsotg_send_reply - send reply to control request
  1364. * @hsotg: The device state
  1365. * @ep: Endpoint 0
  1366. * @buff: Buffer for request
  1367. * @length: Length of reply.
  1368. *
  1369. * Create a request and queue it on the given endpoint. This is useful as
  1370. * an internal method of sending replies to certain control requests, etc.
  1371. */
  1372. static int dwc2_hsotg_send_reply(struct dwc2_hsotg *hsotg,
  1373. struct dwc2_hsotg_ep *ep,
  1374. void *buff,
  1375. int length)
  1376. {
  1377. struct usb_request *req;
  1378. int ret;
  1379. dev_dbg(hsotg->dev, "%s: buff %p, len %d\n", __func__, buff, length);
  1380. req = dwc2_hsotg_ep_alloc_request(&ep->ep, GFP_ATOMIC);
  1381. hsotg->ep0_reply = req;
  1382. if (!req) {
  1383. dev_warn(hsotg->dev, "%s: cannot alloc req\n", __func__);
  1384. return -ENOMEM;
  1385. }
  1386. req->buf = hsotg->ep0_buff;
  1387. req->length = length;
  1388. /*
  1389. * zero flag is for sending zlp in DATA IN stage. It has no impact on
  1390. * STATUS stage.
  1391. */
  1392. req->zero = 0;
  1393. req->complete = dwc2_hsotg_complete_oursetup;
  1394. if (length)
  1395. memcpy(req->buf, buff, length);
  1396. ret = dwc2_hsotg_ep_queue(&ep->ep, req, GFP_ATOMIC);
  1397. if (ret) {
  1398. dev_warn(hsotg->dev, "%s: cannot queue req\n", __func__);
  1399. return ret;
  1400. }
  1401. return 0;
  1402. }
  1403. /**
  1404. * dwc2_hsotg_process_req_status - process request GET_STATUS
  1405. * @hsotg: The device state
  1406. * @ctrl: USB control request
  1407. */
  1408. static int dwc2_hsotg_process_req_status(struct dwc2_hsotg *hsotg,
  1409. struct usb_ctrlrequest *ctrl)
  1410. {
  1411. struct dwc2_hsotg_ep *ep0 = hsotg->eps_out[0];
  1412. struct dwc2_hsotg_ep *ep;
  1413. __le16 reply;
  1414. u16 status;
  1415. int ret;
  1416. dev_dbg(hsotg->dev, "%s: USB_REQ_GET_STATUS\n", __func__);
  1417. if (!ep0->dir_in) {
  1418. dev_warn(hsotg->dev, "%s: direction out?\n", __func__);
  1419. return -EINVAL;
  1420. }
  1421. switch (ctrl->bRequestType & USB_RECIP_MASK) {
  1422. case USB_RECIP_DEVICE:
  1423. status = hsotg->gadget.is_selfpowered <<
  1424. USB_DEVICE_SELF_POWERED;
  1425. status |= hsotg->remote_wakeup_allowed <<
  1426. USB_DEVICE_REMOTE_WAKEUP;
  1427. reply = cpu_to_le16(status);
  1428. break;
  1429. case USB_RECIP_INTERFACE:
  1430. /* currently, the data result should be zero */
  1431. reply = cpu_to_le16(0);
  1432. break;
  1433. case USB_RECIP_ENDPOINT:
  1434. ep = ep_from_windex(hsotg, le16_to_cpu(ctrl->wIndex));
  1435. if (!ep)
  1436. return -ENOENT;
  1437. reply = cpu_to_le16(ep->halted ? 1 : 0);
  1438. break;
  1439. default:
  1440. return 0;
  1441. }
  1442. if (le16_to_cpu(ctrl->wLength) != 2)
  1443. return -EINVAL;
  1444. ret = dwc2_hsotg_send_reply(hsotg, ep0, &reply, 2);
  1445. if (ret) {
  1446. dev_err(hsotg->dev, "%s: failed to send reply\n", __func__);
  1447. return ret;
  1448. }
  1449. return 1;
  1450. }
  1451. static int dwc2_hsotg_ep_sethalt(struct usb_ep *ep, int value, bool now);
  1452. /**
  1453. * get_ep_head - return the first request on the endpoint
  1454. * @hs_ep: The controller endpoint to get
  1455. *
  1456. * Get the first request on the endpoint.
  1457. */
  1458. static struct dwc2_hsotg_req *get_ep_head(struct dwc2_hsotg_ep *hs_ep)
  1459. {
  1460. return list_first_entry_or_null(&hs_ep->queue, struct dwc2_hsotg_req,
  1461. queue);
  1462. }
  1463. /**
  1464. * dwc2_gadget_start_next_request - Starts next request from ep queue
  1465. * @hs_ep: Endpoint structure
  1466. *
  1467. * If queue is empty and EP is ISOC-OUT - unmasks OUTTKNEPDIS which is masked
  1468. * in its handler. Hence we need to unmask it here to be able to do
  1469. * resynchronization.
  1470. */
  1471. static void dwc2_gadget_start_next_request(struct dwc2_hsotg_ep *hs_ep)
  1472. {
  1473. struct dwc2_hsotg *hsotg = hs_ep->parent;
  1474. int dir_in = hs_ep->dir_in;
  1475. struct dwc2_hsotg_req *hs_req;
  1476. if (!list_empty(&hs_ep->queue)) {
  1477. hs_req = get_ep_head(hs_ep);
  1478. dwc2_hsotg_start_req(hsotg, hs_ep, hs_req, false);
  1479. return;
  1480. }
  1481. if (!hs_ep->isochronous)
  1482. return;
  1483. if (dir_in) {
  1484. dev_dbg(hsotg->dev, "%s: No more ISOC-IN requests\n",
  1485. __func__);
  1486. } else {
  1487. dev_dbg(hsotg->dev, "%s: No more ISOC-OUT requests\n",
  1488. __func__);
  1489. }
  1490. }
  1491. /**
  1492. * dwc2_hsotg_process_req_feature - process request {SET,CLEAR}_FEATURE
  1493. * @hsotg: The device state
  1494. * @ctrl: USB control request
  1495. */
  1496. static int dwc2_hsotg_process_req_feature(struct dwc2_hsotg *hsotg,
  1497. struct usb_ctrlrequest *ctrl)
  1498. {
  1499. struct dwc2_hsotg_ep *ep0 = hsotg->eps_out[0];
  1500. struct dwc2_hsotg_req *hs_req;
  1501. bool set = (ctrl->bRequest == USB_REQ_SET_FEATURE);
  1502. struct dwc2_hsotg_ep *ep;
  1503. int ret;
  1504. bool halted;
  1505. u32 recip;
  1506. u32 wValue;
  1507. u32 wIndex;
  1508. dev_dbg(hsotg->dev, "%s: %s_FEATURE\n",
  1509. __func__, set ? "SET" : "CLEAR");
  1510. wValue = le16_to_cpu(ctrl->wValue);
  1511. wIndex = le16_to_cpu(ctrl->wIndex);
  1512. recip = ctrl->bRequestType & USB_RECIP_MASK;
  1513. switch (recip) {
  1514. case USB_RECIP_DEVICE:
  1515. switch (wValue) {
  1516. case USB_DEVICE_REMOTE_WAKEUP:
  1517. if (set)
  1518. hsotg->remote_wakeup_allowed = 1;
  1519. else
  1520. hsotg->remote_wakeup_allowed = 0;
  1521. break;
  1522. case USB_DEVICE_TEST_MODE:
  1523. if ((wIndex & 0xff) != 0)
  1524. return -EINVAL;
  1525. if (!set)
  1526. return -EINVAL;
  1527. hsotg->test_mode = wIndex >> 8;
  1528. break;
  1529. default:
  1530. return -ENOENT;
  1531. }
  1532. ret = dwc2_hsotg_send_reply(hsotg, ep0, NULL, 0);
  1533. if (ret) {
  1534. dev_err(hsotg->dev,
  1535. "%s: failed to send reply\n", __func__);
  1536. return ret;
  1537. }
  1538. break;
  1539. case USB_RECIP_ENDPOINT:
  1540. ep = ep_from_windex(hsotg, wIndex);
  1541. if (!ep) {
  1542. dev_dbg(hsotg->dev, "%s: no endpoint for 0x%04x\n",
  1543. __func__, wIndex);
  1544. return -ENOENT;
  1545. }
  1546. switch (wValue) {
  1547. case USB_ENDPOINT_HALT:
  1548. halted = ep->halted;
  1549. dwc2_hsotg_ep_sethalt(&ep->ep, set, true);
  1550. ret = dwc2_hsotg_send_reply(hsotg, ep0, NULL, 0);
  1551. if (ret) {
  1552. dev_err(hsotg->dev,
  1553. "%s: failed to send reply\n", __func__);
  1554. return ret;
  1555. }
  1556. /*
  1557. * we have to complete all requests for ep if it was
  1558. * halted, and the halt was cleared by CLEAR_FEATURE
  1559. */
  1560. if (!set && halted) {
  1561. /*
  1562. * If we have request in progress,
  1563. * then complete it
  1564. */
  1565. if (ep->req) {
  1566. hs_req = ep->req;
  1567. ep->req = NULL;
  1568. list_del_init(&hs_req->queue);
  1569. if (hs_req->req.complete) {
  1570. spin_unlock(&hsotg->lock);
  1571. usb_gadget_giveback_request(
  1572. &ep->ep, &hs_req->req);
  1573. spin_lock(&hsotg->lock);
  1574. }
  1575. }
  1576. /* If we have pending request, then start it */
  1577. if (!ep->req)
  1578. dwc2_gadget_start_next_request(ep);
  1579. }
  1580. break;
  1581. default:
  1582. return -ENOENT;
  1583. }
  1584. break;
  1585. default:
  1586. return -ENOENT;
  1587. }
  1588. return 1;
  1589. }
  1590. static void dwc2_hsotg_enqueue_setup(struct dwc2_hsotg *hsotg);
  1591. /**
  1592. * dwc2_hsotg_stall_ep0 - stall ep0
  1593. * @hsotg: The device state
  1594. *
  1595. * Set stall for ep0 as response for setup request.
  1596. */
  1597. static void dwc2_hsotg_stall_ep0(struct dwc2_hsotg *hsotg)
  1598. {
  1599. struct dwc2_hsotg_ep *ep0 = hsotg->eps_out[0];
  1600. u32 reg;
  1601. u32 ctrl;
  1602. dev_dbg(hsotg->dev, "ep0 stall (dir=%d)\n", ep0->dir_in);
  1603. reg = (ep0->dir_in) ? DIEPCTL0 : DOEPCTL0;
  1604. /*
  1605. * DxEPCTL_Stall will be cleared by EP once it has
  1606. * taken effect, so no need to clear later.
  1607. */
  1608. ctrl = dwc2_readl(hsotg, reg);
  1609. ctrl |= DXEPCTL_STALL;
  1610. ctrl |= DXEPCTL_CNAK;
  1611. dwc2_writel(hsotg, ctrl, reg);
  1612. dev_dbg(hsotg->dev,
  1613. "written DXEPCTL=0x%08x to %08x (DXEPCTL=0x%08x)\n",
  1614. ctrl, reg, dwc2_readl(hsotg, reg));
  1615. /*
  1616. * complete won't be called, so we enqueue
  1617. * setup request here
  1618. */
  1619. dwc2_hsotg_enqueue_setup(hsotg);
  1620. }
  1621. /**
  1622. * dwc2_hsotg_process_control - process a control request
  1623. * @hsotg: The device state
  1624. * @ctrl: The control request received
  1625. *
  1626. * The controller has received the SETUP phase of a control request, and
  1627. * needs to work out what to do next (and whether to pass it on to the
  1628. * gadget driver).
  1629. */
  1630. static void dwc2_hsotg_process_control(struct dwc2_hsotg *hsotg,
  1631. struct usb_ctrlrequest *ctrl)
  1632. {
  1633. struct dwc2_hsotg_ep *ep0 = hsotg->eps_out[0];
  1634. int ret = 0;
  1635. u32 dcfg;
  1636. dev_dbg(hsotg->dev,
  1637. "ctrl Type=%02x, Req=%02x, V=%04x, I=%04x, L=%04x\n",
  1638. ctrl->bRequestType, ctrl->bRequest, ctrl->wValue,
  1639. ctrl->wIndex, ctrl->wLength);
  1640. if (ctrl->wLength == 0) {
  1641. ep0->dir_in = 1;
  1642. hsotg->ep0_state = DWC2_EP0_STATUS_IN;
  1643. } else if (ctrl->bRequestType & USB_DIR_IN) {
  1644. ep0->dir_in = 1;
  1645. hsotg->ep0_state = DWC2_EP0_DATA_IN;
  1646. } else {
  1647. ep0->dir_in = 0;
  1648. hsotg->ep0_state = DWC2_EP0_DATA_OUT;
  1649. }
  1650. if ((ctrl->bRequestType & USB_TYPE_MASK) == USB_TYPE_STANDARD) {
  1651. switch (ctrl->bRequest) {
  1652. case USB_REQ_SET_ADDRESS:
  1653. hsotg->connected = 1;
  1654. dcfg = dwc2_readl(hsotg, DCFG);
  1655. dcfg &= ~DCFG_DEVADDR_MASK;
  1656. dcfg |= (le16_to_cpu(ctrl->wValue) <<
  1657. DCFG_DEVADDR_SHIFT) & DCFG_DEVADDR_MASK;
  1658. dwc2_writel(hsotg, dcfg, DCFG);
  1659. dev_info(hsotg->dev, "new address %d\n", ctrl->wValue);
  1660. ret = dwc2_hsotg_send_reply(hsotg, ep0, NULL, 0);
  1661. return;
  1662. case USB_REQ_GET_STATUS:
  1663. ret = dwc2_hsotg_process_req_status(hsotg, ctrl);
  1664. break;
  1665. case USB_REQ_CLEAR_FEATURE:
  1666. case USB_REQ_SET_FEATURE:
  1667. ret = dwc2_hsotg_process_req_feature(hsotg, ctrl);
  1668. break;
  1669. }
  1670. }
  1671. /* as a fallback, try delivering it to the driver to deal with */
  1672. if (ret == 0 && hsotg->driver) {
  1673. spin_unlock(&hsotg->lock);
  1674. ret = hsotg->driver->setup(&hsotg->gadget, ctrl);
  1675. spin_lock(&hsotg->lock);
  1676. if (ret < 0)
  1677. dev_dbg(hsotg->dev, "driver->setup() ret %d\n", ret);
  1678. }
  1679. hsotg->delayed_status = false;
  1680. if (ret == USB_GADGET_DELAYED_STATUS)
  1681. hsotg->delayed_status = true;
  1682. /*
  1683. * the request is either unhandlable, or is not formatted correctly
  1684. * so respond with a STALL for the status stage to indicate failure.
  1685. */
  1686. if (ret < 0)
  1687. dwc2_hsotg_stall_ep0(hsotg);
  1688. }
  1689. /**
  1690. * dwc2_hsotg_complete_setup - completion of a setup transfer
  1691. * @ep: The endpoint the request was on.
  1692. * @req: The request completed.
  1693. *
  1694. * Called on completion of any requests the driver itself submitted for
  1695. * EP0 setup packets
  1696. */
  1697. static void dwc2_hsotg_complete_setup(struct usb_ep *ep,
  1698. struct usb_request *req)
  1699. {
  1700. struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
  1701. struct dwc2_hsotg *hsotg = hs_ep->parent;
  1702. if (req->status < 0) {
  1703. dev_dbg(hsotg->dev, "%s: failed %d\n", __func__, req->status);
  1704. return;
  1705. }
  1706. spin_lock(&hsotg->lock);
  1707. if (req->actual == 0)
  1708. dwc2_hsotg_enqueue_setup(hsotg);
  1709. else
  1710. dwc2_hsotg_process_control(hsotg, req->buf);
  1711. spin_unlock(&hsotg->lock);
  1712. }
  1713. /**
  1714. * dwc2_hsotg_enqueue_setup - start a request for EP0 packets
  1715. * @hsotg: The device state.
  1716. *
  1717. * Enqueue a request on EP0 if necessary to received any SETUP packets
  1718. * received from the host.
  1719. */
  1720. static void dwc2_hsotg_enqueue_setup(struct dwc2_hsotg *hsotg)
  1721. {
  1722. struct usb_request *req = hsotg->ctrl_req;
  1723. struct dwc2_hsotg_req *hs_req = our_req(req);
  1724. int ret;
  1725. dev_dbg(hsotg->dev, "%s: queueing setup request\n", __func__);
  1726. req->zero = 0;
  1727. req->length = 8;
  1728. req->buf = hsotg->ctrl_buff;
  1729. req->complete = dwc2_hsotg_complete_setup;
  1730. if (!list_empty(&hs_req->queue)) {
  1731. dev_dbg(hsotg->dev, "%s already queued???\n", __func__);
  1732. return;
  1733. }
  1734. hsotg->eps_out[0]->dir_in = 0;
  1735. hsotg->eps_out[0]->send_zlp = 0;
  1736. hsotg->ep0_state = DWC2_EP0_SETUP;
  1737. ret = dwc2_hsotg_ep_queue(&hsotg->eps_out[0]->ep, req, GFP_ATOMIC);
  1738. if (ret < 0) {
  1739. dev_err(hsotg->dev, "%s: failed queue (%d)\n", __func__, ret);
  1740. /*
  1741. * Don't think there's much we can do other than watch the
  1742. * driver fail.
  1743. */
  1744. }
  1745. }
  1746. static void dwc2_hsotg_program_zlp(struct dwc2_hsotg *hsotg,
  1747. struct dwc2_hsotg_ep *hs_ep)
  1748. {
  1749. u32 ctrl;
  1750. u8 index = hs_ep->index;
  1751. u32 epctl_reg = hs_ep->dir_in ? DIEPCTL(index) : DOEPCTL(index);
  1752. u32 epsiz_reg = hs_ep->dir_in ? DIEPTSIZ(index) : DOEPTSIZ(index);
  1753. if (hs_ep->dir_in)
  1754. dev_dbg(hsotg->dev, "Sending zero-length packet on ep%d\n",
  1755. index);
  1756. else
  1757. dev_dbg(hsotg->dev, "Receiving zero-length packet on ep%d\n",
  1758. index);
  1759. if (using_desc_dma(hsotg)) {
  1760. /* Not specific buffer needed for ep0 ZLP */
  1761. dma_addr_t dma = hs_ep->desc_list_dma;
  1762. if (!index)
  1763. dwc2_gadget_set_ep0_desc_chain(hsotg, hs_ep);
  1764. dwc2_gadget_config_nonisoc_xfer_ddma(hs_ep, dma, 0);
  1765. } else {
  1766. dwc2_writel(hsotg, DXEPTSIZ_MC(1) | DXEPTSIZ_PKTCNT(1) |
  1767. DXEPTSIZ_XFERSIZE(0),
  1768. epsiz_reg);
  1769. }
  1770. ctrl = dwc2_readl(hsotg, epctl_reg);
  1771. ctrl |= DXEPCTL_CNAK; /* clear NAK set by core */
  1772. ctrl |= DXEPCTL_EPENA; /* ensure ep enabled */
  1773. ctrl |= DXEPCTL_USBACTEP;
  1774. dwc2_writel(hsotg, ctrl, epctl_reg);
  1775. }
  1776. /**
  1777. * dwc2_hsotg_complete_request - complete a request given to us
  1778. * @hsotg: The device state.
  1779. * @hs_ep: The endpoint the request was on.
  1780. * @hs_req: The request to complete.
  1781. * @result: The result code (0 => Ok, otherwise errno)
  1782. *
  1783. * The given request has finished, so call the necessary completion
  1784. * if it has one and then look to see if we can start a new request
  1785. * on the endpoint.
  1786. *
  1787. * Note, expects the ep to already be locked as appropriate.
  1788. */
  1789. static void dwc2_hsotg_complete_request(struct dwc2_hsotg *hsotg,
  1790. struct dwc2_hsotg_ep *hs_ep,
  1791. struct dwc2_hsotg_req *hs_req,
  1792. int result)
  1793. {
  1794. if (!hs_req) {
  1795. dev_dbg(hsotg->dev, "%s: nothing to complete?\n", __func__);
  1796. return;
  1797. }
  1798. dev_dbg(hsotg->dev, "complete: ep %p %s, req %p, %d => %p\n",
  1799. hs_ep, hs_ep->ep.name, hs_req, result, hs_req->req.complete);
  1800. /*
  1801. * only replace the status if we've not already set an error
  1802. * from a previous transaction
  1803. */
  1804. if (hs_req->req.status == -EINPROGRESS)
  1805. hs_req->req.status = result;
  1806. if (using_dma(hsotg))
  1807. dwc2_hsotg_unmap_dma(hsotg, hs_ep, hs_req);
  1808. dwc2_hsotg_handle_unaligned_buf_complete(hsotg, hs_ep, hs_req);
  1809. hs_ep->req = NULL;
  1810. list_del_init(&hs_req->queue);
  1811. /*
  1812. * call the complete request with the locks off, just in case the
  1813. * request tries to queue more work for this endpoint.
  1814. */
  1815. if (hs_req->req.complete) {
  1816. spin_unlock(&hsotg->lock);
  1817. usb_gadget_giveback_request(&hs_ep->ep, &hs_req->req);
  1818. spin_lock(&hsotg->lock);
  1819. }
  1820. /* In DDMA don't need to proceed to starting of next ISOC request */
  1821. if (using_desc_dma(hsotg) && hs_ep->isochronous)
  1822. return;
  1823. /*
  1824. * Look to see if there is anything else to do. Note, the completion
  1825. * of the previous request may have caused a new request to be started
  1826. * so be careful when doing this.
  1827. */
  1828. if (!hs_ep->req && result >= 0)
  1829. dwc2_gadget_start_next_request(hs_ep);
  1830. }
  1831. /*
  1832. * dwc2_gadget_complete_isoc_request_ddma - complete an isoc request in DDMA
  1833. * @hs_ep: The endpoint the request was on.
  1834. *
  1835. * Get first request from the ep queue, determine descriptor on which complete
  1836. * happened. SW discovers which descriptor currently in use by HW, adjusts
  1837. * dma_address and calculates index of completed descriptor based on the value
  1838. * of DEPDMA register. Update actual length of request, giveback to gadget.
  1839. */
  1840. static void dwc2_gadget_complete_isoc_request_ddma(struct dwc2_hsotg_ep *hs_ep)
  1841. {
  1842. struct dwc2_hsotg *hsotg = hs_ep->parent;
  1843. struct dwc2_hsotg_req *hs_req;
  1844. struct usb_request *ureq;
  1845. u32 desc_sts;
  1846. u32 mask;
  1847. desc_sts = hs_ep->desc_list[hs_ep->compl_desc].status;
  1848. /* Process only descriptors with buffer status set to DMA done */
  1849. while ((desc_sts & DEV_DMA_BUFF_STS_MASK) >>
  1850. DEV_DMA_BUFF_STS_SHIFT == DEV_DMA_BUFF_STS_DMADONE) {
  1851. hs_req = get_ep_head(hs_ep);
  1852. if (!hs_req) {
  1853. dev_warn(hsotg->dev, "%s: ISOC EP queue empty\n", __func__);
  1854. return;
  1855. }
  1856. ureq = &hs_req->req;
  1857. /* Check completion status */
  1858. if ((desc_sts & DEV_DMA_STS_MASK) >> DEV_DMA_STS_SHIFT ==
  1859. DEV_DMA_STS_SUCC) {
  1860. mask = hs_ep->dir_in ? DEV_DMA_ISOC_TX_NBYTES_MASK :
  1861. DEV_DMA_ISOC_RX_NBYTES_MASK;
  1862. ureq->actual = ureq->length - ((desc_sts & mask) >>
  1863. DEV_DMA_ISOC_NBYTES_SHIFT);
  1864. /* Adjust actual len for ISOC Out if len is
  1865. * not align of 4
  1866. */
  1867. if (!hs_ep->dir_in && ureq->length & 0x3)
  1868. ureq->actual += 4 - (ureq->length & 0x3);
  1869. /* Set actual frame number for completed transfers */
  1870. ureq->frame_number =
  1871. (desc_sts & DEV_DMA_ISOC_FRNUM_MASK) >>
  1872. DEV_DMA_ISOC_FRNUM_SHIFT;
  1873. }
  1874. dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0);
  1875. hs_ep->compl_desc++;
  1876. if (hs_ep->compl_desc > (MAX_DMA_DESC_NUM_HS_ISOC - 1))
  1877. hs_ep->compl_desc = 0;
  1878. desc_sts = hs_ep->desc_list[hs_ep->compl_desc].status;
  1879. }
  1880. }
  1881. /*
  1882. * dwc2_gadget_handle_isoc_bna - handle BNA interrupt for ISOC.
  1883. * @hs_ep: The isochronous endpoint.
  1884. *
  1885. * If EP ISOC OUT then need to flush RX FIFO to remove source of BNA
  1886. * interrupt. Reset target frame and next_desc to allow to start
  1887. * ISOC's on NAK interrupt for IN direction or on OUTTKNEPDIS
  1888. * interrupt for OUT direction.
  1889. */
  1890. static void dwc2_gadget_handle_isoc_bna(struct dwc2_hsotg_ep *hs_ep)
  1891. {
  1892. struct dwc2_hsotg *hsotg = hs_ep->parent;
  1893. if (!hs_ep->dir_in)
  1894. dwc2_flush_rx_fifo(hsotg);
  1895. dwc2_hsotg_complete_request(hsotg, hs_ep, get_ep_head(hs_ep), 0);
  1896. hs_ep->target_frame = TARGET_FRAME_INITIAL;
  1897. hs_ep->next_desc = 0;
  1898. hs_ep->compl_desc = 0;
  1899. }
  1900. /**
  1901. * dwc2_hsotg_rx_data - receive data from the FIFO for an endpoint
  1902. * @hsotg: The device state.
  1903. * @ep_idx: The endpoint index for the data
  1904. * @size: The size of data in the fifo, in bytes
  1905. *
  1906. * The FIFO status shows there is data to read from the FIFO for a given
  1907. * endpoint, so sort out whether we need to read the data into a request
  1908. * that has been made for that endpoint.
  1909. */
  1910. static void dwc2_hsotg_rx_data(struct dwc2_hsotg *hsotg, int ep_idx, int size)
  1911. {
  1912. struct dwc2_hsotg_ep *hs_ep = hsotg->eps_out[ep_idx];
  1913. struct dwc2_hsotg_req *hs_req = hs_ep->req;
  1914. int to_read;
  1915. int max_req;
  1916. int read_ptr;
  1917. if (!hs_req) {
  1918. u32 epctl = dwc2_readl(hsotg, DOEPCTL(ep_idx));
  1919. int ptr;
  1920. dev_dbg(hsotg->dev,
  1921. "%s: FIFO %d bytes on ep%d but no req (DXEPCTl=0x%08x)\n",
  1922. __func__, size, ep_idx, epctl);
  1923. /* dump the data from the FIFO, we've nothing we can do */
  1924. for (ptr = 0; ptr < size; ptr += 4)
  1925. (void)dwc2_readl(hsotg, EPFIFO(ep_idx));
  1926. return;
  1927. }
  1928. to_read = size;
  1929. read_ptr = hs_req->req.actual;
  1930. max_req = hs_req->req.length - read_ptr;
  1931. dev_dbg(hsotg->dev, "%s: read %d/%d, done %d/%d\n",
  1932. __func__, to_read, max_req, read_ptr, hs_req->req.length);
  1933. if (to_read > max_req) {
  1934. /*
  1935. * more data appeared than we where willing
  1936. * to deal with in this request.
  1937. */
  1938. /* currently we don't deal this */
  1939. WARN_ON_ONCE(1);
  1940. }
  1941. hs_ep->total_data += to_read;
  1942. hs_req->req.actual += to_read;
  1943. to_read = DIV_ROUND_UP(to_read, 4);
  1944. /*
  1945. * note, we might over-write the buffer end by 3 bytes depending on
  1946. * alignment of the data.
  1947. */
  1948. dwc2_readl_rep(hsotg, EPFIFO(ep_idx),
  1949. hs_req->req.buf + read_ptr, to_read);
  1950. }
  1951. /**
  1952. * dwc2_hsotg_ep0_zlp - send/receive zero-length packet on control endpoint
  1953. * @hsotg: The device instance
  1954. * @dir_in: If IN zlp
  1955. *
  1956. * Generate a zero-length IN packet request for terminating a SETUP
  1957. * transaction.
  1958. *
  1959. * Note, since we don't write any data to the TxFIFO, then it is
  1960. * currently believed that we do not need to wait for any space in
  1961. * the TxFIFO.
  1962. */
  1963. static void dwc2_hsotg_ep0_zlp(struct dwc2_hsotg *hsotg, bool dir_in)
  1964. {
  1965. /* eps_out[0] is used in both directions */
  1966. hsotg->eps_out[0]->dir_in = dir_in;
  1967. hsotg->ep0_state = dir_in ? DWC2_EP0_STATUS_IN : DWC2_EP0_STATUS_OUT;
  1968. dwc2_hsotg_program_zlp(hsotg, hsotg->eps_out[0]);
  1969. }
  1970. /*
  1971. * dwc2_gadget_get_xfersize_ddma - get transferred bytes amount from desc
  1972. * @hs_ep - The endpoint on which transfer went
  1973. *
  1974. * Iterate over endpoints descriptor chain and get info on bytes remained
  1975. * in DMA descriptors after transfer has completed. Used for non isoc EPs.
  1976. */
  1977. static unsigned int dwc2_gadget_get_xfersize_ddma(struct dwc2_hsotg_ep *hs_ep)
  1978. {
  1979. const struct usb_endpoint_descriptor *ep_desc = hs_ep->ep.desc;
  1980. struct dwc2_hsotg *hsotg = hs_ep->parent;
  1981. unsigned int bytes_rem = 0;
  1982. unsigned int bytes_rem_correction = 0;
  1983. struct dwc2_dma_desc *desc = hs_ep->desc_list;
  1984. int i;
  1985. u32 status;
  1986. u32 mps = hs_ep->ep.maxpacket;
  1987. int dir_in = hs_ep->dir_in;
  1988. if (!desc)
  1989. return -EINVAL;
  1990. /* Interrupt OUT EP with mps not multiple of 4 */
  1991. if (hs_ep->index)
  1992. if (usb_endpoint_xfer_int(ep_desc) && !dir_in && (mps % 4))
  1993. bytes_rem_correction = 4 - (mps % 4);
  1994. for (i = 0; i < hs_ep->desc_count; ++i) {
  1995. status = desc->status;
  1996. bytes_rem += status & DEV_DMA_NBYTES_MASK;
  1997. bytes_rem -= bytes_rem_correction;
  1998. if (status & DEV_DMA_STS_MASK)
  1999. dev_err(hsotg->dev, "descriptor %d closed with %x\n",
  2000. i, status & DEV_DMA_STS_MASK);
  2001. if (status & DEV_DMA_L)
  2002. break;
  2003. desc++;
  2004. }
  2005. return bytes_rem;
  2006. }
  2007. /**
  2008. * dwc2_hsotg_handle_outdone - handle receiving OutDone/SetupDone from RXFIFO
  2009. * @hsotg: The device instance
  2010. * @epnum: The endpoint received from
  2011. *
  2012. * The RXFIFO has delivered an OutDone event, which means that the data
  2013. * transfer for an OUT endpoint has been completed, either by a short
  2014. * packet or by the finish of a transfer.
  2015. */
  2016. static void dwc2_hsotg_handle_outdone(struct dwc2_hsotg *hsotg, int epnum)
  2017. {
  2018. u32 epsize = dwc2_readl(hsotg, DOEPTSIZ(epnum));
  2019. struct dwc2_hsotg_ep *hs_ep = hsotg->eps_out[epnum];
  2020. struct dwc2_hsotg_req *hs_req = hs_ep->req;
  2021. struct usb_request *req = &hs_req->req;
  2022. unsigned int size_left = DXEPTSIZ_XFERSIZE_GET(epsize);
  2023. int result = 0;
  2024. if (!hs_req) {
  2025. dev_dbg(hsotg->dev, "%s: no request active\n", __func__);
  2026. return;
  2027. }
  2028. if (epnum == 0 && hsotg->ep0_state == DWC2_EP0_STATUS_OUT) {
  2029. dev_dbg(hsotg->dev, "zlp packet received\n");
  2030. dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0);
  2031. dwc2_hsotg_enqueue_setup(hsotg);
  2032. return;
  2033. }
  2034. if (using_desc_dma(hsotg))
  2035. size_left = dwc2_gadget_get_xfersize_ddma(hs_ep);
  2036. if (using_dma(hsotg)) {
  2037. unsigned int size_done;
  2038. /*
  2039. * Calculate the size of the transfer by checking how much
  2040. * is left in the endpoint size register and then working it
  2041. * out from the amount we loaded for the transfer.
  2042. *
  2043. * We need to do this as DMA pointers are always 32bit aligned
  2044. * so may overshoot/undershoot the transfer.
  2045. */
  2046. size_done = hs_ep->size_loaded - size_left;
  2047. size_done += hs_ep->last_load;
  2048. req->actual = size_done;
  2049. }
  2050. /* if there is more request to do, schedule new transfer */
  2051. if (req->actual < req->length && size_left == 0) {
  2052. dwc2_hsotg_start_req(hsotg, hs_ep, hs_req, true);
  2053. return;
  2054. }
  2055. if (req->actual < req->length && req->short_not_ok) {
  2056. dev_dbg(hsotg->dev, "%s: got %d/%d (short not ok) => error\n",
  2057. __func__, req->actual, req->length);
  2058. /*
  2059. * todo - what should we return here? there's no one else
  2060. * even bothering to check the status.
  2061. */
  2062. }
  2063. /* DDMA IN status phase will start from StsPhseRcvd interrupt */
  2064. if (!using_desc_dma(hsotg) && epnum == 0 &&
  2065. hsotg->ep0_state == DWC2_EP0_DATA_OUT) {
  2066. /* Move to STATUS IN */
  2067. if (!hsotg->delayed_status)
  2068. dwc2_hsotg_ep0_zlp(hsotg, true);
  2069. }
  2070. /* Set actual frame number for completed transfers */
  2071. if (!using_desc_dma(hsotg) && hs_ep->isochronous) {
  2072. req->frame_number = hs_ep->target_frame;
  2073. dwc2_gadget_incr_frame_num(hs_ep);
  2074. }
  2075. dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, result);
  2076. }
  2077. /**
  2078. * dwc2_hsotg_handle_rx - RX FIFO has data
  2079. * @hsotg: The device instance
  2080. *
  2081. * The IRQ handler has detected that the RX FIFO has some data in it
  2082. * that requires processing, so find out what is in there and do the
  2083. * appropriate read.
  2084. *
  2085. * The RXFIFO is a true FIFO, the packets coming out are still in packet
  2086. * chunks, so if you have x packets received on an endpoint you'll get x
  2087. * FIFO events delivered, each with a packet's worth of data in it.
  2088. *
  2089. * When using DMA, we should not be processing events from the RXFIFO
  2090. * as the actual data should be sent to the memory directly and we turn
  2091. * on the completion interrupts to get notifications of transfer completion.
  2092. */
  2093. static void dwc2_hsotg_handle_rx(struct dwc2_hsotg *hsotg)
  2094. {
  2095. u32 grxstsr = dwc2_readl(hsotg, GRXSTSP);
  2096. u32 epnum, status, size;
  2097. WARN_ON(using_dma(hsotg));
  2098. epnum = grxstsr & GRXSTS_EPNUM_MASK;
  2099. status = grxstsr & GRXSTS_PKTSTS_MASK;
  2100. size = grxstsr & GRXSTS_BYTECNT_MASK;
  2101. size >>= GRXSTS_BYTECNT_SHIFT;
  2102. dev_dbg(hsotg->dev, "%s: GRXSTSP=0x%08x (%d@%d)\n",
  2103. __func__, grxstsr, size, epnum);
  2104. switch ((status & GRXSTS_PKTSTS_MASK) >> GRXSTS_PKTSTS_SHIFT) {
  2105. case GRXSTS_PKTSTS_GLOBALOUTNAK:
  2106. dev_dbg(hsotg->dev, "GLOBALOUTNAK\n");
  2107. break;
  2108. case GRXSTS_PKTSTS_OUTDONE:
  2109. dev_dbg(hsotg->dev, "OutDone (Frame=0x%08x)\n",
  2110. dwc2_hsotg_read_frameno(hsotg));
  2111. if (!using_dma(hsotg))
  2112. dwc2_hsotg_handle_outdone(hsotg, epnum);
  2113. break;
  2114. case GRXSTS_PKTSTS_SETUPDONE:
  2115. dev_dbg(hsotg->dev,
  2116. "SetupDone (Frame=0x%08x, DOPEPCTL=0x%08x)\n",
  2117. dwc2_hsotg_read_frameno(hsotg),
  2118. dwc2_readl(hsotg, DOEPCTL(0)));
  2119. /*
  2120. * Call dwc2_hsotg_handle_outdone here if it was not called from
  2121. * GRXSTS_PKTSTS_OUTDONE. That is, if the core didn't
  2122. * generate GRXSTS_PKTSTS_OUTDONE for setup packet.
  2123. */
  2124. if (hsotg->ep0_state == DWC2_EP0_SETUP)
  2125. dwc2_hsotg_handle_outdone(hsotg, epnum);
  2126. break;
  2127. case GRXSTS_PKTSTS_OUTRX:
  2128. dwc2_hsotg_rx_data(hsotg, epnum, size);
  2129. break;
  2130. case GRXSTS_PKTSTS_SETUPRX:
  2131. dev_dbg(hsotg->dev,
  2132. "SetupRX (Frame=0x%08x, DOPEPCTL=0x%08x)\n",
  2133. dwc2_hsotg_read_frameno(hsotg),
  2134. dwc2_readl(hsotg, DOEPCTL(0)));
  2135. WARN_ON(hsotg->ep0_state != DWC2_EP0_SETUP);
  2136. dwc2_hsotg_rx_data(hsotg, epnum, size);
  2137. break;
  2138. default:
  2139. dev_warn(hsotg->dev, "%s: unknown status %08x\n",
  2140. __func__, grxstsr);
  2141. dwc2_hsotg_dump(hsotg);
  2142. break;
  2143. }
  2144. }
  2145. /**
  2146. * dwc2_hsotg_ep0_mps - turn max packet size into register setting
  2147. * @mps: The maximum packet size in bytes.
  2148. */
  2149. static u32 dwc2_hsotg_ep0_mps(unsigned int mps)
  2150. {
  2151. switch (mps) {
  2152. case 64:
  2153. return D0EPCTL_MPS_64;
  2154. case 32:
  2155. return D0EPCTL_MPS_32;
  2156. case 16:
  2157. return D0EPCTL_MPS_16;
  2158. case 8:
  2159. return D0EPCTL_MPS_8;
  2160. }
  2161. /* bad max packet size, warn and return invalid result */
  2162. WARN_ON(1);
  2163. return (u32)-1;
  2164. }
  2165. /**
  2166. * dwc2_hsotg_set_ep_maxpacket - set endpoint's max-packet field
  2167. * @hsotg: The driver state.
  2168. * @ep: The index number of the endpoint
  2169. * @mps: The maximum packet size in bytes
  2170. * @mc: The multicount value
  2171. * @dir_in: True if direction is in.
  2172. *
  2173. * Configure the maximum packet size for the given endpoint, updating
  2174. * the hardware control registers to reflect this.
  2175. */
  2176. static void dwc2_hsotg_set_ep_maxpacket(struct dwc2_hsotg *hsotg,
  2177. unsigned int ep, unsigned int mps,
  2178. unsigned int mc, unsigned int dir_in)
  2179. {
  2180. struct dwc2_hsotg_ep *hs_ep;
  2181. u32 reg;
  2182. hs_ep = index_to_ep(hsotg, ep, dir_in);
  2183. if (!hs_ep)
  2184. return;
  2185. if (ep == 0) {
  2186. u32 mps_bytes = mps;
  2187. /* EP0 is a special case */
  2188. mps = dwc2_hsotg_ep0_mps(mps_bytes);
  2189. if (mps > 3)
  2190. goto bad_mps;
  2191. hs_ep->ep.maxpacket = mps_bytes;
  2192. hs_ep->mc = 1;
  2193. } else {
  2194. if (mps > 1024)
  2195. goto bad_mps;
  2196. hs_ep->mc = mc;
  2197. if (mc > 3)
  2198. goto bad_mps;
  2199. hs_ep->ep.maxpacket = mps;
  2200. }
  2201. if (dir_in) {
  2202. reg = dwc2_readl(hsotg, DIEPCTL(ep));
  2203. reg &= ~DXEPCTL_MPS_MASK;
  2204. reg |= mps;
  2205. dwc2_writel(hsotg, reg, DIEPCTL(ep));
  2206. } else {
  2207. reg = dwc2_readl(hsotg, DOEPCTL(ep));
  2208. reg &= ~DXEPCTL_MPS_MASK;
  2209. reg |= mps;
  2210. dwc2_writel(hsotg, reg, DOEPCTL(ep));
  2211. }
  2212. return;
  2213. bad_mps:
  2214. dev_err(hsotg->dev, "ep%d: bad mps of %d\n", ep, mps);
  2215. }
  2216. /**
  2217. * dwc2_hsotg_txfifo_flush - flush Tx FIFO
  2218. * @hsotg: The driver state
  2219. * @idx: The index for the endpoint (0..15)
  2220. */
  2221. static void dwc2_hsotg_txfifo_flush(struct dwc2_hsotg *hsotg, unsigned int idx)
  2222. {
  2223. dwc2_writel(hsotg, GRSTCTL_TXFNUM(idx) | GRSTCTL_TXFFLSH,
  2224. GRSTCTL);
  2225. /* wait until the fifo is flushed */
  2226. if (dwc2_hsotg_wait_bit_clear(hsotg, GRSTCTL, GRSTCTL_TXFFLSH, 100))
  2227. dev_warn(hsotg->dev, "%s: timeout flushing fifo GRSTCTL_TXFFLSH\n",
  2228. __func__);
  2229. }
  2230. /**
  2231. * dwc2_hsotg_trytx - check to see if anything needs transmitting
  2232. * @hsotg: The driver state
  2233. * @hs_ep: The driver endpoint to check.
  2234. *
  2235. * Check to see if there is a request that has data to send, and if so
  2236. * make an attempt to write data into the FIFO.
  2237. */
  2238. static int dwc2_hsotg_trytx(struct dwc2_hsotg *hsotg,
  2239. struct dwc2_hsotg_ep *hs_ep)
  2240. {
  2241. struct dwc2_hsotg_req *hs_req = hs_ep->req;
  2242. if (!hs_ep->dir_in || !hs_req) {
  2243. /**
  2244. * if request is not enqueued, we disable interrupts
  2245. * for endpoints, excepting ep0
  2246. */
  2247. if (hs_ep->index != 0)
  2248. dwc2_hsotg_ctrl_epint(hsotg, hs_ep->index,
  2249. hs_ep->dir_in, 0);
  2250. return 0;
  2251. }
  2252. if (hs_req->req.actual < hs_req->req.length) {
  2253. dev_dbg(hsotg->dev, "trying to write more for ep%d\n",
  2254. hs_ep->index);
  2255. return dwc2_hsotg_write_fifo(hsotg, hs_ep, hs_req);
  2256. }
  2257. return 0;
  2258. }
  2259. /**
  2260. * dwc2_hsotg_complete_in - complete IN transfer
  2261. * @hsotg: The device state.
  2262. * @hs_ep: The endpoint that has just completed.
  2263. *
  2264. * An IN transfer has been completed, update the transfer's state and then
  2265. * call the relevant completion routines.
  2266. */
  2267. static void dwc2_hsotg_complete_in(struct dwc2_hsotg *hsotg,
  2268. struct dwc2_hsotg_ep *hs_ep)
  2269. {
  2270. struct dwc2_hsotg_req *hs_req = hs_ep->req;
  2271. u32 epsize = dwc2_readl(hsotg, DIEPTSIZ(hs_ep->index));
  2272. int size_left, size_done;
  2273. if (!hs_req) {
  2274. dev_dbg(hsotg->dev, "XferCompl but no req\n");
  2275. return;
  2276. }
  2277. /* Finish ZLP handling for IN EP0 transactions */
  2278. if (hs_ep->index == 0 && hsotg->ep0_state == DWC2_EP0_STATUS_IN) {
  2279. dev_dbg(hsotg->dev, "zlp packet sent\n");
  2280. /*
  2281. * While send zlp for DWC2_EP0_STATUS_IN EP direction was
  2282. * changed to IN. Change back to complete OUT transfer request
  2283. */
  2284. hs_ep->dir_in = 0;
  2285. dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0);
  2286. if (hsotg->test_mode) {
  2287. int ret;
  2288. ret = dwc2_hsotg_set_test_mode(hsotg, hsotg->test_mode);
  2289. if (ret < 0) {
  2290. dev_dbg(hsotg->dev, "Invalid Test #%d\n",
  2291. hsotg->test_mode);
  2292. dwc2_hsotg_stall_ep0(hsotg);
  2293. return;
  2294. }
  2295. }
  2296. dwc2_hsotg_enqueue_setup(hsotg);
  2297. return;
  2298. }
  2299. /*
  2300. * Calculate the size of the transfer by checking how much is left
  2301. * in the endpoint size register and then working it out from
  2302. * the amount we loaded for the transfer.
  2303. *
  2304. * We do this even for DMA, as the transfer may have incremented
  2305. * past the end of the buffer (DMA transfers are always 32bit
  2306. * aligned).
  2307. */
  2308. if (using_desc_dma(hsotg)) {
  2309. size_left = dwc2_gadget_get_xfersize_ddma(hs_ep);
  2310. if (size_left < 0)
  2311. dev_err(hsotg->dev, "error parsing DDMA results %d\n",
  2312. size_left);
  2313. } else {
  2314. size_left = DXEPTSIZ_XFERSIZE_GET(epsize);
  2315. }
  2316. size_done = hs_ep->size_loaded - size_left;
  2317. size_done += hs_ep->last_load;
  2318. if (hs_req->req.actual != size_done)
  2319. dev_dbg(hsotg->dev, "%s: adjusting size done %d => %d\n",
  2320. __func__, hs_req->req.actual, size_done);
  2321. hs_req->req.actual = size_done;
  2322. dev_dbg(hsotg->dev, "req->length:%d req->actual:%d req->zero:%d\n",
  2323. hs_req->req.length, hs_req->req.actual, hs_req->req.zero);
  2324. if (!size_left && hs_req->req.actual < hs_req->req.length) {
  2325. dev_dbg(hsotg->dev, "%s trying more for req...\n", __func__);
  2326. dwc2_hsotg_start_req(hsotg, hs_ep, hs_req, true);
  2327. return;
  2328. }
  2329. /* Zlp for all endpoints in non DDMA, for ep0 only in DATA IN stage */
  2330. if (hs_ep->send_zlp) {
  2331. hs_ep->send_zlp = 0;
  2332. if (!using_desc_dma(hsotg)) {
  2333. dwc2_hsotg_program_zlp(hsotg, hs_ep);
  2334. /* transfer will be completed on next complete interrupt */
  2335. return;
  2336. }
  2337. }
  2338. if (hs_ep->index == 0 && hsotg->ep0_state == DWC2_EP0_DATA_IN) {
  2339. /* Move to STATUS OUT */
  2340. dwc2_hsotg_ep0_zlp(hsotg, false);
  2341. return;
  2342. }
  2343. /* Set actual frame number for completed transfers */
  2344. if (!using_desc_dma(hsotg) && hs_ep->isochronous) {
  2345. hs_req->req.frame_number = hs_ep->target_frame;
  2346. dwc2_gadget_incr_frame_num(hs_ep);
  2347. }
  2348. dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0);
  2349. }
  2350. /**
  2351. * dwc2_gadget_read_ep_interrupts - reads interrupts for given ep
  2352. * @hsotg: The device state.
  2353. * @idx: Index of ep.
  2354. * @dir_in: Endpoint direction 1-in 0-out.
  2355. *
  2356. * Reads for endpoint with given index and direction, by masking
  2357. * epint_reg with coresponding mask.
  2358. */
  2359. static u32 dwc2_gadget_read_ep_interrupts(struct dwc2_hsotg *hsotg,
  2360. unsigned int idx, int dir_in)
  2361. {
  2362. u32 epmsk_reg = dir_in ? DIEPMSK : DOEPMSK;
  2363. u32 epint_reg = dir_in ? DIEPINT(idx) : DOEPINT(idx);
  2364. u32 ints;
  2365. u32 mask;
  2366. u32 diepempmsk;
  2367. mask = dwc2_readl(hsotg, epmsk_reg);
  2368. diepempmsk = dwc2_readl(hsotg, DIEPEMPMSK);
  2369. mask |= ((diepempmsk >> idx) & 0x1) ? DIEPMSK_TXFIFOEMPTY : 0;
  2370. mask |= DXEPINT_SETUP_RCVD;
  2371. ints = dwc2_readl(hsotg, epint_reg);
  2372. ints &= mask;
  2373. return ints;
  2374. }
  2375. /**
  2376. * dwc2_gadget_handle_ep_disabled - handle DXEPINT_EPDISBLD
  2377. * @hs_ep: The endpoint on which interrupt is asserted.
  2378. *
  2379. * This interrupt indicates that the endpoint has been disabled per the
  2380. * application's request.
  2381. *
  2382. * For IN endpoints flushes txfifo, in case of BULK clears DCTL_CGNPINNAK,
  2383. * in case of ISOC completes current request.
  2384. *
  2385. * For ISOC-OUT endpoints completes expired requests. If there is remaining
  2386. * request starts it.
  2387. */
  2388. static void dwc2_gadget_handle_ep_disabled(struct dwc2_hsotg_ep *hs_ep)
  2389. {
  2390. struct dwc2_hsotg *hsotg = hs_ep->parent;
  2391. struct dwc2_hsotg_req *hs_req;
  2392. unsigned char idx = hs_ep->index;
  2393. int dir_in = hs_ep->dir_in;
  2394. u32 epctl_reg = dir_in ? DIEPCTL(idx) : DOEPCTL(idx);
  2395. int dctl = dwc2_readl(hsotg, DCTL);
  2396. dev_dbg(hsotg->dev, "%s: EPDisbld\n", __func__);
  2397. if (dir_in) {
  2398. int epctl = dwc2_readl(hsotg, epctl_reg);
  2399. dwc2_hsotg_txfifo_flush(hsotg, hs_ep->fifo_index);
  2400. if ((epctl & DXEPCTL_STALL) && (epctl & DXEPCTL_EPTYPE_BULK)) {
  2401. int dctl = dwc2_readl(hsotg, DCTL);
  2402. dctl |= DCTL_CGNPINNAK;
  2403. dwc2_writel(hsotg, dctl, DCTL);
  2404. }
  2405. } else {
  2406. if (dctl & DCTL_GOUTNAKSTS) {
  2407. dctl |= DCTL_CGOUTNAK;
  2408. dwc2_writel(hsotg, dctl, DCTL);
  2409. }
  2410. }
  2411. if (!hs_ep->isochronous)
  2412. return;
  2413. if (list_empty(&hs_ep->queue)) {
  2414. dev_dbg(hsotg->dev, "%s: complete_ep 0x%p, ep->queue empty!\n",
  2415. __func__, hs_ep);
  2416. return;
  2417. }
  2418. do {
  2419. hs_req = get_ep_head(hs_ep);
  2420. if (hs_req) {
  2421. hs_req->req.frame_number = hs_ep->target_frame;
  2422. hs_req->req.actual = 0;
  2423. dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req,
  2424. -ENODATA);
  2425. }
  2426. dwc2_gadget_incr_frame_num(hs_ep);
  2427. /* Update current frame number value. */
  2428. hsotg->frame_number = dwc2_hsotg_read_frameno(hsotg);
  2429. } while (dwc2_gadget_target_frame_elapsed(hs_ep));
  2430. }
  2431. /**
  2432. * dwc2_gadget_handle_out_token_ep_disabled - handle DXEPINT_OUTTKNEPDIS
  2433. * @ep: The endpoint on which interrupt is asserted.
  2434. *
  2435. * This is starting point for ISOC-OUT transfer, synchronization done with
  2436. * first out token received from host while corresponding EP is disabled.
  2437. *
  2438. * Device does not know initial frame in which out token will come. For this
  2439. * HW generates OUTTKNEPDIS - out token is received while EP is disabled. Upon
  2440. * getting this interrupt SW starts calculation for next transfer frame.
  2441. */
  2442. static void dwc2_gadget_handle_out_token_ep_disabled(struct dwc2_hsotg_ep *ep)
  2443. {
  2444. struct dwc2_hsotg *hsotg = ep->parent;
  2445. struct dwc2_hsotg_req *hs_req;
  2446. int dir_in = ep->dir_in;
  2447. if (dir_in || !ep->isochronous)
  2448. return;
  2449. if (using_desc_dma(hsotg)) {
  2450. if (ep->target_frame == TARGET_FRAME_INITIAL) {
  2451. /* Start first ISO Out */
  2452. ep->target_frame = hsotg->frame_number;
  2453. dwc2_gadget_start_isoc_ddma(ep);
  2454. }
  2455. return;
  2456. }
  2457. if (ep->target_frame == TARGET_FRAME_INITIAL) {
  2458. u32 ctrl;
  2459. ep->target_frame = hsotg->frame_number;
  2460. if (ep->interval > 1) {
  2461. ctrl = dwc2_readl(hsotg, DOEPCTL(ep->index));
  2462. if (ep->target_frame & 0x1)
  2463. ctrl |= DXEPCTL_SETODDFR;
  2464. else
  2465. ctrl |= DXEPCTL_SETEVENFR;
  2466. dwc2_writel(hsotg, ctrl, DOEPCTL(ep->index));
  2467. }
  2468. }
  2469. while (dwc2_gadget_target_frame_elapsed(ep)) {
  2470. hs_req = get_ep_head(ep);
  2471. if (hs_req) {
  2472. hs_req->req.frame_number = ep->target_frame;
  2473. hs_req->req.actual = 0;
  2474. dwc2_hsotg_complete_request(hsotg, ep, hs_req, -ENODATA);
  2475. }
  2476. dwc2_gadget_incr_frame_num(ep);
  2477. /* Update current frame number value. */
  2478. hsotg->frame_number = dwc2_hsotg_read_frameno(hsotg);
  2479. }
  2480. if (!ep->req)
  2481. dwc2_gadget_start_next_request(ep);
  2482. }
  2483. static void dwc2_hsotg_ep_stop_xfr(struct dwc2_hsotg *hsotg,
  2484. struct dwc2_hsotg_ep *hs_ep);
  2485. /**
  2486. * dwc2_gadget_handle_nak - handle NAK interrupt
  2487. * @hs_ep: The endpoint on which interrupt is asserted.
  2488. *
  2489. * This is starting point for ISOC-IN transfer, synchronization done with
  2490. * first IN token received from host while corresponding EP is disabled.
  2491. *
  2492. * Device does not know when first one token will arrive from host. On first
  2493. * token arrival HW generates 2 interrupts: 'in token received while FIFO empty'
  2494. * and 'NAK'. NAK interrupt for ISOC-IN means that token has arrived and ZLP was
  2495. * sent in response to that as there was no data in FIFO. SW is basing on this
  2496. * interrupt to obtain frame in which token has come and then based on the
  2497. * interval calculates next frame for transfer.
  2498. */
  2499. static void dwc2_gadget_handle_nak(struct dwc2_hsotg_ep *hs_ep)
  2500. {
  2501. struct dwc2_hsotg *hsotg = hs_ep->parent;
  2502. struct dwc2_hsotg_req *hs_req;
  2503. int dir_in = hs_ep->dir_in;
  2504. u32 ctrl;
  2505. if (!dir_in || !hs_ep->isochronous)
  2506. return;
  2507. if (hs_ep->target_frame == TARGET_FRAME_INITIAL) {
  2508. if (using_desc_dma(hsotg)) {
  2509. hs_ep->target_frame = hsotg->frame_number;
  2510. dwc2_gadget_incr_frame_num(hs_ep);
  2511. /* In service interval mode target_frame must
  2512. * be set to last (u)frame of the service interval.
  2513. */
  2514. if (hsotg->params.service_interval) {
  2515. /* Set target_frame to the first (u)frame of
  2516. * the service interval
  2517. */
  2518. hs_ep->target_frame &= ~hs_ep->interval + 1;
  2519. /* Set target_frame to the last (u)frame of
  2520. * the service interval
  2521. */
  2522. dwc2_gadget_incr_frame_num(hs_ep);
  2523. dwc2_gadget_dec_frame_num_by_one(hs_ep);
  2524. }
  2525. dwc2_gadget_start_isoc_ddma(hs_ep);
  2526. return;
  2527. }
  2528. hs_ep->target_frame = hsotg->frame_number;
  2529. if (hs_ep->interval > 1) {
  2530. u32 ctrl = dwc2_readl(hsotg,
  2531. DIEPCTL(hs_ep->index));
  2532. if (hs_ep->target_frame & 0x1)
  2533. ctrl |= DXEPCTL_SETODDFR;
  2534. else
  2535. ctrl |= DXEPCTL_SETEVENFR;
  2536. dwc2_writel(hsotg, ctrl, DIEPCTL(hs_ep->index));
  2537. }
  2538. }
  2539. if (using_desc_dma(hsotg))
  2540. return;
  2541. ctrl = dwc2_readl(hsotg, DIEPCTL(hs_ep->index));
  2542. if (ctrl & DXEPCTL_EPENA)
  2543. dwc2_hsotg_ep_stop_xfr(hsotg, hs_ep);
  2544. else
  2545. dwc2_hsotg_txfifo_flush(hsotg, hs_ep->fifo_index);
  2546. while (dwc2_gadget_target_frame_elapsed(hs_ep)) {
  2547. hs_req = get_ep_head(hs_ep);
  2548. if (hs_req) {
  2549. hs_req->req.frame_number = hs_ep->target_frame;
  2550. hs_req->req.actual = 0;
  2551. dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, -ENODATA);
  2552. }
  2553. dwc2_gadget_incr_frame_num(hs_ep);
  2554. /* Update current frame number value. */
  2555. hsotg->frame_number = dwc2_hsotg_read_frameno(hsotg);
  2556. }
  2557. if (!hs_ep->req)
  2558. dwc2_gadget_start_next_request(hs_ep);
  2559. }
  2560. /**
  2561. * dwc2_hsotg_epint - handle an in/out endpoint interrupt
  2562. * @hsotg: The driver state
  2563. * @idx: The index for the endpoint (0..15)
  2564. * @dir_in: Set if this is an IN endpoint
  2565. *
  2566. * Process and clear any interrupt pending for an individual endpoint
  2567. */
  2568. static void dwc2_hsotg_epint(struct dwc2_hsotg *hsotg, unsigned int idx,
  2569. int dir_in)
  2570. {
  2571. struct dwc2_hsotg_ep *hs_ep = index_to_ep(hsotg, idx, dir_in);
  2572. u32 epint_reg = dir_in ? DIEPINT(idx) : DOEPINT(idx);
  2573. u32 epctl_reg = dir_in ? DIEPCTL(idx) : DOEPCTL(idx);
  2574. u32 epsiz_reg = dir_in ? DIEPTSIZ(idx) : DOEPTSIZ(idx);
  2575. u32 ints;
  2576. ints = dwc2_gadget_read_ep_interrupts(hsotg, idx, dir_in);
  2577. /* Clear endpoint interrupts */
  2578. dwc2_writel(hsotg, ints, epint_reg);
  2579. if (!hs_ep) {
  2580. dev_err(hsotg->dev, "%s:Interrupt for unconfigured ep%d(%s)\n",
  2581. __func__, idx, dir_in ? "in" : "out");
  2582. return;
  2583. }
  2584. dev_dbg(hsotg->dev, "%s: ep%d(%s) DxEPINT=0x%08x\n",
  2585. __func__, idx, dir_in ? "in" : "out", ints);
  2586. /* Don't process XferCompl interrupt if it is a setup packet */
  2587. if (idx == 0 && (ints & (DXEPINT_SETUP | DXEPINT_SETUP_RCVD)))
  2588. ints &= ~DXEPINT_XFERCOMPL;
  2589. /*
  2590. * Don't process XferCompl interrupt in DDMA if EP0 is still in SETUP
  2591. * stage and xfercomplete was generated without SETUP phase done
  2592. * interrupt. SW should parse received setup packet only after host's
  2593. * exit from setup phase of control transfer.
  2594. */
  2595. if (using_desc_dma(hsotg) && idx == 0 && !hs_ep->dir_in &&
  2596. hsotg->ep0_state == DWC2_EP0_SETUP && !(ints & DXEPINT_SETUP))
  2597. ints &= ~DXEPINT_XFERCOMPL;
  2598. if (ints & DXEPINT_XFERCOMPL) {
  2599. dev_dbg(hsotg->dev,
  2600. "%s: XferCompl: DxEPCTL=0x%08x, DXEPTSIZ=%08x\n",
  2601. __func__, dwc2_readl(hsotg, epctl_reg),
  2602. dwc2_readl(hsotg, epsiz_reg));
  2603. /* In DDMA handle isochronous requests separately */
  2604. if (using_desc_dma(hsotg) && hs_ep->isochronous) {
  2605. dwc2_gadget_complete_isoc_request_ddma(hs_ep);
  2606. } else if (dir_in) {
  2607. /*
  2608. * We get OutDone from the FIFO, so we only
  2609. * need to look at completing IN requests here
  2610. * if operating slave mode
  2611. */
  2612. if (!hs_ep->isochronous || !(ints & DXEPINT_NAKINTRPT))
  2613. dwc2_hsotg_complete_in(hsotg, hs_ep);
  2614. if (idx == 0 && !hs_ep->req)
  2615. dwc2_hsotg_enqueue_setup(hsotg);
  2616. } else if (using_dma(hsotg)) {
  2617. /*
  2618. * We're using DMA, we need to fire an OutDone here
  2619. * as we ignore the RXFIFO.
  2620. */
  2621. if (!hs_ep->isochronous || !(ints & DXEPINT_OUTTKNEPDIS))
  2622. dwc2_hsotg_handle_outdone(hsotg, idx);
  2623. }
  2624. }
  2625. if (ints & DXEPINT_EPDISBLD)
  2626. dwc2_gadget_handle_ep_disabled(hs_ep);
  2627. if (ints & DXEPINT_OUTTKNEPDIS)
  2628. dwc2_gadget_handle_out_token_ep_disabled(hs_ep);
  2629. if (ints & DXEPINT_NAKINTRPT)
  2630. dwc2_gadget_handle_nak(hs_ep);
  2631. if (ints & DXEPINT_AHBERR)
  2632. dev_dbg(hsotg->dev, "%s: AHBErr\n", __func__);
  2633. if (ints & DXEPINT_SETUP) { /* Setup or Timeout */
  2634. dev_dbg(hsotg->dev, "%s: Setup/Timeout\n", __func__);
  2635. if (using_dma(hsotg) && idx == 0) {
  2636. /*
  2637. * this is the notification we've received a
  2638. * setup packet. In non-DMA mode we'd get this
  2639. * from the RXFIFO, instead we need to process
  2640. * the setup here.
  2641. */
  2642. if (dir_in)
  2643. WARN_ON_ONCE(1);
  2644. else
  2645. dwc2_hsotg_handle_outdone(hsotg, 0);
  2646. }
  2647. }
  2648. if (ints & DXEPINT_STSPHSERCVD) {
  2649. dev_dbg(hsotg->dev, "%s: StsPhseRcvd\n", __func__);
  2650. /* Safety check EP0 state when STSPHSERCVD asserted */
  2651. if (hsotg->ep0_state == DWC2_EP0_DATA_OUT) {
  2652. /* Move to STATUS IN for DDMA */
  2653. if (using_desc_dma(hsotg)) {
  2654. if (!hsotg->delayed_status)
  2655. dwc2_hsotg_ep0_zlp(hsotg, true);
  2656. else
  2657. /* In case of 3 stage Control Write with delayed
  2658. * status, when Status IN transfer started
  2659. * before STSPHSERCVD asserted, NAKSTS bit not
  2660. * cleared by CNAK in dwc2_hsotg_start_req()
  2661. * function. Clear now NAKSTS to allow complete
  2662. * transfer.
  2663. */
  2664. dwc2_set_bit(hsotg, DIEPCTL(0),
  2665. DXEPCTL_CNAK);
  2666. }
  2667. }
  2668. }
  2669. if (ints & DXEPINT_BACK2BACKSETUP)
  2670. dev_dbg(hsotg->dev, "%s: B2BSetup/INEPNakEff\n", __func__);
  2671. if (ints & DXEPINT_BNAINTR) {
  2672. dev_dbg(hsotg->dev, "%s: BNA interrupt\n", __func__);
  2673. if (hs_ep->isochronous)
  2674. dwc2_gadget_handle_isoc_bna(hs_ep);
  2675. }
  2676. if (dir_in && !hs_ep->isochronous) {
  2677. /* not sure if this is important, but we'll clear it anyway */
  2678. if (ints & DXEPINT_INTKNTXFEMP) {
  2679. dev_dbg(hsotg->dev, "%s: ep%d: INTknTXFEmpMsk\n",
  2680. __func__, idx);
  2681. }
  2682. /* this probably means something bad is happening */
  2683. if (ints & DXEPINT_INTKNEPMIS) {
  2684. dev_warn(hsotg->dev, "%s: ep%d: INTknEP\n",
  2685. __func__, idx);
  2686. }
  2687. /* FIFO has space or is empty (see GAHBCFG) */
  2688. if (hsotg->dedicated_fifos &&
  2689. ints & DXEPINT_TXFEMP) {
  2690. dev_dbg(hsotg->dev, "%s: ep%d: TxFIFOEmpty\n",
  2691. __func__, idx);
  2692. if (!using_dma(hsotg))
  2693. dwc2_hsotg_trytx(hsotg, hs_ep);
  2694. }
  2695. }
  2696. }
  2697. /**
  2698. * dwc2_hsotg_irq_enumdone - Handle EnumDone interrupt (enumeration done)
  2699. * @hsotg: The device state.
  2700. *
  2701. * Handle updating the device settings after the enumeration phase has
  2702. * been completed.
  2703. */
  2704. static void dwc2_hsotg_irq_enumdone(struct dwc2_hsotg *hsotg)
  2705. {
  2706. u32 dsts = dwc2_readl(hsotg, DSTS);
  2707. int ep0_mps = 0, ep_mps = 8;
  2708. /*
  2709. * This should signal the finish of the enumeration phase
  2710. * of the USB handshaking, so we should now know what rate
  2711. * we connected at.
  2712. */
  2713. dev_dbg(hsotg->dev, "EnumDone (DSTS=0x%08x)\n", dsts);
  2714. /*
  2715. * note, since we're limited by the size of transfer on EP0, and
  2716. * it seems IN transfers must be a even number of packets we do
  2717. * not advertise a 64byte MPS on EP0.
  2718. */
  2719. /* catch both EnumSpd_FS and EnumSpd_FS48 */
  2720. switch ((dsts & DSTS_ENUMSPD_MASK) >> DSTS_ENUMSPD_SHIFT) {
  2721. case DSTS_ENUMSPD_FS:
  2722. case DSTS_ENUMSPD_FS48:
  2723. hsotg->gadget.speed = USB_SPEED_FULL;
  2724. ep0_mps = EP0_MPS_LIMIT;
  2725. ep_mps = 1023;
  2726. break;
  2727. case DSTS_ENUMSPD_HS:
  2728. hsotg->gadget.speed = USB_SPEED_HIGH;
  2729. ep0_mps = EP0_MPS_LIMIT;
  2730. ep_mps = 1024;
  2731. break;
  2732. case DSTS_ENUMSPD_LS:
  2733. hsotg->gadget.speed = USB_SPEED_LOW;
  2734. ep0_mps = 8;
  2735. ep_mps = 8;
  2736. /*
  2737. * note, we don't actually support LS in this driver at the
  2738. * moment, and the documentation seems to imply that it isn't
  2739. * supported by the PHYs on some of the devices.
  2740. */
  2741. break;
  2742. }
  2743. dev_info(hsotg->dev, "new device is %s\n",
  2744. usb_speed_string(hsotg->gadget.speed));
  2745. /*
  2746. * we should now know the maximum packet size for an
  2747. * endpoint, so set the endpoints to a default value.
  2748. */
  2749. if (ep0_mps) {
  2750. int i;
  2751. /* Initialize ep0 for both in and out directions */
  2752. dwc2_hsotg_set_ep_maxpacket(hsotg, 0, ep0_mps, 0, 1);
  2753. dwc2_hsotg_set_ep_maxpacket(hsotg, 0, ep0_mps, 0, 0);
  2754. for (i = 1; i < hsotg->num_of_eps; i++) {
  2755. if (hsotg->eps_in[i])
  2756. dwc2_hsotg_set_ep_maxpacket(hsotg, i, ep_mps,
  2757. 0, 1);
  2758. if (hsotg->eps_out[i])
  2759. dwc2_hsotg_set_ep_maxpacket(hsotg, i, ep_mps,
  2760. 0, 0);
  2761. }
  2762. }
  2763. /* ensure after enumeration our EP0 is active */
  2764. dwc2_hsotg_enqueue_setup(hsotg);
  2765. dev_dbg(hsotg->dev, "EP0: DIEPCTL0=0x%08x, DOEPCTL0=0x%08x\n",
  2766. dwc2_readl(hsotg, DIEPCTL0),
  2767. dwc2_readl(hsotg, DOEPCTL0));
  2768. }
  2769. /**
  2770. * kill_all_requests - remove all requests from the endpoint's queue
  2771. * @hsotg: The device state.
  2772. * @ep: The endpoint the requests may be on.
  2773. * @result: The result code to use.
  2774. *
  2775. * Go through the requests on the given endpoint and mark them
  2776. * completed with the given result code.
  2777. */
  2778. static void kill_all_requests(struct dwc2_hsotg *hsotg,
  2779. struct dwc2_hsotg_ep *ep,
  2780. int result)
  2781. {
  2782. unsigned int size;
  2783. ep->req = NULL;
  2784. while (!list_empty(&ep->queue)) {
  2785. struct dwc2_hsotg_req *req = get_ep_head(ep);
  2786. dwc2_hsotg_complete_request(hsotg, ep, req, result);
  2787. }
  2788. if (!hsotg->dedicated_fifos)
  2789. return;
  2790. size = (dwc2_readl(hsotg, DTXFSTS(ep->fifo_index)) & 0xffff) * 4;
  2791. if (size < ep->fifo_size)
  2792. dwc2_hsotg_txfifo_flush(hsotg, ep->fifo_index);
  2793. }
  2794. /**
  2795. * dwc2_hsotg_disconnect - disconnect service
  2796. * @hsotg: The device state.
  2797. *
  2798. * The device has been disconnected. Remove all current
  2799. * transactions and signal the gadget driver that this
  2800. * has happened.
  2801. */
  2802. void dwc2_hsotg_disconnect(struct dwc2_hsotg *hsotg)
  2803. {
  2804. unsigned int ep;
  2805. if (!hsotg->connected)
  2806. return;
  2807. hsotg->connected = 0;
  2808. hsotg->test_mode = 0;
  2809. /* all endpoints should be shutdown */
  2810. for (ep = 0; ep < hsotg->num_of_eps; ep++) {
  2811. if (hsotg->eps_in[ep])
  2812. kill_all_requests(hsotg, hsotg->eps_in[ep],
  2813. -ESHUTDOWN);
  2814. if (hsotg->eps_out[ep])
  2815. kill_all_requests(hsotg, hsotg->eps_out[ep],
  2816. -ESHUTDOWN);
  2817. }
  2818. call_gadget(hsotg, disconnect);
  2819. hsotg->lx_state = DWC2_L3;
  2820. usb_gadget_set_state(&hsotg->gadget, USB_STATE_NOTATTACHED);
  2821. }
  2822. /**
  2823. * dwc2_hsotg_irq_fifoempty - TX FIFO empty interrupt handler
  2824. * @hsotg: The device state:
  2825. * @periodic: True if this is a periodic FIFO interrupt
  2826. */
  2827. static void dwc2_hsotg_irq_fifoempty(struct dwc2_hsotg *hsotg, bool periodic)
  2828. {
  2829. struct dwc2_hsotg_ep *ep;
  2830. int epno, ret;
  2831. /* look through for any more data to transmit */
  2832. for (epno = 0; epno < hsotg->num_of_eps; epno++) {
  2833. ep = index_to_ep(hsotg, epno, 1);
  2834. if (!ep)
  2835. continue;
  2836. if (!ep->dir_in)
  2837. continue;
  2838. if ((periodic && !ep->periodic) ||
  2839. (!periodic && ep->periodic))
  2840. continue;
  2841. ret = dwc2_hsotg_trytx(hsotg, ep);
  2842. if (ret < 0)
  2843. break;
  2844. }
  2845. }
  2846. /* IRQ flags which will trigger a retry around the IRQ loop */
  2847. #define IRQ_RETRY_MASK (GINTSTS_NPTXFEMP | \
  2848. GINTSTS_PTXFEMP | \
  2849. GINTSTS_RXFLVL)
  2850. static int dwc2_hsotg_ep_disable(struct usb_ep *ep);
  2851. /**
  2852. * dwc2_hsotg_core_init - issue softreset to the core
  2853. * @hsotg: The device state
  2854. * @is_usb_reset: Usb resetting flag
  2855. *
  2856. * Issue a soft reset to the core, and await the core finishing it.
  2857. */
  2858. void dwc2_hsotg_core_init_disconnected(struct dwc2_hsotg *hsotg,
  2859. bool is_usb_reset)
  2860. {
  2861. u32 intmsk;
  2862. u32 val;
  2863. u32 usbcfg;
  2864. u32 dcfg = 0;
  2865. int ep;
  2866. /* Kill any ep0 requests as controller will be reinitialized */
  2867. kill_all_requests(hsotg, hsotg->eps_out[0], -ECONNRESET);
  2868. if (!is_usb_reset) {
  2869. if (dwc2_core_reset(hsotg, true))
  2870. return;
  2871. } else {
  2872. /* all endpoints should be shutdown */
  2873. for (ep = 1; ep < hsotg->num_of_eps; ep++) {
  2874. if (hsotg->eps_in[ep])
  2875. dwc2_hsotg_ep_disable(&hsotg->eps_in[ep]->ep);
  2876. if (hsotg->eps_out[ep])
  2877. dwc2_hsotg_ep_disable(&hsotg->eps_out[ep]->ep);
  2878. }
  2879. }
  2880. /*
  2881. * we must now enable ep0 ready for host detection and then
  2882. * set configuration.
  2883. */
  2884. /* keep other bits untouched (so e.g. forced modes are not lost) */
  2885. usbcfg = dwc2_readl(hsotg, GUSBCFG);
  2886. usbcfg &= ~GUSBCFG_TOUTCAL_MASK;
  2887. usbcfg |= GUSBCFG_TOUTCAL(7);
  2888. /* remove the HNP/SRP and set the PHY */
  2889. usbcfg &= ~(GUSBCFG_SRPCAP | GUSBCFG_HNPCAP);
  2890. dwc2_writel(hsotg, usbcfg, GUSBCFG);
  2891. dwc2_phy_init(hsotg, true);
  2892. dwc2_hsotg_init_fifo(hsotg);
  2893. if (!is_usb_reset)
  2894. dwc2_set_bit(hsotg, DCTL, DCTL_SFTDISCON);
  2895. dcfg |= DCFG_EPMISCNT(1);
  2896. switch (hsotg->params.speed) {
  2897. case DWC2_SPEED_PARAM_LOW:
  2898. dcfg |= DCFG_DEVSPD_LS;
  2899. break;
  2900. case DWC2_SPEED_PARAM_FULL:
  2901. if (hsotg->params.phy_type == DWC2_PHY_TYPE_PARAM_FS)
  2902. dcfg |= DCFG_DEVSPD_FS48;
  2903. else
  2904. dcfg |= DCFG_DEVSPD_FS;
  2905. break;
  2906. default:
  2907. dcfg |= DCFG_DEVSPD_HS;
  2908. }
  2909. if (hsotg->params.ipg_isoc_en)
  2910. dcfg |= DCFG_IPG_ISOC_SUPPORDED;
  2911. dwc2_writel(hsotg, dcfg, DCFG);
  2912. /* Clear any pending OTG interrupts */
  2913. dwc2_writel(hsotg, 0xffffffff, GOTGINT);
  2914. /* Clear any pending interrupts */
  2915. dwc2_writel(hsotg, 0xffffffff, GINTSTS);
  2916. intmsk = GINTSTS_ERLYSUSP | GINTSTS_SESSREQINT |
  2917. GINTSTS_GOUTNAKEFF | GINTSTS_GINNAKEFF |
  2918. GINTSTS_USBRST | GINTSTS_RESETDET |
  2919. GINTSTS_ENUMDONE | GINTSTS_OTGINT |
  2920. GINTSTS_USBSUSP | GINTSTS_WKUPINT |
  2921. GINTSTS_LPMTRANRCVD;
  2922. if (!using_desc_dma(hsotg))
  2923. intmsk |= GINTSTS_INCOMPL_SOIN | GINTSTS_INCOMPL_SOOUT;
  2924. if (!hsotg->params.external_id_pin_ctl)
  2925. intmsk |= GINTSTS_CONIDSTSCHNG;
  2926. dwc2_writel(hsotg, intmsk, GINTMSK);
  2927. if (using_dma(hsotg)) {
  2928. dwc2_writel(hsotg, GAHBCFG_GLBL_INTR_EN | GAHBCFG_DMA_EN |
  2929. hsotg->params.ahbcfg,
  2930. GAHBCFG);
  2931. /* Set DDMA mode support in the core if needed */
  2932. if (using_desc_dma(hsotg))
  2933. dwc2_set_bit(hsotg, DCFG, DCFG_DESCDMA_EN);
  2934. } else {
  2935. dwc2_writel(hsotg, ((hsotg->dedicated_fifos) ?
  2936. (GAHBCFG_NP_TXF_EMP_LVL |
  2937. GAHBCFG_P_TXF_EMP_LVL) : 0) |
  2938. GAHBCFG_GLBL_INTR_EN, GAHBCFG);
  2939. }
  2940. /*
  2941. * If INTknTXFEmpMsk is enabled, it's important to disable ep interrupts
  2942. * when we have no data to transfer. Otherwise we get being flooded by
  2943. * interrupts.
  2944. */
  2945. dwc2_writel(hsotg, ((hsotg->dedicated_fifos && !using_dma(hsotg)) ?
  2946. DIEPMSK_TXFIFOEMPTY | DIEPMSK_INTKNTXFEMPMSK : 0) |
  2947. DIEPMSK_EPDISBLDMSK | DIEPMSK_XFERCOMPLMSK |
  2948. DIEPMSK_TIMEOUTMSK | DIEPMSK_AHBERRMSK,
  2949. DIEPMSK);
  2950. /*
  2951. * don't need XferCompl, we get that from RXFIFO in slave mode. In
  2952. * DMA mode we may need this and StsPhseRcvd.
  2953. */
  2954. dwc2_writel(hsotg, (using_dma(hsotg) ? (DIEPMSK_XFERCOMPLMSK |
  2955. DOEPMSK_STSPHSERCVDMSK) : 0) |
  2956. DOEPMSK_EPDISBLDMSK | DOEPMSK_AHBERRMSK |
  2957. DOEPMSK_SETUPMSK,
  2958. DOEPMSK);
  2959. /* Enable BNA interrupt for DDMA */
  2960. if (using_desc_dma(hsotg)) {
  2961. dwc2_set_bit(hsotg, DOEPMSK, DOEPMSK_BNAMSK);
  2962. dwc2_set_bit(hsotg, DIEPMSK, DIEPMSK_BNAININTRMSK);
  2963. }
  2964. /* Enable Service Interval mode if supported */
  2965. if (using_desc_dma(hsotg) && hsotg->params.service_interval)
  2966. dwc2_set_bit(hsotg, DCTL, DCTL_SERVICE_INTERVAL_SUPPORTED);
  2967. dwc2_writel(hsotg, 0, DAINTMSK);
  2968. dev_dbg(hsotg->dev, "EP0: DIEPCTL0=0x%08x, DOEPCTL0=0x%08x\n",
  2969. dwc2_readl(hsotg, DIEPCTL0),
  2970. dwc2_readl(hsotg, DOEPCTL0));
  2971. /* enable in and out endpoint interrupts */
  2972. dwc2_hsotg_en_gsint(hsotg, GINTSTS_OEPINT | GINTSTS_IEPINT);
  2973. /*
  2974. * Enable the RXFIFO when in slave mode, as this is how we collect
  2975. * the data. In DMA mode, we get events from the FIFO but also
  2976. * things we cannot process, so do not use it.
  2977. */
  2978. if (!using_dma(hsotg))
  2979. dwc2_hsotg_en_gsint(hsotg, GINTSTS_RXFLVL);
  2980. /* Enable interrupts for EP0 in and out */
  2981. dwc2_hsotg_ctrl_epint(hsotg, 0, 0, 1);
  2982. dwc2_hsotg_ctrl_epint(hsotg, 0, 1, 1);
  2983. if (!is_usb_reset) {
  2984. dwc2_set_bit(hsotg, DCTL, DCTL_PWRONPRGDONE);
  2985. udelay(10); /* see openiboot */
  2986. dwc2_clear_bit(hsotg, DCTL, DCTL_PWRONPRGDONE);
  2987. }
  2988. dev_dbg(hsotg->dev, "DCTL=0x%08x\n", dwc2_readl(hsotg, DCTL));
  2989. /*
  2990. * DxEPCTL_USBActEp says RO in manual, but seems to be set by
  2991. * writing to the EPCTL register..
  2992. */
  2993. /* set to read 1 8byte packet */
  2994. dwc2_writel(hsotg, DXEPTSIZ_MC(1) | DXEPTSIZ_PKTCNT(1) |
  2995. DXEPTSIZ_XFERSIZE(8), DOEPTSIZ0);
  2996. dwc2_writel(hsotg, dwc2_hsotg_ep0_mps(hsotg->eps_out[0]->ep.maxpacket) |
  2997. DXEPCTL_CNAK | DXEPCTL_EPENA |
  2998. DXEPCTL_USBACTEP,
  2999. DOEPCTL0);
  3000. /* enable, but don't activate EP0in */
  3001. dwc2_writel(hsotg, dwc2_hsotg_ep0_mps(hsotg->eps_out[0]->ep.maxpacket) |
  3002. DXEPCTL_USBACTEP, DIEPCTL0);
  3003. /* clear global NAKs */
  3004. val = DCTL_CGOUTNAK | DCTL_CGNPINNAK;
  3005. if (!is_usb_reset)
  3006. val |= DCTL_SFTDISCON;
  3007. dwc2_set_bit(hsotg, DCTL, val);
  3008. /* configure the core to support LPM */
  3009. dwc2_gadget_init_lpm(hsotg);
  3010. /* program GREFCLK register if needed */
  3011. if (using_desc_dma(hsotg) && hsotg->params.service_interval)
  3012. dwc2_gadget_program_ref_clk(hsotg);
  3013. /* must be at-least 3ms to allow bus to see disconnect */
  3014. mdelay(3);
  3015. hsotg->lx_state = DWC2_L0;
  3016. dwc2_hsotg_enqueue_setup(hsotg);
  3017. dev_dbg(hsotg->dev, "EP0: DIEPCTL0=0x%08x, DOEPCTL0=0x%08x\n",
  3018. dwc2_readl(hsotg, DIEPCTL0),
  3019. dwc2_readl(hsotg, DOEPCTL0));
  3020. }
  3021. void dwc2_hsotg_core_disconnect(struct dwc2_hsotg *hsotg)
  3022. {
  3023. /* set the soft-disconnect bit */
  3024. dwc2_set_bit(hsotg, DCTL, DCTL_SFTDISCON);
  3025. }
  3026. void dwc2_hsotg_core_connect(struct dwc2_hsotg *hsotg)
  3027. {
  3028. /* remove the soft-disconnect and let's go */
  3029. dwc2_clear_bit(hsotg, DCTL, DCTL_SFTDISCON);
  3030. }
  3031. /**
  3032. * dwc2_gadget_handle_incomplete_isoc_in - handle incomplete ISO IN Interrupt.
  3033. * @hsotg: The device state:
  3034. *
  3035. * This interrupt indicates one of the following conditions occurred while
  3036. * transmitting an ISOC transaction.
  3037. * - Corrupted IN Token for ISOC EP.
  3038. * - Packet not complete in FIFO.
  3039. *
  3040. * The following actions will be taken:
  3041. * - Determine the EP
  3042. * - Disable EP; when 'Endpoint Disabled' interrupt is received Flush FIFO
  3043. */
  3044. static void dwc2_gadget_handle_incomplete_isoc_in(struct dwc2_hsotg *hsotg)
  3045. {
  3046. struct dwc2_hsotg_ep *hs_ep;
  3047. u32 epctrl;
  3048. u32 daintmsk;
  3049. u32 idx;
  3050. dev_dbg(hsotg->dev, "Incomplete isoc in interrupt received:\n");
  3051. daintmsk = dwc2_readl(hsotg, DAINTMSK);
  3052. for (idx = 1; idx < hsotg->num_of_eps; idx++) {
  3053. hs_ep = hsotg->eps_in[idx];
  3054. /* Proceed only unmasked ISOC EPs */
  3055. if ((BIT(idx) & ~daintmsk) || !hs_ep->isochronous)
  3056. continue;
  3057. epctrl = dwc2_readl(hsotg, DIEPCTL(idx));
  3058. if ((epctrl & DXEPCTL_EPENA) &&
  3059. dwc2_gadget_target_frame_elapsed(hs_ep)) {
  3060. epctrl |= DXEPCTL_SNAK;
  3061. epctrl |= DXEPCTL_EPDIS;
  3062. dwc2_writel(hsotg, epctrl, DIEPCTL(idx));
  3063. }
  3064. }
  3065. /* Clear interrupt */
  3066. dwc2_writel(hsotg, GINTSTS_INCOMPL_SOIN, GINTSTS);
  3067. }
  3068. /**
  3069. * dwc2_gadget_handle_incomplete_isoc_out - handle incomplete ISO OUT Interrupt
  3070. * @hsotg: The device state:
  3071. *
  3072. * This interrupt indicates one of the following conditions occurred while
  3073. * transmitting an ISOC transaction.
  3074. * - Corrupted OUT Token for ISOC EP.
  3075. * - Packet not complete in FIFO.
  3076. *
  3077. * The following actions will be taken:
  3078. * - Determine the EP
  3079. * - Set DCTL_SGOUTNAK and unmask GOUTNAKEFF if target frame elapsed.
  3080. */
  3081. static void dwc2_gadget_handle_incomplete_isoc_out(struct dwc2_hsotg *hsotg)
  3082. {
  3083. u32 gintsts;
  3084. u32 gintmsk;
  3085. u32 daintmsk;
  3086. u32 epctrl;
  3087. struct dwc2_hsotg_ep *hs_ep;
  3088. int idx;
  3089. dev_dbg(hsotg->dev, "%s: GINTSTS_INCOMPL_SOOUT\n", __func__);
  3090. daintmsk = dwc2_readl(hsotg, DAINTMSK);
  3091. daintmsk >>= DAINT_OUTEP_SHIFT;
  3092. for (idx = 1; idx < hsotg->num_of_eps; idx++) {
  3093. hs_ep = hsotg->eps_out[idx];
  3094. /* Proceed only unmasked ISOC EPs */
  3095. if ((BIT(idx) & ~daintmsk) || !hs_ep->isochronous)
  3096. continue;
  3097. epctrl = dwc2_readl(hsotg, DOEPCTL(idx));
  3098. if ((epctrl & DXEPCTL_EPENA) &&
  3099. dwc2_gadget_target_frame_elapsed(hs_ep)) {
  3100. /* Unmask GOUTNAKEFF interrupt */
  3101. gintmsk = dwc2_readl(hsotg, GINTMSK);
  3102. gintmsk |= GINTSTS_GOUTNAKEFF;
  3103. dwc2_writel(hsotg, gintmsk, GINTMSK);
  3104. gintsts = dwc2_readl(hsotg, GINTSTS);
  3105. if (!(gintsts & GINTSTS_GOUTNAKEFF)) {
  3106. dwc2_set_bit(hsotg, DCTL, DCTL_SGOUTNAK);
  3107. break;
  3108. }
  3109. }
  3110. }
  3111. /* Clear interrupt */
  3112. dwc2_writel(hsotg, GINTSTS_INCOMPL_SOOUT, GINTSTS);
  3113. }
  3114. /**
  3115. * dwc2_hsotg_irq - handle device interrupt
  3116. * @irq: The IRQ number triggered
  3117. * @pw: The pw value when registered the handler.
  3118. */
  3119. static irqreturn_t dwc2_hsotg_irq(int irq, void *pw)
  3120. {
  3121. struct dwc2_hsotg *hsotg = pw;
  3122. int retry_count = 8;
  3123. u32 gintsts;
  3124. u32 gintmsk;
  3125. if (!dwc2_is_device_mode(hsotg))
  3126. return IRQ_NONE;
  3127. spin_lock(&hsotg->lock);
  3128. irq_retry:
  3129. gintsts = dwc2_readl(hsotg, GINTSTS);
  3130. gintmsk = dwc2_readl(hsotg, GINTMSK);
  3131. dev_dbg(hsotg->dev, "%s: %08x %08x (%08x) retry %d\n",
  3132. __func__, gintsts, gintsts & gintmsk, gintmsk, retry_count);
  3133. gintsts &= gintmsk;
  3134. if (gintsts & GINTSTS_RESETDET) {
  3135. dev_dbg(hsotg->dev, "%s: USBRstDet\n", __func__);
  3136. dwc2_writel(hsotg, GINTSTS_RESETDET, GINTSTS);
  3137. /* This event must be used only if controller is suspended */
  3138. if (hsotg->lx_state == DWC2_L2) {
  3139. dwc2_exit_partial_power_down(hsotg, true);
  3140. hsotg->lx_state = DWC2_L0;
  3141. }
  3142. }
  3143. if (gintsts & (GINTSTS_USBRST | GINTSTS_RESETDET)) {
  3144. u32 usb_status = dwc2_readl(hsotg, GOTGCTL);
  3145. u32 connected = hsotg->connected;
  3146. dev_dbg(hsotg->dev, "%s: USBRst\n", __func__);
  3147. dev_dbg(hsotg->dev, "GNPTXSTS=%08x\n",
  3148. dwc2_readl(hsotg, GNPTXSTS));
  3149. dwc2_writel(hsotg, GINTSTS_USBRST, GINTSTS);
  3150. /* Report disconnection if it is not already done. */
  3151. dwc2_hsotg_disconnect(hsotg);
  3152. /* Reset device address to zero */
  3153. dwc2_clear_bit(hsotg, DCFG, DCFG_DEVADDR_MASK);
  3154. if (usb_status & GOTGCTL_BSESVLD && connected)
  3155. dwc2_hsotg_core_init_disconnected(hsotg, true);
  3156. }
  3157. if (gintsts & GINTSTS_ENUMDONE) {
  3158. dwc2_writel(hsotg, GINTSTS_ENUMDONE, GINTSTS);
  3159. dwc2_hsotg_irq_enumdone(hsotg);
  3160. }
  3161. if (gintsts & (GINTSTS_OEPINT | GINTSTS_IEPINT)) {
  3162. u32 daint = dwc2_readl(hsotg, DAINT);
  3163. u32 daintmsk = dwc2_readl(hsotg, DAINTMSK);
  3164. u32 daint_out, daint_in;
  3165. int ep;
  3166. daint &= daintmsk;
  3167. daint_out = daint >> DAINT_OUTEP_SHIFT;
  3168. daint_in = daint & ~(daint_out << DAINT_OUTEP_SHIFT);
  3169. dev_dbg(hsotg->dev, "%s: daint=%08x\n", __func__, daint);
  3170. for (ep = 0; ep < hsotg->num_of_eps && daint_out;
  3171. ep++, daint_out >>= 1) {
  3172. if (daint_out & 1)
  3173. dwc2_hsotg_epint(hsotg, ep, 0);
  3174. }
  3175. for (ep = 0; ep < hsotg->num_of_eps && daint_in;
  3176. ep++, daint_in >>= 1) {
  3177. if (daint_in & 1)
  3178. dwc2_hsotg_epint(hsotg, ep, 1);
  3179. }
  3180. }
  3181. /* check both FIFOs */
  3182. if (gintsts & GINTSTS_NPTXFEMP) {
  3183. dev_dbg(hsotg->dev, "NPTxFEmp\n");
  3184. /*
  3185. * Disable the interrupt to stop it happening again
  3186. * unless one of these endpoint routines decides that
  3187. * it needs re-enabling
  3188. */
  3189. dwc2_hsotg_disable_gsint(hsotg, GINTSTS_NPTXFEMP);
  3190. dwc2_hsotg_irq_fifoempty(hsotg, false);
  3191. }
  3192. if (gintsts & GINTSTS_PTXFEMP) {
  3193. dev_dbg(hsotg->dev, "PTxFEmp\n");
  3194. /* See note in GINTSTS_NPTxFEmp */
  3195. dwc2_hsotg_disable_gsint(hsotg, GINTSTS_PTXFEMP);
  3196. dwc2_hsotg_irq_fifoempty(hsotg, true);
  3197. }
  3198. if (gintsts & GINTSTS_RXFLVL) {
  3199. /*
  3200. * note, since GINTSTS_RxFLvl doubles as FIFO-not-empty,
  3201. * we need to retry dwc2_hsotg_handle_rx if this is still
  3202. * set.
  3203. */
  3204. dwc2_hsotg_handle_rx(hsotg);
  3205. }
  3206. if (gintsts & GINTSTS_ERLYSUSP) {
  3207. dev_dbg(hsotg->dev, "GINTSTS_ErlySusp\n");
  3208. dwc2_writel(hsotg, GINTSTS_ERLYSUSP, GINTSTS);
  3209. }
  3210. /*
  3211. * these next two seem to crop-up occasionally causing the core
  3212. * to shutdown the USB transfer, so try clearing them and logging
  3213. * the occurrence.
  3214. */
  3215. if (gintsts & GINTSTS_GOUTNAKEFF) {
  3216. u8 idx;
  3217. u32 epctrl;
  3218. u32 gintmsk;
  3219. u32 daintmsk;
  3220. struct dwc2_hsotg_ep *hs_ep;
  3221. daintmsk = dwc2_readl(hsotg, DAINTMSK);
  3222. daintmsk >>= DAINT_OUTEP_SHIFT;
  3223. /* Mask this interrupt */
  3224. gintmsk = dwc2_readl(hsotg, GINTMSK);
  3225. gintmsk &= ~GINTSTS_GOUTNAKEFF;
  3226. dwc2_writel(hsotg, gintmsk, GINTMSK);
  3227. dev_dbg(hsotg->dev, "GOUTNakEff triggered\n");
  3228. for (idx = 1; idx < hsotg->num_of_eps; idx++) {
  3229. hs_ep = hsotg->eps_out[idx];
  3230. /* Proceed only unmasked ISOC EPs */
  3231. if (BIT(idx) & ~daintmsk)
  3232. continue;
  3233. epctrl = dwc2_readl(hsotg, DOEPCTL(idx));
  3234. //ISOC Ep's only
  3235. if ((epctrl & DXEPCTL_EPENA) && hs_ep->isochronous) {
  3236. epctrl |= DXEPCTL_SNAK;
  3237. epctrl |= DXEPCTL_EPDIS;
  3238. dwc2_writel(hsotg, epctrl, DOEPCTL(idx));
  3239. continue;
  3240. }
  3241. //Non-ISOC EP's
  3242. if (hs_ep->halted) {
  3243. if (!(epctrl & DXEPCTL_EPENA))
  3244. epctrl |= DXEPCTL_EPENA;
  3245. epctrl |= DXEPCTL_EPDIS;
  3246. epctrl |= DXEPCTL_STALL;
  3247. dwc2_writel(hsotg, epctrl, DOEPCTL(idx));
  3248. }
  3249. }
  3250. /* This interrupt bit is cleared in DXEPINT_EPDISBLD handler */
  3251. }
  3252. if (gintsts & GINTSTS_GINNAKEFF) {
  3253. dev_info(hsotg->dev, "GINNakEff triggered\n");
  3254. dwc2_set_bit(hsotg, DCTL, DCTL_CGNPINNAK);
  3255. dwc2_hsotg_dump(hsotg);
  3256. }
  3257. if (gintsts & GINTSTS_INCOMPL_SOIN)
  3258. dwc2_gadget_handle_incomplete_isoc_in(hsotg);
  3259. if (gintsts & GINTSTS_INCOMPL_SOOUT)
  3260. dwc2_gadget_handle_incomplete_isoc_out(hsotg);
  3261. /*
  3262. * if we've had fifo events, we should try and go around the
  3263. * loop again to see if there's any point in returning yet.
  3264. */
  3265. if (gintsts & IRQ_RETRY_MASK && --retry_count > 0)
  3266. goto irq_retry;
  3267. /* Check WKUP_ALERT interrupt*/
  3268. if (hsotg->params.service_interval)
  3269. dwc2_gadget_wkup_alert_handler(hsotg);
  3270. spin_unlock(&hsotg->lock);
  3271. return IRQ_HANDLED;
  3272. }
  3273. static void dwc2_hsotg_ep_stop_xfr(struct dwc2_hsotg *hsotg,
  3274. struct dwc2_hsotg_ep *hs_ep)
  3275. {
  3276. u32 epctrl_reg;
  3277. u32 epint_reg;
  3278. epctrl_reg = hs_ep->dir_in ? DIEPCTL(hs_ep->index) :
  3279. DOEPCTL(hs_ep->index);
  3280. epint_reg = hs_ep->dir_in ? DIEPINT(hs_ep->index) :
  3281. DOEPINT(hs_ep->index);
  3282. dev_dbg(hsotg->dev, "%s: stopping transfer on %s\n", __func__,
  3283. hs_ep->name);
  3284. if (hs_ep->dir_in) {
  3285. if (hsotg->dedicated_fifos || hs_ep->periodic) {
  3286. dwc2_set_bit(hsotg, epctrl_reg, DXEPCTL_SNAK);
  3287. /* Wait for Nak effect */
  3288. if (dwc2_hsotg_wait_bit_set(hsotg, epint_reg,
  3289. DXEPINT_INEPNAKEFF, 100))
  3290. dev_warn(hsotg->dev,
  3291. "%s: timeout DIEPINT.NAKEFF\n",
  3292. __func__);
  3293. } else {
  3294. dwc2_set_bit(hsotg, DCTL, DCTL_SGNPINNAK);
  3295. /* Wait for Nak effect */
  3296. if (dwc2_hsotg_wait_bit_set(hsotg, GINTSTS,
  3297. GINTSTS_GINNAKEFF, 100))
  3298. dev_warn(hsotg->dev,
  3299. "%s: timeout GINTSTS.GINNAKEFF\n",
  3300. __func__);
  3301. }
  3302. } else {
  3303. /* Mask GINTSTS_GOUTNAKEFF interrupt */
  3304. dwc2_hsotg_disable_gsint(hsotg, GINTSTS_GOUTNAKEFF);
  3305. if (!(dwc2_readl(hsotg, GINTSTS) & GINTSTS_GOUTNAKEFF))
  3306. dwc2_set_bit(hsotg, DCTL, DCTL_SGOUTNAK);
  3307. if (!using_dma(hsotg)) {
  3308. /* Wait for GINTSTS_RXFLVL interrupt */
  3309. if (dwc2_hsotg_wait_bit_set(hsotg, GINTSTS,
  3310. GINTSTS_RXFLVL, 100)) {
  3311. dev_warn(hsotg->dev, "%s: timeout GINTSTS.RXFLVL\n",
  3312. __func__);
  3313. } else {
  3314. /*
  3315. * Pop GLOBAL OUT NAK status packet from RxFIFO
  3316. * to assert GOUTNAKEFF interrupt
  3317. */
  3318. dwc2_readl(hsotg, GRXSTSP);
  3319. }
  3320. }
  3321. /* Wait for global nak to take effect */
  3322. if (dwc2_hsotg_wait_bit_set(hsotg, GINTSTS,
  3323. GINTSTS_GOUTNAKEFF, 100))
  3324. dev_warn(hsotg->dev, "%s: timeout GINTSTS.GOUTNAKEFF\n",
  3325. __func__);
  3326. }
  3327. /* Disable ep */
  3328. dwc2_set_bit(hsotg, epctrl_reg, DXEPCTL_EPDIS | DXEPCTL_SNAK);
  3329. /* Wait for ep to be disabled */
  3330. if (dwc2_hsotg_wait_bit_set(hsotg, epint_reg, DXEPINT_EPDISBLD, 100))
  3331. dev_warn(hsotg->dev,
  3332. "%s: timeout DOEPCTL.EPDisable\n", __func__);
  3333. /* Clear EPDISBLD interrupt */
  3334. dwc2_set_bit(hsotg, epint_reg, DXEPINT_EPDISBLD);
  3335. if (hs_ep->dir_in) {
  3336. unsigned short fifo_index;
  3337. if (hsotg->dedicated_fifos || hs_ep->periodic)
  3338. fifo_index = hs_ep->fifo_index;
  3339. else
  3340. fifo_index = 0;
  3341. /* Flush TX FIFO */
  3342. dwc2_flush_tx_fifo(hsotg, fifo_index);
  3343. /* Clear Global In NP NAK in Shared FIFO for non periodic ep */
  3344. if (!hsotg->dedicated_fifos && !hs_ep->periodic)
  3345. dwc2_set_bit(hsotg, DCTL, DCTL_CGNPINNAK);
  3346. } else {
  3347. /* Remove global NAKs */
  3348. dwc2_set_bit(hsotg, DCTL, DCTL_CGOUTNAK);
  3349. }
  3350. }
  3351. /**
  3352. * dwc2_hsotg_ep_enable - enable the given endpoint
  3353. * @ep: The USB endpint to configure
  3354. * @desc: The USB endpoint descriptor to configure with.
  3355. *
  3356. * This is called from the USB gadget code's usb_ep_enable().
  3357. */
  3358. static int dwc2_hsotg_ep_enable(struct usb_ep *ep,
  3359. const struct usb_endpoint_descriptor *desc)
  3360. {
  3361. struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
  3362. struct dwc2_hsotg *hsotg = hs_ep->parent;
  3363. unsigned long flags;
  3364. unsigned int index = hs_ep->index;
  3365. u32 epctrl_reg;
  3366. u32 epctrl;
  3367. u32 mps;
  3368. u32 mc;
  3369. u32 mask;
  3370. unsigned int dir_in;
  3371. unsigned int i, val, size;
  3372. int ret = 0;
  3373. unsigned char ep_type;
  3374. int desc_num;
  3375. dev_dbg(hsotg->dev,
  3376. "%s: ep %s: a 0x%02x, attr 0x%02x, mps 0x%04x, intr %d\n",
  3377. __func__, ep->name, desc->bEndpointAddress, desc->bmAttributes,
  3378. desc->wMaxPacketSize, desc->bInterval);
  3379. /* not to be called for EP0 */
  3380. if (index == 0) {
  3381. dev_err(hsotg->dev, "%s: called for EP 0\n", __func__);
  3382. return -EINVAL;
  3383. }
  3384. dir_in = (desc->bEndpointAddress & USB_ENDPOINT_DIR_MASK) ? 1 : 0;
  3385. if (dir_in != hs_ep->dir_in) {
  3386. dev_err(hsotg->dev, "%s: direction mismatch!\n", __func__);
  3387. return -EINVAL;
  3388. }
  3389. ep_type = desc->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK;
  3390. mps = usb_endpoint_maxp(desc);
  3391. mc = usb_endpoint_maxp_mult(desc);
  3392. /* ISOC IN in DDMA supported bInterval up to 10 */
  3393. if (using_desc_dma(hsotg) && ep_type == USB_ENDPOINT_XFER_ISOC &&
  3394. dir_in && desc->bInterval > 10) {
  3395. dev_err(hsotg->dev,
  3396. "%s: ISOC IN, DDMA: bInterval>10 not supported!\n", __func__);
  3397. return -EINVAL;
  3398. }
  3399. /* High bandwidth ISOC OUT in DDMA not supported */
  3400. if (using_desc_dma(hsotg) && ep_type == USB_ENDPOINT_XFER_ISOC &&
  3401. !dir_in && mc > 1) {
  3402. dev_err(hsotg->dev,
  3403. "%s: ISOC OUT, DDMA: HB not supported!\n", __func__);
  3404. return -EINVAL;
  3405. }
  3406. /* note, we handle this here instead of dwc2_hsotg_set_ep_maxpacket */
  3407. epctrl_reg = dir_in ? DIEPCTL(index) : DOEPCTL(index);
  3408. epctrl = dwc2_readl(hsotg, epctrl_reg);
  3409. dev_dbg(hsotg->dev, "%s: read DxEPCTL=0x%08x from 0x%08x\n",
  3410. __func__, epctrl, epctrl_reg);
  3411. if (using_desc_dma(hsotg) && ep_type == USB_ENDPOINT_XFER_ISOC)
  3412. desc_num = MAX_DMA_DESC_NUM_HS_ISOC;
  3413. else
  3414. desc_num = MAX_DMA_DESC_NUM_GENERIC;
  3415. /* Allocate DMA descriptor chain for non-ctrl endpoints */
  3416. if (using_desc_dma(hsotg) && !hs_ep->desc_list) {
  3417. hs_ep->desc_list = dmam_alloc_coherent(hsotg->dev,
  3418. desc_num * sizeof(struct dwc2_dma_desc),
  3419. &hs_ep->desc_list_dma, GFP_ATOMIC);
  3420. if (!hs_ep->desc_list) {
  3421. ret = -ENOMEM;
  3422. goto error2;
  3423. }
  3424. }
  3425. spin_lock_irqsave(&hsotg->lock, flags);
  3426. epctrl &= ~(DXEPCTL_EPTYPE_MASK | DXEPCTL_MPS_MASK);
  3427. epctrl |= DXEPCTL_MPS(mps);
  3428. /*
  3429. * mark the endpoint as active, otherwise the core may ignore
  3430. * transactions entirely for this endpoint
  3431. */
  3432. epctrl |= DXEPCTL_USBACTEP;
  3433. /* update the endpoint state */
  3434. dwc2_hsotg_set_ep_maxpacket(hsotg, hs_ep->index, mps, mc, dir_in);
  3435. /* default, set to non-periodic */
  3436. hs_ep->isochronous = 0;
  3437. hs_ep->periodic = 0;
  3438. hs_ep->halted = 0;
  3439. hs_ep->interval = desc->bInterval;
  3440. switch (ep_type) {
  3441. case USB_ENDPOINT_XFER_ISOC:
  3442. epctrl |= DXEPCTL_EPTYPE_ISO;
  3443. epctrl |= DXEPCTL_SETEVENFR;
  3444. hs_ep->isochronous = 1;
  3445. hs_ep->interval = 1 << (desc->bInterval - 1);
  3446. hs_ep->target_frame = TARGET_FRAME_INITIAL;
  3447. hs_ep->next_desc = 0;
  3448. hs_ep->compl_desc = 0;
  3449. if (dir_in) {
  3450. hs_ep->periodic = 1;
  3451. mask = dwc2_readl(hsotg, DIEPMSK);
  3452. mask |= DIEPMSK_NAKMSK;
  3453. dwc2_writel(hsotg, mask, DIEPMSK);
  3454. } else {
  3455. epctrl |= DXEPCTL_SNAK;
  3456. mask = dwc2_readl(hsotg, DOEPMSK);
  3457. mask |= DOEPMSK_OUTTKNEPDISMSK;
  3458. dwc2_writel(hsotg, mask, DOEPMSK);
  3459. }
  3460. break;
  3461. case USB_ENDPOINT_XFER_BULK:
  3462. epctrl |= DXEPCTL_EPTYPE_BULK;
  3463. break;
  3464. case USB_ENDPOINT_XFER_INT:
  3465. if (dir_in)
  3466. hs_ep->periodic = 1;
  3467. if (hsotg->gadget.speed == USB_SPEED_HIGH)
  3468. hs_ep->interval = 1 << (desc->bInterval - 1);
  3469. epctrl |= DXEPCTL_EPTYPE_INTERRUPT;
  3470. break;
  3471. case USB_ENDPOINT_XFER_CONTROL:
  3472. epctrl |= DXEPCTL_EPTYPE_CONTROL;
  3473. break;
  3474. }
  3475. /*
  3476. * if the hardware has dedicated fifos, we must give each IN EP
  3477. * a unique tx-fifo even if it is non-periodic.
  3478. */
  3479. if (dir_in && hsotg->dedicated_fifos) {
  3480. unsigned fifo_count = dwc2_hsotg_tx_fifo_count(hsotg);
  3481. u32 fifo_index = 0;
  3482. u32 fifo_size = UINT_MAX;
  3483. size = hs_ep->ep.maxpacket * hs_ep->mc;
  3484. for (i = 1; i <= fifo_count; ++i) {
  3485. if (hsotg->fifo_map & (1 << i))
  3486. continue;
  3487. val = dwc2_readl(hsotg, DPTXFSIZN(i));
  3488. val = (val >> FIFOSIZE_DEPTH_SHIFT) * 4;
  3489. if (val < size)
  3490. continue;
  3491. /* Search for smallest acceptable fifo */
  3492. if (val < fifo_size) {
  3493. fifo_size = val;
  3494. fifo_index = i;
  3495. }
  3496. }
  3497. if (!fifo_index) {
  3498. dev_err(hsotg->dev,
  3499. "%s: No suitable fifo found\n", __func__);
  3500. ret = -ENOMEM;
  3501. goto error1;
  3502. }
  3503. epctrl &= ~(DXEPCTL_TXFNUM_LIMIT << DXEPCTL_TXFNUM_SHIFT);
  3504. hsotg->fifo_map |= 1 << fifo_index;
  3505. epctrl |= DXEPCTL_TXFNUM(fifo_index);
  3506. hs_ep->fifo_index = fifo_index;
  3507. hs_ep->fifo_size = fifo_size;
  3508. }
  3509. /* for non control endpoints, set PID to D0 */
  3510. if (index && !hs_ep->isochronous)
  3511. epctrl |= DXEPCTL_SETD0PID;
  3512. /* WA for Full speed ISOC IN in DDMA mode.
  3513. * By Clear NAK status of EP, core will send ZLP
  3514. * to IN token and assert NAK interrupt relying
  3515. * on TxFIFO status only
  3516. */
  3517. if (hsotg->gadget.speed == USB_SPEED_FULL &&
  3518. hs_ep->isochronous && dir_in) {
  3519. /* The WA applies only to core versions from 2.72a
  3520. * to 4.00a (including both). Also for FS_IOT_1.00a
  3521. * and HS_IOT_1.00a.
  3522. */
  3523. u32 gsnpsid = dwc2_readl(hsotg, GSNPSID);
  3524. if ((gsnpsid >= DWC2_CORE_REV_2_72a &&
  3525. gsnpsid <= DWC2_CORE_REV_4_00a) ||
  3526. gsnpsid == DWC2_FS_IOT_REV_1_00a ||
  3527. gsnpsid == DWC2_HS_IOT_REV_1_00a)
  3528. epctrl |= DXEPCTL_CNAK;
  3529. }
  3530. dev_dbg(hsotg->dev, "%s: write DxEPCTL=0x%08x\n",
  3531. __func__, epctrl);
  3532. dwc2_writel(hsotg, epctrl, epctrl_reg);
  3533. dev_dbg(hsotg->dev, "%s: read DxEPCTL=0x%08x\n",
  3534. __func__, dwc2_readl(hsotg, epctrl_reg));
  3535. /* enable the endpoint interrupt */
  3536. dwc2_hsotg_ctrl_epint(hsotg, index, dir_in, 1);
  3537. error1:
  3538. spin_unlock_irqrestore(&hsotg->lock, flags);
  3539. error2:
  3540. if (ret && using_desc_dma(hsotg) && hs_ep->desc_list) {
  3541. dmam_free_coherent(hsotg->dev, desc_num *
  3542. sizeof(struct dwc2_dma_desc),
  3543. hs_ep->desc_list, hs_ep->desc_list_dma);
  3544. hs_ep->desc_list = NULL;
  3545. }
  3546. return ret;
  3547. }
  3548. /**
  3549. * dwc2_hsotg_ep_disable - disable given endpoint
  3550. * @ep: The endpoint to disable.
  3551. */
  3552. static int dwc2_hsotg_ep_disable(struct usb_ep *ep)
  3553. {
  3554. struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
  3555. struct dwc2_hsotg *hsotg = hs_ep->parent;
  3556. int dir_in = hs_ep->dir_in;
  3557. int index = hs_ep->index;
  3558. u32 epctrl_reg;
  3559. u32 ctrl;
  3560. dev_dbg(hsotg->dev, "%s(ep %p)\n", __func__, ep);
  3561. if (ep == &hsotg->eps_out[0]->ep) {
  3562. dev_err(hsotg->dev, "%s: called for ep0\n", __func__);
  3563. return -EINVAL;
  3564. }
  3565. if (hsotg->op_state != OTG_STATE_B_PERIPHERAL) {
  3566. dev_err(hsotg->dev, "%s: called in host mode?\n", __func__);
  3567. return -EINVAL;
  3568. }
  3569. epctrl_reg = dir_in ? DIEPCTL(index) : DOEPCTL(index);
  3570. ctrl = dwc2_readl(hsotg, epctrl_reg);
  3571. if (ctrl & DXEPCTL_EPENA)
  3572. dwc2_hsotg_ep_stop_xfr(hsotg, hs_ep);
  3573. ctrl &= ~DXEPCTL_EPENA;
  3574. ctrl &= ~DXEPCTL_USBACTEP;
  3575. ctrl |= DXEPCTL_SNAK;
  3576. dev_dbg(hsotg->dev, "%s: DxEPCTL=0x%08x\n", __func__, ctrl);
  3577. dwc2_writel(hsotg, ctrl, epctrl_reg);
  3578. /* disable endpoint interrupts */
  3579. dwc2_hsotg_ctrl_epint(hsotg, hs_ep->index, hs_ep->dir_in, 0);
  3580. /* terminate all requests with shutdown */
  3581. kill_all_requests(hsotg, hs_ep, -ESHUTDOWN);
  3582. hsotg->fifo_map &= ~(1 << hs_ep->fifo_index);
  3583. hs_ep->fifo_index = 0;
  3584. hs_ep->fifo_size = 0;
  3585. return 0;
  3586. }
  3587. static int dwc2_hsotg_ep_disable_lock(struct usb_ep *ep)
  3588. {
  3589. struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
  3590. struct dwc2_hsotg *hsotg = hs_ep->parent;
  3591. unsigned long flags;
  3592. int ret;
  3593. spin_lock_irqsave(&hsotg->lock, flags);
  3594. ret = dwc2_hsotg_ep_disable(ep);
  3595. spin_unlock_irqrestore(&hsotg->lock, flags);
  3596. return ret;
  3597. }
  3598. /**
  3599. * on_list - check request is on the given endpoint
  3600. * @ep: The endpoint to check.
  3601. * @test: The request to test if it is on the endpoint.
  3602. */
  3603. static bool on_list(struct dwc2_hsotg_ep *ep, struct dwc2_hsotg_req *test)
  3604. {
  3605. struct dwc2_hsotg_req *req, *treq;
  3606. list_for_each_entry_safe(req, treq, &ep->queue, queue) {
  3607. if (req == test)
  3608. return true;
  3609. }
  3610. return false;
  3611. }
  3612. /**
  3613. * dwc2_hsotg_ep_dequeue - dequeue given endpoint
  3614. * @ep: The endpoint to dequeue.
  3615. * @req: The request to be removed from a queue.
  3616. */
  3617. static int dwc2_hsotg_ep_dequeue(struct usb_ep *ep, struct usb_request *req)
  3618. {
  3619. struct dwc2_hsotg_req *hs_req = our_req(req);
  3620. struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
  3621. struct dwc2_hsotg *hs = hs_ep->parent;
  3622. unsigned long flags;
  3623. dev_dbg(hs->dev, "ep_dequeue(%p,%p)\n", ep, req);
  3624. spin_lock_irqsave(&hs->lock, flags);
  3625. if (!on_list(hs_ep, hs_req)) {
  3626. spin_unlock_irqrestore(&hs->lock, flags);
  3627. return -EINVAL;
  3628. }
  3629. /* Dequeue already started request */
  3630. if (req == &hs_ep->req->req)
  3631. dwc2_hsotg_ep_stop_xfr(hs, hs_ep);
  3632. dwc2_hsotg_complete_request(hs, hs_ep, hs_req, -ECONNRESET);
  3633. spin_unlock_irqrestore(&hs->lock, flags);
  3634. return 0;
  3635. }
  3636. /**
  3637. * dwc2_hsotg_ep_sethalt - set halt on a given endpoint
  3638. * @ep: The endpoint to set halt.
  3639. * @value: Set or unset the halt.
  3640. * @now: If true, stall the endpoint now. Otherwise return -EAGAIN if
  3641. * the endpoint is busy processing requests.
  3642. *
  3643. * We need to stall the endpoint immediately if request comes from set_feature
  3644. * protocol command handler.
  3645. */
  3646. static int dwc2_hsotg_ep_sethalt(struct usb_ep *ep, int value, bool now)
  3647. {
  3648. struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
  3649. struct dwc2_hsotg *hs = hs_ep->parent;
  3650. int index = hs_ep->index;
  3651. u32 epreg;
  3652. u32 epctl;
  3653. u32 xfertype;
  3654. dev_info(hs->dev, "%s(ep %p %s, %d)\n", __func__, ep, ep->name, value);
  3655. if (index == 0) {
  3656. if (value)
  3657. dwc2_hsotg_stall_ep0(hs);
  3658. else
  3659. dev_warn(hs->dev,
  3660. "%s: can't clear halt on ep0\n", __func__);
  3661. return 0;
  3662. }
  3663. if (hs_ep->isochronous) {
  3664. dev_err(hs->dev, "%s is Isochronous Endpoint\n", ep->name);
  3665. return -EINVAL;
  3666. }
  3667. if (!now && value && !list_empty(&hs_ep->queue)) {
  3668. dev_dbg(hs->dev, "%s request is pending, cannot halt\n",
  3669. ep->name);
  3670. return -EAGAIN;
  3671. }
  3672. if (hs_ep->dir_in) {
  3673. epreg = DIEPCTL(index);
  3674. epctl = dwc2_readl(hs, epreg);
  3675. if (value) {
  3676. epctl |= DXEPCTL_STALL | DXEPCTL_SNAK;
  3677. if (epctl & DXEPCTL_EPENA)
  3678. epctl |= DXEPCTL_EPDIS;
  3679. } else {
  3680. epctl &= ~DXEPCTL_STALL;
  3681. xfertype = epctl & DXEPCTL_EPTYPE_MASK;
  3682. if (xfertype == DXEPCTL_EPTYPE_BULK ||
  3683. xfertype == DXEPCTL_EPTYPE_INTERRUPT)
  3684. epctl |= DXEPCTL_SETD0PID;
  3685. }
  3686. dwc2_writel(hs, epctl, epreg);
  3687. } else {
  3688. epreg = DOEPCTL(index);
  3689. epctl = dwc2_readl(hs, epreg);
  3690. if (value) {
  3691. /* Unmask GOUTNAKEFF interrupt */
  3692. dwc2_hsotg_en_gsint(hs, GINTSTS_GOUTNAKEFF);
  3693. if (!(dwc2_readl(hs, GINTSTS) & GINTSTS_GOUTNAKEFF))
  3694. dwc2_set_bit(hs, DCTL, DCTL_SGOUTNAK);
  3695. // STALL bit will be set in GOUTNAKEFF interrupt handler
  3696. } else {
  3697. epctl &= ~DXEPCTL_STALL;
  3698. xfertype = epctl & DXEPCTL_EPTYPE_MASK;
  3699. if (xfertype == DXEPCTL_EPTYPE_BULK ||
  3700. xfertype == DXEPCTL_EPTYPE_INTERRUPT)
  3701. epctl |= DXEPCTL_SETD0PID;
  3702. dwc2_writel(hs, epctl, epreg);
  3703. }
  3704. }
  3705. hs_ep->halted = value;
  3706. return 0;
  3707. }
  3708. /**
  3709. * dwc2_hsotg_ep_sethalt_lock - set halt on a given endpoint with lock held
  3710. * @ep: The endpoint to set halt.
  3711. * @value: Set or unset the halt.
  3712. */
  3713. static int dwc2_hsotg_ep_sethalt_lock(struct usb_ep *ep, int value)
  3714. {
  3715. struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
  3716. struct dwc2_hsotg *hs = hs_ep->parent;
  3717. unsigned long flags = 0;
  3718. int ret = 0;
  3719. spin_lock_irqsave(&hs->lock, flags);
  3720. ret = dwc2_hsotg_ep_sethalt(ep, value, false);
  3721. spin_unlock_irqrestore(&hs->lock, flags);
  3722. return ret;
  3723. }
  3724. static const struct usb_ep_ops dwc2_hsotg_ep_ops = {
  3725. .enable = dwc2_hsotg_ep_enable,
  3726. .disable = dwc2_hsotg_ep_disable_lock,
  3727. .alloc_request = dwc2_hsotg_ep_alloc_request,
  3728. .free_request = dwc2_hsotg_ep_free_request,
  3729. .queue = dwc2_hsotg_ep_queue_lock,
  3730. .dequeue = dwc2_hsotg_ep_dequeue,
  3731. .set_halt = dwc2_hsotg_ep_sethalt_lock,
  3732. /* note, don't believe we have any call for the fifo routines */
  3733. };
  3734. /**
  3735. * dwc2_hsotg_init - initialize the usb core
  3736. * @hsotg: The driver state
  3737. */
  3738. static void dwc2_hsotg_init(struct dwc2_hsotg *hsotg)
  3739. {
  3740. /* unmask subset of endpoint interrupts */
  3741. dwc2_writel(hsotg, DIEPMSK_TIMEOUTMSK | DIEPMSK_AHBERRMSK |
  3742. DIEPMSK_EPDISBLDMSK | DIEPMSK_XFERCOMPLMSK,
  3743. DIEPMSK);
  3744. dwc2_writel(hsotg, DOEPMSK_SETUPMSK | DOEPMSK_AHBERRMSK |
  3745. DOEPMSK_EPDISBLDMSK | DOEPMSK_XFERCOMPLMSK,
  3746. DOEPMSK);
  3747. dwc2_writel(hsotg, 0, DAINTMSK);
  3748. /* Be in disconnected state until gadget is registered */
  3749. dwc2_set_bit(hsotg, DCTL, DCTL_SFTDISCON);
  3750. /* setup fifos */
  3751. dev_dbg(hsotg->dev, "GRXFSIZ=0x%08x, GNPTXFSIZ=0x%08x\n",
  3752. dwc2_readl(hsotg, GRXFSIZ),
  3753. dwc2_readl(hsotg, GNPTXFSIZ));
  3754. dwc2_hsotg_init_fifo(hsotg);
  3755. if (using_dma(hsotg))
  3756. dwc2_set_bit(hsotg, GAHBCFG, GAHBCFG_DMA_EN);
  3757. }
  3758. /**
  3759. * dwc2_hsotg_udc_start - prepare the udc for work
  3760. * @gadget: The usb gadget state
  3761. * @driver: The usb gadget driver
  3762. *
  3763. * Perform initialization to prepare udc device and driver
  3764. * to work.
  3765. */
  3766. static int dwc2_hsotg_udc_start(struct usb_gadget *gadget,
  3767. struct usb_gadget_driver *driver)
  3768. {
  3769. struct dwc2_hsotg *hsotg = to_hsotg(gadget);
  3770. unsigned long flags;
  3771. int ret;
  3772. if (!hsotg) {
  3773. pr_err("%s: called with no device\n", __func__);
  3774. return -ENODEV;
  3775. }
  3776. if (!driver) {
  3777. dev_err(hsotg->dev, "%s: no driver\n", __func__);
  3778. return -EINVAL;
  3779. }
  3780. if (driver->max_speed < USB_SPEED_FULL)
  3781. dev_err(hsotg->dev, "%s: bad speed\n", __func__);
  3782. if (!driver->setup) {
  3783. dev_err(hsotg->dev, "%s: missing entry points\n", __func__);
  3784. return -EINVAL;
  3785. }
  3786. WARN_ON(hsotg->driver);
  3787. driver->driver.bus = NULL;
  3788. hsotg->driver = driver;
  3789. hsotg->gadget.dev.of_node = hsotg->dev->of_node;
  3790. hsotg->gadget.speed = USB_SPEED_UNKNOWN;
  3791. if (hsotg->dr_mode == USB_DR_MODE_PERIPHERAL) {
  3792. ret = dwc2_lowlevel_hw_enable(hsotg);
  3793. if (ret)
  3794. goto err;
  3795. }
  3796. if (!IS_ERR_OR_NULL(hsotg->uphy))
  3797. otg_set_peripheral(hsotg->uphy->otg, &hsotg->gadget);
  3798. spin_lock_irqsave(&hsotg->lock, flags);
  3799. if (dwc2_hw_is_device(hsotg)) {
  3800. dwc2_hsotg_init(hsotg);
  3801. dwc2_hsotg_core_init_disconnected(hsotg, false);
  3802. }
  3803. hsotg->enabled = 0;
  3804. spin_unlock_irqrestore(&hsotg->lock, flags);
  3805. gadget->sg_supported = using_desc_dma(hsotg);
  3806. dev_info(hsotg->dev, "bound driver %s\n", driver->driver.name);
  3807. return 0;
  3808. err:
  3809. hsotg->driver = NULL;
  3810. return ret;
  3811. }
  3812. /**
  3813. * dwc2_hsotg_udc_stop - stop the udc
  3814. * @gadget: The usb gadget state
  3815. *
  3816. * Stop udc hw block and stay tunned for future transmissions
  3817. */
  3818. static int dwc2_hsotg_udc_stop(struct usb_gadget *gadget)
  3819. {
  3820. struct dwc2_hsotg *hsotg = to_hsotg(gadget);
  3821. unsigned long flags = 0;
  3822. int ep;
  3823. if (!hsotg)
  3824. return -ENODEV;
  3825. /* all endpoints should be shutdown */
  3826. for (ep = 1; ep < hsotg->num_of_eps; ep++) {
  3827. if (hsotg->eps_in[ep])
  3828. dwc2_hsotg_ep_disable_lock(&hsotg->eps_in[ep]->ep);
  3829. if (hsotg->eps_out[ep])
  3830. dwc2_hsotg_ep_disable_lock(&hsotg->eps_out[ep]->ep);
  3831. }
  3832. spin_lock_irqsave(&hsotg->lock, flags);
  3833. hsotg->driver = NULL;
  3834. hsotg->gadget.speed = USB_SPEED_UNKNOWN;
  3835. hsotg->enabled = 0;
  3836. spin_unlock_irqrestore(&hsotg->lock, flags);
  3837. if (!IS_ERR_OR_NULL(hsotg->uphy))
  3838. otg_set_peripheral(hsotg->uphy->otg, NULL);
  3839. if (hsotg->dr_mode == USB_DR_MODE_PERIPHERAL)
  3840. dwc2_lowlevel_hw_disable(hsotg);
  3841. return 0;
  3842. }
  3843. /**
  3844. * dwc2_hsotg_gadget_getframe - read the frame number
  3845. * @gadget: The usb gadget state
  3846. *
  3847. * Read the {micro} frame number
  3848. */
  3849. static int dwc2_hsotg_gadget_getframe(struct usb_gadget *gadget)
  3850. {
  3851. return dwc2_hsotg_read_frameno(to_hsotg(gadget));
  3852. }
  3853. /**
  3854. * dwc2_hsotg_set_selfpowered - set if device is self/bus powered
  3855. * @gadget: The usb gadget state
  3856. * @is_selfpowered: Whether the device is self-powered
  3857. *
  3858. * Set if the device is self or bus powered.
  3859. */
  3860. static int dwc2_hsotg_set_selfpowered(struct usb_gadget *gadget,
  3861. int is_selfpowered)
  3862. {
  3863. struct dwc2_hsotg *hsotg = to_hsotg(gadget);
  3864. unsigned long flags;
  3865. spin_lock_irqsave(&hsotg->lock, flags);
  3866. gadget->is_selfpowered = !!is_selfpowered;
  3867. spin_unlock_irqrestore(&hsotg->lock, flags);
  3868. return 0;
  3869. }
  3870. /**
  3871. * dwc2_hsotg_pullup - connect/disconnect the USB PHY
  3872. * @gadget: The usb gadget state
  3873. * @is_on: Current state of the USB PHY
  3874. *
  3875. * Connect/Disconnect the USB PHY pullup
  3876. */
  3877. static int dwc2_hsotg_pullup(struct usb_gadget *gadget, int is_on)
  3878. {
  3879. struct dwc2_hsotg *hsotg = to_hsotg(gadget);
  3880. unsigned long flags = 0;
  3881. dev_dbg(hsotg->dev, "%s: is_on: %d op_state: %d\n", __func__, is_on,
  3882. hsotg->op_state);
  3883. /* Don't modify pullup state while in host mode */
  3884. if (hsotg->op_state != OTG_STATE_B_PERIPHERAL) {
  3885. hsotg->enabled = is_on;
  3886. return 0;
  3887. }
  3888. spin_lock_irqsave(&hsotg->lock, flags);
  3889. if (is_on) {
  3890. hsotg->enabled = 1;
  3891. dwc2_hsotg_core_init_disconnected(hsotg, false);
  3892. /* Enable ACG feature in device mode,if supported */
  3893. dwc2_enable_acg(hsotg);
  3894. dwc2_hsotg_core_connect(hsotg);
  3895. } else {
  3896. dwc2_hsotg_core_disconnect(hsotg);
  3897. dwc2_hsotg_disconnect(hsotg);
  3898. hsotg->enabled = 0;
  3899. }
  3900. hsotg->gadget.speed = USB_SPEED_UNKNOWN;
  3901. spin_unlock_irqrestore(&hsotg->lock, flags);
  3902. return 0;
  3903. }
  3904. static int dwc2_hsotg_vbus_session(struct usb_gadget *gadget, int is_active)
  3905. {
  3906. struct dwc2_hsotg *hsotg = to_hsotg(gadget);
  3907. unsigned long flags;
  3908. dev_dbg(hsotg->dev, "%s: is_active: %d\n", __func__, is_active);
  3909. spin_lock_irqsave(&hsotg->lock, flags);
  3910. /*
  3911. * If controller is hibernated, it must exit from power_down
  3912. * before being initialized / de-initialized
  3913. */
  3914. if (hsotg->lx_state == DWC2_L2)
  3915. dwc2_exit_partial_power_down(hsotg, false);
  3916. if (is_active) {
  3917. hsotg->op_state = OTG_STATE_B_PERIPHERAL;
  3918. dwc2_hsotg_core_init_disconnected(hsotg, false);
  3919. if (hsotg->enabled) {
  3920. /* Enable ACG feature in device mode,if supported */
  3921. dwc2_enable_acg(hsotg);
  3922. dwc2_hsotg_core_connect(hsotg);
  3923. }
  3924. } else {
  3925. dwc2_hsotg_core_disconnect(hsotg);
  3926. dwc2_hsotg_disconnect(hsotg);
  3927. }
  3928. spin_unlock_irqrestore(&hsotg->lock, flags);
  3929. return 0;
  3930. }
  3931. /**
  3932. * dwc2_hsotg_vbus_draw - report bMaxPower field
  3933. * @gadget: The usb gadget state
  3934. * @mA: Amount of current
  3935. *
  3936. * Report how much power the device may consume to the phy.
  3937. */
  3938. static int dwc2_hsotg_vbus_draw(struct usb_gadget *gadget, unsigned int mA)
  3939. {
  3940. struct dwc2_hsotg *hsotg = to_hsotg(gadget);
  3941. if (IS_ERR_OR_NULL(hsotg->uphy))
  3942. return -ENOTSUPP;
  3943. return usb_phy_set_power(hsotg->uphy, mA);
  3944. }
  3945. static const struct usb_gadget_ops dwc2_hsotg_gadget_ops = {
  3946. .get_frame = dwc2_hsotg_gadget_getframe,
  3947. .set_selfpowered = dwc2_hsotg_set_selfpowered,
  3948. .udc_start = dwc2_hsotg_udc_start,
  3949. .udc_stop = dwc2_hsotg_udc_stop,
  3950. .pullup = dwc2_hsotg_pullup,
  3951. .vbus_session = dwc2_hsotg_vbus_session,
  3952. .vbus_draw = dwc2_hsotg_vbus_draw,
  3953. };
  3954. /**
  3955. * dwc2_hsotg_initep - initialise a single endpoint
  3956. * @hsotg: The device state.
  3957. * @hs_ep: The endpoint to be initialised.
  3958. * @epnum: The endpoint number
  3959. * @dir_in: True if direction is in.
  3960. *
  3961. * Initialise the given endpoint (as part of the probe and device state
  3962. * creation) to give to the gadget driver. Setup the endpoint name, any
  3963. * direction information and other state that may be required.
  3964. */
  3965. static void dwc2_hsotg_initep(struct dwc2_hsotg *hsotg,
  3966. struct dwc2_hsotg_ep *hs_ep,
  3967. int epnum,
  3968. bool dir_in)
  3969. {
  3970. char *dir;
  3971. if (epnum == 0)
  3972. dir = "";
  3973. else if (dir_in)
  3974. dir = "in";
  3975. else
  3976. dir = "out";
  3977. hs_ep->dir_in = dir_in;
  3978. hs_ep->index = epnum;
  3979. snprintf(hs_ep->name, sizeof(hs_ep->name), "ep%d%s", epnum, dir);
  3980. INIT_LIST_HEAD(&hs_ep->queue);
  3981. INIT_LIST_HEAD(&hs_ep->ep.ep_list);
  3982. /* add to the list of endpoints known by the gadget driver */
  3983. if (epnum)
  3984. list_add_tail(&hs_ep->ep.ep_list, &hsotg->gadget.ep_list);
  3985. hs_ep->parent = hsotg;
  3986. hs_ep->ep.name = hs_ep->name;
  3987. if (hsotg->params.speed == DWC2_SPEED_PARAM_LOW)
  3988. usb_ep_set_maxpacket_limit(&hs_ep->ep, 8);
  3989. else
  3990. usb_ep_set_maxpacket_limit(&hs_ep->ep,
  3991. epnum ? 1024 : EP0_MPS_LIMIT);
  3992. hs_ep->ep.ops = &dwc2_hsotg_ep_ops;
  3993. if (epnum == 0) {
  3994. hs_ep->ep.caps.type_control = true;
  3995. } else {
  3996. if (hsotg->params.speed != DWC2_SPEED_PARAM_LOW) {
  3997. hs_ep->ep.caps.type_iso = true;
  3998. hs_ep->ep.caps.type_bulk = true;
  3999. }
  4000. hs_ep->ep.caps.type_int = true;
  4001. }
  4002. if (dir_in)
  4003. hs_ep->ep.caps.dir_in = true;
  4004. else
  4005. hs_ep->ep.caps.dir_out = true;
  4006. /*
  4007. * if we're using dma, we need to set the next-endpoint pointer
  4008. * to be something valid.
  4009. */
  4010. if (using_dma(hsotg)) {
  4011. u32 next = DXEPCTL_NEXTEP((epnum + 1) % 15);
  4012. if (dir_in)
  4013. dwc2_writel(hsotg, next, DIEPCTL(epnum));
  4014. else
  4015. dwc2_writel(hsotg, next, DOEPCTL(epnum));
  4016. }
  4017. }
  4018. /**
  4019. * dwc2_hsotg_hw_cfg - read HW configuration registers
  4020. * @hsotg: Programming view of the DWC_otg controller
  4021. *
  4022. * Read the USB core HW configuration registers
  4023. */
  4024. static int dwc2_hsotg_hw_cfg(struct dwc2_hsotg *hsotg)
  4025. {
  4026. u32 cfg;
  4027. u32 ep_type;
  4028. u32 i;
  4029. /* check hardware configuration */
  4030. hsotg->num_of_eps = hsotg->hw_params.num_dev_ep;
  4031. /* Add ep0 */
  4032. hsotg->num_of_eps++;
  4033. hsotg->eps_in[0] = devm_kzalloc(hsotg->dev,
  4034. sizeof(struct dwc2_hsotg_ep),
  4035. GFP_KERNEL);
  4036. if (!hsotg->eps_in[0])
  4037. return -ENOMEM;
  4038. /* Same dwc2_hsotg_ep is used in both directions for ep0 */
  4039. hsotg->eps_out[0] = hsotg->eps_in[0];
  4040. cfg = hsotg->hw_params.dev_ep_dirs;
  4041. for (i = 1, cfg >>= 2; i < hsotg->num_of_eps; i++, cfg >>= 2) {
  4042. ep_type = cfg & 3;
  4043. /* Direction in or both */
  4044. if (!(ep_type & 2)) {
  4045. hsotg->eps_in[i] = devm_kzalloc(hsotg->dev,
  4046. sizeof(struct dwc2_hsotg_ep), GFP_KERNEL);
  4047. if (!hsotg->eps_in[i])
  4048. return -ENOMEM;
  4049. }
  4050. /* Direction out or both */
  4051. if (!(ep_type & 1)) {
  4052. hsotg->eps_out[i] = devm_kzalloc(hsotg->dev,
  4053. sizeof(struct dwc2_hsotg_ep), GFP_KERNEL);
  4054. if (!hsotg->eps_out[i])
  4055. return -ENOMEM;
  4056. }
  4057. }
  4058. hsotg->fifo_mem = hsotg->hw_params.total_fifo_size;
  4059. hsotg->dedicated_fifos = hsotg->hw_params.en_multiple_tx_fifo;
  4060. dev_info(hsotg->dev, "EPs: %d, %s fifos, %d entries in SPRAM\n",
  4061. hsotg->num_of_eps,
  4062. hsotg->dedicated_fifos ? "dedicated" : "shared",
  4063. hsotg->fifo_mem);
  4064. return 0;
  4065. }
  4066. /**
  4067. * dwc2_hsotg_dump - dump state of the udc
  4068. * @hsotg: Programming view of the DWC_otg controller
  4069. *
  4070. */
  4071. static void dwc2_hsotg_dump(struct dwc2_hsotg *hsotg)
  4072. {
  4073. #ifdef DEBUG
  4074. struct device *dev = hsotg->dev;
  4075. u32 val;
  4076. int idx;
  4077. dev_info(dev, "DCFG=0x%08x, DCTL=0x%08x, DIEPMSK=%08x\n",
  4078. dwc2_readl(hsotg, DCFG), dwc2_readl(hsotg, DCTL),
  4079. dwc2_readl(hsotg, DIEPMSK));
  4080. dev_info(dev, "GAHBCFG=0x%08x, GHWCFG1=0x%08x\n",
  4081. dwc2_readl(hsotg, GAHBCFG), dwc2_readl(hsotg, GHWCFG1));
  4082. dev_info(dev, "GRXFSIZ=0x%08x, GNPTXFSIZ=0x%08x\n",
  4083. dwc2_readl(hsotg, GRXFSIZ), dwc2_readl(hsotg, GNPTXFSIZ));
  4084. /* show periodic fifo settings */
  4085. for (idx = 1; idx < hsotg->num_of_eps; idx++) {
  4086. val = dwc2_readl(hsotg, DPTXFSIZN(idx));
  4087. dev_info(dev, "DPTx[%d] FSize=%d, StAddr=0x%08x\n", idx,
  4088. val >> FIFOSIZE_DEPTH_SHIFT,
  4089. val & FIFOSIZE_STARTADDR_MASK);
  4090. }
  4091. for (idx = 0; idx < hsotg->num_of_eps; idx++) {
  4092. dev_info(dev,
  4093. "ep%d-in: EPCTL=0x%08x, SIZ=0x%08x, DMA=0x%08x\n", idx,
  4094. dwc2_readl(hsotg, DIEPCTL(idx)),
  4095. dwc2_readl(hsotg, DIEPTSIZ(idx)),
  4096. dwc2_readl(hsotg, DIEPDMA(idx)));
  4097. val = dwc2_readl(hsotg, DOEPCTL(idx));
  4098. dev_info(dev,
  4099. "ep%d-out: EPCTL=0x%08x, SIZ=0x%08x, DMA=0x%08x\n",
  4100. idx, dwc2_readl(hsotg, DOEPCTL(idx)),
  4101. dwc2_readl(hsotg, DOEPTSIZ(idx)),
  4102. dwc2_readl(hsotg, DOEPDMA(idx)));
  4103. }
  4104. dev_info(dev, "DVBUSDIS=0x%08x, DVBUSPULSE=%08x\n",
  4105. dwc2_readl(hsotg, DVBUSDIS), dwc2_readl(hsotg, DVBUSPULSE));
  4106. #endif
  4107. }
  4108. /**
  4109. * dwc2_gadget_init - init function for gadget
  4110. * @hsotg: Programming view of the DWC_otg controller
  4111. *
  4112. */
  4113. int dwc2_gadget_init(struct dwc2_hsotg *hsotg)
  4114. {
  4115. struct device *dev = hsotg->dev;
  4116. int epnum;
  4117. int ret;
  4118. /* Dump fifo information */
  4119. dev_dbg(dev, "NonPeriodic TXFIFO size: %d\n",
  4120. hsotg->params.g_np_tx_fifo_size);
  4121. dev_dbg(dev, "RXFIFO size: %d\n", hsotg->params.g_rx_fifo_size);
  4122. hsotg->gadget.max_speed = USB_SPEED_HIGH;
  4123. hsotg->gadget.ops = &dwc2_hsotg_gadget_ops;
  4124. hsotg->gadget.name = dev_name(dev);
  4125. hsotg->remote_wakeup_allowed = 0;
  4126. if (hsotg->params.lpm)
  4127. hsotg->gadget.lpm_capable = true;
  4128. if (hsotg->dr_mode == USB_DR_MODE_OTG)
  4129. hsotg->gadget.is_otg = 1;
  4130. else if (hsotg->dr_mode == USB_DR_MODE_PERIPHERAL)
  4131. hsotg->op_state = OTG_STATE_B_PERIPHERAL;
  4132. ret = dwc2_hsotg_hw_cfg(hsotg);
  4133. if (ret) {
  4134. dev_err(hsotg->dev, "Hardware configuration failed: %d\n", ret);
  4135. return ret;
  4136. }
  4137. hsotg->ctrl_buff = devm_kzalloc(hsotg->dev,
  4138. DWC2_CTRL_BUFF_SIZE, GFP_KERNEL);
  4139. if (!hsotg->ctrl_buff)
  4140. return -ENOMEM;
  4141. hsotg->ep0_buff = devm_kzalloc(hsotg->dev,
  4142. DWC2_CTRL_BUFF_SIZE, GFP_KERNEL);
  4143. if (!hsotg->ep0_buff)
  4144. return -ENOMEM;
  4145. if (using_desc_dma(hsotg)) {
  4146. ret = dwc2_gadget_alloc_ctrl_desc_chains(hsotg);
  4147. if (ret < 0)
  4148. return ret;
  4149. }
  4150. ret = devm_request_irq(hsotg->dev, hsotg->irq, dwc2_hsotg_irq,
  4151. IRQF_SHARED, dev_name(hsotg->dev), hsotg);
  4152. if (ret < 0) {
  4153. dev_err(dev, "cannot claim IRQ for gadget\n");
  4154. return ret;
  4155. }
  4156. /* hsotg->num_of_eps holds number of EPs other than ep0 */
  4157. if (hsotg->num_of_eps == 0) {
  4158. dev_err(dev, "wrong number of EPs (zero)\n");
  4159. return -EINVAL;
  4160. }
  4161. /* setup endpoint information */
  4162. INIT_LIST_HEAD(&hsotg->gadget.ep_list);
  4163. hsotg->gadget.ep0 = &hsotg->eps_out[0]->ep;
  4164. /* allocate EP0 request */
  4165. hsotg->ctrl_req = dwc2_hsotg_ep_alloc_request(&hsotg->eps_out[0]->ep,
  4166. GFP_KERNEL);
  4167. if (!hsotg->ctrl_req) {
  4168. dev_err(dev, "failed to allocate ctrl req\n");
  4169. return -ENOMEM;
  4170. }
  4171. /* initialise the endpoints now the core has been initialised */
  4172. for (epnum = 0; epnum < hsotg->num_of_eps; epnum++) {
  4173. if (hsotg->eps_in[epnum])
  4174. dwc2_hsotg_initep(hsotg, hsotg->eps_in[epnum],
  4175. epnum, 1);
  4176. if (hsotg->eps_out[epnum])
  4177. dwc2_hsotg_initep(hsotg, hsotg->eps_out[epnum],
  4178. epnum, 0);
  4179. }
  4180. dwc2_hsotg_dump(hsotg);
  4181. return 0;
  4182. }
  4183. /**
  4184. * dwc2_hsotg_remove - remove function for hsotg driver
  4185. * @hsotg: Programming view of the DWC_otg controller
  4186. *
  4187. */
  4188. int dwc2_hsotg_remove(struct dwc2_hsotg *hsotg)
  4189. {
  4190. usb_del_gadget_udc(&hsotg->gadget);
  4191. dwc2_hsotg_ep_free_request(&hsotg->eps_out[0]->ep, hsotg->ctrl_req);
  4192. return 0;
  4193. }
  4194. int dwc2_hsotg_suspend(struct dwc2_hsotg *hsotg)
  4195. {
  4196. unsigned long flags;
  4197. if (hsotg->lx_state != DWC2_L0)
  4198. return 0;
  4199. if (hsotg->driver) {
  4200. int ep;
  4201. dev_info(hsotg->dev, "suspending usb gadget %s\n",
  4202. hsotg->driver->driver.name);
  4203. spin_lock_irqsave(&hsotg->lock, flags);
  4204. if (hsotg->enabled)
  4205. dwc2_hsotg_core_disconnect(hsotg);
  4206. dwc2_hsotg_disconnect(hsotg);
  4207. hsotg->gadget.speed = USB_SPEED_UNKNOWN;
  4208. spin_unlock_irqrestore(&hsotg->lock, flags);
  4209. for (ep = 1; ep < hsotg->num_of_eps; ep++) {
  4210. if (hsotg->eps_in[ep])
  4211. dwc2_hsotg_ep_disable_lock(&hsotg->eps_in[ep]->ep);
  4212. if (hsotg->eps_out[ep])
  4213. dwc2_hsotg_ep_disable_lock(&hsotg->eps_out[ep]->ep);
  4214. }
  4215. }
  4216. return 0;
  4217. }
  4218. int dwc2_hsotg_resume(struct dwc2_hsotg *hsotg)
  4219. {
  4220. unsigned long flags;
  4221. if (hsotg->lx_state == DWC2_L2)
  4222. return 0;
  4223. if (hsotg->driver) {
  4224. dev_info(hsotg->dev, "resuming usb gadget %s\n",
  4225. hsotg->driver->driver.name);
  4226. spin_lock_irqsave(&hsotg->lock, flags);
  4227. dwc2_hsotg_core_init_disconnected(hsotg, false);
  4228. if (hsotg->enabled) {
  4229. /* Enable ACG feature in device mode,if supported */
  4230. dwc2_enable_acg(hsotg);
  4231. dwc2_hsotg_core_connect(hsotg);
  4232. }
  4233. spin_unlock_irqrestore(&hsotg->lock, flags);
  4234. }
  4235. return 0;
  4236. }
  4237. /**
  4238. * dwc2_backup_device_registers() - Backup controller device registers.
  4239. * When suspending usb bus, registers needs to be backuped
  4240. * if controller power is disabled once suspended.
  4241. *
  4242. * @hsotg: Programming view of the DWC_otg controller
  4243. */
  4244. int dwc2_backup_device_registers(struct dwc2_hsotg *hsotg)
  4245. {
  4246. struct dwc2_dregs_backup *dr;
  4247. int i;
  4248. dev_dbg(hsotg->dev, "%s\n", __func__);
  4249. /* Backup dev regs */
  4250. dr = &hsotg->dr_backup;
  4251. dr->dcfg = dwc2_readl(hsotg, DCFG);
  4252. dr->dctl = dwc2_readl(hsotg, DCTL);
  4253. dr->daintmsk = dwc2_readl(hsotg, DAINTMSK);
  4254. dr->diepmsk = dwc2_readl(hsotg, DIEPMSK);
  4255. dr->doepmsk = dwc2_readl(hsotg, DOEPMSK);
  4256. for (i = 0; i < hsotg->num_of_eps; i++) {
  4257. /* Backup IN EPs */
  4258. dr->diepctl[i] = dwc2_readl(hsotg, DIEPCTL(i));
  4259. /* Ensure DATA PID is correctly configured */
  4260. if (dr->diepctl[i] & DXEPCTL_DPID)
  4261. dr->diepctl[i] |= DXEPCTL_SETD1PID;
  4262. else
  4263. dr->diepctl[i] |= DXEPCTL_SETD0PID;
  4264. dr->dieptsiz[i] = dwc2_readl(hsotg, DIEPTSIZ(i));
  4265. dr->diepdma[i] = dwc2_readl(hsotg, DIEPDMA(i));
  4266. /* Backup OUT EPs */
  4267. dr->doepctl[i] = dwc2_readl(hsotg, DOEPCTL(i));
  4268. /* Ensure DATA PID is correctly configured */
  4269. if (dr->doepctl[i] & DXEPCTL_DPID)
  4270. dr->doepctl[i] |= DXEPCTL_SETD1PID;
  4271. else
  4272. dr->doepctl[i] |= DXEPCTL_SETD0PID;
  4273. dr->doeptsiz[i] = dwc2_readl(hsotg, DOEPTSIZ(i));
  4274. dr->doepdma[i] = dwc2_readl(hsotg, DOEPDMA(i));
  4275. dr->dtxfsiz[i] = dwc2_readl(hsotg, DPTXFSIZN(i));
  4276. }
  4277. dr->valid = true;
  4278. return 0;
  4279. }
  4280. /**
  4281. * dwc2_restore_device_registers() - Restore controller device registers.
  4282. * When resuming usb bus, device registers needs to be restored
  4283. * if controller power were disabled.
  4284. *
  4285. * @hsotg: Programming view of the DWC_otg controller
  4286. * @remote_wakeup: Indicates whether resume is initiated by Device or Host.
  4287. *
  4288. * Return: 0 if successful, negative error code otherwise
  4289. */
  4290. int dwc2_restore_device_registers(struct dwc2_hsotg *hsotg, int remote_wakeup)
  4291. {
  4292. struct dwc2_dregs_backup *dr;
  4293. int i;
  4294. dev_dbg(hsotg->dev, "%s\n", __func__);
  4295. /* Restore dev regs */
  4296. dr = &hsotg->dr_backup;
  4297. if (!dr->valid) {
  4298. dev_err(hsotg->dev, "%s: no device registers to restore\n",
  4299. __func__);
  4300. return -EINVAL;
  4301. }
  4302. dr->valid = false;
  4303. if (!remote_wakeup)
  4304. dwc2_writel(hsotg, dr->dctl, DCTL);
  4305. dwc2_writel(hsotg, dr->daintmsk, DAINTMSK);
  4306. dwc2_writel(hsotg, dr->diepmsk, DIEPMSK);
  4307. dwc2_writel(hsotg, dr->doepmsk, DOEPMSK);
  4308. for (i = 0; i < hsotg->num_of_eps; i++) {
  4309. /* Restore IN EPs */
  4310. dwc2_writel(hsotg, dr->dieptsiz[i], DIEPTSIZ(i));
  4311. dwc2_writel(hsotg, dr->diepdma[i], DIEPDMA(i));
  4312. dwc2_writel(hsotg, dr->doeptsiz[i], DOEPTSIZ(i));
  4313. /** WA for enabled EPx's IN in DDMA mode. On entering to
  4314. * hibernation wrong value read and saved from DIEPDMAx,
  4315. * as result BNA interrupt asserted on hibernation exit
  4316. * by restoring from saved area.
  4317. */
  4318. if (hsotg->params.g_dma_desc &&
  4319. (dr->diepctl[i] & DXEPCTL_EPENA))
  4320. dr->diepdma[i] = hsotg->eps_in[i]->desc_list_dma;
  4321. dwc2_writel(hsotg, dr->dtxfsiz[i], DPTXFSIZN(i));
  4322. dwc2_writel(hsotg, dr->diepctl[i], DIEPCTL(i));
  4323. /* Restore OUT EPs */
  4324. dwc2_writel(hsotg, dr->doeptsiz[i], DOEPTSIZ(i));
  4325. /* WA for enabled EPx's OUT in DDMA mode. On entering to
  4326. * hibernation wrong value read and saved from DOEPDMAx,
  4327. * as result BNA interrupt asserted on hibernation exit
  4328. * by restoring from saved area.
  4329. */
  4330. if (hsotg->params.g_dma_desc &&
  4331. (dr->doepctl[i] & DXEPCTL_EPENA))
  4332. dr->doepdma[i] = hsotg->eps_out[i]->desc_list_dma;
  4333. dwc2_writel(hsotg, dr->doepdma[i], DOEPDMA(i));
  4334. dwc2_writel(hsotg, dr->doepctl[i], DOEPCTL(i));
  4335. }
  4336. return 0;
  4337. }
  4338. /**
  4339. * dwc2_gadget_init_lpm - Configure the core to support LPM in device mode
  4340. *
  4341. * @hsotg: Programming view of DWC_otg controller
  4342. *
  4343. */
  4344. void dwc2_gadget_init_lpm(struct dwc2_hsotg *hsotg)
  4345. {
  4346. u32 val;
  4347. if (!hsotg->params.lpm)
  4348. return;
  4349. val = GLPMCFG_LPMCAP | GLPMCFG_APPL1RES;
  4350. val |= hsotg->params.hird_threshold_en ? GLPMCFG_HIRD_THRES_EN : 0;
  4351. val |= hsotg->params.lpm_clock_gating ? GLPMCFG_ENBLSLPM : 0;
  4352. val |= hsotg->params.hird_threshold << GLPMCFG_HIRD_THRES_SHIFT;
  4353. val |= hsotg->params.besl ? GLPMCFG_ENBESL : 0;
  4354. val |= GLPMCFG_LPM_REJECT_CTRL_CONTROL;
  4355. val |= GLPMCFG_LPM_ACCEPT_CTRL_ISOC;
  4356. dwc2_writel(hsotg, val, GLPMCFG);
  4357. dev_dbg(hsotg->dev, "GLPMCFG=0x%08x\n", dwc2_readl(hsotg, GLPMCFG));
  4358. /* Unmask WKUP_ALERT Interrupt */
  4359. if (hsotg->params.service_interval)
  4360. dwc2_set_bit(hsotg, GINTMSK2, GINTMSK2_WKUP_ALERT_INT_MSK);
  4361. }
  4362. /**
  4363. * dwc2_gadget_program_ref_clk - Program GREFCLK register in device mode
  4364. *
  4365. * @hsotg: Programming view of DWC_otg controller
  4366. *
  4367. */
  4368. void dwc2_gadget_program_ref_clk(struct dwc2_hsotg *hsotg)
  4369. {
  4370. u32 val = 0;
  4371. val |= GREFCLK_REF_CLK_MODE;
  4372. val |= hsotg->params.ref_clk_per << GREFCLK_REFCLKPER_SHIFT;
  4373. val |= hsotg->params.sof_cnt_wkup_alert <<
  4374. GREFCLK_SOF_CNT_WKUP_ALERT_SHIFT;
  4375. dwc2_writel(hsotg, val, GREFCLK);
  4376. dev_dbg(hsotg->dev, "GREFCLK=0x%08x\n", dwc2_readl(hsotg, GREFCLK));
  4377. }
  4378. /**
  4379. * dwc2_gadget_enter_hibernation() - Put controller in Hibernation.
  4380. *
  4381. * @hsotg: Programming view of the DWC_otg controller
  4382. *
  4383. * Return non-zero if failed to enter to hibernation.
  4384. */
  4385. int dwc2_gadget_enter_hibernation(struct dwc2_hsotg *hsotg)
  4386. {
  4387. u32 gpwrdn;
  4388. int ret = 0;
  4389. /* Change to L2(suspend) state */
  4390. hsotg->lx_state = DWC2_L2;
  4391. dev_dbg(hsotg->dev, "Start of hibernation completed\n");
  4392. ret = dwc2_backup_global_registers(hsotg);
  4393. if (ret) {
  4394. dev_err(hsotg->dev, "%s: failed to backup global registers\n",
  4395. __func__);
  4396. return ret;
  4397. }
  4398. ret = dwc2_backup_device_registers(hsotg);
  4399. if (ret) {
  4400. dev_err(hsotg->dev, "%s: failed to backup device registers\n",
  4401. __func__);
  4402. return ret;
  4403. }
  4404. gpwrdn = GPWRDN_PWRDNRSTN;
  4405. gpwrdn |= GPWRDN_PMUACTV;
  4406. dwc2_writel(hsotg, gpwrdn, GPWRDN);
  4407. udelay(10);
  4408. /* Set flag to indicate that we are in hibernation */
  4409. hsotg->hibernated = 1;
  4410. /* Enable interrupts from wake up logic */
  4411. gpwrdn = dwc2_readl(hsotg, GPWRDN);
  4412. gpwrdn |= GPWRDN_PMUINTSEL;
  4413. dwc2_writel(hsotg, gpwrdn, GPWRDN);
  4414. udelay(10);
  4415. /* Unmask device mode interrupts in GPWRDN */
  4416. gpwrdn = dwc2_readl(hsotg, GPWRDN);
  4417. gpwrdn |= GPWRDN_RST_DET_MSK;
  4418. gpwrdn |= GPWRDN_LNSTSCHG_MSK;
  4419. gpwrdn |= GPWRDN_STS_CHGINT_MSK;
  4420. dwc2_writel(hsotg, gpwrdn, GPWRDN);
  4421. udelay(10);
  4422. /* Enable Power Down Clamp */
  4423. gpwrdn = dwc2_readl(hsotg, GPWRDN);
  4424. gpwrdn |= GPWRDN_PWRDNCLMP;
  4425. dwc2_writel(hsotg, gpwrdn, GPWRDN);
  4426. udelay(10);
  4427. /* Switch off VDD */
  4428. gpwrdn = dwc2_readl(hsotg, GPWRDN);
  4429. gpwrdn |= GPWRDN_PWRDNSWTCH;
  4430. dwc2_writel(hsotg, gpwrdn, GPWRDN);
  4431. udelay(10);
  4432. /* Save gpwrdn register for further usage if stschng interrupt */
  4433. hsotg->gr_backup.gpwrdn = dwc2_readl(hsotg, GPWRDN);
  4434. dev_dbg(hsotg->dev, "Hibernation completed\n");
  4435. return ret;
  4436. }
  4437. /**
  4438. * dwc2_gadget_exit_hibernation()
  4439. * This function is for exiting from Device mode hibernation by host initiated
  4440. * resume/reset and device initiated remote-wakeup.
  4441. *
  4442. * @hsotg: Programming view of the DWC_otg controller
  4443. * @rem_wakeup: indicates whether resume is initiated by Device or Host.
  4444. * @reset: indicates whether resume is initiated by Reset.
  4445. *
  4446. * Return non-zero if failed to exit from hibernation.
  4447. */
  4448. int dwc2_gadget_exit_hibernation(struct dwc2_hsotg *hsotg,
  4449. int rem_wakeup, int reset)
  4450. {
  4451. u32 pcgcctl;
  4452. u32 gpwrdn;
  4453. u32 dctl;
  4454. int ret = 0;
  4455. struct dwc2_gregs_backup *gr;
  4456. struct dwc2_dregs_backup *dr;
  4457. gr = &hsotg->gr_backup;
  4458. dr = &hsotg->dr_backup;
  4459. if (!hsotg->hibernated) {
  4460. dev_dbg(hsotg->dev, "Already exited from Hibernation\n");
  4461. return 1;
  4462. }
  4463. dev_dbg(hsotg->dev,
  4464. "%s: called with rem_wakeup = %d reset = %d\n",
  4465. __func__, rem_wakeup, reset);
  4466. dwc2_hib_restore_common(hsotg, rem_wakeup, 0);
  4467. if (!reset) {
  4468. /* Clear all pending interupts */
  4469. dwc2_writel(hsotg, 0xffffffff, GINTSTS);
  4470. }
  4471. /* De-assert Restore */
  4472. gpwrdn = dwc2_readl(hsotg, GPWRDN);
  4473. gpwrdn &= ~GPWRDN_RESTORE;
  4474. dwc2_writel(hsotg, gpwrdn, GPWRDN);
  4475. udelay(10);
  4476. if (!rem_wakeup) {
  4477. pcgcctl = dwc2_readl(hsotg, PCGCTL);
  4478. pcgcctl &= ~PCGCTL_RSTPDWNMODULE;
  4479. dwc2_writel(hsotg, pcgcctl, PCGCTL);
  4480. }
  4481. /* Restore GUSBCFG, DCFG and DCTL */
  4482. dwc2_writel(hsotg, gr->gusbcfg, GUSBCFG);
  4483. dwc2_writel(hsotg, dr->dcfg, DCFG);
  4484. dwc2_writel(hsotg, dr->dctl, DCTL);
  4485. /* De-assert Wakeup Logic */
  4486. gpwrdn = dwc2_readl(hsotg, GPWRDN);
  4487. gpwrdn &= ~GPWRDN_PMUACTV;
  4488. dwc2_writel(hsotg, gpwrdn, GPWRDN);
  4489. if (rem_wakeup) {
  4490. udelay(10);
  4491. /* Start Remote Wakeup Signaling */
  4492. dwc2_writel(hsotg, dr->dctl | DCTL_RMTWKUPSIG, DCTL);
  4493. } else {
  4494. udelay(50);
  4495. /* Set Device programming done bit */
  4496. dctl = dwc2_readl(hsotg, DCTL);
  4497. dctl |= DCTL_PWRONPRGDONE;
  4498. dwc2_writel(hsotg, dctl, DCTL);
  4499. }
  4500. /* Wait for interrupts which must be cleared */
  4501. mdelay(2);
  4502. /* Clear all pending interupts */
  4503. dwc2_writel(hsotg, 0xffffffff, GINTSTS);
  4504. /* Restore global registers */
  4505. ret = dwc2_restore_global_registers(hsotg);
  4506. if (ret) {
  4507. dev_err(hsotg->dev, "%s: failed to restore registers\n",
  4508. __func__);
  4509. return ret;
  4510. }
  4511. /* Restore device registers */
  4512. ret = dwc2_restore_device_registers(hsotg, rem_wakeup);
  4513. if (ret) {
  4514. dev_err(hsotg->dev, "%s: failed to restore device registers\n",
  4515. __func__);
  4516. return ret;
  4517. }
  4518. if (rem_wakeup) {
  4519. mdelay(10);
  4520. dctl = dwc2_readl(hsotg, DCTL);
  4521. dctl &= ~DCTL_RMTWKUPSIG;
  4522. dwc2_writel(hsotg, dctl, DCTL);
  4523. }
  4524. hsotg->hibernated = 0;
  4525. hsotg->lx_state = DWC2_L0;
  4526. dev_dbg(hsotg->dev, "Hibernation recovery completes here\n");
  4527. return ret;
  4528. }