k3-udma.c 92 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
  4. * Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
  5. */
  6. #include <linux/kernel.h>
  7. #include <linux/delay.h>
  8. #include <linux/dmaengine.h>
  9. #include <linux/dma-mapping.h>
  10. #include <linux/dmapool.h>
  11. #include <linux/err.h>
  12. #include <linux/init.h>
  13. #include <linux/interrupt.h>
  14. #include <linux/list.h>
  15. #include <linux/platform_device.h>
  16. #include <linux/slab.h>
  17. #include <linux/spinlock.h>
  18. #include <linux/sys_soc.h>
  19. #include <linux/of.h>
  20. #include <linux/of_dma.h>
  21. #include <linux/of_device.h>
  22. #include <linux/of_irq.h>
  23. #include <linux/workqueue.h>
  24. #include <linux/completion.h>
  25. #include <linux/soc/ti/k3-ringacc.h>
  26. #include <linux/soc/ti/ti_sci_protocol.h>
  27. #include <linux/soc/ti/ti_sci_inta_msi.h>
  28. #include <linux/dma/ti-cppi5.h>
  29. #include "../virt-dma.h"
  30. #include "k3-udma.h"
  31. #include "k3-psil-priv.h"
  32. struct udma_static_tr {
  33. u8 elsize; /* RPSTR0 */
  34. u16 elcnt; /* RPSTR0 */
  35. u16 bstcnt; /* RPSTR1 */
  36. };
  37. #define K3_UDMA_MAX_RFLOWS 1024
  38. #define K3_UDMA_DEFAULT_RING_SIZE 16
  39. /* How SRC/DST tag should be updated by UDMA in the descriptor's Word 3 */
  40. #define UDMA_RFLOW_SRCTAG_NONE 0
  41. #define UDMA_RFLOW_SRCTAG_CFG_TAG 1
  42. #define UDMA_RFLOW_SRCTAG_FLOW_ID 2
  43. #define UDMA_RFLOW_SRCTAG_SRC_TAG 4
  44. #define UDMA_RFLOW_DSTTAG_NONE 0
  45. #define UDMA_RFLOW_DSTTAG_CFG_TAG 1
  46. #define UDMA_RFLOW_DSTTAG_FLOW_ID 2
  47. #define UDMA_RFLOW_DSTTAG_DST_TAG_LO 4
  48. #define UDMA_RFLOW_DSTTAG_DST_TAG_HI 5
  49. struct udma_chan;
  50. enum udma_mmr {
  51. MMR_GCFG = 0,
  52. MMR_RCHANRT,
  53. MMR_TCHANRT,
  54. MMR_LAST,
  55. };
  56. static const char * const mmr_names[] = { "gcfg", "rchanrt", "tchanrt" };
  57. struct udma_tchan {
  58. void __iomem *reg_rt;
  59. int id;
  60. struct k3_ring *t_ring; /* Transmit ring */
  61. struct k3_ring *tc_ring; /* Transmit Completion ring */
  62. };
  63. struct udma_rflow {
  64. int id;
  65. struct k3_ring *fd_ring; /* Free Descriptor ring */
  66. struct k3_ring *r_ring; /* Receive ring */
  67. };
  68. struct udma_rchan {
  69. void __iomem *reg_rt;
  70. int id;
  71. };
  72. #define UDMA_FLAG_PDMA_ACC32 BIT(0)
  73. #define UDMA_FLAG_PDMA_BURST BIT(1)
  74. struct udma_match_data {
  75. u32 psil_base;
  76. bool enable_memcpy_support;
  77. u32 flags;
  78. u32 statictr_z_mask;
  79. };
  80. struct udma_soc_data {
  81. u32 rchan_oes_offset;
  82. };
  83. struct udma_hwdesc {
  84. size_t cppi5_desc_size;
  85. void *cppi5_desc_vaddr;
  86. dma_addr_t cppi5_desc_paddr;
  87. /* TR descriptor internal pointers */
  88. void *tr_req_base;
  89. struct cppi5_tr_resp_t *tr_resp_base;
  90. };
  91. struct udma_rx_flush {
  92. struct udma_hwdesc hwdescs[2];
  93. size_t buffer_size;
  94. void *buffer_vaddr;
  95. dma_addr_t buffer_paddr;
  96. };
  97. struct udma_dev {
  98. struct dma_device ddev;
  99. struct device *dev;
  100. void __iomem *mmrs[MMR_LAST];
  101. const struct udma_match_data *match_data;
  102. const struct udma_soc_data *soc_data;
  103. u8 tpl_levels;
  104. u32 tpl_start_idx[3];
  105. size_t desc_align; /* alignment to use for descriptors */
  106. struct udma_tisci_rm tisci_rm;
  107. struct k3_ringacc *ringacc;
  108. struct work_struct purge_work;
  109. struct list_head desc_to_purge;
  110. spinlock_t lock;
  111. struct udma_rx_flush rx_flush;
  112. int tchan_cnt;
  113. int echan_cnt;
  114. int rchan_cnt;
  115. int rflow_cnt;
  116. unsigned long *tchan_map;
  117. unsigned long *rchan_map;
  118. unsigned long *rflow_gp_map;
  119. unsigned long *rflow_gp_map_allocated;
  120. unsigned long *rflow_in_use;
  121. struct udma_tchan *tchans;
  122. struct udma_rchan *rchans;
  123. struct udma_rflow *rflows;
  124. struct udma_chan *channels;
  125. u32 psil_base;
  126. u32 atype;
  127. };
  128. struct udma_desc {
  129. struct virt_dma_desc vd;
  130. bool terminated;
  131. enum dma_transfer_direction dir;
  132. struct udma_static_tr static_tr;
  133. u32 residue;
  134. unsigned int sglen;
  135. unsigned int desc_idx; /* Only used for cyclic in packet mode */
  136. unsigned int tr_idx;
  137. u32 metadata_size;
  138. void *metadata; /* pointer to provided metadata buffer (EPIP, PSdata) */
  139. unsigned int hwdesc_count;
  140. struct udma_hwdesc hwdesc[];
  141. };
  142. enum udma_chan_state {
  143. UDMA_CHAN_IS_IDLE = 0, /* not active, no teardown is in progress */
  144. UDMA_CHAN_IS_ACTIVE, /* Normal operation */
  145. UDMA_CHAN_IS_TERMINATING, /* channel is being terminated */
  146. };
  147. struct udma_tx_drain {
  148. struct delayed_work work;
  149. ktime_t tstamp;
  150. u32 residue;
  151. };
  152. struct udma_chan_config {
  153. bool pkt_mode; /* TR or packet */
  154. bool needs_epib; /* EPIB is needed for the communication or not */
  155. u32 psd_size; /* size of Protocol Specific Data */
  156. u32 metadata_size; /* (needs_epib ? 16:0) + psd_size */
  157. u32 hdesc_size; /* Size of a packet descriptor in packet mode */
  158. bool notdpkt; /* Suppress sending TDC packet */
  159. int remote_thread_id;
  160. u32 atype;
  161. u32 src_thread;
  162. u32 dst_thread;
  163. enum psil_endpoint_type ep_type;
  164. bool enable_acc32;
  165. bool enable_burst;
  166. enum udma_tp_level channel_tpl; /* Channel Throughput Level */
  167. enum dma_transfer_direction dir;
  168. };
  169. struct udma_chan {
  170. struct virt_dma_chan vc;
  171. struct dma_slave_config cfg;
  172. struct udma_dev *ud;
  173. struct udma_desc *desc;
  174. struct udma_desc *terminated_desc;
  175. struct udma_static_tr static_tr;
  176. char *name;
  177. struct udma_tchan *tchan;
  178. struct udma_rchan *rchan;
  179. struct udma_rflow *rflow;
  180. bool psil_paired;
  181. int irq_num_ring;
  182. int irq_num_udma;
  183. bool cyclic;
  184. bool paused;
  185. enum udma_chan_state state;
  186. struct completion teardown_completed;
  187. struct udma_tx_drain tx_drain;
  188. u32 bcnt; /* number of bytes completed since the start of the channel */
  189. /* Channel configuration parameters */
  190. struct udma_chan_config config;
  191. /* dmapool for packet mode descriptors */
  192. bool use_dma_pool;
  193. struct dma_pool *hdesc_pool;
  194. u32 id;
  195. };
  196. static inline struct udma_dev *to_udma_dev(struct dma_device *d)
  197. {
  198. return container_of(d, struct udma_dev, ddev);
  199. }
  200. static inline struct udma_chan *to_udma_chan(struct dma_chan *c)
  201. {
  202. return container_of(c, struct udma_chan, vc.chan);
  203. }
  204. static inline struct udma_desc *to_udma_desc(struct dma_async_tx_descriptor *t)
  205. {
  206. return container_of(t, struct udma_desc, vd.tx);
  207. }
  208. /* Generic register access functions */
  209. static inline u32 udma_read(void __iomem *base, int reg)
  210. {
  211. return readl(base + reg);
  212. }
  213. static inline void udma_write(void __iomem *base, int reg, u32 val)
  214. {
  215. writel(val, base + reg);
  216. }
  217. static inline void udma_update_bits(void __iomem *base, int reg,
  218. u32 mask, u32 val)
  219. {
  220. u32 tmp, orig;
  221. orig = readl(base + reg);
  222. tmp = orig & ~mask;
  223. tmp |= (val & mask);
  224. if (tmp != orig)
  225. writel(tmp, base + reg);
  226. }
  227. /* TCHANRT */
  228. static inline u32 udma_tchanrt_read(struct udma_chan *uc, int reg)
  229. {
  230. if (!uc->tchan)
  231. return 0;
  232. return udma_read(uc->tchan->reg_rt, reg);
  233. }
  234. static inline void udma_tchanrt_write(struct udma_chan *uc, int reg, u32 val)
  235. {
  236. if (!uc->tchan)
  237. return;
  238. udma_write(uc->tchan->reg_rt, reg, val);
  239. }
  240. static inline void udma_tchanrt_update_bits(struct udma_chan *uc, int reg,
  241. u32 mask, u32 val)
  242. {
  243. if (!uc->tchan)
  244. return;
  245. udma_update_bits(uc->tchan->reg_rt, reg, mask, val);
  246. }
  247. /* RCHANRT */
  248. static inline u32 udma_rchanrt_read(struct udma_chan *uc, int reg)
  249. {
  250. if (!uc->rchan)
  251. return 0;
  252. return udma_read(uc->rchan->reg_rt, reg);
  253. }
  254. static inline void udma_rchanrt_write(struct udma_chan *uc, int reg, u32 val)
  255. {
  256. if (!uc->rchan)
  257. return;
  258. udma_write(uc->rchan->reg_rt, reg, val);
  259. }
  260. static inline void udma_rchanrt_update_bits(struct udma_chan *uc, int reg,
  261. u32 mask, u32 val)
  262. {
  263. if (!uc->rchan)
  264. return;
  265. udma_update_bits(uc->rchan->reg_rt, reg, mask, val);
  266. }
  267. static int navss_psil_pair(struct udma_dev *ud, u32 src_thread, u32 dst_thread)
  268. {
  269. struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
  270. dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;
  271. return tisci_rm->tisci_psil_ops->pair(tisci_rm->tisci,
  272. tisci_rm->tisci_navss_dev_id,
  273. src_thread, dst_thread);
  274. }
  275. static int navss_psil_unpair(struct udma_dev *ud, u32 src_thread,
  276. u32 dst_thread)
  277. {
  278. struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
  279. dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;
  280. return tisci_rm->tisci_psil_ops->unpair(tisci_rm->tisci,
  281. tisci_rm->tisci_navss_dev_id,
  282. src_thread, dst_thread);
  283. }
  284. static void udma_reset_uchan(struct udma_chan *uc)
  285. {
  286. memset(&uc->config, 0, sizeof(uc->config));
  287. uc->config.remote_thread_id = -1;
  288. uc->state = UDMA_CHAN_IS_IDLE;
  289. }
  290. static void udma_dump_chan_stdata(struct udma_chan *uc)
  291. {
  292. struct device *dev = uc->ud->dev;
  293. u32 offset;
  294. int i;
  295. if (uc->config.dir == DMA_MEM_TO_DEV || uc->config.dir == DMA_MEM_TO_MEM) {
  296. dev_dbg(dev, "TCHAN State data:\n");
  297. for (i = 0; i < 32; i++) {
  298. offset = UDMA_CHAN_RT_STDATA_REG + i * 4;
  299. dev_dbg(dev, "TRT_STDATA[%02d]: 0x%08x\n", i,
  300. udma_tchanrt_read(uc, offset));
  301. }
  302. }
  303. if (uc->config.dir == DMA_DEV_TO_MEM || uc->config.dir == DMA_MEM_TO_MEM) {
  304. dev_dbg(dev, "RCHAN State data:\n");
  305. for (i = 0; i < 32; i++) {
  306. offset = UDMA_CHAN_RT_STDATA_REG + i * 4;
  307. dev_dbg(dev, "RRT_STDATA[%02d]: 0x%08x\n", i,
  308. udma_rchanrt_read(uc, offset));
  309. }
  310. }
  311. }
  312. static inline dma_addr_t udma_curr_cppi5_desc_paddr(struct udma_desc *d,
  313. int idx)
  314. {
  315. return d->hwdesc[idx].cppi5_desc_paddr;
  316. }
  317. static inline void *udma_curr_cppi5_desc_vaddr(struct udma_desc *d, int idx)
  318. {
  319. return d->hwdesc[idx].cppi5_desc_vaddr;
  320. }
  321. static struct udma_desc *udma_udma_desc_from_paddr(struct udma_chan *uc,
  322. dma_addr_t paddr)
  323. {
  324. struct udma_desc *d = uc->terminated_desc;
  325. if (d) {
  326. dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d,
  327. d->desc_idx);
  328. if (desc_paddr != paddr)
  329. d = NULL;
  330. }
  331. if (!d) {
  332. d = uc->desc;
  333. if (d) {
  334. dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d,
  335. d->desc_idx);
  336. if (desc_paddr != paddr)
  337. d = NULL;
  338. }
  339. }
  340. return d;
  341. }
  342. static void udma_free_hwdesc(struct udma_chan *uc, struct udma_desc *d)
  343. {
  344. if (uc->use_dma_pool) {
  345. int i;
  346. for (i = 0; i < d->hwdesc_count; i++) {
  347. if (!d->hwdesc[i].cppi5_desc_vaddr)
  348. continue;
  349. dma_pool_free(uc->hdesc_pool,
  350. d->hwdesc[i].cppi5_desc_vaddr,
  351. d->hwdesc[i].cppi5_desc_paddr);
  352. d->hwdesc[i].cppi5_desc_vaddr = NULL;
  353. }
  354. } else if (d->hwdesc[0].cppi5_desc_vaddr) {
  355. struct udma_dev *ud = uc->ud;
  356. dma_free_coherent(ud->dev, d->hwdesc[0].cppi5_desc_size,
  357. d->hwdesc[0].cppi5_desc_vaddr,
  358. d->hwdesc[0].cppi5_desc_paddr);
  359. d->hwdesc[0].cppi5_desc_vaddr = NULL;
  360. }
  361. }
  362. static void udma_purge_desc_work(struct work_struct *work)
  363. {
  364. struct udma_dev *ud = container_of(work, typeof(*ud), purge_work);
  365. struct virt_dma_desc *vd, *_vd;
  366. unsigned long flags;
  367. LIST_HEAD(head);
  368. spin_lock_irqsave(&ud->lock, flags);
  369. list_splice_tail_init(&ud->desc_to_purge, &head);
  370. spin_unlock_irqrestore(&ud->lock, flags);
  371. list_for_each_entry_safe(vd, _vd, &head, node) {
  372. struct udma_chan *uc = to_udma_chan(vd->tx.chan);
  373. struct udma_desc *d = to_udma_desc(&vd->tx);
  374. udma_free_hwdesc(uc, d);
  375. list_del(&vd->node);
  376. kfree(d);
  377. }
  378. /* If more to purge, schedule the work again */
  379. if (!list_empty(&ud->desc_to_purge))
  380. schedule_work(&ud->purge_work);
  381. }
  382. static void udma_desc_free(struct virt_dma_desc *vd)
  383. {
  384. struct udma_dev *ud = to_udma_dev(vd->tx.chan->device);
  385. struct udma_chan *uc = to_udma_chan(vd->tx.chan);
  386. struct udma_desc *d = to_udma_desc(&vd->tx);
  387. unsigned long flags;
  388. if (uc->terminated_desc == d)
  389. uc->terminated_desc = NULL;
  390. if (uc->use_dma_pool) {
  391. udma_free_hwdesc(uc, d);
  392. kfree(d);
  393. return;
  394. }
  395. spin_lock_irqsave(&ud->lock, flags);
  396. list_add_tail(&vd->node, &ud->desc_to_purge);
  397. spin_unlock_irqrestore(&ud->lock, flags);
  398. schedule_work(&ud->purge_work);
  399. }
  400. static bool udma_is_chan_running(struct udma_chan *uc)
  401. {
  402. u32 trt_ctl = 0;
  403. u32 rrt_ctl = 0;
  404. if (uc->tchan)
  405. trt_ctl = udma_tchanrt_read(uc, UDMA_CHAN_RT_CTL_REG);
  406. if (uc->rchan)
  407. rrt_ctl = udma_rchanrt_read(uc, UDMA_CHAN_RT_CTL_REG);
  408. if (trt_ctl & UDMA_CHAN_RT_CTL_EN || rrt_ctl & UDMA_CHAN_RT_CTL_EN)
  409. return true;
  410. return false;
  411. }
  412. static bool udma_is_chan_paused(struct udma_chan *uc)
  413. {
  414. u32 val, pause_mask;
  415. switch (uc->config.dir) {
  416. case DMA_DEV_TO_MEM:
  417. val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PEER_RT_EN_REG);
  418. pause_mask = UDMA_PEER_RT_EN_PAUSE;
  419. break;
  420. case DMA_MEM_TO_DEV:
  421. val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_RT_EN_REG);
  422. pause_mask = UDMA_PEER_RT_EN_PAUSE;
  423. break;
  424. case DMA_MEM_TO_MEM:
  425. val = udma_tchanrt_read(uc, UDMA_CHAN_RT_CTL_REG);
  426. pause_mask = UDMA_CHAN_RT_CTL_PAUSE;
  427. break;
  428. default:
  429. return false;
  430. }
  431. if (val & pause_mask)
  432. return true;
  433. return false;
  434. }
  435. static inline dma_addr_t udma_get_rx_flush_hwdesc_paddr(struct udma_chan *uc)
  436. {
  437. return uc->ud->rx_flush.hwdescs[uc->config.pkt_mode].cppi5_desc_paddr;
  438. }
  439. static int udma_push_to_ring(struct udma_chan *uc, int idx)
  440. {
  441. struct udma_desc *d = uc->desc;
  442. struct k3_ring *ring = NULL;
  443. dma_addr_t paddr;
  444. switch (uc->config.dir) {
  445. case DMA_DEV_TO_MEM:
  446. ring = uc->rflow->fd_ring;
  447. break;
  448. case DMA_MEM_TO_DEV:
  449. case DMA_MEM_TO_MEM:
  450. ring = uc->tchan->t_ring;
  451. break;
  452. default:
  453. return -EINVAL;
  454. }
  455. /* RX flush packet: idx == -1 is only passed in case of DEV_TO_MEM */
  456. if (idx == -1) {
  457. paddr = udma_get_rx_flush_hwdesc_paddr(uc);
  458. } else {
  459. paddr = udma_curr_cppi5_desc_paddr(d, idx);
  460. wmb(); /* Ensure that writes are not moved over this point */
  461. }
  462. return k3_ringacc_ring_push(ring, &paddr);
  463. }
  464. static bool udma_desc_is_rx_flush(struct udma_chan *uc, dma_addr_t addr)
  465. {
  466. if (uc->config.dir != DMA_DEV_TO_MEM)
  467. return false;
  468. if (addr == udma_get_rx_flush_hwdesc_paddr(uc))
  469. return true;
  470. return false;
  471. }
  472. static int udma_pop_from_ring(struct udma_chan *uc, dma_addr_t *addr)
  473. {
  474. struct k3_ring *ring = NULL;
  475. int ret;
  476. switch (uc->config.dir) {
  477. case DMA_DEV_TO_MEM:
  478. ring = uc->rflow->r_ring;
  479. break;
  480. case DMA_MEM_TO_DEV:
  481. case DMA_MEM_TO_MEM:
  482. ring = uc->tchan->tc_ring;
  483. break;
  484. default:
  485. return -ENOENT;
  486. }
  487. ret = k3_ringacc_ring_pop(ring, addr);
  488. if (ret)
  489. return ret;
  490. rmb(); /* Ensure that reads are not moved before this point */
  491. /* Teardown completion */
  492. if (cppi5_desc_is_tdcm(*addr))
  493. return 0;
  494. /* Check for flush descriptor */
  495. if (udma_desc_is_rx_flush(uc, *addr))
  496. return -ENOENT;
  497. return 0;
  498. }
  499. static void udma_reset_rings(struct udma_chan *uc)
  500. {
  501. struct k3_ring *ring1 = NULL;
  502. struct k3_ring *ring2 = NULL;
  503. switch (uc->config.dir) {
  504. case DMA_DEV_TO_MEM:
  505. if (uc->rchan) {
  506. ring1 = uc->rflow->fd_ring;
  507. ring2 = uc->rflow->r_ring;
  508. }
  509. break;
  510. case DMA_MEM_TO_DEV:
  511. case DMA_MEM_TO_MEM:
  512. if (uc->tchan) {
  513. ring1 = uc->tchan->t_ring;
  514. ring2 = uc->tchan->tc_ring;
  515. }
  516. break;
  517. default:
  518. break;
  519. }
  520. if (ring1)
  521. k3_ringacc_ring_reset_dma(ring1,
  522. k3_ringacc_ring_get_occ(ring1));
  523. if (ring2)
  524. k3_ringacc_ring_reset(ring2);
  525. /* make sure we are not leaking memory by stalled descriptor */
  526. if (uc->terminated_desc) {
  527. udma_desc_free(&uc->terminated_desc->vd);
  528. uc->terminated_desc = NULL;
  529. }
  530. }
  531. static void udma_reset_counters(struct udma_chan *uc)
  532. {
  533. u32 val;
  534. if (uc->tchan) {
  535. val = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);
  536. udma_tchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val);
  537. val = udma_tchanrt_read(uc, UDMA_CHAN_RT_SBCNT_REG);
  538. udma_tchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val);
  539. val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PCNT_REG);
  540. udma_tchanrt_write(uc, UDMA_CHAN_RT_PCNT_REG, val);
  541. val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG);
  542. udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val);
  543. }
  544. if (uc->rchan) {
  545. val = udma_rchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);
  546. udma_rchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val);
  547. val = udma_rchanrt_read(uc, UDMA_CHAN_RT_SBCNT_REG);
  548. udma_rchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val);
  549. val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PCNT_REG);
  550. udma_rchanrt_write(uc, UDMA_CHAN_RT_PCNT_REG, val);
  551. val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG);
  552. udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val);
  553. }
  554. uc->bcnt = 0;
  555. }
  556. static int udma_reset_chan(struct udma_chan *uc, bool hard)
  557. {
  558. switch (uc->config.dir) {
  559. case DMA_DEV_TO_MEM:
  560. udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 0);
  561. udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0);
  562. break;
  563. case DMA_MEM_TO_DEV:
  564. udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0);
  565. udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 0);
  566. break;
  567. case DMA_MEM_TO_MEM:
  568. udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0);
  569. udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0);
  570. break;
  571. default:
  572. return -EINVAL;
  573. }
  574. /* Reset all counters */
  575. udma_reset_counters(uc);
  576. /* Hard reset: re-initialize the channel to reset */
  577. if (hard) {
  578. struct udma_chan_config ucc_backup;
  579. int ret;
  580. memcpy(&ucc_backup, &uc->config, sizeof(uc->config));
  581. uc->ud->ddev.device_free_chan_resources(&uc->vc.chan);
  582. /* restore the channel configuration */
  583. memcpy(&uc->config, &ucc_backup, sizeof(uc->config));
  584. ret = uc->ud->ddev.device_alloc_chan_resources(&uc->vc.chan);
  585. if (ret)
  586. return ret;
  587. /*
  588. * Setting forced teardown after forced reset helps recovering
  589. * the rchan.
  590. */
  591. if (uc->config.dir == DMA_DEV_TO_MEM)
  592. udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
  593. UDMA_CHAN_RT_CTL_EN |
  594. UDMA_CHAN_RT_CTL_TDOWN |
  595. UDMA_CHAN_RT_CTL_FTDOWN);
  596. }
  597. uc->state = UDMA_CHAN_IS_IDLE;
  598. return 0;
  599. }
  600. static void udma_start_desc(struct udma_chan *uc)
  601. {
  602. struct udma_chan_config *ucc = &uc->config;
  603. if (ucc->pkt_mode && (uc->cyclic || ucc->dir == DMA_DEV_TO_MEM)) {
  604. int i;
  605. /* Push all descriptors to ring for packet mode cyclic or RX */
  606. for (i = 0; i < uc->desc->sglen; i++)
  607. udma_push_to_ring(uc, i);
  608. } else {
  609. udma_push_to_ring(uc, 0);
  610. }
  611. }
  612. static bool udma_chan_needs_reconfiguration(struct udma_chan *uc)
  613. {
  614. /* Only PDMAs have staticTR */
  615. if (uc->config.ep_type == PSIL_EP_NATIVE)
  616. return false;
  617. /* Check if the staticTR configuration has changed for TX */
  618. if (memcmp(&uc->static_tr, &uc->desc->static_tr, sizeof(uc->static_tr)))
  619. return true;
  620. return false;
  621. }
  622. static int udma_start(struct udma_chan *uc)
  623. {
  624. struct virt_dma_desc *vd = vchan_next_desc(&uc->vc);
  625. if (!vd) {
  626. uc->desc = NULL;
  627. return -ENOENT;
  628. }
  629. list_del(&vd->node);
  630. uc->desc = to_udma_desc(&vd->tx);
  631. /* Channel is already running and does not need reconfiguration */
  632. if (udma_is_chan_running(uc) && !udma_chan_needs_reconfiguration(uc)) {
  633. udma_start_desc(uc);
  634. goto out;
  635. }
  636. /* Make sure that we clear the teardown bit, if it is set */
  637. udma_reset_chan(uc, false);
  638. /* Push descriptors before we start the channel */
  639. udma_start_desc(uc);
  640. switch (uc->desc->dir) {
  641. case DMA_DEV_TO_MEM:
  642. /* Config remote TR */
  643. if (uc->config.ep_type == PSIL_EP_PDMA_XY) {
  644. u32 val = PDMA_STATIC_TR_Y(uc->desc->static_tr.elcnt) |
  645. PDMA_STATIC_TR_X(uc->desc->static_tr.elsize);
  646. const struct udma_match_data *match_data =
  647. uc->ud->match_data;
  648. if (uc->config.enable_acc32)
  649. val |= PDMA_STATIC_TR_XY_ACC32;
  650. if (uc->config.enable_burst)
  651. val |= PDMA_STATIC_TR_XY_BURST;
  652. udma_rchanrt_write(uc,
  653. UDMA_CHAN_RT_PEER_STATIC_TR_XY_REG,
  654. val);
  655. udma_rchanrt_write(uc,
  656. UDMA_CHAN_RT_PEER_STATIC_TR_Z_REG,
  657. PDMA_STATIC_TR_Z(uc->desc->static_tr.bstcnt,
  658. match_data->statictr_z_mask));
  659. /* save the current staticTR configuration */
  660. memcpy(&uc->static_tr, &uc->desc->static_tr,
  661. sizeof(uc->static_tr));
  662. }
  663. udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
  664. UDMA_CHAN_RT_CTL_EN);
  665. /* Enable remote */
  666. udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
  667. UDMA_PEER_RT_EN_ENABLE);
  668. break;
  669. case DMA_MEM_TO_DEV:
  670. /* Config remote TR */
  671. if (uc->config.ep_type == PSIL_EP_PDMA_XY) {
  672. u32 val = PDMA_STATIC_TR_Y(uc->desc->static_tr.elcnt) |
  673. PDMA_STATIC_TR_X(uc->desc->static_tr.elsize);
  674. if (uc->config.enable_acc32)
  675. val |= PDMA_STATIC_TR_XY_ACC32;
  676. if (uc->config.enable_burst)
  677. val |= PDMA_STATIC_TR_XY_BURST;
  678. udma_tchanrt_write(uc,
  679. UDMA_CHAN_RT_PEER_STATIC_TR_XY_REG,
  680. val);
  681. /* save the current staticTR configuration */
  682. memcpy(&uc->static_tr, &uc->desc->static_tr,
  683. sizeof(uc->static_tr));
  684. }
  685. /* Enable remote */
  686. udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
  687. UDMA_PEER_RT_EN_ENABLE);
  688. udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
  689. UDMA_CHAN_RT_CTL_EN);
  690. break;
  691. case DMA_MEM_TO_MEM:
  692. udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
  693. UDMA_CHAN_RT_CTL_EN);
  694. udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
  695. UDMA_CHAN_RT_CTL_EN);
  696. break;
  697. default:
  698. return -EINVAL;
  699. }
  700. uc->state = UDMA_CHAN_IS_ACTIVE;
  701. out:
  702. return 0;
  703. }
  704. static int udma_stop(struct udma_chan *uc)
  705. {
  706. enum udma_chan_state old_state = uc->state;
  707. uc->state = UDMA_CHAN_IS_TERMINATING;
  708. reinit_completion(&uc->teardown_completed);
  709. switch (uc->config.dir) {
  710. case DMA_DEV_TO_MEM:
  711. if (!uc->cyclic && !uc->desc)
  712. udma_push_to_ring(uc, -1);
  713. udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
  714. UDMA_PEER_RT_EN_ENABLE |
  715. UDMA_PEER_RT_EN_TEARDOWN);
  716. break;
  717. case DMA_MEM_TO_DEV:
  718. udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
  719. UDMA_PEER_RT_EN_ENABLE |
  720. UDMA_PEER_RT_EN_FLUSH);
  721. udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
  722. UDMA_CHAN_RT_CTL_EN |
  723. UDMA_CHAN_RT_CTL_TDOWN);
  724. break;
  725. case DMA_MEM_TO_MEM:
  726. udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
  727. UDMA_CHAN_RT_CTL_EN |
  728. UDMA_CHAN_RT_CTL_TDOWN);
  729. break;
  730. default:
  731. uc->state = old_state;
  732. complete_all(&uc->teardown_completed);
  733. return -EINVAL;
  734. }
  735. return 0;
  736. }
  737. static void udma_cyclic_packet_elapsed(struct udma_chan *uc)
  738. {
  739. struct udma_desc *d = uc->desc;
  740. struct cppi5_host_desc_t *h_desc;
  741. h_desc = d->hwdesc[d->desc_idx].cppi5_desc_vaddr;
  742. cppi5_hdesc_reset_to_original(h_desc);
  743. udma_push_to_ring(uc, d->desc_idx);
  744. d->desc_idx = (d->desc_idx + 1) % d->sglen;
  745. }
  746. static inline void udma_fetch_epib(struct udma_chan *uc, struct udma_desc *d)
  747. {
  748. struct cppi5_host_desc_t *h_desc = d->hwdesc[0].cppi5_desc_vaddr;
  749. memcpy(d->metadata, h_desc->epib, d->metadata_size);
  750. }
  751. static bool udma_is_desc_really_done(struct udma_chan *uc, struct udma_desc *d)
  752. {
  753. u32 peer_bcnt, bcnt;
  754. /* Only TX towards PDMA is affected */
  755. if (uc->config.ep_type == PSIL_EP_NATIVE ||
  756. uc->config.dir != DMA_MEM_TO_DEV)
  757. return true;
  758. peer_bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG);
  759. bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);
  760. /* Transfer is incomplete, store current residue and time stamp */
  761. if (peer_bcnt < bcnt) {
  762. uc->tx_drain.residue = bcnt - peer_bcnt;
  763. uc->tx_drain.tstamp = ktime_get();
  764. return false;
  765. }
  766. return true;
  767. }
  768. static void udma_check_tx_completion(struct work_struct *work)
  769. {
  770. struct udma_chan *uc = container_of(work, typeof(*uc),
  771. tx_drain.work.work);
  772. bool desc_done = true;
  773. u32 residue_diff;
  774. ktime_t time_diff;
  775. unsigned long delay;
  776. while (1) {
  777. if (uc->desc) {
  778. /* Get previous residue and time stamp */
  779. residue_diff = uc->tx_drain.residue;
  780. time_diff = uc->tx_drain.tstamp;
  781. /*
  782. * Get current residue and time stamp or see if
  783. * transfer is complete
  784. */
  785. desc_done = udma_is_desc_really_done(uc, uc->desc);
  786. }
  787. if (!desc_done) {
  788. /*
  789. * Find the time delta and residue delta w.r.t
  790. * previous poll
  791. */
  792. time_diff = ktime_sub(uc->tx_drain.tstamp,
  793. time_diff) + 1;
  794. residue_diff -= uc->tx_drain.residue;
  795. if (residue_diff) {
  796. /*
  797. * Try to guess when we should check
  798. * next time by calculating rate at
  799. * which data is being drained at the
  800. * peer device
  801. */
  802. delay = (time_diff / residue_diff) *
  803. uc->tx_drain.residue;
  804. } else {
  805. /* No progress, check again in 1 second */
  806. schedule_delayed_work(&uc->tx_drain.work, HZ);
  807. break;
  808. }
  809. usleep_range(ktime_to_us(delay),
  810. ktime_to_us(delay) + 10);
  811. continue;
  812. }
  813. if (uc->desc) {
  814. struct udma_desc *d = uc->desc;
  815. uc->bcnt += d->residue;
  816. udma_start(uc);
  817. vchan_cookie_complete(&d->vd);
  818. break;
  819. }
  820. break;
  821. }
  822. }
  823. static irqreturn_t udma_ring_irq_handler(int irq, void *data)
  824. {
  825. struct udma_chan *uc = data;
  826. struct udma_desc *d;
  827. unsigned long flags;
  828. dma_addr_t paddr = 0;
  829. if (udma_pop_from_ring(uc, &paddr) || !paddr)
  830. return IRQ_HANDLED;
  831. spin_lock_irqsave(&uc->vc.lock, flags);
  832. /* Teardown completion message */
  833. if (cppi5_desc_is_tdcm(paddr)) {
  834. complete_all(&uc->teardown_completed);
  835. if (uc->terminated_desc) {
  836. udma_desc_free(&uc->terminated_desc->vd);
  837. uc->terminated_desc = NULL;
  838. }
  839. if (!uc->desc)
  840. udma_start(uc);
  841. goto out;
  842. }
  843. d = udma_udma_desc_from_paddr(uc, paddr);
  844. if (d) {
  845. dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d,
  846. d->desc_idx);
  847. if (desc_paddr != paddr) {
  848. dev_err(uc->ud->dev, "not matching descriptors!\n");
  849. goto out;
  850. }
  851. if (d == uc->desc) {
  852. /* active descriptor */
  853. if (uc->cyclic) {
  854. udma_cyclic_packet_elapsed(uc);
  855. vchan_cyclic_callback(&d->vd);
  856. } else {
  857. if (udma_is_desc_really_done(uc, d)) {
  858. uc->bcnt += d->residue;
  859. udma_start(uc);
  860. vchan_cookie_complete(&d->vd);
  861. } else {
  862. schedule_delayed_work(&uc->tx_drain.work,
  863. 0);
  864. }
  865. }
  866. } else {
  867. /*
  868. * terminated descriptor, mark the descriptor as
  869. * completed to update the channel's cookie marker
  870. */
  871. dma_cookie_complete(&d->vd.tx);
  872. }
  873. }
  874. out:
  875. spin_unlock_irqrestore(&uc->vc.lock, flags);
  876. return IRQ_HANDLED;
  877. }
  878. static irqreturn_t udma_udma_irq_handler(int irq, void *data)
  879. {
  880. struct udma_chan *uc = data;
  881. struct udma_desc *d;
  882. unsigned long flags;
  883. spin_lock_irqsave(&uc->vc.lock, flags);
  884. d = uc->desc;
  885. if (d) {
  886. d->tr_idx = (d->tr_idx + 1) % d->sglen;
  887. if (uc->cyclic) {
  888. vchan_cyclic_callback(&d->vd);
  889. } else {
  890. /* TODO: figure out the real amount of data */
  891. uc->bcnt += d->residue;
  892. udma_start(uc);
  893. vchan_cookie_complete(&d->vd);
  894. }
  895. }
  896. spin_unlock_irqrestore(&uc->vc.lock, flags);
  897. return IRQ_HANDLED;
  898. }
  899. /**
  900. * __udma_alloc_gp_rflow_range - alloc range of GP RX flows
  901. * @ud: UDMA device
  902. * @from: Start the search from this flow id number
  903. * @cnt: Number of consecutive flow ids to allocate
  904. *
  905. * Allocate range of RX flow ids for future use, those flows can be requested
  906. * only using explicit flow id number. if @from is set to -1 it will try to find
  907. * first free range. if @from is positive value it will force allocation only
  908. * of the specified range of flows.
  909. *
  910. * Returns -ENOMEM if can't find free range.
  911. * -EEXIST if requested range is busy.
  912. * -EINVAL if wrong input values passed.
  913. * Returns flow id on success.
  914. */
  915. static int __udma_alloc_gp_rflow_range(struct udma_dev *ud, int from, int cnt)
  916. {
  917. int start, tmp_from;
  918. DECLARE_BITMAP(tmp, K3_UDMA_MAX_RFLOWS);
  919. tmp_from = from;
  920. if (tmp_from < 0)
  921. tmp_from = ud->rchan_cnt;
  922. /* default flows can't be allocated and accessible only by id */
  923. if (tmp_from < ud->rchan_cnt)
  924. return -EINVAL;
  925. if (tmp_from + cnt > ud->rflow_cnt)
  926. return -EINVAL;
  927. bitmap_or(tmp, ud->rflow_gp_map, ud->rflow_gp_map_allocated,
  928. ud->rflow_cnt);
  929. start = bitmap_find_next_zero_area(tmp,
  930. ud->rflow_cnt,
  931. tmp_from, cnt, 0);
  932. if (start >= ud->rflow_cnt)
  933. return -ENOMEM;
  934. if (from >= 0 && start != from)
  935. return -EEXIST;
  936. bitmap_set(ud->rflow_gp_map_allocated, start, cnt);
  937. return start;
  938. }
  939. static int __udma_free_gp_rflow_range(struct udma_dev *ud, int from, int cnt)
  940. {
  941. if (from < ud->rchan_cnt)
  942. return -EINVAL;
  943. if (from + cnt > ud->rflow_cnt)
  944. return -EINVAL;
  945. bitmap_clear(ud->rflow_gp_map_allocated, from, cnt);
  946. return 0;
  947. }
  948. static struct udma_rflow *__udma_get_rflow(struct udma_dev *ud, int id)
  949. {
  950. /*
  951. * Attempt to request rflow by ID can be made for any rflow
  952. * if not in use with assumption that caller knows what's doing.
  953. * TI-SCI FW will perform additional permission check ant way, it's
  954. * safe
  955. */
  956. if (id < 0 || id >= ud->rflow_cnt)
  957. return ERR_PTR(-ENOENT);
  958. if (test_bit(id, ud->rflow_in_use))
  959. return ERR_PTR(-ENOENT);
  960. /* GP rflow has to be allocated first */
  961. if (!test_bit(id, ud->rflow_gp_map) &&
  962. !test_bit(id, ud->rflow_gp_map_allocated))
  963. return ERR_PTR(-EINVAL);
  964. dev_dbg(ud->dev, "get rflow%d\n", id);
  965. set_bit(id, ud->rflow_in_use);
  966. return &ud->rflows[id];
  967. }
  968. static void __udma_put_rflow(struct udma_dev *ud, struct udma_rflow *rflow)
  969. {
  970. if (!test_bit(rflow->id, ud->rflow_in_use)) {
  971. dev_err(ud->dev, "attempt to put unused rflow%d\n", rflow->id);
  972. return;
  973. }
  974. dev_dbg(ud->dev, "put rflow%d\n", rflow->id);
  975. clear_bit(rflow->id, ud->rflow_in_use);
  976. }
  977. #define UDMA_RESERVE_RESOURCE(res) \
  978. static struct udma_##res *__udma_reserve_##res(struct udma_dev *ud, \
  979. enum udma_tp_level tpl, \
  980. int id) \
  981. { \
  982. if (id >= 0) { \
  983. if (test_bit(id, ud->res##_map)) { \
  984. dev_err(ud->dev, "res##%d is in use\n", id); \
  985. return ERR_PTR(-ENOENT); \
  986. } \
  987. } else { \
  988. int start; \
  989. \
  990. if (tpl >= ud->tpl_levels) \
  991. tpl = ud->tpl_levels - 1; \
  992. \
  993. start = ud->tpl_start_idx[tpl]; \
  994. \
  995. id = find_next_zero_bit(ud->res##_map, ud->res##_cnt, \
  996. start); \
  997. if (id == ud->res##_cnt) { \
  998. return ERR_PTR(-ENOENT); \
  999. } \
  1000. } \
  1001. \
  1002. set_bit(id, ud->res##_map); \
  1003. return &ud->res##s[id]; \
  1004. }
  1005. UDMA_RESERVE_RESOURCE(tchan);
  1006. UDMA_RESERVE_RESOURCE(rchan);
  1007. static int udma_get_tchan(struct udma_chan *uc)
  1008. {
  1009. struct udma_dev *ud = uc->ud;
  1010. if (uc->tchan) {
  1011. dev_dbg(ud->dev, "chan%d: already have tchan%d allocated\n",
  1012. uc->id, uc->tchan->id);
  1013. return 0;
  1014. }
  1015. uc->tchan = __udma_reserve_tchan(ud, uc->config.channel_tpl, -1);
  1016. return PTR_ERR_OR_ZERO(uc->tchan);
  1017. }
  1018. static int udma_get_rchan(struct udma_chan *uc)
  1019. {
  1020. struct udma_dev *ud = uc->ud;
  1021. if (uc->rchan) {
  1022. dev_dbg(ud->dev, "chan%d: already have rchan%d allocated\n",
  1023. uc->id, uc->rchan->id);
  1024. return 0;
  1025. }
  1026. uc->rchan = __udma_reserve_rchan(ud, uc->config.channel_tpl, -1);
  1027. return PTR_ERR_OR_ZERO(uc->rchan);
  1028. }
  1029. static int udma_get_chan_pair(struct udma_chan *uc)
  1030. {
  1031. struct udma_dev *ud = uc->ud;
  1032. int chan_id, end;
  1033. if ((uc->tchan && uc->rchan) && uc->tchan->id == uc->rchan->id) {
  1034. dev_info(ud->dev, "chan%d: already have %d pair allocated\n",
  1035. uc->id, uc->tchan->id);
  1036. return 0;
  1037. }
  1038. if (uc->tchan) {
  1039. dev_err(ud->dev, "chan%d: already have tchan%d allocated\n",
  1040. uc->id, uc->tchan->id);
  1041. return -EBUSY;
  1042. } else if (uc->rchan) {
  1043. dev_err(ud->dev, "chan%d: already have rchan%d allocated\n",
  1044. uc->id, uc->rchan->id);
  1045. return -EBUSY;
  1046. }
  1047. /* Can be optimized, but let's have it like this for now */
  1048. end = min(ud->tchan_cnt, ud->rchan_cnt);
  1049. /* Try to use the highest TPL channel pair for MEM_TO_MEM channels */
  1050. chan_id = ud->tpl_start_idx[ud->tpl_levels - 1];
  1051. for (; chan_id < end; chan_id++) {
  1052. if (!test_bit(chan_id, ud->tchan_map) &&
  1053. !test_bit(chan_id, ud->rchan_map))
  1054. break;
  1055. }
  1056. if (chan_id == end)
  1057. return -ENOENT;
  1058. set_bit(chan_id, ud->tchan_map);
  1059. set_bit(chan_id, ud->rchan_map);
  1060. uc->tchan = &ud->tchans[chan_id];
  1061. uc->rchan = &ud->rchans[chan_id];
  1062. return 0;
  1063. }
  1064. static int udma_get_rflow(struct udma_chan *uc, int flow_id)
  1065. {
  1066. struct udma_dev *ud = uc->ud;
  1067. if (!uc->rchan) {
  1068. dev_err(ud->dev, "chan%d: does not have rchan??\n", uc->id);
  1069. return -EINVAL;
  1070. }
  1071. if (uc->rflow) {
  1072. dev_dbg(ud->dev, "chan%d: already have rflow%d allocated\n",
  1073. uc->id, uc->rflow->id);
  1074. return 0;
  1075. }
  1076. uc->rflow = __udma_get_rflow(ud, flow_id);
  1077. return PTR_ERR_OR_ZERO(uc->rflow);
  1078. }
  1079. static void udma_put_rchan(struct udma_chan *uc)
  1080. {
  1081. struct udma_dev *ud = uc->ud;
  1082. if (uc->rchan) {
  1083. dev_dbg(ud->dev, "chan%d: put rchan%d\n", uc->id,
  1084. uc->rchan->id);
  1085. clear_bit(uc->rchan->id, ud->rchan_map);
  1086. uc->rchan = NULL;
  1087. }
  1088. }
  1089. static void udma_put_tchan(struct udma_chan *uc)
  1090. {
  1091. struct udma_dev *ud = uc->ud;
  1092. if (uc->tchan) {
  1093. dev_dbg(ud->dev, "chan%d: put tchan%d\n", uc->id,
  1094. uc->tchan->id);
  1095. clear_bit(uc->tchan->id, ud->tchan_map);
  1096. uc->tchan = NULL;
  1097. }
  1098. }
  1099. static void udma_put_rflow(struct udma_chan *uc)
  1100. {
  1101. struct udma_dev *ud = uc->ud;
  1102. if (uc->rflow) {
  1103. dev_dbg(ud->dev, "chan%d: put rflow%d\n", uc->id,
  1104. uc->rflow->id);
  1105. __udma_put_rflow(ud, uc->rflow);
  1106. uc->rflow = NULL;
  1107. }
  1108. }
  1109. static void udma_free_tx_resources(struct udma_chan *uc)
  1110. {
  1111. if (!uc->tchan)
  1112. return;
  1113. k3_ringacc_ring_free(uc->tchan->t_ring);
  1114. k3_ringacc_ring_free(uc->tchan->tc_ring);
  1115. uc->tchan->t_ring = NULL;
  1116. uc->tchan->tc_ring = NULL;
  1117. udma_put_tchan(uc);
  1118. }
  1119. static int udma_alloc_tx_resources(struct udma_chan *uc)
  1120. {
  1121. struct k3_ring_cfg ring_cfg;
  1122. struct udma_dev *ud = uc->ud;
  1123. int ret;
  1124. ret = udma_get_tchan(uc);
  1125. if (ret)
  1126. return ret;
  1127. ret = k3_ringacc_request_rings_pair(ud->ringacc, uc->tchan->id, -1,
  1128. &uc->tchan->t_ring,
  1129. &uc->tchan->tc_ring);
  1130. if (ret) {
  1131. ret = -EBUSY;
  1132. goto err_ring;
  1133. }
  1134. memset(&ring_cfg, 0, sizeof(ring_cfg));
  1135. ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE;
  1136. ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8;
  1137. ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE;
  1138. ret = k3_ringacc_ring_cfg(uc->tchan->t_ring, &ring_cfg);
  1139. ret |= k3_ringacc_ring_cfg(uc->tchan->tc_ring, &ring_cfg);
  1140. if (ret)
  1141. goto err_ringcfg;
  1142. return 0;
  1143. err_ringcfg:
  1144. k3_ringacc_ring_free(uc->tchan->tc_ring);
  1145. uc->tchan->tc_ring = NULL;
  1146. k3_ringacc_ring_free(uc->tchan->t_ring);
  1147. uc->tchan->t_ring = NULL;
  1148. err_ring:
  1149. udma_put_tchan(uc);
  1150. return ret;
  1151. }
  1152. static void udma_free_rx_resources(struct udma_chan *uc)
  1153. {
  1154. if (!uc->rchan)
  1155. return;
  1156. if (uc->rflow) {
  1157. struct udma_rflow *rflow = uc->rflow;
  1158. k3_ringacc_ring_free(rflow->fd_ring);
  1159. k3_ringacc_ring_free(rflow->r_ring);
  1160. rflow->fd_ring = NULL;
  1161. rflow->r_ring = NULL;
  1162. udma_put_rflow(uc);
  1163. }
  1164. udma_put_rchan(uc);
  1165. }
  1166. static int udma_alloc_rx_resources(struct udma_chan *uc)
  1167. {
  1168. struct udma_dev *ud = uc->ud;
  1169. struct k3_ring_cfg ring_cfg;
  1170. struct udma_rflow *rflow;
  1171. int fd_ring_id;
  1172. int ret;
  1173. ret = udma_get_rchan(uc);
  1174. if (ret)
  1175. return ret;
  1176. /* For MEM_TO_MEM we don't need rflow or rings */
  1177. if (uc->config.dir == DMA_MEM_TO_MEM)
  1178. return 0;
  1179. ret = udma_get_rflow(uc, uc->rchan->id);
  1180. if (ret) {
  1181. ret = -EBUSY;
  1182. goto err_rflow;
  1183. }
  1184. rflow = uc->rflow;
  1185. fd_ring_id = ud->tchan_cnt + ud->echan_cnt + uc->rchan->id;
  1186. ret = k3_ringacc_request_rings_pair(ud->ringacc, fd_ring_id, -1,
  1187. &rflow->fd_ring, &rflow->r_ring);
  1188. if (ret) {
  1189. ret = -EBUSY;
  1190. goto err_ring;
  1191. }
  1192. memset(&ring_cfg, 0, sizeof(ring_cfg));
  1193. if (uc->config.pkt_mode)
  1194. ring_cfg.size = SG_MAX_SEGMENTS;
  1195. else
  1196. ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE;
  1197. ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8;
  1198. ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE;
  1199. ret = k3_ringacc_ring_cfg(rflow->fd_ring, &ring_cfg);
  1200. ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE;
  1201. ret |= k3_ringacc_ring_cfg(rflow->r_ring, &ring_cfg);
  1202. if (ret)
  1203. goto err_ringcfg;
  1204. return 0;
  1205. err_ringcfg:
  1206. k3_ringacc_ring_free(rflow->r_ring);
  1207. rflow->r_ring = NULL;
  1208. k3_ringacc_ring_free(rflow->fd_ring);
  1209. rflow->fd_ring = NULL;
  1210. err_ring:
  1211. udma_put_rflow(uc);
  1212. err_rflow:
  1213. udma_put_rchan(uc);
  1214. return ret;
  1215. }
  1216. #define TISCI_TCHAN_VALID_PARAMS ( \
  1217. TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \
  1218. TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID | \
  1219. TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID | \
  1220. TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | \
  1221. TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID | \
  1222. TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | \
  1223. TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | \
  1224. TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID)
  1225. #define TISCI_RCHAN_VALID_PARAMS ( \
  1226. TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \
  1227. TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | \
  1228. TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | \
  1229. TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | \
  1230. TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_SHORT_VALID | \
  1231. TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_LONG_VALID | \
  1232. TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID | \
  1233. TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID | \
  1234. TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID)
  1235. static int udma_tisci_m2m_channel_config(struct udma_chan *uc)
  1236. {
  1237. struct udma_dev *ud = uc->ud;
  1238. struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
  1239. const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
  1240. struct udma_tchan *tchan = uc->tchan;
  1241. struct udma_rchan *rchan = uc->rchan;
  1242. int ret = 0;
  1243. /* Non synchronized - mem to mem type of transfer */
  1244. int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring);
  1245. struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 };
  1246. struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 };
  1247. req_tx.valid_params = TISCI_TCHAN_VALID_PARAMS;
  1248. req_tx.nav_id = tisci_rm->tisci_dev_id;
  1249. req_tx.index = tchan->id;
  1250. req_tx.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR;
  1251. req_tx.tx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2;
  1252. req_tx.txcq_qnum = tc_ring;
  1253. req_tx.tx_atype = ud->atype;
  1254. ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);
  1255. if (ret) {
  1256. dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret);
  1257. return ret;
  1258. }
  1259. req_rx.valid_params = TISCI_RCHAN_VALID_PARAMS;
  1260. req_rx.nav_id = tisci_rm->tisci_dev_id;
  1261. req_rx.index = rchan->id;
  1262. req_rx.rx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2;
  1263. req_rx.rxcq_qnum = tc_ring;
  1264. req_rx.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR;
  1265. req_rx.rx_atype = ud->atype;
  1266. ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx);
  1267. if (ret)
  1268. dev_err(ud->dev, "rchan%d alloc failed %d\n", rchan->id, ret);
  1269. return ret;
  1270. }
  1271. static int udma_tisci_tx_channel_config(struct udma_chan *uc)
  1272. {
  1273. struct udma_dev *ud = uc->ud;
  1274. struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
  1275. const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
  1276. struct udma_tchan *tchan = uc->tchan;
  1277. int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring);
  1278. struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 };
  1279. u32 mode, fetch_size;
  1280. int ret = 0;
  1281. if (uc->config.pkt_mode) {
  1282. mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
  1283. fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib,
  1284. uc->config.psd_size, 0);
  1285. } else {
  1286. mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_PBRR;
  1287. fetch_size = sizeof(struct cppi5_desc_hdr_t);
  1288. }
  1289. req_tx.valid_params = TISCI_TCHAN_VALID_PARAMS;
  1290. req_tx.nav_id = tisci_rm->tisci_dev_id;
  1291. req_tx.index = tchan->id;
  1292. req_tx.tx_chan_type = mode;
  1293. req_tx.tx_supr_tdpkt = uc->config.notdpkt;
  1294. req_tx.tx_fetch_size = fetch_size >> 2;
  1295. req_tx.txcq_qnum = tc_ring;
  1296. req_tx.tx_atype = uc->config.atype;
  1297. ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);
  1298. if (ret)
  1299. dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret);
  1300. return ret;
  1301. }
  1302. static int udma_tisci_rx_channel_config(struct udma_chan *uc)
  1303. {
  1304. struct udma_dev *ud = uc->ud;
  1305. struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
  1306. const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
  1307. struct udma_rchan *rchan = uc->rchan;
  1308. int fd_ring = k3_ringacc_get_ring_id(uc->rflow->fd_ring);
  1309. int rx_ring = k3_ringacc_get_ring_id(uc->rflow->r_ring);
  1310. struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 };
  1311. struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 };
  1312. u32 mode, fetch_size;
  1313. int ret = 0;
  1314. if (uc->config.pkt_mode) {
  1315. mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
  1316. fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib,
  1317. uc->config.psd_size, 0);
  1318. } else {
  1319. mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_PBRR;
  1320. fetch_size = sizeof(struct cppi5_desc_hdr_t);
  1321. }
  1322. req_rx.valid_params = TISCI_RCHAN_VALID_PARAMS;
  1323. req_rx.nav_id = tisci_rm->tisci_dev_id;
  1324. req_rx.index = rchan->id;
  1325. req_rx.rx_fetch_size = fetch_size >> 2;
  1326. req_rx.rxcq_qnum = rx_ring;
  1327. req_rx.rx_chan_type = mode;
  1328. req_rx.rx_atype = uc->config.atype;
  1329. ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx);
  1330. if (ret) {
  1331. dev_err(ud->dev, "rchan%d cfg failed %d\n", rchan->id, ret);
  1332. return ret;
  1333. }
  1334. flow_req.valid_params =
  1335. TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID |
  1336. TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID |
  1337. TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID |
  1338. TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID |
  1339. TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
  1340. TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID |
  1341. TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID |
  1342. TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID |
  1343. TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID |
  1344. TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
  1345. TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
  1346. TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
  1347. TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID;
  1348. flow_req.nav_id = tisci_rm->tisci_dev_id;
  1349. flow_req.flow_index = rchan->id;
  1350. if (uc->config.needs_epib)
  1351. flow_req.rx_einfo_present = 1;
  1352. else
  1353. flow_req.rx_einfo_present = 0;
  1354. if (uc->config.psd_size)
  1355. flow_req.rx_psinfo_present = 1;
  1356. else
  1357. flow_req.rx_psinfo_present = 0;
  1358. flow_req.rx_error_handling = 1;
  1359. flow_req.rx_dest_qnum = rx_ring;
  1360. flow_req.rx_src_tag_hi_sel = UDMA_RFLOW_SRCTAG_NONE;
  1361. flow_req.rx_src_tag_lo_sel = UDMA_RFLOW_SRCTAG_SRC_TAG;
  1362. flow_req.rx_dest_tag_hi_sel = UDMA_RFLOW_DSTTAG_DST_TAG_HI;
  1363. flow_req.rx_dest_tag_lo_sel = UDMA_RFLOW_DSTTAG_DST_TAG_LO;
  1364. flow_req.rx_fdq0_sz0_qnum = fd_ring;
  1365. flow_req.rx_fdq1_qnum = fd_ring;
  1366. flow_req.rx_fdq2_qnum = fd_ring;
  1367. flow_req.rx_fdq3_qnum = fd_ring;
  1368. ret = tisci_ops->rx_flow_cfg(tisci_rm->tisci, &flow_req);
  1369. if (ret)
  1370. dev_err(ud->dev, "flow%d config failed: %d\n", rchan->id, ret);
  1371. return 0;
  1372. }
  1373. static int udma_alloc_chan_resources(struct dma_chan *chan)
  1374. {
  1375. struct udma_chan *uc = to_udma_chan(chan);
  1376. struct udma_dev *ud = to_udma_dev(chan->device);
  1377. const struct udma_soc_data *soc_data = ud->soc_data;
  1378. struct k3_ring *irq_ring;
  1379. u32 irq_udma_idx;
  1380. int ret;
  1381. if (uc->config.pkt_mode || uc->config.dir == DMA_MEM_TO_MEM) {
  1382. uc->use_dma_pool = true;
  1383. /* in case of MEM_TO_MEM we have maximum of two TRs */
  1384. if (uc->config.dir == DMA_MEM_TO_MEM) {
  1385. uc->config.hdesc_size = cppi5_trdesc_calc_size(
  1386. sizeof(struct cppi5_tr_type15_t), 2);
  1387. uc->config.pkt_mode = false;
  1388. }
  1389. }
  1390. if (uc->use_dma_pool) {
  1391. uc->hdesc_pool = dma_pool_create(uc->name, ud->ddev.dev,
  1392. uc->config.hdesc_size,
  1393. ud->desc_align,
  1394. 0);
  1395. if (!uc->hdesc_pool) {
  1396. dev_err(ud->ddev.dev,
  1397. "Descriptor pool allocation failed\n");
  1398. uc->use_dma_pool = false;
  1399. ret = -ENOMEM;
  1400. goto err_cleanup;
  1401. }
  1402. }
  1403. /*
  1404. * Make sure that the completion is in a known state:
  1405. * No teardown, the channel is idle
  1406. */
  1407. reinit_completion(&uc->teardown_completed);
  1408. complete_all(&uc->teardown_completed);
  1409. uc->state = UDMA_CHAN_IS_IDLE;
  1410. switch (uc->config.dir) {
  1411. case DMA_MEM_TO_MEM:
  1412. /* Non synchronized - mem to mem type of transfer */
  1413. dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-MEM\n", __func__,
  1414. uc->id);
  1415. ret = udma_get_chan_pair(uc);
  1416. if (ret)
  1417. goto err_cleanup;
  1418. ret = udma_alloc_tx_resources(uc);
  1419. if (ret) {
  1420. udma_put_rchan(uc);
  1421. goto err_cleanup;
  1422. }
  1423. ret = udma_alloc_rx_resources(uc);
  1424. if (ret) {
  1425. udma_free_tx_resources(uc);
  1426. goto err_cleanup;
  1427. }
  1428. uc->config.src_thread = ud->psil_base + uc->tchan->id;
  1429. uc->config.dst_thread = (ud->psil_base + uc->rchan->id) |
  1430. K3_PSIL_DST_THREAD_ID_OFFSET;
  1431. irq_ring = uc->tchan->tc_ring;
  1432. irq_udma_idx = uc->tchan->id;
  1433. ret = udma_tisci_m2m_channel_config(uc);
  1434. break;
  1435. case DMA_MEM_TO_DEV:
  1436. /* Slave transfer synchronized - mem to dev (TX) trasnfer */
  1437. dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__,
  1438. uc->id);
  1439. ret = udma_alloc_tx_resources(uc);
  1440. if (ret)
  1441. goto err_cleanup;
  1442. uc->config.src_thread = ud->psil_base + uc->tchan->id;
  1443. uc->config.dst_thread = uc->config.remote_thread_id;
  1444. uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;
  1445. irq_ring = uc->tchan->tc_ring;
  1446. irq_udma_idx = uc->tchan->id;
  1447. ret = udma_tisci_tx_channel_config(uc);
  1448. break;
  1449. case DMA_DEV_TO_MEM:
  1450. /* Slave transfer synchronized - dev to mem (RX) trasnfer */
  1451. dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__,
  1452. uc->id);
  1453. ret = udma_alloc_rx_resources(uc);
  1454. if (ret)
  1455. goto err_cleanup;
  1456. uc->config.src_thread = uc->config.remote_thread_id;
  1457. uc->config.dst_thread = (ud->psil_base + uc->rchan->id) |
  1458. K3_PSIL_DST_THREAD_ID_OFFSET;
  1459. irq_ring = uc->rflow->r_ring;
  1460. irq_udma_idx = soc_data->rchan_oes_offset + uc->rchan->id;
  1461. ret = udma_tisci_rx_channel_config(uc);
  1462. break;
  1463. default:
  1464. /* Can not happen */
  1465. dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n",
  1466. __func__, uc->id, uc->config.dir);
  1467. ret = -EINVAL;
  1468. goto err_cleanup;
  1469. }
  1470. /* check if the channel configuration was successful */
  1471. if (ret)
  1472. goto err_res_free;
  1473. if (udma_is_chan_running(uc)) {
  1474. dev_warn(ud->dev, "chan%d: is running!\n", uc->id);
  1475. udma_reset_chan(uc, false);
  1476. if (udma_is_chan_running(uc)) {
  1477. dev_err(ud->dev, "chan%d: won't stop!\n", uc->id);
  1478. ret = -EBUSY;
  1479. goto err_res_free;
  1480. }
  1481. }
  1482. /* PSI-L pairing */
  1483. ret = navss_psil_pair(ud, uc->config.src_thread, uc->config.dst_thread);
  1484. if (ret) {
  1485. dev_err(ud->dev, "PSI-L pairing failed: 0x%04x -> 0x%04x\n",
  1486. uc->config.src_thread, uc->config.dst_thread);
  1487. goto err_res_free;
  1488. }
  1489. uc->psil_paired = true;
  1490. uc->irq_num_ring = k3_ringacc_get_ring_irq_num(irq_ring);
  1491. if (uc->irq_num_ring <= 0) {
  1492. dev_err(ud->dev, "Failed to get ring irq (index: %u)\n",
  1493. k3_ringacc_get_ring_id(irq_ring));
  1494. ret = -EINVAL;
  1495. goto err_psi_free;
  1496. }
  1497. ret = request_irq(uc->irq_num_ring, udma_ring_irq_handler,
  1498. IRQF_TRIGGER_HIGH, uc->name, uc);
  1499. if (ret) {
  1500. dev_err(ud->dev, "chan%d: ring irq request failed\n", uc->id);
  1501. goto err_irq_free;
  1502. }
  1503. /* Event from UDMA (TR events) only needed for slave TR mode channels */
  1504. if (is_slave_direction(uc->config.dir) && !uc->config.pkt_mode) {
  1505. uc->irq_num_udma = ti_sci_inta_msi_get_virq(ud->dev,
  1506. irq_udma_idx);
  1507. if (uc->irq_num_udma <= 0) {
  1508. dev_err(ud->dev, "Failed to get udma irq (index: %u)\n",
  1509. irq_udma_idx);
  1510. free_irq(uc->irq_num_ring, uc);
  1511. ret = -EINVAL;
  1512. goto err_irq_free;
  1513. }
  1514. ret = request_irq(uc->irq_num_udma, udma_udma_irq_handler, 0,
  1515. uc->name, uc);
  1516. if (ret) {
  1517. dev_err(ud->dev, "chan%d: UDMA irq request failed\n",
  1518. uc->id);
  1519. free_irq(uc->irq_num_ring, uc);
  1520. goto err_irq_free;
  1521. }
  1522. } else {
  1523. uc->irq_num_udma = 0;
  1524. }
  1525. udma_reset_rings(uc);
  1526. return 0;
  1527. err_irq_free:
  1528. uc->irq_num_ring = 0;
  1529. uc->irq_num_udma = 0;
  1530. err_psi_free:
  1531. navss_psil_unpair(ud, uc->config.src_thread, uc->config.dst_thread);
  1532. uc->psil_paired = false;
  1533. err_res_free:
  1534. udma_free_tx_resources(uc);
  1535. udma_free_rx_resources(uc);
  1536. err_cleanup:
  1537. udma_reset_uchan(uc);
  1538. if (uc->use_dma_pool) {
  1539. dma_pool_destroy(uc->hdesc_pool);
  1540. uc->use_dma_pool = false;
  1541. }
  1542. return ret;
  1543. }
  1544. static int udma_slave_config(struct dma_chan *chan,
  1545. struct dma_slave_config *cfg)
  1546. {
  1547. struct udma_chan *uc = to_udma_chan(chan);
  1548. memcpy(&uc->cfg, cfg, sizeof(uc->cfg));
  1549. return 0;
  1550. }
  1551. static struct udma_desc *udma_alloc_tr_desc(struct udma_chan *uc,
  1552. size_t tr_size, int tr_count,
  1553. enum dma_transfer_direction dir)
  1554. {
  1555. struct udma_hwdesc *hwdesc;
  1556. struct cppi5_desc_hdr_t *tr_desc;
  1557. struct udma_desc *d;
  1558. u32 reload_count = 0;
  1559. u32 ring_id;
  1560. switch (tr_size) {
  1561. case 16:
  1562. case 32:
  1563. case 64:
  1564. case 128:
  1565. break;
  1566. default:
  1567. dev_err(uc->ud->dev, "Unsupported TR size of %zu\n", tr_size);
  1568. return NULL;
  1569. }
  1570. /* We have only one descriptor containing multiple TRs */
  1571. d = kzalloc(sizeof(*d) + sizeof(d->hwdesc[0]), GFP_NOWAIT);
  1572. if (!d)
  1573. return NULL;
  1574. d->sglen = tr_count;
  1575. d->hwdesc_count = 1;
  1576. hwdesc = &d->hwdesc[0];
  1577. /* Allocate memory for DMA ring descriptor */
  1578. if (uc->use_dma_pool) {
  1579. hwdesc->cppi5_desc_size = uc->config.hdesc_size;
  1580. hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool,
  1581. GFP_NOWAIT,
  1582. &hwdesc->cppi5_desc_paddr);
  1583. } else {
  1584. hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size,
  1585. tr_count);
  1586. hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size,
  1587. uc->ud->desc_align);
  1588. hwdesc->cppi5_desc_vaddr = dma_alloc_coherent(uc->ud->dev,
  1589. hwdesc->cppi5_desc_size,
  1590. &hwdesc->cppi5_desc_paddr,
  1591. GFP_NOWAIT);
  1592. }
  1593. if (!hwdesc->cppi5_desc_vaddr) {
  1594. kfree(d);
  1595. return NULL;
  1596. }
  1597. /* Start of the TR req records */
  1598. hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size;
  1599. /* Start address of the TR response array */
  1600. hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size * tr_count;
  1601. tr_desc = hwdesc->cppi5_desc_vaddr;
  1602. if (uc->cyclic)
  1603. reload_count = CPPI5_INFO0_TRDESC_RLDCNT_INFINITE;
  1604. if (dir == DMA_DEV_TO_MEM)
  1605. ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring);
  1606. else
  1607. ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring);
  1608. cppi5_trdesc_init(tr_desc, tr_count, tr_size, 0, reload_count);
  1609. cppi5_desc_set_pktids(tr_desc, uc->id,
  1610. CPPI5_INFO1_DESC_FLOWID_DEFAULT);
  1611. cppi5_desc_set_retpolicy(tr_desc, 0, ring_id);
  1612. return d;
  1613. }
  1614. /**
  1615. * udma_get_tr_counters - calculate TR counters for a given length
  1616. * @len: Length of the trasnfer
  1617. * @align_to: Preferred alignment
  1618. * @tr0_cnt0: First TR icnt0
  1619. * @tr0_cnt1: First TR icnt1
  1620. * @tr1_cnt0: Second (if used) TR icnt0
  1621. *
  1622. * For len < SZ_64K only one TR is enough, tr1_cnt0 is not updated
  1623. * For len >= SZ_64K two TRs are used in a simple way:
  1624. * First TR: SZ_64K-alignment blocks (tr0_cnt0, tr0_cnt1)
  1625. * Second TR: the remaining length (tr1_cnt0)
  1626. *
  1627. * Returns the number of TRs the length needs (1 or 2)
  1628. * -EINVAL if the length can not be supported
  1629. */
  1630. static int udma_get_tr_counters(size_t len, unsigned long align_to,
  1631. u16 *tr0_cnt0, u16 *tr0_cnt1, u16 *tr1_cnt0)
  1632. {
  1633. if (len < SZ_64K) {
  1634. *tr0_cnt0 = len;
  1635. *tr0_cnt1 = 1;
  1636. return 1;
  1637. }
  1638. if (align_to > 3)
  1639. align_to = 3;
  1640. realign:
  1641. *tr0_cnt0 = SZ_64K - BIT(align_to);
  1642. if (len / *tr0_cnt0 >= SZ_64K) {
  1643. if (align_to) {
  1644. align_to--;
  1645. goto realign;
  1646. }
  1647. return -EINVAL;
  1648. }
  1649. *tr0_cnt1 = len / *tr0_cnt0;
  1650. *tr1_cnt0 = len % *tr0_cnt0;
  1651. return 2;
  1652. }
  1653. static struct udma_desc *
  1654. udma_prep_slave_sg_tr(struct udma_chan *uc, struct scatterlist *sgl,
  1655. unsigned int sglen, enum dma_transfer_direction dir,
  1656. unsigned long tx_flags, void *context)
  1657. {
  1658. struct scatterlist *sgent;
  1659. struct udma_desc *d;
  1660. struct cppi5_tr_type1_t *tr_req = NULL;
  1661. u16 tr0_cnt0, tr0_cnt1, tr1_cnt0;
  1662. unsigned int i;
  1663. size_t tr_size;
  1664. int num_tr = 0;
  1665. int tr_idx = 0;
  1666. /* estimate the number of TRs we will need */
  1667. for_each_sg(sgl, sgent, sglen, i) {
  1668. if (sg_dma_len(sgent) < SZ_64K)
  1669. num_tr++;
  1670. else
  1671. num_tr += 2;
  1672. }
  1673. /* Now allocate and setup the descriptor. */
  1674. tr_size = sizeof(struct cppi5_tr_type1_t);
  1675. d = udma_alloc_tr_desc(uc, tr_size, num_tr, dir);
  1676. if (!d)
  1677. return NULL;
  1678. d->sglen = sglen;
  1679. tr_req = d->hwdesc[0].tr_req_base;
  1680. for_each_sg(sgl, sgent, sglen, i) {
  1681. dma_addr_t sg_addr = sg_dma_address(sgent);
  1682. num_tr = udma_get_tr_counters(sg_dma_len(sgent), __ffs(sg_addr),
  1683. &tr0_cnt0, &tr0_cnt1, &tr1_cnt0);
  1684. if (num_tr < 0) {
  1685. dev_err(uc->ud->dev, "size %u is not supported\n",
  1686. sg_dma_len(sgent));
  1687. udma_free_hwdesc(uc, d);
  1688. kfree(d);
  1689. return NULL;
  1690. }
  1691. cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, false,
  1692. false, CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
  1693. cppi5_tr_csf_set(&tr_req[tr_idx].flags, CPPI5_TR_CSF_SUPR_EVT);
  1694. tr_req[tr_idx].addr = sg_addr;
  1695. tr_req[tr_idx].icnt0 = tr0_cnt0;
  1696. tr_req[tr_idx].icnt1 = tr0_cnt1;
  1697. tr_req[tr_idx].dim1 = tr0_cnt0;
  1698. tr_idx++;
  1699. if (num_tr == 2) {
  1700. cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1,
  1701. false, false,
  1702. CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
  1703. cppi5_tr_csf_set(&tr_req[tr_idx].flags,
  1704. CPPI5_TR_CSF_SUPR_EVT);
  1705. tr_req[tr_idx].addr = sg_addr + tr0_cnt1 * tr0_cnt0;
  1706. tr_req[tr_idx].icnt0 = tr1_cnt0;
  1707. tr_req[tr_idx].icnt1 = 1;
  1708. tr_req[tr_idx].dim1 = tr1_cnt0;
  1709. tr_idx++;
  1710. }
  1711. d->residue += sg_dma_len(sgent);
  1712. }
  1713. cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags,
  1714. CPPI5_TR_CSF_SUPR_EVT | CPPI5_TR_CSF_EOP);
  1715. return d;
  1716. }
  1717. static int udma_configure_statictr(struct udma_chan *uc, struct udma_desc *d,
  1718. enum dma_slave_buswidth dev_width,
  1719. u16 elcnt)
  1720. {
  1721. if (uc->config.ep_type != PSIL_EP_PDMA_XY)
  1722. return 0;
  1723. /* Bus width translates to the element size (ES) */
  1724. switch (dev_width) {
  1725. case DMA_SLAVE_BUSWIDTH_1_BYTE:
  1726. d->static_tr.elsize = 0;
  1727. break;
  1728. case DMA_SLAVE_BUSWIDTH_2_BYTES:
  1729. d->static_tr.elsize = 1;
  1730. break;
  1731. case DMA_SLAVE_BUSWIDTH_3_BYTES:
  1732. d->static_tr.elsize = 2;
  1733. break;
  1734. case DMA_SLAVE_BUSWIDTH_4_BYTES:
  1735. d->static_tr.elsize = 3;
  1736. break;
  1737. case DMA_SLAVE_BUSWIDTH_8_BYTES:
  1738. d->static_tr.elsize = 4;
  1739. break;
  1740. default: /* not reached */
  1741. return -EINVAL;
  1742. }
  1743. d->static_tr.elcnt = elcnt;
  1744. /*
  1745. * PDMA must to close the packet when the channel is in packet mode.
  1746. * For TR mode when the channel is not cyclic we also need PDMA to close
  1747. * the packet otherwise the transfer will stall because PDMA holds on
  1748. * the data it has received from the peripheral.
  1749. */
  1750. if (uc->config.pkt_mode || !uc->cyclic) {
  1751. unsigned int div = dev_width * elcnt;
  1752. if (uc->cyclic)
  1753. d->static_tr.bstcnt = d->residue / d->sglen / div;
  1754. else
  1755. d->static_tr.bstcnt = d->residue / div;
  1756. if (uc->config.dir == DMA_DEV_TO_MEM &&
  1757. d->static_tr.bstcnt > uc->ud->match_data->statictr_z_mask)
  1758. return -EINVAL;
  1759. } else {
  1760. d->static_tr.bstcnt = 0;
  1761. }
  1762. return 0;
  1763. }
  1764. static struct udma_desc *
  1765. udma_prep_slave_sg_pkt(struct udma_chan *uc, struct scatterlist *sgl,
  1766. unsigned int sglen, enum dma_transfer_direction dir,
  1767. unsigned long tx_flags, void *context)
  1768. {
  1769. struct scatterlist *sgent;
  1770. struct cppi5_host_desc_t *h_desc = NULL;
  1771. struct udma_desc *d;
  1772. u32 ring_id;
  1773. unsigned int i;
  1774. d = kzalloc(struct_size(d, hwdesc, sglen), GFP_NOWAIT);
  1775. if (!d)
  1776. return NULL;
  1777. d->sglen = sglen;
  1778. d->hwdesc_count = sglen;
  1779. if (dir == DMA_DEV_TO_MEM)
  1780. ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring);
  1781. else
  1782. ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring);
  1783. for_each_sg(sgl, sgent, sglen, i) {
  1784. struct udma_hwdesc *hwdesc = &d->hwdesc[i];
  1785. dma_addr_t sg_addr = sg_dma_address(sgent);
  1786. struct cppi5_host_desc_t *desc;
  1787. size_t sg_len = sg_dma_len(sgent);
  1788. hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool,
  1789. GFP_NOWAIT,
  1790. &hwdesc->cppi5_desc_paddr);
  1791. if (!hwdesc->cppi5_desc_vaddr) {
  1792. dev_err(uc->ud->dev,
  1793. "descriptor%d allocation failed\n", i);
  1794. udma_free_hwdesc(uc, d);
  1795. kfree(d);
  1796. return NULL;
  1797. }
  1798. d->residue += sg_len;
  1799. hwdesc->cppi5_desc_size = uc->config.hdesc_size;
  1800. desc = hwdesc->cppi5_desc_vaddr;
  1801. if (i == 0) {
  1802. cppi5_hdesc_init(desc, 0, 0);
  1803. /* Flow and Packed ID */
  1804. cppi5_desc_set_pktids(&desc->hdr, uc->id,
  1805. CPPI5_INFO1_DESC_FLOWID_DEFAULT);
  1806. cppi5_desc_set_retpolicy(&desc->hdr, 0, ring_id);
  1807. } else {
  1808. cppi5_hdesc_reset_hbdesc(desc);
  1809. cppi5_desc_set_retpolicy(&desc->hdr, 0, 0xffff);
  1810. }
  1811. /* attach the sg buffer to the descriptor */
  1812. cppi5_hdesc_attach_buf(desc, sg_addr, sg_len, sg_addr, sg_len);
  1813. /* Attach link as host buffer descriptor */
  1814. if (h_desc)
  1815. cppi5_hdesc_link_hbdesc(h_desc,
  1816. hwdesc->cppi5_desc_paddr);
  1817. if (dir == DMA_MEM_TO_DEV)
  1818. h_desc = desc;
  1819. }
  1820. if (d->residue >= SZ_4M) {
  1821. dev_err(uc->ud->dev,
  1822. "%s: Transfer size %u is over the supported 4M range\n",
  1823. __func__, d->residue);
  1824. udma_free_hwdesc(uc, d);
  1825. kfree(d);
  1826. return NULL;
  1827. }
  1828. h_desc = d->hwdesc[0].cppi5_desc_vaddr;
  1829. cppi5_hdesc_set_pktlen(h_desc, d->residue);
  1830. return d;
  1831. }
  1832. static int udma_attach_metadata(struct dma_async_tx_descriptor *desc,
  1833. void *data, size_t len)
  1834. {
  1835. struct udma_desc *d = to_udma_desc(desc);
  1836. struct udma_chan *uc = to_udma_chan(desc->chan);
  1837. struct cppi5_host_desc_t *h_desc;
  1838. u32 psd_size = len;
  1839. u32 flags = 0;
  1840. if (!uc->config.pkt_mode || !uc->config.metadata_size)
  1841. return -ENOTSUPP;
  1842. if (!data || len > uc->config.metadata_size)
  1843. return -EINVAL;
  1844. if (uc->config.needs_epib && len < CPPI5_INFO0_HDESC_EPIB_SIZE)
  1845. return -EINVAL;
  1846. h_desc = d->hwdesc[0].cppi5_desc_vaddr;
  1847. if (d->dir == DMA_MEM_TO_DEV)
  1848. memcpy(h_desc->epib, data, len);
  1849. if (uc->config.needs_epib)
  1850. psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE;
  1851. d->metadata = data;
  1852. d->metadata_size = len;
  1853. if (uc->config.needs_epib)
  1854. flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT;
  1855. cppi5_hdesc_update_flags(h_desc, flags);
  1856. cppi5_hdesc_update_psdata_size(h_desc, psd_size);
  1857. return 0;
  1858. }
  1859. static void *udma_get_metadata_ptr(struct dma_async_tx_descriptor *desc,
  1860. size_t *payload_len, size_t *max_len)
  1861. {
  1862. struct udma_desc *d = to_udma_desc(desc);
  1863. struct udma_chan *uc = to_udma_chan(desc->chan);
  1864. struct cppi5_host_desc_t *h_desc;
  1865. if (!uc->config.pkt_mode || !uc->config.metadata_size)
  1866. return ERR_PTR(-ENOTSUPP);
  1867. h_desc = d->hwdesc[0].cppi5_desc_vaddr;
  1868. *max_len = uc->config.metadata_size;
  1869. *payload_len = cppi5_hdesc_epib_present(&h_desc->hdr) ?
  1870. CPPI5_INFO0_HDESC_EPIB_SIZE : 0;
  1871. *payload_len += cppi5_hdesc_get_psdata_size(h_desc);
  1872. return h_desc->epib;
  1873. }
  1874. static int udma_set_metadata_len(struct dma_async_tx_descriptor *desc,
  1875. size_t payload_len)
  1876. {
  1877. struct udma_desc *d = to_udma_desc(desc);
  1878. struct udma_chan *uc = to_udma_chan(desc->chan);
  1879. struct cppi5_host_desc_t *h_desc;
  1880. u32 psd_size = payload_len;
  1881. u32 flags = 0;
  1882. if (!uc->config.pkt_mode || !uc->config.metadata_size)
  1883. return -ENOTSUPP;
  1884. if (payload_len > uc->config.metadata_size)
  1885. return -EINVAL;
  1886. if (uc->config.needs_epib && payload_len < CPPI5_INFO0_HDESC_EPIB_SIZE)
  1887. return -EINVAL;
  1888. h_desc = d->hwdesc[0].cppi5_desc_vaddr;
  1889. if (uc->config.needs_epib) {
  1890. psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE;
  1891. flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT;
  1892. }
  1893. cppi5_hdesc_update_flags(h_desc, flags);
  1894. cppi5_hdesc_update_psdata_size(h_desc, psd_size);
  1895. return 0;
  1896. }
  1897. static struct dma_descriptor_metadata_ops metadata_ops = {
  1898. .attach = udma_attach_metadata,
  1899. .get_ptr = udma_get_metadata_ptr,
  1900. .set_len = udma_set_metadata_len,
  1901. };
  1902. static struct dma_async_tx_descriptor *
  1903. udma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
  1904. unsigned int sglen, enum dma_transfer_direction dir,
  1905. unsigned long tx_flags, void *context)
  1906. {
  1907. struct udma_chan *uc = to_udma_chan(chan);
  1908. enum dma_slave_buswidth dev_width;
  1909. struct udma_desc *d;
  1910. u32 burst;
  1911. if (dir != uc->config.dir) {
  1912. dev_err(chan->device->dev,
  1913. "%s: chan%d is for %s, not supporting %s\n",
  1914. __func__, uc->id,
  1915. dmaengine_get_direction_text(uc->config.dir),
  1916. dmaengine_get_direction_text(dir));
  1917. return NULL;
  1918. }
  1919. if (dir == DMA_DEV_TO_MEM) {
  1920. dev_width = uc->cfg.src_addr_width;
  1921. burst = uc->cfg.src_maxburst;
  1922. } else if (dir == DMA_MEM_TO_DEV) {
  1923. dev_width = uc->cfg.dst_addr_width;
  1924. burst = uc->cfg.dst_maxburst;
  1925. } else {
  1926. dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
  1927. return NULL;
  1928. }
  1929. if (!burst)
  1930. burst = 1;
  1931. if (uc->config.pkt_mode)
  1932. d = udma_prep_slave_sg_pkt(uc, sgl, sglen, dir, tx_flags,
  1933. context);
  1934. else
  1935. d = udma_prep_slave_sg_tr(uc, sgl, sglen, dir, tx_flags,
  1936. context);
  1937. if (!d)
  1938. return NULL;
  1939. d->dir = dir;
  1940. d->desc_idx = 0;
  1941. d->tr_idx = 0;
  1942. /* static TR for remote PDMA */
  1943. if (udma_configure_statictr(uc, d, dev_width, burst)) {
  1944. dev_err(uc->ud->dev,
  1945. "%s: StaticTR Z is limited to maximum 4095 (%u)\n",
  1946. __func__, d->static_tr.bstcnt);
  1947. udma_free_hwdesc(uc, d);
  1948. kfree(d);
  1949. return NULL;
  1950. }
  1951. if (uc->config.metadata_size)
  1952. d->vd.tx.metadata_ops = &metadata_ops;
  1953. return vchan_tx_prep(&uc->vc, &d->vd, tx_flags);
  1954. }
  1955. static struct udma_desc *
  1956. udma_prep_dma_cyclic_tr(struct udma_chan *uc, dma_addr_t buf_addr,
  1957. size_t buf_len, size_t period_len,
  1958. enum dma_transfer_direction dir, unsigned long flags)
  1959. {
  1960. struct udma_desc *d;
  1961. size_t tr_size, period_addr;
  1962. struct cppi5_tr_type1_t *tr_req;
  1963. unsigned int periods = buf_len / period_len;
  1964. u16 tr0_cnt0, tr0_cnt1, tr1_cnt0;
  1965. unsigned int i;
  1966. int num_tr;
  1967. num_tr = udma_get_tr_counters(period_len, __ffs(buf_addr), &tr0_cnt0,
  1968. &tr0_cnt1, &tr1_cnt0);
  1969. if (num_tr < 0) {
  1970. dev_err(uc->ud->dev, "size %zu is not supported\n",
  1971. period_len);
  1972. return NULL;
  1973. }
  1974. /* Now allocate and setup the descriptor. */
  1975. tr_size = sizeof(struct cppi5_tr_type1_t);
  1976. d = udma_alloc_tr_desc(uc, tr_size, periods * num_tr, dir);
  1977. if (!d)
  1978. return NULL;
  1979. tr_req = d->hwdesc[0].tr_req_base;
  1980. period_addr = buf_addr;
  1981. for (i = 0; i < periods; i++) {
  1982. int tr_idx = i * num_tr;
  1983. cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, false,
  1984. false, CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
  1985. tr_req[tr_idx].addr = period_addr;
  1986. tr_req[tr_idx].icnt0 = tr0_cnt0;
  1987. tr_req[tr_idx].icnt1 = tr0_cnt1;
  1988. tr_req[tr_idx].dim1 = tr0_cnt0;
  1989. if (num_tr == 2) {
  1990. cppi5_tr_csf_set(&tr_req[tr_idx].flags,
  1991. CPPI5_TR_CSF_SUPR_EVT);
  1992. tr_idx++;
  1993. cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1,
  1994. false, false,
  1995. CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
  1996. tr_req[tr_idx].addr = period_addr + tr0_cnt1 * tr0_cnt0;
  1997. tr_req[tr_idx].icnt0 = tr1_cnt0;
  1998. tr_req[tr_idx].icnt1 = 1;
  1999. tr_req[tr_idx].dim1 = tr1_cnt0;
  2000. }
  2001. if (!(flags & DMA_PREP_INTERRUPT))
  2002. cppi5_tr_csf_set(&tr_req[tr_idx].flags,
  2003. CPPI5_TR_CSF_SUPR_EVT);
  2004. period_addr += period_len;
  2005. }
  2006. return d;
  2007. }
  2008. static struct udma_desc *
  2009. udma_prep_dma_cyclic_pkt(struct udma_chan *uc, dma_addr_t buf_addr,
  2010. size_t buf_len, size_t period_len,
  2011. enum dma_transfer_direction dir, unsigned long flags)
  2012. {
  2013. struct udma_desc *d;
  2014. u32 ring_id;
  2015. int i;
  2016. int periods = buf_len / period_len;
  2017. if (periods > (K3_UDMA_DEFAULT_RING_SIZE - 1))
  2018. return NULL;
  2019. if (period_len >= SZ_4M)
  2020. return NULL;
  2021. d = kzalloc(struct_size(d, hwdesc, periods), GFP_NOWAIT);
  2022. if (!d)
  2023. return NULL;
  2024. d->hwdesc_count = periods;
  2025. /* TODO: re-check this... */
  2026. if (dir == DMA_DEV_TO_MEM)
  2027. ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring);
  2028. else
  2029. ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring);
  2030. for (i = 0; i < periods; i++) {
  2031. struct udma_hwdesc *hwdesc = &d->hwdesc[i];
  2032. dma_addr_t period_addr = buf_addr + (period_len * i);
  2033. struct cppi5_host_desc_t *h_desc;
  2034. hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool,
  2035. GFP_NOWAIT,
  2036. &hwdesc->cppi5_desc_paddr);
  2037. if (!hwdesc->cppi5_desc_vaddr) {
  2038. dev_err(uc->ud->dev,
  2039. "descriptor%d allocation failed\n", i);
  2040. udma_free_hwdesc(uc, d);
  2041. kfree(d);
  2042. return NULL;
  2043. }
  2044. hwdesc->cppi5_desc_size = uc->config.hdesc_size;
  2045. h_desc = hwdesc->cppi5_desc_vaddr;
  2046. cppi5_hdesc_init(h_desc, 0, 0);
  2047. cppi5_hdesc_set_pktlen(h_desc, period_len);
  2048. /* Flow and Packed ID */
  2049. cppi5_desc_set_pktids(&h_desc->hdr, uc->id,
  2050. CPPI5_INFO1_DESC_FLOWID_DEFAULT);
  2051. cppi5_desc_set_retpolicy(&h_desc->hdr, 0, ring_id);
  2052. /* attach each period to a new descriptor */
  2053. cppi5_hdesc_attach_buf(h_desc,
  2054. period_addr, period_len,
  2055. period_addr, period_len);
  2056. }
  2057. return d;
  2058. }
  2059. static struct dma_async_tx_descriptor *
  2060. udma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
  2061. size_t period_len, enum dma_transfer_direction dir,
  2062. unsigned long flags)
  2063. {
  2064. struct udma_chan *uc = to_udma_chan(chan);
  2065. enum dma_slave_buswidth dev_width;
  2066. struct udma_desc *d;
  2067. u32 burst;
  2068. if (dir != uc->config.dir) {
  2069. dev_err(chan->device->dev,
  2070. "%s: chan%d is for %s, not supporting %s\n",
  2071. __func__, uc->id,
  2072. dmaengine_get_direction_text(uc->config.dir),
  2073. dmaengine_get_direction_text(dir));
  2074. return NULL;
  2075. }
  2076. uc->cyclic = true;
  2077. if (dir == DMA_DEV_TO_MEM) {
  2078. dev_width = uc->cfg.src_addr_width;
  2079. burst = uc->cfg.src_maxburst;
  2080. } else if (dir == DMA_MEM_TO_DEV) {
  2081. dev_width = uc->cfg.dst_addr_width;
  2082. burst = uc->cfg.dst_maxburst;
  2083. } else {
  2084. dev_err(uc->ud->dev, "%s: bad direction?\n", __func__);
  2085. return NULL;
  2086. }
  2087. if (!burst)
  2088. burst = 1;
  2089. if (uc->config.pkt_mode)
  2090. d = udma_prep_dma_cyclic_pkt(uc, buf_addr, buf_len, period_len,
  2091. dir, flags);
  2092. else
  2093. d = udma_prep_dma_cyclic_tr(uc, buf_addr, buf_len, period_len,
  2094. dir, flags);
  2095. if (!d)
  2096. return NULL;
  2097. d->sglen = buf_len / period_len;
  2098. d->dir = dir;
  2099. d->residue = buf_len;
  2100. /* static TR for remote PDMA */
  2101. if (udma_configure_statictr(uc, d, dev_width, burst)) {
  2102. dev_err(uc->ud->dev,
  2103. "%s: StaticTR Z is limited to maximum 4095 (%u)\n",
  2104. __func__, d->static_tr.bstcnt);
  2105. udma_free_hwdesc(uc, d);
  2106. kfree(d);
  2107. return NULL;
  2108. }
  2109. if (uc->config.metadata_size)
  2110. d->vd.tx.metadata_ops = &metadata_ops;
  2111. return vchan_tx_prep(&uc->vc, &d->vd, flags);
  2112. }
  2113. static struct dma_async_tx_descriptor *
  2114. udma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
  2115. size_t len, unsigned long tx_flags)
  2116. {
  2117. struct udma_chan *uc = to_udma_chan(chan);
  2118. struct udma_desc *d;
  2119. struct cppi5_tr_type15_t *tr_req;
  2120. int num_tr;
  2121. size_t tr_size = sizeof(struct cppi5_tr_type15_t);
  2122. u16 tr0_cnt0, tr0_cnt1, tr1_cnt0;
  2123. if (uc->config.dir != DMA_MEM_TO_MEM) {
  2124. dev_err(chan->device->dev,
  2125. "%s: chan%d is for %s, not supporting %s\n",
  2126. __func__, uc->id,
  2127. dmaengine_get_direction_text(uc->config.dir),
  2128. dmaengine_get_direction_text(DMA_MEM_TO_MEM));
  2129. return NULL;
  2130. }
  2131. num_tr = udma_get_tr_counters(len, __ffs(src | dest), &tr0_cnt0,
  2132. &tr0_cnt1, &tr1_cnt0);
  2133. if (num_tr < 0) {
  2134. dev_err(uc->ud->dev, "size %zu is not supported\n",
  2135. len);
  2136. return NULL;
  2137. }
  2138. d = udma_alloc_tr_desc(uc, tr_size, num_tr, DMA_MEM_TO_MEM);
  2139. if (!d)
  2140. return NULL;
  2141. d->dir = DMA_MEM_TO_MEM;
  2142. d->desc_idx = 0;
  2143. d->tr_idx = 0;
  2144. d->residue = len;
  2145. tr_req = d->hwdesc[0].tr_req_base;
  2146. cppi5_tr_init(&tr_req[0].flags, CPPI5_TR_TYPE15, false, true,
  2147. CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
  2148. cppi5_tr_csf_set(&tr_req[0].flags, CPPI5_TR_CSF_SUPR_EVT);
  2149. tr_req[0].addr = src;
  2150. tr_req[0].icnt0 = tr0_cnt0;
  2151. tr_req[0].icnt1 = tr0_cnt1;
  2152. tr_req[0].icnt2 = 1;
  2153. tr_req[0].icnt3 = 1;
  2154. tr_req[0].dim1 = tr0_cnt0;
  2155. tr_req[0].daddr = dest;
  2156. tr_req[0].dicnt0 = tr0_cnt0;
  2157. tr_req[0].dicnt1 = tr0_cnt1;
  2158. tr_req[0].dicnt2 = 1;
  2159. tr_req[0].dicnt3 = 1;
  2160. tr_req[0].ddim1 = tr0_cnt0;
  2161. if (num_tr == 2) {
  2162. cppi5_tr_init(&tr_req[1].flags, CPPI5_TR_TYPE15, false, true,
  2163. CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
  2164. cppi5_tr_csf_set(&tr_req[1].flags, CPPI5_TR_CSF_SUPR_EVT);
  2165. tr_req[1].addr = src + tr0_cnt1 * tr0_cnt0;
  2166. tr_req[1].icnt0 = tr1_cnt0;
  2167. tr_req[1].icnt1 = 1;
  2168. tr_req[1].icnt2 = 1;
  2169. tr_req[1].icnt3 = 1;
  2170. tr_req[1].daddr = dest + tr0_cnt1 * tr0_cnt0;
  2171. tr_req[1].dicnt0 = tr1_cnt0;
  2172. tr_req[1].dicnt1 = 1;
  2173. tr_req[1].dicnt2 = 1;
  2174. tr_req[1].dicnt3 = 1;
  2175. }
  2176. cppi5_tr_csf_set(&tr_req[num_tr - 1].flags,
  2177. CPPI5_TR_CSF_SUPR_EVT | CPPI5_TR_CSF_EOP);
  2178. if (uc->config.metadata_size)
  2179. d->vd.tx.metadata_ops = &metadata_ops;
  2180. return vchan_tx_prep(&uc->vc, &d->vd, tx_flags);
  2181. }
  2182. static void udma_issue_pending(struct dma_chan *chan)
  2183. {
  2184. struct udma_chan *uc = to_udma_chan(chan);
  2185. unsigned long flags;
  2186. spin_lock_irqsave(&uc->vc.lock, flags);
  2187. /* If we have something pending and no active descriptor, then */
  2188. if (vchan_issue_pending(&uc->vc) && !uc->desc) {
  2189. /*
  2190. * start a descriptor if the channel is NOT [marked as
  2191. * terminating _and_ it is still running (teardown has not
  2192. * completed yet)].
  2193. */
  2194. if (!(uc->state == UDMA_CHAN_IS_TERMINATING &&
  2195. udma_is_chan_running(uc)))
  2196. udma_start(uc);
  2197. }
  2198. spin_unlock_irqrestore(&uc->vc.lock, flags);
  2199. }
  2200. static enum dma_status udma_tx_status(struct dma_chan *chan,
  2201. dma_cookie_t cookie,
  2202. struct dma_tx_state *txstate)
  2203. {
  2204. struct udma_chan *uc = to_udma_chan(chan);
  2205. enum dma_status ret;
  2206. unsigned long flags;
  2207. spin_lock_irqsave(&uc->vc.lock, flags);
  2208. ret = dma_cookie_status(chan, cookie, txstate);
  2209. if (!udma_is_chan_running(uc))
  2210. ret = DMA_COMPLETE;
  2211. if (ret == DMA_IN_PROGRESS && udma_is_chan_paused(uc))
  2212. ret = DMA_PAUSED;
  2213. if (ret == DMA_COMPLETE || !txstate)
  2214. goto out;
  2215. if (uc->desc && uc->desc->vd.tx.cookie == cookie) {
  2216. u32 peer_bcnt = 0;
  2217. u32 bcnt = 0;
  2218. u32 residue = uc->desc->residue;
  2219. u32 delay = 0;
  2220. if (uc->desc->dir == DMA_MEM_TO_DEV) {
  2221. bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_SBCNT_REG);
  2222. if (uc->config.ep_type != PSIL_EP_NATIVE) {
  2223. peer_bcnt = udma_tchanrt_read(uc,
  2224. UDMA_CHAN_RT_PEER_BCNT_REG);
  2225. if (bcnt > peer_bcnt)
  2226. delay = bcnt - peer_bcnt;
  2227. }
  2228. } else if (uc->desc->dir == DMA_DEV_TO_MEM) {
  2229. bcnt = udma_rchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);
  2230. if (uc->config.ep_type != PSIL_EP_NATIVE) {
  2231. peer_bcnt = udma_rchanrt_read(uc,
  2232. UDMA_CHAN_RT_PEER_BCNT_REG);
  2233. if (peer_bcnt > bcnt)
  2234. delay = peer_bcnt - bcnt;
  2235. }
  2236. } else {
  2237. bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);
  2238. }
  2239. bcnt -= uc->bcnt;
  2240. if (bcnt && !(bcnt % uc->desc->residue))
  2241. residue = 0;
  2242. else
  2243. residue -= bcnt % uc->desc->residue;
  2244. if (!residue && (uc->config.dir == DMA_DEV_TO_MEM || !delay)) {
  2245. ret = DMA_COMPLETE;
  2246. delay = 0;
  2247. }
  2248. dma_set_residue(txstate, residue);
  2249. dma_set_in_flight_bytes(txstate, delay);
  2250. } else {
  2251. ret = DMA_COMPLETE;
  2252. }
  2253. out:
  2254. spin_unlock_irqrestore(&uc->vc.lock, flags);
  2255. return ret;
  2256. }
  2257. static int udma_pause(struct dma_chan *chan)
  2258. {
  2259. struct udma_chan *uc = to_udma_chan(chan);
  2260. /* pause the channel */
  2261. switch (uc->config.dir) {
  2262. case DMA_DEV_TO_MEM:
  2263. udma_rchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
  2264. UDMA_PEER_RT_EN_PAUSE,
  2265. UDMA_PEER_RT_EN_PAUSE);
  2266. break;
  2267. case DMA_MEM_TO_DEV:
  2268. udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
  2269. UDMA_PEER_RT_EN_PAUSE,
  2270. UDMA_PEER_RT_EN_PAUSE);
  2271. break;
  2272. case DMA_MEM_TO_MEM:
  2273. udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_CTL_REG,
  2274. UDMA_CHAN_RT_CTL_PAUSE,
  2275. UDMA_CHAN_RT_CTL_PAUSE);
  2276. break;
  2277. default:
  2278. return -EINVAL;
  2279. }
  2280. return 0;
  2281. }
  2282. static int udma_resume(struct dma_chan *chan)
  2283. {
  2284. struct udma_chan *uc = to_udma_chan(chan);
  2285. /* resume the channel */
  2286. switch (uc->config.dir) {
  2287. case DMA_DEV_TO_MEM:
  2288. udma_rchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
  2289. UDMA_PEER_RT_EN_PAUSE, 0);
  2290. break;
  2291. case DMA_MEM_TO_DEV:
  2292. udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
  2293. UDMA_PEER_RT_EN_PAUSE, 0);
  2294. break;
  2295. case DMA_MEM_TO_MEM:
  2296. udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_CTL_REG,
  2297. UDMA_CHAN_RT_CTL_PAUSE, 0);
  2298. break;
  2299. default:
  2300. return -EINVAL;
  2301. }
  2302. return 0;
  2303. }
  2304. static int udma_terminate_all(struct dma_chan *chan)
  2305. {
  2306. struct udma_chan *uc = to_udma_chan(chan);
  2307. unsigned long flags;
  2308. LIST_HEAD(head);
  2309. spin_lock_irqsave(&uc->vc.lock, flags);
  2310. if (udma_is_chan_running(uc))
  2311. udma_stop(uc);
  2312. if (uc->desc) {
  2313. uc->terminated_desc = uc->desc;
  2314. uc->desc = NULL;
  2315. uc->terminated_desc->terminated = true;
  2316. cancel_delayed_work(&uc->tx_drain.work);
  2317. }
  2318. uc->paused = false;
  2319. vchan_get_all_descriptors(&uc->vc, &head);
  2320. spin_unlock_irqrestore(&uc->vc.lock, flags);
  2321. vchan_dma_desc_free_list(&uc->vc, &head);
  2322. return 0;
  2323. }
  2324. static void udma_synchronize(struct dma_chan *chan)
  2325. {
  2326. struct udma_chan *uc = to_udma_chan(chan);
  2327. unsigned long timeout = msecs_to_jiffies(1000);
  2328. vchan_synchronize(&uc->vc);
  2329. if (uc->state == UDMA_CHAN_IS_TERMINATING) {
  2330. timeout = wait_for_completion_timeout(&uc->teardown_completed,
  2331. timeout);
  2332. if (!timeout) {
  2333. dev_warn(uc->ud->dev, "chan%d teardown timeout!\n",
  2334. uc->id);
  2335. udma_dump_chan_stdata(uc);
  2336. udma_reset_chan(uc, true);
  2337. }
  2338. }
  2339. udma_reset_chan(uc, false);
  2340. if (udma_is_chan_running(uc))
  2341. dev_warn(uc->ud->dev, "chan%d refused to stop!\n", uc->id);
  2342. cancel_delayed_work_sync(&uc->tx_drain.work);
  2343. udma_reset_rings(uc);
  2344. }
  2345. static void udma_desc_pre_callback(struct virt_dma_chan *vc,
  2346. struct virt_dma_desc *vd,
  2347. struct dmaengine_result *result)
  2348. {
  2349. struct udma_chan *uc = to_udma_chan(&vc->chan);
  2350. struct udma_desc *d;
  2351. if (!vd)
  2352. return;
  2353. d = to_udma_desc(&vd->tx);
  2354. if (d->metadata_size)
  2355. udma_fetch_epib(uc, d);
  2356. /* Provide residue information for the client */
  2357. if (result) {
  2358. void *desc_vaddr = udma_curr_cppi5_desc_vaddr(d, d->desc_idx);
  2359. if (cppi5_desc_get_type(desc_vaddr) ==
  2360. CPPI5_INFO0_DESC_TYPE_VAL_HOST) {
  2361. result->residue = d->residue -
  2362. cppi5_hdesc_get_pktlen(desc_vaddr);
  2363. if (result->residue)
  2364. result->result = DMA_TRANS_ABORTED;
  2365. else
  2366. result->result = DMA_TRANS_NOERROR;
  2367. } else {
  2368. result->residue = 0;
  2369. result->result = DMA_TRANS_NOERROR;
  2370. }
  2371. }
  2372. }
  2373. /*
  2374. * This tasklet handles the completion of a DMA descriptor by
  2375. * calling its callback and freeing it.
  2376. */
  2377. static void udma_vchan_complete(struct tasklet_struct *t)
  2378. {
  2379. struct virt_dma_chan *vc = from_tasklet(vc, t, task);
  2380. struct virt_dma_desc *vd, *_vd;
  2381. struct dmaengine_desc_callback cb;
  2382. LIST_HEAD(head);
  2383. spin_lock_irq(&vc->lock);
  2384. list_splice_tail_init(&vc->desc_completed, &head);
  2385. vd = vc->cyclic;
  2386. if (vd) {
  2387. vc->cyclic = NULL;
  2388. dmaengine_desc_get_callback(&vd->tx, &cb);
  2389. } else {
  2390. memset(&cb, 0, sizeof(cb));
  2391. }
  2392. spin_unlock_irq(&vc->lock);
  2393. udma_desc_pre_callback(vc, vd, NULL);
  2394. dmaengine_desc_callback_invoke(&cb, NULL);
  2395. list_for_each_entry_safe(vd, _vd, &head, node) {
  2396. struct dmaengine_result result;
  2397. dmaengine_desc_get_callback(&vd->tx, &cb);
  2398. list_del(&vd->node);
  2399. udma_desc_pre_callback(vc, vd, &result);
  2400. dmaengine_desc_callback_invoke(&cb, &result);
  2401. vchan_vdesc_fini(vd);
  2402. }
  2403. }
  2404. static void udma_free_chan_resources(struct dma_chan *chan)
  2405. {
  2406. struct udma_chan *uc = to_udma_chan(chan);
  2407. struct udma_dev *ud = to_udma_dev(chan->device);
  2408. udma_terminate_all(chan);
  2409. if (uc->terminated_desc) {
  2410. udma_reset_chan(uc, false);
  2411. udma_reset_rings(uc);
  2412. }
  2413. cancel_delayed_work_sync(&uc->tx_drain.work);
  2414. if (uc->irq_num_ring > 0) {
  2415. free_irq(uc->irq_num_ring, uc);
  2416. uc->irq_num_ring = 0;
  2417. }
  2418. if (uc->irq_num_udma > 0) {
  2419. free_irq(uc->irq_num_udma, uc);
  2420. uc->irq_num_udma = 0;
  2421. }
  2422. /* Release PSI-L pairing */
  2423. if (uc->psil_paired) {
  2424. navss_psil_unpair(ud, uc->config.src_thread,
  2425. uc->config.dst_thread);
  2426. uc->psil_paired = false;
  2427. }
  2428. vchan_free_chan_resources(&uc->vc);
  2429. tasklet_kill(&uc->vc.task);
  2430. udma_free_tx_resources(uc);
  2431. udma_free_rx_resources(uc);
  2432. udma_reset_uchan(uc);
  2433. if (uc->use_dma_pool) {
  2434. dma_pool_destroy(uc->hdesc_pool);
  2435. uc->use_dma_pool = false;
  2436. }
  2437. }
  2438. static struct platform_driver udma_driver;
  2439. struct udma_filter_param {
  2440. int remote_thread_id;
  2441. u32 atype;
  2442. };
  2443. static bool udma_dma_filter_fn(struct dma_chan *chan, void *param)
  2444. {
  2445. struct udma_chan_config *ucc;
  2446. struct psil_endpoint_config *ep_config;
  2447. struct udma_filter_param *filter_param;
  2448. struct udma_chan *uc;
  2449. struct udma_dev *ud;
  2450. if (chan->device->dev->driver != &udma_driver.driver)
  2451. return false;
  2452. uc = to_udma_chan(chan);
  2453. ucc = &uc->config;
  2454. ud = uc->ud;
  2455. filter_param = param;
  2456. if (filter_param->atype > 2) {
  2457. dev_err(ud->dev, "Invalid channel atype: %u\n",
  2458. filter_param->atype);
  2459. return false;
  2460. }
  2461. ucc->remote_thread_id = filter_param->remote_thread_id;
  2462. ucc->atype = filter_param->atype;
  2463. if (ucc->remote_thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)
  2464. ucc->dir = DMA_MEM_TO_DEV;
  2465. else
  2466. ucc->dir = DMA_DEV_TO_MEM;
  2467. ep_config = psil_get_ep_config(ucc->remote_thread_id);
  2468. if (IS_ERR(ep_config)) {
  2469. dev_err(ud->dev, "No configuration for psi-l thread 0x%04x\n",
  2470. ucc->remote_thread_id);
  2471. ucc->dir = DMA_MEM_TO_MEM;
  2472. ucc->remote_thread_id = -1;
  2473. ucc->atype = 0;
  2474. return false;
  2475. }
  2476. ucc->pkt_mode = ep_config->pkt_mode;
  2477. ucc->channel_tpl = ep_config->channel_tpl;
  2478. ucc->notdpkt = ep_config->notdpkt;
  2479. ucc->ep_type = ep_config->ep_type;
  2480. if (ucc->ep_type != PSIL_EP_NATIVE) {
  2481. const struct udma_match_data *match_data = ud->match_data;
  2482. if (match_data->flags & UDMA_FLAG_PDMA_ACC32)
  2483. ucc->enable_acc32 = ep_config->pdma_acc32;
  2484. if (match_data->flags & UDMA_FLAG_PDMA_BURST)
  2485. ucc->enable_burst = ep_config->pdma_burst;
  2486. }
  2487. ucc->needs_epib = ep_config->needs_epib;
  2488. ucc->psd_size = ep_config->psd_size;
  2489. ucc->metadata_size =
  2490. (ucc->needs_epib ? CPPI5_INFO0_HDESC_EPIB_SIZE : 0) +
  2491. ucc->psd_size;
  2492. if (ucc->pkt_mode)
  2493. ucc->hdesc_size = ALIGN(sizeof(struct cppi5_host_desc_t) +
  2494. ucc->metadata_size, ud->desc_align);
  2495. dev_dbg(ud->dev, "chan%d: Remote thread: 0x%04x (%s)\n", uc->id,
  2496. ucc->remote_thread_id, dmaengine_get_direction_text(ucc->dir));
  2497. return true;
  2498. }
  2499. static struct dma_chan *udma_of_xlate(struct of_phandle_args *dma_spec,
  2500. struct of_dma *ofdma)
  2501. {
  2502. struct udma_dev *ud = ofdma->of_dma_data;
  2503. dma_cap_mask_t mask = ud->ddev.cap_mask;
  2504. struct udma_filter_param filter_param;
  2505. struct dma_chan *chan;
  2506. if (dma_spec->args_count != 1 && dma_spec->args_count != 2)
  2507. return NULL;
  2508. filter_param.remote_thread_id = dma_spec->args[0];
  2509. if (dma_spec->args_count == 2)
  2510. filter_param.atype = dma_spec->args[1];
  2511. else
  2512. filter_param.atype = 0;
  2513. chan = __dma_request_channel(&mask, udma_dma_filter_fn, &filter_param,
  2514. ofdma->of_node);
  2515. if (!chan) {
  2516. dev_err(ud->dev, "get channel fail in %s.\n", __func__);
  2517. return ERR_PTR(-EINVAL);
  2518. }
  2519. return chan;
  2520. }
  2521. static struct udma_match_data am654_main_data = {
  2522. .psil_base = 0x1000,
  2523. .enable_memcpy_support = true,
  2524. .statictr_z_mask = GENMASK(11, 0),
  2525. };
  2526. static struct udma_match_data am654_mcu_data = {
  2527. .psil_base = 0x6000,
  2528. .enable_memcpy_support = false,
  2529. .statictr_z_mask = GENMASK(11, 0),
  2530. };
  2531. static struct udma_match_data j721e_main_data = {
  2532. .psil_base = 0x1000,
  2533. .enable_memcpy_support = true,
  2534. .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST,
  2535. .statictr_z_mask = GENMASK(23, 0),
  2536. };
  2537. static struct udma_match_data j721e_mcu_data = {
  2538. .psil_base = 0x6000,
  2539. .enable_memcpy_support = false, /* MEM_TO_MEM is slow via MCU UDMA */
  2540. .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST,
  2541. .statictr_z_mask = GENMASK(23, 0),
  2542. };
  2543. static const struct of_device_id udma_of_match[] = {
  2544. {
  2545. .compatible = "ti,am654-navss-main-udmap",
  2546. .data = &am654_main_data,
  2547. },
  2548. {
  2549. .compatible = "ti,am654-navss-mcu-udmap",
  2550. .data = &am654_mcu_data,
  2551. }, {
  2552. .compatible = "ti,j721e-navss-main-udmap",
  2553. .data = &j721e_main_data,
  2554. }, {
  2555. .compatible = "ti,j721e-navss-mcu-udmap",
  2556. .data = &j721e_mcu_data,
  2557. },
  2558. { /* Sentinel */ },
  2559. };
  2560. static struct udma_soc_data am654_soc_data = {
  2561. .rchan_oes_offset = 0x200,
  2562. };
  2563. static struct udma_soc_data j721e_soc_data = {
  2564. .rchan_oes_offset = 0x400,
  2565. };
  2566. static struct udma_soc_data j7200_soc_data = {
  2567. .rchan_oes_offset = 0x80,
  2568. };
  2569. static const struct soc_device_attribute k3_soc_devices[] = {
  2570. { .family = "AM65X", .data = &am654_soc_data },
  2571. { .family = "J721E", .data = &j721e_soc_data },
  2572. { .family = "J7200", .data = &j7200_soc_data },
  2573. { /* sentinel */ }
  2574. };
  2575. static int udma_get_mmrs(struct platform_device *pdev, struct udma_dev *ud)
  2576. {
  2577. int i;
  2578. for (i = 0; i < MMR_LAST; i++) {
  2579. ud->mmrs[i] = devm_platform_ioremap_resource_byname(pdev, mmr_names[i]);
  2580. if (IS_ERR(ud->mmrs[i]))
  2581. return PTR_ERR(ud->mmrs[i]);
  2582. }
  2583. return 0;
  2584. }
  2585. static int udma_setup_resources(struct udma_dev *ud)
  2586. {
  2587. struct device *dev = ud->dev;
  2588. int ch_count, ret, i, j;
  2589. u32 cap2, cap3;
  2590. struct ti_sci_resource_desc *rm_desc;
  2591. struct ti_sci_resource *rm_res, irq_res;
  2592. struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
  2593. static const char * const range_names[] = { "ti,sci-rm-range-tchan",
  2594. "ti,sci-rm-range-rchan",
  2595. "ti,sci-rm-range-rflow" };
  2596. cap2 = udma_read(ud->mmrs[MMR_GCFG], UDMA_CAP_REG(2));
  2597. cap3 = udma_read(ud->mmrs[MMR_GCFG], UDMA_CAP_REG(3));
  2598. ud->rflow_cnt = UDMA_CAP3_RFLOW_CNT(cap3);
  2599. ud->tchan_cnt = UDMA_CAP2_TCHAN_CNT(cap2);
  2600. ud->echan_cnt = UDMA_CAP2_ECHAN_CNT(cap2);
  2601. ud->rchan_cnt = UDMA_CAP2_RCHAN_CNT(cap2);
  2602. ch_count = ud->tchan_cnt + ud->rchan_cnt;
  2603. /* Set up the throughput level start indexes */
  2604. if (of_device_is_compatible(dev->of_node,
  2605. "ti,am654-navss-main-udmap")) {
  2606. ud->tpl_levels = 2;
  2607. ud->tpl_start_idx[0] = 8;
  2608. } else if (of_device_is_compatible(dev->of_node,
  2609. "ti,am654-navss-mcu-udmap")) {
  2610. ud->tpl_levels = 2;
  2611. ud->tpl_start_idx[0] = 2;
  2612. } else if (UDMA_CAP3_UCHAN_CNT(cap3)) {
  2613. ud->tpl_levels = 3;
  2614. ud->tpl_start_idx[1] = UDMA_CAP3_UCHAN_CNT(cap3);
  2615. ud->tpl_start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3);
  2616. } else if (UDMA_CAP3_HCHAN_CNT(cap3)) {
  2617. ud->tpl_levels = 2;
  2618. ud->tpl_start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3);
  2619. } else {
  2620. ud->tpl_levels = 1;
  2621. }
  2622. ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt),
  2623. sizeof(unsigned long), GFP_KERNEL);
  2624. ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans),
  2625. GFP_KERNEL);
  2626. ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt),
  2627. sizeof(unsigned long), GFP_KERNEL);
  2628. ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans),
  2629. GFP_KERNEL);
  2630. ud->rflow_gp_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rflow_cnt),
  2631. sizeof(unsigned long),
  2632. GFP_KERNEL);
  2633. ud->rflow_gp_map_allocated = devm_kcalloc(dev,
  2634. BITS_TO_LONGS(ud->rflow_cnt),
  2635. sizeof(unsigned long),
  2636. GFP_KERNEL);
  2637. ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rflow_cnt),
  2638. sizeof(unsigned long),
  2639. GFP_KERNEL);
  2640. ud->rflows = devm_kcalloc(dev, ud->rflow_cnt, sizeof(*ud->rflows),
  2641. GFP_KERNEL);
  2642. if (!ud->tchan_map || !ud->rchan_map || !ud->rflow_gp_map ||
  2643. !ud->rflow_gp_map_allocated || !ud->tchans || !ud->rchans ||
  2644. !ud->rflows || !ud->rflow_in_use)
  2645. return -ENOMEM;
  2646. /*
  2647. * RX flows with the same Ids as RX channels are reserved to be used
  2648. * as default flows if remote HW can't generate flow_ids. Those
  2649. * RX flows can be requested only explicitly by id.
  2650. */
  2651. bitmap_set(ud->rflow_gp_map_allocated, 0, ud->rchan_cnt);
  2652. /* by default no GP rflows are assigned to Linux */
  2653. bitmap_set(ud->rflow_gp_map, 0, ud->rflow_cnt);
  2654. /* Get resource ranges from tisci */
  2655. for (i = 0; i < RM_RANGE_LAST; i++)
  2656. tisci_rm->rm_ranges[i] =
  2657. devm_ti_sci_get_of_resource(tisci_rm->tisci, dev,
  2658. tisci_rm->tisci_dev_id,
  2659. (char *)range_names[i]);
  2660. /* tchan ranges */
  2661. rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
  2662. if (IS_ERR(rm_res)) {
  2663. bitmap_zero(ud->tchan_map, ud->tchan_cnt);
  2664. } else {
  2665. bitmap_fill(ud->tchan_map, ud->tchan_cnt);
  2666. for (i = 0; i < rm_res->sets; i++) {
  2667. rm_desc = &rm_res->desc[i];
  2668. bitmap_clear(ud->tchan_map, rm_desc->start,
  2669. rm_desc->num);
  2670. dev_dbg(dev, "ti-sci-res: tchan: %d:%d\n",
  2671. rm_desc->start, rm_desc->num);
  2672. }
  2673. }
  2674. irq_res.sets = rm_res->sets;
  2675. /* rchan and matching default flow ranges */
  2676. rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
  2677. if (IS_ERR(rm_res)) {
  2678. bitmap_zero(ud->rchan_map, ud->rchan_cnt);
  2679. } else {
  2680. bitmap_fill(ud->rchan_map, ud->rchan_cnt);
  2681. for (i = 0; i < rm_res->sets; i++) {
  2682. rm_desc = &rm_res->desc[i];
  2683. bitmap_clear(ud->rchan_map, rm_desc->start,
  2684. rm_desc->num);
  2685. dev_dbg(dev, "ti-sci-res: rchan: %d:%d\n",
  2686. rm_desc->start, rm_desc->num);
  2687. }
  2688. }
  2689. irq_res.sets += rm_res->sets;
  2690. irq_res.desc = kcalloc(irq_res.sets, sizeof(*irq_res.desc), GFP_KERNEL);
  2691. rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
  2692. for (i = 0; i < rm_res->sets; i++) {
  2693. irq_res.desc[i].start = rm_res->desc[i].start;
  2694. irq_res.desc[i].num = rm_res->desc[i].num;
  2695. }
  2696. rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
  2697. for (j = 0; j < rm_res->sets; j++, i++) {
  2698. irq_res.desc[i].start = rm_res->desc[j].start +
  2699. ud->soc_data->rchan_oes_offset;
  2700. irq_res.desc[i].num = rm_res->desc[j].num;
  2701. }
  2702. ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res);
  2703. kfree(irq_res.desc);
  2704. if (ret) {
  2705. dev_err(ud->dev, "Failed to allocate MSI interrupts\n");
  2706. return ret;
  2707. }
  2708. /* GP rflow ranges */
  2709. rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW];
  2710. if (IS_ERR(rm_res)) {
  2711. /* all gp flows are assigned exclusively to Linux */
  2712. bitmap_clear(ud->rflow_gp_map, ud->rchan_cnt,
  2713. ud->rflow_cnt - ud->rchan_cnt);
  2714. } else {
  2715. for (i = 0; i < rm_res->sets; i++) {
  2716. rm_desc = &rm_res->desc[i];
  2717. bitmap_clear(ud->rflow_gp_map, rm_desc->start,
  2718. rm_desc->num);
  2719. dev_dbg(dev, "ti-sci-res: rflow: %d:%d\n",
  2720. rm_desc->start, rm_desc->num);
  2721. }
  2722. }
  2723. ch_count -= bitmap_weight(ud->tchan_map, ud->tchan_cnt);
  2724. ch_count -= bitmap_weight(ud->rchan_map, ud->rchan_cnt);
  2725. if (!ch_count)
  2726. return -ENODEV;
  2727. ud->channels = devm_kcalloc(dev, ch_count, sizeof(*ud->channels),
  2728. GFP_KERNEL);
  2729. if (!ud->channels)
  2730. return -ENOMEM;
  2731. dev_info(dev, "Channels: %d (tchan: %u, rchan: %u, gp-rflow: %u)\n",
  2732. ch_count,
  2733. ud->tchan_cnt - bitmap_weight(ud->tchan_map, ud->tchan_cnt),
  2734. ud->rchan_cnt - bitmap_weight(ud->rchan_map, ud->rchan_cnt),
  2735. ud->rflow_cnt - bitmap_weight(ud->rflow_gp_map,
  2736. ud->rflow_cnt));
  2737. return ch_count;
  2738. }
  2739. static int udma_setup_rx_flush(struct udma_dev *ud)
  2740. {
  2741. struct udma_rx_flush *rx_flush = &ud->rx_flush;
  2742. struct cppi5_desc_hdr_t *tr_desc;
  2743. struct cppi5_tr_type1_t *tr_req;
  2744. struct cppi5_host_desc_t *desc;
  2745. struct device *dev = ud->dev;
  2746. struct udma_hwdesc *hwdesc;
  2747. size_t tr_size;
  2748. /* Allocate 1K buffer for discarded data on RX channel teardown */
  2749. rx_flush->buffer_size = SZ_1K;
  2750. rx_flush->buffer_vaddr = devm_kzalloc(dev, rx_flush->buffer_size,
  2751. GFP_KERNEL);
  2752. if (!rx_flush->buffer_vaddr)
  2753. return -ENOMEM;
  2754. rx_flush->buffer_paddr = dma_map_single(dev, rx_flush->buffer_vaddr,
  2755. rx_flush->buffer_size,
  2756. DMA_TO_DEVICE);
  2757. if (dma_mapping_error(dev, rx_flush->buffer_paddr))
  2758. return -ENOMEM;
  2759. /* Set up descriptor to be used for TR mode */
  2760. hwdesc = &rx_flush->hwdescs[0];
  2761. tr_size = sizeof(struct cppi5_tr_type1_t);
  2762. hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size, 1);
  2763. hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size,
  2764. ud->desc_align);
  2765. hwdesc->cppi5_desc_vaddr = devm_kzalloc(dev, hwdesc->cppi5_desc_size,
  2766. GFP_KERNEL);
  2767. if (!hwdesc->cppi5_desc_vaddr)
  2768. return -ENOMEM;
  2769. hwdesc->cppi5_desc_paddr = dma_map_single(dev, hwdesc->cppi5_desc_vaddr,
  2770. hwdesc->cppi5_desc_size,
  2771. DMA_TO_DEVICE);
  2772. if (dma_mapping_error(dev, hwdesc->cppi5_desc_paddr))
  2773. return -ENOMEM;
  2774. /* Start of the TR req records */
  2775. hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size;
  2776. /* Start address of the TR response array */
  2777. hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size;
  2778. tr_desc = hwdesc->cppi5_desc_vaddr;
  2779. cppi5_trdesc_init(tr_desc, 1, tr_size, 0, 0);
  2780. cppi5_desc_set_pktids(tr_desc, 0, CPPI5_INFO1_DESC_FLOWID_DEFAULT);
  2781. cppi5_desc_set_retpolicy(tr_desc, 0, 0);
  2782. tr_req = hwdesc->tr_req_base;
  2783. cppi5_tr_init(&tr_req->flags, CPPI5_TR_TYPE1, false, false,
  2784. CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
  2785. cppi5_tr_csf_set(&tr_req->flags, CPPI5_TR_CSF_SUPR_EVT);
  2786. tr_req->addr = rx_flush->buffer_paddr;
  2787. tr_req->icnt0 = rx_flush->buffer_size;
  2788. tr_req->icnt1 = 1;
  2789. dma_sync_single_for_device(dev, hwdesc->cppi5_desc_paddr,
  2790. hwdesc->cppi5_desc_size, DMA_TO_DEVICE);
  2791. /* Set up descriptor to be used for packet mode */
  2792. hwdesc = &rx_flush->hwdescs[1];
  2793. hwdesc->cppi5_desc_size = ALIGN(sizeof(struct cppi5_host_desc_t) +
  2794. CPPI5_INFO0_HDESC_EPIB_SIZE +
  2795. CPPI5_INFO0_HDESC_PSDATA_MAX_SIZE,
  2796. ud->desc_align);
  2797. hwdesc->cppi5_desc_vaddr = devm_kzalloc(dev, hwdesc->cppi5_desc_size,
  2798. GFP_KERNEL);
  2799. if (!hwdesc->cppi5_desc_vaddr)
  2800. return -ENOMEM;
  2801. hwdesc->cppi5_desc_paddr = dma_map_single(dev, hwdesc->cppi5_desc_vaddr,
  2802. hwdesc->cppi5_desc_size,
  2803. DMA_TO_DEVICE);
  2804. if (dma_mapping_error(dev, hwdesc->cppi5_desc_paddr))
  2805. return -ENOMEM;
  2806. desc = hwdesc->cppi5_desc_vaddr;
  2807. cppi5_hdesc_init(desc, 0, 0);
  2808. cppi5_desc_set_pktids(&desc->hdr, 0, CPPI5_INFO1_DESC_FLOWID_DEFAULT);
  2809. cppi5_desc_set_retpolicy(&desc->hdr, 0, 0);
  2810. cppi5_hdesc_attach_buf(desc,
  2811. rx_flush->buffer_paddr, rx_flush->buffer_size,
  2812. rx_flush->buffer_paddr, rx_flush->buffer_size);
  2813. dma_sync_single_for_device(dev, hwdesc->cppi5_desc_paddr,
  2814. hwdesc->cppi5_desc_size, DMA_TO_DEVICE);
  2815. return 0;
  2816. }
  2817. #ifdef CONFIG_DEBUG_FS
  2818. static void udma_dbg_summary_show_chan(struct seq_file *s,
  2819. struct dma_chan *chan)
  2820. {
  2821. struct udma_chan *uc = to_udma_chan(chan);
  2822. struct udma_chan_config *ucc = &uc->config;
  2823. seq_printf(s, " %-13s| %s", dma_chan_name(chan),
  2824. chan->dbg_client_name ?: "in-use");
  2825. seq_printf(s, " (%s, ", dmaengine_get_direction_text(uc->config.dir));
  2826. switch (uc->config.dir) {
  2827. case DMA_MEM_TO_MEM:
  2828. seq_printf(s, "chan%d pair [0x%04x -> 0x%04x], ", uc->tchan->id,
  2829. ucc->src_thread, ucc->dst_thread);
  2830. break;
  2831. case DMA_DEV_TO_MEM:
  2832. seq_printf(s, "rchan%d [0x%04x -> 0x%04x], ", uc->rchan->id,
  2833. ucc->src_thread, ucc->dst_thread);
  2834. break;
  2835. case DMA_MEM_TO_DEV:
  2836. seq_printf(s, "tchan%d [0x%04x -> 0x%04x], ", uc->tchan->id,
  2837. ucc->src_thread, ucc->dst_thread);
  2838. break;
  2839. default:
  2840. seq_printf(s, ")\n");
  2841. return;
  2842. }
  2843. if (ucc->ep_type == PSIL_EP_NATIVE) {
  2844. seq_printf(s, "PSI-L Native");
  2845. if (ucc->metadata_size) {
  2846. seq_printf(s, "[%s", ucc->needs_epib ? " EPIB" : "");
  2847. if (ucc->psd_size)
  2848. seq_printf(s, " PSDsize:%u", ucc->psd_size);
  2849. seq_printf(s, " ]");
  2850. }
  2851. } else {
  2852. seq_printf(s, "PDMA");
  2853. if (ucc->enable_acc32 || ucc->enable_burst)
  2854. seq_printf(s, "[%s%s ]",
  2855. ucc->enable_acc32 ? " ACC32" : "",
  2856. ucc->enable_burst ? " BURST" : "");
  2857. }
  2858. seq_printf(s, ", %s)\n", ucc->pkt_mode ? "Packet mode" : "TR mode");
  2859. }
  2860. static void udma_dbg_summary_show(struct seq_file *s,
  2861. struct dma_device *dma_dev)
  2862. {
  2863. struct dma_chan *chan;
  2864. list_for_each_entry(chan, &dma_dev->channels, device_node) {
  2865. if (chan->client_count)
  2866. udma_dbg_summary_show_chan(s, chan);
  2867. }
  2868. }
  2869. #endif /* CONFIG_DEBUG_FS */
  2870. #define TI_UDMAC_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
  2871. BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
  2872. BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
  2873. BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \
  2874. BIT(DMA_SLAVE_BUSWIDTH_8_BYTES))
  2875. static int udma_probe(struct platform_device *pdev)
  2876. {
  2877. struct device_node *navss_node = pdev->dev.parent->of_node;
  2878. const struct soc_device_attribute *soc;
  2879. struct device *dev = &pdev->dev;
  2880. struct udma_dev *ud;
  2881. const struct of_device_id *match;
  2882. int i, ret;
  2883. int ch_count;
  2884. ret = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(48));
  2885. if (ret)
  2886. dev_err(dev, "failed to set dma mask stuff\n");
  2887. ud = devm_kzalloc(dev, sizeof(*ud), GFP_KERNEL);
  2888. if (!ud)
  2889. return -ENOMEM;
  2890. ret = udma_get_mmrs(pdev, ud);
  2891. if (ret)
  2892. return ret;
  2893. ud->tisci_rm.tisci = ti_sci_get_by_phandle(dev->of_node, "ti,sci");
  2894. if (IS_ERR(ud->tisci_rm.tisci))
  2895. return PTR_ERR(ud->tisci_rm.tisci);
  2896. ret = of_property_read_u32(dev->of_node, "ti,sci-dev-id",
  2897. &ud->tisci_rm.tisci_dev_id);
  2898. if (ret) {
  2899. dev_err(dev, "ti,sci-dev-id read failure %d\n", ret);
  2900. return ret;
  2901. }
  2902. pdev->id = ud->tisci_rm.tisci_dev_id;
  2903. ret = of_property_read_u32(navss_node, "ti,sci-dev-id",
  2904. &ud->tisci_rm.tisci_navss_dev_id);
  2905. if (ret) {
  2906. dev_err(dev, "NAVSS ti,sci-dev-id read failure %d\n", ret);
  2907. return ret;
  2908. }
  2909. ret = of_property_read_u32(dev->of_node, "ti,udma-atype", &ud->atype);
  2910. if (!ret && ud->atype > 2) {
  2911. dev_err(dev, "Invalid atype: %u\n", ud->atype);
  2912. return -EINVAL;
  2913. }
  2914. ud->tisci_rm.tisci_udmap_ops = &ud->tisci_rm.tisci->ops.rm_udmap_ops;
  2915. ud->tisci_rm.tisci_psil_ops = &ud->tisci_rm.tisci->ops.rm_psil_ops;
  2916. ud->ringacc = of_k3_ringacc_get_by_phandle(dev->of_node, "ti,ringacc");
  2917. if (IS_ERR(ud->ringacc))
  2918. return PTR_ERR(ud->ringacc);
  2919. dev->msi_domain = of_msi_get_domain(dev, dev->of_node,
  2920. DOMAIN_BUS_TI_SCI_INTA_MSI);
  2921. if (!dev->msi_domain) {
  2922. dev_err(dev, "Failed to get MSI domain\n");
  2923. return -EPROBE_DEFER;
  2924. }
  2925. match = of_match_node(udma_of_match, dev->of_node);
  2926. if (!match) {
  2927. dev_err(dev, "No compatible match found\n");
  2928. return -ENODEV;
  2929. }
  2930. ud->match_data = match->data;
  2931. soc = soc_device_match(k3_soc_devices);
  2932. if (!soc) {
  2933. dev_err(dev, "No compatible SoC found\n");
  2934. return -ENODEV;
  2935. }
  2936. ud->soc_data = soc->data;
  2937. dma_cap_set(DMA_SLAVE, ud->ddev.cap_mask);
  2938. dma_cap_set(DMA_CYCLIC, ud->ddev.cap_mask);
  2939. ud->ddev.device_alloc_chan_resources = udma_alloc_chan_resources;
  2940. ud->ddev.device_config = udma_slave_config;
  2941. ud->ddev.device_prep_slave_sg = udma_prep_slave_sg;
  2942. ud->ddev.device_prep_dma_cyclic = udma_prep_dma_cyclic;
  2943. ud->ddev.device_issue_pending = udma_issue_pending;
  2944. ud->ddev.device_tx_status = udma_tx_status;
  2945. ud->ddev.device_pause = udma_pause;
  2946. ud->ddev.device_resume = udma_resume;
  2947. ud->ddev.device_terminate_all = udma_terminate_all;
  2948. ud->ddev.device_synchronize = udma_synchronize;
  2949. #ifdef CONFIG_DEBUG_FS
  2950. ud->ddev.dbg_summary_show = udma_dbg_summary_show;
  2951. #endif
  2952. ud->ddev.device_free_chan_resources = udma_free_chan_resources;
  2953. ud->ddev.src_addr_widths = TI_UDMAC_BUSWIDTHS;
  2954. ud->ddev.dst_addr_widths = TI_UDMAC_BUSWIDTHS;
  2955. ud->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
  2956. ud->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
  2957. ud->ddev.copy_align = DMAENGINE_ALIGN_8_BYTES;
  2958. ud->ddev.desc_metadata_modes = DESC_METADATA_CLIENT |
  2959. DESC_METADATA_ENGINE;
  2960. if (ud->match_data->enable_memcpy_support) {
  2961. dma_cap_set(DMA_MEMCPY, ud->ddev.cap_mask);
  2962. ud->ddev.device_prep_dma_memcpy = udma_prep_dma_memcpy;
  2963. ud->ddev.directions |= BIT(DMA_MEM_TO_MEM);
  2964. }
  2965. ud->ddev.dev = dev;
  2966. ud->dev = dev;
  2967. ud->psil_base = ud->match_data->psil_base;
  2968. INIT_LIST_HEAD(&ud->ddev.channels);
  2969. INIT_LIST_HEAD(&ud->desc_to_purge);
  2970. ch_count = udma_setup_resources(ud);
  2971. if (ch_count <= 0)
  2972. return ch_count;
  2973. spin_lock_init(&ud->lock);
  2974. INIT_WORK(&ud->purge_work, udma_purge_desc_work);
  2975. ud->desc_align = 64;
  2976. if (ud->desc_align < dma_get_cache_alignment())
  2977. ud->desc_align = dma_get_cache_alignment();
  2978. ret = udma_setup_rx_flush(ud);
  2979. if (ret)
  2980. return ret;
  2981. for (i = 0; i < ud->tchan_cnt; i++) {
  2982. struct udma_tchan *tchan = &ud->tchans[i];
  2983. tchan->id = i;
  2984. tchan->reg_rt = ud->mmrs[MMR_TCHANRT] + i * 0x1000;
  2985. }
  2986. for (i = 0; i < ud->rchan_cnt; i++) {
  2987. struct udma_rchan *rchan = &ud->rchans[i];
  2988. rchan->id = i;
  2989. rchan->reg_rt = ud->mmrs[MMR_RCHANRT] + i * 0x1000;
  2990. }
  2991. for (i = 0; i < ud->rflow_cnt; i++) {
  2992. struct udma_rflow *rflow = &ud->rflows[i];
  2993. rflow->id = i;
  2994. }
  2995. for (i = 0; i < ch_count; i++) {
  2996. struct udma_chan *uc = &ud->channels[i];
  2997. uc->ud = ud;
  2998. uc->vc.desc_free = udma_desc_free;
  2999. uc->id = i;
  3000. uc->tchan = NULL;
  3001. uc->rchan = NULL;
  3002. uc->config.remote_thread_id = -1;
  3003. uc->config.dir = DMA_MEM_TO_MEM;
  3004. uc->name = devm_kasprintf(dev, GFP_KERNEL, "%s chan%d",
  3005. dev_name(dev), i);
  3006. vchan_init(&uc->vc, &ud->ddev);
  3007. /* Use custom vchan completion handling */
  3008. tasklet_setup(&uc->vc.task, udma_vchan_complete);
  3009. init_completion(&uc->teardown_completed);
  3010. INIT_DELAYED_WORK(&uc->tx_drain.work, udma_check_tx_completion);
  3011. }
  3012. ret = dma_async_device_register(&ud->ddev);
  3013. if (ret) {
  3014. dev_err(dev, "failed to register slave DMA engine: %d\n", ret);
  3015. return ret;
  3016. }
  3017. platform_set_drvdata(pdev, ud);
  3018. ret = of_dma_controller_register(dev->of_node, udma_of_xlate, ud);
  3019. if (ret) {
  3020. dev_err(dev, "failed to register of_dma controller\n");
  3021. dma_async_device_unregister(&ud->ddev);
  3022. }
  3023. return ret;
  3024. }
  3025. static struct platform_driver udma_driver = {
  3026. .driver = {
  3027. .name = "ti-udma",
  3028. .of_match_table = udma_of_match,
  3029. .suppress_bind_attrs = true,
  3030. },
  3031. .probe = udma_probe,
  3032. };
  3033. builtin_platform_driver(udma_driver);
  3034. /* Private interfaces to UDMA */
  3035. #include "k3-udma-private.c"