adma.c 124 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * Copyright (C) 2006-2009 DENX Software Engineering.
  4. *
  5. * Author: Yuri Tikhonov <yur@emcraft.com>
  6. *
  7. * Further porting to arch/powerpc by
  8. * Anatolij Gustschin <agust@denx.de>
  9. */
  10. /*
  11. * This driver supports the asynchrounous DMA copy and RAID engines available
  12. * on the AMCC PPC440SPe Processors.
  13. * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x)
  14. * ADMA driver written by D.Williams.
  15. */
  16. #include <linux/init.h>
  17. #include <linux/module.h>
  18. #include <linux/async_tx.h>
  19. #include <linux/delay.h>
  20. #include <linux/dma-mapping.h>
  21. #include <linux/spinlock.h>
  22. #include <linux/interrupt.h>
  23. #include <linux/slab.h>
  24. #include <linux/uaccess.h>
  25. #include <linux/proc_fs.h>
  26. #include <linux/of.h>
  27. #include <linux/of_address.h>
  28. #include <linux/of_irq.h>
  29. #include <linux/of_platform.h>
  30. #include <asm/dcr.h>
  31. #include <asm/dcr-regs.h>
  32. #include "adma.h"
  33. #include "../dmaengine.h"
  34. enum ppc_adma_init_code {
  35. PPC_ADMA_INIT_OK = 0,
  36. PPC_ADMA_INIT_MEMRES,
  37. PPC_ADMA_INIT_MEMREG,
  38. PPC_ADMA_INIT_ALLOC,
  39. PPC_ADMA_INIT_COHERENT,
  40. PPC_ADMA_INIT_CHANNEL,
  41. PPC_ADMA_INIT_IRQ1,
  42. PPC_ADMA_INIT_IRQ2,
  43. PPC_ADMA_INIT_REGISTER
  44. };
  45. static char *ppc_adma_errors[] = {
  46. [PPC_ADMA_INIT_OK] = "ok",
  47. [PPC_ADMA_INIT_MEMRES] = "failed to get memory resource",
  48. [PPC_ADMA_INIT_MEMREG] = "failed to request memory region",
  49. [PPC_ADMA_INIT_ALLOC] = "failed to allocate memory for adev "
  50. "structure",
  51. [PPC_ADMA_INIT_COHERENT] = "failed to allocate coherent memory for "
  52. "hardware descriptors",
  53. [PPC_ADMA_INIT_CHANNEL] = "failed to allocate memory for channel",
  54. [PPC_ADMA_INIT_IRQ1] = "failed to request first irq",
  55. [PPC_ADMA_INIT_IRQ2] = "failed to request second irq",
  56. [PPC_ADMA_INIT_REGISTER] = "failed to register dma async device",
  57. };
  58. static enum ppc_adma_init_code
  59. ppc440spe_adma_devices[PPC440SPE_ADMA_ENGINES_NUM];
  60. struct ppc_dma_chan_ref {
  61. struct dma_chan *chan;
  62. struct list_head node;
  63. };
  64. /* The list of channels exported by ppc440spe ADMA */
  65. struct list_head
  66. ppc440spe_adma_chan_list = LIST_HEAD_INIT(ppc440spe_adma_chan_list);
  67. /* This flag is set when want to refetch the xor chain in the interrupt
  68. * handler
  69. */
  70. static u32 do_xor_refetch;
  71. /* Pointer to DMA0, DMA1 CP/CS FIFO */
  72. static void *ppc440spe_dma_fifo_buf;
  73. /* Pointers to last submitted to DMA0, DMA1 CDBs */
  74. static struct ppc440spe_adma_desc_slot *chan_last_sub[3];
  75. static struct ppc440spe_adma_desc_slot *chan_first_cdb[3];
  76. /* Pointer to last linked and submitted xor CB */
  77. static struct ppc440spe_adma_desc_slot *xor_last_linked;
  78. static struct ppc440spe_adma_desc_slot *xor_last_submit;
  79. /* This array is used in data-check operations for storing a pattern */
  80. static char ppc440spe_qword[16];
  81. static atomic_t ppc440spe_adma_err_irq_ref;
  82. static dcr_host_t ppc440spe_mq_dcr_host;
  83. static unsigned int ppc440spe_mq_dcr_len;
  84. /* Since RXOR operations use the common register (MQ0_CF2H) for setting-up
  85. * the block size in transactions, then we do not allow to activate more than
  86. * only one RXOR transactions simultaneously. So use this var to store
  87. * the information about is RXOR currently active (PPC440SPE_RXOR_RUN bit is
  88. * set) or not (PPC440SPE_RXOR_RUN is clear).
  89. */
  90. static unsigned long ppc440spe_rxor_state;
  91. /* These are used in enable & check routines
  92. */
  93. static u32 ppc440spe_r6_enabled;
  94. static struct ppc440spe_adma_chan *ppc440spe_r6_tchan;
  95. static struct completion ppc440spe_r6_test_comp;
  96. static int ppc440spe_adma_dma2rxor_prep_src(
  97. struct ppc440spe_adma_desc_slot *desc,
  98. struct ppc440spe_rxor *cursor, int index,
  99. int src_cnt, u32 addr);
  100. static void ppc440spe_adma_dma2rxor_set_src(
  101. struct ppc440spe_adma_desc_slot *desc,
  102. int index, dma_addr_t addr);
  103. static void ppc440spe_adma_dma2rxor_set_mult(
  104. struct ppc440spe_adma_desc_slot *desc,
  105. int index, u8 mult);
  106. #ifdef ADMA_LL_DEBUG
  107. #define ADMA_LL_DBG(x) ({ if (1) x; 0; })
  108. #else
  109. #define ADMA_LL_DBG(x) ({ if (0) x; 0; })
  110. #endif
  111. static void print_cb(struct ppc440spe_adma_chan *chan, void *block)
  112. {
  113. struct dma_cdb *cdb;
  114. struct xor_cb *cb;
  115. int i;
  116. switch (chan->device->id) {
  117. case 0:
  118. case 1:
  119. cdb = block;
  120. pr_debug("CDB at %p [%d]:\n"
  121. "\t attr 0x%02x opc 0x%02x cnt 0x%08x\n"
  122. "\t sg1u 0x%08x sg1l 0x%08x\n"
  123. "\t sg2u 0x%08x sg2l 0x%08x\n"
  124. "\t sg3u 0x%08x sg3l 0x%08x\n",
  125. cdb, chan->device->id,
  126. cdb->attr, cdb->opc, le32_to_cpu(cdb->cnt),
  127. le32_to_cpu(cdb->sg1u), le32_to_cpu(cdb->sg1l),
  128. le32_to_cpu(cdb->sg2u), le32_to_cpu(cdb->sg2l),
  129. le32_to_cpu(cdb->sg3u), le32_to_cpu(cdb->sg3l)
  130. );
  131. break;
  132. case 2:
  133. cb = block;
  134. pr_debug("CB at %p [%d]:\n"
  135. "\t cbc 0x%08x cbbc 0x%08x cbs 0x%08x\n"
  136. "\t cbtah 0x%08x cbtal 0x%08x\n"
  137. "\t cblah 0x%08x cblal 0x%08x\n",
  138. cb, chan->device->id,
  139. cb->cbc, cb->cbbc, cb->cbs,
  140. cb->cbtah, cb->cbtal,
  141. cb->cblah, cb->cblal);
  142. for (i = 0; i < 16; i++) {
  143. if (i && !cb->ops[i].h && !cb->ops[i].l)
  144. continue;
  145. pr_debug("\t ops[%2d]: h 0x%08x l 0x%08x\n",
  146. i, cb->ops[i].h, cb->ops[i].l);
  147. }
  148. break;
  149. }
  150. }
  151. static void print_cb_list(struct ppc440spe_adma_chan *chan,
  152. struct ppc440spe_adma_desc_slot *iter)
  153. {
  154. for (; iter; iter = iter->hw_next)
  155. print_cb(chan, iter->hw_desc);
  156. }
  157. static void prep_dma_xor_dbg(int id, dma_addr_t dst, dma_addr_t *src,
  158. unsigned int src_cnt)
  159. {
  160. int i;
  161. pr_debug("\n%s(%d):\nsrc: ", __func__, id);
  162. for (i = 0; i < src_cnt; i++)
  163. pr_debug("\t0x%016llx ", src[i]);
  164. pr_debug("dst:\n\t0x%016llx\n", dst);
  165. }
  166. static void prep_dma_pq_dbg(int id, dma_addr_t *dst, dma_addr_t *src,
  167. unsigned int src_cnt)
  168. {
  169. int i;
  170. pr_debug("\n%s(%d):\nsrc: ", __func__, id);
  171. for (i = 0; i < src_cnt; i++)
  172. pr_debug("\t0x%016llx ", src[i]);
  173. pr_debug("dst: ");
  174. for (i = 0; i < 2; i++)
  175. pr_debug("\t0x%016llx ", dst[i]);
  176. }
  177. static void prep_dma_pqzero_sum_dbg(int id, dma_addr_t *src,
  178. unsigned int src_cnt,
  179. const unsigned char *scf)
  180. {
  181. int i;
  182. pr_debug("\n%s(%d):\nsrc(coef): ", __func__, id);
  183. if (scf) {
  184. for (i = 0; i < src_cnt; i++)
  185. pr_debug("\t0x%016llx(0x%02x) ", src[i], scf[i]);
  186. } else {
  187. for (i = 0; i < src_cnt; i++)
  188. pr_debug("\t0x%016llx(no) ", src[i]);
  189. }
  190. pr_debug("dst: ");
  191. for (i = 0; i < 2; i++)
  192. pr_debug("\t0x%016llx ", src[src_cnt + i]);
  193. }
  194. /******************************************************************************
  195. * Command (Descriptor) Blocks low-level routines
  196. ******************************************************************************/
  197. /**
  198. * ppc440spe_desc_init_interrupt - initialize the descriptor for INTERRUPT
  199. * pseudo operation
  200. */
  201. static void ppc440spe_desc_init_interrupt(struct ppc440spe_adma_desc_slot *desc,
  202. struct ppc440spe_adma_chan *chan)
  203. {
  204. struct xor_cb *p;
  205. switch (chan->device->id) {
  206. case PPC440SPE_XOR_ID:
  207. p = desc->hw_desc;
  208. memset(desc->hw_desc, 0, sizeof(struct xor_cb));
  209. /* NOP with Command Block Complete Enable */
  210. p->cbc = XOR_CBCR_CBCE_BIT;
  211. break;
  212. case PPC440SPE_DMA0_ID:
  213. case PPC440SPE_DMA1_ID:
  214. memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
  215. /* NOP with interrupt */
  216. set_bit(PPC440SPE_DESC_INT, &desc->flags);
  217. break;
  218. default:
  219. printk(KERN_ERR "Unsupported id %d in %s\n", chan->device->id,
  220. __func__);
  221. break;
  222. }
  223. }
  224. /**
  225. * ppc440spe_desc_init_null_xor - initialize the descriptor for NULL XOR
  226. * pseudo operation
  227. */
  228. static void ppc440spe_desc_init_null_xor(struct ppc440spe_adma_desc_slot *desc)
  229. {
  230. memset(desc->hw_desc, 0, sizeof(struct xor_cb));
  231. desc->hw_next = NULL;
  232. desc->src_cnt = 0;
  233. desc->dst_cnt = 1;
  234. }
  235. /**
  236. * ppc440spe_desc_init_xor - initialize the descriptor for XOR operation
  237. */
  238. static void ppc440spe_desc_init_xor(struct ppc440spe_adma_desc_slot *desc,
  239. int src_cnt, unsigned long flags)
  240. {
  241. struct xor_cb *hw_desc = desc->hw_desc;
  242. memset(desc->hw_desc, 0, sizeof(struct xor_cb));
  243. desc->hw_next = NULL;
  244. desc->src_cnt = src_cnt;
  245. desc->dst_cnt = 1;
  246. hw_desc->cbc = XOR_CBCR_TGT_BIT | src_cnt;
  247. if (flags & DMA_PREP_INTERRUPT)
  248. /* Enable interrupt on completion */
  249. hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
  250. }
  251. /**
  252. * ppc440spe_desc_init_dma2pq - initialize the descriptor for PQ
  253. * operation in DMA2 controller
  254. */
  255. static void ppc440spe_desc_init_dma2pq(struct ppc440spe_adma_desc_slot *desc,
  256. int dst_cnt, int src_cnt, unsigned long flags)
  257. {
  258. struct xor_cb *hw_desc = desc->hw_desc;
  259. memset(desc->hw_desc, 0, sizeof(struct xor_cb));
  260. desc->hw_next = NULL;
  261. desc->src_cnt = src_cnt;
  262. desc->dst_cnt = dst_cnt;
  263. memset(desc->reverse_flags, 0, sizeof(desc->reverse_flags));
  264. desc->descs_per_op = 0;
  265. hw_desc->cbc = XOR_CBCR_TGT_BIT;
  266. if (flags & DMA_PREP_INTERRUPT)
  267. /* Enable interrupt on completion */
  268. hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
  269. }
  270. #define DMA_CTRL_FLAGS_LAST DMA_PREP_FENCE
  271. #define DMA_PREP_ZERO_P (DMA_CTRL_FLAGS_LAST << 1)
  272. #define DMA_PREP_ZERO_Q (DMA_PREP_ZERO_P << 1)
  273. /**
  274. * ppc440spe_desc_init_dma01pq - initialize the descriptors for PQ operation
  275. * with DMA0/1
  276. */
  277. static void ppc440spe_desc_init_dma01pq(struct ppc440spe_adma_desc_slot *desc,
  278. int dst_cnt, int src_cnt, unsigned long flags,
  279. unsigned long op)
  280. {
  281. struct dma_cdb *hw_desc;
  282. struct ppc440spe_adma_desc_slot *iter;
  283. u8 dopc;
  284. /* Common initialization of a PQ descriptors chain */
  285. set_bits(op, &desc->flags);
  286. desc->src_cnt = src_cnt;
  287. desc->dst_cnt = dst_cnt;
  288. /* WXOR MULTICAST if both P and Q are being computed
  289. * MV_SG1_SG2 if Q only
  290. */
  291. dopc = (desc->dst_cnt == DMA_DEST_MAX_NUM) ?
  292. DMA_CDB_OPC_MULTICAST : DMA_CDB_OPC_MV_SG1_SG2;
  293. list_for_each_entry(iter, &desc->group_list, chain_node) {
  294. hw_desc = iter->hw_desc;
  295. memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
  296. if (likely(!list_is_last(&iter->chain_node,
  297. &desc->group_list))) {
  298. /* set 'next' pointer */
  299. iter->hw_next = list_entry(iter->chain_node.next,
  300. struct ppc440spe_adma_desc_slot, chain_node);
  301. clear_bit(PPC440SPE_DESC_INT, &iter->flags);
  302. } else {
  303. /* this is the last descriptor.
  304. * this slot will be pasted from ADMA level
  305. * each time it wants to configure parameters
  306. * of the transaction (src, dst, ...)
  307. */
  308. iter->hw_next = NULL;
  309. if (flags & DMA_PREP_INTERRUPT)
  310. set_bit(PPC440SPE_DESC_INT, &iter->flags);
  311. else
  312. clear_bit(PPC440SPE_DESC_INT, &iter->flags);
  313. }
  314. }
  315. /* Set OPS depending on WXOR/RXOR type of operation */
  316. if (!test_bit(PPC440SPE_DESC_RXOR, &desc->flags)) {
  317. /* This is a WXOR only chain:
  318. * - first descriptors are for zeroing destinations
  319. * if PPC440SPE_ZERO_P/Q set;
  320. * - descriptors remained are for GF-XOR operations.
  321. */
  322. iter = list_first_entry(&desc->group_list,
  323. struct ppc440spe_adma_desc_slot,
  324. chain_node);
  325. if (test_bit(PPC440SPE_ZERO_P, &desc->flags)) {
  326. hw_desc = iter->hw_desc;
  327. hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
  328. iter = list_first_entry(&iter->chain_node,
  329. struct ppc440spe_adma_desc_slot,
  330. chain_node);
  331. }
  332. if (test_bit(PPC440SPE_ZERO_Q, &desc->flags)) {
  333. hw_desc = iter->hw_desc;
  334. hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
  335. iter = list_first_entry(&iter->chain_node,
  336. struct ppc440spe_adma_desc_slot,
  337. chain_node);
  338. }
  339. list_for_each_entry_from(iter, &desc->group_list, chain_node) {
  340. hw_desc = iter->hw_desc;
  341. hw_desc->opc = dopc;
  342. }
  343. } else {
  344. /* This is either RXOR-only or mixed RXOR/WXOR */
  345. /* The first 1 or 2 slots in chain are always RXOR,
  346. * if need to calculate P & Q, then there are two
  347. * RXOR slots; if only P or only Q, then there is one
  348. */
  349. iter = list_first_entry(&desc->group_list,
  350. struct ppc440spe_adma_desc_slot,
  351. chain_node);
  352. hw_desc = iter->hw_desc;
  353. hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
  354. if (desc->dst_cnt == DMA_DEST_MAX_NUM) {
  355. iter = list_first_entry(&iter->chain_node,
  356. struct ppc440spe_adma_desc_slot,
  357. chain_node);
  358. hw_desc = iter->hw_desc;
  359. hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
  360. }
  361. /* The remaining descs (if any) are WXORs */
  362. if (test_bit(PPC440SPE_DESC_WXOR, &desc->flags)) {
  363. iter = list_first_entry(&iter->chain_node,
  364. struct ppc440spe_adma_desc_slot,
  365. chain_node);
  366. list_for_each_entry_from(iter, &desc->group_list,
  367. chain_node) {
  368. hw_desc = iter->hw_desc;
  369. hw_desc->opc = dopc;
  370. }
  371. }
  372. }
  373. }
  374. /**
  375. * ppc440spe_desc_init_dma01pqzero_sum - initialize the descriptor
  376. * for PQ_ZERO_SUM operation
  377. */
  378. static void ppc440spe_desc_init_dma01pqzero_sum(
  379. struct ppc440spe_adma_desc_slot *desc,
  380. int dst_cnt, int src_cnt)
  381. {
  382. struct dma_cdb *hw_desc;
  383. struct ppc440spe_adma_desc_slot *iter;
  384. int i = 0;
  385. u8 dopc = (dst_cnt == 2) ? DMA_CDB_OPC_MULTICAST :
  386. DMA_CDB_OPC_MV_SG1_SG2;
  387. /*
  388. * Initialize starting from 2nd or 3rd descriptor dependent
  389. * on dst_cnt. First one or two slots are for cloning P
  390. * and/or Q to chan->pdest and/or chan->qdest as we have
  391. * to preserve original P/Q.
  392. */
  393. iter = list_first_entry(&desc->group_list,
  394. struct ppc440spe_adma_desc_slot, chain_node);
  395. iter = list_entry(iter->chain_node.next,
  396. struct ppc440spe_adma_desc_slot, chain_node);
  397. if (dst_cnt > 1) {
  398. iter = list_entry(iter->chain_node.next,
  399. struct ppc440spe_adma_desc_slot, chain_node);
  400. }
  401. /* initialize each source descriptor in chain */
  402. list_for_each_entry_from(iter, &desc->group_list, chain_node) {
  403. hw_desc = iter->hw_desc;
  404. memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
  405. iter->src_cnt = 0;
  406. iter->dst_cnt = 0;
  407. /* This is a ZERO_SUM operation:
  408. * - <src_cnt> descriptors starting from 2nd or 3rd
  409. * descriptor are for GF-XOR operations;
  410. * - remaining <dst_cnt> descriptors are for checking the result
  411. */
  412. if (i++ < src_cnt)
  413. /* MV_SG1_SG2 if only Q is being verified
  414. * MULTICAST if both P and Q are being verified
  415. */
  416. hw_desc->opc = dopc;
  417. else
  418. /* DMA_CDB_OPC_DCHECK128 operation */
  419. hw_desc->opc = DMA_CDB_OPC_DCHECK128;
  420. if (likely(!list_is_last(&iter->chain_node,
  421. &desc->group_list))) {
  422. /* set 'next' pointer */
  423. iter->hw_next = list_entry(iter->chain_node.next,
  424. struct ppc440spe_adma_desc_slot,
  425. chain_node);
  426. } else {
  427. /* this is the last descriptor.
  428. * this slot will be pasted from ADMA level
  429. * each time it wants to configure parameters
  430. * of the transaction (src, dst, ...)
  431. */
  432. iter->hw_next = NULL;
  433. /* always enable interrupt generation since we get
  434. * the status of pqzero from the handler
  435. */
  436. set_bit(PPC440SPE_DESC_INT, &iter->flags);
  437. }
  438. }
  439. desc->src_cnt = src_cnt;
  440. desc->dst_cnt = dst_cnt;
  441. }
  442. /**
  443. * ppc440spe_desc_init_memcpy - initialize the descriptor for MEMCPY operation
  444. */
  445. static void ppc440spe_desc_init_memcpy(struct ppc440spe_adma_desc_slot *desc,
  446. unsigned long flags)
  447. {
  448. struct dma_cdb *hw_desc = desc->hw_desc;
  449. memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
  450. desc->hw_next = NULL;
  451. desc->src_cnt = 1;
  452. desc->dst_cnt = 1;
  453. if (flags & DMA_PREP_INTERRUPT)
  454. set_bit(PPC440SPE_DESC_INT, &desc->flags);
  455. else
  456. clear_bit(PPC440SPE_DESC_INT, &desc->flags);
  457. hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
  458. }
  459. /**
  460. * ppc440spe_desc_set_src_addr - set source address into the descriptor
  461. */
  462. static void ppc440spe_desc_set_src_addr(struct ppc440spe_adma_desc_slot *desc,
  463. struct ppc440spe_adma_chan *chan,
  464. int src_idx, dma_addr_t addrh,
  465. dma_addr_t addrl)
  466. {
  467. struct dma_cdb *dma_hw_desc;
  468. struct xor_cb *xor_hw_desc;
  469. phys_addr_t addr64, tmplow, tmphi;
  470. switch (chan->device->id) {
  471. case PPC440SPE_DMA0_ID:
  472. case PPC440SPE_DMA1_ID:
  473. if (!addrh) {
  474. addr64 = addrl;
  475. tmphi = (addr64 >> 32);
  476. tmplow = (addr64 & 0xFFFFFFFF);
  477. } else {
  478. tmphi = addrh;
  479. tmplow = addrl;
  480. }
  481. dma_hw_desc = desc->hw_desc;
  482. dma_hw_desc->sg1l = cpu_to_le32((u32)tmplow);
  483. dma_hw_desc->sg1u |= cpu_to_le32((u32)tmphi);
  484. break;
  485. case PPC440SPE_XOR_ID:
  486. xor_hw_desc = desc->hw_desc;
  487. xor_hw_desc->ops[src_idx].l = addrl;
  488. xor_hw_desc->ops[src_idx].h |= addrh;
  489. break;
  490. }
  491. }
  492. /**
  493. * ppc440spe_desc_set_src_mult - set source address mult into the descriptor
  494. */
  495. static void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc,
  496. struct ppc440spe_adma_chan *chan, u32 mult_index,
  497. int sg_index, unsigned char mult_value)
  498. {
  499. struct dma_cdb *dma_hw_desc;
  500. struct xor_cb *xor_hw_desc;
  501. u32 *psgu;
  502. switch (chan->device->id) {
  503. case PPC440SPE_DMA0_ID:
  504. case PPC440SPE_DMA1_ID:
  505. dma_hw_desc = desc->hw_desc;
  506. switch (sg_index) {
  507. /* for RXOR operations set multiplier
  508. * into source cued address
  509. */
  510. case DMA_CDB_SG_SRC:
  511. psgu = &dma_hw_desc->sg1u;
  512. break;
  513. /* for WXOR operations set multiplier
  514. * into destination cued address(es)
  515. */
  516. case DMA_CDB_SG_DST1:
  517. psgu = &dma_hw_desc->sg2u;
  518. break;
  519. case DMA_CDB_SG_DST2:
  520. psgu = &dma_hw_desc->sg3u;
  521. break;
  522. default:
  523. BUG();
  524. }
  525. *psgu |= cpu_to_le32(mult_value << mult_index);
  526. break;
  527. case PPC440SPE_XOR_ID:
  528. xor_hw_desc = desc->hw_desc;
  529. break;
  530. default:
  531. BUG();
  532. }
  533. }
  534. /**
  535. * ppc440spe_desc_set_dest_addr - set destination address into the descriptor
  536. */
  537. static void ppc440spe_desc_set_dest_addr(struct ppc440spe_adma_desc_slot *desc,
  538. struct ppc440spe_adma_chan *chan,
  539. dma_addr_t addrh, dma_addr_t addrl,
  540. u32 dst_idx)
  541. {
  542. struct dma_cdb *dma_hw_desc;
  543. struct xor_cb *xor_hw_desc;
  544. phys_addr_t addr64, tmphi, tmplow;
  545. u32 *psgu, *psgl;
  546. switch (chan->device->id) {
  547. case PPC440SPE_DMA0_ID:
  548. case PPC440SPE_DMA1_ID:
  549. if (!addrh) {
  550. addr64 = addrl;
  551. tmphi = (addr64 >> 32);
  552. tmplow = (addr64 & 0xFFFFFFFF);
  553. } else {
  554. tmphi = addrh;
  555. tmplow = addrl;
  556. }
  557. dma_hw_desc = desc->hw_desc;
  558. psgu = dst_idx ? &dma_hw_desc->sg3u : &dma_hw_desc->sg2u;
  559. psgl = dst_idx ? &dma_hw_desc->sg3l : &dma_hw_desc->sg2l;
  560. *psgl = cpu_to_le32((u32)tmplow);
  561. *psgu |= cpu_to_le32((u32)tmphi);
  562. break;
  563. case PPC440SPE_XOR_ID:
  564. xor_hw_desc = desc->hw_desc;
  565. xor_hw_desc->cbtal = addrl;
  566. xor_hw_desc->cbtah |= addrh;
  567. break;
  568. }
  569. }
  570. /**
  571. * ppc440spe_desc_set_byte_count - set number of data bytes involved
  572. * into the operation
  573. */
  574. static void ppc440spe_desc_set_byte_count(struct ppc440spe_adma_desc_slot *desc,
  575. struct ppc440spe_adma_chan *chan,
  576. u32 byte_count)
  577. {
  578. struct dma_cdb *dma_hw_desc;
  579. struct xor_cb *xor_hw_desc;
  580. switch (chan->device->id) {
  581. case PPC440SPE_DMA0_ID:
  582. case PPC440SPE_DMA1_ID:
  583. dma_hw_desc = desc->hw_desc;
  584. dma_hw_desc->cnt = cpu_to_le32(byte_count);
  585. break;
  586. case PPC440SPE_XOR_ID:
  587. xor_hw_desc = desc->hw_desc;
  588. xor_hw_desc->cbbc = byte_count;
  589. break;
  590. }
  591. }
  592. /**
  593. * ppc440spe_desc_set_rxor_block_size - set RXOR block size
  594. */
  595. static inline void ppc440spe_desc_set_rxor_block_size(u32 byte_count)
  596. {
  597. /* assume that byte_count is aligned on the 512-boundary;
  598. * thus write it directly to the register (bits 23:31 are
  599. * reserved there).
  600. */
  601. dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CF2H, byte_count);
  602. }
  603. /**
  604. * ppc440spe_desc_set_dcheck - set CHECK pattern
  605. */
  606. static void ppc440spe_desc_set_dcheck(struct ppc440spe_adma_desc_slot *desc,
  607. struct ppc440spe_adma_chan *chan, u8 *qword)
  608. {
  609. struct dma_cdb *dma_hw_desc;
  610. switch (chan->device->id) {
  611. case PPC440SPE_DMA0_ID:
  612. case PPC440SPE_DMA1_ID:
  613. dma_hw_desc = desc->hw_desc;
  614. iowrite32(qword[0], &dma_hw_desc->sg3l);
  615. iowrite32(qword[4], &dma_hw_desc->sg3u);
  616. iowrite32(qword[8], &dma_hw_desc->sg2l);
  617. iowrite32(qword[12], &dma_hw_desc->sg2u);
  618. break;
  619. default:
  620. BUG();
  621. }
  622. }
  623. /**
  624. * ppc440spe_xor_set_link - set link address in xor CB
  625. */
  626. static void ppc440spe_xor_set_link(struct ppc440spe_adma_desc_slot *prev_desc,
  627. struct ppc440spe_adma_desc_slot *next_desc)
  628. {
  629. struct xor_cb *xor_hw_desc = prev_desc->hw_desc;
  630. if (unlikely(!next_desc || !(next_desc->phys))) {
  631. printk(KERN_ERR "%s: next_desc=0x%p; next_desc->phys=0x%llx\n",
  632. __func__, next_desc,
  633. next_desc ? next_desc->phys : 0);
  634. BUG();
  635. }
  636. xor_hw_desc->cbs = 0;
  637. xor_hw_desc->cblal = next_desc->phys;
  638. xor_hw_desc->cblah = 0;
  639. xor_hw_desc->cbc |= XOR_CBCR_LNK_BIT;
  640. }
  641. /**
  642. * ppc440spe_desc_set_link - set the address of descriptor following this
  643. * descriptor in chain
  644. */
  645. static void ppc440spe_desc_set_link(struct ppc440spe_adma_chan *chan,
  646. struct ppc440spe_adma_desc_slot *prev_desc,
  647. struct ppc440spe_adma_desc_slot *next_desc)
  648. {
  649. unsigned long flags;
  650. struct ppc440spe_adma_desc_slot *tail = next_desc;
  651. if (unlikely(!prev_desc || !next_desc ||
  652. (prev_desc->hw_next && prev_desc->hw_next != next_desc))) {
  653. /* If previous next is overwritten something is wrong.
  654. * though we may refetch from append to initiate list
  655. * processing; in this case - it's ok.
  656. */
  657. printk(KERN_ERR "%s: prev_desc=0x%p; next_desc=0x%p; "
  658. "prev->hw_next=0x%p\n", __func__, prev_desc,
  659. next_desc, prev_desc ? prev_desc->hw_next : 0);
  660. BUG();
  661. }
  662. local_irq_save(flags);
  663. /* do s/w chaining both for DMA and XOR descriptors */
  664. prev_desc->hw_next = next_desc;
  665. switch (chan->device->id) {
  666. case PPC440SPE_DMA0_ID:
  667. case PPC440SPE_DMA1_ID:
  668. break;
  669. case PPC440SPE_XOR_ID:
  670. /* bind descriptor to the chain */
  671. while (tail->hw_next)
  672. tail = tail->hw_next;
  673. xor_last_linked = tail;
  674. if (prev_desc == xor_last_submit)
  675. /* do not link to the last submitted CB */
  676. break;
  677. ppc440spe_xor_set_link(prev_desc, next_desc);
  678. break;
  679. }
  680. local_irq_restore(flags);
  681. }
  682. /**
  683. * ppc440spe_desc_get_link - get the address of the descriptor that
  684. * follows this one
  685. */
  686. static inline u32 ppc440spe_desc_get_link(struct ppc440spe_adma_desc_slot *desc,
  687. struct ppc440spe_adma_chan *chan)
  688. {
  689. if (!desc->hw_next)
  690. return 0;
  691. return desc->hw_next->phys;
  692. }
  693. /**
  694. * ppc440spe_desc_is_aligned - check alignment
  695. */
  696. static inline int ppc440spe_desc_is_aligned(
  697. struct ppc440spe_adma_desc_slot *desc, int num_slots)
  698. {
  699. return (desc->idx & (num_slots - 1)) ? 0 : 1;
  700. }
  701. /**
  702. * ppc440spe_chan_xor_slot_count - get the number of slots necessary for
  703. * XOR operation
  704. */
  705. static int ppc440spe_chan_xor_slot_count(size_t len, int src_cnt,
  706. int *slots_per_op)
  707. {
  708. int slot_cnt;
  709. /* each XOR descriptor provides up to 16 source operands */
  710. slot_cnt = *slots_per_op = (src_cnt + XOR_MAX_OPS - 1)/XOR_MAX_OPS;
  711. if (likely(len <= PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT))
  712. return slot_cnt;
  713. printk(KERN_ERR "%s: len %d > max %d !!\n",
  714. __func__, len, PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
  715. BUG();
  716. return slot_cnt;
  717. }
  718. /**
  719. * ppc440spe_dma2_pq_slot_count - get the number of slots necessary for
  720. * DMA2 PQ operation
  721. */
  722. static int ppc440spe_dma2_pq_slot_count(dma_addr_t *srcs,
  723. int src_cnt, size_t len)
  724. {
  725. signed long long order = 0;
  726. int state = 0;
  727. int addr_count = 0;
  728. int i;
  729. for (i = 1; i < src_cnt; i++) {
  730. dma_addr_t cur_addr = srcs[i];
  731. dma_addr_t old_addr = srcs[i-1];
  732. switch (state) {
  733. case 0:
  734. if (cur_addr == old_addr + len) {
  735. /* direct RXOR */
  736. order = 1;
  737. state = 1;
  738. if (i == src_cnt-1)
  739. addr_count++;
  740. } else if (old_addr == cur_addr + len) {
  741. /* reverse RXOR */
  742. order = -1;
  743. state = 1;
  744. if (i == src_cnt-1)
  745. addr_count++;
  746. } else {
  747. state = 3;
  748. }
  749. break;
  750. case 1:
  751. if (i == src_cnt-2 || (order == -1
  752. && cur_addr != old_addr - len)) {
  753. order = 0;
  754. state = 0;
  755. addr_count++;
  756. } else if (cur_addr == old_addr + len*order) {
  757. state = 2;
  758. if (i == src_cnt-1)
  759. addr_count++;
  760. } else if (cur_addr == old_addr + 2*len) {
  761. state = 2;
  762. if (i == src_cnt-1)
  763. addr_count++;
  764. } else if (cur_addr == old_addr + 3*len) {
  765. state = 2;
  766. if (i == src_cnt-1)
  767. addr_count++;
  768. } else {
  769. order = 0;
  770. state = 0;
  771. addr_count++;
  772. }
  773. break;
  774. case 2:
  775. order = 0;
  776. state = 0;
  777. addr_count++;
  778. break;
  779. }
  780. if (state == 3)
  781. break;
  782. }
  783. if (src_cnt <= 1 || (state != 1 && state != 2)) {
  784. pr_err("%s: src_cnt=%d, state=%d, addr_count=%d, order=%lld\n",
  785. __func__, src_cnt, state, addr_count, order);
  786. for (i = 0; i < src_cnt; i++)
  787. pr_err("\t[%d] 0x%llx \n", i, srcs[i]);
  788. BUG();
  789. }
  790. return (addr_count + XOR_MAX_OPS - 1) / XOR_MAX_OPS;
  791. }
  792. /******************************************************************************
  793. * ADMA channel low-level routines
  794. ******************************************************************************/
  795. static u32
  796. ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan);
  797. static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan);
  798. /**
  799. * ppc440spe_adma_device_clear_eot_status - interrupt ack to XOR or DMA engine
  800. */
  801. static void ppc440spe_adma_device_clear_eot_status(
  802. struct ppc440spe_adma_chan *chan)
  803. {
  804. struct dma_regs *dma_reg;
  805. struct xor_regs *xor_reg;
  806. u8 *p = chan->device->dma_desc_pool_virt;
  807. struct dma_cdb *cdb;
  808. u32 rv, i;
  809. switch (chan->device->id) {
  810. case PPC440SPE_DMA0_ID:
  811. case PPC440SPE_DMA1_ID:
  812. /* read FIFO to ack */
  813. dma_reg = chan->device->dma_reg;
  814. while ((rv = ioread32(&dma_reg->csfpl))) {
  815. i = rv & DMA_CDB_ADDR_MSK;
  816. cdb = (struct dma_cdb *)&p[i -
  817. (u32)chan->device->dma_desc_pool];
  818. /* Clear opcode to ack. This is necessary for
  819. * ZeroSum operations only
  820. */
  821. cdb->opc = 0;
  822. if (test_bit(PPC440SPE_RXOR_RUN,
  823. &ppc440spe_rxor_state)) {
  824. /* probably this is a completed RXOR op,
  825. * get pointer to CDB using the fact that
  826. * physical and virtual addresses of CDB
  827. * in pools have the same offsets
  828. */
  829. if (le32_to_cpu(cdb->sg1u) &
  830. DMA_CUED_XOR_BASE) {
  831. /* this is a RXOR */
  832. clear_bit(PPC440SPE_RXOR_RUN,
  833. &ppc440spe_rxor_state);
  834. }
  835. }
  836. if (rv & DMA_CDB_STATUS_MSK) {
  837. /* ZeroSum check failed
  838. */
  839. struct ppc440spe_adma_desc_slot *iter;
  840. dma_addr_t phys = rv & ~DMA_CDB_MSK;
  841. /*
  842. * Update the status of corresponding
  843. * descriptor.
  844. */
  845. list_for_each_entry(iter, &chan->chain,
  846. chain_node) {
  847. if (iter->phys == phys)
  848. break;
  849. }
  850. /*
  851. * if cannot find the corresponding
  852. * slot it's a bug
  853. */
  854. BUG_ON(&iter->chain_node == &chan->chain);
  855. if (iter->xor_check_result) {
  856. if (test_bit(PPC440SPE_DESC_PCHECK,
  857. &iter->flags)) {
  858. *iter->xor_check_result |=
  859. SUM_CHECK_P_RESULT;
  860. } else
  861. if (test_bit(PPC440SPE_DESC_QCHECK,
  862. &iter->flags)) {
  863. *iter->xor_check_result |=
  864. SUM_CHECK_Q_RESULT;
  865. } else
  866. BUG();
  867. }
  868. }
  869. }
  870. rv = ioread32(&dma_reg->dsts);
  871. if (rv) {
  872. pr_err("DMA%d err status: 0x%x\n",
  873. chan->device->id, rv);
  874. /* write back to clear */
  875. iowrite32(rv, &dma_reg->dsts);
  876. }
  877. break;
  878. case PPC440SPE_XOR_ID:
  879. /* reset status bits to ack */
  880. xor_reg = chan->device->xor_reg;
  881. rv = ioread32be(&xor_reg->sr);
  882. iowrite32be(rv, &xor_reg->sr);
  883. if (rv & (XOR_IE_ICBIE_BIT|XOR_IE_ICIE_BIT|XOR_IE_RPTIE_BIT)) {
  884. if (rv & XOR_IE_RPTIE_BIT) {
  885. /* Read PLB Timeout Error.
  886. * Try to resubmit the CB
  887. */
  888. u32 val = ioread32be(&xor_reg->ccbalr);
  889. iowrite32be(val, &xor_reg->cblalr);
  890. val = ioread32be(&xor_reg->crsr);
  891. iowrite32be(val | XOR_CRSR_XAE_BIT,
  892. &xor_reg->crsr);
  893. } else
  894. pr_err("XOR ERR 0x%x status\n", rv);
  895. break;
  896. }
  897. /* if the XORcore is idle, but there are unprocessed CBs
  898. * then refetch the s/w chain here
  899. */
  900. if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) &&
  901. do_xor_refetch)
  902. ppc440spe_chan_append(chan);
  903. break;
  904. }
  905. }
  906. /**
  907. * ppc440spe_chan_is_busy - get the channel status
  908. */
  909. static int ppc440spe_chan_is_busy(struct ppc440spe_adma_chan *chan)
  910. {
  911. struct dma_regs *dma_reg;
  912. struct xor_regs *xor_reg;
  913. int busy = 0;
  914. switch (chan->device->id) {
  915. case PPC440SPE_DMA0_ID:
  916. case PPC440SPE_DMA1_ID:
  917. dma_reg = chan->device->dma_reg;
  918. /* if command FIFO's head and tail pointers are equal and
  919. * status tail is the same as command, then channel is free
  920. */
  921. if (ioread16(&dma_reg->cpfhp) != ioread16(&dma_reg->cpftp) ||
  922. ioread16(&dma_reg->cpftp) != ioread16(&dma_reg->csftp))
  923. busy = 1;
  924. break;
  925. case PPC440SPE_XOR_ID:
  926. /* use the special status bit for the XORcore
  927. */
  928. xor_reg = chan->device->xor_reg;
  929. busy = (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) ? 1 : 0;
  930. break;
  931. }
  932. return busy;
  933. }
  934. /**
  935. * ppc440spe_chan_set_first_xor_descriptor - init XORcore chain
  936. */
  937. static void ppc440spe_chan_set_first_xor_descriptor(
  938. struct ppc440spe_adma_chan *chan,
  939. struct ppc440spe_adma_desc_slot *next_desc)
  940. {
  941. struct xor_regs *xor_reg = chan->device->xor_reg;
  942. if (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)
  943. printk(KERN_INFO "%s: Warn: XORcore is running "
  944. "when try to set the first CDB!\n",
  945. __func__);
  946. xor_last_submit = xor_last_linked = next_desc;
  947. iowrite32be(XOR_CRSR_64BA_BIT, &xor_reg->crsr);
  948. iowrite32be(next_desc->phys, &xor_reg->cblalr);
  949. iowrite32be(0, &xor_reg->cblahr);
  950. iowrite32be(ioread32be(&xor_reg->cbcr) | XOR_CBCR_LNK_BIT,
  951. &xor_reg->cbcr);
  952. chan->hw_chain_inited = 1;
  953. }
  954. /**
  955. * ppc440spe_dma_put_desc - put DMA0,1 descriptor to FIFO.
  956. * called with irqs disabled
  957. */
  958. static void ppc440spe_dma_put_desc(struct ppc440spe_adma_chan *chan,
  959. struct ppc440spe_adma_desc_slot *desc)
  960. {
  961. u32 pcdb;
  962. struct dma_regs *dma_reg = chan->device->dma_reg;
  963. pcdb = desc->phys;
  964. if (!test_bit(PPC440SPE_DESC_INT, &desc->flags))
  965. pcdb |= DMA_CDB_NO_INT;
  966. chan_last_sub[chan->device->id] = desc;
  967. ADMA_LL_DBG(print_cb(chan, desc->hw_desc));
  968. iowrite32(pcdb, &dma_reg->cpfpl);
  969. }
  970. /**
  971. * ppc440spe_chan_append - update the h/w chain in the channel
  972. */
  973. static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan)
  974. {
  975. struct xor_regs *xor_reg;
  976. struct ppc440spe_adma_desc_slot *iter;
  977. struct xor_cb *xcb;
  978. u32 cur_desc;
  979. unsigned long flags;
  980. local_irq_save(flags);
  981. switch (chan->device->id) {
  982. case PPC440SPE_DMA0_ID:
  983. case PPC440SPE_DMA1_ID:
  984. cur_desc = ppc440spe_chan_get_current_descriptor(chan);
  985. if (likely(cur_desc)) {
  986. iter = chan_last_sub[chan->device->id];
  987. BUG_ON(!iter);
  988. } else {
  989. /* first peer */
  990. iter = chan_first_cdb[chan->device->id];
  991. BUG_ON(!iter);
  992. ppc440spe_dma_put_desc(chan, iter);
  993. chan->hw_chain_inited = 1;
  994. }
  995. /* is there something new to append */
  996. if (!iter->hw_next)
  997. break;
  998. /* flush descriptors from the s/w queue to fifo */
  999. list_for_each_entry_continue(iter, &chan->chain, chain_node) {
  1000. ppc440spe_dma_put_desc(chan, iter);
  1001. if (!iter->hw_next)
  1002. break;
  1003. }
  1004. break;
  1005. case PPC440SPE_XOR_ID:
  1006. /* update h/w links and refetch */
  1007. if (!xor_last_submit->hw_next)
  1008. break;
  1009. xor_reg = chan->device->xor_reg;
  1010. /* the last linked CDB has to generate an interrupt
  1011. * that we'd be able to append the next lists to h/w
  1012. * regardless of the XOR engine state at the moment of
  1013. * appending of these next lists
  1014. */
  1015. xcb = xor_last_linked->hw_desc;
  1016. xcb->cbc |= XOR_CBCR_CBCE_BIT;
  1017. if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)) {
  1018. /* XORcore is idle. Refetch now */
  1019. do_xor_refetch = 0;
  1020. ppc440spe_xor_set_link(xor_last_submit,
  1021. xor_last_submit->hw_next);
  1022. ADMA_LL_DBG(print_cb_list(chan,
  1023. xor_last_submit->hw_next));
  1024. xor_last_submit = xor_last_linked;
  1025. iowrite32be(ioread32be(&xor_reg->crsr) |
  1026. XOR_CRSR_RCBE_BIT | XOR_CRSR_64BA_BIT,
  1027. &xor_reg->crsr);
  1028. } else {
  1029. /* XORcore is running. Refetch later in the handler */
  1030. do_xor_refetch = 1;
  1031. }
  1032. break;
  1033. }
  1034. local_irq_restore(flags);
  1035. }
  1036. /**
  1037. * ppc440spe_chan_get_current_descriptor - get the currently executed descriptor
  1038. */
  1039. static u32
  1040. ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan)
  1041. {
  1042. struct dma_regs *dma_reg;
  1043. struct xor_regs *xor_reg;
  1044. if (unlikely(!chan->hw_chain_inited))
  1045. /* h/w descriptor chain is not initialized yet */
  1046. return 0;
  1047. switch (chan->device->id) {
  1048. case PPC440SPE_DMA0_ID:
  1049. case PPC440SPE_DMA1_ID:
  1050. dma_reg = chan->device->dma_reg;
  1051. return ioread32(&dma_reg->acpl) & (~DMA_CDB_MSK);
  1052. case PPC440SPE_XOR_ID:
  1053. xor_reg = chan->device->xor_reg;
  1054. return ioread32be(&xor_reg->ccbalr);
  1055. }
  1056. return 0;
  1057. }
  1058. /**
  1059. * ppc440spe_chan_run - enable the channel
  1060. */
  1061. static void ppc440spe_chan_run(struct ppc440spe_adma_chan *chan)
  1062. {
  1063. struct xor_regs *xor_reg;
  1064. switch (chan->device->id) {
  1065. case PPC440SPE_DMA0_ID:
  1066. case PPC440SPE_DMA1_ID:
  1067. /* DMAs are always enabled, do nothing */
  1068. break;
  1069. case PPC440SPE_XOR_ID:
  1070. /* drain write buffer */
  1071. xor_reg = chan->device->xor_reg;
  1072. /* fetch descriptor pointed to in <link> */
  1073. iowrite32be(XOR_CRSR_64BA_BIT | XOR_CRSR_XAE_BIT,
  1074. &xor_reg->crsr);
  1075. break;
  1076. }
  1077. }
  1078. /******************************************************************************
  1079. * ADMA device level
  1080. ******************************************************************************/
  1081. static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan);
  1082. static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan);
  1083. static dma_cookie_t
  1084. ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx);
  1085. static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *tx,
  1086. dma_addr_t addr, int index);
  1087. static void
  1088. ppc440spe_adma_memcpy_xor_set_src(struct ppc440spe_adma_desc_slot *tx,
  1089. dma_addr_t addr, int index);
  1090. static void
  1091. ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *tx,
  1092. dma_addr_t *paddr, unsigned long flags);
  1093. static void
  1094. ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *tx,
  1095. dma_addr_t addr, int index);
  1096. static void
  1097. ppc440spe_adma_pq_set_src_mult(struct ppc440spe_adma_desc_slot *tx,
  1098. unsigned char mult, int index, int dst_pos);
  1099. static void
  1100. ppc440spe_adma_pqzero_sum_set_dest(struct ppc440spe_adma_desc_slot *tx,
  1101. dma_addr_t paddr, dma_addr_t qaddr);
  1102. static struct page *ppc440spe_rxor_srcs[32];
  1103. /**
  1104. * ppc440spe_can_rxor - check if the operands may be processed with RXOR
  1105. */
  1106. static int ppc440spe_can_rxor(struct page **srcs, int src_cnt, size_t len)
  1107. {
  1108. int i, order = 0, state = 0;
  1109. int idx = 0;
  1110. if (unlikely(!(src_cnt > 1)))
  1111. return 0;
  1112. BUG_ON(src_cnt > ARRAY_SIZE(ppc440spe_rxor_srcs));
  1113. /* Skip holes in the source list before checking */
  1114. for (i = 0; i < src_cnt; i++) {
  1115. if (!srcs[i])
  1116. continue;
  1117. ppc440spe_rxor_srcs[idx++] = srcs[i];
  1118. }
  1119. src_cnt = idx;
  1120. for (i = 1; i < src_cnt; i++) {
  1121. char *cur_addr = page_address(ppc440spe_rxor_srcs[i]);
  1122. char *old_addr = page_address(ppc440spe_rxor_srcs[i - 1]);
  1123. switch (state) {
  1124. case 0:
  1125. if (cur_addr == old_addr + len) {
  1126. /* direct RXOR */
  1127. order = 1;
  1128. state = 1;
  1129. } else if (old_addr == cur_addr + len) {
  1130. /* reverse RXOR */
  1131. order = -1;
  1132. state = 1;
  1133. } else
  1134. goto out;
  1135. break;
  1136. case 1:
  1137. if ((i == src_cnt - 2) ||
  1138. (order == -1 && cur_addr != old_addr - len)) {
  1139. order = 0;
  1140. state = 0;
  1141. } else if ((cur_addr == old_addr + len * order) ||
  1142. (cur_addr == old_addr + 2 * len) ||
  1143. (cur_addr == old_addr + 3 * len)) {
  1144. state = 2;
  1145. } else {
  1146. order = 0;
  1147. state = 0;
  1148. }
  1149. break;
  1150. case 2:
  1151. order = 0;
  1152. state = 0;
  1153. break;
  1154. }
  1155. }
  1156. out:
  1157. if (state == 1 || state == 2)
  1158. return 1;
  1159. return 0;
  1160. }
  1161. /**
  1162. * ppc440spe_adma_device_estimate - estimate the efficiency of processing
  1163. * the operation given on this channel. It's assumed that 'chan' is
  1164. * capable to process 'cap' type of operation.
  1165. * @chan: channel to use
  1166. * @cap: type of transaction
  1167. * @dst_lst: array of destination pointers
  1168. * @dst_cnt: number of destination operands
  1169. * @src_lst: array of source pointers
  1170. * @src_cnt: number of source operands
  1171. * @src_sz: size of each source operand
  1172. */
  1173. static int ppc440spe_adma_estimate(struct dma_chan *chan,
  1174. enum dma_transaction_type cap, struct page **dst_lst, int dst_cnt,
  1175. struct page **src_lst, int src_cnt, size_t src_sz)
  1176. {
  1177. int ef = 1;
  1178. if (cap == DMA_PQ || cap == DMA_PQ_VAL) {
  1179. /* If RAID-6 capabilities were not activated don't try
  1180. * to use them
  1181. */
  1182. if (unlikely(!ppc440spe_r6_enabled))
  1183. return -1;
  1184. }
  1185. /* In the current implementation of ppc440spe ADMA driver it
  1186. * makes sense to pick out only pq case, because it may be
  1187. * processed:
  1188. * (1) either using Biskup method on DMA2;
  1189. * (2) or on DMA0/1.
  1190. * Thus we give a favour to (1) if the sources are suitable;
  1191. * else let it be processed on one of the DMA0/1 engines.
  1192. * In the sum_product case where destination is also the
  1193. * source process it on DMA0/1 only.
  1194. */
  1195. if (cap == DMA_PQ && chan->chan_id == PPC440SPE_XOR_ID) {
  1196. if (dst_cnt == 1 && src_cnt == 2 && dst_lst[0] == src_lst[1])
  1197. ef = 0; /* sum_product case, process on DMA0/1 */
  1198. else if (ppc440spe_can_rxor(src_lst, src_cnt, src_sz))
  1199. ef = 3; /* override (DMA0/1 + idle) */
  1200. else
  1201. ef = 0; /* can't process on DMA2 if !rxor */
  1202. }
  1203. /* channel idleness increases the priority */
  1204. if (likely(ef) &&
  1205. !ppc440spe_chan_is_busy(to_ppc440spe_adma_chan(chan)))
  1206. ef++;
  1207. return ef;
  1208. }
  1209. struct dma_chan *
  1210. ppc440spe_async_tx_find_best_channel(enum dma_transaction_type cap,
  1211. struct page **dst_lst, int dst_cnt, struct page **src_lst,
  1212. int src_cnt, size_t src_sz)
  1213. {
  1214. struct dma_chan *best_chan = NULL;
  1215. struct ppc_dma_chan_ref *ref;
  1216. int best_rank = -1;
  1217. if (unlikely(!src_sz))
  1218. return NULL;
  1219. if (src_sz > PAGE_SIZE) {
  1220. /*
  1221. * should a user of the api ever pass > PAGE_SIZE requests
  1222. * we sort out cases where temporary page-sized buffers
  1223. * are used.
  1224. */
  1225. switch (cap) {
  1226. case DMA_PQ:
  1227. if (src_cnt == 1 && dst_lst[1] == src_lst[0])
  1228. return NULL;
  1229. if (src_cnt == 2 && dst_lst[1] == src_lst[1])
  1230. return NULL;
  1231. break;
  1232. case DMA_PQ_VAL:
  1233. case DMA_XOR_VAL:
  1234. return NULL;
  1235. default:
  1236. break;
  1237. }
  1238. }
  1239. list_for_each_entry(ref, &ppc440spe_adma_chan_list, node) {
  1240. if (dma_has_cap(cap, ref->chan->device->cap_mask)) {
  1241. int rank;
  1242. rank = ppc440spe_adma_estimate(ref->chan, cap, dst_lst,
  1243. dst_cnt, src_lst, src_cnt, src_sz);
  1244. if (rank > best_rank) {
  1245. best_rank = rank;
  1246. best_chan = ref->chan;
  1247. }
  1248. }
  1249. }
  1250. return best_chan;
  1251. }
  1252. EXPORT_SYMBOL_GPL(ppc440spe_async_tx_find_best_channel);
  1253. /**
  1254. * ppc440spe_get_group_entry - get group entry with index idx
  1255. * @tdesc: is the last allocated slot in the group.
  1256. */
  1257. static struct ppc440spe_adma_desc_slot *
  1258. ppc440spe_get_group_entry(struct ppc440spe_adma_desc_slot *tdesc, u32 entry_idx)
  1259. {
  1260. struct ppc440spe_adma_desc_slot *iter = tdesc->group_head;
  1261. int i = 0;
  1262. if (entry_idx < 0 || entry_idx >= (tdesc->src_cnt + tdesc->dst_cnt)) {
  1263. printk("%s: entry_idx %d, src_cnt %d, dst_cnt %d\n",
  1264. __func__, entry_idx, tdesc->src_cnt, tdesc->dst_cnt);
  1265. BUG();
  1266. }
  1267. list_for_each_entry(iter, &tdesc->group_list, chain_node) {
  1268. if (i++ == entry_idx)
  1269. break;
  1270. }
  1271. return iter;
  1272. }
  1273. /**
  1274. * ppc440spe_adma_free_slots - flags descriptor slots for reuse
  1275. * @slot: Slot to free
  1276. * Caller must hold &ppc440spe_chan->lock while calling this function
  1277. */
  1278. static void ppc440spe_adma_free_slots(struct ppc440spe_adma_desc_slot *slot,
  1279. struct ppc440spe_adma_chan *chan)
  1280. {
  1281. int stride = slot->slots_per_op;
  1282. while (stride--) {
  1283. slot->slots_per_op = 0;
  1284. slot = list_entry(slot->slot_node.next,
  1285. struct ppc440spe_adma_desc_slot,
  1286. slot_node);
  1287. }
  1288. }
  1289. /**
  1290. * ppc440spe_adma_run_tx_complete_actions - call functions to be called
  1291. * upon completion
  1292. */
  1293. static dma_cookie_t ppc440spe_adma_run_tx_complete_actions(
  1294. struct ppc440spe_adma_desc_slot *desc,
  1295. struct ppc440spe_adma_chan *chan,
  1296. dma_cookie_t cookie)
  1297. {
  1298. BUG_ON(desc->async_tx.cookie < 0);
  1299. if (desc->async_tx.cookie > 0) {
  1300. cookie = desc->async_tx.cookie;
  1301. desc->async_tx.cookie = 0;
  1302. dma_descriptor_unmap(&desc->async_tx);
  1303. /* call the callback (must not sleep or submit new
  1304. * operations to this channel)
  1305. */
  1306. dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL);
  1307. }
  1308. /* run dependent operations */
  1309. dma_run_dependencies(&desc->async_tx);
  1310. return cookie;
  1311. }
  1312. /**
  1313. * ppc440spe_adma_clean_slot - clean up CDB slot (if ack is set)
  1314. */
  1315. static int ppc440spe_adma_clean_slot(struct ppc440spe_adma_desc_slot *desc,
  1316. struct ppc440spe_adma_chan *chan)
  1317. {
  1318. /* the client is allowed to attach dependent operations
  1319. * until 'ack' is set
  1320. */
  1321. if (!async_tx_test_ack(&desc->async_tx))
  1322. return 0;
  1323. /* leave the last descriptor in the chain
  1324. * so we can append to it
  1325. */
  1326. if (list_is_last(&desc->chain_node, &chan->chain) ||
  1327. desc->phys == ppc440spe_chan_get_current_descriptor(chan))
  1328. return 1;
  1329. if (chan->device->id != PPC440SPE_XOR_ID) {
  1330. /* our DMA interrupt handler clears opc field of
  1331. * each processed descriptor. For all types of
  1332. * operations except for ZeroSum we do not actually
  1333. * need ack from the interrupt handler. ZeroSum is a
  1334. * special case since the result of this operation
  1335. * is available from the handler only, so if we see
  1336. * such type of descriptor (which is unprocessed yet)
  1337. * then leave it in chain.
  1338. */
  1339. struct dma_cdb *cdb = desc->hw_desc;
  1340. if (cdb->opc == DMA_CDB_OPC_DCHECK128)
  1341. return 1;
  1342. }
  1343. dev_dbg(chan->device->common.dev, "\tfree slot %llx: %d stride: %d\n",
  1344. desc->phys, desc->idx, desc->slots_per_op);
  1345. list_del(&desc->chain_node);
  1346. ppc440spe_adma_free_slots(desc, chan);
  1347. return 0;
  1348. }
  1349. /**
  1350. * __ppc440spe_adma_slot_cleanup - this is the common clean-up routine
  1351. * which runs through the channel CDBs list until reach the descriptor
  1352. * currently processed. When routine determines that all CDBs of group
  1353. * are completed then corresponding callbacks (if any) are called and slots
  1354. * are freed.
  1355. */
  1356. static void __ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
  1357. {
  1358. struct ppc440spe_adma_desc_slot *iter, *_iter, *group_start = NULL;
  1359. dma_cookie_t cookie = 0;
  1360. u32 current_desc = ppc440spe_chan_get_current_descriptor(chan);
  1361. int busy = ppc440spe_chan_is_busy(chan);
  1362. int seen_current = 0, slot_cnt = 0, slots_per_op = 0;
  1363. dev_dbg(chan->device->common.dev, "ppc440spe adma%d: %s\n",
  1364. chan->device->id, __func__);
  1365. if (!current_desc) {
  1366. /* There were no transactions yet, so
  1367. * nothing to clean
  1368. */
  1369. return;
  1370. }
  1371. /* free completed slots from the chain starting with
  1372. * the oldest descriptor
  1373. */
  1374. list_for_each_entry_safe(iter, _iter, &chan->chain,
  1375. chain_node) {
  1376. dev_dbg(chan->device->common.dev, "\tcookie: %d slot: %d "
  1377. "busy: %d this_desc: %#llx next_desc: %#x "
  1378. "cur: %#x ack: %d\n",
  1379. iter->async_tx.cookie, iter->idx, busy, iter->phys,
  1380. ppc440spe_desc_get_link(iter, chan), current_desc,
  1381. async_tx_test_ack(&iter->async_tx));
  1382. prefetch(_iter);
  1383. prefetch(&_iter->async_tx);
  1384. /* do not advance past the current descriptor loaded into the
  1385. * hardware channel,subsequent descriptors are either in process
  1386. * or have not been submitted
  1387. */
  1388. if (seen_current)
  1389. break;
  1390. /* stop the search if we reach the current descriptor and the
  1391. * channel is busy, or if it appears that the current descriptor
  1392. * needs to be re-read (i.e. has been appended to)
  1393. */
  1394. if (iter->phys == current_desc) {
  1395. BUG_ON(seen_current++);
  1396. if (busy || ppc440spe_desc_get_link(iter, chan)) {
  1397. /* not all descriptors of the group have
  1398. * been completed; exit.
  1399. */
  1400. break;
  1401. }
  1402. }
  1403. /* detect the start of a group transaction */
  1404. if (!slot_cnt && !slots_per_op) {
  1405. slot_cnt = iter->slot_cnt;
  1406. slots_per_op = iter->slots_per_op;
  1407. if (slot_cnt <= slots_per_op) {
  1408. slot_cnt = 0;
  1409. slots_per_op = 0;
  1410. }
  1411. }
  1412. if (slot_cnt) {
  1413. if (!group_start)
  1414. group_start = iter;
  1415. slot_cnt -= slots_per_op;
  1416. }
  1417. /* all the members of a group are complete */
  1418. if (slots_per_op != 0 && slot_cnt == 0) {
  1419. struct ppc440spe_adma_desc_slot *grp_iter, *_grp_iter;
  1420. int end_of_chain = 0;
  1421. /* clean up the group */
  1422. slot_cnt = group_start->slot_cnt;
  1423. grp_iter = group_start;
  1424. list_for_each_entry_safe_from(grp_iter, _grp_iter,
  1425. &chan->chain, chain_node) {
  1426. cookie = ppc440spe_adma_run_tx_complete_actions(
  1427. grp_iter, chan, cookie);
  1428. slot_cnt -= slots_per_op;
  1429. end_of_chain = ppc440spe_adma_clean_slot(
  1430. grp_iter, chan);
  1431. if (end_of_chain && slot_cnt) {
  1432. /* Should wait for ZeroSum completion */
  1433. if (cookie > 0)
  1434. chan->common.completed_cookie = cookie;
  1435. return;
  1436. }
  1437. if (slot_cnt == 0 || end_of_chain)
  1438. break;
  1439. }
  1440. /* the group should be complete at this point */
  1441. BUG_ON(slot_cnt);
  1442. slots_per_op = 0;
  1443. group_start = NULL;
  1444. if (end_of_chain)
  1445. break;
  1446. else
  1447. continue;
  1448. } else if (slots_per_op) /* wait for group completion */
  1449. continue;
  1450. cookie = ppc440spe_adma_run_tx_complete_actions(iter, chan,
  1451. cookie);
  1452. if (ppc440spe_adma_clean_slot(iter, chan))
  1453. break;
  1454. }
  1455. BUG_ON(!seen_current);
  1456. if (cookie > 0) {
  1457. chan->common.completed_cookie = cookie;
  1458. pr_debug("\tcompleted cookie %d\n", cookie);
  1459. }
  1460. }
  1461. /**
  1462. * ppc440spe_adma_tasklet - clean up watch-dog initiator
  1463. */
  1464. static void ppc440spe_adma_tasklet(struct tasklet_struct *t)
  1465. {
  1466. struct ppc440spe_adma_chan *chan = from_tasklet(chan, t, irq_tasklet);
  1467. spin_lock_nested(&chan->lock, SINGLE_DEPTH_NESTING);
  1468. __ppc440spe_adma_slot_cleanup(chan);
  1469. spin_unlock(&chan->lock);
  1470. }
  1471. /**
  1472. * ppc440spe_adma_slot_cleanup - clean up scheduled initiator
  1473. */
  1474. static void ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
  1475. {
  1476. spin_lock_bh(&chan->lock);
  1477. __ppc440spe_adma_slot_cleanup(chan);
  1478. spin_unlock_bh(&chan->lock);
  1479. }
  1480. /**
  1481. * ppc440spe_adma_alloc_slots - allocate free slots (if any)
  1482. */
  1483. static struct ppc440spe_adma_desc_slot *ppc440spe_adma_alloc_slots(
  1484. struct ppc440spe_adma_chan *chan, int num_slots,
  1485. int slots_per_op)
  1486. {
  1487. struct ppc440spe_adma_desc_slot *iter = NULL, *_iter;
  1488. struct ppc440spe_adma_desc_slot *alloc_start = NULL;
  1489. struct list_head chain = LIST_HEAD_INIT(chain);
  1490. int slots_found, retry = 0;
  1491. BUG_ON(!num_slots || !slots_per_op);
  1492. /* start search from the last allocated descrtiptor
  1493. * if a contiguous allocation can not be found start searching
  1494. * from the beginning of the list
  1495. */
  1496. retry:
  1497. slots_found = 0;
  1498. if (retry == 0)
  1499. iter = chan->last_used;
  1500. else
  1501. iter = list_entry(&chan->all_slots,
  1502. struct ppc440spe_adma_desc_slot,
  1503. slot_node);
  1504. list_for_each_entry_safe_continue(iter, _iter, &chan->all_slots,
  1505. slot_node) {
  1506. prefetch(_iter);
  1507. prefetch(&_iter->async_tx);
  1508. if (iter->slots_per_op) {
  1509. slots_found = 0;
  1510. continue;
  1511. }
  1512. /* start the allocation if the slot is correctly aligned */
  1513. if (!slots_found++)
  1514. alloc_start = iter;
  1515. if (slots_found == num_slots) {
  1516. struct ppc440spe_adma_desc_slot *alloc_tail = NULL;
  1517. struct ppc440spe_adma_desc_slot *last_used = NULL;
  1518. iter = alloc_start;
  1519. while (num_slots) {
  1520. int i;
  1521. /* pre-ack all but the last descriptor */
  1522. if (num_slots != slots_per_op)
  1523. async_tx_ack(&iter->async_tx);
  1524. list_add_tail(&iter->chain_node, &chain);
  1525. alloc_tail = iter;
  1526. iter->async_tx.cookie = 0;
  1527. iter->hw_next = NULL;
  1528. iter->flags = 0;
  1529. iter->slot_cnt = num_slots;
  1530. iter->xor_check_result = NULL;
  1531. for (i = 0; i < slots_per_op; i++) {
  1532. iter->slots_per_op = slots_per_op - i;
  1533. last_used = iter;
  1534. iter = list_entry(iter->slot_node.next,
  1535. struct ppc440spe_adma_desc_slot,
  1536. slot_node);
  1537. }
  1538. num_slots -= slots_per_op;
  1539. }
  1540. alloc_tail->group_head = alloc_start;
  1541. alloc_tail->async_tx.cookie = -EBUSY;
  1542. list_splice(&chain, &alloc_tail->group_list);
  1543. chan->last_used = last_used;
  1544. return alloc_tail;
  1545. }
  1546. }
  1547. if (!retry++)
  1548. goto retry;
  1549. /* try to free some slots if the allocation fails */
  1550. tasklet_schedule(&chan->irq_tasklet);
  1551. return NULL;
  1552. }
  1553. /**
  1554. * ppc440spe_adma_alloc_chan_resources - allocate pools for CDB slots
  1555. */
  1556. static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan)
  1557. {
  1558. struct ppc440spe_adma_chan *ppc440spe_chan;
  1559. struct ppc440spe_adma_desc_slot *slot = NULL;
  1560. char *hw_desc;
  1561. int i, db_sz;
  1562. int init;
  1563. ppc440spe_chan = to_ppc440spe_adma_chan(chan);
  1564. init = ppc440spe_chan->slots_allocated ? 0 : 1;
  1565. chan->chan_id = ppc440spe_chan->device->id;
  1566. /* Allocate descriptor slots */
  1567. i = ppc440spe_chan->slots_allocated;
  1568. if (ppc440spe_chan->device->id != PPC440SPE_XOR_ID)
  1569. db_sz = sizeof(struct dma_cdb);
  1570. else
  1571. db_sz = sizeof(struct xor_cb);
  1572. for (; i < (ppc440spe_chan->device->pool_size / db_sz); i++) {
  1573. slot = kzalloc(sizeof(struct ppc440spe_adma_desc_slot),
  1574. GFP_KERNEL);
  1575. if (!slot) {
  1576. printk(KERN_INFO "SPE ADMA Channel only initialized"
  1577. " %d descriptor slots", i--);
  1578. break;
  1579. }
  1580. hw_desc = (char *) ppc440spe_chan->device->dma_desc_pool_virt;
  1581. slot->hw_desc = (void *) &hw_desc[i * db_sz];
  1582. dma_async_tx_descriptor_init(&slot->async_tx, chan);
  1583. slot->async_tx.tx_submit = ppc440spe_adma_tx_submit;
  1584. INIT_LIST_HEAD(&slot->chain_node);
  1585. INIT_LIST_HEAD(&slot->slot_node);
  1586. INIT_LIST_HEAD(&slot->group_list);
  1587. slot->phys = ppc440spe_chan->device->dma_desc_pool + i * db_sz;
  1588. slot->idx = i;
  1589. spin_lock_bh(&ppc440spe_chan->lock);
  1590. ppc440spe_chan->slots_allocated++;
  1591. list_add_tail(&slot->slot_node, &ppc440spe_chan->all_slots);
  1592. spin_unlock_bh(&ppc440spe_chan->lock);
  1593. }
  1594. if (i && !ppc440spe_chan->last_used) {
  1595. ppc440spe_chan->last_used =
  1596. list_entry(ppc440spe_chan->all_slots.next,
  1597. struct ppc440spe_adma_desc_slot,
  1598. slot_node);
  1599. }
  1600. dev_dbg(ppc440spe_chan->device->common.dev,
  1601. "ppc440spe adma%d: allocated %d descriptor slots\n",
  1602. ppc440spe_chan->device->id, i);
  1603. /* initialize the channel and the chain with a null operation */
  1604. if (init) {
  1605. switch (ppc440spe_chan->device->id) {
  1606. case PPC440SPE_DMA0_ID:
  1607. case PPC440SPE_DMA1_ID:
  1608. ppc440spe_chan->hw_chain_inited = 0;
  1609. /* Use WXOR for self-testing */
  1610. if (!ppc440spe_r6_tchan)
  1611. ppc440spe_r6_tchan = ppc440spe_chan;
  1612. break;
  1613. case PPC440SPE_XOR_ID:
  1614. ppc440spe_chan_start_null_xor(ppc440spe_chan);
  1615. break;
  1616. default:
  1617. BUG();
  1618. }
  1619. ppc440spe_chan->needs_unmap = 1;
  1620. }
  1621. return (i > 0) ? i : -ENOMEM;
  1622. }
  1623. /**
  1624. * ppc440spe_rxor_set_region_data -
  1625. */
  1626. static void ppc440spe_rxor_set_region(struct ppc440spe_adma_desc_slot *desc,
  1627. u8 xor_arg_no, u32 mask)
  1628. {
  1629. struct xor_cb *xcb = desc->hw_desc;
  1630. xcb->ops[xor_arg_no].h |= mask;
  1631. }
  1632. /**
  1633. * ppc440spe_rxor_set_src -
  1634. */
  1635. static void ppc440spe_rxor_set_src(struct ppc440spe_adma_desc_slot *desc,
  1636. u8 xor_arg_no, dma_addr_t addr)
  1637. {
  1638. struct xor_cb *xcb = desc->hw_desc;
  1639. xcb->ops[xor_arg_no].h |= DMA_CUED_XOR_BASE;
  1640. xcb->ops[xor_arg_no].l = addr;
  1641. }
  1642. /**
  1643. * ppc440spe_rxor_set_mult -
  1644. */
  1645. static void ppc440spe_rxor_set_mult(struct ppc440spe_adma_desc_slot *desc,
  1646. u8 xor_arg_no, u8 idx, u8 mult)
  1647. {
  1648. struct xor_cb *xcb = desc->hw_desc;
  1649. xcb->ops[xor_arg_no].h |= mult << (DMA_CUED_MULT1_OFF + idx * 8);
  1650. }
  1651. /**
  1652. * ppc440spe_adma_check_threshold - append CDBs to h/w chain if threshold
  1653. * has been achieved
  1654. */
  1655. static void ppc440spe_adma_check_threshold(struct ppc440spe_adma_chan *chan)
  1656. {
  1657. dev_dbg(chan->device->common.dev, "ppc440spe adma%d: pending: %d\n",
  1658. chan->device->id, chan->pending);
  1659. if (chan->pending >= PPC440SPE_ADMA_THRESHOLD) {
  1660. chan->pending = 0;
  1661. ppc440spe_chan_append(chan);
  1662. }
  1663. }
  1664. /**
  1665. * ppc440spe_adma_tx_submit - submit new descriptor group to the channel
  1666. * (it's not necessary that descriptors will be submitted to the h/w
  1667. * chains too right now)
  1668. */
  1669. static dma_cookie_t ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx)
  1670. {
  1671. struct ppc440spe_adma_desc_slot *sw_desc;
  1672. struct ppc440spe_adma_chan *chan = to_ppc440spe_adma_chan(tx->chan);
  1673. struct ppc440spe_adma_desc_slot *group_start, *old_chain_tail;
  1674. int slot_cnt;
  1675. int slots_per_op;
  1676. dma_cookie_t cookie;
  1677. sw_desc = tx_to_ppc440spe_adma_slot(tx);
  1678. group_start = sw_desc->group_head;
  1679. slot_cnt = group_start->slot_cnt;
  1680. slots_per_op = group_start->slots_per_op;
  1681. spin_lock_bh(&chan->lock);
  1682. cookie = dma_cookie_assign(tx);
  1683. if (unlikely(list_empty(&chan->chain))) {
  1684. /* first peer */
  1685. list_splice_init(&sw_desc->group_list, &chan->chain);
  1686. chan_first_cdb[chan->device->id] = group_start;
  1687. } else {
  1688. /* isn't first peer, bind CDBs to chain */
  1689. old_chain_tail = list_entry(chan->chain.prev,
  1690. struct ppc440spe_adma_desc_slot,
  1691. chain_node);
  1692. list_splice_init(&sw_desc->group_list,
  1693. &old_chain_tail->chain_node);
  1694. /* fix up the hardware chain */
  1695. ppc440spe_desc_set_link(chan, old_chain_tail, group_start);
  1696. }
  1697. /* increment the pending count by the number of operations */
  1698. chan->pending += slot_cnt / slots_per_op;
  1699. ppc440spe_adma_check_threshold(chan);
  1700. spin_unlock_bh(&chan->lock);
  1701. dev_dbg(chan->device->common.dev,
  1702. "ppc440spe adma%d: %s cookie: %d slot: %d tx %p\n",
  1703. chan->device->id, __func__,
  1704. sw_desc->async_tx.cookie, sw_desc->idx, sw_desc);
  1705. return cookie;
  1706. }
  1707. /**
  1708. * ppc440spe_adma_prep_dma_interrupt - prepare CDB for a pseudo DMA operation
  1709. */
  1710. static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_interrupt(
  1711. struct dma_chan *chan, unsigned long flags)
  1712. {
  1713. struct ppc440spe_adma_chan *ppc440spe_chan;
  1714. struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
  1715. int slot_cnt, slots_per_op;
  1716. ppc440spe_chan = to_ppc440spe_adma_chan(chan);
  1717. dev_dbg(ppc440spe_chan->device->common.dev,
  1718. "ppc440spe adma%d: %s\n", ppc440spe_chan->device->id,
  1719. __func__);
  1720. spin_lock_bh(&ppc440spe_chan->lock);
  1721. slot_cnt = slots_per_op = 1;
  1722. sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
  1723. slots_per_op);
  1724. if (sw_desc) {
  1725. group_start = sw_desc->group_head;
  1726. ppc440spe_desc_init_interrupt(group_start, ppc440spe_chan);
  1727. group_start->unmap_len = 0;
  1728. sw_desc->async_tx.flags = flags;
  1729. }
  1730. spin_unlock_bh(&ppc440spe_chan->lock);
  1731. return sw_desc ? &sw_desc->async_tx : NULL;
  1732. }
  1733. /**
  1734. * ppc440spe_adma_prep_dma_memcpy - prepare CDB for a MEMCPY operation
  1735. */
  1736. static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memcpy(
  1737. struct dma_chan *chan, dma_addr_t dma_dest,
  1738. dma_addr_t dma_src, size_t len, unsigned long flags)
  1739. {
  1740. struct ppc440spe_adma_chan *ppc440spe_chan;
  1741. struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
  1742. int slot_cnt, slots_per_op;
  1743. ppc440spe_chan = to_ppc440spe_adma_chan(chan);
  1744. if (unlikely(!len))
  1745. return NULL;
  1746. BUG_ON(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT);
  1747. spin_lock_bh(&ppc440spe_chan->lock);
  1748. dev_dbg(ppc440spe_chan->device->common.dev,
  1749. "ppc440spe adma%d: %s len: %u int_en %d\n",
  1750. ppc440spe_chan->device->id, __func__, len,
  1751. flags & DMA_PREP_INTERRUPT ? 1 : 0);
  1752. slot_cnt = slots_per_op = 1;
  1753. sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
  1754. slots_per_op);
  1755. if (sw_desc) {
  1756. group_start = sw_desc->group_head;
  1757. ppc440spe_desc_init_memcpy(group_start, flags);
  1758. ppc440spe_adma_set_dest(group_start, dma_dest, 0);
  1759. ppc440spe_adma_memcpy_xor_set_src(group_start, dma_src, 0);
  1760. ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
  1761. sw_desc->unmap_len = len;
  1762. sw_desc->async_tx.flags = flags;
  1763. }
  1764. spin_unlock_bh(&ppc440spe_chan->lock);
  1765. return sw_desc ? &sw_desc->async_tx : NULL;
  1766. }
  1767. /**
  1768. * ppc440spe_adma_prep_dma_xor - prepare CDB for a XOR operation
  1769. */
  1770. static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor(
  1771. struct dma_chan *chan, dma_addr_t dma_dest,
  1772. dma_addr_t *dma_src, u32 src_cnt, size_t len,
  1773. unsigned long flags)
  1774. {
  1775. struct ppc440spe_adma_chan *ppc440spe_chan;
  1776. struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
  1777. int slot_cnt, slots_per_op;
  1778. ppc440spe_chan = to_ppc440spe_adma_chan(chan);
  1779. ADMA_LL_DBG(prep_dma_xor_dbg(ppc440spe_chan->device->id,
  1780. dma_dest, dma_src, src_cnt));
  1781. if (unlikely(!len))
  1782. return NULL;
  1783. BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
  1784. dev_dbg(ppc440spe_chan->device->common.dev,
  1785. "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
  1786. ppc440spe_chan->device->id, __func__, src_cnt, len,
  1787. flags & DMA_PREP_INTERRUPT ? 1 : 0);
  1788. spin_lock_bh(&ppc440spe_chan->lock);
  1789. slot_cnt = ppc440spe_chan_xor_slot_count(len, src_cnt, &slots_per_op);
  1790. sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
  1791. slots_per_op);
  1792. if (sw_desc) {
  1793. group_start = sw_desc->group_head;
  1794. ppc440spe_desc_init_xor(group_start, src_cnt, flags);
  1795. ppc440spe_adma_set_dest(group_start, dma_dest, 0);
  1796. while (src_cnt--)
  1797. ppc440spe_adma_memcpy_xor_set_src(group_start,
  1798. dma_src[src_cnt], src_cnt);
  1799. ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
  1800. sw_desc->unmap_len = len;
  1801. sw_desc->async_tx.flags = flags;
  1802. }
  1803. spin_unlock_bh(&ppc440spe_chan->lock);
  1804. return sw_desc ? &sw_desc->async_tx : NULL;
  1805. }
  1806. static inline void
  1807. ppc440spe_desc_set_xor_src_cnt(struct ppc440spe_adma_desc_slot *desc,
  1808. int src_cnt);
  1809. static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor);
  1810. /**
  1811. * ppc440spe_adma_init_dma2rxor_slot -
  1812. */
  1813. static void ppc440spe_adma_init_dma2rxor_slot(
  1814. struct ppc440spe_adma_desc_slot *desc,
  1815. dma_addr_t *src, int src_cnt)
  1816. {
  1817. int i;
  1818. /* initialize CDB */
  1819. for (i = 0; i < src_cnt; i++) {
  1820. ppc440spe_adma_dma2rxor_prep_src(desc, &desc->rxor_cursor, i,
  1821. desc->src_cnt, (u32)src[i]);
  1822. }
  1823. }
  1824. /**
  1825. * ppc440spe_dma01_prep_mult -
  1826. * for Q operation where destination is also the source
  1827. */
  1828. static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_mult(
  1829. struct ppc440spe_adma_chan *ppc440spe_chan,
  1830. dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
  1831. const unsigned char *scf, size_t len, unsigned long flags)
  1832. {
  1833. struct ppc440spe_adma_desc_slot *sw_desc = NULL;
  1834. unsigned long op = 0;
  1835. int slot_cnt;
  1836. set_bit(PPC440SPE_DESC_WXOR, &op);
  1837. slot_cnt = 2;
  1838. spin_lock_bh(&ppc440spe_chan->lock);
  1839. /* use WXOR, each descriptor occupies one slot */
  1840. sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
  1841. if (sw_desc) {
  1842. struct ppc440spe_adma_chan *chan;
  1843. struct ppc440spe_adma_desc_slot *iter;
  1844. struct dma_cdb *hw_desc;
  1845. chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
  1846. set_bits(op, &sw_desc->flags);
  1847. sw_desc->src_cnt = src_cnt;
  1848. sw_desc->dst_cnt = dst_cnt;
  1849. /* First descriptor, zero data in the destination and copy it
  1850. * to q page using MULTICAST transfer.
  1851. */
  1852. iter = list_first_entry(&sw_desc->group_list,
  1853. struct ppc440spe_adma_desc_slot,
  1854. chain_node);
  1855. memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
  1856. /* set 'next' pointer */
  1857. iter->hw_next = list_entry(iter->chain_node.next,
  1858. struct ppc440spe_adma_desc_slot,
  1859. chain_node);
  1860. clear_bit(PPC440SPE_DESC_INT, &iter->flags);
  1861. hw_desc = iter->hw_desc;
  1862. hw_desc->opc = DMA_CDB_OPC_MULTICAST;
  1863. ppc440spe_desc_set_dest_addr(iter, chan,
  1864. DMA_CUED_XOR_BASE, dst[0], 0);
  1865. ppc440spe_desc_set_dest_addr(iter, chan, 0, dst[1], 1);
  1866. ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
  1867. src[0]);
  1868. ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
  1869. iter->unmap_len = len;
  1870. /*
  1871. * Second descriptor, multiply data from the q page
  1872. * and store the result in real destination.
  1873. */
  1874. iter = list_first_entry(&iter->chain_node,
  1875. struct ppc440spe_adma_desc_slot,
  1876. chain_node);
  1877. memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
  1878. iter->hw_next = NULL;
  1879. if (flags & DMA_PREP_INTERRUPT)
  1880. set_bit(PPC440SPE_DESC_INT, &iter->flags);
  1881. else
  1882. clear_bit(PPC440SPE_DESC_INT, &iter->flags);
  1883. hw_desc = iter->hw_desc;
  1884. hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
  1885. ppc440spe_desc_set_src_addr(iter, chan, 0,
  1886. DMA_CUED_XOR_HB, dst[1]);
  1887. ppc440spe_desc_set_dest_addr(iter, chan,
  1888. DMA_CUED_XOR_BASE, dst[0], 0);
  1889. ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
  1890. DMA_CDB_SG_DST1, scf[0]);
  1891. ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
  1892. iter->unmap_len = len;
  1893. sw_desc->async_tx.flags = flags;
  1894. }
  1895. spin_unlock_bh(&ppc440spe_chan->lock);
  1896. return sw_desc;
  1897. }
  1898. /**
  1899. * ppc440spe_dma01_prep_sum_product -
  1900. * Dx = A*(P+Pxy) + B*(Q+Qxy) operation where destination is also
  1901. * the source.
  1902. */
  1903. static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_sum_product(
  1904. struct ppc440spe_adma_chan *ppc440spe_chan,
  1905. dma_addr_t *dst, dma_addr_t *src, int src_cnt,
  1906. const unsigned char *scf, size_t len, unsigned long flags)
  1907. {
  1908. struct ppc440spe_adma_desc_slot *sw_desc = NULL;
  1909. unsigned long op = 0;
  1910. int slot_cnt;
  1911. set_bit(PPC440SPE_DESC_WXOR, &op);
  1912. slot_cnt = 3;
  1913. spin_lock_bh(&ppc440spe_chan->lock);
  1914. /* WXOR, each descriptor occupies one slot */
  1915. sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
  1916. if (sw_desc) {
  1917. struct ppc440spe_adma_chan *chan;
  1918. struct ppc440spe_adma_desc_slot *iter;
  1919. struct dma_cdb *hw_desc;
  1920. chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
  1921. set_bits(op, &sw_desc->flags);
  1922. sw_desc->src_cnt = src_cnt;
  1923. sw_desc->dst_cnt = 1;
  1924. /* 1st descriptor, src[1] data to q page and zero destination */
  1925. iter = list_first_entry(&sw_desc->group_list,
  1926. struct ppc440spe_adma_desc_slot,
  1927. chain_node);
  1928. memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
  1929. iter->hw_next = list_entry(iter->chain_node.next,
  1930. struct ppc440spe_adma_desc_slot,
  1931. chain_node);
  1932. clear_bit(PPC440SPE_DESC_INT, &iter->flags);
  1933. hw_desc = iter->hw_desc;
  1934. hw_desc->opc = DMA_CDB_OPC_MULTICAST;
  1935. ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
  1936. *dst, 0);
  1937. ppc440spe_desc_set_dest_addr(iter, chan, 0,
  1938. ppc440spe_chan->qdest, 1);
  1939. ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
  1940. src[1]);
  1941. ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
  1942. iter->unmap_len = len;
  1943. /* 2nd descriptor, multiply src[1] data and store the
  1944. * result in destination */
  1945. iter = list_first_entry(&iter->chain_node,
  1946. struct ppc440spe_adma_desc_slot,
  1947. chain_node);
  1948. memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
  1949. /* set 'next' pointer */
  1950. iter->hw_next = list_entry(iter->chain_node.next,
  1951. struct ppc440spe_adma_desc_slot,
  1952. chain_node);
  1953. if (flags & DMA_PREP_INTERRUPT)
  1954. set_bit(PPC440SPE_DESC_INT, &iter->flags);
  1955. else
  1956. clear_bit(PPC440SPE_DESC_INT, &iter->flags);
  1957. hw_desc = iter->hw_desc;
  1958. hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
  1959. ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
  1960. ppc440spe_chan->qdest);
  1961. ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
  1962. *dst, 0);
  1963. ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
  1964. DMA_CDB_SG_DST1, scf[1]);
  1965. ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
  1966. iter->unmap_len = len;
  1967. /*
  1968. * 3rd descriptor, multiply src[0] data and xor it
  1969. * with destination
  1970. */
  1971. iter = list_first_entry(&iter->chain_node,
  1972. struct ppc440spe_adma_desc_slot,
  1973. chain_node);
  1974. memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
  1975. iter->hw_next = NULL;
  1976. if (flags & DMA_PREP_INTERRUPT)
  1977. set_bit(PPC440SPE_DESC_INT, &iter->flags);
  1978. else
  1979. clear_bit(PPC440SPE_DESC_INT, &iter->flags);
  1980. hw_desc = iter->hw_desc;
  1981. hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
  1982. ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
  1983. src[0]);
  1984. ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
  1985. *dst, 0);
  1986. ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
  1987. DMA_CDB_SG_DST1, scf[0]);
  1988. ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
  1989. iter->unmap_len = len;
  1990. sw_desc->async_tx.flags = flags;
  1991. }
  1992. spin_unlock_bh(&ppc440spe_chan->lock);
  1993. return sw_desc;
  1994. }
  1995. static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_pq(
  1996. struct ppc440spe_adma_chan *ppc440spe_chan,
  1997. dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
  1998. const unsigned char *scf, size_t len, unsigned long flags)
  1999. {
  2000. int slot_cnt;
  2001. struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
  2002. unsigned long op = 0;
  2003. unsigned char mult = 1;
  2004. pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
  2005. __func__, dst_cnt, src_cnt, len);
  2006. /* select operations WXOR/RXOR depending on the
  2007. * source addresses of operators and the number
  2008. * of destinations (RXOR support only Q-parity calculations)
  2009. */
  2010. set_bit(PPC440SPE_DESC_WXOR, &op);
  2011. if (!test_and_set_bit(PPC440SPE_RXOR_RUN, &ppc440spe_rxor_state)) {
  2012. /* no active RXOR;
  2013. * do RXOR if:
  2014. * - there are more than 1 source,
  2015. * - len is aligned on 512-byte boundary,
  2016. * - source addresses fit to one of 4 possible regions.
  2017. */
  2018. if (src_cnt > 1 &&
  2019. !(len & MQ0_CF2H_RXOR_BS_MASK) &&
  2020. (src[0] + len) == src[1]) {
  2021. /* may do RXOR R1 R2 */
  2022. set_bit(PPC440SPE_DESC_RXOR, &op);
  2023. if (src_cnt != 2) {
  2024. /* may try to enhance region of RXOR */
  2025. if ((src[1] + len) == src[2]) {
  2026. /* do RXOR R1 R2 R3 */
  2027. set_bit(PPC440SPE_DESC_RXOR123,
  2028. &op);
  2029. } else if ((src[1] + len * 2) == src[2]) {
  2030. /* do RXOR R1 R2 R4 */
  2031. set_bit(PPC440SPE_DESC_RXOR124, &op);
  2032. } else if ((src[1] + len * 3) == src[2]) {
  2033. /* do RXOR R1 R2 R5 */
  2034. set_bit(PPC440SPE_DESC_RXOR125,
  2035. &op);
  2036. } else {
  2037. /* do RXOR R1 R2 */
  2038. set_bit(PPC440SPE_DESC_RXOR12,
  2039. &op);
  2040. }
  2041. } else {
  2042. /* do RXOR R1 R2 */
  2043. set_bit(PPC440SPE_DESC_RXOR12, &op);
  2044. }
  2045. }
  2046. if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
  2047. /* can not do this operation with RXOR */
  2048. clear_bit(PPC440SPE_RXOR_RUN,
  2049. &ppc440spe_rxor_state);
  2050. } else {
  2051. /* can do; set block size right now */
  2052. ppc440spe_desc_set_rxor_block_size(len);
  2053. }
  2054. }
  2055. /* Number of necessary slots depends on operation type selected */
  2056. if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
  2057. /* This is a WXOR only chain. Need descriptors for each
  2058. * source to GF-XOR them with WXOR, and need descriptors
  2059. * for each destination to zero them with WXOR
  2060. */
  2061. slot_cnt = src_cnt;
  2062. if (flags & DMA_PREP_ZERO_P) {
  2063. slot_cnt++;
  2064. set_bit(PPC440SPE_ZERO_P, &op);
  2065. }
  2066. if (flags & DMA_PREP_ZERO_Q) {
  2067. slot_cnt++;
  2068. set_bit(PPC440SPE_ZERO_Q, &op);
  2069. }
  2070. } else {
  2071. /* Need 1/2 descriptor for RXOR operation, and
  2072. * need (src_cnt - (2 or 3)) for WXOR of sources
  2073. * remained (if any)
  2074. */
  2075. slot_cnt = dst_cnt;
  2076. if (flags & DMA_PREP_ZERO_P)
  2077. set_bit(PPC440SPE_ZERO_P, &op);
  2078. if (flags & DMA_PREP_ZERO_Q)
  2079. set_bit(PPC440SPE_ZERO_Q, &op);
  2080. if (test_bit(PPC440SPE_DESC_RXOR12, &op))
  2081. slot_cnt += src_cnt - 2;
  2082. else
  2083. slot_cnt += src_cnt - 3;
  2084. /* Thus we have either RXOR only chain or
  2085. * mixed RXOR/WXOR
  2086. */
  2087. if (slot_cnt == dst_cnt)
  2088. /* RXOR only chain */
  2089. clear_bit(PPC440SPE_DESC_WXOR, &op);
  2090. }
  2091. spin_lock_bh(&ppc440spe_chan->lock);
  2092. /* for both RXOR/WXOR each descriptor occupies one slot */
  2093. sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
  2094. if (sw_desc) {
  2095. ppc440spe_desc_init_dma01pq(sw_desc, dst_cnt, src_cnt,
  2096. flags, op);
  2097. /* setup dst/src/mult */
  2098. pr_debug("%s: set dst descriptor 0, 1: 0x%016llx, 0x%016llx\n",
  2099. __func__, dst[0], dst[1]);
  2100. ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
  2101. while (src_cnt--) {
  2102. ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
  2103. src_cnt);
  2104. /* NOTE: "Multi = 0 is equivalent to = 1" as it
  2105. * stated in 440SPSPe_RAID6_Addendum_UM_1_17.pdf
  2106. * doesn't work for RXOR with DMA0/1! Instead, multi=0
  2107. * leads to zeroing source data after RXOR.
  2108. * So, for P case set-up mult=1 explicitly.
  2109. */
  2110. if (!(flags & DMA_PREP_PQ_DISABLE_Q))
  2111. mult = scf[src_cnt];
  2112. ppc440spe_adma_pq_set_src_mult(sw_desc,
  2113. mult, src_cnt, dst_cnt - 1);
  2114. }
  2115. /* Setup byte count foreach slot just allocated */
  2116. sw_desc->async_tx.flags = flags;
  2117. list_for_each_entry(iter, &sw_desc->group_list,
  2118. chain_node) {
  2119. ppc440spe_desc_set_byte_count(iter,
  2120. ppc440spe_chan, len);
  2121. iter->unmap_len = len;
  2122. }
  2123. }
  2124. spin_unlock_bh(&ppc440spe_chan->lock);
  2125. return sw_desc;
  2126. }
  2127. static struct ppc440spe_adma_desc_slot *ppc440spe_dma2_prep_pq(
  2128. struct ppc440spe_adma_chan *ppc440spe_chan,
  2129. dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
  2130. const unsigned char *scf, size_t len, unsigned long flags)
  2131. {
  2132. int slot_cnt, descs_per_op;
  2133. struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
  2134. unsigned long op = 0;
  2135. unsigned char mult = 1;
  2136. BUG_ON(!dst_cnt);
  2137. /*pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
  2138. __func__, dst_cnt, src_cnt, len);*/
  2139. spin_lock_bh(&ppc440spe_chan->lock);
  2140. descs_per_op = ppc440spe_dma2_pq_slot_count(src, src_cnt, len);
  2141. if (descs_per_op < 0) {
  2142. spin_unlock_bh(&ppc440spe_chan->lock);
  2143. return NULL;
  2144. }
  2145. /* depending on number of sources we have 1 or 2 RXOR chains */
  2146. slot_cnt = descs_per_op * dst_cnt;
  2147. sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
  2148. if (sw_desc) {
  2149. op = slot_cnt;
  2150. sw_desc->async_tx.flags = flags;
  2151. list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
  2152. ppc440spe_desc_init_dma2pq(iter, dst_cnt, src_cnt,
  2153. --op ? 0 : flags);
  2154. ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
  2155. len);
  2156. iter->unmap_len = len;
  2157. ppc440spe_init_rxor_cursor(&(iter->rxor_cursor));
  2158. iter->rxor_cursor.len = len;
  2159. iter->descs_per_op = descs_per_op;
  2160. }
  2161. op = 0;
  2162. list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
  2163. op++;
  2164. if (op % descs_per_op == 0)
  2165. ppc440spe_adma_init_dma2rxor_slot(iter, src,
  2166. src_cnt);
  2167. if (likely(!list_is_last(&iter->chain_node,
  2168. &sw_desc->group_list))) {
  2169. /* set 'next' pointer */
  2170. iter->hw_next =
  2171. list_entry(iter->chain_node.next,
  2172. struct ppc440spe_adma_desc_slot,
  2173. chain_node);
  2174. ppc440spe_xor_set_link(iter, iter->hw_next);
  2175. } else {
  2176. /* this is the last descriptor. */
  2177. iter->hw_next = NULL;
  2178. }
  2179. }
  2180. /* fixup head descriptor */
  2181. sw_desc->dst_cnt = dst_cnt;
  2182. if (flags & DMA_PREP_ZERO_P)
  2183. set_bit(PPC440SPE_ZERO_P, &sw_desc->flags);
  2184. if (flags & DMA_PREP_ZERO_Q)
  2185. set_bit(PPC440SPE_ZERO_Q, &sw_desc->flags);
  2186. /* setup dst/src/mult */
  2187. ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
  2188. while (src_cnt--) {
  2189. /* handle descriptors (if dst_cnt == 2) inside
  2190. * the ppc440spe_adma_pq_set_srcxxx() functions
  2191. */
  2192. ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
  2193. src_cnt);
  2194. if (!(flags & DMA_PREP_PQ_DISABLE_Q))
  2195. mult = scf[src_cnt];
  2196. ppc440spe_adma_pq_set_src_mult(sw_desc,
  2197. mult, src_cnt, dst_cnt - 1);
  2198. }
  2199. }
  2200. spin_unlock_bh(&ppc440spe_chan->lock);
  2201. ppc440spe_desc_set_rxor_block_size(len);
  2202. return sw_desc;
  2203. }
  2204. /**
  2205. * ppc440spe_adma_prep_dma_pq - prepare CDB (group) for a GF-XOR operation
  2206. */
  2207. static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pq(
  2208. struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
  2209. unsigned int src_cnt, const unsigned char *scf,
  2210. size_t len, unsigned long flags)
  2211. {
  2212. struct ppc440spe_adma_chan *ppc440spe_chan;
  2213. struct ppc440spe_adma_desc_slot *sw_desc = NULL;
  2214. int dst_cnt = 0;
  2215. ppc440spe_chan = to_ppc440spe_adma_chan(chan);
  2216. ADMA_LL_DBG(prep_dma_pq_dbg(ppc440spe_chan->device->id,
  2217. dst, src, src_cnt));
  2218. BUG_ON(!len);
  2219. BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
  2220. BUG_ON(!src_cnt);
  2221. if (src_cnt == 1 && dst[1] == src[0]) {
  2222. dma_addr_t dest[2];
  2223. /* dst[1] is real destination (Q) */
  2224. dest[0] = dst[1];
  2225. /* this is the page to multicast source data to */
  2226. dest[1] = ppc440spe_chan->qdest;
  2227. sw_desc = ppc440spe_dma01_prep_mult(ppc440spe_chan,
  2228. dest, 2, src, src_cnt, scf, len, flags);
  2229. return sw_desc ? &sw_desc->async_tx : NULL;
  2230. }
  2231. if (src_cnt == 2 && dst[1] == src[1]) {
  2232. sw_desc = ppc440spe_dma01_prep_sum_product(ppc440spe_chan,
  2233. &dst[1], src, 2, scf, len, flags);
  2234. return sw_desc ? &sw_desc->async_tx : NULL;
  2235. }
  2236. if (!(flags & DMA_PREP_PQ_DISABLE_P)) {
  2237. BUG_ON(!dst[0]);
  2238. dst_cnt++;
  2239. flags |= DMA_PREP_ZERO_P;
  2240. }
  2241. if (!(flags & DMA_PREP_PQ_DISABLE_Q)) {
  2242. BUG_ON(!dst[1]);
  2243. dst_cnt++;
  2244. flags |= DMA_PREP_ZERO_Q;
  2245. }
  2246. BUG_ON(!dst_cnt);
  2247. dev_dbg(ppc440spe_chan->device->common.dev,
  2248. "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
  2249. ppc440spe_chan->device->id, __func__, src_cnt, len,
  2250. flags & DMA_PREP_INTERRUPT ? 1 : 0);
  2251. switch (ppc440spe_chan->device->id) {
  2252. case PPC440SPE_DMA0_ID:
  2253. case PPC440SPE_DMA1_ID:
  2254. sw_desc = ppc440spe_dma01_prep_pq(ppc440spe_chan,
  2255. dst, dst_cnt, src, src_cnt, scf,
  2256. len, flags);
  2257. break;
  2258. case PPC440SPE_XOR_ID:
  2259. sw_desc = ppc440spe_dma2_prep_pq(ppc440spe_chan,
  2260. dst, dst_cnt, src, src_cnt, scf,
  2261. len, flags);
  2262. break;
  2263. }
  2264. return sw_desc ? &sw_desc->async_tx : NULL;
  2265. }
  2266. /**
  2267. * ppc440spe_adma_prep_dma_pqzero_sum - prepare CDB group for
  2268. * a PQ_ZERO_SUM operation
  2269. */
  2270. static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pqzero_sum(
  2271. struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
  2272. unsigned int src_cnt, const unsigned char *scf, size_t len,
  2273. enum sum_check_flags *pqres, unsigned long flags)
  2274. {
  2275. struct ppc440spe_adma_chan *ppc440spe_chan;
  2276. struct ppc440spe_adma_desc_slot *sw_desc, *iter;
  2277. dma_addr_t pdest, qdest;
  2278. int slot_cnt, slots_per_op, idst, dst_cnt;
  2279. ppc440spe_chan = to_ppc440spe_adma_chan(chan);
  2280. if (flags & DMA_PREP_PQ_DISABLE_P)
  2281. pdest = 0;
  2282. else
  2283. pdest = pq[0];
  2284. if (flags & DMA_PREP_PQ_DISABLE_Q)
  2285. qdest = 0;
  2286. else
  2287. qdest = pq[1];
  2288. ADMA_LL_DBG(prep_dma_pqzero_sum_dbg(ppc440spe_chan->device->id,
  2289. src, src_cnt, scf));
  2290. /* Always use WXOR for P/Q calculations (two destinations).
  2291. * Need 1 or 2 extra slots to verify results are zero.
  2292. */
  2293. idst = dst_cnt = (pdest && qdest) ? 2 : 1;
  2294. /* One additional slot per destination to clone P/Q
  2295. * before calculation (we have to preserve destinations).
  2296. */
  2297. slot_cnt = src_cnt + dst_cnt * 2;
  2298. slots_per_op = 1;
  2299. spin_lock_bh(&ppc440spe_chan->lock);
  2300. sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
  2301. slots_per_op);
  2302. if (sw_desc) {
  2303. ppc440spe_desc_init_dma01pqzero_sum(sw_desc, dst_cnt, src_cnt);
  2304. /* Setup byte count for each slot just allocated */
  2305. sw_desc->async_tx.flags = flags;
  2306. list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
  2307. ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
  2308. len);
  2309. iter->unmap_len = len;
  2310. }
  2311. if (pdest) {
  2312. struct dma_cdb *hw_desc;
  2313. struct ppc440spe_adma_chan *chan;
  2314. iter = sw_desc->group_head;
  2315. chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
  2316. memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
  2317. iter->hw_next = list_entry(iter->chain_node.next,
  2318. struct ppc440spe_adma_desc_slot,
  2319. chain_node);
  2320. hw_desc = iter->hw_desc;
  2321. hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
  2322. iter->src_cnt = 0;
  2323. iter->dst_cnt = 0;
  2324. ppc440spe_desc_set_dest_addr(iter, chan, 0,
  2325. ppc440spe_chan->pdest, 0);
  2326. ppc440spe_desc_set_src_addr(iter, chan, 0, 0, pdest);
  2327. ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
  2328. len);
  2329. iter->unmap_len = 0;
  2330. /* override pdest to preserve original P */
  2331. pdest = ppc440spe_chan->pdest;
  2332. }
  2333. if (qdest) {
  2334. struct dma_cdb *hw_desc;
  2335. struct ppc440spe_adma_chan *chan;
  2336. iter = list_first_entry(&sw_desc->group_list,
  2337. struct ppc440spe_adma_desc_slot,
  2338. chain_node);
  2339. chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
  2340. if (pdest) {
  2341. iter = list_entry(iter->chain_node.next,
  2342. struct ppc440spe_adma_desc_slot,
  2343. chain_node);
  2344. }
  2345. memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
  2346. iter->hw_next = list_entry(iter->chain_node.next,
  2347. struct ppc440spe_adma_desc_slot,
  2348. chain_node);
  2349. hw_desc = iter->hw_desc;
  2350. hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
  2351. iter->src_cnt = 0;
  2352. iter->dst_cnt = 0;
  2353. ppc440spe_desc_set_dest_addr(iter, chan, 0,
  2354. ppc440spe_chan->qdest, 0);
  2355. ppc440spe_desc_set_src_addr(iter, chan, 0, 0, qdest);
  2356. ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
  2357. len);
  2358. iter->unmap_len = 0;
  2359. /* override qdest to preserve original Q */
  2360. qdest = ppc440spe_chan->qdest;
  2361. }
  2362. /* Setup destinations for P/Q ops */
  2363. ppc440spe_adma_pqzero_sum_set_dest(sw_desc, pdest, qdest);
  2364. /* Setup zero QWORDs into DCHECK CDBs */
  2365. idst = dst_cnt;
  2366. list_for_each_entry_reverse(iter, &sw_desc->group_list,
  2367. chain_node) {
  2368. /*
  2369. * The last CDB corresponds to Q-parity check,
  2370. * the one before last CDB corresponds
  2371. * P-parity check
  2372. */
  2373. if (idst == DMA_DEST_MAX_NUM) {
  2374. if (idst == dst_cnt) {
  2375. set_bit(PPC440SPE_DESC_QCHECK,
  2376. &iter->flags);
  2377. } else {
  2378. set_bit(PPC440SPE_DESC_PCHECK,
  2379. &iter->flags);
  2380. }
  2381. } else {
  2382. if (qdest) {
  2383. set_bit(PPC440SPE_DESC_QCHECK,
  2384. &iter->flags);
  2385. } else {
  2386. set_bit(PPC440SPE_DESC_PCHECK,
  2387. &iter->flags);
  2388. }
  2389. }
  2390. iter->xor_check_result = pqres;
  2391. /*
  2392. * set it to zero, if check fail then result will
  2393. * be updated
  2394. */
  2395. *iter->xor_check_result = 0;
  2396. ppc440spe_desc_set_dcheck(iter, ppc440spe_chan,
  2397. ppc440spe_qword);
  2398. if (!(--dst_cnt))
  2399. break;
  2400. }
  2401. /* Setup sources and mults for P/Q ops */
  2402. list_for_each_entry_continue_reverse(iter, &sw_desc->group_list,
  2403. chain_node) {
  2404. struct ppc440spe_adma_chan *chan;
  2405. u32 mult_dst;
  2406. chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
  2407. ppc440spe_desc_set_src_addr(iter, chan, 0,
  2408. DMA_CUED_XOR_HB,
  2409. src[src_cnt - 1]);
  2410. if (qdest) {
  2411. mult_dst = (dst_cnt - 1) ? DMA_CDB_SG_DST2 :
  2412. DMA_CDB_SG_DST1;
  2413. ppc440spe_desc_set_src_mult(iter, chan,
  2414. DMA_CUED_MULT1_OFF,
  2415. mult_dst,
  2416. scf[src_cnt - 1]);
  2417. }
  2418. if (!(--src_cnt))
  2419. break;
  2420. }
  2421. }
  2422. spin_unlock_bh(&ppc440spe_chan->lock);
  2423. return sw_desc ? &sw_desc->async_tx : NULL;
  2424. }
  2425. /**
  2426. * ppc440spe_adma_prep_dma_xor_zero_sum - prepare CDB group for
  2427. * XOR ZERO_SUM operation
  2428. */
  2429. static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor_zero_sum(
  2430. struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt,
  2431. size_t len, enum sum_check_flags *result, unsigned long flags)
  2432. {
  2433. struct dma_async_tx_descriptor *tx;
  2434. dma_addr_t pq[2];
  2435. /* validate P, disable Q */
  2436. pq[0] = src[0];
  2437. pq[1] = 0;
  2438. flags |= DMA_PREP_PQ_DISABLE_Q;
  2439. tx = ppc440spe_adma_prep_dma_pqzero_sum(chan, pq, &src[1],
  2440. src_cnt - 1, 0, len,
  2441. result, flags);
  2442. return tx;
  2443. }
  2444. /**
  2445. * ppc440spe_adma_set_dest - set destination address into descriptor
  2446. */
  2447. static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
  2448. dma_addr_t addr, int index)
  2449. {
  2450. struct ppc440spe_adma_chan *chan;
  2451. BUG_ON(index >= sw_desc->dst_cnt);
  2452. chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
  2453. switch (chan->device->id) {
  2454. case PPC440SPE_DMA0_ID:
  2455. case PPC440SPE_DMA1_ID:
  2456. /* to do: support transfers lengths >
  2457. * PPC440SPE_ADMA_DMA/XOR_MAX_BYTE_COUNT
  2458. */
  2459. ppc440spe_desc_set_dest_addr(sw_desc->group_head,
  2460. chan, 0, addr, index);
  2461. break;
  2462. case PPC440SPE_XOR_ID:
  2463. sw_desc = ppc440spe_get_group_entry(sw_desc, index);
  2464. ppc440spe_desc_set_dest_addr(sw_desc,
  2465. chan, 0, addr, index);
  2466. break;
  2467. }
  2468. }
  2469. static void ppc440spe_adma_pq_zero_op(struct ppc440spe_adma_desc_slot *iter,
  2470. struct ppc440spe_adma_chan *chan, dma_addr_t addr)
  2471. {
  2472. /* To clear destinations update the descriptor
  2473. * (P or Q depending on index) as follows:
  2474. * addr is destination (0 corresponds to SG2):
  2475. */
  2476. ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, addr, 0);
  2477. /* ... and the addr is source: */
  2478. ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, addr);
  2479. /* addr is always SG2 then the mult is always DST1 */
  2480. ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
  2481. DMA_CDB_SG_DST1, 1);
  2482. }
  2483. /**
  2484. * ppc440spe_adma_pq_set_dest - set destination address into descriptor
  2485. * for the PQXOR operation
  2486. */
  2487. static void ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
  2488. dma_addr_t *addrs, unsigned long flags)
  2489. {
  2490. struct ppc440spe_adma_desc_slot *iter;
  2491. struct ppc440spe_adma_chan *chan;
  2492. dma_addr_t paddr, qaddr;
  2493. dma_addr_t addr = 0, ppath, qpath;
  2494. int index = 0, i;
  2495. chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
  2496. if (flags & DMA_PREP_PQ_DISABLE_P)
  2497. paddr = 0;
  2498. else
  2499. paddr = addrs[0];
  2500. if (flags & DMA_PREP_PQ_DISABLE_Q)
  2501. qaddr = 0;
  2502. else
  2503. qaddr = addrs[1];
  2504. if (!paddr || !qaddr)
  2505. addr = paddr ? paddr : qaddr;
  2506. switch (chan->device->id) {
  2507. case PPC440SPE_DMA0_ID:
  2508. case PPC440SPE_DMA1_ID:
  2509. /* walk through the WXOR source list and set P/Q-destinations
  2510. * for each slot:
  2511. */
  2512. if (!test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
  2513. /* This is WXOR-only chain; may have 1/2 zero descs */
  2514. if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
  2515. index++;
  2516. if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
  2517. index++;
  2518. iter = ppc440spe_get_group_entry(sw_desc, index);
  2519. if (addr) {
  2520. /* one destination */
  2521. list_for_each_entry_from(iter,
  2522. &sw_desc->group_list, chain_node)
  2523. ppc440spe_desc_set_dest_addr(iter, chan,
  2524. DMA_CUED_XOR_BASE, addr, 0);
  2525. } else {
  2526. /* two destinations */
  2527. list_for_each_entry_from(iter,
  2528. &sw_desc->group_list, chain_node) {
  2529. ppc440spe_desc_set_dest_addr(iter, chan,
  2530. DMA_CUED_XOR_BASE, paddr, 0);
  2531. ppc440spe_desc_set_dest_addr(iter, chan,
  2532. DMA_CUED_XOR_BASE, qaddr, 1);
  2533. }
  2534. }
  2535. if (index) {
  2536. /* To clear destinations update the descriptor
  2537. * (1st,2nd, or both depending on flags)
  2538. */
  2539. index = 0;
  2540. if (test_bit(PPC440SPE_ZERO_P,
  2541. &sw_desc->flags)) {
  2542. iter = ppc440spe_get_group_entry(
  2543. sw_desc, index++);
  2544. ppc440spe_adma_pq_zero_op(iter, chan,
  2545. paddr);
  2546. }
  2547. if (test_bit(PPC440SPE_ZERO_Q,
  2548. &sw_desc->flags)) {
  2549. iter = ppc440spe_get_group_entry(
  2550. sw_desc, index++);
  2551. ppc440spe_adma_pq_zero_op(iter, chan,
  2552. qaddr);
  2553. }
  2554. return;
  2555. }
  2556. } else {
  2557. /* This is RXOR-only or RXOR/WXOR mixed chain */
  2558. /* If we want to include destination into calculations,
  2559. * then make dest addresses cued with mult=1 (XOR).
  2560. */
  2561. ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
  2562. DMA_CUED_XOR_HB :
  2563. DMA_CUED_XOR_BASE |
  2564. (1 << DMA_CUED_MULT1_OFF);
  2565. qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
  2566. DMA_CUED_XOR_HB :
  2567. DMA_CUED_XOR_BASE |
  2568. (1 << DMA_CUED_MULT1_OFF);
  2569. /* Setup destination(s) in RXOR slot(s) */
  2570. iter = ppc440spe_get_group_entry(sw_desc, index++);
  2571. ppc440spe_desc_set_dest_addr(iter, chan,
  2572. paddr ? ppath : qpath,
  2573. paddr ? paddr : qaddr, 0);
  2574. if (!addr) {
  2575. /* two destinations */
  2576. iter = ppc440spe_get_group_entry(sw_desc,
  2577. index++);
  2578. ppc440spe_desc_set_dest_addr(iter, chan,
  2579. qpath, qaddr, 0);
  2580. }
  2581. if (test_bit(PPC440SPE_DESC_WXOR, &sw_desc->flags)) {
  2582. /* Setup destination(s) in remaining WXOR
  2583. * slots
  2584. */
  2585. iter = ppc440spe_get_group_entry(sw_desc,
  2586. index);
  2587. if (addr) {
  2588. /* one destination */
  2589. list_for_each_entry_from(iter,
  2590. &sw_desc->group_list,
  2591. chain_node)
  2592. ppc440spe_desc_set_dest_addr(
  2593. iter, chan,
  2594. DMA_CUED_XOR_BASE,
  2595. addr, 0);
  2596. } else {
  2597. /* two destinations */
  2598. list_for_each_entry_from(iter,
  2599. &sw_desc->group_list,
  2600. chain_node) {
  2601. ppc440spe_desc_set_dest_addr(
  2602. iter, chan,
  2603. DMA_CUED_XOR_BASE,
  2604. paddr, 0);
  2605. ppc440spe_desc_set_dest_addr(
  2606. iter, chan,
  2607. DMA_CUED_XOR_BASE,
  2608. qaddr, 1);
  2609. }
  2610. }
  2611. }
  2612. }
  2613. break;
  2614. case PPC440SPE_XOR_ID:
  2615. /* DMA2 descriptors have only 1 destination, so there are
  2616. * two chains - one for each dest.
  2617. * If we want to include destination into calculations,
  2618. * then make dest addresses cued with mult=1 (XOR).
  2619. */
  2620. ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
  2621. DMA_CUED_XOR_HB :
  2622. DMA_CUED_XOR_BASE |
  2623. (1 << DMA_CUED_MULT1_OFF);
  2624. qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
  2625. DMA_CUED_XOR_HB :
  2626. DMA_CUED_XOR_BASE |
  2627. (1 << DMA_CUED_MULT1_OFF);
  2628. iter = ppc440spe_get_group_entry(sw_desc, 0);
  2629. for (i = 0; i < sw_desc->descs_per_op; i++) {
  2630. ppc440spe_desc_set_dest_addr(iter, chan,
  2631. paddr ? ppath : qpath,
  2632. paddr ? paddr : qaddr, 0);
  2633. iter = list_entry(iter->chain_node.next,
  2634. struct ppc440spe_adma_desc_slot,
  2635. chain_node);
  2636. }
  2637. if (!addr) {
  2638. /* Two destinations; setup Q here */
  2639. iter = ppc440spe_get_group_entry(sw_desc,
  2640. sw_desc->descs_per_op);
  2641. for (i = 0; i < sw_desc->descs_per_op; i++) {
  2642. ppc440spe_desc_set_dest_addr(iter,
  2643. chan, qpath, qaddr, 0);
  2644. iter = list_entry(iter->chain_node.next,
  2645. struct ppc440spe_adma_desc_slot,
  2646. chain_node);
  2647. }
  2648. }
  2649. break;
  2650. }
  2651. }
  2652. /**
  2653. * ppc440spe_adma_pq_zero_sum_set_dest - set destination address into descriptor
  2654. * for the PQ_ZERO_SUM operation
  2655. */
  2656. static void ppc440spe_adma_pqzero_sum_set_dest(
  2657. struct ppc440spe_adma_desc_slot *sw_desc,
  2658. dma_addr_t paddr, dma_addr_t qaddr)
  2659. {
  2660. struct ppc440spe_adma_desc_slot *iter, *end;
  2661. struct ppc440spe_adma_chan *chan;
  2662. dma_addr_t addr = 0;
  2663. int idx;
  2664. chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
  2665. /* walk through the WXOR source list and set P/Q-destinations
  2666. * for each slot
  2667. */
  2668. idx = (paddr && qaddr) ? 2 : 1;
  2669. /* set end */
  2670. list_for_each_entry_reverse(end, &sw_desc->group_list,
  2671. chain_node) {
  2672. if (!(--idx))
  2673. break;
  2674. }
  2675. /* set start */
  2676. idx = (paddr && qaddr) ? 2 : 1;
  2677. iter = ppc440spe_get_group_entry(sw_desc, idx);
  2678. if (paddr && qaddr) {
  2679. /* two destinations */
  2680. list_for_each_entry_from(iter, &sw_desc->group_list,
  2681. chain_node) {
  2682. if (unlikely(iter == end))
  2683. break;
  2684. ppc440spe_desc_set_dest_addr(iter, chan,
  2685. DMA_CUED_XOR_BASE, paddr, 0);
  2686. ppc440spe_desc_set_dest_addr(iter, chan,
  2687. DMA_CUED_XOR_BASE, qaddr, 1);
  2688. }
  2689. } else {
  2690. /* one destination */
  2691. addr = paddr ? paddr : qaddr;
  2692. list_for_each_entry_from(iter, &sw_desc->group_list,
  2693. chain_node) {
  2694. if (unlikely(iter == end))
  2695. break;
  2696. ppc440spe_desc_set_dest_addr(iter, chan,
  2697. DMA_CUED_XOR_BASE, addr, 0);
  2698. }
  2699. }
  2700. /* The remaining descriptors are DATACHECK. These have no need in
  2701. * destination. Actually, these destinations are used there
  2702. * as sources for check operation. So, set addr as source.
  2703. */
  2704. ppc440spe_desc_set_src_addr(end, chan, 0, 0, addr ? addr : paddr);
  2705. if (!addr) {
  2706. end = list_entry(end->chain_node.next,
  2707. struct ppc440spe_adma_desc_slot, chain_node);
  2708. ppc440spe_desc_set_src_addr(end, chan, 0, 0, qaddr);
  2709. }
  2710. }
  2711. /**
  2712. * ppc440spe_desc_set_xor_src_cnt - set source count into descriptor
  2713. */
  2714. static inline void ppc440spe_desc_set_xor_src_cnt(
  2715. struct ppc440spe_adma_desc_slot *desc,
  2716. int src_cnt)
  2717. {
  2718. struct xor_cb *hw_desc = desc->hw_desc;
  2719. hw_desc->cbc &= ~XOR_CDCR_OAC_MSK;
  2720. hw_desc->cbc |= src_cnt;
  2721. }
  2722. /**
  2723. * ppc440spe_adma_pq_set_src - set source address into descriptor
  2724. */
  2725. static void ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *sw_desc,
  2726. dma_addr_t addr, int index)
  2727. {
  2728. struct ppc440spe_adma_chan *chan;
  2729. dma_addr_t haddr = 0;
  2730. struct ppc440spe_adma_desc_slot *iter = NULL;
  2731. chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
  2732. switch (chan->device->id) {
  2733. case PPC440SPE_DMA0_ID:
  2734. case PPC440SPE_DMA1_ID:
  2735. /* DMA0,1 may do: WXOR, RXOR, RXOR+WXORs chain
  2736. */
  2737. if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
  2738. /* RXOR-only or RXOR/WXOR operation */
  2739. int iskip = test_bit(PPC440SPE_DESC_RXOR12,
  2740. &sw_desc->flags) ? 2 : 3;
  2741. if (index == 0) {
  2742. /* 1st slot (RXOR) */
  2743. /* setup sources region (R1-2-3, R1-2-4,
  2744. * or R1-2-5)
  2745. */
  2746. if (test_bit(PPC440SPE_DESC_RXOR12,
  2747. &sw_desc->flags))
  2748. haddr = DMA_RXOR12 <<
  2749. DMA_CUED_REGION_OFF;
  2750. else if (test_bit(PPC440SPE_DESC_RXOR123,
  2751. &sw_desc->flags))
  2752. haddr = DMA_RXOR123 <<
  2753. DMA_CUED_REGION_OFF;
  2754. else if (test_bit(PPC440SPE_DESC_RXOR124,
  2755. &sw_desc->flags))
  2756. haddr = DMA_RXOR124 <<
  2757. DMA_CUED_REGION_OFF;
  2758. else if (test_bit(PPC440SPE_DESC_RXOR125,
  2759. &sw_desc->flags))
  2760. haddr = DMA_RXOR125 <<
  2761. DMA_CUED_REGION_OFF;
  2762. else
  2763. BUG();
  2764. haddr |= DMA_CUED_XOR_BASE;
  2765. iter = ppc440spe_get_group_entry(sw_desc, 0);
  2766. } else if (index < iskip) {
  2767. /* 1st slot (RXOR)
  2768. * shall actually set source address only once
  2769. * instead of first <iskip>
  2770. */
  2771. iter = NULL;
  2772. } else {
  2773. /* 2nd/3d and next slots (WXOR);
  2774. * skip first slot with RXOR
  2775. */
  2776. haddr = DMA_CUED_XOR_HB;
  2777. iter = ppc440spe_get_group_entry(sw_desc,
  2778. index - iskip + sw_desc->dst_cnt);
  2779. }
  2780. } else {
  2781. int znum = 0;
  2782. /* WXOR-only operation; skip first slots with
  2783. * zeroing destinations
  2784. */
  2785. if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
  2786. znum++;
  2787. if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
  2788. znum++;
  2789. haddr = DMA_CUED_XOR_HB;
  2790. iter = ppc440spe_get_group_entry(sw_desc,
  2791. index + znum);
  2792. }
  2793. if (likely(iter)) {
  2794. ppc440spe_desc_set_src_addr(iter, chan, 0, haddr, addr);
  2795. if (!index &&
  2796. test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags) &&
  2797. sw_desc->dst_cnt == 2) {
  2798. /* if we have two destinations for RXOR, then
  2799. * setup source in the second descr too
  2800. */
  2801. iter = ppc440spe_get_group_entry(sw_desc, 1);
  2802. ppc440spe_desc_set_src_addr(iter, chan, 0,
  2803. haddr, addr);
  2804. }
  2805. }
  2806. break;
  2807. case PPC440SPE_XOR_ID:
  2808. /* DMA2 may do Biskup */
  2809. iter = sw_desc->group_head;
  2810. if (iter->dst_cnt == 2) {
  2811. /* both P & Q calculations required; set P src here */
  2812. ppc440spe_adma_dma2rxor_set_src(iter, index, addr);
  2813. /* this is for Q */
  2814. iter = ppc440spe_get_group_entry(sw_desc,
  2815. sw_desc->descs_per_op);
  2816. }
  2817. ppc440spe_adma_dma2rxor_set_src(iter, index, addr);
  2818. break;
  2819. }
  2820. }
  2821. /**
  2822. * ppc440spe_adma_memcpy_xor_set_src - set source address into descriptor
  2823. */
  2824. static void ppc440spe_adma_memcpy_xor_set_src(
  2825. struct ppc440spe_adma_desc_slot *sw_desc,
  2826. dma_addr_t addr, int index)
  2827. {
  2828. struct ppc440spe_adma_chan *chan;
  2829. chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
  2830. sw_desc = sw_desc->group_head;
  2831. if (likely(sw_desc))
  2832. ppc440spe_desc_set_src_addr(sw_desc, chan, index, 0, addr);
  2833. }
  2834. /**
  2835. * ppc440spe_adma_dma2rxor_inc_addr -
  2836. */
  2837. static void ppc440spe_adma_dma2rxor_inc_addr(
  2838. struct ppc440spe_adma_desc_slot *desc,
  2839. struct ppc440spe_rxor *cursor, int index, int src_cnt)
  2840. {
  2841. cursor->addr_count++;
  2842. if (index == src_cnt - 1) {
  2843. ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);
  2844. } else if (cursor->addr_count == XOR_MAX_OPS) {
  2845. ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);
  2846. cursor->addr_count = 0;
  2847. cursor->desc_count++;
  2848. }
  2849. }
  2850. /**
  2851. * ppc440spe_adma_dma2rxor_prep_src - setup RXOR types in DMA2 CDB
  2852. */
  2853. static int ppc440spe_adma_dma2rxor_prep_src(
  2854. struct ppc440spe_adma_desc_slot *hdesc,
  2855. struct ppc440spe_rxor *cursor, int index,
  2856. int src_cnt, u32 addr)
  2857. {
  2858. int rval = 0;
  2859. u32 sign;
  2860. struct ppc440spe_adma_desc_slot *desc = hdesc;
  2861. int i;
  2862. for (i = 0; i < cursor->desc_count; i++) {
  2863. desc = list_entry(hdesc->chain_node.next,
  2864. struct ppc440spe_adma_desc_slot,
  2865. chain_node);
  2866. }
  2867. switch (cursor->state) {
  2868. case 0:
  2869. if (addr == cursor->addrl + cursor->len) {
  2870. /* direct RXOR */
  2871. cursor->state = 1;
  2872. cursor->xor_count++;
  2873. if (index == src_cnt-1) {
  2874. ppc440spe_rxor_set_region(desc,
  2875. cursor->addr_count,
  2876. DMA_RXOR12 << DMA_CUED_REGION_OFF);
  2877. ppc440spe_adma_dma2rxor_inc_addr(
  2878. desc, cursor, index, src_cnt);
  2879. }
  2880. } else if (cursor->addrl == addr + cursor->len) {
  2881. /* reverse RXOR */
  2882. cursor->state = 1;
  2883. cursor->xor_count++;
  2884. set_bit(cursor->addr_count, &desc->reverse_flags[0]);
  2885. if (index == src_cnt-1) {
  2886. ppc440spe_rxor_set_region(desc,
  2887. cursor->addr_count,
  2888. DMA_RXOR12 << DMA_CUED_REGION_OFF);
  2889. ppc440spe_adma_dma2rxor_inc_addr(
  2890. desc, cursor, index, src_cnt);
  2891. }
  2892. } else {
  2893. printk(KERN_ERR "Cannot build "
  2894. "DMA2 RXOR command block.\n");
  2895. BUG();
  2896. }
  2897. break;
  2898. case 1:
  2899. sign = test_bit(cursor->addr_count,
  2900. desc->reverse_flags)
  2901. ? -1 : 1;
  2902. if (index == src_cnt-2 || (sign == -1
  2903. && addr != cursor->addrl - 2*cursor->len)) {
  2904. cursor->state = 0;
  2905. cursor->xor_count = 1;
  2906. cursor->addrl = addr;
  2907. ppc440spe_rxor_set_region(desc,
  2908. cursor->addr_count,
  2909. DMA_RXOR12 << DMA_CUED_REGION_OFF);
  2910. ppc440spe_adma_dma2rxor_inc_addr(
  2911. desc, cursor, index, src_cnt);
  2912. } else if (addr == cursor->addrl + 2*sign*cursor->len) {
  2913. cursor->state = 2;
  2914. cursor->xor_count = 0;
  2915. ppc440spe_rxor_set_region(desc,
  2916. cursor->addr_count,
  2917. DMA_RXOR123 << DMA_CUED_REGION_OFF);
  2918. if (index == src_cnt-1) {
  2919. ppc440spe_adma_dma2rxor_inc_addr(
  2920. desc, cursor, index, src_cnt);
  2921. }
  2922. } else if (addr == cursor->addrl + 3*cursor->len) {
  2923. cursor->state = 2;
  2924. cursor->xor_count = 0;
  2925. ppc440spe_rxor_set_region(desc,
  2926. cursor->addr_count,
  2927. DMA_RXOR124 << DMA_CUED_REGION_OFF);
  2928. if (index == src_cnt-1) {
  2929. ppc440spe_adma_dma2rxor_inc_addr(
  2930. desc, cursor, index, src_cnt);
  2931. }
  2932. } else if (addr == cursor->addrl + 4*cursor->len) {
  2933. cursor->state = 2;
  2934. cursor->xor_count = 0;
  2935. ppc440spe_rxor_set_region(desc,
  2936. cursor->addr_count,
  2937. DMA_RXOR125 << DMA_CUED_REGION_OFF);
  2938. if (index == src_cnt-1) {
  2939. ppc440spe_adma_dma2rxor_inc_addr(
  2940. desc, cursor, index, src_cnt);
  2941. }
  2942. } else {
  2943. cursor->state = 0;
  2944. cursor->xor_count = 1;
  2945. cursor->addrl = addr;
  2946. ppc440spe_rxor_set_region(desc,
  2947. cursor->addr_count,
  2948. DMA_RXOR12 << DMA_CUED_REGION_OFF);
  2949. ppc440spe_adma_dma2rxor_inc_addr(
  2950. desc, cursor, index, src_cnt);
  2951. }
  2952. break;
  2953. case 2:
  2954. cursor->state = 0;
  2955. cursor->addrl = addr;
  2956. cursor->xor_count++;
  2957. if (index) {
  2958. ppc440spe_adma_dma2rxor_inc_addr(
  2959. desc, cursor, index, src_cnt);
  2960. }
  2961. break;
  2962. }
  2963. return rval;
  2964. }
  2965. /**
  2966. * ppc440spe_adma_dma2rxor_set_src - set RXOR source address; it's assumed that
  2967. * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
  2968. */
  2969. static void ppc440spe_adma_dma2rxor_set_src(
  2970. struct ppc440spe_adma_desc_slot *desc,
  2971. int index, dma_addr_t addr)
  2972. {
  2973. struct xor_cb *xcb = desc->hw_desc;
  2974. int k = 0, op = 0, lop = 0;
  2975. /* get the RXOR operand which corresponds to index addr */
  2976. while (op <= index) {
  2977. lop = op;
  2978. if (k == XOR_MAX_OPS) {
  2979. k = 0;
  2980. desc = list_entry(desc->chain_node.next,
  2981. struct ppc440spe_adma_desc_slot, chain_node);
  2982. xcb = desc->hw_desc;
  2983. }
  2984. if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
  2985. (DMA_RXOR12 << DMA_CUED_REGION_OFF))
  2986. op += 2;
  2987. else
  2988. op += 3;
  2989. }
  2990. BUG_ON(k < 1);
  2991. if (test_bit(k-1, desc->reverse_flags)) {
  2992. /* reverse operand order; put last op in RXOR group */
  2993. if (index == op - 1)
  2994. ppc440spe_rxor_set_src(desc, k - 1, addr);
  2995. } else {
  2996. /* direct operand order; put first op in RXOR group */
  2997. if (index == lop)
  2998. ppc440spe_rxor_set_src(desc, k - 1, addr);
  2999. }
  3000. }
  3001. /**
  3002. * ppc440spe_adma_dma2rxor_set_mult - set RXOR multipliers; it's assumed that
  3003. * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
  3004. */
  3005. static void ppc440spe_adma_dma2rxor_set_mult(
  3006. struct ppc440spe_adma_desc_slot *desc,
  3007. int index, u8 mult)
  3008. {
  3009. struct xor_cb *xcb = desc->hw_desc;
  3010. int k = 0, op = 0, lop = 0;
  3011. /* get the RXOR operand which corresponds to index mult */
  3012. while (op <= index) {
  3013. lop = op;
  3014. if (k == XOR_MAX_OPS) {
  3015. k = 0;
  3016. desc = list_entry(desc->chain_node.next,
  3017. struct ppc440spe_adma_desc_slot,
  3018. chain_node);
  3019. xcb = desc->hw_desc;
  3020. }
  3021. if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
  3022. (DMA_RXOR12 << DMA_CUED_REGION_OFF))
  3023. op += 2;
  3024. else
  3025. op += 3;
  3026. }
  3027. BUG_ON(k < 1);
  3028. if (test_bit(k-1, desc->reverse_flags)) {
  3029. /* reverse order */
  3030. ppc440spe_rxor_set_mult(desc, k - 1, op - index - 1, mult);
  3031. } else {
  3032. /* direct order */
  3033. ppc440spe_rxor_set_mult(desc, k - 1, index - lop, mult);
  3034. }
  3035. }
  3036. /**
  3037. * ppc440spe_init_rxor_cursor -
  3038. */
  3039. static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor)
  3040. {
  3041. memset(cursor, 0, sizeof(struct ppc440spe_rxor));
  3042. cursor->state = 2;
  3043. }
  3044. /**
  3045. * ppc440spe_adma_pq_set_src_mult - set multiplication coefficient into
  3046. * descriptor for the PQXOR operation
  3047. */
  3048. static void ppc440spe_adma_pq_set_src_mult(
  3049. struct ppc440spe_adma_desc_slot *sw_desc,
  3050. unsigned char mult, int index, int dst_pos)
  3051. {
  3052. struct ppc440spe_adma_chan *chan;
  3053. u32 mult_idx, mult_dst;
  3054. struct ppc440spe_adma_desc_slot *iter = NULL, *iter1 = NULL;
  3055. chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
  3056. switch (chan->device->id) {
  3057. case PPC440SPE_DMA0_ID:
  3058. case PPC440SPE_DMA1_ID:
  3059. if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
  3060. int region = test_bit(PPC440SPE_DESC_RXOR12,
  3061. &sw_desc->flags) ? 2 : 3;
  3062. if (index < region) {
  3063. /* RXOR multipliers */
  3064. iter = ppc440spe_get_group_entry(sw_desc,
  3065. sw_desc->dst_cnt - 1);
  3066. if (sw_desc->dst_cnt == 2)
  3067. iter1 = ppc440spe_get_group_entry(
  3068. sw_desc, 0);
  3069. mult_idx = DMA_CUED_MULT1_OFF + (index << 3);
  3070. mult_dst = DMA_CDB_SG_SRC;
  3071. } else {
  3072. /* WXOR multiplier */
  3073. iter = ppc440spe_get_group_entry(sw_desc,
  3074. index - region +
  3075. sw_desc->dst_cnt);
  3076. mult_idx = DMA_CUED_MULT1_OFF;
  3077. mult_dst = dst_pos ? DMA_CDB_SG_DST2 :
  3078. DMA_CDB_SG_DST1;
  3079. }
  3080. } else {
  3081. int znum = 0;
  3082. /* WXOR-only;
  3083. * skip first slots with destinations (if ZERO_DST has
  3084. * place)
  3085. */
  3086. if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
  3087. znum++;
  3088. if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
  3089. znum++;
  3090. iter = ppc440spe_get_group_entry(sw_desc, index + znum);
  3091. mult_idx = DMA_CUED_MULT1_OFF;
  3092. mult_dst = dst_pos ? DMA_CDB_SG_DST2 : DMA_CDB_SG_DST1;
  3093. }
  3094. if (likely(iter)) {
  3095. ppc440spe_desc_set_src_mult(iter, chan,
  3096. mult_idx, mult_dst, mult);
  3097. if (unlikely(iter1)) {
  3098. /* if we have two destinations for RXOR, then
  3099. * we've just set Q mult. Set-up P now.
  3100. */
  3101. ppc440spe_desc_set_src_mult(iter1, chan,
  3102. mult_idx, mult_dst, 1);
  3103. }
  3104. }
  3105. break;
  3106. case PPC440SPE_XOR_ID:
  3107. iter = sw_desc->group_head;
  3108. if (sw_desc->dst_cnt == 2) {
  3109. /* both P & Q calculations required; set P mult here */
  3110. ppc440spe_adma_dma2rxor_set_mult(iter, index, 1);
  3111. /* and then set Q mult */
  3112. iter = ppc440spe_get_group_entry(sw_desc,
  3113. sw_desc->descs_per_op);
  3114. }
  3115. ppc440spe_adma_dma2rxor_set_mult(iter, index, mult);
  3116. break;
  3117. }
  3118. }
  3119. /**
  3120. * ppc440spe_adma_free_chan_resources - free the resources allocated
  3121. */
  3122. static void ppc440spe_adma_free_chan_resources(struct dma_chan *chan)
  3123. {
  3124. struct ppc440spe_adma_chan *ppc440spe_chan;
  3125. struct ppc440spe_adma_desc_slot *iter, *_iter;
  3126. int in_use_descs = 0;
  3127. ppc440spe_chan = to_ppc440spe_adma_chan(chan);
  3128. ppc440spe_adma_slot_cleanup(ppc440spe_chan);
  3129. spin_lock_bh(&ppc440spe_chan->lock);
  3130. list_for_each_entry_safe(iter, _iter, &ppc440spe_chan->chain,
  3131. chain_node) {
  3132. in_use_descs++;
  3133. list_del(&iter->chain_node);
  3134. }
  3135. list_for_each_entry_safe_reverse(iter, _iter,
  3136. &ppc440spe_chan->all_slots, slot_node) {
  3137. list_del(&iter->slot_node);
  3138. kfree(iter);
  3139. ppc440spe_chan->slots_allocated--;
  3140. }
  3141. ppc440spe_chan->last_used = NULL;
  3142. dev_dbg(ppc440spe_chan->device->common.dev,
  3143. "ppc440spe adma%d %s slots_allocated %d\n",
  3144. ppc440spe_chan->device->id,
  3145. __func__, ppc440spe_chan->slots_allocated);
  3146. spin_unlock_bh(&ppc440spe_chan->lock);
  3147. /* one is ok since we left it on there on purpose */
  3148. if (in_use_descs > 1)
  3149. printk(KERN_ERR "SPE: Freeing %d in use descriptors!\n",
  3150. in_use_descs - 1);
  3151. }
  3152. /**
  3153. * ppc440spe_adma_tx_status - poll the status of an ADMA transaction
  3154. * @chan: ADMA channel handle
  3155. * @cookie: ADMA transaction identifier
  3156. * @txstate: a holder for the current state of the channel
  3157. */
  3158. static enum dma_status ppc440spe_adma_tx_status(struct dma_chan *chan,
  3159. dma_cookie_t cookie, struct dma_tx_state *txstate)
  3160. {
  3161. struct ppc440spe_adma_chan *ppc440spe_chan;
  3162. enum dma_status ret;
  3163. ppc440spe_chan = to_ppc440spe_adma_chan(chan);
  3164. ret = dma_cookie_status(chan, cookie, txstate);
  3165. if (ret == DMA_COMPLETE)
  3166. return ret;
  3167. ppc440spe_adma_slot_cleanup(ppc440spe_chan);
  3168. return dma_cookie_status(chan, cookie, txstate);
  3169. }
  3170. /**
  3171. * ppc440spe_adma_eot_handler - end of transfer interrupt handler
  3172. */
  3173. static irqreturn_t ppc440spe_adma_eot_handler(int irq, void *data)
  3174. {
  3175. struct ppc440spe_adma_chan *chan = data;
  3176. dev_dbg(chan->device->common.dev,
  3177. "ppc440spe adma%d: %s\n", chan->device->id, __func__);
  3178. tasklet_schedule(&chan->irq_tasklet);
  3179. ppc440spe_adma_device_clear_eot_status(chan);
  3180. return IRQ_HANDLED;
  3181. }
  3182. /**
  3183. * ppc440spe_adma_err_handler - DMA error interrupt handler;
  3184. * do the same things as a eot handler
  3185. */
  3186. static irqreturn_t ppc440spe_adma_err_handler(int irq, void *data)
  3187. {
  3188. struct ppc440spe_adma_chan *chan = data;
  3189. dev_dbg(chan->device->common.dev,
  3190. "ppc440spe adma%d: %s\n", chan->device->id, __func__);
  3191. tasklet_schedule(&chan->irq_tasklet);
  3192. ppc440spe_adma_device_clear_eot_status(chan);
  3193. return IRQ_HANDLED;
  3194. }
  3195. /**
  3196. * ppc440spe_test_callback - called when test operation has been done
  3197. */
  3198. static void ppc440spe_test_callback(void *unused)
  3199. {
  3200. complete(&ppc440spe_r6_test_comp);
  3201. }
  3202. /**
  3203. * ppc440spe_adma_issue_pending - flush all pending descriptors to h/w
  3204. */
  3205. static void ppc440spe_adma_issue_pending(struct dma_chan *chan)
  3206. {
  3207. struct ppc440spe_adma_chan *ppc440spe_chan;
  3208. ppc440spe_chan = to_ppc440spe_adma_chan(chan);
  3209. dev_dbg(ppc440spe_chan->device->common.dev,
  3210. "ppc440spe adma%d: %s %d \n", ppc440spe_chan->device->id,
  3211. __func__, ppc440spe_chan->pending);
  3212. if (ppc440spe_chan->pending) {
  3213. ppc440spe_chan->pending = 0;
  3214. ppc440spe_chan_append(ppc440spe_chan);
  3215. }
  3216. }
  3217. /**
  3218. * ppc440spe_chan_start_null_xor - initiate the first XOR operation (DMA engines
  3219. * use FIFOs (as opposite to chains used in XOR) so this is a XOR
  3220. * specific operation)
  3221. */
  3222. static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan)
  3223. {
  3224. struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
  3225. dma_cookie_t cookie;
  3226. int slot_cnt, slots_per_op;
  3227. dev_dbg(chan->device->common.dev,
  3228. "ppc440spe adma%d: %s\n", chan->device->id, __func__);
  3229. spin_lock_bh(&chan->lock);
  3230. slot_cnt = ppc440spe_chan_xor_slot_count(0, 2, &slots_per_op);
  3231. sw_desc = ppc440spe_adma_alloc_slots(chan, slot_cnt, slots_per_op);
  3232. if (sw_desc) {
  3233. group_start = sw_desc->group_head;
  3234. list_splice_init(&sw_desc->group_list, &chan->chain);
  3235. async_tx_ack(&sw_desc->async_tx);
  3236. ppc440spe_desc_init_null_xor(group_start);
  3237. cookie = dma_cookie_assign(&sw_desc->async_tx);
  3238. /* initialize the completed cookie to be less than
  3239. * the most recently used cookie
  3240. */
  3241. chan->common.completed_cookie = cookie - 1;
  3242. /* channel should not be busy */
  3243. BUG_ON(ppc440spe_chan_is_busy(chan));
  3244. /* set the descriptor address */
  3245. ppc440spe_chan_set_first_xor_descriptor(chan, sw_desc);
  3246. /* run the descriptor */
  3247. ppc440spe_chan_run(chan);
  3248. } else
  3249. printk(KERN_ERR "ppc440spe adma%d"
  3250. " failed to allocate null descriptor\n",
  3251. chan->device->id);
  3252. spin_unlock_bh(&chan->lock);
  3253. }
  3254. /**
  3255. * ppc440spe_test_raid6 - test are RAID-6 capabilities enabled successfully.
  3256. * For this we just perform one WXOR operation with the same source
  3257. * and destination addresses, the GF-multiplier is 1; so if RAID-6
  3258. * capabilities are enabled then we'll get src/dst filled with zero.
  3259. */
  3260. static int ppc440spe_test_raid6(struct ppc440spe_adma_chan *chan)
  3261. {
  3262. struct ppc440spe_adma_desc_slot *sw_desc, *iter;
  3263. struct page *pg;
  3264. char *a;
  3265. dma_addr_t dma_addr, addrs[2];
  3266. unsigned long op = 0;
  3267. int rval = 0;
  3268. set_bit(PPC440SPE_DESC_WXOR, &op);
  3269. pg = alloc_page(GFP_KERNEL);
  3270. if (!pg)
  3271. return -ENOMEM;
  3272. spin_lock_bh(&chan->lock);
  3273. sw_desc = ppc440spe_adma_alloc_slots(chan, 1, 1);
  3274. if (sw_desc) {
  3275. /* 1 src, 1 dsr, int_ena, WXOR */
  3276. ppc440spe_desc_init_dma01pq(sw_desc, 1, 1, 1, op);
  3277. list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
  3278. ppc440spe_desc_set_byte_count(iter, chan, PAGE_SIZE);
  3279. iter->unmap_len = PAGE_SIZE;
  3280. }
  3281. } else {
  3282. rval = -EFAULT;
  3283. spin_unlock_bh(&chan->lock);
  3284. goto exit;
  3285. }
  3286. spin_unlock_bh(&chan->lock);
  3287. /* Fill the test page with ones */
  3288. memset(page_address(pg), 0xFF, PAGE_SIZE);
  3289. dma_addr = dma_map_page(chan->device->dev, pg, 0,
  3290. PAGE_SIZE, DMA_BIDIRECTIONAL);
  3291. /* Setup addresses */
  3292. ppc440spe_adma_pq_set_src(sw_desc, dma_addr, 0);
  3293. ppc440spe_adma_pq_set_src_mult(sw_desc, 1, 0, 0);
  3294. addrs[0] = dma_addr;
  3295. addrs[1] = 0;
  3296. ppc440spe_adma_pq_set_dest(sw_desc, addrs, DMA_PREP_PQ_DISABLE_Q);
  3297. async_tx_ack(&sw_desc->async_tx);
  3298. sw_desc->async_tx.callback = ppc440spe_test_callback;
  3299. sw_desc->async_tx.callback_param = NULL;
  3300. init_completion(&ppc440spe_r6_test_comp);
  3301. ppc440spe_adma_tx_submit(&sw_desc->async_tx);
  3302. ppc440spe_adma_issue_pending(&chan->common);
  3303. wait_for_completion(&ppc440spe_r6_test_comp);
  3304. /* Now check if the test page is zeroed */
  3305. a = page_address(pg);
  3306. if ((*(u32 *)a) == 0 && memcmp(a, a+4, PAGE_SIZE-4) == 0) {
  3307. /* page is zero - RAID-6 enabled */
  3308. rval = 0;
  3309. } else {
  3310. /* RAID-6 was not enabled */
  3311. rval = -EINVAL;
  3312. }
  3313. exit:
  3314. __free_page(pg);
  3315. return rval;
  3316. }
  3317. static void ppc440spe_adma_init_capabilities(struct ppc440spe_adma_device *adev)
  3318. {
  3319. switch (adev->id) {
  3320. case PPC440SPE_DMA0_ID:
  3321. case PPC440SPE_DMA1_ID:
  3322. dma_cap_set(DMA_MEMCPY, adev->common.cap_mask);
  3323. dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
  3324. dma_cap_set(DMA_PQ, adev->common.cap_mask);
  3325. dma_cap_set(DMA_PQ_VAL, adev->common.cap_mask);
  3326. dma_cap_set(DMA_XOR_VAL, adev->common.cap_mask);
  3327. break;
  3328. case PPC440SPE_XOR_ID:
  3329. dma_cap_set(DMA_XOR, adev->common.cap_mask);
  3330. dma_cap_set(DMA_PQ, adev->common.cap_mask);
  3331. dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
  3332. adev->common.cap_mask = adev->common.cap_mask;
  3333. break;
  3334. }
  3335. /* Set base routines */
  3336. adev->common.device_alloc_chan_resources =
  3337. ppc440spe_adma_alloc_chan_resources;
  3338. adev->common.device_free_chan_resources =
  3339. ppc440spe_adma_free_chan_resources;
  3340. adev->common.device_tx_status = ppc440spe_adma_tx_status;
  3341. adev->common.device_issue_pending = ppc440spe_adma_issue_pending;
  3342. /* Set prep routines based on capability */
  3343. if (dma_has_cap(DMA_MEMCPY, adev->common.cap_mask)) {
  3344. adev->common.device_prep_dma_memcpy =
  3345. ppc440spe_adma_prep_dma_memcpy;
  3346. }
  3347. if (dma_has_cap(DMA_XOR, adev->common.cap_mask)) {
  3348. adev->common.max_xor = XOR_MAX_OPS;
  3349. adev->common.device_prep_dma_xor =
  3350. ppc440spe_adma_prep_dma_xor;
  3351. }
  3352. if (dma_has_cap(DMA_PQ, adev->common.cap_mask)) {
  3353. switch (adev->id) {
  3354. case PPC440SPE_DMA0_ID:
  3355. dma_set_maxpq(&adev->common,
  3356. DMA0_FIFO_SIZE / sizeof(struct dma_cdb), 0);
  3357. break;
  3358. case PPC440SPE_DMA1_ID:
  3359. dma_set_maxpq(&adev->common,
  3360. DMA1_FIFO_SIZE / sizeof(struct dma_cdb), 0);
  3361. break;
  3362. case PPC440SPE_XOR_ID:
  3363. adev->common.max_pq = XOR_MAX_OPS * 3;
  3364. break;
  3365. }
  3366. adev->common.device_prep_dma_pq =
  3367. ppc440spe_adma_prep_dma_pq;
  3368. }
  3369. if (dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask)) {
  3370. switch (adev->id) {
  3371. case PPC440SPE_DMA0_ID:
  3372. adev->common.max_pq = DMA0_FIFO_SIZE /
  3373. sizeof(struct dma_cdb);
  3374. break;
  3375. case PPC440SPE_DMA1_ID:
  3376. adev->common.max_pq = DMA1_FIFO_SIZE /
  3377. sizeof(struct dma_cdb);
  3378. break;
  3379. }
  3380. adev->common.device_prep_dma_pq_val =
  3381. ppc440spe_adma_prep_dma_pqzero_sum;
  3382. }
  3383. if (dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask)) {
  3384. switch (adev->id) {
  3385. case PPC440SPE_DMA0_ID:
  3386. adev->common.max_xor = DMA0_FIFO_SIZE /
  3387. sizeof(struct dma_cdb);
  3388. break;
  3389. case PPC440SPE_DMA1_ID:
  3390. adev->common.max_xor = DMA1_FIFO_SIZE /
  3391. sizeof(struct dma_cdb);
  3392. break;
  3393. }
  3394. adev->common.device_prep_dma_xor_val =
  3395. ppc440spe_adma_prep_dma_xor_zero_sum;
  3396. }
  3397. if (dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask)) {
  3398. adev->common.device_prep_dma_interrupt =
  3399. ppc440spe_adma_prep_dma_interrupt;
  3400. }
  3401. pr_info("%s: AMCC(R) PPC440SP(E) ADMA Engine: "
  3402. "( %s%s%s%s%s%s)\n",
  3403. dev_name(adev->dev),
  3404. dma_has_cap(DMA_PQ, adev->common.cap_mask) ? "pq " : "",
  3405. dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask) ? "pq_val " : "",
  3406. dma_has_cap(DMA_XOR, adev->common.cap_mask) ? "xor " : "",
  3407. dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask) ? "xor_val " : "",
  3408. dma_has_cap(DMA_MEMCPY, adev->common.cap_mask) ? "memcpy " : "",
  3409. dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask) ? "intr " : "");
  3410. }
  3411. static int ppc440spe_adma_setup_irqs(struct ppc440spe_adma_device *adev,
  3412. struct ppc440spe_adma_chan *chan,
  3413. int *initcode)
  3414. {
  3415. struct platform_device *ofdev;
  3416. struct device_node *np;
  3417. int ret;
  3418. ofdev = container_of(adev->dev, struct platform_device, dev);
  3419. np = ofdev->dev.of_node;
  3420. if (adev->id != PPC440SPE_XOR_ID) {
  3421. adev->err_irq = irq_of_parse_and_map(np, 1);
  3422. if (!adev->err_irq) {
  3423. dev_warn(adev->dev, "no err irq resource?\n");
  3424. *initcode = PPC_ADMA_INIT_IRQ2;
  3425. adev->err_irq = -ENXIO;
  3426. } else
  3427. atomic_inc(&ppc440spe_adma_err_irq_ref);
  3428. } else {
  3429. adev->err_irq = -ENXIO;
  3430. }
  3431. adev->irq = irq_of_parse_and_map(np, 0);
  3432. if (!adev->irq) {
  3433. dev_err(adev->dev, "no irq resource\n");
  3434. *initcode = PPC_ADMA_INIT_IRQ1;
  3435. ret = -ENXIO;
  3436. goto err_irq_map;
  3437. }
  3438. dev_dbg(adev->dev, "irq %d, err irq %d\n",
  3439. adev->irq, adev->err_irq);
  3440. ret = request_irq(adev->irq, ppc440spe_adma_eot_handler,
  3441. 0, dev_driver_string(adev->dev), chan);
  3442. if (ret) {
  3443. dev_err(adev->dev, "can't request irq %d\n",
  3444. adev->irq);
  3445. *initcode = PPC_ADMA_INIT_IRQ1;
  3446. ret = -EIO;
  3447. goto err_req1;
  3448. }
  3449. /* only DMA engines have a separate error IRQ
  3450. * so it's Ok if err_irq < 0 in XOR engine case.
  3451. */
  3452. if (adev->err_irq > 0) {
  3453. /* both DMA engines share common error IRQ */
  3454. ret = request_irq(adev->err_irq,
  3455. ppc440spe_adma_err_handler,
  3456. IRQF_SHARED,
  3457. dev_driver_string(adev->dev),
  3458. chan);
  3459. if (ret) {
  3460. dev_err(adev->dev, "can't request irq %d\n",
  3461. adev->err_irq);
  3462. *initcode = PPC_ADMA_INIT_IRQ2;
  3463. ret = -EIO;
  3464. goto err_req2;
  3465. }
  3466. }
  3467. if (adev->id == PPC440SPE_XOR_ID) {
  3468. /* enable XOR engine interrupts */
  3469. iowrite32be(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
  3470. XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT,
  3471. &adev->xor_reg->ier);
  3472. } else {
  3473. u32 mask, enable;
  3474. np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe");
  3475. if (!np) {
  3476. pr_err("%s: can't find I2O device tree node\n",
  3477. __func__);
  3478. ret = -ENODEV;
  3479. goto err_req2;
  3480. }
  3481. adev->i2o_reg = of_iomap(np, 0);
  3482. if (!adev->i2o_reg) {
  3483. pr_err("%s: failed to map I2O registers\n", __func__);
  3484. of_node_put(np);
  3485. ret = -EINVAL;
  3486. goto err_req2;
  3487. }
  3488. of_node_put(np);
  3489. /* Unmask 'CS FIFO Attention' interrupts and
  3490. * enable generating interrupts on errors
  3491. */
  3492. enable = (adev->id == PPC440SPE_DMA0_ID) ?
  3493. ~(I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
  3494. ~(I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
  3495. mask = ioread32(&adev->i2o_reg->iopim) & enable;
  3496. iowrite32(mask, &adev->i2o_reg->iopim);
  3497. }
  3498. return 0;
  3499. err_req2:
  3500. free_irq(adev->irq, chan);
  3501. err_req1:
  3502. irq_dispose_mapping(adev->irq);
  3503. err_irq_map:
  3504. if (adev->err_irq > 0) {
  3505. if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref))
  3506. irq_dispose_mapping(adev->err_irq);
  3507. }
  3508. return ret;
  3509. }
  3510. static void ppc440spe_adma_release_irqs(struct ppc440spe_adma_device *adev,
  3511. struct ppc440spe_adma_chan *chan)
  3512. {
  3513. u32 mask, disable;
  3514. if (adev->id == PPC440SPE_XOR_ID) {
  3515. /* disable XOR engine interrupts */
  3516. mask = ioread32be(&adev->xor_reg->ier);
  3517. mask &= ~(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
  3518. XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT);
  3519. iowrite32be(mask, &adev->xor_reg->ier);
  3520. } else {
  3521. /* disable DMAx engine interrupts */
  3522. disable = (adev->id == PPC440SPE_DMA0_ID) ?
  3523. (I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
  3524. (I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
  3525. mask = ioread32(&adev->i2o_reg->iopim) | disable;
  3526. iowrite32(mask, &adev->i2o_reg->iopim);
  3527. }
  3528. free_irq(adev->irq, chan);
  3529. irq_dispose_mapping(adev->irq);
  3530. if (adev->err_irq > 0) {
  3531. free_irq(adev->err_irq, chan);
  3532. if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) {
  3533. irq_dispose_mapping(adev->err_irq);
  3534. iounmap(adev->i2o_reg);
  3535. }
  3536. }
  3537. }
  3538. /**
  3539. * ppc440spe_adma_probe - probe the asynch device
  3540. */
  3541. static int ppc440spe_adma_probe(struct platform_device *ofdev)
  3542. {
  3543. struct device_node *np = ofdev->dev.of_node;
  3544. struct resource res;
  3545. struct ppc440spe_adma_device *adev;
  3546. struct ppc440spe_adma_chan *chan;
  3547. struct ppc_dma_chan_ref *ref, *_ref;
  3548. int ret = 0, initcode = PPC_ADMA_INIT_OK;
  3549. const u32 *idx;
  3550. int len;
  3551. void *regs;
  3552. u32 id, pool_size;
  3553. if (of_device_is_compatible(np, "amcc,xor-accelerator")) {
  3554. id = PPC440SPE_XOR_ID;
  3555. /* As far as the XOR engine is concerned, it does not
  3556. * use FIFOs but uses linked list. So there is no dependency
  3557. * between pool size to allocate and the engine configuration.
  3558. */
  3559. pool_size = PAGE_SIZE << 1;
  3560. } else {
  3561. /* it is DMA0 or DMA1 */
  3562. idx = of_get_property(np, "cell-index", &len);
  3563. if (!idx || (len != sizeof(u32))) {
  3564. dev_err(&ofdev->dev, "Device node %pOF has missing "
  3565. "or invalid cell-index property\n",
  3566. np);
  3567. return -EINVAL;
  3568. }
  3569. id = *idx;
  3570. /* DMA0,1 engines use FIFO to maintain CDBs, so we
  3571. * should allocate the pool accordingly to size of this
  3572. * FIFO. Thus, the pool size depends on the FIFO depth:
  3573. * how much CDBs pointers the FIFO may contain then so
  3574. * much CDBs we should provide in the pool.
  3575. * That is
  3576. * CDB size = 32B;
  3577. * CDBs number = (DMA0_FIFO_SIZE >> 3);
  3578. * Pool size = CDBs number * CDB size =
  3579. * = (DMA0_FIFO_SIZE >> 3) << 5 = DMA0_FIFO_SIZE << 2.
  3580. */
  3581. pool_size = (id == PPC440SPE_DMA0_ID) ?
  3582. DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
  3583. pool_size <<= 2;
  3584. }
  3585. if (of_address_to_resource(np, 0, &res)) {
  3586. dev_err(&ofdev->dev, "failed to get memory resource\n");
  3587. initcode = PPC_ADMA_INIT_MEMRES;
  3588. ret = -ENODEV;
  3589. goto out;
  3590. }
  3591. if (!request_mem_region(res.start, resource_size(&res),
  3592. dev_driver_string(&ofdev->dev))) {
  3593. dev_err(&ofdev->dev, "failed to request memory region %pR\n",
  3594. &res);
  3595. initcode = PPC_ADMA_INIT_MEMREG;
  3596. ret = -EBUSY;
  3597. goto out;
  3598. }
  3599. /* create a device */
  3600. adev = kzalloc(sizeof(*adev), GFP_KERNEL);
  3601. if (!adev) {
  3602. initcode = PPC_ADMA_INIT_ALLOC;
  3603. ret = -ENOMEM;
  3604. goto err_adev_alloc;
  3605. }
  3606. adev->id = id;
  3607. adev->pool_size = pool_size;
  3608. /* allocate coherent memory for hardware descriptors */
  3609. adev->dma_desc_pool_virt = dma_alloc_coherent(&ofdev->dev,
  3610. adev->pool_size, &adev->dma_desc_pool,
  3611. GFP_KERNEL);
  3612. if (adev->dma_desc_pool_virt == NULL) {
  3613. dev_err(&ofdev->dev, "failed to allocate %d bytes of coherent "
  3614. "memory for hardware descriptors\n",
  3615. adev->pool_size);
  3616. initcode = PPC_ADMA_INIT_COHERENT;
  3617. ret = -ENOMEM;
  3618. goto err_dma_alloc;
  3619. }
  3620. dev_dbg(&ofdev->dev, "allocated descriptor pool virt 0x%p phys 0x%llx\n",
  3621. adev->dma_desc_pool_virt, (u64)adev->dma_desc_pool);
  3622. regs = ioremap(res.start, resource_size(&res));
  3623. if (!regs) {
  3624. dev_err(&ofdev->dev, "failed to ioremap regs!\n");
  3625. ret = -ENOMEM;
  3626. goto err_regs_alloc;
  3627. }
  3628. if (adev->id == PPC440SPE_XOR_ID) {
  3629. adev->xor_reg = regs;
  3630. /* Reset XOR */
  3631. iowrite32be(XOR_CRSR_XASR_BIT, &adev->xor_reg->crsr);
  3632. iowrite32be(XOR_CRSR_64BA_BIT, &adev->xor_reg->crrr);
  3633. } else {
  3634. size_t fifo_size = (adev->id == PPC440SPE_DMA0_ID) ?
  3635. DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
  3636. adev->dma_reg = regs;
  3637. /* DMAx_FIFO_SIZE is defined in bytes,
  3638. * <fsiz> - is defined in number of CDB pointers (8byte).
  3639. * DMA FIFO Length = CSlength + CPlength, where
  3640. * CSlength = CPlength = (fsiz + 1) * 8.
  3641. */
  3642. iowrite32(DMA_FIFO_ENABLE | ((fifo_size >> 3) - 2),
  3643. &adev->dma_reg->fsiz);
  3644. /* Configure DMA engine */
  3645. iowrite32(DMA_CFG_DXEPR_HP | DMA_CFG_DFMPP_HP | DMA_CFG_FALGN,
  3646. &adev->dma_reg->cfg);
  3647. /* Clear Status */
  3648. iowrite32(~0, &adev->dma_reg->dsts);
  3649. }
  3650. adev->dev = &ofdev->dev;
  3651. adev->common.dev = &ofdev->dev;
  3652. INIT_LIST_HEAD(&adev->common.channels);
  3653. platform_set_drvdata(ofdev, adev);
  3654. /* create a channel */
  3655. chan = kzalloc(sizeof(*chan), GFP_KERNEL);
  3656. if (!chan) {
  3657. initcode = PPC_ADMA_INIT_CHANNEL;
  3658. ret = -ENOMEM;
  3659. goto err_chan_alloc;
  3660. }
  3661. spin_lock_init(&chan->lock);
  3662. INIT_LIST_HEAD(&chan->chain);
  3663. INIT_LIST_HEAD(&chan->all_slots);
  3664. chan->device = adev;
  3665. chan->common.device = &adev->common;
  3666. dma_cookie_init(&chan->common);
  3667. list_add_tail(&chan->common.device_node, &adev->common.channels);
  3668. tasklet_setup(&chan->irq_tasklet, ppc440spe_adma_tasklet);
  3669. /* allocate and map helper pages for async validation or
  3670. * async_mult/async_sum_product operations on DMA0/1.
  3671. */
  3672. if (adev->id != PPC440SPE_XOR_ID) {
  3673. chan->pdest_page = alloc_page(GFP_KERNEL);
  3674. chan->qdest_page = alloc_page(GFP_KERNEL);
  3675. if (!chan->pdest_page ||
  3676. !chan->qdest_page) {
  3677. if (chan->pdest_page)
  3678. __free_page(chan->pdest_page);
  3679. if (chan->qdest_page)
  3680. __free_page(chan->qdest_page);
  3681. ret = -ENOMEM;
  3682. goto err_page_alloc;
  3683. }
  3684. chan->pdest = dma_map_page(&ofdev->dev, chan->pdest_page, 0,
  3685. PAGE_SIZE, DMA_BIDIRECTIONAL);
  3686. chan->qdest = dma_map_page(&ofdev->dev, chan->qdest_page, 0,
  3687. PAGE_SIZE, DMA_BIDIRECTIONAL);
  3688. }
  3689. ref = kmalloc(sizeof(*ref), GFP_KERNEL);
  3690. if (ref) {
  3691. ref->chan = &chan->common;
  3692. INIT_LIST_HEAD(&ref->node);
  3693. list_add_tail(&ref->node, &ppc440spe_adma_chan_list);
  3694. } else {
  3695. dev_err(&ofdev->dev, "failed to allocate channel reference!\n");
  3696. ret = -ENOMEM;
  3697. goto err_ref_alloc;
  3698. }
  3699. ret = ppc440spe_adma_setup_irqs(adev, chan, &initcode);
  3700. if (ret)
  3701. goto err_irq;
  3702. ppc440spe_adma_init_capabilities(adev);
  3703. ret = dma_async_device_register(&adev->common);
  3704. if (ret) {
  3705. initcode = PPC_ADMA_INIT_REGISTER;
  3706. dev_err(&ofdev->dev, "failed to register dma device\n");
  3707. goto err_dev_reg;
  3708. }
  3709. goto out;
  3710. err_dev_reg:
  3711. ppc440spe_adma_release_irqs(adev, chan);
  3712. err_irq:
  3713. list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, node) {
  3714. if (chan == to_ppc440spe_adma_chan(ref->chan)) {
  3715. list_del(&ref->node);
  3716. kfree(ref);
  3717. }
  3718. }
  3719. err_ref_alloc:
  3720. if (adev->id != PPC440SPE_XOR_ID) {
  3721. dma_unmap_page(&ofdev->dev, chan->pdest,
  3722. PAGE_SIZE, DMA_BIDIRECTIONAL);
  3723. dma_unmap_page(&ofdev->dev, chan->qdest,
  3724. PAGE_SIZE, DMA_BIDIRECTIONAL);
  3725. __free_page(chan->pdest_page);
  3726. __free_page(chan->qdest_page);
  3727. }
  3728. err_page_alloc:
  3729. kfree(chan);
  3730. err_chan_alloc:
  3731. if (adev->id == PPC440SPE_XOR_ID)
  3732. iounmap(adev->xor_reg);
  3733. else
  3734. iounmap(adev->dma_reg);
  3735. err_regs_alloc:
  3736. dma_free_coherent(adev->dev, adev->pool_size,
  3737. adev->dma_desc_pool_virt,
  3738. adev->dma_desc_pool);
  3739. err_dma_alloc:
  3740. kfree(adev);
  3741. err_adev_alloc:
  3742. release_mem_region(res.start, resource_size(&res));
  3743. out:
  3744. if (id < PPC440SPE_ADMA_ENGINES_NUM)
  3745. ppc440spe_adma_devices[id] = initcode;
  3746. return ret;
  3747. }
  3748. /**
  3749. * ppc440spe_adma_remove - remove the asynch device
  3750. */
  3751. static int ppc440spe_adma_remove(struct platform_device *ofdev)
  3752. {
  3753. struct ppc440spe_adma_device *adev = platform_get_drvdata(ofdev);
  3754. struct device_node *np = ofdev->dev.of_node;
  3755. struct resource res;
  3756. struct dma_chan *chan, *_chan;
  3757. struct ppc_dma_chan_ref *ref, *_ref;
  3758. struct ppc440spe_adma_chan *ppc440spe_chan;
  3759. if (adev->id < PPC440SPE_ADMA_ENGINES_NUM)
  3760. ppc440spe_adma_devices[adev->id] = -1;
  3761. dma_async_device_unregister(&adev->common);
  3762. list_for_each_entry_safe(chan, _chan, &adev->common.channels,
  3763. device_node) {
  3764. ppc440spe_chan = to_ppc440spe_adma_chan(chan);
  3765. ppc440spe_adma_release_irqs(adev, ppc440spe_chan);
  3766. tasklet_kill(&ppc440spe_chan->irq_tasklet);
  3767. if (adev->id != PPC440SPE_XOR_ID) {
  3768. dma_unmap_page(&ofdev->dev, ppc440spe_chan->pdest,
  3769. PAGE_SIZE, DMA_BIDIRECTIONAL);
  3770. dma_unmap_page(&ofdev->dev, ppc440spe_chan->qdest,
  3771. PAGE_SIZE, DMA_BIDIRECTIONAL);
  3772. __free_page(ppc440spe_chan->pdest_page);
  3773. __free_page(ppc440spe_chan->qdest_page);
  3774. }
  3775. list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list,
  3776. node) {
  3777. if (ppc440spe_chan ==
  3778. to_ppc440spe_adma_chan(ref->chan)) {
  3779. list_del(&ref->node);
  3780. kfree(ref);
  3781. }
  3782. }
  3783. list_del(&chan->device_node);
  3784. kfree(ppc440spe_chan);
  3785. }
  3786. dma_free_coherent(adev->dev, adev->pool_size,
  3787. adev->dma_desc_pool_virt, adev->dma_desc_pool);
  3788. if (adev->id == PPC440SPE_XOR_ID)
  3789. iounmap(adev->xor_reg);
  3790. else
  3791. iounmap(adev->dma_reg);
  3792. of_address_to_resource(np, 0, &res);
  3793. release_mem_region(res.start, resource_size(&res));
  3794. kfree(adev);
  3795. return 0;
  3796. }
  3797. /*
  3798. * /sys driver interface to enable h/w RAID-6 capabilities
  3799. * Files created in e.g. /sys/devices/plb.0/400100100.dma0/driver/
  3800. * directory are "devices", "enable" and "poly".
  3801. * "devices" shows available engines.
  3802. * "enable" is used to enable RAID-6 capabilities or to check
  3803. * whether these has been activated.
  3804. * "poly" allows setting/checking used polynomial (for PPC440SPe only).
  3805. */
  3806. static ssize_t devices_show(struct device_driver *dev, char *buf)
  3807. {
  3808. ssize_t size = 0;
  3809. int i;
  3810. for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) {
  3811. if (ppc440spe_adma_devices[i] == -1)
  3812. continue;
  3813. size += scnprintf(buf + size, PAGE_SIZE - size,
  3814. "PPC440SP(E)-ADMA.%d: %s\n", i,
  3815. ppc_adma_errors[ppc440spe_adma_devices[i]]);
  3816. }
  3817. return size;
  3818. }
  3819. static DRIVER_ATTR_RO(devices);
  3820. static ssize_t enable_show(struct device_driver *dev, char *buf)
  3821. {
  3822. return snprintf(buf, PAGE_SIZE,
  3823. "PPC440SP(e) RAID-6 capabilities are %sABLED.\n",
  3824. ppc440spe_r6_enabled ? "EN" : "DIS");
  3825. }
  3826. static ssize_t enable_store(struct device_driver *dev, const char *buf,
  3827. size_t count)
  3828. {
  3829. unsigned long val;
  3830. if (!count || count > 11)
  3831. return -EINVAL;
  3832. if (!ppc440spe_r6_tchan)
  3833. return -EFAULT;
  3834. /* Write a key */
  3835. sscanf(buf, "%lx", &val);
  3836. dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_XORBA, val);
  3837. isync();
  3838. /* Verify whether it really works now */
  3839. if (ppc440spe_test_raid6(ppc440spe_r6_tchan) == 0) {
  3840. pr_info("PPC440SP(e) RAID-6 has been activated "
  3841. "successfully\n");
  3842. ppc440spe_r6_enabled = 1;
  3843. } else {
  3844. pr_info("PPC440SP(e) RAID-6 hasn't been activated!"
  3845. " Error key ?\n");
  3846. ppc440spe_r6_enabled = 0;
  3847. }
  3848. return count;
  3849. }
  3850. static DRIVER_ATTR_RW(enable);
  3851. static ssize_t poly_show(struct device_driver *dev, char *buf)
  3852. {
  3853. ssize_t size = 0;
  3854. u32 reg;
  3855. #ifdef CONFIG_440SP
  3856. /* 440SP has fixed polynomial */
  3857. reg = 0x4d;
  3858. #else
  3859. reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
  3860. reg >>= MQ0_CFBHL_POLY;
  3861. reg &= 0xFF;
  3862. #endif
  3863. size = snprintf(buf, PAGE_SIZE, "PPC440SP(e) RAID-6 driver "
  3864. "uses 0x1%02x polynomial.\n", reg);
  3865. return size;
  3866. }
  3867. static ssize_t poly_store(struct device_driver *dev, const char *buf,
  3868. size_t count)
  3869. {
  3870. unsigned long reg, val;
  3871. #ifdef CONFIG_440SP
  3872. /* 440SP uses default 0x14D polynomial only */
  3873. return -EINVAL;
  3874. #endif
  3875. if (!count || count > 6)
  3876. return -EINVAL;
  3877. /* e.g., 0x14D or 0x11D */
  3878. sscanf(buf, "%lx", &val);
  3879. if (val & ~0x1FF)
  3880. return -EINVAL;
  3881. val &= 0xFF;
  3882. reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
  3883. reg &= ~(0xFF << MQ0_CFBHL_POLY);
  3884. reg |= val << MQ0_CFBHL_POLY;
  3885. dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, reg);
  3886. return count;
  3887. }
  3888. static DRIVER_ATTR_RW(poly);
  3889. /*
  3890. * Common initialisation for RAID engines; allocate memory for
  3891. * DMAx FIFOs, perform configuration common for all DMA engines.
  3892. * Further DMA engine specific configuration is done at probe time.
  3893. */
  3894. static int ppc440spe_configure_raid_devices(void)
  3895. {
  3896. struct device_node *np;
  3897. struct resource i2o_res;
  3898. struct i2o_regs __iomem *i2o_reg;
  3899. dcr_host_t i2o_dcr_host;
  3900. unsigned int dcr_base, dcr_len;
  3901. int i, ret;
  3902. np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe");
  3903. if (!np) {
  3904. pr_err("%s: can't find I2O device tree node\n",
  3905. __func__);
  3906. return -ENODEV;
  3907. }
  3908. if (of_address_to_resource(np, 0, &i2o_res)) {
  3909. of_node_put(np);
  3910. return -EINVAL;
  3911. }
  3912. i2o_reg = of_iomap(np, 0);
  3913. if (!i2o_reg) {
  3914. pr_err("%s: failed to map I2O registers\n", __func__);
  3915. of_node_put(np);
  3916. return -EINVAL;
  3917. }
  3918. /* Get I2O DCRs base */
  3919. dcr_base = dcr_resource_start(np, 0);
  3920. dcr_len = dcr_resource_len(np, 0);
  3921. if (!dcr_base && !dcr_len) {
  3922. pr_err("%pOF: can't get DCR registers base/len!\n", np);
  3923. of_node_put(np);
  3924. iounmap(i2o_reg);
  3925. return -ENODEV;
  3926. }
  3927. i2o_dcr_host = dcr_map(np, dcr_base, dcr_len);
  3928. if (!DCR_MAP_OK(i2o_dcr_host)) {
  3929. pr_err("%pOF: failed to map DCRs!\n", np);
  3930. of_node_put(np);
  3931. iounmap(i2o_reg);
  3932. return -ENODEV;
  3933. }
  3934. of_node_put(np);
  3935. /* Provide memory regions for DMA's FIFOs: I2O, DMA0 and DMA1 share
  3936. * the base address of FIFO memory space.
  3937. * Actually we need twice more physical memory than programmed in the
  3938. * <fsiz> register (because there are two FIFOs for each DMA: CP and CS)
  3939. */
  3940. ppc440spe_dma_fifo_buf = kmalloc((DMA0_FIFO_SIZE + DMA1_FIFO_SIZE) << 1,
  3941. GFP_KERNEL);
  3942. if (!ppc440spe_dma_fifo_buf) {
  3943. pr_err("%s: DMA FIFO buffer allocation failed.\n", __func__);
  3944. iounmap(i2o_reg);
  3945. dcr_unmap(i2o_dcr_host, dcr_len);
  3946. return -ENOMEM;
  3947. }
  3948. /*
  3949. * Configure h/w
  3950. */
  3951. /* Reset I2O/DMA */
  3952. mtdcri(SDR0, DCRN_SDR0_SRST, DCRN_SDR0_SRST_I2ODMA);
  3953. mtdcri(SDR0, DCRN_SDR0_SRST, 0);
  3954. /* Setup the base address of mmaped registers */
  3955. dcr_write(i2o_dcr_host, DCRN_I2O0_IBAH, (u32)(i2o_res.start >> 32));
  3956. dcr_write(i2o_dcr_host, DCRN_I2O0_IBAL, (u32)(i2o_res.start) |
  3957. I2O_REG_ENABLE);
  3958. dcr_unmap(i2o_dcr_host, dcr_len);
  3959. /* Setup FIFO memory space base address */
  3960. iowrite32(0, &i2o_reg->ifbah);
  3961. iowrite32(((u32)__pa(ppc440spe_dma_fifo_buf)), &i2o_reg->ifbal);
  3962. /* set zero FIFO size for I2O, so the whole
  3963. * ppc440spe_dma_fifo_buf is used by DMAs.
  3964. * DMAx_FIFOs will be configured while probe.
  3965. */
  3966. iowrite32(0, &i2o_reg->ifsiz);
  3967. iounmap(i2o_reg);
  3968. /* To prepare WXOR/RXOR functionality we need access to
  3969. * Memory Queue Module DCRs (finally it will be enabled
  3970. * via /sys interface of the ppc440spe ADMA driver).
  3971. */
  3972. np = of_find_compatible_node(NULL, NULL, "ibm,mq-440spe");
  3973. if (!np) {
  3974. pr_err("%s: can't find MQ device tree node\n",
  3975. __func__);
  3976. ret = -ENODEV;
  3977. goto out_free;
  3978. }
  3979. /* Get MQ DCRs base */
  3980. dcr_base = dcr_resource_start(np, 0);
  3981. dcr_len = dcr_resource_len(np, 0);
  3982. if (!dcr_base && !dcr_len) {
  3983. pr_err("%pOF: can't get DCR registers base/len!\n", np);
  3984. ret = -ENODEV;
  3985. goto out_mq;
  3986. }
  3987. ppc440spe_mq_dcr_host = dcr_map(np, dcr_base, dcr_len);
  3988. if (!DCR_MAP_OK(ppc440spe_mq_dcr_host)) {
  3989. pr_err("%pOF: failed to map DCRs!\n", np);
  3990. ret = -ENODEV;
  3991. goto out_mq;
  3992. }
  3993. of_node_put(np);
  3994. ppc440spe_mq_dcr_len = dcr_len;
  3995. /* Set HB alias */
  3996. dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_BAUH, DMA_CUED_XOR_HB);
  3997. /* Set:
  3998. * - LL transaction passing limit to 1;
  3999. * - Memory controller cycle limit to 1;
  4000. * - Galois Polynomial to 0x14d (default)
  4001. */
  4002. dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL,
  4003. (1 << MQ0_CFBHL_TPLM) | (1 << MQ0_CFBHL_HBCL) |
  4004. (PPC440SPE_DEFAULT_POLY << MQ0_CFBHL_POLY));
  4005. atomic_set(&ppc440spe_adma_err_irq_ref, 0);
  4006. for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++)
  4007. ppc440spe_adma_devices[i] = -1;
  4008. return 0;
  4009. out_mq:
  4010. of_node_put(np);
  4011. out_free:
  4012. kfree(ppc440spe_dma_fifo_buf);
  4013. return ret;
  4014. }
  4015. static const struct of_device_id ppc440spe_adma_of_match[] = {
  4016. { .compatible = "ibm,dma-440spe", },
  4017. { .compatible = "amcc,xor-accelerator", },
  4018. {},
  4019. };
  4020. MODULE_DEVICE_TABLE(of, ppc440spe_adma_of_match);
  4021. static struct platform_driver ppc440spe_adma_driver = {
  4022. .probe = ppc440spe_adma_probe,
  4023. .remove = ppc440spe_adma_remove,
  4024. .driver = {
  4025. .name = "PPC440SP(E)-ADMA",
  4026. .of_match_table = ppc440spe_adma_of_match,
  4027. },
  4028. };
  4029. static __init int ppc440spe_adma_init(void)
  4030. {
  4031. int ret;
  4032. ret = ppc440spe_configure_raid_devices();
  4033. if (ret)
  4034. return ret;
  4035. ret = platform_driver_register(&ppc440spe_adma_driver);
  4036. if (ret) {
  4037. pr_err("%s: failed to register platform driver\n",
  4038. __func__);
  4039. goto out_reg;
  4040. }
  4041. /* Initialization status */
  4042. ret = driver_create_file(&ppc440spe_adma_driver.driver,
  4043. &driver_attr_devices);
  4044. if (ret)
  4045. goto out_dev;
  4046. /* RAID-6 h/w enable entry */
  4047. ret = driver_create_file(&ppc440spe_adma_driver.driver,
  4048. &driver_attr_enable);
  4049. if (ret)
  4050. goto out_en;
  4051. /* GF polynomial to use */
  4052. ret = driver_create_file(&ppc440spe_adma_driver.driver,
  4053. &driver_attr_poly);
  4054. if (!ret)
  4055. return ret;
  4056. driver_remove_file(&ppc440spe_adma_driver.driver,
  4057. &driver_attr_enable);
  4058. out_en:
  4059. driver_remove_file(&ppc440spe_adma_driver.driver,
  4060. &driver_attr_devices);
  4061. out_dev:
  4062. /* User will not be able to enable h/w RAID-6 */
  4063. pr_err("%s: failed to create RAID-6 driver interface\n",
  4064. __func__);
  4065. platform_driver_unregister(&ppc440spe_adma_driver);
  4066. out_reg:
  4067. dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
  4068. kfree(ppc440spe_dma_fifo_buf);
  4069. return ret;
  4070. }
  4071. static void __exit ppc440spe_adma_exit(void)
  4072. {
  4073. driver_remove_file(&ppc440spe_adma_driver.driver,
  4074. &driver_attr_poly);
  4075. driver_remove_file(&ppc440spe_adma_driver.driver,
  4076. &driver_attr_enable);
  4077. driver_remove_file(&ppc440spe_adma_driver.driver,
  4078. &driver_attr_devices);
  4079. platform_driver_unregister(&ppc440spe_adma_driver);
  4080. dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
  4081. kfree(ppc440spe_dma_fifo_buf);
  4082. }
  4083. arch_initcall(ppc440spe_adma_init);
  4084. module_exit(ppc440spe_adma_exit);
  4085. MODULE_AUTHOR("Yuri Tikhonov <yur@emcraft.com>");
  4086. MODULE_DESCRIPTION("PPC440SPE ADMA Engine Driver");
  4087. MODULE_LICENSE("GPL");