irq-gic-v3-its.c 133 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
  4. * Author: Marc Zyngier <marc.zyngier@arm.com>
  5. */
  6. #include <linux/acpi.h>
  7. #include <linux/acpi_iort.h>
  8. #include <linux/bitfield.h>
  9. #include <linux/bitmap.h>
  10. #include <linux/cpu.h>
  11. #include <linux/crash_dump.h>
  12. #include <linux/delay.h>
  13. #include <linux/dma-iommu.h>
  14. #include <linux/efi.h>
  15. #include <linux/interrupt.h>
  16. #include <linux/iopoll.h>
  17. #include <linux/irqdomain.h>
  18. #include <linux/list.h>
  19. #include <linux/log2.h>
  20. #include <linux/memblock.h>
  21. #include <linux/mm.h>
  22. #include <linux/msi.h>
  23. #include <linux/of.h>
  24. #include <linux/of_address.h>
  25. #include <linux/of_irq.h>
  26. #include <linux/of_pci.h>
  27. #include <linux/of_platform.h>
  28. #include <linux/percpu.h>
  29. #include <linux/slab.h>
  30. #include <linux/syscore_ops.h>
  31. #include <linux/irqchip.h>
  32. #include <linux/irqchip/arm-gic-v3.h>
  33. #include <linux/irqchip/arm-gic-v4.h>
  34. #include <asm/cputype.h>
  35. #include <asm/exception.h>
  36. #include "irq-gic-common.h"
  37. #define ITS_FLAGS_CMDQ_NEEDS_FLUSHING (1ULL << 0)
  38. #define ITS_FLAGS_WORKAROUND_CAVIUM_22375 (1ULL << 1)
  39. #define ITS_FLAGS_WORKAROUND_CAVIUM_23144 (1ULL << 2)
  40. #define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING (1 << 0)
  41. #define RDIST_FLAGS_RD_TABLES_PREALLOCATED (1 << 1)
  42. static u32 lpi_id_bits;
  43. /*
  44. * We allocate memory for PROPBASE to cover 2 ^ lpi_id_bits LPIs to
  45. * deal with (one configuration byte per interrupt). PENDBASE has to
  46. * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI).
  47. */
  48. #define LPI_NRBITS lpi_id_bits
  49. #define LPI_PROPBASE_SZ ALIGN(BIT(LPI_NRBITS), SZ_64K)
  50. #define LPI_PENDBASE_SZ ALIGN(BIT(LPI_NRBITS) / 8, SZ_64K)
  51. #define LPI_PROP_DEFAULT_PRIO GICD_INT_DEF_PRI
  52. /*
  53. * Collection structure - just an ID, and a redistributor address to
  54. * ping. We use one per CPU as a bag of interrupts assigned to this
  55. * CPU.
  56. */
  57. struct its_collection {
  58. u64 target_address;
  59. u16 col_id;
  60. };
  61. /*
  62. * The ITS_BASER structure - contains memory information, cached
  63. * value of BASER register configuration and ITS page size.
  64. */
  65. struct its_baser {
  66. void *base;
  67. u64 val;
  68. u32 order;
  69. u32 psz;
  70. };
  71. struct its_device;
  72. /*
  73. * The ITS structure - contains most of the infrastructure, with the
  74. * top-level MSI domain, the command queue, the collections, and the
  75. * list of devices writing to it.
  76. *
  77. * dev_alloc_lock has to be taken for device allocations, while the
  78. * spinlock must be taken to parse data structures such as the device
  79. * list.
  80. */
  81. struct its_node {
  82. raw_spinlock_t lock;
  83. struct mutex dev_alloc_lock;
  84. struct list_head entry;
  85. void __iomem *base;
  86. void __iomem *sgir_base;
  87. phys_addr_t phys_base;
  88. struct its_cmd_block *cmd_base;
  89. struct its_cmd_block *cmd_write;
  90. struct its_baser tables[GITS_BASER_NR_REGS];
  91. struct its_collection *collections;
  92. struct fwnode_handle *fwnode_handle;
  93. u64 (*get_msi_base)(struct its_device *its_dev);
  94. u64 typer;
  95. u64 cbaser_save;
  96. u32 ctlr_save;
  97. u32 mpidr;
  98. struct list_head its_device_list;
  99. u64 flags;
  100. unsigned long list_nr;
  101. int numa_node;
  102. unsigned int msi_domain_flags;
  103. u32 pre_its_base; /* for Socionext Synquacer */
  104. int vlpi_redist_offset;
  105. };
  106. #define is_v4(its) (!!((its)->typer & GITS_TYPER_VLPIS))
  107. #define is_v4_1(its) (!!((its)->typer & GITS_TYPER_VMAPP))
  108. #define device_ids(its) (FIELD_GET(GITS_TYPER_DEVBITS, (its)->typer) + 1)
  109. #define ITS_ITT_ALIGN SZ_256
  110. /* The maximum number of VPEID bits supported by VLPI commands */
  111. #define ITS_MAX_VPEID_BITS \
  112. ({ \
  113. int nvpeid = 16; \
  114. if (gic_rdists->has_rvpeid && \
  115. gic_rdists->gicd_typer2 & GICD_TYPER2_VIL) \
  116. nvpeid = 1 + (gic_rdists->gicd_typer2 & \
  117. GICD_TYPER2_VID); \
  118. \
  119. nvpeid; \
  120. })
  121. #define ITS_MAX_VPEID (1 << (ITS_MAX_VPEID_BITS))
  122. /* Convert page order to size in bytes */
  123. #define PAGE_ORDER_TO_SIZE(o) (PAGE_SIZE << (o))
  124. struct event_lpi_map {
  125. unsigned long *lpi_map;
  126. u16 *col_map;
  127. irq_hw_number_t lpi_base;
  128. int nr_lpis;
  129. raw_spinlock_t vlpi_lock;
  130. struct its_vm *vm;
  131. struct its_vlpi_map *vlpi_maps;
  132. int nr_vlpis;
  133. };
  134. /*
  135. * The ITS view of a device - belongs to an ITS, owns an interrupt
  136. * translation table, and a list of interrupts. If it some of its
  137. * LPIs are injected into a guest (GICv4), the event_map.vm field
  138. * indicates which one.
  139. */
  140. struct its_device {
  141. struct list_head entry;
  142. struct its_node *its;
  143. struct event_lpi_map event_map;
  144. void *itt;
  145. u32 nr_ites;
  146. u32 device_id;
  147. bool shared;
  148. };
  149. static struct {
  150. raw_spinlock_t lock;
  151. struct its_device *dev;
  152. struct its_vpe **vpes;
  153. int next_victim;
  154. } vpe_proxy;
  155. struct cpu_lpi_count {
  156. atomic_t managed;
  157. atomic_t unmanaged;
  158. };
  159. static DEFINE_PER_CPU(struct cpu_lpi_count, cpu_lpi_count);
  160. static LIST_HEAD(its_nodes);
  161. static DEFINE_RAW_SPINLOCK(its_lock);
  162. static struct rdists *gic_rdists;
  163. static struct irq_domain *its_parent;
  164. static unsigned long its_list_map;
  165. static u16 vmovp_seq_num;
  166. static DEFINE_RAW_SPINLOCK(vmovp_lock);
  167. static DEFINE_IDA(its_vpeid_ida);
  168. #define gic_data_rdist() (raw_cpu_ptr(gic_rdists->rdist))
  169. #define gic_data_rdist_cpu(cpu) (per_cpu_ptr(gic_rdists->rdist, cpu))
  170. #define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base)
  171. #define gic_data_rdist_vlpi_base() (gic_data_rdist_rd_base() + SZ_128K)
  172. /*
  173. * Skip ITSs that have no vLPIs mapped, unless we're on GICv4.1, as we
  174. * always have vSGIs mapped.
  175. */
  176. static bool require_its_list_vmovp(struct its_vm *vm, struct its_node *its)
  177. {
  178. return (gic_rdists->has_rvpeid || vm->vlpi_count[its->list_nr]);
  179. }
  180. static u16 get_its_list(struct its_vm *vm)
  181. {
  182. struct its_node *its;
  183. unsigned long its_list = 0;
  184. list_for_each_entry(its, &its_nodes, entry) {
  185. if (!is_v4(its))
  186. continue;
  187. if (require_its_list_vmovp(vm, its))
  188. __set_bit(its->list_nr, &its_list);
  189. }
  190. return (u16)its_list;
  191. }
  192. static inline u32 its_get_event_id(struct irq_data *d)
  193. {
  194. struct its_device *its_dev = irq_data_get_irq_chip_data(d);
  195. return d->hwirq - its_dev->event_map.lpi_base;
  196. }
  197. static struct its_collection *dev_event_to_col(struct its_device *its_dev,
  198. u32 event)
  199. {
  200. struct its_node *its = its_dev->its;
  201. return its->collections + its_dev->event_map.col_map[event];
  202. }
  203. static struct its_vlpi_map *dev_event_to_vlpi_map(struct its_device *its_dev,
  204. u32 event)
  205. {
  206. if (WARN_ON_ONCE(event >= its_dev->event_map.nr_lpis))
  207. return NULL;
  208. return &its_dev->event_map.vlpi_maps[event];
  209. }
  210. static struct its_vlpi_map *get_vlpi_map(struct irq_data *d)
  211. {
  212. if (irqd_is_forwarded_to_vcpu(d)) {
  213. struct its_device *its_dev = irq_data_get_irq_chip_data(d);
  214. u32 event = its_get_event_id(d);
  215. return dev_event_to_vlpi_map(its_dev, event);
  216. }
  217. return NULL;
  218. }
  219. static int vpe_to_cpuid_lock(struct its_vpe *vpe, unsigned long *flags)
  220. {
  221. raw_spin_lock_irqsave(&vpe->vpe_lock, *flags);
  222. return vpe->col_idx;
  223. }
  224. static void vpe_to_cpuid_unlock(struct its_vpe *vpe, unsigned long flags)
  225. {
  226. raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);
  227. }
  228. static int irq_to_cpuid_lock(struct irq_data *d, unsigned long *flags)
  229. {
  230. struct its_vlpi_map *map = get_vlpi_map(d);
  231. int cpu;
  232. if (map) {
  233. cpu = vpe_to_cpuid_lock(map->vpe, flags);
  234. } else {
  235. /* Physical LPIs are already locked via the irq_desc lock */
  236. struct its_device *its_dev = irq_data_get_irq_chip_data(d);
  237. cpu = its_dev->event_map.col_map[its_get_event_id(d)];
  238. /* Keep GCC quiet... */
  239. *flags = 0;
  240. }
  241. return cpu;
  242. }
  243. static void irq_to_cpuid_unlock(struct irq_data *d, unsigned long flags)
  244. {
  245. struct its_vlpi_map *map = get_vlpi_map(d);
  246. if (map)
  247. vpe_to_cpuid_unlock(map->vpe, flags);
  248. }
  249. static struct its_collection *valid_col(struct its_collection *col)
  250. {
  251. if (WARN_ON_ONCE(col->target_address & GENMASK_ULL(15, 0)))
  252. return NULL;
  253. return col;
  254. }
  255. static struct its_vpe *valid_vpe(struct its_node *its, struct its_vpe *vpe)
  256. {
  257. if (valid_col(its->collections + vpe->col_idx))
  258. return vpe;
  259. return NULL;
  260. }
  261. /*
  262. * ITS command descriptors - parameters to be encoded in a command
  263. * block.
  264. */
  265. struct its_cmd_desc {
  266. union {
  267. struct {
  268. struct its_device *dev;
  269. u32 event_id;
  270. } its_inv_cmd;
  271. struct {
  272. struct its_device *dev;
  273. u32 event_id;
  274. } its_clear_cmd;
  275. struct {
  276. struct its_device *dev;
  277. u32 event_id;
  278. } its_int_cmd;
  279. struct {
  280. struct its_device *dev;
  281. int valid;
  282. } its_mapd_cmd;
  283. struct {
  284. struct its_collection *col;
  285. int valid;
  286. } its_mapc_cmd;
  287. struct {
  288. struct its_device *dev;
  289. u32 phys_id;
  290. u32 event_id;
  291. } its_mapti_cmd;
  292. struct {
  293. struct its_device *dev;
  294. struct its_collection *col;
  295. u32 event_id;
  296. } its_movi_cmd;
  297. struct {
  298. struct its_device *dev;
  299. u32 event_id;
  300. } its_discard_cmd;
  301. struct {
  302. struct its_collection *col;
  303. } its_invall_cmd;
  304. struct {
  305. struct its_vpe *vpe;
  306. } its_vinvall_cmd;
  307. struct {
  308. struct its_vpe *vpe;
  309. struct its_collection *col;
  310. bool valid;
  311. } its_vmapp_cmd;
  312. struct {
  313. struct its_vpe *vpe;
  314. struct its_device *dev;
  315. u32 virt_id;
  316. u32 event_id;
  317. bool db_enabled;
  318. } its_vmapti_cmd;
  319. struct {
  320. struct its_vpe *vpe;
  321. struct its_device *dev;
  322. u32 event_id;
  323. bool db_enabled;
  324. } its_vmovi_cmd;
  325. struct {
  326. struct its_vpe *vpe;
  327. struct its_collection *col;
  328. u16 seq_num;
  329. u16 its_list;
  330. } its_vmovp_cmd;
  331. struct {
  332. struct its_vpe *vpe;
  333. } its_invdb_cmd;
  334. struct {
  335. struct its_vpe *vpe;
  336. u8 sgi;
  337. u8 priority;
  338. bool enable;
  339. bool group;
  340. bool clear;
  341. } its_vsgi_cmd;
  342. };
  343. };
  344. /*
  345. * The ITS command block, which is what the ITS actually parses.
  346. */
  347. struct its_cmd_block {
  348. union {
  349. u64 raw_cmd[4];
  350. __le64 raw_cmd_le[4];
  351. };
  352. };
  353. #define ITS_CMD_QUEUE_SZ SZ_64K
  354. #define ITS_CMD_QUEUE_NR_ENTRIES (ITS_CMD_QUEUE_SZ / sizeof(struct its_cmd_block))
  355. typedef struct its_collection *(*its_cmd_builder_t)(struct its_node *,
  356. struct its_cmd_block *,
  357. struct its_cmd_desc *);
  358. typedef struct its_vpe *(*its_cmd_vbuilder_t)(struct its_node *,
  359. struct its_cmd_block *,
  360. struct its_cmd_desc *);
  361. static void its_mask_encode(u64 *raw_cmd, u64 val, int h, int l)
  362. {
  363. u64 mask = GENMASK_ULL(h, l);
  364. *raw_cmd &= ~mask;
  365. *raw_cmd |= (val << l) & mask;
  366. }
  367. static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr)
  368. {
  369. its_mask_encode(&cmd->raw_cmd[0], cmd_nr, 7, 0);
  370. }
  371. static void its_encode_devid(struct its_cmd_block *cmd, u32 devid)
  372. {
  373. its_mask_encode(&cmd->raw_cmd[0], devid, 63, 32);
  374. }
  375. static void its_encode_event_id(struct its_cmd_block *cmd, u32 id)
  376. {
  377. its_mask_encode(&cmd->raw_cmd[1], id, 31, 0);
  378. }
  379. static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id)
  380. {
  381. its_mask_encode(&cmd->raw_cmd[1], phys_id, 63, 32);
  382. }
  383. static void its_encode_size(struct its_cmd_block *cmd, u8 size)
  384. {
  385. its_mask_encode(&cmd->raw_cmd[1], size, 4, 0);
  386. }
  387. static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr)
  388. {
  389. its_mask_encode(&cmd->raw_cmd[2], itt_addr >> 8, 51, 8);
  390. }
  391. static void its_encode_valid(struct its_cmd_block *cmd, int valid)
  392. {
  393. its_mask_encode(&cmd->raw_cmd[2], !!valid, 63, 63);
  394. }
  395. static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr)
  396. {
  397. its_mask_encode(&cmd->raw_cmd[2], target_addr >> 16, 51, 16);
  398. }
  399. static void its_encode_collection(struct its_cmd_block *cmd, u16 col)
  400. {
  401. its_mask_encode(&cmd->raw_cmd[2], col, 15, 0);
  402. }
  403. static void its_encode_vpeid(struct its_cmd_block *cmd, u16 vpeid)
  404. {
  405. its_mask_encode(&cmd->raw_cmd[1], vpeid, 47, 32);
  406. }
  407. static void its_encode_virt_id(struct its_cmd_block *cmd, u32 virt_id)
  408. {
  409. its_mask_encode(&cmd->raw_cmd[2], virt_id, 31, 0);
  410. }
  411. static void its_encode_db_phys_id(struct its_cmd_block *cmd, u32 db_phys_id)
  412. {
  413. its_mask_encode(&cmd->raw_cmd[2], db_phys_id, 63, 32);
  414. }
  415. static void its_encode_db_valid(struct its_cmd_block *cmd, bool db_valid)
  416. {
  417. its_mask_encode(&cmd->raw_cmd[2], db_valid, 0, 0);
  418. }
  419. static void its_encode_seq_num(struct its_cmd_block *cmd, u16 seq_num)
  420. {
  421. its_mask_encode(&cmd->raw_cmd[0], seq_num, 47, 32);
  422. }
  423. static void its_encode_its_list(struct its_cmd_block *cmd, u16 its_list)
  424. {
  425. its_mask_encode(&cmd->raw_cmd[1], its_list, 15, 0);
  426. }
  427. static void its_encode_vpt_addr(struct its_cmd_block *cmd, u64 vpt_pa)
  428. {
  429. its_mask_encode(&cmd->raw_cmd[3], vpt_pa >> 16, 51, 16);
  430. }
  431. static void its_encode_vpt_size(struct its_cmd_block *cmd, u8 vpt_size)
  432. {
  433. its_mask_encode(&cmd->raw_cmd[3], vpt_size, 4, 0);
  434. }
  435. static void its_encode_vconf_addr(struct its_cmd_block *cmd, u64 vconf_pa)
  436. {
  437. its_mask_encode(&cmd->raw_cmd[0], vconf_pa >> 16, 51, 16);
  438. }
  439. static void its_encode_alloc(struct its_cmd_block *cmd, bool alloc)
  440. {
  441. its_mask_encode(&cmd->raw_cmd[0], alloc, 8, 8);
  442. }
  443. static void its_encode_ptz(struct its_cmd_block *cmd, bool ptz)
  444. {
  445. its_mask_encode(&cmd->raw_cmd[0], ptz, 9, 9);
  446. }
  447. static void its_encode_vmapp_default_db(struct its_cmd_block *cmd,
  448. u32 vpe_db_lpi)
  449. {
  450. its_mask_encode(&cmd->raw_cmd[1], vpe_db_lpi, 31, 0);
  451. }
  452. static void its_encode_vmovp_default_db(struct its_cmd_block *cmd,
  453. u32 vpe_db_lpi)
  454. {
  455. its_mask_encode(&cmd->raw_cmd[3], vpe_db_lpi, 31, 0);
  456. }
  457. static void its_encode_db(struct its_cmd_block *cmd, bool db)
  458. {
  459. its_mask_encode(&cmd->raw_cmd[2], db, 63, 63);
  460. }
  461. static void its_encode_sgi_intid(struct its_cmd_block *cmd, u8 sgi)
  462. {
  463. its_mask_encode(&cmd->raw_cmd[0], sgi, 35, 32);
  464. }
  465. static void its_encode_sgi_priority(struct its_cmd_block *cmd, u8 prio)
  466. {
  467. its_mask_encode(&cmd->raw_cmd[0], prio >> 4, 23, 20);
  468. }
  469. static void its_encode_sgi_group(struct its_cmd_block *cmd, bool grp)
  470. {
  471. its_mask_encode(&cmd->raw_cmd[0], grp, 10, 10);
  472. }
  473. static void its_encode_sgi_clear(struct its_cmd_block *cmd, bool clr)
  474. {
  475. its_mask_encode(&cmd->raw_cmd[0], clr, 9, 9);
  476. }
  477. static void its_encode_sgi_enable(struct its_cmd_block *cmd, bool en)
  478. {
  479. its_mask_encode(&cmd->raw_cmd[0], en, 8, 8);
  480. }
  481. static inline void its_fixup_cmd(struct its_cmd_block *cmd)
  482. {
  483. /* Let's fixup BE commands */
  484. cmd->raw_cmd_le[0] = cpu_to_le64(cmd->raw_cmd[0]);
  485. cmd->raw_cmd_le[1] = cpu_to_le64(cmd->raw_cmd[1]);
  486. cmd->raw_cmd_le[2] = cpu_to_le64(cmd->raw_cmd[2]);
  487. cmd->raw_cmd_le[3] = cpu_to_le64(cmd->raw_cmd[3]);
  488. }
  489. static struct its_collection *its_build_mapd_cmd(struct its_node *its,
  490. struct its_cmd_block *cmd,
  491. struct its_cmd_desc *desc)
  492. {
  493. unsigned long itt_addr;
  494. u8 size = ilog2(desc->its_mapd_cmd.dev->nr_ites);
  495. itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt);
  496. itt_addr = ALIGN(itt_addr, ITS_ITT_ALIGN);
  497. its_encode_cmd(cmd, GITS_CMD_MAPD);
  498. its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id);
  499. its_encode_size(cmd, size - 1);
  500. its_encode_itt(cmd, itt_addr);
  501. its_encode_valid(cmd, desc->its_mapd_cmd.valid);
  502. its_fixup_cmd(cmd);
  503. return NULL;
  504. }
  505. static struct its_collection *its_build_mapc_cmd(struct its_node *its,
  506. struct its_cmd_block *cmd,
  507. struct its_cmd_desc *desc)
  508. {
  509. its_encode_cmd(cmd, GITS_CMD_MAPC);
  510. its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id);
  511. its_encode_target(cmd, desc->its_mapc_cmd.col->target_address);
  512. its_encode_valid(cmd, desc->its_mapc_cmd.valid);
  513. its_fixup_cmd(cmd);
  514. return desc->its_mapc_cmd.col;
  515. }
  516. static struct its_collection *its_build_mapti_cmd(struct its_node *its,
  517. struct its_cmd_block *cmd,
  518. struct its_cmd_desc *desc)
  519. {
  520. struct its_collection *col;
  521. col = dev_event_to_col(desc->its_mapti_cmd.dev,
  522. desc->its_mapti_cmd.event_id);
  523. its_encode_cmd(cmd, GITS_CMD_MAPTI);
  524. its_encode_devid(cmd, desc->its_mapti_cmd.dev->device_id);
  525. its_encode_event_id(cmd, desc->its_mapti_cmd.event_id);
  526. its_encode_phys_id(cmd, desc->its_mapti_cmd.phys_id);
  527. its_encode_collection(cmd, col->col_id);
  528. its_fixup_cmd(cmd);
  529. return valid_col(col);
  530. }
  531. static struct its_collection *its_build_movi_cmd(struct its_node *its,
  532. struct its_cmd_block *cmd,
  533. struct its_cmd_desc *desc)
  534. {
  535. struct its_collection *col;
  536. col = dev_event_to_col(desc->its_movi_cmd.dev,
  537. desc->its_movi_cmd.event_id);
  538. its_encode_cmd(cmd, GITS_CMD_MOVI);
  539. its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id);
  540. its_encode_event_id(cmd, desc->its_movi_cmd.event_id);
  541. its_encode_collection(cmd, desc->its_movi_cmd.col->col_id);
  542. its_fixup_cmd(cmd);
  543. return valid_col(col);
  544. }
  545. static struct its_collection *its_build_discard_cmd(struct its_node *its,
  546. struct its_cmd_block *cmd,
  547. struct its_cmd_desc *desc)
  548. {
  549. struct its_collection *col;
  550. col = dev_event_to_col(desc->its_discard_cmd.dev,
  551. desc->its_discard_cmd.event_id);
  552. its_encode_cmd(cmd, GITS_CMD_DISCARD);
  553. its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id);
  554. its_encode_event_id(cmd, desc->its_discard_cmd.event_id);
  555. its_fixup_cmd(cmd);
  556. return valid_col(col);
  557. }
  558. static struct its_collection *its_build_inv_cmd(struct its_node *its,
  559. struct its_cmd_block *cmd,
  560. struct its_cmd_desc *desc)
  561. {
  562. struct its_collection *col;
  563. col = dev_event_to_col(desc->its_inv_cmd.dev,
  564. desc->its_inv_cmd.event_id);
  565. its_encode_cmd(cmd, GITS_CMD_INV);
  566. its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);
  567. its_encode_event_id(cmd, desc->its_inv_cmd.event_id);
  568. its_fixup_cmd(cmd);
  569. return valid_col(col);
  570. }
  571. static struct its_collection *its_build_int_cmd(struct its_node *its,
  572. struct its_cmd_block *cmd,
  573. struct its_cmd_desc *desc)
  574. {
  575. struct its_collection *col;
  576. col = dev_event_to_col(desc->its_int_cmd.dev,
  577. desc->its_int_cmd.event_id);
  578. its_encode_cmd(cmd, GITS_CMD_INT);
  579. its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);
  580. its_encode_event_id(cmd, desc->its_int_cmd.event_id);
  581. its_fixup_cmd(cmd);
  582. return valid_col(col);
  583. }
  584. static struct its_collection *its_build_clear_cmd(struct its_node *its,
  585. struct its_cmd_block *cmd,
  586. struct its_cmd_desc *desc)
  587. {
  588. struct its_collection *col;
  589. col = dev_event_to_col(desc->its_clear_cmd.dev,
  590. desc->its_clear_cmd.event_id);
  591. its_encode_cmd(cmd, GITS_CMD_CLEAR);
  592. its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);
  593. its_encode_event_id(cmd, desc->its_clear_cmd.event_id);
  594. its_fixup_cmd(cmd);
  595. return valid_col(col);
  596. }
  597. static struct its_collection *its_build_invall_cmd(struct its_node *its,
  598. struct its_cmd_block *cmd,
  599. struct its_cmd_desc *desc)
  600. {
  601. its_encode_cmd(cmd, GITS_CMD_INVALL);
  602. its_encode_collection(cmd, desc->its_invall_cmd.col->col_id);
  603. its_fixup_cmd(cmd);
  604. return desc->its_invall_cmd.col;
  605. }
  606. static struct its_vpe *its_build_vinvall_cmd(struct its_node *its,
  607. struct its_cmd_block *cmd,
  608. struct its_cmd_desc *desc)
  609. {
  610. its_encode_cmd(cmd, GITS_CMD_VINVALL);
  611. its_encode_vpeid(cmd, desc->its_vinvall_cmd.vpe->vpe_id);
  612. its_fixup_cmd(cmd);
  613. return valid_vpe(its, desc->its_vinvall_cmd.vpe);
  614. }
  615. static struct its_vpe *its_build_vmapp_cmd(struct its_node *its,
  616. struct its_cmd_block *cmd,
  617. struct its_cmd_desc *desc)
  618. {
  619. unsigned long vpt_addr, vconf_addr;
  620. u64 target;
  621. bool alloc;
  622. its_encode_cmd(cmd, GITS_CMD_VMAPP);
  623. its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id);
  624. its_encode_valid(cmd, desc->its_vmapp_cmd.valid);
  625. if (!desc->its_vmapp_cmd.valid) {
  626. if (is_v4_1(its)) {
  627. alloc = !atomic_dec_return(&desc->its_vmapp_cmd.vpe->vmapp_count);
  628. its_encode_alloc(cmd, alloc);
  629. }
  630. goto out;
  631. }
  632. vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page));
  633. target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset;
  634. its_encode_target(cmd, target);
  635. its_encode_vpt_addr(cmd, vpt_addr);
  636. its_encode_vpt_size(cmd, LPI_NRBITS - 1);
  637. if (!is_v4_1(its))
  638. goto out;
  639. vconf_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->its_vm->vprop_page));
  640. alloc = !atomic_fetch_inc(&desc->its_vmapp_cmd.vpe->vmapp_count);
  641. its_encode_alloc(cmd, alloc);
  642. /* We can only signal PTZ when alloc==1. Why do we have two bits? */
  643. its_encode_ptz(cmd, alloc);
  644. its_encode_vconf_addr(cmd, vconf_addr);
  645. its_encode_vmapp_default_db(cmd, desc->its_vmapp_cmd.vpe->vpe_db_lpi);
  646. out:
  647. its_fixup_cmd(cmd);
  648. return valid_vpe(its, desc->its_vmapp_cmd.vpe);
  649. }
  650. static struct its_vpe *its_build_vmapti_cmd(struct its_node *its,
  651. struct its_cmd_block *cmd,
  652. struct its_cmd_desc *desc)
  653. {
  654. u32 db;
  655. if (!is_v4_1(its) && desc->its_vmapti_cmd.db_enabled)
  656. db = desc->its_vmapti_cmd.vpe->vpe_db_lpi;
  657. else
  658. db = 1023;
  659. its_encode_cmd(cmd, GITS_CMD_VMAPTI);
  660. its_encode_devid(cmd, desc->its_vmapti_cmd.dev->device_id);
  661. its_encode_vpeid(cmd, desc->its_vmapti_cmd.vpe->vpe_id);
  662. its_encode_event_id(cmd, desc->its_vmapti_cmd.event_id);
  663. its_encode_db_phys_id(cmd, db);
  664. its_encode_virt_id(cmd, desc->its_vmapti_cmd.virt_id);
  665. its_fixup_cmd(cmd);
  666. return valid_vpe(its, desc->its_vmapti_cmd.vpe);
  667. }
  668. static struct its_vpe *its_build_vmovi_cmd(struct its_node *its,
  669. struct its_cmd_block *cmd,
  670. struct its_cmd_desc *desc)
  671. {
  672. u32 db;
  673. if (!is_v4_1(its) && desc->its_vmovi_cmd.db_enabled)
  674. db = desc->its_vmovi_cmd.vpe->vpe_db_lpi;
  675. else
  676. db = 1023;
  677. its_encode_cmd(cmd, GITS_CMD_VMOVI);
  678. its_encode_devid(cmd, desc->its_vmovi_cmd.dev->device_id);
  679. its_encode_vpeid(cmd, desc->its_vmovi_cmd.vpe->vpe_id);
  680. its_encode_event_id(cmd, desc->its_vmovi_cmd.event_id);
  681. its_encode_db_phys_id(cmd, db);
  682. its_encode_db_valid(cmd, true);
  683. its_fixup_cmd(cmd);
  684. return valid_vpe(its, desc->its_vmovi_cmd.vpe);
  685. }
  686. static struct its_vpe *its_build_vmovp_cmd(struct its_node *its,
  687. struct its_cmd_block *cmd,
  688. struct its_cmd_desc *desc)
  689. {
  690. u64 target;
  691. target = desc->its_vmovp_cmd.col->target_address + its->vlpi_redist_offset;
  692. its_encode_cmd(cmd, GITS_CMD_VMOVP);
  693. its_encode_seq_num(cmd, desc->its_vmovp_cmd.seq_num);
  694. its_encode_its_list(cmd, desc->its_vmovp_cmd.its_list);
  695. its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id);
  696. its_encode_target(cmd, target);
  697. if (is_v4_1(its)) {
  698. its_encode_db(cmd, true);
  699. its_encode_vmovp_default_db(cmd, desc->its_vmovp_cmd.vpe->vpe_db_lpi);
  700. }
  701. its_fixup_cmd(cmd);
  702. return valid_vpe(its, desc->its_vmovp_cmd.vpe);
  703. }
  704. static struct its_vpe *its_build_vinv_cmd(struct its_node *its,
  705. struct its_cmd_block *cmd,
  706. struct its_cmd_desc *desc)
  707. {
  708. struct its_vlpi_map *map;
  709. map = dev_event_to_vlpi_map(desc->its_inv_cmd.dev,
  710. desc->its_inv_cmd.event_id);
  711. its_encode_cmd(cmd, GITS_CMD_INV);
  712. its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);
  713. its_encode_event_id(cmd, desc->its_inv_cmd.event_id);
  714. its_fixup_cmd(cmd);
  715. return valid_vpe(its, map->vpe);
  716. }
  717. static struct its_vpe *its_build_vint_cmd(struct its_node *its,
  718. struct its_cmd_block *cmd,
  719. struct its_cmd_desc *desc)
  720. {
  721. struct its_vlpi_map *map;
  722. map = dev_event_to_vlpi_map(desc->its_int_cmd.dev,
  723. desc->its_int_cmd.event_id);
  724. its_encode_cmd(cmd, GITS_CMD_INT);
  725. its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);
  726. its_encode_event_id(cmd, desc->its_int_cmd.event_id);
  727. its_fixup_cmd(cmd);
  728. return valid_vpe(its, map->vpe);
  729. }
  730. static struct its_vpe *its_build_vclear_cmd(struct its_node *its,
  731. struct its_cmd_block *cmd,
  732. struct its_cmd_desc *desc)
  733. {
  734. struct its_vlpi_map *map;
  735. map = dev_event_to_vlpi_map(desc->its_clear_cmd.dev,
  736. desc->its_clear_cmd.event_id);
  737. its_encode_cmd(cmd, GITS_CMD_CLEAR);
  738. its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);
  739. its_encode_event_id(cmd, desc->its_clear_cmd.event_id);
  740. its_fixup_cmd(cmd);
  741. return valid_vpe(its, map->vpe);
  742. }
  743. static struct its_vpe *its_build_invdb_cmd(struct its_node *its,
  744. struct its_cmd_block *cmd,
  745. struct its_cmd_desc *desc)
  746. {
  747. if (WARN_ON(!is_v4_1(its)))
  748. return NULL;
  749. its_encode_cmd(cmd, GITS_CMD_INVDB);
  750. its_encode_vpeid(cmd, desc->its_invdb_cmd.vpe->vpe_id);
  751. its_fixup_cmd(cmd);
  752. return valid_vpe(its, desc->its_invdb_cmd.vpe);
  753. }
  754. static struct its_vpe *its_build_vsgi_cmd(struct its_node *its,
  755. struct its_cmd_block *cmd,
  756. struct its_cmd_desc *desc)
  757. {
  758. if (WARN_ON(!is_v4_1(its)))
  759. return NULL;
  760. its_encode_cmd(cmd, GITS_CMD_VSGI);
  761. its_encode_vpeid(cmd, desc->its_vsgi_cmd.vpe->vpe_id);
  762. its_encode_sgi_intid(cmd, desc->its_vsgi_cmd.sgi);
  763. its_encode_sgi_priority(cmd, desc->its_vsgi_cmd.priority);
  764. its_encode_sgi_group(cmd, desc->its_vsgi_cmd.group);
  765. its_encode_sgi_clear(cmd, desc->its_vsgi_cmd.clear);
  766. its_encode_sgi_enable(cmd, desc->its_vsgi_cmd.enable);
  767. its_fixup_cmd(cmd);
  768. return valid_vpe(its, desc->its_vsgi_cmd.vpe);
  769. }
  770. static u64 its_cmd_ptr_to_offset(struct its_node *its,
  771. struct its_cmd_block *ptr)
  772. {
  773. return (ptr - its->cmd_base) * sizeof(*ptr);
  774. }
  775. static int its_queue_full(struct its_node *its)
  776. {
  777. int widx;
  778. int ridx;
  779. widx = its->cmd_write - its->cmd_base;
  780. ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block);
  781. /* This is incredibly unlikely to happen, unless the ITS locks up. */
  782. if (((widx + 1) % ITS_CMD_QUEUE_NR_ENTRIES) == ridx)
  783. return 1;
  784. return 0;
  785. }
  786. static struct its_cmd_block *its_allocate_entry(struct its_node *its)
  787. {
  788. struct its_cmd_block *cmd;
  789. u32 count = 1000000; /* 1s! */
  790. while (its_queue_full(its)) {
  791. count--;
  792. if (!count) {
  793. pr_err_ratelimited("ITS queue not draining\n");
  794. return NULL;
  795. }
  796. cpu_relax();
  797. udelay(1);
  798. }
  799. cmd = its->cmd_write++;
  800. /* Handle queue wrapping */
  801. if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES))
  802. its->cmd_write = its->cmd_base;
  803. /* Clear command */
  804. cmd->raw_cmd[0] = 0;
  805. cmd->raw_cmd[1] = 0;
  806. cmd->raw_cmd[2] = 0;
  807. cmd->raw_cmd[3] = 0;
  808. return cmd;
  809. }
  810. static struct its_cmd_block *its_post_commands(struct its_node *its)
  811. {
  812. u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write);
  813. writel_relaxed(wr, its->base + GITS_CWRITER);
  814. return its->cmd_write;
  815. }
  816. static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd)
  817. {
  818. /*
  819. * Make sure the commands written to memory are observable by
  820. * the ITS.
  821. */
  822. if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING)
  823. gic_flush_dcache_to_poc(cmd, sizeof(*cmd));
  824. else
  825. dsb(ishst);
  826. }
  827. static int its_wait_for_range_completion(struct its_node *its,
  828. u64 prev_idx,
  829. struct its_cmd_block *to)
  830. {
  831. u64 rd_idx, to_idx, linear_idx;
  832. u32 count = 1000000; /* 1s! */
  833. /* Linearize to_idx if the command set has wrapped around */
  834. to_idx = its_cmd_ptr_to_offset(its, to);
  835. if (to_idx < prev_idx)
  836. to_idx += ITS_CMD_QUEUE_SZ;
  837. linear_idx = prev_idx;
  838. while (1) {
  839. s64 delta;
  840. rd_idx = readl_relaxed(its->base + GITS_CREADR);
  841. /*
  842. * Compute the read pointer progress, taking the
  843. * potential wrap-around into account.
  844. */
  845. delta = rd_idx - prev_idx;
  846. if (rd_idx < prev_idx)
  847. delta += ITS_CMD_QUEUE_SZ;
  848. linear_idx += delta;
  849. if (linear_idx >= to_idx)
  850. break;
  851. count--;
  852. if (!count) {
  853. pr_err_ratelimited("ITS queue timeout (%llu %llu)\n",
  854. to_idx, linear_idx);
  855. return -1;
  856. }
  857. prev_idx = rd_idx;
  858. cpu_relax();
  859. udelay(1);
  860. }
  861. return 0;
  862. }
  863. /* Warning, macro hell follows */
  864. #define BUILD_SINGLE_CMD_FUNC(name, buildtype, synctype, buildfn) \
  865. void name(struct its_node *its, \
  866. buildtype builder, \
  867. struct its_cmd_desc *desc) \
  868. { \
  869. struct its_cmd_block *cmd, *sync_cmd, *next_cmd; \
  870. synctype *sync_obj; \
  871. unsigned long flags; \
  872. u64 rd_idx; \
  873. \
  874. raw_spin_lock_irqsave(&its->lock, flags); \
  875. \
  876. cmd = its_allocate_entry(its); \
  877. if (!cmd) { /* We're soooooo screewed... */ \
  878. raw_spin_unlock_irqrestore(&its->lock, flags); \
  879. return; \
  880. } \
  881. sync_obj = builder(its, cmd, desc); \
  882. its_flush_cmd(its, cmd); \
  883. \
  884. if (sync_obj) { \
  885. sync_cmd = its_allocate_entry(its); \
  886. if (!sync_cmd) \
  887. goto post; \
  888. \
  889. buildfn(its, sync_cmd, sync_obj); \
  890. its_flush_cmd(its, sync_cmd); \
  891. } \
  892. \
  893. post: \
  894. rd_idx = readl_relaxed(its->base + GITS_CREADR); \
  895. next_cmd = its_post_commands(its); \
  896. raw_spin_unlock_irqrestore(&its->lock, flags); \
  897. \
  898. if (its_wait_for_range_completion(its, rd_idx, next_cmd)) \
  899. pr_err_ratelimited("ITS cmd %ps failed\n", builder); \
  900. }
  901. static void its_build_sync_cmd(struct its_node *its,
  902. struct its_cmd_block *sync_cmd,
  903. struct its_collection *sync_col)
  904. {
  905. its_encode_cmd(sync_cmd, GITS_CMD_SYNC);
  906. its_encode_target(sync_cmd, sync_col->target_address);
  907. its_fixup_cmd(sync_cmd);
  908. }
  909. static BUILD_SINGLE_CMD_FUNC(its_send_single_command, its_cmd_builder_t,
  910. struct its_collection, its_build_sync_cmd)
  911. static void its_build_vsync_cmd(struct its_node *its,
  912. struct its_cmd_block *sync_cmd,
  913. struct its_vpe *sync_vpe)
  914. {
  915. its_encode_cmd(sync_cmd, GITS_CMD_VSYNC);
  916. its_encode_vpeid(sync_cmd, sync_vpe->vpe_id);
  917. its_fixup_cmd(sync_cmd);
  918. }
  919. static BUILD_SINGLE_CMD_FUNC(its_send_single_vcommand, its_cmd_vbuilder_t,
  920. struct its_vpe, its_build_vsync_cmd)
  921. static void its_send_int(struct its_device *dev, u32 event_id)
  922. {
  923. struct its_cmd_desc desc;
  924. desc.its_int_cmd.dev = dev;
  925. desc.its_int_cmd.event_id = event_id;
  926. its_send_single_command(dev->its, its_build_int_cmd, &desc);
  927. }
  928. static void its_send_clear(struct its_device *dev, u32 event_id)
  929. {
  930. struct its_cmd_desc desc;
  931. desc.its_clear_cmd.dev = dev;
  932. desc.its_clear_cmd.event_id = event_id;
  933. its_send_single_command(dev->its, its_build_clear_cmd, &desc);
  934. }
  935. static void its_send_inv(struct its_device *dev, u32 event_id)
  936. {
  937. struct its_cmd_desc desc;
  938. desc.its_inv_cmd.dev = dev;
  939. desc.its_inv_cmd.event_id = event_id;
  940. its_send_single_command(dev->its, its_build_inv_cmd, &desc);
  941. }
  942. static void its_send_mapd(struct its_device *dev, int valid)
  943. {
  944. struct its_cmd_desc desc;
  945. desc.its_mapd_cmd.dev = dev;
  946. desc.its_mapd_cmd.valid = !!valid;
  947. its_send_single_command(dev->its, its_build_mapd_cmd, &desc);
  948. }
  949. static void its_send_mapc(struct its_node *its, struct its_collection *col,
  950. int valid)
  951. {
  952. struct its_cmd_desc desc;
  953. desc.its_mapc_cmd.col = col;
  954. desc.its_mapc_cmd.valid = !!valid;
  955. its_send_single_command(its, its_build_mapc_cmd, &desc);
  956. }
  957. static void its_send_mapti(struct its_device *dev, u32 irq_id, u32 id)
  958. {
  959. struct its_cmd_desc desc;
  960. desc.its_mapti_cmd.dev = dev;
  961. desc.its_mapti_cmd.phys_id = irq_id;
  962. desc.its_mapti_cmd.event_id = id;
  963. its_send_single_command(dev->its, its_build_mapti_cmd, &desc);
  964. }
  965. static void its_send_movi(struct its_device *dev,
  966. struct its_collection *col, u32 id)
  967. {
  968. struct its_cmd_desc desc;
  969. desc.its_movi_cmd.dev = dev;
  970. desc.its_movi_cmd.col = col;
  971. desc.its_movi_cmd.event_id = id;
  972. its_send_single_command(dev->its, its_build_movi_cmd, &desc);
  973. }
  974. static void its_send_discard(struct its_device *dev, u32 id)
  975. {
  976. struct its_cmd_desc desc;
  977. desc.its_discard_cmd.dev = dev;
  978. desc.its_discard_cmd.event_id = id;
  979. its_send_single_command(dev->its, its_build_discard_cmd, &desc);
  980. }
  981. static void its_send_invall(struct its_node *its, struct its_collection *col)
  982. {
  983. struct its_cmd_desc desc;
  984. desc.its_invall_cmd.col = col;
  985. its_send_single_command(its, its_build_invall_cmd, &desc);
  986. }
  987. static void its_send_vmapti(struct its_device *dev, u32 id)
  988. {
  989. struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id);
  990. struct its_cmd_desc desc;
  991. desc.its_vmapti_cmd.vpe = map->vpe;
  992. desc.its_vmapti_cmd.dev = dev;
  993. desc.its_vmapti_cmd.virt_id = map->vintid;
  994. desc.its_vmapti_cmd.event_id = id;
  995. desc.its_vmapti_cmd.db_enabled = map->db_enabled;
  996. its_send_single_vcommand(dev->its, its_build_vmapti_cmd, &desc);
  997. }
  998. static void its_send_vmovi(struct its_device *dev, u32 id)
  999. {
  1000. struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id);
  1001. struct its_cmd_desc desc;
  1002. desc.its_vmovi_cmd.vpe = map->vpe;
  1003. desc.its_vmovi_cmd.dev = dev;
  1004. desc.its_vmovi_cmd.event_id = id;
  1005. desc.its_vmovi_cmd.db_enabled = map->db_enabled;
  1006. its_send_single_vcommand(dev->its, its_build_vmovi_cmd, &desc);
  1007. }
  1008. static void its_send_vmapp(struct its_node *its,
  1009. struct its_vpe *vpe, bool valid)
  1010. {
  1011. struct its_cmd_desc desc;
  1012. desc.its_vmapp_cmd.vpe = vpe;
  1013. desc.its_vmapp_cmd.valid = valid;
  1014. desc.its_vmapp_cmd.col = &its->collections[vpe->col_idx];
  1015. its_send_single_vcommand(its, its_build_vmapp_cmd, &desc);
  1016. }
  1017. static void its_send_vmovp(struct its_vpe *vpe)
  1018. {
  1019. struct its_cmd_desc desc = {};
  1020. struct its_node *its;
  1021. unsigned long flags;
  1022. int col_id = vpe->col_idx;
  1023. desc.its_vmovp_cmd.vpe = vpe;
  1024. if (!its_list_map) {
  1025. its = list_first_entry(&its_nodes, struct its_node, entry);
  1026. desc.its_vmovp_cmd.col = &its->collections[col_id];
  1027. its_send_single_vcommand(its, its_build_vmovp_cmd, &desc);
  1028. return;
  1029. }
  1030. /*
  1031. * Yet another marvel of the architecture. If using the
  1032. * its_list "feature", we need to make sure that all ITSs
  1033. * receive all VMOVP commands in the same order. The only way
  1034. * to guarantee this is to make vmovp a serialization point.
  1035. *
  1036. * Wall <-- Head.
  1037. */
  1038. raw_spin_lock_irqsave(&vmovp_lock, flags);
  1039. desc.its_vmovp_cmd.seq_num = vmovp_seq_num++;
  1040. desc.its_vmovp_cmd.its_list = get_its_list(vpe->its_vm);
  1041. /* Emit VMOVPs */
  1042. list_for_each_entry(its, &its_nodes, entry) {
  1043. if (!is_v4(its))
  1044. continue;
  1045. if (!require_its_list_vmovp(vpe->its_vm, its))
  1046. continue;
  1047. desc.its_vmovp_cmd.col = &its->collections[col_id];
  1048. its_send_single_vcommand(its, its_build_vmovp_cmd, &desc);
  1049. }
  1050. raw_spin_unlock_irqrestore(&vmovp_lock, flags);
  1051. }
  1052. static void its_send_vinvall(struct its_node *its, struct its_vpe *vpe)
  1053. {
  1054. struct its_cmd_desc desc;
  1055. desc.its_vinvall_cmd.vpe = vpe;
  1056. its_send_single_vcommand(its, its_build_vinvall_cmd, &desc);
  1057. }
  1058. static void its_send_vinv(struct its_device *dev, u32 event_id)
  1059. {
  1060. struct its_cmd_desc desc;
  1061. /*
  1062. * There is no real VINV command. This is just a normal INV,
  1063. * with a VSYNC instead of a SYNC.
  1064. */
  1065. desc.its_inv_cmd.dev = dev;
  1066. desc.its_inv_cmd.event_id = event_id;
  1067. its_send_single_vcommand(dev->its, its_build_vinv_cmd, &desc);
  1068. }
  1069. static void its_send_vint(struct its_device *dev, u32 event_id)
  1070. {
  1071. struct its_cmd_desc desc;
  1072. /*
  1073. * There is no real VINT command. This is just a normal INT,
  1074. * with a VSYNC instead of a SYNC.
  1075. */
  1076. desc.its_int_cmd.dev = dev;
  1077. desc.its_int_cmd.event_id = event_id;
  1078. its_send_single_vcommand(dev->its, its_build_vint_cmd, &desc);
  1079. }
  1080. static void its_send_vclear(struct its_device *dev, u32 event_id)
  1081. {
  1082. struct its_cmd_desc desc;
  1083. /*
  1084. * There is no real VCLEAR command. This is just a normal CLEAR,
  1085. * with a VSYNC instead of a SYNC.
  1086. */
  1087. desc.its_clear_cmd.dev = dev;
  1088. desc.its_clear_cmd.event_id = event_id;
  1089. its_send_single_vcommand(dev->its, its_build_vclear_cmd, &desc);
  1090. }
  1091. static void its_send_invdb(struct its_node *its, struct its_vpe *vpe)
  1092. {
  1093. struct its_cmd_desc desc;
  1094. desc.its_invdb_cmd.vpe = vpe;
  1095. its_send_single_vcommand(its, its_build_invdb_cmd, &desc);
  1096. }
  1097. /*
  1098. * irqchip functions - assumes MSI, mostly.
  1099. */
  1100. static void lpi_write_config(struct irq_data *d, u8 clr, u8 set)
  1101. {
  1102. struct its_vlpi_map *map = get_vlpi_map(d);
  1103. irq_hw_number_t hwirq;
  1104. void *va;
  1105. u8 *cfg;
  1106. if (map) {
  1107. va = page_address(map->vm->vprop_page);
  1108. hwirq = map->vintid;
  1109. /* Remember the updated property */
  1110. map->properties &= ~clr;
  1111. map->properties |= set | LPI_PROP_GROUP1;
  1112. } else {
  1113. va = gic_rdists->prop_table_va;
  1114. hwirq = d->hwirq;
  1115. }
  1116. cfg = va + hwirq - 8192;
  1117. *cfg &= ~clr;
  1118. *cfg |= set | LPI_PROP_GROUP1;
  1119. /*
  1120. * Make the above write visible to the redistributors.
  1121. * And yes, we're flushing exactly: One. Single. Byte.
  1122. * Humpf...
  1123. */
  1124. if (gic_rdists->flags & RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING)
  1125. gic_flush_dcache_to_poc(cfg, sizeof(*cfg));
  1126. else
  1127. dsb(ishst);
  1128. }
  1129. static void wait_for_syncr(void __iomem *rdbase)
  1130. {
  1131. while (readl_relaxed(rdbase + GICR_SYNCR) & 1)
  1132. cpu_relax();
  1133. }
  1134. static void direct_lpi_inv(struct irq_data *d)
  1135. {
  1136. struct its_vlpi_map *map = get_vlpi_map(d);
  1137. void __iomem *rdbase;
  1138. unsigned long flags;
  1139. u64 val;
  1140. int cpu;
  1141. if (map) {
  1142. struct its_device *its_dev = irq_data_get_irq_chip_data(d);
  1143. WARN_ON(!is_v4_1(its_dev->its));
  1144. val = GICR_INVLPIR_V;
  1145. val |= FIELD_PREP(GICR_INVLPIR_VPEID, map->vpe->vpe_id);
  1146. val |= FIELD_PREP(GICR_INVLPIR_INTID, map->vintid);
  1147. } else {
  1148. val = d->hwirq;
  1149. }
  1150. /* Target the redistributor this LPI is currently routed to */
  1151. cpu = irq_to_cpuid_lock(d, &flags);
  1152. raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
  1153. rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base;
  1154. gic_write_lpir(val, rdbase + GICR_INVLPIR);
  1155. wait_for_syncr(rdbase);
  1156. raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
  1157. irq_to_cpuid_unlock(d, flags);
  1158. }
  1159. static void lpi_update_config(struct irq_data *d, u8 clr, u8 set)
  1160. {
  1161. struct its_device *its_dev = irq_data_get_irq_chip_data(d);
  1162. lpi_write_config(d, clr, set);
  1163. if (gic_rdists->has_direct_lpi &&
  1164. (is_v4_1(its_dev->its) || !irqd_is_forwarded_to_vcpu(d)))
  1165. direct_lpi_inv(d);
  1166. else if (!irqd_is_forwarded_to_vcpu(d))
  1167. its_send_inv(its_dev, its_get_event_id(d));
  1168. else
  1169. its_send_vinv(its_dev, its_get_event_id(d));
  1170. }
  1171. static void its_vlpi_set_doorbell(struct irq_data *d, bool enable)
  1172. {
  1173. struct its_device *its_dev = irq_data_get_irq_chip_data(d);
  1174. u32 event = its_get_event_id(d);
  1175. struct its_vlpi_map *map;
  1176. /*
  1177. * GICv4.1 does away with the per-LPI nonsense, nothing to do
  1178. * here.
  1179. */
  1180. if (is_v4_1(its_dev->its))
  1181. return;
  1182. map = dev_event_to_vlpi_map(its_dev, event);
  1183. if (map->db_enabled == enable)
  1184. return;
  1185. map->db_enabled = enable;
  1186. /*
  1187. * More fun with the architecture:
  1188. *
  1189. * Ideally, we'd issue a VMAPTI to set the doorbell to its LPI
  1190. * value or to 1023, depending on the enable bit. But that
  1191. * would be issueing a mapping for an /existing/ DevID+EventID
  1192. * pair, which is UNPREDICTABLE. Instead, let's issue a VMOVI
  1193. * to the /same/ vPE, using this opportunity to adjust the
  1194. * doorbell. Mouahahahaha. We loves it, Precious.
  1195. */
  1196. its_send_vmovi(its_dev, event);
  1197. }
  1198. static void its_mask_irq(struct irq_data *d)
  1199. {
  1200. if (irqd_is_forwarded_to_vcpu(d))
  1201. its_vlpi_set_doorbell(d, false);
  1202. lpi_update_config(d, LPI_PROP_ENABLED, 0);
  1203. }
  1204. static void its_unmask_irq(struct irq_data *d)
  1205. {
  1206. if (irqd_is_forwarded_to_vcpu(d))
  1207. its_vlpi_set_doorbell(d, true);
  1208. lpi_update_config(d, 0, LPI_PROP_ENABLED);
  1209. }
  1210. static __maybe_unused u32 its_read_lpi_count(struct irq_data *d, int cpu)
  1211. {
  1212. if (irqd_affinity_is_managed(d))
  1213. return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
  1214. return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
  1215. }
  1216. static void its_inc_lpi_count(struct irq_data *d, int cpu)
  1217. {
  1218. if (irqd_affinity_is_managed(d))
  1219. atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
  1220. else
  1221. atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
  1222. }
  1223. static void its_dec_lpi_count(struct irq_data *d, int cpu)
  1224. {
  1225. if (irqd_affinity_is_managed(d))
  1226. atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
  1227. else
  1228. atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
  1229. }
  1230. static unsigned int cpumask_pick_least_loaded(struct irq_data *d,
  1231. const struct cpumask *cpu_mask)
  1232. {
  1233. unsigned int cpu = nr_cpu_ids, tmp;
  1234. int count = S32_MAX;
  1235. for_each_cpu(tmp, cpu_mask) {
  1236. int this_count = its_read_lpi_count(d, tmp);
  1237. if (this_count < count) {
  1238. cpu = tmp;
  1239. count = this_count;
  1240. }
  1241. }
  1242. return cpu;
  1243. }
  1244. /*
  1245. * As suggested by Thomas Gleixner in:
  1246. * https://lore.kernel.org/r/87h80q2aoc.fsf@nanos.tec.linutronix.de
  1247. */
  1248. static int its_select_cpu(struct irq_data *d,
  1249. const struct cpumask *aff_mask)
  1250. {
  1251. struct its_device *its_dev = irq_data_get_irq_chip_data(d);
  1252. cpumask_var_t tmpmask;
  1253. int cpu, node;
  1254. if (!alloc_cpumask_var(&tmpmask, GFP_ATOMIC))
  1255. return -ENOMEM;
  1256. node = its_dev->its->numa_node;
  1257. if (!irqd_affinity_is_managed(d)) {
  1258. /* First try the NUMA node */
  1259. if (node != NUMA_NO_NODE) {
  1260. /*
  1261. * Try the intersection of the affinity mask and the
  1262. * node mask (and the online mask, just to be safe).
  1263. */
  1264. cpumask_and(tmpmask, cpumask_of_node(node), aff_mask);
  1265. cpumask_and(tmpmask, tmpmask, cpu_online_mask);
  1266. /*
  1267. * Ideally, we would check if the mask is empty, and
  1268. * try again on the full node here.
  1269. *
  1270. * But it turns out that the way ACPI describes the
  1271. * affinity for ITSs only deals about memory, and
  1272. * not target CPUs, so it cannot describe a single
  1273. * ITS placed next to two NUMA nodes.
  1274. *
  1275. * Instead, just fallback on the online mask. This
  1276. * diverges from Thomas' suggestion above.
  1277. */
  1278. cpu = cpumask_pick_least_loaded(d, tmpmask);
  1279. if (cpu < nr_cpu_ids)
  1280. goto out;
  1281. /* If we can't cross sockets, give up */
  1282. if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144))
  1283. goto out;
  1284. /* If the above failed, expand the search */
  1285. }
  1286. /* Try the intersection of the affinity and online masks */
  1287. cpumask_and(tmpmask, aff_mask, cpu_online_mask);
  1288. /* If that doesn't fly, the online mask is the last resort */
  1289. if (cpumask_empty(tmpmask))
  1290. cpumask_copy(tmpmask, cpu_online_mask);
  1291. cpu = cpumask_pick_least_loaded(d, tmpmask);
  1292. } else {
  1293. cpumask_and(tmpmask, irq_data_get_affinity_mask(d), cpu_online_mask);
  1294. /* If we cannot cross sockets, limit the search to that node */
  1295. if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) &&
  1296. node != NUMA_NO_NODE)
  1297. cpumask_and(tmpmask, tmpmask, cpumask_of_node(node));
  1298. cpu = cpumask_pick_least_loaded(d, tmpmask);
  1299. }
  1300. out:
  1301. free_cpumask_var(tmpmask);
  1302. pr_debug("IRQ%d -> %*pbl CPU%d\n", d->irq, cpumask_pr_args(aff_mask), cpu);
  1303. return cpu;
  1304. }
  1305. static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
  1306. bool force)
  1307. {
  1308. struct its_device *its_dev = irq_data_get_irq_chip_data(d);
  1309. struct its_collection *target_col;
  1310. u32 id = its_get_event_id(d);
  1311. int cpu, prev_cpu;
  1312. /* A forwarded interrupt should use irq_set_vcpu_affinity */
  1313. if (irqd_is_forwarded_to_vcpu(d))
  1314. return -EINVAL;
  1315. prev_cpu = its_dev->event_map.col_map[id];
  1316. its_dec_lpi_count(d, prev_cpu);
  1317. if (!force)
  1318. cpu = its_select_cpu(d, mask_val);
  1319. else
  1320. cpu = cpumask_pick_least_loaded(d, mask_val);
  1321. if (cpu < 0 || cpu >= nr_cpu_ids)
  1322. goto err;
  1323. /* don't set the affinity when the target cpu is same as current one */
  1324. if (cpu != prev_cpu) {
  1325. target_col = &its_dev->its->collections[cpu];
  1326. its_send_movi(its_dev, target_col, id);
  1327. its_dev->event_map.col_map[id] = cpu;
  1328. irq_data_update_effective_affinity(d, cpumask_of(cpu));
  1329. }
  1330. its_inc_lpi_count(d, cpu);
  1331. return IRQ_SET_MASK_OK_DONE;
  1332. err:
  1333. its_inc_lpi_count(d, prev_cpu);
  1334. return -EINVAL;
  1335. }
  1336. static u64 its_irq_get_msi_base(struct its_device *its_dev)
  1337. {
  1338. struct its_node *its = its_dev->its;
  1339. return its->phys_base + GITS_TRANSLATER;
  1340. }
  1341. static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg)
  1342. {
  1343. struct its_device *its_dev = irq_data_get_irq_chip_data(d);
  1344. struct its_node *its;
  1345. u64 addr;
  1346. its = its_dev->its;
  1347. addr = its->get_msi_base(its_dev);
  1348. msg->address_lo = lower_32_bits(addr);
  1349. msg->address_hi = upper_32_bits(addr);
  1350. msg->data = its_get_event_id(d);
  1351. iommu_dma_compose_msi_msg(irq_data_get_msi_desc(d), msg);
  1352. }
  1353. static int its_irq_set_irqchip_state(struct irq_data *d,
  1354. enum irqchip_irq_state which,
  1355. bool state)
  1356. {
  1357. struct its_device *its_dev = irq_data_get_irq_chip_data(d);
  1358. u32 event = its_get_event_id(d);
  1359. if (which != IRQCHIP_STATE_PENDING)
  1360. return -EINVAL;
  1361. if (irqd_is_forwarded_to_vcpu(d)) {
  1362. if (state)
  1363. its_send_vint(its_dev, event);
  1364. else
  1365. its_send_vclear(its_dev, event);
  1366. } else {
  1367. if (state)
  1368. its_send_int(its_dev, event);
  1369. else
  1370. its_send_clear(its_dev, event);
  1371. }
  1372. return 0;
  1373. }
  1374. static int its_irq_retrigger(struct irq_data *d)
  1375. {
  1376. return !its_irq_set_irqchip_state(d, IRQCHIP_STATE_PENDING, true);
  1377. }
  1378. /*
  1379. * Two favourable cases:
  1380. *
  1381. * (a) Either we have a GICv4.1, and all vPEs have to be mapped at all times
  1382. * for vSGI delivery
  1383. *
  1384. * (b) Or the ITSs do not use a list map, meaning that VMOVP is cheap enough
  1385. * and we're better off mapping all VPEs always
  1386. *
  1387. * If neither (a) nor (b) is true, then we map vPEs on demand.
  1388. *
  1389. */
  1390. static bool gic_requires_eager_mapping(void)
  1391. {
  1392. if (!its_list_map || gic_rdists->has_rvpeid)
  1393. return true;
  1394. return false;
  1395. }
  1396. static void its_map_vm(struct its_node *its, struct its_vm *vm)
  1397. {
  1398. unsigned long flags;
  1399. if (gic_requires_eager_mapping())
  1400. return;
  1401. raw_spin_lock_irqsave(&vmovp_lock, flags);
  1402. /*
  1403. * If the VM wasn't mapped yet, iterate over the vpes and get
  1404. * them mapped now.
  1405. */
  1406. vm->vlpi_count[its->list_nr]++;
  1407. if (vm->vlpi_count[its->list_nr] == 1) {
  1408. int i;
  1409. for (i = 0; i < vm->nr_vpes; i++) {
  1410. struct its_vpe *vpe = vm->vpes[i];
  1411. struct irq_data *d = irq_get_irq_data(vpe->irq);
  1412. /* Map the VPE to the first possible CPU */
  1413. vpe->col_idx = cpumask_first(cpu_online_mask);
  1414. its_send_vmapp(its, vpe, true);
  1415. its_send_vinvall(its, vpe);
  1416. irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx));
  1417. }
  1418. }
  1419. raw_spin_unlock_irqrestore(&vmovp_lock, flags);
  1420. }
  1421. static void its_unmap_vm(struct its_node *its, struct its_vm *vm)
  1422. {
  1423. unsigned long flags;
  1424. /* Not using the ITS list? Everything is always mapped. */
  1425. if (gic_requires_eager_mapping())
  1426. return;
  1427. raw_spin_lock_irqsave(&vmovp_lock, flags);
  1428. if (!--vm->vlpi_count[its->list_nr]) {
  1429. int i;
  1430. for (i = 0; i < vm->nr_vpes; i++)
  1431. its_send_vmapp(its, vm->vpes[i], false);
  1432. }
  1433. raw_spin_unlock_irqrestore(&vmovp_lock, flags);
  1434. }
  1435. static int its_vlpi_map(struct irq_data *d, struct its_cmd_info *info)
  1436. {
  1437. struct its_device *its_dev = irq_data_get_irq_chip_data(d);
  1438. u32 event = its_get_event_id(d);
  1439. int ret = 0;
  1440. if (!info->map)
  1441. return -EINVAL;
  1442. raw_spin_lock(&its_dev->event_map.vlpi_lock);
  1443. if (!its_dev->event_map.vm) {
  1444. struct its_vlpi_map *maps;
  1445. maps = kcalloc(its_dev->event_map.nr_lpis, sizeof(*maps),
  1446. GFP_ATOMIC);
  1447. if (!maps) {
  1448. ret = -ENOMEM;
  1449. goto out;
  1450. }
  1451. its_dev->event_map.vm = info->map->vm;
  1452. its_dev->event_map.vlpi_maps = maps;
  1453. } else if (its_dev->event_map.vm != info->map->vm) {
  1454. ret = -EINVAL;
  1455. goto out;
  1456. }
  1457. /* Get our private copy of the mapping information */
  1458. its_dev->event_map.vlpi_maps[event] = *info->map;
  1459. if (irqd_is_forwarded_to_vcpu(d)) {
  1460. /* Already mapped, move it around */
  1461. its_send_vmovi(its_dev, event);
  1462. } else {
  1463. /* Ensure all the VPEs are mapped on this ITS */
  1464. its_map_vm(its_dev->its, info->map->vm);
  1465. /*
  1466. * Flag the interrupt as forwarded so that we can
  1467. * start poking the virtual property table.
  1468. */
  1469. irqd_set_forwarded_to_vcpu(d);
  1470. /* Write out the property to the prop table */
  1471. lpi_write_config(d, 0xff, info->map->properties);
  1472. /* Drop the physical mapping */
  1473. its_send_discard(its_dev, event);
  1474. /* and install the virtual one */
  1475. its_send_vmapti(its_dev, event);
  1476. /* Increment the number of VLPIs */
  1477. its_dev->event_map.nr_vlpis++;
  1478. }
  1479. out:
  1480. raw_spin_unlock(&its_dev->event_map.vlpi_lock);
  1481. return ret;
  1482. }
  1483. static int its_vlpi_get(struct irq_data *d, struct its_cmd_info *info)
  1484. {
  1485. struct its_device *its_dev = irq_data_get_irq_chip_data(d);
  1486. struct its_vlpi_map *map;
  1487. int ret = 0;
  1488. raw_spin_lock(&its_dev->event_map.vlpi_lock);
  1489. map = get_vlpi_map(d);
  1490. if (!its_dev->event_map.vm || !map) {
  1491. ret = -EINVAL;
  1492. goto out;
  1493. }
  1494. /* Copy our mapping information to the incoming request */
  1495. *info->map = *map;
  1496. out:
  1497. raw_spin_unlock(&its_dev->event_map.vlpi_lock);
  1498. return ret;
  1499. }
  1500. static int its_vlpi_unmap(struct irq_data *d)
  1501. {
  1502. struct its_device *its_dev = irq_data_get_irq_chip_data(d);
  1503. u32 event = its_get_event_id(d);
  1504. int ret = 0;
  1505. raw_spin_lock(&its_dev->event_map.vlpi_lock);
  1506. if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d)) {
  1507. ret = -EINVAL;
  1508. goto out;
  1509. }
  1510. /* Drop the virtual mapping */
  1511. its_send_discard(its_dev, event);
  1512. /* and restore the physical one */
  1513. irqd_clr_forwarded_to_vcpu(d);
  1514. its_send_mapti(its_dev, d->hwirq, event);
  1515. lpi_update_config(d, 0xff, (LPI_PROP_DEFAULT_PRIO |
  1516. LPI_PROP_ENABLED |
  1517. LPI_PROP_GROUP1));
  1518. /* Potentially unmap the VM from this ITS */
  1519. its_unmap_vm(its_dev->its, its_dev->event_map.vm);
  1520. /*
  1521. * Drop the refcount and make the device available again if
  1522. * this was the last VLPI.
  1523. */
  1524. if (!--its_dev->event_map.nr_vlpis) {
  1525. its_dev->event_map.vm = NULL;
  1526. kfree(its_dev->event_map.vlpi_maps);
  1527. }
  1528. out:
  1529. raw_spin_unlock(&its_dev->event_map.vlpi_lock);
  1530. return ret;
  1531. }
  1532. static int its_vlpi_prop_update(struct irq_data *d, struct its_cmd_info *info)
  1533. {
  1534. struct its_device *its_dev = irq_data_get_irq_chip_data(d);
  1535. if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d))
  1536. return -EINVAL;
  1537. if (info->cmd_type == PROP_UPDATE_AND_INV_VLPI)
  1538. lpi_update_config(d, 0xff, info->config);
  1539. else
  1540. lpi_write_config(d, 0xff, info->config);
  1541. its_vlpi_set_doorbell(d, !!(info->config & LPI_PROP_ENABLED));
  1542. return 0;
  1543. }
  1544. static int its_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
  1545. {
  1546. struct its_device *its_dev = irq_data_get_irq_chip_data(d);
  1547. struct its_cmd_info *info = vcpu_info;
  1548. /* Need a v4 ITS */
  1549. if (!is_v4(its_dev->its))
  1550. return -EINVAL;
  1551. /* Unmap request? */
  1552. if (!info)
  1553. return its_vlpi_unmap(d);
  1554. switch (info->cmd_type) {
  1555. case MAP_VLPI:
  1556. return its_vlpi_map(d, info);
  1557. case GET_VLPI:
  1558. return its_vlpi_get(d, info);
  1559. case PROP_UPDATE_VLPI:
  1560. case PROP_UPDATE_AND_INV_VLPI:
  1561. return its_vlpi_prop_update(d, info);
  1562. default:
  1563. return -EINVAL;
  1564. }
  1565. }
  1566. static struct irq_chip its_irq_chip = {
  1567. .name = "ITS",
  1568. .irq_mask = its_mask_irq,
  1569. .irq_unmask = its_unmask_irq,
  1570. .irq_eoi = irq_chip_eoi_parent,
  1571. .irq_set_affinity = its_set_affinity,
  1572. .irq_compose_msi_msg = its_irq_compose_msi_msg,
  1573. .irq_set_irqchip_state = its_irq_set_irqchip_state,
  1574. .irq_retrigger = its_irq_retrigger,
  1575. .irq_set_vcpu_affinity = its_irq_set_vcpu_affinity,
  1576. };
  1577. /*
  1578. * How we allocate LPIs:
  1579. *
  1580. * lpi_range_list contains ranges of LPIs that are to available to
  1581. * allocate from. To allocate LPIs, just pick the first range that
  1582. * fits the required allocation, and reduce it by the required
  1583. * amount. Once empty, remove the range from the list.
  1584. *
  1585. * To free a range of LPIs, add a free range to the list, sort it and
  1586. * merge the result if the new range happens to be adjacent to an
  1587. * already free block.
  1588. *
  1589. * The consequence of the above is that allocation is cost is low, but
  1590. * freeing is expensive. We assumes that freeing rarely occurs.
  1591. */
  1592. #define ITS_MAX_LPI_NRBITS 16 /* 64K LPIs */
  1593. static DEFINE_MUTEX(lpi_range_lock);
  1594. static LIST_HEAD(lpi_range_list);
  1595. struct lpi_range {
  1596. struct list_head entry;
  1597. u32 base_id;
  1598. u32 span;
  1599. };
  1600. static struct lpi_range *mk_lpi_range(u32 base, u32 span)
  1601. {
  1602. struct lpi_range *range;
  1603. range = kmalloc(sizeof(*range), GFP_KERNEL);
  1604. if (range) {
  1605. range->base_id = base;
  1606. range->span = span;
  1607. }
  1608. return range;
  1609. }
  1610. static int alloc_lpi_range(u32 nr_lpis, u32 *base)
  1611. {
  1612. struct lpi_range *range, *tmp;
  1613. int err = -ENOSPC;
  1614. mutex_lock(&lpi_range_lock);
  1615. list_for_each_entry_safe(range, tmp, &lpi_range_list, entry) {
  1616. if (range->span >= nr_lpis) {
  1617. *base = range->base_id;
  1618. range->base_id += nr_lpis;
  1619. range->span -= nr_lpis;
  1620. if (range->span == 0) {
  1621. list_del(&range->entry);
  1622. kfree(range);
  1623. }
  1624. err = 0;
  1625. break;
  1626. }
  1627. }
  1628. mutex_unlock(&lpi_range_lock);
  1629. pr_debug("ITS: alloc %u:%u\n", *base, nr_lpis);
  1630. return err;
  1631. }
  1632. static void merge_lpi_ranges(struct lpi_range *a, struct lpi_range *b)
  1633. {
  1634. if (&a->entry == &lpi_range_list || &b->entry == &lpi_range_list)
  1635. return;
  1636. if (a->base_id + a->span != b->base_id)
  1637. return;
  1638. b->base_id = a->base_id;
  1639. b->span += a->span;
  1640. list_del(&a->entry);
  1641. kfree(a);
  1642. }
  1643. static int free_lpi_range(u32 base, u32 nr_lpis)
  1644. {
  1645. struct lpi_range *new, *old;
  1646. new = mk_lpi_range(base, nr_lpis);
  1647. if (!new)
  1648. return -ENOMEM;
  1649. mutex_lock(&lpi_range_lock);
  1650. list_for_each_entry_reverse(old, &lpi_range_list, entry) {
  1651. if (old->base_id < base)
  1652. break;
  1653. }
  1654. /*
  1655. * old is the last element with ->base_id smaller than base,
  1656. * so new goes right after it. If there are no elements with
  1657. * ->base_id smaller than base, &old->entry ends up pointing
  1658. * at the head of the list, and inserting new it the start of
  1659. * the list is the right thing to do in that case as well.
  1660. */
  1661. list_add(&new->entry, &old->entry);
  1662. /*
  1663. * Now check if we can merge with the preceding and/or
  1664. * following ranges.
  1665. */
  1666. merge_lpi_ranges(old, new);
  1667. merge_lpi_ranges(new, list_next_entry(new, entry));
  1668. mutex_unlock(&lpi_range_lock);
  1669. return 0;
  1670. }
  1671. static int __init its_lpi_init(u32 id_bits)
  1672. {
  1673. u32 lpis = (1UL << id_bits) - 8192;
  1674. u32 numlpis;
  1675. int err;
  1676. numlpis = 1UL << GICD_TYPER_NUM_LPIS(gic_rdists->gicd_typer);
  1677. if (numlpis > 2 && !WARN_ON(numlpis > lpis)) {
  1678. lpis = numlpis;
  1679. pr_info("ITS: Using hypervisor restricted LPI range [%u]\n",
  1680. lpis);
  1681. }
  1682. /*
  1683. * Initializing the allocator is just the same as freeing the
  1684. * full range of LPIs.
  1685. */
  1686. err = free_lpi_range(8192, lpis);
  1687. pr_debug("ITS: Allocator initialized for %u LPIs\n", lpis);
  1688. return err;
  1689. }
  1690. static unsigned long *its_lpi_alloc(int nr_irqs, u32 *base, int *nr_ids)
  1691. {
  1692. unsigned long *bitmap = NULL;
  1693. int err = 0;
  1694. do {
  1695. err = alloc_lpi_range(nr_irqs, base);
  1696. if (!err)
  1697. break;
  1698. nr_irqs /= 2;
  1699. } while (nr_irqs > 0);
  1700. if (!nr_irqs)
  1701. err = -ENOSPC;
  1702. if (err)
  1703. goto out;
  1704. bitmap = kcalloc(BITS_TO_LONGS(nr_irqs), sizeof (long), GFP_ATOMIC);
  1705. if (!bitmap)
  1706. goto out;
  1707. *nr_ids = nr_irqs;
  1708. out:
  1709. if (!bitmap)
  1710. *base = *nr_ids = 0;
  1711. return bitmap;
  1712. }
  1713. static void its_lpi_free(unsigned long *bitmap, u32 base, u32 nr_ids)
  1714. {
  1715. WARN_ON(free_lpi_range(base, nr_ids));
  1716. kfree(bitmap);
  1717. }
  1718. static void gic_reset_prop_table(void *va)
  1719. {
  1720. /* Priority 0xa0, Group-1, disabled */
  1721. memset(va, LPI_PROP_DEFAULT_PRIO | LPI_PROP_GROUP1, LPI_PROPBASE_SZ);
  1722. /* Make sure the GIC will observe the written configuration */
  1723. gic_flush_dcache_to_poc(va, LPI_PROPBASE_SZ);
  1724. }
  1725. static struct page *its_allocate_prop_table(gfp_t gfp_flags)
  1726. {
  1727. struct page *prop_page;
  1728. prop_page = alloc_pages(gfp_flags, get_order(LPI_PROPBASE_SZ));
  1729. if (!prop_page)
  1730. return NULL;
  1731. gic_reset_prop_table(page_address(prop_page));
  1732. return prop_page;
  1733. }
  1734. static void its_free_prop_table(struct page *prop_page)
  1735. {
  1736. free_pages((unsigned long)page_address(prop_page),
  1737. get_order(LPI_PROPBASE_SZ));
  1738. }
  1739. static bool gic_check_reserved_range(phys_addr_t addr, unsigned long size)
  1740. {
  1741. phys_addr_t start, end, addr_end;
  1742. u64 i;
  1743. /*
  1744. * We don't bother checking for a kdump kernel as by
  1745. * construction, the LPI tables are out of this kernel's
  1746. * memory map.
  1747. */
  1748. if (is_kdump_kernel())
  1749. return true;
  1750. addr_end = addr + size - 1;
  1751. for_each_reserved_mem_range(i, &start, &end) {
  1752. if (addr >= start && addr_end <= end)
  1753. return true;
  1754. }
  1755. /* Not found, not a good sign... */
  1756. pr_warn("GICv3: Expected reserved range [%pa:%pa], not found\n",
  1757. &addr, &addr_end);
  1758. add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
  1759. return false;
  1760. }
  1761. static int gic_reserve_range(phys_addr_t addr, unsigned long size)
  1762. {
  1763. if (efi_enabled(EFI_CONFIG_TABLES))
  1764. return efi_mem_reserve_persistent(addr, size);
  1765. return 0;
  1766. }
  1767. static int __init its_setup_lpi_prop_table(void)
  1768. {
  1769. if (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) {
  1770. u64 val;
  1771. val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER);
  1772. lpi_id_bits = (val & GICR_PROPBASER_IDBITS_MASK) + 1;
  1773. gic_rdists->prop_table_pa = val & GENMASK_ULL(51, 12);
  1774. gic_rdists->prop_table_va = memremap(gic_rdists->prop_table_pa,
  1775. LPI_PROPBASE_SZ,
  1776. MEMREMAP_WB);
  1777. gic_reset_prop_table(gic_rdists->prop_table_va);
  1778. } else {
  1779. struct page *page;
  1780. lpi_id_bits = min_t(u32,
  1781. GICD_TYPER_ID_BITS(gic_rdists->gicd_typer),
  1782. ITS_MAX_LPI_NRBITS);
  1783. page = its_allocate_prop_table(GFP_NOWAIT);
  1784. if (!page) {
  1785. pr_err("Failed to allocate PROPBASE\n");
  1786. return -ENOMEM;
  1787. }
  1788. gic_rdists->prop_table_pa = page_to_phys(page);
  1789. gic_rdists->prop_table_va = page_address(page);
  1790. WARN_ON(gic_reserve_range(gic_rdists->prop_table_pa,
  1791. LPI_PROPBASE_SZ));
  1792. }
  1793. pr_info("GICv3: using LPI property table @%pa\n",
  1794. &gic_rdists->prop_table_pa);
  1795. return its_lpi_init(lpi_id_bits);
  1796. }
  1797. static const char *its_base_type_string[] = {
  1798. [GITS_BASER_TYPE_DEVICE] = "Devices",
  1799. [GITS_BASER_TYPE_VCPU] = "Virtual CPUs",
  1800. [GITS_BASER_TYPE_RESERVED3] = "Reserved (3)",
  1801. [GITS_BASER_TYPE_COLLECTION] = "Interrupt Collections",
  1802. [GITS_BASER_TYPE_RESERVED5] = "Reserved (5)",
  1803. [GITS_BASER_TYPE_RESERVED6] = "Reserved (6)",
  1804. [GITS_BASER_TYPE_RESERVED7] = "Reserved (7)",
  1805. };
  1806. static u64 its_read_baser(struct its_node *its, struct its_baser *baser)
  1807. {
  1808. u32 idx = baser - its->tables;
  1809. return gits_read_baser(its->base + GITS_BASER + (idx << 3));
  1810. }
  1811. static void its_write_baser(struct its_node *its, struct its_baser *baser,
  1812. u64 val)
  1813. {
  1814. u32 idx = baser - its->tables;
  1815. gits_write_baser(val, its->base + GITS_BASER + (idx << 3));
  1816. baser->val = its_read_baser(its, baser);
  1817. }
  1818. static int its_setup_baser(struct its_node *its, struct its_baser *baser,
  1819. u64 cache, u64 shr, u32 order, bool indirect)
  1820. {
  1821. u64 val = its_read_baser(its, baser);
  1822. u64 esz = GITS_BASER_ENTRY_SIZE(val);
  1823. u64 type = GITS_BASER_TYPE(val);
  1824. u64 baser_phys, tmp;
  1825. u32 alloc_pages, psz;
  1826. struct page *page;
  1827. void *base;
  1828. psz = baser->psz;
  1829. alloc_pages = (PAGE_ORDER_TO_SIZE(order) / psz);
  1830. if (alloc_pages > GITS_BASER_PAGES_MAX) {
  1831. pr_warn("ITS@%pa: %s too large, reduce ITS pages %u->%u\n",
  1832. &its->phys_base, its_base_type_string[type],
  1833. alloc_pages, GITS_BASER_PAGES_MAX);
  1834. alloc_pages = GITS_BASER_PAGES_MAX;
  1835. order = get_order(GITS_BASER_PAGES_MAX * psz);
  1836. }
  1837. page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, order);
  1838. if (!page)
  1839. return -ENOMEM;
  1840. base = (void *)page_address(page);
  1841. baser_phys = virt_to_phys(base);
  1842. /* Check if the physical address of the memory is above 48bits */
  1843. if (IS_ENABLED(CONFIG_ARM64_64K_PAGES) && (baser_phys >> 48)) {
  1844. /* 52bit PA is supported only when PageSize=64K */
  1845. if (psz != SZ_64K) {
  1846. pr_err("ITS: no 52bit PA support when psz=%d\n", psz);
  1847. free_pages((unsigned long)base, order);
  1848. return -ENXIO;
  1849. }
  1850. /* Convert 52bit PA to 48bit field */
  1851. baser_phys = GITS_BASER_PHYS_52_to_48(baser_phys);
  1852. }
  1853. retry_baser:
  1854. val = (baser_phys |
  1855. (type << GITS_BASER_TYPE_SHIFT) |
  1856. ((esz - 1) << GITS_BASER_ENTRY_SIZE_SHIFT) |
  1857. ((alloc_pages - 1) << GITS_BASER_PAGES_SHIFT) |
  1858. cache |
  1859. shr |
  1860. GITS_BASER_VALID);
  1861. val |= indirect ? GITS_BASER_INDIRECT : 0x0;
  1862. switch (psz) {
  1863. case SZ_4K:
  1864. val |= GITS_BASER_PAGE_SIZE_4K;
  1865. break;
  1866. case SZ_16K:
  1867. val |= GITS_BASER_PAGE_SIZE_16K;
  1868. break;
  1869. case SZ_64K:
  1870. val |= GITS_BASER_PAGE_SIZE_64K;
  1871. break;
  1872. }
  1873. its_write_baser(its, baser, val);
  1874. tmp = baser->val;
  1875. if ((val ^ tmp) & GITS_BASER_SHAREABILITY_MASK) {
  1876. /*
  1877. * Shareability didn't stick. Just use
  1878. * whatever the read reported, which is likely
  1879. * to be the only thing this redistributor
  1880. * supports. If that's zero, make it
  1881. * non-cacheable as well.
  1882. */
  1883. shr = tmp & GITS_BASER_SHAREABILITY_MASK;
  1884. if (!shr) {
  1885. cache = GITS_BASER_nC;
  1886. gic_flush_dcache_to_poc(base, PAGE_ORDER_TO_SIZE(order));
  1887. }
  1888. goto retry_baser;
  1889. }
  1890. if (val != tmp) {
  1891. pr_err("ITS@%pa: %s doesn't stick: %llx %llx\n",
  1892. &its->phys_base, its_base_type_string[type],
  1893. val, tmp);
  1894. free_pages((unsigned long)base, order);
  1895. return -ENXIO;
  1896. }
  1897. baser->order = order;
  1898. baser->base = base;
  1899. baser->psz = psz;
  1900. tmp = indirect ? GITS_LVL1_ENTRY_SIZE : esz;
  1901. pr_info("ITS@%pa: allocated %d %s @%lx (%s, esz %d, psz %dK, shr %d)\n",
  1902. &its->phys_base, (int)(PAGE_ORDER_TO_SIZE(order) / (int)tmp),
  1903. its_base_type_string[type],
  1904. (unsigned long)virt_to_phys(base),
  1905. indirect ? "indirect" : "flat", (int)esz,
  1906. psz / SZ_1K, (int)shr >> GITS_BASER_SHAREABILITY_SHIFT);
  1907. return 0;
  1908. }
  1909. static bool its_parse_indirect_baser(struct its_node *its,
  1910. struct its_baser *baser,
  1911. u32 *order, u32 ids)
  1912. {
  1913. u64 tmp = its_read_baser(its, baser);
  1914. u64 type = GITS_BASER_TYPE(tmp);
  1915. u64 esz = GITS_BASER_ENTRY_SIZE(tmp);
  1916. u64 val = GITS_BASER_InnerShareable | GITS_BASER_RaWaWb;
  1917. u32 new_order = *order;
  1918. u32 psz = baser->psz;
  1919. bool indirect = false;
  1920. /* No need to enable Indirection if memory requirement < (psz*2)bytes */
  1921. if ((esz << ids) > (psz * 2)) {
  1922. /*
  1923. * Find out whether hw supports a single or two-level table by
  1924. * table by reading bit at offset '62' after writing '1' to it.
  1925. */
  1926. its_write_baser(its, baser, val | GITS_BASER_INDIRECT);
  1927. indirect = !!(baser->val & GITS_BASER_INDIRECT);
  1928. if (indirect) {
  1929. /*
  1930. * The size of the lvl2 table is equal to ITS page size
  1931. * which is 'psz'. For computing lvl1 table size,
  1932. * subtract ID bits that sparse lvl2 table from 'ids'
  1933. * which is reported by ITS hardware times lvl1 table
  1934. * entry size.
  1935. */
  1936. ids -= ilog2(psz / (int)esz);
  1937. esz = GITS_LVL1_ENTRY_SIZE;
  1938. }
  1939. }
  1940. /*
  1941. * Allocate as many entries as required to fit the
  1942. * range of device IDs that the ITS can grok... The ID
  1943. * space being incredibly sparse, this results in a
  1944. * massive waste of memory if two-level device table
  1945. * feature is not supported by hardware.
  1946. */
  1947. new_order = max_t(u32, get_order(esz << ids), new_order);
  1948. if (new_order >= MAX_ORDER) {
  1949. new_order = MAX_ORDER - 1;
  1950. ids = ilog2(PAGE_ORDER_TO_SIZE(new_order) / (int)esz);
  1951. pr_warn("ITS@%pa: %s Table too large, reduce ids %llu->%u\n",
  1952. &its->phys_base, its_base_type_string[type],
  1953. device_ids(its), ids);
  1954. }
  1955. *order = new_order;
  1956. return indirect;
  1957. }
  1958. static u32 compute_common_aff(u64 val)
  1959. {
  1960. u32 aff, clpiaff;
  1961. aff = FIELD_GET(GICR_TYPER_AFFINITY, val);
  1962. clpiaff = FIELD_GET(GICR_TYPER_COMMON_LPI_AFF, val);
  1963. return aff & ~(GENMASK(31, 0) >> (clpiaff * 8));
  1964. }
  1965. static u32 compute_its_aff(struct its_node *its)
  1966. {
  1967. u64 val;
  1968. u32 svpet;
  1969. /*
  1970. * Reencode the ITS SVPET and MPIDR as a GICR_TYPER, and compute
  1971. * the resulting affinity. We then use that to see if this match
  1972. * our own affinity.
  1973. */
  1974. svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer);
  1975. val = FIELD_PREP(GICR_TYPER_COMMON_LPI_AFF, svpet);
  1976. val |= FIELD_PREP(GICR_TYPER_AFFINITY, its->mpidr);
  1977. return compute_common_aff(val);
  1978. }
  1979. static struct its_node *find_sibling_its(struct its_node *cur_its)
  1980. {
  1981. struct its_node *its;
  1982. u32 aff;
  1983. if (!FIELD_GET(GITS_TYPER_SVPET, cur_its->typer))
  1984. return NULL;
  1985. aff = compute_its_aff(cur_its);
  1986. list_for_each_entry(its, &its_nodes, entry) {
  1987. u64 baser;
  1988. if (!is_v4_1(its) || its == cur_its)
  1989. continue;
  1990. if (!FIELD_GET(GITS_TYPER_SVPET, its->typer))
  1991. continue;
  1992. if (aff != compute_its_aff(its))
  1993. continue;
  1994. /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */
  1995. baser = its->tables[2].val;
  1996. if (!(baser & GITS_BASER_VALID))
  1997. continue;
  1998. return its;
  1999. }
  2000. return NULL;
  2001. }
  2002. static void its_free_tables(struct its_node *its)
  2003. {
  2004. int i;
  2005. for (i = 0; i < GITS_BASER_NR_REGS; i++) {
  2006. if (its->tables[i].base) {
  2007. free_pages((unsigned long)its->tables[i].base,
  2008. its->tables[i].order);
  2009. its->tables[i].base = NULL;
  2010. }
  2011. }
  2012. }
  2013. static int its_probe_baser_psz(struct its_node *its, struct its_baser *baser)
  2014. {
  2015. u64 psz = SZ_64K;
  2016. while (psz) {
  2017. u64 val, gpsz;
  2018. val = its_read_baser(its, baser);
  2019. val &= ~GITS_BASER_PAGE_SIZE_MASK;
  2020. switch (psz) {
  2021. case SZ_64K:
  2022. gpsz = GITS_BASER_PAGE_SIZE_64K;
  2023. break;
  2024. case SZ_16K:
  2025. gpsz = GITS_BASER_PAGE_SIZE_16K;
  2026. break;
  2027. case SZ_4K:
  2028. default:
  2029. gpsz = GITS_BASER_PAGE_SIZE_4K;
  2030. break;
  2031. }
  2032. gpsz >>= GITS_BASER_PAGE_SIZE_SHIFT;
  2033. val |= FIELD_PREP(GITS_BASER_PAGE_SIZE_MASK, gpsz);
  2034. its_write_baser(its, baser, val);
  2035. if (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser->val) == gpsz)
  2036. break;
  2037. switch (psz) {
  2038. case SZ_64K:
  2039. psz = SZ_16K;
  2040. break;
  2041. case SZ_16K:
  2042. psz = SZ_4K;
  2043. break;
  2044. case SZ_4K:
  2045. default:
  2046. return -1;
  2047. }
  2048. }
  2049. baser->psz = psz;
  2050. return 0;
  2051. }
  2052. static int its_alloc_tables(struct its_node *its)
  2053. {
  2054. u64 shr = GITS_BASER_InnerShareable;
  2055. u64 cache = GITS_BASER_RaWaWb;
  2056. int err, i;
  2057. if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375)
  2058. /* erratum 24313: ignore memory access type */
  2059. cache = GITS_BASER_nCnB;
  2060. for (i = 0; i < GITS_BASER_NR_REGS; i++) {
  2061. struct its_baser *baser = its->tables + i;
  2062. u64 val = its_read_baser(its, baser);
  2063. u64 type = GITS_BASER_TYPE(val);
  2064. bool indirect = false;
  2065. u32 order;
  2066. if (type == GITS_BASER_TYPE_NONE)
  2067. continue;
  2068. if (its_probe_baser_psz(its, baser)) {
  2069. its_free_tables(its);
  2070. return -ENXIO;
  2071. }
  2072. order = get_order(baser->psz);
  2073. switch (type) {
  2074. case GITS_BASER_TYPE_DEVICE:
  2075. indirect = its_parse_indirect_baser(its, baser, &order,
  2076. device_ids(its));
  2077. break;
  2078. case GITS_BASER_TYPE_VCPU:
  2079. if (is_v4_1(its)) {
  2080. struct its_node *sibling;
  2081. WARN_ON(i != 2);
  2082. if ((sibling = find_sibling_its(its))) {
  2083. *baser = sibling->tables[2];
  2084. its_write_baser(its, baser, baser->val);
  2085. continue;
  2086. }
  2087. }
  2088. indirect = its_parse_indirect_baser(its, baser, &order,
  2089. ITS_MAX_VPEID_BITS);
  2090. break;
  2091. }
  2092. err = its_setup_baser(its, baser, cache, shr, order, indirect);
  2093. if (err < 0) {
  2094. its_free_tables(its);
  2095. return err;
  2096. }
  2097. /* Update settings which will be used for next BASERn */
  2098. cache = baser->val & GITS_BASER_CACHEABILITY_MASK;
  2099. shr = baser->val & GITS_BASER_SHAREABILITY_MASK;
  2100. }
  2101. return 0;
  2102. }
  2103. static u64 inherit_vpe_l1_table_from_its(void)
  2104. {
  2105. struct its_node *its;
  2106. u64 val;
  2107. u32 aff;
  2108. val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
  2109. aff = compute_common_aff(val);
  2110. list_for_each_entry(its, &its_nodes, entry) {
  2111. u64 baser, addr;
  2112. if (!is_v4_1(its))
  2113. continue;
  2114. if (!FIELD_GET(GITS_TYPER_SVPET, its->typer))
  2115. continue;
  2116. if (aff != compute_its_aff(its))
  2117. continue;
  2118. /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */
  2119. baser = its->tables[2].val;
  2120. if (!(baser & GITS_BASER_VALID))
  2121. continue;
  2122. /* We have a winner! */
  2123. gic_data_rdist()->vpe_l1_base = its->tables[2].base;
  2124. val = GICR_VPROPBASER_4_1_VALID;
  2125. if (baser & GITS_BASER_INDIRECT)
  2126. val |= GICR_VPROPBASER_4_1_INDIRECT;
  2127. val |= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE,
  2128. FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser));
  2129. switch (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser)) {
  2130. case GIC_PAGE_SIZE_64K:
  2131. addr = GITS_BASER_ADDR_48_to_52(baser);
  2132. break;
  2133. default:
  2134. addr = baser & GENMASK_ULL(47, 12);
  2135. break;
  2136. }
  2137. val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, addr >> 12);
  2138. val |= FIELD_PREP(GICR_VPROPBASER_SHAREABILITY_MASK,
  2139. FIELD_GET(GITS_BASER_SHAREABILITY_MASK, baser));
  2140. val |= FIELD_PREP(GICR_VPROPBASER_INNER_CACHEABILITY_MASK,
  2141. FIELD_GET(GITS_BASER_INNER_CACHEABILITY_MASK, baser));
  2142. val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, GITS_BASER_NR_PAGES(baser) - 1);
  2143. return val;
  2144. }
  2145. return 0;
  2146. }
  2147. static u64 inherit_vpe_l1_table_from_rd(cpumask_t **mask)
  2148. {
  2149. u32 aff;
  2150. u64 val;
  2151. int cpu;
  2152. val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
  2153. aff = compute_common_aff(val);
  2154. for_each_possible_cpu(cpu) {
  2155. void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;
  2156. if (!base || cpu == smp_processor_id())
  2157. continue;
  2158. val = gic_read_typer(base + GICR_TYPER);
  2159. if (aff != compute_common_aff(val))
  2160. continue;
  2161. /*
  2162. * At this point, we have a victim. This particular CPU
  2163. * has already booted, and has an affinity that matches
  2164. * ours wrt CommonLPIAff. Let's use its own VPROPBASER.
  2165. * Make sure we don't write the Z bit in that case.
  2166. */
  2167. val = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER);
  2168. val &= ~GICR_VPROPBASER_4_1_Z;
  2169. gic_data_rdist()->vpe_l1_base = gic_data_rdist_cpu(cpu)->vpe_l1_base;
  2170. *mask = gic_data_rdist_cpu(cpu)->vpe_table_mask;
  2171. return val;
  2172. }
  2173. return 0;
  2174. }
  2175. static bool allocate_vpe_l2_table(int cpu, u32 id)
  2176. {
  2177. void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;
  2178. unsigned int psz, esz, idx, npg, gpsz;
  2179. u64 val;
  2180. struct page *page;
  2181. __le64 *table;
  2182. if (!gic_rdists->has_rvpeid)
  2183. return true;
  2184. /* Skip non-present CPUs */
  2185. if (!base)
  2186. return true;
  2187. val = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER);
  2188. esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val) + 1;
  2189. gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val);
  2190. npg = FIELD_GET(GICR_VPROPBASER_4_1_SIZE, val) + 1;
  2191. switch (gpsz) {
  2192. default:
  2193. WARN_ON(1);
  2194. fallthrough;
  2195. case GIC_PAGE_SIZE_4K:
  2196. psz = SZ_4K;
  2197. break;
  2198. case GIC_PAGE_SIZE_16K:
  2199. psz = SZ_16K;
  2200. break;
  2201. case GIC_PAGE_SIZE_64K:
  2202. psz = SZ_64K;
  2203. break;
  2204. }
  2205. /* Don't allow vpe_id that exceeds single, flat table limit */
  2206. if (!(val & GICR_VPROPBASER_4_1_INDIRECT))
  2207. return (id < (npg * psz / (esz * SZ_8)));
  2208. /* Compute 1st level table index & check if that exceeds table limit */
  2209. idx = id >> ilog2(psz / (esz * SZ_8));
  2210. if (idx >= (npg * psz / GITS_LVL1_ENTRY_SIZE))
  2211. return false;
  2212. table = gic_data_rdist_cpu(cpu)->vpe_l1_base;
  2213. /* Allocate memory for 2nd level table */
  2214. if (!table[idx]) {
  2215. page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(psz));
  2216. if (!page)
  2217. return false;
  2218. /* Flush Lvl2 table to PoC if hw doesn't support coherency */
  2219. if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK))
  2220. gic_flush_dcache_to_poc(page_address(page), psz);
  2221. table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID);
  2222. /* Flush Lvl1 entry to PoC if hw doesn't support coherency */
  2223. if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK))
  2224. gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE);
  2225. /* Ensure updated table contents are visible to RD hardware */
  2226. dsb(sy);
  2227. }
  2228. return true;
  2229. }
  2230. static int allocate_vpe_l1_table(void)
  2231. {
  2232. void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
  2233. u64 val, gpsz, npg, pa;
  2234. unsigned int psz = SZ_64K;
  2235. unsigned int np, epp, esz;
  2236. struct page *page;
  2237. if (!gic_rdists->has_rvpeid)
  2238. return 0;
  2239. /*
  2240. * if VPENDBASER.Valid is set, disable any previously programmed
  2241. * VPE by setting PendingLast while clearing Valid. This has the
  2242. * effect of making sure no doorbell will be generated and we can
  2243. * then safely clear VPROPBASER.Valid.
  2244. */
  2245. if (gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER) & GICR_VPENDBASER_Valid)
  2246. gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast,
  2247. vlpi_base + GICR_VPENDBASER);
  2248. /*
  2249. * If we can inherit the configuration from another RD, let's do
  2250. * so. Otherwise, we have to go through the allocation process. We
  2251. * assume that all RDs have the exact same requirements, as
  2252. * nothing will work otherwise.
  2253. */
  2254. val = inherit_vpe_l1_table_from_rd(&gic_data_rdist()->vpe_table_mask);
  2255. if (val & GICR_VPROPBASER_4_1_VALID)
  2256. goto out;
  2257. gic_data_rdist()->vpe_table_mask = kzalloc(sizeof(cpumask_t), GFP_ATOMIC);
  2258. if (!gic_data_rdist()->vpe_table_mask)
  2259. return -ENOMEM;
  2260. val = inherit_vpe_l1_table_from_its();
  2261. if (val & GICR_VPROPBASER_4_1_VALID)
  2262. goto out;
  2263. /* First probe the page size */
  2264. val = FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, GIC_PAGE_SIZE_64K);
  2265. gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
  2266. val = gicr_read_vpropbaser(vlpi_base + GICR_VPROPBASER);
  2267. gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val);
  2268. esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val);
  2269. switch (gpsz) {
  2270. default:
  2271. gpsz = GIC_PAGE_SIZE_4K;
  2272. fallthrough;
  2273. case GIC_PAGE_SIZE_4K:
  2274. psz = SZ_4K;
  2275. break;
  2276. case GIC_PAGE_SIZE_16K:
  2277. psz = SZ_16K;
  2278. break;
  2279. case GIC_PAGE_SIZE_64K:
  2280. psz = SZ_64K;
  2281. break;
  2282. }
  2283. /*
  2284. * Start populating the register from scratch, including RO fields
  2285. * (which we want to print in debug cases...)
  2286. */
  2287. val = 0;
  2288. val |= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, gpsz);
  2289. val |= FIELD_PREP(GICR_VPROPBASER_4_1_ENTRY_SIZE, esz);
  2290. /* How many entries per GIC page? */
  2291. esz++;
  2292. epp = psz / (esz * SZ_8);
  2293. /*
  2294. * If we need more than just a single L1 page, flag the table
  2295. * as indirect and compute the number of required L1 pages.
  2296. */
  2297. if (epp < ITS_MAX_VPEID) {
  2298. int nl2;
  2299. val |= GICR_VPROPBASER_4_1_INDIRECT;
  2300. /* Number of L2 pages required to cover the VPEID space */
  2301. nl2 = DIV_ROUND_UP(ITS_MAX_VPEID, epp);
  2302. /* Number of L1 pages to point to the L2 pages */
  2303. npg = DIV_ROUND_UP(nl2 * SZ_8, psz);
  2304. } else {
  2305. npg = 1;
  2306. }
  2307. val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, npg - 1);
  2308. /* Right, that's the number of CPU pages we need for L1 */
  2309. np = DIV_ROUND_UP(npg * psz, PAGE_SIZE);
  2310. pr_debug("np = %d, npg = %lld, psz = %d, epp = %d, esz = %d\n",
  2311. np, npg, psz, epp, esz);
  2312. page = alloc_pages(GFP_ATOMIC | __GFP_ZERO, get_order(np * PAGE_SIZE));
  2313. if (!page)
  2314. return -ENOMEM;
  2315. gic_data_rdist()->vpe_l1_base = page_address(page);
  2316. pa = virt_to_phys(page_address(page));
  2317. WARN_ON(!IS_ALIGNED(pa, psz));
  2318. val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, pa >> 12);
  2319. val |= GICR_VPROPBASER_RaWb;
  2320. val |= GICR_VPROPBASER_InnerShareable;
  2321. val |= GICR_VPROPBASER_4_1_Z;
  2322. val |= GICR_VPROPBASER_4_1_VALID;
  2323. out:
  2324. gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
  2325. cpumask_set_cpu(smp_processor_id(), gic_data_rdist()->vpe_table_mask);
  2326. pr_debug("CPU%d: VPROPBASER = %llx %*pbl\n",
  2327. smp_processor_id(), val,
  2328. cpumask_pr_args(gic_data_rdist()->vpe_table_mask));
  2329. return 0;
  2330. }
  2331. static int its_alloc_collections(struct its_node *its)
  2332. {
  2333. int i;
  2334. its->collections = kcalloc(nr_cpu_ids, sizeof(*its->collections),
  2335. GFP_KERNEL);
  2336. if (!its->collections)
  2337. return -ENOMEM;
  2338. for (i = 0; i < nr_cpu_ids; i++)
  2339. its->collections[i].target_address = ~0ULL;
  2340. return 0;
  2341. }
  2342. static struct page *its_allocate_pending_table(gfp_t gfp_flags)
  2343. {
  2344. struct page *pend_page;
  2345. pend_page = alloc_pages(gfp_flags | __GFP_ZERO,
  2346. get_order(LPI_PENDBASE_SZ));
  2347. if (!pend_page)
  2348. return NULL;
  2349. /* Make sure the GIC will observe the zero-ed page */
  2350. gic_flush_dcache_to_poc(page_address(pend_page), LPI_PENDBASE_SZ);
  2351. return pend_page;
  2352. }
  2353. static void its_free_pending_table(struct page *pt)
  2354. {
  2355. free_pages((unsigned long)page_address(pt), get_order(LPI_PENDBASE_SZ));
  2356. }
  2357. /*
  2358. * Booting with kdump and LPIs enabled is generally fine. Any other
  2359. * case is wrong in the absence of firmware/EFI support.
  2360. */
  2361. static bool enabled_lpis_allowed(void)
  2362. {
  2363. phys_addr_t addr;
  2364. u64 val;
  2365. /* Check whether the property table is in a reserved region */
  2366. val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER);
  2367. addr = val & GENMASK_ULL(51, 12);
  2368. return gic_check_reserved_range(addr, LPI_PROPBASE_SZ);
  2369. }
  2370. static int __init allocate_lpi_tables(void)
  2371. {
  2372. u64 val;
  2373. int err, cpu;
  2374. /*
  2375. * If LPIs are enabled while we run this from the boot CPU,
  2376. * flag the RD tables as pre-allocated if the stars do align.
  2377. */
  2378. val = readl_relaxed(gic_data_rdist_rd_base() + GICR_CTLR);
  2379. if ((val & GICR_CTLR_ENABLE_LPIS) && enabled_lpis_allowed()) {
  2380. gic_rdists->flags |= (RDIST_FLAGS_RD_TABLES_PREALLOCATED |
  2381. RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING);
  2382. pr_info("GICv3: Using preallocated redistributor tables\n");
  2383. }
  2384. err = its_setup_lpi_prop_table();
  2385. if (err)
  2386. return err;
  2387. /*
  2388. * We allocate all the pending tables anyway, as we may have a
  2389. * mix of RDs that have had LPIs enabled, and some that
  2390. * don't. We'll free the unused ones as each CPU comes online.
  2391. */
  2392. for_each_possible_cpu(cpu) {
  2393. struct page *pend_page;
  2394. pend_page = its_allocate_pending_table(GFP_NOWAIT);
  2395. if (!pend_page) {
  2396. pr_err("Failed to allocate PENDBASE for CPU%d\n", cpu);
  2397. return -ENOMEM;
  2398. }
  2399. gic_data_rdist_cpu(cpu)->pend_page = pend_page;
  2400. }
  2401. return 0;
  2402. }
  2403. static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set)
  2404. {
  2405. u32 count = 1000000; /* 1s! */
  2406. bool clean;
  2407. u64 val;
  2408. val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
  2409. val &= ~GICR_VPENDBASER_Valid;
  2410. val &= ~clr;
  2411. val |= set;
  2412. gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
  2413. do {
  2414. val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
  2415. clean = !(val & GICR_VPENDBASER_Dirty);
  2416. if (!clean) {
  2417. count--;
  2418. cpu_relax();
  2419. udelay(1);
  2420. }
  2421. } while (!clean && count);
  2422. if (unlikely(val & GICR_VPENDBASER_Dirty)) {
  2423. pr_err_ratelimited("ITS virtual pending table not cleaning\n");
  2424. val |= GICR_VPENDBASER_PendingLast;
  2425. }
  2426. return val;
  2427. }
  2428. static void its_cpu_init_lpis(void)
  2429. {
  2430. void __iomem *rbase = gic_data_rdist_rd_base();
  2431. struct page *pend_page;
  2432. phys_addr_t paddr;
  2433. u64 val, tmp;
  2434. if (gic_data_rdist()->lpi_enabled)
  2435. return;
  2436. val = readl_relaxed(rbase + GICR_CTLR);
  2437. if ((gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) &&
  2438. (val & GICR_CTLR_ENABLE_LPIS)) {
  2439. /*
  2440. * Check that we get the same property table on all
  2441. * RDs. If we don't, this is hopeless.
  2442. */
  2443. paddr = gicr_read_propbaser(rbase + GICR_PROPBASER);
  2444. paddr &= GENMASK_ULL(51, 12);
  2445. if (WARN_ON(gic_rdists->prop_table_pa != paddr))
  2446. add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
  2447. paddr = gicr_read_pendbaser(rbase + GICR_PENDBASER);
  2448. paddr &= GENMASK_ULL(51, 16);
  2449. WARN_ON(!gic_check_reserved_range(paddr, LPI_PENDBASE_SZ));
  2450. its_free_pending_table(gic_data_rdist()->pend_page);
  2451. gic_data_rdist()->pend_page = NULL;
  2452. goto out;
  2453. }
  2454. pend_page = gic_data_rdist()->pend_page;
  2455. paddr = page_to_phys(pend_page);
  2456. WARN_ON(gic_reserve_range(paddr, LPI_PENDBASE_SZ));
  2457. /* set PROPBASE */
  2458. val = (gic_rdists->prop_table_pa |
  2459. GICR_PROPBASER_InnerShareable |
  2460. GICR_PROPBASER_RaWaWb |
  2461. ((LPI_NRBITS - 1) & GICR_PROPBASER_IDBITS_MASK));
  2462. gicr_write_propbaser(val, rbase + GICR_PROPBASER);
  2463. tmp = gicr_read_propbaser(rbase + GICR_PROPBASER);
  2464. if ((tmp ^ val) & GICR_PROPBASER_SHAREABILITY_MASK) {
  2465. if (!(tmp & GICR_PROPBASER_SHAREABILITY_MASK)) {
  2466. /*
  2467. * The HW reports non-shareable, we must
  2468. * remove the cacheability attributes as
  2469. * well.
  2470. */
  2471. val &= ~(GICR_PROPBASER_SHAREABILITY_MASK |
  2472. GICR_PROPBASER_CACHEABILITY_MASK);
  2473. val |= GICR_PROPBASER_nC;
  2474. gicr_write_propbaser(val, rbase + GICR_PROPBASER);
  2475. }
  2476. pr_info_once("GIC: using cache flushing for LPI property table\n");
  2477. gic_rdists->flags |= RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING;
  2478. }
  2479. /* set PENDBASE */
  2480. val = (page_to_phys(pend_page) |
  2481. GICR_PENDBASER_InnerShareable |
  2482. GICR_PENDBASER_RaWaWb);
  2483. gicr_write_pendbaser(val, rbase + GICR_PENDBASER);
  2484. tmp = gicr_read_pendbaser(rbase + GICR_PENDBASER);
  2485. if (!(tmp & GICR_PENDBASER_SHAREABILITY_MASK)) {
  2486. /*
  2487. * The HW reports non-shareable, we must remove the
  2488. * cacheability attributes as well.
  2489. */
  2490. val &= ~(GICR_PENDBASER_SHAREABILITY_MASK |
  2491. GICR_PENDBASER_CACHEABILITY_MASK);
  2492. val |= GICR_PENDBASER_nC;
  2493. gicr_write_pendbaser(val, rbase + GICR_PENDBASER);
  2494. }
  2495. /* Enable LPIs */
  2496. val = readl_relaxed(rbase + GICR_CTLR);
  2497. val |= GICR_CTLR_ENABLE_LPIS;
  2498. writel_relaxed(val, rbase + GICR_CTLR);
  2499. if (gic_rdists->has_vlpis && !gic_rdists->has_rvpeid) {
  2500. void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
  2501. /*
  2502. * It's possible for CPU to receive VLPIs before it is
  2503. * sheduled as a vPE, especially for the first CPU, and the
  2504. * VLPI with INTID larger than 2^(IDbits+1) will be considered
  2505. * as out of range and dropped by GIC.
  2506. * So we initialize IDbits to known value to avoid VLPI drop.
  2507. */
  2508. val = (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;
  2509. pr_debug("GICv4: CPU%d: Init IDbits to 0x%llx for GICR_VPROPBASER\n",
  2510. smp_processor_id(), val);
  2511. gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
  2512. /*
  2513. * Also clear Valid bit of GICR_VPENDBASER, in case some
  2514. * ancient programming gets left in and has possibility of
  2515. * corrupting memory.
  2516. */
  2517. val = its_clear_vpend_valid(vlpi_base, 0, 0);
  2518. }
  2519. if (allocate_vpe_l1_table()) {
  2520. /*
  2521. * If the allocation has failed, we're in massive trouble.
  2522. * Disable direct injection, and pray that no VM was
  2523. * already running...
  2524. */
  2525. gic_rdists->has_rvpeid = false;
  2526. gic_rdists->has_vlpis = false;
  2527. }
  2528. /* Make sure the GIC has seen the above */
  2529. dsb(sy);
  2530. out:
  2531. gic_data_rdist()->lpi_enabled = true;
  2532. pr_info("GICv3: CPU%d: using %s LPI pending table @%pa\n",
  2533. smp_processor_id(),
  2534. gic_data_rdist()->pend_page ? "allocated" : "reserved",
  2535. &paddr);
  2536. }
  2537. static void its_cpu_init_collection(struct its_node *its)
  2538. {
  2539. int cpu = smp_processor_id();
  2540. u64 target;
  2541. /* avoid cross node collections and its mapping */
  2542. if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) {
  2543. struct device_node *cpu_node;
  2544. cpu_node = of_get_cpu_node(cpu, NULL);
  2545. if (its->numa_node != NUMA_NO_NODE &&
  2546. its->numa_node != of_node_to_nid(cpu_node))
  2547. return;
  2548. }
  2549. /*
  2550. * We now have to bind each collection to its target
  2551. * redistributor.
  2552. */
  2553. if (gic_read_typer(its->base + GITS_TYPER) & GITS_TYPER_PTA) {
  2554. /*
  2555. * This ITS wants the physical address of the
  2556. * redistributor.
  2557. */
  2558. target = gic_data_rdist()->phys_base;
  2559. } else {
  2560. /* This ITS wants a linear CPU number. */
  2561. target = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
  2562. target = GICR_TYPER_CPU_NUMBER(target) << 16;
  2563. }
  2564. /* Perform collection mapping */
  2565. its->collections[cpu].target_address = target;
  2566. its->collections[cpu].col_id = cpu;
  2567. its_send_mapc(its, &its->collections[cpu], 1);
  2568. its_send_invall(its, &its->collections[cpu]);
  2569. }
  2570. static void its_cpu_init_collections(void)
  2571. {
  2572. struct its_node *its;
  2573. raw_spin_lock(&its_lock);
  2574. list_for_each_entry(its, &its_nodes, entry)
  2575. its_cpu_init_collection(its);
  2576. raw_spin_unlock(&its_lock);
  2577. }
  2578. static struct its_device *its_find_device(struct its_node *its, u32 dev_id)
  2579. {
  2580. struct its_device *its_dev = NULL, *tmp;
  2581. unsigned long flags;
  2582. raw_spin_lock_irqsave(&its->lock, flags);
  2583. list_for_each_entry(tmp, &its->its_device_list, entry) {
  2584. if (tmp->device_id == dev_id) {
  2585. its_dev = tmp;
  2586. break;
  2587. }
  2588. }
  2589. raw_spin_unlock_irqrestore(&its->lock, flags);
  2590. return its_dev;
  2591. }
  2592. static struct its_baser *its_get_baser(struct its_node *its, u32 type)
  2593. {
  2594. int i;
  2595. for (i = 0; i < GITS_BASER_NR_REGS; i++) {
  2596. if (GITS_BASER_TYPE(its->tables[i].val) == type)
  2597. return &its->tables[i];
  2598. }
  2599. return NULL;
  2600. }
  2601. static bool its_alloc_table_entry(struct its_node *its,
  2602. struct its_baser *baser, u32 id)
  2603. {
  2604. struct page *page;
  2605. u32 esz, idx;
  2606. __le64 *table;
  2607. /* Don't allow device id that exceeds single, flat table limit */
  2608. esz = GITS_BASER_ENTRY_SIZE(baser->val);
  2609. if (!(baser->val & GITS_BASER_INDIRECT))
  2610. return (id < (PAGE_ORDER_TO_SIZE(baser->order) / esz));
  2611. /* Compute 1st level table index & check if that exceeds table limit */
  2612. idx = id >> ilog2(baser->psz / esz);
  2613. if (idx >= (PAGE_ORDER_TO_SIZE(baser->order) / GITS_LVL1_ENTRY_SIZE))
  2614. return false;
  2615. table = baser->base;
  2616. /* Allocate memory for 2nd level table */
  2617. if (!table[idx]) {
  2618. page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO,
  2619. get_order(baser->psz));
  2620. if (!page)
  2621. return false;
  2622. /* Flush Lvl2 table to PoC if hw doesn't support coherency */
  2623. if (!(baser->val & GITS_BASER_SHAREABILITY_MASK))
  2624. gic_flush_dcache_to_poc(page_address(page), baser->psz);
  2625. table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID);
  2626. /* Flush Lvl1 entry to PoC if hw doesn't support coherency */
  2627. if (!(baser->val & GITS_BASER_SHAREABILITY_MASK))
  2628. gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE);
  2629. /* Ensure updated table contents are visible to ITS hardware */
  2630. dsb(sy);
  2631. }
  2632. return true;
  2633. }
  2634. static bool its_alloc_device_table(struct its_node *its, u32 dev_id)
  2635. {
  2636. struct its_baser *baser;
  2637. baser = its_get_baser(its, GITS_BASER_TYPE_DEVICE);
  2638. /* Don't allow device id that exceeds ITS hardware limit */
  2639. if (!baser)
  2640. return (ilog2(dev_id) < device_ids(its));
  2641. return its_alloc_table_entry(its, baser, dev_id);
  2642. }
  2643. static bool its_alloc_vpe_table(u32 vpe_id)
  2644. {
  2645. struct its_node *its;
  2646. int cpu;
  2647. /*
  2648. * Make sure the L2 tables are allocated on *all* v4 ITSs. We
  2649. * could try and only do it on ITSs corresponding to devices
  2650. * that have interrupts targeted at this VPE, but the
  2651. * complexity becomes crazy (and you have tons of memory
  2652. * anyway, right?).
  2653. */
  2654. list_for_each_entry(its, &its_nodes, entry) {
  2655. struct its_baser *baser;
  2656. if (!is_v4(its))
  2657. continue;
  2658. baser = its_get_baser(its, GITS_BASER_TYPE_VCPU);
  2659. if (!baser)
  2660. return false;
  2661. if (!its_alloc_table_entry(its, baser, vpe_id))
  2662. return false;
  2663. }
  2664. /* Non v4.1? No need to iterate RDs and go back early. */
  2665. if (!gic_rdists->has_rvpeid)
  2666. return true;
  2667. /*
  2668. * Make sure the L2 tables are allocated for all copies of
  2669. * the L1 table on *all* v4.1 RDs.
  2670. */
  2671. for_each_possible_cpu(cpu) {
  2672. if (!allocate_vpe_l2_table(cpu, vpe_id))
  2673. return false;
  2674. }
  2675. return true;
  2676. }
  2677. static struct its_device *its_create_device(struct its_node *its, u32 dev_id,
  2678. int nvecs, bool alloc_lpis)
  2679. {
  2680. struct its_device *dev;
  2681. unsigned long *lpi_map = NULL;
  2682. unsigned long flags;
  2683. u16 *col_map = NULL;
  2684. void *itt;
  2685. int lpi_base;
  2686. int nr_lpis;
  2687. int nr_ites;
  2688. int sz;
  2689. if (!its_alloc_device_table(its, dev_id))
  2690. return NULL;
  2691. if (WARN_ON(!is_power_of_2(nvecs)))
  2692. nvecs = roundup_pow_of_two(nvecs);
  2693. dev = kzalloc(sizeof(*dev), GFP_KERNEL);
  2694. /*
  2695. * Even if the device wants a single LPI, the ITT must be
  2696. * sized as a power of two (and you need at least one bit...).
  2697. */
  2698. nr_ites = max(2, nvecs);
  2699. sz = nr_ites * (FIELD_GET(GITS_TYPER_ITT_ENTRY_SIZE, its->typer) + 1);
  2700. sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1;
  2701. itt = kzalloc_node(sz, GFP_KERNEL, its->numa_node);
  2702. if (alloc_lpis) {
  2703. lpi_map = its_lpi_alloc(nvecs, &lpi_base, &nr_lpis);
  2704. if (lpi_map)
  2705. col_map = kcalloc(nr_lpis, sizeof(*col_map),
  2706. GFP_KERNEL);
  2707. } else {
  2708. col_map = kcalloc(nr_ites, sizeof(*col_map), GFP_KERNEL);
  2709. nr_lpis = 0;
  2710. lpi_base = 0;
  2711. }
  2712. if (!dev || !itt || !col_map || (!lpi_map && alloc_lpis)) {
  2713. kfree(dev);
  2714. kfree(itt);
  2715. kfree(lpi_map);
  2716. kfree(col_map);
  2717. return NULL;
  2718. }
  2719. gic_flush_dcache_to_poc(itt, sz);
  2720. dev->its = its;
  2721. dev->itt = itt;
  2722. dev->nr_ites = nr_ites;
  2723. dev->event_map.lpi_map = lpi_map;
  2724. dev->event_map.col_map = col_map;
  2725. dev->event_map.lpi_base = lpi_base;
  2726. dev->event_map.nr_lpis = nr_lpis;
  2727. raw_spin_lock_init(&dev->event_map.vlpi_lock);
  2728. dev->device_id = dev_id;
  2729. INIT_LIST_HEAD(&dev->entry);
  2730. raw_spin_lock_irqsave(&its->lock, flags);
  2731. list_add(&dev->entry, &its->its_device_list);
  2732. raw_spin_unlock_irqrestore(&its->lock, flags);
  2733. /* Map device to its ITT */
  2734. its_send_mapd(dev, 1);
  2735. return dev;
  2736. }
  2737. static void its_free_device(struct its_device *its_dev)
  2738. {
  2739. unsigned long flags;
  2740. raw_spin_lock_irqsave(&its_dev->its->lock, flags);
  2741. list_del(&its_dev->entry);
  2742. raw_spin_unlock_irqrestore(&its_dev->its->lock, flags);
  2743. kfree(its_dev->event_map.col_map);
  2744. kfree(its_dev->itt);
  2745. kfree(its_dev);
  2746. }
  2747. static int its_alloc_device_irq(struct its_device *dev, int nvecs, irq_hw_number_t *hwirq)
  2748. {
  2749. int idx;
  2750. /* Find a free LPI region in lpi_map and allocate them. */
  2751. idx = bitmap_find_free_region(dev->event_map.lpi_map,
  2752. dev->event_map.nr_lpis,
  2753. get_count_order(nvecs));
  2754. if (idx < 0)
  2755. return -ENOSPC;
  2756. *hwirq = dev->event_map.lpi_base + idx;
  2757. return 0;
  2758. }
  2759. static int its_msi_prepare(struct irq_domain *domain, struct device *dev,
  2760. int nvec, msi_alloc_info_t *info)
  2761. {
  2762. struct its_node *its;
  2763. struct its_device *its_dev;
  2764. struct msi_domain_info *msi_info;
  2765. u32 dev_id;
  2766. int err = 0;
  2767. /*
  2768. * We ignore "dev" entirely, and rely on the dev_id that has
  2769. * been passed via the scratchpad. This limits this domain's
  2770. * usefulness to upper layers that definitely know that they
  2771. * are built on top of the ITS.
  2772. */
  2773. dev_id = info->scratchpad[0].ul;
  2774. msi_info = msi_get_domain_info(domain);
  2775. its = msi_info->data;
  2776. if (!gic_rdists->has_direct_lpi &&
  2777. vpe_proxy.dev &&
  2778. vpe_proxy.dev->its == its &&
  2779. dev_id == vpe_proxy.dev->device_id) {
  2780. /* Bad luck. Get yourself a better implementation */
  2781. WARN_ONCE(1, "DevId %x clashes with GICv4 VPE proxy device\n",
  2782. dev_id);
  2783. return -EINVAL;
  2784. }
  2785. mutex_lock(&its->dev_alloc_lock);
  2786. its_dev = its_find_device(its, dev_id);
  2787. if (its_dev) {
  2788. /*
  2789. * We already have seen this ID, probably through
  2790. * another alias (PCI bridge of some sort). No need to
  2791. * create the device.
  2792. */
  2793. its_dev->shared = true;
  2794. pr_debug("Reusing ITT for devID %x\n", dev_id);
  2795. goto out;
  2796. }
  2797. its_dev = its_create_device(its, dev_id, nvec, true);
  2798. if (!its_dev) {
  2799. err = -ENOMEM;
  2800. goto out;
  2801. }
  2802. pr_debug("ITT %d entries, %d bits\n", nvec, ilog2(nvec));
  2803. out:
  2804. mutex_unlock(&its->dev_alloc_lock);
  2805. info->scratchpad[0].ptr = its_dev;
  2806. return err;
  2807. }
  2808. static struct msi_domain_ops its_msi_domain_ops = {
  2809. .msi_prepare = its_msi_prepare,
  2810. };
  2811. static int its_irq_gic_domain_alloc(struct irq_domain *domain,
  2812. unsigned int virq,
  2813. irq_hw_number_t hwirq)
  2814. {
  2815. struct irq_fwspec fwspec;
  2816. if (irq_domain_get_of_node(domain->parent)) {
  2817. fwspec.fwnode = domain->parent->fwnode;
  2818. fwspec.param_count = 3;
  2819. fwspec.param[0] = GIC_IRQ_TYPE_LPI;
  2820. fwspec.param[1] = hwirq;
  2821. fwspec.param[2] = IRQ_TYPE_EDGE_RISING;
  2822. } else if (is_fwnode_irqchip(domain->parent->fwnode)) {
  2823. fwspec.fwnode = domain->parent->fwnode;
  2824. fwspec.param_count = 2;
  2825. fwspec.param[0] = hwirq;
  2826. fwspec.param[1] = IRQ_TYPE_EDGE_RISING;
  2827. } else {
  2828. return -EINVAL;
  2829. }
  2830. return irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec);
  2831. }
  2832. static int its_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
  2833. unsigned int nr_irqs, void *args)
  2834. {
  2835. msi_alloc_info_t *info = args;
  2836. struct its_device *its_dev = info->scratchpad[0].ptr;
  2837. struct its_node *its = its_dev->its;
  2838. struct irq_data *irqd;
  2839. irq_hw_number_t hwirq;
  2840. int err;
  2841. int i;
  2842. err = its_alloc_device_irq(its_dev, nr_irqs, &hwirq);
  2843. if (err)
  2844. return err;
  2845. err = iommu_dma_prepare_msi(info->desc, its->get_msi_base(its_dev));
  2846. if (err)
  2847. return err;
  2848. for (i = 0; i < nr_irqs; i++) {
  2849. err = its_irq_gic_domain_alloc(domain, virq + i, hwirq + i);
  2850. if (err)
  2851. return err;
  2852. irq_domain_set_hwirq_and_chip(domain, virq + i,
  2853. hwirq + i, &its_irq_chip, its_dev);
  2854. irqd = irq_get_irq_data(virq + i);
  2855. irqd_set_single_target(irqd);
  2856. irqd_set_affinity_on_activate(irqd);
  2857. pr_debug("ID:%d pID:%d vID:%d\n",
  2858. (int)(hwirq + i - its_dev->event_map.lpi_base),
  2859. (int)(hwirq + i), virq + i);
  2860. }
  2861. return 0;
  2862. }
  2863. static int its_irq_domain_activate(struct irq_domain *domain,
  2864. struct irq_data *d, bool reserve)
  2865. {
  2866. struct its_device *its_dev = irq_data_get_irq_chip_data(d);
  2867. u32 event = its_get_event_id(d);
  2868. int cpu;
  2869. cpu = its_select_cpu(d, cpu_online_mask);
  2870. if (cpu < 0 || cpu >= nr_cpu_ids)
  2871. return -EINVAL;
  2872. its_inc_lpi_count(d, cpu);
  2873. its_dev->event_map.col_map[event] = cpu;
  2874. irq_data_update_effective_affinity(d, cpumask_of(cpu));
  2875. /* Map the GIC IRQ and event to the device */
  2876. its_send_mapti(its_dev, d->hwirq, event);
  2877. return 0;
  2878. }
  2879. static void its_irq_domain_deactivate(struct irq_domain *domain,
  2880. struct irq_data *d)
  2881. {
  2882. struct its_device *its_dev = irq_data_get_irq_chip_data(d);
  2883. u32 event = its_get_event_id(d);
  2884. its_dec_lpi_count(d, its_dev->event_map.col_map[event]);
  2885. /* Stop the delivery of interrupts */
  2886. its_send_discard(its_dev, event);
  2887. }
  2888. static void its_irq_domain_free(struct irq_domain *domain, unsigned int virq,
  2889. unsigned int nr_irqs)
  2890. {
  2891. struct irq_data *d = irq_domain_get_irq_data(domain, virq);
  2892. struct its_device *its_dev = irq_data_get_irq_chip_data(d);
  2893. struct its_node *its = its_dev->its;
  2894. int i;
  2895. bitmap_release_region(its_dev->event_map.lpi_map,
  2896. its_get_event_id(irq_domain_get_irq_data(domain, virq)),
  2897. get_count_order(nr_irqs));
  2898. for (i = 0; i < nr_irqs; i++) {
  2899. struct irq_data *data = irq_domain_get_irq_data(domain,
  2900. virq + i);
  2901. /* Nuke the entry in the domain */
  2902. irq_domain_reset_irq_data(data);
  2903. }
  2904. mutex_lock(&its->dev_alloc_lock);
  2905. /*
  2906. * If all interrupts have been freed, start mopping the
  2907. * floor. This is conditionned on the device not being shared.
  2908. */
  2909. if (!its_dev->shared &&
  2910. bitmap_empty(its_dev->event_map.lpi_map,
  2911. its_dev->event_map.nr_lpis)) {
  2912. its_lpi_free(its_dev->event_map.lpi_map,
  2913. its_dev->event_map.lpi_base,
  2914. its_dev->event_map.nr_lpis);
  2915. /* Unmap device/itt */
  2916. its_send_mapd(its_dev, 0);
  2917. its_free_device(its_dev);
  2918. }
  2919. mutex_unlock(&its->dev_alloc_lock);
  2920. irq_domain_free_irqs_parent(domain, virq, nr_irqs);
  2921. }
  2922. static const struct irq_domain_ops its_domain_ops = {
  2923. .alloc = its_irq_domain_alloc,
  2924. .free = its_irq_domain_free,
  2925. .activate = its_irq_domain_activate,
  2926. .deactivate = its_irq_domain_deactivate,
  2927. };
  2928. /*
  2929. * This is insane.
  2930. *
  2931. * If a GICv4.0 doesn't implement Direct LPIs (which is extremely
  2932. * likely), the only way to perform an invalidate is to use a fake
  2933. * device to issue an INV command, implying that the LPI has first
  2934. * been mapped to some event on that device. Since this is not exactly
  2935. * cheap, we try to keep that mapping around as long as possible, and
  2936. * only issue an UNMAP if we're short on available slots.
  2937. *
  2938. * Broken by design(tm).
  2939. *
  2940. * GICv4.1, on the other hand, mandates that we're able to invalidate
  2941. * by writing to a MMIO register. It doesn't implement the whole of
  2942. * DirectLPI, but that's good enough. And most of the time, we don't
  2943. * even have to invalidate anything, as the redistributor can be told
  2944. * whether to generate a doorbell or not (we thus leave it enabled,
  2945. * always).
  2946. */
  2947. static void its_vpe_db_proxy_unmap_locked(struct its_vpe *vpe)
  2948. {
  2949. /* GICv4.1 doesn't use a proxy, so nothing to do here */
  2950. if (gic_rdists->has_rvpeid)
  2951. return;
  2952. /* Already unmapped? */
  2953. if (vpe->vpe_proxy_event == -1)
  2954. return;
  2955. its_send_discard(vpe_proxy.dev, vpe->vpe_proxy_event);
  2956. vpe_proxy.vpes[vpe->vpe_proxy_event] = NULL;
  2957. /*
  2958. * We don't track empty slots at all, so let's move the
  2959. * next_victim pointer if we can quickly reuse that slot
  2960. * instead of nuking an existing entry. Not clear that this is
  2961. * always a win though, and this might just generate a ripple
  2962. * effect... Let's just hope VPEs don't migrate too often.
  2963. */
  2964. if (vpe_proxy.vpes[vpe_proxy.next_victim])
  2965. vpe_proxy.next_victim = vpe->vpe_proxy_event;
  2966. vpe->vpe_proxy_event = -1;
  2967. }
  2968. static void its_vpe_db_proxy_unmap(struct its_vpe *vpe)
  2969. {
  2970. /* GICv4.1 doesn't use a proxy, so nothing to do here */
  2971. if (gic_rdists->has_rvpeid)
  2972. return;
  2973. if (!gic_rdists->has_direct_lpi) {
  2974. unsigned long flags;
  2975. raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
  2976. its_vpe_db_proxy_unmap_locked(vpe);
  2977. raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
  2978. }
  2979. }
  2980. static void its_vpe_db_proxy_map_locked(struct its_vpe *vpe)
  2981. {
  2982. /* GICv4.1 doesn't use a proxy, so nothing to do here */
  2983. if (gic_rdists->has_rvpeid)
  2984. return;
  2985. /* Already mapped? */
  2986. if (vpe->vpe_proxy_event != -1)
  2987. return;
  2988. /* This slot was already allocated. Kick the other VPE out. */
  2989. if (vpe_proxy.vpes[vpe_proxy.next_victim])
  2990. its_vpe_db_proxy_unmap_locked(vpe_proxy.vpes[vpe_proxy.next_victim]);
  2991. /* Map the new VPE instead */
  2992. vpe_proxy.vpes[vpe_proxy.next_victim] = vpe;
  2993. vpe->vpe_proxy_event = vpe_proxy.next_victim;
  2994. vpe_proxy.next_victim = (vpe_proxy.next_victim + 1) % vpe_proxy.dev->nr_ites;
  2995. vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = vpe->col_idx;
  2996. its_send_mapti(vpe_proxy.dev, vpe->vpe_db_lpi, vpe->vpe_proxy_event);
  2997. }
  2998. static void its_vpe_db_proxy_move(struct its_vpe *vpe, int from, int to)
  2999. {
  3000. unsigned long flags;
  3001. struct its_collection *target_col;
  3002. /* GICv4.1 doesn't use a proxy, so nothing to do here */
  3003. if (gic_rdists->has_rvpeid)
  3004. return;
  3005. if (gic_rdists->has_direct_lpi) {
  3006. void __iomem *rdbase;
  3007. rdbase = per_cpu_ptr(gic_rdists->rdist, from)->rd_base;
  3008. gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
  3009. wait_for_syncr(rdbase);
  3010. return;
  3011. }
  3012. raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
  3013. its_vpe_db_proxy_map_locked(vpe);
  3014. target_col = &vpe_proxy.dev->its->collections[to];
  3015. its_send_movi(vpe_proxy.dev, target_col, vpe->vpe_proxy_event);
  3016. vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = to;
  3017. raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
  3018. }
  3019. static int its_vpe_set_affinity(struct irq_data *d,
  3020. const struct cpumask *mask_val,
  3021. bool force)
  3022. {
  3023. struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
  3024. int from, cpu = cpumask_first(mask_val);
  3025. unsigned long flags;
  3026. /*
  3027. * Changing affinity is mega expensive, so let's be as lazy as
  3028. * we can and only do it if we really have to. Also, if mapped
  3029. * into the proxy device, we need to move the doorbell
  3030. * interrupt to its new location.
  3031. *
  3032. * Another thing is that changing the affinity of a vPE affects
  3033. * *other interrupts* such as all the vLPIs that are routed to
  3034. * this vPE. This means that the irq_desc lock is not enough to
  3035. * protect us, and that we must ensure nobody samples vpe->col_idx
  3036. * during the update, hence the lock below which must also be
  3037. * taken on any vLPI handling path that evaluates vpe->col_idx.
  3038. */
  3039. from = vpe_to_cpuid_lock(vpe, &flags);
  3040. if (from == cpu)
  3041. goto out;
  3042. vpe->col_idx = cpu;
  3043. /*
  3044. * GICv4.1 allows us to skip VMOVP if moving to a cpu whose RD
  3045. * is sharing its VPE table with the current one.
  3046. */
  3047. if (gic_data_rdist_cpu(cpu)->vpe_table_mask &&
  3048. cpumask_test_cpu(from, gic_data_rdist_cpu(cpu)->vpe_table_mask))
  3049. goto out;
  3050. its_send_vmovp(vpe);
  3051. its_vpe_db_proxy_move(vpe, from, cpu);
  3052. out:
  3053. irq_data_update_effective_affinity(d, cpumask_of(cpu));
  3054. vpe_to_cpuid_unlock(vpe, flags);
  3055. return IRQ_SET_MASK_OK_DONE;
  3056. }
  3057. static void its_wait_vpt_parse_complete(void)
  3058. {
  3059. void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
  3060. u64 val;
  3061. if (!gic_rdists->has_vpend_valid_dirty)
  3062. return;
  3063. WARN_ON_ONCE(readq_relaxed_poll_timeout_atomic(vlpi_base + GICR_VPENDBASER,
  3064. val,
  3065. !(val & GICR_VPENDBASER_Dirty),
  3066. 10, 500));
  3067. }
  3068. static void its_vpe_schedule(struct its_vpe *vpe)
  3069. {
  3070. void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
  3071. u64 val;
  3072. /* Schedule the VPE */
  3073. val = virt_to_phys(page_address(vpe->its_vm->vprop_page)) &
  3074. GENMASK_ULL(51, 12);
  3075. val |= (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;
  3076. val |= GICR_VPROPBASER_RaWb;
  3077. val |= GICR_VPROPBASER_InnerShareable;
  3078. gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
  3079. val = virt_to_phys(page_address(vpe->vpt_page)) &
  3080. GENMASK_ULL(51, 16);
  3081. val |= GICR_VPENDBASER_RaWaWb;
  3082. val |= GICR_VPENDBASER_InnerShareable;
  3083. /*
  3084. * There is no good way of finding out if the pending table is
  3085. * empty as we can race against the doorbell interrupt very
  3086. * easily. So in the end, vpe->pending_last is only an
  3087. * indication that the vcpu has something pending, not one
  3088. * that the pending table is empty. A good implementation
  3089. * would be able to read its coarse map pretty quickly anyway,
  3090. * making this a tolerable issue.
  3091. */
  3092. val |= GICR_VPENDBASER_PendingLast;
  3093. val |= vpe->idai ? GICR_VPENDBASER_IDAI : 0;
  3094. val |= GICR_VPENDBASER_Valid;
  3095. gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
  3096. }
  3097. static void its_vpe_deschedule(struct its_vpe *vpe)
  3098. {
  3099. void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
  3100. u64 val;
  3101. val = its_clear_vpend_valid(vlpi_base, 0, 0);
  3102. vpe->idai = !!(val & GICR_VPENDBASER_IDAI);
  3103. vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
  3104. }
  3105. static void its_vpe_invall(struct its_vpe *vpe)
  3106. {
  3107. struct its_node *its;
  3108. list_for_each_entry(its, &its_nodes, entry) {
  3109. if (!is_v4(its))
  3110. continue;
  3111. if (its_list_map && !vpe->its_vm->vlpi_count[its->list_nr])
  3112. continue;
  3113. /*
  3114. * Sending a VINVALL to a single ITS is enough, as all
  3115. * we need is to reach the redistributors.
  3116. */
  3117. its_send_vinvall(its, vpe);
  3118. return;
  3119. }
  3120. }
  3121. static int its_vpe_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
  3122. {
  3123. struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
  3124. struct its_cmd_info *info = vcpu_info;
  3125. switch (info->cmd_type) {
  3126. case SCHEDULE_VPE:
  3127. its_vpe_schedule(vpe);
  3128. return 0;
  3129. case DESCHEDULE_VPE:
  3130. its_vpe_deschedule(vpe);
  3131. return 0;
  3132. case COMMIT_VPE:
  3133. its_wait_vpt_parse_complete();
  3134. return 0;
  3135. case INVALL_VPE:
  3136. its_vpe_invall(vpe);
  3137. return 0;
  3138. default:
  3139. return -EINVAL;
  3140. }
  3141. }
  3142. static void its_vpe_send_cmd(struct its_vpe *vpe,
  3143. void (*cmd)(struct its_device *, u32))
  3144. {
  3145. unsigned long flags;
  3146. raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
  3147. its_vpe_db_proxy_map_locked(vpe);
  3148. cmd(vpe_proxy.dev, vpe->vpe_proxy_event);
  3149. raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
  3150. }
  3151. static void its_vpe_send_inv(struct irq_data *d)
  3152. {
  3153. struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
  3154. if (gic_rdists->has_direct_lpi) {
  3155. void __iomem *rdbase;
  3156. /* Target the redistributor this VPE is currently known on */
  3157. raw_spin_lock(&gic_data_rdist_cpu(vpe->col_idx)->rd_lock);
  3158. rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base;
  3159. gic_write_lpir(d->parent_data->hwirq, rdbase + GICR_INVLPIR);
  3160. wait_for_syncr(rdbase);
  3161. raw_spin_unlock(&gic_data_rdist_cpu(vpe->col_idx)->rd_lock);
  3162. } else {
  3163. its_vpe_send_cmd(vpe, its_send_inv);
  3164. }
  3165. }
  3166. static void its_vpe_mask_irq(struct irq_data *d)
  3167. {
  3168. /*
  3169. * We need to unmask the LPI, which is described by the parent
  3170. * irq_data. Instead of calling into the parent (which won't
  3171. * exactly do the right thing, let's simply use the
  3172. * parent_data pointer. Yes, I'm naughty.
  3173. */
  3174. lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0);
  3175. its_vpe_send_inv(d);
  3176. }
  3177. static void its_vpe_unmask_irq(struct irq_data *d)
  3178. {
  3179. /* Same hack as above... */
  3180. lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED);
  3181. its_vpe_send_inv(d);
  3182. }
  3183. static int its_vpe_set_irqchip_state(struct irq_data *d,
  3184. enum irqchip_irq_state which,
  3185. bool state)
  3186. {
  3187. struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
  3188. if (which != IRQCHIP_STATE_PENDING)
  3189. return -EINVAL;
  3190. if (gic_rdists->has_direct_lpi) {
  3191. void __iomem *rdbase;
  3192. rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base;
  3193. if (state) {
  3194. gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_SETLPIR);
  3195. } else {
  3196. gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
  3197. wait_for_syncr(rdbase);
  3198. }
  3199. } else {
  3200. if (state)
  3201. its_vpe_send_cmd(vpe, its_send_int);
  3202. else
  3203. its_vpe_send_cmd(vpe, its_send_clear);
  3204. }
  3205. return 0;
  3206. }
  3207. static int its_vpe_retrigger(struct irq_data *d)
  3208. {
  3209. return !its_vpe_set_irqchip_state(d, IRQCHIP_STATE_PENDING, true);
  3210. }
  3211. static struct irq_chip its_vpe_irq_chip = {
  3212. .name = "GICv4-vpe",
  3213. .irq_mask = its_vpe_mask_irq,
  3214. .irq_unmask = its_vpe_unmask_irq,
  3215. .irq_eoi = irq_chip_eoi_parent,
  3216. .irq_set_affinity = its_vpe_set_affinity,
  3217. .irq_retrigger = its_vpe_retrigger,
  3218. .irq_set_irqchip_state = its_vpe_set_irqchip_state,
  3219. .irq_set_vcpu_affinity = its_vpe_set_vcpu_affinity,
  3220. };
  3221. static struct its_node *find_4_1_its(void)
  3222. {
  3223. static struct its_node *its = NULL;
  3224. if (!its) {
  3225. list_for_each_entry(its, &its_nodes, entry) {
  3226. if (is_v4_1(its))
  3227. return its;
  3228. }
  3229. /* Oops? */
  3230. its = NULL;
  3231. }
  3232. return its;
  3233. }
  3234. static void its_vpe_4_1_send_inv(struct irq_data *d)
  3235. {
  3236. struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
  3237. struct its_node *its;
  3238. /*
  3239. * GICv4.1 wants doorbells to be invalidated using the
  3240. * INVDB command in order to be broadcast to all RDs. Send
  3241. * it to the first valid ITS, and let the HW do its magic.
  3242. */
  3243. its = find_4_1_its();
  3244. if (its)
  3245. its_send_invdb(its, vpe);
  3246. }
  3247. static void its_vpe_4_1_mask_irq(struct irq_data *d)
  3248. {
  3249. lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0);
  3250. its_vpe_4_1_send_inv(d);
  3251. }
  3252. static void its_vpe_4_1_unmask_irq(struct irq_data *d)
  3253. {
  3254. lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED);
  3255. its_vpe_4_1_send_inv(d);
  3256. }
  3257. static void its_vpe_4_1_schedule(struct its_vpe *vpe,
  3258. struct its_cmd_info *info)
  3259. {
  3260. void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
  3261. u64 val = 0;
  3262. /* Schedule the VPE */
  3263. val |= GICR_VPENDBASER_Valid;
  3264. val |= info->g0en ? GICR_VPENDBASER_4_1_VGRP0EN : 0;
  3265. val |= info->g1en ? GICR_VPENDBASER_4_1_VGRP1EN : 0;
  3266. val |= FIELD_PREP(GICR_VPENDBASER_4_1_VPEID, vpe->vpe_id);
  3267. gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
  3268. }
  3269. static void its_vpe_4_1_deschedule(struct its_vpe *vpe,
  3270. struct its_cmd_info *info)
  3271. {
  3272. void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
  3273. u64 val;
  3274. if (info->req_db) {
  3275. unsigned long flags;
  3276. /*
  3277. * vPE is going to block: make the vPE non-resident with
  3278. * PendingLast clear and DB set. The GIC guarantees that if
  3279. * we read-back PendingLast clear, then a doorbell will be
  3280. * delivered when an interrupt comes.
  3281. *
  3282. * Note the locking to deal with the concurrent update of
  3283. * pending_last from the doorbell interrupt handler that can
  3284. * run concurrently.
  3285. */
  3286. raw_spin_lock_irqsave(&vpe->vpe_lock, flags);
  3287. val = its_clear_vpend_valid(vlpi_base,
  3288. GICR_VPENDBASER_PendingLast,
  3289. GICR_VPENDBASER_4_1_DB);
  3290. vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
  3291. raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);
  3292. } else {
  3293. /*
  3294. * We're not blocking, so just make the vPE non-resident
  3295. * with PendingLast set, indicating that we'll be back.
  3296. */
  3297. val = its_clear_vpend_valid(vlpi_base,
  3298. 0,
  3299. GICR_VPENDBASER_PendingLast);
  3300. vpe->pending_last = true;
  3301. }
  3302. }
  3303. static void its_vpe_4_1_invall(struct its_vpe *vpe)
  3304. {
  3305. void __iomem *rdbase;
  3306. unsigned long flags;
  3307. u64 val;
  3308. int cpu;
  3309. val = GICR_INVALLR_V;
  3310. val |= FIELD_PREP(GICR_INVALLR_VPEID, vpe->vpe_id);
  3311. /* Target the redistributor this vPE is currently known on */
  3312. cpu = vpe_to_cpuid_lock(vpe, &flags);
  3313. raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
  3314. rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base;
  3315. gic_write_lpir(val, rdbase + GICR_INVALLR);
  3316. wait_for_syncr(rdbase);
  3317. raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
  3318. vpe_to_cpuid_unlock(vpe, flags);
  3319. }
  3320. static int its_vpe_4_1_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
  3321. {
  3322. struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
  3323. struct its_cmd_info *info = vcpu_info;
  3324. switch (info->cmd_type) {
  3325. case SCHEDULE_VPE:
  3326. its_vpe_4_1_schedule(vpe, info);
  3327. return 0;
  3328. case DESCHEDULE_VPE:
  3329. its_vpe_4_1_deschedule(vpe, info);
  3330. return 0;
  3331. case COMMIT_VPE:
  3332. its_wait_vpt_parse_complete();
  3333. return 0;
  3334. case INVALL_VPE:
  3335. its_vpe_4_1_invall(vpe);
  3336. return 0;
  3337. default:
  3338. return -EINVAL;
  3339. }
  3340. }
  3341. static struct irq_chip its_vpe_4_1_irq_chip = {
  3342. .name = "GICv4.1-vpe",
  3343. .irq_mask = its_vpe_4_1_mask_irq,
  3344. .irq_unmask = its_vpe_4_1_unmask_irq,
  3345. .irq_eoi = irq_chip_eoi_parent,
  3346. .irq_set_affinity = its_vpe_set_affinity,
  3347. .irq_set_vcpu_affinity = its_vpe_4_1_set_vcpu_affinity,
  3348. };
  3349. static void its_configure_sgi(struct irq_data *d, bool clear)
  3350. {
  3351. struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
  3352. struct its_cmd_desc desc;
  3353. desc.its_vsgi_cmd.vpe = vpe;
  3354. desc.its_vsgi_cmd.sgi = d->hwirq;
  3355. desc.its_vsgi_cmd.priority = vpe->sgi_config[d->hwirq].priority;
  3356. desc.its_vsgi_cmd.enable = vpe->sgi_config[d->hwirq].enabled;
  3357. desc.its_vsgi_cmd.group = vpe->sgi_config[d->hwirq].group;
  3358. desc.its_vsgi_cmd.clear = clear;
  3359. /*
  3360. * GICv4.1 allows us to send VSGI commands to any ITS as long as the
  3361. * destination VPE is mapped there. Since we map them eagerly at
  3362. * activation time, we're pretty sure the first GICv4.1 ITS will do.
  3363. */
  3364. its_send_single_vcommand(find_4_1_its(), its_build_vsgi_cmd, &desc);
  3365. }
  3366. static void its_sgi_mask_irq(struct irq_data *d)
  3367. {
  3368. struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
  3369. vpe->sgi_config[d->hwirq].enabled = false;
  3370. its_configure_sgi(d, false);
  3371. }
  3372. static void its_sgi_unmask_irq(struct irq_data *d)
  3373. {
  3374. struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
  3375. vpe->sgi_config[d->hwirq].enabled = true;
  3376. its_configure_sgi(d, false);
  3377. }
  3378. static int its_sgi_set_affinity(struct irq_data *d,
  3379. const struct cpumask *mask_val,
  3380. bool force)
  3381. {
  3382. /*
  3383. * There is no notion of affinity for virtual SGIs, at least
  3384. * not on the host (since they can only be targetting a vPE).
  3385. * Tell the kernel we've done whatever it asked for.
  3386. */
  3387. irq_data_update_effective_affinity(d, mask_val);
  3388. return IRQ_SET_MASK_OK;
  3389. }
  3390. static int its_sgi_set_irqchip_state(struct irq_data *d,
  3391. enum irqchip_irq_state which,
  3392. bool state)
  3393. {
  3394. if (which != IRQCHIP_STATE_PENDING)
  3395. return -EINVAL;
  3396. if (state) {
  3397. struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
  3398. struct its_node *its = find_4_1_its();
  3399. u64 val;
  3400. val = FIELD_PREP(GITS_SGIR_VPEID, vpe->vpe_id);
  3401. val |= FIELD_PREP(GITS_SGIR_VINTID, d->hwirq);
  3402. writeq_relaxed(val, its->sgir_base + GITS_SGIR - SZ_128K);
  3403. } else {
  3404. its_configure_sgi(d, true);
  3405. }
  3406. return 0;
  3407. }
  3408. static int its_sgi_get_irqchip_state(struct irq_data *d,
  3409. enum irqchip_irq_state which, bool *val)
  3410. {
  3411. struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
  3412. void __iomem *base;
  3413. unsigned long flags;
  3414. u32 count = 1000000; /* 1s! */
  3415. u32 status;
  3416. int cpu;
  3417. if (which != IRQCHIP_STATE_PENDING)
  3418. return -EINVAL;
  3419. /*
  3420. * Locking galore! We can race against two different events:
  3421. *
  3422. * - Concurent vPE affinity change: we must make sure it cannot
  3423. * happen, or we'll talk to the wrong redistributor. This is
  3424. * identical to what happens with vLPIs.
  3425. *
  3426. * - Concurrent VSGIPENDR access: As it involves accessing two
  3427. * MMIO registers, this must be made atomic one way or another.
  3428. */
  3429. cpu = vpe_to_cpuid_lock(vpe, &flags);
  3430. raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
  3431. base = gic_data_rdist_cpu(cpu)->rd_base + SZ_128K;
  3432. writel_relaxed(vpe->vpe_id, base + GICR_VSGIR);
  3433. do {
  3434. status = readl_relaxed(base + GICR_VSGIPENDR);
  3435. if (!(status & GICR_VSGIPENDR_BUSY))
  3436. goto out;
  3437. count--;
  3438. if (!count) {
  3439. pr_err_ratelimited("Unable to get SGI status\n");
  3440. goto out;
  3441. }
  3442. cpu_relax();
  3443. udelay(1);
  3444. } while (count);
  3445. out:
  3446. raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
  3447. vpe_to_cpuid_unlock(vpe, flags);
  3448. if (!count)
  3449. return -ENXIO;
  3450. *val = !!(status & (1 << d->hwirq));
  3451. return 0;
  3452. }
  3453. static int its_sgi_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
  3454. {
  3455. struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
  3456. struct its_cmd_info *info = vcpu_info;
  3457. switch (info->cmd_type) {
  3458. case PROP_UPDATE_VSGI:
  3459. vpe->sgi_config[d->hwirq].priority = info->priority;
  3460. vpe->sgi_config[d->hwirq].group = info->group;
  3461. its_configure_sgi(d, false);
  3462. return 0;
  3463. default:
  3464. return -EINVAL;
  3465. }
  3466. }
  3467. static struct irq_chip its_sgi_irq_chip = {
  3468. .name = "GICv4.1-sgi",
  3469. .irq_mask = its_sgi_mask_irq,
  3470. .irq_unmask = its_sgi_unmask_irq,
  3471. .irq_set_affinity = its_sgi_set_affinity,
  3472. .irq_set_irqchip_state = its_sgi_set_irqchip_state,
  3473. .irq_get_irqchip_state = its_sgi_get_irqchip_state,
  3474. .irq_set_vcpu_affinity = its_sgi_set_vcpu_affinity,
  3475. };
  3476. static int its_sgi_irq_domain_alloc(struct irq_domain *domain,
  3477. unsigned int virq, unsigned int nr_irqs,
  3478. void *args)
  3479. {
  3480. struct its_vpe *vpe = args;
  3481. int i;
  3482. /* Yes, we do want 16 SGIs */
  3483. WARN_ON(nr_irqs != 16);
  3484. for (i = 0; i < 16; i++) {
  3485. vpe->sgi_config[i].priority = 0;
  3486. vpe->sgi_config[i].enabled = false;
  3487. vpe->sgi_config[i].group = false;
  3488. irq_domain_set_hwirq_and_chip(domain, virq + i, i,
  3489. &its_sgi_irq_chip, vpe);
  3490. irq_set_status_flags(virq + i, IRQ_DISABLE_UNLAZY);
  3491. }
  3492. return 0;
  3493. }
  3494. static void its_sgi_irq_domain_free(struct irq_domain *domain,
  3495. unsigned int virq,
  3496. unsigned int nr_irqs)
  3497. {
  3498. /* Nothing to do */
  3499. }
  3500. static int its_sgi_irq_domain_activate(struct irq_domain *domain,
  3501. struct irq_data *d, bool reserve)
  3502. {
  3503. /* Write out the initial SGI configuration */
  3504. its_configure_sgi(d, false);
  3505. return 0;
  3506. }
  3507. static void its_sgi_irq_domain_deactivate(struct irq_domain *domain,
  3508. struct irq_data *d)
  3509. {
  3510. struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
  3511. /*
  3512. * The VSGI command is awkward:
  3513. *
  3514. * - To change the configuration, CLEAR must be set to false,
  3515. * leaving the pending bit unchanged.
  3516. * - To clear the pending bit, CLEAR must be set to true, leaving
  3517. * the configuration unchanged.
  3518. *
  3519. * You just can't do both at once, hence the two commands below.
  3520. */
  3521. vpe->sgi_config[d->hwirq].enabled = false;
  3522. its_configure_sgi(d, false);
  3523. its_configure_sgi(d, true);
  3524. }
  3525. static const struct irq_domain_ops its_sgi_domain_ops = {
  3526. .alloc = its_sgi_irq_domain_alloc,
  3527. .free = its_sgi_irq_domain_free,
  3528. .activate = its_sgi_irq_domain_activate,
  3529. .deactivate = its_sgi_irq_domain_deactivate,
  3530. };
  3531. static int its_vpe_id_alloc(void)
  3532. {
  3533. return ida_simple_get(&its_vpeid_ida, 0, ITS_MAX_VPEID, GFP_KERNEL);
  3534. }
  3535. static void its_vpe_id_free(u16 id)
  3536. {
  3537. ida_simple_remove(&its_vpeid_ida, id);
  3538. }
  3539. static int its_vpe_init(struct its_vpe *vpe)
  3540. {
  3541. struct page *vpt_page;
  3542. int vpe_id;
  3543. /* Allocate vpe_id */
  3544. vpe_id = its_vpe_id_alloc();
  3545. if (vpe_id < 0)
  3546. return vpe_id;
  3547. /* Allocate VPT */
  3548. vpt_page = its_allocate_pending_table(GFP_KERNEL);
  3549. if (!vpt_page) {
  3550. its_vpe_id_free(vpe_id);
  3551. return -ENOMEM;
  3552. }
  3553. if (!its_alloc_vpe_table(vpe_id)) {
  3554. its_vpe_id_free(vpe_id);
  3555. its_free_pending_table(vpt_page);
  3556. return -ENOMEM;
  3557. }
  3558. raw_spin_lock_init(&vpe->vpe_lock);
  3559. vpe->vpe_id = vpe_id;
  3560. vpe->vpt_page = vpt_page;
  3561. if (gic_rdists->has_rvpeid)
  3562. atomic_set(&vpe->vmapp_count, 0);
  3563. else
  3564. vpe->vpe_proxy_event = -1;
  3565. return 0;
  3566. }
  3567. static void its_vpe_teardown(struct its_vpe *vpe)
  3568. {
  3569. its_vpe_db_proxy_unmap(vpe);
  3570. its_vpe_id_free(vpe->vpe_id);
  3571. its_free_pending_table(vpe->vpt_page);
  3572. }
  3573. static void its_vpe_irq_domain_free(struct irq_domain *domain,
  3574. unsigned int virq,
  3575. unsigned int nr_irqs)
  3576. {
  3577. struct its_vm *vm = domain->host_data;
  3578. int i;
  3579. irq_domain_free_irqs_parent(domain, virq, nr_irqs);
  3580. for (i = 0; i < nr_irqs; i++) {
  3581. struct irq_data *data = irq_domain_get_irq_data(domain,
  3582. virq + i);
  3583. struct its_vpe *vpe = irq_data_get_irq_chip_data(data);
  3584. BUG_ON(vm != vpe->its_vm);
  3585. clear_bit(data->hwirq, vm->db_bitmap);
  3586. its_vpe_teardown(vpe);
  3587. irq_domain_reset_irq_data(data);
  3588. }
  3589. if (bitmap_empty(vm->db_bitmap, vm->nr_db_lpis)) {
  3590. its_lpi_free(vm->db_bitmap, vm->db_lpi_base, vm->nr_db_lpis);
  3591. its_free_prop_table(vm->vprop_page);
  3592. }
  3593. }
  3594. static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
  3595. unsigned int nr_irqs, void *args)
  3596. {
  3597. struct irq_chip *irqchip = &its_vpe_irq_chip;
  3598. struct its_vm *vm = args;
  3599. unsigned long *bitmap;
  3600. struct page *vprop_page;
  3601. int base, nr_ids, i, err = 0;
  3602. BUG_ON(!vm);
  3603. bitmap = its_lpi_alloc(roundup_pow_of_two(nr_irqs), &base, &nr_ids);
  3604. if (!bitmap)
  3605. return -ENOMEM;
  3606. if (nr_ids < nr_irqs) {
  3607. its_lpi_free(bitmap, base, nr_ids);
  3608. return -ENOMEM;
  3609. }
  3610. vprop_page = its_allocate_prop_table(GFP_KERNEL);
  3611. if (!vprop_page) {
  3612. its_lpi_free(bitmap, base, nr_ids);
  3613. return -ENOMEM;
  3614. }
  3615. vm->db_bitmap = bitmap;
  3616. vm->db_lpi_base = base;
  3617. vm->nr_db_lpis = nr_ids;
  3618. vm->vprop_page = vprop_page;
  3619. if (gic_rdists->has_rvpeid)
  3620. irqchip = &its_vpe_4_1_irq_chip;
  3621. for (i = 0; i < nr_irqs; i++) {
  3622. vm->vpes[i]->vpe_db_lpi = base + i;
  3623. err = its_vpe_init(vm->vpes[i]);
  3624. if (err)
  3625. break;
  3626. err = its_irq_gic_domain_alloc(domain, virq + i,
  3627. vm->vpes[i]->vpe_db_lpi);
  3628. if (err)
  3629. break;
  3630. irq_domain_set_hwirq_and_chip(domain, virq + i, i,
  3631. irqchip, vm->vpes[i]);
  3632. set_bit(i, bitmap);
  3633. }
  3634. if (err) {
  3635. if (i > 0)
  3636. its_vpe_irq_domain_free(domain, virq, i);
  3637. its_lpi_free(bitmap, base, nr_ids);
  3638. its_free_prop_table(vprop_page);
  3639. }
  3640. return err;
  3641. }
  3642. static int its_vpe_irq_domain_activate(struct irq_domain *domain,
  3643. struct irq_data *d, bool reserve)
  3644. {
  3645. struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
  3646. struct its_node *its;
  3647. /*
  3648. * If we use the list map, we issue VMAPP on demand... Unless
  3649. * we're on a GICv4.1 and we eagerly map the VPE on all ITSs
  3650. * so that VSGIs can work.
  3651. */
  3652. if (!gic_requires_eager_mapping())
  3653. return 0;
  3654. /* Map the VPE to the first possible CPU */
  3655. vpe->col_idx = cpumask_first(cpu_online_mask);
  3656. list_for_each_entry(its, &its_nodes, entry) {
  3657. if (!is_v4(its))
  3658. continue;
  3659. its_send_vmapp(its, vpe, true);
  3660. its_send_vinvall(its, vpe);
  3661. }
  3662. irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx));
  3663. return 0;
  3664. }
  3665. static void its_vpe_irq_domain_deactivate(struct irq_domain *domain,
  3666. struct irq_data *d)
  3667. {
  3668. struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
  3669. struct its_node *its;
  3670. /*
  3671. * If we use the list map on GICv4.0, we unmap the VPE once no
  3672. * VLPIs are associated with the VM.
  3673. */
  3674. if (!gic_requires_eager_mapping())
  3675. return;
  3676. list_for_each_entry(its, &its_nodes, entry) {
  3677. if (!is_v4(its))
  3678. continue;
  3679. its_send_vmapp(its, vpe, false);
  3680. }
  3681. }
  3682. static const struct irq_domain_ops its_vpe_domain_ops = {
  3683. .alloc = its_vpe_irq_domain_alloc,
  3684. .free = its_vpe_irq_domain_free,
  3685. .activate = its_vpe_irq_domain_activate,
  3686. .deactivate = its_vpe_irq_domain_deactivate,
  3687. };
  3688. static int its_force_quiescent(void __iomem *base)
  3689. {
  3690. u32 count = 1000000; /* 1s */
  3691. u32 val;
  3692. val = readl_relaxed(base + GITS_CTLR);
  3693. /*
  3694. * GIC architecture specification requires the ITS to be both
  3695. * disabled and quiescent for writes to GITS_BASER<n> or
  3696. * GITS_CBASER to not have UNPREDICTABLE results.
  3697. */
  3698. if ((val & GITS_CTLR_QUIESCENT) && !(val & GITS_CTLR_ENABLE))
  3699. return 0;
  3700. /* Disable the generation of all interrupts to this ITS */
  3701. val &= ~(GITS_CTLR_ENABLE | GITS_CTLR_ImDe);
  3702. writel_relaxed(val, base + GITS_CTLR);
  3703. /* Poll GITS_CTLR and wait until ITS becomes quiescent */
  3704. while (1) {
  3705. val = readl_relaxed(base + GITS_CTLR);
  3706. if (val & GITS_CTLR_QUIESCENT)
  3707. return 0;
  3708. count--;
  3709. if (!count)
  3710. return -EBUSY;
  3711. cpu_relax();
  3712. udelay(1);
  3713. }
  3714. }
  3715. static bool __maybe_unused its_enable_quirk_cavium_22375(void *data)
  3716. {
  3717. struct its_node *its = data;
  3718. /* erratum 22375: only alloc 8MB table size (20 bits) */
  3719. its->typer &= ~GITS_TYPER_DEVBITS;
  3720. its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, 20 - 1);
  3721. its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_22375;
  3722. return true;
  3723. }
  3724. static bool __maybe_unused its_enable_quirk_cavium_23144(void *data)
  3725. {
  3726. struct its_node *its = data;
  3727. its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_23144;
  3728. return true;
  3729. }
  3730. static bool __maybe_unused its_enable_quirk_qdf2400_e0065(void *data)
  3731. {
  3732. struct its_node *its = data;
  3733. /* On QDF2400, the size of the ITE is 16Bytes */
  3734. its->typer &= ~GITS_TYPER_ITT_ENTRY_SIZE;
  3735. its->typer |= FIELD_PREP(GITS_TYPER_ITT_ENTRY_SIZE, 16 - 1);
  3736. return true;
  3737. }
  3738. static u64 its_irq_get_msi_base_pre_its(struct its_device *its_dev)
  3739. {
  3740. struct its_node *its = its_dev->its;
  3741. /*
  3742. * The Socionext Synquacer SoC has a so-called 'pre-ITS',
  3743. * which maps 32-bit writes targeted at a separate window of
  3744. * size '4 << device_id_bits' onto writes to GITS_TRANSLATER
  3745. * with device ID taken from bits [device_id_bits + 1:2] of
  3746. * the window offset.
  3747. */
  3748. return its->pre_its_base + (its_dev->device_id << 2);
  3749. }
  3750. static bool __maybe_unused its_enable_quirk_socionext_synquacer(void *data)
  3751. {
  3752. struct its_node *its = data;
  3753. u32 pre_its_window[2];
  3754. u32 ids;
  3755. if (!fwnode_property_read_u32_array(its->fwnode_handle,
  3756. "socionext,synquacer-pre-its",
  3757. pre_its_window,
  3758. ARRAY_SIZE(pre_its_window))) {
  3759. its->pre_its_base = pre_its_window[0];
  3760. its->get_msi_base = its_irq_get_msi_base_pre_its;
  3761. ids = ilog2(pre_its_window[1]) - 2;
  3762. if (device_ids(its) > ids) {
  3763. its->typer &= ~GITS_TYPER_DEVBITS;
  3764. its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, ids - 1);
  3765. }
  3766. /* the pre-ITS breaks isolation, so disable MSI remapping */
  3767. its->msi_domain_flags &= ~IRQ_DOMAIN_FLAG_MSI_REMAP;
  3768. return true;
  3769. }
  3770. return false;
  3771. }
  3772. static bool __maybe_unused its_enable_quirk_hip07_161600802(void *data)
  3773. {
  3774. struct its_node *its = data;
  3775. /*
  3776. * Hip07 insists on using the wrong address for the VLPI
  3777. * page. Trick it into doing the right thing...
  3778. */
  3779. its->vlpi_redist_offset = SZ_128K;
  3780. return true;
  3781. }
  3782. static const struct gic_quirk its_quirks[] = {
  3783. #ifdef CONFIG_CAVIUM_ERRATUM_22375
  3784. {
  3785. .desc = "ITS: Cavium errata 22375, 24313",
  3786. .iidr = 0xa100034c, /* ThunderX pass 1.x */
  3787. .mask = 0xffff0fff,
  3788. .init = its_enable_quirk_cavium_22375,
  3789. },
  3790. #endif
  3791. #ifdef CONFIG_CAVIUM_ERRATUM_23144
  3792. {
  3793. .desc = "ITS: Cavium erratum 23144",
  3794. .iidr = 0xa100034c, /* ThunderX pass 1.x */
  3795. .mask = 0xffff0fff,
  3796. .init = its_enable_quirk_cavium_23144,
  3797. },
  3798. #endif
  3799. #ifdef CONFIG_QCOM_QDF2400_ERRATUM_0065
  3800. {
  3801. .desc = "ITS: QDF2400 erratum 0065",
  3802. .iidr = 0x00001070, /* QDF2400 ITS rev 1.x */
  3803. .mask = 0xffffffff,
  3804. .init = its_enable_quirk_qdf2400_e0065,
  3805. },
  3806. #endif
  3807. #ifdef CONFIG_SOCIONEXT_SYNQUACER_PREITS
  3808. {
  3809. /*
  3810. * The Socionext Synquacer SoC incorporates ARM's own GIC-500
  3811. * implementation, but with a 'pre-ITS' added that requires
  3812. * special handling in software.
  3813. */
  3814. .desc = "ITS: Socionext Synquacer pre-ITS",
  3815. .iidr = 0x0001143b,
  3816. .mask = 0xffffffff,
  3817. .init = its_enable_quirk_socionext_synquacer,
  3818. },
  3819. #endif
  3820. #ifdef CONFIG_HISILICON_ERRATUM_161600802
  3821. {
  3822. .desc = "ITS: Hip07 erratum 161600802",
  3823. .iidr = 0x00000004,
  3824. .mask = 0xffffffff,
  3825. .init = its_enable_quirk_hip07_161600802,
  3826. },
  3827. #endif
  3828. {
  3829. }
  3830. };
  3831. static void its_enable_quirks(struct its_node *its)
  3832. {
  3833. u32 iidr = readl_relaxed(its->base + GITS_IIDR);
  3834. gic_enable_quirks(iidr, its_quirks, its);
  3835. }
  3836. static int its_save_disable(void)
  3837. {
  3838. struct its_node *its;
  3839. int err = 0;
  3840. raw_spin_lock(&its_lock);
  3841. list_for_each_entry(its, &its_nodes, entry) {
  3842. void __iomem *base;
  3843. base = its->base;
  3844. its->ctlr_save = readl_relaxed(base + GITS_CTLR);
  3845. err = its_force_quiescent(base);
  3846. if (err) {
  3847. pr_err("ITS@%pa: failed to quiesce: %d\n",
  3848. &its->phys_base, err);
  3849. writel_relaxed(its->ctlr_save, base + GITS_CTLR);
  3850. goto err;
  3851. }
  3852. its->cbaser_save = gits_read_cbaser(base + GITS_CBASER);
  3853. }
  3854. err:
  3855. if (err) {
  3856. list_for_each_entry_continue_reverse(its, &its_nodes, entry) {
  3857. void __iomem *base;
  3858. base = its->base;
  3859. writel_relaxed(its->ctlr_save, base + GITS_CTLR);
  3860. }
  3861. }
  3862. raw_spin_unlock(&its_lock);
  3863. return err;
  3864. }
  3865. static void its_restore_enable(void)
  3866. {
  3867. struct its_node *its;
  3868. int ret;
  3869. raw_spin_lock(&its_lock);
  3870. list_for_each_entry(its, &its_nodes, entry) {
  3871. void __iomem *base;
  3872. int i;
  3873. base = its->base;
  3874. /*
  3875. * Make sure that the ITS is disabled. If it fails to quiesce,
  3876. * don't restore it since writing to CBASER or BASER<n>
  3877. * registers is undefined according to the GIC v3 ITS
  3878. * Specification.
  3879. *
  3880. * Firmware resuming with the ITS enabled is terminally broken.
  3881. */
  3882. WARN_ON(readl_relaxed(base + GITS_CTLR) & GITS_CTLR_ENABLE);
  3883. ret = its_force_quiescent(base);
  3884. if (ret) {
  3885. pr_err("ITS@%pa: failed to quiesce on resume: %d\n",
  3886. &its->phys_base, ret);
  3887. continue;
  3888. }
  3889. gits_write_cbaser(its->cbaser_save, base + GITS_CBASER);
  3890. /*
  3891. * Writing CBASER resets CREADR to 0, so make CWRITER and
  3892. * cmd_write line up with it.
  3893. */
  3894. its->cmd_write = its->cmd_base;
  3895. gits_write_cwriter(0, base + GITS_CWRITER);
  3896. /* Restore GITS_BASER from the value cache. */
  3897. for (i = 0; i < GITS_BASER_NR_REGS; i++) {
  3898. struct its_baser *baser = &its->tables[i];
  3899. if (!(baser->val & GITS_BASER_VALID))
  3900. continue;
  3901. its_write_baser(its, baser, baser->val);
  3902. }
  3903. writel_relaxed(its->ctlr_save, base + GITS_CTLR);
  3904. /*
  3905. * Reinit the collection if it's stored in the ITS. This is
  3906. * indicated by the col_id being less than the HCC field.
  3907. * CID < HCC as specified in the GIC v3 Documentation.
  3908. */
  3909. if (its->collections[smp_processor_id()].col_id <
  3910. GITS_TYPER_HCC(gic_read_typer(base + GITS_TYPER)))
  3911. its_cpu_init_collection(its);
  3912. }
  3913. raw_spin_unlock(&its_lock);
  3914. }
  3915. static struct syscore_ops its_syscore_ops = {
  3916. .suspend = its_save_disable,
  3917. .resume = its_restore_enable,
  3918. };
  3919. static int its_init_domain(struct fwnode_handle *handle, struct its_node *its)
  3920. {
  3921. struct irq_domain *inner_domain;
  3922. struct msi_domain_info *info;
  3923. info = kzalloc(sizeof(*info), GFP_KERNEL);
  3924. if (!info)
  3925. return -ENOMEM;
  3926. inner_domain = irq_domain_create_tree(handle, &its_domain_ops, its);
  3927. if (!inner_domain) {
  3928. kfree(info);
  3929. return -ENOMEM;
  3930. }
  3931. inner_domain->parent = its_parent;
  3932. irq_domain_update_bus_token(inner_domain, DOMAIN_BUS_NEXUS);
  3933. inner_domain->flags |= its->msi_domain_flags;
  3934. info->ops = &its_msi_domain_ops;
  3935. info->data = its;
  3936. inner_domain->host_data = info;
  3937. return 0;
  3938. }
  3939. static int its_init_vpe_domain(void)
  3940. {
  3941. struct its_node *its;
  3942. u32 devid;
  3943. int entries;
  3944. if (gic_rdists->has_direct_lpi) {
  3945. pr_info("ITS: Using DirectLPI for VPE invalidation\n");
  3946. return 0;
  3947. }
  3948. /* Any ITS will do, even if not v4 */
  3949. its = list_first_entry(&its_nodes, struct its_node, entry);
  3950. entries = roundup_pow_of_two(nr_cpu_ids);
  3951. vpe_proxy.vpes = kcalloc(entries, sizeof(*vpe_proxy.vpes),
  3952. GFP_KERNEL);
  3953. if (!vpe_proxy.vpes) {
  3954. pr_err("ITS: Can't allocate GICv4 proxy device array\n");
  3955. return -ENOMEM;
  3956. }
  3957. /* Use the last possible DevID */
  3958. devid = GENMASK(device_ids(its) - 1, 0);
  3959. vpe_proxy.dev = its_create_device(its, devid, entries, false);
  3960. if (!vpe_proxy.dev) {
  3961. kfree(vpe_proxy.vpes);
  3962. pr_err("ITS: Can't allocate GICv4 proxy device\n");
  3963. return -ENOMEM;
  3964. }
  3965. BUG_ON(entries > vpe_proxy.dev->nr_ites);
  3966. raw_spin_lock_init(&vpe_proxy.lock);
  3967. vpe_proxy.next_victim = 0;
  3968. pr_info("ITS: Allocated DevID %x as GICv4 proxy device (%d slots)\n",
  3969. devid, vpe_proxy.dev->nr_ites);
  3970. return 0;
  3971. }
  3972. static int __init its_compute_its_list_map(struct resource *res,
  3973. void __iomem *its_base)
  3974. {
  3975. int its_number;
  3976. u32 ctlr;
  3977. /*
  3978. * This is assumed to be done early enough that we're
  3979. * guaranteed to be single-threaded, hence no
  3980. * locking. Should this change, we should address
  3981. * this.
  3982. */
  3983. its_number = find_first_zero_bit(&its_list_map, GICv4_ITS_LIST_MAX);
  3984. if (its_number >= GICv4_ITS_LIST_MAX) {
  3985. pr_err("ITS@%pa: No ITSList entry available!\n",
  3986. &res->start);
  3987. return -EINVAL;
  3988. }
  3989. ctlr = readl_relaxed(its_base + GITS_CTLR);
  3990. ctlr &= ~GITS_CTLR_ITS_NUMBER;
  3991. ctlr |= its_number << GITS_CTLR_ITS_NUMBER_SHIFT;
  3992. writel_relaxed(ctlr, its_base + GITS_CTLR);
  3993. ctlr = readl_relaxed(its_base + GITS_CTLR);
  3994. if ((ctlr & GITS_CTLR_ITS_NUMBER) != (its_number << GITS_CTLR_ITS_NUMBER_SHIFT)) {
  3995. its_number = ctlr & GITS_CTLR_ITS_NUMBER;
  3996. its_number >>= GITS_CTLR_ITS_NUMBER_SHIFT;
  3997. }
  3998. if (test_and_set_bit(its_number, &its_list_map)) {
  3999. pr_err("ITS@%pa: Duplicate ITSList entry %d\n",
  4000. &res->start, its_number);
  4001. return -EINVAL;
  4002. }
  4003. return its_number;
  4004. }
  4005. static int __init its_probe_one(struct resource *res,
  4006. struct fwnode_handle *handle, int numa_node)
  4007. {
  4008. struct its_node *its;
  4009. void __iomem *its_base;
  4010. u32 val, ctlr;
  4011. u64 baser, tmp, typer;
  4012. struct page *page;
  4013. int err;
  4014. its_base = ioremap(res->start, SZ_64K);
  4015. if (!its_base) {
  4016. pr_warn("ITS@%pa: Unable to map ITS registers\n", &res->start);
  4017. return -ENOMEM;
  4018. }
  4019. val = readl_relaxed(its_base + GITS_PIDR2) & GIC_PIDR2_ARCH_MASK;
  4020. if (val != 0x30 && val != 0x40) {
  4021. pr_warn("ITS@%pa: No ITS detected, giving up\n", &res->start);
  4022. err = -ENODEV;
  4023. goto out_unmap;
  4024. }
  4025. err = its_force_quiescent(its_base);
  4026. if (err) {
  4027. pr_warn("ITS@%pa: Failed to quiesce, giving up\n", &res->start);
  4028. goto out_unmap;
  4029. }
  4030. pr_info("ITS %pR\n", res);
  4031. its = kzalloc(sizeof(*its), GFP_KERNEL);
  4032. if (!its) {
  4033. err = -ENOMEM;
  4034. goto out_unmap;
  4035. }
  4036. raw_spin_lock_init(&its->lock);
  4037. mutex_init(&its->dev_alloc_lock);
  4038. INIT_LIST_HEAD(&its->entry);
  4039. INIT_LIST_HEAD(&its->its_device_list);
  4040. typer = gic_read_typer(its_base + GITS_TYPER);
  4041. its->typer = typer;
  4042. its->base = its_base;
  4043. its->phys_base = res->start;
  4044. if (is_v4(its)) {
  4045. if (!(typer & GITS_TYPER_VMOVP)) {
  4046. err = its_compute_its_list_map(res, its_base);
  4047. if (err < 0)
  4048. goto out_free_its;
  4049. its->list_nr = err;
  4050. pr_info("ITS@%pa: Using ITS number %d\n",
  4051. &res->start, err);
  4052. } else {
  4053. pr_info("ITS@%pa: Single VMOVP capable\n", &res->start);
  4054. }
  4055. if (is_v4_1(its)) {
  4056. u32 svpet = FIELD_GET(GITS_TYPER_SVPET, typer);
  4057. its->sgir_base = ioremap(res->start + SZ_128K, SZ_64K);
  4058. if (!its->sgir_base) {
  4059. err = -ENOMEM;
  4060. goto out_free_its;
  4061. }
  4062. its->mpidr = readl_relaxed(its_base + GITS_MPIDR);
  4063. pr_info("ITS@%pa: Using GICv4.1 mode %08x %08x\n",
  4064. &res->start, its->mpidr, svpet);
  4065. }
  4066. }
  4067. its->numa_node = numa_node;
  4068. page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO,
  4069. get_order(ITS_CMD_QUEUE_SZ));
  4070. if (!page) {
  4071. err = -ENOMEM;
  4072. goto out_unmap_sgir;
  4073. }
  4074. its->cmd_base = (void *)page_address(page);
  4075. its->cmd_write = its->cmd_base;
  4076. its->fwnode_handle = handle;
  4077. its->get_msi_base = its_irq_get_msi_base;
  4078. its->msi_domain_flags = IRQ_DOMAIN_FLAG_MSI_REMAP;
  4079. its_enable_quirks(its);
  4080. err = its_alloc_tables(its);
  4081. if (err)
  4082. goto out_free_cmd;
  4083. err = its_alloc_collections(its);
  4084. if (err)
  4085. goto out_free_tables;
  4086. baser = (virt_to_phys(its->cmd_base) |
  4087. GITS_CBASER_RaWaWb |
  4088. GITS_CBASER_InnerShareable |
  4089. (ITS_CMD_QUEUE_SZ / SZ_4K - 1) |
  4090. GITS_CBASER_VALID);
  4091. gits_write_cbaser(baser, its->base + GITS_CBASER);
  4092. tmp = gits_read_cbaser(its->base + GITS_CBASER);
  4093. if ((tmp ^ baser) & GITS_CBASER_SHAREABILITY_MASK) {
  4094. if (!(tmp & GITS_CBASER_SHAREABILITY_MASK)) {
  4095. /*
  4096. * The HW reports non-shareable, we must
  4097. * remove the cacheability attributes as
  4098. * well.
  4099. */
  4100. baser &= ~(GITS_CBASER_SHAREABILITY_MASK |
  4101. GITS_CBASER_CACHEABILITY_MASK);
  4102. baser |= GITS_CBASER_nC;
  4103. gits_write_cbaser(baser, its->base + GITS_CBASER);
  4104. }
  4105. pr_info("ITS: using cache flushing for cmd queue\n");
  4106. its->flags |= ITS_FLAGS_CMDQ_NEEDS_FLUSHING;
  4107. }
  4108. gits_write_cwriter(0, its->base + GITS_CWRITER);
  4109. ctlr = readl_relaxed(its->base + GITS_CTLR);
  4110. ctlr |= GITS_CTLR_ENABLE;
  4111. if (is_v4(its))
  4112. ctlr |= GITS_CTLR_ImDe;
  4113. writel_relaxed(ctlr, its->base + GITS_CTLR);
  4114. err = its_init_domain(handle, its);
  4115. if (err)
  4116. goto out_free_tables;
  4117. raw_spin_lock(&its_lock);
  4118. list_add(&its->entry, &its_nodes);
  4119. raw_spin_unlock(&its_lock);
  4120. return 0;
  4121. out_free_tables:
  4122. its_free_tables(its);
  4123. out_free_cmd:
  4124. free_pages((unsigned long)its->cmd_base, get_order(ITS_CMD_QUEUE_SZ));
  4125. out_unmap_sgir:
  4126. if (its->sgir_base)
  4127. iounmap(its->sgir_base);
  4128. out_free_its:
  4129. kfree(its);
  4130. out_unmap:
  4131. iounmap(its_base);
  4132. pr_err("ITS@%pa: failed probing (%d)\n", &res->start, err);
  4133. return err;
  4134. }
  4135. static bool gic_rdists_supports_plpis(void)
  4136. {
  4137. return !!(gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER) & GICR_TYPER_PLPIS);
  4138. }
  4139. static int redist_disable_lpis(void)
  4140. {
  4141. void __iomem *rbase = gic_data_rdist_rd_base();
  4142. u64 timeout = USEC_PER_SEC;
  4143. u64 val;
  4144. if (!gic_rdists_supports_plpis()) {
  4145. pr_info("CPU%d: LPIs not supported\n", smp_processor_id());
  4146. return -ENXIO;
  4147. }
  4148. val = readl_relaxed(rbase + GICR_CTLR);
  4149. if (!(val & GICR_CTLR_ENABLE_LPIS))
  4150. return 0;
  4151. /*
  4152. * If coming via a CPU hotplug event, we don't need to disable
  4153. * LPIs before trying to re-enable them. They are already
  4154. * configured and all is well in the world.
  4155. *
  4156. * If running with preallocated tables, there is nothing to do.
  4157. */
  4158. if (gic_data_rdist()->lpi_enabled ||
  4159. (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED))
  4160. return 0;
  4161. /*
  4162. * From that point on, we only try to do some damage control.
  4163. */
  4164. pr_warn("GICv3: CPU%d: Booted with LPIs enabled, memory probably corrupted\n",
  4165. smp_processor_id());
  4166. add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
  4167. /* Disable LPIs */
  4168. val &= ~GICR_CTLR_ENABLE_LPIS;
  4169. writel_relaxed(val, rbase + GICR_CTLR);
  4170. /* Make sure any change to GICR_CTLR is observable by the GIC */
  4171. dsb(sy);
  4172. /*
  4173. * Software must observe RWP==0 after clearing GICR_CTLR.EnableLPIs
  4174. * from 1 to 0 before programming GICR_PEND{PROP}BASER registers.
  4175. * Error out if we time out waiting for RWP to clear.
  4176. */
  4177. while (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_RWP) {
  4178. if (!timeout) {
  4179. pr_err("CPU%d: Timeout while disabling LPIs\n",
  4180. smp_processor_id());
  4181. return -ETIMEDOUT;
  4182. }
  4183. udelay(1);
  4184. timeout--;
  4185. }
  4186. /*
  4187. * After it has been written to 1, it is IMPLEMENTATION
  4188. * DEFINED whether GICR_CTLR.EnableLPI becomes RES1 or can be
  4189. * cleared to 0. Error out if clearing the bit failed.
  4190. */
  4191. if (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_ENABLE_LPIS) {
  4192. pr_err("CPU%d: Failed to disable LPIs\n", smp_processor_id());
  4193. return -EBUSY;
  4194. }
  4195. return 0;
  4196. }
  4197. int its_cpu_init(void)
  4198. {
  4199. if (!list_empty(&its_nodes)) {
  4200. int ret;
  4201. ret = redist_disable_lpis();
  4202. if (ret)
  4203. return ret;
  4204. its_cpu_init_lpis();
  4205. its_cpu_init_collections();
  4206. }
  4207. return 0;
  4208. }
  4209. static const struct of_device_id its_device_id[] = {
  4210. { .compatible = "arm,gic-v3-its", },
  4211. {},
  4212. };
  4213. static int __init its_of_probe(struct device_node *node)
  4214. {
  4215. struct device_node *np;
  4216. struct resource res;
  4217. for (np = of_find_matching_node(node, its_device_id); np;
  4218. np = of_find_matching_node(np, its_device_id)) {
  4219. if (!of_device_is_available(np))
  4220. continue;
  4221. if (!of_property_read_bool(np, "msi-controller")) {
  4222. pr_warn("%pOF: no msi-controller property, ITS ignored\n",
  4223. np);
  4224. continue;
  4225. }
  4226. if (of_address_to_resource(np, 0, &res)) {
  4227. pr_warn("%pOF: no regs?\n", np);
  4228. continue;
  4229. }
  4230. its_probe_one(&res, &np->fwnode, of_node_to_nid(np));
  4231. }
  4232. return 0;
  4233. }
  4234. #ifdef CONFIG_ACPI
  4235. #define ACPI_GICV3_ITS_MEM_SIZE (SZ_128K)
  4236. #ifdef CONFIG_ACPI_NUMA
  4237. struct its_srat_map {
  4238. /* numa node id */
  4239. u32 numa_node;
  4240. /* GIC ITS ID */
  4241. u32 its_id;
  4242. };
  4243. static struct its_srat_map *its_srat_maps __initdata;
  4244. static int its_in_srat __initdata;
  4245. static int __init acpi_get_its_numa_node(u32 its_id)
  4246. {
  4247. int i;
  4248. for (i = 0; i < its_in_srat; i++) {
  4249. if (its_id == its_srat_maps[i].its_id)
  4250. return its_srat_maps[i].numa_node;
  4251. }
  4252. return NUMA_NO_NODE;
  4253. }
  4254. static int __init gic_acpi_match_srat_its(union acpi_subtable_headers *header,
  4255. const unsigned long end)
  4256. {
  4257. return 0;
  4258. }
  4259. static int __init gic_acpi_parse_srat_its(union acpi_subtable_headers *header,
  4260. const unsigned long end)
  4261. {
  4262. int node;
  4263. struct acpi_srat_gic_its_affinity *its_affinity;
  4264. its_affinity = (struct acpi_srat_gic_its_affinity *)header;
  4265. if (!its_affinity)
  4266. return -EINVAL;
  4267. if (its_affinity->header.length < sizeof(*its_affinity)) {
  4268. pr_err("SRAT: Invalid header length %d in ITS affinity\n",
  4269. its_affinity->header.length);
  4270. return -EINVAL;
  4271. }
  4272. /*
  4273. * Note that in theory a new proximity node could be created by this
  4274. * entry as it is an SRAT resource allocation structure.
  4275. * We do not currently support doing so.
  4276. */
  4277. node = pxm_to_node(its_affinity->proximity_domain);
  4278. if (node == NUMA_NO_NODE || node >= MAX_NUMNODES) {
  4279. pr_err("SRAT: Invalid NUMA node %d in ITS affinity\n", node);
  4280. return 0;
  4281. }
  4282. its_srat_maps[its_in_srat].numa_node = node;
  4283. its_srat_maps[its_in_srat].its_id = its_affinity->its_id;
  4284. its_in_srat++;
  4285. pr_info("SRAT: PXM %d -> ITS %d -> Node %d\n",
  4286. its_affinity->proximity_domain, its_affinity->its_id, node);
  4287. return 0;
  4288. }
  4289. static void __init acpi_table_parse_srat_its(void)
  4290. {
  4291. int count;
  4292. count = acpi_table_parse_entries(ACPI_SIG_SRAT,
  4293. sizeof(struct acpi_table_srat),
  4294. ACPI_SRAT_TYPE_GIC_ITS_AFFINITY,
  4295. gic_acpi_match_srat_its, 0);
  4296. if (count <= 0)
  4297. return;
  4298. its_srat_maps = kmalloc_array(count, sizeof(struct its_srat_map),
  4299. GFP_KERNEL);
  4300. if (!its_srat_maps) {
  4301. pr_warn("SRAT: Failed to allocate memory for its_srat_maps!\n");
  4302. return;
  4303. }
  4304. acpi_table_parse_entries(ACPI_SIG_SRAT,
  4305. sizeof(struct acpi_table_srat),
  4306. ACPI_SRAT_TYPE_GIC_ITS_AFFINITY,
  4307. gic_acpi_parse_srat_its, 0);
  4308. }
  4309. /* free the its_srat_maps after ITS probing */
  4310. static void __init acpi_its_srat_maps_free(void)
  4311. {
  4312. kfree(its_srat_maps);
  4313. }
  4314. #else
  4315. static void __init acpi_table_parse_srat_its(void) { }
  4316. static int __init acpi_get_its_numa_node(u32 its_id) { return NUMA_NO_NODE; }
  4317. static void __init acpi_its_srat_maps_free(void) { }
  4318. #endif
  4319. static int __init gic_acpi_parse_madt_its(union acpi_subtable_headers *header,
  4320. const unsigned long end)
  4321. {
  4322. struct acpi_madt_generic_translator *its_entry;
  4323. struct fwnode_handle *dom_handle;
  4324. struct resource res;
  4325. int err;
  4326. its_entry = (struct acpi_madt_generic_translator *)header;
  4327. memset(&res, 0, sizeof(res));
  4328. res.start = its_entry->base_address;
  4329. res.end = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1;
  4330. res.flags = IORESOURCE_MEM;
  4331. dom_handle = irq_domain_alloc_fwnode(&res.start);
  4332. if (!dom_handle) {
  4333. pr_err("ITS@%pa: Unable to allocate GICv3 ITS domain token\n",
  4334. &res.start);
  4335. return -ENOMEM;
  4336. }
  4337. err = iort_register_domain_token(its_entry->translation_id, res.start,
  4338. dom_handle);
  4339. if (err) {
  4340. pr_err("ITS@%pa: Unable to register GICv3 ITS domain token (ITS ID %d) to IORT\n",
  4341. &res.start, its_entry->translation_id);
  4342. goto dom_err;
  4343. }
  4344. err = its_probe_one(&res, dom_handle,
  4345. acpi_get_its_numa_node(its_entry->translation_id));
  4346. if (!err)
  4347. return 0;
  4348. iort_deregister_domain_token(its_entry->translation_id);
  4349. dom_err:
  4350. irq_domain_free_fwnode(dom_handle);
  4351. return err;
  4352. }
  4353. static void __init its_acpi_probe(void)
  4354. {
  4355. acpi_table_parse_srat_its();
  4356. acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR,
  4357. gic_acpi_parse_madt_its, 0);
  4358. acpi_its_srat_maps_free();
  4359. }
  4360. #else
  4361. static void __init its_acpi_probe(void) { }
  4362. #endif
  4363. int __init its_init(struct fwnode_handle *handle, struct rdists *rdists,
  4364. struct irq_domain *parent_domain)
  4365. {
  4366. struct device_node *of_node;
  4367. struct its_node *its;
  4368. bool has_v4 = false;
  4369. bool has_v4_1 = false;
  4370. int err;
  4371. gic_rdists = rdists;
  4372. its_parent = parent_domain;
  4373. of_node = to_of_node(handle);
  4374. if (of_node)
  4375. its_of_probe(of_node);
  4376. else
  4377. its_acpi_probe();
  4378. if (list_empty(&its_nodes)) {
  4379. pr_warn("ITS: No ITS available, not enabling LPIs\n");
  4380. return -ENXIO;
  4381. }
  4382. err = allocate_lpi_tables();
  4383. if (err)
  4384. return err;
  4385. list_for_each_entry(its, &its_nodes, entry) {
  4386. has_v4 |= is_v4(its);
  4387. has_v4_1 |= is_v4_1(its);
  4388. }
  4389. /* Don't bother with inconsistent systems */
  4390. if (WARN_ON(!has_v4_1 && rdists->has_rvpeid))
  4391. rdists->has_rvpeid = false;
  4392. if (has_v4 & rdists->has_vlpis) {
  4393. const struct irq_domain_ops *sgi_ops;
  4394. if (has_v4_1)
  4395. sgi_ops = &its_sgi_domain_ops;
  4396. else
  4397. sgi_ops = NULL;
  4398. if (its_init_vpe_domain() ||
  4399. its_init_v4(parent_domain, &its_vpe_domain_ops, sgi_ops)) {
  4400. rdists->has_vlpis = false;
  4401. pr_err("ITS: Disabling GICv4 support\n");
  4402. }
  4403. }
  4404. register_syscore_ops(&its_syscore_ops);
  4405. return 0;
  4406. }