pci.c 174 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654565556565657565856595660566156625663566456655666566756685669567056715672567356745675567656775678567956805681568256835684568556865687568856895690569156925693569456955696569756985699570057015702570357045705570657075708570957105711571257135714571557165717571857195720572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759576057615762576357645765576657675768576957705771577257735774577557765777577857795780578157825783578457855786578757885789579057915792579357945795579657975798579958005801580258035804580558065807580858095810581158125813581458155816581758185819582058215822582358245825582658275828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855585658575858585958605861586258635864586558665867586858695870587158725873587458755876587758785879588058815882588358845885588658875888588958905891589258935894589558965897589858995900590159025903590459055906590759085909591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936593759385939594059415942594359445945594659475948594959505951595259535954595559565957595859595960596159625963596459655966596759685969597059715972597359745975597659775978597959805981598259835984598559865987598859895990599159925993599459955996599759985999600060016002600360046005600660076008600960106011601260136014601560166017601860196020602160226023602460256026602760286029603060316032603360346035603660376038603960406041604260436044604560466047604860496050605160526053605460556056605760586059606060616062606360646065606660676068606960706071607260736074607560766077607860796080608160826083608460856086608760886089609060916092609360946095609660976098609961006101610261036104610561066107610861096110611161126113611461156116611761186119612061216122612361246125612661276128612961306131613261336134613561366137613861396140614161426143614461456146614761486149615061516152615361546155615661576158615961606161616261636164616561666167616861696170617161726173617461756176617761786179618061816182618361846185618661876188618961906191619261936194619561966197619861996200620162026203620462056206620762086209621062116212621362146215621662176218621962206221622262236224622562266227622862296230623162326233623462356236623762386239624062416242624362446245624662476248624962506251625262536254625562566257625862596260626162626263626462656266626762686269627062716272627362746275627662776278627962806281628262836284628562866287628862896290629162926293629462956296629762986299630063016302630363046305630663076308630963106311631263136314631563166317631863196320632163226323632463256326632763286329633063316332633363346335633663376338633963406341634263436344634563466347634863496350635163526353635463556356635763586359636063616362636363646365636663676368636963706371637263736374637563766377637863796380638163826383638463856386638763886389639063916392639363946395639663976398639964006401640264036404640564066407640864096410641164126413641464156416641764186419642064216422642364246425642664276428642964306431643264336434643564366437643864396440644164426443644464456446644764486449645064516452645364546455645664576458645964606461646264636464646564666467646864696470647164726473647464756476647764786479648064816482648364846485648664876488648964906491649264936494649564966497649864996500650165026503650465056506650765086509651065116512651365146515651665176518651965206521652265236524652565266527652865296530653165326533653465356536653765386539654065416542654365446545654665476548654965506551655265536554655565566557655865596560656165626563656465656566656765686569657065716572657365746575657665776578657965806581658265836584658565866587658865896590659165926593659465956596659765986599660066016602660366046605
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * PCI Bus Services, see include/linux/pci.h for further explanation.
  4. *
  5. * Copyright 1993 -- 1997 Drew Eckhardt, Frederic Potter,
  6. * David Mosberger-Tang
  7. *
  8. * Copyright 1997 -- 2000 Martin Mares <mj@ucw.cz>
  9. */
  10. #include <linux/acpi.h>
  11. #include <linux/kernel.h>
  12. #include <linux/delay.h>
  13. #include <linux/dmi.h>
  14. #include <linux/init.h>
  15. #include <linux/msi.h>
  16. #include <linux/of.h>
  17. #include <linux/pci.h>
  18. #include <linux/pm.h>
  19. #include <linux/slab.h>
  20. #include <linux/module.h>
  21. #include <linux/spinlock.h>
  22. #include <linux/string.h>
  23. #include <linux/log2.h>
  24. #include <linux/logic_pio.h>
  25. #include <linux/pm_wakeup.h>
  26. #include <linux/interrupt.h>
  27. #include <linux/device.h>
  28. #include <linux/pm_runtime.h>
  29. #include <linux/pci_hotplug.h>
  30. #include <linux/vmalloc.h>
  31. #include <asm/dma.h>
  32. #include <linux/aer.h>
  33. #ifndef __GENKSYMS__
  34. #include <trace/hooks/pci.h>
  35. #endif
  36. #include "pci.h"
  37. DEFINE_MUTEX(pci_slot_mutex);
  38. const char *pci_power_names[] = {
  39. "error", "D0", "D1", "D2", "D3hot", "D3cold", "unknown",
  40. };
  41. EXPORT_SYMBOL_GPL(pci_power_names);
  42. int isa_dma_bridge_buggy;
  43. EXPORT_SYMBOL(isa_dma_bridge_buggy);
  44. int pci_pci_problems;
  45. EXPORT_SYMBOL(pci_pci_problems);
  46. unsigned int pci_pm_d3hot_delay;
  47. static void pci_pme_list_scan(struct work_struct *work);
  48. static LIST_HEAD(pci_pme_list);
  49. static DEFINE_MUTEX(pci_pme_list_mutex);
  50. static DECLARE_DELAYED_WORK(pci_pme_work, pci_pme_list_scan);
  51. struct pci_pme_device {
  52. struct list_head list;
  53. struct pci_dev *dev;
  54. };
  55. #define PME_TIMEOUT 1000 /* How long between PME checks */
  56. static void pci_dev_d3_sleep(struct pci_dev *dev)
  57. {
  58. unsigned int delay = dev->d3hot_delay;
  59. int err = -EOPNOTSUPP;
  60. if (delay < pci_pm_d3hot_delay)
  61. delay = pci_pm_d3hot_delay;
  62. if (delay) {
  63. trace_android_rvh_pci_d3_sleep(dev, delay, &err);
  64. if (err == -EOPNOTSUPP)
  65. msleep(delay);
  66. }
  67. }
  68. #ifdef CONFIG_PCI_DOMAINS
  69. int pci_domains_supported = 1;
  70. #endif
  71. #define DEFAULT_CARDBUS_IO_SIZE (256)
  72. #define DEFAULT_CARDBUS_MEM_SIZE (64*1024*1024)
  73. /* pci=cbmemsize=nnM,cbiosize=nn can override this */
  74. unsigned long pci_cardbus_io_size = DEFAULT_CARDBUS_IO_SIZE;
  75. unsigned long pci_cardbus_mem_size = DEFAULT_CARDBUS_MEM_SIZE;
  76. #define DEFAULT_HOTPLUG_IO_SIZE (256)
  77. #define DEFAULT_HOTPLUG_MMIO_SIZE (2*1024*1024)
  78. #define DEFAULT_HOTPLUG_MMIO_PREF_SIZE (2*1024*1024)
  79. /* hpiosize=nn can override this */
  80. unsigned long pci_hotplug_io_size = DEFAULT_HOTPLUG_IO_SIZE;
  81. /*
  82. * pci=hpmmiosize=nnM overrides non-prefetchable MMIO size,
  83. * pci=hpmmioprefsize=nnM overrides prefetchable MMIO size;
  84. * pci=hpmemsize=nnM overrides both
  85. */
  86. unsigned long pci_hotplug_mmio_size = DEFAULT_HOTPLUG_MMIO_SIZE;
  87. unsigned long pci_hotplug_mmio_pref_size = DEFAULT_HOTPLUG_MMIO_PREF_SIZE;
  88. #define DEFAULT_HOTPLUG_BUS_SIZE 1
  89. unsigned long pci_hotplug_bus_size = DEFAULT_HOTPLUG_BUS_SIZE;
  90. /* PCIe MPS/MRRS strategy; can be overridden by kernel command-line param */
  91. #ifdef CONFIG_PCIE_BUS_TUNE_OFF
  92. enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_TUNE_OFF;
  93. #elif defined CONFIG_PCIE_BUS_SAFE
  94. enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_SAFE;
  95. #elif defined CONFIG_PCIE_BUS_PERFORMANCE
  96. enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_PERFORMANCE;
  97. #elif defined CONFIG_PCIE_BUS_PEER2PEER
  98. enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_PEER2PEER;
  99. #else
  100. enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_DEFAULT;
  101. #endif
  102. /*
  103. * The default CLS is used if arch didn't set CLS explicitly and not
  104. * all pci devices agree on the same value. Arch can override either
  105. * the dfl or actual value as it sees fit. Don't forget this is
  106. * measured in 32-bit words, not bytes.
  107. */
  108. u8 pci_dfl_cache_line_size = L1_CACHE_BYTES >> 2;
  109. u8 pci_cache_line_size;
  110. /*
  111. * If we set up a device for bus mastering, we need to check the latency
  112. * timer as certain BIOSes forget to set it properly.
  113. */
  114. unsigned int pcibios_max_latency = 255;
  115. /* If set, the PCIe ARI capability will not be used. */
  116. static bool pcie_ari_disabled;
  117. /* If set, the PCIe ATS capability will not be used. */
  118. static bool pcie_ats_disabled;
  119. /* If set, the PCI config space of each device is printed during boot. */
  120. bool pci_early_dump;
  121. bool pci_ats_disabled(void)
  122. {
  123. return pcie_ats_disabled;
  124. }
  125. EXPORT_SYMBOL_GPL(pci_ats_disabled);
  126. /* Disable bridge_d3 for all PCIe ports */
  127. static bool pci_bridge_d3_disable;
  128. /* Force bridge_d3 for all PCIe ports */
  129. static bool pci_bridge_d3_force;
  130. static int __init pcie_port_pm_setup(char *str)
  131. {
  132. if (!strcmp(str, "off"))
  133. pci_bridge_d3_disable = true;
  134. else if (!strcmp(str, "force"))
  135. pci_bridge_d3_force = true;
  136. return 1;
  137. }
  138. __setup("pcie_port_pm=", pcie_port_pm_setup);
  139. /* Time to wait after a reset for device to become responsive */
  140. #define PCIE_RESET_READY_POLL_MS 60000
  141. /**
  142. * pci_bus_max_busnr - returns maximum PCI bus number of given bus' children
  143. * @bus: pointer to PCI bus structure to search
  144. *
  145. * Given a PCI bus, returns the highest PCI bus number present in the set
  146. * including the given PCI bus and its list of child PCI buses.
  147. */
  148. unsigned char pci_bus_max_busnr(struct pci_bus *bus)
  149. {
  150. struct pci_bus *tmp;
  151. unsigned char max, n;
  152. max = bus->busn_res.end;
  153. list_for_each_entry(tmp, &bus->children, node) {
  154. n = pci_bus_max_busnr(tmp);
  155. if (n > max)
  156. max = n;
  157. }
  158. return max;
  159. }
  160. EXPORT_SYMBOL_GPL(pci_bus_max_busnr);
  161. /**
  162. * pci_status_get_and_clear_errors - return and clear error bits in PCI_STATUS
  163. * @pdev: the PCI device
  164. *
  165. * Returns error bits set in PCI_STATUS and clears them.
  166. */
  167. int pci_status_get_and_clear_errors(struct pci_dev *pdev)
  168. {
  169. u16 status;
  170. int ret;
  171. ret = pci_read_config_word(pdev, PCI_STATUS, &status);
  172. if (ret != PCIBIOS_SUCCESSFUL)
  173. return -EIO;
  174. status &= PCI_STATUS_ERROR_BITS;
  175. if (status)
  176. pci_write_config_word(pdev, PCI_STATUS, status);
  177. return status;
  178. }
  179. EXPORT_SYMBOL_GPL(pci_status_get_and_clear_errors);
  180. #ifdef CONFIG_HAS_IOMEM
  181. void __iomem *pci_ioremap_bar(struct pci_dev *pdev, int bar)
  182. {
  183. struct resource *res = &pdev->resource[bar];
  184. /*
  185. * Make sure the BAR is actually a memory resource, not an IO resource
  186. */
  187. if (res->flags & IORESOURCE_UNSET || !(res->flags & IORESOURCE_MEM)) {
  188. pci_warn(pdev, "can't ioremap BAR %d: %pR\n", bar, res);
  189. return NULL;
  190. }
  191. return ioremap(res->start, resource_size(res));
  192. }
  193. EXPORT_SYMBOL_GPL(pci_ioremap_bar);
  194. void __iomem *pci_ioremap_wc_bar(struct pci_dev *pdev, int bar)
  195. {
  196. /*
  197. * Make sure the BAR is actually a memory resource, not an IO resource
  198. */
  199. if (!(pci_resource_flags(pdev, bar) & IORESOURCE_MEM)) {
  200. WARN_ON(1);
  201. return NULL;
  202. }
  203. return ioremap_wc(pci_resource_start(pdev, bar),
  204. pci_resource_len(pdev, bar));
  205. }
  206. EXPORT_SYMBOL_GPL(pci_ioremap_wc_bar);
  207. #endif
  208. /**
  209. * pci_dev_str_match_path - test if a path string matches a device
  210. * @dev: the PCI device to test
  211. * @path: string to match the device against
  212. * @endptr: pointer to the string after the match
  213. *
  214. * Test if a string (typically from a kernel parameter) formatted as a
  215. * path of device/function addresses matches a PCI device. The string must
  216. * be of the form:
  217. *
  218. * [<domain>:]<bus>:<device>.<func>[/<device>.<func>]*
  219. *
  220. * A path for a device can be obtained using 'lspci -t'. Using a path
  221. * is more robust against bus renumbering than using only a single bus,
  222. * device and function address.
  223. *
  224. * Returns 1 if the string matches the device, 0 if it does not and
  225. * a negative error code if it fails to parse the string.
  226. */
  227. static int pci_dev_str_match_path(struct pci_dev *dev, const char *path,
  228. const char **endptr)
  229. {
  230. int ret;
  231. int seg, bus, slot, func;
  232. char *wpath, *p;
  233. char end;
  234. *endptr = strchrnul(path, ';');
  235. wpath = kmemdup_nul(path, *endptr - path, GFP_ATOMIC);
  236. if (!wpath)
  237. return -ENOMEM;
  238. while (1) {
  239. p = strrchr(wpath, '/');
  240. if (!p)
  241. break;
  242. ret = sscanf(p, "/%x.%x%c", &slot, &func, &end);
  243. if (ret != 2) {
  244. ret = -EINVAL;
  245. goto free_and_exit;
  246. }
  247. if (dev->devfn != PCI_DEVFN(slot, func)) {
  248. ret = 0;
  249. goto free_and_exit;
  250. }
  251. /*
  252. * Note: we don't need to get a reference to the upstream
  253. * bridge because we hold a reference to the top level
  254. * device which should hold a reference to the bridge,
  255. * and so on.
  256. */
  257. dev = pci_upstream_bridge(dev);
  258. if (!dev) {
  259. ret = 0;
  260. goto free_and_exit;
  261. }
  262. *p = 0;
  263. }
  264. ret = sscanf(wpath, "%x:%x:%x.%x%c", &seg, &bus, &slot,
  265. &func, &end);
  266. if (ret != 4) {
  267. seg = 0;
  268. ret = sscanf(wpath, "%x:%x.%x%c", &bus, &slot, &func, &end);
  269. if (ret != 3) {
  270. ret = -EINVAL;
  271. goto free_and_exit;
  272. }
  273. }
  274. ret = (seg == pci_domain_nr(dev->bus) &&
  275. bus == dev->bus->number &&
  276. dev->devfn == PCI_DEVFN(slot, func));
  277. free_and_exit:
  278. kfree(wpath);
  279. return ret;
  280. }
  281. /**
  282. * pci_dev_str_match - test if a string matches a device
  283. * @dev: the PCI device to test
  284. * @p: string to match the device against
  285. * @endptr: pointer to the string after the match
  286. *
  287. * Test if a string (typically from a kernel parameter) matches a specified
  288. * PCI device. The string may be of one of the following formats:
  289. *
  290. * [<domain>:]<bus>:<device>.<func>[/<device>.<func>]*
  291. * pci:<vendor>:<device>[:<subvendor>:<subdevice>]
  292. *
  293. * The first format specifies a PCI bus/device/function address which
  294. * may change if new hardware is inserted, if motherboard firmware changes,
  295. * or due to changes caused in kernel parameters. If the domain is
  296. * left unspecified, it is taken to be 0. In order to be robust against
  297. * bus renumbering issues, a path of PCI device/function numbers may be used
  298. * to address the specific device. The path for a device can be determined
  299. * through the use of 'lspci -t'.
  300. *
  301. * The second format matches devices using IDs in the configuration
  302. * space which may match multiple devices in the system. A value of 0
  303. * for any field will match all devices. (Note: this differs from
  304. * in-kernel code that uses PCI_ANY_ID which is ~0; this is for
  305. * legacy reasons and convenience so users don't have to specify
  306. * FFFFFFFFs on the command line.)
  307. *
  308. * Returns 1 if the string matches the device, 0 if it does not and
  309. * a negative error code if the string cannot be parsed.
  310. */
  311. static int pci_dev_str_match(struct pci_dev *dev, const char *p,
  312. const char **endptr)
  313. {
  314. int ret;
  315. int count;
  316. unsigned short vendor, device, subsystem_vendor, subsystem_device;
  317. if (strncmp(p, "pci:", 4) == 0) {
  318. /* PCI vendor/device (subvendor/subdevice) IDs are specified */
  319. p += 4;
  320. ret = sscanf(p, "%hx:%hx:%hx:%hx%n", &vendor, &device,
  321. &subsystem_vendor, &subsystem_device, &count);
  322. if (ret != 4) {
  323. ret = sscanf(p, "%hx:%hx%n", &vendor, &device, &count);
  324. if (ret != 2)
  325. return -EINVAL;
  326. subsystem_vendor = 0;
  327. subsystem_device = 0;
  328. }
  329. p += count;
  330. if ((!vendor || vendor == dev->vendor) &&
  331. (!device || device == dev->device) &&
  332. (!subsystem_vendor ||
  333. subsystem_vendor == dev->subsystem_vendor) &&
  334. (!subsystem_device ||
  335. subsystem_device == dev->subsystem_device))
  336. goto found;
  337. } else {
  338. /*
  339. * PCI Bus, Device, Function IDs are specified
  340. * (optionally, may include a path of devfns following it)
  341. */
  342. ret = pci_dev_str_match_path(dev, p, &p);
  343. if (ret < 0)
  344. return ret;
  345. else if (ret)
  346. goto found;
  347. }
  348. *endptr = p;
  349. return 0;
  350. found:
  351. *endptr = p;
  352. return 1;
  353. }
  354. static int __pci_find_next_cap_ttl(struct pci_bus *bus, unsigned int devfn,
  355. u8 pos, int cap, int *ttl)
  356. {
  357. u8 id;
  358. u16 ent;
  359. pci_bus_read_config_byte(bus, devfn, pos, &pos);
  360. while ((*ttl)--) {
  361. if (pos < 0x40)
  362. break;
  363. pos &= ~3;
  364. pci_bus_read_config_word(bus, devfn, pos, &ent);
  365. id = ent & 0xff;
  366. if (id == 0xff)
  367. break;
  368. if (id == cap)
  369. return pos;
  370. pos = (ent >> 8);
  371. }
  372. return 0;
  373. }
  374. static int __pci_find_next_cap(struct pci_bus *bus, unsigned int devfn,
  375. u8 pos, int cap)
  376. {
  377. int ttl = PCI_FIND_CAP_TTL;
  378. return __pci_find_next_cap_ttl(bus, devfn, pos, cap, &ttl);
  379. }
  380. int pci_find_next_capability(struct pci_dev *dev, u8 pos, int cap)
  381. {
  382. return __pci_find_next_cap(dev->bus, dev->devfn,
  383. pos + PCI_CAP_LIST_NEXT, cap);
  384. }
  385. EXPORT_SYMBOL_GPL(pci_find_next_capability);
  386. static int __pci_bus_find_cap_start(struct pci_bus *bus,
  387. unsigned int devfn, u8 hdr_type)
  388. {
  389. u16 status;
  390. pci_bus_read_config_word(bus, devfn, PCI_STATUS, &status);
  391. if (!(status & PCI_STATUS_CAP_LIST))
  392. return 0;
  393. switch (hdr_type) {
  394. case PCI_HEADER_TYPE_NORMAL:
  395. case PCI_HEADER_TYPE_BRIDGE:
  396. return PCI_CAPABILITY_LIST;
  397. case PCI_HEADER_TYPE_CARDBUS:
  398. return PCI_CB_CAPABILITY_LIST;
  399. }
  400. return 0;
  401. }
  402. /**
  403. * pci_find_capability - query for devices' capabilities
  404. * @dev: PCI device to query
  405. * @cap: capability code
  406. *
  407. * Tell if a device supports a given PCI capability.
  408. * Returns the address of the requested capability structure within the
  409. * device's PCI configuration space or 0 in case the device does not
  410. * support it. Possible values for @cap include:
  411. *
  412. * %PCI_CAP_ID_PM Power Management
  413. * %PCI_CAP_ID_AGP Accelerated Graphics Port
  414. * %PCI_CAP_ID_VPD Vital Product Data
  415. * %PCI_CAP_ID_SLOTID Slot Identification
  416. * %PCI_CAP_ID_MSI Message Signalled Interrupts
  417. * %PCI_CAP_ID_CHSWP CompactPCI HotSwap
  418. * %PCI_CAP_ID_PCIX PCI-X
  419. * %PCI_CAP_ID_EXP PCI Express
  420. */
  421. int pci_find_capability(struct pci_dev *dev, int cap)
  422. {
  423. int pos;
  424. pos = __pci_bus_find_cap_start(dev->bus, dev->devfn, dev->hdr_type);
  425. if (pos)
  426. pos = __pci_find_next_cap(dev->bus, dev->devfn, pos, cap);
  427. return pos;
  428. }
  429. EXPORT_SYMBOL(pci_find_capability);
  430. /**
  431. * pci_bus_find_capability - query for devices' capabilities
  432. * @bus: the PCI bus to query
  433. * @devfn: PCI device to query
  434. * @cap: capability code
  435. *
  436. * Like pci_find_capability() but works for PCI devices that do not have a
  437. * pci_dev structure set up yet.
  438. *
  439. * Returns the address of the requested capability structure within the
  440. * device's PCI configuration space or 0 in case the device does not
  441. * support it.
  442. */
  443. int pci_bus_find_capability(struct pci_bus *bus, unsigned int devfn, int cap)
  444. {
  445. int pos;
  446. u8 hdr_type;
  447. pci_bus_read_config_byte(bus, devfn, PCI_HEADER_TYPE, &hdr_type);
  448. pos = __pci_bus_find_cap_start(bus, devfn, hdr_type & 0x7f);
  449. if (pos)
  450. pos = __pci_find_next_cap(bus, devfn, pos, cap);
  451. return pos;
  452. }
  453. EXPORT_SYMBOL(pci_bus_find_capability);
  454. /**
  455. * pci_find_next_ext_capability - Find an extended capability
  456. * @dev: PCI device to query
  457. * @start: address at which to start looking (0 to start at beginning of list)
  458. * @cap: capability code
  459. *
  460. * Returns the address of the next matching extended capability structure
  461. * within the device's PCI configuration space or 0 if the device does
  462. * not support it. Some capabilities can occur several times, e.g., the
  463. * vendor-specific capability, and this provides a way to find them all.
  464. */
  465. int pci_find_next_ext_capability(struct pci_dev *dev, int start, int cap)
  466. {
  467. u32 header;
  468. int ttl;
  469. int pos = PCI_CFG_SPACE_SIZE;
  470. /* minimum 8 bytes per capability */
  471. ttl = (PCI_CFG_SPACE_EXP_SIZE - PCI_CFG_SPACE_SIZE) / 8;
  472. if (dev->cfg_size <= PCI_CFG_SPACE_SIZE)
  473. return 0;
  474. if (start)
  475. pos = start;
  476. if (pci_read_config_dword(dev, pos, &header) != PCIBIOS_SUCCESSFUL)
  477. return 0;
  478. /*
  479. * If we have no capabilities, this is indicated by cap ID,
  480. * cap version and next pointer all being 0.
  481. */
  482. if (header == 0)
  483. return 0;
  484. while (ttl-- > 0) {
  485. if (PCI_EXT_CAP_ID(header) == cap && pos != start)
  486. return pos;
  487. pos = PCI_EXT_CAP_NEXT(header);
  488. if (pos < PCI_CFG_SPACE_SIZE)
  489. break;
  490. if (pci_read_config_dword(dev, pos, &header) != PCIBIOS_SUCCESSFUL)
  491. break;
  492. }
  493. return 0;
  494. }
  495. EXPORT_SYMBOL_GPL(pci_find_next_ext_capability);
  496. /**
  497. * pci_find_ext_capability - Find an extended capability
  498. * @dev: PCI device to query
  499. * @cap: capability code
  500. *
  501. * Returns the address of the requested extended capability structure
  502. * within the device's PCI configuration space or 0 if the device does
  503. * not support it. Possible values for @cap include:
  504. *
  505. * %PCI_EXT_CAP_ID_ERR Advanced Error Reporting
  506. * %PCI_EXT_CAP_ID_VC Virtual Channel
  507. * %PCI_EXT_CAP_ID_DSN Device Serial Number
  508. * %PCI_EXT_CAP_ID_PWR Power Budgeting
  509. */
  510. int pci_find_ext_capability(struct pci_dev *dev, int cap)
  511. {
  512. return pci_find_next_ext_capability(dev, 0, cap);
  513. }
  514. EXPORT_SYMBOL_GPL(pci_find_ext_capability);
  515. /**
  516. * pci_get_dsn - Read and return the 8-byte Device Serial Number
  517. * @dev: PCI device to query
  518. *
  519. * Looks up the PCI_EXT_CAP_ID_DSN and reads the 8 bytes of the Device Serial
  520. * Number.
  521. *
  522. * Returns the DSN, or zero if the capability does not exist.
  523. */
  524. u64 pci_get_dsn(struct pci_dev *dev)
  525. {
  526. u32 dword;
  527. u64 dsn;
  528. int pos;
  529. pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DSN);
  530. if (!pos)
  531. return 0;
  532. /*
  533. * The Device Serial Number is two dwords offset 4 bytes from the
  534. * capability position. The specification says that the first dword is
  535. * the lower half, and the second dword is the upper half.
  536. */
  537. pos += 4;
  538. pci_read_config_dword(dev, pos, &dword);
  539. dsn = (u64)dword;
  540. pci_read_config_dword(dev, pos + 4, &dword);
  541. dsn |= ((u64)dword) << 32;
  542. return dsn;
  543. }
  544. EXPORT_SYMBOL_GPL(pci_get_dsn);
  545. static int __pci_find_next_ht_cap(struct pci_dev *dev, int pos, int ht_cap)
  546. {
  547. int rc, ttl = PCI_FIND_CAP_TTL;
  548. u8 cap, mask;
  549. if (ht_cap == HT_CAPTYPE_SLAVE || ht_cap == HT_CAPTYPE_HOST)
  550. mask = HT_3BIT_CAP_MASK;
  551. else
  552. mask = HT_5BIT_CAP_MASK;
  553. pos = __pci_find_next_cap_ttl(dev->bus, dev->devfn, pos,
  554. PCI_CAP_ID_HT, &ttl);
  555. while (pos) {
  556. rc = pci_read_config_byte(dev, pos + 3, &cap);
  557. if (rc != PCIBIOS_SUCCESSFUL)
  558. return 0;
  559. if ((cap & mask) == ht_cap)
  560. return pos;
  561. pos = __pci_find_next_cap_ttl(dev->bus, dev->devfn,
  562. pos + PCI_CAP_LIST_NEXT,
  563. PCI_CAP_ID_HT, &ttl);
  564. }
  565. return 0;
  566. }
  567. /**
  568. * pci_find_next_ht_capability - query a device's Hypertransport capabilities
  569. * @dev: PCI device to query
  570. * @pos: Position from which to continue searching
  571. * @ht_cap: Hypertransport capability code
  572. *
  573. * To be used in conjunction with pci_find_ht_capability() to search for
  574. * all capabilities matching @ht_cap. @pos should always be a value returned
  575. * from pci_find_ht_capability().
  576. *
  577. * NB. To be 100% safe against broken PCI devices, the caller should take
  578. * steps to avoid an infinite loop.
  579. */
  580. int pci_find_next_ht_capability(struct pci_dev *dev, int pos, int ht_cap)
  581. {
  582. return __pci_find_next_ht_cap(dev, pos + PCI_CAP_LIST_NEXT, ht_cap);
  583. }
  584. EXPORT_SYMBOL_GPL(pci_find_next_ht_capability);
  585. /**
  586. * pci_find_ht_capability - query a device's Hypertransport capabilities
  587. * @dev: PCI device to query
  588. * @ht_cap: Hypertransport capability code
  589. *
  590. * Tell if a device supports a given Hypertransport capability.
  591. * Returns an address within the device's PCI configuration space
  592. * or 0 in case the device does not support the request capability.
  593. * The address points to the PCI capability, of type PCI_CAP_ID_HT,
  594. * which has a Hypertransport capability matching @ht_cap.
  595. */
  596. int pci_find_ht_capability(struct pci_dev *dev, int ht_cap)
  597. {
  598. int pos;
  599. pos = __pci_bus_find_cap_start(dev->bus, dev->devfn, dev->hdr_type);
  600. if (pos)
  601. pos = __pci_find_next_ht_cap(dev, pos, ht_cap);
  602. return pos;
  603. }
  604. EXPORT_SYMBOL_GPL(pci_find_ht_capability);
  605. /**
  606. * pci_find_parent_resource - return resource region of parent bus of given
  607. * region
  608. * @dev: PCI device structure contains resources to be searched
  609. * @res: child resource record for which parent is sought
  610. *
  611. * For given resource region of given device, return the resource region of
  612. * parent bus the given region is contained in.
  613. */
  614. struct resource *pci_find_parent_resource(const struct pci_dev *dev,
  615. struct resource *res)
  616. {
  617. const struct pci_bus *bus = dev->bus;
  618. struct resource *r;
  619. int i;
  620. pci_bus_for_each_resource(bus, r, i) {
  621. if (!r)
  622. continue;
  623. if (resource_contains(r, res)) {
  624. /*
  625. * If the window is prefetchable but the BAR is
  626. * not, the allocator made a mistake.
  627. */
  628. if (r->flags & IORESOURCE_PREFETCH &&
  629. !(res->flags & IORESOURCE_PREFETCH))
  630. return NULL;
  631. /*
  632. * If we're below a transparent bridge, there may
  633. * be both a positively-decoded aperture and a
  634. * subtractively-decoded region that contain the BAR.
  635. * We want the positively-decoded one, so this depends
  636. * on pci_bus_for_each_resource() giving us those
  637. * first.
  638. */
  639. return r;
  640. }
  641. }
  642. return NULL;
  643. }
  644. EXPORT_SYMBOL(pci_find_parent_resource);
  645. /**
  646. * pci_find_resource - Return matching PCI device resource
  647. * @dev: PCI device to query
  648. * @res: Resource to look for
  649. *
  650. * Goes over standard PCI resources (BARs) and checks if the given resource
  651. * is partially or fully contained in any of them. In that case the
  652. * matching resource is returned, %NULL otherwise.
  653. */
  654. struct resource *pci_find_resource(struct pci_dev *dev, struct resource *res)
  655. {
  656. int i;
  657. for (i = 0; i < PCI_STD_NUM_BARS; i++) {
  658. struct resource *r = &dev->resource[i];
  659. if (r->start && resource_contains(r, res))
  660. return r;
  661. }
  662. return NULL;
  663. }
  664. EXPORT_SYMBOL(pci_find_resource);
  665. /**
  666. * pci_wait_for_pending - wait for @mask bit(s) to clear in status word @pos
  667. * @dev: the PCI device to operate on
  668. * @pos: config space offset of status word
  669. * @mask: mask of bit(s) to care about in status word
  670. *
  671. * Return 1 when mask bit(s) in status word clear, 0 otherwise.
  672. */
  673. int pci_wait_for_pending(struct pci_dev *dev, int pos, u16 mask)
  674. {
  675. int i;
  676. /* Wait for Transaction Pending bit clean */
  677. for (i = 0; i < 4; i++) {
  678. u16 status;
  679. if (i)
  680. msleep((1 << (i - 1)) * 100);
  681. pci_read_config_word(dev, pos, &status);
  682. if (!(status & mask))
  683. return 1;
  684. }
  685. return 0;
  686. }
  687. static int pci_acs_enable;
  688. /**
  689. * pci_request_acs - ask for ACS to be enabled if supported
  690. */
  691. void pci_request_acs(void)
  692. {
  693. pci_acs_enable = 1;
  694. }
  695. static const char *disable_acs_redir_param;
  696. /**
  697. * pci_disable_acs_redir - disable ACS redirect capabilities
  698. * @dev: the PCI device
  699. *
  700. * For only devices specified in the disable_acs_redir parameter.
  701. */
  702. static void pci_disable_acs_redir(struct pci_dev *dev)
  703. {
  704. int ret = 0;
  705. const char *p;
  706. int pos;
  707. u16 ctrl;
  708. if (!disable_acs_redir_param)
  709. return;
  710. p = disable_acs_redir_param;
  711. while (*p) {
  712. ret = pci_dev_str_match(dev, p, &p);
  713. if (ret < 0) {
  714. pr_info_once("PCI: Can't parse disable_acs_redir parameter: %s\n",
  715. disable_acs_redir_param);
  716. break;
  717. } else if (ret == 1) {
  718. /* Found a match */
  719. break;
  720. }
  721. if (*p != ';' && *p != ',') {
  722. /* End of param or invalid format */
  723. break;
  724. }
  725. p++;
  726. }
  727. if (ret != 1)
  728. return;
  729. if (!pci_dev_specific_disable_acs_redir(dev))
  730. return;
  731. pos = dev->acs_cap;
  732. if (!pos) {
  733. pci_warn(dev, "cannot disable ACS redirect for this hardware as it does not have ACS capabilities\n");
  734. return;
  735. }
  736. pci_read_config_word(dev, pos + PCI_ACS_CTRL, &ctrl);
  737. /* P2P Request & Completion Redirect */
  738. ctrl &= ~(PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_EC);
  739. pci_write_config_word(dev, pos + PCI_ACS_CTRL, ctrl);
  740. pci_info(dev, "disabled ACS redirect\n");
  741. }
  742. /**
  743. * pci_std_enable_acs - enable ACS on devices using standard ACS capabilities
  744. * @dev: the PCI device
  745. */
  746. static void pci_std_enable_acs(struct pci_dev *dev)
  747. {
  748. int pos;
  749. u16 cap;
  750. u16 ctrl;
  751. pos = dev->acs_cap;
  752. if (!pos)
  753. return;
  754. pci_read_config_word(dev, pos + PCI_ACS_CAP, &cap);
  755. pci_read_config_word(dev, pos + PCI_ACS_CTRL, &ctrl);
  756. /* Source Validation */
  757. ctrl |= (cap & PCI_ACS_SV);
  758. /* P2P Request Redirect */
  759. ctrl |= (cap & PCI_ACS_RR);
  760. /* P2P Completion Redirect */
  761. ctrl |= (cap & PCI_ACS_CR);
  762. /* Upstream Forwarding */
  763. ctrl |= (cap & PCI_ACS_UF);
  764. /* Enable Translation Blocking for external devices */
  765. if (dev->external_facing || dev->untrusted)
  766. ctrl |= (cap & PCI_ACS_TB);
  767. pci_write_config_word(dev, pos + PCI_ACS_CTRL, ctrl);
  768. }
  769. /**
  770. * pci_enable_acs - enable ACS if hardware support it
  771. * @dev: the PCI device
  772. */
  773. static void pci_enable_acs(struct pci_dev *dev)
  774. {
  775. if (!pci_acs_enable)
  776. goto disable_acs_redir;
  777. if (!pci_dev_specific_enable_acs(dev))
  778. goto disable_acs_redir;
  779. pci_std_enable_acs(dev);
  780. disable_acs_redir:
  781. /*
  782. * Note: pci_disable_acs_redir() must be called even if ACS was not
  783. * enabled by the kernel because it may have been enabled by
  784. * platform firmware. So if we are told to disable it, we should
  785. * always disable it after setting the kernel's default
  786. * preferences.
  787. */
  788. pci_disable_acs_redir(dev);
  789. }
  790. /**
  791. * pci_restore_bars - restore a device's BAR values (e.g. after wake-up)
  792. * @dev: PCI device to have its BARs restored
  793. *
  794. * Restore the BAR values for a given device, so as to make it
  795. * accessible by its driver.
  796. */
  797. static void pci_restore_bars(struct pci_dev *dev)
  798. {
  799. int i;
  800. for (i = 0; i < PCI_BRIDGE_RESOURCES; i++)
  801. pci_update_resource(dev, i);
  802. }
  803. static const struct pci_platform_pm_ops *pci_platform_pm;
  804. int pci_set_platform_pm(const struct pci_platform_pm_ops *ops)
  805. {
  806. if (!ops->is_manageable || !ops->set_state || !ops->get_state ||
  807. !ops->choose_state || !ops->set_wakeup || !ops->need_resume)
  808. return -EINVAL;
  809. pci_platform_pm = ops;
  810. return 0;
  811. }
  812. static inline bool platform_pci_power_manageable(struct pci_dev *dev)
  813. {
  814. return pci_platform_pm ? pci_platform_pm->is_manageable(dev) : false;
  815. }
  816. static inline int platform_pci_set_power_state(struct pci_dev *dev,
  817. pci_power_t t)
  818. {
  819. return pci_platform_pm ? pci_platform_pm->set_state(dev, t) : -ENOSYS;
  820. }
  821. static inline pci_power_t platform_pci_get_power_state(struct pci_dev *dev)
  822. {
  823. return pci_platform_pm ? pci_platform_pm->get_state(dev) : PCI_UNKNOWN;
  824. }
  825. static inline void platform_pci_refresh_power_state(struct pci_dev *dev)
  826. {
  827. if (pci_platform_pm && pci_platform_pm->refresh_state)
  828. pci_platform_pm->refresh_state(dev);
  829. }
  830. static inline pci_power_t platform_pci_choose_state(struct pci_dev *dev)
  831. {
  832. return pci_platform_pm ?
  833. pci_platform_pm->choose_state(dev) : PCI_POWER_ERROR;
  834. }
  835. static inline int platform_pci_set_wakeup(struct pci_dev *dev, bool enable)
  836. {
  837. return pci_platform_pm ?
  838. pci_platform_pm->set_wakeup(dev, enable) : -ENODEV;
  839. }
  840. static inline bool platform_pci_need_resume(struct pci_dev *dev)
  841. {
  842. return pci_platform_pm ? pci_platform_pm->need_resume(dev) : false;
  843. }
  844. static inline bool platform_pci_bridge_d3(struct pci_dev *dev)
  845. {
  846. if (pci_platform_pm && pci_platform_pm->bridge_d3)
  847. return pci_platform_pm->bridge_d3(dev);
  848. return false;
  849. }
  850. /**
  851. * pci_raw_set_power_state - Use PCI PM registers to set the power state of
  852. * given PCI device
  853. * @dev: PCI device to handle.
  854. * @state: PCI power state (D0, D1, D2, D3hot) to put the device into.
  855. *
  856. * RETURN VALUE:
  857. * -EINVAL if the requested state is invalid.
  858. * -EIO if device does not support PCI PM or its PM capabilities register has a
  859. * wrong version, or device doesn't support the requested state.
  860. * 0 if device already is in the requested state.
  861. * 0 if device's power state has been successfully changed.
  862. */
  863. static int pci_raw_set_power_state(struct pci_dev *dev, pci_power_t state)
  864. {
  865. u16 pmcsr;
  866. bool need_restore = false;
  867. /* Check if we're already there */
  868. if (dev->current_state == state)
  869. return 0;
  870. if (!dev->pm_cap)
  871. return -EIO;
  872. if (state < PCI_D0 || state > PCI_D3hot)
  873. return -EINVAL;
  874. /*
  875. * Validate transition: We can enter D0 from any state, but if
  876. * we're already in a low-power state, we can only go deeper. E.g.,
  877. * we can go from D1 to D3, but we can't go directly from D3 to D1;
  878. * we'd have to go from D3 to D0, then to D1.
  879. */
  880. if (state != PCI_D0 && dev->current_state <= PCI_D3cold
  881. && dev->current_state > state) {
  882. pci_err(dev, "invalid power transition (from %s to %s)\n",
  883. pci_power_name(dev->current_state),
  884. pci_power_name(state));
  885. return -EINVAL;
  886. }
  887. /* Check if this device supports the desired state */
  888. if ((state == PCI_D1 && !dev->d1_support)
  889. || (state == PCI_D2 && !dev->d2_support))
  890. return -EIO;
  891. pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
  892. if (pmcsr == (u16) ~0) {
  893. pci_err(dev, "can't change power state from %s to %s (config space inaccessible)\n",
  894. pci_power_name(dev->current_state),
  895. pci_power_name(state));
  896. return -EIO;
  897. }
  898. /*
  899. * If we're (effectively) in D3, force entire word to 0.
  900. * This doesn't affect PME_Status, disables PME_En, and
  901. * sets PowerState to 0.
  902. */
  903. switch (dev->current_state) {
  904. case PCI_D0:
  905. case PCI_D1:
  906. case PCI_D2:
  907. pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
  908. pmcsr |= state;
  909. break;
  910. case PCI_D3hot:
  911. case PCI_D3cold:
  912. case PCI_UNKNOWN: /* Boot-up */
  913. if ((pmcsr & PCI_PM_CTRL_STATE_MASK) == PCI_D3hot
  914. && !(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET))
  915. need_restore = true;
  916. fallthrough; /* force to D0 */
  917. default:
  918. pmcsr = 0;
  919. break;
  920. }
  921. /* Enter specified state */
  922. pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr);
  923. /*
  924. * Mandatory power management transition delays; see PCI PM 1.1
  925. * 5.6.1 table 18
  926. */
  927. if (state == PCI_D3hot || dev->current_state == PCI_D3hot)
  928. pci_dev_d3_sleep(dev);
  929. else if (state == PCI_D2 || dev->current_state == PCI_D2)
  930. udelay(PCI_PM_D2_DELAY);
  931. pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
  932. dev->current_state = (pmcsr & PCI_PM_CTRL_STATE_MASK);
  933. if (dev->current_state != state)
  934. pci_info_ratelimited(dev, "refused to change power state from %s to %s\n",
  935. pci_power_name(dev->current_state),
  936. pci_power_name(state));
  937. /*
  938. * According to section 5.4.1 of the "PCI BUS POWER MANAGEMENT
  939. * INTERFACE SPECIFICATION, REV. 1.2", a device transitioning
  940. * from D3hot to D0 _may_ perform an internal reset, thereby
  941. * going to "D0 Uninitialized" rather than "D0 Initialized".
  942. * For example, at least some versions of the 3c905B and the
  943. * 3c556B exhibit this behaviour.
  944. *
  945. * At least some laptop BIOSen (e.g. the Thinkpad T21) leave
  946. * devices in a D3hot state at boot. Consequently, we need to
  947. * restore at least the BARs so that the device will be
  948. * accessible to its driver.
  949. */
  950. if (need_restore)
  951. pci_restore_bars(dev);
  952. if (dev->bus->self)
  953. pcie_aspm_pm_state_change(dev->bus->self);
  954. return 0;
  955. }
  956. /**
  957. * pci_update_current_state - Read power state of given device and cache it
  958. * @dev: PCI device to handle.
  959. * @state: State to cache in case the device doesn't have the PM capability
  960. *
  961. * The power state is read from the PMCSR register, which however is
  962. * inaccessible in D3cold. The platform firmware is therefore queried first
  963. * to detect accessibility of the register. In case the platform firmware
  964. * reports an incorrect state or the device isn't power manageable by the
  965. * platform at all, we try to detect D3cold by testing accessibility of the
  966. * vendor ID in config space.
  967. */
  968. void pci_update_current_state(struct pci_dev *dev, pci_power_t state)
  969. {
  970. if (platform_pci_get_power_state(dev) == PCI_D3cold ||
  971. !pci_device_is_present(dev)) {
  972. dev->current_state = PCI_D3cold;
  973. } else if (dev->pm_cap) {
  974. u16 pmcsr;
  975. pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
  976. dev->current_state = (pmcsr & PCI_PM_CTRL_STATE_MASK);
  977. } else {
  978. dev->current_state = state;
  979. }
  980. }
  981. /**
  982. * pci_refresh_power_state - Refresh the given device's power state data
  983. * @dev: Target PCI device.
  984. *
  985. * Ask the platform to refresh the devices power state information and invoke
  986. * pci_update_current_state() to update its current PCI power state.
  987. */
  988. void pci_refresh_power_state(struct pci_dev *dev)
  989. {
  990. if (platform_pci_power_manageable(dev))
  991. platform_pci_refresh_power_state(dev);
  992. pci_update_current_state(dev, dev->current_state);
  993. }
  994. /**
  995. * pci_platform_power_transition - Use platform to change device power state
  996. * @dev: PCI device to handle.
  997. * @state: State to put the device into.
  998. */
  999. int pci_platform_power_transition(struct pci_dev *dev, pci_power_t state)
  1000. {
  1001. int error;
  1002. if (platform_pci_power_manageable(dev)) {
  1003. error = platform_pci_set_power_state(dev, state);
  1004. if (!error)
  1005. pci_update_current_state(dev, state);
  1006. } else
  1007. error = -ENODEV;
  1008. if (error && !dev->pm_cap) /* Fall back to PCI_D0 */
  1009. dev->current_state = PCI_D0;
  1010. return error;
  1011. }
  1012. EXPORT_SYMBOL_GPL(pci_platform_power_transition);
  1013. /**
  1014. * pci_wakeup - Wake up a PCI device
  1015. * @pci_dev: Device to handle.
  1016. * @ign: ignored parameter
  1017. */
  1018. static int pci_wakeup(struct pci_dev *pci_dev, void *ign)
  1019. {
  1020. pci_wakeup_event(pci_dev);
  1021. pm_request_resume(&pci_dev->dev);
  1022. return 0;
  1023. }
  1024. /**
  1025. * pci_wakeup_bus - Walk given bus and wake up devices on it
  1026. * @bus: Top bus of the subtree to walk.
  1027. */
  1028. void pci_wakeup_bus(struct pci_bus *bus)
  1029. {
  1030. if (bus)
  1031. pci_walk_bus(bus, pci_wakeup, NULL);
  1032. }
  1033. static int pci_dev_wait(struct pci_dev *dev, char *reset_type, int timeout)
  1034. {
  1035. int delay = 1;
  1036. u32 id;
  1037. /*
  1038. * After reset, the device should not silently discard config
  1039. * requests, but it may still indicate that it needs more time by
  1040. * responding to them with CRS completions. The Root Port will
  1041. * generally synthesize ~0 data to complete the read (except when
  1042. * CRS SV is enabled and the read was for the Vendor ID; in that
  1043. * case it synthesizes 0x0001 data).
  1044. *
  1045. * Wait for the device to return a non-CRS completion. Read the
  1046. * Command register instead of Vendor ID so we don't have to
  1047. * contend with the CRS SV value.
  1048. */
  1049. pci_read_config_dword(dev, PCI_COMMAND, &id);
  1050. while (id == ~0) {
  1051. if (delay > timeout) {
  1052. pci_warn(dev, "not ready %dms after %s; giving up\n",
  1053. delay - 1, reset_type);
  1054. return -ENOTTY;
  1055. }
  1056. if (delay > 1000)
  1057. pci_info(dev, "not ready %dms after %s; waiting\n",
  1058. delay - 1, reset_type);
  1059. msleep(delay);
  1060. delay *= 2;
  1061. pci_read_config_dword(dev, PCI_COMMAND, &id);
  1062. }
  1063. if (delay > 1000)
  1064. pci_info(dev, "ready %dms after %s\n", delay - 1,
  1065. reset_type);
  1066. return 0;
  1067. }
  1068. /**
  1069. * pci_power_up - Put the given device into D0
  1070. * @dev: PCI device to power up
  1071. */
  1072. int pci_power_up(struct pci_dev *dev)
  1073. {
  1074. pci_platform_power_transition(dev, PCI_D0);
  1075. /*
  1076. * Mandatory power management transition delays are handled in
  1077. * pci_pm_resume_noirq() and pci_pm_runtime_resume() of the
  1078. * corresponding bridge.
  1079. */
  1080. if (dev->runtime_d3cold) {
  1081. /*
  1082. * When powering on a bridge from D3cold, the whole hierarchy
  1083. * may be powered on into D0uninitialized state, resume them to
  1084. * give them a chance to suspend again
  1085. */
  1086. pci_wakeup_bus(dev->subordinate);
  1087. }
  1088. return pci_raw_set_power_state(dev, PCI_D0);
  1089. }
  1090. /**
  1091. * __pci_dev_set_current_state - Set current state of a PCI device
  1092. * @dev: Device to handle
  1093. * @data: pointer to state to be set
  1094. */
  1095. static int __pci_dev_set_current_state(struct pci_dev *dev, void *data)
  1096. {
  1097. pci_power_t state = *(pci_power_t *)data;
  1098. dev->current_state = state;
  1099. return 0;
  1100. }
  1101. /**
  1102. * pci_bus_set_current_state - Walk given bus and set current state of devices
  1103. * @bus: Top bus of the subtree to walk.
  1104. * @state: state to be set
  1105. */
  1106. void pci_bus_set_current_state(struct pci_bus *bus, pci_power_t state)
  1107. {
  1108. if (bus)
  1109. pci_walk_bus(bus, __pci_dev_set_current_state, &state);
  1110. }
  1111. /**
  1112. * pci_set_power_state - Set the power state of a PCI device
  1113. * @dev: PCI device to handle.
  1114. * @state: PCI power state (D0, D1, D2, D3hot) to put the device into.
  1115. *
  1116. * Transition a device to a new power state, using the platform firmware and/or
  1117. * the device's PCI PM registers.
  1118. *
  1119. * RETURN VALUE:
  1120. * -EINVAL if the requested state is invalid.
  1121. * -EIO if device does not support PCI PM or its PM capabilities register has a
  1122. * wrong version, or device doesn't support the requested state.
  1123. * 0 if the transition is to D1 or D2 but D1 and D2 are not supported.
  1124. * 0 if device already is in the requested state.
  1125. * 0 if the transition is to D3 but D3 is not supported.
  1126. * 0 if device's power state has been successfully changed.
  1127. */
  1128. int pci_set_power_state(struct pci_dev *dev, pci_power_t state)
  1129. {
  1130. int error;
  1131. /* Bound the state we're entering */
  1132. if (state > PCI_D3cold)
  1133. state = PCI_D3cold;
  1134. else if (state < PCI_D0)
  1135. state = PCI_D0;
  1136. else if ((state == PCI_D1 || state == PCI_D2) && pci_no_d1d2(dev))
  1137. /*
  1138. * If the device or the parent bridge do not support PCI
  1139. * PM, ignore the request if we're doing anything other
  1140. * than putting it into D0 (which would only happen on
  1141. * boot).
  1142. */
  1143. return 0;
  1144. /* Check if we're already there */
  1145. if (dev->current_state == state)
  1146. return 0;
  1147. if (state == PCI_D0)
  1148. return pci_power_up(dev);
  1149. /*
  1150. * This device is quirked not to be put into D3, so don't put it in
  1151. * D3
  1152. */
  1153. if (state >= PCI_D3hot && (dev->dev_flags & PCI_DEV_FLAGS_NO_D3))
  1154. return 0;
  1155. /*
  1156. * To put device in D3cold, we put device into D3hot in native
  1157. * way, then put device into D3cold with platform ops
  1158. */
  1159. error = pci_raw_set_power_state(dev, state > PCI_D3hot ?
  1160. PCI_D3hot : state);
  1161. if (pci_platform_power_transition(dev, state))
  1162. return error;
  1163. /* Powering off a bridge may power off the whole hierarchy */
  1164. if (state == PCI_D3cold)
  1165. pci_bus_set_current_state(dev->subordinate, PCI_D3cold);
  1166. return 0;
  1167. }
  1168. EXPORT_SYMBOL(pci_set_power_state);
  1169. /**
  1170. * pci_choose_state - Choose the power state of a PCI device
  1171. * @dev: PCI device to be suspended
  1172. * @state: target sleep state for the whole system. This is the value
  1173. * that is passed to suspend() function.
  1174. *
  1175. * Returns PCI power state suitable for given device and given system
  1176. * message.
  1177. */
  1178. pci_power_t pci_choose_state(struct pci_dev *dev, pm_message_t state)
  1179. {
  1180. pci_power_t ret;
  1181. if (!dev->pm_cap)
  1182. return PCI_D0;
  1183. ret = platform_pci_choose_state(dev);
  1184. if (ret != PCI_POWER_ERROR)
  1185. return ret;
  1186. switch (state.event) {
  1187. case PM_EVENT_ON:
  1188. return PCI_D0;
  1189. case PM_EVENT_FREEZE:
  1190. case PM_EVENT_PRETHAW:
  1191. /* REVISIT both freeze and pre-thaw "should" use D0 */
  1192. case PM_EVENT_SUSPEND:
  1193. case PM_EVENT_HIBERNATE:
  1194. return PCI_D3hot;
  1195. default:
  1196. pci_info(dev, "unrecognized suspend event %d\n",
  1197. state.event);
  1198. BUG();
  1199. }
  1200. return PCI_D0;
  1201. }
  1202. EXPORT_SYMBOL(pci_choose_state);
  1203. #define PCI_EXP_SAVE_REGS 7
  1204. static struct pci_cap_saved_state *_pci_find_saved_cap(struct pci_dev *pci_dev,
  1205. u16 cap, bool extended)
  1206. {
  1207. struct pci_cap_saved_state *tmp;
  1208. hlist_for_each_entry(tmp, &pci_dev->saved_cap_space, next) {
  1209. if (tmp->cap.cap_extended == extended && tmp->cap.cap_nr == cap)
  1210. return tmp;
  1211. }
  1212. return NULL;
  1213. }
  1214. struct pci_cap_saved_state *pci_find_saved_cap(struct pci_dev *dev, char cap)
  1215. {
  1216. return _pci_find_saved_cap(dev, cap, false);
  1217. }
  1218. struct pci_cap_saved_state *pci_find_saved_ext_cap(struct pci_dev *dev, u16 cap)
  1219. {
  1220. return _pci_find_saved_cap(dev, cap, true);
  1221. }
  1222. static int pci_save_pcie_state(struct pci_dev *dev)
  1223. {
  1224. int i = 0;
  1225. struct pci_cap_saved_state *save_state;
  1226. u16 *cap;
  1227. if (!pci_is_pcie(dev))
  1228. return 0;
  1229. save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP);
  1230. if (!save_state) {
  1231. pci_err(dev, "buffer not found in %s\n", __func__);
  1232. return -ENOMEM;
  1233. }
  1234. cap = (u16 *)&save_state->cap.data[0];
  1235. pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &cap[i++]);
  1236. pcie_capability_read_word(dev, PCI_EXP_LNKCTL, &cap[i++]);
  1237. pcie_capability_read_word(dev, PCI_EXP_SLTCTL, &cap[i++]);
  1238. pcie_capability_read_word(dev, PCI_EXP_RTCTL, &cap[i++]);
  1239. pcie_capability_read_word(dev, PCI_EXP_DEVCTL2, &cap[i++]);
  1240. pcie_capability_read_word(dev, PCI_EXP_LNKCTL2, &cap[i++]);
  1241. pcie_capability_read_word(dev, PCI_EXP_SLTCTL2, &cap[i++]);
  1242. return 0;
  1243. }
  1244. static void pci_restore_pcie_state(struct pci_dev *dev)
  1245. {
  1246. int i = 0;
  1247. struct pci_cap_saved_state *save_state;
  1248. u16 *cap;
  1249. save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP);
  1250. if (!save_state)
  1251. return;
  1252. cap = (u16 *)&save_state->cap.data[0];
  1253. pcie_capability_write_word(dev, PCI_EXP_DEVCTL, cap[i++]);
  1254. pcie_capability_write_word(dev, PCI_EXP_LNKCTL, cap[i++]);
  1255. pcie_capability_write_word(dev, PCI_EXP_SLTCTL, cap[i++]);
  1256. pcie_capability_write_word(dev, PCI_EXP_RTCTL, cap[i++]);
  1257. pcie_capability_write_word(dev, PCI_EXP_DEVCTL2, cap[i++]);
  1258. pcie_capability_write_word(dev, PCI_EXP_LNKCTL2, cap[i++]);
  1259. pcie_capability_write_word(dev, PCI_EXP_SLTCTL2, cap[i++]);
  1260. }
  1261. static int pci_save_pcix_state(struct pci_dev *dev)
  1262. {
  1263. int pos;
  1264. struct pci_cap_saved_state *save_state;
  1265. pos = pci_find_capability(dev, PCI_CAP_ID_PCIX);
  1266. if (!pos)
  1267. return 0;
  1268. save_state = pci_find_saved_cap(dev, PCI_CAP_ID_PCIX);
  1269. if (!save_state) {
  1270. pci_err(dev, "buffer not found in %s\n", __func__);
  1271. return -ENOMEM;
  1272. }
  1273. pci_read_config_word(dev, pos + PCI_X_CMD,
  1274. (u16 *)save_state->cap.data);
  1275. return 0;
  1276. }
  1277. static void pci_restore_pcix_state(struct pci_dev *dev)
  1278. {
  1279. int i = 0, pos;
  1280. struct pci_cap_saved_state *save_state;
  1281. u16 *cap;
  1282. save_state = pci_find_saved_cap(dev, PCI_CAP_ID_PCIX);
  1283. pos = pci_find_capability(dev, PCI_CAP_ID_PCIX);
  1284. if (!save_state || !pos)
  1285. return;
  1286. cap = (u16 *)&save_state->cap.data[0];
  1287. pci_write_config_word(dev, pos + PCI_X_CMD, cap[i++]);
  1288. }
  1289. static void pci_save_ltr_state(struct pci_dev *dev)
  1290. {
  1291. int ltr;
  1292. struct pci_cap_saved_state *save_state;
  1293. u16 *cap;
  1294. if (!pci_is_pcie(dev))
  1295. return;
  1296. ltr = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_LTR);
  1297. if (!ltr)
  1298. return;
  1299. save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_LTR);
  1300. if (!save_state) {
  1301. pci_err(dev, "no suspend buffer for LTR; ASPM issues possible after resume\n");
  1302. return;
  1303. }
  1304. cap = (u16 *)&save_state->cap.data[0];
  1305. pci_read_config_word(dev, ltr + PCI_LTR_MAX_SNOOP_LAT, cap++);
  1306. pci_read_config_word(dev, ltr + PCI_LTR_MAX_NOSNOOP_LAT, cap++);
  1307. }
  1308. static void pci_restore_ltr_state(struct pci_dev *dev)
  1309. {
  1310. struct pci_cap_saved_state *save_state;
  1311. int ltr;
  1312. u16 *cap;
  1313. save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_LTR);
  1314. ltr = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_LTR);
  1315. if (!save_state || !ltr)
  1316. return;
  1317. cap = (u16 *)&save_state->cap.data[0];
  1318. pci_write_config_word(dev, ltr + PCI_LTR_MAX_SNOOP_LAT, *cap++);
  1319. pci_write_config_word(dev, ltr + PCI_LTR_MAX_NOSNOOP_LAT, *cap++);
  1320. }
  1321. /**
  1322. * pci_save_state - save the PCI configuration space of a device before
  1323. * suspending
  1324. * @dev: PCI device that we're dealing with
  1325. */
  1326. int pci_save_state(struct pci_dev *dev)
  1327. {
  1328. int i;
  1329. /* XXX: 100% dword access ok here? */
  1330. for (i = 0; i < 16; i++) {
  1331. pci_read_config_dword(dev, i * 4, &dev->saved_config_space[i]);
  1332. pci_dbg(dev, "saving config space at offset %#x (reading %#x)\n",
  1333. i * 4, dev->saved_config_space[i]);
  1334. }
  1335. dev->state_saved = true;
  1336. i = pci_save_pcie_state(dev);
  1337. if (i != 0)
  1338. return i;
  1339. i = pci_save_pcix_state(dev);
  1340. if (i != 0)
  1341. return i;
  1342. pci_save_ltr_state(dev);
  1343. pci_save_dpc_state(dev);
  1344. pci_save_aer_state(dev);
  1345. return pci_save_vc_state(dev);
  1346. }
  1347. EXPORT_SYMBOL(pci_save_state);
  1348. static void pci_restore_config_dword(struct pci_dev *pdev, int offset,
  1349. u32 saved_val, int retry, bool force)
  1350. {
  1351. u32 val;
  1352. pci_read_config_dword(pdev, offset, &val);
  1353. if (!force && val == saved_val)
  1354. return;
  1355. for (;;) {
  1356. pci_dbg(pdev, "restoring config space at offset %#x (was %#x, writing %#x)\n",
  1357. offset, val, saved_val);
  1358. pci_write_config_dword(pdev, offset, saved_val);
  1359. if (retry-- <= 0)
  1360. return;
  1361. pci_read_config_dword(pdev, offset, &val);
  1362. if (val == saved_val)
  1363. return;
  1364. mdelay(1);
  1365. }
  1366. }
  1367. static void pci_restore_config_space_range(struct pci_dev *pdev,
  1368. int start, int end, int retry,
  1369. bool force)
  1370. {
  1371. int index;
  1372. for (index = end; index >= start; index--)
  1373. pci_restore_config_dword(pdev, 4 * index,
  1374. pdev->saved_config_space[index],
  1375. retry, force);
  1376. }
  1377. static void pci_restore_config_space(struct pci_dev *pdev)
  1378. {
  1379. if (pdev->hdr_type == PCI_HEADER_TYPE_NORMAL) {
  1380. pci_restore_config_space_range(pdev, 10, 15, 0, false);
  1381. /* Restore BARs before the command register. */
  1382. pci_restore_config_space_range(pdev, 4, 9, 10, false);
  1383. pci_restore_config_space_range(pdev, 0, 3, 0, false);
  1384. } else if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
  1385. pci_restore_config_space_range(pdev, 12, 15, 0, false);
  1386. /*
  1387. * Force rewriting of prefetch registers to avoid S3 resume
  1388. * issues on Intel PCI bridges that occur when these
  1389. * registers are not explicitly written.
  1390. */
  1391. pci_restore_config_space_range(pdev, 9, 11, 0, true);
  1392. pci_restore_config_space_range(pdev, 0, 8, 0, false);
  1393. } else {
  1394. pci_restore_config_space_range(pdev, 0, 15, 0, false);
  1395. }
  1396. }
  1397. static void pci_restore_rebar_state(struct pci_dev *pdev)
  1398. {
  1399. unsigned int pos, nbars, i;
  1400. u32 ctrl;
  1401. pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR);
  1402. if (!pos)
  1403. return;
  1404. pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
  1405. nbars = (ctrl & PCI_REBAR_CTRL_NBAR_MASK) >>
  1406. PCI_REBAR_CTRL_NBAR_SHIFT;
  1407. for (i = 0; i < nbars; i++, pos += 8) {
  1408. struct resource *res;
  1409. int bar_idx, size;
  1410. pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
  1411. bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX;
  1412. res = pdev->resource + bar_idx;
  1413. size = ilog2(resource_size(res)) - 20;
  1414. ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE;
  1415. ctrl |= size << PCI_REBAR_CTRL_BAR_SHIFT;
  1416. pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl);
  1417. }
  1418. }
  1419. /**
  1420. * pci_restore_state - Restore the saved state of a PCI device
  1421. * @dev: PCI device that we're dealing with
  1422. */
  1423. void pci_restore_state(struct pci_dev *dev)
  1424. {
  1425. if (!dev->state_saved)
  1426. return;
  1427. /*
  1428. * Restore max latencies (in the LTR capability) before enabling
  1429. * LTR itself (in the PCIe capability).
  1430. */
  1431. pci_restore_ltr_state(dev);
  1432. pci_restore_pcie_state(dev);
  1433. pci_restore_pasid_state(dev);
  1434. pci_restore_pri_state(dev);
  1435. pci_restore_ats_state(dev);
  1436. pci_restore_vc_state(dev);
  1437. pci_restore_rebar_state(dev);
  1438. pci_restore_dpc_state(dev);
  1439. pci_aer_clear_status(dev);
  1440. pci_restore_aer_state(dev);
  1441. pci_restore_config_space(dev);
  1442. pci_restore_pcix_state(dev);
  1443. pci_restore_msi_state(dev);
  1444. /* Restore ACS and IOV configuration state */
  1445. pci_enable_acs(dev);
  1446. pci_restore_iov_state(dev);
  1447. dev->state_saved = false;
  1448. }
  1449. EXPORT_SYMBOL(pci_restore_state);
  1450. struct pci_saved_state {
  1451. u32 config_space[16];
  1452. struct pci_cap_saved_data cap[];
  1453. };
  1454. /**
  1455. * pci_store_saved_state - Allocate and return an opaque struct containing
  1456. * the device saved state.
  1457. * @dev: PCI device that we're dealing with
  1458. *
  1459. * Return NULL if no state or error.
  1460. */
  1461. struct pci_saved_state *pci_store_saved_state(struct pci_dev *dev)
  1462. {
  1463. struct pci_saved_state *state;
  1464. struct pci_cap_saved_state *tmp;
  1465. struct pci_cap_saved_data *cap;
  1466. size_t size;
  1467. if (!dev->state_saved)
  1468. return NULL;
  1469. size = sizeof(*state) + sizeof(struct pci_cap_saved_data);
  1470. hlist_for_each_entry(tmp, &dev->saved_cap_space, next)
  1471. size += sizeof(struct pci_cap_saved_data) + tmp->cap.size;
  1472. state = kzalloc(size, GFP_KERNEL);
  1473. if (!state)
  1474. return NULL;
  1475. memcpy(state->config_space, dev->saved_config_space,
  1476. sizeof(state->config_space));
  1477. cap = state->cap;
  1478. hlist_for_each_entry(tmp, &dev->saved_cap_space, next) {
  1479. size_t len = sizeof(struct pci_cap_saved_data) + tmp->cap.size;
  1480. memcpy(cap, &tmp->cap, len);
  1481. cap = (struct pci_cap_saved_data *)((u8 *)cap + len);
  1482. }
  1483. /* Empty cap_save terminates list */
  1484. return state;
  1485. }
  1486. EXPORT_SYMBOL_GPL(pci_store_saved_state);
  1487. /**
  1488. * pci_load_saved_state - Reload the provided save state into struct pci_dev.
  1489. * @dev: PCI device that we're dealing with
  1490. * @state: Saved state returned from pci_store_saved_state()
  1491. */
  1492. int pci_load_saved_state(struct pci_dev *dev,
  1493. struct pci_saved_state *state)
  1494. {
  1495. struct pci_cap_saved_data *cap;
  1496. dev->state_saved = false;
  1497. if (!state)
  1498. return 0;
  1499. memcpy(dev->saved_config_space, state->config_space,
  1500. sizeof(state->config_space));
  1501. cap = state->cap;
  1502. while (cap->size) {
  1503. struct pci_cap_saved_state *tmp;
  1504. tmp = _pci_find_saved_cap(dev, cap->cap_nr, cap->cap_extended);
  1505. if (!tmp || tmp->cap.size != cap->size)
  1506. return -EINVAL;
  1507. memcpy(tmp->cap.data, cap->data, tmp->cap.size);
  1508. cap = (struct pci_cap_saved_data *)((u8 *)cap +
  1509. sizeof(struct pci_cap_saved_data) + cap->size);
  1510. }
  1511. dev->state_saved = true;
  1512. return 0;
  1513. }
  1514. EXPORT_SYMBOL_GPL(pci_load_saved_state);
  1515. /**
  1516. * pci_load_and_free_saved_state - Reload the save state pointed to by state,
  1517. * and free the memory allocated for it.
  1518. * @dev: PCI device that we're dealing with
  1519. * @state: Pointer to saved state returned from pci_store_saved_state()
  1520. */
  1521. int pci_load_and_free_saved_state(struct pci_dev *dev,
  1522. struct pci_saved_state **state)
  1523. {
  1524. int ret = pci_load_saved_state(dev, *state);
  1525. kfree(*state);
  1526. *state = NULL;
  1527. return ret;
  1528. }
  1529. EXPORT_SYMBOL_GPL(pci_load_and_free_saved_state);
  1530. int __weak pcibios_enable_device(struct pci_dev *dev, int bars)
  1531. {
  1532. return pci_enable_resources(dev, bars);
  1533. }
  1534. static int do_pci_enable_device(struct pci_dev *dev, int bars)
  1535. {
  1536. int err;
  1537. struct pci_dev *bridge;
  1538. u16 cmd;
  1539. u8 pin;
  1540. err = pci_set_power_state(dev, PCI_D0);
  1541. if (err < 0 && err != -EIO)
  1542. return err;
  1543. bridge = pci_upstream_bridge(dev);
  1544. if (bridge)
  1545. pcie_aspm_powersave_config_link(bridge);
  1546. err = pcibios_enable_device(dev, bars);
  1547. if (err < 0)
  1548. return err;
  1549. pci_fixup_device(pci_fixup_enable, dev);
  1550. if (dev->msi_enabled || dev->msix_enabled)
  1551. return 0;
  1552. pci_read_config_byte(dev, PCI_INTERRUPT_PIN, &pin);
  1553. if (pin) {
  1554. pci_read_config_word(dev, PCI_COMMAND, &cmd);
  1555. if (cmd & PCI_COMMAND_INTX_DISABLE)
  1556. pci_write_config_word(dev, PCI_COMMAND,
  1557. cmd & ~PCI_COMMAND_INTX_DISABLE);
  1558. }
  1559. return 0;
  1560. }
  1561. /**
  1562. * pci_reenable_device - Resume abandoned device
  1563. * @dev: PCI device to be resumed
  1564. *
  1565. * NOTE: This function is a backend of pci_default_resume() and is not supposed
  1566. * to be called by normal code, write proper resume handler and use it instead.
  1567. */
  1568. int pci_reenable_device(struct pci_dev *dev)
  1569. {
  1570. if (pci_is_enabled(dev))
  1571. return do_pci_enable_device(dev, (1 << PCI_NUM_RESOURCES) - 1);
  1572. return 0;
  1573. }
  1574. EXPORT_SYMBOL(pci_reenable_device);
  1575. static void pci_enable_bridge(struct pci_dev *dev)
  1576. {
  1577. struct pci_dev *bridge;
  1578. int retval;
  1579. bridge = pci_upstream_bridge(dev);
  1580. if (bridge)
  1581. pci_enable_bridge(bridge);
  1582. if (pci_is_enabled(dev)) {
  1583. if (!dev->is_busmaster)
  1584. pci_set_master(dev);
  1585. return;
  1586. }
  1587. retval = pci_enable_device(dev);
  1588. if (retval)
  1589. pci_err(dev, "Error enabling bridge (%d), continuing\n",
  1590. retval);
  1591. pci_set_master(dev);
  1592. }
  1593. static int pci_enable_device_flags(struct pci_dev *dev, unsigned long flags)
  1594. {
  1595. struct pci_dev *bridge;
  1596. int err;
  1597. int i, bars = 0;
  1598. /*
  1599. * Power state could be unknown at this point, either due to a fresh
  1600. * boot or a device removal call. So get the current power state
  1601. * so that things like MSI message writing will behave as expected
  1602. * (e.g. if the device really is in D0 at enable time).
  1603. */
  1604. pci_update_current_state(dev, dev->current_state);
  1605. if (atomic_inc_return(&dev->enable_cnt) > 1)
  1606. return 0; /* already enabled */
  1607. bridge = pci_upstream_bridge(dev);
  1608. if (bridge)
  1609. pci_enable_bridge(bridge);
  1610. /* only skip sriov related */
  1611. for (i = 0; i <= PCI_ROM_RESOURCE; i++)
  1612. if (dev->resource[i].flags & flags)
  1613. bars |= (1 << i);
  1614. for (i = PCI_BRIDGE_RESOURCES; i < DEVICE_COUNT_RESOURCE; i++)
  1615. if (dev->resource[i].flags & flags)
  1616. bars |= (1 << i);
  1617. err = do_pci_enable_device(dev, bars);
  1618. if (err < 0)
  1619. atomic_dec(&dev->enable_cnt);
  1620. return err;
  1621. }
  1622. /**
  1623. * pci_enable_device_io - Initialize a device for use with IO space
  1624. * @dev: PCI device to be initialized
  1625. *
  1626. * Initialize device before it's used by a driver. Ask low-level code
  1627. * to enable I/O resources. Wake up the device if it was suspended.
  1628. * Beware, this function can fail.
  1629. */
  1630. int pci_enable_device_io(struct pci_dev *dev)
  1631. {
  1632. return pci_enable_device_flags(dev, IORESOURCE_IO);
  1633. }
  1634. EXPORT_SYMBOL(pci_enable_device_io);
  1635. /**
  1636. * pci_enable_device_mem - Initialize a device for use with Memory space
  1637. * @dev: PCI device to be initialized
  1638. *
  1639. * Initialize device before it's used by a driver. Ask low-level code
  1640. * to enable Memory resources. Wake up the device if it was suspended.
  1641. * Beware, this function can fail.
  1642. */
  1643. int pci_enable_device_mem(struct pci_dev *dev)
  1644. {
  1645. return pci_enable_device_flags(dev, IORESOURCE_MEM);
  1646. }
  1647. EXPORT_SYMBOL(pci_enable_device_mem);
  1648. /**
  1649. * pci_enable_device - Initialize device before it's used by a driver.
  1650. * @dev: PCI device to be initialized
  1651. *
  1652. * Initialize device before it's used by a driver. Ask low-level code
  1653. * to enable I/O and memory. Wake up the device if it was suspended.
  1654. * Beware, this function can fail.
  1655. *
  1656. * Note we don't actually enable the device many times if we call
  1657. * this function repeatedly (we just increment the count).
  1658. */
  1659. int pci_enable_device(struct pci_dev *dev)
  1660. {
  1661. return pci_enable_device_flags(dev, IORESOURCE_MEM | IORESOURCE_IO);
  1662. }
  1663. EXPORT_SYMBOL(pci_enable_device);
  1664. /*
  1665. * Managed PCI resources. This manages device on/off, INTx/MSI/MSI-X
  1666. * on/off and BAR regions. pci_dev itself records MSI/MSI-X status, so
  1667. * there's no need to track it separately. pci_devres is initialized
  1668. * when a device is enabled using managed PCI device enable interface.
  1669. */
  1670. struct pci_devres {
  1671. unsigned int enabled:1;
  1672. unsigned int pinned:1;
  1673. unsigned int orig_intx:1;
  1674. unsigned int restore_intx:1;
  1675. unsigned int mwi:1;
  1676. u32 region_mask;
  1677. };
  1678. static void pcim_release(struct device *gendev, void *res)
  1679. {
  1680. struct pci_dev *dev = to_pci_dev(gendev);
  1681. struct pci_devres *this = res;
  1682. int i;
  1683. if (dev->msi_enabled)
  1684. pci_disable_msi(dev);
  1685. if (dev->msix_enabled)
  1686. pci_disable_msix(dev);
  1687. for (i = 0; i < DEVICE_COUNT_RESOURCE; i++)
  1688. if (this->region_mask & (1 << i))
  1689. pci_release_region(dev, i);
  1690. if (this->mwi)
  1691. pci_clear_mwi(dev);
  1692. if (this->restore_intx)
  1693. pci_intx(dev, this->orig_intx);
  1694. if (this->enabled && !this->pinned)
  1695. pci_disable_device(dev);
  1696. }
  1697. static struct pci_devres *get_pci_dr(struct pci_dev *pdev)
  1698. {
  1699. struct pci_devres *dr, *new_dr;
  1700. dr = devres_find(&pdev->dev, pcim_release, NULL, NULL);
  1701. if (dr)
  1702. return dr;
  1703. new_dr = devres_alloc(pcim_release, sizeof(*new_dr), GFP_KERNEL);
  1704. if (!new_dr)
  1705. return NULL;
  1706. return devres_get(&pdev->dev, new_dr, NULL, NULL);
  1707. }
  1708. static struct pci_devres *find_pci_dr(struct pci_dev *pdev)
  1709. {
  1710. if (pci_is_managed(pdev))
  1711. return devres_find(&pdev->dev, pcim_release, NULL, NULL);
  1712. return NULL;
  1713. }
  1714. /**
  1715. * pcim_enable_device - Managed pci_enable_device()
  1716. * @pdev: PCI device to be initialized
  1717. *
  1718. * Managed pci_enable_device().
  1719. */
  1720. int pcim_enable_device(struct pci_dev *pdev)
  1721. {
  1722. struct pci_devres *dr;
  1723. int rc;
  1724. dr = get_pci_dr(pdev);
  1725. if (unlikely(!dr))
  1726. return -ENOMEM;
  1727. if (dr->enabled)
  1728. return 0;
  1729. rc = pci_enable_device(pdev);
  1730. if (!rc) {
  1731. pdev->is_managed = 1;
  1732. dr->enabled = 1;
  1733. }
  1734. return rc;
  1735. }
  1736. EXPORT_SYMBOL(pcim_enable_device);
  1737. /**
  1738. * pcim_pin_device - Pin managed PCI device
  1739. * @pdev: PCI device to pin
  1740. *
  1741. * Pin managed PCI device @pdev. Pinned device won't be disabled on
  1742. * driver detach. @pdev must have been enabled with
  1743. * pcim_enable_device().
  1744. */
  1745. void pcim_pin_device(struct pci_dev *pdev)
  1746. {
  1747. struct pci_devres *dr;
  1748. dr = find_pci_dr(pdev);
  1749. WARN_ON(!dr || !dr->enabled);
  1750. if (dr)
  1751. dr->pinned = 1;
  1752. }
  1753. EXPORT_SYMBOL(pcim_pin_device);
  1754. /*
  1755. * pcibios_add_device - provide arch specific hooks when adding device dev
  1756. * @dev: the PCI device being added
  1757. *
  1758. * Permits the platform to provide architecture specific functionality when
  1759. * devices are added. This is the default implementation. Architecture
  1760. * implementations can override this.
  1761. */
  1762. int __weak pcibios_add_device(struct pci_dev *dev)
  1763. {
  1764. return 0;
  1765. }
  1766. /**
  1767. * pcibios_release_device - provide arch specific hooks when releasing
  1768. * device dev
  1769. * @dev: the PCI device being released
  1770. *
  1771. * Permits the platform to provide architecture specific functionality when
  1772. * devices are released. This is the default implementation. Architecture
  1773. * implementations can override this.
  1774. */
  1775. void __weak pcibios_release_device(struct pci_dev *dev) {}
  1776. /**
  1777. * pcibios_disable_device - disable arch specific PCI resources for device dev
  1778. * @dev: the PCI device to disable
  1779. *
  1780. * Disables architecture specific PCI resources for the device. This
  1781. * is the default implementation. Architecture implementations can
  1782. * override this.
  1783. */
  1784. void __weak pcibios_disable_device(struct pci_dev *dev) {}
  1785. /**
  1786. * pcibios_penalize_isa_irq - penalize an ISA IRQ
  1787. * @irq: ISA IRQ to penalize
  1788. * @active: IRQ active or not
  1789. *
  1790. * Permits the platform to provide architecture-specific functionality when
  1791. * penalizing ISA IRQs. This is the default implementation. Architecture
  1792. * implementations can override this.
  1793. */
  1794. void __weak pcibios_penalize_isa_irq(int irq, int active) {}
  1795. static void do_pci_disable_device(struct pci_dev *dev)
  1796. {
  1797. u16 pci_command;
  1798. pci_read_config_word(dev, PCI_COMMAND, &pci_command);
  1799. if (pci_command & PCI_COMMAND_MASTER) {
  1800. pci_command &= ~PCI_COMMAND_MASTER;
  1801. pci_write_config_word(dev, PCI_COMMAND, pci_command);
  1802. }
  1803. pcibios_disable_device(dev);
  1804. }
  1805. /**
  1806. * pci_disable_enabled_device - Disable device without updating enable_cnt
  1807. * @dev: PCI device to disable
  1808. *
  1809. * NOTE: This function is a backend of PCI power management routines and is
  1810. * not supposed to be called drivers.
  1811. */
  1812. void pci_disable_enabled_device(struct pci_dev *dev)
  1813. {
  1814. if (pci_is_enabled(dev))
  1815. do_pci_disable_device(dev);
  1816. }
  1817. /**
  1818. * pci_disable_device - Disable PCI device after use
  1819. * @dev: PCI device to be disabled
  1820. *
  1821. * Signal to the system that the PCI device is not in use by the system
  1822. * anymore. This only involves disabling PCI bus-mastering, if active.
  1823. *
  1824. * Note we don't actually disable the device until all callers of
  1825. * pci_enable_device() have called pci_disable_device().
  1826. */
  1827. void pci_disable_device(struct pci_dev *dev)
  1828. {
  1829. struct pci_devres *dr;
  1830. dr = find_pci_dr(dev);
  1831. if (dr)
  1832. dr->enabled = 0;
  1833. dev_WARN_ONCE(&dev->dev, atomic_read(&dev->enable_cnt) <= 0,
  1834. "disabling already-disabled device");
  1835. if (atomic_dec_return(&dev->enable_cnt) != 0)
  1836. return;
  1837. do_pci_disable_device(dev);
  1838. dev->is_busmaster = 0;
  1839. }
  1840. EXPORT_SYMBOL(pci_disable_device);
  1841. /**
  1842. * pcibios_set_pcie_reset_state - set reset state for device dev
  1843. * @dev: the PCIe device reset
  1844. * @state: Reset state to enter into
  1845. *
  1846. * Set the PCIe reset state for the device. This is the default
  1847. * implementation. Architecture implementations can override this.
  1848. */
  1849. int __weak pcibios_set_pcie_reset_state(struct pci_dev *dev,
  1850. enum pcie_reset_state state)
  1851. {
  1852. return -EINVAL;
  1853. }
  1854. /**
  1855. * pci_set_pcie_reset_state - set reset state for device dev
  1856. * @dev: the PCIe device reset
  1857. * @state: Reset state to enter into
  1858. *
  1859. * Sets the PCI reset state for the device.
  1860. */
  1861. int pci_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state state)
  1862. {
  1863. return pcibios_set_pcie_reset_state(dev, state);
  1864. }
  1865. EXPORT_SYMBOL_GPL(pci_set_pcie_reset_state);
  1866. void pcie_clear_device_status(struct pci_dev *dev)
  1867. {
  1868. u16 sta;
  1869. pcie_capability_read_word(dev, PCI_EXP_DEVSTA, &sta);
  1870. pcie_capability_write_word(dev, PCI_EXP_DEVSTA, sta);
  1871. }
  1872. /**
  1873. * pcie_clear_root_pme_status - Clear root port PME interrupt status.
  1874. * @dev: PCIe root port or event collector.
  1875. */
  1876. void pcie_clear_root_pme_status(struct pci_dev *dev)
  1877. {
  1878. pcie_capability_set_dword(dev, PCI_EXP_RTSTA, PCI_EXP_RTSTA_PME);
  1879. }
  1880. /**
  1881. * pci_check_pme_status - Check if given device has generated PME.
  1882. * @dev: Device to check.
  1883. *
  1884. * Check the PME status of the device and if set, clear it and clear PME enable
  1885. * (if set). Return 'true' if PME status and PME enable were both set or
  1886. * 'false' otherwise.
  1887. */
  1888. bool pci_check_pme_status(struct pci_dev *dev)
  1889. {
  1890. int pmcsr_pos;
  1891. u16 pmcsr;
  1892. bool ret = false;
  1893. if (!dev->pm_cap)
  1894. return false;
  1895. pmcsr_pos = dev->pm_cap + PCI_PM_CTRL;
  1896. pci_read_config_word(dev, pmcsr_pos, &pmcsr);
  1897. if (!(pmcsr & PCI_PM_CTRL_PME_STATUS))
  1898. return false;
  1899. /* Clear PME status. */
  1900. pmcsr |= PCI_PM_CTRL_PME_STATUS;
  1901. if (pmcsr & PCI_PM_CTRL_PME_ENABLE) {
  1902. /* Disable PME to avoid interrupt flood. */
  1903. pmcsr &= ~PCI_PM_CTRL_PME_ENABLE;
  1904. ret = true;
  1905. }
  1906. pci_write_config_word(dev, pmcsr_pos, pmcsr);
  1907. return ret;
  1908. }
  1909. /**
  1910. * pci_pme_wakeup - Wake up a PCI device if its PME Status bit is set.
  1911. * @dev: Device to handle.
  1912. * @pme_poll_reset: Whether or not to reset the device's pme_poll flag.
  1913. *
  1914. * Check if @dev has generated PME and queue a resume request for it in that
  1915. * case.
  1916. */
  1917. static int pci_pme_wakeup(struct pci_dev *dev, void *pme_poll_reset)
  1918. {
  1919. if (pme_poll_reset && dev->pme_poll)
  1920. dev->pme_poll = false;
  1921. if (pci_check_pme_status(dev)) {
  1922. pci_wakeup_event(dev);
  1923. pm_request_resume(&dev->dev);
  1924. }
  1925. return 0;
  1926. }
  1927. /**
  1928. * pci_pme_wakeup_bus - Walk given bus and wake up devices on it, if necessary.
  1929. * @bus: Top bus of the subtree to walk.
  1930. */
  1931. void pci_pme_wakeup_bus(struct pci_bus *bus)
  1932. {
  1933. if (bus)
  1934. pci_walk_bus(bus, pci_pme_wakeup, (void *)true);
  1935. }
  1936. /**
  1937. * pci_pme_capable - check the capability of PCI device to generate PME#
  1938. * @dev: PCI device to handle.
  1939. * @state: PCI state from which device will issue PME#.
  1940. */
  1941. bool pci_pme_capable(struct pci_dev *dev, pci_power_t state)
  1942. {
  1943. if (!dev->pm_cap)
  1944. return false;
  1945. return !!(dev->pme_support & (1 << state));
  1946. }
  1947. EXPORT_SYMBOL(pci_pme_capable);
  1948. static void pci_pme_list_scan(struct work_struct *work)
  1949. {
  1950. struct pci_pme_device *pme_dev, *n;
  1951. mutex_lock(&pci_pme_list_mutex);
  1952. list_for_each_entry_safe(pme_dev, n, &pci_pme_list, list) {
  1953. if (pme_dev->dev->pme_poll) {
  1954. struct pci_dev *bridge;
  1955. bridge = pme_dev->dev->bus->self;
  1956. /*
  1957. * If bridge is in low power state, the
  1958. * configuration space of subordinate devices
  1959. * may be not accessible
  1960. */
  1961. if (bridge && bridge->current_state != PCI_D0)
  1962. continue;
  1963. /*
  1964. * If the device is in D3cold it should not be
  1965. * polled either.
  1966. */
  1967. if (pme_dev->dev->current_state == PCI_D3cold)
  1968. continue;
  1969. pci_pme_wakeup(pme_dev->dev, NULL);
  1970. } else {
  1971. list_del(&pme_dev->list);
  1972. kfree(pme_dev);
  1973. }
  1974. }
  1975. if (!list_empty(&pci_pme_list))
  1976. queue_delayed_work(system_freezable_wq, &pci_pme_work,
  1977. msecs_to_jiffies(PME_TIMEOUT));
  1978. mutex_unlock(&pci_pme_list_mutex);
  1979. }
  1980. static void __pci_pme_active(struct pci_dev *dev, bool enable)
  1981. {
  1982. u16 pmcsr;
  1983. if (!dev->pme_support)
  1984. return;
  1985. pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
  1986. /* Clear PME_Status by writing 1 to it and enable PME# */
  1987. pmcsr |= PCI_PM_CTRL_PME_STATUS | PCI_PM_CTRL_PME_ENABLE;
  1988. if (!enable)
  1989. pmcsr &= ~PCI_PM_CTRL_PME_ENABLE;
  1990. pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr);
  1991. }
  1992. /**
  1993. * pci_pme_restore - Restore PME configuration after config space restore.
  1994. * @dev: PCI device to update.
  1995. */
  1996. void pci_pme_restore(struct pci_dev *dev)
  1997. {
  1998. u16 pmcsr;
  1999. if (!dev->pme_support)
  2000. return;
  2001. pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
  2002. if (dev->wakeup_prepared) {
  2003. pmcsr |= PCI_PM_CTRL_PME_ENABLE;
  2004. pmcsr &= ~PCI_PM_CTRL_PME_STATUS;
  2005. } else {
  2006. pmcsr &= ~PCI_PM_CTRL_PME_ENABLE;
  2007. pmcsr |= PCI_PM_CTRL_PME_STATUS;
  2008. }
  2009. pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr);
  2010. }
  2011. /**
  2012. * pci_pme_active - enable or disable PCI device's PME# function
  2013. * @dev: PCI device to handle.
  2014. * @enable: 'true' to enable PME# generation; 'false' to disable it.
  2015. *
  2016. * The caller must verify that the device is capable of generating PME# before
  2017. * calling this function with @enable equal to 'true'.
  2018. */
  2019. void pci_pme_active(struct pci_dev *dev, bool enable)
  2020. {
  2021. __pci_pme_active(dev, enable);
  2022. /*
  2023. * PCI (as opposed to PCIe) PME requires that the device have
  2024. * its PME# line hooked up correctly. Not all hardware vendors
  2025. * do this, so the PME never gets delivered and the device
  2026. * remains asleep. The easiest way around this is to
  2027. * periodically walk the list of suspended devices and check
  2028. * whether any have their PME flag set. The assumption is that
  2029. * we'll wake up often enough anyway that this won't be a huge
  2030. * hit, and the power savings from the devices will still be a
  2031. * win.
  2032. *
  2033. * Although PCIe uses in-band PME message instead of PME# line
  2034. * to report PME, PME does not work for some PCIe devices in
  2035. * reality. For example, there are devices that set their PME
  2036. * status bits, but don't really bother to send a PME message;
  2037. * there are PCI Express Root Ports that don't bother to
  2038. * trigger interrupts when they receive PME messages from the
  2039. * devices below. So PME poll is used for PCIe devices too.
  2040. */
  2041. if (dev->pme_poll) {
  2042. struct pci_pme_device *pme_dev;
  2043. if (enable) {
  2044. pme_dev = kmalloc(sizeof(struct pci_pme_device),
  2045. GFP_KERNEL);
  2046. if (!pme_dev) {
  2047. pci_warn(dev, "can't enable PME#\n");
  2048. return;
  2049. }
  2050. pme_dev->dev = dev;
  2051. mutex_lock(&pci_pme_list_mutex);
  2052. list_add(&pme_dev->list, &pci_pme_list);
  2053. if (list_is_singular(&pci_pme_list))
  2054. queue_delayed_work(system_freezable_wq,
  2055. &pci_pme_work,
  2056. msecs_to_jiffies(PME_TIMEOUT));
  2057. mutex_unlock(&pci_pme_list_mutex);
  2058. } else {
  2059. mutex_lock(&pci_pme_list_mutex);
  2060. list_for_each_entry(pme_dev, &pci_pme_list, list) {
  2061. if (pme_dev->dev == dev) {
  2062. list_del(&pme_dev->list);
  2063. kfree(pme_dev);
  2064. break;
  2065. }
  2066. }
  2067. mutex_unlock(&pci_pme_list_mutex);
  2068. }
  2069. }
  2070. pci_dbg(dev, "PME# %s\n", enable ? "enabled" : "disabled");
  2071. }
  2072. EXPORT_SYMBOL(pci_pme_active);
  2073. /**
  2074. * __pci_enable_wake - enable PCI device as wakeup event source
  2075. * @dev: PCI device affected
  2076. * @state: PCI state from which device will issue wakeup events
  2077. * @enable: True to enable event generation; false to disable
  2078. *
  2079. * This enables the device as a wakeup event source, or disables it.
  2080. * When such events involves platform-specific hooks, those hooks are
  2081. * called automatically by this routine.
  2082. *
  2083. * Devices with legacy power management (no standard PCI PM capabilities)
  2084. * always require such platform hooks.
  2085. *
  2086. * RETURN VALUE:
  2087. * 0 is returned on success
  2088. * -EINVAL is returned if device is not supposed to wake up the system
  2089. * Error code depending on the platform is returned if both the platform and
  2090. * the native mechanism fail to enable the generation of wake-up events
  2091. */
  2092. static int __pci_enable_wake(struct pci_dev *dev, pci_power_t state, bool enable)
  2093. {
  2094. int ret = 0;
  2095. /*
  2096. * Bridges that are not power-manageable directly only signal
  2097. * wakeup on behalf of subordinate devices which is set up
  2098. * elsewhere, so skip them. However, bridges that are
  2099. * power-manageable may signal wakeup for themselves (for example,
  2100. * on a hotplug event) and they need to be covered here.
  2101. */
  2102. if (!pci_power_manageable(dev))
  2103. return 0;
  2104. /* Don't do the same thing twice in a row for one device. */
  2105. if (!!enable == !!dev->wakeup_prepared)
  2106. return 0;
  2107. /*
  2108. * According to "PCI System Architecture" 4th ed. by Tom Shanley & Don
  2109. * Anderson we should be doing PME# wake enable followed by ACPI wake
  2110. * enable. To disable wake-up we call the platform first, for symmetry.
  2111. */
  2112. if (enable) {
  2113. int error;
  2114. /*
  2115. * Enable PME signaling if the device can signal PME from
  2116. * D3cold regardless of whether or not it can signal PME from
  2117. * the current target state, because that will allow it to
  2118. * signal PME when the hierarchy above it goes into D3cold and
  2119. * the device itself ends up in D3cold as a result of that.
  2120. */
  2121. if (pci_pme_capable(dev, state) || pci_pme_capable(dev, PCI_D3cold))
  2122. pci_pme_active(dev, true);
  2123. else
  2124. ret = 1;
  2125. error = platform_pci_set_wakeup(dev, true);
  2126. if (ret)
  2127. ret = error;
  2128. if (!ret)
  2129. dev->wakeup_prepared = true;
  2130. } else {
  2131. platform_pci_set_wakeup(dev, false);
  2132. pci_pme_active(dev, false);
  2133. dev->wakeup_prepared = false;
  2134. }
  2135. return ret;
  2136. }
  2137. /**
  2138. * pci_enable_wake - change wakeup settings for a PCI device
  2139. * @pci_dev: Target device
  2140. * @state: PCI state from which device will issue wakeup events
  2141. * @enable: Whether or not to enable event generation
  2142. *
  2143. * If @enable is set, check device_may_wakeup() for the device before calling
  2144. * __pci_enable_wake() for it.
  2145. */
  2146. int pci_enable_wake(struct pci_dev *pci_dev, pci_power_t state, bool enable)
  2147. {
  2148. if (enable && !device_may_wakeup(&pci_dev->dev))
  2149. return -EINVAL;
  2150. return __pci_enable_wake(pci_dev, state, enable);
  2151. }
  2152. EXPORT_SYMBOL(pci_enable_wake);
  2153. /**
  2154. * pci_wake_from_d3 - enable/disable device to wake up from D3_hot or D3_cold
  2155. * @dev: PCI device to prepare
  2156. * @enable: True to enable wake-up event generation; false to disable
  2157. *
  2158. * Many drivers want the device to wake up the system from D3_hot or D3_cold
  2159. * and this function allows them to set that up cleanly - pci_enable_wake()
  2160. * should not be called twice in a row to enable wake-up due to PCI PM vs ACPI
  2161. * ordering constraints.
  2162. *
  2163. * This function only returns error code if the device is not allowed to wake
  2164. * up the system from sleep or it is not capable of generating PME# from both
  2165. * D3_hot and D3_cold and the platform is unable to enable wake-up power for it.
  2166. */
  2167. int pci_wake_from_d3(struct pci_dev *dev, bool enable)
  2168. {
  2169. return pci_pme_capable(dev, PCI_D3cold) ?
  2170. pci_enable_wake(dev, PCI_D3cold, enable) :
  2171. pci_enable_wake(dev, PCI_D3hot, enable);
  2172. }
  2173. EXPORT_SYMBOL(pci_wake_from_d3);
  2174. /**
  2175. * pci_target_state - find an appropriate low power state for a given PCI dev
  2176. * @dev: PCI device
  2177. * @wakeup: Whether or not wakeup functionality will be enabled for the device.
  2178. *
  2179. * Use underlying platform code to find a supported low power state for @dev.
  2180. * If the platform can't manage @dev, return the deepest state from which it
  2181. * can generate wake events, based on any available PME info.
  2182. */
  2183. static pci_power_t pci_target_state(struct pci_dev *dev, bool wakeup)
  2184. {
  2185. pci_power_t target_state = PCI_D3hot;
  2186. if (platform_pci_power_manageable(dev)) {
  2187. /*
  2188. * Call the platform to find the target state for the device.
  2189. */
  2190. pci_power_t state = platform_pci_choose_state(dev);
  2191. switch (state) {
  2192. case PCI_POWER_ERROR:
  2193. case PCI_UNKNOWN:
  2194. break;
  2195. case PCI_D1:
  2196. case PCI_D2:
  2197. if (pci_no_d1d2(dev))
  2198. break;
  2199. fallthrough;
  2200. default:
  2201. target_state = state;
  2202. }
  2203. return target_state;
  2204. }
  2205. if (!dev->pm_cap)
  2206. target_state = PCI_D0;
  2207. /*
  2208. * If the device is in D3cold even though it's not power-manageable by
  2209. * the platform, it may have been powered down by non-standard means.
  2210. * Best to let it slumber.
  2211. */
  2212. if (dev->current_state == PCI_D3cold)
  2213. target_state = PCI_D3cold;
  2214. if (wakeup && dev->pme_support) {
  2215. pci_power_t state = target_state;
  2216. /*
  2217. * Find the deepest state from which the device can generate
  2218. * PME#.
  2219. */
  2220. while (state && !(dev->pme_support & (1 << state)))
  2221. state--;
  2222. if (state)
  2223. return state;
  2224. else if (dev->pme_support & 1)
  2225. return PCI_D0;
  2226. }
  2227. return target_state;
  2228. }
  2229. /**
  2230. * pci_prepare_to_sleep - prepare PCI device for system-wide transition
  2231. * into a sleep state
  2232. * @dev: Device to handle.
  2233. *
  2234. * Choose the power state appropriate for the device depending on whether
  2235. * it can wake up the system and/or is power manageable by the platform
  2236. * (PCI_D3hot is the default) and put the device into that state.
  2237. */
  2238. int pci_prepare_to_sleep(struct pci_dev *dev)
  2239. {
  2240. bool wakeup = device_may_wakeup(&dev->dev);
  2241. pci_power_t target_state = pci_target_state(dev, wakeup);
  2242. int error;
  2243. if (target_state == PCI_POWER_ERROR)
  2244. return -EIO;
  2245. pci_enable_wake(dev, target_state, wakeup);
  2246. error = pci_set_power_state(dev, target_state);
  2247. if (error)
  2248. pci_enable_wake(dev, target_state, false);
  2249. return error;
  2250. }
  2251. EXPORT_SYMBOL(pci_prepare_to_sleep);
  2252. /**
  2253. * pci_back_from_sleep - turn PCI device on during system-wide transition
  2254. * into working state
  2255. * @dev: Device to handle.
  2256. *
  2257. * Disable device's system wake-up capability and put it into D0.
  2258. */
  2259. int pci_back_from_sleep(struct pci_dev *dev)
  2260. {
  2261. pci_enable_wake(dev, PCI_D0, false);
  2262. return pci_set_power_state(dev, PCI_D0);
  2263. }
  2264. EXPORT_SYMBOL(pci_back_from_sleep);
  2265. /**
  2266. * pci_finish_runtime_suspend - Carry out PCI-specific part of runtime suspend.
  2267. * @dev: PCI device being suspended.
  2268. *
  2269. * Prepare @dev to generate wake-up events at run time and put it into a low
  2270. * power state.
  2271. */
  2272. int pci_finish_runtime_suspend(struct pci_dev *dev)
  2273. {
  2274. pci_power_t target_state;
  2275. int error;
  2276. target_state = pci_target_state(dev, device_can_wakeup(&dev->dev));
  2277. if (target_state == PCI_POWER_ERROR)
  2278. return -EIO;
  2279. dev->runtime_d3cold = target_state == PCI_D3cold;
  2280. __pci_enable_wake(dev, target_state, pci_dev_run_wake(dev));
  2281. error = pci_set_power_state(dev, target_state);
  2282. if (error) {
  2283. pci_enable_wake(dev, target_state, false);
  2284. dev->runtime_d3cold = false;
  2285. }
  2286. return error;
  2287. }
  2288. /**
  2289. * pci_dev_run_wake - Check if device can generate run-time wake-up events.
  2290. * @dev: Device to check.
  2291. *
  2292. * Return true if the device itself is capable of generating wake-up events
  2293. * (through the platform or using the native PCIe PME) or if the device supports
  2294. * PME and one of its upstream bridges can generate wake-up events.
  2295. */
  2296. bool pci_dev_run_wake(struct pci_dev *dev)
  2297. {
  2298. struct pci_bus *bus = dev->bus;
  2299. if (!dev->pme_support)
  2300. return false;
  2301. /* PME-capable in principle, but not from the target power state */
  2302. if (!pci_pme_capable(dev, pci_target_state(dev, true)))
  2303. return false;
  2304. if (device_can_wakeup(&dev->dev))
  2305. return true;
  2306. while (bus->parent) {
  2307. struct pci_dev *bridge = bus->self;
  2308. if (device_can_wakeup(&bridge->dev))
  2309. return true;
  2310. bus = bus->parent;
  2311. }
  2312. /* We have reached the root bus. */
  2313. if (bus->bridge)
  2314. return device_can_wakeup(bus->bridge);
  2315. return false;
  2316. }
  2317. EXPORT_SYMBOL_GPL(pci_dev_run_wake);
  2318. /**
  2319. * pci_dev_need_resume - Check if it is necessary to resume the device.
  2320. * @pci_dev: Device to check.
  2321. *
  2322. * Return 'true' if the device is not runtime-suspended or it has to be
  2323. * reconfigured due to wakeup settings difference between system and runtime
  2324. * suspend, or the current power state of it is not suitable for the upcoming
  2325. * (system-wide) transition.
  2326. */
  2327. bool pci_dev_need_resume(struct pci_dev *pci_dev)
  2328. {
  2329. struct device *dev = &pci_dev->dev;
  2330. pci_power_t target_state;
  2331. if (!pm_runtime_suspended(dev) || platform_pci_need_resume(pci_dev))
  2332. return true;
  2333. target_state = pci_target_state(pci_dev, device_may_wakeup(dev));
  2334. /*
  2335. * If the earlier platform check has not triggered, D3cold is just power
  2336. * removal on top of D3hot, so no need to resume the device in that
  2337. * case.
  2338. */
  2339. return target_state != pci_dev->current_state &&
  2340. target_state != PCI_D3cold &&
  2341. pci_dev->current_state != PCI_D3hot;
  2342. }
  2343. /**
  2344. * pci_dev_adjust_pme - Adjust PME setting for a suspended device.
  2345. * @pci_dev: Device to check.
  2346. *
  2347. * If the device is suspended and it is not configured for system wakeup,
  2348. * disable PME for it to prevent it from waking up the system unnecessarily.
  2349. *
  2350. * Note that if the device's power state is D3cold and the platform check in
  2351. * pci_dev_need_resume() has not triggered, the device's configuration need not
  2352. * be changed.
  2353. */
  2354. void pci_dev_adjust_pme(struct pci_dev *pci_dev)
  2355. {
  2356. struct device *dev = &pci_dev->dev;
  2357. spin_lock_irq(&dev->power.lock);
  2358. if (pm_runtime_suspended(dev) && !device_may_wakeup(dev) &&
  2359. pci_dev->current_state < PCI_D3cold)
  2360. __pci_pme_active(pci_dev, false);
  2361. spin_unlock_irq(&dev->power.lock);
  2362. }
  2363. /**
  2364. * pci_dev_complete_resume - Finalize resume from system sleep for a device.
  2365. * @pci_dev: Device to handle.
  2366. *
  2367. * If the device is runtime suspended and wakeup-capable, enable PME for it as
  2368. * it might have been disabled during the prepare phase of system suspend if
  2369. * the device was not configured for system wakeup.
  2370. */
  2371. void pci_dev_complete_resume(struct pci_dev *pci_dev)
  2372. {
  2373. struct device *dev = &pci_dev->dev;
  2374. if (!pci_dev_run_wake(pci_dev))
  2375. return;
  2376. spin_lock_irq(&dev->power.lock);
  2377. if (pm_runtime_suspended(dev) && pci_dev->current_state < PCI_D3cold)
  2378. __pci_pme_active(pci_dev, true);
  2379. spin_unlock_irq(&dev->power.lock);
  2380. }
  2381. void pci_config_pm_runtime_get(struct pci_dev *pdev)
  2382. {
  2383. struct device *dev = &pdev->dev;
  2384. struct device *parent = dev->parent;
  2385. if (parent)
  2386. pm_runtime_get_sync(parent);
  2387. pm_runtime_get_noresume(dev);
  2388. /*
  2389. * pdev->current_state is set to PCI_D3cold during suspending,
  2390. * so wait until suspending completes
  2391. */
  2392. pm_runtime_barrier(dev);
  2393. /*
  2394. * Only need to resume devices in D3cold, because config
  2395. * registers are still accessible for devices suspended but
  2396. * not in D3cold.
  2397. */
  2398. if (pdev->current_state == PCI_D3cold)
  2399. pm_runtime_resume(dev);
  2400. }
  2401. void pci_config_pm_runtime_put(struct pci_dev *pdev)
  2402. {
  2403. struct device *dev = &pdev->dev;
  2404. struct device *parent = dev->parent;
  2405. pm_runtime_put(dev);
  2406. if (parent)
  2407. pm_runtime_put_sync(parent);
  2408. }
  2409. static const struct dmi_system_id bridge_d3_blacklist[] = {
  2410. #ifdef CONFIG_X86
  2411. {
  2412. /*
  2413. * Gigabyte X299 root port is not marked as hotplug capable
  2414. * which allows Linux to power manage it. However, this
  2415. * confuses the BIOS SMI handler so don't power manage root
  2416. * ports on that system.
  2417. */
  2418. .ident = "X299 DESIGNARE EX-CF",
  2419. .matches = {
  2420. DMI_MATCH(DMI_BOARD_VENDOR, "Gigabyte Technology Co., Ltd."),
  2421. DMI_MATCH(DMI_BOARD_NAME, "X299 DESIGNARE EX-CF"),
  2422. },
  2423. },
  2424. #endif
  2425. { }
  2426. };
  2427. /**
  2428. * pci_bridge_d3_possible - Is it possible to put the bridge into D3
  2429. * @bridge: Bridge to check
  2430. *
  2431. * This function checks if it is possible to move the bridge to D3.
  2432. * Currently we only allow D3 for recent enough PCIe ports and Thunderbolt.
  2433. */
  2434. bool pci_bridge_d3_possible(struct pci_dev *bridge)
  2435. {
  2436. if (!pci_is_pcie(bridge))
  2437. return false;
  2438. switch (pci_pcie_type(bridge)) {
  2439. case PCI_EXP_TYPE_ROOT_PORT:
  2440. case PCI_EXP_TYPE_UPSTREAM:
  2441. case PCI_EXP_TYPE_DOWNSTREAM:
  2442. if (pci_bridge_d3_disable)
  2443. return false;
  2444. /*
  2445. * Hotplug ports handled by firmware in System Management Mode
  2446. * may not be put into D3 by the OS (Thunderbolt on non-Macs).
  2447. */
  2448. if (bridge->is_hotplug_bridge && !pciehp_is_native(bridge))
  2449. return false;
  2450. if (pci_bridge_d3_force)
  2451. return true;
  2452. /* Even the oldest 2010 Thunderbolt controller supports D3. */
  2453. if (bridge->is_thunderbolt)
  2454. return true;
  2455. /* Platform might know better if the bridge supports D3 */
  2456. if (platform_pci_bridge_d3(bridge))
  2457. return true;
  2458. /*
  2459. * Hotplug ports handled natively by the OS were not validated
  2460. * by vendors for runtime D3 at least until 2018 because there
  2461. * was no OS support.
  2462. */
  2463. if (bridge->is_hotplug_bridge)
  2464. return false;
  2465. if (dmi_check_system(bridge_d3_blacklist))
  2466. return false;
  2467. /*
  2468. * It should be safe to put PCIe ports from 2015 or newer
  2469. * to D3.
  2470. */
  2471. if (dmi_get_bios_year() >= 2015)
  2472. return true;
  2473. break;
  2474. }
  2475. return false;
  2476. }
  2477. static int pci_dev_check_d3cold(struct pci_dev *dev, void *data)
  2478. {
  2479. bool *d3cold_ok = data;
  2480. if (/* The device needs to be allowed to go D3cold ... */
  2481. dev->no_d3cold || !dev->d3cold_allowed ||
  2482. /* ... and if it is wakeup capable to do so from D3cold. */
  2483. (device_may_wakeup(&dev->dev) &&
  2484. !pci_pme_capable(dev, PCI_D3cold)) ||
  2485. /* If it is a bridge it must be allowed to go to D3. */
  2486. !pci_power_manageable(dev))
  2487. *d3cold_ok = false;
  2488. return !*d3cold_ok;
  2489. }
  2490. /*
  2491. * pci_bridge_d3_update - Update bridge D3 capabilities
  2492. * @dev: PCI device which is changed
  2493. *
  2494. * Update upstream bridge PM capabilities accordingly depending on if the
  2495. * device PM configuration was changed or the device is being removed. The
  2496. * change is also propagated upstream.
  2497. */
  2498. void pci_bridge_d3_update(struct pci_dev *dev)
  2499. {
  2500. bool remove = !device_is_registered(&dev->dev);
  2501. struct pci_dev *bridge;
  2502. bool d3cold_ok = true;
  2503. bridge = pci_upstream_bridge(dev);
  2504. if (!bridge || !pci_bridge_d3_possible(bridge))
  2505. return;
  2506. /*
  2507. * If D3 is currently allowed for the bridge, removing one of its
  2508. * children won't change that.
  2509. */
  2510. if (remove && bridge->bridge_d3)
  2511. return;
  2512. /*
  2513. * If D3 is currently allowed for the bridge and a child is added or
  2514. * changed, disallowance of D3 can only be caused by that child, so
  2515. * we only need to check that single device, not any of its siblings.
  2516. *
  2517. * If D3 is currently not allowed for the bridge, checking the device
  2518. * first may allow us to skip checking its siblings.
  2519. */
  2520. if (!remove)
  2521. pci_dev_check_d3cold(dev, &d3cold_ok);
  2522. /*
  2523. * If D3 is currently not allowed for the bridge, this may be caused
  2524. * either by the device being changed/removed or any of its siblings,
  2525. * so we need to go through all children to find out if one of them
  2526. * continues to block D3.
  2527. */
  2528. if (d3cold_ok && !bridge->bridge_d3)
  2529. pci_walk_bus(bridge->subordinate, pci_dev_check_d3cold,
  2530. &d3cold_ok);
  2531. if (bridge->bridge_d3 != d3cold_ok) {
  2532. bridge->bridge_d3 = d3cold_ok;
  2533. /* Propagate change to upstream bridges */
  2534. pci_bridge_d3_update(bridge);
  2535. }
  2536. }
  2537. /**
  2538. * pci_d3cold_enable - Enable D3cold for device
  2539. * @dev: PCI device to handle
  2540. *
  2541. * This function can be used in drivers to enable D3cold from the device
  2542. * they handle. It also updates upstream PCI bridge PM capabilities
  2543. * accordingly.
  2544. */
  2545. void pci_d3cold_enable(struct pci_dev *dev)
  2546. {
  2547. if (dev->no_d3cold) {
  2548. dev->no_d3cold = false;
  2549. pci_bridge_d3_update(dev);
  2550. }
  2551. }
  2552. EXPORT_SYMBOL_GPL(pci_d3cold_enable);
  2553. /**
  2554. * pci_d3cold_disable - Disable D3cold for device
  2555. * @dev: PCI device to handle
  2556. *
  2557. * This function can be used in drivers to disable D3cold from the device
  2558. * they handle. It also updates upstream PCI bridge PM capabilities
  2559. * accordingly.
  2560. */
  2561. void pci_d3cold_disable(struct pci_dev *dev)
  2562. {
  2563. if (!dev->no_d3cold) {
  2564. dev->no_d3cold = true;
  2565. pci_bridge_d3_update(dev);
  2566. }
  2567. }
  2568. EXPORT_SYMBOL_GPL(pci_d3cold_disable);
  2569. /**
  2570. * pci_pm_init - Initialize PM functions of given PCI device
  2571. * @dev: PCI device to handle.
  2572. */
  2573. void pci_pm_init(struct pci_dev *dev)
  2574. {
  2575. int pm;
  2576. u16 status;
  2577. u16 pmc;
  2578. pm_runtime_forbid(&dev->dev);
  2579. pm_runtime_set_active(&dev->dev);
  2580. pm_runtime_enable(&dev->dev);
  2581. device_enable_async_suspend(&dev->dev);
  2582. dev->wakeup_prepared = false;
  2583. dev->pm_cap = 0;
  2584. dev->pme_support = 0;
  2585. /* find PCI PM capability in list */
  2586. pm = pci_find_capability(dev, PCI_CAP_ID_PM);
  2587. if (!pm)
  2588. return;
  2589. /* Check device's ability to generate PME# */
  2590. pci_read_config_word(dev, pm + PCI_PM_PMC, &pmc);
  2591. if ((pmc & PCI_PM_CAP_VER_MASK) > 3) {
  2592. pci_err(dev, "unsupported PM cap regs version (%u)\n",
  2593. pmc & PCI_PM_CAP_VER_MASK);
  2594. return;
  2595. }
  2596. dev->pm_cap = pm;
  2597. dev->d3hot_delay = PCI_PM_D3HOT_WAIT;
  2598. dev->d3cold_delay = PCI_PM_D3COLD_WAIT;
  2599. dev->bridge_d3 = pci_bridge_d3_possible(dev);
  2600. dev->d3cold_allowed = true;
  2601. dev->d1_support = false;
  2602. dev->d2_support = false;
  2603. if (!pci_no_d1d2(dev)) {
  2604. if (pmc & PCI_PM_CAP_D1)
  2605. dev->d1_support = true;
  2606. if (pmc & PCI_PM_CAP_D2)
  2607. dev->d2_support = true;
  2608. if (dev->d1_support || dev->d2_support)
  2609. pci_info(dev, "supports%s%s\n",
  2610. dev->d1_support ? " D1" : "",
  2611. dev->d2_support ? " D2" : "");
  2612. }
  2613. pmc &= PCI_PM_CAP_PME_MASK;
  2614. if (pmc) {
  2615. pci_info(dev, "PME# supported from%s%s%s%s%s\n",
  2616. (pmc & PCI_PM_CAP_PME_D0) ? " D0" : "",
  2617. (pmc & PCI_PM_CAP_PME_D1) ? " D1" : "",
  2618. (pmc & PCI_PM_CAP_PME_D2) ? " D2" : "",
  2619. (pmc & PCI_PM_CAP_PME_D3hot) ? " D3hot" : "",
  2620. (pmc & PCI_PM_CAP_PME_D3cold) ? " D3cold" : "");
  2621. dev->pme_support = pmc >> PCI_PM_CAP_PME_SHIFT;
  2622. dev->pme_poll = true;
  2623. /*
  2624. * Make device's PM flags reflect the wake-up capability, but
  2625. * let the user space enable it to wake up the system as needed.
  2626. */
  2627. device_set_wakeup_capable(&dev->dev, true);
  2628. /* Disable the PME# generation functionality */
  2629. pci_pme_active(dev, false);
  2630. }
  2631. pci_read_config_word(dev, PCI_STATUS, &status);
  2632. if (status & PCI_STATUS_IMM_READY)
  2633. dev->imm_ready = 1;
  2634. }
  2635. static unsigned long pci_ea_flags(struct pci_dev *dev, u8 prop)
  2636. {
  2637. unsigned long flags = IORESOURCE_PCI_FIXED | IORESOURCE_PCI_EA_BEI;
  2638. switch (prop) {
  2639. case PCI_EA_P_MEM:
  2640. case PCI_EA_P_VF_MEM:
  2641. flags |= IORESOURCE_MEM;
  2642. break;
  2643. case PCI_EA_P_MEM_PREFETCH:
  2644. case PCI_EA_P_VF_MEM_PREFETCH:
  2645. flags |= IORESOURCE_MEM | IORESOURCE_PREFETCH;
  2646. break;
  2647. case PCI_EA_P_IO:
  2648. flags |= IORESOURCE_IO;
  2649. break;
  2650. default:
  2651. return 0;
  2652. }
  2653. return flags;
  2654. }
  2655. static struct resource *pci_ea_get_resource(struct pci_dev *dev, u8 bei,
  2656. u8 prop)
  2657. {
  2658. if (bei <= PCI_EA_BEI_BAR5 && prop <= PCI_EA_P_IO)
  2659. return &dev->resource[bei];
  2660. #ifdef CONFIG_PCI_IOV
  2661. else if (bei >= PCI_EA_BEI_VF_BAR0 && bei <= PCI_EA_BEI_VF_BAR5 &&
  2662. (prop == PCI_EA_P_VF_MEM || prop == PCI_EA_P_VF_MEM_PREFETCH))
  2663. return &dev->resource[PCI_IOV_RESOURCES +
  2664. bei - PCI_EA_BEI_VF_BAR0];
  2665. #endif
  2666. else if (bei == PCI_EA_BEI_ROM)
  2667. return &dev->resource[PCI_ROM_RESOURCE];
  2668. else
  2669. return NULL;
  2670. }
  2671. /* Read an Enhanced Allocation (EA) entry */
  2672. static int pci_ea_read(struct pci_dev *dev, int offset)
  2673. {
  2674. struct resource *res;
  2675. int ent_size, ent_offset = offset;
  2676. resource_size_t start, end;
  2677. unsigned long flags;
  2678. u32 dw0, bei, base, max_offset;
  2679. u8 prop;
  2680. bool support_64 = (sizeof(resource_size_t) >= 8);
  2681. pci_read_config_dword(dev, ent_offset, &dw0);
  2682. ent_offset += 4;
  2683. /* Entry size field indicates DWORDs after 1st */
  2684. ent_size = ((dw0 & PCI_EA_ES) + 1) << 2;
  2685. if (!(dw0 & PCI_EA_ENABLE)) /* Entry not enabled */
  2686. goto out;
  2687. bei = (dw0 & PCI_EA_BEI) >> 4;
  2688. prop = (dw0 & PCI_EA_PP) >> 8;
  2689. /*
  2690. * If the Property is in the reserved range, try the Secondary
  2691. * Property instead.
  2692. */
  2693. if (prop > PCI_EA_P_BRIDGE_IO && prop < PCI_EA_P_MEM_RESERVED)
  2694. prop = (dw0 & PCI_EA_SP) >> 16;
  2695. if (prop > PCI_EA_P_BRIDGE_IO)
  2696. goto out;
  2697. res = pci_ea_get_resource(dev, bei, prop);
  2698. if (!res) {
  2699. pci_err(dev, "Unsupported EA entry BEI: %u\n", bei);
  2700. goto out;
  2701. }
  2702. flags = pci_ea_flags(dev, prop);
  2703. if (!flags) {
  2704. pci_err(dev, "Unsupported EA properties: %#x\n", prop);
  2705. goto out;
  2706. }
  2707. /* Read Base */
  2708. pci_read_config_dword(dev, ent_offset, &base);
  2709. start = (base & PCI_EA_FIELD_MASK);
  2710. ent_offset += 4;
  2711. /* Read MaxOffset */
  2712. pci_read_config_dword(dev, ent_offset, &max_offset);
  2713. ent_offset += 4;
  2714. /* Read Base MSBs (if 64-bit entry) */
  2715. if (base & PCI_EA_IS_64) {
  2716. u32 base_upper;
  2717. pci_read_config_dword(dev, ent_offset, &base_upper);
  2718. ent_offset += 4;
  2719. flags |= IORESOURCE_MEM_64;
  2720. /* entry starts above 32-bit boundary, can't use */
  2721. if (!support_64 && base_upper)
  2722. goto out;
  2723. if (support_64)
  2724. start |= ((u64)base_upper << 32);
  2725. }
  2726. end = start + (max_offset | 0x03);
  2727. /* Read MaxOffset MSBs (if 64-bit entry) */
  2728. if (max_offset & PCI_EA_IS_64) {
  2729. u32 max_offset_upper;
  2730. pci_read_config_dword(dev, ent_offset, &max_offset_upper);
  2731. ent_offset += 4;
  2732. flags |= IORESOURCE_MEM_64;
  2733. /* entry too big, can't use */
  2734. if (!support_64 && max_offset_upper)
  2735. goto out;
  2736. if (support_64)
  2737. end += ((u64)max_offset_upper << 32);
  2738. }
  2739. if (end < start) {
  2740. pci_err(dev, "EA Entry crosses address boundary\n");
  2741. goto out;
  2742. }
  2743. if (ent_size != ent_offset - offset) {
  2744. pci_err(dev, "EA Entry Size (%d) does not match length read (%d)\n",
  2745. ent_size, ent_offset - offset);
  2746. goto out;
  2747. }
  2748. res->name = pci_name(dev);
  2749. res->start = start;
  2750. res->end = end;
  2751. res->flags = flags;
  2752. if (bei <= PCI_EA_BEI_BAR5)
  2753. pci_info(dev, "BAR %d: %pR (from Enhanced Allocation, properties %#02x)\n",
  2754. bei, res, prop);
  2755. else if (bei == PCI_EA_BEI_ROM)
  2756. pci_info(dev, "ROM: %pR (from Enhanced Allocation, properties %#02x)\n",
  2757. res, prop);
  2758. else if (bei >= PCI_EA_BEI_VF_BAR0 && bei <= PCI_EA_BEI_VF_BAR5)
  2759. pci_info(dev, "VF BAR %d: %pR (from Enhanced Allocation, properties %#02x)\n",
  2760. bei - PCI_EA_BEI_VF_BAR0, res, prop);
  2761. else
  2762. pci_info(dev, "BEI %d res: %pR (from Enhanced Allocation, properties %#02x)\n",
  2763. bei, res, prop);
  2764. out:
  2765. return offset + ent_size;
  2766. }
  2767. /* Enhanced Allocation Initialization */
  2768. void pci_ea_init(struct pci_dev *dev)
  2769. {
  2770. int ea;
  2771. u8 num_ent;
  2772. int offset;
  2773. int i;
  2774. /* find PCI EA capability in list */
  2775. ea = pci_find_capability(dev, PCI_CAP_ID_EA);
  2776. if (!ea)
  2777. return;
  2778. /* determine the number of entries */
  2779. pci_bus_read_config_byte(dev->bus, dev->devfn, ea + PCI_EA_NUM_ENT,
  2780. &num_ent);
  2781. num_ent &= PCI_EA_NUM_ENT_MASK;
  2782. offset = ea + PCI_EA_FIRST_ENT;
  2783. /* Skip DWORD 2 for type 1 functions */
  2784. if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE)
  2785. offset += 4;
  2786. /* parse each EA entry */
  2787. for (i = 0; i < num_ent; ++i)
  2788. offset = pci_ea_read(dev, offset);
  2789. }
  2790. static void pci_add_saved_cap(struct pci_dev *pci_dev,
  2791. struct pci_cap_saved_state *new_cap)
  2792. {
  2793. hlist_add_head(&new_cap->next, &pci_dev->saved_cap_space);
  2794. }
  2795. /**
  2796. * _pci_add_cap_save_buffer - allocate buffer for saving given
  2797. * capability registers
  2798. * @dev: the PCI device
  2799. * @cap: the capability to allocate the buffer for
  2800. * @extended: Standard or Extended capability ID
  2801. * @size: requested size of the buffer
  2802. */
  2803. static int _pci_add_cap_save_buffer(struct pci_dev *dev, u16 cap,
  2804. bool extended, unsigned int size)
  2805. {
  2806. int pos;
  2807. struct pci_cap_saved_state *save_state;
  2808. if (extended)
  2809. pos = pci_find_ext_capability(dev, cap);
  2810. else
  2811. pos = pci_find_capability(dev, cap);
  2812. if (!pos)
  2813. return 0;
  2814. save_state = kzalloc(sizeof(*save_state) + size, GFP_KERNEL);
  2815. if (!save_state)
  2816. return -ENOMEM;
  2817. save_state->cap.cap_nr = cap;
  2818. save_state->cap.cap_extended = extended;
  2819. save_state->cap.size = size;
  2820. pci_add_saved_cap(dev, save_state);
  2821. return 0;
  2822. }
  2823. int pci_add_cap_save_buffer(struct pci_dev *dev, char cap, unsigned int size)
  2824. {
  2825. return _pci_add_cap_save_buffer(dev, cap, false, size);
  2826. }
  2827. int pci_add_ext_cap_save_buffer(struct pci_dev *dev, u16 cap, unsigned int size)
  2828. {
  2829. return _pci_add_cap_save_buffer(dev, cap, true, size);
  2830. }
  2831. /**
  2832. * pci_allocate_cap_save_buffers - allocate buffers for saving capabilities
  2833. * @dev: the PCI device
  2834. */
  2835. void pci_allocate_cap_save_buffers(struct pci_dev *dev)
  2836. {
  2837. int error;
  2838. error = pci_add_cap_save_buffer(dev, PCI_CAP_ID_EXP,
  2839. PCI_EXP_SAVE_REGS * sizeof(u16));
  2840. if (error)
  2841. pci_err(dev, "unable to preallocate PCI Express save buffer\n");
  2842. error = pci_add_cap_save_buffer(dev, PCI_CAP_ID_PCIX, sizeof(u16));
  2843. if (error)
  2844. pci_err(dev, "unable to preallocate PCI-X save buffer\n");
  2845. error = pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_LTR,
  2846. 2 * sizeof(u16));
  2847. if (error)
  2848. pci_err(dev, "unable to allocate suspend buffer for LTR\n");
  2849. pci_allocate_vc_save_buffers(dev);
  2850. }
  2851. void pci_free_cap_save_buffers(struct pci_dev *dev)
  2852. {
  2853. struct pci_cap_saved_state *tmp;
  2854. struct hlist_node *n;
  2855. hlist_for_each_entry_safe(tmp, n, &dev->saved_cap_space, next)
  2856. kfree(tmp);
  2857. }
  2858. /**
  2859. * pci_configure_ari - enable or disable ARI forwarding
  2860. * @dev: the PCI device
  2861. *
  2862. * If @dev and its upstream bridge both support ARI, enable ARI in the
  2863. * bridge. Otherwise, disable ARI in the bridge.
  2864. */
  2865. void pci_configure_ari(struct pci_dev *dev)
  2866. {
  2867. u32 cap;
  2868. struct pci_dev *bridge;
  2869. if (pcie_ari_disabled || !pci_is_pcie(dev) || dev->devfn)
  2870. return;
  2871. bridge = dev->bus->self;
  2872. if (!bridge)
  2873. return;
  2874. pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &cap);
  2875. if (!(cap & PCI_EXP_DEVCAP2_ARI))
  2876. return;
  2877. if (pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ARI)) {
  2878. pcie_capability_set_word(bridge, PCI_EXP_DEVCTL2,
  2879. PCI_EXP_DEVCTL2_ARI);
  2880. bridge->ari_enabled = 1;
  2881. } else {
  2882. pcie_capability_clear_word(bridge, PCI_EXP_DEVCTL2,
  2883. PCI_EXP_DEVCTL2_ARI);
  2884. bridge->ari_enabled = 0;
  2885. }
  2886. }
  2887. static bool pci_acs_flags_enabled(struct pci_dev *pdev, u16 acs_flags)
  2888. {
  2889. int pos;
  2890. u16 cap, ctrl;
  2891. pos = pdev->acs_cap;
  2892. if (!pos)
  2893. return false;
  2894. /*
  2895. * Except for egress control, capabilities are either required
  2896. * or only required if controllable. Features missing from the
  2897. * capability field can therefore be assumed as hard-wired enabled.
  2898. */
  2899. pci_read_config_word(pdev, pos + PCI_ACS_CAP, &cap);
  2900. acs_flags &= (cap | PCI_ACS_EC);
  2901. pci_read_config_word(pdev, pos + PCI_ACS_CTRL, &ctrl);
  2902. return (ctrl & acs_flags) == acs_flags;
  2903. }
  2904. /**
  2905. * pci_acs_enabled - test ACS against required flags for a given device
  2906. * @pdev: device to test
  2907. * @acs_flags: required PCI ACS flags
  2908. *
  2909. * Return true if the device supports the provided flags. Automatically
  2910. * filters out flags that are not implemented on multifunction devices.
  2911. *
  2912. * Note that this interface checks the effective ACS capabilities of the
  2913. * device rather than the actual capabilities. For instance, most single
  2914. * function endpoints are not required to support ACS because they have no
  2915. * opportunity for peer-to-peer access. We therefore return 'true'
  2916. * regardless of whether the device exposes an ACS capability. This makes
  2917. * it much easier for callers of this function to ignore the actual type
  2918. * or topology of the device when testing ACS support.
  2919. */
  2920. bool pci_acs_enabled(struct pci_dev *pdev, u16 acs_flags)
  2921. {
  2922. int ret;
  2923. ret = pci_dev_specific_acs_enabled(pdev, acs_flags);
  2924. if (ret >= 0)
  2925. return ret > 0;
  2926. /*
  2927. * Conventional PCI and PCI-X devices never support ACS, either
  2928. * effectively or actually. The shared bus topology implies that
  2929. * any device on the bus can receive or snoop DMA.
  2930. */
  2931. if (!pci_is_pcie(pdev))
  2932. return false;
  2933. switch (pci_pcie_type(pdev)) {
  2934. /*
  2935. * PCI/X-to-PCIe bridges are not specifically mentioned by the spec,
  2936. * but since their primary interface is PCI/X, we conservatively
  2937. * handle them as we would a non-PCIe device.
  2938. */
  2939. case PCI_EXP_TYPE_PCIE_BRIDGE:
  2940. /*
  2941. * PCIe 3.0, 6.12.1 excludes ACS on these devices. "ACS is never
  2942. * applicable... must never implement an ACS Extended Capability...".
  2943. * This seems arbitrary, but we take a conservative interpretation
  2944. * of this statement.
  2945. */
  2946. case PCI_EXP_TYPE_PCI_BRIDGE:
  2947. case PCI_EXP_TYPE_RC_EC:
  2948. return false;
  2949. /*
  2950. * PCIe 3.0, 6.12.1.1 specifies that downstream and root ports should
  2951. * implement ACS in order to indicate their peer-to-peer capabilities,
  2952. * regardless of whether they are single- or multi-function devices.
  2953. */
  2954. case PCI_EXP_TYPE_DOWNSTREAM:
  2955. case PCI_EXP_TYPE_ROOT_PORT:
  2956. return pci_acs_flags_enabled(pdev, acs_flags);
  2957. /*
  2958. * PCIe 3.0, 6.12.1.2 specifies ACS capabilities that should be
  2959. * implemented by the remaining PCIe types to indicate peer-to-peer
  2960. * capabilities, but only when they are part of a multifunction
  2961. * device. The footnote for section 6.12 indicates the specific
  2962. * PCIe types included here.
  2963. */
  2964. case PCI_EXP_TYPE_ENDPOINT:
  2965. case PCI_EXP_TYPE_UPSTREAM:
  2966. case PCI_EXP_TYPE_LEG_END:
  2967. case PCI_EXP_TYPE_RC_END:
  2968. if (!pdev->multifunction)
  2969. break;
  2970. return pci_acs_flags_enabled(pdev, acs_flags);
  2971. }
  2972. /*
  2973. * PCIe 3.0, 6.12.1.3 specifies no ACS capabilities are applicable
  2974. * to single function devices with the exception of downstream ports.
  2975. */
  2976. return true;
  2977. }
  2978. /**
  2979. * pci_acs_path_enable - test ACS flags from start to end in a hierarchy
  2980. * @start: starting downstream device
  2981. * @end: ending upstream device or NULL to search to the root bus
  2982. * @acs_flags: required flags
  2983. *
  2984. * Walk up a device tree from start to end testing PCI ACS support. If
  2985. * any step along the way does not support the required flags, return false.
  2986. */
  2987. bool pci_acs_path_enabled(struct pci_dev *start,
  2988. struct pci_dev *end, u16 acs_flags)
  2989. {
  2990. struct pci_dev *pdev, *parent = start;
  2991. do {
  2992. pdev = parent;
  2993. if (!pci_acs_enabled(pdev, acs_flags))
  2994. return false;
  2995. if (pci_is_root_bus(pdev->bus))
  2996. return (end == NULL);
  2997. parent = pdev->bus->self;
  2998. } while (pdev != end);
  2999. return true;
  3000. }
  3001. /**
  3002. * pci_acs_init - Initialize ACS if hardware supports it
  3003. * @dev: the PCI device
  3004. */
  3005. void pci_acs_init(struct pci_dev *dev)
  3006. {
  3007. dev->acs_cap = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ACS);
  3008. /*
  3009. * Attempt to enable ACS regardless of capability because some Root
  3010. * Ports (e.g. those quirked with *_intel_pch_acs_*) do not have
  3011. * the standard ACS capability but still support ACS via those
  3012. * quirks.
  3013. */
  3014. pci_enable_acs(dev);
  3015. }
  3016. /**
  3017. * pci_rebar_find_pos - find position of resize ctrl reg for BAR
  3018. * @pdev: PCI device
  3019. * @bar: BAR to find
  3020. *
  3021. * Helper to find the position of the ctrl register for a BAR.
  3022. * Returns -ENOTSUPP if resizable BARs are not supported at all.
  3023. * Returns -ENOENT if no ctrl register for the BAR could be found.
  3024. */
  3025. static int pci_rebar_find_pos(struct pci_dev *pdev, int bar)
  3026. {
  3027. unsigned int pos, nbars, i;
  3028. u32 ctrl;
  3029. pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR);
  3030. if (!pos)
  3031. return -ENOTSUPP;
  3032. pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
  3033. nbars = (ctrl & PCI_REBAR_CTRL_NBAR_MASK) >>
  3034. PCI_REBAR_CTRL_NBAR_SHIFT;
  3035. for (i = 0; i < nbars; i++, pos += 8) {
  3036. int bar_idx;
  3037. pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
  3038. bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX;
  3039. if (bar_idx == bar)
  3040. return pos;
  3041. }
  3042. return -ENOENT;
  3043. }
  3044. /**
  3045. * pci_rebar_get_possible_sizes - get possible sizes for BAR
  3046. * @pdev: PCI device
  3047. * @bar: BAR to query
  3048. *
  3049. * Get the possible sizes of a resizable BAR as bitmask defined in the spec
  3050. * (bit 0=1MB, bit 19=512GB). Returns 0 if BAR isn't resizable.
  3051. */
  3052. u32 pci_rebar_get_possible_sizes(struct pci_dev *pdev, int bar)
  3053. {
  3054. int pos;
  3055. u32 cap;
  3056. pos = pci_rebar_find_pos(pdev, bar);
  3057. if (pos < 0)
  3058. return 0;
  3059. pci_read_config_dword(pdev, pos + PCI_REBAR_CAP, &cap);
  3060. cap &= PCI_REBAR_CAP_SIZES;
  3061. /* Sapphire RX 5600 XT Pulse has an invalid cap dword for BAR 0 */
  3062. if (pdev->vendor == PCI_VENDOR_ID_ATI && pdev->device == 0x731f &&
  3063. bar == 0 && cap == 0x7000)
  3064. cap = 0x3f000;
  3065. return cap >> 4;
  3066. }
  3067. /**
  3068. * pci_rebar_get_current_size - get the current size of a BAR
  3069. * @pdev: PCI device
  3070. * @bar: BAR to set size to
  3071. *
  3072. * Read the size of a BAR from the resizable BAR config.
  3073. * Returns size if found or negative error code.
  3074. */
  3075. int pci_rebar_get_current_size(struct pci_dev *pdev, int bar)
  3076. {
  3077. int pos;
  3078. u32 ctrl;
  3079. pos = pci_rebar_find_pos(pdev, bar);
  3080. if (pos < 0)
  3081. return pos;
  3082. pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
  3083. return (ctrl & PCI_REBAR_CTRL_BAR_SIZE) >> PCI_REBAR_CTRL_BAR_SHIFT;
  3084. }
  3085. /**
  3086. * pci_rebar_set_size - set a new size for a BAR
  3087. * @pdev: PCI device
  3088. * @bar: BAR to set size to
  3089. * @size: new size as defined in the spec (0=1MB, 19=512GB)
  3090. *
  3091. * Set the new size of a BAR as defined in the spec.
  3092. * Returns zero if resizing was successful, error code otherwise.
  3093. */
  3094. int pci_rebar_set_size(struct pci_dev *pdev, int bar, int size)
  3095. {
  3096. int pos;
  3097. u32 ctrl;
  3098. pos = pci_rebar_find_pos(pdev, bar);
  3099. if (pos < 0)
  3100. return pos;
  3101. pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
  3102. ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE;
  3103. ctrl |= size << PCI_REBAR_CTRL_BAR_SHIFT;
  3104. pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl);
  3105. return 0;
  3106. }
  3107. /**
  3108. * pci_enable_atomic_ops_to_root - enable AtomicOp requests to root port
  3109. * @dev: the PCI device
  3110. * @cap_mask: mask of desired AtomicOp sizes, including one or more of:
  3111. * PCI_EXP_DEVCAP2_ATOMIC_COMP32
  3112. * PCI_EXP_DEVCAP2_ATOMIC_COMP64
  3113. * PCI_EXP_DEVCAP2_ATOMIC_COMP128
  3114. *
  3115. * Return 0 if all upstream bridges support AtomicOp routing, egress
  3116. * blocking is disabled on all upstream ports, and the root port supports
  3117. * the requested completion capabilities (32-bit, 64-bit and/or 128-bit
  3118. * AtomicOp completion), or negative otherwise.
  3119. */
  3120. int pci_enable_atomic_ops_to_root(struct pci_dev *dev, u32 cap_mask)
  3121. {
  3122. struct pci_bus *bus = dev->bus;
  3123. struct pci_dev *bridge;
  3124. u32 cap, ctl2;
  3125. if (!pci_is_pcie(dev))
  3126. return -EINVAL;
  3127. /*
  3128. * Per PCIe r4.0, sec 6.15, endpoints and root ports may be
  3129. * AtomicOp requesters. For now, we only support endpoints as
  3130. * requesters and root ports as completers. No endpoints as
  3131. * completers, and no peer-to-peer.
  3132. */
  3133. switch (pci_pcie_type(dev)) {
  3134. case PCI_EXP_TYPE_ENDPOINT:
  3135. case PCI_EXP_TYPE_LEG_END:
  3136. case PCI_EXP_TYPE_RC_END:
  3137. break;
  3138. default:
  3139. return -EINVAL;
  3140. }
  3141. while (bus->parent) {
  3142. bridge = bus->self;
  3143. pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &cap);
  3144. switch (pci_pcie_type(bridge)) {
  3145. /* Ensure switch ports support AtomicOp routing */
  3146. case PCI_EXP_TYPE_UPSTREAM:
  3147. case PCI_EXP_TYPE_DOWNSTREAM:
  3148. if (!(cap & PCI_EXP_DEVCAP2_ATOMIC_ROUTE))
  3149. return -EINVAL;
  3150. break;
  3151. /* Ensure root port supports all the sizes we care about */
  3152. case PCI_EXP_TYPE_ROOT_PORT:
  3153. if ((cap & cap_mask) != cap_mask)
  3154. return -EINVAL;
  3155. break;
  3156. }
  3157. /* Ensure upstream ports don't block AtomicOps on egress */
  3158. if (pci_pcie_type(bridge) == PCI_EXP_TYPE_UPSTREAM) {
  3159. pcie_capability_read_dword(bridge, PCI_EXP_DEVCTL2,
  3160. &ctl2);
  3161. if (ctl2 & PCI_EXP_DEVCTL2_ATOMIC_EGRESS_BLOCK)
  3162. return -EINVAL;
  3163. }
  3164. bus = bus->parent;
  3165. }
  3166. pcie_capability_set_word(dev, PCI_EXP_DEVCTL2,
  3167. PCI_EXP_DEVCTL2_ATOMIC_REQ);
  3168. return 0;
  3169. }
  3170. EXPORT_SYMBOL(pci_enable_atomic_ops_to_root);
  3171. /**
  3172. * pci_swizzle_interrupt_pin - swizzle INTx for device behind bridge
  3173. * @dev: the PCI device
  3174. * @pin: the INTx pin (1=INTA, 2=INTB, 3=INTC, 4=INTD)
  3175. *
  3176. * Perform INTx swizzling for a device behind one level of bridge. This is
  3177. * required by section 9.1 of the PCI-to-PCI bridge specification for devices
  3178. * behind bridges on add-in cards. For devices with ARI enabled, the slot
  3179. * number is always 0 (see the Implementation Note in section 2.2.8.1 of
  3180. * the PCI Express Base Specification, Revision 2.1)
  3181. */
  3182. u8 pci_swizzle_interrupt_pin(const struct pci_dev *dev, u8 pin)
  3183. {
  3184. int slot;
  3185. if (pci_ari_enabled(dev->bus))
  3186. slot = 0;
  3187. else
  3188. slot = PCI_SLOT(dev->devfn);
  3189. return (((pin - 1) + slot) % 4) + 1;
  3190. }
  3191. int pci_get_interrupt_pin(struct pci_dev *dev, struct pci_dev **bridge)
  3192. {
  3193. u8 pin;
  3194. pin = dev->pin;
  3195. if (!pin)
  3196. return -1;
  3197. while (!pci_is_root_bus(dev->bus)) {
  3198. pin = pci_swizzle_interrupt_pin(dev, pin);
  3199. dev = dev->bus->self;
  3200. }
  3201. *bridge = dev;
  3202. return pin;
  3203. }
  3204. /**
  3205. * pci_common_swizzle - swizzle INTx all the way to root bridge
  3206. * @dev: the PCI device
  3207. * @pinp: pointer to the INTx pin value (1=INTA, 2=INTB, 3=INTD, 4=INTD)
  3208. *
  3209. * Perform INTx swizzling for a device. This traverses through all PCI-to-PCI
  3210. * bridges all the way up to a PCI root bus.
  3211. */
  3212. u8 pci_common_swizzle(struct pci_dev *dev, u8 *pinp)
  3213. {
  3214. u8 pin = *pinp;
  3215. while (!pci_is_root_bus(dev->bus)) {
  3216. pin = pci_swizzle_interrupt_pin(dev, pin);
  3217. dev = dev->bus->self;
  3218. }
  3219. *pinp = pin;
  3220. return PCI_SLOT(dev->devfn);
  3221. }
  3222. EXPORT_SYMBOL_GPL(pci_common_swizzle);
  3223. /**
  3224. * pci_release_region - Release a PCI bar
  3225. * @pdev: PCI device whose resources were previously reserved by
  3226. * pci_request_region()
  3227. * @bar: BAR to release
  3228. *
  3229. * Releases the PCI I/O and memory resources previously reserved by a
  3230. * successful call to pci_request_region(). Call this function only
  3231. * after all use of the PCI regions has ceased.
  3232. */
  3233. void pci_release_region(struct pci_dev *pdev, int bar)
  3234. {
  3235. struct pci_devres *dr;
  3236. if (pci_resource_len(pdev, bar) == 0)
  3237. return;
  3238. if (pci_resource_flags(pdev, bar) & IORESOURCE_IO)
  3239. release_region(pci_resource_start(pdev, bar),
  3240. pci_resource_len(pdev, bar));
  3241. else if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM)
  3242. release_mem_region(pci_resource_start(pdev, bar),
  3243. pci_resource_len(pdev, bar));
  3244. dr = find_pci_dr(pdev);
  3245. if (dr)
  3246. dr->region_mask &= ~(1 << bar);
  3247. }
  3248. EXPORT_SYMBOL(pci_release_region);
  3249. /**
  3250. * __pci_request_region - Reserved PCI I/O and memory resource
  3251. * @pdev: PCI device whose resources are to be reserved
  3252. * @bar: BAR to be reserved
  3253. * @res_name: Name to be associated with resource.
  3254. * @exclusive: whether the region access is exclusive or not
  3255. *
  3256. * Mark the PCI region associated with PCI device @pdev BAR @bar as
  3257. * being reserved by owner @res_name. Do not access any
  3258. * address inside the PCI regions unless this call returns
  3259. * successfully.
  3260. *
  3261. * If @exclusive is set, then the region is marked so that userspace
  3262. * is explicitly not allowed to map the resource via /dev/mem or
  3263. * sysfs MMIO access.
  3264. *
  3265. * Returns 0 on success, or %EBUSY on error. A warning
  3266. * message is also printed on failure.
  3267. */
  3268. static int __pci_request_region(struct pci_dev *pdev, int bar,
  3269. const char *res_name, int exclusive)
  3270. {
  3271. struct pci_devres *dr;
  3272. if (pci_resource_len(pdev, bar) == 0)
  3273. return 0;
  3274. if (pci_resource_flags(pdev, bar) & IORESOURCE_IO) {
  3275. if (!request_region(pci_resource_start(pdev, bar),
  3276. pci_resource_len(pdev, bar), res_name))
  3277. goto err_out;
  3278. } else if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM) {
  3279. if (!__request_mem_region(pci_resource_start(pdev, bar),
  3280. pci_resource_len(pdev, bar), res_name,
  3281. exclusive))
  3282. goto err_out;
  3283. }
  3284. dr = find_pci_dr(pdev);
  3285. if (dr)
  3286. dr->region_mask |= 1 << bar;
  3287. return 0;
  3288. err_out:
  3289. pci_warn(pdev, "BAR %d: can't reserve %pR\n", bar,
  3290. &pdev->resource[bar]);
  3291. return -EBUSY;
  3292. }
  3293. /**
  3294. * pci_request_region - Reserve PCI I/O and memory resource
  3295. * @pdev: PCI device whose resources are to be reserved
  3296. * @bar: BAR to be reserved
  3297. * @res_name: Name to be associated with resource
  3298. *
  3299. * Mark the PCI region associated with PCI device @pdev BAR @bar as
  3300. * being reserved by owner @res_name. Do not access any
  3301. * address inside the PCI regions unless this call returns
  3302. * successfully.
  3303. *
  3304. * Returns 0 on success, or %EBUSY on error. A warning
  3305. * message is also printed on failure.
  3306. */
  3307. int pci_request_region(struct pci_dev *pdev, int bar, const char *res_name)
  3308. {
  3309. return __pci_request_region(pdev, bar, res_name, 0);
  3310. }
  3311. EXPORT_SYMBOL(pci_request_region);
  3312. /**
  3313. * pci_release_selected_regions - Release selected PCI I/O and memory resources
  3314. * @pdev: PCI device whose resources were previously reserved
  3315. * @bars: Bitmask of BARs to be released
  3316. *
  3317. * Release selected PCI I/O and memory resources previously reserved.
  3318. * Call this function only after all use of the PCI regions has ceased.
  3319. */
  3320. void pci_release_selected_regions(struct pci_dev *pdev, int bars)
  3321. {
  3322. int i;
  3323. for (i = 0; i < PCI_STD_NUM_BARS; i++)
  3324. if (bars & (1 << i))
  3325. pci_release_region(pdev, i);
  3326. }
  3327. EXPORT_SYMBOL(pci_release_selected_regions);
  3328. static int __pci_request_selected_regions(struct pci_dev *pdev, int bars,
  3329. const char *res_name, int excl)
  3330. {
  3331. int i;
  3332. for (i = 0; i < PCI_STD_NUM_BARS; i++)
  3333. if (bars & (1 << i))
  3334. if (__pci_request_region(pdev, i, res_name, excl))
  3335. goto err_out;
  3336. return 0;
  3337. err_out:
  3338. while (--i >= 0)
  3339. if (bars & (1 << i))
  3340. pci_release_region(pdev, i);
  3341. return -EBUSY;
  3342. }
  3343. /**
  3344. * pci_request_selected_regions - Reserve selected PCI I/O and memory resources
  3345. * @pdev: PCI device whose resources are to be reserved
  3346. * @bars: Bitmask of BARs to be requested
  3347. * @res_name: Name to be associated with resource
  3348. */
  3349. int pci_request_selected_regions(struct pci_dev *pdev, int bars,
  3350. const char *res_name)
  3351. {
  3352. return __pci_request_selected_regions(pdev, bars, res_name, 0);
  3353. }
  3354. EXPORT_SYMBOL(pci_request_selected_regions);
  3355. int pci_request_selected_regions_exclusive(struct pci_dev *pdev, int bars,
  3356. const char *res_name)
  3357. {
  3358. return __pci_request_selected_regions(pdev, bars, res_name,
  3359. IORESOURCE_EXCLUSIVE);
  3360. }
  3361. EXPORT_SYMBOL(pci_request_selected_regions_exclusive);
  3362. /**
  3363. * pci_release_regions - Release reserved PCI I/O and memory resources
  3364. * @pdev: PCI device whose resources were previously reserved by
  3365. * pci_request_regions()
  3366. *
  3367. * Releases all PCI I/O and memory resources previously reserved by a
  3368. * successful call to pci_request_regions(). Call this function only
  3369. * after all use of the PCI regions has ceased.
  3370. */
  3371. void pci_release_regions(struct pci_dev *pdev)
  3372. {
  3373. pci_release_selected_regions(pdev, (1 << PCI_STD_NUM_BARS) - 1);
  3374. }
  3375. EXPORT_SYMBOL(pci_release_regions);
  3376. /**
  3377. * pci_request_regions - Reserve PCI I/O and memory resources
  3378. * @pdev: PCI device whose resources are to be reserved
  3379. * @res_name: Name to be associated with resource.
  3380. *
  3381. * Mark all PCI regions associated with PCI device @pdev as
  3382. * being reserved by owner @res_name. Do not access any
  3383. * address inside the PCI regions unless this call returns
  3384. * successfully.
  3385. *
  3386. * Returns 0 on success, or %EBUSY on error. A warning
  3387. * message is also printed on failure.
  3388. */
  3389. int pci_request_regions(struct pci_dev *pdev, const char *res_name)
  3390. {
  3391. return pci_request_selected_regions(pdev,
  3392. ((1 << PCI_STD_NUM_BARS) - 1), res_name);
  3393. }
  3394. EXPORT_SYMBOL(pci_request_regions);
  3395. /**
  3396. * pci_request_regions_exclusive - Reserve PCI I/O and memory resources
  3397. * @pdev: PCI device whose resources are to be reserved
  3398. * @res_name: Name to be associated with resource.
  3399. *
  3400. * Mark all PCI regions associated with PCI device @pdev as being reserved
  3401. * by owner @res_name. Do not access any address inside the PCI regions
  3402. * unless this call returns successfully.
  3403. *
  3404. * pci_request_regions_exclusive() will mark the region so that /dev/mem
  3405. * and the sysfs MMIO access will not be allowed.
  3406. *
  3407. * Returns 0 on success, or %EBUSY on error. A warning message is also
  3408. * printed on failure.
  3409. */
  3410. int pci_request_regions_exclusive(struct pci_dev *pdev, const char *res_name)
  3411. {
  3412. return pci_request_selected_regions_exclusive(pdev,
  3413. ((1 << PCI_STD_NUM_BARS) - 1), res_name);
  3414. }
  3415. EXPORT_SYMBOL(pci_request_regions_exclusive);
  3416. /*
  3417. * Record the PCI IO range (expressed as CPU physical address + size).
  3418. * Return a negative value if an error has occurred, zero otherwise
  3419. */
  3420. int pci_register_io_range(struct fwnode_handle *fwnode, phys_addr_t addr,
  3421. resource_size_t size)
  3422. {
  3423. int ret = 0;
  3424. #ifdef PCI_IOBASE
  3425. struct logic_pio_hwaddr *range;
  3426. if (!size || addr + size < addr)
  3427. return -EINVAL;
  3428. range = kzalloc(sizeof(*range), GFP_ATOMIC);
  3429. if (!range)
  3430. return -ENOMEM;
  3431. range->fwnode = fwnode;
  3432. range->size = size;
  3433. range->hw_start = addr;
  3434. range->flags = LOGIC_PIO_CPU_MMIO;
  3435. ret = logic_pio_register_range(range);
  3436. if (ret)
  3437. kfree(range);
  3438. /* Ignore duplicates due to deferred probing */
  3439. if (ret == -EEXIST)
  3440. ret = 0;
  3441. #endif
  3442. return ret;
  3443. }
  3444. phys_addr_t pci_pio_to_address(unsigned long pio)
  3445. {
  3446. phys_addr_t address = (phys_addr_t)OF_BAD_ADDR;
  3447. #ifdef PCI_IOBASE
  3448. if (pio >= MMIO_UPPER_LIMIT)
  3449. return address;
  3450. address = logic_pio_to_hwaddr(pio);
  3451. #endif
  3452. return address;
  3453. }
  3454. EXPORT_SYMBOL_GPL(pci_pio_to_address);
  3455. unsigned long __weak pci_address_to_pio(phys_addr_t address)
  3456. {
  3457. #ifdef PCI_IOBASE
  3458. return logic_pio_trans_cpuaddr(address);
  3459. #else
  3460. if (address > IO_SPACE_LIMIT)
  3461. return (unsigned long)-1;
  3462. return (unsigned long) address;
  3463. #endif
  3464. }
  3465. /**
  3466. * pci_remap_iospace - Remap the memory mapped I/O space
  3467. * @res: Resource describing the I/O space
  3468. * @phys_addr: physical address of range to be mapped
  3469. *
  3470. * Remap the memory mapped I/O space described by the @res and the CPU
  3471. * physical address @phys_addr into virtual address space. Only
  3472. * architectures that have memory mapped IO functions defined (and the
  3473. * PCI_IOBASE value defined) should call this function.
  3474. */
  3475. int pci_remap_iospace(const struct resource *res, phys_addr_t phys_addr)
  3476. {
  3477. #if defined(PCI_IOBASE) && defined(CONFIG_MMU)
  3478. unsigned long vaddr = (unsigned long)PCI_IOBASE + res->start;
  3479. if (!(res->flags & IORESOURCE_IO))
  3480. return -EINVAL;
  3481. if (res->end > IO_SPACE_LIMIT)
  3482. return -EINVAL;
  3483. return ioremap_page_range(vaddr, vaddr + resource_size(res), phys_addr,
  3484. pgprot_device(PAGE_KERNEL));
  3485. #else
  3486. /*
  3487. * This architecture does not have memory mapped I/O space,
  3488. * so this function should never be called
  3489. */
  3490. WARN_ONCE(1, "This architecture does not support memory mapped I/O\n");
  3491. return -ENODEV;
  3492. #endif
  3493. }
  3494. EXPORT_SYMBOL(pci_remap_iospace);
  3495. /**
  3496. * pci_unmap_iospace - Unmap the memory mapped I/O space
  3497. * @res: resource to be unmapped
  3498. *
  3499. * Unmap the CPU virtual address @res from virtual address space. Only
  3500. * architectures that have memory mapped IO functions defined (and the
  3501. * PCI_IOBASE value defined) should call this function.
  3502. */
  3503. void pci_unmap_iospace(struct resource *res)
  3504. {
  3505. #if defined(PCI_IOBASE) && defined(CONFIG_MMU)
  3506. unsigned long vaddr = (unsigned long)PCI_IOBASE + res->start;
  3507. unmap_kernel_range(vaddr, resource_size(res));
  3508. #endif
  3509. }
  3510. EXPORT_SYMBOL(pci_unmap_iospace);
  3511. static void devm_pci_unmap_iospace(struct device *dev, void *ptr)
  3512. {
  3513. struct resource **res = ptr;
  3514. pci_unmap_iospace(*res);
  3515. }
  3516. /**
  3517. * devm_pci_remap_iospace - Managed pci_remap_iospace()
  3518. * @dev: Generic device to remap IO address for
  3519. * @res: Resource describing the I/O space
  3520. * @phys_addr: physical address of range to be mapped
  3521. *
  3522. * Managed pci_remap_iospace(). Map is automatically unmapped on driver
  3523. * detach.
  3524. */
  3525. int devm_pci_remap_iospace(struct device *dev, const struct resource *res,
  3526. phys_addr_t phys_addr)
  3527. {
  3528. const struct resource **ptr;
  3529. int error;
  3530. ptr = devres_alloc(devm_pci_unmap_iospace, sizeof(*ptr), GFP_KERNEL);
  3531. if (!ptr)
  3532. return -ENOMEM;
  3533. error = pci_remap_iospace(res, phys_addr);
  3534. if (error) {
  3535. devres_free(ptr);
  3536. } else {
  3537. *ptr = res;
  3538. devres_add(dev, ptr);
  3539. }
  3540. return error;
  3541. }
  3542. EXPORT_SYMBOL(devm_pci_remap_iospace);
  3543. /**
  3544. * devm_pci_remap_cfgspace - Managed pci_remap_cfgspace()
  3545. * @dev: Generic device to remap IO address for
  3546. * @offset: Resource address to map
  3547. * @size: Size of map
  3548. *
  3549. * Managed pci_remap_cfgspace(). Map is automatically unmapped on driver
  3550. * detach.
  3551. */
  3552. void __iomem *devm_pci_remap_cfgspace(struct device *dev,
  3553. resource_size_t offset,
  3554. resource_size_t size)
  3555. {
  3556. void __iomem **ptr, *addr;
  3557. ptr = devres_alloc(devm_ioremap_release, sizeof(*ptr), GFP_KERNEL);
  3558. if (!ptr)
  3559. return NULL;
  3560. addr = pci_remap_cfgspace(offset, size);
  3561. if (addr) {
  3562. *ptr = addr;
  3563. devres_add(dev, ptr);
  3564. } else
  3565. devres_free(ptr);
  3566. return addr;
  3567. }
  3568. EXPORT_SYMBOL(devm_pci_remap_cfgspace);
  3569. /**
  3570. * devm_pci_remap_cfg_resource - check, request region and ioremap cfg resource
  3571. * @dev: generic device to handle the resource for
  3572. * @res: configuration space resource to be handled
  3573. *
  3574. * Checks that a resource is a valid memory region, requests the memory
  3575. * region and ioremaps with pci_remap_cfgspace() API that ensures the
  3576. * proper PCI configuration space memory attributes are guaranteed.
  3577. *
  3578. * All operations are managed and will be undone on driver detach.
  3579. *
  3580. * Returns a pointer to the remapped memory or an ERR_PTR() encoded error code
  3581. * on failure. Usage example::
  3582. *
  3583. * res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
  3584. * base = devm_pci_remap_cfg_resource(&pdev->dev, res);
  3585. * if (IS_ERR(base))
  3586. * return PTR_ERR(base);
  3587. */
  3588. void __iomem *devm_pci_remap_cfg_resource(struct device *dev,
  3589. struct resource *res)
  3590. {
  3591. resource_size_t size;
  3592. const char *name;
  3593. void __iomem *dest_ptr;
  3594. BUG_ON(!dev);
  3595. if (!res || resource_type(res) != IORESOURCE_MEM) {
  3596. dev_err(dev, "invalid resource\n");
  3597. return IOMEM_ERR_PTR(-EINVAL);
  3598. }
  3599. size = resource_size(res);
  3600. name = res->name ?: dev_name(dev);
  3601. if (!devm_request_mem_region(dev, res->start, size, name)) {
  3602. dev_err(dev, "can't request region for resource %pR\n", res);
  3603. return IOMEM_ERR_PTR(-EBUSY);
  3604. }
  3605. dest_ptr = devm_pci_remap_cfgspace(dev, res->start, size);
  3606. if (!dest_ptr) {
  3607. dev_err(dev, "ioremap failed for resource %pR\n", res);
  3608. devm_release_mem_region(dev, res->start, size);
  3609. dest_ptr = IOMEM_ERR_PTR(-ENOMEM);
  3610. }
  3611. return dest_ptr;
  3612. }
  3613. EXPORT_SYMBOL(devm_pci_remap_cfg_resource);
  3614. static void __pci_set_master(struct pci_dev *dev, bool enable)
  3615. {
  3616. u16 old_cmd, cmd;
  3617. pci_read_config_word(dev, PCI_COMMAND, &old_cmd);
  3618. if (enable)
  3619. cmd = old_cmd | PCI_COMMAND_MASTER;
  3620. else
  3621. cmd = old_cmd & ~PCI_COMMAND_MASTER;
  3622. if (cmd != old_cmd) {
  3623. pci_dbg(dev, "%s bus mastering\n",
  3624. enable ? "enabling" : "disabling");
  3625. pci_write_config_word(dev, PCI_COMMAND, cmd);
  3626. }
  3627. dev->is_busmaster = enable;
  3628. }
  3629. /**
  3630. * pcibios_setup - process "pci=" kernel boot arguments
  3631. * @str: string used to pass in "pci=" kernel boot arguments
  3632. *
  3633. * Process kernel boot arguments. This is the default implementation.
  3634. * Architecture specific implementations can override this as necessary.
  3635. */
  3636. char * __weak __init pcibios_setup(char *str)
  3637. {
  3638. return str;
  3639. }
  3640. /**
  3641. * pcibios_set_master - enable PCI bus-mastering for device dev
  3642. * @dev: the PCI device to enable
  3643. *
  3644. * Enables PCI bus-mastering for the device. This is the default
  3645. * implementation. Architecture specific implementations can override
  3646. * this if necessary.
  3647. */
  3648. void __weak pcibios_set_master(struct pci_dev *dev)
  3649. {
  3650. u8 lat;
  3651. /* The latency timer doesn't apply to PCIe (either Type 0 or Type 1) */
  3652. if (pci_is_pcie(dev))
  3653. return;
  3654. pci_read_config_byte(dev, PCI_LATENCY_TIMER, &lat);
  3655. if (lat < 16)
  3656. lat = (64 <= pcibios_max_latency) ? 64 : pcibios_max_latency;
  3657. else if (lat > pcibios_max_latency)
  3658. lat = pcibios_max_latency;
  3659. else
  3660. return;
  3661. pci_write_config_byte(dev, PCI_LATENCY_TIMER, lat);
  3662. }
  3663. /**
  3664. * pci_set_master - enables bus-mastering for device dev
  3665. * @dev: the PCI device to enable
  3666. *
  3667. * Enables bus-mastering on the device and calls pcibios_set_master()
  3668. * to do the needed arch specific settings.
  3669. */
  3670. void pci_set_master(struct pci_dev *dev)
  3671. {
  3672. __pci_set_master(dev, true);
  3673. pcibios_set_master(dev);
  3674. }
  3675. EXPORT_SYMBOL(pci_set_master);
  3676. /**
  3677. * pci_clear_master - disables bus-mastering for device dev
  3678. * @dev: the PCI device to disable
  3679. */
  3680. void pci_clear_master(struct pci_dev *dev)
  3681. {
  3682. __pci_set_master(dev, false);
  3683. }
  3684. EXPORT_SYMBOL(pci_clear_master);
  3685. /**
  3686. * pci_set_cacheline_size - ensure the CACHE_LINE_SIZE register is programmed
  3687. * @dev: the PCI device for which MWI is to be enabled
  3688. *
  3689. * Helper function for pci_set_mwi.
  3690. * Originally copied from drivers/net/acenic.c.
  3691. * Copyright 1998-2001 by Jes Sorensen, <jes@trained-monkey.org>.
  3692. *
  3693. * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
  3694. */
  3695. int pci_set_cacheline_size(struct pci_dev *dev)
  3696. {
  3697. u8 cacheline_size;
  3698. if (!pci_cache_line_size)
  3699. return -EINVAL;
  3700. /* Validate current setting: the PCI_CACHE_LINE_SIZE must be
  3701. equal to or multiple of the right value. */
  3702. pci_read_config_byte(dev, PCI_CACHE_LINE_SIZE, &cacheline_size);
  3703. if (cacheline_size >= pci_cache_line_size &&
  3704. (cacheline_size % pci_cache_line_size) == 0)
  3705. return 0;
  3706. /* Write the correct value. */
  3707. pci_write_config_byte(dev, PCI_CACHE_LINE_SIZE, pci_cache_line_size);
  3708. /* Read it back. */
  3709. pci_read_config_byte(dev, PCI_CACHE_LINE_SIZE, &cacheline_size);
  3710. if (cacheline_size == pci_cache_line_size)
  3711. return 0;
  3712. pci_info(dev, "cache line size of %d is not supported\n",
  3713. pci_cache_line_size << 2);
  3714. return -EINVAL;
  3715. }
  3716. EXPORT_SYMBOL_GPL(pci_set_cacheline_size);
  3717. /**
  3718. * pci_set_mwi - enables memory-write-invalidate PCI transaction
  3719. * @dev: the PCI device for which MWI is enabled
  3720. *
  3721. * Enables the Memory-Write-Invalidate transaction in %PCI_COMMAND.
  3722. *
  3723. * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
  3724. */
  3725. int pci_set_mwi(struct pci_dev *dev)
  3726. {
  3727. #ifdef PCI_DISABLE_MWI
  3728. return 0;
  3729. #else
  3730. int rc;
  3731. u16 cmd;
  3732. rc = pci_set_cacheline_size(dev);
  3733. if (rc)
  3734. return rc;
  3735. pci_read_config_word(dev, PCI_COMMAND, &cmd);
  3736. if (!(cmd & PCI_COMMAND_INVALIDATE)) {
  3737. pci_dbg(dev, "enabling Mem-Wr-Inval\n");
  3738. cmd |= PCI_COMMAND_INVALIDATE;
  3739. pci_write_config_word(dev, PCI_COMMAND, cmd);
  3740. }
  3741. return 0;
  3742. #endif
  3743. }
  3744. EXPORT_SYMBOL(pci_set_mwi);
  3745. /**
  3746. * pcim_set_mwi - a device-managed pci_set_mwi()
  3747. * @dev: the PCI device for which MWI is enabled
  3748. *
  3749. * Managed pci_set_mwi().
  3750. *
  3751. * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
  3752. */
  3753. int pcim_set_mwi(struct pci_dev *dev)
  3754. {
  3755. struct pci_devres *dr;
  3756. dr = find_pci_dr(dev);
  3757. if (!dr)
  3758. return -ENOMEM;
  3759. dr->mwi = 1;
  3760. return pci_set_mwi(dev);
  3761. }
  3762. EXPORT_SYMBOL(pcim_set_mwi);
  3763. /**
  3764. * pci_try_set_mwi - enables memory-write-invalidate PCI transaction
  3765. * @dev: the PCI device for which MWI is enabled
  3766. *
  3767. * Enables the Memory-Write-Invalidate transaction in %PCI_COMMAND.
  3768. * Callers are not required to check the return value.
  3769. *
  3770. * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
  3771. */
  3772. int pci_try_set_mwi(struct pci_dev *dev)
  3773. {
  3774. #ifdef PCI_DISABLE_MWI
  3775. return 0;
  3776. #else
  3777. return pci_set_mwi(dev);
  3778. #endif
  3779. }
  3780. EXPORT_SYMBOL(pci_try_set_mwi);
  3781. /**
  3782. * pci_clear_mwi - disables Memory-Write-Invalidate for device dev
  3783. * @dev: the PCI device to disable
  3784. *
  3785. * Disables PCI Memory-Write-Invalidate transaction on the device
  3786. */
  3787. void pci_clear_mwi(struct pci_dev *dev)
  3788. {
  3789. #ifndef PCI_DISABLE_MWI
  3790. u16 cmd;
  3791. pci_read_config_word(dev, PCI_COMMAND, &cmd);
  3792. if (cmd & PCI_COMMAND_INVALIDATE) {
  3793. cmd &= ~PCI_COMMAND_INVALIDATE;
  3794. pci_write_config_word(dev, PCI_COMMAND, cmd);
  3795. }
  3796. #endif
  3797. }
  3798. EXPORT_SYMBOL(pci_clear_mwi);
  3799. /**
  3800. * pci_intx - enables/disables PCI INTx for device dev
  3801. * @pdev: the PCI device to operate on
  3802. * @enable: boolean: whether to enable or disable PCI INTx
  3803. *
  3804. * Enables/disables PCI INTx for device @pdev
  3805. */
  3806. void pci_intx(struct pci_dev *pdev, int enable)
  3807. {
  3808. u16 pci_command, new;
  3809. pci_read_config_word(pdev, PCI_COMMAND, &pci_command);
  3810. if (enable)
  3811. new = pci_command & ~PCI_COMMAND_INTX_DISABLE;
  3812. else
  3813. new = pci_command | PCI_COMMAND_INTX_DISABLE;
  3814. if (new != pci_command) {
  3815. struct pci_devres *dr;
  3816. pci_write_config_word(pdev, PCI_COMMAND, new);
  3817. dr = find_pci_dr(pdev);
  3818. if (dr && !dr->restore_intx) {
  3819. dr->restore_intx = 1;
  3820. dr->orig_intx = !enable;
  3821. }
  3822. }
  3823. }
  3824. EXPORT_SYMBOL_GPL(pci_intx);
  3825. static bool pci_check_and_set_intx_mask(struct pci_dev *dev, bool mask)
  3826. {
  3827. struct pci_bus *bus = dev->bus;
  3828. bool mask_updated = true;
  3829. u32 cmd_status_dword;
  3830. u16 origcmd, newcmd;
  3831. unsigned long flags;
  3832. bool irq_pending;
  3833. /*
  3834. * We do a single dword read to retrieve both command and status.
  3835. * Document assumptions that make this possible.
  3836. */
  3837. BUILD_BUG_ON(PCI_COMMAND % 4);
  3838. BUILD_BUG_ON(PCI_COMMAND + 2 != PCI_STATUS);
  3839. raw_spin_lock_irqsave(&pci_lock, flags);
  3840. bus->ops->read(bus, dev->devfn, PCI_COMMAND, 4, &cmd_status_dword);
  3841. irq_pending = (cmd_status_dword >> 16) & PCI_STATUS_INTERRUPT;
  3842. /*
  3843. * Check interrupt status register to see whether our device
  3844. * triggered the interrupt (when masking) or the next IRQ is
  3845. * already pending (when unmasking).
  3846. */
  3847. if (mask != irq_pending) {
  3848. mask_updated = false;
  3849. goto done;
  3850. }
  3851. origcmd = cmd_status_dword;
  3852. newcmd = origcmd & ~PCI_COMMAND_INTX_DISABLE;
  3853. if (mask)
  3854. newcmd |= PCI_COMMAND_INTX_DISABLE;
  3855. if (newcmd != origcmd)
  3856. bus->ops->write(bus, dev->devfn, PCI_COMMAND, 2, newcmd);
  3857. done:
  3858. raw_spin_unlock_irqrestore(&pci_lock, flags);
  3859. return mask_updated;
  3860. }
  3861. /**
  3862. * pci_check_and_mask_intx - mask INTx on pending interrupt
  3863. * @dev: the PCI device to operate on
  3864. *
  3865. * Check if the device dev has its INTx line asserted, mask it and return
  3866. * true in that case. False is returned if no interrupt was pending.
  3867. */
  3868. bool pci_check_and_mask_intx(struct pci_dev *dev)
  3869. {
  3870. return pci_check_and_set_intx_mask(dev, true);
  3871. }
  3872. EXPORT_SYMBOL_GPL(pci_check_and_mask_intx);
  3873. /**
  3874. * pci_check_and_unmask_intx - unmask INTx if no interrupt is pending
  3875. * @dev: the PCI device to operate on
  3876. *
  3877. * Check if the device dev has its INTx line asserted, unmask it if not and
  3878. * return true. False is returned and the mask remains active if there was
  3879. * still an interrupt pending.
  3880. */
  3881. bool pci_check_and_unmask_intx(struct pci_dev *dev)
  3882. {
  3883. return pci_check_and_set_intx_mask(dev, false);
  3884. }
  3885. EXPORT_SYMBOL_GPL(pci_check_and_unmask_intx);
  3886. /**
  3887. * pci_wait_for_pending_transaction - wait for pending transaction
  3888. * @dev: the PCI device to operate on
  3889. *
  3890. * Return 0 if transaction is pending 1 otherwise.
  3891. */
  3892. int pci_wait_for_pending_transaction(struct pci_dev *dev)
  3893. {
  3894. if (!pci_is_pcie(dev))
  3895. return 1;
  3896. return pci_wait_for_pending(dev, pci_pcie_cap(dev) + PCI_EXP_DEVSTA,
  3897. PCI_EXP_DEVSTA_TRPND);
  3898. }
  3899. EXPORT_SYMBOL(pci_wait_for_pending_transaction);
  3900. /**
  3901. * pcie_has_flr - check if a device supports function level resets
  3902. * @dev: device to check
  3903. *
  3904. * Returns true if the device advertises support for PCIe function level
  3905. * resets.
  3906. */
  3907. bool pcie_has_flr(struct pci_dev *dev)
  3908. {
  3909. u32 cap;
  3910. if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET)
  3911. return false;
  3912. pcie_capability_read_dword(dev, PCI_EXP_DEVCAP, &cap);
  3913. return cap & PCI_EXP_DEVCAP_FLR;
  3914. }
  3915. EXPORT_SYMBOL_GPL(pcie_has_flr);
  3916. /**
  3917. * pcie_flr - initiate a PCIe function level reset
  3918. * @dev: device to reset
  3919. *
  3920. * Initiate a function level reset on @dev. The caller should ensure the
  3921. * device supports FLR before calling this function, e.g. by using the
  3922. * pcie_has_flr() helper.
  3923. */
  3924. int pcie_flr(struct pci_dev *dev)
  3925. {
  3926. if (!pci_wait_for_pending_transaction(dev))
  3927. pci_err(dev, "timed out waiting for pending transaction; performing function level reset anyway\n");
  3928. pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
  3929. if (dev->imm_ready)
  3930. return 0;
  3931. /*
  3932. * Per PCIe r4.0, sec 6.6.2, a device must complete an FLR within
  3933. * 100ms, but may silently discard requests while the FLR is in
  3934. * progress. Wait 100ms before trying to access the device.
  3935. */
  3936. msleep(100);
  3937. return pci_dev_wait(dev, "FLR", PCIE_RESET_READY_POLL_MS);
  3938. }
  3939. EXPORT_SYMBOL_GPL(pcie_flr);
  3940. static int pci_af_flr(struct pci_dev *dev, int probe)
  3941. {
  3942. int pos;
  3943. u8 cap;
  3944. pos = pci_find_capability(dev, PCI_CAP_ID_AF);
  3945. if (!pos)
  3946. return -ENOTTY;
  3947. if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET)
  3948. return -ENOTTY;
  3949. pci_read_config_byte(dev, pos + PCI_AF_CAP, &cap);
  3950. if (!(cap & PCI_AF_CAP_TP) || !(cap & PCI_AF_CAP_FLR))
  3951. return -ENOTTY;
  3952. if (probe)
  3953. return 0;
  3954. /*
  3955. * Wait for Transaction Pending bit to clear. A word-aligned test
  3956. * is used, so we use the control offset rather than status and shift
  3957. * the test bit to match.
  3958. */
  3959. if (!pci_wait_for_pending(dev, pos + PCI_AF_CTRL,
  3960. PCI_AF_STATUS_TP << 8))
  3961. pci_err(dev, "timed out waiting for pending transaction; performing AF function level reset anyway\n");
  3962. pci_write_config_byte(dev, pos + PCI_AF_CTRL, PCI_AF_CTRL_FLR);
  3963. if (dev->imm_ready)
  3964. return 0;
  3965. /*
  3966. * Per Advanced Capabilities for Conventional PCI ECN, 13 April 2006,
  3967. * updated 27 July 2006; a device must complete an FLR within
  3968. * 100ms, but may silently discard requests while the FLR is in
  3969. * progress. Wait 100ms before trying to access the device.
  3970. */
  3971. msleep(100);
  3972. return pci_dev_wait(dev, "AF_FLR", PCIE_RESET_READY_POLL_MS);
  3973. }
  3974. /**
  3975. * pci_pm_reset - Put device into PCI_D3 and back into PCI_D0.
  3976. * @dev: Device to reset.
  3977. * @probe: If set, only check if the device can be reset this way.
  3978. *
  3979. * If @dev supports native PCI PM and its PCI_PM_CTRL_NO_SOFT_RESET flag is
  3980. * unset, it will be reinitialized internally when going from PCI_D3hot to
  3981. * PCI_D0. If that's the case and the device is not in a low-power state
  3982. * already, force it into PCI_D3hot and back to PCI_D0, causing it to be reset.
  3983. *
  3984. * NOTE: This causes the caller to sleep for twice the device power transition
  3985. * cooldown period, which for the D0->D3hot and D3hot->D0 transitions is 10 ms
  3986. * by default (i.e. unless the @dev's d3hot_delay field has a different value).
  3987. * Moreover, only devices in D0 can be reset by this function.
  3988. */
  3989. static int pci_pm_reset(struct pci_dev *dev, int probe)
  3990. {
  3991. u16 csr;
  3992. if (!dev->pm_cap || dev->dev_flags & PCI_DEV_FLAGS_NO_PM_RESET)
  3993. return -ENOTTY;
  3994. pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &csr);
  3995. if (csr & PCI_PM_CTRL_NO_SOFT_RESET)
  3996. return -ENOTTY;
  3997. if (probe)
  3998. return 0;
  3999. if (dev->current_state != PCI_D0)
  4000. return -EINVAL;
  4001. csr &= ~PCI_PM_CTRL_STATE_MASK;
  4002. csr |= PCI_D3hot;
  4003. pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, csr);
  4004. pci_dev_d3_sleep(dev);
  4005. csr &= ~PCI_PM_CTRL_STATE_MASK;
  4006. csr |= PCI_D0;
  4007. pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, csr);
  4008. pci_dev_d3_sleep(dev);
  4009. return pci_dev_wait(dev, "PM D3hot->D0", PCIE_RESET_READY_POLL_MS);
  4010. }
  4011. /**
  4012. * pcie_wait_for_link_delay - Wait until link is active or inactive
  4013. * @pdev: Bridge device
  4014. * @active: waiting for active or inactive?
  4015. * @delay: Delay to wait after link has become active (in ms)
  4016. *
  4017. * Use this to wait till link becomes active or inactive.
  4018. */
  4019. static bool pcie_wait_for_link_delay(struct pci_dev *pdev, bool active,
  4020. int delay)
  4021. {
  4022. int timeout = 1000;
  4023. bool ret;
  4024. u16 lnk_status;
  4025. /*
  4026. * Some controllers might not implement link active reporting. In this
  4027. * case, we wait for 1000 ms + any delay requested by the caller.
  4028. */
  4029. if (!pdev->link_active_reporting) {
  4030. msleep(timeout + delay);
  4031. return true;
  4032. }
  4033. /*
  4034. * PCIe r4.0 sec 6.6.1, a component must enter LTSSM Detect within 20ms,
  4035. * after which we should expect an link active if the reset was
  4036. * successful. If so, software must wait a minimum 100ms before sending
  4037. * configuration requests to devices downstream this port.
  4038. *
  4039. * If the link fails to activate, either the device was physically
  4040. * removed or the link is permanently failed.
  4041. */
  4042. if (active)
  4043. msleep(20);
  4044. for (;;) {
  4045. pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnk_status);
  4046. ret = !!(lnk_status & PCI_EXP_LNKSTA_DLLLA);
  4047. if (ret == active)
  4048. break;
  4049. if (timeout <= 0)
  4050. break;
  4051. msleep(10);
  4052. timeout -= 10;
  4053. }
  4054. if (active && ret)
  4055. msleep(delay);
  4056. return ret == active;
  4057. }
  4058. /**
  4059. * pcie_wait_for_link - Wait until link is active or inactive
  4060. * @pdev: Bridge device
  4061. * @active: waiting for active or inactive?
  4062. *
  4063. * Use this to wait till link becomes active or inactive.
  4064. */
  4065. bool pcie_wait_for_link(struct pci_dev *pdev, bool active)
  4066. {
  4067. return pcie_wait_for_link_delay(pdev, active, 100);
  4068. }
  4069. /*
  4070. * Find maximum D3cold delay required by all the devices on the bus. The
  4071. * spec says 100 ms, but firmware can lower it and we allow drivers to
  4072. * increase it as well.
  4073. *
  4074. * Called with @pci_bus_sem locked for reading.
  4075. */
  4076. static int pci_bus_max_d3cold_delay(const struct pci_bus *bus)
  4077. {
  4078. const struct pci_dev *pdev;
  4079. int min_delay = 100;
  4080. int max_delay = 0;
  4081. list_for_each_entry(pdev, &bus->devices, bus_list) {
  4082. if (pdev->d3cold_delay < min_delay)
  4083. min_delay = pdev->d3cold_delay;
  4084. if (pdev->d3cold_delay > max_delay)
  4085. max_delay = pdev->d3cold_delay;
  4086. }
  4087. return max(min_delay, max_delay);
  4088. }
  4089. /**
  4090. * pci_bridge_wait_for_secondary_bus - Wait for secondary bus to be accessible
  4091. * @dev: PCI bridge
  4092. *
  4093. * Handle necessary delays before access to the devices on the secondary
  4094. * side of the bridge are permitted after D3cold to D0 transition.
  4095. *
  4096. * For PCIe this means the delays in PCIe 5.0 section 6.6.1. For
  4097. * conventional PCI it means Tpvrh + Trhfa specified in PCI 3.0 section
  4098. * 4.3.2.
  4099. */
  4100. void pci_bridge_wait_for_secondary_bus(struct pci_dev *dev)
  4101. {
  4102. struct pci_dev *child;
  4103. int delay;
  4104. if (pci_dev_is_disconnected(dev))
  4105. return;
  4106. if (!pci_is_bridge(dev) || !dev->bridge_d3)
  4107. return;
  4108. down_read(&pci_bus_sem);
  4109. /*
  4110. * We only deal with devices that are present currently on the bus.
  4111. * For any hot-added devices the access delay is handled in pciehp
  4112. * board_added(). In case of ACPI hotplug the firmware is expected
  4113. * to configure the devices before OS is notified.
  4114. */
  4115. if (!dev->subordinate || list_empty(&dev->subordinate->devices)) {
  4116. up_read(&pci_bus_sem);
  4117. return;
  4118. }
  4119. /* Take d3cold_delay requirements into account */
  4120. delay = pci_bus_max_d3cold_delay(dev->subordinate);
  4121. if (!delay) {
  4122. up_read(&pci_bus_sem);
  4123. return;
  4124. }
  4125. child = list_first_entry(&dev->subordinate->devices, struct pci_dev,
  4126. bus_list);
  4127. up_read(&pci_bus_sem);
  4128. /*
  4129. * Conventional PCI and PCI-X we need to wait Tpvrh + Trhfa before
  4130. * accessing the device after reset (that is 1000 ms + 100 ms). In
  4131. * practice this should not be needed because we don't do power
  4132. * management for them (see pci_bridge_d3_possible()).
  4133. */
  4134. if (!pci_is_pcie(dev)) {
  4135. pci_dbg(dev, "waiting %d ms for secondary bus\n", 1000 + delay);
  4136. msleep(1000 + delay);
  4137. return;
  4138. }
  4139. /*
  4140. * For PCIe downstream and root ports that do not support speeds
  4141. * greater than 5 GT/s need to wait minimum 100 ms. For higher
  4142. * speeds (gen3) we need to wait first for the data link layer to
  4143. * become active.
  4144. *
  4145. * However, 100 ms is the minimum and the PCIe spec says the
  4146. * software must allow at least 1s before it can determine that the
  4147. * device that did not respond is a broken device. There is
  4148. * evidence that 100 ms is not always enough, for example certain
  4149. * Titan Ridge xHCI controller does not always respond to
  4150. * configuration requests if we only wait for 100 ms (see
  4151. * https://bugzilla.kernel.org/show_bug.cgi?id=203885).
  4152. *
  4153. * Therefore we wait for 100 ms and check for the device presence.
  4154. * If it is still not present give it an additional 100 ms.
  4155. */
  4156. if (!pcie_downstream_port(dev))
  4157. return;
  4158. if (pcie_get_speed_cap(dev) <= PCIE_SPEED_5_0GT) {
  4159. pci_dbg(dev, "waiting %d ms for downstream link\n", delay);
  4160. msleep(delay);
  4161. } else {
  4162. pci_dbg(dev, "waiting %d ms for downstream link, after activation\n",
  4163. delay);
  4164. if (!pcie_wait_for_link_delay(dev, true, delay)) {
  4165. /* Did not train, no need to wait any further */
  4166. pci_info(dev, "Data Link Layer Link Active not set in 1000 msec\n");
  4167. return;
  4168. }
  4169. }
  4170. if (!pci_device_is_present(child)) {
  4171. pci_dbg(child, "waiting additional %d ms to become accessible\n", delay);
  4172. msleep(delay);
  4173. }
  4174. }
  4175. void pci_reset_secondary_bus(struct pci_dev *dev)
  4176. {
  4177. u16 ctrl;
  4178. pci_read_config_word(dev, PCI_BRIDGE_CONTROL, &ctrl);
  4179. ctrl |= PCI_BRIDGE_CTL_BUS_RESET;
  4180. pci_write_config_word(dev, PCI_BRIDGE_CONTROL, ctrl);
  4181. /*
  4182. * PCI spec v3.0 7.6.4.2 requires minimum Trst of 1ms. Double
  4183. * this to 2ms to ensure that we meet the minimum requirement.
  4184. */
  4185. msleep(2);
  4186. ctrl &= ~PCI_BRIDGE_CTL_BUS_RESET;
  4187. pci_write_config_word(dev, PCI_BRIDGE_CONTROL, ctrl);
  4188. /*
  4189. * Trhfa for conventional PCI is 2^25 clock cycles.
  4190. * Assuming a minimum 33MHz clock this results in a 1s
  4191. * delay before we can consider subordinate devices to
  4192. * be re-initialized. PCIe has some ways to shorten this,
  4193. * but we don't make use of them yet.
  4194. */
  4195. ssleep(1);
  4196. }
  4197. void __weak pcibios_reset_secondary_bus(struct pci_dev *dev)
  4198. {
  4199. pci_reset_secondary_bus(dev);
  4200. }
  4201. /**
  4202. * pci_bridge_secondary_bus_reset - Reset the secondary bus on a PCI bridge.
  4203. * @dev: Bridge device
  4204. *
  4205. * Use the bridge control register to assert reset on the secondary bus.
  4206. * Devices on the secondary bus are left in power-on state.
  4207. */
  4208. int pci_bridge_secondary_bus_reset(struct pci_dev *dev)
  4209. {
  4210. pcibios_reset_secondary_bus(dev);
  4211. return pci_dev_wait(dev, "bus reset", PCIE_RESET_READY_POLL_MS);
  4212. }
  4213. EXPORT_SYMBOL_GPL(pci_bridge_secondary_bus_reset);
  4214. static int pci_parent_bus_reset(struct pci_dev *dev, int probe)
  4215. {
  4216. struct pci_dev *pdev;
  4217. if (pci_is_root_bus(dev->bus) || dev->subordinate ||
  4218. !dev->bus->self || dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)
  4219. return -ENOTTY;
  4220. list_for_each_entry(pdev, &dev->bus->devices, bus_list)
  4221. if (pdev != dev)
  4222. return -ENOTTY;
  4223. if (probe)
  4224. return 0;
  4225. return pci_bridge_secondary_bus_reset(dev->bus->self);
  4226. }
  4227. static int pci_reset_hotplug_slot(struct hotplug_slot *hotplug, int probe)
  4228. {
  4229. int rc = -ENOTTY;
  4230. if (!hotplug || !try_module_get(hotplug->owner))
  4231. return rc;
  4232. if (hotplug->ops->reset_slot)
  4233. rc = hotplug->ops->reset_slot(hotplug, probe);
  4234. module_put(hotplug->owner);
  4235. return rc;
  4236. }
  4237. static int pci_dev_reset_slot_function(struct pci_dev *dev, int probe)
  4238. {
  4239. if (dev->multifunction || dev->subordinate || !dev->slot ||
  4240. dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)
  4241. return -ENOTTY;
  4242. return pci_reset_hotplug_slot(dev->slot->hotplug, probe);
  4243. }
  4244. static void pci_dev_lock(struct pci_dev *dev)
  4245. {
  4246. pci_cfg_access_lock(dev);
  4247. /* block PM suspend, driver probe, etc. */
  4248. device_lock(&dev->dev);
  4249. }
  4250. /* Return 1 on successful lock, 0 on contention */
  4251. static int pci_dev_trylock(struct pci_dev *dev)
  4252. {
  4253. if (pci_cfg_access_trylock(dev)) {
  4254. if (device_trylock(&dev->dev))
  4255. return 1;
  4256. pci_cfg_access_unlock(dev);
  4257. }
  4258. return 0;
  4259. }
  4260. static void pci_dev_unlock(struct pci_dev *dev)
  4261. {
  4262. device_unlock(&dev->dev);
  4263. pci_cfg_access_unlock(dev);
  4264. }
  4265. static void pci_dev_save_and_disable(struct pci_dev *dev)
  4266. {
  4267. const struct pci_error_handlers *err_handler =
  4268. dev->driver ? dev->driver->err_handler : NULL;
  4269. /*
  4270. * dev->driver->err_handler->reset_prepare() is protected against
  4271. * races with ->remove() by the device lock, which must be held by
  4272. * the caller.
  4273. */
  4274. if (err_handler && err_handler->reset_prepare)
  4275. err_handler->reset_prepare(dev);
  4276. /*
  4277. * Wake-up device prior to save. PM registers default to D0 after
  4278. * reset and a simple register restore doesn't reliably return
  4279. * to a non-D0 state anyway.
  4280. */
  4281. pci_set_power_state(dev, PCI_D0);
  4282. pci_save_state(dev);
  4283. /*
  4284. * Disable the device by clearing the Command register, except for
  4285. * INTx-disable which is set. This not only disables MMIO and I/O port
  4286. * BARs, but also prevents the device from being Bus Master, preventing
  4287. * DMA from the device including MSI/MSI-X interrupts. For PCI 2.3
  4288. * compliant devices, INTx-disable prevents legacy interrupts.
  4289. */
  4290. pci_write_config_word(dev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE);
  4291. }
  4292. static void pci_dev_restore(struct pci_dev *dev)
  4293. {
  4294. const struct pci_error_handlers *err_handler =
  4295. dev->driver ? dev->driver->err_handler : NULL;
  4296. pci_restore_state(dev);
  4297. /*
  4298. * dev->driver->err_handler->reset_done() is protected against
  4299. * races with ->remove() by the device lock, which must be held by
  4300. * the caller.
  4301. */
  4302. if (err_handler && err_handler->reset_done)
  4303. err_handler->reset_done(dev);
  4304. }
  4305. /**
  4306. * __pci_reset_function_locked - reset a PCI device function while holding
  4307. * the @dev mutex lock.
  4308. * @dev: PCI device to reset
  4309. *
  4310. * Some devices allow an individual function to be reset without affecting
  4311. * other functions in the same device. The PCI device must be responsive
  4312. * to PCI config space in order to use this function.
  4313. *
  4314. * The device function is presumed to be unused and the caller is holding
  4315. * the device mutex lock when this function is called.
  4316. *
  4317. * Resetting the device will make the contents of PCI configuration space
  4318. * random, so any caller of this must be prepared to reinitialise the
  4319. * device including MSI, bus mastering, BARs, decoding IO and memory spaces,
  4320. * etc.
  4321. *
  4322. * Returns 0 if the device function was successfully reset or negative if the
  4323. * device doesn't support resetting a single function.
  4324. */
  4325. int __pci_reset_function_locked(struct pci_dev *dev)
  4326. {
  4327. int rc;
  4328. might_sleep();
  4329. /*
  4330. * A reset method returns -ENOTTY if it doesn't support this device
  4331. * and we should try the next method.
  4332. *
  4333. * If it returns 0 (success), we're finished. If it returns any
  4334. * other error, we're also finished: this indicates that further
  4335. * reset mechanisms might be broken on the device.
  4336. */
  4337. rc = pci_dev_specific_reset(dev, 0);
  4338. if (rc != -ENOTTY)
  4339. return rc;
  4340. if (pcie_has_flr(dev)) {
  4341. rc = pcie_flr(dev);
  4342. if (rc != -ENOTTY)
  4343. return rc;
  4344. }
  4345. rc = pci_af_flr(dev, 0);
  4346. if (rc != -ENOTTY)
  4347. return rc;
  4348. rc = pci_pm_reset(dev, 0);
  4349. if (rc != -ENOTTY)
  4350. return rc;
  4351. rc = pci_dev_reset_slot_function(dev, 0);
  4352. if (rc != -ENOTTY)
  4353. return rc;
  4354. return pci_parent_bus_reset(dev, 0);
  4355. }
  4356. EXPORT_SYMBOL_GPL(__pci_reset_function_locked);
  4357. /**
  4358. * pci_probe_reset_function - check whether the device can be safely reset
  4359. * @dev: PCI device to reset
  4360. *
  4361. * Some devices allow an individual function to be reset without affecting
  4362. * other functions in the same device. The PCI device must be responsive
  4363. * to PCI config space in order to use this function.
  4364. *
  4365. * Returns 0 if the device function can be reset or negative if the
  4366. * device doesn't support resetting a single function.
  4367. */
  4368. int pci_probe_reset_function(struct pci_dev *dev)
  4369. {
  4370. int rc;
  4371. might_sleep();
  4372. rc = pci_dev_specific_reset(dev, 1);
  4373. if (rc != -ENOTTY)
  4374. return rc;
  4375. if (pcie_has_flr(dev))
  4376. return 0;
  4377. rc = pci_af_flr(dev, 1);
  4378. if (rc != -ENOTTY)
  4379. return rc;
  4380. rc = pci_pm_reset(dev, 1);
  4381. if (rc != -ENOTTY)
  4382. return rc;
  4383. rc = pci_dev_reset_slot_function(dev, 1);
  4384. if (rc != -ENOTTY)
  4385. return rc;
  4386. return pci_parent_bus_reset(dev, 1);
  4387. }
  4388. /**
  4389. * pci_reset_function - quiesce and reset a PCI device function
  4390. * @dev: PCI device to reset
  4391. *
  4392. * Some devices allow an individual function to be reset without affecting
  4393. * other functions in the same device. The PCI device must be responsive
  4394. * to PCI config space in order to use this function.
  4395. *
  4396. * This function does not just reset the PCI portion of a device, but
  4397. * clears all the state associated with the device. This function differs
  4398. * from __pci_reset_function_locked() in that it saves and restores device state
  4399. * over the reset and takes the PCI device lock.
  4400. *
  4401. * Returns 0 if the device function was successfully reset or negative if the
  4402. * device doesn't support resetting a single function.
  4403. */
  4404. int pci_reset_function(struct pci_dev *dev)
  4405. {
  4406. int rc;
  4407. if (!dev->reset_fn)
  4408. return -ENOTTY;
  4409. pci_dev_lock(dev);
  4410. pci_dev_save_and_disable(dev);
  4411. rc = __pci_reset_function_locked(dev);
  4412. pci_dev_restore(dev);
  4413. pci_dev_unlock(dev);
  4414. return rc;
  4415. }
  4416. EXPORT_SYMBOL_GPL(pci_reset_function);
  4417. /**
  4418. * pci_reset_function_locked - quiesce and reset a PCI device function
  4419. * @dev: PCI device to reset
  4420. *
  4421. * Some devices allow an individual function to be reset without affecting
  4422. * other functions in the same device. The PCI device must be responsive
  4423. * to PCI config space in order to use this function.
  4424. *
  4425. * This function does not just reset the PCI portion of a device, but
  4426. * clears all the state associated with the device. This function differs
  4427. * from __pci_reset_function_locked() in that it saves and restores device state
  4428. * over the reset. It also differs from pci_reset_function() in that it
  4429. * requires the PCI device lock to be held.
  4430. *
  4431. * Returns 0 if the device function was successfully reset or negative if the
  4432. * device doesn't support resetting a single function.
  4433. */
  4434. int pci_reset_function_locked(struct pci_dev *dev)
  4435. {
  4436. int rc;
  4437. if (!dev->reset_fn)
  4438. return -ENOTTY;
  4439. pci_dev_save_and_disable(dev);
  4440. rc = __pci_reset_function_locked(dev);
  4441. pci_dev_restore(dev);
  4442. return rc;
  4443. }
  4444. EXPORT_SYMBOL_GPL(pci_reset_function_locked);
  4445. /**
  4446. * pci_try_reset_function - quiesce and reset a PCI device function
  4447. * @dev: PCI device to reset
  4448. *
  4449. * Same as above, except return -EAGAIN if unable to lock device.
  4450. */
  4451. int pci_try_reset_function(struct pci_dev *dev)
  4452. {
  4453. int rc;
  4454. if (!dev->reset_fn)
  4455. return -ENOTTY;
  4456. if (!pci_dev_trylock(dev))
  4457. return -EAGAIN;
  4458. pci_dev_save_and_disable(dev);
  4459. rc = __pci_reset_function_locked(dev);
  4460. pci_dev_restore(dev);
  4461. pci_dev_unlock(dev);
  4462. return rc;
  4463. }
  4464. EXPORT_SYMBOL_GPL(pci_try_reset_function);
  4465. /* Do any devices on or below this bus prevent a bus reset? */
  4466. static bool pci_bus_resetable(struct pci_bus *bus)
  4467. {
  4468. struct pci_dev *dev;
  4469. if (bus->self && (bus->self->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET))
  4470. return false;
  4471. list_for_each_entry(dev, &bus->devices, bus_list) {
  4472. if (dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET ||
  4473. (dev->subordinate && !pci_bus_resetable(dev->subordinate)))
  4474. return false;
  4475. }
  4476. return true;
  4477. }
  4478. /* Lock devices from the top of the tree down */
  4479. static void pci_bus_lock(struct pci_bus *bus)
  4480. {
  4481. struct pci_dev *dev;
  4482. list_for_each_entry(dev, &bus->devices, bus_list) {
  4483. pci_dev_lock(dev);
  4484. if (dev->subordinate)
  4485. pci_bus_lock(dev->subordinate);
  4486. }
  4487. }
  4488. /* Unlock devices from the bottom of the tree up */
  4489. static void pci_bus_unlock(struct pci_bus *bus)
  4490. {
  4491. struct pci_dev *dev;
  4492. list_for_each_entry(dev, &bus->devices, bus_list) {
  4493. if (dev->subordinate)
  4494. pci_bus_unlock(dev->subordinate);
  4495. pci_dev_unlock(dev);
  4496. }
  4497. }
  4498. /* Return 1 on successful lock, 0 on contention */
  4499. static int pci_bus_trylock(struct pci_bus *bus)
  4500. {
  4501. struct pci_dev *dev;
  4502. list_for_each_entry(dev, &bus->devices, bus_list) {
  4503. if (!pci_dev_trylock(dev))
  4504. goto unlock;
  4505. if (dev->subordinate) {
  4506. if (!pci_bus_trylock(dev->subordinate)) {
  4507. pci_dev_unlock(dev);
  4508. goto unlock;
  4509. }
  4510. }
  4511. }
  4512. return 1;
  4513. unlock:
  4514. list_for_each_entry_continue_reverse(dev, &bus->devices, bus_list) {
  4515. if (dev->subordinate)
  4516. pci_bus_unlock(dev->subordinate);
  4517. pci_dev_unlock(dev);
  4518. }
  4519. return 0;
  4520. }
  4521. /* Do any devices on or below this slot prevent a bus reset? */
  4522. static bool pci_slot_resetable(struct pci_slot *slot)
  4523. {
  4524. struct pci_dev *dev;
  4525. if (slot->bus->self &&
  4526. (slot->bus->self->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET))
  4527. return false;
  4528. list_for_each_entry(dev, &slot->bus->devices, bus_list) {
  4529. if (!dev->slot || dev->slot != slot)
  4530. continue;
  4531. if (dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET ||
  4532. (dev->subordinate && !pci_bus_resetable(dev->subordinate)))
  4533. return false;
  4534. }
  4535. return true;
  4536. }
  4537. /* Lock devices from the top of the tree down */
  4538. static void pci_slot_lock(struct pci_slot *slot)
  4539. {
  4540. struct pci_dev *dev;
  4541. list_for_each_entry(dev, &slot->bus->devices, bus_list) {
  4542. if (!dev->slot || dev->slot != slot)
  4543. continue;
  4544. pci_dev_lock(dev);
  4545. if (dev->subordinate)
  4546. pci_bus_lock(dev->subordinate);
  4547. }
  4548. }
  4549. /* Unlock devices from the bottom of the tree up */
  4550. static void pci_slot_unlock(struct pci_slot *slot)
  4551. {
  4552. struct pci_dev *dev;
  4553. list_for_each_entry(dev, &slot->bus->devices, bus_list) {
  4554. if (!dev->slot || dev->slot != slot)
  4555. continue;
  4556. if (dev->subordinate)
  4557. pci_bus_unlock(dev->subordinate);
  4558. pci_dev_unlock(dev);
  4559. }
  4560. }
  4561. /* Return 1 on successful lock, 0 on contention */
  4562. static int pci_slot_trylock(struct pci_slot *slot)
  4563. {
  4564. struct pci_dev *dev;
  4565. list_for_each_entry(dev, &slot->bus->devices, bus_list) {
  4566. if (!dev->slot || dev->slot != slot)
  4567. continue;
  4568. if (!pci_dev_trylock(dev))
  4569. goto unlock;
  4570. if (dev->subordinate) {
  4571. if (!pci_bus_trylock(dev->subordinate)) {
  4572. pci_dev_unlock(dev);
  4573. goto unlock;
  4574. }
  4575. }
  4576. }
  4577. return 1;
  4578. unlock:
  4579. list_for_each_entry_continue_reverse(dev,
  4580. &slot->bus->devices, bus_list) {
  4581. if (!dev->slot || dev->slot != slot)
  4582. continue;
  4583. if (dev->subordinate)
  4584. pci_bus_unlock(dev->subordinate);
  4585. pci_dev_unlock(dev);
  4586. }
  4587. return 0;
  4588. }
  4589. /*
  4590. * Save and disable devices from the top of the tree down while holding
  4591. * the @dev mutex lock for the entire tree.
  4592. */
  4593. static void pci_bus_save_and_disable_locked(struct pci_bus *bus)
  4594. {
  4595. struct pci_dev *dev;
  4596. list_for_each_entry(dev, &bus->devices, bus_list) {
  4597. pci_dev_save_and_disable(dev);
  4598. if (dev->subordinate)
  4599. pci_bus_save_and_disable_locked(dev->subordinate);
  4600. }
  4601. }
  4602. /*
  4603. * Restore devices from top of the tree down while holding @dev mutex lock
  4604. * for the entire tree. Parent bridges need to be restored before we can
  4605. * get to subordinate devices.
  4606. */
  4607. static void pci_bus_restore_locked(struct pci_bus *bus)
  4608. {
  4609. struct pci_dev *dev;
  4610. list_for_each_entry(dev, &bus->devices, bus_list) {
  4611. pci_dev_restore(dev);
  4612. if (dev->subordinate)
  4613. pci_bus_restore_locked(dev->subordinate);
  4614. }
  4615. }
  4616. /*
  4617. * Save and disable devices from the top of the tree down while holding
  4618. * the @dev mutex lock for the entire tree.
  4619. */
  4620. static void pci_slot_save_and_disable_locked(struct pci_slot *slot)
  4621. {
  4622. struct pci_dev *dev;
  4623. list_for_each_entry(dev, &slot->bus->devices, bus_list) {
  4624. if (!dev->slot || dev->slot != slot)
  4625. continue;
  4626. pci_dev_save_and_disable(dev);
  4627. if (dev->subordinate)
  4628. pci_bus_save_and_disable_locked(dev->subordinate);
  4629. }
  4630. }
  4631. /*
  4632. * Restore devices from top of the tree down while holding @dev mutex lock
  4633. * for the entire tree. Parent bridges need to be restored before we can
  4634. * get to subordinate devices.
  4635. */
  4636. static void pci_slot_restore_locked(struct pci_slot *slot)
  4637. {
  4638. struct pci_dev *dev;
  4639. list_for_each_entry(dev, &slot->bus->devices, bus_list) {
  4640. if (!dev->slot || dev->slot != slot)
  4641. continue;
  4642. pci_dev_restore(dev);
  4643. if (dev->subordinate)
  4644. pci_bus_restore_locked(dev->subordinate);
  4645. }
  4646. }
  4647. static int pci_slot_reset(struct pci_slot *slot, int probe)
  4648. {
  4649. int rc;
  4650. if (!slot || !pci_slot_resetable(slot))
  4651. return -ENOTTY;
  4652. if (!probe)
  4653. pci_slot_lock(slot);
  4654. might_sleep();
  4655. rc = pci_reset_hotplug_slot(slot->hotplug, probe);
  4656. if (!probe)
  4657. pci_slot_unlock(slot);
  4658. return rc;
  4659. }
  4660. /**
  4661. * pci_probe_reset_slot - probe whether a PCI slot can be reset
  4662. * @slot: PCI slot to probe
  4663. *
  4664. * Return 0 if slot can be reset, negative if a slot reset is not supported.
  4665. */
  4666. int pci_probe_reset_slot(struct pci_slot *slot)
  4667. {
  4668. return pci_slot_reset(slot, 1);
  4669. }
  4670. EXPORT_SYMBOL_GPL(pci_probe_reset_slot);
  4671. /**
  4672. * __pci_reset_slot - Try to reset a PCI slot
  4673. * @slot: PCI slot to reset
  4674. *
  4675. * A PCI bus may host multiple slots, each slot may support a reset mechanism
  4676. * independent of other slots. For instance, some slots may support slot power
  4677. * control. In the case of a 1:1 bus to slot architecture, this function may
  4678. * wrap the bus reset to avoid spurious slot related events such as hotplug.
  4679. * Generally a slot reset should be attempted before a bus reset. All of the
  4680. * function of the slot and any subordinate buses behind the slot are reset
  4681. * through this function. PCI config space of all devices in the slot and
  4682. * behind the slot is saved before and restored after reset.
  4683. *
  4684. * Same as above except return -EAGAIN if the slot cannot be locked
  4685. */
  4686. static int __pci_reset_slot(struct pci_slot *slot)
  4687. {
  4688. int rc;
  4689. rc = pci_slot_reset(slot, 1);
  4690. if (rc)
  4691. return rc;
  4692. if (pci_slot_trylock(slot)) {
  4693. pci_slot_save_and_disable_locked(slot);
  4694. might_sleep();
  4695. rc = pci_reset_hotplug_slot(slot->hotplug, 0);
  4696. pci_slot_restore_locked(slot);
  4697. pci_slot_unlock(slot);
  4698. } else
  4699. rc = -EAGAIN;
  4700. return rc;
  4701. }
  4702. static int pci_bus_reset(struct pci_bus *bus, int probe)
  4703. {
  4704. int ret;
  4705. if (!bus->self || !pci_bus_resetable(bus))
  4706. return -ENOTTY;
  4707. if (probe)
  4708. return 0;
  4709. pci_bus_lock(bus);
  4710. might_sleep();
  4711. ret = pci_bridge_secondary_bus_reset(bus->self);
  4712. pci_bus_unlock(bus);
  4713. return ret;
  4714. }
  4715. /**
  4716. * pci_bus_error_reset - reset the bridge's subordinate bus
  4717. * @bridge: The parent device that connects to the bus to reset
  4718. *
  4719. * This function will first try to reset the slots on this bus if the method is
  4720. * available. If slot reset fails or is not available, this will fall back to a
  4721. * secondary bus reset.
  4722. */
  4723. int pci_bus_error_reset(struct pci_dev *bridge)
  4724. {
  4725. struct pci_bus *bus = bridge->subordinate;
  4726. struct pci_slot *slot;
  4727. if (!bus)
  4728. return -ENOTTY;
  4729. mutex_lock(&pci_slot_mutex);
  4730. if (list_empty(&bus->slots))
  4731. goto bus_reset;
  4732. list_for_each_entry(slot, &bus->slots, list)
  4733. if (pci_probe_reset_slot(slot))
  4734. goto bus_reset;
  4735. list_for_each_entry(slot, &bus->slots, list)
  4736. if (pci_slot_reset(slot, 0))
  4737. goto bus_reset;
  4738. mutex_unlock(&pci_slot_mutex);
  4739. return 0;
  4740. bus_reset:
  4741. mutex_unlock(&pci_slot_mutex);
  4742. return pci_bus_reset(bridge->subordinate, 0);
  4743. }
  4744. /**
  4745. * pci_probe_reset_bus - probe whether a PCI bus can be reset
  4746. * @bus: PCI bus to probe
  4747. *
  4748. * Return 0 if bus can be reset, negative if a bus reset is not supported.
  4749. */
  4750. int pci_probe_reset_bus(struct pci_bus *bus)
  4751. {
  4752. return pci_bus_reset(bus, 1);
  4753. }
  4754. EXPORT_SYMBOL_GPL(pci_probe_reset_bus);
  4755. /**
  4756. * __pci_reset_bus - Try to reset a PCI bus
  4757. * @bus: top level PCI bus to reset
  4758. *
  4759. * Same as above except return -EAGAIN if the bus cannot be locked
  4760. */
  4761. static int __pci_reset_bus(struct pci_bus *bus)
  4762. {
  4763. int rc;
  4764. rc = pci_bus_reset(bus, 1);
  4765. if (rc)
  4766. return rc;
  4767. if (pci_bus_trylock(bus)) {
  4768. pci_bus_save_and_disable_locked(bus);
  4769. might_sleep();
  4770. rc = pci_bridge_secondary_bus_reset(bus->self);
  4771. pci_bus_restore_locked(bus);
  4772. pci_bus_unlock(bus);
  4773. } else
  4774. rc = -EAGAIN;
  4775. return rc;
  4776. }
  4777. /**
  4778. * pci_reset_bus - Try to reset a PCI bus
  4779. * @pdev: top level PCI device to reset via slot/bus
  4780. *
  4781. * Same as above except return -EAGAIN if the bus cannot be locked
  4782. */
  4783. int pci_reset_bus(struct pci_dev *pdev)
  4784. {
  4785. return (!pci_probe_reset_slot(pdev->slot)) ?
  4786. __pci_reset_slot(pdev->slot) : __pci_reset_bus(pdev->bus);
  4787. }
  4788. EXPORT_SYMBOL_GPL(pci_reset_bus);
  4789. /**
  4790. * pcix_get_max_mmrbc - get PCI-X maximum designed memory read byte count
  4791. * @dev: PCI device to query
  4792. *
  4793. * Returns mmrbc: maximum designed memory read count in bytes or
  4794. * appropriate error value.
  4795. */
  4796. int pcix_get_max_mmrbc(struct pci_dev *dev)
  4797. {
  4798. int cap;
  4799. u32 stat;
  4800. cap = pci_find_capability(dev, PCI_CAP_ID_PCIX);
  4801. if (!cap)
  4802. return -EINVAL;
  4803. if (pci_read_config_dword(dev, cap + PCI_X_STATUS, &stat))
  4804. return -EINVAL;
  4805. return 512 << ((stat & PCI_X_STATUS_MAX_READ) >> 21);
  4806. }
  4807. EXPORT_SYMBOL(pcix_get_max_mmrbc);
  4808. /**
  4809. * pcix_get_mmrbc - get PCI-X maximum memory read byte count
  4810. * @dev: PCI device to query
  4811. *
  4812. * Returns mmrbc: maximum memory read count in bytes or appropriate error
  4813. * value.
  4814. */
  4815. int pcix_get_mmrbc(struct pci_dev *dev)
  4816. {
  4817. int cap;
  4818. u16 cmd;
  4819. cap = pci_find_capability(dev, PCI_CAP_ID_PCIX);
  4820. if (!cap)
  4821. return -EINVAL;
  4822. if (pci_read_config_word(dev, cap + PCI_X_CMD, &cmd))
  4823. return -EINVAL;
  4824. return 512 << ((cmd & PCI_X_CMD_MAX_READ) >> 2);
  4825. }
  4826. EXPORT_SYMBOL(pcix_get_mmrbc);
  4827. /**
  4828. * pcix_set_mmrbc - set PCI-X maximum memory read byte count
  4829. * @dev: PCI device to query
  4830. * @mmrbc: maximum memory read count in bytes
  4831. * valid values are 512, 1024, 2048, 4096
  4832. *
  4833. * If possible sets maximum memory read byte count, some bridges have errata
  4834. * that prevent this.
  4835. */
  4836. int pcix_set_mmrbc(struct pci_dev *dev, int mmrbc)
  4837. {
  4838. int cap;
  4839. u32 stat, v, o;
  4840. u16 cmd;
  4841. if (mmrbc < 512 || mmrbc > 4096 || !is_power_of_2(mmrbc))
  4842. return -EINVAL;
  4843. v = ffs(mmrbc) - 10;
  4844. cap = pci_find_capability(dev, PCI_CAP_ID_PCIX);
  4845. if (!cap)
  4846. return -EINVAL;
  4847. if (pci_read_config_dword(dev, cap + PCI_X_STATUS, &stat))
  4848. return -EINVAL;
  4849. if (v > (stat & PCI_X_STATUS_MAX_READ) >> 21)
  4850. return -E2BIG;
  4851. if (pci_read_config_word(dev, cap + PCI_X_CMD, &cmd))
  4852. return -EINVAL;
  4853. o = (cmd & PCI_X_CMD_MAX_READ) >> 2;
  4854. if (o != v) {
  4855. if (v > o && (dev->bus->bus_flags & PCI_BUS_FLAGS_NO_MMRBC))
  4856. return -EIO;
  4857. cmd &= ~PCI_X_CMD_MAX_READ;
  4858. cmd |= v << 2;
  4859. if (pci_write_config_word(dev, cap + PCI_X_CMD, cmd))
  4860. return -EIO;
  4861. }
  4862. return 0;
  4863. }
  4864. EXPORT_SYMBOL(pcix_set_mmrbc);
  4865. /**
  4866. * pcie_get_readrq - get PCI Express read request size
  4867. * @dev: PCI device to query
  4868. *
  4869. * Returns maximum memory read request in bytes or appropriate error value.
  4870. */
  4871. int pcie_get_readrq(struct pci_dev *dev)
  4872. {
  4873. u16 ctl;
  4874. pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &ctl);
  4875. return 128 << ((ctl & PCI_EXP_DEVCTL_READRQ) >> 12);
  4876. }
  4877. EXPORT_SYMBOL(pcie_get_readrq);
  4878. /**
  4879. * pcie_set_readrq - set PCI Express maximum memory read request
  4880. * @dev: PCI device to query
  4881. * @rq: maximum memory read count in bytes
  4882. * valid values are 128, 256, 512, 1024, 2048, 4096
  4883. *
  4884. * If possible sets maximum memory read request in bytes
  4885. */
  4886. int pcie_set_readrq(struct pci_dev *dev, int rq)
  4887. {
  4888. u16 v;
  4889. int ret;
  4890. if (rq < 128 || rq > 4096 || !is_power_of_2(rq))
  4891. return -EINVAL;
  4892. /*
  4893. * If using the "performance" PCIe config, we clamp the read rq
  4894. * size to the max packet size to keep the host bridge from
  4895. * generating requests larger than we can cope with.
  4896. */
  4897. if (pcie_bus_config == PCIE_BUS_PERFORMANCE) {
  4898. int mps = pcie_get_mps(dev);
  4899. if (mps < rq)
  4900. rq = mps;
  4901. }
  4902. v = (ffs(rq) - 8) << 12;
  4903. ret = pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL,
  4904. PCI_EXP_DEVCTL_READRQ, v);
  4905. return pcibios_err_to_errno(ret);
  4906. }
  4907. EXPORT_SYMBOL(pcie_set_readrq);
  4908. /**
  4909. * pcie_get_mps - get PCI Express maximum payload size
  4910. * @dev: PCI device to query
  4911. *
  4912. * Returns maximum payload size in bytes
  4913. */
  4914. int pcie_get_mps(struct pci_dev *dev)
  4915. {
  4916. u16 ctl;
  4917. pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &ctl);
  4918. return 128 << ((ctl & PCI_EXP_DEVCTL_PAYLOAD) >> 5);
  4919. }
  4920. EXPORT_SYMBOL(pcie_get_mps);
  4921. /**
  4922. * pcie_set_mps - set PCI Express maximum payload size
  4923. * @dev: PCI device to query
  4924. * @mps: maximum payload size in bytes
  4925. * valid values are 128, 256, 512, 1024, 2048, 4096
  4926. *
  4927. * If possible sets maximum payload size
  4928. */
  4929. int pcie_set_mps(struct pci_dev *dev, int mps)
  4930. {
  4931. u16 v;
  4932. int ret;
  4933. if (mps < 128 || mps > 4096 || !is_power_of_2(mps))
  4934. return -EINVAL;
  4935. v = ffs(mps) - 8;
  4936. if (v > dev->pcie_mpss)
  4937. return -EINVAL;
  4938. v <<= 5;
  4939. ret = pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL,
  4940. PCI_EXP_DEVCTL_PAYLOAD, v);
  4941. return pcibios_err_to_errno(ret);
  4942. }
  4943. EXPORT_SYMBOL(pcie_set_mps);
  4944. /**
  4945. * pcie_bandwidth_available - determine minimum link settings of a PCIe
  4946. * device and its bandwidth limitation
  4947. * @dev: PCI device to query
  4948. * @limiting_dev: storage for device causing the bandwidth limitation
  4949. * @speed: storage for speed of limiting device
  4950. * @width: storage for width of limiting device
  4951. *
  4952. * Walk up the PCI device chain and find the point where the minimum
  4953. * bandwidth is available. Return the bandwidth available there and (if
  4954. * limiting_dev, speed, and width pointers are supplied) information about
  4955. * that point. The bandwidth returned is in Mb/s, i.e., megabits/second of
  4956. * raw bandwidth.
  4957. */
  4958. u32 pcie_bandwidth_available(struct pci_dev *dev, struct pci_dev **limiting_dev,
  4959. enum pci_bus_speed *speed,
  4960. enum pcie_link_width *width)
  4961. {
  4962. u16 lnksta;
  4963. enum pci_bus_speed next_speed;
  4964. enum pcie_link_width next_width;
  4965. u32 bw, next_bw;
  4966. if (speed)
  4967. *speed = PCI_SPEED_UNKNOWN;
  4968. if (width)
  4969. *width = PCIE_LNK_WIDTH_UNKNOWN;
  4970. bw = 0;
  4971. while (dev) {
  4972. pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &lnksta);
  4973. next_speed = pcie_link_speed[lnksta & PCI_EXP_LNKSTA_CLS];
  4974. next_width = (lnksta & PCI_EXP_LNKSTA_NLW) >>
  4975. PCI_EXP_LNKSTA_NLW_SHIFT;
  4976. next_bw = next_width * PCIE_SPEED2MBS_ENC(next_speed);
  4977. /* Check if current device limits the total bandwidth */
  4978. if (!bw || next_bw <= bw) {
  4979. bw = next_bw;
  4980. if (limiting_dev)
  4981. *limiting_dev = dev;
  4982. if (speed)
  4983. *speed = next_speed;
  4984. if (width)
  4985. *width = next_width;
  4986. }
  4987. dev = pci_upstream_bridge(dev);
  4988. }
  4989. return bw;
  4990. }
  4991. EXPORT_SYMBOL(pcie_bandwidth_available);
  4992. /**
  4993. * pcie_get_speed_cap - query for the PCI device's link speed capability
  4994. * @dev: PCI device to query
  4995. *
  4996. * Query the PCI device speed capability. Return the maximum link speed
  4997. * supported by the device.
  4998. */
  4999. enum pci_bus_speed pcie_get_speed_cap(struct pci_dev *dev)
  5000. {
  5001. u32 lnkcap2, lnkcap;
  5002. /*
  5003. * Link Capabilities 2 was added in PCIe r3.0, sec 7.8.18. The
  5004. * implementation note there recommends using the Supported Link
  5005. * Speeds Vector in Link Capabilities 2 when supported.
  5006. *
  5007. * Without Link Capabilities 2, i.e., prior to PCIe r3.0, software
  5008. * should use the Supported Link Speeds field in Link Capabilities,
  5009. * where only 2.5 GT/s and 5.0 GT/s speeds were defined.
  5010. */
  5011. pcie_capability_read_dword(dev, PCI_EXP_LNKCAP2, &lnkcap2);
  5012. /* PCIe r3.0-compliant */
  5013. if (lnkcap2)
  5014. return PCIE_LNKCAP2_SLS2SPEED(lnkcap2);
  5015. pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &lnkcap);
  5016. if ((lnkcap & PCI_EXP_LNKCAP_SLS) == PCI_EXP_LNKCAP_SLS_5_0GB)
  5017. return PCIE_SPEED_5_0GT;
  5018. else if ((lnkcap & PCI_EXP_LNKCAP_SLS) == PCI_EXP_LNKCAP_SLS_2_5GB)
  5019. return PCIE_SPEED_2_5GT;
  5020. return PCI_SPEED_UNKNOWN;
  5021. }
  5022. EXPORT_SYMBOL(pcie_get_speed_cap);
  5023. /**
  5024. * pcie_get_width_cap - query for the PCI device's link width capability
  5025. * @dev: PCI device to query
  5026. *
  5027. * Query the PCI device width capability. Return the maximum link width
  5028. * supported by the device.
  5029. */
  5030. enum pcie_link_width pcie_get_width_cap(struct pci_dev *dev)
  5031. {
  5032. u32 lnkcap;
  5033. pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &lnkcap);
  5034. if (lnkcap)
  5035. return (lnkcap & PCI_EXP_LNKCAP_MLW) >> 4;
  5036. return PCIE_LNK_WIDTH_UNKNOWN;
  5037. }
  5038. EXPORT_SYMBOL(pcie_get_width_cap);
  5039. /**
  5040. * pcie_bandwidth_capable - calculate a PCI device's link bandwidth capability
  5041. * @dev: PCI device
  5042. * @speed: storage for link speed
  5043. * @width: storage for link width
  5044. *
  5045. * Calculate a PCI device's link bandwidth by querying for its link speed
  5046. * and width, multiplying them, and applying encoding overhead. The result
  5047. * is in Mb/s, i.e., megabits/second of raw bandwidth.
  5048. */
  5049. u32 pcie_bandwidth_capable(struct pci_dev *dev, enum pci_bus_speed *speed,
  5050. enum pcie_link_width *width)
  5051. {
  5052. *speed = pcie_get_speed_cap(dev);
  5053. *width = pcie_get_width_cap(dev);
  5054. if (*speed == PCI_SPEED_UNKNOWN || *width == PCIE_LNK_WIDTH_UNKNOWN)
  5055. return 0;
  5056. return *width * PCIE_SPEED2MBS_ENC(*speed);
  5057. }
  5058. /**
  5059. * __pcie_print_link_status - Report the PCI device's link speed and width
  5060. * @dev: PCI device to query
  5061. * @verbose: Print info even when enough bandwidth is available
  5062. *
  5063. * If the available bandwidth at the device is less than the device is
  5064. * capable of, report the device's maximum possible bandwidth and the
  5065. * upstream link that limits its performance. If @verbose, always print
  5066. * the available bandwidth, even if the device isn't constrained.
  5067. */
  5068. void __pcie_print_link_status(struct pci_dev *dev, bool verbose)
  5069. {
  5070. enum pcie_link_width width, width_cap;
  5071. enum pci_bus_speed speed, speed_cap;
  5072. struct pci_dev *limiting_dev = NULL;
  5073. u32 bw_avail, bw_cap;
  5074. bw_cap = pcie_bandwidth_capable(dev, &speed_cap, &width_cap);
  5075. bw_avail = pcie_bandwidth_available(dev, &limiting_dev, &speed, &width);
  5076. if (bw_avail >= bw_cap && verbose)
  5077. pci_info(dev, "%u.%03u Gb/s available PCIe bandwidth (%s x%d link)\n",
  5078. bw_cap / 1000, bw_cap % 1000,
  5079. pci_speed_string(speed_cap), width_cap);
  5080. else if (bw_avail < bw_cap)
  5081. pci_info(dev, "%u.%03u Gb/s available PCIe bandwidth, limited by %s x%d link at %s (capable of %u.%03u Gb/s with %s x%d link)\n",
  5082. bw_avail / 1000, bw_avail % 1000,
  5083. pci_speed_string(speed), width,
  5084. limiting_dev ? pci_name(limiting_dev) : "<unknown>",
  5085. bw_cap / 1000, bw_cap % 1000,
  5086. pci_speed_string(speed_cap), width_cap);
  5087. }
  5088. /**
  5089. * pcie_print_link_status - Report the PCI device's link speed and width
  5090. * @dev: PCI device to query
  5091. *
  5092. * Report the available bandwidth at the device.
  5093. */
  5094. void pcie_print_link_status(struct pci_dev *dev)
  5095. {
  5096. __pcie_print_link_status(dev, true);
  5097. }
  5098. EXPORT_SYMBOL(pcie_print_link_status);
  5099. /**
  5100. * pci_select_bars - Make BAR mask from the type of resource
  5101. * @dev: the PCI device for which BAR mask is made
  5102. * @flags: resource type mask to be selected
  5103. *
  5104. * This helper routine makes bar mask from the type of resource.
  5105. */
  5106. int pci_select_bars(struct pci_dev *dev, unsigned long flags)
  5107. {
  5108. int i, bars = 0;
  5109. for (i = 0; i < PCI_NUM_RESOURCES; i++)
  5110. if (pci_resource_flags(dev, i) & flags)
  5111. bars |= (1 << i);
  5112. return bars;
  5113. }
  5114. EXPORT_SYMBOL(pci_select_bars);
  5115. /* Some architectures require additional programming to enable VGA */
  5116. static arch_set_vga_state_t arch_set_vga_state;
  5117. void __init pci_register_set_vga_state(arch_set_vga_state_t func)
  5118. {
  5119. arch_set_vga_state = func; /* NULL disables */
  5120. }
  5121. static int pci_set_vga_state_arch(struct pci_dev *dev, bool decode,
  5122. unsigned int command_bits, u32 flags)
  5123. {
  5124. if (arch_set_vga_state)
  5125. return arch_set_vga_state(dev, decode, command_bits,
  5126. flags);
  5127. return 0;
  5128. }
  5129. /**
  5130. * pci_set_vga_state - set VGA decode state on device and parents if requested
  5131. * @dev: the PCI device
  5132. * @decode: true = enable decoding, false = disable decoding
  5133. * @command_bits: PCI_COMMAND_IO and/or PCI_COMMAND_MEMORY
  5134. * @flags: traverse ancestors and change bridges
  5135. * CHANGE_BRIDGE_ONLY / CHANGE_BRIDGE
  5136. */
  5137. int pci_set_vga_state(struct pci_dev *dev, bool decode,
  5138. unsigned int command_bits, u32 flags)
  5139. {
  5140. struct pci_bus *bus;
  5141. struct pci_dev *bridge;
  5142. u16 cmd;
  5143. int rc;
  5144. WARN_ON((flags & PCI_VGA_STATE_CHANGE_DECODES) && (command_bits & ~(PCI_COMMAND_IO|PCI_COMMAND_MEMORY)));
  5145. /* ARCH specific VGA enables */
  5146. rc = pci_set_vga_state_arch(dev, decode, command_bits, flags);
  5147. if (rc)
  5148. return rc;
  5149. if (flags & PCI_VGA_STATE_CHANGE_DECODES) {
  5150. pci_read_config_word(dev, PCI_COMMAND, &cmd);
  5151. if (decode)
  5152. cmd |= command_bits;
  5153. else
  5154. cmd &= ~command_bits;
  5155. pci_write_config_word(dev, PCI_COMMAND, cmd);
  5156. }
  5157. if (!(flags & PCI_VGA_STATE_CHANGE_BRIDGE))
  5158. return 0;
  5159. bus = dev->bus;
  5160. while (bus) {
  5161. bridge = bus->self;
  5162. if (bridge) {
  5163. pci_read_config_word(bridge, PCI_BRIDGE_CONTROL,
  5164. &cmd);
  5165. if (decode)
  5166. cmd |= PCI_BRIDGE_CTL_VGA;
  5167. else
  5168. cmd &= ~PCI_BRIDGE_CTL_VGA;
  5169. pci_write_config_word(bridge, PCI_BRIDGE_CONTROL,
  5170. cmd);
  5171. }
  5172. bus = bus->parent;
  5173. }
  5174. return 0;
  5175. }
  5176. #ifdef CONFIG_ACPI
  5177. bool pci_pr3_present(struct pci_dev *pdev)
  5178. {
  5179. struct acpi_device *adev;
  5180. if (acpi_disabled)
  5181. return false;
  5182. adev = ACPI_COMPANION(&pdev->dev);
  5183. if (!adev)
  5184. return false;
  5185. return adev->power.flags.power_resources &&
  5186. acpi_has_method(adev->handle, "_PR3");
  5187. }
  5188. EXPORT_SYMBOL_GPL(pci_pr3_present);
  5189. #endif
  5190. /**
  5191. * pci_add_dma_alias - Add a DMA devfn alias for a device
  5192. * @dev: the PCI device for which alias is added
  5193. * @devfn_from: alias slot and function
  5194. * @nr_devfns: number of subsequent devfns to alias
  5195. *
  5196. * This helper encodes an 8-bit devfn as a bit number in dma_alias_mask
  5197. * which is used to program permissible bus-devfn source addresses for DMA
  5198. * requests in an IOMMU. These aliases factor into IOMMU group creation
  5199. * and are useful for devices generating DMA requests beyond or different
  5200. * from their logical bus-devfn. Examples include device quirks where the
  5201. * device simply uses the wrong devfn, as well as non-transparent bridges
  5202. * where the alias may be a proxy for devices in another domain.
  5203. *
  5204. * IOMMU group creation is performed during device discovery or addition,
  5205. * prior to any potential DMA mapping and therefore prior to driver probing
  5206. * (especially for userspace assigned devices where IOMMU group definition
  5207. * cannot be left as a userspace activity). DMA aliases should therefore
  5208. * be configured via quirks, such as the PCI fixup header quirk.
  5209. */
  5210. void pci_add_dma_alias(struct pci_dev *dev, u8 devfn_from, unsigned nr_devfns)
  5211. {
  5212. int devfn_to;
  5213. nr_devfns = min(nr_devfns, (unsigned) MAX_NR_DEVFNS - devfn_from);
  5214. devfn_to = devfn_from + nr_devfns - 1;
  5215. if (!dev->dma_alias_mask)
  5216. dev->dma_alias_mask = bitmap_zalloc(MAX_NR_DEVFNS, GFP_KERNEL);
  5217. if (!dev->dma_alias_mask) {
  5218. pci_warn(dev, "Unable to allocate DMA alias mask\n");
  5219. return;
  5220. }
  5221. bitmap_set(dev->dma_alias_mask, devfn_from, nr_devfns);
  5222. if (nr_devfns == 1)
  5223. pci_info(dev, "Enabling fixed DMA alias to %02x.%d\n",
  5224. PCI_SLOT(devfn_from), PCI_FUNC(devfn_from));
  5225. else if (nr_devfns > 1)
  5226. pci_info(dev, "Enabling fixed DMA alias for devfn range from %02x.%d to %02x.%d\n",
  5227. PCI_SLOT(devfn_from), PCI_FUNC(devfn_from),
  5228. PCI_SLOT(devfn_to), PCI_FUNC(devfn_to));
  5229. }
  5230. bool pci_devs_are_dma_aliases(struct pci_dev *dev1, struct pci_dev *dev2)
  5231. {
  5232. return (dev1->dma_alias_mask &&
  5233. test_bit(dev2->devfn, dev1->dma_alias_mask)) ||
  5234. (dev2->dma_alias_mask &&
  5235. test_bit(dev1->devfn, dev2->dma_alias_mask)) ||
  5236. pci_real_dma_dev(dev1) == dev2 ||
  5237. pci_real_dma_dev(dev2) == dev1;
  5238. }
  5239. bool pci_device_is_present(struct pci_dev *pdev)
  5240. {
  5241. u32 v;
  5242. if (pci_dev_is_disconnected(pdev))
  5243. return false;
  5244. return pci_bus_read_dev_vendor_id(pdev->bus, pdev->devfn, &v, 0);
  5245. }
  5246. EXPORT_SYMBOL_GPL(pci_device_is_present);
  5247. void pci_ignore_hotplug(struct pci_dev *dev)
  5248. {
  5249. struct pci_dev *bridge = dev->bus->self;
  5250. dev->ignore_hotplug = 1;
  5251. /* Propagate the "ignore hotplug" setting to the parent bridge. */
  5252. if (bridge)
  5253. bridge->ignore_hotplug = 1;
  5254. }
  5255. EXPORT_SYMBOL_GPL(pci_ignore_hotplug);
  5256. /**
  5257. * pci_real_dma_dev - Get PCI DMA device for PCI device
  5258. * @dev: the PCI device that may have a PCI DMA alias
  5259. *
  5260. * Permits the platform to provide architecture-specific functionality to
  5261. * devices needing to alias DMA to another PCI device on another PCI bus. If
  5262. * the PCI device is on the same bus, it is recommended to use
  5263. * pci_add_dma_alias(). This is the default implementation. Architecture
  5264. * implementations can override this.
  5265. */
  5266. struct pci_dev __weak *pci_real_dma_dev(struct pci_dev *dev)
  5267. {
  5268. return dev;
  5269. }
  5270. resource_size_t __weak pcibios_default_alignment(void)
  5271. {
  5272. return 0;
  5273. }
  5274. /*
  5275. * Arches that don't want to expose struct resource to userland as-is in
  5276. * sysfs and /proc can implement their own pci_resource_to_user().
  5277. */
  5278. void __weak pci_resource_to_user(const struct pci_dev *dev, int bar,
  5279. const struct resource *rsrc,
  5280. resource_size_t *start, resource_size_t *end)
  5281. {
  5282. *start = rsrc->start;
  5283. *end = rsrc->end;
  5284. }
  5285. static char *resource_alignment_param;
  5286. static DEFINE_SPINLOCK(resource_alignment_lock);
  5287. /**
  5288. * pci_specified_resource_alignment - get resource alignment specified by user.
  5289. * @dev: the PCI device to get
  5290. * @resize: whether or not to change resources' size when reassigning alignment
  5291. *
  5292. * RETURNS: Resource alignment if it is specified.
  5293. * Zero if it is not specified.
  5294. */
  5295. static resource_size_t pci_specified_resource_alignment(struct pci_dev *dev,
  5296. bool *resize)
  5297. {
  5298. int align_order, count;
  5299. resource_size_t align = pcibios_default_alignment();
  5300. const char *p;
  5301. int ret;
  5302. spin_lock(&resource_alignment_lock);
  5303. p = resource_alignment_param;
  5304. if (!p || !*p)
  5305. goto out;
  5306. if (pci_has_flag(PCI_PROBE_ONLY)) {
  5307. align = 0;
  5308. pr_info_once("PCI: Ignoring requested alignments (PCI_PROBE_ONLY)\n");
  5309. goto out;
  5310. }
  5311. while (*p) {
  5312. count = 0;
  5313. if (sscanf(p, "%d%n", &align_order, &count) == 1 &&
  5314. p[count] == '@') {
  5315. p += count + 1;
  5316. if (align_order > 63) {
  5317. pr_err("PCI: Invalid requested alignment (order %d)\n",
  5318. align_order);
  5319. align_order = PAGE_SHIFT;
  5320. }
  5321. } else {
  5322. align_order = PAGE_SHIFT;
  5323. }
  5324. ret = pci_dev_str_match(dev, p, &p);
  5325. if (ret == 1) {
  5326. *resize = true;
  5327. align = 1ULL << align_order;
  5328. break;
  5329. } else if (ret < 0) {
  5330. pr_err("PCI: Can't parse resource_alignment parameter: %s\n",
  5331. p);
  5332. break;
  5333. }
  5334. if (*p != ';' && *p != ',') {
  5335. /* End of param or invalid format */
  5336. break;
  5337. }
  5338. p++;
  5339. }
  5340. out:
  5341. spin_unlock(&resource_alignment_lock);
  5342. return align;
  5343. }
  5344. static void pci_request_resource_alignment(struct pci_dev *dev, int bar,
  5345. resource_size_t align, bool resize)
  5346. {
  5347. struct resource *r = &dev->resource[bar];
  5348. resource_size_t size;
  5349. if (!(r->flags & IORESOURCE_MEM))
  5350. return;
  5351. if (r->flags & IORESOURCE_PCI_FIXED) {
  5352. pci_info(dev, "BAR%d %pR: ignoring requested alignment %#llx\n",
  5353. bar, r, (unsigned long long)align);
  5354. return;
  5355. }
  5356. size = resource_size(r);
  5357. if (size >= align)
  5358. return;
  5359. /*
  5360. * Increase the alignment of the resource. There are two ways we
  5361. * can do this:
  5362. *
  5363. * 1) Increase the size of the resource. BARs are aligned on their
  5364. * size, so when we reallocate space for this resource, we'll
  5365. * allocate it with the larger alignment. This also prevents
  5366. * assignment of any other BARs inside the alignment region, so
  5367. * if we're requesting page alignment, this means no other BARs
  5368. * will share the page.
  5369. *
  5370. * The disadvantage is that this makes the resource larger than
  5371. * the hardware BAR, which may break drivers that compute things
  5372. * based on the resource size, e.g., to find registers at a
  5373. * fixed offset before the end of the BAR.
  5374. *
  5375. * 2) Retain the resource size, but use IORESOURCE_STARTALIGN and
  5376. * set r->start to the desired alignment. By itself this
  5377. * doesn't prevent other BARs being put inside the alignment
  5378. * region, but if we realign *every* resource of every device in
  5379. * the system, none of them will share an alignment region.
  5380. *
  5381. * When the user has requested alignment for only some devices via
  5382. * the "pci=resource_alignment" argument, "resize" is true and we
  5383. * use the first method. Otherwise we assume we're aligning all
  5384. * devices and we use the second.
  5385. */
  5386. pci_info(dev, "BAR%d %pR: requesting alignment to %#llx\n",
  5387. bar, r, (unsigned long long)align);
  5388. if (resize) {
  5389. r->start = 0;
  5390. r->end = align - 1;
  5391. } else {
  5392. r->flags &= ~IORESOURCE_SIZEALIGN;
  5393. r->flags |= IORESOURCE_STARTALIGN;
  5394. r->start = align;
  5395. r->end = r->start + size - 1;
  5396. }
  5397. r->flags |= IORESOURCE_UNSET;
  5398. }
  5399. /*
  5400. * This function disables memory decoding and releases memory resources
  5401. * of the device specified by kernel's boot parameter 'pci=resource_alignment='.
  5402. * It also rounds up size to specified alignment.
  5403. * Later on, the kernel will assign page-aligned memory resource back
  5404. * to the device.
  5405. */
  5406. void pci_reassigndev_resource_alignment(struct pci_dev *dev)
  5407. {
  5408. int i;
  5409. struct resource *r;
  5410. resource_size_t align;
  5411. u16 command;
  5412. bool resize = false;
  5413. /*
  5414. * VF BARs are read-only zero according to SR-IOV spec r1.1, sec
  5415. * 3.4.1.11. Their resources are allocated from the space
  5416. * described by the VF BARx register in the PF's SR-IOV capability.
  5417. * We can't influence their alignment here.
  5418. */
  5419. if (dev->is_virtfn)
  5420. return;
  5421. /* check if specified PCI is target device to reassign */
  5422. align = pci_specified_resource_alignment(dev, &resize);
  5423. if (!align)
  5424. return;
  5425. if (dev->hdr_type == PCI_HEADER_TYPE_NORMAL &&
  5426. (dev->class >> 8) == PCI_CLASS_BRIDGE_HOST) {
  5427. pci_warn(dev, "Can't reassign resources to host bridge\n");
  5428. return;
  5429. }
  5430. pci_read_config_word(dev, PCI_COMMAND, &command);
  5431. command &= ~PCI_COMMAND_MEMORY;
  5432. pci_write_config_word(dev, PCI_COMMAND, command);
  5433. for (i = 0; i <= PCI_ROM_RESOURCE; i++)
  5434. pci_request_resource_alignment(dev, i, align, resize);
  5435. /*
  5436. * Need to disable bridge's resource window,
  5437. * to enable the kernel to reassign new resource
  5438. * window later on.
  5439. */
  5440. if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
  5441. for (i = PCI_BRIDGE_RESOURCES; i < PCI_NUM_RESOURCES; i++) {
  5442. r = &dev->resource[i];
  5443. if (!(r->flags & IORESOURCE_MEM))
  5444. continue;
  5445. r->flags |= IORESOURCE_UNSET;
  5446. r->end = resource_size(r) - 1;
  5447. r->start = 0;
  5448. }
  5449. pci_disable_bridge_window(dev);
  5450. }
  5451. }
  5452. static ssize_t resource_alignment_show(struct bus_type *bus, char *buf)
  5453. {
  5454. size_t count = 0;
  5455. spin_lock(&resource_alignment_lock);
  5456. if (resource_alignment_param)
  5457. count = scnprintf(buf, PAGE_SIZE, "%s", resource_alignment_param);
  5458. spin_unlock(&resource_alignment_lock);
  5459. /*
  5460. * When set by the command line, resource_alignment_param will not
  5461. * have a trailing line feed, which is ugly. So conditionally add
  5462. * it here.
  5463. */
  5464. if (count >= 2 && buf[count - 2] != '\n' && count < PAGE_SIZE - 1) {
  5465. buf[count - 1] = '\n';
  5466. buf[count++] = 0;
  5467. }
  5468. return count;
  5469. }
  5470. static ssize_t resource_alignment_store(struct bus_type *bus,
  5471. const char *buf, size_t count)
  5472. {
  5473. char *param = kstrndup(buf, count, GFP_KERNEL);
  5474. if (!param)
  5475. return -ENOMEM;
  5476. spin_lock(&resource_alignment_lock);
  5477. kfree(resource_alignment_param);
  5478. resource_alignment_param = param;
  5479. spin_unlock(&resource_alignment_lock);
  5480. return count;
  5481. }
  5482. static BUS_ATTR_RW(resource_alignment);
  5483. static int __init pci_resource_alignment_sysfs_init(void)
  5484. {
  5485. return bus_create_file(&pci_bus_type,
  5486. &bus_attr_resource_alignment);
  5487. }
  5488. late_initcall(pci_resource_alignment_sysfs_init);
  5489. static void pci_no_domains(void)
  5490. {
  5491. #ifdef CONFIG_PCI_DOMAINS
  5492. pci_domains_supported = 0;
  5493. #endif
  5494. }
  5495. #ifdef CONFIG_PCI_DOMAINS_GENERIC
  5496. static atomic_t __domain_nr = ATOMIC_INIT(-1);
  5497. static int pci_get_new_domain_nr(void)
  5498. {
  5499. return atomic_inc_return(&__domain_nr);
  5500. }
  5501. static int of_pci_bus_find_domain_nr(struct device *parent)
  5502. {
  5503. static int use_dt_domains = -1;
  5504. int domain = -1;
  5505. if (parent)
  5506. domain = of_get_pci_domain_nr(parent->of_node);
  5507. /*
  5508. * Check DT domain and use_dt_domains values.
  5509. *
  5510. * If DT domain property is valid (domain >= 0) and
  5511. * use_dt_domains != 0, the DT assignment is valid since this means
  5512. * we have not previously allocated a domain number by using
  5513. * pci_get_new_domain_nr(); we should also update use_dt_domains to
  5514. * 1, to indicate that we have just assigned a domain number from
  5515. * DT.
  5516. *
  5517. * If DT domain property value is not valid (ie domain < 0), and we
  5518. * have not previously assigned a domain number from DT
  5519. * (use_dt_domains != 1) we should assign a domain number by
  5520. * using the:
  5521. *
  5522. * pci_get_new_domain_nr()
  5523. *
  5524. * API and update the use_dt_domains value to keep track of method we
  5525. * are using to assign domain numbers (use_dt_domains = 0).
  5526. *
  5527. * All other combinations imply we have a platform that is trying
  5528. * to mix domain numbers obtained from DT and pci_get_new_domain_nr(),
  5529. * which is a recipe for domain mishandling and it is prevented by
  5530. * invalidating the domain value (domain = -1) and printing a
  5531. * corresponding error.
  5532. */
  5533. if (domain >= 0 && use_dt_domains) {
  5534. use_dt_domains = 1;
  5535. } else if (domain < 0 && use_dt_domains != 1) {
  5536. use_dt_domains = 0;
  5537. domain = pci_get_new_domain_nr();
  5538. } else {
  5539. if (parent)
  5540. pr_err("Node %pOF has ", parent->of_node);
  5541. pr_err("Inconsistent \"linux,pci-domain\" property in DT\n");
  5542. domain = -1;
  5543. }
  5544. return domain;
  5545. }
  5546. int pci_bus_find_domain_nr(struct pci_bus *bus, struct device *parent)
  5547. {
  5548. return acpi_disabled ? of_pci_bus_find_domain_nr(parent) :
  5549. acpi_pci_bus_find_domain_nr(bus);
  5550. }
  5551. #endif
  5552. /**
  5553. * pci_ext_cfg_avail - can we access extended PCI config space?
  5554. *
  5555. * Returns 1 if we can access PCI extended config space (offsets
  5556. * greater than 0xff). This is the default implementation. Architecture
  5557. * implementations can override this.
  5558. */
  5559. int __weak pci_ext_cfg_avail(void)
  5560. {
  5561. return 1;
  5562. }
  5563. void __weak pci_fixup_cardbus(struct pci_bus *bus)
  5564. {
  5565. }
  5566. EXPORT_SYMBOL(pci_fixup_cardbus);
  5567. static int __init pci_setup(char *str)
  5568. {
  5569. while (str) {
  5570. char *k = strchr(str, ',');
  5571. if (k)
  5572. *k++ = 0;
  5573. if (*str && (str = pcibios_setup(str)) && *str) {
  5574. if (!strcmp(str, "nomsi")) {
  5575. pci_no_msi();
  5576. } else if (!strncmp(str, "noats", 5)) {
  5577. pr_info("PCIe: ATS is disabled\n");
  5578. pcie_ats_disabled = true;
  5579. } else if (!strcmp(str, "noaer")) {
  5580. pci_no_aer();
  5581. } else if (!strcmp(str, "earlydump")) {
  5582. pci_early_dump = true;
  5583. } else if (!strncmp(str, "realloc=", 8)) {
  5584. pci_realloc_get_opt(str + 8);
  5585. } else if (!strncmp(str, "realloc", 7)) {
  5586. pci_realloc_get_opt("on");
  5587. } else if (!strcmp(str, "nodomains")) {
  5588. pci_no_domains();
  5589. } else if (!strncmp(str, "noari", 5)) {
  5590. pcie_ari_disabled = true;
  5591. } else if (!strncmp(str, "cbiosize=", 9)) {
  5592. pci_cardbus_io_size = memparse(str + 9, &str);
  5593. } else if (!strncmp(str, "cbmemsize=", 10)) {
  5594. pci_cardbus_mem_size = memparse(str + 10, &str);
  5595. } else if (!strncmp(str, "resource_alignment=", 19)) {
  5596. resource_alignment_param = str + 19;
  5597. } else if (!strncmp(str, "ecrc=", 5)) {
  5598. pcie_ecrc_get_policy(str + 5);
  5599. } else if (!strncmp(str, "hpiosize=", 9)) {
  5600. pci_hotplug_io_size = memparse(str + 9, &str);
  5601. } else if (!strncmp(str, "hpmmiosize=", 11)) {
  5602. pci_hotplug_mmio_size = memparse(str + 11, &str);
  5603. } else if (!strncmp(str, "hpmmioprefsize=", 15)) {
  5604. pci_hotplug_mmio_pref_size = memparse(str + 15, &str);
  5605. } else if (!strncmp(str, "hpmemsize=", 10)) {
  5606. pci_hotplug_mmio_size = memparse(str + 10, &str);
  5607. pci_hotplug_mmio_pref_size = pci_hotplug_mmio_size;
  5608. } else if (!strncmp(str, "hpbussize=", 10)) {
  5609. pci_hotplug_bus_size =
  5610. simple_strtoul(str + 10, &str, 0);
  5611. if (pci_hotplug_bus_size > 0xff)
  5612. pci_hotplug_bus_size = DEFAULT_HOTPLUG_BUS_SIZE;
  5613. } else if (!strncmp(str, "pcie_bus_tune_off", 17)) {
  5614. pcie_bus_config = PCIE_BUS_TUNE_OFF;
  5615. } else if (!strncmp(str, "pcie_bus_safe", 13)) {
  5616. pcie_bus_config = PCIE_BUS_SAFE;
  5617. } else if (!strncmp(str, "pcie_bus_perf", 13)) {
  5618. pcie_bus_config = PCIE_BUS_PERFORMANCE;
  5619. } else if (!strncmp(str, "pcie_bus_peer2peer", 18)) {
  5620. pcie_bus_config = PCIE_BUS_PEER2PEER;
  5621. } else if (!strncmp(str, "pcie_scan_all", 13)) {
  5622. pci_add_flags(PCI_SCAN_ALL_PCIE_DEVS);
  5623. } else if (!strncmp(str, "disable_acs_redir=", 18)) {
  5624. disable_acs_redir_param = str + 18;
  5625. } else {
  5626. pr_err("PCI: Unknown option `%s'\n", str);
  5627. }
  5628. }
  5629. str = k;
  5630. }
  5631. return 0;
  5632. }
  5633. early_param("pci", pci_setup);
  5634. /*
  5635. * 'resource_alignment_param' and 'disable_acs_redir_param' are initialized
  5636. * in pci_setup(), above, to point to data in the __initdata section which
  5637. * will be freed after the init sequence is complete. We can't allocate memory
  5638. * in pci_setup() because some architectures do not have any memory allocation
  5639. * service available during an early_param() call. So we allocate memory and
  5640. * copy the variable here before the init section is freed.
  5641. *
  5642. */
  5643. static int __init pci_realloc_setup_params(void)
  5644. {
  5645. resource_alignment_param = kstrdup(resource_alignment_param,
  5646. GFP_KERNEL);
  5647. disable_acs_redir_param = kstrdup(disable_acs_redir_param, GFP_KERNEL);
  5648. return 0;
  5649. }
  5650. pure_initcall(pci_realloc_setup_params);