page_alloc.c 253 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654565556565657565856595660566156625663566456655666566756685669567056715672567356745675567656775678567956805681568256835684568556865687568856895690569156925693569456955696569756985699570057015702570357045705570657075708570957105711571257135714571557165717571857195720572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759576057615762576357645765576657675768576957705771577257735774577557765777577857795780578157825783578457855786578757885789579057915792579357945795579657975798579958005801580258035804580558065807580858095810581158125813581458155816581758185819582058215822582358245825582658275828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855585658575858585958605861586258635864586558665867586858695870587158725873587458755876587758785879588058815882588358845885588658875888588958905891589258935894589558965897589858995900590159025903590459055906590759085909591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936593759385939594059415942594359445945594659475948594959505951595259535954595559565957595859595960596159625963596459655966596759685969597059715972597359745975597659775978597959805981598259835984598559865987598859895990599159925993599459955996599759985999600060016002600360046005600660076008600960106011601260136014601560166017601860196020602160226023602460256026602760286029603060316032603360346035603660376038603960406041604260436044604560466047604860496050605160526053605460556056605760586059606060616062606360646065606660676068606960706071607260736074607560766077607860796080608160826083608460856086608760886089609060916092609360946095609660976098609961006101610261036104610561066107610861096110611161126113611461156116611761186119612061216122612361246125612661276128612961306131613261336134613561366137613861396140614161426143614461456146614761486149615061516152615361546155615661576158615961606161616261636164616561666167616861696170617161726173617461756176617761786179618061816182618361846185618661876188618961906191619261936194619561966197619861996200620162026203620462056206620762086209621062116212621362146215621662176218621962206221622262236224622562266227622862296230623162326233623462356236623762386239624062416242624362446245624662476248624962506251625262536254625562566257625862596260626162626263626462656266626762686269627062716272627362746275627662776278627962806281628262836284628562866287628862896290629162926293629462956296629762986299630063016302630363046305630663076308630963106311631263136314631563166317631863196320632163226323632463256326632763286329633063316332633363346335633663376338633963406341634263436344634563466347634863496350635163526353635463556356635763586359636063616362636363646365636663676368636963706371637263736374637563766377637863796380638163826383638463856386638763886389639063916392639363946395639663976398639964006401640264036404640564066407640864096410641164126413641464156416641764186419642064216422642364246425642664276428642964306431643264336434643564366437643864396440644164426443644464456446644764486449645064516452645364546455645664576458645964606461646264636464646564666467646864696470647164726473647464756476647764786479648064816482648364846485648664876488648964906491649264936494649564966497649864996500650165026503650465056506650765086509651065116512651365146515651665176518651965206521652265236524652565266527652865296530653165326533653465356536653765386539654065416542654365446545654665476548654965506551655265536554655565566557655865596560656165626563656465656566656765686569657065716572657365746575657665776578657965806581658265836584658565866587658865896590659165926593659465956596659765986599660066016602660366046605660666076608660966106611661266136614661566166617661866196620662166226623662466256626662766286629663066316632663366346635663666376638663966406641664266436644664566466647664866496650665166526653665466556656665766586659666066616662666366646665666666676668666966706671667266736674667566766677667866796680668166826683668466856686668766886689669066916692669366946695669666976698669967006701670267036704670567066707670867096710671167126713671467156716671767186719672067216722672367246725672667276728672967306731673267336734673567366737673867396740674167426743674467456746674767486749675067516752675367546755675667576758675967606761676267636764676567666767676867696770677167726773677467756776677767786779678067816782678367846785678667876788678967906791679267936794679567966797679867996800680168026803680468056806680768086809681068116812681368146815681668176818681968206821682268236824682568266827682868296830683168326833683468356836683768386839684068416842684368446845684668476848684968506851685268536854685568566857685868596860686168626863686468656866686768686869687068716872687368746875687668776878687968806881688268836884688568866887688868896890689168926893689468956896689768986899690069016902690369046905690669076908690969106911691269136914691569166917691869196920692169226923692469256926692769286929693069316932693369346935693669376938693969406941694269436944694569466947694869496950695169526953695469556956695769586959696069616962696369646965696669676968696969706971697269736974697569766977697869796980698169826983698469856986698769886989699069916992699369946995699669976998699970007001700270037004700570067007700870097010701170127013701470157016701770187019702070217022702370247025702670277028702970307031703270337034703570367037703870397040704170427043704470457046704770487049705070517052705370547055705670577058705970607061706270637064706570667067706870697070707170727073707470757076707770787079708070817082708370847085708670877088708970907091709270937094709570967097709870997100710171027103710471057106710771087109711071117112711371147115711671177118711971207121712271237124712571267127712871297130713171327133713471357136713771387139714071417142714371447145714671477148714971507151715271537154715571567157715871597160716171627163716471657166716771687169717071717172717371747175717671777178717971807181718271837184718571867187718871897190719171927193719471957196719771987199720072017202720372047205720672077208720972107211721272137214721572167217721872197220722172227223722472257226722772287229723072317232723372347235723672377238723972407241724272437244724572467247724872497250725172527253725472557256725772587259726072617262726372647265726672677268726972707271727272737274727572767277727872797280728172827283728472857286728772887289729072917292729372947295729672977298729973007301730273037304730573067307730873097310731173127313731473157316731773187319732073217322732373247325732673277328732973307331733273337334733573367337733873397340734173427343734473457346734773487349735073517352735373547355735673577358735973607361736273637364736573667367736873697370737173727373737473757376737773787379738073817382738373847385738673877388738973907391739273937394739573967397739873997400740174027403740474057406740774087409741074117412741374147415741674177418741974207421742274237424742574267427742874297430743174327433743474357436743774387439744074417442744374447445744674477448744974507451745274537454745574567457745874597460746174627463746474657466746774687469747074717472747374747475747674777478747974807481748274837484748574867487748874897490749174927493749474957496749774987499750075017502750375047505750675077508750975107511751275137514751575167517751875197520752175227523752475257526752775287529753075317532753375347535753675377538753975407541754275437544754575467547754875497550755175527553755475557556755775587559756075617562756375647565756675677568756975707571757275737574757575767577757875797580758175827583758475857586758775887589759075917592759375947595759675977598759976007601760276037604760576067607760876097610761176127613761476157616761776187619762076217622762376247625762676277628762976307631763276337634763576367637763876397640764176427643764476457646764776487649765076517652765376547655765676577658765976607661766276637664766576667667766876697670767176727673767476757676767776787679768076817682768376847685768676877688768976907691769276937694769576967697769876997700770177027703770477057706770777087709771077117712771377147715771677177718771977207721772277237724772577267727772877297730773177327733773477357736773777387739774077417742774377447745774677477748774977507751775277537754775577567757775877597760776177627763776477657766776777687769777077717772777377747775777677777778777977807781778277837784778577867787778877897790779177927793779477957796779777987799780078017802780378047805780678077808780978107811781278137814781578167817781878197820782178227823782478257826782778287829783078317832783378347835783678377838783978407841784278437844784578467847784878497850785178527853785478557856785778587859786078617862786378647865786678677868786978707871787278737874787578767877787878797880788178827883788478857886788778887889789078917892789378947895789678977898789979007901790279037904790579067907790879097910791179127913791479157916791779187919792079217922792379247925792679277928792979307931793279337934793579367937793879397940794179427943794479457946794779487949795079517952795379547955795679577958795979607961796279637964796579667967796879697970797179727973797479757976797779787979798079817982798379847985798679877988798979907991799279937994799579967997799879998000800180028003800480058006800780088009801080118012801380148015801680178018801980208021802280238024802580268027802880298030803180328033803480358036803780388039804080418042804380448045804680478048804980508051805280538054805580568057805880598060806180628063806480658066806780688069807080718072807380748075807680778078807980808081808280838084808580868087808880898090809180928093809480958096809780988099810081018102810381048105810681078108810981108111811281138114811581168117811881198120812181228123812481258126812781288129813081318132813381348135813681378138813981408141814281438144814581468147814881498150815181528153815481558156815781588159816081618162816381648165816681678168816981708171817281738174817581768177817881798180818181828183818481858186818781888189819081918192819381948195819681978198819982008201820282038204820582068207820882098210821182128213821482158216821782188219822082218222822382248225822682278228822982308231823282338234823582368237823882398240824182428243824482458246824782488249825082518252825382548255825682578258825982608261826282638264826582668267826882698270827182728273827482758276827782788279828082818282828382848285828682878288828982908291829282938294829582968297829882998300830183028303830483058306830783088309831083118312831383148315831683178318831983208321832283238324832583268327832883298330833183328333833483358336833783388339834083418342834383448345834683478348834983508351835283538354835583568357835883598360836183628363836483658366836783688369837083718372837383748375837683778378837983808381838283838384838583868387838883898390839183928393839483958396839783988399840084018402840384048405840684078408840984108411841284138414841584168417841884198420842184228423842484258426842784288429843084318432843384348435843684378438843984408441844284438444844584468447844884498450845184528453845484558456845784588459846084618462846384648465846684678468846984708471847284738474847584768477847884798480848184828483848484858486848784888489849084918492849384948495849684978498849985008501850285038504850585068507850885098510851185128513851485158516851785188519852085218522852385248525852685278528852985308531853285338534853585368537853885398540854185428543854485458546854785488549855085518552855385548555855685578558855985608561856285638564856585668567856885698570857185728573857485758576857785788579858085818582858385848585858685878588858985908591859285938594859585968597859885998600860186028603860486058606860786088609861086118612861386148615861686178618861986208621862286238624862586268627862886298630863186328633863486358636863786388639864086418642864386448645864686478648864986508651865286538654865586568657865886598660866186628663866486658666866786688669867086718672867386748675867686778678867986808681868286838684868586868687868886898690869186928693869486958696869786988699870087018702870387048705870687078708870987108711871287138714871587168717871887198720872187228723872487258726872787288729873087318732873387348735873687378738873987408741874287438744874587468747874887498750875187528753875487558756875787588759876087618762876387648765876687678768876987708771877287738774877587768777877887798780878187828783878487858786878787888789879087918792879387948795879687978798879988008801880288038804880588068807880888098810881188128813881488158816881788188819882088218822882388248825882688278828882988308831883288338834883588368837883888398840884188428843884488458846884788488849885088518852885388548855885688578858885988608861886288638864886588668867886888698870887188728873887488758876887788788879888088818882888388848885888688878888888988908891889288938894889588968897889888998900890189028903890489058906890789088909891089118912891389148915891689178918891989208921892289238924892589268927892889298930893189328933893489358936893789388939894089418942894389448945894689478948894989508951895289538954895589568957895889598960896189628963896489658966896789688969897089718972897389748975897689778978897989808981898289838984898589868987898889898990899189928993899489958996899789988999900090019002900390049005900690079008900990109011901290139014901590169017901890199020902190229023902490259026902790289029903090319032903390349035903690379038903990409041904290439044904590469047904890499050905190529053905490559056905790589059906090619062906390649065906690679068906990709071907290739074907590769077907890799080908190829083908490859086908790889089909090919092909390949095909690979098909991009101910291039104910591069107910891099110911191129113911491159116911791189119912091219122912391249125912691279128912991309131913291339134913591369137913891399140914191429143914491459146914791489149915091519152915391549155915691579158915991609161916291639164916591669167916891699170
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * linux/mm/page_alloc.c
  4. *
  5. * Manages the free list, the system allocates free pages here.
  6. * Note that kmalloc() lives in slab.c
  7. *
  8. * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
  9. * Swap reorganised 29.12.95, Stephen Tweedie
  10. * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
  11. * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999
  12. * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
  13. * Zone balancing, Kanoj Sarcar, SGI, Jan 2000
  14. * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002
  15. * (lots of bits borrowed from Ingo Molnar & Andrew Morton)
  16. */
  17. #include <linux/stddef.h>
  18. #include <linux/mm.h>
  19. #include <linux/highmem.h>
  20. #include <linux/swap.h>
  21. #include <linux/interrupt.h>
  22. #include <linux/pagemap.h>
  23. #include <linux/jiffies.h>
  24. #include <linux/memblock.h>
  25. #include <linux/compiler.h>
  26. #include <linux/kernel.h>
  27. #include <linux/kasan.h>
  28. #include <linux/module.h>
  29. #include <linux/suspend.h>
  30. #include <linux/pagevec.h>
  31. #include <linux/blkdev.h>
  32. #include <linux/slab.h>
  33. #include <linux/ratelimit.h>
  34. #include <linux/oom.h>
  35. #include <linux/topology.h>
  36. #include <linux/sysctl.h>
  37. #include <linux/cpu.h>
  38. #include <linux/cpuset.h>
  39. #include <linux/memory_hotplug.h>
  40. #include <linux/nodemask.h>
  41. #include <linux/vmalloc.h>
  42. #include <linux/vmstat.h>
  43. #include <linux/mempolicy.h>
  44. #include <linux/memremap.h>
  45. #include <linux/stop_machine.h>
  46. #include <linux/random.h>
  47. #include <linux/sort.h>
  48. #include <linux/pfn.h>
  49. #include <linux/backing-dev.h>
  50. #include <linux/fault-inject.h>
  51. #include <linux/page-isolation.h>
  52. #include <linux/debugobjects.h>
  53. #include <linux/kmemleak.h>
  54. #include <linux/compaction.h>
  55. #include <trace/events/kmem.h>
  56. #include <trace/events/oom.h>
  57. #include <linux/prefetch.h>
  58. #include <linux/mm_inline.h>
  59. #include <linux/migrate.h>
  60. #include <linux/hugetlb.h>
  61. #include <linux/sched/rt.h>
  62. #include <linux/sched/mm.h>
  63. #include <linux/page_owner.h>
  64. #include <linux/page_pinner.h>
  65. #include <linux/kthread.h>
  66. #include <linux/memcontrol.h>
  67. #include <linux/ftrace.h>
  68. #include <linux/lockdep.h>
  69. #include <linux/nmi.h>
  70. #include <linux/psi.h>
  71. #include <linux/padata.h>
  72. #include <linux/khugepaged.h>
  73. #include <trace/hooks/mm.h>
  74. #include <asm/sections.h>
  75. #include <asm/tlbflush.h>
  76. #include <asm/div64.h>
  77. #include "internal.h"
  78. #include "shuffle.h"
  79. #include "page_reporting.h"
  80. /* Free Page Internal flags: for internal, non-pcp variants of free_pages(). */
  81. typedef int __bitwise fpi_t;
  82. /* No special request */
  83. #define FPI_NONE ((__force fpi_t)0)
  84. /*
  85. * Skip free page reporting notification for the (possibly merged) page.
  86. * This does not hinder free page reporting from grabbing the page,
  87. * reporting it and marking it "reported" - it only skips notifying
  88. * the free page reporting infrastructure about a newly freed page. For
  89. * example, used when temporarily pulling a page from a freelist and
  90. * putting it back unmodified.
  91. */
  92. #define FPI_SKIP_REPORT_NOTIFY ((__force fpi_t)BIT(0))
  93. /*
  94. * Place the (possibly merged) page to the tail of the freelist. Will ignore
  95. * page shuffling (relevant code - e.g., memory onlining - is expected to
  96. * shuffle the whole zone).
  97. *
  98. * Note: No code should rely on this flag for correctness - it's purely
  99. * to allow for optimizations when handing back either fresh pages
  100. * (memory onlining) or untouched pages (page isolation, free page
  101. * reporting).
  102. */
  103. #define FPI_TO_TAIL ((__force fpi_t)BIT(1))
  104. /*
  105. * Don't poison memory with KASAN (only for the tag-based modes).
  106. * During boot, all non-reserved memblock memory is exposed to page_alloc.
  107. * Poisoning all that memory lengthens boot time, especially on systems with
  108. * large amount of RAM. This flag is used to skip that poisoning.
  109. * This is only done for the tag-based KASAN modes, as those are able to
  110. * detect memory corruptions with the memory tags assigned by default.
  111. * All memory allocated normally after boot gets poisoned as usual.
  112. */
  113. #define FPI_SKIP_KASAN_POISON ((__force fpi_t)BIT(2))
  114. /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */
  115. static DEFINE_MUTEX(pcp_batch_high_lock);
  116. #define MIN_PERCPU_PAGELIST_FRACTION (8)
  117. #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
  118. DEFINE_PER_CPU(int, numa_node);
  119. EXPORT_PER_CPU_SYMBOL(numa_node);
  120. #endif
  121. DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key);
  122. #ifdef CONFIG_HAVE_MEMORYLESS_NODES
  123. /*
  124. * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
  125. * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.
  126. * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem()
  127. * defined in <linux/topology.h>.
  128. */
  129. DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */
  130. EXPORT_PER_CPU_SYMBOL(_numa_mem_);
  131. #endif
  132. /* work_structs for global per-cpu drains */
  133. struct pcpu_drain {
  134. struct zone *zone;
  135. struct work_struct work;
  136. };
  137. static DEFINE_MUTEX(pcpu_drain_mutex);
  138. static DEFINE_PER_CPU(struct pcpu_drain, pcpu_drain);
  139. #ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY
  140. volatile unsigned long latent_entropy __latent_entropy;
  141. EXPORT_SYMBOL(latent_entropy);
  142. #endif
  143. /*
  144. * Array of node states.
  145. */
  146. nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
  147. [N_POSSIBLE] = NODE_MASK_ALL,
  148. [N_ONLINE] = { { [0] = 1UL } },
  149. #ifndef CONFIG_NUMA
  150. [N_NORMAL_MEMORY] = { { [0] = 1UL } },
  151. #ifdef CONFIG_HIGHMEM
  152. [N_HIGH_MEMORY] = { { [0] = 1UL } },
  153. #endif
  154. [N_MEMORY] = { { [0] = 1UL } },
  155. [N_CPU] = { { [0] = 1UL } },
  156. #endif /* NUMA */
  157. };
  158. EXPORT_SYMBOL(node_states);
  159. atomic_long_t _totalram_pages __read_mostly;
  160. EXPORT_SYMBOL(_totalram_pages);
  161. unsigned long totalreserve_pages __read_mostly;
  162. unsigned long totalcma_pages __read_mostly;
  163. int percpu_pagelist_fraction;
  164. gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
  165. DEFINE_STATIC_KEY_FALSE(init_on_alloc);
  166. EXPORT_SYMBOL(init_on_alloc);
  167. DEFINE_STATIC_KEY_FALSE(init_on_free);
  168. EXPORT_SYMBOL(init_on_free);
  169. static bool _init_on_alloc_enabled_early __read_mostly
  170. = IS_ENABLED(CONFIG_INIT_ON_ALLOC_DEFAULT_ON);
  171. static int __init early_init_on_alloc(char *buf)
  172. {
  173. return kstrtobool(buf, &_init_on_alloc_enabled_early);
  174. }
  175. early_param("init_on_alloc", early_init_on_alloc);
  176. static bool _init_on_free_enabled_early __read_mostly
  177. = IS_ENABLED(CONFIG_INIT_ON_FREE_DEFAULT_ON);
  178. static int __init early_init_on_free(char *buf)
  179. {
  180. return kstrtobool(buf, &_init_on_free_enabled_early);
  181. }
  182. early_param("init_on_free", early_init_on_free);
  183. /*
  184. * A cached value of the page's pageblock's migratetype, used when the page is
  185. * put on a pcplist. Used to avoid the pageblock migratetype lookup when
  186. * freeing from pcplists in most cases, at the cost of possibly becoming stale.
  187. * Also the migratetype set in the page does not necessarily match the pcplist
  188. * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any
  189. * other index - this ensures that it will be put on the correct CMA freelist.
  190. */
  191. static inline int get_pcppage_migratetype(struct page *page)
  192. {
  193. return page->index;
  194. }
  195. static inline void set_pcppage_migratetype(struct page *page, int migratetype)
  196. {
  197. page->index = migratetype;
  198. }
  199. #ifdef CONFIG_PM_SLEEP
  200. /*
  201. * The following functions are used by the suspend/hibernate code to temporarily
  202. * change gfp_allowed_mask in order to avoid using I/O during memory allocations
  203. * while devices are suspended. To avoid races with the suspend/hibernate code,
  204. * they should always be called with system_transition_mutex held
  205. * (gfp_allowed_mask also should only be modified with system_transition_mutex
  206. * held, unless the suspend/hibernate code is guaranteed not to run in parallel
  207. * with that modification).
  208. */
  209. static gfp_t saved_gfp_mask;
  210. void pm_restore_gfp_mask(void)
  211. {
  212. WARN_ON(!mutex_is_locked(&system_transition_mutex));
  213. if (saved_gfp_mask) {
  214. gfp_allowed_mask = saved_gfp_mask;
  215. saved_gfp_mask = 0;
  216. }
  217. }
  218. void pm_restrict_gfp_mask(void)
  219. {
  220. WARN_ON(!mutex_is_locked(&system_transition_mutex));
  221. WARN_ON(saved_gfp_mask);
  222. saved_gfp_mask = gfp_allowed_mask;
  223. gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS);
  224. }
  225. bool pm_suspended_storage(void)
  226. {
  227. if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS))
  228. return false;
  229. return true;
  230. }
  231. #endif /* CONFIG_PM_SLEEP */
  232. #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
  233. unsigned int pageblock_order __read_mostly;
  234. #endif
  235. static void __free_pages_ok(struct page *page, unsigned int order,
  236. fpi_t fpi_flags);
  237. /*
  238. * results with 256, 32 in the lowmem_reserve sysctl:
  239. * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high)
  240. * 1G machine -> (16M dma, 784M normal, 224M high)
  241. * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA
  242. * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL
  243. * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA
  244. *
  245. * TBD: should special case ZONE_DMA32 machines here - in those we normally
  246. * don't need any ZONE_NORMAL reservation
  247. */
  248. int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = {
  249. #ifdef CONFIG_ZONE_DMA
  250. [ZONE_DMA] = 256,
  251. #endif
  252. #ifdef CONFIG_ZONE_DMA32
  253. [ZONE_DMA32] = 256,
  254. #endif
  255. [ZONE_NORMAL] = 32,
  256. #ifdef CONFIG_HIGHMEM
  257. [ZONE_HIGHMEM] = 0,
  258. #endif
  259. [ZONE_MOVABLE] = 0,
  260. };
  261. static char * const zone_names[MAX_NR_ZONES] = {
  262. #ifdef CONFIG_ZONE_DMA
  263. "DMA",
  264. #endif
  265. #ifdef CONFIG_ZONE_DMA32
  266. "DMA32",
  267. #endif
  268. "Normal",
  269. #ifdef CONFIG_HIGHMEM
  270. "HighMem",
  271. #endif
  272. "Movable",
  273. #ifdef CONFIG_ZONE_DEVICE
  274. "Device",
  275. #endif
  276. };
  277. const char * const migratetype_names[MIGRATE_TYPES] = {
  278. "Unmovable",
  279. "Movable",
  280. "Reclaimable",
  281. #ifdef CONFIG_CMA
  282. "CMA",
  283. #endif
  284. "HighAtomic",
  285. #ifdef CONFIG_MEMORY_ISOLATION
  286. "Isolate",
  287. #endif
  288. };
  289. compound_page_dtor * const compound_page_dtors[NR_COMPOUND_DTORS] = {
  290. [NULL_COMPOUND_DTOR] = NULL,
  291. [COMPOUND_PAGE_DTOR] = free_compound_page,
  292. #ifdef CONFIG_HUGETLB_PAGE
  293. [HUGETLB_PAGE_DTOR] = free_huge_page,
  294. #endif
  295. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  296. [TRANSHUGE_PAGE_DTOR] = free_transhuge_page,
  297. #endif
  298. };
  299. /*
  300. * Try to keep at least this much lowmem free. Do not allow normal
  301. * allocations below this point, only high priority ones. Automatically
  302. * tuned according to the amount of memory in the system.
  303. */
  304. int min_free_kbytes = 1024;
  305. int user_min_free_kbytes = -1;
  306. #ifdef CONFIG_DISCONTIGMEM
  307. /*
  308. * DiscontigMem defines memory ranges as separate pg_data_t even if the ranges
  309. * are not on separate NUMA nodes. Functionally this works but with
  310. * watermark_boost_factor, it can reclaim prematurely as the ranges can be
  311. * quite small. By default, do not boost watermarks on discontigmem as in
  312. * many cases very high-order allocations like THP are likely to be
  313. * unsupported and the premature reclaim offsets the advantage of long-term
  314. * fragmentation avoidance.
  315. */
  316. int watermark_boost_factor __read_mostly;
  317. #else
  318. int watermark_boost_factor __read_mostly = 15000;
  319. #endif
  320. int watermark_scale_factor = 10;
  321. /*
  322. * Extra memory for the system to try freeing. Used to temporarily
  323. * free memory, to make space for new workloads. Anyone can allocate
  324. * down to the min watermarks controlled by min_free_kbytes above.
  325. */
  326. int extra_free_kbytes = 0;
  327. static unsigned long nr_kernel_pages __initdata;
  328. static unsigned long nr_all_pages __initdata;
  329. static unsigned long dma_reserve __initdata;
  330. static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata;
  331. static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata;
  332. static unsigned long required_kernelcore __initdata;
  333. static unsigned long required_kernelcore_percent __initdata;
  334. static unsigned long required_movablecore __initdata;
  335. static unsigned long required_movablecore_percent __initdata;
  336. static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata;
  337. static bool mirrored_kernelcore __meminitdata;
  338. /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */
  339. int movable_zone;
  340. EXPORT_SYMBOL(movable_zone);
  341. #if MAX_NUMNODES > 1
  342. unsigned int nr_node_ids __read_mostly = MAX_NUMNODES;
  343. unsigned int nr_online_nodes __read_mostly = 1;
  344. EXPORT_SYMBOL(nr_node_ids);
  345. EXPORT_SYMBOL(nr_online_nodes);
  346. #endif
  347. int page_group_by_mobility_disabled __read_mostly;
  348. #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
  349. /*
  350. * During boot we initialize deferred pages on-demand, as needed, but once
  351. * page_alloc_init_late() has finished, the deferred pages are all initialized,
  352. * and we can permanently disable that path.
  353. */
  354. static DEFINE_STATIC_KEY_TRUE(deferred_pages);
  355. /*
  356. * Calling kasan_poison_pages() only after deferred memory initialization
  357. * has completed. Poisoning pages during deferred memory init will greatly
  358. * lengthen the process and cause problem in large memory systems as the
  359. * deferred pages initialization is done with interrupt disabled.
  360. *
  361. * Assuming that there will be no reference to those newly initialized
  362. * pages before they are ever allocated, this should have no effect on
  363. * KASAN memory tracking as the poison will be properly inserted at page
  364. * allocation time. The only corner case is when pages are allocated by
  365. * on-demand allocation and then freed again before the deferred pages
  366. * initialization is done, but this is not likely to happen.
  367. */
  368. static inline bool should_skip_kasan_poison(struct page *page, fpi_t fpi_flags)
  369. {
  370. return static_branch_unlikely(&deferred_pages) ||
  371. (!IS_ENABLED(CONFIG_KASAN_GENERIC) &&
  372. (fpi_flags & FPI_SKIP_KASAN_POISON)) ||
  373. PageSkipKASanPoison(page);
  374. }
  375. /* Returns true if the struct page for the pfn is uninitialised */
  376. static inline bool __meminit early_page_uninitialised(unsigned long pfn)
  377. {
  378. int nid = early_pfn_to_nid(pfn);
  379. if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn)
  380. return true;
  381. return false;
  382. }
  383. /*
  384. * Returns true when the remaining initialisation should be deferred until
  385. * later in the boot cycle when it can be parallelised.
  386. */
  387. static bool __meminit
  388. defer_init(int nid, unsigned long pfn, unsigned long end_pfn)
  389. {
  390. static unsigned long prev_end_pfn, nr_initialised;
  391. /*
  392. * prev_end_pfn static that contains the end of previous zone
  393. * No need to protect because called very early in boot before smp_init.
  394. */
  395. if (prev_end_pfn != end_pfn) {
  396. prev_end_pfn = end_pfn;
  397. nr_initialised = 0;
  398. }
  399. /* Always populate low zones for address-constrained allocations */
  400. if (end_pfn < pgdat_end_pfn(NODE_DATA(nid)))
  401. return false;
  402. if (NODE_DATA(nid)->first_deferred_pfn != ULONG_MAX)
  403. return true;
  404. /*
  405. * We start only with one section of pages, more pages are added as
  406. * needed until the rest of deferred pages are initialized.
  407. */
  408. nr_initialised++;
  409. if ((nr_initialised > PAGES_PER_SECTION) &&
  410. (pfn & (PAGES_PER_SECTION - 1)) == 0) {
  411. NODE_DATA(nid)->first_deferred_pfn = pfn;
  412. return true;
  413. }
  414. return false;
  415. }
  416. #else
  417. static inline bool should_skip_kasan_poison(struct page *page, fpi_t fpi_flags)
  418. {
  419. return (!IS_ENABLED(CONFIG_KASAN_GENERIC) &&
  420. (fpi_flags & FPI_SKIP_KASAN_POISON)) ||
  421. PageSkipKASanPoison(page);
  422. }
  423. static inline bool early_page_uninitialised(unsigned long pfn)
  424. {
  425. return false;
  426. }
  427. static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn)
  428. {
  429. return false;
  430. }
  431. #endif
  432. /* Return a pointer to the bitmap storing bits affecting a block of pages */
  433. static inline unsigned long *get_pageblock_bitmap(struct page *page,
  434. unsigned long pfn)
  435. {
  436. #ifdef CONFIG_SPARSEMEM
  437. return section_to_usemap(__pfn_to_section(pfn));
  438. #else
  439. return page_zone(page)->pageblock_flags;
  440. #endif /* CONFIG_SPARSEMEM */
  441. }
  442. static inline int pfn_to_bitidx(struct page *page, unsigned long pfn)
  443. {
  444. #ifdef CONFIG_SPARSEMEM
  445. pfn &= (PAGES_PER_SECTION-1);
  446. #else
  447. pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages);
  448. #endif /* CONFIG_SPARSEMEM */
  449. return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
  450. }
  451. /**
  452. * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages
  453. * @page: The page within the block of interest
  454. * @pfn: The target page frame number
  455. * @mask: mask of bits that the caller is interested in
  456. *
  457. * Return: pageblock_bits flags
  458. */
  459. static __always_inline
  460. unsigned long __get_pfnblock_flags_mask(struct page *page,
  461. unsigned long pfn,
  462. unsigned long mask)
  463. {
  464. unsigned long *bitmap;
  465. unsigned long bitidx, word_bitidx;
  466. unsigned long word;
  467. bitmap = get_pageblock_bitmap(page, pfn);
  468. bitidx = pfn_to_bitidx(page, pfn);
  469. word_bitidx = bitidx / BITS_PER_LONG;
  470. bitidx &= (BITS_PER_LONG-1);
  471. word = bitmap[word_bitidx];
  472. return (word >> bitidx) & mask;
  473. }
  474. unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn,
  475. unsigned long mask)
  476. {
  477. return __get_pfnblock_flags_mask(page, pfn, mask);
  478. }
  479. EXPORT_SYMBOL_GPL(get_pfnblock_flags_mask);
  480. int isolate_anon_lru_page(struct page *page)
  481. {
  482. int ret;
  483. if (!PageLRU(page) || !PageAnon(page))
  484. return -EINVAL;
  485. if (!get_page_unless_zero(page))
  486. return -EINVAL;
  487. ret = isolate_lru_page(page);
  488. put_page(page);
  489. return ret;
  490. }
  491. EXPORT_SYMBOL_GPL(isolate_anon_lru_page);
  492. static __always_inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn)
  493. {
  494. return __get_pfnblock_flags_mask(page, pfn, MIGRATETYPE_MASK);
  495. }
  496. /**
  497. * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages
  498. * @page: The page within the block of interest
  499. * @flags: The flags to set
  500. * @pfn: The target page frame number
  501. * @mask: mask of bits that the caller is interested in
  502. */
  503. void set_pfnblock_flags_mask(struct page *page, unsigned long flags,
  504. unsigned long pfn,
  505. unsigned long mask)
  506. {
  507. unsigned long *bitmap;
  508. unsigned long bitidx, word_bitidx;
  509. unsigned long old_word, word;
  510. BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);
  511. BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits));
  512. bitmap = get_pageblock_bitmap(page, pfn);
  513. bitidx = pfn_to_bitidx(page, pfn);
  514. word_bitidx = bitidx / BITS_PER_LONG;
  515. bitidx &= (BITS_PER_LONG-1);
  516. VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);
  517. mask <<= bitidx;
  518. flags <<= bitidx;
  519. word = READ_ONCE(bitmap[word_bitidx]);
  520. for (;;) {
  521. old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags);
  522. if (word == old_word)
  523. break;
  524. word = old_word;
  525. }
  526. }
  527. void set_pageblock_migratetype(struct page *page, int migratetype)
  528. {
  529. if (unlikely(page_group_by_mobility_disabled &&
  530. migratetype < MIGRATE_PCPTYPES))
  531. migratetype = MIGRATE_UNMOVABLE;
  532. set_pfnblock_flags_mask(page, (unsigned long)migratetype,
  533. page_to_pfn(page), MIGRATETYPE_MASK);
  534. }
  535. #ifdef CONFIG_DEBUG_VM
  536. static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
  537. {
  538. int ret = 0;
  539. unsigned seq;
  540. unsigned long pfn = page_to_pfn(page);
  541. unsigned long sp, start_pfn;
  542. do {
  543. seq = zone_span_seqbegin(zone);
  544. start_pfn = zone->zone_start_pfn;
  545. sp = zone->spanned_pages;
  546. if (!zone_spans_pfn(zone, pfn))
  547. ret = 1;
  548. } while (zone_span_seqretry(zone, seq));
  549. if (ret)
  550. pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n",
  551. pfn, zone_to_nid(zone), zone->name,
  552. start_pfn, start_pfn + sp);
  553. return ret;
  554. }
  555. static int page_is_consistent(struct zone *zone, struct page *page)
  556. {
  557. if (!pfn_valid_within(page_to_pfn(page)))
  558. return 0;
  559. if (zone != page_zone(page))
  560. return 0;
  561. return 1;
  562. }
  563. /*
  564. * Temporary debugging check for pages not lying within a given zone.
  565. */
  566. static int __maybe_unused bad_range(struct zone *zone, struct page *page)
  567. {
  568. if (page_outside_zone_boundaries(zone, page))
  569. return 1;
  570. if (!page_is_consistent(zone, page))
  571. return 1;
  572. return 0;
  573. }
  574. #else
  575. static inline int __maybe_unused bad_range(struct zone *zone, struct page *page)
  576. {
  577. return 0;
  578. }
  579. #endif
  580. static void bad_page(struct page *page, const char *reason)
  581. {
  582. static unsigned long resume;
  583. static unsigned long nr_shown;
  584. static unsigned long nr_unshown;
  585. /*
  586. * Allow a burst of 60 reports, then keep quiet for that minute;
  587. * or allow a steady drip of one report per second.
  588. */
  589. if (nr_shown == 60) {
  590. if (time_before(jiffies, resume)) {
  591. nr_unshown++;
  592. goto out;
  593. }
  594. if (nr_unshown) {
  595. pr_alert(
  596. "BUG: Bad page state: %lu messages suppressed\n",
  597. nr_unshown);
  598. nr_unshown = 0;
  599. }
  600. nr_shown = 0;
  601. }
  602. if (nr_shown++ == 0)
  603. resume = jiffies + 60 * HZ;
  604. pr_alert("BUG: Bad page state in process %s pfn:%05lx\n",
  605. current->comm, page_to_pfn(page));
  606. __dump_page(page, reason);
  607. dump_page_owner(page);
  608. print_modules();
  609. dump_stack();
  610. out:
  611. /* Leave bad fields for debug, except PageBuddy could make trouble */
  612. page_mapcount_reset(page); /* remove PageBuddy */
  613. add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
  614. }
  615. /*
  616. * Higher-order pages are called "compound pages". They are structured thusly:
  617. *
  618. * The first PAGE_SIZE page is called the "head page" and have PG_head set.
  619. *
  620. * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded
  621. * in bit 0 of page->compound_head. The rest of bits is pointer to head page.
  622. *
  623. * The first tail page's ->compound_dtor holds the offset in array of compound
  624. * page destructors. See compound_page_dtors.
  625. *
  626. * The first tail page's ->compound_order holds the order of allocation.
  627. * This usage means that zero-order pages may not be compound.
  628. */
  629. void free_compound_page(struct page *page)
  630. {
  631. mem_cgroup_uncharge(page);
  632. __free_pages_ok(page, compound_order(page), FPI_NONE);
  633. }
  634. void prep_compound_page(struct page *page, unsigned int order)
  635. {
  636. int i;
  637. int nr_pages = 1 << order;
  638. __SetPageHead(page);
  639. for (i = 1; i < nr_pages; i++) {
  640. struct page *p = page + i;
  641. set_page_count(p, 0);
  642. p->mapping = TAIL_MAPPING;
  643. set_compound_head(p, page);
  644. }
  645. set_compound_page_dtor(page, COMPOUND_PAGE_DTOR);
  646. set_compound_order(page, order);
  647. atomic_set(compound_mapcount_ptr(page), -1);
  648. if (hpage_pincount_available(page))
  649. atomic_set(compound_pincount_ptr(page), 0);
  650. }
  651. #ifdef CONFIG_DEBUG_PAGEALLOC
  652. unsigned int _debug_guardpage_minorder;
  653. bool _debug_pagealloc_enabled_early __read_mostly
  654. = IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT);
  655. EXPORT_SYMBOL(_debug_pagealloc_enabled_early);
  656. DEFINE_STATIC_KEY_FALSE(_debug_pagealloc_enabled);
  657. EXPORT_SYMBOL(_debug_pagealloc_enabled);
  658. DEFINE_STATIC_KEY_FALSE(_debug_guardpage_enabled);
  659. static int __init early_debug_pagealloc(char *buf)
  660. {
  661. return kstrtobool(buf, &_debug_pagealloc_enabled_early);
  662. }
  663. early_param("debug_pagealloc", early_debug_pagealloc);
  664. static int __init debug_guardpage_minorder_setup(char *buf)
  665. {
  666. unsigned long res;
  667. if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) {
  668. pr_err("Bad debug_guardpage_minorder value\n");
  669. return 0;
  670. }
  671. _debug_guardpage_minorder = res;
  672. pr_info("Setting debug_guardpage_minorder to %lu\n", res);
  673. return 0;
  674. }
  675. early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup);
  676. static inline bool set_page_guard(struct zone *zone, struct page *page,
  677. unsigned int order, int migratetype)
  678. {
  679. if (!debug_guardpage_enabled())
  680. return false;
  681. if (order >= debug_guardpage_minorder())
  682. return false;
  683. __SetPageGuard(page);
  684. INIT_LIST_HEAD(&page->lru);
  685. set_page_private(page, order);
  686. /* Guard pages are not available for any usage */
  687. __mod_zone_freepage_state(zone, -(1 << order), migratetype);
  688. return true;
  689. }
  690. static inline void clear_page_guard(struct zone *zone, struct page *page,
  691. unsigned int order, int migratetype)
  692. {
  693. if (!debug_guardpage_enabled())
  694. return;
  695. __ClearPageGuard(page);
  696. set_page_private(page, 0);
  697. if (!is_migrate_isolate(migratetype))
  698. __mod_zone_freepage_state(zone, (1 << order), migratetype);
  699. }
  700. #else
  701. static inline bool set_page_guard(struct zone *zone, struct page *page,
  702. unsigned int order, int migratetype) { return false; }
  703. static inline void clear_page_guard(struct zone *zone, struct page *page,
  704. unsigned int order, int migratetype) {}
  705. #endif
  706. /*
  707. * Enable static keys related to various memory debugging and hardening options.
  708. * Some override others, and depend on early params that are evaluated in the
  709. * order of appearance. So we need to first gather the full picture of what was
  710. * enabled, and then make decisions.
  711. */
  712. void init_mem_debugging_and_hardening(void)
  713. {
  714. bool page_poisoning_requested = false;
  715. #ifdef CONFIG_PAGE_POISONING
  716. /*
  717. * Page poisoning is debug page alloc for some arches. If
  718. * either of those options are enabled, enable poisoning.
  719. */
  720. if (page_poisoning_enabled() ||
  721. (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC) &&
  722. debug_pagealloc_enabled())) {
  723. static_branch_enable(&_page_poisoning_enabled);
  724. page_poisoning_requested = true;
  725. }
  726. #endif
  727. if (_init_on_alloc_enabled_early) {
  728. if (page_poisoning_requested)
  729. pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, "
  730. "will take precedence over init_on_alloc\n");
  731. else
  732. static_branch_enable(&init_on_alloc);
  733. }
  734. if (_init_on_free_enabled_early) {
  735. if (page_poisoning_requested)
  736. pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, "
  737. "will take precedence over init_on_free\n");
  738. else
  739. static_branch_enable(&init_on_free);
  740. }
  741. #ifdef CONFIG_DEBUG_PAGEALLOC
  742. if (!debug_pagealloc_enabled())
  743. return;
  744. static_branch_enable(&_debug_pagealloc_enabled);
  745. if (!debug_guardpage_minorder())
  746. return;
  747. static_branch_enable(&_debug_guardpage_enabled);
  748. #endif
  749. }
  750. static inline void set_buddy_order(struct page *page, unsigned int order)
  751. {
  752. set_page_private(page, order);
  753. __SetPageBuddy(page);
  754. }
  755. /*
  756. * This function checks whether a page is free && is the buddy
  757. * we can coalesce a page and its buddy if
  758. * (a) the buddy is not in a hole (check before calling!) &&
  759. * (b) the buddy is in the buddy system &&
  760. * (c) a page and its buddy have the same order &&
  761. * (d) a page and its buddy are in the same zone.
  762. *
  763. * For recording whether a page is in the buddy system, we set PageBuddy.
  764. * Setting, clearing, and testing PageBuddy is serialized by zone->lock.
  765. *
  766. * For recording page's order, we use page_private(page).
  767. */
  768. static inline bool page_is_buddy(struct page *page, struct page *buddy,
  769. unsigned int order)
  770. {
  771. if (!page_is_guard(buddy) && !PageBuddy(buddy))
  772. return false;
  773. if (buddy_order(buddy) != order)
  774. return false;
  775. /*
  776. * zone check is done late to avoid uselessly calculating
  777. * zone/node ids for pages that could never merge.
  778. */
  779. if (page_zone_id(page) != page_zone_id(buddy))
  780. return false;
  781. VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
  782. return true;
  783. }
  784. #ifdef CONFIG_COMPACTION
  785. static inline struct capture_control *task_capc(struct zone *zone)
  786. {
  787. struct capture_control *capc = current->capture_control;
  788. return unlikely(capc) &&
  789. !(current->flags & PF_KTHREAD) &&
  790. !capc->page &&
  791. capc->cc->zone == zone ? capc : NULL;
  792. }
  793. static inline bool
  794. compaction_capture(struct capture_control *capc, struct page *page,
  795. int order, int migratetype)
  796. {
  797. if (!capc || order != capc->cc->order)
  798. return false;
  799. /* Do not accidentally pollute CMA or isolated regions*/
  800. if (is_migrate_cma(migratetype) ||
  801. is_migrate_isolate(migratetype))
  802. return false;
  803. /*
  804. * Do not let lower order allocations polluate a movable pageblock.
  805. * This might let an unmovable request use a reclaimable pageblock
  806. * and vice-versa but no more than normal fallback logic which can
  807. * have trouble finding a high-order free page.
  808. */
  809. if (order < pageblock_order && migratetype == MIGRATE_MOVABLE)
  810. return false;
  811. capc->page = page;
  812. return true;
  813. }
  814. #else
  815. static inline struct capture_control *task_capc(struct zone *zone)
  816. {
  817. return NULL;
  818. }
  819. static inline bool
  820. compaction_capture(struct capture_control *capc, struct page *page,
  821. int order, int migratetype)
  822. {
  823. return false;
  824. }
  825. #endif /* CONFIG_COMPACTION */
  826. /* Used for pages not on another list */
  827. static inline void add_to_free_list(struct page *page, struct zone *zone,
  828. unsigned int order, int migratetype)
  829. {
  830. struct free_area *area = &zone->free_area[order];
  831. list_add(&page->lru, &area->free_list[migratetype]);
  832. area->nr_free++;
  833. }
  834. /* Used for pages not on another list */
  835. static inline void add_to_free_list_tail(struct page *page, struct zone *zone,
  836. unsigned int order, int migratetype)
  837. {
  838. struct free_area *area = &zone->free_area[order];
  839. list_add_tail(&page->lru, &area->free_list[migratetype]);
  840. area->nr_free++;
  841. }
  842. /*
  843. * Used for pages which are on another list. Move the pages to the tail
  844. * of the list - so the moved pages won't immediately be considered for
  845. * allocation again (e.g., optimization for memory onlining).
  846. */
  847. static inline void move_to_free_list(struct page *page, struct zone *zone,
  848. unsigned int order, int migratetype)
  849. {
  850. struct free_area *area = &zone->free_area[order];
  851. list_move_tail(&page->lru, &area->free_list[migratetype]);
  852. }
  853. static inline void del_page_from_free_list(struct page *page, struct zone *zone,
  854. unsigned int order)
  855. {
  856. /* clear reported state and update reported page count */
  857. if (page_reported(page))
  858. __ClearPageReported(page);
  859. list_del(&page->lru);
  860. __ClearPageBuddy(page);
  861. set_page_private(page, 0);
  862. zone->free_area[order].nr_free--;
  863. }
  864. /*
  865. * If this is not the largest possible page, check if the buddy
  866. * of the next-highest order is free. If it is, it's possible
  867. * that pages are being freed that will coalesce soon. In case,
  868. * that is happening, add the free page to the tail of the list
  869. * so it's less likely to be used soon and more likely to be merged
  870. * as a higher order page
  871. */
  872. static inline bool
  873. buddy_merge_likely(unsigned long pfn, unsigned long buddy_pfn,
  874. struct page *page, unsigned int order)
  875. {
  876. struct page *higher_page, *higher_buddy;
  877. unsigned long combined_pfn;
  878. if (order >= MAX_ORDER - 2)
  879. return false;
  880. if (!pfn_valid_within(buddy_pfn))
  881. return false;
  882. combined_pfn = buddy_pfn & pfn;
  883. higher_page = page + (combined_pfn - pfn);
  884. buddy_pfn = __find_buddy_pfn(combined_pfn, order + 1);
  885. higher_buddy = higher_page + (buddy_pfn - combined_pfn);
  886. return pfn_valid_within(buddy_pfn) &&
  887. page_is_buddy(higher_page, higher_buddy, order + 1);
  888. }
  889. /*
  890. * Freeing function for a buddy system allocator.
  891. *
  892. * The concept of a buddy system is to maintain direct-mapped table
  893. * (containing bit values) for memory blocks of various "orders".
  894. * The bottom level table contains the map for the smallest allocatable
  895. * units of memory (here, pages), and each level above it describes
  896. * pairs of units from the levels below, hence, "buddies".
  897. * At a high level, all that happens here is marking the table entry
  898. * at the bottom level available, and propagating the changes upward
  899. * as necessary, plus some accounting needed to play nicely with other
  900. * parts of the VM system.
  901. * At each level, we keep a list of pages, which are heads of continuous
  902. * free pages of length of (1 << order) and marked with PageBuddy.
  903. * Page's order is recorded in page_private(page) field.
  904. * So when we are allocating or freeing one, we can derive the state of the
  905. * other. That is, if we allocate a small block, and both were
  906. * free, the remainder of the region must be split into blocks.
  907. * If a block is freed, and its buddy is also free, then this
  908. * triggers coalescing into a block of larger size.
  909. *
  910. * -- nyc
  911. */
  912. static inline void __free_one_page(struct page *page,
  913. unsigned long pfn,
  914. struct zone *zone, unsigned int order,
  915. int migratetype, fpi_t fpi_flags)
  916. {
  917. struct capture_control *capc = task_capc(zone);
  918. unsigned long buddy_pfn;
  919. unsigned long combined_pfn;
  920. unsigned int max_order;
  921. struct page *buddy;
  922. bool to_tail;
  923. max_order = min_t(unsigned int, MAX_ORDER - 1, pageblock_order);
  924. VM_BUG_ON(!zone_is_initialized(zone));
  925. VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);
  926. VM_BUG_ON(migratetype == -1);
  927. if (likely(!is_migrate_isolate(migratetype)))
  928. __mod_zone_freepage_state(zone, 1 << order, migratetype);
  929. VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page);
  930. VM_BUG_ON_PAGE(bad_range(zone, page), page);
  931. continue_merging:
  932. while (order < max_order) {
  933. if (compaction_capture(capc, page, order, migratetype)) {
  934. __mod_zone_freepage_state(zone, -(1 << order),
  935. migratetype);
  936. return;
  937. }
  938. buddy_pfn = __find_buddy_pfn(pfn, order);
  939. buddy = page + (buddy_pfn - pfn);
  940. if (!pfn_valid_within(buddy_pfn))
  941. goto done_merging;
  942. if (!page_is_buddy(page, buddy, order))
  943. goto done_merging;
  944. /*
  945. * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
  946. * merge with it and move up one order.
  947. */
  948. if (page_is_guard(buddy))
  949. clear_page_guard(zone, buddy, order, migratetype);
  950. else
  951. del_page_from_free_list(buddy, zone, order);
  952. combined_pfn = buddy_pfn & pfn;
  953. page = page + (combined_pfn - pfn);
  954. pfn = combined_pfn;
  955. order++;
  956. }
  957. if (order < MAX_ORDER - 1) {
  958. /* If we are here, it means order is >= pageblock_order.
  959. * We want to prevent merge between freepages on isolate
  960. * pageblock and normal pageblock. Without this, pageblock
  961. * isolation could cause incorrect freepage or CMA accounting.
  962. *
  963. * We don't want to hit this code for the more frequent
  964. * low-order merging.
  965. */
  966. if (unlikely(has_isolate_pageblock(zone))) {
  967. int buddy_mt;
  968. buddy_pfn = __find_buddy_pfn(pfn, order);
  969. buddy = page + (buddy_pfn - pfn);
  970. if (!page_is_buddy(page, buddy, order))
  971. goto done_merging;
  972. buddy_mt = get_pageblock_migratetype(buddy);
  973. if (migratetype != buddy_mt
  974. && (is_migrate_isolate(migratetype) ||
  975. is_migrate_isolate(buddy_mt)))
  976. goto done_merging;
  977. }
  978. max_order = order + 1;
  979. goto continue_merging;
  980. }
  981. done_merging:
  982. set_buddy_order(page, order);
  983. if (fpi_flags & FPI_TO_TAIL)
  984. to_tail = true;
  985. else if (is_shuffle_order(order))
  986. to_tail = shuffle_pick_tail();
  987. else
  988. to_tail = buddy_merge_likely(pfn, buddy_pfn, page, order);
  989. if (to_tail)
  990. add_to_free_list_tail(page, zone, order, migratetype);
  991. else
  992. add_to_free_list(page, zone, order, migratetype);
  993. /* Notify page reporting subsystem of freed page */
  994. if (!(fpi_flags & FPI_SKIP_REPORT_NOTIFY))
  995. page_reporting_notify_free(order);
  996. }
  997. /*
  998. * A bad page could be due to a number of fields. Instead of multiple branches,
  999. * try and check multiple fields with one check. The caller must do a detailed
  1000. * check if necessary.
  1001. */
  1002. static inline bool page_expected_state(struct page *page,
  1003. unsigned long check_flags)
  1004. {
  1005. if (unlikely(atomic_read(&page->_mapcount) != -1))
  1006. return false;
  1007. if (unlikely((unsigned long)page->mapping |
  1008. page_ref_count(page) |
  1009. #ifdef CONFIG_MEMCG
  1010. (unsigned long)page->mem_cgroup |
  1011. #endif
  1012. (page->flags & check_flags)))
  1013. return false;
  1014. return true;
  1015. }
  1016. static const char *page_bad_reason(struct page *page, unsigned long flags)
  1017. {
  1018. const char *bad_reason = NULL;
  1019. if (unlikely(atomic_read(&page->_mapcount) != -1))
  1020. bad_reason = "nonzero mapcount";
  1021. if (unlikely(page->mapping != NULL))
  1022. bad_reason = "non-NULL mapping";
  1023. if (unlikely(page_ref_count(page) != 0))
  1024. bad_reason = "nonzero _refcount";
  1025. if (unlikely(page->flags & flags)) {
  1026. if (flags == PAGE_FLAGS_CHECK_AT_PREP)
  1027. bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag(s) set";
  1028. else
  1029. bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set";
  1030. }
  1031. #ifdef CONFIG_MEMCG
  1032. if (unlikely(page->mem_cgroup))
  1033. bad_reason = "page still charged to cgroup";
  1034. #endif
  1035. return bad_reason;
  1036. }
  1037. static void check_free_page_bad(struct page *page)
  1038. {
  1039. bad_page(page,
  1040. page_bad_reason(page, PAGE_FLAGS_CHECK_AT_FREE));
  1041. }
  1042. static inline int check_free_page(struct page *page)
  1043. {
  1044. if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE)))
  1045. return 0;
  1046. /* Something has gone sideways, find it */
  1047. check_free_page_bad(page);
  1048. return 1;
  1049. }
  1050. static int free_tail_pages_check(struct page *head_page, struct page *page)
  1051. {
  1052. int ret = 1;
  1053. /*
  1054. * We rely page->lru.next never has bit 0 set, unless the page
  1055. * is PageTail(). Let's make sure that's true even for poisoned ->lru.
  1056. */
  1057. BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1);
  1058. if (!IS_ENABLED(CONFIG_DEBUG_VM)) {
  1059. ret = 0;
  1060. goto out;
  1061. }
  1062. switch (page - head_page) {
  1063. case 1:
  1064. /* the first tail page: ->mapping may be compound_mapcount() */
  1065. if (unlikely(compound_mapcount(page))) {
  1066. bad_page(page, "nonzero compound_mapcount");
  1067. goto out;
  1068. }
  1069. break;
  1070. case 2:
  1071. /*
  1072. * the second tail page: ->mapping is
  1073. * deferred_list.next -- ignore value.
  1074. */
  1075. break;
  1076. default:
  1077. if (page->mapping != TAIL_MAPPING) {
  1078. bad_page(page, "corrupted mapping in tail page");
  1079. goto out;
  1080. }
  1081. break;
  1082. }
  1083. if (unlikely(!PageTail(page))) {
  1084. bad_page(page, "PageTail not set");
  1085. goto out;
  1086. }
  1087. if (unlikely(compound_head(page) != head_page)) {
  1088. bad_page(page, "compound_head not consistent");
  1089. goto out;
  1090. }
  1091. ret = 0;
  1092. out:
  1093. page->mapping = NULL;
  1094. clear_compound_head(page);
  1095. return ret;
  1096. }
  1097. static void kernel_init_free_pages(struct page *page, int numpages, bool zero_tags)
  1098. {
  1099. int i;
  1100. if (zero_tags) {
  1101. for (i = 0; i < numpages; i++)
  1102. tag_clear_highpage(page + i);
  1103. return;
  1104. }
  1105. /* s390's use of memset() could override KASAN redzones. */
  1106. kasan_disable_current();
  1107. for (i = 0; i < numpages; i++) {
  1108. u8 tag = page_kasan_tag(page + i);
  1109. page_kasan_tag_reset(page + i);
  1110. clear_highpage(page + i);
  1111. page_kasan_tag_set(page + i, tag);
  1112. }
  1113. kasan_enable_current();
  1114. }
  1115. static __always_inline bool free_pages_prepare(struct page *page,
  1116. unsigned int order, bool check_free, fpi_t fpi_flags)
  1117. {
  1118. int bad = 0;
  1119. bool skip_kasan_poison = should_skip_kasan_poison(page, fpi_flags);
  1120. VM_BUG_ON_PAGE(PageTail(page), page);
  1121. trace_mm_page_free(page, order);
  1122. if (unlikely(PageHWPoison(page)) && !order) {
  1123. /*
  1124. * Do not let hwpoison pages hit pcplists/buddy
  1125. * Untie memcg state and reset page's owner
  1126. */
  1127. if (memcg_kmem_enabled() && PageKmemcg(page))
  1128. __memcg_kmem_uncharge_page(page, order);
  1129. reset_page_owner(page, order);
  1130. free_page_pinner(page, order);
  1131. return false;
  1132. }
  1133. /*
  1134. * Check tail pages before head page information is cleared to
  1135. * avoid checking PageCompound for order-0 pages.
  1136. */
  1137. if (unlikely(order)) {
  1138. bool compound = PageCompound(page);
  1139. int i;
  1140. VM_BUG_ON_PAGE(compound && compound_order(page) != order, page);
  1141. if (compound)
  1142. ClearPageDoubleMap(page);
  1143. for (i = 1; i < (1 << order); i++) {
  1144. if (compound)
  1145. bad += free_tail_pages_check(page, page + i);
  1146. if (unlikely(check_free_page(page + i))) {
  1147. bad++;
  1148. continue;
  1149. }
  1150. (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
  1151. }
  1152. }
  1153. if (PageMappingFlags(page))
  1154. page->mapping = NULL;
  1155. if (memcg_kmem_enabled() && PageKmemcg(page))
  1156. __memcg_kmem_uncharge_page(page, order);
  1157. if (check_free)
  1158. bad += check_free_page(page);
  1159. if (bad)
  1160. return false;
  1161. page_cpupid_reset_last(page);
  1162. page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
  1163. reset_page_owner(page, order);
  1164. free_page_pinner(page, order);
  1165. if (!PageHighMem(page)) {
  1166. debug_check_no_locks_freed(page_address(page),
  1167. PAGE_SIZE << order);
  1168. debug_check_no_obj_freed(page_address(page),
  1169. PAGE_SIZE << order);
  1170. }
  1171. kernel_poison_pages(page, 1 << order);
  1172. /*
  1173. * As memory initialization might be integrated into KASAN,
  1174. * kasan_free_pages and kernel_init_free_pages must be
  1175. * kept together to avoid discrepancies in behavior.
  1176. *
  1177. * With hardware tag-based KASAN, memory tags must be set before the
  1178. * page becomes unavailable via debug_pagealloc or arch_free_page.
  1179. */
  1180. if (kasan_has_integrated_init()) {
  1181. if (!skip_kasan_poison)
  1182. kasan_free_pages(page, order);
  1183. } else {
  1184. bool init = want_init_on_free();
  1185. if (init)
  1186. kernel_init_free_pages(page, 1 << order, false);
  1187. if (!skip_kasan_poison)
  1188. kasan_poison_pages(page, order, init);
  1189. }
  1190. /*
  1191. * arch_free_page() can make the page's contents inaccessible. s390
  1192. * does this. So nothing which can access the page's contents should
  1193. * happen after this.
  1194. */
  1195. arch_free_page(page, order);
  1196. debug_pagealloc_unmap_pages(page, 1 << order);
  1197. return true;
  1198. }
  1199. #ifdef CONFIG_DEBUG_VM
  1200. /*
  1201. * With DEBUG_VM enabled, order-0 pages are checked immediately when being freed
  1202. * to pcp lists. With debug_pagealloc also enabled, they are also rechecked when
  1203. * moved from pcp lists to free lists.
  1204. */
  1205. static bool free_pcp_prepare(struct page *page)
  1206. {
  1207. return free_pages_prepare(page, 0, true, FPI_NONE);
  1208. }
  1209. static bool bulkfree_pcp_prepare(struct page *page)
  1210. {
  1211. if (debug_pagealloc_enabled_static())
  1212. return check_free_page(page);
  1213. else
  1214. return false;
  1215. }
  1216. #else
  1217. /*
  1218. * With DEBUG_VM disabled, order-0 pages being freed are checked only when
  1219. * moving from pcp lists to free list in order to reduce overhead. With
  1220. * debug_pagealloc enabled, they are checked also immediately when being freed
  1221. * to the pcp lists.
  1222. */
  1223. static bool free_pcp_prepare(struct page *page)
  1224. {
  1225. if (debug_pagealloc_enabled_static())
  1226. return free_pages_prepare(page, 0, true, FPI_NONE);
  1227. else
  1228. return free_pages_prepare(page, 0, false, FPI_NONE);
  1229. }
  1230. static bool bulkfree_pcp_prepare(struct page *page)
  1231. {
  1232. return check_free_page(page);
  1233. }
  1234. #endif /* CONFIG_DEBUG_VM */
  1235. static inline void prefetch_buddy(struct page *page)
  1236. {
  1237. unsigned long pfn = page_to_pfn(page);
  1238. unsigned long buddy_pfn = __find_buddy_pfn(pfn, 0);
  1239. struct page *buddy = page + (buddy_pfn - pfn);
  1240. prefetch(buddy);
  1241. }
  1242. /*
  1243. * Frees a number of pages from the PCP lists
  1244. * Assumes all pages on list are in same zone, and of same order.
  1245. * count is the number of pages to free.
  1246. *
  1247. * If the zone was previously in an "all pages pinned" state then look to
  1248. * see if this freeing clears that state.
  1249. *
  1250. * And clear the zone's pages_scanned counter, to hold off the "all pages are
  1251. * pinned" detection logic.
  1252. */
  1253. static void free_pcppages_bulk(struct zone *zone, int count,
  1254. struct per_cpu_pages *pcp)
  1255. {
  1256. int migratetype = 0;
  1257. int batch_free = 0;
  1258. int prefetch_nr = 0;
  1259. bool isolated_pageblocks;
  1260. struct page *page, *tmp;
  1261. LIST_HEAD(head);
  1262. /*
  1263. * Ensure proper count is passed which otherwise would stuck in the
  1264. * below while (list_empty(list)) loop.
  1265. */
  1266. count = min(pcp->count, count);
  1267. while (count) {
  1268. struct list_head *list;
  1269. /*
  1270. * Remove pages from lists in a round-robin fashion. A
  1271. * batch_free count is maintained that is incremented when an
  1272. * empty list is encountered. This is so more pages are freed
  1273. * off fuller lists instead of spinning excessively around empty
  1274. * lists
  1275. */
  1276. do {
  1277. batch_free++;
  1278. if (++migratetype == MIGRATE_PCPTYPES)
  1279. migratetype = 0;
  1280. list = &pcp->lists[migratetype];
  1281. } while (list_empty(list));
  1282. /* This is the only non-empty list. Free them all. */
  1283. if (batch_free == MIGRATE_PCPTYPES)
  1284. batch_free = count;
  1285. do {
  1286. page = list_last_entry(list, struct page, lru);
  1287. /* must delete to avoid corrupting pcp list */
  1288. list_del(&page->lru);
  1289. pcp->count--;
  1290. if (bulkfree_pcp_prepare(page))
  1291. continue;
  1292. list_add_tail(&page->lru, &head);
  1293. /*
  1294. * We are going to put the page back to the global
  1295. * pool, prefetch its buddy to speed up later access
  1296. * under zone->lock. It is believed the overhead of
  1297. * an additional test and calculating buddy_pfn here
  1298. * can be offset by reduced memory latency later. To
  1299. * avoid excessive prefetching due to large count, only
  1300. * prefetch buddy for the first pcp->batch nr of pages.
  1301. */
  1302. if (prefetch_nr++ < pcp->batch)
  1303. prefetch_buddy(page);
  1304. } while (--count && --batch_free && !list_empty(list));
  1305. }
  1306. spin_lock(&zone->lock);
  1307. isolated_pageblocks = has_isolate_pageblock(zone);
  1308. /*
  1309. * Use safe version since after __free_one_page(),
  1310. * page->lru.next will not point to original list.
  1311. */
  1312. list_for_each_entry_safe(page, tmp, &head, lru) {
  1313. int mt = get_pcppage_migratetype(page);
  1314. /* MIGRATE_ISOLATE page should not go to pcplists */
  1315. VM_BUG_ON_PAGE(is_migrate_isolate(mt), page);
  1316. /* Pageblock could have been isolated meanwhile */
  1317. if (unlikely(isolated_pageblocks))
  1318. mt = get_pageblock_migratetype(page);
  1319. __free_one_page(page, page_to_pfn(page), zone, 0, mt, FPI_NONE);
  1320. trace_mm_page_pcpu_drain(page, 0, mt);
  1321. }
  1322. spin_unlock(&zone->lock);
  1323. }
  1324. static void free_one_page(struct zone *zone,
  1325. struct page *page, unsigned long pfn,
  1326. unsigned int order,
  1327. int migratetype, fpi_t fpi_flags)
  1328. {
  1329. spin_lock(&zone->lock);
  1330. if (unlikely(has_isolate_pageblock(zone) ||
  1331. is_migrate_isolate(migratetype))) {
  1332. migratetype = get_pfnblock_migratetype(page, pfn);
  1333. }
  1334. __free_one_page(page, pfn, zone, order, migratetype, fpi_flags);
  1335. spin_unlock(&zone->lock);
  1336. }
  1337. static void __meminit __init_single_page(struct page *page, unsigned long pfn,
  1338. unsigned long zone, int nid)
  1339. {
  1340. mm_zero_struct_page(page);
  1341. set_page_links(page, zone, nid, pfn);
  1342. init_page_count(page);
  1343. page_mapcount_reset(page);
  1344. page_cpupid_reset_last(page);
  1345. page_kasan_tag_reset(page);
  1346. INIT_LIST_HEAD(&page->lru);
  1347. #ifdef WANT_PAGE_VIRTUAL
  1348. /* The shift won't overflow because ZONE_NORMAL is below 4G. */
  1349. if (!is_highmem_idx(zone))
  1350. set_page_address(page, __va(pfn << PAGE_SHIFT));
  1351. #endif
  1352. }
  1353. #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
  1354. static void __meminit init_reserved_page(unsigned long pfn)
  1355. {
  1356. pg_data_t *pgdat;
  1357. int nid, zid;
  1358. if (!early_page_uninitialised(pfn))
  1359. return;
  1360. nid = early_pfn_to_nid(pfn);
  1361. pgdat = NODE_DATA(nid);
  1362. for (zid = 0; zid < MAX_NR_ZONES; zid++) {
  1363. struct zone *zone = &pgdat->node_zones[zid];
  1364. if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone))
  1365. break;
  1366. }
  1367. __init_single_page(pfn_to_page(pfn), pfn, zid, nid);
  1368. }
  1369. #else
  1370. static inline void init_reserved_page(unsigned long pfn)
  1371. {
  1372. }
  1373. #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
  1374. /*
  1375. * Initialised pages do not have PageReserved set. This function is
  1376. * called for each range allocated by the bootmem allocator and
  1377. * marks the pages PageReserved. The remaining valid pages are later
  1378. * sent to the buddy page allocator.
  1379. */
  1380. void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end)
  1381. {
  1382. unsigned long start_pfn = PFN_DOWN(start);
  1383. unsigned long end_pfn = PFN_UP(end);
  1384. for (; start_pfn < end_pfn; start_pfn++) {
  1385. if (pfn_valid(start_pfn)) {
  1386. struct page *page = pfn_to_page(start_pfn);
  1387. init_reserved_page(start_pfn);
  1388. /* Avoid false-positive PageTail() */
  1389. INIT_LIST_HEAD(&page->lru);
  1390. /*
  1391. * no need for atomic set_bit because the struct
  1392. * page is not visible yet so nobody should
  1393. * access it yet.
  1394. */
  1395. __SetPageReserved(page);
  1396. }
  1397. }
  1398. }
  1399. static void __free_pages_ok(struct page *page, unsigned int order,
  1400. fpi_t fpi_flags)
  1401. {
  1402. unsigned long flags;
  1403. int migratetype;
  1404. unsigned long pfn = page_to_pfn(page);
  1405. if (!free_pages_prepare(page, order, true, fpi_flags))
  1406. return;
  1407. migratetype = get_pfnblock_migratetype(page, pfn);
  1408. local_irq_save(flags);
  1409. __count_vm_events(PGFREE, 1 << order);
  1410. free_one_page(page_zone(page), page, pfn, order, migratetype,
  1411. fpi_flags);
  1412. local_irq_restore(flags);
  1413. }
  1414. void __free_pages_core(struct page *page, unsigned int order)
  1415. {
  1416. unsigned int nr_pages = 1 << order;
  1417. struct page *p = page;
  1418. unsigned int loop;
  1419. /*
  1420. * When initializing the memmap, __init_single_page() sets the refcount
  1421. * of all pages to 1 ("allocated"/"not free"). We have to set the
  1422. * refcount of all involved pages to 0.
  1423. */
  1424. prefetchw(p);
  1425. for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
  1426. prefetchw(p + 1);
  1427. __ClearPageReserved(p);
  1428. set_page_count(p, 0);
  1429. }
  1430. __ClearPageReserved(p);
  1431. set_page_count(p, 0);
  1432. atomic_long_add(nr_pages, &page_zone(page)->managed_pages);
  1433. /*
  1434. * Bypass PCP and place fresh pages right to the tail, primarily
  1435. * relevant for memory onlining.
  1436. */
  1437. __free_pages_ok(page, order, FPI_TO_TAIL | FPI_SKIP_KASAN_POISON);
  1438. }
  1439. #ifdef CONFIG_NEED_MULTIPLE_NODES
  1440. static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata;
  1441. #ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
  1442. /*
  1443. * Required by SPARSEMEM. Given a PFN, return what node the PFN is on.
  1444. */
  1445. int __meminit __early_pfn_to_nid(unsigned long pfn,
  1446. struct mminit_pfnnid_cache *state)
  1447. {
  1448. unsigned long start_pfn, end_pfn;
  1449. int nid;
  1450. if (state->last_start <= pfn && pfn < state->last_end)
  1451. return state->last_nid;
  1452. nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn);
  1453. if (nid != NUMA_NO_NODE) {
  1454. state->last_start = start_pfn;
  1455. state->last_end = end_pfn;
  1456. state->last_nid = nid;
  1457. }
  1458. return nid;
  1459. }
  1460. #endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */
  1461. int __meminit early_pfn_to_nid(unsigned long pfn)
  1462. {
  1463. static DEFINE_SPINLOCK(early_pfn_lock);
  1464. int nid;
  1465. spin_lock(&early_pfn_lock);
  1466. nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache);
  1467. if (nid < 0)
  1468. nid = first_online_node;
  1469. spin_unlock(&early_pfn_lock);
  1470. return nid;
  1471. }
  1472. #endif /* CONFIG_NEED_MULTIPLE_NODES */
  1473. void __init memblock_free_pages(struct page *page, unsigned long pfn,
  1474. unsigned int order)
  1475. {
  1476. if (early_page_uninitialised(pfn))
  1477. return;
  1478. __free_pages_core(page, order);
  1479. }
  1480. /*
  1481. * Check that the whole (or subset of) a pageblock given by the interval of
  1482. * [start_pfn, end_pfn) is valid and within the same zone, before scanning it
  1483. * with the migration of free compaction scanner. The scanners then need to
  1484. * use only pfn_valid_within() check for arches that allow holes within
  1485. * pageblocks.
  1486. *
  1487. * Return struct page pointer of start_pfn, or NULL if checks were not passed.
  1488. *
  1489. * It's possible on some configurations to have a setup like node0 node1 node0
  1490. * i.e. it's possible that all pages within a zones range of pages do not
  1491. * belong to a single zone. We assume that a border between node0 and node1
  1492. * can occur within a single pageblock, but not a node0 node1 node0
  1493. * interleaving within a single pageblock. It is therefore sufficient to check
  1494. * the first and last page of a pageblock and avoid checking each individual
  1495. * page in a pageblock.
  1496. */
  1497. struct page *__pageblock_pfn_to_page(unsigned long start_pfn,
  1498. unsigned long end_pfn, struct zone *zone)
  1499. {
  1500. struct page *start_page;
  1501. struct page *end_page;
  1502. /* end_pfn is one past the range we are checking */
  1503. end_pfn--;
  1504. if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn))
  1505. return NULL;
  1506. start_page = pfn_to_online_page(start_pfn);
  1507. if (!start_page)
  1508. return NULL;
  1509. if (page_zone(start_page) != zone)
  1510. return NULL;
  1511. end_page = pfn_to_page(end_pfn);
  1512. /* This gives a shorter code than deriving page_zone(end_page) */
  1513. if (page_zone_id(start_page) != page_zone_id(end_page))
  1514. return NULL;
  1515. return start_page;
  1516. }
  1517. void set_zone_contiguous(struct zone *zone)
  1518. {
  1519. unsigned long block_start_pfn = zone->zone_start_pfn;
  1520. unsigned long block_end_pfn;
  1521. block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages);
  1522. for (; block_start_pfn < zone_end_pfn(zone);
  1523. block_start_pfn = block_end_pfn,
  1524. block_end_pfn += pageblock_nr_pages) {
  1525. block_end_pfn = min(block_end_pfn, zone_end_pfn(zone));
  1526. if (!__pageblock_pfn_to_page(block_start_pfn,
  1527. block_end_pfn, zone))
  1528. return;
  1529. cond_resched();
  1530. }
  1531. /* We confirm that there is no hole */
  1532. zone->contiguous = true;
  1533. }
  1534. void clear_zone_contiguous(struct zone *zone)
  1535. {
  1536. zone->contiguous = false;
  1537. }
  1538. #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
  1539. static void __init deferred_free_range(unsigned long pfn,
  1540. unsigned long nr_pages)
  1541. {
  1542. struct page *page;
  1543. unsigned long i;
  1544. if (!nr_pages)
  1545. return;
  1546. page = pfn_to_page(pfn);
  1547. /* Free a large naturally-aligned chunk if possible */
  1548. if (nr_pages == pageblock_nr_pages &&
  1549. (pfn & (pageblock_nr_pages - 1)) == 0) {
  1550. set_pageblock_migratetype(page, MIGRATE_MOVABLE);
  1551. __free_pages_core(page, pageblock_order);
  1552. return;
  1553. }
  1554. for (i = 0; i < nr_pages; i++, page++, pfn++) {
  1555. if ((pfn & (pageblock_nr_pages - 1)) == 0)
  1556. set_pageblock_migratetype(page, MIGRATE_MOVABLE);
  1557. __free_pages_core(page, 0);
  1558. }
  1559. }
  1560. /* Completion tracking for deferred_init_memmap() threads */
  1561. static atomic_t pgdat_init_n_undone __initdata;
  1562. static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp);
  1563. static inline void __init pgdat_init_report_one_done(void)
  1564. {
  1565. if (atomic_dec_and_test(&pgdat_init_n_undone))
  1566. complete(&pgdat_init_all_done_comp);
  1567. }
  1568. /*
  1569. * Returns true if page needs to be initialized or freed to buddy allocator.
  1570. *
  1571. * First we check if pfn is valid on architectures where it is possible to have
  1572. * holes within pageblock_nr_pages. On systems where it is not possible, this
  1573. * function is optimized out.
  1574. *
  1575. * Then, we check if a current large page is valid by only checking the validity
  1576. * of the head pfn.
  1577. */
  1578. static inline bool __init deferred_pfn_valid(unsigned long pfn)
  1579. {
  1580. if (!pfn_valid_within(pfn))
  1581. return false;
  1582. if (!(pfn & (pageblock_nr_pages - 1)) && !pfn_valid(pfn))
  1583. return false;
  1584. return true;
  1585. }
  1586. /*
  1587. * Free pages to buddy allocator. Try to free aligned pages in
  1588. * pageblock_nr_pages sizes.
  1589. */
  1590. static void __init deferred_free_pages(unsigned long pfn,
  1591. unsigned long end_pfn)
  1592. {
  1593. unsigned long nr_pgmask = pageblock_nr_pages - 1;
  1594. unsigned long nr_free = 0;
  1595. for (; pfn < end_pfn; pfn++) {
  1596. if (!deferred_pfn_valid(pfn)) {
  1597. deferred_free_range(pfn - nr_free, nr_free);
  1598. nr_free = 0;
  1599. } else if (!(pfn & nr_pgmask)) {
  1600. deferred_free_range(pfn - nr_free, nr_free);
  1601. nr_free = 1;
  1602. } else {
  1603. nr_free++;
  1604. }
  1605. }
  1606. /* Free the last block of pages to allocator */
  1607. deferred_free_range(pfn - nr_free, nr_free);
  1608. }
  1609. /*
  1610. * Initialize struct pages. We minimize pfn page lookups and scheduler checks
  1611. * by performing it only once every pageblock_nr_pages.
  1612. * Return number of pages initialized.
  1613. */
  1614. static unsigned long __init deferred_init_pages(struct zone *zone,
  1615. unsigned long pfn,
  1616. unsigned long end_pfn)
  1617. {
  1618. unsigned long nr_pgmask = pageblock_nr_pages - 1;
  1619. int nid = zone_to_nid(zone);
  1620. unsigned long nr_pages = 0;
  1621. int zid = zone_idx(zone);
  1622. struct page *page = NULL;
  1623. for (; pfn < end_pfn; pfn++) {
  1624. if (!deferred_pfn_valid(pfn)) {
  1625. page = NULL;
  1626. continue;
  1627. } else if (!page || !(pfn & nr_pgmask)) {
  1628. page = pfn_to_page(pfn);
  1629. } else {
  1630. page++;
  1631. }
  1632. __init_single_page(page, pfn, zid, nid);
  1633. nr_pages++;
  1634. }
  1635. return (nr_pages);
  1636. }
  1637. /*
  1638. * This function is meant to pre-load the iterator for the zone init.
  1639. * Specifically it walks through the ranges until we are caught up to the
  1640. * first_init_pfn value and exits there. If we never encounter the value we
  1641. * return false indicating there are no valid ranges left.
  1642. */
  1643. static bool __init
  1644. deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone,
  1645. unsigned long *spfn, unsigned long *epfn,
  1646. unsigned long first_init_pfn)
  1647. {
  1648. u64 j;
  1649. /*
  1650. * Start out by walking through the ranges in this zone that have
  1651. * already been initialized. We don't need to do anything with them
  1652. * so we just need to flush them out of the system.
  1653. */
  1654. for_each_free_mem_pfn_range_in_zone(j, zone, spfn, epfn) {
  1655. if (*epfn <= first_init_pfn)
  1656. continue;
  1657. if (*spfn < first_init_pfn)
  1658. *spfn = first_init_pfn;
  1659. *i = j;
  1660. return true;
  1661. }
  1662. return false;
  1663. }
  1664. /*
  1665. * Initialize and free pages. We do it in two loops: first we initialize
  1666. * struct page, then free to buddy allocator, because while we are
  1667. * freeing pages we can access pages that are ahead (computing buddy
  1668. * page in __free_one_page()).
  1669. *
  1670. * In order to try and keep some memory in the cache we have the loop
  1671. * broken along max page order boundaries. This way we will not cause
  1672. * any issues with the buddy page computation.
  1673. */
  1674. static unsigned long __init
  1675. deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn,
  1676. unsigned long *end_pfn)
  1677. {
  1678. unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES);
  1679. unsigned long spfn = *start_pfn, epfn = *end_pfn;
  1680. unsigned long nr_pages = 0;
  1681. u64 j = *i;
  1682. /* First we loop through and initialize the page values */
  1683. for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) {
  1684. unsigned long t;
  1685. if (mo_pfn <= *start_pfn)
  1686. break;
  1687. t = min(mo_pfn, *end_pfn);
  1688. nr_pages += deferred_init_pages(zone, *start_pfn, t);
  1689. if (mo_pfn < *end_pfn) {
  1690. *start_pfn = mo_pfn;
  1691. break;
  1692. }
  1693. }
  1694. /* Reset values and now loop through freeing pages as needed */
  1695. swap(j, *i);
  1696. for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) {
  1697. unsigned long t;
  1698. if (mo_pfn <= spfn)
  1699. break;
  1700. t = min(mo_pfn, epfn);
  1701. deferred_free_pages(spfn, t);
  1702. if (mo_pfn <= epfn)
  1703. break;
  1704. }
  1705. return nr_pages;
  1706. }
  1707. static void __init
  1708. deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn,
  1709. void *arg)
  1710. {
  1711. unsigned long spfn, epfn;
  1712. struct zone *zone = arg;
  1713. u64 i;
  1714. deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn);
  1715. /*
  1716. * Initialize and free pages in MAX_ORDER sized increments so that we
  1717. * can avoid introducing any issues with the buddy allocator.
  1718. */
  1719. while (spfn < end_pfn) {
  1720. deferred_init_maxorder(&i, zone, &spfn, &epfn);
  1721. cond_resched();
  1722. }
  1723. }
  1724. /* An arch may override for more concurrency. */
  1725. __weak int __init
  1726. deferred_page_init_max_threads(const struct cpumask *node_cpumask)
  1727. {
  1728. return 1;
  1729. }
  1730. /* Initialise remaining memory on a node */
  1731. static int __init deferred_init_memmap(void *data)
  1732. {
  1733. pg_data_t *pgdat = data;
  1734. const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
  1735. unsigned long spfn = 0, epfn = 0;
  1736. unsigned long first_init_pfn, flags;
  1737. unsigned long start = jiffies;
  1738. struct zone *zone;
  1739. int zid, max_threads;
  1740. u64 i;
  1741. /* Bind memory initialisation thread to a local node if possible */
  1742. if (!cpumask_empty(cpumask))
  1743. set_cpus_allowed_ptr(current, cpumask);
  1744. pgdat_resize_lock(pgdat, &flags);
  1745. first_init_pfn = pgdat->first_deferred_pfn;
  1746. if (first_init_pfn == ULONG_MAX) {
  1747. pgdat_resize_unlock(pgdat, &flags);
  1748. pgdat_init_report_one_done();
  1749. return 0;
  1750. }
  1751. /* Sanity check boundaries */
  1752. BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn);
  1753. BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat));
  1754. pgdat->first_deferred_pfn = ULONG_MAX;
  1755. /*
  1756. * Once we unlock here, the zone cannot be grown anymore, thus if an
  1757. * interrupt thread must allocate this early in boot, zone must be
  1758. * pre-grown prior to start of deferred page initialization.
  1759. */
  1760. pgdat_resize_unlock(pgdat, &flags);
  1761. /* Only the highest zone is deferred so find it */
  1762. for (zid = 0; zid < MAX_NR_ZONES; zid++) {
  1763. zone = pgdat->node_zones + zid;
  1764. if (first_init_pfn < zone_end_pfn(zone))
  1765. break;
  1766. }
  1767. /* If the zone is empty somebody else may have cleared out the zone */
  1768. if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
  1769. first_init_pfn))
  1770. goto zone_empty;
  1771. max_threads = deferred_page_init_max_threads(cpumask);
  1772. while (spfn < epfn) {
  1773. unsigned long epfn_align = ALIGN(epfn, PAGES_PER_SECTION);
  1774. struct padata_mt_job job = {
  1775. .thread_fn = deferred_init_memmap_chunk,
  1776. .fn_arg = zone,
  1777. .start = spfn,
  1778. .size = epfn_align - spfn,
  1779. .align = PAGES_PER_SECTION,
  1780. .min_chunk = PAGES_PER_SECTION,
  1781. .max_threads = max_threads,
  1782. };
  1783. padata_do_multithreaded(&job);
  1784. deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
  1785. epfn_align);
  1786. }
  1787. zone_empty:
  1788. /* Sanity check that the next zone really is unpopulated */
  1789. WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone));
  1790. pr_info("node %d deferred pages initialised in %ums\n",
  1791. pgdat->node_id, jiffies_to_msecs(jiffies - start));
  1792. pgdat_init_report_one_done();
  1793. return 0;
  1794. }
  1795. /*
  1796. * If this zone has deferred pages, try to grow it by initializing enough
  1797. * deferred pages to satisfy the allocation specified by order, rounded up to
  1798. * the nearest PAGES_PER_SECTION boundary. So we're adding memory in increments
  1799. * of SECTION_SIZE bytes by initializing struct pages in increments of
  1800. * PAGES_PER_SECTION * sizeof(struct page) bytes.
  1801. *
  1802. * Return true when zone was grown, otherwise return false. We return true even
  1803. * when we grow less than requested, to let the caller decide if there are
  1804. * enough pages to satisfy the allocation.
  1805. *
  1806. * Note: We use noinline because this function is needed only during boot, and
  1807. * it is called from a __ref function _deferred_grow_zone. This way we are
  1808. * making sure that it is not inlined into permanent text section.
  1809. */
  1810. static noinline bool __init
  1811. deferred_grow_zone(struct zone *zone, unsigned int order)
  1812. {
  1813. unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION);
  1814. pg_data_t *pgdat = zone->zone_pgdat;
  1815. unsigned long first_deferred_pfn = pgdat->first_deferred_pfn;
  1816. unsigned long spfn, epfn, flags;
  1817. unsigned long nr_pages = 0;
  1818. u64 i;
  1819. /* Only the last zone may have deferred pages */
  1820. if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat))
  1821. return false;
  1822. pgdat_resize_lock(pgdat, &flags);
  1823. /*
  1824. * If someone grew this zone while we were waiting for spinlock, return
  1825. * true, as there might be enough pages already.
  1826. */
  1827. if (first_deferred_pfn != pgdat->first_deferred_pfn) {
  1828. pgdat_resize_unlock(pgdat, &flags);
  1829. return true;
  1830. }
  1831. /* If the zone is empty somebody else may have cleared out the zone */
  1832. if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
  1833. first_deferred_pfn)) {
  1834. pgdat->first_deferred_pfn = ULONG_MAX;
  1835. pgdat_resize_unlock(pgdat, &flags);
  1836. /* Retry only once. */
  1837. return first_deferred_pfn != ULONG_MAX;
  1838. }
  1839. /*
  1840. * Initialize and free pages in MAX_ORDER sized increments so
  1841. * that we can avoid introducing any issues with the buddy
  1842. * allocator.
  1843. */
  1844. while (spfn < epfn) {
  1845. /* update our first deferred PFN for this section */
  1846. first_deferred_pfn = spfn;
  1847. nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn);
  1848. touch_nmi_watchdog();
  1849. /* We should only stop along section boundaries */
  1850. if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION)
  1851. continue;
  1852. /* If our quota has been met we can stop here */
  1853. if (nr_pages >= nr_pages_needed)
  1854. break;
  1855. }
  1856. pgdat->first_deferred_pfn = spfn;
  1857. pgdat_resize_unlock(pgdat, &flags);
  1858. return nr_pages > 0;
  1859. }
  1860. /*
  1861. * deferred_grow_zone() is __init, but it is called from
  1862. * get_page_from_freelist() during early boot until deferred_pages permanently
  1863. * disables this call. This is why we have refdata wrapper to avoid warning,
  1864. * and to ensure that the function body gets unloaded.
  1865. */
  1866. static bool __ref
  1867. _deferred_grow_zone(struct zone *zone, unsigned int order)
  1868. {
  1869. return deferred_grow_zone(zone, order);
  1870. }
  1871. #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
  1872. void __init page_alloc_init_late(void)
  1873. {
  1874. struct zone *zone;
  1875. int nid;
  1876. #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
  1877. /* There will be num_node_state(N_MEMORY) threads */
  1878. atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY));
  1879. for_each_node_state(nid, N_MEMORY) {
  1880. kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid);
  1881. }
  1882. /* Block until all are initialised */
  1883. wait_for_completion(&pgdat_init_all_done_comp);
  1884. /*
  1885. * The number of managed pages has changed due to the initialisation
  1886. * so the pcpu batch and high limits needs to be updated or the limits
  1887. * will be artificially small.
  1888. */
  1889. for_each_populated_zone(zone)
  1890. zone_pcp_update(zone);
  1891. /*
  1892. * We initialized the rest of the deferred pages. Permanently disable
  1893. * on-demand struct page initialization.
  1894. */
  1895. static_branch_disable(&deferred_pages);
  1896. /* Reinit limits that are based on free pages after the kernel is up */
  1897. files_maxfiles_init();
  1898. #endif
  1899. /* Discard memblock private memory */
  1900. memblock_discard();
  1901. for_each_node_state(nid, N_MEMORY)
  1902. shuffle_free_memory(NODE_DATA(nid));
  1903. for_each_populated_zone(zone)
  1904. set_zone_contiguous(zone);
  1905. }
  1906. #ifdef CONFIG_CMA
  1907. /* Free whole pageblock and set its migration type to MIGRATE_CMA. */
  1908. void __init init_cma_reserved_pageblock(struct page *page)
  1909. {
  1910. unsigned i = pageblock_nr_pages;
  1911. struct page *p = page;
  1912. do {
  1913. __ClearPageReserved(p);
  1914. set_page_count(p, 0);
  1915. } while (++p, --i);
  1916. set_pageblock_migratetype(page, MIGRATE_CMA);
  1917. if (pageblock_order >= MAX_ORDER) {
  1918. i = pageblock_nr_pages;
  1919. p = page;
  1920. do {
  1921. set_page_refcounted(p);
  1922. __free_pages(p, MAX_ORDER - 1);
  1923. p += MAX_ORDER_NR_PAGES;
  1924. } while (i -= MAX_ORDER_NR_PAGES);
  1925. } else {
  1926. set_page_refcounted(page);
  1927. __free_pages(page, pageblock_order);
  1928. }
  1929. adjust_managed_page_count(page, pageblock_nr_pages);
  1930. page_zone(page)->cma_pages += pageblock_nr_pages;
  1931. }
  1932. #endif
  1933. /*
  1934. * The order of subdivision here is critical for the IO subsystem.
  1935. * Please do not alter this order without good reasons and regression
  1936. * testing. Specifically, as large blocks of memory are subdivided,
  1937. * the order in which smaller blocks are delivered depends on the order
  1938. * they're subdivided in this function. This is the primary factor
  1939. * influencing the order in which pages are delivered to the IO
  1940. * subsystem according to empirical testing, and this is also justified
  1941. * by considering the behavior of a buddy system containing a single
  1942. * large block of memory acted on by a series of small allocations.
  1943. * This behavior is a critical factor in sglist merging's success.
  1944. *
  1945. * -- nyc
  1946. */
  1947. static inline void expand(struct zone *zone, struct page *page,
  1948. int low, int high, int migratetype)
  1949. {
  1950. unsigned long size = 1 << high;
  1951. while (high > low) {
  1952. high--;
  1953. size >>= 1;
  1954. VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]);
  1955. /*
  1956. * Mark as guard pages (or page), that will allow to
  1957. * merge back to allocator when buddy will be freed.
  1958. * Corresponding page table entries will not be touched,
  1959. * pages will stay not present in virtual address space
  1960. */
  1961. if (set_page_guard(zone, &page[size], high, migratetype))
  1962. continue;
  1963. add_to_free_list(&page[size], zone, high, migratetype);
  1964. set_buddy_order(&page[size], high);
  1965. }
  1966. }
  1967. static void check_new_page_bad(struct page *page)
  1968. {
  1969. if (unlikely(page->flags & __PG_HWPOISON)) {
  1970. /* Don't complain about hwpoisoned pages */
  1971. page_mapcount_reset(page); /* remove PageBuddy */
  1972. return;
  1973. }
  1974. bad_page(page,
  1975. page_bad_reason(page, PAGE_FLAGS_CHECK_AT_PREP));
  1976. }
  1977. /*
  1978. * This page is about to be returned from the page allocator
  1979. */
  1980. static inline int check_new_page(struct page *page)
  1981. {
  1982. if (likely(page_expected_state(page,
  1983. PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON)))
  1984. return 0;
  1985. check_new_page_bad(page);
  1986. return 1;
  1987. }
  1988. #ifdef CONFIG_DEBUG_VM
  1989. /*
  1990. * With DEBUG_VM enabled, order-0 pages are checked for expected state when
  1991. * being allocated from pcp lists. With debug_pagealloc also enabled, they are
  1992. * also checked when pcp lists are refilled from the free lists.
  1993. */
  1994. static inline bool check_pcp_refill(struct page *page)
  1995. {
  1996. if (debug_pagealloc_enabled_static())
  1997. return check_new_page(page);
  1998. else
  1999. return false;
  2000. }
  2001. static inline bool check_new_pcp(struct page *page)
  2002. {
  2003. return check_new_page(page);
  2004. }
  2005. #else
  2006. /*
  2007. * With DEBUG_VM disabled, free order-0 pages are checked for expected state
  2008. * when pcp lists are being refilled from the free lists. With debug_pagealloc
  2009. * enabled, they are also checked when being allocated from the pcp lists.
  2010. */
  2011. static inline bool check_pcp_refill(struct page *page)
  2012. {
  2013. return check_new_page(page);
  2014. }
  2015. static inline bool check_new_pcp(struct page *page)
  2016. {
  2017. if (debug_pagealloc_enabled_static())
  2018. return check_new_page(page);
  2019. else
  2020. return false;
  2021. }
  2022. #endif /* CONFIG_DEBUG_VM */
  2023. static bool check_new_pages(struct page *page, unsigned int order)
  2024. {
  2025. int i;
  2026. for (i = 0; i < (1 << order); i++) {
  2027. struct page *p = page + i;
  2028. if (unlikely(check_new_page(p)))
  2029. return true;
  2030. }
  2031. return false;
  2032. }
  2033. inline void post_alloc_hook(struct page *page, unsigned int order,
  2034. gfp_t gfp_flags)
  2035. {
  2036. set_page_private(page, 0);
  2037. set_page_refcounted(page);
  2038. arch_alloc_page(page, order);
  2039. debug_pagealloc_map_pages(page, 1 << order);
  2040. /*
  2041. * Page unpoisoning must happen before memory initialization.
  2042. * Otherwise, the poison pattern will be overwritten for __GFP_ZERO
  2043. * allocations and the page unpoisoning code will complain.
  2044. */
  2045. kernel_unpoison_pages(page, 1 << order);
  2046. /*
  2047. * As memory initialization might be integrated into KASAN,
  2048. * kasan_alloc_pages and kernel_init_free_pages must be
  2049. * kept together to avoid discrepancies in behavior.
  2050. */
  2051. if (kasan_has_integrated_init()) {
  2052. kasan_alloc_pages(page, order, gfp_flags);
  2053. } else {
  2054. bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags);
  2055. kasan_unpoison_pages(page, order, init);
  2056. if (init)
  2057. kernel_init_free_pages(page, 1 << order,
  2058. gfp_flags & __GFP_ZEROTAGS);
  2059. }
  2060. set_page_owner(page, order, gfp_flags);
  2061. }
  2062. static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
  2063. unsigned int alloc_flags)
  2064. {
  2065. post_alloc_hook(page, order, gfp_flags);
  2066. if (order && (gfp_flags & __GFP_COMP))
  2067. prep_compound_page(page, order);
  2068. /*
  2069. * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to
  2070. * allocate the page. The expectation is that the caller is taking
  2071. * steps that will free more memory. The caller should avoid the page
  2072. * being used for !PFMEMALLOC purposes.
  2073. */
  2074. if (alloc_flags & ALLOC_NO_WATERMARKS)
  2075. set_page_pfmemalloc(page);
  2076. else
  2077. clear_page_pfmemalloc(page);
  2078. }
  2079. /*
  2080. * Go through the free lists for the given migratetype and remove
  2081. * the smallest available page from the freelists
  2082. */
  2083. static __always_inline
  2084. struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
  2085. int migratetype)
  2086. {
  2087. unsigned int current_order;
  2088. struct free_area *area;
  2089. struct page *page;
  2090. /* Find a page of the appropriate size in the preferred list */
  2091. for (current_order = order; current_order < MAX_ORDER; ++current_order) {
  2092. area = &(zone->free_area[current_order]);
  2093. page = get_page_from_free_area(area, migratetype);
  2094. if (!page)
  2095. continue;
  2096. del_page_from_free_list(page, zone, current_order);
  2097. expand(zone, page, order, current_order, migratetype);
  2098. set_pcppage_migratetype(page, migratetype);
  2099. return page;
  2100. }
  2101. return NULL;
  2102. }
  2103. /*
  2104. * This array describes the order lists are fallen back to when
  2105. * the free lists for the desirable migrate type are depleted
  2106. */
  2107. static int fallbacks[MIGRATE_TYPES][3] = {
  2108. [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_TYPES },
  2109. [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES },
  2110. [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_TYPES },
  2111. #ifdef CONFIG_CMA
  2112. [MIGRATE_CMA] = { MIGRATE_TYPES }, /* Never used */
  2113. #endif
  2114. #ifdef CONFIG_MEMORY_ISOLATION
  2115. [MIGRATE_ISOLATE] = { MIGRATE_TYPES }, /* Never used */
  2116. #endif
  2117. };
  2118. #ifdef CONFIG_CMA
  2119. static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone,
  2120. unsigned int order)
  2121. {
  2122. return __rmqueue_smallest(zone, order, MIGRATE_CMA);
  2123. }
  2124. #else
  2125. static inline struct page *__rmqueue_cma_fallback(struct zone *zone,
  2126. unsigned int order) { return NULL; }
  2127. #endif
  2128. /*
  2129. * Move the free pages in a range to the freelist tail of the requested type.
  2130. * Note that start_page and end_pages are not aligned on a pageblock
  2131. * boundary. If alignment is required, use move_freepages_block()
  2132. */
  2133. static int move_freepages(struct zone *zone,
  2134. struct page *start_page, struct page *end_page,
  2135. int migratetype, int *num_movable)
  2136. {
  2137. struct page *page;
  2138. unsigned int order;
  2139. int pages_moved = 0;
  2140. for (page = start_page; page <= end_page;) {
  2141. if (!pfn_valid_within(page_to_pfn(page))) {
  2142. page++;
  2143. continue;
  2144. }
  2145. if (!PageBuddy(page)) {
  2146. /*
  2147. * We assume that pages that could be isolated for
  2148. * migration are movable. But we don't actually try
  2149. * isolating, as that would be expensive.
  2150. */
  2151. if (num_movable &&
  2152. (PageLRU(page) || __PageMovable(page)))
  2153. (*num_movable)++;
  2154. page++;
  2155. continue;
  2156. }
  2157. /* Make sure we are not inadvertently changing nodes */
  2158. VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page);
  2159. VM_BUG_ON_PAGE(page_zone(page) != zone, page);
  2160. order = buddy_order(page);
  2161. move_to_free_list(page, zone, order, migratetype);
  2162. page += 1 << order;
  2163. pages_moved += 1 << order;
  2164. }
  2165. return pages_moved;
  2166. }
  2167. int move_freepages_block(struct zone *zone, struct page *page,
  2168. int migratetype, int *num_movable)
  2169. {
  2170. unsigned long start_pfn, end_pfn;
  2171. struct page *start_page, *end_page;
  2172. if (num_movable)
  2173. *num_movable = 0;
  2174. start_pfn = page_to_pfn(page);
  2175. start_pfn = start_pfn & ~(pageblock_nr_pages-1);
  2176. start_page = pfn_to_page(start_pfn);
  2177. end_page = start_page + pageblock_nr_pages - 1;
  2178. end_pfn = start_pfn + pageblock_nr_pages - 1;
  2179. /* Do not cross zone boundaries */
  2180. if (!zone_spans_pfn(zone, start_pfn))
  2181. start_page = page;
  2182. if (!zone_spans_pfn(zone, end_pfn))
  2183. return 0;
  2184. return move_freepages(zone, start_page, end_page, migratetype,
  2185. num_movable);
  2186. }
  2187. static void change_pageblock_range(struct page *pageblock_page,
  2188. int start_order, int migratetype)
  2189. {
  2190. int nr_pageblocks = 1 << (start_order - pageblock_order);
  2191. while (nr_pageblocks--) {
  2192. set_pageblock_migratetype(pageblock_page, migratetype);
  2193. pageblock_page += pageblock_nr_pages;
  2194. }
  2195. }
  2196. /*
  2197. * When we are falling back to another migratetype during allocation, try to
  2198. * steal extra free pages from the same pageblocks to satisfy further
  2199. * allocations, instead of polluting multiple pageblocks.
  2200. *
  2201. * If we are stealing a relatively large buddy page, it is likely there will
  2202. * be more free pages in the pageblock, so try to steal them all. For
  2203. * reclaimable and unmovable allocations, we steal regardless of page size,
  2204. * as fragmentation caused by those allocations polluting movable pageblocks
  2205. * is worse than movable allocations stealing from unmovable and reclaimable
  2206. * pageblocks.
  2207. */
  2208. static bool can_steal_fallback(unsigned int order, int start_mt)
  2209. {
  2210. /*
  2211. * Leaving this order check is intended, although there is
  2212. * relaxed order check in next check. The reason is that
  2213. * we can actually steal whole pageblock if this condition met,
  2214. * but, below check doesn't guarantee it and that is just heuristic
  2215. * so could be changed anytime.
  2216. */
  2217. if (order >= pageblock_order)
  2218. return true;
  2219. if (order >= pageblock_order / 2 ||
  2220. start_mt == MIGRATE_RECLAIMABLE ||
  2221. start_mt == MIGRATE_UNMOVABLE ||
  2222. page_group_by_mobility_disabled)
  2223. return true;
  2224. return false;
  2225. }
  2226. static inline bool boost_watermark(struct zone *zone)
  2227. {
  2228. unsigned long max_boost;
  2229. if (!watermark_boost_factor)
  2230. return false;
  2231. /*
  2232. * Don't bother in zones that are unlikely to produce results.
  2233. * On small machines, including kdump capture kernels running
  2234. * in a small area, boosting the watermark can cause an out of
  2235. * memory situation immediately.
  2236. */
  2237. if ((pageblock_nr_pages * 4) > zone_managed_pages(zone))
  2238. return false;
  2239. max_boost = mult_frac(zone->_watermark[WMARK_HIGH],
  2240. watermark_boost_factor, 10000);
  2241. /*
  2242. * high watermark may be uninitialised if fragmentation occurs
  2243. * very early in boot so do not boost. We do not fall
  2244. * through and boost by pageblock_nr_pages as failing
  2245. * allocations that early means that reclaim is not going
  2246. * to help and it may even be impossible to reclaim the
  2247. * boosted watermark resulting in a hang.
  2248. */
  2249. if (!max_boost)
  2250. return false;
  2251. max_boost = max(pageblock_nr_pages, max_boost);
  2252. zone->watermark_boost = min(zone->watermark_boost + pageblock_nr_pages,
  2253. max_boost);
  2254. return true;
  2255. }
  2256. /*
  2257. * This function implements actual steal behaviour. If order is large enough,
  2258. * we can steal whole pageblock. If not, we first move freepages in this
  2259. * pageblock to our migratetype and determine how many already-allocated pages
  2260. * are there in the pageblock with a compatible migratetype. If at least half
  2261. * of pages are free or compatible, we can change migratetype of the pageblock
  2262. * itself, so pages freed in the future will be put on the correct free list.
  2263. */
  2264. static void steal_suitable_fallback(struct zone *zone, struct page *page,
  2265. unsigned int alloc_flags, int start_type, bool whole_block)
  2266. {
  2267. unsigned int current_order = buddy_order(page);
  2268. int free_pages, movable_pages, alike_pages;
  2269. int old_block_type;
  2270. old_block_type = get_pageblock_migratetype(page);
  2271. /*
  2272. * This can happen due to races and we want to prevent broken
  2273. * highatomic accounting.
  2274. */
  2275. if (is_migrate_highatomic(old_block_type))
  2276. goto single_page;
  2277. /* Take ownership for orders >= pageblock_order */
  2278. if (current_order >= pageblock_order) {
  2279. change_pageblock_range(page, current_order, start_type);
  2280. goto single_page;
  2281. }
  2282. /*
  2283. * Boost watermarks to increase reclaim pressure to reduce the
  2284. * likelihood of future fallbacks. Wake kswapd now as the node
  2285. * may be balanced overall and kswapd will not wake naturally.
  2286. */
  2287. if (boost_watermark(zone) && (alloc_flags & ALLOC_KSWAPD))
  2288. set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);
  2289. /* We are not allowed to try stealing from the whole block */
  2290. if (!whole_block)
  2291. goto single_page;
  2292. free_pages = move_freepages_block(zone, page, start_type,
  2293. &movable_pages);
  2294. /*
  2295. * Determine how many pages are compatible with our allocation.
  2296. * For movable allocation, it's the number of movable pages which
  2297. * we just obtained. For other types it's a bit more tricky.
  2298. */
  2299. if (start_type == MIGRATE_MOVABLE) {
  2300. alike_pages = movable_pages;
  2301. } else {
  2302. /*
  2303. * If we are falling back a RECLAIMABLE or UNMOVABLE allocation
  2304. * to MOVABLE pageblock, consider all non-movable pages as
  2305. * compatible. If it's UNMOVABLE falling back to RECLAIMABLE or
  2306. * vice versa, be conservative since we can't distinguish the
  2307. * exact migratetype of non-movable pages.
  2308. */
  2309. if (old_block_type == MIGRATE_MOVABLE)
  2310. alike_pages = pageblock_nr_pages
  2311. - (free_pages + movable_pages);
  2312. else
  2313. alike_pages = 0;
  2314. }
  2315. /* moving whole block can fail due to zone boundary conditions */
  2316. if (!free_pages)
  2317. goto single_page;
  2318. /*
  2319. * If a sufficient number of pages in the block are either free or of
  2320. * comparable migratability as our allocation, claim the whole block.
  2321. */
  2322. if (free_pages + alike_pages >= (1 << (pageblock_order-1)) ||
  2323. page_group_by_mobility_disabled)
  2324. set_pageblock_migratetype(page, start_type);
  2325. return;
  2326. single_page:
  2327. move_to_free_list(page, zone, current_order, start_type);
  2328. }
  2329. /*
  2330. * Check whether there is a suitable fallback freepage with requested order.
  2331. * If only_stealable is true, this function returns fallback_mt only if
  2332. * we can steal other freepages all together. This would help to reduce
  2333. * fragmentation due to mixed migratetype pages in one pageblock.
  2334. */
  2335. int find_suitable_fallback(struct free_area *area, unsigned int order,
  2336. int migratetype, bool only_stealable, bool *can_steal)
  2337. {
  2338. int i;
  2339. int fallback_mt;
  2340. if (area->nr_free == 0)
  2341. return -1;
  2342. *can_steal = false;
  2343. for (i = 0;; i++) {
  2344. fallback_mt = fallbacks[migratetype][i];
  2345. if (fallback_mt == MIGRATE_TYPES)
  2346. break;
  2347. if (free_area_empty(area, fallback_mt))
  2348. continue;
  2349. if (can_steal_fallback(order, migratetype))
  2350. *can_steal = true;
  2351. if (!only_stealable)
  2352. return fallback_mt;
  2353. if (*can_steal)
  2354. return fallback_mt;
  2355. }
  2356. return -1;
  2357. }
  2358. /*
  2359. * Reserve a pageblock for exclusive use of high-order atomic allocations if
  2360. * there are no empty page blocks that contain a page with a suitable order
  2361. */
  2362. static void reserve_highatomic_pageblock(struct page *page, struct zone *zone,
  2363. unsigned int alloc_order)
  2364. {
  2365. int mt;
  2366. unsigned long max_managed, flags;
  2367. /*
  2368. * Limit the number reserved to 1 pageblock or roughly 1% of a zone.
  2369. * Check is race-prone but harmless.
  2370. */
  2371. max_managed = (zone_managed_pages(zone) / 100) + pageblock_nr_pages;
  2372. if (zone->nr_reserved_highatomic >= max_managed)
  2373. return;
  2374. spin_lock_irqsave(&zone->lock, flags);
  2375. /* Recheck the nr_reserved_highatomic limit under the lock */
  2376. if (zone->nr_reserved_highatomic >= max_managed)
  2377. goto out_unlock;
  2378. /* Yoink! */
  2379. mt = get_pageblock_migratetype(page);
  2380. if (!is_migrate_highatomic(mt) && !is_migrate_isolate(mt)
  2381. && !is_migrate_cma(mt)) {
  2382. zone->nr_reserved_highatomic += pageblock_nr_pages;
  2383. set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC);
  2384. move_freepages_block(zone, page, MIGRATE_HIGHATOMIC, NULL);
  2385. }
  2386. out_unlock:
  2387. spin_unlock_irqrestore(&zone->lock, flags);
  2388. }
  2389. /*
  2390. * Used when an allocation is about to fail under memory pressure. This
  2391. * potentially hurts the reliability of high-order allocations when under
  2392. * intense memory pressure but failed atomic allocations should be easier
  2393. * to recover from than an OOM.
  2394. *
  2395. * If @force is true, try to unreserve a pageblock even though highatomic
  2396. * pageblock is exhausted.
  2397. */
  2398. static bool unreserve_highatomic_pageblock(const struct alloc_context *ac,
  2399. bool force)
  2400. {
  2401. struct zonelist *zonelist = ac->zonelist;
  2402. unsigned long flags;
  2403. struct zoneref *z;
  2404. struct zone *zone;
  2405. struct page *page;
  2406. int order;
  2407. bool ret;
  2408. for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->highest_zoneidx,
  2409. ac->nodemask) {
  2410. /*
  2411. * Preserve at least one pageblock unless memory pressure
  2412. * is really high.
  2413. */
  2414. if (!force && zone->nr_reserved_highatomic <=
  2415. pageblock_nr_pages)
  2416. continue;
  2417. spin_lock_irqsave(&zone->lock, flags);
  2418. for (order = 0; order < MAX_ORDER; order++) {
  2419. struct free_area *area = &(zone->free_area[order]);
  2420. page = get_page_from_free_area(area, MIGRATE_HIGHATOMIC);
  2421. if (!page)
  2422. continue;
  2423. /*
  2424. * In page freeing path, migratetype change is racy so
  2425. * we can counter several free pages in a pageblock
  2426. * in this loop althoug we changed the pageblock type
  2427. * from highatomic to ac->migratetype. So we should
  2428. * adjust the count once.
  2429. */
  2430. if (is_migrate_highatomic_page(page)) {
  2431. /*
  2432. * It should never happen but changes to
  2433. * locking could inadvertently allow a per-cpu
  2434. * drain to add pages to MIGRATE_HIGHATOMIC
  2435. * while unreserving so be safe and watch for
  2436. * underflows.
  2437. */
  2438. zone->nr_reserved_highatomic -= min(
  2439. pageblock_nr_pages,
  2440. zone->nr_reserved_highatomic);
  2441. }
  2442. /*
  2443. * Convert to ac->migratetype and avoid the normal
  2444. * pageblock stealing heuristics. Minimally, the caller
  2445. * is doing the work and needs the pages. More
  2446. * importantly, if the block was always converted to
  2447. * MIGRATE_UNMOVABLE or another type then the number
  2448. * of pageblocks that cannot be completely freed
  2449. * may increase.
  2450. */
  2451. set_pageblock_migratetype(page, ac->migratetype);
  2452. ret = move_freepages_block(zone, page, ac->migratetype,
  2453. NULL);
  2454. if (ret) {
  2455. spin_unlock_irqrestore(&zone->lock, flags);
  2456. return ret;
  2457. }
  2458. }
  2459. spin_unlock_irqrestore(&zone->lock, flags);
  2460. }
  2461. return false;
  2462. }
  2463. /*
  2464. * Try finding a free buddy page on the fallback list and put it on the free
  2465. * list of requested migratetype, possibly along with other pages from the same
  2466. * block, depending on fragmentation avoidance heuristics. Returns true if
  2467. * fallback was found so that __rmqueue_smallest() can grab it.
  2468. *
  2469. * The use of signed ints for order and current_order is a deliberate
  2470. * deviation from the rest of this file, to make the for loop
  2471. * condition simpler.
  2472. */
  2473. static __always_inline bool
  2474. __rmqueue_fallback(struct zone *zone, int order, int start_migratetype,
  2475. unsigned int alloc_flags)
  2476. {
  2477. struct free_area *area;
  2478. int current_order;
  2479. int min_order = order;
  2480. struct page *page;
  2481. int fallback_mt;
  2482. bool can_steal;
  2483. /*
  2484. * Do not steal pages from freelists belonging to other pageblocks
  2485. * i.e. orders < pageblock_order. If there are no local zones free,
  2486. * the zonelists will be reiterated without ALLOC_NOFRAGMENT.
  2487. */
  2488. if (alloc_flags & ALLOC_NOFRAGMENT)
  2489. min_order = pageblock_order;
  2490. /*
  2491. * Find the largest available free page in the other list. This roughly
  2492. * approximates finding the pageblock with the most free pages, which
  2493. * would be too costly to do exactly.
  2494. */
  2495. for (current_order = MAX_ORDER - 1; current_order >= min_order;
  2496. --current_order) {
  2497. area = &(zone->free_area[current_order]);
  2498. fallback_mt = find_suitable_fallback(area, current_order,
  2499. start_migratetype, false, &can_steal);
  2500. if (fallback_mt == -1)
  2501. continue;
  2502. /*
  2503. * We cannot steal all free pages from the pageblock and the
  2504. * requested migratetype is movable. In that case it's better to
  2505. * steal and split the smallest available page instead of the
  2506. * largest available page, because even if the next movable
  2507. * allocation falls back into a different pageblock than this
  2508. * one, it won't cause permanent fragmentation.
  2509. */
  2510. if (!can_steal && start_migratetype == MIGRATE_MOVABLE
  2511. && current_order > order)
  2512. goto find_smallest;
  2513. goto do_steal;
  2514. }
  2515. return false;
  2516. find_smallest:
  2517. for (current_order = order; current_order < MAX_ORDER;
  2518. current_order++) {
  2519. area = &(zone->free_area[current_order]);
  2520. fallback_mt = find_suitable_fallback(area, current_order,
  2521. start_migratetype, false, &can_steal);
  2522. if (fallback_mt != -1)
  2523. break;
  2524. }
  2525. /*
  2526. * This should not happen - we already found a suitable fallback
  2527. * when looking for the largest page.
  2528. */
  2529. VM_BUG_ON(current_order == MAX_ORDER);
  2530. do_steal:
  2531. page = get_page_from_free_area(area, fallback_mt);
  2532. steal_suitable_fallback(zone, page, alloc_flags, start_migratetype,
  2533. can_steal);
  2534. trace_mm_page_alloc_extfrag(page, order, current_order,
  2535. start_migratetype, fallback_mt);
  2536. return true;
  2537. }
  2538. /*
  2539. * Do the hard work of removing an element from the buddy allocator.
  2540. * Call me with the zone->lock already held.
  2541. */
  2542. static __always_inline struct page *
  2543. __rmqueue(struct zone *zone, unsigned int order, int migratetype,
  2544. unsigned int alloc_flags)
  2545. {
  2546. struct page *page;
  2547. retry:
  2548. page = __rmqueue_smallest(zone, order, migratetype);
  2549. if (unlikely(!page) && __rmqueue_fallback(zone, order, migratetype,
  2550. alloc_flags))
  2551. goto retry;
  2552. trace_mm_page_alloc_zone_locked(page, order, migratetype);
  2553. return page;
  2554. }
  2555. #ifdef CONFIG_CMA
  2556. static struct page *__rmqueue_cma(struct zone *zone, unsigned int order,
  2557. int migratetype,
  2558. unsigned int alloc_flags)
  2559. {
  2560. struct page *page = __rmqueue_cma_fallback(zone, order);
  2561. trace_mm_page_alloc_zone_locked(page, order, MIGRATE_CMA);
  2562. return page;
  2563. }
  2564. #else
  2565. static inline struct page *__rmqueue_cma(struct zone *zone, unsigned int order,
  2566. int migratetype,
  2567. unsigned int alloc_flags)
  2568. {
  2569. return NULL;
  2570. }
  2571. #endif
  2572. /*
  2573. * Obtain a specified number of elements from the buddy allocator, all under
  2574. * a single hold of the lock, for efficiency. Add them to the supplied list.
  2575. * Returns the number of new pages which were placed at *list.
  2576. */
  2577. static int rmqueue_bulk(struct zone *zone, unsigned int order,
  2578. unsigned long count, struct list_head *list,
  2579. int migratetype, unsigned int alloc_flags)
  2580. {
  2581. int i, alloced = 0;
  2582. spin_lock(&zone->lock);
  2583. for (i = 0; i < count; ++i) {
  2584. struct page *page;
  2585. if (is_migrate_cma(migratetype))
  2586. page = __rmqueue_cma(zone, order, migratetype,
  2587. alloc_flags);
  2588. else
  2589. page = __rmqueue(zone, order, migratetype, alloc_flags);
  2590. if (unlikely(page == NULL))
  2591. break;
  2592. if (unlikely(check_pcp_refill(page)))
  2593. continue;
  2594. /*
  2595. * Split buddy pages returned by expand() are received here in
  2596. * physical page order. The page is added to the tail of
  2597. * caller's list. From the callers perspective, the linked list
  2598. * is ordered by page number under some conditions. This is
  2599. * useful for IO devices that can forward direction from the
  2600. * head, thus also in the physical page order. This is useful
  2601. * for IO devices that can merge IO requests if the physical
  2602. * pages are ordered properly.
  2603. */
  2604. list_add_tail(&page->lru, list);
  2605. alloced++;
  2606. if (is_migrate_cma(get_pcppage_migratetype(page)))
  2607. __mod_zone_page_state(zone, NR_FREE_CMA_PAGES,
  2608. -(1 << order));
  2609. }
  2610. /*
  2611. * i pages were removed from the buddy list even if some leak due
  2612. * to check_pcp_refill failing so adjust NR_FREE_PAGES based
  2613. * on i. Do not confuse with 'alloced' which is the number of
  2614. * pages added to the pcp list.
  2615. */
  2616. __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
  2617. spin_unlock(&zone->lock);
  2618. return alloced;
  2619. }
  2620. /*
  2621. * Return the pcp list that corresponds to the migrate type if that list isn't
  2622. * empty.
  2623. * If the list is empty return NULL.
  2624. */
  2625. static struct list_head *get_populated_pcp_list(struct zone *zone,
  2626. unsigned int order, struct per_cpu_pages *pcp,
  2627. int migratetype, unsigned int alloc_flags)
  2628. {
  2629. struct list_head *list = &pcp->lists[migratetype];
  2630. if (list_empty(list)) {
  2631. pcp->count += rmqueue_bulk(zone, order,
  2632. pcp->batch, list,
  2633. migratetype, alloc_flags);
  2634. if (list_empty(list))
  2635. list = NULL;
  2636. }
  2637. return list;
  2638. }
  2639. #ifdef CONFIG_NUMA
  2640. /*
  2641. * Called from the vmstat counter updater to drain pagesets of this
  2642. * currently executing processor on remote nodes after they have
  2643. * expired.
  2644. *
  2645. * Note that this function must be called with the thread pinned to
  2646. * a single processor.
  2647. */
  2648. void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
  2649. {
  2650. unsigned long flags;
  2651. int to_drain, batch;
  2652. local_irq_save(flags);
  2653. batch = READ_ONCE(pcp->batch);
  2654. to_drain = min(pcp->count, batch);
  2655. if (to_drain > 0)
  2656. free_pcppages_bulk(zone, to_drain, pcp);
  2657. local_irq_restore(flags);
  2658. }
  2659. #endif
  2660. /*
  2661. * Drain pcplists of the indicated processor and zone.
  2662. *
  2663. * The processor must either be the current processor and the
  2664. * thread pinned to the current processor or a processor that
  2665. * is not online.
  2666. */
  2667. static void drain_pages_zone(unsigned int cpu, struct zone *zone)
  2668. {
  2669. unsigned long flags;
  2670. struct per_cpu_pageset *pset;
  2671. struct per_cpu_pages *pcp;
  2672. local_irq_save(flags);
  2673. pset = per_cpu_ptr(zone->pageset, cpu);
  2674. pcp = &pset->pcp;
  2675. if (pcp->count)
  2676. free_pcppages_bulk(zone, pcp->count, pcp);
  2677. local_irq_restore(flags);
  2678. }
  2679. /*
  2680. * Drain pcplists of all zones on the indicated processor.
  2681. *
  2682. * The processor must either be the current processor and the
  2683. * thread pinned to the current processor or a processor that
  2684. * is not online.
  2685. */
  2686. static void drain_pages(unsigned int cpu)
  2687. {
  2688. struct zone *zone;
  2689. for_each_populated_zone(zone) {
  2690. drain_pages_zone(cpu, zone);
  2691. }
  2692. }
  2693. /*
  2694. * Spill all of this CPU's per-cpu pages back into the buddy allocator.
  2695. *
  2696. * The CPU has to be pinned. When zone parameter is non-NULL, spill just
  2697. * the single zone's pages.
  2698. */
  2699. void drain_local_pages(struct zone *zone)
  2700. {
  2701. int cpu = smp_processor_id();
  2702. if (zone)
  2703. drain_pages_zone(cpu, zone);
  2704. else
  2705. drain_pages(cpu);
  2706. }
  2707. static void drain_local_pages_wq(struct work_struct *work)
  2708. {
  2709. struct pcpu_drain *drain;
  2710. drain = container_of(work, struct pcpu_drain, work);
  2711. /*
  2712. * drain_all_pages doesn't use proper cpu hotplug protection so
  2713. * we can race with cpu offline when the WQ can move this from
  2714. * a cpu pinned worker to an unbound one. We can operate on a different
  2715. * cpu which is allright but we also have to make sure to not move to
  2716. * a different one.
  2717. */
  2718. preempt_disable();
  2719. drain_local_pages(drain->zone);
  2720. preempt_enable();
  2721. }
  2722. /*
  2723. * Spill all the per-cpu pages from all CPUs back into the buddy allocator.
  2724. *
  2725. * When zone parameter is non-NULL, spill just the single zone's pages.
  2726. *
  2727. * Note that this can be extremely slow as the draining happens in a workqueue.
  2728. */
  2729. void drain_all_pages(struct zone *zone)
  2730. {
  2731. int cpu;
  2732. /*
  2733. * Allocate in the BSS so we wont require allocation in
  2734. * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y
  2735. */
  2736. static cpumask_t cpus_with_pcps;
  2737. /*
  2738. * Make sure nobody triggers this path before mm_percpu_wq is fully
  2739. * initialized.
  2740. */
  2741. if (WARN_ON_ONCE(!mm_percpu_wq))
  2742. return;
  2743. /*
  2744. * Do not drain if one is already in progress unless it's specific to
  2745. * a zone. Such callers are primarily CMA and memory hotplug and need
  2746. * the drain to be complete when the call returns.
  2747. */
  2748. if (unlikely(!mutex_trylock(&pcpu_drain_mutex))) {
  2749. if (!zone)
  2750. return;
  2751. mutex_lock(&pcpu_drain_mutex);
  2752. }
  2753. /*
  2754. * We don't care about racing with CPU hotplug event
  2755. * as offline notification will cause the notified
  2756. * cpu to drain that CPU pcps and on_each_cpu_mask
  2757. * disables preemption as part of its processing
  2758. */
  2759. for_each_online_cpu(cpu) {
  2760. struct per_cpu_pageset *pcp;
  2761. struct zone *z;
  2762. bool has_pcps = false;
  2763. if (zone) {
  2764. pcp = per_cpu_ptr(zone->pageset, cpu);
  2765. if (pcp->pcp.count)
  2766. has_pcps = true;
  2767. } else {
  2768. for_each_populated_zone(z) {
  2769. pcp = per_cpu_ptr(z->pageset, cpu);
  2770. if (pcp->pcp.count) {
  2771. has_pcps = true;
  2772. break;
  2773. }
  2774. }
  2775. }
  2776. if (has_pcps)
  2777. cpumask_set_cpu(cpu, &cpus_with_pcps);
  2778. else
  2779. cpumask_clear_cpu(cpu, &cpus_with_pcps);
  2780. }
  2781. for_each_cpu(cpu, &cpus_with_pcps) {
  2782. struct pcpu_drain *drain = per_cpu_ptr(&pcpu_drain, cpu);
  2783. drain->zone = zone;
  2784. INIT_WORK(&drain->work, drain_local_pages_wq);
  2785. queue_work_on(cpu, mm_percpu_wq, &drain->work);
  2786. }
  2787. for_each_cpu(cpu, &cpus_with_pcps)
  2788. flush_work(&per_cpu_ptr(&pcpu_drain, cpu)->work);
  2789. mutex_unlock(&pcpu_drain_mutex);
  2790. }
  2791. #ifdef CONFIG_HIBERNATION
  2792. /*
  2793. * Touch the watchdog for every WD_PAGE_COUNT pages.
  2794. */
  2795. #define WD_PAGE_COUNT (128*1024)
  2796. void mark_free_pages(struct zone *zone)
  2797. {
  2798. unsigned long pfn, max_zone_pfn, page_count = WD_PAGE_COUNT;
  2799. unsigned long flags;
  2800. unsigned int order, t;
  2801. struct page *page;
  2802. if (zone_is_empty(zone))
  2803. return;
  2804. spin_lock_irqsave(&zone->lock, flags);
  2805. max_zone_pfn = zone_end_pfn(zone);
  2806. for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
  2807. if (pfn_valid(pfn)) {
  2808. page = pfn_to_page(pfn);
  2809. if (!--page_count) {
  2810. touch_nmi_watchdog();
  2811. page_count = WD_PAGE_COUNT;
  2812. }
  2813. if (page_zone(page) != zone)
  2814. continue;
  2815. if (!swsusp_page_is_forbidden(page))
  2816. swsusp_unset_page_free(page);
  2817. }
  2818. for_each_migratetype_order(order, t) {
  2819. list_for_each_entry(page,
  2820. &zone->free_area[order].free_list[t], lru) {
  2821. unsigned long i;
  2822. pfn = page_to_pfn(page);
  2823. for (i = 0; i < (1UL << order); i++) {
  2824. if (!--page_count) {
  2825. touch_nmi_watchdog();
  2826. page_count = WD_PAGE_COUNT;
  2827. }
  2828. swsusp_set_page_free(pfn_to_page(pfn + i));
  2829. }
  2830. }
  2831. }
  2832. spin_unlock_irqrestore(&zone->lock, flags);
  2833. }
  2834. #endif /* CONFIG_PM */
  2835. static bool free_unref_page_prepare(struct page *page, unsigned long pfn)
  2836. {
  2837. int migratetype;
  2838. if (!free_pcp_prepare(page))
  2839. return false;
  2840. migratetype = get_pfnblock_migratetype(page, pfn);
  2841. set_pcppage_migratetype(page, migratetype);
  2842. return true;
  2843. }
  2844. static void free_unref_page_commit(struct page *page, unsigned long pfn)
  2845. {
  2846. struct zone *zone = page_zone(page);
  2847. struct per_cpu_pages *pcp;
  2848. int migratetype;
  2849. bool pcp_skip_cma_pages = false;
  2850. migratetype = get_pcppage_migratetype(page);
  2851. __count_vm_event(PGFREE);
  2852. /*
  2853. * We only track unmovable, reclaimable and movable on pcp lists.
  2854. * Free ISOLATE pages back to the allocator because they are being
  2855. * offlined but treat HIGHATOMIC as movable pages so we can get those
  2856. * areas back if necessary. Otherwise, we may have to free
  2857. * excessively into the page allocator
  2858. */
  2859. if (migratetype >= MIGRATE_PCPTYPES) {
  2860. trace_android_vh_pcplist_add_cma_pages_bypass(migratetype,
  2861. &pcp_skip_cma_pages);
  2862. if (unlikely(is_migrate_isolate(migratetype)) ||
  2863. pcp_skip_cma_pages) {
  2864. free_one_page(zone, page, pfn, 0, migratetype,
  2865. FPI_NONE);
  2866. return;
  2867. }
  2868. migratetype = MIGRATE_MOVABLE;
  2869. }
  2870. pcp = &this_cpu_ptr(zone->pageset)->pcp;
  2871. list_add(&page->lru, &pcp->lists[migratetype]);
  2872. pcp->count++;
  2873. if (pcp->count >= pcp->high) {
  2874. unsigned long batch = READ_ONCE(pcp->batch);
  2875. free_pcppages_bulk(zone, batch, pcp);
  2876. }
  2877. }
  2878. /*
  2879. * Free a 0-order page
  2880. */
  2881. void free_unref_page(struct page *page)
  2882. {
  2883. unsigned long flags;
  2884. unsigned long pfn = page_to_pfn(page);
  2885. if (!free_unref_page_prepare(page, pfn))
  2886. return;
  2887. local_irq_save(flags);
  2888. free_unref_page_commit(page, pfn);
  2889. local_irq_restore(flags);
  2890. }
  2891. /*
  2892. * Free a list of 0-order pages
  2893. */
  2894. void free_unref_page_list(struct list_head *list)
  2895. {
  2896. struct page *page, *next;
  2897. unsigned long flags, pfn;
  2898. int batch_count = 0;
  2899. /* Prepare pages for freeing */
  2900. list_for_each_entry_safe(page, next, list, lru) {
  2901. pfn = page_to_pfn(page);
  2902. if (!free_unref_page_prepare(page, pfn))
  2903. list_del(&page->lru);
  2904. set_page_private(page, pfn);
  2905. }
  2906. local_irq_save(flags);
  2907. list_for_each_entry_safe(page, next, list, lru) {
  2908. unsigned long pfn = page_private(page);
  2909. set_page_private(page, 0);
  2910. trace_mm_page_free_batched(page);
  2911. free_unref_page_commit(page, pfn);
  2912. /*
  2913. * Guard against excessive IRQ disabled times when we get
  2914. * a large list of pages to free.
  2915. */
  2916. if (++batch_count == SWAP_CLUSTER_MAX) {
  2917. local_irq_restore(flags);
  2918. batch_count = 0;
  2919. local_irq_save(flags);
  2920. }
  2921. }
  2922. local_irq_restore(flags);
  2923. }
  2924. /*
  2925. * split_page takes a non-compound higher-order page, and splits it into
  2926. * n (1<<order) sub-pages: page[0..n]
  2927. * Each sub-page must be freed individually.
  2928. *
  2929. * Note: this is probably too low level an operation for use in drivers.
  2930. * Please consult with lkml before using this in your driver.
  2931. */
  2932. void split_page(struct page *page, unsigned int order)
  2933. {
  2934. int i;
  2935. VM_BUG_ON_PAGE(PageCompound(page), page);
  2936. VM_BUG_ON_PAGE(!page_count(page), page);
  2937. for (i = 1; i < (1 << order); i++)
  2938. set_page_refcounted(page + i);
  2939. split_page_owner(page, 1 << order);
  2940. split_page_memcg(page, 1 << order);
  2941. }
  2942. EXPORT_SYMBOL_GPL(split_page);
  2943. int __isolate_free_page(struct page *page, unsigned int order)
  2944. {
  2945. unsigned long watermark;
  2946. struct zone *zone;
  2947. int mt;
  2948. BUG_ON(!PageBuddy(page));
  2949. zone = page_zone(page);
  2950. mt = get_pageblock_migratetype(page);
  2951. if (!is_migrate_isolate(mt)) {
  2952. /*
  2953. * Obey watermarks as if the page was being allocated. We can
  2954. * emulate a high-order watermark check with a raised order-0
  2955. * watermark, because we already know our high-order page
  2956. * exists.
  2957. */
  2958. watermark = zone->_watermark[WMARK_MIN] + (1UL << order);
  2959. if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA))
  2960. return 0;
  2961. __mod_zone_freepage_state(zone, -(1UL << order), mt);
  2962. }
  2963. /* Remove page from free list */
  2964. del_page_from_free_list(page, zone, order);
  2965. /*
  2966. * Set the pageblock if the isolated page is at least half of a
  2967. * pageblock
  2968. */
  2969. if (order >= pageblock_order - 1) {
  2970. struct page *endpage = page + (1 << order) - 1;
  2971. for (; page < endpage; page += pageblock_nr_pages) {
  2972. int mt = get_pageblock_migratetype(page);
  2973. if (!is_migrate_isolate(mt) && !is_migrate_cma(mt)
  2974. && !is_migrate_highatomic(mt))
  2975. set_pageblock_migratetype(page,
  2976. MIGRATE_MOVABLE);
  2977. }
  2978. }
  2979. return 1UL << order;
  2980. }
  2981. /**
  2982. * __putback_isolated_page - Return a now-isolated page back where we got it
  2983. * @page: Page that was isolated
  2984. * @order: Order of the isolated page
  2985. * @mt: The page's pageblock's migratetype
  2986. *
  2987. * This function is meant to return a page pulled from the free lists via
  2988. * __isolate_free_page back to the free lists they were pulled from.
  2989. */
  2990. void __putback_isolated_page(struct page *page, unsigned int order, int mt)
  2991. {
  2992. struct zone *zone = page_zone(page);
  2993. /* zone lock should be held when this function is called */
  2994. lockdep_assert_held(&zone->lock);
  2995. /* Return isolated page to tail of freelist. */
  2996. __free_one_page(page, page_to_pfn(page), zone, order, mt,
  2997. FPI_SKIP_REPORT_NOTIFY | FPI_TO_TAIL);
  2998. }
  2999. /*
  3000. * Update NUMA hit/miss statistics
  3001. *
  3002. * Must be called with interrupts disabled.
  3003. */
  3004. static inline void zone_statistics(struct zone *preferred_zone, struct zone *z)
  3005. {
  3006. #ifdef CONFIG_NUMA
  3007. enum numa_stat_item local_stat = NUMA_LOCAL;
  3008. /* skip numa counters update if numa stats is disabled */
  3009. if (!static_branch_likely(&vm_numa_stat_key))
  3010. return;
  3011. if (zone_to_nid(z) != numa_node_id())
  3012. local_stat = NUMA_OTHER;
  3013. if (zone_to_nid(z) == zone_to_nid(preferred_zone))
  3014. __inc_numa_state(z, NUMA_HIT);
  3015. else {
  3016. __inc_numa_state(z, NUMA_MISS);
  3017. __inc_numa_state(preferred_zone, NUMA_FOREIGN);
  3018. }
  3019. __inc_numa_state(z, local_stat);
  3020. #endif
  3021. }
  3022. /* Remove page from the per-cpu list, caller must protect the list */
  3023. static struct page *__rmqueue_pcplist(struct zone *zone, int migratetype,
  3024. unsigned int alloc_flags,
  3025. struct per_cpu_pages *pcp,
  3026. gfp_t gfp_flags)
  3027. {
  3028. struct page *page = NULL;
  3029. struct list_head *list = NULL;
  3030. do {
  3031. /* First try to get CMA pages */
  3032. if (migratetype == MIGRATE_MOVABLE &&
  3033. alloc_flags & ALLOC_CMA) {
  3034. list = get_populated_pcp_list(zone, 0, pcp,
  3035. get_cma_migrate_type(), alloc_flags);
  3036. }
  3037. if (list == NULL) {
  3038. /*
  3039. * Either CMA is not suitable or there are no
  3040. * free CMA pages.
  3041. */
  3042. list = get_populated_pcp_list(zone, 0, pcp,
  3043. migratetype, alloc_flags);
  3044. if (unlikely(list == NULL) ||
  3045. unlikely(list_empty(list)))
  3046. return NULL;
  3047. }
  3048. page = list_first_entry(list, struct page, lru);
  3049. list_del(&page->lru);
  3050. pcp->count--;
  3051. } while (check_new_pcp(page));
  3052. return page;
  3053. }
  3054. /* Lock and remove page from the per-cpu list */
  3055. static struct page *rmqueue_pcplist(struct zone *preferred_zone,
  3056. struct zone *zone, gfp_t gfp_flags,
  3057. int migratetype, unsigned int alloc_flags)
  3058. {
  3059. struct per_cpu_pages *pcp;
  3060. struct page *page;
  3061. unsigned long flags;
  3062. local_irq_save(flags);
  3063. pcp = &this_cpu_ptr(zone->pageset)->pcp;
  3064. page = __rmqueue_pcplist(zone, migratetype, alloc_flags, pcp,
  3065. gfp_flags);
  3066. if (page) {
  3067. __count_zid_vm_events(PGALLOC, page_zonenum(page), 1);
  3068. zone_statistics(preferred_zone, zone);
  3069. }
  3070. local_irq_restore(flags);
  3071. return page;
  3072. }
  3073. /*
  3074. * Allocate a page from the given zone. Use pcplists for order-0 allocations.
  3075. */
  3076. static inline
  3077. struct page *rmqueue(struct zone *preferred_zone,
  3078. struct zone *zone, unsigned int order,
  3079. gfp_t gfp_flags, unsigned int alloc_flags,
  3080. int migratetype)
  3081. {
  3082. unsigned long flags;
  3083. struct page *page;
  3084. if (likely(order == 0)) {
  3085. page = rmqueue_pcplist(preferred_zone, zone, gfp_flags,
  3086. migratetype, alloc_flags);
  3087. goto out;
  3088. }
  3089. /*
  3090. * We most definitely don't want callers attempting to
  3091. * allocate greater than order-1 page units with __GFP_NOFAIL.
  3092. */
  3093. WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1));
  3094. spin_lock_irqsave(&zone->lock, flags);
  3095. do {
  3096. page = NULL;
  3097. /*
  3098. * order-0 request can reach here when the pcplist is skipped
  3099. * due to non-CMA allocation context. HIGHATOMIC area is
  3100. * reserved for high-order atomic allocation, so order-0
  3101. * request should skip it.
  3102. */
  3103. if (order > 0 && alloc_flags & ALLOC_HARDER) {
  3104. page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC);
  3105. if (page)
  3106. trace_mm_page_alloc_zone_locked(page, order, migratetype);
  3107. }
  3108. if (!page) {
  3109. if (migratetype == MIGRATE_MOVABLE &&
  3110. alloc_flags & ALLOC_CMA)
  3111. page = __rmqueue_cma(zone, order, migratetype,
  3112. alloc_flags);
  3113. if (!page)
  3114. page = __rmqueue(zone, order, migratetype,
  3115. alloc_flags);
  3116. }
  3117. } while (page && check_new_pages(page, order));
  3118. spin_unlock(&zone->lock);
  3119. if (!page)
  3120. goto failed;
  3121. __mod_zone_freepage_state(zone, -(1 << order),
  3122. get_pcppage_migratetype(page));
  3123. __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);
  3124. zone_statistics(preferred_zone, zone);
  3125. trace_android_vh_rmqueue(preferred_zone, zone, order,
  3126. gfp_flags, alloc_flags, migratetype);
  3127. local_irq_restore(flags);
  3128. out:
  3129. /* Separate test+clear to avoid unnecessary atomics */
  3130. if (test_bit(ZONE_BOOSTED_WATERMARK, &zone->flags)) {
  3131. clear_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);
  3132. wakeup_kswapd(zone, 0, 0, zone_idx(zone));
  3133. }
  3134. VM_BUG_ON_PAGE(page && bad_range(zone, page), page);
  3135. return page;
  3136. failed:
  3137. local_irq_restore(flags);
  3138. return NULL;
  3139. }
  3140. #ifdef CONFIG_FAIL_PAGE_ALLOC
  3141. static struct {
  3142. struct fault_attr attr;
  3143. bool ignore_gfp_highmem;
  3144. bool ignore_gfp_reclaim;
  3145. u32 min_order;
  3146. } fail_page_alloc = {
  3147. .attr = FAULT_ATTR_INITIALIZER,
  3148. .ignore_gfp_reclaim = true,
  3149. .ignore_gfp_highmem = true,
  3150. .min_order = 1,
  3151. };
  3152. static int __init setup_fail_page_alloc(char *str)
  3153. {
  3154. return setup_fault_attr(&fail_page_alloc.attr, str);
  3155. }
  3156. __setup("fail_page_alloc=", setup_fail_page_alloc);
  3157. static bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
  3158. {
  3159. if (order < fail_page_alloc.min_order)
  3160. return false;
  3161. if (gfp_mask & __GFP_NOFAIL)
  3162. return false;
  3163. if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM))
  3164. return false;
  3165. if (fail_page_alloc.ignore_gfp_reclaim &&
  3166. (gfp_mask & __GFP_DIRECT_RECLAIM))
  3167. return false;
  3168. return should_fail(&fail_page_alloc.attr, 1 << order);
  3169. }
  3170. #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
  3171. static int __init fail_page_alloc_debugfs(void)
  3172. {
  3173. umode_t mode = S_IFREG | 0600;
  3174. struct dentry *dir;
  3175. dir = fault_create_debugfs_attr("fail_page_alloc", NULL,
  3176. &fail_page_alloc.attr);
  3177. debugfs_create_bool("ignore-gfp-wait", mode, dir,
  3178. &fail_page_alloc.ignore_gfp_reclaim);
  3179. debugfs_create_bool("ignore-gfp-highmem", mode, dir,
  3180. &fail_page_alloc.ignore_gfp_highmem);
  3181. debugfs_create_u32("min-order", mode, dir, &fail_page_alloc.min_order);
  3182. return 0;
  3183. }
  3184. late_initcall(fail_page_alloc_debugfs);
  3185. #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
  3186. #else /* CONFIG_FAIL_PAGE_ALLOC */
  3187. static inline bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
  3188. {
  3189. return false;
  3190. }
  3191. #endif /* CONFIG_FAIL_PAGE_ALLOC */
  3192. noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
  3193. {
  3194. return __should_fail_alloc_page(gfp_mask, order);
  3195. }
  3196. ALLOW_ERROR_INJECTION(should_fail_alloc_page, TRUE);
  3197. static inline long __zone_watermark_unusable_free(struct zone *z,
  3198. unsigned int order, unsigned int alloc_flags)
  3199. {
  3200. const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM));
  3201. long unusable_free = (1 << order) - 1;
  3202. /*
  3203. * If the caller does not have rights to ALLOC_HARDER then subtract
  3204. * the high-atomic reserves. This will over-estimate the size of the
  3205. * atomic reserve but it avoids a search.
  3206. */
  3207. if (likely(!alloc_harder))
  3208. unusable_free += z->nr_reserved_highatomic;
  3209. #ifdef CONFIG_CMA
  3210. /* If allocation can't use CMA areas don't use free CMA pages */
  3211. if (!(alloc_flags & ALLOC_CMA))
  3212. unusable_free += zone_page_state(z, NR_FREE_CMA_PAGES);
  3213. #endif
  3214. return unusable_free;
  3215. }
  3216. /*
  3217. * Return true if free base pages are above 'mark'. For high-order checks it
  3218. * will return true of the order-0 watermark is reached and there is at least
  3219. * one free page of a suitable size. Checking now avoids taking the zone lock
  3220. * to check in the allocation paths if no pages are free.
  3221. */
  3222. bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
  3223. int highest_zoneidx, unsigned int alloc_flags,
  3224. long free_pages)
  3225. {
  3226. long min = mark;
  3227. int o;
  3228. const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM));
  3229. /* free_pages may go negative - that's OK */
  3230. free_pages -= __zone_watermark_unusable_free(z, order, alloc_flags);
  3231. if (alloc_flags & ALLOC_HIGH)
  3232. min -= min / 2;
  3233. if (unlikely(alloc_harder)) {
  3234. /*
  3235. * OOM victims can try even harder than normal ALLOC_HARDER
  3236. * users on the grounds that it's definitely going to be in
  3237. * the exit path shortly and free memory. Any allocation it
  3238. * makes during the free path will be small and short-lived.
  3239. */
  3240. if (alloc_flags & ALLOC_OOM)
  3241. min -= min / 2;
  3242. else
  3243. min -= min / 4;
  3244. }
  3245. /*
  3246. * Check watermarks for an order-0 allocation request. If these
  3247. * are not met, then a high-order request also cannot go ahead
  3248. * even if a suitable page happened to be free.
  3249. */
  3250. if (free_pages <= min + z->lowmem_reserve[highest_zoneidx])
  3251. return false;
  3252. /* If this is an order-0 request then the watermark is fine */
  3253. if (!order)
  3254. return true;
  3255. /* For a high-order request, check at least one suitable page is free */
  3256. for (o = order; o < MAX_ORDER; o++) {
  3257. struct free_area *area = &z->free_area[o];
  3258. int mt;
  3259. if (!area->nr_free)
  3260. continue;
  3261. for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) {
  3262. #ifdef CONFIG_CMA
  3263. /*
  3264. * Note that this check is needed only
  3265. * when MIGRATE_CMA < MIGRATE_PCPTYPES.
  3266. */
  3267. if (mt == MIGRATE_CMA)
  3268. continue;
  3269. #endif
  3270. if (!free_area_empty(area, mt))
  3271. return true;
  3272. }
  3273. #ifdef CONFIG_CMA
  3274. if ((alloc_flags & ALLOC_CMA) &&
  3275. !free_area_empty(area, MIGRATE_CMA)) {
  3276. return true;
  3277. }
  3278. #endif
  3279. if (alloc_harder && !free_area_empty(area, MIGRATE_HIGHATOMIC))
  3280. return true;
  3281. }
  3282. return false;
  3283. }
  3284. bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
  3285. int highest_zoneidx, unsigned int alloc_flags)
  3286. {
  3287. return __zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags,
  3288. zone_page_state(z, NR_FREE_PAGES));
  3289. }
  3290. EXPORT_SYMBOL_GPL(zone_watermark_ok);
  3291. static inline bool zone_watermark_fast(struct zone *z, unsigned int order,
  3292. unsigned long mark, int highest_zoneidx,
  3293. unsigned int alloc_flags, gfp_t gfp_mask)
  3294. {
  3295. long free_pages;
  3296. free_pages = zone_page_state(z, NR_FREE_PAGES);
  3297. /*
  3298. * Fast check for order-0 only. If this fails then the reserves
  3299. * need to be calculated.
  3300. */
  3301. if (!order) {
  3302. long fast_free;
  3303. fast_free = free_pages;
  3304. fast_free -= __zone_watermark_unusable_free(z, 0, alloc_flags);
  3305. if (fast_free > mark + z->lowmem_reserve[highest_zoneidx])
  3306. return true;
  3307. }
  3308. if (__zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags,
  3309. free_pages))
  3310. return true;
  3311. /*
  3312. * Ignore watermark boosting for GFP_ATOMIC order-0 allocations
  3313. * when checking the min watermark. The min watermark is the
  3314. * point where boosting is ignored so that kswapd is woken up
  3315. * when below the low watermark.
  3316. */
  3317. if (unlikely(!order && (gfp_mask & __GFP_ATOMIC) && z->watermark_boost
  3318. && ((alloc_flags & ALLOC_WMARK_MASK) == WMARK_MIN))) {
  3319. mark = z->_watermark[WMARK_MIN];
  3320. return __zone_watermark_ok(z, order, mark, highest_zoneidx,
  3321. alloc_flags, free_pages);
  3322. }
  3323. return false;
  3324. }
  3325. bool zone_watermark_ok_safe(struct zone *z, unsigned int order,
  3326. unsigned long mark, int highest_zoneidx)
  3327. {
  3328. long free_pages = zone_page_state(z, NR_FREE_PAGES);
  3329. if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark)
  3330. free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES);
  3331. return __zone_watermark_ok(z, order, mark, highest_zoneidx, 0,
  3332. free_pages);
  3333. }
  3334. EXPORT_SYMBOL_GPL(zone_watermark_ok_safe);
  3335. #ifdef CONFIG_NUMA
  3336. static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
  3337. {
  3338. return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <=
  3339. node_reclaim_distance;
  3340. }
  3341. #else /* CONFIG_NUMA */
  3342. static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
  3343. {
  3344. return true;
  3345. }
  3346. #endif /* CONFIG_NUMA */
  3347. /*
  3348. * The restriction on ZONE_DMA32 as being a suitable zone to use to avoid
  3349. * fragmentation is subtle. If the preferred zone was HIGHMEM then
  3350. * premature use of a lower zone may cause lowmem pressure problems that
  3351. * are worse than fragmentation. If the next zone is ZONE_DMA then it is
  3352. * probably too small. It only makes sense to spread allocations to avoid
  3353. * fragmentation between the Normal and DMA32 zones.
  3354. */
  3355. static inline unsigned int
  3356. alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask)
  3357. {
  3358. unsigned int alloc_flags;
  3359. /*
  3360. * __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD
  3361. * to save a branch.
  3362. */
  3363. alloc_flags = (__force int) (gfp_mask & __GFP_KSWAPD_RECLAIM);
  3364. #ifdef CONFIG_ZONE_DMA32
  3365. if (!zone)
  3366. return alloc_flags;
  3367. if (zone_idx(zone) != ZONE_NORMAL)
  3368. return alloc_flags;
  3369. /*
  3370. * If ZONE_DMA32 exists, assume it is the one after ZONE_NORMAL and
  3371. * the pointer is within zone->zone_pgdat->node_zones[]. Also assume
  3372. * on UMA that if Normal is populated then so is DMA32.
  3373. */
  3374. BUILD_BUG_ON(ZONE_NORMAL - ZONE_DMA32 != 1);
  3375. if (nr_online_nodes > 1 && !populated_zone(--zone))
  3376. return alloc_flags;
  3377. alloc_flags |= ALLOC_NOFRAGMENT;
  3378. #endif /* CONFIG_ZONE_DMA32 */
  3379. return alloc_flags;
  3380. }
  3381. static inline unsigned int current_alloc_flags(gfp_t gfp_mask,
  3382. unsigned int alloc_flags)
  3383. {
  3384. #ifdef CONFIG_CMA
  3385. unsigned int pflags = current->flags;
  3386. if (!(pflags & PF_MEMALLOC_NOCMA) &&
  3387. gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE &&
  3388. gfp_mask & __GFP_CMA)
  3389. alloc_flags |= ALLOC_CMA;
  3390. #endif
  3391. return alloc_flags;
  3392. }
  3393. /*
  3394. * get_page_from_freelist goes through the zonelist trying to allocate
  3395. * a page.
  3396. */
  3397. static struct page *
  3398. get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
  3399. const struct alloc_context *ac)
  3400. {
  3401. struct zoneref *z;
  3402. struct zone *zone;
  3403. struct pglist_data *last_pgdat_dirty_limit = NULL;
  3404. bool no_fallback;
  3405. retry:
  3406. /*
  3407. * Scan zonelist, looking for a zone with enough free.
  3408. * See also __cpuset_node_allowed() comment in kernel/cpuset.c.
  3409. */
  3410. no_fallback = alloc_flags & ALLOC_NOFRAGMENT;
  3411. z = ac->preferred_zoneref;
  3412. for_next_zone_zonelist_nodemask(zone, z, ac->highest_zoneidx,
  3413. ac->nodemask) {
  3414. struct page *page;
  3415. unsigned long mark;
  3416. if (cpusets_enabled() &&
  3417. (alloc_flags & ALLOC_CPUSET) &&
  3418. !__cpuset_zone_allowed(zone, gfp_mask))
  3419. continue;
  3420. /*
  3421. * When allocating a page cache page for writing, we
  3422. * want to get it from a node that is within its dirty
  3423. * limit, such that no single node holds more than its
  3424. * proportional share of globally allowed dirty pages.
  3425. * The dirty limits take into account the node's
  3426. * lowmem reserves and high watermark so that kswapd
  3427. * should be able to balance it without having to
  3428. * write pages from its LRU list.
  3429. *
  3430. * XXX: For now, allow allocations to potentially
  3431. * exceed the per-node dirty limit in the slowpath
  3432. * (spread_dirty_pages unset) before going into reclaim,
  3433. * which is important when on a NUMA setup the allowed
  3434. * nodes are together not big enough to reach the
  3435. * global limit. The proper fix for these situations
  3436. * will require awareness of nodes in the
  3437. * dirty-throttling and the flusher threads.
  3438. */
  3439. if (ac->spread_dirty_pages) {
  3440. if (last_pgdat_dirty_limit == zone->zone_pgdat)
  3441. continue;
  3442. if (!node_dirty_ok(zone->zone_pgdat)) {
  3443. last_pgdat_dirty_limit = zone->zone_pgdat;
  3444. continue;
  3445. }
  3446. }
  3447. if (no_fallback && nr_online_nodes > 1 &&
  3448. zone != ac->preferred_zoneref->zone) {
  3449. int local_nid;
  3450. /*
  3451. * If moving to a remote node, retry but allow
  3452. * fragmenting fallbacks. Locality is more important
  3453. * than fragmentation avoidance.
  3454. */
  3455. local_nid = zone_to_nid(ac->preferred_zoneref->zone);
  3456. if (zone_to_nid(zone) != local_nid) {
  3457. alloc_flags &= ~ALLOC_NOFRAGMENT;
  3458. goto retry;
  3459. }
  3460. }
  3461. mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK);
  3462. if (!zone_watermark_fast(zone, order, mark,
  3463. ac->highest_zoneidx, alloc_flags,
  3464. gfp_mask)) {
  3465. int ret;
  3466. #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
  3467. /*
  3468. * Watermark failed for this zone, but see if we can
  3469. * grow this zone if it contains deferred pages.
  3470. */
  3471. if (static_branch_unlikely(&deferred_pages)) {
  3472. if (_deferred_grow_zone(zone, order))
  3473. goto try_this_zone;
  3474. }
  3475. #endif
  3476. /* Checked here to keep the fast path fast */
  3477. BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
  3478. if (alloc_flags & ALLOC_NO_WATERMARKS)
  3479. goto try_this_zone;
  3480. if (node_reclaim_mode == 0 ||
  3481. !zone_allows_reclaim(ac->preferred_zoneref->zone, zone))
  3482. continue;
  3483. ret = node_reclaim(zone->zone_pgdat, gfp_mask, order);
  3484. switch (ret) {
  3485. case NODE_RECLAIM_NOSCAN:
  3486. /* did not scan */
  3487. continue;
  3488. case NODE_RECLAIM_FULL:
  3489. /* scanned but unreclaimable */
  3490. continue;
  3491. default:
  3492. /* did we reclaim enough */
  3493. if (zone_watermark_ok(zone, order, mark,
  3494. ac->highest_zoneidx, alloc_flags))
  3495. goto try_this_zone;
  3496. continue;
  3497. }
  3498. }
  3499. try_this_zone:
  3500. page = rmqueue(ac->preferred_zoneref->zone, zone, order,
  3501. gfp_mask, alloc_flags, ac->migratetype);
  3502. if (page) {
  3503. prep_new_page(page, order, gfp_mask, alloc_flags);
  3504. /*
  3505. * If this is a high-order atomic allocation then check
  3506. * if the pageblock should be reserved for the future
  3507. */
  3508. if (unlikely(order && (alloc_flags & ALLOC_HARDER)))
  3509. reserve_highatomic_pageblock(page, zone, order);
  3510. return page;
  3511. } else {
  3512. #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
  3513. /* Try again if zone has deferred pages */
  3514. if (static_branch_unlikely(&deferred_pages)) {
  3515. if (_deferred_grow_zone(zone, order))
  3516. goto try_this_zone;
  3517. }
  3518. #endif
  3519. }
  3520. }
  3521. /*
  3522. * It's possible on a UMA machine to get through all zones that are
  3523. * fragmented. If avoiding fragmentation, reset and try again.
  3524. */
  3525. if (no_fallback) {
  3526. alloc_flags &= ~ALLOC_NOFRAGMENT;
  3527. goto retry;
  3528. }
  3529. return NULL;
  3530. }
  3531. static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask)
  3532. {
  3533. unsigned int filter = SHOW_MEM_FILTER_NODES;
  3534. /*
  3535. * This documents exceptions given to allocations in certain
  3536. * contexts that are allowed to allocate outside current's set
  3537. * of allowed nodes.
  3538. */
  3539. if (!(gfp_mask & __GFP_NOMEMALLOC))
  3540. if (tsk_is_oom_victim(current) ||
  3541. (current->flags & (PF_MEMALLOC | PF_EXITING)))
  3542. filter &= ~SHOW_MEM_FILTER_NODES;
  3543. if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM))
  3544. filter &= ~SHOW_MEM_FILTER_NODES;
  3545. show_mem(filter, nodemask);
  3546. }
  3547. void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...)
  3548. {
  3549. struct va_format vaf;
  3550. va_list args;
  3551. static DEFINE_RATELIMIT_STATE(nopage_rs, 10*HZ, 1);
  3552. if ((gfp_mask & __GFP_NOWARN) ||
  3553. !__ratelimit(&nopage_rs) ||
  3554. ((gfp_mask & __GFP_DMA) && !has_managed_dma()))
  3555. return;
  3556. va_start(args, fmt);
  3557. vaf.fmt = fmt;
  3558. vaf.va = &args;
  3559. pr_warn("%s: %pV, mode:%#x(%pGg), nodemask=%*pbl",
  3560. current->comm, &vaf, gfp_mask, &gfp_mask,
  3561. nodemask_pr_args(nodemask));
  3562. va_end(args);
  3563. cpuset_print_current_mems_allowed();
  3564. pr_cont("\n");
  3565. dump_stack();
  3566. warn_alloc_show_mem(gfp_mask, nodemask);
  3567. }
  3568. static inline struct page *
  3569. __alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order,
  3570. unsigned int alloc_flags,
  3571. const struct alloc_context *ac)
  3572. {
  3573. struct page *page;
  3574. page = get_page_from_freelist(gfp_mask, order,
  3575. alloc_flags|ALLOC_CPUSET, ac);
  3576. /*
  3577. * fallback to ignore cpuset restriction if our nodes
  3578. * are depleted
  3579. */
  3580. if (!page)
  3581. page = get_page_from_freelist(gfp_mask, order,
  3582. alloc_flags, ac);
  3583. return page;
  3584. }
  3585. static inline struct page *
  3586. __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
  3587. const struct alloc_context *ac, unsigned long *did_some_progress)
  3588. {
  3589. struct oom_control oc = {
  3590. .zonelist = ac->zonelist,
  3591. .nodemask = ac->nodemask,
  3592. .memcg = NULL,
  3593. .gfp_mask = gfp_mask,
  3594. .order = order,
  3595. };
  3596. struct page *page;
  3597. *did_some_progress = 0;
  3598. /*
  3599. * Acquire the oom lock. If that fails, somebody else is
  3600. * making progress for us.
  3601. */
  3602. if (!mutex_trylock(&oom_lock)) {
  3603. *did_some_progress = 1;
  3604. schedule_timeout_uninterruptible(1);
  3605. return NULL;
  3606. }
  3607. /*
  3608. * Go through the zonelist yet one more time, keep very high watermark
  3609. * here, this is only to catch a parallel oom killing, we must fail if
  3610. * we're still under heavy pressure. But make sure that this reclaim
  3611. * attempt shall not depend on __GFP_DIRECT_RECLAIM && !__GFP_NORETRY
  3612. * allocation which will never fail due to oom_lock already held.
  3613. */
  3614. page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) &
  3615. ~__GFP_DIRECT_RECLAIM, order,
  3616. ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac);
  3617. if (page)
  3618. goto out;
  3619. /* Coredumps can quickly deplete all memory reserves */
  3620. if (current->flags & PF_DUMPCORE)
  3621. goto out;
  3622. /* The OOM killer will not help higher order allocs */
  3623. if (order > PAGE_ALLOC_COSTLY_ORDER)
  3624. goto out;
  3625. /*
  3626. * We have already exhausted all our reclaim opportunities without any
  3627. * success so it is time to admit defeat. We will skip the OOM killer
  3628. * because it is very likely that the caller has a more reasonable
  3629. * fallback than shooting a random task.
  3630. *
  3631. * The OOM killer may not free memory on a specific node.
  3632. */
  3633. if (gfp_mask & (__GFP_RETRY_MAYFAIL | __GFP_THISNODE))
  3634. goto out;
  3635. /* The OOM killer does not needlessly kill tasks for lowmem */
  3636. if (ac->highest_zoneidx < ZONE_NORMAL)
  3637. goto out;
  3638. if (pm_suspended_storage())
  3639. goto out;
  3640. /*
  3641. * XXX: GFP_NOFS allocations should rather fail than rely on
  3642. * other request to make a forward progress.
  3643. * We are in an unfortunate situation where out_of_memory cannot
  3644. * do much for this context but let's try it to at least get
  3645. * access to memory reserved if the current task is killed (see
  3646. * out_of_memory). Once filesystems are ready to handle allocation
  3647. * failures more gracefully we should just bail out here.
  3648. */
  3649. /* Exhausted what can be done so it's blame time */
  3650. if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) {
  3651. *did_some_progress = 1;
  3652. /*
  3653. * Help non-failing allocations by giving them access to memory
  3654. * reserves
  3655. */
  3656. if (gfp_mask & __GFP_NOFAIL)
  3657. page = __alloc_pages_cpuset_fallback(gfp_mask, order,
  3658. ALLOC_NO_WATERMARKS, ac);
  3659. }
  3660. out:
  3661. mutex_unlock(&oom_lock);
  3662. return page;
  3663. }
  3664. /*
  3665. * Maximum number of compaction retries wit a progress before OOM
  3666. * killer is consider as the only way to move forward.
  3667. */
  3668. #define MAX_COMPACT_RETRIES 16
  3669. #ifdef CONFIG_COMPACTION
  3670. /* Try memory compaction for high-order allocations before reclaim */
  3671. static struct page *
  3672. __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
  3673. unsigned int alloc_flags, const struct alloc_context *ac,
  3674. enum compact_priority prio, enum compact_result *compact_result)
  3675. {
  3676. struct page *page = NULL;
  3677. unsigned long pflags;
  3678. unsigned int noreclaim_flag;
  3679. if (!order)
  3680. return NULL;
  3681. psi_memstall_enter(&pflags);
  3682. noreclaim_flag = memalloc_noreclaim_save();
  3683. *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac,
  3684. prio, &page);
  3685. memalloc_noreclaim_restore(noreclaim_flag);
  3686. psi_memstall_leave(&pflags);
  3687. /*
  3688. * At least in one zone compaction wasn't deferred or skipped, so let's
  3689. * count a compaction stall
  3690. */
  3691. count_vm_event(COMPACTSTALL);
  3692. /* Prep a captured page if available */
  3693. if (page)
  3694. prep_new_page(page, order, gfp_mask, alloc_flags);
  3695. /* Try get a page from the freelist if available */
  3696. if (!page)
  3697. page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
  3698. if (page) {
  3699. struct zone *zone = page_zone(page);
  3700. zone->compact_blockskip_flush = false;
  3701. compaction_defer_reset(zone, order, true);
  3702. count_vm_event(COMPACTSUCCESS);
  3703. return page;
  3704. }
  3705. /*
  3706. * It's bad if compaction run occurs and fails. The most likely reason
  3707. * is that pages exist, but not enough to satisfy watermarks.
  3708. */
  3709. count_vm_event(COMPACTFAIL);
  3710. cond_resched();
  3711. return NULL;
  3712. }
  3713. static inline bool
  3714. should_compact_retry(struct alloc_context *ac, int order, int alloc_flags,
  3715. enum compact_result compact_result,
  3716. enum compact_priority *compact_priority,
  3717. int *compaction_retries)
  3718. {
  3719. int max_retries = MAX_COMPACT_RETRIES;
  3720. int min_priority;
  3721. bool ret = false;
  3722. int retries = *compaction_retries;
  3723. enum compact_priority priority = *compact_priority;
  3724. if (!order)
  3725. return false;
  3726. if (compaction_made_progress(compact_result))
  3727. (*compaction_retries)++;
  3728. /*
  3729. * compaction considers all the zone as desperately out of memory
  3730. * so it doesn't really make much sense to retry except when the
  3731. * failure could be caused by insufficient priority
  3732. */
  3733. if (compaction_failed(compact_result))
  3734. goto check_priority;
  3735. /*
  3736. * compaction was skipped because there are not enough order-0 pages
  3737. * to work with, so we retry only if it looks like reclaim can help.
  3738. */
  3739. if (compaction_needs_reclaim(compact_result)) {
  3740. ret = compaction_zonelist_suitable(ac, order, alloc_flags);
  3741. goto out;
  3742. }
  3743. /*
  3744. * make sure the compaction wasn't deferred or didn't bail out early
  3745. * due to locks contention before we declare that we should give up.
  3746. * But the next retry should use a higher priority if allowed, so
  3747. * we don't just keep bailing out endlessly.
  3748. */
  3749. if (compaction_withdrawn(compact_result)) {
  3750. goto check_priority;
  3751. }
  3752. /*
  3753. * !costly requests are much more important than __GFP_RETRY_MAYFAIL
  3754. * costly ones because they are de facto nofail and invoke OOM
  3755. * killer to move on while costly can fail and users are ready
  3756. * to cope with that. 1/4 retries is rather arbitrary but we
  3757. * would need much more detailed feedback from compaction to
  3758. * make a better decision.
  3759. */
  3760. if (order > PAGE_ALLOC_COSTLY_ORDER)
  3761. max_retries /= 4;
  3762. if (*compaction_retries <= max_retries) {
  3763. ret = true;
  3764. goto out;
  3765. }
  3766. /*
  3767. * Make sure there are attempts at the highest priority if we exhausted
  3768. * all retries or failed at the lower priorities.
  3769. */
  3770. check_priority:
  3771. min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ?
  3772. MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY;
  3773. if (*compact_priority > min_priority) {
  3774. (*compact_priority)--;
  3775. *compaction_retries = 0;
  3776. ret = true;
  3777. }
  3778. out:
  3779. trace_compact_retry(order, priority, compact_result, retries, max_retries, ret);
  3780. return ret;
  3781. }
  3782. #else
  3783. static inline struct page *
  3784. __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
  3785. unsigned int alloc_flags, const struct alloc_context *ac,
  3786. enum compact_priority prio, enum compact_result *compact_result)
  3787. {
  3788. *compact_result = COMPACT_SKIPPED;
  3789. return NULL;
  3790. }
  3791. static inline bool
  3792. should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags,
  3793. enum compact_result compact_result,
  3794. enum compact_priority *compact_priority,
  3795. int *compaction_retries)
  3796. {
  3797. struct zone *zone;
  3798. struct zoneref *z;
  3799. if (!order || order > PAGE_ALLOC_COSTLY_ORDER)
  3800. return false;
  3801. /*
  3802. * There are setups with compaction disabled which would prefer to loop
  3803. * inside the allocator rather than hit the oom killer prematurely.
  3804. * Let's give them a good hope and keep retrying while the order-0
  3805. * watermarks are OK.
  3806. */
  3807. for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
  3808. ac->highest_zoneidx, ac->nodemask) {
  3809. if (zone_watermark_ok(zone, 0, min_wmark_pages(zone),
  3810. ac->highest_zoneidx, alloc_flags))
  3811. return true;
  3812. }
  3813. return false;
  3814. }
  3815. #endif /* CONFIG_COMPACTION */
  3816. #ifdef CONFIG_LOCKDEP
  3817. static struct lockdep_map __fs_reclaim_map =
  3818. STATIC_LOCKDEP_MAP_INIT("fs_reclaim", &__fs_reclaim_map);
  3819. static bool __need_fs_reclaim(gfp_t gfp_mask)
  3820. {
  3821. gfp_mask = current_gfp_context(gfp_mask);
  3822. /* no reclaim without waiting on it */
  3823. if (!(gfp_mask & __GFP_DIRECT_RECLAIM))
  3824. return false;
  3825. /* this guy won't enter reclaim */
  3826. if (current->flags & PF_MEMALLOC)
  3827. return false;
  3828. /* We're only interested __GFP_FS allocations for now */
  3829. if (!(gfp_mask & __GFP_FS))
  3830. return false;
  3831. if (gfp_mask & __GFP_NOLOCKDEP)
  3832. return false;
  3833. return true;
  3834. }
  3835. void __fs_reclaim_acquire(void)
  3836. {
  3837. lock_map_acquire(&__fs_reclaim_map);
  3838. }
  3839. void __fs_reclaim_release(void)
  3840. {
  3841. lock_map_release(&__fs_reclaim_map);
  3842. }
  3843. void fs_reclaim_acquire(gfp_t gfp_mask)
  3844. {
  3845. if (__need_fs_reclaim(gfp_mask))
  3846. __fs_reclaim_acquire();
  3847. }
  3848. EXPORT_SYMBOL_GPL(fs_reclaim_acquire);
  3849. void fs_reclaim_release(gfp_t gfp_mask)
  3850. {
  3851. if (__need_fs_reclaim(gfp_mask))
  3852. __fs_reclaim_release();
  3853. }
  3854. EXPORT_SYMBOL_GPL(fs_reclaim_release);
  3855. #endif
  3856. /* Perform direct synchronous page reclaim */
  3857. static unsigned long
  3858. __perform_reclaim(gfp_t gfp_mask, unsigned int order,
  3859. const struct alloc_context *ac)
  3860. {
  3861. unsigned int noreclaim_flag;
  3862. unsigned long progress;
  3863. cond_resched();
  3864. /* We now go into synchronous reclaim */
  3865. cpuset_memory_pressure_bump();
  3866. fs_reclaim_acquire(gfp_mask);
  3867. noreclaim_flag = memalloc_noreclaim_save();
  3868. progress = try_to_free_pages(ac->zonelist, order, gfp_mask,
  3869. ac->nodemask);
  3870. memalloc_noreclaim_restore(noreclaim_flag);
  3871. fs_reclaim_release(gfp_mask);
  3872. cond_resched();
  3873. return progress;
  3874. }
  3875. /* The really slow allocator path where we enter direct reclaim */
  3876. static inline struct page *
  3877. __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
  3878. unsigned int alloc_flags, const struct alloc_context *ac,
  3879. unsigned long *did_some_progress)
  3880. {
  3881. struct page *page = NULL;
  3882. unsigned long pflags;
  3883. bool drained = false;
  3884. bool skip_pcp_drain = false;
  3885. psi_memstall_enter(&pflags);
  3886. *did_some_progress = __perform_reclaim(gfp_mask, order, ac);
  3887. if (unlikely(!(*did_some_progress)))
  3888. goto out;
  3889. retry:
  3890. page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
  3891. /*
  3892. * If an allocation failed after direct reclaim, it could be because
  3893. * pages are pinned on the per-cpu lists or in high alloc reserves.
  3894. * Shrink them and try again
  3895. */
  3896. if (!page && !drained) {
  3897. unreserve_highatomic_pageblock(ac, false);
  3898. trace_android_vh_drain_all_pages_bypass(gfp_mask, order,
  3899. alloc_flags, ac->migratetype, *did_some_progress, &skip_pcp_drain);
  3900. if (!skip_pcp_drain)
  3901. drain_all_pages(NULL);
  3902. drained = true;
  3903. goto retry;
  3904. }
  3905. out:
  3906. psi_memstall_leave(&pflags);
  3907. return page;
  3908. }
  3909. static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask,
  3910. const struct alloc_context *ac)
  3911. {
  3912. struct zoneref *z;
  3913. struct zone *zone;
  3914. pg_data_t *last_pgdat = NULL;
  3915. enum zone_type highest_zoneidx = ac->highest_zoneidx;
  3916. for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, highest_zoneidx,
  3917. ac->nodemask) {
  3918. if (last_pgdat != zone->zone_pgdat)
  3919. wakeup_kswapd(zone, gfp_mask, order, highest_zoneidx);
  3920. last_pgdat = zone->zone_pgdat;
  3921. }
  3922. }
  3923. static inline unsigned int
  3924. gfp_to_alloc_flags(gfp_t gfp_mask)
  3925. {
  3926. unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
  3927. /*
  3928. * __GFP_HIGH is assumed to be the same as ALLOC_HIGH
  3929. * and __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD
  3930. * to save two branches.
  3931. */
  3932. BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH);
  3933. BUILD_BUG_ON(__GFP_KSWAPD_RECLAIM != (__force gfp_t) ALLOC_KSWAPD);
  3934. /*
  3935. * The caller may dip into page reserves a bit more if the caller
  3936. * cannot run direct reclaim, or if the caller has realtime scheduling
  3937. * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will
  3938. * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH).
  3939. */
  3940. alloc_flags |= (__force int)
  3941. (gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM));
  3942. if (gfp_mask & __GFP_ATOMIC) {
  3943. /*
  3944. * Not worth trying to allocate harder for __GFP_NOMEMALLOC even
  3945. * if it can't schedule.
  3946. */
  3947. if (!(gfp_mask & __GFP_NOMEMALLOC))
  3948. alloc_flags |= ALLOC_HARDER;
  3949. /*
  3950. * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the
  3951. * comment for __cpuset_node_allowed().
  3952. */
  3953. alloc_flags &= ~ALLOC_CPUSET;
  3954. } else if (unlikely(rt_task(current)) && !in_interrupt())
  3955. alloc_flags |= ALLOC_HARDER;
  3956. alloc_flags = current_alloc_flags(gfp_mask, alloc_flags);
  3957. return alloc_flags;
  3958. }
  3959. static bool oom_reserves_allowed(struct task_struct *tsk)
  3960. {
  3961. if (!tsk_is_oom_victim(tsk))
  3962. return false;
  3963. /*
  3964. * !MMU doesn't have oom reaper so give access to memory reserves
  3965. * only to the thread with TIF_MEMDIE set
  3966. */
  3967. if (!IS_ENABLED(CONFIG_MMU) && !test_thread_flag(TIF_MEMDIE))
  3968. return false;
  3969. return true;
  3970. }
  3971. /*
  3972. * Distinguish requests which really need access to full memory
  3973. * reserves from oom victims which can live with a portion of it
  3974. */
  3975. static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask)
  3976. {
  3977. if (unlikely(gfp_mask & __GFP_NOMEMALLOC))
  3978. return 0;
  3979. if (gfp_mask & __GFP_MEMALLOC)
  3980. return ALLOC_NO_WATERMARKS;
  3981. if (in_serving_softirq() && (current->flags & PF_MEMALLOC))
  3982. return ALLOC_NO_WATERMARKS;
  3983. if (!in_interrupt()) {
  3984. if (current->flags & PF_MEMALLOC)
  3985. return ALLOC_NO_WATERMARKS;
  3986. else if (oom_reserves_allowed(current))
  3987. return ALLOC_OOM;
  3988. }
  3989. return 0;
  3990. }
  3991. bool gfp_pfmemalloc_allowed(gfp_t gfp_mask)
  3992. {
  3993. return !!__gfp_pfmemalloc_flags(gfp_mask);
  3994. }
  3995. /*
  3996. * Checks whether it makes sense to retry the reclaim to make a forward progress
  3997. * for the given allocation request.
  3998. *
  3999. * We give up when we either have tried MAX_RECLAIM_RETRIES in a row
  4000. * without success, or when we couldn't even meet the watermark if we
  4001. * reclaimed all remaining pages on the LRU lists.
  4002. *
  4003. * Returns true if a retry is viable or false to enter the oom path.
  4004. */
  4005. static inline bool
  4006. should_reclaim_retry(gfp_t gfp_mask, unsigned order,
  4007. struct alloc_context *ac, int alloc_flags,
  4008. bool did_some_progress, int *no_progress_loops)
  4009. {
  4010. struct zone *zone;
  4011. struct zoneref *z;
  4012. bool ret = false;
  4013. /*
  4014. * Costly allocations might have made a progress but this doesn't mean
  4015. * their order will become available due to high fragmentation so
  4016. * always increment the no progress counter for them
  4017. */
  4018. if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER)
  4019. *no_progress_loops = 0;
  4020. else
  4021. (*no_progress_loops)++;
  4022. /*
  4023. * Make sure we converge to OOM if we cannot make any progress
  4024. * several times in the row.
  4025. */
  4026. if (*no_progress_loops > MAX_RECLAIM_RETRIES) {
  4027. /* Before OOM, exhaust highatomic_reserve */
  4028. return unreserve_highatomic_pageblock(ac, true);
  4029. }
  4030. /*
  4031. * Keep reclaiming pages while there is a chance this will lead
  4032. * somewhere. If none of the target zones can satisfy our allocation
  4033. * request even if all reclaimable pages are considered then we are
  4034. * screwed and have to go OOM.
  4035. */
  4036. for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
  4037. ac->highest_zoneidx, ac->nodemask) {
  4038. unsigned long available;
  4039. unsigned long reclaimable;
  4040. unsigned long min_wmark = min_wmark_pages(zone);
  4041. bool wmark;
  4042. available = reclaimable = zone_reclaimable_pages(zone);
  4043. available += zone_page_state_snapshot(zone, NR_FREE_PAGES);
  4044. /*
  4045. * Would the allocation succeed if we reclaimed all
  4046. * reclaimable pages?
  4047. */
  4048. wmark = __zone_watermark_ok(zone, order, min_wmark,
  4049. ac->highest_zoneidx, alloc_flags, available);
  4050. trace_reclaim_retry_zone(z, order, reclaimable,
  4051. available, min_wmark, *no_progress_loops, wmark);
  4052. if (wmark) {
  4053. /*
  4054. * If we didn't make any progress and have a lot of
  4055. * dirty + writeback pages then we should wait for
  4056. * an IO to complete to slow down the reclaim and
  4057. * prevent from pre mature OOM
  4058. */
  4059. if (!did_some_progress) {
  4060. unsigned long write_pending;
  4061. write_pending = zone_page_state_snapshot(zone,
  4062. NR_ZONE_WRITE_PENDING);
  4063. if (2 * write_pending > reclaimable) {
  4064. congestion_wait(BLK_RW_ASYNC, HZ/10);
  4065. return true;
  4066. }
  4067. }
  4068. ret = true;
  4069. goto out;
  4070. }
  4071. }
  4072. out:
  4073. /*
  4074. * Memory allocation/reclaim might be called from a WQ context and the
  4075. * current implementation of the WQ concurrency control doesn't
  4076. * recognize that a particular WQ is congested if the worker thread is
  4077. * looping without ever sleeping. Therefore we have to do a short sleep
  4078. * here rather than calling cond_resched().
  4079. */
  4080. if (current->flags & PF_WQ_WORKER)
  4081. schedule_timeout_uninterruptible(1);
  4082. else
  4083. cond_resched();
  4084. return ret;
  4085. }
  4086. static inline bool
  4087. check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac)
  4088. {
  4089. /*
  4090. * It's possible that cpuset's mems_allowed and the nodemask from
  4091. * mempolicy don't intersect. This should be normally dealt with by
  4092. * policy_nodemask(), but it's possible to race with cpuset update in
  4093. * such a way the check therein was true, and then it became false
  4094. * before we got our cpuset_mems_cookie here.
  4095. * This assumes that for all allocations, ac->nodemask can come only
  4096. * from MPOL_BIND mempolicy (whose documented semantics is to be ignored
  4097. * when it does not intersect with the cpuset restrictions) or the
  4098. * caller can deal with a violated nodemask.
  4099. */
  4100. if (cpusets_enabled() && ac->nodemask &&
  4101. !cpuset_nodemask_valid_mems_allowed(ac->nodemask)) {
  4102. ac->nodemask = NULL;
  4103. return true;
  4104. }
  4105. /*
  4106. * When updating a task's mems_allowed or mempolicy nodemask, it is
  4107. * possible to race with parallel threads in such a way that our
  4108. * allocation can fail while the mask is being updated. If we are about
  4109. * to fail, check if the cpuset changed during allocation and if so,
  4110. * retry.
  4111. */
  4112. if (read_mems_allowed_retry(cpuset_mems_cookie))
  4113. return true;
  4114. return false;
  4115. }
  4116. static inline struct page *
  4117. __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
  4118. struct alloc_context *ac)
  4119. {
  4120. bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM;
  4121. const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER;
  4122. struct page *page = NULL;
  4123. unsigned int alloc_flags;
  4124. unsigned long did_some_progress;
  4125. enum compact_priority compact_priority;
  4126. enum compact_result compact_result;
  4127. int compaction_retries;
  4128. int no_progress_loops;
  4129. unsigned int cpuset_mems_cookie;
  4130. int reserve_flags;
  4131. /*
  4132. * We also sanity check to catch abuse of atomic reserves being used by
  4133. * callers that are not in atomic context.
  4134. */
  4135. if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) ==
  4136. (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)))
  4137. gfp_mask &= ~__GFP_ATOMIC;
  4138. retry_cpuset:
  4139. compaction_retries = 0;
  4140. no_progress_loops = 0;
  4141. compact_priority = DEF_COMPACT_PRIORITY;
  4142. cpuset_mems_cookie = read_mems_allowed_begin();
  4143. /*
  4144. * The fast path uses conservative alloc_flags to succeed only until
  4145. * kswapd needs to be woken up, and to avoid the cost of setting up
  4146. * alloc_flags precisely. So we do that now.
  4147. */
  4148. alloc_flags = gfp_to_alloc_flags(gfp_mask);
  4149. /*
  4150. * We need to recalculate the starting point for the zonelist iterator
  4151. * because we might have used different nodemask in the fast path, or
  4152. * there was a cpuset modification and we are retrying - otherwise we
  4153. * could end up iterating over non-eligible zones endlessly.
  4154. */
  4155. ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
  4156. ac->highest_zoneidx, ac->nodemask);
  4157. if (!ac->preferred_zoneref->zone)
  4158. goto nopage;
  4159. if (alloc_flags & ALLOC_KSWAPD)
  4160. wake_all_kswapds(order, gfp_mask, ac);
  4161. /*
  4162. * The adjusted alloc_flags might result in immediate success, so try
  4163. * that first
  4164. */
  4165. page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
  4166. if (page)
  4167. goto got_pg;
  4168. /*
  4169. * For costly allocations, try direct compaction first, as it's likely
  4170. * that we have enough base pages and don't need to reclaim. For non-
  4171. * movable high-order allocations, do that as well, as compaction will
  4172. * try prevent permanent fragmentation by migrating from blocks of the
  4173. * same migratetype.
  4174. * Don't try this for allocations that are allowed to ignore
  4175. * watermarks, as the ALLOC_NO_WATERMARKS attempt didn't yet happen.
  4176. */
  4177. if (can_direct_reclaim &&
  4178. (costly_order ||
  4179. (order > 0 && ac->migratetype != MIGRATE_MOVABLE))
  4180. && !gfp_pfmemalloc_allowed(gfp_mask)) {
  4181. page = __alloc_pages_direct_compact(gfp_mask, order,
  4182. alloc_flags, ac,
  4183. INIT_COMPACT_PRIORITY,
  4184. &compact_result);
  4185. if (page)
  4186. goto got_pg;
  4187. /*
  4188. * Checks for costly allocations with __GFP_NORETRY, which
  4189. * includes some THP page fault allocations
  4190. */
  4191. if (costly_order && (gfp_mask & __GFP_NORETRY)) {
  4192. /*
  4193. * If allocating entire pageblock(s) and compaction
  4194. * failed because all zones are below low watermarks
  4195. * or is prohibited because it recently failed at this
  4196. * order, fail immediately unless the allocator has
  4197. * requested compaction and reclaim retry.
  4198. *
  4199. * Reclaim is
  4200. * - potentially very expensive because zones are far
  4201. * below their low watermarks or this is part of very
  4202. * bursty high order allocations,
  4203. * - not guaranteed to help because isolate_freepages()
  4204. * may not iterate over freed pages as part of its
  4205. * linear scan, and
  4206. * - unlikely to make entire pageblocks free on its
  4207. * own.
  4208. */
  4209. if (compact_result == COMPACT_SKIPPED ||
  4210. compact_result == COMPACT_DEFERRED)
  4211. goto nopage;
  4212. /*
  4213. * Looks like reclaim/compaction is worth trying, but
  4214. * sync compaction could be very expensive, so keep
  4215. * using async compaction.
  4216. */
  4217. compact_priority = INIT_COMPACT_PRIORITY;
  4218. }
  4219. }
  4220. retry:
  4221. /* Ensure kswapd doesn't accidentally go to sleep as long as we loop */
  4222. if (alloc_flags & ALLOC_KSWAPD)
  4223. wake_all_kswapds(order, gfp_mask, ac);
  4224. reserve_flags = __gfp_pfmemalloc_flags(gfp_mask);
  4225. if (reserve_flags)
  4226. alloc_flags = current_alloc_flags(gfp_mask, reserve_flags);
  4227. /*
  4228. * Reset the nodemask and zonelist iterators if memory policies can be
  4229. * ignored. These allocations are high priority and system rather than
  4230. * user oriented.
  4231. */
  4232. if (!(alloc_flags & ALLOC_CPUSET) || reserve_flags) {
  4233. ac->nodemask = NULL;
  4234. ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
  4235. ac->highest_zoneidx, ac->nodemask);
  4236. }
  4237. /* Attempt with potentially adjusted zonelist and alloc_flags */
  4238. page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
  4239. if (page)
  4240. goto got_pg;
  4241. /* Caller is not willing to reclaim, we can't balance anything */
  4242. if (!can_direct_reclaim)
  4243. goto nopage;
  4244. /* Avoid recursion of direct reclaim */
  4245. if (current->flags & PF_MEMALLOC)
  4246. goto nopage;
  4247. /* Try direct reclaim and then allocating */
  4248. page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac,
  4249. &did_some_progress);
  4250. if (page)
  4251. goto got_pg;
  4252. /* Try direct compaction and then allocating */
  4253. page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac,
  4254. compact_priority, &compact_result);
  4255. if (page)
  4256. goto got_pg;
  4257. /* Do not loop if specifically requested */
  4258. if (gfp_mask & __GFP_NORETRY)
  4259. goto nopage;
  4260. /*
  4261. * Do not retry costly high order allocations unless they are
  4262. * __GFP_RETRY_MAYFAIL
  4263. */
  4264. if (costly_order && !(gfp_mask & __GFP_RETRY_MAYFAIL))
  4265. goto nopage;
  4266. if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags,
  4267. did_some_progress > 0, &no_progress_loops))
  4268. goto retry;
  4269. /*
  4270. * It doesn't make any sense to retry for the compaction if the order-0
  4271. * reclaim is not able to make any progress because the current
  4272. * implementation of the compaction depends on the sufficient amount
  4273. * of free memory (see __compaction_suitable)
  4274. */
  4275. if (did_some_progress > 0 &&
  4276. should_compact_retry(ac, order, alloc_flags,
  4277. compact_result, &compact_priority,
  4278. &compaction_retries))
  4279. goto retry;
  4280. /* Deal with possible cpuset update races before we start OOM killing */
  4281. if (check_retry_cpuset(cpuset_mems_cookie, ac))
  4282. goto retry_cpuset;
  4283. /* Reclaim has failed us, start killing things */
  4284. page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress);
  4285. if (page)
  4286. goto got_pg;
  4287. /* Avoid allocations with no watermarks from looping endlessly */
  4288. if (tsk_is_oom_victim(current) &&
  4289. (alloc_flags & ALLOC_OOM ||
  4290. (gfp_mask & __GFP_NOMEMALLOC)))
  4291. goto nopage;
  4292. /* Retry as long as the OOM killer is making progress */
  4293. if (did_some_progress) {
  4294. no_progress_loops = 0;
  4295. goto retry;
  4296. }
  4297. nopage:
  4298. /* Deal with possible cpuset update races before we fail */
  4299. if (check_retry_cpuset(cpuset_mems_cookie, ac))
  4300. goto retry_cpuset;
  4301. /*
  4302. * Make sure that __GFP_NOFAIL request doesn't leak out and make sure
  4303. * we always retry
  4304. */
  4305. if (gfp_mask & __GFP_NOFAIL) {
  4306. /*
  4307. * All existing users of the __GFP_NOFAIL are blockable, so warn
  4308. * of any new users that actually require GFP_NOWAIT
  4309. */
  4310. if (WARN_ON_ONCE(!can_direct_reclaim))
  4311. goto fail;
  4312. /*
  4313. * PF_MEMALLOC request from this context is rather bizarre
  4314. * because we cannot reclaim anything and only can loop waiting
  4315. * for somebody to do a work for us
  4316. */
  4317. WARN_ON_ONCE(current->flags & PF_MEMALLOC);
  4318. /*
  4319. * non failing costly orders are a hard requirement which we
  4320. * are not prepared for much so let's warn about these users
  4321. * so that we can identify them and convert them to something
  4322. * else.
  4323. */
  4324. WARN_ON_ONCE(order > PAGE_ALLOC_COSTLY_ORDER);
  4325. /*
  4326. * Help non-failing allocations by giving them access to memory
  4327. * reserves but do not use ALLOC_NO_WATERMARKS because this
  4328. * could deplete whole memory reserves which would just make
  4329. * the situation worse
  4330. */
  4331. page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_HARDER, ac);
  4332. if (page)
  4333. goto got_pg;
  4334. cond_resched();
  4335. goto retry;
  4336. }
  4337. fail:
  4338. warn_alloc(gfp_mask, ac->nodemask,
  4339. "page allocation failure: order:%u", order);
  4340. got_pg:
  4341. return page;
  4342. }
  4343. static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order,
  4344. int preferred_nid, nodemask_t *nodemask,
  4345. struct alloc_context *ac, gfp_t *alloc_mask,
  4346. unsigned int *alloc_flags)
  4347. {
  4348. ac->highest_zoneidx = gfp_zone(gfp_mask);
  4349. ac->zonelist = node_zonelist(preferred_nid, gfp_mask);
  4350. ac->nodemask = nodemask;
  4351. ac->migratetype = gfp_migratetype(gfp_mask);
  4352. if (cpusets_enabled()) {
  4353. *alloc_mask |= __GFP_HARDWALL;
  4354. /*
  4355. * When we are in the interrupt context, it is irrelevant
  4356. * to the current task context. It means that any node ok.
  4357. */
  4358. if (!in_interrupt() && !ac->nodemask)
  4359. ac->nodemask = &cpuset_current_mems_allowed;
  4360. else
  4361. *alloc_flags |= ALLOC_CPUSET;
  4362. }
  4363. fs_reclaim_acquire(gfp_mask);
  4364. fs_reclaim_release(gfp_mask);
  4365. might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM);
  4366. if (should_fail_alloc_page(gfp_mask, order))
  4367. return false;
  4368. *alloc_flags = current_alloc_flags(gfp_mask, *alloc_flags);
  4369. /* Dirty zone balancing only done in the fast path */
  4370. ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE);
  4371. /*
  4372. * The preferred zone is used for statistics but crucially it is
  4373. * also used as the starting point for the zonelist iterator. It
  4374. * may get reset for allocations that ignore memory policies.
  4375. */
  4376. ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
  4377. ac->highest_zoneidx, ac->nodemask);
  4378. return true;
  4379. }
  4380. /*
  4381. * This is the 'heart' of the zoned buddy allocator.
  4382. */
  4383. struct page *
  4384. __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid,
  4385. nodemask_t *nodemask)
  4386. {
  4387. struct page *page;
  4388. unsigned int alloc_flags = ALLOC_WMARK_LOW;
  4389. gfp_t alloc_mask; /* The gfp_t that was actually used for allocation */
  4390. struct alloc_context ac = { };
  4391. /*
  4392. * There are several places where we assume that the order value is sane
  4393. * so bail out early if the request is out of bound.
  4394. */
  4395. if (unlikely(order >= MAX_ORDER)) {
  4396. WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN));
  4397. return NULL;
  4398. }
  4399. gfp_mask &= gfp_allowed_mask;
  4400. alloc_mask = gfp_mask;
  4401. if (!prepare_alloc_pages(gfp_mask, order, preferred_nid, nodemask, &ac, &alloc_mask, &alloc_flags))
  4402. return NULL;
  4403. /*
  4404. * Forbid the first pass from falling back to types that fragment
  4405. * memory until all local zones are considered.
  4406. */
  4407. alloc_flags |= alloc_flags_nofragment(ac.preferred_zoneref->zone, gfp_mask);
  4408. /* First allocation attempt */
  4409. page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac);
  4410. if (likely(page))
  4411. goto out;
  4412. /*
  4413. * Apply scoped allocation constraints. This is mainly about GFP_NOFS
  4414. * resp. GFP_NOIO which has to be inherited for all allocation requests
  4415. * from a particular context which has been marked by
  4416. * memalloc_no{fs,io}_{save,restore}.
  4417. */
  4418. alloc_mask = current_gfp_context(gfp_mask);
  4419. ac.spread_dirty_pages = false;
  4420. /*
  4421. * Restore the original nodemask if it was potentially replaced with
  4422. * &cpuset_current_mems_allowed to optimize the fast-path attempt.
  4423. */
  4424. ac.nodemask = nodemask;
  4425. page = __alloc_pages_slowpath(alloc_mask, order, &ac);
  4426. out:
  4427. if (memcg_kmem_enabled() && (gfp_mask & __GFP_ACCOUNT) && page &&
  4428. unlikely(__memcg_kmem_charge_page(page, gfp_mask, order) != 0)) {
  4429. __free_pages(page, order);
  4430. page = NULL;
  4431. }
  4432. trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype);
  4433. return page;
  4434. }
  4435. EXPORT_SYMBOL(__alloc_pages_nodemask);
  4436. /*
  4437. * Common helper functions. Never use with __GFP_HIGHMEM because the returned
  4438. * address cannot represent highmem pages. Use alloc_pages and then kmap if
  4439. * you need to access high mem.
  4440. */
  4441. unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order)
  4442. {
  4443. struct page *page;
  4444. page = alloc_pages(gfp_mask & ~__GFP_HIGHMEM, order);
  4445. if (!page)
  4446. return 0;
  4447. return (unsigned long) page_address(page);
  4448. }
  4449. EXPORT_SYMBOL(__get_free_pages);
  4450. unsigned long get_zeroed_page(gfp_t gfp_mask)
  4451. {
  4452. return __get_free_pages(gfp_mask | __GFP_ZERO, 0);
  4453. }
  4454. EXPORT_SYMBOL(get_zeroed_page);
  4455. static inline void free_the_page(struct page *page, unsigned int order)
  4456. {
  4457. if (order == 0) /* Via pcp? */
  4458. free_unref_page(page);
  4459. else
  4460. __free_pages_ok(page, order, FPI_NONE);
  4461. }
  4462. void __free_pages(struct page *page, unsigned int order)
  4463. {
  4464. if (put_page_testzero(page))
  4465. free_the_page(page, order);
  4466. else if (!PageHead(page))
  4467. while (order-- > 0)
  4468. free_the_page(page + (1 << order), order);
  4469. }
  4470. EXPORT_SYMBOL(__free_pages);
  4471. void free_pages(unsigned long addr, unsigned int order)
  4472. {
  4473. if (addr != 0) {
  4474. VM_BUG_ON(!virt_addr_valid((void *)addr));
  4475. __free_pages(virt_to_page((void *)addr), order);
  4476. }
  4477. }
  4478. EXPORT_SYMBOL(free_pages);
  4479. /*
  4480. * Page Fragment:
  4481. * An arbitrary-length arbitrary-offset area of memory which resides
  4482. * within a 0 or higher order page. Multiple fragments within that page
  4483. * are individually refcounted, in the page's reference counter.
  4484. *
  4485. * The page_frag functions below provide a simple allocation framework for
  4486. * page fragments. This is used by the network stack and network device
  4487. * drivers to provide a backing region of memory for use as either an
  4488. * sk_buff->head, or to be used in the "frags" portion of skb_shared_info.
  4489. */
  4490. static struct page *__page_frag_cache_refill(struct page_frag_cache *nc,
  4491. gfp_t gfp_mask)
  4492. {
  4493. struct page *page = NULL;
  4494. gfp_t gfp = gfp_mask;
  4495. #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
  4496. gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY |
  4497. __GFP_NOMEMALLOC;
  4498. page = alloc_pages_node(NUMA_NO_NODE, gfp_mask,
  4499. PAGE_FRAG_CACHE_MAX_ORDER);
  4500. nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE;
  4501. #endif
  4502. if (unlikely(!page))
  4503. page = alloc_pages_node(NUMA_NO_NODE, gfp, 0);
  4504. nc->va = page ? page_address(page) : NULL;
  4505. return page;
  4506. }
  4507. void __page_frag_cache_drain(struct page *page, unsigned int count)
  4508. {
  4509. VM_BUG_ON_PAGE(page_ref_count(page) == 0, page);
  4510. if (page_ref_sub_and_test(page, count))
  4511. free_the_page(page, compound_order(page));
  4512. }
  4513. EXPORT_SYMBOL(__page_frag_cache_drain);
  4514. void *page_frag_alloc(struct page_frag_cache *nc,
  4515. unsigned int fragsz, gfp_t gfp_mask)
  4516. {
  4517. unsigned int size = PAGE_SIZE;
  4518. struct page *page;
  4519. int offset;
  4520. if (unlikely(!nc->va)) {
  4521. refill:
  4522. page = __page_frag_cache_refill(nc, gfp_mask);
  4523. if (!page)
  4524. return NULL;
  4525. #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
  4526. /* if size can vary use size else just use PAGE_SIZE */
  4527. size = nc->size;
  4528. #endif
  4529. /* Even if we own the page, we do not use atomic_set().
  4530. * This would break get_page_unless_zero() users.
  4531. */
  4532. page_ref_add(page, PAGE_FRAG_CACHE_MAX_SIZE);
  4533. /* reset page count bias and offset to start of new frag */
  4534. nc->pfmemalloc = page_is_pfmemalloc(page);
  4535. nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
  4536. nc->offset = size;
  4537. }
  4538. offset = nc->offset - fragsz;
  4539. if (unlikely(offset < 0)) {
  4540. page = virt_to_page(nc->va);
  4541. if (!page_ref_sub_and_test(page, nc->pagecnt_bias))
  4542. goto refill;
  4543. if (unlikely(nc->pfmemalloc)) {
  4544. free_the_page(page, compound_order(page));
  4545. goto refill;
  4546. }
  4547. #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
  4548. /* if size can vary use size else just use PAGE_SIZE */
  4549. size = nc->size;
  4550. #endif
  4551. /* OK, page count is 0, we can safely set it */
  4552. set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1);
  4553. /* reset page count bias and offset to start of new frag */
  4554. nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
  4555. offset = size - fragsz;
  4556. }
  4557. nc->pagecnt_bias--;
  4558. nc->offset = offset;
  4559. return nc->va + offset;
  4560. }
  4561. EXPORT_SYMBOL(page_frag_alloc);
  4562. /*
  4563. * Frees a page fragment allocated out of either a compound or order 0 page.
  4564. */
  4565. void page_frag_free(void *addr)
  4566. {
  4567. struct page *page = virt_to_head_page(addr);
  4568. if (unlikely(put_page_testzero(page)))
  4569. free_the_page(page, compound_order(page));
  4570. }
  4571. EXPORT_SYMBOL(page_frag_free);
  4572. static void *make_alloc_exact(unsigned long addr, unsigned int order,
  4573. size_t size)
  4574. {
  4575. if (addr) {
  4576. unsigned long alloc_end = addr + (PAGE_SIZE << order);
  4577. unsigned long used = addr + PAGE_ALIGN(size);
  4578. split_page(virt_to_page((void *)addr), order);
  4579. while (used < alloc_end) {
  4580. free_page(used);
  4581. used += PAGE_SIZE;
  4582. }
  4583. }
  4584. return (void *)addr;
  4585. }
  4586. /**
  4587. * alloc_pages_exact - allocate an exact number physically-contiguous pages.
  4588. * @size: the number of bytes to allocate
  4589. * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP
  4590. *
  4591. * This function is similar to alloc_pages(), except that it allocates the
  4592. * minimum number of pages to satisfy the request. alloc_pages() can only
  4593. * allocate memory in power-of-two pages.
  4594. *
  4595. * This function is also limited by MAX_ORDER.
  4596. *
  4597. * Memory allocated by this function must be released by free_pages_exact().
  4598. *
  4599. * Return: pointer to the allocated area or %NULL in case of error.
  4600. */
  4601. void *alloc_pages_exact(size_t size, gfp_t gfp_mask)
  4602. {
  4603. unsigned int order = get_order(size);
  4604. unsigned long addr;
  4605. if (WARN_ON_ONCE(gfp_mask & __GFP_COMP))
  4606. gfp_mask &= ~__GFP_COMP;
  4607. addr = __get_free_pages(gfp_mask, order);
  4608. return make_alloc_exact(addr, order, size);
  4609. }
  4610. EXPORT_SYMBOL(alloc_pages_exact);
  4611. /**
  4612. * alloc_pages_exact_nid - allocate an exact number of physically-contiguous
  4613. * pages on a node.
  4614. * @nid: the preferred node ID where memory should be allocated
  4615. * @size: the number of bytes to allocate
  4616. * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP
  4617. *
  4618. * Like alloc_pages_exact(), but try to allocate on node nid first before falling
  4619. * back.
  4620. *
  4621. * Return: pointer to the allocated area or %NULL in case of error.
  4622. */
  4623. void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask)
  4624. {
  4625. unsigned int order = get_order(size);
  4626. struct page *p;
  4627. if (WARN_ON_ONCE(gfp_mask & __GFP_COMP))
  4628. gfp_mask &= ~__GFP_COMP;
  4629. p = alloc_pages_node(nid, gfp_mask, order);
  4630. if (!p)
  4631. return NULL;
  4632. return make_alloc_exact((unsigned long)page_address(p), order, size);
  4633. }
  4634. /**
  4635. * free_pages_exact - release memory allocated via alloc_pages_exact()
  4636. * @virt: the value returned by alloc_pages_exact.
  4637. * @size: size of allocation, same value as passed to alloc_pages_exact().
  4638. *
  4639. * Release the memory allocated by a previous call to alloc_pages_exact.
  4640. */
  4641. void free_pages_exact(void *virt, size_t size)
  4642. {
  4643. unsigned long addr = (unsigned long)virt;
  4644. unsigned long end = addr + PAGE_ALIGN(size);
  4645. while (addr < end) {
  4646. free_page(addr);
  4647. addr += PAGE_SIZE;
  4648. }
  4649. }
  4650. EXPORT_SYMBOL(free_pages_exact);
  4651. /**
  4652. * nr_free_zone_pages - count number of pages beyond high watermark
  4653. * @offset: The zone index of the highest zone
  4654. *
  4655. * nr_free_zone_pages() counts the number of pages which are beyond the
  4656. * high watermark within all zones at or below a given zone index. For each
  4657. * zone, the number of pages is calculated as:
  4658. *
  4659. * nr_free_zone_pages = managed_pages - high_pages
  4660. *
  4661. * Return: number of pages beyond high watermark.
  4662. */
  4663. static unsigned long nr_free_zone_pages(int offset)
  4664. {
  4665. struct zoneref *z;
  4666. struct zone *zone;
  4667. /* Just pick one node, since fallback list is circular */
  4668. unsigned long sum = 0;
  4669. struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL);
  4670. for_each_zone_zonelist(zone, z, zonelist, offset) {
  4671. unsigned long size = zone_managed_pages(zone);
  4672. unsigned long high = high_wmark_pages(zone);
  4673. if (size > high)
  4674. sum += size - high;
  4675. }
  4676. return sum;
  4677. }
  4678. /**
  4679. * nr_free_buffer_pages - count number of pages beyond high watermark
  4680. *
  4681. * nr_free_buffer_pages() counts the number of pages which are beyond the high
  4682. * watermark within ZONE_DMA and ZONE_NORMAL.
  4683. *
  4684. * Return: number of pages beyond high watermark within ZONE_DMA and
  4685. * ZONE_NORMAL.
  4686. */
  4687. unsigned long nr_free_buffer_pages(void)
  4688. {
  4689. return nr_free_zone_pages(gfp_zone(GFP_USER));
  4690. }
  4691. EXPORT_SYMBOL_GPL(nr_free_buffer_pages);
  4692. static inline void show_node(struct zone *zone)
  4693. {
  4694. if (IS_ENABLED(CONFIG_NUMA))
  4695. printk("Node %d ", zone_to_nid(zone));
  4696. }
  4697. long si_mem_available(void)
  4698. {
  4699. long available;
  4700. unsigned long pagecache;
  4701. unsigned long wmark_low = 0;
  4702. unsigned long pages[NR_LRU_LISTS];
  4703. unsigned long reclaimable;
  4704. struct zone *zone;
  4705. int lru;
  4706. for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++)
  4707. pages[lru] = global_node_page_state(NR_LRU_BASE + lru);
  4708. for_each_zone(zone)
  4709. wmark_low += low_wmark_pages(zone);
  4710. /*
  4711. * Estimate the amount of memory available for userspace allocations,
  4712. * without causing swapping.
  4713. */
  4714. available = global_zone_page_state(NR_FREE_PAGES) - totalreserve_pages;
  4715. /*
  4716. * Not all the page cache can be freed, otherwise the system will
  4717. * start swapping. Assume at least half of the page cache, or the
  4718. * low watermark worth of cache, needs to stay.
  4719. */
  4720. pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE];
  4721. pagecache -= min(pagecache / 2, wmark_low);
  4722. available += pagecache;
  4723. /*
  4724. * Part of the reclaimable slab and other kernel memory consists of
  4725. * items that are in use, and cannot be freed. Cap this estimate at the
  4726. * low watermark.
  4727. */
  4728. reclaimable = global_node_page_state_pages(NR_SLAB_RECLAIMABLE_B) +
  4729. global_node_page_state(NR_KERNEL_MISC_RECLAIMABLE);
  4730. available += reclaimable - min(reclaimable / 2, wmark_low);
  4731. if (available < 0)
  4732. available = 0;
  4733. return available;
  4734. }
  4735. EXPORT_SYMBOL_GPL(si_mem_available);
  4736. void si_meminfo(struct sysinfo *val)
  4737. {
  4738. val->totalram = totalram_pages();
  4739. val->sharedram = global_node_page_state(NR_SHMEM);
  4740. val->freeram = global_zone_page_state(NR_FREE_PAGES);
  4741. val->bufferram = nr_blockdev_pages();
  4742. val->totalhigh = totalhigh_pages();
  4743. val->freehigh = nr_free_highpages();
  4744. val->mem_unit = PAGE_SIZE;
  4745. }
  4746. EXPORT_SYMBOL(si_meminfo);
  4747. #ifdef CONFIG_NUMA
  4748. void si_meminfo_node(struct sysinfo *val, int nid)
  4749. {
  4750. int zone_type; /* needs to be signed */
  4751. unsigned long managed_pages = 0;
  4752. unsigned long managed_highpages = 0;
  4753. unsigned long free_highpages = 0;
  4754. pg_data_t *pgdat = NODE_DATA(nid);
  4755. for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++)
  4756. managed_pages += zone_managed_pages(&pgdat->node_zones[zone_type]);
  4757. val->totalram = managed_pages;
  4758. val->sharedram = node_page_state(pgdat, NR_SHMEM);
  4759. val->freeram = sum_zone_node_page_state(nid, NR_FREE_PAGES);
  4760. #ifdef CONFIG_HIGHMEM
  4761. for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
  4762. struct zone *zone = &pgdat->node_zones[zone_type];
  4763. if (is_highmem(zone)) {
  4764. managed_highpages += zone_managed_pages(zone);
  4765. free_highpages += zone_page_state(zone, NR_FREE_PAGES);
  4766. }
  4767. }
  4768. val->totalhigh = managed_highpages;
  4769. val->freehigh = free_highpages;
  4770. #else
  4771. val->totalhigh = managed_highpages;
  4772. val->freehigh = free_highpages;
  4773. #endif
  4774. val->mem_unit = PAGE_SIZE;
  4775. }
  4776. #endif
  4777. /*
  4778. * Determine whether the node should be displayed or not, depending on whether
  4779. * SHOW_MEM_FILTER_NODES was passed to show_free_areas().
  4780. */
  4781. static bool show_mem_node_skip(unsigned int flags, int nid, nodemask_t *nodemask)
  4782. {
  4783. if (!(flags & SHOW_MEM_FILTER_NODES))
  4784. return false;
  4785. /*
  4786. * no node mask - aka implicit memory numa policy. Do not bother with
  4787. * the synchronization - read_mems_allowed_begin - because we do not
  4788. * have to be precise here.
  4789. */
  4790. if (!nodemask)
  4791. nodemask = &cpuset_current_mems_allowed;
  4792. return !node_isset(nid, *nodemask);
  4793. }
  4794. #define K(x) ((x) << (PAGE_SHIFT-10))
  4795. static void show_migration_types(unsigned char type)
  4796. {
  4797. static const char types[MIGRATE_TYPES] = {
  4798. [MIGRATE_UNMOVABLE] = 'U',
  4799. [MIGRATE_MOVABLE] = 'M',
  4800. [MIGRATE_RECLAIMABLE] = 'E',
  4801. [MIGRATE_HIGHATOMIC] = 'H',
  4802. #ifdef CONFIG_CMA
  4803. [MIGRATE_CMA] = 'C',
  4804. #endif
  4805. #ifdef CONFIG_MEMORY_ISOLATION
  4806. [MIGRATE_ISOLATE] = 'I',
  4807. #endif
  4808. };
  4809. char tmp[MIGRATE_TYPES + 1];
  4810. char *p = tmp;
  4811. int i;
  4812. for (i = 0; i < MIGRATE_TYPES; i++) {
  4813. if (type & (1 << i))
  4814. *p++ = types[i];
  4815. }
  4816. *p = '\0';
  4817. printk(KERN_CONT "(%s) ", tmp);
  4818. }
  4819. /*
  4820. * Show free area list (used inside shift_scroll-lock stuff)
  4821. * We also calculate the percentage fragmentation. We do this by counting the
  4822. * memory on each free list with the exception of the first item on the list.
  4823. *
  4824. * Bits in @filter:
  4825. * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's
  4826. * cpuset.
  4827. */
  4828. void show_free_areas(unsigned int filter, nodemask_t *nodemask)
  4829. {
  4830. unsigned long free_pcp = 0;
  4831. int cpu;
  4832. struct zone *zone;
  4833. pg_data_t *pgdat;
  4834. for_each_populated_zone(zone) {
  4835. if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
  4836. continue;
  4837. for_each_online_cpu(cpu)
  4838. free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count;
  4839. }
  4840. printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n"
  4841. " active_file:%lu inactive_file:%lu isolated_file:%lu\n"
  4842. " unevictable:%lu dirty:%lu writeback:%lu\n"
  4843. " slab_reclaimable:%lu slab_unreclaimable:%lu\n"
  4844. " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n"
  4845. " free:%lu free_pcp:%lu free_cma:%lu\n",
  4846. global_node_page_state(NR_ACTIVE_ANON),
  4847. global_node_page_state(NR_INACTIVE_ANON),
  4848. global_node_page_state(NR_ISOLATED_ANON),
  4849. global_node_page_state(NR_ACTIVE_FILE),
  4850. global_node_page_state(NR_INACTIVE_FILE),
  4851. global_node_page_state(NR_ISOLATED_FILE),
  4852. global_node_page_state(NR_UNEVICTABLE),
  4853. global_node_page_state(NR_FILE_DIRTY),
  4854. global_node_page_state(NR_WRITEBACK),
  4855. global_node_page_state_pages(NR_SLAB_RECLAIMABLE_B),
  4856. global_node_page_state_pages(NR_SLAB_UNRECLAIMABLE_B),
  4857. global_node_page_state(NR_FILE_MAPPED),
  4858. global_node_page_state(NR_SHMEM),
  4859. global_zone_page_state(NR_PAGETABLE),
  4860. global_zone_page_state(NR_BOUNCE),
  4861. global_zone_page_state(NR_FREE_PAGES),
  4862. free_pcp,
  4863. global_zone_page_state(NR_FREE_CMA_PAGES));
  4864. for_each_online_pgdat(pgdat) {
  4865. if (show_mem_node_skip(filter, pgdat->node_id, nodemask))
  4866. continue;
  4867. printk("Node %d"
  4868. " active_anon:%lukB"
  4869. " inactive_anon:%lukB"
  4870. " active_file:%lukB"
  4871. " inactive_file:%lukB"
  4872. " unevictable:%lukB"
  4873. " isolated(anon):%lukB"
  4874. " isolated(file):%lukB"
  4875. " mapped:%lukB"
  4876. " dirty:%lukB"
  4877. " writeback:%lukB"
  4878. " shmem:%lukB"
  4879. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  4880. " shmem_thp: %lukB"
  4881. " shmem_pmdmapped: %lukB"
  4882. " anon_thp: %lukB"
  4883. #endif
  4884. " writeback_tmp:%lukB"
  4885. " kernel_stack:%lukB"
  4886. #ifdef CONFIG_SHADOW_CALL_STACK
  4887. " shadow_call_stack:%lukB"
  4888. #endif
  4889. " all_unreclaimable? %s"
  4890. "\n",
  4891. pgdat->node_id,
  4892. K(node_page_state(pgdat, NR_ACTIVE_ANON)),
  4893. K(node_page_state(pgdat, NR_INACTIVE_ANON)),
  4894. K(node_page_state(pgdat, NR_ACTIVE_FILE)),
  4895. K(node_page_state(pgdat, NR_INACTIVE_FILE)),
  4896. K(node_page_state(pgdat, NR_UNEVICTABLE)),
  4897. K(node_page_state(pgdat, NR_ISOLATED_ANON)),
  4898. K(node_page_state(pgdat, NR_ISOLATED_FILE)),
  4899. K(node_page_state(pgdat, NR_FILE_MAPPED)),
  4900. K(node_page_state(pgdat, NR_FILE_DIRTY)),
  4901. K(node_page_state(pgdat, NR_WRITEBACK)),
  4902. K(node_page_state(pgdat, NR_SHMEM)),
  4903. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  4904. K(node_page_state(pgdat, NR_SHMEM_THPS) * HPAGE_PMD_NR),
  4905. K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)
  4906. * HPAGE_PMD_NR),
  4907. K(node_page_state(pgdat, NR_ANON_THPS) * HPAGE_PMD_NR),
  4908. #endif
  4909. K(node_page_state(pgdat, NR_WRITEBACK_TEMP)),
  4910. node_page_state(pgdat, NR_KERNEL_STACK_KB),
  4911. #ifdef CONFIG_SHADOW_CALL_STACK
  4912. node_page_state(pgdat, NR_KERNEL_SCS_KB),
  4913. #endif
  4914. pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ?
  4915. "yes" : "no");
  4916. }
  4917. for_each_populated_zone(zone) {
  4918. int i;
  4919. if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
  4920. continue;
  4921. free_pcp = 0;
  4922. for_each_online_cpu(cpu)
  4923. free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count;
  4924. show_node(zone);
  4925. printk(KERN_CONT
  4926. "%s"
  4927. " free:%lukB"
  4928. " min:%lukB"
  4929. " low:%lukB"
  4930. " high:%lukB"
  4931. " reserved_highatomic:%luKB"
  4932. " active_anon:%lukB"
  4933. " inactive_anon:%lukB"
  4934. " active_file:%lukB"
  4935. " inactive_file:%lukB"
  4936. " unevictable:%lukB"
  4937. " writepending:%lukB"
  4938. " present:%lukB"
  4939. " managed:%lukB"
  4940. " mlocked:%lukB"
  4941. " pagetables:%lukB"
  4942. " bounce:%lukB"
  4943. " free_pcp:%lukB"
  4944. " local_pcp:%ukB"
  4945. " free_cma:%lukB"
  4946. "\n",
  4947. zone->name,
  4948. K(zone_page_state(zone, NR_FREE_PAGES)),
  4949. K(min_wmark_pages(zone)),
  4950. K(low_wmark_pages(zone)),
  4951. K(high_wmark_pages(zone)),
  4952. K(zone->nr_reserved_highatomic),
  4953. K(zone_page_state(zone, NR_ZONE_ACTIVE_ANON)),
  4954. K(zone_page_state(zone, NR_ZONE_INACTIVE_ANON)),
  4955. K(zone_page_state(zone, NR_ZONE_ACTIVE_FILE)),
  4956. K(zone_page_state(zone, NR_ZONE_INACTIVE_FILE)),
  4957. K(zone_page_state(zone, NR_ZONE_UNEVICTABLE)),
  4958. K(zone_page_state(zone, NR_ZONE_WRITE_PENDING)),
  4959. K(zone->present_pages),
  4960. K(zone_managed_pages(zone)),
  4961. K(zone_page_state(zone, NR_MLOCK)),
  4962. K(zone_page_state(zone, NR_PAGETABLE)),
  4963. K(zone_page_state(zone, NR_BOUNCE)),
  4964. K(free_pcp),
  4965. K(this_cpu_read(zone->pageset->pcp.count)),
  4966. K(zone_page_state(zone, NR_FREE_CMA_PAGES)));
  4967. printk("lowmem_reserve[]:");
  4968. for (i = 0; i < MAX_NR_ZONES; i++)
  4969. printk(KERN_CONT " %ld", zone->lowmem_reserve[i]);
  4970. printk(KERN_CONT "\n");
  4971. }
  4972. for_each_populated_zone(zone) {
  4973. unsigned int order;
  4974. unsigned long nr[MAX_ORDER], flags, total = 0;
  4975. unsigned char types[MAX_ORDER];
  4976. if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
  4977. continue;
  4978. show_node(zone);
  4979. printk(KERN_CONT "%s: ", zone->name);
  4980. spin_lock_irqsave(&zone->lock, flags);
  4981. for (order = 0; order < MAX_ORDER; order++) {
  4982. struct free_area *area = &zone->free_area[order];
  4983. int type;
  4984. nr[order] = area->nr_free;
  4985. total += nr[order] << order;
  4986. types[order] = 0;
  4987. for (type = 0; type < MIGRATE_TYPES; type++) {
  4988. if (!free_area_empty(area, type))
  4989. types[order] |= 1 << type;
  4990. }
  4991. }
  4992. spin_unlock_irqrestore(&zone->lock, flags);
  4993. for (order = 0; order < MAX_ORDER; order++) {
  4994. printk(KERN_CONT "%lu*%lukB ",
  4995. nr[order], K(1UL) << order);
  4996. if (nr[order])
  4997. show_migration_types(types[order]);
  4998. }
  4999. printk(KERN_CONT "= %lukB\n", K(total));
  5000. }
  5001. hugetlb_show_meminfo();
  5002. printk("%ld total pagecache pages\n", global_node_page_state(NR_FILE_PAGES));
  5003. show_swap_cache_info();
  5004. }
  5005. static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref)
  5006. {
  5007. zoneref->zone = zone;
  5008. zoneref->zone_idx = zone_idx(zone);
  5009. }
  5010. /*
  5011. * Builds allocation fallback zone lists.
  5012. *
  5013. * Add all populated zones of a node to the zonelist.
  5014. */
  5015. static int build_zonerefs_node(pg_data_t *pgdat, struct zoneref *zonerefs)
  5016. {
  5017. struct zone *zone;
  5018. enum zone_type zone_type = MAX_NR_ZONES;
  5019. int nr_zones = 0;
  5020. do {
  5021. zone_type--;
  5022. zone = pgdat->node_zones + zone_type;
  5023. if (populated_zone(zone)) {
  5024. zoneref_set_zone(zone, &zonerefs[nr_zones++]);
  5025. check_highest_zone(zone_type);
  5026. }
  5027. } while (zone_type);
  5028. return nr_zones;
  5029. }
  5030. #ifdef CONFIG_NUMA
  5031. static int __parse_numa_zonelist_order(char *s)
  5032. {
  5033. /*
  5034. * We used to support different zonlists modes but they turned
  5035. * out to be just not useful. Let's keep the warning in place
  5036. * if somebody still use the cmd line parameter so that we do
  5037. * not fail it silently
  5038. */
  5039. if (!(*s == 'd' || *s == 'D' || *s == 'n' || *s == 'N')) {
  5040. pr_warn("Ignoring unsupported numa_zonelist_order value: %s\n", s);
  5041. return -EINVAL;
  5042. }
  5043. return 0;
  5044. }
  5045. char numa_zonelist_order[] = "Node";
  5046. /*
  5047. * sysctl handler for numa_zonelist_order
  5048. */
  5049. int numa_zonelist_order_handler(struct ctl_table *table, int write,
  5050. void *buffer, size_t *length, loff_t *ppos)
  5051. {
  5052. if (write)
  5053. return __parse_numa_zonelist_order(buffer);
  5054. return proc_dostring(table, write, buffer, length, ppos);
  5055. }
  5056. #define MAX_NODE_LOAD (nr_online_nodes)
  5057. static int node_load[MAX_NUMNODES];
  5058. /**
  5059. * find_next_best_node - find the next node that should appear in a given node's fallback list
  5060. * @node: node whose fallback list we're appending
  5061. * @used_node_mask: nodemask_t of already used nodes
  5062. *
  5063. * We use a number of factors to determine which is the next node that should
  5064. * appear on a given node's fallback list. The node should not have appeared
  5065. * already in @node's fallback list, and it should be the next closest node
  5066. * according to the distance array (which contains arbitrary distance values
  5067. * from each node to each node in the system), and should also prefer nodes
  5068. * with no CPUs, since presumably they'll have very little allocation pressure
  5069. * on them otherwise.
  5070. *
  5071. * Return: node id of the found node or %NUMA_NO_NODE if no node is found.
  5072. */
  5073. static int find_next_best_node(int node, nodemask_t *used_node_mask)
  5074. {
  5075. int n, val;
  5076. int min_val = INT_MAX;
  5077. int best_node = NUMA_NO_NODE;
  5078. /* Use the local node if we haven't already */
  5079. if (!node_isset(node, *used_node_mask)) {
  5080. node_set(node, *used_node_mask);
  5081. return node;
  5082. }
  5083. for_each_node_state(n, N_MEMORY) {
  5084. /* Don't want a node to appear more than once */
  5085. if (node_isset(n, *used_node_mask))
  5086. continue;
  5087. /* Use the distance array to find the distance */
  5088. val = node_distance(node, n);
  5089. /* Penalize nodes under us ("prefer the next node") */
  5090. val += (n < node);
  5091. /* Give preference to headless and unused nodes */
  5092. if (!cpumask_empty(cpumask_of_node(n)))
  5093. val += PENALTY_FOR_NODE_WITH_CPUS;
  5094. /* Slight preference for less loaded node */
  5095. val *= (MAX_NODE_LOAD*MAX_NUMNODES);
  5096. val += node_load[n];
  5097. if (val < min_val) {
  5098. min_val = val;
  5099. best_node = n;
  5100. }
  5101. }
  5102. if (best_node >= 0)
  5103. node_set(best_node, *used_node_mask);
  5104. return best_node;
  5105. }
  5106. /*
  5107. * Build zonelists ordered by node and zones within node.
  5108. * This results in maximum locality--normal zone overflows into local
  5109. * DMA zone, if any--but risks exhausting DMA zone.
  5110. */
  5111. static void build_zonelists_in_node_order(pg_data_t *pgdat, int *node_order,
  5112. unsigned nr_nodes)
  5113. {
  5114. struct zoneref *zonerefs;
  5115. int i;
  5116. zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs;
  5117. for (i = 0; i < nr_nodes; i++) {
  5118. int nr_zones;
  5119. pg_data_t *node = NODE_DATA(node_order[i]);
  5120. nr_zones = build_zonerefs_node(node, zonerefs);
  5121. zonerefs += nr_zones;
  5122. }
  5123. zonerefs->zone = NULL;
  5124. zonerefs->zone_idx = 0;
  5125. }
  5126. /*
  5127. * Build gfp_thisnode zonelists
  5128. */
  5129. static void build_thisnode_zonelists(pg_data_t *pgdat)
  5130. {
  5131. struct zoneref *zonerefs;
  5132. int nr_zones;
  5133. zonerefs = pgdat->node_zonelists[ZONELIST_NOFALLBACK]._zonerefs;
  5134. nr_zones = build_zonerefs_node(pgdat, zonerefs);
  5135. zonerefs += nr_zones;
  5136. zonerefs->zone = NULL;
  5137. zonerefs->zone_idx = 0;
  5138. }
  5139. /*
  5140. * Build zonelists ordered by zone and nodes within zones.
  5141. * This results in conserving DMA zone[s] until all Normal memory is
  5142. * exhausted, but results in overflowing to remote node while memory
  5143. * may still exist in local DMA zone.
  5144. */
  5145. static void build_zonelists(pg_data_t *pgdat)
  5146. {
  5147. static int node_order[MAX_NUMNODES];
  5148. int node, load, nr_nodes = 0;
  5149. nodemask_t used_mask = NODE_MASK_NONE;
  5150. int local_node, prev_node;
  5151. /* NUMA-aware ordering of nodes */
  5152. local_node = pgdat->node_id;
  5153. load = nr_online_nodes;
  5154. prev_node = local_node;
  5155. memset(node_order, 0, sizeof(node_order));
  5156. while ((node = find_next_best_node(local_node, &used_mask)) >= 0) {
  5157. /*
  5158. * We don't want to pressure a particular node.
  5159. * So adding penalty to the first node in same
  5160. * distance group to make it round-robin.
  5161. */
  5162. if (node_distance(local_node, node) !=
  5163. node_distance(local_node, prev_node))
  5164. node_load[node] = load;
  5165. node_order[nr_nodes++] = node;
  5166. prev_node = node;
  5167. load--;
  5168. }
  5169. build_zonelists_in_node_order(pgdat, node_order, nr_nodes);
  5170. build_thisnode_zonelists(pgdat);
  5171. }
  5172. #ifdef CONFIG_HAVE_MEMORYLESS_NODES
  5173. /*
  5174. * Return node id of node used for "local" allocations.
  5175. * I.e., first node id of first zone in arg node's generic zonelist.
  5176. * Used for initializing percpu 'numa_mem', which is used primarily
  5177. * for kernel allocations, so use GFP_KERNEL flags to locate zonelist.
  5178. */
  5179. int local_memory_node(int node)
  5180. {
  5181. struct zoneref *z;
  5182. z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL),
  5183. gfp_zone(GFP_KERNEL),
  5184. NULL);
  5185. return zone_to_nid(z->zone);
  5186. }
  5187. #endif
  5188. static void setup_min_unmapped_ratio(void);
  5189. static void setup_min_slab_ratio(void);
  5190. #else /* CONFIG_NUMA */
  5191. static void build_zonelists(pg_data_t *pgdat)
  5192. {
  5193. int node, local_node;
  5194. struct zoneref *zonerefs;
  5195. int nr_zones;
  5196. local_node = pgdat->node_id;
  5197. zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs;
  5198. nr_zones = build_zonerefs_node(pgdat, zonerefs);
  5199. zonerefs += nr_zones;
  5200. /*
  5201. * Now we build the zonelist so that it contains the zones
  5202. * of all the other nodes.
  5203. * We don't want to pressure a particular node, so when
  5204. * building the zones for node N, we make sure that the
  5205. * zones coming right after the local ones are those from
  5206. * node N+1 (modulo N)
  5207. */
  5208. for (node = local_node + 1; node < MAX_NUMNODES; node++) {
  5209. if (!node_online(node))
  5210. continue;
  5211. nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs);
  5212. zonerefs += nr_zones;
  5213. }
  5214. for (node = 0; node < local_node; node++) {
  5215. if (!node_online(node))
  5216. continue;
  5217. nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs);
  5218. zonerefs += nr_zones;
  5219. }
  5220. zonerefs->zone = NULL;
  5221. zonerefs->zone_idx = 0;
  5222. }
  5223. #endif /* CONFIG_NUMA */
  5224. /*
  5225. * Boot pageset table. One per cpu which is going to be used for all
  5226. * zones and all nodes. The parameters will be set in such a way
  5227. * that an item put on a list will immediately be handed over to
  5228. * the buddy list. This is safe since pageset manipulation is done
  5229. * with interrupts disabled.
  5230. *
  5231. * The boot_pagesets must be kept even after bootup is complete for
  5232. * unused processors and/or zones. They do play a role for bootstrapping
  5233. * hotplugged processors.
  5234. *
  5235. * zoneinfo_show() and maybe other functions do
  5236. * not check if the processor is online before following the pageset pointer.
  5237. * Other parts of the kernel may not check if the zone is available.
  5238. */
  5239. static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch);
  5240. static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset);
  5241. static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats);
  5242. static void __build_all_zonelists(void *data)
  5243. {
  5244. int nid;
  5245. int __maybe_unused cpu;
  5246. pg_data_t *self = data;
  5247. static DEFINE_SPINLOCK(lock);
  5248. spin_lock(&lock);
  5249. #ifdef CONFIG_NUMA
  5250. memset(node_load, 0, sizeof(node_load));
  5251. #endif
  5252. /*
  5253. * This node is hotadded and no memory is yet present. So just
  5254. * building zonelists is fine - no need to touch other nodes.
  5255. */
  5256. if (self && !node_online(self->node_id)) {
  5257. build_zonelists(self);
  5258. } else {
  5259. for_each_online_node(nid) {
  5260. pg_data_t *pgdat = NODE_DATA(nid);
  5261. build_zonelists(pgdat);
  5262. }
  5263. #ifdef CONFIG_HAVE_MEMORYLESS_NODES
  5264. /*
  5265. * We now know the "local memory node" for each node--
  5266. * i.e., the node of the first zone in the generic zonelist.
  5267. * Set up numa_mem percpu variable for on-line cpus. During
  5268. * boot, only the boot cpu should be on-line; we'll init the
  5269. * secondary cpus' numa_mem as they come on-line. During
  5270. * node/memory hotplug, we'll fixup all on-line cpus.
  5271. */
  5272. for_each_online_cpu(cpu)
  5273. set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu)));
  5274. #endif
  5275. }
  5276. spin_unlock(&lock);
  5277. }
  5278. static noinline void __init
  5279. build_all_zonelists_init(void)
  5280. {
  5281. int cpu;
  5282. __build_all_zonelists(NULL);
  5283. /*
  5284. * Initialize the boot_pagesets that are going to be used
  5285. * for bootstrapping processors. The real pagesets for
  5286. * each zone will be allocated later when the per cpu
  5287. * allocator is available.
  5288. *
  5289. * boot_pagesets are used also for bootstrapping offline
  5290. * cpus if the system is already booted because the pagesets
  5291. * are needed to initialize allocators on a specific cpu too.
  5292. * F.e. the percpu allocator needs the page allocator which
  5293. * needs the percpu allocator in order to allocate its pagesets
  5294. * (a chicken-egg dilemma).
  5295. */
  5296. for_each_possible_cpu(cpu)
  5297. setup_pageset(&per_cpu(boot_pageset, cpu), 0);
  5298. mminit_verify_zonelist();
  5299. cpuset_init_current_mems_allowed();
  5300. }
  5301. /*
  5302. * unless system_state == SYSTEM_BOOTING.
  5303. *
  5304. * __ref due to call of __init annotated helper build_all_zonelists_init
  5305. * [protected by SYSTEM_BOOTING].
  5306. */
  5307. void __ref build_all_zonelists(pg_data_t *pgdat)
  5308. {
  5309. unsigned long vm_total_pages;
  5310. if (system_state == SYSTEM_BOOTING) {
  5311. build_all_zonelists_init();
  5312. } else {
  5313. __build_all_zonelists(pgdat);
  5314. /* cpuset refresh routine should be here */
  5315. }
  5316. /* Get the number of free pages beyond high watermark in all zones. */
  5317. vm_total_pages = nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE));
  5318. /*
  5319. * Disable grouping by mobility if the number of pages in the
  5320. * system is too low to allow the mechanism to work. It would be
  5321. * more accurate, but expensive to check per-zone. This check is
  5322. * made on memory-hotadd so a system can start with mobility
  5323. * disabled and enable it later
  5324. */
  5325. if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES))
  5326. page_group_by_mobility_disabled = 1;
  5327. else
  5328. page_group_by_mobility_disabled = 0;
  5329. pr_info("Built %u zonelists, mobility grouping %s. Total pages: %ld\n",
  5330. nr_online_nodes,
  5331. page_group_by_mobility_disabled ? "off" : "on",
  5332. vm_total_pages);
  5333. #ifdef CONFIG_NUMA
  5334. pr_info("Policy zone: %s\n", zone_names[policy_zone]);
  5335. #endif
  5336. }
  5337. /* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */
  5338. static bool __meminit
  5339. overlap_memmap_init(unsigned long zone, unsigned long *pfn)
  5340. {
  5341. static struct memblock_region *r;
  5342. if (mirrored_kernelcore && zone == ZONE_MOVABLE) {
  5343. if (!r || *pfn >= memblock_region_memory_end_pfn(r)) {
  5344. for_each_mem_region(r) {
  5345. if (*pfn < memblock_region_memory_end_pfn(r))
  5346. break;
  5347. }
  5348. }
  5349. if (*pfn >= memblock_region_memory_base_pfn(r) &&
  5350. memblock_is_mirror(r)) {
  5351. *pfn = memblock_region_memory_end_pfn(r);
  5352. return true;
  5353. }
  5354. }
  5355. return false;
  5356. }
  5357. /*
  5358. * Initially all pages are reserved - free ones are freed
  5359. * up by memblock_free_all() once the early boot process is
  5360. * done. Non-atomic initialization, single-pass.
  5361. *
  5362. * All aligned pageblocks are initialized to the specified migratetype
  5363. * (usually MIGRATE_MOVABLE). Besides setting the migratetype, no related
  5364. * zone stats (e.g., nr_isolate_pageblock) are touched.
  5365. */
  5366. void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
  5367. unsigned long start_pfn, unsigned long zone_end_pfn,
  5368. enum meminit_context context,
  5369. struct vmem_altmap *altmap, int migratetype)
  5370. {
  5371. unsigned long pfn, end_pfn = start_pfn + size;
  5372. struct page *page;
  5373. if (highest_memmap_pfn < end_pfn - 1)
  5374. highest_memmap_pfn = end_pfn - 1;
  5375. #ifdef CONFIG_ZONE_DEVICE
  5376. /*
  5377. * Honor reservation requested by the driver for this ZONE_DEVICE
  5378. * memory. We limit the total number of pages to initialize to just
  5379. * those that might contain the memory mapping. We will defer the
  5380. * ZONE_DEVICE page initialization until after we have released
  5381. * the hotplug lock.
  5382. */
  5383. if (zone == ZONE_DEVICE) {
  5384. if (!altmap)
  5385. return;
  5386. if (start_pfn == altmap->base_pfn)
  5387. start_pfn += altmap->reserve;
  5388. end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap);
  5389. }
  5390. #endif
  5391. for (pfn = start_pfn; pfn < end_pfn; ) {
  5392. /*
  5393. * There can be holes in boot-time mem_map[]s handed to this
  5394. * function. They do not exist on hotplugged memory.
  5395. */
  5396. if (context == MEMINIT_EARLY) {
  5397. if (overlap_memmap_init(zone, &pfn))
  5398. continue;
  5399. if (defer_init(nid, pfn, zone_end_pfn))
  5400. break;
  5401. }
  5402. page = pfn_to_page(pfn);
  5403. __init_single_page(page, pfn, zone, nid);
  5404. if (context == MEMINIT_HOTPLUG)
  5405. __SetPageReserved(page);
  5406. /*
  5407. * Usually, we want to mark the pageblock MIGRATE_MOVABLE,
  5408. * such that unmovable allocations won't be scattered all
  5409. * over the place during system boot.
  5410. */
  5411. if (IS_ALIGNED(pfn, pageblock_nr_pages)) {
  5412. set_pageblock_migratetype(page, migratetype);
  5413. cond_resched();
  5414. }
  5415. pfn++;
  5416. }
  5417. }
  5418. #ifdef CONFIG_ZONE_DEVICE
  5419. void __ref memmap_init_zone_device(struct zone *zone,
  5420. unsigned long start_pfn,
  5421. unsigned long nr_pages,
  5422. struct dev_pagemap *pgmap)
  5423. {
  5424. unsigned long pfn, end_pfn = start_pfn + nr_pages;
  5425. struct pglist_data *pgdat = zone->zone_pgdat;
  5426. struct vmem_altmap *altmap = pgmap_altmap(pgmap);
  5427. unsigned long zone_idx = zone_idx(zone);
  5428. unsigned long start = jiffies;
  5429. int nid = pgdat->node_id;
  5430. if (WARN_ON_ONCE(!pgmap || zone_idx(zone) != ZONE_DEVICE))
  5431. return;
  5432. /*
  5433. * The call to memmap_init should have already taken care
  5434. * of the pages reserved for the memmap, so we can just jump to
  5435. * the end of that region and start processing the device pages.
  5436. */
  5437. if (altmap) {
  5438. start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap);
  5439. nr_pages = end_pfn - start_pfn;
  5440. }
  5441. for (pfn = start_pfn; pfn < end_pfn; pfn++) {
  5442. struct page *page = pfn_to_page(pfn);
  5443. __init_single_page(page, pfn, zone_idx, nid);
  5444. /*
  5445. * Mark page reserved as it will need to wait for onlining
  5446. * phase for it to be fully associated with a zone.
  5447. *
  5448. * We can use the non-atomic __set_bit operation for setting
  5449. * the flag as we are still initializing the pages.
  5450. */
  5451. __SetPageReserved(page);
  5452. /*
  5453. * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer
  5454. * and zone_device_data. It is a bug if a ZONE_DEVICE page is
  5455. * ever freed or placed on a driver-private list.
  5456. */
  5457. page->pgmap = pgmap;
  5458. page->zone_device_data = NULL;
  5459. /*
  5460. * Mark the block movable so that blocks are reserved for
  5461. * movable at startup. This will force kernel allocations
  5462. * to reserve their blocks rather than leaking throughout
  5463. * the address space during boot when many long-lived
  5464. * kernel allocations are made.
  5465. *
  5466. * Please note that MEMINIT_HOTPLUG path doesn't clear memmap
  5467. * because this is done early in section_activate()
  5468. */
  5469. if (IS_ALIGNED(pfn, pageblock_nr_pages)) {
  5470. set_pageblock_migratetype(page, MIGRATE_MOVABLE);
  5471. cond_resched();
  5472. }
  5473. }
  5474. pr_info("%s initialised %lu pages in %ums\n", __func__,
  5475. nr_pages, jiffies_to_msecs(jiffies - start));
  5476. }
  5477. #endif
  5478. static void __meminit zone_init_free_lists(struct zone *zone)
  5479. {
  5480. unsigned int order, t;
  5481. for_each_migratetype_order(order, t) {
  5482. INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
  5483. zone->free_area[order].nr_free = 0;
  5484. }
  5485. }
  5486. #if !defined(CONFIG_FLAT_NODE_MEM_MAP)
  5487. /*
  5488. * Only struct pages that correspond to ranges defined by memblock.memory
  5489. * are zeroed and initialized by going through __init_single_page() during
  5490. * memmap_init_zone_range().
  5491. *
  5492. * But, there could be struct pages that correspond to holes in
  5493. * memblock.memory. This can happen because of the following reasons:
  5494. * - physical memory bank size is not necessarily the exact multiple of the
  5495. * arbitrary section size
  5496. * - early reserved memory may not be listed in memblock.memory
  5497. * - memory layouts defined with memmap= kernel parameter may not align
  5498. * nicely with memmap sections
  5499. *
  5500. * Explicitly initialize those struct pages so that:
  5501. * - PG_Reserved is set
  5502. * - zone and node links point to zone and node that span the page if the
  5503. * hole is in the middle of a zone
  5504. * - zone and node links point to adjacent zone/node if the hole falls on
  5505. * the zone boundary; the pages in such holes will be prepended to the
  5506. * zone/node above the hole except for the trailing pages in the last
  5507. * section that will be appended to the zone/node below.
  5508. */
  5509. static void __init init_unavailable_range(unsigned long spfn,
  5510. unsigned long epfn,
  5511. int zone, int node)
  5512. {
  5513. unsigned long pfn;
  5514. u64 pgcnt = 0;
  5515. for (pfn = spfn; pfn < epfn; pfn++) {
  5516. if (!pfn_valid(ALIGN_DOWN(pfn, pageblock_nr_pages))) {
  5517. pfn = ALIGN_DOWN(pfn, pageblock_nr_pages)
  5518. + pageblock_nr_pages - 1;
  5519. continue;
  5520. }
  5521. __init_single_page(pfn_to_page(pfn), pfn, zone, node);
  5522. __SetPageReserved(pfn_to_page(pfn));
  5523. pgcnt++;
  5524. }
  5525. if (pgcnt)
  5526. pr_info("On node %d, zone %s: %lld pages in unavailable ranges",
  5527. node, zone_names[zone], pgcnt);
  5528. }
  5529. #else
  5530. static inline void init_unavailable_range(unsigned long spfn,
  5531. unsigned long epfn,
  5532. int zone, int node)
  5533. {
  5534. }
  5535. #endif
  5536. static void __init memmap_init_zone_range(struct zone *zone,
  5537. unsigned long start_pfn,
  5538. unsigned long end_pfn,
  5539. unsigned long *hole_pfn)
  5540. {
  5541. unsigned long zone_start_pfn = zone->zone_start_pfn;
  5542. unsigned long zone_end_pfn = zone_start_pfn + zone->spanned_pages;
  5543. int nid = zone_to_nid(zone), zone_id = zone_idx(zone);
  5544. start_pfn = clamp(start_pfn, zone_start_pfn, zone_end_pfn);
  5545. end_pfn = clamp(end_pfn, zone_start_pfn, zone_end_pfn);
  5546. if (start_pfn >= end_pfn)
  5547. return;
  5548. memmap_init_zone(end_pfn - start_pfn, nid, zone_id, start_pfn,
  5549. zone_end_pfn, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE);
  5550. if (*hole_pfn < start_pfn)
  5551. init_unavailable_range(*hole_pfn, start_pfn, zone_id, nid);
  5552. *hole_pfn = end_pfn;
  5553. }
  5554. void __init __weak memmap_init(void)
  5555. {
  5556. unsigned long start_pfn, end_pfn;
  5557. unsigned long hole_pfn = 0;
  5558. int i, j, zone_id, nid;
  5559. for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
  5560. struct pglist_data *node = NODE_DATA(nid);
  5561. for (j = 0; j < MAX_NR_ZONES; j++) {
  5562. struct zone *zone = node->node_zones + j;
  5563. if (!populated_zone(zone))
  5564. continue;
  5565. memmap_init_zone_range(zone, start_pfn, end_pfn,
  5566. &hole_pfn);
  5567. zone_id = j;
  5568. }
  5569. }
  5570. #ifdef CONFIG_SPARSEMEM
  5571. /*
  5572. * Initialize the memory map for hole in the range [memory_end,
  5573. * section_end].
  5574. * Append the pages in this hole to the highest zone in the last
  5575. * node.
  5576. * The call to init_unavailable_range() is outside the ifdef to
  5577. * silence the compiler warining about zone_id set but not used;
  5578. * for FLATMEM it is a nop anyway
  5579. */
  5580. end_pfn = round_up(end_pfn, PAGES_PER_SECTION);
  5581. if (hole_pfn < end_pfn)
  5582. #endif
  5583. init_unavailable_range(hole_pfn, end_pfn, zone_id, nid);
  5584. }
  5585. /* A stub for backwards compatibility with custom implementatin on IA-64 */
  5586. void __meminit __weak arch_memmap_init(unsigned long size, int nid,
  5587. unsigned long zone,
  5588. unsigned long range_start_pfn)
  5589. {
  5590. }
  5591. static int zone_batchsize(struct zone *zone)
  5592. {
  5593. #ifdef CONFIG_MMU
  5594. int batch;
  5595. /*
  5596. * The per-cpu-pages pools are set to around 1000th of the
  5597. * size of the zone.
  5598. */
  5599. batch = zone_managed_pages(zone) / 1024;
  5600. /* But no more than a meg. */
  5601. if (batch * PAGE_SIZE > 1024 * 1024)
  5602. batch = (1024 * 1024) / PAGE_SIZE;
  5603. batch /= 4; /* We effectively *= 4 below */
  5604. if (batch < 1)
  5605. batch = 1;
  5606. /*
  5607. * Clamp the batch to a 2^n - 1 value. Having a power
  5608. * of 2 value was found to be more likely to have
  5609. * suboptimal cache aliasing properties in some cases.
  5610. *
  5611. * For example if 2 tasks are alternately allocating
  5612. * batches of pages, one task can end up with a lot
  5613. * of pages of one half of the possible page colors
  5614. * and the other with pages of the other colors.
  5615. */
  5616. batch = rounddown_pow_of_two(batch + batch/2) - 1;
  5617. return batch;
  5618. #else
  5619. /* The deferral and batching of frees should be suppressed under NOMMU
  5620. * conditions.
  5621. *
  5622. * The problem is that NOMMU needs to be able to allocate large chunks
  5623. * of contiguous memory as there's no hardware page translation to
  5624. * assemble apparent contiguous memory from discontiguous pages.
  5625. *
  5626. * Queueing large contiguous runs of pages for batching, however,
  5627. * causes the pages to actually be freed in smaller chunks. As there
  5628. * can be a significant delay between the individual batches being
  5629. * recycled, this leads to the once large chunks of space being
  5630. * fragmented and becoming unavailable for high-order allocations.
  5631. */
  5632. return 0;
  5633. #endif
  5634. }
  5635. /*
  5636. * pcp->high and pcp->batch values are related and dependent on one another:
  5637. * ->batch must never be higher then ->high.
  5638. * The following function updates them in a safe manner without read side
  5639. * locking.
  5640. *
  5641. * Any new users of pcp->batch and pcp->high should ensure they can cope with
  5642. * those fields changing asynchronously (acording to the above rule).
  5643. *
  5644. * mutex_is_locked(&pcp_batch_high_lock) required when calling this function
  5645. * outside of boot time (or some other assurance that no concurrent updaters
  5646. * exist).
  5647. */
  5648. static void pageset_update(struct per_cpu_pages *pcp, unsigned long high,
  5649. unsigned long batch)
  5650. {
  5651. /* start with a fail safe value for batch */
  5652. pcp->batch = 1;
  5653. smp_wmb();
  5654. /* Update high, then batch, in order */
  5655. pcp->high = high;
  5656. smp_wmb();
  5657. pcp->batch = batch;
  5658. }
  5659. /* a companion to pageset_set_high() */
  5660. static void pageset_set_batch(struct per_cpu_pageset *p, unsigned long batch)
  5661. {
  5662. pageset_update(&p->pcp, 6 * batch, max(1UL, 1 * batch));
  5663. }
  5664. static void pageset_init(struct per_cpu_pageset *p)
  5665. {
  5666. struct per_cpu_pages *pcp;
  5667. int migratetype;
  5668. memset(p, 0, sizeof(*p));
  5669. pcp = &p->pcp;
  5670. for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++)
  5671. INIT_LIST_HEAD(&pcp->lists[migratetype]);
  5672. }
  5673. static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch)
  5674. {
  5675. pageset_init(p);
  5676. pageset_set_batch(p, batch);
  5677. }
  5678. /*
  5679. * pageset_set_high() sets the high water mark for hot per_cpu_pagelist
  5680. * to the value high for the pageset p.
  5681. */
  5682. static void pageset_set_high(struct per_cpu_pageset *p,
  5683. unsigned long high)
  5684. {
  5685. unsigned long batch = max(1UL, high / 4);
  5686. if ((high / 4) > (PAGE_SHIFT * 8))
  5687. batch = PAGE_SHIFT * 8;
  5688. pageset_update(&p->pcp, high, batch);
  5689. }
  5690. static void pageset_set_high_and_batch(struct zone *zone,
  5691. struct per_cpu_pageset *pcp)
  5692. {
  5693. if (percpu_pagelist_fraction)
  5694. pageset_set_high(pcp,
  5695. (zone_managed_pages(zone) /
  5696. percpu_pagelist_fraction));
  5697. else
  5698. pageset_set_batch(pcp, zone_batchsize(zone));
  5699. }
  5700. static void __meminit zone_pageset_init(struct zone *zone, int cpu)
  5701. {
  5702. struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu);
  5703. pageset_init(pcp);
  5704. pageset_set_high_and_batch(zone, pcp);
  5705. }
  5706. void __meminit setup_zone_pageset(struct zone *zone)
  5707. {
  5708. int cpu;
  5709. zone->pageset = alloc_percpu(struct per_cpu_pageset);
  5710. for_each_possible_cpu(cpu)
  5711. zone_pageset_init(zone, cpu);
  5712. }
  5713. /*
  5714. * Allocate per cpu pagesets and initialize them.
  5715. * Before this call only boot pagesets were available.
  5716. */
  5717. void __init setup_per_cpu_pageset(void)
  5718. {
  5719. struct pglist_data *pgdat;
  5720. struct zone *zone;
  5721. int __maybe_unused cpu;
  5722. for_each_populated_zone(zone)
  5723. setup_zone_pageset(zone);
  5724. #ifdef CONFIG_NUMA
  5725. /*
  5726. * Unpopulated zones continue using the boot pagesets.
  5727. * The numa stats for these pagesets need to be reset.
  5728. * Otherwise, they will end up skewing the stats of
  5729. * the nodes these zones are associated with.
  5730. */
  5731. for_each_possible_cpu(cpu) {
  5732. struct per_cpu_pageset *pcp = &per_cpu(boot_pageset, cpu);
  5733. memset(pcp->vm_numa_stat_diff, 0,
  5734. sizeof(pcp->vm_numa_stat_diff));
  5735. }
  5736. #endif
  5737. for_each_online_pgdat(pgdat)
  5738. pgdat->per_cpu_nodestats =
  5739. alloc_percpu(struct per_cpu_nodestat);
  5740. }
  5741. static __meminit void zone_pcp_init(struct zone *zone)
  5742. {
  5743. /*
  5744. * per cpu subsystem is not up at this point. The following code
  5745. * relies on the ability of the linker to provide the
  5746. * offset of a (static) per cpu variable into the per cpu area.
  5747. */
  5748. zone->pageset = &boot_pageset;
  5749. if (populated_zone(zone))
  5750. printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n",
  5751. zone->name, zone->present_pages,
  5752. zone_batchsize(zone));
  5753. }
  5754. void __meminit init_currently_empty_zone(struct zone *zone,
  5755. unsigned long zone_start_pfn,
  5756. unsigned long size)
  5757. {
  5758. struct pglist_data *pgdat = zone->zone_pgdat;
  5759. int zone_idx = zone_idx(zone) + 1;
  5760. if (zone_idx > pgdat->nr_zones)
  5761. pgdat->nr_zones = zone_idx;
  5762. zone->zone_start_pfn = zone_start_pfn;
  5763. mminit_dprintk(MMINIT_TRACE, "memmap_init",
  5764. "Initialising map node %d zone %lu pfns %lu -> %lu\n",
  5765. pgdat->node_id,
  5766. (unsigned long)zone_idx(zone),
  5767. zone_start_pfn, (zone_start_pfn + size));
  5768. zone_init_free_lists(zone);
  5769. zone->initialized = 1;
  5770. }
  5771. /**
  5772. * get_pfn_range_for_nid - Return the start and end page frames for a node
  5773. * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned.
  5774. * @start_pfn: Passed by reference. On return, it will have the node start_pfn.
  5775. * @end_pfn: Passed by reference. On return, it will have the node end_pfn.
  5776. *
  5777. * It returns the start and end page frame of a node based on information
  5778. * provided by memblock_set_node(). If called for a node
  5779. * with no available memory, a warning is printed and the start and end
  5780. * PFNs will be 0.
  5781. */
  5782. void __init get_pfn_range_for_nid(unsigned int nid,
  5783. unsigned long *start_pfn, unsigned long *end_pfn)
  5784. {
  5785. unsigned long this_start_pfn, this_end_pfn;
  5786. int i;
  5787. *start_pfn = -1UL;
  5788. *end_pfn = 0;
  5789. for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) {
  5790. *start_pfn = min(*start_pfn, this_start_pfn);
  5791. *end_pfn = max(*end_pfn, this_end_pfn);
  5792. }
  5793. if (*start_pfn == -1UL)
  5794. *start_pfn = 0;
  5795. }
  5796. /*
  5797. * This finds a zone that can be used for ZONE_MOVABLE pages. The
  5798. * assumption is made that zones within a node are ordered in monotonic
  5799. * increasing memory addresses so that the "highest" populated zone is used
  5800. */
  5801. static void __init find_usable_zone_for_movable(void)
  5802. {
  5803. int zone_index;
  5804. for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) {
  5805. if (zone_index == ZONE_MOVABLE)
  5806. continue;
  5807. if (arch_zone_highest_possible_pfn[zone_index] >
  5808. arch_zone_lowest_possible_pfn[zone_index])
  5809. break;
  5810. }
  5811. VM_BUG_ON(zone_index == -1);
  5812. movable_zone = zone_index;
  5813. }
  5814. /*
  5815. * The zone ranges provided by the architecture do not include ZONE_MOVABLE
  5816. * because it is sized independent of architecture. Unlike the other zones,
  5817. * the starting point for ZONE_MOVABLE is not fixed. It may be different
  5818. * in each node depending on the size of each node and how evenly kernelcore
  5819. * is distributed. This helper function adjusts the zone ranges
  5820. * provided by the architecture for a given node by using the end of the
  5821. * highest usable zone for ZONE_MOVABLE. This preserves the assumption that
  5822. * zones within a node are in order of monotonic increases memory addresses
  5823. */
  5824. static void __init adjust_zone_range_for_zone_movable(int nid,
  5825. unsigned long zone_type,
  5826. unsigned long node_start_pfn,
  5827. unsigned long node_end_pfn,
  5828. unsigned long *zone_start_pfn,
  5829. unsigned long *zone_end_pfn)
  5830. {
  5831. /* Only adjust if ZONE_MOVABLE is on this node */
  5832. if (zone_movable_pfn[nid]) {
  5833. /* Size ZONE_MOVABLE */
  5834. if (zone_type == ZONE_MOVABLE) {
  5835. *zone_start_pfn = zone_movable_pfn[nid];
  5836. *zone_end_pfn = min(node_end_pfn,
  5837. arch_zone_highest_possible_pfn[movable_zone]);
  5838. /* Adjust for ZONE_MOVABLE starting within this range */
  5839. } else if (!mirrored_kernelcore &&
  5840. *zone_start_pfn < zone_movable_pfn[nid] &&
  5841. *zone_end_pfn > zone_movable_pfn[nid]) {
  5842. *zone_end_pfn = zone_movable_pfn[nid];
  5843. /* Check if this whole range is within ZONE_MOVABLE */
  5844. } else if (*zone_start_pfn >= zone_movable_pfn[nid])
  5845. *zone_start_pfn = *zone_end_pfn;
  5846. }
  5847. }
  5848. /*
  5849. * Return the number of pages a zone spans in a node, including holes
  5850. * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node()
  5851. */
  5852. static unsigned long __init zone_spanned_pages_in_node(int nid,
  5853. unsigned long zone_type,
  5854. unsigned long node_start_pfn,
  5855. unsigned long node_end_pfn,
  5856. unsigned long *zone_start_pfn,
  5857. unsigned long *zone_end_pfn)
  5858. {
  5859. unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
  5860. unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
  5861. /* When hotadd a new node from cpu_up(), the node should be empty */
  5862. if (!node_start_pfn && !node_end_pfn)
  5863. return 0;
  5864. /* Get the start and end of the zone */
  5865. *zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
  5866. *zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
  5867. adjust_zone_range_for_zone_movable(nid, zone_type,
  5868. node_start_pfn, node_end_pfn,
  5869. zone_start_pfn, zone_end_pfn);
  5870. /* Check that this node has pages within the zone's required range */
  5871. if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn)
  5872. return 0;
  5873. /* Move the zone boundaries inside the node if necessary */
  5874. *zone_end_pfn = min(*zone_end_pfn, node_end_pfn);
  5875. *zone_start_pfn = max(*zone_start_pfn, node_start_pfn);
  5876. /* Return the spanned pages */
  5877. return *zone_end_pfn - *zone_start_pfn;
  5878. }
  5879. /*
  5880. * Return the number of holes in a range on a node. If nid is MAX_NUMNODES,
  5881. * then all holes in the requested range will be accounted for.
  5882. */
  5883. unsigned long __init __absent_pages_in_range(int nid,
  5884. unsigned long range_start_pfn,
  5885. unsigned long range_end_pfn)
  5886. {
  5887. unsigned long nr_absent = range_end_pfn - range_start_pfn;
  5888. unsigned long start_pfn, end_pfn;
  5889. int i;
  5890. for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
  5891. start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn);
  5892. end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn);
  5893. nr_absent -= end_pfn - start_pfn;
  5894. }
  5895. return nr_absent;
  5896. }
  5897. /**
  5898. * absent_pages_in_range - Return number of page frames in holes within a range
  5899. * @start_pfn: The start PFN to start searching for holes
  5900. * @end_pfn: The end PFN to stop searching for holes
  5901. *
  5902. * Return: the number of pages frames in memory holes within a range.
  5903. */
  5904. unsigned long __init absent_pages_in_range(unsigned long start_pfn,
  5905. unsigned long end_pfn)
  5906. {
  5907. return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn);
  5908. }
  5909. /* Return the number of page frames in holes in a zone on a node */
  5910. static unsigned long __init zone_absent_pages_in_node(int nid,
  5911. unsigned long zone_type,
  5912. unsigned long node_start_pfn,
  5913. unsigned long node_end_pfn)
  5914. {
  5915. unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
  5916. unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
  5917. unsigned long zone_start_pfn, zone_end_pfn;
  5918. unsigned long nr_absent;
  5919. /* When hotadd a new node from cpu_up(), the node should be empty */
  5920. if (!node_start_pfn && !node_end_pfn)
  5921. return 0;
  5922. zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
  5923. zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
  5924. adjust_zone_range_for_zone_movable(nid, zone_type,
  5925. node_start_pfn, node_end_pfn,
  5926. &zone_start_pfn, &zone_end_pfn);
  5927. nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn);
  5928. /*
  5929. * ZONE_MOVABLE handling.
  5930. * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages
  5931. * and vice versa.
  5932. */
  5933. if (mirrored_kernelcore && zone_movable_pfn[nid]) {
  5934. unsigned long start_pfn, end_pfn;
  5935. struct memblock_region *r;
  5936. for_each_mem_region(r) {
  5937. start_pfn = clamp(memblock_region_memory_base_pfn(r),
  5938. zone_start_pfn, zone_end_pfn);
  5939. end_pfn = clamp(memblock_region_memory_end_pfn(r),
  5940. zone_start_pfn, zone_end_pfn);
  5941. if (zone_type == ZONE_MOVABLE &&
  5942. memblock_is_mirror(r))
  5943. nr_absent += end_pfn - start_pfn;
  5944. if (zone_type == ZONE_NORMAL &&
  5945. !memblock_is_mirror(r))
  5946. nr_absent += end_pfn - start_pfn;
  5947. }
  5948. }
  5949. return nr_absent;
  5950. }
  5951. static void __init calculate_node_totalpages(struct pglist_data *pgdat,
  5952. unsigned long node_start_pfn,
  5953. unsigned long node_end_pfn)
  5954. {
  5955. unsigned long realtotalpages = 0, totalpages = 0;
  5956. enum zone_type i;
  5957. for (i = 0; i < MAX_NR_ZONES; i++) {
  5958. struct zone *zone = pgdat->node_zones + i;
  5959. unsigned long zone_start_pfn, zone_end_pfn;
  5960. unsigned long spanned, absent;
  5961. unsigned long size, real_size;
  5962. spanned = zone_spanned_pages_in_node(pgdat->node_id, i,
  5963. node_start_pfn,
  5964. node_end_pfn,
  5965. &zone_start_pfn,
  5966. &zone_end_pfn);
  5967. absent = zone_absent_pages_in_node(pgdat->node_id, i,
  5968. node_start_pfn,
  5969. node_end_pfn);
  5970. size = spanned;
  5971. real_size = size - absent;
  5972. if (size)
  5973. zone->zone_start_pfn = zone_start_pfn;
  5974. else
  5975. zone->zone_start_pfn = 0;
  5976. zone->spanned_pages = size;
  5977. zone->present_pages = real_size;
  5978. totalpages += size;
  5979. realtotalpages += real_size;
  5980. }
  5981. pgdat->node_spanned_pages = totalpages;
  5982. pgdat->node_present_pages = realtotalpages;
  5983. printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id,
  5984. realtotalpages);
  5985. }
  5986. #ifndef CONFIG_SPARSEMEM
  5987. /*
  5988. * Calculate the size of the zone->blockflags rounded to an unsigned long
  5989. * Start by making sure zonesize is a multiple of pageblock_order by rounding
  5990. * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally
  5991. * round what is now in bits to nearest long in bits, then return it in
  5992. * bytes.
  5993. */
  5994. static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize)
  5995. {
  5996. unsigned long usemapsize;
  5997. zonesize += zone_start_pfn & (pageblock_nr_pages-1);
  5998. usemapsize = roundup(zonesize, pageblock_nr_pages);
  5999. usemapsize = usemapsize >> pageblock_order;
  6000. usemapsize *= NR_PAGEBLOCK_BITS;
  6001. usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long));
  6002. return usemapsize / 8;
  6003. }
  6004. static void __ref setup_usemap(struct pglist_data *pgdat,
  6005. struct zone *zone,
  6006. unsigned long zone_start_pfn,
  6007. unsigned long zonesize)
  6008. {
  6009. unsigned long usemapsize = usemap_size(zone_start_pfn, zonesize);
  6010. zone->pageblock_flags = NULL;
  6011. if (usemapsize) {
  6012. zone->pageblock_flags =
  6013. memblock_alloc_node(usemapsize, SMP_CACHE_BYTES,
  6014. pgdat->node_id);
  6015. if (!zone->pageblock_flags)
  6016. panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n",
  6017. usemapsize, zone->name, pgdat->node_id);
  6018. }
  6019. }
  6020. #else
  6021. static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone,
  6022. unsigned long zone_start_pfn, unsigned long zonesize) {}
  6023. #endif /* CONFIG_SPARSEMEM */
  6024. #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
  6025. /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */
  6026. void __init set_pageblock_order(void)
  6027. {
  6028. unsigned int order;
  6029. /* Check that pageblock_nr_pages has not already been setup */
  6030. if (pageblock_order)
  6031. return;
  6032. if (HPAGE_SHIFT > PAGE_SHIFT)
  6033. order = HUGETLB_PAGE_ORDER;
  6034. else
  6035. order = MAX_ORDER - 1;
  6036. /*
  6037. * Assume the largest contiguous order of interest is a huge page.
  6038. * This value may be variable depending on boot parameters on IA64 and
  6039. * powerpc.
  6040. */
  6041. pageblock_order = order;
  6042. }
  6043. #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
  6044. /*
  6045. * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order()
  6046. * is unused as pageblock_order is set at compile-time. See
  6047. * include/linux/pageblock-flags.h for the values of pageblock_order based on
  6048. * the kernel config
  6049. */
  6050. void __init set_pageblock_order(void)
  6051. {
  6052. }
  6053. #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
  6054. static unsigned long __init calc_memmap_size(unsigned long spanned_pages,
  6055. unsigned long present_pages)
  6056. {
  6057. unsigned long pages = spanned_pages;
  6058. /*
  6059. * Provide a more accurate estimation if there are holes within
  6060. * the zone and SPARSEMEM is in use. If there are holes within the
  6061. * zone, each populated memory region may cost us one or two extra
  6062. * memmap pages due to alignment because memmap pages for each
  6063. * populated regions may not be naturally aligned on page boundary.
  6064. * So the (present_pages >> 4) heuristic is a tradeoff for that.
  6065. */
  6066. if (spanned_pages > present_pages + (present_pages >> 4) &&
  6067. IS_ENABLED(CONFIG_SPARSEMEM))
  6068. pages = present_pages;
  6069. return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT;
  6070. }
  6071. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  6072. static void pgdat_init_split_queue(struct pglist_data *pgdat)
  6073. {
  6074. struct deferred_split *ds_queue = &pgdat->deferred_split_queue;
  6075. spin_lock_init(&ds_queue->split_queue_lock);
  6076. INIT_LIST_HEAD(&ds_queue->split_queue);
  6077. ds_queue->split_queue_len = 0;
  6078. }
  6079. #else
  6080. static void pgdat_init_split_queue(struct pglist_data *pgdat) {}
  6081. #endif
  6082. #ifdef CONFIG_COMPACTION
  6083. static void pgdat_init_kcompactd(struct pglist_data *pgdat)
  6084. {
  6085. init_waitqueue_head(&pgdat->kcompactd_wait);
  6086. }
  6087. #else
  6088. static void pgdat_init_kcompactd(struct pglist_data *pgdat) {}
  6089. #endif
  6090. static void __meminit pgdat_init_internals(struct pglist_data *pgdat)
  6091. {
  6092. pgdat_resize_init(pgdat);
  6093. pgdat_init_split_queue(pgdat);
  6094. pgdat_init_kcompactd(pgdat);
  6095. init_waitqueue_head(&pgdat->kswapd_wait);
  6096. init_waitqueue_head(&pgdat->pfmemalloc_wait);
  6097. pgdat_page_ext_init(pgdat);
  6098. spin_lock_init(&pgdat->lru_lock);
  6099. lruvec_init(&pgdat->__lruvec);
  6100. }
  6101. static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid,
  6102. unsigned long remaining_pages)
  6103. {
  6104. atomic_long_set(&zone->managed_pages, remaining_pages);
  6105. zone_set_nid(zone, nid);
  6106. zone->name = zone_names[idx];
  6107. zone->zone_pgdat = NODE_DATA(nid);
  6108. spin_lock_init(&zone->lock);
  6109. zone_seqlock_init(zone);
  6110. zone_pcp_init(zone);
  6111. }
  6112. /*
  6113. * Set up the zone data structures
  6114. * - init pgdat internals
  6115. * - init all zones belonging to this node
  6116. *
  6117. * NOTE: this function is only called during memory hotplug
  6118. */
  6119. #ifdef CONFIG_MEMORY_HOTPLUG
  6120. void __ref free_area_init_core_hotplug(int nid)
  6121. {
  6122. enum zone_type z;
  6123. pg_data_t *pgdat = NODE_DATA(nid);
  6124. pgdat_init_internals(pgdat);
  6125. for (z = 0; z < MAX_NR_ZONES; z++)
  6126. zone_init_internals(&pgdat->node_zones[z], z, nid, 0);
  6127. }
  6128. #endif
  6129. /*
  6130. * Set up the zone data structures:
  6131. * - mark all pages reserved
  6132. * - mark all memory queues empty
  6133. * - clear the memory bitmaps
  6134. *
  6135. * NOTE: pgdat should get zeroed by caller.
  6136. * NOTE: this function is only called during early init.
  6137. */
  6138. static void __init free_area_init_core(struct pglist_data *pgdat)
  6139. {
  6140. enum zone_type j;
  6141. int nid = pgdat->node_id;
  6142. pgdat_init_internals(pgdat);
  6143. pgdat->per_cpu_nodestats = &boot_nodestats;
  6144. for (j = 0; j < MAX_NR_ZONES; j++) {
  6145. struct zone *zone = pgdat->node_zones + j;
  6146. unsigned long size, freesize, memmap_pages;
  6147. unsigned long zone_start_pfn = zone->zone_start_pfn;
  6148. size = zone->spanned_pages;
  6149. freesize = zone->present_pages;
  6150. /*
  6151. * Adjust freesize so that it accounts for how much memory
  6152. * is used by this zone for memmap. This affects the watermark
  6153. * and per-cpu initialisations
  6154. */
  6155. memmap_pages = calc_memmap_size(size, freesize);
  6156. if (!is_highmem_idx(j)) {
  6157. if (freesize >= memmap_pages) {
  6158. freesize -= memmap_pages;
  6159. if (memmap_pages)
  6160. printk(KERN_DEBUG
  6161. " %s zone: %lu pages used for memmap\n",
  6162. zone_names[j], memmap_pages);
  6163. } else
  6164. pr_warn(" %s zone: %lu pages exceeds freesize %lu\n",
  6165. zone_names[j], memmap_pages, freesize);
  6166. }
  6167. /* Account for reserved pages */
  6168. if (j == 0 && freesize > dma_reserve) {
  6169. freesize -= dma_reserve;
  6170. printk(KERN_DEBUG " %s zone: %lu pages reserved\n",
  6171. zone_names[0], dma_reserve);
  6172. }
  6173. if (!is_highmem_idx(j))
  6174. nr_kernel_pages += freesize;
  6175. /* Charge for highmem memmap if there are enough kernel pages */
  6176. else if (nr_kernel_pages > memmap_pages * 2)
  6177. nr_kernel_pages -= memmap_pages;
  6178. nr_all_pages += freesize;
  6179. /*
  6180. * Set an approximate value for lowmem here, it will be adjusted
  6181. * when the bootmem allocator frees pages into the buddy system.
  6182. * And all highmem pages will be managed by the buddy system.
  6183. */
  6184. zone_init_internals(zone, j, nid, freesize);
  6185. if (!size)
  6186. continue;
  6187. set_pageblock_order();
  6188. setup_usemap(pgdat, zone, zone_start_pfn, size);
  6189. init_currently_empty_zone(zone, zone_start_pfn, size);
  6190. arch_memmap_init(size, nid, j, zone_start_pfn);
  6191. }
  6192. }
  6193. #ifdef CONFIG_FLAT_NODE_MEM_MAP
  6194. static void __ref alloc_node_mem_map(struct pglist_data *pgdat)
  6195. {
  6196. unsigned long __maybe_unused start = 0;
  6197. unsigned long __maybe_unused offset = 0;
  6198. /* Skip empty nodes */
  6199. if (!pgdat->node_spanned_pages)
  6200. return;
  6201. start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1);
  6202. offset = pgdat->node_start_pfn - start;
  6203. /* ia64 gets its own node_mem_map, before this, without bootmem */
  6204. if (!pgdat->node_mem_map) {
  6205. unsigned long size, end;
  6206. struct page *map;
  6207. /*
  6208. * The zone's endpoints aren't required to be MAX_ORDER
  6209. * aligned but the node_mem_map endpoints must be in order
  6210. * for the buddy allocator to function correctly.
  6211. */
  6212. end = pgdat_end_pfn(pgdat);
  6213. end = ALIGN(end, MAX_ORDER_NR_PAGES);
  6214. size = (end - start) * sizeof(struct page);
  6215. map = memblock_alloc_node(size, SMP_CACHE_BYTES,
  6216. pgdat->node_id);
  6217. if (!map)
  6218. panic("Failed to allocate %ld bytes for node %d memory map\n",
  6219. size, pgdat->node_id);
  6220. pgdat->node_mem_map = map + offset;
  6221. }
  6222. pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n",
  6223. __func__, pgdat->node_id, (unsigned long)pgdat,
  6224. (unsigned long)pgdat->node_mem_map);
  6225. #ifndef CONFIG_NEED_MULTIPLE_NODES
  6226. /*
  6227. * With no DISCONTIG, the global mem_map is just set as node 0's
  6228. */
  6229. if (pgdat == NODE_DATA(0)) {
  6230. mem_map = NODE_DATA(0)->node_mem_map;
  6231. if (page_to_pfn(mem_map) != pgdat->node_start_pfn)
  6232. mem_map -= offset;
  6233. }
  6234. #endif
  6235. }
  6236. #else
  6237. static void __ref alloc_node_mem_map(struct pglist_data *pgdat) { }
  6238. #endif /* CONFIG_FLAT_NODE_MEM_MAP */
  6239. #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
  6240. static inline void pgdat_set_deferred_range(pg_data_t *pgdat)
  6241. {
  6242. pgdat->first_deferred_pfn = ULONG_MAX;
  6243. }
  6244. #else
  6245. static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {}
  6246. #endif
  6247. static void __init free_area_init_node(int nid)
  6248. {
  6249. pg_data_t *pgdat = NODE_DATA(nid);
  6250. unsigned long start_pfn = 0;
  6251. unsigned long end_pfn = 0;
  6252. /* pg_data_t should be reset to zero when it's allocated */
  6253. WARN_ON(pgdat->nr_zones || pgdat->kswapd_highest_zoneidx);
  6254. get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
  6255. pgdat->node_id = nid;
  6256. pgdat->node_start_pfn = start_pfn;
  6257. pgdat->per_cpu_nodestats = NULL;
  6258. pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid,
  6259. (u64)start_pfn << PAGE_SHIFT,
  6260. end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0);
  6261. calculate_node_totalpages(pgdat, start_pfn, end_pfn);
  6262. alloc_node_mem_map(pgdat);
  6263. pgdat_set_deferred_range(pgdat);
  6264. free_area_init_core(pgdat);
  6265. }
  6266. void __init free_area_init_memoryless_node(int nid)
  6267. {
  6268. free_area_init_node(nid);
  6269. }
  6270. #if MAX_NUMNODES > 1
  6271. /*
  6272. * Figure out the number of possible node ids.
  6273. */
  6274. void __init setup_nr_node_ids(void)
  6275. {
  6276. unsigned int highest;
  6277. highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES);
  6278. nr_node_ids = highest + 1;
  6279. }
  6280. #endif
  6281. /**
  6282. * node_map_pfn_alignment - determine the maximum internode alignment
  6283. *
  6284. * This function should be called after node map is populated and sorted.
  6285. * It calculates the maximum power of two alignment which can distinguish
  6286. * all the nodes.
  6287. *
  6288. * For example, if all nodes are 1GiB and aligned to 1GiB, the return value
  6289. * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the
  6290. * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is
  6291. * shifted, 1GiB is enough and this function will indicate so.
  6292. *
  6293. * This is used to test whether pfn -> nid mapping of the chosen memory
  6294. * model has fine enough granularity to avoid incorrect mapping for the
  6295. * populated node map.
  6296. *
  6297. * Return: the determined alignment in pfn's. 0 if there is no alignment
  6298. * requirement (single node).
  6299. */
  6300. unsigned long __init node_map_pfn_alignment(void)
  6301. {
  6302. unsigned long accl_mask = 0, last_end = 0;
  6303. unsigned long start, end, mask;
  6304. int last_nid = NUMA_NO_NODE;
  6305. int i, nid;
  6306. for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) {
  6307. if (!start || last_nid < 0 || last_nid == nid) {
  6308. last_nid = nid;
  6309. last_end = end;
  6310. continue;
  6311. }
  6312. /*
  6313. * Start with a mask granular enough to pin-point to the
  6314. * start pfn and tick off bits one-by-one until it becomes
  6315. * too coarse to separate the current node from the last.
  6316. */
  6317. mask = ~((1 << __ffs(start)) - 1);
  6318. while (mask && last_end <= (start & (mask << 1)))
  6319. mask <<= 1;
  6320. /* accumulate all internode masks */
  6321. accl_mask |= mask;
  6322. }
  6323. /* convert mask to number of pages */
  6324. return ~accl_mask + 1;
  6325. }
  6326. /**
  6327. * find_min_pfn_with_active_regions - Find the minimum PFN registered
  6328. *
  6329. * Return: the minimum PFN based on information provided via
  6330. * memblock_set_node().
  6331. */
  6332. unsigned long __init find_min_pfn_with_active_regions(void)
  6333. {
  6334. return PHYS_PFN(memblock_start_of_DRAM());
  6335. }
  6336. /*
  6337. * early_calculate_totalpages()
  6338. * Sum pages in active regions for movable zone.
  6339. * Populate N_MEMORY for calculating usable_nodes.
  6340. */
  6341. static unsigned long __init early_calculate_totalpages(void)
  6342. {
  6343. unsigned long totalpages = 0;
  6344. unsigned long start_pfn, end_pfn;
  6345. int i, nid;
  6346. for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
  6347. unsigned long pages = end_pfn - start_pfn;
  6348. totalpages += pages;
  6349. if (pages)
  6350. node_set_state(nid, N_MEMORY);
  6351. }
  6352. return totalpages;
  6353. }
  6354. /*
  6355. * Find the PFN the Movable zone begins in each node. Kernel memory
  6356. * is spread evenly between nodes as long as the nodes have enough
  6357. * memory. When they don't, some nodes will have more kernelcore than
  6358. * others
  6359. */
  6360. static void __init find_zone_movable_pfns_for_nodes(void)
  6361. {
  6362. int i, nid;
  6363. unsigned long usable_startpfn;
  6364. unsigned long kernelcore_node, kernelcore_remaining;
  6365. /* save the state before borrow the nodemask */
  6366. nodemask_t saved_node_state = node_states[N_MEMORY];
  6367. unsigned long totalpages = early_calculate_totalpages();
  6368. int usable_nodes = nodes_weight(node_states[N_MEMORY]);
  6369. struct memblock_region *r;
  6370. /* Need to find movable_zone earlier when movable_node is specified. */
  6371. find_usable_zone_for_movable();
  6372. /*
  6373. * If movable_node is specified, ignore kernelcore and movablecore
  6374. * options.
  6375. */
  6376. if (movable_node_is_enabled()) {
  6377. for_each_mem_region(r) {
  6378. if (!memblock_is_hotpluggable(r))
  6379. continue;
  6380. nid = memblock_get_region_node(r);
  6381. usable_startpfn = PFN_DOWN(r->base);
  6382. zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
  6383. min(usable_startpfn, zone_movable_pfn[nid]) :
  6384. usable_startpfn;
  6385. }
  6386. goto out2;
  6387. }
  6388. /*
  6389. * If kernelcore=mirror is specified, ignore movablecore option
  6390. */
  6391. if (mirrored_kernelcore) {
  6392. bool mem_below_4gb_not_mirrored = false;
  6393. for_each_mem_region(r) {
  6394. if (memblock_is_mirror(r))
  6395. continue;
  6396. nid = memblock_get_region_node(r);
  6397. usable_startpfn = memblock_region_memory_base_pfn(r);
  6398. if (usable_startpfn < 0x100000) {
  6399. mem_below_4gb_not_mirrored = true;
  6400. continue;
  6401. }
  6402. zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
  6403. min(usable_startpfn, zone_movable_pfn[nid]) :
  6404. usable_startpfn;
  6405. }
  6406. if (mem_below_4gb_not_mirrored)
  6407. pr_warn("This configuration results in unmirrored kernel memory.\n");
  6408. goto out2;
  6409. }
  6410. /*
  6411. * If kernelcore=nn% or movablecore=nn% was specified, calculate the
  6412. * amount of necessary memory.
  6413. */
  6414. if (required_kernelcore_percent)
  6415. required_kernelcore = (totalpages * 100 * required_kernelcore_percent) /
  6416. 10000UL;
  6417. if (required_movablecore_percent)
  6418. required_movablecore = (totalpages * 100 * required_movablecore_percent) /
  6419. 10000UL;
  6420. /*
  6421. * If movablecore= was specified, calculate what size of
  6422. * kernelcore that corresponds so that memory usable for
  6423. * any allocation type is evenly spread. If both kernelcore
  6424. * and movablecore are specified, then the value of kernelcore
  6425. * will be used for required_kernelcore if it's greater than
  6426. * what movablecore would have allowed.
  6427. */
  6428. if (required_movablecore) {
  6429. unsigned long corepages;
  6430. /*
  6431. * Round-up so that ZONE_MOVABLE is at least as large as what
  6432. * was requested by the user
  6433. */
  6434. required_movablecore =
  6435. roundup(required_movablecore, MAX_ORDER_NR_PAGES);
  6436. required_movablecore = min(totalpages, required_movablecore);
  6437. corepages = totalpages - required_movablecore;
  6438. required_kernelcore = max(required_kernelcore, corepages);
  6439. }
  6440. /*
  6441. * If kernelcore was not specified or kernelcore size is larger
  6442. * than totalpages, there is no ZONE_MOVABLE.
  6443. */
  6444. if (!required_kernelcore || required_kernelcore >= totalpages)
  6445. goto out;
  6446. /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */
  6447. usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone];
  6448. restart:
  6449. /* Spread kernelcore memory as evenly as possible throughout nodes */
  6450. kernelcore_node = required_kernelcore / usable_nodes;
  6451. for_each_node_state(nid, N_MEMORY) {
  6452. unsigned long start_pfn, end_pfn;
  6453. /*
  6454. * Recalculate kernelcore_node if the division per node
  6455. * now exceeds what is necessary to satisfy the requested
  6456. * amount of memory for the kernel
  6457. */
  6458. if (required_kernelcore < kernelcore_node)
  6459. kernelcore_node = required_kernelcore / usable_nodes;
  6460. /*
  6461. * As the map is walked, we track how much memory is usable
  6462. * by the kernel using kernelcore_remaining. When it is
  6463. * 0, the rest of the node is usable by ZONE_MOVABLE
  6464. */
  6465. kernelcore_remaining = kernelcore_node;
  6466. /* Go through each range of PFNs within this node */
  6467. for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
  6468. unsigned long size_pages;
  6469. start_pfn = max(start_pfn, zone_movable_pfn[nid]);
  6470. if (start_pfn >= end_pfn)
  6471. continue;
  6472. /* Account for what is only usable for kernelcore */
  6473. if (start_pfn < usable_startpfn) {
  6474. unsigned long kernel_pages;
  6475. kernel_pages = min(end_pfn, usable_startpfn)
  6476. - start_pfn;
  6477. kernelcore_remaining -= min(kernel_pages,
  6478. kernelcore_remaining);
  6479. required_kernelcore -= min(kernel_pages,
  6480. required_kernelcore);
  6481. /* Continue if range is now fully accounted */
  6482. if (end_pfn <= usable_startpfn) {
  6483. /*
  6484. * Push zone_movable_pfn to the end so
  6485. * that if we have to rebalance
  6486. * kernelcore across nodes, we will
  6487. * not double account here
  6488. */
  6489. zone_movable_pfn[nid] = end_pfn;
  6490. continue;
  6491. }
  6492. start_pfn = usable_startpfn;
  6493. }
  6494. /*
  6495. * The usable PFN range for ZONE_MOVABLE is from
  6496. * start_pfn->end_pfn. Calculate size_pages as the
  6497. * number of pages used as kernelcore
  6498. */
  6499. size_pages = end_pfn - start_pfn;
  6500. if (size_pages > kernelcore_remaining)
  6501. size_pages = kernelcore_remaining;
  6502. zone_movable_pfn[nid] = start_pfn + size_pages;
  6503. /*
  6504. * Some kernelcore has been met, update counts and
  6505. * break if the kernelcore for this node has been
  6506. * satisfied
  6507. */
  6508. required_kernelcore -= min(required_kernelcore,
  6509. size_pages);
  6510. kernelcore_remaining -= size_pages;
  6511. if (!kernelcore_remaining)
  6512. break;
  6513. }
  6514. }
  6515. /*
  6516. * If there is still required_kernelcore, we do another pass with one
  6517. * less node in the count. This will push zone_movable_pfn[nid] further
  6518. * along on the nodes that still have memory until kernelcore is
  6519. * satisfied
  6520. */
  6521. usable_nodes--;
  6522. if (usable_nodes && required_kernelcore > usable_nodes)
  6523. goto restart;
  6524. out2:
  6525. /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */
  6526. for (nid = 0; nid < MAX_NUMNODES; nid++) {
  6527. unsigned long start_pfn, end_pfn;
  6528. zone_movable_pfn[nid] =
  6529. roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
  6530. get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
  6531. if (zone_movable_pfn[nid] >= end_pfn)
  6532. zone_movable_pfn[nid] = 0;
  6533. }
  6534. out:
  6535. /* restore the node_state */
  6536. node_states[N_MEMORY] = saved_node_state;
  6537. }
  6538. /* Any regular or high memory on that node ? */
  6539. static void check_for_memory(pg_data_t *pgdat, int nid)
  6540. {
  6541. enum zone_type zone_type;
  6542. for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) {
  6543. struct zone *zone = &pgdat->node_zones[zone_type];
  6544. if (populated_zone(zone)) {
  6545. if (IS_ENABLED(CONFIG_HIGHMEM))
  6546. node_set_state(nid, N_HIGH_MEMORY);
  6547. if (zone_type <= ZONE_NORMAL)
  6548. node_set_state(nid, N_NORMAL_MEMORY);
  6549. break;
  6550. }
  6551. }
  6552. }
  6553. /*
  6554. * Some architecturs, e.g. ARC may have ZONE_HIGHMEM below ZONE_NORMAL. For
  6555. * such cases we allow max_zone_pfn sorted in the descending order
  6556. */
  6557. bool __weak arch_has_descending_max_zone_pfns(void)
  6558. {
  6559. return false;
  6560. }
  6561. /**
  6562. * free_area_init - Initialise all pg_data_t and zone data
  6563. * @max_zone_pfn: an array of max PFNs for each zone
  6564. *
  6565. * This will call free_area_init_node() for each active node in the system.
  6566. * Using the page ranges provided by memblock_set_node(), the size of each
  6567. * zone in each node and their holes is calculated. If the maximum PFN
  6568. * between two adjacent zones match, it is assumed that the zone is empty.
  6569. * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed
  6570. * that arch_max_dma32_pfn has no pages. It is also assumed that a zone
  6571. * starts where the previous one ended. For example, ZONE_DMA32 starts
  6572. * at arch_max_dma_pfn.
  6573. */
  6574. void __init free_area_init(unsigned long *max_zone_pfn)
  6575. {
  6576. unsigned long start_pfn, end_pfn;
  6577. int i, nid, zone;
  6578. bool descending;
  6579. /* Record where the zone boundaries are */
  6580. memset(arch_zone_lowest_possible_pfn, 0,
  6581. sizeof(arch_zone_lowest_possible_pfn));
  6582. memset(arch_zone_highest_possible_pfn, 0,
  6583. sizeof(arch_zone_highest_possible_pfn));
  6584. start_pfn = find_min_pfn_with_active_regions();
  6585. descending = arch_has_descending_max_zone_pfns();
  6586. for (i = 0; i < MAX_NR_ZONES; i++) {
  6587. if (descending)
  6588. zone = MAX_NR_ZONES - i - 1;
  6589. else
  6590. zone = i;
  6591. if (zone == ZONE_MOVABLE)
  6592. continue;
  6593. end_pfn = max(max_zone_pfn[zone], start_pfn);
  6594. arch_zone_lowest_possible_pfn[zone] = start_pfn;
  6595. arch_zone_highest_possible_pfn[zone] = end_pfn;
  6596. start_pfn = end_pfn;
  6597. }
  6598. /* Find the PFNs that ZONE_MOVABLE begins at in each node */
  6599. memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn));
  6600. find_zone_movable_pfns_for_nodes();
  6601. /* Print out the zone ranges */
  6602. pr_info("Zone ranges:\n");
  6603. for (i = 0; i < MAX_NR_ZONES; i++) {
  6604. if (i == ZONE_MOVABLE)
  6605. continue;
  6606. pr_info(" %-8s ", zone_names[i]);
  6607. if (arch_zone_lowest_possible_pfn[i] ==
  6608. arch_zone_highest_possible_pfn[i])
  6609. pr_cont("empty\n");
  6610. else
  6611. pr_cont("[mem %#018Lx-%#018Lx]\n",
  6612. (u64)arch_zone_lowest_possible_pfn[i]
  6613. << PAGE_SHIFT,
  6614. ((u64)arch_zone_highest_possible_pfn[i]
  6615. << PAGE_SHIFT) - 1);
  6616. }
  6617. /* Print out the PFNs ZONE_MOVABLE begins at in each node */
  6618. pr_info("Movable zone start for each node\n");
  6619. for (i = 0; i < MAX_NUMNODES; i++) {
  6620. if (zone_movable_pfn[i])
  6621. pr_info(" Node %d: %#018Lx\n", i,
  6622. (u64)zone_movable_pfn[i] << PAGE_SHIFT);
  6623. }
  6624. /*
  6625. * Print out the early node map, and initialize the
  6626. * subsection-map relative to active online memory ranges to
  6627. * enable future "sub-section" extensions of the memory map.
  6628. */
  6629. pr_info("Early memory node ranges\n");
  6630. for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
  6631. pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid,
  6632. (u64)start_pfn << PAGE_SHIFT,
  6633. ((u64)end_pfn << PAGE_SHIFT) - 1);
  6634. subsection_map_init(start_pfn, end_pfn - start_pfn);
  6635. }
  6636. /* Initialise every node */
  6637. mminit_verify_pageflags_layout();
  6638. setup_nr_node_ids();
  6639. for_each_online_node(nid) {
  6640. pg_data_t *pgdat = NODE_DATA(nid);
  6641. free_area_init_node(nid);
  6642. /* Any memory on that node */
  6643. if (pgdat->node_present_pages)
  6644. node_set_state(nid, N_MEMORY);
  6645. check_for_memory(pgdat, nid);
  6646. }
  6647. memmap_init();
  6648. }
  6649. static int __init cmdline_parse_core(char *p, unsigned long *core,
  6650. unsigned long *percent)
  6651. {
  6652. unsigned long long coremem;
  6653. char *endptr;
  6654. if (!p)
  6655. return -EINVAL;
  6656. /* Value may be a percentage of total memory, otherwise bytes */
  6657. coremem = simple_strtoull(p, &endptr, 0);
  6658. if (*endptr == '%') {
  6659. /* Paranoid check for percent values greater than 100 */
  6660. WARN_ON(coremem > 100);
  6661. *percent = coremem;
  6662. } else {
  6663. coremem = memparse(p, &p);
  6664. /* Paranoid check that UL is enough for the coremem value */
  6665. WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX);
  6666. *core = coremem >> PAGE_SHIFT;
  6667. *percent = 0UL;
  6668. }
  6669. return 0;
  6670. }
  6671. /*
  6672. * kernelcore=size sets the amount of memory for use for allocations that
  6673. * cannot be reclaimed or migrated.
  6674. */
  6675. static int __init cmdline_parse_kernelcore(char *p)
  6676. {
  6677. /* parse kernelcore=mirror */
  6678. if (parse_option_str(p, "mirror")) {
  6679. mirrored_kernelcore = true;
  6680. return 0;
  6681. }
  6682. return cmdline_parse_core(p, &required_kernelcore,
  6683. &required_kernelcore_percent);
  6684. }
  6685. /*
  6686. * movablecore=size sets the amount of memory for use for allocations that
  6687. * can be reclaimed or migrated.
  6688. */
  6689. static int __init cmdline_parse_movablecore(char *p)
  6690. {
  6691. return cmdline_parse_core(p, &required_movablecore,
  6692. &required_movablecore_percent);
  6693. }
  6694. early_param("kernelcore", cmdline_parse_kernelcore);
  6695. early_param("movablecore", cmdline_parse_movablecore);
  6696. void adjust_managed_page_count(struct page *page, long count)
  6697. {
  6698. atomic_long_add(count, &page_zone(page)->managed_pages);
  6699. totalram_pages_add(count);
  6700. #ifdef CONFIG_HIGHMEM
  6701. if (PageHighMem(page))
  6702. totalhigh_pages_add(count);
  6703. #endif
  6704. }
  6705. EXPORT_SYMBOL(adjust_managed_page_count);
  6706. unsigned long free_reserved_area(void *start, void *end, int poison, const char *s)
  6707. {
  6708. void *pos;
  6709. unsigned long pages = 0;
  6710. start = (void *)PAGE_ALIGN((unsigned long)start);
  6711. end = (void *)((unsigned long)end & PAGE_MASK);
  6712. for (pos = start; pos < end; pos += PAGE_SIZE, pages++) {
  6713. struct page *page = virt_to_page(pos);
  6714. void *direct_map_addr;
  6715. /*
  6716. * 'direct_map_addr' might be different from 'pos'
  6717. * because some architectures' virt_to_page()
  6718. * work with aliases. Getting the direct map
  6719. * address ensures that we get a _writeable_
  6720. * alias for the memset().
  6721. */
  6722. direct_map_addr = page_address(page);
  6723. /*
  6724. * Perform a kasan-unchecked memset() since this memory
  6725. * has not been initialized.
  6726. */
  6727. direct_map_addr = kasan_reset_tag(direct_map_addr);
  6728. if ((unsigned int)poison <= 0xFF)
  6729. memset(direct_map_addr, poison, PAGE_SIZE);
  6730. free_reserved_page(page);
  6731. }
  6732. if (pages && s)
  6733. pr_info("Freeing %s memory: %ldK\n",
  6734. s, pages << (PAGE_SHIFT - 10));
  6735. return pages;
  6736. }
  6737. #ifdef CONFIG_HIGHMEM
  6738. void free_highmem_page(struct page *page)
  6739. {
  6740. __free_reserved_page(page);
  6741. totalram_pages_inc();
  6742. atomic_long_inc(&page_zone(page)->managed_pages);
  6743. totalhigh_pages_inc();
  6744. }
  6745. #endif
  6746. void __init mem_init_print_info(const char *str)
  6747. {
  6748. unsigned long physpages, codesize, datasize, rosize, bss_size;
  6749. unsigned long init_code_size, init_data_size;
  6750. physpages = get_num_physpages();
  6751. codesize = _etext - _stext;
  6752. datasize = _edata - _sdata;
  6753. rosize = __end_rodata - __start_rodata;
  6754. bss_size = __bss_stop - __bss_start;
  6755. init_data_size = __init_end - __init_begin;
  6756. init_code_size = _einittext - _sinittext;
  6757. /*
  6758. * Detect special cases and adjust section sizes accordingly:
  6759. * 1) .init.* may be embedded into .data sections
  6760. * 2) .init.text.* may be out of [__init_begin, __init_end],
  6761. * please refer to arch/tile/kernel/vmlinux.lds.S.
  6762. * 3) .rodata.* may be embedded into .text or .data sections.
  6763. */
  6764. #define adj_init_size(start, end, size, pos, adj) \
  6765. do { \
  6766. if (&start[0] <= &pos[0] && &pos[0] < &end[0] && size > adj) \
  6767. size -= adj; \
  6768. } while (0)
  6769. adj_init_size(__init_begin, __init_end, init_data_size,
  6770. _sinittext, init_code_size);
  6771. adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size);
  6772. adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size);
  6773. adj_init_size(_stext, _etext, codesize, __start_rodata, rosize);
  6774. adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize);
  6775. #undef adj_init_size
  6776. pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved"
  6777. #ifdef CONFIG_HIGHMEM
  6778. ", %luK highmem"
  6779. #endif
  6780. "%s%s)\n",
  6781. nr_free_pages() << (PAGE_SHIFT - 10),
  6782. physpages << (PAGE_SHIFT - 10),
  6783. codesize >> 10, datasize >> 10, rosize >> 10,
  6784. (init_data_size + init_code_size) >> 10, bss_size >> 10,
  6785. (physpages - totalram_pages() - totalcma_pages) << (PAGE_SHIFT - 10),
  6786. totalcma_pages << (PAGE_SHIFT - 10),
  6787. #ifdef CONFIG_HIGHMEM
  6788. totalhigh_pages() << (PAGE_SHIFT - 10),
  6789. #endif
  6790. str ? ", " : "", str ? str : "");
  6791. }
  6792. /**
  6793. * set_dma_reserve - set the specified number of pages reserved in the first zone
  6794. * @new_dma_reserve: The number of pages to mark reserved
  6795. *
  6796. * The per-cpu batchsize and zone watermarks are determined by managed_pages.
  6797. * In the DMA zone, a significant percentage may be consumed by kernel image
  6798. * and other unfreeable allocations which can skew the watermarks badly. This
  6799. * function may optionally be used to account for unfreeable pages in the
  6800. * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and
  6801. * smaller per-cpu batchsize.
  6802. */
  6803. void __init set_dma_reserve(unsigned long new_dma_reserve)
  6804. {
  6805. dma_reserve = new_dma_reserve;
  6806. }
  6807. static int page_alloc_cpu_dead(unsigned int cpu)
  6808. {
  6809. lru_add_drain_cpu(cpu);
  6810. drain_pages(cpu);
  6811. /*
  6812. * Spill the event counters of the dead processor
  6813. * into the current processors event counters.
  6814. * This artificially elevates the count of the current
  6815. * processor.
  6816. */
  6817. vm_events_fold_cpu(cpu);
  6818. /*
  6819. * Zero the differential counters of the dead processor
  6820. * so that the vm statistics are consistent.
  6821. *
  6822. * This is only okay since the processor is dead and cannot
  6823. * race with what we are doing.
  6824. */
  6825. cpu_vm_stats_fold(cpu);
  6826. return 0;
  6827. }
  6828. #ifdef CONFIG_NUMA
  6829. int hashdist = HASHDIST_DEFAULT;
  6830. static int __init set_hashdist(char *str)
  6831. {
  6832. if (!str)
  6833. return 0;
  6834. hashdist = simple_strtoul(str, &str, 0);
  6835. return 1;
  6836. }
  6837. __setup("hashdist=", set_hashdist);
  6838. #endif
  6839. void __init page_alloc_init(void)
  6840. {
  6841. int ret;
  6842. #ifdef CONFIG_NUMA
  6843. if (num_node_state(N_MEMORY) == 1)
  6844. hashdist = 0;
  6845. #endif
  6846. ret = cpuhp_setup_state_nocalls(CPUHP_PAGE_ALLOC_DEAD,
  6847. "mm/page_alloc:dead", NULL,
  6848. page_alloc_cpu_dead);
  6849. WARN_ON(ret < 0);
  6850. }
  6851. /*
  6852. * calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio
  6853. * or min_free_kbytes changes.
  6854. */
  6855. static void calculate_totalreserve_pages(void)
  6856. {
  6857. struct pglist_data *pgdat;
  6858. unsigned long reserve_pages = 0;
  6859. enum zone_type i, j;
  6860. for_each_online_pgdat(pgdat) {
  6861. pgdat->totalreserve_pages = 0;
  6862. for (i = 0; i < MAX_NR_ZONES; i++) {
  6863. struct zone *zone = pgdat->node_zones + i;
  6864. long max = 0;
  6865. unsigned long managed_pages = zone_managed_pages(zone);
  6866. /* Find valid and maximum lowmem_reserve in the zone */
  6867. for (j = i; j < MAX_NR_ZONES; j++) {
  6868. if (zone->lowmem_reserve[j] > max)
  6869. max = zone->lowmem_reserve[j];
  6870. }
  6871. /* we treat the high watermark as reserved pages. */
  6872. max += high_wmark_pages(zone);
  6873. if (max > managed_pages)
  6874. max = managed_pages;
  6875. pgdat->totalreserve_pages += max;
  6876. reserve_pages += max;
  6877. }
  6878. }
  6879. totalreserve_pages = reserve_pages;
  6880. }
  6881. /*
  6882. * setup_per_zone_lowmem_reserve - called whenever
  6883. * sysctl_lowmem_reserve_ratio changes. Ensures that each zone
  6884. * has a correct pages reserved value, so an adequate number of
  6885. * pages are left in the zone after a successful __alloc_pages().
  6886. */
  6887. static void setup_per_zone_lowmem_reserve(void)
  6888. {
  6889. struct pglist_data *pgdat;
  6890. enum zone_type i, j;
  6891. for_each_online_pgdat(pgdat) {
  6892. for (i = 0; i < MAX_NR_ZONES - 1; i++) {
  6893. struct zone *zone = &pgdat->node_zones[i];
  6894. int ratio = sysctl_lowmem_reserve_ratio[i];
  6895. bool clear = !ratio || !zone_managed_pages(zone);
  6896. unsigned long managed_pages = 0;
  6897. for (j = i + 1; j < MAX_NR_ZONES; j++) {
  6898. struct zone *upper_zone = &pgdat->node_zones[j];
  6899. managed_pages += zone_managed_pages(upper_zone);
  6900. if (clear)
  6901. zone->lowmem_reserve[j] = 0;
  6902. else
  6903. zone->lowmem_reserve[j] = managed_pages / ratio;
  6904. }
  6905. }
  6906. }
  6907. /* update totalreserve_pages */
  6908. calculate_totalreserve_pages();
  6909. }
  6910. static void __setup_per_zone_wmarks(void)
  6911. {
  6912. unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);
  6913. unsigned long pages_low = extra_free_kbytes >> (PAGE_SHIFT - 10);
  6914. unsigned long lowmem_pages = 0;
  6915. struct zone *zone;
  6916. unsigned long flags;
  6917. /* Calculate total number of !ZONE_HIGHMEM pages */
  6918. for_each_zone(zone) {
  6919. if (!is_highmem(zone))
  6920. lowmem_pages += zone_managed_pages(zone);
  6921. }
  6922. for_each_zone(zone) {
  6923. u64 tmp, low;
  6924. spin_lock_irqsave(&zone->lock, flags);
  6925. tmp = (u64)pages_min * zone_managed_pages(zone);
  6926. do_div(tmp, lowmem_pages);
  6927. low = (u64)pages_low * zone_managed_pages(zone);
  6928. do_div(low, nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE)));
  6929. if (is_highmem(zone)) {
  6930. /*
  6931. * __GFP_HIGH and PF_MEMALLOC allocations usually don't
  6932. * need highmem pages, so cap pages_min to a small
  6933. * value here.
  6934. *
  6935. * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN)
  6936. * deltas control async page reclaim, and so should
  6937. * not be capped for highmem.
  6938. */
  6939. unsigned long min_pages;
  6940. min_pages = zone_managed_pages(zone) / 1024;
  6941. min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL);
  6942. zone->_watermark[WMARK_MIN] = min_pages;
  6943. } else {
  6944. /*
  6945. * If it's a lowmem zone, reserve a number of pages
  6946. * proportionate to the zone's size.
  6947. */
  6948. zone->_watermark[WMARK_MIN] = tmp;
  6949. }
  6950. /*
  6951. * Set the kswapd watermarks distance according to the
  6952. * scale factor in proportion to available memory, but
  6953. * ensure a minimum size on small systems.
  6954. */
  6955. tmp = max_t(u64, tmp >> 2,
  6956. mult_frac(zone_managed_pages(zone),
  6957. watermark_scale_factor, 10000));
  6958. zone->watermark_boost = 0;
  6959. zone->_watermark[WMARK_LOW] = min_wmark_pages(zone) + low + tmp;
  6960. zone->_watermark[WMARK_HIGH] = min_wmark_pages(zone) + low + tmp * 2;
  6961. spin_unlock_irqrestore(&zone->lock, flags);
  6962. }
  6963. /* update totalreserve_pages */
  6964. calculate_totalreserve_pages();
  6965. }
  6966. /**
  6967. * setup_per_zone_wmarks - called when min_free_kbytes changes
  6968. * or when memory is hot-{added|removed}
  6969. *
  6970. * Ensures that the watermark[min,low,high] values for each zone are set
  6971. * correctly with respect to min_free_kbytes.
  6972. */
  6973. void setup_per_zone_wmarks(void)
  6974. {
  6975. static DEFINE_SPINLOCK(lock);
  6976. spin_lock(&lock);
  6977. __setup_per_zone_wmarks();
  6978. spin_unlock(&lock);
  6979. }
  6980. /*
  6981. * Initialise min_free_kbytes.
  6982. *
  6983. * For small machines we want it small (128k min). For large machines
  6984. * we want it large (256MB max). But it is not linear, because network
  6985. * bandwidth does not increase linearly with machine size. We use
  6986. *
  6987. * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy:
  6988. * min_free_kbytes = sqrt(lowmem_kbytes * 16)
  6989. *
  6990. * which yields
  6991. *
  6992. * 16MB: 512k
  6993. * 32MB: 724k
  6994. * 64MB: 1024k
  6995. * 128MB: 1448k
  6996. * 256MB: 2048k
  6997. * 512MB: 2896k
  6998. * 1024MB: 4096k
  6999. * 2048MB: 5792k
  7000. * 4096MB: 8192k
  7001. * 8192MB: 11584k
  7002. * 16384MB: 16384k
  7003. */
  7004. int __meminit init_per_zone_wmark_min(void)
  7005. {
  7006. unsigned long lowmem_kbytes;
  7007. int new_min_free_kbytes;
  7008. lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10);
  7009. new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16);
  7010. if (new_min_free_kbytes > user_min_free_kbytes) {
  7011. min_free_kbytes = new_min_free_kbytes;
  7012. if (min_free_kbytes < 128)
  7013. min_free_kbytes = 128;
  7014. if (min_free_kbytes > 262144)
  7015. min_free_kbytes = 262144;
  7016. } else {
  7017. pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n",
  7018. new_min_free_kbytes, user_min_free_kbytes);
  7019. }
  7020. setup_per_zone_wmarks();
  7021. refresh_zone_stat_thresholds();
  7022. setup_per_zone_lowmem_reserve();
  7023. #ifdef CONFIG_NUMA
  7024. setup_min_unmapped_ratio();
  7025. setup_min_slab_ratio();
  7026. #endif
  7027. khugepaged_min_free_kbytes_update();
  7028. return 0;
  7029. }
  7030. postcore_initcall(init_per_zone_wmark_min)
  7031. /*
  7032. * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so
  7033. * that we can call two helper functions whenever min_free_kbytes
  7034. * or extra_free_kbytes changes.
  7035. */
  7036. int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write,
  7037. void *buffer, size_t *length, loff_t *ppos)
  7038. {
  7039. int rc;
  7040. rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
  7041. if (rc)
  7042. return rc;
  7043. if (write) {
  7044. user_min_free_kbytes = min_free_kbytes;
  7045. setup_per_zone_wmarks();
  7046. }
  7047. return 0;
  7048. }
  7049. int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write,
  7050. void *buffer, size_t *length, loff_t *ppos)
  7051. {
  7052. int rc;
  7053. rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
  7054. if (rc)
  7055. return rc;
  7056. if (write)
  7057. setup_per_zone_wmarks();
  7058. return 0;
  7059. }
  7060. #ifdef CONFIG_NUMA
  7061. static void setup_min_unmapped_ratio(void)
  7062. {
  7063. pg_data_t *pgdat;
  7064. struct zone *zone;
  7065. for_each_online_pgdat(pgdat)
  7066. pgdat->min_unmapped_pages = 0;
  7067. for_each_zone(zone)
  7068. zone->zone_pgdat->min_unmapped_pages += (zone_managed_pages(zone) *
  7069. sysctl_min_unmapped_ratio) / 100;
  7070. }
  7071. int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write,
  7072. void *buffer, size_t *length, loff_t *ppos)
  7073. {
  7074. int rc;
  7075. rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
  7076. if (rc)
  7077. return rc;
  7078. setup_min_unmapped_ratio();
  7079. return 0;
  7080. }
  7081. static void setup_min_slab_ratio(void)
  7082. {
  7083. pg_data_t *pgdat;
  7084. struct zone *zone;
  7085. for_each_online_pgdat(pgdat)
  7086. pgdat->min_slab_pages = 0;
  7087. for_each_zone(zone)
  7088. zone->zone_pgdat->min_slab_pages += (zone_managed_pages(zone) *
  7089. sysctl_min_slab_ratio) / 100;
  7090. }
  7091. int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write,
  7092. void *buffer, size_t *length, loff_t *ppos)
  7093. {
  7094. int rc;
  7095. rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
  7096. if (rc)
  7097. return rc;
  7098. setup_min_slab_ratio();
  7099. return 0;
  7100. }
  7101. #endif
  7102. /*
  7103. * lowmem_reserve_ratio_sysctl_handler - just a wrapper around
  7104. * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve()
  7105. * whenever sysctl_lowmem_reserve_ratio changes.
  7106. *
  7107. * The reserve ratio obviously has absolutely no relation with the
  7108. * minimum watermarks. The lowmem reserve ratio can only make sense
  7109. * if in function of the boot time zone sizes.
  7110. */
  7111. int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write,
  7112. void *buffer, size_t *length, loff_t *ppos)
  7113. {
  7114. int i;
  7115. proc_dointvec_minmax(table, write, buffer, length, ppos);
  7116. for (i = 0; i < MAX_NR_ZONES; i++) {
  7117. if (sysctl_lowmem_reserve_ratio[i] < 1)
  7118. sysctl_lowmem_reserve_ratio[i] = 0;
  7119. }
  7120. setup_per_zone_lowmem_reserve();
  7121. return 0;
  7122. }
  7123. static void __zone_pcp_update(struct zone *zone)
  7124. {
  7125. unsigned int cpu;
  7126. for_each_possible_cpu(cpu)
  7127. pageset_set_high_and_batch(zone,
  7128. per_cpu_ptr(zone->pageset, cpu));
  7129. }
  7130. /*
  7131. * percpu_pagelist_fraction - changes the pcp->high for each zone on each
  7132. * cpu. It is the fraction of total pages in each zone that a hot per cpu
  7133. * pagelist can have before it gets flushed back to buddy allocator.
  7134. */
  7135. int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *table, int write,
  7136. void *buffer, size_t *length, loff_t *ppos)
  7137. {
  7138. struct zone *zone;
  7139. int old_percpu_pagelist_fraction;
  7140. int ret;
  7141. mutex_lock(&pcp_batch_high_lock);
  7142. old_percpu_pagelist_fraction = percpu_pagelist_fraction;
  7143. ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
  7144. if (!write || ret < 0)
  7145. goto out;
  7146. /* Sanity checking to avoid pcp imbalance */
  7147. if (percpu_pagelist_fraction &&
  7148. percpu_pagelist_fraction < MIN_PERCPU_PAGELIST_FRACTION) {
  7149. percpu_pagelist_fraction = old_percpu_pagelist_fraction;
  7150. ret = -EINVAL;
  7151. goto out;
  7152. }
  7153. /* No change? */
  7154. if (percpu_pagelist_fraction == old_percpu_pagelist_fraction)
  7155. goto out;
  7156. for_each_populated_zone(zone)
  7157. __zone_pcp_update(zone);
  7158. out:
  7159. mutex_unlock(&pcp_batch_high_lock);
  7160. return ret;
  7161. }
  7162. #ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES
  7163. /*
  7164. * Returns the number of pages that arch has reserved but
  7165. * is not known to alloc_large_system_hash().
  7166. */
  7167. static unsigned long __init arch_reserved_kernel_pages(void)
  7168. {
  7169. return 0;
  7170. }
  7171. #endif
  7172. /*
  7173. * Adaptive scale is meant to reduce sizes of hash tables on large memory
  7174. * machines. As memory size is increased the scale is also increased but at
  7175. * slower pace. Starting from ADAPT_SCALE_BASE (64G), every time memory
  7176. * quadruples the scale is increased by one, which means the size of hash table
  7177. * only doubles, instead of quadrupling as well.
  7178. * Because 32-bit systems cannot have large physical memory, where this scaling
  7179. * makes sense, it is disabled on such platforms.
  7180. */
  7181. #if __BITS_PER_LONG > 32
  7182. #define ADAPT_SCALE_BASE (64ul << 30)
  7183. #define ADAPT_SCALE_SHIFT 2
  7184. #define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT)
  7185. #endif
  7186. /*
  7187. * allocate a large system hash table from bootmem
  7188. * - it is assumed that the hash table must contain an exact power-of-2
  7189. * quantity of entries
  7190. * - limit is the number of hash buckets, not the total allocation size
  7191. */
  7192. void *__init alloc_large_system_hash(const char *tablename,
  7193. unsigned long bucketsize,
  7194. unsigned long numentries,
  7195. int scale,
  7196. int flags,
  7197. unsigned int *_hash_shift,
  7198. unsigned int *_hash_mask,
  7199. unsigned long low_limit,
  7200. unsigned long high_limit)
  7201. {
  7202. unsigned long long max = high_limit;
  7203. unsigned long log2qty, size;
  7204. void *table = NULL;
  7205. gfp_t gfp_flags;
  7206. bool virt;
  7207. /* allow the kernel cmdline to have a say */
  7208. if (!numentries) {
  7209. /* round applicable memory size up to nearest megabyte */
  7210. numentries = nr_kernel_pages;
  7211. numentries -= arch_reserved_kernel_pages();
  7212. /* It isn't necessary when PAGE_SIZE >= 1MB */
  7213. if (PAGE_SHIFT < 20)
  7214. numentries = round_up(numentries, (1<<20)/PAGE_SIZE);
  7215. #if __BITS_PER_LONG > 32
  7216. if (!high_limit) {
  7217. unsigned long adapt;
  7218. for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries;
  7219. adapt <<= ADAPT_SCALE_SHIFT)
  7220. scale++;
  7221. }
  7222. #endif
  7223. /* limit to 1 bucket per 2^scale bytes of low memory */
  7224. if (scale > PAGE_SHIFT)
  7225. numentries >>= (scale - PAGE_SHIFT);
  7226. else
  7227. numentries <<= (PAGE_SHIFT - scale);
  7228. /* Make sure we've got at least a 0-order allocation.. */
  7229. if (unlikely(flags & HASH_SMALL)) {
  7230. /* Makes no sense without HASH_EARLY */
  7231. WARN_ON(!(flags & HASH_EARLY));
  7232. if (!(numentries >> *_hash_shift)) {
  7233. numentries = 1UL << *_hash_shift;
  7234. BUG_ON(!numentries);
  7235. }
  7236. } else if (unlikely((numentries * bucketsize) < PAGE_SIZE))
  7237. numentries = PAGE_SIZE / bucketsize;
  7238. }
  7239. numentries = roundup_pow_of_two(numentries);
  7240. /* limit allocation size to 1/16 total memory by default */
  7241. if (max == 0) {
  7242. max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4;
  7243. do_div(max, bucketsize);
  7244. }
  7245. max = min(max, 0x80000000ULL);
  7246. if (numentries < low_limit)
  7247. numentries = low_limit;
  7248. if (numentries > max)
  7249. numentries = max;
  7250. log2qty = ilog2(numentries);
  7251. gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC;
  7252. do {
  7253. virt = false;
  7254. size = bucketsize << log2qty;
  7255. if (flags & HASH_EARLY) {
  7256. if (flags & HASH_ZERO)
  7257. table = memblock_alloc(size, SMP_CACHE_BYTES);
  7258. else
  7259. table = memblock_alloc_raw(size,
  7260. SMP_CACHE_BYTES);
  7261. } else if (get_order(size) >= MAX_ORDER || hashdist) {
  7262. table = __vmalloc(size, gfp_flags);
  7263. virt = true;
  7264. } else {
  7265. /*
  7266. * If bucketsize is not a power-of-two, we may free
  7267. * some pages at the end of hash table which
  7268. * alloc_pages_exact() automatically does
  7269. */
  7270. table = alloc_pages_exact(size, gfp_flags);
  7271. kmemleak_alloc(table, size, 1, gfp_flags);
  7272. }
  7273. } while (!table && size > PAGE_SIZE && --log2qty);
  7274. if (!table)
  7275. panic("Failed to allocate %s hash table\n", tablename);
  7276. pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n",
  7277. tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size,
  7278. virt ? "vmalloc" : "linear");
  7279. if (_hash_shift)
  7280. *_hash_shift = log2qty;
  7281. if (_hash_mask)
  7282. *_hash_mask = (1 << log2qty) - 1;
  7283. return table;
  7284. }
  7285. /*
  7286. * This function checks whether pageblock includes unmovable pages or not.
  7287. *
  7288. * PageLRU check without isolation or lru_lock could race so that
  7289. * MIGRATE_MOVABLE block might include unmovable pages. And __PageMovable
  7290. * check without lock_page also may miss some movable non-lru pages at
  7291. * race condition. So you can't expect this function should be exact.
  7292. *
  7293. * Returns a page without holding a reference. If the caller wants to
  7294. * dereference that page (e.g., dumping), it has to make sure that it
  7295. * cannot get removed (e.g., via memory unplug) concurrently.
  7296. *
  7297. */
  7298. struct page *has_unmovable_pages(struct zone *zone, struct page *page,
  7299. int migratetype, int flags)
  7300. {
  7301. unsigned long iter = 0;
  7302. unsigned long pfn = page_to_pfn(page);
  7303. unsigned long offset = pfn % pageblock_nr_pages;
  7304. if (is_migrate_cma_page(page)) {
  7305. /*
  7306. * CMA allocations (alloc_contig_range) really need to mark
  7307. * isolate CMA pageblocks even when they are not movable in fact
  7308. * so consider them movable here.
  7309. */
  7310. if (is_migrate_cma(migratetype))
  7311. return NULL;
  7312. return page;
  7313. }
  7314. for (; iter < pageblock_nr_pages - offset; iter++) {
  7315. if (!pfn_valid_within(pfn + iter))
  7316. continue;
  7317. page = pfn_to_page(pfn + iter);
  7318. /*
  7319. * Both, bootmem allocations and memory holes are marked
  7320. * PG_reserved and are unmovable. We can even have unmovable
  7321. * allocations inside ZONE_MOVABLE, for example when
  7322. * specifying "movablecore".
  7323. */
  7324. if (PageReserved(page))
  7325. return page;
  7326. /*
  7327. * If the zone is movable and we have ruled out all reserved
  7328. * pages then it should be reasonably safe to assume the rest
  7329. * is movable.
  7330. */
  7331. if (zone_idx(zone) == ZONE_MOVABLE)
  7332. continue;
  7333. /*
  7334. * Hugepages are not in LRU lists, but they're movable.
  7335. * THPs are on the LRU, but need to be counted as #small pages.
  7336. * We need not scan over tail pages because we don't
  7337. * handle each tail page individually in migration.
  7338. */
  7339. if (PageHuge(page) || PageTransCompound(page)) {
  7340. struct page *head = compound_head(page);
  7341. unsigned int skip_pages;
  7342. if (PageHuge(page)) {
  7343. if (!hugepage_migration_supported(page_hstate(head)))
  7344. return page;
  7345. } else if (!PageLRU(head) && !__PageMovable(head)) {
  7346. return page;
  7347. }
  7348. skip_pages = compound_nr(head) - (page - head);
  7349. iter += skip_pages - 1;
  7350. continue;
  7351. }
  7352. /*
  7353. * We can't use page_count without pin a page
  7354. * because another CPU can free compound page.
  7355. * This check already skips compound tails of THP
  7356. * because their page->_refcount is zero at all time.
  7357. */
  7358. if (!page_ref_count(page)) {
  7359. if (PageBuddy(page))
  7360. iter += (1 << buddy_order(page)) - 1;
  7361. continue;
  7362. }
  7363. /*
  7364. * The HWPoisoned page may be not in buddy system, and
  7365. * page_count() is not 0.
  7366. */
  7367. if ((flags & MEMORY_OFFLINE) && PageHWPoison(page))
  7368. continue;
  7369. /*
  7370. * We treat all PageOffline() pages as movable when offlining
  7371. * to give drivers a chance to decrement their reference count
  7372. * in MEM_GOING_OFFLINE in order to indicate that these pages
  7373. * can be offlined as there are no direct references anymore.
  7374. * For actually unmovable PageOffline() where the driver does
  7375. * not support this, we will fail later when trying to actually
  7376. * move these pages that still have a reference count > 0.
  7377. * (false negatives in this function only)
  7378. */
  7379. if ((flags & MEMORY_OFFLINE) && PageOffline(page))
  7380. continue;
  7381. if (__PageMovable(page) || PageLRU(page))
  7382. continue;
  7383. /*
  7384. * If there are RECLAIMABLE pages, we need to check
  7385. * it. But now, memory offline itself doesn't call
  7386. * shrink_node_slabs() and it still to be fixed.
  7387. */
  7388. return page;
  7389. }
  7390. return NULL;
  7391. }
  7392. #ifdef CONFIG_CONTIG_ALLOC
  7393. static unsigned long pfn_max_align_down(unsigned long pfn)
  7394. {
  7395. return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES,
  7396. pageblock_nr_pages) - 1);
  7397. }
  7398. unsigned long pfn_max_align_up(unsigned long pfn)
  7399. {
  7400. return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES,
  7401. pageblock_nr_pages));
  7402. }
  7403. #if defined(CONFIG_DYNAMIC_DEBUG) || \
  7404. (defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE))
  7405. /* Usage: See admin-guide/dynamic-debug-howto.rst */
  7406. static void alloc_contig_dump_pages(struct list_head *page_list)
  7407. {
  7408. DEFINE_DYNAMIC_DEBUG_METADATA(descriptor, "migrate failure");
  7409. if (DYNAMIC_DEBUG_BRANCH(descriptor)) {
  7410. struct page *page;
  7411. unsigned long nr_skip = 0;
  7412. unsigned long nr_pages = 0;
  7413. dump_stack();
  7414. list_for_each_entry(page, page_list, lru) {
  7415. nr_pages++;
  7416. /* The page will be freed by putback_movable_pages soon */
  7417. if (page_count(page) == 1) {
  7418. nr_skip++;
  7419. continue;
  7420. }
  7421. dump_page(page, "migration failure");
  7422. }
  7423. pr_warn("total dump_pages %lu skipping %lu\n", nr_pages, nr_skip);
  7424. }
  7425. }
  7426. #else
  7427. static inline void alloc_contig_dump_pages(struct list_head *page_list)
  7428. {
  7429. }
  7430. #endif
  7431. /* [start, end) must belong to a single zone. */
  7432. static int __alloc_contig_migrate_range(struct compact_control *cc,
  7433. unsigned long start, unsigned long end,
  7434. struct acr_info *info)
  7435. {
  7436. /* This function is based on compact_zone() from compaction.c. */
  7437. unsigned int nr_reclaimed;
  7438. unsigned long pfn = start;
  7439. unsigned int tries = 0;
  7440. unsigned int max_tries = 5;
  7441. int ret = 0;
  7442. struct page *page;
  7443. struct migration_target_control mtc = {
  7444. .nid = zone_to_nid(cc->zone),
  7445. .gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL,
  7446. };
  7447. if (cc->alloc_contig && cc->mode == MIGRATE_ASYNC)
  7448. max_tries = 1;
  7449. lru_cache_disable();
  7450. while (pfn < end || !list_empty(&cc->migratepages)) {
  7451. if (fatal_signal_pending(current)) {
  7452. ret = -EINTR;
  7453. break;
  7454. }
  7455. if (list_empty(&cc->migratepages)) {
  7456. cc->nr_migratepages = 0;
  7457. pfn = isolate_migratepages_range(cc, pfn, end);
  7458. if (!pfn) {
  7459. ret = -EINTR;
  7460. break;
  7461. }
  7462. tries = 0;
  7463. } else if (++tries == max_tries) {
  7464. ret = ret < 0 ? ret : -EBUSY;
  7465. break;
  7466. }
  7467. nr_reclaimed = reclaim_clean_pages_from_list(cc->zone,
  7468. &cc->migratepages);
  7469. info->nr_reclaimed += nr_reclaimed;
  7470. cc->nr_migratepages -= nr_reclaimed;
  7471. list_for_each_entry(page, &cc->migratepages, lru)
  7472. info->nr_mapped += page_mapcount(page);
  7473. ret = migrate_pages(&cc->migratepages, alloc_migration_target,
  7474. NULL, (unsigned long)&mtc, cc->mode, MR_CONTIG_RANGE);
  7475. if (!ret)
  7476. info->nr_migrated += cc->nr_migratepages;
  7477. }
  7478. lru_cache_enable();
  7479. if (ret < 0) {
  7480. if (ret == -EBUSY) {
  7481. alloc_contig_dump_pages(&cc->migratepages);
  7482. page_pinner_mark_migration_failed_pages(&cc->migratepages);
  7483. }
  7484. if (!list_empty(&cc->migratepages)) {
  7485. page = list_first_entry(&cc->migratepages, struct page , lru);
  7486. info->failed_pfn = page_to_pfn(page);
  7487. }
  7488. putback_movable_pages(&cc->migratepages);
  7489. info->err |= ACR_ERR_MIGRATE;
  7490. return ret;
  7491. }
  7492. return 0;
  7493. }
  7494. /**
  7495. * alloc_contig_range() -- tries to allocate given range of pages
  7496. * @start: start PFN to allocate
  7497. * @end: one-past-the-last PFN to allocate
  7498. * @migratetype: migratetype of the underlaying pageblocks (either
  7499. * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks
  7500. * in range must have the same migratetype and it must
  7501. * be either of the two.
  7502. * @gfp_mask: GFP mask to use during compaction
  7503. *
  7504. * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES
  7505. * aligned. The PFN range must belong to a single zone.
  7506. *
  7507. * The first thing this routine does is attempt to MIGRATE_ISOLATE all
  7508. * pageblocks in the range. Once isolated, the pageblocks should not
  7509. * be modified by others.
  7510. *
  7511. * Return: zero on success or negative error code. On success all
  7512. * pages which PFN is in [start, end) are allocated for the caller and
  7513. * need to be freed with free_contig_range().
  7514. */
  7515. int alloc_contig_range(unsigned long start, unsigned long end,
  7516. unsigned migratetype, gfp_t gfp_mask,
  7517. struct acr_info *info)
  7518. {
  7519. unsigned long outer_start, outer_end;
  7520. unsigned int order;
  7521. int ret = 0;
  7522. bool skip_drain_all_pages = false;
  7523. struct compact_control cc = {
  7524. .nr_migratepages = 0,
  7525. .order = -1,
  7526. .zone = page_zone(pfn_to_page(start)),
  7527. .mode = gfp_mask & __GFP_NORETRY ? MIGRATE_ASYNC : MIGRATE_SYNC,
  7528. .ignore_skip_hint = true,
  7529. .no_set_skip_hint = true,
  7530. .gfp_mask = current_gfp_context(gfp_mask),
  7531. .alloc_contig = true,
  7532. };
  7533. INIT_LIST_HEAD(&cc.migratepages);
  7534. /*
  7535. * What we do here is we mark all pageblocks in range as
  7536. * MIGRATE_ISOLATE. Because pageblock and max order pages may
  7537. * have different sizes, and due to the way page allocator
  7538. * work, we align the range to biggest of the two pages so
  7539. * that page allocator won't try to merge buddies from
  7540. * different pageblocks and change MIGRATE_ISOLATE to some
  7541. * other migration type.
  7542. *
  7543. * Once the pageblocks are marked as MIGRATE_ISOLATE, we
  7544. * migrate the pages from an unaligned range (ie. pages that
  7545. * we are interested in). This will put all the pages in
  7546. * range back to page allocator as MIGRATE_ISOLATE.
  7547. *
  7548. * When this is done, we take the pages in range from page
  7549. * allocator removing them from the buddy system. This way
  7550. * page allocator will never consider using them.
  7551. *
  7552. * This lets us mark the pageblocks back as
  7553. * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the
  7554. * aligned range but not in the unaligned, original range are
  7555. * put back to page allocator so that buddy can use them.
  7556. */
  7557. ret = start_isolate_page_range(pfn_max_align_down(start),
  7558. pfn_max_align_up(end), migratetype, 0,
  7559. &info->failed_pfn);
  7560. if (ret) {
  7561. info->err |= ACR_ERR_ISOLATE;
  7562. return ret;
  7563. }
  7564. trace_android_vh_cma_drain_all_pages_bypass(migratetype,
  7565. &skip_drain_all_pages);
  7566. if (skip_drain_all_pages)
  7567. drain_all_pages(cc.zone);
  7568. /*
  7569. * In case of -EBUSY, we'd like to know which page causes problem.
  7570. * So, just fall through. test_pages_isolated() has a tracepoint
  7571. * which will report the busy page.
  7572. *
  7573. * It is possible that busy pages could become available before
  7574. * the call to test_pages_isolated, and the range will actually be
  7575. * allocated. So, if we fall through be sure to clear ret so that
  7576. * -EBUSY is not accidentally used or returned to caller.
  7577. */
  7578. ret = __alloc_contig_migrate_range(&cc, start, end, info);
  7579. if (ret && (ret != -EBUSY || (gfp_mask & __GFP_NORETRY)))
  7580. goto done;
  7581. ret =0;
  7582. /*
  7583. * Pages from [start, end) are within a MAX_ORDER_NR_PAGES
  7584. * aligned blocks that are marked as MIGRATE_ISOLATE. What's
  7585. * more, all pages in [start, end) are free in page allocator.
  7586. * What we are going to do is to allocate all pages from
  7587. * [start, end) (that is remove them from page allocator).
  7588. *
  7589. * The only problem is that pages at the beginning and at the
  7590. * end of interesting range may be not aligned with pages that
  7591. * page allocator holds, ie. they can be part of higher order
  7592. * pages. Because of this, we reserve the bigger range and
  7593. * once this is done free the pages we are not interested in.
  7594. *
  7595. * We don't have to hold zone->lock here because the pages are
  7596. * isolated thus they won't get removed from buddy.
  7597. */
  7598. order = 0;
  7599. outer_start = start;
  7600. while (!PageBuddy(pfn_to_page(outer_start))) {
  7601. if (++order >= MAX_ORDER) {
  7602. outer_start = start;
  7603. break;
  7604. }
  7605. outer_start &= ~0UL << order;
  7606. }
  7607. if (outer_start != start) {
  7608. order = buddy_order(pfn_to_page(outer_start));
  7609. /*
  7610. * outer_start page could be small order buddy page and
  7611. * it doesn't include start page. Adjust outer_start
  7612. * in this case to report failed page properly
  7613. * on tracepoint in test_pages_isolated()
  7614. */
  7615. if (outer_start + (1UL << order) <= start)
  7616. outer_start = start;
  7617. }
  7618. /* Make sure the range is really isolated. */
  7619. if (test_pages_isolated(outer_start, end, 0, &info->failed_pfn)) {
  7620. pr_info_ratelimited("%s: [%lx, %lx) PFNs busy\n",
  7621. __func__, outer_start, end);
  7622. ret = -EBUSY;
  7623. info->err |= ACR_ERR_TEST;
  7624. goto done;
  7625. }
  7626. /* Grab isolated pages from freelists. */
  7627. outer_end = isolate_freepages_range(&cc, outer_start, end);
  7628. if (!outer_end) {
  7629. ret = -EBUSY;
  7630. goto done;
  7631. }
  7632. /* Free head and tail (if any) */
  7633. if (start != outer_start)
  7634. free_contig_range(outer_start, start - outer_start);
  7635. if (end != outer_end)
  7636. free_contig_range(end, outer_end - end);
  7637. done:
  7638. undo_isolate_page_range(pfn_max_align_down(start),
  7639. pfn_max_align_up(end), migratetype);
  7640. return ret;
  7641. }
  7642. EXPORT_SYMBOL(alloc_contig_range);
  7643. static int __alloc_contig_pages(unsigned long start_pfn,
  7644. unsigned long nr_pages, gfp_t gfp_mask)
  7645. {
  7646. struct acr_info dummy;
  7647. unsigned long end_pfn = start_pfn + nr_pages;
  7648. return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE,
  7649. gfp_mask, &dummy);
  7650. }
  7651. static bool pfn_range_valid_contig(struct zone *z, unsigned long start_pfn,
  7652. unsigned long nr_pages)
  7653. {
  7654. unsigned long i, end_pfn = start_pfn + nr_pages;
  7655. struct page *page;
  7656. for (i = start_pfn; i < end_pfn; i++) {
  7657. page = pfn_to_online_page(i);
  7658. if (!page)
  7659. return false;
  7660. if (page_zone(page) != z)
  7661. return false;
  7662. if (PageReserved(page))
  7663. return false;
  7664. if (page_count(page) > 0)
  7665. return false;
  7666. if (PageHuge(page))
  7667. return false;
  7668. }
  7669. return true;
  7670. }
  7671. static bool zone_spans_last_pfn(const struct zone *zone,
  7672. unsigned long start_pfn, unsigned long nr_pages)
  7673. {
  7674. unsigned long last_pfn = start_pfn + nr_pages - 1;
  7675. return zone_spans_pfn(zone, last_pfn);
  7676. }
  7677. /**
  7678. * alloc_contig_pages() -- tries to find and allocate contiguous range of pages
  7679. * @nr_pages: Number of contiguous pages to allocate
  7680. * @gfp_mask: GFP mask to limit search and used during compaction
  7681. * @nid: Target node
  7682. * @nodemask: Mask for other possible nodes
  7683. *
  7684. * This routine is a wrapper around alloc_contig_range(). It scans over zones
  7685. * on an applicable zonelist to find a contiguous pfn range which can then be
  7686. * tried for allocation with alloc_contig_range(). This routine is intended
  7687. * for allocation requests which can not be fulfilled with the buddy allocator.
  7688. *
  7689. * The allocated memory is always aligned to a page boundary. If nr_pages is a
  7690. * power of two then the alignment is guaranteed to be to the given nr_pages
  7691. * (e.g. 1GB request would be aligned to 1GB).
  7692. *
  7693. * Allocated pages can be freed with free_contig_range() or by manually calling
  7694. * __free_page() on each allocated page.
  7695. *
  7696. * Return: pointer to contiguous pages on success, or NULL if not successful.
  7697. */
  7698. struct page *alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask,
  7699. int nid, nodemask_t *nodemask)
  7700. {
  7701. unsigned long ret, pfn, flags;
  7702. struct zonelist *zonelist;
  7703. struct zone *zone;
  7704. struct zoneref *z;
  7705. zonelist = node_zonelist(nid, gfp_mask);
  7706. for_each_zone_zonelist_nodemask(zone, z, zonelist,
  7707. gfp_zone(gfp_mask), nodemask) {
  7708. spin_lock_irqsave(&zone->lock, flags);
  7709. pfn = ALIGN(zone->zone_start_pfn, nr_pages);
  7710. while (zone_spans_last_pfn(zone, pfn, nr_pages)) {
  7711. if (pfn_range_valid_contig(zone, pfn, nr_pages)) {
  7712. /*
  7713. * We release the zone lock here because
  7714. * alloc_contig_range() will also lock the zone
  7715. * at some point. If there's an allocation
  7716. * spinning on this lock, it may win the race
  7717. * and cause alloc_contig_range() to fail...
  7718. */
  7719. spin_unlock_irqrestore(&zone->lock, flags);
  7720. ret = __alloc_contig_pages(pfn, nr_pages,
  7721. gfp_mask);
  7722. if (!ret)
  7723. return pfn_to_page(pfn);
  7724. spin_lock_irqsave(&zone->lock, flags);
  7725. }
  7726. pfn += nr_pages;
  7727. }
  7728. spin_unlock_irqrestore(&zone->lock, flags);
  7729. }
  7730. return NULL;
  7731. }
  7732. #endif /* CONFIG_CONTIG_ALLOC */
  7733. void free_contig_range(unsigned long pfn, unsigned int nr_pages)
  7734. {
  7735. unsigned int count = 0;
  7736. for (; nr_pages--; pfn++) {
  7737. struct page *page = pfn_to_page(pfn);
  7738. count += page_count(page) != 1;
  7739. __free_page(page);
  7740. }
  7741. WARN(count != 0, "%d pages are still in use!\n", count);
  7742. }
  7743. EXPORT_SYMBOL(free_contig_range);
  7744. /*
  7745. * The zone indicated has a new number of managed_pages; batch sizes and percpu
  7746. * page high values need to be recalulated.
  7747. */
  7748. void __meminit zone_pcp_update(struct zone *zone)
  7749. {
  7750. mutex_lock(&pcp_batch_high_lock);
  7751. __zone_pcp_update(zone);
  7752. mutex_unlock(&pcp_batch_high_lock);
  7753. }
  7754. void zone_pcp_reset(struct zone *zone)
  7755. {
  7756. unsigned long flags;
  7757. int cpu;
  7758. struct per_cpu_pageset *pset;
  7759. /* avoid races with drain_pages() */
  7760. local_irq_save(flags);
  7761. if (zone->pageset != &boot_pageset) {
  7762. for_each_online_cpu(cpu) {
  7763. pset = per_cpu_ptr(zone->pageset, cpu);
  7764. drain_zonestat(zone, pset);
  7765. }
  7766. free_percpu(zone->pageset);
  7767. zone->pageset = &boot_pageset;
  7768. }
  7769. local_irq_restore(flags);
  7770. }
  7771. #ifdef CONFIG_MEMORY_HOTREMOVE
  7772. /*
  7773. * All pages in the range must be in a single zone, must not contain holes,
  7774. * must span full sections, and must be isolated before calling this function.
  7775. */
  7776. void __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
  7777. {
  7778. unsigned long pfn = start_pfn;
  7779. struct page *page;
  7780. struct zone *zone;
  7781. unsigned int order;
  7782. unsigned long flags;
  7783. offline_mem_sections(pfn, end_pfn);
  7784. zone = page_zone(pfn_to_page(pfn));
  7785. spin_lock_irqsave(&zone->lock, flags);
  7786. while (pfn < end_pfn) {
  7787. page = pfn_to_page(pfn);
  7788. /*
  7789. * The HWPoisoned page may be not in buddy system, and
  7790. * page_count() is not 0.
  7791. */
  7792. if (unlikely(!PageBuddy(page) && PageHWPoison(page))) {
  7793. pfn++;
  7794. continue;
  7795. }
  7796. /*
  7797. * At this point all remaining PageOffline() pages have a
  7798. * reference count of 0 and can simply be skipped.
  7799. */
  7800. if (PageOffline(page)) {
  7801. BUG_ON(page_count(page));
  7802. BUG_ON(PageBuddy(page));
  7803. pfn++;
  7804. continue;
  7805. }
  7806. BUG_ON(page_count(page));
  7807. BUG_ON(!PageBuddy(page));
  7808. order = buddy_order(page);
  7809. del_page_from_free_list(page, zone, order);
  7810. pfn += (1 << order);
  7811. }
  7812. spin_unlock_irqrestore(&zone->lock, flags);
  7813. }
  7814. #endif
  7815. bool is_free_buddy_page(struct page *page)
  7816. {
  7817. struct zone *zone = page_zone(page);
  7818. unsigned long pfn = page_to_pfn(page);
  7819. unsigned long flags;
  7820. unsigned int order;
  7821. spin_lock_irqsave(&zone->lock, flags);
  7822. for (order = 0; order < MAX_ORDER; order++) {
  7823. struct page *page_head = page - (pfn & ((1 << order) - 1));
  7824. if (PageBuddy(page_head) && buddy_order(page_head) >= order)
  7825. break;
  7826. }
  7827. spin_unlock_irqrestore(&zone->lock, flags);
  7828. return order < MAX_ORDER;
  7829. }
  7830. #ifdef CONFIG_MEMORY_FAILURE
  7831. /*
  7832. * Break down a higher-order page in sub-pages, and keep our target out of
  7833. * buddy allocator.
  7834. */
  7835. static void break_down_buddy_pages(struct zone *zone, struct page *page,
  7836. struct page *target, int low, int high,
  7837. int migratetype)
  7838. {
  7839. unsigned long size = 1 << high;
  7840. struct page *current_buddy, *next_page;
  7841. while (high > low) {
  7842. high--;
  7843. size >>= 1;
  7844. if (target >= &page[size]) {
  7845. next_page = page + size;
  7846. current_buddy = page;
  7847. } else {
  7848. next_page = page;
  7849. current_buddy = page + size;
  7850. }
  7851. if (set_page_guard(zone, current_buddy, high, migratetype))
  7852. continue;
  7853. if (current_buddy != target) {
  7854. add_to_free_list(current_buddy, zone, high, migratetype);
  7855. set_buddy_order(current_buddy, high);
  7856. page = next_page;
  7857. }
  7858. }
  7859. }
  7860. /*
  7861. * Take a page that will be marked as poisoned off the buddy allocator.
  7862. */
  7863. bool take_page_off_buddy(struct page *page)
  7864. {
  7865. struct zone *zone = page_zone(page);
  7866. unsigned long pfn = page_to_pfn(page);
  7867. unsigned long flags;
  7868. unsigned int order;
  7869. bool ret = false;
  7870. spin_lock_irqsave(&zone->lock, flags);
  7871. for (order = 0; order < MAX_ORDER; order++) {
  7872. struct page *page_head = page - (pfn & ((1 << order) - 1));
  7873. int page_order = buddy_order(page_head);
  7874. if (PageBuddy(page_head) && page_order >= order) {
  7875. unsigned long pfn_head = page_to_pfn(page_head);
  7876. int migratetype = get_pfnblock_migratetype(page_head,
  7877. pfn_head);
  7878. del_page_from_free_list(page_head, zone, page_order);
  7879. break_down_buddy_pages(zone, page_head, page, 0,
  7880. page_order, migratetype);
  7881. if (!is_migrate_isolate(migratetype))
  7882. __mod_zone_freepage_state(zone, -1, migratetype);
  7883. ret = true;
  7884. break;
  7885. }
  7886. if (page_count(page_head) > 0)
  7887. break;
  7888. }
  7889. spin_unlock_irqrestore(&zone->lock, flags);
  7890. return ret;
  7891. }
  7892. #endif
  7893. #ifdef CONFIG_ZONE_DMA
  7894. bool has_managed_dma(void)
  7895. {
  7896. struct pglist_data *pgdat;
  7897. for_each_online_pgdat(pgdat) {
  7898. struct zone *zone = &pgdat->node_zones[ZONE_DMA];
  7899. if (managed_zone(zone))
  7900. return true;
  7901. }
  7902. return false;
  7903. }
  7904. #endif /* CONFIG_ZONE_DMA */