memcontrol.c 193 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645664666476648664966506651665266536654665566566657665866596660666166626663666466656666666766686669667066716672667366746675667666776678667966806681668266836684668566866687668866896690669166926693669466956696669766986699670067016702670367046705670667076708670967106711671267136714671567166717671867196720672167226723672467256726672767286729673067316732673367346735673667376738673967406741674267436744674567466747674867496750675167526753675467556756675767586759676067616762676367646765676667676768676967706771677267736774677567766777677867796780678167826783678467856786678767886789679067916792679367946795679667976798679968006801680268036804680568066807680868096810681168126813681468156816681768186819682068216822682368246825682668276828682968306831683268336834683568366837683868396840684168426843684468456846684768486849685068516852685368546855685668576858685968606861686268636864686568666867686868696870687168726873687468756876687768786879688068816882688368846885688668876888688968906891689268936894689568966897689868996900690169026903690469056906690769086909691069116912691369146915691669176918691969206921692269236924692569266927692869296930693169326933693469356936693769386939694069416942694369446945694669476948694969506951695269536954695569566957695869596960696169626963696469656966696769686969697069716972697369746975697669776978697969806981698269836984698569866987698869896990699169926993699469956996699769986999700070017002700370047005700670077008700970107011701270137014701570167017701870197020702170227023702470257026702770287029703070317032703370347035703670377038703970407041704270437044704570467047704870497050705170527053705470557056705770587059706070617062706370647065706670677068706970707071707270737074707570767077707870797080708170827083708470857086708770887089709070917092709370947095709670977098709971007101710271037104710571067107710871097110711171127113711471157116711771187119712071217122712371247125712671277128712971307131713271337134713571367137713871397140714171427143714471457146714771487149715071517152715371547155715671577158715971607161716271637164716571667167716871697170717171727173717471757176717771787179718071817182718371847185718671877188718971907191719271937194719571967197719871997200720172027203720472057206720772087209721072117212721372147215721672177218721972207221722272237224722572267227722872297230723172327233723472357236723772387239724072417242724372447245724672477248724972507251725272537254725572567257725872597260726172627263726472657266726772687269727072717272727372747275727672777278727972807281728272837284728572867287728872897290729172927293729472957296729772987299730073017302730373047305730673077308730973107311731273137314731573167317731873197320732173227323732473257326732773287329733073317332733373347335733673377338733973407341734273437344734573467347734873497350735173527353735473557356735773587359736073617362736373647365736673677368736973707371737273737374737573767377737873797380738173827383738473857386738773887389739073917392739373947395739673977398739974007401740274037404740574067407740874097410741174127413741474157416741774187419742074217422742374247425742674277428742974307431743274337434743574367437743874397440744174427443744474457446744774487449745074517452745374547455745674577458745974607461746274637464746574667467746874697470747174727473747474757476747774787479748074817482748374847485748674877488748974907491749274937494749574967497749874997500750175027503750475057506750775087509751075117512751375147515751675177518751975207521
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /* memcontrol.c - Memory Controller
  3. *
  4. * Copyright IBM Corporation, 2007
  5. * Author Balbir Singh <balbir@linux.vnet.ibm.com>
  6. *
  7. * Copyright 2007 OpenVZ SWsoft Inc
  8. * Author: Pavel Emelianov <xemul@openvz.org>
  9. *
  10. * Memory thresholds
  11. * Copyright (C) 2009 Nokia Corporation
  12. * Author: Kirill A. Shutemov
  13. *
  14. * Kernel Memory Controller
  15. * Copyright (C) 2012 Parallels Inc. and Google Inc.
  16. * Authors: Glauber Costa and Suleiman Souhlal
  17. *
  18. * Native page reclaim
  19. * Charge lifetime sanitation
  20. * Lockless page tracking & accounting
  21. * Unified hierarchy configuration model
  22. * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner
  23. */
  24. #include <linux/page_counter.h>
  25. #include <linux/memcontrol.h>
  26. #include <linux/cgroup.h>
  27. #include <linux/pagewalk.h>
  28. #include <linux/sched/mm.h>
  29. #include <linux/shmem_fs.h>
  30. #include <linux/hugetlb.h>
  31. #include <linux/pagemap.h>
  32. #include <linux/vm_event_item.h>
  33. #include <linux/smp.h>
  34. #include <linux/page-flags.h>
  35. #include <linux/backing-dev.h>
  36. #include <linux/bit_spinlock.h>
  37. #include <linux/rcupdate.h>
  38. #include <linux/limits.h>
  39. #include <linux/export.h>
  40. #include <linux/mutex.h>
  41. #include <linux/rbtree.h>
  42. #include <linux/slab.h>
  43. #include <linux/swap.h>
  44. #include <linux/swapops.h>
  45. #include <linux/spinlock.h>
  46. #include <linux/eventfd.h>
  47. #include <linux/poll.h>
  48. #include <linux/sort.h>
  49. #include <linux/fs.h>
  50. #include <linux/seq_file.h>
  51. #include <linux/vmpressure.h>
  52. #include <linux/mm_inline.h>
  53. #include <linux/swap_cgroup.h>
  54. #include <linux/cpu.h>
  55. #include <linux/oom.h>
  56. #include <linux/lockdep.h>
  57. #include <linux/file.h>
  58. #include <linux/tracehook.h>
  59. #include <linux/psi.h>
  60. #include <linux/seq_buf.h>
  61. #include "internal.h"
  62. #include <net/sock.h>
  63. #include <net/ip.h>
  64. #include "slab.h"
  65. #include <linux/uaccess.h>
  66. #include <trace/events/vmscan.h>
  67. #include <trace/hooks/mm.h>
  68. struct cgroup_subsys memory_cgrp_subsys __read_mostly;
  69. EXPORT_SYMBOL(memory_cgrp_subsys);
  70. struct mem_cgroup *root_mem_cgroup __read_mostly;
  71. /* Active memory cgroup to use from an interrupt context */
  72. DEFINE_PER_CPU(struct mem_cgroup *, int_active_memcg);
  73. /* Socket memory accounting disabled? */
  74. static bool cgroup_memory_nosocket;
  75. /* Kernel memory accounting disabled? */
  76. static bool cgroup_memory_nokmem;
  77. /* Whether the swap controller is active */
  78. #ifdef CONFIG_MEMCG_SWAP
  79. bool cgroup_memory_noswap __read_mostly;
  80. #else
  81. #define cgroup_memory_noswap 1
  82. #endif
  83. #ifdef CONFIG_CGROUP_WRITEBACK
  84. static DECLARE_WAIT_QUEUE_HEAD(memcg_cgwb_frn_waitq);
  85. #endif
  86. /* Whether legacy memory+swap accounting is active */
  87. static bool do_memsw_account(void)
  88. {
  89. return !cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_noswap;
  90. }
  91. #define THRESHOLDS_EVENTS_TARGET 128
  92. #define SOFTLIMIT_EVENTS_TARGET 1024
  93. /*
  94. * Cgroups above their limits are maintained in a RB-Tree, independent of
  95. * their hierarchy representation
  96. */
  97. struct mem_cgroup_tree_per_node {
  98. struct rb_root rb_root;
  99. struct rb_node *rb_rightmost;
  100. spinlock_t lock;
  101. };
  102. struct mem_cgroup_tree {
  103. struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
  104. };
  105. static struct mem_cgroup_tree soft_limit_tree __read_mostly;
  106. /* for OOM */
  107. struct mem_cgroup_eventfd_list {
  108. struct list_head list;
  109. struct eventfd_ctx *eventfd;
  110. };
  111. /*
  112. * cgroup_event represents events which userspace want to receive.
  113. */
  114. struct mem_cgroup_event {
  115. /*
  116. * memcg which the event belongs to.
  117. */
  118. struct mem_cgroup *memcg;
  119. /*
  120. * eventfd to signal userspace about the event.
  121. */
  122. struct eventfd_ctx *eventfd;
  123. /*
  124. * Each of these stored in a list by the cgroup.
  125. */
  126. struct list_head list;
  127. /*
  128. * register_event() callback will be used to add new userspace
  129. * waiter for changes related to this event. Use eventfd_signal()
  130. * on eventfd to send notification to userspace.
  131. */
  132. int (*register_event)(struct mem_cgroup *memcg,
  133. struct eventfd_ctx *eventfd, const char *args);
  134. /*
  135. * unregister_event() callback will be called when userspace closes
  136. * the eventfd or on cgroup removing. This callback must be set,
  137. * if you want provide notification functionality.
  138. */
  139. void (*unregister_event)(struct mem_cgroup *memcg,
  140. struct eventfd_ctx *eventfd);
  141. /*
  142. * All fields below needed to unregister event when
  143. * userspace closes eventfd.
  144. */
  145. poll_table pt;
  146. wait_queue_head_t *wqh;
  147. wait_queue_entry_t wait;
  148. struct work_struct remove;
  149. };
  150. static void mem_cgroup_threshold(struct mem_cgroup *memcg);
  151. static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
  152. /* Stuffs for move charges at task migration. */
  153. /*
  154. * Types of charges to be moved.
  155. */
  156. #define MOVE_ANON 0x1U
  157. #define MOVE_FILE 0x2U
  158. #define MOVE_MASK (MOVE_ANON | MOVE_FILE)
  159. /* "mc" and its members are protected by cgroup_mutex */
  160. static struct move_charge_struct {
  161. spinlock_t lock; /* for from, to */
  162. struct mm_struct *mm;
  163. struct mem_cgroup *from;
  164. struct mem_cgroup *to;
  165. unsigned long flags;
  166. unsigned long precharge;
  167. unsigned long moved_charge;
  168. unsigned long moved_swap;
  169. struct task_struct *moving_task; /* a task moving charges */
  170. wait_queue_head_t waitq; /* a waitq for other context */
  171. } mc = {
  172. .lock = __SPIN_LOCK_UNLOCKED(mc.lock),
  173. .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
  174. };
  175. /*
  176. * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
  177. * limit reclaim to prevent infinite loops, if they ever occur.
  178. */
  179. #define MEM_CGROUP_MAX_RECLAIM_LOOPS 100
  180. #define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2
  181. /* for encoding cft->private value on file */
  182. enum res_type {
  183. _MEM,
  184. _MEMSWAP,
  185. _OOM_TYPE,
  186. _KMEM,
  187. _TCP,
  188. };
  189. #define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val))
  190. #define MEMFILE_TYPE(val) ((val) >> 16 & 0xffff)
  191. #define MEMFILE_ATTR(val) ((val) & 0xffff)
  192. /* Used for OOM nofiier */
  193. #define OOM_CONTROL (0)
  194. /*
  195. * Iteration constructs for visiting all cgroups (under a tree). If
  196. * loops are exited prematurely (break), mem_cgroup_iter_break() must
  197. * be used for reference counting.
  198. */
  199. #define for_each_mem_cgroup_tree(iter, root) \
  200. for (iter = mem_cgroup_iter(root, NULL, NULL); \
  201. iter != NULL; \
  202. iter = mem_cgroup_iter(root, iter, NULL))
  203. #define for_each_mem_cgroup(iter) \
  204. for (iter = mem_cgroup_iter(NULL, NULL, NULL); \
  205. iter != NULL; \
  206. iter = mem_cgroup_iter(NULL, iter, NULL))
  207. static inline bool task_is_dying(void)
  208. {
  209. return tsk_is_oom_victim(current) || fatal_signal_pending(current) ||
  210. (current->flags & PF_EXITING);
  211. }
  212. /* Some nice accessors for the vmpressure. */
  213. struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
  214. {
  215. if (!memcg)
  216. memcg = root_mem_cgroup;
  217. return &memcg->vmpressure;
  218. }
  219. struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr)
  220. {
  221. return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
  222. }
  223. #ifdef CONFIG_MEMCG_KMEM
  224. static DEFINE_SPINLOCK(objcg_lock);
  225. static void obj_cgroup_release(struct percpu_ref *ref)
  226. {
  227. struct obj_cgroup *objcg = container_of(ref, struct obj_cgroup, refcnt);
  228. struct mem_cgroup *memcg;
  229. unsigned int nr_bytes;
  230. unsigned int nr_pages;
  231. unsigned long flags;
  232. /*
  233. * At this point all allocated objects are freed, and
  234. * objcg->nr_charged_bytes can't have an arbitrary byte value.
  235. * However, it can be PAGE_SIZE or (x * PAGE_SIZE).
  236. *
  237. * The following sequence can lead to it:
  238. * 1) CPU0: objcg == stock->cached_objcg
  239. * 2) CPU1: we do a small allocation (e.g. 92 bytes),
  240. * PAGE_SIZE bytes are charged
  241. * 3) CPU1: a process from another memcg is allocating something,
  242. * the stock if flushed,
  243. * objcg->nr_charged_bytes = PAGE_SIZE - 92
  244. * 5) CPU0: we do release this object,
  245. * 92 bytes are added to stock->nr_bytes
  246. * 6) CPU0: stock is flushed,
  247. * 92 bytes are added to objcg->nr_charged_bytes
  248. *
  249. * In the result, nr_charged_bytes == PAGE_SIZE.
  250. * This page will be uncharged in obj_cgroup_release().
  251. */
  252. nr_bytes = atomic_read(&objcg->nr_charged_bytes);
  253. WARN_ON_ONCE(nr_bytes & (PAGE_SIZE - 1));
  254. nr_pages = nr_bytes >> PAGE_SHIFT;
  255. spin_lock_irqsave(&objcg_lock, flags);
  256. memcg = obj_cgroup_memcg(objcg);
  257. if (nr_pages)
  258. __memcg_kmem_uncharge(memcg, nr_pages);
  259. list_del(&objcg->list);
  260. mem_cgroup_put(memcg);
  261. spin_unlock_irqrestore(&objcg_lock, flags);
  262. percpu_ref_exit(ref);
  263. kfree_rcu(objcg, rcu);
  264. }
  265. static struct obj_cgroup *obj_cgroup_alloc(void)
  266. {
  267. struct obj_cgroup *objcg;
  268. int ret;
  269. objcg = kzalloc(sizeof(struct obj_cgroup), GFP_KERNEL);
  270. if (!objcg)
  271. return NULL;
  272. ret = percpu_ref_init(&objcg->refcnt, obj_cgroup_release, 0,
  273. GFP_KERNEL);
  274. if (ret) {
  275. kfree(objcg);
  276. return NULL;
  277. }
  278. INIT_LIST_HEAD(&objcg->list);
  279. return objcg;
  280. }
  281. static void memcg_reparent_objcgs(struct mem_cgroup *memcg,
  282. struct mem_cgroup *parent)
  283. {
  284. struct obj_cgroup *objcg, *iter;
  285. objcg = rcu_replace_pointer(memcg->objcg, NULL, true);
  286. spin_lock_irq(&objcg_lock);
  287. /* Move active objcg to the parent's list */
  288. xchg(&objcg->memcg, parent);
  289. css_get(&parent->css);
  290. list_add(&objcg->list, &parent->objcg_list);
  291. /* Move already reparented objcgs to the parent's list */
  292. list_for_each_entry(iter, &memcg->objcg_list, list) {
  293. css_get(&parent->css);
  294. xchg(&iter->memcg, parent);
  295. css_put(&memcg->css);
  296. }
  297. list_splice(&memcg->objcg_list, &parent->objcg_list);
  298. spin_unlock_irq(&objcg_lock);
  299. percpu_ref_kill(&objcg->refcnt);
  300. }
  301. /*
  302. * This will be used as a shrinker list's index.
  303. * The main reason for not using cgroup id for this:
  304. * this works better in sparse environments, where we have a lot of memcgs,
  305. * but only a few kmem-limited. Or also, if we have, for instance, 200
  306. * memcgs, and none but the 200th is kmem-limited, we'd have to have a
  307. * 200 entry array for that.
  308. *
  309. * The current size of the caches array is stored in memcg_nr_cache_ids. It
  310. * will double each time we have to increase it.
  311. */
  312. static DEFINE_IDA(memcg_cache_ida);
  313. int memcg_nr_cache_ids;
  314. /* Protects memcg_nr_cache_ids */
  315. static DECLARE_RWSEM(memcg_cache_ids_sem);
  316. void memcg_get_cache_ids(void)
  317. {
  318. down_read(&memcg_cache_ids_sem);
  319. }
  320. void memcg_put_cache_ids(void)
  321. {
  322. up_read(&memcg_cache_ids_sem);
  323. }
  324. /*
  325. * MIN_SIZE is different than 1, because we would like to avoid going through
  326. * the alloc/free process all the time. In a small machine, 4 kmem-limited
  327. * cgroups is a reasonable guess. In the future, it could be a parameter or
  328. * tunable, but that is strictly not necessary.
  329. *
  330. * MAX_SIZE should be as large as the number of cgrp_ids. Ideally, we could get
  331. * this constant directly from cgroup, but it is understandable that this is
  332. * better kept as an internal representation in cgroup.c. In any case, the
  333. * cgrp_id space is not getting any smaller, and we don't have to necessarily
  334. * increase ours as well if it increases.
  335. */
  336. #define MEMCG_CACHES_MIN_SIZE 4
  337. #define MEMCG_CACHES_MAX_SIZE MEM_CGROUP_ID_MAX
  338. /*
  339. * A lot of the calls to the cache allocation functions are expected to be
  340. * inlined by the compiler. Since the calls to memcg_slab_pre_alloc_hook() are
  341. * conditional to this static branch, we'll have to allow modules that does
  342. * kmem_cache_alloc and the such to see this symbol as well
  343. */
  344. DEFINE_STATIC_KEY_FALSE(memcg_kmem_enabled_key);
  345. EXPORT_SYMBOL(memcg_kmem_enabled_key);
  346. #endif
  347. static int memcg_shrinker_map_size;
  348. static DEFINE_MUTEX(memcg_shrinker_map_mutex);
  349. static void memcg_free_shrinker_map_rcu(struct rcu_head *head)
  350. {
  351. kvfree(container_of(head, struct memcg_shrinker_map, rcu));
  352. }
  353. static int memcg_expand_one_shrinker_map(struct mem_cgroup *memcg,
  354. int size, int old_size)
  355. {
  356. struct memcg_shrinker_map *new, *old;
  357. int nid;
  358. lockdep_assert_held(&memcg_shrinker_map_mutex);
  359. for_each_node(nid) {
  360. old = rcu_dereference_protected(
  361. mem_cgroup_nodeinfo(memcg, nid)->shrinker_map, true);
  362. /* Not yet online memcg */
  363. if (!old)
  364. return 0;
  365. new = kvmalloc_node(sizeof(*new) + size, GFP_KERNEL, nid);
  366. if (!new)
  367. return -ENOMEM;
  368. /* Set all old bits, clear all new bits */
  369. memset(new->map, (int)0xff, old_size);
  370. memset((void *)new->map + old_size, 0, size - old_size);
  371. rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, new);
  372. call_rcu(&old->rcu, memcg_free_shrinker_map_rcu);
  373. }
  374. return 0;
  375. }
  376. static void memcg_free_shrinker_maps(struct mem_cgroup *memcg)
  377. {
  378. struct mem_cgroup_per_node *pn;
  379. struct memcg_shrinker_map *map;
  380. int nid;
  381. if (mem_cgroup_is_root(memcg))
  382. return;
  383. for_each_node(nid) {
  384. pn = mem_cgroup_nodeinfo(memcg, nid);
  385. map = rcu_dereference_protected(pn->shrinker_map, true);
  386. if (map)
  387. kvfree(map);
  388. rcu_assign_pointer(pn->shrinker_map, NULL);
  389. }
  390. }
  391. static int memcg_alloc_shrinker_maps(struct mem_cgroup *memcg)
  392. {
  393. struct memcg_shrinker_map *map;
  394. int nid, size, ret = 0;
  395. if (mem_cgroup_is_root(memcg))
  396. return 0;
  397. mutex_lock(&memcg_shrinker_map_mutex);
  398. size = memcg_shrinker_map_size;
  399. for_each_node(nid) {
  400. map = kvzalloc_node(sizeof(*map) + size, GFP_KERNEL, nid);
  401. if (!map) {
  402. memcg_free_shrinker_maps(memcg);
  403. ret = -ENOMEM;
  404. break;
  405. }
  406. rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, map);
  407. }
  408. mutex_unlock(&memcg_shrinker_map_mutex);
  409. return ret;
  410. }
  411. int memcg_expand_shrinker_maps(int new_id)
  412. {
  413. int size, old_size, ret = 0;
  414. struct mem_cgroup *memcg;
  415. size = DIV_ROUND_UP(new_id + 1, BITS_PER_LONG) * sizeof(unsigned long);
  416. old_size = memcg_shrinker_map_size;
  417. if (size <= old_size)
  418. return 0;
  419. mutex_lock(&memcg_shrinker_map_mutex);
  420. if (!root_mem_cgroup)
  421. goto unlock;
  422. for_each_mem_cgroup(memcg) {
  423. if (mem_cgroup_is_root(memcg))
  424. continue;
  425. ret = memcg_expand_one_shrinker_map(memcg, size, old_size);
  426. if (ret) {
  427. mem_cgroup_iter_break(NULL, memcg);
  428. goto unlock;
  429. }
  430. }
  431. unlock:
  432. if (!ret)
  433. memcg_shrinker_map_size = size;
  434. mutex_unlock(&memcg_shrinker_map_mutex);
  435. return ret;
  436. }
  437. void memcg_set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id)
  438. {
  439. if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) {
  440. struct memcg_shrinker_map *map;
  441. rcu_read_lock();
  442. map = rcu_dereference(memcg->nodeinfo[nid]->shrinker_map);
  443. /* Pairs with smp mb in shrink_slab() */
  444. smp_mb__before_atomic();
  445. set_bit(shrinker_id, map->map);
  446. rcu_read_unlock();
  447. }
  448. }
  449. /**
  450. * mem_cgroup_css_from_page - css of the memcg associated with a page
  451. * @page: page of interest
  452. *
  453. * If memcg is bound to the default hierarchy, css of the memcg associated
  454. * with @page is returned. The returned css remains associated with @page
  455. * until it is released.
  456. *
  457. * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup
  458. * is returned.
  459. */
  460. struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page)
  461. {
  462. struct mem_cgroup *memcg;
  463. memcg = page->mem_cgroup;
  464. if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
  465. memcg = root_mem_cgroup;
  466. return &memcg->css;
  467. }
  468. /**
  469. * page_cgroup_ino - return inode number of the memcg a page is charged to
  470. * @page: the page
  471. *
  472. * Look up the closest online ancestor of the memory cgroup @page is charged to
  473. * and return its inode number or 0 if @page is not charged to any cgroup. It
  474. * is safe to call this function without holding a reference to @page.
  475. *
  476. * Note, this function is inherently racy, because there is nothing to prevent
  477. * the cgroup inode from getting torn down and potentially reallocated a moment
  478. * after page_cgroup_ino() returns, so it only should be used by callers that
  479. * do not care (such as procfs interfaces).
  480. */
  481. ino_t page_cgroup_ino(struct page *page)
  482. {
  483. struct mem_cgroup *memcg;
  484. unsigned long ino = 0;
  485. rcu_read_lock();
  486. memcg = page->mem_cgroup;
  487. /*
  488. * The lowest bit set means that memcg isn't a valid
  489. * memcg pointer, but a obj_cgroups pointer.
  490. * In this case the page is shared and doesn't belong
  491. * to any specific memory cgroup.
  492. */
  493. if ((unsigned long) memcg & 0x1UL)
  494. memcg = NULL;
  495. while (memcg && !(memcg->css.flags & CSS_ONLINE))
  496. memcg = parent_mem_cgroup(memcg);
  497. if (memcg)
  498. ino = cgroup_ino(memcg->css.cgroup);
  499. rcu_read_unlock();
  500. return ino;
  501. }
  502. static struct mem_cgroup_per_node *
  503. mem_cgroup_page_nodeinfo(struct mem_cgroup *memcg, struct page *page)
  504. {
  505. int nid = page_to_nid(page);
  506. return memcg->nodeinfo[nid];
  507. }
  508. static struct mem_cgroup_tree_per_node *
  509. soft_limit_tree_node(int nid)
  510. {
  511. return soft_limit_tree.rb_tree_per_node[nid];
  512. }
  513. static struct mem_cgroup_tree_per_node *
  514. soft_limit_tree_from_page(struct page *page)
  515. {
  516. int nid = page_to_nid(page);
  517. return soft_limit_tree.rb_tree_per_node[nid];
  518. }
  519. static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node *mz,
  520. struct mem_cgroup_tree_per_node *mctz,
  521. unsigned long new_usage_in_excess)
  522. {
  523. struct rb_node **p = &mctz->rb_root.rb_node;
  524. struct rb_node *parent = NULL;
  525. struct mem_cgroup_per_node *mz_node;
  526. bool rightmost = true;
  527. if (mz->on_tree)
  528. return;
  529. mz->usage_in_excess = new_usage_in_excess;
  530. if (!mz->usage_in_excess)
  531. return;
  532. while (*p) {
  533. parent = *p;
  534. mz_node = rb_entry(parent, struct mem_cgroup_per_node,
  535. tree_node);
  536. if (mz->usage_in_excess < mz_node->usage_in_excess) {
  537. p = &(*p)->rb_left;
  538. rightmost = false;
  539. }
  540. /*
  541. * We can't avoid mem cgroups that are over their soft
  542. * limit by the same amount
  543. */
  544. else if (mz->usage_in_excess >= mz_node->usage_in_excess)
  545. p = &(*p)->rb_right;
  546. }
  547. if (rightmost)
  548. mctz->rb_rightmost = &mz->tree_node;
  549. rb_link_node(&mz->tree_node, parent, p);
  550. rb_insert_color(&mz->tree_node, &mctz->rb_root);
  551. mz->on_tree = true;
  552. }
  553. static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
  554. struct mem_cgroup_tree_per_node *mctz)
  555. {
  556. if (!mz->on_tree)
  557. return;
  558. if (&mz->tree_node == mctz->rb_rightmost)
  559. mctz->rb_rightmost = rb_prev(&mz->tree_node);
  560. rb_erase(&mz->tree_node, &mctz->rb_root);
  561. mz->on_tree = false;
  562. }
  563. static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
  564. struct mem_cgroup_tree_per_node *mctz)
  565. {
  566. unsigned long flags;
  567. spin_lock_irqsave(&mctz->lock, flags);
  568. __mem_cgroup_remove_exceeded(mz, mctz);
  569. spin_unlock_irqrestore(&mctz->lock, flags);
  570. }
  571. static unsigned long soft_limit_excess(struct mem_cgroup *memcg)
  572. {
  573. unsigned long nr_pages = page_counter_read(&memcg->memory);
  574. unsigned long soft_limit = READ_ONCE(memcg->soft_limit);
  575. unsigned long excess = 0;
  576. if (nr_pages > soft_limit)
  577. excess = nr_pages - soft_limit;
  578. return excess;
  579. }
  580. static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
  581. {
  582. unsigned long excess;
  583. struct mem_cgroup_per_node *mz;
  584. struct mem_cgroup_tree_per_node *mctz;
  585. mctz = soft_limit_tree_from_page(page);
  586. if (!mctz)
  587. return;
  588. /*
  589. * Necessary to update all ancestors when hierarchy is used.
  590. * because their event counter is not touched.
  591. */
  592. for (; memcg; memcg = parent_mem_cgroup(memcg)) {
  593. mz = mem_cgroup_page_nodeinfo(memcg, page);
  594. excess = soft_limit_excess(memcg);
  595. /*
  596. * We have to update the tree if mz is on RB-tree or
  597. * mem is over its softlimit.
  598. */
  599. if (excess || mz->on_tree) {
  600. unsigned long flags;
  601. spin_lock_irqsave(&mctz->lock, flags);
  602. /* if on-tree, remove it */
  603. if (mz->on_tree)
  604. __mem_cgroup_remove_exceeded(mz, mctz);
  605. /*
  606. * Insert again. mz->usage_in_excess will be updated.
  607. * If excess is 0, no tree ops.
  608. */
  609. __mem_cgroup_insert_exceeded(mz, mctz, excess);
  610. spin_unlock_irqrestore(&mctz->lock, flags);
  611. }
  612. }
  613. }
  614. static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
  615. {
  616. struct mem_cgroup_tree_per_node *mctz;
  617. struct mem_cgroup_per_node *mz;
  618. int nid;
  619. for_each_node(nid) {
  620. mz = mem_cgroup_nodeinfo(memcg, nid);
  621. mctz = soft_limit_tree_node(nid);
  622. if (mctz)
  623. mem_cgroup_remove_exceeded(mz, mctz);
  624. }
  625. }
  626. static struct mem_cgroup_per_node *
  627. __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
  628. {
  629. struct mem_cgroup_per_node *mz;
  630. retry:
  631. mz = NULL;
  632. if (!mctz->rb_rightmost)
  633. goto done; /* Nothing to reclaim from */
  634. mz = rb_entry(mctz->rb_rightmost,
  635. struct mem_cgroup_per_node, tree_node);
  636. /*
  637. * Remove the node now but someone else can add it back,
  638. * we will to add it back at the end of reclaim to its correct
  639. * position in the tree.
  640. */
  641. __mem_cgroup_remove_exceeded(mz, mctz);
  642. if (!soft_limit_excess(mz->memcg) ||
  643. !css_tryget(&mz->memcg->css))
  644. goto retry;
  645. done:
  646. return mz;
  647. }
  648. static struct mem_cgroup_per_node *
  649. mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
  650. {
  651. struct mem_cgroup_per_node *mz;
  652. spin_lock_irq(&mctz->lock);
  653. mz = __mem_cgroup_largest_soft_limit_node(mctz);
  654. spin_unlock_irq(&mctz->lock);
  655. return mz;
  656. }
  657. /**
  658. * __mod_memcg_state - update cgroup memory statistics
  659. * @memcg: the memory cgroup
  660. * @idx: the stat item - can be enum memcg_stat_item or enum node_stat_item
  661. * @val: delta to add to the counter, can be negative
  662. */
  663. void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val)
  664. {
  665. long x, threshold = MEMCG_CHARGE_BATCH;
  666. if (mem_cgroup_disabled())
  667. return;
  668. if (memcg_stat_item_in_bytes(idx))
  669. threshold <<= PAGE_SHIFT;
  670. x = val + __this_cpu_read(memcg->vmstats_percpu->stat[idx]);
  671. if (unlikely(abs(x) > threshold)) {
  672. struct mem_cgroup *mi;
  673. /*
  674. * Batch local counters to keep them in sync with
  675. * the hierarchical ones.
  676. */
  677. __this_cpu_add(memcg->vmstats_local->stat[idx], x);
  678. for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
  679. atomic_long_add(x, &mi->vmstats[idx]);
  680. x = 0;
  681. }
  682. __this_cpu_write(memcg->vmstats_percpu->stat[idx], x);
  683. }
  684. static struct mem_cgroup_per_node *
  685. parent_nodeinfo(struct mem_cgroup_per_node *pn, int nid)
  686. {
  687. struct mem_cgroup *parent;
  688. parent = parent_mem_cgroup(pn->memcg);
  689. if (!parent)
  690. return NULL;
  691. return mem_cgroup_nodeinfo(parent, nid);
  692. }
  693. void __mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
  694. int val)
  695. {
  696. struct mem_cgroup_per_node *pn;
  697. struct mem_cgroup *memcg;
  698. long x, threshold = MEMCG_CHARGE_BATCH;
  699. pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
  700. memcg = pn->memcg;
  701. /* Update memcg */
  702. __mod_memcg_state(memcg, idx, val);
  703. /* Update lruvec */
  704. __this_cpu_add(pn->lruvec_stat_local->count[idx], val);
  705. if (vmstat_item_in_bytes(idx))
  706. threshold <<= PAGE_SHIFT;
  707. x = val + __this_cpu_read(pn->lruvec_stat_cpu->count[idx]);
  708. if (unlikely(abs(x) > threshold)) {
  709. pg_data_t *pgdat = lruvec_pgdat(lruvec);
  710. struct mem_cgroup_per_node *pi;
  711. for (pi = pn; pi; pi = parent_nodeinfo(pi, pgdat->node_id))
  712. atomic_long_add(x, &pi->lruvec_stat[idx]);
  713. x = 0;
  714. }
  715. __this_cpu_write(pn->lruvec_stat_cpu->count[idx], x);
  716. }
  717. /**
  718. * __mod_lruvec_state - update lruvec memory statistics
  719. * @lruvec: the lruvec
  720. * @idx: the stat item
  721. * @val: delta to add to the counter, can be negative
  722. *
  723. * The lruvec is the intersection of the NUMA node and a cgroup. This
  724. * function updates the all three counters that are affected by a
  725. * change of state at this level: per-node, per-cgroup, per-lruvec.
  726. */
  727. void __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
  728. int val)
  729. {
  730. /* Update node */
  731. __mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
  732. /* Update memcg and lruvec */
  733. if (!mem_cgroup_disabled())
  734. __mod_memcg_lruvec_state(lruvec, idx, val);
  735. }
  736. void __mod_lruvec_slab_state(void *p, enum node_stat_item idx, int val)
  737. {
  738. pg_data_t *pgdat = page_pgdat(virt_to_page(p));
  739. struct mem_cgroup *memcg;
  740. struct lruvec *lruvec;
  741. rcu_read_lock();
  742. memcg = mem_cgroup_from_obj(p);
  743. /*
  744. * Untracked pages have no memcg, no lruvec. Update only the
  745. * node. If we reparent the slab objects to the root memcg,
  746. * when we free the slab object, we need to update the per-memcg
  747. * vmstats to keep it correct for the root memcg.
  748. */
  749. if (!memcg) {
  750. __mod_node_page_state(pgdat, idx, val);
  751. } else {
  752. lruvec = mem_cgroup_lruvec(memcg, pgdat);
  753. __mod_lruvec_state(lruvec, idx, val);
  754. }
  755. rcu_read_unlock();
  756. }
  757. void mod_memcg_obj_state(void *p, int idx, int val)
  758. {
  759. struct mem_cgroup *memcg;
  760. rcu_read_lock();
  761. memcg = mem_cgroup_from_obj(p);
  762. if (memcg)
  763. mod_memcg_state(memcg, idx, val);
  764. rcu_read_unlock();
  765. }
  766. /**
  767. * __count_memcg_events - account VM events in a cgroup
  768. * @memcg: the memory cgroup
  769. * @idx: the event item
  770. * @count: the number of events that occured
  771. */
  772. void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
  773. unsigned long count)
  774. {
  775. unsigned long x;
  776. if (mem_cgroup_disabled())
  777. return;
  778. x = count + __this_cpu_read(memcg->vmstats_percpu->events[idx]);
  779. if (unlikely(x > MEMCG_CHARGE_BATCH)) {
  780. struct mem_cgroup *mi;
  781. /*
  782. * Batch local counters to keep them in sync with
  783. * the hierarchical ones.
  784. */
  785. __this_cpu_add(memcg->vmstats_local->events[idx], x);
  786. for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
  787. atomic_long_add(x, &mi->vmevents[idx]);
  788. x = 0;
  789. }
  790. __this_cpu_write(memcg->vmstats_percpu->events[idx], x);
  791. }
  792. static unsigned long memcg_events(struct mem_cgroup *memcg, int event)
  793. {
  794. return atomic_long_read(&memcg->vmevents[event]);
  795. }
  796. static unsigned long memcg_events_local(struct mem_cgroup *memcg, int event)
  797. {
  798. long x = 0;
  799. int cpu;
  800. for_each_possible_cpu(cpu)
  801. x += per_cpu(memcg->vmstats_local->events[event], cpu);
  802. return x;
  803. }
  804. static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
  805. struct page *page,
  806. int nr_pages)
  807. {
  808. /* pagein of a big page is an event. So, ignore page size */
  809. if (nr_pages > 0)
  810. __count_memcg_events(memcg, PGPGIN, 1);
  811. else {
  812. __count_memcg_events(memcg, PGPGOUT, 1);
  813. nr_pages = -nr_pages; /* for event */
  814. }
  815. __this_cpu_add(memcg->vmstats_percpu->nr_page_events, nr_pages);
  816. }
  817. static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
  818. enum mem_cgroup_events_target target)
  819. {
  820. unsigned long val, next;
  821. val = __this_cpu_read(memcg->vmstats_percpu->nr_page_events);
  822. next = __this_cpu_read(memcg->vmstats_percpu->targets[target]);
  823. /* from time_after() in jiffies.h */
  824. if ((long)(next - val) < 0) {
  825. switch (target) {
  826. case MEM_CGROUP_TARGET_THRESH:
  827. next = val + THRESHOLDS_EVENTS_TARGET;
  828. break;
  829. case MEM_CGROUP_TARGET_SOFTLIMIT:
  830. next = val + SOFTLIMIT_EVENTS_TARGET;
  831. break;
  832. default:
  833. break;
  834. }
  835. __this_cpu_write(memcg->vmstats_percpu->targets[target], next);
  836. return true;
  837. }
  838. return false;
  839. }
  840. /*
  841. * Check events in order.
  842. *
  843. */
  844. static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
  845. {
  846. /* threshold event is triggered in finer grain than soft limit */
  847. if (unlikely(mem_cgroup_event_ratelimit(memcg,
  848. MEM_CGROUP_TARGET_THRESH))) {
  849. bool do_softlimit;
  850. do_softlimit = mem_cgroup_event_ratelimit(memcg,
  851. MEM_CGROUP_TARGET_SOFTLIMIT);
  852. mem_cgroup_threshold(memcg);
  853. if (unlikely(do_softlimit))
  854. mem_cgroup_update_tree(memcg, page);
  855. }
  856. }
  857. struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
  858. {
  859. /*
  860. * mm_update_next_owner() may clear mm->owner to NULL
  861. * if it races with swapoff, page migration, etc.
  862. * So this can be called with p == NULL.
  863. */
  864. if (unlikely(!p))
  865. return NULL;
  866. return mem_cgroup_from_css(task_css(p, memory_cgrp_id));
  867. }
  868. EXPORT_SYMBOL(mem_cgroup_from_task);
  869. /**
  870. * get_mem_cgroup_from_mm: Obtain a reference on given mm_struct's memcg.
  871. * @mm: mm from which memcg should be extracted. It can be NULL.
  872. *
  873. * Obtain a reference on mm->memcg and returns it if successful. Otherwise
  874. * root_mem_cgroup is returned. However if mem_cgroup is disabled, NULL is
  875. * returned.
  876. */
  877. struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
  878. {
  879. struct mem_cgroup *memcg;
  880. if (mem_cgroup_disabled())
  881. return NULL;
  882. rcu_read_lock();
  883. do {
  884. /*
  885. * Page cache insertions can happen withou an
  886. * actual mm context, e.g. during disk probing
  887. * on boot, loopback IO, acct() writes etc.
  888. */
  889. if (unlikely(!mm))
  890. memcg = root_mem_cgroup;
  891. else {
  892. memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
  893. if (unlikely(!memcg))
  894. memcg = root_mem_cgroup;
  895. }
  896. } while (!css_tryget(&memcg->css));
  897. rcu_read_unlock();
  898. return memcg;
  899. }
  900. EXPORT_SYMBOL(get_mem_cgroup_from_mm);
  901. /**
  902. * get_mem_cgroup_from_page: Obtain a reference on given page's memcg.
  903. * @page: page from which memcg should be extracted.
  904. *
  905. * Obtain a reference on page->memcg and returns it if successful. Otherwise
  906. * root_mem_cgroup is returned.
  907. */
  908. struct mem_cgroup *get_mem_cgroup_from_page(struct page *page)
  909. {
  910. struct mem_cgroup *memcg = page->mem_cgroup;
  911. if (mem_cgroup_disabled())
  912. return NULL;
  913. rcu_read_lock();
  914. /* Page should not get uncharged and freed memcg under us. */
  915. if (!memcg || WARN_ON_ONCE(!css_tryget(&memcg->css)))
  916. memcg = root_mem_cgroup;
  917. rcu_read_unlock();
  918. return memcg;
  919. }
  920. EXPORT_SYMBOL(get_mem_cgroup_from_page);
  921. static __always_inline struct mem_cgroup *active_memcg(void)
  922. {
  923. if (in_interrupt())
  924. return this_cpu_read(int_active_memcg);
  925. else
  926. return current->active_memcg;
  927. }
  928. static __always_inline struct mem_cgroup *get_active_memcg(void)
  929. {
  930. struct mem_cgroup *memcg;
  931. rcu_read_lock();
  932. memcg = active_memcg();
  933. /* remote memcg must hold a ref. */
  934. if (memcg && WARN_ON_ONCE(!css_tryget(&memcg->css)))
  935. memcg = root_mem_cgroup;
  936. rcu_read_unlock();
  937. return memcg;
  938. }
  939. static __always_inline bool memcg_kmem_bypass(void)
  940. {
  941. /* Allow remote memcg charging from any context. */
  942. if (unlikely(active_memcg()))
  943. return false;
  944. /* Memcg to charge can't be determined. */
  945. if (in_interrupt() || !current->mm || (current->flags & PF_KTHREAD))
  946. return true;
  947. return false;
  948. }
  949. /**
  950. * If active memcg is set, do not fallback to current->mm->memcg.
  951. */
  952. static __always_inline struct mem_cgroup *get_mem_cgroup_from_current(void)
  953. {
  954. if (memcg_kmem_bypass())
  955. return NULL;
  956. if (unlikely(active_memcg()))
  957. return get_active_memcg();
  958. return get_mem_cgroup_from_mm(current->mm);
  959. }
  960. /**
  961. * mem_cgroup_iter - iterate over memory cgroup hierarchy
  962. * @root: hierarchy root
  963. * @prev: previously returned memcg, NULL on first invocation
  964. * @reclaim: cookie for shared reclaim walks, NULL for full walks
  965. *
  966. * Returns references to children of the hierarchy below @root, or
  967. * @root itself, or %NULL after a full round-trip.
  968. *
  969. * Caller must pass the return value in @prev on subsequent
  970. * invocations for reference counting, or use mem_cgroup_iter_break()
  971. * to cancel a hierarchy walk before the round-trip is complete.
  972. *
  973. * Reclaimers can specify a node in @reclaim to divide up the memcgs
  974. * in the hierarchy among all concurrent reclaimers operating on the
  975. * same node.
  976. */
  977. struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
  978. struct mem_cgroup *prev,
  979. struct mem_cgroup_reclaim_cookie *reclaim)
  980. {
  981. struct mem_cgroup_reclaim_iter *iter;
  982. struct cgroup_subsys_state *css = NULL;
  983. struct mem_cgroup *memcg = NULL;
  984. struct mem_cgroup *pos = NULL;
  985. if (mem_cgroup_disabled())
  986. return NULL;
  987. if (!root)
  988. root = root_mem_cgroup;
  989. if (prev && !reclaim)
  990. pos = prev;
  991. if (!root->use_hierarchy && root != root_mem_cgroup) {
  992. if (prev)
  993. goto out;
  994. return root;
  995. }
  996. rcu_read_lock();
  997. if (reclaim) {
  998. struct mem_cgroup_per_node *mz;
  999. mz = mem_cgroup_nodeinfo(root, reclaim->pgdat->node_id);
  1000. iter = &mz->iter;
  1001. if (prev && reclaim->generation != iter->generation)
  1002. goto out_unlock;
  1003. while (1) {
  1004. pos = READ_ONCE(iter->position);
  1005. if (!pos || css_tryget(&pos->css))
  1006. break;
  1007. /*
  1008. * css reference reached zero, so iter->position will
  1009. * be cleared by ->css_released. However, we should not
  1010. * rely on this happening soon, because ->css_released
  1011. * is called from a work queue, and by busy-waiting we
  1012. * might block it. So we clear iter->position right
  1013. * away.
  1014. */
  1015. (void)cmpxchg(&iter->position, pos, NULL);
  1016. }
  1017. }
  1018. if (pos)
  1019. css = &pos->css;
  1020. for (;;) {
  1021. css = css_next_descendant_pre(css, &root->css);
  1022. if (!css) {
  1023. /*
  1024. * Reclaimers share the hierarchy walk, and a
  1025. * new one might jump in right at the end of
  1026. * the hierarchy - make sure they see at least
  1027. * one group and restart from the beginning.
  1028. */
  1029. if (!prev)
  1030. continue;
  1031. break;
  1032. }
  1033. /*
  1034. * Verify the css and acquire a reference. The root
  1035. * is provided by the caller, so we know it's alive
  1036. * and kicking, and don't take an extra reference.
  1037. */
  1038. memcg = mem_cgroup_from_css(css);
  1039. if (css == &root->css)
  1040. break;
  1041. if (css_tryget(css))
  1042. break;
  1043. memcg = NULL;
  1044. }
  1045. if (reclaim) {
  1046. /*
  1047. * The position could have already been updated by a competing
  1048. * thread, so check that the value hasn't changed since we read
  1049. * it to avoid reclaiming from the same cgroup twice.
  1050. */
  1051. (void)cmpxchg(&iter->position, pos, memcg);
  1052. if (pos)
  1053. css_put(&pos->css);
  1054. if (!memcg)
  1055. iter->generation++;
  1056. else if (!prev)
  1057. reclaim->generation = iter->generation;
  1058. }
  1059. out_unlock:
  1060. rcu_read_unlock();
  1061. out:
  1062. if (prev && prev != root)
  1063. css_put(&prev->css);
  1064. return memcg;
  1065. }
  1066. /**
  1067. * mem_cgroup_iter_break - abort a hierarchy walk prematurely
  1068. * @root: hierarchy root
  1069. * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
  1070. */
  1071. void mem_cgroup_iter_break(struct mem_cgroup *root,
  1072. struct mem_cgroup *prev)
  1073. {
  1074. if (!root)
  1075. root = root_mem_cgroup;
  1076. if (prev && prev != root)
  1077. css_put(&prev->css);
  1078. }
  1079. static void __invalidate_reclaim_iterators(struct mem_cgroup *from,
  1080. struct mem_cgroup *dead_memcg)
  1081. {
  1082. struct mem_cgroup_reclaim_iter *iter;
  1083. struct mem_cgroup_per_node *mz;
  1084. int nid;
  1085. for_each_node(nid) {
  1086. mz = mem_cgroup_nodeinfo(from, nid);
  1087. iter = &mz->iter;
  1088. cmpxchg(&iter->position, dead_memcg, NULL);
  1089. }
  1090. }
  1091. static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg)
  1092. {
  1093. struct mem_cgroup *memcg = dead_memcg;
  1094. struct mem_cgroup *last;
  1095. do {
  1096. __invalidate_reclaim_iterators(memcg, dead_memcg);
  1097. last = memcg;
  1098. } while ((memcg = parent_mem_cgroup(memcg)));
  1099. /*
  1100. * When cgruop1 non-hierarchy mode is used,
  1101. * parent_mem_cgroup() does not walk all the way up to the
  1102. * cgroup root (root_mem_cgroup). So we have to handle
  1103. * dead_memcg from cgroup root separately.
  1104. */
  1105. if (last != root_mem_cgroup)
  1106. __invalidate_reclaim_iterators(root_mem_cgroup,
  1107. dead_memcg);
  1108. }
  1109. /**
  1110. * mem_cgroup_scan_tasks - iterate over tasks of a memory cgroup hierarchy
  1111. * @memcg: hierarchy root
  1112. * @fn: function to call for each task
  1113. * @arg: argument passed to @fn
  1114. *
  1115. * This function iterates over tasks attached to @memcg or to any of its
  1116. * descendants and calls @fn for each task. If @fn returns a non-zero
  1117. * value, the function breaks the iteration loop and returns the value.
  1118. * Otherwise, it will iterate over all tasks and return 0.
  1119. *
  1120. * This function must not be called for the root memory cgroup.
  1121. */
  1122. int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
  1123. int (*fn)(struct task_struct *, void *), void *arg)
  1124. {
  1125. struct mem_cgroup *iter;
  1126. int ret = 0;
  1127. BUG_ON(memcg == root_mem_cgroup);
  1128. for_each_mem_cgroup_tree(iter, memcg) {
  1129. struct css_task_iter it;
  1130. struct task_struct *task;
  1131. css_task_iter_start(&iter->css, CSS_TASK_ITER_PROCS, &it);
  1132. while (!ret && (task = css_task_iter_next(&it)))
  1133. ret = fn(task, arg);
  1134. css_task_iter_end(&it);
  1135. if (ret) {
  1136. mem_cgroup_iter_break(memcg, iter);
  1137. break;
  1138. }
  1139. }
  1140. return ret;
  1141. }
  1142. /**
  1143. * mem_cgroup_page_lruvec - return lruvec for isolating/putting an LRU page
  1144. * @page: the page
  1145. * @pgdat: pgdat of the page
  1146. *
  1147. * This function relies on page->mem_cgroup being stable - see the
  1148. * access rules in commit_charge().
  1149. */
  1150. struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct pglist_data *pgdat)
  1151. {
  1152. struct mem_cgroup_per_node *mz;
  1153. struct mem_cgroup *memcg;
  1154. struct lruvec *lruvec;
  1155. if (mem_cgroup_disabled()) {
  1156. lruvec = &pgdat->__lruvec;
  1157. goto out;
  1158. }
  1159. memcg = page->mem_cgroup;
  1160. /*
  1161. * Swapcache readahead pages are added to the LRU - and
  1162. * possibly migrated - before they are charged.
  1163. */
  1164. if (!memcg)
  1165. memcg = root_mem_cgroup;
  1166. mz = mem_cgroup_page_nodeinfo(memcg, page);
  1167. lruvec = &mz->lruvec;
  1168. out:
  1169. /*
  1170. * Since a node can be onlined after the mem_cgroup was created,
  1171. * we have to be prepared to initialize lruvec->zone here;
  1172. * and if offlined then reonlined, we need to reinitialize it.
  1173. */
  1174. if (unlikely(lruvec->pgdat != pgdat))
  1175. lruvec->pgdat = pgdat;
  1176. return lruvec;
  1177. }
  1178. /**
  1179. * mem_cgroup_update_lru_size - account for adding or removing an lru page
  1180. * @lruvec: mem_cgroup per zone lru vector
  1181. * @lru: index of lru list the page is sitting on
  1182. * @zid: zone id of the accounted pages
  1183. * @nr_pages: positive when adding or negative when removing
  1184. *
  1185. * This function must be called under lru_lock, just before a page is added
  1186. * to or just after a page is removed from an lru list (that ordering being
  1187. * so as to allow it to check that lru_size 0 is consistent with list_empty).
  1188. */
  1189. void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
  1190. int zid, int nr_pages)
  1191. {
  1192. struct mem_cgroup_per_node *mz;
  1193. unsigned long *lru_size;
  1194. long size;
  1195. if (mem_cgroup_disabled())
  1196. return;
  1197. mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
  1198. lru_size = &mz->lru_zone_size[zid][lru];
  1199. if (nr_pages < 0)
  1200. *lru_size += nr_pages;
  1201. size = *lru_size;
  1202. if (WARN_ONCE(size < 0,
  1203. "%s(%p, %d, %d): lru_size %ld\n",
  1204. __func__, lruvec, lru, nr_pages, size)) {
  1205. VM_BUG_ON(1);
  1206. *lru_size = 0;
  1207. }
  1208. if (nr_pages > 0)
  1209. *lru_size += nr_pages;
  1210. }
  1211. /**
  1212. * mem_cgroup_margin - calculate chargeable space of a memory cgroup
  1213. * @memcg: the memory cgroup
  1214. *
  1215. * Returns the maximum amount of memory @mem can be charged with, in
  1216. * pages.
  1217. */
  1218. static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
  1219. {
  1220. unsigned long margin = 0;
  1221. unsigned long count;
  1222. unsigned long limit;
  1223. count = page_counter_read(&memcg->memory);
  1224. limit = READ_ONCE(memcg->memory.max);
  1225. if (count < limit)
  1226. margin = limit - count;
  1227. if (do_memsw_account()) {
  1228. count = page_counter_read(&memcg->memsw);
  1229. limit = READ_ONCE(memcg->memsw.max);
  1230. if (count < limit)
  1231. margin = min(margin, limit - count);
  1232. else
  1233. margin = 0;
  1234. }
  1235. return margin;
  1236. }
  1237. /*
  1238. * A routine for checking "mem" is under move_account() or not.
  1239. *
  1240. * Checking a cgroup is mc.from or mc.to or under hierarchy of
  1241. * moving cgroups. This is for waiting at high-memory pressure
  1242. * caused by "move".
  1243. */
  1244. static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
  1245. {
  1246. struct mem_cgroup *from;
  1247. struct mem_cgroup *to;
  1248. bool ret = false;
  1249. /*
  1250. * Unlike task_move routines, we access mc.to, mc.from not under
  1251. * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
  1252. */
  1253. spin_lock(&mc.lock);
  1254. from = mc.from;
  1255. to = mc.to;
  1256. if (!from)
  1257. goto unlock;
  1258. ret = mem_cgroup_is_descendant(from, memcg) ||
  1259. mem_cgroup_is_descendant(to, memcg);
  1260. unlock:
  1261. spin_unlock(&mc.lock);
  1262. return ret;
  1263. }
  1264. static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
  1265. {
  1266. if (mc.moving_task && current != mc.moving_task) {
  1267. if (mem_cgroup_under_move(memcg)) {
  1268. DEFINE_WAIT(wait);
  1269. prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
  1270. /* moving charge context might have finished. */
  1271. if (mc.moving_task)
  1272. schedule();
  1273. finish_wait(&mc.waitq, &wait);
  1274. return true;
  1275. }
  1276. }
  1277. return false;
  1278. }
  1279. struct memory_stat {
  1280. const char *name;
  1281. unsigned int ratio;
  1282. unsigned int idx;
  1283. };
  1284. static struct memory_stat memory_stats[] = {
  1285. { "anon", PAGE_SIZE, NR_ANON_MAPPED },
  1286. { "file", PAGE_SIZE, NR_FILE_PAGES },
  1287. { "kernel_stack", 1024, NR_KERNEL_STACK_KB },
  1288. { "percpu", 1, MEMCG_PERCPU_B },
  1289. { "sock", PAGE_SIZE, MEMCG_SOCK },
  1290. { "shmem", PAGE_SIZE, NR_SHMEM },
  1291. { "file_mapped", PAGE_SIZE, NR_FILE_MAPPED },
  1292. { "file_dirty", PAGE_SIZE, NR_FILE_DIRTY },
  1293. { "file_writeback", PAGE_SIZE, NR_WRITEBACK },
  1294. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  1295. /*
  1296. * The ratio will be initialized in memory_stats_init(). Because
  1297. * on some architectures, the macro of HPAGE_PMD_SIZE is not
  1298. * constant(e.g. powerpc).
  1299. */
  1300. { "anon_thp", 0, NR_ANON_THPS },
  1301. #endif
  1302. { "inactive_anon", PAGE_SIZE, NR_INACTIVE_ANON },
  1303. { "active_anon", PAGE_SIZE, NR_ACTIVE_ANON },
  1304. { "inactive_file", PAGE_SIZE, NR_INACTIVE_FILE },
  1305. { "active_file", PAGE_SIZE, NR_ACTIVE_FILE },
  1306. { "unevictable", PAGE_SIZE, NR_UNEVICTABLE },
  1307. /*
  1308. * Note: The slab_reclaimable and slab_unreclaimable must be
  1309. * together and slab_reclaimable must be in front.
  1310. */
  1311. { "slab_reclaimable", 1, NR_SLAB_RECLAIMABLE_B },
  1312. { "slab_unreclaimable", 1, NR_SLAB_UNRECLAIMABLE_B },
  1313. /* The memory events */
  1314. { "workingset_refault_anon", 1, WORKINGSET_REFAULT_ANON },
  1315. { "workingset_refault_file", 1, WORKINGSET_REFAULT_FILE },
  1316. { "workingset_activate_anon", 1, WORKINGSET_ACTIVATE_ANON },
  1317. { "workingset_activate_file", 1, WORKINGSET_ACTIVATE_FILE },
  1318. { "workingset_restore_anon", 1, WORKINGSET_RESTORE_ANON },
  1319. { "workingset_restore_file", 1, WORKINGSET_RESTORE_FILE },
  1320. { "workingset_nodereclaim", 1, WORKINGSET_NODERECLAIM },
  1321. };
  1322. static int __init memory_stats_init(void)
  1323. {
  1324. int i;
  1325. for (i = 0; i < ARRAY_SIZE(memory_stats); i++) {
  1326. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  1327. if (memory_stats[i].idx == NR_ANON_THPS)
  1328. memory_stats[i].ratio = HPAGE_PMD_SIZE;
  1329. #endif
  1330. VM_BUG_ON(!memory_stats[i].ratio);
  1331. VM_BUG_ON(memory_stats[i].idx >= MEMCG_NR_STAT);
  1332. }
  1333. return 0;
  1334. }
  1335. pure_initcall(memory_stats_init);
  1336. static char *memory_stat_format(struct mem_cgroup *memcg)
  1337. {
  1338. struct seq_buf s;
  1339. int i;
  1340. seq_buf_init(&s, kmalloc(PAGE_SIZE, GFP_KERNEL), PAGE_SIZE);
  1341. if (!s.buffer)
  1342. return NULL;
  1343. /*
  1344. * Provide statistics on the state of the memory subsystem as
  1345. * well as cumulative event counters that show past behavior.
  1346. *
  1347. * This list is ordered following a combination of these gradients:
  1348. * 1) generic big picture -> specifics and details
  1349. * 2) reflecting userspace activity -> reflecting kernel heuristics
  1350. *
  1351. * Current memory state:
  1352. */
  1353. for (i = 0; i < ARRAY_SIZE(memory_stats); i++) {
  1354. u64 size;
  1355. size = memcg_page_state(memcg, memory_stats[i].idx);
  1356. size *= memory_stats[i].ratio;
  1357. seq_buf_printf(&s, "%s %llu\n", memory_stats[i].name, size);
  1358. if (unlikely(memory_stats[i].idx == NR_SLAB_UNRECLAIMABLE_B)) {
  1359. size = memcg_page_state(memcg, NR_SLAB_RECLAIMABLE_B) +
  1360. memcg_page_state(memcg, NR_SLAB_UNRECLAIMABLE_B);
  1361. seq_buf_printf(&s, "slab %llu\n", size);
  1362. }
  1363. }
  1364. /* Accumulated memory events */
  1365. seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGFAULT),
  1366. memcg_events(memcg, PGFAULT));
  1367. seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGMAJFAULT),
  1368. memcg_events(memcg, PGMAJFAULT));
  1369. seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGREFILL),
  1370. memcg_events(memcg, PGREFILL));
  1371. seq_buf_printf(&s, "pgscan %lu\n",
  1372. memcg_events(memcg, PGSCAN_KSWAPD) +
  1373. memcg_events(memcg, PGSCAN_DIRECT));
  1374. seq_buf_printf(&s, "pgsteal %lu\n",
  1375. memcg_events(memcg, PGSTEAL_KSWAPD) +
  1376. memcg_events(memcg, PGSTEAL_DIRECT));
  1377. seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGACTIVATE),
  1378. memcg_events(memcg, PGACTIVATE));
  1379. seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGDEACTIVATE),
  1380. memcg_events(memcg, PGDEACTIVATE));
  1381. seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGLAZYFREE),
  1382. memcg_events(memcg, PGLAZYFREE));
  1383. seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGLAZYFREED),
  1384. memcg_events(memcg, PGLAZYFREED));
  1385. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  1386. seq_buf_printf(&s, "%s %lu\n", vm_event_name(THP_FAULT_ALLOC),
  1387. memcg_events(memcg, THP_FAULT_ALLOC));
  1388. seq_buf_printf(&s, "%s %lu\n", vm_event_name(THP_COLLAPSE_ALLOC),
  1389. memcg_events(memcg, THP_COLLAPSE_ALLOC));
  1390. #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
  1391. /* The above should easily fit into one page */
  1392. WARN_ON_ONCE(seq_buf_has_overflowed(&s));
  1393. return s.buffer;
  1394. }
  1395. #define K(x) ((x) << (PAGE_SHIFT-10))
  1396. /**
  1397. * mem_cgroup_print_oom_context: Print OOM information relevant to
  1398. * memory controller.
  1399. * @memcg: The memory cgroup that went over limit
  1400. * @p: Task that is going to be killed
  1401. *
  1402. * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
  1403. * enabled
  1404. */
  1405. void mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p)
  1406. {
  1407. rcu_read_lock();
  1408. if (memcg) {
  1409. pr_cont(",oom_memcg=");
  1410. pr_cont_cgroup_path(memcg->css.cgroup);
  1411. } else
  1412. pr_cont(",global_oom");
  1413. if (p) {
  1414. pr_cont(",task_memcg=");
  1415. pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id));
  1416. }
  1417. rcu_read_unlock();
  1418. }
  1419. /**
  1420. * mem_cgroup_print_oom_meminfo: Print OOM memory information relevant to
  1421. * memory controller.
  1422. * @memcg: The memory cgroup that went over limit
  1423. */
  1424. void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg)
  1425. {
  1426. char *buf;
  1427. pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n",
  1428. K((u64)page_counter_read(&memcg->memory)),
  1429. K((u64)READ_ONCE(memcg->memory.max)), memcg->memory.failcnt);
  1430. if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
  1431. pr_info("swap: usage %llukB, limit %llukB, failcnt %lu\n",
  1432. K((u64)page_counter_read(&memcg->swap)),
  1433. K((u64)READ_ONCE(memcg->swap.max)), memcg->swap.failcnt);
  1434. else {
  1435. pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n",
  1436. K((u64)page_counter_read(&memcg->memsw)),
  1437. K((u64)memcg->memsw.max), memcg->memsw.failcnt);
  1438. pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n",
  1439. K((u64)page_counter_read(&memcg->kmem)),
  1440. K((u64)memcg->kmem.max), memcg->kmem.failcnt);
  1441. }
  1442. pr_info("Memory cgroup stats for ");
  1443. pr_cont_cgroup_path(memcg->css.cgroup);
  1444. pr_cont(":");
  1445. buf = memory_stat_format(memcg);
  1446. if (!buf)
  1447. return;
  1448. pr_info("%s", buf);
  1449. kfree(buf);
  1450. }
  1451. /*
  1452. * Return the memory (and swap, if configured) limit for a memcg.
  1453. */
  1454. unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg)
  1455. {
  1456. unsigned long max = READ_ONCE(memcg->memory.max);
  1457. if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
  1458. if (mem_cgroup_swappiness(memcg))
  1459. max += min(READ_ONCE(memcg->swap.max),
  1460. (unsigned long)total_swap_pages);
  1461. } else { /* v1 */
  1462. if (mem_cgroup_swappiness(memcg)) {
  1463. /* Calculate swap excess capacity from memsw limit */
  1464. unsigned long swap = READ_ONCE(memcg->memsw.max) - max;
  1465. max += min(swap, (unsigned long)total_swap_pages);
  1466. }
  1467. }
  1468. return max;
  1469. }
  1470. unsigned long mem_cgroup_size(struct mem_cgroup *memcg)
  1471. {
  1472. return page_counter_read(&memcg->memory);
  1473. }
  1474. static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
  1475. int order)
  1476. {
  1477. struct oom_control oc = {
  1478. .zonelist = NULL,
  1479. .nodemask = NULL,
  1480. .memcg = memcg,
  1481. .gfp_mask = gfp_mask,
  1482. .order = order,
  1483. };
  1484. bool ret = true;
  1485. if (mutex_lock_killable(&oom_lock))
  1486. return true;
  1487. if (mem_cgroup_margin(memcg) >= (1 << order))
  1488. goto unlock;
  1489. /*
  1490. * A few threads which were not waiting at mutex_lock_killable() can
  1491. * fail to bail out. Therefore, check again after holding oom_lock.
  1492. */
  1493. ret = task_is_dying() || out_of_memory(&oc);
  1494. unlock:
  1495. mutex_unlock(&oom_lock);
  1496. return ret;
  1497. }
  1498. static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
  1499. pg_data_t *pgdat,
  1500. gfp_t gfp_mask,
  1501. unsigned long *total_scanned)
  1502. {
  1503. struct mem_cgroup *victim = NULL;
  1504. int total = 0;
  1505. int loop = 0;
  1506. unsigned long excess;
  1507. unsigned long nr_scanned;
  1508. struct mem_cgroup_reclaim_cookie reclaim = {
  1509. .pgdat = pgdat,
  1510. };
  1511. excess = soft_limit_excess(root_memcg);
  1512. while (1) {
  1513. victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
  1514. if (!victim) {
  1515. loop++;
  1516. if (loop >= 2) {
  1517. /*
  1518. * If we have not been able to reclaim
  1519. * anything, it might because there are
  1520. * no reclaimable pages under this hierarchy
  1521. */
  1522. if (!total)
  1523. break;
  1524. /*
  1525. * We want to do more targeted reclaim.
  1526. * excess >> 2 is not to excessive so as to
  1527. * reclaim too much, nor too less that we keep
  1528. * coming back to reclaim from this cgroup
  1529. */
  1530. if (total >= (excess >> 2) ||
  1531. (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
  1532. break;
  1533. }
  1534. continue;
  1535. }
  1536. total += mem_cgroup_shrink_node(victim, gfp_mask, false,
  1537. pgdat, &nr_scanned);
  1538. *total_scanned += nr_scanned;
  1539. if (!soft_limit_excess(root_memcg))
  1540. break;
  1541. }
  1542. mem_cgroup_iter_break(root_memcg, victim);
  1543. return total;
  1544. }
  1545. #ifdef CONFIG_LOCKDEP
  1546. static struct lockdep_map memcg_oom_lock_dep_map = {
  1547. .name = "memcg_oom_lock",
  1548. };
  1549. #endif
  1550. static DEFINE_SPINLOCK(memcg_oom_lock);
  1551. /*
  1552. * Check OOM-Killer is already running under our hierarchy.
  1553. * If someone is running, return false.
  1554. */
  1555. static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
  1556. {
  1557. struct mem_cgroup *iter, *failed = NULL;
  1558. spin_lock(&memcg_oom_lock);
  1559. for_each_mem_cgroup_tree(iter, memcg) {
  1560. if (iter->oom_lock) {
  1561. /*
  1562. * this subtree of our hierarchy is already locked
  1563. * so we cannot give a lock.
  1564. */
  1565. failed = iter;
  1566. mem_cgroup_iter_break(memcg, iter);
  1567. break;
  1568. } else
  1569. iter->oom_lock = true;
  1570. }
  1571. if (failed) {
  1572. /*
  1573. * OK, we failed to lock the whole subtree so we have
  1574. * to clean up what we set up to the failing subtree
  1575. */
  1576. for_each_mem_cgroup_tree(iter, memcg) {
  1577. if (iter == failed) {
  1578. mem_cgroup_iter_break(memcg, iter);
  1579. break;
  1580. }
  1581. iter->oom_lock = false;
  1582. }
  1583. } else
  1584. mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_);
  1585. spin_unlock(&memcg_oom_lock);
  1586. return !failed;
  1587. }
  1588. static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
  1589. {
  1590. struct mem_cgroup *iter;
  1591. spin_lock(&memcg_oom_lock);
  1592. mutex_release(&memcg_oom_lock_dep_map, _RET_IP_);
  1593. for_each_mem_cgroup_tree(iter, memcg)
  1594. iter->oom_lock = false;
  1595. spin_unlock(&memcg_oom_lock);
  1596. }
  1597. static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
  1598. {
  1599. struct mem_cgroup *iter;
  1600. spin_lock(&memcg_oom_lock);
  1601. for_each_mem_cgroup_tree(iter, memcg)
  1602. iter->under_oom++;
  1603. spin_unlock(&memcg_oom_lock);
  1604. }
  1605. static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
  1606. {
  1607. struct mem_cgroup *iter;
  1608. /*
  1609. * Be careful about under_oom underflows becase a child memcg
  1610. * could have been added after mem_cgroup_mark_under_oom.
  1611. */
  1612. spin_lock(&memcg_oom_lock);
  1613. for_each_mem_cgroup_tree(iter, memcg)
  1614. if (iter->under_oom > 0)
  1615. iter->under_oom--;
  1616. spin_unlock(&memcg_oom_lock);
  1617. }
  1618. static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
  1619. struct oom_wait_info {
  1620. struct mem_cgroup *memcg;
  1621. wait_queue_entry_t wait;
  1622. };
  1623. static int memcg_oom_wake_function(wait_queue_entry_t *wait,
  1624. unsigned mode, int sync, void *arg)
  1625. {
  1626. struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
  1627. struct mem_cgroup *oom_wait_memcg;
  1628. struct oom_wait_info *oom_wait_info;
  1629. oom_wait_info = container_of(wait, struct oom_wait_info, wait);
  1630. oom_wait_memcg = oom_wait_info->memcg;
  1631. if (!mem_cgroup_is_descendant(wake_memcg, oom_wait_memcg) &&
  1632. !mem_cgroup_is_descendant(oom_wait_memcg, wake_memcg))
  1633. return 0;
  1634. return autoremove_wake_function(wait, mode, sync, arg);
  1635. }
  1636. static void memcg_oom_recover(struct mem_cgroup *memcg)
  1637. {
  1638. /*
  1639. * For the following lockless ->under_oom test, the only required
  1640. * guarantee is that it must see the state asserted by an OOM when
  1641. * this function is called as a result of userland actions
  1642. * triggered by the notification of the OOM. This is trivially
  1643. * achieved by invoking mem_cgroup_mark_under_oom() before
  1644. * triggering notification.
  1645. */
  1646. if (memcg && memcg->under_oom)
  1647. __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
  1648. }
  1649. enum oom_status {
  1650. OOM_SUCCESS,
  1651. OOM_FAILED,
  1652. OOM_ASYNC,
  1653. OOM_SKIPPED
  1654. };
  1655. static enum oom_status mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
  1656. {
  1657. enum oom_status ret;
  1658. bool locked;
  1659. if (order > PAGE_ALLOC_COSTLY_ORDER)
  1660. return OOM_SKIPPED;
  1661. memcg_memory_event(memcg, MEMCG_OOM);
  1662. /*
  1663. * We are in the middle of the charge context here, so we
  1664. * don't want to block when potentially sitting on a callstack
  1665. * that holds all kinds of filesystem and mm locks.
  1666. *
  1667. * cgroup1 allows disabling the OOM killer and waiting for outside
  1668. * handling until the charge can succeed; remember the context and put
  1669. * the task to sleep at the end of the page fault when all locks are
  1670. * released.
  1671. *
  1672. * On the other hand, in-kernel OOM killer allows for an async victim
  1673. * memory reclaim (oom_reaper) and that means that we are not solely
  1674. * relying on the oom victim to make a forward progress and we can
  1675. * invoke the oom killer here.
  1676. *
  1677. * Please note that mem_cgroup_out_of_memory might fail to find a
  1678. * victim and then we have to bail out from the charge path.
  1679. */
  1680. if (memcg->oom_kill_disable) {
  1681. if (!current->in_user_fault)
  1682. return OOM_SKIPPED;
  1683. css_get(&memcg->css);
  1684. current->memcg_in_oom = memcg;
  1685. current->memcg_oom_gfp_mask = mask;
  1686. current->memcg_oom_order = order;
  1687. return OOM_ASYNC;
  1688. }
  1689. mem_cgroup_mark_under_oom(memcg);
  1690. locked = mem_cgroup_oom_trylock(memcg);
  1691. if (locked)
  1692. mem_cgroup_oom_notify(memcg);
  1693. mem_cgroup_unmark_under_oom(memcg);
  1694. if (mem_cgroup_out_of_memory(memcg, mask, order))
  1695. ret = OOM_SUCCESS;
  1696. else
  1697. ret = OOM_FAILED;
  1698. if (locked)
  1699. mem_cgroup_oom_unlock(memcg);
  1700. return ret;
  1701. }
  1702. /**
  1703. * mem_cgroup_oom_synchronize - complete memcg OOM handling
  1704. * @handle: actually kill/wait or just clean up the OOM state
  1705. *
  1706. * This has to be called at the end of a page fault if the memcg OOM
  1707. * handler was enabled.
  1708. *
  1709. * Memcg supports userspace OOM handling where failed allocations must
  1710. * sleep on a waitqueue until the userspace task resolves the
  1711. * situation. Sleeping directly in the charge context with all kinds
  1712. * of locks held is not a good idea, instead we remember an OOM state
  1713. * in the task and mem_cgroup_oom_synchronize() has to be called at
  1714. * the end of the page fault to complete the OOM handling.
  1715. *
  1716. * Returns %true if an ongoing memcg OOM situation was detected and
  1717. * completed, %false otherwise.
  1718. */
  1719. bool mem_cgroup_oom_synchronize(bool handle)
  1720. {
  1721. struct mem_cgroup *memcg = current->memcg_in_oom;
  1722. struct oom_wait_info owait;
  1723. bool locked;
  1724. /* OOM is global, do not handle */
  1725. if (!memcg)
  1726. return false;
  1727. if (!handle)
  1728. goto cleanup;
  1729. owait.memcg = memcg;
  1730. owait.wait.flags = 0;
  1731. owait.wait.func = memcg_oom_wake_function;
  1732. owait.wait.private = current;
  1733. INIT_LIST_HEAD(&owait.wait.entry);
  1734. prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
  1735. mem_cgroup_mark_under_oom(memcg);
  1736. locked = mem_cgroup_oom_trylock(memcg);
  1737. if (locked)
  1738. mem_cgroup_oom_notify(memcg);
  1739. if (locked && !memcg->oom_kill_disable) {
  1740. mem_cgroup_unmark_under_oom(memcg);
  1741. finish_wait(&memcg_oom_waitq, &owait.wait);
  1742. mem_cgroup_out_of_memory(memcg, current->memcg_oom_gfp_mask,
  1743. current->memcg_oom_order);
  1744. } else {
  1745. schedule();
  1746. mem_cgroup_unmark_under_oom(memcg);
  1747. finish_wait(&memcg_oom_waitq, &owait.wait);
  1748. }
  1749. if (locked) {
  1750. mem_cgroup_oom_unlock(memcg);
  1751. /*
  1752. * There is no guarantee that an OOM-lock contender
  1753. * sees the wakeups triggered by the OOM kill
  1754. * uncharges. Wake any sleepers explicitely.
  1755. */
  1756. memcg_oom_recover(memcg);
  1757. }
  1758. cleanup:
  1759. current->memcg_in_oom = NULL;
  1760. css_put(&memcg->css);
  1761. return true;
  1762. }
  1763. /**
  1764. * mem_cgroup_get_oom_group - get a memory cgroup to clean up after OOM
  1765. * @victim: task to be killed by the OOM killer
  1766. * @oom_domain: memcg in case of memcg OOM, NULL in case of system-wide OOM
  1767. *
  1768. * Returns a pointer to a memory cgroup, which has to be cleaned up
  1769. * by killing all belonging OOM-killable tasks.
  1770. *
  1771. * Caller has to call mem_cgroup_put() on the returned non-NULL memcg.
  1772. */
  1773. struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim,
  1774. struct mem_cgroup *oom_domain)
  1775. {
  1776. struct mem_cgroup *oom_group = NULL;
  1777. struct mem_cgroup *memcg;
  1778. if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
  1779. return NULL;
  1780. if (!oom_domain)
  1781. oom_domain = root_mem_cgroup;
  1782. rcu_read_lock();
  1783. memcg = mem_cgroup_from_task(victim);
  1784. if (memcg == root_mem_cgroup)
  1785. goto out;
  1786. /*
  1787. * If the victim task has been asynchronously moved to a different
  1788. * memory cgroup, we might end up killing tasks outside oom_domain.
  1789. * In this case it's better to ignore memory.group.oom.
  1790. */
  1791. if (unlikely(!mem_cgroup_is_descendant(memcg, oom_domain)))
  1792. goto out;
  1793. /*
  1794. * Traverse the memory cgroup hierarchy from the victim task's
  1795. * cgroup up to the OOMing cgroup (or root) to find the
  1796. * highest-level memory cgroup with oom.group set.
  1797. */
  1798. for (; memcg; memcg = parent_mem_cgroup(memcg)) {
  1799. if (memcg->oom_group)
  1800. oom_group = memcg;
  1801. if (memcg == oom_domain)
  1802. break;
  1803. }
  1804. if (oom_group)
  1805. css_get(&oom_group->css);
  1806. out:
  1807. rcu_read_unlock();
  1808. return oom_group;
  1809. }
  1810. void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)
  1811. {
  1812. pr_info("Tasks in ");
  1813. pr_cont_cgroup_path(memcg->css.cgroup);
  1814. pr_cont(" are going to be killed due to memory.oom.group set\n");
  1815. }
  1816. /**
  1817. * lock_page_memcg - lock a page->mem_cgroup binding
  1818. * @page: the page
  1819. *
  1820. * This function protects unlocked LRU pages from being moved to
  1821. * another cgroup.
  1822. *
  1823. * It ensures lifetime of the returned memcg. Caller is responsible
  1824. * for the lifetime of the page; __unlock_page_memcg() is available
  1825. * when @page might get freed inside the locked section.
  1826. */
  1827. struct mem_cgroup *lock_page_memcg(struct page *page)
  1828. {
  1829. struct page *head = compound_head(page); /* rmap on tail pages */
  1830. struct mem_cgroup *memcg;
  1831. unsigned long flags;
  1832. /*
  1833. * The RCU lock is held throughout the transaction. The fast
  1834. * path can get away without acquiring the memcg->move_lock
  1835. * because page moving starts with an RCU grace period.
  1836. *
  1837. * The RCU lock also protects the memcg from being freed when
  1838. * the page state that is going to change is the only thing
  1839. * preventing the page itself from being freed. E.g. writeback
  1840. * doesn't hold a page reference and relies on PG_writeback to
  1841. * keep off truncation, migration and so forth.
  1842. */
  1843. rcu_read_lock();
  1844. if (mem_cgroup_disabled())
  1845. return NULL;
  1846. again:
  1847. memcg = head->mem_cgroup;
  1848. if (unlikely(!memcg))
  1849. return NULL;
  1850. if (atomic_read(&memcg->moving_account) <= 0)
  1851. return memcg;
  1852. spin_lock_irqsave(&memcg->move_lock, flags);
  1853. if (memcg != head->mem_cgroup) {
  1854. spin_unlock_irqrestore(&memcg->move_lock, flags);
  1855. goto again;
  1856. }
  1857. /*
  1858. * When charge migration first begins, we can have locked and
  1859. * unlocked page stat updates happening concurrently. Track
  1860. * the task who has the lock for unlock_page_memcg().
  1861. */
  1862. memcg->move_lock_task = current;
  1863. memcg->move_lock_flags = flags;
  1864. return memcg;
  1865. }
  1866. EXPORT_SYMBOL(lock_page_memcg);
  1867. /**
  1868. * __unlock_page_memcg - unlock and unpin a memcg
  1869. * @memcg: the memcg
  1870. *
  1871. * Unlock and unpin a memcg returned by lock_page_memcg().
  1872. */
  1873. void __unlock_page_memcg(struct mem_cgroup *memcg)
  1874. {
  1875. if (memcg && memcg->move_lock_task == current) {
  1876. unsigned long flags = memcg->move_lock_flags;
  1877. memcg->move_lock_task = NULL;
  1878. memcg->move_lock_flags = 0;
  1879. spin_unlock_irqrestore(&memcg->move_lock, flags);
  1880. }
  1881. rcu_read_unlock();
  1882. }
  1883. /**
  1884. * unlock_page_memcg - unlock a page->mem_cgroup binding
  1885. * @page: the page
  1886. */
  1887. void unlock_page_memcg(struct page *page)
  1888. {
  1889. struct page *head = compound_head(page);
  1890. __unlock_page_memcg(head->mem_cgroup);
  1891. }
  1892. EXPORT_SYMBOL(unlock_page_memcg);
  1893. struct memcg_stock_pcp {
  1894. struct mem_cgroup *cached; /* this never be root cgroup */
  1895. unsigned int nr_pages;
  1896. #ifdef CONFIG_MEMCG_KMEM
  1897. struct obj_cgroup *cached_objcg;
  1898. unsigned int nr_bytes;
  1899. #endif
  1900. struct work_struct work;
  1901. unsigned long flags;
  1902. #define FLUSHING_CACHED_CHARGE 0
  1903. };
  1904. static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
  1905. static DEFINE_MUTEX(percpu_charge_mutex);
  1906. #ifdef CONFIG_MEMCG_KMEM
  1907. static void drain_obj_stock(struct memcg_stock_pcp *stock);
  1908. static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
  1909. struct mem_cgroup *root_memcg);
  1910. #else
  1911. static inline void drain_obj_stock(struct memcg_stock_pcp *stock)
  1912. {
  1913. }
  1914. static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
  1915. struct mem_cgroup *root_memcg)
  1916. {
  1917. return false;
  1918. }
  1919. #endif
  1920. /**
  1921. * consume_stock: Try to consume stocked charge on this cpu.
  1922. * @memcg: memcg to consume from.
  1923. * @nr_pages: how many pages to charge.
  1924. *
  1925. * The charges will only happen if @memcg matches the current cpu's memcg
  1926. * stock, and at least @nr_pages are available in that stock. Failure to
  1927. * service an allocation will refill the stock.
  1928. *
  1929. * returns true if successful, false otherwise.
  1930. */
  1931. static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
  1932. {
  1933. struct memcg_stock_pcp *stock;
  1934. unsigned long flags;
  1935. bool ret = false;
  1936. if (nr_pages > MEMCG_CHARGE_BATCH)
  1937. return ret;
  1938. local_irq_save(flags);
  1939. stock = this_cpu_ptr(&memcg_stock);
  1940. if (memcg == stock->cached && stock->nr_pages >= nr_pages) {
  1941. stock->nr_pages -= nr_pages;
  1942. ret = true;
  1943. }
  1944. local_irq_restore(flags);
  1945. return ret;
  1946. }
  1947. /*
  1948. * Returns stocks cached in percpu and reset cached information.
  1949. */
  1950. static void drain_stock(struct memcg_stock_pcp *stock)
  1951. {
  1952. struct mem_cgroup *old = stock->cached;
  1953. if (!old)
  1954. return;
  1955. if (stock->nr_pages) {
  1956. page_counter_uncharge(&old->memory, stock->nr_pages);
  1957. if (do_memsw_account())
  1958. page_counter_uncharge(&old->memsw, stock->nr_pages);
  1959. stock->nr_pages = 0;
  1960. }
  1961. css_put(&old->css);
  1962. stock->cached = NULL;
  1963. }
  1964. static void drain_local_stock(struct work_struct *dummy)
  1965. {
  1966. struct memcg_stock_pcp *stock;
  1967. unsigned long flags;
  1968. /*
  1969. * The only protection from memory hotplug vs. drain_stock races is
  1970. * that we always operate on local CPU stock here with IRQ disabled
  1971. */
  1972. local_irq_save(flags);
  1973. stock = this_cpu_ptr(&memcg_stock);
  1974. drain_obj_stock(stock);
  1975. drain_stock(stock);
  1976. clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
  1977. local_irq_restore(flags);
  1978. }
  1979. /*
  1980. * Cache charges(val) to local per_cpu area.
  1981. * This will be consumed by consume_stock() function, later.
  1982. */
  1983. static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
  1984. {
  1985. struct memcg_stock_pcp *stock;
  1986. unsigned long flags;
  1987. local_irq_save(flags);
  1988. stock = this_cpu_ptr(&memcg_stock);
  1989. if (stock->cached != memcg) { /* reset if necessary */
  1990. drain_stock(stock);
  1991. css_get(&memcg->css);
  1992. stock->cached = memcg;
  1993. }
  1994. stock->nr_pages += nr_pages;
  1995. if (stock->nr_pages > MEMCG_CHARGE_BATCH)
  1996. drain_stock(stock);
  1997. local_irq_restore(flags);
  1998. }
  1999. /*
  2000. * Drains all per-CPU charge caches for given root_memcg resp. subtree
  2001. * of the hierarchy under it.
  2002. */
  2003. static void drain_all_stock(struct mem_cgroup *root_memcg)
  2004. {
  2005. int cpu, curcpu;
  2006. /* If someone's already draining, avoid adding running more workers. */
  2007. if (!mutex_trylock(&percpu_charge_mutex))
  2008. return;
  2009. /*
  2010. * Notify other cpus that system-wide "drain" is running
  2011. * We do not care about races with the cpu hotplug because cpu down
  2012. * as well as workers from this path always operate on the local
  2013. * per-cpu data. CPU up doesn't touch memcg_stock at all.
  2014. */
  2015. curcpu = get_cpu();
  2016. for_each_online_cpu(cpu) {
  2017. struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
  2018. struct mem_cgroup *memcg;
  2019. bool flush = false;
  2020. rcu_read_lock();
  2021. memcg = stock->cached;
  2022. if (memcg && stock->nr_pages &&
  2023. mem_cgroup_is_descendant(memcg, root_memcg))
  2024. flush = true;
  2025. if (obj_stock_flush_required(stock, root_memcg))
  2026. flush = true;
  2027. rcu_read_unlock();
  2028. if (flush &&
  2029. !test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
  2030. if (cpu == curcpu)
  2031. drain_local_stock(&stock->work);
  2032. else
  2033. schedule_work_on(cpu, &stock->work);
  2034. }
  2035. }
  2036. put_cpu();
  2037. mutex_unlock(&percpu_charge_mutex);
  2038. }
  2039. static int memcg_hotplug_cpu_dead(unsigned int cpu)
  2040. {
  2041. struct memcg_stock_pcp *stock;
  2042. struct mem_cgroup *memcg, *mi;
  2043. stock = &per_cpu(memcg_stock, cpu);
  2044. drain_stock(stock);
  2045. for_each_mem_cgroup(memcg) {
  2046. int i;
  2047. for (i = 0; i < MEMCG_NR_STAT; i++) {
  2048. int nid;
  2049. long x;
  2050. x = this_cpu_xchg(memcg->vmstats_percpu->stat[i], 0);
  2051. if (x)
  2052. for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
  2053. atomic_long_add(x, &memcg->vmstats[i]);
  2054. if (i >= NR_VM_NODE_STAT_ITEMS)
  2055. continue;
  2056. for_each_node(nid) {
  2057. struct mem_cgroup_per_node *pn;
  2058. pn = mem_cgroup_nodeinfo(memcg, nid);
  2059. x = this_cpu_xchg(pn->lruvec_stat_cpu->count[i], 0);
  2060. if (x)
  2061. do {
  2062. atomic_long_add(x, &pn->lruvec_stat[i]);
  2063. } while ((pn = parent_nodeinfo(pn, nid)));
  2064. }
  2065. }
  2066. for (i = 0; i < NR_VM_EVENT_ITEMS; i++) {
  2067. long x;
  2068. x = this_cpu_xchg(memcg->vmstats_percpu->events[i], 0);
  2069. if (x)
  2070. for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
  2071. atomic_long_add(x, &memcg->vmevents[i]);
  2072. }
  2073. }
  2074. return 0;
  2075. }
  2076. static unsigned long reclaim_high(struct mem_cgroup *memcg,
  2077. unsigned int nr_pages,
  2078. gfp_t gfp_mask)
  2079. {
  2080. unsigned long nr_reclaimed = 0;
  2081. do {
  2082. unsigned long pflags;
  2083. if (page_counter_read(&memcg->memory) <=
  2084. READ_ONCE(memcg->memory.high))
  2085. continue;
  2086. memcg_memory_event(memcg, MEMCG_HIGH);
  2087. psi_memstall_enter(&pflags);
  2088. nr_reclaimed += try_to_free_mem_cgroup_pages(memcg, nr_pages,
  2089. gfp_mask, true);
  2090. psi_memstall_leave(&pflags);
  2091. } while ((memcg = parent_mem_cgroup(memcg)) &&
  2092. !mem_cgroup_is_root(memcg));
  2093. return nr_reclaimed;
  2094. }
  2095. static void high_work_func(struct work_struct *work)
  2096. {
  2097. struct mem_cgroup *memcg;
  2098. memcg = container_of(work, struct mem_cgroup, high_work);
  2099. reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL);
  2100. }
  2101. /*
  2102. * Clamp the maximum sleep time per allocation batch to 2 seconds. This is
  2103. * enough to still cause a significant slowdown in most cases, while still
  2104. * allowing diagnostics and tracing to proceed without becoming stuck.
  2105. */
  2106. #define MEMCG_MAX_HIGH_DELAY_JIFFIES (2UL*HZ)
  2107. /*
  2108. * When calculating the delay, we use these either side of the exponentiation to
  2109. * maintain precision and scale to a reasonable number of jiffies (see the table
  2110. * below.
  2111. *
  2112. * - MEMCG_DELAY_PRECISION_SHIFT: Extra precision bits while translating the
  2113. * overage ratio to a delay.
  2114. * - MEMCG_DELAY_SCALING_SHIFT: The number of bits to scale down the
  2115. * proposed penalty in order to reduce to a reasonable number of jiffies, and
  2116. * to produce a reasonable delay curve.
  2117. *
  2118. * MEMCG_DELAY_SCALING_SHIFT just happens to be a number that produces a
  2119. * reasonable delay curve compared to precision-adjusted overage, not
  2120. * penalising heavily at first, but still making sure that growth beyond the
  2121. * limit penalises misbehaviour cgroups by slowing them down exponentially. For
  2122. * example, with a high of 100 megabytes:
  2123. *
  2124. * +-------+------------------------+
  2125. * | usage | time to allocate in ms |
  2126. * +-------+------------------------+
  2127. * | 100M | 0 |
  2128. * | 101M | 6 |
  2129. * | 102M | 25 |
  2130. * | 103M | 57 |
  2131. * | 104M | 102 |
  2132. * | 105M | 159 |
  2133. * | 106M | 230 |
  2134. * | 107M | 313 |
  2135. * | 108M | 409 |
  2136. * | 109M | 518 |
  2137. * | 110M | 639 |
  2138. * | 111M | 774 |
  2139. * | 112M | 921 |
  2140. * | 113M | 1081 |
  2141. * | 114M | 1254 |
  2142. * | 115M | 1439 |
  2143. * | 116M | 1638 |
  2144. * | 117M | 1849 |
  2145. * | 118M | 2000 |
  2146. * | 119M | 2000 |
  2147. * | 120M | 2000 |
  2148. * +-------+------------------------+
  2149. */
  2150. #define MEMCG_DELAY_PRECISION_SHIFT 20
  2151. #define MEMCG_DELAY_SCALING_SHIFT 14
  2152. static u64 calculate_overage(unsigned long usage, unsigned long high)
  2153. {
  2154. u64 overage;
  2155. if (usage <= high)
  2156. return 0;
  2157. /*
  2158. * Prevent division by 0 in overage calculation by acting as if
  2159. * it was a threshold of 1 page
  2160. */
  2161. high = max(high, 1UL);
  2162. overage = usage - high;
  2163. overage <<= MEMCG_DELAY_PRECISION_SHIFT;
  2164. return div64_u64(overage, high);
  2165. }
  2166. static u64 mem_find_max_overage(struct mem_cgroup *memcg)
  2167. {
  2168. u64 overage, max_overage = 0;
  2169. do {
  2170. overage = calculate_overage(page_counter_read(&memcg->memory),
  2171. READ_ONCE(memcg->memory.high));
  2172. max_overage = max(overage, max_overage);
  2173. } while ((memcg = parent_mem_cgroup(memcg)) &&
  2174. !mem_cgroup_is_root(memcg));
  2175. return max_overage;
  2176. }
  2177. static u64 swap_find_max_overage(struct mem_cgroup *memcg)
  2178. {
  2179. u64 overage, max_overage = 0;
  2180. do {
  2181. overage = calculate_overage(page_counter_read(&memcg->swap),
  2182. READ_ONCE(memcg->swap.high));
  2183. if (overage)
  2184. memcg_memory_event(memcg, MEMCG_SWAP_HIGH);
  2185. max_overage = max(overage, max_overage);
  2186. } while ((memcg = parent_mem_cgroup(memcg)) &&
  2187. !mem_cgroup_is_root(memcg));
  2188. return max_overage;
  2189. }
  2190. /*
  2191. * Get the number of jiffies that we should penalise a mischievous cgroup which
  2192. * is exceeding its memory.high by checking both it and its ancestors.
  2193. */
  2194. static unsigned long calculate_high_delay(struct mem_cgroup *memcg,
  2195. unsigned int nr_pages,
  2196. u64 max_overage)
  2197. {
  2198. unsigned long penalty_jiffies;
  2199. if (!max_overage)
  2200. return 0;
  2201. /*
  2202. * We use overage compared to memory.high to calculate the number of
  2203. * jiffies to sleep (penalty_jiffies). Ideally this value should be
  2204. * fairly lenient on small overages, and increasingly harsh when the
  2205. * memcg in question makes it clear that it has no intention of stopping
  2206. * its crazy behaviour, so we exponentially increase the delay based on
  2207. * overage amount.
  2208. */
  2209. penalty_jiffies = max_overage * max_overage * HZ;
  2210. penalty_jiffies >>= MEMCG_DELAY_PRECISION_SHIFT;
  2211. penalty_jiffies >>= MEMCG_DELAY_SCALING_SHIFT;
  2212. /*
  2213. * Factor in the task's own contribution to the overage, such that four
  2214. * N-sized allocations are throttled approximately the same as one
  2215. * 4N-sized allocation.
  2216. *
  2217. * MEMCG_CHARGE_BATCH pages is nominal, so work out how much smaller or
  2218. * larger the current charge patch is than that.
  2219. */
  2220. return penalty_jiffies * nr_pages / MEMCG_CHARGE_BATCH;
  2221. }
  2222. /*
  2223. * Scheduled by try_charge() to be executed from the userland return path
  2224. * and reclaims memory over the high limit.
  2225. */
  2226. void mem_cgroup_handle_over_high(void)
  2227. {
  2228. unsigned long penalty_jiffies;
  2229. unsigned long pflags;
  2230. unsigned long nr_reclaimed;
  2231. unsigned int nr_pages = current->memcg_nr_pages_over_high;
  2232. int nr_retries = MAX_RECLAIM_RETRIES;
  2233. struct mem_cgroup *memcg;
  2234. bool in_retry = false;
  2235. if (likely(!nr_pages))
  2236. return;
  2237. memcg = get_mem_cgroup_from_mm(current->mm);
  2238. current->memcg_nr_pages_over_high = 0;
  2239. retry_reclaim:
  2240. /*
  2241. * The allocating task should reclaim at least the batch size, but for
  2242. * subsequent retries we only want to do what's necessary to prevent oom
  2243. * or breaching resource isolation.
  2244. *
  2245. * This is distinct from memory.max or page allocator behaviour because
  2246. * memory.high is currently batched, whereas memory.max and the page
  2247. * allocator run every time an allocation is made.
  2248. */
  2249. nr_reclaimed = reclaim_high(memcg,
  2250. in_retry ? SWAP_CLUSTER_MAX : nr_pages,
  2251. GFP_KERNEL);
  2252. /*
  2253. * memory.high is breached and reclaim is unable to keep up. Throttle
  2254. * allocators proactively to slow down excessive growth.
  2255. */
  2256. penalty_jiffies = calculate_high_delay(memcg, nr_pages,
  2257. mem_find_max_overage(memcg));
  2258. penalty_jiffies += calculate_high_delay(memcg, nr_pages,
  2259. swap_find_max_overage(memcg));
  2260. /*
  2261. * Clamp the max delay per usermode return so as to still keep the
  2262. * application moving forwards and also permit diagnostics, albeit
  2263. * extremely slowly.
  2264. */
  2265. penalty_jiffies = min(penalty_jiffies, MEMCG_MAX_HIGH_DELAY_JIFFIES);
  2266. /*
  2267. * Don't sleep if the amount of jiffies this memcg owes us is so low
  2268. * that it's not even worth doing, in an attempt to be nice to those who
  2269. * go only a small amount over their memory.high value and maybe haven't
  2270. * been aggressively reclaimed enough yet.
  2271. */
  2272. if (penalty_jiffies <= HZ / 100)
  2273. goto out;
  2274. /*
  2275. * If reclaim is making forward progress but we're still over
  2276. * memory.high, we want to encourage that rather than doing allocator
  2277. * throttling.
  2278. */
  2279. if (nr_reclaimed || nr_retries--) {
  2280. in_retry = true;
  2281. goto retry_reclaim;
  2282. }
  2283. /*
  2284. * If we exit early, we're guaranteed to die (since
  2285. * schedule_timeout_killable sets TASK_KILLABLE). This means we don't
  2286. * need to account for any ill-begotten jiffies to pay them off later.
  2287. */
  2288. psi_memstall_enter(&pflags);
  2289. schedule_timeout_killable(penalty_jiffies);
  2290. psi_memstall_leave(&pflags);
  2291. out:
  2292. css_put(&memcg->css);
  2293. }
  2294. static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
  2295. unsigned int nr_pages)
  2296. {
  2297. unsigned int batch = max(MEMCG_CHARGE_BATCH, nr_pages);
  2298. int nr_retries = MAX_RECLAIM_RETRIES;
  2299. struct mem_cgroup *mem_over_limit;
  2300. struct page_counter *counter;
  2301. enum oom_status oom_status;
  2302. unsigned long nr_reclaimed;
  2303. bool passed_oom = false;
  2304. bool may_swap = true;
  2305. bool drained = false;
  2306. unsigned long pflags;
  2307. if (mem_cgroup_is_root(memcg))
  2308. return 0;
  2309. retry:
  2310. if (consume_stock(memcg, nr_pages))
  2311. return 0;
  2312. if (!do_memsw_account() ||
  2313. page_counter_try_charge(&memcg->memsw, batch, &counter)) {
  2314. if (page_counter_try_charge(&memcg->memory, batch, &counter))
  2315. goto done_restock;
  2316. if (do_memsw_account())
  2317. page_counter_uncharge(&memcg->memsw, batch);
  2318. mem_over_limit = mem_cgroup_from_counter(counter, memory);
  2319. } else {
  2320. mem_over_limit = mem_cgroup_from_counter(counter, memsw);
  2321. may_swap = false;
  2322. }
  2323. if (batch > nr_pages) {
  2324. batch = nr_pages;
  2325. goto retry;
  2326. }
  2327. /*
  2328. * Memcg doesn't have a dedicated reserve for atomic
  2329. * allocations. But like the global atomic pool, we need to
  2330. * put the burden of reclaim on regular allocation requests
  2331. * and let these go through as privileged allocations.
  2332. */
  2333. if (gfp_mask & __GFP_ATOMIC)
  2334. goto force;
  2335. /*
  2336. * Prevent unbounded recursion when reclaim operations need to
  2337. * allocate memory. This might exceed the limits temporarily,
  2338. * but we prefer facilitating memory reclaim and getting back
  2339. * under the limit over triggering OOM kills in these cases.
  2340. */
  2341. if (unlikely(current->flags & PF_MEMALLOC))
  2342. goto force;
  2343. if (unlikely(task_in_memcg_oom(current)))
  2344. goto nomem;
  2345. if (!gfpflags_allow_blocking(gfp_mask))
  2346. goto nomem;
  2347. memcg_memory_event(mem_over_limit, MEMCG_MAX);
  2348. psi_memstall_enter(&pflags);
  2349. nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages,
  2350. gfp_mask, may_swap);
  2351. psi_memstall_leave(&pflags);
  2352. if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
  2353. goto retry;
  2354. if (!drained) {
  2355. drain_all_stock(mem_over_limit);
  2356. drained = true;
  2357. goto retry;
  2358. }
  2359. if (gfp_mask & __GFP_NORETRY)
  2360. goto nomem;
  2361. /*
  2362. * Even though the limit is exceeded at this point, reclaim
  2363. * may have been able to free some pages. Retry the charge
  2364. * before killing the task.
  2365. *
  2366. * Only for regular pages, though: huge pages are rather
  2367. * unlikely to succeed so close to the limit, and we fall back
  2368. * to regular pages anyway in case of failure.
  2369. */
  2370. if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER))
  2371. goto retry;
  2372. /*
  2373. * At task move, charge accounts can be doubly counted. So, it's
  2374. * better to wait until the end of task_move if something is going on.
  2375. */
  2376. if (mem_cgroup_wait_acct_move(mem_over_limit))
  2377. goto retry;
  2378. if (nr_retries--)
  2379. goto retry;
  2380. if (gfp_mask & __GFP_RETRY_MAYFAIL)
  2381. goto nomem;
  2382. if (gfp_mask & __GFP_NOFAIL)
  2383. goto force;
  2384. /* Avoid endless loop for tasks bypassed by the oom killer */
  2385. if (passed_oom && task_is_dying())
  2386. goto nomem;
  2387. /*
  2388. * keep retrying as long as the memcg oom killer is able to make
  2389. * a forward progress or bypass the charge if the oom killer
  2390. * couldn't make any progress.
  2391. */
  2392. oom_status = mem_cgroup_oom(mem_over_limit, gfp_mask,
  2393. get_order(nr_pages * PAGE_SIZE));
  2394. if (oom_status == OOM_SUCCESS) {
  2395. passed_oom = true;
  2396. nr_retries = MAX_RECLAIM_RETRIES;
  2397. goto retry;
  2398. }
  2399. nomem:
  2400. if (!(gfp_mask & __GFP_NOFAIL))
  2401. return -ENOMEM;
  2402. force:
  2403. /*
  2404. * The allocation either can't fail or will lead to more memory
  2405. * being freed very soon. Allow memory usage go over the limit
  2406. * temporarily by force charging it.
  2407. */
  2408. page_counter_charge(&memcg->memory, nr_pages);
  2409. if (do_memsw_account())
  2410. page_counter_charge(&memcg->memsw, nr_pages);
  2411. return 0;
  2412. done_restock:
  2413. if (batch > nr_pages)
  2414. refill_stock(memcg, batch - nr_pages);
  2415. /*
  2416. * If the hierarchy is above the normal consumption range, schedule
  2417. * reclaim on returning to userland. We can perform reclaim here
  2418. * if __GFP_RECLAIM but let's always punt for simplicity and so that
  2419. * GFP_KERNEL can consistently be used during reclaim. @memcg is
  2420. * not recorded as it most likely matches current's and won't
  2421. * change in the meantime. As high limit is checked again before
  2422. * reclaim, the cost of mismatch is negligible.
  2423. */
  2424. do {
  2425. bool mem_high, swap_high;
  2426. mem_high = page_counter_read(&memcg->memory) >
  2427. READ_ONCE(memcg->memory.high);
  2428. swap_high = page_counter_read(&memcg->swap) >
  2429. READ_ONCE(memcg->swap.high);
  2430. /* Don't bother a random interrupted task */
  2431. if (in_interrupt()) {
  2432. if (mem_high) {
  2433. schedule_work(&memcg->high_work);
  2434. break;
  2435. }
  2436. continue;
  2437. }
  2438. if (mem_high || swap_high) {
  2439. /*
  2440. * The allocating tasks in this cgroup will need to do
  2441. * reclaim or be throttled to prevent further growth
  2442. * of the memory or swap footprints.
  2443. *
  2444. * Target some best-effort fairness between the tasks,
  2445. * and distribute reclaim work and delay penalties
  2446. * based on how much each task is actually allocating.
  2447. */
  2448. current->memcg_nr_pages_over_high += batch;
  2449. set_notify_resume(current);
  2450. break;
  2451. }
  2452. } while ((memcg = parent_mem_cgroup(memcg)));
  2453. return 0;
  2454. }
  2455. #if defined(CONFIG_MEMCG_KMEM) || defined(CONFIG_MMU)
  2456. static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
  2457. {
  2458. if (mem_cgroup_is_root(memcg))
  2459. return;
  2460. page_counter_uncharge(&memcg->memory, nr_pages);
  2461. if (do_memsw_account())
  2462. page_counter_uncharge(&memcg->memsw, nr_pages);
  2463. }
  2464. #endif
  2465. static void commit_charge(struct page *page, struct mem_cgroup *memcg)
  2466. {
  2467. VM_BUG_ON_PAGE(page->mem_cgroup, page);
  2468. /*
  2469. * Any of the following ensures page->mem_cgroup stability:
  2470. *
  2471. * - the page lock
  2472. * - LRU isolation
  2473. * - lock_page_memcg()
  2474. * - exclusive reference
  2475. */
  2476. page->mem_cgroup = memcg;
  2477. }
  2478. #ifdef CONFIG_MEMCG_KMEM
  2479. /*
  2480. * The allocated objcg pointers array is not accounted directly.
  2481. * Moreover, it should not come from DMA buffer and is not readily
  2482. * reclaimable. So those GFP bits should be masked off.
  2483. */
  2484. #define OBJCGS_CLEAR_MASK (__GFP_DMA | __GFP_RECLAIMABLE | __GFP_ACCOUNT)
  2485. int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
  2486. gfp_t gfp)
  2487. {
  2488. unsigned int objects = objs_per_slab_page(s, page);
  2489. void *vec;
  2490. gfp &= ~OBJCGS_CLEAR_MASK;
  2491. vec = kcalloc_node(objects, sizeof(struct obj_cgroup *), gfp,
  2492. page_to_nid(page));
  2493. if (!vec)
  2494. return -ENOMEM;
  2495. if (cmpxchg(&page->obj_cgroups, NULL,
  2496. (struct obj_cgroup **) ((unsigned long)vec | 0x1UL)))
  2497. kfree(vec);
  2498. else
  2499. kmemleak_not_leak(vec);
  2500. return 0;
  2501. }
  2502. /*
  2503. * Returns a pointer to the memory cgroup to which the kernel object is charged.
  2504. *
  2505. * The caller must ensure the memcg lifetime, e.g. by taking rcu_read_lock(),
  2506. * cgroup_mutex, etc.
  2507. */
  2508. struct mem_cgroup *mem_cgroup_from_obj(void *p)
  2509. {
  2510. struct page *page;
  2511. if (mem_cgroup_disabled())
  2512. return NULL;
  2513. page = virt_to_head_page(p);
  2514. /*
  2515. * If page->mem_cgroup is set, it's either a simple mem_cgroup pointer
  2516. * or a pointer to obj_cgroup vector. In the latter case the lowest
  2517. * bit of the pointer is set.
  2518. * The page->mem_cgroup pointer can be asynchronously changed
  2519. * from NULL to (obj_cgroup_vec | 0x1UL), but can't be changed
  2520. * from a valid memcg pointer to objcg vector or back.
  2521. */
  2522. if (!page->mem_cgroup)
  2523. return NULL;
  2524. /*
  2525. * Slab objects are accounted individually, not per-page.
  2526. * Memcg membership data for each individual object is saved in
  2527. * the page->obj_cgroups.
  2528. */
  2529. if (page_has_obj_cgroups(page)) {
  2530. struct obj_cgroup *objcg;
  2531. unsigned int off;
  2532. off = obj_to_index(page->slab_cache, page, p);
  2533. objcg = page_obj_cgroups(page)[off];
  2534. if (objcg)
  2535. return obj_cgroup_memcg(objcg);
  2536. return NULL;
  2537. }
  2538. /* All other pages use page->mem_cgroup */
  2539. return page->mem_cgroup;
  2540. }
  2541. __always_inline struct obj_cgroup *get_obj_cgroup_from_current(void)
  2542. {
  2543. struct obj_cgroup *objcg = NULL;
  2544. struct mem_cgroup *memcg;
  2545. if (memcg_kmem_bypass())
  2546. return NULL;
  2547. rcu_read_lock();
  2548. if (unlikely(active_memcg()))
  2549. memcg = active_memcg();
  2550. else
  2551. memcg = mem_cgroup_from_task(current);
  2552. for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) {
  2553. objcg = rcu_dereference(memcg->objcg);
  2554. if (objcg && obj_cgroup_tryget(objcg))
  2555. break;
  2556. objcg = NULL;
  2557. }
  2558. rcu_read_unlock();
  2559. return objcg;
  2560. }
  2561. static int memcg_alloc_cache_id(void)
  2562. {
  2563. int id, size;
  2564. int err;
  2565. id = ida_simple_get(&memcg_cache_ida,
  2566. 0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
  2567. if (id < 0)
  2568. return id;
  2569. if (id < memcg_nr_cache_ids)
  2570. return id;
  2571. /*
  2572. * There's no space for the new id in memcg_caches arrays,
  2573. * so we have to grow them.
  2574. */
  2575. down_write(&memcg_cache_ids_sem);
  2576. size = 2 * (id + 1);
  2577. if (size < MEMCG_CACHES_MIN_SIZE)
  2578. size = MEMCG_CACHES_MIN_SIZE;
  2579. else if (size > MEMCG_CACHES_MAX_SIZE)
  2580. size = MEMCG_CACHES_MAX_SIZE;
  2581. err = memcg_update_all_list_lrus(size);
  2582. if (!err)
  2583. memcg_nr_cache_ids = size;
  2584. up_write(&memcg_cache_ids_sem);
  2585. if (err) {
  2586. ida_simple_remove(&memcg_cache_ida, id);
  2587. return err;
  2588. }
  2589. return id;
  2590. }
  2591. static void memcg_free_cache_id(int id)
  2592. {
  2593. ida_simple_remove(&memcg_cache_ida, id);
  2594. }
  2595. /**
  2596. * __memcg_kmem_charge: charge a number of kernel pages to a memcg
  2597. * @memcg: memory cgroup to charge
  2598. * @gfp: reclaim mode
  2599. * @nr_pages: number of pages to charge
  2600. *
  2601. * Returns 0 on success, an error code on failure.
  2602. */
  2603. int __memcg_kmem_charge(struct mem_cgroup *memcg, gfp_t gfp,
  2604. unsigned int nr_pages)
  2605. {
  2606. struct page_counter *counter;
  2607. int ret;
  2608. ret = try_charge(memcg, gfp, nr_pages);
  2609. if (ret)
  2610. return ret;
  2611. if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) &&
  2612. !page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) {
  2613. /*
  2614. * Enforce __GFP_NOFAIL allocation because callers are not
  2615. * prepared to see failures and likely do not have any failure
  2616. * handling code.
  2617. */
  2618. if (gfp & __GFP_NOFAIL) {
  2619. page_counter_charge(&memcg->kmem, nr_pages);
  2620. return 0;
  2621. }
  2622. cancel_charge(memcg, nr_pages);
  2623. return -ENOMEM;
  2624. }
  2625. return 0;
  2626. }
  2627. /**
  2628. * __memcg_kmem_uncharge: uncharge a number of kernel pages from a memcg
  2629. * @memcg: memcg to uncharge
  2630. * @nr_pages: number of pages to uncharge
  2631. */
  2632. void __memcg_kmem_uncharge(struct mem_cgroup *memcg, unsigned int nr_pages)
  2633. {
  2634. if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
  2635. page_counter_uncharge(&memcg->kmem, nr_pages);
  2636. refill_stock(memcg, nr_pages);
  2637. }
  2638. /**
  2639. * __memcg_kmem_charge_page: charge a kmem page to the current memory cgroup
  2640. * @page: page to charge
  2641. * @gfp: reclaim mode
  2642. * @order: allocation order
  2643. *
  2644. * Returns 0 on success, an error code on failure.
  2645. */
  2646. int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order)
  2647. {
  2648. struct mem_cgroup *memcg;
  2649. int ret = 0;
  2650. memcg = get_mem_cgroup_from_current();
  2651. if (memcg && !mem_cgroup_is_root(memcg)) {
  2652. ret = __memcg_kmem_charge(memcg, gfp, 1 << order);
  2653. if (!ret) {
  2654. page->mem_cgroup = memcg;
  2655. __SetPageKmemcg(page);
  2656. return 0;
  2657. }
  2658. css_put(&memcg->css);
  2659. }
  2660. return ret;
  2661. }
  2662. /**
  2663. * __memcg_kmem_uncharge_page: uncharge a kmem page
  2664. * @page: page to uncharge
  2665. * @order: allocation order
  2666. */
  2667. void __memcg_kmem_uncharge_page(struct page *page, int order)
  2668. {
  2669. struct mem_cgroup *memcg = page->mem_cgroup;
  2670. unsigned int nr_pages = 1 << order;
  2671. if (!memcg)
  2672. return;
  2673. VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page);
  2674. __memcg_kmem_uncharge(memcg, nr_pages);
  2675. page->mem_cgroup = NULL;
  2676. css_put(&memcg->css);
  2677. /* slab pages do not have PageKmemcg flag set */
  2678. if (PageKmemcg(page))
  2679. __ClearPageKmemcg(page);
  2680. }
  2681. static bool consume_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes)
  2682. {
  2683. struct memcg_stock_pcp *stock;
  2684. unsigned long flags;
  2685. bool ret = false;
  2686. local_irq_save(flags);
  2687. stock = this_cpu_ptr(&memcg_stock);
  2688. if (objcg == stock->cached_objcg && stock->nr_bytes >= nr_bytes) {
  2689. stock->nr_bytes -= nr_bytes;
  2690. ret = true;
  2691. }
  2692. local_irq_restore(flags);
  2693. return ret;
  2694. }
  2695. static void drain_obj_stock(struct memcg_stock_pcp *stock)
  2696. {
  2697. struct obj_cgroup *old = stock->cached_objcg;
  2698. if (!old)
  2699. return;
  2700. if (stock->nr_bytes) {
  2701. unsigned int nr_pages = stock->nr_bytes >> PAGE_SHIFT;
  2702. unsigned int nr_bytes = stock->nr_bytes & (PAGE_SIZE - 1);
  2703. if (nr_pages) {
  2704. struct mem_cgroup *memcg;
  2705. rcu_read_lock();
  2706. retry:
  2707. memcg = obj_cgroup_memcg(old);
  2708. if (unlikely(!css_tryget(&memcg->css)))
  2709. goto retry;
  2710. rcu_read_unlock();
  2711. __memcg_kmem_uncharge(memcg, nr_pages);
  2712. css_put(&memcg->css);
  2713. }
  2714. /*
  2715. * The leftover is flushed to the centralized per-memcg value.
  2716. * On the next attempt to refill obj stock it will be moved
  2717. * to a per-cpu stock (probably, on an other CPU), see
  2718. * refill_obj_stock().
  2719. *
  2720. * How often it's flushed is a trade-off between the memory
  2721. * limit enforcement accuracy and potential CPU contention,
  2722. * so it might be changed in the future.
  2723. */
  2724. atomic_add(nr_bytes, &old->nr_charged_bytes);
  2725. stock->nr_bytes = 0;
  2726. }
  2727. obj_cgroup_put(old);
  2728. stock->cached_objcg = NULL;
  2729. }
  2730. static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
  2731. struct mem_cgroup *root_memcg)
  2732. {
  2733. struct mem_cgroup *memcg;
  2734. if (stock->cached_objcg) {
  2735. memcg = obj_cgroup_memcg(stock->cached_objcg);
  2736. if (memcg && mem_cgroup_is_descendant(memcg, root_memcg))
  2737. return true;
  2738. }
  2739. return false;
  2740. }
  2741. static void refill_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes)
  2742. {
  2743. struct memcg_stock_pcp *stock;
  2744. unsigned long flags;
  2745. local_irq_save(flags);
  2746. stock = this_cpu_ptr(&memcg_stock);
  2747. if (stock->cached_objcg != objcg) { /* reset if necessary */
  2748. drain_obj_stock(stock);
  2749. obj_cgroup_get(objcg);
  2750. stock->cached_objcg = objcg;
  2751. stock->nr_bytes = atomic_xchg(&objcg->nr_charged_bytes, 0);
  2752. }
  2753. stock->nr_bytes += nr_bytes;
  2754. if (stock->nr_bytes > PAGE_SIZE)
  2755. drain_obj_stock(stock);
  2756. local_irq_restore(flags);
  2757. }
  2758. int obj_cgroup_charge(struct obj_cgroup *objcg, gfp_t gfp, size_t size)
  2759. {
  2760. struct mem_cgroup *memcg;
  2761. unsigned int nr_pages, nr_bytes;
  2762. int ret;
  2763. if (consume_obj_stock(objcg, size))
  2764. return 0;
  2765. /*
  2766. * In theory, memcg->nr_charged_bytes can have enough
  2767. * pre-charged bytes to satisfy the allocation. However,
  2768. * flushing memcg->nr_charged_bytes requires two atomic
  2769. * operations, and memcg->nr_charged_bytes can't be big,
  2770. * so it's better to ignore it and try grab some new pages.
  2771. * memcg->nr_charged_bytes will be flushed in
  2772. * refill_obj_stock(), called from this function or
  2773. * independently later.
  2774. */
  2775. rcu_read_lock();
  2776. retry:
  2777. memcg = obj_cgroup_memcg(objcg);
  2778. if (unlikely(!css_tryget(&memcg->css)))
  2779. goto retry;
  2780. rcu_read_unlock();
  2781. nr_pages = size >> PAGE_SHIFT;
  2782. nr_bytes = size & (PAGE_SIZE - 1);
  2783. if (nr_bytes)
  2784. nr_pages += 1;
  2785. ret = __memcg_kmem_charge(memcg, gfp, nr_pages);
  2786. if (!ret && nr_bytes)
  2787. refill_obj_stock(objcg, PAGE_SIZE - nr_bytes);
  2788. css_put(&memcg->css);
  2789. return ret;
  2790. }
  2791. void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size)
  2792. {
  2793. refill_obj_stock(objcg, size);
  2794. }
  2795. #endif /* CONFIG_MEMCG_KMEM */
  2796. /*
  2797. * Because head->mem_cgroup is not set on tails, set it now.
  2798. */
  2799. void split_page_memcg(struct page *head, unsigned int nr)
  2800. {
  2801. struct mem_cgroup *memcg = head->mem_cgroup;
  2802. int kmemcg = PageKmemcg(head);
  2803. int i;
  2804. if (mem_cgroup_disabled() || !memcg)
  2805. return;
  2806. for (i = 1; i < nr; i++) {
  2807. head[i].mem_cgroup = memcg;
  2808. if (kmemcg)
  2809. __SetPageKmemcg(head + i);
  2810. }
  2811. css_get_many(&memcg->css, nr - 1);
  2812. }
  2813. #ifdef CONFIG_MEMCG_SWAP
  2814. /**
  2815. * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
  2816. * @entry: swap entry to be moved
  2817. * @from: mem_cgroup which the entry is moved from
  2818. * @to: mem_cgroup which the entry is moved to
  2819. *
  2820. * It succeeds only when the swap_cgroup's record for this entry is the same
  2821. * as the mem_cgroup's id of @from.
  2822. *
  2823. * Returns 0 on success, -EINVAL on failure.
  2824. *
  2825. * The caller must have charged to @to, IOW, called page_counter_charge() about
  2826. * both res and memsw, and called css_get().
  2827. */
  2828. static int mem_cgroup_move_swap_account(swp_entry_t entry,
  2829. struct mem_cgroup *from, struct mem_cgroup *to)
  2830. {
  2831. unsigned short old_id, new_id;
  2832. old_id = mem_cgroup_id(from);
  2833. new_id = mem_cgroup_id(to);
  2834. if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
  2835. mod_memcg_state(from, MEMCG_SWAP, -1);
  2836. mod_memcg_state(to, MEMCG_SWAP, 1);
  2837. return 0;
  2838. }
  2839. return -EINVAL;
  2840. }
  2841. #else
  2842. static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
  2843. struct mem_cgroup *from, struct mem_cgroup *to)
  2844. {
  2845. return -EINVAL;
  2846. }
  2847. #endif
  2848. static DEFINE_MUTEX(memcg_max_mutex);
  2849. static int mem_cgroup_resize_max(struct mem_cgroup *memcg,
  2850. unsigned long max, bool memsw)
  2851. {
  2852. bool enlarge = false;
  2853. bool drained = false;
  2854. int ret;
  2855. bool limits_invariant;
  2856. struct page_counter *counter = memsw ? &memcg->memsw : &memcg->memory;
  2857. do {
  2858. if (signal_pending(current)) {
  2859. ret = -EINTR;
  2860. break;
  2861. }
  2862. mutex_lock(&memcg_max_mutex);
  2863. /*
  2864. * Make sure that the new limit (memsw or memory limit) doesn't
  2865. * break our basic invariant rule memory.max <= memsw.max.
  2866. */
  2867. limits_invariant = memsw ? max >= READ_ONCE(memcg->memory.max) :
  2868. max <= memcg->memsw.max;
  2869. if (!limits_invariant) {
  2870. mutex_unlock(&memcg_max_mutex);
  2871. ret = -EINVAL;
  2872. break;
  2873. }
  2874. if (max > counter->max)
  2875. enlarge = true;
  2876. ret = page_counter_set_max(counter, max);
  2877. mutex_unlock(&memcg_max_mutex);
  2878. if (!ret)
  2879. break;
  2880. if (!drained) {
  2881. drain_all_stock(memcg);
  2882. drained = true;
  2883. continue;
  2884. }
  2885. if (!try_to_free_mem_cgroup_pages(memcg, 1,
  2886. GFP_KERNEL, !memsw)) {
  2887. ret = -EBUSY;
  2888. break;
  2889. }
  2890. } while (true);
  2891. if (!ret && enlarge)
  2892. memcg_oom_recover(memcg);
  2893. return ret;
  2894. }
  2895. unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
  2896. gfp_t gfp_mask,
  2897. unsigned long *total_scanned)
  2898. {
  2899. unsigned long nr_reclaimed = 0;
  2900. struct mem_cgroup_per_node *mz, *next_mz = NULL;
  2901. unsigned long reclaimed;
  2902. int loop = 0;
  2903. struct mem_cgroup_tree_per_node *mctz;
  2904. unsigned long excess;
  2905. unsigned long nr_scanned;
  2906. if (order > 0)
  2907. return 0;
  2908. mctz = soft_limit_tree_node(pgdat->node_id);
  2909. /*
  2910. * Do not even bother to check the largest node if the root
  2911. * is empty. Do it lockless to prevent lock bouncing. Races
  2912. * are acceptable as soft limit is best effort anyway.
  2913. */
  2914. if (!mctz || RB_EMPTY_ROOT(&mctz->rb_root))
  2915. return 0;
  2916. /*
  2917. * This loop can run a while, specially if mem_cgroup's continuously
  2918. * keep exceeding their soft limit and putting the system under
  2919. * pressure
  2920. */
  2921. do {
  2922. if (next_mz)
  2923. mz = next_mz;
  2924. else
  2925. mz = mem_cgroup_largest_soft_limit_node(mctz);
  2926. if (!mz)
  2927. break;
  2928. nr_scanned = 0;
  2929. reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat,
  2930. gfp_mask, &nr_scanned);
  2931. nr_reclaimed += reclaimed;
  2932. *total_scanned += nr_scanned;
  2933. spin_lock_irq(&mctz->lock);
  2934. __mem_cgroup_remove_exceeded(mz, mctz);
  2935. /*
  2936. * If we failed to reclaim anything from this memory cgroup
  2937. * it is time to move on to the next cgroup
  2938. */
  2939. next_mz = NULL;
  2940. if (!reclaimed)
  2941. next_mz = __mem_cgroup_largest_soft_limit_node(mctz);
  2942. excess = soft_limit_excess(mz->memcg);
  2943. /*
  2944. * One school of thought says that we should not add
  2945. * back the node to the tree if reclaim returns 0.
  2946. * But our reclaim could return 0, simply because due
  2947. * to priority we are exposing a smaller subset of
  2948. * memory to reclaim from. Consider this as a longer
  2949. * term TODO.
  2950. */
  2951. /* If excess == 0, no tree ops */
  2952. __mem_cgroup_insert_exceeded(mz, mctz, excess);
  2953. spin_unlock_irq(&mctz->lock);
  2954. css_put(&mz->memcg->css);
  2955. loop++;
  2956. /*
  2957. * Could not reclaim anything and there are no more
  2958. * mem cgroups to try or we seem to be looping without
  2959. * reclaiming anything.
  2960. */
  2961. if (!nr_reclaimed &&
  2962. (next_mz == NULL ||
  2963. loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
  2964. break;
  2965. } while (!nr_reclaimed);
  2966. if (next_mz)
  2967. css_put(&next_mz->memcg->css);
  2968. return nr_reclaimed;
  2969. }
  2970. /*
  2971. * Test whether @memcg has children, dead or alive. Note that this
  2972. * function doesn't care whether @memcg has use_hierarchy enabled and
  2973. * returns %true if there are child csses according to the cgroup
  2974. * hierarchy. Testing use_hierarchy is the caller's responsibility.
  2975. */
  2976. static inline bool memcg_has_children(struct mem_cgroup *memcg)
  2977. {
  2978. bool ret;
  2979. rcu_read_lock();
  2980. ret = css_next_child(NULL, &memcg->css);
  2981. rcu_read_unlock();
  2982. return ret;
  2983. }
  2984. /*
  2985. * Reclaims as many pages from the given memcg as possible.
  2986. *
  2987. * Caller is responsible for holding css reference for memcg.
  2988. */
  2989. static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
  2990. {
  2991. int nr_retries = MAX_RECLAIM_RETRIES;
  2992. /* we call try-to-free pages for make this cgroup empty */
  2993. lru_add_drain_all();
  2994. drain_all_stock(memcg);
  2995. /* try to free all pages in this cgroup */
  2996. while (nr_retries && page_counter_read(&memcg->memory)) {
  2997. int progress;
  2998. if (signal_pending(current))
  2999. return -EINTR;
  3000. progress = try_to_free_mem_cgroup_pages(memcg, 1,
  3001. GFP_KERNEL, true);
  3002. if (!progress) {
  3003. nr_retries--;
  3004. /* maybe some writeback is necessary */
  3005. congestion_wait(BLK_RW_ASYNC, HZ/10);
  3006. }
  3007. }
  3008. return 0;
  3009. }
  3010. static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of,
  3011. char *buf, size_t nbytes,
  3012. loff_t off)
  3013. {
  3014. struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
  3015. if (mem_cgroup_is_root(memcg))
  3016. return -EINVAL;
  3017. return mem_cgroup_force_empty(memcg) ?: nbytes;
  3018. }
  3019. static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css,
  3020. struct cftype *cft)
  3021. {
  3022. return mem_cgroup_from_css(css)->use_hierarchy;
  3023. }
  3024. static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
  3025. struct cftype *cft, u64 val)
  3026. {
  3027. int retval = 0;
  3028. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  3029. struct mem_cgroup *parent_memcg = mem_cgroup_from_css(memcg->css.parent);
  3030. if (memcg->use_hierarchy == val)
  3031. return 0;
  3032. /*
  3033. * If parent's use_hierarchy is set, we can't make any modifications
  3034. * in the child subtrees. If it is unset, then the change can
  3035. * occur, provided the current cgroup has no children.
  3036. *
  3037. * For the root cgroup, parent_mem is NULL, we allow value to be
  3038. * set if there are no children.
  3039. */
  3040. if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
  3041. (val == 1 || val == 0)) {
  3042. if (!memcg_has_children(memcg))
  3043. memcg->use_hierarchy = val;
  3044. else
  3045. retval = -EBUSY;
  3046. } else
  3047. retval = -EINVAL;
  3048. return retval;
  3049. }
  3050. static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
  3051. {
  3052. unsigned long val;
  3053. if (mem_cgroup_is_root(memcg)) {
  3054. val = memcg_page_state(memcg, NR_FILE_PAGES) +
  3055. memcg_page_state(memcg, NR_ANON_MAPPED);
  3056. if (swap)
  3057. val += memcg_page_state(memcg, MEMCG_SWAP);
  3058. } else {
  3059. if (!swap)
  3060. val = page_counter_read(&memcg->memory);
  3061. else
  3062. val = page_counter_read(&memcg->memsw);
  3063. }
  3064. return val;
  3065. }
  3066. enum {
  3067. RES_USAGE,
  3068. RES_LIMIT,
  3069. RES_MAX_USAGE,
  3070. RES_FAILCNT,
  3071. RES_SOFT_LIMIT,
  3072. };
  3073. static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
  3074. struct cftype *cft)
  3075. {
  3076. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  3077. struct page_counter *counter;
  3078. switch (MEMFILE_TYPE(cft->private)) {
  3079. case _MEM:
  3080. counter = &memcg->memory;
  3081. break;
  3082. case _MEMSWAP:
  3083. counter = &memcg->memsw;
  3084. break;
  3085. case _KMEM:
  3086. counter = &memcg->kmem;
  3087. break;
  3088. case _TCP:
  3089. counter = &memcg->tcpmem;
  3090. break;
  3091. default:
  3092. BUG();
  3093. }
  3094. switch (MEMFILE_ATTR(cft->private)) {
  3095. case RES_USAGE:
  3096. if (counter == &memcg->memory)
  3097. return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE;
  3098. if (counter == &memcg->memsw)
  3099. return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE;
  3100. return (u64)page_counter_read(counter) * PAGE_SIZE;
  3101. case RES_LIMIT:
  3102. return (u64)counter->max * PAGE_SIZE;
  3103. case RES_MAX_USAGE:
  3104. return (u64)counter->watermark * PAGE_SIZE;
  3105. case RES_FAILCNT:
  3106. return counter->failcnt;
  3107. case RES_SOFT_LIMIT:
  3108. return (u64)memcg->soft_limit * PAGE_SIZE;
  3109. default:
  3110. BUG();
  3111. }
  3112. }
  3113. static void memcg_flush_percpu_vmstats(struct mem_cgroup *memcg)
  3114. {
  3115. unsigned long stat[MEMCG_NR_STAT] = {0};
  3116. struct mem_cgroup *mi;
  3117. int node, cpu, i;
  3118. for_each_online_cpu(cpu)
  3119. for (i = 0; i < MEMCG_NR_STAT; i++)
  3120. stat[i] += per_cpu(memcg->vmstats_percpu->stat[i], cpu);
  3121. for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
  3122. for (i = 0; i < MEMCG_NR_STAT; i++)
  3123. atomic_long_add(stat[i], &mi->vmstats[i]);
  3124. for_each_node(node) {
  3125. struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
  3126. struct mem_cgroup_per_node *pi;
  3127. for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
  3128. stat[i] = 0;
  3129. for_each_online_cpu(cpu)
  3130. for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
  3131. stat[i] += per_cpu(
  3132. pn->lruvec_stat_cpu->count[i], cpu);
  3133. for (pi = pn; pi; pi = parent_nodeinfo(pi, node))
  3134. for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
  3135. atomic_long_add(stat[i], &pi->lruvec_stat[i]);
  3136. }
  3137. }
  3138. static void memcg_flush_percpu_vmevents(struct mem_cgroup *memcg)
  3139. {
  3140. unsigned long events[NR_VM_EVENT_ITEMS];
  3141. struct mem_cgroup *mi;
  3142. int cpu, i;
  3143. for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
  3144. events[i] = 0;
  3145. for_each_online_cpu(cpu)
  3146. for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
  3147. events[i] += per_cpu(memcg->vmstats_percpu->events[i],
  3148. cpu);
  3149. for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
  3150. for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
  3151. atomic_long_add(events[i], &mi->vmevents[i]);
  3152. }
  3153. #ifdef CONFIG_MEMCG_KMEM
  3154. static int memcg_online_kmem(struct mem_cgroup *memcg)
  3155. {
  3156. struct obj_cgroup *objcg;
  3157. int memcg_id;
  3158. if (cgroup_memory_nokmem)
  3159. return 0;
  3160. BUG_ON(memcg->kmemcg_id >= 0);
  3161. BUG_ON(memcg->kmem_state);
  3162. memcg_id = memcg_alloc_cache_id();
  3163. if (memcg_id < 0)
  3164. return memcg_id;
  3165. objcg = obj_cgroup_alloc();
  3166. if (!objcg) {
  3167. memcg_free_cache_id(memcg_id);
  3168. return -ENOMEM;
  3169. }
  3170. objcg->memcg = memcg;
  3171. rcu_assign_pointer(memcg->objcg, objcg);
  3172. static_branch_enable(&memcg_kmem_enabled_key);
  3173. /*
  3174. * A memory cgroup is considered kmem-online as soon as it gets
  3175. * kmemcg_id. Setting the id after enabling static branching will
  3176. * guarantee no one starts accounting before all call sites are
  3177. * patched.
  3178. */
  3179. memcg->kmemcg_id = memcg_id;
  3180. memcg->kmem_state = KMEM_ONLINE;
  3181. return 0;
  3182. }
  3183. static void memcg_offline_kmem(struct mem_cgroup *memcg)
  3184. {
  3185. struct cgroup_subsys_state *css;
  3186. struct mem_cgroup *parent, *child;
  3187. int kmemcg_id;
  3188. if (memcg->kmem_state != KMEM_ONLINE)
  3189. return;
  3190. memcg->kmem_state = KMEM_ALLOCATED;
  3191. parent = parent_mem_cgroup(memcg);
  3192. if (!parent)
  3193. parent = root_mem_cgroup;
  3194. memcg_reparent_objcgs(memcg, parent);
  3195. kmemcg_id = memcg->kmemcg_id;
  3196. BUG_ON(kmemcg_id < 0);
  3197. /*
  3198. * Change kmemcg_id of this cgroup and all its descendants to the
  3199. * parent's id, and then move all entries from this cgroup's list_lrus
  3200. * to ones of the parent. After we have finished, all list_lrus
  3201. * corresponding to this cgroup are guaranteed to remain empty. The
  3202. * ordering is imposed by list_lru_node->lock taken by
  3203. * memcg_drain_all_list_lrus().
  3204. */
  3205. rcu_read_lock(); /* can be called from css_free w/o cgroup_mutex */
  3206. css_for_each_descendant_pre(css, &memcg->css) {
  3207. child = mem_cgroup_from_css(css);
  3208. BUG_ON(child->kmemcg_id != kmemcg_id);
  3209. child->kmemcg_id = parent->kmemcg_id;
  3210. if (!memcg->use_hierarchy)
  3211. break;
  3212. }
  3213. rcu_read_unlock();
  3214. memcg_drain_all_list_lrus(kmemcg_id, parent);
  3215. memcg_free_cache_id(kmemcg_id);
  3216. }
  3217. static void memcg_free_kmem(struct mem_cgroup *memcg)
  3218. {
  3219. /* css_alloc() failed, offlining didn't happen */
  3220. if (unlikely(memcg->kmem_state == KMEM_ONLINE))
  3221. memcg_offline_kmem(memcg);
  3222. }
  3223. #else
  3224. static int memcg_online_kmem(struct mem_cgroup *memcg)
  3225. {
  3226. return 0;
  3227. }
  3228. static void memcg_offline_kmem(struct mem_cgroup *memcg)
  3229. {
  3230. }
  3231. static void memcg_free_kmem(struct mem_cgroup *memcg)
  3232. {
  3233. }
  3234. #endif /* CONFIG_MEMCG_KMEM */
  3235. static int memcg_update_kmem_max(struct mem_cgroup *memcg,
  3236. unsigned long max)
  3237. {
  3238. int ret;
  3239. mutex_lock(&memcg_max_mutex);
  3240. ret = page_counter_set_max(&memcg->kmem, max);
  3241. mutex_unlock(&memcg_max_mutex);
  3242. return ret;
  3243. }
  3244. static int memcg_update_tcp_max(struct mem_cgroup *memcg, unsigned long max)
  3245. {
  3246. int ret;
  3247. mutex_lock(&memcg_max_mutex);
  3248. ret = page_counter_set_max(&memcg->tcpmem, max);
  3249. if (ret)
  3250. goto out;
  3251. if (!memcg->tcpmem_active) {
  3252. /*
  3253. * The active flag needs to be written after the static_key
  3254. * update. This is what guarantees that the socket activation
  3255. * function is the last one to run. See mem_cgroup_sk_alloc()
  3256. * for details, and note that we don't mark any socket as
  3257. * belonging to this memcg until that flag is up.
  3258. *
  3259. * We need to do this, because static_keys will span multiple
  3260. * sites, but we can't control their order. If we mark a socket
  3261. * as accounted, but the accounting functions are not patched in
  3262. * yet, we'll lose accounting.
  3263. *
  3264. * We never race with the readers in mem_cgroup_sk_alloc(),
  3265. * because when this value change, the code to process it is not
  3266. * patched in yet.
  3267. */
  3268. static_branch_inc(&memcg_sockets_enabled_key);
  3269. memcg->tcpmem_active = true;
  3270. }
  3271. out:
  3272. mutex_unlock(&memcg_max_mutex);
  3273. return ret;
  3274. }
  3275. /*
  3276. * The user of this function is...
  3277. * RES_LIMIT.
  3278. */
  3279. static ssize_t mem_cgroup_write(struct kernfs_open_file *of,
  3280. char *buf, size_t nbytes, loff_t off)
  3281. {
  3282. struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
  3283. unsigned long nr_pages;
  3284. int ret;
  3285. buf = strstrip(buf);
  3286. ret = page_counter_memparse(buf, "-1", &nr_pages);
  3287. if (ret)
  3288. return ret;
  3289. switch (MEMFILE_ATTR(of_cft(of)->private)) {
  3290. case RES_LIMIT:
  3291. if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
  3292. ret = -EINVAL;
  3293. break;
  3294. }
  3295. switch (MEMFILE_TYPE(of_cft(of)->private)) {
  3296. case _MEM:
  3297. ret = mem_cgroup_resize_max(memcg, nr_pages, false);
  3298. break;
  3299. case _MEMSWAP:
  3300. ret = mem_cgroup_resize_max(memcg, nr_pages, true);
  3301. break;
  3302. case _KMEM:
  3303. pr_warn_once("kmem.limit_in_bytes is deprecated and will be removed. "
  3304. "Please report your usecase to linux-mm@kvack.org if you "
  3305. "depend on this functionality.\n");
  3306. ret = memcg_update_kmem_max(memcg, nr_pages);
  3307. break;
  3308. case _TCP:
  3309. ret = memcg_update_tcp_max(memcg, nr_pages);
  3310. break;
  3311. }
  3312. break;
  3313. case RES_SOFT_LIMIT:
  3314. memcg->soft_limit = nr_pages;
  3315. ret = 0;
  3316. break;
  3317. }
  3318. return ret ?: nbytes;
  3319. }
  3320. static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf,
  3321. size_t nbytes, loff_t off)
  3322. {
  3323. struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
  3324. struct page_counter *counter;
  3325. switch (MEMFILE_TYPE(of_cft(of)->private)) {
  3326. case _MEM:
  3327. counter = &memcg->memory;
  3328. break;
  3329. case _MEMSWAP:
  3330. counter = &memcg->memsw;
  3331. break;
  3332. case _KMEM:
  3333. counter = &memcg->kmem;
  3334. break;
  3335. case _TCP:
  3336. counter = &memcg->tcpmem;
  3337. break;
  3338. default:
  3339. BUG();
  3340. }
  3341. switch (MEMFILE_ATTR(of_cft(of)->private)) {
  3342. case RES_MAX_USAGE:
  3343. page_counter_reset_watermark(counter);
  3344. break;
  3345. case RES_FAILCNT:
  3346. counter->failcnt = 0;
  3347. break;
  3348. default:
  3349. BUG();
  3350. }
  3351. return nbytes;
  3352. }
  3353. static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
  3354. struct cftype *cft)
  3355. {
  3356. return mem_cgroup_from_css(css)->move_charge_at_immigrate;
  3357. }
  3358. #ifdef CONFIG_MMU
  3359. static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
  3360. struct cftype *cft, u64 val)
  3361. {
  3362. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  3363. if (val & ~MOVE_MASK)
  3364. return -EINVAL;
  3365. /*
  3366. * No kind of locking is needed in here, because ->can_attach() will
  3367. * check this value once in the beginning of the process, and then carry
  3368. * on with stale data. This means that changes to this value will only
  3369. * affect task migrations starting after the change.
  3370. */
  3371. memcg->move_charge_at_immigrate = val;
  3372. return 0;
  3373. }
  3374. #else
  3375. static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
  3376. struct cftype *cft, u64 val)
  3377. {
  3378. return -ENOSYS;
  3379. }
  3380. #endif
  3381. #ifdef CONFIG_NUMA
  3382. #define LRU_ALL_FILE (BIT(LRU_INACTIVE_FILE) | BIT(LRU_ACTIVE_FILE))
  3383. #define LRU_ALL_ANON (BIT(LRU_INACTIVE_ANON) | BIT(LRU_ACTIVE_ANON))
  3384. #define LRU_ALL ((1 << NR_LRU_LISTS) - 1)
  3385. static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
  3386. int nid, unsigned int lru_mask, bool tree)
  3387. {
  3388. struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
  3389. unsigned long nr = 0;
  3390. enum lru_list lru;
  3391. VM_BUG_ON((unsigned)nid >= nr_node_ids);
  3392. for_each_lru(lru) {
  3393. if (!(BIT(lru) & lru_mask))
  3394. continue;
  3395. if (tree)
  3396. nr += lruvec_page_state(lruvec, NR_LRU_BASE + lru);
  3397. else
  3398. nr += lruvec_page_state_local(lruvec, NR_LRU_BASE + lru);
  3399. }
  3400. return nr;
  3401. }
  3402. static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
  3403. unsigned int lru_mask,
  3404. bool tree)
  3405. {
  3406. unsigned long nr = 0;
  3407. enum lru_list lru;
  3408. for_each_lru(lru) {
  3409. if (!(BIT(lru) & lru_mask))
  3410. continue;
  3411. if (tree)
  3412. nr += memcg_page_state(memcg, NR_LRU_BASE + lru);
  3413. else
  3414. nr += memcg_page_state_local(memcg, NR_LRU_BASE + lru);
  3415. }
  3416. return nr;
  3417. }
  3418. static int memcg_numa_stat_show(struct seq_file *m, void *v)
  3419. {
  3420. struct numa_stat {
  3421. const char *name;
  3422. unsigned int lru_mask;
  3423. };
  3424. static const struct numa_stat stats[] = {
  3425. { "total", LRU_ALL },
  3426. { "file", LRU_ALL_FILE },
  3427. { "anon", LRU_ALL_ANON },
  3428. { "unevictable", BIT(LRU_UNEVICTABLE) },
  3429. };
  3430. const struct numa_stat *stat;
  3431. int nid;
  3432. struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
  3433. for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
  3434. seq_printf(m, "%s=%lu", stat->name,
  3435. mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
  3436. false));
  3437. for_each_node_state(nid, N_MEMORY)
  3438. seq_printf(m, " N%d=%lu", nid,
  3439. mem_cgroup_node_nr_lru_pages(memcg, nid,
  3440. stat->lru_mask, false));
  3441. seq_putc(m, '\n');
  3442. }
  3443. for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
  3444. seq_printf(m, "hierarchical_%s=%lu", stat->name,
  3445. mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
  3446. true));
  3447. for_each_node_state(nid, N_MEMORY)
  3448. seq_printf(m, " N%d=%lu", nid,
  3449. mem_cgroup_node_nr_lru_pages(memcg, nid,
  3450. stat->lru_mask, true));
  3451. seq_putc(m, '\n');
  3452. }
  3453. return 0;
  3454. }
  3455. #endif /* CONFIG_NUMA */
  3456. static const unsigned int memcg1_stats[] = {
  3457. NR_FILE_PAGES,
  3458. NR_ANON_MAPPED,
  3459. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  3460. NR_ANON_THPS,
  3461. #endif
  3462. NR_SHMEM,
  3463. NR_FILE_MAPPED,
  3464. NR_FILE_DIRTY,
  3465. NR_WRITEBACK,
  3466. MEMCG_SWAP,
  3467. };
  3468. static const char *const memcg1_stat_names[] = {
  3469. "cache",
  3470. "rss",
  3471. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  3472. "rss_huge",
  3473. #endif
  3474. "shmem",
  3475. "mapped_file",
  3476. "dirty",
  3477. "writeback",
  3478. "swap",
  3479. };
  3480. /* Universal VM events cgroup1 shows, original sort order */
  3481. static const unsigned int memcg1_events[] = {
  3482. PGPGIN,
  3483. PGPGOUT,
  3484. PGFAULT,
  3485. PGMAJFAULT,
  3486. };
  3487. static int memcg_stat_show(struct seq_file *m, void *v)
  3488. {
  3489. struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
  3490. unsigned long memory, memsw;
  3491. struct mem_cgroup *mi;
  3492. unsigned int i;
  3493. BUILD_BUG_ON(ARRAY_SIZE(memcg1_stat_names) != ARRAY_SIZE(memcg1_stats));
  3494. for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
  3495. unsigned long nr;
  3496. if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
  3497. continue;
  3498. nr = memcg_page_state_local(memcg, memcg1_stats[i]);
  3499. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  3500. if (memcg1_stats[i] == NR_ANON_THPS)
  3501. nr *= HPAGE_PMD_NR;
  3502. #endif
  3503. seq_printf(m, "%s %lu\n", memcg1_stat_names[i], nr * PAGE_SIZE);
  3504. }
  3505. for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
  3506. seq_printf(m, "%s %lu\n", vm_event_name(memcg1_events[i]),
  3507. memcg_events_local(memcg, memcg1_events[i]));
  3508. for (i = 0; i < NR_LRU_LISTS; i++)
  3509. seq_printf(m, "%s %lu\n", lru_list_name(i),
  3510. memcg_page_state_local(memcg, NR_LRU_BASE + i) *
  3511. PAGE_SIZE);
  3512. /* Hierarchical information */
  3513. memory = memsw = PAGE_COUNTER_MAX;
  3514. for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) {
  3515. memory = min(memory, READ_ONCE(mi->memory.max));
  3516. memsw = min(memsw, READ_ONCE(mi->memsw.max));
  3517. }
  3518. seq_printf(m, "hierarchical_memory_limit %llu\n",
  3519. (u64)memory * PAGE_SIZE);
  3520. if (do_memsw_account())
  3521. seq_printf(m, "hierarchical_memsw_limit %llu\n",
  3522. (u64)memsw * PAGE_SIZE);
  3523. for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
  3524. unsigned long nr;
  3525. if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
  3526. continue;
  3527. nr = memcg_page_state(memcg, memcg1_stats[i]);
  3528. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  3529. if (memcg1_stats[i] == NR_ANON_THPS)
  3530. nr *= HPAGE_PMD_NR;
  3531. #endif
  3532. seq_printf(m, "total_%s %llu\n", memcg1_stat_names[i],
  3533. (u64)nr * PAGE_SIZE);
  3534. }
  3535. for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
  3536. seq_printf(m, "total_%s %llu\n",
  3537. vm_event_name(memcg1_events[i]),
  3538. (u64)memcg_events(memcg, memcg1_events[i]));
  3539. for (i = 0; i < NR_LRU_LISTS; i++)
  3540. seq_printf(m, "total_%s %llu\n", lru_list_name(i),
  3541. (u64)memcg_page_state(memcg, NR_LRU_BASE + i) *
  3542. PAGE_SIZE);
  3543. #ifdef CONFIG_DEBUG_VM
  3544. {
  3545. pg_data_t *pgdat;
  3546. struct mem_cgroup_per_node *mz;
  3547. unsigned long anon_cost = 0;
  3548. unsigned long file_cost = 0;
  3549. for_each_online_pgdat(pgdat) {
  3550. mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
  3551. anon_cost += mz->lruvec.anon_cost;
  3552. file_cost += mz->lruvec.file_cost;
  3553. }
  3554. seq_printf(m, "anon_cost %lu\n", anon_cost);
  3555. seq_printf(m, "file_cost %lu\n", file_cost);
  3556. }
  3557. #endif
  3558. return 0;
  3559. }
  3560. static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css,
  3561. struct cftype *cft)
  3562. {
  3563. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  3564. return mem_cgroup_swappiness(memcg);
  3565. }
  3566. static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css,
  3567. struct cftype *cft, u64 val)
  3568. {
  3569. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  3570. if (val > 100)
  3571. return -EINVAL;
  3572. if (css->parent)
  3573. memcg->swappiness = val;
  3574. else
  3575. vm_swappiness = val;
  3576. return 0;
  3577. }
  3578. static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
  3579. {
  3580. struct mem_cgroup_threshold_ary *t;
  3581. unsigned long usage;
  3582. int i;
  3583. rcu_read_lock();
  3584. if (!swap)
  3585. t = rcu_dereference(memcg->thresholds.primary);
  3586. else
  3587. t = rcu_dereference(memcg->memsw_thresholds.primary);
  3588. if (!t)
  3589. goto unlock;
  3590. usage = mem_cgroup_usage(memcg, swap);
  3591. /*
  3592. * current_threshold points to threshold just below or equal to usage.
  3593. * If it's not true, a threshold was crossed after last
  3594. * call of __mem_cgroup_threshold().
  3595. */
  3596. i = t->current_threshold;
  3597. /*
  3598. * Iterate backward over array of thresholds starting from
  3599. * current_threshold and check if a threshold is crossed.
  3600. * If none of thresholds below usage is crossed, we read
  3601. * only one element of the array here.
  3602. */
  3603. for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
  3604. eventfd_signal(t->entries[i].eventfd, 1);
  3605. /* i = current_threshold + 1 */
  3606. i++;
  3607. /*
  3608. * Iterate forward over array of thresholds starting from
  3609. * current_threshold+1 and check if a threshold is crossed.
  3610. * If none of thresholds above usage is crossed, we read
  3611. * only one element of the array here.
  3612. */
  3613. for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
  3614. eventfd_signal(t->entries[i].eventfd, 1);
  3615. /* Update current_threshold */
  3616. t->current_threshold = i - 1;
  3617. unlock:
  3618. rcu_read_unlock();
  3619. }
  3620. static void mem_cgroup_threshold(struct mem_cgroup *memcg)
  3621. {
  3622. while (memcg) {
  3623. __mem_cgroup_threshold(memcg, false);
  3624. if (do_memsw_account())
  3625. __mem_cgroup_threshold(memcg, true);
  3626. memcg = parent_mem_cgroup(memcg);
  3627. }
  3628. }
  3629. static int compare_thresholds(const void *a, const void *b)
  3630. {
  3631. const struct mem_cgroup_threshold *_a = a;
  3632. const struct mem_cgroup_threshold *_b = b;
  3633. if (_a->threshold > _b->threshold)
  3634. return 1;
  3635. if (_a->threshold < _b->threshold)
  3636. return -1;
  3637. return 0;
  3638. }
  3639. static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
  3640. {
  3641. struct mem_cgroup_eventfd_list *ev;
  3642. spin_lock(&memcg_oom_lock);
  3643. list_for_each_entry(ev, &memcg->oom_notify, list)
  3644. eventfd_signal(ev->eventfd, 1);
  3645. spin_unlock(&memcg_oom_lock);
  3646. return 0;
  3647. }
  3648. static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
  3649. {
  3650. struct mem_cgroup *iter;
  3651. for_each_mem_cgroup_tree(iter, memcg)
  3652. mem_cgroup_oom_notify_cb(iter);
  3653. }
  3654. static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
  3655. struct eventfd_ctx *eventfd, const char *args, enum res_type type)
  3656. {
  3657. struct mem_cgroup_thresholds *thresholds;
  3658. struct mem_cgroup_threshold_ary *new;
  3659. unsigned long threshold;
  3660. unsigned long usage;
  3661. int i, size, ret;
  3662. ret = page_counter_memparse(args, "-1", &threshold);
  3663. if (ret)
  3664. return ret;
  3665. mutex_lock(&memcg->thresholds_lock);
  3666. if (type == _MEM) {
  3667. thresholds = &memcg->thresholds;
  3668. usage = mem_cgroup_usage(memcg, false);
  3669. } else if (type == _MEMSWAP) {
  3670. thresholds = &memcg->memsw_thresholds;
  3671. usage = mem_cgroup_usage(memcg, true);
  3672. } else
  3673. BUG();
  3674. /* Check if a threshold crossed before adding a new one */
  3675. if (thresholds->primary)
  3676. __mem_cgroup_threshold(memcg, type == _MEMSWAP);
  3677. size = thresholds->primary ? thresholds->primary->size + 1 : 1;
  3678. /* Allocate memory for new array of thresholds */
  3679. new = kmalloc(struct_size(new, entries, size), GFP_KERNEL);
  3680. if (!new) {
  3681. ret = -ENOMEM;
  3682. goto unlock;
  3683. }
  3684. new->size = size;
  3685. /* Copy thresholds (if any) to new array */
  3686. if (thresholds->primary)
  3687. memcpy(new->entries, thresholds->primary->entries,
  3688. flex_array_size(new, entries, size - 1));
  3689. /* Add new threshold */
  3690. new->entries[size - 1].eventfd = eventfd;
  3691. new->entries[size - 1].threshold = threshold;
  3692. /* Sort thresholds. Registering of new threshold isn't time-critical */
  3693. sort(new->entries, size, sizeof(*new->entries),
  3694. compare_thresholds, NULL);
  3695. /* Find current threshold */
  3696. new->current_threshold = -1;
  3697. for (i = 0; i < size; i++) {
  3698. if (new->entries[i].threshold <= usage) {
  3699. /*
  3700. * new->current_threshold will not be used until
  3701. * rcu_assign_pointer(), so it's safe to increment
  3702. * it here.
  3703. */
  3704. ++new->current_threshold;
  3705. } else
  3706. break;
  3707. }
  3708. /* Free old spare buffer and save old primary buffer as spare */
  3709. kfree(thresholds->spare);
  3710. thresholds->spare = thresholds->primary;
  3711. rcu_assign_pointer(thresholds->primary, new);
  3712. /* To be sure that nobody uses thresholds */
  3713. synchronize_rcu();
  3714. unlock:
  3715. mutex_unlock(&memcg->thresholds_lock);
  3716. return ret;
  3717. }
  3718. static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
  3719. struct eventfd_ctx *eventfd, const char *args)
  3720. {
  3721. return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM);
  3722. }
  3723. static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg,
  3724. struct eventfd_ctx *eventfd, const char *args)
  3725. {
  3726. return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP);
  3727. }
  3728. static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
  3729. struct eventfd_ctx *eventfd, enum res_type type)
  3730. {
  3731. struct mem_cgroup_thresholds *thresholds;
  3732. struct mem_cgroup_threshold_ary *new;
  3733. unsigned long usage;
  3734. int i, j, size, entries;
  3735. mutex_lock(&memcg->thresholds_lock);
  3736. if (type == _MEM) {
  3737. thresholds = &memcg->thresholds;
  3738. usage = mem_cgroup_usage(memcg, false);
  3739. } else if (type == _MEMSWAP) {
  3740. thresholds = &memcg->memsw_thresholds;
  3741. usage = mem_cgroup_usage(memcg, true);
  3742. } else
  3743. BUG();
  3744. if (!thresholds->primary)
  3745. goto unlock;
  3746. /* Check if a threshold crossed before removing */
  3747. __mem_cgroup_threshold(memcg, type == _MEMSWAP);
  3748. /* Calculate new number of threshold */
  3749. size = entries = 0;
  3750. for (i = 0; i < thresholds->primary->size; i++) {
  3751. if (thresholds->primary->entries[i].eventfd != eventfd)
  3752. size++;
  3753. else
  3754. entries++;
  3755. }
  3756. new = thresholds->spare;
  3757. /* If no items related to eventfd have been cleared, nothing to do */
  3758. if (!entries)
  3759. goto unlock;
  3760. /* Set thresholds array to NULL if we don't have thresholds */
  3761. if (!size) {
  3762. kfree(new);
  3763. new = NULL;
  3764. goto swap_buffers;
  3765. }
  3766. new->size = size;
  3767. /* Copy thresholds and find current threshold */
  3768. new->current_threshold = -1;
  3769. for (i = 0, j = 0; i < thresholds->primary->size; i++) {
  3770. if (thresholds->primary->entries[i].eventfd == eventfd)
  3771. continue;
  3772. new->entries[j] = thresholds->primary->entries[i];
  3773. if (new->entries[j].threshold <= usage) {
  3774. /*
  3775. * new->current_threshold will not be used
  3776. * until rcu_assign_pointer(), so it's safe to increment
  3777. * it here.
  3778. */
  3779. ++new->current_threshold;
  3780. }
  3781. j++;
  3782. }
  3783. swap_buffers:
  3784. /* Swap primary and spare array */
  3785. thresholds->spare = thresholds->primary;
  3786. rcu_assign_pointer(thresholds->primary, new);
  3787. /* To be sure that nobody uses thresholds */
  3788. synchronize_rcu();
  3789. /* If all events are unregistered, free the spare array */
  3790. if (!new) {
  3791. kfree(thresholds->spare);
  3792. thresholds->spare = NULL;
  3793. }
  3794. unlock:
  3795. mutex_unlock(&memcg->thresholds_lock);
  3796. }
  3797. static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
  3798. struct eventfd_ctx *eventfd)
  3799. {
  3800. return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM);
  3801. }
  3802. static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
  3803. struct eventfd_ctx *eventfd)
  3804. {
  3805. return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP);
  3806. }
  3807. static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg,
  3808. struct eventfd_ctx *eventfd, const char *args)
  3809. {
  3810. struct mem_cgroup_eventfd_list *event;
  3811. event = kmalloc(sizeof(*event), GFP_KERNEL);
  3812. if (!event)
  3813. return -ENOMEM;
  3814. spin_lock(&memcg_oom_lock);
  3815. event->eventfd = eventfd;
  3816. list_add(&event->list, &memcg->oom_notify);
  3817. /* already in OOM ? */
  3818. if (memcg->under_oom)
  3819. eventfd_signal(eventfd, 1);
  3820. spin_unlock(&memcg_oom_lock);
  3821. return 0;
  3822. }
  3823. static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg,
  3824. struct eventfd_ctx *eventfd)
  3825. {
  3826. struct mem_cgroup_eventfd_list *ev, *tmp;
  3827. spin_lock(&memcg_oom_lock);
  3828. list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
  3829. if (ev->eventfd == eventfd) {
  3830. list_del(&ev->list);
  3831. kfree(ev);
  3832. }
  3833. }
  3834. spin_unlock(&memcg_oom_lock);
  3835. }
  3836. static int mem_cgroup_oom_control_read(struct seq_file *sf, void *v)
  3837. {
  3838. struct mem_cgroup *memcg = mem_cgroup_from_seq(sf);
  3839. seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable);
  3840. seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom);
  3841. seq_printf(sf, "oom_kill %lu\n",
  3842. atomic_long_read(&memcg->memory_events[MEMCG_OOM_KILL]));
  3843. return 0;
  3844. }
  3845. static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css,
  3846. struct cftype *cft, u64 val)
  3847. {
  3848. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  3849. /* cannot set to root cgroup and only 0 and 1 are allowed */
  3850. if (!css->parent || !((val == 0) || (val == 1)))
  3851. return -EINVAL;
  3852. memcg->oom_kill_disable = val;
  3853. if (!val)
  3854. memcg_oom_recover(memcg);
  3855. return 0;
  3856. }
  3857. #ifdef CONFIG_CGROUP_WRITEBACK
  3858. #include <trace/events/writeback.h>
  3859. static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
  3860. {
  3861. return wb_domain_init(&memcg->cgwb_domain, gfp);
  3862. }
  3863. static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
  3864. {
  3865. wb_domain_exit(&memcg->cgwb_domain);
  3866. }
  3867. static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
  3868. {
  3869. wb_domain_size_changed(&memcg->cgwb_domain);
  3870. }
  3871. struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
  3872. {
  3873. struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
  3874. if (!memcg->css.parent)
  3875. return NULL;
  3876. return &memcg->cgwb_domain;
  3877. }
  3878. /*
  3879. * idx can be of type enum memcg_stat_item or node_stat_item.
  3880. * Keep in sync with memcg_exact_page().
  3881. */
  3882. static unsigned long memcg_exact_page_state(struct mem_cgroup *memcg, int idx)
  3883. {
  3884. long x = atomic_long_read(&memcg->vmstats[idx]);
  3885. int cpu;
  3886. for_each_online_cpu(cpu)
  3887. x += per_cpu_ptr(memcg->vmstats_percpu, cpu)->stat[idx];
  3888. if (x < 0)
  3889. x = 0;
  3890. return x;
  3891. }
  3892. /**
  3893. * mem_cgroup_wb_stats - retrieve writeback related stats from its memcg
  3894. * @wb: bdi_writeback in question
  3895. * @pfilepages: out parameter for number of file pages
  3896. * @pheadroom: out parameter for number of allocatable pages according to memcg
  3897. * @pdirty: out parameter for number of dirty pages
  3898. * @pwriteback: out parameter for number of pages under writeback
  3899. *
  3900. * Determine the numbers of file, headroom, dirty, and writeback pages in
  3901. * @wb's memcg. File, dirty and writeback are self-explanatory. Headroom
  3902. * is a bit more involved.
  3903. *
  3904. * A memcg's headroom is "min(max, high) - used". In the hierarchy, the
  3905. * headroom is calculated as the lowest headroom of itself and the
  3906. * ancestors. Note that this doesn't consider the actual amount of
  3907. * available memory in the system. The caller should further cap
  3908. * *@pheadroom accordingly.
  3909. */
  3910. void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
  3911. unsigned long *pheadroom, unsigned long *pdirty,
  3912. unsigned long *pwriteback)
  3913. {
  3914. struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
  3915. struct mem_cgroup *parent;
  3916. *pdirty = memcg_exact_page_state(memcg, NR_FILE_DIRTY);
  3917. *pwriteback = memcg_exact_page_state(memcg, NR_WRITEBACK);
  3918. *pfilepages = memcg_exact_page_state(memcg, NR_INACTIVE_FILE) +
  3919. memcg_exact_page_state(memcg, NR_ACTIVE_FILE);
  3920. *pheadroom = PAGE_COUNTER_MAX;
  3921. while ((parent = parent_mem_cgroup(memcg))) {
  3922. unsigned long ceiling = min(READ_ONCE(memcg->memory.max),
  3923. READ_ONCE(memcg->memory.high));
  3924. unsigned long used = page_counter_read(&memcg->memory);
  3925. *pheadroom = min(*pheadroom, ceiling - min(ceiling, used));
  3926. memcg = parent;
  3927. }
  3928. }
  3929. /*
  3930. * Foreign dirty flushing
  3931. *
  3932. * There's an inherent mismatch between memcg and writeback. The former
  3933. * trackes ownership per-page while the latter per-inode. This was a
  3934. * deliberate design decision because honoring per-page ownership in the
  3935. * writeback path is complicated, may lead to higher CPU and IO overheads
  3936. * and deemed unnecessary given that write-sharing an inode across
  3937. * different cgroups isn't a common use-case.
  3938. *
  3939. * Combined with inode majority-writer ownership switching, this works well
  3940. * enough in most cases but there are some pathological cases. For
  3941. * example, let's say there are two cgroups A and B which keep writing to
  3942. * different but confined parts of the same inode. B owns the inode and
  3943. * A's memory is limited far below B's. A's dirty ratio can rise enough to
  3944. * trigger balance_dirty_pages() sleeps but B's can be low enough to avoid
  3945. * triggering background writeback. A will be slowed down without a way to
  3946. * make writeback of the dirty pages happen.
  3947. *
  3948. * Conditions like the above can lead to a cgroup getting repatedly and
  3949. * severely throttled after making some progress after each
  3950. * dirty_expire_interval while the underyling IO device is almost
  3951. * completely idle.
  3952. *
  3953. * Solving this problem completely requires matching the ownership tracking
  3954. * granularities between memcg and writeback in either direction. However,
  3955. * the more egregious behaviors can be avoided by simply remembering the
  3956. * most recent foreign dirtying events and initiating remote flushes on
  3957. * them when local writeback isn't enough to keep the memory clean enough.
  3958. *
  3959. * The following two functions implement such mechanism. When a foreign
  3960. * page - a page whose memcg and writeback ownerships don't match - is
  3961. * dirtied, mem_cgroup_track_foreign_dirty() records the inode owning
  3962. * bdi_writeback on the page owning memcg. When balance_dirty_pages()
  3963. * decides that the memcg needs to sleep due to high dirty ratio, it calls
  3964. * mem_cgroup_flush_foreign() which queues writeback on the recorded
  3965. * foreign bdi_writebacks which haven't expired. Both the numbers of
  3966. * recorded bdi_writebacks and concurrent in-flight foreign writebacks are
  3967. * limited to MEMCG_CGWB_FRN_CNT.
  3968. *
  3969. * The mechanism only remembers IDs and doesn't hold any object references.
  3970. * As being wrong occasionally doesn't matter, updates and accesses to the
  3971. * records are lockless and racy.
  3972. */
  3973. void mem_cgroup_track_foreign_dirty_slowpath(struct page *page,
  3974. struct bdi_writeback *wb)
  3975. {
  3976. struct mem_cgroup *memcg = page->mem_cgroup;
  3977. struct memcg_cgwb_frn *frn;
  3978. u64 now = get_jiffies_64();
  3979. u64 oldest_at = now;
  3980. int oldest = -1;
  3981. int i;
  3982. trace_track_foreign_dirty(page, wb);
  3983. /*
  3984. * Pick the slot to use. If there is already a slot for @wb, keep
  3985. * using it. If not replace the oldest one which isn't being
  3986. * written out.
  3987. */
  3988. for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) {
  3989. frn = &memcg->cgwb_frn[i];
  3990. if (frn->bdi_id == wb->bdi->id &&
  3991. frn->memcg_id == wb->memcg_css->id)
  3992. break;
  3993. if (time_before64(frn->at, oldest_at) &&
  3994. atomic_read(&frn->done.cnt) == 1) {
  3995. oldest = i;
  3996. oldest_at = frn->at;
  3997. }
  3998. }
  3999. if (i < MEMCG_CGWB_FRN_CNT) {
  4000. /*
  4001. * Re-using an existing one. Update timestamp lazily to
  4002. * avoid making the cacheline hot. We want them to be
  4003. * reasonably up-to-date and significantly shorter than
  4004. * dirty_expire_interval as that's what expires the record.
  4005. * Use the shorter of 1s and dirty_expire_interval / 8.
  4006. */
  4007. unsigned long update_intv =
  4008. min_t(unsigned long, HZ,
  4009. msecs_to_jiffies(dirty_expire_interval * 10) / 8);
  4010. if (time_before64(frn->at, now - update_intv))
  4011. frn->at = now;
  4012. } else if (oldest >= 0) {
  4013. /* replace the oldest free one */
  4014. frn = &memcg->cgwb_frn[oldest];
  4015. frn->bdi_id = wb->bdi->id;
  4016. frn->memcg_id = wb->memcg_css->id;
  4017. frn->at = now;
  4018. }
  4019. }
  4020. /* issue foreign writeback flushes for recorded foreign dirtying events */
  4021. void mem_cgroup_flush_foreign(struct bdi_writeback *wb)
  4022. {
  4023. struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
  4024. unsigned long intv = msecs_to_jiffies(dirty_expire_interval * 10);
  4025. u64 now = jiffies_64;
  4026. int i;
  4027. for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) {
  4028. struct memcg_cgwb_frn *frn = &memcg->cgwb_frn[i];
  4029. /*
  4030. * If the record is older than dirty_expire_interval,
  4031. * writeback on it has already started. No need to kick it
  4032. * off again. Also, don't start a new one if there's
  4033. * already one in flight.
  4034. */
  4035. if (time_after64(frn->at, now - intv) &&
  4036. atomic_read(&frn->done.cnt) == 1) {
  4037. frn->at = 0;
  4038. trace_flush_foreign(wb, frn->bdi_id, frn->memcg_id);
  4039. cgroup_writeback_by_id(frn->bdi_id, frn->memcg_id, 0,
  4040. WB_REASON_FOREIGN_FLUSH,
  4041. &frn->done);
  4042. }
  4043. }
  4044. }
  4045. #else /* CONFIG_CGROUP_WRITEBACK */
  4046. static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
  4047. {
  4048. return 0;
  4049. }
  4050. static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
  4051. {
  4052. }
  4053. static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
  4054. {
  4055. }
  4056. #endif /* CONFIG_CGROUP_WRITEBACK */
  4057. /*
  4058. * DO NOT USE IN NEW FILES.
  4059. *
  4060. * "cgroup.event_control" implementation.
  4061. *
  4062. * This is way over-engineered. It tries to support fully configurable
  4063. * events for each user. Such level of flexibility is completely
  4064. * unnecessary especially in the light of the planned unified hierarchy.
  4065. *
  4066. * Please deprecate this and replace with something simpler if at all
  4067. * possible.
  4068. */
  4069. /*
  4070. * Unregister event and free resources.
  4071. *
  4072. * Gets called from workqueue.
  4073. */
  4074. static void memcg_event_remove(struct work_struct *work)
  4075. {
  4076. struct mem_cgroup_event *event =
  4077. container_of(work, struct mem_cgroup_event, remove);
  4078. struct mem_cgroup *memcg = event->memcg;
  4079. remove_wait_queue(event->wqh, &event->wait);
  4080. event->unregister_event(memcg, event->eventfd);
  4081. /* Notify userspace the event is going away. */
  4082. eventfd_signal(event->eventfd, 1);
  4083. eventfd_ctx_put(event->eventfd);
  4084. kfree(event);
  4085. css_put(&memcg->css);
  4086. }
  4087. /*
  4088. * Gets called on EPOLLHUP on eventfd when user closes it.
  4089. *
  4090. * Called with wqh->lock held and interrupts disabled.
  4091. */
  4092. static int memcg_event_wake(wait_queue_entry_t *wait, unsigned mode,
  4093. int sync, void *key)
  4094. {
  4095. struct mem_cgroup_event *event =
  4096. container_of(wait, struct mem_cgroup_event, wait);
  4097. struct mem_cgroup *memcg = event->memcg;
  4098. __poll_t flags = key_to_poll(key);
  4099. if (flags & EPOLLHUP) {
  4100. /*
  4101. * If the event has been detached at cgroup removal, we
  4102. * can simply return knowing the other side will cleanup
  4103. * for us.
  4104. *
  4105. * We can't race against event freeing since the other
  4106. * side will require wqh->lock via remove_wait_queue(),
  4107. * which we hold.
  4108. */
  4109. spin_lock(&memcg->event_list_lock);
  4110. if (!list_empty(&event->list)) {
  4111. list_del_init(&event->list);
  4112. /*
  4113. * We are in atomic context, but cgroup_event_remove()
  4114. * may sleep, so we have to call it in workqueue.
  4115. */
  4116. schedule_work(&event->remove);
  4117. }
  4118. spin_unlock(&memcg->event_list_lock);
  4119. }
  4120. return 0;
  4121. }
  4122. static void memcg_event_ptable_queue_proc(struct file *file,
  4123. wait_queue_head_t *wqh, poll_table *pt)
  4124. {
  4125. struct mem_cgroup_event *event =
  4126. container_of(pt, struct mem_cgroup_event, pt);
  4127. event->wqh = wqh;
  4128. add_wait_queue(wqh, &event->wait);
  4129. }
  4130. /*
  4131. * DO NOT USE IN NEW FILES.
  4132. *
  4133. * Parse input and register new cgroup event handler.
  4134. *
  4135. * Input must be in format '<event_fd> <control_fd> <args>'.
  4136. * Interpretation of args is defined by control file implementation.
  4137. */
  4138. static ssize_t memcg_write_event_control(struct kernfs_open_file *of,
  4139. char *buf, size_t nbytes, loff_t off)
  4140. {
  4141. struct cgroup_subsys_state *css = of_css(of);
  4142. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  4143. struct mem_cgroup_event *event;
  4144. struct cgroup_subsys_state *cfile_css;
  4145. unsigned int efd, cfd;
  4146. struct fd efile;
  4147. struct fd cfile;
  4148. const char *name;
  4149. char *endp;
  4150. int ret;
  4151. buf = strstrip(buf);
  4152. efd = simple_strtoul(buf, &endp, 10);
  4153. if (*endp != ' ')
  4154. return -EINVAL;
  4155. buf = endp + 1;
  4156. cfd = simple_strtoul(buf, &endp, 10);
  4157. if ((*endp != ' ') && (*endp != '\0'))
  4158. return -EINVAL;
  4159. buf = endp + 1;
  4160. event = kzalloc(sizeof(*event), GFP_KERNEL);
  4161. if (!event)
  4162. return -ENOMEM;
  4163. event->memcg = memcg;
  4164. INIT_LIST_HEAD(&event->list);
  4165. init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc);
  4166. init_waitqueue_func_entry(&event->wait, memcg_event_wake);
  4167. INIT_WORK(&event->remove, memcg_event_remove);
  4168. efile = fdget(efd);
  4169. if (!efile.file) {
  4170. ret = -EBADF;
  4171. goto out_kfree;
  4172. }
  4173. event->eventfd = eventfd_ctx_fileget(efile.file);
  4174. if (IS_ERR(event->eventfd)) {
  4175. ret = PTR_ERR(event->eventfd);
  4176. goto out_put_efile;
  4177. }
  4178. cfile = fdget(cfd);
  4179. if (!cfile.file) {
  4180. ret = -EBADF;
  4181. goto out_put_eventfd;
  4182. }
  4183. /* the process need read permission on control file */
  4184. /* AV: shouldn't we check that it's been opened for read instead? */
  4185. ret = inode_permission(file_inode(cfile.file), MAY_READ);
  4186. if (ret < 0)
  4187. goto out_put_cfile;
  4188. /*
  4189. * Determine the event callbacks and set them in @event. This used
  4190. * to be done via struct cftype but cgroup core no longer knows
  4191. * about these events. The following is crude but the whole thing
  4192. * is for compatibility anyway.
  4193. *
  4194. * DO NOT ADD NEW FILES.
  4195. */
  4196. name = cfile.file->f_path.dentry->d_name.name;
  4197. if (!strcmp(name, "memory.usage_in_bytes")) {
  4198. event->register_event = mem_cgroup_usage_register_event;
  4199. event->unregister_event = mem_cgroup_usage_unregister_event;
  4200. } else if (!strcmp(name, "memory.oom_control")) {
  4201. event->register_event = mem_cgroup_oom_register_event;
  4202. event->unregister_event = mem_cgroup_oom_unregister_event;
  4203. } else if (!strcmp(name, "memory.pressure_level")) {
  4204. event->register_event = vmpressure_register_event;
  4205. event->unregister_event = vmpressure_unregister_event;
  4206. } else if (!strcmp(name, "memory.memsw.usage_in_bytes")) {
  4207. event->register_event = memsw_cgroup_usage_register_event;
  4208. event->unregister_event = memsw_cgroup_usage_unregister_event;
  4209. } else {
  4210. ret = -EINVAL;
  4211. goto out_put_cfile;
  4212. }
  4213. /*
  4214. * Verify @cfile should belong to @css. Also, remaining events are
  4215. * automatically removed on cgroup destruction but the removal is
  4216. * asynchronous, so take an extra ref on @css.
  4217. */
  4218. cfile_css = css_tryget_online_from_dir(cfile.file->f_path.dentry->d_parent,
  4219. &memory_cgrp_subsys);
  4220. ret = -EINVAL;
  4221. if (IS_ERR(cfile_css))
  4222. goto out_put_cfile;
  4223. if (cfile_css != css) {
  4224. css_put(cfile_css);
  4225. goto out_put_cfile;
  4226. }
  4227. ret = event->register_event(memcg, event->eventfd, buf);
  4228. if (ret)
  4229. goto out_put_css;
  4230. vfs_poll(efile.file, &event->pt);
  4231. spin_lock(&memcg->event_list_lock);
  4232. list_add(&event->list, &memcg->event_list);
  4233. spin_unlock(&memcg->event_list_lock);
  4234. fdput(cfile);
  4235. fdput(efile);
  4236. return nbytes;
  4237. out_put_css:
  4238. css_put(css);
  4239. out_put_cfile:
  4240. fdput(cfile);
  4241. out_put_eventfd:
  4242. eventfd_ctx_put(event->eventfd);
  4243. out_put_efile:
  4244. fdput(efile);
  4245. out_kfree:
  4246. kfree(event);
  4247. return ret;
  4248. }
  4249. static struct cftype mem_cgroup_legacy_files[] = {
  4250. {
  4251. .name = "usage_in_bytes",
  4252. .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
  4253. .read_u64 = mem_cgroup_read_u64,
  4254. },
  4255. {
  4256. .name = "max_usage_in_bytes",
  4257. .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
  4258. .write = mem_cgroup_reset,
  4259. .read_u64 = mem_cgroup_read_u64,
  4260. },
  4261. {
  4262. .name = "limit_in_bytes",
  4263. .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
  4264. .write = mem_cgroup_write,
  4265. .read_u64 = mem_cgroup_read_u64,
  4266. },
  4267. {
  4268. .name = "soft_limit_in_bytes",
  4269. .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
  4270. .write = mem_cgroup_write,
  4271. .read_u64 = mem_cgroup_read_u64,
  4272. },
  4273. {
  4274. .name = "failcnt",
  4275. .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
  4276. .write = mem_cgroup_reset,
  4277. .read_u64 = mem_cgroup_read_u64,
  4278. },
  4279. {
  4280. .name = "stat",
  4281. .seq_show = memcg_stat_show,
  4282. },
  4283. {
  4284. .name = "force_empty",
  4285. .write = mem_cgroup_force_empty_write,
  4286. },
  4287. {
  4288. .name = "use_hierarchy",
  4289. .write_u64 = mem_cgroup_hierarchy_write,
  4290. .read_u64 = mem_cgroup_hierarchy_read,
  4291. },
  4292. {
  4293. .name = "cgroup.event_control", /* XXX: for compat */
  4294. .write = memcg_write_event_control,
  4295. .flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE,
  4296. },
  4297. {
  4298. .name = "swappiness",
  4299. .read_u64 = mem_cgroup_swappiness_read,
  4300. .write_u64 = mem_cgroup_swappiness_write,
  4301. },
  4302. {
  4303. .name = "move_charge_at_immigrate",
  4304. .read_u64 = mem_cgroup_move_charge_read,
  4305. .write_u64 = mem_cgroup_move_charge_write,
  4306. },
  4307. {
  4308. .name = "oom_control",
  4309. .seq_show = mem_cgroup_oom_control_read,
  4310. .write_u64 = mem_cgroup_oom_control_write,
  4311. .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
  4312. },
  4313. {
  4314. .name = "pressure_level",
  4315. },
  4316. #ifdef CONFIG_NUMA
  4317. {
  4318. .name = "numa_stat",
  4319. .seq_show = memcg_numa_stat_show,
  4320. },
  4321. #endif
  4322. {
  4323. .name = "kmem.limit_in_bytes",
  4324. .private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
  4325. .write = mem_cgroup_write,
  4326. .read_u64 = mem_cgroup_read_u64,
  4327. },
  4328. {
  4329. .name = "kmem.usage_in_bytes",
  4330. .private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
  4331. .read_u64 = mem_cgroup_read_u64,
  4332. },
  4333. {
  4334. .name = "kmem.failcnt",
  4335. .private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
  4336. .write = mem_cgroup_reset,
  4337. .read_u64 = mem_cgroup_read_u64,
  4338. },
  4339. {
  4340. .name = "kmem.max_usage_in_bytes",
  4341. .private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
  4342. .write = mem_cgroup_reset,
  4343. .read_u64 = mem_cgroup_read_u64,
  4344. },
  4345. #if defined(CONFIG_MEMCG_KMEM) && \
  4346. (defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG))
  4347. {
  4348. .name = "kmem.slabinfo",
  4349. .seq_show = memcg_slab_show,
  4350. },
  4351. #endif
  4352. {
  4353. .name = "kmem.tcp.limit_in_bytes",
  4354. .private = MEMFILE_PRIVATE(_TCP, RES_LIMIT),
  4355. .write = mem_cgroup_write,
  4356. .read_u64 = mem_cgroup_read_u64,
  4357. },
  4358. {
  4359. .name = "kmem.tcp.usage_in_bytes",
  4360. .private = MEMFILE_PRIVATE(_TCP, RES_USAGE),
  4361. .read_u64 = mem_cgroup_read_u64,
  4362. },
  4363. {
  4364. .name = "kmem.tcp.failcnt",
  4365. .private = MEMFILE_PRIVATE(_TCP, RES_FAILCNT),
  4366. .write = mem_cgroup_reset,
  4367. .read_u64 = mem_cgroup_read_u64,
  4368. },
  4369. {
  4370. .name = "kmem.tcp.max_usage_in_bytes",
  4371. .private = MEMFILE_PRIVATE(_TCP, RES_MAX_USAGE),
  4372. .write = mem_cgroup_reset,
  4373. .read_u64 = mem_cgroup_read_u64,
  4374. },
  4375. { }, /* terminate */
  4376. };
  4377. /*
  4378. * Private memory cgroup IDR
  4379. *
  4380. * Swap-out records and page cache shadow entries need to store memcg
  4381. * references in constrained space, so we maintain an ID space that is
  4382. * limited to 16 bit (MEM_CGROUP_ID_MAX), limiting the total number of
  4383. * memory-controlled cgroups to 64k.
  4384. *
  4385. * However, there usually are many references to the offline CSS after
  4386. * the cgroup has been destroyed, such as page cache or reclaimable
  4387. * slab objects, that don't need to hang on to the ID. We want to keep
  4388. * those dead CSS from occupying IDs, or we might quickly exhaust the
  4389. * relatively small ID space and prevent the creation of new cgroups
  4390. * even when there are much fewer than 64k cgroups - possibly none.
  4391. *
  4392. * Maintain a private 16-bit ID space for memcg, and allow the ID to
  4393. * be freed and recycled when it's no longer needed, which is usually
  4394. * when the CSS is offlined.
  4395. *
  4396. * The only exception to that are records of swapped out tmpfs/shmem
  4397. * pages that need to be attributed to live ancestors on swapin. But
  4398. * those references are manageable from userspace.
  4399. */
  4400. static DEFINE_IDR(mem_cgroup_idr);
  4401. static void mem_cgroup_id_remove(struct mem_cgroup *memcg)
  4402. {
  4403. if (memcg->id.id > 0) {
  4404. trace_android_vh_mem_cgroup_id_remove(memcg);
  4405. idr_remove(&mem_cgroup_idr, memcg->id.id);
  4406. memcg->id.id = 0;
  4407. }
  4408. }
  4409. static void __maybe_unused mem_cgroup_id_get_many(struct mem_cgroup *memcg,
  4410. unsigned int n)
  4411. {
  4412. refcount_add(n, &memcg->id.ref);
  4413. }
  4414. static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n)
  4415. {
  4416. if (refcount_sub_and_test(n, &memcg->id.ref)) {
  4417. mem_cgroup_id_remove(memcg);
  4418. /* Memcg ID pins CSS */
  4419. css_put(&memcg->css);
  4420. }
  4421. }
  4422. static inline void mem_cgroup_id_put(struct mem_cgroup *memcg)
  4423. {
  4424. mem_cgroup_id_put_many(memcg, 1);
  4425. }
  4426. /**
  4427. * mem_cgroup_from_id - look up a memcg from a memcg id
  4428. * @id: the memcg id to look up
  4429. *
  4430. * Caller must hold rcu_read_lock().
  4431. */
  4432. struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
  4433. {
  4434. WARN_ON_ONCE(!rcu_read_lock_held());
  4435. return idr_find(&mem_cgroup_idr, id);
  4436. }
  4437. EXPORT_SYMBOL_GPL(mem_cgroup_from_id);
  4438. static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
  4439. {
  4440. struct mem_cgroup_per_node *pn;
  4441. int tmp = node;
  4442. /*
  4443. * This routine is called against possible nodes.
  4444. * But it's BUG to call kmalloc() against offline node.
  4445. *
  4446. * TODO: this routine can waste much memory for nodes which will
  4447. * never be onlined. It's better to use memory hotplug callback
  4448. * function.
  4449. */
  4450. if (!node_state(node, N_NORMAL_MEMORY))
  4451. tmp = -1;
  4452. pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
  4453. if (!pn)
  4454. return 1;
  4455. pn->lruvec_stat_local = alloc_percpu_gfp(struct lruvec_stat,
  4456. GFP_KERNEL_ACCOUNT);
  4457. if (!pn->lruvec_stat_local) {
  4458. kfree(pn);
  4459. return 1;
  4460. }
  4461. pn->lruvec_stat_cpu = alloc_percpu_gfp(struct lruvec_stat,
  4462. GFP_KERNEL_ACCOUNT);
  4463. if (!pn->lruvec_stat_cpu) {
  4464. free_percpu(pn->lruvec_stat_local);
  4465. kfree(pn);
  4466. return 1;
  4467. }
  4468. lruvec_init(&pn->lruvec);
  4469. pn->usage_in_excess = 0;
  4470. pn->on_tree = false;
  4471. pn->memcg = memcg;
  4472. memcg->nodeinfo[node] = pn;
  4473. return 0;
  4474. }
  4475. static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
  4476. {
  4477. struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
  4478. if (!pn)
  4479. return;
  4480. free_percpu(pn->lruvec_stat_cpu);
  4481. free_percpu(pn->lruvec_stat_local);
  4482. kfree(pn);
  4483. }
  4484. static void __mem_cgroup_free(struct mem_cgroup *memcg)
  4485. {
  4486. int node;
  4487. trace_android_vh_mem_cgroup_free(memcg);
  4488. for_each_node(node)
  4489. free_mem_cgroup_per_node_info(memcg, node);
  4490. free_percpu(memcg->vmstats_percpu);
  4491. free_percpu(memcg->vmstats_local);
  4492. kfree(memcg);
  4493. }
  4494. static void mem_cgroup_free(struct mem_cgroup *memcg)
  4495. {
  4496. memcg_wb_domain_exit(memcg);
  4497. /*
  4498. * Flush percpu vmstats and vmevents to guarantee the value correctness
  4499. * on parent's and all ancestor levels.
  4500. */
  4501. memcg_flush_percpu_vmstats(memcg);
  4502. memcg_flush_percpu_vmevents(memcg);
  4503. __mem_cgroup_free(memcg);
  4504. }
  4505. static struct mem_cgroup *mem_cgroup_alloc(void)
  4506. {
  4507. struct mem_cgroup *memcg;
  4508. unsigned int size;
  4509. int node;
  4510. int __maybe_unused i;
  4511. long error = -ENOMEM;
  4512. size = sizeof(struct mem_cgroup);
  4513. size += nr_node_ids * sizeof(struct mem_cgroup_per_node *);
  4514. memcg = kzalloc(size, GFP_KERNEL);
  4515. if (!memcg)
  4516. return ERR_PTR(error);
  4517. memcg->id.id = idr_alloc(&mem_cgroup_idr, NULL,
  4518. 1, MEM_CGROUP_ID_MAX,
  4519. GFP_KERNEL);
  4520. if (memcg->id.id < 0) {
  4521. error = memcg->id.id;
  4522. goto fail;
  4523. }
  4524. memcg->vmstats_local = alloc_percpu_gfp(struct memcg_vmstats_percpu,
  4525. GFP_KERNEL_ACCOUNT);
  4526. if (!memcg->vmstats_local)
  4527. goto fail;
  4528. memcg->vmstats_percpu = alloc_percpu_gfp(struct memcg_vmstats_percpu,
  4529. GFP_KERNEL_ACCOUNT);
  4530. if (!memcg->vmstats_percpu)
  4531. goto fail;
  4532. for_each_node(node)
  4533. if (alloc_mem_cgroup_per_node_info(memcg, node))
  4534. goto fail;
  4535. if (memcg_wb_domain_init(memcg, GFP_KERNEL))
  4536. goto fail;
  4537. INIT_WORK(&memcg->high_work, high_work_func);
  4538. INIT_LIST_HEAD(&memcg->oom_notify);
  4539. mutex_init(&memcg->thresholds_lock);
  4540. spin_lock_init(&memcg->move_lock);
  4541. vmpressure_init(&memcg->vmpressure);
  4542. INIT_LIST_HEAD(&memcg->event_list);
  4543. spin_lock_init(&memcg->event_list_lock);
  4544. memcg->socket_pressure = jiffies;
  4545. #ifdef CONFIG_MEMCG_KMEM
  4546. memcg->kmemcg_id = -1;
  4547. INIT_LIST_HEAD(&memcg->objcg_list);
  4548. #endif
  4549. #ifdef CONFIG_CGROUP_WRITEBACK
  4550. INIT_LIST_HEAD(&memcg->cgwb_list);
  4551. for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++)
  4552. memcg->cgwb_frn[i].done =
  4553. __WB_COMPLETION_INIT(&memcg_cgwb_frn_waitq);
  4554. #endif
  4555. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  4556. spin_lock_init(&memcg->deferred_split_queue.split_queue_lock);
  4557. INIT_LIST_HEAD(&memcg->deferred_split_queue.split_queue);
  4558. memcg->deferred_split_queue.split_queue_len = 0;
  4559. #endif
  4560. idr_replace(&mem_cgroup_idr, memcg, memcg->id.id);
  4561. trace_android_vh_mem_cgroup_alloc(memcg);
  4562. return memcg;
  4563. fail:
  4564. mem_cgroup_id_remove(memcg);
  4565. __mem_cgroup_free(memcg);
  4566. return ERR_PTR(error);
  4567. }
  4568. static struct cgroup_subsys_state * __ref
  4569. mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
  4570. {
  4571. struct mem_cgroup *parent = mem_cgroup_from_css(parent_css);
  4572. struct mem_cgroup *memcg, *old_memcg;
  4573. long error = -ENOMEM;
  4574. old_memcg = set_active_memcg(parent);
  4575. memcg = mem_cgroup_alloc();
  4576. set_active_memcg(old_memcg);
  4577. if (IS_ERR(memcg))
  4578. return ERR_CAST(memcg);
  4579. page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);
  4580. memcg->soft_limit = PAGE_COUNTER_MAX;
  4581. page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
  4582. if (parent) {
  4583. memcg->swappiness = mem_cgroup_swappiness(parent);
  4584. memcg->oom_kill_disable = parent->oom_kill_disable;
  4585. }
  4586. if (!parent) {
  4587. page_counter_init(&memcg->memory, NULL);
  4588. page_counter_init(&memcg->swap, NULL);
  4589. page_counter_init(&memcg->kmem, NULL);
  4590. page_counter_init(&memcg->tcpmem, NULL);
  4591. } else if (parent->use_hierarchy) {
  4592. memcg->use_hierarchy = true;
  4593. page_counter_init(&memcg->memory, &parent->memory);
  4594. page_counter_init(&memcg->swap, &parent->swap);
  4595. page_counter_init(&memcg->kmem, &parent->kmem);
  4596. page_counter_init(&memcg->tcpmem, &parent->tcpmem);
  4597. } else {
  4598. page_counter_init(&memcg->memory, &root_mem_cgroup->memory);
  4599. page_counter_init(&memcg->swap, &root_mem_cgroup->swap);
  4600. page_counter_init(&memcg->kmem, &root_mem_cgroup->kmem);
  4601. page_counter_init(&memcg->tcpmem, &root_mem_cgroup->tcpmem);
  4602. /*
  4603. * Deeper hierachy with use_hierarchy == false doesn't make
  4604. * much sense so let cgroup subsystem know about this
  4605. * unfortunate state in our controller.
  4606. */
  4607. if (parent != root_mem_cgroup)
  4608. memory_cgrp_subsys.broken_hierarchy = true;
  4609. }
  4610. /* The following stuff does not apply to the root */
  4611. if (!parent) {
  4612. root_mem_cgroup = memcg;
  4613. return &memcg->css;
  4614. }
  4615. error = memcg_online_kmem(memcg);
  4616. if (error)
  4617. goto fail;
  4618. if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
  4619. static_branch_inc(&memcg_sockets_enabled_key);
  4620. return &memcg->css;
  4621. fail:
  4622. mem_cgroup_id_remove(memcg);
  4623. mem_cgroup_free(memcg);
  4624. return ERR_PTR(error);
  4625. }
  4626. static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
  4627. {
  4628. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  4629. /*
  4630. * A memcg must be visible for memcg_expand_shrinker_maps()
  4631. * by the time the maps are allocated. So, we allocate maps
  4632. * here, when for_each_mem_cgroup() can't skip it.
  4633. */
  4634. if (memcg_alloc_shrinker_maps(memcg)) {
  4635. mem_cgroup_id_remove(memcg);
  4636. return -ENOMEM;
  4637. }
  4638. /* Online state pins memcg ID, memcg ID pins CSS */
  4639. refcount_set(&memcg->id.ref, 1);
  4640. css_get(css);
  4641. trace_android_vh_mem_cgroup_css_online(css, memcg);
  4642. return 0;
  4643. }
  4644. static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
  4645. {
  4646. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  4647. struct mem_cgroup_event *event, *tmp;
  4648. trace_android_vh_mem_cgroup_css_offline(css, memcg);
  4649. /*
  4650. * Unregister events and notify userspace.
  4651. * Notify userspace about cgroup removing only after rmdir of cgroup
  4652. * directory to avoid race between userspace and kernelspace.
  4653. */
  4654. spin_lock(&memcg->event_list_lock);
  4655. list_for_each_entry_safe(event, tmp, &memcg->event_list, list) {
  4656. list_del_init(&event->list);
  4657. schedule_work(&event->remove);
  4658. }
  4659. spin_unlock(&memcg->event_list_lock);
  4660. page_counter_set_min(&memcg->memory, 0);
  4661. page_counter_set_low(&memcg->memory, 0);
  4662. memcg_offline_kmem(memcg);
  4663. wb_memcg_offline(memcg);
  4664. drain_all_stock(memcg);
  4665. mem_cgroup_id_put(memcg);
  4666. }
  4667. static void mem_cgroup_css_released(struct cgroup_subsys_state *css)
  4668. {
  4669. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  4670. invalidate_reclaim_iterators(memcg);
  4671. }
  4672. static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
  4673. {
  4674. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  4675. int __maybe_unused i;
  4676. #ifdef CONFIG_CGROUP_WRITEBACK
  4677. for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++)
  4678. wb_wait_for_completion(&memcg->cgwb_frn[i].done);
  4679. #endif
  4680. if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
  4681. static_branch_dec(&memcg_sockets_enabled_key);
  4682. if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active)
  4683. static_branch_dec(&memcg_sockets_enabled_key);
  4684. vmpressure_cleanup(&memcg->vmpressure);
  4685. cancel_work_sync(&memcg->high_work);
  4686. mem_cgroup_remove_from_trees(memcg);
  4687. memcg_free_shrinker_maps(memcg);
  4688. memcg_free_kmem(memcg);
  4689. mem_cgroup_free(memcg);
  4690. }
  4691. /**
  4692. * mem_cgroup_css_reset - reset the states of a mem_cgroup
  4693. * @css: the target css
  4694. *
  4695. * Reset the states of the mem_cgroup associated with @css. This is
  4696. * invoked when the userland requests disabling on the default hierarchy
  4697. * but the memcg is pinned through dependency. The memcg should stop
  4698. * applying policies and should revert to the vanilla state as it may be
  4699. * made visible again.
  4700. *
  4701. * The current implementation only resets the essential configurations.
  4702. * This needs to be expanded to cover all the visible parts.
  4703. */
  4704. static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
  4705. {
  4706. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  4707. page_counter_set_max(&memcg->memory, PAGE_COUNTER_MAX);
  4708. page_counter_set_max(&memcg->swap, PAGE_COUNTER_MAX);
  4709. page_counter_set_max(&memcg->kmem, PAGE_COUNTER_MAX);
  4710. page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX);
  4711. page_counter_set_min(&memcg->memory, 0);
  4712. page_counter_set_low(&memcg->memory, 0);
  4713. page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);
  4714. memcg->soft_limit = PAGE_COUNTER_MAX;
  4715. page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
  4716. memcg_wb_domain_size_changed(memcg);
  4717. }
  4718. #ifdef CONFIG_MMU
  4719. /* Handlers for move charge at task migration. */
  4720. static int mem_cgroup_do_precharge(unsigned long count)
  4721. {
  4722. int ret;
  4723. /* Try a single bulk charge without reclaim first, kswapd may wake */
  4724. ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_DIRECT_RECLAIM, count);
  4725. if (!ret) {
  4726. mc.precharge += count;
  4727. return ret;
  4728. }
  4729. /* Try charges one by one with reclaim, but do not retry */
  4730. while (count--) {
  4731. ret = try_charge(mc.to, GFP_KERNEL | __GFP_NORETRY, 1);
  4732. if (ret)
  4733. return ret;
  4734. mc.precharge++;
  4735. cond_resched();
  4736. }
  4737. return 0;
  4738. }
  4739. union mc_target {
  4740. struct page *page;
  4741. swp_entry_t ent;
  4742. };
  4743. enum mc_target_type {
  4744. MC_TARGET_NONE = 0,
  4745. MC_TARGET_PAGE,
  4746. MC_TARGET_SWAP,
  4747. MC_TARGET_DEVICE,
  4748. };
  4749. static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
  4750. unsigned long addr, pte_t ptent)
  4751. {
  4752. struct page *page = vm_normal_page(vma, addr, ptent);
  4753. if (!page || !page_mapped(page))
  4754. return NULL;
  4755. if (PageAnon(page)) {
  4756. if (!(mc.flags & MOVE_ANON))
  4757. return NULL;
  4758. } else {
  4759. if (!(mc.flags & MOVE_FILE))
  4760. return NULL;
  4761. }
  4762. if (!get_page_unless_zero(page))
  4763. return NULL;
  4764. return page;
  4765. }
  4766. #if defined(CONFIG_SWAP) || defined(CONFIG_DEVICE_PRIVATE)
  4767. static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
  4768. pte_t ptent, swp_entry_t *entry)
  4769. {
  4770. struct page *page = NULL;
  4771. swp_entry_t ent = pte_to_swp_entry(ptent);
  4772. if (!(mc.flags & MOVE_ANON))
  4773. return NULL;
  4774. /*
  4775. * Handle MEMORY_DEVICE_PRIVATE which are ZONE_DEVICE page belonging to
  4776. * a device and because they are not accessible by CPU they are store
  4777. * as special swap entry in the CPU page table.
  4778. */
  4779. if (is_device_private_entry(ent)) {
  4780. page = device_private_entry_to_page(ent);
  4781. /*
  4782. * MEMORY_DEVICE_PRIVATE means ZONE_DEVICE page and which have
  4783. * a refcount of 1 when free (unlike normal page)
  4784. */
  4785. if (!page_ref_add_unless(page, 1, 1))
  4786. return NULL;
  4787. return page;
  4788. }
  4789. if (non_swap_entry(ent))
  4790. return NULL;
  4791. /*
  4792. * Because lookup_swap_cache() updates some statistics counter,
  4793. * we call find_get_page() with swapper_space directly.
  4794. */
  4795. page = find_get_page(swap_address_space(ent), swp_offset(ent));
  4796. entry->val = ent.val;
  4797. return page;
  4798. }
  4799. #else
  4800. static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
  4801. pte_t ptent, swp_entry_t *entry)
  4802. {
  4803. return NULL;
  4804. }
  4805. #endif
  4806. static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
  4807. unsigned long addr, pte_t ptent, swp_entry_t *entry)
  4808. {
  4809. if (!vma->vm_file) /* anonymous vma */
  4810. return NULL;
  4811. if (!(mc.flags & MOVE_FILE))
  4812. return NULL;
  4813. /* page is moved even if it's not RSS of this task(page-faulted). */
  4814. /* shmem/tmpfs may report page out on swap: account for that too. */
  4815. return find_get_incore_page(vma->vm_file->f_mapping,
  4816. linear_page_index(vma, addr));
  4817. }
  4818. /**
  4819. * mem_cgroup_move_account - move account of the page
  4820. * @page: the page
  4821. * @compound: charge the page as compound or small page
  4822. * @from: mem_cgroup which the page is moved from.
  4823. * @to: mem_cgroup which the page is moved to. @from != @to.
  4824. *
  4825. * The caller must make sure the page is not on LRU (isolate_page() is useful.)
  4826. *
  4827. * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
  4828. * from old cgroup.
  4829. */
  4830. static int mem_cgroup_move_account(struct page *page,
  4831. bool compound,
  4832. struct mem_cgroup *from,
  4833. struct mem_cgroup *to)
  4834. {
  4835. struct lruvec *from_vec, *to_vec;
  4836. struct pglist_data *pgdat;
  4837. unsigned int nr_pages = compound ? thp_nr_pages(page) : 1;
  4838. int ret;
  4839. VM_BUG_ON(from == to);
  4840. VM_BUG_ON_PAGE(PageLRU(page), page);
  4841. VM_BUG_ON(compound && !PageTransHuge(page));
  4842. /*
  4843. * Prevent mem_cgroup_migrate() from looking at
  4844. * page->mem_cgroup of its source page while we change it.
  4845. */
  4846. ret = -EBUSY;
  4847. if (!trylock_page(page))
  4848. goto out;
  4849. ret = -EINVAL;
  4850. if (page->mem_cgroup != from)
  4851. goto out_unlock;
  4852. pgdat = page_pgdat(page);
  4853. from_vec = mem_cgroup_lruvec(from, pgdat);
  4854. to_vec = mem_cgroup_lruvec(to, pgdat);
  4855. lock_page_memcg(page);
  4856. if (PageAnon(page)) {
  4857. if (page_mapped(page)) {
  4858. __mod_lruvec_state(from_vec, NR_ANON_MAPPED, -nr_pages);
  4859. __mod_lruvec_state(to_vec, NR_ANON_MAPPED, nr_pages);
  4860. if (PageTransHuge(page)) {
  4861. __dec_lruvec_state(from_vec, NR_ANON_THPS);
  4862. __inc_lruvec_state(to_vec, NR_ANON_THPS);
  4863. }
  4864. }
  4865. } else {
  4866. __mod_lruvec_state(from_vec, NR_FILE_PAGES, -nr_pages);
  4867. __mod_lruvec_state(to_vec, NR_FILE_PAGES, nr_pages);
  4868. if (PageSwapBacked(page)) {
  4869. __mod_lruvec_state(from_vec, NR_SHMEM, -nr_pages);
  4870. __mod_lruvec_state(to_vec, NR_SHMEM, nr_pages);
  4871. }
  4872. if (page_mapped(page)) {
  4873. __mod_lruvec_state(from_vec, NR_FILE_MAPPED, -nr_pages);
  4874. __mod_lruvec_state(to_vec, NR_FILE_MAPPED, nr_pages);
  4875. }
  4876. if (PageDirty(page)) {
  4877. struct address_space *mapping = page_mapping(page);
  4878. if (mapping_can_writeback(mapping)) {
  4879. __mod_lruvec_state(from_vec, NR_FILE_DIRTY,
  4880. -nr_pages);
  4881. __mod_lruvec_state(to_vec, NR_FILE_DIRTY,
  4882. nr_pages);
  4883. }
  4884. }
  4885. }
  4886. if (PageWriteback(page)) {
  4887. __mod_lruvec_state(from_vec, NR_WRITEBACK, -nr_pages);
  4888. __mod_lruvec_state(to_vec, NR_WRITEBACK, nr_pages);
  4889. }
  4890. /*
  4891. * All state has been migrated, let's switch to the new memcg.
  4892. *
  4893. * It is safe to change page->mem_cgroup here because the page
  4894. * is referenced, charged, isolated, and locked: we can't race
  4895. * with (un)charging, migration, LRU putback, or anything else
  4896. * that would rely on a stable page->mem_cgroup.
  4897. *
  4898. * Note that lock_page_memcg is a memcg lock, not a page lock,
  4899. * to save space. As soon as we switch page->mem_cgroup to a
  4900. * new memcg that isn't locked, the above state can change
  4901. * concurrently again. Make sure we're truly done with it.
  4902. */
  4903. smp_mb();
  4904. css_get(&to->css);
  4905. css_put(&from->css);
  4906. page->mem_cgroup = to;
  4907. __unlock_page_memcg(from);
  4908. ret = 0;
  4909. local_irq_disable();
  4910. mem_cgroup_charge_statistics(to, page, nr_pages);
  4911. memcg_check_events(to, page);
  4912. mem_cgroup_charge_statistics(from, page, -nr_pages);
  4913. memcg_check_events(from, page);
  4914. local_irq_enable();
  4915. out_unlock:
  4916. unlock_page(page);
  4917. out:
  4918. return ret;
  4919. }
  4920. /**
  4921. * get_mctgt_type - get target type of moving charge
  4922. * @vma: the vma the pte to be checked belongs
  4923. * @addr: the address corresponding to the pte to be checked
  4924. * @ptent: the pte to be checked
  4925. * @target: the pointer the target page or swap ent will be stored(can be NULL)
  4926. *
  4927. * Returns
  4928. * 0(MC_TARGET_NONE): if the pte is not a target for move charge.
  4929. * 1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
  4930. * move charge. if @target is not NULL, the page is stored in target->page
  4931. * with extra refcnt got(Callers should handle it).
  4932. * 2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
  4933. * target for charge migration. if @target is not NULL, the entry is stored
  4934. * in target->ent.
  4935. * 3(MC_TARGET_DEVICE): like MC_TARGET_PAGE but page is MEMORY_DEVICE_PRIVATE
  4936. * (so ZONE_DEVICE page and thus not on the lru).
  4937. * For now we such page is charge like a regular page would be as for all
  4938. * intent and purposes it is just special memory taking the place of a
  4939. * regular page.
  4940. *
  4941. * See Documentations/vm/hmm.txt and include/linux/hmm.h
  4942. *
  4943. * Called with pte lock held.
  4944. */
  4945. static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
  4946. unsigned long addr, pte_t ptent, union mc_target *target)
  4947. {
  4948. struct page *page = NULL;
  4949. enum mc_target_type ret = MC_TARGET_NONE;
  4950. swp_entry_t ent = { .val = 0 };
  4951. if (pte_present(ptent))
  4952. page = mc_handle_present_pte(vma, addr, ptent);
  4953. else if (is_swap_pte(ptent))
  4954. page = mc_handle_swap_pte(vma, ptent, &ent);
  4955. else if (pte_none(ptent))
  4956. page = mc_handle_file_pte(vma, addr, ptent, &ent);
  4957. if (!page && !ent.val)
  4958. return ret;
  4959. if (page) {
  4960. /*
  4961. * Do only loose check w/o serialization.
  4962. * mem_cgroup_move_account() checks the page is valid or
  4963. * not under LRU exclusion.
  4964. */
  4965. if (page->mem_cgroup == mc.from) {
  4966. ret = MC_TARGET_PAGE;
  4967. if (is_device_private_page(page))
  4968. ret = MC_TARGET_DEVICE;
  4969. if (target)
  4970. target->page = page;
  4971. }
  4972. if (!ret || !target)
  4973. put_page(page);
  4974. }
  4975. /*
  4976. * There is a swap entry and a page doesn't exist or isn't charged.
  4977. * But we cannot move a tail-page in a THP.
  4978. */
  4979. if (ent.val && !ret && (!page || !PageTransCompound(page)) &&
  4980. mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) {
  4981. ret = MC_TARGET_SWAP;
  4982. if (target)
  4983. target->ent = ent;
  4984. }
  4985. return ret;
  4986. }
  4987. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  4988. /*
  4989. * We don't consider PMD mapped swapping or file mapped pages because THP does
  4990. * not support them for now.
  4991. * Caller should make sure that pmd_trans_huge(pmd) is true.
  4992. */
  4993. static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
  4994. unsigned long addr, pmd_t pmd, union mc_target *target)
  4995. {
  4996. struct page *page = NULL;
  4997. enum mc_target_type ret = MC_TARGET_NONE;
  4998. if (unlikely(is_swap_pmd(pmd))) {
  4999. VM_BUG_ON(thp_migration_supported() &&
  5000. !is_pmd_migration_entry(pmd));
  5001. return ret;
  5002. }
  5003. page = pmd_page(pmd);
  5004. VM_BUG_ON_PAGE(!page || !PageHead(page), page);
  5005. if (!(mc.flags & MOVE_ANON))
  5006. return ret;
  5007. if (page->mem_cgroup == mc.from) {
  5008. ret = MC_TARGET_PAGE;
  5009. if (target) {
  5010. get_page(page);
  5011. target->page = page;
  5012. }
  5013. }
  5014. return ret;
  5015. }
  5016. #else
  5017. static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
  5018. unsigned long addr, pmd_t pmd, union mc_target *target)
  5019. {
  5020. return MC_TARGET_NONE;
  5021. }
  5022. #endif
  5023. static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
  5024. unsigned long addr, unsigned long end,
  5025. struct mm_walk *walk)
  5026. {
  5027. struct vm_area_struct *vma = walk->vma;
  5028. pte_t *pte;
  5029. spinlock_t *ptl;
  5030. ptl = pmd_trans_huge_lock(pmd, vma);
  5031. if (ptl) {
  5032. /*
  5033. * Note their can not be MC_TARGET_DEVICE for now as we do not
  5034. * support transparent huge page with MEMORY_DEVICE_PRIVATE but
  5035. * this might change.
  5036. */
  5037. if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
  5038. mc.precharge += HPAGE_PMD_NR;
  5039. spin_unlock(ptl);
  5040. return 0;
  5041. }
  5042. if (pmd_trans_unstable(pmd))
  5043. return 0;
  5044. pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
  5045. for (; addr != end; pte++, addr += PAGE_SIZE)
  5046. if (get_mctgt_type(vma, addr, *pte, NULL))
  5047. mc.precharge++; /* increment precharge temporarily */
  5048. pte_unmap_unlock(pte - 1, ptl);
  5049. cond_resched();
  5050. return 0;
  5051. }
  5052. static const struct mm_walk_ops precharge_walk_ops = {
  5053. .pmd_entry = mem_cgroup_count_precharge_pte_range,
  5054. };
  5055. static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
  5056. {
  5057. unsigned long precharge;
  5058. mmap_read_lock(mm);
  5059. walk_page_range(mm, 0, mm->highest_vm_end, &precharge_walk_ops, NULL);
  5060. mmap_read_unlock(mm);
  5061. precharge = mc.precharge;
  5062. mc.precharge = 0;
  5063. return precharge;
  5064. }
  5065. static int mem_cgroup_precharge_mc(struct mm_struct *mm)
  5066. {
  5067. unsigned long precharge = mem_cgroup_count_precharge(mm);
  5068. VM_BUG_ON(mc.moving_task);
  5069. mc.moving_task = current;
  5070. return mem_cgroup_do_precharge(precharge);
  5071. }
  5072. /* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
  5073. static void __mem_cgroup_clear_mc(void)
  5074. {
  5075. struct mem_cgroup *from = mc.from;
  5076. struct mem_cgroup *to = mc.to;
  5077. /* we must uncharge all the leftover precharges from mc.to */
  5078. if (mc.precharge) {
  5079. cancel_charge(mc.to, mc.precharge);
  5080. mc.precharge = 0;
  5081. }
  5082. /*
  5083. * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
  5084. * we must uncharge here.
  5085. */
  5086. if (mc.moved_charge) {
  5087. cancel_charge(mc.from, mc.moved_charge);
  5088. mc.moved_charge = 0;
  5089. }
  5090. /* we must fixup refcnts and charges */
  5091. if (mc.moved_swap) {
  5092. /* uncharge swap account from the old cgroup */
  5093. if (!mem_cgroup_is_root(mc.from))
  5094. page_counter_uncharge(&mc.from->memsw, mc.moved_swap);
  5095. mem_cgroup_id_put_many(mc.from, mc.moved_swap);
  5096. /*
  5097. * we charged both to->memory and to->memsw, so we
  5098. * should uncharge to->memory.
  5099. */
  5100. if (!mem_cgroup_is_root(mc.to))
  5101. page_counter_uncharge(&mc.to->memory, mc.moved_swap);
  5102. mc.moved_swap = 0;
  5103. }
  5104. memcg_oom_recover(from);
  5105. memcg_oom_recover(to);
  5106. wake_up_all(&mc.waitq);
  5107. }
  5108. static void mem_cgroup_clear_mc(void)
  5109. {
  5110. struct mm_struct *mm = mc.mm;
  5111. /*
  5112. * we must clear moving_task before waking up waiters at the end of
  5113. * task migration.
  5114. */
  5115. mc.moving_task = NULL;
  5116. __mem_cgroup_clear_mc();
  5117. spin_lock(&mc.lock);
  5118. mc.from = NULL;
  5119. mc.to = NULL;
  5120. mc.mm = NULL;
  5121. spin_unlock(&mc.lock);
  5122. mmput(mm);
  5123. }
  5124. static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
  5125. {
  5126. struct cgroup_subsys_state *css;
  5127. struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */
  5128. struct mem_cgroup *from;
  5129. struct task_struct *leader, *p;
  5130. struct mm_struct *mm;
  5131. unsigned long move_flags;
  5132. int ret = 0;
  5133. /* charge immigration isn't supported on the default hierarchy */
  5134. if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
  5135. return 0;
  5136. /*
  5137. * Multi-process migrations only happen on the default hierarchy
  5138. * where charge immigration is not used. Perform charge
  5139. * immigration if @tset contains a leader and whine if there are
  5140. * multiple.
  5141. */
  5142. p = NULL;
  5143. cgroup_taskset_for_each_leader(leader, css, tset) {
  5144. WARN_ON_ONCE(p);
  5145. p = leader;
  5146. memcg = mem_cgroup_from_css(css);
  5147. }
  5148. if (!p)
  5149. return 0;
  5150. /*
  5151. * We are now commited to this value whatever it is. Changes in this
  5152. * tunable will only affect upcoming migrations, not the current one.
  5153. * So we need to save it, and keep it going.
  5154. */
  5155. move_flags = READ_ONCE(memcg->move_charge_at_immigrate);
  5156. if (!move_flags)
  5157. return 0;
  5158. from = mem_cgroup_from_task(p);
  5159. VM_BUG_ON(from == memcg);
  5160. mm = get_task_mm(p);
  5161. if (!mm)
  5162. return 0;
  5163. /* We move charges only when we move a owner of the mm */
  5164. if (mm->owner == p) {
  5165. VM_BUG_ON(mc.from);
  5166. VM_BUG_ON(mc.to);
  5167. VM_BUG_ON(mc.precharge);
  5168. VM_BUG_ON(mc.moved_charge);
  5169. VM_BUG_ON(mc.moved_swap);
  5170. spin_lock(&mc.lock);
  5171. mc.mm = mm;
  5172. mc.from = from;
  5173. mc.to = memcg;
  5174. mc.flags = move_flags;
  5175. spin_unlock(&mc.lock);
  5176. /* We set mc.moving_task later */
  5177. ret = mem_cgroup_precharge_mc(mm);
  5178. if (ret)
  5179. mem_cgroup_clear_mc();
  5180. } else {
  5181. mmput(mm);
  5182. }
  5183. return ret;
  5184. }
  5185. static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
  5186. {
  5187. if (mc.to)
  5188. mem_cgroup_clear_mc();
  5189. }
  5190. static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
  5191. unsigned long addr, unsigned long end,
  5192. struct mm_walk *walk)
  5193. {
  5194. int ret = 0;
  5195. struct vm_area_struct *vma = walk->vma;
  5196. pte_t *pte;
  5197. spinlock_t *ptl;
  5198. enum mc_target_type target_type;
  5199. union mc_target target;
  5200. struct page *page;
  5201. ptl = pmd_trans_huge_lock(pmd, vma);
  5202. if (ptl) {
  5203. if (mc.precharge < HPAGE_PMD_NR) {
  5204. spin_unlock(ptl);
  5205. return 0;
  5206. }
  5207. target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
  5208. if (target_type == MC_TARGET_PAGE) {
  5209. page = target.page;
  5210. if (!isolate_lru_page(page)) {
  5211. if (!mem_cgroup_move_account(page, true,
  5212. mc.from, mc.to)) {
  5213. mc.precharge -= HPAGE_PMD_NR;
  5214. mc.moved_charge += HPAGE_PMD_NR;
  5215. }
  5216. putback_lru_page(page);
  5217. }
  5218. put_page(page);
  5219. } else if (target_type == MC_TARGET_DEVICE) {
  5220. page = target.page;
  5221. if (!mem_cgroup_move_account(page, true,
  5222. mc.from, mc.to)) {
  5223. mc.precharge -= HPAGE_PMD_NR;
  5224. mc.moved_charge += HPAGE_PMD_NR;
  5225. }
  5226. put_page(page);
  5227. }
  5228. spin_unlock(ptl);
  5229. return 0;
  5230. }
  5231. if (pmd_trans_unstable(pmd))
  5232. return 0;
  5233. retry:
  5234. pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
  5235. for (; addr != end; addr += PAGE_SIZE) {
  5236. pte_t ptent = *(pte++);
  5237. bool device = false;
  5238. swp_entry_t ent;
  5239. if (!mc.precharge)
  5240. break;
  5241. switch (get_mctgt_type(vma, addr, ptent, &target)) {
  5242. case MC_TARGET_DEVICE:
  5243. device = true;
  5244. fallthrough;
  5245. case MC_TARGET_PAGE:
  5246. page = target.page;
  5247. /*
  5248. * We can have a part of the split pmd here. Moving it
  5249. * can be done but it would be too convoluted so simply
  5250. * ignore such a partial THP and keep it in original
  5251. * memcg. There should be somebody mapping the head.
  5252. */
  5253. if (PageTransCompound(page))
  5254. goto put;
  5255. if (!device && isolate_lru_page(page))
  5256. goto put;
  5257. if (!mem_cgroup_move_account(page, false,
  5258. mc.from, mc.to)) {
  5259. mc.precharge--;
  5260. /* we uncharge from mc.from later. */
  5261. mc.moved_charge++;
  5262. }
  5263. if (!device)
  5264. putback_lru_page(page);
  5265. put: /* get_mctgt_type() gets the page */
  5266. put_page(page);
  5267. break;
  5268. case MC_TARGET_SWAP:
  5269. ent = target.ent;
  5270. if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
  5271. mc.precharge--;
  5272. mem_cgroup_id_get_many(mc.to, 1);
  5273. /* we fixup other refcnts and charges later. */
  5274. mc.moved_swap++;
  5275. }
  5276. break;
  5277. default:
  5278. break;
  5279. }
  5280. }
  5281. pte_unmap_unlock(pte - 1, ptl);
  5282. cond_resched();
  5283. if (addr != end) {
  5284. /*
  5285. * We have consumed all precharges we got in can_attach().
  5286. * We try charge one by one, but don't do any additional
  5287. * charges to mc.to if we have failed in charge once in attach()
  5288. * phase.
  5289. */
  5290. ret = mem_cgroup_do_precharge(1);
  5291. if (!ret)
  5292. goto retry;
  5293. }
  5294. return ret;
  5295. }
  5296. static const struct mm_walk_ops charge_walk_ops = {
  5297. .pmd_entry = mem_cgroup_move_charge_pte_range,
  5298. };
  5299. static void mem_cgroup_move_charge(void)
  5300. {
  5301. lru_add_drain_all();
  5302. /*
  5303. * Signal lock_page_memcg() to take the memcg's move_lock
  5304. * while we're moving its pages to another memcg. Then wait
  5305. * for already started RCU-only updates to finish.
  5306. */
  5307. atomic_inc(&mc.from->moving_account);
  5308. synchronize_rcu();
  5309. retry:
  5310. if (unlikely(!mmap_read_trylock(mc.mm))) {
  5311. /*
  5312. * Someone who are holding the mmap_lock might be waiting in
  5313. * waitq. So we cancel all extra charges, wake up all waiters,
  5314. * and retry. Because we cancel precharges, we might not be able
  5315. * to move enough charges, but moving charge is a best-effort
  5316. * feature anyway, so it wouldn't be a big problem.
  5317. */
  5318. __mem_cgroup_clear_mc();
  5319. cond_resched();
  5320. goto retry;
  5321. }
  5322. /*
  5323. * When we have consumed all precharges and failed in doing
  5324. * additional charge, the page walk just aborts.
  5325. */
  5326. walk_page_range(mc.mm, 0, mc.mm->highest_vm_end, &charge_walk_ops,
  5327. NULL);
  5328. mmap_read_unlock(mc.mm);
  5329. atomic_dec(&mc.from->moving_account);
  5330. }
  5331. static void mem_cgroup_move_task(void)
  5332. {
  5333. if (mc.to) {
  5334. mem_cgroup_move_charge();
  5335. mem_cgroup_clear_mc();
  5336. }
  5337. }
  5338. #else /* !CONFIG_MMU */
  5339. static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
  5340. {
  5341. return 0;
  5342. }
  5343. static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
  5344. {
  5345. }
  5346. static void mem_cgroup_move_task(void)
  5347. {
  5348. }
  5349. #endif
  5350. /*
  5351. * Cgroup retains root cgroups across [un]mount cycles making it necessary
  5352. * to verify whether we're attached to the default hierarchy on each mount
  5353. * attempt.
  5354. */
  5355. static void mem_cgroup_bind(struct cgroup_subsys_state *root_css)
  5356. {
  5357. /*
  5358. * use_hierarchy is forced on the default hierarchy. cgroup core
  5359. * guarantees that @root doesn't have any children, so turning it
  5360. * on for the root memcg is enough.
  5361. */
  5362. if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
  5363. root_mem_cgroup->use_hierarchy = true;
  5364. else
  5365. root_mem_cgroup->use_hierarchy = false;
  5366. }
  5367. static int seq_puts_memcg_tunable(struct seq_file *m, unsigned long value)
  5368. {
  5369. if (value == PAGE_COUNTER_MAX)
  5370. seq_puts(m, "max\n");
  5371. else
  5372. seq_printf(m, "%llu\n", (u64)value * PAGE_SIZE);
  5373. return 0;
  5374. }
  5375. static u64 memory_current_read(struct cgroup_subsys_state *css,
  5376. struct cftype *cft)
  5377. {
  5378. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  5379. return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE;
  5380. }
  5381. static int memory_min_show(struct seq_file *m, void *v)
  5382. {
  5383. return seq_puts_memcg_tunable(m,
  5384. READ_ONCE(mem_cgroup_from_seq(m)->memory.min));
  5385. }
  5386. static ssize_t memory_min_write(struct kernfs_open_file *of,
  5387. char *buf, size_t nbytes, loff_t off)
  5388. {
  5389. struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
  5390. unsigned long min;
  5391. int err;
  5392. buf = strstrip(buf);
  5393. err = page_counter_memparse(buf, "max", &min);
  5394. if (err)
  5395. return err;
  5396. page_counter_set_min(&memcg->memory, min);
  5397. return nbytes;
  5398. }
  5399. static int memory_low_show(struct seq_file *m, void *v)
  5400. {
  5401. return seq_puts_memcg_tunable(m,
  5402. READ_ONCE(mem_cgroup_from_seq(m)->memory.low));
  5403. }
  5404. static ssize_t memory_low_write(struct kernfs_open_file *of,
  5405. char *buf, size_t nbytes, loff_t off)
  5406. {
  5407. struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
  5408. unsigned long low;
  5409. int err;
  5410. buf = strstrip(buf);
  5411. err = page_counter_memparse(buf, "max", &low);
  5412. if (err)
  5413. return err;
  5414. page_counter_set_low(&memcg->memory, low);
  5415. return nbytes;
  5416. }
  5417. static int memory_high_show(struct seq_file *m, void *v)
  5418. {
  5419. return seq_puts_memcg_tunable(m,
  5420. READ_ONCE(mem_cgroup_from_seq(m)->memory.high));
  5421. }
  5422. static ssize_t memory_high_write(struct kernfs_open_file *of,
  5423. char *buf, size_t nbytes, loff_t off)
  5424. {
  5425. struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
  5426. unsigned int nr_retries = MAX_RECLAIM_RETRIES;
  5427. bool drained = false;
  5428. unsigned long high;
  5429. int err;
  5430. buf = strstrip(buf);
  5431. err = page_counter_memparse(buf, "max", &high);
  5432. if (err)
  5433. return err;
  5434. page_counter_set_high(&memcg->memory, high);
  5435. for (;;) {
  5436. unsigned long nr_pages = page_counter_read(&memcg->memory);
  5437. unsigned long reclaimed;
  5438. if (nr_pages <= high)
  5439. break;
  5440. if (signal_pending(current))
  5441. break;
  5442. if (!drained) {
  5443. drain_all_stock(memcg);
  5444. drained = true;
  5445. continue;
  5446. }
  5447. reclaimed = try_to_free_mem_cgroup_pages(memcg, nr_pages - high,
  5448. GFP_KERNEL, true);
  5449. if (!reclaimed && !nr_retries--)
  5450. break;
  5451. }
  5452. memcg_wb_domain_size_changed(memcg);
  5453. return nbytes;
  5454. }
  5455. static int memory_max_show(struct seq_file *m, void *v)
  5456. {
  5457. return seq_puts_memcg_tunable(m,
  5458. READ_ONCE(mem_cgroup_from_seq(m)->memory.max));
  5459. }
  5460. static ssize_t memory_max_write(struct kernfs_open_file *of,
  5461. char *buf, size_t nbytes, loff_t off)
  5462. {
  5463. struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
  5464. unsigned int nr_reclaims = MAX_RECLAIM_RETRIES;
  5465. bool drained = false;
  5466. unsigned long max;
  5467. int err;
  5468. buf = strstrip(buf);
  5469. err = page_counter_memparse(buf, "max", &max);
  5470. if (err)
  5471. return err;
  5472. xchg(&memcg->memory.max, max);
  5473. for (;;) {
  5474. unsigned long nr_pages = page_counter_read(&memcg->memory);
  5475. if (nr_pages <= max)
  5476. break;
  5477. if (signal_pending(current))
  5478. break;
  5479. if (!drained) {
  5480. drain_all_stock(memcg);
  5481. drained = true;
  5482. continue;
  5483. }
  5484. if (nr_reclaims) {
  5485. if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max,
  5486. GFP_KERNEL, true))
  5487. nr_reclaims--;
  5488. continue;
  5489. }
  5490. memcg_memory_event(memcg, MEMCG_OOM);
  5491. if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0))
  5492. break;
  5493. }
  5494. memcg_wb_domain_size_changed(memcg);
  5495. return nbytes;
  5496. }
  5497. static void __memory_events_show(struct seq_file *m, atomic_long_t *events)
  5498. {
  5499. seq_printf(m, "low %lu\n", atomic_long_read(&events[MEMCG_LOW]));
  5500. seq_printf(m, "high %lu\n", atomic_long_read(&events[MEMCG_HIGH]));
  5501. seq_printf(m, "max %lu\n", atomic_long_read(&events[MEMCG_MAX]));
  5502. seq_printf(m, "oom %lu\n", atomic_long_read(&events[MEMCG_OOM]));
  5503. seq_printf(m, "oom_kill %lu\n",
  5504. atomic_long_read(&events[MEMCG_OOM_KILL]));
  5505. }
  5506. static int memory_events_show(struct seq_file *m, void *v)
  5507. {
  5508. struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
  5509. __memory_events_show(m, memcg->memory_events);
  5510. return 0;
  5511. }
  5512. static int memory_events_local_show(struct seq_file *m, void *v)
  5513. {
  5514. struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
  5515. __memory_events_show(m, memcg->memory_events_local);
  5516. return 0;
  5517. }
  5518. static int memory_stat_show(struct seq_file *m, void *v)
  5519. {
  5520. struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
  5521. char *buf;
  5522. buf = memory_stat_format(memcg);
  5523. if (!buf)
  5524. return -ENOMEM;
  5525. seq_puts(m, buf);
  5526. kfree(buf);
  5527. return 0;
  5528. }
  5529. #ifdef CONFIG_NUMA
  5530. static int memory_numa_stat_show(struct seq_file *m, void *v)
  5531. {
  5532. int i;
  5533. struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
  5534. for (i = 0; i < ARRAY_SIZE(memory_stats); i++) {
  5535. int nid;
  5536. if (memory_stats[i].idx >= NR_VM_NODE_STAT_ITEMS)
  5537. continue;
  5538. seq_printf(m, "%s", memory_stats[i].name);
  5539. for_each_node_state(nid, N_MEMORY) {
  5540. u64 size;
  5541. struct lruvec *lruvec;
  5542. lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
  5543. size = lruvec_page_state(lruvec, memory_stats[i].idx);
  5544. size *= memory_stats[i].ratio;
  5545. seq_printf(m, " N%d=%llu", nid, size);
  5546. }
  5547. seq_putc(m, '\n');
  5548. }
  5549. return 0;
  5550. }
  5551. #endif
  5552. static int memory_oom_group_show(struct seq_file *m, void *v)
  5553. {
  5554. struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
  5555. seq_printf(m, "%d\n", memcg->oom_group);
  5556. return 0;
  5557. }
  5558. static ssize_t memory_oom_group_write(struct kernfs_open_file *of,
  5559. char *buf, size_t nbytes, loff_t off)
  5560. {
  5561. struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
  5562. int ret, oom_group;
  5563. buf = strstrip(buf);
  5564. if (!buf)
  5565. return -EINVAL;
  5566. ret = kstrtoint(buf, 0, &oom_group);
  5567. if (ret)
  5568. return ret;
  5569. if (oom_group != 0 && oom_group != 1)
  5570. return -EINVAL;
  5571. memcg->oom_group = oom_group;
  5572. return nbytes;
  5573. }
  5574. static struct cftype memory_files[] = {
  5575. {
  5576. .name = "current",
  5577. .flags = CFTYPE_NOT_ON_ROOT,
  5578. .read_u64 = memory_current_read,
  5579. },
  5580. {
  5581. .name = "min",
  5582. .flags = CFTYPE_NOT_ON_ROOT,
  5583. .seq_show = memory_min_show,
  5584. .write = memory_min_write,
  5585. },
  5586. {
  5587. .name = "low",
  5588. .flags = CFTYPE_NOT_ON_ROOT,
  5589. .seq_show = memory_low_show,
  5590. .write = memory_low_write,
  5591. },
  5592. {
  5593. .name = "high",
  5594. .flags = CFTYPE_NOT_ON_ROOT,
  5595. .seq_show = memory_high_show,
  5596. .write = memory_high_write,
  5597. },
  5598. {
  5599. .name = "max",
  5600. .flags = CFTYPE_NOT_ON_ROOT,
  5601. .seq_show = memory_max_show,
  5602. .write = memory_max_write,
  5603. },
  5604. {
  5605. .name = "events",
  5606. .flags = CFTYPE_NOT_ON_ROOT,
  5607. .file_offset = offsetof(struct mem_cgroup, events_file),
  5608. .seq_show = memory_events_show,
  5609. },
  5610. {
  5611. .name = "events.local",
  5612. .flags = CFTYPE_NOT_ON_ROOT,
  5613. .file_offset = offsetof(struct mem_cgroup, events_local_file),
  5614. .seq_show = memory_events_local_show,
  5615. },
  5616. {
  5617. .name = "stat",
  5618. .seq_show = memory_stat_show,
  5619. },
  5620. #ifdef CONFIG_NUMA
  5621. {
  5622. .name = "numa_stat",
  5623. .seq_show = memory_numa_stat_show,
  5624. },
  5625. #endif
  5626. {
  5627. .name = "oom.group",
  5628. .flags = CFTYPE_NOT_ON_ROOT | CFTYPE_NS_DELEGATABLE,
  5629. .seq_show = memory_oom_group_show,
  5630. .write = memory_oom_group_write,
  5631. },
  5632. { } /* terminate */
  5633. };
  5634. struct cgroup_subsys memory_cgrp_subsys = {
  5635. .css_alloc = mem_cgroup_css_alloc,
  5636. .css_online = mem_cgroup_css_online,
  5637. .css_offline = mem_cgroup_css_offline,
  5638. .css_released = mem_cgroup_css_released,
  5639. .css_free = mem_cgroup_css_free,
  5640. .css_reset = mem_cgroup_css_reset,
  5641. .can_attach = mem_cgroup_can_attach,
  5642. .cancel_attach = mem_cgroup_cancel_attach,
  5643. .post_attach = mem_cgroup_move_task,
  5644. .bind = mem_cgroup_bind,
  5645. .dfl_cftypes = memory_files,
  5646. .legacy_cftypes = mem_cgroup_legacy_files,
  5647. .early_init = 0,
  5648. };
  5649. /*
  5650. * This function calculates an individual cgroup's effective
  5651. * protection which is derived from its own memory.min/low, its
  5652. * parent's and siblings' settings, as well as the actual memory
  5653. * distribution in the tree.
  5654. *
  5655. * The following rules apply to the effective protection values:
  5656. *
  5657. * 1. At the first level of reclaim, effective protection is equal to
  5658. * the declared protection in memory.min and memory.low.
  5659. *
  5660. * 2. To enable safe delegation of the protection configuration, at
  5661. * subsequent levels the effective protection is capped to the
  5662. * parent's effective protection.
  5663. *
  5664. * 3. To make complex and dynamic subtrees easier to configure, the
  5665. * user is allowed to overcommit the declared protection at a given
  5666. * level. If that is the case, the parent's effective protection is
  5667. * distributed to the children in proportion to how much protection
  5668. * they have declared and how much of it they are utilizing.
  5669. *
  5670. * This makes distribution proportional, but also work-conserving:
  5671. * if one cgroup claims much more protection than it uses memory,
  5672. * the unused remainder is available to its siblings.
  5673. *
  5674. * 4. Conversely, when the declared protection is undercommitted at a
  5675. * given level, the distribution of the larger parental protection
  5676. * budget is NOT proportional. A cgroup's protection from a sibling
  5677. * is capped to its own memory.min/low setting.
  5678. *
  5679. * 5. However, to allow protecting recursive subtrees from each other
  5680. * without having to declare each individual cgroup's fixed share
  5681. * of the ancestor's claim to protection, any unutilized -
  5682. * "floating" - protection from up the tree is distributed in
  5683. * proportion to each cgroup's *usage*. This makes the protection
  5684. * neutral wrt sibling cgroups and lets them compete freely over
  5685. * the shared parental protection budget, but it protects the
  5686. * subtree as a whole from neighboring subtrees.
  5687. *
  5688. * Note that 4. and 5. are not in conflict: 4. is about protecting
  5689. * against immediate siblings whereas 5. is about protecting against
  5690. * neighboring subtrees.
  5691. */
  5692. static unsigned long effective_protection(unsigned long usage,
  5693. unsigned long parent_usage,
  5694. unsigned long setting,
  5695. unsigned long parent_effective,
  5696. unsigned long siblings_protected)
  5697. {
  5698. unsigned long protected;
  5699. unsigned long ep;
  5700. protected = min(usage, setting);
  5701. /*
  5702. * If all cgroups at this level combined claim and use more
  5703. * protection then what the parent affords them, distribute
  5704. * shares in proportion to utilization.
  5705. *
  5706. * We are using actual utilization rather than the statically
  5707. * claimed protection in order to be work-conserving: claimed
  5708. * but unused protection is available to siblings that would
  5709. * otherwise get a smaller chunk than what they claimed.
  5710. */
  5711. if (siblings_protected > parent_effective)
  5712. return protected * parent_effective / siblings_protected;
  5713. /*
  5714. * Ok, utilized protection of all children is within what the
  5715. * parent affords them, so we know whatever this child claims
  5716. * and utilizes is effectively protected.
  5717. *
  5718. * If there is unprotected usage beyond this value, reclaim
  5719. * will apply pressure in proportion to that amount.
  5720. *
  5721. * If there is unutilized protection, the cgroup will be fully
  5722. * shielded from reclaim, but we do return a smaller value for
  5723. * protection than what the group could enjoy in theory. This
  5724. * is okay. With the overcommit distribution above, effective
  5725. * protection is always dependent on how memory is actually
  5726. * consumed among the siblings anyway.
  5727. */
  5728. ep = protected;
  5729. /*
  5730. * If the children aren't claiming (all of) the protection
  5731. * afforded to them by the parent, distribute the remainder in
  5732. * proportion to the (unprotected) memory of each cgroup. That
  5733. * way, cgroups that aren't explicitly prioritized wrt each
  5734. * other compete freely over the allowance, but they are
  5735. * collectively protected from neighboring trees.
  5736. *
  5737. * We're using unprotected memory for the weight so that if
  5738. * some cgroups DO claim explicit protection, we don't protect
  5739. * the same bytes twice.
  5740. *
  5741. * Check both usage and parent_usage against the respective
  5742. * protected values. One should imply the other, but they
  5743. * aren't read atomically - make sure the division is sane.
  5744. */
  5745. if (!(cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_RECURSIVE_PROT))
  5746. return ep;
  5747. if (parent_effective > siblings_protected &&
  5748. parent_usage > siblings_protected &&
  5749. usage > protected) {
  5750. unsigned long unclaimed;
  5751. unclaimed = parent_effective - siblings_protected;
  5752. unclaimed *= usage - protected;
  5753. unclaimed /= parent_usage - siblings_protected;
  5754. ep += unclaimed;
  5755. }
  5756. return ep;
  5757. }
  5758. /**
  5759. * mem_cgroup_protected - check if memory consumption is in the normal range
  5760. * @root: the top ancestor of the sub-tree being checked
  5761. * @memcg: the memory cgroup to check
  5762. *
  5763. * WARNING: This function is not stateless! It can only be used as part
  5764. * of a top-down tree iteration, not for isolated queries.
  5765. */
  5766. void mem_cgroup_calculate_protection(struct mem_cgroup *root,
  5767. struct mem_cgroup *memcg)
  5768. {
  5769. unsigned long usage, parent_usage;
  5770. struct mem_cgroup *parent;
  5771. if (mem_cgroup_disabled())
  5772. return;
  5773. if (!root)
  5774. root = root_mem_cgroup;
  5775. /*
  5776. * Effective values of the reclaim targets are ignored so they
  5777. * can be stale. Have a look at mem_cgroup_protection for more
  5778. * details.
  5779. * TODO: calculation should be more robust so that we do not need
  5780. * that special casing.
  5781. */
  5782. if (memcg == root)
  5783. return;
  5784. usage = page_counter_read(&memcg->memory);
  5785. if (!usage)
  5786. return;
  5787. parent = parent_mem_cgroup(memcg);
  5788. /* No parent means a non-hierarchical mode on v1 memcg */
  5789. if (!parent)
  5790. return;
  5791. if (parent == root) {
  5792. memcg->memory.emin = READ_ONCE(memcg->memory.min);
  5793. memcg->memory.elow = READ_ONCE(memcg->memory.low);
  5794. return;
  5795. }
  5796. parent_usage = page_counter_read(&parent->memory);
  5797. WRITE_ONCE(memcg->memory.emin, effective_protection(usage, parent_usage,
  5798. READ_ONCE(memcg->memory.min),
  5799. READ_ONCE(parent->memory.emin),
  5800. atomic_long_read(&parent->memory.children_min_usage)));
  5801. WRITE_ONCE(memcg->memory.elow, effective_protection(usage, parent_usage,
  5802. READ_ONCE(memcg->memory.low),
  5803. READ_ONCE(parent->memory.elow),
  5804. atomic_long_read(&parent->memory.children_low_usage)));
  5805. }
  5806. /**
  5807. * __mem_cgroup_charge - charge a newly allocated page to a cgroup
  5808. * @page: page to charge
  5809. * @mm: mm context of the victim
  5810. * @gfp_mask: reclaim mode
  5811. *
  5812. * Try to charge @page to the memcg that @mm belongs to, reclaiming
  5813. * pages according to @gfp_mask if necessary.
  5814. *
  5815. * Returns 0 on success. Otherwise, an error code is returned.
  5816. */
  5817. int __mem_cgroup_charge(struct page *page, struct mm_struct *mm,
  5818. gfp_t gfp_mask)
  5819. {
  5820. unsigned int nr_pages = thp_nr_pages(page);
  5821. struct mem_cgroup *memcg = NULL;
  5822. int ret = 0;
  5823. if (PageSwapCache(page)) {
  5824. swp_entry_t ent = { .val = page_private(page), };
  5825. unsigned short id;
  5826. /*
  5827. * Every swap fault against a single page tries to charge the
  5828. * page, bail as early as possible. shmem_unuse() encounters
  5829. * already charged pages, too. page->mem_cgroup is protected
  5830. * by the page lock, which serializes swap cache removal, which
  5831. * in turn serializes uncharging.
  5832. */
  5833. VM_BUG_ON_PAGE(!PageLocked(page), page);
  5834. if (compound_head(page)->mem_cgroup)
  5835. goto out;
  5836. id = lookup_swap_cgroup_id(ent);
  5837. rcu_read_lock();
  5838. memcg = mem_cgroup_from_id(id);
  5839. if (memcg && !css_tryget_online(&memcg->css))
  5840. memcg = NULL;
  5841. rcu_read_unlock();
  5842. }
  5843. if (!memcg)
  5844. memcg = get_mem_cgroup_from_mm(mm);
  5845. ret = try_charge(memcg, gfp_mask, nr_pages);
  5846. if (ret)
  5847. goto out_put;
  5848. css_get(&memcg->css);
  5849. commit_charge(page, memcg);
  5850. local_irq_disable();
  5851. mem_cgroup_charge_statistics(memcg, page, nr_pages);
  5852. memcg_check_events(memcg, page);
  5853. local_irq_enable();
  5854. /*
  5855. * Cgroup1's unified memory+swap counter has been charged with the
  5856. * new swapcache page, finish the transfer by uncharging the swap
  5857. * slot. The swap slot would also get uncharged when it dies, but
  5858. * it can stick around indefinitely and we'd count the page twice
  5859. * the entire time.
  5860. *
  5861. * Cgroup2 has separate resource counters for memory and swap,
  5862. * so this is a non-issue here. Memory and swap charge lifetimes
  5863. * correspond 1:1 to page and swap slot lifetimes: we charge the
  5864. * page to memory here, and uncharge swap when the slot is freed.
  5865. */
  5866. if (do_memsw_account() && PageSwapCache(page)) {
  5867. swp_entry_t entry = { .val = page_private(page) };
  5868. /*
  5869. * The swap entry might not get freed for a long time,
  5870. * let's not wait for it. The page already received a
  5871. * memory+swap charge, drop the swap entry duplicate.
  5872. */
  5873. mem_cgroup_uncharge_swap(entry, nr_pages);
  5874. }
  5875. out_put:
  5876. css_put(&memcg->css);
  5877. out:
  5878. return ret;
  5879. }
  5880. struct uncharge_gather {
  5881. struct mem_cgroup *memcg;
  5882. unsigned long nr_pages;
  5883. unsigned long pgpgout;
  5884. unsigned long nr_kmem;
  5885. struct page *dummy_page;
  5886. };
  5887. static inline void uncharge_gather_clear(struct uncharge_gather *ug)
  5888. {
  5889. memset(ug, 0, sizeof(*ug));
  5890. }
  5891. static void uncharge_batch(const struct uncharge_gather *ug)
  5892. {
  5893. unsigned long flags;
  5894. if (!mem_cgroup_is_root(ug->memcg)) {
  5895. page_counter_uncharge(&ug->memcg->memory, ug->nr_pages);
  5896. if (do_memsw_account())
  5897. page_counter_uncharge(&ug->memcg->memsw, ug->nr_pages);
  5898. if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && ug->nr_kmem)
  5899. page_counter_uncharge(&ug->memcg->kmem, ug->nr_kmem);
  5900. memcg_oom_recover(ug->memcg);
  5901. }
  5902. local_irq_save(flags);
  5903. __count_memcg_events(ug->memcg, PGPGOUT, ug->pgpgout);
  5904. __this_cpu_add(ug->memcg->vmstats_percpu->nr_page_events, ug->nr_pages);
  5905. memcg_check_events(ug->memcg, ug->dummy_page);
  5906. local_irq_restore(flags);
  5907. /* drop reference from uncharge_page */
  5908. css_put(&ug->memcg->css);
  5909. }
  5910. static void uncharge_page(struct page *page, struct uncharge_gather *ug)
  5911. {
  5912. unsigned long nr_pages;
  5913. VM_BUG_ON_PAGE(PageLRU(page), page);
  5914. if (!page->mem_cgroup)
  5915. return;
  5916. /*
  5917. * Nobody should be changing or seriously looking at
  5918. * page->mem_cgroup at this point, we have fully
  5919. * exclusive access to the page.
  5920. */
  5921. if (ug->memcg != page->mem_cgroup) {
  5922. if (ug->memcg) {
  5923. uncharge_batch(ug);
  5924. uncharge_gather_clear(ug);
  5925. }
  5926. ug->memcg = page->mem_cgroup;
  5927. /* pairs with css_put in uncharge_batch */
  5928. css_get(&ug->memcg->css);
  5929. }
  5930. nr_pages = compound_nr(page);
  5931. ug->nr_pages += nr_pages;
  5932. if (!PageKmemcg(page)) {
  5933. ug->pgpgout++;
  5934. } else {
  5935. ug->nr_kmem += nr_pages;
  5936. __ClearPageKmemcg(page);
  5937. }
  5938. ug->dummy_page = page;
  5939. page->mem_cgroup = NULL;
  5940. css_put(&ug->memcg->css);
  5941. }
  5942. static void uncharge_list(struct list_head *page_list)
  5943. {
  5944. struct uncharge_gather ug;
  5945. struct list_head *next;
  5946. uncharge_gather_clear(&ug);
  5947. /*
  5948. * Note that the list can be a single page->lru; hence the
  5949. * do-while loop instead of a simple list_for_each_entry().
  5950. */
  5951. next = page_list->next;
  5952. do {
  5953. struct page *page;
  5954. page = list_entry(next, struct page, lru);
  5955. next = page->lru.next;
  5956. uncharge_page(page, &ug);
  5957. } while (next != page_list);
  5958. if (ug.memcg)
  5959. uncharge_batch(&ug);
  5960. }
  5961. /**
  5962. * __mem_cgroup_uncharge - uncharge a page
  5963. * @page: page to uncharge
  5964. *
  5965. * Uncharge a page previously charged with __mem_cgroup_charge().
  5966. */
  5967. void __mem_cgroup_uncharge(struct page *page)
  5968. {
  5969. struct uncharge_gather ug;
  5970. /* Don't touch page->lru of any random page, pre-check: */
  5971. if (!page->mem_cgroup)
  5972. return;
  5973. uncharge_gather_clear(&ug);
  5974. uncharge_page(page, &ug);
  5975. uncharge_batch(&ug);
  5976. }
  5977. /**
  5978. * __mem_cgroup_uncharge_list - uncharge a list of page
  5979. * @page_list: list of pages to uncharge
  5980. *
  5981. * Uncharge a list of pages previously charged with
  5982. * __mem_cgroup_charge().
  5983. */
  5984. void __mem_cgroup_uncharge_list(struct list_head *page_list)
  5985. {
  5986. if (!list_empty(page_list))
  5987. uncharge_list(page_list);
  5988. }
  5989. /**
  5990. * mem_cgroup_migrate - charge a page's replacement
  5991. * @oldpage: currently circulating page
  5992. * @newpage: replacement page
  5993. *
  5994. * Charge @newpage as a replacement page for @oldpage. @oldpage will
  5995. * be uncharged upon free.
  5996. *
  5997. * Both pages must be locked, @newpage->mapping must be set up.
  5998. */
  5999. void mem_cgroup_migrate(struct page *oldpage, struct page *newpage)
  6000. {
  6001. struct mem_cgroup *memcg;
  6002. unsigned int nr_pages;
  6003. unsigned long flags;
  6004. VM_BUG_ON_PAGE(!PageLocked(oldpage), oldpage);
  6005. VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
  6006. VM_BUG_ON_PAGE(PageAnon(oldpage) != PageAnon(newpage), newpage);
  6007. VM_BUG_ON_PAGE(PageTransHuge(oldpage) != PageTransHuge(newpage),
  6008. newpage);
  6009. if (mem_cgroup_disabled())
  6010. return;
  6011. /* Page cache replacement: new page already charged? */
  6012. if (newpage->mem_cgroup)
  6013. return;
  6014. /* Swapcache readahead pages can get replaced before being charged */
  6015. memcg = oldpage->mem_cgroup;
  6016. if (!memcg)
  6017. return;
  6018. /* Force-charge the new page. The old one will be freed soon */
  6019. nr_pages = thp_nr_pages(newpage);
  6020. page_counter_charge(&memcg->memory, nr_pages);
  6021. if (do_memsw_account())
  6022. page_counter_charge(&memcg->memsw, nr_pages);
  6023. css_get(&memcg->css);
  6024. commit_charge(newpage, memcg);
  6025. local_irq_save(flags);
  6026. mem_cgroup_charge_statistics(memcg, newpage, nr_pages);
  6027. memcg_check_events(memcg, newpage);
  6028. local_irq_restore(flags);
  6029. }
  6030. DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key);
  6031. EXPORT_SYMBOL(memcg_sockets_enabled_key);
  6032. void mem_cgroup_sk_alloc(struct sock *sk)
  6033. {
  6034. struct mem_cgroup *memcg;
  6035. if (!mem_cgroup_sockets_enabled)
  6036. return;
  6037. /* Do not associate the sock with unrelated interrupted task's memcg. */
  6038. if (in_interrupt())
  6039. return;
  6040. rcu_read_lock();
  6041. memcg = mem_cgroup_from_task(current);
  6042. if (memcg == root_mem_cgroup)
  6043. goto out;
  6044. if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active)
  6045. goto out;
  6046. if (css_tryget(&memcg->css))
  6047. sk->sk_memcg = memcg;
  6048. out:
  6049. rcu_read_unlock();
  6050. }
  6051. void mem_cgroup_sk_free(struct sock *sk)
  6052. {
  6053. if (sk->sk_memcg)
  6054. css_put(&sk->sk_memcg->css);
  6055. }
  6056. /**
  6057. * mem_cgroup_charge_skmem - charge socket memory
  6058. * @memcg: memcg to charge
  6059. * @nr_pages: number of pages to charge
  6060. *
  6061. * Charges @nr_pages to @memcg. Returns %true if the charge fit within
  6062. * @memcg's configured limit, %false if the charge had to be forced.
  6063. */
  6064. bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
  6065. {
  6066. gfp_t gfp_mask = GFP_KERNEL;
  6067. if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
  6068. struct page_counter *fail;
  6069. if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) {
  6070. memcg->tcpmem_pressure = 0;
  6071. return true;
  6072. }
  6073. page_counter_charge(&memcg->tcpmem, nr_pages);
  6074. memcg->tcpmem_pressure = 1;
  6075. return false;
  6076. }
  6077. /* Don't block in the packet receive path */
  6078. if (in_softirq())
  6079. gfp_mask = GFP_NOWAIT;
  6080. mod_memcg_state(memcg, MEMCG_SOCK, nr_pages);
  6081. if (try_charge(memcg, gfp_mask, nr_pages) == 0)
  6082. return true;
  6083. try_charge(memcg, gfp_mask|__GFP_NOFAIL, nr_pages);
  6084. return false;
  6085. }
  6086. /**
  6087. * mem_cgroup_uncharge_skmem - uncharge socket memory
  6088. * @memcg: memcg to uncharge
  6089. * @nr_pages: number of pages to uncharge
  6090. */
  6091. void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
  6092. {
  6093. if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
  6094. page_counter_uncharge(&memcg->tcpmem, nr_pages);
  6095. return;
  6096. }
  6097. mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages);
  6098. refill_stock(memcg, nr_pages);
  6099. }
  6100. static int __init cgroup_memory(char *s)
  6101. {
  6102. char *token;
  6103. while ((token = strsep(&s, ",")) != NULL) {
  6104. if (!*token)
  6105. continue;
  6106. if (!strcmp(token, "nosocket"))
  6107. cgroup_memory_nosocket = true;
  6108. if (!strcmp(token, "nokmem"))
  6109. cgroup_memory_nokmem = true;
  6110. }
  6111. return 1;
  6112. }
  6113. __setup("cgroup.memory=", cgroup_memory);
  6114. /*
  6115. * subsys_initcall() for memory controller.
  6116. *
  6117. * Some parts like memcg_hotplug_cpu_dead() have to be initialized from this
  6118. * context because of lock dependencies (cgroup_lock -> cpu hotplug) but
  6119. * basically everything that doesn't depend on a specific mem_cgroup structure
  6120. * should be initialized from here.
  6121. */
  6122. static int __init mem_cgroup_init(void)
  6123. {
  6124. int cpu, node;
  6125. cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL,
  6126. memcg_hotplug_cpu_dead);
  6127. for_each_possible_cpu(cpu)
  6128. INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work,
  6129. drain_local_stock);
  6130. for_each_node(node) {
  6131. struct mem_cgroup_tree_per_node *rtpn;
  6132. rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL,
  6133. node_online(node) ? node : NUMA_NO_NODE);
  6134. rtpn->rb_root = RB_ROOT;
  6135. rtpn->rb_rightmost = NULL;
  6136. spin_lock_init(&rtpn->lock);
  6137. soft_limit_tree.rb_tree_per_node[node] = rtpn;
  6138. }
  6139. return 0;
  6140. }
  6141. subsys_initcall(mem_cgroup_init);
  6142. #ifdef CONFIG_MEMCG_SWAP
  6143. static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg)
  6144. {
  6145. while (!refcount_inc_not_zero(&memcg->id.ref)) {
  6146. /*
  6147. * The root cgroup cannot be destroyed, so it's refcount must
  6148. * always be >= 1.
  6149. */
  6150. if (WARN_ON_ONCE(memcg == root_mem_cgroup)) {
  6151. VM_BUG_ON(1);
  6152. break;
  6153. }
  6154. memcg = parent_mem_cgroup(memcg);
  6155. if (!memcg)
  6156. memcg = root_mem_cgroup;
  6157. }
  6158. return memcg;
  6159. }
  6160. /**
  6161. * mem_cgroup_swapout - transfer a memsw charge to swap
  6162. * @page: page whose memsw charge to transfer
  6163. * @entry: swap entry to move the charge to
  6164. *
  6165. * Transfer the memsw charge of @page to @entry.
  6166. */
  6167. void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
  6168. {
  6169. struct mem_cgroup *memcg, *swap_memcg;
  6170. unsigned int nr_entries;
  6171. unsigned short oldid;
  6172. VM_BUG_ON_PAGE(PageLRU(page), page);
  6173. VM_BUG_ON_PAGE(page_count(page), page);
  6174. if (mem_cgroup_disabled())
  6175. return;
  6176. if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
  6177. return;
  6178. memcg = page->mem_cgroup;
  6179. /* Readahead page, never charged */
  6180. if (!memcg)
  6181. return;
  6182. /*
  6183. * In case the memcg owning these pages has been offlined and doesn't
  6184. * have an ID allocated to it anymore, charge the closest online
  6185. * ancestor for the swap instead and transfer the memory+swap charge.
  6186. */
  6187. swap_memcg = mem_cgroup_id_get_online(memcg);
  6188. nr_entries = thp_nr_pages(page);
  6189. /* Get references for the tail pages, too */
  6190. if (nr_entries > 1)
  6191. mem_cgroup_id_get_many(swap_memcg, nr_entries - 1);
  6192. oldid = swap_cgroup_record(entry, mem_cgroup_id(swap_memcg),
  6193. nr_entries);
  6194. VM_BUG_ON_PAGE(oldid, page);
  6195. mod_memcg_state(swap_memcg, MEMCG_SWAP, nr_entries);
  6196. page->mem_cgroup = NULL;
  6197. if (!mem_cgroup_is_root(memcg))
  6198. page_counter_uncharge(&memcg->memory, nr_entries);
  6199. if (!cgroup_memory_noswap && memcg != swap_memcg) {
  6200. if (!mem_cgroup_is_root(swap_memcg))
  6201. page_counter_charge(&swap_memcg->memsw, nr_entries);
  6202. page_counter_uncharge(&memcg->memsw, nr_entries);
  6203. }
  6204. /*
  6205. * Interrupts should be disabled here because the caller holds the
  6206. * i_pages lock which is taken with interrupts-off. It is
  6207. * important here to have the interrupts disabled because it is the
  6208. * only synchronisation we have for updating the per-CPU variables.
  6209. */
  6210. VM_BUG_ON(!irqs_disabled());
  6211. mem_cgroup_charge_statistics(memcg, page, -nr_entries);
  6212. memcg_check_events(memcg, page);
  6213. css_put(&memcg->css);
  6214. }
  6215. /**
  6216. * __mem_cgroup_try_charge_swap - try charging swap space for a page
  6217. * @page: page being added to swap
  6218. * @entry: swap entry to charge
  6219. *
  6220. * Try to charge @page's memcg for the swap space at @entry.
  6221. *
  6222. * Returns 0 on success, -ENOMEM on failure.
  6223. */
  6224. int __mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
  6225. {
  6226. unsigned int nr_pages = thp_nr_pages(page);
  6227. struct page_counter *counter;
  6228. struct mem_cgroup *memcg;
  6229. unsigned short oldid;
  6230. if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
  6231. return 0;
  6232. memcg = page->mem_cgroup;
  6233. /* Readahead page, never charged */
  6234. if (!memcg)
  6235. return 0;
  6236. if (!entry.val) {
  6237. memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
  6238. return 0;
  6239. }
  6240. memcg = mem_cgroup_id_get_online(memcg);
  6241. if (!cgroup_memory_noswap && !mem_cgroup_is_root(memcg) &&
  6242. !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) {
  6243. memcg_memory_event(memcg, MEMCG_SWAP_MAX);
  6244. memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
  6245. mem_cgroup_id_put(memcg);
  6246. return -ENOMEM;
  6247. }
  6248. /* Get references for the tail pages, too */
  6249. if (nr_pages > 1)
  6250. mem_cgroup_id_get_many(memcg, nr_pages - 1);
  6251. oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg), nr_pages);
  6252. VM_BUG_ON_PAGE(oldid, page);
  6253. mod_memcg_state(memcg, MEMCG_SWAP, nr_pages);
  6254. return 0;
  6255. }
  6256. /**
  6257. * __mem_cgroup_uncharge_swap - uncharge swap space
  6258. * @entry: swap entry to uncharge
  6259. * @nr_pages: the amount of swap space to uncharge
  6260. */
  6261. void __mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
  6262. {
  6263. struct mem_cgroup *memcg;
  6264. unsigned short id;
  6265. id = swap_cgroup_record(entry, 0, nr_pages);
  6266. rcu_read_lock();
  6267. memcg = mem_cgroup_from_id(id);
  6268. if (memcg) {
  6269. if (!cgroup_memory_noswap && !mem_cgroup_is_root(memcg)) {
  6270. if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
  6271. page_counter_uncharge(&memcg->swap, nr_pages);
  6272. else
  6273. page_counter_uncharge(&memcg->memsw, nr_pages);
  6274. }
  6275. mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages);
  6276. mem_cgroup_id_put_many(memcg, nr_pages);
  6277. }
  6278. rcu_read_unlock();
  6279. }
  6280. long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
  6281. {
  6282. long nr_swap_pages = get_nr_swap_pages();
  6283. if (cgroup_memory_noswap || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
  6284. return nr_swap_pages;
  6285. for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
  6286. nr_swap_pages = min_t(long, nr_swap_pages,
  6287. READ_ONCE(memcg->swap.max) -
  6288. page_counter_read(&memcg->swap));
  6289. return nr_swap_pages;
  6290. }
  6291. bool mem_cgroup_swap_full(struct page *page)
  6292. {
  6293. struct mem_cgroup *memcg;
  6294. VM_BUG_ON_PAGE(!PageLocked(page), page);
  6295. if (vm_swap_full())
  6296. return true;
  6297. if (cgroup_memory_noswap || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
  6298. return false;
  6299. memcg = page->mem_cgroup;
  6300. if (!memcg)
  6301. return false;
  6302. for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) {
  6303. unsigned long usage = page_counter_read(&memcg->swap);
  6304. if (usage * 2 >= READ_ONCE(memcg->swap.high) ||
  6305. usage * 2 >= READ_ONCE(memcg->swap.max))
  6306. return true;
  6307. }
  6308. return false;
  6309. }
  6310. static int __init setup_swap_account(char *s)
  6311. {
  6312. if (!strcmp(s, "1"))
  6313. cgroup_memory_noswap = 0;
  6314. else if (!strcmp(s, "0"))
  6315. cgroup_memory_noswap = 1;
  6316. return 1;
  6317. }
  6318. __setup("swapaccount=", setup_swap_account);
  6319. static u64 swap_current_read(struct cgroup_subsys_state *css,
  6320. struct cftype *cft)
  6321. {
  6322. struct mem_cgroup *memcg = mem_cgroup_from_css(css);
  6323. return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE;
  6324. }
  6325. static int swap_high_show(struct seq_file *m, void *v)
  6326. {
  6327. return seq_puts_memcg_tunable(m,
  6328. READ_ONCE(mem_cgroup_from_seq(m)->swap.high));
  6329. }
  6330. static ssize_t swap_high_write(struct kernfs_open_file *of,
  6331. char *buf, size_t nbytes, loff_t off)
  6332. {
  6333. struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
  6334. unsigned long high;
  6335. int err;
  6336. buf = strstrip(buf);
  6337. err = page_counter_memparse(buf, "max", &high);
  6338. if (err)
  6339. return err;
  6340. page_counter_set_high(&memcg->swap, high);
  6341. return nbytes;
  6342. }
  6343. static int swap_max_show(struct seq_file *m, void *v)
  6344. {
  6345. return seq_puts_memcg_tunable(m,
  6346. READ_ONCE(mem_cgroup_from_seq(m)->swap.max));
  6347. }
  6348. static ssize_t swap_max_write(struct kernfs_open_file *of,
  6349. char *buf, size_t nbytes, loff_t off)
  6350. {
  6351. struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
  6352. unsigned long max;
  6353. int err;
  6354. buf = strstrip(buf);
  6355. err = page_counter_memparse(buf, "max", &max);
  6356. if (err)
  6357. return err;
  6358. xchg(&memcg->swap.max, max);
  6359. return nbytes;
  6360. }
  6361. static int swap_events_show(struct seq_file *m, void *v)
  6362. {
  6363. struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
  6364. seq_printf(m, "high %lu\n",
  6365. atomic_long_read(&memcg->memory_events[MEMCG_SWAP_HIGH]));
  6366. seq_printf(m, "max %lu\n",
  6367. atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]));
  6368. seq_printf(m, "fail %lu\n",
  6369. atomic_long_read(&memcg->memory_events[MEMCG_SWAP_FAIL]));
  6370. return 0;
  6371. }
  6372. static struct cftype swap_files[] = {
  6373. {
  6374. .name = "swap.current",
  6375. .flags = CFTYPE_NOT_ON_ROOT,
  6376. .read_u64 = swap_current_read,
  6377. },
  6378. {
  6379. .name = "swap.high",
  6380. .flags = CFTYPE_NOT_ON_ROOT,
  6381. .seq_show = swap_high_show,
  6382. .write = swap_high_write,
  6383. },
  6384. {
  6385. .name = "swap.max",
  6386. .flags = CFTYPE_NOT_ON_ROOT,
  6387. .seq_show = swap_max_show,
  6388. .write = swap_max_write,
  6389. },
  6390. {
  6391. .name = "swap.events",
  6392. .flags = CFTYPE_NOT_ON_ROOT,
  6393. .file_offset = offsetof(struct mem_cgroup, swap_events_file),
  6394. .seq_show = swap_events_show,
  6395. },
  6396. { } /* terminate */
  6397. };
  6398. static struct cftype memsw_files[] = {
  6399. {
  6400. .name = "memsw.usage_in_bytes",
  6401. .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
  6402. .read_u64 = mem_cgroup_read_u64,
  6403. },
  6404. {
  6405. .name = "memsw.max_usage_in_bytes",
  6406. .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
  6407. .write = mem_cgroup_reset,
  6408. .read_u64 = mem_cgroup_read_u64,
  6409. },
  6410. {
  6411. .name = "memsw.limit_in_bytes",
  6412. .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
  6413. .write = mem_cgroup_write,
  6414. .read_u64 = mem_cgroup_read_u64,
  6415. },
  6416. {
  6417. .name = "memsw.failcnt",
  6418. .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
  6419. .write = mem_cgroup_reset,
  6420. .read_u64 = mem_cgroup_read_u64,
  6421. },
  6422. { }, /* terminate */
  6423. };
  6424. /*
  6425. * If mem_cgroup_swap_init() is implemented as a subsys_initcall()
  6426. * instead of a core_initcall(), this could mean cgroup_memory_noswap still
  6427. * remains set to false even when memcg is disabled via "cgroup_disable=memory"
  6428. * boot parameter. This may result in premature OOPS inside
  6429. * mem_cgroup_get_nr_swap_pages() function in corner cases.
  6430. */
  6431. static int __init mem_cgroup_swap_init(void)
  6432. {
  6433. /* No memory control -> no swap control */
  6434. if (mem_cgroup_disabled())
  6435. cgroup_memory_noswap = true;
  6436. if (cgroup_memory_noswap)
  6437. return 0;
  6438. WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys, swap_files));
  6439. WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys, memsw_files));
  6440. return 0;
  6441. }
  6442. core_initcall(mem_cgroup_swap_init);
  6443. #endif /* CONFIG_MEMCG_SWAP */