slub.c 144 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * SLUB: A slab allocator that limits cache line use instead of queuing
  4. * objects in per cpu and per node lists.
  5. *
  6. * The allocator synchronizes using per slab locks or atomic operatios
  7. * and only uses a centralized lock to manage a pool of partial slabs.
  8. *
  9. * (C) 2007 SGI, Christoph Lameter
  10. * (C) 2011 Linux Foundation, Christoph Lameter
  11. */
  12. #include <linux/mm.h>
  13. #include <linux/swap.h> /* struct reclaim_state */
  14. #include <linux/module.h>
  15. #include <linux/bit_spinlock.h>
  16. #include <linux/interrupt.h>
  17. #include <linux/swab.h>
  18. #include <linux/bitops.h>
  19. #include <linux/slab.h>
  20. #include "slab.h"
  21. #include <linux/proc_fs.h>
  22. #include <linux/seq_file.h>
  23. #include <linux/kasan.h>
  24. #include <linux/cpu.h>
  25. #include <linux/cpuset.h>
  26. #include <linux/mempolicy.h>
  27. #include <linux/ctype.h>
  28. #include <linux/debugobjects.h>
  29. #include <linux/kallsyms.h>
  30. #include <linux/kfence.h>
  31. #include <linux/memory.h>
  32. #include <linux/math64.h>
  33. #include <linux/fault-inject.h>
  34. #include <linux/stacktrace.h>
  35. #include <linux/prefetch.h>
  36. #include <linux/memcontrol.h>
  37. #include <linux/random.h>
  38. #include <linux/debugfs.h>
  39. #include <trace/events/kmem.h>
  40. #include <trace/hooks/mm.h>
  41. #include "internal.h"
  42. /*
  43. * Lock order:
  44. * 1. slab_mutex (Global Mutex)
  45. * 2. node->list_lock
  46. * 3. slab_lock(page) (Only on some arches and for debugging)
  47. *
  48. * slab_mutex
  49. *
  50. * The role of the slab_mutex is to protect the list of all the slabs
  51. * and to synchronize major metadata changes to slab cache structures.
  52. *
  53. * The slab_lock is only used for debugging and on arches that do not
  54. * have the ability to do a cmpxchg_double. It only protects:
  55. * A. page->freelist -> List of object free in a page
  56. * B. page->inuse -> Number of objects in use
  57. * C. page->objects -> Number of objects in page
  58. * D. page->frozen -> frozen state
  59. *
  60. * If a slab is frozen then it is exempt from list management. It is not
  61. * on any list except per cpu partial list. The processor that froze the
  62. * slab is the one who can perform list operations on the page. Other
  63. * processors may put objects onto the freelist but the processor that
  64. * froze the slab is the only one that can retrieve the objects from the
  65. * page's freelist.
  66. *
  67. * The list_lock protects the partial and full list on each node and
  68. * the partial slab counter. If taken then no new slabs may be added or
  69. * removed from the lists nor make the number of partial slabs be modified.
  70. * (Note that the total number of slabs is an atomic value that may be
  71. * modified without taking the list lock).
  72. *
  73. * The list_lock is a centralized lock and thus we avoid taking it as
  74. * much as possible. As long as SLUB does not have to handle partial
  75. * slabs, operations can continue without any centralized lock. F.e.
  76. * allocating a long series of objects that fill up slabs does not require
  77. * the list lock.
  78. * Interrupts are disabled during allocation and deallocation in order to
  79. * make the slab allocator safe to use in the context of an irq. In addition
  80. * interrupts are disabled to ensure that the processor does not change
  81. * while handling per_cpu slabs, due to kernel preemption.
  82. *
  83. * SLUB assigns one slab for allocation to each processor.
  84. * Allocations only occur from these slabs called cpu slabs.
  85. *
  86. * Slabs with free elements are kept on a partial list and during regular
  87. * operations no list for full slabs is used. If an object in a full slab is
  88. * freed then the slab will show up again on the partial lists.
  89. * We track full slabs for debugging purposes though because otherwise we
  90. * cannot scan all objects.
  91. *
  92. * Slabs are freed when they become empty. Teardown and setup is
  93. * minimal so we rely on the page allocators per cpu caches for
  94. * fast frees and allocs.
  95. *
  96. * page->frozen The slab is frozen and exempt from list processing.
  97. * This means that the slab is dedicated to a purpose
  98. * such as satisfying allocations for a specific
  99. * processor. Objects may be freed in the slab while
  100. * it is frozen but slab_free will then skip the usual
  101. * list operations. It is up to the processor holding
  102. * the slab to integrate the slab into the slab lists
  103. * when the slab is no longer needed.
  104. *
  105. * One use of this flag is to mark slabs that are
  106. * used for allocations. Then such a slab becomes a cpu
  107. * slab. The cpu slab may be equipped with an additional
  108. * freelist that allows lockless access to
  109. * free objects in addition to the regular freelist
  110. * that requires the slab lock.
  111. *
  112. * SLAB_DEBUG_FLAGS Slab requires special handling due to debug
  113. * options set. This moves slab handling out of
  114. * the fast path and disables lockless freelists.
  115. */
  116. #ifdef CONFIG_SLUB_DEBUG
  117. #ifdef CONFIG_SLUB_DEBUG_ON
  118. DEFINE_STATIC_KEY_TRUE(slub_debug_enabled);
  119. #else
  120. DEFINE_STATIC_KEY_FALSE(slub_debug_enabled);
  121. #endif
  122. #endif
  123. static inline bool kmem_cache_debug(struct kmem_cache *s)
  124. {
  125. return kmem_cache_debug_flags(s, SLAB_DEBUG_FLAGS);
  126. }
  127. void *fixup_red_left(struct kmem_cache *s, void *p)
  128. {
  129. if (kmem_cache_debug_flags(s, SLAB_RED_ZONE))
  130. p += s->red_left_pad;
  131. return p;
  132. }
  133. static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s)
  134. {
  135. #ifdef CONFIG_SLUB_CPU_PARTIAL
  136. return !kmem_cache_debug(s);
  137. #else
  138. return false;
  139. #endif
  140. }
  141. /*
  142. * Issues still to be resolved:
  143. *
  144. * - Support PAGE_ALLOC_DEBUG. Should be easy to do.
  145. *
  146. * - Variable sizing of the per node arrays
  147. */
  148. /* Enable to test recovery from slab corruption on boot */
  149. #undef SLUB_RESILIENCY_TEST
  150. /* Enable to log cmpxchg failures */
  151. #undef SLUB_DEBUG_CMPXCHG
  152. /*
  153. * Mininum number of partial slabs. These will be left on the partial
  154. * lists even if they are empty. kmem_cache_shrink may reclaim them.
  155. */
  156. #define MIN_PARTIAL 5
  157. /*
  158. * Maximum number of desirable partial slabs.
  159. * The existence of more partial slabs makes kmem_cache_shrink
  160. * sort the partial list by the number of objects in use.
  161. */
  162. #define MAX_PARTIAL 10
  163. #define DEBUG_DEFAULT_FLAGS (SLAB_CONSISTENCY_CHECKS | SLAB_RED_ZONE | \
  164. SLAB_POISON | SLAB_STORE_USER)
  165. /*
  166. * These debug flags cannot use CMPXCHG because there might be consistency
  167. * issues when checking or reading debug information
  168. */
  169. #define SLAB_NO_CMPXCHG (SLAB_CONSISTENCY_CHECKS | SLAB_STORE_USER | \
  170. SLAB_TRACE)
  171. /*
  172. * Debugging flags that require metadata to be stored in the slab. These get
  173. * disabled when slub_debug=O is used and a cache's min order increases with
  174. * metadata.
  175. */
  176. #define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
  177. #define OO_SHIFT 16
  178. #define OO_MASK ((1 << OO_SHIFT) - 1)
  179. #define MAX_OBJS_PER_PAGE 32767 /* since page.objects is u15 */
  180. /* Internal SLUB flags */
  181. /* Poison object */
  182. #define __OBJECT_POISON ((slab_flags_t __force)0x80000000U)
  183. /* Use cmpxchg_double */
  184. #define __CMPXCHG_DOUBLE ((slab_flags_t __force)0x40000000U)
  185. #ifdef CONFIG_SYSFS
  186. static int sysfs_slab_add(struct kmem_cache *);
  187. static int sysfs_slab_alias(struct kmem_cache *, const char *);
  188. #else
  189. static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; }
  190. static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p)
  191. { return 0; }
  192. #endif
  193. #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_SLUB_DEBUG)
  194. static void debugfs_slab_add(struct kmem_cache *);
  195. #else
  196. static inline void debugfs_slab_add(struct kmem_cache *s) { }
  197. #endif
  198. static inline void stat(const struct kmem_cache *s, enum stat_item si)
  199. {
  200. #ifdef CONFIG_SLUB_STATS
  201. /*
  202. * The rmw is racy on a preemptible kernel but this is acceptable, so
  203. * avoid this_cpu_add()'s irq-disable overhead.
  204. */
  205. raw_cpu_inc(s->cpu_slab->stat[si]);
  206. #endif
  207. }
  208. /********************************************************************
  209. * Core slab cache functions
  210. *******************************************************************/
  211. /*
  212. * Returns freelist pointer (ptr). With hardening, this is obfuscated
  213. * with an XOR of the address where the pointer is held and a per-cache
  214. * random number.
  215. */
  216. static inline void *freelist_ptr(const struct kmem_cache *s, void *ptr,
  217. unsigned long ptr_addr)
  218. {
  219. #ifdef CONFIG_SLAB_FREELIST_HARDENED
  220. /*
  221. * When CONFIG_KASAN_SW/HW_TAGS is enabled, ptr_addr might be tagged.
  222. * Normally, this doesn't cause any issues, as both set_freepointer()
  223. * and get_freepointer() are called with a pointer with the same tag.
  224. * However, there are some issues with CONFIG_SLUB_DEBUG code. For
  225. * example, when __free_slub() iterates over objects in a cache, it
  226. * passes untagged pointers to check_object(). check_object() in turns
  227. * calls get_freepointer() with an untagged pointer, which causes the
  228. * freepointer to be restored incorrectly.
  229. */
  230. return (void *)((unsigned long)ptr ^ s->random ^
  231. swab((unsigned long)kasan_reset_tag((void *)ptr_addr)));
  232. #else
  233. return ptr;
  234. #endif
  235. }
  236. /* Returns the freelist pointer recorded at location ptr_addr. */
  237. static inline void *freelist_dereference(const struct kmem_cache *s,
  238. void *ptr_addr)
  239. {
  240. return freelist_ptr(s, (void *)*(unsigned long *)(ptr_addr),
  241. (unsigned long)ptr_addr);
  242. }
  243. static inline void *get_freepointer(struct kmem_cache *s, void *object)
  244. {
  245. object = kasan_reset_tag(object);
  246. return freelist_dereference(s, object + s->offset);
  247. }
  248. static void prefetch_freepointer(const struct kmem_cache *s, void *object)
  249. {
  250. prefetch(object + s->offset);
  251. }
  252. static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)
  253. {
  254. unsigned long freepointer_addr;
  255. void *p;
  256. if (!debug_pagealloc_enabled_static())
  257. return get_freepointer(s, object);
  258. object = kasan_reset_tag(object);
  259. freepointer_addr = (unsigned long)object + s->offset;
  260. copy_from_kernel_nofault(&p, (void **)freepointer_addr, sizeof(p));
  261. return freelist_ptr(s, p, freepointer_addr);
  262. }
  263. static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
  264. {
  265. unsigned long freeptr_addr = (unsigned long)object + s->offset;
  266. #ifdef CONFIG_SLAB_FREELIST_HARDENED
  267. BUG_ON(object == fp); /* naive detection of double free or corruption */
  268. #endif
  269. freeptr_addr = (unsigned long)kasan_reset_tag((void *)freeptr_addr);
  270. *(void **)freeptr_addr = freelist_ptr(s, fp, freeptr_addr);
  271. }
  272. /* Loop over all objects in a slab */
  273. #define for_each_object(__p, __s, __addr, __objects) \
  274. for (__p = fixup_red_left(__s, __addr); \
  275. __p < (__addr) + (__objects) * (__s)->size; \
  276. __p += (__s)->size)
  277. static inline unsigned int order_objects(unsigned int order, unsigned int size)
  278. {
  279. return ((unsigned int)PAGE_SIZE << order) / size;
  280. }
  281. static inline struct kmem_cache_order_objects oo_make(unsigned int order,
  282. unsigned int size)
  283. {
  284. struct kmem_cache_order_objects x = {
  285. (order << OO_SHIFT) + order_objects(order, size)
  286. };
  287. return x;
  288. }
  289. static inline unsigned int oo_order(struct kmem_cache_order_objects x)
  290. {
  291. return x.x >> OO_SHIFT;
  292. }
  293. static inline unsigned int oo_objects(struct kmem_cache_order_objects x)
  294. {
  295. return x.x & OO_MASK;
  296. }
  297. /*
  298. * Per slab locking using the pagelock
  299. */
  300. static __always_inline void slab_lock(struct page *page)
  301. {
  302. VM_BUG_ON_PAGE(PageTail(page), page);
  303. bit_spin_lock(PG_locked, &page->flags);
  304. }
  305. static __always_inline void slab_unlock(struct page *page)
  306. {
  307. VM_BUG_ON_PAGE(PageTail(page), page);
  308. __bit_spin_unlock(PG_locked, &page->flags);
  309. }
  310. /* Interrupts must be disabled (for the fallback code to work right) */
  311. static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
  312. void *freelist_old, unsigned long counters_old,
  313. void *freelist_new, unsigned long counters_new,
  314. const char *n)
  315. {
  316. VM_BUG_ON(!irqs_disabled());
  317. #if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
  318. defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
  319. if (s->flags & __CMPXCHG_DOUBLE) {
  320. if (cmpxchg_double(&page->freelist, &page->counters,
  321. freelist_old, counters_old,
  322. freelist_new, counters_new))
  323. return true;
  324. } else
  325. #endif
  326. {
  327. slab_lock(page);
  328. if (page->freelist == freelist_old &&
  329. page->counters == counters_old) {
  330. page->freelist = freelist_new;
  331. page->counters = counters_new;
  332. slab_unlock(page);
  333. return true;
  334. }
  335. slab_unlock(page);
  336. }
  337. cpu_relax();
  338. stat(s, CMPXCHG_DOUBLE_FAIL);
  339. #ifdef SLUB_DEBUG_CMPXCHG
  340. pr_info("%s %s: cmpxchg double redo ", n, s->name);
  341. #endif
  342. return false;
  343. }
  344. static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
  345. void *freelist_old, unsigned long counters_old,
  346. void *freelist_new, unsigned long counters_new,
  347. const char *n)
  348. {
  349. #if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
  350. defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
  351. if (s->flags & __CMPXCHG_DOUBLE) {
  352. if (cmpxchg_double(&page->freelist, &page->counters,
  353. freelist_old, counters_old,
  354. freelist_new, counters_new))
  355. return true;
  356. } else
  357. #endif
  358. {
  359. unsigned long flags;
  360. local_irq_save(flags);
  361. slab_lock(page);
  362. if (page->freelist == freelist_old &&
  363. page->counters == counters_old) {
  364. page->freelist = freelist_new;
  365. page->counters = counters_new;
  366. slab_unlock(page);
  367. local_irq_restore(flags);
  368. return true;
  369. }
  370. slab_unlock(page);
  371. local_irq_restore(flags);
  372. }
  373. cpu_relax();
  374. stat(s, CMPXCHG_DOUBLE_FAIL);
  375. #ifdef SLUB_DEBUG_CMPXCHG
  376. pr_info("%s %s: cmpxchg double redo ", n, s->name);
  377. #endif
  378. return false;
  379. }
  380. #ifdef CONFIG_SLUB_DEBUG
  381. static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];
  382. static DEFINE_SPINLOCK(object_map_lock);
  383. static void __fill_map(unsigned long *obj_map, struct kmem_cache *s,
  384. struct page *page)
  385. {
  386. void *addr = page_address(page);
  387. void *p;
  388. bitmap_zero(obj_map, page->objects);
  389. for (p = page->freelist; p; p = get_freepointer(s, p))
  390. set_bit(__obj_to_index(s, addr, p), obj_map);
  391. }
  392. /*
  393. * Determine a map of object in use on a page.
  394. *
  395. * Node listlock must be held to guarantee that the page does
  396. * not vanish from under us.
  397. */
  398. static unsigned long *get_map(struct kmem_cache *s, struct page *page)
  399. __acquires(&object_map_lock)
  400. {
  401. VM_BUG_ON(!irqs_disabled());
  402. spin_lock(&object_map_lock);
  403. __fill_map(object_map, s, page);
  404. return object_map;
  405. }
  406. static void put_map(unsigned long *map) __releases(&object_map_lock)
  407. {
  408. VM_BUG_ON(map != object_map);
  409. spin_unlock(&object_map_lock);
  410. }
  411. static inline unsigned int size_from_object(struct kmem_cache *s)
  412. {
  413. if (s->flags & SLAB_RED_ZONE)
  414. return s->size - s->red_left_pad;
  415. return s->size;
  416. }
  417. static inline void *restore_red_left(struct kmem_cache *s, void *p)
  418. {
  419. if (s->flags & SLAB_RED_ZONE)
  420. p -= s->red_left_pad;
  421. return p;
  422. }
  423. /*
  424. * Debug settings:
  425. */
  426. #if defined(CONFIG_SLUB_DEBUG_ON)
  427. slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;
  428. #else
  429. slab_flags_t slub_debug;
  430. #endif
  431. static char *slub_debug_string;
  432. static int disable_higher_order_debug;
  433. /*
  434. * slub is about to manipulate internal object metadata. This memory lies
  435. * outside the range of the allocated object, so accessing it would normally
  436. * be reported by kasan as a bounds error. metadata_access_enable() is used
  437. * to tell kasan that these accesses are OK.
  438. */
  439. static inline void metadata_access_enable(void)
  440. {
  441. kasan_disable_current();
  442. }
  443. static inline void metadata_access_disable(void)
  444. {
  445. kasan_enable_current();
  446. }
  447. /*
  448. * Object debugging
  449. */
  450. /* Verify that a pointer has an address that is valid within a slab page */
  451. static inline int check_valid_pointer(struct kmem_cache *s,
  452. struct page *page, void *object)
  453. {
  454. void *base;
  455. if (!object)
  456. return 1;
  457. base = page_address(page);
  458. object = kasan_reset_tag(object);
  459. object = restore_red_left(s, object);
  460. if (object < base || object >= base + page->objects * s->size ||
  461. (object - base) % s->size) {
  462. return 0;
  463. }
  464. return 1;
  465. }
  466. static void print_section(char *level, char *text, u8 *addr,
  467. unsigned int length)
  468. {
  469. metadata_access_enable();
  470. print_hex_dump(level, text, DUMP_PREFIX_ADDRESS,
  471. 16, 1, kasan_reset_tag((void *)addr), length, 1);
  472. metadata_access_disable();
  473. }
  474. /*
  475. * See comment in calculate_sizes().
  476. */
  477. static inline bool freeptr_outside_object(struct kmem_cache *s)
  478. {
  479. return s->offset >= s->inuse;
  480. }
  481. /*
  482. * Return offset of the end of info block which is inuse + free pointer if
  483. * not overlapping with object.
  484. */
  485. static inline unsigned int get_info_end(struct kmem_cache *s)
  486. {
  487. if (freeptr_outside_object(s))
  488. return s->inuse + sizeof(void *);
  489. else
  490. return s->inuse;
  491. }
  492. static struct track *get_track(struct kmem_cache *s, void *object,
  493. enum track_item alloc)
  494. {
  495. struct track *p;
  496. p = object + get_info_end(s);
  497. return kasan_reset_tag(p + alloc);
  498. }
  499. /*
  500. * This function will be used to loop through all the slab objects in
  501. * a page to give track structure for each object, the function fn will
  502. * be using this track structure and extract required info into its private
  503. * data, the return value will be the number of track structures that are
  504. * processed.
  505. */
  506. unsigned long get_each_object_track(struct kmem_cache *s,
  507. struct page *page, enum track_item alloc,
  508. int (*fn)(const struct kmem_cache *, const void *,
  509. const struct track *, void *), void *private)
  510. {
  511. void *p;
  512. struct track *t;
  513. int ret;
  514. unsigned long num_track = 0;
  515. if (!slub_debug || !(s->flags & SLAB_STORE_USER))
  516. return 0;
  517. slab_lock(page);
  518. for_each_object(p, s, page_address(page), page->objects) {
  519. t = get_track(s, p, alloc);
  520. metadata_access_enable();
  521. ret = fn(s, p, t, private);
  522. metadata_access_disable();
  523. if (ret < 0)
  524. break;
  525. num_track += 1;
  526. }
  527. slab_unlock(page);
  528. return num_track;
  529. }
  530. EXPORT_SYMBOL_GPL(get_each_object_track);
  531. static void set_track(struct kmem_cache *s, void *object,
  532. enum track_item alloc, unsigned long addr)
  533. {
  534. struct track *p = get_track(s, object, alloc);
  535. if (addr) {
  536. #ifdef CONFIG_STACKTRACE
  537. unsigned int nr_entries;
  538. metadata_access_enable();
  539. nr_entries = stack_trace_save(kasan_reset_tag(p->addrs),
  540. TRACK_ADDRS_COUNT, 3);
  541. metadata_access_disable();
  542. if (nr_entries < TRACK_ADDRS_COUNT)
  543. p->addrs[nr_entries] = 0;
  544. trace_android_vh_save_track_hash(alloc == TRACK_ALLOC,
  545. (unsigned long)p);
  546. #endif
  547. p->addr = addr;
  548. p->cpu = smp_processor_id();
  549. p->pid = current->pid;
  550. p->when = jiffies;
  551. } else {
  552. memset(p, 0, sizeof(struct track));
  553. }
  554. }
  555. static void init_tracking(struct kmem_cache *s, void *object)
  556. {
  557. if (!(s->flags & SLAB_STORE_USER))
  558. return;
  559. set_track(s, object, TRACK_FREE, 0UL);
  560. set_track(s, object, TRACK_ALLOC, 0UL);
  561. }
  562. static void print_track(const char *s, struct track *t, unsigned long pr_time)
  563. {
  564. if (!t->addr)
  565. return;
  566. pr_err("INFO: %s in %pS age=%lu cpu=%u pid=%d\n",
  567. s, (void *)t->addr, pr_time - t->when, t->cpu, t->pid);
  568. #ifdef CONFIG_STACKTRACE
  569. {
  570. int i;
  571. for (i = 0; i < TRACK_ADDRS_COUNT; i++)
  572. if (t->addrs[i])
  573. pr_err("\t%pS\n", (void *)t->addrs[i]);
  574. else
  575. break;
  576. }
  577. #endif
  578. }
  579. void print_tracking(struct kmem_cache *s, void *object)
  580. {
  581. unsigned long pr_time = jiffies;
  582. if (!(s->flags & SLAB_STORE_USER))
  583. return;
  584. print_track("Allocated", get_track(s, object, TRACK_ALLOC), pr_time);
  585. print_track("Freed", get_track(s, object, TRACK_FREE), pr_time);
  586. }
  587. static void print_page_info(struct page *page)
  588. {
  589. pr_err("INFO: Slab 0x%p objects=%u used=%u fp=0x%p flags=0x%04lx\n",
  590. page, page->objects, page->inuse, page->freelist, page->flags);
  591. }
  592. static void slab_bug(struct kmem_cache *s, char *fmt, ...)
  593. {
  594. struct va_format vaf;
  595. va_list args;
  596. va_start(args, fmt);
  597. vaf.fmt = fmt;
  598. vaf.va = &args;
  599. pr_err("=============================================================================\n");
  600. pr_err("BUG %s (%s): %pV\n", s->name, print_tainted(), &vaf);
  601. pr_err("-----------------------------------------------------------------------------\n\n");
  602. va_end(args);
  603. }
  604. static void slab_fix(struct kmem_cache *s, char *fmt, ...)
  605. {
  606. struct va_format vaf;
  607. va_list args;
  608. va_start(args, fmt);
  609. vaf.fmt = fmt;
  610. vaf.va = &args;
  611. pr_err("FIX %s: %pV\n", s->name, &vaf);
  612. va_end(args);
  613. }
  614. static bool freelist_corrupted(struct kmem_cache *s, struct page *page,
  615. void **freelist, void *nextfree)
  616. {
  617. if ((s->flags & SLAB_CONSISTENCY_CHECKS) &&
  618. !check_valid_pointer(s, page, nextfree) && freelist) {
  619. object_err(s, page, *freelist, "Freechain corrupt");
  620. *freelist = NULL;
  621. slab_fix(s, "Isolate corrupted freechain");
  622. return true;
  623. }
  624. return false;
  625. }
  626. static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p)
  627. {
  628. unsigned int off; /* Offset of last byte */
  629. u8 *addr = page_address(page);
  630. print_tracking(s, p);
  631. print_page_info(page);
  632. pr_err("INFO: Object 0x%p @offset=%tu fp=0x%p\n\n",
  633. p, p - addr, get_freepointer(s, p));
  634. if (s->flags & SLAB_RED_ZONE)
  635. print_section(KERN_ERR, "Redzone ", p - s->red_left_pad,
  636. s->red_left_pad);
  637. else if (p > addr + 16)
  638. print_section(KERN_ERR, "Bytes b4 ", p - 16, 16);
  639. print_section(KERN_ERR, "Object ", p,
  640. min_t(unsigned int, s->object_size, PAGE_SIZE));
  641. if (s->flags & SLAB_RED_ZONE)
  642. print_section(KERN_ERR, "Redzone ", p + s->object_size,
  643. s->inuse - s->object_size);
  644. off = get_info_end(s);
  645. if (s->flags & SLAB_STORE_USER)
  646. off += 2 * sizeof(struct track);
  647. off += kasan_metadata_size(s);
  648. if (off != size_from_object(s))
  649. /* Beginning of the filler is the free pointer */
  650. print_section(KERN_ERR, "Padding ", p + off,
  651. size_from_object(s) - off);
  652. dump_stack();
  653. }
  654. void object_err(struct kmem_cache *s, struct page *page,
  655. u8 *object, char *reason)
  656. {
  657. slab_bug(s, "%s", reason);
  658. print_trailer(s, page, object);
  659. add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
  660. }
  661. static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
  662. const char *fmt, ...)
  663. {
  664. va_list args;
  665. char buf[100];
  666. va_start(args, fmt);
  667. vsnprintf(buf, sizeof(buf), fmt, args);
  668. va_end(args);
  669. slab_bug(s, "%s", buf);
  670. print_page_info(page);
  671. dump_stack();
  672. add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
  673. }
  674. static void init_object(struct kmem_cache *s, void *object, u8 val)
  675. {
  676. u8 *p = kasan_reset_tag(object);
  677. if (s->flags & SLAB_RED_ZONE)
  678. memset(p - s->red_left_pad, val, s->red_left_pad);
  679. if (s->flags & __OBJECT_POISON) {
  680. memset(p, POISON_FREE, s->object_size - 1);
  681. p[s->object_size - 1] = POISON_END;
  682. }
  683. if (s->flags & SLAB_RED_ZONE)
  684. memset(p + s->object_size, val, s->inuse - s->object_size);
  685. }
  686. static void restore_bytes(struct kmem_cache *s, char *message, u8 data,
  687. void *from, void *to)
  688. {
  689. slab_fix(s, "Restoring 0x%p-0x%p=0x%x\n", from, to - 1, data);
  690. memset(from, data, to - from);
  691. }
  692. static int check_bytes_and_report(struct kmem_cache *s, struct page *page,
  693. u8 *object, char *what,
  694. u8 *start, unsigned int value, unsigned int bytes)
  695. {
  696. u8 *fault;
  697. u8 *end;
  698. u8 *addr = page_address(page);
  699. metadata_access_enable();
  700. fault = memchr_inv(kasan_reset_tag(start), value, bytes);
  701. metadata_access_disable();
  702. if (!fault)
  703. return 1;
  704. end = start + bytes;
  705. while (end > fault && end[-1] == value)
  706. end--;
  707. slab_bug(s, "%s overwritten", what);
  708. pr_err("INFO: 0x%p-0x%p @offset=%tu. First byte 0x%x instead of 0x%x\n",
  709. fault, end - 1, fault - addr,
  710. fault[0], value);
  711. print_trailer(s, page, object);
  712. add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
  713. restore_bytes(s, what, value, fault, end);
  714. return 0;
  715. }
  716. /*
  717. * Object layout:
  718. *
  719. * object address
  720. * Bytes of the object to be managed.
  721. * If the freepointer may overlay the object then the free
  722. * pointer is at the middle of the object.
  723. *
  724. * Poisoning uses 0x6b (POISON_FREE) and the last byte is
  725. * 0xa5 (POISON_END)
  726. *
  727. * object + s->object_size
  728. * Padding to reach word boundary. This is also used for Redzoning.
  729. * Padding is extended by another word if Redzoning is enabled and
  730. * object_size == inuse.
  731. *
  732. * We fill with 0xbb (RED_INACTIVE) for inactive objects and with
  733. * 0xcc (RED_ACTIVE) for objects in use.
  734. *
  735. * object + s->inuse
  736. * Meta data starts here.
  737. *
  738. * A. Free pointer (if we cannot overwrite object on free)
  739. * B. Tracking data for SLAB_STORE_USER
  740. * C. Padding to reach required alignment boundary or at mininum
  741. * one word if debugging is on to be able to detect writes
  742. * before the word boundary.
  743. *
  744. * Padding is done using 0x5a (POISON_INUSE)
  745. *
  746. * object + s->size
  747. * Nothing is used beyond s->size.
  748. *
  749. * If slabcaches are merged then the object_size and inuse boundaries are mostly
  750. * ignored. And therefore no slab options that rely on these boundaries
  751. * may be used with merged slabcaches.
  752. */
  753. static int check_pad_bytes(struct kmem_cache *s, struct page *page, u8 *p)
  754. {
  755. unsigned long off = get_info_end(s); /* The end of info */
  756. if (s->flags & SLAB_STORE_USER)
  757. /* We also have user information there */
  758. off += 2 * sizeof(struct track);
  759. off += kasan_metadata_size(s);
  760. if (size_from_object(s) == off)
  761. return 1;
  762. return check_bytes_and_report(s, page, p, "Object padding",
  763. p + off, POISON_INUSE, size_from_object(s) - off);
  764. }
  765. /* Check the pad bytes at the end of a slab page */
  766. static int slab_pad_check(struct kmem_cache *s, struct page *page)
  767. {
  768. u8 *start;
  769. u8 *fault;
  770. u8 *end;
  771. u8 *pad;
  772. int length;
  773. int remainder;
  774. if (!(s->flags & SLAB_POISON))
  775. return 1;
  776. start = page_address(page);
  777. length = page_size(page);
  778. end = start + length;
  779. remainder = length % s->size;
  780. if (!remainder)
  781. return 1;
  782. pad = end - remainder;
  783. metadata_access_enable();
  784. fault = memchr_inv(kasan_reset_tag(pad), POISON_INUSE, remainder);
  785. metadata_access_disable();
  786. if (!fault)
  787. return 1;
  788. while (end > fault && end[-1] == POISON_INUSE)
  789. end--;
  790. slab_err(s, page, "Padding overwritten. 0x%p-0x%p @offset=%tu",
  791. fault, end - 1, fault - start);
  792. print_section(KERN_ERR, "Padding ", pad, remainder);
  793. restore_bytes(s, "slab padding", POISON_INUSE, fault, end);
  794. return 0;
  795. }
  796. static int check_object(struct kmem_cache *s, struct page *page,
  797. void *object, u8 val)
  798. {
  799. u8 *p = object;
  800. u8 *endobject = object + s->object_size;
  801. if (s->flags & SLAB_RED_ZONE) {
  802. if (!check_bytes_and_report(s, page, object, "Left Redzone",
  803. object - s->red_left_pad, val, s->red_left_pad))
  804. return 0;
  805. if (!check_bytes_and_report(s, page, object, "Right Redzone",
  806. endobject, val, s->inuse - s->object_size))
  807. return 0;
  808. } else {
  809. if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) {
  810. check_bytes_and_report(s, page, p, "Alignment padding",
  811. endobject, POISON_INUSE,
  812. s->inuse - s->object_size);
  813. }
  814. }
  815. if (s->flags & SLAB_POISON) {
  816. if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON) &&
  817. (!check_bytes_and_report(s, page, p, "Poison", p,
  818. POISON_FREE, s->object_size - 1) ||
  819. !check_bytes_and_report(s, page, p, "End Poison",
  820. p + s->object_size - 1, POISON_END, 1)))
  821. return 0;
  822. /*
  823. * check_pad_bytes cleans up on its own.
  824. */
  825. check_pad_bytes(s, page, p);
  826. }
  827. if (!freeptr_outside_object(s) && val == SLUB_RED_ACTIVE)
  828. /*
  829. * Object and freepointer overlap. Cannot check
  830. * freepointer while object is allocated.
  831. */
  832. return 1;
  833. /* Check free pointer validity */
  834. if (!check_valid_pointer(s, page, get_freepointer(s, p))) {
  835. object_err(s, page, p, "Freepointer corrupt");
  836. /*
  837. * No choice but to zap it and thus lose the remainder
  838. * of the free objects in this slab. May cause
  839. * another error because the object count is now wrong.
  840. */
  841. set_freepointer(s, p, NULL);
  842. return 0;
  843. }
  844. return 1;
  845. }
  846. static int check_slab(struct kmem_cache *s, struct page *page)
  847. {
  848. int maxobj;
  849. VM_BUG_ON(!irqs_disabled());
  850. if (!PageSlab(page)) {
  851. slab_err(s, page, "Not a valid slab page");
  852. return 0;
  853. }
  854. maxobj = order_objects(compound_order(page), s->size);
  855. if (page->objects > maxobj) {
  856. slab_err(s, page, "objects %u > max %u",
  857. page->objects, maxobj);
  858. return 0;
  859. }
  860. if (page->inuse > page->objects) {
  861. slab_err(s, page, "inuse %u > max %u",
  862. page->inuse, page->objects);
  863. return 0;
  864. }
  865. /* Slab_pad_check fixes things up after itself */
  866. slab_pad_check(s, page);
  867. return 1;
  868. }
  869. /*
  870. * Determine if a certain object on a page is on the freelist. Must hold the
  871. * slab lock to guarantee that the chains are in a consistent state.
  872. */
  873. static int on_freelist(struct kmem_cache *s, struct page *page, void *search)
  874. {
  875. int nr = 0;
  876. void *fp;
  877. void *object = NULL;
  878. int max_objects;
  879. fp = page->freelist;
  880. while (fp && nr <= page->objects) {
  881. if (fp == search)
  882. return 1;
  883. if (!check_valid_pointer(s, page, fp)) {
  884. if (object) {
  885. object_err(s, page, object,
  886. "Freechain corrupt");
  887. set_freepointer(s, object, NULL);
  888. } else {
  889. slab_err(s, page, "Freepointer corrupt");
  890. page->freelist = NULL;
  891. page->inuse = page->objects;
  892. slab_fix(s, "Freelist cleared");
  893. return 0;
  894. }
  895. break;
  896. }
  897. object = fp;
  898. fp = get_freepointer(s, object);
  899. nr++;
  900. }
  901. max_objects = order_objects(compound_order(page), s->size);
  902. if (max_objects > MAX_OBJS_PER_PAGE)
  903. max_objects = MAX_OBJS_PER_PAGE;
  904. if (page->objects != max_objects) {
  905. slab_err(s, page, "Wrong number of objects. Found %d but should be %d",
  906. page->objects, max_objects);
  907. page->objects = max_objects;
  908. slab_fix(s, "Number of objects adjusted.");
  909. }
  910. if (page->inuse != page->objects - nr) {
  911. slab_err(s, page, "Wrong object count. Counter is %d but counted were %d",
  912. page->inuse, page->objects - nr);
  913. page->inuse = page->objects - nr;
  914. slab_fix(s, "Object count adjusted.");
  915. }
  916. return search == NULL;
  917. }
  918. static void trace(struct kmem_cache *s, struct page *page, void *object,
  919. int alloc)
  920. {
  921. if (s->flags & SLAB_TRACE) {
  922. pr_info("TRACE %s %s 0x%p inuse=%d fp=0x%p\n",
  923. s->name,
  924. alloc ? "alloc" : "free",
  925. object, page->inuse,
  926. page->freelist);
  927. if (!alloc)
  928. print_section(KERN_INFO, "Object ", (void *)object,
  929. s->object_size);
  930. dump_stack();
  931. }
  932. }
  933. /*
  934. * Tracking of fully allocated slabs for debugging purposes.
  935. */
  936. static void add_full(struct kmem_cache *s,
  937. struct kmem_cache_node *n, struct page *page)
  938. {
  939. if (!(s->flags & SLAB_STORE_USER))
  940. return;
  941. lockdep_assert_held(&n->list_lock);
  942. list_add(&page->slab_list, &n->full);
  943. }
  944. static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct page *page)
  945. {
  946. if (!(s->flags & SLAB_STORE_USER))
  947. return;
  948. lockdep_assert_held(&n->list_lock);
  949. list_del(&page->slab_list);
  950. }
  951. /* Tracking of the number of slabs for debugging purposes */
  952. static inline unsigned long slabs_node(struct kmem_cache *s, int node)
  953. {
  954. struct kmem_cache_node *n = get_node(s, node);
  955. return atomic_long_read(&n->nr_slabs);
  956. }
  957. static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
  958. {
  959. return atomic_long_read(&n->nr_slabs);
  960. }
  961. static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects)
  962. {
  963. struct kmem_cache_node *n = get_node(s, node);
  964. /*
  965. * May be called early in order to allocate a slab for the
  966. * kmem_cache_node structure. Solve the chicken-egg
  967. * dilemma by deferring the increment of the count during
  968. * bootstrap (see early_kmem_cache_node_alloc).
  969. */
  970. if (likely(n)) {
  971. atomic_long_inc(&n->nr_slabs);
  972. atomic_long_add(objects, &n->total_objects);
  973. }
  974. }
  975. static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects)
  976. {
  977. struct kmem_cache_node *n = get_node(s, node);
  978. atomic_long_dec(&n->nr_slabs);
  979. atomic_long_sub(objects, &n->total_objects);
  980. }
  981. /* Object debug checks for alloc/free paths */
  982. static void setup_object_debug(struct kmem_cache *s, struct page *page,
  983. void *object)
  984. {
  985. if (!kmem_cache_debug_flags(s, SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON))
  986. return;
  987. init_object(s, object, SLUB_RED_INACTIVE);
  988. init_tracking(s, object);
  989. }
  990. static
  991. void setup_page_debug(struct kmem_cache *s, struct page *page, void *addr)
  992. {
  993. if (!kmem_cache_debug_flags(s, SLAB_POISON))
  994. return;
  995. metadata_access_enable();
  996. memset(kasan_reset_tag(addr), POISON_INUSE, page_size(page));
  997. metadata_access_disable();
  998. }
  999. static inline int alloc_consistency_checks(struct kmem_cache *s,
  1000. struct page *page, void *object)
  1001. {
  1002. if (!check_slab(s, page))
  1003. return 0;
  1004. if (!check_valid_pointer(s, page, object)) {
  1005. object_err(s, page, object, "Freelist Pointer check fails");
  1006. return 0;
  1007. }
  1008. if (!check_object(s, page, object, SLUB_RED_INACTIVE))
  1009. return 0;
  1010. return 1;
  1011. }
  1012. static noinline int alloc_debug_processing(struct kmem_cache *s,
  1013. struct page *page,
  1014. void *object, unsigned long addr)
  1015. {
  1016. if (s->flags & SLAB_CONSISTENCY_CHECKS) {
  1017. if (!alloc_consistency_checks(s, page, object))
  1018. goto bad;
  1019. }
  1020. /* Success perform special debug activities for allocs */
  1021. if (s->flags & SLAB_STORE_USER)
  1022. set_track(s, object, TRACK_ALLOC, addr);
  1023. trace(s, page, object, 1);
  1024. init_object(s, object, SLUB_RED_ACTIVE);
  1025. return 1;
  1026. bad:
  1027. if (PageSlab(page)) {
  1028. /*
  1029. * If this is a slab page then lets do the best we can
  1030. * to avoid issues in the future. Marking all objects
  1031. * as used avoids touching the remaining objects.
  1032. */
  1033. slab_fix(s, "Marking all objects used");
  1034. page->inuse = page->objects;
  1035. page->freelist = NULL;
  1036. }
  1037. return 0;
  1038. }
  1039. static inline int free_consistency_checks(struct kmem_cache *s,
  1040. struct page *page, void *object, unsigned long addr)
  1041. {
  1042. if (!check_valid_pointer(s, page, object)) {
  1043. slab_err(s, page, "Invalid object pointer 0x%p", object);
  1044. return 0;
  1045. }
  1046. if (on_freelist(s, page, object)) {
  1047. object_err(s, page, object, "Object already free");
  1048. return 0;
  1049. }
  1050. if (!check_object(s, page, object, SLUB_RED_ACTIVE))
  1051. return 0;
  1052. if (unlikely(s != page->slab_cache)) {
  1053. if (!PageSlab(page)) {
  1054. slab_err(s, page, "Attempt to free object(0x%p) outside of slab",
  1055. object);
  1056. } else if (!page->slab_cache) {
  1057. pr_err("SLUB <none>: no slab for object 0x%p.\n",
  1058. object);
  1059. dump_stack();
  1060. } else
  1061. object_err(s, page, object,
  1062. "page slab pointer corrupt.");
  1063. return 0;
  1064. }
  1065. return 1;
  1066. }
  1067. /* Supports checking bulk free of a constructed freelist */
  1068. static noinline int free_debug_processing(
  1069. struct kmem_cache *s, struct page *page,
  1070. void *head, void *tail, int bulk_cnt,
  1071. unsigned long addr)
  1072. {
  1073. struct kmem_cache_node *n = get_node(s, page_to_nid(page));
  1074. void *object = head;
  1075. int cnt = 0;
  1076. unsigned long flags;
  1077. int ret = 0;
  1078. spin_lock_irqsave(&n->list_lock, flags);
  1079. slab_lock(page);
  1080. if (s->flags & SLAB_CONSISTENCY_CHECKS) {
  1081. if (!check_slab(s, page))
  1082. goto out;
  1083. }
  1084. next_object:
  1085. cnt++;
  1086. if (s->flags & SLAB_CONSISTENCY_CHECKS) {
  1087. if (!free_consistency_checks(s, page, object, addr))
  1088. goto out;
  1089. }
  1090. if (s->flags & SLAB_STORE_USER)
  1091. set_track(s, object, TRACK_FREE, addr);
  1092. trace(s, page, object, 0);
  1093. /* Freepointer not overwritten by init_object(), SLAB_POISON moved it */
  1094. init_object(s, object, SLUB_RED_INACTIVE);
  1095. /* Reached end of constructed freelist yet? */
  1096. if (object != tail) {
  1097. object = get_freepointer(s, object);
  1098. goto next_object;
  1099. }
  1100. ret = 1;
  1101. out:
  1102. if (cnt != bulk_cnt)
  1103. slab_err(s, page, "Bulk freelist count(%d) invalid(%d)\n",
  1104. bulk_cnt, cnt);
  1105. slab_unlock(page);
  1106. spin_unlock_irqrestore(&n->list_lock, flags);
  1107. if (!ret)
  1108. slab_fix(s, "Object at 0x%p not freed", object);
  1109. return ret;
  1110. }
  1111. /*
  1112. * Parse a block of slub_debug options. Blocks are delimited by ';'
  1113. *
  1114. * @str: start of block
  1115. * @flags: returns parsed flags, or DEBUG_DEFAULT_FLAGS if none specified
  1116. * @slabs: return start of list of slabs, or NULL when there's no list
  1117. * @init: assume this is initial parsing and not per-kmem-create parsing
  1118. *
  1119. * returns the start of next block if there's any, or NULL
  1120. */
  1121. static char *
  1122. parse_slub_debug_flags(char *str, slab_flags_t *flags, char **slabs, bool init)
  1123. {
  1124. bool higher_order_disable = false;
  1125. /* Skip any completely empty blocks */
  1126. while (*str && *str == ';')
  1127. str++;
  1128. if (*str == ',') {
  1129. /*
  1130. * No options but restriction on slabs. This means full
  1131. * debugging for slabs matching a pattern.
  1132. */
  1133. *flags = DEBUG_DEFAULT_FLAGS;
  1134. goto check_slabs;
  1135. }
  1136. *flags = 0;
  1137. /* Determine which debug features should be switched on */
  1138. for (; *str && *str != ',' && *str != ';'; str++) {
  1139. switch (tolower(*str)) {
  1140. case '-':
  1141. *flags = 0;
  1142. break;
  1143. case 'f':
  1144. *flags |= SLAB_CONSISTENCY_CHECKS;
  1145. break;
  1146. case 'z':
  1147. *flags |= SLAB_RED_ZONE;
  1148. break;
  1149. case 'p':
  1150. *flags |= SLAB_POISON;
  1151. break;
  1152. case 'u':
  1153. *flags |= SLAB_STORE_USER;
  1154. break;
  1155. case 't':
  1156. *flags |= SLAB_TRACE;
  1157. break;
  1158. case 'a':
  1159. *flags |= SLAB_FAILSLAB;
  1160. break;
  1161. case 'o':
  1162. /*
  1163. * Avoid enabling debugging on caches if its minimum
  1164. * order would increase as a result.
  1165. */
  1166. higher_order_disable = true;
  1167. break;
  1168. default:
  1169. if (init)
  1170. pr_err("slub_debug option '%c' unknown. skipped\n", *str);
  1171. }
  1172. }
  1173. check_slabs:
  1174. if (*str == ',')
  1175. *slabs = ++str;
  1176. else
  1177. *slabs = NULL;
  1178. /* Skip over the slab list */
  1179. while (*str && *str != ';')
  1180. str++;
  1181. /* Skip any completely empty blocks */
  1182. while (*str && *str == ';')
  1183. str++;
  1184. if (init && higher_order_disable)
  1185. disable_higher_order_debug = 1;
  1186. if (*str)
  1187. return str;
  1188. else
  1189. return NULL;
  1190. }
  1191. static int __init setup_slub_debug(char *str)
  1192. {
  1193. slab_flags_t flags;
  1194. slab_flags_t global_flags;
  1195. char *saved_str;
  1196. char *slab_list;
  1197. bool global_slub_debug_changed = false;
  1198. bool slab_list_specified = false;
  1199. global_flags = DEBUG_DEFAULT_FLAGS;
  1200. if (*str++ != '=' || !*str)
  1201. /*
  1202. * No options specified. Switch on full debugging.
  1203. */
  1204. goto out;
  1205. saved_str = str;
  1206. while (str) {
  1207. str = parse_slub_debug_flags(str, &flags, &slab_list, true);
  1208. if (!slab_list) {
  1209. global_flags = flags;
  1210. global_slub_debug_changed = true;
  1211. } else {
  1212. slab_list_specified = true;
  1213. }
  1214. }
  1215. /*
  1216. * For backwards compatibility, a single list of flags with list of
  1217. * slabs means debugging is only changed for those slabs, so the global
  1218. * slub_debug should be unchanged (0 or DEBUG_DEFAULT_FLAGS, depending
  1219. * on CONFIG_SLUB_DEBUG_ON). We can extended that to multiple lists as
  1220. * long as there is no option specifying flags without a slab list.
  1221. */
  1222. if (slab_list_specified) {
  1223. if (!global_slub_debug_changed)
  1224. global_flags = slub_debug;
  1225. slub_debug_string = saved_str;
  1226. }
  1227. out:
  1228. slub_debug = global_flags;
  1229. if (slub_debug != 0 || slub_debug_string)
  1230. static_branch_enable(&slub_debug_enabled);
  1231. if ((static_branch_unlikely(&init_on_alloc) ||
  1232. static_branch_unlikely(&init_on_free)) &&
  1233. (slub_debug & SLAB_POISON))
  1234. pr_info("mem auto-init: SLAB_POISON will take precedence over init_on_alloc/init_on_free\n");
  1235. return 1;
  1236. }
  1237. __setup("slub_debug", setup_slub_debug);
  1238. /*
  1239. * kmem_cache_flags - apply debugging options to the cache
  1240. * @object_size: the size of an object without meta data
  1241. * @flags: flags to set
  1242. * @name: name of the cache
  1243. *
  1244. * Debug option(s) are applied to @flags. In addition to the debug
  1245. * option(s), if a slab name (or multiple) is specified i.e.
  1246. * slub_debug=<Debug-Options>,<slab name1>,<slab name2> ...
  1247. * then only the select slabs will receive the debug option(s).
  1248. */
  1249. slab_flags_t kmem_cache_flags(unsigned int object_size,
  1250. slab_flags_t flags, const char *name)
  1251. {
  1252. char *iter;
  1253. size_t len;
  1254. char *next_block;
  1255. slab_flags_t block_flags;
  1256. len = strlen(name);
  1257. next_block = slub_debug_string;
  1258. /* Go through all blocks of debug options, see if any matches our slab's name */
  1259. while (next_block) {
  1260. next_block = parse_slub_debug_flags(next_block, &block_flags, &iter, false);
  1261. if (!iter)
  1262. continue;
  1263. /* Found a block that has a slab list, search it */
  1264. while (*iter) {
  1265. char *end, *glob;
  1266. size_t cmplen;
  1267. end = strchrnul(iter, ',');
  1268. if (next_block && next_block < end)
  1269. end = next_block - 1;
  1270. glob = strnchr(iter, end - iter, '*');
  1271. if (glob)
  1272. cmplen = glob - iter;
  1273. else
  1274. cmplen = max_t(size_t, len, (end - iter));
  1275. if (!strncmp(name, iter, cmplen)) {
  1276. flags |= block_flags;
  1277. return flags;
  1278. }
  1279. if (!*end || *end == ';')
  1280. break;
  1281. iter = end + 1;
  1282. }
  1283. }
  1284. return flags | slub_debug;
  1285. }
  1286. #else /* !CONFIG_SLUB_DEBUG */
  1287. static inline void setup_object_debug(struct kmem_cache *s,
  1288. struct page *page, void *object) {}
  1289. static inline
  1290. void setup_page_debug(struct kmem_cache *s, struct page *page, void *addr) {}
  1291. static inline int alloc_debug_processing(struct kmem_cache *s,
  1292. struct page *page, void *object, unsigned long addr) { return 0; }
  1293. static inline int free_debug_processing(
  1294. struct kmem_cache *s, struct page *page,
  1295. void *head, void *tail, int bulk_cnt,
  1296. unsigned long addr) { return 0; }
  1297. static inline int slab_pad_check(struct kmem_cache *s, struct page *page)
  1298. { return 1; }
  1299. static inline int check_object(struct kmem_cache *s, struct page *page,
  1300. void *object, u8 val) { return 1; }
  1301. static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n,
  1302. struct page *page) {}
  1303. static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n,
  1304. struct page *page) {}
  1305. slab_flags_t kmem_cache_flags(unsigned int object_size,
  1306. slab_flags_t flags, const char *name)
  1307. {
  1308. return flags;
  1309. }
  1310. #define slub_debug 0
  1311. #define disable_higher_order_debug 0
  1312. static inline unsigned long slabs_node(struct kmem_cache *s, int node)
  1313. { return 0; }
  1314. static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
  1315. { return 0; }
  1316. static inline void inc_slabs_node(struct kmem_cache *s, int node,
  1317. int objects) {}
  1318. static inline void dec_slabs_node(struct kmem_cache *s, int node,
  1319. int objects) {}
  1320. static bool freelist_corrupted(struct kmem_cache *s, struct page *page,
  1321. void **freelist, void *nextfree)
  1322. {
  1323. return false;
  1324. }
  1325. #endif /* CONFIG_SLUB_DEBUG */
  1326. /*
  1327. * Hooks for other subsystems that check memory allocations. In a typical
  1328. * production configuration these hooks all should produce no code at all.
  1329. */
  1330. static inline void *kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags)
  1331. {
  1332. ptr = kasan_kmalloc_large(ptr, size, flags);
  1333. /* As ptr might get tagged, call kmemleak hook after KASAN. */
  1334. kmemleak_alloc(ptr, size, 1, flags);
  1335. return ptr;
  1336. }
  1337. static __always_inline void kfree_hook(void *x)
  1338. {
  1339. kmemleak_free(x);
  1340. kasan_kfree_large(x);
  1341. }
  1342. static __always_inline bool slab_free_hook(struct kmem_cache *s,
  1343. void *x, bool init)
  1344. {
  1345. kmemleak_free_recursive(x, s->flags);
  1346. /*
  1347. * Trouble is that we may no longer disable interrupts in the fast path
  1348. * So in order to make the debug calls that expect irqs to be
  1349. * disabled we need to disable interrupts temporarily.
  1350. */
  1351. #ifdef CONFIG_LOCKDEP
  1352. {
  1353. unsigned long flags;
  1354. local_irq_save(flags);
  1355. debug_check_no_locks_freed(x, s->object_size);
  1356. local_irq_restore(flags);
  1357. }
  1358. #endif
  1359. if (!(s->flags & SLAB_DEBUG_OBJECTS))
  1360. debug_check_no_obj_freed(x, s->object_size);
  1361. /* Use KCSAN to help debug racy use-after-free. */
  1362. if (!(s->flags & SLAB_TYPESAFE_BY_RCU))
  1363. __kcsan_check_access(x, s->object_size,
  1364. KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ASSERT);
  1365. /*
  1366. * As memory initialization might be integrated into KASAN,
  1367. * kasan_slab_free and initialization memset's must be
  1368. * kept together to avoid discrepancies in behavior.
  1369. *
  1370. * The initialization memset's clear the object and the metadata,
  1371. * but don't touch the SLAB redzone.
  1372. */
  1373. if (init) {
  1374. int rsize;
  1375. if (!kasan_has_integrated_init())
  1376. memset(kasan_reset_tag(x), 0, s->object_size);
  1377. rsize = (s->flags & SLAB_RED_ZONE) ? s->red_left_pad : 0;
  1378. memset((char *)kasan_reset_tag(x) + s->inuse, 0,
  1379. s->size - s->inuse - rsize);
  1380. }
  1381. /* KASAN might put x into memory quarantine, delaying its reuse. */
  1382. return kasan_slab_free(s, x, init);
  1383. }
  1384. static inline bool slab_free_freelist_hook(struct kmem_cache *s,
  1385. void **head, void **tail,
  1386. int *cnt)
  1387. {
  1388. void *object;
  1389. void *next = *head;
  1390. void *old_tail = *tail ? *tail : *head;
  1391. if (is_kfence_address(next)) {
  1392. slab_free_hook(s, next, false);
  1393. return true;
  1394. }
  1395. /* Head and tail of the reconstructed freelist */
  1396. *head = NULL;
  1397. *tail = NULL;
  1398. do {
  1399. object = next;
  1400. next = get_freepointer(s, object);
  1401. /* If object's reuse doesn't have to be delayed */
  1402. if (!slab_free_hook(s, object, slab_want_init_on_free(s))) {
  1403. /* Move object to the new freelist */
  1404. set_freepointer(s, object, *head);
  1405. *head = object;
  1406. if (!*tail)
  1407. *tail = object;
  1408. } else {
  1409. /*
  1410. * Adjust the reconstructed freelist depth
  1411. * accordingly if object's reuse is delayed.
  1412. */
  1413. --(*cnt);
  1414. }
  1415. } while (object != old_tail);
  1416. if (*head == *tail)
  1417. *tail = NULL;
  1418. return *head != NULL;
  1419. }
  1420. static void *setup_object(struct kmem_cache *s, struct page *page,
  1421. void *object)
  1422. {
  1423. setup_object_debug(s, page, object);
  1424. object = kasan_init_slab_obj(s, object);
  1425. if (unlikely(s->ctor)) {
  1426. kasan_unpoison_object_data(s, object);
  1427. s->ctor(object);
  1428. kasan_poison_object_data(s, object);
  1429. }
  1430. return object;
  1431. }
  1432. /*
  1433. * Slab allocation and freeing
  1434. */
  1435. static inline struct page *alloc_slab_page(struct kmem_cache *s,
  1436. gfp_t flags, int node, struct kmem_cache_order_objects oo)
  1437. {
  1438. struct page *page;
  1439. unsigned int order = oo_order(oo);
  1440. if (node == NUMA_NO_NODE)
  1441. page = alloc_pages(flags, order);
  1442. else
  1443. page = __alloc_pages_node(node, flags, order);
  1444. if (page)
  1445. account_slab_page(page, order, s);
  1446. return page;
  1447. }
  1448. #ifdef CONFIG_SLAB_FREELIST_RANDOM
  1449. /* Pre-initialize the random sequence cache */
  1450. static int init_cache_random_seq(struct kmem_cache *s)
  1451. {
  1452. unsigned int count = oo_objects(s->oo);
  1453. int err;
  1454. /* Bailout if already initialised */
  1455. if (s->random_seq)
  1456. return 0;
  1457. err = cache_random_seq_create(s, count, GFP_KERNEL);
  1458. if (err) {
  1459. pr_err("SLUB: Unable to initialize free list for %s\n",
  1460. s->name);
  1461. return err;
  1462. }
  1463. /* Transform to an offset on the set of pages */
  1464. if (s->random_seq) {
  1465. unsigned int i;
  1466. for (i = 0; i < count; i++)
  1467. s->random_seq[i] *= s->size;
  1468. }
  1469. return 0;
  1470. }
  1471. /* Initialize each random sequence freelist per cache */
  1472. static void __init init_freelist_randomization(void)
  1473. {
  1474. struct kmem_cache *s;
  1475. mutex_lock(&slab_mutex);
  1476. list_for_each_entry(s, &slab_caches, list)
  1477. init_cache_random_seq(s);
  1478. mutex_unlock(&slab_mutex);
  1479. }
  1480. /* Get the next entry on the pre-computed freelist randomized */
  1481. static void *next_freelist_entry(struct kmem_cache *s, struct page *page,
  1482. unsigned long *pos, void *start,
  1483. unsigned long page_limit,
  1484. unsigned long freelist_count)
  1485. {
  1486. unsigned int idx;
  1487. /*
  1488. * If the target page allocation failed, the number of objects on the
  1489. * page might be smaller than the usual size defined by the cache.
  1490. */
  1491. do {
  1492. idx = s->random_seq[*pos];
  1493. *pos += 1;
  1494. if (*pos >= freelist_count)
  1495. *pos = 0;
  1496. } while (unlikely(idx >= page_limit));
  1497. return (char *)start + idx;
  1498. }
  1499. /* Shuffle the single linked freelist based on a random pre-computed sequence */
  1500. static bool shuffle_freelist(struct kmem_cache *s, struct page *page)
  1501. {
  1502. void *start;
  1503. void *cur;
  1504. void *next;
  1505. unsigned long idx, pos, page_limit, freelist_count;
  1506. if (page->objects < 2 || !s->random_seq)
  1507. return false;
  1508. freelist_count = oo_objects(s->oo);
  1509. pos = get_random_int() % freelist_count;
  1510. page_limit = page->objects * s->size;
  1511. start = fixup_red_left(s, page_address(page));
  1512. /* First entry is used as the base of the freelist */
  1513. cur = next_freelist_entry(s, page, &pos, start, page_limit,
  1514. freelist_count);
  1515. cur = setup_object(s, page, cur);
  1516. page->freelist = cur;
  1517. for (idx = 1; idx < page->objects; idx++) {
  1518. next = next_freelist_entry(s, page, &pos, start, page_limit,
  1519. freelist_count);
  1520. next = setup_object(s, page, next);
  1521. set_freepointer(s, cur, next);
  1522. cur = next;
  1523. }
  1524. set_freepointer(s, cur, NULL);
  1525. return true;
  1526. }
  1527. #else
  1528. static inline int init_cache_random_seq(struct kmem_cache *s)
  1529. {
  1530. return 0;
  1531. }
  1532. static inline void init_freelist_randomization(void) { }
  1533. static inline bool shuffle_freelist(struct kmem_cache *s, struct page *page)
  1534. {
  1535. return false;
  1536. }
  1537. #endif /* CONFIG_SLAB_FREELIST_RANDOM */
  1538. static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
  1539. {
  1540. struct page *page;
  1541. struct kmem_cache_order_objects oo = s->oo;
  1542. gfp_t alloc_gfp;
  1543. void *start, *p, *next;
  1544. int idx;
  1545. bool shuffle;
  1546. flags &= gfp_allowed_mask;
  1547. if (gfpflags_allow_blocking(flags))
  1548. local_irq_enable();
  1549. flags |= s->allocflags;
  1550. /*
  1551. * Let the initial higher-order allocation fail under memory pressure
  1552. * so we fall-back to the minimum order allocation.
  1553. */
  1554. alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL;
  1555. if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(oo) > oo_order(s->min))
  1556. alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~(__GFP_RECLAIM|__GFP_NOFAIL);
  1557. page = alloc_slab_page(s, alloc_gfp, node, oo);
  1558. if (unlikely(!page)) {
  1559. oo = s->min;
  1560. alloc_gfp = flags;
  1561. /*
  1562. * Allocation may have failed due to fragmentation.
  1563. * Try a lower order alloc if possible
  1564. */
  1565. page = alloc_slab_page(s, alloc_gfp, node, oo);
  1566. if (unlikely(!page))
  1567. goto out;
  1568. stat(s, ORDER_FALLBACK);
  1569. }
  1570. page->objects = oo_objects(oo);
  1571. page->slab_cache = s;
  1572. __SetPageSlab(page);
  1573. if (page_is_pfmemalloc(page))
  1574. SetPageSlabPfmemalloc(page);
  1575. kasan_poison_slab(page);
  1576. start = page_address(page);
  1577. setup_page_debug(s, page, start);
  1578. shuffle = shuffle_freelist(s, page);
  1579. if (!shuffle) {
  1580. start = fixup_red_left(s, start);
  1581. start = setup_object(s, page, start);
  1582. page->freelist = start;
  1583. for (idx = 0, p = start; idx < page->objects - 1; idx++) {
  1584. next = p + s->size;
  1585. next = setup_object(s, page, next);
  1586. set_freepointer(s, p, next);
  1587. p = next;
  1588. }
  1589. set_freepointer(s, p, NULL);
  1590. }
  1591. page->inuse = page->objects;
  1592. page->frozen = 1;
  1593. out:
  1594. if (gfpflags_allow_blocking(flags))
  1595. local_irq_disable();
  1596. if (!page)
  1597. return NULL;
  1598. inc_slabs_node(s, page_to_nid(page), page->objects);
  1599. return page;
  1600. }
  1601. static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
  1602. {
  1603. if (unlikely(flags & GFP_SLAB_BUG_MASK))
  1604. flags = kmalloc_fix_flags(flags);
  1605. return allocate_slab(s,
  1606. flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
  1607. }
  1608. static void __free_slab(struct kmem_cache *s, struct page *page)
  1609. {
  1610. int order = compound_order(page);
  1611. int pages = 1 << order;
  1612. if (kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS)) {
  1613. void *p;
  1614. slab_pad_check(s, page);
  1615. for_each_object(p, s, page_address(page),
  1616. page->objects)
  1617. check_object(s, page, p, SLUB_RED_INACTIVE);
  1618. }
  1619. __ClearPageSlabPfmemalloc(page);
  1620. __ClearPageSlab(page);
  1621. page->mapping = NULL;
  1622. if (current->reclaim_state)
  1623. current->reclaim_state->reclaimed_slab += pages;
  1624. unaccount_slab_page(page, order, s);
  1625. __free_pages(page, order);
  1626. }
  1627. static void rcu_free_slab(struct rcu_head *h)
  1628. {
  1629. struct page *page = container_of(h, struct page, rcu_head);
  1630. __free_slab(page->slab_cache, page);
  1631. }
  1632. static void free_slab(struct kmem_cache *s, struct page *page)
  1633. {
  1634. if (unlikely(s->flags & SLAB_TYPESAFE_BY_RCU)) {
  1635. call_rcu(&page->rcu_head, rcu_free_slab);
  1636. } else
  1637. __free_slab(s, page);
  1638. }
  1639. static void discard_slab(struct kmem_cache *s, struct page *page)
  1640. {
  1641. dec_slabs_node(s, page_to_nid(page), page->objects);
  1642. free_slab(s, page);
  1643. }
  1644. /*
  1645. * Management of partially allocated slabs.
  1646. */
  1647. static inline void
  1648. __add_partial(struct kmem_cache_node *n, struct page *page, int tail)
  1649. {
  1650. n->nr_partial++;
  1651. if (tail == DEACTIVATE_TO_TAIL)
  1652. list_add_tail(&page->slab_list, &n->partial);
  1653. else
  1654. list_add(&page->slab_list, &n->partial);
  1655. }
  1656. static inline void add_partial(struct kmem_cache_node *n,
  1657. struct page *page, int tail)
  1658. {
  1659. lockdep_assert_held(&n->list_lock);
  1660. __add_partial(n, page, tail);
  1661. }
  1662. static inline void remove_partial(struct kmem_cache_node *n,
  1663. struct page *page)
  1664. {
  1665. lockdep_assert_held(&n->list_lock);
  1666. list_del(&page->slab_list);
  1667. n->nr_partial--;
  1668. }
  1669. /*
  1670. * Remove slab from the partial list, freeze it and
  1671. * return the pointer to the freelist.
  1672. *
  1673. * Returns a list of objects or NULL if it fails.
  1674. */
  1675. static inline void *acquire_slab(struct kmem_cache *s,
  1676. struct kmem_cache_node *n, struct page *page,
  1677. int mode, int *objects)
  1678. {
  1679. void *freelist;
  1680. unsigned long counters;
  1681. struct page new;
  1682. lockdep_assert_held(&n->list_lock);
  1683. /*
  1684. * Zap the freelist and set the frozen bit.
  1685. * The old freelist is the list of objects for the
  1686. * per cpu allocation list.
  1687. */
  1688. freelist = page->freelist;
  1689. counters = page->counters;
  1690. new.counters = counters;
  1691. *objects = new.objects - new.inuse;
  1692. if (mode) {
  1693. new.inuse = page->objects;
  1694. new.freelist = NULL;
  1695. } else {
  1696. new.freelist = freelist;
  1697. }
  1698. VM_BUG_ON(new.frozen);
  1699. new.frozen = 1;
  1700. if (!__cmpxchg_double_slab(s, page,
  1701. freelist, counters,
  1702. new.freelist, new.counters,
  1703. "acquire_slab"))
  1704. return NULL;
  1705. remove_partial(n, page);
  1706. WARN_ON(!freelist);
  1707. return freelist;
  1708. }
  1709. static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);
  1710. static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);
  1711. /*
  1712. * Try to allocate a partial slab from a specific node.
  1713. */
  1714. static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n,
  1715. struct kmem_cache_cpu *c, gfp_t flags)
  1716. {
  1717. struct page *page, *page2;
  1718. void *object = NULL;
  1719. unsigned int available = 0;
  1720. int objects;
  1721. /*
  1722. * Racy check. If we mistakenly see no partial slabs then we
  1723. * just allocate an empty slab. If we mistakenly try to get a
  1724. * partial slab and there is none available then get_partial()
  1725. * will return NULL.
  1726. */
  1727. if (!n || !n->nr_partial)
  1728. return NULL;
  1729. spin_lock(&n->list_lock);
  1730. list_for_each_entry_safe(page, page2, &n->partial, slab_list) {
  1731. void *t;
  1732. if (!pfmemalloc_match(page, flags))
  1733. continue;
  1734. t = acquire_slab(s, n, page, object == NULL, &objects);
  1735. if (!t)
  1736. break;
  1737. available += objects;
  1738. if (!object) {
  1739. c->page = page;
  1740. stat(s, ALLOC_FROM_PARTIAL);
  1741. object = t;
  1742. } else {
  1743. put_cpu_partial(s, page, 0);
  1744. stat(s, CPU_PARTIAL_NODE);
  1745. }
  1746. if (!kmem_cache_has_cpu_partial(s)
  1747. || available > slub_cpu_partial(s) / 2)
  1748. break;
  1749. }
  1750. spin_unlock(&n->list_lock);
  1751. return object;
  1752. }
  1753. /*
  1754. * Get a page from somewhere. Search in increasing NUMA distances.
  1755. */
  1756. static void *get_any_partial(struct kmem_cache *s, gfp_t flags,
  1757. struct kmem_cache_cpu *c)
  1758. {
  1759. #ifdef CONFIG_NUMA
  1760. struct zonelist *zonelist;
  1761. struct zoneref *z;
  1762. struct zone *zone;
  1763. enum zone_type highest_zoneidx = gfp_zone(flags);
  1764. void *object;
  1765. unsigned int cpuset_mems_cookie;
  1766. /*
  1767. * The defrag ratio allows a configuration of the tradeoffs between
  1768. * inter node defragmentation and node local allocations. A lower
  1769. * defrag_ratio increases the tendency to do local allocations
  1770. * instead of attempting to obtain partial slabs from other nodes.
  1771. *
  1772. * If the defrag_ratio is set to 0 then kmalloc() always
  1773. * returns node local objects. If the ratio is higher then kmalloc()
  1774. * may return off node objects because partial slabs are obtained
  1775. * from other nodes and filled up.
  1776. *
  1777. * If /sys/kernel/slab/xx/remote_node_defrag_ratio is set to 100
  1778. * (which makes defrag_ratio = 1000) then every (well almost)
  1779. * allocation will first attempt to defrag slab caches on other nodes.
  1780. * This means scanning over all nodes to look for partial slabs which
  1781. * may be expensive if we do it every time we are trying to find a slab
  1782. * with available objects.
  1783. */
  1784. if (!s->remote_node_defrag_ratio ||
  1785. get_cycles() % 1024 > s->remote_node_defrag_ratio)
  1786. return NULL;
  1787. do {
  1788. cpuset_mems_cookie = read_mems_allowed_begin();
  1789. zonelist = node_zonelist(mempolicy_slab_node(), flags);
  1790. for_each_zone_zonelist(zone, z, zonelist, highest_zoneidx) {
  1791. struct kmem_cache_node *n;
  1792. n = get_node(s, zone_to_nid(zone));
  1793. if (n && cpuset_zone_allowed(zone, flags) &&
  1794. n->nr_partial > s->min_partial) {
  1795. object = get_partial_node(s, n, c, flags);
  1796. if (object) {
  1797. /*
  1798. * Don't check read_mems_allowed_retry()
  1799. * here - if mems_allowed was updated in
  1800. * parallel, that was a harmless race
  1801. * between allocation and the cpuset
  1802. * update
  1803. */
  1804. return object;
  1805. }
  1806. }
  1807. }
  1808. } while (read_mems_allowed_retry(cpuset_mems_cookie));
  1809. #endif /* CONFIG_NUMA */
  1810. return NULL;
  1811. }
  1812. /*
  1813. * Get a partial page, lock it and return it.
  1814. */
  1815. static void *get_partial(struct kmem_cache *s, gfp_t flags, int node,
  1816. struct kmem_cache_cpu *c)
  1817. {
  1818. void *object;
  1819. int searchnode = node;
  1820. if (node == NUMA_NO_NODE)
  1821. searchnode = numa_mem_id();
  1822. object = get_partial_node(s, get_node(s, searchnode), c, flags);
  1823. if (object || node != NUMA_NO_NODE)
  1824. return object;
  1825. return get_any_partial(s, flags, c);
  1826. }
  1827. #ifdef CONFIG_PREEMPTION
  1828. /*
  1829. * Calculate the next globally unique transaction for disambiguation
  1830. * during cmpxchg. The transactions start with the cpu number and are then
  1831. * incremented by CONFIG_NR_CPUS.
  1832. */
  1833. #define TID_STEP roundup_pow_of_two(CONFIG_NR_CPUS)
  1834. #else
  1835. /*
  1836. * No preemption supported therefore also no need to check for
  1837. * different cpus.
  1838. */
  1839. #define TID_STEP 1
  1840. #endif
  1841. static inline unsigned long next_tid(unsigned long tid)
  1842. {
  1843. return tid + TID_STEP;
  1844. }
  1845. #ifdef SLUB_DEBUG_CMPXCHG
  1846. static inline unsigned int tid_to_cpu(unsigned long tid)
  1847. {
  1848. return tid % TID_STEP;
  1849. }
  1850. static inline unsigned long tid_to_event(unsigned long tid)
  1851. {
  1852. return tid / TID_STEP;
  1853. }
  1854. #endif
  1855. static inline unsigned int init_tid(int cpu)
  1856. {
  1857. return cpu;
  1858. }
  1859. static inline void note_cmpxchg_failure(const char *n,
  1860. const struct kmem_cache *s, unsigned long tid)
  1861. {
  1862. #ifdef SLUB_DEBUG_CMPXCHG
  1863. unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid);
  1864. pr_info("%s %s: cmpxchg redo ", n, s->name);
  1865. #ifdef CONFIG_PREEMPTION
  1866. if (tid_to_cpu(tid) != tid_to_cpu(actual_tid))
  1867. pr_warn("due to cpu change %d -> %d\n",
  1868. tid_to_cpu(tid), tid_to_cpu(actual_tid));
  1869. else
  1870. #endif
  1871. if (tid_to_event(tid) != tid_to_event(actual_tid))
  1872. pr_warn("due to cpu running other code. Event %ld->%ld\n",
  1873. tid_to_event(tid), tid_to_event(actual_tid));
  1874. else
  1875. pr_warn("for unknown reason: actual=%lx was=%lx target=%lx\n",
  1876. actual_tid, tid, next_tid(tid));
  1877. #endif
  1878. stat(s, CMPXCHG_DOUBLE_CPU_FAIL);
  1879. }
  1880. static void init_kmem_cache_cpus(struct kmem_cache *s)
  1881. {
  1882. int cpu;
  1883. for_each_possible_cpu(cpu)
  1884. per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu);
  1885. }
  1886. /*
  1887. * Remove the cpu slab
  1888. */
  1889. static void deactivate_slab(struct kmem_cache *s, struct page *page,
  1890. void *freelist, struct kmem_cache_cpu *c)
  1891. {
  1892. enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE };
  1893. struct kmem_cache_node *n = get_node(s, page_to_nid(page));
  1894. int lock = 0;
  1895. enum slab_modes l = M_NONE, m = M_NONE;
  1896. void *nextfree;
  1897. int tail = DEACTIVATE_TO_HEAD;
  1898. struct page new;
  1899. struct page old;
  1900. if (page->freelist) {
  1901. stat(s, DEACTIVATE_REMOTE_FREES);
  1902. tail = DEACTIVATE_TO_TAIL;
  1903. }
  1904. /*
  1905. * Stage one: Free all available per cpu objects back
  1906. * to the page freelist while it is still frozen. Leave the
  1907. * last one.
  1908. *
  1909. * There is no need to take the list->lock because the page
  1910. * is still frozen.
  1911. */
  1912. while (freelist && (nextfree = get_freepointer(s, freelist))) {
  1913. void *prior;
  1914. unsigned long counters;
  1915. /*
  1916. * If 'nextfree' is invalid, it is possible that the object at
  1917. * 'freelist' is already corrupted. So isolate all objects
  1918. * starting at 'freelist'.
  1919. */
  1920. if (freelist_corrupted(s, page, &freelist, nextfree))
  1921. break;
  1922. do {
  1923. prior = page->freelist;
  1924. counters = page->counters;
  1925. set_freepointer(s, freelist, prior);
  1926. new.counters = counters;
  1927. new.inuse--;
  1928. VM_BUG_ON(!new.frozen);
  1929. } while (!__cmpxchg_double_slab(s, page,
  1930. prior, counters,
  1931. freelist, new.counters,
  1932. "drain percpu freelist"));
  1933. freelist = nextfree;
  1934. }
  1935. /*
  1936. * Stage two: Ensure that the page is unfrozen while the
  1937. * list presence reflects the actual number of objects
  1938. * during unfreeze.
  1939. *
  1940. * We setup the list membership and then perform a cmpxchg
  1941. * with the count. If there is a mismatch then the page
  1942. * is not unfrozen but the page is on the wrong list.
  1943. *
  1944. * Then we restart the process which may have to remove
  1945. * the page from the list that we just put it on again
  1946. * because the number of objects in the slab may have
  1947. * changed.
  1948. */
  1949. redo:
  1950. old.freelist = page->freelist;
  1951. old.counters = page->counters;
  1952. VM_BUG_ON(!old.frozen);
  1953. /* Determine target state of the slab */
  1954. new.counters = old.counters;
  1955. if (freelist) {
  1956. new.inuse--;
  1957. set_freepointer(s, freelist, old.freelist);
  1958. new.freelist = freelist;
  1959. } else
  1960. new.freelist = old.freelist;
  1961. new.frozen = 0;
  1962. if (!new.inuse && n->nr_partial >= s->min_partial)
  1963. m = M_FREE;
  1964. else if (new.freelist) {
  1965. m = M_PARTIAL;
  1966. if (!lock) {
  1967. lock = 1;
  1968. /*
  1969. * Taking the spinlock removes the possibility
  1970. * that acquire_slab() will see a slab page that
  1971. * is frozen
  1972. */
  1973. spin_lock(&n->list_lock);
  1974. }
  1975. } else {
  1976. m = M_FULL;
  1977. #ifdef CONFIG_SLUB_DEBUG
  1978. if ((s->flags & SLAB_STORE_USER) && !lock) {
  1979. lock = 1;
  1980. /*
  1981. * This also ensures that the scanning of full
  1982. * slabs from diagnostic functions will not see
  1983. * any frozen slabs.
  1984. */
  1985. spin_lock(&n->list_lock);
  1986. }
  1987. #endif
  1988. }
  1989. if (l != m) {
  1990. if (l == M_PARTIAL)
  1991. remove_partial(n, page);
  1992. else if (l == M_FULL)
  1993. remove_full(s, n, page);
  1994. if (m == M_PARTIAL)
  1995. add_partial(n, page, tail);
  1996. else if (m == M_FULL)
  1997. add_full(s, n, page);
  1998. }
  1999. l = m;
  2000. if (!__cmpxchg_double_slab(s, page,
  2001. old.freelist, old.counters,
  2002. new.freelist, new.counters,
  2003. "unfreezing slab"))
  2004. goto redo;
  2005. if (lock)
  2006. spin_unlock(&n->list_lock);
  2007. if (m == M_PARTIAL)
  2008. stat(s, tail);
  2009. else if (m == M_FULL)
  2010. stat(s, DEACTIVATE_FULL);
  2011. else if (m == M_FREE) {
  2012. stat(s, DEACTIVATE_EMPTY);
  2013. discard_slab(s, page);
  2014. stat(s, FREE_SLAB);
  2015. }
  2016. c->page = NULL;
  2017. c->freelist = NULL;
  2018. }
  2019. /*
  2020. * Unfreeze all the cpu partial slabs.
  2021. *
  2022. * This function must be called with interrupts disabled
  2023. * for the cpu using c (or some other guarantee must be there
  2024. * to guarantee no concurrent accesses).
  2025. */
  2026. static void unfreeze_partials(struct kmem_cache *s,
  2027. struct kmem_cache_cpu *c)
  2028. {
  2029. #ifdef CONFIG_SLUB_CPU_PARTIAL
  2030. struct kmem_cache_node *n = NULL, *n2 = NULL;
  2031. struct page *page, *discard_page = NULL;
  2032. while ((page = slub_percpu_partial(c))) {
  2033. struct page new;
  2034. struct page old;
  2035. slub_set_percpu_partial(c, page);
  2036. n2 = get_node(s, page_to_nid(page));
  2037. if (n != n2) {
  2038. if (n)
  2039. spin_unlock(&n->list_lock);
  2040. n = n2;
  2041. spin_lock(&n->list_lock);
  2042. }
  2043. do {
  2044. old.freelist = page->freelist;
  2045. old.counters = page->counters;
  2046. VM_BUG_ON(!old.frozen);
  2047. new.counters = old.counters;
  2048. new.freelist = old.freelist;
  2049. new.frozen = 0;
  2050. } while (!__cmpxchg_double_slab(s, page,
  2051. old.freelist, old.counters,
  2052. new.freelist, new.counters,
  2053. "unfreezing slab"));
  2054. if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) {
  2055. page->next = discard_page;
  2056. discard_page = page;
  2057. } else {
  2058. add_partial(n, page, DEACTIVATE_TO_TAIL);
  2059. stat(s, FREE_ADD_PARTIAL);
  2060. }
  2061. }
  2062. if (n)
  2063. spin_unlock(&n->list_lock);
  2064. while (discard_page) {
  2065. page = discard_page;
  2066. discard_page = discard_page->next;
  2067. stat(s, DEACTIVATE_EMPTY);
  2068. discard_slab(s, page);
  2069. stat(s, FREE_SLAB);
  2070. }
  2071. #endif /* CONFIG_SLUB_CPU_PARTIAL */
  2072. }
  2073. /*
  2074. * Put a page that was just frozen (in __slab_free|get_partial_node) into a
  2075. * partial page slot if available.
  2076. *
  2077. * If we did not find a slot then simply move all the partials to the
  2078. * per node partial list.
  2079. */
  2080. static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
  2081. {
  2082. #ifdef CONFIG_SLUB_CPU_PARTIAL
  2083. struct page *oldpage;
  2084. int pages;
  2085. int pobjects;
  2086. preempt_disable();
  2087. do {
  2088. pages = 0;
  2089. pobjects = 0;
  2090. oldpage = this_cpu_read(s->cpu_slab->partial);
  2091. if (oldpage) {
  2092. pobjects = oldpage->pobjects;
  2093. pages = oldpage->pages;
  2094. if (drain && pobjects > slub_cpu_partial(s)) {
  2095. unsigned long flags;
  2096. /*
  2097. * partial array is full. Move the existing
  2098. * set to the per node partial list.
  2099. */
  2100. local_irq_save(flags);
  2101. unfreeze_partials(s, this_cpu_ptr(s->cpu_slab));
  2102. local_irq_restore(flags);
  2103. oldpage = NULL;
  2104. pobjects = 0;
  2105. pages = 0;
  2106. stat(s, CPU_PARTIAL_DRAIN);
  2107. }
  2108. }
  2109. pages++;
  2110. pobjects += page->objects - page->inuse;
  2111. page->pages = pages;
  2112. page->pobjects = pobjects;
  2113. page->next = oldpage;
  2114. } while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page)
  2115. != oldpage);
  2116. if (unlikely(!slub_cpu_partial(s))) {
  2117. unsigned long flags;
  2118. local_irq_save(flags);
  2119. unfreeze_partials(s, this_cpu_ptr(s->cpu_slab));
  2120. local_irq_restore(flags);
  2121. }
  2122. preempt_enable();
  2123. #endif /* CONFIG_SLUB_CPU_PARTIAL */
  2124. }
  2125. static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
  2126. {
  2127. stat(s, CPUSLAB_FLUSH);
  2128. deactivate_slab(s, c->page, c->freelist, c);
  2129. c->tid = next_tid(c->tid);
  2130. }
  2131. /*
  2132. * Flush cpu slab.
  2133. *
  2134. * Called from IPI handler with interrupts disabled.
  2135. */
  2136. static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
  2137. {
  2138. struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
  2139. if (c->page)
  2140. flush_slab(s, c);
  2141. unfreeze_partials(s, c);
  2142. }
  2143. static void flush_cpu_slab(void *d)
  2144. {
  2145. struct kmem_cache *s = d;
  2146. __flush_cpu_slab(s, smp_processor_id());
  2147. }
  2148. static bool has_cpu_slab(int cpu, void *info)
  2149. {
  2150. struct kmem_cache *s = info;
  2151. struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
  2152. return c->page || slub_percpu_partial(c);
  2153. }
  2154. static void flush_all(struct kmem_cache *s)
  2155. {
  2156. on_each_cpu_cond(has_cpu_slab, flush_cpu_slab, s, 1);
  2157. }
  2158. /*
  2159. * Use the cpu notifier to insure that the cpu slabs are flushed when
  2160. * necessary.
  2161. */
  2162. static int slub_cpu_dead(unsigned int cpu)
  2163. {
  2164. struct kmem_cache *s;
  2165. unsigned long flags;
  2166. mutex_lock(&slab_mutex);
  2167. list_for_each_entry(s, &slab_caches, list) {
  2168. local_irq_save(flags);
  2169. __flush_cpu_slab(s, cpu);
  2170. local_irq_restore(flags);
  2171. }
  2172. mutex_unlock(&slab_mutex);
  2173. return 0;
  2174. }
  2175. /*
  2176. * Check if the objects in a per cpu structure fit numa
  2177. * locality expectations.
  2178. */
  2179. static inline int node_match(struct page *page, int node)
  2180. {
  2181. #ifdef CONFIG_NUMA
  2182. if (node != NUMA_NO_NODE && page_to_nid(page) != node)
  2183. return 0;
  2184. #endif
  2185. return 1;
  2186. }
  2187. #ifdef CONFIG_SLUB_DEBUG
  2188. static int count_free(struct page *page)
  2189. {
  2190. return page->objects - page->inuse;
  2191. }
  2192. static inline unsigned long node_nr_objs(struct kmem_cache_node *n)
  2193. {
  2194. return atomic_long_read(&n->total_objects);
  2195. }
  2196. #endif /* CONFIG_SLUB_DEBUG */
  2197. #if defined(CONFIG_SLUB_DEBUG) || defined(CONFIG_SYSFS)
  2198. static unsigned long count_partial(struct kmem_cache_node *n,
  2199. int (*get_count)(struct page *))
  2200. {
  2201. unsigned long flags;
  2202. unsigned long x = 0;
  2203. struct page *page;
  2204. spin_lock_irqsave(&n->list_lock, flags);
  2205. list_for_each_entry(page, &n->partial, slab_list)
  2206. x += get_count(page);
  2207. spin_unlock_irqrestore(&n->list_lock, flags);
  2208. return x;
  2209. }
  2210. #endif /* CONFIG_SLUB_DEBUG || CONFIG_SYSFS */
  2211. static noinline void
  2212. slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid)
  2213. {
  2214. #ifdef CONFIG_SLUB_DEBUG
  2215. static DEFINE_RATELIMIT_STATE(slub_oom_rs, DEFAULT_RATELIMIT_INTERVAL,
  2216. DEFAULT_RATELIMIT_BURST);
  2217. int node;
  2218. struct kmem_cache_node *n;
  2219. if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slub_oom_rs))
  2220. return;
  2221. pr_warn("SLUB: Unable to allocate memory on node %d, gfp=%#x(%pGg)\n",
  2222. nid, gfpflags, &gfpflags);
  2223. pr_warn(" cache: %s, object size: %u, buffer size: %u, default order: %u, min order: %u\n",
  2224. s->name, s->object_size, s->size, oo_order(s->oo),
  2225. oo_order(s->min));
  2226. if (oo_order(s->min) > get_order(s->object_size))
  2227. pr_warn(" %s debugging increased min order, use slub_debug=O to disable.\n",
  2228. s->name);
  2229. for_each_kmem_cache_node(s, node, n) {
  2230. unsigned long nr_slabs;
  2231. unsigned long nr_objs;
  2232. unsigned long nr_free;
  2233. nr_free = count_partial(n, count_free);
  2234. nr_slabs = node_nr_slabs(n);
  2235. nr_objs = node_nr_objs(n);
  2236. pr_warn(" node %d: slabs: %ld, objs: %ld, free: %ld\n",
  2237. node, nr_slabs, nr_objs, nr_free);
  2238. }
  2239. #endif
  2240. }
  2241. static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags,
  2242. int node, struct kmem_cache_cpu **pc)
  2243. {
  2244. void *freelist;
  2245. struct kmem_cache_cpu *c = *pc;
  2246. struct page *page;
  2247. WARN_ON_ONCE(s->ctor && (flags & __GFP_ZERO));
  2248. freelist = get_partial(s, flags, node, c);
  2249. if (freelist)
  2250. return freelist;
  2251. page = new_slab(s, flags, node);
  2252. if (page) {
  2253. c = raw_cpu_ptr(s->cpu_slab);
  2254. if (c->page)
  2255. flush_slab(s, c);
  2256. /*
  2257. * No other reference to the page yet so we can
  2258. * muck around with it freely without cmpxchg
  2259. */
  2260. freelist = page->freelist;
  2261. page->freelist = NULL;
  2262. stat(s, ALLOC_SLAB);
  2263. c->page = page;
  2264. *pc = c;
  2265. }
  2266. return freelist;
  2267. }
  2268. static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags)
  2269. {
  2270. if (unlikely(PageSlabPfmemalloc(page)))
  2271. return gfp_pfmemalloc_allowed(gfpflags);
  2272. return true;
  2273. }
  2274. /*
  2275. * Check the page->freelist of a page and either transfer the freelist to the
  2276. * per cpu freelist or deactivate the page.
  2277. *
  2278. * The page is still frozen if the return value is not NULL.
  2279. *
  2280. * If this function returns NULL then the page has been unfrozen.
  2281. *
  2282. * This function must be called with interrupt disabled.
  2283. */
  2284. static inline void *get_freelist(struct kmem_cache *s, struct page *page)
  2285. {
  2286. struct page new;
  2287. unsigned long counters;
  2288. void *freelist;
  2289. do {
  2290. freelist = page->freelist;
  2291. counters = page->counters;
  2292. new.counters = counters;
  2293. VM_BUG_ON(!new.frozen);
  2294. new.inuse = page->objects;
  2295. new.frozen = freelist != NULL;
  2296. } while (!__cmpxchg_double_slab(s, page,
  2297. freelist, counters,
  2298. NULL, new.counters,
  2299. "get_freelist"));
  2300. return freelist;
  2301. }
  2302. /*
  2303. * Slow path. The lockless freelist is empty or we need to perform
  2304. * debugging duties.
  2305. *
  2306. * Processing is still very fast if new objects have been freed to the
  2307. * regular freelist. In that case we simply take over the regular freelist
  2308. * as the lockless freelist and zap the regular freelist.
  2309. *
  2310. * If that is not working then we fall back to the partial lists. We take the
  2311. * first element of the freelist as the object to allocate now and move the
  2312. * rest of the freelist to the lockless freelist.
  2313. *
  2314. * And if we were unable to get a new slab from the partial slab lists then
  2315. * we need to allocate a new slab. This is the slowest path since it involves
  2316. * a call to the page allocator and the setup of a new slab.
  2317. *
  2318. * Version of __slab_alloc to use when we know that interrupts are
  2319. * already disabled (which is the case for bulk allocation).
  2320. */
  2321. static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
  2322. unsigned long addr, struct kmem_cache_cpu *c)
  2323. {
  2324. void *freelist;
  2325. struct page *page;
  2326. stat(s, ALLOC_SLOWPATH);
  2327. page = c->page;
  2328. if (!page) {
  2329. /*
  2330. * if the node is not online or has no normal memory, just
  2331. * ignore the node constraint
  2332. */
  2333. if (unlikely(node != NUMA_NO_NODE &&
  2334. !node_state(node, N_NORMAL_MEMORY)))
  2335. node = NUMA_NO_NODE;
  2336. goto new_slab;
  2337. }
  2338. redo:
  2339. if (unlikely(!node_match(page, node))) {
  2340. /*
  2341. * same as above but node_match() being false already
  2342. * implies node != NUMA_NO_NODE
  2343. */
  2344. if (!node_state(node, N_NORMAL_MEMORY)) {
  2345. node = NUMA_NO_NODE;
  2346. goto redo;
  2347. } else {
  2348. stat(s, ALLOC_NODE_MISMATCH);
  2349. deactivate_slab(s, page, c->freelist, c);
  2350. goto new_slab;
  2351. }
  2352. }
  2353. /*
  2354. * By rights, we should be searching for a slab page that was
  2355. * PFMEMALLOC but right now, we are losing the pfmemalloc
  2356. * information when the page leaves the per-cpu allocator
  2357. */
  2358. if (unlikely(!pfmemalloc_match(page, gfpflags))) {
  2359. deactivate_slab(s, page, c->freelist, c);
  2360. goto new_slab;
  2361. }
  2362. /* must check again c->freelist in case of cpu migration or IRQ */
  2363. freelist = c->freelist;
  2364. if (freelist)
  2365. goto load_freelist;
  2366. freelist = get_freelist(s, page);
  2367. if (!freelist) {
  2368. c->page = NULL;
  2369. stat(s, DEACTIVATE_BYPASS);
  2370. goto new_slab;
  2371. }
  2372. stat(s, ALLOC_REFILL);
  2373. load_freelist:
  2374. /*
  2375. * freelist is pointing to the list of objects to be used.
  2376. * page is pointing to the page from which the objects are obtained.
  2377. * That page must be frozen for per cpu allocations to work.
  2378. */
  2379. VM_BUG_ON(!c->page->frozen);
  2380. c->freelist = get_freepointer(s, freelist);
  2381. c->tid = next_tid(c->tid);
  2382. return freelist;
  2383. new_slab:
  2384. if (slub_percpu_partial(c)) {
  2385. page = c->page = slub_percpu_partial(c);
  2386. slub_set_percpu_partial(c, page);
  2387. stat(s, CPU_PARTIAL_ALLOC);
  2388. goto redo;
  2389. }
  2390. freelist = new_slab_objects(s, gfpflags, node, &c);
  2391. if (unlikely(!freelist)) {
  2392. slab_out_of_memory(s, gfpflags, node);
  2393. return NULL;
  2394. }
  2395. page = c->page;
  2396. if (likely(!kmem_cache_debug(s) && pfmemalloc_match(page, gfpflags)))
  2397. goto load_freelist;
  2398. /* Only entered in the debug case */
  2399. if (kmem_cache_debug(s) &&
  2400. !alloc_debug_processing(s, page, freelist, addr))
  2401. goto new_slab; /* Slab failed checks. Next slab needed */
  2402. deactivate_slab(s, page, get_freepointer(s, freelist), c);
  2403. return freelist;
  2404. }
  2405. /*
  2406. * Another one that disabled interrupt and compensates for possible
  2407. * cpu changes by refetching the per cpu area pointer.
  2408. */
  2409. static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
  2410. unsigned long addr, struct kmem_cache_cpu *c)
  2411. {
  2412. void *p;
  2413. unsigned long flags;
  2414. local_irq_save(flags);
  2415. #ifdef CONFIG_PREEMPTION
  2416. /*
  2417. * We may have been preempted and rescheduled on a different
  2418. * cpu before disabling interrupts. Need to reload cpu area
  2419. * pointer.
  2420. */
  2421. c = this_cpu_ptr(s->cpu_slab);
  2422. #endif
  2423. p = ___slab_alloc(s, gfpflags, node, addr, c);
  2424. local_irq_restore(flags);
  2425. return p;
  2426. }
  2427. /*
  2428. * If the object has been wiped upon free, make sure it's fully initialized by
  2429. * zeroing out freelist pointer.
  2430. */
  2431. static __always_inline void maybe_wipe_obj_freeptr(struct kmem_cache *s,
  2432. void *obj)
  2433. {
  2434. if (unlikely(slab_want_init_on_free(s)) && obj)
  2435. memset((void *)((char *)kasan_reset_tag(obj) + s->offset),
  2436. 0, sizeof(void *));
  2437. }
  2438. /*
  2439. * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc)
  2440. * have the fastpath folded into their functions. So no function call
  2441. * overhead for requests that can be satisfied on the fastpath.
  2442. *
  2443. * The fastpath works by first checking if the lockless freelist can be used.
  2444. * If not then __slab_alloc is called for slow processing.
  2445. *
  2446. * Otherwise we can simply pick the next object from the lockless free list.
  2447. */
  2448. static __always_inline void *slab_alloc_node(struct kmem_cache *s,
  2449. gfp_t gfpflags, int node, unsigned long addr, size_t orig_size)
  2450. {
  2451. void *object;
  2452. struct kmem_cache_cpu *c;
  2453. struct page *page;
  2454. unsigned long tid;
  2455. struct obj_cgroup *objcg = NULL;
  2456. bool init = false;
  2457. s = slab_pre_alloc_hook(s, &objcg, 1, gfpflags);
  2458. if (!s)
  2459. return NULL;
  2460. object = kfence_alloc(s, orig_size, gfpflags);
  2461. if (unlikely(object))
  2462. goto out;
  2463. redo:
  2464. /*
  2465. * Must read kmem_cache cpu data via this cpu ptr. Preemption is
  2466. * enabled. We may switch back and forth between cpus while
  2467. * reading from one cpu area. That does not matter as long
  2468. * as we end up on the original cpu again when doing the cmpxchg.
  2469. *
  2470. * We should guarantee that tid and kmem_cache are retrieved on
  2471. * the same cpu. It could be different if CONFIG_PREEMPTION so we need
  2472. * to check if it is matched or not.
  2473. */
  2474. do {
  2475. tid = this_cpu_read(s->cpu_slab->tid);
  2476. c = raw_cpu_ptr(s->cpu_slab);
  2477. } while (IS_ENABLED(CONFIG_PREEMPTION) &&
  2478. unlikely(tid != READ_ONCE(c->tid)));
  2479. /*
  2480. * Irqless object alloc/free algorithm used here depends on sequence
  2481. * of fetching cpu_slab's data. tid should be fetched before anything
  2482. * on c to guarantee that object and page associated with previous tid
  2483. * won't be used with current tid. If we fetch tid first, object and
  2484. * page could be one associated with next tid and our alloc/free
  2485. * request will be failed. In this case, we will retry. So, no problem.
  2486. */
  2487. barrier();
  2488. /*
  2489. * The transaction ids are globally unique per cpu and per operation on
  2490. * a per cpu queue. Thus they can be guarantee that the cmpxchg_double
  2491. * occurs on the right processor and that there was no operation on the
  2492. * linked list in between.
  2493. */
  2494. object = c->freelist;
  2495. page = c->page;
  2496. if (unlikely(!object || !page || !node_match(page, node))) {
  2497. object = __slab_alloc(s, gfpflags, node, addr, c);
  2498. } else {
  2499. void *next_object = get_freepointer_safe(s, object);
  2500. /*
  2501. * The cmpxchg will only match if there was no additional
  2502. * operation and if we are on the right processor.
  2503. *
  2504. * The cmpxchg does the following atomically (without lock
  2505. * semantics!)
  2506. * 1. Relocate first pointer to the current per cpu area.
  2507. * 2. Verify that tid and freelist have not been changed
  2508. * 3. If they were not changed replace tid and freelist
  2509. *
  2510. * Since this is without lock semantics the protection is only
  2511. * against code executing on this cpu *not* from access by
  2512. * other cpus.
  2513. */
  2514. if (unlikely(!this_cpu_cmpxchg_double(
  2515. s->cpu_slab->freelist, s->cpu_slab->tid,
  2516. object, tid,
  2517. next_object, next_tid(tid)))) {
  2518. note_cmpxchg_failure("slab_alloc", s, tid);
  2519. goto redo;
  2520. }
  2521. prefetch_freepointer(s, next_object);
  2522. stat(s, ALLOC_FASTPATH);
  2523. }
  2524. maybe_wipe_obj_freeptr(s, object);
  2525. init = slab_want_init_on_alloc(gfpflags, s);
  2526. out:
  2527. slab_post_alloc_hook(s, objcg, gfpflags, 1, &object, init);
  2528. return object;
  2529. }
  2530. static __always_inline void *slab_alloc(struct kmem_cache *s,
  2531. gfp_t gfpflags, unsigned long addr, size_t orig_size)
  2532. {
  2533. return slab_alloc_node(s, gfpflags, NUMA_NO_NODE, addr, orig_size);
  2534. }
  2535. void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
  2536. {
  2537. void *ret = slab_alloc(s, gfpflags, _RET_IP_, s->object_size);
  2538. trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size,
  2539. s->size, gfpflags);
  2540. return ret;
  2541. }
  2542. EXPORT_SYMBOL(kmem_cache_alloc);
  2543. #ifdef CONFIG_TRACING
  2544. void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)
  2545. {
  2546. void *ret = slab_alloc(s, gfpflags, _RET_IP_, size);
  2547. trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags);
  2548. ret = kasan_kmalloc(s, ret, size, gfpflags);
  2549. return ret;
  2550. }
  2551. EXPORT_SYMBOL(kmem_cache_alloc_trace);
  2552. #endif
  2553. #ifdef CONFIG_NUMA
  2554. void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
  2555. {
  2556. void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_, s->object_size);
  2557. trace_kmem_cache_alloc_node(_RET_IP_, ret,
  2558. s->object_size, s->size, gfpflags, node);
  2559. return ret;
  2560. }
  2561. EXPORT_SYMBOL(kmem_cache_alloc_node);
  2562. #ifdef CONFIG_TRACING
  2563. void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
  2564. gfp_t gfpflags,
  2565. int node, size_t size)
  2566. {
  2567. void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_, size);
  2568. trace_kmalloc_node(_RET_IP_, ret,
  2569. size, s->size, gfpflags, node);
  2570. ret = kasan_kmalloc(s, ret, size, gfpflags);
  2571. return ret;
  2572. }
  2573. EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
  2574. #endif
  2575. #endif /* CONFIG_NUMA */
  2576. /*
  2577. * Slow path handling. This may still be called frequently since objects
  2578. * have a longer lifetime than the cpu slabs in most processing loads.
  2579. *
  2580. * So we still attempt to reduce cache line usage. Just take the slab
  2581. * lock and free the item. If there is no additional partial page
  2582. * handling required then we can return immediately.
  2583. */
  2584. static void __slab_free(struct kmem_cache *s, struct page *page,
  2585. void *head, void *tail, int cnt,
  2586. unsigned long addr)
  2587. {
  2588. void *prior;
  2589. int was_frozen;
  2590. struct page new;
  2591. unsigned long counters;
  2592. struct kmem_cache_node *n = NULL;
  2593. unsigned long flags;
  2594. stat(s, FREE_SLOWPATH);
  2595. if (kfence_free(head))
  2596. return;
  2597. if (kmem_cache_debug(s) &&
  2598. !free_debug_processing(s, page, head, tail, cnt, addr))
  2599. return;
  2600. do {
  2601. if (unlikely(n)) {
  2602. spin_unlock_irqrestore(&n->list_lock, flags);
  2603. n = NULL;
  2604. }
  2605. prior = page->freelist;
  2606. counters = page->counters;
  2607. set_freepointer(s, tail, prior);
  2608. new.counters = counters;
  2609. was_frozen = new.frozen;
  2610. new.inuse -= cnt;
  2611. if ((!new.inuse || !prior) && !was_frozen) {
  2612. if (kmem_cache_has_cpu_partial(s) && !prior) {
  2613. /*
  2614. * Slab was on no list before and will be
  2615. * partially empty
  2616. * We can defer the list move and instead
  2617. * freeze it.
  2618. */
  2619. new.frozen = 1;
  2620. } else { /* Needs to be taken off a list */
  2621. n = get_node(s, page_to_nid(page));
  2622. /*
  2623. * Speculatively acquire the list_lock.
  2624. * If the cmpxchg does not succeed then we may
  2625. * drop the list_lock without any processing.
  2626. *
  2627. * Otherwise the list_lock will synchronize with
  2628. * other processors updating the list of slabs.
  2629. */
  2630. spin_lock_irqsave(&n->list_lock, flags);
  2631. }
  2632. }
  2633. } while (!cmpxchg_double_slab(s, page,
  2634. prior, counters,
  2635. head, new.counters,
  2636. "__slab_free"));
  2637. if (likely(!n)) {
  2638. if (likely(was_frozen)) {
  2639. /*
  2640. * The list lock was not taken therefore no list
  2641. * activity can be necessary.
  2642. */
  2643. stat(s, FREE_FROZEN);
  2644. } else if (new.frozen) {
  2645. /*
  2646. * If we just froze the page then put it onto the
  2647. * per cpu partial list.
  2648. */
  2649. put_cpu_partial(s, page, 1);
  2650. stat(s, CPU_PARTIAL_FREE);
  2651. }
  2652. return;
  2653. }
  2654. if (unlikely(!new.inuse && n->nr_partial >= s->min_partial))
  2655. goto slab_empty;
  2656. /*
  2657. * Objects left in the slab. If it was not on the partial list before
  2658. * then add it.
  2659. */
  2660. if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) {
  2661. remove_full(s, n, page);
  2662. add_partial(n, page, DEACTIVATE_TO_TAIL);
  2663. stat(s, FREE_ADD_PARTIAL);
  2664. }
  2665. spin_unlock_irqrestore(&n->list_lock, flags);
  2666. return;
  2667. slab_empty:
  2668. if (prior) {
  2669. /*
  2670. * Slab on the partial list.
  2671. */
  2672. remove_partial(n, page);
  2673. stat(s, FREE_REMOVE_PARTIAL);
  2674. } else {
  2675. /* Slab must be on the full list */
  2676. remove_full(s, n, page);
  2677. }
  2678. spin_unlock_irqrestore(&n->list_lock, flags);
  2679. stat(s, FREE_SLAB);
  2680. discard_slab(s, page);
  2681. }
  2682. /*
  2683. * Fastpath with forced inlining to produce a kfree and kmem_cache_free that
  2684. * can perform fastpath freeing without additional function calls.
  2685. *
  2686. * The fastpath is only possible if we are freeing to the current cpu slab
  2687. * of this processor. This typically the case if we have just allocated
  2688. * the item before.
  2689. *
  2690. * If fastpath is not possible then fall back to __slab_free where we deal
  2691. * with all sorts of special processing.
  2692. *
  2693. * Bulk free of a freelist with several objects (all pointing to the
  2694. * same page) possible by specifying head and tail ptr, plus objects
  2695. * count (cnt). Bulk free indicated by tail pointer being set.
  2696. */
  2697. static __always_inline void do_slab_free(struct kmem_cache *s,
  2698. struct page *page, void *head, void *tail,
  2699. int cnt, unsigned long addr)
  2700. {
  2701. void *tail_obj = tail ? : head;
  2702. struct kmem_cache_cpu *c;
  2703. unsigned long tid;
  2704. /* memcg_slab_free_hook() is already called for bulk free. */
  2705. if (!tail)
  2706. memcg_slab_free_hook(s, &head, 1);
  2707. redo:
  2708. /*
  2709. * Determine the currently cpus per cpu slab.
  2710. * The cpu may change afterward. However that does not matter since
  2711. * data is retrieved via this pointer. If we are on the same cpu
  2712. * during the cmpxchg then the free will succeed.
  2713. */
  2714. do {
  2715. tid = this_cpu_read(s->cpu_slab->tid);
  2716. c = raw_cpu_ptr(s->cpu_slab);
  2717. } while (IS_ENABLED(CONFIG_PREEMPTION) &&
  2718. unlikely(tid != READ_ONCE(c->tid)));
  2719. /* Same with comment on barrier() in slab_alloc_node() */
  2720. barrier();
  2721. if (likely(page == c->page)) {
  2722. void **freelist = READ_ONCE(c->freelist);
  2723. set_freepointer(s, tail_obj, freelist);
  2724. if (unlikely(!this_cpu_cmpxchg_double(
  2725. s->cpu_slab->freelist, s->cpu_slab->tid,
  2726. freelist, tid,
  2727. head, next_tid(tid)))) {
  2728. note_cmpxchg_failure("slab_free", s, tid);
  2729. goto redo;
  2730. }
  2731. stat(s, FREE_FASTPATH);
  2732. } else
  2733. __slab_free(s, page, head, tail_obj, cnt, addr);
  2734. }
  2735. static __always_inline void slab_free(struct kmem_cache *s, struct page *page,
  2736. void *head, void *tail, int cnt,
  2737. unsigned long addr)
  2738. {
  2739. /*
  2740. * With KASAN enabled slab_free_freelist_hook modifies the freelist
  2741. * to remove objects, whose reuse must be delayed.
  2742. */
  2743. if (slab_free_freelist_hook(s, &head, &tail, &cnt))
  2744. do_slab_free(s, page, head, tail, cnt, addr);
  2745. }
  2746. #ifdef CONFIG_KASAN_GENERIC
  2747. void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr)
  2748. {
  2749. do_slab_free(cache, virt_to_head_page(x), x, NULL, 1, addr);
  2750. }
  2751. #endif
  2752. void kmem_cache_free(struct kmem_cache *s, void *x)
  2753. {
  2754. s = cache_from_obj(s, x);
  2755. if (!s)
  2756. return;
  2757. slab_free(s, virt_to_head_page(x), x, NULL, 1, _RET_IP_);
  2758. trace_kmem_cache_free(_RET_IP_, x);
  2759. }
  2760. EXPORT_SYMBOL(kmem_cache_free);
  2761. struct detached_freelist {
  2762. struct page *page;
  2763. void *tail;
  2764. void *freelist;
  2765. int cnt;
  2766. struct kmem_cache *s;
  2767. };
  2768. /*
  2769. * This function progressively scans the array with free objects (with
  2770. * a limited look ahead) and extract objects belonging to the same
  2771. * page. It builds a detached freelist directly within the given
  2772. * page/objects. This can happen without any need for
  2773. * synchronization, because the objects are owned by running process.
  2774. * The freelist is build up as a single linked list in the objects.
  2775. * The idea is, that this detached freelist can then be bulk
  2776. * transferred to the real freelist(s), but only requiring a single
  2777. * synchronization primitive. Look ahead in the array is limited due
  2778. * to performance reasons.
  2779. */
  2780. static inline
  2781. int build_detached_freelist(struct kmem_cache *s, size_t size,
  2782. void **p, struct detached_freelist *df)
  2783. {
  2784. size_t first_skipped_index = 0;
  2785. int lookahead = 3;
  2786. void *object;
  2787. struct page *page;
  2788. /* Always re-init detached_freelist */
  2789. df->page = NULL;
  2790. do {
  2791. object = p[--size];
  2792. /* Do we need !ZERO_OR_NULL_PTR(object) here? (for kfree) */
  2793. } while (!object && size);
  2794. if (!object)
  2795. return 0;
  2796. page = virt_to_head_page(object);
  2797. if (!s) {
  2798. /* Handle kalloc'ed objects */
  2799. if (unlikely(!PageSlab(page))) {
  2800. BUG_ON(!PageCompound(page));
  2801. kfree_hook(object);
  2802. __free_pages(page, compound_order(page));
  2803. p[size] = NULL; /* mark object processed */
  2804. return size;
  2805. }
  2806. /* Derive kmem_cache from object */
  2807. df->s = page->slab_cache;
  2808. } else {
  2809. df->s = cache_from_obj(s, object); /* Support for memcg */
  2810. }
  2811. if (is_kfence_address(object)) {
  2812. slab_free_hook(df->s, object, false);
  2813. __kfence_free(object);
  2814. p[size] = NULL; /* mark object processed */
  2815. return size;
  2816. }
  2817. /* Start new detached freelist */
  2818. df->page = page;
  2819. set_freepointer(df->s, object, NULL);
  2820. df->tail = object;
  2821. df->freelist = object;
  2822. p[size] = NULL; /* mark object processed */
  2823. df->cnt = 1;
  2824. while (size) {
  2825. object = p[--size];
  2826. if (!object)
  2827. continue; /* Skip processed objects */
  2828. /* df->page is always set at this point */
  2829. if (df->page == virt_to_head_page(object)) {
  2830. /* Opportunity build freelist */
  2831. set_freepointer(df->s, object, df->freelist);
  2832. df->freelist = object;
  2833. df->cnt++;
  2834. p[size] = NULL; /* mark object processed */
  2835. continue;
  2836. }
  2837. /* Limit look ahead search */
  2838. if (!--lookahead)
  2839. break;
  2840. if (!first_skipped_index)
  2841. first_skipped_index = size + 1;
  2842. }
  2843. return first_skipped_index;
  2844. }
  2845. /* Note that interrupts must be enabled when calling this function. */
  2846. void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
  2847. {
  2848. if (WARN_ON(!size))
  2849. return;
  2850. memcg_slab_free_hook(s, p, size);
  2851. do {
  2852. struct detached_freelist df;
  2853. size = build_detached_freelist(s, size, p, &df);
  2854. if (!df.page)
  2855. continue;
  2856. slab_free(df.s, df.page, df.freelist, df.tail, df.cnt,_RET_IP_);
  2857. } while (likely(size));
  2858. }
  2859. EXPORT_SYMBOL(kmem_cache_free_bulk);
  2860. /* Note that interrupts must be enabled when calling this function. */
  2861. int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
  2862. void **p)
  2863. {
  2864. struct kmem_cache_cpu *c;
  2865. int i;
  2866. struct obj_cgroup *objcg = NULL;
  2867. /* memcg and kmem_cache debug support */
  2868. s = slab_pre_alloc_hook(s, &objcg, size, flags);
  2869. if (unlikely(!s))
  2870. return false;
  2871. /*
  2872. * Drain objects in the per cpu slab, while disabling local
  2873. * IRQs, which protects against PREEMPT and interrupts
  2874. * handlers invoking normal fastpath.
  2875. */
  2876. local_irq_disable();
  2877. c = this_cpu_ptr(s->cpu_slab);
  2878. for (i = 0; i < size; i++) {
  2879. void *object = kfence_alloc(s, s->object_size, flags);
  2880. if (unlikely(object)) {
  2881. p[i] = object;
  2882. continue;
  2883. }
  2884. object = c->freelist;
  2885. if (unlikely(!object)) {
  2886. /*
  2887. * We may have removed an object from c->freelist using
  2888. * the fastpath in the previous iteration; in that case,
  2889. * c->tid has not been bumped yet.
  2890. * Since ___slab_alloc() may reenable interrupts while
  2891. * allocating memory, we should bump c->tid now.
  2892. */
  2893. c->tid = next_tid(c->tid);
  2894. /*
  2895. * Invoking slow path likely have side-effect
  2896. * of re-populating per CPU c->freelist
  2897. */
  2898. p[i] = ___slab_alloc(s, flags, NUMA_NO_NODE,
  2899. _RET_IP_, c);
  2900. if (unlikely(!p[i]))
  2901. goto error;
  2902. c = this_cpu_ptr(s->cpu_slab);
  2903. maybe_wipe_obj_freeptr(s, p[i]);
  2904. continue; /* goto for-loop */
  2905. }
  2906. c->freelist = get_freepointer(s, object);
  2907. p[i] = object;
  2908. maybe_wipe_obj_freeptr(s, p[i]);
  2909. }
  2910. c->tid = next_tid(c->tid);
  2911. local_irq_enable();
  2912. /*
  2913. * memcg and kmem_cache debug support and memory initialization.
  2914. * Done outside of the IRQ disabled fastpath loop.
  2915. */
  2916. slab_post_alloc_hook(s, objcg, flags, size, p,
  2917. slab_want_init_on_alloc(flags, s));
  2918. return i;
  2919. error:
  2920. local_irq_enable();
  2921. slab_post_alloc_hook(s, objcg, flags, i, p, false);
  2922. __kmem_cache_free_bulk(s, i, p);
  2923. return 0;
  2924. }
  2925. EXPORT_SYMBOL(kmem_cache_alloc_bulk);
  2926. /*
  2927. * Object placement in a slab is made very easy because we always start at
  2928. * offset 0. If we tune the size of the object to the alignment then we can
  2929. * get the required alignment by putting one properly sized object after
  2930. * another.
  2931. *
  2932. * Notice that the allocation order determines the sizes of the per cpu
  2933. * caches. Each processor has always one slab available for allocations.
  2934. * Increasing the allocation order reduces the number of times that slabs
  2935. * must be moved on and off the partial lists and is therefore a factor in
  2936. * locking overhead.
  2937. */
  2938. /*
  2939. * Mininum / Maximum order of slab pages. This influences locking overhead
  2940. * and slab fragmentation. A higher order reduces the number of partial slabs
  2941. * and increases the number of allocations possible without having to
  2942. * take the list_lock.
  2943. */
  2944. static unsigned int slub_min_order;
  2945. static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;
  2946. static unsigned int slub_min_objects;
  2947. /*
  2948. * Calculate the order of allocation given an slab object size.
  2949. *
  2950. * The order of allocation has significant impact on performance and other
  2951. * system components. Generally order 0 allocations should be preferred since
  2952. * order 0 does not cause fragmentation in the page allocator. Larger objects
  2953. * be problematic to put into order 0 slabs because there may be too much
  2954. * unused space left. We go to a higher order if more than 1/16th of the slab
  2955. * would be wasted.
  2956. *
  2957. * In order to reach satisfactory performance we must ensure that a minimum
  2958. * number of objects is in one slab. Otherwise we may generate too much
  2959. * activity on the partial lists which requires taking the list_lock. This is
  2960. * less a concern for large slabs though which are rarely used.
  2961. *
  2962. * slub_max_order specifies the order where we begin to stop considering the
  2963. * number of objects in a slab as critical. If we reach slub_max_order then
  2964. * we try to keep the page order as low as possible. So we accept more waste
  2965. * of space in favor of a small page order.
  2966. *
  2967. * Higher order allocations also allow the placement of more objects in a
  2968. * slab and thereby reduce object handling overhead. If the user has
  2969. * requested a higher mininum order then we start with that one instead of
  2970. * the smallest order which will fit the object.
  2971. */
  2972. static inline unsigned int slab_order(unsigned int size,
  2973. unsigned int min_objects, unsigned int max_order,
  2974. unsigned int fract_leftover)
  2975. {
  2976. unsigned int min_order = slub_min_order;
  2977. unsigned int order;
  2978. if (order_objects(min_order, size) > MAX_OBJS_PER_PAGE)
  2979. return get_order(size * MAX_OBJS_PER_PAGE) - 1;
  2980. for (order = max(min_order, (unsigned int)get_order(min_objects * size));
  2981. order <= max_order; order++) {
  2982. unsigned int slab_size = (unsigned int)PAGE_SIZE << order;
  2983. unsigned int rem;
  2984. rem = slab_size % size;
  2985. if (rem <= slab_size / fract_leftover)
  2986. break;
  2987. }
  2988. return order;
  2989. }
  2990. static inline int calculate_order(unsigned int size)
  2991. {
  2992. unsigned int order;
  2993. unsigned int min_objects;
  2994. unsigned int max_objects;
  2995. /*
  2996. * Attempt to find best configuration for a slab. This
  2997. * works by first attempting to generate a layout with
  2998. * the best configuration and backing off gradually.
  2999. *
  3000. * First we increase the acceptable waste in a slab. Then
  3001. * we reduce the minimum objects required in a slab.
  3002. */
  3003. min_objects = slub_min_objects;
  3004. if (!min_objects)
  3005. min_objects = 4 * (fls(nr_cpu_ids) + 1);
  3006. max_objects = order_objects(slub_max_order, size);
  3007. min_objects = min(min_objects, max_objects);
  3008. while (min_objects > 1) {
  3009. unsigned int fraction;
  3010. fraction = 16;
  3011. while (fraction >= 4) {
  3012. order = slab_order(size, min_objects,
  3013. slub_max_order, fraction);
  3014. if (order <= slub_max_order)
  3015. return order;
  3016. fraction /= 2;
  3017. }
  3018. min_objects--;
  3019. }
  3020. /*
  3021. * We were unable to place multiple objects in a slab. Now
  3022. * lets see if we can place a single object there.
  3023. */
  3024. order = slab_order(size, 1, slub_max_order, 1);
  3025. if (order <= slub_max_order)
  3026. return order;
  3027. /*
  3028. * Doh this slab cannot be placed using slub_max_order.
  3029. */
  3030. order = slab_order(size, 1, MAX_ORDER, 1);
  3031. if (order < MAX_ORDER)
  3032. return order;
  3033. return -ENOSYS;
  3034. }
  3035. static void
  3036. init_kmem_cache_node(struct kmem_cache_node *n)
  3037. {
  3038. n->nr_partial = 0;
  3039. spin_lock_init(&n->list_lock);
  3040. INIT_LIST_HEAD(&n->partial);
  3041. #ifdef CONFIG_SLUB_DEBUG
  3042. atomic_long_set(&n->nr_slabs, 0);
  3043. atomic_long_set(&n->total_objects, 0);
  3044. INIT_LIST_HEAD(&n->full);
  3045. #endif
  3046. }
  3047. static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
  3048. {
  3049. BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE <
  3050. KMALLOC_SHIFT_HIGH * sizeof(struct kmem_cache_cpu));
  3051. /*
  3052. * Must align to double word boundary for the double cmpxchg
  3053. * instructions to work; see __pcpu_double_call_return_bool().
  3054. */
  3055. s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu),
  3056. 2 * sizeof(void *));
  3057. if (!s->cpu_slab)
  3058. return 0;
  3059. init_kmem_cache_cpus(s);
  3060. return 1;
  3061. }
  3062. static struct kmem_cache *kmem_cache_node;
  3063. /*
  3064. * No kmalloc_node yet so do it by hand. We know that this is the first
  3065. * slab on the node for this slabcache. There are no concurrent accesses
  3066. * possible.
  3067. *
  3068. * Note that this function only works on the kmem_cache_node
  3069. * when allocating for the kmem_cache_node. This is used for bootstrapping
  3070. * memory on a fresh node that has no slab structures yet.
  3071. */
  3072. static void early_kmem_cache_node_alloc(int node)
  3073. {
  3074. struct page *page;
  3075. struct kmem_cache_node *n;
  3076. BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node));
  3077. page = new_slab(kmem_cache_node, GFP_NOWAIT, node);
  3078. BUG_ON(!page);
  3079. if (page_to_nid(page) != node) {
  3080. pr_err("SLUB: Unable to allocate memory from node %d\n", node);
  3081. pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n");
  3082. }
  3083. n = page->freelist;
  3084. BUG_ON(!n);
  3085. #ifdef CONFIG_SLUB_DEBUG
  3086. init_object(kmem_cache_node, n, SLUB_RED_ACTIVE);
  3087. init_tracking(kmem_cache_node, n);
  3088. #endif
  3089. n = kasan_slab_alloc(kmem_cache_node, n, GFP_KERNEL, false);
  3090. page->freelist = get_freepointer(kmem_cache_node, n);
  3091. page->inuse = 1;
  3092. page->frozen = 0;
  3093. kmem_cache_node->node[node] = n;
  3094. init_kmem_cache_node(n);
  3095. inc_slabs_node(kmem_cache_node, node, page->objects);
  3096. /*
  3097. * No locks need to be taken here as it has just been
  3098. * initialized and there is no concurrent access.
  3099. */
  3100. __add_partial(n, page, DEACTIVATE_TO_HEAD);
  3101. }
  3102. static void free_kmem_cache_nodes(struct kmem_cache *s)
  3103. {
  3104. int node;
  3105. struct kmem_cache_node *n;
  3106. for_each_kmem_cache_node(s, node, n) {
  3107. s->node[node] = NULL;
  3108. kmem_cache_free(kmem_cache_node, n);
  3109. }
  3110. }
  3111. void __kmem_cache_release(struct kmem_cache *s)
  3112. {
  3113. cache_random_seq_destroy(s);
  3114. free_percpu(s->cpu_slab);
  3115. free_kmem_cache_nodes(s);
  3116. }
  3117. static int init_kmem_cache_nodes(struct kmem_cache *s)
  3118. {
  3119. int node;
  3120. for_each_node_state(node, N_NORMAL_MEMORY) {
  3121. struct kmem_cache_node *n;
  3122. if (slab_state == DOWN) {
  3123. early_kmem_cache_node_alloc(node);
  3124. continue;
  3125. }
  3126. n = kmem_cache_alloc_node(kmem_cache_node,
  3127. GFP_KERNEL, node);
  3128. if (!n) {
  3129. free_kmem_cache_nodes(s);
  3130. return 0;
  3131. }
  3132. init_kmem_cache_node(n);
  3133. s->node[node] = n;
  3134. }
  3135. return 1;
  3136. }
  3137. static void set_min_partial(struct kmem_cache *s, unsigned long min)
  3138. {
  3139. if (min < MIN_PARTIAL)
  3140. min = MIN_PARTIAL;
  3141. else if (min > MAX_PARTIAL)
  3142. min = MAX_PARTIAL;
  3143. s->min_partial = min;
  3144. }
  3145. static void set_cpu_partial(struct kmem_cache *s)
  3146. {
  3147. #ifdef CONFIG_SLUB_CPU_PARTIAL
  3148. /*
  3149. * cpu_partial determined the maximum number of objects kept in the
  3150. * per cpu partial lists of a processor.
  3151. *
  3152. * Per cpu partial lists mainly contain slabs that just have one
  3153. * object freed. If they are used for allocation then they can be
  3154. * filled up again with minimal effort. The slab will never hit the
  3155. * per node partial lists and therefore no locking will be required.
  3156. *
  3157. * This setting also determines
  3158. *
  3159. * A) The number of objects from per cpu partial slabs dumped to the
  3160. * per node list when we reach the limit.
  3161. * B) The number of objects in cpu partial slabs to extract from the
  3162. * per node list when we run out of per cpu objects. We only fetch
  3163. * 50% to keep some capacity around for frees.
  3164. */
  3165. if (!kmem_cache_has_cpu_partial(s))
  3166. slub_set_cpu_partial(s, 0);
  3167. else if (s->size >= PAGE_SIZE)
  3168. slub_set_cpu_partial(s, 2);
  3169. else if (s->size >= 1024)
  3170. slub_set_cpu_partial(s, 6);
  3171. else if (s->size >= 256)
  3172. slub_set_cpu_partial(s, 13);
  3173. else
  3174. slub_set_cpu_partial(s, 30);
  3175. #endif
  3176. }
  3177. /*
  3178. * calculate_sizes() determines the order and the distribution of data within
  3179. * a slab object.
  3180. */
  3181. static int calculate_sizes(struct kmem_cache *s, int forced_order)
  3182. {
  3183. slab_flags_t flags = s->flags;
  3184. unsigned int size = s->object_size;
  3185. unsigned int order;
  3186. /*
  3187. * Round up object size to the next word boundary. We can only
  3188. * place the free pointer at word boundaries and this determines
  3189. * the possible location of the free pointer.
  3190. */
  3191. size = ALIGN(size, sizeof(void *));
  3192. #ifdef CONFIG_SLUB_DEBUG
  3193. /*
  3194. * Determine if we can poison the object itself. If the user of
  3195. * the slab may touch the object after free or before allocation
  3196. * then we should never poison the object itself.
  3197. */
  3198. if ((flags & SLAB_POISON) && !(flags & SLAB_TYPESAFE_BY_RCU) &&
  3199. !s->ctor)
  3200. s->flags |= __OBJECT_POISON;
  3201. else
  3202. s->flags &= ~__OBJECT_POISON;
  3203. /*
  3204. * If we are Redzoning then check if there is some space between the
  3205. * end of the object and the free pointer. If not then add an
  3206. * additional word to have some bytes to store Redzone information.
  3207. */
  3208. if ((flags & SLAB_RED_ZONE) && size == s->object_size)
  3209. size += sizeof(void *);
  3210. #endif
  3211. /*
  3212. * With that we have determined the number of bytes in actual use
  3213. * by the object and redzoning.
  3214. */
  3215. s->inuse = size;
  3216. if ((flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)) ||
  3217. ((flags & SLAB_RED_ZONE) && s->object_size < sizeof(void *)) ||
  3218. s->ctor) {
  3219. /*
  3220. * Relocate free pointer after the object if it is not
  3221. * permitted to overwrite the first word of the object on
  3222. * kmem_cache_free.
  3223. *
  3224. * This is the case if we do RCU, have a constructor or
  3225. * destructor, are poisoning the objects, or are
  3226. * redzoning an object smaller than sizeof(void *).
  3227. *
  3228. * The assumption that s->offset >= s->inuse means free
  3229. * pointer is outside of the object is used in the
  3230. * freeptr_outside_object() function. If that is no
  3231. * longer true, the function needs to be modified.
  3232. */
  3233. s->offset = size;
  3234. size += sizeof(void *);
  3235. } else {
  3236. /*
  3237. * Store freelist pointer near middle of object to keep
  3238. * it away from the edges of the object to avoid small
  3239. * sized over/underflows from neighboring allocations.
  3240. */
  3241. s->offset = ALIGN_DOWN(s->object_size / 2, sizeof(void *));
  3242. }
  3243. #ifdef CONFIG_SLUB_DEBUG
  3244. if (flags & SLAB_STORE_USER)
  3245. /*
  3246. * Need to store information about allocs and frees after
  3247. * the object.
  3248. */
  3249. size += 2 * sizeof(struct track);
  3250. #endif
  3251. kasan_cache_create(s, &size, &s->flags);
  3252. #ifdef CONFIG_SLUB_DEBUG
  3253. if (flags & SLAB_RED_ZONE) {
  3254. /*
  3255. * Add some empty padding so that we can catch
  3256. * overwrites from earlier objects rather than let
  3257. * tracking information or the free pointer be
  3258. * corrupted if a user writes before the start
  3259. * of the object.
  3260. */
  3261. size += sizeof(void *);
  3262. s->red_left_pad = sizeof(void *);
  3263. s->red_left_pad = ALIGN(s->red_left_pad, s->align);
  3264. size += s->red_left_pad;
  3265. }
  3266. #endif
  3267. /*
  3268. * SLUB stores one object immediately after another beginning from
  3269. * offset 0. In order to align the objects we have to simply size
  3270. * each object to conform to the alignment.
  3271. */
  3272. size = ALIGN(size, s->align);
  3273. s->size = size;
  3274. s->reciprocal_size = reciprocal_value(size);
  3275. if (forced_order >= 0)
  3276. order = forced_order;
  3277. else
  3278. order = calculate_order(size);
  3279. if ((int)order < 0)
  3280. return 0;
  3281. s->allocflags = 0;
  3282. if (order)
  3283. s->allocflags |= __GFP_COMP;
  3284. if (s->flags & SLAB_CACHE_DMA)
  3285. s->allocflags |= GFP_DMA;
  3286. if (s->flags & SLAB_CACHE_DMA32)
  3287. s->allocflags |= GFP_DMA32;
  3288. if (s->flags & SLAB_RECLAIM_ACCOUNT)
  3289. s->allocflags |= __GFP_RECLAIMABLE;
  3290. /*
  3291. * Determine the number of objects per slab
  3292. */
  3293. s->oo = oo_make(order, size);
  3294. s->min = oo_make(get_order(size), size);
  3295. if (oo_objects(s->oo) > oo_objects(s->max))
  3296. s->max = s->oo;
  3297. return !!oo_objects(s->oo);
  3298. }
  3299. static int kmem_cache_open(struct kmem_cache *s, slab_flags_t flags)
  3300. {
  3301. s->flags = kmem_cache_flags(s->size, flags, s->name);
  3302. #ifdef CONFIG_SLAB_FREELIST_HARDENED
  3303. s->random = get_random_long();
  3304. #endif
  3305. if (!calculate_sizes(s, -1))
  3306. goto error;
  3307. if (disable_higher_order_debug) {
  3308. /*
  3309. * Disable debugging flags that store metadata if the min slab
  3310. * order increased.
  3311. */
  3312. if (get_order(s->size) > get_order(s->object_size)) {
  3313. s->flags &= ~DEBUG_METADATA_FLAGS;
  3314. s->offset = 0;
  3315. if (!calculate_sizes(s, -1))
  3316. goto error;
  3317. }
  3318. }
  3319. #if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
  3320. defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
  3321. if (system_has_cmpxchg_double() && (s->flags & SLAB_NO_CMPXCHG) == 0)
  3322. /* Enable fast mode */
  3323. s->flags |= __CMPXCHG_DOUBLE;
  3324. #endif
  3325. /*
  3326. * The larger the object size is, the more pages we want on the partial
  3327. * list to avoid pounding the page allocator excessively.
  3328. */
  3329. set_min_partial(s, ilog2(s->size) / 2);
  3330. set_cpu_partial(s);
  3331. #ifdef CONFIG_NUMA
  3332. s->remote_node_defrag_ratio = 1000;
  3333. #endif
  3334. /* Initialize the pre-computed randomized freelist if slab is up */
  3335. if (slab_state >= UP) {
  3336. if (init_cache_random_seq(s))
  3337. goto error;
  3338. }
  3339. if (!init_kmem_cache_nodes(s))
  3340. goto error;
  3341. if (alloc_kmem_cache_cpus(s))
  3342. return 0;
  3343. error:
  3344. __kmem_cache_release(s);
  3345. return -EINVAL;
  3346. }
  3347. static void list_slab_objects(struct kmem_cache *s, struct page *page,
  3348. const char *text)
  3349. {
  3350. #ifdef CONFIG_SLUB_DEBUG
  3351. void *addr = page_address(page);
  3352. unsigned long *map;
  3353. void *p;
  3354. slab_err(s, page, text, s->name);
  3355. slab_lock(page);
  3356. map = get_map(s, page);
  3357. for_each_object(p, s, addr, page->objects) {
  3358. if (!test_bit(__obj_to_index(s, addr, p), map)) {
  3359. pr_err("INFO: Object 0x%p @offset=%tu\n", p, p - addr);
  3360. print_tracking(s, p);
  3361. }
  3362. }
  3363. put_map(map);
  3364. slab_unlock(page);
  3365. #endif
  3366. }
  3367. /*
  3368. * Attempt to free all partial slabs on a node.
  3369. * This is called from __kmem_cache_shutdown(). We must take list_lock
  3370. * because sysfs file might still access partial list after the shutdowning.
  3371. */
  3372. static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n)
  3373. {
  3374. LIST_HEAD(discard);
  3375. struct page *page, *h;
  3376. BUG_ON(irqs_disabled());
  3377. spin_lock_irq(&n->list_lock);
  3378. list_for_each_entry_safe(page, h, &n->partial, slab_list) {
  3379. if (!page->inuse) {
  3380. remove_partial(n, page);
  3381. list_add(&page->slab_list, &discard);
  3382. } else {
  3383. list_slab_objects(s, page,
  3384. "Objects remaining in %s on __kmem_cache_shutdown()");
  3385. }
  3386. }
  3387. spin_unlock_irq(&n->list_lock);
  3388. list_for_each_entry_safe(page, h, &discard, slab_list)
  3389. discard_slab(s, page);
  3390. }
  3391. bool __kmem_cache_empty(struct kmem_cache *s)
  3392. {
  3393. int node;
  3394. struct kmem_cache_node *n;
  3395. for_each_kmem_cache_node(s, node, n)
  3396. if (n->nr_partial || slabs_node(s, node))
  3397. return false;
  3398. return true;
  3399. }
  3400. /*
  3401. * Release all resources used by a slab cache.
  3402. */
  3403. int __kmem_cache_shutdown(struct kmem_cache *s)
  3404. {
  3405. int node;
  3406. struct kmem_cache_node *n;
  3407. flush_all(s);
  3408. /* Attempt to free all objects */
  3409. for_each_kmem_cache_node(s, node, n) {
  3410. free_partial(s, n);
  3411. if (n->nr_partial || slabs_node(s, node))
  3412. return 1;
  3413. }
  3414. return 0;
  3415. }
  3416. /********************************************************************
  3417. * Kmalloc subsystem
  3418. *******************************************************************/
  3419. static int __init setup_slub_min_order(char *str)
  3420. {
  3421. get_option(&str, (int *)&slub_min_order);
  3422. return 1;
  3423. }
  3424. __setup("slub_min_order=", setup_slub_min_order);
  3425. static int __init setup_slub_max_order(char *str)
  3426. {
  3427. get_option(&str, (int *)&slub_max_order);
  3428. slub_max_order = min(slub_max_order, (unsigned int)MAX_ORDER - 1);
  3429. return 1;
  3430. }
  3431. __setup("slub_max_order=", setup_slub_max_order);
  3432. static int __init setup_slub_min_objects(char *str)
  3433. {
  3434. get_option(&str, (int *)&slub_min_objects);
  3435. return 1;
  3436. }
  3437. __setup("slub_min_objects=", setup_slub_min_objects);
  3438. void *__kmalloc(size_t size, gfp_t flags)
  3439. {
  3440. struct kmem_cache *s;
  3441. void *ret;
  3442. if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
  3443. return kmalloc_large(size, flags);
  3444. s = kmalloc_slab(size, flags);
  3445. if (unlikely(ZERO_OR_NULL_PTR(s)))
  3446. return s;
  3447. ret = slab_alloc(s, flags, _RET_IP_, size);
  3448. trace_kmalloc(_RET_IP_, ret, size, s->size, flags);
  3449. ret = kasan_kmalloc(s, ret, size, flags);
  3450. return ret;
  3451. }
  3452. EXPORT_SYMBOL(__kmalloc);
  3453. #ifdef CONFIG_NUMA
  3454. static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
  3455. {
  3456. struct page *page;
  3457. void *ptr = NULL;
  3458. unsigned int order = get_order(size);
  3459. flags |= __GFP_COMP;
  3460. page = alloc_pages_node(node, flags, order);
  3461. if (page) {
  3462. ptr = page_address(page);
  3463. mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B,
  3464. PAGE_SIZE << order);
  3465. }
  3466. return kmalloc_large_node_hook(ptr, size, flags);
  3467. }
  3468. void *__kmalloc_node(size_t size, gfp_t flags, int node)
  3469. {
  3470. struct kmem_cache *s;
  3471. void *ret;
  3472. if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
  3473. ret = kmalloc_large_node(size, flags, node);
  3474. trace_kmalloc_node(_RET_IP_, ret,
  3475. size, PAGE_SIZE << get_order(size),
  3476. flags, node);
  3477. return ret;
  3478. }
  3479. s = kmalloc_slab(size, flags);
  3480. if (unlikely(ZERO_OR_NULL_PTR(s)))
  3481. return s;
  3482. ret = slab_alloc_node(s, flags, node, _RET_IP_, size);
  3483. trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node);
  3484. ret = kasan_kmalloc(s, ret, size, flags);
  3485. return ret;
  3486. }
  3487. EXPORT_SYMBOL(__kmalloc_node);
  3488. #endif /* CONFIG_NUMA */
  3489. #ifdef CONFIG_HARDENED_USERCOPY
  3490. /*
  3491. * Rejects incorrectly sized objects and objects that are to be copied
  3492. * to/from userspace but do not fall entirely within the containing slab
  3493. * cache's usercopy region.
  3494. *
  3495. * Returns NULL if check passes, otherwise const char * to name of cache
  3496. * to indicate an error.
  3497. */
  3498. void __check_heap_object(const void *ptr, unsigned long n, struct page *page,
  3499. bool to_user)
  3500. {
  3501. struct kmem_cache *s;
  3502. unsigned int offset;
  3503. size_t object_size;
  3504. bool is_kfence = is_kfence_address(ptr);
  3505. ptr = kasan_reset_tag(ptr);
  3506. /* Find object and usable object size. */
  3507. s = page->slab_cache;
  3508. /* Reject impossible pointers. */
  3509. if (ptr < page_address(page))
  3510. usercopy_abort("SLUB object not in SLUB page?!", NULL,
  3511. to_user, 0, n);
  3512. /* Find offset within object. */
  3513. if (is_kfence)
  3514. offset = ptr - kfence_object_start(ptr);
  3515. else
  3516. offset = (ptr - page_address(page)) % s->size;
  3517. /* Adjust for redzone and reject if within the redzone. */
  3518. if (!is_kfence && kmem_cache_debug_flags(s, SLAB_RED_ZONE)) {
  3519. if (offset < s->red_left_pad)
  3520. usercopy_abort("SLUB object in left red zone",
  3521. s->name, to_user, offset, n);
  3522. offset -= s->red_left_pad;
  3523. }
  3524. /* Allow address range falling entirely within usercopy region. */
  3525. if (offset >= s->useroffset &&
  3526. offset - s->useroffset <= s->usersize &&
  3527. n <= s->useroffset - offset + s->usersize)
  3528. return;
  3529. /*
  3530. * If the copy is still within the allocated object, produce
  3531. * a warning instead of rejecting the copy. This is intended
  3532. * to be a temporary method to find any missing usercopy
  3533. * whitelists.
  3534. */
  3535. object_size = slab_ksize(s);
  3536. if (usercopy_fallback &&
  3537. offset <= object_size && n <= object_size - offset) {
  3538. usercopy_warn("SLUB object", s->name, to_user, offset, n);
  3539. return;
  3540. }
  3541. usercopy_abort("SLUB object", s->name, to_user, offset, n);
  3542. }
  3543. #endif /* CONFIG_HARDENED_USERCOPY */
  3544. size_t __ksize(const void *object)
  3545. {
  3546. struct page *page;
  3547. if (unlikely(object == ZERO_SIZE_PTR))
  3548. return 0;
  3549. page = virt_to_head_page(object);
  3550. if (unlikely(!PageSlab(page))) {
  3551. WARN_ON(!PageCompound(page));
  3552. return page_size(page);
  3553. }
  3554. return slab_ksize(page->slab_cache);
  3555. }
  3556. EXPORT_SYMBOL(__ksize);
  3557. void kfree(const void *x)
  3558. {
  3559. struct page *page;
  3560. void *object = (void *)x;
  3561. trace_kfree(_RET_IP_, x);
  3562. if (unlikely(ZERO_OR_NULL_PTR(x)))
  3563. return;
  3564. page = virt_to_head_page(x);
  3565. if (unlikely(!PageSlab(page))) {
  3566. unsigned int order = compound_order(page);
  3567. BUG_ON(!PageCompound(page));
  3568. kfree_hook(object);
  3569. mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B,
  3570. -(PAGE_SIZE << order));
  3571. __free_pages(page, order);
  3572. return;
  3573. }
  3574. slab_free(page->slab_cache, page, object, NULL, 1, _RET_IP_);
  3575. }
  3576. EXPORT_SYMBOL(kfree);
  3577. #define SHRINK_PROMOTE_MAX 32
  3578. /*
  3579. * kmem_cache_shrink discards empty slabs and promotes the slabs filled
  3580. * up most to the head of the partial lists. New allocations will then
  3581. * fill those up and thus they can be removed from the partial lists.
  3582. *
  3583. * The slabs with the least items are placed last. This results in them
  3584. * being allocated from last increasing the chance that the last objects
  3585. * are freed in them.
  3586. */
  3587. int __kmem_cache_shrink(struct kmem_cache *s)
  3588. {
  3589. int node;
  3590. int i;
  3591. struct kmem_cache_node *n;
  3592. struct page *page;
  3593. struct page *t;
  3594. struct list_head discard;
  3595. struct list_head promote[SHRINK_PROMOTE_MAX];
  3596. unsigned long flags;
  3597. int ret = 0;
  3598. flush_all(s);
  3599. for_each_kmem_cache_node(s, node, n) {
  3600. INIT_LIST_HEAD(&discard);
  3601. for (i = 0; i < SHRINK_PROMOTE_MAX; i++)
  3602. INIT_LIST_HEAD(promote + i);
  3603. spin_lock_irqsave(&n->list_lock, flags);
  3604. /*
  3605. * Build lists of slabs to discard or promote.
  3606. *
  3607. * Note that concurrent frees may occur while we hold the
  3608. * list_lock. page->inuse here is the upper limit.
  3609. */
  3610. list_for_each_entry_safe(page, t, &n->partial, slab_list) {
  3611. int free = page->objects - page->inuse;
  3612. /* Do not reread page->inuse */
  3613. barrier();
  3614. /* We do not keep full slabs on the list */
  3615. BUG_ON(free <= 0);
  3616. if (free == page->objects) {
  3617. list_move(&page->slab_list, &discard);
  3618. n->nr_partial--;
  3619. } else if (free <= SHRINK_PROMOTE_MAX)
  3620. list_move(&page->slab_list, promote + free - 1);
  3621. }
  3622. /*
  3623. * Promote the slabs filled up most to the head of the
  3624. * partial list.
  3625. */
  3626. for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--)
  3627. list_splice(promote + i, &n->partial);
  3628. spin_unlock_irqrestore(&n->list_lock, flags);
  3629. /* Release empty slabs */
  3630. list_for_each_entry_safe(page, t, &discard, slab_list)
  3631. discard_slab(s, page);
  3632. if (slabs_node(s, node))
  3633. ret = 1;
  3634. }
  3635. return ret;
  3636. }
  3637. static int slab_mem_going_offline_callback(void *arg)
  3638. {
  3639. struct kmem_cache *s;
  3640. mutex_lock(&slab_mutex);
  3641. list_for_each_entry(s, &slab_caches, list)
  3642. __kmem_cache_shrink(s);
  3643. mutex_unlock(&slab_mutex);
  3644. return 0;
  3645. }
  3646. static void slab_mem_offline_callback(void *arg)
  3647. {
  3648. struct kmem_cache_node *n;
  3649. struct kmem_cache *s;
  3650. struct memory_notify *marg = arg;
  3651. int offline_node;
  3652. offline_node = marg->status_change_nid_normal;
  3653. /*
  3654. * If the node still has available memory. we need kmem_cache_node
  3655. * for it yet.
  3656. */
  3657. if (offline_node < 0)
  3658. return;
  3659. mutex_lock(&slab_mutex);
  3660. list_for_each_entry(s, &slab_caches, list) {
  3661. n = get_node(s, offline_node);
  3662. if (n) {
  3663. /*
  3664. * if n->nr_slabs > 0, slabs still exist on the node
  3665. * that is going down. We were unable to free them,
  3666. * and offline_pages() function shouldn't call this
  3667. * callback. So, we must fail.
  3668. */
  3669. BUG_ON(slabs_node(s, offline_node));
  3670. s->node[offline_node] = NULL;
  3671. kmem_cache_free(kmem_cache_node, n);
  3672. }
  3673. }
  3674. mutex_unlock(&slab_mutex);
  3675. }
  3676. static int slab_mem_going_online_callback(void *arg)
  3677. {
  3678. struct kmem_cache_node *n;
  3679. struct kmem_cache *s;
  3680. struct memory_notify *marg = arg;
  3681. int nid = marg->status_change_nid_normal;
  3682. int ret = 0;
  3683. /*
  3684. * If the node's memory is already available, then kmem_cache_node is
  3685. * already created. Nothing to do.
  3686. */
  3687. if (nid < 0)
  3688. return 0;
  3689. /*
  3690. * We are bringing a node online. No memory is available yet. We must
  3691. * allocate a kmem_cache_node structure in order to bring the node
  3692. * online.
  3693. */
  3694. mutex_lock(&slab_mutex);
  3695. list_for_each_entry(s, &slab_caches, list) {
  3696. /*
  3697. * XXX: kmem_cache_alloc_node will fallback to other nodes
  3698. * since memory is not yet available from the node that
  3699. * is brought up.
  3700. */
  3701. n = kmem_cache_alloc(kmem_cache_node, GFP_KERNEL);
  3702. if (!n) {
  3703. ret = -ENOMEM;
  3704. goto out;
  3705. }
  3706. init_kmem_cache_node(n);
  3707. s->node[nid] = n;
  3708. }
  3709. out:
  3710. mutex_unlock(&slab_mutex);
  3711. return ret;
  3712. }
  3713. static int slab_memory_callback(struct notifier_block *self,
  3714. unsigned long action, void *arg)
  3715. {
  3716. int ret = 0;
  3717. switch (action) {
  3718. case MEM_GOING_ONLINE:
  3719. ret = slab_mem_going_online_callback(arg);
  3720. break;
  3721. case MEM_GOING_OFFLINE:
  3722. ret = slab_mem_going_offline_callback(arg);
  3723. break;
  3724. case MEM_OFFLINE:
  3725. case MEM_CANCEL_ONLINE:
  3726. slab_mem_offline_callback(arg);
  3727. break;
  3728. case MEM_ONLINE:
  3729. case MEM_CANCEL_OFFLINE:
  3730. break;
  3731. }
  3732. if (ret)
  3733. ret = notifier_from_errno(ret);
  3734. else
  3735. ret = NOTIFY_OK;
  3736. return ret;
  3737. }
  3738. static struct notifier_block slab_memory_callback_nb = {
  3739. .notifier_call = slab_memory_callback,
  3740. .priority = SLAB_CALLBACK_PRI,
  3741. };
  3742. /********************************************************************
  3743. * Basic setup of slabs
  3744. *******************************************************************/
  3745. /*
  3746. * Used for early kmem_cache structures that were allocated using
  3747. * the page allocator. Allocate them properly then fix up the pointers
  3748. * that may be pointing to the wrong kmem_cache structure.
  3749. */
  3750. static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache)
  3751. {
  3752. int node;
  3753. struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
  3754. struct kmem_cache_node *n;
  3755. memcpy(s, static_cache, kmem_cache->object_size);
  3756. /*
  3757. * This runs very early, and only the boot processor is supposed to be
  3758. * up. Even if it weren't true, IRQs are not up so we couldn't fire
  3759. * IPIs around.
  3760. */
  3761. __flush_cpu_slab(s, smp_processor_id());
  3762. for_each_kmem_cache_node(s, node, n) {
  3763. struct page *p;
  3764. list_for_each_entry(p, &n->partial, slab_list)
  3765. p->slab_cache = s;
  3766. #ifdef CONFIG_SLUB_DEBUG
  3767. list_for_each_entry(p, &n->full, slab_list)
  3768. p->slab_cache = s;
  3769. #endif
  3770. }
  3771. list_add(&s->list, &slab_caches);
  3772. return s;
  3773. }
  3774. void __init kmem_cache_init(void)
  3775. {
  3776. static __initdata struct kmem_cache boot_kmem_cache,
  3777. boot_kmem_cache_node;
  3778. if (debug_guardpage_minorder())
  3779. slub_max_order = 0;
  3780. kmem_cache_node = &boot_kmem_cache_node;
  3781. kmem_cache = &boot_kmem_cache;
  3782. create_boot_cache(kmem_cache_node, "kmem_cache_node",
  3783. sizeof(struct kmem_cache_node), SLAB_HWCACHE_ALIGN, 0, 0);
  3784. register_hotmemory_notifier(&slab_memory_callback_nb);
  3785. /* Able to allocate the per node structures */
  3786. slab_state = PARTIAL;
  3787. create_boot_cache(kmem_cache, "kmem_cache",
  3788. offsetof(struct kmem_cache, node) +
  3789. nr_node_ids * sizeof(struct kmem_cache_node *),
  3790. SLAB_HWCACHE_ALIGN, 0, 0);
  3791. kmem_cache = bootstrap(&boot_kmem_cache);
  3792. kmem_cache_node = bootstrap(&boot_kmem_cache_node);
  3793. /* Now we can use the kmem_cache to allocate kmalloc slabs */
  3794. setup_kmalloc_cache_index_table();
  3795. create_kmalloc_caches(0);
  3796. /* Setup random freelists for each cache */
  3797. init_freelist_randomization();
  3798. cpuhp_setup_state_nocalls(CPUHP_SLUB_DEAD, "slub:dead", NULL,
  3799. slub_cpu_dead);
  3800. pr_info("SLUB: HWalign=%d, Order=%u-%u, MinObjects=%u, CPUs=%u, Nodes=%u\n",
  3801. cache_line_size(),
  3802. slub_min_order, slub_max_order, slub_min_objects,
  3803. nr_cpu_ids, nr_node_ids);
  3804. }
  3805. void __init kmem_cache_init_late(void)
  3806. {
  3807. }
  3808. struct kmem_cache *
  3809. __kmem_cache_alias(const char *name, unsigned int size, unsigned int align,
  3810. slab_flags_t flags, void (*ctor)(void *))
  3811. {
  3812. struct kmem_cache *s;
  3813. s = find_mergeable(size, align, flags, name, ctor);
  3814. if (s) {
  3815. s->refcount++;
  3816. /*
  3817. * Adjust the object sizes so that we clear
  3818. * the complete object on kzalloc.
  3819. */
  3820. s->object_size = max(s->object_size, size);
  3821. s->inuse = max(s->inuse, ALIGN(size, sizeof(void *)));
  3822. if (sysfs_slab_alias(s, name)) {
  3823. s->refcount--;
  3824. s = NULL;
  3825. }
  3826. }
  3827. return s;
  3828. }
  3829. int __kmem_cache_create(struct kmem_cache *s, slab_flags_t flags)
  3830. {
  3831. int err;
  3832. err = kmem_cache_open(s, flags);
  3833. if (err)
  3834. return err;
  3835. /* Mutex is not taken during early boot */
  3836. if (slab_state <= UP)
  3837. return 0;
  3838. err = sysfs_slab_add(s);
  3839. if (err) {
  3840. __kmem_cache_release(s);
  3841. return err;
  3842. }
  3843. if (s->flags & SLAB_STORE_USER)
  3844. debugfs_slab_add(s);
  3845. return 0;
  3846. }
  3847. void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
  3848. {
  3849. struct kmem_cache *s;
  3850. void *ret;
  3851. if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
  3852. return kmalloc_large(size, gfpflags);
  3853. s = kmalloc_slab(size, gfpflags);
  3854. if (unlikely(ZERO_OR_NULL_PTR(s)))
  3855. return s;
  3856. ret = slab_alloc(s, gfpflags, caller, size);
  3857. /* Honor the call site pointer we received. */
  3858. trace_kmalloc(caller, ret, size, s->size, gfpflags);
  3859. return ret;
  3860. }
  3861. EXPORT_SYMBOL(__kmalloc_track_caller);
  3862. #ifdef CONFIG_NUMA
  3863. void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
  3864. int node, unsigned long caller)
  3865. {
  3866. struct kmem_cache *s;
  3867. void *ret;
  3868. if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
  3869. ret = kmalloc_large_node(size, gfpflags, node);
  3870. trace_kmalloc_node(caller, ret,
  3871. size, PAGE_SIZE << get_order(size),
  3872. gfpflags, node);
  3873. return ret;
  3874. }
  3875. s = kmalloc_slab(size, gfpflags);
  3876. if (unlikely(ZERO_OR_NULL_PTR(s)))
  3877. return s;
  3878. ret = slab_alloc_node(s, gfpflags, node, caller, size);
  3879. /* Honor the call site pointer we received. */
  3880. trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node);
  3881. return ret;
  3882. }
  3883. EXPORT_SYMBOL(__kmalloc_node_track_caller);
  3884. #endif
  3885. #ifdef CONFIG_SYSFS
  3886. static int count_inuse(struct page *page)
  3887. {
  3888. return page->inuse;
  3889. }
  3890. static int count_total(struct page *page)
  3891. {
  3892. return page->objects;
  3893. }
  3894. #endif
  3895. #ifdef CONFIG_SLUB_DEBUG
  3896. static void validate_slab(struct kmem_cache *s, struct page *page)
  3897. {
  3898. void *p;
  3899. void *addr = page_address(page);
  3900. unsigned long *map;
  3901. slab_lock(page);
  3902. if (!check_slab(s, page) || !on_freelist(s, page, NULL))
  3903. goto unlock;
  3904. /* Now we know that a valid freelist exists */
  3905. map = get_map(s, page);
  3906. for_each_object(p, s, addr, page->objects) {
  3907. u8 val = test_bit(__obj_to_index(s, addr, p), map) ?
  3908. SLUB_RED_INACTIVE : SLUB_RED_ACTIVE;
  3909. if (!check_object(s, page, p, val))
  3910. break;
  3911. }
  3912. put_map(map);
  3913. unlock:
  3914. slab_unlock(page);
  3915. }
  3916. static int validate_slab_node(struct kmem_cache *s,
  3917. struct kmem_cache_node *n)
  3918. {
  3919. unsigned long count = 0;
  3920. struct page *page;
  3921. unsigned long flags;
  3922. spin_lock_irqsave(&n->list_lock, flags);
  3923. list_for_each_entry(page, &n->partial, slab_list) {
  3924. validate_slab(s, page);
  3925. count++;
  3926. }
  3927. if (count != n->nr_partial)
  3928. pr_err("SLUB %s: %ld partial slabs counted but counter=%ld\n",
  3929. s->name, count, n->nr_partial);
  3930. if (!(s->flags & SLAB_STORE_USER))
  3931. goto out;
  3932. list_for_each_entry(page, &n->full, slab_list) {
  3933. validate_slab(s, page);
  3934. count++;
  3935. }
  3936. if (count != atomic_long_read(&n->nr_slabs))
  3937. pr_err("SLUB: %s %ld slabs counted but counter=%ld\n",
  3938. s->name, count, atomic_long_read(&n->nr_slabs));
  3939. out:
  3940. spin_unlock_irqrestore(&n->list_lock, flags);
  3941. return count;
  3942. }
  3943. static long validate_slab_cache(struct kmem_cache *s)
  3944. {
  3945. int node;
  3946. unsigned long count = 0;
  3947. struct kmem_cache_node *n;
  3948. flush_all(s);
  3949. for_each_kmem_cache_node(s, node, n)
  3950. count += validate_slab_node(s, n);
  3951. return count;
  3952. }
  3953. #ifdef CONFIG_DEBUG_FS
  3954. /*
  3955. * Generate lists of code addresses where slabcache objects are allocated
  3956. * and freed.
  3957. */
  3958. struct location {
  3959. unsigned long count;
  3960. unsigned long addr;
  3961. long long sum_time;
  3962. long min_time;
  3963. long max_time;
  3964. long min_pid;
  3965. long max_pid;
  3966. DECLARE_BITMAP(cpus, NR_CPUS);
  3967. nodemask_t nodes;
  3968. };
  3969. struct loc_track {
  3970. unsigned long max;
  3971. unsigned long count;
  3972. struct location *loc;
  3973. loff_t idx;
  3974. };
  3975. static struct dentry *slab_debugfs_root;
  3976. static void free_loc_track(struct loc_track *t)
  3977. {
  3978. if (t->max)
  3979. free_pages((unsigned long)t->loc,
  3980. get_order(sizeof(struct location) * t->max));
  3981. }
  3982. static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags)
  3983. {
  3984. struct location *l;
  3985. int order;
  3986. order = get_order(sizeof(struct location) * max);
  3987. l = (void *)__get_free_pages(flags, order);
  3988. if (!l)
  3989. return 0;
  3990. if (t->count) {
  3991. memcpy(l, t->loc, sizeof(struct location) * t->count);
  3992. free_loc_track(t);
  3993. }
  3994. t->max = max;
  3995. t->loc = l;
  3996. return 1;
  3997. }
  3998. static int add_location(struct loc_track *t, struct kmem_cache *s,
  3999. const struct track *track)
  4000. {
  4001. long start, end, pos;
  4002. struct location *l;
  4003. unsigned long caddr;
  4004. unsigned long age = jiffies - track->when;
  4005. start = -1;
  4006. end = t->count;
  4007. for ( ; ; ) {
  4008. pos = start + (end - start + 1) / 2;
  4009. /*
  4010. * There is nothing at "end". If we end up there
  4011. * we need to add something to before end.
  4012. */
  4013. if (pos == end)
  4014. break;
  4015. caddr = t->loc[pos].addr;
  4016. if (track->addr == caddr) {
  4017. l = &t->loc[pos];
  4018. l->count++;
  4019. if (track->when) {
  4020. l->sum_time += age;
  4021. if (age < l->min_time)
  4022. l->min_time = age;
  4023. if (age > l->max_time)
  4024. l->max_time = age;
  4025. if (track->pid < l->min_pid)
  4026. l->min_pid = track->pid;
  4027. if (track->pid > l->max_pid)
  4028. l->max_pid = track->pid;
  4029. cpumask_set_cpu(track->cpu,
  4030. to_cpumask(l->cpus));
  4031. }
  4032. node_set(page_to_nid(virt_to_page(track)), l->nodes);
  4033. return 1;
  4034. }
  4035. if (track->addr < caddr)
  4036. end = pos;
  4037. else
  4038. start = pos;
  4039. }
  4040. /*
  4041. * Not found. Insert new tracking element.
  4042. */
  4043. if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC))
  4044. return 0;
  4045. l = t->loc + pos;
  4046. if (pos < t->count)
  4047. memmove(l + 1, l,
  4048. (t->count - pos) * sizeof(struct location));
  4049. t->count++;
  4050. l->count = 1;
  4051. l->addr = track->addr;
  4052. l->sum_time = age;
  4053. l->min_time = age;
  4054. l->max_time = age;
  4055. l->min_pid = track->pid;
  4056. l->max_pid = track->pid;
  4057. cpumask_clear(to_cpumask(l->cpus));
  4058. cpumask_set_cpu(track->cpu, to_cpumask(l->cpus));
  4059. nodes_clear(l->nodes);
  4060. node_set(page_to_nid(virt_to_page(track)), l->nodes);
  4061. return 1;
  4062. }
  4063. static void process_slab(struct loc_track *t, struct kmem_cache *s,
  4064. struct page *page, enum track_item alloc,
  4065. unsigned long *obj_map)
  4066. {
  4067. void *addr = page_address(page);
  4068. void *p;
  4069. __fill_map(obj_map, s, page);
  4070. for_each_object(p, s, addr, page->objects)
  4071. if (!test_bit(__obj_to_index(s, addr, p), obj_map))
  4072. add_location(t, s, get_track(s, p, alloc));
  4073. }
  4074. #endif /* CONFIG_DEBUG_FS */
  4075. #endif /* CONFIG_SLUB_DEBUG */
  4076. #ifdef SLUB_RESILIENCY_TEST
  4077. static void __init resiliency_test(void)
  4078. {
  4079. u8 *p;
  4080. int type = KMALLOC_NORMAL;
  4081. BUILD_BUG_ON(KMALLOC_MIN_SIZE > 16 || KMALLOC_SHIFT_HIGH < 10);
  4082. pr_err("SLUB resiliency testing\n");
  4083. pr_err("-----------------------\n");
  4084. pr_err("A. Corruption after allocation\n");
  4085. p = kzalloc(16, GFP_KERNEL);
  4086. p[16] = 0x12;
  4087. pr_err("\n1. kmalloc-16: Clobber Redzone/next pointer 0x12->0x%p\n\n",
  4088. p + 16);
  4089. validate_slab_cache(kmalloc_caches[type][4]);
  4090. /* Hmmm... The next two are dangerous */
  4091. p = kzalloc(32, GFP_KERNEL);
  4092. p[32 + sizeof(void *)] = 0x34;
  4093. pr_err("\n2. kmalloc-32: Clobber next pointer/next slab 0x34 -> -0x%p\n",
  4094. p);
  4095. pr_err("If allocated object is overwritten then not detectable\n\n");
  4096. validate_slab_cache(kmalloc_caches[type][5]);
  4097. p = kzalloc(64, GFP_KERNEL);
  4098. p += 64 + (get_cycles() & 0xff) * sizeof(void *);
  4099. *p = 0x56;
  4100. pr_err("\n3. kmalloc-64: corrupting random byte 0x56->0x%p\n",
  4101. p);
  4102. pr_err("If allocated object is overwritten then not detectable\n\n");
  4103. validate_slab_cache(kmalloc_caches[type][6]);
  4104. pr_err("\nB. Corruption after free\n");
  4105. p = kzalloc(128, GFP_KERNEL);
  4106. kfree(p);
  4107. *p = 0x78;
  4108. pr_err("1. kmalloc-128: Clobber first word 0x78->0x%p\n\n", p);
  4109. validate_slab_cache(kmalloc_caches[type][7]);
  4110. p = kzalloc(256, GFP_KERNEL);
  4111. kfree(p);
  4112. p[50] = 0x9a;
  4113. pr_err("\n2. kmalloc-256: Clobber 50th byte 0x9a->0x%p\n\n", p);
  4114. validate_slab_cache(kmalloc_caches[type][8]);
  4115. p = kzalloc(512, GFP_KERNEL);
  4116. kfree(p);
  4117. p[512] = 0xab;
  4118. pr_err("\n3. kmalloc-512: Clobber redzone 0xab->0x%p\n\n", p);
  4119. validate_slab_cache(kmalloc_caches[type][9]);
  4120. }
  4121. #else
  4122. #ifdef CONFIG_SYSFS
  4123. static void resiliency_test(void) {};
  4124. #endif
  4125. #endif /* SLUB_RESILIENCY_TEST */
  4126. #ifdef CONFIG_SYSFS
  4127. enum slab_stat_type {
  4128. SL_ALL, /* All slabs */
  4129. SL_PARTIAL, /* Only partially allocated slabs */
  4130. SL_CPU, /* Only slabs used for cpu caches */
  4131. SL_OBJECTS, /* Determine allocated objects not slabs */
  4132. SL_TOTAL /* Determine object capacity not slabs */
  4133. };
  4134. #define SO_ALL (1 << SL_ALL)
  4135. #define SO_PARTIAL (1 << SL_PARTIAL)
  4136. #define SO_CPU (1 << SL_CPU)
  4137. #define SO_OBJECTS (1 << SL_OBJECTS)
  4138. #define SO_TOTAL (1 << SL_TOTAL)
  4139. #ifdef CONFIG_MEMCG
  4140. static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);
  4141. static int __init setup_slub_memcg_sysfs(char *str)
  4142. {
  4143. int v;
  4144. if (get_option(&str, &v) > 0)
  4145. memcg_sysfs_enabled = v;
  4146. return 1;
  4147. }
  4148. __setup("slub_memcg_sysfs=", setup_slub_memcg_sysfs);
  4149. #endif
  4150. static ssize_t show_slab_objects(struct kmem_cache *s,
  4151. char *buf, unsigned long flags)
  4152. {
  4153. unsigned long total = 0;
  4154. int node;
  4155. int x;
  4156. unsigned long *nodes;
  4157. nodes = kcalloc(nr_node_ids, sizeof(unsigned long), GFP_KERNEL);
  4158. if (!nodes)
  4159. return -ENOMEM;
  4160. if (flags & SO_CPU) {
  4161. int cpu;
  4162. for_each_possible_cpu(cpu) {
  4163. struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab,
  4164. cpu);
  4165. int node;
  4166. struct page *page;
  4167. page = READ_ONCE(c->page);
  4168. if (!page)
  4169. continue;
  4170. node = page_to_nid(page);
  4171. if (flags & SO_TOTAL)
  4172. x = page->objects;
  4173. else if (flags & SO_OBJECTS)
  4174. x = page->inuse;
  4175. else
  4176. x = 1;
  4177. total += x;
  4178. nodes[node] += x;
  4179. page = slub_percpu_partial_read_once(c);
  4180. if (page) {
  4181. node = page_to_nid(page);
  4182. if (flags & SO_TOTAL)
  4183. WARN_ON_ONCE(1);
  4184. else if (flags & SO_OBJECTS)
  4185. WARN_ON_ONCE(1);
  4186. else
  4187. x = page->pages;
  4188. total += x;
  4189. nodes[node] += x;
  4190. }
  4191. }
  4192. }
  4193. /*
  4194. * It is impossible to take "mem_hotplug_lock" here with "kernfs_mutex"
  4195. * already held which will conflict with an existing lock order:
  4196. *
  4197. * mem_hotplug_lock->slab_mutex->kernfs_mutex
  4198. *
  4199. * We don't really need mem_hotplug_lock (to hold off
  4200. * slab_mem_going_offline_callback) here because slab's memory hot
  4201. * unplug code doesn't destroy the kmem_cache->node[] data.
  4202. */
  4203. #ifdef CONFIG_SLUB_DEBUG
  4204. if (flags & SO_ALL) {
  4205. struct kmem_cache_node *n;
  4206. for_each_kmem_cache_node(s, node, n) {
  4207. if (flags & SO_TOTAL)
  4208. x = atomic_long_read(&n->total_objects);
  4209. else if (flags & SO_OBJECTS)
  4210. x = atomic_long_read(&n->total_objects) -
  4211. count_partial(n, count_free);
  4212. else
  4213. x = atomic_long_read(&n->nr_slabs);
  4214. total += x;
  4215. nodes[node] += x;
  4216. }
  4217. } else
  4218. #endif
  4219. if (flags & SO_PARTIAL) {
  4220. struct kmem_cache_node *n;
  4221. for_each_kmem_cache_node(s, node, n) {
  4222. if (flags & SO_TOTAL)
  4223. x = count_partial(n, count_total);
  4224. else if (flags & SO_OBJECTS)
  4225. x = count_partial(n, count_inuse);
  4226. else
  4227. x = n->nr_partial;
  4228. total += x;
  4229. nodes[node] += x;
  4230. }
  4231. }
  4232. x = sprintf(buf, "%lu", total);
  4233. #ifdef CONFIG_NUMA
  4234. for (node = 0; node < nr_node_ids; node++)
  4235. if (nodes[node])
  4236. x += sprintf(buf + x, " N%d=%lu",
  4237. node, nodes[node]);
  4238. #endif
  4239. kfree(nodes);
  4240. return x + sprintf(buf + x, "\n");
  4241. }
  4242. #define to_slab_attr(n) container_of(n, struct slab_attribute, attr)
  4243. #define to_slab(n) container_of(n, struct kmem_cache, kobj)
  4244. struct slab_attribute {
  4245. struct attribute attr;
  4246. ssize_t (*show)(struct kmem_cache *s, char *buf);
  4247. ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
  4248. };
  4249. #define SLAB_ATTR_RO(_name) \
  4250. static struct slab_attribute _name##_attr = \
  4251. __ATTR(_name, 0400, _name##_show, NULL)
  4252. #define SLAB_ATTR(_name) \
  4253. static struct slab_attribute _name##_attr = \
  4254. __ATTR(_name, 0600, _name##_show, _name##_store)
  4255. static ssize_t slab_size_show(struct kmem_cache *s, char *buf)
  4256. {
  4257. return sprintf(buf, "%u\n", s->size);
  4258. }
  4259. SLAB_ATTR_RO(slab_size);
  4260. static ssize_t align_show(struct kmem_cache *s, char *buf)
  4261. {
  4262. return sprintf(buf, "%u\n", s->align);
  4263. }
  4264. SLAB_ATTR_RO(align);
  4265. static ssize_t object_size_show(struct kmem_cache *s, char *buf)
  4266. {
  4267. return sprintf(buf, "%u\n", s->object_size);
  4268. }
  4269. SLAB_ATTR_RO(object_size);
  4270. static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf)
  4271. {
  4272. return sprintf(buf, "%u\n", oo_objects(s->oo));
  4273. }
  4274. SLAB_ATTR_RO(objs_per_slab);
  4275. static ssize_t order_show(struct kmem_cache *s, char *buf)
  4276. {
  4277. return sprintf(buf, "%u\n", oo_order(s->oo));
  4278. }
  4279. SLAB_ATTR_RO(order);
  4280. static ssize_t min_partial_show(struct kmem_cache *s, char *buf)
  4281. {
  4282. return sprintf(buf, "%lu\n", s->min_partial);
  4283. }
  4284. static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,
  4285. size_t length)
  4286. {
  4287. unsigned long min;
  4288. int err;
  4289. err = kstrtoul(buf, 10, &min);
  4290. if (err)
  4291. return err;
  4292. set_min_partial(s, min);
  4293. return length;
  4294. }
  4295. SLAB_ATTR(min_partial);
  4296. static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf)
  4297. {
  4298. return sprintf(buf, "%u\n", slub_cpu_partial(s));
  4299. }
  4300. static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,
  4301. size_t length)
  4302. {
  4303. unsigned int objects;
  4304. int err;
  4305. err = kstrtouint(buf, 10, &objects);
  4306. if (err)
  4307. return err;
  4308. if (objects && !kmem_cache_has_cpu_partial(s))
  4309. return -EINVAL;
  4310. slub_set_cpu_partial(s, objects);
  4311. flush_all(s);
  4312. return length;
  4313. }
  4314. SLAB_ATTR(cpu_partial);
  4315. static ssize_t ctor_show(struct kmem_cache *s, char *buf)
  4316. {
  4317. if (!s->ctor)
  4318. return 0;
  4319. return sprintf(buf, "%pS\n", s->ctor);
  4320. }
  4321. SLAB_ATTR_RO(ctor);
  4322. static ssize_t aliases_show(struct kmem_cache *s, char *buf)
  4323. {
  4324. return sprintf(buf, "%d\n", s->refcount < 0 ? 0 : s->refcount - 1);
  4325. }
  4326. SLAB_ATTR_RO(aliases);
  4327. static ssize_t partial_show(struct kmem_cache *s, char *buf)
  4328. {
  4329. return show_slab_objects(s, buf, SO_PARTIAL);
  4330. }
  4331. SLAB_ATTR_RO(partial);
  4332. static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf)
  4333. {
  4334. return show_slab_objects(s, buf, SO_CPU);
  4335. }
  4336. SLAB_ATTR_RO(cpu_slabs);
  4337. static ssize_t objects_show(struct kmem_cache *s, char *buf)
  4338. {
  4339. return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS);
  4340. }
  4341. SLAB_ATTR_RO(objects);
  4342. static ssize_t objects_partial_show(struct kmem_cache *s, char *buf)
  4343. {
  4344. return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS);
  4345. }
  4346. SLAB_ATTR_RO(objects_partial);
  4347. static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf)
  4348. {
  4349. int objects = 0;
  4350. int pages = 0;
  4351. int cpu;
  4352. int len;
  4353. for_each_online_cpu(cpu) {
  4354. struct page *page;
  4355. page = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu));
  4356. if (page) {
  4357. pages += page->pages;
  4358. objects += page->pobjects;
  4359. }
  4360. }
  4361. len = sprintf(buf, "%d(%d)", objects, pages);
  4362. #ifdef CONFIG_SMP
  4363. for_each_online_cpu(cpu) {
  4364. struct page *page;
  4365. page = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu));
  4366. if (page && len < PAGE_SIZE - 20)
  4367. len += sprintf(buf + len, " C%d=%d(%d)", cpu,
  4368. page->pobjects, page->pages);
  4369. }
  4370. #endif
  4371. return len + sprintf(buf + len, "\n");
  4372. }
  4373. SLAB_ATTR_RO(slabs_cpu_partial);
  4374. static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf)
  4375. {
  4376. return sprintf(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT));
  4377. }
  4378. SLAB_ATTR_RO(reclaim_account);
  4379. static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf)
  4380. {
  4381. return sprintf(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN));
  4382. }
  4383. SLAB_ATTR_RO(hwcache_align);
  4384. #ifdef CONFIG_ZONE_DMA
  4385. static ssize_t cache_dma_show(struct kmem_cache *s, char *buf)
  4386. {
  4387. return sprintf(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA));
  4388. }
  4389. SLAB_ATTR_RO(cache_dma);
  4390. #endif
  4391. static ssize_t usersize_show(struct kmem_cache *s, char *buf)
  4392. {
  4393. return sprintf(buf, "%u\n", s->usersize);
  4394. }
  4395. SLAB_ATTR_RO(usersize);
  4396. static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf)
  4397. {
  4398. return sprintf(buf, "%d\n", !!(s->flags & SLAB_TYPESAFE_BY_RCU));
  4399. }
  4400. SLAB_ATTR_RO(destroy_by_rcu);
  4401. #ifdef CONFIG_SLUB_DEBUG
  4402. static ssize_t slabs_show(struct kmem_cache *s, char *buf)
  4403. {
  4404. return show_slab_objects(s, buf, SO_ALL);
  4405. }
  4406. SLAB_ATTR_RO(slabs);
  4407. static ssize_t total_objects_show(struct kmem_cache *s, char *buf)
  4408. {
  4409. return show_slab_objects(s, buf, SO_ALL|SO_TOTAL);
  4410. }
  4411. SLAB_ATTR_RO(total_objects);
  4412. static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf)
  4413. {
  4414. return sprintf(buf, "%d\n", !!(s->flags & SLAB_CONSISTENCY_CHECKS));
  4415. }
  4416. SLAB_ATTR_RO(sanity_checks);
  4417. static ssize_t trace_show(struct kmem_cache *s, char *buf)
  4418. {
  4419. return sprintf(buf, "%d\n", !!(s->flags & SLAB_TRACE));
  4420. }
  4421. SLAB_ATTR_RO(trace);
  4422. static ssize_t red_zone_show(struct kmem_cache *s, char *buf)
  4423. {
  4424. return sprintf(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE));
  4425. }
  4426. SLAB_ATTR_RO(red_zone);
  4427. static ssize_t poison_show(struct kmem_cache *s, char *buf)
  4428. {
  4429. return sprintf(buf, "%d\n", !!(s->flags & SLAB_POISON));
  4430. }
  4431. SLAB_ATTR_RO(poison);
  4432. static ssize_t store_user_show(struct kmem_cache *s, char *buf)
  4433. {
  4434. return sprintf(buf, "%d\n", !!(s->flags & SLAB_STORE_USER));
  4435. }
  4436. SLAB_ATTR_RO(store_user);
  4437. static ssize_t validate_show(struct kmem_cache *s, char *buf)
  4438. {
  4439. return 0;
  4440. }
  4441. static ssize_t validate_store(struct kmem_cache *s,
  4442. const char *buf, size_t length)
  4443. {
  4444. int ret = -EINVAL;
  4445. if (buf[0] == '1') {
  4446. ret = validate_slab_cache(s);
  4447. if (ret >= 0)
  4448. ret = length;
  4449. }
  4450. return ret;
  4451. }
  4452. SLAB_ATTR(validate);
  4453. #endif /* CONFIG_SLUB_DEBUG */
  4454. #ifdef CONFIG_FAILSLAB
  4455. static ssize_t failslab_show(struct kmem_cache *s, char *buf)
  4456. {
  4457. return sprintf(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB));
  4458. }
  4459. SLAB_ATTR_RO(failslab);
  4460. #endif
  4461. static ssize_t shrink_show(struct kmem_cache *s, char *buf)
  4462. {
  4463. return 0;
  4464. }
  4465. static ssize_t shrink_store(struct kmem_cache *s,
  4466. const char *buf, size_t length)
  4467. {
  4468. if (buf[0] == '1')
  4469. kmem_cache_shrink(s);
  4470. else
  4471. return -EINVAL;
  4472. return length;
  4473. }
  4474. SLAB_ATTR(shrink);
  4475. #ifdef CONFIG_NUMA
  4476. static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf)
  4477. {
  4478. return sprintf(buf, "%u\n", s->remote_node_defrag_ratio / 10);
  4479. }
  4480. static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s,
  4481. const char *buf, size_t length)
  4482. {
  4483. unsigned int ratio;
  4484. int err;
  4485. err = kstrtouint(buf, 10, &ratio);
  4486. if (err)
  4487. return err;
  4488. if (ratio > 100)
  4489. return -ERANGE;
  4490. s->remote_node_defrag_ratio = ratio * 10;
  4491. return length;
  4492. }
  4493. SLAB_ATTR(remote_node_defrag_ratio);
  4494. #endif
  4495. #ifdef CONFIG_SLUB_STATS
  4496. static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si)
  4497. {
  4498. unsigned long sum = 0;
  4499. int cpu;
  4500. int len;
  4501. int *data = kmalloc_array(nr_cpu_ids, sizeof(int), GFP_KERNEL);
  4502. if (!data)
  4503. return -ENOMEM;
  4504. for_each_online_cpu(cpu) {
  4505. unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si];
  4506. data[cpu] = x;
  4507. sum += x;
  4508. }
  4509. len = sprintf(buf, "%lu", sum);
  4510. #ifdef CONFIG_SMP
  4511. for_each_online_cpu(cpu) {
  4512. if (data[cpu] && len < PAGE_SIZE - 20)
  4513. len += sprintf(buf + len, " C%d=%u", cpu, data[cpu]);
  4514. }
  4515. #endif
  4516. kfree(data);
  4517. return len + sprintf(buf + len, "\n");
  4518. }
  4519. static void clear_stat(struct kmem_cache *s, enum stat_item si)
  4520. {
  4521. int cpu;
  4522. for_each_online_cpu(cpu)
  4523. per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0;
  4524. }
  4525. #define STAT_ATTR(si, text) \
  4526. static ssize_t text##_show(struct kmem_cache *s, char *buf) \
  4527. { \
  4528. return show_stat(s, buf, si); \
  4529. } \
  4530. static ssize_t text##_store(struct kmem_cache *s, \
  4531. const char *buf, size_t length) \
  4532. { \
  4533. if (buf[0] != '0') \
  4534. return -EINVAL; \
  4535. clear_stat(s, si); \
  4536. return length; \
  4537. } \
  4538. SLAB_ATTR(text); \
  4539. STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath);
  4540. STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath);
  4541. STAT_ATTR(FREE_FASTPATH, free_fastpath);
  4542. STAT_ATTR(FREE_SLOWPATH, free_slowpath);
  4543. STAT_ATTR(FREE_FROZEN, free_frozen);
  4544. STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial);
  4545. STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial);
  4546. STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial);
  4547. STAT_ATTR(ALLOC_SLAB, alloc_slab);
  4548. STAT_ATTR(ALLOC_REFILL, alloc_refill);
  4549. STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch);
  4550. STAT_ATTR(FREE_SLAB, free_slab);
  4551. STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush);
  4552. STAT_ATTR(DEACTIVATE_FULL, deactivate_full);
  4553. STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty);
  4554. STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head);
  4555. STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail);
  4556. STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees);
  4557. STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass);
  4558. STAT_ATTR(ORDER_FALLBACK, order_fallback);
  4559. STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail);
  4560. STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail);
  4561. STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc);
  4562. STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free);
  4563. STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node);
  4564. STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain);
  4565. #endif /* CONFIG_SLUB_STATS */
  4566. static struct attribute *slab_attrs[] = {
  4567. &slab_size_attr.attr,
  4568. &object_size_attr.attr,
  4569. &objs_per_slab_attr.attr,
  4570. &order_attr.attr,
  4571. &min_partial_attr.attr,
  4572. &cpu_partial_attr.attr,
  4573. &objects_attr.attr,
  4574. &objects_partial_attr.attr,
  4575. &partial_attr.attr,
  4576. &cpu_slabs_attr.attr,
  4577. &ctor_attr.attr,
  4578. &aliases_attr.attr,
  4579. &align_attr.attr,
  4580. &hwcache_align_attr.attr,
  4581. &reclaim_account_attr.attr,
  4582. &destroy_by_rcu_attr.attr,
  4583. &shrink_attr.attr,
  4584. &slabs_cpu_partial_attr.attr,
  4585. #ifdef CONFIG_SLUB_DEBUG
  4586. &total_objects_attr.attr,
  4587. &slabs_attr.attr,
  4588. &sanity_checks_attr.attr,
  4589. &trace_attr.attr,
  4590. &red_zone_attr.attr,
  4591. &poison_attr.attr,
  4592. &store_user_attr.attr,
  4593. &validate_attr.attr,
  4594. #endif
  4595. #ifdef CONFIG_ZONE_DMA
  4596. &cache_dma_attr.attr,
  4597. #endif
  4598. #ifdef CONFIG_NUMA
  4599. &remote_node_defrag_ratio_attr.attr,
  4600. #endif
  4601. #ifdef CONFIG_SLUB_STATS
  4602. &alloc_fastpath_attr.attr,
  4603. &alloc_slowpath_attr.attr,
  4604. &free_fastpath_attr.attr,
  4605. &free_slowpath_attr.attr,
  4606. &free_frozen_attr.attr,
  4607. &free_add_partial_attr.attr,
  4608. &free_remove_partial_attr.attr,
  4609. &alloc_from_partial_attr.attr,
  4610. &alloc_slab_attr.attr,
  4611. &alloc_refill_attr.attr,
  4612. &alloc_node_mismatch_attr.attr,
  4613. &free_slab_attr.attr,
  4614. &cpuslab_flush_attr.attr,
  4615. &deactivate_full_attr.attr,
  4616. &deactivate_empty_attr.attr,
  4617. &deactivate_to_head_attr.attr,
  4618. &deactivate_to_tail_attr.attr,
  4619. &deactivate_remote_frees_attr.attr,
  4620. &deactivate_bypass_attr.attr,
  4621. &order_fallback_attr.attr,
  4622. &cmpxchg_double_fail_attr.attr,
  4623. &cmpxchg_double_cpu_fail_attr.attr,
  4624. &cpu_partial_alloc_attr.attr,
  4625. &cpu_partial_free_attr.attr,
  4626. &cpu_partial_node_attr.attr,
  4627. &cpu_partial_drain_attr.attr,
  4628. #endif
  4629. #ifdef CONFIG_FAILSLAB
  4630. &failslab_attr.attr,
  4631. #endif
  4632. &usersize_attr.attr,
  4633. NULL
  4634. };
  4635. static const struct attribute_group slab_attr_group = {
  4636. .attrs = slab_attrs,
  4637. };
  4638. static ssize_t slab_attr_show(struct kobject *kobj,
  4639. struct attribute *attr,
  4640. char *buf)
  4641. {
  4642. struct slab_attribute *attribute;
  4643. struct kmem_cache *s;
  4644. int err;
  4645. attribute = to_slab_attr(attr);
  4646. s = to_slab(kobj);
  4647. if (!attribute->show)
  4648. return -EIO;
  4649. err = attribute->show(s, buf);
  4650. return err;
  4651. }
  4652. static ssize_t slab_attr_store(struct kobject *kobj,
  4653. struct attribute *attr,
  4654. const char *buf, size_t len)
  4655. {
  4656. struct slab_attribute *attribute;
  4657. struct kmem_cache *s;
  4658. int err;
  4659. attribute = to_slab_attr(attr);
  4660. s = to_slab(kobj);
  4661. if (!attribute->store)
  4662. return -EIO;
  4663. err = attribute->store(s, buf, len);
  4664. return err;
  4665. }
  4666. static void kmem_cache_release(struct kobject *k)
  4667. {
  4668. slab_kmem_cache_release(to_slab(k));
  4669. }
  4670. static const struct sysfs_ops slab_sysfs_ops = {
  4671. .show = slab_attr_show,
  4672. .store = slab_attr_store,
  4673. };
  4674. static struct kobj_type slab_ktype = {
  4675. .sysfs_ops = &slab_sysfs_ops,
  4676. .release = kmem_cache_release,
  4677. };
  4678. static struct kset *slab_kset;
  4679. static inline struct kset *cache_kset(struct kmem_cache *s)
  4680. {
  4681. return slab_kset;
  4682. }
  4683. #define ID_STR_LENGTH 64
  4684. /* Create a unique string id for a slab cache:
  4685. *
  4686. * Format :[flags-]size
  4687. */
  4688. static char *create_unique_id(struct kmem_cache *s)
  4689. {
  4690. char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL);
  4691. char *p = name;
  4692. BUG_ON(!name);
  4693. *p++ = ':';
  4694. /*
  4695. * First flags affecting slabcache operations. We will only
  4696. * get here for aliasable slabs so we do not need to support
  4697. * too many flags. The flags here must cover all flags that
  4698. * are matched during merging to guarantee that the id is
  4699. * unique.
  4700. */
  4701. if (s->flags & SLAB_CACHE_DMA)
  4702. *p++ = 'd';
  4703. if (s->flags & SLAB_CACHE_DMA32)
  4704. *p++ = 'D';
  4705. if (s->flags & SLAB_RECLAIM_ACCOUNT)
  4706. *p++ = 'a';
  4707. if (s->flags & SLAB_CONSISTENCY_CHECKS)
  4708. *p++ = 'F';
  4709. if (s->flags & SLAB_ACCOUNT)
  4710. *p++ = 'A';
  4711. if (p != name + 1)
  4712. *p++ = '-';
  4713. p += sprintf(p, "%07u", s->size);
  4714. BUG_ON(p > name + ID_STR_LENGTH - 1);
  4715. return name;
  4716. }
  4717. static int sysfs_slab_add(struct kmem_cache *s)
  4718. {
  4719. int err;
  4720. const char *name;
  4721. struct kset *kset = cache_kset(s);
  4722. int unmergeable = slab_unmergeable(s);
  4723. if (!kset) {
  4724. kobject_init(&s->kobj, &slab_ktype);
  4725. return 0;
  4726. }
  4727. if (!unmergeable && disable_higher_order_debug &&
  4728. (slub_debug & DEBUG_METADATA_FLAGS))
  4729. unmergeable = 1;
  4730. if (unmergeable) {
  4731. /*
  4732. * Slabcache can never be merged so we can use the name proper.
  4733. * This is typically the case for debug situations. In that
  4734. * case we can catch duplicate names easily.
  4735. */
  4736. sysfs_remove_link(&slab_kset->kobj, s->name);
  4737. name = s->name;
  4738. } else {
  4739. /*
  4740. * Create a unique name for the slab as a target
  4741. * for the symlinks.
  4742. */
  4743. name = create_unique_id(s);
  4744. }
  4745. s->kobj.kset = kset;
  4746. err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name);
  4747. if (err)
  4748. goto out;
  4749. err = sysfs_create_group(&s->kobj, &slab_attr_group);
  4750. if (err)
  4751. goto out_del_kobj;
  4752. if (!unmergeable) {
  4753. /* Setup first alias */
  4754. sysfs_slab_alias(s, s->name);
  4755. }
  4756. out:
  4757. if (!unmergeable)
  4758. kfree(name);
  4759. return err;
  4760. out_del_kobj:
  4761. kobject_del(&s->kobj);
  4762. goto out;
  4763. }
  4764. void sysfs_slab_unlink(struct kmem_cache *s)
  4765. {
  4766. if (slab_state >= FULL)
  4767. kobject_del(&s->kobj);
  4768. }
  4769. void sysfs_slab_release(struct kmem_cache *s)
  4770. {
  4771. if (slab_state >= FULL)
  4772. kobject_put(&s->kobj);
  4773. }
  4774. /*
  4775. * Need to buffer aliases during bootup until sysfs becomes
  4776. * available lest we lose that information.
  4777. */
  4778. struct saved_alias {
  4779. struct kmem_cache *s;
  4780. const char *name;
  4781. struct saved_alias *next;
  4782. };
  4783. static struct saved_alias *alias_list;
  4784. static int sysfs_slab_alias(struct kmem_cache *s, const char *name)
  4785. {
  4786. struct saved_alias *al;
  4787. if (slab_state == FULL) {
  4788. /*
  4789. * If we have a leftover link then remove it.
  4790. */
  4791. sysfs_remove_link(&slab_kset->kobj, name);
  4792. return sysfs_create_link(&slab_kset->kobj, &s->kobj, name);
  4793. }
  4794. al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL);
  4795. if (!al)
  4796. return -ENOMEM;
  4797. al->s = s;
  4798. al->name = name;
  4799. al->next = alias_list;
  4800. alias_list = al;
  4801. return 0;
  4802. }
  4803. static int __init slab_sysfs_init(void)
  4804. {
  4805. struct kmem_cache *s;
  4806. int err;
  4807. mutex_lock(&slab_mutex);
  4808. slab_kset = kset_create_and_add("slab", NULL, kernel_kobj);
  4809. if (!slab_kset) {
  4810. mutex_unlock(&slab_mutex);
  4811. pr_err("Cannot register slab subsystem.\n");
  4812. return -ENOSYS;
  4813. }
  4814. slab_state = FULL;
  4815. list_for_each_entry(s, &slab_caches, list) {
  4816. err = sysfs_slab_add(s);
  4817. if (err)
  4818. pr_err("SLUB: Unable to add boot slab %s to sysfs\n",
  4819. s->name);
  4820. }
  4821. while (alias_list) {
  4822. struct saved_alias *al = alias_list;
  4823. alias_list = alias_list->next;
  4824. err = sysfs_slab_alias(al->s, al->name);
  4825. if (err)
  4826. pr_err("SLUB: Unable to add boot slab alias %s to sysfs\n",
  4827. al->name);
  4828. kfree(al);
  4829. }
  4830. mutex_unlock(&slab_mutex);
  4831. resiliency_test();
  4832. return 0;
  4833. }
  4834. __initcall(slab_sysfs_init);
  4835. #endif /* CONFIG_SYSFS */
  4836. #if defined(CONFIG_SLUB_DEBUG) && defined(CONFIG_DEBUG_FS)
  4837. static int slab_debugfs_show(struct seq_file *seq, void *v)
  4838. {
  4839. struct loc_track *t = seq->private;
  4840. struct location *l;
  4841. unsigned long idx;
  4842. idx = (unsigned long) t->idx;
  4843. if (idx < t->count) {
  4844. l = &t->loc[idx];
  4845. seq_printf(seq, "%7ld ", l->count);
  4846. if (l->addr)
  4847. seq_printf(seq, "%pS", (void *)l->addr);
  4848. else
  4849. seq_puts(seq, "<not-available>");
  4850. if (l->sum_time != l->min_time) {
  4851. seq_printf(seq, " age=%ld/%llu/%ld",
  4852. l->min_time, div_u64(l->sum_time, l->count),
  4853. l->max_time);
  4854. } else
  4855. seq_printf(seq, " age=%ld", l->min_time);
  4856. if (l->min_pid != l->max_pid)
  4857. seq_printf(seq, " pid=%ld-%ld", l->min_pid, l->max_pid);
  4858. else
  4859. seq_printf(seq, " pid=%ld",
  4860. l->min_pid);
  4861. if (num_online_cpus() > 1 && !cpumask_empty(to_cpumask(l->cpus)))
  4862. seq_printf(seq, " cpus=%*pbl",
  4863. cpumask_pr_args(to_cpumask(l->cpus)));
  4864. if (nr_online_nodes > 1 && !nodes_empty(l->nodes))
  4865. seq_printf(seq, " nodes=%*pbl",
  4866. nodemask_pr_args(&l->nodes));
  4867. seq_puts(seq, "\n");
  4868. }
  4869. if (!idx && !t->count)
  4870. seq_puts(seq, "No data\n");
  4871. return 0;
  4872. }
  4873. static void slab_debugfs_stop(struct seq_file *seq, void *v)
  4874. {
  4875. }
  4876. static void *slab_debugfs_next(struct seq_file *seq, void *v, loff_t *ppos)
  4877. {
  4878. struct loc_track *t = seq->private;
  4879. t->idx = ++(*ppos);
  4880. if (*ppos <= t->count)
  4881. return ppos;
  4882. return NULL;
  4883. }
  4884. static void *slab_debugfs_start(struct seq_file *seq, loff_t *ppos)
  4885. {
  4886. struct loc_track *t = seq->private;
  4887. t->idx = *ppos;
  4888. return ppos;
  4889. }
  4890. static const struct seq_operations slab_debugfs_sops = {
  4891. .start = slab_debugfs_start,
  4892. .next = slab_debugfs_next,
  4893. .stop = slab_debugfs_stop,
  4894. .show = slab_debugfs_show,
  4895. };
  4896. static int slab_debug_trace_open(struct inode *inode, struct file *filep)
  4897. {
  4898. struct kmem_cache_node *n;
  4899. enum track_item alloc;
  4900. int node;
  4901. struct loc_track *t = __seq_open_private(filep, &slab_debugfs_sops,
  4902. sizeof(struct loc_track));
  4903. struct kmem_cache *s = file_inode(filep)->i_private;
  4904. unsigned long *obj_map;
  4905. if (!t)
  4906. return -ENOMEM;
  4907. obj_map = bitmap_alloc(oo_objects(s->oo), GFP_KERNEL);
  4908. if (!obj_map) {
  4909. seq_release_private(inode, filep);
  4910. return -ENOMEM;
  4911. }
  4912. if (strcmp(filep->f_path.dentry->d_name.name, "alloc_traces") == 0)
  4913. alloc = TRACK_ALLOC;
  4914. else
  4915. alloc = TRACK_FREE;
  4916. if (!alloc_loc_track(t, PAGE_SIZE / sizeof(struct location), GFP_KERNEL)) {
  4917. bitmap_free(obj_map);
  4918. seq_release_private(inode, filep);
  4919. return -ENOMEM;
  4920. }
  4921. /* Push back cpu slabs */
  4922. flush_all(s);
  4923. for_each_kmem_cache_node(s, node, n) {
  4924. unsigned long flags;
  4925. struct page *page;
  4926. if (!atomic_long_read(&n->nr_slabs))
  4927. continue;
  4928. spin_lock_irqsave(&n->list_lock, flags);
  4929. list_for_each_entry(page, &n->partial, slab_list)
  4930. process_slab(t, s, page, alloc, obj_map);
  4931. list_for_each_entry(page, &n->full, slab_list)
  4932. process_slab(t, s, page, alloc, obj_map);
  4933. spin_unlock_irqrestore(&n->list_lock, flags);
  4934. }
  4935. bitmap_free(obj_map);
  4936. return 0;
  4937. }
  4938. static int slab_debug_trace_release(struct inode *inode, struct file *file)
  4939. {
  4940. struct seq_file *seq = file->private_data;
  4941. struct loc_track *t = seq->private;
  4942. free_loc_track(t);
  4943. return seq_release_private(inode, file);
  4944. }
  4945. static const struct file_operations slab_debugfs_fops = {
  4946. .open = slab_debug_trace_open,
  4947. .read = seq_read,
  4948. .llseek = seq_lseek,
  4949. .release = slab_debug_trace_release,
  4950. };
  4951. static void debugfs_slab_add(struct kmem_cache *s)
  4952. {
  4953. struct dentry *slab_cache_dir;
  4954. if (unlikely(!slab_debugfs_root))
  4955. return;
  4956. slab_cache_dir = debugfs_create_dir(s->name, slab_debugfs_root);
  4957. debugfs_create_file("alloc_traces", 0400,
  4958. slab_cache_dir, s, &slab_debugfs_fops);
  4959. debugfs_create_file("free_traces", 0400,
  4960. slab_cache_dir, s, &slab_debugfs_fops);
  4961. }
  4962. void debugfs_slab_release(struct kmem_cache *s)
  4963. {
  4964. debugfs_remove_recursive(debugfs_lookup(s->name, slab_debugfs_root));
  4965. }
  4966. static int __init slab_debugfs_init(void)
  4967. {
  4968. struct kmem_cache *s;
  4969. slab_debugfs_root = debugfs_create_dir("slab", NULL);
  4970. list_for_each_entry(s, &slab_caches, list)
  4971. if (s->flags & SLAB_STORE_USER)
  4972. debugfs_slab_add(s);
  4973. return 0;
  4974. }
  4975. __initcall(slab_debugfs_init);
  4976. #endif
  4977. /*
  4978. * The /proc/slabinfo ABI
  4979. */
  4980. #ifdef CONFIG_SLUB_DEBUG
  4981. void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo)
  4982. {
  4983. unsigned long nr_slabs = 0;
  4984. unsigned long nr_objs = 0;
  4985. unsigned long nr_free = 0;
  4986. int node;
  4987. struct kmem_cache_node *n;
  4988. for_each_kmem_cache_node(s, node, n) {
  4989. nr_slabs += node_nr_slabs(n);
  4990. nr_objs += node_nr_objs(n);
  4991. nr_free += count_partial(n, count_free);
  4992. }
  4993. sinfo->active_objs = nr_objs - nr_free;
  4994. sinfo->num_objs = nr_objs;
  4995. sinfo->active_slabs = nr_slabs;
  4996. sinfo->num_slabs = nr_slabs;
  4997. sinfo->objects_per_slab = oo_objects(s->oo);
  4998. sinfo->cache_order = oo_order(s->oo);
  4999. }
  5000. EXPORT_SYMBOL_GPL(get_slabinfo);
  5001. void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s)
  5002. {
  5003. }
  5004. ssize_t slabinfo_write(struct file *file, const char __user *buffer,
  5005. size_t count, loff_t *ppos)
  5006. {
  5007. return -EIO;
  5008. }
  5009. #endif /* CONFIG_SLUB_DEBUG */