partition_alloc_unittest.cc 196 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914
  1. // Copyright (c) 2013 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "base/allocator/partition_allocator/partition_alloc.h"
  5. #include <algorithm>
  6. #include <cstddef>
  7. #include <cstdint>
  8. #include <cstdlib>
  9. #include <cstring>
  10. #include <iostream>
  11. #include <limits>
  12. #include <memory>
  13. #include <random>
  14. #include <set>
  15. #include <tuple>
  16. #include <vector>
  17. #include "base/allocator/partition_allocator/address_space_randomization.h"
  18. #include "base/allocator/partition_allocator/chromecast_buildflags.h"
  19. #include "base/allocator/partition_allocator/dangling_raw_ptr_checks.h"
  20. #include "base/allocator/partition_allocator/page_allocator_constants.h"
  21. #include "base/allocator/partition_allocator/partition_address_space.h"
  22. #include "base/allocator/partition_allocator/partition_alloc_base/bits.h"
  23. #include "base/allocator/partition_allocator/partition_alloc_base/compiler_specific.h"
  24. #include "base/allocator/partition_allocator/partition_alloc_base/cpu.h"
  25. #include "base/allocator/partition_allocator/partition_alloc_base/debug/debugging_buildflags.h"
  26. #include "base/allocator/partition_allocator/partition_alloc_base/logging.h"
  27. #include "base/allocator/partition_allocator/partition_alloc_base/numerics/checked_math.h"
  28. #include "base/allocator/partition_allocator/partition_alloc_base/rand_util.h"
  29. #include "base/allocator/partition_allocator/partition_alloc_base/strings/stringprintf.h"
  30. #include "base/allocator/partition_allocator/partition_alloc_base/thread_annotations.h"
  31. #include "base/allocator/partition_allocator/partition_alloc_base/threading/platform_thread_for_testing.h"
  32. #include "base/allocator/partition_allocator/partition_alloc_buildflags.h"
  33. #include "base/allocator/partition_allocator/partition_alloc_config.h"
  34. #include "base/allocator/partition_allocator/partition_alloc_constants.h"
  35. #include "base/allocator/partition_allocator/partition_bucket.h"
  36. #include "base/allocator/partition_allocator/partition_cookie.h"
  37. #include "base/allocator/partition_allocator/partition_freelist_entry.h"
  38. #include "base/allocator/partition_allocator/partition_page.h"
  39. #include "base/allocator/partition_allocator/partition_ref_count.h"
  40. #include "base/allocator/partition_allocator/partition_root.h"
  41. #include "base/allocator/partition_allocator/partition_tag.h"
  42. #include "base/allocator/partition_allocator/partition_tag_bitmap.h"
  43. #include "base/allocator/partition_allocator/partition_tag_types.h"
  44. #include "base/allocator/partition_allocator/reservation_offset_table.h"
  45. #include "base/allocator/partition_allocator/tagging.h"
  46. #include "base/system/sys_info.h"
  47. #include "build/build_config.h"
  48. #include "testing/gtest/include/gtest/gtest.h"
  49. #if defined(__ARM_FEATURE_MEMORY_TAGGING)
  50. #include <arm_acle.h>
  51. #endif
  52. #if BUILDFLAG(IS_POSIX)
  53. #include <sys/mman.h>
  54. #include <sys/resource.h>
  55. #include <sys/time.h>
  56. #endif // BUILDFLAG(IS_POSIX)
  57. #if BUILDFLAG(USE_PARTITION_ALLOC_AS_MALLOC) && BUILDFLAG(IS_MAC)
  58. #include <OpenCL/opencl.h>
  59. #endif
  60. // In the MTE world, the upper bits of a pointer can be decorated with a tag,
  61. // thus allowing many versions of the same pointer to exist. These macros take
  62. // that into account when comparing.
  63. #define PA_EXPECT_PTR_EQ(ptr1, ptr2) \
  64. { EXPECT_EQ(UntagPtr(ptr1), UntagPtr(ptr2)); }
  65. #define PA_EXPECT_PTR_NE(ptr1, ptr2) \
  66. { EXPECT_NE(UntagPtr(ptr1), UntagPtr(ptr2)); }
  67. #if !defined(MEMORY_TOOL_REPLACES_ALLOCATOR)
  68. namespace {
  69. bool IsLargeMemoryDevice() {
  70. // Treat any device with 4GiB or more of physical memory as a "large memory
  71. // device". We check for slightly less than GiB so that devices with a small
  72. // amount of memory not accessible to the OS still count as "large".
  73. //
  74. // Set to 4GiB, since we have 2GiB Android devices where tests flakily fail
  75. // (e.g. Nexus 5X, crbug.com/1191195).
  76. return base::SysInfo::AmountOfPhysicalMemory() >= 4000ULL * 1024 * 1024;
  77. }
  78. bool SetAddressSpaceLimit() {
  79. #if !defined(ARCH_CPU_64_BITS) || !BUILDFLAG(IS_POSIX)
  80. // 32 bits => address space is limited already.
  81. return true;
  82. #elif BUILDFLAG(IS_POSIX) && !BUILDFLAG(IS_APPLE)
  83. // macOS will accept, but not enforce, |RLIMIT_AS| changes. See
  84. // https://crbug.com/435269 and rdar://17576114.
  85. //
  86. // Note: This number must be not less than 6 GB, because with
  87. // sanitizer_coverage_flags=edge, it reserves > 5 GB of address space. See
  88. // https://crbug.com/674665.
  89. const size_t kAddressSpaceLimit = static_cast<size_t>(6144) * 1024 * 1024;
  90. struct rlimit limit;
  91. if (getrlimit(RLIMIT_DATA, &limit) != 0)
  92. return false;
  93. if (limit.rlim_cur == RLIM_INFINITY || limit.rlim_cur > kAddressSpaceLimit) {
  94. limit.rlim_cur = kAddressSpaceLimit;
  95. if (setrlimit(RLIMIT_DATA, &limit) != 0)
  96. return false;
  97. }
  98. return true;
  99. #else
  100. return false;
  101. #endif
  102. }
  103. bool ClearAddressSpaceLimit() {
  104. #if !defined(ARCH_CPU_64_BITS) || !BUILDFLAG(IS_POSIX)
  105. return true;
  106. #elif BUILDFLAG(IS_POSIX)
  107. struct rlimit limit;
  108. if (getrlimit(RLIMIT_DATA, &limit) != 0)
  109. return false;
  110. limit.rlim_cur = limit.rlim_max;
  111. if (setrlimit(RLIMIT_DATA, &limit) != 0)
  112. return false;
  113. return true;
  114. #else
  115. return false;
  116. #endif
  117. }
  118. const size_t kTestSizes[] = {
  119. 1,
  120. 17,
  121. 100,
  122. partition_alloc::internal::SystemPageSize(),
  123. partition_alloc::internal::SystemPageSize() + 1,
  124. partition_alloc::PartitionRoot<
  125. partition_alloc::internal::ThreadSafe>::GetDirectMapSlotSize(100),
  126. 1 << 20,
  127. 1 << 21,
  128. };
  129. constexpr size_t kTestSizesCount = std::size(kTestSizes);
  130. void AllocateRandomly(
  131. partition_alloc::PartitionRoot<partition_alloc::internal::ThreadSafe>* root,
  132. size_t count,
  133. unsigned int flags) {
  134. std::vector<void*> allocations(count, nullptr);
  135. for (size_t i = 0; i < count; ++i) {
  136. const size_t size =
  137. kTestSizes[partition_alloc::internal::base::RandGenerator(
  138. kTestSizesCount)];
  139. allocations[i] = root->AllocWithFlags(flags, size, nullptr);
  140. EXPECT_NE(nullptr, allocations[i]) << " size: " << size << " i: " << i;
  141. }
  142. for (size_t i = 0; i < count; ++i) {
  143. if (allocations[i])
  144. root->Free(allocations[i]);
  145. }
  146. }
  147. void HandleOOM(size_t unused_size) {
  148. PA_LOG(FATAL) << "Out of memory";
  149. }
  150. int g_dangling_raw_ptr_detected_count = 0;
  151. int g_dangling_raw_ptr_released_count = 0;
  152. class CountDanglingRawPtr {
  153. public:
  154. CountDanglingRawPtr() {
  155. g_dangling_raw_ptr_detected_count = 0;
  156. g_dangling_raw_ptr_released_count = 0;
  157. old_detected_fn_ = partition_alloc::GetDanglingRawPtrDetectedFn();
  158. old_released_fn_ = partition_alloc::GetDanglingRawPtrReleasedFn();
  159. partition_alloc::SetDanglingRawPtrDetectedFn(
  160. CountDanglingRawPtr::DanglingRawPtrDetected);
  161. partition_alloc::SetDanglingRawPtrReleasedFn(
  162. CountDanglingRawPtr::DanglingRawPtrReleased);
  163. }
  164. ~CountDanglingRawPtr() {
  165. partition_alloc::SetDanglingRawPtrDetectedFn(old_detected_fn_);
  166. partition_alloc::SetDanglingRawPtrReleasedFn(old_released_fn_);
  167. }
  168. private:
  169. static void DanglingRawPtrDetected(uintptr_t) {
  170. g_dangling_raw_ptr_detected_count++;
  171. }
  172. static void DanglingRawPtrReleased(uintptr_t) {
  173. g_dangling_raw_ptr_released_count++;
  174. }
  175. partition_alloc::DanglingRawPtrDetectedFn* old_detected_fn_;
  176. partition_alloc::DanglingRawPtrReleasedFn* old_released_fn_;
  177. };
  178. } // namespace
  179. // Note: This test exercises interfaces inside the `partition_alloc`
  180. // namespace, but inspects objects inside `partition_alloc::internal`.
  181. // For ease of reading, the tests are placed into the latter namespace.
  182. namespace partition_alloc::internal {
  183. using SlotSpan = SlotSpanMetadata<ThreadSafe>;
  184. const size_t kTestAllocSize = 16;
  185. #if !BUILDFLAG(PA_DCHECK_IS_ON)
  186. const size_t kPointerOffset = kPartitionRefCountOffsetAdjustment;
  187. const size_t kExtraAllocSize = kInSlotRefCountBufferSize;
  188. #else
  189. const size_t kPointerOffset = kPartitionRefCountOffsetAdjustment;
  190. const size_t kExtraAllocSize = kCookieSize + kInSlotRefCountBufferSize;
  191. #endif
  192. const size_t kRealAllocSize = partition_alloc::internal::base::bits::AlignUp(
  193. kTestAllocSize + kExtraAllocSize,
  194. kAlignment);
  195. const char* type_name = nullptr;
  196. class ScopedPageAllocation {
  197. public:
  198. ScopedPageAllocation(PartitionAllocator<internal::ThreadSafe>& allocator,
  199. base::CheckedNumeric<size_t> npages)
  200. : allocator_(allocator),
  201. npages_(npages),
  202. ptr_(static_cast<char*>(allocator_.root()->Alloc(
  203. (npages * SystemPageSize() - kExtraAllocSize).ValueOrDie(),
  204. type_name))) {}
  205. ~ScopedPageAllocation() { allocator_.root()->Free(ptr_); }
  206. void TouchAllPages() {
  207. memset(ptr_, 'A',
  208. ((npages_ * SystemPageSize()) - kExtraAllocSize).ValueOrDie());
  209. }
  210. void* PageAtIndex(size_t index) {
  211. return ptr_ - kPointerOffset + (SystemPageSize() * index);
  212. }
  213. private:
  214. PartitionAllocator<internal::ThreadSafe>& allocator_;
  215. const base::CheckedNumeric<size_t> npages_;
  216. char* ptr_;
  217. };
  218. class PartitionAllocTest : public testing::TestWithParam<bool> {
  219. protected:
  220. PartitionAllocTest() = default;
  221. ~PartitionAllocTest() override = default;
  222. void SetUp() override {
  223. PartitionRoot<ThreadSafe>::EnableSortActiveSlotSpans();
  224. PartitionAllocGlobalInit(HandleOOM);
  225. allocator.init({
  226. #if !BUILDFLAG(USE_BACKUP_REF_PTR) || BUILDFLAG(PUT_REF_COUNT_IN_PREVIOUS_SLOT)
  227. // AlignedAllocWithFlags() can't be called when BRP is in the "before
  228. // allocation" mode, because this mode adds extras before the allocation.
  229. // Extras after the allocation are ok.
  230. PartitionOptions::AlignedAlloc::kAllowed,
  231. #else
  232. PartitionOptions::AlignedAlloc::kDisallowed,
  233. #endif
  234. PartitionOptions::ThreadCache::kDisabled,
  235. PartitionOptions::Quarantine::kDisallowed,
  236. PartitionOptions::Cookie::kAllowed,
  237. #if BUILDFLAG(USE_BACKUP_REF_PTR)
  238. PartitionOptions::BackupRefPtr::kEnabled,
  239. PartitionOptions::BackupRefPtrZapping::kEnabled,
  240. #else
  241. PartitionOptions::BackupRefPtr::kDisabled,
  242. PartitionOptions::BackupRefPtrZapping::kDisabled,
  243. #endif
  244. PartitionOptions::UseConfigurablePool::kNo,
  245. });
  246. aligned_allocator.init({
  247. PartitionOptions::AlignedAlloc::kAllowed,
  248. PartitionOptions::ThreadCache::kDisabled,
  249. PartitionOptions::Quarantine::kDisallowed,
  250. PartitionOptions::Cookie::kDisallowed,
  251. PartitionOptions::BackupRefPtr::kDisabled,
  252. PartitionOptions::BackupRefPtrZapping::kDisabled,
  253. PartitionOptions::UseConfigurablePool::kNo,
  254. });
  255. test_bucket_index_ = SizeToIndex(kRealAllocSize);
  256. allocator.root()->UncapEmptySlotSpanMemoryForTesting();
  257. aligned_allocator.root()->UncapEmptySlotSpanMemoryForTesting();
  258. if (GetParam())
  259. allocator.root()->SwitchToDenserBucketDistribution();
  260. else
  261. allocator.root()->ResetBucketDistributionForTesting();
  262. }
  263. size_t SizeToIndex(size_t size) {
  264. const bool with_denser_bucket_distribution = GetParam();
  265. return PartitionRoot<internal::ThreadSafe>::SizeToBucketIndex(
  266. size, with_denser_bucket_distribution);
  267. }
  268. void TearDown() override {
  269. allocator.root()->PurgeMemory(PurgeFlags::kDecommitEmptySlotSpans |
  270. PurgeFlags::kDiscardUnusedSystemPages);
  271. PartitionAllocGlobalUninitForTesting();
  272. }
  273. size_t GetNumPagesPerSlotSpan(size_t size) {
  274. size_t real_size = size + kExtraAllocSize;
  275. size_t bucket_index = SizeToIndex(real_size);
  276. PartitionRoot<ThreadSafe>::Bucket* bucket =
  277. &allocator.root()->buckets[bucket_index];
  278. // TODO(tasak): make get_pages_per_slot_span() available at
  279. // partition_alloc_unittest.cc. Is it allowable to make the code from
  280. // partition_bucet.cc to partition_bucket.h?
  281. return (bucket->num_system_pages_per_slot_span +
  282. (NumSystemPagesPerPartitionPage() - 1)) /
  283. NumSystemPagesPerPartitionPage();
  284. }
  285. SlotSpan* GetFullSlotSpan(size_t size) {
  286. size_t real_size = size + kExtraAllocSize;
  287. size_t bucket_index = SizeToIndex(real_size);
  288. PartitionRoot<ThreadSafe>::Bucket* bucket =
  289. &allocator.root()->buckets[bucket_index];
  290. size_t num_slots =
  291. (bucket->num_system_pages_per_slot_span * SystemPageSize()) /
  292. bucket->slot_size;
  293. uintptr_t first = 0;
  294. uintptr_t last = 0;
  295. size_t i;
  296. for (i = 0; i < num_slots; ++i) {
  297. void* ptr = allocator.root()->Alloc(size, type_name);
  298. EXPECT_TRUE(ptr);
  299. if (!i)
  300. first = allocator.root()->ObjectToSlotStart(ptr);
  301. else if (i == num_slots - 1)
  302. last = allocator.root()->ObjectToSlotStart(ptr);
  303. }
  304. EXPECT_EQ(SlotSpan::FromSlotStart(first), SlotSpan::FromSlotStart(last));
  305. if (bucket->num_system_pages_per_slot_span ==
  306. NumSystemPagesPerPartitionPage())
  307. EXPECT_EQ(first & PartitionPageBaseMask(),
  308. last & PartitionPageBaseMask());
  309. EXPECT_EQ(num_slots, bucket->active_slot_spans_head->num_allocated_slots);
  310. EXPECT_EQ(nullptr, bucket->active_slot_spans_head->get_freelist_head());
  311. EXPECT_TRUE(bucket->is_valid());
  312. EXPECT_TRUE(bucket->active_slot_spans_head !=
  313. SlotSpan::get_sentinel_slot_span());
  314. EXPECT_TRUE(bucket->active_slot_spans_head->is_full());
  315. return bucket->active_slot_spans_head;
  316. }
  317. void CycleFreeCache(size_t size) {
  318. for (size_t i = 0; i < kMaxFreeableSpans; ++i) {
  319. void* ptr = allocator.root()->Alloc(size, type_name);
  320. auto* slot_span =
  321. SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr));
  322. auto* bucket = slot_span->bucket;
  323. EXPECT_EQ(1u, bucket->active_slot_spans_head->num_allocated_slots);
  324. allocator.root()->Free(ptr);
  325. EXPECT_EQ(0u, bucket->active_slot_spans_head->num_allocated_slots);
  326. EXPECT_TRUE(bucket->active_slot_spans_head->in_empty_cache() ||
  327. bucket->active_slot_spans_head ==
  328. SlotSpanMetadata<ThreadSafe>::get_sentinel_slot_span());
  329. }
  330. }
  331. enum ReturnNullTestMode {
  332. kPartitionAllocWithFlags,
  333. kPartitionReallocWithFlags,
  334. kPartitionRootTryRealloc,
  335. };
  336. void DoReturnNullTest(size_t alloc_size, ReturnNullTestMode mode) {
  337. // TODO(crbug.com/678782): Where necessary and possible, disable the
  338. // platform's OOM-killing behavior. OOM-killing makes this test flaky on
  339. // low-memory devices.
  340. if (!IsLargeMemoryDevice()) {
  341. PA_LOG(WARNING)
  342. << "Skipping test on this device because of crbug.com/678782";
  343. PA_LOG(FATAL) << "Passed DoReturnNullTest";
  344. }
  345. ASSERT_TRUE(SetAddressSpaceLimit());
  346. // Work out the number of allocations for 6 GB of memory.
  347. const int num_allocations = (6 * 1024 * 1024) / (alloc_size / 1024);
  348. void** ptrs = static_cast<void**>(
  349. allocator.root()->Alloc(num_allocations * sizeof(void*), type_name));
  350. int i;
  351. for (i = 0; i < num_allocations; ++i) {
  352. switch (mode) {
  353. case kPartitionAllocWithFlags: {
  354. ptrs[i] = allocator.root()->AllocWithFlags(AllocFlags::kReturnNull,
  355. alloc_size, type_name);
  356. break;
  357. }
  358. case kPartitionReallocWithFlags: {
  359. ptrs[i] = allocator.root()->AllocWithFlags(AllocFlags::kReturnNull, 1,
  360. type_name);
  361. ptrs[i] = allocator.root()->ReallocWithFlags(
  362. AllocFlags::kReturnNull, ptrs[i], alloc_size, type_name);
  363. break;
  364. }
  365. case kPartitionRootTryRealloc: {
  366. ptrs[i] = allocator.root()->AllocWithFlags(AllocFlags::kReturnNull, 1,
  367. type_name);
  368. ptrs[i] =
  369. allocator.root()->TryRealloc(ptrs[i], alloc_size, type_name);
  370. }
  371. }
  372. if (!i)
  373. EXPECT_TRUE(ptrs[0]);
  374. if (!ptrs[i]) {
  375. ptrs[i] = allocator.root()->AllocWithFlags(AllocFlags::kReturnNull,
  376. alloc_size, type_name);
  377. EXPECT_FALSE(ptrs[i]);
  378. break;
  379. }
  380. }
  381. // We shouldn't succeed in allocating all 6 GB of memory. If we do, then
  382. // we're not actually testing anything here.
  383. EXPECT_LT(i, num_allocations);
  384. // Free, reallocate and free again each block we allocated. We do this to
  385. // check that freeing memory also works correctly after a failed allocation.
  386. for (--i; i >= 0; --i) {
  387. allocator.root()->Free(ptrs[i]);
  388. ptrs[i] = allocator.root()->AllocWithFlags(AllocFlags::kReturnNull,
  389. alloc_size, type_name);
  390. EXPECT_TRUE(ptrs[i]);
  391. allocator.root()->Free(ptrs[i]);
  392. }
  393. allocator.root()->Free(ptrs);
  394. EXPECT_TRUE(ClearAddressSpaceLimit());
  395. PA_LOG(FATAL) << "Passed DoReturnNullTest";
  396. }
  397. void RunRefCountReallocSubtest(size_t orig_size, size_t new_size);
  398. PA_NOINLINE PA_MALLOC_FN void* Alloc(size_t size) {
  399. return allocator.root()->Alloc(size, "");
  400. }
  401. PA_NOINLINE void Free(void* ptr) { allocator.root()->Free(ptr); }
  402. PartitionAllocator<internal::ThreadSafe> allocator;
  403. PartitionAllocator<internal::ThreadSafe> aligned_allocator;
  404. size_t test_bucket_index_;
  405. };
  406. class PartitionAllocDeathTest : public PartitionAllocTest {};
  407. namespace {
  408. void FreeFullSlotSpan(PartitionRoot<internal::ThreadSafe>* root,
  409. SlotSpan* slot_span) {
  410. EXPECT_TRUE(slot_span->is_full());
  411. size_t size = slot_span->bucket->slot_size;
  412. size_t num_slots =
  413. (slot_span->bucket->num_system_pages_per_slot_span * SystemPageSize()) /
  414. size;
  415. EXPECT_EQ(num_slots, slot_span->num_allocated_slots);
  416. uintptr_t address = SlotSpan::ToSlotSpanStart(slot_span);
  417. size_t i;
  418. for (i = 0; i < num_slots; ++i) {
  419. root->Free(root->SlotStartToObject(address));
  420. address += size;
  421. }
  422. EXPECT_TRUE(slot_span->is_empty());
  423. }
  424. #if BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)
  425. bool CheckPageInCore(void* ptr, bool in_core) {
  426. unsigned char ret = 0;
  427. EXPECT_EQ(0, mincore(ptr, SystemPageSize(), &ret));
  428. return in_core == (ret & 1);
  429. }
  430. #define CHECK_PAGE_IN_CORE(ptr, in_core) \
  431. EXPECT_TRUE(CheckPageInCore(ptr, in_core))
  432. #else
  433. #define CHECK_PAGE_IN_CORE(ptr, in_core) (void)(0)
  434. #endif // BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS)
  435. class MockPartitionStatsDumper : public PartitionStatsDumper {
  436. public:
  437. MockPartitionStatsDumper() = default;
  438. void PartitionDumpTotals(const char* partition_name,
  439. const PartitionMemoryStats* stats) override {
  440. EXPECT_GE(stats->total_mmapped_bytes, stats->total_resident_bytes);
  441. EXPECT_EQ(total_resident_bytes, stats->total_resident_bytes);
  442. EXPECT_EQ(total_active_bytes, stats->total_active_bytes);
  443. EXPECT_EQ(total_decommittable_bytes, stats->total_decommittable_bytes);
  444. EXPECT_EQ(total_discardable_bytes, stats->total_discardable_bytes);
  445. }
  446. void PartitionsDumpBucketStats(
  447. [[maybe_unused]] const char* partition_name,
  448. const PartitionBucketMemoryStats* stats) override {
  449. EXPECT_TRUE(stats->is_valid);
  450. EXPECT_EQ(0u, stats->bucket_slot_size & sizeof(void*));
  451. bucket_stats.push_back(*stats);
  452. total_resident_bytes += stats->resident_bytes;
  453. total_active_bytes += stats->active_bytes;
  454. total_decommittable_bytes += stats->decommittable_bytes;
  455. total_discardable_bytes += stats->discardable_bytes;
  456. }
  457. bool IsMemoryAllocationRecorded() {
  458. return total_resident_bytes != 0 && total_active_bytes != 0;
  459. }
  460. const PartitionBucketMemoryStats* GetBucketStats(size_t bucket_size) {
  461. for (auto& stat : bucket_stats) {
  462. if (stat.bucket_slot_size == bucket_size)
  463. return &stat;
  464. }
  465. return nullptr;
  466. }
  467. private:
  468. size_t total_resident_bytes = 0;
  469. size_t total_active_bytes = 0;
  470. size_t total_decommittable_bytes = 0;
  471. size_t total_discardable_bytes = 0;
  472. std::vector<PartitionBucketMemoryStats> bucket_stats;
  473. };
  474. } // namespace
  475. INSTANTIATE_TEST_SUITE_P(AlternateBucketDistribution,
  476. PartitionAllocTest,
  477. testing::Values(false, true));
  478. // Check that the most basic of allocate / free pairs work.
  479. TEST_P(PartitionAllocTest, Basic) {
  480. PartitionRoot<ThreadSafe>::Bucket* bucket =
  481. &allocator.root()->buckets[test_bucket_index_];
  482. auto* seed_slot_span = SlotSpan::get_sentinel_slot_span();
  483. EXPECT_FALSE(bucket->empty_slot_spans_head);
  484. EXPECT_FALSE(bucket->decommitted_slot_spans_head);
  485. EXPECT_EQ(seed_slot_span, bucket->active_slot_spans_head);
  486. EXPECT_EQ(nullptr, bucket->active_slot_spans_head->next_slot_span);
  487. void* ptr = allocator.root()->Alloc(kTestAllocSize, type_name);
  488. EXPECT_TRUE(ptr);
  489. EXPECT_EQ(kPointerOffset, UntagPtr(ptr) & PartitionPageOffsetMask());
  490. // Check that the offset appears to include a guard page.
  491. EXPECT_EQ(PartitionPageSize() +
  492. partition_alloc::internal::ReservedTagBitmapSize() +
  493. kPointerOffset,
  494. UntagPtr(ptr) & kSuperPageOffsetMask);
  495. allocator.root()->Free(ptr);
  496. // Expect that the last active slot span gets noticed as empty but doesn't get
  497. // decommitted.
  498. EXPECT_TRUE(bucket->empty_slot_spans_head);
  499. EXPECT_FALSE(bucket->decommitted_slot_spans_head);
  500. }
  501. // Test multiple allocations, and freelist handling.
  502. TEST_P(PartitionAllocTest, MultiAlloc) {
  503. void* ptr1 = allocator.root()->Alloc(kTestAllocSize, type_name);
  504. void* ptr2 = allocator.root()->Alloc(kTestAllocSize, type_name);
  505. EXPECT_TRUE(ptr1);
  506. EXPECT_TRUE(ptr2);
  507. ptrdiff_t diff = UntagPtr(ptr2) - UntagPtr(ptr1);
  508. EXPECT_EQ(static_cast<ptrdiff_t>(kRealAllocSize), diff);
  509. // Check that we re-use the just-freed slot.
  510. allocator.root()->Free(ptr2);
  511. ptr2 = allocator.root()->Alloc(kTestAllocSize, type_name);
  512. EXPECT_TRUE(ptr2);
  513. diff = UntagPtr(ptr2) - UntagPtr(ptr1);
  514. EXPECT_EQ(static_cast<ptrdiff_t>(kRealAllocSize), diff);
  515. allocator.root()->Free(ptr1);
  516. ptr1 = allocator.root()->Alloc(kTestAllocSize, type_name);
  517. EXPECT_TRUE(ptr1);
  518. diff = UntagPtr(ptr2) - UntagPtr(ptr1);
  519. EXPECT_EQ(static_cast<ptrdiff_t>(kRealAllocSize), diff);
  520. void* ptr3 = allocator.root()->Alloc(kTestAllocSize, type_name);
  521. EXPECT_TRUE(ptr3);
  522. diff = UntagPtr(ptr3) - UntagPtr(ptr1);
  523. EXPECT_EQ(static_cast<ptrdiff_t>(kRealAllocSize * 2), diff);
  524. allocator.root()->Free(ptr1);
  525. allocator.root()->Free(ptr2);
  526. allocator.root()->Free(ptr3);
  527. }
  528. // Test a bucket with multiple slot spans.
  529. TEST_P(PartitionAllocTest, MultiSlotSpans) {
  530. PartitionRoot<ThreadSafe>::Bucket* bucket =
  531. &allocator.root()->buckets[test_bucket_index_];
  532. auto* slot_span = GetFullSlotSpan(kTestAllocSize);
  533. FreeFullSlotSpan(allocator.root(), slot_span);
  534. EXPECT_TRUE(bucket->empty_slot_spans_head);
  535. EXPECT_EQ(SlotSpan::get_sentinel_slot_span(), bucket->active_slot_spans_head);
  536. EXPECT_EQ(nullptr, slot_span->next_slot_span);
  537. EXPECT_EQ(0u, slot_span->num_allocated_slots);
  538. slot_span = GetFullSlotSpan(kTestAllocSize);
  539. auto* slot_span2 = GetFullSlotSpan(kTestAllocSize);
  540. EXPECT_EQ(slot_span2, bucket->active_slot_spans_head);
  541. EXPECT_EQ(nullptr, slot_span2->next_slot_span);
  542. EXPECT_EQ(SlotSpan::ToSlotSpanStart(slot_span) & kSuperPageBaseMask,
  543. SlotSpan::ToSlotSpanStart(slot_span2) & kSuperPageBaseMask);
  544. // Fully free the non-current slot span. This will leave us with no current
  545. // active slot span because one is empty and the other is full.
  546. FreeFullSlotSpan(allocator.root(), slot_span);
  547. EXPECT_EQ(0u, slot_span->num_allocated_slots);
  548. EXPECT_TRUE(bucket->empty_slot_spans_head);
  549. EXPECT_EQ(SlotSpanMetadata<ThreadSafe>::get_sentinel_slot_span(),
  550. bucket->active_slot_spans_head);
  551. // Allocate a new slot span, it should pull from the freelist.
  552. slot_span = GetFullSlotSpan(kTestAllocSize);
  553. EXPECT_FALSE(bucket->empty_slot_spans_head);
  554. EXPECT_EQ(slot_span, bucket->active_slot_spans_head);
  555. FreeFullSlotSpan(allocator.root(), slot_span);
  556. FreeFullSlotSpan(allocator.root(), slot_span2);
  557. EXPECT_EQ(0u, slot_span->num_allocated_slots);
  558. EXPECT_EQ(0u, slot_span2->num_allocated_slots);
  559. EXPECT_EQ(0u, slot_span2->num_unprovisioned_slots);
  560. EXPECT_TRUE(slot_span2->in_empty_cache());
  561. }
  562. // Test some finer aspects of internal slot span transitions.
  563. TEST_P(PartitionAllocTest, SlotSpanTransitions) {
  564. PartitionRoot<ThreadSafe>::Bucket* bucket =
  565. &allocator.root()->buckets[test_bucket_index_];
  566. auto* slot_span1 = GetFullSlotSpan(kTestAllocSize);
  567. EXPECT_EQ(slot_span1, bucket->active_slot_spans_head);
  568. EXPECT_EQ(nullptr, slot_span1->next_slot_span);
  569. auto* slot_span2 = GetFullSlotSpan(kTestAllocSize);
  570. EXPECT_EQ(slot_span2, bucket->active_slot_spans_head);
  571. EXPECT_EQ(nullptr, slot_span2->next_slot_span);
  572. // Bounce slot_span1 back into the non-full list then fill it up again.
  573. void* ptr = allocator.root()->SlotStartToObject(
  574. SlotSpan::ToSlotSpanStart(slot_span1));
  575. allocator.root()->Free(ptr);
  576. EXPECT_EQ(slot_span1, bucket->active_slot_spans_head);
  577. std::ignore = allocator.root()->Alloc(kTestAllocSize, type_name);
  578. EXPECT_EQ(slot_span1, bucket->active_slot_spans_head);
  579. EXPECT_EQ(slot_span2, bucket->active_slot_spans_head->next_slot_span);
  580. // Allocating another slot span at this point should cause us to scan over
  581. // slot_span1 (which is both full and NOT our current slot span), and evict it
  582. // from the freelist. Older code had a O(n^2) condition due to failure to do
  583. // this.
  584. auto* slot_span3 = GetFullSlotSpan(kTestAllocSize);
  585. EXPECT_EQ(slot_span3, bucket->active_slot_spans_head);
  586. EXPECT_EQ(nullptr, slot_span3->next_slot_span);
  587. // Work out a pointer into slot_span2 and free it.
  588. ptr = allocator.root()->SlotStartToObject(
  589. SlotSpan::ToSlotSpanStart(slot_span2));
  590. allocator.root()->Free(ptr);
  591. // Trying to allocate at this time should cause us to cycle around to
  592. // slot_span2 and find the recently freed slot.
  593. void* ptr2 = allocator.root()->Alloc(kTestAllocSize, type_name);
  594. PA_EXPECT_PTR_EQ(ptr, ptr2);
  595. EXPECT_EQ(slot_span2, bucket->active_slot_spans_head);
  596. EXPECT_EQ(slot_span3, slot_span2->next_slot_span);
  597. // Work out a pointer into slot_span1 and free it. This should pull the slot
  598. // span back into the list of available slot spans.
  599. ptr = allocator.root()->SlotStartToObject(
  600. SlotSpan::ToSlotSpanStart(slot_span1));
  601. allocator.root()->Free(ptr);
  602. // This allocation should be satisfied by slot_span1.
  603. ptr2 = allocator.root()->Alloc(kTestAllocSize, type_name);
  604. PA_EXPECT_PTR_EQ(ptr, ptr2);
  605. EXPECT_EQ(slot_span1, bucket->active_slot_spans_head);
  606. EXPECT_EQ(slot_span2, slot_span1->next_slot_span);
  607. FreeFullSlotSpan(allocator.root(), slot_span3);
  608. FreeFullSlotSpan(allocator.root(), slot_span2);
  609. FreeFullSlotSpan(allocator.root(), slot_span1);
  610. // Allocating whilst in this state exposed a bug, so keep the test.
  611. ptr = allocator.root()->Alloc(kTestAllocSize, type_name);
  612. allocator.root()->Free(ptr);
  613. }
  614. TEST_P(PartitionAllocTest, PreferSlotSpansWithProvisionedEntries) {
  615. size_t size = SystemPageSize() - kExtraAllocSize;
  616. size_t real_size = size + kExtraAllocSize;
  617. size_t bucket_index =
  618. allocator.root()->SizeToBucketIndex(real_size, GetParam());
  619. PartitionRoot<ThreadSafe>::Bucket* bucket =
  620. &allocator.root()->buckets[bucket_index];
  621. ASSERT_EQ(bucket->slot_size, real_size);
  622. size_t slots_per_span = bucket->num_system_pages_per_slot_span;
  623. // Make 10 full slot spans.
  624. constexpr int kSpans = 10;
  625. std::vector<std::vector<void*>> allocated_memory_spans(kSpans);
  626. for (int span_index = 0; span_index < kSpans; span_index++) {
  627. for (size_t i = 0; i < slots_per_span; i++) {
  628. allocated_memory_spans[span_index].push_back(
  629. allocator.root()->Alloc(size, ""));
  630. }
  631. }
  632. // Reverse ordering, since a newly non-full span is placed at the head of the
  633. // active list.
  634. for (int span_index = kSpans - 1; span_index >= 0; span_index--) {
  635. allocator.root()->Free(allocated_memory_spans[span_index].back());
  636. allocated_memory_spans[span_index].pop_back();
  637. }
  638. // Since slot spans are large enough and we freed memory from the end, the
  639. // slot spans become partially provisioned after PurgeMemory().
  640. allocator.root()->PurgeMemory(PurgeFlags::kDecommitEmptySlotSpans |
  641. PurgeFlags::kDiscardUnusedSystemPages);
  642. std::vector<SlotSpanMetadata<ThreadSafe>*> active_slot_spans;
  643. for (auto* span = bucket->active_slot_spans_head; span;
  644. span = span->next_slot_span) {
  645. active_slot_spans.push_back(span);
  646. ASSERT_EQ(span->num_unprovisioned_slots, 1u);
  647. // But no freelist entries.
  648. ASSERT_FALSE(span->get_freelist_head());
  649. }
  650. // Free one entry in the middle span, creating a freelist entry.
  651. constexpr size_t kSpanIndex = 5;
  652. allocator.root()->Free(allocated_memory_spans[kSpanIndex].back());
  653. allocated_memory_spans[kSpanIndex].pop_back();
  654. ASSERT_TRUE(active_slot_spans[kSpanIndex]->get_freelist_head());
  655. ASSERT_FALSE(bucket->active_slot_spans_head->get_freelist_head());
  656. // It must come from the middle slot span even though the first one has
  657. // unprovisioned space.
  658. void* new_ptr = allocator.root()->Alloc(size, "");
  659. // Comes from the middle slot span, since it has a freelist entry.
  660. auto* new_active_slot_span = active_slot_spans[kSpanIndex];
  661. ASSERT_FALSE(new_active_slot_span->get_freelist_head());
  662. // The middle slot span was moved to the front.
  663. active_slot_spans.erase(active_slot_spans.begin() + kSpanIndex);
  664. active_slot_spans.insert(active_slot_spans.begin(), new_active_slot_span);
  665. // Check slot span ordering.
  666. int index = 0;
  667. for (auto* span = bucket->active_slot_spans_head; span;
  668. span = span->next_slot_span) {
  669. EXPECT_EQ(span, active_slot_spans[index]);
  670. index++;
  671. }
  672. EXPECT_EQ(index, kSpans);
  673. allocator.root()->Free(new_ptr);
  674. for (int span_index = 0; span_index < kSpans; span_index++) {
  675. for (void* ptr : allocated_memory_spans[span_index]) {
  676. allocator.root()->Free(ptr);
  677. }
  678. }
  679. }
  680. // Test some corner cases relating to slot span transitions in the internal
  681. // free slot span list metadata bucket.
  682. TEST_P(PartitionAllocTest, FreeSlotSpanListSlotSpanTransitions) {
  683. PartitionRoot<ThreadSafe>::Bucket* bucket =
  684. &allocator.root()->buckets[test_bucket_index_];
  685. size_t num_to_fill_free_list_slot_span =
  686. PartitionPageSize() / (sizeof(SlotSpan) + kExtraAllocSize);
  687. // The +1 is because we need to account for the fact that the current slot
  688. // span never gets thrown on the freelist.
  689. ++num_to_fill_free_list_slot_span;
  690. auto slot_spans =
  691. std::make_unique<SlotSpan*[]>(num_to_fill_free_list_slot_span);
  692. size_t i;
  693. for (i = 0; i < num_to_fill_free_list_slot_span; ++i) {
  694. slot_spans[i] = GetFullSlotSpan(kTestAllocSize);
  695. }
  696. EXPECT_EQ(slot_spans[num_to_fill_free_list_slot_span - 1],
  697. bucket->active_slot_spans_head);
  698. for (i = 0; i < num_to_fill_free_list_slot_span; ++i)
  699. FreeFullSlotSpan(allocator.root(), slot_spans[i]);
  700. EXPECT_EQ(SlotSpan::get_sentinel_slot_span(), bucket->active_slot_spans_head);
  701. EXPECT_TRUE(bucket->empty_slot_spans_head);
  702. // Allocate / free in a different bucket size so we get control of a
  703. // different free slot span list. We need two slot spans because one will be
  704. // the last active slot span and not get freed.
  705. auto* slot_span1 = GetFullSlotSpan(kTestAllocSize * 2);
  706. auto* slot_span2 = GetFullSlotSpan(kTestAllocSize * 2);
  707. FreeFullSlotSpan(allocator.root(), slot_span1);
  708. FreeFullSlotSpan(allocator.root(), slot_span2);
  709. for (i = 0; i < num_to_fill_free_list_slot_span; ++i) {
  710. slot_spans[i] = GetFullSlotSpan(kTestAllocSize);
  711. }
  712. EXPECT_EQ(slot_spans[num_to_fill_free_list_slot_span - 1],
  713. bucket->active_slot_spans_head);
  714. for (i = 0; i < num_to_fill_free_list_slot_span; ++i)
  715. FreeFullSlotSpan(allocator.root(), slot_spans[i]);
  716. EXPECT_EQ(SlotSpan::get_sentinel_slot_span(), bucket->active_slot_spans_head);
  717. EXPECT_TRUE(bucket->empty_slot_spans_head);
  718. }
  719. // Test a large series of allocations that cross more than one underlying
  720. // super page.
  721. TEST_P(PartitionAllocTest, MultiPageAllocs) {
  722. size_t num_pages_per_slot_span = GetNumPagesPerSlotSpan(kTestAllocSize);
  723. // 1 super page has 2 guard partition pages and a tag bitmap.
  724. size_t num_slot_spans_needed =
  725. (NumPartitionPagesPerSuperPage() - 2 -
  726. partition_alloc::internal::NumPartitionPagesPerTagBitmap()) /
  727. num_pages_per_slot_span;
  728. // We need one more slot span in order to cross super page boundary.
  729. ++num_slot_spans_needed;
  730. EXPECT_GT(num_slot_spans_needed, 1u);
  731. auto slot_spans = std::make_unique<SlotSpan*[]>(num_slot_spans_needed);
  732. uintptr_t first_super_page_base = 0;
  733. size_t i;
  734. for (i = 0; i < num_slot_spans_needed; ++i) {
  735. slot_spans[i] = GetFullSlotSpan(kTestAllocSize);
  736. uintptr_t slot_span_start = SlotSpan::ToSlotSpanStart(slot_spans[i]);
  737. if (!i)
  738. first_super_page_base = slot_span_start & kSuperPageBaseMask;
  739. if (i == num_slot_spans_needed - 1) {
  740. uintptr_t second_super_page_base = slot_span_start & kSuperPageBaseMask;
  741. uintptr_t second_super_page_offset =
  742. slot_span_start & kSuperPageOffsetMask;
  743. EXPECT_FALSE(second_super_page_base == first_super_page_base);
  744. // Check that we allocated a guard page and the reserved tag bitmap for
  745. // the second page.
  746. EXPECT_EQ(PartitionPageSize() +
  747. partition_alloc::internal::ReservedTagBitmapSize(),
  748. second_super_page_offset);
  749. }
  750. }
  751. for (i = 0; i < num_slot_spans_needed; ++i)
  752. FreeFullSlotSpan(allocator.root(), slot_spans[i]);
  753. }
  754. // Test the generic allocation functions that can handle arbitrary sizes and
  755. // reallocing etc.
  756. TEST_P(PartitionAllocTest, Alloc) {
  757. void* ptr = allocator.root()->Alloc(1, type_name);
  758. EXPECT_TRUE(ptr);
  759. allocator.root()->Free(ptr);
  760. ptr = allocator.root()->Alloc(kMaxBucketed + 1, type_name);
  761. EXPECT_TRUE(ptr);
  762. allocator.root()->Free(ptr);
  763. // To make both alloc(x + 1) and alloc(x + kSmallestBucket) to allocate from
  764. // the same bucket, partition_alloc::internal::base::bits::AlignUp(1 + x +
  765. // kExtraAllocSize, kAlignment)
  766. // == partition_alloc::internal::base::bits::AlignUp(kSmallestBucket + x +
  767. // kExtraAllocSize, kAlignment), because slot_size is multiples of
  768. // kAlignment. So (x + kExtraAllocSize) must be multiples of
  769. // kAlignment. x =
  770. // partition_alloc::internal::base::bits::AlignUp(kExtraAllocSize,
  771. // kAlignment) - kExtraAllocSize;
  772. size_t base_size = partition_alloc::internal::base::bits::AlignUp(
  773. kExtraAllocSize, kAlignment) -
  774. kExtraAllocSize;
  775. ptr = allocator.root()->Alloc(base_size + 1, type_name);
  776. EXPECT_TRUE(ptr);
  777. void* orig_ptr = ptr;
  778. char* char_ptr = static_cast<char*>(ptr);
  779. *char_ptr = 'A';
  780. // Change the size of the realloc, remaining inside the same bucket.
  781. void* new_ptr = allocator.root()->Realloc(ptr, base_size + 2, type_name);
  782. PA_EXPECT_PTR_EQ(ptr, new_ptr);
  783. new_ptr = allocator.root()->Realloc(ptr, base_size + 1, type_name);
  784. PA_EXPECT_PTR_EQ(ptr, new_ptr);
  785. new_ptr =
  786. allocator.root()->Realloc(ptr, base_size + kSmallestBucket, type_name);
  787. PA_EXPECT_PTR_EQ(ptr, new_ptr);
  788. // Change the size of the realloc, switching buckets.
  789. new_ptr = allocator.root()->Realloc(ptr, base_size + kSmallestBucket + 1,
  790. type_name);
  791. PA_EXPECT_PTR_NE(new_ptr, ptr);
  792. // Check that the realloc copied correctly.
  793. char* new_char_ptr = static_cast<char*>(new_ptr);
  794. EXPECT_EQ(*new_char_ptr, 'A');
  795. #if BUILDFLAG(PA_EXPENSIVE_DCHECKS_ARE_ON)
  796. // Subtle: this checks for an old bug where we copied too much from the
  797. // source of the realloc. The condition can be detected by a trashing of
  798. // the uninitialized value in the space of the upsized allocation.
  799. EXPECT_EQ(kUninitializedByte,
  800. static_cast<unsigned char>(*(new_char_ptr + kSmallestBucket)));
  801. #endif
  802. *new_char_ptr = 'B';
  803. // The realloc moved. To check that the old allocation was freed, we can
  804. // do an alloc of the old allocation size and check that the old allocation
  805. // address is at the head of the freelist and reused.
  806. void* reused_ptr = allocator.root()->Alloc(base_size + 1, type_name);
  807. PA_EXPECT_PTR_EQ(reused_ptr, orig_ptr);
  808. allocator.root()->Free(reused_ptr);
  809. // Downsize the realloc.
  810. ptr = new_ptr;
  811. new_ptr = allocator.root()->Realloc(ptr, base_size + 1, type_name);
  812. PA_EXPECT_PTR_EQ(new_ptr, orig_ptr);
  813. new_char_ptr = static_cast<char*>(new_ptr);
  814. EXPECT_EQ(*new_char_ptr, 'B');
  815. *new_char_ptr = 'C';
  816. // Upsize the realloc to outside the partition.
  817. ptr = new_ptr;
  818. new_ptr = allocator.root()->Realloc(ptr, kMaxBucketed + 1, type_name);
  819. PA_EXPECT_PTR_NE(new_ptr, ptr);
  820. new_char_ptr = static_cast<char*>(new_ptr);
  821. EXPECT_EQ(*new_char_ptr, 'C');
  822. *new_char_ptr = 'D';
  823. // Upsize and downsize the realloc, remaining outside the partition.
  824. ptr = new_ptr;
  825. new_ptr = allocator.root()->Realloc(ptr, kMaxBucketed * 10, type_name);
  826. new_char_ptr = static_cast<char*>(new_ptr);
  827. EXPECT_EQ(*new_char_ptr, 'D');
  828. *new_char_ptr = 'E';
  829. ptr = new_ptr;
  830. new_ptr = allocator.root()->Realloc(ptr, kMaxBucketed * 2, type_name);
  831. new_char_ptr = static_cast<char*>(new_ptr);
  832. EXPECT_EQ(*new_char_ptr, 'E');
  833. *new_char_ptr = 'F';
  834. // Downsize the realloc to inside the partition.
  835. ptr = new_ptr;
  836. new_ptr = allocator.root()->Realloc(ptr, base_size + 1, type_name);
  837. PA_EXPECT_PTR_NE(new_ptr, ptr);
  838. PA_EXPECT_PTR_EQ(new_ptr, orig_ptr);
  839. new_char_ptr = static_cast<char*>(new_ptr);
  840. EXPECT_EQ(*new_char_ptr, 'F');
  841. allocator.root()->Free(new_ptr);
  842. }
  843. // Test the generic allocation functions can handle some specific sizes of
  844. // interest.
  845. TEST_P(PartitionAllocTest, AllocSizes) {
  846. {
  847. void* ptr = allocator.root()->Alloc(0, type_name);
  848. EXPECT_TRUE(ptr);
  849. allocator.root()->Free(ptr);
  850. }
  851. {
  852. // PartitionPageSize() is interesting because it results in just one
  853. // allocation per page, which tripped up some corner cases.
  854. const size_t size = PartitionPageSize() - kExtraAllocSize;
  855. void* ptr = allocator.root()->Alloc(size, type_name);
  856. EXPECT_TRUE(ptr);
  857. void* ptr2 = allocator.root()->Alloc(size, type_name);
  858. EXPECT_TRUE(ptr2);
  859. allocator.root()->Free(ptr);
  860. // Should be freeable at this point.
  861. auto* slot_span =
  862. SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr));
  863. EXPECT_TRUE(slot_span->in_empty_cache());
  864. allocator.root()->Free(ptr2);
  865. }
  866. {
  867. const size_t size =
  868. (((PartitionPageSize() * kMaxPartitionPagesPerRegularSlotSpan) -
  869. SystemPageSize()) /
  870. 2) -
  871. kExtraAllocSize;
  872. void* ptr = allocator.root()->Alloc(size, type_name);
  873. EXPECT_TRUE(ptr);
  874. memset(ptr, 'A', size);
  875. void* ptr2 = allocator.root()->Alloc(size, type_name);
  876. EXPECT_TRUE(ptr2);
  877. void* ptr3 = allocator.root()->Alloc(size, type_name);
  878. EXPECT_TRUE(ptr3);
  879. void* ptr4 = allocator.root()->Alloc(size, type_name);
  880. EXPECT_TRUE(ptr4);
  881. auto* slot_span = SlotSpanMetadata<internal::ThreadSafe>::FromSlotStart(
  882. allocator.root()->ObjectToSlotStart(ptr));
  883. auto* slot_span2 =
  884. SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr3));
  885. EXPECT_NE(slot_span, slot_span2);
  886. allocator.root()->Free(ptr);
  887. allocator.root()->Free(ptr3);
  888. allocator.root()->Free(ptr2);
  889. // Should be freeable at this point.
  890. EXPECT_TRUE(slot_span->in_empty_cache());
  891. EXPECT_EQ(0u, slot_span->num_allocated_slots);
  892. EXPECT_EQ(0u, slot_span->num_unprovisioned_slots);
  893. void* new_ptr_1 = allocator.root()->Alloc(size, type_name);
  894. PA_EXPECT_PTR_EQ(ptr2, new_ptr_1);
  895. void* new_ptr_2 = allocator.root()->Alloc(size, type_name);
  896. PA_EXPECT_PTR_EQ(ptr3, new_ptr_2);
  897. allocator.root()->Free(new_ptr_1);
  898. allocator.root()->Free(new_ptr_2);
  899. allocator.root()->Free(ptr4);
  900. #if BUILDFLAG(PA_EXPENSIVE_DCHECKS_ARE_ON)
  901. // |SlotSpanMetadata::Free| must poison the slot's contents with
  902. // |kFreedByte|.
  903. EXPECT_EQ(kFreedByte,
  904. *(static_cast<unsigned char*>(new_ptr_1) + (size - 1)));
  905. #endif
  906. }
  907. // Can we allocate a massive (128MB) size?
  908. // Add +1, to test for cookie writing alignment issues.
  909. // Test this only if the device has enough memory or it might fail due
  910. // to OOM.
  911. if (IsLargeMemoryDevice()) {
  912. void* ptr = allocator.root()->Alloc(128 * 1024 * 1024 + 1, type_name);
  913. allocator.root()->Free(ptr);
  914. }
  915. {
  916. // Check a more reasonable, but still direct mapped, size.
  917. // Chop a system page and a byte off to test for rounding errors.
  918. size_t size = 20 * 1024 * 1024;
  919. ASSERT_GT(size, kMaxBucketed);
  920. size -= SystemPageSize();
  921. size -= 1;
  922. void* ptr = allocator.root()->Alloc(size, type_name);
  923. char* char_ptr = static_cast<char*>(ptr);
  924. *(char_ptr + (size - 1)) = 'A';
  925. allocator.root()->Free(ptr);
  926. // Can we free null?
  927. allocator.root()->Free(nullptr);
  928. // Do we correctly get a null for a failed allocation?
  929. EXPECT_EQ(nullptr,
  930. allocator.root()->AllocWithFlags(
  931. AllocFlags::kReturnNull, 3u * 1024 * 1024 * 1024, type_name));
  932. }
  933. }
  934. // Test that we can fetch the real allocated size after an allocation.
  935. TEST_P(PartitionAllocTest, AllocGetSizeAndStart) {
  936. void* ptr;
  937. size_t requested_size, actual_capacity, predicted_capacity;
  938. // Allocate something small.
  939. requested_size = 511 - kExtraAllocSize;
  940. predicted_capacity =
  941. allocator.root()->AllocationCapacityFromRequestedSize(requested_size);
  942. ptr = allocator.root()->Alloc(requested_size, type_name);
  943. EXPECT_TRUE(ptr);
  944. uintptr_t slot_start = allocator.root()->ObjectToSlotStart(ptr);
  945. actual_capacity =
  946. allocator.root()->AllocationCapacityFromSlotStart(slot_start);
  947. EXPECT_EQ(predicted_capacity, actual_capacity);
  948. EXPECT_LT(requested_size, actual_capacity);
  949. #if BUILDFLAG(USE_BACKUP_REF_PTR)
  950. uintptr_t address = UntagPtr(ptr);
  951. for (size_t offset = 0; offset < requested_size; ++offset) {
  952. EXPECT_EQ(PartitionAllocGetSlotStartInBRPPool(address + offset),
  953. slot_start);
  954. }
  955. #endif // BUILDFLAG(USE_BACKUP_REF_PTR)
  956. allocator.root()->Free(ptr);
  957. // Allocate a size that should be a perfect match for a bucket, because it
  958. // is an exact power of 2.
  959. requested_size = (256 * 1024) - kExtraAllocSize;
  960. predicted_capacity =
  961. allocator.root()->AllocationCapacityFromRequestedSize(requested_size);
  962. ptr = allocator.root()->Alloc(requested_size, type_name);
  963. EXPECT_TRUE(ptr);
  964. slot_start = allocator.root()->ObjectToSlotStart(ptr);
  965. actual_capacity =
  966. allocator.root()->AllocationCapacityFromSlotStart(slot_start);
  967. EXPECT_EQ(predicted_capacity, actual_capacity);
  968. EXPECT_EQ(requested_size, actual_capacity);
  969. #if BUILDFLAG(USE_BACKUP_REF_PTR)
  970. address = UntagPtr(ptr);
  971. for (size_t offset = 0; offset < requested_size; offset += 877) {
  972. EXPECT_EQ(PartitionAllocGetSlotStartInBRPPool(address + offset),
  973. slot_start);
  974. }
  975. #endif // BUILDFLAG(USE_BACKUP_REF_PTR)
  976. allocator.root()->Free(ptr);
  977. // Allocate a size that is a system page smaller than a bucket.
  978. // AllocationCapacityFromSlotStart() should return a larger size than we asked
  979. // for now.
  980. size_t num = 64;
  981. while (num * SystemPageSize() >= 1024 * 1024) {
  982. num /= 2;
  983. }
  984. requested_size = num * SystemPageSize() - SystemPageSize() - kExtraAllocSize;
  985. predicted_capacity =
  986. allocator.root()->AllocationCapacityFromRequestedSize(requested_size);
  987. ptr = allocator.root()->Alloc(requested_size, type_name);
  988. EXPECT_TRUE(ptr);
  989. slot_start = allocator.root()->ObjectToSlotStart(ptr);
  990. actual_capacity =
  991. allocator.root()->AllocationCapacityFromSlotStart(slot_start);
  992. EXPECT_EQ(predicted_capacity, actual_capacity);
  993. EXPECT_EQ(requested_size + SystemPageSize(), actual_capacity);
  994. #if BUILDFLAG(USE_BACKUP_REF_PTR)
  995. address = UntagPtr(ptr);
  996. for (size_t offset = 0; offset < requested_size; offset += 4999) {
  997. EXPECT_EQ(PartitionAllocGetSlotStartInBRPPool(address + offset),
  998. slot_start);
  999. }
  1000. #endif // BUILDFLAG(USE_BACKUP_REF_PTR)
  1001. // Allocate the maximum allowed bucketed size.
  1002. requested_size = kMaxBucketed - kExtraAllocSize;
  1003. predicted_capacity =
  1004. allocator.root()->AllocationCapacityFromRequestedSize(requested_size);
  1005. ptr = allocator.root()->Alloc(requested_size, type_name);
  1006. EXPECT_TRUE(ptr);
  1007. slot_start = allocator.root()->ObjectToSlotStart(ptr);
  1008. actual_capacity =
  1009. allocator.root()->AllocationCapacityFromSlotStart(slot_start);
  1010. EXPECT_EQ(predicted_capacity, actual_capacity);
  1011. EXPECT_EQ(requested_size, actual_capacity);
  1012. #if BUILDFLAG(USE_BACKUP_REF_PTR)
  1013. address = UntagPtr(ptr);
  1014. for (size_t offset = 0; offset < requested_size; offset += 4999) {
  1015. EXPECT_EQ(PartitionAllocGetSlotStartInBRPPool(address + offset),
  1016. slot_start);
  1017. }
  1018. #endif // BUILDFLAG(USE_BACKUP_REF_PTR)
  1019. // Check that we can write at the end of the reported size too.
  1020. char* char_ptr = static_cast<char*>(ptr);
  1021. *(char_ptr + (actual_capacity - 1)) = 'A';
  1022. allocator.root()->Free(ptr);
  1023. // Allocate something very large, and uneven.
  1024. if (IsLargeMemoryDevice()) {
  1025. requested_size = 128 * 1024 * 1024 - 1;
  1026. predicted_capacity =
  1027. allocator.root()->AllocationCapacityFromRequestedSize(requested_size);
  1028. ptr = allocator.root()->Alloc(requested_size, type_name);
  1029. EXPECT_TRUE(ptr);
  1030. slot_start = allocator.root()->ObjectToSlotStart(ptr);
  1031. actual_capacity =
  1032. allocator.root()->AllocationCapacityFromSlotStart(slot_start);
  1033. EXPECT_EQ(predicted_capacity, actual_capacity);
  1034. EXPECT_LT(requested_size, actual_capacity);
  1035. #if BUILDFLAG(USE_BACKUP_REF_PTR)
  1036. address = UntagPtr(ptr);
  1037. for (size_t offset = 0; offset < requested_size; offset += 16111) {
  1038. EXPECT_EQ(PartitionAllocGetSlotStartInBRPPool(address + offset),
  1039. slot_start);
  1040. }
  1041. #endif // BUILDFLAG(USE_BACKUP_REF_PTR)
  1042. allocator.root()->Free(ptr);
  1043. }
  1044. // Too large allocation.
  1045. requested_size = MaxDirectMapped() + 1;
  1046. predicted_capacity =
  1047. allocator.root()->AllocationCapacityFromRequestedSize(requested_size);
  1048. EXPECT_EQ(requested_size, predicted_capacity);
  1049. }
  1050. #if BUILDFLAG(USE_BACKUP_REF_PTR)
  1051. TEST_P(PartitionAllocTest, IsValidPtrDelta) {
  1052. const size_t kMinReasonableTestSize =
  1053. partition_alloc::internal::base::bits::AlignUp(kExtraAllocSize + 1,
  1054. kAlignment);
  1055. ASSERT_GT(kMinReasonableTestSize, kExtraAllocSize);
  1056. const size_t kSizes[] = {kMinReasonableTestSize,
  1057. 256,
  1058. SystemPageSize(),
  1059. PartitionPageSize(),
  1060. MaxRegularSlotSpanSize(),
  1061. MaxRegularSlotSpanSize() + 1,
  1062. MaxRegularSlotSpanSize() + SystemPageSize(),
  1063. MaxRegularSlotSpanSize() + PartitionPageSize(),
  1064. kMaxBucketed,
  1065. kMaxBucketed + 1,
  1066. kMaxBucketed + SystemPageSize(),
  1067. kMaxBucketed + PartitionPageSize(),
  1068. kSuperPageSize};
  1069. #if defined(PA_HAS_64_BITS_POINTERS)
  1070. constexpr size_t kFarFarAwayDelta = 512 * kGiB;
  1071. #else
  1072. constexpr size_t kFarFarAwayDelta = kGiB;
  1073. #endif
  1074. for (size_t size : kSizes) {
  1075. size_t requested_size = size - kExtraAllocSize;
  1076. // For regular slot-span allocations, confirm the size fills the entire
  1077. // slot. Otherwise the test would be ineffective, as Partition Alloc has no
  1078. // ability to check against the actual allocated size.
  1079. // Single-slot slot-spans and direct map don't have that problem.
  1080. if (size <= MaxRegularSlotSpanSize()) {
  1081. ASSERT_EQ(requested_size,
  1082. allocator.root()->AllocationCapacityFromRequestedSize(
  1083. requested_size));
  1084. }
  1085. constexpr size_t kNumRepeats = 3;
  1086. void* ptrs[kNumRepeats];
  1087. for (void*& ptr : ptrs) {
  1088. ptr = allocator.root()->Alloc(requested_size, type_name);
  1089. // Double check.
  1090. if (size <= MaxRegularSlotSpanSize()) {
  1091. uintptr_t slot_start = allocator.root()->ObjectToSlotStart(ptr);
  1092. EXPECT_EQ(
  1093. requested_size,
  1094. allocator.root()->AllocationCapacityFromSlotStart(slot_start));
  1095. }
  1096. uintptr_t address = UntagPtr(ptr);
  1097. EXPECT_FALSE(PartitionAllocIsValidPtrDelta(address, -kFarFarAwayDelta));
  1098. EXPECT_FALSE(PartitionAllocIsValidPtrDelta(address, -kSuperPageSize));
  1099. EXPECT_FALSE(PartitionAllocIsValidPtrDelta(address, -1));
  1100. EXPECT_TRUE(PartitionAllocIsValidPtrDelta(address, 0));
  1101. EXPECT_TRUE(PartitionAllocIsValidPtrDelta(address, requested_size / 2));
  1102. EXPECT_TRUE(PartitionAllocIsValidPtrDelta(address, requested_size));
  1103. EXPECT_FALSE(PartitionAllocIsValidPtrDelta(address, requested_size + 1));
  1104. EXPECT_FALSE(PartitionAllocIsValidPtrDelta(
  1105. address, requested_size + kSuperPageSize));
  1106. EXPECT_FALSE(PartitionAllocIsValidPtrDelta(
  1107. address, requested_size + kFarFarAwayDelta));
  1108. EXPECT_FALSE(PartitionAllocIsValidPtrDelta(address + requested_size,
  1109. kFarFarAwayDelta));
  1110. EXPECT_FALSE(PartitionAllocIsValidPtrDelta(address + requested_size,
  1111. kSuperPageSize));
  1112. EXPECT_FALSE(PartitionAllocIsValidPtrDelta(address + requested_size, 1));
  1113. EXPECT_TRUE(PartitionAllocIsValidPtrDelta(address + requested_size, 0));
  1114. EXPECT_TRUE(PartitionAllocIsValidPtrDelta(address + requested_size,
  1115. -(requested_size / 2)));
  1116. EXPECT_TRUE(PartitionAllocIsValidPtrDelta(address + requested_size,
  1117. -requested_size));
  1118. EXPECT_FALSE(PartitionAllocIsValidPtrDelta(address + requested_size,
  1119. -requested_size - 1));
  1120. EXPECT_FALSE(PartitionAllocIsValidPtrDelta(
  1121. address + requested_size, -requested_size - kSuperPageSize));
  1122. EXPECT_FALSE(PartitionAllocIsValidPtrDelta(
  1123. address + requested_size, -requested_size - kFarFarAwayDelta));
  1124. }
  1125. for (void* ptr : ptrs)
  1126. allocator.root()->Free(ptr);
  1127. }
  1128. }
  1129. TEST_P(PartitionAllocTest, GetSlotStartMultiplePages) {
  1130. auto* root = allocator.root();
  1131. // Find the smallest bucket with multiple PartitionPages. When searching for
  1132. // a bucket here, we need to check two conditions:
  1133. // (1) The bucket is used in our current bucket distribution.
  1134. // (2) The bucket is large enough that our requested size (see below) will be
  1135. // non-zero.
  1136. size_t real_size = 0;
  1137. for (const auto& bucket : root->buckets) {
  1138. if ((root->buckets + SizeToIndex(bucket.slot_size))->slot_size !=
  1139. bucket.slot_size)
  1140. continue;
  1141. if (bucket.slot_size <= kExtraAllocSize)
  1142. continue;
  1143. if (bucket.num_system_pages_per_slot_span >
  1144. NumSystemPagesPerPartitionPage()) {
  1145. real_size = bucket.slot_size;
  1146. break;
  1147. }
  1148. }
  1149. // Make sure that we've managed to find an appropriate bucket.
  1150. ASSERT_GT(real_size, 0u);
  1151. const size_t requested_size = real_size - kExtraAllocSize;
  1152. // Double check we don't end up with 0 or negative size.
  1153. EXPECT_GT(requested_size, 0u);
  1154. EXPECT_LE(requested_size, real_size);
  1155. const auto* bucket = allocator.root()->buckets + SizeToIndex(real_size);
  1156. EXPECT_EQ(bucket->slot_size, real_size);
  1157. // Make sure the test is testing multiple partition pages case.
  1158. EXPECT_GT(bucket->num_system_pages_per_slot_span,
  1159. PartitionPageSize() / SystemPageSize());
  1160. size_t num_slots =
  1161. (bucket->num_system_pages_per_slot_span * SystemPageSize()) / real_size;
  1162. std::vector<void*> ptrs;
  1163. for (size_t i = 0; i < num_slots; ++i) {
  1164. ptrs.push_back(allocator.root()->Alloc(requested_size, type_name));
  1165. }
  1166. for (void* ptr : ptrs) {
  1167. uintptr_t address = UntagPtr(ptr);
  1168. uintptr_t slot_start = allocator.root()->ObjectToSlotStart(ptr);
  1169. EXPECT_EQ(allocator.root()->AllocationCapacityFromSlotStart(slot_start),
  1170. requested_size);
  1171. for (size_t offset = 0; offset < requested_size; offset += 13) {
  1172. EXPECT_EQ(PartitionAllocGetSlotStartInBRPPool(address + offset),
  1173. slot_start);
  1174. }
  1175. allocator.root()->Free(ptr);
  1176. }
  1177. }
  1178. #endif // BUILDFLAG(USE_BACKUP_REF_PTR)
  1179. // Test the realloc() contract.
  1180. TEST_P(PartitionAllocTest, Realloc) {
  1181. // realloc(0, size) should be equivalent to malloc().
  1182. void* ptr = allocator.root()->Realloc(nullptr, kTestAllocSize, type_name);
  1183. memset(ptr, 'A', kTestAllocSize);
  1184. auto* slot_span =
  1185. SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr));
  1186. // realloc(ptr, 0) should be equivalent to free().
  1187. void* ptr2 = allocator.root()->Realloc(ptr, 0, type_name);
  1188. EXPECT_EQ(nullptr, ptr2);
  1189. EXPECT_EQ(allocator.root()->ObjectToSlotStart(ptr),
  1190. UntagPtr(slot_span->get_freelist_head()));
  1191. // Test that growing an allocation with realloc() copies everything from the
  1192. // old allocation.
  1193. size_t size = SystemPageSize() - kExtraAllocSize;
  1194. // Confirm size fills the entire slot.
  1195. ASSERT_EQ(size, allocator.root()->AllocationCapacityFromRequestedSize(size));
  1196. ptr = allocator.root()->Alloc(size, type_name);
  1197. memset(ptr, 'A', size);
  1198. ptr2 = allocator.root()->Realloc(ptr, size + 1, type_name);
  1199. PA_EXPECT_PTR_NE(ptr, ptr2);
  1200. char* char_ptr2 = static_cast<char*>(ptr2);
  1201. EXPECT_EQ('A', char_ptr2[0]);
  1202. EXPECT_EQ('A', char_ptr2[size - 1]);
  1203. #if BUILDFLAG(PA_EXPENSIVE_DCHECKS_ARE_ON)
  1204. EXPECT_EQ(kUninitializedByte, static_cast<unsigned char>(char_ptr2[size]));
  1205. #endif
  1206. // Test that shrinking an allocation with realloc() also copies everything
  1207. // from the old allocation. Use |size - 1| to test what happens to the extra
  1208. // space before the cookie.
  1209. ptr = allocator.root()->Realloc(ptr2, size - 1, type_name);
  1210. PA_EXPECT_PTR_NE(ptr2, ptr);
  1211. char* char_ptr = static_cast<char*>(ptr);
  1212. EXPECT_EQ('A', char_ptr[0]);
  1213. EXPECT_EQ('A', char_ptr[size - 2]);
  1214. #if BUILDFLAG(PA_EXPENSIVE_DCHECKS_ARE_ON)
  1215. EXPECT_EQ(kUninitializedByte, static_cast<unsigned char>(char_ptr[size - 1]));
  1216. #endif
  1217. allocator.root()->Free(ptr);
  1218. // Single-slot slot spans...
  1219. // Test that growing an allocation with realloc() copies everything from the
  1220. // old allocation.
  1221. size = MaxRegularSlotSpanSize() + 1;
  1222. ASSERT_LE(2 * size, kMaxBucketed); // should be in single-slot span range
  1223. // Confirm size doesn't fill the entire slot.
  1224. ASSERT_LT(size, allocator.root()->AllocationCapacityFromRequestedSize(size));
  1225. ptr = allocator.root()->Alloc(size, type_name);
  1226. memset(ptr, 'A', size);
  1227. ptr2 = allocator.root()->Realloc(ptr, size * 2, type_name);
  1228. PA_EXPECT_PTR_NE(ptr, ptr2);
  1229. char_ptr2 = static_cast<char*>(ptr2);
  1230. EXPECT_EQ('A', char_ptr2[0]);
  1231. EXPECT_EQ('A', char_ptr2[size - 1]);
  1232. #if BUILDFLAG(PA_EXPENSIVE_DCHECKS_ARE_ON)
  1233. EXPECT_EQ(kUninitializedByte, static_cast<unsigned char>(char_ptr2[size]));
  1234. #endif
  1235. allocator.root()->Free(ptr2);
  1236. // Test that shrinking an allocation with realloc() also copies everything
  1237. // from the old allocation.
  1238. size = 2 * (MaxRegularSlotSpanSize() + 1);
  1239. ASSERT_GT(size / 2, MaxRegularSlotSpanSize()); // in single-slot span range
  1240. ptr = allocator.root()->Alloc(size, type_name);
  1241. memset(ptr, 'A', size);
  1242. ptr2 = allocator.root()->Realloc(ptr2, size / 2, type_name);
  1243. PA_EXPECT_PTR_NE(ptr, ptr2);
  1244. char_ptr2 = static_cast<char*>(ptr2);
  1245. EXPECT_EQ('A', char_ptr2[0]);
  1246. EXPECT_EQ('A', char_ptr2[size / 2 - 1]);
  1247. #if BUILDFLAG(PA_DCHECK_IS_ON)
  1248. // For single-slot slot spans, the cookie is always placed immediately after
  1249. // the allocation.
  1250. EXPECT_EQ(kCookieValue[0], static_cast<unsigned char>(char_ptr2[size / 2]));
  1251. #endif
  1252. allocator.root()->Free(ptr2);
  1253. // Test that shrinking a direct mapped allocation happens in-place.
  1254. // Pick a large size so that Realloc doesn't think it's worthwhile to
  1255. // downsize even if one less super page is used (due to high granularity on
  1256. // 64-bit systems).
  1257. size = 10 * kSuperPageSize + SystemPageSize() - 42;
  1258. ASSERT_GT(size - 32 * SystemPageSize(), kMaxBucketed);
  1259. ptr = allocator.root()->Alloc(size, type_name);
  1260. uintptr_t slot_start = allocator.root()->ObjectToSlotStart(ptr);
  1261. size_t actual_capacity =
  1262. allocator.root()->AllocationCapacityFromSlotStart(slot_start);
  1263. ptr2 = allocator.root()->Realloc(ptr, size - SystemPageSize(), type_name);
  1264. uintptr_t slot_start2 = allocator.root()->ObjectToSlotStart(ptr2);
  1265. EXPECT_EQ(slot_start, slot_start2);
  1266. EXPECT_EQ(actual_capacity - SystemPageSize(),
  1267. allocator.root()->AllocationCapacityFromSlotStart(slot_start2));
  1268. void* ptr3 =
  1269. allocator.root()->Realloc(ptr2, size - 32 * SystemPageSize(), type_name);
  1270. uintptr_t slot_start3 = allocator.root()->ObjectToSlotStart(ptr3);
  1271. EXPECT_EQ(slot_start2, slot_start3);
  1272. EXPECT_EQ(actual_capacity - 32 * SystemPageSize(),
  1273. allocator.root()->AllocationCapacityFromSlotStart(slot_start3));
  1274. // Test that a previously in-place shrunk direct mapped allocation can be
  1275. // expanded up again up to its original size.
  1276. ptr = allocator.root()->Realloc(ptr3, size, type_name);
  1277. slot_start = allocator.root()->ObjectToSlotStart(ptr);
  1278. EXPECT_EQ(slot_start3, slot_start);
  1279. EXPECT_EQ(actual_capacity,
  1280. allocator.root()->AllocationCapacityFromSlotStart(slot_start));
  1281. // Test that the allocation can be expanded in place up to its capacity.
  1282. ptr2 = allocator.root()->Realloc(ptr, actual_capacity, type_name);
  1283. slot_start2 = allocator.root()->ObjectToSlotStart(ptr2);
  1284. EXPECT_EQ(slot_start, slot_start2);
  1285. EXPECT_EQ(actual_capacity,
  1286. allocator.root()->AllocationCapacityFromSlotStart(slot_start2));
  1287. // Test that a direct mapped allocation is performed not in-place when the
  1288. // new size is small enough.
  1289. ptr3 = allocator.root()->Realloc(ptr2, SystemPageSize(), type_name);
  1290. slot_start3 = allocator.root()->ObjectToSlotStart(ptr3);
  1291. EXPECT_NE(slot_start, slot_start3);
  1292. allocator.root()->Free(ptr3);
  1293. }
  1294. TEST_P(PartitionAllocTest, ReallocDirectMapAligned) {
  1295. size_t alignments[] = {
  1296. PartitionPageSize(),
  1297. 2 * PartitionPageSize(),
  1298. kMaxSupportedAlignment / 2,
  1299. kMaxSupportedAlignment,
  1300. };
  1301. for (size_t alignment : alignments) {
  1302. // Test that shrinking a direct mapped allocation happens in-place.
  1303. // Pick a large size so that Realloc doesn't think it's worthwhile to
  1304. // downsize even if one less super page is used (due to high granularity on
  1305. // 64-bit systems), even if the alignment padding is taken out.
  1306. size_t size = 10 * kSuperPageSize + SystemPageSize() - 42;
  1307. ASSERT_GT(size, kMaxBucketed);
  1308. void* ptr =
  1309. allocator.root()->AllocWithFlagsInternal(0, size, alignment, type_name);
  1310. uintptr_t slot_start = allocator.root()->ObjectToSlotStart(ptr);
  1311. size_t actual_capacity =
  1312. allocator.root()->AllocationCapacityFromSlotStart(slot_start);
  1313. void* ptr2 =
  1314. allocator.root()->Realloc(ptr, size - SystemPageSize(), type_name);
  1315. uintptr_t slot_start2 = allocator.root()->ObjectToSlotStart(ptr2);
  1316. EXPECT_EQ(slot_start, slot_start2);
  1317. EXPECT_EQ(actual_capacity - SystemPageSize(),
  1318. allocator.root()->AllocationCapacityFromSlotStart(slot_start2));
  1319. void* ptr3 = allocator.root()->Realloc(ptr2, size - 32 * SystemPageSize(),
  1320. type_name);
  1321. uintptr_t slot_start3 = allocator.root()->ObjectToSlotStart(ptr3);
  1322. EXPECT_EQ(slot_start2, slot_start3);
  1323. EXPECT_EQ(actual_capacity - 32 * SystemPageSize(),
  1324. allocator.root()->AllocationCapacityFromSlotStart(slot_start3));
  1325. // Test that a previously in-place shrunk direct mapped allocation can be
  1326. // expanded up again up to its original size.
  1327. ptr = allocator.root()->Realloc(ptr3, size, type_name);
  1328. slot_start = allocator.root()->ObjectToSlotStart(ptr);
  1329. EXPECT_EQ(slot_start3, slot_start);
  1330. EXPECT_EQ(actual_capacity,
  1331. allocator.root()->AllocationCapacityFromSlotStart(slot_start));
  1332. // Test that the allocation can be expanded in place up to its capacity.
  1333. ptr2 = allocator.root()->Realloc(ptr, actual_capacity, type_name);
  1334. slot_start2 = allocator.root()->ObjectToSlotStart(ptr2);
  1335. EXPECT_EQ(slot_start, slot_start2);
  1336. EXPECT_EQ(actual_capacity,
  1337. allocator.root()->AllocationCapacityFromSlotStart(slot_start2));
  1338. // Test that a direct mapped allocation is performed not in-place when the
  1339. // new size is small enough.
  1340. ptr3 = allocator.root()->Realloc(ptr2, SystemPageSize(), type_name);
  1341. slot_start3 = allocator.root()->ObjectToSlotStart(ptr3);
  1342. EXPECT_NE(slot_start2, slot_start3);
  1343. allocator.root()->Free(ptr3);
  1344. }
  1345. }
  1346. TEST_P(PartitionAllocTest, ReallocDirectMapAlignedRelocate) {
  1347. // Pick size such that the alignment will put it cross the super page
  1348. // boundary.
  1349. size_t size = 2 * kSuperPageSize - kMaxSupportedAlignment + SystemPageSize();
  1350. ASSERT_GT(size, kMaxBucketed);
  1351. void* ptr = allocator.root()->AllocWithFlagsInternal(
  1352. 0, size, kMaxSupportedAlignment, type_name);
  1353. // Reallocating with the same size will actually relocate, because without a
  1354. // need for alignment we can downsize the reservation significantly.
  1355. void* ptr2 = allocator.root()->Realloc(ptr, size, type_name);
  1356. PA_EXPECT_PTR_NE(ptr, ptr2);
  1357. allocator.root()->Free(ptr2);
  1358. // Again pick size such that the alignment will put it cross the super page
  1359. // boundary, but this time make it so large that Realloc doesn't fing it worth
  1360. // shrinking.
  1361. size = 10 * kSuperPageSize - kMaxSupportedAlignment + SystemPageSize();
  1362. ASSERT_GT(size, kMaxBucketed);
  1363. ptr = allocator.root()->AllocWithFlagsInternal(
  1364. 0, size, kMaxSupportedAlignment, type_name);
  1365. ptr2 = allocator.root()->Realloc(ptr, size, type_name);
  1366. EXPECT_EQ(ptr, ptr2);
  1367. allocator.root()->Free(ptr2);
  1368. }
  1369. // Tests the handing out of freelists for partial slot spans.
  1370. TEST_P(PartitionAllocTest, PartialPageFreelists) {
  1371. size_t big_size = SystemPageSize() - kExtraAllocSize;
  1372. size_t bucket_index = SizeToIndex(big_size + kExtraAllocSize);
  1373. PartitionRoot<ThreadSafe>::Bucket* bucket =
  1374. &allocator.root()->buckets[bucket_index];
  1375. EXPECT_EQ(nullptr, bucket->empty_slot_spans_head);
  1376. void* ptr = allocator.root()->Alloc(big_size, type_name);
  1377. EXPECT_TRUE(ptr);
  1378. auto* slot_span =
  1379. SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr));
  1380. size_t total_slots =
  1381. (slot_span->bucket->num_system_pages_per_slot_span * SystemPageSize()) /
  1382. (big_size + kExtraAllocSize);
  1383. EXPECT_EQ(4u, total_slots);
  1384. // The freelist should have one entry, because we were able to exactly fit
  1385. // one object slot and one freelist pointer (the null that the head points
  1386. // to) into a system page.
  1387. EXPECT_FALSE(slot_span->get_freelist_head());
  1388. EXPECT_EQ(1u, slot_span->num_allocated_slots);
  1389. EXPECT_EQ(3u, slot_span->num_unprovisioned_slots);
  1390. void* ptr2 = allocator.root()->Alloc(big_size, type_name);
  1391. EXPECT_TRUE(ptr2);
  1392. EXPECT_FALSE(slot_span->get_freelist_head());
  1393. EXPECT_EQ(2u, slot_span->num_allocated_slots);
  1394. EXPECT_EQ(2u, slot_span->num_unprovisioned_slots);
  1395. void* ptr3 = allocator.root()->Alloc(big_size, type_name);
  1396. EXPECT_TRUE(ptr3);
  1397. EXPECT_FALSE(slot_span->get_freelist_head());
  1398. EXPECT_EQ(3u, slot_span->num_allocated_slots);
  1399. EXPECT_EQ(1u, slot_span->num_unprovisioned_slots);
  1400. void* ptr4 = allocator.root()->Alloc(big_size, type_name);
  1401. EXPECT_TRUE(ptr4);
  1402. EXPECT_FALSE(slot_span->get_freelist_head());
  1403. EXPECT_EQ(4u, slot_span->num_allocated_slots);
  1404. EXPECT_EQ(0u, slot_span->num_unprovisioned_slots);
  1405. void* ptr5 = allocator.root()->Alloc(big_size, type_name);
  1406. EXPECT_TRUE(ptr5);
  1407. auto* slot_span2 =
  1408. SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr5));
  1409. EXPECT_EQ(1u, slot_span2->num_allocated_slots);
  1410. // Churn things a little whilst there's a partial slot span freelist.
  1411. allocator.root()->Free(ptr);
  1412. ptr = allocator.root()->Alloc(big_size, type_name);
  1413. void* ptr6 = allocator.root()->Alloc(big_size, type_name);
  1414. allocator.root()->Free(ptr);
  1415. allocator.root()->Free(ptr2);
  1416. allocator.root()->Free(ptr3);
  1417. allocator.root()->Free(ptr4);
  1418. allocator.root()->Free(ptr5);
  1419. allocator.root()->Free(ptr6);
  1420. EXPECT_TRUE(slot_span->in_empty_cache());
  1421. EXPECT_TRUE(slot_span2->in_empty_cache());
  1422. EXPECT_TRUE(slot_span2->get_freelist_head());
  1423. EXPECT_EQ(0u, slot_span2->num_allocated_slots);
  1424. // Size that's just above half a page.
  1425. size_t non_dividing_size = SystemPageSize() / 2 + 1 - kExtraAllocSize;
  1426. bucket_index = SizeToIndex(non_dividing_size + kExtraAllocSize);
  1427. bucket = &allocator.root()->buckets[bucket_index];
  1428. EXPECT_EQ(nullptr, bucket->empty_slot_spans_head);
  1429. ptr = allocator.root()->Alloc(non_dividing_size, type_name);
  1430. EXPECT_TRUE(ptr);
  1431. slot_span = SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr));
  1432. total_slots =
  1433. (slot_span->bucket->num_system_pages_per_slot_span * SystemPageSize()) /
  1434. bucket->slot_size;
  1435. EXPECT_FALSE(slot_span->get_freelist_head());
  1436. EXPECT_EQ(1u, slot_span->num_allocated_slots);
  1437. EXPECT_EQ(total_slots - 1, slot_span->num_unprovisioned_slots);
  1438. ptr2 = allocator.root()->Alloc(non_dividing_size, type_name);
  1439. EXPECT_TRUE(ptr2);
  1440. EXPECT_TRUE(slot_span->get_freelist_head());
  1441. EXPECT_EQ(2u, slot_span->num_allocated_slots);
  1442. // 2 slots got provisioned: the first one fills the rest of the first (already
  1443. // provision page) and exceeds it by just a tad, thus leading to provisioning
  1444. // a new page, and the second one fully fits within that new page.
  1445. EXPECT_EQ(total_slots - 3, slot_span->num_unprovisioned_slots);
  1446. ptr3 = allocator.root()->Alloc(non_dividing_size, type_name);
  1447. EXPECT_TRUE(ptr3);
  1448. EXPECT_FALSE(slot_span->get_freelist_head());
  1449. EXPECT_EQ(3u, slot_span->num_allocated_slots);
  1450. EXPECT_EQ(total_slots - 3, slot_span->num_unprovisioned_slots);
  1451. allocator.root()->Free(ptr);
  1452. allocator.root()->Free(ptr2);
  1453. allocator.root()->Free(ptr3);
  1454. EXPECT_TRUE(slot_span->in_empty_cache());
  1455. EXPECT_TRUE(slot_span2->get_freelist_head());
  1456. EXPECT_EQ(0u, slot_span2->num_allocated_slots);
  1457. // And test a couple of sizes that do not cross SystemPageSize() with a
  1458. // single allocation.
  1459. size_t medium_size = (SystemPageSize() / 2) - kExtraAllocSize;
  1460. bucket_index = SizeToIndex(medium_size + kExtraAllocSize);
  1461. bucket = &allocator.root()->buckets[bucket_index];
  1462. EXPECT_EQ(nullptr, bucket->empty_slot_spans_head);
  1463. ptr = allocator.root()->Alloc(medium_size, type_name);
  1464. EXPECT_TRUE(ptr);
  1465. slot_span = SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr));
  1466. EXPECT_EQ(1u, slot_span->num_allocated_slots);
  1467. total_slots =
  1468. (slot_span->bucket->num_system_pages_per_slot_span * SystemPageSize()) /
  1469. (medium_size + kExtraAllocSize);
  1470. size_t first_slot_span_slots =
  1471. SystemPageSize() / (medium_size + kExtraAllocSize);
  1472. EXPECT_EQ(2u, first_slot_span_slots);
  1473. EXPECT_EQ(total_slots - first_slot_span_slots,
  1474. slot_span->num_unprovisioned_slots);
  1475. allocator.root()->Free(ptr);
  1476. size_t small_size = (SystemPageSize() / 4) - kExtraAllocSize;
  1477. bucket_index = SizeToIndex(small_size + kExtraAllocSize);
  1478. bucket = &allocator.root()->buckets[bucket_index];
  1479. EXPECT_EQ(nullptr, bucket->empty_slot_spans_head);
  1480. ptr = allocator.root()->Alloc(small_size, type_name);
  1481. EXPECT_TRUE(ptr);
  1482. slot_span = SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr));
  1483. EXPECT_EQ(1u, slot_span->num_allocated_slots);
  1484. total_slots =
  1485. (slot_span->bucket->num_system_pages_per_slot_span * SystemPageSize()) /
  1486. (small_size + kExtraAllocSize);
  1487. first_slot_span_slots = SystemPageSize() / (small_size + kExtraAllocSize);
  1488. EXPECT_EQ(total_slots - first_slot_span_slots,
  1489. slot_span->num_unprovisioned_slots);
  1490. allocator.root()->Free(ptr);
  1491. EXPECT_TRUE(slot_span->get_freelist_head());
  1492. EXPECT_EQ(0u, slot_span->num_allocated_slots);
  1493. static_assert(kExtraAllocSize < 64, "");
  1494. size_t very_small_size =
  1495. (kExtraAllocSize <= 32) ? (32 - kExtraAllocSize) : (64 - kExtraAllocSize);
  1496. size_t very_small_adjusted_size =
  1497. allocator.root()->AdjustSize0IfNeeded(very_small_size);
  1498. bucket_index = SizeToIndex(very_small_adjusted_size + kExtraAllocSize);
  1499. bucket = &allocator.root()->buckets[bucket_index];
  1500. EXPECT_EQ(nullptr, bucket->empty_slot_spans_head);
  1501. ptr = allocator.root()->Alloc(very_small_size, type_name);
  1502. EXPECT_TRUE(ptr);
  1503. slot_span = SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr));
  1504. EXPECT_EQ(1u, slot_span->num_allocated_slots);
  1505. size_t very_small_actual_size = allocator.root()->GetUsableSize(ptr);
  1506. total_slots =
  1507. (slot_span->bucket->num_system_pages_per_slot_span * SystemPageSize()) /
  1508. (very_small_actual_size + kExtraAllocSize);
  1509. first_slot_span_slots =
  1510. SystemPageSize() / (very_small_actual_size + kExtraAllocSize);
  1511. EXPECT_EQ(total_slots - first_slot_span_slots,
  1512. slot_span->num_unprovisioned_slots);
  1513. allocator.root()->Free(ptr);
  1514. EXPECT_TRUE(slot_span->get_freelist_head());
  1515. EXPECT_EQ(0u, slot_span->num_allocated_slots);
  1516. // And try an allocation size (against the generic allocator) that is
  1517. // larger than a system page.
  1518. size_t page_and_a_half_size =
  1519. (SystemPageSize() + (SystemPageSize() / 2)) - kExtraAllocSize;
  1520. ptr = allocator.root()->Alloc(page_and_a_half_size, type_name);
  1521. EXPECT_TRUE(ptr);
  1522. slot_span = SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr));
  1523. EXPECT_EQ(1u, slot_span->num_allocated_slots);
  1524. // Only the first slot was provisioned, and that's the one that was just
  1525. // allocated so the free list is empty.
  1526. EXPECT_TRUE(!slot_span->get_freelist_head());
  1527. total_slots =
  1528. (slot_span->bucket->num_system_pages_per_slot_span * SystemPageSize()) /
  1529. (page_and_a_half_size + kExtraAllocSize);
  1530. EXPECT_EQ(total_slots - 1, slot_span->num_unprovisioned_slots);
  1531. ptr2 = allocator.root()->Alloc(page_and_a_half_size, type_name);
  1532. EXPECT_TRUE(ptr);
  1533. slot_span = SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr));
  1534. EXPECT_EQ(2u, slot_span->num_allocated_slots);
  1535. // As above, only one slot was provisioned.
  1536. EXPECT_TRUE(!slot_span->get_freelist_head());
  1537. EXPECT_EQ(total_slots - 2, slot_span->num_unprovisioned_slots);
  1538. allocator.root()->Free(ptr);
  1539. allocator.root()->Free(ptr2);
  1540. // And then make sure than exactly the page size only faults one page.
  1541. size_t page_size = SystemPageSize() - kExtraAllocSize;
  1542. ptr = allocator.root()->Alloc(page_size, type_name);
  1543. EXPECT_TRUE(ptr);
  1544. slot_span = SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr));
  1545. EXPECT_EQ(1u, slot_span->num_allocated_slots);
  1546. EXPECT_TRUE(slot_span->get_freelist_head());
  1547. total_slots =
  1548. (slot_span->bucket->num_system_pages_per_slot_span * SystemPageSize()) /
  1549. (page_size + kExtraAllocSize);
  1550. EXPECT_EQ(total_slots - 2, slot_span->num_unprovisioned_slots);
  1551. allocator.root()->Free(ptr);
  1552. }
  1553. // Test some of the fragmentation-resistant properties of the allocator.
  1554. TEST_P(PartitionAllocTest, SlotSpanRefilling) {
  1555. PartitionRoot<ThreadSafe>::Bucket* bucket =
  1556. &allocator.root()->buckets[test_bucket_index_];
  1557. // Grab two full slot spans and a non-full slot span.
  1558. auto* slot_span1 = GetFullSlotSpan(kTestAllocSize);
  1559. auto* slot_span2 = GetFullSlotSpan(kTestAllocSize);
  1560. void* ptr = allocator.root()->Alloc(kTestAllocSize, type_name);
  1561. EXPECT_TRUE(ptr);
  1562. EXPECT_NE(slot_span1, bucket->active_slot_spans_head);
  1563. EXPECT_NE(slot_span2, bucket->active_slot_spans_head);
  1564. auto* slot_span =
  1565. SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr));
  1566. EXPECT_EQ(1u, slot_span->num_allocated_slots);
  1567. // Work out a pointer into slot_span2 and free it; and then slot_span1 and
  1568. // free it.
  1569. void* ptr2 = allocator.root()->SlotStartToObject(
  1570. SlotSpan::ToSlotSpanStart(slot_span1));
  1571. allocator.root()->Free(ptr2);
  1572. ptr2 = allocator.root()->SlotStartToObject(
  1573. SlotSpan::ToSlotSpanStart(slot_span2));
  1574. allocator.root()->Free(ptr2);
  1575. // If we perform two allocations from the same bucket now, we expect to
  1576. // refill both the nearly full slot spans.
  1577. std::ignore = allocator.root()->Alloc(kTestAllocSize, type_name);
  1578. std::ignore = allocator.root()->Alloc(kTestAllocSize, type_name);
  1579. EXPECT_EQ(1u, slot_span->num_allocated_slots);
  1580. FreeFullSlotSpan(allocator.root(), slot_span2);
  1581. FreeFullSlotSpan(allocator.root(), slot_span1);
  1582. allocator.root()->Free(ptr);
  1583. }
  1584. // Basic tests to ensure that allocations work for partial page buckets.
  1585. TEST_P(PartitionAllocTest, PartialPages) {
  1586. // Find a size that is backed by a partial partition page.
  1587. size_t size = sizeof(void*);
  1588. size_t bucket_index;
  1589. PartitionRoot<ThreadSafe>::Bucket* bucket = nullptr;
  1590. constexpr size_t kMaxSize = 4000u;
  1591. while (size < kMaxSize) {
  1592. bucket_index = SizeToIndex(size + kExtraAllocSize);
  1593. bucket = &allocator.root()->buckets[bucket_index];
  1594. if (bucket->num_system_pages_per_slot_span %
  1595. NumSystemPagesPerPartitionPage()) {
  1596. break;
  1597. }
  1598. size += sizeof(void*);
  1599. }
  1600. EXPECT_LT(size, kMaxSize);
  1601. auto* slot_span1 = GetFullSlotSpan(size);
  1602. auto* slot_span2 = GetFullSlotSpan(size);
  1603. FreeFullSlotSpan(allocator.root(), slot_span2);
  1604. FreeFullSlotSpan(allocator.root(), slot_span1);
  1605. }
  1606. // Test correct handling if our mapping collides with another.
  1607. TEST_P(PartitionAllocTest, MappingCollision) {
  1608. size_t num_pages_per_slot_span = GetNumPagesPerSlotSpan(kTestAllocSize);
  1609. // The -2 is because the first and last partition pages in a super page are
  1610. // guard pages. We also discount the partition pages used for the tag bitmap.
  1611. size_t num_slot_span_needed =
  1612. (NumPartitionPagesPerSuperPage() - 2 -
  1613. partition_alloc::internal::NumPartitionPagesPerTagBitmap()) /
  1614. num_pages_per_slot_span;
  1615. size_t num_partition_pages_needed =
  1616. num_slot_span_needed * num_pages_per_slot_span;
  1617. auto first_super_page_pages =
  1618. std::make_unique<SlotSpan*[]>(num_partition_pages_needed);
  1619. auto second_super_page_pages =
  1620. std::make_unique<SlotSpan*[]>(num_partition_pages_needed);
  1621. size_t i;
  1622. for (i = 0; i < num_partition_pages_needed; ++i)
  1623. first_super_page_pages[i] = GetFullSlotSpan(kTestAllocSize);
  1624. uintptr_t slot_spart_start =
  1625. SlotSpan::ToSlotSpanStart(first_super_page_pages[0]);
  1626. EXPECT_EQ(
  1627. PartitionPageSize() + partition_alloc::internal::ReservedTagBitmapSize(),
  1628. slot_spart_start & kSuperPageOffsetMask);
  1629. uintptr_t super_page = slot_spart_start - PartitionPageSize() -
  1630. partition_alloc::internal::ReservedTagBitmapSize();
  1631. // Map a single system page either side of the mapping for our allocations,
  1632. // with the goal of tripping up alignment of the next mapping.
  1633. uintptr_t map1 = AllocPages(
  1634. super_page - PageAllocationGranularity(), PageAllocationGranularity(),
  1635. PageAllocationGranularity(),
  1636. PageAccessibilityConfiguration::kInaccessible, PageTag::kPartitionAlloc);
  1637. EXPECT_TRUE(map1);
  1638. uintptr_t map2 = AllocPages(
  1639. super_page + kSuperPageSize, PageAllocationGranularity(),
  1640. PageAllocationGranularity(),
  1641. PageAccessibilityConfiguration::kInaccessible, PageTag::kPartitionAlloc);
  1642. EXPECT_TRUE(map2);
  1643. for (i = 0; i < num_partition_pages_needed; ++i)
  1644. second_super_page_pages[i] = GetFullSlotSpan(kTestAllocSize);
  1645. FreePages(map1, PageAllocationGranularity());
  1646. FreePages(map2, PageAllocationGranularity());
  1647. super_page = SlotSpan::ToSlotSpanStart(second_super_page_pages[0]);
  1648. EXPECT_EQ(
  1649. PartitionPageSize() + partition_alloc::internal::ReservedTagBitmapSize(),
  1650. super_page & kSuperPageOffsetMask);
  1651. super_page -=
  1652. PartitionPageSize() - partition_alloc::internal::ReservedTagBitmapSize();
  1653. // Map a single system page either side of the mapping for our allocations,
  1654. // with the goal of tripping up alignment of the next mapping.
  1655. map1 = AllocPages(super_page - PageAllocationGranularity(),
  1656. PageAllocationGranularity(), PageAllocationGranularity(),
  1657. PageAccessibilityConfiguration::kReadWriteTagged,
  1658. PageTag::kPartitionAlloc);
  1659. EXPECT_TRUE(map1);
  1660. map2 = AllocPages(super_page + kSuperPageSize, PageAllocationGranularity(),
  1661. PageAllocationGranularity(),
  1662. PageAccessibilityConfiguration::kReadWriteTagged,
  1663. PageTag::kPartitionAlloc);
  1664. EXPECT_TRUE(map2);
  1665. EXPECT_TRUE(
  1666. TrySetSystemPagesAccess(map1, PageAllocationGranularity(),
  1667. PageAccessibilityConfiguration::kInaccessible));
  1668. EXPECT_TRUE(
  1669. TrySetSystemPagesAccess(map2, PageAllocationGranularity(),
  1670. PageAccessibilityConfiguration::kInaccessible));
  1671. auto* slot_span_in_third_super_page = GetFullSlotSpan(kTestAllocSize);
  1672. FreePages(map1, PageAllocationGranularity());
  1673. FreePages(map2, PageAllocationGranularity());
  1674. EXPECT_EQ(0u, SlotSpan::ToSlotSpanStart(slot_span_in_third_super_page) &
  1675. PartitionPageOffsetMask());
  1676. // And make sure we really did get a page in a new superpage.
  1677. EXPECT_NE(
  1678. SlotSpan::ToSlotSpanStart(first_super_page_pages[0]) & kSuperPageBaseMask,
  1679. SlotSpan::ToSlotSpanStart(slot_span_in_third_super_page) &
  1680. kSuperPageBaseMask);
  1681. EXPECT_NE(SlotSpan::ToSlotSpanStart(second_super_page_pages[0]) &
  1682. kSuperPageBaseMask,
  1683. SlotSpan::ToSlotSpanStart(slot_span_in_third_super_page) &
  1684. kSuperPageBaseMask);
  1685. FreeFullSlotSpan(allocator.root(), slot_span_in_third_super_page);
  1686. for (i = 0; i < num_partition_pages_needed; ++i) {
  1687. FreeFullSlotSpan(allocator.root(), first_super_page_pages[i]);
  1688. FreeFullSlotSpan(allocator.root(), second_super_page_pages[i]);
  1689. }
  1690. }
  1691. // Tests that slot spans in the free slot span cache do get freed as
  1692. // appropriate.
  1693. TEST_P(PartitionAllocTest, FreeCache) {
  1694. EXPECT_EQ(0U, allocator.root()->get_total_size_of_committed_pages());
  1695. size_t big_size = 1000 - kExtraAllocSize;
  1696. size_t bucket_index = SizeToIndex(big_size + kExtraAllocSize);
  1697. PartitionBucket<internal::ThreadSafe>* bucket =
  1698. &allocator.root()->buckets[bucket_index];
  1699. void* ptr = allocator.root()->Alloc(big_size, type_name);
  1700. EXPECT_TRUE(ptr);
  1701. auto* slot_span =
  1702. SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr));
  1703. EXPECT_EQ(nullptr, bucket->empty_slot_spans_head);
  1704. EXPECT_EQ(1u, slot_span->num_allocated_slots);
  1705. // Lazy commit commits only needed pages.
  1706. size_t expected_committed_size =
  1707. kUseLazyCommit ? SystemPageSize() : PartitionPageSize();
  1708. EXPECT_EQ(expected_committed_size,
  1709. allocator.root()->get_total_size_of_committed_pages());
  1710. allocator.root()->Free(ptr);
  1711. EXPECT_EQ(0u, slot_span->num_allocated_slots);
  1712. EXPECT_TRUE(slot_span->in_empty_cache());
  1713. EXPECT_TRUE(slot_span->get_freelist_head());
  1714. CycleFreeCache(kTestAllocSize);
  1715. // Flushing the cache should have really freed the unused slot spans.
  1716. EXPECT_FALSE(slot_span->get_freelist_head());
  1717. EXPECT_FALSE(slot_span->in_empty_cache());
  1718. EXPECT_EQ(0u, slot_span->num_allocated_slots);
  1719. size_t num_system_pages_per_slot_span = allocator.root()
  1720. ->buckets[test_bucket_index_]
  1721. .num_system_pages_per_slot_span;
  1722. size_t expected_size =
  1723. kUseLazyCommit ? SystemPageSize()
  1724. : num_system_pages_per_slot_span * SystemPageSize();
  1725. EXPECT_EQ(expected_size,
  1726. allocator.root()->get_total_size_of_committed_pages());
  1727. // Check that an allocation works ok whilst in this state (a free'd slot span
  1728. // as the active slot spans head).
  1729. ptr = allocator.root()->Alloc(big_size, type_name);
  1730. EXPECT_FALSE(bucket->empty_slot_spans_head);
  1731. allocator.root()->Free(ptr);
  1732. // Also check that a slot span that is bouncing immediately between empty and
  1733. // used does not get freed.
  1734. for (size_t i = 0; i < kMaxFreeableSpans * 2; ++i) {
  1735. ptr = allocator.root()->Alloc(big_size, type_name);
  1736. EXPECT_TRUE(slot_span->get_freelist_head());
  1737. allocator.root()->Free(ptr);
  1738. EXPECT_TRUE(slot_span->get_freelist_head());
  1739. }
  1740. EXPECT_EQ(expected_committed_size,
  1741. allocator.root()->get_total_size_of_committed_pages());
  1742. }
  1743. // Tests for a bug we had with losing references to free slot spans.
  1744. TEST_P(PartitionAllocTest, LostFreeSlotSpansBug) {
  1745. size_t size = PartitionPageSize() - kExtraAllocSize;
  1746. void* ptr = allocator.root()->Alloc(size, type_name);
  1747. EXPECT_TRUE(ptr);
  1748. void* ptr2 = allocator.root()->Alloc(size, type_name);
  1749. EXPECT_TRUE(ptr2);
  1750. SlotSpanMetadata<internal::ThreadSafe>* slot_span =
  1751. SlotSpanMetadata<internal::ThreadSafe>::FromSlotStart(
  1752. allocator.root()->ObjectToSlotStart(ptr));
  1753. SlotSpanMetadata<internal::ThreadSafe>* slot_span2 =
  1754. SlotSpanMetadata<internal::ThreadSafe>::FromSlotStart(
  1755. allocator.root()->ObjectToSlotStart(ptr2));
  1756. PartitionBucket<internal::ThreadSafe>* bucket = slot_span->bucket;
  1757. EXPECT_EQ(nullptr, bucket->empty_slot_spans_head);
  1758. EXPECT_EQ(1u, slot_span->num_allocated_slots);
  1759. EXPECT_EQ(1u, slot_span2->num_allocated_slots);
  1760. EXPECT_TRUE(slot_span->is_full());
  1761. EXPECT_TRUE(slot_span2->is_full());
  1762. // The first span was kicked out from the active list, but the second one
  1763. // wasn't.
  1764. EXPECT_TRUE(slot_span->marked_full);
  1765. EXPECT_FALSE(slot_span2->marked_full);
  1766. allocator.root()->Free(ptr);
  1767. allocator.root()->Free(ptr2);
  1768. EXPECT_TRUE(bucket->empty_slot_spans_head);
  1769. EXPECT_TRUE(bucket->empty_slot_spans_head->next_slot_span);
  1770. EXPECT_EQ(0u, slot_span->num_allocated_slots);
  1771. EXPECT_EQ(0u, slot_span2->num_allocated_slots);
  1772. EXPECT_FALSE(slot_span->is_full());
  1773. EXPECT_FALSE(slot_span->is_full());
  1774. EXPECT_FALSE(slot_span->marked_full);
  1775. EXPECT_FALSE(slot_span2->marked_full);
  1776. EXPECT_TRUE(slot_span->get_freelist_head());
  1777. EXPECT_TRUE(slot_span2->get_freelist_head());
  1778. CycleFreeCache(kTestAllocSize);
  1779. EXPECT_FALSE(slot_span->get_freelist_head());
  1780. EXPECT_FALSE(slot_span2->get_freelist_head());
  1781. EXPECT_TRUE(bucket->empty_slot_spans_head);
  1782. EXPECT_TRUE(bucket->empty_slot_spans_head->next_slot_span);
  1783. EXPECT_EQ(SlotSpanMetadata<internal::ThreadSafe>::get_sentinel_slot_span(),
  1784. bucket->active_slot_spans_head);
  1785. // At this moment, we have two decommitted slot spans, on the empty list.
  1786. ptr = allocator.root()->Alloc(size, type_name);
  1787. EXPECT_TRUE(ptr);
  1788. allocator.root()->Free(ptr);
  1789. EXPECT_EQ(SlotSpanMetadata<internal::ThreadSafe>::get_sentinel_slot_span(),
  1790. bucket->active_slot_spans_head);
  1791. EXPECT_TRUE(bucket->empty_slot_spans_head);
  1792. EXPECT_TRUE(bucket->decommitted_slot_spans_head);
  1793. CycleFreeCache(kTestAllocSize);
  1794. // We're now set up to trigger a historical bug by scanning over the active
  1795. // slot spans list. The current code gets into a different state, but we'll
  1796. // keep the test as being an interesting corner case.
  1797. ptr = allocator.root()->Alloc(size, type_name);
  1798. EXPECT_TRUE(ptr);
  1799. allocator.root()->Free(ptr);
  1800. EXPECT_TRUE(bucket->is_valid());
  1801. EXPECT_TRUE(bucket->empty_slot_spans_head);
  1802. EXPECT_TRUE(bucket->decommitted_slot_spans_head);
  1803. }
  1804. // Death tests misbehave on Android, http://crbug.com/643760.
  1805. #if defined(GTEST_HAS_DEATH_TEST) && !BUILDFLAG(IS_ANDROID)
  1806. INSTANTIATE_TEST_SUITE_P(AlternateBucketDistribution,
  1807. PartitionAllocDeathTest,
  1808. testing::Values(false, true));
  1809. // Unit tests that check if an allocation fails in "return null" mode,
  1810. // repeating it doesn't crash, and still returns null. The tests need to
  1811. // stress memory subsystem limits to do so, hence they try to allocate
  1812. // 6 GB of memory, each with a different per-allocation block sizes.
  1813. //
  1814. // On 64-bit systems we need to restrict the address space to force allocation
  1815. // failure, so these tests run only on POSIX systems that provide setrlimit(),
  1816. // and use it to limit address space to 6GB.
  1817. //
  1818. // Disable these tests on Android because, due to the allocation-heavy behavior,
  1819. // they tend to get OOM-killed rather than pass.
  1820. // TODO(https://crbug.com/779645): Fuchsia currently sets OS_POSIX, but does
  1821. // not provide a working setrlimit().
  1822. //
  1823. // Disable these test on Windows, since they run slower, so tend to timout and
  1824. // cause flake.
  1825. #if !BUILDFLAG(IS_WIN) && \
  1826. (!defined(ARCH_CPU_64_BITS) || \
  1827. (BUILDFLAG(IS_POSIX) && !(BUILDFLAG(IS_APPLE) || BUILDFLAG(IS_ANDROID))))
  1828. // The following four tests wrap a called function in an expect death statement
  1829. // to perform their test, because they are non-hermetic. Specifically they are
  1830. // going to attempt to exhaust the allocatable memory, which leaves the
  1831. // allocator in a bad global state.
  1832. // Performing them as death tests causes them to be forked into their own
  1833. // process, so they won't pollute other tests.
  1834. //
  1835. // These tests are *very* slow when BUILDFLAG(PA_DCHECK_IS_ON), because they
  1836. // memset() many GiB of data (see crbug.com/1168168).
  1837. // TODO(lizeb): make these tests faster.
  1838. TEST_P(PartitionAllocDeathTest, RepeatedAllocReturnNullDirect) {
  1839. // A direct-mapped allocation size.
  1840. size_t direct_map_size = 32 * 1024 * 1024;
  1841. ASSERT_GT(direct_map_size, kMaxBucketed);
  1842. EXPECT_DEATH(DoReturnNullTest(direct_map_size, kPartitionAllocWithFlags),
  1843. "Passed DoReturnNullTest");
  1844. }
  1845. // Repeating above test with Realloc
  1846. TEST_P(PartitionAllocDeathTest, RepeatedReallocReturnNullDirect) {
  1847. size_t direct_map_size = 32 * 1024 * 1024;
  1848. ASSERT_GT(direct_map_size, kMaxBucketed);
  1849. EXPECT_DEATH(DoReturnNullTest(direct_map_size, kPartitionReallocWithFlags),
  1850. "Passed DoReturnNullTest");
  1851. }
  1852. // Repeating above test with TryRealloc
  1853. TEST_P(PartitionAllocDeathTest, RepeatedTryReallocReturnNullDirect) {
  1854. size_t direct_map_size = 32 * 1024 * 1024;
  1855. ASSERT_GT(direct_map_size, kMaxBucketed);
  1856. EXPECT_DEATH(DoReturnNullTest(direct_map_size, kPartitionRootTryRealloc),
  1857. "Passed DoReturnNullTest");
  1858. }
  1859. // TODO(crbug.com/1348221) re-enable the tests below, once the allocator
  1860. // actually returns nullptr for non direct-mapped allocations.
  1861. //
  1862. // Tests "return null" with a 512 kB block size.
  1863. TEST_P(PartitionAllocDeathTest, DISABLED_RepeatedAllocReturnNull) {
  1864. // A single-slot but non-direct-mapped allocation size.
  1865. size_t single_slot_size = 512 * 1024;
  1866. ASSERT_GT(single_slot_size, MaxRegularSlotSpanSize());
  1867. ASSERT_LE(single_slot_size, kMaxBucketed);
  1868. EXPECT_DEATH(DoReturnNullTest(single_slot_size, kPartitionAllocWithFlags),
  1869. "Passed DoReturnNullTest");
  1870. }
  1871. // Repeating above test with Realloc.
  1872. TEST_P(PartitionAllocDeathTest, DISABLED_RepeatedReallocReturnNull) {
  1873. size_t single_slot_size = 512 * 1024;
  1874. ASSERT_GT(single_slot_size, MaxRegularSlotSpanSize());
  1875. ASSERT_LE(single_slot_size, kMaxBucketed);
  1876. EXPECT_DEATH(DoReturnNullTest(single_slot_size, kPartitionReallocWithFlags),
  1877. "Passed DoReturnNullTest");
  1878. }
  1879. // Repeating above test with TryRealloc.
  1880. TEST_P(PartitionAllocDeathTest, DISABLED_RepeatedTryReallocReturnNull) {
  1881. size_t single_slot_size = 512 * 1024;
  1882. ASSERT_GT(single_slot_size, MaxRegularSlotSpanSize());
  1883. ASSERT_LE(single_slot_size, kMaxBucketed);
  1884. EXPECT_DEATH(DoReturnNullTest(single_slot_size, kPartitionRootTryRealloc),
  1885. "Passed DoReturnNullTest");
  1886. }
  1887. #endif // !defined(ARCH_CPU_64_BITS) || (BUILDFLAG(IS_POSIX) &&
  1888. // !(BUILDFLAG(IS_APPLE) || BUILDFLAG(IS_ANDROID)))
  1889. // Make sure that malloc(-1) dies.
  1890. // In the past, we had an integer overflow that would alias malloc(-1) to
  1891. // malloc(0), which is not good.
  1892. TEST_P(PartitionAllocDeathTest, LargeAllocs) {
  1893. // Largest alloc.
  1894. EXPECT_DEATH(allocator.root()->Alloc(static_cast<size_t>(-1), type_name), "");
  1895. // And the smallest allocation we expect to die.
  1896. // TODO(bartekn): Separate into its own test, as it wouldn't run (same below).
  1897. EXPECT_DEATH(allocator.root()->Alloc(MaxDirectMapped() + 1, type_name), "");
  1898. }
  1899. // These tests don't work deterministically when BRP is enabled on certain
  1900. // architectures. On Free(), BRP's ref-count gets overwritten by an encoded
  1901. // freelist pointer. On little-endian 64-bit architectures, this happens to be
  1902. // always an even number, which will triggers BRP's own CHECK (sic!). On other
  1903. // architectures, it's likely to be an odd number >1, which will fool BRP into
  1904. // thinking the memory isn't freed and still referenced, thus making it
  1905. // quarantine it and return early, before PA_CHECK(slot_start != freelist_head)
  1906. // is reached.
  1907. // TODO(bartekn): Enable in the BUILDFLAG(PUT_REF_COUNT_IN_PREVIOUS_SLOT) case.
  1908. #if !BUILDFLAG(USE_BACKUP_REF_PTR) || \
  1909. (defined(PA_HAS_64_BITS_POINTERS) && defined(ARCH_CPU_LITTLE_ENDIAN))
  1910. // Check that our immediate double-free detection works.
  1911. TEST_P(PartitionAllocDeathTest, ImmediateDoubleFree) {
  1912. void* ptr = allocator.root()->Alloc(kTestAllocSize, type_name);
  1913. EXPECT_TRUE(ptr);
  1914. allocator.root()->Free(ptr);
  1915. EXPECT_DEATH(allocator.root()->Free(ptr), "");
  1916. }
  1917. // As above, but when this isn't the only slot in the span.
  1918. TEST_P(PartitionAllocDeathTest, ImmediateDoubleFree2ndSlot) {
  1919. void* ptr0 = allocator.root()->Alloc(kTestAllocSize, type_name);
  1920. EXPECT_TRUE(ptr0);
  1921. void* ptr = allocator.root()->Alloc(kTestAllocSize, type_name);
  1922. EXPECT_TRUE(ptr);
  1923. allocator.root()->Free(ptr);
  1924. EXPECT_DEATH(allocator.root()->Free(ptr), "");
  1925. allocator.root()->Free(ptr0);
  1926. }
  1927. // Check that our double-free detection based on |num_allocated_slots| not going
  1928. // below 0 works.
  1929. //
  1930. // Unlike in ImmediateDoubleFree test, we can't have a 2ndSlot version, as this
  1931. // protection wouldn't work when there is another slot present in the span. It
  1932. // will prevent |num_allocated_slots| from going below 0.
  1933. TEST_P(PartitionAllocDeathTest, NumAllocatedSlotsDoubleFree) {
  1934. void* ptr = allocator.root()->Alloc(kTestAllocSize, type_name);
  1935. EXPECT_TRUE(ptr);
  1936. void* ptr2 = allocator.root()->Alloc(kTestAllocSize, type_name);
  1937. EXPECT_TRUE(ptr2);
  1938. allocator.root()->Free(ptr);
  1939. allocator.root()->Free(ptr2);
  1940. // This is not an immediate double-free so our immediate detection won't
  1941. // fire. However, it does take |num_allocated_slots| to -1, which is illegal
  1942. // and should be trapped.
  1943. EXPECT_DEATH(allocator.root()->Free(ptr), "");
  1944. }
  1945. #endif // !BUILDFLAG(USE_BACKUP_REF_PTR) || \
  1946. // (defined(PA_HAS_64_BITS_POINTERS) && defined(ARCH_CPU_LITTLE_ENDIAN))
  1947. // Check that guard pages are present where expected.
  1948. TEST_P(PartitionAllocDeathTest, DirectMapGuardPages) {
  1949. const size_t kSizes[] = {
  1950. kMaxBucketed + kExtraAllocSize + 1, kMaxBucketed + SystemPageSize(),
  1951. kMaxBucketed + PartitionPageSize(),
  1952. partition_alloc::internal::base::bits::AlignUp(
  1953. kMaxBucketed + kSuperPageSize, kSuperPageSize) -
  1954. PartitionRoot<ThreadSafe>::GetDirectMapMetadataAndGuardPagesSize()};
  1955. for (size_t size : kSizes) {
  1956. ASSERT_GT(size, kMaxBucketed);
  1957. size -= kExtraAllocSize;
  1958. EXPECT_GT(size, kMaxBucketed)
  1959. << "allocation not large enough for direct allocation";
  1960. void* ptr = allocator.root()->Alloc(size, type_name);
  1961. EXPECT_TRUE(ptr);
  1962. char* char_ptr = static_cast<char*>(ptr) - kPointerOffset;
  1963. EXPECT_DEATH(*(char_ptr - 1) = 'A', "");
  1964. EXPECT_DEATH(*(char_ptr + partition_alloc::internal::base::bits::AlignUp(
  1965. size, SystemPageSize())) = 'A',
  1966. "");
  1967. allocator.root()->Free(ptr);
  1968. }
  1969. }
  1970. #if (BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_ANDROID)) && defined(ARCH_CPU_ARM64)
  1971. TEST_P(PartitionAllocTest, MTEProtectsFreedPtr) {
  1972. // This test checks that Arm's memory tagging extension is correctly
  1973. // protecting freed pointers. Writes to a freed pointer should cause a crash.
  1974. base::CPU cpu;
  1975. if (!cpu.has_mte()) {
  1976. // This test won't pass on non-MTE systems.
  1977. GTEST_SKIP();
  1978. }
  1979. constexpr uint64_t kCookie = 0x1234567890ABCDEF;
  1980. constexpr uint64_t kQuarantined = 0xEFEFEFEFEFEFEFEF;
  1981. size_t alloc_size = 64 - kExtraAllocSize;
  1982. uint64_t* ptr1 =
  1983. static_cast<uint64_t*>(allocator.root()->Alloc(alloc_size, type_name));
  1984. EXPECT_TRUE(ptr1);
  1985. // Write to the pointer whilst it's live
  1986. *ptr1 = kCookie;
  1987. // Invalidate the pointer on free.
  1988. allocator.root()->Free(ptr1);
  1989. // Writing to ptr1 after free should crash.
  1990. EXPECT_EXIT(
  1991. {
  1992. // Should be in synchronous MTE mode for running this test.
  1993. *ptr1 = kQuarantined;
  1994. },
  1995. testing::KilledBySignal(SIGSEGV), "");
  1996. }
  1997. #endif
  1998. // These tests rely on precise layout. They handle cookie, not ref-count.
  1999. #if !BUILDFLAG(USE_BACKUP_REF_PTR) && defined(PA_HAS_FREELIST_SHADOW_ENTRY)
  2000. TEST_P(PartitionAllocDeathTest, UseAfterFreeDetection) {
  2001. base::CPU cpu;
  2002. void* data = allocator.root()->Alloc(100, "");
  2003. allocator.root()->Free(data);
  2004. // use after free, not crashing here, but the next allocation should crash,
  2005. // since we corrupted the freelist.
  2006. memset(data, 0x42, 100);
  2007. EXPECT_DEATH(allocator.root()->Alloc(100, ""), "");
  2008. }
  2009. TEST_P(PartitionAllocDeathTest, FreelistCorruption) {
  2010. base::CPU cpu;
  2011. const size_t alloc_size = 2 * sizeof(void*);
  2012. void** fake_freelist_entry =
  2013. static_cast<void**>(allocator.root()->Alloc(alloc_size, ""));
  2014. fake_freelist_entry[0] = nullptr;
  2015. fake_freelist_entry[1] = nullptr;
  2016. void** uaf_data =
  2017. static_cast<void**>(allocator.root()->Alloc(alloc_size, ""));
  2018. allocator.root()->Free(uaf_data);
  2019. // Try to confuse the allocator. This is still easy to circumvent willingly,
  2020. // "just" need to set uaf_data[1] to ~uaf_data[0].
  2021. void* previous_uaf_data = uaf_data[0];
  2022. uaf_data[0] = fake_freelist_entry;
  2023. EXPECT_DEATH(allocator.root()->Alloc(alloc_size, ""), "");
  2024. // Restore the freelist entry value, otherwise freelist corruption is detected
  2025. // in TearDown(), crashing this process.
  2026. uaf_data[0] = previous_uaf_data;
  2027. }
  2028. // With BUILDFLAG(PA_DCHECK_IS_ON), cookie already handles off-by-one detection.
  2029. #if !BUILDFLAG(PA_DCHECK_IS_ON)
  2030. TEST_P(PartitionAllocDeathTest, OffByOneDetection) {
  2031. base::CPU cpu;
  2032. const size_t alloc_size = 2 * sizeof(void*);
  2033. char* array = static_cast<char*>(allocator.root()->Alloc(alloc_size, ""));
  2034. if (cpu.has_mte()) {
  2035. EXPECT_DEATH(array[alloc_size] = 'A', "");
  2036. } else {
  2037. char previous_value = array[alloc_size];
  2038. // volatile is required to prevent the compiler from getting too clever and
  2039. // eliding the out-of-bounds write. The root cause is that the PA_MALLOC_FN
  2040. // annotation tells the compiler (among other things) that the returned
  2041. // value cannot alias anything.
  2042. *const_cast<volatile char*>(&array[alloc_size]) = 'A';
  2043. // Crash at the next allocation. This assumes that we are touching a new,
  2044. // non-randomized slot span, where the next slot to be handed over to the
  2045. // application directly follows the current one.
  2046. EXPECT_DEATH(allocator.root()->Alloc(alloc_size, ""), "");
  2047. // Restore integrity, otherwise the process will crash in TearDown().
  2048. array[alloc_size] = previous_value;
  2049. }
  2050. }
  2051. TEST_P(PartitionAllocDeathTest, OffByOneDetectionWithRealisticData) {
  2052. base::CPU cpu;
  2053. const size_t alloc_size = 2 * sizeof(void*);
  2054. void** array = static_cast<void**>(allocator.root()->Alloc(alloc_size, ""));
  2055. char valid;
  2056. if (cpu.has_mte()) {
  2057. EXPECT_DEATH(array[2] = &valid, "");
  2058. } else {
  2059. void* previous_value = array[2];
  2060. // As above, needs volatile to convince the compiler to perform the write.
  2061. *const_cast<void* volatile*>(&array[2]) = &valid;
  2062. // Crash at the next allocation. This assumes that we are touching a new,
  2063. // non-randomized slot span, where the next slot to be handed over to the
  2064. // application directly follows the current one.
  2065. EXPECT_DEATH(allocator.root()->Alloc(alloc_size, ""), "");
  2066. array[2] = previous_value;
  2067. }
  2068. }
  2069. #endif // !BUILDFLAG(PA_DCHECK_IS_ON)
  2070. #endif // !BUILDFLAG(USE_BACKUP_REF_PTR) &&
  2071. // defined(PA_HAS_FREELIST_SHADOW_ENTRY)
  2072. #endif // !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS)
  2073. // Tests that |PartitionDumpStats| and |PartitionDumpStats| run without
  2074. // crashing and return non-zero values when memory is allocated.
  2075. TEST_P(PartitionAllocTest, DumpMemoryStats) {
  2076. {
  2077. void* ptr = allocator.root()->Alloc(kTestAllocSize, type_name);
  2078. MockPartitionStatsDumper mock_stats_dumper;
  2079. allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
  2080. &mock_stats_dumper);
  2081. EXPECT_TRUE(mock_stats_dumper.IsMemoryAllocationRecorded());
  2082. allocator.root()->Free(ptr);
  2083. }
  2084. // This series of tests checks the active -> empty -> decommitted states.
  2085. {
  2086. {
  2087. void* ptr = allocator.root()->Alloc(2048 - kExtraAllocSize, type_name);
  2088. MockPartitionStatsDumper dumper;
  2089. allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
  2090. &dumper);
  2091. EXPECT_TRUE(dumper.IsMemoryAllocationRecorded());
  2092. const PartitionBucketMemoryStats* stats = dumper.GetBucketStats(2048);
  2093. EXPECT_TRUE(stats);
  2094. EXPECT_TRUE(stats->is_valid);
  2095. EXPECT_EQ(2048u, stats->bucket_slot_size);
  2096. EXPECT_EQ(2048u, stats->active_bytes);
  2097. EXPECT_EQ(1u, stats->active_count);
  2098. EXPECT_EQ(SystemPageSize(), stats->resident_bytes);
  2099. EXPECT_EQ(0u, stats->decommittable_bytes);
  2100. EXPECT_EQ(0u, stats->discardable_bytes);
  2101. EXPECT_EQ(0u, stats->num_full_slot_spans);
  2102. EXPECT_EQ(1u, stats->num_active_slot_spans);
  2103. EXPECT_EQ(0u, stats->num_empty_slot_spans);
  2104. EXPECT_EQ(0u, stats->num_decommitted_slot_spans);
  2105. allocator.root()->Free(ptr);
  2106. }
  2107. {
  2108. MockPartitionStatsDumper dumper;
  2109. allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
  2110. &dumper);
  2111. EXPECT_FALSE(dumper.IsMemoryAllocationRecorded());
  2112. const PartitionBucketMemoryStats* stats = dumper.GetBucketStats(2048);
  2113. EXPECT_TRUE(stats);
  2114. EXPECT_TRUE(stats->is_valid);
  2115. EXPECT_EQ(2048u, stats->bucket_slot_size);
  2116. EXPECT_EQ(0u, stats->active_bytes);
  2117. EXPECT_EQ(0u, stats->active_count);
  2118. EXPECT_EQ(SystemPageSize(), stats->resident_bytes);
  2119. EXPECT_EQ(SystemPageSize(), stats->decommittable_bytes);
  2120. EXPECT_EQ(0u, stats->discardable_bytes);
  2121. EXPECT_EQ(0u, stats->num_full_slot_spans);
  2122. EXPECT_EQ(0u, stats->num_active_slot_spans);
  2123. EXPECT_EQ(1u, stats->num_empty_slot_spans);
  2124. EXPECT_EQ(0u, stats->num_decommitted_slot_spans);
  2125. }
  2126. // TODO(crbug.com/722911): Commenting this out causes this test to fail when
  2127. // run singly (--gtest_filter=PartitionAllocTest.DumpMemoryStats), but not
  2128. // when run with the others (--gtest_filter=PartitionAllocTest.*).
  2129. CycleFreeCache(kTestAllocSize);
  2130. {
  2131. MockPartitionStatsDumper dumper;
  2132. allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
  2133. &dumper);
  2134. EXPECT_FALSE(dumper.IsMemoryAllocationRecorded());
  2135. const PartitionBucketMemoryStats* stats = dumper.GetBucketStats(2048);
  2136. EXPECT_TRUE(stats);
  2137. EXPECT_TRUE(stats->is_valid);
  2138. EXPECT_EQ(2048u, stats->bucket_slot_size);
  2139. EXPECT_EQ(0u, stats->active_bytes);
  2140. EXPECT_EQ(0u, stats->active_count);
  2141. EXPECT_EQ(0u, stats->resident_bytes);
  2142. EXPECT_EQ(0u, stats->decommittable_bytes);
  2143. EXPECT_EQ(0u, stats->discardable_bytes);
  2144. EXPECT_EQ(0u, stats->num_full_slot_spans);
  2145. EXPECT_EQ(0u, stats->num_active_slot_spans);
  2146. EXPECT_EQ(0u, stats->num_empty_slot_spans);
  2147. EXPECT_EQ(1u, stats->num_decommitted_slot_spans);
  2148. }
  2149. }
  2150. // This test checks for correct empty slot span list accounting.
  2151. {
  2152. size_t size = PartitionPageSize() - kExtraAllocSize;
  2153. void* ptr1 = allocator.root()->Alloc(size, type_name);
  2154. void* ptr2 = allocator.root()->Alloc(size, type_name);
  2155. allocator.root()->Free(ptr1);
  2156. allocator.root()->Free(ptr2);
  2157. CycleFreeCache(kTestAllocSize);
  2158. ptr1 = allocator.root()->Alloc(size, type_name);
  2159. {
  2160. MockPartitionStatsDumper dumper;
  2161. allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
  2162. &dumper);
  2163. EXPECT_TRUE(dumper.IsMemoryAllocationRecorded());
  2164. const PartitionBucketMemoryStats* stats =
  2165. dumper.GetBucketStats(PartitionPageSize());
  2166. EXPECT_TRUE(stats);
  2167. EXPECT_TRUE(stats->is_valid);
  2168. EXPECT_EQ(PartitionPageSize(), stats->bucket_slot_size);
  2169. EXPECT_EQ(PartitionPageSize(), stats->active_bytes);
  2170. EXPECT_EQ(1u, stats->active_count);
  2171. EXPECT_EQ(PartitionPageSize(), stats->resident_bytes);
  2172. EXPECT_EQ(0u, stats->decommittable_bytes);
  2173. EXPECT_EQ(0u, stats->discardable_bytes);
  2174. EXPECT_EQ(1u, stats->num_full_slot_spans);
  2175. EXPECT_EQ(0u, stats->num_active_slot_spans);
  2176. EXPECT_EQ(0u, stats->num_empty_slot_spans);
  2177. EXPECT_EQ(1u, stats->num_decommitted_slot_spans);
  2178. }
  2179. allocator.root()->Free(ptr1);
  2180. }
  2181. // This test checks for correct direct mapped accounting.
  2182. {
  2183. size_t size_smaller = kMaxBucketed + 1;
  2184. size_t size_bigger = (kMaxBucketed * 2) + 1;
  2185. size_t real_size_smaller =
  2186. (size_smaller + SystemPageOffsetMask()) & SystemPageBaseMask();
  2187. size_t real_size_bigger =
  2188. (size_bigger + SystemPageOffsetMask()) & SystemPageBaseMask();
  2189. void* ptr = allocator.root()->Alloc(size_smaller, type_name);
  2190. void* ptr2 = allocator.root()->Alloc(size_bigger, type_name);
  2191. {
  2192. MockPartitionStatsDumper dumper;
  2193. allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
  2194. &dumper);
  2195. EXPECT_TRUE(dumper.IsMemoryAllocationRecorded());
  2196. const PartitionBucketMemoryStats* stats =
  2197. dumper.GetBucketStats(real_size_smaller);
  2198. EXPECT_TRUE(stats);
  2199. EXPECT_TRUE(stats->is_valid);
  2200. EXPECT_TRUE(stats->is_direct_map);
  2201. EXPECT_EQ(real_size_smaller, stats->bucket_slot_size);
  2202. EXPECT_EQ(real_size_smaller, stats->active_bytes);
  2203. EXPECT_EQ(1u, stats->active_count);
  2204. EXPECT_EQ(real_size_smaller, stats->resident_bytes);
  2205. EXPECT_EQ(0u, stats->decommittable_bytes);
  2206. EXPECT_EQ(0u, stats->discardable_bytes);
  2207. EXPECT_EQ(1u, stats->num_full_slot_spans);
  2208. EXPECT_EQ(0u, stats->num_active_slot_spans);
  2209. EXPECT_EQ(0u, stats->num_empty_slot_spans);
  2210. EXPECT_EQ(0u, stats->num_decommitted_slot_spans);
  2211. stats = dumper.GetBucketStats(real_size_bigger);
  2212. EXPECT_TRUE(stats);
  2213. EXPECT_TRUE(stats->is_valid);
  2214. EXPECT_TRUE(stats->is_direct_map);
  2215. EXPECT_EQ(real_size_bigger, stats->bucket_slot_size);
  2216. EXPECT_EQ(real_size_bigger, stats->active_bytes);
  2217. EXPECT_EQ(1u, stats->active_count);
  2218. EXPECT_EQ(real_size_bigger, stats->resident_bytes);
  2219. EXPECT_EQ(0u, stats->decommittable_bytes);
  2220. EXPECT_EQ(0u, stats->discardable_bytes);
  2221. EXPECT_EQ(1u, stats->num_full_slot_spans);
  2222. EXPECT_EQ(0u, stats->num_active_slot_spans);
  2223. EXPECT_EQ(0u, stats->num_empty_slot_spans);
  2224. EXPECT_EQ(0u, stats->num_decommitted_slot_spans);
  2225. }
  2226. allocator.root()->Free(ptr2);
  2227. allocator.root()->Free(ptr);
  2228. // Whilst we're here, allocate again and free with different ordering to
  2229. // give a workout to our linked list code.
  2230. ptr = allocator.root()->Alloc(size_smaller, type_name);
  2231. ptr2 = allocator.root()->Alloc(size_bigger, type_name);
  2232. allocator.root()->Free(ptr);
  2233. allocator.root()->Free(ptr2);
  2234. }
  2235. // This test checks large-but-not-quite-direct allocations.
  2236. {
  2237. const size_t requested_size = 16 * SystemPageSize();
  2238. void* ptr = allocator.root()->Alloc(requested_size + 1, type_name);
  2239. {
  2240. MockPartitionStatsDumper dumper;
  2241. allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
  2242. &dumper);
  2243. EXPECT_TRUE(dumper.IsMemoryAllocationRecorded());
  2244. size_t slot_size =
  2245. requested_size + (requested_size / kNumBucketsPerOrder);
  2246. const PartitionBucketMemoryStats* stats =
  2247. dumper.GetBucketStats(slot_size);
  2248. ASSERT_TRUE(stats);
  2249. EXPECT_TRUE(stats->is_valid);
  2250. EXPECT_FALSE(stats->is_direct_map);
  2251. EXPECT_EQ(slot_size, stats->bucket_slot_size);
  2252. EXPECT_EQ(requested_size + 1 + kExtraAllocSize, stats->active_bytes);
  2253. EXPECT_EQ(1u, stats->active_count);
  2254. EXPECT_EQ(slot_size, stats->resident_bytes);
  2255. EXPECT_EQ(0u, stats->decommittable_bytes);
  2256. EXPECT_EQ(3 * SystemPageSize(), stats->discardable_bytes);
  2257. EXPECT_EQ(1u, stats->num_full_slot_spans);
  2258. EXPECT_EQ(0u, stats->num_active_slot_spans);
  2259. EXPECT_EQ(0u, stats->num_empty_slot_spans);
  2260. EXPECT_EQ(0u, stats->num_decommitted_slot_spans);
  2261. }
  2262. allocator.root()->Free(ptr);
  2263. {
  2264. MockPartitionStatsDumper dumper;
  2265. allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
  2266. &dumper);
  2267. EXPECT_FALSE(dumper.IsMemoryAllocationRecorded());
  2268. size_t slot_size =
  2269. requested_size + (requested_size / kNumBucketsPerOrder);
  2270. const PartitionBucketMemoryStats* stats =
  2271. dumper.GetBucketStats(slot_size);
  2272. EXPECT_TRUE(stats);
  2273. EXPECT_TRUE(stats->is_valid);
  2274. EXPECT_FALSE(stats->is_direct_map);
  2275. EXPECT_EQ(slot_size, stats->bucket_slot_size);
  2276. EXPECT_EQ(0u, stats->active_bytes);
  2277. EXPECT_EQ(0u, stats->active_count);
  2278. EXPECT_EQ(slot_size, stats->resident_bytes);
  2279. EXPECT_EQ(slot_size, stats->decommittable_bytes);
  2280. EXPECT_EQ(0u, stats->num_full_slot_spans);
  2281. EXPECT_EQ(0u, stats->num_active_slot_spans);
  2282. EXPECT_EQ(1u, stats->num_empty_slot_spans);
  2283. EXPECT_EQ(0u, stats->num_decommitted_slot_spans);
  2284. }
  2285. void* ptr2 = allocator.root()->Alloc(requested_size + SystemPageSize() + 1,
  2286. type_name);
  2287. EXPECT_EQ(ptr, ptr2);
  2288. {
  2289. MockPartitionStatsDumper dumper;
  2290. allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
  2291. &dumper);
  2292. EXPECT_TRUE(dumper.IsMemoryAllocationRecorded());
  2293. size_t slot_size =
  2294. requested_size + (requested_size / kNumBucketsPerOrder);
  2295. const PartitionBucketMemoryStats* stats =
  2296. dumper.GetBucketStats(slot_size);
  2297. EXPECT_TRUE(stats);
  2298. EXPECT_TRUE(stats->is_valid);
  2299. EXPECT_FALSE(stats->is_direct_map);
  2300. EXPECT_EQ(slot_size, stats->bucket_slot_size);
  2301. EXPECT_EQ(requested_size + SystemPageSize() + 1 + kExtraAllocSize,
  2302. stats->active_bytes);
  2303. EXPECT_EQ(1u, stats->active_count);
  2304. EXPECT_EQ(slot_size, stats->resident_bytes);
  2305. EXPECT_EQ(0u, stats->decommittable_bytes);
  2306. EXPECT_EQ(2 * SystemPageSize(), stats->discardable_bytes);
  2307. EXPECT_EQ(1u, stats->num_full_slot_spans);
  2308. EXPECT_EQ(0u, stats->num_active_slot_spans);
  2309. EXPECT_EQ(0u, stats->num_empty_slot_spans);
  2310. EXPECT_EQ(0u, stats->num_decommitted_slot_spans);
  2311. }
  2312. allocator.root()->Free(ptr2);
  2313. }
  2314. }
  2315. // Tests the API to purge freeable memory.
  2316. TEST_P(PartitionAllocTest, Purge) {
  2317. char* ptr = static_cast<char*>(
  2318. allocator.root()->Alloc(2048 - kExtraAllocSize, type_name));
  2319. allocator.root()->Free(ptr);
  2320. {
  2321. MockPartitionStatsDumper dumper;
  2322. allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
  2323. &dumper);
  2324. EXPECT_FALSE(dumper.IsMemoryAllocationRecorded());
  2325. const PartitionBucketMemoryStats* stats = dumper.GetBucketStats(2048);
  2326. EXPECT_TRUE(stats);
  2327. EXPECT_TRUE(stats->is_valid);
  2328. EXPECT_EQ(SystemPageSize(), stats->decommittable_bytes);
  2329. EXPECT_EQ(SystemPageSize(), stats->resident_bytes);
  2330. }
  2331. allocator.root()->PurgeMemory(PurgeFlags::kDecommitEmptySlotSpans);
  2332. {
  2333. MockPartitionStatsDumper dumper;
  2334. allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
  2335. &dumper);
  2336. EXPECT_FALSE(dumper.IsMemoryAllocationRecorded());
  2337. const PartitionBucketMemoryStats* stats = dumper.GetBucketStats(2048);
  2338. EXPECT_TRUE(stats);
  2339. EXPECT_TRUE(stats->is_valid);
  2340. EXPECT_EQ(0u, stats->decommittable_bytes);
  2341. EXPECT_EQ(0u, stats->resident_bytes);
  2342. }
  2343. // Calling purge again here is a good way of testing we didn't mess up the
  2344. // state of the free cache ring.
  2345. allocator.root()->PurgeMemory(PurgeFlags::kDecommitEmptySlotSpans);
  2346. // A single-slot but non-direct-mapped allocation size.
  2347. size_t single_slot_size = 512 * 1024;
  2348. ASSERT_GT(single_slot_size, MaxRegularSlotSpanSize());
  2349. ASSERT_LE(single_slot_size, kMaxBucketed);
  2350. char* big_ptr =
  2351. static_cast<char*>(allocator.root()->Alloc(single_slot_size, type_name));
  2352. allocator.root()->Free(big_ptr);
  2353. allocator.root()->PurgeMemory(PurgeFlags::kDecommitEmptySlotSpans);
  2354. CHECK_PAGE_IN_CORE(ptr - kPointerOffset, false);
  2355. CHECK_PAGE_IN_CORE(big_ptr - kPointerOffset, false);
  2356. }
  2357. // Tests that we prefer to allocate into a non-empty partition page over an
  2358. // empty one. This is an important aspect of minimizing memory usage for some
  2359. // allocation sizes, particularly larger ones.
  2360. TEST_P(PartitionAllocTest, PreferActiveOverEmpty) {
  2361. size_t size = (SystemPageSize() * 2) - kExtraAllocSize;
  2362. // Allocate 3 full slot spans worth of 8192-byte allocations.
  2363. // Each slot span for this size is 16384 bytes, or 1 partition page and 2
  2364. // slots.
  2365. void* ptr1 = allocator.root()->Alloc(size, type_name);
  2366. void* ptr2 = allocator.root()->Alloc(size, type_name);
  2367. void* ptr3 = allocator.root()->Alloc(size, type_name);
  2368. void* ptr4 = allocator.root()->Alloc(size, type_name);
  2369. void* ptr5 = allocator.root()->Alloc(size, type_name);
  2370. void* ptr6 = allocator.root()->Alloc(size, type_name);
  2371. SlotSpanMetadata<internal::ThreadSafe>* slot_span1 =
  2372. SlotSpanMetadata<internal::ThreadSafe>::FromSlotStart(
  2373. allocator.root()->ObjectToSlotStart(ptr1));
  2374. SlotSpanMetadata<internal::ThreadSafe>* slot_span2 =
  2375. SlotSpanMetadata<internal::ThreadSafe>::FromSlotStart(
  2376. allocator.root()->ObjectToSlotStart(ptr3));
  2377. SlotSpanMetadata<internal::ThreadSafe>* slot_span3 =
  2378. SlotSpanMetadata<internal::ThreadSafe>::FromSlotStart(
  2379. allocator.root()->ObjectToSlotStart(ptr6));
  2380. EXPECT_NE(slot_span1, slot_span2);
  2381. EXPECT_NE(slot_span2, slot_span3);
  2382. PartitionBucket<internal::ThreadSafe>* bucket = slot_span1->bucket;
  2383. EXPECT_EQ(slot_span3, bucket->active_slot_spans_head);
  2384. // Free up the 2nd slot in each slot span.
  2385. // This leaves the active list containing 3 slot spans, each with 1 used and 1
  2386. // free slot. The active slot span will be the one containing ptr1.
  2387. allocator.root()->Free(ptr6);
  2388. allocator.root()->Free(ptr4);
  2389. allocator.root()->Free(ptr2);
  2390. EXPECT_EQ(slot_span1, bucket->active_slot_spans_head);
  2391. // Empty the middle slot span in the active list.
  2392. allocator.root()->Free(ptr3);
  2393. EXPECT_EQ(slot_span1, bucket->active_slot_spans_head);
  2394. // Empty the first slot span in the active list -- also the current slot span.
  2395. allocator.root()->Free(ptr1);
  2396. // A good choice here is to re-fill the third slot span since the first two
  2397. // are empty. We used to fail that.
  2398. void* ptr7 = allocator.root()->Alloc(size, type_name);
  2399. PA_EXPECT_PTR_EQ(ptr6, ptr7);
  2400. EXPECT_EQ(slot_span3, bucket->active_slot_spans_head);
  2401. allocator.root()->Free(ptr5);
  2402. allocator.root()->Free(ptr7);
  2403. }
  2404. // Tests the API to purge discardable memory.
  2405. TEST_P(PartitionAllocTest, PurgeDiscardableSecondPage) {
  2406. // Free the second of two 4096 byte allocations and then purge.
  2407. void* ptr1 =
  2408. allocator.root()->Alloc(SystemPageSize() - kExtraAllocSize, type_name);
  2409. char* ptr2 = static_cast<char*>(
  2410. allocator.root()->Alloc(SystemPageSize() - kExtraAllocSize, type_name));
  2411. allocator.root()->Free(ptr2);
  2412. SlotSpanMetadata<internal::ThreadSafe>* slot_span =
  2413. SlotSpanMetadata<internal::ThreadSafe>::FromSlotStart(
  2414. allocator.root()->ObjectToSlotStart(ptr1));
  2415. EXPECT_EQ(2u, slot_span->num_unprovisioned_slots);
  2416. {
  2417. MockPartitionStatsDumper dumper;
  2418. allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
  2419. &dumper);
  2420. EXPECT_TRUE(dumper.IsMemoryAllocationRecorded());
  2421. const PartitionBucketMemoryStats* stats =
  2422. dumper.GetBucketStats(SystemPageSize());
  2423. EXPECT_TRUE(stats);
  2424. EXPECT_TRUE(stats->is_valid);
  2425. EXPECT_EQ(0u, stats->decommittable_bytes);
  2426. EXPECT_EQ(SystemPageSize(), stats->discardable_bytes);
  2427. EXPECT_EQ(SystemPageSize(), stats->active_bytes);
  2428. EXPECT_EQ(2 * SystemPageSize(), stats->resident_bytes);
  2429. }
  2430. CHECK_PAGE_IN_CORE(ptr2 - kPointerOffset, true);
  2431. allocator.root()->PurgeMemory(PurgeFlags::kDiscardUnusedSystemPages);
  2432. CHECK_PAGE_IN_CORE(ptr2 - kPointerOffset, false);
  2433. EXPECT_EQ(3u, slot_span->num_unprovisioned_slots);
  2434. allocator.root()->Free(ptr1);
  2435. }
  2436. TEST_P(PartitionAllocTest, PurgeDiscardableFirstPage) {
  2437. // Free the first of two 4096 byte allocations and then purge.
  2438. char* ptr1 = static_cast<char*>(
  2439. allocator.root()->Alloc(SystemPageSize() - kExtraAllocSize, type_name));
  2440. void* ptr2 =
  2441. allocator.root()->Alloc(SystemPageSize() - kExtraAllocSize, type_name);
  2442. allocator.root()->Free(ptr1);
  2443. {
  2444. MockPartitionStatsDumper dumper;
  2445. allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
  2446. &dumper);
  2447. EXPECT_TRUE(dumper.IsMemoryAllocationRecorded());
  2448. const PartitionBucketMemoryStats* stats =
  2449. dumper.GetBucketStats(SystemPageSize());
  2450. EXPECT_TRUE(stats);
  2451. EXPECT_TRUE(stats->is_valid);
  2452. EXPECT_EQ(0u, stats->decommittable_bytes);
  2453. #if BUILDFLAG(IS_WIN)
  2454. EXPECT_EQ(0u, stats->discardable_bytes);
  2455. #else
  2456. EXPECT_EQ(SystemPageSize(), stats->discardable_bytes);
  2457. #endif
  2458. EXPECT_EQ(SystemPageSize(), stats->active_bytes);
  2459. EXPECT_EQ(2 * SystemPageSize(), stats->resident_bytes);
  2460. }
  2461. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset, true);
  2462. allocator.root()->PurgeMemory(PurgeFlags::kDiscardUnusedSystemPages);
  2463. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset, false);
  2464. allocator.root()->Free(ptr2);
  2465. }
  2466. TEST_P(PartitionAllocTest, PurgeDiscardableNonPageSizedAlloc) {
  2467. const size_t requested_size = 2.5 * SystemPageSize();
  2468. char* ptr1 = static_cast<char*>(
  2469. allocator.root()->Alloc(requested_size - kExtraAllocSize, type_name));
  2470. void* ptr2 =
  2471. allocator.root()->Alloc(requested_size - kExtraAllocSize, type_name);
  2472. void* ptr3 =
  2473. allocator.root()->Alloc(requested_size - kExtraAllocSize, type_name);
  2474. void* ptr4 =
  2475. allocator.root()->Alloc(requested_size - kExtraAllocSize, type_name);
  2476. memset(ptr1, 'A', requested_size - kExtraAllocSize);
  2477. memset(ptr2, 'A', requested_size - kExtraAllocSize);
  2478. allocator.root()->Free(ptr1);
  2479. allocator.root()->Free(ptr2);
  2480. {
  2481. MockPartitionStatsDumper dumper;
  2482. allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
  2483. &dumper);
  2484. EXPECT_TRUE(dumper.IsMemoryAllocationRecorded());
  2485. const PartitionBucketMemoryStats* stats =
  2486. dumper.GetBucketStats(requested_size);
  2487. EXPECT_TRUE(stats);
  2488. EXPECT_TRUE(stats->is_valid);
  2489. EXPECT_EQ(0u, stats->decommittable_bytes);
  2490. #if BUILDFLAG(IS_WIN)
  2491. EXPECT_EQ(3 * SystemPageSize(), stats->discardable_bytes);
  2492. #else
  2493. EXPECT_EQ(4 * SystemPageSize(), stats->discardable_bytes);
  2494. #endif
  2495. EXPECT_EQ(requested_size * 2, stats->active_bytes);
  2496. EXPECT_EQ(10 * SystemPageSize(), stats->resident_bytes);
  2497. }
  2498. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset, true);
  2499. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + SystemPageSize(), true);
  2500. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 2), true);
  2501. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 3), true);
  2502. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 4), true);
  2503. allocator.root()->PurgeMemory(PurgeFlags::kDiscardUnusedSystemPages);
  2504. // Except for Windows, the first page is discardable because the freelist
  2505. // pointer on this page is nullptr. Note that CHECK_PAGE_IN_CORE only executes
  2506. // checks for Linux and ChromeOS, not for Windows.
  2507. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset, false);
  2508. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + SystemPageSize(), false);
  2509. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 2), true);
  2510. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 3), false);
  2511. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 4), false);
  2512. allocator.root()->Free(ptr3);
  2513. allocator.root()->Free(ptr4);
  2514. }
  2515. TEST_P(PartitionAllocTest, PurgeDiscardableNonPageSizedAllocOnSlotBoundary) {
  2516. const size_t requested_size = 2.5 * SystemPageSize();
  2517. char* ptr1 = static_cast<char*>(
  2518. allocator.root()->Alloc(requested_size - kExtraAllocSize, type_name));
  2519. void* ptr2 =
  2520. allocator.root()->Alloc(requested_size - kExtraAllocSize, type_name);
  2521. void* ptr3 =
  2522. allocator.root()->Alloc(requested_size - kExtraAllocSize, type_name);
  2523. void* ptr4 =
  2524. allocator.root()->Alloc(requested_size - kExtraAllocSize, type_name);
  2525. memset(ptr1, 'A', requested_size - kExtraAllocSize);
  2526. memset(ptr2, 'A', requested_size - kExtraAllocSize);
  2527. allocator.root()->Free(ptr2);
  2528. allocator.root()->Free(ptr1);
  2529. {
  2530. MockPartitionStatsDumper dumper;
  2531. allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
  2532. &dumper);
  2533. EXPECT_TRUE(dumper.IsMemoryAllocationRecorded());
  2534. const PartitionBucketMemoryStats* stats =
  2535. dumper.GetBucketStats(requested_size);
  2536. EXPECT_TRUE(stats);
  2537. EXPECT_TRUE(stats->is_valid);
  2538. EXPECT_EQ(0u, stats->decommittable_bytes);
  2539. #if BUILDFLAG(IS_WIN)
  2540. EXPECT_EQ(3 * SystemPageSize(), stats->discardable_bytes);
  2541. #else
  2542. EXPECT_EQ(4 * SystemPageSize(), stats->discardable_bytes);
  2543. #endif
  2544. EXPECT_EQ(requested_size * 2, stats->active_bytes);
  2545. EXPECT_EQ(10 * SystemPageSize(), stats->resident_bytes);
  2546. }
  2547. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset, true);
  2548. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + SystemPageSize(), true);
  2549. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 2), true);
  2550. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 3), true);
  2551. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 4), true);
  2552. allocator.root()->PurgeMemory(PurgeFlags::kDiscardUnusedSystemPages);
  2553. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset, true);
  2554. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + SystemPageSize(), false);
  2555. // Except for Windows, the third page is discardable because the freelist
  2556. // pointer on this page is nullptr. Note that CHECK_PAGE_IN_CORE only executes
  2557. // checks for Linux and ChromeOS, not for Windows.
  2558. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 2), false);
  2559. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 3), false);
  2560. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 4), false);
  2561. allocator.root()->Free(ptr3);
  2562. allocator.root()->Free(ptr4);
  2563. }
  2564. TEST_P(PartitionAllocTest, PurgeDiscardableManyPages) {
  2565. // On systems with large pages, use less pages because:
  2566. // 1) There must be a bucket for kFirstAllocPages * SystemPageSize(), and
  2567. // 2) On low-end systems, using too many large pages can OOM during the test
  2568. const bool kHasLargePages = SystemPageSize() > 4096;
  2569. const size_t kFirstAllocPages = kHasLargePages ? 32 : 64;
  2570. const size_t kSecondAllocPages = kHasLargePages ? 31 : 61;
  2571. // Detect case (1) from above.
  2572. PA_DCHECK(kFirstAllocPages * SystemPageSize() < (1UL << kMaxBucketedOrder));
  2573. const size_t kDeltaPages = kFirstAllocPages - kSecondAllocPages;
  2574. {
  2575. ScopedPageAllocation p(allocator, kFirstAllocPages);
  2576. p.TouchAllPages();
  2577. }
  2578. ScopedPageAllocation p(allocator, kSecondAllocPages);
  2579. MockPartitionStatsDumper dumper;
  2580. allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
  2581. &dumper);
  2582. EXPECT_TRUE(dumper.IsMemoryAllocationRecorded());
  2583. const PartitionBucketMemoryStats* stats =
  2584. dumper.GetBucketStats(kFirstAllocPages * SystemPageSize());
  2585. EXPECT_TRUE(stats);
  2586. EXPECT_TRUE(stats->is_valid);
  2587. EXPECT_EQ(0u, stats->decommittable_bytes);
  2588. EXPECT_EQ(kDeltaPages * SystemPageSize(), stats->discardable_bytes);
  2589. EXPECT_EQ(kSecondAllocPages * SystemPageSize(), stats->active_bytes);
  2590. EXPECT_EQ(kFirstAllocPages * SystemPageSize(), stats->resident_bytes);
  2591. for (size_t i = 0; i < kFirstAllocPages; i++)
  2592. CHECK_PAGE_IN_CORE(p.PageAtIndex(i), true);
  2593. allocator.root()->PurgeMemory(PurgeFlags::kDiscardUnusedSystemPages);
  2594. for (size_t i = 0; i < kSecondAllocPages; i++)
  2595. CHECK_PAGE_IN_CORE(p.PageAtIndex(i), true);
  2596. for (size_t i = kSecondAllocPages; i < kFirstAllocPages; i++)
  2597. CHECK_PAGE_IN_CORE(p.PageAtIndex(i), false);
  2598. }
  2599. TEST_P(PartitionAllocTest, PurgeDiscardableWithFreeListRewrite) {
  2600. // This sub-test tests truncation of the provisioned slots in a trickier
  2601. // case where the freelist is rewritten.
  2602. allocator.root()->PurgeMemory(PurgeFlags::kDecommitEmptySlotSpans);
  2603. char* ptr1 = static_cast<char*>(
  2604. allocator.root()->Alloc(SystemPageSize() - kExtraAllocSize, type_name));
  2605. void* ptr2 =
  2606. allocator.root()->Alloc(SystemPageSize() - kExtraAllocSize, type_name);
  2607. void* ptr3 =
  2608. allocator.root()->Alloc(SystemPageSize() - kExtraAllocSize, type_name);
  2609. void* ptr4 =
  2610. allocator.root()->Alloc(SystemPageSize() - kExtraAllocSize, type_name);
  2611. ptr1[0] = 'A';
  2612. ptr1[SystemPageSize()] = 'A';
  2613. ptr1[SystemPageSize() * 2] = 'A';
  2614. ptr1[SystemPageSize() * 3] = 'A';
  2615. SlotSpanMetadata<internal::ThreadSafe>* slot_span =
  2616. SlotSpanMetadata<internal::ThreadSafe>::FromSlotStart(
  2617. allocator.root()->ObjectToSlotStart(ptr1));
  2618. allocator.root()->Free(ptr2);
  2619. allocator.root()->Free(ptr4);
  2620. allocator.root()->Free(ptr1);
  2621. EXPECT_EQ(0u, slot_span->num_unprovisioned_slots);
  2622. {
  2623. MockPartitionStatsDumper dumper;
  2624. allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
  2625. &dumper);
  2626. EXPECT_TRUE(dumper.IsMemoryAllocationRecorded());
  2627. const PartitionBucketMemoryStats* stats =
  2628. dumper.GetBucketStats(SystemPageSize());
  2629. EXPECT_TRUE(stats);
  2630. EXPECT_TRUE(stats->is_valid);
  2631. EXPECT_EQ(0u, stats->decommittable_bytes);
  2632. #if BUILDFLAG(IS_WIN)
  2633. EXPECT_EQ(SystemPageSize(), stats->discardable_bytes);
  2634. #else
  2635. EXPECT_EQ(2 * SystemPageSize(), stats->discardable_bytes);
  2636. #endif
  2637. EXPECT_EQ(SystemPageSize(), stats->active_bytes);
  2638. EXPECT_EQ(4 * SystemPageSize(), stats->resident_bytes);
  2639. }
  2640. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset, true);
  2641. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + SystemPageSize(), true);
  2642. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 2), true);
  2643. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 3), true);
  2644. allocator.root()->PurgeMemory(PurgeFlags::kDiscardUnusedSystemPages);
  2645. EXPECT_EQ(1u, slot_span->num_unprovisioned_slots);
  2646. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset, true);
  2647. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + SystemPageSize(), false);
  2648. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 2), true);
  2649. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 3), false);
  2650. // Let's check we didn't brick the freelist.
  2651. void* ptr1b =
  2652. allocator.root()->Alloc(SystemPageSize() - kExtraAllocSize, type_name);
  2653. PA_EXPECT_PTR_EQ(ptr1, ptr1b);
  2654. void* ptr2b =
  2655. allocator.root()->Alloc(SystemPageSize() - kExtraAllocSize, type_name);
  2656. PA_EXPECT_PTR_EQ(ptr2, ptr2b);
  2657. EXPECT_FALSE(slot_span->get_freelist_head());
  2658. allocator.root()->Free(ptr1);
  2659. allocator.root()->Free(ptr2);
  2660. allocator.root()->Free(ptr3);
  2661. }
  2662. TEST_P(PartitionAllocTest, PurgeDiscardableDoubleTruncateFreeList) {
  2663. // This sub-test is similar, but tests a double-truncation.
  2664. allocator.root()->PurgeMemory(PurgeFlags::kDecommitEmptySlotSpans);
  2665. char* ptr1 = static_cast<char*>(
  2666. allocator.root()->Alloc(SystemPageSize() - kExtraAllocSize, type_name));
  2667. void* ptr2 =
  2668. allocator.root()->Alloc(SystemPageSize() - kExtraAllocSize, type_name);
  2669. void* ptr3 =
  2670. allocator.root()->Alloc(SystemPageSize() - kExtraAllocSize, type_name);
  2671. void* ptr4 =
  2672. allocator.root()->Alloc(SystemPageSize() - kExtraAllocSize, type_name);
  2673. ptr1[0] = 'A';
  2674. ptr1[SystemPageSize()] = 'A';
  2675. ptr1[SystemPageSize() * 2] = 'A';
  2676. ptr1[SystemPageSize() * 3] = 'A';
  2677. SlotSpanMetadata<internal::ThreadSafe>* slot_span =
  2678. SlotSpanMetadata<internal::ThreadSafe>::FromSlotStart(
  2679. allocator.root()->ObjectToSlotStart(ptr1));
  2680. allocator.root()->Free(ptr4);
  2681. allocator.root()->Free(ptr3);
  2682. EXPECT_EQ(0u, slot_span->num_unprovisioned_slots);
  2683. {
  2684. MockPartitionStatsDumper dumper;
  2685. allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
  2686. &dumper);
  2687. EXPECT_TRUE(dumper.IsMemoryAllocationRecorded());
  2688. const PartitionBucketMemoryStats* stats =
  2689. dumper.GetBucketStats(SystemPageSize());
  2690. EXPECT_TRUE(stats);
  2691. EXPECT_TRUE(stats->is_valid);
  2692. EXPECT_EQ(0u, stats->decommittable_bytes);
  2693. EXPECT_EQ(2 * SystemPageSize(), stats->discardable_bytes);
  2694. EXPECT_EQ(2 * SystemPageSize(), stats->active_bytes);
  2695. EXPECT_EQ(4 * SystemPageSize(), stats->resident_bytes);
  2696. }
  2697. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset, true);
  2698. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + SystemPageSize(), true);
  2699. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 2), true);
  2700. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 3), true);
  2701. allocator.root()->PurgeMemory(PurgeFlags::kDiscardUnusedSystemPages);
  2702. EXPECT_EQ(2u, slot_span->num_unprovisioned_slots);
  2703. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset, true);
  2704. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + SystemPageSize(), true);
  2705. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 2), false);
  2706. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + (SystemPageSize() * 3), false);
  2707. EXPECT_FALSE(slot_span->get_freelist_head());
  2708. allocator.root()->Free(ptr1);
  2709. allocator.root()->Free(ptr2);
  2710. }
  2711. TEST_P(PartitionAllocTest, PurgeDiscardableSmallSlotsWithTruncate) {
  2712. size_t requested_size = 0.5 * SystemPageSize();
  2713. char* ptr1 = static_cast<char*>(
  2714. allocator.root()->Alloc(requested_size - kExtraAllocSize, type_name));
  2715. void* ptr2 =
  2716. allocator.root()->Alloc(requested_size - kExtraAllocSize, type_name);
  2717. void* ptr3 =
  2718. allocator.root()->Alloc(requested_size - kExtraAllocSize, type_name);
  2719. void* ptr4 =
  2720. allocator.root()->Alloc(requested_size - kExtraAllocSize, type_name);
  2721. allocator.root()->Free(ptr3);
  2722. allocator.root()->Free(ptr4);
  2723. SlotSpanMetadata<internal::ThreadSafe>* slot_span =
  2724. SlotSpanMetadata<internal::ThreadSafe>::FromSlotStart(
  2725. allocator.root()->ObjectToSlotStart(ptr1));
  2726. EXPECT_EQ(4u, slot_span->num_unprovisioned_slots);
  2727. {
  2728. MockPartitionStatsDumper dumper;
  2729. allocator.root()->DumpStats("mock_allocator", false /* detailed dump */,
  2730. &dumper);
  2731. EXPECT_TRUE(dumper.IsMemoryAllocationRecorded());
  2732. const PartitionBucketMemoryStats* stats =
  2733. dumper.GetBucketStats(requested_size);
  2734. EXPECT_TRUE(stats);
  2735. EXPECT_TRUE(stats->is_valid);
  2736. EXPECT_EQ(0u, stats->decommittable_bytes);
  2737. EXPECT_EQ(SystemPageSize(), stats->discardable_bytes);
  2738. EXPECT_EQ(requested_size * 2, stats->active_bytes);
  2739. EXPECT_EQ(2 * SystemPageSize(), stats->resident_bytes);
  2740. }
  2741. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset, true);
  2742. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + SystemPageSize(), true);
  2743. allocator.root()->PurgeMemory(PurgeFlags::kDiscardUnusedSystemPages);
  2744. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset, true);
  2745. CHECK_PAGE_IN_CORE(ptr1 - kPointerOffset + SystemPageSize(), false);
  2746. EXPECT_EQ(6u, slot_span->num_unprovisioned_slots);
  2747. allocator.root()->Free(ptr1);
  2748. allocator.root()->Free(ptr2);
  2749. }
  2750. TEST_P(PartitionAllocTest, ActiveListMaintenance) {
  2751. size_t size = SystemPageSize() - kExtraAllocSize;
  2752. size_t real_size = size + kExtraAllocSize;
  2753. size_t bucket_index =
  2754. allocator.root()->SizeToBucketIndex(real_size, GetParam());
  2755. PartitionRoot<ThreadSafe>::Bucket* bucket =
  2756. &allocator.root()->buckets[bucket_index];
  2757. ASSERT_EQ(bucket->slot_size, real_size);
  2758. size_t slots_per_span = bucket->num_system_pages_per_slot_span;
  2759. // Make 10 full slot spans.
  2760. constexpr int kSpans = 10;
  2761. std::vector<std::vector<void*>> allocated_memory_spans(kSpans);
  2762. for (int span_index = 0; span_index < kSpans; span_index++) {
  2763. for (size_t i = 0; i < slots_per_span; i++) {
  2764. allocated_memory_spans[span_index].push_back(
  2765. allocator.root()->Alloc(size, ""));
  2766. }
  2767. }
  2768. // Free one entry in the middle span, creating a partial slot span.
  2769. constexpr size_t kSpanIndex = 5;
  2770. allocator.root()->Free(allocated_memory_spans[kSpanIndex].back());
  2771. allocated_memory_spans[kSpanIndex].pop_back();
  2772. // Empty the last slot span.
  2773. for (void* ptr : allocated_memory_spans[kSpans - 1])
  2774. allocator.root()->Free(ptr);
  2775. allocated_memory_spans.pop_back();
  2776. // The active list now is:
  2777. // Partial -> Empty -> Full -> Full -> ... -> Full
  2778. bucket->MaintainActiveList();
  2779. // Only one entry in the active list.
  2780. ASSERT_NE(bucket->active_slot_spans_head,
  2781. SlotSpanMetadata<ThreadSafe>::get_sentinel_slot_span());
  2782. EXPECT_FALSE(bucket->active_slot_spans_head->next_slot_span);
  2783. // The empty list has 1 entry.
  2784. ASSERT_NE(bucket->empty_slot_spans_head,
  2785. SlotSpanMetadata<ThreadSafe>::get_sentinel_slot_span());
  2786. EXPECT_FALSE(bucket->empty_slot_spans_head->next_slot_span);
  2787. // The rest are full slot spans.
  2788. EXPECT_EQ(8u, bucket->num_full_slot_spans);
  2789. // Free all memory.
  2790. for (const auto& span : allocated_memory_spans) {
  2791. for (void* ptr : span)
  2792. allocator.root()->Free(ptr);
  2793. }
  2794. }
  2795. TEST_P(PartitionAllocTest, ReallocMovesCookie) {
  2796. // Resize so as to be sure to hit a "resize in place" case, and ensure that
  2797. // use of the entire result is compatible with the debug mode's cookie, even
  2798. // when the bucket size is large enough to span more than one partition page
  2799. // and we can track the "raw" size. See https://crbug.com/709271
  2800. static const size_t kSize = MaxRegularSlotSpanSize();
  2801. void* ptr = allocator.root()->Alloc(kSize + 1, type_name);
  2802. EXPECT_TRUE(ptr);
  2803. memset(ptr, 0xbd, kSize + 1);
  2804. ptr = allocator.root()->Realloc(ptr, kSize + 2, type_name);
  2805. EXPECT_TRUE(ptr);
  2806. memset(ptr, 0xbd, kSize + 2);
  2807. allocator.root()->Free(ptr);
  2808. }
  2809. TEST_P(PartitionAllocTest, SmallReallocDoesNotMoveTrailingCookie) {
  2810. // For crbug.com/781473
  2811. static constexpr size_t kSize = 264;
  2812. void* ptr = allocator.root()->Alloc(kSize, type_name);
  2813. EXPECT_TRUE(ptr);
  2814. ptr = allocator.root()->Realloc(ptr, kSize + 16, type_name);
  2815. EXPECT_TRUE(ptr);
  2816. allocator.root()->Free(ptr);
  2817. }
  2818. TEST_P(PartitionAllocTest, ZeroFill) {
  2819. constexpr static size_t kAllZerosSentinel =
  2820. std::numeric_limits<size_t>::max();
  2821. for (size_t size : kTestSizes) {
  2822. char* p = static_cast<char*>(
  2823. allocator.root()->AllocWithFlags(AllocFlags::kZeroFill, size, nullptr));
  2824. size_t non_zero_position = kAllZerosSentinel;
  2825. for (size_t i = 0; i < size; ++i) {
  2826. if (0 != p[i]) {
  2827. non_zero_position = i;
  2828. break;
  2829. }
  2830. }
  2831. EXPECT_EQ(kAllZerosSentinel, non_zero_position)
  2832. << "test allocation size: " << size;
  2833. allocator.root()->Free(p);
  2834. }
  2835. for (int i = 0; i < 10; ++i) {
  2836. SCOPED_TRACE(i);
  2837. AllocateRandomly(allocator.root(), 250, AllocFlags::kZeroFill);
  2838. }
  2839. }
  2840. TEST_P(PartitionAllocTest, Bug_897585) {
  2841. // Need sizes big enough to be direct mapped and a delta small enough to
  2842. // allow re-use of the slot span when cookied. These numbers fall out of the
  2843. // test case in the indicated bug.
  2844. size_t kInitialSize = 983040;
  2845. size_t kDesiredSize = 983100;
  2846. ASSERT_GT(kInitialSize, kMaxBucketed);
  2847. ASSERT_GT(kDesiredSize, kMaxBucketed);
  2848. void* ptr = allocator.root()->AllocWithFlags(AllocFlags::kReturnNull,
  2849. kInitialSize, nullptr);
  2850. ASSERT_NE(nullptr, ptr);
  2851. ptr = allocator.root()->ReallocWithFlags(AllocFlags::kReturnNull, ptr,
  2852. kDesiredSize, nullptr);
  2853. ASSERT_NE(nullptr, ptr);
  2854. memset(ptr, 0xbd, kDesiredSize);
  2855. allocator.root()->Free(ptr);
  2856. }
  2857. TEST_P(PartitionAllocTest, OverrideHooks) {
  2858. constexpr size_t kOverriddenSize = 1234;
  2859. constexpr const char* kOverriddenType = "Overridden type";
  2860. constexpr unsigned char kOverriddenChar = 'A';
  2861. // Marked static so that we can use them in non-capturing lambdas below.
  2862. // (Non-capturing lambdas convert directly to function pointers.)
  2863. static volatile bool free_called = false;
  2864. static void* overridden_allocation = nullptr;
  2865. overridden_allocation = malloc(kOverriddenSize);
  2866. memset(overridden_allocation, kOverriddenChar, kOverriddenSize);
  2867. PartitionAllocHooks::SetOverrideHooks(
  2868. [](void** out, unsigned int flags, size_t size,
  2869. const char* type_name) -> bool {
  2870. if (size == kOverriddenSize && type_name == kOverriddenType) {
  2871. *out = overridden_allocation;
  2872. return true;
  2873. }
  2874. return false;
  2875. },
  2876. [](void* address) -> bool {
  2877. if (address == overridden_allocation) {
  2878. free_called = true;
  2879. return true;
  2880. }
  2881. return false;
  2882. },
  2883. [](size_t* out, void* address) -> bool {
  2884. if (address == overridden_allocation) {
  2885. *out = kOverriddenSize;
  2886. return true;
  2887. }
  2888. return false;
  2889. });
  2890. void* ptr = allocator.root()->AllocWithFlags(
  2891. AllocFlags::kReturnNull, kOverriddenSize, kOverriddenType);
  2892. ASSERT_EQ(ptr, overridden_allocation);
  2893. allocator.root()->Free(ptr);
  2894. EXPECT_TRUE(free_called);
  2895. // overridden_allocation has not actually been freed so we can now immediately
  2896. // realloc it.
  2897. free_called = false;
  2898. ptr = allocator.root()->ReallocWithFlags(AllocFlags::kReturnNull, ptr, 1,
  2899. nullptr);
  2900. ASSERT_NE(ptr, nullptr);
  2901. EXPECT_NE(ptr, overridden_allocation);
  2902. EXPECT_TRUE(free_called);
  2903. EXPECT_EQ(*(char*)ptr, kOverriddenChar);
  2904. allocator.root()->Free(ptr);
  2905. PartitionAllocHooks::SetOverrideHooks(nullptr, nullptr, nullptr);
  2906. free(overridden_allocation);
  2907. }
  2908. TEST_P(PartitionAllocTest, Alignment) {
  2909. std::vector<void*> allocated_ptrs;
  2910. for (size_t size = 1; size <= PartitionPageSize(); size <<= 1) {
  2911. if (size <= kExtraAllocSize)
  2912. continue;
  2913. size_t requested_size = size - kExtraAllocSize;
  2914. // All allocations which are not direct-mapped occupy contiguous slots of a
  2915. // span, starting on a page boundary. This means that allocations are first
  2916. // rounded up to the nearest bucket size, then have an address of the form:
  2917. // (partition-page-aligned address) + i * bucket_size.
  2918. //
  2919. // All powers of two are bucket sizes, meaning that all power of two
  2920. // allocations smaller than a page will be aligned on the allocation size.
  2921. size_t expected_alignment = size;
  2922. for (int index = 0; index < 3; index++) {
  2923. void* ptr = allocator.root()->Alloc(requested_size, "");
  2924. allocated_ptrs.push_back(ptr);
  2925. EXPECT_EQ(0u,
  2926. allocator.root()->ObjectToSlotStart(ptr) % expected_alignment)
  2927. << (index + 1) << "-th allocation of size=" << size;
  2928. }
  2929. }
  2930. for (void* ptr : allocated_ptrs)
  2931. allocator.root()->Free(ptr);
  2932. }
  2933. TEST_P(PartitionAllocTest, FundamentalAlignment) {
  2934. // See the test above for details. Essentially, checking the bucket size is
  2935. // sufficient to ensure that alignment will always be respected, as long as
  2936. // the fundamental alignment is <= 16 bytes.
  2937. size_t fundamental_alignment = kAlignment;
  2938. for (size_t size = 0; size < SystemPageSize(); size++) {
  2939. // Allocate several pointers, as the first one in use in a size class will
  2940. // be aligned on a page boundary.
  2941. void* ptr = allocator.root()->Alloc(size, "");
  2942. void* ptr2 = allocator.root()->Alloc(size, "");
  2943. void* ptr3 = allocator.root()->Alloc(size, "");
  2944. EXPECT_EQ(UntagPtr(ptr) % fundamental_alignment, 0u);
  2945. EXPECT_EQ(UntagPtr(ptr2) % fundamental_alignment, 0u);
  2946. EXPECT_EQ(UntagPtr(ptr3) % fundamental_alignment, 0u);
  2947. uintptr_t slot_start = allocator.root()->ObjectToSlotStart(ptr);
  2948. #if BUILDFLAG(PUT_REF_COUNT_IN_PREVIOUS_SLOT)
  2949. // The capacity(C) is slot size - kExtraAllocSize.
  2950. // Since slot size is multiples of kAlignment,
  2951. // C % kAlignment == (slot_size - kExtraAllocSize) % kAlignment.
  2952. // C % kAlignment == (-kExtraAllocSize) % kAlignment.
  2953. // Since kCookieSize is a multiple of kAlignment,
  2954. // C % kAlignment == (-kInSlotRefCountBufferSize) % kAlignment
  2955. // == (kAlignment - kInSlotRefCountBufferSize) % kAlignment.
  2956. EXPECT_EQ(allocator.root()->AllocationCapacityFromSlotStart(slot_start) %
  2957. fundamental_alignment,
  2958. fundamental_alignment - kInSlotRefCountBufferSize);
  2959. #else
  2960. EXPECT_EQ(allocator.root()->AllocationCapacityFromSlotStart(slot_start) %
  2961. fundamental_alignment,
  2962. 0u);
  2963. #endif
  2964. allocator.root()->Free(ptr);
  2965. allocator.root()->Free(ptr2);
  2966. allocator.root()->Free(ptr3);
  2967. }
  2968. }
  2969. void VerifyAlignment(PartitionRoot<ThreadSafe>* root,
  2970. size_t size,
  2971. size_t alignment) {
  2972. std::vector<void*> allocated_ptrs;
  2973. for (int index = 0; index < 3; index++) {
  2974. void* ptr = root->AlignedAllocWithFlags(0, alignment, size);
  2975. ASSERT_TRUE(ptr);
  2976. allocated_ptrs.push_back(ptr);
  2977. EXPECT_EQ(0ull, UntagPtr(ptr) % alignment)
  2978. << (index + 1) << "-th allocation of size=" << size
  2979. << ", alignment=" << alignment;
  2980. }
  2981. for (void* ptr : allocated_ptrs)
  2982. PartitionRoot<ThreadSafe>::Free(ptr);
  2983. }
  2984. TEST_P(PartitionAllocTest, AlignedAllocations) {
  2985. size_t alloc_sizes[] = {1,
  2986. 10,
  2987. 100,
  2988. 1000,
  2989. 10000,
  2990. 60000,
  2991. 70000,
  2992. 130000,
  2993. 500000,
  2994. 900000,
  2995. kMaxBucketed + 1,
  2996. 2 * kMaxBucketed,
  2997. kSuperPageSize - 2 * PartitionPageSize(),
  2998. 4 * kMaxBucketed};
  2999. for (size_t alloc_size : alloc_sizes) {
  3000. for (size_t alignment = 1; alignment <= kMaxSupportedAlignment;
  3001. alignment <<= 1) {
  3002. VerifyAlignment(aligned_allocator.root(), alloc_size, alignment);
  3003. // Verify alignment on the regular allocator only when BRP is off, or when
  3004. // it's on in the "previous slot" mode. See the comment in SetUp().
  3005. #if !BUILDFLAG(USE_BACKUP_REF_PTR) || BUILDFLAG(PUT_REF_COUNT_IN_PREVIOUS_SLOT)
  3006. VerifyAlignment(allocator.root(), alloc_size, alignment);
  3007. #endif
  3008. }
  3009. }
  3010. }
  3011. // Test that the optimized `GetSlotNumber` implementation produces valid
  3012. // results.
  3013. TEST_P(PartitionAllocTest, OptimizedGetSlotNumber) {
  3014. for (size_t i = 0; i < kNumBuckets; ++i) {
  3015. auto& bucket = allocator.root()->buckets[i];
  3016. if (SizeToIndex(bucket.slot_size) != i)
  3017. continue;
  3018. for (size_t slot = 0, offset = 0; slot < bucket.get_slots_per_span();
  3019. ++slot, offset += bucket.slot_size) {
  3020. EXPECT_EQ(slot, bucket.GetSlotNumber(offset));
  3021. EXPECT_EQ(slot, bucket.GetSlotNumber(offset + bucket.slot_size / 2));
  3022. EXPECT_EQ(slot, bucket.GetSlotNumber(offset + bucket.slot_size - 1));
  3023. }
  3024. }
  3025. }
  3026. TEST_P(PartitionAllocTest, GetUsableSizeNull) {
  3027. EXPECT_EQ(0ULL, PartitionRoot<ThreadSafe>::GetUsableSize(nullptr));
  3028. }
  3029. TEST_P(PartitionAllocTest, GetUsableSize) {
  3030. size_t delta = SystemPageSize() + 1;
  3031. for (size_t size = 1; size <= kMinDirectMappedDownsize; size += delta) {
  3032. void* ptr = allocator.root()->Alloc(size, "");
  3033. EXPECT_TRUE(ptr);
  3034. size_t usable_size = PartitionRoot<ThreadSafe>::GetUsableSize(ptr);
  3035. EXPECT_LE(size, usable_size);
  3036. memset(ptr, 0xDE, usable_size);
  3037. // Should not crash when free the ptr.
  3038. allocator.root()->Free(ptr);
  3039. }
  3040. }
  3041. TEST_P(PartitionAllocTest, Bookkeeping) {
  3042. auto& root = *allocator.root();
  3043. EXPECT_EQ(0U, root.total_size_of_committed_pages);
  3044. EXPECT_EQ(0U, root.max_size_of_committed_pages);
  3045. EXPECT_EQ(0U, root.get_total_size_of_allocated_bytes());
  3046. EXPECT_EQ(0U, root.get_max_size_of_allocated_bytes());
  3047. EXPECT_EQ(0U, root.total_size_of_super_pages);
  3048. size_t small_size = 1000;
  3049. // A full slot span of size 1 partition page is committed.
  3050. void* ptr = root.Alloc(small_size - kExtraAllocSize, type_name);
  3051. // Lazy commit commits only needed pages.
  3052. size_t expected_committed_size =
  3053. kUseLazyCommit ? SystemPageSize() : PartitionPageSize();
  3054. size_t expected_super_pages_size = kSuperPageSize;
  3055. size_t expected_max_committed_size = expected_committed_size;
  3056. size_t bucket_index = SizeToIndex(small_size - kExtraAllocSize);
  3057. PartitionBucket<internal::ThreadSafe>* bucket = &root.buckets[bucket_index];
  3058. size_t expected_total_allocated_size = bucket->slot_size;
  3059. size_t expected_max_allocated_size = expected_total_allocated_size;
  3060. EXPECT_EQ(expected_committed_size, root.total_size_of_committed_pages);
  3061. EXPECT_EQ(expected_max_committed_size, root.max_size_of_committed_pages);
  3062. EXPECT_EQ(expected_total_allocated_size,
  3063. root.get_total_size_of_allocated_bytes());
  3064. EXPECT_EQ(expected_max_allocated_size,
  3065. root.get_max_size_of_allocated_bytes());
  3066. EXPECT_EQ(expected_super_pages_size, root.total_size_of_super_pages);
  3067. // Freeing memory doesn't result in decommitting pages right away.
  3068. root.Free(ptr);
  3069. expected_total_allocated_size = 0U;
  3070. EXPECT_EQ(expected_committed_size, root.total_size_of_committed_pages);
  3071. EXPECT_EQ(expected_max_committed_size, root.max_size_of_committed_pages);
  3072. EXPECT_EQ(expected_total_allocated_size,
  3073. root.get_total_size_of_allocated_bytes());
  3074. EXPECT_EQ(expected_max_allocated_size,
  3075. root.get_max_size_of_allocated_bytes());
  3076. EXPECT_EQ(expected_super_pages_size, root.total_size_of_super_pages);
  3077. // Allocating the same size lands it in the same slot span.
  3078. ptr = root.Alloc(small_size - kExtraAllocSize, type_name);
  3079. EXPECT_EQ(expected_committed_size, root.total_size_of_committed_pages);
  3080. EXPECT_EQ(expected_max_committed_size, root.max_size_of_committed_pages);
  3081. EXPECT_EQ(expected_max_allocated_size,
  3082. root.get_max_size_of_allocated_bytes());
  3083. EXPECT_EQ(expected_super_pages_size, root.total_size_of_super_pages);
  3084. // Freeing memory doesn't result in decommitting pages right away.
  3085. root.Free(ptr);
  3086. EXPECT_EQ(expected_committed_size, root.total_size_of_committed_pages);
  3087. EXPECT_EQ(expected_max_committed_size, root.max_size_of_committed_pages);
  3088. EXPECT_EQ(expected_max_allocated_size,
  3089. root.get_max_size_of_allocated_bytes());
  3090. EXPECT_EQ(expected_super_pages_size, root.total_size_of_super_pages);
  3091. // Allocating another size commits another slot span.
  3092. ptr = root.Alloc(2 * small_size - kExtraAllocSize, type_name);
  3093. expected_committed_size +=
  3094. kUseLazyCommit ? SystemPageSize() : PartitionPageSize();
  3095. expected_max_committed_size =
  3096. std::max(expected_max_committed_size, expected_committed_size);
  3097. expected_max_allocated_size =
  3098. std::max(expected_max_allocated_size, static_cast<size_t>(2048));
  3099. EXPECT_EQ(expected_committed_size, root.total_size_of_committed_pages);
  3100. EXPECT_EQ(expected_max_committed_size, root.max_size_of_committed_pages);
  3101. EXPECT_EQ(expected_max_allocated_size,
  3102. root.get_max_size_of_allocated_bytes());
  3103. EXPECT_EQ(expected_super_pages_size, root.total_size_of_super_pages);
  3104. // Freeing memory doesn't result in decommitting pages right away.
  3105. root.Free(ptr);
  3106. EXPECT_EQ(expected_committed_size, root.total_size_of_committed_pages);
  3107. EXPECT_EQ(expected_max_committed_size, root.max_size_of_committed_pages);
  3108. EXPECT_EQ(expected_max_allocated_size,
  3109. root.get_max_size_of_allocated_bytes());
  3110. EXPECT_EQ(expected_super_pages_size, root.total_size_of_super_pages);
  3111. // Single-slot slot spans...
  3112. size_t big_size = kMaxBucketed - SystemPageSize();
  3113. ASSERT_GT(big_size, MaxRegularSlotSpanSize());
  3114. ASSERT_LE(big_size, kMaxBucketed);
  3115. bucket_index = SizeToIndex(big_size - kExtraAllocSize);
  3116. bucket = &root.buckets[bucket_index];
  3117. // Assert the allocation doesn't fill the entire span nor entire partition
  3118. // page, to make the test more interesting.
  3119. ASSERT_LT(big_size, bucket->get_bytes_per_span());
  3120. ASSERT_NE(big_size % PartitionPageSize(), 0U);
  3121. ptr = root.Alloc(big_size - kExtraAllocSize, type_name);
  3122. expected_committed_size += bucket->get_bytes_per_span();
  3123. expected_max_committed_size =
  3124. std::max(expected_max_committed_size, expected_committed_size);
  3125. expected_total_allocated_size += bucket->get_bytes_per_span();
  3126. expected_max_allocated_size =
  3127. std::max(expected_max_allocated_size, expected_total_allocated_size);
  3128. EXPECT_EQ(expected_committed_size, root.total_size_of_committed_pages);
  3129. EXPECT_EQ(expected_max_committed_size, root.max_size_of_committed_pages);
  3130. EXPECT_EQ(expected_total_allocated_size,
  3131. root.get_total_size_of_allocated_bytes());
  3132. EXPECT_EQ(expected_max_allocated_size,
  3133. root.get_max_size_of_allocated_bytes());
  3134. EXPECT_EQ(expected_super_pages_size, root.total_size_of_super_pages);
  3135. // Allocating 2nd time doesn't overflow the super page...
  3136. void* ptr2 = root.Alloc(big_size - kExtraAllocSize, type_name);
  3137. expected_committed_size += bucket->get_bytes_per_span();
  3138. expected_max_committed_size =
  3139. std::max(expected_max_committed_size, expected_committed_size);
  3140. expected_total_allocated_size += bucket->get_bytes_per_span();
  3141. expected_max_allocated_size =
  3142. std::max(expected_max_allocated_size, expected_total_allocated_size);
  3143. EXPECT_EQ(expected_committed_size, root.total_size_of_committed_pages);
  3144. EXPECT_EQ(expected_max_committed_size, root.max_size_of_committed_pages);
  3145. EXPECT_EQ(expected_total_allocated_size,
  3146. root.get_total_size_of_allocated_bytes());
  3147. EXPECT_EQ(expected_max_allocated_size,
  3148. root.get_max_size_of_allocated_bytes());
  3149. EXPECT_EQ(expected_super_pages_size, root.total_size_of_super_pages);
  3150. // ... but 3rd time does.
  3151. void* ptr3 = root.Alloc(big_size - kExtraAllocSize, type_name);
  3152. expected_committed_size += bucket->get_bytes_per_span();
  3153. expected_max_committed_size =
  3154. std::max(expected_max_committed_size, expected_committed_size);
  3155. expected_total_allocated_size += bucket->get_bytes_per_span();
  3156. expected_max_allocated_size =
  3157. std::max(expected_max_allocated_size, expected_total_allocated_size);
  3158. expected_super_pages_size += kSuperPageSize;
  3159. EXPECT_EQ(expected_committed_size, root.total_size_of_committed_pages);
  3160. EXPECT_EQ(expected_max_committed_size, root.max_size_of_committed_pages);
  3161. EXPECT_EQ(expected_total_allocated_size,
  3162. root.get_total_size_of_allocated_bytes());
  3163. EXPECT_EQ(expected_max_allocated_size,
  3164. root.get_max_size_of_allocated_bytes());
  3165. EXPECT_EQ(expected_super_pages_size, root.total_size_of_super_pages);
  3166. // Freeing memory doesn't result in decommitting pages right away.
  3167. root.Free(ptr);
  3168. root.Free(ptr2);
  3169. root.Free(ptr3);
  3170. expected_total_allocated_size -= 3 * bucket->get_bytes_per_span();
  3171. expected_max_allocated_size =
  3172. std::max(expected_max_allocated_size, expected_total_allocated_size);
  3173. EXPECT_EQ(expected_committed_size, root.total_size_of_committed_pages);
  3174. EXPECT_EQ(expected_max_committed_size, root.max_size_of_committed_pages);
  3175. EXPECT_EQ(expected_total_allocated_size,
  3176. root.get_total_size_of_allocated_bytes());
  3177. EXPECT_EQ(expected_max_allocated_size,
  3178. root.get_max_size_of_allocated_bytes());
  3179. EXPECT_EQ(expected_super_pages_size, root.total_size_of_super_pages);
  3180. // Now everything should be decommitted. The reserved space for super pages
  3181. // stays the same and will never go away (by design).
  3182. root.PurgeMemory(PurgeFlags::kDecommitEmptySlotSpans);
  3183. expected_committed_size = 0;
  3184. EXPECT_EQ(expected_committed_size, root.total_size_of_committed_pages);
  3185. EXPECT_EQ(expected_max_committed_size, root.max_size_of_committed_pages);
  3186. EXPECT_EQ(expected_total_allocated_size,
  3187. root.get_total_size_of_allocated_bytes());
  3188. EXPECT_EQ(expected_max_allocated_size,
  3189. root.get_max_size_of_allocated_bytes());
  3190. EXPECT_EQ(expected_super_pages_size, root.total_size_of_super_pages);
  3191. // None of the above should affect the direct map space.
  3192. EXPECT_EQ(0U, root.total_size_of_direct_mapped_pages);
  3193. size_t huge_sizes[] = {
  3194. kMaxBucketed + SystemPageSize(),
  3195. kMaxBucketed + SystemPageSize() + 123,
  3196. kSuperPageSize - PageAllocationGranularity(),
  3197. kSuperPageSize - SystemPageSize() - PartitionPageSize(),
  3198. kSuperPageSize - PartitionPageSize(),
  3199. kSuperPageSize - SystemPageSize(),
  3200. kSuperPageSize,
  3201. kSuperPageSize + SystemPageSize(),
  3202. kSuperPageSize + PartitionPageSize(),
  3203. kSuperPageSize + SystemPageSize() + PartitionPageSize(),
  3204. kSuperPageSize + PageAllocationGranularity(),
  3205. kSuperPageSize + DirectMapAllocationGranularity(),
  3206. };
  3207. size_t alignments[] = {
  3208. PartitionPageSize(),
  3209. 2 * PartitionPageSize(),
  3210. kMaxSupportedAlignment / 2,
  3211. kMaxSupportedAlignment,
  3212. };
  3213. for (size_t huge_size : huge_sizes) {
  3214. ASSERT_GT(huge_size, kMaxBucketed);
  3215. for (size_t alignment : alignments) {
  3216. // For direct map, we commit only as many pages as needed.
  3217. size_t aligned_size = partition_alloc::internal::base::bits::AlignUp(
  3218. huge_size, SystemPageSize());
  3219. ptr = root.AllocWithFlagsInternal(0, huge_size - kExtraAllocSize,
  3220. alignment, type_name);
  3221. expected_committed_size += aligned_size;
  3222. expected_max_committed_size =
  3223. std::max(expected_max_committed_size, expected_committed_size);
  3224. expected_total_allocated_size += aligned_size;
  3225. expected_max_allocated_size =
  3226. std::max(expected_max_allocated_size, expected_total_allocated_size);
  3227. // The total reserved map includes metadata and guard pages at the ends.
  3228. // It also includes alignment. However, these would double count the first
  3229. // partition page, so it needs to be subtracted.
  3230. size_t surrounding_pages_size =
  3231. PartitionRoot<ThreadSafe>::GetDirectMapMetadataAndGuardPagesSize() +
  3232. alignment - PartitionPageSize();
  3233. size_t expected_direct_map_size =
  3234. partition_alloc::internal::base::bits::AlignUp(
  3235. aligned_size + surrounding_pages_size,
  3236. DirectMapAllocationGranularity());
  3237. EXPECT_EQ(expected_committed_size, root.total_size_of_committed_pages);
  3238. EXPECT_EQ(expected_max_committed_size, root.max_size_of_committed_pages);
  3239. EXPECT_EQ(expected_total_allocated_size,
  3240. root.get_total_size_of_allocated_bytes());
  3241. EXPECT_EQ(expected_max_allocated_size,
  3242. root.get_max_size_of_allocated_bytes());
  3243. EXPECT_EQ(expected_super_pages_size, root.total_size_of_super_pages);
  3244. EXPECT_EQ(expected_direct_map_size,
  3245. root.total_size_of_direct_mapped_pages);
  3246. // Freeing memory in the diret map decommits pages right away. The address
  3247. // space is released for re-use too.
  3248. root.Free(ptr);
  3249. expected_committed_size -= aligned_size;
  3250. expected_direct_map_size = 0;
  3251. expected_max_committed_size =
  3252. std::max(expected_max_committed_size, expected_committed_size);
  3253. expected_total_allocated_size -= aligned_size;
  3254. expected_max_allocated_size =
  3255. std::max(expected_max_allocated_size, expected_total_allocated_size);
  3256. EXPECT_EQ(expected_committed_size, root.total_size_of_committed_pages);
  3257. EXPECT_EQ(expected_max_committed_size, root.max_size_of_committed_pages);
  3258. EXPECT_EQ(expected_total_allocated_size,
  3259. root.get_total_size_of_allocated_bytes());
  3260. EXPECT_EQ(expected_max_allocated_size,
  3261. root.get_max_size_of_allocated_bytes());
  3262. EXPECT_EQ(expected_super_pages_size, root.total_size_of_super_pages);
  3263. EXPECT_EQ(expected_direct_map_size,
  3264. root.total_size_of_direct_mapped_pages);
  3265. }
  3266. }
  3267. }
  3268. #if BUILDFLAG(USE_BACKUP_REF_PTR)
  3269. TEST_P(PartitionAllocTest, RefCountBasic) {
  3270. constexpr uint64_t kCookie = 0x1234567890ABCDEF;
  3271. constexpr uint64_t kQuarantined = 0xEFEFEFEFEFEFEFEF;
  3272. size_t alloc_size = 64 - kExtraAllocSize;
  3273. uint64_t* ptr1 =
  3274. static_cast<uint64_t*>(allocator.root()->Alloc(alloc_size, type_name));
  3275. EXPECT_TRUE(ptr1);
  3276. *ptr1 = kCookie;
  3277. auto* ref_count =
  3278. PartitionRefCountPointer(allocator.root()->ObjectToSlotStart(ptr1));
  3279. EXPECT_TRUE(ref_count->IsAliveWithNoKnownRefs());
  3280. ref_count->Acquire();
  3281. EXPECT_FALSE(ref_count->Release());
  3282. EXPECT_TRUE(ref_count->IsAliveWithNoKnownRefs());
  3283. EXPECT_EQ(*ptr1, kCookie);
  3284. ref_count->Acquire();
  3285. EXPECT_FALSE(ref_count->IsAliveWithNoKnownRefs());
  3286. allocator.root()->Free(ptr1);
  3287. // The allocation shouldn't be reclaimed, and its contents should be zapped.
  3288. // Retag ptr1 to get its correct MTE tag.
  3289. ptr1 = TagPtr(ptr1);
  3290. EXPECT_NE(*ptr1, kCookie);
  3291. EXPECT_EQ(*ptr1, kQuarantined);
  3292. // The allocator should not reuse the original slot since its reference count
  3293. // doesn't equal zero.
  3294. uint64_t* ptr2 =
  3295. static_cast<uint64_t*>(allocator.root()->Alloc(alloc_size, type_name));
  3296. EXPECT_NE(ptr1, ptr2);
  3297. allocator.root()->Free(ptr2);
  3298. // When the last reference is released, the slot should become reusable.
  3299. // Retag ref_count because PartitionAlloc retags ptr to enforce quarantine.
  3300. ref_count = TagPtr(ref_count);
  3301. EXPECT_TRUE(ref_count->Release());
  3302. PartitionAllocFreeForRefCounting(allocator.root()->ObjectToSlotStart(ptr1));
  3303. uint64_t* ptr3 =
  3304. static_cast<uint64_t*>(allocator.root()->Alloc(alloc_size, type_name));
  3305. EXPECT_EQ(ptr1, ptr3);
  3306. allocator.root()->Free(ptr3);
  3307. }
  3308. void PartitionAllocTest::RunRefCountReallocSubtest(size_t orig_size,
  3309. size_t new_size) {
  3310. void* ptr1 = allocator.root()->Alloc(orig_size, type_name);
  3311. EXPECT_TRUE(ptr1);
  3312. auto* ref_count1 =
  3313. PartitionRefCountPointer(allocator.root()->ObjectToSlotStart(ptr1));
  3314. EXPECT_TRUE(ref_count1->IsAliveWithNoKnownRefs());
  3315. ref_count1->Acquire();
  3316. EXPECT_FALSE(ref_count1->IsAliveWithNoKnownRefs());
  3317. void* ptr2 = allocator.root()->Realloc(ptr1, new_size, type_name);
  3318. EXPECT_TRUE(ptr2);
  3319. // PartitionAlloc may retag memory areas on realloc (even if they
  3320. // do not move), so recover the true tag here.
  3321. ref_count1 = TagPtr(ref_count1);
  3322. // Re-query ref-count. It may have moved if Realloc changed the slot.
  3323. auto* ref_count2 =
  3324. PartitionRefCountPointer(allocator.root()->ObjectToSlotStart(ptr2));
  3325. if (UntagPtr(ptr1) == UntagPtr(ptr2)) {
  3326. // If the slot didn't change, ref-count should stay the same.
  3327. EXPECT_EQ(ref_count1, ref_count2);
  3328. EXPECT_FALSE(ref_count2->IsAliveWithNoKnownRefs());
  3329. EXPECT_FALSE(ref_count2->Release());
  3330. } else {
  3331. // If the allocation was moved to another slot, the old ref-count stayed
  3332. // in the same location in memory, is no longer alive, but still has a
  3333. // reference. The new ref-count is alive, but has no references.
  3334. EXPECT_NE(ref_count1, ref_count2);
  3335. EXPECT_FALSE(ref_count1->IsAlive());
  3336. EXPECT_FALSE(ref_count1->IsAliveWithNoKnownRefs());
  3337. EXPECT_TRUE(ref_count2->IsAliveWithNoKnownRefs());
  3338. EXPECT_TRUE(ref_count1->Release());
  3339. }
  3340. allocator.root()->Free(ptr2);
  3341. }
  3342. TEST_P(PartitionAllocTest, RefCountRealloc) {
  3343. size_t alloc_sizes[] = {500, 5000, 50000, 400000};
  3344. for (size_t alloc_size : alloc_sizes) {
  3345. alloc_size -= kExtraAllocSize;
  3346. RunRefCountReallocSubtest(alloc_size, alloc_size - 9);
  3347. RunRefCountReallocSubtest(alloc_size, alloc_size + 9);
  3348. RunRefCountReallocSubtest(alloc_size, alloc_size * 2);
  3349. RunRefCountReallocSubtest(alloc_size, alloc_size / 2);
  3350. }
  3351. }
  3352. #endif // BUILDFLAG(USE_BACKUP_REF_PTR)
  3353. #if BUILDFLAG(ENABLE_DANGLING_RAW_PTR_CHECKS)
  3354. // Allocate memory, and reference it from 3 raw_ptr. Among them 2 will be
  3355. // dangling.
  3356. TEST_P(PartitionAllocTest, DanglingPtr) {
  3357. CountDanglingRawPtr dangling_checks;
  3358. // Allocate memory, and reference it from 3 raw_ptr.
  3359. uint64_t* ptr = static_cast<uint64_t*>(
  3360. allocator.root()->Alloc(64 - kExtraAllocSize, type_name));
  3361. auto* ref_count =
  3362. PartitionRefCountPointer(allocator.root()->ObjectToSlotStart(ptr));
  3363. ref_count->Acquire();
  3364. ref_count->Acquire();
  3365. ref_count->Acquire();
  3366. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
  3367. EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
  3368. // The first raw_ptr stops referencing it, before the memory has been
  3369. // released.
  3370. EXPECT_FALSE(ref_count->Release());
  3371. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
  3372. EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
  3373. // Free it. This creates two dangling pointer.
  3374. allocator.root()->Free(ptr);
  3375. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 1);
  3376. EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
  3377. // The dangling raw_ptr stop referencing it.
  3378. EXPECT_FALSE(ref_count->Release());
  3379. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 1);
  3380. EXPECT_EQ(g_dangling_raw_ptr_released_count, 1);
  3381. // The dangling raw_ptr stop referencing it again.
  3382. EXPECT_TRUE(ref_count->Release());
  3383. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 1);
  3384. EXPECT_EQ(g_dangling_raw_ptr_released_count, 2);
  3385. }
  3386. // Allocate memory, and reference it from 3
  3387. // raw_ptr<T, DisableDanglingPtrDetection>. Among them 2 will be dangling. This
  3388. // doesn't trigger any dangling raw_ptr checks.
  3389. TEST_P(PartitionAllocTest, DanglingDanglingPtr) {
  3390. CountDanglingRawPtr dangling_checks;
  3391. // Allocate memory, and reference it from 3 raw_ptr.
  3392. uint64_t* ptr = static_cast<uint64_t*>(
  3393. allocator.root()->Alloc(64 - kExtraAllocSize, type_name));
  3394. auto* ref_count =
  3395. PartitionRefCountPointer(allocator.root()->ObjectToSlotStart(ptr));
  3396. ref_count->AcquireFromUnprotectedPtr();
  3397. ref_count->AcquireFromUnprotectedPtr();
  3398. ref_count->AcquireFromUnprotectedPtr();
  3399. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
  3400. EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
  3401. // The first raw_ptr<T, DisableDanglingPtrDetection> stops referencing it,
  3402. // before the memory has been released.
  3403. EXPECT_FALSE(ref_count->ReleaseFromUnprotectedPtr());
  3404. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
  3405. EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
  3406. // Free it. This creates two dangling raw_ptr<T, DisableDanglingPtrDetection>.
  3407. allocator.root()->Free(ptr);
  3408. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
  3409. EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
  3410. // The dangling raw_ptr<T, DisableDanglingPtrDetection> stop referencing it.
  3411. EXPECT_FALSE(ref_count->ReleaseFromUnprotectedPtr());
  3412. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
  3413. EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
  3414. // The dangling raw_ptr<T, DisableDanglingPtrDetection> stop referencing it
  3415. // again.
  3416. EXPECT_TRUE(ref_count->ReleaseFromUnprotectedPtr());
  3417. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
  3418. EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
  3419. }
  3420. // When 'free' is called, it remain one raw_ptr<> and one
  3421. // raw_ptr<T, DisableDanglingPtrDetection>. The raw_ptr<> is released first.
  3422. TEST_P(PartitionAllocTest, DanglingMixedReleaseRawPtrFirst) {
  3423. CountDanglingRawPtr dangling_checks;
  3424. uint64_t* ptr = static_cast<uint64_t*>(
  3425. allocator.root()->Alloc(64 - kExtraAllocSize, type_name));
  3426. auto* ref_count =
  3427. PartitionRefCountPointer(allocator.root()->ObjectToSlotStart(ptr));
  3428. // Acquire a raw_ptr<T, DisableDanglingPtrDetection> and a raw_ptr<>.
  3429. ref_count->AcquireFromUnprotectedPtr();
  3430. ref_count->Acquire();
  3431. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
  3432. EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
  3433. // Free it.
  3434. allocator.root()->Free(ptr);
  3435. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 1);
  3436. EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
  3437. // The raw_ptr<> stops referencing it.
  3438. EXPECT_FALSE(ref_count->Release());
  3439. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 1);
  3440. EXPECT_EQ(g_dangling_raw_ptr_released_count, 1);
  3441. // The raw_ptr<T, DisableDanglingPtrDetection> stop referencing it.
  3442. EXPECT_TRUE(ref_count->ReleaseFromUnprotectedPtr());
  3443. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 1);
  3444. EXPECT_EQ(g_dangling_raw_ptr_released_count, 1);
  3445. }
  3446. // When 'free' is called, it remain one raw_ptr<> and one
  3447. // raw_ptr<T, DisableDanglingPtrDetection>.
  3448. // The raw_ptr<T, DisableDanglingPtrDetection> is released first. This
  3449. // triggers the dangling raw_ptr<> checks.
  3450. TEST_P(PartitionAllocTest, DanglingMixedReleaseDanglingPtrFirst) {
  3451. CountDanglingRawPtr dangling_checks;
  3452. void* ptr = allocator.root()->Alloc(64 - kExtraAllocSize, type_name);
  3453. auto* ref_count =
  3454. PartitionRefCountPointer(allocator.root()->ObjectToSlotStart(ptr));
  3455. // Acquire a raw_ptr<T, DisableDanglingPtrDetection> and a raw_ptr<>.
  3456. ref_count->AcquireFromUnprotectedPtr();
  3457. ref_count->Acquire();
  3458. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
  3459. EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
  3460. // Free it.
  3461. allocator.root()->Free(ptr);
  3462. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 1);
  3463. EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
  3464. // The raw_ptr<> stops referencing it.
  3465. EXPECT_FALSE(ref_count->ReleaseFromUnprotectedPtr());
  3466. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 1);
  3467. EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
  3468. // The raw_ptr<T, DisableDanglingPtrDetection> stop referencing it.
  3469. EXPECT_TRUE(ref_count->Release());
  3470. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 1);
  3471. EXPECT_EQ(g_dangling_raw_ptr_released_count, 1);
  3472. }
  3473. // When 'free' is called, it remains one
  3474. // raw_ptr<T, DisableDanglingPtrDetection>, then it is used to acquire one
  3475. // dangling raw_ptr<>. Release the raw_ptr<> first.
  3476. TEST_P(PartitionAllocTest, DanglingPtrUsedToAcquireNewRawPtr) {
  3477. CountDanglingRawPtr dangling_checks;
  3478. void* ptr = allocator.root()->Alloc(64 - kExtraAllocSize, type_name);
  3479. auto* ref_count =
  3480. PartitionRefCountPointer(allocator.root()->ObjectToSlotStart(ptr));
  3481. // Acquire a raw_ptr<T, DisableDanglingPtrDetection>.
  3482. ref_count->AcquireFromUnprotectedPtr();
  3483. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
  3484. EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
  3485. // Free it once.
  3486. allocator.root()->Free(ptr);
  3487. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
  3488. EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
  3489. // A raw_ptr<> starts referencing it.
  3490. ref_count->Acquire();
  3491. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
  3492. EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
  3493. // The raw_ptr<> stops referencing it.
  3494. EXPECT_FALSE(ref_count->Release());
  3495. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
  3496. EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
  3497. // The raw_ptr<T, DisableDanglingPtrDetection> stops referencing it.
  3498. EXPECT_TRUE(ref_count->ReleaseFromUnprotectedPtr());
  3499. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
  3500. EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
  3501. }
  3502. // Same as 'DanglingPtrUsedToAcquireNewRawPtr', but release the
  3503. // raw_ptr<T, DisableDanglingPtrDetection> before the raw_ptr<>.
  3504. TEST_P(PartitionAllocTest, DanglingPtrUsedToAcquireNewRawPtrVariant) {
  3505. CountDanglingRawPtr dangling_checks;
  3506. void* ptr = allocator.root()->Alloc(64 - kExtraAllocSize, type_name);
  3507. auto* ref_count =
  3508. PartitionRefCountPointer(allocator.root()->ObjectToSlotStart(ptr));
  3509. // Acquire a raw_ptr<T, DisableDanglingPtrDetection>.
  3510. ref_count->AcquireFromUnprotectedPtr();
  3511. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
  3512. EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
  3513. // Free it.
  3514. allocator.root()->Free(ptr);
  3515. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
  3516. EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
  3517. // A raw_ptr<> starts referencing it.
  3518. ref_count->Acquire();
  3519. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
  3520. EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
  3521. // The raw_ptr<> stops referencing it.
  3522. EXPECT_FALSE(ref_count->ReleaseFromUnprotectedPtr());
  3523. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
  3524. EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
  3525. // The raw_ptr<T, DisableDanglingPtrDetection> stops referencing it.
  3526. EXPECT_TRUE(ref_count->Release());
  3527. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
  3528. EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
  3529. }
  3530. // Acquire a raw_ptr<T>, and release it before freeing memory. In the
  3531. // background, there is one raw_ptr<T, DisableDanglingPtrDetection>. This
  3532. // doesn't trigger any dangling raw_ptr<T> checks.
  3533. TEST_P(PartitionAllocTest, RawPtrReleasedBeforeFree) {
  3534. CountDanglingRawPtr dangling_checks;
  3535. void* ptr = allocator.root()->Alloc(64 - kExtraAllocSize, type_name);
  3536. auto* ref_count =
  3537. PartitionRefCountPointer(allocator.root()->ObjectToSlotStart(ptr));
  3538. // Acquire a raw_ptr<T, DisableDanglingPtrDetection> and a raw_ptr<>.
  3539. ref_count->Acquire();
  3540. ref_count->AcquireFromUnprotectedPtr();
  3541. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
  3542. EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
  3543. // Release the raw_ptr<>.
  3544. EXPECT_FALSE(ref_count->Release());
  3545. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
  3546. EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
  3547. // Free it.
  3548. allocator.root()->Free(ptr);
  3549. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
  3550. EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
  3551. // The raw_ptr<T, DisableDanglingPtrDetection> stop referencing it.
  3552. EXPECT_TRUE(ref_count->ReleaseFromUnprotectedPtr());
  3553. EXPECT_EQ(g_dangling_raw_ptr_detected_count, 0);
  3554. EXPECT_EQ(g_dangling_raw_ptr_released_count, 0);
  3555. }
  3556. // Acquire() once, Release() twice => CRASH
  3557. TEST_P(PartitionAllocDeathTest, ReleaseUnderflowRawPtr) {
  3558. void* ptr = allocator.root()->Alloc(64 - kExtraAllocSize, type_name);
  3559. auto* ref_count =
  3560. PartitionRefCountPointer(allocator.root()->ObjectToSlotStart(ptr));
  3561. ref_count->Acquire();
  3562. EXPECT_FALSE(ref_count->Release());
  3563. EXPECT_DEATH(ref_count->Release(), "");
  3564. allocator.root()->Free(ptr);
  3565. }
  3566. // AcquireFromUnprotectedPtr() once, ReleaseFromUnprotectedPtr() twice => CRASH
  3567. TEST_P(PartitionAllocDeathTest, ReleaseUnderflowDanglingPtr) {
  3568. void* ptr = allocator.root()->Alloc(64 - kExtraAllocSize, type_name);
  3569. auto* ref_count =
  3570. PartitionRefCountPointer(allocator.root()->ObjectToSlotStart(ptr));
  3571. ref_count->AcquireFromUnprotectedPtr();
  3572. EXPECT_FALSE(ref_count->ReleaseFromUnprotectedPtr());
  3573. EXPECT_DEATH(ref_count->ReleaseFromUnprotectedPtr(), "");
  3574. allocator.root()->Free(ptr);
  3575. }
  3576. #endif // BUILDFLAG(ENABLE_DANGLING_RAW_PTR_CHECKS)
  3577. TEST_P(PartitionAllocTest, ReservationOffset) {
  3578. // For normal buckets, offset should be kOffsetTagNormalBuckets.
  3579. void* ptr = allocator.root()->Alloc(kTestAllocSize, type_name);
  3580. EXPECT_TRUE(ptr);
  3581. uintptr_t address = UntagPtr(ptr);
  3582. EXPECT_EQ(kOffsetTagNormalBuckets, *ReservationOffsetPointer(address));
  3583. allocator.root()->Free(ptr);
  3584. // For direct-map,
  3585. size_t large_size = kSuperPageSize * 5 + PartitionPageSize() * .5f;
  3586. ASSERT_GT(large_size, kMaxBucketed);
  3587. ptr = allocator.root()->Alloc(large_size, type_name);
  3588. EXPECT_TRUE(ptr);
  3589. address = UntagPtr(ptr);
  3590. EXPECT_EQ(0U, *ReservationOffsetPointer(address));
  3591. EXPECT_EQ(1U, *ReservationOffsetPointer(address + kSuperPageSize));
  3592. EXPECT_EQ(2U, *ReservationOffsetPointer(address + kSuperPageSize * 2));
  3593. EXPECT_EQ(3U, *ReservationOffsetPointer(address + kSuperPageSize * 3));
  3594. EXPECT_EQ(4U, *ReservationOffsetPointer(address + kSuperPageSize * 4));
  3595. EXPECT_EQ(5U, *ReservationOffsetPointer(address + kSuperPageSize * 5));
  3596. // In-place realloc doesn't affect the offsets.
  3597. void* new_ptr = allocator.root()->Realloc(ptr, large_size * .8, type_name);
  3598. EXPECT_EQ(new_ptr, ptr);
  3599. EXPECT_EQ(0U, *ReservationOffsetPointer(address));
  3600. EXPECT_EQ(1U, *ReservationOffsetPointer(address + kSuperPageSize));
  3601. EXPECT_EQ(2U, *ReservationOffsetPointer(address + kSuperPageSize * 2));
  3602. EXPECT_EQ(3U, *ReservationOffsetPointer(address + kSuperPageSize * 3));
  3603. EXPECT_EQ(4U, *ReservationOffsetPointer(address + kSuperPageSize * 4));
  3604. EXPECT_EQ(5U, *ReservationOffsetPointer(address + kSuperPageSize * 5));
  3605. allocator.root()->Free(ptr);
  3606. // After free, the offsets must be kOffsetTagNotAllocated.
  3607. EXPECT_EQ(kOffsetTagNotAllocated, *ReservationOffsetPointer(address));
  3608. EXPECT_EQ(kOffsetTagNotAllocated,
  3609. *ReservationOffsetPointer(address + kSuperPageSize));
  3610. EXPECT_EQ(kOffsetTagNotAllocated,
  3611. *ReservationOffsetPointer(address + kSuperPageSize * 2));
  3612. EXPECT_EQ(kOffsetTagNotAllocated,
  3613. *ReservationOffsetPointer(address + kSuperPageSize * 3));
  3614. EXPECT_EQ(kOffsetTagNotAllocated,
  3615. *ReservationOffsetPointer(address + kSuperPageSize * 4));
  3616. EXPECT_EQ(kOffsetTagNotAllocated,
  3617. *ReservationOffsetPointer(address + kSuperPageSize * 5));
  3618. }
  3619. TEST_P(PartitionAllocTest, GetReservationStart) {
  3620. size_t large_size = kSuperPageSize * 3 + PartitionPageSize() * .5f;
  3621. ASSERT_GT(large_size, kMaxBucketed);
  3622. void* ptr = allocator.root()->Alloc(large_size, type_name);
  3623. EXPECT_TRUE(ptr);
  3624. uintptr_t slot_start = allocator.root()->ObjectToSlotStart(ptr);
  3625. uintptr_t reservation_start = slot_start - PartitionPageSize();
  3626. EXPECT_EQ(0U, reservation_start & DirectMapAllocationGranularityOffsetMask());
  3627. uintptr_t address = UntagPtr(ptr);
  3628. for (uintptr_t a = address; a < address + large_size; ++a) {
  3629. uintptr_t address2 = GetDirectMapReservationStart(a) + PartitionPageSize();
  3630. EXPECT_EQ(slot_start, address2);
  3631. }
  3632. EXPECT_EQ(reservation_start, GetDirectMapReservationStart(slot_start));
  3633. allocator.root()->Free(ptr);
  3634. }
  3635. TEST_P(PartitionAllocTest, CheckReservationType) {
  3636. void* ptr = allocator.root()->Alloc(kTestAllocSize, type_name);
  3637. EXPECT_TRUE(ptr);
  3638. uintptr_t address = UntagPtr(ptr);
  3639. uintptr_t address_to_check = address;
  3640. EXPECT_FALSE(IsReservationStart(address_to_check));
  3641. EXPECT_TRUE(IsManagedByNormalBuckets(address_to_check));
  3642. EXPECT_FALSE(IsManagedByDirectMap(address_to_check));
  3643. EXPECT_TRUE(IsManagedByNormalBucketsOrDirectMap(address_to_check));
  3644. address_to_check = address + kTestAllocSize - 1;
  3645. EXPECT_FALSE(IsReservationStart(address_to_check));
  3646. EXPECT_TRUE(IsManagedByNormalBuckets(address_to_check));
  3647. EXPECT_FALSE(IsManagedByDirectMap(address_to_check));
  3648. EXPECT_TRUE(IsManagedByNormalBucketsOrDirectMap(address_to_check));
  3649. address_to_check =
  3650. partition_alloc::internal::base::bits::AlignDown(address, kSuperPageSize);
  3651. EXPECT_TRUE(IsReservationStart(address_to_check));
  3652. EXPECT_TRUE(IsManagedByNormalBuckets(address_to_check));
  3653. EXPECT_FALSE(IsManagedByDirectMap(address_to_check));
  3654. EXPECT_TRUE(IsManagedByNormalBucketsOrDirectMap(address_to_check));
  3655. allocator.root()->Free(ptr);
  3656. // Freeing keeps a normal-bucket super page in memory.
  3657. address_to_check =
  3658. partition_alloc::internal::base::bits::AlignDown(address, kSuperPageSize);
  3659. EXPECT_TRUE(IsReservationStart(address_to_check));
  3660. EXPECT_TRUE(IsManagedByNormalBuckets(address_to_check));
  3661. EXPECT_FALSE(IsManagedByDirectMap(address_to_check));
  3662. EXPECT_TRUE(IsManagedByNormalBucketsOrDirectMap(address_to_check));
  3663. size_t large_size = 2 * kSuperPageSize;
  3664. ASSERT_GT(large_size, kMaxBucketed);
  3665. ptr = allocator.root()->Alloc(large_size, type_name);
  3666. EXPECT_TRUE(ptr);
  3667. address = UntagPtr(ptr);
  3668. address_to_check = address;
  3669. EXPECT_FALSE(IsReservationStart(address_to_check));
  3670. EXPECT_FALSE(IsManagedByNormalBuckets(address_to_check));
  3671. EXPECT_TRUE(IsManagedByDirectMap(address_to_check));
  3672. EXPECT_TRUE(IsManagedByNormalBucketsOrDirectMap(address_to_check));
  3673. address_to_check =
  3674. partition_alloc::internal::base::bits::AlignUp(address, kSuperPageSize);
  3675. EXPECT_FALSE(IsReservationStart(address_to_check));
  3676. EXPECT_FALSE(IsManagedByNormalBuckets(address_to_check));
  3677. EXPECT_TRUE(IsManagedByDirectMap(address_to_check));
  3678. EXPECT_TRUE(IsManagedByNormalBucketsOrDirectMap(address_to_check));
  3679. address_to_check = address + large_size - 1;
  3680. EXPECT_FALSE(IsReservationStart(address_to_check));
  3681. EXPECT_FALSE(IsManagedByNormalBuckets(address_to_check));
  3682. EXPECT_TRUE(IsManagedByDirectMap(address_to_check));
  3683. EXPECT_TRUE(IsManagedByNormalBucketsOrDirectMap(address_to_check));
  3684. address_to_check =
  3685. partition_alloc::internal::base::bits::AlignDown(address, kSuperPageSize);
  3686. EXPECT_TRUE(IsReservationStart(address_to_check));
  3687. EXPECT_FALSE(IsManagedByNormalBuckets(address_to_check));
  3688. EXPECT_TRUE(IsManagedByDirectMap(address_to_check));
  3689. EXPECT_TRUE(IsManagedByNormalBucketsOrDirectMap(address_to_check));
  3690. allocator.root()->Free(ptr);
  3691. // Freeing releases direct-map super pages.
  3692. address_to_check =
  3693. partition_alloc::internal::base::bits::AlignDown(address, kSuperPageSize);
  3694. #if BUILDFLAG(PA_DCHECK_IS_ON)
  3695. // Expect to DCHECK on unallocated region.
  3696. EXPECT_DEATH_IF_SUPPORTED(IsReservationStart(address_to_check), "");
  3697. #endif
  3698. EXPECT_FALSE(IsManagedByNormalBuckets(address_to_check));
  3699. EXPECT_FALSE(IsManagedByDirectMap(address_to_check));
  3700. EXPECT_FALSE(IsManagedByNormalBucketsOrDirectMap(address_to_check));
  3701. }
  3702. // Test for crash http://crbug.com/1169003.
  3703. TEST_P(PartitionAllocTest, CrossPartitionRootRealloc) {
  3704. // Size is large enough to satisfy it from a single-slot slot span
  3705. size_t test_size = MaxRegularSlotSpanSize() - kExtraAllocSize;
  3706. void* ptr = allocator.root()->AllocWithFlags(AllocFlags::kReturnNull,
  3707. test_size, nullptr);
  3708. EXPECT_TRUE(ptr);
  3709. // Create new root and call PurgeMemory to simulate ConfigurePartitions().
  3710. allocator.root()->PurgeMemory(PurgeFlags::kDecommitEmptySlotSpans |
  3711. PurgeFlags::kDiscardUnusedSystemPages);
  3712. auto* new_root = new PartitionRoot<ThreadSafe>({
  3713. PartitionOptions::AlignedAlloc::kDisallowed,
  3714. PartitionOptions::ThreadCache::kDisabled,
  3715. PartitionOptions::Quarantine::kDisallowed,
  3716. PartitionOptions::Cookie::kAllowed,
  3717. PartitionOptions::BackupRefPtr::kDisabled,
  3718. PartitionOptions::BackupRefPtrZapping::kDisabled,
  3719. PartitionOptions::UseConfigurablePool::kNo,
  3720. });
  3721. // Realloc from |allocator.root()| into |new_root|.
  3722. void* ptr2 = new_root->ReallocWithFlags(AllocFlags::kReturnNull, ptr,
  3723. test_size + 1024, nullptr);
  3724. EXPECT_TRUE(ptr2);
  3725. PA_EXPECT_PTR_NE(ptr, ptr2);
  3726. }
  3727. TEST_P(PartitionAllocTest, FastPathOrReturnNull) {
  3728. size_t allocation_size = 64;
  3729. // The very first allocation is never a fast path one, since it needs a new
  3730. // super page and a new partition page.
  3731. EXPECT_FALSE(allocator.root()->AllocWithFlags(
  3732. AllocFlags::kFastPathOrReturnNull, allocation_size, ""));
  3733. void* ptr = allocator.root()->AllocWithFlags(0, allocation_size, "");
  3734. ASSERT_TRUE(ptr);
  3735. // Next one is, since the partition page has been activated.
  3736. void* ptr2 = allocator.root()->AllocWithFlags(
  3737. AllocFlags::kFastPathOrReturnNull, allocation_size, "");
  3738. EXPECT_TRUE(ptr2);
  3739. // First allocation of a different bucket is slow.
  3740. EXPECT_FALSE(allocator.root()->AllocWithFlags(
  3741. AllocFlags::kFastPathOrReturnNull, 2 * allocation_size, ""));
  3742. size_t allocated_size = 2 * allocation_size;
  3743. std::vector<void*> ptrs;
  3744. while (void* new_ptr = allocator.root()->AllocWithFlags(
  3745. AllocFlags::kFastPathOrReturnNull, allocation_size, "")) {
  3746. ptrs.push_back(new_ptr);
  3747. allocated_size += allocation_size;
  3748. }
  3749. EXPECT_LE(allocated_size,
  3750. PartitionPageSize() * kMaxPartitionPagesPerRegularSlotSpan);
  3751. for (void* ptr_to_free : ptrs)
  3752. allocator.root()->FreeNoHooks(ptr_to_free);
  3753. allocator.root()->FreeNoHooks(ptr);
  3754. allocator.root()->FreeNoHooks(ptr2);
  3755. }
  3756. // Death tests misbehave on Android, http://crbug.com/643760.
  3757. #if defined(GTEST_HAS_DEATH_TEST) && !BUILDFLAG(IS_ANDROID)
  3758. #if !defined(OFFICIAL_BUILD) || !defined(NDEBUG)
  3759. TEST_P(PartitionAllocDeathTest, CheckTriggered) {
  3760. using ::testing::ContainsRegex;
  3761. #if BUILDFLAG(PA_DCHECK_IS_ON)
  3762. EXPECT_DEATH(PA_CHECK(5 == 7), ContainsRegex("Check failed.*5 == 7"));
  3763. #endif
  3764. EXPECT_DEATH(PA_CHECK(5 == 7), ContainsRegex("Check failed.*5 == 7"));
  3765. }
  3766. #endif // !defined(OFFICIAL_BUILD) && !defined(NDEBUG)
  3767. #endif // defined(GTEST_HAS_DEATH_TEST) && !BUILDFLAG(IS_ANDROID)
  3768. // Not on chromecast, since gtest considers extra output from itself as a test
  3769. // failure:
  3770. // https://ci.chromium.org/ui/p/chromium/builders/ci/Cast%20Audio%20Linux/98492/overview
  3771. #if BUILDFLAG(USE_PARTITION_ALLOC_AS_MALLOC) && \
  3772. defined(GTEST_HAS_DEATH_TEST) && !BUILDFLAG(IS_ANDROID) && \
  3773. !BUILDFLAG(PA_IS_CASTOS)
  3774. namespace {
  3775. PA_NOINLINE void FreeForTest(void* data) {
  3776. free(data);
  3777. }
  3778. class ThreadDelegateForPreforkHandler
  3779. : public base::PlatformThreadForTesting::Delegate {
  3780. public:
  3781. ThreadDelegateForPreforkHandler(std::atomic<bool>& please_stop,
  3782. std::atomic<int>& started_threads,
  3783. const int alloc_size)
  3784. : please_stop_(please_stop),
  3785. started_threads_(started_threads),
  3786. alloc_size_(alloc_size) {}
  3787. void ThreadMain() override {
  3788. started_threads_++;
  3789. while (!please_stop_.load(std::memory_order_relaxed)) {
  3790. void* ptr = malloc(alloc_size_);
  3791. // A simple malloc() / free() pair can be discarded by the compiler (and
  3792. // is), making the test fail. It is sufficient to make |FreeForTest()| a
  3793. // PA_NOINLINE function for the call to not be eliminated, but it is
  3794. // required.
  3795. FreeForTest(ptr);
  3796. }
  3797. }
  3798. private:
  3799. std::atomic<bool>& please_stop_;
  3800. std::atomic<int>& started_threads_;
  3801. const int alloc_size_;
  3802. };
  3803. } // namespace
  3804. // Disabled because executing it causes Gtest to show a warning in the output,
  3805. // which confuses the runner on some platforms, making the test report an
  3806. // "UNKNOWN" status even though it succeeded.
  3807. TEST_P(PartitionAllocTest, DISABLED_PreforkHandler) {
  3808. std::atomic<bool> please_stop;
  3809. std::atomic<int> started_threads{0};
  3810. // Continuously allocates / frees memory, bypassing the thread cache. This
  3811. // makes it likely that this thread will own the lock, and that the
  3812. // EXPECT_EXIT() part will deadlock.
  3813. constexpr size_t kAllocSize = ThreadCache::kLargeSizeThreshold + 1;
  3814. ThreadDelegateForPreforkHandler delegate(please_stop, started_threads,
  3815. kAllocSize);
  3816. constexpr int kThreads = 4;
  3817. base::PlatformThreadHandle thread_handles[kThreads];
  3818. for (auto& thread_handle : thread_handles) {
  3819. base::PlatformThreadForTesting::Create(0, &delegate, &thread_handle);
  3820. }
  3821. // Make sure all threads are actually already running.
  3822. while (started_threads != kThreads) {
  3823. }
  3824. EXPECT_EXIT(
  3825. {
  3826. void* ptr = malloc(kAllocSize);
  3827. FreeForTest(ptr);
  3828. exit(1);
  3829. },
  3830. ::testing::ExitedWithCode(1), "");
  3831. please_stop.store(true);
  3832. for (auto& thread_handle : thread_handles) {
  3833. base::PlatformThreadForTesting::Join(thread_handle);
  3834. }
  3835. }
  3836. #endif // BUILDFLAG(USE_PARTITION_ALLOC_AS_MALLOC) &&
  3837. // defined(GTEST_HAS_DEATH_TEST) && !BUILDFLAG(IS_ANDROID) &&
  3838. // !BUILDFLAG(PA_IS_CASTOS)
  3839. // Checks the bucket index logic.
  3840. TEST_P(PartitionAllocTest, GetIndex) {
  3841. BucketIndexLookup lookup{};
  3842. for (size_t size = 0; size < kMaxBucketed; size++) {
  3843. size_t index = BucketIndexLookup::GetIndex(size);
  3844. ASSERT_GE(lookup.bucket_sizes()[index], size);
  3845. }
  3846. // Make sure that power-of-two have exactly matching buckets.
  3847. for (size_t size = (1 << (kMinBucketedOrder - 1)); size < kMaxBucketed;
  3848. size <<= 1) {
  3849. size_t index = BucketIndexLookup::GetIndex(size);
  3850. ASSERT_EQ(lookup.bucket_sizes()[index], size);
  3851. }
  3852. }
  3853. // Used to check alignment. If the compiler understands the annotations, the
  3854. // zeroing in the constructor uses aligned SIMD instructions.
  3855. TEST_P(PartitionAllocTest, MallocFunctionAnnotations) {
  3856. struct TestStruct {
  3857. uint64_t a = 0;
  3858. uint64_t b = 0;
  3859. };
  3860. void* buffer = Alloc(sizeof(TestStruct));
  3861. // Should use "mov*a*ps" on x86_64.
  3862. auto* x = new (buffer) TestStruct();
  3863. EXPECT_EQ(x->a, 0u);
  3864. Free(buffer);
  3865. }
  3866. // Test that the ConfigurablePool works properly.
  3867. TEST_P(PartitionAllocTest, ConfigurablePool) {
  3868. EXPECT_FALSE(IsConfigurablePoolAvailable());
  3869. // The rest is only applicable to 64-bit mode
  3870. #if defined(ARCH_CPU_64_BITS)
  3871. // Repeat the test for every possible Pool size
  3872. const size_t max_pool_size = PartitionAddressSpace::ConfigurablePoolMaxSize();
  3873. const size_t min_pool_size = PartitionAddressSpace::ConfigurablePoolMinSize();
  3874. for (size_t pool_size = max_pool_size; pool_size >= min_pool_size;
  3875. pool_size /= 2) {
  3876. PA_DCHECK(partition_alloc::internal::base::bits::IsPowerOfTwo(pool_size));
  3877. EXPECT_FALSE(IsConfigurablePoolAvailable());
  3878. uintptr_t pool_base = AllocPages(
  3879. pool_size, pool_size, PageAccessibilityConfiguration::kInaccessible,
  3880. PageTag::kPartitionAlloc);
  3881. EXPECT_NE(0u, pool_base);
  3882. PartitionAddressSpace::InitConfigurablePool(pool_base, pool_size);
  3883. EXPECT_TRUE(IsConfigurablePoolAvailable());
  3884. auto* root = new PartitionRoot<ThreadSafe>({
  3885. PartitionOptions::AlignedAlloc::kDisallowed,
  3886. PartitionOptions::ThreadCache::kDisabled,
  3887. PartitionOptions::Quarantine::kDisallowed,
  3888. PartitionOptions::Cookie::kAllowed,
  3889. PartitionOptions::BackupRefPtr::kDisabled,
  3890. PartitionOptions::BackupRefPtrZapping::kDisabled,
  3891. PartitionOptions::UseConfigurablePool::kIfAvailable,
  3892. });
  3893. root->UncapEmptySlotSpanMemoryForTesting();
  3894. const size_t count = 250;
  3895. std::vector<void*> allocations(count, nullptr);
  3896. for (size_t i = 0; i < count; ++i) {
  3897. const size_t size = kTestSizes[base::RandGenerator(kTestSizesCount)];
  3898. allocations[i] = root->Alloc(size, nullptr);
  3899. EXPECT_NE(nullptr, allocations[i]);
  3900. uintptr_t allocation_base = UntagPtr(allocations[i]);
  3901. EXPECT_TRUE(allocation_base >= pool_base &&
  3902. allocation_base < pool_base + pool_size);
  3903. }
  3904. PartitionAddressSpace::UninitConfigurablePoolForTesting();
  3905. FreePages(pool_base, pool_size);
  3906. }
  3907. #endif // defined(ARCH_CPU_64_BITS)
  3908. }
  3909. TEST_P(PartitionAllocTest, EmptySlotSpanSizeIsCapped) {
  3910. // Use another root, since the ones from the test harness disable the empty
  3911. // slot span size cap.
  3912. PartitionRoot<ThreadSafe> root;
  3913. root.Init({
  3914. PartitionOptions::AlignedAlloc::kDisallowed,
  3915. PartitionOptions::ThreadCache::kDisabled,
  3916. PartitionOptions::Quarantine::kDisallowed,
  3917. PartitionOptions::Cookie::kAllowed,
  3918. PartitionOptions::BackupRefPtr::kDisabled,
  3919. PartitionOptions::BackupRefPtrZapping::kDisabled,
  3920. PartitionOptions::UseConfigurablePool::kNo,
  3921. });
  3922. // Allocate some memory, don't free it to keep committed memory.
  3923. std::vector<void*> allocated_memory;
  3924. const size_t size = SystemPageSize();
  3925. const size_t count = 400;
  3926. for (size_t i = 0; i < count; i++) {
  3927. void* ptr = root.Alloc(size, "");
  3928. allocated_memory.push_back(ptr);
  3929. }
  3930. ASSERT_GE(root.total_size_of_committed_pages.load(std::memory_order_relaxed),
  3931. size * count);
  3932. // To create empty slot spans, allocate from single-slot slot spans, 128kiB at
  3933. // a time.
  3934. std::vector<void*> single_slot_allocated_memory;
  3935. constexpr size_t single_slot_count = kDefaultEmptySlotSpanRingSize - 1;
  3936. const size_t single_slot_size = MaxRegularSlotSpanSize() + 1;
  3937. // Make sure that even with allocation size rounding up, a single allocation
  3938. // is still below the threshold.
  3939. ASSERT_LT(MaxRegularSlotSpanSize() * 2,
  3940. ((count * size) >> root.max_empty_slot_spans_dirty_bytes_shift));
  3941. for (size_t i = 0; i < single_slot_count; i++) {
  3942. void* ptr = root.Alloc(single_slot_size, "");
  3943. single_slot_allocated_memory.push_back(ptr);
  3944. }
  3945. // Free everything at once, creating as many empty slot spans as there are
  3946. // allocations (since they are from single-slot slot spans).
  3947. for (void* ptr : single_slot_allocated_memory)
  3948. root.Free(ptr);
  3949. // Still have some committed empty slot spans.
  3950. // PA_TS_UNCHECKED_READ() is not an issue here, since everything is
  3951. // single-threaded.
  3952. EXPECT_GT(PA_TS_UNCHECKED_READ(root.empty_slot_spans_dirty_bytes), 0u);
  3953. // But not all, as the cap triggered.
  3954. EXPECT_LT(PA_TS_UNCHECKED_READ(root.empty_slot_spans_dirty_bytes),
  3955. single_slot_count * single_slot_size);
  3956. // Nothing left after explicit purge.
  3957. root.PurgeMemory(PurgeFlags::kDecommitEmptySlotSpans);
  3958. EXPECT_EQ(PA_TS_UNCHECKED_READ(root.empty_slot_spans_dirty_bytes), 0u);
  3959. for (void* ptr : allocated_memory)
  3960. root.Free(ptr);
  3961. }
  3962. TEST_P(PartitionAllocTest, IncreaseEmptySlotSpanRingSize) {
  3963. PartitionRoot<ThreadSafe> root({
  3964. PartitionOptions::AlignedAlloc::kDisallowed,
  3965. PartitionOptions::ThreadCache::kDisabled,
  3966. PartitionOptions::Quarantine::kDisallowed,
  3967. PartitionOptions::Cookie::kAllowed,
  3968. PartitionOptions::BackupRefPtr::kDisabled,
  3969. PartitionOptions::BackupRefPtrZapping::kDisabled,
  3970. PartitionOptions::UseConfigurablePool::kIfAvailable,
  3971. });
  3972. root.UncapEmptySlotSpanMemoryForTesting();
  3973. std::vector<void*> single_slot_allocated_memory;
  3974. constexpr size_t single_slot_count = kDefaultEmptySlotSpanRingSize + 10;
  3975. const size_t single_slot_size = MaxRegularSlotSpanSize() + 1;
  3976. const size_t bucket_size =
  3977. root.buckets[SizeToIndex(single_slot_size)].slot_size;
  3978. for (size_t i = 0; i < single_slot_count; i++) {
  3979. void* ptr = root.Alloc(single_slot_size, "");
  3980. single_slot_allocated_memory.push_back(ptr);
  3981. }
  3982. // Free everything at once, creating as many empty slot spans as there are
  3983. // allocations (since they are from single-slot slot spans).
  3984. for (void* ptr : single_slot_allocated_memory)
  3985. root.Free(ptr);
  3986. single_slot_allocated_memory.clear();
  3987. // Some of the free()-s above overflowed the slot span ring.
  3988. EXPECT_EQ(PA_TS_UNCHECKED_READ(root.empty_slot_spans_dirty_bytes),
  3989. kDefaultEmptySlotSpanRingSize * bucket_size);
  3990. // Now can cache more slot spans.
  3991. root.EnableLargeEmptySlotSpanRing();
  3992. constexpr size_t single_slot_large_count = kDefaultEmptySlotSpanRingSize + 10;
  3993. for (size_t i = 0; i < single_slot_large_count; i++) {
  3994. void* ptr = root.Alloc(single_slot_size, "");
  3995. single_slot_allocated_memory.push_back(ptr);
  3996. }
  3997. for (void* ptr : single_slot_allocated_memory)
  3998. root.Free(ptr);
  3999. single_slot_allocated_memory.clear();
  4000. // No overflow this time.
  4001. EXPECT_EQ(PA_TS_UNCHECKED_READ(root.empty_slot_spans_dirty_bytes),
  4002. single_slot_large_count * bucket_size);
  4003. constexpr size_t single_slot_too_many_count = kMaxFreeableSpans + 10;
  4004. for (size_t i = 0; i < single_slot_too_many_count; i++) {
  4005. void* ptr = root.Alloc(single_slot_size, "");
  4006. single_slot_allocated_memory.push_back(ptr);
  4007. }
  4008. for (void* ptr : single_slot_allocated_memory)
  4009. root.Free(ptr);
  4010. single_slot_allocated_memory.clear();
  4011. // Overflow still works.
  4012. EXPECT_EQ(PA_TS_UNCHECKED_READ(root.empty_slot_spans_dirty_bytes),
  4013. kMaxFreeableSpans * bucket_size);
  4014. }
  4015. #if defined(PA_USE_MTE_CHECKED_PTR_WITH_64_BITS_POINTERS)
  4016. // Verifies basic PA support for `MTECheckedPtr`.
  4017. TEST_P(PartitionAllocTest, PartitionTagBasic) {
  4018. const size_t alloc_size = 64 - kExtraAllocSize;
  4019. void* ptr1 = allocator.root()->Alloc(alloc_size, type_name);
  4020. void* ptr2 = allocator.root()->Alloc(alloc_size, type_name);
  4021. void* ptr3 = allocator.root()->Alloc(alloc_size, type_name);
  4022. EXPECT_TRUE(ptr1);
  4023. EXPECT_TRUE(ptr2);
  4024. EXPECT_TRUE(ptr3);
  4025. auto* slot_span = SlotSpan::FromObject(ptr1);
  4026. EXPECT_TRUE(slot_span);
  4027. char* char_ptr1 = static_cast<char*>(ptr1);
  4028. char* char_ptr2 = static_cast<char*>(ptr2);
  4029. char* char_ptr3 = static_cast<char*>(ptr3);
  4030. EXPECT_LT(kTestAllocSize, slot_span->bucket->slot_size);
  4031. EXPECT_EQ(char_ptr1 + slot_span->bucket->slot_size, char_ptr2);
  4032. EXPECT_EQ(char_ptr2 + slot_span->bucket->slot_size, char_ptr3);
  4033. constexpr partition_alloc::PartitionTag kTag1 =
  4034. static_cast<partition_alloc::PartitionTag>(0xBADA);
  4035. constexpr partition_alloc::PartitionTag kTag2 =
  4036. static_cast<partition_alloc::PartitionTag>(0xDB8A);
  4037. constexpr partition_alloc::PartitionTag kTag3 =
  4038. static_cast<partition_alloc::PartitionTag>(0xA3C4);
  4039. partition_alloc::internal::NormalBucketPartitionTagSetValue(
  4040. allocator.root()->ObjectToSlotStart(ptr1), slot_span->bucket->slot_size,
  4041. kTag1);
  4042. partition_alloc::internal::NormalBucketPartitionTagSetValue(
  4043. allocator.root()->ObjectToSlotStart(ptr2), slot_span->bucket->slot_size,
  4044. kTag2);
  4045. partition_alloc::internal::NormalBucketPartitionTagSetValue(
  4046. allocator.root()->ObjectToSlotStart(ptr3), slot_span->bucket->slot_size,
  4047. kTag3);
  4048. memset(ptr1, 0, alloc_size);
  4049. memset(ptr2, 0, alloc_size);
  4050. memset(ptr3, 0, alloc_size);
  4051. EXPECT_EQ(kTag1, partition_alloc::internal::PartitionTagGetValue(ptr1));
  4052. EXPECT_EQ(kTag2, partition_alloc::internal::PartitionTagGetValue(ptr2));
  4053. EXPECT_EQ(kTag3, partition_alloc::internal::PartitionTagGetValue(ptr3));
  4054. EXPECT_TRUE(!memchr(ptr1, static_cast<uint8_t>(kTag1), alloc_size));
  4055. EXPECT_TRUE(!memchr(ptr2, static_cast<uint8_t>(kTag2), alloc_size));
  4056. allocator.root()->Free(ptr1);
  4057. EXPECT_EQ(kTag2, partition_alloc::internal::PartitionTagGetValue(ptr2));
  4058. size_t request_size = slot_span->bucket->slot_size - kExtraAllocSize;
  4059. void* new_ptr2 = allocator.root()->Realloc(ptr2, request_size, type_name);
  4060. EXPECT_EQ(ptr2, new_ptr2);
  4061. EXPECT_EQ(kTag3, partition_alloc::internal::PartitionTagGetValue(ptr3));
  4062. // Add 1B to ensure the object is rellocated to a larger slot.
  4063. request_size = slot_span->bucket->slot_size - kExtraAllocSize + 1;
  4064. new_ptr2 = allocator.root()->Realloc(ptr2, request_size, type_name);
  4065. EXPECT_TRUE(new_ptr2);
  4066. EXPECT_NE(ptr2, new_ptr2);
  4067. allocator.root()->Free(new_ptr2);
  4068. EXPECT_EQ(kTag3, partition_alloc::internal::PartitionTagGetValue(ptr3));
  4069. allocator.root()->Free(ptr3);
  4070. }
  4071. // Verifies basic PA support for MTECheckedPtr used with direct map
  4072. // allocations.
  4073. TEST_P(PartitionAllocTest, PartitionTagDirectMapBasic) {
  4074. constexpr size_t kAllocSize = partition_alloc::internal::kSuperPageSize * 3;
  4075. void* object = allocator.root()->AllocWithFlags(AllocFlags::kZeroFill,
  4076. kAllocSize, type_name);
  4077. ASSERT_TRUE(object);
  4078. ASSERT_TRUE(IsManagedByDirectMap(UntagPtr(object)));
  4079. constexpr partition_alloc::PartitionTag kTag =
  4080. static_cast<partition_alloc::PartitionTag>(0xBADA);
  4081. partition_alloc::internal::DirectMapPartitionTagSetValue(
  4082. allocator.root()->ObjectToSlotStart(object), kTag);
  4083. EXPECT_EQ(kTag, partition_alloc::internal::PartitionTagGetValue(object));
  4084. // As the allocation spans four (bumped over three by metadata) super
  4085. // pages, we expect offsets into the two subsequent super pages to
  4086. // also bear the same tag.
  4087. EXPECT_EQ(kTag, partition_alloc::internal::PartitionTagGetValue(
  4088. static_cast<char*>(object) + kSuperPageSize));
  4089. EXPECT_EQ(kTag, partition_alloc::internal::PartitionTagGetValue(
  4090. static_cast<char*>(object) + (2 * kSuperPageSize)));
  4091. EXPECT_EQ(kTag, partition_alloc::internal::PartitionTagGetValue(
  4092. static_cast<char*>(object) + (3 * kSuperPageSize) - 1));
  4093. allocator.root()->Free(object);
  4094. }
  4095. #endif // defined(PA_USE_MTE_CHECKED_PTR_WITH_64_BITS_POINTERS)
  4096. #if BUILDFLAG(PA_IS_CAST_ANDROID) && BUILDFLAG(USE_PARTITION_ALLOC_AS_MALLOC)
  4097. extern "C" {
  4098. void* __real_malloc(size_t);
  4099. } // extern "C"
  4100. TEST_P(PartitionAllocTest, HandleMixedAllocations) {
  4101. void* ptr = __real_malloc(12);
  4102. // Should not crash, no test assertion.
  4103. free(ptr);
  4104. }
  4105. #endif
  4106. TEST_P(PartitionAllocTest, SortFreelist) {
  4107. const size_t count = 100;
  4108. const size_t allocation_size = 1;
  4109. void* first_ptr = allocator.root()->Alloc(allocation_size, "");
  4110. std::vector<void*> allocations;
  4111. for (size_t i = 0; i < count; ++i)
  4112. allocations.push_back(allocator.root()->Alloc(allocation_size, ""));
  4113. // Shuffle and free memory out of order.
  4114. std::random_device rd;
  4115. std::mt19937 generator(rd());
  4116. std::shuffle(allocations.begin(), allocations.end(), generator);
  4117. // Keep one allocation alive (first_ptr), so that the SlotSpan is not fully
  4118. // empty.
  4119. for (void* ptr : allocations)
  4120. allocator.root()->Free(ptr);
  4121. allocations.clear();
  4122. allocator.root()->PurgeMemory(PurgeFlags::kDiscardUnusedSystemPages);
  4123. size_t bucket_index = SizeToIndex(allocation_size + kExtraAllocSize);
  4124. auto& bucket = allocator.root()->buckets[bucket_index];
  4125. EXPECT_TRUE(bucket.active_slot_spans_head->freelist_is_sorted());
  4126. // Can sort again.
  4127. allocator.root()->PurgeMemory(PurgeFlags::kDiscardUnusedSystemPages);
  4128. EXPECT_TRUE(bucket.active_slot_spans_head->freelist_is_sorted());
  4129. for (size_t i = 0; i < count; ++i) {
  4130. allocations.push_back(allocator.root()->Alloc(allocation_size, ""));
  4131. // Allocating keeps the freelist sorted.
  4132. EXPECT_TRUE(bucket.active_slot_spans_head->freelist_is_sorted());
  4133. }
  4134. // Check that it is sorted.
  4135. for (size_t i = 1; i < allocations.size(); i++) {
  4136. EXPECT_LT(UntagPtr(allocations[i - 1]), UntagPtr(allocations[i]));
  4137. }
  4138. for (void* ptr : allocations) {
  4139. allocator.root()->Free(ptr);
  4140. // Free()-ing memory destroys order. Not looking at the head of the active
  4141. // list, as it is not necessarily the one from which |ptr| came from.
  4142. auto* slot_span =
  4143. SlotSpan::FromSlotStart(allocator.root()->ObjectToSlotStart(ptr));
  4144. EXPECT_FALSE(slot_span->freelist_is_sorted());
  4145. }
  4146. allocator.root()->Free(first_ptr);
  4147. }
  4148. #if BUILDFLAG(USE_PARTITION_ALLOC_AS_MALLOC) && BUILDFLAG(IS_LINUX) && \
  4149. defined(ARCH_CPU_64_BITS)
  4150. TEST_P(PartitionAllocTest, CrashOnUnknownPointer) {
  4151. int not_a_heap_object = 42;
  4152. EXPECT_DEATH(allocator.root()->Free(&not_a_heap_object), "");
  4153. }
  4154. #endif // BUILDFLAG(USE_PARTITION_ALLOC_AS_MALLOC) && BUILDFLAG(IS_LINUX) &&
  4155. // defined(ARCH_CPU_64_BITS)
  4156. #if BUILDFLAG(USE_PARTITION_ALLOC_AS_MALLOC) && BUILDFLAG(IS_MAC)
  4157. // Adapted from crashpad tests.
  4158. class ScopedOpenCLNoOpKernel {
  4159. public:
  4160. ScopedOpenCLNoOpKernel()
  4161. : context_(nullptr),
  4162. program_(nullptr),
  4163. kernel_(nullptr),
  4164. success_(false) {}
  4165. ScopedOpenCLNoOpKernel(const ScopedOpenCLNoOpKernel&) = delete;
  4166. ScopedOpenCLNoOpKernel& operator=(const ScopedOpenCLNoOpKernel&) = delete;
  4167. ~ScopedOpenCLNoOpKernel() {
  4168. if (kernel_) {
  4169. cl_int rv = clReleaseKernel(kernel_);
  4170. EXPECT_EQ(rv, CL_SUCCESS) << "clReleaseKernel";
  4171. }
  4172. if (program_) {
  4173. cl_int rv = clReleaseProgram(program_);
  4174. EXPECT_EQ(rv, CL_SUCCESS) << "clReleaseProgram";
  4175. }
  4176. if (context_) {
  4177. cl_int rv = clReleaseContext(context_);
  4178. EXPECT_EQ(rv, CL_SUCCESS) << "clReleaseContext";
  4179. }
  4180. }
  4181. void SetUp() {
  4182. cl_platform_id platform_id;
  4183. cl_int rv = clGetPlatformIDs(1, &platform_id, nullptr);
  4184. ASSERT_EQ(rv, CL_SUCCESS) << "clGetPlatformIDs";
  4185. cl_device_id device_id;
  4186. rv =
  4187. clGetDeviceIDs(platform_id, CL_DEVICE_TYPE_CPU, 1, &device_id, nullptr);
  4188. #if defined(ARCH_CPU_ARM64)
  4189. // CL_DEVICE_TYPE_CPU doesn’t seem to work at all on arm64, meaning that
  4190. // these weird OpenCL modules probably don’t show up there at all. Keep this
  4191. // test even on arm64 in case this ever does start working.
  4192. if (rv == CL_INVALID_VALUE) {
  4193. return;
  4194. }
  4195. #endif // ARCH_CPU_ARM64
  4196. ASSERT_EQ(rv, CL_SUCCESS) << "clGetDeviceIDs";
  4197. context_ = clCreateContext(nullptr, 1, &device_id, nullptr, nullptr, &rv);
  4198. ASSERT_EQ(rv, CL_SUCCESS) << "clCreateContext";
  4199. const char* sources[] = {
  4200. "__kernel void NoOp(void) {barrier(CLK_LOCAL_MEM_FENCE);}",
  4201. };
  4202. const size_t source_lengths[] = {
  4203. strlen(sources[0]),
  4204. };
  4205. static_assert(std::size(sources) == std::size(source_lengths),
  4206. "arrays must be parallel");
  4207. program_ = clCreateProgramWithSource(context_, std::size(sources), sources,
  4208. source_lengths, &rv);
  4209. ASSERT_EQ(rv, CL_SUCCESS) << "clCreateProgramWithSource";
  4210. rv = clBuildProgram(program_, 1, &device_id, "-cl-opt-disable", nullptr,
  4211. nullptr);
  4212. ASSERT_EQ(rv, CL_SUCCESS) << "clBuildProgram";
  4213. kernel_ = clCreateKernel(program_, "NoOp", &rv);
  4214. ASSERT_EQ(rv, CL_SUCCESS) << "clCreateKernel";
  4215. success_ = true;
  4216. }
  4217. bool success() const { return success_; }
  4218. private:
  4219. cl_context context_;
  4220. cl_program program_;
  4221. cl_kernel kernel_;
  4222. bool success_;
  4223. };
  4224. // On macOS 10.11, allocations are made with PartitionAlloc, but the pointer
  4225. // is incorrectly passed by CoreFoundation to the previous default zone,
  4226. // causing crashes. This is intended to detect these issues regressing in future
  4227. // versions of macOS.
  4228. TEST_P(PartitionAllocTest, OpenCL) {
  4229. ScopedOpenCLNoOpKernel kernel;
  4230. kernel.SetUp();
  4231. #if !defined(ARCH_CPU_ARM64)
  4232. ASSERT_TRUE(kernel.success());
  4233. #endif
  4234. }
  4235. #endif // BUILDFLAG(USE_PARTITION_ALLOC_AS_MALLOC) && BUILDFLAG(IS_MAC)
  4236. TEST_P(PartitionAllocTest, SmallSlotSpanWaste) {
  4237. for (PartitionRoot<ThreadSafe>::Bucket& bucket : allocator.root()->buckets) {
  4238. size_t slot_size = bucket.slot_size;
  4239. if (slot_size == kInvalidBucketSize)
  4240. continue;
  4241. size_t small_system_page_count =
  4242. partition_alloc::internal::ComputeSystemPagesPerSlotSpan(
  4243. bucket.slot_size, true);
  4244. size_t small_waste =
  4245. (small_system_page_count * SystemPageSize()) % slot_size;
  4246. EXPECT_LT(small_waste, .05 * SystemPageSize());
  4247. if (slot_size <= MaxRegularSlotSpanSize())
  4248. EXPECT_LE(small_system_page_count, MaxSystemPagesPerRegularSlotSpan());
  4249. }
  4250. }
  4251. TEST_P(PartitionAllocTest, SortActiveSlotSpans) {
  4252. auto run_test = [](size_t count) {
  4253. PartitionBucket<ThreadSafe> bucket;
  4254. bucket.Init(16);
  4255. bucket.active_slot_spans_head = nullptr;
  4256. std::vector<SlotSpanMetadata<ThreadSafe>> slot_spans;
  4257. slot_spans.reserve(count);
  4258. // Add slot spans with random freelist length.
  4259. for (size_t i = 0; i < count; i++) {
  4260. slot_spans.emplace_back(&bucket);
  4261. auto& slot_span = slot_spans.back();
  4262. slot_span.num_unprovisioned_slots =
  4263. partition_alloc::internal::base::RandGenerator(
  4264. bucket.get_slots_per_span() / 2);
  4265. slot_span.num_allocated_slots =
  4266. partition_alloc::internal::base::RandGenerator(
  4267. bucket.get_slots_per_span() - slot_span.num_unprovisioned_slots);
  4268. slot_span.next_slot_span = bucket.active_slot_spans_head;
  4269. bucket.active_slot_spans_head = &slot_span;
  4270. }
  4271. bucket.SortActiveSlotSpans();
  4272. std::set<SlotSpanMetadata<ThreadSafe>*> seen_slot_spans;
  4273. std::vector<SlotSpanMetadata<ThreadSafe>*> sorted_slot_spans;
  4274. for (auto* slot_span = bucket.active_slot_spans_head; slot_span;
  4275. slot_span = slot_span->next_slot_span) {
  4276. sorted_slot_spans.push_back(slot_span);
  4277. seen_slot_spans.insert(slot_span);
  4278. }
  4279. // None repeated, none missing.
  4280. EXPECT_EQ(seen_slot_spans.size(), sorted_slot_spans.size());
  4281. EXPECT_EQ(seen_slot_spans.size(), slot_spans.size());
  4282. // The first slot spans are sorted.
  4283. size_t sorted_spans_count =
  4284. std::min(PartitionBucket<ThreadSafe>::kMaxSlotSpansToSort, count);
  4285. EXPECT_TRUE(std::is_sorted(sorted_slot_spans.begin(),
  4286. sorted_slot_spans.begin() + sorted_spans_count,
  4287. partition_alloc::internal::CompareSlotSpans));
  4288. // Slot spans with no freelist entries are at the end of the sorted run.
  4289. auto has_empty_freelist = [](SlotSpanMetadata<ThreadSafe>* a) {
  4290. return a->GetFreelistLength() == 0;
  4291. };
  4292. auto it = std::find_if(sorted_slot_spans.begin(),
  4293. sorted_slot_spans.begin() + sorted_spans_count,
  4294. has_empty_freelist);
  4295. if (it != sorted_slot_spans.end()) {
  4296. EXPECT_TRUE(std::all_of(it,
  4297. sorted_slot_spans.begin() + sorted_spans_count,
  4298. has_empty_freelist));
  4299. }
  4300. };
  4301. // Everything is sorted.
  4302. run_test(PartitionBucket<ThreadSafe>::kMaxSlotSpansToSort / 2);
  4303. // Only the first slot spans are sorted.
  4304. run_test(PartitionBucket<ThreadSafe>::kMaxSlotSpansToSort * 2);
  4305. // Corner cases.
  4306. run_test(0);
  4307. run_test(1);
  4308. }
  4309. } // namespace partition_alloc::internal
  4310. #endif // !defined(MEMORY_TOOL_REPLACES_ALLOCATOR)