memory.c 160 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * linux/mm/memory.c
  4. *
  5. * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
  6. */
  7. /*
  8. * demand-loading started 01.12.91 - seems it is high on the list of
  9. * things wanted, and it should be easy to implement. - Linus
  10. */
  11. /*
  12. * Ok, demand-loading was easy, shared pages a little bit tricker. Shared
  13. * pages started 02.12.91, seems to work. - Linus.
  14. *
  15. * Tested sharing by executing about 30 /bin/sh: under the old kernel it
  16. * would have taken more than the 6M I have free, but it worked well as
  17. * far as I could see.
  18. *
  19. * Also corrected some "invalidate()"s - I wasn't doing enough of them.
  20. */
  21. /*
  22. * Real VM (paging to/from disk) started 18.12.91. Much more work and
  23. * thought has to go into this. Oh, well..
  24. * 19.12.91 - works, somewhat. Sometimes I get faults, don't know why.
  25. * Found it. Everything seems to work now.
  26. * 20.12.91 - Ok, making the swap-device changeable like the root.
  27. */
  28. /*
  29. * 05.04.94 - Multi-page memory management added for v1.1.
  30. * Idea by Alex Bligh (alex@cconcepts.co.uk)
  31. *
  32. * 16.07.99 - Support of BIGMEM added by Gerhard Wichert, Siemens AG
  33. * (Gerhard.Wichert@pdb.siemens.de)
  34. *
  35. * Aug/Sep 2004 Changed to four level page tables (Andi Kleen)
  36. */
  37. #include <linux/kernel_stat.h>
  38. #include <linux/mm.h>
  39. #include <linux/sched/mm.h>
  40. #include <linux/sched/coredump.h>
  41. #include <linux/sched/numa_balancing.h>
  42. #include <linux/sched/task.h>
  43. #include <linux/hugetlb.h>
  44. #include <linux/mman.h>
  45. #include <linux/swap.h>
  46. #include <linux/highmem.h>
  47. #include <linux/pagemap.h>
  48. #include <linux/memremap.h>
  49. #include <linux/ksm.h>
  50. #include <linux/rmap.h>
  51. #include <linux/export.h>
  52. #include <linux/delayacct.h>
  53. #include <linux/init.h>
  54. #include <linux/pfn_t.h>
  55. #include <linux/writeback.h>
  56. #include <linux/memcontrol.h>
  57. #include <linux/mmu_notifier.h>
  58. #include <linux/swapops.h>
  59. #include <linux/elf.h>
  60. #include <linux/gfp.h>
  61. #include <linux/migrate.h>
  62. #include <linux/string.h>
  63. #include <linux/debugfs.h>
  64. #include <linux/userfaultfd_k.h>
  65. #include <linux/dax.h>
  66. #include <linux/oom.h>
  67. #include <linux/numa.h>
  68. #include <linux/perf_event.h>
  69. #include <linux/ptrace.h>
  70. #include <linux/vmalloc.h>
  71. #include <trace/hooks/mm.h>
  72. #include <trace/events/kmem.h>
  73. #include <asm/io.h>
  74. #include <asm/mmu_context.h>
  75. #include <asm/pgalloc.h>
  76. #include <linux/uaccess.h>
  77. #include <asm/tlb.h>
  78. #include <asm/tlbflush.h>
  79. #include "pgalloc-track.h"
  80. #include "internal.h"
  81. #define CREATE_TRACE_POINTS
  82. #include <trace/events/pagefault.h>
  83. #if defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) && !defined(CONFIG_COMPILE_TEST)
  84. #warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid.
  85. #endif
  86. #ifndef CONFIG_NEED_MULTIPLE_NODES
  87. /* use the per-pgdat data instead for discontigmem - mbligh */
  88. unsigned long max_mapnr;
  89. EXPORT_SYMBOL(max_mapnr);
  90. struct page *mem_map;
  91. EXPORT_SYMBOL(mem_map);
  92. #endif
  93. /*
  94. * A number of key systems in x86 including ioremap() rely on the assumption
  95. * that high_memory defines the upper bound on direct map memory, then end
  96. * of ZONE_NORMAL. Under CONFIG_DISCONTIG this means that max_low_pfn and
  97. * highstart_pfn must be the same; there must be no gap between ZONE_NORMAL
  98. * and ZONE_HIGHMEM.
  99. */
  100. void *high_memory;
  101. EXPORT_SYMBOL(high_memory);
  102. /*
  103. * Randomize the address space (stacks, mmaps, brk, etc.).
  104. *
  105. * ( When CONFIG_COMPAT_BRK=y we exclude brk from randomization,
  106. * as ancient (libc5 based) binaries can segfault. )
  107. */
  108. int randomize_va_space __read_mostly =
  109. #ifdef CONFIG_COMPAT_BRK
  110. 1;
  111. #else
  112. 2;
  113. #endif
  114. #ifndef arch_faults_on_old_pte
  115. static inline bool arch_faults_on_old_pte(void)
  116. {
  117. /*
  118. * Those arches which don't have hw access flag feature need to
  119. * implement their own helper. By default, "true" means pagefault
  120. * will be hit on old pte.
  121. */
  122. return true;
  123. }
  124. #endif
  125. #ifndef arch_wants_old_prefaulted_pte
  126. static inline bool arch_wants_old_prefaulted_pte(void)
  127. {
  128. /*
  129. * Transitioning a PTE from 'old' to 'young' can be expensive on
  130. * some architectures, even if it's performed in hardware. By
  131. * default, "false" means prefaulted entries will be 'young'.
  132. */
  133. return false;
  134. }
  135. #endif
  136. static int __init disable_randmaps(char *s)
  137. {
  138. randomize_va_space = 0;
  139. return 1;
  140. }
  141. __setup("norandmaps", disable_randmaps);
  142. unsigned long zero_pfn __read_mostly;
  143. EXPORT_SYMBOL(zero_pfn);
  144. unsigned long highest_memmap_pfn __read_mostly;
  145. /*
  146. * CONFIG_MMU architectures set up ZERO_PAGE in their paging_init()
  147. */
  148. static int __init init_zero_pfn(void)
  149. {
  150. zero_pfn = page_to_pfn(ZERO_PAGE(0));
  151. return 0;
  152. }
  153. early_initcall(init_zero_pfn);
  154. /*
  155. * Only trace rss_stat when there is a 512kb cross over.
  156. * Smaller changes may be lost unless every small change is
  157. * crossing into or returning to a 512kb boundary.
  158. */
  159. #define TRACE_MM_COUNTER_THRESHOLD 128
  160. void mm_trace_rss_stat(struct mm_struct *mm, int member, long count,
  161. long value)
  162. {
  163. long thresh_mask = ~(TRACE_MM_COUNTER_THRESHOLD - 1);
  164. /* Threshold roll-over, trace it */
  165. if ((count & thresh_mask) != ((count - value) & thresh_mask))
  166. trace_rss_stat(mm, member, count);
  167. }
  168. EXPORT_SYMBOL_GPL(mm_trace_rss_stat);
  169. #if defined(SPLIT_RSS_COUNTING)
  170. void sync_mm_rss(struct mm_struct *mm)
  171. {
  172. int i;
  173. for (i = 0; i < NR_MM_COUNTERS; i++) {
  174. if (current->rss_stat.count[i]) {
  175. add_mm_counter(mm, i, current->rss_stat.count[i]);
  176. current->rss_stat.count[i] = 0;
  177. }
  178. }
  179. current->rss_stat.events = 0;
  180. }
  181. static void add_mm_counter_fast(struct mm_struct *mm, int member, int val)
  182. {
  183. struct task_struct *task = current;
  184. if (likely(task->mm == mm))
  185. task->rss_stat.count[member] += val;
  186. else
  187. add_mm_counter(mm, member, val);
  188. }
  189. #define inc_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, 1)
  190. #define dec_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, -1)
  191. /* sync counter once per 64 page faults */
  192. #define TASK_RSS_EVENTS_THRESH (64)
  193. static void check_sync_rss_stat(struct task_struct *task)
  194. {
  195. if (unlikely(task != current))
  196. return;
  197. if (unlikely(task->rss_stat.events++ > TASK_RSS_EVENTS_THRESH))
  198. sync_mm_rss(task->mm);
  199. }
  200. #else /* SPLIT_RSS_COUNTING */
  201. #define inc_mm_counter_fast(mm, member) inc_mm_counter(mm, member)
  202. #define dec_mm_counter_fast(mm, member) dec_mm_counter(mm, member)
  203. static void check_sync_rss_stat(struct task_struct *task)
  204. {
  205. }
  206. #endif /* SPLIT_RSS_COUNTING */
  207. /*
  208. * Note: this doesn't free the actual pages themselves. That
  209. * has been handled earlier when unmapping all the memory regions.
  210. */
  211. static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
  212. unsigned long addr)
  213. {
  214. pgtable_t token = pmd_pgtable(*pmd);
  215. pmd_clear(pmd);
  216. pte_free_tlb(tlb, token, addr);
  217. mm_dec_nr_ptes(tlb->mm);
  218. }
  219. static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
  220. unsigned long addr, unsigned long end,
  221. unsigned long floor, unsigned long ceiling)
  222. {
  223. pmd_t *pmd;
  224. unsigned long next;
  225. unsigned long start;
  226. start = addr;
  227. pmd = pmd_offset(pud, addr);
  228. do {
  229. next = pmd_addr_end(addr, end);
  230. if (pmd_none_or_clear_bad(pmd))
  231. continue;
  232. free_pte_range(tlb, pmd, addr);
  233. } while (pmd++, addr = next, addr != end);
  234. start &= PUD_MASK;
  235. if (start < floor)
  236. return;
  237. if (ceiling) {
  238. ceiling &= PUD_MASK;
  239. if (!ceiling)
  240. return;
  241. }
  242. if (end - 1 > ceiling - 1)
  243. return;
  244. pmd = pmd_offset(pud, start);
  245. pud_clear(pud);
  246. pmd_free_tlb(tlb, pmd, start);
  247. mm_dec_nr_pmds(tlb->mm);
  248. }
  249. static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,
  250. unsigned long addr, unsigned long end,
  251. unsigned long floor, unsigned long ceiling)
  252. {
  253. pud_t *pud;
  254. unsigned long next;
  255. unsigned long start;
  256. start = addr;
  257. pud = pud_offset(p4d, addr);
  258. do {
  259. next = pud_addr_end(addr, end);
  260. if (pud_none_or_clear_bad(pud))
  261. continue;
  262. free_pmd_range(tlb, pud, addr, next, floor, ceiling);
  263. } while (pud++, addr = next, addr != end);
  264. start &= P4D_MASK;
  265. if (start < floor)
  266. return;
  267. if (ceiling) {
  268. ceiling &= P4D_MASK;
  269. if (!ceiling)
  270. return;
  271. }
  272. if (end - 1 > ceiling - 1)
  273. return;
  274. pud = pud_offset(p4d, start);
  275. p4d_clear(p4d);
  276. pud_free_tlb(tlb, pud, start);
  277. mm_dec_nr_puds(tlb->mm);
  278. }
  279. static inline void free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd,
  280. unsigned long addr, unsigned long end,
  281. unsigned long floor, unsigned long ceiling)
  282. {
  283. p4d_t *p4d;
  284. unsigned long next;
  285. unsigned long start;
  286. start = addr;
  287. p4d = p4d_offset(pgd, addr);
  288. do {
  289. next = p4d_addr_end(addr, end);
  290. if (p4d_none_or_clear_bad(p4d))
  291. continue;
  292. free_pud_range(tlb, p4d, addr, next, floor, ceiling);
  293. } while (p4d++, addr = next, addr != end);
  294. start &= PGDIR_MASK;
  295. if (start < floor)
  296. return;
  297. if (ceiling) {
  298. ceiling &= PGDIR_MASK;
  299. if (!ceiling)
  300. return;
  301. }
  302. if (end - 1 > ceiling - 1)
  303. return;
  304. p4d = p4d_offset(pgd, start);
  305. pgd_clear(pgd);
  306. p4d_free_tlb(tlb, p4d, start);
  307. }
  308. /*
  309. * This function frees user-level page tables of a process.
  310. */
  311. void free_pgd_range(struct mmu_gather *tlb,
  312. unsigned long addr, unsigned long end,
  313. unsigned long floor, unsigned long ceiling)
  314. {
  315. pgd_t *pgd;
  316. unsigned long next;
  317. /*
  318. * The next few lines have given us lots of grief...
  319. *
  320. * Why are we testing PMD* at this top level? Because often
  321. * there will be no work to do at all, and we'd prefer not to
  322. * go all the way down to the bottom just to discover that.
  323. *
  324. * Why all these "- 1"s? Because 0 represents both the bottom
  325. * of the address space and the top of it (using -1 for the
  326. * top wouldn't help much: the masks would do the wrong thing).
  327. * The rule is that addr 0 and floor 0 refer to the bottom of
  328. * the address space, but end 0 and ceiling 0 refer to the top
  329. * Comparisons need to use "end - 1" and "ceiling - 1" (though
  330. * that end 0 case should be mythical).
  331. *
  332. * Wherever addr is brought up or ceiling brought down, we must
  333. * be careful to reject "the opposite 0" before it confuses the
  334. * subsequent tests. But what about where end is brought down
  335. * by PMD_SIZE below? no, end can't go down to 0 there.
  336. *
  337. * Whereas we round start (addr) and ceiling down, by different
  338. * masks at different levels, in order to test whether a table
  339. * now has no other vmas using it, so can be freed, we don't
  340. * bother to round floor or end up - the tests don't need that.
  341. */
  342. addr &= PMD_MASK;
  343. if (addr < floor) {
  344. addr += PMD_SIZE;
  345. if (!addr)
  346. return;
  347. }
  348. if (ceiling) {
  349. ceiling &= PMD_MASK;
  350. if (!ceiling)
  351. return;
  352. }
  353. if (end - 1 > ceiling - 1)
  354. end -= PMD_SIZE;
  355. if (addr > end - 1)
  356. return;
  357. /*
  358. * We add page table cache pages with PAGE_SIZE,
  359. * (see pte_free_tlb()), flush the tlb if we need
  360. */
  361. tlb_change_page_size(tlb, PAGE_SIZE);
  362. pgd = pgd_offset(tlb->mm, addr);
  363. do {
  364. next = pgd_addr_end(addr, end);
  365. if (pgd_none_or_clear_bad(pgd))
  366. continue;
  367. free_p4d_range(tlb, pgd, addr, next, floor, ceiling);
  368. } while (pgd++, addr = next, addr != end);
  369. }
  370. void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma,
  371. unsigned long floor, unsigned long ceiling)
  372. {
  373. while (vma) {
  374. struct vm_area_struct *next = vma->vm_next;
  375. unsigned long addr = vma->vm_start;
  376. /*
  377. * Hide vma from rmap and truncate_pagecache before freeing
  378. * pgtables
  379. */
  380. vm_write_begin(vma);
  381. unlink_anon_vmas(vma);
  382. vm_write_end(vma);
  383. unlink_file_vma(vma);
  384. if (is_vm_hugetlb_page(vma)) {
  385. hugetlb_free_pgd_range(tlb, addr, vma->vm_end,
  386. floor, next ? next->vm_start : ceiling);
  387. } else {
  388. /*
  389. * Optimization: gather nearby vmas into one call down
  390. */
  391. while (next && next->vm_start <= vma->vm_end + PMD_SIZE
  392. && !is_vm_hugetlb_page(next)) {
  393. vma = next;
  394. next = vma->vm_next;
  395. vm_write_begin(vma);
  396. unlink_anon_vmas(vma);
  397. vm_write_end(vma);
  398. unlink_file_vma(vma);
  399. }
  400. free_pgd_range(tlb, addr, vma->vm_end,
  401. floor, next ? next->vm_start : ceiling);
  402. }
  403. vma = next;
  404. }
  405. }
  406. int __pte_alloc(struct mm_struct *mm, pmd_t *pmd)
  407. {
  408. spinlock_t *ptl;
  409. pgtable_t new = pte_alloc_one(mm);
  410. if (!new)
  411. return -ENOMEM;
  412. /*
  413. * Ensure all pte setup (eg. pte page lock and page clearing) are
  414. * visible before the pte is made visible to other CPUs by being
  415. * put into page tables.
  416. *
  417. * The other side of the story is the pointer chasing in the page
  418. * table walking code (when walking the page table without locking;
  419. * ie. most of the time). Fortunately, these data accesses consist
  420. * of a chain of data-dependent loads, meaning most CPUs (alpha
  421. * being the notable exception) will already guarantee loads are
  422. * seen in-order. See the alpha page table accessors for the
  423. * smp_rmb() barriers in page table walking code.
  424. */
  425. smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */
  426. ptl = pmd_lock(mm, pmd);
  427. if (likely(pmd_none(*pmd))) { /* Has another populated it ? */
  428. mm_inc_nr_ptes(mm);
  429. pmd_populate(mm, pmd, new);
  430. new = NULL;
  431. }
  432. spin_unlock(ptl);
  433. if (new)
  434. pte_free(mm, new);
  435. return 0;
  436. }
  437. int __pte_alloc_kernel(pmd_t *pmd)
  438. {
  439. pte_t *new = pte_alloc_one_kernel(&init_mm);
  440. if (!new)
  441. return -ENOMEM;
  442. smp_wmb(); /* See comment in __pte_alloc */
  443. spin_lock(&init_mm.page_table_lock);
  444. if (likely(pmd_none(*pmd))) { /* Has another populated it ? */
  445. pmd_populate_kernel(&init_mm, pmd, new);
  446. new = NULL;
  447. }
  448. spin_unlock(&init_mm.page_table_lock);
  449. if (new)
  450. pte_free_kernel(&init_mm, new);
  451. return 0;
  452. }
  453. static inline void init_rss_vec(int *rss)
  454. {
  455. memset(rss, 0, sizeof(int) * NR_MM_COUNTERS);
  456. }
  457. static inline void add_mm_rss_vec(struct mm_struct *mm, int *rss)
  458. {
  459. int i;
  460. if (current->mm == mm)
  461. sync_mm_rss(mm);
  462. for (i = 0; i < NR_MM_COUNTERS; i++)
  463. if (rss[i])
  464. add_mm_counter(mm, i, rss[i]);
  465. }
  466. /*
  467. * This function is called to print an error when a bad pte
  468. * is found. For example, we might have a PFN-mapped pte in
  469. * a region that doesn't allow it.
  470. *
  471. * The calling function must still handle the error.
  472. */
  473. static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr,
  474. pte_t pte, struct page *page)
  475. {
  476. pgd_t *pgd = pgd_offset(vma->vm_mm, addr);
  477. p4d_t *p4d = p4d_offset(pgd, addr);
  478. pud_t *pud = pud_offset(p4d, addr);
  479. pmd_t *pmd = pmd_offset(pud, addr);
  480. struct address_space *mapping;
  481. pgoff_t index;
  482. static unsigned long resume;
  483. static unsigned long nr_shown;
  484. static unsigned long nr_unshown;
  485. /*
  486. * Allow a burst of 60 reports, then keep quiet for that minute;
  487. * or allow a steady drip of one report per second.
  488. */
  489. if (nr_shown == 60) {
  490. if (time_before(jiffies, resume)) {
  491. nr_unshown++;
  492. return;
  493. }
  494. if (nr_unshown) {
  495. pr_alert("BUG: Bad page map: %lu messages suppressed\n",
  496. nr_unshown);
  497. nr_unshown = 0;
  498. }
  499. nr_shown = 0;
  500. }
  501. if (nr_shown++ == 0)
  502. resume = jiffies + 60 * HZ;
  503. mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL;
  504. index = linear_page_index(vma, addr);
  505. pr_alert("BUG: Bad page map in process %s pte:%08llx pmd:%08llx\n",
  506. current->comm,
  507. (long long)pte_val(pte), (long long)pmd_val(*pmd));
  508. if (page)
  509. dump_page(page, "bad pte");
  510. pr_alert("addr:%px vm_flags:%08lx anon_vma:%px mapping:%px index:%lx\n",
  511. (void *)addr, READ_ONCE(vma->vm_flags), vma->anon_vma, mapping, index);
  512. pr_alert("file:%pD fault:%ps mmap:%ps readpage:%ps\n",
  513. vma->vm_file,
  514. vma->vm_ops ? vma->vm_ops->fault : NULL,
  515. vma->vm_file ? vma->vm_file->f_op->mmap : NULL,
  516. mapping ? mapping->a_ops->readpage : NULL);
  517. dump_stack();
  518. add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
  519. }
  520. /*
  521. * __vm_normal_page -- This function gets the "struct page" associated with
  522. * a pte.
  523. *
  524. * "Special" mappings do not wish to be associated with a "struct page" (either
  525. * it doesn't exist, or it exists but they don't want to touch it). In this
  526. * case, NULL is returned here. "Normal" mappings do have a struct page.
  527. *
  528. * There are 2 broad cases. Firstly, an architecture may define a pte_special()
  529. * pte bit, in which case this function is trivial. Secondly, an architecture
  530. * may not have a spare pte bit, which requires a more complicated scheme,
  531. * described below.
  532. *
  533. * A raw VM_PFNMAP mapping (ie. one that is not COWed) is always considered a
  534. * special mapping (even if there are underlying and valid "struct pages").
  535. * COWed pages of a VM_PFNMAP are always normal.
  536. *
  537. * The way we recognize COWed pages within VM_PFNMAP mappings is through the
  538. * rules set up by "remap_pfn_range()": the vma will have the VM_PFNMAP bit
  539. * set, and the vm_pgoff will point to the first PFN mapped: thus every special
  540. * mapping will always honor the rule
  541. *
  542. * pfn_of_page == vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT)
  543. *
  544. * And for normal mappings this is false.
  545. *
  546. * This restricts such mappings to be a linear translation from virtual address
  547. * to pfn. To get around this restriction, we allow arbitrary mappings so long
  548. * as the vma is not a COW mapping; in that case, we know that all ptes are
  549. * special (because none can have been COWed).
  550. *
  551. *
  552. * In order to support COW of arbitrary special mappings, we have VM_MIXEDMAP.
  553. *
  554. * VM_MIXEDMAP mappings can likewise contain memory with or without "struct
  555. * page" backing, however the difference is that _all_ pages with a struct
  556. * page (that is, those where pfn_valid is true) are refcounted and considered
  557. * normal pages by the VM. The disadvantage is that pages are refcounted
  558. * (which can be slower and simply not an option for some PFNMAP users). The
  559. * advantage is that we don't have to follow the strict linearity rule of
  560. * PFNMAP mappings in order to support COWable mappings.
  561. *
  562. */
  563. struct page *_vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
  564. pte_t pte, unsigned long vma_flags)
  565. {
  566. unsigned long pfn = pte_pfn(pte);
  567. if (IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL)) {
  568. if (likely(!pte_special(pte)))
  569. goto check_pfn;
  570. if (vma->vm_ops && vma->vm_ops->find_special_page)
  571. return vma->vm_ops->find_special_page(vma, addr);
  572. if (vma_flags & (VM_PFNMAP | VM_MIXEDMAP))
  573. return NULL;
  574. if (is_zero_pfn(pfn))
  575. return NULL;
  576. if (pte_devmap(pte))
  577. return NULL;
  578. print_bad_pte(vma, addr, pte, NULL);
  579. return NULL;
  580. }
  581. /* !CONFIG_ARCH_HAS_PTE_SPECIAL case follows: */
  582. /*
  583. * This part should never get called when CONFIG_SPECULATIVE_PAGE_FAULT
  584. * is set. This is mainly because we can't rely on vm_start.
  585. */
  586. if (unlikely(vma_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
  587. if (vma_flags & VM_MIXEDMAP) {
  588. if (!pfn_valid(pfn))
  589. return NULL;
  590. goto out;
  591. } else {
  592. unsigned long off;
  593. off = (addr - vma->vm_start) >> PAGE_SHIFT;
  594. if (pfn == vma->vm_pgoff + off)
  595. return NULL;
  596. if (!is_cow_mapping(vma_flags))
  597. return NULL;
  598. }
  599. }
  600. if (is_zero_pfn(pfn))
  601. return NULL;
  602. check_pfn:
  603. if (unlikely(pfn > highest_memmap_pfn)) {
  604. print_bad_pte(vma, addr, pte, NULL);
  605. return NULL;
  606. }
  607. /*
  608. * NOTE! We still have PageReserved() pages in the page tables.
  609. * eg. VDSO mappings can cause them to exist.
  610. */
  611. out:
  612. return pfn_to_page(pfn);
  613. }
  614. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  615. struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
  616. pmd_t pmd)
  617. {
  618. unsigned long pfn = pmd_pfn(pmd);
  619. /*
  620. * There is no pmd_special() but there may be special pmds, e.g.
  621. * in a direct-access (dax) mapping, so let's just replicate the
  622. * !CONFIG_ARCH_HAS_PTE_SPECIAL case from vm_normal_page() here.
  623. */
  624. if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
  625. if (vma->vm_flags & VM_MIXEDMAP) {
  626. if (!pfn_valid(pfn))
  627. return NULL;
  628. goto out;
  629. } else {
  630. unsigned long off;
  631. off = (addr - vma->vm_start) >> PAGE_SHIFT;
  632. if (pfn == vma->vm_pgoff + off)
  633. return NULL;
  634. if (!is_cow_mapping(vma->vm_flags))
  635. return NULL;
  636. }
  637. }
  638. if (pmd_devmap(pmd))
  639. return NULL;
  640. if (is_huge_zero_pmd(pmd))
  641. return NULL;
  642. if (unlikely(pfn > highest_memmap_pfn))
  643. return NULL;
  644. /*
  645. * NOTE! We still have PageReserved() pages in the page tables.
  646. * eg. VDSO mappings can cause them to exist.
  647. */
  648. out:
  649. return pfn_to_page(pfn);
  650. }
  651. #endif
  652. /*
  653. * copy one vm_area from one task to the other. Assumes the page tables
  654. * already present in the new task to be cleared in the whole range
  655. * covered by this vma.
  656. */
  657. static unsigned long
  658. copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
  659. pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *dst_vma,
  660. struct vm_area_struct *src_vma, unsigned long addr, int *rss)
  661. {
  662. unsigned long vm_flags = dst_vma->vm_flags;
  663. pte_t pte = *src_pte;
  664. struct page *page;
  665. swp_entry_t entry = pte_to_swp_entry(pte);
  666. if (likely(!non_swap_entry(entry))) {
  667. if (swap_duplicate(entry) < 0)
  668. return entry.val;
  669. /* make sure dst_mm is on swapoff's mmlist. */
  670. if (unlikely(list_empty(&dst_mm->mmlist))) {
  671. spin_lock(&mmlist_lock);
  672. if (list_empty(&dst_mm->mmlist))
  673. list_add(&dst_mm->mmlist,
  674. &src_mm->mmlist);
  675. spin_unlock(&mmlist_lock);
  676. }
  677. rss[MM_SWAPENTS]++;
  678. } else if (is_migration_entry(entry)) {
  679. page = migration_entry_to_page(entry);
  680. rss[mm_counter(page)]++;
  681. if (is_write_migration_entry(entry) &&
  682. is_cow_mapping(vm_flags)) {
  683. /*
  684. * COW mappings require pages in both
  685. * parent and child to be set to read.
  686. */
  687. make_migration_entry_read(&entry);
  688. pte = swp_entry_to_pte(entry);
  689. if (pte_swp_soft_dirty(*src_pte))
  690. pte = pte_swp_mksoft_dirty(pte);
  691. if (pte_swp_uffd_wp(*src_pte))
  692. pte = pte_swp_mkuffd_wp(pte);
  693. set_pte_at(src_mm, addr, src_pte, pte);
  694. }
  695. } else if (is_device_private_entry(entry)) {
  696. page = device_private_entry_to_page(entry);
  697. /*
  698. * Update rss count even for unaddressable pages, as
  699. * they should treated just like normal pages in this
  700. * respect.
  701. *
  702. * We will likely want to have some new rss counters
  703. * for unaddressable pages, at some point. But for now
  704. * keep things as they are.
  705. */
  706. get_page(page);
  707. rss[mm_counter(page)]++;
  708. page_dup_rmap(page, false);
  709. /*
  710. * We do not preserve soft-dirty information, because so
  711. * far, checkpoint/restore is the only feature that
  712. * requires that. And checkpoint/restore does not work
  713. * when a device driver is involved (you cannot easily
  714. * save and restore device driver state).
  715. */
  716. if (is_write_device_private_entry(entry) &&
  717. is_cow_mapping(vm_flags)) {
  718. make_device_private_entry_read(&entry);
  719. pte = swp_entry_to_pte(entry);
  720. if (pte_swp_uffd_wp(*src_pte))
  721. pte = pte_swp_mkuffd_wp(pte);
  722. set_pte_at(src_mm, addr, src_pte, pte);
  723. }
  724. }
  725. if (!userfaultfd_wp(dst_vma))
  726. pte = pte_swp_clear_uffd_wp(pte);
  727. set_pte_at(dst_mm, addr, dst_pte, pte);
  728. return 0;
  729. }
  730. /*
  731. * Copy a present and normal page if necessary.
  732. *
  733. * NOTE! The usual case is that this doesn't need to do
  734. * anything, and can just return a positive value. That
  735. * will let the caller know that it can just increase
  736. * the page refcount and re-use the pte the traditional
  737. * way.
  738. *
  739. * But _if_ we need to copy it because it needs to be
  740. * pinned in the parent (and the child should get its own
  741. * copy rather than just a reference to the same page),
  742. * we'll do that here and return zero to let the caller
  743. * know we're done.
  744. *
  745. * And if we need a pre-allocated page but don't yet have
  746. * one, return a negative error to let the preallocation
  747. * code know so that it can do so outside the page table
  748. * lock.
  749. */
  750. static inline int
  751. copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
  752. pte_t *dst_pte, pte_t *src_pte, unsigned long addr, int *rss,
  753. struct page **prealloc, pte_t pte, struct page *page)
  754. {
  755. struct mm_struct *src_mm = src_vma->vm_mm;
  756. struct page *new_page;
  757. if (!is_cow_mapping(src_vma->vm_flags))
  758. return 1;
  759. /*
  760. * What we want to do is to check whether this page may
  761. * have been pinned by the parent process. If so,
  762. * instead of wrprotect the pte on both sides, we copy
  763. * the page immediately so that we'll always guarantee
  764. * the pinned page won't be randomly replaced in the
  765. * future.
  766. *
  767. * The page pinning checks are just "has this mm ever
  768. * seen pinning", along with the (inexact) check of
  769. * the page count. That might give false positives for
  770. * for pinning, but it will work correctly.
  771. */
  772. if (likely(!atomic_read(&src_mm->has_pinned)))
  773. return 1;
  774. if (likely(!page_maybe_dma_pinned(page)))
  775. return 1;
  776. new_page = *prealloc;
  777. if (!new_page)
  778. return -EAGAIN;
  779. /*
  780. * We have a prealloc page, all good! Take it
  781. * over and copy the page & arm it.
  782. */
  783. *prealloc = NULL;
  784. copy_user_highpage(new_page, page, addr, src_vma);
  785. __SetPageUptodate(new_page);
  786. page_add_new_anon_rmap(new_page, dst_vma, addr, false);
  787. lru_cache_add_inactive_or_unevictable(new_page, dst_vma);
  788. rss[mm_counter(new_page)]++;
  789. /* All done, just insert the new page copy in the child */
  790. pte = mk_pte(new_page, dst_vma->vm_page_prot);
  791. pte = maybe_mkwrite(pte_mkdirty(pte), dst_vma->vm_flags);
  792. if (userfaultfd_pte_wp(dst_vma, *src_pte))
  793. /* Uffd-wp needs to be delivered to dest pte as well */
  794. pte = pte_wrprotect(pte_mkuffd_wp(pte));
  795. set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte);
  796. return 0;
  797. }
  798. /*
  799. * Copy one pte. Returns 0 if succeeded, or -EAGAIN if one preallocated page
  800. * is required to copy this pte.
  801. */
  802. static inline int
  803. copy_present_pte(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
  804. pte_t *dst_pte, pte_t *src_pte, unsigned long addr, int *rss,
  805. struct page **prealloc)
  806. {
  807. struct mm_struct *src_mm = src_vma->vm_mm;
  808. unsigned long vm_flags = src_vma->vm_flags;
  809. pte_t pte = *src_pte;
  810. struct page *page;
  811. page = vm_normal_page(src_vma, addr, pte);
  812. if (page) {
  813. int retval;
  814. retval = copy_present_page(dst_vma, src_vma, dst_pte, src_pte,
  815. addr, rss, prealloc, pte, page);
  816. if (retval <= 0)
  817. return retval;
  818. get_page(page);
  819. page_dup_rmap(page, false);
  820. rss[mm_counter(page)]++;
  821. }
  822. /*
  823. * If it's a COW mapping, write protect it both
  824. * in the parent and the child
  825. */
  826. if (is_cow_mapping(vm_flags) && pte_write(pte)) {
  827. ptep_set_wrprotect(src_mm, addr, src_pte);
  828. pte = pte_wrprotect(pte);
  829. }
  830. /*
  831. * If it's a shared mapping, mark it clean in
  832. * the child
  833. */
  834. if (vm_flags & VM_SHARED)
  835. pte = pte_mkclean(pte);
  836. pte = pte_mkold(pte);
  837. if (!userfaultfd_wp(dst_vma))
  838. pte = pte_clear_uffd_wp(pte);
  839. set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte);
  840. return 0;
  841. }
  842. static inline struct page *
  843. page_copy_prealloc(struct mm_struct *src_mm, struct vm_area_struct *vma,
  844. unsigned long addr)
  845. {
  846. struct page *new_page;
  847. new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, addr);
  848. if (!new_page)
  849. return NULL;
  850. if (mem_cgroup_charge(new_page, src_mm, GFP_KERNEL)) {
  851. put_page(new_page);
  852. return NULL;
  853. }
  854. cgroup_throttle_swaprate(new_page, GFP_KERNEL);
  855. return new_page;
  856. }
  857. static int
  858. copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
  859. pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
  860. unsigned long end)
  861. {
  862. struct mm_struct *dst_mm = dst_vma->vm_mm;
  863. struct mm_struct *src_mm = src_vma->vm_mm;
  864. pte_t *orig_src_pte, *orig_dst_pte;
  865. pte_t *src_pte, *dst_pte;
  866. spinlock_t *src_ptl, *dst_ptl;
  867. int progress, ret = 0;
  868. int rss[NR_MM_COUNTERS];
  869. swp_entry_t entry = (swp_entry_t){0};
  870. struct page *prealloc = NULL;
  871. again:
  872. progress = 0;
  873. init_rss_vec(rss);
  874. dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl);
  875. if (!dst_pte) {
  876. ret = -ENOMEM;
  877. goto out;
  878. }
  879. src_pte = pte_offset_map(src_pmd, addr);
  880. src_ptl = pte_lockptr(src_mm, src_pmd);
  881. spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
  882. orig_src_pte = src_pte;
  883. orig_dst_pte = dst_pte;
  884. arch_enter_lazy_mmu_mode();
  885. do {
  886. /*
  887. * We are holding two locks at this point - either of them
  888. * could generate latencies in another task on another CPU.
  889. */
  890. if (progress >= 32) {
  891. progress = 0;
  892. if (need_resched() ||
  893. spin_needbreak(src_ptl) || spin_needbreak(dst_ptl))
  894. break;
  895. }
  896. if (pte_none(*src_pte)) {
  897. progress++;
  898. continue;
  899. }
  900. if (unlikely(!pte_present(*src_pte))) {
  901. entry.val = copy_nonpresent_pte(dst_mm, src_mm,
  902. dst_pte, src_pte,
  903. dst_vma, src_vma,
  904. addr, rss);
  905. if (entry.val)
  906. break;
  907. progress += 8;
  908. continue;
  909. }
  910. /* copy_present_pte() will clear `*prealloc' if consumed */
  911. ret = copy_present_pte(dst_vma, src_vma, dst_pte, src_pte,
  912. addr, rss, &prealloc);
  913. /*
  914. * If we need a pre-allocated page for this pte, drop the
  915. * locks, allocate, and try again.
  916. */
  917. if (unlikely(ret == -EAGAIN))
  918. break;
  919. if (unlikely(prealloc)) {
  920. /*
  921. * pre-alloc page cannot be reused by next time so as
  922. * to strictly follow mempolicy (e.g., alloc_page_vma()
  923. * will allocate page according to address). This
  924. * could only happen if one pinned pte changed.
  925. */
  926. put_page(prealloc);
  927. prealloc = NULL;
  928. }
  929. progress += 8;
  930. } while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end);
  931. arch_leave_lazy_mmu_mode();
  932. spin_unlock(src_ptl);
  933. pte_unmap(orig_src_pte);
  934. add_mm_rss_vec(dst_mm, rss);
  935. pte_unmap_unlock(orig_dst_pte, dst_ptl);
  936. cond_resched();
  937. if (entry.val) {
  938. if (add_swap_count_continuation(entry, GFP_KERNEL) < 0) {
  939. ret = -ENOMEM;
  940. goto out;
  941. }
  942. entry.val = 0;
  943. } else if (ret) {
  944. WARN_ON_ONCE(ret != -EAGAIN);
  945. prealloc = page_copy_prealloc(src_mm, src_vma, addr);
  946. if (!prealloc)
  947. return -ENOMEM;
  948. /* We've captured and resolved the error. Reset, try again. */
  949. ret = 0;
  950. }
  951. if (addr != end)
  952. goto again;
  953. out:
  954. if (unlikely(prealloc))
  955. put_page(prealloc);
  956. return ret;
  957. }
  958. static inline int
  959. copy_pmd_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
  960. pud_t *dst_pud, pud_t *src_pud, unsigned long addr,
  961. unsigned long end)
  962. {
  963. struct mm_struct *dst_mm = dst_vma->vm_mm;
  964. struct mm_struct *src_mm = src_vma->vm_mm;
  965. pmd_t *src_pmd, *dst_pmd;
  966. unsigned long next;
  967. dst_pmd = pmd_alloc(dst_mm, dst_pud, addr);
  968. if (!dst_pmd)
  969. return -ENOMEM;
  970. src_pmd = pmd_offset(src_pud, addr);
  971. do {
  972. next = pmd_addr_end(addr, end);
  973. if (is_swap_pmd(*src_pmd) || pmd_trans_huge(*src_pmd)
  974. || pmd_devmap(*src_pmd)) {
  975. int err;
  976. VM_BUG_ON_VMA(next-addr != HPAGE_PMD_SIZE, src_vma);
  977. err = copy_huge_pmd(dst_mm, src_mm, dst_pmd, src_pmd,
  978. addr, dst_vma, src_vma);
  979. if (err == -ENOMEM)
  980. return -ENOMEM;
  981. if (!err)
  982. continue;
  983. /* fall through */
  984. }
  985. if (pmd_none_or_clear_bad(src_pmd))
  986. continue;
  987. if (copy_pte_range(dst_vma, src_vma, dst_pmd, src_pmd,
  988. addr, next))
  989. return -ENOMEM;
  990. } while (dst_pmd++, src_pmd++, addr = next, addr != end);
  991. return 0;
  992. }
  993. static inline int
  994. copy_pud_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
  995. p4d_t *dst_p4d, p4d_t *src_p4d, unsigned long addr,
  996. unsigned long end)
  997. {
  998. struct mm_struct *dst_mm = dst_vma->vm_mm;
  999. struct mm_struct *src_mm = src_vma->vm_mm;
  1000. pud_t *src_pud, *dst_pud;
  1001. unsigned long next;
  1002. dst_pud = pud_alloc(dst_mm, dst_p4d, addr);
  1003. if (!dst_pud)
  1004. return -ENOMEM;
  1005. src_pud = pud_offset(src_p4d, addr);
  1006. do {
  1007. next = pud_addr_end(addr, end);
  1008. if (pud_trans_huge(*src_pud) || pud_devmap(*src_pud)) {
  1009. int err;
  1010. VM_BUG_ON_VMA(next-addr != HPAGE_PUD_SIZE, src_vma);
  1011. err = copy_huge_pud(dst_mm, src_mm,
  1012. dst_pud, src_pud, addr, src_vma);
  1013. if (err == -ENOMEM)
  1014. return -ENOMEM;
  1015. if (!err)
  1016. continue;
  1017. /* fall through */
  1018. }
  1019. if (pud_none_or_clear_bad(src_pud))
  1020. continue;
  1021. if (copy_pmd_range(dst_vma, src_vma, dst_pud, src_pud,
  1022. addr, next))
  1023. return -ENOMEM;
  1024. } while (dst_pud++, src_pud++, addr = next, addr != end);
  1025. return 0;
  1026. }
  1027. static inline int
  1028. copy_p4d_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
  1029. pgd_t *dst_pgd, pgd_t *src_pgd, unsigned long addr,
  1030. unsigned long end)
  1031. {
  1032. struct mm_struct *dst_mm = dst_vma->vm_mm;
  1033. p4d_t *src_p4d, *dst_p4d;
  1034. unsigned long next;
  1035. dst_p4d = p4d_alloc(dst_mm, dst_pgd, addr);
  1036. if (!dst_p4d)
  1037. return -ENOMEM;
  1038. src_p4d = p4d_offset(src_pgd, addr);
  1039. do {
  1040. next = p4d_addr_end(addr, end);
  1041. if (p4d_none_or_clear_bad(src_p4d))
  1042. continue;
  1043. if (copy_pud_range(dst_vma, src_vma, dst_p4d, src_p4d,
  1044. addr, next))
  1045. return -ENOMEM;
  1046. } while (dst_p4d++, src_p4d++, addr = next, addr != end);
  1047. return 0;
  1048. }
  1049. int
  1050. copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
  1051. {
  1052. pgd_t *src_pgd, *dst_pgd;
  1053. unsigned long next;
  1054. unsigned long addr = src_vma->vm_start;
  1055. unsigned long end = src_vma->vm_end;
  1056. struct mm_struct *dst_mm = dst_vma->vm_mm;
  1057. struct mm_struct *src_mm = src_vma->vm_mm;
  1058. struct mmu_notifier_range range;
  1059. bool is_cow;
  1060. int ret;
  1061. /*
  1062. * Don't copy ptes where a page fault will fill them correctly.
  1063. * Fork becomes much lighter when there are big shared or private
  1064. * readonly mappings. The tradeoff is that copy_page_range is more
  1065. * efficient than faulting.
  1066. */
  1067. if (!(src_vma->vm_flags & (VM_HUGETLB | VM_PFNMAP | VM_MIXEDMAP)) &&
  1068. !src_vma->anon_vma)
  1069. return 0;
  1070. if (is_vm_hugetlb_page(src_vma))
  1071. return copy_hugetlb_page_range(dst_mm, src_mm, src_vma);
  1072. if (unlikely(src_vma->vm_flags & VM_PFNMAP)) {
  1073. /*
  1074. * We do not free on error cases below as remove_vma
  1075. * gets called on error from higher level routine
  1076. */
  1077. ret = track_pfn_copy(src_vma);
  1078. if (ret)
  1079. return ret;
  1080. }
  1081. /*
  1082. * We need to invalidate the secondary MMU mappings only when
  1083. * there could be a permission downgrade on the ptes of the
  1084. * parent mm. And a permission downgrade will only happen if
  1085. * is_cow_mapping() returns true.
  1086. */
  1087. is_cow = is_cow_mapping(src_vma->vm_flags);
  1088. if (is_cow) {
  1089. mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE,
  1090. 0, src_vma, src_mm, addr, end);
  1091. mmu_notifier_invalidate_range_start(&range);
  1092. /*
  1093. * Disabling preemption is not needed for the write side, as
  1094. * the read side doesn't spin, but goes to the mmap_lock.
  1095. *
  1096. * Use the raw variant of the seqcount_t write API to avoid
  1097. * lockdep complaining about preemptibility.
  1098. */
  1099. mmap_assert_write_locked(src_mm);
  1100. raw_write_seqcount_begin(&src_mm->write_protect_seq);
  1101. }
  1102. ret = 0;
  1103. dst_pgd = pgd_offset(dst_mm, addr);
  1104. src_pgd = pgd_offset(src_mm, addr);
  1105. do {
  1106. next = pgd_addr_end(addr, end);
  1107. if (pgd_none_or_clear_bad(src_pgd))
  1108. continue;
  1109. if (unlikely(copy_p4d_range(dst_vma, src_vma, dst_pgd, src_pgd,
  1110. addr, next))) {
  1111. ret = -ENOMEM;
  1112. break;
  1113. }
  1114. } while (dst_pgd++, src_pgd++, addr = next, addr != end);
  1115. if (is_cow) {
  1116. raw_write_seqcount_end(&src_mm->write_protect_seq);
  1117. mmu_notifier_invalidate_range_end(&range);
  1118. }
  1119. return ret;
  1120. }
  1121. /* Whether we should zap all COWed (private) pages too */
  1122. static inline bool should_zap_cows(struct zap_details *details)
  1123. {
  1124. /* By default, zap all pages */
  1125. if (!details)
  1126. return true;
  1127. /* Or, we zap COWed pages only if the caller wants to */
  1128. return !details->check_mapping;
  1129. }
  1130. static unsigned long zap_pte_range(struct mmu_gather *tlb,
  1131. struct vm_area_struct *vma, pmd_t *pmd,
  1132. unsigned long addr, unsigned long end,
  1133. struct zap_details *details)
  1134. {
  1135. struct mm_struct *mm = tlb->mm;
  1136. int force_flush = 0;
  1137. int rss[NR_MM_COUNTERS];
  1138. spinlock_t *ptl;
  1139. pte_t *start_pte;
  1140. pte_t *pte;
  1141. swp_entry_t entry;
  1142. tlb_change_page_size(tlb, PAGE_SIZE);
  1143. again:
  1144. init_rss_vec(rss);
  1145. start_pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
  1146. pte = start_pte;
  1147. flush_tlb_batched_pending(mm);
  1148. arch_enter_lazy_mmu_mode();
  1149. do {
  1150. pte_t ptent = *pte;
  1151. if (pte_none(ptent))
  1152. continue;
  1153. if (need_resched())
  1154. break;
  1155. if (pte_present(ptent)) {
  1156. struct page *page;
  1157. page = vm_normal_page(vma, addr, ptent);
  1158. if (unlikely(details) && page) {
  1159. /*
  1160. * unmap_shared_mapping_pages() wants to
  1161. * invalidate cache without truncating:
  1162. * unmap shared but keep private pages.
  1163. */
  1164. if (details->check_mapping &&
  1165. details->check_mapping != page_rmapping(page))
  1166. continue;
  1167. }
  1168. ptent = ptep_get_and_clear_full(mm, addr, pte,
  1169. tlb->fullmm);
  1170. tlb_remove_tlb_entry(tlb, pte, addr);
  1171. if (unlikely(!page))
  1172. continue;
  1173. if (!PageAnon(page)) {
  1174. if (pte_dirty(ptent)) {
  1175. force_flush = 1;
  1176. set_page_dirty(page);
  1177. }
  1178. if (pte_young(ptent) &&
  1179. likely(!(vma->vm_flags & VM_SEQ_READ)))
  1180. mark_page_accessed(page);
  1181. }
  1182. rss[mm_counter(page)]--;
  1183. page_remove_rmap(page, false);
  1184. if (unlikely(page_mapcount(page) < 0))
  1185. print_bad_pte(vma, addr, ptent, page);
  1186. if (unlikely(__tlb_remove_page(tlb, page)) ||
  1187. lru_cache_disabled()) {
  1188. force_flush = 1;
  1189. addr += PAGE_SIZE;
  1190. break;
  1191. }
  1192. continue;
  1193. }
  1194. entry = pte_to_swp_entry(ptent);
  1195. if (is_device_private_entry(entry)) {
  1196. struct page *page = device_private_entry_to_page(entry);
  1197. if (unlikely(details && details->check_mapping)) {
  1198. /*
  1199. * unmap_shared_mapping_pages() wants to
  1200. * invalidate cache without truncating:
  1201. * unmap shared but keep private pages.
  1202. */
  1203. if (details->check_mapping !=
  1204. page_rmapping(page))
  1205. continue;
  1206. }
  1207. pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);
  1208. rss[mm_counter(page)]--;
  1209. page_remove_rmap(page, false);
  1210. put_page(page);
  1211. continue;
  1212. }
  1213. if (!non_swap_entry(entry)) {
  1214. /* Genuine swap entry, hence a private anon page */
  1215. if (!should_zap_cows(details))
  1216. continue;
  1217. rss[MM_SWAPENTS]--;
  1218. } else if (is_migration_entry(entry)) {
  1219. struct page *page;
  1220. page = migration_entry_to_page(entry);
  1221. if (details && details->check_mapping &&
  1222. details->check_mapping != page_rmapping(page))
  1223. continue;
  1224. rss[mm_counter(page)]--;
  1225. }
  1226. if (unlikely(!free_swap_and_cache(entry)))
  1227. print_bad_pte(vma, addr, ptent, NULL);
  1228. pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);
  1229. } while (pte++, addr += PAGE_SIZE, addr != end);
  1230. add_mm_rss_vec(mm, rss);
  1231. arch_leave_lazy_mmu_mode();
  1232. /* Do the actual TLB flush before dropping ptl */
  1233. if (force_flush)
  1234. tlb_flush_mmu_tlbonly(tlb);
  1235. pte_unmap_unlock(start_pte, ptl);
  1236. /*
  1237. * If we forced a TLB flush (either due to running out of
  1238. * batch buffers or because we needed to flush dirty TLB
  1239. * entries before releasing the ptl), free the batched
  1240. * memory too. Restart if we didn't do everything.
  1241. */
  1242. if (force_flush) {
  1243. force_flush = 0;
  1244. tlb_flush_mmu(tlb);
  1245. }
  1246. if (addr != end) {
  1247. cond_resched();
  1248. goto again;
  1249. }
  1250. return addr;
  1251. }
  1252. static inline unsigned long zap_pmd_range(struct mmu_gather *tlb,
  1253. struct vm_area_struct *vma, pud_t *pud,
  1254. unsigned long addr, unsigned long end,
  1255. struct zap_details *details)
  1256. {
  1257. pmd_t *pmd;
  1258. unsigned long next;
  1259. pmd = pmd_offset(pud, addr);
  1260. do {
  1261. next = pmd_addr_end(addr, end);
  1262. if (is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) {
  1263. if (next - addr != HPAGE_PMD_SIZE)
  1264. __split_huge_pmd(vma, pmd, addr, false, NULL);
  1265. else if (zap_huge_pmd(tlb, vma, pmd, addr))
  1266. goto next;
  1267. /* fall through */
  1268. } else if (details && details->single_page &&
  1269. PageTransCompound(details->single_page) &&
  1270. next - addr == HPAGE_PMD_SIZE && pmd_none(*pmd)) {
  1271. spinlock_t *ptl = pmd_lock(tlb->mm, pmd);
  1272. /*
  1273. * Take and drop THP pmd lock so that we cannot return
  1274. * prematurely, while zap_huge_pmd() has cleared *pmd,
  1275. * but not yet decremented compound_mapcount().
  1276. */
  1277. spin_unlock(ptl);
  1278. }
  1279. /*
  1280. * Here there can be other concurrent MADV_DONTNEED or
  1281. * trans huge page faults running, and if the pmd is
  1282. * none or trans huge it can change under us. This is
  1283. * because MADV_DONTNEED holds the mmap_lock in read
  1284. * mode.
  1285. */
  1286. if (pmd_none_or_trans_huge_or_clear_bad(pmd))
  1287. goto next;
  1288. next = zap_pte_range(tlb, vma, pmd, addr, next, details);
  1289. next:
  1290. cond_resched();
  1291. } while (pmd++, addr = next, addr != end);
  1292. return addr;
  1293. }
  1294. static inline unsigned long zap_pud_range(struct mmu_gather *tlb,
  1295. struct vm_area_struct *vma, p4d_t *p4d,
  1296. unsigned long addr, unsigned long end,
  1297. struct zap_details *details)
  1298. {
  1299. pud_t *pud;
  1300. unsigned long next;
  1301. pud = pud_offset(p4d, addr);
  1302. do {
  1303. next = pud_addr_end(addr, end);
  1304. if (pud_trans_huge(*pud) || pud_devmap(*pud)) {
  1305. if (next - addr != HPAGE_PUD_SIZE) {
  1306. mmap_assert_locked(tlb->mm);
  1307. split_huge_pud(vma, pud, addr);
  1308. } else if (zap_huge_pud(tlb, vma, pud, addr))
  1309. goto next;
  1310. /* fall through */
  1311. }
  1312. if (pud_none_or_clear_bad(pud))
  1313. continue;
  1314. next = zap_pmd_range(tlb, vma, pud, addr, next, details);
  1315. next:
  1316. cond_resched();
  1317. } while (pud++, addr = next, addr != end);
  1318. return addr;
  1319. }
  1320. static inline unsigned long zap_p4d_range(struct mmu_gather *tlb,
  1321. struct vm_area_struct *vma, pgd_t *pgd,
  1322. unsigned long addr, unsigned long end,
  1323. struct zap_details *details)
  1324. {
  1325. p4d_t *p4d;
  1326. unsigned long next;
  1327. p4d = p4d_offset(pgd, addr);
  1328. do {
  1329. next = p4d_addr_end(addr, end);
  1330. if (p4d_none_or_clear_bad(p4d))
  1331. continue;
  1332. next = zap_pud_range(tlb, vma, p4d, addr, next, details);
  1333. } while (p4d++, addr = next, addr != end);
  1334. return addr;
  1335. }
  1336. void unmap_page_range(struct mmu_gather *tlb,
  1337. struct vm_area_struct *vma,
  1338. unsigned long addr, unsigned long end,
  1339. struct zap_details *details)
  1340. {
  1341. pgd_t *pgd;
  1342. unsigned long next;
  1343. BUG_ON(addr >= end);
  1344. vm_write_begin(vma);
  1345. tlb_start_vma(tlb, vma);
  1346. pgd = pgd_offset(vma->vm_mm, addr);
  1347. do {
  1348. next = pgd_addr_end(addr, end);
  1349. if (pgd_none_or_clear_bad(pgd))
  1350. continue;
  1351. next = zap_p4d_range(tlb, vma, pgd, addr, next, details);
  1352. } while (pgd++, addr = next, addr != end);
  1353. tlb_end_vma(tlb, vma);
  1354. vm_write_end(vma);
  1355. }
  1356. static void unmap_single_vma(struct mmu_gather *tlb,
  1357. struct vm_area_struct *vma, unsigned long start_addr,
  1358. unsigned long end_addr,
  1359. struct zap_details *details)
  1360. {
  1361. unsigned long start = max(vma->vm_start, start_addr);
  1362. unsigned long end;
  1363. if (start >= vma->vm_end)
  1364. return;
  1365. end = min(vma->vm_end, end_addr);
  1366. if (end <= vma->vm_start)
  1367. return;
  1368. if (vma->vm_file)
  1369. uprobe_munmap(vma, start, end);
  1370. if (unlikely(vma->vm_flags & VM_PFNMAP))
  1371. untrack_pfn(vma, 0, 0);
  1372. if (start != end) {
  1373. if (unlikely(is_vm_hugetlb_page(vma))) {
  1374. /*
  1375. * It is undesirable to test vma->vm_file as it
  1376. * should be non-null for valid hugetlb area.
  1377. * However, vm_file will be NULL in the error
  1378. * cleanup path of mmap_region. When
  1379. * hugetlbfs ->mmap method fails,
  1380. * mmap_region() nullifies vma->vm_file
  1381. * before calling this function to clean up.
  1382. * Since no pte has actually been setup, it is
  1383. * safe to do nothing in this case.
  1384. */
  1385. if (vma->vm_file) {
  1386. i_mmap_lock_write(vma->vm_file->f_mapping);
  1387. __unmap_hugepage_range_final(tlb, vma, start, end, NULL);
  1388. i_mmap_unlock_write(vma->vm_file->f_mapping);
  1389. }
  1390. } else
  1391. unmap_page_range(tlb, vma, start, end, details);
  1392. }
  1393. }
  1394. /**
  1395. * unmap_vmas - unmap a range of memory covered by a list of vma's
  1396. * @tlb: address of the caller's struct mmu_gather
  1397. * @vma: the starting vma
  1398. * @start_addr: virtual address at which to start unmapping
  1399. * @end_addr: virtual address at which to end unmapping
  1400. *
  1401. * Unmap all pages in the vma list.
  1402. *
  1403. * Only addresses between `start' and `end' will be unmapped.
  1404. *
  1405. * The VMA list must be sorted in ascending virtual address order.
  1406. *
  1407. * unmap_vmas() assumes that the caller will flush the whole unmapped address
  1408. * range after unmap_vmas() returns. So the only responsibility here is to
  1409. * ensure that any thus-far unmapped pages are flushed before unmap_vmas()
  1410. * drops the lock and schedules.
  1411. */
  1412. void unmap_vmas(struct mmu_gather *tlb,
  1413. struct vm_area_struct *vma, unsigned long start_addr,
  1414. unsigned long end_addr)
  1415. {
  1416. struct mmu_notifier_range range;
  1417. mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, vma->vm_mm,
  1418. start_addr, end_addr);
  1419. mmu_notifier_invalidate_range_start(&range);
  1420. for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next)
  1421. unmap_single_vma(tlb, vma, start_addr, end_addr, NULL);
  1422. mmu_notifier_invalidate_range_end(&range);
  1423. }
  1424. /**
  1425. * zap_page_range - remove user pages in a given range
  1426. * @vma: vm_area_struct holding the applicable pages
  1427. * @start: starting address of pages to zap
  1428. * @size: number of bytes to zap
  1429. *
  1430. * Caller must protect the VMA list
  1431. */
  1432. void zap_page_range(struct vm_area_struct *vma, unsigned long start,
  1433. unsigned long size)
  1434. {
  1435. struct mmu_notifier_range range;
  1436. struct mmu_gather tlb;
  1437. lru_add_drain();
  1438. mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,
  1439. start, start + size);
  1440. tlb_gather_mmu(&tlb, vma->vm_mm, start, range.end);
  1441. update_hiwater_rss(vma->vm_mm);
  1442. mmu_notifier_invalidate_range_start(&range);
  1443. for ( ; vma && vma->vm_start < range.end; vma = vma->vm_next)
  1444. unmap_single_vma(&tlb, vma, start, range.end, NULL);
  1445. mmu_notifier_invalidate_range_end(&range);
  1446. tlb_finish_mmu(&tlb, start, range.end);
  1447. }
  1448. /**
  1449. * zap_page_range_single - remove user pages in a given range
  1450. * @vma: vm_area_struct holding the applicable pages
  1451. * @address: starting address of pages to zap
  1452. * @size: number of bytes to zap
  1453. * @details: details of shared cache invalidation
  1454. *
  1455. * The range must fit into one VMA.
  1456. */
  1457. static void zap_page_range_single(struct vm_area_struct *vma, unsigned long address,
  1458. unsigned long size, struct zap_details *details)
  1459. {
  1460. struct mmu_notifier_range range;
  1461. struct mmu_gather tlb;
  1462. lru_add_drain();
  1463. mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,
  1464. address, address + size);
  1465. tlb_gather_mmu(&tlb, vma->vm_mm, address, range.end);
  1466. update_hiwater_rss(vma->vm_mm);
  1467. mmu_notifier_invalidate_range_start(&range);
  1468. unmap_single_vma(&tlb, vma, address, range.end, details);
  1469. mmu_notifier_invalidate_range_end(&range);
  1470. tlb_finish_mmu(&tlb, address, range.end);
  1471. }
  1472. /**
  1473. * zap_vma_ptes - remove ptes mapping the vma
  1474. * @vma: vm_area_struct holding ptes to be zapped
  1475. * @address: starting address of pages to zap
  1476. * @size: number of bytes to zap
  1477. *
  1478. * This function only unmaps ptes assigned to VM_PFNMAP vmas.
  1479. *
  1480. * The entire address range must be fully contained within the vma.
  1481. *
  1482. */
  1483. void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
  1484. unsigned long size)
  1485. {
  1486. if (address < vma->vm_start || address + size > vma->vm_end ||
  1487. !(vma->vm_flags & VM_PFNMAP))
  1488. return;
  1489. zap_page_range_single(vma, address, size, NULL);
  1490. }
  1491. EXPORT_SYMBOL_GPL(zap_vma_ptes);
  1492. static pmd_t *walk_to_pmd(struct mm_struct *mm, unsigned long addr)
  1493. {
  1494. pgd_t *pgd;
  1495. p4d_t *p4d;
  1496. pud_t *pud;
  1497. pmd_t *pmd;
  1498. pgd = pgd_offset(mm, addr);
  1499. p4d = p4d_alloc(mm, pgd, addr);
  1500. if (!p4d)
  1501. return NULL;
  1502. pud = pud_alloc(mm, p4d, addr);
  1503. if (!pud)
  1504. return NULL;
  1505. pmd = pmd_alloc(mm, pud, addr);
  1506. if (!pmd)
  1507. return NULL;
  1508. VM_BUG_ON(pmd_trans_huge(*pmd));
  1509. return pmd;
  1510. }
  1511. pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
  1512. spinlock_t **ptl)
  1513. {
  1514. pmd_t *pmd = walk_to_pmd(mm, addr);
  1515. if (!pmd)
  1516. return NULL;
  1517. return pte_alloc_map_lock(mm, pmd, addr, ptl);
  1518. }
  1519. static int validate_page_before_insert(struct page *page)
  1520. {
  1521. if (PageAnon(page) || PageSlab(page) || page_has_type(page))
  1522. return -EINVAL;
  1523. flush_dcache_page(page);
  1524. return 0;
  1525. }
  1526. static int insert_page_into_pte_locked(struct mm_struct *mm, pte_t *pte,
  1527. unsigned long addr, struct page *page, pgprot_t prot)
  1528. {
  1529. if (!pte_none(*pte))
  1530. return -EBUSY;
  1531. /* Ok, finally just insert the thing.. */
  1532. get_page(page);
  1533. inc_mm_counter_fast(mm, mm_counter_file(page));
  1534. page_add_file_rmap(page, false);
  1535. set_pte_at(mm, addr, pte, mk_pte(page, prot));
  1536. return 0;
  1537. }
  1538. /*
  1539. * This is the old fallback for page remapping.
  1540. *
  1541. * For historical reasons, it only allows reserved pages. Only
  1542. * old drivers should use this, and they needed to mark their
  1543. * pages reserved for the old functions anyway.
  1544. */
  1545. static int insert_page(struct vm_area_struct *vma, unsigned long addr,
  1546. struct page *page, pgprot_t prot)
  1547. {
  1548. struct mm_struct *mm = vma->vm_mm;
  1549. int retval;
  1550. pte_t *pte;
  1551. spinlock_t *ptl;
  1552. retval = validate_page_before_insert(page);
  1553. if (retval)
  1554. goto out;
  1555. retval = -ENOMEM;
  1556. pte = get_locked_pte(mm, addr, &ptl);
  1557. if (!pte)
  1558. goto out;
  1559. retval = insert_page_into_pte_locked(mm, pte, addr, page, prot);
  1560. pte_unmap_unlock(pte, ptl);
  1561. out:
  1562. return retval;
  1563. }
  1564. #ifdef pte_index
  1565. static int insert_page_in_batch_locked(struct mm_struct *mm, pte_t *pte,
  1566. unsigned long addr, struct page *page, pgprot_t prot)
  1567. {
  1568. int err;
  1569. if (!page_count(page))
  1570. return -EINVAL;
  1571. err = validate_page_before_insert(page);
  1572. if (err)
  1573. return err;
  1574. return insert_page_into_pte_locked(mm, pte, addr, page, prot);
  1575. }
  1576. /* insert_pages() amortizes the cost of spinlock operations
  1577. * when inserting pages in a loop. Arch *must* define pte_index.
  1578. */
  1579. static int insert_pages(struct vm_area_struct *vma, unsigned long addr,
  1580. struct page **pages, unsigned long *num, pgprot_t prot)
  1581. {
  1582. pmd_t *pmd = NULL;
  1583. pte_t *start_pte, *pte;
  1584. spinlock_t *pte_lock;
  1585. struct mm_struct *const mm = vma->vm_mm;
  1586. unsigned long curr_page_idx = 0;
  1587. unsigned long remaining_pages_total = *num;
  1588. unsigned long pages_to_write_in_pmd;
  1589. int ret;
  1590. more:
  1591. ret = -EFAULT;
  1592. pmd = walk_to_pmd(mm, addr);
  1593. if (!pmd)
  1594. goto out;
  1595. pages_to_write_in_pmd = min_t(unsigned long,
  1596. remaining_pages_total, PTRS_PER_PTE - pte_index(addr));
  1597. /* Allocate the PTE if necessary; takes PMD lock once only. */
  1598. ret = -ENOMEM;
  1599. if (pte_alloc(mm, pmd))
  1600. goto out;
  1601. while (pages_to_write_in_pmd) {
  1602. int pte_idx = 0;
  1603. const int batch_size = min_t(int, pages_to_write_in_pmd, 8);
  1604. start_pte = pte_offset_map_lock(mm, pmd, addr, &pte_lock);
  1605. for (pte = start_pte; pte_idx < batch_size; ++pte, ++pte_idx) {
  1606. int err = insert_page_in_batch_locked(mm, pte,
  1607. addr, pages[curr_page_idx], prot);
  1608. if (unlikely(err)) {
  1609. pte_unmap_unlock(start_pte, pte_lock);
  1610. ret = err;
  1611. remaining_pages_total -= pte_idx;
  1612. goto out;
  1613. }
  1614. addr += PAGE_SIZE;
  1615. ++curr_page_idx;
  1616. }
  1617. pte_unmap_unlock(start_pte, pte_lock);
  1618. pages_to_write_in_pmd -= batch_size;
  1619. remaining_pages_total -= batch_size;
  1620. }
  1621. if (remaining_pages_total)
  1622. goto more;
  1623. ret = 0;
  1624. out:
  1625. *num = remaining_pages_total;
  1626. return ret;
  1627. }
  1628. #endif /* ifdef pte_index */
  1629. /**
  1630. * vm_insert_pages - insert multiple pages into user vma, batching the pmd lock.
  1631. * @vma: user vma to map to
  1632. * @addr: target start user address of these pages
  1633. * @pages: source kernel pages
  1634. * @num: in: number of pages to map. out: number of pages that were *not*
  1635. * mapped. (0 means all pages were successfully mapped).
  1636. *
  1637. * Preferred over vm_insert_page() when inserting multiple pages.
  1638. *
  1639. * In case of error, we may have mapped a subset of the provided
  1640. * pages. It is the caller's responsibility to account for this case.
  1641. *
  1642. * The same restrictions apply as in vm_insert_page().
  1643. */
  1644. int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr,
  1645. struct page **pages, unsigned long *num)
  1646. {
  1647. #ifdef pte_index
  1648. const unsigned long end_addr = addr + (*num * PAGE_SIZE) - 1;
  1649. if (addr < vma->vm_start || end_addr >= vma->vm_end)
  1650. return -EFAULT;
  1651. if (!(vma->vm_flags & VM_MIXEDMAP)) {
  1652. BUG_ON(mmap_read_trylock(vma->vm_mm));
  1653. BUG_ON(vma->vm_flags & VM_PFNMAP);
  1654. vma->vm_flags |= VM_MIXEDMAP;
  1655. }
  1656. /* Defer page refcount checking till we're about to map that page. */
  1657. return insert_pages(vma, addr, pages, num, vma->vm_page_prot);
  1658. #else
  1659. unsigned long idx = 0, pgcount = *num;
  1660. int err = -EINVAL;
  1661. for (; idx < pgcount; ++idx) {
  1662. err = vm_insert_page(vma, addr + (PAGE_SIZE * idx), pages[idx]);
  1663. if (err)
  1664. break;
  1665. }
  1666. *num = pgcount - idx;
  1667. return err;
  1668. #endif /* ifdef pte_index */
  1669. }
  1670. EXPORT_SYMBOL(vm_insert_pages);
  1671. /**
  1672. * vm_insert_page - insert single page into user vma
  1673. * @vma: user vma to map to
  1674. * @addr: target user address of this page
  1675. * @page: source kernel page
  1676. *
  1677. * This allows drivers to insert individual pages they've allocated
  1678. * into a user vma.
  1679. *
  1680. * The page has to be a nice clean _individual_ kernel allocation.
  1681. * If you allocate a compound page, you need to have marked it as
  1682. * such (__GFP_COMP), or manually just split the page up yourself
  1683. * (see split_page()).
  1684. *
  1685. * NOTE! Traditionally this was done with "remap_pfn_range()" which
  1686. * took an arbitrary page protection parameter. This doesn't allow
  1687. * that. Your vma protection will have to be set up correctly, which
  1688. * means that if you want a shared writable mapping, you'd better
  1689. * ask for a shared writable mapping!
  1690. *
  1691. * The page does not need to be reserved.
  1692. *
  1693. * Usually this function is called from f_op->mmap() handler
  1694. * under mm->mmap_lock write-lock, so it can change vma->vm_flags.
  1695. * Caller must set VM_MIXEDMAP on vma if it wants to call this
  1696. * function from other places, for example from page-fault handler.
  1697. *
  1698. * Return: %0 on success, negative error code otherwise.
  1699. */
  1700. int vm_insert_page(struct vm_area_struct *vma, unsigned long addr,
  1701. struct page *page)
  1702. {
  1703. if (addr < vma->vm_start || addr >= vma->vm_end)
  1704. return -EFAULT;
  1705. if (!page_count(page))
  1706. return -EINVAL;
  1707. if (!(vma->vm_flags & VM_MIXEDMAP)) {
  1708. BUG_ON(mmap_read_trylock(vma->vm_mm));
  1709. BUG_ON(vma->vm_flags & VM_PFNMAP);
  1710. vma->vm_flags |= VM_MIXEDMAP;
  1711. }
  1712. return insert_page(vma, addr, page, vma->vm_page_prot);
  1713. }
  1714. EXPORT_SYMBOL(vm_insert_page);
  1715. /*
  1716. * __vm_map_pages - maps range of kernel pages into user vma
  1717. * @vma: user vma to map to
  1718. * @pages: pointer to array of source kernel pages
  1719. * @num: number of pages in page array
  1720. * @offset: user's requested vm_pgoff
  1721. *
  1722. * This allows drivers to map range of kernel pages into a user vma.
  1723. *
  1724. * Return: 0 on success and error code otherwise.
  1725. */
  1726. static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages,
  1727. unsigned long num, unsigned long offset)
  1728. {
  1729. unsigned long count = vma_pages(vma);
  1730. unsigned long uaddr = vma->vm_start;
  1731. int ret, i;
  1732. /* Fail if the user requested offset is beyond the end of the object */
  1733. if (offset >= num)
  1734. return -ENXIO;
  1735. /* Fail if the user requested size exceeds available object size */
  1736. if (count > num - offset)
  1737. return -ENXIO;
  1738. for (i = 0; i < count; i++) {
  1739. ret = vm_insert_page(vma, uaddr, pages[offset + i]);
  1740. if (ret < 0)
  1741. return ret;
  1742. uaddr += PAGE_SIZE;
  1743. }
  1744. return 0;
  1745. }
  1746. /**
  1747. * vm_map_pages - maps range of kernel pages starts with non zero offset
  1748. * @vma: user vma to map to
  1749. * @pages: pointer to array of source kernel pages
  1750. * @num: number of pages in page array
  1751. *
  1752. * Maps an object consisting of @num pages, catering for the user's
  1753. * requested vm_pgoff
  1754. *
  1755. * If we fail to insert any page into the vma, the function will return
  1756. * immediately leaving any previously inserted pages present. Callers
  1757. * from the mmap handler may immediately return the error as their caller
  1758. * will destroy the vma, removing any successfully inserted pages. Other
  1759. * callers should make their own arrangements for calling unmap_region().
  1760. *
  1761. * Context: Process context. Called by mmap handlers.
  1762. * Return: 0 on success and error code otherwise.
  1763. */
  1764. int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
  1765. unsigned long num)
  1766. {
  1767. return __vm_map_pages(vma, pages, num, vma->vm_pgoff);
  1768. }
  1769. EXPORT_SYMBOL(vm_map_pages);
  1770. /**
  1771. * vm_map_pages_zero - map range of kernel pages starts with zero offset
  1772. * @vma: user vma to map to
  1773. * @pages: pointer to array of source kernel pages
  1774. * @num: number of pages in page array
  1775. *
  1776. * Similar to vm_map_pages(), except that it explicitly sets the offset
  1777. * to 0. This function is intended for the drivers that did not consider
  1778. * vm_pgoff.
  1779. *
  1780. * Context: Process context. Called by mmap handlers.
  1781. * Return: 0 on success and error code otherwise.
  1782. */
  1783. int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages,
  1784. unsigned long num)
  1785. {
  1786. return __vm_map_pages(vma, pages, num, 0);
  1787. }
  1788. EXPORT_SYMBOL(vm_map_pages_zero);
  1789. static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr,
  1790. pfn_t pfn, pgprot_t prot, bool mkwrite)
  1791. {
  1792. struct mm_struct *mm = vma->vm_mm;
  1793. pte_t *pte, entry;
  1794. spinlock_t *ptl;
  1795. pte = get_locked_pte(mm, addr, &ptl);
  1796. if (!pte)
  1797. return VM_FAULT_OOM;
  1798. if (!pte_none(*pte)) {
  1799. if (mkwrite) {
  1800. /*
  1801. * For read faults on private mappings the PFN passed
  1802. * in may not match the PFN we have mapped if the
  1803. * mapped PFN is a writeable COW page. In the mkwrite
  1804. * case we are creating a writable PTE for a shared
  1805. * mapping and we expect the PFNs to match. If they
  1806. * don't match, we are likely racing with block
  1807. * allocation and mapping invalidation so just skip the
  1808. * update.
  1809. */
  1810. if (pte_pfn(*pte) != pfn_t_to_pfn(pfn)) {
  1811. WARN_ON_ONCE(!is_zero_pfn(pte_pfn(*pte)));
  1812. goto out_unlock;
  1813. }
  1814. entry = pte_mkyoung(*pte);
  1815. entry = maybe_mkwrite(pte_mkdirty(entry),
  1816. vma->vm_flags);
  1817. if (ptep_set_access_flags(vma, addr, pte, entry, 1))
  1818. update_mmu_cache(vma, addr, pte);
  1819. }
  1820. goto out_unlock;
  1821. }
  1822. /* Ok, finally just insert the thing.. */
  1823. if (pfn_t_devmap(pfn))
  1824. entry = pte_mkdevmap(pfn_t_pte(pfn, prot));
  1825. else
  1826. entry = pte_mkspecial(pfn_t_pte(pfn, prot));
  1827. if (mkwrite) {
  1828. entry = pte_mkyoung(entry);
  1829. entry = maybe_mkwrite(pte_mkdirty(entry), vma->vm_flags);
  1830. }
  1831. set_pte_at(mm, addr, pte, entry);
  1832. update_mmu_cache(vma, addr, pte); /* XXX: why not for insert_page? */
  1833. out_unlock:
  1834. pte_unmap_unlock(pte, ptl);
  1835. return VM_FAULT_NOPAGE;
  1836. }
  1837. /**
  1838. * vmf_insert_pfn_prot - insert single pfn into user vma with specified pgprot
  1839. * @vma: user vma to map to
  1840. * @addr: target user address of this page
  1841. * @pfn: source kernel pfn
  1842. * @pgprot: pgprot flags for the inserted page
  1843. *
  1844. * This is exactly like vmf_insert_pfn(), except that it allows drivers
  1845. * to override pgprot on a per-page basis.
  1846. *
  1847. * This only makes sense for IO mappings, and it makes no sense for
  1848. * COW mappings. In general, using multiple vmas is preferable;
  1849. * vmf_insert_pfn_prot should only be used if using multiple VMAs is
  1850. * impractical.
  1851. *
  1852. * See vmf_insert_mixed_prot() for a discussion of the implication of using
  1853. * a value of @pgprot different from that of @vma->vm_page_prot.
  1854. *
  1855. * Context: Process context. May allocate using %GFP_KERNEL.
  1856. * Return: vm_fault_t value.
  1857. */
  1858. vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr,
  1859. unsigned long pfn, pgprot_t pgprot)
  1860. {
  1861. /*
  1862. * Technically, architectures with pte_special can avoid all these
  1863. * restrictions (same for remap_pfn_range). However we would like
  1864. * consistency in testing and feature parity among all, so we should
  1865. * try to keep these invariants in place for everybody.
  1866. */
  1867. BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
  1868. BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
  1869. (VM_PFNMAP|VM_MIXEDMAP));
  1870. BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
  1871. BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn));
  1872. if (addr < vma->vm_start || addr >= vma->vm_end)
  1873. return VM_FAULT_SIGBUS;
  1874. if (!pfn_modify_allowed(pfn, pgprot))
  1875. return VM_FAULT_SIGBUS;
  1876. track_pfn_insert(vma, &pgprot, __pfn_to_pfn_t(pfn, PFN_DEV));
  1877. return insert_pfn(vma, addr, __pfn_to_pfn_t(pfn, PFN_DEV), pgprot,
  1878. false);
  1879. }
  1880. EXPORT_SYMBOL(vmf_insert_pfn_prot);
  1881. /**
  1882. * vmf_insert_pfn - insert single pfn into user vma
  1883. * @vma: user vma to map to
  1884. * @addr: target user address of this page
  1885. * @pfn: source kernel pfn
  1886. *
  1887. * Similar to vm_insert_page, this allows drivers to insert individual pages
  1888. * they've allocated into a user vma. Same comments apply.
  1889. *
  1890. * This function should only be called from a vm_ops->fault handler, and
  1891. * in that case the handler should return the result of this function.
  1892. *
  1893. * vma cannot be a COW mapping.
  1894. *
  1895. * As this is called only for pages that do not currently exist, we
  1896. * do not need to flush old virtual caches or the TLB.
  1897. *
  1898. * Context: Process context. May allocate using %GFP_KERNEL.
  1899. * Return: vm_fault_t value.
  1900. */
  1901. vm_fault_t vmf_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
  1902. unsigned long pfn)
  1903. {
  1904. return vmf_insert_pfn_prot(vma, addr, pfn, vma->vm_page_prot);
  1905. }
  1906. EXPORT_SYMBOL(vmf_insert_pfn);
  1907. static bool vm_mixed_ok(struct vm_area_struct *vma, pfn_t pfn)
  1908. {
  1909. /* these checks mirror the abort conditions in vm_normal_page */
  1910. if (vma->vm_flags & VM_MIXEDMAP)
  1911. return true;
  1912. if (pfn_t_devmap(pfn))
  1913. return true;
  1914. if (pfn_t_special(pfn))
  1915. return true;
  1916. if (is_zero_pfn(pfn_t_to_pfn(pfn)))
  1917. return true;
  1918. return false;
  1919. }
  1920. static vm_fault_t __vm_insert_mixed(struct vm_area_struct *vma,
  1921. unsigned long addr, pfn_t pfn, pgprot_t pgprot,
  1922. bool mkwrite)
  1923. {
  1924. int err;
  1925. BUG_ON(!vm_mixed_ok(vma, pfn));
  1926. if (addr < vma->vm_start || addr >= vma->vm_end)
  1927. return VM_FAULT_SIGBUS;
  1928. track_pfn_insert(vma, &pgprot, pfn);
  1929. if (!pfn_modify_allowed(pfn_t_to_pfn(pfn), pgprot))
  1930. return VM_FAULT_SIGBUS;
  1931. /*
  1932. * If we don't have pte special, then we have to use the pfn_valid()
  1933. * based VM_MIXEDMAP scheme (see vm_normal_page), and thus we *must*
  1934. * refcount the page if pfn_valid is true (hence insert_page rather
  1935. * than insert_pfn). If a zero_pfn were inserted into a VM_MIXEDMAP
  1936. * without pte special, it would there be refcounted as a normal page.
  1937. */
  1938. if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL) &&
  1939. !pfn_t_devmap(pfn) && pfn_t_valid(pfn)) {
  1940. struct page *page;
  1941. /*
  1942. * At this point we are committed to insert_page()
  1943. * regardless of whether the caller specified flags that
  1944. * result in pfn_t_has_page() == false.
  1945. */
  1946. page = pfn_to_page(pfn_t_to_pfn(pfn));
  1947. err = insert_page(vma, addr, page, pgprot);
  1948. } else {
  1949. return insert_pfn(vma, addr, pfn, pgprot, mkwrite);
  1950. }
  1951. if (err == -ENOMEM)
  1952. return VM_FAULT_OOM;
  1953. if (err < 0 && err != -EBUSY)
  1954. return VM_FAULT_SIGBUS;
  1955. return VM_FAULT_NOPAGE;
  1956. }
  1957. /**
  1958. * vmf_insert_mixed_prot - insert single pfn into user vma with specified pgprot
  1959. * @vma: user vma to map to
  1960. * @addr: target user address of this page
  1961. * @pfn: source kernel pfn
  1962. * @pgprot: pgprot flags for the inserted page
  1963. *
  1964. * This is exactly like vmf_insert_mixed(), except that it allows drivers
  1965. * to override pgprot on a per-page basis.
  1966. *
  1967. * Typically this function should be used by drivers to set caching- and
  1968. * encryption bits different than those of @vma->vm_page_prot, because
  1969. * the caching- or encryption mode may not be known at mmap() time.
  1970. * This is ok as long as @vma->vm_page_prot is not used by the core vm
  1971. * to set caching and encryption bits for those vmas (except for COW pages).
  1972. * This is ensured by core vm only modifying these page table entries using
  1973. * functions that don't touch caching- or encryption bits, using pte_modify()
  1974. * if needed. (See for example mprotect()).
  1975. * Also when new page-table entries are created, this is only done using the
  1976. * fault() callback, and never using the value of vma->vm_page_prot,
  1977. * except for page-table entries that point to anonymous pages as the result
  1978. * of COW.
  1979. *
  1980. * Context: Process context. May allocate using %GFP_KERNEL.
  1981. * Return: vm_fault_t value.
  1982. */
  1983. vm_fault_t vmf_insert_mixed_prot(struct vm_area_struct *vma, unsigned long addr,
  1984. pfn_t pfn, pgprot_t pgprot)
  1985. {
  1986. return __vm_insert_mixed(vma, addr, pfn, pgprot, false);
  1987. }
  1988. EXPORT_SYMBOL(vmf_insert_mixed_prot);
  1989. vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
  1990. pfn_t pfn)
  1991. {
  1992. return __vm_insert_mixed(vma, addr, pfn, vma->vm_page_prot, false);
  1993. }
  1994. EXPORT_SYMBOL(vmf_insert_mixed);
  1995. /*
  1996. * If the insertion of PTE failed because someone else already added a
  1997. * different entry in the mean time, we treat that as success as we assume
  1998. * the same entry was actually inserted.
  1999. */
  2000. vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma,
  2001. unsigned long addr, pfn_t pfn)
  2002. {
  2003. return __vm_insert_mixed(vma, addr, pfn, vma->vm_page_prot, true);
  2004. }
  2005. EXPORT_SYMBOL(vmf_insert_mixed_mkwrite);
  2006. /*
  2007. * maps a range of physical memory into the requested pages. the old
  2008. * mappings are removed. any references to nonexistent pages results
  2009. * in null mappings (currently treated as "copy-on-access")
  2010. */
  2011. static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd,
  2012. unsigned long addr, unsigned long end,
  2013. unsigned long pfn, pgprot_t prot)
  2014. {
  2015. pte_t *pte, *mapped_pte;
  2016. spinlock_t *ptl;
  2017. int err = 0;
  2018. mapped_pte = pte = pte_alloc_map_lock(mm, pmd, addr, &ptl);
  2019. if (!pte)
  2020. return -ENOMEM;
  2021. arch_enter_lazy_mmu_mode();
  2022. do {
  2023. BUG_ON(!pte_none(*pte));
  2024. if (!pfn_modify_allowed(pfn, prot)) {
  2025. err = -EACCES;
  2026. break;
  2027. }
  2028. set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot)));
  2029. pfn++;
  2030. } while (pte++, addr += PAGE_SIZE, addr != end);
  2031. arch_leave_lazy_mmu_mode();
  2032. pte_unmap_unlock(mapped_pte, ptl);
  2033. return err;
  2034. }
  2035. static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud,
  2036. unsigned long addr, unsigned long end,
  2037. unsigned long pfn, pgprot_t prot)
  2038. {
  2039. pmd_t *pmd;
  2040. unsigned long next;
  2041. int err;
  2042. pfn -= addr >> PAGE_SHIFT;
  2043. pmd = pmd_alloc(mm, pud, addr);
  2044. if (!pmd)
  2045. return -ENOMEM;
  2046. VM_BUG_ON(pmd_trans_huge(*pmd));
  2047. do {
  2048. next = pmd_addr_end(addr, end);
  2049. err = remap_pte_range(mm, pmd, addr, next,
  2050. pfn + (addr >> PAGE_SHIFT), prot);
  2051. if (err)
  2052. return err;
  2053. } while (pmd++, addr = next, addr != end);
  2054. return 0;
  2055. }
  2056. static inline int remap_pud_range(struct mm_struct *mm, p4d_t *p4d,
  2057. unsigned long addr, unsigned long end,
  2058. unsigned long pfn, pgprot_t prot)
  2059. {
  2060. pud_t *pud;
  2061. unsigned long next;
  2062. int err;
  2063. pfn -= addr >> PAGE_SHIFT;
  2064. pud = pud_alloc(mm, p4d, addr);
  2065. if (!pud)
  2066. return -ENOMEM;
  2067. do {
  2068. next = pud_addr_end(addr, end);
  2069. err = remap_pmd_range(mm, pud, addr, next,
  2070. pfn + (addr >> PAGE_SHIFT), prot);
  2071. if (err)
  2072. return err;
  2073. } while (pud++, addr = next, addr != end);
  2074. return 0;
  2075. }
  2076. static inline int remap_p4d_range(struct mm_struct *mm, pgd_t *pgd,
  2077. unsigned long addr, unsigned long end,
  2078. unsigned long pfn, pgprot_t prot)
  2079. {
  2080. p4d_t *p4d;
  2081. unsigned long next;
  2082. int err;
  2083. pfn -= addr >> PAGE_SHIFT;
  2084. p4d = p4d_alloc(mm, pgd, addr);
  2085. if (!p4d)
  2086. return -ENOMEM;
  2087. do {
  2088. next = p4d_addr_end(addr, end);
  2089. err = remap_pud_range(mm, p4d, addr, next,
  2090. pfn + (addr >> PAGE_SHIFT), prot);
  2091. if (err)
  2092. return err;
  2093. } while (p4d++, addr = next, addr != end);
  2094. return 0;
  2095. }
  2096. /**
  2097. * remap_pfn_range - remap kernel memory to userspace
  2098. * @vma: user vma to map to
  2099. * @addr: target page aligned user address to start at
  2100. * @pfn: page frame number of kernel physical memory address
  2101. * @size: size of mapping area
  2102. * @prot: page protection flags for this mapping
  2103. *
  2104. * Note: this is only safe if the mm semaphore is held when called.
  2105. *
  2106. * Return: %0 on success, negative error code otherwise.
  2107. */
  2108. int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
  2109. unsigned long pfn, unsigned long size, pgprot_t prot)
  2110. {
  2111. pgd_t *pgd;
  2112. unsigned long next;
  2113. unsigned long end = addr + PAGE_ALIGN(size);
  2114. struct mm_struct *mm = vma->vm_mm;
  2115. unsigned long remap_pfn = pfn;
  2116. int err;
  2117. if (WARN_ON_ONCE(!PAGE_ALIGNED(addr)))
  2118. return -EINVAL;
  2119. /*
  2120. * Physically remapped pages are special. Tell the
  2121. * rest of the world about it:
  2122. * VM_IO tells people not to look at these pages
  2123. * (accesses can have side effects).
  2124. * VM_PFNMAP tells the core MM that the base pages are just
  2125. * raw PFN mappings, and do not have a "struct page" associated
  2126. * with them.
  2127. * VM_DONTEXPAND
  2128. * Disable vma merging and expanding with mremap().
  2129. * VM_DONTDUMP
  2130. * Omit vma from core dump, even when VM_IO turned off.
  2131. *
  2132. * There's a horrible special case to handle copy-on-write
  2133. * behaviour that some programs depend on. We mark the "original"
  2134. * un-COW'ed pages by matching them up with "vma->vm_pgoff".
  2135. * See vm_normal_page() for details.
  2136. */
  2137. if (is_cow_mapping(vma->vm_flags)) {
  2138. if (addr != vma->vm_start || end != vma->vm_end)
  2139. return -EINVAL;
  2140. vma->vm_pgoff = pfn;
  2141. }
  2142. err = track_pfn_remap(vma, &prot, remap_pfn, addr, PAGE_ALIGN(size));
  2143. if (err)
  2144. return -EINVAL;
  2145. vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
  2146. BUG_ON(addr >= end);
  2147. pfn -= addr >> PAGE_SHIFT;
  2148. pgd = pgd_offset(mm, addr);
  2149. flush_cache_range(vma, addr, end);
  2150. do {
  2151. next = pgd_addr_end(addr, end);
  2152. err = remap_p4d_range(mm, pgd, addr, next,
  2153. pfn + (addr >> PAGE_SHIFT), prot);
  2154. if (err)
  2155. break;
  2156. } while (pgd++, addr = next, addr != end);
  2157. if (err)
  2158. untrack_pfn(vma, remap_pfn, PAGE_ALIGN(size));
  2159. return err;
  2160. }
  2161. EXPORT_SYMBOL(remap_pfn_range);
  2162. /**
  2163. * vm_iomap_memory - remap memory to userspace
  2164. * @vma: user vma to map to
  2165. * @start: start of the physical memory to be mapped
  2166. * @len: size of area
  2167. *
  2168. * This is a simplified io_remap_pfn_range() for common driver use. The
  2169. * driver just needs to give us the physical memory range to be mapped,
  2170. * we'll figure out the rest from the vma information.
  2171. *
  2172. * NOTE! Some drivers might want to tweak vma->vm_page_prot first to get
  2173. * whatever write-combining details or similar.
  2174. *
  2175. * Return: %0 on success, negative error code otherwise.
  2176. */
  2177. int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len)
  2178. {
  2179. unsigned long vm_len, pfn, pages;
  2180. /* Check that the physical memory area passed in looks valid */
  2181. if (start + len < start)
  2182. return -EINVAL;
  2183. /*
  2184. * You *really* shouldn't map things that aren't page-aligned,
  2185. * but we've historically allowed it because IO memory might
  2186. * just have smaller alignment.
  2187. */
  2188. len += start & ~PAGE_MASK;
  2189. pfn = start >> PAGE_SHIFT;
  2190. pages = (len + ~PAGE_MASK) >> PAGE_SHIFT;
  2191. if (pfn + pages < pfn)
  2192. return -EINVAL;
  2193. /* We start the mapping 'vm_pgoff' pages into the area */
  2194. if (vma->vm_pgoff > pages)
  2195. return -EINVAL;
  2196. pfn += vma->vm_pgoff;
  2197. pages -= vma->vm_pgoff;
  2198. /* Can we fit all of the mapping? */
  2199. vm_len = vma->vm_end - vma->vm_start;
  2200. if (vm_len >> PAGE_SHIFT > pages)
  2201. return -EINVAL;
  2202. /* Ok, let it rip */
  2203. return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot);
  2204. }
  2205. EXPORT_SYMBOL(vm_iomap_memory);
  2206. static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd,
  2207. unsigned long addr, unsigned long end,
  2208. pte_fn_t fn, void *data, bool create,
  2209. pgtbl_mod_mask *mask)
  2210. {
  2211. pte_t *pte;
  2212. int err = 0;
  2213. spinlock_t *ptl;
  2214. if (create) {
  2215. pte = (mm == &init_mm) ?
  2216. pte_alloc_kernel_track(pmd, addr, mask) :
  2217. pte_alloc_map_lock(mm, pmd, addr, &ptl);
  2218. if (!pte)
  2219. return -ENOMEM;
  2220. } else {
  2221. pte = (mm == &init_mm) ?
  2222. pte_offset_kernel(pmd, addr) :
  2223. pte_offset_map_lock(mm, pmd, addr, &ptl);
  2224. }
  2225. BUG_ON(pmd_huge(*pmd));
  2226. arch_enter_lazy_mmu_mode();
  2227. if (fn) {
  2228. do {
  2229. if (create || !pte_none(*pte)) {
  2230. err = fn(pte++, addr, data);
  2231. if (err)
  2232. break;
  2233. }
  2234. } while (addr += PAGE_SIZE, addr != end);
  2235. }
  2236. *mask |= PGTBL_PTE_MODIFIED;
  2237. arch_leave_lazy_mmu_mode();
  2238. if (mm != &init_mm)
  2239. pte_unmap_unlock(pte-1, ptl);
  2240. return err;
  2241. }
  2242. static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud,
  2243. unsigned long addr, unsigned long end,
  2244. pte_fn_t fn, void *data, bool create,
  2245. pgtbl_mod_mask *mask)
  2246. {
  2247. pmd_t *pmd;
  2248. unsigned long next;
  2249. int err = 0;
  2250. BUG_ON(pud_huge(*pud));
  2251. if (create) {
  2252. pmd = pmd_alloc_track(mm, pud, addr, mask);
  2253. if (!pmd)
  2254. return -ENOMEM;
  2255. } else {
  2256. pmd = pmd_offset(pud, addr);
  2257. }
  2258. do {
  2259. next = pmd_addr_end(addr, end);
  2260. if (create || !pmd_none_or_clear_bad(pmd)) {
  2261. err = apply_to_pte_range(mm, pmd, addr, next, fn, data,
  2262. create, mask);
  2263. if (err)
  2264. break;
  2265. }
  2266. } while (pmd++, addr = next, addr != end);
  2267. return err;
  2268. }
  2269. static int apply_to_pud_range(struct mm_struct *mm, p4d_t *p4d,
  2270. unsigned long addr, unsigned long end,
  2271. pte_fn_t fn, void *data, bool create,
  2272. pgtbl_mod_mask *mask)
  2273. {
  2274. pud_t *pud;
  2275. unsigned long next;
  2276. int err = 0;
  2277. if (create) {
  2278. pud = pud_alloc_track(mm, p4d, addr, mask);
  2279. if (!pud)
  2280. return -ENOMEM;
  2281. } else {
  2282. pud = pud_offset(p4d, addr);
  2283. }
  2284. do {
  2285. next = pud_addr_end(addr, end);
  2286. if (create || !pud_none_or_clear_bad(pud)) {
  2287. err = apply_to_pmd_range(mm, pud, addr, next, fn, data,
  2288. create, mask);
  2289. if (err)
  2290. break;
  2291. }
  2292. } while (pud++, addr = next, addr != end);
  2293. return err;
  2294. }
  2295. static int apply_to_p4d_range(struct mm_struct *mm, pgd_t *pgd,
  2296. unsigned long addr, unsigned long end,
  2297. pte_fn_t fn, void *data, bool create,
  2298. pgtbl_mod_mask *mask)
  2299. {
  2300. p4d_t *p4d;
  2301. unsigned long next;
  2302. int err = 0;
  2303. if (create) {
  2304. p4d = p4d_alloc_track(mm, pgd, addr, mask);
  2305. if (!p4d)
  2306. return -ENOMEM;
  2307. } else {
  2308. p4d = p4d_offset(pgd, addr);
  2309. }
  2310. do {
  2311. next = p4d_addr_end(addr, end);
  2312. if (create || !p4d_none_or_clear_bad(p4d)) {
  2313. err = apply_to_pud_range(mm, p4d, addr, next, fn, data,
  2314. create, mask);
  2315. if (err)
  2316. break;
  2317. }
  2318. } while (p4d++, addr = next, addr != end);
  2319. return err;
  2320. }
  2321. static int __apply_to_page_range(struct mm_struct *mm, unsigned long addr,
  2322. unsigned long size, pte_fn_t fn,
  2323. void *data, bool create)
  2324. {
  2325. pgd_t *pgd;
  2326. unsigned long start = addr, next;
  2327. unsigned long end = addr + size;
  2328. pgtbl_mod_mask mask = 0;
  2329. int err = 0;
  2330. if (WARN_ON(addr >= end))
  2331. return -EINVAL;
  2332. pgd = pgd_offset(mm, addr);
  2333. do {
  2334. next = pgd_addr_end(addr, end);
  2335. if (!create && pgd_none_or_clear_bad(pgd))
  2336. continue;
  2337. err = apply_to_p4d_range(mm, pgd, addr, next, fn, data, create, &mask);
  2338. if (err)
  2339. break;
  2340. } while (pgd++, addr = next, addr != end);
  2341. if (mask & ARCH_PAGE_TABLE_SYNC_MASK)
  2342. arch_sync_kernel_mappings(start, start + size);
  2343. return err;
  2344. }
  2345. /*
  2346. * Scan a region of virtual memory, filling in page tables as necessary
  2347. * and calling a provided function on each leaf page table.
  2348. */
  2349. int apply_to_page_range(struct mm_struct *mm, unsigned long addr,
  2350. unsigned long size, pte_fn_t fn, void *data)
  2351. {
  2352. return __apply_to_page_range(mm, addr, size, fn, data, true);
  2353. }
  2354. EXPORT_SYMBOL_GPL(apply_to_page_range);
  2355. #ifdef CONFIG_SPECULATIVE_PAGE_FAULT
  2356. static bool pte_spinlock(struct vm_fault *vmf)
  2357. {
  2358. bool ret = false;
  2359. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  2360. pmd_t pmdval;
  2361. #endif
  2362. /* Check if vma is still valid */
  2363. if (!(vmf->flags & FAULT_FLAG_SPECULATIVE)) {
  2364. vmf->ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd);
  2365. spin_lock(vmf->ptl);
  2366. return true;
  2367. }
  2368. local_irq_disable();
  2369. if (vma_has_changed(vmf)) {
  2370. trace_spf_vma_changed(_RET_IP_, vmf->vma, vmf->address);
  2371. goto out;
  2372. }
  2373. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  2374. /*
  2375. * We check if the pmd value is still the same to ensure that there
  2376. * is not a huge collapse operation in progress in our back.
  2377. */
  2378. pmdval = READ_ONCE(*vmf->pmd);
  2379. if (!pmd_same(pmdval, vmf->orig_pmd)) {
  2380. trace_spf_pmd_changed(_RET_IP_, vmf->vma, vmf->address);
  2381. goto out;
  2382. }
  2383. #endif
  2384. vmf->ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd);
  2385. if (unlikely(!spin_trylock(vmf->ptl))) {
  2386. trace_spf_pte_lock(_RET_IP_, vmf->vma, vmf->address);
  2387. goto out;
  2388. }
  2389. if (vma_has_changed(vmf)) {
  2390. spin_unlock(vmf->ptl);
  2391. trace_spf_vma_changed(_RET_IP_, vmf->vma, vmf->address);
  2392. goto out;
  2393. }
  2394. ret = true;
  2395. out:
  2396. local_irq_enable();
  2397. return ret;
  2398. }
  2399. static bool __pte_map_lock_speculative(struct vm_fault *vmf, unsigned long addr)
  2400. {
  2401. bool ret = false;
  2402. pte_t *pte;
  2403. spinlock_t *ptl;
  2404. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  2405. pmd_t pmdval;
  2406. #endif
  2407. /*
  2408. * The first vma_has_changed() guarantees the page-tables are still
  2409. * valid, having IRQs disabled ensures they stay around, hence the
  2410. * second vma_has_changed() to make sure they are still valid once
  2411. * we've got the lock. After that a concurrent zap_pte_range() will
  2412. * block on the PTL and thus we're safe.
  2413. */
  2414. local_irq_disable();
  2415. if (vma_has_changed(vmf)) {
  2416. trace_spf_vma_changed(_RET_IP_, vmf->vma, addr);
  2417. goto out;
  2418. }
  2419. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  2420. /*
  2421. * We check if the pmd value is still the same to ensure that there
  2422. * is not a huge collapse operation in progress in our back.
  2423. */
  2424. pmdval = READ_ONCE(*vmf->pmd);
  2425. if (!pmd_same(pmdval, vmf->orig_pmd)) {
  2426. trace_spf_pmd_changed(_RET_IP_, vmf->vma, addr);
  2427. goto out;
  2428. }
  2429. #endif
  2430. /*
  2431. * Same as pte_offset_map_lock() except that we call
  2432. * spin_trylock() in place of spin_lock() to avoid race with
  2433. * unmap path which may have the lock and wait for this CPU
  2434. * to invalidate TLB but this CPU has irq disabled.
  2435. * Since we are in a speculative patch, accept it could fail
  2436. */
  2437. ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd);
  2438. pte = pte_offset_map(vmf->pmd, addr);
  2439. if (unlikely(!spin_trylock(ptl))) {
  2440. pte_unmap(pte);
  2441. trace_spf_pte_lock(_RET_IP_, vmf->vma, addr);
  2442. goto out;
  2443. }
  2444. if (vma_has_changed(vmf)) {
  2445. pte_unmap_unlock(pte, ptl);
  2446. trace_spf_vma_changed(_RET_IP_, vmf->vma, addr);
  2447. goto out;
  2448. }
  2449. vmf->pte = pte;
  2450. vmf->ptl = ptl;
  2451. ret = true;
  2452. out:
  2453. local_irq_enable();
  2454. return ret;
  2455. }
  2456. static bool pte_map_lock(struct vm_fault *vmf)
  2457. {
  2458. if (!(vmf->flags & FAULT_FLAG_SPECULATIVE)) {
  2459. vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd,
  2460. vmf->address, &vmf->ptl);
  2461. return true;
  2462. }
  2463. return __pte_map_lock_speculative(vmf, vmf->address);
  2464. }
  2465. bool pte_map_lock_addr(struct vm_fault *vmf, unsigned long addr)
  2466. {
  2467. if (!(vmf->flags & FAULT_FLAG_SPECULATIVE)) {
  2468. vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd,
  2469. addr, &vmf->ptl);
  2470. return true;
  2471. }
  2472. return __pte_map_lock_speculative(vmf, addr);
  2473. }
  2474. static bool __read_mostly allow_file_spec_access;
  2475. static int __init allow_file_spec_access_setup(char *str)
  2476. {
  2477. allow_file_spec_access = true;
  2478. return 1;
  2479. }
  2480. __setup("allow_file_spec_access", allow_file_spec_access_setup);
  2481. static bool vmf_allows_speculation(struct vm_fault *vmf)
  2482. {
  2483. if (vma_is_anonymous(vmf->vma)) {
  2484. /*
  2485. * __anon_vma_prepare() requires the mmap_sem to be held
  2486. * because vm_next and vm_prev must be safe. This can't be
  2487. * guaranteed in the speculative path.
  2488. */
  2489. if (!vmf->vma->anon_vma) {
  2490. trace_spf_vma_notsup(_RET_IP_, vmf->vma, vmf->address);
  2491. return false;
  2492. }
  2493. return true;
  2494. }
  2495. if (!allow_file_spec_access) {
  2496. /*
  2497. * Can't call vm_ops service has we don't know what they would
  2498. * do with the VMA.
  2499. * This include huge page from hugetlbfs.
  2500. */
  2501. trace_spf_vma_notsup(_RET_IP_, vmf->vma, vmf->address);
  2502. return false;
  2503. }
  2504. if (!(vmf->vma->vm_flags & VM_SHARED) &&
  2505. (vmf->flags & FAULT_FLAG_WRITE) &&
  2506. !vmf->vma->anon_vma) {
  2507. /*
  2508. * non-anonymous private COW without anon_vma.
  2509. * See above.
  2510. */
  2511. trace_spf_vma_notsup(_RET_IP_, vmf->vma, vmf->address);
  2512. return false;
  2513. }
  2514. if (vmf->vma->vm_ops->allow_speculation &&
  2515. vmf->vma->vm_ops->allow_speculation()) {
  2516. return true;
  2517. }
  2518. trace_spf_vma_notsup(_RET_IP_, vmf->vma, vmf->address);
  2519. return false;
  2520. }
  2521. #else
  2522. static inline bool pte_spinlock(struct vm_fault *vmf)
  2523. {
  2524. vmf->ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd);
  2525. spin_lock(vmf->ptl);
  2526. return true;
  2527. }
  2528. static inline bool pte_map_lock(struct vm_fault *vmf)
  2529. {
  2530. vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd,
  2531. vmf->address, &vmf->ptl);
  2532. return true;
  2533. }
  2534. inline bool pte_map_lock_addr(struct vm_fault *vmf, unsigned long addr)
  2535. {
  2536. vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd,
  2537. addr, &vmf->ptl);
  2538. return true;
  2539. }
  2540. static inline bool vmf_allows_speculation(struct vm_fault *vmf)
  2541. {
  2542. return false;
  2543. }
  2544. #endif /* CONFIG_SPECULATIVE_PAGE_FAULT */
  2545. /*
  2546. * Scan a region of virtual memory, calling a provided function on
  2547. * each leaf page table where it exists.
  2548. *
  2549. * Unlike apply_to_page_range, this does _not_ fill in page tables
  2550. * where they are absent.
  2551. */
  2552. int apply_to_existing_page_range(struct mm_struct *mm, unsigned long addr,
  2553. unsigned long size, pte_fn_t fn, void *data)
  2554. {
  2555. return __apply_to_page_range(mm, addr, size, fn, data, false);
  2556. }
  2557. EXPORT_SYMBOL_GPL(apply_to_existing_page_range);
  2558. /*
  2559. * handle_pte_fault chooses page fault handler according to an entry which was
  2560. * read non-atomically. Before making any commitment, on those architectures
  2561. * or configurations (e.g. i386 with PAE) which might give a mix of unmatched
  2562. * parts, do_swap_page must check under lock before unmapping the pte and
  2563. * proceeding (but do_wp_page is only called after already making such a check;
  2564. * and do_anonymous_page can safely check later on).
  2565. *
  2566. * pte_unmap_same() returns:
  2567. * 0 if the PTE are the same
  2568. * VM_FAULT_PTNOTSAME if the PTE are different
  2569. * VM_FAULT_RETRY if the VMA has changed in our back during
  2570. * a speculative page fault handling.
  2571. */
  2572. static inline int pte_unmap_same(struct vm_fault *vmf)
  2573. {
  2574. int ret = 0;
  2575. #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT)
  2576. if (sizeof(pte_t) > sizeof(unsigned long)) {
  2577. if (pte_spinlock(vmf)) {
  2578. if (!pte_same(*vmf->pte, vmf->orig_pte))
  2579. ret = VM_FAULT_PTNOTSAME;
  2580. spin_unlock(vmf->ptl);
  2581. } else
  2582. ret = VM_FAULT_RETRY;
  2583. }
  2584. #endif
  2585. pte_unmap(vmf->pte);
  2586. return ret;
  2587. }
  2588. static inline bool cow_user_page(struct page *dst, struct page *src,
  2589. struct vm_fault *vmf)
  2590. {
  2591. bool ret;
  2592. void *kaddr;
  2593. void __user *uaddr;
  2594. bool locked = false;
  2595. struct vm_area_struct *vma = vmf->vma;
  2596. struct mm_struct *mm = vma->vm_mm;
  2597. unsigned long addr = vmf->address;
  2598. if (likely(src)) {
  2599. copy_user_highpage(dst, src, addr, vma);
  2600. return true;
  2601. }
  2602. /*
  2603. * If the source page was a PFN mapping, we don't have
  2604. * a "struct page" for it. We do a best-effort copy by
  2605. * just copying from the original user address. If that
  2606. * fails, we just zero-fill it. Live with it.
  2607. */
  2608. kaddr = kmap_atomic(dst);
  2609. uaddr = (void __user *)(addr & PAGE_MASK);
  2610. /*
  2611. * On architectures with software "accessed" bits, we would
  2612. * take a double page fault, so mark it accessed here.
  2613. */
  2614. if (arch_faults_on_old_pte() && !pte_young(vmf->orig_pte)) {
  2615. pte_t entry;
  2616. vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl);
  2617. locked = true;
  2618. if (!likely(pte_same(*vmf->pte, vmf->orig_pte))) {
  2619. /*
  2620. * Other thread has already handled the fault
  2621. * and update local tlb only
  2622. */
  2623. update_mmu_tlb(vma, addr, vmf->pte);
  2624. ret = false;
  2625. goto pte_unlock;
  2626. }
  2627. entry = pte_mkyoung(vmf->orig_pte);
  2628. if (ptep_set_access_flags(vma, addr, vmf->pte, entry, 0))
  2629. update_mmu_cache(vma, addr, vmf->pte);
  2630. }
  2631. /*
  2632. * This really shouldn't fail, because the page is there
  2633. * in the page tables. But it might just be unreadable,
  2634. * in which case we just give up and fill the result with
  2635. * zeroes.
  2636. */
  2637. if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) {
  2638. if (locked)
  2639. goto warn;
  2640. /* Re-validate under PTL if the page is still mapped */
  2641. vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl);
  2642. locked = true;
  2643. if (!likely(pte_same(*vmf->pte, vmf->orig_pte))) {
  2644. /* The PTE changed under us, update local tlb */
  2645. update_mmu_tlb(vma, addr, vmf->pte);
  2646. ret = false;
  2647. goto pte_unlock;
  2648. }
  2649. /*
  2650. * The same page can be mapped back since last copy attempt.
  2651. * Try to copy again under PTL.
  2652. */
  2653. if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) {
  2654. /*
  2655. * Give a warn in case there can be some obscure
  2656. * use-case
  2657. */
  2658. warn:
  2659. WARN_ON_ONCE(1);
  2660. clear_page(kaddr);
  2661. }
  2662. }
  2663. ret = true;
  2664. pte_unlock:
  2665. if (locked)
  2666. pte_unmap_unlock(vmf->pte, vmf->ptl);
  2667. kunmap_atomic(kaddr);
  2668. flush_dcache_page(dst);
  2669. return ret;
  2670. }
  2671. static gfp_t __get_fault_gfp_mask(struct vm_area_struct *vma)
  2672. {
  2673. struct file *vm_file = vma->vm_file;
  2674. if (vm_file)
  2675. return mapping_gfp_mask(vm_file->f_mapping) | __GFP_FS | __GFP_IO;
  2676. /*
  2677. * Special mappings (e.g. VDSO) do not have any file so fake
  2678. * a default GFP_KERNEL for them.
  2679. */
  2680. return GFP_KERNEL;
  2681. }
  2682. /*
  2683. * Notify the address space that the page is about to become writable so that
  2684. * it can prohibit this or wait for the page to get into an appropriate state.
  2685. *
  2686. * We do this without the lock held, so that it can sleep if it needs to.
  2687. */
  2688. static vm_fault_t do_page_mkwrite(struct vm_fault *vmf)
  2689. {
  2690. vm_fault_t ret;
  2691. struct page *page = vmf->page;
  2692. unsigned int old_flags = vmf->flags;
  2693. vmf->flags = FAULT_FLAG_WRITE|FAULT_FLAG_MKWRITE;
  2694. if (vmf->vma->vm_file &&
  2695. IS_SWAPFILE(vmf->vma->vm_file->f_mapping->host))
  2696. return VM_FAULT_SIGBUS;
  2697. ret = vmf->vma->vm_ops->page_mkwrite(vmf);
  2698. /* Restore original flags so that caller is not surprised */
  2699. vmf->flags = old_flags;
  2700. if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))
  2701. return ret;
  2702. if (unlikely(!(ret & VM_FAULT_LOCKED))) {
  2703. lock_page(page);
  2704. if (!page->mapping) {
  2705. unlock_page(page);
  2706. return 0; /* retry */
  2707. }
  2708. ret |= VM_FAULT_LOCKED;
  2709. } else
  2710. VM_BUG_ON_PAGE(!PageLocked(page), page);
  2711. return ret;
  2712. }
  2713. /*
  2714. * Handle dirtying of a page in shared file mapping on a write fault.
  2715. *
  2716. * The function expects the page to be locked and unlocks it.
  2717. */
  2718. static vm_fault_t fault_dirty_shared_page(struct vm_fault *vmf)
  2719. {
  2720. struct vm_area_struct *vma = vmf->vma;
  2721. struct address_space *mapping;
  2722. struct page *page = vmf->page;
  2723. bool dirtied;
  2724. bool page_mkwrite = vma->vm_ops && vma->vm_ops->page_mkwrite;
  2725. dirtied = set_page_dirty(page);
  2726. VM_BUG_ON_PAGE(PageAnon(page), page);
  2727. /*
  2728. * Take a local copy of the address_space - page.mapping may be zeroed
  2729. * by truncate after unlock_page(). The address_space itself remains
  2730. * pinned by vma->vm_file's reference. We rely on unlock_page()'s
  2731. * release semantics to prevent the compiler from undoing this copying.
  2732. */
  2733. mapping = page_rmapping(page);
  2734. unlock_page(page);
  2735. if (!page_mkwrite)
  2736. file_update_time(vma->vm_file);
  2737. /*
  2738. * Throttle page dirtying rate down to writeback speed.
  2739. *
  2740. * mapping may be NULL here because some device drivers do not
  2741. * set page.mapping but still dirty their pages
  2742. *
  2743. * Drop the mmap_lock before waiting on IO, if we can. The file
  2744. * is pinning the mapping, as per above.
  2745. */
  2746. if ((dirtied || page_mkwrite) && mapping) {
  2747. struct file *fpin;
  2748. fpin = maybe_unlock_mmap_for_io(vmf, NULL);
  2749. balance_dirty_pages_ratelimited(mapping);
  2750. if (fpin) {
  2751. fput(fpin);
  2752. return VM_FAULT_RETRY;
  2753. }
  2754. }
  2755. return 0;
  2756. }
  2757. /*
  2758. * Handle write page faults for pages that can be reused in the current vma
  2759. *
  2760. * This can happen either due to the mapping being with the VM_SHARED flag,
  2761. * or due to us being the last reference standing to the page. In either
  2762. * case, all we need to do here is to mark the page as writable and update
  2763. * any related book-keeping.
  2764. */
  2765. static inline void wp_page_reuse(struct vm_fault *vmf)
  2766. __releases(vmf->ptl)
  2767. {
  2768. struct vm_area_struct *vma = vmf->vma;
  2769. struct page *page = vmf->page;
  2770. pte_t entry;
  2771. /*
  2772. * Clear the pages cpupid information as the existing
  2773. * information potentially belongs to a now completely
  2774. * unrelated process.
  2775. */
  2776. if (page)
  2777. page_cpupid_xchg_last(page, (1 << LAST_CPUPID_SHIFT) - 1);
  2778. flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
  2779. entry = pte_mkyoung(vmf->orig_pte);
  2780. entry = maybe_mkwrite(pte_mkdirty(entry), vmf->vma_flags);
  2781. if (ptep_set_access_flags(vma, vmf->address, vmf->pte, entry, 1))
  2782. update_mmu_cache(vma, vmf->address, vmf->pte);
  2783. pte_unmap_unlock(vmf->pte, vmf->ptl);
  2784. count_vm_event(PGREUSE);
  2785. }
  2786. /*
  2787. * Handle the case of a page which we actually need to copy to a new page.
  2788. *
  2789. * Called with mmap_lock locked and the old page referenced, but
  2790. * without the ptl held.
  2791. *
  2792. * High level logic flow:
  2793. *
  2794. * - Allocate a page, copy the content of the old page to the new one.
  2795. * - Handle book keeping and accounting - cgroups, mmu-notifiers, etc.
  2796. * - Take the PTL. If the pte changed, bail out and release the allocated page
  2797. * - If the pte is still the way we remember it, update the page table and all
  2798. * relevant references. This includes dropping the reference the page-table
  2799. * held to the old page, as well as updating the rmap.
  2800. * - In any case, unlock the PTL and drop the reference we took to the old page.
  2801. */
  2802. static vm_fault_t wp_page_copy(struct vm_fault *vmf)
  2803. {
  2804. struct vm_area_struct *vma = vmf->vma;
  2805. struct mm_struct *mm = vma->vm_mm;
  2806. struct page *old_page = vmf->page;
  2807. struct page *new_page = NULL;
  2808. pte_t entry;
  2809. int page_copied = 0;
  2810. struct mmu_notifier_range range;
  2811. vm_fault_t ret = VM_FAULT_OOM;
  2812. if (unlikely(anon_vma_prepare(vma)))
  2813. goto out;
  2814. if (is_zero_pfn(pte_pfn(vmf->orig_pte))) {
  2815. new_page = alloc_zeroed_user_highpage_movable(vma,
  2816. vmf->address);
  2817. if (!new_page)
  2818. goto out;
  2819. } else {
  2820. new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma,
  2821. vmf->address);
  2822. if (!new_page)
  2823. goto out;
  2824. if (!cow_user_page(new_page, old_page, vmf)) {
  2825. /*
  2826. * COW failed, if the fault was solved by other,
  2827. * it's fine. If not, userspace would re-fault on
  2828. * the same address and we will handle the fault
  2829. * from the second attempt.
  2830. */
  2831. put_page(new_page);
  2832. if (old_page)
  2833. put_page(old_page);
  2834. return 0;
  2835. }
  2836. }
  2837. if (mem_cgroup_charge(new_page, mm, GFP_KERNEL))
  2838. goto out_free_new;
  2839. cgroup_throttle_swaprate(new_page, GFP_KERNEL);
  2840. __SetPageUptodate(new_page);
  2841. mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm,
  2842. vmf->address & PAGE_MASK,
  2843. (vmf->address & PAGE_MASK) + PAGE_SIZE);
  2844. mmu_notifier_invalidate_range_start(&range);
  2845. /*
  2846. * Re-check the pte - we dropped the lock
  2847. */
  2848. if (!pte_map_lock(vmf)) {
  2849. ret = VM_FAULT_RETRY;
  2850. goto out_invalidate_end;
  2851. }
  2852. if (likely(pte_same(*vmf->pte, vmf->orig_pte))) {
  2853. if (old_page) {
  2854. if (!PageAnon(old_page)) {
  2855. dec_mm_counter_fast(mm,
  2856. mm_counter_file(old_page));
  2857. inc_mm_counter_fast(mm, MM_ANONPAGES);
  2858. }
  2859. } else {
  2860. inc_mm_counter_fast(mm, MM_ANONPAGES);
  2861. }
  2862. flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
  2863. entry = mk_pte(new_page, vmf->vma_page_prot);
  2864. entry = pte_sw_mkyoung(entry);
  2865. entry = maybe_mkwrite(pte_mkdirty(entry), vmf->vma_flags);
  2866. /*
  2867. * Clear the pte entry and flush it first, before updating the
  2868. * pte with the new entry. This will avoid a race condition
  2869. * seen in the presence of one thread doing SMC and another
  2870. * thread doing COW.
  2871. */
  2872. ptep_clear_flush_notify(vma, vmf->address, vmf->pte);
  2873. __page_add_new_anon_rmap(new_page, vma, vmf->address, false);
  2874. __lru_cache_add_inactive_or_unevictable(new_page, vmf->vma_flags);
  2875. /*
  2876. * We call the notify macro here because, when using secondary
  2877. * mmu page tables (such as kvm shadow page tables), we want the
  2878. * new page to be mapped directly into the secondary page table.
  2879. */
  2880. set_pte_at_notify(mm, vmf->address, vmf->pte, entry);
  2881. update_mmu_cache(vma, vmf->address, vmf->pte);
  2882. if (old_page) {
  2883. /*
  2884. * Only after switching the pte to the new page may
  2885. * we remove the mapcount here. Otherwise another
  2886. * process may come and find the rmap count decremented
  2887. * before the pte is switched to the new page, and
  2888. * "reuse" the old page writing into it while our pte
  2889. * here still points into it and can be read by other
  2890. * threads.
  2891. *
  2892. * The critical issue is to order this
  2893. * page_remove_rmap with the ptp_clear_flush above.
  2894. * Those stores are ordered by (if nothing else,)
  2895. * the barrier present in the atomic_add_negative
  2896. * in page_remove_rmap.
  2897. *
  2898. * Then the TLB flush in ptep_clear_flush ensures that
  2899. * no process can access the old page before the
  2900. * decremented mapcount is visible. And the old page
  2901. * cannot be reused until after the decremented
  2902. * mapcount is visible. So transitively, TLBs to
  2903. * old page will be flushed before it can be reused.
  2904. */
  2905. page_remove_rmap(old_page, false);
  2906. }
  2907. /* Free the old page.. */
  2908. new_page = old_page;
  2909. page_copied = 1;
  2910. } else {
  2911. update_mmu_tlb(vma, vmf->address, vmf->pte);
  2912. }
  2913. if (new_page)
  2914. put_page(new_page);
  2915. pte_unmap_unlock(vmf->pte, vmf->ptl);
  2916. /*
  2917. * No need to double call mmu_notifier->invalidate_range() callback as
  2918. * the above ptep_clear_flush_notify() did already call it.
  2919. */
  2920. mmu_notifier_invalidate_range_only_end(&range);
  2921. if (old_page) {
  2922. /*
  2923. * Don't let another task, with possibly unlocked vma,
  2924. * keep the mlocked page.
  2925. */
  2926. if (page_copied && (vmf->vma_flags & VM_LOCKED)) {
  2927. lock_page(old_page); /* LRU manipulation */
  2928. if (PageMlocked(old_page))
  2929. munlock_vma_page(old_page);
  2930. unlock_page(old_page);
  2931. }
  2932. put_page(old_page);
  2933. }
  2934. return page_copied ? VM_FAULT_WRITE : 0;
  2935. out_invalidate_end:
  2936. mmu_notifier_invalidate_range_only_end(&range);
  2937. out_free_new:
  2938. put_page(new_page);
  2939. out:
  2940. if (old_page)
  2941. put_page(old_page);
  2942. return ret;
  2943. }
  2944. /**
  2945. * finish_mkwrite_fault - finish page fault for a shared mapping, making PTE
  2946. * writeable once the page is prepared
  2947. *
  2948. * @vmf: structure describing the fault
  2949. *
  2950. * This function handles all that is needed to finish a write page fault in a
  2951. * shared mapping due to PTE being read-only once the mapped page is prepared.
  2952. * It handles locking of PTE and modifying it.
  2953. *
  2954. * The function expects the page to be locked or other protection against
  2955. * concurrent faults / writeback (such as DAX radix tree locks).
  2956. *
  2957. * Return: %VM_FAULT_WRITE on success, %0 when PTE got changed before
  2958. * we acquired PTE lock.
  2959. */
  2960. vm_fault_t finish_mkwrite_fault(struct vm_fault *vmf)
  2961. {
  2962. WARN_ON_ONCE(!(vmf->vma_flags & VM_SHARED));
  2963. if (!pte_map_lock(vmf))
  2964. return VM_FAULT_RETRY;
  2965. /*
  2966. * We might have raced with another page fault while we released the
  2967. * pte_offset_map_lock.
  2968. */
  2969. if (!pte_same(*vmf->pte, vmf->orig_pte)) {
  2970. update_mmu_tlb(vmf->vma, vmf->address, vmf->pte);
  2971. pte_unmap_unlock(vmf->pte, vmf->ptl);
  2972. return VM_FAULT_NOPAGE;
  2973. }
  2974. wp_page_reuse(vmf);
  2975. return 0;
  2976. }
  2977. /*
  2978. * Handle write page faults for VM_MIXEDMAP or VM_PFNMAP for a VM_SHARED
  2979. * mapping
  2980. */
  2981. static vm_fault_t wp_pfn_shared(struct vm_fault *vmf)
  2982. {
  2983. struct vm_area_struct *vma = vmf->vma;
  2984. if (vma->vm_ops && vma->vm_ops->pfn_mkwrite) {
  2985. vm_fault_t ret;
  2986. pte_unmap_unlock(vmf->pte, vmf->ptl);
  2987. vmf->flags |= FAULT_FLAG_MKWRITE;
  2988. ret = vma->vm_ops->pfn_mkwrite(vmf);
  2989. if (ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))
  2990. return ret;
  2991. return finish_mkwrite_fault(vmf);
  2992. }
  2993. wp_page_reuse(vmf);
  2994. return VM_FAULT_WRITE;
  2995. }
  2996. static vm_fault_t wp_page_shared(struct vm_fault *vmf)
  2997. __releases(vmf->ptl)
  2998. {
  2999. struct vm_area_struct *vma = vmf->vma;
  3000. vm_fault_t ret = VM_FAULT_WRITE;
  3001. get_page(vmf->page);
  3002. if (vma->vm_ops && vma->vm_ops->page_mkwrite) {
  3003. vm_fault_t tmp;
  3004. pte_unmap_unlock(vmf->pte, vmf->ptl);
  3005. tmp = do_page_mkwrite(vmf);
  3006. if (unlikely(!tmp || (tmp &
  3007. (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
  3008. put_page(vmf->page);
  3009. return tmp;
  3010. }
  3011. tmp = finish_mkwrite_fault(vmf);
  3012. if (unlikely(tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) {
  3013. unlock_page(vmf->page);
  3014. put_page(vmf->page);
  3015. return tmp;
  3016. }
  3017. } else {
  3018. wp_page_reuse(vmf);
  3019. lock_page(vmf->page);
  3020. }
  3021. ret |= fault_dirty_shared_page(vmf);
  3022. put_page(vmf->page);
  3023. return ret;
  3024. }
  3025. /*
  3026. * This routine handles present pages, when users try to write
  3027. * to a shared page. It is done by copying the page to a new address
  3028. * and decrementing the shared-page counter for the old page.
  3029. *
  3030. * Note that this routine assumes that the protection checks have been
  3031. * done by the caller (the low-level page fault routine in most cases).
  3032. * Thus we can safely just mark it writable once we've done any necessary
  3033. * COW.
  3034. *
  3035. * We also mark the page dirty at this point even though the page will
  3036. * change only once the write actually happens. This avoids a few races,
  3037. * and potentially makes it more efficient.
  3038. *
  3039. * We enter with non-exclusive mmap_lock (to exclude vma changes,
  3040. * but allow concurrent faults), with pte both mapped and locked.
  3041. * We return with mmap_lock still held, but pte unmapped and unlocked.
  3042. */
  3043. static vm_fault_t do_wp_page(struct vm_fault *vmf)
  3044. __releases(vmf->ptl)
  3045. {
  3046. struct vm_area_struct *vma = vmf->vma;
  3047. if (userfaultfd_pte_wp(vma, *vmf->pte)) {
  3048. pte_unmap_unlock(vmf->pte, vmf->ptl);
  3049. return handle_userfault(vmf, VM_UFFD_WP);
  3050. }
  3051. /*
  3052. * Userfaultfd write-protect can defer flushes. Ensure the TLB
  3053. * is flushed in this case before copying.
  3054. */
  3055. if (unlikely(userfaultfd_wp(vmf->vma) &&
  3056. mm_tlb_flush_pending(vmf->vma->vm_mm)))
  3057. flush_tlb_page(vmf->vma, vmf->address);
  3058. vmf->page = _vm_normal_page(vma, vmf->address, vmf->orig_pte,
  3059. vmf->vma_flags);
  3060. if (!vmf->page) {
  3061. /*
  3062. * VM_MIXEDMAP !pfn_valid() case, or VM_SOFTDIRTY clear on a
  3063. * VM_PFNMAP VMA.
  3064. *
  3065. * We should not cow pages in a shared writeable mapping.
  3066. * Just mark the pages writable and/or call ops->pfn_mkwrite.
  3067. */
  3068. if ((vmf->vma_flags & (VM_WRITE|VM_SHARED)) ==
  3069. (VM_WRITE|VM_SHARED))
  3070. return wp_pfn_shared(vmf);
  3071. pte_unmap_unlock(vmf->pte, vmf->ptl);
  3072. return wp_page_copy(vmf);
  3073. }
  3074. /*
  3075. * Take out anonymous pages first, anonymous shared vmas are
  3076. * not dirty accountable.
  3077. */
  3078. if (PageAnon(vmf->page)) {
  3079. struct page *page = vmf->page;
  3080. /* PageKsm() doesn't necessarily raise the page refcount */
  3081. if (PageKsm(page) || page_count(page) != 1)
  3082. goto copy;
  3083. if (!trylock_page(page))
  3084. goto copy;
  3085. if (PageKsm(page) || page_mapcount(page) != 1 || page_count(page) != 1) {
  3086. unlock_page(page);
  3087. goto copy;
  3088. }
  3089. /*
  3090. * Ok, we've got the only map reference, and the only
  3091. * page count reference, and the page is locked,
  3092. * it's dark out, and we're wearing sunglasses. Hit it.
  3093. */
  3094. unlock_page(page);
  3095. wp_page_reuse(vmf);
  3096. return VM_FAULT_WRITE;
  3097. } else if (unlikely((vmf->vma_flags & (VM_WRITE|VM_SHARED)) ==
  3098. (VM_WRITE|VM_SHARED))) {
  3099. return wp_page_shared(vmf);
  3100. }
  3101. copy:
  3102. /*
  3103. * Ok, we need to copy. Oh, well..
  3104. */
  3105. get_page(vmf->page);
  3106. pte_unmap_unlock(vmf->pte, vmf->ptl);
  3107. return wp_page_copy(vmf);
  3108. }
  3109. static void unmap_mapping_range_vma(struct vm_area_struct *vma,
  3110. unsigned long start_addr, unsigned long end_addr,
  3111. struct zap_details *details)
  3112. {
  3113. zap_page_range_single(vma, start_addr, end_addr - start_addr, details);
  3114. }
  3115. static inline void unmap_mapping_range_tree(struct rb_root_cached *root,
  3116. struct zap_details *details)
  3117. {
  3118. struct vm_area_struct *vma;
  3119. pgoff_t vba, vea, zba, zea;
  3120. vma_interval_tree_foreach(vma, root,
  3121. details->first_index, details->last_index) {
  3122. vba = vma->vm_pgoff;
  3123. vea = vba + vma_pages(vma) - 1;
  3124. zba = details->first_index;
  3125. if (zba < vba)
  3126. zba = vba;
  3127. zea = details->last_index;
  3128. if (zea > vea)
  3129. zea = vea;
  3130. unmap_mapping_range_vma(vma,
  3131. ((zba - vba) << PAGE_SHIFT) + vma->vm_start,
  3132. ((zea - vba + 1) << PAGE_SHIFT) + vma->vm_start,
  3133. details);
  3134. }
  3135. }
  3136. /**
  3137. * unmap_mapping_page() - Unmap single page from processes.
  3138. * @page: The locked page to be unmapped.
  3139. *
  3140. * Unmap this page from any userspace process which still has it mmaped.
  3141. * Typically, for efficiency, the range of nearby pages has already been
  3142. * unmapped by unmap_mapping_pages() or unmap_mapping_range(). But once
  3143. * truncation or invalidation holds the lock on a page, it may find that
  3144. * the page has been remapped again: and then uses unmap_mapping_page()
  3145. * to unmap it finally.
  3146. */
  3147. void unmap_mapping_page(struct page *page)
  3148. {
  3149. struct address_space *mapping = page->mapping;
  3150. struct zap_details details = { };
  3151. VM_BUG_ON(!PageLocked(page));
  3152. VM_BUG_ON(PageTail(page));
  3153. details.check_mapping = mapping;
  3154. details.first_index = page->index;
  3155. details.last_index = page->index + thp_nr_pages(page) - 1;
  3156. details.single_page = page;
  3157. i_mmap_lock_write(mapping);
  3158. if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root)))
  3159. unmap_mapping_range_tree(&mapping->i_mmap, &details);
  3160. i_mmap_unlock_write(mapping);
  3161. }
  3162. /**
  3163. * unmap_mapping_pages() - Unmap pages from processes.
  3164. * @mapping: The address space containing pages to be unmapped.
  3165. * @start: Index of first page to be unmapped.
  3166. * @nr: Number of pages to be unmapped. 0 to unmap to end of file.
  3167. * @even_cows: Whether to unmap even private COWed pages.
  3168. *
  3169. * Unmap the pages in this address space from any userspace process which
  3170. * has them mmaped. Generally, you want to remove COWed pages as well when
  3171. * a file is being truncated, but not when invalidating pages from the page
  3172. * cache.
  3173. */
  3174. void unmap_mapping_pages(struct address_space *mapping, pgoff_t start,
  3175. pgoff_t nr, bool even_cows)
  3176. {
  3177. struct zap_details details = { };
  3178. details.check_mapping = even_cows ? NULL : mapping;
  3179. details.first_index = start;
  3180. details.last_index = start + nr - 1;
  3181. if (details.last_index < details.first_index)
  3182. details.last_index = ULONG_MAX;
  3183. i_mmap_lock_write(mapping);
  3184. if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root)))
  3185. unmap_mapping_range_tree(&mapping->i_mmap, &details);
  3186. i_mmap_unlock_write(mapping);
  3187. }
  3188. /**
  3189. * unmap_mapping_range - unmap the portion of all mmaps in the specified
  3190. * address_space corresponding to the specified byte range in the underlying
  3191. * file.
  3192. *
  3193. * @mapping: the address space containing mmaps to be unmapped.
  3194. * @holebegin: byte in first page to unmap, relative to the start of
  3195. * the underlying file. This will be rounded down to a PAGE_SIZE
  3196. * boundary. Note that this is different from truncate_pagecache(), which
  3197. * must keep the partial page. In contrast, we must get rid of
  3198. * partial pages.
  3199. * @holelen: size of prospective hole in bytes. This will be rounded
  3200. * up to a PAGE_SIZE boundary. A holelen of zero truncates to the
  3201. * end of the file.
  3202. * @even_cows: 1 when truncating a file, unmap even private COWed pages;
  3203. * but 0 when invalidating pagecache, don't throw away private data.
  3204. */
  3205. void unmap_mapping_range(struct address_space *mapping,
  3206. loff_t const holebegin, loff_t const holelen, int even_cows)
  3207. {
  3208. pgoff_t hba = holebegin >> PAGE_SHIFT;
  3209. pgoff_t hlen = (holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
  3210. /* Check for overflow. */
  3211. if (sizeof(holelen) > sizeof(hlen)) {
  3212. long long holeend =
  3213. (holebegin + holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
  3214. if (holeend & ~(long long)ULONG_MAX)
  3215. hlen = ULONG_MAX - hba + 1;
  3216. }
  3217. unmap_mapping_pages(mapping, hba, hlen, even_cows);
  3218. }
  3219. EXPORT_SYMBOL(unmap_mapping_range);
  3220. /*
  3221. * We enter with non-exclusive mmap_lock (to exclude vma changes,
  3222. * but allow concurrent faults), and pte mapped but not yet locked.
  3223. * We return with pte unmapped and unlocked.
  3224. *
  3225. * We return with the mmap_lock locked or unlocked in the same cases
  3226. * as does filemap_fault().
  3227. */
  3228. vm_fault_t do_swap_page(struct vm_fault *vmf)
  3229. {
  3230. struct vm_area_struct *vma = vmf->vma;
  3231. struct page *page = NULL, *swapcache;
  3232. swp_entry_t entry;
  3233. pte_t pte;
  3234. int locked;
  3235. int exclusive = 0;
  3236. vm_fault_t ret;
  3237. void *shadow = NULL;
  3238. ret = pte_unmap_same(vmf);
  3239. if (ret) {
  3240. /*
  3241. * If pte != orig_pte, this means another thread did the
  3242. * swap operation in our back.
  3243. * So nothing else to do.
  3244. */
  3245. if (ret == VM_FAULT_PTNOTSAME)
  3246. ret = 0;
  3247. goto out;
  3248. }
  3249. entry = pte_to_swp_entry(vmf->orig_pte);
  3250. if (unlikely(non_swap_entry(entry))) {
  3251. if (is_migration_entry(entry)) {
  3252. migration_entry_wait(vma->vm_mm, vmf->pmd,
  3253. vmf->address);
  3254. } else if (is_device_private_entry(entry)) {
  3255. vmf->page = device_private_entry_to_page(entry);
  3256. ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
  3257. } else if (is_hwpoison_entry(entry)) {
  3258. ret = VM_FAULT_HWPOISON;
  3259. } else {
  3260. print_bad_pte(vma, vmf->address, vmf->orig_pte, NULL);
  3261. ret = VM_FAULT_SIGBUS;
  3262. }
  3263. goto out;
  3264. }
  3265. delayacct_set_flag(DELAYACCT_PF_SWAPIN);
  3266. page = lookup_swap_cache(entry, vma, vmf->address);
  3267. swapcache = page;
  3268. if (!page) {
  3269. struct swap_info_struct *si = swp_swap_info(entry);
  3270. if (data_race(si->flags & SWP_SYNCHRONOUS_IO) &&
  3271. __swap_count(entry) == 1) {
  3272. /* skip swapcache */
  3273. gfp_t flags = GFP_HIGHUSER_MOVABLE;
  3274. trace_android_rvh_set_skip_swapcache_flags(&flags);
  3275. page = alloc_page_vma(flags, vma, vmf->address);
  3276. if (page) {
  3277. int err;
  3278. __SetPageLocked(page);
  3279. __SetPageSwapBacked(page);
  3280. set_page_private(page, entry.val);
  3281. /* Tell memcg to use swap ownership records */
  3282. SetPageSwapCache(page);
  3283. err = mem_cgroup_charge(page, vma->vm_mm,
  3284. GFP_KERNEL);
  3285. ClearPageSwapCache(page);
  3286. if (err) {
  3287. ret = VM_FAULT_OOM;
  3288. goto out_page;
  3289. }
  3290. shadow = get_shadow_from_swap_cache(entry);
  3291. if (shadow)
  3292. workingset_refault(page, shadow);
  3293. lru_cache_add(page);
  3294. swap_readpage(page, true);
  3295. }
  3296. } else if (vmf->flags & FAULT_FLAG_SPECULATIVE) {
  3297. /*
  3298. * Don't try readahead during a speculative page fault
  3299. * as the VMA's boundaries may change in our back.
  3300. * If the page is not in the swap cache and synchronous
  3301. * read is disabled, fall back to the regular page fault
  3302. * mechanism.
  3303. */
  3304. delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
  3305. ret = VM_FAULT_RETRY;
  3306. goto out;
  3307. } else {
  3308. page = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE,
  3309. vmf);
  3310. swapcache = page;
  3311. }
  3312. if (!page) {
  3313. /*
  3314. * Back out if the VMA has changed in our back during
  3315. * a speculative page fault or if somebody else
  3316. * faulted in this pte while we released the pte lock.
  3317. */
  3318. if (!pte_map_lock(vmf)) {
  3319. delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
  3320. ret = VM_FAULT_RETRY;
  3321. goto out;
  3322. }
  3323. if (likely(pte_same(*vmf->pte, vmf->orig_pte)))
  3324. ret = VM_FAULT_OOM;
  3325. delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
  3326. goto unlock;
  3327. }
  3328. /* Had to read the page from swap area: Major fault */
  3329. ret = VM_FAULT_MAJOR;
  3330. count_vm_event(PGMAJFAULT);
  3331. count_memcg_event_mm(vma->vm_mm, PGMAJFAULT);
  3332. } else if (PageHWPoison(page)) {
  3333. /*
  3334. * hwpoisoned dirty swapcache pages are kept for killing
  3335. * owner processes (which may be unknown at hwpoison time)
  3336. */
  3337. ret = VM_FAULT_HWPOISON;
  3338. delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
  3339. goto out_release;
  3340. }
  3341. locked = lock_page_or_retry(page, vma->vm_mm, vmf->flags);
  3342. delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
  3343. if (!locked) {
  3344. ret |= VM_FAULT_RETRY;
  3345. goto out_release;
  3346. }
  3347. /*
  3348. * Make sure try_to_free_swap or reuse_swap_page or swapoff did not
  3349. * release the swapcache from under us. The page pin, and pte_same
  3350. * test below, are not enough to exclude that. Even if it is still
  3351. * swapcache, we need to check that the page's swap has not changed.
  3352. */
  3353. if (unlikely((!PageSwapCache(page) ||
  3354. page_private(page) != entry.val)) && swapcache)
  3355. goto out_page;
  3356. page = ksm_might_need_to_copy(page, vma, vmf->address);
  3357. if (unlikely(!page)) {
  3358. ret = VM_FAULT_OOM;
  3359. page = swapcache;
  3360. goto out_page;
  3361. }
  3362. cgroup_throttle_swaprate(page, GFP_KERNEL);
  3363. /*
  3364. * Back out if the VMA has changed in our back during a speculative
  3365. * page fault or if somebody else already faulted in this pte.
  3366. */
  3367. if (!pte_map_lock(vmf)) {
  3368. ret = VM_FAULT_RETRY;
  3369. goto out_page;
  3370. }
  3371. if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte)))
  3372. goto out_nomap;
  3373. if (unlikely(!PageUptodate(page))) {
  3374. ret = VM_FAULT_SIGBUS;
  3375. goto out_nomap;
  3376. }
  3377. /*
  3378. * The page isn't present yet, go ahead with the fault.
  3379. *
  3380. * Be careful about the sequence of operations here.
  3381. * To get its accounting right, reuse_swap_page() must be called
  3382. * while the page is counted on swap but not yet in mapcount i.e.
  3383. * before page_add_anon_rmap() and swap_free(); try_to_free_swap()
  3384. * must be called after the swap_free(), or it will never succeed.
  3385. */
  3386. inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
  3387. dec_mm_counter_fast(vma->vm_mm, MM_SWAPENTS);
  3388. pte = mk_pte(page, vmf->vma_page_prot);
  3389. if ((vmf->flags & FAULT_FLAG_WRITE) && reuse_swap_page(page, NULL)) {
  3390. pte = maybe_mkwrite(pte_mkdirty(pte), vmf->vma_flags);
  3391. vmf->flags &= ~FAULT_FLAG_WRITE;
  3392. ret |= VM_FAULT_WRITE;
  3393. exclusive = RMAP_EXCLUSIVE;
  3394. }
  3395. flush_icache_page(vma, page);
  3396. if (pte_swp_soft_dirty(vmf->orig_pte))
  3397. pte = pte_mksoft_dirty(pte);
  3398. if (pte_swp_uffd_wp(vmf->orig_pte)) {
  3399. pte = pte_mkuffd_wp(pte);
  3400. pte = pte_wrprotect(pte);
  3401. }
  3402. set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte);
  3403. arch_do_swap_page(vma->vm_mm, vma, vmf->address, pte, vmf->orig_pte);
  3404. vmf->orig_pte = pte;
  3405. /* ksm created a completely new copy */
  3406. if (unlikely(page != swapcache && swapcache)) {
  3407. __page_add_new_anon_rmap(page, vma, vmf->address, false);
  3408. __lru_cache_add_inactive_or_unevictable(page, vmf->vma_flags);
  3409. } else {
  3410. do_page_add_anon_rmap(page, vma, vmf->address, exclusive);
  3411. }
  3412. swap_free(entry);
  3413. if (mem_cgroup_swap_full(page) ||
  3414. (vmf->vma_flags & VM_LOCKED) || PageMlocked(page))
  3415. try_to_free_swap(page);
  3416. unlock_page(page);
  3417. if (page != swapcache && swapcache) {
  3418. /*
  3419. * Hold the lock to avoid the swap entry to be reused
  3420. * until we take the PT lock for the pte_same() check
  3421. * (to avoid false positives from pte_same). For
  3422. * further safety release the lock after the swap_free
  3423. * so that the swap count won't change under a
  3424. * parallel locked swapcache.
  3425. */
  3426. unlock_page(swapcache);
  3427. put_page(swapcache);
  3428. }
  3429. if (vmf->flags & FAULT_FLAG_WRITE) {
  3430. ret |= do_wp_page(vmf);
  3431. if (ret & VM_FAULT_ERROR)
  3432. ret &= VM_FAULT_ERROR;
  3433. goto out;
  3434. }
  3435. /* No need to invalidate - it was non-present before */
  3436. update_mmu_cache(vma, vmf->address, vmf->pte);
  3437. unlock:
  3438. pte_unmap_unlock(vmf->pte, vmf->ptl);
  3439. out:
  3440. return ret;
  3441. out_nomap:
  3442. pte_unmap_unlock(vmf->pte, vmf->ptl);
  3443. out_page:
  3444. unlock_page(page);
  3445. out_release:
  3446. put_page(page);
  3447. if (page != swapcache && swapcache) {
  3448. unlock_page(swapcache);
  3449. put_page(swapcache);
  3450. }
  3451. return ret;
  3452. }
  3453. /*
  3454. * We enter with non-exclusive mmap_lock (to exclude vma changes,
  3455. * but allow concurrent faults), and pte mapped but not yet locked.
  3456. * We return with mmap_lock still held, but pte unmapped and unlocked.
  3457. */
  3458. static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
  3459. {
  3460. struct vm_area_struct *vma = vmf->vma;
  3461. struct page *page;
  3462. vm_fault_t ret = 0;
  3463. pte_t entry;
  3464. /* File mapping without ->vm_ops ? */
  3465. if (vmf->vma_flags & VM_SHARED)
  3466. return VM_FAULT_SIGBUS;
  3467. /*
  3468. * Use pte_alloc() instead of pte_alloc_map(). We can't run
  3469. * pte_offset_map() on pmds where a huge pmd might be created
  3470. * from a different thread.
  3471. *
  3472. * pte_alloc_map() is safe to use under mmap_write_lock(mm) or when
  3473. * parallel threads are excluded by other means.
  3474. *
  3475. * Here we only have mmap_read_lock(mm).
  3476. */
  3477. if (pte_alloc(vma->vm_mm, vmf->pmd))
  3478. return VM_FAULT_OOM;
  3479. /* See comment in handle_pte_fault() */
  3480. if (unlikely(pmd_trans_unstable(vmf->pmd)))
  3481. return 0;
  3482. /* Use the zero-page for reads */
  3483. if (!(vmf->flags & FAULT_FLAG_WRITE) &&
  3484. !mm_forbids_zeropage(vma->vm_mm)) {
  3485. entry = pte_mkspecial(pfn_pte(my_zero_pfn(vmf->address),
  3486. vmf->vma_page_prot));
  3487. if (!pte_map_lock(vmf))
  3488. return VM_FAULT_RETRY;
  3489. if (!pte_none(*vmf->pte)) {
  3490. update_mmu_tlb(vma, vmf->address, vmf->pte);
  3491. goto unlock;
  3492. }
  3493. ret = check_stable_address_space(vma->vm_mm);
  3494. if (ret)
  3495. goto unlock;
  3496. /*
  3497. * Don't call the userfaultfd during the speculative path.
  3498. * We already checked for the VMA to not be managed through
  3499. * userfaultfd, but it may be set in our back once we have lock
  3500. * the pte. In such a case we can ignore it this time.
  3501. */
  3502. if (vmf->flags & FAULT_FLAG_SPECULATIVE)
  3503. goto setpte;
  3504. /* Deliver the page fault to userland, check inside PT lock */
  3505. if (userfaultfd_missing(vma)) {
  3506. pte_unmap_unlock(vmf->pte, vmf->ptl);
  3507. return handle_userfault(vmf, VM_UFFD_MISSING);
  3508. }
  3509. goto setpte;
  3510. }
  3511. /* Allocate our own private page. */
  3512. if (unlikely(anon_vma_prepare(vma)))
  3513. goto oom;
  3514. page = alloc_zeroed_user_highpage_movable(vma, vmf->address);
  3515. if (!page)
  3516. goto oom;
  3517. if (mem_cgroup_charge(page, vma->vm_mm, GFP_KERNEL))
  3518. goto oom_free_page;
  3519. cgroup_throttle_swaprate(page, GFP_KERNEL);
  3520. /*
  3521. * The memory barrier inside __SetPageUptodate makes sure that
  3522. * preceding stores to the page contents become visible before
  3523. * the set_pte_at() write.
  3524. */
  3525. __SetPageUptodate(page);
  3526. entry = mk_pte(page, vmf->vma_page_prot);
  3527. entry = pte_sw_mkyoung(entry);
  3528. if (vmf->vma_flags & VM_WRITE)
  3529. entry = pte_mkwrite(pte_mkdirty(entry));
  3530. if (!pte_map_lock(vmf)) {
  3531. ret = VM_FAULT_RETRY;
  3532. goto release;
  3533. }
  3534. if (!pte_none(*vmf->pte)) {
  3535. update_mmu_cache(vma, vmf->address, vmf->pte);
  3536. goto unlock_and_release;
  3537. }
  3538. ret = check_stable_address_space(vma->vm_mm);
  3539. if (ret)
  3540. goto unlock_and_release;
  3541. /* Deliver the page fault to userland, check inside PT lock */
  3542. if (!(vmf->flags & FAULT_FLAG_SPECULATIVE) &&
  3543. userfaultfd_missing(vma)) {
  3544. pte_unmap_unlock(vmf->pte, vmf->ptl);
  3545. put_page(page);
  3546. return handle_userfault(vmf, VM_UFFD_MISSING);
  3547. }
  3548. inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
  3549. __page_add_new_anon_rmap(page, vma, vmf->address, false);
  3550. __lru_cache_add_inactive_or_unevictable(page, vmf->vma_flags);
  3551. setpte:
  3552. set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry);
  3553. /* No need to invalidate - it was non-present before */
  3554. update_mmu_cache(vma, vmf->address, vmf->pte);
  3555. unlock:
  3556. pte_unmap_unlock(vmf->pte, vmf->ptl);
  3557. return ret;
  3558. unlock_and_release:
  3559. pte_unmap_unlock(vmf->pte, vmf->ptl);
  3560. release:
  3561. put_page(page);
  3562. return ret;
  3563. oom_free_page:
  3564. put_page(page);
  3565. oom:
  3566. return VM_FAULT_OOM;
  3567. }
  3568. /*
  3569. * The mmap_lock must have been held on entry, and may have been
  3570. * released depending on flags and vma->vm_ops->fault() return value.
  3571. * See filemap_fault() and __lock_page_retry().
  3572. */
  3573. static vm_fault_t __do_fault(struct vm_fault *vmf)
  3574. {
  3575. struct vm_area_struct *vma = vmf->vma;
  3576. vm_fault_t ret;
  3577. /*
  3578. * Preallocate pte before we take page_lock because this might lead to
  3579. * deadlocks for memcg reclaim which waits for pages under writeback:
  3580. * lock_page(A)
  3581. * SetPageWriteback(A)
  3582. * unlock_page(A)
  3583. * lock_page(B)
  3584. * lock_page(B)
  3585. * pte_alloc_one
  3586. * shrink_page_list
  3587. * wait_on_page_writeback(A)
  3588. * SetPageWriteback(B)
  3589. * unlock_page(B)
  3590. * # flush A, B to clear the writeback
  3591. */
  3592. if (pmd_none(*vmf->pmd) && !vmf->prealloc_pte) {
  3593. vmf->prealloc_pte = pte_alloc_one(vma->vm_mm);
  3594. if (!vmf->prealloc_pte)
  3595. return VM_FAULT_OOM;
  3596. smp_wmb(); /* See comment in __pte_alloc() */
  3597. }
  3598. ret = vma->vm_ops->fault(vmf);
  3599. if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY |
  3600. VM_FAULT_DONE_COW)))
  3601. return ret;
  3602. if (unlikely(PageHWPoison(vmf->page))) {
  3603. struct page *page = vmf->page;
  3604. vm_fault_t poisonret = VM_FAULT_HWPOISON;
  3605. if (ret & VM_FAULT_LOCKED) {
  3606. if (page_mapped(page))
  3607. unmap_mapping_pages(page_mapping(page),
  3608. page->index, 1, false);
  3609. /* Retry if a clean page was removed from the cache. */
  3610. if (invalidate_inode_page(page))
  3611. poisonret = VM_FAULT_NOPAGE;
  3612. unlock_page(page);
  3613. }
  3614. put_page(page);
  3615. vmf->page = NULL;
  3616. return poisonret;
  3617. }
  3618. if (unlikely(!(ret & VM_FAULT_LOCKED)))
  3619. lock_page(vmf->page);
  3620. else
  3621. VM_BUG_ON_PAGE(!PageLocked(vmf->page), vmf->page);
  3622. return ret;
  3623. }
  3624. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  3625. static void deposit_prealloc_pte(struct vm_fault *vmf)
  3626. {
  3627. struct vm_area_struct *vma = vmf->vma;
  3628. pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, vmf->prealloc_pte);
  3629. /*
  3630. * We are going to consume the prealloc table,
  3631. * count that as nr_ptes.
  3632. */
  3633. mm_inc_nr_ptes(vma->vm_mm);
  3634. vmf->prealloc_pte = NULL;
  3635. }
  3636. vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page)
  3637. {
  3638. struct vm_area_struct *vma = vmf->vma;
  3639. bool write = vmf->flags & FAULT_FLAG_WRITE;
  3640. unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
  3641. pmd_t entry;
  3642. int i;
  3643. vm_fault_t ret = VM_FAULT_FALLBACK;
  3644. if (!transhuge_vma_suitable(vma, haddr))
  3645. return ret;
  3646. page = compound_head(page);
  3647. if (compound_order(page) != HPAGE_PMD_ORDER)
  3648. return ret;
  3649. /*
  3650. * Archs like ppc64 need additonal space to store information
  3651. * related to pte entry. Use the preallocated table for that.
  3652. */
  3653. if (arch_needs_pgtable_deposit() && !vmf->prealloc_pte) {
  3654. vmf->prealloc_pte = pte_alloc_one(vma->vm_mm);
  3655. if (!vmf->prealloc_pte)
  3656. return VM_FAULT_OOM;
  3657. smp_wmb(); /* See comment in __pte_alloc() */
  3658. }
  3659. vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
  3660. if (unlikely(!pmd_none(*vmf->pmd)))
  3661. goto out;
  3662. for (i = 0; i < HPAGE_PMD_NR; i++)
  3663. flush_icache_page(vma, page + i);
  3664. entry = mk_huge_pmd(page, vmf->vma_page_prot);
  3665. if (write)
  3666. entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
  3667. add_mm_counter(vma->vm_mm, mm_counter_file(page), HPAGE_PMD_NR);
  3668. page_add_file_rmap(page, true);
  3669. /*
  3670. * deposit and withdraw with pmd lock held
  3671. */
  3672. if (arch_needs_pgtable_deposit())
  3673. deposit_prealloc_pte(vmf);
  3674. set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);
  3675. update_mmu_cache_pmd(vma, haddr, vmf->pmd);
  3676. /* fault is handled */
  3677. ret = 0;
  3678. count_vm_event(THP_FILE_MAPPED);
  3679. out:
  3680. spin_unlock(vmf->ptl);
  3681. return ret;
  3682. }
  3683. #else
  3684. vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page)
  3685. {
  3686. return VM_FAULT_FALLBACK;
  3687. }
  3688. #endif
  3689. void do_set_pte(struct vm_fault *vmf, struct page *page, unsigned long addr)
  3690. {
  3691. struct vm_area_struct *vma = vmf->vma;
  3692. bool write = vmf->flags & FAULT_FLAG_WRITE;
  3693. bool prefault = vmf->address != addr;
  3694. pte_t entry;
  3695. flush_icache_page(vma, page);
  3696. entry = mk_pte(page, vmf->vma_page_prot);
  3697. if (prefault && arch_wants_old_prefaulted_pte())
  3698. entry = pte_mkold(entry);
  3699. else
  3700. entry = pte_sw_mkyoung(entry);
  3701. if (write)
  3702. entry = maybe_mkwrite(pte_mkdirty(entry), vmf->vma_flags);
  3703. /* copy-on-write page */
  3704. if (write && !(vmf->vma_flags & VM_SHARED)) {
  3705. inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
  3706. __page_add_new_anon_rmap(page, vma, addr, false);
  3707. __lru_cache_add_inactive_or_unevictable(page, vmf->vma_flags);
  3708. } else {
  3709. inc_mm_counter_fast(vma->vm_mm, mm_counter_file(page));
  3710. page_add_file_rmap(page, false);
  3711. }
  3712. set_pte_at(vma->vm_mm, addr, vmf->pte, entry);
  3713. }
  3714. /**
  3715. * finish_fault - finish page fault once we have prepared the page to fault
  3716. *
  3717. * @vmf: structure describing the fault
  3718. *
  3719. * This function handles all that is needed to finish a page fault once the
  3720. * page to fault in is prepared. It handles locking of PTEs, inserts PTE for
  3721. * given page, adds reverse page mapping, handles memcg charges and LRU
  3722. * addition.
  3723. *
  3724. * The function expects the page to be locked and on success it consumes a
  3725. * reference of a page being mapped (for the PTE which maps it).
  3726. *
  3727. * Return: %0 on success, %VM_FAULT_ code in case of error.
  3728. */
  3729. vm_fault_t finish_fault(struct vm_fault *vmf)
  3730. {
  3731. struct vm_area_struct *vma = vmf->vma;
  3732. struct page *page;
  3733. vm_fault_t ret;
  3734. /* Did we COW the page? */
  3735. if ((vmf->flags & FAULT_FLAG_WRITE) &&
  3736. !(vmf->vma_flags & VM_SHARED))
  3737. page = vmf->cow_page;
  3738. else
  3739. page = vmf->page;
  3740. /*
  3741. * check even for read faults because we might have lost our CoWed
  3742. * page
  3743. */
  3744. if (!(vma->vm_flags & VM_SHARED)) {
  3745. ret = check_stable_address_space(vma->vm_mm);
  3746. if (ret)
  3747. return ret;
  3748. }
  3749. if (pmd_none(*vmf->pmd) && !(vmf->flags & FAULT_FLAG_SPECULATIVE)) {
  3750. if (PageTransCompound(page)) {
  3751. ret = do_set_pmd(vmf, page);
  3752. if (ret != VM_FAULT_FALLBACK)
  3753. return ret;
  3754. }
  3755. if (vmf->prealloc_pte) {
  3756. vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
  3757. if (likely(pmd_none(*vmf->pmd))) {
  3758. mm_inc_nr_ptes(vma->vm_mm);
  3759. pmd_populate(vma->vm_mm, vmf->pmd, vmf->prealloc_pte);
  3760. vmf->prealloc_pte = NULL;
  3761. }
  3762. spin_unlock(vmf->ptl);
  3763. } else if (unlikely(pte_alloc(vma->vm_mm, vmf->pmd))) {
  3764. return VM_FAULT_OOM;
  3765. }
  3766. }
  3767. /* See comment in handle_pte_fault() */
  3768. if (pmd_devmap_trans_unstable(vmf->pmd))
  3769. return 0;
  3770. if (!pte_map_lock(vmf))
  3771. return VM_FAULT_RETRY;
  3772. ret = 0;
  3773. /* Re-check under ptl */
  3774. if (likely(pte_none(*vmf->pte)))
  3775. do_set_pte(vmf, page, vmf->address);
  3776. else
  3777. ret = VM_FAULT_NOPAGE;
  3778. update_mmu_tlb(vma, vmf->address, vmf->pte);
  3779. pte_unmap_unlock(vmf->pte, vmf->ptl);
  3780. return ret;
  3781. }
  3782. static unsigned long fault_around_bytes __read_mostly =
  3783. rounddown_pow_of_two(65536);
  3784. #ifdef CONFIG_DEBUG_FS
  3785. static int fault_around_bytes_get(void *data, u64 *val)
  3786. {
  3787. *val = fault_around_bytes;
  3788. return 0;
  3789. }
  3790. /*
  3791. * fault_around_bytes must be rounded down to the nearest page order as it's
  3792. * what do_fault_around() expects to see.
  3793. */
  3794. static int fault_around_bytes_set(void *data, u64 val)
  3795. {
  3796. if (val / PAGE_SIZE > PTRS_PER_PTE)
  3797. return -EINVAL;
  3798. if (val > PAGE_SIZE)
  3799. fault_around_bytes = rounddown_pow_of_two(val);
  3800. else
  3801. fault_around_bytes = PAGE_SIZE; /* rounddown_pow_of_two(0) is undefined */
  3802. return 0;
  3803. }
  3804. DEFINE_DEBUGFS_ATTRIBUTE(fault_around_bytes_fops,
  3805. fault_around_bytes_get, fault_around_bytes_set, "%llu\n");
  3806. static int __init fault_around_debugfs(void)
  3807. {
  3808. debugfs_create_file_unsafe("fault_around_bytes", 0644, NULL, NULL,
  3809. &fault_around_bytes_fops);
  3810. return 0;
  3811. }
  3812. late_initcall(fault_around_debugfs);
  3813. #endif
  3814. /*
  3815. * do_fault_around() tries to map few pages around the fault address. The hope
  3816. * is that the pages will be needed soon and this will lower the number of
  3817. * faults to handle.
  3818. *
  3819. * It uses vm_ops->map_pages() to map the pages, which skips the page if it's
  3820. * not ready to be mapped: not up-to-date, locked, etc.
  3821. *
  3822. * This function is called with the page table lock taken. In the split ptlock
  3823. * case the page table lock only protects only those entries which belong to
  3824. * the page table corresponding to the fault address.
  3825. *
  3826. * This function doesn't cross the VMA boundaries, in order to call map_pages()
  3827. * only once.
  3828. *
  3829. * fault_around_bytes defines how many bytes we'll try to map.
  3830. * do_fault_around() expects it to be set to a power of two less than or equal
  3831. * to PTRS_PER_PTE.
  3832. *
  3833. * The virtual address of the area that we map is naturally aligned to
  3834. * fault_around_bytes rounded down to the machine page size
  3835. * (and therefore to page order). This way it's easier to guarantee
  3836. * that we don't cross page table boundaries.
  3837. */
  3838. static vm_fault_t do_fault_around(struct vm_fault *vmf)
  3839. {
  3840. unsigned long address = vmf->address, nr_pages, mask;
  3841. pgoff_t start_pgoff = vmf->pgoff;
  3842. pgoff_t end_pgoff;
  3843. int off;
  3844. nr_pages = READ_ONCE(fault_around_bytes) >> PAGE_SHIFT;
  3845. mask = ~(nr_pages * PAGE_SIZE - 1) & PAGE_MASK;
  3846. address = max(address & mask, vmf->vma->vm_start);
  3847. off = ((vmf->address - address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1);
  3848. start_pgoff -= off;
  3849. /*
  3850. * end_pgoff is either the end of the page table, the end of
  3851. * the vma or nr_pages from start_pgoff, depending what is nearest.
  3852. */
  3853. end_pgoff = start_pgoff -
  3854. ((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) +
  3855. PTRS_PER_PTE - 1;
  3856. end_pgoff = min3(end_pgoff, vma_pages(vmf->vma) + vmf->vma->vm_pgoff - 1,
  3857. start_pgoff + nr_pages - 1);
  3858. if (pmd_none(*vmf->pmd)) {
  3859. vmf->prealloc_pte = pte_alloc_one(vmf->vma->vm_mm);
  3860. if (!vmf->prealloc_pte)
  3861. return VM_FAULT_OOM;
  3862. smp_wmb(); /* See comment in __pte_alloc() */
  3863. }
  3864. return vmf->vma->vm_ops->map_pages(vmf, start_pgoff, end_pgoff);
  3865. }
  3866. static vm_fault_t do_read_fault(struct vm_fault *vmf)
  3867. {
  3868. struct vm_area_struct *vma = vmf->vma;
  3869. vm_fault_t ret = 0;
  3870. /*
  3871. * Let's call ->map_pages() first and use ->fault() as fallback
  3872. * if page by the offset is not ready to be mapped (cold cache or
  3873. * something).
  3874. */
  3875. if (vma->vm_ops->map_pages && fault_around_bytes >> PAGE_SHIFT > 1) {
  3876. if (likely(!userfaultfd_minor(vmf->vma))) {
  3877. ret = do_fault_around(vmf);
  3878. if (ret)
  3879. return ret;
  3880. }
  3881. }
  3882. ret = __do_fault(vmf);
  3883. if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
  3884. return ret;
  3885. ret |= finish_fault(vmf);
  3886. unlock_page(vmf->page);
  3887. if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
  3888. put_page(vmf->page);
  3889. return ret;
  3890. }
  3891. static vm_fault_t do_cow_fault(struct vm_fault *vmf)
  3892. {
  3893. struct vm_area_struct *vma = vmf->vma;
  3894. vm_fault_t ret;
  3895. if (unlikely(anon_vma_prepare(vma)))
  3896. return VM_FAULT_OOM;
  3897. vmf->cow_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vmf->address);
  3898. if (!vmf->cow_page)
  3899. return VM_FAULT_OOM;
  3900. if (mem_cgroup_charge(vmf->cow_page, vma->vm_mm, GFP_KERNEL)) {
  3901. put_page(vmf->cow_page);
  3902. return VM_FAULT_OOM;
  3903. }
  3904. cgroup_throttle_swaprate(vmf->cow_page, GFP_KERNEL);
  3905. ret = __do_fault(vmf);
  3906. if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
  3907. goto uncharge_out;
  3908. if (ret & VM_FAULT_DONE_COW)
  3909. return ret;
  3910. copy_user_highpage(vmf->cow_page, vmf->page, vmf->address, vma);
  3911. __SetPageUptodate(vmf->cow_page);
  3912. ret |= finish_fault(vmf);
  3913. unlock_page(vmf->page);
  3914. put_page(vmf->page);
  3915. if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
  3916. goto uncharge_out;
  3917. return ret;
  3918. uncharge_out:
  3919. put_page(vmf->cow_page);
  3920. return ret;
  3921. }
  3922. static vm_fault_t do_shared_fault(struct vm_fault *vmf)
  3923. {
  3924. struct vm_area_struct *vma = vmf->vma;
  3925. vm_fault_t ret, tmp;
  3926. ret = __do_fault(vmf);
  3927. if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
  3928. return ret;
  3929. /*
  3930. * Check if the backing address space wants to know that the page is
  3931. * about to become writable
  3932. */
  3933. if (vma->vm_ops->page_mkwrite) {
  3934. unlock_page(vmf->page);
  3935. tmp = do_page_mkwrite(vmf);
  3936. if (unlikely(!tmp ||
  3937. (tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
  3938. put_page(vmf->page);
  3939. return tmp;
  3940. }
  3941. }
  3942. ret |= finish_fault(vmf);
  3943. if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE |
  3944. VM_FAULT_RETRY))) {
  3945. unlock_page(vmf->page);
  3946. put_page(vmf->page);
  3947. return ret;
  3948. }
  3949. ret |= fault_dirty_shared_page(vmf);
  3950. return ret;
  3951. }
  3952. /*
  3953. * We enter with non-exclusive mmap_lock (to exclude vma changes,
  3954. * but allow concurrent faults).
  3955. * The mmap_lock may have been released depending on flags and our
  3956. * return value. See filemap_fault() and __lock_page_or_retry().
  3957. * If mmap_lock is released, vma may become invalid (for example
  3958. * by other thread calling munmap()).
  3959. */
  3960. static vm_fault_t do_fault(struct vm_fault *vmf)
  3961. {
  3962. struct vm_area_struct *vma = vmf->vma;
  3963. struct mm_struct *vm_mm = vma->vm_mm;
  3964. vm_fault_t ret;
  3965. /*
  3966. * The VMA was not fully populated on mmap() or missing VM_DONTEXPAND
  3967. */
  3968. if (!vma->vm_ops->fault) {
  3969. /*
  3970. * If we find a migration pmd entry or a none pmd entry, which
  3971. * should never happen, return SIGBUS
  3972. */
  3973. if (unlikely(!pmd_present(*vmf->pmd)))
  3974. ret = VM_FAULT_SIGBUS;
  3975. else {
  3976. vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm,
  3977. vmf->pmd,
  3978. vmf->address,
  3979. &vmf->ptl);
  3980. /*
  3981. * Make sure this is not a temporary clearing of pte
  3982. * by holding ptl and checking again. A R/M/W update
  3983. * of pte involves: take ptl, clearing the pte so that
  3984. * we don't have concurrent modification by hardware
  3985. * followed by an update.
  3986. */
  3987. if (unlikely(pte_none(*vmf->pte)))
  3988. ret = VM_FAULT_SIGBUS;
  3989. else
  3990. ret = VM_FAULT_NOPAGE;
  3991. pte_unmap_unlock(vmf->pte, vmf->ptl);
  3992. }
  3993. } else if (!(vmf->flags & FAULT_FLAG_WRITE))
  3994. ret = do_read_fault(vmf);
  3995. else if (!(vmf->vma_flags & VM_SHARED))
  3996. ret = do_cow_fault(vmf);
  3997. else
  3998. ret = do_shared_fault(vmf);
  3999. /* preallocated pagetable is unused: free it */
  4000. if (vmf->prealloc_pte) {
  4001. pte_free(vm_mm, vmf->prealloc_pte);
  4002. vmf->prealloc_pte = NULL;
  4003. }
  4004. return ret;
  4005. }
  4006. static int numa_migrate_prep(struct page *page, struct vm_area_struct *vma,
  4007. unsigned long addr, int page_nid,
  4008. int *flags)
  4009. {
  4010. get_page(page);
  4011. count_vm_numa_event(NUMA_HINT_FAULTS);
  4012. if (page_nid == numa_node_id()) {
  4013. count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
  4014. *flags |= TNF_FAULT_LOCAL;
  4015. }
  4016. return mpol_misplaced(page, vma, addr);
  4017. }
  4018. static vm_fault_t do_numa_page(struct vm_fault *vmf)
  4019. {
  4020. struct vm_area_struct *vma = vmf->vma;
  4021. struct page *page = NULL;
  4022. int page_nid = NUMA_NO_NODE;
  4023. int last_cpupid;
  4024. int target_nid;
  4025. bool migrated = false;
  4026. pte_t pte, old_pte;
  4027. bool was_writable = pte_savedwrite(vmf->orig_pte);
  4028. int flags = 0;
  4029. /*
  4030. * The "pte" at this point cannot be used safely without
  4031. * validation through pte_unmap_same(). It's of NUMA type but
  4032. * the pfn may be screwed if the read is non atomic.
  4033. */
  4034. if (!pte_spinlock(vmf))
  4035. return VM_FAULT_RETRY;
  4036. if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte))) {
  4037. pte_unmap_unlock(vmf->pte, vmf->ptl);
  4038. goto out;
  4039. }
  4040. /*
  4041. * Make it present again, Depending on how arch implementes non
  4042. * accessible ptes, some can allow access by kernel mode.
  4043. */
  4044. old_pte = ptep_modify_prot_start(vma, vmf->address, vmf->pte);
  4045. pte = pte_modify(old_pte, vmf->vma_page_prot);
  4046. pte = pte_mkyoung(pte);
  4047. if (was_writable)
  4048. pte = pte_mkwrite(pte);
  4049. ptep_modify_prot_commit(vma, vmf->address, vmf->pte, old_pte, pte);
  4050. update_mmu_cache(vma, vmf->address, vmf->pte);
  4051. page = _vm_normal_page(vma, vmf->address, pte, vmf->vma_flags);
  4052. if (!page) {
  4053. pte_unmap_unlock(vmf->pte, vmf->ptl);
  4054. return 0;
  4055. }
  4056. /* TODO: handle PTE-mapped THP */
  4057. if (PageCompound(page)) {
  4058. pte_unmap_unlock(vmf->pte, vmf->ptl);
  4059. return 0;
  4060. }
  4061. /*
  4062. * Avoid grouping on RO pages in general. RO pages shouldn't hurt as
  4063. * much anyway since they can be in shared cache state. This misses
  4064. * the case where a mapping is writable but the process never writes
  4065. * to it but pte_write gets cleared during protection updates and
  4066. * pte_dirty has unpredictable behaviour between PTE scan updates,
  4067. * background writeback, dirty balancing and application behaviour.
  4068. */
  4069. if (!pte_write(pte))
  4070. flags |= TNF_NO_GROUP;
  4071. /*
  4072. * Flag if the page is shared between multiple address spaces. This
  4073. * is later used when determining whether to group tasks together
  4074. */
  4075. if (page_mapcount(page) > 1 && (vmf->vma_flags & VM_SHARED))
  4076. flags |= TNF_SHARED;
  4077. last_cpupid = page_cpupid_last(page);
  4078. page_nid = page_to_nid(page);
  4079. target_nid = numa_migrate_prep(page, vma, vmf->address, page_nid,
  4080. &flags);
  4081. pte_unmap_unlock(vmf->pte, vmf->ptl);
  4082. if (target_nid == NUMA_NO_NODE) {
  4083. put_page(page);
  4084. goto out;
  4085. }
  4086. /* Migrate to the requested node */
  4087. migrated = migrate_misplaced_page(page, vmf, target_nid);
  4088. if (migrated) {
  4089. page_nid = target_nid;
  4090. flags |= TNF_MIGRATED;
  4091. } else
  4092. flags |= TNF_MIGRATE_FAIL;
  4093. out:
  4094. if (page_nid != NUMA_NO_NODE)
  4095. task_numa_fault(last_cpupid, page_nid, 1, flags);
  4096. return 0;
  4097. }
  4098. static inline vm_fault_t create_huge_pmd(struct vm_fault *vmf)
  4099. {
  4100. if (vma_is_anonymous(vmf->vma))
  4101. return do_huge_pmd_anonymous_page(vmf);
  4102. if (vmf->vma->vm_ops->huge_fault)
  4103. return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PMD);
  4104. return VM_FAULT_FALLBACK;
  4105. }
  4106. /* `inline' is required to avoid gcc 4.1.2 build error */
  4107. static inline vm_fault_t wp_huge_pmd(struct vm_fault *vmf, pmd_t orig_pmd)
  4108. {
  4109. if (vma_is_anonymous(vmf->vma)) {
  4110. if (userfaultfd_huge_pmd_wp(vmf->vma, orig_pmd))
  4111. return handle_userfault(vmf, VM_UFFD_WP);
  4112. return do_huge_pmd_wp_page(vmf, orig_pmd);
  4113. }
  4114. if (vmf->vma->vm_ops->huge_fault) {
  4115. vm_fault_t ret = vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PMD);
  4116. if (!(ret & VM_FAULT_FALLBACK))
  4117. return ret;
  4118. }
  4119. /* COW or write-notify handled on pte level: split pmd. */
  4120. __split_huge_pmd(vmf->vma, vmf->pmd, vmf->address, false, NULL);
  4121. return VM_FAULT_FALLBACK;
  4122. }
  4123. static vm_fault_t create_huge_pud(struct vm_fault *vmf)
  4124. {
  4125. #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
  4126. defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
  4127. /* No support for anonymous transparent PUD pages yet */
  4128. if (vma_is_anonymous(vmf->vma))
  4129. goto split;
  4130. if (vmf->vma->vm_ops->huge_fault) {
  4131. vm_fault_t ret = vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PUD);
  4132. if (!(ret & VM_FAULT_FALLBACK))
  4133. return ret;
  4134. }
  4135. split:
  4136. /* COW or write-notify not handled on PUD level: split pud.*/
  4137. __split_huge_pud(vmf->vma, vmf->pud, vmf->address);
  4138. #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
  4139. return VM_FAULT_FALLBACK;
  4140. }
  4141. static vm_fault_t wp_huge_pud(struct vm_fault *vmf, pud_t orig_pud)
  4142. {
  4143. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  4144. /* No support for anonymous transparent PUD pages yet */
  4145. if (vma_is_anonymous(vmf->vma))
  4146. return VM_FAULT_FALLBACK;
  4147. if (vmf->vma->vm_ops->huge_fault)
  4148. return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PUD);
  4149. #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
  4150. return VM_FAULT_FALLBACK;
  4151. }
  4152. /*
  4153. * These routines also need to handle stuff like marking pages dirty
  4154. * and/or accessed for architectures that don't do it in hardware (most
  4155. * RISC architectures). The early dirtying is also good on the i386.
  4156. *
  4157. * There is also a hook called "update_mmu_cache()" that architectures
  4158. * with external mmu caches can use to update those (ie the Sparc or
  4159. * PowerPC hashed page tables that act as extended TLBs).
  4160. *
  4161. * We enter with non-exclusive mmap_lock (to exclude vma changes, but allow
  4162. * concurrent faults).
  4163. *
  4164. * The mmap_lock may have been released depending on flags and our return value.
  4165. * See filemap_fault() and __lock_page_or_retry().
  4166. */
  4167. static vm_fault_t handle_pte_fault(struct vm_fault *vmf)
  4168. {
  4169. pte_t entry;
  4170. vm_fault_t ret = 0;
  4171. if (unlikely(pmd_none(*vmf->pmd))) {
  4172. /*
  4173. * In the case of the speculative page fault handler we abort
  4174. * the speculative path immediately as the pmd is probably
  4175. * in the way to be converted in a huge one. We will try
  4176. * again holding the mmap_sem (which implies that the collapse
  4177. * operation is done).
  4178. */
  4179. if (vmf->flags & FAULT_FLAG_SPECULATIVE)
  4180. return VM_FAULT_RETRY;
  4181. /*
  4182. * Leave __pte_alloc() until later: because vm_ops->fault may
  4183. * want to allocate huge page, and if we expose page table
  4184. * for an instant, it will be difficult to retract from
  4185. * concurrent faults and from rmap lookups.
  4186. */
  4187. vmf->pte = NULL;
  4188. } else if (!(vmf->flags & FAULT_FLAG_SPECULATIVE)) {
  4189. /*
  4190. * If a huge pmd materialized under us just retry later. Use
  4191. * pmd_trans_unstable() via pmd_devmap_trans_unstable() instead
  4192. * of pmd_trans_huge() to ensure the pmd didn't become
  4193. * pmd_trans_huge under us and then back to pmd_none, as a
  4194. * result of MADV_DONTNEED running immediately after a huge pmd
  4195. * fault in a different thread of this mm, in turn leading to a
  4196. * misleading pmd_trans_huge() retval. All we have to ensure is
  4197. * that it is a regular pmd that we can walk with
  4198. * pte_offset_map() and we can do that through an atomic read
  4199. * in C, which is what pmd_trans_unstable() provides.
  4200. */
  4201. if (pmd_devmap_trans_unstable(vmf->pmd))
  4202. return 0;
  4203. /*
  4204. * A regular pmd is established and it can't morph into a huge
  4205. * pmd from under us anymore at this point because we hold the
  4206. * mmap_lock read mode and khugepaged takes it in write mode.
  4207. * So now it's safe to run pte_offset_map().
  4208. * This is not applicable to the speculative page fault handler
  4209. * but in that case, the pte is fetched earlier in
  4210. * handle_speculative_fault().
  4211. */
  4212. vmf->pte = pte_offset_map(vmf->pmd, vmf->address);
  4213. vmf->orig_pte = *vmf->pte;
  4214. /*
  4215. * some architectures can have larger ptes than wordsize,
  4216. * e.g.ppc44x-defconfig has CONFIG_PTE_64BIT=y and
  4217. * CONFIG_32BIT=y, so READ_ONCE cannot guarantee atomic
  4218. * accesses. The code below just needs a consistent view
  4219. * for the ifs and we later double check anyway with the
  4220. * ptl lock held. So here a barrier will do.
  4221. */
  4222. barrier();
  4223. if (pte_none(vmf->orig_pte)) {
  4224. pte_unmap(vmf->pte);
  4225. vmf->pte = NULL;
  4226. }
  4227. }
  4228. if (!vmf->pte) {
  4229. if (vma_is_anonymous(vmf->vma))
  4230. return do_anonymous_page(vmf);
  4231. else if ((vmf->flags & FAULT_FLAG_SPECULATIVE) &&
  4232. !vmf_allows_speculation(vmf))
  4233. return VM_FAULT_RETRY;
  4234. else
  4235. return do_fault(vmf);
  4236. }
  4237. if (!pte_present(vmf->orig_pte))
  4238. return do_swap_page(vmf);
  4239. if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma))
  4240. return do_numa_page(vmf);
  4241. if (!pte_spinlock(vmf))
  4242. return VM_FAULT_RETRY;
  4243. entry = vmf->orig_pte;
  4244. if (unlikely(!pte_same(*vmf->pte, entry))) {
  4245. update_mmu_tlb(vmf->vma, vmf->address, vmf->pte);
  4246. goto unlock;
  4247. }
  4248. if (vmf->flags & FAULT_FLAG_WRITE) {
  4249. if (!pte_write(entry)) {
  4250. if (!(vmf->flags & FAULT_FLAG_SPECULATIVE))
  4251. return do_wp_page(vmf);
  4252. if (!mmu_notifier_trylock(vmf->vma->vm_mm)) {
  4253. ret = VM_FAULT_RETRY;
  4254. goto unlock;
  4255. }
  4256. ret = do_wp_page(vmf);
  4257. mmu_notifier_unlock(vmf->vma->vm_mm);
  4258. return ret;
  4259. }
  4260. entry = pte_mkdirty(entry);
  4261. }
  4262. entry = pte_mkyoung(entry);
  4263. if (ptep_set_access_flags(vmf->vma, vmf->address, vmf->pte, entry,
  4264. vmf->flags & FAULT_FLAG_WRITE)) {
  4265. update_mmu_cache(vmf->vma, vmf->address, vmf->pte);
  4266. } else {
  4267. /* Skip spurious TLB flush for retried page fault */
  4268. if (vmf->flags & FAULT_FLAG_TRIED)
  4269. goto unlock;
  4270. if (vmf->flags & FAULT_FLAG_SPECULATIVE)
  4271. ret = VM_FAULT_RETRY;
  4272. /*
  4273. * This is needed only for protection faults but the arch code
  4274. * is not yet telling us if this is a protection fault or not.
  4275. * This still avoids useless tlb flushes for .text page faults
  4276. * with threads.
  4277. */
  4278. if (vmf->flags & FAULT_FLAG_WRITE)
  4279. flush_tlb_fix_spurious_fault(vmf->vma, vmf->address);
  4280. }
  4281. unlock:
  4282. pte_unmap_unlock(vmf->pte, vmf->ptl);
  4283. return ret;
  4284. }
  4285. /*
  4286. * By the time we get here, we already hold the mm semaphore
  4287. *
  4288. * The mmap_lock may have been released depending on flags and our
  4289. * return value. See filemap_fault() and __lock_page_or_retry().
  4290. */
  4291. static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,
  4292. unsigned long address, unsigned int flags)
  4293. {
  4294. struct vm_fault vmf = {
  4295. .vma = vma,
  4296. .address = address & PAGE_MASK,
  4297. .flags = flags,
  4298. .pgoff = linear_page_index(vma, address),
  4299. .gfp_mask = __get_fault_gfp_mask(vma),
  4300. .vma_flags = vma->vm_flags,
  4301. .vma_page_prot = vma->vm_page_prot,
  4302. };
  4303. unsigned int dirty = flags & FAULT_FLAG_WRITE;
  4304. struct mm_struct *mm = vma->vm_mm;
  4305. pgd_t *pgd;
  4306. p4d_t *p4d;
  4307. vm_fault_t ret;
  4308. pgd = pgd_offset(mm, address);
  4309. p4d = p4d_alloc(mm, pgd, address);
  4310. if (!p4d)
  4311. return VM_FAULT_OOM;
  4312. vmf.pud = pud_alloc(mm, p4d, address);
  4313. if (!vmf.pud)
  4314. return VM_FAULT_OOM;
  4315. retry_pud:
  4316. if (pud_none(*vmf.pud) && __transparent_hugepage_enabled(vma)) {
  4317. ret = create_huge_pud(&vmf);
  4318. if (!(ret & VM_FAULT_FALLBACK))
  4319. return ret;
  4320. } else {
  4321. pud_t orig_pud = *vmf.pud;
  4322. barrier();
  4323. if (pud_trans_huge(orig_pud) || pud_devmap(orig_pud)) {
  4324. /* NUMA case for anonymous PUDs would go here */
  4325. if (dirty && !pud_write(orig_pud)) {
  4326. ret = wp_huge_pud(&vmf, orig_pud);
  4327. if (!(ret & VM_FAULT_FALLBACK))
  4328. return ret;
  4329. } else {
  4330. huge_pud_set_accessed(&vmf, orig_pud);
  4331. return 0;
  4332. }
  4333. }
  4334. }
  4335. vmf.pmd = pmd_alloc(mm, vmf.pud, address);
  4336. if (!vmf.pmd)
  4337. return VM_FAULT_OOM;
  4338. /* Huge pud page fault raced with pmd_alloc? */
  4339. if (pud_trans_unstable(vmf.pud))
  4340. goto retry_pud;
  4341. #ifdef CONFIG_SPECULATIVE_PAGE_FAULT
  4342. vmf.sequence = raw_read_seqcount(&vma->vm_sequence);
  4343. #endif
  4344. if (pmd_none(*vmf.pmd) && __transparent_hugepage_enabled(vma)) {
  4345. ret = create_huge_pmd(&vmf);
  4346. if (!(ret & VM_FAULT_FALLBACK))
  4347. return ret;
  4348. } else {
  4349. pmd_t orig_pmd = *vmf.pmd;
  4350. barrier();
  4351. if (unlikely(is_swap_pmd(orig_pmd))) {
  4352. VM_BUG_ON(thp_migration_supported() &&
  4353. !is_pmd_migration_entry(orig_pmd));
  4354. if (is_pmd_migration_entry(orig_pmd))
  4355. pmd_migration_entry_wait(mm, vmf.pmd);
  4356. return 0;
  4357. }
  4358. if (pmd_trans_huge(orig_pmd) || pmd_devmap(orig_pmd)) {
  4359. if (pmd_protnone(orig_pmd) && vma_is_accessible(vma))
  4360. return do_huge_pmd_numa_page(&vmf, orig_pmd);
  4361. if (dirty && !pmd_write(orig_pmd)) {
  4362. ret = wp_huge_pmd(&vmf, orig_pmd);
  4363. if (!(ret & VM_FAULT_FALLBACK))
  4364. return ret;
  4365. } else {
  4366. huge_pmd_set_accessed(&vmf, orig_pmd);
  4367. return 0;
  4368. }
  4369. }
  4370. }
  4371. return handle_pte_fault(&vmf);
  4372. }
  4373. /**
  4374. * mm_account_fault - Do page fault accountings
  4375. *
  4376. * @regs: the pt_regs struct pointer. When set to NULL, will skip accounting
  4377. * of perf event counters, but we'll still do the per-task accounting to
  4378. * the task who triggered this page fault.
  4379. * @address: the faulted address.
  4380. * @flags: the fault flags.
  4381. * @ret: the fault retcode.
  4382. *
  4383. * This will take care of most of the page fault accountings. Meanwhile, it
  4384. * will also include the PERF_COUNT_SW_PAGE_FAULTS_[MAJ|MIN] perf counter
  4385. * updates. However note that the handling of PERF_COUNT_SW_PAGE_FAULTS should
  4386. * still be in per-arch page fault handlers at the entry of page fault.
  4387. */
  4388. static inline void mm_account_fault(struct pt_regs *regs,
  4389. unsigned long address, unsigned int flags,
  4390. vm_fault_t ret)
  4391. {
  4392. bool major;
  4393. /*
  4394. * We don't do accounting for some specific faults:
  4395. *
  4396. * - Unsuccessful faults (e.g. when the address wasn't valid). That
  4397. * includes arch_vma_access_permitted() failing before reaching here.
  4398. * So this is not a "this many hardware page faults" counter. We
  4399. * should use the hw profiling for that.
  4400. *
  4401. * - Incomplete faults (VM_FAULT_RETRY). They will only be counted
  4402. * once they're completed.
  4403. */
  4404. if (ret & (VM_FAULT_ERROR | VM_FAULT_RETRY))
  4405. return;
  4406. /*
  4407. * We define the fault as a major fault when the final successful fault
  4408. * is VM_FAULT_MAJOR, or if it retried (which implies that we couldn't
  4409. * handle it immediately previously).
  4410. */
  4411. major = (ret & VM_FAULT_MAJOR) || (flags & FAULT_FLAG_TRIED);
  4412. if (major)
  4413. current->maj_flt++;
  4414. else
  4415. current->min_flt++;
  4416. /*
  4417. * If the fault is done for GUP, regs will be NULL. We only do the
  4418. * accounting for the per thread fault counters who triggered the
  4419. * fault, and we skip the perf event updates.
  4420. */
  4421. if (!regs)
  4422. return;
  4423. if (major)
  4424. perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address);
  4425. else
  4426. perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address);
  4427. }
  4428. #ifdef CONFIG_SPECULATIVE_PAGE_FAULT
  4429. #ifndef CONFIG_ARCH_HAS_PTE_SPECIAL
  4430. /* This is required by vm_normal_page() */
  4431. #error "Speculative page fault handler requires CONFIG_ARCH_HAS_PTE_SPECIAL"
  4432. #endif
  4433. /*
  4434. * vm_normal_page() adds some processing which should be done while
  4435. * hodling the mmap_sem.
  4436. */
  4437. /*
  4438. * Tries to handle the page fault in a speculative way, without grabbing the
  4439. * mmap_sem.
  4440. * When VM_FAULT_RETRY is returned, the vma pointer is valid and this vma must
  4441. * be checked later when the mmap_sem has been grabbed by calling
  4442. * can_reuse_spf_vma().
  4443. * This is needed as the returned vma is kept in memory until the call to
  4444. * can_reuse_spf_vma() is made.
  4445. */
  4446. static vm_fault_t ___handle_speculative_fault(struct mm_struct *mm,
  4447. unsigned long address, unsigned int flags,
  4448. struct vm_area_struct *vma)
  4449. {
  4450. struct vm_fault vmf = {
  4451. .address = address,
  4452. .pgoff = linear_page_index(vma, address),
  4453. .vma = vma,
  4454. .gfp_mask = __get_fault_gfp_mask(vma),
  4455. .flags = flags,
  4456. };
  4457. #ifdef CONFIG_NUMA
  4458. struct mempolicy *pol;
  4459. #endif
  4460. pgd_t *pgd, pgdval;
  4461. p4d_t *p4d, p4dval;
  4462. pud_t pudval;
  4463. int seq;
  4464. vm_fault_t ret;
  4465. /* Clear flags that may lead to release the mmap_sem to retry */
  4466. flags &= ~(FAULT_FLAG_ALLOW_RETRY|FAULT_FLAG_KILLABLE);
  4467. flags |= FAULT_FLAG_SPECULATIVE;
  4468. /* rmb <-> seqlock,vma_rb_erase() */
  4469. seq = raw_read_seqcount(&vmf.vma->vm_sequence);
  4470. if (seq & 1) {
  4471. trace_spf_vma_changed(_RET_IP_, vmf.vma, address);
  4472. return VM_FAULT_RETRY;
  4473. }
  4474. if (!vmf_allows_speculation(&vmf))
  4475. return VM_FAULT_RETRY;
  4476. vmf.vma_flags = READ_ONCE(vmf.vma->vm_flags);
  4477. vmf.vma_page_prot = READ_ONCE(vmf.vma->vm_page_prot);
  4478. /* Can't call userland page fault handler in the speculative path */
  4479. if (unlikely(vmf.vma_flags & VM_UFFD_MISSING)) {
  4480. trace_spf_vma_notsup(_RET_IP_, vmf.vma, address);
  4481. return VM_FAULT_RETRY;
  4482. }
  4483. if (vmf.vma_flags & VM_GROWSDOWN || vmf.vma_flags & VM_GROWSUP) {
  4484. /*
  4485. * This could be detected by the check address against VMA's
  4486. * boundaries but we want to trace it as not supported instead
  4487. * of changed.
  4488. */
  4489. trace_spf_vma_notsup(_RET_IP_, vmf.vma, address);
  4490. return VM_FAULT_RETRY;
  4491. }
  4492. if (address < READ_ONCE(vmf.vma->vm_start)
  4493. || READ_ONCE(vmf.vma->vm_end) <= address) {
  4494. trace_spf_vma_changed(_RET_IP_, vmf.vma, address);
  4495. return VM_FAULT_RETRY;
  4496. }
  4497. if (!arch_vma_access_permitted(vmf.vma, flags & FAULT_FLAG_WRITE,
  4498. flags & FAULT_FLAG_INSTRUCTION,
  4499. flags & FAULT_FLAG_REMOTE))
  4500. goto out_segv;
  4501. /* This is one is required to check that the VMA has write access set */
  4502. if (flags & FAULT_FLAG_WRITE) {
  4503. if (unlikely(!(vmf.vma_flags & VM_WRITE)))
  4504. goto out_segv;
  4505. } else if (unlikely(!(vmf.vma_flags & (VM_READ|VM_EXEC|VM_WRITE))))
  4506. goto out_segv;
  4507. #ifdef CONFIG_NUMA
  4508. /*
  4509. * MPOL_INTERLEAVE implies additional checks in
  4510. * mpol_misplaced() which are not compatible with the
  4511. *speculative page fault processing.
  4512. */
  4513. pol = __get_vma_policy(vmf.vma, address);
  4514. if (!pol)
  4515. pol = get_task_policy(current);
  4516. if (!pol)
  4517. if (pol && pol->mode == MPOL_INTERLEAVE) {
  4518. trace_spf_vma_notsup(_RET_IP_, vmf.vma, address);
  4519. return VM_FAULT_RETRY;
  4520. }
  4521. #endif
  4522. /*
  4523. * Do a speculative lookup of the PTE entry.
  4524. */
  4525. local_irq_disable();
  4526. pgd = pgd_offset(mm, address);
  4527. pgdval = READ_ONCE(*pgd);
  4528. if (pgd_none(pgdval) || unlikely(pgd_bad(pgdval)))
  4529. goto out_walk;
  4530. p4d = p4d_offset(pgd, address);
  4531. if (pgd_val(READ_ONCE(*pgd)) != pgd_val(pgdval))
  4532. goto out_walk;
  4533. p4dval = READ_ONCE(*p4d);
  4534. if (p4d_none(p4dval) || unlikely(p4d_bad(p4dval)))
  4535. goto out_walk;
  4536. vmf.pud = pud_offset(p4d, address);
  4537. if (p4d_val(READ_ONCE(*p4d)) != p4d_val(p4dval))
  4538. goto out_walk;
  4539. pudval = READ_ONCE(*vmf.pud);
  4540. if (pud_none(pudval) || unlikely(pud_bad(pudval)))
  4541. goto out_walk;
  4542. /* Huge pages at PUD level are not supported. */
  4543. if (unlikely(pud_trans_huge(pudval)))
  4544. goto out_walk;
  4545. vmf.pmd = pmd_offset(vmf.pud, address);
  4546. if (pud_val(READ_ONCE(*vmf.pud)) != pud_val(pudval))
  4547. goto out_walk;
  4548. vmf.orig_pmd = READ_ONCE(*vmf.pmd);
  4549. /*
  4550. * pmd_none could mean that a hugepage collapse is in progress
  4551. * in our back as collapse_huge_page() mark it before
  4552. * invalidating the pte (which is done once the IPI is catched
  4553. * by all CPU and we have interrupt disabled).
  4554. * For this reason we cannot handle THP in a speculative way since we
  4555. * can't safely indentify an in progress collapse operation done in our
  4556. * back on that PMD.
  4557. * Regarding the order of the following checks, see comment in
  4558. * pmd_devmap_trans_unstable()
  4559. */
  4560. if (unlikely(pmd_devmap(vmf.orig_pmd) ||
  4561. pmd_none(vmf.orig_pmd) || pmd_trans_huge(vmf.orig_pmd) ||
  4562. is_swap_pmd(vmf.orig_pmd)))
  4563. goto out_walk;
  4564. /*
  4565. * The above does not allocate/instantiate page-tables because doing so
  4566. * would lead to the possibility of instantiating page-tables after
  4567. * free_pgtables() -- and consequently leaking them.
  4568. *
  4569. * The result is that we take at least one !speculative fault per PMD
  4570. * in order to instantiate it.
  4571. */
  4572. vmf.pte = pte_offset_map(vmf.pmd, address);
  4573. if (pmd_val(READ_ONCE(*vmf.pmd)) != pmd_val(vmf.orig_pmd)) {
  4574. pte_unmap(vmf.pte);
  4575. vmf.pte = NULL;
  4576. goto out_walk;
  4577. }
  4578. vmf.orig_pte = READ_ONCE(*vmf.pte);
  4579. barrier(); /* See comment in handle_pte_fault() */
  4580. if (pte_none(vmf.orig_pte)) {
  4581. pte_unmap(vmf.pte);
  4582. vmf.pte = NULL;
  4583. }
  4584. vmf.sequence = seq;
  4585. vmf.flags = flags;
  4586. local_irq_enable();
  4587. /*
  4588. * We need to re-validate the VMA after checking the bounds, otherwise
  4589. * we might have a false positive on the bounds.
  4590. */
  4591. if (read_seqcount_retry(&vmf.vma->vm_sequence, seq)) {
  4592. trace_spf_vma_changed(_RET_IP_, vmf.vma, address);
  4593. return VM_FAULT_RETRY;
  4594. }
  4595. mem_cgroup_enter_user_fault();
  4596. ret = handle_pte_fault(&vmf);
  4597. mem_cgroup_exit_user_fault();
  4598. if (ret != VM_FAULT_RETRY) {
  4599. if (vma_is_anonymous(vmf.vma))
  4600. count_vm_event(SPECULATIVE_PGFAULT_ANON);
  4601. else
  4602. count_vm_event(SPECULATIVE_PGFAULT_FILE);
  4603. }
  4604. /*
  4605. * The task may have entered a memcg OOM situation but
  4606. * if the allocation error was handled gracefully (no
  4607. * VM_FAULT_OOM), there is no need to kill anything.
  4608. * Just clean up the OOM state peacefully.
  4609. */
  4610. if (task_in_memcg_oom(current) && !(ret & VM_FAULT_OOM))
  4611. mem_cgroup_oom_synchronize(false);
  4612. return ret;
  4613. out_walk:
  4614. trace_spf_vma_notsup(_RET_IP_, vmf.vma, address);
  4615. local_irq_enable();
  4616. return VM_FAULT_RETRY;
  4617. out_segv:
  4618. trace_spf_vma_access(_RET_IP_, vmf.vma, address);
  4619. return VM_FAULT_SIGSEGV;
  4620. }
  4621. vm_fault_t __handle_speculative_fault(struct mm_struct *mm,
  4622. unsigned long address, unsigned int flags,
  4623. struct vm_area_struct **vma,
  4624. struct pt_regs *regs)
  4625. {
  4626. vm_fault_t ret;
  4627. check_sync_rss_stat(current);
  4628. *vma = get_vma(mm, address);
  4629. if (!*vma)
  4630. return VM_FAULT_RETRY;
  4631. ret = ___handle_speculative_fault(mm, address, flags, *vma);
  4632. /*
  4633. * If there is no need to retry, don't return the vma to the caller.
  4634. */
  4635. if (ret != VM_FAULT_RETRY) {
  4636. put_vma(*vma);
  4637. *vma = NULL;
  4638. mm_account_fault(regs, address, flags, ret);
  4639. }
  4640. return ret;
  4641. }
  4642. /*
  4643. * This is used to know if the vma fetch in the speculative page fault handler
  4644. * is still valid when trying the regular fault path while holding the
  4645. * mmap_sem.
  4646. * The call to put_vma(vma) must be made after checking the vma's fields, as
  4647. * the vma may be freed by put_vma(). In such a case it is expected that false
  4648. * is returned.
  4649. */
  4650. bool can_reuse_spf_vma(struct vm_area_struct *vma, unsigned long address)
  4651. {
  4652. bool ret;
  4653. ret = !RB_EMPTY_NODE(&vma->vm_rb) &&
  4654. vma->vm_start <= address && address < vma->vm_end;
  4655. put_vma(vma);
  4656. return ret;
  4657. }
  4658. #endif /* CONFIG_SPECULATIVE_PAGE_FAULT */
  4659. /*
  4660. * By the time we get here, we already hold the mm semaphore
  4661. *
  4662. * The mmap_lock may have been released depending on flags and our
  4663. * return value. See filemap_fault() and __lock_page_or_retry().
  4664. */
  4665. vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
  4666. unsigned int flags, struct pt_regs *regs)
  4667. {
  4668. vm_fault_t ret;
  4669. __set_current_state(TASK_RUNNING);
  4670. count_vm_event(PGFAULT);
  4671. count_memcg_event_mm(vma->vm_mm, PGFAULT);
  4672. /* do counter updates before entering really critical section. */
  4673. check_sync_rss_stat(current);
  4674. if (!arch_vma_access_permitted(vma, flags & FAULT_FLAG_WRITE,
  4675. flags & FAULT_FLAG_INSTRUCTION,
  4676. flags & FAULT_FLAG_REMOTE))
  4677. return VM_FAULT_SIGSEGV;
  4678. /*
  4679. * Enable the memcg OOM handling for faults triggered in user
  4680. * space. Kernel faults are handled more gracefully.
  4681. */
  4682. if (flags & FAULT_FLAG_USER)
  4683. mem_cgroup_enter_user_fault();
  4684. if (unlikely(is_vm_hugetlb_page(vma)))
  4685. ret = hugetlb_fault(vma->vm_mm, vma, address, flags);
  4686. else
  4687. ret = __handle_mm_fault(vma, address, flags);
  4688. if (flags & FAULT_FLAG_USER) {
  4689. mem_cgroup_exit_user_fault();
  4690. /*
  4691. * The task may have entered a memcg OOM situation but
  4692. * if the allocation error was handled gracefully (no
  4693. * VM_FAULT_OOM), there is no need to kill anything.
  4694. * Just clean up the OOM state peacefully.
  4695. */
  4696. if (task_in_memcg_oom(current) && !(ret & VM_FAULT_OOM))
  4697. mem_cgroup_oom_synchronize(false);
  4698. }
  4699. mm_account_fault(regs, address, flags, ret);
  4700. return ret;
  4701. }
  4702. EXPORT_SYMBOL_GPL(handle_mm_fault);
  4703. #ifndef __PAGETABLE_P4D_FOLDED
  4704. /*
  4705. * Allocate p4d page table.
  4706. * We've already handled the fast-path in-line.
  4707. */
  4708. int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
  4709. {
  4710. p4d_t *new = p4d_alloc_one(mm, address);
  4711. if (!new)
  4712. return -ENOMEM;
  4713. smp_wmb(); /* See comment in __pte_alloc */
  4714. spin_lock(&mm->page_table_lock);
  4715. if (pgd_present(*pgd)) /* Another has populated it */
  4716. p4d_free(mm, new);
  4717. else
  4718. pgd_populate(mm, pgd, new);
  4719. spin_unlock(&mm->page_table_lock);
  4720. return 0;
  4721. }
  4722. #endif /* __PAGETABLE_P4D_FOLDED */
  4723. #ifndef __PAGETABLE_PUD_FOLDED
  4724. /*
  4725. * Allocate page upper directory.
  4726. * We've already handled the fast-path in-line.
  4727. */
  4728. int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address)
  4729. {
  4730. pud_t *new = pud_alloc_one(mm, address);
  4731. if (!new)
  4732. return -ENOMEM;
  4733. smp_wmb(); /* See comment in __pte_alloc */
  4734. spin_lock(&mm->page_table_lock);
  4735. if (!p4d_present(*p4d)) {
  4736. mm_inc_nr_puds(mm);
  4737. p4d_populate(mm, p4d, new);
  4738. } else /* Another has populated it */
  4739. pud_free(mm, new);
  4740. spin_unlock(&mm->page_table_lock);
  4741. return 0;
  4742. }
  4743. #endif /* __PAGETABLE_PUD_FOLDED */
  4744. #ifndef __PAGETABLE_PMD_FOLDED
  4745. /*
  4746. * Allocate page middle directory.
  4747. * We've already handled the fast-path in-line.
  4748. */
  4749. int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
  4750. {
  4751. spinlock_t *ptl;
  4752. pmd_t *new = pmd_alloc_one(mm, address);
  4753. if (!new)
  4754. return -ENOMEM;
  4755. smp_wmb(); /* See comment in __pte_alloc */
  4756. ptl = pud_lock(mm, pud);
  4757. if (!pud_present(*pud)) {
  4758. mm_inc_nr_pmds(mm);
  4759. pud_populate(mm, pud, new);
  4760. } else /* Another has populated it */
  4761. pmd_free(mm, new);
  4762. spin_unlock(ptl);
  4763. return 0;
  4764. }
  4765. #endif /* __PAGETABLE_PMD_FOLDED */
  4766. int follow_invalidate_pte(struct mm_struct *mm, unsigned long address,
  4767. struct mmu_notifier_range *range, pte_t **ptepp,
  4768. pmd_t **pmdpp, spinlock_t **ptlp)
  4769. {
  4770. pgd_t *pgd;
  4771. p4d_t *p4d;
  4772. pud_t *pud;
  4773. pmd_t *pmd;
  4774. pte_t *ptep;
  4775. pgd = pgd_offset(mm, address);
  4776. if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
  4777. goto out;
  4778. p4d = p4d_offset(pgd, address);
  4779. if (p4d_none(*p4d) || unlikely(p4d_bad(*p4d)))
  4780. goto out;
  4781. pud = pud_offset(p4d, address);
  4782. if (pud_none(*pud) || unlikely(pud_bad(*pud)))
  4783. goto out;
  4784. pmd = pmd_offset(pud, address);
  4785. VM_BUG_ON(pmd_trans_huge(*pmd));
  4786. if (pmd_huge(*pmd)) {
  4787. if (!pmdpp)
  4788. goto out;
  4789. if (range) {
  4790. mmu_notifier_range_init(range, MMU_NOTIFY_CLEAR, 0,
  4791. NULL, mm, address & PMD_MASK,
  4792. (address & PMD_MASK) + PMD_SIZE);
  4793. mmu_notifier_invalidate_range_start(range);
  4794. }
  4795. *ptlp = pmd_lock(mm, pmd);
  4796. if (pmd_huge(*pmd)) {
  4797. *pmdpp = pmd;
  4798. return 0;
  4799. }
  4800. spin_unlock(*ptlp);
  4801. if (range)
  4802. mmu_notifier_invalidate_range_end(range);
  4803. }
  4804. if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd)))
  4805. goto out;
  4806. if (range) {
  4807. mmu_notifier_range_init(range, MMU_NOTIFY_CLEAR, 0, NULL, mm,
  4808. address & PAGE_MASK,
  4809. (address & PAGE_MASK) + PAGE_SIZE);
  4810. mmu_notifier_invalidate_range_start(range);
  4811. }
  4812. ptep = pte_offset_map_lock(mm, pmd, address, ptlp);
  4813. if (!pte_present(*ptep))
  4814. goto unlock;
  4815. *ptepp = ptep;
  4816. return 0;
  4817. unlock:
  4818. pte_unmap_unlock(ptep, *ptlp);
  4819. if (range)
  4820. mmu_notifier_invalidate_range_end(range);
  4821. out:
  4822. return -EINVAL;
  4823. }
  4824. /**
  4825. * follow_pte - look up PTE at a user virtual address
  4826. * @mm: the mm_struct of the target address space
  4827. * @address: user virtual address
  4828. * @ptepp: location to store found PTE
  4829. * @ptlp: location to store the lock for the PTE
  4830. *
  4831. * On a successful return, the pointer to the PTE is stored in @ptepp;
  4832. * the corresponding lock is taken and its location is stored in @ptlp.
  4833. * The contents of the PTE are only stable until @ptlp is released;
  4834. * any further use, if any, must be protected against invalidation
  4835. * with MMU notifiers.
  4836. *
  4837. * Only IO mappings and raw PFN mappings are allowed. The mmap semaphore
  4838. * should be taken for read.
  4839. *
  4840. * KVM uses this function. While it is arguably less bad than ``follow_pfn``,
  4841. * it is not a good general-purpose API.
  4842. *
  4843. * Return: zero on success, -ve otherwise.
  4844. */
  4845. int follow_pte(struct mm_struct *mm, unsigned long address,
  4846. pte_t **ptepp, spinlock_t **ptlp)
  4847. {
  4848. return follow_invalidate_pte(mm, address, NULL, ptepp, NULL, ptlp);
  4849. }
  4850. EXPORT_SYMBOL_GPL(follow_pte);
  4851. /**
  4852. * follow_pfn - look up PFN at a user virtual address
  4853. * @vma: memory mapping
  4854. * @address: user virtual address
  4855. * @pfn: location to store found PFN
  4856. *
  4857. * Only IO mappings and raw PFN mappings are allowed.
  4858. *
  4859. * This function does not allow the caller to read the permissions
  4860. * of the PTE. Do not use it.
  4861. *
  4862. * Return: zero and the pfn at @pfn on success, -ve otherwise.
  4863. */
  4864. int follow_pfn(struct vm_area_struct *vma, unsigned long address,
  4865. unsigned long *pfn)
  4866. {
  4867. int ret = -EINVAL;
  4868. spinlock_t *ptl;
  4869. pte_t *ptep;
  4870. if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
  4871. return ret;
  4872. ret = follow_pte(vma->vm_mm, address, &ptep, &ptl);
  4873. if (ret)
  4874. return ret;
  4875. *pfn = pte_pfn(*ptep);
  4876. pte_unmap_unlock(ptep, ptl);
  4877. return 0;
  4878. }
  4879. EXPORT_SYMBOL(follow_pfn);
  4880. #ifdef CONFIG_HAVE_IOREMAP_PROT
  4881. int follow_phys(struct vm_area_struct *vma,
  4882. unsigned long address, unsigned int flags,
  4883. unsigned long *prot, resource_size_t *phys)
  4884. {
  4885. int ret = -EINVAL;
  4886. pte_t *ptep, pte;
  4887. spinlock_t *ptl;
  4888. if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
  4889. goto out;
  4890. if (follow_pte(vma->vm_mm, address, &ptep, &ptl))
  4891. goto out;
  4892. pte = *ptep;
  4893. if ((flags & FOLL_WRITE) && !pte_write(pte))
  4894. goto unlock;
  4895. *prot = pgprot_val(pte_pgprot(pte));
  4896. *phys = (resource_size_t)pte_pfn(pte) << PAGE_SHIFT;
  4897. ret = 0;
  4898. unlock:
  4899. pte_unmap_unlock(ptep, ptl);
  4900. out:
  4901. return ret;
  4902. }
  4903. int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
  4904. void *buf, int len, int write)
  4905. {
  4906. resource_size_t phys_addr;
  4907. unsigned long prot = 0;
  4908. void __iomem *maddr;
  4909. int offset = addr & (PAGE_SIZE-1);
  4910. if (follow_phys(vma, addr, write, &prot, &phys_addr))
  4911. return -EINVAL;
  4912. maddr = ioremap_prot(phys_addr, PAGE_ALIGN(len + offset), prot);
  4913. if (!maddr)
  4914. return -ENOMEM;
  4915. if (write)
  4916. memcpy_toio(maddr + offset, buf, len);
  4917. else
  4918. memcpy_fromio(buf, maddr + offset, len);
  4919. iounmap(maddr);
  4920. return len;
  4921. }
  4922. EXPORT_SYMBOL_GPL(generic_access_phys);
  4923. #endif
  4924. /*
  4925. * Access another process' address space as given in mm. If non-NULL, use the
  4926. * given task for page fault accounting.
  4927. */
  4928. int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
  4929. unsigned long addr, void *buf, int len, unsigned int gup_flags)
  4930. {
  4931. struct vm_area_struct *vma;
  4932. void *old_buf = buf;
  4933. int write = gup_flags & FOLL_WRITE;
  4934. if (mmap_read_lock_killable(mm))
  4935. return 0;
  4936. /* ignore errors, just check how much was successfully transferred */
  4937. while (len) {
  4938. int bytes, ret, offset;
  4939. void *maddr;
  4940. struct page *page = NULL;
  4941. ret = get_user_pages_remote(mm, addr, 1,
  4942. gup_flags, &page, &vma, NULL);
  4943. if (ret <= 0) {
  4944. #ifndef CONFIG_HAVE_IOREMAP_PROT
  4945. break;
  4946. #else
  4947. /*
  4948. * Check if this is a VM_IO | VM_PFNMAP VMA, which
  4949. * we can access using slightly different code.
  4950. */
  4951. vma = find_vma(mm, addr);
  4952. if (!vma || vma->vm_start > addr)
  4953. break;
  4954. if (vma->vm_ops && vma->vm_ops->access)
  4955. ret = vma->vm_ops->access(vma, addr, buf,
  4956. len, write);
  4957. if (ret <= 0)
  4958. break;
  4959. bytes = ret;
  4960. #endif
  4961. } else {
  4962. bytes = len;
  4963. offset = addr & (PAGE_SIZE-1);
  4964. if (bytes > PAGE_SIZE-offset)
  4965. bytes = PAGE_SIZE-offset;
  4966. maddr = kmap(page);
  4967. if (write) {
  4968. copy_to_user_page(vma, page, addr,
  4969. maddr + offset, buf, bytes);
  4970. set_page_dirty_lock(page);
  4971. } else {
  4972. copy_from_user_page(vma, page, addr,
  4973. buf, maddr + offset, bytes);
  4974. }
  4975. kunmap(page);
  4976. put_user_page(page);
  4977. }
  4978. len -= bytes;
  4979. buf += bytes;
  4980. addr += bytes;
  4981. }
  4982. mmap_read_unlock(mm);
  4983. return buf - old_buf;
  4984. }
  4985. /**
  4986. * access_remote_vm - access another process' address space
  4987. * @mm: the mm_struct of the target address space
  4988. * @addr: start address to access
  4989. * @buf: source or destination buffer
  4990. * @len: number of bytes to transfer
  4991. * @gup_flags: flags modifying lookup behaviour
  4992. *
  4993. * The caller must hold a reference on @mm.
  4994. *
  4995. * Return: number of bytes copied from source to destination.
  4996. */
  4997. int access_remote_vm(struct mm_struct *mm, unsigned long addr,
  4998. void *buf, int len, unsigned int gup_flags)
  4999. {
  5000. return __access_remote_vm(NULL, mm, addr, buf, len, gup_flags);
  5001. }
  5002. /*
  5003. * Access another process' address space.
  5004. * Source/target buffer must be kernel space,
  5005. * Do not walk the page table directly, use get_user_pages
  5006. */
  5007. int access_process_vm(struct task_struct *tsk, unsigned long addr,
  5008. void *buf, int len, unsigned int gup_flags)
  5009. {
  5010. struct mm_struct *mm;
  5011. int ret;
  5012. mm = get_task_mm(tsk);
  5013. if (!mm)
  5014. return 0;
  5015. ret = __access_remote_vm(tsk, mm, addr, buf, len, gup_flags);
  5016. mmput(mm);
  5017. return ret;
  5018. }
  5019. EXPORT_SYMBOL_GPL(access_process_vm);
  5020. /*
  5021. * Print the name of a VMA.
  5022. */
  5023. void print_vma_addr(char *prefix, unsigned long ip)
  5024. {
  5025. struct mm_struct *mm = current->mm;
  5026. struct vm_area_struct *vma;
  5027. /*
  5028. * we might be running from an atomic context so we cannot sleep
  5029. */
  5030. if (!mmap_read_trylock(mm))
  5031. return;
  5032. vma = find_vma(mm, ip);
  5033. if (vma && vma->vm_file) {
  5034. struct file *f = vma->vm_file;
  5035. char *buf = (char *)__get_free_page(GFP_NOWAIT);
  5036. if (buf) {
  5037. char *p;
  5038. p = file_path(f, buf, PAGE_SIZE);
  5039. if (IS_ERR(p))
  5040. p = "?";
  5041. printk("%s%s[%lx+%lx]", prefix, kbasename(p),
  5042. vma->vm_start,
  5043. vma->vm_end - vma->vm_start);
  5044. free_page((unsigned long)buf);
  5045. }
  5046. }
  5047. mmap_read_unlock(mm);
  5048. }
  5049. #if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP)
  5050. void __might_fault(const char *file, int line)
  5051. {
  5052. /*
  5053. * Some code (nfs/sunrpc) uses socket ops on kernel memory while
  5054. * holding the mmap_lock, this is safe because kernel memory doesn't
  5055. * get paged out, therefore we'll never actually fault, and the
  5056. * below annotations will generate false positives.
  5057. */
  5058. if (uaccess_kernel())
  5059. return;
  5060. if (pagefault_disabled())
  5061. return;
  5062. __might_sleep(file, line, 0);
  5063. #if defined(CONFIG_DEBUG_ATOMIC_SLEEP)
  5064. if (current->mm)
  5065. might_lock_read(&current->mm->mmap_lock);
  5066. #endif
  5067. }
  5068. EXPORT_SYMBOL(__might_fault);
  5069. #endif
  5070. #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
  5071. /*
  5072. * Process all subpages of the specified huge page with the specified
  5073. * operation. The target subpage will be processed last to keep its
  5074. * cache lines hot.
  5075. */
  5076. static inline void process_huge_page(
  5077. unsigned long addr_hint, unsigned int pages_per_huge_page,
  5078. void (*process_subpage)(unsigned long addr, int idx, void *arg),
  5079. void *arg)
  5080. {
  5081. int i, n, base, l;
  5082. unsigned long addr = addr_hint &
  5083. ~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
  5084. /* Process target subpage last to keep its cache lines hot */
  5085. might_sleep();
  5086. n = (addr_hint - addr) / PAGE_SIZE;
  5087. if (2 * n <= pages_per_huge_page) {
  5088. /* If target subpage in first half of huge page */
  5089. base = 0;
  5090. l = n;
  5091. /* Process subpages at the end of huge page */
  5092. for (i = pages_per_huge_page - 1; i >= 2 * n; i--) {
  5093. cond_resched();
  5094. process_subpage(addr + i * PAGE_SIZE, i, arg);
  5095. }
  5096. } else {
  5097. /* If target subpage in second half of huge page */
  5098. base = pages_per_huge_page - 2 * (pages_per_huge_page - n);
  5099. l = pages_per_huge_page - n;
  5100. /* Process subpages at the begin of huge page */
  5101. for (i = 0; i < base; i++) {
  5102. cond_resched();
  5103. process_subpage(addr + i * PAGE_SIZE, i, arg);
  5104. }
  5105. }
  5106. /*
  5107. * Process remaining subpages in left-right-left-right pattern
  5108. * towards the target subpage
  5109. */
  5110. for (i = 0; i < l; i++) {
  5111. int left_idx = base + i;
  5112. int right_idx = base + 2 * l - 1 - i;
  5113. cond_resched();
  5114. process_subpage(addr + left_idx * PAGE_SIZE, left_idx, arg);
  5115. cond_resched();
  5116. process_subpage(addr + right_idx * PAGE_SIZE, right_idx, arg);
  5117. }
  5118. }
  5119. static void clear_gigantic_page(struct page *page,
  5120. unsigned long addr,
  5121. unsigned int pages_per_huge_page)
  5122. {
  5123. int i;
  5124. struct page *p = page;
  5125. might_sleep();
  5126. for (i = 0; i < pages_per_huge_page;
  5127. i++, p = mem_map_next(p, page, i)) {
  5128. cond_resched();
  5129. clear_user_highpage(p, addr + i * PAGE_SIZE);
  5130. }
  5131. }
  5132. static void clear_subpage(unsigned long addr, int idx, void *arg)
  5133. {
  5134. struct page *page = arg;
  5135. clear_user_highpage(page + idx, addr);
  5136. }
  5137. void clear_huge_page(struct page *page,
  5138. unsigned long addr_hint, unsigned int pages_per_huge_page)
  5139. {
  5140. unsigned long addr = addr_hint &
  5141. ~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
  5142. if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) {
  5143. clear_gigantic_page(page, addr, pages_per_huge_page);
  5144. return;
  5145. }
  5146. process_huge_page(addr_hint, pages_per_huge_page, clear_subpage, page);
  5147. }
  5148. static void copy_user_gigantic_page(struct page *dst, struct page *src,
  5149. unsigned long addr,
  5150. struct vm_area_struct *vma,
  5151. unsigned int pages_per_huge_page)
  5152. {
  5153. int i;
  5154. struct page *dst_base = dst;
  5155. struct page *src_base = src;
  5156. for (i = 0; i < pages_per_huge_page; ) {
  5157. cond_resched();
  5158. copy_user_highpage(dst, src, addr + i*PAGE_SIZE, vma);
  5159. i++;
  5160. dst = mem_map_next(dst, dst_base, i);
  5161. src = mem_map_next(src, src_base, i);
  5162. }
  5163. }
  5164. struct copy_subpage_arg {
  5165. struct page *dst;
  5166. struct page *src;
  5167. struct vm_area_struct *vma;
  5168. };
  5169. static void copy_subpage(unsigned long addr, int idx, void *arg)
  5170. {
  5171. struct copy_subpage_arg *copy_arg = arg;
  5172. copy_user_highpage(copy_arg->dst + idx, copy_arg->src + idx,
  5173. addr, copy_arg->vma);
  5174. }
  5175. void copy_user_huge_page(struct page *dst, struct page *src,
  5176. unsigned long addr_hint, struct vm_area_struct *vma,
  5177. unsigned int pages_per_huge_page)
  5178. {
  5179. unsigned long addr = addr_hint &
  5180. ~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
  5181. struct copy_subpage_arg arg = {
  5182. .dst = dst,
  5183. .src = src,
  5184. .vma = vma,
  5185. };
  5186. if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) {
  5187. copy_user_gigantic_page(dst, src, addr, vma,
  5188. pages_per_huge_page);
  5189. return;
  5190. }
  5191. process_huge_page(addr_hint, pages_per_huge_page, copy_subpage, &arg);
  5192. }
  5193. long copy_huge_page_from_user(struct page *dst_page,
  5194. const void __user *usr_src,
  5195. unsigned int pages_per_huge_page,
  5196. bool allow_pagefault)
  5197. {
  5198. void *src = (void *)usr_src;
  5199. void *page_kaddr;
  5200. unsigned long i, rc = 0;
  5201. unsigned long ret_val = pages_per_huge_page * PAGE_SIZE;
  5202. struct page *subpage = dst_page;
  5203. for (i = 0; i < pages_per_huge_page;
  5204. i++, subpage = mem_map_next(subpage, dst_page, i)) {
  5205. if (allow_pagefault)
  5206. page_kaddr = kmap(subpage);
  5207. else
  5208. page_kaddr = kmap_atomic(subpage);
  5209. rc = copy_from_user(page_kaddr,
  5210. (const void __user *)(src + i * PAGE_SIZE),
  5211. PAGE_SIZE);
  5212. if (allow_pagefault)
  5213. kunmap(subpage);
  5214. else
  5215. kunmap_atomic(page_kaddr);
  5216. ret_val -= (PAGE_SIZE - rc);
  5217. if (rc)
  5218. break;
  5219. cond_resched();
  5220. }
  5221. return ret_val;
  5222. }
  5223. #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
  5224. #if USE_SPLIT_PTE_PTLOCKS && ALLOC_SPLIT_PTLOCKS
  5225. static struct kmem_cache *page_ptl_cachep;
  5226. void __init ptlock_cache_init(void)
  5227. {
  5228. page_ptl_cachep = kmem_cache_create("page->ptl", sizeof(spinlock_t), 0,
  5229. SLAB_PANIC, NULL);
  5230. }
  5231. bool ptlock_alloc(struct page *page)
  5232. {
  5233. spinlock_t *ptl;
  5234. ptl = kmem_cache_alloc(page_ptl_cachep, GFP_KERNEL);
  5235. if (!ptl)
  5236. return false;
  5237. page->ptl = ptl;
  5238. return true;
  5239. }
  5240. void ptlock_free(struct page *page)
  5241. {
  5242. kmem_cache_free(page_ptl_cachep, page->ptl);
  5243. }
  5244. #endif