core.c 313 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106610761086109611061116112611361146115611661176118611961206121612261236124612561266127612861296130613161326133613461356136613761386139614061416142614361446145614661476148614961506151615261536154615561566157615861596160616161626163616461656166616761686169617061716172617361746175617661776178617961806181618261836184618561866187618861896190619161926193619461956196619761986199620062016202620362046205620662076208620962106211621262136214621562166217621862196220622162226223622462256226622762286229623062316232623362346235623662376238623962406241624262436244624562466247624862496250625162526253625462556256625762586259626062616262626362646265626662676268626962706271627262736274627562766277627862796280628162826283628462856286628762886289629062916292629362946295629662976298629963006301630263036304630563066307630863096310631163126313631463156316631763186319632063216322632363246325632663276328632963306331633263336334633563366337633863396340634163426343634463456346634763486349635063516352635363546355635663576358635963606361636263636364636563666367636863696370637163726373637463756376637763786379638063816382638363846385638663876388638963906391639263936394639563966397639863996400640164026403640464056406640764086409641064116412641364146415641664176418641964206421642264236424642564266427642864296430643164326433643464356436643764386439644064416442644364446445644664476448644964506451645264536454645564566457645864596460646164626463646464656466646764686469647064716472647364746475647664776478647964806481648264836484648564866487648864896490649164926493649464956496649764986499650065016502650365046505650665076508650965106511651265136514651565166517651865196520652165226523652465256526652765286529653065316532653365346535653665376538653965406541654265436544654565466547654865496550655165526553655465556556655765586559656065616562656365646565656665676568656965706571657265736574657565766577657865796580658165826583658465856586658765886589659065916592659365946595659665976598659966006601660266036604660566066607660866096610661166126613661466156616661766186619662066216622662366246625662666276628662966306631663266336634663566366637663866396640664166426643664466456646664766486649665066516652665366546655665666576658665966606661666266636664666566666667666866696670667166726673667466756676667766786679668066816682668366846685668666876688668966906691669266936694669566966697669866996700670167026703670467056706670767086709671067116712671367146715671667176718671967206721672267236724672567266727672867296730673167326733673467356736673767386739674067416742674367446745674667476748674967506751675267536754675567566757675867596760676167626763676467656766676767686769677067716772677367746775677667776778677967806781678267836784678567866787678867896790679167926793679467956796679767986799680068016802680368046805680668076808680968106811681268136814681568166817681868196820682168226823682468256826682768286829683068316832683368346835683668376838683968406841684268436844684568466847684868496850685168526853685468556856685768586859686068616862686368646865686668676868686968706871687268736874687568766877687868796880688168826883688468856886688768886889689068916892689368946895689668976898689969006901690269036904690569066907690869096910691169126913691469156916691769186919692069216922692369246925692669276928692969306931693269336934693569366937693869396940694169426943694469456946694769486949695069516952695369546955695669576958695969606961696269636964696569666967696869696970697169726973697469756976697769786979698069816982698369846985698669876988698969906991699269936994699569966997699869997000700170027003700470057006700770087009701070117012701370147015701670177018701970207021702270237024702570267027702870297030703170327033703470357036703770387039704070417042704370447045704670477048704970507051705270537054705570567057705870597060706170627063706470657066706770687069707070717072707370747075707670777078707970807081708270837084708570867087708870897090709170927093709470957096709770987099710071017102710371047105710671077108710971107111711271137114711571167117711871197120712171227123712471257126712771287129713071317132713371347135713671377138713971407141714271437144714571467147714871497150715171527153715471557156715771587159716071617162716371647165716671677168716971707171717271737174717571767177717871797180718171827183718471857186718771887189719071917192719371947195719671977198719972007201720272037204720572067207720872097210721172127213721472157216721772187219722072217222722372247225722672277228722972307231723272337234723572367237723872397240724172427243724472457246724772487249725072517252725372547255725672577258725972607261726272637264726572667267726872697270727172727273727472757276727772787279728072817282728372847285728672877288728972907291729272937294729572967297729872997300730173027303730473057306730773087309731073117312731373147315731673177318731973207321732273237324732573267327732873297330733173327333733473357336733773387339734073417342734373447345734673477348734973507351735273537354735573567357735873597360736173627363736473657366736773687369737073717372737373747375737673777378737973807381738273837384738573867387738873897390739173927393739473957396739773987399740074017402740374047405740674077408740974107411741274137414741574167417741874197420742174227423742474257426742774287429743074317432743374347435743674377438743974407441744274437444744574467447744874497450745174527453745474557456745774587459746074617462746374647465746674677468746974707471747274737474747574767477747874797480748174827483748474857486748774887489749074917492749374947495749674977498749975007501750275037504750575067507750875097510751175127513751475157516751775187519752075217522752375247525752675277528752975307531753275337534753575367537753875397540754175427543754475457546754775487549755075517552755375547555755675577558755975607561756275637564756575667567756875697570757175727573757475757576757775787579758075817582758375847585758675877588758975907591759275937594759575967597759875997600760176027603760476057606760776087609761076117612761376147615761676177618761976207621762276237624762576267627762876297630763176327633763476357636763776387639764076417642764376447645764676477648764976507651765276537654765576567657765876597660766176627663766476657666766776687669767076717672767376747675767676777678767976807681768276837684768576867687768876897690769176927693769476957696769776987699770077017702770377047705770677077708770977107711771277137714771577167717771877197720772177227723772477257726772777287729773077317732773377347735773677377738773977407741774277437744774577467747774877497750775177527753775477557756775777587759776077617762776377647765776677677768776977707771777277737774777577767777777877797780778177827783778477857786778777887789779077917792779377947795779677977798779978007801780278037804780578067807780878097810781178127813781478157816781778187819782078217822782378247825782678277828782978307831783278337834783578367837783878397840784178427843784478457846784778487849785078517852785378547855785678577858785978607861786278637864786578667867786878697870787178727873787478757876787778787879788078817882788378847885788678877888788978907891789278937894789578967897789878997900790179027903790479057906790779087909791079117912791379147915791679177918791979207921792279237924792579267927792879297930793179327933793479357936793779387939794079417942794379447945794679477948794979507951795279537954795579567957795879597960796179627963796479657966796779687969797079717972797379747975797679777978797979807981798279837984798579867987798879897990799179927993799479957996799779987999800080018002800380048005800680078008800980108011801280138014801580168017801880198020802180228023802480258026802780288029803080318032803380348035803680378038803980408041804280438044804580468047804880498050805180528053805480558056805780588059806080618062806380648065806680678068806980708071807280738074807580768077807880798080808180828083808480858086808780888089809080918092809380948095809680978098809981008101810281038104810581068107810881098110811181128113811481158116811781188119812081218122812381248125812681278128812981308131813281338134813581368137813881398140814181428143814481458146814781488149815081518152815381548155815681578158815981608161816281638164816581668167816881698170817181728173817481758176817781788179818081818182818381848185818681878188818981908191819281938194819581968197819881998200820182028203820482058206820782088209821082118212821382148215821682178218821982208221822282238224822582268227822882298230823182328233823482358236823782388239824082418242824382448245824682478248824982508251825282538254825582568257825882598260826182628263826482658266826782688269827082718272827382748275827682778278827982808281828282838284828582868287828882898290829182928293829482958296829782988299830083018302830383048305830683078308830983108311831283138314831583168317831883198320832183228323832483258326832783288329833083318332833383348335833683378338833983408341834283438344834583468347834883498350835183528353835483558356835783588359836083618362836383648365836683678368836983708371837283738374837583768377837883798380838183828383838483858386838783888389839083918392839383948395839683978398839984008401840284038404840584068407840884098410841184128413841484158416841784188419842084218422842384248425842684278428842984308431843284338434843584368437843884398440844184428443844484458446844784488449845084518452845384548455845684578458845984608461846284638464846584668467846884698470847184728473847484758476847784788479848084818482848384848485848684878488848984908491849284938494849584968497849884998500850185028503850485058506850785088509851085118512851385148515851685178518851985208521852285238524852585268527852885298530853185328533853485358536853785388539854085418542854385448545854685478548854985508551855285538554855585568557855885598560856185628563856485658566856785688569857085718572857385748575857685778578857985808581858285838584858585868587858885898590859185928593859485958596859785988599860086018602860386048605860686078608860986108611861286138614861586168617861886198620862186228623862486258626862786288629863086318632863386348635863686378638863986408641864286438644864586468647864886498650865186528653865486558656865786588659866086618662866386648665866686678668866986708671867286738674867586768677867886798680868186828683868486858686868786888689869086918692869386948695869686978698869987008701870287038704870587068707870887098710871187128713871487158716871787188719872087218722872387248725872687278728872987308731873287338734873587368737873887398740874187428743874487458746874787488749875087518752875387548755875687578758875987608761876287638764876587668767876887698770877187728773877487758776877787788779878087818782878387848785878687878788878987908791879287938794879587968797879887998800880188028803880488058806880788088809881088118812881388148815881688178818881988208821882288238824882588268827882888298830883188328833883488358836883788388839884088418842884388448845884688478848884988508851885288538854885588568857885888598860886188628863886488658866886788688869887088718872887388748875887688778878887988808881888288838884888588868887888888898890889188928893889488958896889788988899890089018902890389048905890689078908890989108911891289138914891589168917891889198920892189228923892489258926892789288929893089318932893389348935893689378938893989408941894289438944894589468947894889498950895189528953895489558956895789588959896089618962896389648965896689678968896989708971897289738974897589768977897889798980898189828983898489858986898789888989899089918992899389948995899689978998899990009001900290039004900590069007900890099010901190129013901490159016901790189019902090219022902390249025902690279028902990309031903290339034903590369037903890399040904190429043904490459046904790489049905090519052905390549055905690579058905990609061906290639064906590669067906890699070907190729073907490759076907790789079908090819082908390849085908690879088908990909091909290939094909590969097909890999100910191029103910491059106910791089109911091119112911391149115911691179118911991209121912291239124912591269127912891299130913191329133913491359136913791389139914091419142914391449145914691479148914991509151915291539154915591569157915891599160916191629163916491659166916791689169917091719172917391749175917691779178917991809181918291839184918591869187918891899190919191929193919491959196919791989199920092019202920392049205920692079208920992109211921292139214921592169217921892199220922192229223922492259226922792289229923092319232923392349235923692379238923992409241924292439244924592469247924892499250925192529253925492559256925792589259926092619262926392649265926692679268926992709271927292739274927592769277927892799280928192829283928492859286928792889289929092919292929392949295929692979298929993009301930293039304930593069307930893099310931193129313931493159316931793189319932093219322932393249325932693279328932993309331933293339334933593369337933893399340934193429343934493459346934793489349935093519352935393549355935693579358935993609361936293639364936593669367936893699370937193729373937493759376937793789379938093819382938393849385938693879388938993909391939293939394939593969397939893999400940194029403940494059406940794089409941094119412941394149415941694179418941994209421942294239424942594269427942894299430943194329433943494359436943794389439944094419442944394449445944694479448944994509451945294539454945594569457945894599460946194629463946494659466946794689469947094719472947394749475947694779478947994809481948294839484948594869487948894899490949194929493949494959496949794989499950095019502950395049505950695079508950995109511951295139514951595169517951895199520952195229523952495259526952795289529953095319532953395349535953695379538953995409541954295439544954595469547954895499550955195529553955495559556955795589559956095619562956395649565956695679568956995709571957295739574957595769577957895799580958195829583958495859586958795889589959095919592959395949595959695979598959996009601960296039604960596069607960896099610961196129613961496159616961796189619962096219622962396249625962696279628962996309631963296339634963596369637963896399640964196429643964496459646964796489649965096519652965396549655965696579658965996609661966296639664966596669667966896699670967196729673967496759676967796789679968096819682968396849685968696879688968996909691969296939694969596969697969896999700970197029703970497059706970797089709971097119712971397149715971697179718971997209721972297239724972597269727972897299730973197329733973497359736973797389739974097419742974397449745974697479748974997509751975297539754975597569757975897599760976197629763976497659766976797689769977097719772977397749775977697779778977997809781978297839784978597869787978897899790979197929793979497959796979797989799980098019802980398049805980698079808980998109811981298139814981598169817981898199820982198229823982498259826982798289829983098319832983398349835983698379838983998409841984298439844984598469847984898499850985198529853985498559856985798589859986098619862986398649865986698679868986998709871987298739874987598769877987898799880988198829883988498859886988798889889989098919892989398949895989698979898989999009901990299039904990599069907990899099910991199129913991499159916991799189919992099219922992399249925992699279928992999309931993299339934993599369937993899399940994199429943994499459946994799489949995099519952995399549955995699579958995999609961996299639964996599669967996899699970997199729973997499759976997799789979998099819982998399849985998699879988998999909991999299939994999599969997999899991000010001100021000310004100051000610007100081000910010100111001210013100141001510016100171001810019100201002110022100231002410025100261002710028100291003010031100321003310034100351003610037100381003910040100411004210043100441004510046100471004810049100501005110052100531005410055100561005710058100591006010061100621006310064100651006610067100681006910070100711007210073100741007510076100771007810079100801008110082100831008410085100861008710088100891009010091100921009310094100951009610097100981009910100101011010210103101041010510106101071010810109101101011110112101131011410115101161011710118101191012010121101221012310124101251012610127101281012910130101311013210133101341013510136101371013810139101401014110142101431014410145101461014710148101491015010151101521015310154101551015610157101581015910160101611016210163101641016510166101671016810169101701017110172101731017410175101761017710178101791018010181101821018310184101851018610187101881018910190101911019210193101941019510196101971019810199102001020110202102031020410205102061020710208102091021010211102121021310214102151021610217102181021910220102211022210223102241022510226102271022810229102301023110232102331023410235102361023710238102391024010241102421024310244102451024610247102481024910250102511025210253102541025510256102571025810259102601026110262102631026410265102661026710268102691027010271102721027310274102751027610277102781027910280102811028210283102841028510286102871028810289102901029110292102931029410295102961029710298102991030010301103021030310304103051030610307103081030910310103111031210313103141031510316103171031810319103201032110322103231032410325103261032710328103291033010331103321033310334103351033610337103381033910340103411034210343103441034510346103471034810349103501035110352103531035410355103561035710358103591036010361103621036310364103651036610367103681036910370103711037210373103741037510376103771037810379103801038110382103831038410385103861038710388103891039010391103921039310394103951039610397103981039910400104011040210403104041040510406104071040810409104101041110412104131041410415104161041710418104191042010421104221042310424104251042610427104281042910430104311043210433104341043510436104371043810439104401044110442104431044410445104461044710448104491045010451104521045310454104551045610457104581045910460104611046210463104641046510466104671046810469104701047110472104731047410475104761047710478104791048010481104821048310484104851048610487104881048910490104911049210493104941049510496104971049810499105001050110502105031050410505105061050710508105091051010511105121051310514105151051610517105181051910520105211052210523105241052510526105271052810529105301053110532105331053410535105361053710538105391054010541105421054310544105451054610547105481054910550105511055210553105541055510556105571055810559105601056110562105631056410565105661056710568105691057010571105721057310574105751057610577105781057910580105811058210583105841058510586105871058810589105901059110592105931059410595105961059710598105991060010601106021060310604106051060610607106081060910610106111061210613106141061510616106171061810619106201062110622106231062410625106261062710628106291063010631106321063310634106351063610637106381063910640106411064210643106441064510646106471064810649106501065110652106531065410655106561065710658106591066010661106621066310664106651066610667106681066910670106711067210673106741067510676106771067810679106801068110682106831068410685106861068710688106891069010691106921069310694106951069610697106981069910700107011070210703107041070510706107071070810709107101071110712107131071410715107161071710718107191072010721107221072310724107251072610727107281072910730107311073210733107341073510736107371073810739107401074110742107431074410745107461074710748107491075010751107521075310754107551075610757107581075910760107611076210763107641076510766107671076810769107701077110772107731077410775107761077710778107791078010781107821078310784107851078610787107881078910790107911079210793107941079510796107971079810799108001080110802108031080410805108061080710808108091081010811108121081310814108151081610817108181081910820108211082210823108241082510826108271082810829108301083110832108331083410835108361083710838108391084010841108421084310844108451084610847108481084910850108511085210853108541085510856108571085810859108601086110862108631086410865108661086710868108691087010871108721087310874108751087610877108781087910880108811088210883108841088510886108871088810889108901089110892108931089410895108961089710898108991090010901109021090310904109051090610907109081090910910109111091210913109141091510916109171091810919109201092110922109231092410925109261092710928109291093010931109321093310934109351093610937109381093910940109411094210943109441094510946109471094810949109501095110952109531095410955109561095710958109591096010961109621096310964109651096610967109681096910970109711097210973109741097510976109771097810979109801098110982109831098410985109861098710988109891099010991109921099310994109951099610997109981099911000110011100211003110041100511006110071100811009110101101111012110131101411015110161101711018110191102011021110221102311024110251102611027110281102911030110311103211033110341103511036110371103811039110401104111042110431104411045110461104711048110491105011051110521105311054110551105611057110581105911060110611106211063110641106511066110671106811069110701107111072110731107411075110761107711078110791108011081110821108311084110851108611087110881108911090110911109211093110941109511096110971109811099111001110111102111031110411105111061110711108111091111011111111121111311114111151111611117111181111911120111211112211123111241112511126111271112811129111301113111132111331113411135111361113711138111391114011141111421114311144111451114611147111481114911150111511115211153111541115511156111571115811159111601116111162111631116411165111661116711168111691117011171111721117311174111751117611177111781117911180111811118211183111841118511186111871118811189111901119111192111931119411195111961119711198111991120011201112021120311204112051120611207112081120911210112111121211213112141121511216112171121811219112201122111222112231122411225112261122711228112291123011231112321123311234112351123611237112381123911240112411124211243112441124511246112471124811249112501125111252112531125411255112561125711258112591126011261112621126311264112651126611267112681126911270112711127211273112741127511276112771127811279112801128111282112831128411285112861128711288112891129011291112921129311294112951129611297112981129911300113011130211303113041130511306113071130811309113101131111312113131131411315113161131711318113191132011321113221132311324113251132611327113281132911330113311133211333113341133511336113371133811339113401134111342113431134411345113461134711348113491135011351113521135311354113551135611357113581135911360113611136211363113641136511366113671136811369113701137111372113731137411375113761137711378113791138011381113821138311384113851138611387113881138911390113911139211393113941139511396113971139811399114001140111402114031140411405114061140711408114091141011411114121141311414114151141611417114181141911420114211142211423114241142511426114271142811429114301143111432114331143411435114361143711438114391144011441114421144311444114451144611447114481144911450114511145211453114541145511456114571145811459114601146111462114631146411465114661146711468114691147011471114721147311474114751147611477114781147911480114811148211483114841148511486114871148811489114901149111492114931149411495114961149711498114991150011501115021150311504115051150611507115081150911510115111151211513115141151511516115171151811519115201152111522115231152411525115261152711528115291153011531115321153311534115351153611537115381153911540115411154211543115441154511546115471154811549115501155111552115531155411555115561155711558115591156011561115621156311564115651156611567115681156911570115711157211573115741157511576115771157811579115801158111582115831158411585115861158711588115891159011591115921159311594115951159611597115981159911600116011160211603116041160511606116071160811609116101161111612116131161411615116161161711618116191162011621116221162311624116251162611627116281162911630116311163211633116341163511636116371163811639116401164111642116431164411645116461164711648116491165011651116521165311654116551165611657116581165911660116611166211663116641166511666116671166811669116701167111672116731167411675116761167711678116791168011681116821168311684116851168611687116881168911690116911169211693116941169511696116971169811699117001170111702117031170411705117061170711708117091171011711117121171311714117151171611717117181171911720117211172211723117241172511726117271172811729117301173111732117331173411735117361173711738117391174011741117421174311744117451174611747117481174911750117511175211753117541175511756117571175811759117601176111762117631176411765117661176711768117691177011771117721177311774117751177611777117781177911780117811178211783117841178511786117871178811789117901179111792117931179411795117961179711798117991180011801118021180311804118051180611807118081180911810118111181211813118141181511816118171181811819118201182111822118231182411825118261182711828118291183011831118321183311834118351183611837118381183911840118411184211843118441184511846118471184811849118501185111852118531185411855118561185711858118591186011861118621186311864118651186611867118681186911870118711187211873118741187511876118771187811879118801188111882118831188411885118861188711888118891189011891118921189311894118951189611897118981189911900119011190211903119041190511906119071190811909119101191111912119131191411915119161191711918119191192011921119221192311924119251192611927119281192911930119311193211933119341193511936119371193811939119401194111942119431194411945119461194711948119491195011951119521195311954119551195611957119581195911960119611196211963119641196511966119671196811969119701197111972119731197411975119761197711978119791198011981119821198311984119851198611987119881198911990119911199211993119941199511996119971199811999120001200112002120031200412005120061200712008120091201012011120121201312014120151201612017120181201912020120211202212023120241202512026120271202812029120301203112032120331203412035120361203712038120391204012041120421204312044120451204612047120481204912050120511205212053120541205512056120571205812059120601206112062120631206412065120661206712068120691207012071120721207312074120751207612077120781207912080120811208212083120841208512086120871208812089120901209112092120931209412095120961209712098120991210012101121021210312104121051210612107121081210912110121111211212113121141211512116121171211812119121201212112122121231212412125121261212712128121291213012131121321213312134121351213612137121381213912140121411214212143121441214512146121471214812149121501215112152121531215412155121561215712158121591216012161121621216312164121651216612167121681216912170121711217212173121741217512176121771217812179121801218112182121831218412185121861218712188121891219012191121921219312194121951219612197121981219912200122011220212203122041220512206122071220812209122101221112212122131221412215122161221712218122191222012221122221222312224122251222612227122281222912230122311223212233122341223512236122371223812239122401224112242122431224412245122461224712248122491225012251122521225312254122551225612257122581225912260122611226212263122641226512266122671226812269122701227112272122731227412275122761227712278122791228012281122821228312284122851228612287122881228912290122911229212293122941229512296122971229812299123001230112302123031230412305123061230712308123091231012311123121231312314123151231612317123181231912320123211232212323123241232512326123271232812329123301233112332123331233412335123361233712338123391234012341123421234312344123451234612347123481234912350123511235212353123541235512356123571235812359123601236112362123631236412365123661236712368123691237012371123721237312374123751237612377123781237912380123811238212383123841238512386123871238812389123901239112392123931239412395123961239712398123991240012401124021240312404124051240612407124081240912410124111241212413124141241512416124171241812419124201242112422124231242412425124261242712428124291243012431124321243312434124351243612437124381243912440124411244212443124441244512446124471244812449124501245112452124531245412455124561245712458124591246012461124621246312464124651246612467124681246912470124711247212473124741247512476124771247812479124801248112482124831248412485124861248712488124891249012491124921249312494124951249612497124981249912500125011250212503125041250512506125071250812509125101251112512125131251412515125161251712518125191252012521125221252312524125251252612527125281252912530125311253212533125341253512536125371253812539125401254112542125431254412545125461254712548125491255012551125521255312554125551255612557125581255912560125611256212563125641256512566125671256812569125701257112572125731257412575125761257712578125791258012581125821258312584125851258612587125881258912590125911259212593125941259512596125971259812599126001260112602126031260412605126061260712608126091261012611126121261312614126151261612617126181261912620126211262212623126241262512626126271262812629126301263112632126331263412635126361263712638126391264012641126421264312644126451264612647126481264912650126511265212653126541265512656126571265812659126601266112662126631266412665126661266712668126691267012671126721267312674126751267612677126781267912680126811268212683126841268512686126871268812689126901269112692126931269412695126961269712698126991270012701127021270312704127051270612707127081270912710127111271212713127141271512716127171271812719127201272112722127231272412725127261272712728127291273012731127321273312734127351273612737127381273912740127411274212743127441274512746127471274812749127501275112752127531275412755127561275712758127591276012761127621276312764127651276612767127681276912770127711277212773127741277512776127771277812779127801278112782127831278412785127861278712788127891279012791127921279312794127951279612797127981279912800128011280212803128041280512806128071280812809128101281112812128131281412815128161281712818128191282012821128221282312824128251282612827128281282912830128311283212833128341283512836128371283812839128401284112842128431284412845128461284712848128491285012851128521285312854128551285612857128581285912860128611286212863128641286512866128671286812869128701287112872128731287412875128761287712878128791288012881128821288312884128851288612887128881288912890128911289212893128941289512896128971289812899129001290112902129031290412905129061290712908129091291012911129121291312914129151291612917129181291912920129211292212923129241292512926129271292812929129301293112932129331293412935129361293712938129391294012941129421294312944129451294612947129481294912950129511295212953129541295512956129571295812959129601296112962129631296412965129661296712968129691297012971129721297312974129751297612977129781297912980129811298212983129841298512986129871298812989129901299112992129931299412995129961299712998129991300013001130021300313004130051300613007130081300913010130111301213013130141301513016130171301813019130201302113022130231302413025130261302713028130291303013031130321303313034130351303613037130381303913040130411304213043130441304513046130471304813049130501305113052130531305413055130561305713058130591306013061130621306313064130651306613067130681306913070130711307213073130741307513076130771307813079130801308113082130831308413085130861308713088130891309013091130921309313094130951309613097130981309913100131011310213103131041310513106131071310813109131101311113112131131311413115131161311713118131191312013121131221312313124131251312613127131281312913130131311313213133131341313513136131371313813139131401314113142131431314413145131461314713148131491315013151131521315313154131551315613157131581315913160131611316213163131641316513166131671316813169
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Performance events core code:
  4. *
  5. * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
  6. * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
  7. * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
  8. * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
  9. */
  10. #include <linux/fs.h>
  11. #include <linux/mm.h>
  12. #include <linux/cpu.h>
  13. #include <linux/smp.h>
  14. #include <linux/idr.h>
  15. #include <linux/file.h>
  16. #include <linux/poll.h>
  17. #include <linux/slab.h>
  18. #include <linux/hash.h>
  19. #include <linux/tick.h>
  20. #include <linux/sysfs.h>
  21. #include <linux/dcache.h>
  22. #include <linux/percpu.h>
  23. #include <linux/ptrace.h>
  24. #include <linux/reboot.h>
  25. #include <linux/vmstat.h>
  26. #include <linux/device.h>
  27. #include <linux/export.h>
  28. #include <linux/vmalloc.h>
  29. #include <linux/hardirq.h>
  30. #include <linux/hugetlb.h>
  31. #include <linux/rculist.h>
  32. #include <linux/uaccess.h>
  33. #include <linux/syscalls.h>
  34. #include <linux/anon_inodes.h>
  35. #include <linux/kernel_stat.h>
  36. #include <linux/cgroup.h>
  37. #include <linux/perf_event.h>
  38. #include <linux/trace_events.h>
  39. #include <linux/hw_breakpoint.h>
  40. #include <linux/mm_types.h>
  41. #include <linux/module.h>
  42. #include <linux/mman.h>
  43. #include <linux/compat.h>
  44. #include <linux/bpf.h>
  45. #include <linux/filter.h>
  46. #include <linux/namei.h>
  47. #include <linux/parser.h>
  48. #include <linux/sched/clock.h>
  49. #include <linux/sched/mm.h>
  50. #include <linux/proc_ns.h>
  51. #include <linux/mount.h>
  52. #include <linux/min_heap.h>
  53. #include "internal.h"
  54. #include <asm/irq_regs.h>
  55. typedef int (*remote_function_f)(void *);
  56. struct remote_function_call {
  57. struct task_struct *p;
  58. remote_function_f func;
  59. void *info;
  60. int ret;
  61. };
  62. static void remote_function(void *data)
  63. {
  64. struct remote_function_call *tfc = data;
  65. struct task_struct *p = tfc->p;
  66. if (p) {
  67. /* -EAGAIN */
  68. if (task_cpu(p) != smp_processor_id())
  69. return;
  70. /*
  71. * Now that we're on right CPU with IRQs disabled, we can test
  72. * if we hit the right task without races.
  73. */
  74. tfc->ret = -ESRCH; /* No such (running) process */
  75. if (p != current)
  76. return;
  77. }
  78. tfc->ret = tfc->func(tfc->info);
  79. }
  80. /**
  81. * task_function_call - call a function on the cpu on which a task runs
  82. * @p: the task to evaluate
  83. * @func: the function to be called
  84. * @info: the function call argument
  85. *
  86. * Calls the function @func when the task is currently running. This might
  87. * be on the current CPU, which just calls the function directly. This will
  88. * retry due to any failures in smp_call_function_single(), such as if the
  89. * task_cpu() goes offline concurrently.
  90. *
  91. * returns @func return value or -ESRCH or -ENXIO when the process isn't running
  92. */
  93. static int
  94. task_function_call(struct task_struct *p, remote_function_f func, void *info)
  95. {
  96. struct remote_function_call data = {
  97. .p = p,
  98. .func = func,
  99. .info = info,
  100. .ret = -EAGAIN,
  101. };
  102. int ret;
  103. for (;;) {
  104. ret = smp_call_function_single(task_cpu(p), remote_function,
  105. &data, 1);
  106. if (!ret)
  107. ret = data.ret;
  108. if (ret != -EAGAIN)
  109. break;
  110. cond_resched();
  111. }
  112. return ret;
  113. }
  114. /**
  115. * cpu_function_call - call a function on the cpu
  116. * @func: the function to be called
  117. * @info: the function call argument
  118. *
  119. * Calls the function @func on the remote cpu.
  120. *
  121. * returns: @func return value or -ENXIO when the cpu is offline
  122. */
  123. static int cpu_function_call(int cpu, remote_function_f func, void *info)
  124. {
  125. struct remote_function_call data = {
  126. .p = NULL,
  127. .func = func,
  128. .info = info,
  129. .ret = -ENXIO, /* No such CPU */
  130. };
  131. smp_call_function_single(cpu, remote_function, &data, 1);
  132. return data.ret;
  133. }
  134. static inline struct perf_cpu_context *
  135. __get_cpu_context(struct perf_event_context *ctx)
  136. {
  137. return this_cpu_ptr(ctx->pmu->pmu_cpu_context);
  138. }
  139. static void perf_ctx_lock(struct perf_cpu_context *cpuctx,
  140. struct perf_event_context *ctx)
  141. {
  142. raw_spin_lock(&cpuctx->ctx.lock);
  143. if (ctx)
  144. raw_spin_lock(&ctx->lock);
  145. }
  146. static void perf_ctx_unlock(struct perf_cpu_context *cpuctx,
  147. struct perf_event_context *ctx)
  148. {
  149. if (ctx)
  150. raw_spin_unlock(&ctx->lock);
  151. raw_spin_unlock(&cpuctx->ctx.lock);
  152. }
  153. #define TASK_TOMBSTONE ((void *)-1L)
  154. static bool is_kernel_event(struct perf_event *event)
  155. {
  156. return READ_ONCE(event->owner) == TASK_TOMBSTONE;
  157. }
  158. /*
  159. * On task ctx scheduling...
  160. *
  161. * When !ctx->nr_events a task context will not be scheduled. This means
  162. * we can disable the scheduler hooks (for performance) without leaving
  163. * pending task ctx state.
  164. *
  165. * This however results in two special cases:
  166. *
  167. * - removing the last event from a task ctx; this is relatively straight
  168. * forward and is done in __perf_remove_from_context.
  169. *
  170. * - adding the first event to a task ctx; this is tricky because we cannot
  171. * rely on ctx->is_active and therefore cannot use event_function_call().
  172. * See perf_install_in_context().
  173. *
  174. * If ctx->nr_events, then ctx->is_active and cpuctx->task_ctx are set.
  175. */
  176. typedef void (*event_f)(struct perf_event *, struct perf_cpu_context *,
  177. struct perf_event_context *, void *);
  178. struct event_function_struct {
  179. struct perf_event *event;
  180. event_f func;
  181. void *data;
  182. };
  183. static int event_function(void *info)
  184. {
  185. struct event_function_struct *efs = info;
  186. struct perf_event *event = efs->event;
  187. struct perf_event_context *ctx = event->ctx;
  188. struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
  189. struct perf_event_context *task_ctx = cpuctx->task_ctx;
  190. int ret = 0;
  191. lockdep_assert_irqs_disabled();
  192. perf_ctx_lock(cpuctx, task_ctx);
  193. /*
  194. * Since we do the IPI call without holding ctx->lock things can have
  195. * changed, double check we hit the task we set out to hit.
  196. */
  197. if (ctx->task) {
  198. if (ctx->task != current) {
  199. ret = -ESRCH;
  200. goto unlock;
  201. }
  202. /*
  203. * We only use event_function_call() on established contexts,
  204. * and event_function() is only ever called when active (or
  205. * rather, we'll have bailed in task_function_call() or the
  206. * above ctx->task != current test), therefore we must have
  207. * ctx->is_active here.
  208. */
  209. WARN_ON_ONCE(!ctx->is_active);
  210. /*
  211. * And since we have ctx->is_active, cpuctx->task_ctx must
  212. * match.
  213. */
  214. WARN_ON_ONCE(task_ctx != ctx);
  215. } else {
  216. WARN_ON_ONCE(&cpuctx->ctx != ctx);
  217. }
  218. efs->func(event, cpuctx, ctx, efs->data);
  219. unlock:
  220. perf_ctx_unlock(cpuctx, task_ctx);
  221. return ret;
  222. }
  223. static void event_function_call(struct perf_event *event, event_f func, void *data)
  224. {
  225. struct perf_event_context *ctx = event->ctx;
  226. struct task_struct *task = READ_ONCE(ctx->task); /* verified in event_function */
  227. struct event_function_struct efs = {
  228. .event = event,
  229. .func = func,
  230. .data = data,
  231. };
  232. if (!event->parent) {
  233. /*
  234. * If this is a !child event, we must hold ctx::mutex to
  235. * stabilize the event->ctx relation. See
  236. * perf_event_ctx_lock().
  237. */
  238. lockdep_assert_held(&ctx->mutex);
  239. }
  240. if (!task) {
  241. cpu_function_call(event->cpu, event_function, &efs);
  242. return;
  243. }
  244. if (task == TASK_TOMBSTONE)
  245. return;
  246. again:
  247. if (!task_function_call(task, event_function, &efs))
  248. return;
  249. raw_spin_lock_irq(&ctx->lock);
  250. /*
  251. * Reload the task pointer, it might have been changed by
  252. * a concurrent perf_event_context_sched_out().
  253. */
  254. task = ctx->task;
  255. if (task == TASK_TOMBSTONE) {
  256. raw_spin_unlock_irq(&ctx->lock);
  257. return;
  258. }
  259. if (ctx->is_active) {
  260. raw_spin_unlock_irq(&ctx->lock);
  261. goto again;
  262. }
  263. func(event, NULL, ctx, data);
  264. raw_spin_unlock_irq(&ctx->lock);
  265. }
  266. /*
  267. * Similar to event_function_call() + event_function(), but hard assumes IRQs
  268. * are already disabled and we're on the right CPU.
  269. */
  270. static void event_function_local(struct perf_event *event, event_f func, void *data)
  271. {
  272. struct perf_event_context *ctx = event->ctx;
  273. struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
  274. struct task_struct *task = READ_ONCE(ctx->task);
  275. struct perf_event_context *task_ctx = NULL;
  276. lockdep_assert_irqs_disabled();
  277. if (task) {
  278. if (task == TASK_TOMBSTONE)
  279. return;
  280. task_ctx = ctx;
  281. }
  282. perf_ctx_lock(cpuctx, task_ctx);
  283. task = ctx->task;
  284. if (task == TASK_TOMBSTONE)
  285. goto unlock;
  286. if (task) {
  287. /*
  288. * We must be either inactive or active and the right task,
  289. * otherwise we're screwed, since we cannot IPI to somewhere
  290. * else.
  291. */
  292. if (ctx->is_active) {
  293. if (WARN_ON_ONCE(task != current))
  294. goto unlock;
  295. if (WARN_ON_ONCE(cpuctx->task_ctx != ctx))
  296. goto unlock;
  297. }
  298. } else {
  299. WARN_ON_ONCE(&cpuctx->ctx != ctx);
  300. }
  301. func(event, cpuctx, ctx, data);
  302. unlock:
  303. perf_ctx_unlock(cpuctx, task_ctx);
  304. }
  305. #define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\
  306. PERF_FLAG_FD_OUTPUT |\
  307. PERF_FLAG_PID_CGROUP |\
  308. PERF_FLAG_FD_CLOEXEC)
  309. /*
  310. * branch priv levels that need permission checks
  311. */
  312. #define PERF_SAMPLE_BRANCH_PERM_PLM \
  313. (PERF_SAMPLE_BRANCH_KERNEL |\
  314. PERF_SAMPLE_BRANCH_HV)
  315. enum event_type_t {
  316. EVENT_FLEXIBLE = 0x1,
  317. EVENT_PINNED = 0x2,
  318. EVENT_TIME = 0x4,
  319. /* see ctx_resched() for details */
  320. EVENT_CPU = 0x8,
  321. EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
  322. };
  323. /*
  324. * perf_sched_events : >0 events exist
  325. * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
  326. */
  327. static void perf_sched_delayed(struct work_struct *work);
  328. DEFINE_STATIC_KEY_FALSE(perf_sched_events);
  329. static DECLARE_DELAYED_WORK(perf_sched_work, perf_sched_delayed);
  330. static DEFINE_MUTEX(perf_sched_mutex);
  331. static atomic_t perf_sched_count;
  332. static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
  333. static DEFINE_PER_CPU(int, perf_sched_cb_usages);
  334. static DEFINE_PER_CPU(struct pmu_event_list, pmu_sb_events);
  335. static atomic_t nr_mmap_events __read_mostly;
  336. static atomic_t nr_comm_events __read_mostly;
  337. static atomic_t nr_namespaces_events __read_mostly;
  338. static atomic_t nr_task_events __read_mostly;
  339. static atomic_t nr_freq_events __read_mostly;
  340. static atomic_t nr_switch_events __read_mostly;
  341. static atomic_t nr_ksymbol_events __read_mostly;
  342. static atomic_t nr_bpf_events __read_mostly;
  343. static atomic_t nr_cgroup_events __read_mostly;
  344. static atomic_t nr_text_poke_events __read_mostly;
  345. static LIST_HEAD(pmus);
  346. static DEFINE_MUTEX(pmus_lock);
  347. static struct srcu_struct pmus_srcu;
  348. static cpumask_var_t perf_online_mask;
  349. /*
  350. * perf event paranoia level:
  351. * -1 - not paranoid at all
  352. * 0 - disallow raw tracepoint access for unpriv
  353. * 1 - disallow cpu events for unpriv
  354. * 2 - disallow kernel profiling for unpriv
  355. */
  356. int sysctl_perf_event_paranoid __read_mostly = 2;
  357. /* Minimum for 512 kiB + 1 user control page */
  358. int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */
  359. /*
  360. * max perf event sample rate
  361. */
  362. #define DEFAULT_MAX_SAMPLE_RATE 100000
  363. #define DEFAULT_SAMPLE_PERIOD_NS (NSEC_PER_SEC / DEFAULT_MAX_SAMPLE_RATE)
  364. #define DEFAULT_CPU_TIME_MAX_PERCENT 25
  365. int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE;
  366. static int max_samples_per_tick __read_mostly = DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ);
  367. static int perf_sample_period_ns __read_mostly = DEFAULT_SAMPLE_PERIOD_NS;
  368. static int perf_sample_allowed_ns __read_mostly =
  369. DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100;
  370. static void update_perf_cpu_limits(void)
  371. {
  372. u64 tmp = perf_sample_period_ns;
  373. tmp *= sysctl_perf_cpu_time_max_percent;
  374. tmp = div_u64(tmp, 100);
  375. if (!tmp)
  376. tmp = 1;
  377. WRITE_ONCE(perf_sample_allowed_ns, tmp);
  378. }
  379. static bool perf_rotate_context(struct perf_cpu_context *cpuctx);
  380. int perf_proc_update_handler(struct ctl_table *table, int write,
  381. void *buffer, size_t *lenp, loff_t *ppos)
  382. {
  383. int ret;
  384. int perf_cpu = sysctl_perf_cpu_time_max_percent;
  385. /*
  386. * If throttling is disabled don't allow the write:
  387. */
  388. if (write && (perf_cpu == 100 || perf_cpu == 0))
  389. return -EINVAL;
  390. ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
  391. if (ret || !write)
  392. return ret;
  393. max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ);
  394. perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
  395. update_perf_cpu_limits();
  396. return 0;
  397. }
  398. int sysctl_perf_cpu_time_max_percent __read_mostly = DEFAULT_CPU_TIME_MAX_PERCENT;
  399. int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write,
  400. void *buffer, size_t *lenp, loff_t *ppos)
  401. {
  402. int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
  403. if (ret || !write)
  404. return ret;
  405. if (sysctl_perf_cpu_time_max_percent == 100 ||
  406. sysctl_perf_cpu_time_max_percent == 0) {
  407. printk(KERN_WARNING
  408. "perf: Dynamic interrupt throttling disabled, can hang your system!\n");
  409. WRITE_ONCE(perf_sample_allowed_ns, 0);
  410. } else {
  411. update_perf_cpu_limits();
  412. }
  413. return 0;
  414. }
  415. /*
  416. * perf samples are done in some very critical code paths (NMIs).
  417. * If they take too much CPU time, the system can lock up and not
  418. * get any real work done. This will drop the sample rate when
  419. * we detect that events are taking too long.
  420. */
  421. #define NR_ACCUMULATED_SAMPLES 128
  422. static DEFINE_PER_CPU(u64, running_sample_length);
  423. static u64 __report_avg;
  424. static u64 __report_allowed;
  425. static void perf_duration_warn(struct irq_work *w)
  426. {
  427. printk_ratelimited(KERN_INFO
  428. "perf: interrupt took too long (%lld > %lld), lowering "
  429. "kernel.perf_event_max_sample_rate to %d\n",
  430. __report_avg, __report_allowed,
  431. sysctl_perf_event_sample_rate);
  432. }
  433. static DEFINE_IRQ_WORK(perf_duration_work, perf_duration_warn);
  434. void perf_sample_event_took(u64 sample_len_ns)
  435. {
  436. u64 max_len = READ_ONCE(perf_sample_allowed_ns);
  437. u64 running_len;
  438. u64 avg_len;
  439. u32 max;
  440. if (max_len == 0)
  441. return;
  442. /* Decay the counter by 1 average sample. */
  443. running_len = __this_cpu_read(running_sample_length);
  444. running_len -= running_len/NR_ACCUMULATED_SAMPLES;
  445. running_len += sample_len_ns;
  446. __this_cpu_write(running_sample_length, running_len);
  447. /*
  448. * Note: this will be biased artifically low until we have
  449. * seen NR_ACCUMULATED_SAMPLES. Doing it this way keeps us
  450. * from having to maintain a count.
  451. */
  452. avg_len = running_len/NR_ACCUMULATED_SAMPLES;
  453. if (avg_len <= max_len)
  454. return;
  455. __report_avg = avg_len;
  456. __report_allowed = max_len;
  457. /*
  458. * Compute a throttle threshold 25% below the current duration.
  459. */
  460. avg_len += avg_len / 4;
  461. max = (TICK_NSEC / 100) * sysctl_perf_cpu_time_max_percent;
  462. if (avg_len < max)
  463. max /= (u32)avg_len;
  464. else
  465. max = 1;
  466. WRITE_ONCE(perf_sample_allowed_ns, avg_len);
  467. WRITE_ONCE(max_samples_per_tick, max);
  468. sysctl_perf_event_sample_rate = max * HZ;
  469. perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
  470. if (!irq_work_queue(&perf_duration_work)) {
  471. early_printk("perf: interrupt took too long (%lld > %lld), lowering "
  472. "kernel.perf_event_max_sample_rate to %d\n",
  473. __report_avg, __report_allowed,
  474. sysctl_perf_event_sample_rate);
  475. }
  476. }
  477. static atomic64_t perf_event_id;
  478. static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
  479. enum event_type_t event_type);
  480. static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
  481. enum event_type_t event_type,
  482. struct task_struct *task);
  483. static void update_context_time(struct perf_event_context *ctx);
  484. static u64 perf_event_time(struct perf_event *event);
  485. void __weak perf_event_print_debug(void) { }
  486. extern __weak const char *perf_pmu_name(void)
  487. {
  488. return "pmu";
  489. }
  490. static inline u64 perf_clock(void)
  491. {
  492. return local_clock();
  493. }
  494. static inline u64 perf_event_clock(struct perf_event *event)
  495. {
  496. return event->clock();
  497. }
  498. /*
  499. * State based event timekeeping...
  500. *
  501. * The basic idea is to use event->state to determine which (if any) time
  502. * fields to increment with the current delta. This means we only need to
  503. * update timestamps when we change state or when they are explicitly requested
  504. * (read).
  505. *
  506. * Event groups make things a little more complicated, but not terribly so. The
  507. * rules for a group are that if the group leader is OFF the entire group is
  508. * OFF, irrespecive of what the group member states are. This results in
  509. * __perf_effective_state().
  510. *
  511. * A futher ramification is that when a group leader flips between OFF and
  512. * !OFF, we need to update all group member times.
  513. *
  514. *
  515. * NOTE: perf_event_time() is based on the (cgroup) context time, and thus we
  516. * need to make sure the relevant context time is updated before we try and
  517. * update our timestamps.
  518. */
  519. static __always_inline enum perf_event_state
  520. __perf_effective_state(struct perf_event *event)
  521. {
  522. struct perf_event *leader = event->group_leader;
  523. if (leader->state <= PERF_EVENT_STATE_OFF)
  524. return leader->state;
  525. return event->state;
  526. }
  527. static __always_inline void
  528. __perf_update_times(struct perf_event *event, u64 now, u64 *enabled, u64 *running)
  529. {
  530. enum perf_event_state state = __perf_effective_state(event);
  531. u64 delta = now - event->tstamp;
  532. *enabled = event->total_time_enabled;
  533. if (state >= PERF_EVENT_STATE_INACTIVE)
  534. *enabled += delta;
  535. *running = event->total_time_running;
  536. if (state >= PERF_EVENT_STATE_ACTIVE)
  537. *running += delta;
  538. }
  539. static void perf_event_update_time(struct perf_event *event)
  540. {
  541. u64 now = perf_event_time(event);
  542. __perf_update_times(event, now, &event->total_time_enabled,
  543. &event->total_time_running);
  544. event->tstamp = now;
  545. }
  546. static void perf_event_update_sibling_time(struct perf_event *leader)
  547. {
  548. struct perf_event *sibling;
  549. for_each_sibling_event(sibling, leader)
  550. perf_event_update_time(sibling);
  551. }
  552. static void
  553. perf_event_set_state(struct perf_event *event, enum perf_event_state state)
  554. {
  555. if (event->state == state)
  556. return;
  557. perf_event_update_time(event);
  558. /*
  559. * If a group leader gets enabled/disabled all its siblings
  560. * are affected too.
  561. */
  562. if ((event->state < 0) ^ (state < 0))
  563. perf_event_update_sibling_time(event);
  564. WRITE_ONCE(event->state, state);
  565. }
  566. #ifdef CONFIG_CGROUP_PERF
  567. static inline bool
  568. perf_cgroup_match(struct perf_event *event)
  569. {
  570. struct perf_event_context *ctx = event->ctx;
  571. struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
  572. /* @event doesn't care about cgroup */
  573. if (!event->cgrp)
  574. return true;
  575. /* wants specific cgroup scope but @cpuctx isn't associated with any */
  576. if (!cpuctx->cgrp)
  577. return false;
  578. /*
  579. * Cgroup scoping is recursive. An event enabled for a cgroup is
  580. * also enabled for all its descendant cgroups. If @cpuctx's
  581. * cgroup is a descendant of @event's (the test covers identity
  582. * case), it's a match.
  583. */
  584. return cgroup_is_descendant(cpuctx->cgrp->css.cgroup,
  585. event->cgrp->css.cgroup);
  586. }
  587. static inline void perf_detach_cgroup(struct perf_event *event)
  588. {
  589. css_put(&event->cgrp->css);
  590. event->cgrp = NULL;
  591. }
  592. static inline int is_cgroup_event(struct perf_event *event)
  593. {
  594. return event->cgrp != NULL;
  595. }
  596. static inline u64 perf_cgroup_event_time(struct perf_event *event)
  597. {
  598. struct perf_cgroup_info *t;
  599. t = per_cpu_ptr(event->cgrp->info, event->cpu);
  600. return t->time;
  601. }
  602. static inline void __update_cgrp_time(struct perf_cgroup *cgrp)
  603. {
  604. struct perf_cgroup_info *info;
  605. u64 now;
  606. now = perf_clock();
  607. info = this_cpu_ptr(cgrp->info);
  608. info->time += now - info->timestamp;
  609. info->timestamp = now;
  610. }
  611. static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
  612. {
  613. struct perf_cgroup *cgrp = cpuctx->cgrp;
  614. struct cgroup_subsys_state *css;
  615. if (cgrp) {
  616. for (css = &cgrp->css; css; css = css->parent) {
  617. cgrp = container_of(css, struct perf_cgroup, css);
  618. __update_cgrp_time(cgrp);
  619. }
  620. }
  621. }
  622. static inline void update_cgrp_time_from_event(struct perf_event *event)
  623. {
  624. struct perf_cgroup *cgrp;
  625. /*
  626. * ensure we access cgroup data only when needed and
  627. * when we know the cgroup is pinned (css_get)
  628. */
  629. if (!is_cgroup_event(event))
  630. return;
  631. cgrp = perf_cgroup_from_task(current, event->ctx);
  632. /*
  633. * Do not update time when cgroup is not active
  634. */
  635. if (cgroup_is_descendant(cgrp->css.cgroup, event->cgrp->css.cgroup))
  636. __update_cgrp_time(event->cgrp);
  637. }
  638. static inline void
  639. perf_cgroup_set_timestamp(struct task_struct *task,
  640. struct perf_event_context *ctx)
  641. {
  642. struct perf_cgroup *cgrp;
  643. struct perf_cgroup_info *info;
  644. struct cgroup_subsys_state *css;
  645. /*
  646. * ctx->lock held by caller
  647. * ensure we do not access cgroup data
  648. * unless we have the cgroup pinned (css_get)
  649. */
  650. if (!task || !ctx->nr_cgroups)
  651. return;
  652. cgrp = perf_cgroup_from_task(task, ctx);
  653. for (css = &cgrp->css; css; css = css->parent) {
  654. cgrp = container_of(css, struct perf_cgroup, css);
  655. info = this_cpu_ptr(cgrp->info);
  656. info->timestamp = ctx->timestamp;
  657. }
  658. }
  659. static DEFINE_PER_CPU(struct list_head, cgrp_cpuctx_list);
  660. #define PERF_CGROUP_SWOUT 0x1 /* cgroup switch out every event */
  661. #define PERF_CGROUP_SWIN 0x2 /* cgroup switch in events based on task */
  662. /*
  663. * reschedule events based on the cgroup constraint of task.
  664. *
  665. * mode SWOUT : schedule out everything
  666. * mode SWIN : schedule in based on cgroup for next
  667. */
  668. static void perf_cgroup_switch(struct task_struct *task, int mode)
  669. {
  670. struct perf_cpu_context *cpuctx, *tmp;
  671. struct list_head *list;
  672. unsigned long flags;
  673. /*
  674. * Disable interrupts and preemption to avoid this CPU's
  675. * cgrp_cpuctx_entry to change under us.
  676. */
  677. local_irq_save(flags);
  678. list = this_cpu_ptr(&cgrp_cpuctx_list);
  679. list_for_each_entry_safe(cpuctx, tmp, list, cgrp_cpuctx_entry) {
  680. WARN_ON_ONCE(cpuctx->ctx.nr_cgroups == 0);
  681. perf_ctx_lock(cpuctx, cpuctx->task_ctx);
  682. perf_pmu_disable(cpuctx->ctx.pmu);
  683. if (mode & PERF_CGROUP_SWOUT) {
  684. cpu_ctx_sched_out(cpuctx, EVENT_ALL);
  685. /*
  686. * must not be done before ctxswout due
  687. * to event_filter_match() in event_sched_out()
  688. */
  689. cpuctx->cgrp = NULL;
  690. }
  691. if (mode & PERF_CGROUP_SWIN) {
  692. WARN_ON_ONCE(cpuctx->cgrp);
  693. /*
  694. * set cgrp before ctxsw in to allow
  695. * event_filter_match() to not have to pass
  696. * task around
  697. * we pass the cpuctx->ctx to perf_cgroup_from_task()
  698. * because cgorup events are only per-cpu
  699. */
  700. cpuctx->cgrp = perf_cgroup_from_task(task,
  701. &cpuctx->ctx);
  702. cpu_ctx_sched_in(cpuctx, EVENT_ALL, task);
  703. }
  704. perf_pmu_enable(cpuctx->ctx.pmu);
  705. perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
  706. }
  707. local_irq_restore(flags);
  708. }
  709. static inline void perf_cgroup_sched_out(struct task_struct *task,
  710. struct task_struct *next)
  711. {
  712. struct perf_cgroup *cgrp1;
  713. struct perf_cgroup *cgrp2 = NULL;
  714. rcu_read_lock();
  715. /*
  716. * we come here when we know perf_cgroup_events > 0
  717. * we do not need to pass the ctx here because we know
  718. * we are holding the rcu lock
  719. */
  720. cgrp1 = perf_cgroup_from_task(task, NULL);
  721. cgrp2 = perf_cgroup_from_task(next, NULL);
  722. /*
  723. * only schedule out current cgroup events if we know
  724. * that we are switching to a different cgroup. Otherwise,
  725. * do no touch the cgroup events.
  726. */
  727. if (cgrp1 != cgrp2)
  728. perf_cgroup_switch(task, PERF_CGROUP_SWOUT);
  729. rcu_read_unlock();
  730. }
  731. static inline void perf_cgroup_sched_in(struct task_struct *prev,
  732. struct task_struct *task)
  733. {
  734. struct perf_cgroup *cgrp1;
  735. struct perf_cgroup *cgrp2 = NULL;
  736. rcu_read_lock();
  737. /*
  738. * we come here when we know perf_cgroup_events > 0
  739. * we do not need to pass the ctx here because we know
  740. * we are holding the rcu lock
  741. */
  742. cgrp1 = perf_cgroup_from_task(task, NULL);
  743. cgrp2 = perf_cgroup_from_task(prev, NULL);
  744. /*
  745. * only need to schedule in cgroup events if we are changing
  746. * cgroup during ctxsw. Cgroup events were not scheduled
  747. * out of ctxsw out if that was not the case.
  748. */
  749. if (cgrp1 != cgrp2)
  750. perf_cgroup_switch(task, PERF_CGROUP_SWIN);
  751. rcu_read_unlock();
  752. }
  753. static int perf_cgroup_ensure_storage(struct perf_event *event,
  754. struct cgroup_subsys_state *css)
  755. {
  756. struct perf_cpu_context *cpuctx;
  757. struct perf_event **storage;
  758. int cpu, heap_size, ret = 0;
  759. /*
  760. * Allow storage to have sufficent space for an iterator for each
  761. * possibly nested cgroup plus an iterator for events with no cgroup.
  762. */
  763. for (heap_size = 1; css; css = css->parent)
  764. heap_size++;
  765. for_each_possible_cpu(cpu) {
  766. cpuctx = per_cpu_ptr(event->pmu->pmu_cpu_context, cpu);
  767. if (heap_size <= cpuctx->heap_size)
  768. continue;
  769. storage = kmalloc_node(heap_size * sizeof(struct perf_event *),
  770. GFP_KERNEL, cpu_to_node(cpu));
  771. if (!storage) {
  772. ret = -ENOMEM;
  773. break;
  774. }
  775. raw_spin_lock_irq(&cpuctx->ctx.lock);
  776. if (cpuctx->heap_size < heap_size) {
  777. swap(cpuctx->heap, storage);
  778. if (storage == cpuctx->heap_default)
  779. storage = NULL;
  780. cpuctx->heap_size = heap_size;
  781. }
  782. raw_spin_unlock_irq(&cpuctx->ctx.lock);
  783. kfree(storage);
  784. }
  785. return ret;
  786. }
  787. static inline int perf_cgroup_connect(int fd, struct perf_event *event,
  788. struct perf_event_attr *attr,
  789. struct perf_event *group_leader)
  790. {
  791. struct perf_cgroup *cgrp;
  792. struct cgroup_subsys_state *css;
  793. struct fd f = fdget(fd);
  794. int ret = 0;
  795. if (!f.file)
  796. return -EBADF;
  797. css = css_tryget_online_from_dir(f.file->f_path.dentry,
  798. &perf_event_cgrp_subsys);
  799. if (IS_ERR(css)) {
  800. ret = PTR_ERR(css);
  801. goto out;
  802. }
  803. ret = perf_cgroup_ensure_storage(event, css);
  804. if (ret)
  805. goto out;
  806. cgrp = container_of(css, struct perf_cgroup, css);
  807. event->cgrp = cgrp;
  808. /*
  809. * all events in a group must monitor
  810. * the same cgroup because a task belongs
  811. * to only one perf cgroup at a time
  812. */
  813. if (group_leader && group_leader->cgrp != cgrp) {
  814. perf_detach_cgroup(event);
  815. ret = -EINVAL;
  816. }
  817. out:
  818. fdput(f);
  819. return ret;
  820. }
  821. static inline void
  822. perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
  823. {
  824. struct perf_cgroup_info *t;
  825. t = per_cpu_ptr(event->cgrp->info, event->cpu);
  826. event->shadow_ctx_time = now - t->timestamp;
  827. }
  828. static inline void
  829. perf_cgroup_event_enable(struct perf_event *event, struct perf_event_context *ctx)
  830. {
  831. struct perf_cpu_context *cpuctx;
  832. if (!is_cgroup_event(event))
  833. return;
  834. /*
  835. * Because cgroup events are always per-cpu events,
  836. * @ctx == &cpuctx->ctx.
  837. */
  838. cpuctx = container_of(ctx, struct perf_cpu_context, ctx);
  839. /*
  840. * Since setting cpuctx->cgrp is conditional on the current @cgrp
  841. * matching the event's cgroup, we must do this for every new event,
  842. * because if the first would mismatch, the second would not try again
  843. * and we would leave cpuctx->cgrp unset.
  844. */
  845. if (ctx->is_active && !cpuctx->cgrp) {
  846. struct perf_cgroup *cgrp = perf_cgroup_from_task(current, ctx);
  847. if (cgroup_is_descendant(cgrp->css.cgroup, event->cgrp->css.cgroup))
  848. cpuctx->cgrp = cgrp;
  849. }
  850. if (ctx->nr_cgroups++)
  851. return;
  852. list_add(&cpuctx->cgrp_cpuctx_entry,
  853. per_cpu_ptr(&cgrp_cpuctx_list, event->cpu));
  854. }
  855. static inline void
  856. perf_cgroup_event_disable(struct perf_event *event, struct perf_event_context *ctx)
  857. {
  858. struct perf_cpu_context *cpuctx;
  859. if (!is_cgroup_event(event))
  860. return;
  861. /*
  862. * Because cgroup events are always per-cpu events,
  863. * @ctx == &cpuctx->ctx.
  864. */
  865. cpuctx = container_of(ctx, struct perf_cpu_context, ctx);
  866. if (--ctx->nr_cgroups)
  867. return;
  868. if (ctx->is_active && cpuctx->cgrp)
  869. cpuctx->cgrp = NULL;
  870. list_del(&cpuctx->cgrp_cpuctx_entry);
  871. }
  872. #else /* !CONFIG_CGROUP_PERF */
  873. static inline bool
  874. perf_cgroup_match(struct perf_event *event)
  875. {
  876. return true;
  877. }
  878. static inline void perf_detach_cgroup(struct perf_event *event)
  879. {}
  880. static inline int is_cgroup_event(struct perf_event *event)
  881. {
  882. return 0;
  883. }
  884. static inline void update_cgrp_time_from_event(struct perf_event *event)
  885. {
  886. }
  887. static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
  888. {
  889. }
  890. static inline void perf_cgroup_sched_out(struct task_struct *task,
  891. struct task_struct *next)
  892. {
  893. }
  894. static inline void perf_cgroup_sched_in(struct task_struct *prev,
  895. struct task_struct *task)
  896. {
  897. }
  898. static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event,
  899. struct perf_event_attr *attr,
  900. struct perf_event *group_leader)
  901. {
  902. return -EINVAL;
  903. }
  904. static inline void
  905. perf_cgroup_set_timestamp(struct task_struct *task,
  906. struct perf_event_context *ctx)
  907. {
  908. }
  909. static inline void
  910. perf_cgroup_switch(struct task_struct *task, struct task_struct *next)
  911. {
  912. }
  913. static inline void
  914. perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
  915. {
  916. }
  917. static inline u64 perf_cgroup_event_time(struct perf_event *event)
  918. {
  919. return 0;
  920. }
  921. static inline void
  922. perf_cgroup_event_enable(struct perf_event *event, struct perf_event_context *ctx)
  923. {
  924. }
  925. static inline void
  926. perf_cgroup_event_disable(struct perf_event *event, struct perf_event_context *ctx)
  927. {
  928. }
  929. #endif
  930. /*
  931. * set default to be dependent on timer tick just
  932. * like original code
  933. */
  934. #define PERF_CPU_HRTIMER (1000 / HZ)
  935. /*
  936. * function must be called with interrupts disabled
  937. */
  938. static enum hrtimer_restart perf_mux_hrtimer_handler(struct hrtimer *hr)
  939. {
  940. struct perf_cpu_context *cpuctx;
  941. bool rotations;
  942. lockdep_assert_irqs_disabled();
  943. cpuctx = container_of(hr, struct perf_cpu_context, hrtimer);
  944. rotations = perf_rotate_context(cpuctx);
  945. raw_spin_lock(&cpuctx->hrtimer_lock);
  946. if (rotations)
  947. hrtimer_forward_now(hr, cpuctx->hrtimer_interval);
  948. else
  949. cpuctx->hrtimer_active = 0;
  950. raw_spin_unlock(&cpuctx->hrtimer_lock);
  951. return rotations ? HRTIMER_RESTART : HRTIMER_NORESTART;
  952. }
  953. static void __perf_mux_hrtimer_init(struct perf_cpu_context *cpuctx, int cpu)
  954. {
  955. struct hrtimer *timer = &cpuctx->hrtimer;
  956. struct pmu *pmu = cpuctx->ctx.pmu;
  957. u64 interval;
  958. /* no multiplexing needed for SW PMU */
  959. if (pmu->task_ctx_nr == perf_sw_context)
  960. return;
  961. /*
  962. * check default is sane, if not set then force to
  963. * default interval (1/tick)
  964. */
  965. interval = pmu->hrtimer_interval_ms;
  966. if (interval < 1)
  967. interval = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER;
  968. cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * interval);
  969. raw_spin_lock_init(&cpuctx->hrtimer_lock);
  970. hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD);
  971. timer->function = perf_mux_hrtimer_handler;
  972. }
  973. static int perf_mux_hrtimer_restart(struct perf_cpu_context *cpuctx)
  974. {
  975. struct hrtimer *timer = &cpuctx->hrtimer;
  976. struct pmu *pmu = cpuctx->ctx.pmu;
  977. unsigned long flags;
  978. /* not for SW PMU */
  979. if (pmu->task_ctx_nr == perf_sw_context)
  980. return 0;
  981. raw_spin_lock_irqsave(&cpuctx->hrtimer_lock, flags);
  982. if (!cpuctx->hrtimer_active) {
  983. cpuctx->hrtimer_active = 1;
  984. hrtimer_forward_now(timer, cpuctx->hrtimer_interval);
  985. hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED_HARD);
  986. }
  987. raw_spin_unlock_irqrestore(&cpuctx->hrtimer_lock, flags);
  988. return 0;
  989. }
  990. void perf_pmu_disable(struct pmu *pmu)
  991. {
  992. int *count = this_cpu_ptr(pmu->pmu_disable_count);
  993. if (!(*count)++)
  994. pmu->pmu_disable(pmu);
  995. }
  996. void perf_pmu_enable(struct pmu *pmu)
  997. {
  998. int *count = this_cpu_ptr(pmu->pmu_disable_count);
  999. if (!--(*count))
  1000. pmu->pmu_enable(pmu);
  1001. }
  1002. static DEFINE_PER_CPU(struct list_head, active_ctx_list);
  1003. /*
  1004. * perf_event_ctx_activate(), perf_event_ctx_deactivate(), and
  1005. * perf_event_task_tick() are fully serialized because they're strictly cpu
  1006. * affine and perf_event_ctx{activate,deactivate} are called with IRQs
  1007. * disabled, while perf_event_task_tick is called from IRQ context.
  1008. */
  1009. static void perf_event_ctx_activate(struct perf_event_context *ctx)
  1010. {
  1011. struct list_head *head = this_cpu_ptr(&active_ctx_list);
  1012. lockdep_assert_irqs_disabled();
  1013. WARN_ON(!list_empty(&ctx->active_ctx_list));
  1014. list_add(&ctx->active_ctx_list, head);
  1015. }
  1016. static void perf_event_ctx_deactivate(struct perf_event_context *ctx)
  1017. {
  1018. lockdep_assert_irqs_disabled();
  1019. WARN_ON(list_empty(&ctx->active_ctx_list));
  1020. list_del_init(&ctx->active_ctx_list);
  1021. }
  1022. static void get_ctx(struct perf_event_context *ctx)
  1023. {
  1024. refcount_inc(&ctx->refcount);
  1025. }
  1026. static void *alloc_task_ctx_data(struct pmu *pmu)
  1027. {
  1028. if (pmu->task_ctx_cache)
  1029. return kmem_cache_zalloc(pmu->task_ctx_cache, GFP_KERNEL);
  1030. return NULL;
  1031. }
  1032. static void free_task_ctx_data(struct pmu *pmu, void *task_ctx_data)
  1033. {
  1034. if (pmu->task_ctx_cache && task_ctx_data)
  1035. kmem_cache_free(pmu->task_ctx_cache, task_ctx_data);
  1036. }
  1037. static void free_ctx(struct rcu_head *head)
  1038. {
  1039. struct perf_event_context *ctx;
  1040. ctx = container_of(head, struct perf_event_context, rcu_head);
  1041. free_task_ctx_data(ctx->pmu, ctx->task_ctx_data);
  1042. kfree(ctx);
  1043. }
  1044. static void put_ctx(struct perf_event_context *ctx)
  1045. {
  1046. if (refcount_dec_and_test(&ctx->refcount)) {
  1047. if (ctx->parent_ctx)
  1048. put_ctx(ctx->parent_ctx);
  1049. if (ctx->task && ctx->task != TASK_TOMBSTONE)
  1050. put_task_struct(ctx->task);
  1051. call_rcu(&ctx->rcu_head, free_ctx);
  1052. }
  1053. }
  1054. /*
  1055. * Because of perf_event::ctx migration in sys_perf_event_open::move_group and
  1056. * perf_pmu_migrate_context() we need some magic.
  1057. *
  1058. * Those places that change perf_event::ctx will hold both
  1059. * perf_event_ctx::mutex of the 'old' and 'new' ctx value.
  1060. *
  1061. * Lock ordering is by mutex address. There are two other sites where
  1062. * perf_event_context::mutex nests and those are:
  1063. *
  1064. * - perf_event_exit_task_context() [ child , 0 ]
  1065. * perf_event_exit_event()
  1066. * put_event() [ parent, 1 ]
  1067. *
  1068. * - perf_event_init_context() [ parent, 0 ]
  1069. * inherit_task_group()
  1070. * inherit_group()
  1071. * inherit_event()
  1072. * perf_event_alloc()
  1073. * perf_init_event()
  1074. * perf_try_init_event() [ child , 1 ]
  1075. *
  1076. * While it appears there is an obvious deadlock here -- the parent and child
  1077. * nesting levels are inverted between the two. This is in fact safe because
  1078. * life-time rules separate them. That is an exiting task cannot fork, and a
  1079. * spawning task cannot (yet) exit.
  1080. *
  1081. * But remember that these are parent<->child context relations, and
  1082. * migration does not affect children, therefore these two orderings should not
  1083. * interact.
  1084. *
  1085. * The change in perf_event::ctx does not affect children (as claimed above)
  1086. * because the sys_perf_event_open() case will install a new event and break
  1087. * the ctx parent<->child relation, and perf_pmu_migrate_context() is only
  1088. * concerned with cpuctx and that doesn't have children.
  1089. *
  1090. * The places that change perf_event::ctx will issue:
  1091. *
  1092. * perf_remove_from_context();
  1093. * synchronize_rcu();
  1094. * perf_install_in_context();
  1095. *
  1096. * to affect the change. The remove_from_context() + synchronize_rcu() should
  1097. * quiesce the event, after which we can install it in the new location. This
  1098. * means that only external vectors (perf_fops, prctl) can perturb the event
  1099. * while in transit. Therefore all such accessors should also acquire
  1100. * perf_event_context::mutex to serialize against this.
  1101. *
  1102. * However; because event->ctx can change while we're waiting to acquire
  1103. * ctx->mutex we must be careful and use the below perf_event_ctx_lock()
  1104. * function.
  1105. *
  1106. * Lock order:
  1107. * exec_update_lock
  1108. * task_struct::perf_event_mutex
  1109. * perf_event_context::mutex
  1110. * perf_event::child_mutex;
  1111. * perf_event_context::lock
  1112. * perf_event::mmap_mutex
  1113. * mmap_lock
  1114. * perf_addr_filters_head::lock
  1115. *
  1116. * cpu_hotplug_lock
  1117. * pmus_lock
  1118. * cpuctx->mutex / perf_event_context::mutex
  1119. */
  1120. static struct perf_event_context *
  1121. perf_event_ctx_lock_nested(struct perf_event *event, int nesting)
  1122. {
  1123. struct perf_event_context *ctx;
  1124. again:
  1125. rcu_read_lock();
  1126. ctx = READ_ONCE(event->ctx);
  1127. if (!refcount_inc_not_zero(&ctx->refcount)) {
  1128. rcu_read_unlock();
  1129. goto again;
  1130. }
  1131. rcu_read_unlock();
  1132. mutex_lock_nested(&ctx->mutex, nesting);
  1133. if (event->ctx != ctx) {
  1134. mutex_unlock(&ctx->mutex);
  1135. put_ctx(ctx);
  1136. goto again;
  1137. }
  1138. return ctx;
  1139. }
  1140. static inline struct perf_event_context *
  1141. perf_event_ctx_lock(struct perf_event *event)
  1142. {
  1143. return perf_event_ctx_lock_nested(event, 0);
  1144. }
  1145. static void perf_event_ctx_unlock(struct perf_event *event,
  1146. struct perf_event_context *ctx)
  1147. {
  1148. mutex_unlock(&ctx->mutex);
  1149. put_ctx(ctx);
  1150. }
  1151. /*
  1152. * This must be done under the ctx->lock, such as to serialize against
  1153. * context_equiv(), therefore we cannot call put_ctx() since that might end up
  1154. * calling scheduler related locks and ctx->lock nests inside those.
  1155. */
  1156. static __must_check struct perf_event_context *
  1157. unclone_ctx(struct perf_event_context *ctx)
  1158. {
  1159. struct perf_event_context *parent_ctx = ctx->parent_ctx;
  1160. lockdep_assert_held(&ctx->lock);
  1161. if (parent_ctx)
  1162. ctx->parent_ctx = NULL;
  1163. ctx->generation++;
  1164. return parent_ctx;
  1165. }
  1166. static u32 perf_event_pid_type(struct perf_event *event, struct task_struct *p,
  1167. enum pid_type type)
  1168. {
  1169. u32 nr;
  1170. /*
  1171. * only top level events have the pid namespace they were created in
  1172. */
  1173. if (event->parent)
  1174. event = event->parent;
  1175. nr = __task_pid_nr_ns(p, type, event->ns);
  1176. /* avoid -1 if it is idle thread or runs in another ns */
  1177. if (!nr && !pid_alive(p))
  1178. nr = -1;
  1179. return nr;
  1180. }
  1181. static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
  1182. {
  1183. return perf_event_pid_type(event, p, PIDTYPE_TGID);
  1184. }
  1185. static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
  1186. {
  1187. return perf_event_pid_type(event, p, PIDTYPE_PID);
  1188. }
  1189. /*
  1190. * If we inherit events we want to return the parent event id
  1191. * to userspace.
  1192. */
  1193. static u64 primary_event_id(struct perf_event *event)
  1194. {
  1195. u64 id = event->id;
  1196. if (event->parent)
  1197. id = event->parent->id;
  1198. return id;
  1199. }
  1200. /*
  1201. * Get the perf_event_context for a task and lock it.
  1202. *
  1203. * This has to cope with the fact that until it is locked,
  1204. * the context could get moved to another task.
  1205. */
  1206. static struct perf_event_context *
  1207. perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags)
  1208. {
  1209. struct perf_event_context *ctx;
  1210. retry:
  1211. /*
  1212. * One of the few rules of preemptible RCU is that one cannot do
  1213. * rcu_read_unlock() while holding a scheduler (or nested) lock when
  1214. * part of the read side critical section was irqs-enabled -- see
  1215. * rcu_read_unlock_special().
  1216. *
  1217. * Since ctx->lock nests under rq->lock we must ensure the entire read
  1218. * side critical section has interrupts disabled.
  1219. */
  1220. local_irq_save(*flags);
  1221. rcu_read_lock();
  1222. ctx = rcu_dereference(task->perf_event_ctxp[ctxn]);
  1223. if (ctx) {
  1224. /*
  1225. * If this context is a clone of another, it might
  1226. * get swapped for another underneath us by
  1227. * perf_event_task_sched_out, though the
  1228. * rcu_read_lock() protects us from any context
  1229. * getting freed. Lock the context and check if it
  1230. * got swapped before we could get the lock, and retry
  1231. * if so. If we locked the right context, then it
  1232. * can't get swapped on us any more.
  1233. */
  1234. raw_spin_lock(&ctx->lock);
  1235. if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) {
  1236. raw_spin_unlock(&ctx->lock);
  1237. rcu_read_unlock();
  1238. local_irq_restore(*flags);
  1239. goto retry;
  1240. }
  1241. if (ctx->task == TASK_TOMBSTONE ||
  1242. !refcount_inc_not_zero(&ctx->refcount)) {
  1243. raw_spin_unlock(&ctx->lock);
  1244. ctx = NULL;
  1245. } else {
  1246. WARN_ON_ONCE(ctx->task != task);
  1247. }
  1248. }
  1249. rcu_read_unlock();
  1250. if (!ctx)
  1251. local_irq_restore(*flags);
  1252. return ctx;
  1253. }
  1254. /*
  1255. * Get the context for a task and increment its pin_count so it
  1256. * can't get swapped to another task. This also increments its
  1257. * reference count so that the context can't get freed.
  1258. */
  1259. static struct perf_event_context *
  1260. perf_pin_task_context(struct task_struct *task, int ctxn)
  1261. {
  1262. struct perf_event_context *ctx;
  1263. unsigned long flags;
  1264. ctx = perf_lock_task_context(task, ctxn, &flags);
  1265. if (ctx) {
  1266. ++ctx->pin_count;
  1267. raw_spin_unlock_irqrestore(&ctx->lock, flags);
  1268. }
  1269. return ctx;
  1270. }
  1271. static void perf_unpin_context(struct perf_event_context *ctx)
  1272. {
  1273. unsigned long flags;
  1274. raw_spin_lock_irqsave(&ctx->lock, flags);
  1275. --ctx->pin_count;
  1276. raw_spin_unlock_irqrestore(&ctx->lock, flags);
  1277. }
  1278. /*
  1279. * Update the record of the current time in a context.
  1280. */
  1281. static void update_context_time(struct perf_event_context *ctx)
  1282. {
  1283. u64 now = perf_clock();
  1284. ctx->time += now - ctx->timestamp;
  1285. ctx->timestamp = now;
  1286. }
  1287. static u64 perf_event_time(struct perf_event *event)
  1288. {
  1289. struct perf_event_context *ctx = event->ctx;
  1290. if (is_cgroup_event(event))
  1291. return perf_cgroup_event_time(event);
  1292. return ctx ? ctx->time : 0;
  1293. }
  1294. static enum event_type_t get_event_type(struct perf_event *event)
  1295. {
  1296. struct perf_event_context *ctx = event->ctx;
  1297. enum event_type_t event_type;
  1298. lockdep_assert_held(&ctx->lock);
  1299. /*
  1300. * It's 'group type', really, because if our group leader is
  1301. * pinned, so are we.
  1302. */
  1303. if (event->group_leader != event)
  1304. event = event->group_leader;
  1305. event_type = event->attr.pinned ? EVENT_PINNED : EVENT_FLEXIBLE;
  1306. if (!ctx->task)
  1307. event_type |= EVENT_CPU;
  1308. return event_type;
  1309. }
  1310. /*
  1311. * Helper function to initialize event group nodes.
  1312. */
  1313. static void init_event_group(struct perf_event *event)
  1314. {
  1315. RB_CLEAR_NODE(&event->group_node);
  1316. event->group_index = 0;
  1317. }
  1318. /*
  1319. * Extract pinned or flexible groups from the context
  1320. * based on event attrs bits.
  1321. */
  1322. static struct perf_event_groups *
  1323. get_event_groups(struct perf_event *event, struct perf_event_context *ctx)
  1324. {
  1325. if (event->attr.pinned)
  1326. return &ctx->pinned_groups;
  1327. else
  1328. return &ctx->flexible_groups;
  1329. }
  1330. /*
  1331. * Helper function to initializes perf_event_group trees.
  1332. */
  1333. static void perf_event_groups_init(struct perf_event_groups *groups)
  1334. {
  1335. groups->tree = RB_ROOT;
  1336. groups->index = 0;
  1337. }
  1338. /*
  1339. * Compare function for event groups;
  1340. *
  1341. * Implements complex key that first sorts by CPU and then by virtual index
  1342. * which provides ordering when rotating groups for the same CPU.
  1343. */
  1344. static bool
  1345. perf_event_groups_less(struct perf_event *left, struct perf_event *right)
  1346. {
  1347. if (left->cpu < right->cpu)
  1348. return true;
  1349. if (left->cpu > right->cpu)
  1350. return false;
  1351. #ifdef CONFIG_CGROUP_PERF
  1352. if (left->cgrp != right->cgrp) {
  1353. if (!left->cgrp || !left->cgrp->css.cgroup) {
  1354. /*
  1355. * Left has no cgroup but right does, no cgroups come
  1356. * first.
  1357. */
  1358. return true;
  1359. }
  1360. if (!right->cgrp || !right->cgrp->css.cgroup) {
  1361. /*
  1362. * Right has no cgroup but left does, no cgroups come
  1363. * first.
  1364. */
  1365. return false;
  1366. }
  1367. /* Two dissimilar cgroups, order by id. */
  1368. if (left->cgrp->css.cgroup->kn->id < right->cgrp->css.cgroup->kn->id)
  1369. return true;
  1370. return false;
  1371. }
  1372. #endif
  1373. if (left->group_index < right->group_index)
  1374. return true;
  1375. if (left->group_index > right->group_index)
  1376. return false;
  1377. return false;
  1378. }
  1379. /*
  1380. * Insert @event into @groups' tree; using {@event->cpu, ++@groups->index} for
  1381. * key (see perf_event_groups_less). This places it last inside the CPU
  1382. * subtree.
  1383. */
  1384. static void
  1385. perf_event_groups_insert(struct perf_event_groups *groups,
  1386. struct perf_event *event)
  1387. {
  1388. struct perf_event *node_event;
  1389. struct rb_node *parent;
  1390. struct rb_node **node;
  1391. event->group_index = ++groups->index;
  1392. node = &groups->tree.rb_node;
  1393. parent = *node;
  1394. while (*node) {
  1395. parent = *node;
  1396. node_event = container_of(*node, struct perf_event, group_node);
  1397. if (perf_event_groups_less(event, node_event))
  1398. node = &parent->rb_left;
  1399. else
  1400. node = &parent->rb_right;
  1401. }
  1402. rb_link_node(&event->group_node, parent, node);
  1403. rb_insert_color(&event->group_node, &groups->tree);
  1404. }
  1405. /*
  1406. * Helper function to insert event into the pinned or flexible groups.
  1407. */
  1408. static void
  1409. add_event_to_groups(struct perf_event *event, struct perf_event_context *ctx)
  1410. {
  1411. struct perf_event_groups *groups;
  1412. groups = get_event_groups(event, ctx);
  1413. perf_event_groups_insert(groups, event);
  1414. }
  1415. /*
  1416. * Delete a group from a tree.
  1417. */
  1418. static void
  1419. perf_event_groups_delete(struct perf_event_groups *groups,
  1420. struct perf_event *event)
  1421. {
  1422. WARN_ON_ONCE(RB_EMPTY_NODE(&event->group_node) ||
  1423. RB_EMPTY_ROOT(&groups->tree));
  1424. rb_erase(&event->group_node, &groups->tree);
  1425. init_event_group(event);
  1426. }
  1427. /*
  1428. * Helper function to delete event from its groups.
  1429. */
  1430. static void
  1431. del_event_from_groups(struct perf_event *event, struct perf_event_context *ctx)
  1432. {
  1433. struct perf_event_groups *groups;
  1434. groups = get_event_groups(event, ctx);
  1435. perf_event_groups_delete(groups, event);
  1436. }
  1437. /*
  1438. * Get the leftmost event in the cpu/cgroup subtree.
  1439. */
  1440. static struct perf_event *
  1441. perf_event_groups_first(struct perf_event_groups *groups, int cpu,
  1442. struct cgroup *cgrp)
  1443. {
  1444. struct perf_event *node_event = NULL, *match = NULL;
  1445. struct rb_node *node = groups->tree.rb_node;
  1446. #ifdef CONFIG_CGROUP_PERF
  1447. u64 node_cgrp_id, cgrp_id = 0;
  1448. if (cgrp)
  1449. cgrp_id = cgrp->kn->id;
  1450. #endif
  1451. while (node) {
  1452. node_event = container_of(node, struct perf_event, group_node);
  1453. if (cpu < node_event->cpu) {
  1454. node = node->rb_left;
  1455. continue;
  1456. }
  1457. if (cpu > node_event->cpu) {
  1458. node = node->rb_right;
  1459. continue;
  1460. }
  1461. #ifdef CONFIG_CGROUP_PERF
  1462. node_cgrp_id = 0;
  1463. if (node_event->cgrp && node_event->cgrp->css.cgroup)
  1464. node_cgrp_id = node_event->cgrp->css.cgroup->kn->id;
  1465. if (cgrp_id < node_cgrp_id) {
  1466. node = node->rb_left;
  1467. continue;
  1468. }
  1469. if (cgrp_id > node_cgrp_id) {
  1470. node = node->rb_right;
  1471. continue;
  1472. }
  1473. #endif
  1474. match = node_event;
  1475. node = node->rb_left;
  1476. }
  1477. return match;
  1478. }
  1479. /*
  1480. * Like rb_entry_next_safe() for the @cpu subtree.
  1481. */
  1482. static struct perf_event *
  1483. perf_event_groups_next(struct perf_event *event)
  1484. {
  1485. struct perf_event *next;
  1486. #ifdef CONFIG_CGROUP_PERF
  1487. u64 curr_cgrp_id = 0;
  1488. u64 next_cgrp_id = 0;
  1489. #endif
  1490. next = rb_entry_safe(rb_next(&event->group_node), typeof(*event), group_node);
  1491. if (next == NULL || next->cpu != event->cpu)
  1492. return NULL;
  1493. #ifdef CONFIG_CGROUP_PERF
  1494. if (event->cgrp && event->cgrp->css.cgroup)
  1495. curr_cgrp_id = event->cgrp->css.cgroup->kn->id;
  1496. if (next->cgrp && next->cgrp->css.cgroup)
  1497. next_cgrp_id = next->cgrp->css.cgroup->kn->id;
  1498. if (curr_cgrp_id != next_cgrp_id)
  1499. return NULL;
  1500. #endif
  1501. return next;
  1502. }
  1503. /*
  1504. * Iterate through the whole groups tree.
  1505. */
  1506. #define perf_event_groups_for_each(event, groups) \
  1507. for (event = rb_entry_safe(rb_first(&((groups)->tree)), \
  1508. typeof(*event), group_node); event; \
  1509. event = rb_entry_safe(rb_next(&event->group_node), \
  1510. typeof(*event), group_node))
  1511. /*
  1512. * Add an event from the lists for its context.
  1513. * Must be called with ctx->mutex and ctx->lock held.
  1514. */
  1515. static void
  1516. list_add_event(struct perf_event *event, struct perf_event_context *ctx)
  1517. {
  1518. lockdep_assert_held(&ctx->lock);
  1519. WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
  1520. event->attach_state |= PERF_ATTACH_CONTEXT;
  1521. event->tstamp = perf_event_time(event);
  1522. /*
  1523. * If we're a stand alone event or group leader, we go to the context
  1524. * list, group events are kept attached to the group so that
  1525. * perf_group_detach can, at all times, locate all siblings.
  1526. */
  1527. if (event->group_leader == event) {
  1528. event->group_caps = event->event_caps;
  1529. add_event_to_groups(event, ctx);
  1530. }
  1531. list_add_rcu(&event->event_entry, &ctx->event_list);
  1532. ctx->nr_events++;
  1533. if (event->attr.inherit_stat)
  1534. ctx->nr_stat++;
  1535. if (event->state > PERF_EVENT_STATE_OFF)
  1536. perf_cgroup_event_enable(event, ctx);
  1537. ctx->generation++;
  1538. }
  1539. /*
  1540. * Initialize event state based on the perf_event_attr::disabled.
  1541. */
  1542. static inline void perf_event__state_init(struct perf_event *event)
  1543. {
  1544. event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF :
  1545. PERF_EVENT_STATE_INACTIVE;
  1546. }
  1547. static void __perf_event_read_size(struct perf_event *event, int nr_siblings)
  1548. {
  1549. int entry = sizeof(u64); /* value */
  1550. int size = 0;
  1551. int nr = 1;
  1552. if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
  1553. size += sizeof(u64);
  1554. if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
  1555. size += sizeof(u64);
  1556. if (event->attr.read_format & PERF_FORMAT_ID)
  1557. entry += sizeof(u64);
  1558. if (event->attr.read_format & PERF_FORMAT_GROUP) {
  1559. nr += nr_siblings;
  1560. size += sizeof(u64);
  1561. }
  1562. size += entry * nr;
  1563. event->read_size = size;
  1564. }
  1565. static void __perf_event_header_size(struct perf_event *event, u64 sample_type)
  1566. {
  1567. struct perf_sample_data *data;
  1568. u16 size = 0;
  1569. if (sample_type & PERF_SAMPLE_IP)
  1570. size += sizeof(data->ip);
  1571. if (sample_type & PERF_SAMPLE_ADDR)
  1572. size += sizeof(data->addr);
  1573. if (sample_type & PERF_SAMPLE_PERIOD)
  1574. size += sizeof(data->period);
  1575. if (sample_type & PERF_SAMPLE_WEIGHT)
  1576. size += sizeof(data->weight);
  1577. if (sample_type & PERF_SAMPLE_READ)
  1578. size += event->read_size;
  1579. if (sample_type & PERF_SAMPLE_DATA_SRC)
  1580. size += sizeof(data->data_src.val);
  1581. if (sample_type & PERF_SAMPLE_TRANSACTION)
  1582. size += sizeof(data->txn);
  1583. if (sample_type & PERF_SAMPLE_PHYS_ADDR)
  1584. size += sizeof(data->phys_addr);
  1585. if (sample_type & PERF_SAMPLE_CGROUP)
  1586. size += sizeof(data->cgroup);
  1587. event->header_size = size;
  1588. }
  1589. /*
  1590. * Called at perf_event creation and when events are attached/detached from a
  1591. * group.
  1592. */
  1593. static void perf_event__header_size(struct perf_event *event)
  1594. {
  1595. __perf_event_read_size(event,
  1596. event->group_leader->nr_siblings);
  1597. __perf_event_header_size(event, event->attr.sample_type);
  1598. }
  1599. static void perf_event__id_header_size(struct perf_event *event)
  1600. {
  1601. struct perf_sample_data *data;
  1602. u64 sample_type = event->attr.sample_type;
  1603. u16 size = 0;
  1604. if (sample_type & PERF_SAMPLE_TID)
  1605. size += sizeof(data->tid_entry);
  1606. if (sample_type & PERF_SAMPLE_TIME)
  1607. size += sizeof(data->time);
  1608. if (sample_type & PERF_SAMPLE_IDENTIFIER)
  1609. size += sizeof(data->id);
  1610. if (sample_type & PERF_SAMPLE_ID)
  1611. size += sizeof(data->id);
  1612. if (sample_type & PERF_SAMPLE_STREAM_ID)
  1613. size += sizeof(data->stream_id);
  1614. if (sample_type & PERF_SAMPLE_CPU)
  1615. size += sizeof(data->cpu_entry);
  1616. event->id_header_size = size;
  1617. }
  1618. static bool perf_event_validate_size(struct perf_event *event)
  1619. {
  1620. /*
  1621. * The values computed here will be over-written when we actually
  1622. * attach the event.
  1623. */
  1624. __perf_event_read_size(event, event->group_leader->nr_siblings + 1);
  1625. __perf_event_header_size(event, event->attr.sample_type & ~PERF_SAMPLE_READ);
  1626. perf_event__id_header_size(event);
  1627. /*
  1628. * Sum the lot; should not exceed the 64k limit we have on records.
  1629. * Conservative limit to allow for callchains and other variable fields.
  1630. */
  1631. if (event->read_size + event->header_size +
  1632. event->id_header_size + sizeof(struct perf_event_header) >= 16*1024)
  1633. return false;
  1634. return true;
  1635. }
  1636. static void perf_group_attach(struct perf_event *event)
  1637. {
  1638. struct perf_event *group_leader = event->group_leader, *pos;
  1639. lockdep_assert_held(&event->ctx->lock);
  1640. /*
  1641. * We can have double attach due to group movement in perf_event_open.
  1642. */
  1643. if (event->attach_state & PERF_ATTACH_GROUP)
  1644. return;
  1645. event->attach_state |= PERF_ATTACH_GROUP;
  1646. if (group_leader == event)
  1647. return;
  1648. WARN_ON_ONCE(group_leader->ctx != event->ctx);
  1649. group_leader->group_caps &= event->event_caps;
  1650. list_add_tail(&event->sibling_list, &group_leader->sibling_list);
  1651. group_leader->nr_siblings++;
  1652. perf_event__header_size(group_leader);
  1653. for_each_sibling_event(pos, group_leader)
  1654. perf_event__header_size(pos);
  1655. }
  1656. /*
  1657. * Remove an event from the lists for its context.
  1658. * Must be called with ctx->mutex and ctx->lock held.
  1659. */
  1660. static void
  1661. list_del_event(struct perf_event *event, struct perf_event_context *ctx)
  1662. {
  1663. WARN_ON_ONCE(event->ctx != ctx);
  1664. lockdep_assert_held(&ctx->lock);
  1665. /*
  1666. * We can have double detach due to exit/hot-unplug + close.
  1667. */
  1668. if (!(event->attach_state & PERF_ATTACH_CONTEXT))
  1669. return;
  1670. event->attach_state &= ~PERF_ATTACH_CONTEXT;
  1671. ctx->nr_events--;
  1672. if (event->attr.inherit_stat)
  1673. ctx->nr_stat--;
  1674. list_del_rcu(&event->event_entry);
  1675. if (event->group_leader == event)
  1676. del_event_from_groups(event, ctx);
  1677. /*
  1678. * If event was in error state, then keep it
  1679. * that way, otherwise bogus counts will be
  1680. * returned on read(). The only way to get out
  1681. * of error state is by explicit re-enabling
  1682. * of the event
  1683. */
  1684. if (event->state > PERF_EVENT_STATE_OFF) {
  1685. perf_cgroup_event_disable(event, ctx);
  1686. perf_event_set_state(event, PERF_EVENT_STATE_OFF);
  1687. }
  1688. ctx->generation++;
  1689. }
  1690. static int
  1691. perf_aux_output_match(struct perf_event *event, struct perf_event *aux_event)
  1692. {
  1693. if (!has_aux(aux_event))
  1694. return 0;
  1695. if (!event->pmu->aux_output_match)
  1696. return 0;
  1697. return event->pmu->aux_output_match(aux_event);
  1698. }
  1699. static void put_event(struct perf_event *event);
  1700. static void event_sched_out(struct perf_event *event,
  1701. struct perf_cpu_context *cpuctx,
  1702. struct perf_event_context *ctx);
  1703. static void perf_put_aux_event(struct perf_event *event)
  1704. {
  1705. struct perf_event_context *ctx = event->ctx;
  1706. struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
  1707. struct perf_event *iter;
  1708. /*
  1709. * If event uses aux_event tear down the link
  1710. */
  1711. if (event->aux_event) {
  1712. iter = event->aux_event;
  1713. event->aux_event = NULL;
  1714. put_event(iter);
  1715. return;
  1716. }
  1717. /*
  1718. * If the event is an aux_event, tear down all links to
  1719. * it from other events.
  1720. */
  1721. for_each_sibling_event(iter, event->group_leader) {
  1722. if (iter->aux_event != event)
  1723. continue;
  1724. iter->aux_event = NULL;
  1725. put_event(event);
  1726. /*
  1727. * If it's ACTIVE, schedule it out and put it into ERROR
  1728. * state so that we don't try to schedule it again. Note
  1729. * that perf_event_enable() will clear the ERROR status.
  1730. */
  1731. event_sched_out(iter, cpuctx, ctx);
  1732. perf_event_set_state(event, PERF_EVENT_STATE_ERROR);
  1733. }
  1734. }
  1735. static bool perf_need_aux_event(struct perf_event *event)
  1736. {
  1737. return !!event->attr.aux_output || !!event->attr.aux_sample_size;
  1738. }
  1739. static int perf_get_aux_event(struct perf_event *event,
  1740. struct perf_event *group_leader)
  1741. {
  1742. /*
  1743. * Our group leader must be an aux event if we want to be
  1744. * an aux_output. This way, the aux event will precede its
  1745. * aux_output events in the group, and therefore will always
  1746. * schedule first.
  1747. */
  1748. if (!group_leader)
  1749. return 0;
  1750. /*
  1751. * aux_output and aux_sample_size are mutually exclusive.
  1752. */
  1753. if (event->attr.aux_output && event->attr.aux_sample_size)
  1754. return 0;
  1755. if (event->attr.aux_output &&
  1756. !perf_aux_output_match(event, group_leader))
  1757. return 0;
  1758. if (event->attr.aux_sample_size && !group_leader->pmu->snapshot_aux)
  1759. return 0;
  1760. if (!atomic_long_inc_not_zero(&group_leader->refcount))
  1761. return 0;
  1762. /*
  1763. * Link aux_outputs to their aux event; this is undone in
  1764. * perf_group_detach() by perf_put_aux_event(). When the
  1765. * group in torn down, the aux_output events loose their
  1766. * link to the aux_event and can't schedule any more.
  1767. */
  1768. event->aux_event = group_leader;
  1769. return 1;
  1770. }
  1771. static inline struct list_head *get_event_list(struct perf_event *event)
  1772. {
  1773. struct perf_event_context *ctx = event->ctx;
  1774. return event->attr.pinned ? &ctx->pinned_active : &ctx->flexible_active;
  1775. }
  1776. /*
  1777. * Events that have PERF_EV_CAP_SIBLING require being part of a group and
  1778. * cannot exist on their own, schedule them out and move them into the ERROR
  1779. * state. Also see _perf_event_enable(), it will not be able to recover
  1780. * this ERROR state.
  1781. */
  1782. static inline void perf_remove_sibling_event(struct perf_event *event)
  1783. {
  1784. struct perf_event_context *ctx = event->ctx;
  1785. struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
  1786. event_sched_out(event, cpuctx, ctx);
  1787. perf_event_set_state(event, PERF_EVENT_STATE_ERROR);
  1788. }
  1789. static void perf_group_detach(struct perf_event *event)
  1790. {
  1791. struct perf_event *leader = event->group_leader;
  1792. struct perf_event *sibling, *tmp;
  1793. struct perf_event_context *ctx = event->ctx;
  1794. lockdep_assert_held(&ctx->lock);
  1795. /*
  1796. * We can have double detach due to exit/hot-unplug + close.
  1797. */
  1798. if (!(event->attach_state & PERF_ATTACH_GROUP))
  1799. return;
  1800. event->attach_state &= ~PERF_ATTACH_GROUP;
  1801. perf_put_aux_event(event);
  1802. /*
  1803. * If this is a sibling, remove it from its group.
  1804. */
  1805. if (leader != event) {
  1806. list_del_init(&event->sibling_list);
  1807. event->group_leader->nr_siblings--;
  1808. goto out;
  1809. }
  1810. /*
  1811. * If this was a group event with sibling events then
  1812. * upgrade the siblings to singleton events by adding them
  1813. * to whatever list we are on.
  1814. */
  1815. list_for_each_entry_safe(sibling, tmp, &event->sibling_list, sibling_list) {
  1816. if (sibling->event_caps & PERF_EV_CAP_SIBLING)
  1817. perf_remove_sibling_event(sibling);
  1818. sibling->group_leader = sibling;
  1819. list_del_init(&sibling->sibling_list);
  1820. /* Inherit group flags from the previous leader */
  1821. sibling->group_caps = event->group_caps;
  1822. if (!RB_EMPTY_NODE(&event->group_node)) {
  1823. add_event_to_groups(sibling, event->ctx);
  1824. if (sibling->state == PERF_EVENT_STATE_ACTIVE)
  1825. list_add_tail(&sibling->active_list, get_event_list(sibling));
  1826. }
  1827. WARN_ON_ONCE(sibling->ctx != event->ctx);
  1828. }
  1829. out:
  1830. for_each_sibling_event(tmp, leader)
  1831. perf_event__header_size(tmp);
  1832. perf_event__header_size(leader);
  1833. }
  1834. static bool is_orphaned_event(struct perf_event *event)
  1835. {
  1836. return event->state == PERF_EVENT_STATE_DEAD;
  1837. }
  1838. static inline int __pmu_filter_match(struct perf_event *event)
  1839. {
  1840. struct pmu *pmu = event->pmu;
  1841. return pmu->filter_match ? pmu->filter_match(event) : 1;
  1842. }
  1843. /*
  1844. * Check whether we should attempt to schedule an event group based on
  1845. * PMU-specific filtering. An event group can consist of HW and SW events,
  1846. * potentially with a SW leader, so we must check all the filters, to
  1847. * determine whether a group is schedulable:
  1848. */
  1849. static inline int pmu_filter_match(struct perf_event *event)
  1850. {
  1851. struct perf_event *sibling;
  1852. if (!__pmu_filter_match(event))
  1853. return 0;
  1854. for_each_sibling_event(sibling, event) {
  1855. if (!__pmu_filter_match(sibling))
  1856. return 0;
  1857. }
  1858. return 1;
  1859. }
  1860. static inline int
  1861. event_filter_match(struct perf_event *event)
  1862. {
  1863. return (event->cpu == -1 || event->cpu == smp_processor_id()) &&
  1864. perf_cgroup_match(event) && pmu_filter_match(event);
  1865. }
  1866. static void
  1867. event_sched_out(struct perf_event *event,
  1868. struct perf_cpu_context *cpuctx,
  1869. struct perf_event_context *ctx)
  1870. {
  1871. enum perf_event_state state = PERF_EVENT_STATE_INACTIVE;
  1872. WARN_ON_ONCE(event->ctx != ctx);
  1873. lockdep_assert_held(&ctx->lock);
  1874. if (event->state != PERF_EVENT_STATE_ACTIVE)
  1875. return;
  1876. /*
  1877. * Asymmetry; we only schedule events _IN_ through ctx_sched_in(), but
  1878. * we can schedule events _OUT_ individually through things like
  1879. * __perf_remove_from_context().
  1880. */
  1881. list_del_init(&event->active_list);
  1882. perf_pmu_disable(event->pmu);
  1883. event->pmu->del(event, 0);
  1884. event->oncpu = -1;
  1885. if (READ_ONCE(event->pending_disable) >= 0) {
  1886. WRITE_ONCE(event->pending_disable, -1);
  1887. perf_cgroup_event_disable(event, ctx);
  1888. state = PERF_EVENT_STATE_OFF;
  1889. }
  1890. perf_event_set_state(event, state);
  1891. if (!is_software_event(event))
  1892. cpuctx->active_oncpu--;
  1893. if (!--ctx->nr_active)
  1894. perf_event_ctx_deactivate(ctx);
  1895. if (event->attr.freq && event->attr.sample_freq)
  1896. ctx->nr_freq--;
  1897. if (event->attr.exclusive || !cpuctx->active_oncpu)
  1898. cpuctx->exclusive = 0;
  1899. perf_pmu_enable(event->pmu);
  1900. }
  1901. static void
  1902. group_sched_out(struct perf_event *group_event,
  1903. struct perf_cpu_context *cpuctx,
  1904. struct perf_event_context *ctx)
  1905. {
  1906. struct perf_event *event;
  1907. if (group_event->state != PERF_EVENT_STATE_ACTIVE)
  1908. return;
  1909. perf_pmu_disable(ctx->pmu);
  1910. event_sched_out(group_event, cpuctx, ctx);
  1911. /*
  1912. * Schedule out siblings (if any):
  1913. */
  1914. for_each_sibling_event(event, group_event)
  1915. event_sched_out(event, cpuctx, ctx);
  1916. perf_pmu_enable(ctx->pmu);
  1917. }
  1918. #define DETACH_GROUP 0x01UL
  1919. /*
  1920. * Cross CPU call to remove a performance event
  1921. *
  1922. * We disable the event on the hardware level first. After that we
  1923. * remove it from the context list.
  1924. */
  1925. static void
  1926. __perf_remove_from_context(struct perf_event *event,
  1927. struct perf_cpu_context *cpuctx,
  1928. struct perf_event_context *ctx,
  1929. void *info)
  1930. {
  1931. unsigned long flags = (unsigned long)info;
  1932. if (ctx->is_active & EVENT_TIME) {
  1933. update_context_time(ctx);
  1934. update_cgrp_time_from_cpuctx(cpuctx);
  1935. }
  1936. event_sched_out(event, cpuctx, ctx);
  1937. if (flags & DETACH_GROUP)
  1938. perf_group_detach(event);
  1939. list_del_event(event, ctx);
  1940. if (!ctx->nr_events && ctx->is_active) {
  1941. ctx->is_active = 0;
  1942. ctx->rotate_necessary = 0;
  1943. if (ctx->task) {
  1944. WARN_ON_ONCE(cpuctx->task_ctx != ctx);
  1945. cpuctx->task_ctx = NULL;
  1946. }
  1947. }
  1948. }
  1949. /*
  1950. * Remove the event from a task's (or a CPU's) list of events.
  1951. *
  1952. * If event->ctx is a cloned context, callers must make sure that
  1953. * every task struct that event->ctx->task could possibly point to
  1954. * remains valid. This is OK when called from perf_release since
  1955. * that only calls us on the top-level context, which can't be a clone.
  1956. * When called from perf_event_exit_task, it's OK because the
  1957. * context has been detached from its task.
  1958. */
  1959. static void perf_remove_from_context(struct perf_event *event, unsigned long flags)
  1960. {
  1961. struct perf_event_context *ctx = event->ctx;
  1962. lockdep_assert_held(&ctx->mutex);
  1963. event_function_call(event, __perf_remove_from_context, (void *)flags);
  1964. /*
  1965. * The above event_function_call() can NO-OP when it hits
  1966. * TASK_TOMBSTONE. In that case we must already have been detached
  1967. * from the context (by perf_event_exit_event()) but the grouping
  1968. * might still be in-tact.
  1969. */
  1970. WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
  1971. if ((flags & DETACH_GROUP) &&
  1972. (event->attach_state & PERF_ATTACH_GROUP)) {
  1973. /*
  1974. * Since in that case we cannot possibly be scheduled, simply
  1975. * detach now.
  1976. */
  1977. raw_spin_lock_irq(&ctx->lock);
  1978. perf_group_detach(event);
  1979. raw_spin_unlock_irq(&ctx->lock);
  1980. }
  1981. }
  1982. /*
  1983. * Cross CPU call to disable a performance event
  1984. */
  1985. static void __perf_event_disable(struct perf_event *event,
  1986. struct perf_cpu_context *cpuctx,
  1987. struct perf_event_context *ctx,
  1988. void *info)
  1989. {
  1990. if (event->state < PERF_EVENT_STATE_INACTIVE)
  1991. return;
  1992. if (ctx->is_active & EVENT_TIME) {
  1993. update_context_time(ctx);
  1994. update_cgrp_time_from_event(event);
  1995. }
  1996. if (event == event->group_leader)
  1997. group_sched_out(event, cpuctx, ctx);
  1998. else
  1999. event_sched_out(event, cpuctx, ctx);
  2000. perf_event_set_state(event, PERF_EVENT_STATE_OFF);
  2001. perf_cgroup_event_disable(event, ctx);
  2002. }
  2003. /*
  2004. * Disable an event.
  2005. *
  2006. * If event->ctx is a cloned context, callers must make sure that
  2007. * every task struct that event->ctx->task could possibly point to
  2008. * remains valid. This condition is satisfied when called through
  2009. * perf_event_for_each_child or perf_event_for_each because they
  2010. * hold the top-level event's child_mutex, so any descendant that
  2011. * goes to exit will block in perf_event_exit_event().
  2012. *
  2013. * When called from perf_pending_event it's OK because event->ctx
  2014. * is the current context on this CPU and preemption is disabled,
  2015. * hence we can't get into perf_event_task_sched_out for this context.
  2016. */
  2017. static void _perf_event_disable(struct perf_event *event)
  2018. {
  2019. struct perf_event_context *ctx = event->ctx;
  2020. raw_spin_lock_irq(&ctx->lock);
  2021. if (event->state <= PERF_EVENT_STATE_OFF) {
  2022. raw_spin_unlock_irq(&ctx->lock);
  2023. return;
  2024. }
  2025. raw_spin_unlock_irq(&ctx->lock);
  2026. event_function_call(event, __perf_event_disable, NULL);
  2027. }
  2028. void perf_event_disable_local(struct perf_event *event)
  2029. {
  2030. event_function_local(event, __perf_event_disable, NULL);
  2031. }
  2032. /*
  2033. * Strictly speaking kernel users cannot create groups and therefore this
  2034. * interface does not need the perf_event_ctx_lock() magic.
  2035. */
  2036. void perf_event_disable(struct perf_event *event)
  2037. {
  2038. struct perf_event_context *ctx;
  2039. ctx = perf_event_ctx_lock(event);
  2040. _perf_event_disable(event);
  2041. perf_event_ctx_unlock(event, ctx);
  2042. }
  2043. EXPORT_SYMBOL_GPL(perf_event_disable);
  2044. void perf_event_disable_inatomic(struct perf_event *event)
  2045. {
  2046. WRITE_ONCE(event->pending_disable, smp_processor_id());
  2047. /* can fail, see perf_pending_event_disable() */
  2048. irq_work_queue(&event->pending);
  2049. }
  2050. static void perf_set_shadow_time(struct perf_event *event,
  2051. struct perf_event_context *ctx)
  2052. {
  2053. /*
  2054. * use the correct time source for the time snapshot
  2055. *
  2056. * We could get by without this by leveraging the
  2057. * fact that to get to this function, the caller
  2058. * has most likely already called update_context_time()
  2059. * and update_cgrp_time_xx() and thus both timestamp
  2060. * are identical (or very close). Given that tstamp is,
  2061. * already adjusted for cgroup, we could say that:
  2062. * tstamp - ctx->timestamp
  2063. * is equivalent to
  2064. * tstamp - cgrp->timestamp.
  2065. *
  2066. * Then, in perf_output_read(), the calculation would
  2067. * work with no changes because:
  2068. * - event is guaranteed scheduled in
  2069. * - no scheduled out in between
  2070. * - thus the timestamp would be the same
  2071. *
  2072. * But this is a bit hairy.
  2073. *
  2074. * So instead, we have an explicit cgroup call to remain
  2075. * within the time source all along. We believe it
  2076. * is cleaner and simpler to understand.
  2077. */
  2078. if (is_cgroup_event(event))
  2079. perf_cgroup_set_shadow_time(event, event->tstamp);
  2080. else
  2081. event->shadow_ctx_time = event->tstamp - ctx->timestamp;
  2082. }
  2083. #define MAX_INTERRUPTS (~0ULL)
  2084. static void perf_log_throttle(struct perf_event *event, int enable);
  2085. static void perf_log_itrace_start(struct perf_event *event);
  2086. static int
  2087. event_sched_in(struct perf_event *event,
  2088. struct perf_cpu_context *cpuctx,
  2089. struct perf_event_context *ctx)
  2090. {
  2091. int ret = 0;
  2092. WARN_ON_ONCE(event->ctx != ctx);
  2093. lockdep_assert_held(&ctx->lock);
  2094. if (event->state <= PERF_EVENT_STATE_OFF)
  2095. return 0;
  2096. WRITE_ONCE(event->oncpu, smp_processor_id());
  2097. /*
  2098. * Order event::oncpu write to happen before the ACTIVE state is
  2099. * visible. This allows perf_event_{stop,read}() to observe the correct
  2100. * ->oncpu if it sees ACTIVE.
  2101. */
  2102. smp_wmb();
  2103. perf_event_set_state(event, PERF_EVENT_STATE_ACTIVE);
  2104. /*
  2105. * Unthrottle events, since we scheduled we might have missed several
  2106. * ticks already, also for a heavily scheduling task there is little
  2107. * guarantee it'll get a tick in a timely manner.
  2108. */
  2109. if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {
  2110. perf_log_throttle(event, 1);
  2111. event->hw.interrupts = 0;
  2112. }
  2113. perf_pmu_disable(event->pmu);
  2114. perf_set_shadow_time(event, ctx);
  2115. perf_log_itrace_start(event);
  2116. if (event->pmu->add(event, PERF_EF_START)) {
  2117. perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE);
  2118. event->oncpu = -1;
  2119. ret = -EAGAIN;
  2120. goto out;
  2121. }
  2122. if (!is_software_event(event))
  2123. cpuctx->active_oncpu++;
  2124. if (!ctx->nr_active++)
  2125. perf_event_ctx_activate(ctx);
  2126. if (event->attr.freq && event->attr.sample_freq)
  2127. ctx->nr_freq++;
  2128. if (event->attr.exclusive)
  2129. cpuctx->exclusive = 1;
  2130. out:
  2131. perf_pmu_enable(event->pmu);
  2132. return ret;
  2133. }
  2134. static int
  2135. group_sched_in(struct perf_event *group_event,
  2136. struct perf_cpu_context *cpuctx,
  2137. struct perf_event_context *ctx)
  2138. {
  2139. struct perf_event *event, *partial_group = NULL;
  2140. struct pmu *pmu = ctx->pmu;
  2141. if (group_event->state == PERF_EVENT_STATE_OFF)
  2142. return 0;
  2143. pmu->start_txn(pmu, PERF_PMU_TXN_ADD);
  2144. if (event_sched_in(group_event, cpuctx, ctx))
  2145. goto error;
  2146. /*
  2147. * Schedule in siblings as one group (if any):
  2148. */
  2149. for_each_sibling_event(event, group_event) {
  2150. if (event_sched_in(event, cpuctx, ctx)) {
  2151. partial_group = event;
  2152. goto group_error;
  2153. }
  2154. }
  2155. if (!pmu->commit_txn(pmu))
  2156. return 0;
  2157. group_error:
  2158. /*
  2159. * Groups can be scheduled in as one unit only, so undo any
  2160. * partial group before returning:
  2161. * The events up to the failed event are scheduled out normally.
  2162. */
  2163. for_each_sibling_event(event, group_event) {
  2164. if (event == partial_group)
  2165. break;
  2166. event_sched_out(event, cpuctx, ctx);
  2167. }
  2168. event_sched_out(group_event, cpuctx, ctx);
  2169. error:
  2170. pmu->cancel_txn(pmu);
  2171. return -EAGAIN;
  2172. }
  2173. /*
  2174. * Work out whether we can put this event group on the CPU now.
  2175. */
  2176. static int group_can_go_on(struct perf_event *event,
  2177. struct perf_cpu_context *cpuctx,
  2178. int can_add_hw)
  2179. {
  2180. /*
  2181. * Groups consisting entirely of software events can always go on.
  2182. */
  2183. if (event->group_caps & PERF_EV_CAP_SOFTWARE)
  2184. return 1;
  2185. /*
  2186. * If an exclusive group is already on, no other hardware
  2187. * events can go on.
  2188. */
  2189. if (cpuctx->exclusive)
  2190. return 0;
  2191. /*
  2192. * If this group is exclusive and there are already
  2193. * events on the CPU, it can't go on.
  2194. */
  2195. if (event->attr.exclusive && !list_empty(get_event_list(event)))
  2196. return 0;
  2197. /*
  2198. * Otherwise, try to add it if all previous groups were able
  2199. * to go on.
  2200. */
  2201. return can_add_hw;
  2202. }
  2203. static void add_event_to_ctx(struct perf_event *event,
  2204. struct perf_event_context *ctx)
  2205. {
  2206. list_add_event(event, ctx);
  2207. perf_group_attach(event);
  2208. }
  2209. static void ctx_sched_out(struct perf_event_context *ctx,
  2210. struct perf_cpu_context *cpuctx,
  2211. enum event_type_t event_type);
  2212. static void
  2213. ctx_sched_in(struct perf_event_context *ctx,
  2214. struct perf_cpu_context *cpuctx,
  2215. enum event_type_t event_type,
  2216. struct task_struct *task);
  2217. static void task_ctx_sched_out(struct perf_cpu_context *cpuctx,
  2218. struct perf_event_context *ctx,
  2219. enum event_type_t event_type)
  2220. {
  2221. if (!cpuctx->task_ctx)
  2222. return;
  2223. if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
  2224. return;
  2225. ctx_sched_out(ctx, cpuctx, event_type);
  2226. }
  2227. static void perf_event_sched_in(struct perf_cpu_context *cpuctx,
  2228. struct perf_event_context *ctx,
  2229. struct task_struct *task)
  2230. {
  2231. cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task);
  2232. if (ctx)
  2233. ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task);
  2234. cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task);
  2235. if (ctx)
  2236. ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task);
  2237. }
  2238. /*
  2239. * We want to maintain the following priority of scheduling:
  2240. * - CPU pinned (EVENT_CPU | EVENT_PINNED)
  2241. * - task pinned (EVENT_PINNED)
  2242. * - CPU flexible (EVENT_CPU | EVENT_FLEXIBLE)
  2243. * - task flexible (EVENT_FLEXIBLE).
  2244. *
  2245. * In order to avoid unscheduling and scheduling back in everything every
  2246. * time an event is added, only do it for the groups of equal priority and
  2247. * below.
  2248. *
  2249. * This can be called after a batch operation on task events, in which case
  2250. * event_type is a bit mask of the types of events involved. For CPU events,
  2251. * event_type is only either EVENT_PINNED or EVENT_FLEXIBLE.
  2252. */
  2253. static void ctx_resched(struct perf_cpu_context *cpuctx,
  2254. struct perf_event_context *task_ctx,
  2255. enum event_type_t event_type)
  2256. {
  2257. enum event_type_t ctx_event_type;
  2258. bool cpu_event = !!(event_type & EVENT_CPU);
  2259. /*
  2260. * If pinned groups are involved, flexible groups also need to be
  2261. * scheduled out.
  2262. */
  2263. if (event_type & EVENT_PINNED)
  2264. event_type |= EVENT_FLEXIBLE;
  2265. ctx_event_type = event_type & EVENT_ALL;
  2266. perf_pmu_disable(cpuctx->ctx.pmu);
  2267. if (task_ctx)
  2268. task_ctx_sched_out(cpuctx, task_ctx, event_type);
  2269. /*
  2270. * Decide which cpu ctx groups to schedule out based on the types
  2271. * of events that caused rescheduling:
  2272. * - EVENT_CPU: schedule out corresponding groups;
  2273. * - EVENT_PINNED task events: schedule out EVENT_FLEXIBLE groups;
  2274. * - otherwise, do nothing more.
  2275. */
  2276. if (cpu_event)
  2277. cpu_ctx_sched_out(cpuctx, ctx_event_type);
  2278. else if (ctx_event_type & EVENT_PINNED)
  2279. cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
  2280. perf_event_sched_in(cpuctx, task_ctx, current);
  2281. perf_pmu_enable(cpuctx->ctx.pmu);
  2282. }
  2283. void perf_pmu_resched(struct pmu *pmu)
  2284. {
  2285. struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
  2286. struct perf_event_context *task_ctx = cpuctx->task_ctx;
  2287. perf_ctx_lock(cpuctx, task_ctx);
  2288. ctx_resched(cpuctx, task_ctx, EVENT_ALL|EVENT_CPU);
  2289. perf_ctx_unlock(cpuctx, task_ctx);
  2290. }
  2291. /*
  2292. * Cross CPU call to install and enable a performance event
  2293. *
  2294. * Very similar to remote_function() + event_function() but cannot assume that
  2295. * things like ctx->is_active and cpuctx->task_ctx are set.
  2296. */
  2297. static int __perf_install_in_context(void *info)
  2298. {
  2299. struct perf_event *event = info;
  2300. struct perf_event_context *ctx = event->ctx;
  2301. struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
  2302. struct perf_event_context *task_ctx = cpuctx->task_ctx;
  2303. bool reprogram = true;
  2304. int ret = 0;
  2305. raw_spin_lock(&cpuctx->ctx.lock);
  2306. if (ctx->task) {
  2307. raw_spin_lock(&ctx->lock);
  2308. task_ctx = ctx;
  2309. reprogram = (ctx->task == current);
  2310. /*
  2311. * If the task is running, it must be running on this CPU,
  2312. * otherwise we cannot reprogram things.
  2313. *
  2314. * If its not running, we don't care, ctx->lock will
  2315. * serialize against it becoming runnable.
  2316. */
  2317. if (task_curr(ctx->task) && !reprogram) {
  2318. ret = -ESRCH;
  2319. goto unlock;
  2320. }
  2321. WARN_ON_ONCE(reprogram && cpuctx->task_ctx && cpuctx->task_ctx != ctx);
  2322. } else if (task_ctx) {
  2323. raw_spin_lock(&task_ctx->lock);
  2324. }
  2325. #ifdef CONFIG_CGROUP_PERF
  2326. if (event->state > PERF_EVENT_STATE_OFF && is_cgroup_event(event)) {
  2327. /*
  2328. * If the current cgroup doesn't match the event's
  2329. * cgroup, we should not try to schedule it.
  2330. */
  2331. struct perf_cgroup *cgrp = perf_cgroup_from_task(current, ctx);
  2332. reprogram = cgroup_is_descendant(cgrp->css.cgroup,
  2333. event->cgrp->css.cgroup);
  2334. }
  2335. #endif
  2336. if (reprogram) {
  2337. ctx_sched_out(ctx, cpuctx, EVENT_TIME);
  2338. add_event_to_ctx(event, ctx);
  2339. ctx_resched(cpuctx, task_ctx, get_event_type(event));
  2340. } else {
  2341. add_event_to_ctx(event, ctx);
  2342. }
  2343. unlock:
  2344. perf_ctx_unlock(cpuctx, task_ctx);
  2345. return ret;
  2346. }
  2347. static bool exclusive_event_installable(struct perf_event *event,
  2348. struct perf_event_context *ctx);
  2349. /*
  2350. * Attach a performance event to a context.
  2351. *
  2352. * Very similar to event_function_call, see comment there.
  2353. */
  2354. static void
  2355. perf_install_in_context(struct perf_event_context *ctx,
  2356. struct perf_event *event,
  2357. int cpu)
  2358. {
  2359. struct task_struct *task = READ_ONCE(ctx->task);
  2360. lockdep_assert_held(&ctx->mutex);
  2361. WARN_ON_ONCE(!exclusive_event_installable(event, ctx));
  2362. if (event->cpu != -1)
  2363. event->cpu = cpu;
  2364. /*
  2365. * Ensures that if we can observe event->ctx, both the event and ctx
  2366. * will be 'complete'. See perf_iterate_sb_cpu().
  2367. */
  2368. smp_store_release(&event->ctx, ctx);
  2369. /*
  2370. * perf_event_attr::disabled events will not run and can be initialized
  2371. * without IPI. Except when this is the first event for the context, in
  2372. * that case we need the magic of the IPI to set ctx->is_active.
  2373. *
  2374. * The IOC_ENABLE that is sure to follow the creation of a disabled
  2375. * event will issue the IPI and reprogram the hardware.
  2376. */
  2377. if (__perf_effective_state(event) == PERF_EVENT_STATE_OFF && ctx->nr_events) {
  2378. raw_spin_lock_irq(&ctx->lock);
  2379. if (ctx->task == TASK_TOMBSTONE) {
  2380. raw_spin_unlock_irq(&ctx->lock);
  2381. return;
  2382. }
  2383. add_event_to_ctx(event, ctx);
  2384. raw_spin_unlock_irq(&ctx->lock);
  2385. return;
  2386. }
  2387. if (!task) {
  2388. cpu_function_call(cpu, __perf_install_in_context, event);
  2389. return;
  2390. }
  2391. /*
  2392. * Should not happen, we validate the ctx is still alive before calling.
  2393. */
  2394. if (WARN_ON_ONCE(task == TASK_TOMBSTONE))
  2395. return;
  2396. /*
  2397. * Installing events is tricky because we cannot rely on ctx->is_active
  2398. * to be set in case this is the nr_events 0 -> 1 transition.
  2399. *
  2400. * Instead we use task_curr(), which tells us if the task is running.
  2401. * However, since we use task_curr() outside of rq::lock, we can race
  2402. * against the actual state. This means the result can be wrong.
  2403. *
  2404. * If we get a false positive, we retry, this is harmless.
  2405. *
  2406. * If we get a false negative, things are complicated. If we are after
  2407. * perf_event_context_sched_in() ctx::lock will serialize us, and the
  2408. * value must be correct. If we're before, it doesn't matter since
  2409. * perf_event_context_sched_in() will program the counter.
  2410. *
  2411. * However, this hinges on the remote context switch having observed
  2412. * our task->perf_event_ctxp[] store, such that it will in fact take
  2413. * ctx::lock in perf_event_context_sched_in().
  2414. *
  2415. * We do this by task_function_call(), if the IPI fails to hit the task
  2416. * we know any future context switch of task must see the
  2417. * perf_event_ctpx[] store.
  2418. */
  2419. /*
  2420. * This smp_mb() orders the task->perf_event_ctxp[] store with the
  2421. * task_cpu() load, such that if the IPI then does not find the task
  2422. * running, a future context switch of that task must observe the
  2423. * store.
  2424. */
  2425. smp_mb();
  2426. again:
  2427. if (!task_function_call(task, __perf_install_in_context, event))
  2428. return;
  2429. raw_spin_lock_irq(&ctx->lock);
  2430. task = ctx->task;
  2431. if (WARN_ON_ONCE(task == TASK_TOMBSTONE)) {
  2432. /*
  2433. * Cannot happen because we already checked above (which also
  2434. * cannot happen), and we hold ctx->mutex, which serializes us
  2435. * against perf_event_exit_task_context().
  2436. */
  2437. raw_spin_unlock_irq(&ctx->lock);
  2438. return;
  2439. }
  2440. /*
  2441. * If the task is not running, ctx->lock will avoid it becoming so,
  2442. * thus we can safely install the event.
  2443. */
  2444. if (task_curr(task)) {
  2445. raw_spin_unlock_irq(&ctx->lock);
  2446. goto again;
  2447. }
  2448. add_event_to_ctx(event, ctx);
  2449. raw_spin_unlock_irq(&ctx->lock);
  2450. }
  2451. /*
  2452. * Cross CPU call to enable a performance event
  2453. */
  2454. static void __perf_event_enable(struct perf_event *event,
  2455. struct perf_cpu_context *cpuctx,
  2456. struct perf_event_context *ctx,
  2457. void *info)
  2458. {
  2459. struct perf_event *leader = event->group_leader;
  2460. struct perf_event_context *task_ctx;
  2461. if (event->state >= PERF_EVENT_STATE_INACTIVE ||
  2462. event->state <= PERF_EVENT_STATE_ERROR)
  2463. return;
  2464. if (ctx->is_active)
  2465. ctx_sched_out(ctx, cpuctx, EVENT_TIME);
  2466. perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE);
  2467. perf_cgroup_event_enable(event, ctx);
  2468. if (!ctx->is_active)
  2469. return;
  2470. if (!event_filter_match(event)) {
  2471. ctx_sched_in(ctx, cpuctx, EVENT_TIME, current);
  2472. return;
  2473. }
  2474. /*
  2475. * If the event is in a group and isn't the group leader,
  2476. * then don't put it on unless the group is on.
  2477. */
  2478. if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE) {
  2479. ctx_sched_in(ctx, cpuctx, EVENT_TIME, current);
  2480. return;
  2481. }
  2482. task_ctx = cpuctx->task_ctx;
  2483. if (ctx->task)
  2484. WARN_ON_ONCE(task_ctx != ctx);
  2485. ctx_resched(cpuctx, task_ctx, get_event_type(event));
  2486. }
  2487. /*
  2488. * Enable an event.
  2489. *
  2490. * If event->ctx is a cloned context, callers must make sure that
  2491. * every task struct that event->ctx->task could possibly point to
  2492. * remains valid. This condition is satisfied when called through
  2493. * perf_event_for_each_child or perf_event_for_each as described
  2494. * for perf_event_disable.
  2495. */
  2496. static void _perf_event_enable(struct perf_event *event)
  2497. {
  2498. struct perf_event_context *ctx = event->ctx;
  2499. raw_spin_lock_irq(&ctx->lock);
  2500. if (event->state >= PERF_EVENT_STATE_INACTIVE ||
  2501. event->state < PERF_EVENT_STATE_ERROR) {
  2502. out:
  2503. raw_spin_unlock_irq(&ctx->lock);
  2504. return;
  2505. }
  2506. /*
  2507. * If the event is in error state, clear that first.
  2508. *
  2509. * That way, if we see the event in error state below, we know that it
  2510. * has gone back into error state, as distinct from the task having
  2511. * been scheduled away before the cross-call arrived.
  2512. */
  2513. if (event->state == PERF_EVENT_STATE_ERROR) {
  2514. /*
  2515. * Detached SIBLING events cannot leave ERROR state.
  2516. */
  2517. if (event->event_caps & PERF_EV_CAP_SIBLING &&
  2518. event->group_leader == event)
  2519. goto out;
  2520. event->state = PERF_EVENT_STATE_OFF;
  2521. }
  2522. raw_spin_unlock_irq(&ctx->lock);
  2523. event_function_call(event, __perf_event_enable, NULL);
  2524. }
  2525. /*
  2526. * See perf_event_disable();
  2527. */
  2528. void perf_event_enable(struct perf_event *event)
  2529. {
  2530. struct perf_event_context *ctx;
  2531. ctx = perf_event_ctx_lock(event);
  2532. _perf_event_enable(event);
  2533. perf_event_ctx_unlock(event, ctx);
  2534. }
  2535. EXPORT_SYMBOL_GPL(perf_event_enable);
  2536. struct stop_event_data {
  2537. struct perf_event *event;
  2538. unsigned int restart;
  2539. };
  2540. static int __perf_event_stop(void *info)
  2541. {
  2542. struct stop_event_data *sd = info;
  2543. struct perf_event *event = sd->event;
  2544. /* if it's already INACTIVE, do nothing */
  2545. if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE)
  2546. return 0;
  2547. /* matches smp_wmb() in event_sched_in() */
  2548. smp_rmb();
  2549. /*
  2550. * There is a window with interrupts enabled before we get here,
  2551. * so we need to check again lest we try to stop another CPU's event.
  2552. */
  2553. if (READ_ONCE(event->oncpu) != smp_processor_id())
  2554. return -EAGAIN;
  2555. event->pmu->stop(event, PERF_EF_UPDATE);
  2556. /*
  2557. * May race with the actual stop (through perf_pmu_output_stop()),
  2558. * but it is only used for events with AUX ring buffer, and such
  2559. * events will refuse to restart because of rb::aux_mmap_count==0,
  2560. * see comments in perf_aux_output_begin().
  2561. *
  2562. * Since this is happening on an event-local CPU, no trace is lost
  2563. * while restarting.
  2564. */
  2565. if (sd->restart)
  2566. event->pmu->start(event, 0);
  2567. return 0;
  2568. }
  2569. static int perf_event_stop(struct perf_event *event, int restart)
  2570. {
  2571. struct stop_event_data sd = {
  2572. .event = event,
  2573. .restart = restart,
  2574. };
  2575. int ret = 0;
  2576. do {
  2577. if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE)
  2578. return 0;
  2579. /* matches smp_wmb() in event_sched_in() */
  2580. smp_rmb();
  2581. /*
  2582. * We only want to restart ACTIVE events, so if the event goes
  2583. * inactive here (event->oncpu==-1), there's nothing more to do;
  2584. * fall through with ret==-ENXIO.
  2585. */
  2586. ret = cpu_function_call(READ_ONCE(event->oncpu),
  2587. __perf_event_stop, &sd);
  2588. } while (ret == -EAGAIN);
  2589. return ret;
  2590. }
  2591. /*
  2592. * In order to contain the amount of racy and tricky in the address filter
  2593. * configuration management, it is a two part process:
  2594. *
  2595. * (p1) when userspace mappings change as a result of (1) or (2) or (3) below,
  2596. * we update the addresses of corresponding vmas in
  2597. * event::addr_filter_ranges array and bump the event::addr_filters_gen;
  2598. * (p2) when an event is scheduled in (pmu::add), it calls
  2599. * perf_event_addr_filters_sync() which calls pmu::addr_filters_sync()
  2600. * if the generation has changed since the previous call.
  2601. *
  2602. * If (p1) happens while the event is active, we restart it to force (p2).
  2603. *
  2604. * (1) perf_addr_filters_apply(): adjusting filters' offsets based on
  2605. * pre-existing mappings, called once when new filters arrive via SET_FILTER
  2606. * ioctl;
  2607. * (2) perf_addr_filters_adjust(): adjusting filters' offsets based on newly
  2608. * registered mapping, called for every new mmap(), with mm::mmap_lock down
  2609. * for reading;
  2610. * (3) perf_event_addr_filters_exec(): clearing filters' offsets in the process
  2611. * of exec.
  2612. */
  2613. void perf_event_addr_filters_sync(struct perf_event *event)
  2614. {
  2615. struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
  2616. if (!has_addr_filter(event))
  2617. return;
  2618. raw_spin_lock(&ifh->lock);
  2619. if (event->addr_filters_gen != event->hw.addr_filters_gen) {
  2620. event->pmu->addr_filters_sync(event);
  2621. event->hw.addr_filters_gen = event->addr_filters_gen;
  2622. }
  2623. raw_spin_unlock(&ifh->lock);
  2624. }
  2625. EXPORT_SYMBOL_GPL(perf_event_addr_filters_sync);
  2626. static int _perf_event_refresh(struct perf_event *event, int refresh)
  2627. {
  2628. /*
  2629. * not supported on inherited events
  2630. */
  2631. if (event->attr.inherit || !is_sampling_event(event))
  2632. return -EINVAL;
  2633. atomic_add(refresh, &event->event_limit);
  2634. _perf_event_enable(event);
  2635. return 0;
  2636. }
  2637. /*
  2638. * See perf_event_disable()
  2639. */
  2640. int perf_event_refresh(struct perf_event *event, int refresh)
  2641. {
  2642. struct perf_event_context *ctx;
  2643. int ret;
  2644. ctx = perf_event_ctx_lock(event);
  2645. ret = _perf_event_refresh(event, refresh);
  2646. perf_event_ctx_unlock(event, ctx);
  2647. return ret;
  2648. }
  2649. EXPORT_SYMBOL_GPL(perf_event_refresh);
  2650. static int perf_event_modify_breakpoint(struct perf_event *bp,
  2651. struct perf_event_attr *attr)
  2652. {
  2653. int err;
  2654. _perf_event_disable(bp);
  2655. err = modify_user_hw_breakpoint_check(bp, attr, true);
  2656. if (!bp->attr.disabled)
  2657. _perf_event_enable(bp);
  2658. return err;
  2659. }
  2660. static int perf_event_modify_attr(struct perf_event *event,
  2661. struct perf_event_attr *attr)
  2662. {
  2663. if (event->attr.type != attr->type)
  2664. return -EINVAL;
  2665. switch (event->attr.type) {
  2666. case PERF_TYPE_BREAKPOINT:
  2667. return perf_event_modify_breakpoint(event, attr);
  2668. default:
  2669. /* Place holder for future additions. */
  2670. return -EOPNOTSUPP;
  2671. }
  2672. }
  2673. static void ctx_sched_out(struct perf_event_context *ctx,
  2674. struct perf_cpu_context *cpuctx,
  2675. enum event_type_t event_type)
  2676. {
  2677. struct perf_event *event, *tmp;
  2678. int is_active = ctx->is_active;
  2679. lockdep_assert_held(&ctx->lock);
  2680. if (likely(!ctx->nr_events)) {
  2681. /*
  2682. * See __perf_remove_from_context().
  2683. */
  2684. WARN_ON_ONCE(ctx->is_active);
  2685. if (ctx->task)
  2686. WARN_ON_ONCE(cpuctx->task_ctx);
  2687. return;
  2688. }
  2689. ctx->is_active &= ~event_type;
  2690. if (!(ctx->is_active & EVENT_ALL))
  2691. ctx->is_active = 0;
  2692. if (ctx->task) {
  2693. WARN_ON_ONCE(cpuctx->task_ctx != ctx);
  2694. if (!ctx->is_active)
  2695. cpuctx->task_ctx = NULL;
  2696. }
  2697. /*
  2698. * Always update time if it was set; not only when it changes.
  2699. * Otherwise we can 'forget' to update time for any but the last
  2700. * context we sched out. For example:
  2701. *
  2702. * ctx_sched_out(.event_type = EVENT_FLEXIBLE)
  2703. * ctx_sched_out(.event_type = EVENT_PINNED)
  2704. *
  2705. * would only update time for the pinned events.
  2706. */
  2707. if (is_active & EVENT_TIME) {
  2708. /* update (and stop) ctx time */
  2709. update_context_time(ctx);
  2710. update_cgrp_time_from_cpuctx(cpuctx);
  2711. }
  2712. is_active ^= ctx->is_active; /* changed bits */
  2713. if (!ctx->nr_active || !(is_active & EVENT_ALL))
  2714. return;
  2715. perf_pmu_disable(ctx->pmu);
  2716. if (is_active & EVENT_PINNED) {
  2717. list_for_each_entry_safe(event, tmp, &ctx->pinned_active, active_list)
  2718. group_sched_out(event, cpuctx, ctx);
  2719. }
  2720. if (is_active & EVENT_FLEXIBLE) {
  2721. list_for_each_entry_safe(event, tmp, &ctx->flexible_active, active_list)
  2722. group_sched_out(event, cpuctx, ctx);
  2723. /*
  2724. * Since we cleared EVENT_FLEXIBLE, also clear
  2725. * rotate_necessary, is will be reset by
  2726. * ctx_flexible_sched_in() when needed.
  2727. */
  2728. ctx->rotate_necessary = 0;
  2729. }
  2730. perf_pmu_enable(ctx->pmu);
  2731. }
  2732. /*
  2733. * Test whether two contexts are equivalent, i.e. whether they have both been
  2734. * cloned from the same version of the same context.
  2735. *
  2736. * Equivalence is measured using a generation number in the context that is
  2737. * incremented on each modification to it; see unclone_ctx(), list_add_event()
  2738. * and list_del_event().
  2739. */
  2740. static int context_equiv(struct perf_event_context *ctx1,
  2741. struct perf_event_context *ctx2)
  2742. {
  2743. lockdep_assert_held(&ctx1->lock);
  2744. lockdep_assert_held(&ctx2->lock);
  2745. /* Pinning disables the swap optimization */
  2746. if (ctx1->pin_count || ctx2->pin_count)
  2747. return 0;
  2748. /* If ctx1 is the parent of ctx2 */
  2749. if (ctx1 == ctx2->parent_ctx && ctx1->generation == ctx2->parent_gen)
  2750. return 1;
  2751. /* If ctx2 is the parent of ctx1 */
  2752. if (ctx1->parent_ctx == ctx2 && ctx1->parent_gen == ctx2->generation)
  2753. return 1;
  2754. /*
  2755. * If ctx1 and ctx2 have the same parent; we flatten the parent
  2756. * hierarchy, see perf_event_init_context().
  2757. */
  2758. if (ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx &&
  2759. ctx1->parent_gen == ctx2->parent_gen)
  2760. return 1;
  2761. /* Unmatched */
  2762. return 0;
  2763. }
  2764. static void __perf_event_sync_stat(struct perf_event *event,
  2765. struct perf_event *next_event)
  2766. {
  2767. u64 value;
  2768. if (!event->attr.inherit_stat)
  2769. return;
  2770. /*
  2771. * Update the event value, we cannot use perf_event_read()
  2772. * because we're in the middle of a context switch and have IRQs
  2773. * disabled, which upsets smp_call_function_single(), however
  2774. * we know the event must be on the current CPU, therefore we
  2775. * don't need to use it.
  2776. */
  2777. if (event->state == PERF_EVENT_STATE_ACTIVE)
  2778. event->pmu->read(event);
  2779. perf_event_update_time(event);
  2780. /*
  2781. * In order to keep per-task stats reliable we need to flip the event
  2782. * values when we flip the contexts.
  2783. */
  2784. value = local64_read(&next_event->count);
  2785. value = local64_xchg(&event->count, value);
  2786. local64_set(&next_event->count, value);
  2787. swap(event->total_time_enabled, next_event->total_time_enabled);
  2788. swap(event->total_time_running, next_event->total_time_running);
  2789. /*
  2790. * Since we swizzled the values, update the user visible data too.
  2791. */
  2792. perf_event_update_userpage(event);
  2793. perf_event_update_userpage(next_event);
  2794. }
  2795. static void perf_event_sync_stat(struct perf_event_context *ctx,
  2796. struct perf_event_context *next_ctx)
  2797. {
  2798. struct perf_event *event, *next_event;
  2799. if (!ctx->nr_stat)
  2800. return;
  2801. update_context_time(ctx);
  2802. event = list_first_entry(&ctx->event_list,
  2803. struct perf_event, event_entry);
  2804. next_event = list_first_entry(&next_ctx->event_list,
  2805. struct perf_event, event_entry);
  2806. while (&event->event_entry != &ctx->event_list &&
  2807. &next_event->event_entry != &next_ctx->event_list) {
  2808. __perf_event_sync_stat(event, next_event);
  2809. event = list_next_entry(event, event_entry);
  2810. next_event = list_next_entry(next_event, event_entry);
  2811. }
  2812. }
  2813. static void perf_event_context_sched_out(struct task_struct *task, int ctxn,
  2814. struct task_struct *next)
  2815. {
  2816. struct perf_event_context *ctx = task->perf_event_ctxp[ctxn];
  2817. struct perf_event_context *next_ctx;
  2818. struct perf_event_context *parent, *next_parent;
  2819. struct perf_cpu_context *cpuctx;
  2820. int do_switch = 1;
  2821. struct pmu *pmu;
  2822. if (likely(!ctx))
  2823. return;
  2824. pmu = ctx->pmu;
  2825. cpuctx = __get_cpu_context(ctx);
  2826. if (!cpuctx->task_ctx)
  2827. return;
  2828. rcu_read_lock();
  2829. next_ctx = next->perf_event_ctxp[ctxn];
  2830. if (!next_ctx)
  2831. goto unlock;
  2832. parent = rcu_dereference(ctx->parent_ctx);
  2833. next_parent = rcu_dereference(next_ctx->parent_ctx);
  2834. /* If neither context have a parent context; they cannot be clones. */
  2835. if (!parent && !next_parent)
  2836. goto unlock;
  2837. if (next_parent == ctx || next_ctx == parent || next_parent == parent) {
  2838. /*
  2839. * Looks like the two contexts are clones, so we might be
  2840. * able to optimize the context switch. We lock both
  2841. * contexts and check that they are clones under the
  2842. * lock (including re-checking that neither has been
  2843. * uncloned in the meantime). It doesn't matter which
  2844. * order we take the locks because no other cpu could
  2845. * be trying to lock both of these tasks.
  2846. */
  2847. raw_spin_lock(&ctx->lock);
  2848. raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
  2849. if (context_equiv(ctx, next_ctx)) {
  2850. WRITE_ONCE(ctx->task, next);
  2851. WRITE_ONCE(next_ctx->task, task);
  2852. perf_pmu_disable(pmu);
  2853. if (cpuctx->sched_cb_usage && pmu->sched_task)
  2854. pmu->sched_task(ctx, false);
  2855. /*
  2856. * PMU specific parts of task perf context can require
  2857. * additional synchronization. As an example of such
  2858. * synchronization see implementation details of Intel
  2859. * LBR call stack data profiling;
  2860. */
  2861. if (pmu->swap_task_ctx)
  2862. pmu->swap_task_ctx(ctx, next_ctx);
  2863. else
  2864. swap(ctx->task_ctx_data, next_ctx->task_ctx_data);
  2865. perf_pmu_enable(pmu);
  2866. /*
  2867. * RCU_INIT_POINTER here is safe because we've not
  2868. * modified the ctx and the above modification of
  2869. * ctx->task and ctx->task_ctx_data are immaterial
  2870. * since those values are always verified under
  2871. * ctx->lock which we're now holding.
  2872. */
  2873. RCU_INIT_POINTER(task->perf_event_ctxp[ctxn], next_ctx);
  2874. RCU_INIT_POINTER(next->perf_event_ctxp[ctxn], ctx);
  2875. do_switch = 0;
  2876. perf_event_sync_stat(ctx, next_ctx);
  2877. }
  2878. raw_spin_unlock(&next_ctx->lock);
  2879. raw_spin_unlock(&ctx->lock);
  2880. }
  2881. unlock:
  2882. rcu_read_unlock();
  2883. if (do_switch) {
  2884. raw_spin_lock(&ctx->lock);
  2885. perf_pmu_disable(pmu);
  2886. if (cpuctx->sched_cb_usage && pmu->sched_task)
  2887. pmu->sched_task(ctx, false);
  2888. task_ctx_sched_out(cpuctx, ctx, EVENT_ALL);
  2889. perf_pmu_enable(pmu);
  2890. raw_spin_unlock(&ctx->lock);
  2891. }
  2892. }
  2893. static DEFINE_PER_CPU(struct list_head, sched_cb_list);
  2894. void perf_sched_cb_dec(struct pmu *pmu)
  2895. {
  2896. struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
  2897. this_cpu_dec(perf_sched_cb_usages);
  2898. if (!--cpuctx->sched_cb_usage)
  2899. list_del(&cpuctx->sched_cb_entry);
  2900. }
  2901. void perf_sched_cb_inc(struct pmu *pmu)
  2902. {
  2903. struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
  2904. if (!cpuctx->sched_cb_usage++)
  2905. list_add(&cpuctx->sched_cb_entry, this_cpu_ptr(&sched_cb_list));
  2906. this_cpu_inc(perf_sched_cb_usages);
  2907. }
  2908. /*
  2909. * This function provides the context switch callback to the lower code
  2910. * layer. It is invoked ONLY when the context switch callback is enabled.
  2911. *
  2912. * This callback is relevant even to per-cpu events; for example multi event
  2913. * PEBS requires this to provide PID/TID information. This requires we flush
  2914. * all queued PEBS records before we context switch to a new task.
  2915. */
  2916. static void __perf_pmu_sched_task(struct perf_cpu_context *cpuctx, bool sched_in)
  2917. {
  2918. struct pmu *pmu;
  2919. pmu = cpuctx->ctx.pmu; /* software PMUs will not have sched_task */
  2920. if (WARN_ON_ONCE(!pmu->sched_task))
  2921. return;
  2922. perf_ctx_lock(cpuctx, cpuctx->task_ctx);
  2923. perf_pmu_disable(pmu);
  2924. pmu->sched_task(cpuctx->task_ctx, sched_in);
  2925. perf_pmu_enable(pmu);
  2926. perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
  2927. }
  2928. static void perf_pmu_sched_task(struct task_struct *prev,
  2929. struct task_struct *next,
  2930. bool sched_in)
  2931. {
  2932. struct perf_cpu_context *cpuctx;
  2933. if (prev == next)
  2934. return;
  2935. list_for_each_entry(cpuctx, this_cpu_ptr(&sched_cb_list), sched_cb_entry) {
  2936. /* will be handled in perf_event_context_sched_in/out */
  2937. if (cpuctx->task_ctx)
  2938. continue;
  2939. __perf_pmu_sched_task(cpuctx, sched_in);
  2940. }
  2941. }
  2942. static void perf_event_switch(struct task_struct *task,
  2943. struct task_struct *next_prev, bool sched_in);
  2944. #define for_each_task_context_nr(ctxn) \
  2945. for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++)
  2946. /*
  2947. * Called from scheduler to remove the events of the current task,
  2948. * with interrupts disabled.
  2949. *
  2950. * We stop each event and update the event value in event->count.
  2951. *
  2952. * This does not protect us against NMI, but disable()
  2953. * sets the disabled bit in the control field of event _before_
  2954. * accessing the event control register. If a NMI hits, then it will
  2955. * not restart the event.
  2956. */
  2957. void __perf_event_task_sched_out(struct task_struct *task,
  2958. struct task_struct *next)
  2959. {
  2960. int ctxn;
  2961. if (__this_cpu_read(perf_sched_cb_usages))
  2962. perf_pmu_sched_task(task, next, false);
  2963. if (atomic_read(&nr_switch_events))
  2964. perf_event_switch(task, next, false);
  2965. for_each_task_context_nr(ctxn)
  2966. perf_event_context_sched_out(task, ctxn, next);
  2967. /*
  2968. * if cgroup events exist on this CPU, then we need
  2969. * to check if we have to switch out PMU state.
  2970. * cgroup event are system-wide mode only
  2971. */
  2972. if (atomic_read(this_cpu_ptr(&perf_cgroup_events)))
  2973. perf_cgroup_sched_out(task, next);
  2974. }
  2975. /*
  2976. * Called with IRQs disabled
  2977. */
  2978. static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
  2979. enum event_type_t event_type)
  2980. {
  2981. ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
  2982. }
  2983. static bool perf_less_group_idx(const void *l, const void *r)
  2984. {
  2985. const struct perf_event *le = *(const struct perf_event **)l;
  2986. const struct perf_event *re = *(const struct perf_event **)r;
  2987. return le->group_index < re->group_index;
  2988. }
  2989. static void swap_ptr(void *l, void *r)
  2990. {
  2991. void **lp = l, **rp = r;
  2992. swap(*lp, *rp);
  2993. }
  2994. static const struct min_heap_callbacks perf_min_heap = {
  2995. .elem_size = sizeof(struct perf_event *),
  2996. .less = perf_less_group_idx,
  2997. .swp = swap_ptr,
  2998. };
  2999. static void __heap_add(struct min_heap *heap, struct perf_event *event)
  3000. {
  3001. struct perf_event **itrs = heap->data;
  3002. if (event) {
  3003. itrs[heap->nr] = event;
  3004. heap->nr++;
  3005. }
  3006. }
  3007. static noinline int visit_groups_merge(struct perf_cpu_context *cpuctx,
  3008. struct perf_event_groups *groups, int cpu,
  3009. int (*func)(struct perf_event *, void *),
  3010. void *data)
  3011. {
  3012. #ifdef CONFIG_CGROUP_PERF
  3013. struct cgroup_subsys_state *css = NULL;
  3014. #endif
  3015. /* Space for per CPU and/or any CPU event iterators. */
  3016. struct perf_event *itrs[2];
  3017. struct min_heap event_heap;
  3018. struct perf_event **evt;
  3019. int ret;
  3020. if (cpuctx) {
  3021. event_heap = (struct min_heap){
  3022. .data = cpuctx->heap,
  3023. .nr = 0,
  3024. .size = cpuctx->heap_size,
  3025. };
  3026. lockdep_assert_held(&cpuctx->ctx.lock);
  3027. #ifdef CONFIG_CGROUP_PERF
  3028. if (cpuctx->cgrp)
  3029. css = &cpuctx->cgrp->css;
  3030. #endif
  3031. } else {
  3032. event_heap = (struct min_heap){
  3033. .data = itrs,
  3034. .nr = 0,
  3035. .size = ARRAY_SIZE(itrs),
  3036. };
  3037. /* Events not within a CPU context may be on any CPU. */
  3038. __heap_add(&event_heap, perf_event_groups_first(groups, -1, NULL));
  3039. }
  3040. evt = event_heap.data;
  3041. __heap_add(&event_heap, perf_event_groups_first(groups, cpu, NULL));
  3042. #ifdef CONFIG_CGROUP_PERF
  3043. for (; css; css = css->parent)
  3044. __heap_add(&event_heap, perf_event_groups_first(groups, cpu, css->cgroup));
  3045. #endif
  3046. min_heapify_all(&event_heap, &perf_min_heap);
  3047. while (event_heap.nr) {
  3048. ret = func(*evt, data);
  3049. if (ret)
  3050. return ret;
  3051. *evt = perf_event_groups_next(*evt);
  3052. if (*evt)
  3053. min_heapify(&event_heap, 0, &perf_min_heap);
  3054. else
  3055. min_heap_pop(&event_heap, &perf_min_heap);
  3056. }
  3057. return 0;
  3058. }
  3059. static inline bool event_update_userpage(struct perf_event *event)
  3060. {
  3061. if (likely(!atomic_read(&event->mmap_count)))
  3062. return false;
  3063. perf_event_update_time(event);
  3064. perf_set_shadow_time(event, event->ctx);
  3065. perf_event_update_userpage(event);
  3066. return true;
  3067. }
  3068. static inline void group_update_userpage(struct perf_event *group_event)
  3069. {
  3070. struct perf_event *event;
  3071. if (!event_update_userpage(group_event))
  3072. return;
  3073. for_each_sibling_event(event, group_event)
  3074. event_update_userpage(event);
  3075. }
  3076. static int merge_sched_in(struct perf_event *event, void *data)
  3077. {
  3078. struct perf_event_context *ctx = event->ctx;
  3079. struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
  3080. int *can_add_hw = data;
  3081. if (event->state <= PERF_EVENT_STATE_OFF)
  3082. return 0;
  3083. if (!event_filter_match(event))
  3084. return 0;
  3085. if (group_can_go_on(event, cpuctx, *can_add_hw)) {
  3086. if (!group_sched_in(event, cpuctx, ctx))
  3087. list_add_tail(&event->active_list, get_event_list(event));
  3088. }
  3089. if (event->state == PERF_EVENT_STATE_INACTIVE) {
  3090. *can_add_hw = 0;
  3091. if (event->attr.pinned) {
  3092. perf_cgroup_event_disable(event, ctx);
  3093. perf_event_set_state(event, PERF_EVENT_STATE_ERROR);
  3094. } else {
  3095. ctx->rotate_necessary = 1;
  3096. perf_mux_hrtimer_restart(cpuctx);
  3097. group_update_userpage(event);
  3098. }
  3099. }
  3100. return 0;
  3101. }
  3102. static void
  3103. ctx_pinned_sched_in(struct perf_event_context *ctx,
  3104. struct perf_cpu_context *cpuctx)
  3105. {
  3106. int can_add_hw = 1;
  3107. if (ctx != &cpuctx->ctx)
  3108. cpuctx = NULL;
  3109. visit_groups_merge(cpuctx, &ctx->pinned_groups,
  3110. smp_processor_id(),
  3111. merge_sched_in, &can_add_hw);
  3112. }
  3113. static void
  3114. ctx_flexible_sched_in(struct perf_event_context *ctx,
  3115. struct perf_cpu_context *cpuctx)
  3116. {
  3117. int can_add_hw = 1;
  3118. if (ctx != &cpuctx->ctx)
  3119. cpuctx = NULL;
  3120. visit_groups_merge(cpuctx, &ctx->flexible_groups,
  3121. smp_processor_id(),
  3122. merge_sched_in, &can_add_hw);
  3123. }
  3124. static void
  3125. ctx_sched_in(struct perf_event_context *ctx,
  3126. struct perf_cpu_context *cpuctx,
  3127. enum event_type_t event_type,
  3128. struct task_struct *task)
  3129. {
  3130. int is_active = ctx->is_active;
  3131. u64 now;
  3132. lockdep_assert_held(&ctx->lock);
  3133. if (likely(!ctx->nr_events))
  3134. return;
  3135. ctx->is_active |= (event_type | EVENT_TIME);
  3136. if (ctx->task) {
  3137. if (!is_active)
  3138. cpuctx->task_ctx = ctx;
  3139. else
  3140. WARN_ON_ONCE(cpuctx->task_ctx != ctx);
  3141. }
  3142. is_active ^= ctx->is_active; /* changed bits */
  3143. if (is_active & EVENT_TIME) {
  3144. /* start ctx time */
  3145. now = perf_clock();
  3146. ctx->timestamp = now;
  3147. perf_cgroup_set_timestamp(task, ctx);
  3148. }
  3149. /*
  3150. * First go through the list and put on any pinned groups
  3151. * in order to give them the best chance of going on.
  3152. */
  3153. if (is_active & EVENT_PINNED)
  3154. ctx_pinned_sched_in(ctx, cpuctx);
  3155. /* Then walk through the lower prio flexible groups */
  3156. if (is_active & EVENT_FLEXIBLE)
  3157. ctx_flexible_sched_in(ctx, cpuctx);
  3158. }
  3159. static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
  3160. enum event_type_t event_type,
  3161. struct task_struct *task)
  3162. {
  3163. struct perf_event_context *ctx = &cpuctx->ctx;
  3164. ctx_sched_in(ctx, cpuctx, event_type, task);
  3165. }
  3166. static void perf_event_context_sched_in(struct perf_event_context *ctx,
  3167. struct task_struct *task)
  3168. {
  3169. struct perf_cpu_context *cpuctx;
  3170. struct pmu *pmu = ctx->pmu;
  3171. cpuctx = __get_cpu_context(ctx);
  3172. if (cpuctx->task_ctx == ctx) {
  3173. if (cpuctx->sched_cb_usage)
  3174. __perf_pmu_sched_task(cpuctx, true);
  3175. return;
  3176. }
  3177. perf_ctx_lock(cpuctx, ctx);
  3178. /*
  3179. * We must check ctx->nr_events while holding ctx->lock, such
  3180. * that we serialize against perf_install_in_context().
  3181. */
  3182. if (!ctx->nr_events)
  3183. goto unlock;
  3184. perf_pmu_disable(pmu);
  3185. /*
  3186. * We want to keep the following priority order:
  3187. * cpu pinned (that don't need to move), task pinned,
  3188. * cpu flexible, task flexible.
  3189. *
  3190. * However, if task's ctx is not carrying any pinned
  3191. * events, no need to flip the cpuctx's events around.
  3192. */
  3193. if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree))
  3194. cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
  3195. perf_event_sched_in(cpuctx, ctx, task);
  3196. if (cpuctx->sched_cb_usage && pmu->sched_task)
  3197. pmu->sched_task(cpuctx->task_ctx, true);
  3198. perf_pmu_enable(pmu);
  3199. unlock:
  3200. perf_ctx_unlock(cpuctx, ctx);
  3201. }
  3202. /*
  3203. * Called from scheduler to add the events of the current task
  3204. * with interrupts disabled.
  3205. *
  3206. * We restore the event value and then enable it.
  3207. *
  3208. * This does not protect us against NMI, but enable()
  3209. * sets the enabled bit in the control field of event _before_
  3210. * accessing the event control register. If a NMI hits, then it will
  3211. * keep the event running.
  3212. */
  3213. void __perf_event_task_sched_in(struct task_struct *prev,
  3214. struct task_struct *task)
  3215. {
  3216. struct perf_event_context *ctx;
  3217. int ctxn;
  3218. /*
  3219. * If cgroup events exist on this CPU, then we need to check if we have
  3220. * to switch in PMU state; cgroup event are system-wide mode only.
  3221. *
  3222. * Since cgroup events are CPU events, we must schedule these in before
  3223. * we schedule in the task events.
  3224. */
  3225. if (atomic_read(this_cpu_ptr(&perf_cgroup_events)))
  3226. perf_cgroup_sched_in(prev, task);
  3227. for_each_task_context_nr(ctxn) {
  3228. ctx = task->perf_event_ctxp[ctxn];
  3229. if (likely(!ctx))
  3230. continue;
  3231. perf_event_context_sched_in(ctx, task);
  3232. }
  3233. if (atomic_read(&nr_switch_events))
  3234. perf_event_switch(task, prev, true);
  3235. if (__this_cpu_read(perf_sched_cb_usages))
  3236. perf_pmu_sched_task(prev, task, true);
  3237. }
  3238. static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
  3239. {
  3240. u64 frequency = event->attr.sample_freq;
  3241. u64 sec = NSEC_PER_SEC;
  3242. u64 divisor, dividend;
  3243. int count_fls, nsec_fls, frequency_fls, sec_fls;
  3244. count_fls = fls64(count);
  3245. nsec_fls = fls64(nsec);
  3246. frequency_fls = fls64(frequency);
  3247. sec_fls = 30;
  3248. /*
  3249. * We got @count in @nsec, with a target of sample_freq HZ
  3250. * the target period becomes:
  3251. *
  3252. * @count * 10^9
  3253. * period = -------------------
  3254. * @nsec * sample_freq
  3255. *
  3256. */
  3257. /*
  3258. * Reduce accuracy by one bit such that @a and @b converge
  3259. * to a similar magnitude.
  3260. */
  3261. #define REDUCE_FLS(a, b) \
  3262. do { \
  3263. if (a##_fls > b##_fls) { \
  3264. a >>= 1; \
  3265. a##_fls--; \
  3266. } else { \
  3267. b >>= 1; \
  3268. b##_fls--; \
  3269. } \
  3270. } while (0)
  3271. /*
  3272. * Reduce accuracy until either term fits in a u64, then proceed with
  3273. * the other, so that finally we can do a u64/u64 division.
  3274. */
  3275. while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
  3276. REDUCE_FLS(nsec, frequency);
  3277. REDUCE_FLS(sec, count);
  3278. }
  3279. if (count_fls + sec_fls > 64) {
  3280. divisor = nsec * frequency;
  3281. while (count_fls + sec_fls > 64) {
  3282. REDUCE_FLS(count, sec);
  3283. divisor >>= 1;
  3284. }
  3285. dividend = count * sec;
  3286. } else {
  3287. dividend = count * sec;
  3288. while (nsec_fls + frequency_fls > 64) {
  3289. REDUCE_FLS(nsec, frequency);
  3290. dividend >>= 1;
  3291. }
  3292. divisor = nsec * frequency;
  3293. }
  3294. if (!divisor)
  3295. return dividend;
  3296. return div64_u64(dividend, divisor);
  3297. }
  3298. static DEFINE_PER_CPU(int, perf_throttled_count);
  3299. static DEFINE_PER_CPU(u64, perf_throttled_seq);
  3300. static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable)
  3301. {
  3302. struct hw_perf_event *hwc = &event->hw;
  3303. s64 period, sample_period;
  3304. s64 delta;
  3305. period = perf_calculate_period(event, nsec, count);
  3306. delta = (s64)(period - hwc->sample_period);
  3307. delta = (delta + 7) / 8; /* low pass filter */
  3308. sample_period = hwc->sample_period + delta;
  3309. if (!sample_period)
  3310. sample_period = 1;
  3311. hwc->sample_period = sample_period;
  3312. if (local64_read(&hwc->period_left) > 8*sample_period) {
  3313. if (disable)
  3314. event->pmu->stop(event, PERF_EF_UPDATE);
  3315. local64_set(&hwc->period_left, 0);
  3316. if (disable)
  3317. event->pmu->start(event, PERF_EF_RELOAD);
  3318. }
  3319. }
  3320. /*
  3321. * combine freq adjustment with unthrottling to avoid two passes over the
  3322. * events. At the same time, make sure, having freq events does not change
  3323. * the rate of unthrottling as that would introduce bias.
  3324. */
  3325. static void perf_adjust_freq_unthr_context(struct perf_event_context *ctx,
  3326. int needs_unthr)
  3327. {
  3328. struct perf_event *event;
  3329. struct hw_perf_event *hwc;
  3330. u64 now, period = TICK_NSEC;
  3331. s64 delta;
  3332. /*
  3333. * only need to iterate over all events iff:
  3334. * - context have events in frequency mode (needs freq adjust)
  3335. * - there are events to unthrottle on this cpu
  3336. */
  3337. if (!(ctx->nr_freq || needs_unthr))
  3338. return;
  3339. raw_spin_lock(&ctx->lock);
  3340. perf_pmu_disable(ctx->pmu);
  3341. list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
  3342. if (event->state != PERF_EVENT_STATE_ACTIVE)
  3343. continue;
  3344. if (!event_filter_match(event))
  3345. continue;
  3346. perf_pmu_disable(event->pmu);
  3347. hwc = &event->hw;
  3348. if (hwc->interrupts == MAX_INTERRUPTS) {
  3349. hwc->interrupts = 0;
  3350. perf_log_throttle(event, 1);
  3351. event->pmu->start(event, 0);
  3352. }
  3353. if (!event->attr.freq || !event->attr.sample_freq)
  3354. goto next;
  3355. /*
  3356. * stop the event and update event->count
  3357. */
  3358. event->pmu->stop(event, PERF_EF_UPDATE);
  3359. now = local64_read(&event->count);
  3360. delta = now - hwc->freq_count_stamp;
  3361. hwc->freq_count_stamp = now;
  3362. /*
  3363. * restart the event
  3364. * reload only if value has changed
  3365. * we have stopped the event so tell that
  3366. * to perf_adjust_period() to avoid stopping it
  3367. * twice.
  3368. */
  3369. if (delta > 0)
  3370. perf_adjust_period(event, period, delta, false);
  3371. event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
  3372. next:
  3373. perf_pmu_enable(event->pmu);
  3374. }
  3375. perf_pmu_enable(ctx->pmu);
  3376. raw_spin_unlock(&ctx->lock);
  3377. }
  3378. /*
  3379. * Move @event to the tail of the @ctx's elegible events.
  3380. */
  3381. static void rotate_ctx(struct perf_event_context *ctx, struct perf_event *event)
  3382. {
  3383. /*
  3384. * Rotate the first entry last of non-pinned groups. Rotation might be
  3385. * disabled by the inheritance code.
  3386. */
  3387. if (ctx->rotate_disable)
  3388. return;
  3389. perf_event_groups_delete(&ctx->flexible_groups, event);
  3390. perf_event_groups_insert(&ctx->flexible_groups, event);
  3391. }
  3392. /* pick an event from the flexible_groups to rotate */
  3393. static inline struct perf_event *
  3394. ctx_event_to_rotate(struct perf_event_context *ctx)
  3395. {
  3396. struct perf_event *event;
  3397. /* pick the first active flexible event */
  3398. event = list_first_entry_or_null(&ctx->flexible_active,
  3399. struct perf_event, active_list);
  3400. /* if no active flexible event, pick the first event */
  3401. if (!event) {
  3402. event = rb_entry_safe(rb_first(&ctx->flexible_groups.tree),
  3403. typeof(*event), group_node);
  3404. }
  3405. /*
  3406. * Unconditionally clear rotate_necessary; if ctx_flexible_sched_in()
  3407. * finds there are unschedulable events, it will set it again.
  3408. */
  3409. ctx->rotate_necessary = 0;
  3410. return event;
  3411. }
  3412. static bool perf_rotate_context(struct perf_cpu_context *cpuctx)
  3413. {
  3414. struct perf_event *cpu_event = NULL, *task_event = NULL;
  3415. struct perf_event_context *task_ctx = NULL;
  3416. int cpu_rotate, task_rotate;
  3417. /*
  3418. * Since we run this from IRQ context, nobody can install new
  3419. * events, thus the event count values are stable.
  3420. */
  3421. cpu_rotate = cpuctx->ctx.rotate_necessary;
  3422. task_ctx = cpuctx->task_ctx;
  3423. task_rotate = task_ctx ? task_ctx->rotate_necessary : 0;
  3424. if (!(cpu_rotate || task_rotate))
  3425. return false;
  3426. perf_ctx_lock(cpuctx, cpuctx->task_ctx);
  3427. perf_pmu_disable(cpuctx->ctx.pmu);
  3428. if (task_rotate)
  3429. task_event = ctx_event_to_rotate(task_ctx);
  3430. if (cpu_rotate)
  3431. cpu_event = ctx_event_to_rotate(&cpuctx->ctx);
  3432. /*
  3433. * As per the order given at ctx_resched() first 'pop' task flexible
  3434. * and then, if needed CPU flexible.
  3435. */
  3436. if (task_event || (task_ctx && cpu_event))
  3437. ctx_sched_out(task_ctx, cpuctx, EVENT_FLEXIBLE);
  3438. if (cpu_event)
  3439. cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
  3440. if (task_event)
  3441. rotate_ctx(task_ctx, task_event);
  3442. if (cpu_event)
  3443. rotate_ctx(&cpuctx->ctx, cpu_event);
  3444. perf_event_sched_in(cpuctx, task_ctx, current);
  3445. perf_pmu_enable(cpuctx->ctx.pmu);
  3446. perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
  3447. return true;
  3448. }
  3449. void perf_event_task_tick(void)
  3450. {
  3451. struct list_head *head = this_cpu_ptr(&active_ctx_list);
  3452. struct perf_event_context *ctx, *tmp;
  3453. int throttled;
  3454. lockdep_assert_irqs_disabled();
  3455. __this_cpu_inc(perf_throttled_seq);
  3456. throttled = __this_cpu_xchg(perf_throttled_count, 0);
  3457. tick_dep_clear_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS);
  3458. list_for_each_entry_safe(ctx, tmp, head, active_ctx_list)
  3459. perf_adjust_freq_unthr_context(ctx, throttled);
  3460. }
  3461. static int event_enable_on_exec(struct perf_event *event,
  3462. struct perf_event_context *ctx)
  3463. {
  3464. if (!event->attr.enable_on_exec)
  3465. return 0;
  3466. event->attr.enable_on_exec = 0;
  3467. if (event->state >= PERF_EVENT_STATE_INACTIVE)
  3468. return 0;
  3469. perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE);
  3470. return 1;
  3471. }
  3472. /*
  3473. * Enable all of a task's events that have been marked enable-on-exec.
  3474. * This expects task == current.
  3475. */
  3476. static void perf_event_enable_on_exec(int ctxn)
  3477. {
  3478. struct perf_event_context *ctx, *clone_ctx = NULL;
  3479. enum event_type_t event_type = 0;
  3480. struct perf_cpu_context *cpuctx;
  3481. struct perf_event *event;
  3482. unsigned long flags;
  3483. int enabled = 0;
  3484. local_irq_save(flags);
  3485. ctx = current->perf_event_ctxp[ctxn];
  3486. if (!ctx || !ctx->nr_events)
  3487. goto out;
  3488. cpuctx = __get_cpu_context(ctx);
  3489. perf_ctx_lock(cpuctx, ctx);
  3490. ctx_sched_out(ctx, cpuctx, EVENT_TIME);
  3491. list_for_each_entry(event, &ctx->event_list, event_entry) {
  3492. enabled |= event_enable_on_exec(event, ctx);
  3493. event_type |= get_event_type(event);
  3494. }
  3495. /*
  3496. * Unclone and reschedule this context if we enabled any event.
  3497. */
  3498. if (enabled) {
  3499. clone_ctx = unclone_ctx(ctx);
  3500. ctx_resched(cpuctx, ctx, event_type);
  3501. } else {
  3502. ctx_sched_in(ctx, cpuctx, EVENT_TIME, current);
  3503. }
  3504. perf_ctx_unlock(cpuctx, ctx);
  3505. out:
  3506. local_irq_restore(flags);
  3507. if (clone_ctx)
  3508. put_ctx(clone_ctx);
  3509. }
  3510. struct perf_read_data {
  3511. struct perf_event *event;
  3512. bool group;
  3513. int ret;
  3514. };
  3515. static int __perf_event_read_cpu(struct perf_event *event, int event_cpu)
  3516. {
  3517. u16 local_pkg, event_pkg;
  3518. if (event->group_caps & PERF_EV_CAP_READ_ACTIVE_PKG) {
  3519. int local_cpu = smp_processor_id();
  3520. event_pkg = topology_physical_package_id(event_cpu);
  3521. local_pkg = topology_physical_package_id(local_cpu);
  3522. if (event_pkg == local_pkg)
  3523. return local_cpu;
  3524. }
  3525. return event_cpu;
  3526. }
  3527. /*
  3528. * Cross CPU call to read the hardware event
  3529. */
  3530. static void __perf_event_read(void *info)
  3531. {
  3532. struct perf_read_data *data = info;
  3533. struct perf_event *sub, *event = data->event;
  3534. struct perf_event_context *ctx = event->ctx;
  3535. struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
  3536. struct pmu *pmu = event->pmu;
  3537. /*
  3538. * If this is a task context, we need to check whether it is
  3539. * the current task context of this cpu. If not it has been
  3540. * scheduled out before the smp call arrived. In that case
  3541. * event->count would have been updated to a recent sample
  3542. * when the event was scheduled out.
  3543. */
  3544. if (ctx->task && cpuctx->task_ctx != ctx)
  3545. return;
  3546. raw_spin_lock(&ctx->lock);
  3547. if (ctx->is_active & EVENT_TIME) {
  3548. update_context_time(ctx);
  3549. update_cgrp_time_from_event(event);
  3550. }
  3551. perf_event_update_time(event);
  3552. if (data->group)
  3553. perf_event_update_sibling_time(event);
  3554. if (event->state != PERF_EVENT_STATE_ACTIVE)
  3555. goto unlock;
  3556. if (!data->group) {
  3557. pmu->read(event);
  3558. data->ret = 0;
  3559. goto unlock;
  3560. }
  3561. pmu->start_txn(pmu, PERF_PMU_TXN_READ);
  3562. pmu->read(event);
  3563. for_each_sibling_event(sub, event) {
  3564. if (sub->state == PERF_EVENT_STATE_ACTIVE) {
  3565. /*
  3566. * Use sibling's PMU rather than @event's since
  3567. * sibling could be on different (eg: software) PMU.
  3568. */
  3569. sub->pmu->read(sub);
  3570. }
  3571. }
  3572. data->ret = pmu->commit_txn(pmu);
  3573. unlock:
  3574. raw_spin_unlock(&ctx->lock);
  3575. }
  3576. static inline u64 perf_event_count(struct perf_event *event)
  3577. {
  3578. return local64_read(&event->count) + atomic64_read(&event->child_count);
  3579. }
  3580. /*
  3581. * NMI-safe method to read a local event, that is an event that
  3582. * is:
  3583. * - either for the current task, or for this CPU
  3584. * - does not have inherit set, for inherited task events
  3585. * will not be local and we cannot read them atomically
  3586. * - must not have a pmu::count method
  3587. */
  3588. int perf_event_read_local(struct perf_event *event, u64 *value,
  3589. u64 *enabled, u64 *running)
  3590. {
  3591. unsigned long flags;
  3592. int ret = 0;
  3593. /*
  3594. * Disabling interrupts avoids all counter scheduling (context
  3595. * switches, timer based rotation and IPIs).
  3596. */
  3597. local_irq_save(flags);
  3598. /*
  3599. * It must not be an event with inherit set, we cannot read
  3600. * all child counters from atomic context.
  3601. */
  3602. if (event->attr.inherit) {
  3603. ret = -EOPNOTSUPP;
  3604. goto out;
  3605. }
  3606. /* If this is a per-task event, it must be for current */
  3607. if ((event->attach_state & PERF_ATTACH_TASK) &&
  3608. event->hw.target != current) {
  3609. ret = -EINVAL;
  3610. goto out;
  3611. }
  3612. /* If this is a per-CPU event, it must be for this CPU */
  3613. if (!(event->attach_state & PERF_ATTACH_TASK) &&
  3614. event->cpu != smp_processor_id()) {
  3615. ret = -EINVAL;
  3616. goto out;
  3617. }
  3618. /* If this is a pinned event it must be running on this CPU */
  3619. if (event->attr.pinned && event->oncpu != smp_processor_id()) {
  3620. ret = -EBUSY;
  3621. goto out;
  3622. }
  3623. /*
  3624. * If the event is currently on this CPU, its either a per-task event,
  3625. * or local to this CPU. Furthermore it means its ACTIVE (otherwise
  3626. * oncpu == -1).
  3627. */
  3628. if (event->oncpu == smp_processor_id())
  3629. event->pmu->read(event);
  3630. *value = local64_read(&event->count);
  3631. if (enabled || running) {
  3632. u64 now = event->shadow_ctx_time + perf_clock();
  3633. u64 __enabled, __running;
  3634. __perf_update_times(event, now, &__enabled, &__running);
  3635. if (enabled)
  3636. *enabled = __enabled;
  3637. if (running)
  3638. *running = __running;
  3639. }
  3640. out:
  3641. local_irq_restore(flags);
  3642. return ret;
  3643. }
  3644. EXPORT_SYMBOL_GPL(perf_event_read_local);
  3645. static int perf_event_read(struct perf_event *event, bool group)
  3646. {
  3647. enum perf_event_state state = READ_ONCE(event->state);
  3648. int event_cpu, ret = 0;
  3649. /*
  3650. * If event is enabled and currently active on a CPU, update the
  3651. * value in the event structure:
  3652. */
  3653. again:
  3654. if (state == PERF_EVENT_STATE_ACTIVE) {
  3655. struct perf_read_data data;
  3656. /*
  3657. * Orders the ->state and ->oncpu loads such that if we see
  3658. * ACTIVE we must also see the right ->oncpu.
  3659. *
  3660. * Matches the smp_wmb() from event_sched_in().
  3661. */
  3662. smp_rmb();
  3663. event_cpu = READ_ONCE(event->oncpu);
  3664. if ((unsigned)event_cpu >= nr_cpu_ids)
  3665. return 0;
  3666. data = (struct perf_read_data){
  3667. .event = event,
  3668. .group = group,
  3669. .ret = 0,
  3670. };
  3671. preempt_disable();
  3672. event_cpu = __perf_event_read_cpu(event, event_cpu);
  3673. /*
  3674. * Purposely ignore the smp_call_function_single() return
  3675. * value.
  3676. *
  3677. * If event_cpu isn't a valid CPU it means the event got
  3678. * scheduled out and that will have updated the event count.
  3679. *
  3680. * Therefore, either way, we'll have an up-to-date event count
  3681. * after this.
  3682. */
  3683. (void)smp_call_function_single(event_cpu, __perf_event_read, &data, 1);
  3684. preempt_enable();
  3685. ret = data.ret;
  3686. } else if (state == PERF_EVENT_STATE_INACTIVE) {
  3687. struct perf_event_context *ctx = event->ctx;
  3688. unsigned long flags;
  3689. raw_spin_lock_irqsave(&ctx->lock, flags);
  3690. state = event->state;
  3691. if (state != PERF_EVENT_STATE_INACTIVE) {
  3692. raw_spin_unlock_irqrestore(&ctx->lock, flags);
  3693. goto again;
  3694. }
  3695. /*
  3696. * May read while context is not active (e.g., thread is
  3697. * blocked), in that case we cannot update context time
  3698. */
  3699. if (ctx->is_active & EVENT_TIME) {
  3700. update_context_time(ctx);
  3701. update_cgrp_time_from_event(event);
  3702. }
  3703. perf_event_update_time(event);
  3704. if (group)
  3705. perf_event_update_sibling_time(event);
  3706. raw_spin_unlock_irqrestore(&ctx->lock, flags);
  3707. }
  3708. return ret;
  3709. }
  3710. /*
  3711. * Initialize the perf_event context in a task_struct:
  3712. */
  3713. static void __perf_event_init_context(struct perf_event_context *ctx)
  3714. {
  3715. raw_spin_lock_init(&ctx->lock);
  3716. mutex_init(&ctx->mutex);
  3717. INIT_LIST_HEAD(&ctx->active_ctx_list);
  3718. perf_event_groups_init(&ctx->pinned_groups);
  3719. perf_event_groups_init(&ctx->flexible_groups);
  3720. INIT_LIST_HEAD(&ctx->event_list);
  3721. INIT_LIST_HEAD(&ctx->pinned_active);
  3722. INIT_LIST_HEAD(&ctx->flexible_active);
  3723. refcount_set(&ctx->refcount, 1);
  3724. }
  3725. static struct perf_event_context *
  3726. alloc_perf_context(struct pmu *pmu, struct task_struct *task)
  3727. {
  3728. struct perf_event_context *ctx;
  3729. ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
  3730. if (!ctx)
  3731. return NULL;
  3732. __perf_event_init_context(ctx);
  3733. if (task)
  3734. ctx->task = get_task_struct(task);
  3735. ctx->pmu = pmu;
  3736. return ctx;
  3737. }
  3738. static struct task_struct *
  3739. find_lively_task_by_vpid(pid_t vpid)
  3740. {
  3741. struct task_struct *task;
  3742. rcu_read_lock();
  3743. if (!vpid)
  3744. task = current;
  3745. else
  3746. task = find_task_by_vpid(vpid);
  3747. if (task)
  3748. get_task_struct(task);
  3749. rcu_read_unlock();
  3750. if (!task)
  3751. return ERR_PTR(-ESRCH);
  3752. return task;
  3753. }
  3754. /*
  3755. * Returns a matching context with refcount and pincount.
  3756. */
  3757. static struct perf_event_context *
  3758. find_get_context(struct pmu *pmu, struct task_struct *task,
  3759. struct perf_event *event)
  3760. {
  3761. struct perf_event_context *ctx, *clone_ctx = NULL;
  3762. struct perf_cpu_context *cpuctx;
  3763. void *task_ctx_data = NULL;
  3764. unsigned long flags;
  3765. int ctxn, err;
  3766. int cpu = event->cpu;
  3767. if (!task) {
  3768. /* Must be root to operate on a CPU event: */
  3769. err = perf_allow_cpu(&event->attr);
  3770. if (err)
  3771. return ERR_PTR(err);
  3772. cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
  3773. ctx = &cpuctx->ctx;
  3774. get_ctx(ctx);
  3775. raw_spin_lock_irqsave(&ctx->lock, flags);
  3776. ++ctx->pin_count;
  3777. raw_spin_unlock_irqrestore(&ctx->lock, flags);
  3778. return ctx;
  3779. }
  3780. err = -EINVAL;
  3781. ctxn = pmu->task_ctx_nr;
  3782. if (ctxn < 0)
  3783. goto errout;
  3784. if (event->attach_state & PERF_ATTACH_TASK_DATA) {
  3785. task_ctx_data = alloc_task_ctx_data(pmu);
  3786. if (!task_ctx_data) {
  3787. err = -ENOMEM;
  3788. goto errout;
  3789. }
  3790. }
  3791. retry:
  3792. ctx = perf_lock_task_context(task, ctxn, &flags);
  3793. if (ctx) {
  3794. clone_ctx = unclone_ctx(ctx);
  3795. ++ctx->pin_count;
  3796. if (task_ctx_data && !ctx->task_ctx_data) {
  3797. ctx->task_ctx_data = task_ctx_data;
  3798. task_ctx_data = NULL;
  3799. }
  3800. raw_spin_unlock_irqrestore(&ctx->lock, flags);
  3801. if (clone_ctx)
  3802. put_ctx(clone_ctx);
  3803. } else {
  3804. ctx = alloc_perf_context(pmu, task);
  3805. err = -ENOMEM;
  3806. if (!ctx)
  3807. goto errout;
  3808. if (task_ctx_data) {
  3809. ctx->task_ctx_data = task_ctx_data;
  3810. task_ctx_data = NULL;
  3811. }
  3812. err = 0;
  3813. mutex_lock(&task->perf_event_mutex);
  3814. /*
  3815. * If it has already passed perf_event_exit_task().
  3816. * we must see PF_EXITING, it takes this mutex too.
  3817. */
  3818. if (task->flags & PF_EXITING)
  3819. err = -ESRCH;
  3820. else if (task->perf_event_ctxp[ctxn])
  3821. err = -EAGAIN;
  3822. else {
  3823. get_ctx(ctx);
  3824. ++ctx->pin_count;
  3825. rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx);
  3826. }
  3827. mutex_unlock(&task->perf_event_mutex);
  3828. if (unlikely(err)) {
  3829. put_ctx(ctx);
  3830. if (err == -EAGAIN)
  3831. goto retry;
  3832. goto errout;
  3833. }
  3834. }
  3835. free_task_ctx_data(pmu, task_ctx_data);
  3836. return ctx;
  3837. errout:
  3838. free_task_ctx_data(pmu, task_ctx_data);
  3839. return ERR_PTR(err);
  3840. }
  3841. static void perf_event_free_filter(struct perf_event *event);
  3842. static void perf_event_free_bpf_prog(struct perf_event *event);
  3843. static void free_event_rcu(struct rcu_head *head)
  3844. {
  3845. struct perf_event *event;
  3846. event = container_of(head, struct perf_event, rcu_head);
  3847. if (event->ns)
  3848. put_pid_ns(event->ns);
  3849. perf_event_free_filter(event);
  3850. kfree(event);
  3851. }
  3852. static void ring_buffer_attach(struct perf_event *event,
  3853. struct perf_buffer *rb);
  3854. static void detach_sb_event(struct perf_event *event)
  3855. {
  3856. struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu);
  3857. raw_spin_lock(&pel->lock);
  3858. list_del_rcu(&event->sb_list);
  3859. raw_spin_unlock(&pel->lock);
  3860. }
  3861. static bool is_sb_event(struct perf_event *event)
  3862. {
  3863. struct perf_event_attr *attr = &event->attr;
  3864. if (event->parent)
  3865. return false;
  3866. if (event->attach_state & PERF_ATTACH_TASK)
  3867. return false;
  3868. if (attr->mmap || attr->mmap_data || attr->mmap2 ||
  3869. attr->comm || attr->comm_exec ||
  3870. attr->task || attr->ksymbol ||
  3871. attr->context_switch || attr->text_poke ||
  3872. attr->bpf_event)
  3873. return true;
  3874. return false;
  3875. }
  3876. static void unaccount_pmu_sb_event(struct perf_event *event)
  3877. {
  3878. if (is_sb_event(event))
  3879. detach_sb_event(event);
  3880. }
  3881. static void unaccount_event_cpu(struct perf_event *event, int cpu)
  3882. {
  3883. if (event->parent)
  3884. return;
  3885. if (is_cgroup_event(event))
  3886. atomic_dec(&per_cpu(perf_cgroup_events, cpu));
  3887. }
  3888. #ifdef CONFIG_NO_HZ_FULL
  3889. static DEFINE_SPINLOCK(nr_freq_lock);
  3890. #endif
  3891. static void unaccount_freq_event_nohz(void)
  3892. {
  3893. #ifdef CONFIG_NO_HZ_FULL
  3894. spin_lock(&nr_freq_lock);
  3895. if (atomic_dec_and_test(&nr_freq_events))
  3896. tick_nohz_dep_clear(TICK_DEP_BIT_PERF_EVENTS);
  3897. spin_unlock(&nr_freq_lock);
  3898. #endif
  3899. }
  3900. static void unaccount_freq_event(void)
  3901. {
  3902. if (tick_nohz_full_enabled())
  3903. unaccount_freq_event_nohz();
  3904. else
  3905. atomic_dec(&nr_freq_events);
  3906. }
  3907. static void unaccount_event(struct perf_event *event)
  3908. {
  3909. bool dec = false;
  3910. if (event->parent)
  3911. return;
  3912. if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB))
  3913. dec = true;
  3914. if (event->attr.mmap || event->attr.mmap_data)
  3915. atomic_dec(&nr_mmap_events);
  3916. if (event->attr.comm)
  3917. atomic_dec(&nr_comm_events);
  3918. if (event->attr.namespaces)
  3919. atomic_dec(&nr_namespaces_events);
  3920. if (event->attr.cgroup)
  3921. atomic_dec(&nr_cgroup_events);
  3922. if (event->attr.task)
  3923. atomic_dec(&nr_task_events);
  3924. if (event->attr.freq)
  3925. unaccount_freq_event();
  3926. if (event->attr.context_switch) {
  3927. dec = true;
  3928. atomic_dec(&nr_switch_events);
  3929. }
  3930. if (is_cgroup_event(event))
  3931. dec = true;
  3932. if (has_branch_stack(event))
  3933. dec = true;
  3934. if (event->attr.ksymbol)
  3935. atomic_dec(&nr_ksymbol_events);
  3936. if (event->attr.bpf_event)
  3937. atomic_dec(&nr_bpf_events);
  3938. if (event->attr.text_poke)
  3939. atomic_dec(&nr_text_poke_events);
  3940. if (dec) {
  3941. if (!atomic_add_unless(&perf_sched_count, -1, 1))
  3942. schedule_delayed_work(&perf_sched_work, HZ);
  3943. }
  3944. unaccount_event_cpu(event, event->cpu);
  3945. unaccount_pmu_sb_event(event);
  3946. }
  3947. static void perf_sched_delayed(struct work_struct *work)
  3948. {
  3949. mutex_lock(&perf_sched_mutex);
  3950. if (atomic_dec_and_test(&perf_sched_count))
  3951. static_branch_disable(&perf_sched_events);
  3952. mutex_unlock(&perf_sched_mutex);
  3953. }
  3954. /*
  3955. * The following implement mutual exclusion of events on "exclusive" pmus
  3956. * (PERF_PMU_CAP_EXCLUSIVE). Such pmus can only have one event scheduled
  3957. * at a time, so we disallow creating events that might conflict, namely:
  3958. *
  3959. * 1) cpu-wide events in the presence of per-task events,
  3960. * 2) per-task events in the presence of cpu-wide events,
  3961. * 3) two matching events on the same context.
  3962. *
  3963. * The former two cases are handled in the allocation path (perf_event_alloc(),
  3964. * _free_event()), the latter -- before the first perf_install_in_context().
  3965. */
  3966. static int exclusive_event_init(struct perf_event *event)
  3967. {
  3968. struct pmu *pmu = event->pmu;
  3969. if (!is_exclusive_pmu(pmu))
  3970. return 0;
  3971. /*
  3972. * Prevent co-existence of per-task and cpu-wide events on the
  3973. * same exclusive pmu.
  3974. *
  3975. * Negative pmu::exclusive_cnt means there are cpu-wide
  3976. * events on this "exclusive" pmu, positive means there are
  3977. * per-task events.
  3978. *
  3979. * Since this is called in perf_event_alloc() path, event::ctx
  3980. * doesn't exist yet; it is, however, safe to use PERF_ATTACH_TASK
  3981. * to mean "per-task event", because unlike other attach states it
  3982. * never gets cleared.
  3983. */
  3984. if (event->attach_state & PERF_ATTACH_TASK) {
  3985. if (!atomic_inc_unless_negative(&pmu->exclusive_cnt))
  3986. return -EBUSY;
  3987. } else {
  3988. if (!atomic_dec_unless_positive(&pmu->exclusive_cnt))
  3989. return -EBUSY;
  3990. }
  3991. return 0;
  3992. }
  3993. static void exclusive_event_destroy(struct perf_event *event)
  3994. {
  3995. struct pmu *pmu = event->pmu;
  3996. if (!is_exclusive_pmu(pmu))
  3997. return;
  3998. /* see comment in exclusive_event_init() */
  3999. if (event->attach_state & PERF_ATTACH_TASK)
  4000. atomic_dec(&pmu->exclusive_cnt);
  4001. else
  4002. atomic_inc(&pmu->exclusive_cnt);
  4003. }
  4004. static bool exclusive_event_match(struct perf_event *e1, struct perf_event *e2)
  4005. {
  4006. if ((e1->pmu == e2->pmu) &&
  4007. (e1->cpu == e2->cpu ||
  4008. e1->cpu == -1 ||
  4009. e2->cpu == -1))
  4010. return true;
  4011. return false;
  4012. }
  4013. static bool exclusive_event_installable(struct perf_event *event,
  4014. struct perf_event_context *ctx)
  4015. {
  4016. struct perf_event *iter_event;
  4017. struct pmu *pmu = event->pmu;
  4018. lockdep_assert_held(&ctx->mutex);
  4019. if (!is_exclusive_pmu(pmu))
  4020. return true;
  4021. list_for_each_entry(iter_event, &ctx->event_list, event_entry) {
  4022. if (exclusive_event_match(iter_event, event))
  4023. return false;
  4024. }
  4025. return true;
  4026. }
  4027. static void perf_addr_filters_splice(struct perf_event *event,
  4028. struct list_head *head);
  4029. static void _free_event(struct perf_event *event)
  4030. {
  4031. irq_work_sync(&event->pending);
  4032. unaccount_event(event);
  4033. security_perf_event_free(event);
  4034. if (event->rb) {
  4035. /*
  4036. * Can happen when we close an event with re-directed output.
  4037. *
  4038. * Since we have a 0 refcount, perf_mmap_close() will skip
  4039. * over us; possibly making our ring_buffer_put() the last.
  4040. */
  4041. mutex_lock(&event->mmap_mutex);
  4042. ring_buffer_attach(event, NULL);
  4043. mutex_unlock(&event->mmap_mutex);
  4044. }
  4045. if (is_cgroup_event(event))
  4046. perf_detach_cgroup(event);
  4047. if (!event->parent) {
  4048. if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
  4049. put_callchain_buffers();
  4050. }
  4051. perf_event_free_bpf_prog(event);
  4052. perf_addr_filters_splice(event, NULL);
  4053. kfree(event->addr_filter_ranges);
  4054. if (event->destroy)
  4055. event->destroy(event);
  4056. /*
  4057. * Must be after ->destroy(), due to uprobe_perf_close() using
  4058. * hw.target.
  4059. */
  4060. if (event->hw.target)
  4061. put_task_struct(event->hw.target);
  4062. /*
  4063. * perf_event_free_task() relies on put_ctx() being 'last', in particular
  4064. * all task references must be cleaned up.
  4065. */
  4066. if (event->ctx)
  4067. put_ctx(event->ctx);
  4068. exclusive_event_destroy(event);
  4069. module_put(event->pmu->module);
  4070. call_rcu(&event->rcu_head, free_event_rcu);
  4071. }
  4072. /*
  4073. * Used to free events which have a known refcount of 1, such as in error paths
  4074. * where the event isn't exposed yet and inherited events.
  4075. */
  4076. static void free_event(struct perf_event *event)
  4077. {
  4078. if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1,
  4079. "unexpected event refcount: %ld; ptr=%p\n",
  4080. atomic_long_read(&event->refcount), event)) {
  4081. /* leak to avoid use-after-free */
  4082. return;
  4083. }
  4084. _free_event(event);
  4085. }
  4086. /*
  4087. * Remove user event from the owner task.
  4088. */
  4089. static void perf_remove_from_owner(struct perf_event *event)
  4090. {
  4091. struct task_struct *owner;
  4092. rcu_read_lock();
  4093. /*
  4094. * Matches the smp_store_release() in perf_event_exit_task(). If we
  4095. * observe !owner it means the list deletion is complete and we can
  4096. * indeed free this event, otherwise we need to serialize on
  4097. * owner->perf_event_mutex.
  4098. */
  4099. owner = READ_ONCE(event->owner);
  4100. if (owner) {
  4101. /*
  4102. * Since delayed_put_task_struct() also drops the last
  4103. * task reference we can safely take a new reference
  4104. * while holding the rcu_read_lock().
  4105. */
  4106. get_task_struct(owner);
  4107. }
  4108. rcu_read_unlock();
  4109. if (owner) {
  4110. /*
  4111. * If we're here through perf_event_exit_task() we're already
  4112. * holding ctx->mutex which would be an inversion wrt. the
  4113. * normal lock order.
  4114. *
  4115. * However we can safely take this lock because its the child
  4116. * ctx->mutex.
  4117. */
  4118. mutex_lock_nested(&owner->perf_event_mutex, SINGLE_DEPTH_NESTING);
  4119. /*
  4120. * We have to re-check the event->owner field, if it is cleared
  4121. * we raced with perf_event_exit_task(), acquiring the mutex
  4122. * ensured they're done, and we can proceed with freeing the
  4123. * event.
  4124. */
  4125. if (event->owner) {
  4126. list_del_init(&event->owner_entry);
  4127. smp_store_release(&event->owner, NULL);
  4128. }
  4129. mutex_unlock(&owner->perf_event_mutex);
  4130. put_task_struct(owner);
  4131. }
  4132. }
  4133. static void put_event(struct perf_event *event)
  4134. {
  4135. if (!atomic_long_dec_and_test(&event->refcount))
  4136. return;
  4137. _free_event(event);
  4138. }
  4139. /*
  4140. * Kill an event dead; while event:refcount will preserve the event
  4141. * object, it will not preserve its functionality. Once the last 'user'
  4142. * gives up the object, we'll destroy the thing.
  4143. */
  4144. int perf_event_release_kernel(struct perf_event *event)
  4145. {
  4146. struct perf_event_context *ctx = event->ctx;
  4147. struct perf_event *child, *tmp;
  4148. LIST_HEAD(free_list);
  4149. /*
  4150. * If we got here through err_file: fput(event_file); we will not have
  4151. * attached to a context yet.
  4152. */
  4153. if (!ctx) {
  4154. WARN_ON_ONCE(event->attach_state &
  4155. (PERF_ATTACH_CONTEXT|PERF_ATTACH_GROUP));
  4156. goto no_ctx;
  4157. }
  4158. if (!is_kernel_event(event))
  4159. perf_remove_from_owner(event);
  4160. ctx = perf_event_ctx_lock(event);
  4161. WARN_ON_ONCE(ctx->parent_ctx);
  4162. perf_remove_from_context(event, DETACH_GROUP);
  4163. raw_spin_lock_irq(&ctx->lock);
  4164. /*
  4165. * Mark this event as STATE_DEAD, there is no external reference to it
  4166. * anymore.
  4167. *
  4168. * Anybody acquiring event->child_mutex after the below loop _must_
  4169. * also see this, most importantly inherit_event() which will avoid
  4170. * placing more children on the list.
  4171. *
  4172. * Thus this guarantees that we will in fact observe and kill _ALL_
  4173. * child events.
  4174. */
  4175. event->state = PERF_EVENT_STATE_DEAD;
  4176. raw_spin_unlock_irq(&ctx->lock);
  4177. perf_event_ctx_unlock(event, ctx);
  4178. again:
  4179. mutex_lock(&event->child_mutex);
  4180. list_for_each_entry(child, &event->child_list, child_list) {
  4181. /*
  4182. * Cannot change, child events are not migrated, see the
  4183. * comment with perf_event_ctx_lock_nested().
  4184. */
  4185. ctx = READ_ONCE(child->ctx);
  4186. /*
  4187. * Since child_mutex nests inside ctx::mutex, we must jump
  4188. * through hoops. We start by grabbing a reference on the ctx.
  4189. *
  4190. * Since the event cannot get freed while we hold the
  4191. * child_mutex, the context must also exist and have a !0
  4192. * reference count.
  4193. */
  4194. get_ctx(ctx);
  4195. /*
  4196. * Now that we have a ctx ref, we can drop child_mutex, and
  4197. * acquire ctx::mutex without fear of it going away. Then we
  4198. * can re-acquire child_mutex.
  4199. */
  4200. mutex_unlock(&event->child_mutex);
  4201. mutex_lock(&ctx->mutex);
  4202. mutex_lock(&event->child_mutex);
  4203. /*
  4204. * Now that we hold ctx::mutex and child_mutex, revalidate our
  4205. * state, if child is still the first entry, it didn't get freed
  4206. * and we can continue doing so.
  4207. */
  4208. tmp = list_first_entry_or_null(&event->child_list,
  4209. struct perf_event, child_list);
  4210. if (tmp == child) {
  4211. perf_remove_from_context(child, DETACH_GROUP);
  4212. list_move(&child->child_list, &free_list);
  4213. /*
  4214. * This matches the refcount bump in inherit_event();
  4215. * this can't be the last reference.
  4216. */
  4217. put_event(event);
  4218. }
  4219. mutex_unlock(&event->child_mutex);
  4220. mutex_unlock(&ctx->mutex);
  4221. put_ctx(ctx);
  4222. goto again;
  4223. }
  4224. mutex_unlock(&event->child_mutex);
  4225. list_for_each_entry_safe(child, tmp, &free_list, child_list) {
  4226. void *var = &child->ctx->refcount;
  4227. list_del(&child->child_list);
  4228. free_event(child);
  4229. /*
  4230. * Wake any perf_event_free_task() waiting for this event to be
  4231. * freed.
  4232. */
  4233. smp_mb(); /* pairs with wait_var_event() */
  4234. wake_up_var(var);
  4235. }
  4236. no_ctx:
  4237. put_event(event); /* Must be the 'last' reference */
  4238. return 0;
  4239. }
  4240. EXPORT_SYMBOL_GPL(perf_event_release_kernel);
  4241. /*
  4242. * Called when the last reference to the file is gone.
  4243. */
  4244. static int perf_release(struct inode *inode, struct file *file)
  4245. {
  4246. perf_event_release_kernel(file->private_data);
  4247. return 0;
  4248. }
  4249. static u64 __perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
  4250. {
  4251. struct perf_event *child;
  4252. u64 total = 0;
  4253. *enabled = 0;
  4254. *running = 0;
  4255. mutex_lock(&event->child_mutex);
  4256. (void)perf_event_read(event, false);
  4257. total += perf_event_count(event);
  4258. *enabled += event->total_time_enabled +
  4259. atomic64_read(&event->child_total_time_enabled);
  4260. *running += event->total_time_running +
  4261. atomic64_read(&event->child_total_time_running);
  4262. list_for_each_entry(child, &event->child_list, child_list) {
  4263. (void)perf_event_read(child, false);
  4264. total += perf_event_count(child);
  4265. *enabled += child->total_time_enabled;
  4266. *running += child->total_time_running;
  4267. }
  4268. mutex_unlock(&event->child_mutex);
  4269. return total;
  4270. }
  4271. u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
  4272. {
  4273. struct perf_event_context *ctx;
  4274. u64 count;
  4275. ctx = perf_event_ctx_lock(event);
  4276. count = __perf_event_read_value(event, enabled, running);
  4277. perf_event_ctx_unlock(event, ctx);
  4278. return count;
  4279. }
  4280. EXPORT_SYMBOL_GPL(perf_event_read_value);
  4281. static int __perf_read_group_add(struct perf_event *leader,
  4282. u64 read_format, u64 *values)
  4283. {
  4284. struct perf_event_context *ctx = leader->ctx;
  4285. struct perf_event *sub;
  4286. unsigned long flags;
  4287. int n = 1; /* skip @nr */
  4288. int ret;
  4289. ret = perf_event_read(leader, true);
  4290. if (ret)
  4291. return ret;
  4292. raw_spin_lock_irqsave(&ctx->lock, flags);
  4293. /*
  4294. * Since we co-schedule groups, {enabled,running} times of siblings
  4295. * will be identical to those of the leader, so we only publish one
  4296. * set.
  4297. */
  4298. if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
  4299. values[n++] += leader->total_time_enabled +
  4300. atomic64_read(&leader->child_total_time_enabled);
  4301. }
  4302. if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
  4303. values[n++] += leader->total_time_running +
  4304. atomic64_read(&leader->child_total_time_running);
  4305. }
  4306. /*
  4307. * Write {count,id} tuples for every sibling.
  4308. */
  4309. values[n++] += perf_event_count(leader);
  4310. if (read_format & PERF_FORMAT_ID)
  4311. values[n++] = primary_event_id(leader);
  4312. for_each_sibling_event(sub, leader) {
  4313. values[n++] += perf_event_count(sub);
  4314. if (read_format & PERF_FORMAT_ID)
  4315. values[n++] = primary_event_id(sub);
  4316. }
  4317. raw_spin_unlock_irqrestore(&ctx->lock, flags);
  4318. return 0;
  4319. }
  4320. static int perf_read_group(struct perf_event *event,
  4321. u64 read_format, char __user *buf)
  4322. {
  4323. struct perf_event *leader = event->group_leader, *child;
  4324. struct perf_event_context *ctx = leader->ctx;
  4325. int ret;
  4326. u64 *values;
  4327. lockdep_assert_held(&ctx->mutex);
  4328. values = kzalloc(event->read_size, GFP_KERNEL);
  4329. if (!values)
  4330. return -ENOMEM;
  4331. values[0] = 1 + leader->nr_siblings;
  4332. /*
  4333. * By locking the child_mutex of the leader we effectively
  4334. * lock the child list of all siblings.. XXX explain how.
  4335. */
  4336. mutex_lock(&leader->child_mutex);
  4337. ret = __perf_read_group_add(leader, read_format, values);
  4338. if (ret)
  4339. goto unlock;
  4340. list_for_each_entry(child, &leader->child_list, child_list) {
  4341. ret = __perf_read_group_add(child, read_format, values);
  4342. if (ret)
  4343. goto unlock;
  4344. }
  4345. mutex_unlock(&leader->child_mutex);
  4346. ret = event->read_size;
  4347. if (copy_to_user(buf, values, event->read_size))
  4348. ret = -EFAULT;
  4349. goto out;
  4350. unlock:
  4351. mutex_unlock(&leader->child_mutex);
  4352. out:
  4353. kfree(values);
  4354. return ret;
  4355. }
  4356. static int perf_read_one(struct perf_event *event,
  4357. u64 read_format, char __user *buf)
  4358. {
  4359. u64 enabled, running;
  4360. u64 values[4];
  4361. int n = 0;
  4362. values[n++] = __perf_event_read_value(event, &enabled, &running);
  4363. if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
  4364. values[n++] = enabled;
  4365. if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
  4366. values[n++] = running;
  4367. if (read_format & PERF_FORMAT_ID)
  4368. values[n++] = primary_event_id(event);
  4369. if (copy_to_user(buf, values, n * sizeof(u64)))
  4370. return -EFAULT;
  4371. return n * sizeof(u64);
  4372. }
  4373. static bool is_event_hup(struct perf_event *event)
  4374. {
  4375. bool no_children;
  4376. if (event->state > PERF_EVENT_STATE_EXIT)
  4377. return false;
  4378. mutex_lock(&event->child_mutex);
  4379. no_children = list_empty(&event->child_list);
  4380. mutex_unlock(&event->child_mutex);
  4381. return no_children;
  4382. }
  4383. /*
  4384. * Read the performance event - simple non blocking version for now
  4385. */
  4386. static ssize_t
  4387. __perf_read(struct perf_event *event, char __user *buf, size_t count)
  4388. {
  4389. u64 read_format = event->attr.read_format;
  4390. int ret;
  4391. /*
  4392. * Return end-of-file for a read on an event that is in
  4393. * error state (i.e. because it was pinned but it couldn't be
  4394. * scheduled on to the CPU at some point).
  4395. */
  4396. if (event->state == PERF_EVENT_STATE_ERROR)
  4397. return 0;
  4398. if (count < event->read_size)
  4399. return -ENOSPC;
  4400. WARN_ON_ONCE(event->ctx->parent_ctx);
  4401. if (read_format & PERF_FORMAT_GROUP)
  4402. ret = perf_read_group(event, read_format, buf);
  4403. else
  4404. ret = perf_read_one(event, read_format, buf);
  4405. return ret;
  4406. }
  4407. static ssize_t
  4408. perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
  4409. {
  4410. struct perf_event *event = file->private_data;
  4411. struct perf_event_context *ctx;
  4412. int ret;
  4413. ret = security_perf_event_read(event);
  4414. if (ret)
  4415. return ret;
  4416. ctx = perf_event_ctx_lock(event);
  4417. ret = __perf_read(event, buf, count);
  4418. perf_event_ctx_unlock(event, ctx);
  4419. return ret;
  4420. }
  4421. static __poll_t perf_poll(struct file *file, poll_table *wait)
  4422. {
  4423. struct perf_event *event = file->private_data;
  4424. struct perf_buffer *rb;
  4425. __poll_t events = EPOLLHUP;
  4426. poll_wait(file, &event->waitq, wait);
  4427. if (is_event_hup(event))
  4428. return events;
  4429. /*
  4430. * Pin the event->rb by taking event->mmap_mutex; otherwise
  4431. * perf_event_set_output() can swizzle our rb and make us miss wakeups.
  4432. */
  4433. mutex_lock(&event->mmap_mutex);
  4434. rb = event->rb;
  4435. if (rb)
  4436. events = atomic_xchg(&rb->poll, 0);
  4437. mutex_unlock(&event->mmap_mutex);
  4438. return events;
  4439. }
  4440. static void _perf_event_reset(struct perf_event *event)
  4441. {
  4442. (void)perf_event_read(event, false);
  4443. local64_set(&event->count, 0);
  4444. perf_event_update_userpage(event);
  4445. }
  4446. /* Assume it's not an event with inherit set. */
  4447. u64 perf_event_pause(struct perf_event *event, bool reset)
  4448. {
  4449. struct perf_event_context *ctx;
  4450. u64 count;
  4451. ctx = perf_event_ctx_lock(event);
  4452. WARN_ON_ONCE(event->attr.inherit);
  4453. _perf_event_disable(event);
  4454. count = local64_read(&event->count);
  4455. if (reset)
  4456. local64_set(&event->count, 0);
  4457. perf_event_ctx_unlock(event, ctx);
  4458. return count;
  4459. }
  4460. EXPORT_SYMBOL_GPL(perf_event_pause);
  4461. /*
  4462. * Holding the top-level event's child_mutex means that any
  4463. * descendant process that has inherited this event will block
  4464. * in perf_event_exit_event() if it goes to exit, thus satisfying the
  4465. * task existence requirements of perf_event_enable/disable.
  4466. */
  4467. static void perf_event_for_each_child(struct perf_event *event,
  4468. void (*func)(struct perf_event *))
  4469. {
  4470. struct perf_event *child;
  4471. WARN_ON_ONCE(event->ctx->parent_ctx);
  4472. mutex_lock(&event->child_mutex);
  4473. func(event);
  4474. list_for_each_entry(child, &event->child_list, child_list)
  4475. func(child);
  4476. mutex_unlock(&event->child_mutex);
  4477. }
  4478. static void perf_event_for_each(struct perf_event *event,
  4479. void (*func)(struct perf_event *))
  4480. {
  4481. struct perf_event_context *ctx = event->ctx;
  4482. struct perf_event *sibling;
  4483. lockdep_assert_held(&ctx->mutex);
  4484. event = event->group_leader;
  4485. perf_event_for_each_child(event, func);
  4486. for_each_sibling_event(sibling, event)
  4487. perf_event_for_each_child(sibling, func);
  4488. }
  4489. static void __perf_event_period(struct perf_event *event,
  4490. struct perf_cpu_context *cpuctx,
  4491. struct perf_event_context *ctx,
  4492. void *info)
  4493. {
  4494. u64 value = *((u64 *)info);
  4495. bool active;
  4496. if (event->attr.freq) {
  4497. event->attr.sample_freq = value;
  4498. } else {
  4499. event->attr.sample_period = value;
  4500. event->hw.sample_period = value;
  4501. }
  4502. active = (event->state == PERF_EVENT_STATE_ACTIVE);
  4503. if (active) {
  4504. perf_pmu_disable(ctx->pmu);
  4505. /*
  4506. * We could be throttled; unthrottle now to avoid the tick
  4507. * trying to unthrottle while we already re-started the event.
  4508. */
  4509. if (event->hw.interrupts == MAX_INTERRUPTS) {
  4510. event->hw.interrupts = 0;
  4511. perf_log_throttle(event, 1);
  4512. }
  4513. event->pmu->stop(event, PERF_EF_UPDATE);
  4514. }
  4515. local64_set(&event->hw.period_left, 0);
  4516. if (active) {
  4517. event->pmu->start(event, PERF_EF_RELOAD);
  4518. perf_pmu_enable(ctx->pmu);
  4519. }
  4520. }
  4521. static int perf_event_check_period(struct perf_event *event, u64 value)
  4522. {
  4523. return event->pmu->check_period(event, value);
  4524. }
  4525. static int _perf_event_period(struct perf_event *event, u64 value)
  4526. {
  4527. if (!is_sampling_event(event))
  4528. return -EINVAL;
  4529. if (!value)
  4530. return -EINVAL;
  4531. if (event->attr.freq && value > sysctl_perf_event_sample_rate)
  4532. return -EINVAL;
  4533. if (perf_event_check_period(event, value))
  4534. return -EINVAL;
  4535. if (!event->attr.freq && (value & (1ULL << 63)))
  4536. return -EINVAL;
  4537. event_function_call(event, __perf_event_period, &value);
  4538. return 0;
  4539. }
  4540. int perf_event_period(struct perf_event *event, u64 value)
  4541. {
  4542. struct perf_event_context *ctx;
  4543. int ret;
  4544. ctx = perf_event_ctx_lock(event);
  4545. ret = _perf_event_period(event, value);
  4546. perf_event_ctx_unlock(event, ctx);
  4547. return ret;
  4548. }
  4549. EXPORT_SYMBOL_GPL(perf_event_period);
  4550. static const struct file_operations perf_fops;
  4551. static inline int perf_fget_light(int fd, struct fd *p)
  4552. {
  4553. struct fd f = fdget(fd);
  4554. if (!f.file)
  4555. return -EBADF;
  4556. if (f.file->f_op != &perf_fops) {
  4557. fdput(f);
  4558. return -EBADF;
  4559. }
  4560. *p = f;
  4561. return 0;
  4562. }
  4563. static int perf_event_set_output(struct perf_event *event,
  4564. struct perf_event *output_event);
  4565. static int perf_event_set_filter(struct perf_event *event, void __user *arg);
  4566. static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd);
  4567. static int perf_copy_attr(struct perf_event_attr __user *uattr,
  4568. struct perf_event_attr *attr);
  4569. static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned long arg)
  4570. {
  4571. void (*func)(struct perf_event *);
  4572. u32 flags = arg;
  4573. switch (cmd) {
  4574. case PERF_EVENT_IOC_ENABLE:
  4575. func = _perf_event_enable;
  4576. break;
  4577. case PERF_EVENT_IOC_DISABLE:
  4578. func = _perf_event_disable;
  4579. break;
  4580. case PERF_EVENT_IOC_RESET:
  4581. func = _perf_event_reset;
  4582. break;
  4583. case PERF_EVENT_IOC_REFRESH:
  4584. return _perf_event_refresh(event, arg);
  4585. case PERF_EVENT_IOC_PERIOD:
  4586. {
  4587. u64 value;
  4588. if (copy_from_user(&value, (u64 __user *)arg, sizeof(value)))
  4589. return -EFAULT;
  4590. return _perf_event_period(event, value);
  4591. }
  4592. case PERF_EVENT_IOC_ID:
  4593. {
  4594. u64 id = primary_event_id(event);
  4595. if (copy_to_user((void __user *)arg, &id, sizeof(id)))
  4596. return -EFAULT;
  4597. return 0;
  4598. }
  4599. case PERF_EVENT_IOC_SET_OUTPUT:
  4600. {
  4601. int ret;
  4602. if (arg != -1) {
  4603. struct perf_event *output_event;
  4604. struct fd output;
  4605. ret = perf_fget_light(arg, &output);
  4606. if (ret)
  4607. return ret;
  4608. output_event = output.file->private_data;
  4609. ret = perf_event_set_output(event, output_event);
  4610. fdput(output);
  4611. } else {
  4612. ret = perf_event_set_output(event, NULL);
  4613. }
  4614. return ret;
  4615. }
  4616. case PERF_EVENT_IOC_SET_FILTER:
  4617. return perf_event_set_filter(event, (void __user *)arg);
  4618. case PERF_EVENT_IOC_SET_BPF:
  4619. return perf_event_set_bpf_prog(event, arg);
  4620. case PERF_EVENT_IOC_PAUSE_OUTPUT: {
  4621. struct perf_buffer *rb;
  4622. rcu_read_lock();
  4623. rb = rcu_dereference(event->rb);
  4624. if (!rb || !rb->nr_pages) {
  4625. rcu_read_unlock();
  4626. return -EINVAL;
  4627. }
  4628. rb_toggle_paused(rb, !!arg);
  4629. rcu_read_unlock();
  4630. return 0;
  4631. }
  4632. case PERF_EVENT_IOC_QUERY_BPF:
  4633. return perf_event_query_prog_array(event, (void __user *)arg);
  4634. case PERF_EVENT_IOC_MODIFY_ATTRIBUTES: {
  4635. struct perf_event_attr new_attr;
  4636. int err = perf_copy_attr((struct perf_event_attr __user *)arg,
  4637. &new_attr);
  4638. if (err)
  4639. return err;
  4640. return perf_event_modify_attr(event, &new_attr);
  4641. }
  4642. default:
  4643. return -ENOTTY;
  4644. }
  4645. if (flags & PERF_IOC_FLAG_GROUP)
  4646. perf_event_for_each(event, func);
  4647. else
  4648. perf_event_for_each_child(event, func);
  4649. return 0;
  4650. }
  4651. static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
  4652. {
  4653. struct perf_event *event = file->private_data;
  4654. struct perf_event_context *ctx;
  4655. long ret;
  4656. /* Treat ioctl like writes as it is likely a mutating operation. */
  4657. ret = security_perf_event_write(event);
  4658. if (ret)
  4659. return ret;
  4660. ctx = perf_event_ctx_lock(event);
  4661. ret = _perf_ioctl(event, cmd, arg);
  4662. perf_event_ctx_unlock(event, ctx);
  4663. return ret;
  4664. }
  4665. #ifdef CONFIG_COMPAT
  4666. static long perf_compat_ioctl(struct file *file, unsigned int cmd,
  4667. unsigned long arg)
  4668. {
  4669. switch (_IOC_NR(cmd)) {
  4670. case _IOC_NR(PERF_EVENT_IOC_SET_FILTER):
  4671. case _IOC_NR(PERF_EVENT_IOC_ID):
  4672. case _IOC_NR(PERF_EVENT_IOC_QUERY_BPF):
  4673. case _IOC_NR(PERF_EVENT_IOC_MODIFY_ATTRIBUTES):
  4674. /* Fix up pointer size (usually 4 -> 8 in 32-on-64-bit case */
  4675. if (_IOC_SIZE(cmd) == sizeof(compat_uptr_t)) {
  4676. cmd &= ~IOCSIZE_MASK;
  4677. cmd |= sizeof(void *) << IOCSIZE_SHIFT;
  4678. }
  4679. break;
  4680. }
  4681. return perf_ioctl(file, cmd, arg);
  4682. }
  4683. #else
  4684. # define perf_compat_ioctl NULL
  4685. #endif
  4686. int perf_event_task_enable(void)
  4687. {
  4688. struct perf_event_context *ctx;
  4689. struct perf_event *event;
  4690. mutex_lock(&current->perf_event_mutex);
  4691. list_for_each_entry(event, &current->perf_event_list, owner_entry) {
  4692. ctx = perf_event_ctx_lock(event);
  4693. perf_event_for_each_child(event, _perf_event_enable);
  4694. perf_event_ctx_unlock(event, ctx);
  4695. }
  4696. mutex_unlock(&current->perf_event_mutex);
  4697. return 0;
  4698. }
  4699. int perf_event_task_disable(void)
  4700. {
  4701. struct perf_event_context *ctx;
  4702. struct perf_event *event;
  4703. mutex_lock(&current->perf_event_mutex);
  4704. list_for_each_entry(event, &current->perf_event_list, owner_entry) {
  4705. ctx = perf_event_ctx_lock(event);
  4706. perf_event_for_each_child(event, _perf_event_disable);
  4707. perf_event_ctx_unlock(event, ctx);
  4708. }
  4709. mutex_unlock(&current->perf_event_mutex);
  4710. return 0;
  4711. }
  4712. static int perf_event_index(struct perf_event *event)
  4713. {
  4714. if (event->hw.state & PERF_HES_STOPPED)
  4715. return 0;
  4716. if (event->state != PERF_EVENT_STATE_ACTIVE)
  4717. return 0;
  4718. return event->pmu->event_idx(event);
  4719. }
  4720. static void calc_timer_values(struct perf_event *event,
  4721. u64 *now,
  4722. u64 *enabled,
  4723. u64 *running)
  4724. {
  4725. u64 ctx_time;
  4726. *now = perf_clock();
  4727. ctx_time = event->shadow_ctx_time + *now;
  4728. __perf_update_times(event, ctx_time, enabled, running);
  4729. }
  4730. static void perf_event_init_userpage(struct perf_event *event)
  4731. {
  4732. struct perf_event_mmap_page *userpg;
  4733. struct perf_buffer *rb;
  4734. rcu_read_lock();
  4735. rb = rcu_dereference(event->rb);
  4736. if (!rb)
  4737. goto unlock;
  4738. userpg = rb->user_page;
  4739. /* Allow new userspace to detect that bit 0 is deprecated */
  4740. userpg->cap_bit0_is_deprecated = 1;
  4741. userpg->size = offsetof(struct perf_event_mmap_page, __reserved);
  4742. userpg->data_offset = PAGE_SIZE;
  4743. userpg->data_size = perf_data_size(rb);
  4744. unlock:
  4745. rcu_read_unlock();
  4746. }
  4747. void __weak arch_perf_update_userpage(
  4748. struct perf_event *event, struct perf_event_mmap_page *userpg, u64 now)
  4749. {
  4750. }
  4751. /*
  4752. * Callers need to ensure there can be no nesting of this function, otherwise
  4753. * the seqlock logic goes bad. We can not serialize this because the arch
  4754. * code calls this from NMI context.
  4755. */
  4756. void perf_event_update_userpage(struct perf_event *event)
  4757. {
  4758. struct perf_event_mmap_page *userpg;
  4759. struct perf_buffer *rb;
  4760. u64 enabled, running, now;
  4761. rcu_read_lock();
  4762. rb = rcu_dereference(event->rb);
  4763. if (!rb)
  4764. goto unlock;
  4765. /*
  4766. * compute total_time_enabled, total_time_running
  4767. * based on snapshot values taken when the event
  4768. * was last scheduled in.
  4769. *
  4770. * we cannot simply called update_context_time()
  4771. * because of locking issue as we can be called in
  4772. * NMI context
  4773. */
  4774. calc_timer_values(event, &now, &enabled, &running);
  4775. userpg = rb->user_page;
  4776. /*
  4777. * Disable preemption to guarantee consistent time stamps are stored to
  4778. * the user page.
  4779. */
  4780. preempt_disable();
  4781. ++userpg->lock;
  4782. barrier();
  4783. userpg->index = perf_event_index(event);
  4784. userpg->offset = perf_event_count(event);
  4785. if (userpg->index)
  4786. userpg->offset -= local64_read(&event->hw.prev_count);
  4787. userpg->time_enabled = enabled +
  4788. atomic64_read(&event->child_total_time_enabled);
  4789. userpg->time_running = running +
  4790. atomic64_read(&event->child_total_time_running);
  4791. arch_perf_update_userpage(event, userpg, now);
  4792. barrier();
  4793. ++userpg->lock;
  4794. preempt_enable();
  4795. unlock:
  4796. rcu_read_unlock();
  4797. }
  4798. EXPORT_SYMBOL_GPL(perf_event_update_userpage);
  4799. static vm_fault_t perf_mmap_fault(struct vm_fault *vmf)
  4800. {
  4801. struct perf_event *event = vmf->vma->vm_file->private_data;
  4802. struct perf_buffer *rb;
  4803. vm_fault_t ret = VM_FAULT_SIGBUS;
  4804. if (vmf->flags & FAULT_FLAG_MKWRITE) {
  4805. if (vmf->pgoff == 0)
  4806. ret = 0;
  4807. return ret;
  4808. }
  4809. rcu_read_lock();
  4810. rb = rcu_dereference(event->rb);
  4811. if (!rb)
  4812. goto unlock;
  4813. if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
  4814. goto unlock;
  4815. vmf->page = perf_mmap_to_page(rb, vmf->pgoff);
  4816. if (!vmf->page)
  4817. goto unlock;
  4818. get_page(vmf->page);
  4819. vmf->page->mapping = vmf->vma->vm_file->f_mapping;
  4820. vmf->page->index = vmf->pgoff;
  4821. ret = 0;
  4822. unlock:
  4823. rcu_read_unlock();
  4824. return ret;
  4825. }
  4826. static void ring_buffer_attach(struct perf_event *event,
  4827. struct perf_buffer *rb)
  4828. {
  4829. struct perf_buffer *old_rb = NULL;
  4830. unsigned long flags;
  4831. WARN_ON_ONCE(event->parent);
  4832. if (event->rb) {
  4833. /*
  4834. * Should be impossible, we set this when removing
  4835. * event->rb_entry and wait/clear when adding event->rb_entry.
  4836. */
  4837. WARN_ON_ONCE(event->rcu_pending);
  4838. old_rb = event->rb;
  4839. spin_lock_irqsave(&old_rb->event_lock, flags);
  4840. list_del_rcu(&event->rb_entry);
  4841. spin_unlock_irqrestore(&old_rb->event_lock, flags);
  4842. event->rcu_batches = get_state_synchronize_rcu();
  4843. event->rcu_pending = 1;
  4844. }
  4845. if (rb) {
  4846. if (event->rcu_pending) {
  4847. cond_synchronize_rcu(event->rcu_batches);
  4848. event->rcu_pending = 0;
  4849. }
  4850. spin_lock_irqsave(&rb->event_lock, flags);
  4851. list_add_rcu(&event->rb_entry, &rb->event_list);
  4852. spin_unlock_irqrestore(&rb->event_lock, flags);
  4853. }
  4854. /*
  4855. * Avoid racing with perf_mmap_close(AUX): stop the event
  4856. * before swizzling the event::rb pointer; if it's getting
  4857. * unmapped, its aux_mmap_count will be 0 and it won't
  4858. * restart. See the comment in __perf_pmu_output_stop().
  4859. *
  4860. * Data will inevitably be lost when set_output is done in
  4861. * mid-air, but then again, whoever does it like this is
  4862. * not in for the data anyway.
  4863. */
  4864. if (has_aux(event))
  4865. perf_event_stop(event, 0);
  4866. rcu_assign_pointer(event->rb, rb);
  4867. if (old_rb) {
  4868. ring_buffer_put(old_rb);
  4869. /*
  4870. * Since we detached before setting the new rb, so that we
  4871. * could attach the new rb, we could have missed a wakeup.
  4872. * Provide it now.
  4873. */
  4874. wake_up_all(&event->waitq);
  4875. }
  4876. }
  4877. static void ring_buffer_wakeup(struct perf_event *event)
  4878. {
  4879. struct perf_buffer *rb;
  4880. if (event->parent)
  4881. event = event->parent;
  4882. rcu_read_lock();
  4883. rb = rcu_dereference(event->rb);
  4884. if (rb) {
  4885. list_for_each_entry_rcu(event, &rb->event_list, rb_entry)
  4886. wake_up_all(&event->waitq);
  4887. }
  4888. rcu_read_unlock();
  4889. }
  4890. struct perf_buffer *ring_buffer_get(struct perf_event *event)
  4891. {
  4892. struct perf_buffer *rb;
  4893. if (event->parent)
  4894. event = event->parent;
  4895. rcu_read_lock();
  4896. rb = rcu_dereference(event->rb);
  4897. if (rb) {
  4898. if (!refcount_inc_not_zero(&rb->refcount))
  4899. rb = NULL;
  4900. }
  4901. rcu_read_unlock();
  4902. return rb;
  4903. }
  4904. void ring_buffer_put(struct perf_buffer *rb)
  4905. {
  4906. if (!refcount_dec_and_test(&rb->refcount))
  4907. return;
  4908. WARN_ON_ONCE(!list_empty(&rb->event_list));
  4909. call_rcu(&rb->rcu_head, rb_free_rcu);
  4910. }
  4911. static void perf_mmap_open(struct vm_area_struct *vma)
  4912. {
  4913. struct perf_event *event = vma->vm_file->private_data;
  4914. atomic_inc(&event->mmap_count);
  4915. atomic_inc(&event->rb->mmap_count);
  4916. if (vma->vm_pgoff)
  4917. atomic_inc(&event->rb->aux_mmap_count);
  4918. if (event->pmu->event_mapped)
  4919. event->pmu->event_mapped(event, vma->vm_mm);
  4920. }
  4921. static void perf_pmu_output_stop(struct perf_event *event);
  4922. /*
  4923. * A buffer can be mmap()ed multiple times; either directly through the same
  4924. * event, or through other events by use of perf_event_set_output().
  4925. *
  4926. * In order to undo the VM accounting done by perf_mmap() we need to destroy
  4927. * the buffer here, where we still have a VM context. This means we need
  4928. * to detach all events redirecting to us.
  4929. */
  4930. static void perf_mmap_close(struct vm_area_struct *vma)
  4931. {
  4932. struct perf_event *event = vma->vm_file->private_data;
  4933. struct perf_buffer *rb = ring_buffer_get(event);
  4934. struct user_struct *mmap_user = rb->mmap_user;
  4935. int mmap_locked = rb->mmap_locked;
  4936. unsigned long size = perf_data_size(rb);
  4937. bool detach_rest = false;
  4938. if (event->pmu->event_unmapped)
  4939. event->pmu->event_unmapped(event, vma->vm_mm);
  4940. /*
  4941. * rb->aux_mmap_count will always drop before rb->mmap_count and
  4942. * event->mmap_count, so it is ok to use event->mmap_mutex to
  4943. * serialize with perf_mmap here.
  4944. */
  4945. if (rb_has_aux(rb) && vma->vm_pgoff == rb->aux_pgoff &&
  4946. atomic_dec_and_mutex_lock(&rb->aux_mmap_count, &event->mmap_mutex)) {
  4947. /*
  4948. * Stop all AUX events that are writing to this buffer,
  4949. * so that we can free its AUX pages and corresponding PMU
  4950. * data. Note that after rb::aux_mmap_count dropped to zero,
  4951. * they won't start any more (see perf_aux_output_begin()).
  4952. */
  4953. perf_pmu_output_stop(event);
  4954. /* now it's safe to free the pages */
  4955. atomic_long_sub(rb->aux_nr_pages - rb->aux_mmap_locked, &mmap_user->locked_vm);
  4956. atomic64_sub(rb->aux_mmap_locked, &vma->vm_mm->pinned_vm);
  4957. /* this has to be the last one */
  4958. rb_free_aux(rb);
  4959. WARN_ON_ONCE(refcount_read(&rb->aux_refcount));
  4960. mutex_unlock(&event->mmap_mutex);
  4961. }
  4962. if (atomic_dec_and_test(&rb->mmap_count))
  4963. detach_rest = true;
  4964. if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex))
  4965. goto out_put;
  4966. ring_buffer_attach(event, NULL);
  4967. mutex_unlock(&event->mmap_mutex);
  4968. /* If there's still other mmap()s of this buffer, we're done. */
  4969. if (!detach_rest)
  4970. goto out_put;
  4971. /*
  4972. * No other mmap()s, detach from all other events that might redirect
  4973. * into the now unreachable buffer. Somewhat complicated by the
  4974. * fact that rb::event_lock otherwise nests inside mmap_mutex.
  4975. */
  4976. again:
  4977. rcu_read_lock();
  4978. list_for_each_entry_rcu(event, &rb->event_list, rb_entry) {
  4979. if (!atomic_long_inc_not_zero(&event->refcount)) {
  4980. /*
  4981. * This event is en-route to free_event() which will
  4982. * detach it and remove it from the list.
  4983. */
  4984. continue;
  4985. }
  4986. rcu_read_unlock();
  4987. mutex_lock(&event->mmap_mutex);
  4988. /*
  4989. * Check we didn't race with perf_event_set_output() which can
  4990. * swizzle the rb from under us while we were waiting to
  4991. * acquire mmap_mutex.
  4992. *
  4993. * If we find a different rb; ignore this event, a next
  4994. * iteration will no longer find it on the list. We have to
  4995. * still restart the iteration to make sure we're not now
  4996. * iterating the wrong list.
  4997. */
  4998. if (event->rb == rb)
  4999. ring_buffer_attach(event, NULL);
  5000. mutex_unlock(&event->mmap_mutex);
  5001. put_event(event);
  5002. /*
  5003. * Restart the iteration; either we're on the wrong list or
  5004. * destroyed its integrity by doing a deletion.
  5005. */
  5006. goto again;
  5007. }
  5008. rcu_read_unlock();
  5009. /*
  5010. * It could be there's still a few 0-ref events on the list; they'll
  5011. * get cleaned up by free_event() -- they'll also still have their
  5012. * ref on the rb and will free it whenever they are done with it.
  5013. *
  5014. * Aside from that, this buffer is 'fully' detached and unmapped,
  5015. * undo the VM accounting.
  5016. */
  5017. atomic_long_sub((size >> PAGE_SHIFT) + 1 - mmap_locked,
  5018. &mmap_user->locked_vm);
  5019. atomic64_sub(mmap_locked, &vma->vm_mm->pinned_vm);
  5020. free_uid(mmap_user);
  5021. out_put:
  5022. ring_buffer_put(rb); /* could be last */
  5023. }
  5024. static const struct vm_operations_struct perf_mmap_vmops = {
  5025. .open = perf_mmap_open,
  5026. .close = perf_mmap_close, /* non mergeable */
  5027. .fault = perf_mmap_fault,
  5028. .page_mkwrite = perf_mmap_fault,
  5029. };
  5030. static int perf_mmap(struct file *file, struct vm_area_struct *vma)
  5031. {
  5032. struct perf_event *event = file->private_data;
  5033. unsigned long user_locked, user_lock_limit;
  5034. struct user_struct *user = current_user();
  5035. struct perf_buffer *rb = NULL;
  5036. unsigned long locked, lock_limit;
  5037. unsigned long vma_size;
  5038. unsigned long nr_pages;
  5039. long user_extra = 0, extra = 0;
  5040. int ret = 0, flags = 0;
  5041. /*
  5042. * Don't allow mmap() of inherited per-task counters. This would
  5043. * create a performance issue due to all children writing to the
  5044. * same rb.
  5045. */
  5046. if (event->cpu == -1 && event->attr.inherit)
  5047. return -EINVAL;
  5048. if (!(vma->vm_flags & VM_SHARED))
  5049. return -EINVAL;
  5050. ret = security_perf_event_read(event);
  5051. if (ret)
  5052. return ret;
  5053. vma_size = vma->vm_end - vma->vm_start;
  5054. if (vma->vm_pgoff == 0) {
  5055. nr_pages = (vma_size / PAGE_SIZE) - 1;
  5056. } else {
  5057. /*
  5058. * AUX area mapping: if rb->aux_nr_pages != 0, it's already
  5059. * mapped, all subsequent mappings should have the same size
  5060. * and offset. Must be above the normal perf buffer.
  5061. */
  5062. u64 aux_offset, aux_size;
  5063. if (!event->rb)
  5064. return -EINVAL;
  5065. nr_pages = vma_size / PAGE_SIZE;
  5066. mutex_lock(&event->mmap_mutex);
  5067. ret = -EINVAL;
  5068. rb = event->rb;
  5069. if (!rb)
  5070. goto aux_unlock;
  5071. aux_offset = READ_ONCE(rb->user_page->aux_offset);
  5072. aux_size = READ_ONCE(rb->user_page->aux_size);
  5073. if (aux_offset < perf_data_size(rb) + PAGE_SIZE)
  5074. goto aux_unlock;
  5075. if (aux_offset != vma->vm_pgoff << PAGE_SHIFT)
  5076. goto aux_unlock;
  5077. /* already mapped with a different offset */
  5078. if (rb_has_aux(rb) && rb->aux_pgoff != vma->vm_pgoff)
  5079. goto aux_unlock;
  5080. if (aux_size != vma_size || aux_size != nr_pages * PAGE_SIZE)
  5081. goto aux_unlock;
  5082. /* already mapped with a different size */
  5083. if (rb_has_aux(rb) && rb->aux_nr_pages != nr_pages)
  5084. goto aux_unlock;
  5085. if (!is_power_of_2(nr_pages))
  5086. goto aux_unlock;
  5087. if (!atomic_inc_not_zero(&rb->mmap_count))
  5088. goto aux_unlock;
  5089. if (rb_has_aux(rb)) {
  5090. atomic_inc(&rb->aux_mmap_count);
  5091. ret = 0;
  5092. goto unlock;
  5093. }
  5094. atomic_set(&rb->aux_mmap_count, 1);
  5095. user_extra = nr_pages;
  5096. goto accounting;
  5097. }
  5098. /*
  5099. * If we have rb pages ensure they're a power-of-two number, so we
  5100. * can do bitmasks instead of modulo.
  5101. */
  5102. if (nr_pages != 0 && !is_power_of_2(nr_pages))
  5103. return -EINVAL;
  5104. if (vma_size != PAGE_SIZE * (1 + nr_pages))
  5105. return -EINVAL;
  5106. WARN_ON_ONCE(event->ctx->parent_ctx);
  5107. again:
  5108. mutex_lock(&event->mmap_mutex);
  5109. if (event->rb) {
  5110. if (data_page_nr(event->rb) != nr_pages) {
  5111. ret = -EINVAL;
  5112. goto unlock;
  5113. }
  5114. if (!atomic_inc_not_zero(&event->rb->mmap_count)) {
  5115. /*
  5116. * Raced against perf_mmap_close() through
  5117. * perf_event_set_output(). Try again, hope for better
  5118. * luck.
  5119. */
  5120. mutex_unlock(&event->mmap_mutex);
  5121. goto again;
  5122. }
  5123. goto unlock;
  5124. }
  5125. user_extra = nr_pages + 1;
  5126. accounting:
  5127. user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
  5128. /*
  5129. * Increase the limit linearly with more CPUs:
  5130. */
  5131. user_lock_limit *= num_online_cpus();
  5132. user_locked = atomic_long_read(&user->locked_vm);
  5133. /*
  5134. * sysctl_perf_event_mlock may have changed, so that
  5135. * user->locked_vm > user_lock_limit
  5136. */
  5137. if (user_locked > user_lock_limit)
  5138. user_locked = user_lock_limit;
  5139. user_locked += user_extra;
  5140. if (user_locked > user_lock_limit) {
  5141. /*
  5142. * charge locked_vm until it hits user_lock_limit;
  5143. * charge the rest from pinned_vm
  5144. */
  5145. extra = user_locked - user_lock_limit;
  5146. user_extra -= extra;
  5147. }
  5148. lock_limit = rlimit(RLIMIT_MEMLOCK);
  5149. lock_limit >>= PAGE_SHIFT;
  5150. locked = atomic64_read(&vma->vm_mm->pinned_vm) + extra;
  5151. if ((locked > lock_limit) && perf_is_paranoid() &&
  5152. !capable(CAP_IPC_LOCK)) {
  5153. ret = -EPERM;
  5154. goto unlock;
  5155. }
  5156. WARN_ON(!rb && event->rb);
  5157. if (vma->vm_flags & VM_WRITE)
  5158. flags |= RING_BUFFER_WRITABLE;
  5159. if (!rb) {
  5160. rb = rb_alloc(nr_pages,
  5161. event->attr.watermark ? event->attr.wakeup_watermark : 0,
  5162. event->cpu, flags);
  5163. if (!rb) {
  5164. ret = -ENOMEM;
  5165. goto unlock;
  5166. }
  5167. atomic_set(&rb->mmap_count, 1);
  5168. rb->mmap_user = get_current_user();
  5169. rb->mmap_locked = extra;
  5170. ring_buffer_attach(event, rb);
  5171. perf_event_update_time(event);
  5172. perf_set_shadow_time(event, event->ctx);
  5173. perf_event_init_userpage(event);
  5174. perf_event_update_userpage(event);
  5175. } else {
  5176. ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages,
  5177. event->attr.aux_watermark, flags);
  5178. if (!ret)
  5179. rb->aux_mmap_locked = extra;
  5180. }
  5181. unlock:
  5182. if (!ret) {
  5183. atomic_long_add(user_extra, &user->locked_vm);
  5184. atomic64_add(extra, &vma->vm_mm->pinned_vm);
  5185. atomic_inc(&event->mmap_count);
  5186. } else if (rb) {
  5187. atomic_dec(&rb->mmap_count);
  5188. }
  5189. aux_unlock:
  5190. mutex_unlock(&event->mmap_mutex);
  5191. /*
  5192. * Since pinned accounting is per vm we cannot allow fork() to copy our
  5193. * vma.
  5194. */
  5195. vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP;
  5196. vma->vm_ops = &perf_mmap_vmops;
  5197. if (event->pmu->event_mapped)
  5198. event->pmu->event_mapped(event, vma->vm_mm);
  5199. return ret;
  5200. }
  5201. static int perf_fasync(int fd, struct file *filp, int on)
  5202. {
  5203. struct inode *inode = file_inode(filp);
  5204. struct perf_event *event = filp->private_data;
  5205. int retval;
  5206. inode_lock(inode);
  5207. retval = fasync_helper(fd, filp, on, &event->fasync);
  5208. inode_unlock(inode);
  5209. if (retval < 0)
  5210. return retval;
  5211. return 0;
  5212. }
  5213. static const struct file_operations perf_fops = {
  5214. .llseek = no_llseek,
  5215. .release = perf_release,
  5216. .read = perf_read,
  5217. .poll = perf_poll,
  5218. .unlocked_ioctl = perf_ioctl,
  5219. .compat_ioctl = perf_compat_ioctl,
  5220. .mmap = perf_mmap,
  5221. .fasync = perf_fasync,
  5222. };
  5223. /*
  5224. * Perf event wakeup
  5225. *
  5226. * If there's data, ensure we set the poll() state and publish everything
  5227. * to user-space before waking everybody up.
  5228. */
  5229. static inline struct fasync_struct **perf_event_fasync(struct perf_event *event)
  5230. {
  5231. /* only the parent has fasync state */
  5232. if (event->parent)
  5233. event = event->parent;
  5234. return &event->fasync;
  5235. }
  5236. void perf_event_wakeup(struct perf_event *event)
  5237. {
  5238. ring_buffer_wakeup(event);
  5239. if (event->pending_kill) {
  5240. kill_fasync(perf_event_fasync(event), SIGIO, event->pending_kill);
  5241. event->pending_kill = 0;
  5242. }
  5243. }
  5244. static void perf_pending_event_disable(struct perf_event *event)
  5245. {
  5246. int cpu = READ_ONCE(event->pending_disable);
  5247. if (cpu < 0)
  5248. return;
  5249. if (cpu == smp_processor_id()) {
  5250. WRITE_ONCE(event->pending_disable, -1);
  5251. perf_event_disable_local(event);
  5252. return;
  5253. }
  5254. /*
  5255. * CPU-A CPU-B
  5256. *
  5257. * perf_event_disable_inatomic()
  5258. * @pending_disable = CPU-A;
  5259. * irq_work_queue();
  5260. *
  5261. * sched-out
  5262. * @pending_disable = -1;
  5263. *
  5264. * sched-in
  5265. * perf_event_disable_inatomic()
  5266. * @pending_disable = CPU-B;
  5267. * irq_work_queue(); // FAILS
  5268. *
  5269. * irq_work_run()
  5270. * perf_pending_event()
  5271. *
  5272. * But the event runs on CPU-B and wants disabling there.
  5273. */
  5274. irq_work_queue_on(&event->pending, cpu);
  5275. }
  5276. static void perf_pending_event(struct irq_work *entry)
  5277. {
  5278. struct perf_event *event = container_of(entry, struct perf_event, pending);
  5279. int rctx;
  5280. rctx = perf_swevent_get_recursion_context();
  5281. /*
  5282. * If we 'fail' here, that's OK, it means recursion is already disabled
  5283. * and we won't recurse 'further'.
  5284. */
  5285. perf_pending_event_disable(event);
  5286. if (event->pending_wakeup) {
  5287. event->pending_wakeup = 0;
  5288. perf_event_wakeup(event);
  5289. }
  5290. if (rctx >= 0)
  5291. perf_swevent_put_recursion_context(rctx);
  5292. }
  5293. /*
  5294. * We assume there is only KVM supporting the callbacks.
  5295. * Later on, we might change it to a list if there is
  5296. * another virtualization implementation supporting the callbacks.
  5297. */
  5298. struct perf_guest_info_callbacks __rcu *perf_guest_cbs;
  5299. int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
  5300. {
  5301. if (WARN_ON_ONCE(rcu_access_pointer(perf_guest_cbs)))
  5302. return -EBUSY;
  5303. rcu_assign_pointer(perf_guest_cbs, cbs);
  5304. return 0;
  5305. }
  5306. EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);
  5307. int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
  5308. {
  5309. if (WARN_ON_ONCE(rcu_access_pointer(perf_guest_cbs) != cbs))
  5310. return -EINVAL;
  5311. rcu_assign_pointer(perf_guest_cbs, NULL);
  5312. synchronize_rcu();
  5313. return 0;
  5314. }
  5315. EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);
  5316. static void
  5317. perf_output_sample_regs(struct perf_output_handle *handle,
  5318. struct pt_regs *regs, u64 mask)
  5319. {
  5320. int bit;
  5321. DECLARE_BITMAP(_mask, 64);
  5322. bitmap_from_u64(_mask, mask);
  5323. for_each_set_bit(bit, _mask, sizeof(mask) * BITS_PER_BYTE) {
  5324. u64 val;
  5325. val = perf_reg_value(regs, bit);
  5326. perf_output_put(handle, val);
  5327. }
  5328. }
  5329. static void perf_sample_regs_user(struct perf_regs *regs_user,
  5330. struct pt_regs *regs)
  5331. {
  5332. if (user_mode(regs)) {
  5333. regs_user->abi = perf_reg_abi(current);
  5334. regs_user->regs = regs;
  5335. } else if (!(current->flags & PF_KTHREAD)) {
  5336. perf_get_regs_user(regs_user, regs);
  5337. } else {
  5338. regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE;
  5339. regs_user->regs = NULL;
  5340. }
  5341. }
  5342. static void perf_sample_regs_intr(struct perf_regs *regs_intr,
  5343. struct pt_regs *regs)
  5344. {
  5345. regs_intr->regs = regs;
  5346. regs_intr->abi = perf_reg_abi(current);
  5347. }
  5348. /*
  5349. * Get remaining task size from user stack pointer.
  5350. *
  5351. * It'd be better to take stack vma map and limit this more
  5352. * precisely, but there's no way to get it safely under interrupt,
  5353. * so using TASK_SIZE as limit.
  5354. */
  5355. static u64 perf_ustack_task_size(struct pt_regs *regs)
  5356. {
  5357. unsigned long addr = perf_user_stack_pointer(regs);
  5358. if (!addr || addr >= TASK_SIZE)
  5359. return 0;
  5360. return TASK_SIZE - addr;
  5361. }
  5362. static u16
  5363. perf_sample_ustack_size(u16 stack_size, u16 header_size,
  5364. struct pt_regs *regs)
  5365. {
  5366. u64 task_size;
  5367. /* No regs, no stack pointer, no dump. */
  5368. if (!regs)
  5369. return 0;
  5370. /*
  5371. * Check if we fit in with the requested stack size into the:
  5372. * - TASK_SIZE
  5373. * If we don't, we limit the size to the TASK_SIZE.
  5374. *
  5375. * - remaining sample size
  5376. * If we don't, we customize the stack size to
  5377. * fit in to the remaining sample size.
  5378. */
  5379. task_size = min((u64) USHRT_MAX, perf_ustack_task_size(regs));
  5380. stack_size = min(stack_size, (u16) task_size);
  5381. /* Current header size plus static size and dynamic size. */
  5382. header_size += 2 * sizeof(u64);
  5383. /* Do we fit in with the current stack dump size? */
  5384. if ((u16) (header_size + stack_size) < header_size) {
  5385. /*
  5386. * If we overflow the maximum size for the sample,
  5387. * we customize the stack dump size to fit in.
  5388. */
  5389. stack_size = USHRT_MAX - header_size - sizeof(u64);
  5390. stack_size = round_up(stack_size, sizeof(u64));
  5391. }
  5392. return stack_size;
  5393. }
  5394. static void
  5395. perf_output_sample_ustack(struct perf_output_handle *handle, u64 dump_size,
  5396. struct pt_regs *regs)
  5397. {
  5398. /* Case of a kernel thread, nothing to dump */
  5399. if (!regs) {
  5400. u64 size = 0;
  5401. perf_output_put(handle, size);
  5402. } else {
  5403. unsigned long sp;
  5404. unsigned int rem;
  5405. u64 dyn_size;
  5406. mm_segment_t fs;
  5407. /*
  5408. * We dump:
  5409. * static size
  5410. * - the size requested by user or the best one we can fit
  5411. * in to the sample max size
  5412. * data
  5413. * - user stack dump data
  5414. * dynamic size
  5415. * - the actual dumped size
  5416. */
  5417. /* Static size. */
  5418. perf_output_put(handle, dump_size);
  5419. /* Data. */
  5420. sp = perf_user_stack_pointer(regs);
  5421. fs = force_uaccess_begin();
  5422. rem = __output_copy_user(handle, (void *) sp, dump_size);
  5423. force_uaccess_end(fs);
  5424. dyn_size = dump_size - rem;
  5425. perf_output_skip(handle, rem);
  5426. /* Dynamic size. */
  5427. perf_output_put(handle, dyn_size);
  5428. }
  5429. }
  5430. static unsigned long perf_prepare_sample_aux(struct perf_event *event,
  5431. struct perf_sample_data *data,
  5432. size_t size)
  5433. {
  5434. struct perf_event *sampler = event->aux_event;
  5435. struct perf_buffer *rb;
  5436. data->aux_size = 0;
  5437. if (!sampler)
  5438. goto out;
  5439. if (WARN_ON_ONCE(READ_ONCE(sampler->state) != PERF_EVENT_STATE_ACTIVE))
  5440. goto out;
  5441. if (WARN_ON_ONCE(READ_ONCE(sampler->oncpu) != smp_processor_id()))
  5442. goto out;
  5443. rb = ring_buffer_get(sampler);
  5444. if (!rb)
  5445. goto out;
  5446. /*
  5447. * If this is an NMI hit inside sampling code, don't take
  5448. * the sample. See also perf_aux_sample_output().
  5449. */
  5450. if (READ_ONCE(rb->aux_in_sampling)) {
  5451. data->aux_size = 0;
  5452. } else {
  5453. size = min_t(size_t, size, perf_aux_size(rb));
  5454. data->aux_size = ALIGN(size, sizeof(u64));
  5455. }
  5456. ring_buffer_put(rb);
  5457. out:
  5458. return data->aux_size;
  5459. }
  5460. long perf_pmu_snapshot_aux(struct perf_buffer *rb,
  5461. struct perf_event *event,
  5462. struct perf_output_handle *handle,
  5463. unsigned long size)
  5464. {
  5465. unsigned long flags;
  5466. long ret;
  5467. /*
  5468. * Normal ->start()/->stop() callbacks run in IRQ mode in scheduler
  5469. * paths. If we start calling them in NMI context, they may race with
  5470. * the IRQ ones, that is, for example, re-starting an event that's just
  5471. * been stopped, which is why we're using a separate callback that
  5472. * doesn't change the event state.
  5473. *
  5474. * IRQs need to be disabled to prevent IPIs from racing with us.
  5475. */
  5476. local_irq_save(flags);
  5477. /*
  5478. * Guard against NMI hits inside the critical section;
  5479. * see also perf_prepare_sample_aux().
  5480. */
  5481. WRITE_ONCE(rb->aux_in_sampling, 1);
  5482. barrier();
  5483. ret = event->pmu->snapshot_aux(event, handle, size);
  5484. barrier();
  5485. WRITE_ONCE(rb->aux_in_sampling, 0);
  5486. local_irq_restore(flags);
  5487. return ret;
  5488. }
  5489. static void perf_aux_sample_output(struct perf_event *event,
  5490. struct perf_output_handle *handle,
  5491. struct perf_sample_data *data)
  5492. {
  5493. struct perf_event *sampler = event->aux_event;
  5494. struct perf_buffer *rb;
  5495. unsigned long pad;
  5496. long size;
  5497. if (WARN_ON_ONCE(!sampler || !data->aux_size))
  5498. return;
  5499. rb = ring_buffer_get(sampler);
  5500. if (!rb)
  5501. return;
  5502. size = perf_pmu_snapshot_aux(rb, sampler, handle, data->aux_size);
  5503. /*
  5504. * An error here means that perf_output_copy() failed (returned a
  5505. * non-zero surplus that it didn't copy), which in its current
  5506. * enlightened implementation is not possible. If that changes, we'd
  5507. * like to know.
  5508. */
  5509. if (WARN_ON_ONCE(size < 0))
  5510. goto out_put;
  5511. /*
  5512. * The pad comes from ALIGN()ing data->aux_size up to u64 in
  5513. * perf_prepare_sample_aux(), so should not be more than that.
  5514. */
  5515. pad = data->aux_size - size;
  5516. if (WARN_ON_ONCE(pad >= sizeof(u64)))
  5517. pad = 8;
  5518. if (pad) {
  5519. u64 zero = 0;
  5520. perf_output_copy(handle, &zero, pad);
  5521. }
  5522. out_put:
  5523. ring_buffer_put(rb);
  5524. }
  5525. static void __perf_event_header__init_id(struct perf_event_header *header,
  5526. struct perf_sample_data *data,
  5527. struct perf_event *event)
  5528. {
  5529. u64 sample_type = event->attr.sample_type;
  5530. data->type = sample_type;
  5531. header->size += event->id_header_size;
  5532. if (sample_type & PERF_SAMPLE_TID) {
  5533. /* namespace issues */
  5534. data->tid_entry.pid = perf_event_pid(event, current);
  5535. data->tid_entry.tid = perf_event_tid(event, current);
  5536. }
  5537. if (sample_type & PERF_SAMPLE_TIME)
  5538. data->time = perf_event_clock(event);
  5539. if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER))
  5540. data->id = primary_event_id(event);
  5541. if (sample_type & PERF_SAMPLE_STREAM_ID)
  5542. data->stream_id = event->id;
  5543. if (sample_type & PERF_SAMPLE_CPU) {
  5544. data->cpu_entry.cpu = raw_smp_processor_id();
  5545. data->cpu_entry.reserved = 0;
  5546. }
  5547. }
  5548. void perf_event_header__init_id(struct perf_event_header *header,
  5549. struct perf_sample_data *data,
  5550. struct perf_event *event)
  5551. {
  5552. if (event->attr.sample_id_all)
  5553. __perf_event_header__init_id(header, data, event);
  5554. }
  5555. static void __perf_event__output_id_sample(struct perf_output_handle *handle,
  5556. struct perf_sample_data *data)
  5557. {
  5558. u64 sample_type = data->type;
  5559. if (sample_type & PERF_SAMPLE_TID)
  5560. perf_output_put(handle, data->tid_entry);
  5561. if (sample_type & PERF_SAMPLE_TIME)
  5562. perf_output_put(handle, data->time);
  5563. if (sample_type & PERF_SAMPLE_ID)
  5564. perf_output_put(handle, data->id);
  5565. if (sample_type & PERF_SAMPLE_STREAM_ID)
  5566. perf_output_put(handle, data->stream_id);
  5567. if (sample_type & PERF_SAMPLE_CPU)
  5568. perf_output_put(handle, data->cpu_entry);
  5569. if (sample_type & PERF_SAMPLE_IDENTIFIER)
  5570. perf_output_put(handle, data->id);
  5571. }
  5572. void perf_event__output_id_sample(struct perf_event *event,
  5573. struct perf_output_handle *handle,
  5574. struct perf_sample_data *sample)
  5575. {
  5576. if (event->attr.sample_id_all)
  5577. __perf_event__output_id_sample(handle, sample);
  5578. }
  5579. static void perf_output_read_one(struct perf_output_handle *handle,
  5580. struct perf_event *event,
  5581. u64 enabled, u64 running)
  5582. {
  5583. u64 read_format = event->attr.read_format;
  5584. u64 values[4];
  5585. int n = 0;
  5586. values[n++] = perf_event_count(event);
  5587. if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
  5588. values[n++] = enabled +
  5589. atomic64_read(&event->child_total_time_enabled);
  5590. }
  5591. if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
  5592. values[n++] = running +
  5593. atomic64_read(&event->child_total_time_running);
  5594. }
  5595. if (read_format & PERF_FORMAT_ID)
  5596. values[n++] = primary_event_id(event);
  5597. __output_copy(handle, values, n * sizeof(u64));
  5598. }
  5599. static void perf_output_read_group(struct perf_output_handle *handle,
  5600. struct perf_event *event,
  5601. u64 enabled, u64 running)
  5602. {
  5603. struct perf_event *leader = event->group_leader, *sub;
  5604. u64 read_format = event->attr.read_format;
  5605. u64 values[5];
  5606. int n = 0;
  5607. values[n++] = 1 + leader->nr_siblings;
  5608. if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
  5609. values[n++] = enabled;
  5610. if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
  5611. values[n++] = running;
  5612. if ((leader != event) &&
  5613. (leader->state == PERF_EVENT_STATE_ACTIVE))
  5614. leader->pmu->read(leader);
  5615. values[n++] = perf_event_count(leader);
  5616. if (read_format & PERF_FORMAT_ID)
  5617. values[n++] = primary_event_id(leader);
  5618. __output_copy(handle, values, n * sizeof(u64));
  5619. for_each_sibling_event(sub, leader) {
  5620. n = 0;
  5621. if ((sub != event) &&
  5622. (sub->state == PERF_EVENT_STATE_ACTIVE))
  5623. sub->pmu->read(sub);
  5624. values[n++] = perf_event_count(sub);
  5625. if (read_format & PERF_FORMAT_ID)
  5626. values[n++] = primary_event_id(sub);
  5627. __output_copy(handle, values, n * sizeof(u64));
  5628. }
  5629. }
  5630. #define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\
  5631. PERF_FORMAT_TOTAL_TIME_RUNNING)
  5632. /*
  5633. * XXX PERF_SAMPLE_READ vs inherited events seems difficult.
  5634. *
  5635. * The problem is that its both hard and excessively expensive to iterate the
  5636. * child list, not to mention that its impossible to IPI the children running
  5637. * on another CPU, from interrupt/NMI context.
  5638. */
  5639. static void perf_output_read(struct perf_output_handle *handle,
  5640. struct perf_event *event)
  5641. {
  5642. u64 enabled = 0, running = 0, now;
  5643. u64 read_format = event->attr.read_format;
  5644. /*
  5645. * compute total_time_enabled, total_time_running
  5646. * based on snapshot values taken when the event
  5647. * was last scheduled in.
  5648. *
  5649. * we cannot simply called update_context_time()
  5650. * because of locking issue as we are called in
  5651. * NMI context
  5652. */
  5653. if (read_format & PERF_FORMAT_TOTAL_TIMES)
  5654. calc_timer_values(event, &now, &enabled, &running);
  5655. if (event->attr.read_format & PERF_FORMAT_GROUP)
  5656. perf_output_read_group(handle, event, enabled, running);
  5657. else
  5658. perf_output_read_one(handle, event, enabled, running);
  5659. }
  5660. static inline bool perf_sample_save_hw_index(struct perf_event *event)
  5661. {
  5662. return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX;
  5663. }
  5664. void perf_output_sample(struct perf_output_handle *handle,
  5665. struct perf_event_header *header,
  5666. struct perf_sample_data *data,
  5667. struct perf_event *event)
  5668. {
  5669. u64 sample_type = data->type;
  5670. perf_output_put(handle, *header);
  5671. if (sample_type & PERF_SAMPLE_IDENTIFIER)
  5672. perf_output_put(handle, data->id);
  5673. if (sample_type & PERF_SAMPLE_IP)
  5674. perf_output_put(handle, data->ip);
  5675. if (sample_type & PERF_SAMPLE_TID)
  5676. perf_output_put(handle, data->tid_entry);
  5677. if (sample_type & PERF_SAMPLE_TIME)
  5678. perf_output_put(handle, data->time);
  5679. if (sample_type & PERF_SAMPLE_ADDR)
  5680. perf_output_put(handle, data->addr);
  5681. if (sample_type & PERF_SAMPLE_ID)
  5682. perf_output_put(handle, data->id);
  5683. if (sample_type & PERF_SAMPLE_STREAM_ID)
  5684. perf_output_put(handle, data->stream_id);
  5685. if (sample_type & PERF_SAMPLE_CPU)
  5686. perf_output_put(handle, data->cpu_entry);
  5687. if (sample_type & PERF_SAMPLE_PERIOD)
  5688. perf_output_put(handle, data->period);
  5689. if (sample_type & PERF_SAMPLE_READ)
  5690. perf_output_read(handle, event);
  5691. if (sample_type & PERF_SAMPLE_CALLCHAIN) {
  5692. int size = 1;
  5693. size += data->callchain->nr;
  5694. size *= sizeof(u64);
  5695. __output_copy(handle, data->callchain, size);
  5696. }
  5697. if (sample_type & PERF_SAMPLE_RAW) {
  5698. struct perf_raw_record *raw = data->raw;
  5699. if (raw) {
  5700. struct perf_raw_frag *frag = &raw->frag;
  5701. perf_output_put(handle, raw->size);
  5702. do {
  5703. if (frag->copy) {
  5704. __output_custom(handle, frag->copy,
  5705. frag->data, frag->size);
  5706. } else {
  5707. __output_copy(handle, frag->data,
  5708. frag->size);
  5709. }
  5710. if (perf_raw_frag_last(frag))
  5711. break;
  5712. frag = frag->next;
  5713. } while (1);
  5714. if (frag->pad)
  5715. __output_skip(handle, NULL, frag->pad);
  5716. } else {
  5717. struct {
  5718. u32 size;
  5719. u32 data;
  5720. } raw = {
  5721. .size = sizeof(u32),
  5722. .data = 0,
  5723. };
  5724. perf_output_put(handle, raw);
  5725. }
  5726. }
  5727. if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
  5728. if (data->br_stack) {
  5729. size_t size;
  5730. size = data->br_stack->nr
  5731. * sizeof(struct perf_branch_entry);
  5732. perf_output_put(handle, data->br_stack->nr);
  5733. if (perf_sample_save_hw_index(event))
  5734. perf_output_put(handle, data->br_stack->hw_idx);
  5735. perf_output_copy(handle, data->br_stack->entries, size);
  5736. } else {
  5737. /*
  5738. * we always store at least the value of nr
  5739. */
  5740. u64 nr = 0;
  5741. perf_output_put(handle, nr);
  5742. }
  5743. }
  5744. if (sample_type & PERF_SAMPLE_REGS_USER) {
  5745. u64 abi = data->regs_user.abi;
  5746. /*
  5747. * If there are no regs to dump, notice it through
  5748. * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE).
  5749. */
  5750. perf_output_put(handle, abi);
  5751. if (abi) {
  5752. u64 mask = event->attr.sample_regs_user;
  5753. perf_output_sample_regs(handle,
  5754. data->regs_user.regs,
  5755. mask);
  5756. }
  5757. }
  5758. if (sample_type & PERF_SAMPLE_STACK_USER) {
  5759. perf_output_sample_ustack(handle,
  5760. data->stack_user_size,
  5761. data->regs_user.regs);
  5762. }
  5763. if (sample_type & PERF_SAMPLE_WEIGHT)
  5764. perf_output_put(handle, data->weight);
  5765. if (sample_type & PERF_SAMPLE_DATA_SRC)
  5766. perf_output_put(handle, data->data_src.val);
  5767. if (sample_type & PERF_SAMPLE_TRANSACTION)
  5768. perf_output_put(handle, data->txn);
  5769. if (sample_type & PERF_SAMPLE_REGS_INTR) {
  5770. u64 abi = data->regs_intr.abi;
  5771. /*
  5772. * If there are no regs to dump, notice it through
  5773. * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE).
  5774. */
  5775. perf_output_put(handle, abi);
  5776. if (abi) {
  5777. u64 mask = event->attr.sample_regs_intr;
  5778. perf_output_sample_regs(handle,
  5779. data->regs_intr.regs,
  5780. mask);
  5781. }
  5782. }
  5783. if (sample_type & PERF_SAMPLE_PHYS_ADDR)
  5784. perf_output_put(handle, data->phys_addr);
  5785. if (sample_type & PERF_SAMPLE_CGROUP)
  5786. perf_output_put(handle, data->cgroup);
  5787. if (sample_type & PERF_SAMPLE_AUX) {
  5788. perf_output_put(handle, data->aux_size);
  5789. if (data->aux_size)
  5790. perf_aux_sample_output(event, handle, data);
  5791. }
  5792. if (!event->attr.watermark) {
  5793. int wakeup_events = event->attr.wakeup_events;
  5794. if (wakeup_events) {
  5795. struct perf_buffer *rb = handle->rb;
  5796. int events = local_inc_return(&rb->events);
  5797. if (events >= wakeup_events) {
  5798. local_sub(wakeup_events, &rb->events);
  5799. local_inc(&rb->wakeup);
  5800. }
  5801. }
  5802. }
  5803. }
  5804. static u64 perf_virt_to_phys(u64 virt)
  5805. {
  5806. u64 phys_addr = 0;
  5807. if (!virt)
  5808. return 0;
  5809. if (virt >= TASK_SIZE) {
  5810. /* If it's vmalloc()d memory, leave phys_addr as 0 */
  5811. if (virt_addr_valid((void *)(uintptr_t)virt) &&
  5812. !(virt >= VMALLOC_START && virt < VMALLOC_END))
  5813. phys_addr = (u64)virt_to_phys((void *)(uintptr_t)virt);
  5814. } else {
  5815. /*
  5816. * Walking the pages tables for user address.
  5817. * Interrupts are disabled, so it prevents any tear down
  5818. * of the page tables.
  5819. * Try IRQ-safe get_user_page_fast_only first.
  5820. * If failed, leave phys_addr as 0.
  5821. */
  5822. if (current->mm != NULL) {
  5823. struct page *p;
  5824. pagefault_disable();
  5825. if (get_user_page_fast_only(virt, 0, &p)) {
  5826. phys_addr = page_to_phys(p) + virt % PAGE_SIZE;
  5827. put_page(p);
  5828. }
  5829. pagefault_enable();
  5830. }
  5831. }
  5832. return phys_addr;
  5833. }
  5834. static struct perf_callchain_entry __empty_callchain = { .nr = 0, };
  5835. struct perf_callchain_entry *
  5836. perf_callchain(struct perf_event *event, struct pt_regs *regs)
  5837. {
  5838. bool kernel = !event->attr.exclude_callchain_kernel;
  5839. bool user = !event->attr.exclude_callchain_user;
  5840. /* Disallow cross-task user callchains. */
  5841. bool crosstask = event->ctx->task && event->ctx->task != current;
  5842. const u32 max_stack = event->attr.sample_max_stack;
  5843. struct perf_callchain_entry *callchain;
  5844. if (!kernel && !user)
  5845. return &__empty_callchain;
  5846. callchain = get_perf_callchain(regs, 0, kernel, user,
  5847. max_stack, crosstask, true);
  5848. return callchain ?: &__empty_callchain;
  5849. }
  5850. void perf_prepare_sample(struct perf_event_header *header,
  5851. struct perf_sample_data *data,
  5852. struct perf_event *event,
  5853. struct pt_regs *regs)
  5854. {
  5855. u64 sample_type = event->attr.sample_type;
  5856. header->type = PERF_RECORD_SAMPLE;
  5857. header->size = sizeof(*header) + event->header_size;
  5858. header->misc = 0;
  5859. header->misc |= perf_misc_flags(regs);
  5860. __perf_event_header__init_id(header, data, event);
  5861. if (sample_type & PERF_SAMPLE_IP)
  5862. data->ip = perf_instruction_pointer(regs);
  5863. if (sample_type & PERF_SAMPLE_CALLCHAIN) {
  5864. int size = 1;
  5865. if (!(sample_type & __PERF_SAMPLE_CALLCHAIN_EARLY))
  5866. data->callchain = perf_callchain(event, regs);
  5867. size += data->callchain->nr;
  5868. header->size += size * sizeof(u64);
  5869. }
  5870. if (sample_type & PERF_SAMPLE_RAW) {
  5871. struct perf_raw_record *raw = data->raw;
  5872. int size;
  5873. if (raw) {
  5874. struct perf_raw_frag *frag = &raw->frag;
  5875. u32 sum = 0;
  5876. do {
  5877. sum += frag->size;
  5878. if (perf_raw_frag_last(frag))
  5879. break;
  5880. frag = frag->next;
  5881. } while (1);
  5882. size = round_up(sum + sizeof(u32), sizeof(u64));
  5883. raw->size = size - sizeof(u32);
  5884. frag->pad = raw->size - sum;
  5885. } else {
  5886. size = sizeof(u64);
  5887. }
  5888. header->size += size;
  5889. }
  5890. if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
  5891. int size = sizeof(u64); /* nr */
  5892. if (data->br_stack) {
  5893. if (perf_sample_save_hw_index(event))
  5894. size += sizeof(u64);
  5895. size += data->br_stack->nr
  5896. * sizeof(struct perf_branch_entry);
  5897. }
  5898. header->size += size;
  5899. }
  5900. if (sample_type & (PERF_SAMPLE_REGS_USER | PERF_SAMPLE_STACK_USER))
  5901. perf_sample_regs_user(&data->regs_user, regs);
  5902. if (sample_type & PERF_SAMPLE_REGS_USER) {
  5903. /* regs dump ABI info */
  5904. int size = sizeof(u64);
  5905. if (data->regs_user.regs) {
  5906. u64 mask = event->attr.sample_regs_user;
  5907. size += hweight64(mask) * sizeof(u64);
  5908. }
  5909. header->size += size;
  5910. }
  5911. if (sample_type & PERF_SAMPLE_STACK_USER) {
  5912. /*
  5913. * Either we need PERF_SAMPLE_STACK_USER bit to be always
  5914. * processed as the last one or have additional check added
  5915. * in case new sample type is added, because we could eat
  5916. * up the rest of the sample size.
  5917. */
  5918. u16 stack_size = event->attr.sample_stack_user;
  5919. u16 size = sizeof(u64);
  5920. stack_size = perf_sample_ustack_size(stack_size, header->size,
  5921. data->regs_user.regs);
  5922. /*
  5923. * If there is something to dump, add space for the dump
  5924. * itself and for the field that tells the dynamic size,
  5925. * which is how many have been actually dumped.
  5926. */
  5927. if (stack_size)
  5928. size += sizeof(u64) + stack_size;
  5929. data->stack_user_size = stack_size;
  5930. header->size += size;
  5931. }
  5932. if (sample_type & PERF_SAMPLE_REGS_INTR) {
  5933. /* regs dump ABI info */
  5934. int size = sizeof(u64);
  5935. perf_sample_regs_intr(&data->regs_intr, regs);
  5936. if (data->regs_intr.regs) {
  5937. u64 mask = event->attr.sample_regs_intr;
  5938. size += hweight64(mask) * sizeof(u64);
  5939. }
  5940. header->size += size;
  5941. }
  5942. if (sample_type & PERF_SAMPLE_PHYS_ADDR)
  5943. data->phys_addr = perf_virt_to_phys(data->addr);
  5944. #ifdef CONFIG_CGROUP_PERF
  5945. if (sample_type & PERF_SAMPLE_CGROUP) {
  5946. struct cgroup *cgrp;
  5947. /* protected by RCU */
  5948. cgrp = task_css_check(current, perf_event_cgrp_id, 1)->cgroup;
  5949. data->cgroup = cgroup_id(cgrp);
  5950. }
  5951. #endif
  5952. if (sample_type & PERF_SAMPLE_AUX) {
  5953. u64 size;
  5954. header->size += sizeof(u64); /* size */
  5955. /*
  5956. * Given the 16bit nature of header::size, an AUX sample can
  5957. * easily overflow it, what with all the preceding sample bits.
  5958. * Make sure this doesn't happen by using up to U16_MAX bytes
  5959. * per sample in total (rounded down to 8 byte boundary).
  5960. */
  5961. size = min_t(size_t, U16_MAX - header->size,
  5962. event->attr.aux_sample_size);
  5963. size = rounddown(size, 8);
  5964. size = perf_prepare_sample_aux(event, data, size);
  5965. WARN_ON_ONCE(size + header->size > U16_MAX);
  5966. header->size += size;
  5967. }
  5968. /*
  5969. * If you're adding more sample types here, you likely need to do
  5970. * something about the overflowing header::size, like repurpose the
  5971. * lowest 3 bits of size, which should be always zero at the moment.
  5972. * This raises a more important question, do we really need 512k sized
  5973. * samples and why, so good argumentation is in order for whatever you
  5974. * do here next.
  5975. */
  5976. WARN_ON_ONCE(header->size & 7);
  5977. }
  5978. static __always_inline int
  5979. __perf_event_output(struct perf_event *event,
  5980. struct perf_sample_data *data,
  5981. struct pt_regs *regs,
  5982. int (*output_begin)(struct perf_output_handle *,
  5983. struct perf_sample_data *,
  5984. struct perf_event *,
  5985. unsigned int))
  5986. {
  5987. struct perf_output_handle handle;
  5988. struct perf_event_header header;
  5989. int err;
  5990. /* protect the callchain buffers */
  5991. rcu_read_lock();
  5992. perf_prepare_sample(&header, data, event, regs);
  5993. err = output_begin(&handle, data, event, header.size);
  5994. if (err)
  5995. goto exit;
  5996. perf_output_sample(&handle, &header, data, event);
  5997. perf_output_end(&handle);
  5998. exit:
  5999. rcu_read_unlock();
  6000. return err;
  6001. }
  6002. void
  6003. perf_event_output_forward(struct perf_event *event,
  6004. struct perf_sample_data *data,
  6005. struct pt_regs *regs)
  6006. {
  6007. __perf_event_output(event, data, regs, perf_output_begin_forward);
  6008. }
  6009. void
  6010. perf_event_output_backward(struct perf_event *event,
  6011. struct perf_sample_data *data,
  6012. struct pt_regs *regs)
  6013. {
  6014. __perf_event_output(event, data, regs, perf_output_begin_backward);
  6015. }
  6016. int
  6017. perf_event_output(struct perf_event *event,
  6018. struct perf_sample_data *data,
  6019. struct pt_regs *regs)
  6020. {
  6021. return __perf_event_output(event, data, regs, perf_output_begin);
  6022. }
  6023. /*
  6024. * read event_id
  6025. */
  6026. struct perf_read_event {
  6027. struct perf_event_header header;
  6028. u32 pid;
  6029. u32 tid;
  6030. };
  6031. static void
  6032. perf_event_read_event(struct perf_event *event,
  6033. struct task_struct *task)
  6034. {
  6035. struct perf_output_handle handle;
  6036. struct perf_sample_data sample;
  6037. struct perf_read_event read_event = {
  6038. .header = {
  6039. .type = PERF_RECORD_READ,
  6040. .misc = 0,
  6041. .size = sizeof(read_event) + event->read_size,
  6042. },
  6043. .pid = perf_event_pid(event, task),
  6044. .tid = perf_event_tid(event, task),
  6045. };
  6046. int ret;
  6047. perf_event_header__init_id(&read_event.header, &sample, event);
  6048. ret = perf_output_begin(&handle, &sample, event, read_event.header.size);
  6049. if (ret)
  6050. return;
  6051. perf_output_put(&handle, read_event);
  6052. perf_output_read(&handle, event);
  6053. perf_event__output_id_sample(event, &handle, &sample);
  6054. perf_output_end(&handle);
  6055. }
  6056. typedef void (perf_iterate_f)(struct perf_event *event, void *data);
  6057. static void
  6058. perf_iterate_ctx(struct perf_event_context *ctx,
  6059. perf_iterate_f output,
  6060. void *data, bool all)
  6061. {
  6062. struct perf_event *event;
  6063. list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
  6064. if (!all) {
  6065. if (event->state < PERF_EVENT_STATE_INACTIVE)
  6066. continue;
  6067. if (!event_filter_match(event))
  6068. continue;
  6069. }
  6070. output(event, data);
  6071. }
  6072. }
  6073. static void perf_iterate_sb_cpu(perf_iterate_f output, void *data)
  6074. {
  6075. struct pmu_event_list *pel = this_cpu_ptr(&pmu_sb_events);
  6076. struct perf_event *event;
  6077. list_for_each_entry_rcu(event, &pel->list, sb_list) {
  6078. /*
  6079. * Skip events that are not fully formed yet; ensure that
  6080. * if we observe event->ctx, both event and ctx will be
  6081. * complete enough. See perf_install_in_context().
  6082. */
  6083. if (!smp_load_acquire(&event->ctx))
  6084. continue;
  6085. if (event->state < PERF_EVENT_STATE_INACTIVE)
  6086. continue;
  6087. if (!event_filter_match(event))
  6088. continue;
  6089. output(event, data);
  6090. }
  6091. }
  6092. /*
  6093. * Iterate all events that need to receive side-band events.
  6094. *
  6095. * For new callers; ensure that account_pmu_sb_event() includes
  6096. * your event, otherwise it might not get delivered.
  6097. */
  6098. static void
  6099. perf_iterate_sb(perf_iterate_f output, void *data,
  6100. struct perf_event_context *task_ctx)
  6101. {
  6102. struct perf_event_context *ctx;
  6103. int ctxn;
  6104. rcu_read_lock();
  6105. preempt_disable();
  6106. /*
  6107. * If we have task_ctx != NULL we only notify the task context itself.
  6108. * The task_ctx is set only for EXIT events before releasing task
  6109. * context.
  6110. */
  6111. if (task_ctx) {
  6112. perf_iterate_ctx(task_ctx, output, data, false);
  6113. goto done;
  6114. }
  6115. perf_iterate_sb_cpu(output, data);
  6116. for_each_task_context_nr(ctxn) {
  6117. ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
  6118. if (ctx)
  6119. perf_iterate_ctx(ctx, output, data, false);
  6120. }
  6121. done:
  6122. preempt_enable();
  6123. rcu_read_unlock();
  6124. }
  6125. /*
  6126. * Clear all file-based filters at exec, they'll have to be
  6127. * re-instated when/if these objects are mmapped again.
  6128. */
  6129. static void perf_event_addr_filters_exec(struct perf_event *event, void *data)
  6130. {
  6131. struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
  6132. struct perf_addr_filter *filter;
  6133. unsigned int restart = 0, count = 0;
  6134. unsigned long flags;
  6135. if (!has_addr_filter(event))
  6136. return;
  6137. raw_spin_lock_irqsave(&ifh->lock, flags);
  6138. list_for_each_entry(filter, &ifh->list, entry) {
  6139. if (filter->path.dentry) {
  6140. event->addr_filter_ranges[count].start = 0;
  6141. event->addr_filter_ranges[count].size = 0;
  6142. restart++;
  6143. }
  6144. count++;
  6145. }
  6146. if (restart)
  6147. event->addr_filters_gen++;
  6148. raw_spin_unlock_irqrestore(&ifh->lock, flags);
  6149. if (restart)
  6150. perf_event_stop(event, 1);
  6151. }
  6152. void perf_event_exec(void)
  6153. {
  6154. struct perf_event_context *ctx;
  6155. int ctxn;
  6156. rcu_read_lock();
  6157. for_each_task_context_nr(ctxn) {
  6158. ctx = current->perf_event_ctxp[ctxn];
  6159. if (!ctx)
  6160. continue;
  6161. perf_event_enable_on_exec(ctxn);
  6162. perf_iterate_ctx(ctx, perf_event_addr_filters_exec, NULL,
  6163. true);
  6164. }
  6165. rcu_read_unlock();
  6166. }
  6167. struct remote_output {
  6168. struct perf_buffer *rb;
  6169. int err;
  6170. };
  6171. static void __perf_event_output_stop(struct perf_event *event, void *data)
  6172. {
  6173. struct perf_event *parent = event->parent;
  6174. struct remote_output *ro = data;
  6175. struct perf_buffer *rb = ro->rb;
  6176. struct stop_event_data sd = {
  6177. .event = event,
  6178. };
  6179. if (!has_aux(event))
  6180. return;
  6181. if (!parent)
  6182. parent = event;
  6183. /*
  6184. * In case of inheritance, it will be the parent that links to the
  6185. * ring-buffer, but it will be the child that's actually using it.
  6186. *
  6187. * We are using event::rb to determine if the event should be stopped,
  6188. * however this may race with ring_buffer_attach() (through set_output),
  6189. * which will make us skip the event that actually needs to be stopped.
  6190. * So ring_buffer_attach() has to stop an aux event before re-assigning
  6191. * its rb pointer.
  6192. */
  6193. if (rcu_dereference(parent->rb) == rb)
  6194. ro->err = __perf_event_stop(&sd);
  6195. }
  6196. static int __perf_pmu_output_stop(void *info)
  6197. {
  6198. struct perf_event *event = info;
  6199. struct pmu *pmu = event->ctx->pmu;
  6200. struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
  6201. struct remote_output ro = {
  6202. .rb = event->rb,
  6203. };
  6204. rcu_read_lock();
  6205. perf_iterate_ctx(&cpuctx->ctx, __perf_event_output_stop, &ro, false);
  6206. if (cpuctx->task_ctx)
  6207. perf_iterate_ctx(cpuctx->task_ctx, __perf_event_output_stop,
  6208. &ro, false);
  6209. rcu_read_unlock();
  6210. return ro.err;
  6211. }
  6212. static void perf_pmu_output_stop(struct perf_event *event)
  6213. {
  6214. struct perf_event *iter;
  6215. int err, cpu;
  6216. restart:
  6217. rcu_read_lock();
  6218. list_for_each_entry_rcu(iter, &event->rb->event_list, rb_entry) {
  6219. /*
  6220. * For per-CPU events, we need to make sure that neither they
  6221. * nor their children are running; for cpu==-1 events it's
  6222. * sufficient to stop the event itself if it's active, since
  6223. * it can't have children.
  6224. */
  6225. cpu = iter->cpu;
  6226. if (cpu == -1)
  6227. cpu = READ_ONCE(iter->oncpu);
  6228. if (cpu == -1)
  6229. continue;
  6230. err = cpu_function_call(cpu, __perf_pmu_output_stop, event);
  6231. if (err == -EAGAIN) {
  6232. rcu_read_unlock();
  6233. goto restart;
  6234. }
  6235. }
  6236. rcu_read_unlock();
  6237. }
  6238. /*
  6239. * task tracking -- fork/exit
  6240. *
  6241. * enabled by: attr.comm | attr.mmap | attr.mmap2 | attr.mmap_data | attr.task
  6242. */
  6243. struct perf_task_event {
  6244. struct task_struct *task;
  6245. struct perf_event_context *task_ctx;
  6246. struct {
  6247. struct perf_event_header header;
  6248. u32 pid;
  6249. u32 ppid;
  6250. u32 tid;
  6251. u32 ptid;
  6252. u64 time;
  6253. } event_id;
  6254. };
  6255. static int perf_event_task_match(struct perf_event *event)
  6256. {
  6257. return event->attr.comm || event->attr.mmap ||
  6258. event->attr.mmap2 || event->attr.mmap_data ||
  6259. event->attr.task;
  6260. }
  6261. static void perf_event_task_output(struct perf_event *event,
  6262. void *data)
  6263. {
  6264. struct perf_task_event *task_event = data;
  6265. struct perf_output_handle handle;
  6266. struct perf_sample_data sample;
  6267. struct task_struct *task = task_event->task;
  6268. int ret, size = task_event->event_id.header.size;
  6269. if (!perf_event_task_match(event))
  6270. return;
  6271. perf_event_header__init_id(&task_event->event_id.header, &sample, event);
  6272. ret = perf_output_begin(&handle, &sample, event,
  6273. task_event->event_id.header.size);
  6274. if (ret)
  6275. goto out;
  6276. task_event->event_id.pid = perf_event_pid(event, task);
  6277. task_event->event_id.tid = perf_event_tid(event, task);
  6278. if (task_event->event_id.header.type == PERF_RECORD_EXIT) {
  6279. task_event->event_id.ppid = perf_event_pid(event,
  6280. task->real_parent);
  6281. task_event->event_id.ptid = perf_event_pid(event,
  6282. task->real_parent);
  6283. } else { /* PERF_RECORD_FORK */
  6284. task_event->event_id.ppid = perf_event_pid(event, current);
  6285. task_event->event_id.ptid = perf_event_tid(event, current);
  6286. }
  6287. task_event->event_id.time = perf_event_clock(event);
  6288. perf_output_put(&handle, task_event->event_id);
  6289. perf_event__output_id_sample(event, &handle, &sample);
  6290. perf_output_end(&handle);
  6291. out:
  6292. task_event->event_id.header.size = size;
  6293. }
  6294. static void perf_event_task(struct task_struct *task,
  6295. struct perf_event_context *task_ctx,
  6296. int new)
  6297. {
  6298. struct perf_task_event task_event;
  6299. if (!atomic_read(&nr_comm_events) &&
  6300. !atomic_read(&nr_mmap_events) &&
  6301. !atomic_read(&nr_task_events))
  6302. return;
  6303. task_event = (struct perf_task_event){
  6304. .task = task,
  6305. .task_ctx = task_ctx,
  6306. .event_id = {
  6307. .header = {
  6308. .type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
  6309. .misc = 0,
  6310. .size = sizeof(task_event.event_id),
  6311. },
  6312. /* .pid */
  6313. /* .ppid */
  6314. /* .tid */
  6315. /* .ptid */
  6316. /* .time */
  6317. },
  6318. };
  6319. perf_iterate_sb(perf_event_task_output,
  6320. &task_event,
  6321. task_ctx);
  6322. }
  6323. void perf_event_fork(struct task_struct *task)
  6324. {
  6325. perf_event_task(task, NULL, 1);
  6326. perf_event_namespaces(task);
  6327. }
  6328. /*
  6329. * comm tracking
  6330. */
  6331. struct perf_comm_event {
  6332. struct task_struct *task;
  6333. char *comm;
  6334. int comm_size;
  6335. struct {
  6336. struct perf_event_header header;
  6337. u32 pid;
  6338. u32 tid;
  6339. } event_id;
  6340. };
  6341. static int perf_event_comm_match(struct perf_event *event)
  6342. {
  6343. return event->attr.comm;
  6344. }
  6345. static void perf_event_comm_output(struct perf_event *event,
  6346. void *data)
  6347. {
  6348. struct perf_comm_event *comm_event = data;
  6349. struct perf_output_handle handle;
  6350. struct perf_sample_data sample;
  6351. int size = comm_event->event_id.header.size;
  6352. int ret;
  6353. if (!perf_event_comm_match(event))
  6354. return;
  6355. perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
  6356. ret = perf_output_begin(&handle, &sample, event,
  6357. comm_event->event_id.header.size);
  6358. if (ret)
  6359. goto out;
  6360. comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
  6361. comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
  6362. perf_output_put(&handle, comm_event->event_id);
  6363. __output_copy(&handle, comm_event->comm,
  6364. comm_event->comm_size);
  6365. perf_event__output_id_sample(event, &handle, &sample);
  6366. perf_output_end(&handle);
  6367. out:
  6368. comm_event->event_id.header.size = size;
  6369. }
  6370. static void perf_event_comm_event(struct perf_comm_event *comm_event)
  6371. {
  6372. char comm[TASK_COMM_LEN];
  6373. unsigned int size;
  6374. memset(comm, 0, sizeof(comm));
  6375. strlcpy(comm, comm_event->task->comm, sizeof(comm));
  6376. size = ALIGN(strlen(comm)+1, sizeof(u64));
  6377. comm_event->comm = comm;
  6378. comm_event->comm_size = size;
  6379. comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
  6380. perf_iterate_sb(perf_event_comm_output,
  6381. comm_event,
  6382. NULL);
  6383. }
  6384. void perf_event_comm(struct task_struct *task, bool exec)
  6385. {
  6386. struct perf_comm_event comm_event;
  6387. if (!atomic_read(&nr_comm_events))
  6388. return;
  6389. comm_event = (struct perf_comm_event){
  6390. .task = task,
  6391. /* .comm */
  6392. /* .comm_size */
  6393. .event_id = {
  6394. .header = {
  6395. .type = PERF_RECORD_COMM,
  6396. .misc = exec ? PERF_RECORD_MISC_COMM_EXEC : 0,
  6397. /* .size */
  6398. },
  6399. /* .pid */
  6400. /* .tid */
  6401. },
  6402. };
  6403. perf_event_comm_event(&comm_event);
  6404. }
  6405. /*
  6406. * namespaces tracking
  6407. */
  6408. struct perf_namespaces_event {
  6409. struct task_struct *task;
  6410. struct {
  6411. struct perf_event_header header;
  6412. u32 pid;
  6413. u32 tid;
  6414. u64 nr_namespaces;
  6415. struct perf_ns_link_info link_info[NR_NAMESPACES];
  6416. } event_id;
  6417. };
  6418. static int perf_event_namespaces_match(struct perf_event *event)
  6419. {
  6420. return event->attr.namespaces;
  6421. }
  6422. static void perf_event_namespaces_output(struct perf_event *event,
  6423. void *data)
  6424. {
  6425. struct perf_namespaces_event *namespaces_event = data;
  6426. struct perf_output_handle handle;
  6427. struct perf_sample_data sample;
  6428. u16 header_size = namespaces_event->event_id.header.size;
  6429. int ret;
  6430. if (!perf_event_namespaces_match(event))
  6431. return;
  6432. perf_event_header__init_id(&namespaces_event->event_id.header,
  6433. &sample, event);
  6434. ret = perf_output_begin(&handle, &sample, event,
  6435. namespaces_event->event_id.header.size);
  6436. if (ret)
  6437. goto out;
  6438. namespaces_event->event_id.pid = perf_event_pid(event,
  6439. namespaces_event->task);
  6440. namespaces_event->event_id.tid = perf_event_tid(event,
  6441. namespaces_event->task);
  6442. perf_output_put(&handle, namespaces_event->event_id);
  6443. perf_event__output_id_sample(event, &handle, &sample);
  6444. perf_output_end(&handle);
  6445. out:
  6446. namespaces_event->event_id.header.size = header_size;
  6447. }
  6448. static void perf_fill_ns_link_info(struct perf_ns_link_info *ns_link_info,
  6449. struct task_struct *task,
  6450. const struct proc_ns_operations *ns_ops)
  6451. {
  6452. struct path ns_path;
  6453. struct inode *ns_inode;
  6454. int error;
  6455. error = ns_get_path(&ns_path, task, ns_ops);
  6456. if (!error) {
  6457. ns_inode = ns_path.dentry->d_inode;
  6458. ns_link_info->dev = new_encode_dev(ns_inode->i_sb->s_dev);
  6459. ns_link_info->ino = ns_inode->i_ino;
  6460. path_put(&ns_path);
  6461. }
  6462. }
  6463. void perf_event_namespaces(struct task_struct *task)
  6464. {
  6465. struct perf_namespaces_event namespaces_event;
  6466. struct perf_ns_link_info *ns_link_info;
  6467. if (!atomic_read(&nr_namespaces_events))
  6468. return;
  6469. namespaces_event = (struct perf_namespaces_event){
  6470. .task = task,
  6471. .event_id = {
  6472. .header = {
  6473. .type = PERF_RECORD_NAMESPACES,
  6474. .misc = 0,
  6475. .size = sizeof(namespaces_event.event_id),
  6476. },
  6477. /* .pid */
  6478. /* .tid */
  6479. .nr_namespaces = NR_NAMESPACES,
  6480. /* .link_info[NR_NAMESPACES] */
  6481. },
  6482. };
  6483. ns_link_info = namespaces_event.event_id.link_info;
  6484. perf_fill_ns_link_info(&ns_link_info[MNT_NS_INDEX],
  6485. task, &mntns_operations);
  6486. #ifdef CONFIG_USER_NS
  6487. perf_fill_ns_link_info(&ns_link_info[USER_NS_INDEX],
  6488. task, &userns_operations);
  6489. #endif
  6490. #ifdef CONFIG_NET_NS
  6491. perf_fill_ns_link_info(&ns_link_info[NET_NS_INDEX],
  6492. task, &netns_operations);
  6493. #endif
  6494. #ifdef CONFIG_UTS_NS
  6495. perf_fill_ns_link_info(&ns_link_info[UTS_NS_INDEX],
  6496. task, &utsns_operations);
  6497. #endif
  6498. #ifdef CONFIG_IPC_NS
  6499. perf_fill_ns_link_info(&ns_link_info[IPC_NS_INDEX],
  6500. task, &ipcns_operations);
  6501. #endif
  6502. #ifdef CONFIG_PID_NS
  6503. perf_fill_ns_link_info(&ns_link_info[PID_NS_INDEX],
  6504. task, &pidns_operations);
  6505. #endif
  6506. #ifdef CONFIG_CGROUPS
  6507. perf_fill_ns_link_info(&ns_link_info[CGROUP_NS_INDEX],
  6508. task, &cgroupns_operations);
  6509. #endif
  6510. perf_iterate_sb(perf_event_namespaces_output,
  6511. &namespaces_event,
  6512. NULL);
  6513. }
  6514. /*
  6515. * cgroup tracking
  6516. */
  6517. #ifdef CONFIG_CGROUP_PERF
  6518. struct perf_cgroup_event {
  6519. char *path;
  6520. int path_size;
  6521. struct {
  6522. struct perf_event_header header;
  6523. u64 id;
  6524. char path[];
  6525. } event_id;
  6526. };
  6527. static int perf_event_cgroup_match(struct perf_event *event)
  6528. {
  6529. return event->attr.cgroup;
  6530. }
  6531. static void perf_event_cgroup_output(struct perf_event *event, void *data)
  6532. {
  6533. struct perf_cgroup_event *cgroup_event = data;
  6534. struct perf_output_handle handle;
  6535. struct perf_sample_data sample;
  6536. u16 header_size = cgroup_event->event_id.header.size;
  6537. int ret;
  6538. if (!perf_event_cgroup_match(event))
  6539. return;
  6540. perf_event_header__init_id(&cgroup_event->event_id.header,
  6541. &sample, event);
  6542. ret = perf_output_begin(&handle, &sample, event,
  6543. cgroup_event->event_id.header.size);
  6544. if (ret)
  6545. goto out;
  6546. perf_output_put(&handle, cgroup_event->event_id);
  6547. __output_copy(&handle, cgroup_event->path, cgroup_event->path_size);
  6548. perf_event__output_id_sample(event, &handle, &sample);
  6549. perf_output_end(&handle);
  6550. out:
  6551. cgroup_event->event_id.header.size = header_size;
  6552. }
  6553. static void perf_event_cgroup(struct cgroup *cgrp)
  6554. {
  6555. struct perf_cgroup_event cgroup_event;
  6556. char path_enomem[16] = "//enomem";
  6557. char *pathname;
  6558. size_t size;
  6559. if (!atomic_read(&nr_cgroup_events))
  6560. return;
  6561. cgroup_event = (struct perf_cgroup_event){
  6562. .event_id = {
  6563. .header = {
  6564. .type = PERF_RECORD_CGROUP,
  6565. .misc = 0,
  6566. .size = sizeof(cgroup_event.event_id),
  6567. },
  6568. .id = cgroup_id(cgrp),
  6569. },
  6570. };
  6571. pathname = kmalloc(PATH_MAX, GFP_KERNEL);
  6572. if (pathname == NULL) {
  6573. cgroup_event.path = path_enomem;
  6574. } else {
  6575. /* just to be sure to have enough space for alignment */
  6576. cgroup_path(cgrp, pathname, PATH_MAX - sizeof(u64));
  6577. cgroup_event.path = pathname;
  6578. }
  6579. /*
  6580. * Since our buffer works in 8 byte units we need to align our string
  6581. * size to a multiple of 8. However, we must guarantee the tail end is
  6582. * zero'd out to avoid leaking random bits to userspace.
  6583. */
  6584. size = strlen(cgroup_event.path) + 1;
  6585. while (!IS_ALIGNED(size, sizeof(u64)))
  6586. cgroup_event.path[size++] = '\0';
  6587. cgroup_event.event_id.header.size += size;
  6588. cgroup_event.path_size = size;
  6589. perf_iterate_sb(perf_event_cgroup_output,
  6590. &cgroup_event,
  6591. NULL);
  6592. kfree(pathname);
  6593. }
  6594. #endif
  6595. /*
  6596. * mmap tracking
  6597. */
  6598. struct perf_mmap_event {
  6599. struct vm_area_struct *vma;
  6600. const char *file_name;
  6601. int file_size;
  6602. int maj, min;
  6603. u64 ino;
  6604. u64 ino_generation;
  6605. u32 prot, flags;
  6606. struct {
  6607. struct perf_event_header header;
  6608. u32 pid;
  6609. u32 tid;
  6610. u64 start;
  6611. u64 len;
  6612. u64 pgoff;
  6613. } event_id;
  6614. };
  6615. static int perf_event_mmap_match(struct perf_event *event,
  6616. void *data)
  6617. {
  6618. struct perf_mmap_event *mmap_event = data;
  6619. struct vm_area_struct *vma = mmap_event->vma;
  6620. int executable = vma->vm_flags & VM_EXEC;
  6621. return (!executable && event->attr.mmap_data) ||
  6622. (executable && (event->attr.mmap || event->attr.mmap2));
  6623. }
  6624. static void perf_event_mmap_output(struct perf_event *event,
  6625. void *data)
  6626. {
  6627. struct perf_mmap_event *mmap_event = data;
  6628. struct perf_output_handle handle;
  6629. struct perf_sample_data sample;
  6630. int size = mmap_event->event_id.header.size;
  6631. u32 type = mmap_event->event_id.header.type;
  6632. int ret;
  6633. if (!perf_event_mmap_match(event, data))
  6634. return;
  6635. if (event->attr.mmap2) {
  6636. mmap_event->event_id.header.type = PERF_RECORD_MMAP2;
  6637. mmap_event->event_id.header.size += sizeof(mmap_event->maj);
  6638. mmap_event->event_id.header.size += sizeof(mmap_event->min);
  6639. mmap_event->event_id.header.size += sizeof(mmap_event->ino);
  6640. mmap_event->event_id.header.size += sizeof(mmap_event->ino_generation);
  6641. mmap_event->event_id.header.size += sizeof(mmap_event->prot);
  6642. mmap_event->event_id.header.size += sizeof(mmap_event->flags);
  6643. }
  6644. perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
  6645. ret = perf_output_begin(&handle, &sample, event,
  6646. mmap_event->event_id.header.size);
  6647. if (ret)
  6648. goto out;
  6649. mmap_event->event_id.pid = perf_event_pid(event, current);
  6650. mmap_event->event_id.tid = perf_event_tid(event, current);
  6651. perf_output_put(&handle, mmap_event->event_id);
  6652. if (event->attr.mmap2) {
  6653. perf_output_put(&handle, mmap_event->maj);
  6654. perf_output_put(&handle, mmap_event->min);
  6655. perf_output_put(&handle, mmap_event->ino);
  6656. perf_output_put(&handle, mmap_event->ino_generation);
  6657. perf_output_put(&handle, mmap_event->prot);
  6658. perf_output_put(&handle, mmap_event->flags);
  6659. }
  6660. __output_copy(&handle, mmap_event->file_name,
  6661. mmap_event->file_size);
  6662. perf_event__output_id_sample(event, &handle, &sample);
  6663. perf_output_end(&handle);
  6664. out:
  6665. mmap_event->event_id.header.size = size;
  6666. mmap_event->event_id.header.type = type;
  6667. }
  6668. static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
  6669. {
  6670. struct vm_area_struct *vma = mmap_event->vma;
  6671. struct file *file = vma->vm_file;
  6672. int maj = 0, min = 0;
  6673. u64 ino = 0, gen = 0;
  6674. u32 prot = 0, flags = 0;
  6675. unsigned int size;
  6676. char tmp[16];
  6677. char *buf = NULL;
  6678. char *name;
  6679. if (vma->vm_flags & VM_READ)
  6680. prot |= PROT_READ;
  6681. if (vma->vm_flags & VM_WRITE)
  6682. prot |= PROT_WRITE;
  6683. if (vma->vm_flags & VM_EXEC)
  6684. prot |= PROT_EXEC;
  6685. if (vma->vm_flags & VM_MAYSHARE)
  6686. flags = MAP_SHARED;
  6687. else
  6688. flags = MAP_PRIVATE;
  6689. if (vma->vm_flags & VM_DENYWRITE)
  6690. flags |= MAP_DENYWRITE;
  6691. if (vma->vm_flags & VM_MAYEXEC)
  6692. flags |= MAP_EXECUTABLE;
  6693. if (vma->vm_flags & VM_LOCKED)
  6694. flags |= MAP_LOCKED;
  6695. if (is_vm_hugetlb_page(vma))
  6696. flags |= MAP_HUGETLB;
  6697. if (file) {
  6698. struct inode *inode;
  6699. dev_t dev;
  6700. buf = kmalloc(PATH_MAX, GFP_KERNEL);
  6701. if (!buf) {
  6702. name = "//enomem";
  6703. goto cpy_name;
  6704. }
  6705. /*
  6706. * d_path() works from the end of the rb backwards, so we
  6707. * need to add enough zero bytes after the string to handle
  6708. * the 64bit alignment we do later.
  6709. */
  6710. name = file_path(file, buf, PATH_MAX - sizeof(u64));
  6711. if (IS_ERR(name)) {
  6712. name = "//toolong";
  6713. goto cpy_name;
  6714. }
  6715. inode = file_inode(vma->vm_file);
  6716. dev = inode->i_sb->s_dev;
  6717. ino = inode->i_ino;
  6718. gen = inode->i_generation;
  6719. maj = MAJOR(dev);
  6720. min = MINOR(dev);
  6721. goto got_name;
  6722. } else {
  6723. if (vma->vm_ops && vma->vm_ops->name) {
  6724. name = (char *) vma->vm_ops->name(vma);
  6725. if (name)
  6726. goto cpy_name;
  6727. }
  6728. name = (char *)arch_vma_name(vma);
  6729. if (name)
  6730. goto cpy_name;
  6731. if (vma->vm_start <= vma->vm_mm->start_brk &&
  6732. vma->vm_end >= vma->vm_mm->brk) {
  6733. name = "[heap]";
  6734. goto cpy_name;
  6735. }
  6736. if (vma->vm_start <= vma->vm_mm->start_stack &&
  6737. vma->vm_end >= vma->vm_mm->start_stack) {
  6738. name = "[stack]";
  6739. goto cpy_name;
  6740. }
  6741. name = "//anon";
  6742. goto cpy_name;
  6743. }
  6744. cpy_name:
  6745. strlcpy(tmp, name, sizeof(tmp));
  6746. name = tmp;
  6747. got_name:
  6748. /*
  6749. * Since our buffer works in 8 byte units we need to align our string
  6750. * size to a multiple of 8. However, we must guarantee the tail end is
  6751. * zero'd out to avoid leaking random bits to userspace.
  6752. */
  6753. size = strlen(name)+1;
  6754. while (!IS_ALIGNED(size, sizeof(u64)))
  6755. name[size++] = '\0';
  6756. mmap_event->file_name = name;
  6757. mmap_event->file_size = size;
  6758. mmap_event->maj = maj;
  6759. mmap_event->min = min;
  6760. mmap_event->ino = ino;
  6761. mmap_event->ino_generation = gen;
  6762. mmap_event->prot = prot;
  6763. mmap_event->flags = flags;
  6764. if (!(vma->vm_flags & VM_EXEC))
  6765. mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA;
  6766. mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
  6767. perf_iterate_sb(perf_event_mmap_output,
  6768. mmap_event,
  6769. NULL);
  6770. kfree(buf);
  6771. }
  6772. /*
  6773. * Check whether inode and address range match filter criteria.
  6774. */
  6775. static bool perf_addr_filter_match(struct perf_addr_filter *filter,
  6776. struct file *file, unsigned long offset,
  6777. unsigned long size)
  6778. {
  6779. /* d_inode(NULL) won't be equal to any mapped user-space file */
  6780. if (!filter->path.dentry)
  6781. return false;
  6782. if (d_inode(filter->path.dentry) != file_inode(file))
  6783. return false;
  6784. if (filter->offset > offset + size)
  6785. return false;
  6786. if (filter->offset + filter->size < offset)
  6787. return false;
  6788. return true;
  6789. }
  6790. static bool perf_addr_filter_vma_adjust(struct perf_addr_filter *filter,
  6791. struct vm_area_struct *vma,
  6792. struct perf_addr_filter_range *fr)
  6793. {
  6794. unsigned long vma_size = vma->vm_end - vma->vm_start;
  6795. unsigned long off = vma->vm_pgoff << PAGE_SHIFT;
  6796. struct file *file = vma->vm_file;
  6797. if (!perf_addr_filter_match(filter, file, off, vma_size))
  6798. return false;
  6799. if (filter->offset < off) {
  6800. fr->start = vma->vm_start;
  6801. fr->size = min(vma_size, filter->size - (off - filter->offset));
  6802. } else {
  6803. fr->start = vma->vm_start + filter->offset - off;
  6804. fr->size = min(vma->vm_end - fr->start, filter->size);
  6805. }
  6806. return true;
  6807. }
  6808. static void __perf_addr_filters_adjust(struct perf_event *event, void *data)
  6809. {
  6810. struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
  6811. struct vm_area_struct *vma = data;
  6812. struct perf_addr_filter *filter;
  6813. unsigned int restart = 0, count = 0;
  6814. unsigned long flags;
  6815. if (!has_addr_filter(event))
  6816. return;
  6817. if (!vma->vm_file)
  6818. return;
  6819. raw_spin_lock_irqsave(&ifh->lock, flags);
  6820. list_for_each_entry(filter, &ifh->list, entry) {
  6821. if (perf_addr_filter_vma_adjust(filter, vma,
  6822. &event->addr_filter_ranges[count]))
  6823. restart++;
  6824. count++;
  6825. }
  6826. if (restart)
  6827. event->addr_filters_gen++;
  6828. raw_spin_unlock_irqrestore(&ifh->lock, flags);
  6829. if (restart)
  6830. perf_event_stop(event, 1);
  6831. }
  6832. /*
  6833. * Adjust all task's events' filters to the new vma
  6834. */
  6835. static void perf_addr_filters_adjust(struct vm_area_struct *vma)
  6836. {
  6837. struct perf_event_context *ctx;
  6838. int ctxn;
  6839. /*
  6840. * Data tracing isn't supported yet and as such there is no need
  6841. * to keep track of anything that isn't related to executable code:
  6842. */
  6843. if (!(vma->vm_flags & VM_EXEC))
  6844. return;
  6845. rcu_read_lock();
  6846. for_each_task_context_nr(ctxn) {
  6847. ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
  6848. if (!ctx)
  6849. continue;
  6850. perf_iterate_ctx(ctx, __perf_addr_filters_adjust, vma, true);
  6851. }
  6852. rcu_read_unlock();
  6853. }
  6854. void perf_event_mmap(struct vm_area_struct *vma)
  6855. {
  6856. struct perf_mmap_event mmap_event;
  6857. if (!atomic_read(&nr_mmap_events))
  6858. return;
  6859. mmap_event = (struct perf_mmap_event){
  6860. .vma = vma,
  6861. /* .file_name */
  6862. /* .file_size */
  6863. .event_id = {
  6864. .header = {
  6865. .type = PERF_RECORD_MMAP,
  6866. .misc = PERF_RECORD_MISC_USER,
  6867. /* .size */
  6868. },
  6869. /* .pid */
  6870. /* .tid */
  6871. .start = vma->vm_start,
  6872. .len = vma->vm_end - vma->vm_start,
  6873. .pgoff = (u64)vma->vm_pgoff << PAGE_SHIFT,
  6874. },
  6875. /* .maj (attr_mmap2 only) */
  6876. /* .min (attr_mmap2 only) */
  6877. /* .ino (attr_mmap2 only) */
  6878. /* .ino_generation (attr_mmap2 only) */
  6879. /* .prot (attr_mmap2 only) */
  6880. /* .flags (attr_mmap2 only) */
  6881. };
  6882. perf_addr_filters_adjust(vma);
  6883. perf_event_mmap_event(&mmap_event);
  6884. }
  6885. void perf_event_aux_event(struct perf_event *event, unsigned long head,
  6886. unsigned long size, u64 flags)
  6887. {
  6888. struct perf_output_handle handle;
  6889. struct perf_sample_data sample;
  6890. struct perf_aux_event {
  6891. struct perf_event_header header;
  6892. u64 offset;
  6893. u64 size;
  6894. u64 flags;
  6895. } rec = {
  6896. .header = {
  6897. .type = PERF_RECORD_AUX,
  6898. .misc = 0,
  6899. .size = sizeof(rec),
  6900. },
  6901. .offset = head,
  6902. .size = size,
  6903. .flags = flags,
  6904. };
  6905. int ret;
  6906. perf_event_header__init_id(&rec.header, &sample, event);
  6907. ret = perf_output_begin(&handle, &sample, event, rec.header.size);
  6908. if (ret)
  6909. return;
  6910. perf_output_put(&handle, rec);
  6911. perf_event__output_id_sample(event, &handle, &sample);
  6912. perf_output_end(&handle);
  6913. }
  6914. /*
  6915. * Lost/dropped samples logging
  6916. */
  6917. void perf_log_lost_samples(struct perf_event *event, u64 lost)
  6918. {
  6919. struct perf_output_handle handle;
  6920. struct perf_sample_data sample;
  6921. int ret;
  6922. struct {
  6923. struct perf_event_header header;
  6924. u64 lost;
  6925. } lost_samples_event = {
  6926. .header = {
  6927. .type = PERF_RECORD_LOST_SAMPLES,
  6928. .misc = 0,
  6929. .size = sizeof(lost_samples_event),
  6930. },
  6931. .lost = lost,
  6932. };
  6933. perf_event_header__init_id(&lost_samples_event.header, &sample, event);
  6934. ret = perf_output_begin(&handle, &sample, event,
  6935. lost_samples_event.header.size);
  6936. if (ret)
  6937. return;
  6938. perf_output_put(&handle, lost_samples_event);
  6939. perf_event__output_id_sample(event, &handle, &sample);
  6940. perf_output_end(&handle);
  6941. }
  6942. /*
  6943. * context_switch tracking
  6944. */
  6945. struct perf_switch_event {
  6946. struct task_struct *task;
  6947. struct task_struct *next_prev;
  6948. struct {
  6949. struct perf_event_header header;
  6950. u32 next_prev_pid;
  6951. u32 next_prev_tid;
  6952. } event_id;
  6953. };
  6954. static int perf_event_switch_match(struct perf_event *event)
  6955. {
  6956. return event->attr.context_switch;
  6957. }
  6958. static void perf_event_switch_output(struct perf_event *event, void *data)
  6959. {
  6960. struct perf_switch_event *se = data;
  6961. struct perf_output_handle handle;
  6962. struct perf_sample_data sample;
  6963. int ret;
  6964. if (!perf_event_switch_match(event))
  6965. return;
  6966. /* Only CPU-wide events are allowed to see next/prev pid/tid */
  6967. if (event->ctx->task) {
  6968. se->event_id.header.type = PERF_RECORD_SWITCH;
  6969. se->event_id.header.size = sizeof(se->event_id.header);
  6970. } else {
  6971. se->event_id.header.type = PERF_RECORD_SWITCH_CPU_WIDE;
  6972. se->event_id.header.size = sizeof(se->event_id);
  6973. se->event_id.next_prev_pid =
  6974. perf_event_pid(event, se->next_prev);
  6975. se->event_id.next_prev_tid =
  6976. perf_event_tid(event, se->next_prev);
  6977. }
  6978. perf_event_header__init_id(&se->event_id.header, &sample, event);
  6979. ret = perf_output_begin(&handle, &sample, event, se->event_id.header.size);
  6980. if (ret)
  6981. return;
  6982. if (event->ctx->task)
  6983. perf_output_put(&handle, se->event_id.header);
  6984. else
  6985. perf_output_put(&handle, se->event_id);
  6986. perf_event__output_id_sample(event, &handle, &sample);
  6987. perf_output_end(&handle);
  6988. }
  6989. static void perf_event_switch(struct task_struct *task,
  6990. struct task_struct *next_prev, bool sched_in)
  6991. {
  6992. struct perf_switch_event switch_event;
  6993. /* N.B. caller checks nr_switch_events != 0 */
  6994. switch_event = (struct perf_switch_event){
  6995. .task = task,
  6996. .next_prev = next_prev,
  6997. .event_id = {
  6998. .header = {
  6999. /* .type */
  7000. .misc = sched_in ? 0 : PERF_RECORD_MISC_SWITCH_OUT,
  7001. /* .size */
  7002. },
  7003. /* .next_prev_pid */
  7004. /* .next_prev_tid */
  7005. },
  7006. };
  7007. if (!sched_in && task->state == TASK_RUNNING)
  7008. switch_event.event_id.header.misc |=
  7009. PERF_RECORD_MISC_SWITCH_OUT_PREEMPT;
  7010. perf_iterate_sb(perf_event_switch_output,
  7011. &switch_event,
  7012. NULL);
  7013. }
  7014. /*
  7015. * IRQ throttle logging
  7016. */
  7017. static void perf_log_throttle(struct perf_event *event, int enable)
  7018. {
  7019. struct perf_output_handle handle;
  7020. struct perf_sample_data sample;
  7021. int ret;
  7022. struct {
  7023. struct perf_event_header header;
  7024. u64 time;
  7025. u64 id;
  7026. u64 stream_id;
  7027. } throttle_event = {
  7028. .header = {
  7029. .type = PERF_RECORD_THROTTLE,
  7030. .misc = 0,
  7031. .size = sizeof(throttle_event),
  7032. },
  7033. .time = perf_event_clock(event),
  7034. .id = primary_event_id(event),
  7035. .stream_id = event->id,
  7036. };
  7037. if (enable)
  7038. throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
  7039. perf_event_header__init_id(&throttle_event.header, &sample, event);
  7040. ret = perf_output_begin(&handle, &sample, event,
  7041. throttle_event.header.size);
  7042. if (ret)
  7043. return;
  7044. perf_output_put(&handle, throttle_event);
  7045. perf_event__output_id_sample(event, &handle, &sample);
  7046. perf_output_end(&handle);
  7047. }
  7048. /*
  7049. * ksymbol register/unregister tracking
  7050. */
  7051. struct perf_ksymbol_event {
  7052. const char *name;
  7053. int name_len;
  7054. struct {
  7055. struct perf_event_header header;
  7056. u64 addr;
  7057. u32 len;
  7058. u16 ksym_type;
  7059. u16 flags;
  7060. } event_id;
  7061. };
  7062. static int perf_event_ksymbol_match(struct perf_event *event)
  7063. {
  7064. return event->attr.ksymbol;
  7065. }
  7066. static void perf_event_ksymbol_output(struct perf_event *event, void *data)
  7067. {
  7068. struct perf_ksymbol_event *ksymbol_event = data;
  7069. struct perf_output_handle handle;
  7070. struct perf_sample_data sample;
  7071. int ret;
  7072. if (!perf_event_ksymbol_match(event))
  7073. return;
  7074. perf_event_header__init_id(&ksymbol_event->event_id.header,
  7075. &sample, event);
  7076. ret = perf_output_begin(&handle, &sample, event,
  7077. ksymbol_event->event_id.header.size);
  7078. if (ret)
  7079. return;
  7080. perf_output_put(&handle, ksymbol_event->event_id);
  7081. __output_copy(&handle, ksymbol_event->name, ksymbol_event->name_len);
  7082. perf_event__output_id_sample(event, &handle, &sample);
  7083. perf_output_end(&handle);
  7084. }
  7085. void perf_event_ksymbol(u16 ksym_type, u64 addr, u32 len, bool unregister,
  7086. const char *sym)
  7087. {
  7088. struct perf_ksymbol_event ksymbol_event;
  7089. char name[KSYM_NAME_LEN];
  7090. u16 flags = 0;
  7091. int name_len;
  7092. if (!atomic_read(&nr_ksymbol_events))
  7093. return;
  7094. if (ksym_type >= PERF_RECORD_KSYMBOL_TYPE_MAX ||
  7095. ksym_type == PERF_RECORD_KSYMBOL_TYPE_UNKNOWN)
  7096. goto err;
  7097. strlcpy(name, sym, KSYM_NAME_LEN);
  7098. name_len = strlen(name) + 1;
  7099. while (!IS_ALIGNED(name_len, sizeof(u64)))
  7100. name[name_len++] = '\0';
  7101. BUILD_BUG_ON(KSYM_NAME_LEN % sizeof(u64));
  7102. if (unregister)
  7103. flags |= PERF_RECORD_KSYMBOL_FLAGS_UNREGISTER;
  7104. ksymbol_event = (struct perf_ksymbol_event){
  7105. .name = name,
  7106. .name_len = name_len,
  7107. .event_id = {
  7108. .header = {
  7109. .type = PERF_RECORD_KSYMBOL,
  7110. .size = sizeof(ksymbol_event.event_id) +
  7111. name_len,
  7112. },
  7113. .addr = addr,
  7114. .len = len,
  7115. .ksym_type = ksym_type,
  7116. .flags = flags,
  7117. },
  7118. };
  7119. perf_iterate_sb(perf_event_ksymbol_output, &ksymbol_event, NULL);
  7120. return;
  7121. err:
  7122. WARN_ONCE(1, "%s: Invalid KSYMBOL type 0x%x\n", __func__, ksym_type);
  7123. }
  7124. /*
  7125. * bpf program load/unload tracking
  7126. */
  7127. struct perf_bpf_event {
  7128. struct bpf_prog *prog;
  7129. struct {
  7130. struct perf_event_header header;
  7131. u16 type;
  7132. u16 flags;
  7133. u32 id;
  7134. u8 tag[BPF_TAG_SIZE];
  7135. } event_id;
  7136. };
  7137. static int perf_event_bpf_match(struct perf_event *event)
  7138. {
  7139. return event->attr.bpf_event;
  7140. }
  7141. static void perf_event_bpf_output(struct perf_event *event, void *data)
  7142. {
  7143. struct perf_bpf_event *bpf_event = data;
  7144. struct perf_output_handle handle;
  7145. struct perf_sample_data sample;
  7146. int ret;
  7147. if (!perf_event_bpf_match(event))
  7148. return;
  7149. perf_event_header__init_id(&bpf_event->event_id.header,
  7150. &sample, event);
  7151. ret = perf_output_begin(&handle, data, event,
  7152. bpf_event->event_id.header.size);
  7153. if (ret)
  7154. return;
  7155. perf_output_put(&handle, bpf_event->event_id);
  7156. perf_event__output_id_sample(event, &handle, &sample);
  7157. perf_output_end(&handle);
  7158. }
  7159. static void perf_event_bpf_emit_ksymbols(struct bpf_prog *prog,
  7160. enum perf_bpf_event_type type)
  7161. {
  7162. bool unregister = type == PERF_BPF_EVENT_PROG_UNLOAD;
  7163. int i;
  7164. if (prog->aux->func_cnt == 0) {
  7165. perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_BPF,
  7166. (u64)(unsigned long)prog->bpf_func,
  7167. prog->jited_len, unregister,
  7168. prog->aux->ksym.name);
  7169. } else {
  7170. for (i = 0; i < prog->aux->func_cnt; i++) {
  7171. struct bpf_prog *subprog = prog->aux->func[i];
  7172. perf_event_ksymbol(
  7173. PERF_RECORD_KSYMBOL_TYPE_BPF,
  7174. (u64)(unsigned long)subprog->bpf_func,
  7175. subprog->jited_len, unregister,
  7176. prog->aux->ksym.name);
  7177. }
  7178. }
  7179. }
  7180. void perf_event_bpf_event(struct bpf_prog *prog,
  7181. enum perf_bpf_event_type type,
  7182. u16 flags)
  7183. {
  7184. struct perf_bpf_event bpf_event;
  7185. if (type <= PERF_BPF_EVENT_UNKNOWN ||
  7186. type >= PERF_BPF_EVENT_MAX)
  7187. return;
  7188. switch (type) {
  7189. case PERF_BPF_EVENT_PROG_LOAD:
  7190. case PERF_BPF_EVENT_PROG_UNLOAD:
  7191. if (atomic_read(&nr_ksymbol_events))
  7192. perf_event_bpf_emit_ksymbols(prog, type);
  7193. break;
  7194. default:
  7195. break;
  7196. }
  7197. if (!atomic_read(&nr_bpf_events))
  7198. return;
  7199. bpf_event = (struct perf_bpf_event){
  7200. .prog = prog,
  7201. .event_id = {
  7202. .header = {
  7203. .type = PERF_RECORD_BPF_EVENT,
  7204. .size = sizeof(bpf_event.event_id),
  7205. },
  7206. .type = type,
  7207. .flags = flags,
  7208. .id = prog->aux->id,
  7209. },
  7210. };
  7211. BUILD_BUG_ON(BPF_TAG_SIZE % sizeof(u64));
  7212. memcpy(bpf_event.event_id.tag, prog->tag, BPF_TAG_SIZE);
  7213. perf_iterate_sb(perf_event_bpf_output, &bpf_event, NULL);
  7214. }
  7215. struct perf_text_poke_event {
  7216. const void *old_bytes;
  7217. const void *new_bytes;
  7218. size_t pad;
  7219. u16 old_len;
  7220. u16 new_len;
  7221. struct {
  7222. struct perf_event_header header;
  7223. u64 addr;
  7224. } event_id;
  7225. };
  7226. static int perf_event_text_poke_match(struct perf_event *event)
  7227. {
  7228. return event->attr.text_poke;
  7229. }
  7230. static void perf_event_text_poke_output(struct perf_event *event, void *data)
  7231. {
  7232. struct perf_text_poke_event *text_poke_event = data;
  7233. struct perf_output_handle handle;
  7234. struct perf_sample_data sample;
  7235. u64 padding = 0;
  7236. int ret;
  7237. if (!perf_event_text_poke_match(event))
  7238. return;
  7239. perf_event_header__init_id(&text_poke_event->event_id.header, &sample, event);
  7240. ret = perf_output_begin(&handle, &sample, event,
  7241. text_poke_event->event_id.header.size);
  7242. if (ret)
  7243. return;
  7244. perf_output_put(&handle, text_poke_event->event_id);
  7245. perf_output_put(&handle, text_poke_event->old_len);
  7246. perf_output_put(&handle, text_poke_event->new_len);
  7247. __output_copy(&handle, text_poke_event->old_bytes, text_poke_event->old_len);
  7248. __output_copy(&handle, text_poke_event->new_bytes, text_poke_event->new_len);
  7249. if (text_poke_event->pad)
  7250. __output_copy(&handle, &padding, text_poke_event->pad);
  7251. perf_event__output_id_sample(event, &handle, &sample);
  7252. perf_output_end(&handle);
  7253. }
  7254. void perf_event_text_poke(const void *addr, const void *old_bytes,
  7255. size_t old_len, const void *new_bytes, size_t new_len)
  7256. {
  7257. struct perf_text_poke_event text_poke_event;
  7258. size_t tot, pad;
  7259. if (!atomic_read(&nr_text_poke_events))
  7260. return;
  7261. tot = sizeof(text_poke_event.old_len) + old_len;
  7262. tot += sizeof(text_poke_event.new_len) + new_len;
  7263. pad = ALIGN(tot, sizeof(u64)) - tot;
  7264. text_poke_event = (struct perf_text_poke_event){
  7265. .old_bytes = old_bytes,
  7266. .new_bytes = new_bytes,
  7267. .pad = pad,
  7268. .old_len = old_len,
  7269. .new_len = new_len,
  7270. .event_id = {
  7271. .header = {
  7272. .type = PERF_RECORD_TEXT_POKE,
  7273. .misc = PERF_RECORD_MISC_KERNEL,
  7274. .size = sizeof(text_poke_event.event_id) + tot + pad,
  7275. },
  7276. .addr = (unsigned long)addr,
  7277. },
  7278. };
  7279. perf_iterate_sb(perf_event_text_poke_output, &text_poke_event, NULL);
  7280. }
  7281. void perf_event_itrace_started(struct perf_event *event)
  7282. {
  7283. event->attach_state |= PERF_ATTACH_ITRACE;
  7284. }
  7285. static void perf_log_itrace_start(struct perf_event *event)
  7286. {
  7287. struct perf_output_handle handle;
  7288. struct perf_sample_data sample;
  7289. struct perf_aux_event {
  7290. struct perf_event_header header;
  7291. u32 pid;
  7292. u32 tid;
  7293. } rec;
  7294. int ret;
  7295. if (event->parent)
  7296. event = event->parent;
  7297. if (!(event->pmu->capabilities & PERF_PMU_CAP_ITRACE) ||
  7298. event->attach_state & PERF_ATTACH_ITRACE)
  7299. return;
  7300. rec.header.type = PERF_RECORD_ITRACE_START;
  7301. rec.header.misc = 0;
  7302. rec.header.size = sizeof(rec);
  7303. rec.pid = perf_event_pid(event, current);
  7304. rec.tid = perf_event_tid(event, current);
  7305. perf_event_header__init_id(&rec.header, &sample, event);
  7306. ret = perf_output_begin(&handle, &sample, event, rec.header.size);
  7307. if (ret)
  7308. return;
  7309. perf_output_put(&handle, rec);
  7310. perf_event__output_id_sample(event, &handle, &sample);
  7311. perf_output_end(&handle);
  7312. }
  7313. static int
  7314. __perf_event_account_interrupt(struct perf_event *event, int throttle)
  7315. {
  7316. struct hw_perf_event *hwc = &event->hw;
  7317. int ret = 0;
  7318. u64 seq;
  7319. seq = __this_cpu_read(perf_throttled_seq);
  7320. if (seq != hwc->interrupts_seq) {
  7321. hwc->interrupts_seq = seq;
  7322. hwc->interrupts = 1;
  7323. } else {
  7324. hwc->interrupts++;
  7325. if (unlikely(throttle
  7326. && hwc->interrupts >= max_samples_per_tick)) {
  7327. __this_cpu_inc(perf_throttled_count);
  7328. tick_dep_set_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS);
  7329. hwc->interrupts = MAX_INTERRUPTS;
  7330. perf_log_throttle(event, 0);
  7331. ret = 1;
  7332. }
  7333. }
  7334. if (event->attr.freq) {
  7335. u64 now = perf_clock();
  7336. s64 delta = now - hwc->freq_time_stamp;
  7337. hwc->freq_time_stamp = now;
  7338. if (delta > 0 && delta < 2*TICK_NSEC)
  7339. perf_adjust_period(event, delta, hwc->last_period, true);
  7340. }
  7341. return ret;
  7342. }
  7343. int perf_event_account_interrupt(struct perf_event *event)
  7344. {
  7345. return __perf_event_account_interrupt(event, 1);
  7346. }
  7347. /*
  7348. * Generic event overflow handling, sampling.
  7349. */
  7350. static int __perf_event_overflow(struct perf_event *event,
  7351. int throttle, struct perf_sample_data *data,
  7352. struct pt_regs *regs)
  7353. {
  7354. int events = atomic_read(&event->event_limit);
  7355. int ret = 0;
  7356. /*
  7357. * Non-sampling counters might still use the PMI to fold short
  7358. * hardware counters, ignore those.
  7359. */
  7360. if (unlikely(!is_sampling_event(event)))
  7361. return 0;
  7362. ret = __perf_event_account_interrupt(event, throttle);
  7363. /*
  7364. * XXX event_limit might not quite work as expected on inherited
  7365. * events
  7366. */
  7367. event->pending_kill = POLL_IN;
  7368. if (events && atomic_dec_and_test(&event->event_limit)) {
  7369. ret = 1;
  7370. event->pending_kill = POLL_HUP;
  7371. perf_event_disable_inatomic(event);
  7372. }
  7373. READ_ONCE(event->overflow_handler)(event, data, regs);
  7374. if (*perf_event_fasync(event) && event->pending_kill) {
  7375. event->pending_wakeup = 1;
  7376. irq_work_queue(&event->pending);
  7377. }
  7378. return ret;
  7379. }
  7380. int perf_event_overflow(struct perf_event *event,
  7381. struct perf_sample_data *data,
  7382. struct pt_regs *regs)
  7383. {
  7384. return __perf_event_overflow(event, 1, data, regs);
  7385. }
  7386. /*
  7387. * Generic software event infrastructure
  7388. */
  7389. struct swevent_htable {
  7390. struct swevent_hlist *swevent_hlist;
  7391. struct mutex hlist_mutex;
  7392. int hlist_refcount;
  7393. /* Recursion avoidance in each contexts */
  7394. int recursion[PERF_NR_CONTEXTS];
  7395. };
  7396. static DEFINE_PER_CPU(struct swevent_htable, swevent_htable);
  7397. /*
  7398. * We directly increment event->count and keep a second value in
  7399. * event->hw.period_left to count intervals. This period event
  7400. * is kept in the range [-sample_period, 0] so that we can use the
  7401. * sign as trigger.
  7402. */
  7403. u64 perf_swevent_set_period(struct perf_event *event)
  7404. {
  7405. struct hw_perf_event *hwc = &event->hw;
  7406. u64 period = hwc->last_period;
  7407. u64 nr, offset;
  7408. s64 old, val;
  7409. hwc->last_period = hwc->sample_period;
  7410. again:
  7411. old = val = local64_read(&hwc->period_left);
  7412. if (val < 0)
  7413. return 0;
  7414. nr = div64_u64(period + val, period);
  7415. offset = nr * period;
  7416. val -= offset;
  7417. if (local64_cmpxchg(&hwc->period_left, old, val) != old)
  7418. goto again;
  7419. return nr;
  7420. }
  7421. static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
  7422. struct perf_sample_data *data,
  7423. struct pt_regs *regs)
  7424. {
  7425. struct hw_perf_event *hwc = &event->hw;
  7426. int throttle = 0;
  7427. if (!overflow)
  7428. overflow = perf_swevent_set_period(event);
  7429. if (hwc->interrupts == MAX_INTERRUPTS)
  7430. return;
  7431. for (; overflow; overflow--) {
  7432. if (__perf_event_overflow(event, throttle,
  7433. data, regs)) {
  7434. /*
  7435. * We inhibit the overflow from happening when
  7436. * hwc->interrupts == MAX_INTERRUPTS.
  7437. */
  7438. break;
  7439. }
  7440. throttle = 1;
  7441. }
  7442. }
  7443. static void perf_swevent_event(struct perf_event *event, u64 nr,
  7444. struct perf_sample_data *data,
  7445. struct pt_regs *regs)
  7446. {
  7447. struct hw_perf_event *hwc = &event->hw;
  7448. local64_add(nr, &event->count);
  7449. if (!regs)
  7450. return;
  7451. if (!is_sampling_event(event))
  7452. return;
  7453. if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) {
  7454. data->period = nr;
  7455. return perf_swevent_overflow(event, 1, data, regs);
  7456. } else
  7457. data->period = event->hw.last_period;
  7458. if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
  7459. return perf_swevent_overflow(event, 1, data, regs);
  7460. if (local64_add_negative(nr, &hwc->period_left))
  7461. return;
  7462. perf_swevent_overflow(event, 0, data, regs);
  7463. }
  7464. static int perf_exclude_event(struct perf_event *event,
  7465. struct pt_regs *regs)
  7466. {
  7467. if (event->hw.state & PERF_HES_STOPPED)
  7468. return 1;
  7469. if (regs) {
  7470. if (event->attr.exclude_user && user_mode(regs))
  7471. return 1;
  7472. if (event->attr.exclude_kernel && !user_mode(regs))
  7473. return 1;
  7474. }
  7475. return 0;
  7476. }
  7477. static int perf_swevent_match(struct perf_event *event,
  7478. enum perf_type_id type,
  7479. u32 event_id,
  7480. struct perf_sample_data *data,
  7481. struct pt_regs *regs)
  7482. {
  7483. if (event->attr.type != type)
  7484. return 0;
  7485. if (event->attr.config != event_id)
  7486. return 0;
  7487. if (perf_exclude_event(event, regs))
  7488. return 0;
  7489. return 1;
  7490. }
  7491. static inline u64 swevent_hash(u64 type, u32 event_id)
  7492. {
  7493. u64 val = event_id | (type << 32);
  7494. return hash_64(val, SWEVENT_HLIST_BITS);
  7495. }
  7496. static inline struct hlist_head *
  7497. __find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id)
  7498. {
  7499. u64 hash = swevent_hash(type, event_id);
  7500. return &hlist->heads[hash];
  7501. }
  7502. /* For the read side: events when they trigger */
  7503. static inline struct hlist_head *
  7504. find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id)
  7505. {
  7506. struct swevent_hlist *hlist;
  7507. hlist = rcu_dereference(swhash->swevent_hlist);
  7508. if (!hlist)
  7509. return NULL;
  7510. return __find_swevent_head(hlist, type, event_id);
  7511. }
  7512. /* For the event head insertion and removal in the hlist */
  7513. static inline struct hlist_head *
  7514. find_swevent_head(struct swevent_htable *swhash, struct perf_event *event)
  7515. {
  7516. struct swevent_hlist *hlist;
  7517. u32 event_id = event->attr.config;
  7518. u64 type = event->attr.type;
  7519. /*
  7520. * Event scheduling is always serialized against hlist allocation
  7521. * and release. Which makes the protected version suitable here.
  7522. * The context lock guarantees that.
  7523. */
  7524. hlist = rcu_dereference_protected(swhash->swevent_hlist,
  7525. lockdep_is_held(&event->ctx->lock));
  7526. if (!hlist)
  7527. return NULL;
  7528. return __find_swevent_head(hlist, type, event_id);
  7529. }
  7530. static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
  7531. u64 nr,
  7532. struct perf_sample_data *data,
  7533. struct pt_regs *regs)
  7534. {
  7535. struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
  7536. struct perf_event *event;
  7537. struct hlist_head *head;
  7538. rcu_read_lock();
  7539. head = find_swevent_head_rcu(swhash, type, event_id);
  7540. if (!head)
  7541. goto end;
  7542. hlist_for_each_entry_rcu(event, head, hlist_entry) {
  7543. if (perf_swevent_match(event, type, event_id, data, regs))
  7544. perf_swevent_event(event, nr, data, regs);
  7545. }
  7546. end:
  7547. rcu_read_unlock();
  7548. }
  7549. DEFINE_PER_CPU(struct pt_regs, __perf_regs[4]);
  7550. int perf_swevent_get_recursion_context(void)
  7551. {
  7552. struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
  7553. return get_recursion_context(swhash->recursion);
  7554. }
  7555. EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
  7556. void perf_swevent_put_recursion_context(int rctx)
  7557. {
  7558. struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
  7559. put_recursion_context(swhash->recursion, rctx);
  7560. }
  7561. void ___perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
  7562. {
  7563. struct perf_sample_data data;
  7564. if (WARN_ON_ONCE(!regs))
  7565. return;
  7566. perf_sample_data_init(&data, addr, 0);
  7567. do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs);
  7568. }
  7569. void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
  7570. {
  7571. int rctx;
  7572. preempt_disable_notrace();
  7573. rctx = perf_swevent_get_recursion_context();
  7574. if (unlikely(rctx < 0))
  7575. goto fail;
  7576. ___perf_sw_event(event_id, nr, regs, addr);
  7577. perf_swevent_put_recursion_context(rctx);
  7578. fail:
  7579. preempt_enable_notrace();
  7580. }
  7581. static void perf_swevent_read(struct perf_event *event)
  7582. {
  7583. }
  7584. static int perf_swevent_add(struct perf_event *event, int flags)
  7585. {
  7586. struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
  7587. struct hw_perf_event *hwc = &event->hw;
  7588. struct hlist_head *head;
  7589. if (is_sampling_event(event)) {
  7590. hwc->last_period = hwc->sample_period;
  7591. perf_swevent_set_period(event);
  7592. }
  7593. hwc->state = !(flags & PERF_EF_START);
  7594. head = find_swevent_head(swhash, event);
  7595. if (WARN_ON_ONCE(!head))
  7596. return -EINVAL;
  7597. hlist_add_head_rcu(&event->hlist_entry, head);
  7598. perf_event_update_userpage(event);
  7599. return 0;
  7600. }
  7601. static void perf_swevent_del(struct perf_event *event, int flags)
  7602. {
  7603. hlist_del_rcu(&event->hlist_entry);
  7604. }
  7605. static void perf_swevent_start(struct perf_event *event, int flags)
  7606. {
  7607. event->hw.state = 0;
  7608. }
  7609. static void perf_swevent_stop(struct perf_event *event, int flags)
  7610. {
  7611. event->hw.state = PERF_HES_STOPPED;
  7612. }
  7613. /* Deref the hlist from the update side */
  7614. static inline struct swevent_hlist *
  7615. swevent_hlist_deref(struct swevent_htable *swhash)
  7616. {
  7617. return rcu_dereference_protected(swhash->swevent_hlist,
  7618. lockdep_is_held(&swhash->hlist_mutex));
  7619. }
  7620. static void swevent_hlist_release(struct swevent_htable *swhash)
  7621. {
  7622. struct swevent_hlist *hlist = swevent_hlist_deref(swhash);
  7623. if (!hlist)
  7624. return;
  7625. RCU_INIT_POINTER(swhash->swevent_hlist, NULL);
  7626. kfree_rcu(hlist, rcu_head);
  7627. }
  7628. static void swevent_hlist_put_cpu(int cpu)
  7629. {
  7630. struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
  7631. mutex_lock(&swhash->hlist_mutex);
  7632. if (!--swhash->hlist_refcount)
  7633. swevent_hlist_release(swhash);
  7634. mutex_unlock(&swhash->hlist_mutex);
  7635. }
  7636. static void swevent_hlist_put(void)
  7637. {
  7638. int cpu;
  7639. for_each_possible_cpu(cpu)
  7640. swevent_hlist_put_cpu(cpu);
  7641. }
  7642. static int swevent_hlist_get_cpu(int cpu)
  7643. {
  7644. struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
  7645. int err = 0;
  7646. mutex_lock(&swhash->hlist_mutex);
  7647. if (!swevent_hlist_deref(swhash) &&
  7648. cpumask_test_cpu(cpu, perf_online_mask)) {
  7649. struct swevent_hlist *hlist;
  7650. hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
  7651. if (!hlist) {
  7652. err = -ENOMEM;
  7653. goto exit;
  7654. }
  7655. rcu_assign_pointer(swhash->swevent_hlist, hlist);
  7656. }
  7657. swhash->hlist_refcount++;
  7658. exit:
  7659. mutex_unlock(&swhash->hlist_mutex);
  7660. return err;
  7661. }
  7662. static int swevent_hlist_get(void)
  7663. {
  7664. int err, cpu, failed_cpu;
  7665. mutex_lock(&pmus_lock);
  7666. for_each_possible_cpu(cpu) {
  7667. err = swevent_hlist_get_cpu(cpu);
  7668. if (err) {
  7669. failed_cpu = cpu;
  7670. goto fail;
  7671. }
  7672. }
  7673. mutex_unlock(&pmus_lock);
  7674. return 0;
  7675. fail:
  7676. for_each_possible_cpu(cpu) {
  7677. if (cpu == failed_cpu)
  7678. break;
  7679. swevent_hlist_put_cpu(cpu);
  7680. }
  7681. mutex_unlock(&pmus_lock);
  7682. return err;
  7683. }
  7684. struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
  7685. static void sw_perf_event_destroy(struct perf_event *event)
  7686. {
  7687. u64 event_id = event->attr.config;
  7688. WARN_ON(event->parent);
  7689. static_key_slow_dec(&perf_swevent_enabled[event_id]);
  7690. swevent_hlist_put();
  7691. }
  7692. static int perf_swevent_init(struct perf_event *event)
  7693. {
  7694. u64 event_id = event->attr.config;
  7695. if (event->attr.type != PERF_TYPE_SOFTWARE)
  7696. return -ENOENT;
  7697. /*
  7698. * no branch sampling for software events
  7699. */
  7700. if (has_branch_stack(event))
  7701. return -EOPNOTSUPP;
  7702. switch (event_id) {
  7703. case PERF_COUNT_SW_CPU_CLOCK:
  7704. case PERF_COUNT_SW_TASK_CLOCK:
  7705. return -ENOENT;
  7706. default:
  7707. break;
  7708. }
  7709. if (event_id >= PERF_COUNT_SW_MAX)
  7710. return -ENOENT;
  7711. if (!event->parent) {
  7712. int err;
  7713. err = swevent_hlist_get();
  7714. if (err)
  7715. return err;
  7716. static_key_slow_inc(&perf_swevent_enabled[event_id]);
  7717. event->destroy = sw_perf_event_destroy;
  7718. }
  7719. return 0;
  7720. }
  7721. static struct pmu perf_swevent = {
  7722. .task_ctx_nr = perf_sw_context,
  7723. .capabilities = PERF_PMU_CAP_NO_NMI,
  7724. .event_init = perf_swevent_init,
  7725. .add = perf_swevent_add,
  7726. .del = perf_swevent_del,
  7727. .start = perf_swevent_start,
  7728. .stop = perf_swevent_stop,
  7729. .read = perf_swevent_read,
  7730. };
  7731. #ifdef CONFIG_EVENT_TRACING
  7732. static int perf_tp_filter_match(struct perf_event *event,
  7733. struct perf_sample_data *data)
  7734. {
  7735. void *record = data->raw->frag.data;
  7736. /* only top level events have filters set */
  7737. if (event->parent)
  7738. event = event->parent;
  7739. if (likely(!event->filter) || filter_match_preds(event->filter, record))
  7740. return 1;
  7741. return 0;
  7742. }
  7743. static int perf_tp_event_match(struct perf_event *event,
  7744. struct perf_sample_data *data,
  7745. struct pt_regs *regs)
  7746. {
  7747. if (event->hw.state & PERF_HES_STOPPED)
  7748. return 0;
  7749. /*
  7750. * If exclude_kernel, only trace user-space tracepoints (uprobes)
  7751. */
  7752. if (event->attr.exclude_kernel && !user_mode(regs))
  7753. return 0;
  7754. if (!perf_tp_filter_match(event, data))
  7755. return 0;
  7756. return 1;
  7757. }
  7758. void perf_trace_run_bpf_submit(void *raw_data, int size, int rctx,
  7759. struct trace_event_call *call, u64 count,
  7760. struct pt_regs *regs, struct hlist_head *head,
  7761. struct task_struct *task)
  7762. {
  7763. if (bpf_prog_array_valid(call)) {
  7764. *(struct pt_regs **)raw_data = regs;
  7765. if (!trace_call_bpf(call, raw_data) || hlist_empty(head)) {
  7766. perf_swevent_put_recursion_context(rctx);
  7767. return;
  7768. }
  7769. }
  7770. perf_tp_event(call->event.type, count, raw_data, size, regs, head,
  7771. rctx, task);
  7772. }
  7773. EXPORT_SYMBOL_GPL(perf_trace_run_bpf_submit);
  7774. void perf_tp_event(u16 event_type, u64 count, void *record, int entry_size,
  7775. struct pt_regs *regs, struct hlist_head *head, int rctx,
  7776. struct task_struct *task)
  7777. {
  7778. struct perf_sample_data data;
  7779. struct perf_event *event;
  7780. struct perf_raw_record raw = {
  7781. .frag = {
  7782. .size = entry_size,
  7783. .data = record,
  7784. },
  7785. };
  7786. perf_sample_data_init(&data, 0, 0);
  7787. data.raw = &raw;
  7788. perf_trace_buf_update(record, event_type);
  7789. hlist_for_each_entry_rcu(event, head, hlist_entry) {
  7790. if (perf_tp_event_match(event, &data, regs))
  7791. perf_swevent_event(event, count, &data, regs);
  7792. }
  7793. /*
  7794. * If we got specified a target task, also iterate its context and
  7795. * deliver this event there too.
  7796. */
  7797. if (task && task != current) {
  7798. struct perf_event_context *ctx;
  7799. struct trace_entry *entry = record;
  7800. rcu_read_lock();
  7801. ctx = rcu_dereference(task->perf_event_ctxp[perf_sw_context]);
  7802. if (!ctx)
  7803. goto unlock;
  7804. list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
  7805. if (event->cpu != smp_processor_id())
  7806. continue;
  7807. if (event->attr.type != PERF_TYPE_TRACEPOINT)
  7808. continue;
  7809. if (event->attr.config != entry->type)
  7810. continue;
  7811. if (perf_tp_event_match(event, &data, regs))
  7812. perf_swevent_event(event, count, &data, regs);
  7813. }
  7814. unlock:
  7815. rcu_read_unlock();
  7816. }
  7817. perf_swevent_put_recursion_context(rctx);
  7818. }
  7819. EXPORT_SYMBOL_GPL(perf_tp_event);
  7820. static void tp_perf_event_destroy(struct perf_event *event)
  7821. {
  7822. perf_trace_destroy(event);
  7823. }
  7824. static int perf_tp_event_init(struct perf_event *event)
  7825. {
  7826. int err;
  7827. if (event->attr.type != PERF_TYPE_TRACEPOINT)
  7828. return -ENOENT;
  7829. /*
  7830. * no branch sampling for tracepoint events
  7831. */
  7832. if (has_branch_stack(event))
  7833. return -EOPNOTSUPP;
  7834. err = perf_trace_init(event);
  7835. if (err)
  7836. return err;
  7837. event->destroy = tp_perf_event_destroy;
  7838. return 0;
  7839. }
  7840. static struct pmu perf_tracepoint = {
  7841. .task_ctx_nr = perf_sw_context,
  7842. .event_init = perf_tp_event_init,
  7843. .add = perf_trace_add,
  7844. .del = perf_trace_del,
  7845. .start = perf_swevent_start,
  7846. .stop = perf_swevent_stop,
  7847. .read = perf_swevent_read,
  7848. };
  7849. #if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS)
  7850. /*
  7851. * Flags in config, used by dynamic PMU kprobe and uprobe
  7852. * The flags should match following PMU_FORMAT_ATTR().
  7853. *
  7854. * PERF_PROBE_CONFIG_IS_RETPROBE if set, create kretprobe/uretprobe
  7855. * if not set, create kprobe/uprobe
  7856. *
  7857. * The following values specify a reference counter (or semaphore in the
  7858. * terminology of tools like dtrace, systemtap, etc.) Userspace Statically
  7859. * Defined Tracepoints (USDT). Currently, we use 40 bit for the offset.
  7860. *
  7861. * PERF_UPROBE_REF_CTR_OFFSET_BITS # of bits in config as th offset
  7862. * PERF_UPROBE_REF_CTR_OFFSET_SHIFT # of bits to shift left
  7863. */
  7864. enum perf_probe_config {
  7865. PERF_PROBE_CONFIG_IS_RETPROBE = 1U << 0, /* [k,u]retprobe */
  7866. PERF_UPROBE_REF_CTR_OFFSET_BITS = 32,
  7867. PERF_UPROBE_REF_CTR_OFFSET_SHIFT = 64 - PERF_UPROBE_REF_CTR_OFFSET_BITS,
  7868. };
  7869. PMU_FORMAT_ATTR(retprobe, "config:0");
  7870. #endif
  7871. #ifdef CONFIG_KPROBE_EVENTS
  7872. static struct attribute *kprobe_attrs[] = {
  7873. &format_attr_retprobe.attr,
  7874. NULL,
  7875. };
  7876. static struct attribute_group kprobe_format_group = {
  7877. .name = "format",
  7878. .attrs = kprobe_attrs,
  7879. };
  7880. static const struct attribute_group *kprobe_attr_groups[] = {
  7881. &kprobe_format_group,
  7882. NULL,
  7883. };
  7884. static int perf_kprobe_event_init(struct perf_event *event);
  7885. static struct pmu perf_kprobe = {
  7886. .task_ctx_nr = perf_sw_context,
  7887. .event_init = perf_kprobe_event_init,
  7888. .add = perf_trace_add,
  7889. .del = perf_trace_del,
  7890. .start = perf_swevent_start,
  7891. .stop = perf_swevent_stop,
  7892. .read = perf_swevent_read,
  7893. .attr_groups = kprobe_attr_groups,
  7894. };
  7895. static int perf_kprobe_event_init(struct perf_event *event)
  7896. {
  7897. int err;
  7898. bool is_retprobe;
  7899. if (event->attr.type != perf_kprobe.type)
  7900. return -ENOENT;
  7901. if (!perfmon_capable())
  7902. return -EACCES;
  7903. /*
  7904. * no branch sampling for probe events
  7905. */
  7906. if (has_branch_stack(event))
  7907. return -EOPNOTSUPP;
  7908. is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE;
  7909. err = perf_kprobe_init(event, is_retprobe);
  7910. if (err)
  7911. return err;
  7912. event->destroy = perf_kprobe_destroy;
  7913. return 0;
  7914. }
  7915. #endif /* CONFIG_KPROBE_EVENTS */
  7916. #ifdef CONFIG_UPROBE_EVENTS
  7917. PMU_FORMAT_ATTR(ref_ctr_offset, "config:32-63");
  7918. static struct attribute *uprobe_attrs[] = {
  7919. &format_attr_retprobe.attr,
  7920. &format_attr_ref_ctr_offset.attr,
  7921. NULL,
  7922. };
  7923. static struct attribute_group uprobe_format_group = {
  7924. .name = "format",
  7925. .attrs = uprobe_attrs,
  7926. };
  7927. static const struct attribute_group *uprobe_attr_groups[] = {
  7928. &uprobe_format_group,
  7929. NULL,
  7930. };
  7931. static int perf_uprobe_event_init(struct perf_event *event);
  7932. static struct pmu perf_uprobe = {
  7933. .task_ctx_nr = perf_sw_context,
  7934. .event_init = perf_uprobe_event_init,
  7935. .add = perf_trace_add,
  7936. .del = perf_trace_del,
  7937. .start = perf_swevent_start,
  7938. .stop = perf_swevent_stop,
  7939. .read = perf_swevent_read,
  7940. .attr_groups = uprobe_attr_groups,
  7941. };
  7942. static int perf_uprobe_event_init(struct perf_event *event)
  7943. {
  7944. int err;
  7945. unsigned long ref_ctr_offset;
  7946. bool is_retprobe;
  7947. if (event->attr.type != perf_uprobe.type)
  7948. return -ENOENT;
  7949. if (!perfmon_capable())
  7950. return -EACCES;
  7951. /*
  7952. * no branch sampling for probe events
  7953. */
  7954. if (has_branch_stack(event))
  7955. return -EOPNOTSUPP;
  7956. is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE;
  7957. ref_ctr_offset = event->attr.config >> PERF_UPROBE_REF_CTR_OFFSET_SHIFT;
  7958. err = perf_uprobe_init(event, ref_ctr_offset, is_retprobe);
  7959. if (err)
  7960. return err;
  7961. event->destroy = perf_uprobe_destroy;
  7962. return 0;
  7963. }
  7964. #endif /* CONFIG_UPROBE_EVENTS */
  7965. static inline void perf_tp_register(void)
  7966. {
  7967. perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT);
  7968. #ifdef CONFIG_KPROBE_EVENTS
  7969. perf_pmu_register(&perf_kprobe, "kprobe", -1);
  7970. #endif
  7971. #ifdef CONFIG_UPROBE_EVENTS
  7972. perf_pmu_register(&perf_uprobe, "uprobe", -1);
  7973. #endif
  7974. }
  7975. static void perf_event_free_filter(struct perf_event *event)
  7976. {
  7977. ftrace_profile_free_filter(event);
  7978. }
  7979. #ifdef CONFIG_BPF_SYSCALL
  7980. static void bpf_overflow_handler(struct perf_event *event,
  7981. struct perf_sample_data *data,
  7982. struct pt_regs *regs)
  7983. {
  7984. struct bpf_perf_event_data_kern ctx = {
  7985. .data = data,
  7986. .event = event,
  7987. };
  7988. int ret = 0;
  7989. ctx.regs = perf_arch_bpf_user_pt_regs(regs);
  7990. if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1))
  7991. goto out;
  7992. rcu_read_lock();
  7993. ret = BPF_PROG_RUN(event->prog, &ctx);
  7994. rcu_read_unlock();
  7995. out:
  7996. __this_cpu_dec(bpf_prog_active);
  7997. if (!ret)
  7998. return;
  7999. event->orig_overflow_handler(event, data, regs);
  8000. }
  8001. static int perf_event_set_bpf_handler(struct perf_event *event, u32 prog_fd)
  8002. {
  8003. struct bpf_prog *prog;
  8004. if (event->overflow_handler_context)
  8005. /* hw breakpoint or kernel counter */
  8006. return -EINVAL;
  8007. if (event->prog)
  8008. return -EEXIST;
  8009. prog = bpf_prog_get_type(prog_fd, BPF_PROG_TYPE_PERF_EVENT);
  8010. if (IS_ERR(prog))
  8011. return PTR_ERR(prog);
  8012. if (event->attr.precise_ip &&
  8013. prog->call_get_stack &&
  8014. (!(event->attr.sample_type & __PERF_SAMPLE_CALLCHAIN_EARLY) ||
  8015. event->attr.exclude_callchain_kernel ||
  8016. event->attr.exclude_callchain_user)) {
  8017. /*
  8018. * On perf_event with precise_ip, calling bpf_get_stack()
  8019. * may trigger unwinder warnings and occasional crashes.
  8020. * bpf_get_[stack|stackid] works around this issue by using
  8021. * callchain attached to perf_sample_data. If the
  8022. * perf_event does not full (kernel and user) callchain
  8023. * attached to perf_sample_data, do not allow attaching BPF
  8024. * program that calls bpf_get_[stack|stackid].
  8025. */
  8026. bpf_prog_put(prog);
  8027. return -EPROTO;
  8028. }
  8029. event->prog = prog;
  8030. event->orig_overflow_handler = READ_ONCE(event->overflow_handler);
  8031. WRITE_ONCE(event->overflow_handler, bpf_overflow_handler);
  8032. return 0;
  8033. }
  8034. static void perf_event_free_bpf_handler(struct perf_event *event)
  8035. {
  8036. struct bpf_prog *prog = event->prog;
  8037. if (!prog)
  8038. return;
  8039. WRITE_ONCE(event->overflow_handler, event->orig_overflow_handler);
  8040. event->prog = NULL;
  8041. bpf_prog_put(prog);
  8042. }
  8043. #else
  8044. static int perf_event_set_bpf_handler(struct perf_event *event, u32 prog_fd)
  8045. {
  8046. return -EOPNOTSUPP;
  8047. }
  8048. static void perf_event_free_bpf_handler(struct perf_event *event)
  8049. {
  8050. }
  8051. #endif
  8052. /*
  8053. * returns true if the event is a tracepoint, or a kprobe/upprobe created
  8054. * with perf_event_open()
  8055. */
  8056. static inline bool perf_event_is_tracing(struct perf_event *event)
  8057. {
  8058. if (event->pmu == &perf_tracepoint)
  8059. return true;
  8060. #ifdef CONFIG_KPROBE_EVENTS
  8061. if (event->pmu == &perf_kprobe)
  8062. return true;
  8063. #endif
  8064. #ifdef CONFIG_UPROBE_EVENTS
  8065. if (event->pmu == &perf_uprobe)
  8066. return true;
  8067. #endif
  8068. return false;
  8069. }
  8070. static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd)
  8071. {
  8072. bool is_kprobe, is_tracepoint, is_syscall_tp;
  8073. struct bpf_prog *prog;
  8074. int ret;
  8075. if (!perf_event_is_tracing(event))
  8076. return perf_event_set_bpf_handler(event, prog_fd);
  8077. is_kprobe = event->tp_event->flags & TRACE_EVENT_FL_UKPROBE;
  8078. is_tracepoint = event->tp_event->flags & TRACE_EVENT_FL_TRACEPOINT;
  8079. is_syscall_tp = is_syscall_trace_event(event->tp_event);
  8080. if (!is_kprobe && !is_tracepoint && !is_syscall_tp)
  8081. /* bpf programs can only be attached to u/kprobe or tracepoint */
  8082. return -EINVAL;
  8083. prog = bpf_prog_get(prog_fd);
  8084. if (IS_ERR(prog))
  8085. return PTR_ERR(prog);
  8086. if ((is_kprobe && prog->type != BPF_PROG_TYPE_KPROBE) ||
  8087. (is_tracepoint && prog->type != BPF_PROG_TYPE_TRACEPOINT) ||
  8088. (is_syscall_tp && prog->type != BPF_PROG_TYPE_TRACEPOINT)) {
  8089. /* valid fd, but invalid bpf program type */
  8090. bpf_prog_put(prog);
  8091. return -EINVAL;
  8092. }
  8093. /* Kprobe override only works for kprobes, not uprobes. */
  8094. if (prog->kprobe_override &&
  8095. !(event->tp_event->flags & TRACE_EVENT_FL_KPROBE)) {
  8096. bpf_prog_put(prog);
  8097. return -EINVAL;
  8098. }
  8099. if (is_tracepoint || is_syscall_tp) {
  8100. int off = trace_event_get_offsets(event->tp_event);
  8101. if (prog->aux->max_ctx_offset > off) {
  8102. bpf_prog_put(prog);
  8103. return -EACCES;
  8104. }
  8105. }
  8106. ret = perf_event_attach_bpf_prog(event, prog);
  8107. if (ret)
  8108. bpf_prog_put(prog);
  8109. return ret;
  8110. }
  8111. static void perf_event_free_bpf_prog(struct perf_event *event)
  8112. {
  8113. if (!perf_event_is_tracing(event)) {
  8114. perf_event_free_bpf_handler(event);
  8115. return;
  8116. }
  8117. perf_event_detach_bpf_prog(event);
  8118. }
  8119. #else
  8120. static inline void perf_tp_register(void)
  8121. {
  8122. }
  8123. static void perf_event_free_filter(struct perf_event *event)
  8124. {
  8125. }
  8126. static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd)
  8127. {
  8128. return -ENOENT;
  8129. }
  8130. static void perf_event_free_bpf_prog(struct perf_event *event)
  8131. {
  8132. }
  8133. #endif /* CONFIG_EVENT_TRACING */
  8134. #ifdef CONFIG_HAVE_HW_BREAKPOINT
  8135. void perf_bp_event(struct perf_event *bp, void *data)
  8136. {
  8137. struct perf_sample_data sample;
  8138. struct pt_regs *regs = data;
  8139. perf_sample_data_init(&sample, bp->attr.bp_addr, 0);
  8140. if (!bp->hw.state && !perf_exclude_event(bp, regs))
  8141. perf_swevent_event(bp, 1, &sample, regs);
  8142. }
  8143. #endif
  8144. /*
  8145. * Allocate a new address filter
  8146. */
  8147. static struct perf_addr_filter *
  8148. perf_addr_filter_new(struct perf_event *event, struct list_head *filters)
  8149. {
  8150. int node = cpu_to_node(event->cpu == -1 ? 0 : event->cpu);
  8151. struct perf_addr_filter *filter;
  8152. filter = kzalloc_node(sizeof(*filter), GFP_KERNEL, node);
  8153. if (!filter)
  8154. return NULL;
  8155. INIT_LIST_HEAD(&filter->entry);
  8156. list_add_tail(&filter->entry, filters);
  8157. return filter;
  8158. }
  8159. static void free_filters_list(struct list_head *filters)
  8160. {
  8161. struct perf_addr_filter *filter, *iter;
  8162. list_for_each_entry_safe(filter, iter, filters, entry) {
  8163. path_put(&filter->path);
  8164. list_del(&filter->entry);
  8165. kfree(filter);
  8166. }
  8167. }
  8168. /*
  8169. * Free existing address filters and optionally install new ones
  8170. */
  8171. static void perf_addr_filters_splice(struct perf_event *event,
  8172. struct list_head *head)
  8173. {
  8174. unsigned long flags;
  8175. LIST_HEAD(list);
  8176. if (!has_addr_filter(event))
  8177. return;
  8178. /* don't bother with children, they don't have their own filters */
  8179. if (event->parent)
  8180. return;
  8181. raw_spin_lock_irqsave(&event->addr_filters.lock, flags);
  8182. list_splice_init(&event->addr_filters.list, &list);
  8183. if (head)
  8184. list_splice(head, &event->addr_filters.list);
  8185. raw_spin_unlock_irqrestore(&event->addr_filters.lock, flags);
  8186. free_filters_list(&list);
  8187. }
  8188. /*
  8189. * Scan through mm's vmas and see if one of them matches the
  8190. * @filter; if so, adjust filter's address range.
  8191. * Called with mm::mmap_lock down for reading.
  8192. */
  8193. static void perf_addr_filter_apply(struct perf_addr_filter *filter,
  8194. struct mm_struct *mm,
  8195. struct perf_addr_filter_range *fr)
  8196. {
  8197. struct vm_area_struct *vma;
  8198. for (vma = mm->mmap; vma; vma = vma->vm_next) {
  8199. if (!vma->vm_file)
  8200. continue;
  8201. if (perf_addr_filter_vma_adjust(filter, vma, fr))
  8202. return;
  8203. }
  8204. }
  8205. /*
  8206. * Update event's address range filters based on the
  8207. * task's existing mappings, if any.
  8208. */
  8209. static void perf_event_addr_filters_apply(struct perf_event *event)
  8210. {
  8211. struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
  8212. struct task_struct *task = READ_ONCE(event->ctx->task);
  8213. struct perf_addr_filter *filter;
  8214. struct mm_struct *mm = NULL;
  8215. unsigned int count = 0;
  8216. unsigned long flags;
  8217. /*
  8218. * We may observe TASK_TOMBSTONE, which means that the event tear-down
  8219. * will stop on the parent's child_mutex that our caller is also holding
  8220. */
  8221. if (task == TASK_TOMBSTONE)
  8222. return;
  8223. if (ifh->nr_file_filters) {
  8224. mm = get_task_mm(task);
  8225. if (!mm)
  8226. goto restart;
  8227. mmap_read_lock(mm);
  8228. }
  8229. raw_spin_lock_irqsave(&ifh->lock, flags);
  8230. list_for_each_entry(filter, &ifh->list, entry) {
  8231. if (filter->path.dentry) {
  8232. /*
  8233. * Adjust base offset if the filter is associated to a
  8234. * binary that needs to be mapped:
  8235. */
  8236. event->addr_filter_ranges[count].start = 0;
  8237. event->addr_filter_ranges[count].size = 0;
  8238. perf_addr_filter_apply(filter, mm, &event->addr_filter_ranges[count]);
  8239. } else {
  8240. event->addr_filter_ranges[count].start = filter->offset;
  8241. event->addr_filter_ranges[count].size = filter->size;
  8242. }
  8243. count++;
  8244. }
  8245. event->addr_filters_gen++;
  8246. raw_spin_unlock_irqrestore(&ifh->lock, flags);
  8247. if (ifh->nr_file_filters) {
  8248. mmap_read_unlock(mm);
  8249. mmput(mm);
  8250. }
  8251. restart:
  8252. perf_event_stop(event, 1);
  8253. }
  8254. /*
  8255. * Address range filtering: limiting the data to certain
  8256. * instruction address ranges. Filters are ioctl()ed to us from
  8257. * userspace as ascii strings.
  8258. *
  8259. * Filter string format:
  8260. *
  8261. * ACTION RANGE_SPEC
  8262. * where ACTION is one of the
  8263. * * "filter": limit the trace to this region
  8264. * * "start": start tracing from this address
  8265. * * "stop": stop tracing at this address/region;
  8266. * RANGE_SPEC is
  8267. * * for kernel addresses: <start address>[/<size>]
  8268. * * for object files: <start address>[/<size>]@</path/to/object/file>
  8269. *
  8270. * if <size> is not specified or is zero, the range is treated as a single
  8271. * address; not valid for ACTION=="filter".
  8272. */
  8273. enum {
  8274. IF_ACT_NONE = -1,
  8275. IF_ACT_FILTER,
  8276. IF_ACT_START,
  8277. IF_ACT_STOP,
  8278. IF_SRC_FILE,
  8279. IF_SRC_KERNEL,
  8280. IF_SRC_FILEADDR,
  8281. IF_SRC_KERNELADDR,
  8282. };
  8283. enum {
  8284. IF_STATE_ACTION = 0,
  8285. IF_STATE_SOURCE,
  8286. IF_STATE_END,
  8287. };
  8288. static const match_table_t if_tokens = {
  8289. { IF_ACT_FILTER, "filter" },
  8290. { IF_ACT_START, "start" },
  8291. { IF_ACT_STOP, "stop" },
  8292. { IF_SRC_FILE, "%u/%u@%s" },
  8293. { IF_SRC_KERNEL, "%u/%u" },
  8294. { IF_SRC_FILEADDR, "%u@%s" },
  8295. { IF_SRC_KERNELADDR, "%u" },
  8296. { IF_ACT_NONE, NULL },
  8297. };
  8298. /*
  8299. * Address filter string parser
  8300. */
  8301. static int
  8302. perf_event_parse_addr_filter(struct perf_event *event, char *fstr,
  8303. struct list_head *filters)
  8304. {
  8305. struct perf_addr_filter *filter = NULL;
  8306. char *start, *orig, *filename = NULL;
  8307. substring_t args[MAX_OPT_ARGS];
  8308. int state = IF_STATE_ACTION, token;
  8309. unsigned int kernel = 0;
  8310. int ret = -EINVAL;
  8311. orig = fstr = kstrdup(fstr, GFP_KERNEL);
  8312. if (!fstr)
  8313. return -ENOMEM;
  8314. while ((start = strsep(&fstr, " ,\n")) != NULL) {
  8315. static const enum perf_addr_filter_action_t actions[] = {
  8316. [IF_ACT_FILTER] = PERF_ADDR_FILTER_ACTION_FILTER,
  8317. [IF_ACT_START] = PERF_ADDR_FILTER_ACTION_START,
  8318. [IF_ACT_STOP] = PERF_ADDR_FILTER_ACTION_STOP,
  8319. };
  8320. ret = -EINVAL;
  8321. if (!*start)
  8322. continue;
  8323. /* filter definition begins */
  8324. if (state == IF_STATE_ACTION) {
  8325. filter = perf_addr_filter_new(event, filters);
  8326. if (!filter)
  8327. goto fail;
  8328. }
  8329. token = match_token(start, if_tokens, args);
  8330. switch (token) {
  8331. case IF_ACT_FILTER:
  8332. case IF_ACT_START:
  8333. case IF_ACT_STOP:
  8334. if (state != IF_STATE_ACTION)
  8335. goto fail;
  8336. filter->action = actions[token];
  8337. state = IF_STATE_SOURCE;
  8338. break;
  8339. case IF_SRC_KERNELADDR:
  8340. case IF_SRC_KERNEL:
  8341. kernel = 1;
  8342. fallthrough;
  8343. case IF_SRC_FILEADDR:
  8344. case IF_SRC_FILE:
  8345. if (state != IF_STATE_SOURCE)
  8346. goto fail;
  8347. *args[0].to = 0;
  8348. ret = kstrtoul(args[0].from, 0, &filter->offset);
  8349. if (ret)
  8350. goto fail;
  8351. if (token == IF_SRC_KERNEL || token == IF_SRC_FILE) {
  8352. *args[1].to = 0;
  8353. ret = kstrtoul(args[1].from, 0, &filter->size);
  8354. if (ret)
  8355. goto fail;
  8356. }
  8357. if (token == IF_SRC_FILE || token == IF_SRC_FILEADDR) {
  8358. int fpos = token == IF_SRC_FILE ? 2 : 1;
  8359. kfree(filename);
  8360. filename = match_strdup(&args[fpos]);
  8361. if (!filename) {
  8362. ret = -ENOMEM;
  8363. goto fail;
  8364. }
  8365. }
  8366. state = IF_STATE_END;
  8367. break;
  8368. default:
  8369. goto fail;
  8370. }
  8371. /*
  8372. * Filter definition is fully parsed, validate and install it.
  8373. * Make sure that it doesn't contradict itself or the event's
  8374. * attribute.
  8375. */
  8376. if (state == IF_STATE_END) {
  8377. ret = -EINVAL;
  8378. if (kernel && event->attr.exclude_kernel)
  8379. goto fail;
  8380. /*
  8381. * ACTION "filter" must have a non-zero length region
  8382. * specified.
  8383. */
  8384. if (filter->action == PERF_ADDR_FILTER_ACTION_FILTER &&
  8385. !filter->size)
  8386. goto fail;
  8387. if (!kernel) {
  8388. if (!filename)
  8389. goto fail;
  8390. /*
  8391. * For now, we only support file-based filters
  8392. * in per-task events; doing so for CPU-wide
  8393. * events requires additional context switching
  8394. * trickery, since same object code will be
  8395. * mapped at different virtual addresses in
  8396. * different processes.
  8397. */
  8398. ret = -EOPNOTSUPP;
  8399. if (!event->ctx->task)
  8400. goto fail;
  8401. /* look up the path and grab its inode */
  8402. ret = kern_path(filename, LOOKUP_FOLLOW,
  8403. &filter->path);
  8404. if (ret)
  8405. goto fail;
  8406. ret = -EINVAL;
  8407. if (!filter->path.dentry ||
  8408. !S_ISREG(d_inode(filter->path.dentry)
  8409. ->i_mode))
  8410. goto fail;
  8411. event->addr_filters.nr_file_filters++;
  8412. }
  8413. /* ready to consume more filters */
  8414. kfree(filename);
  8415. filename = NULL;
  8416. state = IF_STATE_ACTION;
  8417. filter = NULL;
  8418. kernel = 0;
  8419. }
  8420. }
  8421. if (state != IF_STATE_ACTION)
  8422. goto fail;
  8423. kfree(filename);
  8424. kfree(orig);
  8425. return 0;
  8426. fail:
  8427. kfree(filename);
  8428. free_filters_list(filters);
  8429. kfree(orig);
  8430. return ret;
  8431. }
  8432. static int
  8433. perf_event_set_addr_filter(struct perf_event *event, char *filter_str)
  8434. {
  8435. LIST_HEAD(filters);
  8436. int ret;
  8437. /*
  8438. * Since this is called in perf_ioctl() path, we're already holding
  8439. * ctx::mutex.
  8440. */
  8441. lockdep_assert_held(&event->ctx->mutex);
  8442. if (WARN_ON_ONCE(event->parent))
  8443. return -EINVAL;
  8444. ret = perf_event_parse_addr_filter(event, filter_str, &filters);
  8445. if (ret)
  8446. goto fail_clear_files;
  8447. ret = event->pmu->addr_filters_validate(&filters);
  8448. if (ret)
  8449. goto fail_free_filters;
  8450. /* remove existing filters, if any */
  8451. perf_addr_filters_splice(event, &filters);
  8452. /* install new filters */
  8453. perf_event_for_each_child(event, perf_event_addr_filters_apply);
  8454. return ret;
  8455. fail_free_filters:
  8456. free_filters_list(&filters);
  8457. fail_clear_files:
  8458. event->addr_filters.nr_file_filters = 0;
  8459. return ret;
  8460. }
  8461. static int perf_event_set_filter(struct perf_event *event, void __user *arg)
  8462. {
  8463. int ret = -EINVAL;
  8464. char *filter_str;
  8465. filter_str = strndup_user(arg, PAGE_SIZE);
  8466. if (IS_ERR(filter_str))
  8467. return PTR_ERR(filter_str);
  8468. #ifdef CONFIG_EVENT_TRACING
  8469. if (perf_event_is_tracing(event)) {
  8470. struct perf_event_context *ctx = event->ctx;
  8471. /*
  8472. * Beware, here be dragons!!
  8473. *
  8474. * the tracepoint muck will deadlock against ctx->mutex, but
  8475. * the tracepoint stuff does not actually need it. So
  8476. * temporarily drop ctx->mutex. As per perf_event_ctx_lock() we
  8477. * already have a reference on ctx.
  8478. *
  8479. * This can result in event getting moved to a different ctx,
  8480. * but that does not affect the tracepoint state.
  8481. */
  8482. mutex_unlock(&ctx->mutex);
  8483. ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);
  8484. mutex_lock(&ctx->mutex);
  8485. } else
  8486. #endif
  8487. if (has_addr_filter(event))
  8488. ret = perf_event_set_addr_filter(event, filter_str);
  8489. kfree(filter_str);
  8490. return ret;
  8491. }
  8492. /*
  8493. * hrtimer based swevent callback
  8494. */
  8495. static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
  8496. {
  8497. enum hrtimer_restart ret = HRTIMER_RESTART;
  8498. struct perf_sample_data data;
  8499. struct pt_regs *regs;
  8500. struct perf_event *event;
  8501. u64 period;
  8502. event = container_of(hrtimer, struct perf_event, hw.hrtimer);
  8503. if (event->state != PERF_EVENT_STATE_ACTIVE)
  8504. return HRTIMER_NORESTART;
  8505. event->pmu->read(event);
  8506. perf_sample_data_init(&data, 0, event->hw.last_period);
  8507. regs = get_irq_regs();
  8508. if (regs && !perf_exclude_event(event, regs)) {
  8509. if (!(event->attr.exclude_idle && is_idle_task(current)))
  8510. if (__perf_event_overflow(event, 1, &data, regs))
  8511. ret = HRTIMER_NORESTART;
  8512. }
  8513. period = max_t(u64, 10000, event->hw.sample_period);
  8514. hrtimer_forward_now(hrtimer, ns_to_ktime(period));
  8515. return ret;
  8516. }
  8517. static void perf_swevent_start_hrtimer(struct perf_event *event)
  8518. {
  8519. struct hw_perf_event *hwc = &event->hw;
  8520. s64 period;
  8521. if (!is_sampling_event(event))
  8522. return;
  8523. period = local64_read(&hwc->period_left);
  8524. if (period) {
  8525. if (period < 0)
  8526. period = 10000;
  8527. local64_set(&hwc->period_left, 0);
  8528. } else {
  8529. period = max_t(u64, 10000, hwc->sample_period);
  8530. }
  8531. hrtimer_start(&hwc->hrtimer, ns_to_ktime(period),
  8532. HRTIMER_MODE_REL_PINNED_HARD);
  8533. }
  8534. static void perf_swevent_cancel_hrtimer(struct perf_event *event)
  8535. {
  8536. struct hw_perf_event *hwc = &event->hw;
  8537. if (is_sampling_event(event)) {
  8538. ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
  8539. local64_set(&hwc->period_left, ktime_to_ns(remaining));
  8540. hrtimer_cancel(&hwc->hrtimer);
  8541. }
  8542. }
  8543. static void perf_swevent_init_hrtimer(struct perf_event *event)
  8544. {
  8545. struct hw_perf_event *hwc = &event->hw;
  8546. if (!is_sampling_event(event))
  8547. return;
  8548. hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
  8549. hwc->hrtimer.function = perf_swevent_hrtimer;
  8550. /*
  8551. * Since hrtimers have a fixed rate, we can do a static freq->period
  8552. * mapping and avoid the whole period adjust feedback stuff.
  8553. */
  8554. if (event->attr.freq) {
  8555. long freq = event->attr.sample_freq;
  8556. event->attr.sample_period = NSEC_PER_SEC / freq;
  8557. hwc->sample_period = event->attr.sample_period;
  8558. local64_set(&hwc->period_left, hwc->sample_period);
  8559. hwc->last_period = hwc->sample_period;
  8560. event->attr.freq = 0;
  8561. }
  8562. }
  8563. /*
  8564. * Software event: cpu wall time clock
  8565. */
  8566. static void cpu_clock_event_update(struct perf_event *event)
  8567. {
  8568. s64 prev;
  8569. u64 now;
  8570. now = local_clock();
  8571. prev = local64_xchg(&event->hw.prev_count, now);
  8572. local64_add(now - prev, &event->count);
  8573. }
  8574. static void cpu_clock_event_start(struct perf_event *event, int flags)
  8575. {
  8576. local64_set(&event->hw.prev_count, local_clock());
  8577. perf_swevent_start_hrtimer(event);
  8578. }
  8579. static void cpu_clock_event_stop(struct perf_event *event, int flags)
  8580. {
  8581. perf_swevent_cancel_hrtimer(event);
  8582. cpu_clock_event_update(event);
  8583. }
  8584. static int cpu_clock_event_add(struct perf_event *event, int flags)
  8585. {
  8586. if (flags & PERF_EF_START)
  8587. cpu_clock_event_start(event, flags);
  8588. perf_event_update_userpage(event);
  8589. return 0;
  8590. }
  8591. static void cpu_clock_event_del(struct perf_event *event, int flags)
  8592. {
  8593. cpu_clock_event_stop(event, flags);
  8594. }
  8595. static void cpu_clock_event_read(struct perf_event *event)
  8596. {
  8597. cpu_clock_event_update(event);
  8598. }
  8599. static int cpu_clock_event_init(struct perf_event *event)
  8600. {
  8601. if (event->attr.type != PERF_TYPE_SOFTWARE)
  8602. return -ENOENT;
  8603. if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
  8604. return -ENOENT;
  8605. /*
  8606. * no branch sampling for software events
  8607. */
  8608. if (has_branch_stack(event))
  8609. return -EOPNOTSUPP;
  8610. perf_swevent_init_hrtimer(event);
  8611. return 0;
  8612. }
  8613. static struct pmu perf_cpu_clock = {
  8614. .task_ctx_nr = perf_sw_context,
  8615. .capabilities = PERF_PMU_CAP_NO_NMI,
  8616. .event_init = cpu_clock_event_init,
  8617. .add = cpu_clock_event_add,
  8618. .del = cpu_clock_event_del,
  8619. .start = cpu_clock_event_start,
  8620. .stop = cpu_clock_event_stop,
  8621. .read = cpu_clock_event_read,
  8622. };
  8623. /*
  8624. * Software event: task time clock
  8625. */
  8626. static void task_clock_event_update(struct perf_event *event, u64 now)
  8627. {
  8628. u64 prev;
  8629. s64 delta;
  8630. prev = local64_xchg(&event->hw.prev_count, now);
  8631. delta = now - prev;
  8632. local64_add(delta, &event->count);
  8633. }
  8634. static void task_clock_event_start(struct perf_event *event, int flags)
  8635. {
  8636. local64_set(&event->hw.prev_count, event->ctx->time);
  8637. perf_swevent_start_hrtimer(event);
  8638. }
  8639. static void task_clock_event_stop(struct perf_event *event, int flags)
  8640. {
  8641. perf_swevent_cancel_hrtimer(event);
  8642. task_clock_event_update(event, event->ctx->time);
  8643. }
  8644. static int task_clock_event_add(struct perf_event *event, int flags)
  8645. {
  8646. if (flags & PERF_EF_START)
  8647. task_clock_event_start(event, flags);
  8648. perf_event_update_userpage(event);
  8649. return 0;
  8650. }
  8651. static void task_clock_event_del(struct perf_event *event, int flags)
  8652. {
  8653. task_clock_event_stop(event, PERF_EF_UPDATE);
  8654. }
  8655. static void task_clock_event_read(struct perf_event *event)
  8656. {
  8657. u64 now = perf_clock();
  8658. u64 delta = now - event->ctx->timestamp;
  8659. u64 time = event->ctx->time + delta;
  8660. task_clock_event_update(event, time);
  8661. }
  8662. static int task_clock_event_init(struct perf_event *event)
  8663. {
  8664. if (event->attr.type != PERF_TYPE_SOFTWARE)
  8665. return -ENOENT;
  8666. if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
  8667. return -ENOENT;
  8668. /*
  8669. * no branch sampling for software events
  8670. */
  8671. if (has_branch_stack(event))
  8672. return -EOPNOTSUPP;
  8673. perf_swevent_init_hrtimer(event);
  8674. return 0;
  8675. }
  8676. static struct pmu perf_task_clock = {
  8677. .task_ctx_nr = perf_sw_context,
  8678. .capabilities = PERF_PMU_CAP_NO_NMI,
  8679. .event_init = task_clock_event_init,
  8680. .add = task_clock_event_add,
  8681. .del = task_clock_event_del,
  8682. .start = task_clock_event_start,
  8683. .stop = task_clock_event_stop,
  8684. .read = task_clock_event_read,
  8685. };
  8686. static void perf_pmu_nop_void(struct pmu *pmu)
  8687. {
  8688. }
  8689. static void perf_pmu_nop_txn(struct pmu *pmu, unsigned int flags)
  8690. {
  8691. }
  8692. static int perf_pmu_nop_int(struct pmu *pmu)
  8693. {
  8694. return 0;
  8695. }
  8696. static int perf_event_nop_int(struct perf_event *event, u64 value)
  8697. {
  8698. return 0;
  8699. }
  8700. static DEFINE_PER_CPU(unsigned int, nop_txn_flags);
  8701. static void perf_pmu_start_txn(struct pmu *pmu, unsigned int flags)
  8702. {
  8703. __this_cpu_write(nop_txn_flags, flags);
  8704. if (flags & ~PERF_PMU_TXN_ADD)
  8705. return;
  8706. perf_pmu_disable(pmu);
  8707. }
  8708. static int perf_pmu_commit_txn(struct pmu *pmu)
  8709. {
  8710. unsigned int flags = __this_cpu_read(nop_txn_flags);
  8711. __this_cpu_write(nop_txn_flags, 0);
  8712. if (flags & ~PERF_PMU_TXN_ADD)
  8713. return 0;
  8714. perf_pmu_enable(pmu);
  8715. return 0;
  8716. }
  8717. static void perf_pmu_cancel_txn(struct pmu *pmu)
  8718. {
  8719. unsigned int flags = __this_cpu_read(nop_txn_flags);
  8720. __this_cpu_write(nop_txn_flags, 0);
  8721. if (flags & ~PERF_PMU_TXN_ADD)
  8722. return;
  8723. perf_pmu_enable(pmu);
  8724. }
  8725. static int perf_event_idx_default(struct perf_event *event)
  8726. {
  8727. return 0;
  8728. }
  8729. /*
  8730. * Ensures all contexts with the same task_ctx_nr have the same
  8731. * pmu_cpu_context too.
  8732. */
  8733. static struct perf_cpu_context __percpu *find_pmu_context(int ctxn)
  8734. {
  8735. struct pmu *pmu;
  8736. if (ctxn < 0)
  8737. return NULL;
  8738. list_for_each_entry(pmu, &pmus, entry) {
  8739. if (pmu->task_ctx_nr == ctxn)
  8740. return pmu->pmu_cpu_context;
  8741. }
  8742. return NULL;
  8743. }
  8744. static void free_pmu_context(struct pmu *pmu)
  8745. {
  8746. /*
  8747. * Static contexts such as perf_sw_context have a global lifetime
  8748. * and may be shared between different PMUs. Avoid freeing them
  8749. * when a single PMU is going away.
  8750. */
  8751. if (pmu->task_ctx_nr > perf_invalid_context)
  8752. return;
  8753. free_percpu(pmu->pmu_cpu_context);
  8754. }
  8755. /*
  8756. * Let userspace know that this PMU supports address range filtering:
  8757. */
  8758. static ssize_t nr_addr_filters_show(struct device *dev,
  8759. struct device_attribute *attr,
  8760. char *page)
  8761. {
  8762. struct pmu *pmu = dev_get_drvdata(dev);
  8763. return snprintf(page, PAGE_SIZE - 1, "%d\n", pmu->nr_addr_filters);
  8764. }
  8765. DEVICE_ATTR_RO(nr_addr_filters);
  8766. static struct idr pmu_idr;
  8767. static ssize_t
  8768. type_show(struct device *dev, struct device_attribute *attr, char *page)
  8769. {
  8770. struct pmu *pmu = dev_get_drvdata(dev);
  8771. return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->type);
  8772. }
  8773. static DEVICE_ATTR_RO(type);
  8774. static ssize_t
  8775. perf_event_mux_interval_ms_show(struct device *dev,
  8776. struct device_attribute *attr,
  8777. char *page)
  8778. {
  8779. struct pmu *pmu = dev_get_drvdata(dev);
  8780. return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->hrtimer_interval_ms);
  8781. }
  8782. static DEFINE_MUTEX(mux_interval_mutex);
  8783. static ssize_t
  8784. perf_event_mux_interval_ms_store(struct device *dev,
  8785. struct device_attribute *attr,
  8786. const char *buf, size_t count)
  8787. {
  8788. struct pmu *pmu = dev_get_drvdata(dev);
  8789. int timer, cpu, ret;
  8790. ret = kstrtoint(buf, 0, &timer);
  8791. if (ret)
  8792. return ret;
  8793. if (timer < 1)
  8794. return -EINVAL;
  8795. /* same value, noting to do */
  8796. if (timer == pmu->hrtimer_interval_ms)
  8797. return count;
  8798. mutex_lock(&mux_interval_mutex);
  8799. pmu->hrtimer_interval_ms = timer;
  8800. /* update all cpuctx for this PMU */
  8801. cpus_read_lock();
  8802. for_each_online_cpu(cpu) {
  8803. struct perf_cpu_context *cpuctx;
  8804. cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
  8805. cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer);
  8806. cpu_function_call(cpu,
  8807. (remote_function_f)perf_mux_hrtimer_restart, cpuctx);
  8808. }
  8809. cpus_read_unlock();
  8810. mutex_unlock(&mux_interval_mutex);
  8811. return count;
  8812. }
  8813. static DEVICE_ATTR_RW(perf_event_mux_interval_ms);
  8814. static struct attribute *pmu_dev_attrs[] = {
  8815. &dev_attr_type.attr,
  8816. &dev_attr_perf_event_mux_interval_ms.attr,
  8817. NULL,
  8818. };
  8819. ATTRIBUTE_GROUPS(pmu_dev);
  8820. static int pmu_bus_running;
  8821. static struct bus_type pmu_bus = {
  8822. .name = "event_source",
  8823. .dev_groups = pmu_dev_groups,
  8824. };
  8825. static void pmu_dev_release(struct device *dev)
  8826. {
  8827. kfree(dev);
  8828. }
  8829. static int pmu_dev_alloc(struct pmu *pmu)
  8830. {
  8831. int ret = -ENOMEM;
  8832. pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL);
  8833. if (!pmu->dev)
  8834. goto out;
  8835. pmu->dev->groups = pmu->attr_groups;
  8836. device_initialize(pmu->dev);
  8837. ret = dev_set_name(pmu->dev, "%s", pmu->name);
  8838. if (ret)
  8839. goto free_dev;
  8840. dev_set_drvdata(pmu->dev, pmu);
  8841. pmu->dev->bus = &pmu_bus;
  8842. pmu->dev->release = pmu_dev_release;
  8843. ret = device_add(pmu->dev);
  8844. if (ret)
  8845. goto free_dev;
  8846. /* For PMUs with address filters, throw in an extra attribute: */
  8847. if (pmu->nr_addr_filters)
  8848. ret = device_create_file(pmu->dev, &dev_attr_nr_addr_filters);
  8849. if (ret)
  8850. goto del_dev;
  8851. if (pmu->attr_update)
  8852. ret = sysfs_update_groups(&pmu->dev->kobj, pmu->attr_update);
  8853. if (ret)
  8854. goto del_dev;
  8855. out:
  8856. return ret;
  8857. del_dev:
  8858. device_del(pmu->dev);
  8859. free_dev:
  8860. put_device(pmu->dev);
  8861. goto out;
  8862. }
  8863. static struct lock_class_key cpuctx_mutex;
  8864. static struct lock_class_key cpuctx_lock;
  8865. int perf_pmu_register(struct pmu *pmu, const char *name, int type)
  8866. {
  8867. int cpu, ret, max = PERF_TYPE_MAX;
  8868. mutex_lock(&pmus_lock);
  8869. ret = -ENOMEM;
  8870. pmu->pmu_disable_count = alloc_percpu(int);
  8871. if (!pmu->pmu_disable_count)
  8872. goto unlock;
  8873. pmu->type = -1;
  8874. if (!name)
  8875. goto skip_type;
  8876. pmu->name = name;
  8877. if (type != PERF_TYPE_SOFTWARE) {
  8878. if (type >= 0)
  8879. max = type;
  8880. ret = idr_alloc(&pmu_idr, pmu, max, 0, GFP_KERNEL);
  8881. if (ret < 0)
  8882. goto free_pdc;
  8883. WARN_ON(type >= 0 && ret != type);
  8884. type = ret;
  8885. }
  8886. pmu->type = type;
  8887. if (pmu_bus_running) {
  8888. ret = pmu_dev_alloc(pmu);
  8889. if (ret)
  8890. goto free_idr;
  8891. }
  8892. skip_type:
  8893. if (pmu->task_ctx_nr == perf_hw_context) {
  8894. static int hw_context_taken = 0;
  8895. /*
  8896. * Other than systems with heterogeneous CPUs, it never makes
  8897. * sense for two PMUs to share perf_hw_context. PMUs which are
  8898. * uncore must use perf_invalid_context.
  8899. */
  8900. if (WARN_ON_ONCE(hw_context_taken &&
  8901. !(pmu->capabilities & PERF_PMU_CAP_HETEROGENEOUS_CPUS)))
  8902. pmu->task_ctx_nr = perf_invalid_context;
  8903. hw_context_taken = 1;
  8904. }
  8905. pmu->pmu_cpu_context = find_pmu_context(pmu->task_ctx_nr);
  8906. if (pmu->pmu_cpu_context)
  8907. goto got_cpu_context;
  8908. ret = -ENOMEM;
  8909. pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context);
  8910. if (!pmu->pmu_cpu_context)
  8911. goto free_dev;
  8912. for_each_possible_cpu(cpu) {
  8913. struct perf_cpu_context *cpuctx;
  8914. cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
  8915. __perf_event_init_context(&cpuctx->ctx);
  8916. lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);
  8917. lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock);
  8918. cpuctx->ctx.pmu = pmu;
  8919. cpuctx->online = cpumask_test_cpu(cpu, perf_online_mask);
  8920. __perf_mux_hrtimer_init(cpuctx, cpu);
  8921. cpuctx->heap_size = ARRAY_SIZE(cpuctx->heap_default);
  8922. cpuctx->heap = cpuctx->heap_default;
  8923. }
  8924. got_cpu_context:
  8925. if (!pmu->start_txn) {
  8926. if (pmu->pmu_enable) {
  8927. /*
  8928. * If we have pmu_enable/pmu_disable calls, install
  8929. * transaction stubs that use that to try and batch
  8930. * hardware accesses.
  8931. */
  8932. pmu->start_txn = perf_pmu_start_txn;
  8933. pmu->commit_txn = perf_pmu_commit_txn;
  8934. pmu->cancel_txn = perf_pmu_cancel_txn;
  8935. } else {
  8936. pmu->start_txn = perf_pmu_nop_txn;
  8937. pmu->commit_txn = perf_pmu_nop_int;
  8938. pmu->cancel_txn = perf_pmu_nop_void;
  8939. }
  8940. }
  8941. if (!pmu->pmu_enable) {
  8942. pmu->pmu_enable = perf_pmu_nop_void;
  8943. pmu->pmu_disable = perf_pmu_nop_void;
  8944. }
  8945. if (!pmu->check_period)
  8946. pmu->check_period = perf_event_nop_int;
  8947. if (!pmu->event_idx)
  8948. pmu->event_idx = perf_event_idx_default;
  8949. /*
  8950. * Ensure the TYPE_SOFTWARE PMUs are at the head of the list,
  8951. * since these cannot be in the IDR. This way the linear search
  8952. * is fast, provided a valid software event is provided.
  8953. */
  8954. if (type == PERF_TYPE_SOFTWARE || !name)
  8955. list_add_rcu(&pmu->entry, &pmus);
  8956. else
  8957. list_add_tail_rcu(&pmu->entry, &pmus);
  8958. atomic_set(&pmu->exclusive_cnt, 0);
  8959. ret = 0;
  8960. unlock:
  8961. mutex_unlock(&pmus_lock);
  8962. return ret;
  8963. free_dev:
  8964. device_del(pmu->dev);
  8965. put_device(pmu->dev);
  8966. free_idr:
  8967. if (pmu->type != PERF_TYPE_SOFTWARE)
  8968. idr_remove(&pmu_idr, pmu->type);
  8969. free_pdc:
  8970. free_percpu(pmu->pmu_disable_count);
  8971. goto unlock;
  8972. }
  8973. EXPORT_SYMBOL_GPL(perf_pmu_register);
  8974. void perf_pmu_unregister(struct pmu *pmu)
  8975. {
  8976. mutex_lock(&pmus_lock);
  8977. list_del_rcu(&pmu->entry);
  8978. /*
  8979. * We dereference the pmu list under both SRCU and regular RCU, so
  8980. * synchronize against both of those.
  8981. */
  8982. synchronize_srcu(&pmus_srcu);
  8983. synchronize_rcu();
  8984. free_percpu(pmu->pmu_disable_count);
  8985. if (pmu->type != PERF_TYPE_SOFTWARE)
  8986. idr_remove(&pmu_idr, pmu->type);
  8987. if (pmu_bus_running) {
  8988. if (pmu->nr_addr_filters)
  8989. device_remove_file(pmu->dev, &dev_attr_nr_addr_filters);
  8990. device_del(pmu->dev);
  8991. put_device(pmu->dev);
  8992. }
  8993. free_pmu_context(pmu);
  8994. mutex_unlock(&pmus_lock);
  8995. }
  8996. EXPORT_SYMBOL_GPL(perf_pmu_unregister);
  8997. static inline bool has_extended_regs(struct perf_event *event)
  8998. {
  8999. return (event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK) ||
  9000. (event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK);
  9001. }
  9002. static int perf_try_init_event(struct pmu *pmu, struct perf_event *event)
  9003. {
  9004. struct perf_event_context *ctx = NULL;
  9005. int ret;
  9006. if (!try_module_get(pmu->module))
  9007. return -ENODEV;
  9008. /*
  9009. * A number of pmu->event_init() methods iterate the sibling_list to,
  9010. * for example, validate if the group fits on the PMU. Therefore,
  9011. * if this is a sibling event, acquire the ctx->mutex to protect
  9012. * the sibling_list.
  9013. */
  9014. if (event->group_leader != event && pmu->task_ctx_nr != perf_sw_context) {
  9015. /*
  9016. * This ctx->mutex can nest when we're called through
  9017. * inheritance. See the perf_event_ctx_lock_nested() comment.
  9018. */
  9019. ctx = perf_event_ctx_lock_nested(event->group_leader,
  9020. SINGLE_DEPTH_NESTING);
  9021. BUG_ON(!ctx);
  9022. }
  9023. event->pmu = pmu;
  9024. ret = pmu->event_init(event);
  9025. if (ctx)
  9026. perf_event_ctx_unlock(event->group_leader, ctx);
  9027. if (!ret) {
  9028. if (!(pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS) &&
  9029. has_extended_regs(event))
  9030. ret = -EOPNOTSUPP;
  9031. if (pmu->capabilities & PERF_PMU_CAP_NO_EXCLUDE &&
  9032. event_has_any_exclude_flag(event))
  9033. ret = -EINVAL;
  9034. if (ret && event->destroy)
  9035. event->destroy(event);
  9036. }
  9037. if (ret)
  9038. module_put(pmu->module);
  9039. return ret;
  9040. }
  9041. static struct pmu *perf_init_event(struct perf_event *event)
  9042. {
  9043. int idx, type, ret;
  9044. struct pmu *pmu;
  9045. idx = srcu_read_lock(&pmus_srcu);
  9046. /* Try parent's PMU first: */
  9047. if (event->parent && event->parent->pmu) {
  9048. pmu = event->parent->pmu;
  9049. ret = perf_try_init_event(pmu, event);
  9050. if (!ret)
  9051. goto unlock;
  9052. }
  9053. /*
  9054. * PERF_TYPE_HARDWARE and PERF_TYPE_HW_CACHE
  9055. * are often aliases for PERF_TYPE_RAW.
  9056. */
  9057. type = event->attr.type;
  9058. if (type == PERF_TYPE_HARDWARE || type == PERF_TYPE_HW_CACHE)
  9059. type = PERF_TYPE_RAW;
  9060. again:
  9061. rcu_read_lock();
  9062. pmu = idr_find(&pmu_idr, type);
  9063. rcu_read_unlock();
  9064. if (pmu) {
  9065. ret = perf_try_init_event(pmu, event);
  9066. if (ret == -ENOENT && event->attr.type != type) {
  9067. type = event->attr.type;
  9068. goto again;
  9069. }
  9070. if (ret)
  9071. pmu = ERR_PTR(ret);
  9072. goto unlock;
  9073. }
  9074. list_for_each_entry_rcu(pmu, &pmus, entry, lockdep_is_held(&pmus_srcu)) {
  9075. ret = perf_try_init_event(pmu, event);
  9076. if (!ret)
  9077. goto unlock;
  9078. if (ret != -ENOENT) {
  9079. pmu = ERR_PTR(ret);
  9080. goto unlock;
  9081. }
  9082. }
  9083. pmu = ERR_PTR(-ENOENT);
  9084. unlock:
  9085. srcu_read_unlock(&pmus_srcu, idx);
  9086. return pmu;
  9087. }
  9088. static void attach_sb_event(struct perf_event *event)
  9089. {
  9090. struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu);
  9091. raw_spin_lock(&pel->lock);
  9092. list_add_rcu(&event->sb_list, &pel->list);
  9093. raw_spin_unlock(&pel->lock);
  9094. }
  9095. /*
  9096. * We keep a list of all !task (and therefore per-cpu) events
  9097. * that need to receive side-band records.
  9098. *
  9099. * This avoids having to scan all the various PMU per-cpu contexts
  9100. * looking for them.
  9101. */
  9102. static void account_pmu_sb_event(struct perf_event *event)
  9103. {
  9104. if (is_sb_event(event))
  9105. attach_sb_event(event);
  9106. }
  9107. static void account_event_cpu(struct perf_event *event, int cpu)
  9108. {
  9109. if (event->parent)
  9110. return;
  9111. if (is_cgroup_event(event))
  9112. atomic_inc(&per_cpu(perf_cgroup_events, cpu));
  9113. }
  9114. /* Freq events need the tick to stay alive (see perf_event_task_tick). */
  9115. static void account_freq_event_nohz(void)
  9116. {
  9117. #ifdef CONFIG_NO_HZ_FULL
  9118. /* Lock so we don't race with concurrent unaccount */
  9119. spin_lock(&nr_freq_lock);
  9120. if (atomic_inc_return(&nr_freq_events) == 1)
  9121. tick_nohz_dep_set(TICK_DEP_BIT_PERF_EVENTS);
  9122. spin_unlock(&nr_freq_lock);
  9123. #endif
  9124. }
  9125. static void account_freq_event(void)
  9126. {
  9127. if (tick_nohz_full_enabled())
  9128. account_freq_event_nohz();
  9129. else
  9130. atomic_inc(&nr_freq_events);
  9131. }
  9132. static void account_event(struct perf_event *event)
  9133. {
  9134. bool inc = false;
  9135. if (event->parent)
  9136. return;
  9137. if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB))
  9138. inc = true;
  9139. if (event->attr.mmap || event->attr.mmap_data)
  9140. atomic_inc(&nr_mmap_events);
  9141. if (event->attr.comm)
  9142. atomic_inc(&nr_comm_events);
  9143. if (event->attr.namespaces)
  9144. atomic_inc(&nr_namespaces_events);
  9145. if (event->attr.cgroup)
  9146. atomic_inc(&nr_cgroup_events);
  9147. if (event->attr.task)
  9148. atomic_inc(&nr_task_events);
  9149. if (event->attr.freq)
  9150. account_freq_event();
  9151. if (event->attr.context_switch) {
  9152. atomic_inc(&nr_switch_events);
  9153. inc = true;
  9154. }
  9155. if (has_branch_stack(event))
  9156. inc = true;
  9157. if (is_cgroup_event(event))
  9158. inc = true;
  9159. if (event->attr.ksymbol)
  9160. atomic_inc(&nr_ksymbol_events);
  9161. if (event->attr.bpf_event)
  9162. atomic_inc(&nr_bpf_events);
  9163. if (event->attr.text_poke)
  9164. atomic_inc(&nr_text_poke_events);
  9165. if (inc) {
  9166. /*
  9167. * We need the mutex here because static_branch_enable()
  9168. * must complete *before* the perf_sched_count increment
  9169. * becomes visible.
  9170. */
  9171. if (atomic_inc_not_zero(&perf_sched_count))
  9172. goto enabled;
  9173. mutex_lock(&perf_sched_mutex);
  9174. if (!atomic_read(&perf_sched_count)) {
  9175. static_branch_enable(&perf_sched_events);
  9176. /*
  9177. * Guarantee that all CPUs observe they key change and
  9178. * call the perf scheduling hooks before proceeding to
  9179. * install events that need them.
  9180. */
  9181. synchronize_rcu();
  9182. }
  9183. /*
  9184. * Now that we have waited for the sync_sched(), allow further
  9185. * increments to by-pass the mutex.
  9186. */
  9187. atomic_inc(&perf_sched_count);
  9188. mutex_unlock(&perf_sched_mutex);
  9189. }
  9190. enabled:
  9191. account_event_cpu(event, event->cpu);
  9192. account_pmu_sb_event(event);
  9193. }
  9194. /*
  9195. * Allocate and initialize an event structure
  9196. */
  9197. static struct perf_event *
  9198. perf_event_alloc(struct perf_event_attr *attr, int cpu,
  9199. struct task_struct *task,
  9200. struct perf_event *group_leader,
  9201. struct perf_event *parent_event,
  9202. perf_overflow_handler_t overflow_handler,
  9203. void *context, int cgroup_fd)
  9204. {
  9205. struct pmu *pmu;
  9206. struct perf_event *event;
  9207. struct hw_perf_event *hwc;
  9208. long err = -EINVAL;
  9209. if ((unsigned)cpu >= nr_cpu_ids) {
  9210. if (!task || cpu != -1)
  9211. return ERR_PTR(-EINVAL);
  9212. }
  9213. event = kzalloc(sizeof(*event), GFP_KERNEL);
  9214. if (!event)
  9215. return ERR_PTR(-ENOMEM);
  9216. /*
  9217. * Single events are their own group leaders, with an
  9218. * empty sibling list:
  9219. */
  9220. if (!group_leader)
  9221. group_leader = event;
  9222. mutex_init(&event->child_mutex);
  9223. INIT_LIST_HEAD(&event->child_list);
  9224. INIT_LIST_HEAD(&event->event_entry);
  9225. INIT_LIST_HEAD(&event->sibling_list);
  9226. INIT_LIST_HEAD(&event->active_list);
  9227. init_event_group(event);
  9228. INIT_LIST_HEAD(&event->rb_entry);
  9229. INIT_LIST_HEAD(&event->active_entry);
  9230. INIT_LIST_HEAD(&event->addr_filters.list);
  9231. INIT_HLIST_NODE(&event->hlist_entry);
  9232. init_waitqueue_head(&event->waitq);
  9233. event->pending_disable = -1;
  9234. init_irq_work(&event->pending, perf_pending_event);
  9235. mutex_init(&event->mmap_mutex);
  9236. raw_spin_lock_init(&event->addr_filters.lock);
  9237. atomic_long_set(&event->refcount, 1);
  9238. event->cpu = cpu;
  9239. event->attr = *attr;
  9240. event->group_leader = group_leader;
  9241. event->pmu = NULL;
  9242. event->oncpu = -1;
  9243. event->parent = parent_event;
  9244. event->ns = get_pid_ns(task_active_pid_ns(current));
  9245. event->id = atomic64_inc_return(&perf_event_id);
  9246. event->state = PERF_EVENT_STATE_INACTIVE;
  9247. if (task) {
  9248. event->attach_state = PERF_ATTACH_TASK;
  9249. /*
  9250. * XXX pmu::event_init needs to know what task to account to
  9251. * and we cannot use the ctx information because we need the
  9252. * pmu before we get a ctx.
  9253. */
  9254. event->hw.target = get_task_struct(task);
  9255. }
  9256. event->clock = &local_clock;
  9257. if (parent_event)
  9258. event->clock = parent_event->clock;
  9259. if (!overflow_handler && parent_event) {
  9260. overflow_handler = parent_event->overflow_handler;
  9261. context = parent_event->overflow_handler_context;
  9262. #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_EVENT_TRACING)
  9263. if (overflow_handler == bpf_overflow_handler) {
  9264. struct bpf_prog *prog = parent_event->prog;
  9265. bpf_prog_inc(prog);
  9266. event->prog = prog;
  9267. event->orig_overflow_handler =
  9268. parent_event->orig_overflow_handler;
  9269. }
  9270. #endif
  9271. }
  9272. if (overflow_handler) {
  9273. event->overflow_handler = overflow_handler;
  9274. event->overflow_handler_context = context;
  9275. } else if (is_write_backward(event)){
  9276. event->overflow_handler = perf_event_output_backward;
  9277. event->overflow_handler_context = NULL;
  9278. } else {
  9279. event->overflow_handler = perf_event_output_forward;
  9280. event->overflow_handler_context = NULL;
  9281. }
  9282. perf_event__state_init(event);
  9283. pmu = NULL;
  9284. hwc = &event->hw;
  9285. hwc->sample_period = attr->sample_period;
  9286. if (attr->freq && attr->sample_freq)
  9287. hwc->sample_period = 1;
  9288. hwc->last_period = hwc->sample_period;
  9289. local64_set(&hwc->period_left, hwc->sample_period);
  9290. /*
  9291. * We currently do not support PERF_SAMPLE_READ on inherited events.
  9292. * See perf_output_read().
  9293. */
  9294. if (attr->inherit && (attr->sample_type & PERF_SAMPLE_READ))
  9295. goto err_ns;
  9296. if (!has_branch_stack(event))
  9297. event->attr.branch_sample_type = 0;
  9298. pmu = perf_init_event(event);
  9299. if (IS_ERR(pmu)) {
  9300. err = PTR_ERR(pmu);
  9301. goto err_ns;
  9302. }
  9303. /*
  9304. * Disallow uncore-cgroup events, they don't make sense as the cgroup will
  9305. * be different on other CPUs in the uncore mask.
  9306. */
  9307. if (pmu->task_ctx_nr == perf_invalid_context && cgroup_fd != -1) {
  9308. err = -EINVAL;
  9309. goto err_pmu;
  9310. }
  9311. if (event->attr.aux_output &&
  9312. !(pmu->capabilities & PERF_PMU_CAP_AUX_OUTPUT)) {
  9313. err = -EOPNOTSUPP;
  9314. goto err_pmu;
  9315. }
  9316. if (cgroup_fd != -1) {
  9317. err = perf_cgroup_connect(cgroup_fd, event, attr, group_leader);
  9318. if (err)
  9319. goto err_pmu;
  9320. }
  9321. err = exclusive_event_init(event);
  9322. if (err)
  9323. goto err_pmu;
  9324. if (has_addr_filter(event)) {
  9325. event->addr_filter_ranges = kcalloc(pmu->nr_addr_filters,
  9326. sizeof(struct perf_addr_filter_range),
  9327. GFP_KERNEL);
  9328. if (!event->addr_filter_ranges) {
  9329. err = -ENOMEM;
  9330. goto err_per_task;
  9331. }
  9332. /*
  9333. * Clone the parent's vma offsets: they are valid until exec()
  9334. * even if the mm is not shared with the parent.
  9335. */
  9336. if (event->parent) {
  9337. struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
  9338. raw_spin_lock_irq(&ifh->lock);
  9339. memcpy(event->addr_filter_ranges,
  9340. event->parent->addr_filter_ranges,
  9341. pmu->nr_addr_filters * sizeof(struct perf_addr_filter_range));
  9342. raw_spin_unlock_irq(&ifh->lock);
  9343. }
  9344. /* force hw sync on the address filters */
  9345. event->addr_filters_gen = 1;
  9346. }
  9347. if (!event->parent) {
  9348. if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {
  9349. err = get_callchain_buffers(attr->sample_max_stack);
  9350. if (err)
  9351. goto err_addr_filters;
  9352. }
  9353. }
  9354. err = security_perf_event_alloc(event);
  9355. if (err)
  9356. goto err_callchain_buffer;
  9357. /* symmetric to unaccount_event() in _free_event() */
  9358. account_event(event);
  9359. return event;
  9360. err_callchain_buffer:
  9361. if (!event->parent) {
  9362. if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
  9363. put_callchain_buffers();
  9364. }
  9365. err_addr_filters:
  9366. kfree(event->addr_filter_ranges);
  9367. err_per_task:
  9368. exclusive_event_destroy(event);
  9369. err_pmu:
  9370. if (is_cgroup_event(event))
  9371. perf_detach_cgroup(event);
  9372. if (event->destroy)
  9373. event->destroy(event);
  9374. module_put(pmu->module);
  9375. err_ns:
  9376. if (event->ns)
  9377. put_pid_ns(event->ns);
  9378. if (event->hw.target)
  9379. put_task_struct(event->hw.target);
  9380. kfree(event);
  9381. return ERR_PTR(err);
  9382. }
  9383. static int perf_copy_attr(struct perf_event_attr __user *uattr,
  9384. struct perf_event_attr *attr)
  9385. {
  9386. u32 size;
  9387. int ret;
  9388. /* Zero the full structure, so that a short copy will be nice. */
  9389. memset(attr, 0, sizeof(*attr));
  9390. ret = get_user(size, &uattr->size);
  9391. if (ret)
  9392. return ret;
  9393. /* ABI compatibility quirk: */
  9394. if (!size)
  9395. size = PERF_ATTR_SIZE_VER0;
  9396. if (size < PERF_ATTR_SIZE_VER0 || size > PAGE_SIZE)
  9397. goto err_size;
  9398. ret = copy_struct_from_user(attr, sizeof(*attr), uattr, size);
  9399. if (ret) {
  9400. if (ret == -E2BIG)
  9401. goto err_size;
  9402. return ret;
  9403. }
  9404. attr->size = size;
  9405. if (attr->__reserved_1 || attr->__reserved_2 || attr->__reserved_3)
  9406. return -EINVAL;
  9407. if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
  9408. return -EINVAL;
  9409. if (attr->read_format & ~(PERF_FORMAT_MAX-1))
  9410. return -EINVAL;
  9411. if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) {
  9412. u64 mask = attr->branch_sample_type;
  9413. /* only using defined bits */
  9414. if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1))
  9415. return -EINVAL;
  9416. /* at least one branch bit must be set */
  9417. if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL))
  9418. return -EINVAL;
  9419. /* propagate priv level, when not set for branch */
  9420. if (!(mask & PERF_SAMPLE_BRANCH_PLM_ALL)) {
  9421. /* exclude_kernel checked on syscall entry */
  9422. if (!attr->exclude_kernel)
  9423. mask |= PERF_SAMPLE_BRANCH_KERNEL;
  9424. if (!attr->exclude_user)
  9425. mask |= PERF_SAMPLE_BRANCH_USER;
  9426. if (!attr->exclude_hv)
  9427. mask |= PERF_SAMPLE_BRANCH_HV;
  9428. /*
  9429. * adjust user setting (for HW filter setup)
  9430. */
  9431. attr->branch_sample_type = mask;
  9432. }
  9433. /* privileged levels capture (kernel, hv): check permissions */
  9434. if (mask & PERF_SAMPLE_BRANCH_PERM_PLM) {
  9435. ret = perf_allow_kernel(attr);
  9436. if (ret)
  9437. return ret;
  9438. }
  9439. }
  9440. if (attr->sample_type & PERF_SAMPLE_REGS_USER) {
  9441. ret = perf_reg_validate(attr->sample_regs_user);
  9442. if (ret)
  9443. return ret;
  9444. }
  9445. if (attr->sample_type & PERF_SAMPLE_STACK_USER) {
  9446. if (!arch_perf_have_user_stack_dump())
  9447. return -ENOSYS;
  9448. /*
  9449. * We have __u32 type for the size, but so far
  9450. * we can only use __u16 as maximum due to the
  9451. * __u16 sample size limit.
  9452. */
  9453. if (attr->sample_stack_user >= USHRT_MAX)
  9454. return -EINVAL;
  9455. else if (!IS_ALIGNED(attr->sample_stack_user, sizeof(u64)))
  9456. return -EINVAL;
  9457. }
  9458. if (!attr->sample_max_stack)
  9459. attr->sample_max_stack = sysctl_perf_event_max_stack;
  9460. if (attr->sample_type & PERF_SAMPLE_REGS_INTR)
  9461. ret = perf_reg_validate(attr->sample_regs_intr);
  9462. #ifndef CONFIG_CGROUP_PERF
  9463. if (attr->sample_type & PERF_SAMPLE_CGROUP)
  9464. return -EINVAL;
  9465. #endif
  9466. out:
  9467. return ret;
  9468. err_size:
  9469. put_user(sizeof(*attr), &uattr->size);
  9470. ret = -E2BIG;
  9471. goto out;
  9472. }
  9473. static int
  9474. perf_event_set_output(struct perf_event *event, struct perf_event *output_event)
  9475. {
  9476. struct perf_buffer *rb = NULL;
  9477. int ret = -EINVAL;
  9478. if (!output_event)
  9479. goto set;
  9480. /* don't allow circular references */
  9481. if (event == output_event)
  9482. goto out;
  9483. /*
  9484. * Don't allow cross-cpu buffers
  9485. */
  9486. if (output_event->cpu != event->cpu)
  9487. goto out;
  9488. /*
  9489. * If its not a per-cpu rb, it must be the same task.
  9490. */
  9491. if (output_event->cpu == -1 && output_event->ctx != event->ctx)
  9492. goto out;
  9493. /*
  9494. * Mixing clocks in the same buffer is trouble you don't need.
  9495. */
  9496. if (output_event->clock != event->clock)
  9497. goto out;
  9498. /*
  9499. * Either writing ring buffer from beginning or from end.
  9500. * Mixing is not allowed.
  9501. */
  9502. if (is_write_backward(output_event) != is_write_backward(event))
  9503. goto out;
  9504. /*
  9505. * If both events generate aux data, they must be on the same PMU
  9506. */
  9507. if (has_aux(event) && has_aux(output_event) &&
  9508. event->pmu != output_event->pmu)
  9509. goto out;
  9510. set:
  9511. mutex_lock(&event->mmap_mutex);
  9512. /* Can't redirect output if we've got an active mmap() */
  9513. if (atomic_read(&event->mmap_count))
  9514. goto unlock;
  9515. if (output_event) {
  9516. /* get the rb we want to redirect to */
  9517. rb = ring_buffer_get(output_event);
  9518. if (!rb)
  9519. goto unlock;
  9520. }
  9521. ring_buffer_attach(event, rb);
  9522. ret = 0;
  9523. unlock:
  9524. mutex_unlock(&event->mmap_mutex);
  9525. out:
  9526. return ret;
  9527. }
  9528. static void mutex_lock_double(struct mutex *a, struct mutex *b)
  9529. {
  9530. if (b < a)
  9531. swap(a, b);
  9532. mutex_lock(a);
  9533. mutex_lock_nested(b, SINGLE_DEPTH_NESTING);
  9534. }
  9535. static int perf_event_set_clock(struct perf_event *event, clockid_t clk_id)
  9536. {
  9537. bool nmi_safe = false;
  9538. switch (clk_id) {
  9539. case CLOCK_MONOTONIC:
  9540. event->clock = &ktime_get_mono_fast_ns;
  9541. nmi_safe = true;
  9542. break;
  9543. case CLOCK_MONOTONIC_RAW:
  9544. event->clock = &ktime_get_raw_fast_ns;
  9545. nmi_safe = true;
  9546. break;
  9547. case CLOCK_REALTIME:
  9548. event->clock = &ktime_get_real_ns;
  9549. break;
  9550. case CLOCK_BOOTTIME:
  9551. event->clock = &ktime_get_boottime_ns;
  9552. break;
  9553. case CLOCK_TAI:
  9554. event->clock = &ktime_get_clocktai_ns;
  9555. break;
  9556. default:
  9557. return -EINVAL;
  9558. }
  9559. if (!nmi_safe && !(event->pmu->capabilities & PERF_PMU_CAP_NO_NMI))
  9560. return -EINVAL;
  9561. return 0;
  9562. }
  9563. /*
  9564. * Variation on perf_event_ctx_lock_nested(), except we take two context
  9565. * mutexes.
  9566. */
  9567. static struct perf_event_context *
  9568. __perf_event_ctx_lock_double(struct perf_event *group_leader,
  9569. struct perf_event_context *ctx)
  9570. {
  9571. struct perf_event_context *gctx;
  9572. again:
  9573. rcu_read_lock();
  9574. gctx = READ_ONCE(group_leader->ctx);
  9575. if (!refcount_inc_not_zero(&gctx->refcount)) {
  9576. rcu_read_unlock();
  9577. goto again;
  9578. }
  9579. rcu_read_unlock();
  9580. mutex_lock_double(&gctx->mutex, &ctx->mutex);
  9581. if (group_leader->ctx != gctx) {
  9582. mutex_unlock(&ctx->mutex);
  9583. mutex_unlock(&gctx->mutex);
  9584. put_ctx(gctx);
  9585. goto again;
  9586. }
  9587. return gctx;
  9588. }
  9589. /**
  9590. * sys_perf_event_open - open a performance event, associate it to a task/cpu
  9591. *
  9592. * @attr_uptr: event_id type attributes for monitoring/sampling
  9593. * @pid: target pid
  9594. * @cpu: target cpu
  9595. * @group_fd: group leader event fd
  9596. */
  9597. SYSCALL_DEFINE5(perf_event_open,
  9598. struct perf_event_attr __user *, attr_uptr,
  9599. pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
  9600. {
  9601. struct perf_event *group_leader = NULL, *output_event = NULL;
  9602. struct perf_event *event, *sibling;
  9603. struct perf_event_attr attr;
  9604. struct perf_event_context *ctx, *gctx;
  9605. struct file *event_file = NULL;
  9606. struct fd group = {NULL, 0};
  9607. struct task_struct *task = NULL;
  9608. struct pmu *pmu;
  9609. int event_fd;
  9610. int move_group = 0;
  9611. int err;
  9612. int f_flags = O_RDWR;
  9613. int cgroup_fd = -1;
  9614. /* for future expandability... */
  9615. if (flags & ~PERF_FLAG_ALL)
  9616. return -EINVAL;
  9617. /* Do we allow access to perf_event_open(2) ? */
  9618. err = security_perf_event_open(&attr, PERF_SECURITY_OPEN);
  9619. if (err)
  9620. return err;
  9621. err = perf_copy_attr(attr_uptr, &attr);
  9622. if (err)
  9623. return err;
  9624. if (!attr.exclude_kernel) {
  9625. err = perf_allow_kernel(&attr);
  9626. if (err)
  9627. return err;
  9628. }
  9629. if (attr.namespaces) {
  9630. if (!perfmon_capable())
  9631. return -EACCES;
  9632. }
  9633. if (attr.freq) {
  9634. if (attr.sample_freq > sysctl_perf_event_sample_rate)
  9635. return -EINVAL;
  9636. } else {
  9637. if (attr.sample_period & (1ULL << 63))
  9638. return -EINVAL;
  9639. }
  9640. /* Only privileged users can get physical addresses */
  9641. if ((attr.sample_type & PERF_SAMPLE_PHYS_ADDR)) {
  9642. err = perf_allow_kernel(&attr);
  9643. if (err)
  9644. return err;
  9645. }
  9646. /* REGS_INTR can leak data, lockdown must prevent this */
  9647. if (attr.sample_type & PERF_SAMPLE_REGS_INTR) {
  9648. err = security_locked_down(LOCKDOWN_PERF);
  9649. if (err)
  9650. return err;
  9651. }
  9652. /*
  9653. * In cgroup mode, the pid argument is used to pass the fd
  9654. * opened to the cgroup directory in cgroupfs. The cpu argument
  9655. * designates the cpu on which to monitor threads from that
  9656. * cgroup.
  9657. */
  9658. if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1))
  9659. return -EINVAL;
  9660. if (flags & PERF_FLAG_FD_CLOEXEC)
  9661. f_flags |= O_CLOEXEC;
  9662. event_fd = get_unused_fd_flags(f_flags);
  9663. if (event_fd < 0)
  9664. return event_fd;
  9665. if (group_fd != -1) {
  9666. err = perf_fget_light(group_fd, &group);
  9667. if (err)
  9668. goto err_fd;
  9669. group_leader = group.file->private_data;
  9670. if (flags & PERF_FLAG_FD_OUTPUT)
  9671. output_event = group_leader;
  9672. if (flags & PERF_FLAG_FD_NO_GROUP)
  9673. group_leader = NULL;
  9674. }
  9675. if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) {
  9676. task = find_lively_task_by_vpid(pid);
  9677. if (IS_ERR(task)) {
  9678. err = PTR_ERR(task);
  9679. goto err_group_fd;
  9680. }
  9681. }
  9682. if (task && group_leader &&
  9683. group_leader->attr.inherit != attr.inherit) {
  9684. err = -EINVAL;
  9685. goto err_task;
  9686. }
  9687. if (flags & PERF_FLAG_PID_CGROUP)
  9688. cgroup_fd = pid;
  9689. event = perf_event_alloc(&attr, cpu, task, group_leader, NULL,
  9690. NULL, NULL, cgroup_fd);
  9691. if (IS_ERR(event)) {
  9692. err = PTR_ERR(event);
  9693. goto err_task;
  9694. }
  9695. if (is_sampling_event(event)) {
  9696. if (event->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT) {
  9697. err = -EOPNOTSUPP;
  9698. goto err_alloc;
  9699. }
  9700. }
  9701. /*
  9702. * Special case software events and allow them to be part of
  9703. * any hardware group.
  9704. */
  9705. pmu = event->pmu;
  9706. if (attr.use_clockid) {
  9707. err = perf_event_set_clock(event, attr.clockid);
  9708. if (err)
  9709. goto err_alloc;
  9710. }
  9711. if (pmu->task_ctx_nr == perf_sw_context)
  9712. event->event_caps |= PERF_EV_CAP_SOFTWARE;
  9713. if (group_leader) {
  9714. if (is_software_event(event) &&
  9715. !in_software_context(group_leader)) {
  9716. /*
  9717. * If the event is a sw event, but the group_leader
  9718. * is on hw context.
  9719. *
  9720. * Allow the addition of software events to hw
  9721. * groups, this is safe because software events
  9722. * never fail to schedule.
  9723. */
  9724. pmu = group_leader->ctx->pmu;
  9725. } else if (!is_software_event(event) &&
  9726. is_software_event(group_leader) &&
  9727. (group_leader->group_caps & PERF_EV_CAP_SOFTWARE)) {
  9728. /*
  9729. * In case the group is a pure software group, and we
  9730. * try to add a hardware event, move the whole group to
  9731. * the hardware context.
  9732. */
  9733. move_group = 1;
  9734. }
  9735. }
  9736. /*
  9737. * Get the target context (task or percpu):
  9738. */
  9739. ctx = find_get_context(pmu, task, event);
  9740. if (IS_ERR(ctx)) {
  9741. err = PTR_ERR(ctx);
  9742. goto err_alloc;
  9743. }
  9744. /*
  9745. * Look up the group leader (we will attach this event to it):
  9746. */
  9747. if (group_leader) {
  9748. err = -EINVAL;
  9749. /*
  9750. * Do not allow a recursive hierarchy (this new sibling
  9751. * becoming part of another group-sibling):
  9752. */
  9753. if (group_leader->group_leader != group_leader)
  9754. goto err_context;
  9755. /* All events in a group should have the same clock */
  9756. if (group_leader->clock != event->clock)
  9757. goto err_context;
  9758. /*
  9759. * Make sure we're both events for the same CPU;
  9760. * grouping events for different CPUs is broken; since
  9761. * you can never concurrently schedule them anyhow.
  9762. */
  9763. if (group_leader->cpu != event->cpu)
  9764. goto err_context;
  9765. /*
  9766. * Make sure we're both on the same task, or both
  9767. * per-CPU events.
  9768. */
  9769. if (group_leader->ctx->task != ctx->task)
  9770. goto err_context;
  9771. /*
  9772. * Do not allow to attach to a group in a different task
  9773. * or CPU context. If we're moving SW events, we'll fix
  9774. * this up later, so allow that.
  9775. */
  9776. if (!move_group && group_leader->ctx != ctx)
  9777. goto err_context;
  9778. /*
  9779. * Only a group leader can be exclusive or pinned
  9780. */
  9781. if (attr.exclusive || attr.pinned)
  9782. goto err_context;
  9783. }
  9784. if (output_event) {
  9785. err = perf_event_set_output(event, output_event);
  9786. if (err)
  9787. goto err_context;
  9788. }
  9789. event_file = anon_inode_getfile("[perf_event]", &perf_fops, event,
  9790. f_flags);
  9791. if (IS_ERR(event_file)) {
  9792. err = PTR_ERR(event_file);
  9793. event_file = NULL;
  9794. goto err_context;
  9795. }
  9796. if (task) {
  9797. err = down_read_interruptible(&task->signal->exec_update_lock);
  9798. if (err)
  9799. goto err_file;
  9800. /*
  9801. * Preserve ptrace permission check for backwards compatibility.
  9802. *
  9803. * We must hold exec_update_lock across this and any potential
  9804. * perf_install_in_context() call for this new event to
  9805. * serialize against exec() altering our credentials (and the
  9806. * perf_event_exit_task() that could imply).
  9807. */
  9808. err = -EACCES;
  9809. if (!perfmon_capable() && !ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS))
  9810. goto err_cred;
  9811. }
  9812. if (move_group) {
  9813. gctx = __perf_event_ctx_lock_double(group_leader, ctx);
  9814. if (gctx->task == TASK_TOMBSTONE) {
  9815. err = -ESRCH;
  9816. goto err_locked;
  9817. }
  9818. /*
  9819. * Check if we raced against another sys_perf_event_open() call
  9820. * moving the software group underneath us.
  9821. */
  9822. if (!(group_leader->group_caps & PERF_EV_CAP_SOFTWARE)) {
  9823. /*
  9824. * If someone moved the group out from under us, check
  9825. * if this new event wound up on the same ctx, if so
  9826. * its the regular !move_group case, otherwise fail.
  9827. */
  9828. if (gctx != ctx) {
  9829. err = -EINVAL;
  9830. goto err_locked;
  9831. } else {
  9832. perf_event_ctx_unlock(group_leader, gctx);
  9833. move_group = 0;
  9834. }
  9835. }
  9836. /*
  9837. * Failure to create exclusive events returns -EBUSY.
  9838. */
  9839. err = -EBUSY;
  9840. if (!exclusive_event_installable(group_leader, ctx))
  9841. goto err_locked;
  9842. for_each_sibling_event(sibling, group_leader) {
  9843. if (!exclusive_event_installable(sibling, ctx))
  9844. goto err_locked;
  9845. }
  9846. } else {
  9847. mutex_lock(&ctx->mutex);
  9848. }
  9849. if (ctx->task == TASK_TOMBSTONE) {
  9850. err = -ESRCH;
  9851. goto err_locked;
  9852. }
  9853. if (!perf_event_validate_size(event)) {
  9854. err = -E2BIG;
  9855. goto err_locked;
  9856. }
  9857. if (!task) {
  9858. /*
  9859. * Check if the @cpu we're creating an event for is online.
  9860. *
  9861. * We use the perf_cpu_context::ctx::mutex to serialize against
  9862. * the hotplug notifiers. See perf_event_{init,exit}_cpu().
  9863. */
  9864. struct perf_cpu_context *cpuctx =
  9865. container_of(ctx, struct perf_cpu_context, ctx);
  9866. if (!cpuctx->online) {
  9867. err = -ENODEV;
  9868. goto err_locked;
  9869. }
  9870. }
  9871. if (perf_need_aux_event(event) && !perf_get_aux_event(event, group_leader)) {
  9872. err = -EINVAL;
  9873. goto err_locked;
  9874. }
  9875. /*
  9876. * Must be under the same ctx::mutex as perf_install_in_context(),
  9877. * because we need to serialize with concurrent event creation.
  9878. */
  9879. if (!exclusive_event_installable(event, ctx)) {
  9880. err = -EBUSY;
  9881. goto err_locked;
  9882. }
  9883. WARN_ON_ONCE(ctx->parent_ctx);
  9884. /*
  9885. * This is the point on no return; we cannot fail hereafter. This is
  9886. * where we start modifying current state.
  9887. */
  9888. if (move_group) {
  9889. /*
  9890. * See perf_event_ctx_lock() for comments on the details
  9891. * of swizzling perf_event::ctx.
  9892. */
  9893. perf_remove_from_context(group_leader, 0);
  9894. put_ctx(gctx);
  9895. for_each_sibling_event(sibling, group_leader) {
  9896. perf_remove_from_context(sibling, 0);
  9897. put_ctx(gctx);
  9898. }
  9899. /*
  9900. * Wait for everybody to stop referencing the events through
  9901. * the old lists, before installing it on new lists.
  9902. */
  9903. synchronize_rcu();
  9904. /*
  9905. * Install the group siblings before the group leader.
  9906. *
  9907. * Because a group leader will try and install the entire group
  9908. * (through the sibling list, which is still in-tact), we can
  9909. * end up with siblings installed in the wrong context.
  9910. *
  9911. * By installing siblings first we NO-OP because they're not
  9912. * reachable through the group lists.
  9913. */
  9914. for_each_sibling_event(sibling, group_leader) {
  9915. perf_event__state_init(sibling);
  9916. perf_install_in_context(ctx, sibling, sibling->cpu);
  9917. get_ctx(ctx);
  9918. }
  9919. /*
  9920. * Removing from the context ends up with disabled
  9921. * event. What we want here is event in the initial
  9922. * startup state, ready to be add into new context.
  9923. */
  9924. perf_event__state_init(group_leader);
  9925. perf_install_in_context(ctx, group_leader, group_leader->cpu);
  9926. get_ctx(ctx);
  9927. }
  9928. /*
  9929. * Precalculate sample_data sizes; do while holding ctx::mutex such
  9930. * that we're serialized against further additions and before
  9931. * perf_install_in_context() which is the point the event is active and
  9932. * can use these values.
  9933. */
  9934. perf_event__header_size(event);
  9935. perf_event__id_header_size(event);
  9936. event->owner = current;
  9937. perf_install_in_context(ctx, event, event->cpu);
  9938. perf_unpin_context(ctx);
  9939. if (move_group)
  9940. perf_event_ctx_unlock(group_leader, gctx);
  9941. mutex_unlock(&ctx->mutex);
  9942. if (task) {
  9943. up_read(&task->signal->exec_update_lock);
  9944. put_task_struct(task);
  9945. }
  9946. mutex_lock(&current->perf_event_mutex);
  9947. list_add_tail(&event->owner_entry, &current->perf_event_list);
  9948. mutex_unlock(&current->perf_event_mutex);
  9949. /*
  9950. * Drop the reference on the group_event after placing the
  9951. * new event on the sibling_list. This ensures destruction
  9952. * of the group leader will find the pointer to itself in
  9953. * perf_group_detach().
  9954. */
  9955. fdput(group);
  9956. fd_install(event_fd, event_file);
  9957. return event_fd;
  9958. err_locked:
  9959. if (move_group)
  9960. perf_event_ctx_unlock(group_leader, gctx);
  9961. mutex_unlock(&ctx->mutex);
  9962. err_cred:
  9963. if (task)
  9964. up_read(&task->signal->exec_update_lock);
  9965. err_file:
  9966. fput(event_file);
  9967. err_context:
  9968. perf_unpin_context(ctx);
  9969. put_ctx(ctx);
  9970. err_alloc:
  9971. /*
  9972. * If event_file is set, the fput() above will have called ->release()
  9973. * and that will take care of freeing the event.
  9974. */
  9975. if (!event_file)
  9976. free_event(event);
  9977. err_task:
  9978. if (task)
  9979. put_task_struct(task);
  9980. err_group_fd:
  9981. fdput(group);
  9982. err_fd:
  9983. put_unused_fd(event_fd);
  9984. return err;
  9985. }
  9986. /**
  9987. * perf_event_create_kernel_counter
  9988. *
  9989. * @attr: attributes of the counter to create
  9990. * @cpu: cpu in which the counter is bound
  9991. * @task: task to profile (NULL for percpu)
  9992. */
  9993. struct perf_event *
  9994. perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
  9995. struct task_struct *task,
  9996. perf_overflow_handler_t overflow_handler,
  9997. void *context)
  9998. {
  9999. struct perf_event_context *ctx;
  10000. struct perf_event *event;
  10001. int err;
  10002. /*
  10003. * Grouping is not supported for kernel events, neither is 'AUX',
  10004. * make sure the caller's intentions are adjusted.
  10005. */
  10006. if (attr->aux_output)
  10007. return ERR_PTR(-EINVAL);
  10008. event = perf_event_alloc(attr, cpu, task, NULL, NULL,
  10009. overflow_handler, context, -1);
  10010. if (IS_ERR(event)) {
  10011. err = PTR_ERR(event);
  10012. goto err;
  10013. }
  10014. /* Mark owner so we could distinguish it from user events. */
  10015. event->owner = TASK_TOMBSTONE;
  10016. /*
  10017. * Get the target context (task or percpu):
  10018. */
  10019. ctx = find_get_context(event->pmu, task, event);
  10020. if (IS_ERR(ctx)) {
  10021. err = PTR_ERR(ctx);
  10022. goto err_free;
  10023. }
  10024. WARN_ON_ONCE(ctx->parent_ctx);
  10025. mutex_lock(&ctx->mutex);
  10026. if (ctx->task == TASK_TOMBSTONE) {
  10027. err = -ESRCH;
  10028. goto err_unlock;
  10029. }
  10030. if (!task) {
  10031. /*
  10032. * Check if the @cpu we're creating an event for is online.
  10033. *
  10034. * We use the perf_cpu_context::ctx::mutex to serialize against
  10035. * the hotplug notifiers. See perf_event_{init,exit}_cpu().
  10036. */
  10037. struct perf_cpu_context *cpuctx =
  10038. container_of(ctx, struct perf_cpu_context, ctx);
  10039. if (!cpuctx->online) {
  10040. err = -ENODEV;
  10041. goto err_unlock;
  10042. }
  10043. }
  10044. if (!exclusive_event_installable(event, ctx)) {
  10045. err = -EBUSY;
  10046. goto err_unlock;
  10047. }
  10048. perf_install_in_context(ctx, event, event->cpu);
  10049. perf_unpin_context(ctx);
  10050. mutex_unlock(&ctx->mutex);
  10051. return event;
  10052. err_unlock:
  10053. mutex_unlock(&ctx->mutex);
  10054. perf_unpin_context(ctx);
  10055. put_ctx(ctx);
  10056. err_free:
  10057. free_event(event);
  10058. err:
  10059. return ERR_PTR(err);
  10060. }
  10061. EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);
  10062. void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu)
  10063. {
  10064. struct perf_event_context *src_ctx;
  10065. struct perf_event_context *dst_ctx;
  10066. struct perf_event *event, *tmp;
  10067. LIST_HEAD(events);
  10068. src_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, src_cpu)->ctx;
  10069. dst_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, dst_cpu)->ctx;
  10070. /*
  10071. * See perf_event_ctx_lock() for comments on the details
  10072. * of swizzling perf_event::ctx.
  10073. */
  10074. mutex_lock_double(&src_ctx->mutex, &dst_ctx->mutex);
  10075. list_for_each_entry_safe(event, tmp, &src_ctx->event_list,
  10076. event_entry) {
  10077. perf_remove_from_context(event, 0);
  10078. unaccount_event_cpu(event, src_cpu);
  10079. put_ctx(src_ctx);
  10080. list_add(&event->migrate_entry, &events);
  10081. }
  10082. /*
  10083. * Wait for the events to quiesce before re-instating them.
  10084. */
  10085. synchronize_rcu();
  10086. /*
  10087. * Re-instate events in 2 passes.
  10088. *
  10089. * Skip over group leaders and only install siblings on this first
  10090. * pass, siblings will not get enabled without a leader, however a
  10091. * leader will enable its siblings, even if those are still on the old
  10092. * context.
  10093. */
  10094. list_for_each_entry_safe(event, tmp, &events, migrate_entry) {
  10095. if (event->group_leader == event)
  10096. continue;
  10097. list_del(&event->migrate_entry);
  10098. if (event->state >= PERF_EVENT_STATE_OFF)
  10099. event->state = PERF_EVENT_STATE_INACTIVE;
  10100. account_event_cpu(event, dst_cpu);
  10101. perf_install_in_context(dst_ctx, event, dst_cpu);
  10102. get_ctx(dst_ctx);
  10103. }
  10104. /*
  10105. * Once all the siblings are setup properly, install the group leaders
  10106. * to make it go.
  10107. */
  10108. list_for_each_entry_safe(event, tmp, &events, migrate_entry) {
  10109. list_del(&event->migrate_entry);
  10110. if (event->state >= PERF_EVENT_STATE_OFF)
  10111. event->state = PERF_EVENT_STATE_INACTIVE;
  10112. account_event_cpu(event, dst_cpu);
  10113. perf_install_in_context(dst_ctx, event, dst_cpu);
  10114. get_ctx(dst_ctx);
  10115. }
  10116. mutex_unlock(&dst_ctx->mutex);
  10117. mutex_unlock(&src_ctx->mutex);
  10118. }
  10119. EXPORT_SYMBOL_GPL(perf_pmu_migrate_context);
  10120. static void sync_child_event(struct perf_event *child_event,
  10121. struct task_struct *child)
  10122. {
  10123. struct perf_event *parent_event = child_event->parent;
  10124. u64 child_val;
  10125. if (child_event->attr.inherit_stat)
  10126. perf_event_read_event(child_event, child);
  10127. child_val = perf_event_count(child_event);
  10128. /*
  10129. * Add back the child's count to the parent's count:
  10130. */
  10131. atomic64_add(child_val, &parent_event->child_count);
  10132. atomic64_add(child_event->total_time_enabled,
  10133. &parent_event->child_total_time_enabled);
  10134. atomic64_add(child_event->total_time_running,
  10135. &parent_event->child_total_time_running);
  10136. }
  10137. static void
  10138. perf_event_exit_event(struct perf_event *child_event,
  10139. struct perf_event_context *child_ctx,
  10140. struct task_struct *child)
  10141. {
  10142. struct perf_event *parent_event = child_event->parent;
  10143. /*
  10144. * Do not destroy the 'original' grouping; because of the context
  10145. * switch optimization the original events could've ended up in a
  10146. * random child task.
  10147. *
  10148. * If we were to destroy the original group, all group related
  10149. * operations would cease to function properly after this random
  10150. * child dies.
  10151. *
  10152. * Do destroy all inherited groups, we don't care about those
  10153. * and being thorough is better.
  10154. */
  10155. raw_spin_lock_irq(&child_ctx->lock);
  10156. WARN_ON_ONCE(child_ctx->is_active);
  10157. if (parent_event)
  10158. perf_group_detach(child_event);
  10159. list_del_event(child_event, child_ctx);
  10160. perf_event_set_state(child_event, PERF_EVENT_STATE_EXIT); /* is_event_hup() */
  10161. raw_spin_unlock_irq(&child_ctx->lock);
  10162. /*
  10163. * Parent events are governed by their filedesc, retain them.
  10164. */
  10165. if (!parent_event) {
  10166. perf_event_wakeup(child_event);
  10167. return;
  10168. }
  10169. /*
  10170. * Child events can be cleaned up.
  10171. */
  10172. sync_child_event(child_event, child);
  10173. /*
  10174. * Remove this event from the parent's list
  10175. */
  10176. WARN_ON_ONCE(parent_event->ctx->parent_ctx);
  10177. mutex_lock(&parent_event->child_mutex);
  10178. list_del_init(&child_event->child_list);
  10179. mutex_unlock(&parent_event->child_mutex);
  10180. /*
  10181. * Kick perf_poll() for is_event_hup().
  10182. */
  10183. perf_event_wakeup(parent_event);
  10184. free_event(child_event);
  10185. put_event(parent_event);
  10186. }
  10187. static void perf_event_exit_task_context(struct task_struct *child, int ctxn)
  10188. {
  10189. struct perf_event_context *child_ctx, *clone_ctx = NULL;
  10190. struct perf_event *child_event, *next;
  10191. WARN_ON_ONCE(child != current);
  10192. child_ctx = perf_pin_task_context(child, ctxn);
  10193. if (!child_ctx)
  10194. return;
  10195. /*
  10196. * In order to reduce the amount of tricky in ctx tear-down, we hold
  10197. * ctx::mutex over the entire thing. This serializes against almost
  10198. * everything that wants to access the ctx.
  10199. *
  10200. * The exception is sys_perf_event_open() /
  10201. * perf_event_create_kernel_count() which does find_get_context()
  10202. * without ctx::mutex (it cannot because of the move_group double mutex
  10203. * lock thing). See the comments in perf_install_in_context().
  10204. */
  10205. mutex_lock(&child_ctx->mutex);
  10206. /*
  10207. * In a single ctx::lock section, de-schedule the events and detach the
  10208. * context from the task such that we cannot ever get it scheduled back
  10209. * in.
  10210. */
  10211. raw_spin_lock_irq(&child_ctx->lock);
  10212. task_ctx_sched_out(__get_cpu_context(child_ctx), child_ctx, EVENT_ALL);
  10213. /*
  10214. * Now that the context is inactive, destroy the task <-> ctx relation
  10215. * and mark the context dead.
  10216. */
  10217. RCU_INIT_POINTER(child->perf_event_ctxp[ctxn], NULL);
  10218. put_ctx(child_ctx); /* cannot be last */
  10219. WRITE_ONCE(child_ctx->task, TASK_TOMBSTONE);
  10220. put_task_struct(current); /* cannot be last */
  10221. clone_ctx = unclone_ctx(child_ctx);
  10222. raw_spin_unlock_irq(&child_ctx->lock);
  10223. if (clone_ctx)
  10224. put_ctx(clone_ctx);
  10225. /*
  10226. * Report the task dead after unscheduling the events so that we
  10227. * won't get any samples after PERF_RECORD_EXIT. We can however still
  10228. * get a few PERF_RECORD_READ events.
  10229. */
  10230. perf_event_task(child, child_ctx, 0);
  10231. list_for_each_entry_safe(child_event, next, &child_ctx->event_list, event_entry)
  10232. perf_event_exit_event(child_event, child_ctx, child);
  10233. mutex_unlock(&child_ctx->mutex);
  10234. put_ctx(child_ctx);
  10235. }
  10236. /*
  10237. * When a child task exits, feed back event values to parent events.
  10238. *
  10239. * Can be called with exec_update_lock held when called from
  10240. * setup_new_exec().
  10241. */
  10242. void perf_event_exit_task(struct task_struct *child)
  10243. {
  10244. struct perf_event *event, *tmp;
  10245. int ctxn;
  10246. mutex_lock(&child->perf_event_mutex);
  10247. list_for_each_entry_safe(event, tmp, &child->perf_event_list,
  10248. owner_entry) {
  10249. list_del_init(&event->owner_entry);
  10250. /*
  10251. * Ensure the list deletion is visible before we clear
  10252. * the owner, closes a race against perf_release() where
  10253. * we need to serialize on the owner->perf_event_mutex.
  10254. */
  10255. smp_store_release(&event->owner, NULL);
  10256. }
  10257. mutex_unlock(&child->perf_event_mutex);
  10258. for_each_task_context_nr(ctxn)
  10259. perf_event_exit_task_context(child, ctxn);
  10260. /*
  10261. * The perf_event_exit_task_context calls perf_event_task
  10262. * with child's task_ctx, which generates EXIT events for
  10263. * child contexts and sets child->perf_event_ctxp[] to NULL.
  10264. * At this point we need to send EXIT events to cpu contexts.
  10265. */
  10266. perf_event_task(child, NULL, 0);
  10267. }
  10268. static void perf_free_event(struct perf_event *event,
  10269. struct perf_event_context *ctx)
  10270. {
  10271. struct perf_event *parent = event->parent;
  10272. if (WARN_ON_ONCE(!parent))
  10273. return;
  10274. mutex_lock(&parent->child_mutex);
  10275. list_del_init(&event->child_list);
  10276. mutex_unlock(&parent->child_mutex);
  10277. put_event(parent);
  10278. raw_spin_lock_irq(&ctx->lock);
  10279. perf_group_detach(event);
  10280. list_del_event(event, ctx);
  10281. raw_spin_unlock_irq(&ctx->lock);
  10282. free_event(event);
  10283. }
  10284. /*
  10285. * Free a context as created by inheritance by perf_event_init_task() below,
  10286. * used by fork() in case of fail.
  10287. *
  10288. * Even though the task has never lived, the context and events have been
  10289. * exposed through the child_list, so we must take care tearing it all down.
  10290. */
  10291. void perf_event_free_task(struct task_struct *task)
  10292. {
  10293. struct perf_event_context *ctx;
  10294. struct perf_event *event, *tmp;
  10295. int ctxn;
  10296. for_each_task_context_nr(ctxn) {
  10297. ctx = task->perf_event_ctxp[ctxn];
  10298. if (!ctx)
  10299. continue;
  10300. mutex_lock(&ctx->mutex);
  10301. raw_spin_lock_irq(&ctx->lock);
  10302. /*
  10303. * Destroy the task <-> ctx relation and mark the context dead.
  10304. *
  10305. * This is important because even though the task hasn't been
  10306. * exposed yet the context has been (through child_list).
  10307. */
  10308. RCU_INIT_POINTER(task->perf_event_ctxp[ctxn], NULL);
  10309. WRITE_ONCE(ctx->task, TASK_TOMBSTONE);
  10310. put_task_struct(task); /* cannot be last */
  10311. raw_spin_unlock_irq(&ctx->lock);
  10312. list_for_each_entry_safe(event, tmp, &ctx->event_list, event_entry)
  10313. perf_free_event(event, ctx);
  10314. mutex_unlock(&ctx->mutex);
  10315. /*
  10316. * perf_event_release_kernel() could've stolen some of our
  10317. * child events and still have them on its free_list. In that
  10318. * case we must wait for these events to have been freed (in
  10319. * particular all their references to this task must've been
  10320. * dropped).
  10321. *
  10322. * Without this copy_process() will unconditionally free this
  10323. * task (irrespective of its reference count) and
  10324. * _free_event()'s put_task_struct(event->hw.target) will be a
  10325. * use-after-free.
  10326. *
  10327. * Wait for all events to drop their context reference.
  10328. */
  10329. wait_var_event(&ctx->refcount, refcount_read(&ctx->refcount) == 1);
  10330. put_ctx(ctx); /* must be last */
  10331. }
  10332. }
  10333. void perf_event_delayed_put(struct task_struct *task)
  10334. {
  10335. int ctxn;
  10336. for_each_task_context_nr(ctxn)
  10337. WARN_ON_ONCE(task->perf_event_ctxp[ctxn]);
  10338. }
  10339. struct file *perf_event_get(unsigned int fd)
  10340. {
  10341. struct file *file = fget(fd);
  10342. if (!file)
  10343. return ERR_PTR(-EBADF);
  10344. if (file->f_op != &perf_fops) {
  10345. fput(file);
  10346. return ERR_PTR(-EBADF);
  10347. }
  10348. return file;
  10349. }
  10350. const struct perf_event *perf_get_event(struct file *file)
  10351. {
  10352. if (file->f_op != &perf_fops)
  10353. return ERR_PTR(-EINVAL);
  10354. return file->private_data;
  10355. }
  10356. const struct perf_event_attr *perf_event_attrs(struct perf_event *event)
  10357. {
  10358. if (!event)
  10359. return ERR_PTR(-EINVAL);
  10360. return &event->attr;
  10361. }
  10362. /*
  10363. * Inherit an event from parent task to child task.
  10364. *
  10365. * Returns:
  10366. * - valid pointer on success
  10367. * - NULL for orphaned events
  10368. * - IS_ERR() on error
  10369. */
  10370. static struct perf_event *
  10371. inherit_event(struct perf_event *parent_event,
  10372. struct task_struct *parent,
  10373. struct perf_event_context *parent_ctx,
  10374. struct task_struct *child,
  10375. struct perf_event *group_leader,
  10376. struct perf_event_context *child_ctx)
  10377. {
  10378. enum perf_event_state parent_state = parent_event->state;
  10379. struct perf_event *child_event;
  10380. unsigned long flags;
  10381. /*
  10382. * Instead of creating recursive hierarchies of events,
  10383. * we link inherited events back to the original parent,
  10384. * which has a filp for sure, which we use as the reference
  10385. * count:
  10386. */
  10387. if (parent_event->parent)
  10388. parent_event = parent_event->parent;
  10389. child_event = perf_event_alloc(&parent_event->attr,
  10390. parent_event->cpu,
  10391. child,
  10392. group_leader, parent_event,
  10393. NULL, NULL, -1);
  10394. if (IS_ERR(child_event))
  10395. return child_event;
  10396. if ((child_event->attach_state & PERF_ATTACH_TASK_DATA) &&
  10397. !child_ctx->task_ctx_data) {
  10398. struct pmu *pmu = child_event->pmu;
  10399. child_ctx->task_ctx_data = alloc_task_ctx_data(pmu);
  10400. if (!child_ctx->task_ctx_data) {
  10401. free_event(child_event);
  10402. return ERR_PTR(-ENOMEM);
  10403. }
  10404. }
  10405. /*
  10406. * is_orphaned_event() and list_add_tail(&parent_event->child_list)
  10407. * must be under the same lock in order to serialize against
  10408. * perf_event_release_kernel(), such that either we must observe
  10409. * is_orphaned_event() or they will observe us on the child_list.
  10410. */
  10411. mutex_lock(&parent_event->child_mutex);
  10412. if (is_orphaned_event(parent_event) ||
  10413. !atomic_long_inc_not_zero(&parent_event->refcount)) {
  10414. mutex_unlock(&parent_event->child_mutex);
  10415. /* task_ctx_data is freed with child_ctx */
  10416. free_event(child_event);
  10417. return NULL;
  10418. }
  10419. get_ctx(child_ctx);
  10420. /*
  10421. * Make the child state follow the state of the parent event,
  10422. * not its attr.disabled bit. We hold the parent's mutex,
  10423. * so we won't race with perf_event_{en, dis}able_family.
  10424. */
  10425. if (parent_state >= PERF_EVENT_STATE_INACTIVE)
  10426. child_event->state = PERF_EVENT_STATE_INACTIVE;
  10427. else
  10428. child_event->state = PERF_EVENT_STATE_OFF;
  10429. if (parent_event->attr.freq) {
  10430. u64 sample_period = parent_event->hw.sample_period;
  10431. struct hw_perf_event *hwc = &child_event->hw;
  10432. hwc->sample_period = sample_period;
  10433. hwc->last_period = sample_period;
  10434. local64_set(&hwc->period_left, sample_period);
  10435. }
  10436. child_event->ctx = child_ctx;
  10437. child_event->overflow_handler = parent_event->overflow_handler;
  10438. child_event->overflow_handler_context
  10439. = parent_event->overflow_handler_context;
  10440. /*
  10441. * Precalculate sample_data sizes
  10442. */
  10443. perf_event__header_size(child_event);
  10444. perf_event__id_header_size(child_event);
  10445. /*
  10446. * Link it up in the child's context:
  10447. */
  10448. raw_spin_lock_irqsave(&child_ctx->lock, flags);
  10449. add_event_to_ctx(child_event, child_ctx);
  10450. raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
  10451. /*
  10452. * Link this into the parent event's child list
  10453. */
  10454. list_add_tail(&child_event->child_list, &parent_event->child_list);
  10455. mutex_unlock(&parent_event->child_mutex);
  10456. return child_event;
  10457. }
  10458. /*
  10459. * Inherits an event group.
  10460. *
  10461. * This will quietly suppress orphaned events; !inherit_event() is not an error.
  10462. * This matches with perf_event_release_kernel() removing all child events.
  10463. *
  10464. * Returns:
  10465. * - 0 on success
  10466. * - <0 on error
  10467. */
  10468. static int inherit_group(struct perf_event *parent_event,
  10469. struct task_struct *parent,
  10470. struct perf_event_context *parent_ctx,
  10471. struct task_struct *child,
  10472. struct perf_event_context *child_ctx)
  10473. {
  10474. struct perf_event *leader;
  10475. struct perf_event *sub;
  10476. struct perf_event *child_ctr;
  10477. leader = inherit_event(parent_event, parent, parent_ctx,
  10478. child, NULL, child_ctx);
  10479. if (IS_ERR(leader))
  10480. return PTR_ERR(leader);
  10481. /*
  10482. * @leader can be NULL here because of is_orphaned_event(). In this
  10483. * case inherit_event() will create individual events, similar to what
  10484. * perf_group_detach() would do anyway.
  10485. */
  10486. for_each_sibling_event(sub, parent_event) {
  10487. child_ctr = inherit_event(sub, parent, parent_ctx,
  10488. child, leader, child_ctx);
  10489. if (IS_ERR(child_ctr))
  10490. return PTR_ERR(child_ctr);
  10491. if (sub->aux_event == parent_event && child_ctr &&
  10492. !perf_get_aux_event(child_ctr, leader))
  10493. return -EINVAL;
  10494. }
  10495. return 0;
  10496. }
  10497. /*
  10498. * Creates the child task context and tries to inherit the event-group.
  10499. *
  10500. * Clears @inherited_all on !attr.inherited or error. Note that we'll leave
  10501. * inherited_all set when we 'fail' to inherit an orphaned event; this is
  10502. * consistent with perf_event_release_kernel() removing all child events.
  10503. *
  10504. * Returns:
  10505. * - 0 on success
  10506. * - <0 on error
  10507. */
  10508. static int
  10509. inherit_task_group(struct perf_event *event, struct task_struct *parent,
  10510. struct perf_event_context *parent_ctx,
  10511. struct task_struct *child, int ctxn,
  10512. int *inherited_all)
  10513. {
  10514. int ret;
  10515. struct perf_event_context *child_ctx;
  10516. if (!event->attr.inherit) {
  10517. *inherited_all = 0;
  10518. return 0;
  10519. }
  10520. child_ctx = child->perf_event_ctxp[ctxn];
  10521. if (!child_ctx) {
  10522. /*
  10523. * This is executed from the parent task context, so
  10524. * inherit events that have been marked for cloning.
  10525. * First allocate and initialize a context for the
  10526. * child.
  10527. */
  10528. child_ctx = alloc_perf_context(parent_ctx->pmu, child);
  10529. if (!child_ctx)
  10530. return -ENOMEM;
  10531. child->perf_event_ctxp[ctxn] = child_ctx;
  10532. }
  10533. ret = inherit_group(event, parent, parent_ctx,
  10534. child, child_ctx);
  10535. if (ret)
  10536. *inherited_all = 0;
  10537. return ret;
  10538. }
  10539. /*
  10540. * Initialize the perf_event context in task_struct
  10541. */
  10542. static int perf_event_init_context(struct task_struct *child, int ctxn)
  10543. {
  10544. struct perf_event_context *child_ctx, *parent_ctx;
  10545. struct perf_event_context *cloned_ctx;
  10546. struct perf_event *event;
  10547. struct task_struct *parent = current;
  10548. int inherited_all = 1;
  10549. unsigned long flags;
  10550. int ret = 0;
  10551. if (likely(!parent->perf_event_ctxp[ctxn]))
  10552. return 0;
  10553. /*
  10554. * If the parent's context is a clone, pin it so it won't get
  10555. * swapped under us.
  10556. */
  10557. parent_ctx = perf_pin_task_context(parent, ctxn);
  10558. if (!parent_ctx)
  10559. return 0;
  10560. /*
  10561. * No need to check if parent_ctx != NULL here; since we saw
  10562. * it non-NULL earlier, the only reason for it to become NULL
  10563. * is if we exit, and since we're currently in the middle of
  10564. * a fork we can't be exiting at the same time.
  10565. */
  10566. /*
  10567. * Lock the parent list. No need to lock the child - not PID
  10568. * hashed yet and not running, so nobody can access it.
  10569. */
  10570. mutex_lock(&parent_ctx->mutex);
  10571. /*
  10572. * We dont have to disable NMIs - we are only looking at
  10573. * the list, not manipulating it:
  10574. */
  10575. perf_event_groups_for_each(event, &parent_ctx->pinned_groups) {
  10576. ret = inherit_task_group(event, parent, parent_ctx,
  10577. child, ctxn, &inherited_all);
  10578. if (ret)
  10579. goto out_unlock;
  10580. }
  10581. /*
  10582. * We can't hold ctx->lock when iterating the ->flexible_group list due
  10583. * to allocations, but we need to prevent rotation because
  10584. * rotate_ctx() will change the list from interrupt context.
  10585. */
  10586. raw_spin_lock_irqsave(&parent_ctx->lock, flags);
  10587. parent_ctx->rotate_disable = 1;
  10588. raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
  10589. perf_event_groups_for_each(event, &parent_ctx->flexible_groups) {
  10590. ret = inherit_task_group(event, parent, parent_ctx,
  10591. child, ctxn, &inherited_all);
  10592. if (ret)
  10593. goto out_unlock;
  10594. }
  10595. raw_spin_lock_irqsave(&parent_ctx->lock, flags);
  10596. parent_ctx->rotate_disable = 0;
  10597. child_ctx = child->perf_event_ctxp[ctxn];
  10598. if (child_ctx && inherited_all) {
  10599. /*
  10600. * Mark the child context as a clone of the parent
  10601. * context, or of whatever the parent is a clone of.
  10602. *
  10603. * Note that if the parent is a clone, the holding of
  10604. * parent_ctx->lock avoids it from being uncloned.
  10605. */
  10606. cloned_ctx = parent_ctx->parent_ctx;
  10607. if (cloned_ctx) {
  10608. child_ctx->parent_ctx = cloned_ctx;
  10609. child_ctx->parent_gen = parent_ctx->parent_gen;
  10610. } else {
  10611. child_ctx->parent_ctx = parent_ctx;
  10612. child_ctx->parent_gen = parent_ctx->generation;
  10613. }
  10614. get_ctx(child_ctx->parent_ctx);
  10615. }
  10616. raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
  10617. out_unlock:
  10618. mutex_unlock(&parent_ctx->mutex);
  10619. perf_unpin_context(parent_ctx);
  10620. put_ctx(parent_ctx);
  10621. return ret;
  10622. }
  10623. /*
  10624. * Initialize the perf_event context in task_struct
  10625. */
  10626. int perf_event_init_task(struct task_struct *child)
  10627. {
  10628. int ctxn, ret;
  10629. memset(child->perf_event_ctxp, 0, sizeof(child->perf_event_ctxp));
  10630. mutex_init(&child->perf_event_mutex);
  10631. INIT_LIST_HEAD(&child->perf_event_list);
  10632. for_each_task_context_nr(ctxn) {
  10633. ret = perf_event_init_context(child, ctxn);
  10634. if (ret) {
  10635. perf_event_free_task(child);
  10636. return ret;
  10637. }
  10638. }
  10639. return 0;
  10640. }
  10641. static void __init perf_event_init_all_cpus(void)
  10642. {
  10643. struct swevent_htable *swhash;
  10644. int cpu;
  10645. zalloc_cpumask_var(&perf_online_mask, GFP_KERNEL);
  10646. for_each_possible_cpu(cpu) {
  10647. swhash = &per_cpu(swevent_htable, cpu);
  10648. mutex_init(&swhash->hlist_mutex);
  10649. INIT_LIST_HEAD(&per_cpu(active_ctx_list, cpu));
  10650. INIT_LIST_HEAD(&per_cpu(pmu_sb_events.list, cpu));
  10651. raw_spin_lock_init(&per_cpu(pmu_sb_events.lock, cpu));
  10652. #ifdef CONFIG_CGROUP_PERF
  10653. INIT_LIST_HEAD(&per_cpu(cgrp_cpuctx_list, cpu));
  10654. #endif
  10655. INIT_LIST_HEAD(&per_cpu(sched_cb_list, cpu));
  10656. }
  10657. }
  10658. static void perf_swevent_init_cpu(unsigned int cpu)
  10659. {
  10660. struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
  10661. mutex_lock(&swhash->hlist_mutex);
  10662. if (swhash->hlist_refcount > 0 && !swevent_hlist_deref(swhash)) {
  10663. struct swevent_hlist *hlist;
  10664. hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu));
  10665. WARN_ON(!hlist);
  10666. rcu_assign_pointer(swhash->swevent_hlist, hlist);
  10667. }
  10668. mutex_unlock(&swhash->hlist_mutex);
  10669. }
  10670. #if defined CONFIG_HOTPLUG_CPU || defined CONFIG_KEXEC_CORE
  10671. static void __perf_event_exit_context(void *__info)
  10672. {
  10673. struct perf_event_context *ctx = __info;
  10674. struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
  10675. struct perf_event *event;
  10676. raw_spin_lock(&ctx->lock);
  10677. ctx_sched_out(ctx, cpuctx, EVENT_TIME);
  10678. list_for_each_entry(event, &ctx->event_list, event_entry)
  10679. __perf_remove_from_context(event, cpuctx, ctx, (void *)DETACH_GROUP);
  10680. raw_spin_unlock(&ctx->lock);
  10681. }
  10682. static void perf_event_exit_cpu_context(int cpu)
  10683. {
  10684. struct perf_cpu_context *cpuctx;
  10685. struct perf_event_context *ctx;
  10686. struct pmu *pmu;
  10687. mutex_lock(&pmus_lock);
  10688. list_for_each_entry(pmu, &pmus, entry) {
  10689. cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
  10690. ctx = &cpuctx->ctx;
  10691. mutex_lock(&ctx->mutex);
  10692. smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1);
  10693. cpuctx->online = 0;
  10694. mutex_unlock(&ctx->mutex);
  10695. }
  10696. cpumask_clear_cpu(cpu, perf_online_mask);
  10697. mutex_unlock(&pmus_lock);
  10698. }
  10699. #else
  10700. static void perf_event_exit_cpu_context(int cpu) { }
  10701. #endif
  10702. int perf_event_init_cpu(unsigned int cpu)
  10703. {
  10704. struct perf_cpu_context *cpuctx;
  10705. struct perf_event_context *ctx;
  10706. struct pmu *pmu;
  10707. perf_swevent_init_cpu(cpu);
  10708. mutex_lock(&pmus_lock);
  10709. cpumask_set_cpu(cpu, perf_online_mask);
  10710. list_for_each_entry(pmu, &pmus, entry) {
  10711. cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
  10712. ctx = &cpuctx->ctx;
  10713. mutex_lock(&ctx->mutex);
  10714. cpuctx->online = 1;
  10715. mutex_unlock(&ctx->mutex);
  10716. }
  10717. mutex_unlock(&pmus_lock);
  10718. return 0;
  10719. }
  10720. int perf_event_exit_cpu(unsigned int cpu)
  10721. {
  10722. perf_event_exit_cpu_context(cpu);
  10723. return 0;
  10724. }
  10725. static int
  10726. perf_reboot(struct notifier_block *notifier, unsigned long val, void *v)
  10727. {
  10728. int cpu;
  10729. for_each_online_cpu(cpu)
  10730. perf_event_exit_cpu(cpu);
  10731. return NOTIFY_OK;
  10732. }
  10733. /*
  10734. * Run the perf reboot notifier at the very last possible moment so that
  10735. * the generic watchdog code runs as long as possible.
  10736. */
  10737. static struct notifier_block perf_reboot_notifier = {
  10738. .notifier_call = perf_reboot,
  10739. .priority = INT_MIN,
  10740. };
  10741. void __init perf_event_init(void)
  10742. {
  10743. int ret;
  10744. idr_init(&pmu_idr);
  10745. perf_event_init_all_cpus();
  10746. init_srcu_struct(&pmus_srcu);
  10747. perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE);
  10748. perf_pmu_register(&perf_cpu_clock, NULL, -1);
  10749. perf_pmu_register(&perf_task_clock, NULL, -1);
  10750. perf_tp_register();
  10751. perf_event_init_cpu(smp_processor_id());
  10752. register_reboot_notifier(&perf_reboot_notifier);
  10753. ret = init_hw_breakpoint();
  10754. WARN(ret, "hw_breakpoint initialization failed with: %d", ret);
  10755. /*
  10756. * Build time assertion that we keep the data_head at the intended
  10757. * location. IOW, validation we got the __reserved[] size right.
  10758. */
  10759. BUILD_BUG_ON((offsetof(struct perf_event_mmap_page, data_head))
  10760. != 1024);
  10761. }
  10762. ssize_t perf_event_sysfs_show(struct device *dev, struct device_attribute *attr,
  10763. char *page)
  10764. {
  10765. struct perf_pmu_events_attr *pmu_attr =
  10766. container_of(attr, struct perf_pmu_events_attr, attr);
  10767. if (pmu_attr->event_str)
  10768. return sprintf(page, "%s\n", pmu_attr->event_str);
  10769. return 0;
  10770. }
  10771. EXPORT_SYMBOL_GPL(perf_event_sysfs_show);
  10772. static int __init perf_event_sysfs_init(void)
  10773. {
  10774. struct pmu *pmu;
  10775. int ret;
  10776. mutex_lock(&pmus_lock);
  10777. ret = bus_register(&pmu_bus);
  10778. if (ret)
  10779. goto unlock;
  10780. list_for_each_entry(pmu, &pmus, entry) {
  10781. if (!pmu->name || pmu->type < 0)
  10782. continue;
  10783. ret = pmu_dev_alloc(pmu);
  10784. WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret);
  10785. }
  10786. pmu_bus_running = 1;
  10787. ret = 0;
  10788. unlock:
  10789. mutex_unlock(&pmus_lock);
  10790. return ret;
  10791. }
  10792. device_initcall(perf_event_sysfs_init);
  10793. #ifdef CONFIG_CGROUP_PERF
  10794. static struct cgroup_subsys_state *
  10795. perf_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
  10796. {
  10797. struct perf_cgroup *jc;
  10798. jc = kzalloc(sizeof(*jc), GFP_KERNEL);
  10799. if (!jc)
  10800. return ERR_PTR(-ENOMEM);
  10801. jc->info = alloc_percpu(struct perf_cgroup_info);
  10802. if (!jc->info) {
  10803. kfree(jc);
  10804. return ERR_PTR(-ENOMEM);
  10805. }
  10806. return &jc->css;
  10807. }
  10808. static void perf_cgroup_css_free(struct cgroup_subsys_state *css)
  10809. {
  10810. struct perf_cgroup *jc = container_of(css, struct perf_cgroup, css);
  10811. free_percpu(jc->info);
  10812. kfree(jc);
  10813. }
  10814. static int perf_cgroup_css_online(struct cgroup_subsys_state *css)
  10815. {
  10816. perf_event_cgroup(css->cgroup);
  10817. return 0;
  10818. }
  10819. static int __perf_cgroup_move(void *info)
  10820. {
  10821. struct task_struct *task = info;
  10822. rcu_read_lock();
  10823. perf_cgroup_switch(task, PERF_CGROUP_SWOUT | PERF_CGROUP_SWIN);
  10824. rcu_read_unlock();
  10825. return 0;
  10826. }
  10827. static void perf_cgroup_attach(struct cgroup_taskset *tset)
  10828. {
  10829. struct task_struct *task;
  10830. struct cgroup_subsys_state *css;
  10831. cgroup_taskset_for_each(task, css, tset)
  10832. task_function_call(task, __perf_cgroup_move, task);
  10833. }
  10834. struct cgroup_subsys perf_event_cgrp_subsys = {
  10835. .css_alloc = perf_cgroup_css_alloc,
  10836. .css_free = perf_cgroup_css_free,
  10837. .css_online = perf_cgroup_css_online,
  10838. .attach = perf_cgroup_attach,
  10839. /*
  10840. * Implicitly enable on dfl hierarchy so that perf events can
  10841. * always be filtered by cgroup2 path as long as perf_event
  10842. * controller is not mounted on a legacy hierarchy.
  10843. */
  10844. .implicit_on_dfl = true,
  10845. .threaded = true,
  10846. };
  10847. #endif /* CONFIG_CGROUP_PERF */