trace.c 235 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645664666476648664966506651665266536654665566566657665866596660666166626663666466656666666766686669667066716672667366746675667666776678667966806681668266836684668566866687668866896690669166926693669466956696669766986699670067016702670367046705670667076708670967106711671267136714671567166717671867196720672167226723672467256726672767286729673067316732673367346735673667376738673967406741674267436744674567466747674867496750675167526753675467556756675767586759676067616762676367646765676667676768676967706771677267736774677567766777677867796780678167826783678467856786678767886789679067916792679367946795679667976798679968006801680268036804680568066807680868096810681168126813681468156816681768186819682068216822682368246825682668276828682968306831683268336834683568366837683868396840684168426843684468456846684768486849685068516852685368546855685668576858685968606861686268636864686568666867686868696870687168726873687468756876687768786879688068816882688368846885688668876888688968906891689268936894689568966897689868996900690169026903690469056906690769086909691069116912691369146915691669176918691969206921692269236924692569266927692869296930693169326933693469356936693769386939694069416942694369446945694669476948694969506951695269536954695569566957695869596960696169626963696469656966696769686969697069716972697369746975697669776978697969806981698269836984698569866987698869896990699169926993699469956996699769986999700070017002700370047005700670077008700970107011701270137014701570167017701870197020702170227023702470257026702770287029703070317032703370347035703670377038703970407041704270437044704570467047704870497050705170527053705470557056705770587059706070617062706370647065706670677068706970707071707270737074707570767077707870797080708170827083708470857086708770887089709070917092709370947095709670977098709971007101710271037104710571067107710871097110711171127113711471157116711771187119712071217122712371247125712671277128712971307131713271337134713571367137713871397140714171427143714471457146714771487149715071517152715371547155715671577158715971607161716271637164716571667167716871697170717171727173717471757176717771787179718071817182718371847185718671877188718971907191719271937194719571967197719871997200720172027203720472057206720772087209721072117212721372147215721672177218721972207221722272237224722572267227722872297230723172327233723472357236723772387239724072417242724372447245724672477248724972507251725272537254725572567257725872597260726172627263726472657266726772687269727072717272727372747275727672777278727972807281728272837284728572867287728872897290729172927293729472957296729772987299730073017302730373047305730673077308730973107311731273137314731573167317731873197320732173227323732473257326732773287329733073317332733373347335733673377338733973407341734273437344734573467347734873497350735173527353735473557356735773587359736073617362736373647365736673677368736973707371737273737374737573767377737873797380738173827383738473857386738773887389739073917392739373947395739673977398739974007401740274037404740574067407740874097410741174127413741474157416741774187419742074217422742374247425742674277428742974307431743274337434743574367437743874397440744174427443744474457446744774487449745074517452745374547455745674577458745974607461746274637464746574667467746874697470747174727473747474757476747774787479748074817482748374847485748674877488748974907491749274937494749574967497749874997500750175027503750475057506750775087509751075117512751375147515751675177518751975207521752275237524752575267527752875297530753175327533753475357536753775387539754075417542754375447545754675477548754975507551755275537554755575567557755875597560756175627563756475657566756775687569757075717572757375747575757675777578757975807581758275837584758575867587758875897590759175927593759475957596759775987599760076017602760376047605760676077608760976107611761276137614761576167617761876197620762176227623762476257626762776287629763076317632763376347635763676377638763976407641764276437644764576467647764876497650765176527653765476557656765776587659766076617662766376647665766676677668766976707671767276737674767576767677767876797680768176827683768476857686768776887689769076917692769376947695769676977698769977007701770277037704770577067707770877097710771177127713771477157716771777187719772077217722772377247725772677277728772977307731773277337734773577367737773877397740774177427743774477457746774777487749775077517752775377547755775677577758775977607761776277637764776577667767776877697770777177727773777477757776777777787779778077817782778377847785778677877788778977907791779277937794779577967797779877997800780178027803780478057806780778087809781078117812781378147815781678177818781978207821782278237824782578267827782878297830783178327833783478357836783778387839784078417842784378447845784678477848784978507851785278537854785578567857785878597860786178627863786478657866786778687869787078717872787378747875787678777878787978807881788278837884788578867887788878897890789178927893789478957896789778987899790079017902790379047905790679077908790979107911791279137914791579167917791879197920792179227923792479257926792779287929793079317932793379347935793679377938793979407941794279437944794579467947794879497950795179527953795479557956795779587959796079617962796379647965796679677968796979707971797279737974797579767977797879797980798179827983798479857986798779887989799079917992799379947995799679977998799980008001800280038004800580068007800880098010801180128013801480158016801780188019802080218022802380248025802680278028802980308031803280338034803580368037803880398040804180428043804480458046804780488049805080518052805380548055805680578058805980608061806280638064806580668067806880698070807180728073807480758076807780788079808080818082808380848085808680878088808980908091809280938094809580968097809880998100810181028103810481058106810781088109811081118112811381148115811681178118811981208121812281238124812581268127812881298130813181328133813481358136813781388139814081418142814381448145814681478148814981508151815281538154815581568157815881598160816181628163816481658166816781688169817081718172817381748175817681778178817981808181818281838184818581868187818881898190819181928193819481958196819781988199820082018202820382048205820682078208820982108211821282138214821582168217821882198220822182228223822482258226822782288229823082318232823382348235823682378238823982408241824282438244824582468247824882498250825182528253825482558256825782588259826082618262826382648265826682678268826982708271827282738274827582768277827882798280828182828283828482858286828782888289829082918292829382948295829682978298829983008301830283038304830583068307830883098310831183128313831483158316831783188319832083218322832383248325832683278328832983308331833283338334833583368337833883398340834183428343834483458346834783488349835083518352835383548355835683578358835983608361836283638364836583668367836883698370837183728373837483758376837783788379838083818382838383848385838683878388838983908391839283938394839583968397839883998400840184028403840484058406840784088409841084118412841384148415841684178418841984208421842284238424842584268427842884298430843184328433843484358436843784388439844084418442844384448445844684478448844984508451845284538454845584568457845884598460846184628463846484658466846784688469847084718472847384748475847684778478847984808481848284838484848584868487848884898490849184928493849484958496849784988499850085018502850385048505850685078508850985108511851285138514851585168517851885198520852185228523852485258526852785288529853085318532853385348535853685378538853985408541854285438544854585468547854885498550855185528553855485558556855785588559856085618562856385648565856685678568856985708571857285738574857585768577857885798580858185828583858485858586858785888589859085918592859385948595859685978598859986008601860286038604860586068607860886098610861186128613861486158616861786188619862086218622862386248625862686278628862986308631863286338634863586368637863886398640864186428643864486458646864786488649865086518652865386548655865686578658865986608661866286638664866586668667866886698670867186728673867486758676867786788679868086818682868386848685868686878688868986908691869286938694869586968697869886998700870187028703870487058706870787088709871087118712871387148715871687178718871987208721872287238724872587268727872887298730873187328733873487358736873787388739874087418742874387448745874687478748874987508751875287538754875587568757875887598760876187628763876487658766876787688769877087718772877387748775877687778778877987808781878287838784878587868787878887898790879187928793879487958796879787988799880088018802880388048805880688078808880988108811881288138814881588168817881888198820882188228823882488258826882788288829883088318832883388348835883688378838883988408841884288438844884588468847884888498850885188528853885488558856885788588859886088618862886388648865886688678868886988708871887288738874887588768877887888798880888188828883888488858886888788888889889088918892889388948895889688978898889989008901890289038904890589068907890889098910891189128913891489158916891789188919892089218922892389248925892689278928892989308931893289338934893589368937893889398940894189428943894489458946894789488949895089518952895389548955895689578958895989608961896289638964896589668967896889698970897189728973897489758976897789788979898089818982898389848985898689878988898989908991899289938994899589968997899889999000900190029003900490059006900790089009901090119012901390149015901690179018901990209021902290239024902590269027902890299030903190329033903490359036903790389039904090419042904390449045904690479048904990509051905290539054905590569057905890599060906190629063906490659066906790689069907090719072907390749075907690779078907990809081908290839084908590869087908890899090909190929093909490959096909790989099910091019102910391049105910691079108910991109111911291139114911591169117911891199120912191229123912491259126912791289129913091319132913391349135913691379138913991409141914291439144914591469147914891499150915191529153915491559156915791589159916091619162916391649165916691679168916991709171917291739174917591769177917891799180918191829183918491859186918791889189919091919192919391949195919691979198919992009201920292039204920592069207920892099210921192129213921492159216921792189219922092219222922392249225922692279228922992309231923292339234923592369237923892399240924192429243924492459246924792489249925092519252925392549255925692579258925992609261926292639264926592669267926892699270927192729273927492759276927792789279928092819282928392849285928692879288928992909291929292939294929592969297929892999300930193029303930493059306930793089309931093119312931393149315931693179318931993209321932293239324932593269327932893299330933193329333933493359336933793389339934093419342934393449345934693479348934993509351935293539354935593569357935893599360936193629363936493659366936793689369937093719372937393749375937693779378937993809381938293839384938593869387938893899390939193929393939493959396939793989399940094019402940394049405940694079408940994109411941294139414941594169417941894199420942194229423942494259426942794289429943094319432943394349435943694379438943994409441944294439444944594469447944894499450945194529453945494559456945794589459946094619462946394649465946694679468946994709471947294739474947594769477947894799480948194829483948494859486948794889489949094919492949394949495949694979498949995009501950295039504950595069507950895099510951195129513951495159516951795189519952095219522952395249525952695279528952995309531953295339534953595369537953895399540954195429543954495459546954795489549955095519552955395549555955695579558955995609561956295639564956595669567956895699570957195729573957495759576957795789579958095819582958395849585958695879588958995909591959295939594959595969597959895999600960196029603960496059606960796089609961096119612961396149615961696179618961996209621962296239624962596269627962896299630963196329633963496359636963796389639964096419642964396449645964696479648964996509651965296539654965596569657965896599660966196629663966496659666966796689669967096719672967396749675967696779678967996809681968296839684968596869687968896899690969196929693969496959696969796989699970097019702970397049705970697079708970997109711971297139714971597169717971897199720972197229723972497259726972797289729
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * ring buffer based function tracer
  4. *
  5. * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
  6. * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
  7. *
  8. * Originally taken from the RT patch by:
  9. * Arnaldo Carvalho de Melo <acme@redhat.com>
  10. *
  11. * Based on code from the latency_tracer, that is:
  12. * Copyright (C) 2004-2006 Ingo Molnar
  13. * Copyright (C) 2004 Nadia Yvette Chambers
  14. */
  15. #include <linux/ring_buffer.h>
  16. #include <generated/utsrelease.h>
  17. #include <linux/stacktrace.h>
  18. #include <linux/writeback.h>
  19. #include <linux/kallsyms.h>
  20. #include <linux/security.h>
  21. #include <linux/seq_file.h>
  22. #include <linux/notifier.h>
  23. #include <linux/irqflags.h>
  24. #include <linux/debugfs.h>
  25. #include <linux/tracefs.h>
  26. #include <linux/pagemap.h>
  27. #include <linux/hardirq.h>
  28. #include <linux/linkage.h>
  29. #include <linux/uaccess.h>
  30. #include <linux/vmalloc.h>
  31. #include <linux/ftrace.h>
  32. #include <linux/module.h>
  33. #include <linux/percpu.h>
  34. #include <linux/splice.h>
  35. #include <linux/kdebug.h>
  36. #include <linux/string.h>
  37. #include <linux/mount.h>
  38. #include <linux/rwsem.h>
  39. #include <linux/slab.h>
  40. #include <linux/ctype.h>
  41. #include <linux/init.h>
  42. #include <linux/poll.h>
  43. #include <linux/nmi.h>
  44. #include <linux/fs.h>
  45. #include <linux/trace.h>
  46. #include <linux/sched/clock.h>
  47. #include <linux/sched/rt.h>
  48. #include <linux/fsnotify.h>
  49. #include <linux/irq_work.h>
  50. #include <linux/workqueue.h>
  51. #include <trace/hooks/ftrace_dump.h>
  52. #include "trace.h"
  53. #include "trace_output.h"
  54. /*
  55. * On boot up, the ring buffer is set to the minimum size, so that
  56. * we do not waste memory on systems that are not using tracing.
  57. */
  58. bool ring_buffer_expanded;
  59. /*
  60. * We need to change this state when a selftest is running.
  61. * A selftest will lurk into the ring-buffer to count the
  62. * entries inserted during the selftest although some concurrent
  63. * insertions into the ring-buffer such as trace_printk could occurred
  64. * at the same time, giving false positive or negative results.
  65. */
  66. static bool __read_mostly tracing_selftest_running;
  67. /*
  68. * If boot-time tracing including tracers/events via kernel cmdline
  69. * is running, we do not want to run SELFTEST.
  70. */
  71. bool __read_mostly tracing_selftest_disabled;
  72. #ifdef CONFIG_FTRACE_STARTUP_TEST
  73. void __init disable_tracing_selftest(const char *reason)
  74. {
  75. if (!tracing_selftest_disabled) {
  76. tracing_selftest_disabled = true;
  77. pr_info("Ftrace startup test is disabled due to %s\n", reason);
  78. }
  79. }
  80. #endif
  81. /* Pipe tracepoints to printk */
  82. struct trace_iterator *tracepoint_print_iter;
  83. int tracepoint_printk;
  84. static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key);
  85. /* For tracers that don't implement custom flags */
  86. static struct tracer_opt dummy_tracer_opt[] = {
  87. { }
  88. };
  89. static int
  90. dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
  91. {
  92. return 0;
  93. }
  94. /*
  95. * To prevent the comm cache from being overwritten when no
  96. * tracing is active, only save the comm when a trace event
  97. * occurred.
  98. */
  99. static DEFINE_PER_CPU(bool, trace_taskinfo_save);
  100. /*
  101. * Kill all tracing for good (never come back).
  102. * It is initialized to 1 but will turn to zero if the initialization
  103. * of the tracer is successful. But that is the only place that sets
  104. * this back to zero.
  105. */
  106. static int tracing_disabled = 1;
  107. cpumask_var_t __read_mostly tracing_buffer_mask;
  108. /*
  109. * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
  110. *
  111. * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
  112. * is set, then ftrace_dump is called. This will output the contents
  113. * of the ftrace buffers to the console. This is very useful for
  114. * capturing traces that lead to crashes and outputing it to a
  115. * serial console.
  116. *
  117. * It is default off, but you can enable it with either specifying
  118. * "ftrace_dump_on_oops" in the kernel command line, or setting
  119. * /proc/sys/kernel/ftrace_dump_on_oops
  120. * Set 1 if you want to dump buffers of all CPUs
  121. * Set 2 if you want to dump the buffer of the CPU that triggered oops
  122. */
  123. enum ftrace_dump_mode ftrace_dump_on_oops;
  124. /* When set, tracing will stop when a WARN*() is hit */
  125. int __disable_trace_on_warning;
  126. #ifdef CONFIG_TRACE_EVAL_MAP_FILE
  127. /* Map of enums to their values, for "eval_map" file */
  128. struct trace_eval_map_head {
  129. struct module *mod;
  130. unsigned long length;
  131. };
  132. union trace_eval_map_item;
  133. struct trace_eval_map_tail {
  134. /*
  135. * "end" is first and points to NULL as it must be different
  136. * than "mod" or "eval_string"
  137. */
  138. union trace_eval_map_item *next;
  139. const char *end; /* points to NULL */
  140. };
  141. static DEFINE_MUTEX(trace_eval_mutex);
  142. /*
  143. * The trace_eval_maps are saved in an array with two extra elements,
  144. * one at the beginning, and one at the end. The beginning item contains
  145. * the count of the saved maps (head.length), and the module they
  146. * belong to if not built in (head.mod). The ending item contains a
  147. * pointer to the next array of saved eval_map items.
  148. */
  149. union trace_eval_map_item {
  150. struct trace_eval_map map;
  151. struct trace_eval_map_head head;
  152. struct trace_eval_map_tail tail;
  153. };
  154. static union trace_eval_map_item *trace_eval_maps;
  155. #endif /* CONFIG_TRACE_EVAL_MAP_FILE */
  156. int tracing_set_tracer(struct trace_array *tr, const char *buf);
  157. static void ftrace_trace_userstack(struct trace_array *tr,
  158. struct trace_buffer *buffer,
  159. unsigned long flags, int pc);
  160. #define MAX_TRACER_SIZE 100
  161. static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
  162. static char *default_bootup_tracer;
  163. static bool allocate_snapshot;
  164. static int __init set_cmdline_ftrace(char *str)
  165. {
  166. strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
  167. default_bootup_tracer = bootup_tracer_buf;
  168. /* We are using ftrace early, expand it */
  169. ring_buffer_expanded = true;
  170. return 1;
  171. }
  172. __setup("ftrace=", set_cmdline_ftrace);
  173. static int __init set_ftrace_dump_on_oops(char *str)
  174. {
  175. if (*str++ != '=' || !*str) {
  176. ftrace_dump_on_oops = DUMP_ALL;
  177. return 1;
  178. }
  179. if (!strcmp("orig_cpu", str)) {
  180. ftrace_dump_on_oops = DUMP_ORIG;
  181. return 1;
  182. }
  183. return 0;
  184. }
  185. __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
  186. static int __init stop_trace_on_warning(char *str)
  187. {
  188. if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
  189. __disable_trace_on_warning = 1;
  190. return 1;
  191. }
  192. __setup("traceoff_on_warning", stop_trace_on_warning);
  193. static int __init boot_alloc_snapshot(char *str)
  194. {
  195. allocate_snapshot = true;
  196. /* We also need the main ring buffer expanded */
  197. ring_buffer_expanded = true;
  198. return 1;
  199. }
  200. __setup("alloc_snapshot", boot_alloc_snapshot);
  201. static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
  202. static int __init set_trace_boot_options(char *str)
  203. {
  204. strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
  205. return 1;
  206. }
  207. __setup("trace_options=", set_trace_boot_options);
  208. static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
  209. static char *trace_boot_clock __initdata;
  210. static int __init set_trace_boot_clock(char *str)
  211. {
  212. strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
  213. trace_boot_clock = trace_boot_clock_buf;
  214. return 1;
  215. }
  216. __setup("trace_clock=", set_trace_boot_clock);
  217. static int __init set_tracepoint_printk(char *str)
  218. {
  219. /* Ignore the "tp_printk_stop_on_boot" param */
  220. if (*str == '_')
  221. return 0;
  222. if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
  223. tracepoint_printk = 1;
  224. return 1;
  225. }
  226. __setup("tp_printk", set_tracepoint_printk);
  227. unsigned long long ns2usecs(u64 nsec)
  228. {
  229. nsec += 500;
  230. do_div(nsec, 1000);
  231. return nsec;
  232. }
  233. static void
  234. trace_process_export(struct trace_export *export,
  235. struct ring_buffer_event *event, int flag)
  236. {
  237. struct trace_entry *entry;
  238. unsigned int size = 0;
  239. if (export->flags & flag) {
  240. entry = ring_buffer_event_data(event);
  241. size = ring_buffer_event_length(event);
  242. export->write(export, entry, size);
  243. }
  244. }
  245. static DEFINE_MUTEX(ftrace_export_lock);
  246. static struct trace_export __rcu *ftrace_exports_list __read_mostly;
  247. static DEFINE_STATIC_KEY_FALSE(trace_function_exports_enabled);
  248. static DEFINE_STATIC_KEY_FALSE(trace_event_exports_enabled);
  249. static DEFINE_STATIC_KEY_FALSE(trace_marker_exports_enabled);
  250. static inline void ftrace_exports_enable(struct trace_export *export)
  251. {
  252. if (export->flags & TRACE_EXPORT_FUNCTION)
  253. static_branch_inc(&trace_function_exports_enabled);
  254. if (export->flags & TRACE_EXPORT_EVENT)
  255. static_branch_inc(&trace_event_exports_enabled);
  256. if (export->flags & TRACE_EXPORT_MARKER)
  257. static_branch_inc(&trace_marker_exports_enabled);
  258. }
  259. static inline void ftrace_exports_disable(struct trace_export *export)
  260. {
  261. if (export->flags & TRACE_EXPORT_FUNCTION)
  262. static_branch_dec(&trace_function_exports_enabled);
  263. if (export->flags & TRACE_EXPORT_EVENT)
  264. static_branch_dec(&trace_event_exports_enabled);
  265. if (export->flags & TRACE_EXPORT_MARKER)
  266. static_branch_dec(&trace_marker_exports_enabled);
  267. }
  268. static void ftrace_exports(struct ring_buffer_event *event, int flag)
  269. {
  270. struct trace_export *export;
  271. preempt_disable_notrace();
  272. export = rcu_dereference_raw_check(ftrace_exports_list);
  273. while (export) {
  274. trace_process_export(export, event, flag);
  275. export = rcu_dereference_raw_check(export->next);
  276. }
  277. preempt_enable_notrace();
  278. }
  279. static inline void
  280. add_trace_export(struct trace_export **list, struct trace_export *export)
  281. {
  282. rcu_assign_pointer(export->next, *list);
  283. /*
  284. * We are entering export into the list but another
  285. * CPU might be walking that list. We need to make sure
  286. * the export->next pointer is valid before another CPU sees
  287. * the export pointer included into the list.
  288. */
  289. rcu_assign_pointer(*list, export);
  290. }
  291. static inline int
  292. rm_trace_export(struct trace_export **list, struct trace_export *export)
  293. {
  294. struct trace_export **p;
  295. for (p = list; *p != NULL; p = &(*p)->next)
  296. if (*p == export)
  297. break;
  298. if (*p != export)
  299. return -1;
  300. rcu_assign_pointer(*p, (*p)->next);
  301. return 0;
  302. }
  303. static inline void
  304. add_ftrace_export(struct trace_export **list, struct trace_export *export)
  305. {
  306. ftrace_exports_enable(export);
  307. add_trace_export(list, export);
  308. }
  309. static inline int
  310. rm_ftrace_export(struct trace_export **list, struct trace_export *export)
  311. {
  312. int ret;
  313. ret = rm_trace_export(list, export);
  314. ftrace_exports_disable(export);
  315. return ret;
  316. }
  317. int register_ftrace_export(struct trace_export *export)
  318. {
  319. if (WARN_ON_ONCE(!export->write))
  320. return -1;
  321. mutex_lock(&ftrace_export_lock);
  322. add_ftrace_export(&ftrace_exports_list, export);
  323. mutex_unlock(&ftrace_export_lock);
  324. return 0;
  325. }
  326. EXPORT_SYMBOL_GPL(register_ftrace_export);
  327. int unregister_ftrace_export(struct trace_export *export)
  328. {
  329. int ret;
  330. mutex_lock(&ftrace_export_lock);
  331. ret = rm_ftrace_export(&ftrace_exports_list, export);
  332. mutex_unlock(&ftrace_export_lock);
  333. return ret;
  334. }
  335. EXPORT_SYMBOL_GPL(unregister_ftrace_export);
  336. /* trace_flags holds trace_options default values */
  337. #define TRACE_DEFAULT_FLAGS \
  338. (FUNCTION_DEFAULT_FLAGS | \
  339. TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \
  340. TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \
  341. TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \
  342. TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS)
  343. /* trace_options that are only supported by global_trace */
  344. #define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \
  345. TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)
  346. /* trace_flags that are default zero for instances */
  347. #define ZEROED_TRACE_FLAGS \
  348. (TRACE_ITER_EVENT_FORK | TRACE_ITER_FUNC_FORK)
  349. /*
  350. * The global_trace is the descriptor that holds the top-level tracing
  351. * buffers for the live tracing.
  352. */
  353. static struct trace_array global_trace = {
  354. .trace_flags = TRACE_DEFAULT_FLAGS,
  355. };
  356. LIST_HEAD(ftrace_trace_arrays);
  357. int trace_array_get(struct trace_array *this_tr)
  358. {
  359. struct trace_array *tr;
  360. int ret = -ENODEV;
  361. mutex_lock(&trace_types_lock);
  362. list_for_each_entry(tr, &ftrace_trace_arrays, list) {
  363. if (tr == this_tr) {
  364. tr->ref++;
  365. ret = 0;
  366. break;
  367. }
  368. }
  369. mutex_unlock(&trace_types_lock);
  370. return ret;
  371. }
  372. static void __trace_array_put(struct trace_array *this_tr)
  373. {
  374. WARN_ON(!this_tr->ref);
  375. this_tr->ref--;
  376. }
  377. /**
  378. * trace_array_put - Decrement the reference counter for this trace array.
  379. *
  380. * NOTE: Use this when we no longer need the trace array returned by
  381. * trace_array_get_by_name(). This ensures the trace array can be later
  382. * destroyed.
  383. *
  384. */
  385. void trace_array_put(struct trace_array *this_tr)
  386. {
  387. if (!this_tr)
  388. return;
  389. mutex_lock(&trace_types_lock);
  390. __trace_array_put(this_tr);
  391. mutex_unlock(&trace_types_lock);
  392. }
  393. EXPORT_SYMBOL_GPL(trace_array_put);
  394. int tracing_check_open_get_tr(struct trace_array *tr)
  395. {
  396. int ret;
  397. ret = security_locked_down(LOCKDOWN_TRACEFS);
  398. if (ret)
  399. return ret;
  400. if (tracing_disabled)
  401. return -ENODEV;
  402. if (tr && trace_array_get(tr) < 0)
  403. return -ENODEV;
  404. return 0;
  405. }
  406. int call_filter_check_discard(struct trace_event_call *call, void *rec,
  407. struct trace_buffer *buffer,
  408. struct ring_buffer_event *event)
  409. {
  410. if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
  411. !filter_match_preds(call->filter, rec)) {
  412. __trace_event_discard_commit(buffer, event);
  413. return 1;
  414. }
  415. return 0;
  416. }
  417. void trace_free_pid_list(struct trace_pid_list *pid_list)
  418. {
  419. vfree(pid_list->pids);
  420. kfree(pid_list);
  421. }
  422. /**
  423. * trace_find_filtered_pid - check if a pid exists in a filtered_pid list
  424. * @filtered_pids: The list of pids to check
  425. * @search_pid: The PID to find in @filtered_pids
  426. *
  427. * Returns true if @search_pid is fonud in @filtered_pids, and false otherwis.
  428. */
  429. bool
  430. trace_find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid)
  431. {
  432. /*
  433. * If pid_max changed after filtered_pids was created, we
  434. * by default ignore all pids greater than the previous pid_max.
  435. */
  436. if (search_pid >= filtered_pids->pid_max)
  437. return false;
  438. return test_bit(search_pid, filtered_pids->pids);
  439. }
  440. /**
  441. * trace_ignore_this_task - should a task be ignored for tracing
  442. * @filtered_pids: The list of pids to check
  443. * @task: The task that should be ignored if not filtered
  444. *
  445. * Checks if @task should be traced or not from @filtered_pids.
  446. * Returns true if @task should *NOT* be traced.
  447. * Returns false if @task should be traced.
  448. */
  449. bool
  450. trace_ignore_this_task(struct trace_pid_list *filtered_pids,
  451. struct trace_pid_list *filtered_no_pids,
  452. struct task_struct *task)
  453. {
  454. /*
  455. * If filterd_no_pids is not empty, and the task's pid is listed
  456. * in filtered_no_pids, then return true.
  457. * Otherwise, if filtered_pids is empty, that means we can
  458. * trace all tasks. If it has content, then only trace pids
  459. * within filtered_pids.
  460. */
  461. return (filtered_pids &&
  462. !trace_find_filtered_pid(filtered_pids, task->pid)) ||
  463. (filtered_no_pids &&
  464. trace_find_filtered_pid(filtered_no_pids, task->pid));
  465. }
  466. /**
  467. * trace_filter_add_remove_task - Add or remove a task from a pid_list
  468. * @pid_list: The list to modify
  469. * @self: The current task for fork or NULL for exit
  470. * @task: The task to add or remove
  471. *
  472. * If adding a task, if @self is defined, the task is only added if @self
  473. * is also included in @pid_list. This happens on fork and tasks should
  474. * only be added when the parent is listed. If @self is NULL, then the
  475. * @task pid will be removed from the list, which would happen on exit
  476. * of a task.
  477. */
  478. void trace_filter_add_remove_task(struct trace_pid_list *pid_list,
  479. struct task_struct *self,
  480. struct task_struct *task)
  481. {
  482. if (!pid_list)
  483. return;
  484. /* For forks, we only add if the forking task is listed */
  485. if (self) {
  486. if (!trace_find_filtered_pid(pid_list, self->pid))
  487. return;
  488. }
  489. /* Sorry, but we don't support pid_max changing after setting */
  490. if (task->pid >= pid_list->pid_max)
  491. return;
  492. /* "self" is set for forks, and NULL for exits */
  493. if (self)
  494. set_bit(task->pid, pid_list->pids);
  495. else
  496. clear_bit(task->pid, pid_list->pids);
  497. }
  498. /**
  499. * trace_pid_next - Used for seq_file to get to the next pid of a pid_list
  500. * @pid_list: The pid list to show
  501. * @v: The last pid that was shown (+1 the actual pid to let zero be displayed)
  502. * @pos: The position of the file
  503. *
  504. * This is used by the seq_file "next" operation to iterate the pids
  505. * listed in a trace_pid_list structure.
  506. *
  507. * Returns the pid+1 as we want to display pid of zero, but NULL would
  508. * stop the iteration.
  509. */
  510. void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos)
  511. {
  512. unsigned long pid = (unsigned long)v;
  513. (*pos)++;
  514. /* pid already is +1 of the actual prevous bit */
  515. pid = find_next_bit(pid_list->pids, pid_list->pid_max, pid);
  516. /* Return pid + 1 to allow zero to be represented */
  517. if (pid < pid_list->pid_max)
  518. return (void *)(pid + 1);
  519. return NULL;
  520. }
  521. /**
  522. * trace_pid_start - Used for seq_file to start reading pid lists
  523. * @pid_list: The pid list to show
  524. * @pos: The position of the file
  525. *
  526. * This is used by seq_file "start" operation to start the iteration
  527. * of listing pids.
  528. *
  529. * Returns the pid+1 as we want to display pid of zero, but NULL would
  530. * stop the iteration.
  531. */
  532. void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos)
  533. {
  534. unsigned long pid;
  535. loff_t l = 0;
  536. pid = find_first_bit(pid_list->pids, pid_list->pid_max);
  537. if (pid >= pid_list->pid_max)
  538. return NULL;
  539. /* Return pid + 1 so that zero can be the exit value */
  540. for (pid++; pid && l < *pos;
  541. pid = (unsigned long)trace_pid_next(pid_list, (void *)pid, &l))
  542. ;
  543. return (void *)pid;
  544. }
  545. /**
  546. * trace_pid_show - show the current pid in seq_file processing
  547. * @m: The seq_file structure to write into
  548. * @v: A void pointer of the pid (+1) value to display
  549. *
  550. * Can be directly used by seq_file operations to display the current
  551. * pid value.
  552. */
  553. int trace_pid_show(struct seq_file *m, void *v)
  554. {
  555. unsigned long pid = (unsigned long)v - 1;
  556. seq_printf(m, "%lu\n", pid);
  557. return 0;
  558. }
  559. /* 128 should be much more than enough */
  560. #define PID_BUF_SIZE 127
  561. int trace_pid_write(struct trace_pid_list *filtered_pids,
  562. struct trace_pid_list **new_pid_list,
  563. const char __user *ubuf, size_t cnt)
  564. {
  565. struct trace_pid_list *pid_list;
  566. struct trace_parser parser;
  567. unsigned long val;
  568. int nr_pids = 0;
  569. ssize_t read = 0;
  570. ssize_t ret = 0;
  571. loff_t pos;
  572. pid_t pid;
  573. if (trace_parser_get_init(&parser, PID_BUF_SIZE + 1))
  574. return -ENOMEM;
  575. /*
  576. * Always recreate a new array. The write is an all or nothing
  577. * operation. Always create a new array when adding new pids by
  578. * the user. If the operation fails, then the current list is
  579. * not modified.
  580. */
  581. pid_list = kmalloc(sizeof(*pid_list), GFP_KERNEL);
  582. if (!pid_list) {
  583. trace_parser_put(&parser);
  584. return -ENOMEM;
  585. }
  586. pid_list->pid_max = READ_ONCE(pid_max);
  587. /* Only truncating will shrink pid_max */
  588. if (filtered_pids && filtered_pids->pid_max > pid_list->pid_max)
  589. pid_list->pid_max = filtered_pids->pid_max;
  590. pid_list->pids = vzalloc((pid_list->pid_max + 7) >> 3);
  591. if (!pid_list->pids) {
  592. trace_parser_put(&parser);
  593. kfree(pid_list);
  594. return -ENOMEM;
  595. }
  596. if (filtered_pids) {
  597. /* copy the current bits to the new max */
  598. for_each_set_bit(pid, filtered_pids->pids,
  599. filtered_pids->pid_max) {
  600. set_bit(pid, pid_list->pids);
  601. nr_pids++;
  602. }
  603. }
  604. while (cnt > 0) {
  605. pos = 0;
  606. ret = trace_get_user(&parser, ubuf, cnt, &pos);
  607. if (ret < 0 || !trace_parser_loaded(&parser))
  608. break;
  609. read += ret;
  610. ubuf += ret;
  611. cnt -= ret;
  612. ret = -EINVAL;
  613. if (kstrtoul(parser.buffer, 0, &val))
  614. break;
  615. if (val >= pid_list->pid_max)
  616. break;
  617. pid = (pid_t)val;
  618. set_bit(pid, pid_list->pids);
  619. nr_pids++;
  620. trace_parser_clear(&parser);
  621. ret = 0;
  622. }
  623. trace_parser_put(&parser);
  624. if (ret < 0) {
  625. trace_free_pid_list(pid_list);
  626. return ret;
  627. }
  628. if (!nr_pids) {
  629. /* Cleared the list of pids */
  630. trace_free_pid_list(pid_list);
  631. read = ret;
  632. pid_list = NULL;
  633. }
  634. *new_pid_list = pid_list;
  635. return read;
  636. }
  637. static u64 buffer_ftrace_now(struct array_buffer *buf, int cpu)
  638. {
  639. u64 ts;
  640. /* Early boot up does not have a buffer yet */
  641. if (!buf->buffer)
  642. return trace_clock_local();
  643. ts = ring_buffer_time_stamp(buf->buffer, cpu);
  644. ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
  645. return ts;
  646. }
  647. u64 ftrace_now(int cpu)
  648. {
  649. return buffer_ftrace_now(&global_trace.array_buffer, cpu);
  650. }
  651. /**
  652. * tracing_is_enabled - Show if global_trace has been disabled
  653. *
  654. * Shows if the global trace has been enabled or not. It uses the
  655. * mirror flag "buffer_disabled" to be used in fast paths such as for
  656. * the irqsoff tracer. But it may be inaccurate due to races. If you
  657. * need to know the accurate state, use tracing_is_on() which is a little
  658. * slower, but accurate.
  659. */
  660. int tracing_is_enabled(void)
  661. {
  662. /*
  663. * For quick access (irqsoff uses this in fast path), just
  664. * return the mirror variable of the state of the ring buffer.
  665. * It's a little racy, but we don't really care.
  666. */
  667. smp_rmb();
  668. return !global_trace.buffer_disabled;
  669. }
  670. /*
  671. * trace_buf_size is the size in bytes that is allocated
  672. * for a buffer. Note, the number of bytes is always rounded
  673. * to page size.
  674. *
  675. * This number is purposely set to a low number of 16384.
  676. * If the dump on oops happens, it will be much appreciated
  677. * to not have to wait for all that output. Anyway this can be
  678. * boot time and run time configurable.
  679. */
  680. #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
  681. static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
  682. /* trace_types holds a link list of available tracers. */
  683. static struct tracer *trace_types __read_mostly;
  684. /*
  685. * trace_types_lock is used to protect the trace_types list.
  686. */
  687. DEFINE_MUTEX(trace_types_lock);
  688. /*
  689. * serialize the access of the ring buffer
  690. *
  691. * ring buffer serializes readers, but it is low level protection.
  692. * The validity of the events (which returns by ring_buffer_peek() ..etc)
  693. * are not protected by ring buffer.
  694. *
  695. * The content of events may become garbage if we allow other process consumes
  696. * these events concurrently:
  697. * A) the page of the consumed events may become a normal page
  698. * (not reader page) in ring buffer, and this page will be rewrited
  699. * by events producer.
  700. * B) The page of the consumed events may become a page for splice_read,
  701. * and this page will be returned to system.
  702. *
  703. * These primitives allow multi process access to different cpu ring buffer
  704. * concurrently.
  705. *
  706. * These primitives don't distinguish read-only and read-consume access.
  707. * Multi read-only access are also serialized.
  708. */
  709. #ifdef CONFIG_SMP
  710. static DECLARE_RWSEM(all_cpu_access_lock);
  711. static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
  712. static inline void trace_access_lock(int cpu)
  713. {
  714. if (cpu == RING_BUFFER_ALL_CPUS) {
  715. /* gain it for accessing the whole ring buffer. */
  716. down_write(&all_cpu_access_lock);
  717. } else {
  718. /* gain it for accessing a cpu ring buffer. */
  719. /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
  720. down_read(&all_cpu_access_lock);
  721. /* Secondly block other access to this @cpu ring buffer. */
  722. mutex_lock(&per_cpu(cpu_access_lock, cpu));
  723. }
  724. }
  725. static inline void trace_access_unlock(int cpu)
  726. {
  727. if (cpu == RING_BUFFER_ALL_CPUS) {
  728. up_write(&all_cpu_access_lock);
  729. } else {
  730. mutex_unlock(&per_cpu(cpu_access_lock, cpu));
  731. up_read(&all_cpu_access_lock);
  732. }
  733. }
  734. static inline void trace_access_lock_init(void)
  735. {
  736. int cpu;
  737. for_each_possible_cpu(cpu)
  738. mutex_init(&per_cpu(cpu_access_lock, cpu));
  739. }
  740. #else
  741. static DEFINE_MUTEX(access_lock);
  742. static inline void trace_access_lock(int cpu)
  743. {
  744. (void)cpu;
  745. mutex_lock(&access_lock);
  746. }
  747. static inline void trace_access_unlock(int cpu)
  748. {
  749. (void)cpu;
  750. mutex_unlock(&access_lock);
  751. }
  752. static inline void trace_access_lock_init(void)
  753. {
  754. }
  755. #endif
  756. #ifdef CONFIG_STACKTRACE
  757. static void __ftrace_trace_stack(struct trace_buffer *buffer,
  758. unsigned long flags,
  759. int skip, int pc, struct pt_regs *regs);
  760. static inline void ftrace_trace_stack(struct trace_array *tr,
  761. struct trace_buffer *buffer,
  762. unsigned long flags,
  763. int skip, int pc, struct pt_regs *regs);
  764. #else
  765. static inline void __ftrace_trace_stack(struct trace_buffer *buffer,
  766. unsigned long flags,
  767. int skip, int pc, struct pt_regs *regs)
  768. {
  769. }
  770. static inline void ftrace_trace_stack(struct trace_array *tr,
  771. struct trace_buffer *buffer,
  772. unsigned long flags,
  773. int skip, int pc, struct pt_regs *regs)
  774. {
  775. }
  776. #endif
  777. static __always_inline void
  778. trace_event_setup(struct ring_buffer_event *event,
  779. int type, unsigned long flags, int pc)
  780. {
  781. struct trace_entry *ent = ring_buffer_event_data(event);
  782. tracing_generic_entry_update(ent, type, flags, pc);
  783. }
  784. static __always_inline struct ring_buffer_event *
  785. __trace_buffer_lock_reserve(struct trace_buffer *buffer,
  786. int type,
  787. unsigned long len,
  788. unsigned long flags, int pc)
  789. {
  790. struct ring_buffer_event *event;
  791. event = ring_buffer_lock_reserve(buffer, len);
  792. if (event != NULL)
  793. trace_event_setup(event, type, flags, pc);
  794. return event;
  795. }
  796. void tracer_tracing_on(struct trace_array *tr)
  797. {
  798. if (tr->array_buffer.buffer)
  799. ring_buffer_record_on(tr->array_buffer.buffer);
  800. /*
  801. * This flag is looked at when buffers haven't been allocated
  802. * yet, or by some tracers (like irqsoff), that just want to
  803. * know if the ring buffer has been disabled, but it can handle
  804. * races of where it gets disabled but we still do a record.
  805. * As the check is in the fast path of the tracers, it is more
  806. * important to be fast than accurate.
  807. */
  808. tr->buffer_disabled = 0;
  809. /* Make the flag seen by readers */
  810. smp_wmb();
  811. }
  812. /**
  813. * tracing_on - enable tracing buffers
  814. *
  815. * This function enables tracing buffers that may have been
  816. * disabled with tracing_off.
  817. */
  818. void tracing_on(void)
  819. {
  820. tracer_tracing_on(&global_trace);
  821. }
  822. EXPORT_SYMBOL_GPL(tracing_on);
  823. static __always_inline void
  824. __buffer_unlock_commit(struct trace_buffer *buffer, struct ring_buffer_event *event)
  825. {
  826. __this_cpu_write(trace_taskinfo_save, true);
  827. /* If this is the temp buffer, we need to commit fully */
  828. if (this_cpu_read(trace_buffered_event) == event) {
  829. /* Length is in event->array[0] */
  830. ring_buffer_write(buffer, event->array[0], &event->array[1]);
  831. /* Release the temp buffer */
  832. this_cpu_dec(trace_buffered_event_cnt);
  833. } else
  834. ring_buffer_unlock_commit(buffer, event);
  835. }
  836. /**
  837. * __trace_puts - write a constant string into the trace buffer.
  838. * @ip: The address of the caller
  839. * @str: The constant string to write
  840. * @size: The size of the string.
  841. */
  842. int __trace_puts(unsigned long ip, const char *str, int size)
  843. {
  844. struct ring_buffer_event *event;
  845. struct trace_buffer *buffer;
  846. struct print_entry *entry;
  847. unsigned long irq_flags;
  848. int alloc;
  849. int pc;
  850. if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
  851. return 0;
  852. pc = preempt_count();
  853. if (unlikely(tracing_selftest_running || tracing_disabled))
  854. return 0;
  855. alloc = sizeof(*entry) + size + 2; /* possible \n added */
  856. local_save_flags(irq_flags);
  857. buffer = global_trace.array_buffer.buffer;
  858. ring_buffer_nest_start(buffer);
  859. event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
  860. irq_flags, pc);
  861. if (!event) {
  862. size = 0;
  863. goto out;
  864. }
  865. entry = ring_buffer_event_data(event);
  866. entry->ip = ip;
  867. memcpy(&entry->buf, str, size);
  868. /* Add a newline if necessary */
  869. if (entry->buf[size - 1] != '\n') {
  870. entry->buf[size] = '\n';
  871. entry->buf[size + 1] = '\0';
  872. } else
  873. entry->buf[size] = '\0';
  874. __buffer_unlock_commit(buffer, event);
  875. ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
  876. out:
  877. ring_buffer_nest_end(buffer);
  878. return size;
  879. }
  880. EXPORT_SYMBOL_GPL(__trace_puts);
  881. /**
  882. * __trace_bputs - write the pointer to a constant string into trace buffer
  883. * @ip: The address of the caller
  884. * @str: The constant string to write to the buffer to
  885. */
  886. int __trace_bputs(unsigned long ip, const char *str)
  887. {
  888. struct ring_buffer_event *event;
  889. struct trace_buffer *buffer;
  890. struct bputs_entry *entry;
  891. unsigned long irq_flags;
  892. int size = sizeof(struct bputs_entry);
  893. int ret = 0;
  894. int pc;
  895. if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
  896. return 0;
  897. pc = preempt_count();
  898. if (unlikely(tracing_selftest_running || tracing_disabled))
  899. return 0;
  900. local_save_flags(irq_flags);
  901. buffer = global_trace.array_buffer.buffer;
  902. ring_buffer_nest_start(buffer);
  903. event = __trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
  904. irq_flags, pc);
  905. if (!event)
  906. goto out;
  907. entry = ring_buffer_event_data(event);
  908. entry->ip = ip;
  909. entry->str = str;
  910. __buffer_unlock_commit(buffer, event);
  911. ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
  912. ret = 1;
  913. out:
  914. ring_buffer_nest_end(buffer);
  915. return ret;
  916. }
  917. EXPORT_SYMBOL_GPL(__trace_bputs);
  918. #ifdef CONFIG_TRACER_SNAPSHOT
  919. static void tracing_snapshot_instance_cond(struct trace_array *tr,
  920. void *cond_data)
  921. {
  922. struct tracer *tracer = tr->current_trace;
  923. unsigned long flags;
  924. if (in_nmi()) {
  925. internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
  926. internal_trace_puts("*** snapshot is being ignored ***\n");
  927. return;
  928. }
  929. if (!tr->allocated_snapshot) {
  930. internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
  931. internal_trace_puts("*** stopping trace here! ***\n");
  932. tracing_off();
  933. return;
  934. }
  935. /* Note, snapshot can not be used when the tracer uses it */
  936. if (tracer->use_max_tr) {
  937. internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
  938. internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
  939. return;
  940. }
  941. local_irq_save(flags);
  942. update_max_tr(tr, current, smp_processor_id(), cond_data);
  943. local_irq_restore(flags);
  944. }
  945. void tracing_snapshot_instance(struct trace_array *tr)
  946. {
  947. tracing_snapshot_instance_cond(tr, NULL);
  948. }
  949. /**
  950. * tracing_snapshot - take a snapshot of the current buffer.
  951. *
  952. * This causes a swap between the snapshot buffer and the current live
  953. * tracing buffer. You can use this to take snapshots of the live
  954. * trace when some condition is triggered, but continue to trace.
  955. *
  956. * Note, make sure to allocate the snapshot with either
  957. * a tracing_snapshot_alloc(), or by doing it manually
  958. * with: echo 1 > /sys/kernel/debug/tracing/snapshot
  959. *
  960. * If the snapshot buffer is not allocated, it will stop tracing.
  961. * Basically making a permanent snapshot.
  962. */
  963. void tracing_snapshot(void)
  964. {
  965. struct trace_array *tr = &global_trace;
  966. tracing_snapshot_instance(tr);
  967. }
  968. EXPORT_SYMBOL_GPL(tracing_snapshot);
  969. /**
  970. * tracing_snapshot_cond - conditionally take a snapshot of the current buffer.
  971. * @tr: The tracing instance to snapshot
  972. * @cond_data: The data to be tested conditionally, and possibly saved
  973. *
  974. * This is the same as tracing_snapshot() except that the snapshot is
  975. * conditional - the snapshot will only happen if the
  976. * cond_snapshot.update() implementation receiving the cond_data
  977. * returns true, which means that the trace array's cond_snapshot
  978. * update() operation used the cond_data to determine whether the
  979. * snapshot should be taken, and if it was, presumably saved it along
  980. * with the snapshot.
  981. */
  982. void tracing_snapshot_cond(struct trace_array *tr, void *cond_data)
  983. {
  984. tracing_snapshot_instance_cond(tr, cond_data);
  985. }
  986. EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
  987. /**
  988. * tracing_snapshot_cond_data - get the user data associated with a snapshot
  989. * @tr: The tracing instance
  990. *
  991. * When the user enables a conditional snapshot using
  992. * tracing_snapshot_cond_enable(), the user-defined cond_data is saved
  993. * with the snapshot. This accessor is used to retrieve it.
  994. *
  995. * Should not be called from cond_snapshot.update(), since it takes
  996. * the tr->max_lock lock, which the code calling
  997. * cond_snapshot.update() has already done.
  998. *
  999. * Returns the cond_data associated with the trace array's snapshot.
  1000. */
  1001. void *tracing_cond_snapshot_data(struct trace_array *tr)
  1002. {
  1003. void *cond_data = NULL;
  1004. arch_spin_lock(&tr->max_lock);
  1005. if (tr->cond_snapshot)
  1006. cond_data = tr->cond_snapshot->cond_data;
  1007. arch_spin_unlock(&tr->max_lock);
  1008. return cond_data;
  1009. }
  1010. EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
  1011. static int resize_buffer_duplicate_size(struct array_buffer *trace_buf,
  1012. struct array_buffer *size_buf, int cpu_id);
  1013. static void set_buffer_entries(struct array_buffer *buf, unsigned long val);
  1014. int tracing_alloc_snapshot_instance(struct trace_array *tr)
  1015. {
  1016. int ret;
  1017. if (!tr->allocated_snapshot) {
  1018. /* allocate spare buffer */
  1019. ret = resize_buffer_duplicate_size(&tr->max_buffer,
  1020. &tr->array_buffer, RING_BUFFER_ALL_CPUS);
  1021. if (ret < 0)
  1022. return ret;
  1023. tr->allocated_snapshot = true;
  1024. }
  1025. return 0;
  1026. }
  1027. static void free_snapshot(struct trace_array *tr)
  1028. {
  1029. /*
  1030. * We don't free the ring buffer. instead, resize it because
  1031. * The max_tr ring buffer has some state (e.g. ring->clock) and
  1032. * we want preserve it.
  1033. */
  1034. ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
  1035. set_buffer_entries(&tr->max_buffer, 1);
  1036. tracing_reset_online_cpus(&tr->max_buffer);
  1037. tr->allocated_snapshot = false;
  1038. }
  1039. /**
  1040. * tracing_alloc_snapshot - allocate snapshot buffer.
  1041. *
  1042. * This only allocates the snapshot buffer if it isn't already
  1043. * allocated - it doesn't also take a snapshot.
  1044. *
  1045. * This is meant to be used in cases where the snapshot buffer needs
  1046. * to be set up for events that can't sleep but need to be able to
  1047. * trigger a snapshot.
  1048. */
  1049. int tracing_alloc_snapshot(void)
  1050. {
  1051. struct trace_array *tr = &global_trace;
  1052. int ret;
  1053. ret = tracing_alloc_snapshot_instance(tr);
  1054. WARN_ON(ret < 0);
  1055. return ret;
  1056. }
  1057. EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
  1058. /**
  1059. * tracing_snapshot_alloc - allocate and take a snapshot of the current buffer.
  1060. *
  1061. * This is similar to tracing_snapshot(), but it will allocate the
  1062. * snapshot buffer if it isn't already allocated. Use this only
  1063. * where it is safe to sleep, as the allocation may sleep.
  1064. *
  1065. * This causes a swap between the snapshot buffer and the current live
  1066. * tracing buffer. You can use this to take snapshots of the live
  1067. * trace when some condition is triggered, but continue to trace.
  1068. */
  1069. void tracing_snapshot_alloc(void)
  1070. {
  1071. int ret;
  1072. ret = tracing_alloc_snapshot();
  1073. if (ret < 0)
  1074. return;
  1075. tracing_snapshot();
  1076. }
  1077. EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
  1078. /**
  1079. * tracing_snapshot_cond_enable - enable conditional snapshot for an instance
  1080. * @tr: The tracing instance
  1081. * @cond_data: User data to associate with the snapshot
  1082. * @update: Implementation of the cond_snapshot update function
  1083. *
  1084. * Check whether the conditional snapshot for the given instance has
  1085. * already been enabled, or if the current tracer is already using a
  1086. * snapshot; if so, return -EBUSY, else create a cond_snapshot and
  1087. * save the cond_data and update function inside.
  1088. *
  1089. * Returns 0 if successful, error otherwise.
  1090. */
  1091. int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data,
  1092. cond_update_fn_t update)
  1093. {
  1094. struct cond_snapshot *cond_snapshot;
  1095. int ret = 0;
  1096. cond_snapshot = kzalloc(sizeof(*cond_snapshot), GFP_KERNEL);
  1097. if (!cond_snapshot)
  1098. return -ENOMEM;
  1099. cond_snapshot->cond_data = cond_data;
  1100. cond_snapshot->update = update;
  1101. mutex_lock(&trace_types_lock);
  1102. ret = tracing_alloc_snapshot_instance(tr);
  1103. if (ret)
  1104. goto fail_unlock;
  1105. if (tr->current_trace->use_max_tr) {
  1106. ret = -EBUSY;
  1107. goto fail_unlock;
  1108. }
  1109. /*
  1110. * The cond_snapshot can only change to NULL without the
  1111. * trace_types_lock. We don't care if we race with it going
  1112. * to NULL, but we want to make sure that it's not set to
  1113. * something other than NULL when we get here, which we can
  1114. * do safely with only holding the trace_types_lock and not
  1115. * having to take the max_lock.
  1116. */
  1117. if (tr->cond_snapshot) {
  1118. ret = -EBUSY;
  1119. goto fail_unlock;
  1120. }
  1121. arch_spin_lock(&tr->max_lock);
  1122. tr->cond_snapshot = cond_snapshot;
  1123. arch_spin_unlock(&tr->max_lock);
  1124. mutex_unlock(&trace_types_lock);
  1125. return ret;
  1126. fail_unlock:
  1127. mutex_unlock(&trace_types_lock);
  1128. kfree(cond_snapshot);
  1129. return ret;
  1130. }
  1131. EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
  1132. /**
  1133. * tracing_snapshot_cond_disable - disable conditional snapshot for an instance
  1134. * @tr: The tracing instance
  1135. *
  1136. * Check whether the conditional snapshot for the given instance is
  1137. * enabled; if so, free the cond_snapshot associated with it,
  1138. * otherwise return -EINVAL.
  1139. *
  1140. * Returns 0 if successful, error otherwise.
  1141. */
  1142. int tracing_snapshot_cond_disable(struct trace_array *tr)
  1143. {
  1144. int ret = 0;
  1145. arch_spin_lock(&tr->max_lock);
  1146. if (!tr->cond_snapshot)
  1147. ret = -EINVAL;
  1148. else {
  1149. kfree(tr->cond_snapshot);
  1150. tr->cond_snapshot = NULL;
  1151. }
  1152. arch_spin_unlock(&tr->max_lock);
  1153. return ret;
  1154. }
  1155. EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
  1156. #else
  1157. void tracing_snapshot(void)
  1158. {
  1159. WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
  1160. }
  1161. EXPORT_SYMBOL_GPL(tracing_snapshot);
  1162. void tracing_snapshot_cond(struct trace_array *tr, void *cond_data)
  1163. {
  1164. WARN_ONCE(1, "Snapshot feature not enabled, but internal conditional snapshot used");
  1165. }
  1166. EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
  1167. int tracing_alloc_snapshot(void)
  1168. {
  1169. WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
  1170. return -ENODEV;
  1171. }
  1172. EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
  1173. void tracing_snapshot_alloc(void)
  1174. {
  1175. /* Give warning */
  1176. tracing_snapshot();
  1177. }
  1178. EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
  1179. void *tracing_cond_snapshot_data(struct trace_array *tr)
  1180. {
  1181. return NULL;
  1182. }
  1183. EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
  1184. int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, cond_update_fn_t update)
  1185. {
  1186. return -ENODEV;
  1187. }
  1188. EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
  1189. int tracing_snapshot_cond_disable(struct trace_array *tr)
  1190. {
  1191. return false;
  1192. }
  1193. EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
  1194. #endif /* CONFIG_TRACER_SNAPSHOT */
  1195. void tracer_tracing_off(struct trace_array *tr)
  1196. {
  1197. if (tr->array_buffer.buffer)
  1198. ring_buffer_record_off(tr->array_buffer.buffer);
  1199. /*
  1200. * This flag is looked at when buffers haven't been allocated
  1201. * yet, or by some tracers (like irqsoff), that just want to
  1202. * know if the ring buffer has been disabled, but it can handle
  1203. * races of where it gets disabled but we still do a record.
  1204. * As the check is in the fast path of the tracers, it is more
  1205. * important to be fast than accurate.
  1206. */
  1207. tr->buffer_disabled = 1;
  1208. /* Make the flag seen by readers */
  1209. smp_wmb();
  1210. }
  1211. /**
  1212. * tracing_off - turn off tracing buffers
  1213. *
  1214. * This function stops the tracing buffers from recording data.
  1215. * It does not disable any overhead the tracers themselves may
  1216. * be causing. This function simply causes all recording to
  1217. * the ring buffers to fail.
  1218. */
  1219. void tracing_off(void)
  1220. {
  1221. tracer_tracing_off(&global_trace);
  1222. }
  1223. EXPORT_SYMBOL_GPL(tracing_off);
  1224. void disable_trace_on_warning(void)
  1225. {
  1226. if (__disable_trace_on_warning) {
  1227. trace_array_printk_buf(global_trace.array_buffer.buffer, _THIS_IP_,
  1228. "Disabling tracing due to warning\n");
  1229. tracing_off();
  1230. }
  1231. }
  1232. /**
  1233. * tracer_tracing_is_on - show real state of ring buffer enabled
  1234. * @tr : the trace array to know if ring buffer is enabled
  1235. *
  1236. * Shows real state of the ring buffer if it is enabled or not.
  1237. */
  1238. bool tracer_tracing_is_on(struct trace_array *tr)
  1239. {
  1240. if (tr->array_buffer.buffer)
  1241. return ring_buffer_record_is_on(tr->array_buffer.buffer);
  1242. return !tr->buffer_disabled;
  1243. }
  1244. /**
  1245. * tracing_is_on - show state of ring buffers enabled
  1246. */
  1247. int tracing_is_on(void)
  1248. {
  1249. return tracer_tracing_is_on(&global_trace);
  1250. }
  1251. EXPORT_SYMBOL_GPL(tracing_is_on);
  1252. static int __init set_buf_size(char *str)
  1253. {
  1254. unsigned long buf_size;
  1255. if (!str)
  1256. return 0;
  1257. buf_size = memparse(str, &str);
  1258. /*
  1259. * nr_entries can not be zero and the startup
  1260. * tests require some buffer space. Therefore
  1261. * ensure we have at least 4096 bytes of buffer.
  1262. */
  1263. trace_buf_size = max(4096UL, buf_size);
  1264. return 1;
  1265. }
  1266. __setup("trace_buf_size=", set_buf_size);
  1267. static int __init set_tracing_thresh(char *str)
  1268. {
  1269. unsigned long threshold;
  1270. int ret;
  1271. if (!str)
  1272. return 0;
  1273. ret = kstrtoul(str, 0, &threshold);
  1274. if (ret < 0)
  1275. return 0;
  1276. tracing_thresh = threshold * 1000;
  1277. return 1;
  1278. }
  1279. __setup("tracing_thresh=", set_tracing_thresh);
  1280. unsigned long nsecs_to_usecs(unsigned long nsecs)
  1281. {
  1282. return nsecs / 1000;
  1283. }
  1284. /*
  1285. * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
  1286. * It uses C(a, b) where 'a' is the eval (enum) name and 'b' is the string that
  1287. * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
  1288. * of strings in the order that the evals (enum) were defined.
  1289. */
  1290. #undef C
  1291. #define C(a, b) b
  1292. /* These must match the bit postions in trace_iterator_flags */
  1293. static const char *trace_options[] = {
  1294. TRACE_FLAGS
  1295. NULL
  1296. };
  1297. static struct {
  1298. u64 (*func)(void);
  1299. const char *name;
  1300. int in_ns; /* is this clock in nanoseconds? */
  1301. } trace_clocks[] = {
  1302. { trace_clock_local, "local", 1 },
  1303. { trace_clock_global, "global", 1 },
  1304. { trace_clock_counter, "counter", 0 },
  1305. { trace_clock_jiffies, "uptime", 0 },
  1306. { trace_clock, "perf", 1 },
  1307. { ktime_get_mono_fast_ns, "mono", 1 },
  1308. { ktime_get_raw_fast_ns, "mono_raw", 1 },
  1309. { ktime_get_boot_fast_ns, "boot", 1 },
  1310. ARCH_TRACE_CLOCKS
  1311. };
  1312. bool trace_clock_in_ns(struct trace_array *tr)
  1313. {
  1314. if (trace_clocks[tr->clock_id].in_ns)
  1315. return true;
  1316. return false;
  1317. }
  1318. /*
  1319. * trace_parser_get_init - gets the buffer for trace parser
  1320. */
  1321. int trace_parser_get_init(struct trace_parser *parser, int size)
  1322. {
  1323. memset(parser, 0, sizeof(*parser));
  1324. parser->buffer = kmalloc(size, GFP_KERNEL);
  1325. if (!parser->buffer)
  1326. return 1;
  1327. parser->size = size;
  1328. return 0;
  1329. }
  1330. /*
  1331. * trace_parser_put - frees the buffer for trace parser
  1332. */
  1333. void trace_parser_put(struct trace_parser *parser)
  1334. {
  1335. kfree(parser->buffer);
  1336. parser->buffer = NULL;
  1337. }
  1338. /*
  1339. * trace_get_user - reads the user input string separated by space
  1340. * (matched by isspace(ch))
  1341. *
  1342. * For each string found the 'struct trace_parser' is updated,
  1343. * and the function returns.
  1344. *
  1345. * Returns number of bytes read.
  1346. *
  1347. * See kernel/trace/trace.h for 'struct trace_parser' details.
  1348. */
  1349. int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
  1350. size_t cnt, loff_t *ppos)
  1351. {
  1352. char ch;
  1353. size_t read = 0;
  1354. ssize_t ret;
  1355. if (!*ppos)
  1356. trace_parser_clear(parser);
  1357. ret = get_user(ch, ubuf++);
  1358. if (ret)
  1359. goto out;
  1360. read++;
  1361. cnt--;
  1362. /*
  1363. * The parser is not finished with the last write,
  1364. * continue reading the user input without skipping spaces.
  1365. */
  1366. if (!parser->cont) {
  1367. /* skip white space */
  1368. while (cnt && isspace(ch)) {
  1369. ret = get_user(ch, ubuf++);
  1370. if (ret)
  1371. goto out;
  1372. read++;
  1373. cnt--;
  1374. }
  1375. parser->idx = 0;
  1376. /* only spaces were written */
  1377. if (isspace(ch) || !ch) {
  1378. *ppos += read;
  1379. ret = read;
  1380. goto out;
  1381. }
  1382. }
  1383. /* read the non-space input */
  1384. while (cnt && !isspace(ch) && ch) {
  1385. if (parser->idx < parser->size - 1)
  1386. parser->buffer[parser->idx++] = ch;
  1387. else {
  1388. ret = -EINVAL;
  1389. goto out;
  1390. }
  1391. ret = get_user(ch, ubuf++);
  1392. if (ret)
  1393. goto out;
  1394. read++;
  1395. cnt--;
  1396. }
  1397. /* We either got finished input or we have to wait for another call. */
  1398. if (isspace(ch) || !ch) {
  1399. parser->buffer[parser->idx] = 0;
  1400. parser->cont = false;
  1401. } else if (parser->idx < parser->size - 1) {
  1402. parser->cont = true;
  1403. parser->buffer[parser->idx++] = ch;
  1404. /* Make sure the parsed string always terminates with '\0'. */
  1405. parser->buffer[parser->idx] = 0;
  1406. } else {
  1407. ret = -EINVAL;
  1408. goto out;
  1409. }
  1410. *ppos += read;
  1411. ret = read;
  1412. out:
  1413. return ret;
  1414. }
  1415. /* TODO add a seq_buf_to_buffer() */
  1416. static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
  1417. {
  1418. int len;
  1419. if (trace_seq_used(s) <= s->seq.readpos)
  1420. return -EBUSY;
  1421. len = trace_seq_used(s) - s->seq.readpos;
  1422. if (cnt > len)
  1423. cnt = len;
  1424. memcpy(buf, s->buffer + s->seq.readpos, cnt);
  1425. s->seq.readpos += cnt;
  1426. return cnt;
  1427. }
  1428. unsigned long __read_mostly tracing_thresh;
  1429. static const struct file_operations tracing_max_lat_fops;
  1430. #if (defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)) && \
  1431. defined(CONFIG_FSNOTIFY)
  1432. static struct workqueue_struct *fsnotify_wq;
  1433. static void latency_fsnotify_workfn(struct work_struct *work)
  1434. {
  1435. struct trace_array *tr = container_of(work, struct trace_array,
  1436. fsnotify_work);
  1437. fsnotify_inode(tr->d_max_latency->d_inode, FS_MODIFY);
  1438. }
  1439. static void latency_fsnotify_workfn_irq(struct irq_work *iwork)
  1440. {
  1441. struct trace_array *tr = container_of(iwork, struct trace_array,
  1442. fsnotify_irqwork);
  1443. queue_work(fsnotify_wq, &tr->fsnotify_work);
  1444. }
  1445. static void trace_create_maxlat_file(struct trace_array *tr,
  1446. struct dentry *d_tracer)
  1447. {
  1448. INIT_WORK(&tr->fsnotify_work, latency_fsnotify_workfn);
  1449. init_irq_work(&tr->fsnotify_irqwork, latency_fsnotify_workfn_irq);
  1450. tr->d_max_latency = trace_create_file("tracing_max_latency", 0644,
  1451. d_tracer, &tr->max_latency,
  1452. &tracing_max_lat_fops);
  1453. }
  1454. __init static int latency_fsnotify_init(void)
  1455. {
  1456. fsnotify_wq = alloc_workqueue("tr_max_lat_wq",
  1457. WQ_UNBOUND | WQ_HIGHPRI, 0);
  1458. if (!fsnotify_wq) {
  1459. pr_err("Unable to allocate tr_max_lat_wq\n");
  1460. return -ENOMEM;
  1461. }
  1462. return 0;
  1463. }
  1464. late_initcall_sync(latency_fsnotify_init);
  1465. void latency_fsnotify(struct trace_array *tr)
  1466. {
  1467. if (!fsnotify_wq)
  1468. return;
  1469. /*
  1470. * We cannot call queue_work(&tr->fsnotify_work) from here because it's
  1471. * possible that we are called from __schedule() or do_idle(), which
  1472. * could cause a deadlock.
  1473. */
  1474. irq_work_queue(&tr->fsnotify_irqwork);
  1475. }
  1476. /*
  1477. * (defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)) && \
  1478. * defined(CONFIG_FSNOTIFY)
  1479. */
  1480. #else
  1481. #define trace_create_maxlat_file(tr, d_tracer) \
  1482. trace_create_file("tracing_max_latency", 0644, d_tracer, \
  1483. &tr->max_latency, &tracing_max_lat_fops)
  1484. #endif
  1485. #ifdef CONFIG_TRACER_MAX_TRACE
  1486. /*
  1487. * Copy the new maximum trace into the separate maximum-trace
  1488. * structure. (this way the maximum trace is permanently saved,
  1489. * for later retrieval via /sys/kernel/tracing/tracing_max_latency)
  1490. */
  1491. static void
  1492. __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
  1493. {
  1494. struct array_buffer *trace_buf = &tr->array_buffer;
  1495. struct array_buffer *max_buf = &tr->max_buffer;
  1496. struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
  1497. struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
  1498. max_buf->cpu = cpu;
  1499. max_buf->time_start = data->preempt_timestamp;
  1500. max_data->saved_latency = tr->max_latency;
  1501. max_data->critical_start = data->critical_start;
  1502. max_data->critical_end = data->critical_end;
  1503. strncpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
  1504. max_data->pid = tsk->pid;
  1505. /*
  1506. * If tsk == current, then use current_uid(), as that does not use
  1507. * RCU. The irq tracer can be called out of RCU scope.
  1508. */
  1509. if (tsk == current)
  1510. max_data->uid = current_uid();
  1511. else
  1512. max_data->uid = task_uid(tsk);
  1513. max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
  1514. max_data->policy = tsk->policy;
  1515. max_data->rt_priority = tsk->rt_priority;
  1516. /* record this tasks comm */
  1517. tracing_record_cmdline(tsk);
  1518. latency_fsnotify(tr);
  1519. }
  1520. /**
  1521. * update_max_tr - snapshot all trace buffers from global_trace to max_tr
  1522. * @tr: tracer
  1523. * @tsk: the task with the latency
  1524. * @cpu: The cpu that initiated the trace.
  1525. * @cond_data: User data associated with a conditional snapshot
  1526. *
  1527. * Flip the buffers between the @tr and the max_tr and record information
  1528. * about which task was the cause of this latency.
  1529. */
  1530. void
  1531. update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu,
  1532. void *cond_data)
  1533. {
  1534. if (tr->stop_count)
  1535. return;
  1536. WARN_ON_ONCE(!irqs_disabled());
  1537. if (!tr->allocated_snapshot) {
  1538. /* Only the nop tracer should hit this when disabling */
  1539. WARN_ON_ONCE(tr->current_trace != &nop_trace);
  1540. return;
  1541. }
  1542. arch_spin_lock(&tr->max_lock);
  1543. /* Inherit the recordable setting from array_buffer */
  1544. if (ring_buffer_record_is_set_on(tr->array_buffer.buffer))
  1545. ring_buffer_record_on(tr->max_buffer.buffer);
  1546. else
  1547. ring_buffer_record_off(tr->max_buffer.buffer);
  1548. #ifdef CONFIG_TRACER_SNAPSHOT
  1549. if (tr->cond_snapshot && !tr->cond_snapshot->update(tr, cond_data))
  1550. goto out_unlock;
  1551. #endif
  1552. swap(tr->array_buffer.buffer, tr->max_buffer.buffer);
  1553. __update_max_tr(tr, tsk, cpu);
  1554. out_unlock:
  1555. arch_spin_unlock(&tr->max_lock);
  1556. }
  1557. /**
  1558. * update_max_tr_single - only copy one trace over, and reset the rest
  1559. * @tr: tracer
  1560. * @tsk: task with the latency
  1561. * @cpu: the cpu of the buffer to copy.
  1562. *
  1563. * Flip the trace of a single CPU buffer between the @tr and the max_tr.
  1564. */
  1565. void
  1566. update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
  1567. {
  1568. int ret;
  1569. if (tr->stop_count)
  1570. return;
  1571. WARN_ON_ONCE(!irqs_disabled());
  1572. if (!tr->allocated_snapshot) {
  1573. /* Only the nop tracer should hit this when disabling */
  1574. WARN_ON_ONCE(tr->current_trace != &nop_trace);
  1575. return;
  1576. }
  1577. arch_spin_lock(&tr->max_lock);
  1578. ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->array_buffer.buffer, cpu);
  1579. if (ret == -EBUSY) {
  1580. /*
  1581. * We failed to swap the buffer due to a commit taking
  1582. * place on this CPU. We fail to record, but we reset
  1583. * the max trace buffer (no one writes directly to it)
  1584. * and flag that it failed.
  1585. */
  1586. trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
  1587. "Failed to swap buffers due to commit in progress\n");
  1588. }
  1589. WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
  1590. __update_max_tr(tr, tsk, cpu);
  1591. arch_spin_unlock(&tr->max_lock);
  1592. }
  1593. #endif /* CONFIG_TRACER_MAX_TRACE */
  1594. static int wait_on_pipe(struct trace_iterator *iter, int full)
  1595. {
  1596. /* Iterators are static, they should be filled or empty */
  1597. if (trace_buffer_iter(iter, iter->cpu_file))
  1598. return 0;
  1599. return ring_buffer_wait(iter->array_buffer->buffer, iter->cpu_file,
  1600. full);
  1601. }
  1602. #ifdef CONFIG_FTRACE_STARTUP_TEST
  1603. static bool selftests_can_run;
  1604. struct trace_selftests {
  1605. struct list_head list;
  1606. struct tracer *type;
  1607. };
  1608. static LIST_HEAD(postponed_selftests);
  1609. static int save_selftest(struct tracer *type)
  1610. {
  1611. struct trace_selftests *selftest;
  1612. selftest = kmalloc(sizeof(*selftest), GFP_KERNEL);
  1613. if (!selftest)
  1614. return -ENOMEM;
  1615. selftest->type = type;
  1616. list_add(&selftest->list, &postponed_selftests);
  1617. return 0;
  1618. }
  1619. static int run_tracer_selftest(struct tracer *type)
  1620. {
  1621. struct trace_array *tr = &global_trace;
  1622. struct tracer *saved_tracer = tr->current_trace;
  1623. int ret;
  1624. if (!type->selftest || tracing_selftest_disabled)
  1625. return 0;
  1626. /*
  1627. * If a tracer registers early in boot up (before scheduling is
  1628. * initialized and such), then do not run its selftests yet.
  1629. * Instead, run it a little later in the boot process.
  1630. */
  1631. if (!selftests_can_run)
  1632. return save_selftest(type);
  1633. /*
  1634. * Run a selftest on this tracer.
  1635. * Here we reset the trace buffer, and set the current
  1636. * tracer to be this tracer. The tracer can then run some
  1637. * internal tracing to verify that everything is in order.
  1638. * If we fail, we do not register this tracer.
  1639. */
  1640. tracing_reset_online_cpus(&tr->array_buffer);
  1641. tr->current_trace = type;
  1642. #ifdef CONFIG_TRACER_MAX_TRACE
  1643. if (type->use_max_tr) {
  1644. /* If we expanded the buffers, make sure the max is expanded too */
  1645. if (ring_buffer_expanded)
  1646. ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
  1647. RING_BUFFER_ALL_CPUS);
  1648. tr->allocated_snapshot = true;
  1649. }
  1650. #endif
  1651. /* the test is responsible for initializing and enabling */
  1652. pr_info("Testing tracer %s: ", type->name);
  1653. ret = type->selftest(type, tr);
  1654. /* the test is responsible for resetting too */
  1655. tr->current_trace = saved_tracer;
  1656. if (ret) {
  1657. printk(KERN_CONT "FAILED!\n");
  1658. /* Add the warning after printing 'FAILED' */
  1659. WARN_ON(1);
  1660. return -1;
  1661. }
  1662. /* Only reset on passing, to avoid touching corrupted buffers */
  1663. tracing_reset_online_cpus(&tr->array_buffer);
  1664. #ifdef CONFIG_TRACER_MAX_TRACE
  1665. if (type->use_max_tr) {
  1666. tr->allocated_snapshot = false;
  1667. /* Shrink the max buffer again */
  1668. if (ring_buffer_expanded)
  1669. ring_buffer_resize(tr->max_buffer.buffer, 1,
  1670. RING_BUFFER_ALL_CPUS);
  1671. }
  1672. #endif
  1673. printk(KERN_CONT "PASSED\n");
  1674. return 0;
  1675. }
  1676. static __init int init_trace_selftests(void)
  1677. {
  1678. struct trace_selftests *p, *n;
  1679. struct tracer *t, **last;
  1680. int ret;
  1681. selftests_can_run = true;
  1682. mutex_lock(&trace_types_lock);
  1683. if (list_empty(&postponed_selftests))
  1684. goto out;
  1685. pr_info("Running postponed tracer tests:\n");
  1686. tracing_selftest_running = true;
  1687. list_for_each_entry_safe(p, n, &postponed_selftests, list) {
  1688. /* This loop can take minutes when sanitizers are enabled, so
  1689. * lets make sure we allow RCU processing.
  1690. */
  1691. cond_resched();
  1692. ret = run_tracer_selftest(p->type);
  1693. /* If the test fails, then warn and remove from available_tracers */
  1694. if (ret < 0) {
  1695. WARN(1, "tracer: %s failed selftest, disabling\n",
  1696. p->type->name);
  1697. last = &trace_types;
  1698. for (t = trace_types; t; t = t->next) {
  1699. if (t == p->type) {
  1700. *last = t->next;
  1701. break;
  1702. }
  1703. last = &t->next;
  1704. }
  1705. }
  1706. list_del(&p->list);
  1707. kfree(p);
  1708. }
  1709. tracing_selftest_running = false;
  1710. out:
  1711. mutex_unlock(&trace_types_lock);
  1712. return 0;
  1713. }
  1714. core_initcall(init_trace_selftests);
  1715. #else
  1716. static inline int run_tracer_selftest(struct tracer *type)
  1717. {
  1718. return 0;
  1719. }
  1720. #endif /* CONFIG_FTRACE_STARTUP_TEST */
  1721. static void add_tracer_options(struct trace_array *tr, struct tracer *t);
  1722. static void __init apply_trace_boot_options(void);
  1723. /**
  1724. * register_tracer - register a tracer with the ftrace system.
  1725. * @type: the plugin for the tracer
  1726. *
  1727. * Register a new plugin tracer.
  1728. */
  1729. int __init register_tracer(struct tracer *type)
  1730. {
  1731. struct tracer *t;
  1732. int ret = 0;
  1733. if (!type->name) {
  1734. pr_info("Tracer must have a name\n");
  1735. return -1;
  1736. }
  1737. if (strlen(type->name) >= MAX_TRACER_SIZE) {
  1738. pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
  1739. return -1;
  1740. }
  1741. if (security_locked_down(LOCKDOWN_TRACEFS)) {
  1742. pr_warn("Can not register tracer %s due to lockdown\n",
  1743. type->name);
  1744. return -EPERM;
  1745. }
  1746. mutex_lock(&trace_types_lock);
  1747. tracing_selftest_running = true;
  1748. for (t = trace_types; t; t = t->next) {
  1749. if (strcmp(type->name, t->name) == 0) {
  1750. /* already found */
  1751. pr_info("Tracer %s already registered\n",
  1752. type->name);
  1753. ret = -1;
  1754. goto out;
  1755. }
  1756. }
  1757. if (!type->set_flag)
  1758. type->set_flag = &dummy_set_flag;
  1759. if (!type->flags) {
  1760. /*allocate a dummy tracer_flags*/
  1761. type->flags = kmalloc(sizeof(*type->flags), GFP_KERNEL);
  1762. if (!type->flags) {
  1763. ret = -ENOMEM;
  1764. goto out;
  1765. }
  1766. type->flags->val = 0;
  1767. type->flags->opts = dummy_tracer_opt;
  1768. } else
  1769. if (!type->flags->opts)
  1770. type->flags->opts = dummy_tracer_opt;
  1771. /* store the tracer for __set_tracer_option */
  1772. type->flags->trace = type;
  1773. ret = run_tracer_selftest(type);
  1774. if (ret < 0)
  1775. goto out;
  1776. type->next = trace_types;
  1777. trace_types = type;
  1778. add_tracer_options(&global_trace, type);
  1779. out:
  1780. tracing_selftest_running = false;
  1781. mutex_unlock(&trace_types_lock);
  1782. if (ret || !default_bootup_tracer)
  1783. goto out_unlock;
  1784. if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
  1785. goto out_unlock;
  1786. printk(KERN_INFO "Starting tracer '%s'\n", type->name);
  1787. /* Do we want this tracer to start on bootup? */
  1788. tracing_set_tracer(&global_trace, type->name);
  1789. default_bootup_tracer = NULL;
  1790. apply_trace_boot_options();
  1791. /* disable other selftests, since this will break it. */
  1792. disable_tracing_selftest("running a tracer");
  1793. out_unlock:
  1794. return ret;
  1795. }
  1796. static void tracing_reset_cpu(struct array_buffer *buf, int cpu)
  1797. {
  1798. struct trace_buffer *buffer = buf->buffer;
  1799. if (!buffer)
  1800. return;
  1801. ring_buffer_record_disable(buffer);
  1802. /* Make sure all commits have finished */
  1803. synchronize_rcu();
  1804. ring_buffer_reset_cpu(buffer, cpu);
  1805. ring_buffer_record_enable(buffer);
  1806. }
  1807. void tracing_reset_online_cpus(struct array_buffer *buf)
  1808. {
  1809. struct trace_buffer *buffer = buf->buffer;
  1810. if (!buffer)
  1811. return;
  1812. ring_buffer_record_disable(buffer);
  1813. /* Make sure all commits have finished */
  1814. synchronize_rcu();
  1815. buf->time_start = buffer_ftrace_now(buf, buf->cpu);
  1816. ring_buffer_reset_online_cpus(buffer);
  1817. ring_buffer_record_enable(buffer);
  1818. }
  1819. /* Must have trace_types_lock held */
  1820. void tracing_reset_all_online_cpus(void)
  1821. {
  1822. struct trace_array *tr;
  1823. list_for_each_entry(tr, &ftrace_trace_arrays, list) {
  1824. if (!tr->clear_trace)
  1825. continue;
  1826. tr->clear_trace = false;
  1827. tracing_reset_online_cpus(&tr->array_buffer);
  1828. #ifdef CONFIG_TRACER_MAX_TRACE
  1829. tracing_reset_online_cpus(&tr->max_buffer);
  1830. #endif
  1831. }
  1832. }
  1833. /*
  1834. * The tgid_map array maps from pid to tgid; i.e. the value stored at index i
  1835. * is the tgid last observed corresponding to pid=i.
  1836. */
  1837. static int *tgid_map;
  1838. /* The maximum valid index into tgid_map. */
  1839. static size_t tgid_map_max;
  1840. #define SAVED_CMDLINES_DEFAULT 128
  1841. #define NO_CMDLINE_MAP UINT_MAX
  1842. static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
  1843. struct saved_cmdlines_buffer {
  1844. unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
  1845. unsigned *map_cmdline_to_pid;
  1846. unsigned cmdline_num;
  1847. int cmdline_idx;
  1848. char *saved_cmdlines;
  1849. };
  1850. static struct saved_cmdlines_buffer *savedcmd;
  1851. static inline char *get_saved_cmdlines(int idx)
  1852. {
  1853. return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
  1854. }
  1855. static inline void set_cmdline(int idx, const char *cmdline)
  1856. {
  1857. strncpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
  1858. }
  1859. static int allocate_cmdlines_buffer(unsigned int val,
  1860. struct saved_cmdlines_buffer *s)
  1861. {
  1862. s->map_cmdline_to_pid = kmalloc_array(val,
  1863. sizeof(*s->map_cmdline_to_pid),
  1864. GFP_KERNEL);
  1865. if (!s->map_cmdline_to_pid)
  1866. return -ENOMEM;
  1867. s->saved_cmdlines = kmalloc_array(TASK_COMM_LEN, val, GFP_KERNEL);
  1868. if (!s->saved_cmdlines) {
  1869. kfree(s->map_cmdline_to_pid);
  1870. return -ENOMEM;
  1871. }
  1872. s->cmdline_idx = 0;
  1873. s->cmdline_num = val;
  1874. memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
  1875. sizeof(s->map_pid_to_cmdline));
  1876. memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
  1877. val * sizeof(*s->map_cmdline_to_pid));
  1878. return 0;
  1879. }
  1880. static int trace_create_savedcmd(void)
  1881. {
  1882. int ret;
  1883. savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
  1884. if (!savedcmd)
  1885. return -ENOMEM;
  1886. ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
  1887. if (ret < 0) {
  1888. kfree(savedcmd);
  1889. savedcmd = NULL;
  1890. return -ENOMEM;
  1891. }
  1892. return 0;
  1893. }
  1894. int is_tracing_stopped(void)
  1895. {
  1896. return global_trace.stop_count;
  1897. }
  1898. /**
  1899. * tracing_start - quick start of the tracer
  1900. *
  1901. * If tracing is enabled but was stopped by tracing_stop,
  1902. * this will start the tracer back up.
  1903. */
  1904. void tracing_start(void)
  1905. {
  1906. struct trace_buffer *buffer;
  1907. unsigned long flags;
  1908. if (tracing_disabled)
  1909. return;
  1910. raw_spin_lock_irqsave(&global_trace.start_lock, flags);
  1911. if (--global_trace.stop_count) {
  1912. if (global_trace.stop_count < 0) {
  1913. /* Someone screwed up their debugging */
  1914. WARN_ON_ONCE(1);
  1915. global_trace.stop_count = 0;
  1916. }
  1917. goto out;
  1918. }
  1919. /* Prevent the buffers from switching */
  1920. arch_spin_lock(&global_trace.max_lock);
  1921. buffer = global_trace.array_buffer.buffer;
  1922. if (buffer)
  1923. ring_buffer_record_enable(buffer);
  1924. #ifdef CONFIG_TRACER_MAX_TRACE
  1925. buffer = global_trace.max_buffer.buffer;
  1926. if (buffer)
  1927. ring_buffer_record_enable(buffer);
  1928. #endif
  1929. arch_spin_unlock(&global_trace.max_lock);
  1930. out:
  1931. raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
  1932. }
  1933. static void tracing_start_tr(struct trace_array *tr)
  1934. {
  1935. struct trace_buffer *buffer;
  1936. unsigned long flags;
  1937. if (tracing_disabled)
  1938. return;
  1939. /* If global, we need to also start the max tracer */
  1940. if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
  1941. return tracing_start();
  1942. raw_spin_lock_irqsave(&tr->start_lock, flags);
  1943. if (--tr->stop_count) {
  1944. if (tr->stop_count < 0) {
  1945. /* Someone screwed up their debugging */
  1946. WARN_ON_ONCE(1);
  1947. tr->stop_count = 0;
  1948. }
  1949. goto out;
  1950. }
  1951. buffer = tr->array_buffer.buffer;
  1952. if (buffer)
  1953. ring_buffer_record_enable(buffer);
  1954. out:
  1955. raw_spin_unlock_irqrestore(&tr->start_lock, flags);
  1956. }
  1957. /**
  1958. * tracing_stop - quick stop of the tracer
  1959. *
  1960. * Light weight way to stop tracing. Use in conjunction with
  1961. * tracing_start.
  1962. */
  1963. void tracing_stop(void)
  1964. {
  1965. struct trace_buffer *buffer;
  1966. unsigned long flags;
  1967. raw_spin_lock_irqsave(&global_trace.start_lock, flags);
  1968. if (global_trace.stop_count++)
  1969. goto out;
  1970. /* Prevent the buffers from switching */
  1971. arch_spin_lock(&global_trace.max_lock);
  1972. buffer = global_trace.array_buffer.buffer;
  1973. if (buffer)
  1974. ring_buffer_record_disable(buffer);
  1975. #ifdef CONFIG_TRACER_MAX_TRACE
  1976. buffer = global_trace.max_buffer.buffer;
  1977. if (buffer)
  1978. ring_buffer_record_disable(buffer);
  1979. #endif
  1980. arch_spin_unlock(&global_trace.max_lock);
  1981. out:
  1982. raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
  1983. }
  1984. static void tracing_stop_tr(struct trace_array *tr)
  1985. {
  1986. struct trace_buffer *buffer;
  1987. unsigned long flags;
  1988. /* If global, we need to also stop the max tracer */
  1989. if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
  1990. return tracing_stop();
  1991. raw_spin_lock_irqsave(&tr->start_lock, flags);
  1992. if (tr->stop_count++)
  1993. goto out;
  1994. buffer = tr->array_buffer.buffer;
  1995. if (buffer)
  1996. ring_buffer_record_disable(buffer);
  1997. out:
  1998. raw_spin_unlock_irqrestore(&tr->start_lock, flags);
  1999. }
  2000. static int trace_save_cmdline(struct task_struct *tsk)
  2001. {
  2002. unsigned tpid, idx;
  2003. /* treat recording of idle task as a success */
  2004. if (!tsk->pid)
  2005. return 1;
  2006. tpid = tsk->pid & (PID_MAX_DEFAULT - 1);
  2007. /*
  2008. * It's not the end of the world if we don't get
  2009. * the lock, but we also don't want to spin
  2010. * nor do we want to disable interrupts,
  2011. * so if we miss here, then better luck next time.
  2012. */
  2013. if (!arch_spin_trylock(&trace_cmdline_lock))
  2014. return 0;
  2015. idx = savedcmd->map_pid_to_cmdline[tpid];
  2016. if (idx == NO_CMDLINE_MAP) {
  2017. idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
  2018. savedcmd->map_pid_to_cmdline[tpid] = idx;
  2019. savedcmd->cmdline_idx = idx;
  2020. }
  2021. savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
  2022. set_cmdline(idx, tsk->comm);
  2023. arch_spin_unlock(&trace_cmdline_lock);
  2024. return 1;
  2025. }
  2026. static void __trace_find_cmdline(int pid, char comm[])
  2027. {
  2028. unsigned map;
  2029. int tpid;
  2030. if (!pid) {
  2031. strcpy(comm, "<idle>");
  2032. return;
  2033. }
  2034. if (WARN_ON_ONCE(pid < 0)) {
  2035. strcpy(comm, "<XXX>");
  2036. return;
  2037. }
  2038. tpid = pid & (PID_MAX_DEFAULT - 1);
  2039. map = savedcmd->map_pid_to_cmdline[tpid];
  2040. if (map != NO_CMDLINE_MAP) {
  2041. tpid = savedcmd->map_cmdline_to_pid[map];
  2042. if (tpid == pid) {
  2043. strlcpy(comm, get_saved_cmdlines(map), TASK_COMM_LEN);
  2044. return;
  2045. }
  2046. }
  2047. strcpy(comm, "<...>");
  2048. }
  2049. void trace_find_cmdline(int pid, char comm[])
  2050. {
  2051. preempt_disable();
  2052. arch_spin_lock(&trace_cmdline_lock);
  2053. __trace_find_cmdline(pid, comm);
  2054. arch_spin_unlock(&trace_cmdline_lock);
  2055. preempt_enable();
  2056. }
  2057. static int *trace_find_tgid_ptr(int pid)
  2058. {
  2059. /*
  2060. * Pairs with the smp_store_release in set_tracer_flag() to ensure that
  2061. * if we observe a non-NULL tgid_map then we also observe the correct
  2062. * tgid_map_max.
  2063. */
  2064. int *map = smp_load_acquire(&tgid_map);
  2065. if (unlikely(!map || pid > tgid_map_max))
  2066. return NULL;
  2067. return &map[pid];
  2068. }
  2069. int trace_find_tgid(int pid)
  2070. {
  2071. int *ptr = trace_find_tgid_ptr(pid);
  2072. return ptr ? *ptr : 0;
  2073. }
  2074. static int trace_save_tgid(struct task_struct *tsk)
  2075. {
  2076. int *ptr;
  2077. /* treat recording of idle task as a success */
  2078. if (!tsk->pid)
  2079. return 1;
  2080. ptr = trace_find_tgid_ptr(tsk->pid);
  2081. if (!ptr)
  2082. return 0;
  2083. *ptr = tsk->tgid;
  2084. return 1;
  2085. }
  2086. static bool tracing_record_taskinfo_skip(int flags)
  2087. {
  2088. if (unlikely(!(flags & (TRACE_RECORD_CMDLINE | TRACE_RECORD_TGID))))
  2089. return true;
  2090. if (!__this_cpu_read(trace_taskinfo_save))
  2091. return true;
  2092. return false;
  2093. }
  2094. /**
  2095. * tracing_record_taskinfo - record the task info of a task
  2096. *
  2097. * @task: task to record
  2098. * @flags: TRACE_RECORD_CMDLINE for recording comm
  2099. * TRACE_RECORD_TGID for recording tgid
  2100. */
  2101. void tracing_record_taskinfo(struct task_struct *task, int flags)
  2102. {
  2103. bool done;
  2104. if (tracing_record_taskinfo_skip(flags))
  2105. return;
  2106. /*
  2107. * Record as much task information as possible. If some fail, continue
  2108. * to try to record the others.
  2109. */
  2110. done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(task);
  2111. done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(task);
  2112. /* If recording any information failed, retry again soon. */
  2113. if (!done)
  2114. return;
  2115. __this_cpu_write(trace_taskinfo_save, false);
  2116. }
  2117. /**
  2118. * tracing_record_taskinfo_sched_switch - record task info for sched_switch
  2119. *
  2120. * @prev: previous task during sched_switch
  2121. * @next: next task during sched_switch
  2122. * @flags: TRACE_RECORD_CMDLINE for recording comm
  2123. * TRACE_RECORD_TGID for recording tgid
  2124. */
  2125. void tracing_record_taskinfo_sched_switch(struct task_struct *prev,
  2126. struct task_struct *next, int flags)
  2127. {
  2128. bool done;
  2129. if (tracing_record_taskinfo_skip(flags))
  2130. return;
  2131. /*
  2132. * Record as much task information as possible. If some fail, continue
  2133. * to try to record the others.
  2134. */
  2135. done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(prev);
  2136. done &= !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(next);
  2137. done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(prev);
  2138. done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(next);
  2139. /* If recording any information failed, retry again soon. */
  2140. if (!done)
  2141. return;
  2142. __this_cpu_write(trace_taskinfo_save, false);
  2143. }
  2144. /* Helpers to record a specific task information */
  2145. void tracing_record_cmdline(struct task_struct *task)
  2146. {
  2147. tracing_record_taskinfo(task, TRACE_RECORD_CMDLINE);
  2148. }
  2149. void tracing_record_tgid(struct task_struct *task)
  2150. {
  2151. tracing_record_taskinfo(task, TRACE_RECORD_TGID);
  2152. }
  2153. /*
  2154. * Several functions return TRACE_TYPE_PARTIAL_LINE if the trace_seq
  2155. * overflowed, and TRACE_TYPE_HANDLED otherwise. This helper function
  2156. * simplifies those functions and keeps them in sync.
  2157. */
  2158. enum print_line_t trace_handle_return(struct trace_seq *s)
  2159. {
  2160. return trace_seq_has_overflowed(s) ?
  2161. TRACE_TYPE_PARTIAL_LINE : TRACE_TYPE_HANDLED;
  2162. }
  2163. EXPORT_SYMBOL_GPL(trace_handle_return);
  2164. void
  2165. tracing_generic_entry_update(struct trace_entry *entry, unsigned short type,
  2166. unsigned long flags, int pc)
  2167. {
  2168. struct task_struct *tsk = current;
  2169. entry->preempt_count = pc & 0xff;
  2170. entry->pid = (tsk) ? tsk->pid : 0;
  2171. entry->type = type;
  2172. entry->flags =
  2173. #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
  2174. (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
  2175. #else
  2176. TRACE_FLAG_IRQS_NOSUPPORT |
  2177. #endif
  2178. ((pc & NMI_MASK ) ? TRACE_FLAG_NMI : 0) |
  2179. ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
  2180. ((pc & SOFTIRQ_OFFSET) ? TRACE_FLAG_SOFTIRQ : 0) |
  2181. (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
  2182. (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
  2183. }
  2184. EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
  2185. struct ring_buffer_event *
  2186. trace_buffer_lock_reserve(struct trace_buffer *buffer,
  2187. int type,
  2188. unsigned long len,
  2189. unsigned long flags, int pc)
  2190. {
  2191. return __trace_buffer_lock_reserve(buffer, type, len, flags, pc);
  2192. }
  2193. DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
  2194. DEFINE_PER_CPU(int, trace_buffered_event_cnt);
  2195. static int trace_buffered_event_ref;
  2196. /**
  2197. * trace_buffered_event_enable - enable buffering events
  2198. *
  2199. * When events are being filtered, it is quicker to use a temporary
  2200. * buffer to write the event data into if there's a likely chance
  2201. * that it will not be committed. The discard of the ring buffer
  2202. * is not as fast as committing, and is much slower than copying
  2203. * a commit.
  2204. *
  2205. * When an event is to be filtered, allocate per cpu buffers to
  2206. * write the event data into, and if the event is filtered and discarded
  2207. * it is simply dropped, otherwise, the entire data is to be committed
  2208. * in one shot.
  2209. */
  2210. void trace_buffered_event_enable(void)
  2211. {
  2212. struct ring_buffer_event *event;
  2213. struct page *page;
  2214. int cpu;
  2215. WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
  2216. if (trace_buffered_event_ref++)
  2217. return;
  2218. for_each_tracing_cpu(cpu) {
  2219. page = alloc_pages_node(cpu_to_node(cpu),
  2220. GFP_KERNEL | __GFP_NORETRY, 0);
  2221. if (!page)
  2222. goto failed;
  2223. event = page_address(page);
  2224. memset(event, 0, sizeof(*event));
  2225. per_cpu(trace_buffered_event, cpu) = event;
  2226. preempt_disable();
  2227. if (cpu == smp_processor_id() &&
  2228. __this_cpu_read(trace_buffered_event) !=
  2229. per_cpu(trace_buffered_event, cpu))
  2230. WARN_ON_ONCE(1);
  2231. preempt_enable();
  2232. }
  2233. return;
  2234. failed:
  2235. trace_buffered_event_disable();
  2236. }
  2237. static void enable_trace_buffered_event(void *data)
  2238. {
  2239. /* Probably not needed, but do it anyway */
  2240. smp_rmb();
  2241. this_cpu_dec(trace_buffered_event_cnt);
  2242. }
  2243. static void disable_trace_buffered_event(void *data)
  2244. {
  2245. this_cpu_inc(trace_buffered_event_cnt);
  2246. }
  2247. /**
  2248. * trace_buffered_event_disable - disable buffering events
  2249. *
  2250. * When a filter is removed, it is faster to not use the buffered
  2251. * events, and to commit directly into the ring buffer. Free up
  2252. * the temp buffers when there are no more users. This requires
  2253. * special synchronization with current events.
  2254. */
  2255. void trace_buffered_event_disable(void)
  2256. {
  2257. int cpu;
  2258. WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
  2259. if (WARN_ON_ONCE(!trace_buffered_event_ref))
  2260. return;
  2261. if (--trace_buffered_event_ref)
  2262. return;
  2263. preempt_disable();
  2264. /* For each CPU, set the buffer as used. */
  2265. smp_call_function_many(tracing_buffer_mask,
  2266. disable_trace_buffered_event, NULL, 1);
  2267. preempt_enable();
  2268. /* Wait for all current users to finish */
  2269. synchronize_rcu();
  2270. for_each_tracing_cpu(cpu) {
  2271. free_page((unsigned long)per_cpu(trace_buffered_event, cpu));
  2272. per_cpu(trace_buffered_event, cpu) = NULL;
  2273. }
  2274. /*
  2275. * Make sure trace_buffered_event is NULL before clearing
  2276. * trace_buffered_event_cnt.
  2277. */
  2278. smp_wmb();
  2279. preempt_disable();
  2280. /* Do the work on each cpu */
  2281. smp_call_function_many(tracing_buffer_mask,
  2282. enable_trace_buffered_event, NULL, 1);
  2283. preempt_enable();
  2284. }
  2285. static struct trace_buffer *temp_buffer;
  2286. struct ring_buffer_event *
  2287. trace_event_buffer_lock_reserve(struct trace_buffer **current_rb,
  2288. struct trace_event_file *trace_file,
  2289. int type, unsigned long len,
  2290. unsigned long flags, int pc)
  2291. {
  2292. struct ring_buffer_event *entry;
  2293. int val;
  2294. *current_rb = trace_file->tr->array_buffer.buffer;
  2295. if (!ring_buffer_time_stamp_abs(*current_rb) && (trace_file->flags &
  2296. (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED)) &&
  2297. (entry = this_cpu_read(trace_buffered_event))) {
  2298. /* Try to use the per cpu buffer first */
  2299. val = this_cpu_inc_return(trace_buffered_event_cnt);
  2300. if ((len < (PAGE_SIZE - sizeof(*entry) - sizeof(entry->array[0]))) && val == 1) {
  2301. trace_event_setup(entry, type, flags, pc);
  2302. entry->array[0] = len;
  2303. return entry;
  2304. }
  2305. this_cpu_dec(trace_buffered_event_cnt);
  2306. }
  2307. entry = __trace_buffer_lock_reserve(*current_rb,
  2308. type, len, flags, pc);
  2309. /*
  2310. * If tracing is off, but we have triggers enabled
  2311. * we still need to look at the event data. Use the temp_buffer
  2312. * to store the trace event for the trigger to use. It's recursive
  2313. * safe and will not be recorded anywhere.
  2314. */
  2315. if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
  2316. *current_rb = temp_buffer;
  2317. entry = __trace_buffer_lock_reserve(*current_rb,
  2318. type, len, flags, pc);
  2319. }
  2320. return entry;
  2321. }
  2322. EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
  2323. static DEFINE_SPINLOCK(tracepoint_iter_lock);
  2324. static DEFINE_MUTEX(tracepoint_printk_mutex);
  2325. static void output_printk(struct trace_event_buffer *fbuffer)
  2326. {
  2327. struct trace_event_call *event_call;
  2328. struct trace_event_file *file;
  2329. struct trace_event *event;
  2330. unsigned long flags;
  2331. struct trace_iterator *iter = tracepoint_print_iter;
  2332. /* We should never get here if iter is NULL */
  2333. if (WARN_ON_ONCE(!iter))
  2334. return;
  2335. event_call = fbuffer->trace_file->event_call;
  2336. if (!event_call || !event_call->event.funcs ||
  2337. !event_call->event.funcs->trace)
  2338. return;
  2339. file = fbuffer->trace_file;
  2340. if (test_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags) ||
  2341. (unlikely(file->flags & EVENT_FILE_FL_FILTERED) &&
  2342. !filter_match_preds(file->filter, fbuffer->entry)))
  2343. return;
  2344. event = &fbuffer->trace_file->event_call->event;
  2345. spin_lock_irqsave(&tracepoint_iter_lock, flags);
  2346. trace_seq_init(&iter->seq);
  2347. iter->ent = fbuffer->entry;
  2348. event_call->event.funcs->trace(iter, 0, event);
  2349. trace_seq_putc(&iter->seq, 0);
  2350. printk("%s", iter->seq.buffer);
  2351. spin_unlock_irqrestore(&tracepoint_iter_lock, flags);
  2352. }
  2353. int tracepoint_printk_sysctl(struct ctl_table *table, int write,
  2354. void *buffer, size_t *lenp,
  2355. loff_t *ppos)
  2356. {
  2357. int save_tracepoint_printk;
  2358. int ret;
  2359. mutex_lock(&tracepoint_printk_mutex);
  2360. save_tracepoint_printk = tracepoint_printk;
  2361. ret = proc_dointvec(table, write, buffer, lenp, ppos);
  2362. /*
  2363. * This will force exiting early, as tracepoint_printk
  2364. * is always zero when tracepoint_printk_iter is not allocated
  2365. */
  2366. if (!tracepoint_print_iter)
  2367. tracepoint_printk = 0;
  2368. if (save_tracepoint_printk == tracepoint_printk)
  2369. goto out;
  2370. if (tracepoint_printk)
  2371. static_key_enable(&tracepoint_printk_key.key);
  2372. else
  2373. static_key_disable(&tracepoint_printk_key.key);
  2374. out:
  2375. mutex_unlock(&tracepoint_printk_mutex);
  2376. return ret;
  2377. }
  2378. void trace_event_buffer_commit(struct trace_event_buffer *fbuffer)
  2379. {
  2380. if (static_key_false(&tracepoint_printk_key.key))
  2381. output_printk(fbuffer);
  2382. if (static_branch_unlikely(&trace_event_exports_enabled))
  2383. ftrace_exports(fbuffer->event, TRACE_EXPORT_EVENT);
  2384. event_trigger_unlock_commit_regs(fbuffer->trace_file, fbuffer->buffer,
  2385. fbuffer->event, fbuffer->entry,
  2386. fbuffer->flags, fbuffer->pc, fbuffer->regs);
  2387. }
  2388. EXPORT_SYMBOL_GPL(trace_event_buffer_commit);
  2389. /*
  2390. * Skip 3:
  2391. *
  2392. * trace_buffer_unlock_commit_regs()
  2393. * trace_event_buffer_commit()
  2394. * trace_event_raw_event_xxx()
  2395. */
  2396. # define STACK_SKIP 3
  2397. void trace_buffer_unlock_commit_regs(struct trace_array *tr,
  2398. struct trace_buffer *buffer,
  2399. struct ring_buffer_event *event,
  2400. unsigned long flags, int pc,
  2401. struct pt_regs *regs)
  2402. {
  2403. __buffer_unlock_commit(buffer, event);
  2404. /*
  2405. * If regs is not set, then skip the necessary functions.
  2406. * Note, we can still get here via blktrace, wakeup tracer
  2407. * and mmiotrace, but that's ok if they lose a function or
  2408. * two. They are not that meaningful.
  2409. */
  2410. ftrace_trace_stack(tr, buffer, flags, regs ? 0 : STACK_SKIP, pc, regs);
  2411. ftrace_trace_userstack(tr, buffer, flags, pc);
  2412. }
  2413. /*
  2414. * Similar to trace_buffer_unlock_commit_regs() but do not dump stack.
  2415. */
  2416. void
  2417. trace_buffer_unlock_commit_nostack(struct trace_buffer *buffer,
  2418. struct ring_buffer_event *event)
  2419. {
  2420. __buffer_unlock_commit(buffer, event);
  2421. }
  2422. void
  2423. trace_function(struct trace_array *tr,
  2424. unsigned long ip, unsigned long parent_ip, unsigned long flags,
  2425. int pc)
  2426. {
  2427. struct trace_event_call *call = &event_function;
  2428. struct trace_buffer *buffer = tr->array_buffer.buffer;
  2429. struct ring_buffer_event *event;
  2430. struct ftrace_entry *entry;
  2431. event = __trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
  2432. flags, pc);
  2433. if (!event)
  2434. return;
  2435. entry = ring_buffer_event_data(event);
  2436. entry->ip = ip;
  2437. entry->parent_ip = parent_ip;
  2438. if (!call_filter_check_discard(call, entry, buffer, event)) {
  2439. if (static_branch_unlikely(&trace_function_exports_enabled))
  2440. ftrace_exports(event, TRACE_EXPORT_FUNCTION);
  2441. __buffer_unlock_commit(buffer, event);
  2442. }
  2443. }
  2444. #ifdef CONFIG_STACKTRACE
  2445. /* Allow 4 levels of nesting: normal, softirq, irq, NMI */
  2446. #define FTRACE_KSTACK_NESTING 4
  2447. #define FTRACE_KSTACK_ENTRIES (PAGE_SIZE / FTRACE_KSTACK_NESTING)
  2448. struct ftrace_stack {
  2449. unsigned long calls[FTRACE_KSTACK_ENTRIES];
  2450. };
  2451. struct ftrace_stacks {
  2452. struct ftrace_stack stacks[FTRACE_KSTACK_NESTING];
  2453. };
  2454. static DEFINE_PER_CPU(struct ftrace_stacks, ftrace_stacks);
  2455. static DEFINE_PER_CPU(int, ftrace_stack_reserve);
  2456. static void __ftrace_trace_stack(struct trace_buffer *buffer,
  2457. unsigned long flags,
  2458. int skip, int pc, struct pt_regs *regs)
  2459. {
  2460. struct trace_event_call *call = &event_kernel_stack;
  2461. struct ring_buffer_event *event;
  2462. unsigned int size, nr_entries;
  2463. struct ftrace_stack *fstack;
  2464. struct stack_entry *entry;
  2465. int stackidx;
  2466. /*
  2467. * Add one, for this function and the call to save_stack_trace()
  2468. * If regs is set, then these functions will not be in the way.
  2469. */
  2470. #ifndef CONFIG_UNWINDER_ORC
  2471. if (!regs)
  2472. skip++;
  2473. #endif
  2474. preempt_disable_notrace();
  2475. stackidx = __this_cpu_inc_return(ftrace_stack_reserve) - 1;
  2476. /* This should never happen. If it does, yell once and skip */
  2477. if (WARN_ON_ONCE(stackidx >= FTRACE_KSTACK_NESTING))
  2478. goto out;
  2479. /*
  2480. * The above __this_cpu_inc_return() is 'atomic' cpu local. An
  2481. * interrupt will either see the value pre increment or post
  2482. * increment. If the interrupt happens pre increment it will have
  2483. * restored the counter when it returns. We just need a barrier to
  2484. * keep gcc from moving things around.
  2485. */
  2486. barrier();
  2487. fstack = this_cpu_ptr(ftrace_stacks.stacks) + stackidx;
  2488. size = ARRAY_SIZE(fstack->calls);
  2489. if (regs) {
  2490. nr_entries = stack_trace_save_regs(regs, fstack->calls,
  2491. size, skip);
  2492. } else {
  2493. nr_entries = stack_trace_save(fstack->calls, size, skip);
  2494. }
  2495. size = nr_entries * sizeof(unsigned long);
  2496. event = __trace_buffer_lock_reserve(buffer, TRACE_STACK,
  2497. (sizeof(*entry) - sizeof(entry->caller)) + size,
  2498. flags, pc);
  2499. if (!event)
  2500. goto out;
  2501. entry = ring_buffer_event_data(event);
  2502. memcpy(&entry->caller, fstack->calls, size);
  2503. entry->size = nr_entries;
  2504. if (!call_filter_check_discard(call, entry, buffer, event))
  2505. __buffer_unlock_commit(buffer, event);
  2506. out:
  2507. /* Again, don't let gcc optimize things here */
  2508. barrier();
  2509. __this_cpu_dec(ftrace_stack_reserve);
  2510. preempt_enable_notrace();
  2511. }
  2512. static inline void ftrace_trace_stack(struct trace_array *tr,
  2513. struct trace_buffer *buffer,
  2514. unsigned long flags,
  2515. int skip, int pc, struct pt_regs *regs)
  2516. {
  2517. if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))
  2518. return;
  2519. __ftrace_trace_stack(buffer, flags, skip, pc, regs);
  2520. }
  2521. void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
  2522. int pc)
  2523. {
  2524. struct trace_buffer *buffer = tr->array_buffer.buffer;
  2525. if (rcu_is_watching()) {
  2526. __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
  2527. return;
  2528. }
  2529. /*
  2530. * When an NMI triggers, RCU is enabled via rcu_nmi_enter(),
  2531. * but if the above rcu_is_watching() failed, then the NMI
  2532. * triggered someplace critical, and rcu_irq_enter() should
  2533. * not be called from NMI.
  2534. */
  2535. if (unlikely(in_nmi()))
  2536. return;
  2537. rcu_irq_enter_irqson();
  2538. __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
  2539. rcu_irq_exit_irqson();
  2540. }
  2541. /**
  2542. * trace_dump_stack - record a stack back trace in the trace buffer
  2543. * @skip: Number of functions to skip (helper handlers)
  2544. */
  2545. void trace_dump_stack(int skip)
  2546. {
  2547. unsigned long flags;
  2548. if (tracing_disabled || tracing_selftest_running)
  2549. return;
  2550. local_save_flags(flags);
  2551. #ifndef CONFIG_UNWINDER_ORC
  2552. /* Skip 1 to skip this function. */
  2553. skip++;
  2554. #endif
  2555. __ftrace_trace_stack(global_trace.array_buffer.buffer,
  2556. flags, skip, preempt_count(), NULL);
  2557. }
  2558. EXPORT_SYMBOL_GPL(trace_dump_stack);
  2559. #ifdef CONFIG_USER_STACKTRACE_SUPPORT
  2560. static DEFINE_PER_CPU(int, user_stack_count);
  2561. static void
  2562. ftrace_trace_userstack(struct trace_array *tr,
  2563. struct trace_buffer *buffer, unsigned long flags, int pc)
  2564. {
  2565. struct trace_event_call *call = &event_user_stack;
  2566. struct ring_buffer_event *event;
  2567. struct userstack_entry *entry;
  2568. if (!(tr->trace_flags & TRACE_ITER_USERSTACKTRACE))
  2569. return;
  2570. /*
  2571. * NMIs can not handle page faults, even with fix ups.
  2572. * The save user stack can (and often does) fault.
  2573. */
  2574. if (unlikely(in_nmi()))
  2575. return;
  2576. /*
  2577. * prevent recursion, since the user stack tracing may
  2578. * trigger other kernel events.
  2579. */
  2580. preempt_disable();
  2581. if (__this_cpu_read(user_stack_count))
  2582. goto out;
  2583. __this_cpu_inc(user_stack_count);
  2584. event = __trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
  2585. sizeof(*entry), flags, pc);
  2586. if (!event)
  2587. goto out_drop_count;
  2588. entry = ring_buffer_event_data(event);
  2589. entry->tgid = current->tgid;
  2590. memset(&entry->caller, 0, sizeof(entry->caller));
  2591. stack_trace_save_user(entry->caller, FTRACE_STACK_ENTRIES);
  2592. if (!call_filter_check_discard(call, entry, buffer, event))
  2593. __buffer_unlock_commit(buffer, event);
  2594. out_drop_count:
  2595. __this_cpu_dec(user_stack_count);
  2596. out:
  2597. preempt_enable();
  2598. }
  2599. #else /* CONFIG_USER_STACKTRACE_SUPPORT */
  2600. static void ftrace_trace_userstack(struct trace_array *tr,
  2601. struct trace_buffer *buffer,
  2602. unsigned long flags, int pc)
  2603. {
  2604. }
  2605. #endif /* !CONFIG_USER_STACKTRACE_SUPPORT */
  2606. #endif /* CONFIG_STACKTRACE */
  2607. /* created for use with alloc_percpu */
  2608. struct trace_buffer_struct {
  2609. int nesting;
  2610. char buffer[4][TRACE_BUF_SIZE];
  2611. };
  2612. static struct trace_buffer_struct __percpu *trace_percpu_buffer;
  2613. /*
  2614. * Thise allows for lockless recording. If we're nested too deeply, then
  2615. * this returns NULL.
  2616. */
  2617. static char *get_trace_buf(void)
  2618. {
  2619. struct trace_buffer_struct *buffer = this_cpu_ptr(trace_percpu_buffer);
  2620. if (!trace_percpu_buffer || buffer->nesting >= 4)
  2621. return NULL;
  2622. buffer->nesting++;
  2623. /* Interrupts must see nesting incremented before we use the buffer */
  2624. barrier();
  2625. return &buffer->buffer[buffer->nesting - 1][0];
  2626. }
  2627. static void put_trace_buf(void)
  2628. {
  2629. /* Don't let the decrement of nesting leak before this */
  2630. barrier();
  2631. this_cpu_dec(trace_percpu_buffer->nesting);
  2632. }
  2633. static int alloc_percpu_trace_buffer(void)
  2634. {
  2635. struct trace_buffer_struct __percpu *buffers;
  2636. if (trace_percpu_buffer)
  2637. return 0;
  2638. buffers = alloc_percpu(struct trace_buffer_struct);
  2639. if (MEM_FAIL(!buffers, "Could not allocate percpu trace_printk buffer"))
  2640. return -ENOMEM;
  2641. trace_percpu_buffer = buffers;
  2642. return 0;
  2643. }
  2644. static int buffers_allocated;
  2645. void trace_printk_init_buffers(void)
  2646. {
  2647. if (buffers_allocated)
  2648. return;
  2649. if (alloc_percpu_trace_buffer())
  2650. return;
  2651. /* trace_printk() is for debug use only. Don't use it in production. */
  2652. pr_warn("\n");
  2653. pr_warn("**********************************************************\n");
  2654. pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
  2655. pr_warn("** **\n");
  2656. pr_warn("** trace_printk() being used. Allocating extra memory. **\n");
  2657. pr_warn("** **\n");
  2658. pr_warn("** This means that this is a DEBUG kernel and it is **\n");
  2659. pr_warn("** unsafe for production use. **\n");
  2660. pr_warn("** **\n");
  2661. pr_warn("** If you see this message and you are not debugging **\n");
  2662. pr_warn("** the kernel, report this immediately to your vendor! **\n");
  2663. pr_warn("** **\n");
  2664. pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
  2665. pr_warn("**********************************************************\n");
  2666. /* Expand the buffers to set size */
  2667. tracing_update_buffers();
  2668. buffers_allocated = 1;
  2669. /*
  2670. * trace_printk_init_buffers() can be called by modules.
  2671. * If that happens, then we need to start cmdline recording
  2672. * directly here. If the global_trace.buffer is already
  2673. * allocated here, then this was called by module code.
  2674. */
  2675. if (global_trace.array_buffer.buffer)
  2676. tracing_start_cmdline_record();
  2677. }
  2678. EXPORT_SYMBOL_GPL(trace_printk_init_buffers);
  2679. void trace_printk_start_comm(void)
  2680. {
  2681. /* Start tracing comms if trace printk is set */
  2682. if (!buffers_allocated)
  2683. return;
  2684. tracing_start_cmdline_record();
  2685. }
  2686. static void trace_printk_start_stop_comm(int enabled)
  2687. {
  2688. if (!buffers_allocated)
  2689. return;
  2690. if (enabled)
  2691. tracing_start_cmdline_record();
  2692. else
  2693. tracing_stop_cmdline_record();
  2694. }
  2695. /**
  2696. * trace_vbprintk - write binary msg to tracing buffer
  2697. * @ip: The address of the caller
  2698. * @fmt: The string format to write to the buffer
  2699. * @args: Arguments for @fmt
  2700. */
  2701. int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
  2702. {
  2703. struct trace_event_call *call = &event_bprint;
  2704. struct ring_buffer_event *event;
  2705. struct trace_buffer *buffer;
  2706. struct trace_array *tr = &global_trace;
  2707. struct bprint_entry *entry;
  2708. unsigned long flags;
  2709. char *tbuffer;
  2710. int len = 0, size, pc;
  2711. if (unlikely(tracing_selftest_running || tracing_disabled))
  2712. return 0;
  2713. /* Don't pollute graph traces with trace_vprintk internals */
  2714. pause_graph_tracing();
  2715. pc = preempt_count();
  2716. preempt_disable_notrace();
  2717. tbuffer = get_trace_buf();
  2718. if (!tbuffer) {
  2719. len = 0;
  2720. goto out_nobuffer;
  2721. }
  2722. len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
  2723. if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
  2724. goto out_put;
  2725. local_save_flags(flags);
  2726. size = sizeof(*entry) + sizeof(u32) * len;
  2727. buffer = tr->array_buffer.buffer;
  2728. ring_buffer_nest_start(buffer);
  2729. event = __trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
  2730. flags, pc);
  2731. if (!event)
  2732. goto out;
  2733. entry = ring_buffer_event_data(event);
  2734. entry->ip = ip;
  2735. entry->fmt = fmt;
  2736. memcpy(entry->buf, tbuffer, sizeof(u32) * len);
  2737. if (!call_filter_check_discard(call, entry, buffer, event)) {
  2738. __buffer_unlock_commit(buffer, event);
  2739. ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL);
  2740. }
  2741. out:
  2742. ring_buffer_nest_end(buffer);
  2743. out_put:
  2744. put_trace_buf();
  2745. out_nobuffer:
  2746. preempt_enable_notrace();
  2747. unpause_graph_tracing();
  2748. return len;
  2749. }
  2750. EXPORT_SYMBOL_GPL(trace_vbprintk);
  2751. __printf(3, 0)
  2752. static int
  2753. __trace_array_vprintk(struct trace_buffer *buffer,
  2754. unsigned long ip, const char *fmt, va_list args)
  2755. {
  2756. struct trace_event_call *call = &event_print;
  2757. struct ring_buffer_event *event;
  2758. int len = 0, size, pc;
  2759. struct print_entry *entry;
  2760. unsigned long flags;
  2761. char *tbuffer;
  2762. if (tracing_disabled || tracing_selftest_running)
  2763. return 0;
  2764. /* Don't pollute graph traces with trace_vprintk internals */
  2765. pause_graph_tracing();
  2766. pc = preempt_count();
  2767. preempt_disable_notrace();
  2768. tbuffer = get_trace_buf();
  2769. if (!tbuffer) {
  2770. len = 0;
  2771. goto out_nobuffer;
  2772. }
  2773. len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
  2774. local_save_flags(flags);
  2775. size = sizeof(*entry) + len + 1;
  2776. ring_buffer_nest_start(buffer);
  2777. event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
  2778. flags, pc);
  2779. if (!event)
  2780. goto out;
  2781. entry = ring_buffer_event_data(event);
  2782. entry->ip = ip;
  2783. memcpy(&entry->buf, tbuffer, len + 1);
  2784. if (!call_filter_check_discard(call, entry, buffer, event)) {
  2785. __buffer_unlock_commit(buffer, event);
  2786. ftrace_trace_stack(&global_trace, buffer, flags, 6, pc, NULL);
  2787. }
  2788. out:
  2789. ring_buffer_nest_end(buffer);
  2790. put_trace_buf();
  2791. out_nobuffer:
  2792. preempt_enable_notrace();
  2793. unpause_graph_tracing();
  2794. return len;
  2795. }
  2796. __printf(3, 0)
  2797. int trace_array_vprintk(struct trace_array *tr,
  2798. unsigned long ip, const char *fmt, va_list args)
  2799. {
  2800. return __trace_array_vprintk(tr->array_buffer.buffer, ip, fmt, args);
  2801. }
  2802. /**
  2803. * trace_array_printk - Print a message to a specific instance
  2804. * @tr: The instance trace_array descriptor
  2805. * @ip: The instruction pointer that this is called from.
  2806. * @fmt: The format to print (printf format)
  2807. *
  2808. * If a subsystem sets up its own instance, they have the right to
  2809. * printk strings into their tracing instance buffer using this
  2810. * function. Note, this function will not write into the top level
  2811. * buffer (use trace_printk() for that), as writing into the top level
  2812. * buffer should only have events that can be individually disabled.
  2813. * trace_printk() is only used for debugging a kernel, and should not
  2814. * be ever encorporated in normal use.
  2815. *
  2816. * trace_array_printk() can be used, as it will not add noise to the
  2817. * top level tracing buffer.
  2818. *
  2819. * Note, trace_array_init_printk() must be called on @tr before this
  2820. * can be used.
  2821. */
  2822. __printf(3, 0)
  2823. int trace_array_printk(struct trace_array *tr,
  2824. unsigned long ip, const char *fmt, ...)
  2825. {
  2826. int ret;
  2827. va_list ap;
  2828. if (!tr)
  2829. return -ENOENT;
  2830. /* This is only allowed for created instances */
  2831. if (tr == &global_trace)
  2832. return 0;
  2833. if (!(tr->trace_flags & TRACE_ITER_PRINTK))
  2834. return 0;
  2835. va_start(ap, fmt);
  2836. ret = trace_array_vprintk(tr, ip, fmt, ap);
  2837. va_end(ap);
  2838. return ret;
  2839. }
  2840. EXPORT_SYMBOL_GPL(trace_array_printk);
  2841. /**
  2842. * trace_array_init_printk - Initialize buffers for trace_array_printk()
  2843. * @tr: The trace array to initialize the buffers for
  2844. *
  2845. * As trace_array_printk() only writes into instances, they are OK to
  2846. * have in the kernel (unlike trace_printk()). This needs to be called
  2847. * before trace_array_printk() can be used on a trace_array.
  2848. */
  2849. int trace_array_init_printk(struct trace_array *tr)
  2850. {
  2851. if (!tr)
  2852. return -ENOENT;
  2853. /* This is only allowed for created instances */
  2854. if (tr == &global_trace)
  2855. return -EINVAL;
  2856. return alloc_percpu_trace_buffer();
  2857. }
  2858. EXPORT_SYMBOL_GPL(trace_array_init_printk);
  2859. __printf(3, 4)
  2860. int trace_array_printk_buf(struct trace_buffer *buffer,
  2861. unsigned long ip, const char *fmt, ...)
  2862. {
  2863. int ret;
  2864. va_list ap;
  2865. if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
  2866. return 0;
  2867. va_start(ap, fmt);
  2868. ret = __trace_array_vprintk(buffer, ip, fmt, ap);
  2869. va_end(ap);
  2870. return ret;
  2871. }
  2872. __printf(2, 0)
  2873. int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
  2874. {
  2875. return trace_array_vprintk(&global_trace, ip, fmt, args);
  2876. }
  2877. EXPORT_SYMBOL_GPL(trace_vprintk);
  2878. static void trace_iterator_increment(struct trace_iterator *iter)
  2879. {
  2880. struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
  2881. iter->idx++;
  2882. if (buf_iter)
  2883. ring_buffer_iter_advance(buf_iter);
  2884. }
  2885. static struct trace_entry *
  2886. peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
  2887. unsigned long *lost_events)
  2888. {
  2889. struct ring_buffer_event *event;
  2890. struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
  2891. if (buf_iter) {
  2892. event = ring_buffer_iter_peek(buf_iter, ts);
  2893. if (lost_events)
  2894. *lost_events = ring_buffer_iter_dropped(buf_iter) ?
  2895. (unsigned long)-1 : 0;
  2896. } else {
  2897. event = ring_buffer_peek(iter->array_buffer->buffer, cpu, ts,
  2898. lost_events);
  2899. }
  2900. if (event) {
  2901. iter->ent_size = ring_buffer_event_length(event);
  2902. return ring_buffer_event_data(event);
  2903. }
  2904. iter->ent_size = 0;
  2905. return NULL;
  2906. }
  2907. static struct trace_entry *
  2908. __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
  2909. unsigned long *missing_events, u64 *ent_ts)
  2910. {
  2911. struct trace_buffer *buffer = iter->array_buffer->buffer;
  2912. struct trace_entry *ent, *next = NULL;
  2913. unsigned long lost_events = 0, next_lost = 0;
  2914. int cpu_file = iter->cpu_file;
  2915. u64 next_ts = 0, ts;
  2916. int next_cpu = -1;
  2917. int next_size = 0;
  2918. int cpu;
  2919. /*
  2920. * If we are in a per_cpu trace file, don't bother by iterating over
  2921. * all cpu and peek directly.
  2922. */
  2923. if (cpu_file > RING_BUFFER_ALL_CPUS) {
  2924. if (ring_buffer_empty_cpu(buffer, cpu_file))
  2925. return NULL;
  2926. ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
  2927. if (ent_cpu)
  2928. *ent_cpu = cpu_file;
  2929. return ent;
  2930. }
  2931. for_each_tracing_cpu(cpu) {
  2932. if (ring_buffer_empty_cpu(buffer, cpu))
  2933. continue;
  2934. ent = peek_next_entry(iter, cpu, &ts, &lost_events);
  2935. /*
  2936. * Pick the entry with the smallest timestamp:
  2937. */
  2938. if (ent && (!next || ts < next_ts)) {
  2939. next = ent;
  2940. next_cpu = cpu;
  2941. next_ts = ts;
  2942. next_lost = lost_events;
  2943. next_size = iter->ent_size;
  2944. }
  2945. }
  2946. iter->ent_size = next_size;
  2947. if (ent_cpu)
  2948. *ent_cpu = next_cpu;
  2949. if (ent_ts)
  2950. *ent_ts = next_ts;
  2951. if (missing_events)
  2952. *missing_events = next_lost;
  2953. return next;
  2954. }
  2955. #define STATIC_TEMP_BUF_SIZE 128
  2956. static char static_temp_buf[STATIC_TEMP_BUF_SIZE] __aligned(4);
  2957. /* Find the next real entry, without updating the iterator itself */
  2958. struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
  2959. int *ent_cpu, u64 *ent_ts)
  2960. {
  2961. /* __find_next_entry will reset ent_size */
  2962. int ent_size = iter->ent_size;
  2963. struct trace_entry *entry;
  2964. /*
  2965. * If called from ftrace_dump(), then the iter->temp buffer
  2966. * will be the static_temp_buf and not created from kmalloc.
  2967. * If the entry size is greater than the buffer, we can
  2968. * not save it. Just return NULL in that case. This is only
  2969. * used to add markers when two consecutive events' time
  2970. * stamps have a large delta. See trace_print_lat_context()
  2971. */
  2972. if (iter->temp == static_temp_buf &&
  2973. STATIC_TEMP_BUF_SIZE < ent_size)
  2974. return NULL;
  2975. /*
  2976. * The __find_next_entry() may call peek_next_entry(), which may
  2977. * call ring_buffer_peek() that may make the contents of iter->ent
  2978. * undefined. Need to copy iter->ent now.
  2979. */
  2980. if (iter->ent && iter->ent != iter->temp) {
  2981. if ((!iter->temp || iter->temp_size < iter->ent_size) &&
  2982. !WARN_ON_ONCE(iter->temp == static_temp_buf)) {
  2983. void *temp;
  2984. temp = kmalloc(iter->ent_size, GFP_KERNEL);
  2985. if (!temp)
  2986. return NULL;
  2987. kfree(iter->temp);
  2988. iter->temp = temp;
  2989. iter->temp_size = iter->ent_size;
  2990. }
  2991. memcpy(iter->temp, iter->ent, iter->ent_size);
  2992. iter->ent = iter->temp;
  2993. }
  2994. entry = __find_next_entry(iter, ent_cpu, NULL, ent_ts);
  2995. /* Put back the original ent_size */
  2996. iter->ent_size = ent_size;
  2997. return entry;
  2998. }
  2999. /* Find the next real entry, and increment the iterator to the next entry */
  3000. void *trace_find_next_entry_inc(struct trace_iterator *iter)
  3001. {
  3002. iter->ent = __find_next_entry(iter, &iter->cpu,
  3003. &iter->lost_events, &iter->ts);
  3004. if (iter->ent)
  3005. trace_iterator_increment(iter);
  3006. return iter->ent ? iter : NULL;
  3007. }
  3008. static void trace_consume(struct trace_iterator *iter)
  3009. {
  3010. ring_buffer_consume(iter->array_buffer->buffer, iter->cpu, &iter->ts,
  3011. &iter->lost_events);
  3012. }
  3013. static void *s_next(struct seq_file *m, void *v, loff_t *pos)
  3014. {
  3015. struct trace_iterator *iter = m->private;
  3016. int i = (int)*pos;
  3017. void *ent;
  3018. WARN_ON_ONCE(iter->leftover);
  3019. (*pos)++;
  3020. /* can't go backwards */
  3021. if (iter->idx > i)
  3022. return NULL;
  3023. if (iter->idx < 0)
  3024. ent = trace_find_next_entry_inc(iter);
  3025. else
  3026. ent = iter;
  3027. while (ent && iter->idx < i)
  3028. ent = trace_find_next_entry_inc(iter);
  3029. iter->pos = *pos;
  3030. return ent;
  3031. }
  3032. void tracing_iter_reset(struct trace_iterator *iter, int cpu)
  3033. {
  3034. struct ring_buffer_iter *buf_iter;
  3035. unsigned long entries = 0;
  3036. u64 ts;
  3037. per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = 0;
  3038. buf_iter = trace_buffer_iter(iter, cpu);
  3039. if (!buf_iter)
  3040. return;
  3041. ring_buffer_iter_reset(buf_iter);
  3042. /*
  3043. * We could have the case with the max latency tracers
  3044. * that a reset never took place on a cpu. This is evident
  3045. * by the timestamp being before the start of the buffer.
  3046. */
  3047. while (ring_buffer_iter_peek(buf_iter, &ts)) {
  3048. if (ts >= iter->array_buffer->time_start)
  3049. break;
  3050. entries++;
  3051. ring_buffer_iter_advance(buf_iter);
  3052. }
  3053. per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = entries;
  3054. }
  3055. /*
  3056. * The current tracer is copied to avoid a global locking
  3057. * all around.
  3058. */
  3059. static void *s_start(struct seq_file *m, loff_t *pos)
  3060. {
  3061. struct trace_iterator *iter = m->private;
  3062. struct trace_array *tr = iter->tr;
  3063. int cpu_file = iter->cpu_file;
  3064. void *p = NULL;
  3065. loff_t l = 0;
  3066. int cpu;
  3067. /*
  3068. * copy the tracer to avoid using a global lock all around.
  3069. * iter->trace is a copy of current_trace, the pointer to the
  3070. * name may be used instead of a strcmp(), as iter->trace->name
  3071. * will point to the same string as current_trace->name.
  3072. */
  3073. mutex_lock(&trace_types_lock);
  3074. if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
  3075. *iter->trace = *tr->current_trace;
  3076. mutex_unlock(&trace_types_lock);
  3077. #ifdef CONFIG_TRACER_MAX_TRACE
  3078. if (iter->snapshot && iter->trace->use_max_tr)
  3079. return ERR_PTR(-EBUSY);
  3080. #endif
  3081. if (*pos != iter->pos) {
  3082. iter->ent = NULL;
  3083. iter->cpu = 0;
  3084. iter->idx = -1;
  3085. if (cpu_file == RING_BUFFER_ALL_CPUS) {
  3086. for_each_tracing_cpu(cpu)
  3087. tracing_iter_reset(iter, cpu);
  3088. } else
  3089. tracing_iter_reset(iter, cpu_file);
  3090. iter->leftover = 0;
  3091. for (p = iter; p && l < *pos; p = s_next(m, p, &l))
  3092. ;
  3093. } else {
  3094. /*
  3095. * If we overflowed the seq_file before, then we want
  3096. * to just reuse the trace_seq buffer again.
  3097. */
  3098. if (iter->leftover)
  3099. p = iter;
  3100. else {
  3101. l = *pos - 1;
  3102. p = s_next(m, p, &l);
  3103. }
  3104. }
  3105. trace_event_read_lock();
  3106. trace_access_lock(cpu_file);
  3107. return p;
  3108. }
  3109. static void s_stop(struct seq_file *m, void *p)
  3110. {
  3111. struct trace_iterator *iter = m->private;
  3112. #ifdef CONFIG_TRACER_MAX_TRACE
  3113. if (iter->snapshot && iter->trace->use_max_tr)
  3114. return;
  3115. #endif
  3116. trace_access_unlock(iter->cpu_file);
  3117. trace_event_read_unlock();
  3118. }
  3119. static void
  3120. get_total_entries_cpu(struct array_buffer *buf, unsigned long *total,
  3121. unsigned long *entries, int cpu)
  3122. {
  3123. unsigned long count;
  3124. count = ring_buffer_entries_cpu(buf->buffer, cpu);
  3125. /*
  3126. * If this buffer has skipped entries, then we hold all
  3127. * entries for the trace and we need to ignore the
  3128. * ones before the time stamp.
  3129. */
  3130. if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
  3131. count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
  3132. /* total is the same as the entries */
  3133. *total = count;
  3134. } else
  3135. *total = count +
  3136. ring_buffer_overrun_cpu(buf->buffer, cpu);
  3137. *entries = count;
  3138. }
  3139. static void
  3140. get_total_entries(struct array_buffer *buf,
  3141. unsigned long *total, unsigned long *entries)
  3142. {
  3143. unsigned long t, e;
  3144. int cpu;
  3145. *total = 0;
  3146. *entries = 0;
  3147. for_each_tracing_cpu(cpu) {
  3148. get_total_entries_cpu(buf, &t, &e, cpu);
  3149. *total += t;
  3150. *entries += e;
  3151. }
  3152. }
  3153. unsigned long trace_total_entries_cpu(struct trace_array *tr, int cpu)
  3154. {
  3155. unsigned long total, entries;
  3156. if (!tr)
  3157. tr = &global_trace;
  3158. get_total_entries_cpu(&tr->array_buffer, &total, &entries, cpu);
  3159. return entries;
  3160. }
  3161. unsigned long trace_total_entries(struct trace_array *tr)
  3162. {
  3163. unsigned long total, entries;
  3164. if (!tr)
  3165. tr = &global_trace;
  3166. get_total_entries(&tr->array_buffer, &total, &entries);
  3167. return entries;
  3168. }
  3169. static void print_lat_help_header(struct seq_file *m)
  3170. {
  3171. seq_puts(m, "# _------=> CPU# \n"
  3172. "# / _-----=> irqs-off \n"
  3173. "# | / _----=> need-resched \n"
  3174. "# || / _---=> hardirq/softirq \n"
  3175. "# ||| / _--=> preempt-depth \n"
  3176. "# |||| / delay \n"
  3177. "# cmd pid ||||| time | caller \n"
  3178. "# \\ / ||||| \\ | / \n");
  3179. }
  3180. static void print_event_info(struct array_buffer *buf, struct seq_file *m)
  3181. {
  3182. unsigned long total;
  3183. unsigned long entries;
  3184. get_total_entries(buf, &total, &entries);
  3185. seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
  3186. entries, total, num_online_cpus());
  3187. seq_puts(m, "#\n");
  3188. }
  3189. static void print_func_help_header(struct array_buffer *buf, struct seq_file *m,
  3190. unsigned int flags)
  3191. {
  3192. bool tgid = flags & TRACE_ITER_RECORD_TGID;
  3193. print_event_info(buf, m);
  3194. seq_printf(m, "# TASK-PID %s CPU# TIMESTAMP FUNCTION\n", tgid ? " TGID " : "");
  3195. seq_printf(m, "# | | %s | | |\n", tgid ? " | " : "");
  3196. }
  3197. static void print_func_help_header_irq(struct array_buffer *buf, struct seq_file *m,
  3198. unsigned int flags)
  3199. {
  3200. bool tgid = flags & TRACE_ITER_RECORD_TGID;
  3201. const char *space = " ";
  3202. int prec = tgid ? 12 : 2;
  3203. print_event_info(buf, m);
  3204. seq_printf(m, "# %.*s _-----=> irqs-off\n", prec, space);
  3205. seq_printf(m, "# %.*s / _----=> need-resched\n", prec, space);
  3206. seq_printf(m, "# %.*s| / _---=> hardirq/softirq\n", prec, space);
  3207. seq_printf(m, "# %.*s|| / _--=> preempt-depth\n", prec, space);
  3208. seq_printf(m, "# %.*s||| / delay\n", prec, space);
  3209. seq_printf(m, "# TASK-PID %.*s CPU# |||| TIMESTAMP FUNCTION\n", prec, " TGID ");
  3210. seq_printf(m, "# | | %.*s | |||| | |\n", prec, " | ");
  3211. }
  3212. void
  3213. print_trace_header(struct seq_file *m, struct trace_iterator *iter)
  3214. {
  3215. unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
  3216. struct array_buffer *buf = iter->array_buffer;
  3217. struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
  3218. struct tracer *type = iter->trace;
  3219. unsigned long entries;
  3220. unsigned long total;
  3221. const char *name = "preemption";
  3222. name = type->name;
  3223. get_total_entries(buf, &total, &entries);
  3224. seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
  3225. name, UTS_RELEASE);
  3226. seq_puts(m, "# -----------------------------------"
  3227. "---------------------------------\n");
  3228. seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
  3229. " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
  3230. nsecs_to_usecs(data->saved_latency),
  3231. entries,
  3232. total,
  3233. buf->cpu,
  3234. #if defined(CONFIG_PREEMPT_NONE)
  3235. "server",
  3236. #elif defined(CONFIG_PREEMPT_VOLUNTARY)
  3237. "desktop",
  3238. #elif defined(CONFIG_PREEMPT)
  3239. "preempt",
  3240. #elif defined(CONFIG_PREEMPT_RT)
  3241. "preempt_rt",
  3242. #else
  3243. "unknown",
  3244. #endif
  3245. /* These are reserved for later use */
  3246. 0, 0, 0, 0);
  3247. #ifdef CONFIG_SMP
  3248. seq_printf(m, " #P:%d)\n", num_online_cpus());
  3249. #else
  3250. seq_puts(m, ")\n");
  3251. #endif
  3252. seq_puts(m, "# -----------------\n");
  3253. seq_printf(m, "# | task: %.16s-%d "
  3254. "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
  3255. data->comm, data->pid,
  3256. from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
  3257. data->policy, data->rt_priority);
  3258. seq_puts(m, "# -----------------\n");
  3259. if (data->critical_start) {
  3260. seq_puts(m, "# => started at: ");
  3261. seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
  3262. trace_print_seq(m, &iter->seq);
  3263. seq_puts(m, "\n# => ended at: ");
  3264. seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
  3265. trace_print_seq(m, &iter->seq);
  3266. seq_puts(m, "\n#\n");
  3267. }
  3268. seq_puts(m, "#\n");
  3269. }
  3270. static void test_cpu_buff_start(struct trace_iterator *iter)
  3271. {
  3272. struct trace_seq *s = &iter->seq;
  3273. struct trace_array *tr = iter->tr;
  3274. if (!(tr->trace_flags & TRACE_ITER_ANNOTATE))
  3275. return;
  3276. if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
  3277. return;
  3278. if (cpumask_available(iter->started) &&
  3279. cpumask_test_cpu(iter->cpu, iter->started))
  3280. return;
  3281. if (per_cpu_ptr(iter->array_buffer->data, iter->cpu)->skipped_entries)
  3282. return;
  3283. if (cpumask_available(iter->started))
  3284. cpumask_set_cpu(iter->cpu, iter->started);
  3285. /* Don't print started cpu buffer for the first entry of the trace */
  3286. if (iter->idx > 1)
  3287. trace_seq_printf(s, "##### CPU %u buffer started ####\n",
  3288. iter->cpu);
  3289. }
  3290. static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
  3291. {
  3292. struct trace_array *tr = iter->tr;
  3293. struct trace_seq *s = &iter->seq;
  3294. unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
  3295. struct trace_entry *entry;
  3296. struct trace_event *event;
  3297. entry = iter->ent;
  3298. test_cpu_buff_start(iter);
  3299. event = ftrace_find_event(entry->type);
  3300. if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
  3301. if (iter->iter_flags & TRACE_FILE_LAT_FMT)
  3302. trace_print_lat_context(iter);
  3303. else
  3304. trace_print_context(iter);
  3305. }
  3306. if (trace_seq_has_overflowed(s))
  3307. return TRACE_TYPE_PARTIAL_LINE;
  3308. if (event)
  3309. return event->funcs->trace(iter, sym_flags, event);
  3310. trace_seq_printf(s, "Unknown type %d\n", entry->type);
  3311. return trace_handle_return(s);
  3312. }
  3313. static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
  3314. {
  3315. struct trace_array *tr = iter->tr;
  3316. struct trace_seq *s = &iter->seq;
  3317. struct trace_entry *entry;
  3318. struct trace_event *event;
  3319. entry = iter->ent;
  3320. if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO)
  3321. trace_seq_printf(s, "%d %d %llu ",
  3322. entry->pid, iter->cpu, iter->ts);
  3323. if (trace_seq_has_overflowed(s))
  3324. return TRACE_TYPE_PARTIAL_LINE;
  3325. event = ftrace_find_event(entry->type);
  3326. if (event)
  3327. return event->funcs->raw(iter, 0, event);
  3328. trace_seq_printf(s, "%d ?\n", entry->type);
  3329. return trace_handle_return(s);
  3330. }
  3331. static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
  3332. {
  3333. struct trace_array *tr = iter->tr;
  3334. struct trace_seq *s = &iter->seq;
  3335. unsigned char newline = '\n';
  3336. struct trace_entry *entry;
  3337. struct trace_event *event;
  3338. entry = iter->ent;
  3339. if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
  3340. SEQ_PUT_HEX_FIELD(s, entry->pid);
  3341. SEQ_PUT_HEX_FIELD(s, iter->cpu);
  3342. SEQ_PUT_HEX_FIELD(s, iter->ts);
  3343. if (trace_seq_has_overflowed(s))
  3344. return TRACE_TYPE_PARTIAL_LINE;
  3345. }
  3346. event = ftrace_find_event(entry->type);
  3347. if (event) {
  3348. enum print_line_t ret = event->funcs->hex(iter, 0, event);
  3349. if (ret != TRACE_TYPE_HANDLED)
  3350. return ret;
  3351. }
  3352. SEQ_PUT_FIELD(s, newline);
  3353. return trace_handle_return(s);
  3354. }
  3355. static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
  3356. {
  3357. struct trace_array *tr = iter->tr;
  3358. struct trace_seq *s = &iter->seq;
  3359. struct trace_entry *entry;
  3360. struct trace_event *event;
  3361. entry = iter->ent;
  3362. if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
  3363. SEQ_PUT_FIELD(s, entry->pid);
  3364. SEQ_PUT_FIELD(s, iter->cpu);
  3365. SEQ_PUT_FIELD(s, iter->ts);
  3366. if (trace_seq_has_overflowed(s))
  3367. return TRACE_TYPE_PARTIAL_LINE;
  3368. }
  3369. event = ftrace_find_event(entry->type);
  3370. return event ? event->funcs->binary(iter, 0, event) :
  3371. TRACE_TYPE_HANDLED;
  3372. }
  3373. int trace_empty(struct trace_iterator *iter)
  3374. {
  3375. struct ring_buffer_iter *buf_iter;
  3376. int cpu;
  3377. /* If we are looking at one CPU buffer, only check that one */
  3378. if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
  3379. cpu = iter->cpu_file;
  3380. buf_iter = trace_buffer_iter(iter, cpu);
  3381. if (buf_iter) {
  3382. if (!ring_buffer_iter_empty(buf_iter))
  3383. return 0;
  3384. } else {
  3385. if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu))
  3386. return 0;
  3387. }
  3388. return 1;
  3389. }
  3390. for_each_tracing_cpu(cpu) {
  3391. buf_iter = trace_buffer_iter(iter, cpu);
  3392. if (buf_iter) {
  3393. if (!ring_buffer_iter_empty(buf_iter))
  3394. return 0;
  3395. } else {
  3396. if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu))
  3397. return 0;
  3398. }
  3399. }
  3400. return 1;
  3401. }
  3402. /* Called with trace_event_read_lock() held. */
  3403. enum print_line_t print_trace_line(struct trace_iterator *iter)
  3404. {
  3405. struct trace_array *tr = iter->tr;
  3406. unsigned long trace_flags = tr->trace_flags;
  3407. enum print_line_t ret;
  3408. if (iter->lost_events) {
  3409. if (iter->lost_events == (unsigned long)-1)
  3410. trace_seq_printf(&iter->seq, "CPU:%d [LOST EVENTS]\n",
  3411. iter->cpu);
  3412. else
  3413. trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
  3414. iter->cpu, iter->lost_events);
  3415. if (trace_seq_has_overflowed(&iter->seq))
  3416. return TRACE_TYPE_PARTIAL_LINE;
  3417. }
  3418. if (iter->trace && iter->trace->print_line) {
  3419. ret = iter->trace->print_line(iter);
  3420. if (ret != TRACE_TYPE_UNHANDLED)
  3421. return ret;
  3422. }
  3423. if (iter->ent->type == TRACE_BPUTS &&
  3424. trace_flags & TRACE_ITER_PRINTK &&
  3425. trace_flags & TRACE_ITER_PRINTK_MSGONLY)
  3426. return trace_print_bputs_msg_only(iter);
  3427. if (iter->ent->type == TRACE_BPRINT &&
  3428. trace_flags & TRACE_ITER_PRINTK &&
  3429. trace_flags & TRACE_ITER_PRINTK_MSGONLY)
  3430. return trace_print_bprintk_msg_only(iter);
  3431. if (iter->ent->type == TRACE_PRINT &&
  3432. trace_flags & TRACE_ITER_PRINTK &&
  3433. trace_flags & TRACE_ITER_PRINTK_MSGONLY)
  3434. return trace_print_printk_msg_only(iter);
  3435. if (trace_flags & TRACE_ITER_BIN)
  3436. return print_bin_fmt(iter);
  3437. if (trace_flags & TRACE_ITER_HEX)
  3438. return print_hex_fmt(iter);
  3439. if (trace_flags & TRACE_ITER_RAW)
  3440. return print_raw_fmt(iter);
  3441. return print_trace_fmt(iter);
  3442. }
  3443. void trace_latency_header(struct seq_file *m)
  3444. {
  3445. struct trace_iterator *iter = m->private;
  3446. struct trace_array *tr = iter->tr;
  3447. /* print nothing if the buffers are empty */
  3448. if (trace_empty(iter))
  3449. return;
  3450. if (iter->iter_flags & TRACE_FILE_LAT_FMT)
  3451. print_trace_header(m, iter);
  3452. if (!(tr->trace_flags & TRACE_ITER_VERBOSE))
  3453. print_lat_help_header(m);
  3454. }
  3455. void trace_default_header(struct seq_file *m)
  3456. {
  3457. struct trace_iterator *iter = m->private;
  3458. struct trace_array *tr = iter->tr;
  3459. unsigned long trace_flags = tr->trace_flags;
  3460. if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
  3461. return;
  3462. if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
  3463. /* print nothing if the buffers are empty */
  3464. if (trace_empty(iter))
  3465. return;
  3466. print_trace_header(m, iter);
  3467. if (!(trace_flags & TRACE_ITER_VERBOSE))
  3468. print_lat_help_header(m);
  3469. } else {
  3470. if (!(trace_flags & TRACE_ITER_VERBOSE)) {
  3471. if (trace_flags & TRACE_ITER_IRQ_INFO)
  3472. print_func_help_header_irq(iter->array_buffer,
  3473. m, trace_flags);
  3474. else
  3475. print_func_help_header(iter->array_buffer, m,
  3476. trace_flags);
  3477. }
  3478. }
  3479. }
  3480. static void test_ftrace_alive(struct seq_file *m)
  3481. {
  3482. if (!ftrace_is_dead())
  3483. return;
  3484. seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
  3485. "# MAY BE MISSING FUNCTION EVENTS\n");
  3486. }
  3487. #ifdef CONFIG_TRACER_MAX_TRACE
  3488. static void show_snapshot_main_help(struct seq_file *m)
  3489. {
  3490. seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
  3491. "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
  3492. "# Takes a snapshot of the main buffer.\n"
  3493. "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
  3494. "# (Doesn't have to be '2' works with any number that\n"
  3495. "# is not a '0' or '1')\n");
  3496. }
  3497. static void show_snapshot_percpu_help(struct seq_file *m)
  3498. {
  3499. seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
  3500. #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
  3501. seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
  3502. "# Takes a snapshot of the main buffer for this cpu.\n");
  3503. #else
  3504. seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
  3505. "# Must use main snapshot file to allocate.\n");
  3506. #endif
  3507. seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
  3508. "# (Doesn't have to be '2' works with any number that\n"
  3509. "# is not a '0' or '1')\n");
  3510. }
  3511. static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
  3512. {
  3513. if (iter->tr->allocated_snapshot)
  3514. seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
  3515. else
  3516. seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
  3517. seq_puts(m, "# Snapshot commands:\n");
  3518. if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
  3519. show_snapshot_main_help(m);
  3520. else
  3521. show_snapshot_percpu_help(m);
  3522. }
  3523. #else
  3524. /* Should never be called */
  3525. static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
  3526. #endif
  3527. static int s_show(struct seq_file *m, void *v)
  3528. {
  3529. struct trace_iterator *iter = v;
  3530. int ret;
  3531. if (iter->ent == NULL) {
  3532. if (iter->tr) {
  3533. seq_printf(m, "# tracer: %s\n", iter->trace->name);
  3534. seq_puts(m, "#\n");
  3535. test_ftrace_alive(m);
  3536. }
  3537. if (iter->snapshot && trace_empty(iter))
  3538. print_snapshot_help(m, iter);
  3539. else if (iter->trace && iter->trace->print_header)
  3540. iter->trace->print_header(m);
  3541. else
  3542. trace_default_header(m);
  3543. } else if (iter->leftover) {
  3544. /*
  3545. * If we filled the seq_file buffer earlier, we
  3546. * want to just show it now.
  3547. */
  3548. ret = trace_print_seq(m, &iter->seq);
  3549. /* ret should this time be zero, but you never know */
  3550. iter->leftover = ret;
  3551. } else {
  3552. print_trace_line(iter);
  3553. ret = trace_print_seq(m, &iter->seq);
  3554. /*
  3555. * If we overflow the seq_file buffer, then it will
  3556. * ask us for this data again at start up.
  3557. * Use that instead.
  3558. * ret is 0 if seq_file write succeeded.
  3559. * -1 otherwise.
  3560. */
  3561. iter->leftover = ret;
  3562. }
  3563. return 0;
  3564. }
  3565. /*
  3566. * Should be used after trace_array_get(), trace_types_lock
  3567. * ensures that i_cdev was already initialized.
  3568. */
  3569. static inline int tracing_get_cpu(struct inode *inode)
  3570. {
  3571. if (inode->i_cdev) /* See trace_create_cpu_file() */
  3572. return (long)inode->i_cdev - 1;
  3573. return RING_BUFFER_ALL_CPUS;
  3574. }
  3575. static const struct seq_operations tracer_seq_ops = {
  3576. .start = s_start,
  3577. .next = s_next,
  3578. .stop = s_stop,
  3579. .show = s_show,
  3580. };
  3581. static struct trace_iterator *
  3582. __tracing_open(struct inode *inode, struct file *file, bool snapshot)
  3583. {
  3584. struct trace_array *tr = inode->i_private;
  3585. struct trace_iterator *iter;
  3586. int cpu;
  3587. if (tracing_disabled)
  3588. return ERR_PTR(-ENODEV);
  3589. iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
  3590. if (!iter)
  3591. return ERR_PTR(-ENOMEM);
  3592. iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter),
  3593. GFP_KERNEL);
  3594. if (!iter->buffer_iter)
  3595. goto release;
  3596. /*
  3597. * trace_find_next_entry() may need to save off iter->ent.
  3598. * It will place it into the iter->temp buffer. As most
  3599. * events are less than 128, allocate a buffer of that size.
  3600. * If one is greater, then trace_find_next_entry() will
  3601. * allocate a new buffer to adjust for the bigger iter->ent.
  3602. * It's not critical if it fails to get allocated here.
  3603. */
  3604. iter->temp = kmalloc(128, GFP_KERNEL);
  3605. if (iter->temp)
  3606. iter->temp_size = 128;
  3607. /*
  3608. * We make a copy of the current tracer to avoid concurrent
  3609. * changes on it while we are reading.
  3610. */
  3611. mutex_lock(&trace_types_lock);
  3612. iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
  3613. if (!iter->trace)
  3614. goto fail;
  3615. *iter->trace = *tr->current_trace;
  3616. if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
  3617. goto fail;
  3618. iter->tr = tr;
  3619. #ifdef CONFIG_TRACER_MAX_TRACE
  3620. /* Currently only the top directory has a snapshot */
  3621. if (tr->current_trace->print_max || snapshot)
  3622. iter->array_buffer = &tr->max_buffer;
  3623. else
  3624. #endif
  3625. iter->array_buffer = &tr->array_buffer;
  3626. iter->snapshot = snapshot;
  3627. iter->pos = -1;
  3628. iter->cpu_file = tracing_get_cpu(inode);
  3629. mutex_init(&iter->mutex);
  3630. /* Notify the tracer early; before we stop tracing. */
  3631. if (iter->trace->open)
  3632. iter->trace->open(iter);
  3633. /* Annotate start of buffers if we had overruns */
  3634. if (ring_buffer_overruns(iter->array_buffer->buffer))
  3635. iter->iter_flags |= TRACE_FILE_ANNOTATE;
  3636. /* Output in nanoseconds only if we are using a clock in nanoseconds. */
  3637. if (trace_clocks[tr->clock_id].in_ns)
  3638. iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
  3639. /*
  3640. * If pause-on-trace is enabled, then stop the trace while
  3641. * dumping, unless this is the "snapshot" file
  3642. */
  3643. if (!iter->snapshot && (tr->trace_flags & TRACE_ITER_PAUSE_ON_TRACE))
  3644. tracing_stop_tr(tr);
  3645. if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
  3646. for_each_tracing_cpu(cpu) {
  3647. iter->buffer_iter[cpu] =
  3648. ring_buffer_read_prepare(iter->array_buffer->buffer,
  3649. cpu, GFP_KERNEL);
  3650. }
  3651. ring_buffer_read_prepare_sync();
  3652. for_each_tracing_cpu(cpu) {
  3653. ring_buffer_read_start(iter->buffer_iter[cpu]);
  3654. tracing_iter_reset(iter, cpu);
  3655. }
  3656. } else {
  3657. cpu = iter->cpu_file;
  3658. iter->buffer_iter[cpu] =
  3659. ring_buffer_read_prepare(iter->array_buffer->buffer,
  3660. cpu, GFP_KERNEL);
  3661. ring_buffer_read_prepare_sync();
  3662. ring_buffer_read_start(iter->buffer_iter[cpu]);
  3663. tracing_iter_reset(iter, cpu);
  3664. }
  3665. mutex_unlock(&trace_types_lock);
  3666. return iter;
  3667. fail:
  3668. mutex_unlock(&trace_types_lock);
  3669. kfree(iter->trace);
  3670. kfree(iter->temp);
  3671. kfree(iter->buffer_iter);
  3672. release:
  3673. seq_release_private(inode, file);
  3674. return ERR_PTR(-ENOMEM);
  3675. }
  3676. int tracing_open_generic(struct inode *inode, struct file *filp)
  3677. {
  3678. int ret;
  3679. ret = tracing_check_open_get_tr(NULL);
  3680. if (ret)
  3681. return ret;
  3682. filp->private_data = inode->i_private;
  3683. return 0;
  3684. }
  3685. bool tracing_is_disabled(void)
  3686. {
  3687. return (tracing_disabled) ? true: false;
  3688. }
  3689. /*
  3690. * Open and update trace_array ref count.
  3691. * Must have the current trace_array passed to it.
  3692. */
  3693. int tracing_open_generic_tr(struct inode *inode, struct file *filp)
  3694. {
  3695. struct trace_array *tr = inode->i_private;
  3696. int ret;
  3697. ret = tracing_check_open_get_tr(tr);
  3698. if (ret)
  3699. return ret;
  3700. filp->private_data = inode->i_private;
  3701. return 0;
  3702. }
  3703. static int tracing_release(struct inode *inode, struct file *file)
  3704. {
  3705. struct trace_array *tr = inode->i_private;
  3706. struct seq_file *m = file->private_data;
  3707. struct trace_iterator *iter;
  3708. int cpu;
  3709. if (!(file->f_mode & FMODE_READ)) {
  3710. trace_array_put(tr);
  3711. return 0;
  3712. }
  3713. /* Writes do not use seq_file */
  3714. iter = m->private;
  3715. mutex_lock(&trace_types_lock);
  3716. for_each_tracing_cpu(cpu) {
  3717. if (iter->buffer_iter[cpu])
  3718. ring_buffer_read_finish(iter->buffer_iter[cpu]);
  3719. }
  3720. if (iter->trace && iter->trace->close)
  3721. iter->trace->close(iter);
  3722. if (!iter->snapshot && tr->stop_count)
  3723. /* reenable tracing if it was previously enabled */
  3724. tracing_start_tr(tr);
  3725. __trace_array_put(tr);
  3726. mutex_unlock(&trace_types_lock);
  3727. mutex_destroy(&iter->mutex);
  3728. free_cpumask_var(iter->started);
  3729. kfree(iter->temp);
  3730. kfree(iter->trace);
  3731. kfree(iter->buffer_iter);
  3732. seq_release_private(inode, file);
  3733. return 0;
  3734. }
  3735. static int tracing_release_generic_tr(struct inode *inode, struct file *file)
  3736. {
  3737. struct trace_array *tr = inode->i_private;
  3738. trace_array_put(tr);
  3739. return 0;
  3740. }
  3741. static int tracing_single_release_tr(struct inode *inode, struct file *file)
  3742. {
  3743. struct trace_array *tr = inode->i_private;
  3744. trace_array_put(tr);
  3745. return single_release(inode, file);
  3746. }
  3747. static int tracing_open(struct inode *inode, struct file *file)
  3748. {
  3749. struct trace_array *tr = inode->i_private;
  3750. struct trace_iterator *iter;
  3751. int ret;
  3752. ret = tracing_check_open_get_tr(tr);
  3753. if (ret)
  3754. return ret;
  3755. /* If this file was open for write, then erase contents */
  3756. if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
  3757. int cpu = tracing_get_cpu(inode);
  3758. struct array_buffer *trace_buf = &tr->array_buffer;
  3759. #ifdef CONFIG_TRACER_MAX_TRACE
  3760. if (tr->current_trace->print_max)
  3761. trace_buf = &tr->max_buffer;
  3762. #endif
  3763. if (cpu == RING_BUFFER_ALL_CPUS)
  3764. tracing_reset_online_cpus(trace_buf);
  3765. else
  3766. tracing_reset_cpu(trace_buf, cpu);
  3767. }
  3768. if (file->f_mode & FMODE_READ) {
  3769. iter = __tracing_open(inode, file, false);
  3770. if (IS_ERR(iter))
  3771. ret = PTR_ERR(iter);
  3772. else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
  3773. iter->iter_flags |= TRACE_FILE_LAT_FMT;
  3774. }
  3775. if (ret < 0)
  3776. trace_array_put(tr);
  3777. return ret;
  3778. }
  3779. /*
  3780. * Some tracers are not suitable for instance buffers.
  3781. * A tracer is always available for the global array (toplevel)
  3782. * or if it explicitly states that it is.
  3783. */
  3784. static bool
  3785. trace_ok_for_array(struct tracer *t, struct trace_array *tr)
  3786. {
  3787. return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
  3788. }
  3789. /* Find the next tracer that this trace array may use */
  3790. static struct tracer *
  3791. get_tracer_for_array(struct trace_array *tr, struct tracer *t)
  3792. {
  3793. while (t && !trace_ok_for_array(t, tr))
  3794. t = t->next;
  3795. return t;
  3796. }
  3797. static void *
  3798. t_next(struct seq_file *m, void *v, loff_t *pos)
  3799. {
  3800. struct trace_array *tr = m->private;
  3801. struct tracer *t = v;
  3802. (*pos)++;
  3803. if (t)
  3804. t = get_tracer_for_array(tr, t->next);
  3805. return t;
  3806. }
  3807. static void *t_start(struct seq_file *m, loff_t *pos)
  3808. {
  3809. struct trace_array *tr = m->private;
  3810. struct tracer *t;
  3811. loff_t l = 0;
  3812. mutex_lock(&trace_types_lock);
  3813. t = get_tracer_for_array(tr, trace_types);
  3814. for (; t && l < *pos; t = t_next(m, t, &l))
  3815. ;
  3816. return t;
  3817. }
  3818. static void t_stop(struct seq_file *m, void *p)
  3819. {
  3820. mutex_unlock(&trace_types_lock);
  3821. }
  3822. static int t_show(struct seq_file *m, void *v)
  3823. {
  3824. struct tracer *t = v;
  3825. if (!t)
  3826. return 0;
  3827. seq_puts(m, t->name);
  3828. if (t->next)
  3829. seq_putc(m, ' ');
  3830. else
  3831. seq_putc(m, '\n');
  3832. return 0;
  3833. }
  3834. static const struct seq_operations show_traces_seq_ops = {
  3835. .start = t_start,
  3836. .next = t_next,
  3837. .stop = t_stop,
  3838. .show = t_show,
  3839. };
  3840. static int show_traces_open(struct inode *inode, struct file *file)
  3841. {
  3842. struct trace_array *tr = inode->i_private;
  3843. struct seq_file *m;
  3844. int ret;
  3845. ret = tracing_check_open_get_tr(tr);
  3846. if (ret)
  3847. return ret;
  3848. ret = seq_open(file, &show_traces_seq_ops);
  3849. if (ret) {
  3850. trace_array_put(tr);
  3851. return ret;
  3852. }
  3853. m = file->private_data;
  3854. m->private = tr;
  3855. return 0;
  3856. }
  3857. static int show_traces_release(struct inode *inode, struct file *file)
  3858. {
  3859. struct trace_array *tr = inode->i_private;
  3860. trace_array_put(tr);
  3861. return seq_release(inode, file);
  3862. }
  3863. static ssize_t
  3864. tracing_write_stub(struct file *filp, const char __user *ubuf,
  3865. size_t count, loff_t *ppos)
  3866. {
  3867. return count;
  3868. }
  3869. loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
  3870. {
  3871. int ret;
  3872. if (file->f_mode & FMODE_READ)
  3873. ret = seq_lseek(file, offset, whence);
  3874. else
  3875. file->f_pos = ret = 0;
  3876. return ret;
  3877. }
  3878. static const struct file_operations tracing_fops = {
  3879. .open = tracing_open,
  3880. .read = seq_read,
  3881. .write = tracing_write_stub,
  3882. .llseek = tracing_lseek,
  3883. .release = tracing_release,
  3884. };
  3885. static const struct file_operations show_traces_fops = {
  3886. .open = show_traces_open,
  3887. .read = seq_read,
  3888. .llseek = seq_lseek,
  3889. .release = show_traces_release,
  3890. };
  3891. static ssize_t
  3892. tracing_cpumask_read(struct file *filp, char __user *ubuf,
  3893. size_t count, loff_t *ppos)
  3894. {
  3895. struct trace_array *tr = file_inode(filp)->i_private;
  3896. char *mask_str;
  3897. int len;
  3898. len = snprintf(NULL, 0, "%*pb\n",
  3899. cpumask_pr_args(tr->tracing_cpumask)) + 1;
  3900. mask_str = kmalloc(len, GFP_KERNEL);
  3901. if (!mask_str)
  3902. return -ENOMEM;
  3903. len = snprintf(mask_str, len, "%*pb\n",
  3904. cpumask_pr_args(tr->tracing_cpumask));
  3905. if (len >= count) {
  3906. count = -EINVAL;
  3907. goto out_err;
  3908. }
  3909. count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
  3910. out_err:
  3911. kfree(mask_str);
  3912. return count;
  3913. }
  3914. int tracing_set_cpumask(struct trace_array *tr,
  3915. cpumask_var_t tracing_cpumask_new)
  3916. {
  3917. int cpu;
  3918. if (!tr)
  3919. return -EINVAL;
  3920. local_irq_disable();
  3921. arch_spin_lock(&tr->max_lock);
  3922. for_each_tracing_cpu(cpu) {
  3923. /*
  3924. * Increase/decrease the disabled counter if we are
  3925. * about to flip a bit in the cpumask:
  3926. */
  3927. if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
  3928. !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
  3929. atomic_inc(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled);
  3930. ring_buffer_record_disable_cpu(tr->array_buffer.buffer, cpu);
  3931. }
  3932. if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
  3933. cpumask_test_cpu(cpu, tracing_cpumask_new)) {
  3934. atomic_dec(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled);
  3935. ring_buffer_record_enable_cpu(tr->array_buffer.buffer, cpu);
  3936. }
  3937. }
  3938. arch_spin_unlock(&tr->max_lock);
  3939. local_irq_enable();
  3940. cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
  3941. return 0;
  3942. }
  3943. static ssize_t
  3944. tracing_cpumask_write(struct file *filp, const char __user *ubuf,
  3945. size_t count, loff_t *ppos)
  3946. {
  3947. struct trace_array *tr = file_inode(filp)->i_private;
  3948. cpumask_var_t tracing_cpumask_new;
  3949. int err;
  3950. if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
  3951. return -ENOMEM;
  3952. err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
  3953. if (err)
  3954. goto err_free;
  3955. err = tracing_set_cpumask(tr, tracing_cpumask_new);
  3956. if (err)
  3957. goto err_free;
  3958. free_cpumask_var(tracing_cpumask_new);
  3959. return count;
  3960. err_free:
  3961. free_cpumask_var(tracing_cpumask_new);
  3962. return err;
  3963. }
  3964. static const struct file_operations tracing_cpumask_fops = {
  3965. .open = tracing_open_generic_tr,
  3966. .read = tracing_cpumask_read,
  3967. .write = tracing_cpumask_write,
  3968. .release = tracing_release_generic_tr,
  3969. .llseek = generic_file_llseek,
  3970. };
  3971. static int tracing_trace_options_show(struct seq_file *m, void *v)
  3972. {
  3973. struct tracer_opt *trace_opts;
  3974. struct trace_array *tr = m->private;
  3975. u32 tracer_flags;
  3976. int i;
  3977. mutex_lock(&trace_types_lock);
  3978. tracer_flags = tr->current_trace->flags->val;
  3979. trace_opts = tr->current_trace->flags->opts;
  3980. for (i = 0; trace_options[i]; i++) {
  3981. if (tr->trace_flags & (1 << i))
  3982. seq_printf(m, "%s\n", trace_options[i]);
  3983. else
  3984. seq_printf(m, "no%s\n", trace_options[i]);
  3985. }
  3986. for (i = 0; trace_opts[i].name; i++) {
  3987. if (tracer_flags & trace_opts[i].bit)
  3988. seq_printf(m, "%s\n", trace_opts[i].name);
  3989. else
  3990. seq_printf(m, "no%s\n", trace_opts[i].name);
  3991. }
  3992. mutex_unlock(&trace_types_lock);
  3993. return 0;
  3994. }
  3995. static int __set_tracer_option(struct trace_array *tr,
  3996. struct tracer_flags *tracer_flags,
  3997. struct tracer_opt *opts, int neg)
  3998. {
  3999. struct tracer *trace = tracer_flags->trace;
  4000. int ret;
  4001. ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
  4002. if (ret)
  4003. return ret;
  4004. if (neg)
  4005. tracer_flags->val &= ~opts->bit;
  4006. else
  4007. tracer_flags->val |= opts->bit;
  4008. return 0;
  4009. }
  4010. /* Try to assign a tracer specific option */
  4011. static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
  4012. {
  4013. struct tracer *trace = tr->current_trace;
  4014. struct tracer_flags *tracer_flags = trace->flags;
  4015. struct tracer_opt *opts = NULL;
  4016. int i;
  4017. for (i = 0; tracer_flags->opts[i].name; i++) {
  4018. opts = &tracer_flags->opts[i];
  4019. if (strcmp(cmp, opts->name) == 0)
  4020. return __set_tracer_option(tr, trace->flags, opts, neg);
  4021. }
  4022. return -EINVAL;
  4023. }
  4024. /* Some tracers require overwrite to stay enabled */
  4025. int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
  4026. {
  4027. if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
  4028. return -1;
  4029. return 0;
  4030. }
  4031. int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
  4032. {
  4033. int *map;
  4034. if ((mask == TRACE_ITER_RECORD_TGID) ||
  4035. (mask == TRACE_ITER_RECORD_CMD))
  4036. lockdep_assert_held(&event_mutex);
  4037. /* do nothing if flag is already set */
  4038. if (!!(tr->trace_flags & mask) == !!enabled)
  4039. return 0;
  4040. /* Give the tracer a chance to approve the change */
  4041. if (tr->current_trace->flag_changed)
  4042. if (tr->current_trace->flag_changed(tr, mask, !!enabled))
  4043. return -EINVAL;
  4044. if (enabled)
  4045. tr->trace_flags |= mask;
  4046. else
  4047. tr->trace_flags &= ~mask;
  4048. if (mask == TRACE_ITER_RECORD_CMD)
  4049. trace_event_enable_cmd_record(enabled);
  4050. if (mask == TRACE_ITER_RECORD_TGID) {
  4051. if (!tgid_map) {
  4052. tgid_map_max = pid_max;
  4053. map = kvcalloc(tgid_map_max + 1, sizeof(*tgid_map),
  4054. GFP_KERNEL);
  4055. /*
  4056. * Pairs with smp_load_acquire() in
  4057. * trace_find_tgid_ptr() to ensure that if it observes
  4058. * the tgid_map we just allocated then it also observes
  4059. * the corresponding tgid_map_max value.
  4060. */
  4061. smp_store_release(&tgid_map, map);
  4062. }
  4063. if (!tgid_map) {
  4064. tr->trace_flags &= ~TRACE_ITER_RECORD_TGID;
  4065. return -ENOMEM;
  4066. }
  4067. trace_event_enable_tgid_record(enabled);
  4068. }
  4069. if (mask == TRACE_ITER_EVENT_FORK)
  4070. trace_event_follow_fork(tr, enabled);
  4071. if (mask == TRACE_ITER_FUNC_FORK)
  4072. ftrace_pid_follow_fork(tr, enabled);
  4073. if (mask == TRACE_ITER_OVERWRITE) {
  4074. ring_buffer_change_overwrite(tr->array_buffer.buffer, enabled);
  4075. #ifdef CONFIG_TRACER_MAX_TRACE
  4076. ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
  4077. #endif
  4078. }
  4079. if (mask == TRACE_ITER_PRINTK) {
  4080. trace_printk_start_stop_comm(enabled);
  4081. trace_printk_control(enabled);
  4082. }
  4083. return 0;
  4084. }
  4085. int trace_set_options(struct trace_array *tr, char *option)
  4086. {
  4087. char *cmp;
  4088. int neg = 0;
  4089. int ret;
  4090. size_t orig_len = strlen(option);
  4091. int len;
  4092. cmp = strstrip(option);
  4093. len = str_has_prefix(cmp, "no");
  4094. if (len)
  4095. neg = 1;
  4096. cmp += len;
  4097. mutex_lock(&event_mutex);
  4098. mutex_lock(&trace_types_lock);
  4099. ret = match_string(trace_options, -1, cmp);
  4100. /* If no option could be set, test the specific tracer options */
  4101. if (ret < 0)
  4102. ret = set_tracer_option(tr, cmp, neg);
  4103. else
  4104. ret = set_tracer_flag(tr, 1 << ret, !neg);
  4105. mutex_unlock(&trace_types_lock);
  4106. mutex_unlock(&event_mutex);
  4107. /*
  4108. * If the first trailing whitespace is replaced with '\0' by strstrip,
  4109. * turn it back into a space.
  4110. */
  4111. if (orig_len > strlen(option))
  4112. option[strlen(option)] = ' ';
  4113. return ret;
  4114. }
  4115. static void __init apply_trace_boot_options(void)
  4116. {
  4117. char *buf = trace_boot_options_buf;
  4118. char *option;
  4119. while (true) {
  4120. option = strsep(&buf, ",");
  4121. if (!option)
  4122. break;
  4123. if (*option)
  4124. trace_set_options(&global_trace, option);
  4125. /* Put back the comma to allow this to be called again */
  4126. if (buf)
  4127. *(buf - 1) = ',';
  4128. }
  4129. }
  4130. static ssize_t
  4131. tracing_trace_options_write(struct file *filp, const char __user *ubuf,
  4132. size_t cnt, loff_t *ppos)
  4133. {
  4134. struct seq_file *m = filp->private_data;
  4135. struct trace_array *tr = m->private;
  4136. char buf[64];
  4137. int ret;
  4138. if (cnt >= sizeof(buf))
  4139. return -EINVAL;
  4140. if (copy_from_user(buf, ubuf, cnt))
  4141. return -EFAULT;
  4142. buf[cnt] = 0;
  4143. ret = trace_set_options(tr, buf);
  4144. if (ret < 0)
  4145. return ret;
  4146. *ppos += cnt;
  4147. return cnt;
  4148. }
  4149. static int tracing_trace_options_open(struct inode *inode, struct file *file)
  4150. {
  4151. struct trace_array *tr = inode->i_private;
  4152. int ret;
  4153. ret = tracing_check_open_get_tr(tr);
  4154. if (ret)
  4155. return ret;
  4156. ret = single_open(file, tracing_trace_options_show, inode->i_private);
  4157. if (ret < 0)
  4158. trace_array_put(tr);
  4159. return ret;
  4160. }
  4161. static const struct file_operations tracing_iter_fops = {
  4162. .open = tracing_trace_options_open,
  4163. .read = seq_read,
  4164. .llseek = seq_lseek,
  4165. .release = tracing_single_release_tr,
  4166. .write = tracing_trace_options_write,
  4167. };
  4168. static const char readme_msg[] =
  4169. "tracing mini-HOWTO:\n\n"
  4170. "# echo 0 > tracing_on : quick way to disable tracing\n"
  4171. "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
  4172. " Important files:\n"
  4173. " trace\t\t\t- The static contents of the buffer\n"
  4174. "\t\t\t To clear the buffer write into this file: echo > trace\n"
  4175. " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
  4176. " current_tracer\t- function and latency tracers\n"
  4177. " available_tracers\t- list of configured tracers for current_tracer\n"
  4178. " error_log\t- error log for failed commands (that support it)\n"
  4179. " buffer_size_kb\t- view and modify size of per cpu buffer\n"
  4180. " buffer_total_size_kb - view total size of all cpu buffers\n\n"
  4181. " trace_clock\t\t-change the clock used to order events\n"
  4182. " local: Per cpu clock but may not be synced across CPUs\n"
  4183. " global: Synced across CPUs but slows tracing down.\n"
  4184. " counter: Not a clock, but just an increment\n"
  4185. " uptime: Jiffy counter from time of boot\n"
  4186. " perf: Same clock that perf events use\n"
  4187. #ifdef CONFIG_X86_64
  4188. " x86-tsc: TSC cycle counter\n"
  4189. #endif
  4190. "\n timestamp_mode\t-view the mode used to timestamp events\n"
  4191. " delta: Delta difference against a buffer-wide timestamp\n"
  4192. " absolute: Absolute (standalone) timestamp\n"
  4193. "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
  4194. "\n trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n"
  4195. " tracing_cpumask\t- Limit which CPUs to trace\n"
  4196. " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
  4197. "\t\t\t Remove sub-buffer with rmdir\n"
  4198. " trace_options\t\t- Set format or modify how tracing happens\n"
  4199. "\t\t\t Disable an option by prefixing 'no' to the\n"
  4200. "\t\t\t option name\n"
  4201. " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
  4202. #ifdef CONFIG_DYNAMIC_FTRACE
  4203. "\n available_filter_functions - list of functions that can be filtered on\n"
  4204. " set_ftrace_filter\t- echo function name in here to only trace these\n"
  4205. "\t\t\t functions\n"
  4206. "\t accepts: func_full_name or glob-matching-pattern\n"
  4207. "\t modules: Can select a group via module\n"
  4208. "\t Format: :mod:<module-name>\n"
  4209. "\t example: echo :mod:ext3 > set_ftrace_filter\n"
  4210. "\t triggers: a command to perform when function is hit\n"
  4211. "\t Format: <function>:<trigger>[:count]\n"
  4212. "\t trigger: traceon, traceoff\n"
  4213. "\t\t enable_event:<system>:<event>\n"
  4214. "\t\t disable_event:<system>:<event>\n"
  4215. #ifdef CONFIG_STACKTRACE
  4216. "\t\t stacktrace\n"
  4217. #endif
  4218. #ifdef CONFIG_TRACER_SNAPSHOT
  4219. "\t\t snapshot\n"
  4220. #endif
  4221. "\t\t dump\n"
  4222. "\t\t cpudump\n"
  4223. "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
  4224. "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
  4225. "\t The first one will disable tracing every time do_fault is hit\n"
  4226. "\t The second will disable tracing at most 3 times when do_trap is hit\n"
  4227. "\t The first time do trap is hit and it disables tracing, the\n"
  4228. "\t counter will decrement to 2. If tracing is already disabled,\n"
  4229. "\t the counter will not decrement. It only decrements when the\n"
  4230. "\t trigger did work\n"
  4231. "\t To remove trigger without count:\n"
  4232. "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
  4233. "\t To remove trigger with a count:\n"
  4234. "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
  4235. " set_ftrace_notrace\t- echo function name in here to never trace.\n"
  4236. "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
  4237. "\t modules: Can select a group via module command :mod:\n"
  4238. "\t Does not accept triggers\n"
  4239. #endif /* CONFIG_DYNAMIC_FTRACE */
  4240. #ifdef CONFIG_FUNCTION_TRACER
  4241. " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
  4242. "\t\t (function)\n"
  4243. " set_ftrace_notrace_pid\t- Write pid(s) to not function trace those pids\n"
  4244. "\t\t (function)\n"
  4245. #endif
  4246. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  4247. " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
  4248. " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
  4249. " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
  4250. #endif
  4251. #ifdef CONFIG_TRACER_SNAPSHOT
  4252. "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
  4253. "\t\t\t snapshot buffer. Read the contents for more\n"
  4254. "\t\t\t information\n"
  4255. #endif
  4256. #ifdef CONFIG_STACK_TRACER
  4257. " stack_trace\t\t- Shows the max stack trace when active\n"
  4258. " stack_max_size\t- Shows current max stack size that was traced\n"
  4259. "\t\t\t Write into this file to reset the max size (trigger a\n"
  4260. "\t\t\t new trace)\n"
  4261. #ifdef CONFIG_DYNAMIC_FTRACE
  4262. " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
  4263. "\t\t\t traces\n"
  4264. #endif
  4265. #endif /* CONFIG_STACK_TRACER */
  4266. #ifdef CONFIG_DYNAMIC_EVENTS
  4267. " dynamic_events\t\t- Create/append/remove/show the generic dynamic events\n"
  4268. "\t\t\t Write into this file to define/undefine new trace events.\n"
  4269. #endif
  4270. #ifdef CONFIG_KPROBE_EVENTS
  4271. " kprobe_events\t\t- Create/append/remove/show the kernel dynamic events\n"
  4272. "\t\t\t Write into this file to define/undefine new trace events.\n"
  4273. #endif
  4274. #ifdef CONFIG_UPROBE_EVENTS
  4275. " uprobe_events\t\t- Create/append/remove/show the userspace dynamic events\n"
  4276. "\t\t\t Write into this file to define/undefine new trace events.\n"
  4277. #endif
  4278. #if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS)
  4279. "\t accepts: event-definitions (one definition per line)\n"
  4280. "\t Format: p[:[<group>/]<event>] <place> [<args>]\n"
  4281. "\t r[maxactive][:[<group>/]<event>] <place> [<args>]\n"
  4282. #ifdef CONFIG_HIST_TRIGGERS
  4283. "\t s:[synthetic/]<event> <field> [<field>]\n"
  4284. #endif
  4285. "\t -:[<group>/]<event>\n"
  4286. #ifdef CONFIG_KPROBE_EVENTS
  4287. "\t place: [<module>:]<symbol>[+<offset>]|<memaddr>\n"
  4288. "place (kretprobe): [<module>:]<symbol>[+<offset>]%return|<memaddr>\n"
  4289. #endif
  4290. #ifdef CONFIG_UPROBE_EVENTS
  4291. " place (uprobe): <path>:<offset>[%return][(ref_ctr_offset)]\n"
  4292. #endif
  4293. "\t args: <name>=fetcharg[:type]\n"
  4294. "\t fetcharg: %<register>, @<address>, @<symbol>[+|-<offset>],\n"
  4295. #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
  4296. "\t $stack<index>, $stack, $retval, $comm, $arg<N>,\n"
  4297. #else
  4298. "\t $stack<index>, $stack, $retval, $comm,\n"
  4299. #endif
  4300. "\t +|-[u]<offset>(<fetcharg>), \\imm-value, \\\"imm-string\"\n"
  4301. "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string, symbol,\n"
  4302. "\t b<bit-width>@<bit-offset>/<container-size>, ustring,\n"
  4303. "\t <type>\\[<array-size>\\]\n"
  4304. #ifdef CONFIG_HIST_TRIGGERS
  4305. "\t field: <stype> <name>;\n"
  4306. "\t stype: u8/u16/u32/u64, s8/s16/s32/s64, pid_t,\n"
  4307. "\t [unsigned] char/int/long\n"
  4308. #endif
  4309. #endif
  4310. " events/\t\t- Directory containing all trace event subsystems:\n"
  4311. " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
  4312. " events/<system>/\t- Directory containing all trace events for <system>:\n"
  4313. " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
  4314. "\t\t\t events\n"
  4315. " filter\t\t- If set, only events passing filter are traced\n"
  4316. " events/<system>/<event>/\t- Directory containing control files for\n"
  4317. "\t\t\t <event>:\n"
  4318. " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
  4319. " filter\t\t- If set, only events passing filter are traced\n"
  4320. " trigger\t\t- If set, a command to perform when event is hit\n"
  4321. "\t Format: <trigger>[:count][if <filter>]\n"
  4322. "\t trigger: traceon, traceoff\n"
  4323. "\t enable_event:<system>:<event>\n"
  4324. "\t disable_event:<system>:<event>\n"
  4325. #ifdef CONFIG_HIST_TRIGGERS
  4326. "\t enable_hist:<system>:<event>\n"
  4327. "\t disable_hist:<system>:<event>\n"
  4328. #endif
  4329. #ifdef CONFIG_STACKTRACE
  4330. "\t\t stacktrace\n"
  4331. #endif
  4332. #ifdef CONFIG_TRACER_SNAPSHOT
  4333. "\t\t snapshot\n"
  4334. #endif
  4335. #ifdef CONFIG_HIST_TRIGGERS
  4336. "\t\t hist (see below)\n"
  4337. #endif
  4338. "\t example: echo traceoff > events/block/block_unplug/trigger\n"
  4339. "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
  4340. "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
  4341. "\t events/block/block_unplug/trigger\n"
  4342. "\t The first disables tracing every time block_unplug is hit.\n"
  4343. "\t The second disables tracing the first 3 times block_unplug is hit.\n"
  4344. "\t The third enables the kmalloc event the first 3 times block_unplug\n"
  4345. "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
  4346. "\t Like function triggers, the counter is only decremented if it\n"
  4347. "\t enabled or disabled tracing.\n"
  4348. "\t To remove a trigger without a count:\n"
  4349. "\t echo '!<trigger> > <system>/<event>/trigger\n"
  4350. "\t To remove a trigger with a count:\n"
  4351. "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
  4352. "\t Filters can be ignored when removing a trigger.\n"
  4353. #ifdef CONFIG_HIST_TRIGGERS
  4354. " hist trigger\t- If set, event hits are aggregated into a hash table\n"
  4355. "\t Format: hist:keys=<field1[,field2,...]>\n"
  4356. "\t [:values=<field1[,field2,...]>]\n"
  4357. "\t [:sort=<field1[,field2,...]>]\n"
  4358. "\t [:size=#entries]\n"
  4359. "\t [:pause][:continue][:clear]\n"
  4360. "\t [:name=histname1]\n"
  4361. "\t [:<handler>.<action>]\n"
  4362. "\t [if <filter>]\n\n"
  4363. "\t Note, special fields can be used as well:\n"
  4364. "\t common_timestamp - to record current timestamp\n"
  4365. "\t common_cpu - to record the CPU the event happened on\n"
  4366. "\n"
  4367. "\t When a matching event is hit, an entry is added to a hash\n"
  4368. "\t table using the key(s) and value(s) named, and the value of a\n"
  4369. "\t sum called 'hitcount' is incremented. Keys and values\n"
  4370. "\t correspond to fields in the event's format description. Keys\n"
  4371. "\t can be any field, or the special string 'stacktrace'.\n"
  4372. "\t Compound keys consisting of up to two fields can be specified\n"
  4373. "\t by the 'keys' keyword. Values must correspond to numeric\n"
  4374. "\t fields. Sort keys consisting of up to two fields can be\n"
  4375. "\t specified using the 'sort' keyword. The sort direction can\n"
  4376. "\t be modified by appending '.descending' or '.ascending' to a\n"
  4377. "\t sort field. The 'size' parameter can be used to specify more\n"
  4378. "\t or fewer than the default 2048 entries for the hashtable size.\n"
  4379. "\t If a hist trigger is given a name using the 'name' parameter,\n"
  4380. "\t its histogram data will be shared with other triggers of the\n"
  4381. "\t same name, and trigger hits will update this common data.\n\n"
  4382. "\t Reading the 'hist' file for the event will dump the hash\n"
  4383. "\t table in its entirety to stdout. If there are multiple hist\n"
  4384. "\t triggers attached to an event, there will be a table for each\n"
  4385. "\t trigger in the output. The table displayed for a named\n"
  4386. "\t trigger will be the same as any other instance having the\n"
  4387. "\t same name. The default format used to display a given field\n"
  4388. "\t can be modified by appending any of the following modifiers\n"
  4389. "\t to the field name, as applicable:\n\n"
  4390. "\t .hex display a number as a hex value\n"
  4391. "\t .sym display an address as a symbol\n"
  4392. "\t .sym-offset display an address as a symbol and offset\n"
  4393. "\t .execname display a common_pid as a program name\n"
  4394. "\t .syscall display a syscall id as a syscall name\n"
  4395. "\t .log2 display log2 value rather than raw number\n"
  4396. "\t .usecs display a common_timestamp in microseconds\n\n"
  4397. "\t The 'pause' parameter can be used to pause an existing hist\n"
  4398. "\t trigger or to start a hist trigger but not log any events\n"
  4399. "\t until told to do so. 'continue' can be used to start or\n"
  4400. "\t restart a paused hist trigger.\n\n"
  4401. "\t The 'clear' parameter will clear the contents of a running\n"
  4402. "\t hist trigger and leave its current paused/active state\n"
  4403. "\t unchanged.\n\n"
  4404. "\t The enable_hist and disable_hist triggers can be used to\n"
  4405. "\t have one event conditionally start and stop another event's\n"
  4406. "\t already-attached hist trigger. The syntax is analogous to\n"
  4407. "\t the enable_event and disable_event triggers.\n\n"
  4408. "\t Hist trigger handlers and actions are executed whenever a\n"
  4409. "\t a histogram entry is added or updated. They take the form:\n\n"
  4410. "\t <handler>.<action>\n\n"
  4411. "\t The available handlers are:\n\n"
  4412. "\t onmatch(matching.event) - invoke on addition or update\n"
  4413. "\t onmax(var) - invoke if var exceeds current max\n"
  4414. "\t onchange(var) - invoke action if var changes\n\n"
  4415. "\t The available actions are:\n\n"
  4416. "\t trace(<synthetic_event>,param list) - generate synthetic event\n"
  4417. "\t save(field,...) - save current event fields\n"
  4418. #ifdef CONFIG_TRACER_SNAPSHOT
  4419. "\t snapshot() - snapshot the trace buffer\n\n"
  4420. #endif
  4421. #ifdef CONFIG_SYNTH_EVENTS
  4422. " events/synthetic_events\t- Create/append/remove/show synthetic events\n"
  4423. "\t Write into this file to define/undefine new synthetic events.\n"
  4424. "\t example: echo 'myevent u64 lat; char name[]' >> synthetic_events\n"
  4425. #endif
  4426. #endif
  4427. ;
  4428. static ssize_t
  4429. tracing_readme_read(struct file *filp, char __user *ubuf,
  4430. size_t cnt, loff_t *ppos)
  4431. {
  4432. return simple_read_from_buffer(ubuf, cnt, ppos,
  4433. readme_msg, strlen(readme_msg));
  4434. }
  4435. static const struct file_operations tracing_readme_fops = {
  4436. .open = tracing_open_generic,
  4437. .read = tracing_readme_read,
  4438. .llseek = generic_file_llseek,
  4439. };
  4440. static void *saved_tgids_next(struct seq_file *m, void *v, loff_t *pos)
  4441. {
  4442. int pid = ++(*pos);
  4443. return trace_find_tgid_ptr(pid);
  4444. }
  4445. static void *saved_tgids_start(struct seq_file *m, loff_t *pos)
  4446. {
  4447. int pid = *pos;
  4448. return trace_find_tgid_ptr(pid);
  4449. }
  4450. static void saved_tgids_stop(struct seq_file *m, void *v)
  4451. {
  4452. }
  4453. static int saved_tgids_show(struct seq_file *m, void *v)
  4454. {
  4455. int *entry = (int *)v;
  4456. int pid = entry - tgid_map;
  4457. int tgid = *entry;
  4458. if (tgid == 0)
  4459. return SEQ_SKIP;
  4460. seq_printf(m, "%d %d\n", pid, tgid);
  4461. return 0;
  4462. }
  4463. static const struct seq_operations tracing_saved_tgids_seq_ops = {
  4464. .start = saved_tgids_start,
  4465. .stop = saved_tgids_stop,
  4466. .next = saved_tgids_next,
  4467. .show = saved_tgids_show,
  4468. };
  4469. static int tracing_saved_tgids_open(struct inode *inode, struct file *filp)
  4470. {
  4471. int ret;
  4472. ret = tracing_check_open_get_tr(NULL);
  4473. if (ret)
  4474. return ret;
  4475. return seq_open(filp, &tracing_saved_tgids_seq_ops);
  4476. }
  4477. static const struct file_operations tracing_saved_tgids_fops = {
  4478. .open = tracing_saved_tgids_open,
  4479. .read = seq_read,
  4480. .llseek = seq_lseek,
  4481. .release = seq_release,
  4482. };
  4483. static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
  4484. {
  4485. unsigned int *ptr = v;
  4486. if (*pos || m->count)
  4487. ptr++;
  4488. (*pos)++;
  4489. for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
  4490. ptr++) {
  4491. if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
  4492. continue;
  4493. return ptr;
  4494. }
  4495. return NULL;
  4496. }
  4497. static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
  4498. {
  4499. void *v;
  4500. loff_t l = 0;
  4501. preempt_disable();
  4502. arch_spin_lock(&trace_cmdline_lock);
  4503. v = &savedcmd->map_cmdline_to_pid[0];
  4504. while (l <= *pos) {
  4505. v = saved_cmdlines_next(m, v, &l);
  4506. if (!v)
  4507. return NULL;
  4508. }
  4509. return v;
  4510. }
  4511. static void saved_cmdlines_stop(struct seq_file *m, void *v)
  4512. {
  4513. arch_spin_unlock(&trace_cmdline_lock);
  4514. preempt_enable();
  4515. }
  4516. static int saved_cmdlines_show(struct seq_file *m, void *v)
  4517. {
  4518. char buf[TASK_COMM_LEN];
  4519. unsigned int *pid = v;
  4520. __trace_find_cmdline(*pid, buf);
  4521. seq_printf(m, "%d %s\n", *pid, buf);
  4522. return 0;
  4523. }
  4524. static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
  4525. .start = saved_cmdlines_start,
  4526. .next = saved_cmdlines_next,
  4527. .stop = saved_cmdlines_stop,
  4528. .show = saved_cmdlines_show,
  4529. };
  4530. static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
  4531. {
  4532. int ret;
  4533. ret = tracing_check_open_get_tr(NULL);
  4534. if (ret)
  4535. return ret;
  4536. return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
  4537. }
  4538. static const struct file_operations tracing_saved_cmdlines_fops = {
  4539. .open = tracing_saved_cmdlines_open,
  4540. .read = seq_read,
  4541. .llseek = seq_lseek,
  4542. .release = seq_release,
  4543. };
  4544. static ssize_t
  4545. tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
  4546. size_t cnt, loff_t *ppos)
  4547. {
  4548. char buf[64];
  4549. int r;
  4550. arch_spin_lock(&trace_cmdline_lock);
  4551. r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
  4552. arch_spin_unlock(&trace_cmdline_lock);
  4553. return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
  4554. }
  4555. static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
  4556. {
  4557. kfree(s->saved_cmdlines);
  4558. kfree(s->map_cmdline_to_pid);
  4559. kfree(s);
  4560. }
  4561. static int tracing_resize_saved_cmdlines(unsigned int val)
  4562. {
  4563. struct saved_cmdlines_buffer *s, *savedcmd_temp;
  4564. s = kmalloc(sizeof(*s), GFP_KERNEL);
  4565. if (!s)
  4566. return -ENOMEM;
  4567. if (allocate_cmdlines_buffer(val, s) < 0) {
  4568. kfree(s);
  4569. return -ENOMEM;
  4570. }
  4571. arch_spin_lock(&trace_cmdline_lock);
  4572. savedcmd_temp = savedcmd;
  4573. savedcmd = s;
  4574. arch_spin_unlock(&trace_cmdline_lock);
  4575. free_saved_cmdlines_buffer(savedcmd_temp);
  4576. return 0;
  4577. }
  4578. static ssize_t
  4579. tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
  4580. size_t cnt, loff_t *ppos)
  4581. {
  4582. unsigned long val;
  4583. int ret;
  4584. ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
  4585. if (ret)
  4586. return ret;
  4587. /* must have at least 1 entry or less than PID_MAX_DEFAULT */
  4588. if (!val || val > PID_MAX_DEFAULT)
  4589. return -EINVAL;
  4590. ret = tracing_resize_saved_cmdlines((unsigned int)val);
  4591. if (ret < 0)
  4592. return ret;
  4593. *ppos += cnt;
  4594. return cnt;
  4595. }
  4596. static const struct file_operations tracing_saved_cmdlines_size_fops = {
  4597. .open = tracing_open_generic,
  4598. .read = tracing_saved_cmdlines_size_read,
  4599. .write = tracing_saved_cmdlines_size_write,
  4600. };
  4601. #ifdef CONFIG_TRACE_EVAL_MAP_FILE
  4602. static union trace_eval_map_item *
  4603. update_eval_map(union trace_eval_map_item *ptr)
  4604. {
  4605. if (!ptr->map.eval_string) {
  4606. if (ptr->tail.next) {
  4607. ptr = ptr->tail.next;
  4608. /* Set ptr to the next real item (skip head) */
  4609. ptr++;
  4610. } else
  4611. return NULL;
  4612. }
  4613. return ptr;
  4614. }
  4615. static void *eval_map_next(struct seq_file *m, void *v, loff_t *pos)
  4616. {
  4617. union trace_eval_map_item *ptr = v;
  4618. /*
  4619. * Paranoid! If ptr points to end, we don't want to increment past it.
  4620. * This really should never happen.
  4621. */
  4622. (*pos)++;
  4623. ptr = update_eval_map(ptr);
  4624. if (WARN_ON_ONCE(!ptr))
  4625. return NULL;
  4626. ptr++;
  4627. ptr = update_eval_map(ptr);
  4628. return ptr;
  4629. }
  4630. static void *eval_map_start(struct seq_file *m, loff_t *pos)
  4631. {
  4632. union trace_eval_map_item *v;
  4633. loff_t l = 0;
  4634. mutex_lock(&trace_eval_mutex);
  4635. v = trace_eval_maps;
  4636. if (v)
  4637. v++;
  4638. while (v && l < *pos) {
  4639. v = eval_map_next(m, v, &l);
  4640. }
  4641. return v;
  4642. }
  4643. static void eval_map_stop(struct seq_file *m, void *v)
  4644. {
  4645. mutex_unlock(&trace_eval_mutex);
  4646. }
  4647. static int eval_map_show(struct seq_file *m, void *v)
  4648. {
  4649. union trace_eval_map_item *ptr = v;
  4650. seq_printf(m, "%s %ld (%s)\n",
  4651. ptr->map.eval_string, ptr->map.eval_value,
  4652. ptr->map.system);
  4653. return 0;
  4654. }
  4655. static const struct seq_operations tracing_eval_map_seq_ops = {
  4656. .start = eval_map_start,
  4657. .next = eval_map_next,
  4658. .stop = eval_map_stop,
  4659. .show = eval_map_show,
  4660. };
  4661. static int tracing_eval_map_open(struct inode *inode, struct file *filp)
  4662. {
  4663. int ret;
  4664. ret = tracing_check_open_get_tr(NULL);
  4665. if (ret)
  4666. return ret;
  4667. return seq_open(filp, &tracing_eval_map_seq_ops);
  4668. }
  4669. static const struct file_operations tracing_eval_map_fops = {
  4670. .open = tracing_eval_map_open,
  4671. .read = seq_read,
  4672. .llseek = seq_lseek,
  4673. .release = seq_release,
  4674. };
  4675. static inline union trace_eval_map_item *
  4676. trace_eval_jmp_to_tail(union trace_eval_map_item *ptr)
  4677. {
  4678. /* Return tail of array given the head */
  4679. return ptr + ptr->head.length + 1;
  4680. }
  4681. static void
  4682. trace_insert_eval_map_file(struct module *mod, struct trace_eval_map **start,
  4683. int len)
  4684. {
  4685. struct trace_eval_map **stop;
  4686. struct trace_eval_map **map;
  4687. union trace_eval_map_item *map_array;
  4688. union trace_eval_map_item *ptr;
  4689. stop = start + len;
  4690. /*
  4691. * The trace_eval_maps contains the map plus a head and tail item,
  4692. * where the head holds the module and length of array, and the
  4693. * tail holds a pointer to the next list.
  4694. */
  4695. map_array = kmalloc_array(len + 2, sizeof(*map_array), GFP_KERNEL);
  4696. if (!map_array) {
  4697. pr_warn("Unable to allocate trace eval mapping\n");
  4698. return;
  4699. }
  4700. mutex_lock(&trace_eval_mutex);
  4701. if (!trace_eval_maps)
  4702. trace_eval_maps = map_array;
  4703. else {
  4704. ptr = trace_eval_maps;
  4705. for (;;) {
  4706. ptr = trace_eval_jmp_to_tail(ptr);
  4707. if (!ptr->tail.next)
  4708. break;
  4709. ptr = ptr->tail.next;
  4710. }
  4711. ptr->tail.next = map_array;
  4712. }
  4713. map_array->head.mod = mod;
  4714. map_array->head.length = len;
  4715. map_array++;
  4716. for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
  4717. map_array->map = **map;
  4718. map_array++;
  4719. }
  4720. memset(map_array, 0, sizeof(*map_array));
  4721. mutex_unlock(&trace_eval_mutex);
  4722. }
  4723. static void trace_create_eval_file(struct dentry *d_tracer)
  4724. {
  4725. trace_create_file("eval_map", 0444, d_tracer,
  4726. NULL, &tracing_eval_map_fops);
  4727. }
  4728. #else /* CONFIG_TRACE_EVAL_MAP_FILE */
  4729. static inline void trace_create_eval_file(struct dentry *d_tracer) { }
  4730. static inline void trace_insert_eval_map_file(struct module *mod,
  4731. struct trace_eval_map **start, int len) { }
  4732. #endif /* !CONFIG_TRACE_EVAL_MAP_FILE */
  4733. static void trace_insert_eval_map(struct module *mod,
  4734. struct trace_eval_map **start, int len)
  4735. {
  4736. struct trace_eval_map **map;
  4737. if (len <= 0)
  4738. return;
  4739. map = start;
  4740. trace_event_eval_update(map, len);
  4741. trace_insert_eval_map_file(mod, start, len);
  4742. }
  4743. static ssize_t
  4744. tracing_set_trace_read(struct file *filp, char __user *ubuf,
  4745. size_t cnt, loff_t *ppos)
  4746. {
  4747. struct trace_array *tr = filp->private_data;
  4748. char buf[MAX_TRACER_SIZE+2];
  4749. int r;
  4750. mutex_lock(&trace_types_lock);
  4751. r = sprintf(buf, "%s\n", tr->current_trace->name);
  4752. mutex_unlock(&trace_types_lock);
  4753. return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
  4754. }
  4755. int tracer_init(struct tracer *t, struct trace_array *tr)
  4756. {
  4757. tracing_reset_online_cpus(&tr->array_buffer);
  4758. return t->init(tr);
  4759. }
  4760. static void set_buffer_entries(struct array_buffer *buf, unsigned long val)
  4761. {
  4762. int cpu;
  4763. for_each_tracing_cpu(cpu)
  4764. per_cpu_ptr(buf->data, cpu)->entries = val;
  4765. }
  4766. #ifdef CONFIG_TRACER_MAX_TRACE
  4767. /* resize @tr's buffer to the size of @size_tr's entries */
  4768. static int resize_buffer_duplicate_size(struct array_buffer *trace_buf,
  4769. struct array_buffer *size_buf, int cpu_id)
  4770. {
  4771. int cpu, ret = 0;
  4772. if (cpu_id == RING_BUFFER_ALL_CPUS) {
  4773. for_each_tracing_cpu(cpu) {
  4774. ret = ring_buffer_resize(trace_buf->buffer,
  4775. per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
  4776. if (ret < 0)
  4777. break;
  4778. per_cpu_ptr(trace_buf->data, cpu)->entries =
  4779. per_cpu_ptr(size_buf->data, cpu)->entries;
  4780. }
  4781. } else {
  4782. ret = ring_buffer_resize(trace_buf->buffer,
  4783. per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
  4784. if (ret == 0)
  4785. per_cpu_ptr(trace_buf->data, cpu_id)->entries =
  4786. per_cpu_ptr(size_buf->data, cpu_id)->entries;
  4787. }
  4788. return ret;
  4789. }
  4790. #endif /* CONFIG_TRACER_MAX_TRACE */
  4791. static int __tracing_resize_ring_buffer(struct trace_array *tr,
  4792. unsigned long size, int cpu)
  4793. {
  4794. int ret;
  4795. /*
  4796. * If kernel or user changes the size of the ring buffer
  4797. * we use the size that was given, and we can forget about
  4798. * expanding it later.
  4799. */
  4800. ring_buffer_expanded = true;
  4801. /* May be called before buffers are initialized */
  4802. if (!tr->array_buffer.buffer)
  4803. return 0;
  4804. ret = ring_buffer_resize(tr->array_buffer.buffer, size, cpu);
  4805. if (ret < 0)
  4806. return ret;
  4807. #ifdef CONFIG_TRACER_MAX_TRACE
  4808. if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
  4809. !tr->current_trace->use_max_tr)
  4810. goto out;
  4811. ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
  4812. if (ret < 0) {
  4813. int r = resize_buffer_duplicate_size(&tr->array_buffer,
  4814. &tr->array_buffer, cpu);
  4815. if (r < 0) {
  4816. /*
  4817. * AARGH! We are left with different
  4818. * size max buffer!!!!
  4819. * The max buffer is our "snapshot" buffer.
  4820. * When a tracer needs a snapshot (one of the
  4821. * latency tracers), it swaps the max buffer
  4822. * with the saved snap shot. We succeeded to
  4823. * update the size of the main buffer, but failed to
  4824. * update the size of the max buffer. But when we tried
  4825. * to reset the main buffer to the original size, we
  4826. * failed there too. This is very unlikely to
  4827. * happen, but if it does, warn and kill all
  4828. * tracing.
  4829. */
  4830. WARN_ON(1);
  4831. tracing_disabled = 1;
  4832. }
  4833. return ret;
  4834. }
  4835. if (cpu == RING_BUFFER_ALL_CPUS)
  4836. set_buffer_entries(&tr->max_buffer, size);
  4837. else
  4838. per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
  4839. out:
  4840. #endif /* CONFIG_TRACER_MAX_TRACE */
  4841. if (cpu == RING_BUFFER_ALL_CPUS)
  4842. set_buffer_entries(&tr->array_buffer, size);
  4843. else
  4844. per_cpu_ptr(tr->array_buffer.data, cpu)->entries = size;
  4845. return ret;
  4846. }
  4847. ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
  4848. unsigned long size, int cpu_id)
  4849. {
  4850. int ret = size;
  4851. mutex_lock(&trace_types_lock);
  4852. if (cpu_id != RING_BUFFER_ALL_CPUS) {
  4853. /* make sure, this cpu is enabled in the mask */
  4854. if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
  4855. ret = -EINVAL;
  4856. goto out;
  4857. }
  4858. }
  4859. ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
  4860. if (ret < 0)
  4861. ret = -ENOMEM;
  4862. out:
  4863. mutex_unlock(&trace_types_lock);
  4864. return ret;
  4865. }
  4866. /**
  4867. * tracing_update_buffers - used by tracing facility to expand ring buffers
  4868. *
  4869. * To save on memory when the tracing is never used on a system with it
  4870. * configured in. The ring buffers are set to a minimum size. But once
  4871. * a user starts to use the tracing facility, then they need to grow
  4872. * to their default size.
  4873. *
  4874. * This function is to be called when a tracer is about to be used.
  4875. */
  4876. int tracing_update_buffers(void)
  4877. {
  4878. int ret = 0;
  4879. mutex_lock(&trace_types_lock);
  4880. if (!ring_buffer_expanded)
  4881. ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
  4882. RING_BUFFER_ALL_CPUS);
  4883. mutex_unlock(&trace_types_lock);
  4884. return ret;
  4885. }
  4886. struct trace_option_dentry;
  4887. static void
  4888. create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
  4889. /*
  4890. * Used to clear out the tracer before deletion of an instance.
  4891. * Must have trace_types_lock held.
  4892. */
  4893. static void tracing_set_nop(struct trace_array *tr)
  4894. {
  4895. if (tr->current_trace == &nop_trace)
  4896. return;
  4897. tr->current_trace->enabled--;
  4898. if (tr->current_trace->reset)
  4899. tr->current_trace->reset(tr);
  4900. tr->current_trace = &nop_trace;
  4901. }
  4902. static void add_tracer_options(struct trace_array *tr, struct tracer *t)
  4903. {
  4904. /* Only enable if the directory has been created already. */
  4905. if (!tr->dir)
  4906. return;
  4907. create_trace_option_files(tr, t);
  4908. }
  4909. int tracing_set_tracer(struct trace_array *tr, const char *buf)
  4910. {
  4911. struct tracer *t;
  4912. #ifdef CONFIG_TRACER_MAX_TRACE
  4913. bool had_max_tr;
  4914. #endif
  4915. int ret = 0;
  4916. mutex_lock(&trace_types_lock);
  4917. if (!ring_buffer_expanded) {
  4918. ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
  4919. RING_BUFFER_ALL_CPUS);
  4920. if (ret < 0)
  4921. goto out;
  4922. ret = 0;
  4923. }
  4924. for (t = trace_types; t; t = t->next) {
  4925. if (strcmp(t->name, buf) == 0)
  4926. break;
  4927. }
  4928. if (!t) {
  4929. ret = -EINVAL;
  4930. goto out;
  4931. }
  4932. if (t == tr->current_trace)
  4933. goto out;
  4934. #ifdef CONFIG_TRACER_SNAPSHOT
  4935. if (t->use_max_tr) {
  4936. arch_spin_lock(&tr->max_lock);
  4937. if (tr->cond_snapshot)
  4938. ret = -EBUSY;
  4939. arch_spin_unlock(&tr->max_lock);
  4940. if (ret)
  4941. goto out;
  4942. }
  4943. #endif
  4944. /* Some tracers won't work on kernel command line */
  4945. if (system_state < SYSTEM_RUNNING && t->noboot) {
  4946. pr_warn("Tracer '%s' is not allowed on command line, ignored\n",
  4947. t->name);
  4948. goto out;
  4949. }
  4950. /* Some tracers are only allowed for the top level buffer */
  4951. if (!trace_ok_for_array(t, tr)) {
  4952. ret = -EINVAL;
  4953. goto out;
  4954. }
  4955. /* If trace pipe files are being read, we can't change the tracer */
  4956. if (tr->trace_ref) {
  4957. ret = -EBUSY;
  4958. goto out;
  4959. }
  4960. trace_branch_disable();
  4961. tr->current_trace->enabled--;
  4962. if (tr->current_trace->reset)
  4963. tr->current_trace->reset(tr);
  4964. /* Current trace needs to be nop_trace before synchronize_rcu */
  4965. tr->current_trace = &nop_trace;
  4966. #ifdef CONFIG_TRACER_MAX_TRACE
  4967. had_max_tr = tr->allocated_snapshot;
  4968. if (had_max_tr && !t->use_max_tr) {
  4969. /*
  4970. * We need to make sure that the update_max_tr sees that
  4971. * current_trace changed to nop_trace to keep it from
  4972. * swapping the buffers after we resize it.
  4973. * The update_max_tr is called from interrupts disabled
  4974. * so a synchronized_sched() is sufficient.
  4975. */
  4976. synchronize_rcu();
  4977. free_snapshot(tr);
  4978. }
  4979. #endif
  4980. #ifdef CONFIG_TRACER_MAX_TRACE
  4981. if (t->use_max_tr && !had_max_tr) {
  4982. ret = tracing_alloc_snapshot_instance(tr);
  4983. if (ret < 0)
  4984. goto out;
  4985. }
  4986. #endif
  4987. if (t->init) {
  4988. ret = tracer_init(t, tr);
  4989. if (ret)
  4990. goto out;
  4991. }
  4992. tr->current_trace = t;
  4993. tr->current_trace->enabled++;
  4994. trace_branch_enable(tr);
  4995. out:
  4996. mutex_unlock(&trace_types_lock);
  4997. return ret;
  4998. }
  4999. static ssize_t
  5000. tracing_set_trace_write(struct file *filp, const char __user *ubuf,
  5001. size_t cnt, loff_t *ppos)
  5002. {
  5003. struct trace_array *tr = filp->private_data;
  5004. char buf[MAX_TRACER_SIZE+1];
  5005. int i;
  5006. size_t ret;
  5007. int err;
  5008. ret = cnt;
  5009. if (cnt > MAX_TRACER_SIZE)
  5010. cnt = MAX_TRACER_SIZE;
  5011. if (copy_from_user(buf, ubuf, cnt))
  5012. return -EFAULT;
  5013. buf[cnt] = 0;
  5014. /* strip ending whitespace. */
  5015. for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
  5016. buf[i] = 0;
  5017. err = tracing_set_tracer(tr, buf);
  5018. if (err)
  5019. return err;
  5020. *ppos += ret;
  5021. return ret;
  5022. }
  5023. static ssize_t
  5024. tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
  5025. size_t cnt, loff_t *ppos)
  5026. {
  5027. char buf[64];
  5028. int r;
  5029. r = snprintf(buf, sizeof(buf), "%ld\n",
  5030. *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
  5031. if (r > sizeof(buf))
  5032. r = sizeof(buf);
  5033. return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
  5034. }
  5035. static ssize_t
  5036. tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
  5037. size_t cnt, loff_t *ppos)
  5038. {
  5039. unsigned long val;
  5040. int ret;
  5041. ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
  5042. if (ret)
  5043. return ret;
  5044. *ptr = val * 1000;
  5045. return cnt;
  5046. }
  5047. static ssize_t
  5048. tracing_thresh_read(struct file *filp, char __user *ubuf,
  5049. size_t cnt, loff_t *ppos)
  5050. {
  5051. return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
  5052. }
  5053. static ssize_t
  5054. tracing_thresh_write(struct file *filp, const char __user *ubuf,
  5055. size_t cnt, loff_t *ppos)
  5056. {
  5057. struct trace_array *tr = filp->private_data;
  5058. int ret;
  5059. mutex_lock(&trace_types_lock);
  5060. ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
  5061. if (ret < 0)
  5062. goto out;
  5063. if (tr->current_trace->update_thresh) {
  5064. ret = tr->current_trace->update_thresh(tr);
  5065. if (ret < 0)
  5066. goto out;
  5067. }
  5068. ret = cnt;
  5069. out:
  5070. mutex_unlock(&trace_types_lock);
  5071. return ret;
  5072. }
  5073. #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
  5074. static ssize_t
  5075. tracing_max_lat_read(struct file *filp, char __user *ubuf,
  5076. size_t cnt, loff_t *ppos)
  5077. {
  5078. return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
  5079. }
  5080. static ssize_t
  5081. tracing_max_lat_write(struct file *filp, const char __user *ubuf,
  5082. size_t cnt, loff_t *ppos)
  5083. {
  5084. return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
  5085. }
  5086. #endif
  5087. static int tracing_open_pipe(struct inode *inode, struct file *filp)
  5088. {
  5089. struct trace_array *tr = inode->i_private;
  5090. struct trace_iterator *iter;
  5091. int ret;
  5092. ret = tracing_check_open_get_tr(tr);
  5093. if (ret)
  5094. return ret;
  5095. mutex_lock(&trace_types_lock);
  5096. /* create a buffer to store the information to pass to userspace */
  5097. iter = kzalloc(sizeof(*iter), GFP_KERNEL);
  5098. if (!iter) {
  5099. ret = -ENOMEM;
  5100. __trace_array_put(tr);
  5101. goto out;
  5102. }
  5103. trace_seq_init(&iter->seq);
  5104. iter->trace = tr->current_trace;
  5105. if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
  5106. ret = -ENOMEM;
  5107. goto fail;
  5108. }
  5109. /* trace pipe does not show start of buffer */
  5110. cpumask_setall(iter->started);
  5111. if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
  5112. iter->iter_flags |= TRACE_FILE_LAT_FMT;
  5113. /* Output in nanoseconds only if we are using a clock in nanoseconds. */
  5114. if (trace_clocks[tr->clock_id].in_ns)
  5115. iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
  5116. iter->tr = tr;
  5117. iter->array_buffer = &tr->array_buffer;
  5118. iter->cpu_file = tracing_get_cpu(inode);
  5119. mutex_init(&iter->mutex);
  5120. filp->private_data = iter;
  5121. if (iter->trace->pipe_open)
  5122. iter->trace->pipe_open(iter);
  5123. nonseekable_open(inode, filp);
  5124. tr->trace_ref++;
  5125. out:
  5126. mutex_unlock(&trace_types_lock);
  5127. return ret;
  5128. fail:
  5129. kfree(iter);
  5130. __trace_array_put(tr);
  5131. mutex_unlock(&trace_types_lock);
  5132. return ret;
  5133. }
  5134. static int tracing_release_pipe(struct inode *inode, struct file *file)
  5135. {
  5136. struct trace_iterator *iter = file->private_data;
  5137. struct trace_array *tr = inode->i_private;
  5138. mutex_lock(&trace_types_lock);
  5139. tr->trace_ref--;
  5140. if (iter->trace->pipe_close)
  5141. iter->trace->pipe_close(iter);
  5142. mutex_unlock(&trace_types_lock);
  5143. free_cpumask_var(iter->started);
  5144. mutex_destroy(&iter->mutex);
  5145. kfree(iter);
  5146. trace_array_put(tr);
  5147. return 0;
  5148. }
  5149. static __poll_t
  5150. trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
  5151. {
  5152. struct trace_array *tr = iter->tr;
  5153. /* Iterators are static, they should be filled or empty */
  5154. if (trace_buffer_iter(iter, iter->cpu_file))
  5155. return EPOLLIN | EPOLLRDNORM;
  5156. if (tr->trace_flags & TRACE_ITER_BLOCK)
  5157. /*
  5158. * Always select as readable when in blocking mode
  5159. */
  5160. return EPOLLIN | EPOLLRDNORM;
  5161. else
  5162. return ring_buffer_poll_wait(iter->array_buffer->buffer, iter->cpu_file,
  5163. filp, poll_table);
  5164. }
  5165. static __poll_t
  5166. tracing_poll_pipe(struct file *filp, poll_table *poll_table)
  5167. {
  5168. struct trace_iterator *iter = filp->private_data;
  5169. return trace_poll(iter, filp, poll_table);
  5170. }
  5171. /* Must be called with iter->mutex held. */
  5172. static int tracing_wait_pipe(struct file *filp)
  5173. {
  5174. struct trace_iterator *iter = filp->private_data;
  5175. int ret;
  5176. while (trace_empty(iter)) {
  5177. if ((filp->f_flags & O_NONBLOCK)) {
  5178. return -EAGAIN;
  5179. }
  5180. /*
  5181. * We block until we read something and tracing is disabled.
  5182. * We still block if tracing is disabled, but we have never
  5183. * read anything. This allows a user to cat this file, and
  5184. * then enable tracing. But after we have read something,
  5185. * we give an EOF when tracing is again disabled.
  5186. *
  5187. * iter->pos will be 0 if we haven't read anything.
  5188. */
  5189. if (!tracer_tracing_is_on(iter->tr) && iter->pos)
  5190. break;
  5191. mutex_unlock(&iter->mutex);
  5192. ret = wait_on_pipe(iter, 0);
  5193. mutex_lock(&iter->mutex);
  5194. if (ret)
  5195. return ret;
  5196. }
  5197. return 1;
  5198. }
  5199. /*
  5200. * Consumer reader.
  5201. */
  5202. static ssize_t
  5203. tracing_read_pipe(struct file *filp, char __user *ubuf,
  5204. size_t cnt, loff_t *ppos)
  5205. {
  5206. struct trace_iterator *iter = filp->private_data;
  5207. ssize_t sret;
  5208. /*
  5209. * Avoid more than one consumer on a single file descriptor
  5210. * This is just a matter of traces coherency, the ring buffer itself
  5211. * is protected.
  5212. */
  5213. mutex_lock(&iter->mutex);
  5214. /* return any leftover data */
  5215. sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
  5216. if (sret != -EBUSY)
  5217. goto out;
  5218. trace_seq_init(&iter->seq);
  5219. if (iter->trace->read) {
  5220. sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
  5221. if (sret)
  5222. goto out;
  5223. }
  5224. waitagain:
  5225. sret = tracing_wait_pipe(filp);
  5226. if (sret <= 0)
  5227. goto out;
  5228. /* stop when tracing is finished */
  5229. if (trace_empty(iter)) {
  5230. sret = 0;
  5231. goto out;
  5232. }
  5233. if (cnt >= PAGE_SIZE)
  5234. cnt = PAGE_SIZE - 1;
  5235. /* reset all but tr, trace, and overruns */
  5236. memset(&iter->seq, 0,
  5237. sizeof(struct trace_iterator) -
  5238. offsetof(struct trace_iterator, seq));
  5239. cpumask_clear(iter->started);
  5240. trace_seq_init(&iter->seq);
  5241. iter->pos = -1;
  5242. trace_event_read_lock();
  5243. trace_access_lock(iter->cpu_file);
  5244. while (trace_find_next_entry_inc(iter) != NULL) {
  5245. enum print_line_t ret;
  5246. int save_len = iter->seq.seq.len;
  5247. ret = print_trace_line(iter);
  5248. if (ret == TRACE_TYPE_PARTIAL_LINE) {
  5249. /* don't print partial lines */
  5250. iter->seq.seq.len = save_len;
  5251. break;
  5252. }
  5253. if (ret != TRACE_TYPE_NO_CONSUME)
  5254. trace_consume(iter);
  5255. if (trace_seq_used(&iter->seq) >= cnt)
  5256. break;
  5257. /*
  5258. * Setting the full flag means we reached the trace_seq buffer
  5259. * size and we should leave by partial output condition above.
  5260. * One of the trace_seq_* functions is not used properly.
  5261. */
  5262. WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
  5263. iter->ent->type);
  5264. }
  5265. trace_access_unlock(iter->cpu_file);
  5266. trace_event_read_unlock();
  5267. /* Now copy what we have to the user */
  5268. sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
  5269. if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
  5270. trace_seq_init(&iter->seq);
  5271. /*
  5272. * If there was nothing to send to user, in spite of consuming trace
  5273. * entries, go back to wait for more entries.
  5274. */
  5275. if (sret == -EBUSY)
  5276. goto waitagain;
  5277. out:
  5278. mutex_unlock(&iter->mutex);
  5279. return sret;
  5280. }
  5281. static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
  5282. unsigned int idx)
  5283. {
  5284. __free_page(spd->pages[idx]);
  5285. }
  5286. static size_t
  5287. tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
  5288. {
  5289. size_t count;
  5290. int save_len;
  5291. int ret;
  5292. /* Seq buffer is page-sized, exactly what we need. */
  5293. for (;;) {
  5294. save_len = iter->seq.seq.len;
  5295. ret = print_trace_line(iter);
  5296. if (trace_seq_has_overflowed(&iter->seq)) {
  5297. iter->seq.seq.len = save_len;
  5298. break;
  5299. }
  5300. /*
  5301. * This should not be hit, because it should only
  5302. * be set if the iter->seq overflowed. But check it
  5303. * anyway to be safe.
  5304. */
  5305. if (ret == TRACE_TYPE_PARTIAL_LINE) {
  5306. iter->seq.seq.len = save_len;
  5307. break;
  5308. }
  5309. count = trace_seq_used(&iter->seq) - save_len;
  5310. if (rem < count) {
  5311. rem = 0;
  5312. iter->seq.seq.len = save_len;
  5313. break;
  5314. }
  5315. if (ret != TRACE_TYPE_NO_CONSUME)
  5316. trace_consume(iter);
  5317. rem -= count;
  5318. if (!trace_find_next_entry_inc(iter)) {
  5319. rem = 0;
  5320. iter->ent = NULL;
  5321. break;
  5322. }
  5323. }
  5324. return rem;
  5325. }
  5326. static ssize_t tracing_splice_read_pipe(struct file *filp,
  5327. loff_t *ppos,
  5328. struct pipe_inode_info *pipe,
  5329. size_t len,
  5330. unsigned int flags)
  5331. {
  5332. struct page *pages_def[PIPE_DEF_BUFFERS];
  5333. struct partial_page partial_def[PIPE_DEF_BUFFERS];
  5334. struct trace_iterator *iter = filp->private_data;
  5335. struct splice_pipe_desc spd = {
  5336. .pages = pages_def,
  5337. .partial = partial_def,
  5338. .nr_pages = 0, /* This gets updated below. */
  5339. .nr_pages_max = PIPE_DEF_BUFFERS,
  5340. .ops = &default_pipe_buf_ops,
  5341. .spd_release = tracing_spd_release_pipe,
  5342. };
  5343. ssize_t ret;
  5344. size_t rem;
  5345. unsigned int i;
  5346. if (splice_grow_spd(pipe, &spd))
  5347. return -ENOMEM;
  5348. mutex_lock(&iter->mutex);
  5349. if (iter->trace->splice_read) {
  5350. ret = iter->trace->splice_read(iter, filp,
  5351. ppos, pipe, len, flags);
  5352. if (ret)
  5353. goto out_err;
  5354. }
  5355. ret = tracing_wait_pipe(filp);
  5356. if (ret <= 0)
  5357. goto out_err;
  5358. if (!iter->ent && !trace_find_next_entry_inc(iter)) {
  5359. ret = -EFAULT;
  5360. goto out_err;
  5361. }
  5362. trace_event_read_lock();
  5363. trace_access_lock(iter->cpu_file);
  5364. /* Fill as many pages as possible. */
  5365. for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
  5366. spd.pages[i] = alloc_page(GFP_KERNEL);
  5367. if (!spd.pages[i])
  5368. break;
  5369. rem = tracing_fill_pipe_page(rem, iter);
  5370. /* Copy the data into the page, so we can start over. */
  5371. ret = trace_seq_to_buffer(&iter->seq,
  5372. page_address(spd.pages[i]),
  5373. trace_seq_used(&iter->seq));
  5374. if (ret < 0) {
  5375. __free_page(spd.pages[i]);
  5376. break;
  5377. }
  5378. spd.partial[i].offset = 0;
  5379. spd.partial[i].len = trace_seq_used(&iter->seq);
  5380. trace_seq_init(&iter->seq);
  5381. }
  5382. trace_access_unlock(iter->cpu_file);
  5383. trace_event_read_unlock();
  5384. mutex_unlock(&iter->mutex);
  5385. spd.nr_pages = i;
  5386. if (i)
  5387. ret = splice_to_pipe(pipe, &spd);
  5388. else
  5389. ret = 0;
  5390. out:
  5391. splice_shrink_spd(&spd);
  5392. return ret;
  5393. out_err:
  5394. mutex_unlock(&iter->mutex);
  5395. goto out;
  5396. }
  5397. static ssize_t
  5398. tracing_entries_read(struct file *filp, char __user *ubuf,
  5399. size_t cnt, loff_t *ppos)
  5400. {
  5401. struct inode *inode = file_inode(filp);
  5402. struct trace_array *tr = inode->i_private;
  5403. int cpu = tracing_get_cpu(inode);
  5404. char buf[64];
  5405. int r = 0;
  5406. ssize_t ret;
  5407. mutex_lock(&trace_types_lock);
  5408. if (cpu == RING_BUFFER_ALL_CPUS) {
  5409. int cpu, buf_size_same;
  5410. unsigned long size;
  5411. size = 0;
  5412. buf_size_same = 1;
  5413. /* check if all cpu sizes are same */
  5414. for_each_tracing_cpu(cpu) {
  5415. /* fill in the size from first enabled cpu */
  5416. if (size == 0)
  5417. size = per_cpu_ptr(tr->array_buffer.data, cpu)->entries;
  5418. if (size != per_cpu_ptr(tr->array_buffer.data, cpu)->entries) {
  5419. buf_size_same = 0;
  5420. break;
  5421. }
  5422. }
  5423. if (buf_size_same) {
  5424. if (!ring_buffer_expanded)
  5425. r = sprintf(buf, "%lu (expanded: %lu)\n",
  5426. size >> 10,
  5427. trace_buf_size >> 10);
  5428. else
  5429. r = sprintf(buf, "%lu\n", size >> 10);
  5430. } else
  5431. r = sprintf(buf, "X\n");
  5432. } else
  5433. r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10);
  5434. mutex_unlock(&trace_types_lock);
  5435. ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
  5436. return ret;
  5437. }
  5438. static ssize_t
  5439. tracing_entries_write(struct file *filp, const char __user *ubuf,
  5440. size_t cnt, loff_t *ppos)
  5441. {
  5442. struct inode *inode = file_inode(filp);
  5443. struct trace_array *tr = inode->i_private;
  5444. unsigned long val;
  5445. int ret;
  5446. ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
  5447. if (ret)
  5448. return ret;
  5449. /* must have at least 1 entry */
  5450. if (!val)
  5451. return -EINVAL;
  5452. /* value is in KB */
  5453. val <<= 10;
  5454. ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
  5455. if (ret < 0)
  5456. return ret;
  5457. *ppos += cnt;
  5458. return cnt;
  5459. }
  5460. static ssize_t
  5461. tracing_total_entries_read(struct file *filp, char __user *ubuf,
  5462. size_t cnt, loff_t *ppos)
  5463. {
  5464. struct trace_array *tr = filp->private_data;
  5465. char buf[64];
  5466. int r, cpu;
  5467. unsigned long size = 0, expanded_size = 0;
  5468. mutex_lock(&trace_types_lock);
  5469. for_each_tracing_cpu(cpu) {
  5470. size += per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10;
  5471. if (!ring_buffer_expanded)
  5472. expanded_size += trace_buf_size >> 10;
  5473. }
  5474. if (ring_buffer_expanded)
  5475. r = sprintf(buf, "%lu\n", size);
  5476. else
  5477. r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
  5478. mutex_unlock(&trace_types_lock);
  5479. return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
  5480. }
  5481. static ssize_t
  5482. tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
  5483. size_t cnt, loff_t *ppos)
  5484. {
  5485. /*
  5486. * There is no need to read what the user has written, this function
  5487. * is just to make sure that there is no error when "echo" is used
  5488. */
  5489. *ppos += cnt;
  5490. return cnt;
  5491. }
  5492. static int
  5493. tracing_free_buffer_release(struct inode *inode, struct file *filp)
  5494. {
  5495. struct trace_array *tr = inode->i_private;
  5496. /* disable tracing ? */
  5497. if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE)
  5498. tracer_tracing_off(tr);
  5499. /* resize the ring buffer to 0 */
  5500. tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
  5501. trace_array_put(tr);
  5502. return 0;
  5503. }
  5504. static ssize_t
  5505. tracing_mark_write(struct file *filp, const char __user *ubuf,
  5506. size_t cnt, loff_t *fpos)
  5507. {
  5508. struct trace_array *tr = filp->private_data;
  5509. struct ring_buffer_event *event;
  5510. enum event_trigger_type tt = ETT_NONE;
  5511. struct trace_buffer *buffer;
  5512. struct print_entry *entry;
  5513. unsigned long irq_flags;
  5514. ssize_t written;
  5515. int size;
  5516. int len;
  5517. /* Used in tracing_mark_raw_write() as well */
  5518. #define FAULTED_STR "<faulted>"
  5519. #define FAULTED_SIZE (sizeof(FAULTED_STR) - 1) /* '\0' is already accounted for */
  5520. if (tracing_disabled)
  5521. return -EINVAL;
  5522. if (!(tr->trace_flags & TRACE_ITER_MARKERS))
  5523. return -EINVAL;
  5524. if (cnt > TRACE_BUF_SIZE)
  5525. cnt = TRACE_BUF_SIZE;
  5526. BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
  5527. local_save_flags(irq_flags);
  5528. size = sizeof(*entry) + cnt + 2; /* add '\0' and possible '\n' */
  5529. /* If less than "<faulted>", then make sure we can still add that */
  5530. if (cnt < FAULTED_SIZE)
  5531. size += FAULTED_SIZE - cnt;
  5532. buffer = tr->array_buffer.buffer;
  5533. event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
  5534. irq_flags, preempt_count());
  5535. if (unlikely(!event))
  5536. /* Ring buffer disabled, return as if not open for write */
  5537. return -EBADF;
  5538. entry = ring_buffer_event_data(event);
  5539. entry->ip = _THIS_IP_;
  5540. len = __copy_from_user_inatomic(&entry->buf, ubuf, cnt);
  5541. if (len) {
  5542. memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);
  5543. cnt = FAULTED_SIZE;
  5544. written = -EFAULT;
  5545. } else
  5546. written = cnt;
  5547. if (tr->trace_marker_file && !list_empty(&tr->trace_marker_file->triggers)) {
  5548. /* do not add \n before testing triggers, but add \0 */
  5549. entry->buf[cnt] = '\0';
  5550. tt = event_triggers_call(tr->trace_marker_file, entry, event);
  5551. }
  5552. if (entry->buf[cnt - 1] != '\n') {
  5553. entry->buf[cnt] = '\n';
  5554. entry->buf[cnt + 1] = '\0';
  5555. } else
  5556. entry->buf[cnt] = '\0';
  5557. if (static_branch_unlikely(&trace_marker_exports_enabled))
  5558. ftrace_exports(event, TRACE_EXPORT_MARKER);
  5559. __buffer_unlock_commit(buffer, event);
  5560. if (tt)
  5561. event_triggers_post_call(tr->trace_marker_file, tt);
  5562. if (written > 0)
  5563. *fpos += written;
  5564. return written;
  5565. }
  5566. /* Limit it for now to 3K (including tag) */
  5567. #define RAW_DATA_MAX_SIZE (1024*3)
  5568. static ssize_t
  5569. tracing_mark_raw_write(struct file *filp, const char __user *ubuf,
  5570. size_t cnt, loff_t *fpos)
  5571. {
  5572. struct trace_array *tr = filp->private_data;
  5573. struct ring_buffer_event *event;
  5574. struct trace_buffer *buffer;
  5575. struct raw_data_entry *entry;
  5576. unsigned long irq_flags;
  5577. ssize_t written;
  5578. int size;
  5579. int len;
  5580. #define FAULT_SIZE_ID (FAULTED_SIZE + sizeof(int))
  5581. if (tracing_disabled)
  5582. return -EINVAL;
  5583. if (!(tr->trace_flags & TRACE_ITER_MARKERS))
  5584. return -EINVAL;
  5585. /* The marker must at least have a tag id */
  5586. if (cnt < sizeof(unsigned int) || cnt > RAW_DATA_MAX_SIZE)
  5587. return -EINVAL;
  5588. if (cnt > TRACE_BUF_SIZE)
  5589. cnt = TRACE_BUF_SIZE;
  5590. BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
  5591. local_save_flags(irq_flags);
  5592. size = sizeof(*entry) + cnt;
  5593. if (cnt < FAULT_SIZE_ID)
  5594. size += FAULT_SIZE_ID - cnt;
  5595. buffer = tr->array_buffer.buffer;
  5596. event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size,
  5597. irq_flags, preempt_count());
  5598. if (!event)
  5599. /* Ring buffer disabled, return as if not open for write */
  5600. return -EBADF;
  5601. entry = ring_buffer_event_data(event);
  5602. len = __copy_from_user_inatomic(&entry->id, ubuf, cnt);
  5603. if (len) {
  5604. entry->id = -1;
  5605. memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);
  5606. written = -EFAULT;
  5607. } else
  5608. written = cnt;
  5609. __buffer_unlock_commit(buffer, event);
  5610. if (written > 0)
  5611. *fpos += written;
  5612. return written;
  5613. }
  5614. static int tracing_clock_show(struct seq_file *m, void *v)
  5615. {
  5616. struct trace_array *tr = m->private;
  5617. int i;
  5618. for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
  5619. seq_printf(m,
  5620. "%s%s%s%s", i ? " " : "",
  5621. i == tr->clock_id ? "[" : "", trace_clocks[i].name,
  5622. i == tr->clock_id ? "]" : "");
  5623. seq_putc(m, '\n');
  5624. return 0;
  5625. }
  5626. int tracing_set_clock(struct trace_array *tr, const char *clockstr)
  5627. {
  5628. int i;
  5629. for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
  5630. if (strcmp(trace_clocks[i].name, clockstr) == 0)
  5631. break;
  5632. }
  5633. if (i == ARRAY_SIZE(trace_clocks))
  5634. return -EINVAL;
  5635. mutex_lock(&trace_types_lock);
  5636. tr->clock_id = i;
  5637. ring_buffer_set_clock(tr->array_buffer.buffer, trace_clocks[i].func);
  5638. /*
  5639. * New clock may not be consistent with the previous clock.
  5640. * Reset the buffer so that it doesn't have incomparable timestamps.
  5641. */
  5642. tracing_reset_online_cpus(&tr->array_buffer);
  5643. #ifdef CONFIG_TRACER_MAX_TRACE
  5644. if (tr->max_buffer.buffer)
  5645. ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
  5646. tracing_reset_online_cpus(&tr->max_buffer);
  5647. #endif
  5648. mutex_unlock(&trace_types_lock);
  5649. return 0;
  5650. }
  5651. static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
  5652. size_t cnt, loff_t *fpos)
  5653. {
  5654. struct seq_file *m = filp->private_data;
  5655. struct trace_array *tr = m->private;
  5656. char buf[64];
  5657. const char *clockstr;
  5658. int ret;
  5659. if (cnt >= sizeof(buf))
  5660. return -EINVAL;
  5661. if (copy_from_user(buf, ubuf, cnt))
  5662. return -EFAULT;
  5663. buf[cnt] = 0;
  5664. clockstr = strstrip(buf);
  5665. ret = tracing_set_clock(tr, clockstr);
  5666. if (ret)
  5667. return ret;
  5668. *fpos += cnt;
  5669. return cnt;
  5670. }
  5671. static int tracing_clock_open(struct inode *inode, struct file *file)
  5672. {
  5673. struct trace_array *tr = inode->i_private;
  5674. int ret;
  5675. ret = tracing_check_open_get_tr(tr);
  5676. if (ret)
  5677. return ret;
  5678. ret = single_open(file, tracing_clock_show, inode->i_private);
  5679. if (ret < 0)
  5680. trace_array_put(tr);
  5681. return ret;
  5682. }
  5683. static int tracing_time_stamp_mode_show(struct seq_file *m, void *v)
  5684. {
  5685. struct trace_array *tr = m->private;
  5686. mutex_lock(&trace_types_lock);
  5687. if (ring_buffer_time_stamp_abs(tr->array_buffer.buffer))
  5688. seq_puts(m, "delta [absolute]\n");
  5689. else
  5690. seq_puts(m, "[delta] absolute\n");
  5691. mutex_unlock(&trace_types_lock);
  5692. return 0;
  5693. }
  5694. static int tracing_time_stamp_mode_open(struct inode *inode, struct file *file)
  5695. {
  5696. struct trace_array *tr = inode->i_private;
  5697. int ret;
  5698. ret = tracing_check_open_get_tr(tr);
  5699. if (ret)
  5700. return ret;
  5701. ret = single_open(file, tracing_time_stamp_mode_show, inode->i_private);
  5702. if (ret < 0)
  5703. trace_array_put(tr);
  5704. return ret;
  5705. }
  5706. int tracing_set_time_stamp_abs(struct trace_array *tr, bool abs)
  5707. {
  5708. int ret = 0;
  5709. mutex_lock(&trace_types_lock);
  5710. if (abs && tr->time_stamp_abs_ref++)
  5711. goto out;
  5712. if (!abs) {
  5713. if (WARN_ON_ONCE(!tr->time_stamp_abs_ref)) {
  5714. ret = -EINVAL;
  5715. goto out;
  5716. }
  5717. if (--tr->time_stamp_abs_ref)
  5718. goto out;
  5719. }
  5720. ring_buffer_set_time_stamp_abs(tr->array_buffer.buffer, abs);
  5721. #ifdef CONFIG_TRACER_MAX_TRACE
  5722. if (tr->max_buffer.buffer)
  5723. ring_buffer_set_time_stamp_abs(tr->max_buffer.buffer, abs);
  5724. #endif
  5725. out:
  5726. mutex_unlock(&trace_types_lock);
  5727. return ret;
  5728. }
  5729. struct ftrace_buffer_info {
  5730. struct trace_iterator iter;
  5731. void *spare;
  5732. unsigned int spare_cpu;
  5733. unsigned int read;
  5734. };
  5735. #ifdef CONFIG_TRACER_SNAPSHOT
  5736. static int tracing_snapshot_open(struct inode *inode, struct file *file)
  5737. {
  5738. struct trace_array *tr = inode->i_private;
  5739. struct trace_iterator *iter;
  5740. struct seq_file *m;
  5741. int ret;
  5742. ret = tracing_check_open_get_tr(tr);
  5743. if (ret)
  5744. return ret;
  5745. if (file->f_mode & FMODE_READ) {
  5746. iter = __tracing_open(inode, file, true);
  5747. if (IS_ERR(iter))
  5748. ret = PTR_ERR(iter);
  5749. } else {
  5750. /* Writes still need the seq_file to hold the private data */
  5751. ret = -ENOMEM;
  5752. m = kzalloc(sizeof(*m), GFP_KERNEL);
  5753. if (!m)
  5754. goto out;
  5755. iter = kzalloc(sizeof(*iter), GFP_KERNEL);
  5756. if (!iter) {
  5757. kfree(m);
  5758. goto out;
  5759. }
  5760. ret = 0;
  5761. iter->tr = tr;
  5762. iter->array_buffer = &tr->max_buffer;
  5763. iter->cpu_file = tracing_get_cpu(inode);
  5764. m->private = iter;
  5765. file->private_data = m;
  5766. }
  5767. out:
  5768. if (ret < 0)
  5769. trace_array_put(tr);
  5770. return ret;
  5771. }
  5772. static ssize_t
  5773. tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
  5774. loff_t *ppos)
  5775. {
  5776. struct seq_file *m = filp->private_data;
  5777. struct trace_iterator *iter = m->private;
  5778. struct trace_array *tr = iter->tr;
  5779. unsigned long val;
  5780. int ret;
  5781. ret = tracing_update_buffers();
  5782. if (ret < 0)
  5783. return ret;
  5784. ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
  5785. if (ret)
  5786. return ret;
  5787. mutex_lock(&trace_types_lock);
  5788. if (tr->current_trace->use_max_tr) {
  5789. ret = -EBUSY;
  5790. goto out;
  5791. }
  5792. arch_spin_lock(&tr->max_lock);
  5793. if (tr->cond_snapshot)
  5794. ret = -EBUSY;
  5795. arch_spin_unlock(&tr->max_lock);
  5796. if (ret)
  5797. goto out;
  5798. switch (val) {
  5799. case 0:
  5800. if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
  5801. ret = -EINVAL;
  5802. break;
  5803. }
  5804. if (tr->allocated_snapshot)
  5805. free_snapshot(tr);
  5806. break;
  5807. case 1:
  5808. /* Only allow per-cpu swap if the ring buffer supports it */
  5809. #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
  5810. if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
  5811. ret = -EINVAL;
  5812. break;
  5813. }
  5814. #endif
  5815. if (tr->allocated_snapshot)
  5816. ret = resize_buffer_duplicate_size(&tr->max_buffer,
  5817. &tr->array_buffer, iter->cpu_file);
  5818. else
  5819. ret = tracing_alloc_snapshot_instance(tr);
  5820. if (ret < 0)
  5821. break;
  5822. local_irq_disable();
  5823. /* Now, we're going to swap */
  5824. if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
  5825. update_max_tr(tr, current, smp_processor_id(), NULL);
  5826. else
  5827. update_max_tr_single(tr, current, iter->cpu_file);
  5828. local_irq_enable();
  5829. break;
  5830. default:
  5831. if (tr->allocated_snapshot) {
  5832. if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
  5833. tracing_reset_online_cpus(&tr->max_buffer);
  5834. else
  5835. tracing_reset_cpu(&tr->max_buffer, iter->cpu_file);
  5836. }
  5837. break;
  5838. }
  5839. if (ret >= 0) {
  5840. *ppos += cnt;
  5841. ret = cnt;
  5842. }
  5843. out:
  5844. mutex_unlock(&trace_types_lock);
  5845. return ret;
  5846. }
  5847. static int tracing_snapshot_release(struct inode *inode, struct file *file)
  5848. {
  5849. struct seq_file *m = file->private_data;
  5850. int ret;
  5851. ret = tracing_release(inode, file);
  5852. if (file->f_mode & FMODE_READ)
  5853. return ret;
  5854. /* If write only, the seq_file is just a stub */
  5855. if (m)
  5856. kfree(m->private);
  5857. kfree(m);
  5858. return 0;
  5859. }
  5860. static int tracing_buffers_open(struct inode *inode, struct file *filp);
  5861. static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
  5862. size_t count, loff_t *ppos);
  5863. static int tracing_buffers_release(struct inode *inode, struct file *file);
  5864. static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
  5865. struct pipe_inode_info *pipe, size_t len, unsigned int flags);
  5866. static int snapshot_raw_open(struct inode *inode, struct file *filp)
  5867. {
  5868. struct ftrace_buffer_info *info;
  5869. int ret;
  5870. /* The following checks for tracefs lockdown */
  5871. ret = tracing_buffers_open(inode, filp);
  5872. if (ret < 0)
  5873. return ret;
  5874. info = filp->private_data;
  5875. if (info->iter.trace->use_max_tr) {
  5876. tracing_buffers_release(inode, filp);
  5877. return -EBUSY;
  5878. }
  5879. info->iter.snapshot = true;
  5880. info->iter.array_buffer = &info->iter.tr->max_buffer;
  5881. return ret;
  5882. }
  5883. #endif /* CONFIG_TRACER_SNAPSHOT */
  5884. static const struct file_operations tracing_thresh_fops = {
  5885. .open = tracing_open_generic,
  5886. .read = tracing_thresh_read,
  5887. .write = tracing_thresh_write,
  5888. .llseek = generic_file_llseek,
  5889. };
  5890. #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
  5891. static const struct file_operations tracing_max_lat_fops = {
  5892. .open = tracing_open_generic,
  5893. .read = tracing_max_lat_read,
  5894. .write = tracing_max_lat_write,
  5895. .llseek = generic_file_llseek,
  5896. };
  5897. #endif
  5898. static const struct file_operations set_tracer_fops = {
  5899. .open = tracing_open_generic,
  5900. .read = tracing_set_trace_read,
  5901. .write = tracing_set_trace_write,
  5902. .llseek = generic_file_llseek,
  5903. };
  5904. static const struct file_operations tracing_pipe_fops = {
  5905. .open = tracing_open_pipe,
  5906. .poll = tracing_poll_pipe,
  5907. .read = tracing_read_pipe,
  5908. .splice_read = tracing_splice_read_pipe,
  5909. .release = tracing_release_pipe,
  5910. .llseek = no_llseek,
  5911. };
  5912. static const struct file_operations tracing_entries_fops = {
  5913. .open = tracing_open_generic_tr,
  5914. .read = tracing_entries_read,
  5915. .write = tracing_entries_write,
  5916. .llseek = generic_file_llseek,
  5917. .release = tracing_release_generic_tr,
  5918. };
  5919. static const struct file_operations tracing_total_entries_fops = {
  5920. .open = tracing_open_generic_tr,
  5921. .read = tracing_total_entries_read,
  5922. .llseek = generic_file_llseek,
  5923. .release = tracing_release_generic_tr,
  5924. };
  5925. static const struct file_operations tracing_free_buffer_fops = {
  5926. .open = tracing_open_generic_tr,
  5927. .write = tracing_free_buffer_write,
  5928. .release = tracing_free_buffer_release,
  5929. };
  5930. static const struct file_operations tracing_mark_fops = {
  5931. .open = tracing_open_generic_tr,
  5932. .write = tracing_mark_write,
  5933. .llseek = generic_file_llseek,
  5934. .release = tracing_release_generic_tr,
  5935. };
  5936. static const struct file_operations tracing_mark_raw_fops = {
  5937. .open = tracing_open_generic_tr,
  5938. .write = tracing_mark_raw_write,
  5939. .llseek = generic_file_llseek,
  5940. .release = tracing_release_generic_tr,
  5941. };
  5942. static const struct file_operations trace_clock_fops = {
  5943. .open = tracing_clock_open,
  5944. .read = seq_read,
  5945. .llseek = seq_lseek,
  5946. .release = tracing_single_release_tr,
  5947. .write = tracing_clock_write,
  5948. };
  5949. static const struct file_operations trace_time_stamp_mode_fops = {
  5950. .open = tracing_time_stamp_mode_open,
  5951. .read = seq_read,
  5952. .llseek = seq_lseek,
  5953. .release = tracing_single_release_tr,
  5954. };
  5955. #ifdef CONFIG_TRACER_SNAPSHOT
  5956. static const struct file_operations snapshot_fops = {
  5957. .open = tracing_snapshot_open,
  5958. .read = seq_read,
  5959. .write = tracing_snapshot_write,
  5960. .llseek = tracing_lseek,
  5961. .release = tracing_snapshot_release,
  5962. };
  5963. static const struct file_operations snapshot_raw_fops = {
  5964. .open = snapshot_raw_open,
  5965. .read = tracing_buffers_read,
  5966. .release = tracing_buffers_release,
  5967. .splice_read = tracing_buffers_splice_read,
  5968. .llseek = no_llseek,
  5969. };
  5970. #endif /* CONFIG_TRACER_SNAPSHOT */
  5971. #define TRACING_LOG_ERRS_MAX 8
  5972. #define TRACING_LOG_LOC_MAX 128
  5973. #define CMD_PREFIX " Command: "
  5974. struct err_info {
  5975. const char **errs; /* ptr to loc-specific array of err strings */
  5976. u8 type; /* index into errs -> specific err string */
  5977. u8 pos; /* MAX_FILTER_STR_VAL = 256 */
  5978. u64 ts;
  5979. };
  5980. struct tracing_log_err {
  5981. struct list_head list;
  5982. struct err_info info;
  5983. char loc[TRACING_LOG_LOC_MAX]; /* err location */
  5984. char cmd[MAX_FILTER_STR_VAL]; /* what caused err */
  5985. };
  5986. static DEFINE_MUTEX(tracing_err_log_lock);
  5987. static struct tracing_log_err *get_tracing_log_err(struct trace_array *tr)
  5988. {
  5989. struct tracing_log_err *err;
  5990. if (tr->n_err_log_entries < TRACING_LOG_ERRS_MAX) {
  5991. err = kzalloc(sizeof(*err), GFP_KERNEL);
  5992. if (!err)
  5993. err = ERR_PTR(-ENOMEM);
  5994. else
  5995. tr->n_err_log_entries++;
  5996. return err;
  5997. }
  5998. err = list_first_entry(&tr->err_log, struct tracing_log_err, list);
  5999. list_del(&err->list);
  6000. return err;
  6001. }
  6002. /**
  6003. * err_pos - find the position of a string within a command for error careting
  6004. * @cmd: The tracing command that caused the error
  6005. * @str: The string to position the caret at within @cmd
  6006. *
  6007. * Finds the position of the first occurence of @str within @cmd. The
  6008. * return value can be passed to tracing_log_err() for caret placement
  6009. * within @cmd.
  6010. *
  6011. * Returns the index within @cmd of the first occurence of @str or 0
  6012. * if @str was not found.
  6013. */
  6014. unsigned int err_pos(char *cmd, const char *str)
  6015. {
  6016. char *found;
  6017. if (WARN_ON(!strlen(cmd)))
  6018. return 0;
  6019. found = strstr(cmd, str);
  6020. if (found)
  6021. return found - cmd;
  6022. return 0;
  6023. }
  6024. /**
  6025. * tracing_log_err - write an error to the tracing error log
  6026. * @tr: The associated trace array for the error (NULL for top level array)
  6027. * @loc: A string describing where the error occurred
  6028. * @cmd: The tracing command that caused the error
  6029. * @errs: The array of loc-specific static error strings
  6030. * @type: The index into errs[], which produces the specific static err string
  6031. * @pos: The position the caret should be placed in the cmd
  6032. *
  6033. * Writes an error into tracing/error_log of the form:
  6034. *
  6035. * <loc>: error: <text>
  6036. * Command: <cmd>
  6037. * ^
  6038. *
  6039. * tracing/error_log is a small log file containing the last
  6040. * TRACING_LOG_ERRS_MAX errors (8). Memory for errors isn't allocated
  6041. * unless there has been a tracing error, and the error log can be
  6042. * cleared and have its memory freed by writing the empty string in
  6043. * truncation mode to it i.e. echo > tracing/error_log.
  6044. *
  6045. * NOTE: the @errs array along with the @type param are used to
  6046. * produce a static error string - this string is not copied and saved
  6047. * when the error is logged - only a pointer to it is saved. See
  6048. * existing callers for examples of how static strings are typically
  6049. * defined for use with tracing_log_err().
  6050. */
  6051. void tracing_log_err(struct trace_array *tr,
  6052. const char *loc, const char *cmd,
  6053. const char **errs, u8 type, u8 pos)
  6054. {
  6055. struct tracing_log_err *err;
  6056. if (!tr)
  6057. tr = &global_trace;
  6058. mutex_lock(&tracing_err_log_lock);
  6059. err = get_tracing_log_err(tr);
  6060. if (PTR_ERR(err) == -ENOMEM) {
  6061. mutex_unlock(&tracing_err_log_lock);
  6062. return;
  6063. }
  6064. snprintf(err->loc, TRACING_LOG_LOC_MAX, "%s: error: ", loc);
  6065. snprintf(err->cmd, MAX_FILTER_STR_VAL,"\n" CMD_PREFIX "%s\n", cmd);
  6066. err->info.errs = errs;
  6067. err->info.type = type;
  6068. err->info.pos = pos;
  6069. err->info.ts = local_clock();
  6070. list_add_tail(&err->list, &tr->err_log);
  6071. mutex_unlock(&tracing_err_log_lock);
  6072. }
  6073. static void clear_tracing_err_log(struct trace_array *tr)
  6074. {
  6075. struct tracing_log_err *err, *next;
  6076. mutex_lock(&tracing_err_log_lock);
  6077. list_for_each_entry_safe(err, next, &tr->err_log, list) {
  6078. list_del(&err->list);
  6079. kfree(err);
  6080. }
  6081. tr->n_err_log_entries = 0;
  6082. mutex_unlock(&tracing_err_log_lock);
  6083. }
  6084. static void *tracing_err_log_seq_start(struct seq_file *m, loff_t *pos)
  6085. {
  6086. struct trace_array *tr = m->private;
  6087. mutex_lock(&tracing_err_log_lock);
  6088. return seq_list_start(&tr->err_log, *pos);
  6089. }
  6090. static void *tracing_err_log_seq_next(struct seq_file *m, void *v, loff_t *pos)
  6091. {
  6092. struct trace_array *tr = m->private;
  6093. return seq_list_next(v, &tr->err_log, pos);
  6094. }
  6095. static void tracing_err_log_seq_stop(struct seq_file *m, void *v)
  6096. {
  6097. mutex_unlock(&tracing_err_log_lock);
  6098. }
  6099. static void tracing_err_log_show_pos(struct seq_file *m, u8 pos)
  6100. {
  6101. u8 i;
  6102. for (i = 0; i < sizeof(CMD_PREFIX) - 1; i++)
  6103. seq_putc(m, ' ');
  6104. for (i = 0; i < pos; i++)
  6105. seq_putc(m, ' ');
  6106. seq_puts(m, "^\n");
  6107. }
  6108. static int tracing_err_log_seq_show(struct seq_file *m, void *v)
  6109. {
  6110. struct tracing_log_err *err = v;
  6111. if (err) {
  6112. const char *err_text = err->info.errs[err->info.type];
  6113. u64 sec = err->info.ts;
  6114. u32 nsec;
  6115. nsec = do_div(sec, NSEC_PER_SEC);
  6116. seq_printf(m, "[%5llu.%06u] %s%s", sec, nsec / 1000,
  6117. err->loc, err_text);
  6118. seq_printf(m, "%s", err->cmd);
  6119. tracing_err_log_show_pos(m, err->info.pos);
  6120. }
  6121. return 0;
  6122. }
  6123. static const struct seq_operations tracing_err_log_seq_ops = {
  6124. .start = tracing_err_log_seq_start,
  6125. .next = tracing_err_log_seq_next,
  6126. .stop = tracing_err_log_seq_stop,
  6127. .show = tracing_err_log_seq_show
  6128. };
  6129. static int tracing_err_log_open(struct inode *inode, struct file *file)
  6130. {
  6131. struct trace_array *tr = inode->i_private;
  6132. int ret = 0;
  6133. ret = tracing_check_open_get_tr(tr);
  6134. if (ret)
  6135. return ret;
  6136. /* If this file was opened for write, then erase contents */
  6137. if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC))
  6138. clear_tracing_err_log(tr);
  6139. if (file->f_mode & FMODE_READ) {
  6140. ret = seq_open(file, &tracing_err_log_seq_ops);
  6141. if (!ret) {
  6142. struct seq_file *m = file->private_data;
  6143. m->private = tr;
  6144. } else {
  6145. trace_array_put(tr);
  6146. }
  6147. }
  6148. return ret;
  6149. }
  6150. static ssize_t tracing_err_log_write(struct file *file,
  6151. const char __user *buffer,
  6152. size_t count, loff_t *ppos)
  6153. {
  6154. return count;
  6155. }
  6156. static int tracing_err_log_release(struct inode *inode, struct file *file)
  6157. {
  6158. struct trace_array *tr = inode->i_private;
  6159. trace_array_put(tr);
  6160. if (file->f_mode & FMODE_READ)
  6161. seq_release(inode, file);
  6162. return 0;
  6163. }
  6164. static const struct file_operations tracing_err_log_fops = {
  6165. .open = tracing_err_log_open,
  6166. .write = tracing_err_log_write,
  6167. .read = seq_read,
  6168. .llseek = seq_lseek,
  6169. .release = tracing_err_log_release,
  6170. };
  6171. static int tracing_buffers_open(struct inode *inode, struct file *filp)
  6172. {
  6173. struct trace_array *tr = inode->i_private;
  6174. struct ftrace_buffer_info *info;
  6175. int ret;
  6176. ret = tracing_check_open_get_tr(tr);
  6177. if (ret)
  6178. return ret;
  6179. info = kvzalloc(sizeof(*info), GFP_KERNEL);
  6180. if (!info) {
  6181. trace_array_put(tr);
  6182. return -ENOMEM;
  6183. }
  6184. mutex_lock(&trace_types_lock);
  6185. info->iter.tr = tr;
  6186. info->iter.cpu_file = tracing_get_cpu(inode);
  6187. info->iter.trace = tr->current_trace;
  6188. info->iter.array_buffer = &tr->array_buffer;
  6189. info->spare = NULL;
  6190. /* Force reading ring buffer for first read */
  6191. info->read = (unsigned int)-1;
  6192. filp->private_data = info;
  6193. tr->trace_ref++;
  6194. mutex_unlock(&trace_types_lock);
  6195. ret = nonseekable_open(inode, filp);
  6196. if (ret < 0)
  6197. trace_array_put(tr);
  6198. return ret;
  6199. }
  6200. static __poll_t
  6201. tracing_buffers_poll(struct file *filp, poll_table *poll_table)
  6202. {
  6203. struct ftrace_buffer_info *info = filp->private_data;
  6204. struct trace_iterator *iter = &info->iter;
  6205. return trace_poll(iter, filp, poll_table);
  6206. }
  6207. static ssize_t
  6208. tracing_buffers_read(struct file *filp, char __user *ubuf,
  6209. size_t count, loff_t *ppos)
  6210. {
  6211. struct ftrace_buffer_info *info = filp->private_data;
  6212. struct trace_iterator *iter = &info->iter;
  6213. ssize_t ret = 0;
  6214. ssize_t size;
  6215. if (!count)
  6216. return 0;
  6217. #ifdef CONFIG_TRACER_MAX_TRACE
  6218. if (iter->snapshot && iter->tr->current_trace->use_max_tr)
  6219. return -EBUSY;
  6220. #endif
  6221. if (!info->spare) {
  6222. info->spare = ring_buffer_alloc_read_page(iter->array_buffer->buffer,
  6223. iter->cpu_file);
  6224. if (IS_ERR(info->spare)) {
  6225. ret = PTR_ERR(info->spare);
  6226. info->spare = NULL;
  6227. } else {
  6228. info->spare_cpu = iter->cpu_file;
  6229. }
  6230. }
  6231. if (!info->spare)
  6232. return ret;
  6233. /* Do we have previous read data to read? */
  6234. if (info->read < PAGE_SIZE)
  6235. goto read;
  6236. again:
  6237. trace_access_lock(iter->cpu_file);
  6238. ret = ring_buffer_read_page(iter->array_buffer->buffer,
  6239. &info->spare,
  6240. count,
  6241. iter->cpu_file, 0);
  6242. trace_access_unlock(iter->cpu_file);
  6243. if (ret < 0) {
  6244. if (trace_empty(iter)) {
  6245. if ((filp->f_flags & O_NONBLOCK))
  6246. return -EAGAIN;
  6247. ret = wait_on_pipe(iter, 0);
  6248. if (ret)
  6249. return ret;
  6250. goto again;
  6251. }
  6252. return 0;
  6253. }
  6254. info->read = 0;
  6255. read:
  6256. size = PAGE_SIZE - info->read;
  6257. if (size > count)
  6258. size = count;
  6259. ret = copy_to_user(ubuf, info->spare + info->read, size);
  6260. if (ret == size)
  6261. return -EFAULT;
  6262. size -= ret;
  6263. *ppos += size;
  6264. info->read += size;
  6265. return size;
  6266. }
  6267. static int tracing_buffers_release(struct inode *inode, struct file *file)
  6268. {
  6269. struct ftrace_buffer_info *info = file->private_data;
  6270. struct trace_iterator *iter = &info->iter;
  6271. mutex_lock(&trace_types_lock);
  6272. iter->tr->trace_ref--;
  6273. __trace_array_put(iter->tr);
  6274. if (info->spare)
  6275. ring_buffer_free_read_page(iter->array_buffer->buffer,
  6276. info->spare_cpu, info->spare);
  6277. kvfree(info);
  6278. mutex_unlock(&trace_types_lock);
  6279. return 0;
  6280. }
  6281. struct buffer_ref {
  6282. struct trace_buffer *buffer;
  6283. void *page;
  6284. int cpu;
  6285. refcount_t refcount;
  6286. };
  6287. static void buffer_ref_release(struct buffer_ref *ref)
  6288. {
  6289. if (!refcount_dec_and_test(&ref->refcount))
  6290. return;
  6291. ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page);
  6292. kfree(ref);
  6293. }
  6294. static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
  6295. struct pipe_buffer *buf)
  6296. {
  6297. struct buffer_ref *ref = (struct buffer_ref *)buf->private;
  6298. buffer_ref_release(ref);
  6299. buf->private = 0;
  6300. }
  6301. static bool buffer_pipe_buf_get(struct pipe_inode_info *pipe,
  6302. struct pipe_buffer *buf)
  6303. {
  6304. struct buffer_ref *ref = (struct buffer_ref *)buf->private;
  6305. if (refcount_read(&ref->refcount) > INT_MAX/2)
  6306. return false;
  6307. refcount_inc(&ref->refcount);
  6308. return true;
  6309. }
  6310. /* Pipe buffer operations for a buffer. */
  6311. static const struct pipe_buf_operations buffer_pipe_buf_ops = {
  6312. .release = buffer_pipe_buf_release,
  6313. .get = buffer_pipe_buf_get,
  6314. };
  6315. /*
  6316. * Callback from splice_to_pipe(), if we need to release some pages
  6317. * at the end of the spd in case we error'ed out in filling the pipe.
  6318. */
  6319. static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
  6320. {
  6321. struct buffer_ref *ref =
  6322. (struct buffer_ref *)spd->partial[i].private;
  6323. buffer_ref_release(ref);
  6324. spd->partial[i].private = 0;
  6325. }
  6326. static ssize_t
  6327. tracing_buffers_splice_read(struct file *file, loff_t *ppos,
  6328. struct pipe_inode_info *pipe, size_t len,
  6329. unsigned int flags)
  6330. {
  6331. struct ftrace_buffer_info *info = file->private_data;
  6332. struct trace_iterator *iter = &info->iter;
  6333. struct partial_page partial_def[PIPE_DEF_BUFFERS];
  6334. struct page *pages_def[PIPE_DEF_BUFFERS];
  6335. struct splice_pipe_desc spd = {
  6336. .pages = pages_def,
  6337. .partial = partial_def,
  6338. .nr_pages_max = PIPE_DEF_BUFFERS,
  6339. .ops = &buffer_pipe_buf_ops,
  6340. .spd_release = buffer_spd_release,
  6341. };
  6342. struct buffer_ref *ref;
  6343. int entries, i;
  6344. ssize_t ret = 0;
  6345. #ifdef CONFIG_TRACER_MAX_TRACE
  6346. if (iter->snapshot && iter->tr->current_trace->use_max_tr)
  6347. return -EBUSY;
  6348. #endif
  6349. if (*ppos & (PAGE_SIZE - 1))
  6350. return -EINVAL;
  6351. if (len & (PAGE_SIZE - 1)) {
  6352. if (len < PAGE_SIZE)
  6353. return -EINVAL;
  6354. len &= PAGE_MASK;
  6355. }
  6356. if (splice_grow_spd(pipe, &spd))
  6357. return -ENOMEM;
  6358. again:
  6359. trace_access_lock(iter->cpu_file);
  6360. entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);
  6361. for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
  6362. struct page *page;
  6363. int r;
  6364. ref = kzalloc(sizeof(*ref), GFP_KERNEL);
  6365. if (!ref) {
  6366. ret = -ENOMEM;
  6367. break;
  6368. }
  6369. refcount_set(&ref->refcount, 1);
  6370. ref->buffer = iter->array_buffer->buffer;
  6371. ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
  6372. if (IS_ERR(ref->page)) {
  6373. ret = PTR_ERR(ref->page);
  6374. ref->page = NULL;
  6375. kfree(ref);
  6376. break;
  6377. }
  6378. ref->cpu = iter->cpu_file;
  6379. r = ring_buffer_read_page(ref->buffer, &ref->page,
  6380. len, iter->cpu_file, 1);
  6381. if (r < 0) {
  6382. ring_buffer_free_read_page(ref->buffer, ref->cpu,
  6383. ref->page);
  6384. kfree(ref);
  6385. break;
  6386. }
  6387. page = virt_to_page(ref->page);
  6388. spd.pages[i] = page;
  6389. spd.partial[i].len = PAGE_SIZE;
  6390. spd.partial[i].offset = 0;
  6391. spd.partial[i].private = (unsigned long)ref;
  6392. spd.nr_pages++;
  6393. *ppos += PAGE_SIZE;
  6394. entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);
  6395. }
  6396. trace_access_unlock(iter->cpu_file);
  6397. spd.nr_pages = i;
  6398. /* did we read anything? */
  6399. if (!spd.nr_pages) {
  6400. if (ret)
  6401. goto out;
  6402. ret = -EAGAIN;
  6403. if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
  6404. goto out;
  6405. ret = wait_on_pipe(iter, iter->tr->buffer_percent);
  6406. if (ret)
  6407. goto out;
  6408. goto again;
  6409. }
  6410. ret = splice_to_pipe(pipe, &spd);
  6411. out:
  6412. splice_shrink_spd(&spd);
  6413. return ret;
  6414. }
  6415. static const struct file_operations tracing_buffers_fops = {
  6416. .open = tracing_buffers_open,
  6417. .read = tracing_buffers_read,
  6418. .poll = tracing_buffers_poll,
  6419. .release = tracing_buffers_release,
  6420. .splice_read = tracing_buffers_splice_read,
  6421. .llseek = no_llseek,
  6422. };
  6423. static ssize_t
  6424. tracing_stats_read(struct file *filp, char __user *ubuf,
  6425. size_t count, loff_t *ppos)
  6426. {
  6427. struct inode *inode = file_inode(filp);
  6428. struct trace_array *tr = inode->i_private;
  6429. struct array_buffer *trace_buf = &tr->array_buffer;
  6430. int cpu = tracing_get_cpu(inode);
  6431. struct trace_seq *s;
  6432. unsigned long cnt;
  6433. unsigned long long t;
  6434. unsigned long usec_rem;
  6435. s = kmalloc(sizeof(*s), GFP_KERNEL);
  6436. if (!s)
  6437. return -ENOMEM;
  6438. trace_seq_init(s);
  6439. cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
  6440. trace_seq_printf(s, "entries: %ld\n", cnt);
  6441. cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
  6442. trace_seq_printf(s, "overrun: %ld\n", cnt);
  6443. cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
  6444. trace_seq_printf(s, "commit overrun: %ld\n", cnt);
  6445. cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
  6446. trace_seq_printf(s, "bytes: %ld\n", cnt);
  6447. if (trace_clocks[tr->clock_id].in_ns) {
  6448. /* local or global for trace_clock */
  6449. t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
  6450. usec_rem = do_div(t, USEC_PER_SEC);
  6451. trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
  6452. t, usec_rem);
  6453. t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
  6454. usec_rem = do_div(t, USEC_PER_SEC);
  6455. trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
  6456. } else {
  6457. /* counter or tsc mode for trace_clock */
  6458. trace_seq_printf(s, "oldest event ts: %llu\n",
  6459. ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
  6460. trace_seq_printf(s, "now ts: %llu\n",
  6461. ring_buffer_time_stamp(trace_buf->buffer, cpu));
  6462. }
  6463. cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
  6464. trace_seq_printf(s, "dropped events: %ld\n", cnt);
  6465. cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
  6466. trace_seq_printf(s, "read events: %ld\n", cnt);
  6467. count = simple_read_from_buffer(ubuf, count, ppos,
  6468. s->buffer, trace_seq_used(s));
  6469. kfree(s);
  6470. return count;
  6471. }
  6472. static const struct file_operations tracing_stats_fops = {
  6473. .open = tracing_open_generic_tr,
  6474. .read = tracing_stats_read,
  6475. .llseek = generic_file_llseek,
  6476. .release = tracing_release_generic_tr,
  6477. };
  6478. #ifdef CONFIG_DYNAMIC_FTRACE
  6479. static ssize_t
  6480. tracing_read_dyn_info(struct file *filp, char __user *ubuf,
  6481. size_t cnt, loff_t *ppos)
  6482. {
  6483. ssize_t ret;
  6484. char *buf;
  6485. int r;
  6486. /* 256 should be plenty to hold the amount needed */
  6487. buf = kmalloc(256, GFP_KERNEL);
  6488. if (!buf)
  6489. return -ENOMEM;
  6490. r = scnprintf(buf, 256, "%ld pages:%ld groups: %ld\n",
  6491. ftrace_update_tot_cnt,
  6492. ftrace_number_of_pages,
  6493. ftrace_number_of_groups);
  6494. ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
  6495. kfree(buf);
  6496. return ret;
  6497. }
  6498. static const struct file_operations tracing_dyn_info_fops = {
  6499. .open = tracing_open_generic,
  6500. .read = tracing_read_dyn_info,
  6501. .llseek = generic_file_llseek,
  6502. };
  6503. #endif /* CONFIG_DYNAMIC_FTRACE */
  6504. #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
  6505. static void
  6506. ftrace_snapshot(unsigned long ip, unsigned long parent_ip,
  6507. struct trace_array *tr, struct ftrace_probe_ops *ops,
  6508. void *data)
  6509. {
  6510. tracing_snapshot_instance(tr);
  6511. }
  6512. static void
  6513. ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip,
  6514. struct trace_array *tr, struct ftrace_probe_ops *ops,
  6515. void *data)
  6516. {
  6517. struct ftrace_func_mapper *mapper = data;
  6518. long *count = NULL;
  6519. if (mapper)
  6520. count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
  6521. if (count) {
  6522. if (*count <= 0)
  6523. return;
  6524. (*count)--;
  6525. }
  6526. tracing_snapshot_instance(tr);
  6527. }
  6528. static int
  6529. ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
  6530. struct ftrace_probe_ops *ops, void *data)
  6531. {
  6532. struct ftrace_func_mapper *mapper = data;
  6533. long *count = NULL;
  6534. seq_printf(m, "%ps:", (void *)ip);
  6535. seq_puts(m, "snapshot");
  6536. if (mapper)
  6537. count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
  6538. if (count)
  6539. seq_printf(m, ":count=%ld\n", *count);
  6540. else
  6541. seq_puts(m, ":unlimited\n");
  6542. return 0;
  6543. }
  6544. static int
  6545. ftrace_snapshot_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
  6546. unsigned long ip, void *init_data, void **data)
  6547. {
  6548. struct ftrace_func_mapper *mapper = *data;
  6549. if (!mapper) {
  6550. mapper = allocate_ftrace_func_mapper();
  6551. if (!mapper)
  6552. return -ENOMEM;
  6553. *data = mapper;
  6554. }
  6555. return ftrace_func_mapper_add_ip(mapper, ip, init_data);
  6556. }
  6557. static void
  6558. ftrace_snapshot_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
  6559. unsigned long ip, void *data)
  6560. {
  6561. struct ftrace_func_mapper *mapper = data;
  6562. if (!ip) {
  6563. if (!mapper)
  6564. return;
  6565. free_ftrace_func_mapper(mapper, NULL);
  6566. return;
  6567. }
  6568. ftrace_func_mapper_remove_ip(mapper, ip);
  6569. }
  6570. static struct ftrace_probe_ops snapshot_probe_ops = {
  6571. .func = ftrace_snapshot,
  6572. .print = ftrace_snapshot_print,
  6573. };
  6574. static struct ftrace_probe_ops snapshot_count_probe_ops = {
  6575. .func = ftrace_count_snapshot,
  6576. .print = ftrace_snapshot_print,
  6577. .init = ftrace_snapshot_init,
  6578. .free = ftrace_snapshot_free,
  6579. };
  6580. static int
  6581. ftrace_trace_snapshot_callback(struct trace_array *tr, struct ftrace_hash *hash,
  6582. char *glob, char *cmd, char *param, int enable)
  6583. {
  6584. struct ftrace_probe_ops *ops;
  6585. void *count = (void *)-1;
  6586. char *number;
  6587. int ret;
  6588. if (!tr)
  6589. return -ENODEV;
  6590. /* hash funcs only work with set_ftrace_filter */
  6591. if (!enable)
  6592. return -EINVAL;
  6593. ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
  6594. if (glob[0] == '!')
  6595. return unregister_ftrace_function_probe_func(glob+1, tr, ops);
  6596. if (!param)
  6597. goto out_reg;
  6598. number = strsep(&param, ":");
  6599. if (!strlen(number))
  6600. goto out_reg;
  6601. /*
  6602. * We use the callback data field (which is a pointer)
  6603. * as our counter.
  6604. */
  6605. ret = kstrtoul(number, 0, (unsigned long *)&count);
  6606. if (ret)
  6607. return ret;
  6608. out_reg:
  6609. ret = tracing_alloc_snapshot_instance(tr);
  6610. if (ret < 0)
  6611. goto out;
  6612. ret = register_ftrace_function_probe(glob, tr, ops, count);
  6613. out:
  6614. return ret < 0 ? ret : 0;
  6615. }
  6616. static struct ftrace_func_command ftrace_snapshot_cmd = {
  6617. .name = "snapshot",
  6618. .func = ftrace_trace_snapshot_callback,
  6619. };
  6620. static __init int register_snapshot_cmd(void)
  6621. {
  6622. return register_ftrace_command(&ftrace_snapshot_cmd);
  6623. }
  6624. #else
  6625. static inline __init int register_snapshot_cmd(void) { return 0; }
  6626. #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
  6627. static struct dentry *tracing_get_dentry(struct trace_array *tr)
  6628. {
  6629. if (WARN_ON(!tr->dir))
  6630. return ERR_PTR(-ENODEV);
  6631. /* Top directory uses NULL as the parent */
  6632. if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
  6633. return NULL;
  6634. /* All sub buffers have a descriptor */
  6635. return tr->dir;
  6636. }
  6637. static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
  6638. {
  6639. struct dentry *d_tracer;
  6640. if (tr->percpu_dir)
  6641. return tr->percpu_dir;
  6642. d_tracer = tracing_get_dentry(tr);
  6643. if (IS_ERR(d_tracer))
  6644. return NULL;
  6645. tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
  6646. MEM_FAIL(!tr->percpu_dir,
  6647. "Could not create tracefs directory 'per_cpu/%d'\n", cpu);
  6648. return tr->percpu_dir;
  6649. }
  6650. static struct dentry *
  6651. trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
  6652. void *data, long cpu, const struct file_operations *fops)
  6653. {
  6654. struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
  6655. if (ret) /* See tracing_get_cpu() */
  6656. d_inode(ret)->i_cdev = (void *)(cpu + 1);
  6657. return ret;
  6658. }
  6659. static void
  6660. tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
  6661. {
  6662. struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
  6663. struct dentry *d_cpu;
  6664. char cpu_dir[30]; /* 30 characters should be more than enough */
  6665. if (!d_percpu)
  6666. return;
  6667. snprintf(cpu_dir, 30, "cpu%ld", cpu);
  6668. d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
  6669. if (!d_cpu) {
  6670. pr_warn("Could not create tracefs '%s' entry\n", cpu_dir);
  6671. return;
  6672. }
  6673. /* per cpu trace_pipe */
  6674. trace_create_cpu_file("trace_pipe", 0444, d_cpu,
  6675. tr, cpu, &tracing_pipe_fops);
  6676. /* per cpu trace */
  6677. trace_create_cpu_file("trace", 0644, d_cpu,
  6678. tr, cpu, &tracing_fops);
  6679. trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
  6680. tr, cpu, &tracing_buffers_fops);
  6681. trace_create_cpu_file("stats", 0444, d_cpu,
  6682. tr, cpu, &tracing_stats_fops);
  6683. trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
  6684. tr, cpu, &tracing_entries_fops);
  6685. #ifdef CONFIG_TRACER_SNAPSHOT
  6686. trace_create_cpu_file("snapshot", 0644, d_cpu,
  6687. tr, cpu, &snapshot_fops);
  6688. trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
  6689. tr, cpu, &snapshot_raw_fops);
  6690. #endif
  6691. }
  6692. #ifdef CONFIG_FTRACE_SELFTEST
  6693. /* Let selftest have access to static functions in this file */
  6694. #include "trace_selftest.c"
  6695. #endif
  6696. static ssize_t
  6697. trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
  6698. loff_t *ppos)
  6699. {
  6700. struct trace_option_dentry *topt = filp->private_data;
  6701. char *buf;
  6702. if (topt->flags->val & topt->opt->bit)
  6703. buf = "1\n";
  6704. else
  6705. buf = "0\n";
  6706. return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
  6707. }
  6708. static ssize_t
  6709. trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
  6710. loff_t *ppos)
  6711. {
  6712. struct trace_option_dentry *topt = filp->private_data;
  6713. unsigned long val;
  6714. int ret;
  6715. ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
  6716. if (ret)
  6717. return ret;
  6718. if (val != 0 && val != 1)
  6719. return -EINVAL;
  6720. if (!!(topt->flags->val & topt->opt->bit) != val) {
  6721. mutex_lock(&trace_types_lock);
  6722. ret = __set_tracer_option(topt->tr, topt->flags,
  6723. topt->opt, !val);
  6724. mutex_unlock(&trace_types_lock);
  6725. if (ret)
  6726. return ret;
  6727. }
  6728. *ppos += cnt;
  6729. return cnt;
  6730. }
  6731. static const struct file_operations trace_options_fops = {
  6732. .open = tracing_open_generic,
  6733. .read = trace_options_read,
  6734. .write = trace_options_write,
  6735. .llseek = generic_file_llseek,
  6736. };
  6737. /*
  6738. * In order to pass in both the trace_array descriptor as well as the index
  6739. * to the flag that the trace option file represents, the trace_array
  6740. * has a character array of trace_flags_index[], which holds the index
  6741. * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
  6742. * The address of this character array is passed to the flag option file
  6743. * read/write callbacks.
  6744. *
  6745. * In order to extract both the index and the trace_array descriptor,
  6746. * get_tr_index() uses the following algorithm.
  6747. *
  6748. * idx = *ptr;
  6749. *
  6750. * As the pointer itself contains the address of the index (remember
  6751. * index[1] == 1).
  6752. *
  6753. * Then to get the trace_array descriptor, by subtracting that index
  6754. * from the ptr, we get to the start of the index itself.
  6755. *
  6756. * ptr - idx == &index[0]
  6757. *
  6758. * Then a simple container_of() from that pointer gets us to the
  6759. * trace_array descriptor.
  6760. */
  6761. static void get_tr_index(void *data, struct trace_array **ptr,
  6762. unsigned int *pindex)
  6763. {
  6764. *pindex = *(unsigned char *)data;
  6765. *ptr = container_of(data - *pindex, struct trace_array,
  6766. trace_flags_index);
  6767. }
  6768. static ssize_t
  6769. trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
  6770. loff_t *ppos)
  6771. {
  6772. void *tr_index = filp->private_data;
  6773. struct trace_array *tr;
  6774. unsigned int index;
  6775. char *buf;
  6776. get_tr_index(tr_index, &tr, &index);
  6777. if (tr->trace_flags & (1 << index))
  6778. buf = "1\n";
  6779. else
  6780. buf = "0\n";
  6781. return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
  6782. }
  6783. static ssize_t
  6784. trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
  6785. loff_t *ppos)
  6786. {
  6787. void *tr_index = filp->private_data;
  6788. struct trace_array *tr;
  6789. unsigned int index;
  6790. unsigned long val;
  6791. int ret;
  6792. get_tr_index(tr_index, &tr, &index);
  6793. ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
  6794. if (ret)
  6795. return ret;
  6796. if (val != 0 && val != 1)
  6797. return -EINVAL;
  6798. mutex_lock(&event_mutex);
  6799. mutex_lock(&trace_types_lock);
  6800. ret = set_tracer_flag(tr, 1 << index, val);
  6801. mutex_unlock(&trace_types_lock);
  6802. mutex_unlock(&event_mutex);
  6803. if (ret < 0)
  6804. return ret;
  6805. *ppos += cnt;
  6806. return cnt;
  6807. }
  6808. static const struct file_operations trace_options_core_fops = {
  6809. .open = tracing_open_generic,
  6810. .read = trace_options_core_read,
  6811. .write = trace_options_core_write,
  6812. .llseek = generic_file_llseek,
  6813. };
  6814. struct dentry *trace_create_file(const char *name,
  6815. umode_t mode,
  6816. struct dentry *parent,
  6817. void *data,
  6818. const struct file_operations *fops)
  6819. {
  6820. struct dentry *ret;
  6821. ret = tracefs_create_file(name, mode, parent, data, fops);
  6822. if (!ret)
  6823. pr_warn("Could not create tracefs '%s' entry\n", name);
  6824. return ret;
  6825. }
  6826. static struct dentry *trace_options_init_dentry(struct trace_array *tr)
  6827. {
  6828. struct dentry *d_tracer;
  6829. if (tr->options)
  6830. return tr->options;
  6831. d_tracer = tracing_get_dentry(tr);
  6832. if (IS_ERR(d_tracer))
  6833. return NULL;
  6834. tr->options = tracefs_create_dir("options", d_tracer);
  6835. if (!tr->options) {
  6836. pr_warn("Could not create tracefs directory 'options'\n");
  6837. return NULL;
  6838. }
  6839. return tr->options;
  6840. }
  6841. static void
  6842. create_trace_option_file(struct trace_array *tr,
  6843. struct trace_option_dentry *topt,
  6844. struct tracer_flags *flags,
  6845. struct tracer_opt *opt)
  6846. {
  6847. struct dentry *t_options;
  6848. t_options = trace_options_init_dentry(tr);
  6849. if (!t_options)
  6850. return;
  6851. topt->flags = flags;
  6852. topt->opt = opt;
  6853. topt->tr = tr;
  6854. topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
  6855. &trace_options_fops);
  6856. }
  6857. static void
  6858. create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
  6859. {
  6860. struct trace_option_dentry *topts;
  6861. struct trace_options *tr_topts;
  6862. struct tracer_flags *flags;
  6863. struct tracer_opt *opts;
  6864. int cnt;
  6865. int i;
  6866. if (!tracer)
  6867. return;
  6868. flags = tracer->flags;
  6869. if (!flags || !flags->opts)
  6870. return;
  6871. /*
  6872. * If this is an instance, only create flags for tracers
  6873. * the instance may have.
  6874. */
  6875. if (!trace_ok_for_array(tracer, tr))
  6876. return;
  6877. for (i = 0; i < tr->nr_topts; i++) {
  6878. /* Make sure there's no duplicate flags. */
  6879. if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags))
  6880. return;
  6881. }
  6882. opts = flags->opts;
  6883. for (cnt = 0; opts[cnt].name; cnt++)
  6884. ;
  6885. topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
  6886. if (!topts)
  6887. return;
  6888. tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1),
  6889. GFP_KERNEL);
  6890. if (!tr_topts) {
  6891. kfree(topts);
  6892. return;
  6893. }
  6894. tr->topts = tr_topts;
  6895. tr->topts[tr->nr_topts].tracer = tracer;
  6896. tr->topts[tr->nr_topts].topts = topts;
  6897. tr->nr_topts++;
  6898. for (cnt = 0; opts[cnt].name; cnt++) {
  6899. create_trace_option_file(tr, &topts[cnt], flags,
  6900. &opts[cnt]);
  6901. MEM_FAIL(topts[cnt].entry == NULL,
  6902. "Failed to create trace option: %s",
  6903. opts[cnt].name);
  6904. }
  6905. }
  6906. static struct dentry *
  6907. create_trace_option_core_file(struct trace_array *tr,
  6908. const char *option, long index)
  6909. {
  6910. struct dentry *t_options;
  6911. t_options = trace_options_init_dentry(tr);
  6912. if (!t_options)
  6913. return NULL;
  6914. return trace_create_file(option, 0644, t_options,
  6915. (void *)&tr->trace_flags_index[index],
  6916. &trace_options_core_fops);
  6917. }
  6918. static void create_trace_options_dir(struct trace_array *tr)
  6919. {
  6920. struct dentry *t_options;
  6921. bool top_level = tr == &global_trace;
  6922. int i;
  6923. t_options = trace_options_init_dentry(tr);
  6924. if (!t_options)
  6925. return;
  6926. for (i = 0; trace_options[i]; i++) {
  6927. if (top_level ||
  6928. !((1 << i) & TOP_LEVEL_TRACE_FLAGS))
  6929. create_trace_option_core_file(tr, trace_options[i], i);
  6930. }
  6931. }
  6932. static ssize_t
  6933. rb_simple_read(struct file *filp, char __user *ubuf,
  6934. size_t cnt, loff_t *ppos)
  6935. {
  6936. struct trace_array *tr = filp->private_data;
  6937. char buf[64];
  6938. int r;
  6939. r = tracer_tracing_is_on(tr);
  6940. r = sprintf(buf, "%d\n", r);
  6941. return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
  6942. }
  6943. static ssize_t
  6944. rb_simple_write(struct file *filp, const char __user *ubuf,
  6945. size_t cnt, loff_t *ppos)
  6946. {
  6947. struct trace_array *tr = filp->private_data;
  6948. struct trace_buffer *buffer = tr->array_buffer.buffer;
  6949. unsigned long val;
  6950. int ret;
  6951. ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
  6952. if (ret)
  6953. return ret;
  6954. if (buffer) {
  6955. mutex_lock(&trace_types_lock);
  6956. if (!!val == tracer_tracing_is_on(tr)) {
  6957. val = 0; /* do nothing */
  6958. } else if (val) {
  6959. tracer_tracing_on(tr);
  6960. if (tr->current_trace->start)
  6961. tr->current_trace->start(tr);
  6962. } else {
  6963. tracer_tracing_off(tr);
  6964. if (tr->current_trace->stop)
  6965. tr->current_trace->stop(tr);
  6966. }
  6967. mutex_unlock(&trace_types_lock);
  6968. }
  6969. (*ppos)++;
  6970. return cnt;
  6971. }
  6972. static const struct file_operations rb_simple_fops = {
  6973. .open = tracing_open_generic_tr,
  6974. .read = rb_simple_read,
  6975. .write = rb_simple_write,
  6976. .release = tracing_release_generic_tr,
  6977. .llseek = default_llseek,
  6978. };
  6979. static ssize_t
  6980. buffer_percent_read(struct file *filp, char __user *ubuf,
  6981. size_t cnt, loff_t *ppos)
  6982. {
  6983. struct trace_array *tr = filp->private_data;
  6984. char buf[64];
  6985. int r;
  6986. r = tr->buffer_percent;
  6987. r = sprintf(buf, "%d\n", r);
  6988. return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
  6989. }
  6990. static ssize_t
  6991. buffer_percent_write(struct file *filp, const char __user *ubuf,
  6992. size_t cnt, loff_t *ppos)
  6993. {
  6994. struct trace_array *tr = filp->private_data;
  6995. unsigned long val;
  6996. int ret;
  6997. ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
  6998. if (ret)
  6999. return ret;
  7000. if (val > 100)
  7001. return -EINVAL;
  7002. if (!val)
  7003. val = 1;
  7004. tr->buffer_percent = val;
  7005. (*ppos)++;
  7006. return cnt;
  7007. }
  7008. static const struct file_operations buffer_percent_fops = {
  7009. .open = tracing_open_generic_tr,
  7010. .read = buffer_percent_read,
  7011. .write = buffer_percent_write,
  7012. .release = tracing_release_generic_tr,
  7013. .llseek = default_llseek,
  7014. };
  7015. static struct dentry *trace_instance_dir;
  7016. static void
  7017. init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
  7018. static int
  7019. allocate_trace_buffer(struct trace_array *tr, struct array_buffer *buf, int size)
  7020. {
  7021. enum ring_buffer_flags rb_flags;
  7022. rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
  7023. buf->tr = tr;
  7024. buf->buffer = ring_buffer_alloc(size, rb_flags);
  7025. if (!buf->buffer)
  7026. return -ENOMEM;
  7027. buf->data = alloc_percpu(struct trace_array_cpu);
  7028. if (!buf->data) {
  7029. ring_buffer_free(buf->buffer);
  7030. buf->buffer = NULL;
  7031. return -ENOMEM;
  7032. }
  7033. /* Allocate the first page for all buffers */
  7034. set_buffer_entries(&tr->array_buffer,
  7035. ring_buffer_size(tr->array_buffer.buffer, 0));
  7036. return 0;
  7037. }
  7038. static int allocate_trace_buffers(struct trace_array *tr, int size)
  7039. {
  7040. int ret;
  7041. ret = allocate_trace_buffer(tr, &tr->array_buffer, size);
  7042. if (ret)
  7043. return ret;
  7044. #ifdef CONFIG_TRACER_MAX_TRACE
  7045. ret = allocate_trace_buffer(tr, &tr->max_buffer,
  7046. allocate_snapshot ? size : 1);
  7047. if (MEM_FAIL(ret, "Failed to allocate trace buffer\n")) {
  7048. ring_buffer_free(tr->array_buffer.buffer);
  7049. tr->array_buffer.buffer = NULL;
  7050. free_percpu(tr->array_buffer.data);
  7051. tr->array_buffer.data = NULL;
  7052. return -ENOMEM;
  7053. }
  7054. tr->allocated_snapshot = allocate_snapshot;
  7055. /*
  7056. * Only the top level trace array gets its snapshot allocated
  7057. * from the kernel command line.
  7058. */
  7059. allocate_snapshot = false;
  7060. #endif
  7061. return 0;
  7062. }
  7063. static void free_trace_buffer(struct array_buffer *buf)
  7064. {
  7065. if (buf->buffer) {
  7066. ring_buffer_free(buf->buffer);
  7067. buf->buffer = NULL;
  7068. free_percpu(buf->data);
  7069. buf->data = NULL;
  7070. }
  7071. }
  7072. static void free_trace_buffers(struct trace_array *tr)
  7073. {
  7074. if (!tr)
  7075. return;
  7076. free_trace_buffer(&tr->array_buffer);
  7077. #ifdef CONFIG_TRACER_MAX_TRACE
  7078. free_trace_buffer(&tr->max_buffer);
  7079. #endif
  7080. }
  7081. static void init_trace_flags_index(struct trace_array *tr)
  7082. {
  7083. int i;
  7084. /* Used by the trace options files */
  7085. for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
  7086. tr->trace_flags_index[i] = i;
  7087. }
  7088. static void __update_tracer_options(struct trace_array *tr)
  7089. {
  7090. struct tracer *t;
  7091. for (t = trace_types; t; t = t->next)
  7092. add_tracer_options(tr, t);
  7093. }
  7094. static void update_tracer_options(struct trace_array *tr)
  7095. {
  7096. mutex_lock(&trace_types_lock);
  7097. __update_tracer_options(tr);
  7098. mutex_unlock(&trace_types_lock);
  7099. }
  7100. /* Must have trace_types_lock held */
  7101. struct trace_array *trace_array_find(const char *instance)
  7102. {
  7103. struct trace_array *tr, *found = NULL;
  7104. list_for_each_entry(tr, &ftrace_trace_arrays, list) {
  7105. if (tr->name && strcmp(tr->name, instance) == 0) {
  7106. found = tr;
  7107. break;
  7108. }
  7109. }
  7110. return found;
  7111. }
  7112. struct trace_array *trace_array_find_get(const char *instance)
  7113. {
  7114. struct trace_array *tr;
  7115. mutex_lock(&trace_types_lock);
  7116. tr = trace_array_find(instance);
  7117. if (tr)
  7118. tr->ref++;
  7119. mutex_unlock(&trace_types_lock);
  7120. return tr;
  7121. }
  7122. static int trace_array_create_dir(struct trace_array *tr)
  7123. {
  7124. int ret;
  7125. tr->dir = tracefs_create_dir(tr->name, trace_instance_dir);
  7126. if (!tr->dir)
  7127. return -EINVAL;
  7128. ret = event_trace_add_tracer(tr->dir, tr);
  7129. if (ret) {
  7130. tracefs_remove(tr->dir);
  7131. return ret;
  7132. }
  7133. init_tracer_tracefs(tr, tr->dir);
  7134. __update_tracer_options(tr);
  7135. return ret;
  7136. }
  7137. static struct trace_array *trace_array_create(const char *name)
  7138. {
  7139. struct trace_array *tr;
  7140. int ret;
  7141. ret = -ENOMEM;
  7142. tr = kzalloc(sizeof(*tr), GFP_KERNEL);
  7143. if (!tr)
  7144. return ERR_PTR(ret);
  7145. tr->name = kstrdup(name, GFP_KERNEL);
  7146. if (!tr->name)
  7147. goto out_free_tr;
  7148. if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
  7149. goto out_free_tr;
  7150. tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS;
  7151. cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
  7152. raw_spin_lock_init(&tr->start_lock);
  7153. tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
  7154. tr->current_trace = &nop_trace;
  7155. INIT_LIST_HEAD(&tr->systems);
  7156. INIT_LIST_HEAD(&tr->events);
  7157. INIT_LIST_HEAD(&tr->hist_vars);
  7158. INIT_LIST_HEAD(&tr->err_log);
  7159. if (allocate_trace_buffers(tr, trace_buf_size) < 0)
  7160. goto out_free_tr;
  7161. if (ftrace_allocate_ftrace_ops(tr) < 0)
  7162. goto out_free_tr;
  7163. ftrace_init_trace_array(tr);
  7164. init_trace_flags_index(tr);
  7165. if (trace_instance_dir) {
  7166. ret = trace_array_create_dir(tr);
  7167. if (ret)
  7168. goto out_free_tr;
  7169. } else
  7170. __trace_early_add_events(tr);
  7171. list_add(&tr->list, &ftrace_trace_arrays);
  7172. tr->ref++;
  7173. return tr;
  7174. out_free_tr:
  7175. ftrace_free_ftrace_ops(tr);
  7176. free_trace_buffers(tr);
  7177. free_cpumask_var(tr->tracing_cpumask);
  7178. kfree(tr->name);
  7179. kfree(tr);
  7180. return ERR_PTR(ret);
  7181. }
  7182. static int instance_mkdir(const char *name)
  7183. {
  7184. struct trace_array *tr;
  7185. int ret;
  7186. mutex_lock(&event_mutex);
  7187. mutex_lock(&trace_types_lock);
  7188. ret = -EEXIST;
  7189. if (trace_array_find(name))
  7190. goto out_unlock;
  7191. tr = trace_array_create(name);
  7192. ret = PTR_ERR_OR_ZERO(tr);
  7193. out_unlock:
  7194. mutex_unlock(&trace_types_lock);
  7195. mutex_unlock(&event_mutex);
  7196. return ret;
  7197. }
  7198. /**
  7199. * trace_array_get_by_name - Create/Lookup a trace array, given its name.
  7200. * @name: The name of the trace array to be looked up/created.
  7201. *
  7202. * Returns pointer to trace array with given name.
  7203. * NULL, if it cannot be created.
  7204. *
  7205. * NOTE: This function increments the reference counter associated with the
  7206. * trace array returned. This makes sure it cannot be freed while in use.
  7207. * Use trace_array_put() once the trace array is no longer needed.
  7208. * If the trace_array is to be freed, trace_array_destroy() needs to
  7209. * be called after the trace_array_put(), or simply let user space delete
  7210. * it from the tracefs instances directory. But until the
  7211. * trace_array_put() is called, user space can not delete it.
  7212. *
  7213. */
  7214. struct trace_array *trace_array_get_by_name(const char *name)
  7215. {
  7216. struct trace_array *tr;
  7217. mutex_lock(&event_mutex);
  7218. mutex_lock(&trace_types_lock);
  7219. list_for_each_entry(tr, &ftrace_trace_arrays, list) {
  7220. if (tr->name && strcmp(tr->name, name) == 0)
  7221. goto out_unlock;
  7222. }
  7223. tr = trace_array_create(name);
  7224. if (IS_ERR(tr))
  7225. tr = NULL;
  7226. out_unlock:
  7227. if (tr)
  7228. tr->ref++;
  7229. mutex_unlock(&trace_types_lock);
  7230. mutex_unlock(&event_mutex);
  7231. return tr;
  7232. }
  7233. EXPORT_SYMBOL_GPL(trace_array_get_by_name);
  7234. static int __remove_instance(struct trace_array *tr)
  7235. {
  7236. int i;
  7237. /* Reference counter for a newly created trace array = 1. */
  7238. if (tr->ref > 1 || (tr->current_trace && tr->trace_ref))
  7239. return -EBUSY;
  7240. list_del(&tr->list);
  7241. /* Disable all the flags that were enabled coming in */
  7242. for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) {
  7243. if ((1 << i) & ZEROED_TRACE_FLAGS)
  7244. set_tracer_flag(tr, 1 << i, 0);
  7245. }
  7246. tracing_set_nop(tr);
  7247. clear_ftrace_function_probes(tr);
  7248. event_trace_del_tracer(tr);
  7249. ftrace_clear_pids(tr);
  7250. ftrace_destroy_function_files(tr);
  7251. tracefs_remove(tr->dir);
  7252. free_trace_buffers(tr);
  7253. for (i = 0; i < tr->nr_topts; i++) {
  7254. kfree(tr->topts[i].topts);
  7255. }
  7256. kfree(tr->topts);
  7257. free_cpumask_var(tr->tracing_cpumask);
  7258. kfree(tr->name);
  7259. kfree(tr);
  7260. return 0;
  7261. }
  7262. int trace_array_destroy(struct trace_array *this_tr)
  7263. {
  7264. struct trace_array *tr;
  7265. int ret;
  7266. if (!this_tr)
  7267. return -EINVAL;
  7268. mutex_lock(&event_mutex);
  7269. mutex_lock(&trace_types_lock);
  7270. ret = -ENODEV;
  7271. /* Making sure trace array exists before destroying it. */
  7272. list_for_each_entry(tr, &ftrace_trace_arrays, list) {
  7273. if (tr == this_tr) {
  7274. ret = __remove_instance(tr);
  7275. break;
  7276. }
  7277. }
  7278. mutex_unlock(&trace_types_lock);
  7279. mutex_unlock(&event_mutex);
  7280. return ret;
  7281. }
  7282. EXPORT_SYMBOL_GPL(trace_array_destroy);
  7283. static int instance_rmdir(const char *name)
  7284. {
  7285. struct trace_array *tr;
  7286. int ret;
  7287. mutex_lock(&event_mutex);
  7288. mutex_lock(&trace_types_lock);
  7289. ret = -ENODEV;
  7290. tr = trace_array_find(name);
  7291. if (tr)
  7292. ret = __remove_instance(tr);
  7293. mutex_unlock(&trace_types_lock);
  7294. mutex_unlock(&event_mutex);
  7295. return ret;
  7296. }
  7297. static __init void create_trace_instances(struct dentry *d_tracer)
  7298. {
  7299. struct trace_array *tr;
  7300. trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
  7301. instance_mkdir,
  7302. instance_rmdir);
  7303. if (MEM_FAIL(!trace_instance_dir, "Failed to create instances directory\n"))
  7304. return;
  7305. mutex_lock(&event_mutex);
  7306. mutex_lock(&trace_types_lock);
  7307. list_for_each_entry(tr, &ftrace_trace_arrays, list) {
  7308. if (!tr->name)
  7309. continue;
  7310. if (MEM_FAIL(trace_array_create_dir(tr) < 0,
  7311. "Failed to create instance directory\n"))
  7312. break;
  7313. }
  7314. mutex_unlock(&trace_types_lock);
  7315. mutex_unlock(&event_mutex);
  7316. }
  7317. static void
  7318. init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
  7319. {
  7320. struct trace_event_file *file;
  7321. int cpu;
  7322. trace_create_file("available_tracers", 0444, d_tracer,
  7323. tr, &show_traces_fops);
  7324. trace_create_file("current_tracer", 0644, d_tracer,
  7325. tr, &set_tracer_fops);
  7326. trace_create_file("tracing_cpumask", 0644, d_tracer,
  7327. tr, &tracing_cpumask_fops);
  7328. trace_create_file("trace_options", 0644, d_tracer,
  7329. tr, &tracing_iter_fops);
  7330. trace_create_file("trace", 0644, d_tracer,
  7331. tr, &tracing_fops);
  7332. trace_create_file("trace_pipe", 0444, d_tracer,
  7333. tr, &tracing_pipe_fops);
  7334. trace_create_file("buffer_size_kb", 0644, d_tracer,
  7335. tr, &tracing_entries_fops);
  7336. trace_create_file("buffer_total_size_kb", 0444, d_tracer,
  7337. tr, &tracing_total_entries_fops);
  7338. trace_create_file("free_buffer", 0200, d_tracer,
  7339. tr, &tracing_free_buffer_fops);
  7340. trace_create_file("trace_marker", 0220, d_tracer,
  7341. tr, &tracing_mark_fops);
  7342. file = __find_event_file(tr, "ftrace", "print");
  7343. if (file && file->dir)
  7344. trace_create_file("trigger", 0644, file->dir, file,
  7345. &event_trigger_fops);
  7346. tr->trace_marker_file = file;
  7347. trace_create_file("trace_marker_raw", 0220, d_tracer,
  7348. tr, &tracing_mark_raw_fops);
  7349. trace_create_file("trace_clock", 0644, d_tracer, tr,
  7350. &trace_clock_fops);
  7351. trace_create_file("tracing_on", 0644, d_tracer,
  7352. tr, &rb_simple_fops);
  7353. trace_create_file("timestamp_mode", 0444, d_tracer, tr,
  7354. &trace_time_stamp_mode_fops);
  7355. tr->buffer_percent = 50;
  7356. trace_create_file("buffer_percent", 0444, d_tracer,
  7357. tr, &buffer_percent_fops);
  7358. create_trace_options_dir(tr);
  7359. #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
  7360. trace_create_maxlat_file(tr, d_tracer);
  7361. #endif
  7362. if (ftrace_create_function_files(tr, d_tracer))
  7363. MEM_FAIL(1, "Could not allocate function filter files");
  7364. #ifdef CONFIG_TRACER_SNAPSHOT
  7365. trace_create_file("snapshot", 0644, d_tracer,
  7366. tr, &snapshot_fops);
  7367. #endif
  7368. trace_create_file("error_log", 0644, d_tracer,
  7369. tr, &tracing_err_log_fops);
  7370. for_each_tracing_cpu(cpu)
  7371. tracing_init_tracefs_percpu(tr, cpu);
  7372. ftrace_init_tracefs(tr, d_tracer);
  7373. }
  7374. #ifndef CONFIG_TRACEFS_DISABLE_AUTOMOUNT
  7375. static struct vfsmount *trace_automount(struct dentry *mntpt, void *ingore)
  7376. {
  7377. struct vfsmount *mnt;
  7378. struct file_system_type *type;
  7379. /*
  7380. * To maintain backward compatibility for tools that mount
  7381. * debugfs to get to the tracing facility, tracefs is automatically
  7382. * mounted to the debugfs/tracing directory.
  7383. */
  7384. type = get_fs_type("tracefs");
  7385. if (!type)
  7386. return NULL;
  7387. mnt = vfs_submount(mntpt, type, "tracefs", NULL);
  7388. put_filesystem(type);
  7389. if (IS_ERR(mnt))
  7390. return NULL;
  7391. mntget(mnt);
  7392. return mnt;
  7393. }
  7394. #endif
  7395. /**
  7396. * tracing_init_dentry - initialize top level trace array
  7397. *
  7398. * This is called when creating files or directories in the tracing
  7399. * directory. It is called via fs_initcall() by any of the boot up code
  7400. * and expects to return the dentry of the top level tracing directory.
  7401. */
  7402. int tracing_init_dentry(void)
  7403. {
  7404. struct trace_array *tr = &global_trace;
  7405. if (security_locked_down(LOCKDOWN_TRACEFS)) {
  7406. pr_warn("Tracing disabled due to lockdown\n");
  7407. return -EPERM;
  7408. }
  7409. /* The top level trace array uses NULL as parent */
  7410. if (tr->dir)
  7411. return 0;
  7412. if (WARN_ON(!tracefs_initialized()))
  7413. return -ENODEV;
  7414. #ifndef CONFIG_TRACEFS_DISABLE_AUTOMOUNT
  7415. /*
  7416. * As there may still be users that expect the tracing
  7417. * files to exist in debugfs/tracing, we must automount
  7418. * the tracefs file system there, so older tools still
  7419. * work with the newer kerenl.
  7420. */
  7421. tr->dir = debugfs_create_automount("tracing", NULL,
  7422. trace_automount, NULL);
  7423. #else
  7424. tr->dir = ERR_PTR(-ENODEV);
  7425. #endif
  7426. return 0;
  7427. }
  7428. extern struct trace_eval_map *__start_ftrace_eval_maps[];
  7429. extern struct trace_eval_map *__stop_ftrace_eval_maps[];
  7430. static void __init trace_eval_init(void)
  7431. {
  7432. int len;
  7433. len = __stop_ftrace_eval_maps - __start_ftrace_eval_maps;
  7434. trace_insert_eval_map(NULL, __start_ftrace_eval_maps, len);
  7435. }
  7436. #ifdef CONFIG_MODULES
  7437. static void trace_module_add_evals(struct module *mod)
  7438. {
  7439. if (!mod->num_trace_evals)
  7440. return;
  7441. /*
  7442. * Modules with bad taint do not have events created, do
  7443. * not bother with enums either.
  7444. */
  7445. if (trace_module_has_bad_taint(mod))
  7446. return;
  7447. trace_insert_eval_map(mod, mod->trace_evals, mod->num_trace_evals);
  7448. }
  7449. #ifdef CONFIG_TRACE_EVAL_MAP_FILE
  7450. static void trace_module_remove_evals(struct module *mod)
  7451. {
  7452. union trace_eval_map_item *map;
  7453. union trace_eval_map_item **last = &trace_eval_maps;
  7454. if (!mod->num_trace_evals)
  7455. return;
  7456. mutex_lock(&trace_eval_mutex);
  7457. map = trace_eval_maps;
  7458. while (map) {
  7459. if (map->head.mod == mod)
  7460. break;
  7461. map = trace_eval_jmp_to_tail(map);
  7462. last = &map->tail.next;
  7463. map = map->tail.next;
  7464. }
  7465. if (!map)
  7466. goto out;
  7467. *last = trace_eval_jmp_to_tail(map)->tail.next;
  7468. kfree(map);
  7469. out:
  7470. mutex_unlock(&trace_eval_mutex);
  7471. }
  7472. #else
  7473. static inline void trace_module_remove_evals(struct module *mod) { }
  7474. #endif /* CONFIG_TRACE_EVAL_MAP_FILE */
  7475. static int trace_module_notify(struct notifier_block *self,
  7476. unsigned long val, void *data)
  7477. {
  7478. struct module *mod = data;
  7479. switch (val) {
  7480. case MODULE_STATE_COMING:
  7481. trace_module_add_evals(mod);
  7482. break;
  7483. case MODULE_STATE_GOING:
  7484. trace_module_remove_evals(mod);
  7485. break;
  7486. }
  7487. return NOTIFY_OK;
  7488. }
  7489. static struct notifier_block trace_module_nb = {
  7490. .notifier_call = trace_module_notify,
  7491. .priority = 0,
  7492. };
  7493. #endif /* CONFIG_MODULES */
  7494. static __init int tracer_init_tracefs(void)
  7495. {
  7496. int ret;
  7497. trace_access_lock_init();
  7498. ret = tracing_init_dentry();
  7499. if (ret)
  7500. return 0;
  7501. event_trace_init();
  7502. init_tracer_tracefs(&global_trace, NULL);
  7503. ftrace_init_tracefs_toplevel(&global_trace, NULL);
  7504. trace_create_file("tracing_thresh", 0644, NULL,
  7505. &global_trace, &tracing_thresh_fops);
  7506. trace_create_file("README", 0444, NULL,
  7507. NULL, &tracing_readme_fops);
  7508. trace_create_file("saved_cmdlines", 0444, NULL,
  7509. NULL, &tracing_saved_cmdlines_fops);
  7510. trace_create_file("saved_cmdlines_size", 0644, NULL,
  7511. NULL, &tracing_saved_cmdlines_size_fops);
  7512. trace_create_file("saved_tgids", 0444, NULL,
  7513. NULL, &tracing_saved_tgids_fops);
  7514. trace_eval_init();
  7515. trace_create_eval_file(NULL);
  7516. #ifdef CONFIG_MODULES
  7517. register_module_notifier(&trace_module_nb);
  7518. #endif
  7519. #ifdef CONFIG_DYNAMIC_FTRACE
  7520. trace_create_file("dyn_ftrace_total_info", 0444, NULL,
  7521. NULL, &tracing_dyn_info_fops);
  7522. #endif
  7523. create_trace_instances(NULL);
  7524. update_tracer_options(&global_trace);
  7525. return 0;
  7526. }
  7527. static int trace_panic_handler(struct notifier_block *this,
  7528. unsigned long event, void *unused)
  7529. {
  7530. bool ftrace_check = false;
  7531. trace_android_vh_ftrace_oops_enter(&ftrace_check);
  7532. if (ftrace_check)
  7533. return NOTIFY_OK;
  7534. if (ftrace_dump_on_oops)
  7535. ftrace_dump(ftrace_dump_on_oops);
  7536. trace_android_vh_ftrace_oops_exit(&ftrace_check);
  7537. return NOTIFY_OK;
  7538. }
  7539. static struct notifier_block trace_panic_notifier = {
  7540. .notifier_call = trace_panic_handler,
  7541. .next = NULL,
  7542. .priority = 150 /* priority: INT_MAX >= x >= 0 */
  7543. };
  7544. static int trace_die_handler(struct notifier_block *self,
  7545. unsigned long val,
  7546. void *data)
  7547. {
  7548. bool ftrace_check = false;
  7549. trace_android_vh_ftrace_oops_enter(&ftrace_check);
  7550. if (ftrace_check)
  7551. return NOTIFY_OK;
  7552. switch (val) {
  7553. case DIE_OOPS:
  7554. if (ftrace_dump_on_oops)
  7555. ftrace_dump(ftrace_dump_on_oops);
  7556. break;
  7557. default:
  7558. break;
  7559. }
  7560. trace_android_vh_ftrace_oops_exit(&ftrace_check);
  7561. return NOTIFY_OK;
  7562. }
  7563. static struct notifier_block trace_die_notifier = {
  7564. .notifier_call = trace_die_handler,
  7565. .priority = 200
  7566. };
  7567. /*
  7568. * printk is set to max of 1024, we really don't need it that big.
  7569. * Nothing should be printing 1000 characters anyway.
  7570. */
  7571. #define TRACE_MAX_PRINT 1000
  7572. /*
  7573. * Define here KERN_TRACE so that we have one place to modify
  7574. * it if we decide to change what log level the ftrace dump
  7575. * should be at.
  7576. */
  7577. #define KERN_TRACE KERN_EMERG
  7578. void
  7579. trace_printk_seq(struct trace_seq *s)
  7580. {
  7581. bool dump_printk = true;
  7582. /* Probably should print a warning here. */
  7583. if (s->seq.len >= TRACE_MAX_PRINT)
  7584. s->seq.len = TRACE_MAX_PRINT;
  7585. /*
  7586. * More paranoid code. Although the buffer size is set to
  7587. * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
  7588. * an extra layer of protection.
  7589. */
  7590. if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
  7591. s->seq.len = s->seq.size - 1;
  7592. /* should be zero ended, but we are paranoid. */
  7593. s->buffer[s->seq.len] = 0;
  7594. trace_android_vh_ftrace_dump_buffer(s, &dump_printk);
  7595. if (dump_printk)
  7596. printk(KERN_TRACE "%s", s->buffer);
  7597. trace_seq_init(s);
  7598. }
  7599. void trace_init_global_iter(struct trace_iterator *iter)
  7600. {
  7601. iter->tr = &global_trace;
  7602. iter->trace = iter->tr->current_trace;
  7603. iter->cpu_file = RING_BUFFER_ALL_CPUS;
  7604. iter->array_buffer = &global_trace.array_buffer;
  7605. if (iter->trace && iter->trace->open)
  7606. iter->trace->open(iter);
  7607. /* Annotate start of buffers if we had overruns */
  7608. if (ring_buffer_overruns(iter->array_buffer->buffer))
  7609. iter->iter_flags |= TRACE_FILE_ANNOTATE;
  7610. /* Output in nanoseconds only if we are using a clock in nanoseconds. */
  7611. if (trace_clocks[iter->tr->clock_id].in_ns)
  7612. iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
  7613. }
  7614. void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
  7615. {
  7616. /* use static because iter can be a bit big for the stack */
  7617. static struct trace_iterator iter;
  7618. static atomic_t dump_running;
  7619. struct trace_array *tr = &global_trace;
  7620. unsigned int old_userobj;
  7621. unsigned long flags;
  7622. int cnt = 0, cpu;
  7623. bool ftrace_check = false;
  7624. unsigned long size;
  7625. /* Only allow one dump user at a time. */
  7626. if (atomic_inc_return(&dump_running) != 1) {
  7627. atomic_dec(&dump_running);
  7628. return;
  7629. }
  7630. /*
  7631. * Always turn off tracing when we dump.
  7632. * We don't need to show trace output of what happens
  7633. * between multiple crashes.
  7634. *
  7635. * If the user does a sysrq-z, then they can re-enable
  7636. * tracing with echo 1 > tracing_on.
  7637. */
  7638. tracing_off();
  7639. local_irq_save(flags);
  7640. printk_nmi_direct_enter();
  7641. /* Simulate the iterator */
  7642. trace_init_global_iter(&iter);
  7643. /* Can not use kmalloc for iter.temp */
  7644. iter.temp = static_temp_buf;
  7645. iter.temp_size = STATIC_TEMP_BUF_SIZE;
  7646. for_each_tracing_cpu(cpu) {
  7647. atomic_inc(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
  7648. size = ring_buffer_size(iter.array_buffer->buffer, cpu);
  7649. trace_android_vh_ftrace_size_check(size, &ftrace_check);
  7650. }
  7651. old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
  7652. /* don't look at user memory in panic mode */
  7653. tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
  7654. if (ftrace_check)
  7655. goto out_enable;
  7656. switch (oops_dump_mode) {
  7657. case DUMP_ALL:
  7658. iter.cpu_file = RING_BUFFER_ALL_CPUS;
  7659. break;
  7660. case DUMP_ORIG:
  7661. iter.cpu_file = raw_smp_processor_id();
  7662. break;
  7663. case DUMP_NONE:
  7664. goto out_enable;
  7665. default:
  7666. printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
  7667. iter.cpu_file = RING_BUFFER_ALL_CPUS;
  7668. }
  7669. printk(KERN_TRACE "Dumping ftrace buffer:\n");
  7670. /* Did function tracer already get disabled? */
  7671. if (ftrace_is_dead()) {
  7672. printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
  7673. printk("# MAY BE MISSING FUNCTION EVENTS\n");
  7674. }
  7675. /*
  7676. * We need to stop all tracing on all CPUS to read
  7677. * the next buffer. This is a bit expensive, but is
  7678. * not done often. We fill all what we can read,
  7679. * and then release the locks again.
  7680. */
  7681. while (!trace_empty(&iter)) {
  7682. ftrace_check = true;
  7683. if (!cnt)
  7684. printk(KERN_TRACE "---------------------------------\n");
  7685. cnt++;
  7686. trace_iterator_reset(&iter);
  7687. trace_android_vh_ftrace_format_check(&ftrace_check);
  7688. if (ftrace_check)
  7689. iter.iter_flags |= TRACE_FILE_LAT_FMT;
  7690. if (trace_find_next_entry_inc(&iter) != NULL) {
  7691. int ret;
  7692. ret = print_trace_line(&iter);
  7693. if (ret != TRACE_TYPE_NO_CONSUME)
  7694. trace_consume(&iter);
  7695. }
  7696. touch_nmi_watchdog();
  7697. trace_printk_seq(&iter.seq);
  7698. }
  7699. if (!cnt)
  7700. printk(KERN_TRACE " (ftrace buffer empty)\n");
  7701. else
  7702. printk(KERN_TRACE "---------------------------------\n");
  7703. out_enable:
  7704. tr->trace_flags |= old_userobj;
  7705. for_each_tracing_cpu(cpu) {
  7706. atomic_dec(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
  7707. }
  7708. atomic_dec(&dump_running);
  7709. printk_nmi_direct_exit();
  7710. local_irq_restore(flags);
  7711. }
  7712. EXPORT_SYMBOL_GPL(ftrace_dump);
  7713. int trace_run_command(const char *buf, int (*createfn)(int, char **))
  7714. {
  7715. char **argv;
  7716. int argc, ret;
  7717. argc = 0;
  7718. ret = 0;
  7719. argv = argv_split(GFP_KERNEL, buf, &argc);
  7720. if (!argv)
  7721. return -ENOMEM;
  7722. if (argc)
  7723. ret = createfn(argc, argv);
  7724. argv_free(argv);
  7725. return ret;
  7726. }
  7727. #define WRITE_BUFSIZE 4096
  7728. ssize_t trace_parse_run_command(struct file *file, const char __user *buffer,
  7729. size_t count, loff_t *ppos,
  7730. int (*createfn)(int, char **))
  7731. {
  7732. char *kbuf, *buf, *tmp;
  7733. int ret = 0;
  7734. size_t done = 0;
  7735. size_t size;
  7736. kbuf = kmalloc(WRITE_BUFSIZE, GFP_KERNEL);
  7737. if (!kbuf)
  7738. return -ENOMEM;
  7739. while (done < count) {
  7740. size = count - done;
  7741. if (size >= WRITE_BUFSIZE)
  7742. size = WRITE_BUFSIZE - 1;
  7743. if (copy_from_user(kbuf, buffer + done, size)) {
  7744. ret = -EFAULT;
  7745. goto out;
  7746. }
  7747. kbuf[size] = '\0';
  7748. buf = kbuf;
  7749. do {
  7750. tmp = strchr(buf, '\n');
  7751. if (tmp) {
  7752. *tmp = '\0';
  7753. size = tmp - buf + 1;
  7754. } else {
  7755. size = strlen(buf);
  7756. if (done + size < count) {
  7757. if (buf != kbuf)
  7758. break;
  7759. /* This can accept WRITE_BUFSIZE - 2 ('\n' + '\0') */
  7760. pr_warn("Line length is too long: Should be less than %d\n",
  7761. WRITE_BUFSIZE - 2);
  7762. ret = -EINVAL;
  7763. goto out;
  7764. }
  7765. }
  7766. done += size;
  7767. /* Remove comments */
  7768. tmp = strchr(buf, '#');
  7769. if (tmp)
  7770. *tmp = '\0';
  7771. ret = trace_run_command(buf, createfn);
  7772. if (ret)
  7773. goto out;
  7774. buf += size;
  7775. } while (done < count);
  7776. }
  7777. ret = done;
  7778. out:
  7779. kfree(kbuf);
  7780. return ret;
  7781. }
  7782. __init static int tracer_alloc_buffers(void)
  7783. {
  7784. int ring_buf_size;
  7785. int ret = -ENOMEM;
  7786. if (security_locked_down(LOCKDOWN_TRACEFS)) {
  7787. pr_warn("Tracing disabled due to lockdown\n");
  7788. return -EPERM;
  7789. }
  7790. /*
  7791. * Make sure we don't accidentally add more trace options
  7792. * than we have bits for.
  7793. */
  7794. BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
  7795. if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
  7796. goto out;
  7797. if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
  7798. goto out_free_buffer_mask;
  7799. /* Only allocate trace_printk buffers if a trace_printk exists */
  7800. if (&__stop___trace_bprintk_fmt != &__start___trace_bprintk_fmt)
  7801. /* Must be called before global_trace.buffer is allocated */
  7802. trace_printk_init_buffers();
  7803. /* To save memory, keep the ring buffer size to its minimum */
  7804. if (ring_buffer_expanded)
  7805. ring_buf_size = trace_buf_size;
  7806. else
  7807. ring_buf_size = 1;
  7808. cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
  7809. cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
  7810. raw_spin_lock_init(&global_trace.start_lock);
  7811. /*
  7812. * The prepare callbacks allocates some memory for the ring buffer. We
  7813. * don't free the buffer if the CPU goes down. If we were to free
  7814. * the buffer, then the user would lose any trace that was in the
  7815. * buffer. The memory will be removed once the "instance" is removed.
  7816. */
  7817. ret = cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE,
  7818. "trace/RB:preapre", trace_rb_cpu_prepare,
  7819. NULL);
  7820. if (ret < 0)
  7821. goto out_free_cpumask;
  7822. /* Used for event triggers */
  7823. ret = -ENOMEM;
  7824. temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
  7825. if (!temp_buffer)
  7826. goto out_rm_hp_state;
  7827. if (trace_create_savedcmd() < 0)
  7828. goto out_free_temp_buffer;
  7829. /* TODO: make the number of buffers hot pluggable with CPUS */
  7830. if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
  7831. MEM_FAIL(1, "tracer: failed to allocate ring buffer!\n");
  7832. goto out_free_savedcmd;
  7833. }
  7834. if (global_trace.buffer_disabled)
  7835. tracing_off();
  7836. if (trace_boot_clock) {
  7837. ret = tracing_set_clock(&global_trace, trace_boot_clock);
  7838. if (ret < 0)
  7839. pr_warn("Trace clock %s not defined, going back to default\n",
  7840. trace_boot_clock);
  7841. }
  7842. /*
  7843. * register_tracer() might reference current_trace, so it
  7844. * needs to be set before we register anything. This is
  7845. * just a bootstrap of current_trace anyway.
  7846. */
  7847. global_trace.current_trace = &nop_trace;
  7848. global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
  7849. ftrace_init_global_array_ops(&global_trace);
  7850. init_trace_flags_index(&global_trace);
  7851. register_tracer(&nop_trace);
  7852. /* Function tracing may start here (via kernel command line) */
  7853. init_function_trace();
  7854. /* All seems OK, enable tracing */
  7855. tracing_disabled = 0;
  7856. atomic_notifier_chain_register(&panic_notifier_list,
  7857. &trace_panic_notifier);
  7858. register_die_notifier(&trace_die_notifier);
  7859. global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
  7860. INIT_LIST_HEAD(&global_trace.systems);
  7861. INIT_LIST_HEAD(&global_trace.events);
  7862. INIT_LIST_HEAD(&global_trace.hist_vars);
  7863. INIT_LIST_HEAD(&global_trace.err_log);
  7864. list_add(&global_trace.list, &ftrace_trace_arrays);
  7865. apply_trace_boot_options();
  7866. register_snapshot_cmd();
  7867. return 0;
  7868. out_free_savedcmd:
  7869. free_saved_cmdlines_buffer(savedcmd);
  7870. out_free_temp_buffer:
  7871. ring_buffer_free(temp_buffer);
  7872. out_rm_hp_state:
  7873. cpuhp_remove_multi_state(CPUHP_TRACE_RB_PREPARE);
  7874. out_free_cpumask:
  7875. free_cpumask_var(global_trace.tracing_cpumask);
  7876. out_free_buffer_mask:
  7877. free_cpumask_var(tracing_buffer_mask);
  7878. out:
  7879. return ret;
  7880. }
  7881. void __init early_trace_init(void)
  7882. {
  7883. if (tracepoint_printk) {
  7884. tracepoint_print_iter =
  7885. kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
  7886. if (MEM_FAIL(!tracepoint_print_iter,
  7887. "Failed to allocate trace iterator\n"))
  7888. tracepoint_printk = 0;
  7889. else
  7890. static_key_enable(&tracepoint_printk_key.key);
  7891. }
  7892. tracer_alloc_buffers();
  7893. }
  7894. void __init trace_init(void)
  7895. {
  7896. trace_event_init();
  7897. }
  7898. __init static int clear_boot_tracer(void)
  7899. {
  7900. /*
  7901. * The default tracer at boot buffer is an init section.
  7902. * This function is called in lateinit. If we did not
  7903. * find the boot tracer, then clear it out, to prevent
  7904. * later registration from accessing the buffer that is
  7905. * about to be freed.
  7906. */
  7907. if (!default_bootup_tracer)
  7908. return 0;
  7909. printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
  7910. default_bootup_tracer);
  7911. default_bootup_tracer = NULL;
  7912. return 0;
  7913. }
  7914. fs_initcall(tracer_init_tracefs);
  7915. late_initcall_sync(clear_boot_tracer);
  7916. #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
  7917. __init static int tracing_set_default_clock(void)
  7918. {
  7919. /* sched_clock_stable() is determined in late_initcall */
  7920. if (!trace_boot_clock && !sched_clock_stable()) {
  7921. if (security_locked_down(LOCKDOWN_TRACEFS)) {
  7922. pr_warn("Can not set tracing clock due to lockdown\n");
  7923. return -EPERM;
  7924. }
  7925. printk(KERN_WARNING
  7926. "Unstable clock detected, switching default tracing clock to \"global\"\n"
  7927. "If you want to keep using the local clock, then add:\n"
  7928. " \"trace_clock=local\"\n"
  7929. "on the kernel command line\n");
  7930. tracing_set_clock(&global_trace, "global");
  7931. }
  7932. return 0;
  7933. }
  7934. late_initcall_sync(tracing_set_default_clock);
  7935. #endif