ftrace.c 180 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106610761086109611061116112611361146115611661176118611961206121612261236124612561266127612861296130613161326133613461356136613761386139614061416142614361446145614661476148614961506151615261536154615561566157615861596160616161626163616461656166616761686169617061716172617361746175617661776178617961806181618261836184618561866187618861896190619161926193619461956196619761986199620062016202620362046205620662076208620962106211621262136214621562166217621862196220622162226223622462256226622762286229623062316232623362346235623662376238623962406241624262436244624562466247624862496250625162526253625462556256625762586259626062616262626362646265626662676268626962706271627262736274627562766277627862796280628162826283628462856286628762886289629062916292629362946295629662976298629963006301630263036304630563066307630863096310631163126313631463156316631763186319632063216322632363246325632663276328632963306331633263336334633563366337633863396340634163426343634463456346634763486349635063516352635363546355635663576358635963606361636263636364636563666367636863696370637163726373637463756376637763786379638063816382638363846385638663876388638963906391639263936394639563966397639863996400640164026403640464056406640764086409641064116412641364146415641664176418641964206421642264236424642564266427642864296430643164326433643464356436643764386439644064416442644364446445644664476448644964506451645264536454645564566457645864596460646164626463646464656466646764686469647064716472647364746475647664776478647964806481648264836484648564866487648864896490649164926493649464956496649764986499650065016502650365046505650665076508650965106511651265136514651565166517651865196520652165226523652465256526652765286529653065316532653365346535653665376538653965406541654265436544654565466547654865496550655165526553655465556556655765586559656065616562656365646565656665676568656965706571657265736574657565766577657865796580658165826583658465856586658765886589659065916592659365946595659665976598659966006601660266036604660566066607660866096610661166126613661466156616661766186619662066216622662366246625662666276628662966306631663266336634663566366637663866396640664166426643664466456646664766486649665066516652665366546655665666576658665966606661666266636664666566666667666866696670667166726673667466756676667766786679668066816682668366846685668666876688668966906691669266936694669566966697669866996700670167026703670467056706670767086709671067116712671367146715671667176718671967206721672267236724672567266727672867296730673167326733673467356736673767386739674067416742674367446745674667476748674967506751675267536754675567566757675867596760676167626763676467656766676767686769677067716772677367746775677667776778677967806781678267836784678567866787678867896790679167926793679467956796679767986799680068016802680368046805680668076808680968106811681268136814681568166817681868196820682168226823682468256826682768286829683068316832683368346835683668376838683968406841684268436844684568466847684868496850685168526853685468556856685768586859686068616862686368646865686668676868686968706871687268736874687568766877687868796880688168826883688468856886688768886889689068916892689368946895689668976898689969006901690269036904690569066907690869096910691169126913691469156916691769186919692069216922692369246925692669276928692969306931693269336934693569366937693869396940694169426943694469456946694769486949695069516952695369546955695669576958695969606961696269636964696569666967696869696970697169726973697469756976697769786979698069816982698369846985698669876988698969906991699269936994699569966997699869997000700170027003700470057006700770087009701070117012701370147015701670177018701970207021702270237024702570267027702870297030703170327033703470357036703770387039704070417042704370447045704670477048704970507051705270537054705570567057705870597060706170627063706470657066706770687069707070717072707370747075707670777078707970807081708270837084708570867087708870897090709170927093709470957096709770987099710071017102710371047105710671077108710971107111711271137114711571167117711871197120712171227123712471257126712771287129713071317132713371347135713671377138713971407141714271437144714571467147714871497150715171527153715471557156715771587159716071617162716371647165716671677168716971707171717271737174717571767177717871797180718171827183718471857186718771887189719071917192719371947195719671977198719972007201720272037204720572067207720872097210721172127213721472157216721772187219722072217222722372247225722672277228722972307231723272337234723572367237723872397240724172427243724472457246724772487249725072517252725372547255725672577258725972607261726272637264726572667267726872697270727172727273727472757276727772787279728072817282728372847285728672877288728972907291729272937294729572967297729872997300730173027303730473057306730773087309731073117312731373147315731673177318731973207321732273237324732573267327732873297330733173327333733473357336733773387339734073417342734373447345734673477348734973507351735273537354735573567357735873597360736173627363736473657366736773687369737073717372737373747375737673777378737973807381738273837384738573867387738873897390739173927393739473957396739773987399740074017402740374047405740674077408740974107411741274137414741574167417741874197420742174227423742474257426742774287429743074317432743374347435743674377438743974407441744274437444744574467447744874497450745174527453745474557456745774587459746074617462746374647465746674677468746974707471747274737474747574767477747874797480748174827483748474857486748774887489749074917492749374947495749674977498749975007501750275037504750575067507750875097510751175127513751475157516751775187519752075217522752375247525752675277528752975307531753275337534753575367537753875397540754175427543754475457546754775487549755075517552755375547555755675577558755975607561756275637564756575667567756875697570757175727573757475757576757775787579758075817582758375847585758675877588758975907591759275937594759575967597759875997600760176027603760476057606760776087609761076117612761376147615761676177618761976207621762276237624762576267627762876297630763176327633763476357636763776387639764076417642
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Infrastructure for profiling code inserted by 'gcc -pg'.
  4. *
  5. * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
  6. * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
  7. *
  8. * Originally ported from the -rt patch by:
  9. * Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
  10. *
  11. * Based on code in the latency_tracer, that is:
  12. *
  13. * Copyright (C) 2004-2006 Ingo Molnar
  14. * Copyright (C) 2004 Nadia Yvette Chambers
  15. */
  16. #include <linux/stop_machine.h>
  17. #include <linux/clocksource.h>
  18. #include <linux/sched/task.h>
  19. #include <linux/kallsyms.h>
  20. #include <linux/security.h>
  21. #include <linux/seq_file.h>
  22. #include <linux/tracefs.h>
  23. #include <linux/hardirq.h>
  24. #include <linux/kthread.h>
  25. #include <linux/uaccess.h>
  26. #include <linux/bsearch.h>
  27. #include <linux/module.h>
  28. #include <linux/ftrace.h>
  29. #include <linux/sysctl.h>
  30. #include <linux/slab.h>
  31. #include <linux/ctype.h>
  32. #include <linux/sort.h>
  33. #include <linux/list.h>
  34. #include <linux/hash.h>
  35. #include <linux/rcupdate.h>
  36. #include <linux/kprobes.h>
  37. #include <trace/events/sched.h>
  38. #include <asm/sections.h>
  39. #include <asm/setup.h>
  40. #include "ftrace_internal.h"
  41. #include "trace_output.h"
  42. #include "trace_stat.h"
  43. #define FTRACE_WARN_ON(cond) \
  44. ({ \
  45. int ___r = cond; \
  46. if (WARN_ON(___r)) \
  47. ftrace_kill(); \
  48. ___r; \
  49. })
  50. #define FTRACE_WARN_ON_ONCE(cond) \
  51. ({ \
  52. int ___r = cond; \
  53. if (WARN_ON_ONCE(___r)) \
  54. ftrace_kill(); \
  55. ___r; \
  56. })
  57. /* hash bits for specific function selection */
  58. #define FTRACE_HASH_DEFAULT_BITS 10
  59. #define FTRACE_HASH_MAX_BITS 12
  60. #ifdef CONFIG_DYNAMIC_FTRACE
  61. #define INIT_OPS_HASH(opsname) \
  62. .func_hash = &opsname.local_hash, \
  63. .local_hash.regex_lock = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
  64. #else
  65. #define INIT_OPS_HASH(opsname)
  66. #endif
  67. enum {
  68. FTRACE_MODIFY_ENABLE_FL = (1 << 0),
  69. FTRACE_MODIFY_MAY_SLEEP_FL = (1 << 1),
  70. };
  71. struct ftrace_ops ftrace_list_end __read_mostly = {
  72. .func = ftrace_stub,
  73. .flags = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_STUB,
  74. INIT_OPS_HASH(ftrace_list_end)
  75. };
  76. /* ftrace_enabled is a method to turn ftrace on or off */
  77. int ftrace_enabled __read_mostly;
  78. static int last_ftrace_enabled;
  79. /* Current function tracing op */
  80. struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end;
  81. /* What to set function_trace_op to */
  82. static struct ftrace_ops *set_function_trace_op;
  83. static bool ftrace_pids_enabled(struct ftrace_ops *ops)
  84. {
  85. struct trace_array *tr;
  86. if (!(ops->flags & FTRACE_OPS_FL_PID) || !ops->private)
  87. return false;
  88. tr = ops->private;
  89. return tr->function_pids != NULL || tr->function_no_pids != NULL;
  90. }
  91. static void ftrace_update_trampoline(struct ftrace_ops *ops);
  92. /*
  93. * ftrace_disabled is set when an anomaly is discovered.
  94. * ftrace_disabled is much stronger than ftrace_enabled.
  95. */
  96. static int ftrace_disabled __read_mostly;
  97. DEFINE_MUTEX(ftrace_lock);
  98. struct ftrace_ops __rcu *ftrace_ops_list __read_mostly = &ftrace_list_end;
  99. ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
  100. struct ftrace_ops global_ops;
  101. #if ARCH_SUPPORTS_FTRACE_OPS
  102. static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
  103. struct ftrace_ops *op, struct pt_regs *regs);
  104. #else
  105. /* See comment below, where ftrace_ops_list_func is defined */
  106. static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip);
  107. #define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops)
  108. #endif
  109. static inline void ftrace_ops_init(struct ftrace_ops *ops)
  110. {
  111. #ifdef CONFIG_DYNAMIC_FTRACE
  112. if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) {
  113. mutex_init(&ops->local_hash.regex_lock);
  114. ops->func_hash = &ops->local_hash;
  115. ops->flags |= FTRACE_OPS_FL_INITIALIZED;
  116. }
  117. #endif
  118. }
  119. static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip,
  120. struct ftrace_ops *op, struct pt_regs *regs)
  121. {
  122. struct trace_array *tr = op->private;
  123. int pid;
  124. if (tr) {
  125. pid = this_cpu_read(tr->array_buffer.data->ftrace_ignore_pid);
  126. if (pid == FTRACE_PID_IGNORE)
  127. return;
  128. if (pid != FTRACE_PID_TRACE &&
  129. pid != current->pid)
  130. return;
  131. }
  132. op->saved_func(ip, parent_ip, op, regs);
  133. }
  134. static void ftrace_sync_ipi(void *data)
  135. {
  136. /* Probably not needed, but do it anyway */
  137. smp_rmb();
  138. }
  139. static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops)
  140. {
  141. /*
  142. * If this is a dynamic, RCU, or per CPU ops, or we force list func,
  143. * then it needs to call the list anyway.
  144. */
  145. if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_RCU) ||
  146. FTRACE_FORCE_LIST_FUNC)
  147. return ftrace_ops_list_func;
  148. return ftrace_ops_get_func(ops);
  149. }
  150. static void update_ftrace_function(void)
  151. {
  152. ftrace_func_t func;
  153. /*
  154. * Prepare the ftrace_ops that the arch callback will use.
  155. * If there's only one ftrace_ops registered, the ftrace_ops_list
  156. * will point to the ops we want.
  157. */
  158. set_function_trace_op = rcu_dereference_protected(ftrace_ops_list,
  159. lockdep_is_held(&ftrace_lock));
  160. /* If there's no ftrace_ops registered, just call the stub function */
  161. if (set_function_trace_op == &ftrace_list_end) {
  162. func = ftrace_stub;
  163. /*
  164. * If we are at the end of the list and this ops is
  165. * recursion safe and not dynamic and the arch supports passing ops,
  166. * then have the mcount trampoline call the function directly.
  167. */
  168. } else if (rcu_dereference_protected(ftrace_ops_list->next,
  169. lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {
  170. func = ftrace_ops_get_list_func(ftrace_ops_list);
  171. } else {
  172. /* Just use the default ftrace_ops */
  173. set_function_trace_op = &ftrace_list_end;
  174. func = ftrace_ops_list_func;
  175. }
  176. update_function_graph_func();
  177. /* If there's no change, then do nothing more here */
  178. if (ftrace_trace_function == func)
  179. return;
  180. /*
  181. * If we are using the list function, it doesn't care
  182. * about the function_trace_ops.
  183. */
  184. if (func == ftrace_ops_list_func) {
  185. ftrace_trace_function = func;
  186. /*
  187. * Don't even bother setting function_trace_ops,
  188. * it would be racy to do so anyway.
  189. */
  190. return;
  191. }
  192. #ifndef CONFIG_DYNAMIC_FTRACE
  193. /*
  194. * For static tracing, we need to be a bit more careful.
  195. * The function change takes affect immediately. Thus,
  196. * we need to coordinate the setting of the function_trace_ops
  197. * with the setting of the ftrace_trace_function.
  198. *
  199. * Set the function to the list ops, which will call the
  200. * function we want, albeit indirectly, but it handles the
  201. * ftrace_ops and doesn't depend on function_trace_op.
  202. */
  203. ftrace_trace_function = ftrace_ops_list_func;
  204. /*
  205. * Make sure all CPUs see this. Yes this is slow, but static
  206. * tracing is slow and nasty to have enabled.
  207. */
  208. synchronize_rcu_tasks_rude();
  209. /* Now all cpus are using the list ops. */
  210. function_trace_op = set_function_trace_op;
  211. /* Make sure the function_trace_op is visible on all CPUs */
  212. smp_wmb();
  213. /* Nasty way to force a rmb on all cpus */
  214. smp_call_function(ftrace_sync_ipi, NULL, 1);
  215. /* OK, we are all set to update the ftrace_trace_function now! */
  216. #endif /* !CONFIG_DYNAMIC_FTRACE */
  217. ftrace_trace_function = func;
  218. }
  219. static void add_ftrace_ops(struct ftrace_ops __rcu **list,
  220. struct ftrace_ops *ops)
  221. {
  222. rcu_assign_pointer(ops->next, *list);
  223. /*
  224. * We are entering ops into the list but another
  225. * CPU might be walking that list. We need to make sure
  226. * the ops->next pointer is valid before another CPU sees
  227. * the ops pointer included into the list.
  228. */
  229. rcu_assign_pointer(*list, ops);
  230. }
  231. static int remove_ftrace_ops(struct ftrace_ops __rcu **list,
  232. struct ftrace_ops *ops)
  233. {
  234. struct ftrace_ops **p;
  235. /*
  236. * If we are removing the last function, then simply point
  237. * to the ftrace_stub.
  238. */
  239. if (rcu_dereference_protected(*list,
  240. lockdep_is_held(&ftrace_lock)) == ops &&
  241. rcu_dereference_protected(ops->next,
  242. lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {
  243. *list = &ftrace_list_end;
  244. return 0;
  245. }
  246. for (p = list; *p != &ftrace_list_end; p = &(*p)->next)
  247. if (*p == ops)
  248. break;
  249. if (*p != ops)
  250. return -1;
  251. *p = (*p)->next;
  252. return 0;
  253. }
  254. static void ftrace_update_trampoline(struct ftrace_ops *ops);
  255. int __register_ftrace_function(struct ftrace_ops *ops)
  256. {
  257. if (ops->flags & FTRACE_OPS_FL_DELETED)
  258. return -EINVAL;
  259. if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
  260. return -EBUSY;
  261. #ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS
  262. /*
  263. * If the ftrace_ops specifies SAVE_REGS, then it only can be used
  264. * if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set.
  265. * Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant.
  266. */
  267. if (ops->flags & FTRACE_OPS_FL_SAVE_REGS &&
  268. !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED))
  269. return -EINVAL;
  270. if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)
  271. ops->flags |= FTRACE_OPS_FL_SAVE_REGS;
  272. #endif
  273. if (!ftrace_enabled && (ops->flags & FTRACE_OPS_FL_PERMANENT))
  274. return -EBUSY;
  275. if (!core_kernel_data((unsigned long)ops))
  276. ops->flags |= FTRACE_OPS_FL_DYNAMIC;
  277. add_ftrace_ops(&ftrace_ops_list, ops);
  278. /* Always save the function, and reset at unregistering */
  279. ops->saved_func = ops->func;
  280. if (ftrace_pids_enabled(ops))
  281. ops->func = ftrace_pid_func;
  282. ftrace_update_trampoline(ops);
  283. if (ftrace_enabled)
  284. update_ftrace_function();
  285. return 0;
  286. }
  287. int __unregister_ftrace_function(struct ftrace_ops *ops)
  288. {
  289. int ret;
  290. if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))
  291. return -EBUSY;
  292. ret = remove_ftrace_ops(&ftrace_ops_list, ops);
  293. if (ret < 0)
  294. return ret;
  295. if (ftrace_enabled)
  296. update_ftrace_function();
  297. ops->func = ops->saved_func;
  298. return 0;
  299. }
  300. static void ftrace_update_pid_func(void)
  301. {
  302. struct ftrace_ops *op;
  303. /* Only do something if we are tracing something */
  304. if (ftrace_trace_function == ftrace_stub)
  305. return;
  306. do_for_each_ftrace_op(op, ftrace_ops_list) {
  307. if (op->flags & FTRACE_OPS_FL_PID) {
  308. op->func = ftrace_pids_enabled(op) ?
  309. ftrace_pid_func : op->saved_func;
  310. ftrace_update_trampoline(op);
  311. }
  312. } while_for_each_ftrace_op(op);
  313. update_ftrace_function();
  314. }
  315. #ifdef CONFIG_FUNCTION_PROFILER
  316. struct ftrace_profile {
  317. struct hlist_node node;
  318. unsigned long ip;
  319. unsigned long counter;
  320. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  321. unsigned long long time;
  322. unsigned long long time_squared;
  323. #endif
  324. };
  325. struct ftrace_profile_page {
  326. struct ftrace_profile_page *next;
  327. unsigned long index;
  328. struct ftrace_profile records[];
  329. };
  330. struct ftrace_profile_stat {
  331. atomic_t disabled;
  332. struct hlist_head *hash;
  333. struct ftrace_profile_page *pages;
  334. struct ftrace_profile_page *start;
  335. struct tracer_stat stat;
  336. };
  337. #define PROFILE_RECORDS_SIZE \
  338. (PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
  339. #define PROFILES_PER_PAGE \
  340. (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
  341. static int ftrace_profile_enabled __read_mostly;
  342. /* ftrace_profile_lock - synchronize the enable and disable of the profiler */
  343. static DEFINE_MUTEX(ftrace_profile_lock);
  344. static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
  345. #define FTRACE_PROFILE_HASH_BITS 10
  346. #define FTRACE_PROFILE_HASH_SIZE (1 << FTRACE_PROFILE_HASH_BITS)
  347. static void *
  348. function_stat_next(void *v, int idx)
  349. {
  350. struct ftrace_profile *rec = v;
  351. struct ftrace_profile_page *pg;
  352. pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
  353. again:
  354. if (idx != 0)
  355. rec++;
  356. if ((void *)rec >= (void *)&pg->records[pg->index]) {
  357. pg = pg->next;
  358. if (!pg)
  359. return NULL;
  360. rec = &pg->records[0];
  361. if (!rec->counter)
  362. goto again;
  363. }
  364. return rec;
  365. }
  366. static void *function_stat_start(struct tracer_stat *trace)
  367. {
  368. struct ftrace_profile_stat *stat =
  369. container_of(trace, struct ftrace_profile_stat, stat);
  370. if (!stat || !stat->start)
  371. return NULL;
  372. return function_stat_next(&stat->start->records[0], 0);
  373. }
  374. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  375. /* function graph compares on total time */
  376. static int function_stat_cmp(const void *p1, const void *p2)
  377. {
  378. const struct ftrace_profile *a = p1;
  379. const struct ftrace_profile *b = p2;
  380. if (a->time < b->time)
  381. return -1;
  382. if (a->time > b->time)
  383. return 1;
  384. else
  385. return 0;
  386. }
  387. #else
  388. /* not function graph compares against hits */
  389. static int function_stat_cmp(const void *p1, const void *p2)
  390. {
  391. const struct ftrace_profile *a = p1;
  392. const struct ftrace_profile *b = p2;
  393. if (a->counter < b->counter)
  394. return -1;
  395. if (a->counter > b->counter)
  396. return 1;
  397. else
  398. return 0;
  399. }
  400. #endif
  401. static int function_stat_headers(struct seq_file *m)
  402. {
  403. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  404. seq_puts(m, " Function "
  405. "Hit Time Avg s^2\n"
  406. " -------- "
  407. "--- ---- --- ---\n");
  408. #else
  409. seq_puts(m, " Function Hit\n"
  410. " -------- ---\n");
  411. #endif
  412. return 0;
  413. }
  414. static int function_stat_show(struct seq_file *m, void *v)
  415. {
  416. struct ftrace_profile *rec = v;
  417. char str[KSYM_SYMBOL_LEN];
  418. int ret = 0;
  419. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  420. static struct trace_seq s;
  421. unsigned long long avg;
  422. unsigned long long stddev;
  423. #endif
  424. mutex_lock(&ftrace_profile_lock);
  425. /* we raced with function_profile_reset() */
  426. if (unlikely(rec->counter == 0)) {
  427. ret = -EBUSY;
  428. goto out;
  429. }
  430. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  431. avg = div64_ul(rec->time, rec->counter);
  432. if (tracing_thresh && (avg < tracing_thresh))
  433. goto out;
  434. #endif
  435. kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
  436. seq_printf(m, " %-30.30s %10lu", str, rec->counter);
  437. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  438. seq_puts(m, " ");
  439. /* Sample standard deviation (s^2) */
  440. if (rec->counter <= 1)
  441. stddev = 0;
  442. else {
  443. /*
  444. * Apply Welford's method:
  445. * s^2 = 1 / (n * (n-1)) * (n * \Sum (x_i)^2 - (\Sum x_i)^2)
  446. */
  447. stddev = rec->counter * rec->time_squared -
  448. rec->time * rec->time;
  449. /*
  450. * Divide only 1000 for ns^2 -> us^2 conversion.
  451. * trace_print_graph_duration will divide 1000 again.
  452. */
  453. stddev = div64_ul(stddev,
  454. rec->counter * (rec->counter - 1) * 1000);
  455. }
  456. trace_seq_init(&s);
  457. trace_print_graph_duration(rec->time, &s);
  458. trace_seq_puts(&s, " ");
  459. trace_print_graph_duration(avg, &s);
  460. trace_seq_puts(&s, " ");
  461. trace_print_graph_duration(stddev, &s);
  462. trace_print_seq(m, &s);
  463. #endif
  464. seq_putc(m, '\n');
  465. out:
  466. mutex_unlock(&ftrace_profile_lock);
  467. return ret;
  468. }
  469. static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
  470. {
  471. struct ftrace_profile_page *pg;
  472. pg = stat->pages = stat->start;
  473. while (pg) {
  474. memset(pg->records, 0, PROFILE_RECORDS_SIZE);
  475. pg->index = 0;
  476. pg = pg->next;
  477. }
  478. memset(stat->hash, 0,
  479. FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
  480. }
  481. int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
  482. {
  483. struct ftrace_profile_page *pg;
  484. int functions;
  485. int pages;
  486. int i;
  487. /* If we already allocated, do nothing */
  488. if (stat->pages)
  489. return 0;
  490. stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
  491. if (!stat->pages)
  492. return -ENOMEM;
  493. #ifdef CONFIG_DYNAMIC_FTRACE
  494. functions = ftrace_update_tot_cnt;
  495. #else
  496. /*
  497. * We do not know the number of functions that exist because
  498. * dynamic tracing is what counts them. With past experience
  499. * we have around 20K functions. That should be more than enough.
  500. * It is highly unlikely we will execute every function in
  501. * the kernel.
  502. */
  503. functions = 20000;
  504. #endif
  505. pg = stat->start = stat->pages;
  506. pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
  507. for (i = 1; i < pages; i++) {
  508. pg->next = (void *)get_zeroed_page(GFP_KERNEL);
  509. if (!pg->next)
  510. goto out_free;
  511. pg = pg->next;
  512. }
  513. return 0;
  514. out_free:
  515. pg = stat->start;
  516. while (pg) {
  517. unsigned long tmp = (unsigned long)pg;
  518. pg = pg->next;
  519. free_page(tmp);
  520. }
  521. stat->pages = NULL;
  522. stat->start = NULL;
  523. return -ENOMEM;
  524. }
  525. static int ftrace_profile_init_cpu(int cpu)
  526. {
  527. struct ftrace_profile_stat *stat;
  528. int size;
  529. stat = &per_cpu(ftrace_profile_stats, cpu);
  530. if (stat->hash) {
  531. /* If the profile is already created, simply reset it */
  532. ftrace_profile_reset(stat);
  533. return 0;
  534. }
  535. /*
  536. * We are profiling all functions, but usually only a few thousand
  537. * functions are hit. We'll make a hash of 1024 items.
  538. */
  539. size = FTRACE_PROFILE_HASH_SIZE;
  540. stat->hash = kcalloc(size, sizeof(struct hlist_head), GFP_KERNEL);
  541. if (!stat->hash)
  542. return -ENOMEM;
  543. /* Preallocate the function profiling pages */
  544. if (ftrace_profile_pages_init(stat) < 0) {
  545. kfree(stat->hash);
  546. stat->hash = NULL;
  547. return -ENOMEM;
  548. }
  549. return 0;
  550. }
  551. static int ftrace_profile_init(void)
  552. {
  553. int cpu;
  554. int ret = 0;
  555. for_each_possible_cpu(cpu) {
  556. ret = ftrace_profile_init_cpu(cpu);
  557. if (ret)
  558. break;
  559. }
  560. return ret;
  561. }
  562. /* interrupts must be disabled */
  563. static struct ftrace_profile *
  564. ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
  565. {
  566. struct ftrace_profile *rec;
  567. struct hlist_head *hhd;
  568. unsigned long key;
  569. key = hash_long(ip, FTRACE_PROFILE_HASH_BITS);
  570. hhd = &stat->hash[key];
  571. if (hlist_empty(hhd))
  572. return NULL;
  573. hlist_for_each_entry_rcu_notrace(rec, hhd, node) {
  574. if (rec->ip == ip)
  575. return rec;
  576. }
  577. return NULL;
  578. }
  579. static void ftrace_add_profile(struct ftrace_profile_stat *stat,
  580. struct ftrace_profile *rec)
  581. {
  582. unsigned long key;
  583. key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS);
  584. hlist_add_head_rcu(&rec->node, &stat->hash[key]);
  585. }
  586. /*
  587. * The memory is already allocated, this simply finds a new record to use.
  588. */
  589. static struct ftrace_profile *
  590. ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
  591. {
  592. struct ftrace_profile *rec = NULL;
  593. /* prevent recursion (from NMIs) */
  594. if (atomic_inc_return(&stat->disabled) != 1)
  595. goto out;
  596. /*
  597. * Try to find the function again since an NMI
  598. * could have added it
  599. */
  600. rec = ftrace_find_profiled_func(stat, ip);
  601. if (rec)
  602. goto out;
  603. if (stat->pages->index == PROFILES_PER_PAGE) {
  604. if (!stat->pages->next)
  605. goto out;
  606. stat->pages = stat->pages->next;
  607. }
  608. rec = &stat->pages->records[stat->pages->index++];
  609. rec->ip = ip;
  610. ftrace_add_profile(stat, rec);
  611. out:
  612. atomic_dec(&stat->disabled);
  613. return rec;
  614. }
  615. static void
  616. function_profile_call(unsigned long ip, unsigned long parent_ip,
  617. struct ftrace_ops *ops, struct pt_regs *regs)
  618. {
  619. struct ftrace_profile_stat *stat;
  620. struct ftrace_profile *rec;
  621. unsigned long flags;
  622. if (!ftrace_profile_enabled)
  623. return;
  624. local_irq_save(flags);
  625. stat = this_cpu_ptr(&ftrace_profile_stats);
  626. if (!stat->hash || !ftrace_profile_enabled)
  627. goto out;
  628. rec = ftrace_find_profiled_func(stat, ip);
  629. if (!rec) {
  630. rec = ftrace_profile_alloc(stat, ip);
  631. if (!rec)
  632. goto out;
  633. }
  634. rec->counter++;
  635. out:
  636. local_irq_restore(flags);
  637. }
  638. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  639. static bool fgraph_graph_time = true;
  640. void ftrace_graph_graph_time_control(bool enable)
  641. {
  642. fgraph_graph_time = enable;
  643. }
  644. static int profile_graph_entry(struct ftrace_graph_ent *trace)
  645. {
  646. struct ftrace_ret_stack *ret_stack;
  647. function_profile_call(trace->func, 0, NULL, NULL);
  648. /* If function graph is shutting down, ret_stack can be NULL */
  649. if (!current->ret_stack)
  650. return 0;
  651. ret_stack = ftrace_graph_get_ret_stack(current, 0);
  652. if (ret_stack)
  653. ret_stack->subtime = 0;
  654. return 1;
  655. }
  656. static void profile_graph_return(struct ftrace_graph_ret *trace)
  657. {
  658. struct ftrace_ret_stack *ret_stack;
  659. struct ftrace_profile_stat *stat;
  660. unsigned long long calltime;
  661. struct ftrace_profile *rec;
  662. unsigned long flags;
  663. local_irq_save(flags);
  664. stat = this_cpu_ptr(&ftrace_profile_stats);
  665. if (!stat->hash || !ftrace_profile_enabled)
  666. goto out;
  667. /* If the calltime was zero'd ignore it */
  668. if (!trace->calltime)
  669. goto out;
  670. calltime = trace->rettime - trace->calltime;
  671. if (!fgraph_graph_time) {
  672. /* Append this call time to the parent time to subtract */
  673. ret_stack = ftrace_graph_get_ret_stack(current, 1);
  674. if (ret_stack)
  675. ret_stack->subtime += calltime;
  676. ret_stack = ftrace_graph_get_ret_stack(current, 0);
  677. if (ret_stack && ret_stack->subtime < calltime)
  678. calltime -= ret_stack->subtime;
  679. else
  680. calltime = 0;
  681. }
  682. rec = ftrace_find_profiled_func(stat, trace->func);
  683. if (rec) {
  684. rec->time += calltime;
  685. rec->time_squared += calltime * calltime;
  686. }
  687. out:
  688. local_irq_restore(flags);
  689. }
  690. static struct fgraph_ops fprofiler_ops = {
  691. .entryfunc = &profile_graph_entry,
  692. .retfunc = &profile_graph_return,
  693. };
  694. static int register_ftrace_profiler(void)
  695. {
  696. return register_ftrace_graph(&fprofiler_ops);
  697. }
  698. static void unregister_ftrace_profiler(void)
  699. {
  700. unregister_ftrace_graph(&fprofiler_ops);
  701. }
  702. #else
  703. static struct ftrace_ops ftrace_profile_ops __read_mostly = {
  704. .func = function_profile_call,
  705. .flags = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED,
  706. INIT_OPS_HASH(ftrace_profile_ops)
  707. };
  708. static int register_ftrace_profiler(void)
  709. {
  710. return register_ftrace_function(&ftrace_profile_ops);
  711. }
  712. static void unregister_ftrace_profiler(void)
  713. {
  714. unregister_ftrace_function(&ftrace_profile_ops);
  715. }
  716. #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
  717. static ssize_t
  718. ftrace_profile_write(struct file *filp, const char __user *ubuf,
  719. size_t cnt, loff_t *ppos)
  720. {
  721. unsigned long val;
  722. int ret;
  723. ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
  724. if (ret)
  725. return ret;
  726. val = !!val;
  727. mutex_lock(&ftrace_profile_lock);
  728. if (ftrace_profile_enabled ^ val) {
  729. if (val) {
  730. ret = ftrace_profile_init();
  731. if (ret < 0) {
  732. cnt = ret;
  733. goto out;
  734. }
  735. ret = register_ftrace_profiler();
  736. if (ret < 0) {
  737. cnt = ret;
  738. goto out;
  739. }
  740. ftrace_profile_enabled = 1;
  741. } else {
  742. ftrace_profile_enabled = 0;
  743. /*
  744. * unregister_ftrace_profiler calls stop_machine
  745. * so this acts like an synchronize_rcu.
  746. */
  747. unregister_ftrace_profiler();
  748. }
  749. }
  750. out:
  751. mutex_unlock(&ftrace_profile_lock);
  752. *ppos += cnt;
  753. return cnt;
  754. }
  755. static ssize_t
  756. ftrace_profile_read(struct file *filp, char __user *ubuf,
  757. size_t cnt, loff_t *ppos)
  758. {
  759. char buf[64]; /* big enough to hold a number */
  760. int r;
  761. r = sprintf(buf, "%u\n", ftrace_profile_enabled);
  762. return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
  763. }
  764. static const struct file_operations ftrace_profile_fops = {
  765. .open = tracing_open_generic,
  766. .read = ftrace_profile_read,
  767. .write = ftrace_profile_write,
  768. .llseek = default_llseek,
  769. };
  770. /* used to initialize the real stat files */
  771. static struct tracer_stat function_stats __initdata = {
  772. .name = "functions",
  773. .stat_start = function_stat_start,
  774. .stat_next = function_stat_next,
  775. .stat_cmp = function_stat_cmp,
  776. .stat_headers = function_stat_headers,
  777. .stat_show = function_stat_show
  778. };
  779. static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
  780. {
  781. struct ftrace_profile_stat *stat;
  782. struct dentry *entry;
  783. char *name;
  784. int ret;
  785. int cpu;
  786. for_each_possible_cpu(cpu) {
  787. stat = &per_cpu(ftrace_profile_stats, cpu);
  788. name = kasprintf(GFP_KERNEL, "function%d", cpu);
  789. if (!name) {
  790. /*
  791. * The files created are permanent, if something happens
  792. * we still do not free memory.
  793. */
  794. WARN(1,
  795. "Could not allocate stat file for cpu %d\n",
  796. cpu);
  797. return;
  798. }
  799. stat->stat = function_stats;
  800. stat->stat.name = name;
  801. ret = register_stat_tracer(&stat->stat);
  802. if (ret) {
  803. WARN(1,
  804. "Could not register function stat for cpu %d\n",
  805. cpu);
  806. kfree(name);
  807. return;
  808. }
  809. }
  810. entry = tracefs_create_file("function_profile_enabled", 0644,
  811. d_tracer, NULL, &ftrace_profile_fops);
  812. if (!entry)
  813. pr_warn("Could not create tracefs 'function_profile_enabled' entry\n");
  814. }
  815. #else /* CONFIG_FUNCTION_PROFILER */
  816. static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
  817. {
  818. }
  819. #endif /* CONFIG_FUNCTION_PROFILER */
  820. #ifdef CONFIG_DYNAMIC_FTRACE
  821. static struct ftrace_ops *removed_ops;
  822. /*
  823. * Set when doing a global update, like enabling all recs or disabling them.
  824. * It is not set when just updating a single ftrace_ops.
  825. */
  826. static bool update_all_ops;
  827. #ifndef CONFIG_FTRACE_MCOUNT_RECORD
  828. # error Dynamic ftrace depends on MCOUNT_RECORD
  829. #endif
  830. struct ftrace_func_probe {
  831. struct ftrace_probe_ops *probe_ops;
  832. struct ftrace_ops ops;
  833. struct trace_array *tr;
  834. struct list_head list;
  835. void *data;
  836. int ref;
  837. };
  838. /*
  839. * We make these constant because no one should touch them,
  840. * but they are used as the default "empty hash", to avoid allocating
  841. * it all the time. These are in a read only section such that if
  842. * anyone does try to modify it, it will cause an exception.
  843. */
  844. static const struct hlist_head empty_buckets[1];
  845. static const struct ftrace_hash empty_hash = {
  846. .buckets = (struct hlist_head *)empty_buckets,
  847. };
  848. #define EMPTY_HASH ((struct ftrace_hash *)&empty_hash)
  849. struct ftrace_ops global_ops = {
  850. .func = ftrace_stub,
  851. .local_hash.notrace_hash = EMPTY_HASH,
  852. .local_hash.filter_hash = EMPTY_HASH,
  853. INIT_OPS_HASH(global_ops)
  854. .flags = FTRACE_OPS_FL_RECURSION_SAFE |
  855. FTRACE_OPS_FL_INITIALIZED |
  856. FTRACE_OPS_FL_PID,
  857. };
  858. /*
  859. * Used by the stack undwinder to know about dynamic ftrace trampolines.
  860. */
  861. struct ftrace_ops *ftrace_ops_trampoline(unsigned long addr)
  862. {
  863. struct ftrace_ops *op = NULL;
  864. /*
  865. * Some of the ops may be dynamically allocated,
  866. * they are freed after a synchronize_rcu().
  867. */
  868. preempt_disable_notrace();
  869. do_for_each_ftrace_op(op, ftrace_ops_list) {
  870. /*
  871. * This is to check for dynamically allocated trampolines.
  872. * Trampolines that are in kernel text will have
  873. * core_kernel_text() return true.
  874. */
  875. if (op->trampoline && op->trampoline_size)
  876. if (addr >= op->trampoline &&
  877. addr < op->trampoline + op->trampoline_size) {
  878. preempt_enable_notrace();
  879. return op;
  880. }
  881. } while_for_each_ftrace_op(op);
  882. preempt_enable_notrace();
  883. return NULL;
  884. }
  885. /*
  886. * This is used by __kernel_text_address() to return true if the
  887. * address is on a dynamically allocated trampoline that would
  888. * not return true for either core_kernel_text() or
  889. * is_module_text_address().
  890. */
  891. bool is_ftrace_trampoline(unsigned long addr)
  892. {
  893. return ftrace_ops_trampoline(addr) != NULL;
  894. }
  895. struct ftrace_page {
  896. struct ftrace_page *next;
  897. struct dyn_ftrace *records;
  898. int index;
  899. int size;
  900. };
  901. #define ENTRY_SIZE sizeof(struct dyn_ftrace)
  902. #define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE)
  903. static struct ftrace_page *ftrace_pages_start;
  904. static struct ftrace_page *ftrace_pages;
  905. static __always_inline unsigned long
  906. ftrace_hash_key(struct ftrace_hash *hash, unsigned long ip)
  907. {
  908. if (hash->size_bits > 0)
  909. return hash_long(ip, hash->size_bits);
  910. return 0;
  911. }
  912. /* Only use this function if ftrace_hash_empty() has already been tested */
  913. static __always_inline struct ftrace_func_entry *
  914. __ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
  915. {
  916. unsigned long key;
  917. struct ftrace_func_entry *entry;
  918. struct hlist_head *hhd;
  919. key = ftrace_hash_key(hash, ip);
  920. hhd = &hash->buckets[key];
  921. hlist_for_each_entry_rcu_notrace(entry, hhd, hlist) {
  922. if (entry->ip == ip)
  923. return entry;
  924. }
  925. return NULL;
  926. }
  927. /**
  928. * ftrace_lookup_ip - Test to see if an ip exists in an ftrace_hash
  929. * @hash: The hash to look at
  930. * @ip: The instruction pointer to test
  931. *
  932. * Search a given @hash to see if a given instruction pointer (@ip)
  933. * exists in it.
  934. *
  935. * Returns the entry that holds the @ip if found. NULL otherwise.
  936. */
  937. struct ftrace_func_entry *
  938. ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
  939. {
  940. if (ftrace_hash_empty(hash))
  941. return NULL;
  942. return __ftrace_lookup_ip(hash, ip);
  943. }
  944. static void __add_hash_entry(struct ftrace_hash *hash,
  945. struct ftrace_func_entry *entry)
  946. {
  947. struct hlist_head *hhd;
  948. unsigned long key;
  949. key = ftrace_hash_key(hash, entry->ip);
  950. hhd = &hash->buckets[key];
  951. hlist_add_head(&entry->hlist, hhd);
  952. hash->count++;
  953. }
  954. static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
  955. {
  956. struct ftrace_func_entry *entry;
  957. entry = kmalloc(sizeof(*entry), GFP_KERNEL);
  958. if (!entry)
  959. return -ENOMEM;
  960. entry->ip = ip;
  961. __add_hash_entry(hash, entry);
  962. return 0;
  963. }
  964. static void
  965. free_hash_entry(struct ftrace_hash *hash,
  966. struct ftrace_func_entry *entry)
  967. {
  968. hlist_del(&entry->hlist);
  969. kfree(entry);
  970. hash->count--;
  971. }
  972. static void
  973. remove_hash_entry(struct ftrace_hash *hash,
  974. struct ftrace_func_entry *entry)
  975. {
  976. hlist_del_rcu(&entry->hlist);
  977. hash->count--;
  978. }
  979. static void ftrace_hash_clear(struct ftrace_hash *hash)
  980. {
  981. struct hlist_head *hhd;
  982. struct hlist_node *tn;
  983. struct ftrace_func_entry *entry;
  984. int size = 1 << hash->size_bits;
  985. int i;
  986. if (!hash->count)
  987. return;
  988. for (i = 0; i < size; i++) {
  989. hhd = &hash->buckets[i];
  990. hlist_for_each_entry_safe(entry, tn, hhd, hlist)
  991. free_hash_entry(hash, entry);
  992. }
  993. FTRACE_WARN_ON(hash->count);
  994. }
  995. static void free_ftrace_mod(struct ftrace_mod_load *ftrace_mod)
  996. {
  997. list_del(&ftrace_mod->list);
  998. kfree(ftrace_mod->module);
  999. kfree(ftrace_mod->func);
  1000. kfree(ftrace_mod);
  1001. }
  1002. static void clear_ftrace_mod_list(struct list_head *head)
  1003. {
  1004. struct ftrace_mod_load *p, *n;
  1005. /* stack tracer isn't supported yet */
  1006. if (!head)
  1007. return;
  1008. mutex_lock(&ftrace_lock);
  1009. list_for_each_entry_safe(p, n, head, list)
  1010. free_ftrace_mod(p);
  1011. mutex_unlock(&ftrace_lock);
  1012. }
  1013. static void free_ftrace_hash(struct ftrace_hash *hash)
  1014. {
  1015. if (!hash || hash == EMPTY_HASH)
  1016. return;
  1017. ftrace_hash_clear(hash);
  1018. kfree(hash->buckets);
  1019. kfree(hash);
  1020. }
  1021. static void __free_ftrace_hash_rcu(struct rcu_head *rcu)
  1022. {
  1023. struct ftrace_hash *hash;
  1024. hash = container_of(rcu, struct ftrace_hash, rcu);
  1025. free_ftrace_hash(hash);
  1026. }
  1027. static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
  1028. {
  1029. if (!hash || hash == EMPTY_HASH)
  1030. return;
  1031. call_rcu(&hash->rcu, __free_ftrace_hash_rcu);
  1032. }
  1033. void ftrace_free_filter(struct ftrace_ops *ops)
  1034. {
  1035. ftrace_ops_init(ops);
  1036. free_ftrace_hash(ops->func_hash->filter_hash);
  1037. free_ftrace_hash(ops->func_hash->notrace_hash);
  1038. }
  1039. static struct ftrace_hash *alloc_ftrace_hash(int size_bits)
  1040. {
  1041. struct ftrace_hash *hash;
  1042. int size;
  1043. hash = kzalloc(sizeof(*hash), GFP_KERNEL);
  1044. if (!hash)
  1045. return NULL;
  1046. size = 1 << size_bits;
  1047. hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL);
  1048. if (!hash->buckets) {
  1049. kfree(hash);
  1050. return NULL;
  1051. }
  1052. hash->size_bits = size_bits;
  1053. return hash;
  1054. }
  1055. static int ftrace_add_mod(struct trace_array *tr,
  1056. const char *func, const char *module,
  1057. int enable)
  1058. {
  1059. struct ftrace_mod_load *ftrace_mod;
  1060. struct list_head *mod_head = enable ? &tr->mod_trace : &tr->mod_notrace;
  1061. ftrace_mod = kzalloc(sizeof(*ftrace_mod), GFP_KERNEL);
  1062. if (!ftrace_mod)
  1063. return -ENOMEM;
  1064. ftrace_mod->func = kstrdup(func, GFP_KERNEL);
  1065. ftrace_mod->module = kstrdup(module, GFP_KERNEL);
  1066. ftrace_mod->enable = enable;
  1067. if (!ftrace_mod->func || !ftrace_mod->module)
  1068. goto out_free;
  1069. list_add(&ftrace_mod->list, mod_head);
  1070. return 0;
  1071. out_free:
  1072. free_ftrace_mod(ftrace_mod);
  1073. return -ENOMEM;
  1074. }
  1075. static struct ftrace_hash *
  1076. alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
  1077. {
  1078. struct ftrace_func_entry *entry;
  1079. struct ftrace_hash *new_hash;
  1080. int size;
  1081. int ret;
  1082. int i;
  1083. new_hash = alloc_ftrace_hash(size_bits);
  1084. if (!new_hash)
  1085. return NULL;
  1086. if (hash)
  1087. new_hash->flags = hash->flags;
  1088. /* Empty hash? */
  1089. if (ftrace_hash_empty(hash))
  1090. return new_hash;
  1091. size = 1 << hash->size_bits;
  1092. for (i = 0; i < size; i++) {
  1093. hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
  1094. ret = add_hash_entry(new_hash, entry->ip);
  1095. if (ret < 0)
  1096. goto free_hash;
  1097. }
  1098. }
  1099. FTRACE_WARN_ON(new_hash->count != hash->count);
  1100. return new_hash;
  1101. free_hash:
  1102. free_ftrace_hash(new_hash);
  1103. return NULL;
  1104. }
  1105. static void
  1106. ftrace_hash_rec_disable_modify(struct ftrace_ops *ops, int filter_hash);
  1107. static void
  1108. ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, int filter_hash);
  1109. static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
  1110. struct ftrace_hash *new_hash);
  1111. static struct ftrace_hash *dup_hash(struct ftrace_hash *src, int size)
  1112. {
  1113. struct ftrace_func_entry *entry;
  1114. struct ftrace_hash *new_hash;
  1115. struct hlist_head *hhd;
  1116. struct hlist_node *tn;
  1117. int bits = 0;
  1118. int i;
  1119. /*
  1120. * Use around half the size (max bit of it), but
  1121. * a minimum of 2 is fine (as size of 0 or 1 both give 1 for bits).
  1122. */
  1123. bits = fls(size / 2);
  1124. /* Don't allocate too much */
  1125. if (bits > FTRACE_HASH_MAX_BITS)
  1126. bits = FTRACE_HASH_MAX_BITS;
  1127. new_hash = alloc_ftrace_hash(bits);
  1128. if (!new_hash)
  1129. return NULL;
  1130. new_hash->flags = src->flags;
  1131. size = 1 << src->size_bits;
  1132. for (i = 0; i < size; i++) {
  1133. hhd = &src->buckets[i];
  1134. hlist_for_each_entry_safe(entry, tn, hhd, hlist) {
  1135. remove_hash_entry(src, entry);
  1136. __add_hash_entry(new_hash, entry);
  1137. }
  1138. }
  1139. return new_hash;
  1140. }
  1141. static struct ftrace_hash *
  1142. __ftrace_hash_move(struct ftrace_hash *src)
  1143. {
  1144. int size = src->count;
  1145. /*
  1146. * If the new source is empty, just return the empty_hash.
  1147. */
  1148. if (ftrace_hash_empty(src))
  1149. return EMPTY_HASH;
  1150. return dup_hash(src, size);
  1151. }
  1152. static int
  1153. ftrace_hash_move(struct ftrace_ops *ops, int enable,
  1154. struct ftrace_hash **dst, struct ftrace_hash *src)
  1155. {
  1156. struct ftrace_hash *new_hash;
  1157. int ret;
  1158. /* Reject setting notrace hash on IPMODIFY ftrace_ops */
  1159. if (ops->flags & FTRACE_OPS_FL_IPMODIFY && !enable)
  1160. return -EINVAL;
  1161. new_hash = __ftrace_hash_move(src);
  1162. if (!new_hash)
  1163. return -ENOMEM;
  1164. /* Make sure this can be applied if it is IPMODIFY ftrace_ops */
  1165. if (enable) {
  1166. /* IPMODIFY should be updated only when filter_hash updating */
  1167. ret = ftrace_hash_ipmodify_update(ops, new_hash);
  1168. if (ret < 0) {
  1169. free_ftrace_hash(new_hash);
  1170. return ret;
  1171. }
  1172. }
  1173. /*
  1174. * Remove the current set, update the hash and add
  1175. * them back.
  1176. */
  1177. ftrace_hash_rec_disable_modify(ops, enable);
  1178. rcu_assign_pointer(*dst, new_hash);
  1179. ftrace_hash_rec_enable_modify(ops, enable);
  1180. return 0;
  1181. }
  1182. static bool hash_contains_ip(unsigned long ip,
  1183. struct ftrace_ops_hash *hash)
  1184. {
  1185. /*
  1186. * The function record is a match if it exists in the filter
  1187. * hash and not in the notrace hash. Note, an empty hash is
  1188. * considered a match for the filter hash, but an empty
  1189. * notrace hash is considered not in the notrace hash.
  1190. */
  1191. return (ftrace_hash_empty(hash->filter_hash) ||
  1192. __ftrace_lookup_ip(hash->filter_hash, ip)) &&
  1193. (ftrace_hash_empty(hash->notrace_hash) ||
  1194. !__ftrace_lookup_ip(hash->notrace_hash, ip));
  1195. }
  1196. /*
  1197. * Test the hashes for this ops to see if we want to call
  1198. * the ops->func or not.
  1199. *
  1200. * It's a match if the ip is in the ops->filter_hash or
  1201. * the filter_hash does not exist or is empty,
  1202. * AND
  1203. * the ip is not in the ops->notrace_hash.
  1204. *
  1205. * This needs to be called with preemption disabled as
  1206. * the hashes are freed with call_rcu().
  1207. */
  1208. int
  1209. ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
  1210. {
  1211. struct ftrace_ops_hash hash;
  1212. int ret;
  1213. #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
  1214. /*
  1215. * There's a small race when adding ops that the ftrace handler
  1216. * that wants regs, may be called without them. We can not
  1217. * allow that handler to be called if regs is NULL.
  1218. */
  1219. if (regs == NULL && (ops->flags & FTRACE_OPS_FL_SAVE_REGS))
  1220. return 0;
  1221. #endif
  1222. rcu_assign_pointer(hash.filter_hash, ops->func_hash->filter_hash);
  1223. rcu_assign_pointer(hash.notrace_hash, ops->func_hash->notrace_hash);
  1224. if (hash_contains_ip(ip, &hash))
  1225. ret = 1;
  1226. else
  1227. ret = 0;
  1228. return ret;
  1229. }
  1230. /*
  1231. * This is a double for. Do not use 'break' to break out of the loop,
  1232. * you must use a goto.
  1233. */
  1234. #define do_for_each_ftrace_rec(pg, rec) \
  1235. for (pg = ftrace_pages_start; pg; pg = pg->next) { \
  1236. int _____i; \
  1237. for (_____i = 0; _____i < pg->index; _____i++) { \
  1238. rec = &pg->records[_____i];
  1239. #define while_for_each_ftrace_rec() \
  1240. } \
  1241. }
  1242. static int ftrace_cmp_recs(const void *a, const void *b)
  1243. {
  1244. const struct dyn_ftrace *key = a;
  1245. const struct dyn_ftrace *rec = b;
  1246. if (key->flags < rec->ip)
  1247. return -1;
  1248. if (key->ip >= rec->ip + MCOUNT_INSN_SIZE)
  1249. return 1;
  1250. return 0;
  1251. }
  1252. static struct dyn_ftrace *lookup_rec(unsigned long start, unsigned long end)
  1253. {
  1254. struct ftrace_page *pg;
  1255. struct dyn_ftrace *rec = NULL;
  1256. struct dyn_ftrace key;
  1257. key.ip = start;
  1258. key.flags = end; /* overload flags, as it is unsigned long */
  1259. for (pg = ftrace_pages_start; pg; pg = pg->next) {
  1260. if (end < pg->records[0].ip ||
  1261. start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
  1262. continue;
  1263. rec = bsearch(&key, pg->records, pg->index,
  1264. sizeof(struct dyn_ftrace),
  1265. ftrace_cmp_recs);
  1266. if (rec)
  1267. break;
  1268. }
  1269. return rec;
  1270. }
  1271. /**
  1272. * ftrace_location_range - return the first address of a traced location
  1273. * if it touches the given ip range
  1274. * @start: start of range to search.
  1275. * @end: end of range to search (inclusive). @end points to the last byte
  1276. * to check.
  1277. *
  1278. * Returns rec->ip if the related ftrace location is a least partly within
  1279. * the given address range. That is, the first address of the instruction
  1280. * that is either a NOP or call to the function tracer. It checks the ftrace
  1281. * internal tables to determine if the address belongs or not.
  1282. */
  1283. unsigned long ftrace_location_range(unsigned long start, unsigned long end)
  1284. {
  1285. struct dyn_ftrace *rec;
  1286. rec = lookup_rec(start, end);
  1287. if (rec)
  1288. return rec->ip;
  1289. return 0;
  1290. }
  1291. /**
  1292. * ftrace_location - return true if the ip giving is a traced location
  1293. * @ip: the instruction pointer to check
  1294. *
  1295. * Returns rec->ip if @ip given is a pointer to a ftrace location.
  1296. * That is, the instruction that is either a NOP or call to
  1297. * the function tracer. It checks the ftrace internal tables to
  1298. * determine if the address belongs or not.
  1299. */
  1300. unsigned long ftrace_location(unsigned long ip)
  1301. {
  1302. return ftrace_location_range(ip, ip);
  1303. }
  1304. /**
  1305. * ftrace_text_reserved - return true if range contains an ftrace location
  1306. * @start: start of range to search
  1307. * @end: end of range to search (inclusive). @end points to the last byte to check.
  1308. *
  1309. * Returns 1 if @start and @end contains a ftrace location.
  1310. * That is, the instruction that is either a NOP or call to
  1311. * the function tracer. It checks the ftrace internal tables to
  1312. * determine if the address belongs or not.
  1313. */
  1314. int ftrace_text_reserved(const void *start, const void *end)
  1315. {
  1316. unsigned long ret;
  1317. ret = ftrace_location_range((unsigned long)start,
  1318. (unsigned long)end);
  1319. return (int)!!ret;
  1320. }
  1321. /* Test if ops registered to this rec needs regs */
  1322. static bool test_rec_ops_needs_regs(struct dyn_ftrace *rec)
  1323. {
  1324. struct ftrace_ops *ops;
  1325. bool keep_regs = false;
  1326. for (ops = ftrace_ops_list;
  1327. ops != &ftrace_list_end; ops = ops->next) {
  1328. /* pass rec in as regs to have non-NULL val */
  1329. if (ftrace_ops_test(ops, rec->ip, rec)) {
  1330. if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
  1331. keep_regs = true;
  1332. break;
  1333. }
  1334. }
  1335. }
  1336. return keep_regs;
  1337. }
  1338. static struct ftrace_ops *
  1339. ftrace_find_tramp_ops_any(struct dyn_ftrace *rec);
  1340. static struct ftrace_ops *
  1341. ftrace_find_tramp_ops_any_other(struct dyn_ftrace *rec, struct ftrace_ops *op_exclude);
  1342. static struct ftrace_ops *
  1343. ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *ops);
  1344. static bool __ftrace_hash_rec_update(struct ftrace_ops *ops,
  1345. int filter_hash,
  1346. bool inc)
  1347. {
  1348. struct ftrace_hash *hash;
  1349. struct ftrace_hash *other_hash;
  1350. struct ftrace_page *pg;
  1351. struct dyn_ftrace *rec;
  1352. bool update = false;
  1353. int count = 0;
  1354. int all = false;
  1355. /* Only update if the ops has been registered */
  1356. if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
  1357. return false;
  1358. /*
  1359. * In the filter_hash case:
  1360. * If the count is zero, we update all records.
  1361. * Otherwise we just update the items in the hash.
  1362. *
  1363. * In the notrace_hash case:
  1364. * We enable the update in the hash.
  1365. * As disabling notrace means enabling the tracing,
  1366. * and enabling notrace means disabling, the inc variable
  1367. * gets inversed.
  1368. */
  1369. if (filter_hash) {
  1370. hash = ops->func_hash->filter_hash;
  1371. other_hash = ops->func_hash->notrace_hash;
  1372. if (ftrace_hash_empty(hash))
  1373. all = true;
  1374. } else {
  1375. inc = !inc;
  1376. hash = ops->func_hash->notrace_hash;
  1377. other_hash = ops->func_hash->filter_hash;
  1378. /*
  1379. * If the notrace hash has no items,
  1380. * then there's nothing to do.
  1381. */
  1382. if (ftrace_hash_empty(hash))
  1383. return false;
  1384. }
  1385. do_for_each_ftrace_rec(pg, rec) {
  1386. int in_other_hash = 0;
  1387. int in_hash = 0;
  1388. int match = 0;
  1389. if (rec->flags & FTRACE_FL_DISABLED)
  1390. continue;
  1391. if (all) {
  1392. /*
  1393. * Only the filter_hash affects all records.
  1394. * Update if the record is not in the notrace hash.
  1395. */
  1396. if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))
  1397. match = 1;
  1398. } else {
  1399. in_hash = !!ftrace_lookup_ip(hash, rec->ip);
  1400. in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip);
  1401. /*
  1402. * If filter_hash is set, we want to match all functions
  1403. * that are in the hash but not in the other hash.
  1404. *
  1405. * If filter_hash is not set, then we are decrementing.
  1406. * That means we match anything that is in the hash
  1407. * and also in the other_hash. That is, we need to turn
  1408. * off functions in the other hash because they are disabled
  1409. * by this hash.
  1410. */
  1411. if (filter_hash && in_hash && !in_other_hash)
  1412. match = 1;
  1413. else if (!filter_hash && in_hash &&
  1414. (in_other_hash || ftrace_hash_empty(other_hash)))
  1415. match = 1;
  1416. }
  1417. if (!match)
  1418. continue;
  1419. if (inc) {
  1420. rec->flags++;
  1421. if (FTRACE_WARN_ON(ftrace_rec_count(rec) == FTRACE_REF_MAX))
  1422. return false;
  1423. if (ops->flags & FTRACE_OPS_FL_DIRECT)
  1424. rec->flags |= FTRACE_FL_DIRECT;
  1425. /*
  1426. * If there's only a single callback registered to a
  1427. * function, and the ops has a trampoline registered
  1428. * for it, then we can call it directly.
  1429. */
  1430. if (ftrace_rec_count(rec) == 1 && ops->trampoline)
  1431. rec->flags |= FTRACE_FL_TRAMP;
  1432. else
  1433. /*
  1434. * If we are adding another function callback
  1435. * to this function, and the previous had a
  1436. * custom trampoline in use, then we need to go
  1437. * back to the default trampoline.
  1438. */
  1439. rec->flags &= ~FTRACE_FL_TRAMP;
  1440. /*
  1441. * If any ops wants regs saved for this function
  1442. * then all ops will get saved regs.
  1443. */
  1444. if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
  1445. rec->flags |= FTRACE_FL_REGS;
  1446. } else {
  1447. if (FTRACE_WARN_ON(ftrace_rec_count(rec) == 0))
  1448. return false;
  1449. rec->flags--;
  1450. /*
  1451. * Only the internal direct_ops should have the
  1452. * DIRECT flag set. Thus, if it is removing a
  1453. * function, then that function should no longer
  1454. * be direct.
  1455. */
  1456. if (ops->flags & FTRACE_OPS_FL_DIRECT)
  1457. rec->flags &= ~FTRACE_FL_DIRECT;
  1458. /*
  1459. * If the rec had REGS enabled and the ops that is
  1460. * being removed had REGS set, then see if there is
  1461. * still any ops for this record that wants regs.
  1462. * If not, we can stop recording them.
  1463. */
  1464. if (ftrace_rec_count(rec) > 0 &&
  1465. rec->flags & FTRACE_FL_REGS &&
  1466. ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
  1467. if (!test_rec_ops_needs_regs(rec))
  1468. rec->flags &= ~FTRACE_FL_REGS;
  1469. }
  1470. /*
  1471. * The TRAMP needs to be set only if rec count
  1472. * is decremented to one, and the ops that is
  1473. * left has a trampoline. As TRAMP can only be
  1474. * enabled if there is only a single ops attached
  1475. * to it.
  1476. */
  1477. if (ftrace_rec_count(rec) == 1 &&
  1478. ftrace_find_tramp_ops_any_other(rec, ops))
  1479. rec->flags |= FTRACE_FL_TRAMP;
  1480. else
  1481. rec->flags &= ~FTRACE_FL_TRAMP;
  1482. /*
  1483. * flags will be cleared in ftrace_check_record()
  1484. * if rec count is zero.
  1485. */
  1486. }
  1487. count++;
  1488. /* Must match FTRACE_UPDATE_CALLS in ftrace_modify_all_code() */
  1489. update |= ftrace_test_record(rec, true) != FTRACE_UPDATE_IGNORE;
  1490. /* Shortcut, if we handled all records, we are done. */
  1491. if (!all && count == hash->count)
  1492. return update;
  1493. } while_for_each_ftrace_rec();
  1494. return update;
  1495. }
  1496. static bool ftrace_hash_rec_disable(struct ftrace_ops *ops,
  1497. int filter_hash)
  1498. {
  1499. return __ftrace_hash_rec_update(ops, filter_hash, 0);
  1500. }
  1501. static bool ftrace_hash_rec_enable(struct ftrace_ops *ops,
  1502. int filter_hash)
  1503. {
  1504. return __ftrace_hash_rec_update(ops, filter_hash, 1);
  1505. }
  1506. static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops,
  1507. int filter_hash, int inc)
  1508. {
  1509. struct ftrace_ops *op;
  1510. __ftrace_hash_rec_update(ops, filter_hash, inc);
  1511. if (ops->func_hash != &global_ops.local_hash)
  1512. return;
  1513. /*
  1514. * If the ops shares the global_ops hash, then we need to update
  1515. * all ops that are enabled and use this hash.
  1516. */
  1517. do_for_each_ftrace_op(op, ftrace_ops_list) {
  1518. /* Already done */
  1519. if (op == ops)
  1520. continue;
  1521. if (op->func_hash == &global_ops.local_hash)
  1522. __ftrace_hash_rec_update(op, filter_hash, inc);
  1523. } while_for_each_ftrace_op(op);
  1524. }
  1525. static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops,
  1526. int filter_hash)
  1527. {
  1528. ftrace_hash_rec_update_modify(ops, filter_hash, 0);
  1529. }
  1530. static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops,
  1531. int filter_hash)
  1532. {
  1533. ftrace_hash_rec_update_modify(ops, filter_hash, 1);
  1534. }
  1535. /*
  1536. * Try to update IPMODIFY flag on each ftrace_rec. Return 0 if it is OK
  1537. * or no-needed to update, -EBUSY if it detects a conflict of the flag
  1538. * on a ftrace_rec, and -EINVAL if the new_hash tries to trace all recs.
  1539. * Note that old_hash and new_hash has below meanings
  1540. * - If the hash is NULL, it hits all recs (if IPMODIFY is set, this is rejected)
  1541. * - If the hash is EMPTY_HASH, it hits nothing
  1542. * - Anything else hits the recs which match the hash entries.
  1543. */
  1544. static int __ftrace_hash_update_ipmodify(struct ftrace_ops *ops,
  1545. struct ftrace_hash *old_hash,
  1546. struct ftrace_hash *new_hash)
  1547. {
  1548. struct ftrace_page *pg;
  1549. struct dyn_ftrace *rec, *end = NULL;
  1550. int in_old, in_new;
  1551. /* Only update if the ops has been registered */
  1552. if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
  1553. return 0;
  1554. if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY))
  1555. return 0;
  1556. /*
  1557. * Since the IPMODIFY is a very address sensitive action, we do not
  1558. * allow ftrace_ops to set all functions to new hash.
  1559. */
  1560. if (!new_hash || !old_hash)
  1561. return -EINVAL;
  1562. /* Update rec->flags */
  1563. do_for_each_ftrace_rec(pg, rec) {
  1564. if (rec->flags & FTRACE_FL_DISABLED)
  1565. continue;
  1566. /* We need to update only differences of filter_hash */
  1567. in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
  1568. in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
  1569. if (in_old == in_new)
  1570. continue;
  1571. if (in_new) {
  1572. /* New entries must ensure no others are using it */
  1573. if (rec->flags & FTRACE_FL_IPMODIFY)
  1574. goto rollback;
  1575. rec->flags |= FTRACE_FL_IPMODIFY;
  1576. } else /* Removed entry */
  1577. rec->flags &= ~FTRACE_FL_IPMODIFY;
  1578. } while_for_each_ftrace_rec();
  1579. return 0;
  1580. rollback:
  1581. end = rec;
  1582. /* Roll back what we did above */
  1583. do_for_each_ftrace_rec(pg, rec) {
  1584. if (rec->flags & FTRACE_FL_DISABLED)
  1585. continue;
  1586. if (rec == end)
  1587. goto err_out;
  1588. in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
  1589. in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
  1590. if (in_old == in_new)
  1591. continue;
  1592. if (in_new)
  1593. rec->flags &= ~FTRACE_FL_IPMODIFY;
  1594. else
  1595. rec->flags |= FTRACE_FL_IPMODIFY;
  1596. } while_for_each_ftrace_rec();
  1597. err_out:
  1598. return -EBUSY;
  1599. }
  1600. static int ftrace_hash_ipmodify_enable(struct ftrace_ops *ops)
  1601. {
  1602. struct ftrace_hash *hash = ops->func_hash->filter_hash;
  1603. if (ftrace_hash_empty(hash))
  1604. hash = NULL;
  1605. return __ftrace_hash_update_ipmodify(ops, EMPTY_HASH, hash);
  1606. }
  1607. /* Disabling always succeeds */
  1608. static void ftrace_hash_ipmodify_disable(struct ftrace_ops *ops)
  1609. {
  1610. struct ftrace_hash *hash = ops->func_hash->filter_hash;
  1611. if (ftrace_hash_empty(hash))
  1612. hash = NULL;
  1613. __ftrace_hash_update_ipmodify(ops, hash, EMPTY_HASH);
  1614. }
  1615. static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
  1616. struct ftrace_hash *new_hash)
  1617. {
  1618. struct ftrace_hash *old_hash = ops->func_hash->filter_hash;
  1619. if (ftrace_hash_empty(old_hash))
  1620. old_hash = NULL;
  1621. if (ftrace_hash_empty(new_hash))
  1622. new_hash = NULL;
  1623. return __ftrace_hash_update_ipmodify(ops, old_hash, new_hash);
  1624. }
  1625. static void print_ip_ins(const char *fmt, const unsigned char *p)
  1626. {
  1627. char ins[MCOUNT_INSN_SIZE];
  1628. int i;
  1629. if (copy_from_kernel_nofault(ins, p, MCOUNT_INSN_SIZE)) {
  1630. printk(KERN_CONT "%s[FAULT] %px\n", fmt, p);
  1631. return;
  1632. }
  1633. printk(KERN_CONT "%s", fmt);
  1634. for (i = 0; i < MCOUNT_INSN_SIZE; i++)
  1635. printk(KERN_CONT "%s%02x", i ? ":" : "", ins[i]);
  1636. }
  1637. enum ftrace_bug_type ftrace_bug_type;
  1638. const void *ftrace_expected;
  1639. static void print_bug_type(void)
  1640. {
  1641. switch (ftrace_bug_type) {
  1642. case FTRACE_BUG_UNKNOWN:
  1643. break;
  1644. case FTRACE_BUG_INIT:
  1645. pr_info("Initializing ftrace call sites\n");
  1646. break;
  1647. case FTRACE_BUG_NOP:
  1648. pr_info("Setting ftrace call site to NOP\n");
  1649. break;
  1650. case FTRACE_BUG_CALL:
  1651. pr_info("Setting ftrace call site to call ftrace function\n");
  1652. break;
  1653. case FTRACE_BUG_UPDATE:
  1654. pr_info("Updating ftrace call site to call a different ftrace function\n");
  1655. break;
  1656. }
  1657. }
  1658. /**
  1659. * ftrace_bug - report and shutdown function tracer
  1660. * @failed: The failed type (EFAULT, EINVAL, EPERM)
  1661. * @rec: The record that failed
  1662. *
  1663. * The arch code that enables or disables the function tracing
  1664. * can call ftrace_bug() when it has detected a problem in
  1665. * modifying the code. @failed should be one of either:
  1666. * EFAULT - if the problem happens on reading the @ip address
  1667. * EINVAL - if what is read at @ip is not what was expected
  1668. * EPERM - if the problem happens on writing to the @ip address
  1669. */
  1670. void ftrace_bug(int failed, struct dyn_ftrace *rec)
  1671. {
  1672. unsigned long ip = rec ? rec->ip : 0;
  1673. pr_info("------------[ ftrace bug ]------------\n");
  1674. switch (failed) {
  1675. case -EFAULT:
  1676. pr_info("ftrace faulted on modifying ");
  1677. print_ip_sym(KERN_INFO, ip);
  1678. break;
  1679. case -EINVAL:
  1680. pr_info("ftrace failed to modify ");
  1681. print_ip_sym(KERN_INFO, ip);
  1682. print_ip_ins(" actual: ", (unsigned char *)ip);
  1683. pr_cont("\n");
  1684. if (ftrace_expected) {
  1685. print_ip_ins(" expected: ", ftrace_expected);
  1686. pr_cont("\n");
  1687. }
  1688. break;
  1689. case -EPERM:
  1690. pr_info("ftrace faulted on writing ");
  1691. print_ip_sym(KERN_INFO, ip);
  1692. break;
  1693. default:
  1694. pr_info("ftrace faulted on unknown error ");
  1695. print_ip_sym(KERN_INFO, ip);
  1696. }
  1697. print_bug_type();
  1698. if (rec) {
  1699. struct ftrace_ops *ops = NULL;
  1700. pr_info("ftrace record flags: %lx\n", rec->flags);
  1701. pr_cont(" (%ld)%s", ftrace_rec_count(rec),
  1702. rec->flags & FTRACE_FL_REGS ? " R" : " ");
  1703. if (rec->flags & FTRACE_FL_TRAMP_EN) {
  1704. ops = ftrace_find_tramp_ops_any(rec);
  1705. if (ops) {
  1706. do {
  1707. pr_cont("\ttramp: %pS (%pS)",
  1708. (void *)ops->trampoline,
  1709. (void *)ops->func);
  1710. ops = ftrace_find_tramp_ops_next(rec, ops);
  1711. } while (ops);
  1712. } else
  1713. pr_cont("\ttramp: ERROR!");
  1714. }
  1715. ip = ftrace_get_addr_curr(rec);
  1716. pr_cont("\n expected tramp: %lx\n", ip);
  1717. }
  1718. FTRACE_WARN_ON_ONCE(1);
  1719. }
  1720. static int ftrace_check_record(struct dyn_ftrace *rec, bool enable, bool update)
  1721. {
  1722. unsigned long flag = 0UL;
  1723. ftrace_bug_type = FTRACE_BUG_UNKNOWN;
  1724. if (rec->flags & FTRACE_FL_DISABLED)
  1725. return FTRACE_UPDATE_IGNORE;
  1726. /*
  1727. * If we are updating calls:
  1728. *
  1729. * If the record has a ref count, then we need to enable it
  1730. * because someone is using it.
  1731. *
  1732. * Otherwise we make sure its disabled.
  1733. *
  1734. * If we are disabling calls, then disable all records that
  1735. * are enabled.
  1736. */
  1737. if (enable && ftrace_rec_count(rec))
  1738. flag = FTRACE_FL_ENABLED;
  1739. /*
  1740. * If enabling and the REGS flag does not match the REGS_EN, or
  1741. * the TRAMP flag doesn't match the TRAMP_EN, then do not ignore
  1742. * this record. Set flags to fail the compare against ENABLED.
  1743. * Same for direct calls.
  1744. */
  1745. if (flag) {
  1746. if (!(rec->flags & FTRACE_FL_REGS) !=
  1747. !(rec->flags & FTRACE_FL_REGS_EN))
  1748. flag |= FTRACE_FL_REGS;
  1749. if (!(rec->flags & FTRACE_FL_TRAMP) !=
  1750. !(rec->flags & FTRACE_FL_TRAMP_EN))
  1751. flag |= FTRACE_FL_TRAMP;
  1752. /*
  1753. * Direct calls are special, as count matters.
  1754. * We must test the record for direct, if the
  1755. * DIRECT and DIRECT_EN do not match, but only
  1756. * if the count is 1. That's because, if the
  1757. * count is something other than one, we do not
  1758. * want the direct enabled (it will be done via the
  1759. * direct helper). But if DIRECT_EN is set, and
  1760. * the count is not one, we need to clear it.
  1761. */
  1762. if (ftrace_rec_count(rec) == 1) {
  1763. if (!(rec->flags & FTRACE_FL_DIRECT) !=
  1764. !(rec->flags & FTRACE_FL_DIRECT_EN))
  1765. flag |= FTRACE_FL_DIRECT;
  1766. } else if (rec->flags & FTRACE_FL_DIRECT_EN) {
  1767. flag |= FTRACE_FL_DIRECT;
  1768. }
  1769. }
  1770. /* If the state of this record hasn't changed, then do nothing */
  1771. if ((rec->flags & FTRACE_FL_ENABLED) == flag)
  1772. return FTRACE_UPDATE_IGNORE;
  1773. if (flag) {
  1774. /* Save off if rec is being enabled (for return value) */
  1775. flag ^= rec->flags & FTRACE_FL_ENABLED;
  1776. if (update) {
  1777. rec->flags |= FTRACE_FL_ENABLED;
  1778. if (flag & FTRACE_FL_REGS) {
  1779. if (rec->flags & FTRACE_FL_REGS)
  1780. rec->flags |= FTRACE_FL_REGS_EN;
  1781. else
  1782. rec->flags &= ~FTRACE_FL_REGS_EN;
  1783. }
  1784. if (flag & FTRACE_FL_TRAMP) {
  1785. if (rec->flags & FTRACE_FL_TRAMP)
  1786. rec->flags |= FTRACE_FL_TRAMP_EN;
  1787. else
  1788. rec->flags &= ~FTRACE_FL_TRAMP_EN;
  1789. }
  1790. if (flag & FTRACE_FL_DIRECT) {
  1791. /*
  1792. * If there's only one user (direct_ops helper)
  1793. * then we can call the direct function
  1794. * directly (no ftrace trampoline).
  1795. */
  1796. if (ftrace_rec_count(rec) == 1) {
  1797. if (rec->flags & FTRACE_FL_DIRECT)
  1798. rec->flags |= FTRACE_FL_DIRECT_EN;
  1799. else
  1800. rec->flags &= ~FTRACE_FL_DIRECT_EN;
  1801. } else {
  1802. /*
  1803. * Can only call directly if there's
  1804. * only one callback to the function.
  1805. */
  1806. rec->flags &= ~FTRACE_FL_DIRECT_EN;
  1807. }
  1808. }
  1809. }
  1810. /*
  1811. * If this record is being updated from a nop, then
  1812. * return UPDATE_MAKE_CALL.
  1813. * Otherwise,
  1814. * return UPDATE_MODIFY_CALL to tell the caller to convert
  1815. * from the save regs, to a non-save regs function or
  1816. * vice versa, or from a trampoline call.
  1817. */
  1818. if (flag & FTRACE_FL_ENABLED) {
  1819. ftrace_bug_type = FTRACE_BUG_CALL;
  1820. return FTRACE_UPDATE_MAKE_CALL;
  1821. }
  1822. ftrace_bug_type = FTRACE_BUG_UPDATE;
  1823. return FTRACE_UPDATE_MODIFY_CALL;
  1824. }
  1825. if (update) {
  1826. /* If there's no more users, clear all flags */
  1827. if (!ftrace_rec_count(rec))
  1828. rec->flags = 0;
  1829. else
  1830. /*
  1831. * Just disable the record, but keep the ops TRAMP
  1832. * and REGS states. The _EN flags must be disabled though.
  1833. */
  1834. rec->flags &= ~(FTRACE_FL_ENABLED | FTRACE_FL_TRAMP_EN |
  1835. FTRACE_FL_REGS_EN | FTRACE_FL_DIRECT_EN);
  1836. }
  1837. ftrace_bug_type = FTRACE_BUG_NOP;
  1838. return FTRACE_UPDATE_MAKE_NOP;
  1839. }
  1840. /**
  1841. * ftrace_update_record, set a record that now is tracing or not
  1842. * @rec: the record to update
  1843. * @enable: set to true if the record is tracing, false to force disable
  1844. *
  1845. * The records that represent all functions that can be traced need
  1846. * to be updated when tracing has been enabled.
  1847. */
  1848. int ftrace_update_record(struct dyn_ftrace *rec, bool enable)
  1849. {
  1850. return ftrace_check_record(rec, enable, true);
  1851. }
  1852. /**
  1853. * ftrace_test_record, check if the record has been enabled or not
  1854. * @rec: the record to test
  1855. * @enable: set to true to check if enabled, false if it is disabled
  1856. *
  1857. * The arch code may need to test if a record is already set to
  1858. * tracing to determine how to modify the function code that it
  1859. * represents.
  1860. */
  1861. int ftrace_test_record(struct dyn_ftrace *rec, bool enable)
  1862. {
  1863. return ftrace_check_record(rec, enable, false);
  1864. }
  1865. static struct ftrace_ops *
  1866. ftrace_find_tramp_ops_any(struct dyn_ftrace *rec)
  1867. {
  1868. struct ftrace_ops *op;
  1869. unsigned long ip = rec->ip;
  1870. do_for_each_ftrace_op(op, ftrace_ops_list) {
  1871. if (!op->trampoline)
  1872. continue;
  1873. if (hash_contains_ip(ip, op->func_hash))
  1874. return op;
  1875. } while_for_each_ftrace_op(op);
  1876. return NULL;
  1877. }
  1878. static struct ftrace_ops *
  1879. ftrace_find_tramp_ops_any_other(struct dyn_ftrace *rec, struct ftrace_ops *op_exclude)
  1880. {
  1881. struct ftrace_ops *op;
  1882. unsigned long ip = rec->ip;
  1883. do_for_each_ftrace_op(op, ftrace_ops_list) {
  1884. if (op == op_exclude || !op->trampoline)
  1885. continue;
  1886. if (hash_contains_ip(ip, op->func_hash))
  1887. return op;
  1888. } while_for_each_ftrace_op(op);
  1889. return NULL;
  1890. }
  1891. static struct ftrace_ops *
  1892. ftrace_find_tramp_ops_next(struct dyn_ftrace *rec,
  1893. struct ftrace_ops *op)
  1894. {
  1895. unsigned long ip = rec->ip;
  1896. while_for_each_ftrace_op(op) {
  1897. if (!op->trampoline)
  1898. continue;
  1899. if (hash_contains_ip(ip, op->func_hash))
  1900. return op;
  1901. }
  1902. return NULL;
  1903. }
  1904. static struct ftrace_ops *
  1905. ftrace_find_tramp_ops_curr(struct dyn_ftrace *rec)
  1906. {
  1907. struct ftrace_ops *op;
  1908. unsigned long ip = rec->ip;
  1909. /*
  1910. * Need to check removed ops first.
  1911. * If they are being removed, and this rec has a tramp,
  1912. * and this rec is in the ops list, then it would be the
  1913. * one with the tramp.
  1914. */
  1915. if (removed_ops) {
  1916. if (hash_contains_ip(ip, &removed_ops->old_hash))
  1917. return removed_ops;
  1918. }
  1919. /*
  1920. * Need to find the current trampoline for a rec.
  1921. * Now, a trampoline is only attached to a rec if there
  1922. * was a single 'ops' attached to it. But this can be called
  1923. * when we are adding another op to the rec or removing the
  1924. * current one. Thus, if the op is being added, we can
  1925. * ignore it because it hasn't attached itself to the rec
  1926. * yet.
  1927. *
  1928. * If an ops is being modified (hooking to different functions)
  1929. * then we don't care about the new functions that are being
  1930. * added, just the old ones (that are probably being removed).
  1931. *
  1932. * If we are adding an ops to a function that already is using
  1933. * a trampoline, it needs to be removed (trampolines are only
  1934. * for single ops connected), then an ops that is not being
  1935. * modified also needs to be checked.
  1936. */
  1937. do_for_each_ftrace_op(op, ftrace_ops_list) {
  1938. if (!op->trampoline)
  1939. continue;
  1940. /*
  1941. * If the ops is being added, it hasn't gotten to
  1942. * the point to be removed from this tree yet.
  1943. */
  1944. if (op->flags & FTRACE_OPS_FL_ADDING)
  1945. continue;
  1946. /*
  1947. * If the ops is being modified and is in the old
  1948. * hash, then it is probably being removed from this
  1949. * function.
  1950. */
  1951. if ((op->flags & FTRACE_OPS_FL_MODIFYING) &&
  1952. hash_contains_ip(ip, &op->old_hash))
  1953. return op;
  1954. /*
  1955. * If the ops is not being added or modified, and it's
  1956. * in its normal filter hash, then this must be the one
  1957. * we want!
  1958. */
  1959. if (!(op->flags & FTRACE_OPS_FL_MODIFYING) &&
  1960. hash_contains_ip(ip, op->func_hash))
  1961. return op;
  1962. } while_for_each_ftrace_op(op);
  1963. return NULL;
  1964. }
  1965. static struct ftrace_ops *
  1966. ftrace_find_tramp_ops_new(struct dyn_ftrace *rec)
  1967. {
  1968. struct ftrace_ops *op;
  1969. unsigned long ip = rec->ip;
  1970. do_for_each_ftrace_op(op, ftrace_ops_list) {
  1971. /* pass rec in as regs to have non-NULL val */
  1972. if (hash_contains_ip(ip, op->func_hash))
  1973. return op;
  1974. } while_for_each_ftrace_op(op);
  1975. return NULL;
  1976. }
  1977. #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
  1978. /* Protected by rcu_tasks for reading, and direct_mutex for writing */
  1979. static struct ftrace_hash *direct_functions = EMPTY_HASH;
  1980. static DEFINE_MUTEX(direct_mutex);
  1981. int ftrace_direct_func_count;
  1982. /*
  1983. * Search the direct_functions hash to see if the given instruction pointer
  1984. * has a direct caller attached to it.
  1985. */
  1986. unsigned long ftrace_find_rec_direct(unsigned long ip)
  1987. {
  1988. struct ftrace_func_entry *entry;
  1989. entry = __ftrace_lookup_ip(direct_functions, ip);
  1990. if (!entry)
  1991. return 0;
  1992. return entry->direct;
  1993. }
  1994. static void call_direct_funcs(unsigned long ip, unsigned long pip,
  1995. struct ftrace_ops *ops, struct pt_regs *regs)
  1996. {
  1997. unsigned long addr;
  1998. addr = ftrace_find_rec_direct(ip);
  1999. if (!addr)
  2000. return;
  2001. arch_ftrace_set_direct_caller(regs, addr);
  2002. }
  2003. struct ftrace_ops direct_ops = {
  2004. .func = call_direct_funcs,
  2005. .flags = FTRACE_OPS_FL_IPMODIFY | FTRACE_OPS_FL_RECURSION_SAFE
  2006. | FTRACE_OPS_FL_DIRECT | FTRACE_OPS_FL_SAVE_REGS
  2007. | FTRACE_OPS_FL_PERMANENT,
  2008. /*
  2009. * By declaring the main trampoline as this trampoline
  2010. * it will never have one allocated for it. Allocated
  2011. * trampolines should not call direct functions.
  2012. * The direct_ops should only be called by the builtin
  2013. * ftrace_regs_caller trampoline.
  2014. */
  2015. .trampoline = FTRACE_REGS_ADDR,
  2016. };
  2017. #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
  2018. /**
  2019. * ftrace_get_addr_new - Get the call address to set to
  2020. * @rec: The ftrace record descriptor
  2021. *
  2022. * If the record has the FTRACE_FL_REGS set, that means that it
  2023. * wants to convert to a callback that saves all regs. If FTRACE_FL_REGS
  2024. * is not set, then it wants to convert to the normal callback.
  2025. *
  2026. * Returns the address of the trampoline to set to
  2027. */
  2028. unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec)
  2029. {
  2030. struct ftrace_ops *ops;
  2031. unsigned long addr;
  2032. if ((rec->flags & FTRACE_FL_DIRECT) &&
  2033. (ftrace_rec_count(rec) == 1)) {
  2034. addr = ftrace_find_rec_direct(rec->ip);
  2035. if (addr)
  2036. return addr;
  2037. WARN_ON_ONCE(1);
  2038. }
  2039. /* Trampolines take precedence over regs */
  2040. if (rec->flags & FTRACE_FL_TRAMP) {
  2041. ops = ftrace_find_tramp_ops_new(rec);
  2042. if (FTRACE_WARN_ON(!ops || !ops->trampoline)) {
  2043. pr_warn("Bad trampoline accounting at: %p (%pS) (%lx)\n",
  2044. (void *)rec->ip, (void *)rec->ip, rec->flags);
  2045. /* Ftrace is shutting down, return anything */
  2046. return (unsigned long)FTRACE_ADDR;
  2047. }
  2048. return ops->trampoline;
  2049. }
  2050. if (rec->flags & FTRACE_FL_REGS)
  2051. return (unsigned long)FTRACE_REGS_ADDR;
  2052. else
  2053. return (unsigned long)FTRACE_ADDR;
  2054. }
  2055. /**
  2056. * ftrace_get_addr_curr - Get the call address that is already there
  2057. * @rec: The ftrace record descriptor
  2058. *
  2059. * The FTRACE_FL_REGS_EN is set when the record already points to
  2060. * a function that saves all the regs. Basically the '_EN' version
  2061. * represents the current state of the function.
  2062. *
  2063. * Returns the address of the trampoline that is currently being called
  2064. */
  2065. unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec)
  2066. {
  2067. struct ftrace_ops *ops;
  2068. unsigned long addr;
  2069. /* Direct calls take precedence over trampolines */
  2070. if (rec->flags & FTRACE_FL_DIRECT_EN) {
  2071. addr = ftrace_find_rec_direct(rec->ip);
  2072. if (addr)
  2073. return addr;
  2074. WARN_ON_ONCE(1);
  2075. }
  2076. /* Trampolines take precedence over regs */
  2077. if (rec->flags & FTRACE_FL_TRAMP_EN) {
  2078. ops = ftrace_find_tramp_ops_curr(rec);
  2079. if (FTRACE_WARN_ON(!ops)) {
  2080. pr_warn("Bad trampoline accounting at: %p (%pS)\n",
  2081. (void *)rec->ip, (void *)rec->ip);
  2082. /* Ftrace is shutting down, return anything */
  2083. return (unsigned long)FTRACE_ADDR;
  2084. }
  2085. return ops->trampoline;
  2086. }
  2087. if (rec->flags & FTRACE_FL_REGS_EN)
  2088. return (unsigned long)FTRACE_REGS_ADDR;
  2089. else
  2090. return (unsigned long)FTRACE_ADDR;
  2091. }
  2092. static int
  2093. __ftrace_replace_code(struct dyn_ftrace *rec, bool enable)
  2094. {
  2095. unsigned long ftrace_old_addr;
  2096. unsigned long ftrace_addr;
  2097. int ret;
  2098. ftrace_addr = ftrace_get_addr_new(rec);
  2099. /* This needs to be done before we call ftrace_update_record */
  2100. ftrace_old_addr = ftrace_get_addr_curr(rec);
  2101. ret = ftrace_update_record(rec, enable);
  2102. ftrace_bug_type = FTRACE_BUG_UNKNOWN;
  2103. switch (ret) {
  2104. case FTRACE_UPDATE_IGNORE:
  2105. return 0;
  2106. case FTRACE_UPDATE_MAKE_CALL:
  2107. ftrace_bug_type = FTRACE_BUG_CALL;
  2108. return ftrace_make_call(rec, ftrace_addr);
  2109. case FTRACE_UPDATE_MAKE_NOP:
  2110. ftrace_bug_type = FTRACE_BUG_NOP;
  2111. return ftrace_make_nop(NULL, rec, ftrace_old_addr);
  2112. case FTRACE_UPDATE_MODIFY_CALL:
  2113. ftrace_bug_type = FTRACE_BUG_UPDATE;
  2114. return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr);
  2115. }
  2116. return -1; /* unknown ftrace bug */
  2117. }
  2118. void __weak ftrace_replace_code(int mod_flags)
  2119. {
  2120. struct dyn_ftrace *rec;
  2121. struct ftrace_page *pg;
  2122. bool enable = mod_flags & FTRACE_MODIFY_ENABLE_FL;
  2123. int schedulable = mod_flags & FTRACE_MODIFY_MAY_SLEEP_FL;
  2124. int failed;
  2125. if (unlikely(ftrace_disabled))
  2126. return;
  2127. do_for_each_ftrace_rec(pg, rec) {
  2128. if (rec->flags & FTRACE_FL_DISABLED)
  2129. continue;
  2130. failed = __ftrace_replace_code(rec, enable);
  2131. if (failed) {
  2132. ftrace_bug(failed, rec);
  2133. /* Stop processing */
  2134. return;
  2135. }
  2136. if (schedulable)
  2137. cond_resched();
  2138. } while_for_each_ftrace_rec();
  2139. }
  2140. struct ftrace_rec_iter {
  2141. struct ftrace_page *pg;
  2142. int index;
  2143. };
  2144. /**
  2145. * ftrace_rec_iter_start, start up iterating over traced functions
  2146. *
  2147. * Returns an iterator handle that is used to iterate over all
  2148. * the records that represent address locations where functions
  2149. * are traced.
  2150. *
  2151. * May return NULL if no records are available.
  2152. */
  2153. struct ftrace_rec_iter *ftrace_rec_iter_start(void)
  2154. {
  2155. /*
  2156. * We only use a single iterator.
  2157. * Protected by the ftrace_lock mutex.
  2158. */
  2159. static struct ftrace_rec_iter ftrace_rec_iter;
  2160. struct ftrace_rec_iter *iter = &ftrace_rec_iter;
  2161. iter->pg = ftrace_pages_start;
  2162. iter->index = 0;
  2163. /* Could have empty pages */
  2164. while (iter->pg && !iter->pg->index)
  2165. iter->pg = iter->pg->next;
  2166. if (!iter->pg)
  2167. return NULL;
  2168. return iter;
  2169. }
  2170. /**
  2171. * ftrace_rec_iter_next, get the next record to process.
  2172. * @iter: The handle to the iterator.
  2173. *
  2174. * Returns the next iterator after the given iterator @iter.
  2175. */
  2176. struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter)
  2177. {
  2178. iter->index++;
  2179. if (iter->index >= iter->pg->index) {
  2180. iter->pg = iter->pg->next;
  2181. iter->index = 0;
  2182. /* Could have empty pages */
  2183. while (iter->pg && !iter->pg->index)
  2184. iter->pg = iter->pg->next;
  2185. }
  2186. if (!iter->pg)
  2187. return NULL;
  2188. return iter;
  2189. }
  2190. /**
  2191. * ftrace_rec_iter_record, get the record at the iterator location
  2192. * @iter: The current iterator location
  2193. *
  2194. * Returns the record that the current @iter is at.
  2195. */
  2196. struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter)
  2197. {
  2198. return &iter->pg->records[iter->index];
  2199. }
  2200. static int
  2201. ftrace_nop_initialize(struct module *mod, struct dyn_ftrace *rec)
  2202. {
  2203. int ret;
  2204. if (unlikely(ftrace_disabled))
  2205. return 0;
  2206. ret = ftrace_init_nop(mod, rec);
  2207. if (ret) {
  2208. ftrace_bug_type = FTRACE_BUG_INIT;
  2209. ftrace_bug(ret, rec);
  2210. return 0;
  2211. }
  2212. return 1;
  2213. }
  2214. /*
  2215. * archs can override this function if they must do something
  2216. * before the modifying code is performed.
  2217. */
  2218. int __weak ftrace_arch_code_modify_prepare(void)
  2219. {
  2220. return 0;
  2221. }
  2222. /*
  2223. * archs can override this function if they must do something
  2224. * after the modifying code is performed.
  2225. */
  2226. int __weak ftrace_arch_code_modify_post_process(void)
  2227. {
  2228. return 0;
  2229. }
  2230. void ftrace_modify_all_code(int command)
  2231. {
  2232. int update = command & FTRACE_UPDATE_TRACE_FUNC;
  2233. int mod_flags = 0;
  2234. int err = 0;
  2235. if (command & FTRACE_MAY_SLEEP)
  2236. mod_flags = FTRACE_MODIFY_MAY_SLEEP_FL;
  2237. /*
  2238. * If the ftrace_caller calls a ftrace_ops func directly,
  2239. * we need to make sure that it only traces functions it
  2240. * expects to trace. When doing the switch of functions,
  2241. * we need to update to the ftrace_ops_list_func first
  2242. * before the transition between old and new calls are set,
  2243. * as the ftrace_ops_list_func will check the ops hashes
  2244. * to make sure the ops are having the right functions
  2245. * traced.
  2246. */
  2247. if (update) {
  2248. err = ftrace_update_ftrace_func(ftrace_ops_list_func);
  2249. if (FTRACE_WARN_ON(err))
  2250. return;
  2251. }
  2252. if (command & FTRACE_UPDATE_CALLS)
  2253. ftrace_replace_code(mod_flags | FTRACE_MODIFY_ENABLE_FL);
  2254. else if (command & FTRACE_DISABLE_CALLS)
  2255. ftrace_replace_code(mod_flags);
  2256. if (update && ftrace_trace_function != ftrace_ops_list_func) {
  2257. function_trace_op = set_function_trace_op;
  2258. smp_wmb();
  2259. /* If irqs are disabled, we are in stop machine */
  2260. if (!irqs_disabled())
  2261. smp_call_function(ftrace_sync_ipi, NULL, 1);
  2262. err = ftrace_update_ftrace_func(ftrace_trace_function);
  2263. if (FTRACE_WARN_ON(err))
  2264. return;
  2265. }
  2266. if (command & FTRACE_START_FUNC_RET)
  2267. err = ftrace_enable_ftrace_graph_caller();
  2268. else if (command & FTRACE_STOP_FUNC_RET)
  2269. err = ftrace_disable_ftrace_graph_caller();
  2270. FTRACE_WARN_ON(err);
  2271. }
  2272. static int __ftrace_modify_code(void *data)
  2273. {
  2274. int *command = data;
  2275. ftrace_modify_all_code(*command);
  2276. return 0;
  2277. }
  2278. /**
  2279. * ftrace_run_stop_machine, go back to the stop machine method
  2280. * @command: The command to tell ftrace what to do
  2281. *
  2282. * If an arch needs to fall back to the stop machine method, the
  2283. * it can call this function.
  2284. */
  2285. void ftrace_run_stop_machine(int command)
  2286. {
  2287. stop_machine(__ftrace_modify_code, &command, NULL);
  2288. }
  2289. /**
  2290. * arch_ftrace_update_code, modify the code to trace or not trace
  2291. * @command: The command that needs to be done
  2292. *
  2293. * Archs can override this function if it does not need to
  2294. * run stop_machine() to modify code.
  2295. */
  2296. void __weak arch_ftrace_update_code(int command)
  2297. {
  2298. ftrace_run_stop_machine(command);
  2299. }
  2300. static void ftrace_run_update_code(int command)
  2301. {
  2302. int ret;
  2303. ret = ftrace_arch_code_modify_prepare();
  2304. FTRACE_WARN_ON(ret);
  2305. if (ret)
  2306. return;
  2307. /*
  2308. * By default we use stop_machine() to modify the code.
  2309. * But archs can do what ever they want as long as it
  2310. * is safe. The stop_machine() is the safest, but also
  2311. * produces the most overhead.
  2312. */
  2313. arch_ftrace_update_code(command);
  2314. ret = ftrace_arch_code_modify_post_process();
  2315. FTRACE_WARN_ON(ret);
  2316. }
  2317. static void ftrace_run_modify_code(struct ftrace_ops *ops, int command,
  2318. struct ftrace_ops_hash *old_hash)
  2319. {
  2320. ops->flags |= FTRACE_OPS_FL_MODIFYING;
  2321. ops->old_hash.filter_hash = old_hash->filter_hash;
  2322. ops->old_hash.notrace_hash = old_hash->notrace_hash;
  2323. ftrace_run_update_code(command);
  2324. ops->old_hash.filter_hash = NULL;
  2325. ops->old_hash.notrace_hash = NULL;
  2326. ops->flags &= ~FTRACE_OPS_FL_MODIFYING;
  2327. }
  2328. static ftrace_func_t saved_ftrace_func;
  2329. static int ftrace_start_up;
  2330. void __weak arch_ftrace_trampoline_free(struct ftrace_ops *ops)
  2331. {
  2332. }
  2333. /* List of trace_ops that have allocated trampolines */
  2334. static LIST_HEAD(ftrace_ops_trampoline_list);
  2335. static void ftrace_add_trampoline_to_kallsyms(struct ftrace_ops *ops)
  2336. {
  2337. lockdep_assert_held(&ftrace_lock);
  2338. list_add_rcu(&ops->list, &ftrace_ops_trampoline_list);
  2339. }
  2340. static void ftrace_remove_trampoline_from_kallsyms(struct ftrace_ops *ops)
  2341. {
  2342. lockdep_assert_held(&ftrace_lock);
  2343. list_del_rcu(&ops->list);
  2344. synchronize_rcu();
  2345. }
  2346. /*
  2347. * "__builtin__ftrace" is used as a module name in /proc/kallsyms for symbols
  2348. * for pages allocated for ftrace purposes, even though "__builtin__ftrace" is
  2349. * not a module.
  2350. */
  2351. #define FTRACE_TRAMPOLINE_MOD "__builtin__ftrace"
  2352. #define FTRACE_TRAMPOLINE_SYM "ftrace_trampoline"
  2353. static void ftrace_trampoline_free(struct ftrace_ops *ops)
  2354. {
  2355. if (ops && (ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP) &&
  2356. ops->trampoline) {
  2357. /*
  2358. * Record the text poke event before the ksymbol unregister
  2359. * event.
  2360. */
  2361. perf_event_text_poke((void *)ops->trampoline,
  2362. (void *)ops->trampoline,
  2363. ops->trampoline_size, NULL, 0);
  2364. perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL,
  2365. ops->trampoline, ops->trampoline_size,
  2366. true, FTRACE_TRAMPOLINE_SYM);
  2367. /* Remove from kallsyms after the perf events */
  2368. ftrace_remove_trampoline_from_kallsyms(ops);
  2369. }
  2370. arch_ftrace_trampoline_free(ops);
  2371. }
  2372. static void ftrace_startup_enable(int command)
  2373. {
  2374. if (saved_ftrace_func != ftrace_trace_function) {
  2375. saved_ftrace_func = ftrace_trace_function;
  2376. command |= FTRACE_UPDATE_TRACE_FUNC;
  2377. }
  2378. if (!command || !ftrace_enabled)
  2379. return;
  2380. ftrace_run_update_code(command);
  2381. }
  2382. static void ftrace_startup_all(int command)
  2383. {
  2384. update_all_ops = true;
  2385. ftrace_startup_enable(command);
  2386. update_all_ops = false;
  2387. }
  2388. int ftrace_startup(struct ftrace_ops *ops, int command)
  2389. {
  2390. int ret;
  2391. if (unlikely(ftrace_disabled))
  2392. return -ENODEV;
  2393. ret = __register_ftrace_function(ops);
  2394. if (ret)
  2395. return ret;
  2396. ftrace_start_up++;
  2397. /*
  2398. * Note that ftrace probes uses this to start up
  2399. * and modify functions it will probe. But we still
  2400. * set the ADDING flag for modification, as probes
  2401. * do not have trampolines. If they add them in the
  2402. * future, then the probes will need to distinguish
  2403. * between adding and updating probes.
  2404. */
  2405. ops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_ADDING;
  2406. ret = ftrace_hash_ipmodify_enable(ops);
  2407. if (ret < 0) {
  2408. /* Rollback registration process */
  2409. __unregister_ftrace_function(ops);
  2410. ftrace_start_up--;
  2411. ops->flags &= ~FTRACE_OPS_FL_ENABLED;
  2412. if (ops->flags & FTRACE_OPS_FL_DYNAMIC)
  2413. ftrace_trampoline_free(ops);
  2414. return ret;
  2415. }
  2416. if (ftrace_hash_rec_enable(ops, 1))
  2417. command |= FTRACE_UPDATE_CALLS;
  2418. ftrace_startup_enable(command);
  2419. ops->flags &= ~FTRACE_OPS_FL_ADDING;
  2420. return 0;
  2421. }
  2422. int ftrace_shutdown(struct ftrace_ops *ops, int command)
  2423. {
  2424. int ret;
  2425. if (unlikely(ftrace_disabled))
  2426. return -ENODEV;
  2427. ret = __unregister_ftrace_function(ops);
  2428. if (ret)
  2429. return ret;
  2430. ftrace_start_up--;
  2431. /*
  2432. * Just warn in case of unbalance, no need to kill ftrace, it's not
  2433. * critical but the ftrace_call callers may be never nopped again after
  2434. * further ftrace uses.
  2435. */
  2436. WARN_ON_ONCE(ftrace_start_up < 0);
  2437. /* Disabling ipmodify never fails */
  2438. ftrace_hash_ipmodify_disable(ops);
  2439. if (ftrace_hash_rec_disable(ops, 1))
  2440. command |= FTRACE_UPDATE_CALLS;
  2441. ops->flags &= ~FTRACE_OPS_FL_ENABLED;
  2442. if (saved_ftrace_func != ftrace_trace_function) {
  2443. saved_ftrace_func = ftrace_trace_function;
  2444. command |= FTRACE_UPDATE_TRACE_FUNC;
  2445. }
  2446. if (!command || !ftrace_enabled) {
  2447. /*
  2448. * If these are dynamic or per_cpu ops, they still
  2449. * need their data freed. Since, function tracing is
  2450. * not currently active, we can just free them
  2451. * without synchronizing all CPUs.
  2452. */
  2453. if (ops->flags & FTRACE_OPS_FL_DYNAMIC)
  2454. goto free_ops;
  2455. return 0;
  2456. }
  2457. /*
  2458. * If the ops uses a trampoline, then it needs to be
  2459. * tested first on update.
  2460. */
  2461. ops->flags |= FTRACE_OPS_FL_REMOVING;
  2462. removed_ops = ops;
  2463. /* The trampoline logic checks the old hashes */
  2464. ops->old_hash.filter_hash = ops->func_hash->filter_hash;
  2465. ops->old_hash.notrace_hash = ops->func_hash->notrace_hash;
  2466. ftrace_run_update_code(command);
  2467. /*
  2468. * If there's no more ops registered with ftrace, run a
  2469. * sanity check to make sure all rec flags are cleared.
  2470. */
  2471. if (rcu_dereference_protected(ftrace_ops_list,
  2472. lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {
  2473. struct ftrace_page *pg;
  2474. struct dyn_ftrace *rec;
  2475. do_for_each_ftrace_rec(pg, rec) {
  2476. if (FTRACE_WARN_ON_ONCE(rec->flags & ~FTRACE_FL_DISABLED))
  2477. pr_warn(" %pS flags:%lx\n",
  2478. (void *)rec->ip, rec->flags);
  2479. } while_for_each_ftrace_rec();
  2480. }
  2481. ops->old_hash.filter_hash = NULL;
  2482. ops->old_hash.notrace_hash = NULL;
  2483. removed_ops = NULL;
  2484. ops->flags &= ~FTRACE_OPS_FL_REMOVING;
  2485. /*
  2486. * Dynamic ops may be freed, we must make sure that all
  2487. * callers are done before leaving this function.
  2488. * The same goes for freeing the per_cpu data of the per_cpu
  2489. * ops.
  2490. */
  2491. if (ops->flags & FTRACE_OPS_FL_DYNAMIC) {
  2492. /*
  2493. * We need to do a hard force of sched synchronization.
  2494. * This is because we use preempt_disable() to do RCU, but
  2495. * the function tracers can be called where RCU is not watching
  2496. * (like before user_exit()). We can not rely on the RCU
  2497. * infrastructure to do the synchronization, thus we must do it
  2498. * ourselves.
  2499. */
  2500. synchronize_rcu_tasks_rude();
  2501. /*
  2502. * When the kernel is preemptive, tasks can be preempted
  2503. * while on a ftrace trampoline. Just scheduling a task on
  2504. * a CPU is not good enough to flush them. Calling
  2505. * synchornize_rcu_tasks() will wait for those tasks to
  2506. * execute and either schedule voluntarily or enter user space.
  2507. */
  2508. if (IS_ENABLED(CONFIG_PREEMPTION))
  2509. synchronize_rcu_tasks();
  2510. free_ops:
  2511. ftrace_trampoline_free(ops);
  2512. }
  2513. return 0;
  2514. }
  2515. static void ftrace_startup_sysctl(void)
  2516. {
  2517. int command;
  2518. if (unlikely(ftrace_disabled))
  2519. return;
  2520. /* Force update next time */
  2521. saved_ftrace_func = NULL;
  2522. /* ftrace_start_up is true if we want ftrace running */
  2523. if (ftrace_start_up) {
  2524. command = FTRACE_UPDATE_CALLS;
  2525. if (ftrace_graph_active)
  2526. command |= FTRACE_START_FUNC_RET;
  2527. ftrace_startup_enable(command);
  2528. }
  2529. }
  2530. static void ftrace_shutdown_sysctl(void)
  2531. {
  2532. int command;
  2533. if (unlikely(ftrace_disabled))
  2534. return;
  2535. /* ftrace_start_up is true if ftrace is running */
  2536. if (ftrace_start_up) {
  2537. command = FTRACE_DISABLE_CALLS;
  2538. if (ftrace_graph_active)
  2539. command |= FTRACE_STOP_FUNC_RET;
  2540. ftrace_run_update_code(command);
  2541. }
  2542. }
  2543. static u64 ftrace_update_time;
  2544. unsigned long ftrace_update_tot_cnt;
  2545. unsigned long ftrace_number_of_pages;
  2546. unsigned long ftrace_number_of_groups;
  2547. static inline int ops_traces_mod(struct ftrace_ops *ops)
  2548. {
  2549. /*
  2550. * Filter_hash being empty will default to trace module.
  2551. * But notrace hash requires a test of individual module functions.
  2552. */
  2553. return ftrace_hash_empty(ops->func_hash->filter_hash) &&
  2554. ftrace_hash_empty(ops->func_hash->notrace_hash);
  2555. }
  2556. /*
  2557. * Check if the current ops references the record.
  2558. *
  2559. * If the ops traces all functions, then it was already accounted for.
  2560. * If the ops does not trace the current record function, skip it.
  2561. * If the ops ignores the function via notrace filter, skip it.
  2562. */
  2563. static inline bool
  2564. ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec)
  2565. {
  2566. /* If ops isn't enabled, ignore it */
  2567. if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
  2568. return false;
  2569. /* If ops traces all then it includes this function */
  2570. if (ops_traces_mod(ops))
  2571. return true;
  2572. /* The function must be in the filter */
  2573. if (!ftrace_hash_empty(ops->func_hash->filter_hash) &&
  2574. !__ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip))
  2575. return false;
  2576. /* If in notrace hash, we ignore it too */
  2577. if (ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip))
  2578. return false;
  2579. return true;
  2580. }
  2581. static int ftrace_update_code(struct module *mod, struct ftrace_page *new_pgs)
  2582. {
  2583. struct ftrace_page *pg;
  2584. struct dyn_ftrace *p;
  2585. u64 start, stop;
  2586. unsigned long update_cnt = 0;
  2587. unsigned long rec_flags = 0;
  2588. int i;
  2589. start = ftrace_now(raw_smp_processor_id());
  2590. /*
  2591. * When a module is loaded, this function is called to convert
  2592. * the calls to mcount in its text to nops, and also to create
  2593. * an entry in the ftrace data. Now, if ftrace is activated
  2594. * after this call, but before the module sets its text to
  2595. * read-only, the modification of enabling ftrace can fail if
  2596. * the read-only is done while ftrace is converting the calls.
  2597. * To prevent this, the module's records are set as disabled
  2598. * and will be enabled after the call to set the module's text
  2599. * to read-only.
  2600. */
  2601. if (mod)
  2602. rec_flags |= FTRACE_FL_DISABLED;
  2603. for (pg = new_pgs; pg; pg = pg->next) {
  2604. for (i = 0; i < pg->index; i++) {
  2605. /* If something went wrong, bail without enabling anything */
  2606. if (unlikely(ftrace_disabled))
  2607. return -1;
  2608. p = &pg->records[i];
  2609. p->flags = rec_flags;
  2610. /*
  2611. * Do the initial record conversion from mcount jump
  2612. * to the NOP instructions.
  2613. */
  2614. if (!__is_defined(CC_USING_NOP_MCOUNT) &&
  2615. !ftrace_nop_initialize(mod, p))
  2616. break;
  2617. update_cnt++;
  2618. }
  2619. }
  2620. stop = ftrace_now(raw_smp_processor_id());
  2621. ftrace_update_time = stop - start;
  2622. ftrace_update_tot_cnt += update_cnt;
  2623. return 0;
  2624. }
  2625. static int ftrace_allocate_records(struct ftrace_page *pg, int count)
  2626. {
  2627. int order;
  2628. int pages;
  2629. int cnt;
  2630. if (WARN_ON(!count))
  2631. return -EINVAL;
  2632. pages = DIV_ROUND_UP(count, ENTRIES_PER_PAGE);
  2633. order = get_count_order(pages);
  2634. /*
  2635. * We want to fill as much as possible. No more than a page
  2636. * may be empty.
  2637. */
  2638. if (!is_power_of_2(pages))
  2639. order--;
  2640. again:
  2641. pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
  2642. if (!pg->records) {
  2643. /* if we can't allocate this size, try something smaller */
  2644. if (!order)
  2645. return -ENOMEM;
  2646. order >>= 1;
  2647. goto again;
  2648. }
  2649. ftrace_number_of_pages += 1 << order;
  2650. ftrace_number_of_groups++;
  2651. cnt = (PAGE_SIZE << order) / ENTRY_SIZE;
  2652. pg->size = cnt;
  2653. if (cnt > count)
  2654. cnt = count;
  2655. return cnt;
  2656. }
  2657. static struct ftrace_page *
  2658. ftrace_allocate_pages(unsigned long num_to_init)
  2659. {
  2660. struct ftrace_page *start_pg;
  2661. struct ftrace_page *pg;
  2662. int order;
  2663. int cnt;
  2664. if (!num_to_init)
  2665. return NULL;
  2666. start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL);
  2667. if (!pg)
  2668. return NULL;
  2669. /*
  2670. * Try to allocate as much as possible in one continues
  2671. * location that fills in all of the space. We want to
  2672. * waste as little space as possible.
  2673. */
  2674. for (;;) {
  2675. cnt = ftrace_allocate_records(pg, num_to_init);
  2676. if (cnt < 0)
  2677. goto free_pages;
  2678. num_to_init -= cnt;
  2679. if (!num_to_init)
  2680. break;
  2681. pg->next = kzalloc(sizeof(*pg), GFP_KERNEL);
  2682. if (!pg->next)
  2683. goto free_pages;
  2684. pg = pg->next;
  2685. }
  2686. return start_pg;
  2687. free_pages:
  2688. pg = start_pg;
  2689. while (pg) {
  2690. order = get_count_order(pg->size / ENTRIES_PER_PAGE);
  2691. if (order >= 0)
  2692. free_pages((unsigned long)pg->records, order);
  2693. start_pg = pg->next;
  2694. kfree(pg);
  2695. pg = start_pg;
  2696. ftrace_number_of_pages -= 1 << order;
  2697. ftrace_number_of_groups--;
  2698. }
  2699. pr_info("ftrace: FAILED to allocate memory for functions\n");
  2700. return NULL;
  2701. }
  2702. #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
  2703. struct ftrace_iterator {
  2704. loff_t pos;
  2705. loff_t func_pos;
  2706. loff_t mod_pos;
  2707. struct ftrace_page *pg;
  2708. struct dyn_ftrace *func;
  2709. struct ftrace_func_probe *probe;
  2710. struct ftrace_func_entry *probe_entry;
  2711. struct trace_parser parser;
  2712. struct ftrace_hash *hash;
  2713. struct ftrace_ops *ops;
  2714. struct trace_array *tr;
  2715. struct list_head *mod_list;
  2716. int pidx;
  2717. int idx;
  2718. unsigned flags;
  2719. };
  2720. static void *
  2721. t_probe_next(struct seq_file *m, loff_t *pos)
  2722. {
  2723. struct ftrace_iterator *iter = m->private;
  2724. struct trace_array *tr = iter->ops->private;
  2725. struct list_head *func_probes;
  2726. struct ftrace_hash *hash;
  2727. struct list_head *next;
  2728. struct hlist_node *hnd = NULL;
  2729. struct hlist_head *hhd;
  2730. int size;
  2731. (*pos)++;
  2732. iter->pos = *pos;
  2733. if (!tr)
  2734. return NULL;
  2735. func_probes = &tr->func_probes;
  2736. if (list_empty(func_probes))
  2737. return NULL;
  2738. if (!iter->probe) {
  2739. next = func_probes->next;
  2740. iter->probe = list_entry(next, struct ftrace_func_probe, list);
  2741. }
  2742. if (iter->probe_entry)
  2743. hnd = &iter->probe_entry->hlist;
  2744. hash = iter->probe->ops.func_hash->filter_hash;
  2745. /*
  2746. * A probe being registered may temporarily have an empty hash
  2747. * and it's at the end of the func_probes list.
  2748. */
  2749. if (!hash || hash == EMPTY_HASH)
  2750. return NULL;
  2751. size = 1 << hash->size_bits;
  2752. retry:
  2753. if (iter->pidx >= size) {
  2754. if (iter->probe->list.next == func_probes)
  2755. return NULL;
  2756. next = iter->probe->list.next;
  2757. iter->probe = list_entry(next, struct ftrace_func_probe, list);
  2758. hash = iter->probe->ops.func_hash->filter_hash;
  2759. size = 1 << hash->size_bits;
  2760. iter->pidx = 0;
  2761. }
  2762. hhd = &hash->buckets[iter->pidx];
  2763. if (hlist_empty(hhd)) {
  2764. iter->pidx++;
  2765. hnd = NULL;
  2766. goto retry;
  2767. }
  2768. if (!hnd)
  2769. hnd = hhd->first;
  2770. else {
  2771. hnd = hnd->next;
  2772. if (!hnd) {
  2773. iter->pidx++;
  2774. goto retry;
  2775. }
  2776. }
  2777. if (WARN_ON_ONCE(!hnd))
  2778. return NULL;
  2779. iter->probe_entry = hlist_entry(hnd, struct ftrace_func_entry, hlist);
  2780. return iter;
  2781. }
  2782. static void *t_probe_start(struct seq_file *m, loff_t *pos)
  2783. {
  2784. struct ftrace_iterator *iter = m->private;
  2785. void *p = NULL;
  2786. loff_t l;
  2787. if (!(iter->flags & FTRACE_ITER_DO_PROBES))
  2788. return NULL;
  2789. if (iter->mod_pos > *pos)
  2790. return NULL;
  2791. iter->probe = NULL;
  2792. iter->probe_entry = NULL;
  2793. iter->pidx = 0;
  2794. for (l = 0; l <= (*pos - iter->mod_pos); ) {
  2795. p = t_probe_next(m, &l);
  2796. if (!p)
  2797. break;
  2798. }
  2799. if (!p)
  2800. return NULL;
  2801. /* Only set this if we have an item */
  2802. iter->flags |= FTRACE_ITER_PROBE;
  2803. return iter;
  2804. }
  2805. static int
  2806. t_probe_show(struct seq_file *m, struct ftrace_iterator *iter)
  2807. {
  2808. struct ftrace_func_entry *probe_entry;
  2809. struct ftrace_probe_ops *probe_ops;
  2810. struct ftrace_func_probe *probe;
  2811. probe = iter->probe;
  2812. probe_entry = iter->probe_entry;
  2813. if (WARN_ON_ONCE(!probe || !probe_entry))
  2814. return -EIO;
  2815. probe_ops = probe->probe_ops;
  2816. if (probe_ops->print)
  2817. return probe_ops->print(m, probe_entry->ip, probe_ops, probe->data);
  2818. seq_printf(m, "%ps:%ps\n", (void *)probe_entry->ip,
  2819. (void *)probe_ops->func);
  2820. return 0;
  2821. }
  2822. static void *
  2823. t_mod_next(struct seq_file *m, loff_t *pos)
  2824. {
  2825. struct ftrace_iterator *iter = m->private;
  2826. struct trace_array *tr = iter->tr;
  2827. (*pos)++;
  2828. iter->pos = *pos;
  2829. iter->mod_list = iter->mod_list->next;
  2830. if (iter->mod_list == &tr->mod_trace ||
  2831. iter->mod_list == &tr->mod_notrace) {
  2832. iter->flags &= ~FTRACE_ITER_MOD;
  2833. return NULL;
  2834. }
  2835. iter->mod_pos = *pos;
  2836. return iter;
  2837. }
  2838. static void *t_mod_start(struct seq_file *m, loff_t *pos)
  2839. {
  2840. struct ftrace_iterator *iter = m->private;
  2841. void *p = NULL;
  2842. loff_t l;
  2843. if (iter->func_pos > *pos)
  2844. return NULL;
  2845. iter->mod_pos = iter->func_pos;
  2846. /* probes are only available if tr is set */
  2847. if (!iter->tr)
  2848. return NULL;
  2849. for (l = 0; l <= (*pos - iter->func_pos); ) {
  2850. p = t_mod_next(m, &l);
  2851. if (!p)
  2852. break;
  2853. }
  2854. if (!p) {
  2855. iter->flags &= ~FTRACE_ITER_MOD;
  2856. return t_probe_start(m, pos);
  2857. }
  2858. /* Only set this if we have an item */
  2859. iter->flags |= FTRACE_ITER_MOD;
  2860. return iter;
  2861. }
  2862. static int
  2863. t_mod_show(struct seq_file *m, struct ftrace_iterator *iter)
  2864. {
  2865. struct ftrace_mod_load *ftrace_mod;
  2866. struct trace_array *tr = iter->tr;
  2867. if (WARN_ON_ONCE(!iter->mod_list) ||
  2868. iter->mod_list == &tr->mod_trace ||
  2869. iter->mod_list == &tr->mod_notrace)
  2870. return -EIO;
  2871. ftrace_mod = list_entry(iter->mod_list, struct ftrace_mod_load, list);
  2872. if (ftrace_mod->func)
  2873. seq_printf(m, "%s", ftrace_mod->func);
  2874. else
  2875. seq_putc(m, '*');
  2876. seq_printf(m, ":mod:%s\n", ftrace_mod->module);
  2877. return 0;
  2878. }
  2879. static void *
  2880. t_func_next(struct seq_file *m, loff_t *pos)
  2881. {
  2882. struct ftrace_iterator *iter = m->private;
  2883. struct dyn_ftrace *rec = NULL;
  2884. (*pos)++;
  2885. retry:
  2886. if (iter->idx >= iter->pg->index) {
  2887. if (iter->pg->next) {
  2888. iter->pg = iter->pg->next;
  2889. iter->idx = 0;
  2890. goto retry;
  2891. }
  2892. } else {
  2893. rec = &iter->pg->records[iter->idx++];
  2894. if (((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) &&
  2895. !ftrace_lookup_ip(iter->hash, rec->ip)) ||
  2896. ((iter->flags & FTRACE_ITER_ENABLED) &&
  2897. !(rec->flags & FTRACE_FL_ENABLED))) {
  2898. rec = NULL;
  2899. goto retry;
  2900. }
  2901. }
  2902. if (!rec)
  2903. return NULL;
  2904. iter->pos = iter->func_pos = *pos;
  2905. iter->func = rec;
  2906. return iter;
  2907. }
  2908. static void *
  2909. t_next(struct seq_file *m, void *v, loff_t *pos)
  2910. {
  2911. struct ftrace_iterator *iter = m->private;
  2912. loff_t l = *pos; /* t_probe_start() must use original pos */
  2913. void *ret;
  2914. if (unlikely(ftrace_disabled))
  2915. return NULL;
  2916. if (iter->flags & FTRACE_ITER_PROBE)
  2917. return t_probe_next(m, pos);
  2918. if (iter->flags & FTRACE_ITER_MOD)
  2919. return t_mod_next(m, pos);
  2920. if (iter->flags & FTRACE_ITER_PRINTALL) {
  2921. /* next must increment pos, and t_probe_start does not */
  2922. (*pos)++;
  2923. return t_mod_start(m, &l);
  2924. }
  2925. ret = t_func_next(m, pos);
  2926. if (!ret)
  2927. return t_mod_start(m, &l);
  2928. return ret;
  2929. }
  2930. static void reset_iter_read(struct ftrace_iterator *iter)
  2931. {
  2932. iter->pos = 0;
  2933. iter->func_pos = 0;
  2934. iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_PROBE | FTRACE_ITER_MOD);
  2935. }
  2936. static void *t_start(struct seq_file *m, loff_t *pos)
  2937. {
  2938. struct ftrace_iterator *iter = m->private;
  2939. void *p = NULL;
  2940. loff_t l;
  2941. mutex_lock(&ftrace_lock);
  2942. if (unlikely(ftrace_disabled))
  2943. return NULL;
  2944. /*
  2945. * If an lseek was done, then reset and start from beginning.
  2946. */
  2947. if (*pos < iter->pos)
  2948. reset_iter_read(iter);
  2949. /*
  2950. * For set_ftrace_filter reading, if we have the filter
  2951. * off, we can short cut and just print out that all
  2952. * functions are enabled.
  2953. */
  2954. if ((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) &&
  2955. ftrace_hash_empty(iter->hash)) {
  2956. iter->func_pos = 1; /* Account for the message */
  2957. if (*pos > 0)
  2958. return t_mod_start(m, pos);
  2959. iter->flags |= FTRACE_ITER_PRINTALL;
  2960. /* reset in case of seek/pread */
  2961. iter->flags &= ~FTRACE_ITER_PROBE;
  2962. return iter;
  2963. }
  2964. if (iter->flags & FTRACE_ITER_MOD)
  2965. return t_mod_start(m, pos);
  2966. /*
  2967. * Unfortunately, we need to restart at ftrace_pages_start
  2968. * every time we let go of the ftrace_mutex. This is because
  2969. * those pointers can change without the lock.
  2970. */
  2971. iter->pg = ftrace_pages_start;
  2972. iter->idx = 0;
  2973. for (l = 0; l <= *pos; ) {
  2974. p = t_func_next(m, &l);
  2975. if (!p)
  2976. break;
  2977. }
  2978. if (!p)
  2979. return t_mod_start(m, pos);
  2980. return iter;
  2981. }
  2982. static void t_stop(struct seq_file *m, void *p)
  2983. {
  2984. mutex_unlock(&ftrace_lock);
  2985. }
  2986. void * __weak
  2987. arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec)
  2988. {
  2989. return NULL;
  2990. }
  2991. static void add_trampoline_func(struct seq_file *m, struct ftrace_ops *ops,
  2992. struct dyn_ftrace *rec)
  2993. {
  2994. void *ptr;
  2995. ptr = arch_ftrace_trampoline_func(ops, rec);
  2996. if (ptr)
  2997. seq_printf(m, " ->%pS", ptr);
  2998. }
  2999. static int t_show(struct seq_file *m, void *v)
  3000. {
  3001. struct ftrace_iterator *iter = m->private;
  3002. struct dyn_ftrace *rec;
  3003. if (iter->flags & FTRACE_ITER_PROBE)
  3004. return t_probe_show(m, iter);
  3005. if (iter->flags & FTRACE_ITER_MOD)
  3006. return t_mod_show(m, iter);
  3007. if (iter->flags & FTRACE_ITER_PRINTALL) {
  3008. if (iter->flags & FTRACE_ITER_NOTRACE)
  3009. seq_puts(m, "#### no functions disabled ####\n");
  3010. else
  3011. seq_puts(m, "#### all functions enabled ####\n");
  3012. return 0;
  3013. }
  3014. rec = iter->func;
  3015. if (!rec)
  3016. return 0;
  3017. seq_printf(m, "%ps", (void *)rec->ip);
  3018. if (iter->flags & FTRACE_ITER_ENABLED) {
  3019. struct ftrace_ops *ops;
  3020. seq_printf(m, " (%ld)%s%s%s",
  3021. ftrace_rec_count(rec),
  3022. rec->flags & FTRACE_FL_REGS ? " R" : " ",
  3023. rec->flags & FTRACE_FL_IPMODIFY ? " I" : " ",
  3024. rec->flags & FTRACE_FL_DIRECT ? " D" : " ");
  3025. if (rec->flags & FTRACE_FL_TRAMP_EN) {
  3026. ops = ftrace_find_tramp_ops_any(rec);
  3027. if (ops) {
  3028. do {
  3029. seq_printf(m, "\ttramp: %pS (%pS)",
  3030. (void *)ops->trampoline,
  3031. (void *)ops->func);
  3032. add_trampoline_func(m, ops, rec);
  3033. ops = ftrace_find_tramp_ops_next(rec, ops);
  3034. } while (ops);
  3035. } else
  3036. seq_puts(m, "\ttramp: ERROR!");
  3037. } else {
  3038. add_trampoline_func(m, NULL, rec);
  3039. }
  3040. if (rec->flags & FTRACE_FL_DIRECT) {
  3041. unsigned long direct;
  3042. direct = ftrace_find_rec_direct(rec->ip);
  3043. if (direct)
  3044. seq_printf(m, "\n\tdirect-->%pS", (void *)direct);
  3045. }
  3046. }
  3047. seq_putc(m, '\n');
  3048. return 0;
  3049. }
  3050. static const struct seq_operations show_ftrace_seq_ops = {
  3051. .start = t_start,
  3052. .next = t_next,
  3053. .stop = t_stop,
  3054. .show = t_show,
  3055. };
  3056. static int
  3057. ftrace_avail_open(struct inode *inode, struct file *file)
  3058. {
  3059. struct ftrace_iterator *iter;
  3060. int ret;
  3061. ret = security_locked_down(LOCKDOWN_TRACEFS);
  3062. if (ret)
  3063. return ret;
  3064. if (unlikely(ftrace_disabled))
  3065. return -ENODEV;
  3066. iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
  3067. if (!iter)
  3068. return -ENOMEM;
  3069. iter->pg = ftrace_pages_start;
  3070. iter->ops = &global_ops;
  3071. return 0;
  3072. }
  3073. static int
  3074. ftrace_enabled_open(struct inode *inode, struct file *file)
  3075. {
  3076. struct ftrace_iterator *iter;
  3077. /*
  3078. * This shows us what functions are currently being
  3079. * traced and by what. Not sure if we want lockdown
  3080. * to hide such critical information for an admin.
  3081. * Although, perhaps it can show information we don't
  3082. * want people to see, but if something is tracing
  3083. * something, we probably want to know about it.
  3084. */
  3085. iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
  3086. if (!iter)
  3087. return -ENOMEM;
  3088. iter->pg = ftrace_pages_start;
  3089. iter->flags = FTRACE_ITER_ENABLED;
  3090. iter->ops = &global_ops;
  3091. return 0;
  3092. }
  3093. /**
  3094. * ftrace_regex_open - initialize function tracer filter files
  3095. * @ops: The ftrace_ops that hold the hash filters
  3096. * @flag: The type of filter to process
  3097. * @inode: The inode, usually passed in to your open routine
  3098. * @file: The file, usually passed in to your open routine
  3099. *
  3100. * ftrace_regex_open() initializes the filter files for the
  3101. * @ops. Depending on @flag it may process the filter hash or
  3102. * the notrace hash of @ops. With this called from the open
  3103. * routine, you can use ftrace_filter_write() for the write
  3104. * routine if @flag has FTRACE_ITER_FILTER set, or
  3105. * ftrace_notrace_write() if @flag has FTRACE_ITER_NOTRACE set.
  3106. * tracing_lseek() should be used as the lseek routine, and
  3107. * release must call ftrace_regex_release().
  3108. */
  3109. int
  3110. ftrace_regex_open(struct ftrace_ops *ops, int flag,
  3111. struct inode *inode, struct file *file)
  3112. {
  3113. struct ftrace_iterator *iter;
  3114. struct ftrace_hash *hash;
  3115. struct list_head *mod_head;
  3116. struct trace_array *tr = ops->private;
  3117. int ret = -ENOMEM;
  3118. ftrace_ops_init(ops);
  3119. if (unlikely(ftrace_disabled))
  3120. return -ENODEV;
  3121. if (tracing_check_open_get_tr(tr))
  3122. return -ENODEV;
  3123. iter = kzalloc(sizeof(*iter), GFP_KERNEL);
  3124. if (!iter)
  3125. goto out;
  3126. if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX))
  3127. goto out;
  3128. iter->ops = ops;
  3129. iter->flags = flag;
  3130. iter->tr = tr;
  3131. mutex_lock(&ops->func_hash->regex_lock);
  3132. if (flag & FTRACE_ITER_NOTRACE) {
  3133. hash = ops->func_hash->notrace_hash;
  3134. mod_head = tr ? &tr->mod_notrace : NULL;
  3135. } else {
  3136. hash = ops->func_hash->filter_hash;
  3137. mod_head = tr ? &tr->mod_trace : NULL;
  3138. }
  3139. iter->mod_list = mod_head;
  3140. if (file->f_mode & FMODE_WRITE) {
  3141. const int size_bits = FTRACE_HASH_DEFAULT_BITS;
  3142. if (file->f_flags & O_TRUNC) {
  3143. iter->hash = alloc_ftrace_hash(size_bits);
  3144. clear_ftrace_mod_list(mod_head);
  3145. } else {
  3146. iter->hash = alloc_and_copy_ftrace_hash(size_bits, hash);
  3147. }
  3148. if (!iter->hash) {
  3149. trace_parser_put(&iter->parser);
  3150. goto out_unlock;
  3151. }
  3152. } else
  3153. iter->hash = hash;
  3154. ret = 0;
  3155. if (file->f_mode & FMODE_READ) {
  3156. iter->pg = ftrace_pages_start;
  3157. ret = seq_open(file, &show_ftrace_seq_ops);
  3158. if (!ret) {
  3159. struct seq_file *m = file->private_data;
  3160. m->private = iter;
  3161. } else {
  3162. /* Failed */
  3163. free_ftrace_hash(iter->hash);
  3164. trace_parser_put(&iter->parser);
  3165. }
  3166. } else
  3167. file->private_data = iter;
  3168. out_unlock:
  3169. mutex_unlock(&ops->func_hash->regex_lock);
  3170. out:
  3171. if (ret) {
  3172. kfree(iter);
  3173. if (tr)
  3174. trace_array_put(tr);
  3175. }
  3176. return ret;
  3177. }
  3178. static int
  3179. ftrace_filter_open(struct inode *inode, struct file *file)
  3180. {
  3181. struct ftrace_ops *ops = inode->i_private;
  3182. /* Checks for tracefs lockdown */
  3183. return ftrace_regex_open(ops,
  3184. FTRACE_ITER_FILTER | FTRACE_ITER_DO_PROBES,
  3185. inode, file);
  3186. }
  3187. static int
  3188. ftrace_notrace_open(struct inode *inode, struct file *file)
  3189. {
  3190. struct ftrace_ops *ops = inode->i_private;
  3191. /* Checks for tracefs lockdown */
  3192. return ftrace_regex_open(ops, FTRACE_ITER_NOTRACE,
  3193. inode, file);
  3194. }
  3195. /* Type for quick search ftrace basic regexes (globs) from filter_parse_regex */
  3196. struct ftrace_glob {
  3197. char *search;
  3198. unsigned len;
  3199. int type;
  3200. };
  3201. /*
  3202. * If symbols in an architecture don't correspond exactly to the user-visible
  3203. * name of what they represent, it is possible to define this function to
  3204. * perform the necessary adjustments.
  3205. */
  3206. char * __weak arch_ftrace_match_adjust(char *str, const char *search)
  3207. {
  3208. return str;
  3209. }
  3210. static int ftrace_match(char *str, struct ftrace_glob *g)
  3211. {
  3212. int matched = 0;
  3213. int slen;
  3214. str = arch_ftrace_match_adjust(str, g->search);
  3215. switch (g->type) {
  3216. case MATCH_FULL:
  3217. if (strcmp(str, g->search) == 0)
  3218. matched = 1;
  3219. break;
  3220. case MATCH_FRONT_ONLY:
  3221. if (strncmp(str, g->search, g->len) == 0)
  3222. matched = 1;
  3223. break;
  3224. case MATCH_MIDDLE_ONLY:
  3225. if (strstr(str, g->search))
  3226. matched = 1;
  3227. break;
  3228. case MATCH_END_ONLY:
  3229. slen = strlen(str);
  3230. if (slen >= g->len &&
  3231. memcmp(str + slen - g->len, g->search, g->len) == 0)
  3232. matched = 1;
  3233. break;
  3234. case MATCH_GLOB:
  3235. if (glob_match(g->search, str))
  3236. matched = 1;
  3237. break;
  3238. }
  3239. return matched;
  3240. }
  3241. static int
  3242. enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int clear_filter)
  3243. {
  3244. struct ftrace_func_entry *entry;
  3245. int ret = 0;
  3246. entry = ftrace_lookup_ip(hash, rec->ip);
  3247. if (clear_filter) {
  3248. /* Do nothing if it doesn't exist */
  3249. if (!entry)
  3250. return 0;
  3251. free_hash_entry(hash, entry);
  3252. } else {
  3253. /* Do nothing if it exists */
  3254. if (entry)
  3255. return 0;
  3256. ret = add_hash_entry(hash, rec->ip);
  3257. }
  3258. return ret;
  3259. }
  3260. static int
  3261. add_rec_by_index(struct ftrace_hash *hash, struct ftrace_glob *func_g,
  3262. int clear_filter)
  3263. {
  3264. long index = simple_strtoul(func_g->search, NULL, 0);
  3265. struct ftrace_page *pg;
  3266. struct dyn_ftrace *rec;
  3267. /* The index starts at 1 */
  3268. if (--index < 0)
  3269. return 0;
  3270. do_for_each_ftrace_rec(pg, rec) {
  3271. if (pg->index <= index) {
  3272. index -= pg->index;
  3273. /* this is a double loop, break goes to the next page */
  3274. break;
  3275. }
  3276. rec = &pg->records[index];
  3277. enter_record(hash, rec, clear_filter);
  3278. return 1;
  3279. } while_for_each_ftrace_rec();
  3280. return 0;
  3281. }
  3282. static int
  3283. ftrace_match_record(struct dyn_ftrace *rec, struct ftrace_glob *func_g,
  3284. struct ftrace_glob *mod_g, int exclude_mod)
  3285. {
  3286. char str[KSYM_SYMBOL_LEN];
  3287. char *modname;
  3288. kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
  3289. if (mod_g) {
  3290. int mod_matches = (modname) ? ftrace_match(modname, mod_g) : 0;
  3291. /* blank module name to match all modules */
  3292. if (!mod_g->len) {
  3293. /* blank module globbing: modname xor exclude_mod */
  3294. if (!exclude_mod != !modname)
  3295. goto func_match;
  3296. return 0;
  3297. }
  3298. /*
  3299. * exclude_mod is set to trace everything but the given
  3300. * module. If it is set and the module matches, then
  3301. * return 0. If it is not set, and the module doesn't match
  3302. * also return 0. Otherwise, check the function to see if
  3303. * that matches.
  3304. */
  3305. if (!mod_matches == !exclude_mod)
  3306. return 0;
  3307. func_match:
  3308. /* blank search means to match all funcs in the mod */
  3309. if (!func_g->len)
  3310. return 1;
  3311. }
  3312. return ftrace_match(str, func_g);
  3313. }
  3314. static int
  3315. match_records(struct ftrace_hash *hash, char *func, int len, char *mod)
  3316. {
  3317. struct ftrace_page *pg;
  3318. struct dyn_ftrace *rec;
  3319. struct ftrace_glob func_g = { .type = MATCH_FULL };
  3320. struct ftrace_glob mod_g = { .type = MATCH_FULL };
  3321. struct ftrace_glob *mod_match = (mod) ? &mod_g : NULL;
  3322. int exclude_mod = 0;
  3323. int found = 0;
  3324. int ret;
  3325. int clear_filter = 0;
  3326. if (func) {
  3327. func_g.type = filter_parse_regex(func, len, &func_g.search,
  3328. &clear_filter);
  3329. func_g.len = strlen(func_g.search);
  3330. }
  3331. if (mod) {
  3332. mod_g.type = filter_parse_regex(mod, strlen(mod),
  3333. &mod_g.search, &exclude_mod);
  3334. mod_g.len = strlen(mod_g.search);
  3335. }
  3336. mutex_lock(&ftrace_lock);
  3337. if (unlikely(ftrace_disabled))
  3338. goto out_unlock;
  3339. if (func_g.type == MATCH_INDEX) {
  3340. found = add_rec_by_index(hash, &func_g, clear_filter);
  3341. goto out_unlock;
  3342. }
  3343. do_for_each_ftrace_rec(pg, rec) {
  3344. if (rec->flags & FTRACE_FL_DISABLED)
  3345. continue;
  3346. if (ftrace_match_record(rec, &func_g, mod_match, exclude_mod)) {
  3347. ret = enter_record(hash, rec, clear_filter);
  3348. if (ret < 0) {
  3349. found = ret;
  3350. goto out_unlock;
  3351. }
  3352. found = 1;
  3353. }
  3354. } while_for_each_ftrace_rec();
  3355. out_unlock:
  3356. mutex_unlock(&ftrace_lock);
  3357. return found;
  3358. }
  3359. static int
  3360. ftrace_match_records(struct ftrace_hash *hash, char *buff, int len)
  3361. {
  3362. return match_records(hash, buff, len, NULL);
  3363. }
  3364. static void ftrace_ops_update_code(struct ftrace_ops *ops,
  3365. struct ftrace_ops_hash *old_hash)
  3366. {
  3367. struct ftrace_ops *op;
  3368. if (!ftrace_enabled)
  3369. return;
  3370. if (ops->flags & FTRACE_OPS_FL_ENABLED) {
  3371. ftrace_run_modify_code(ops, FTRACE_UPDATE_CALLS, old_hash);
  3372. return;
  3373. }
  3374. /*
  3375. * If this is the shared global_ops filter, then we need to
  3376. * check if there is another ops that shares it, is enabled.
  3377. * If so, we still need to run the modify code.
  3378. */
  3379. if (ops->func_hash != &global_ops.local_hash)
  3380. return;
  3381. do_for_each_ftrace_op(op, ftrace_ops_list) {
  3382. if (op->func_hash == &global_ops.local_hash &&
  3383. op->flags & FTRACE_OPS_FL_ENABLED) {
  3384. ftrace_run_modify_code(op, FTRACE_UPDATE_CALLS, old_hash);
  3385. /* Only need to do this once */
  3386. return;
  3387. }
  3388. } while_for_each_ftrace_op(op);
  3389. }
  3390. static int ftrace_hash_move_and_update_ops(struct ftrace_ops *ops,
  3391. struct ftrace_hash **orig_hash,
  3392. struct ftrace_hash *hash,
  3393. int enable)
  3394. {
  3395. struct ftrace_ops_hash old_hash_ops;
  3396. struct ftrace_hash *old_hash;
  3397. int ret;
  3398. old_hash = *orig_hash;
  3399. old_hash_ops.filter_hash = ops->func_hash->filter_hash;
  3400. old_hash_ops.notrace_hash = ops->func_hash->notrace_hash;
  3401. ret = ftrace_hash_move(ops, enable, orig_hash, hash);
  3402. if (!ret) {
  3403. ftrace_ops_update_code(ops, &old_hash_ops);
  3404. free_ftrace_hash_rcu(old_hash);
  3405. }
  3406. return ret;
  3407. }
  3408. static bool module_exists(const char *module)
  3409. {
  3410. /* All modules have the symbol __this_module */
  3411. static const char this_mod[] = "__this_module";
  3412. char modname[MAX_PARAM_PREFIX_LEN + sizeof(this_mod) + 2];
  3413. unsigned long val;
  3414. int n;
  3415. n = snprintf(modname, sizeof(modname), "%s:%s", module, this_mod);
  3416. if (n > sizeof(modname) - 1)
  3417. return false;
  3418. val = module_kallsyms_lookup_name(modname);
  3419. return val != 0;
  3420. }
  3421. static int cache_mod(struct trace_array *tr,
  3422. const char *func, char *module, int enable)
  3423. {
  3424. struct ftrace_mod_load *ftrace_mod, *n;
  3425. struct list_head *head = enable ? &tr->mod_trace : &tr->mod_notrace;
  3426. int ret;
  3427. mutex_lock(&ftrace_lock);
  3428. /* We do not cache inverse filters */
  3429. if (func[0] == '!') {
  3430. func++;
  3431. ret = -EINVAL;
  3432. /* Look to remove this hash */
  3433. list_for_each_entry_safe(ftrace_mod, n, head, list) {
  3434. if (strcmp(ftrace_mod->module, module) != 0)
  3435. continue;
  3436. /* no func matches all */
  3437. if (strcmp(func, "*") == 0 ||
  3438. (ftrace_mod->func &&
  3439. strcmp(ftrace_mod->func, func) == 0)) {
  3440. ret = 0;
  3441. free_ftrace_mod(ftrace_mod);
  3442. continue;
  3443. }
  3444. }
  3445. goto out;
  3446. }
  3447. ret = -EINVAL;
  3448. /* We only care about modules that have not been loaded yet */
  3449. if (module_exists(module))
  3450. goto out;
  3451. /* Save this string off, and execute it when the module is loaded */
  3452. ret = ftrace_add_mod(tr, func, module, enable);
  3453. out:
  3454. mutex_unlock(&ftrace_lock);
  3455. return ret;
  3456. }
  3457. static int
  3458. ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
  3459. int reset, int enable);
  3460. #ifdef CONFIG_MODULES
  3461. static void process_mod_list(struct list_head *head, struct ftrace_ops *ops,
  3462. char *mod, bool enable)
  3463. {
  3464. struct ftrace_mod_load *ftrace_mod, *n;
  3465. struct ftrace_hash **orig_hash, *new_hash;
  3466. LIST_HEAD(process_mods);
  3467. char *func;
  3468. int ret;
  3469. mutex_lock(&ops->func_hash->regex_lock);
  3470. if (enable)
  3471. orig_hash = &ops->func_hash->filter_hash;
  3472. else
  3473. orig_hash = &ops->func_hash->notrace_hash;
  3474. new_hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS,
  3475. *orig_hash);
  3476. if (!new_hash)
  3477. goto out; /* warn? */
  3478. mutex_lock(&ftrace_lock);
  3479. list_for_each_entry_safe(ftrace_mod, n, head, list) {
  3480. if (strcmp(ftrace_mod->module, mod) != 0)
  3481. continue;
  3482. if (ftrace_mod->func)
  3483. func = kstrdup(ftrace_mod->func, GFP_KERNEL);
  3484. else
  3485. func = kstrdup("*", GFP_KERNEL);
  3486. if (!func) /* warn? */
  3487. continue;
  3488. list_del(&ftrace_mod->list);
  3489. list_add(&ftrace_mod->list, &process_mods);
  3490. /* Use the newly allocated func, as it may be "*" */
  3491. kfree(ftrace_mod->func);
  3492. ftrace_mod->func = func;
  3493. }
  3494. mutex_unlock(&ftrace_lock);
  3495. list_for_each_entry_safe(ftrace_mod, n, &process_mods, list) {
  3496. func = ftrace_mod->func;
  3497. /* Grabs ftrace_lock, which is why we have this extra step */
  3498. match_records(new_hash, func, strlen(func), mod);
  3499. free_ftrace_mod(ftrace_mod);
  3500. }
  3501. if (enable && list_empty(head))
  3502. new_hash->flags &= ~FTRACE_HASH_FL_MOD;
  3503. mutex_lock(&ftrace_lock);
  3504. ret = ftrace_hash_move_and_update_ops(ops, orig_hash,
  3505. new_hash, enable);
  3506. mutex_unlock(&ftrace_lock);
  3507. out:
  3508. mutex_unlock(&ops->func_hash->regex_lock);
  3509. free_ftrace_hash(new_hash);
  3510. }
  3511. static void process_cached_mods(const char *mod_name)
  3512. {
  3513. struct trace_array *tr;
  3514. char *mod;
  3515. mod = kstrdup(mod_name, GFP_KERNEL);
  3516. if (!mod)
  3517. return;
  3518. mutex_lock(&trace_types_lock);
  3519. list_for_each_entry(tr, &ftrace_trace_arrays, list) {
  3520. if (!list_empty(&tr->mod_trace))
  3521. process_mod_list(&tr->mod_trace, tr->ops, mod, true);
  3522. if (!list_empty(&tr->mod_notrace))
  3523. process_mod_list(&tr->mod_notrace, tr->ops, mod, false);
  3524. }
  3525. mutex_unlock(&trace_types_lock);
  3526. kfree(mod);
  3527. }
  3528. #endif
  3529. /*
  3530. * We register the module command as a template to show others how
  3531. * to register the a command as well.
  3532. */
  3533. static int
  3534. ftrace_mod_callback(struct trace_array *tr, struct ftrace_hash *hash,
  3535. char *func_orig, char *cmd, char *module, int enable)
  3536. {
  3537. char *func;
  3538. int ret;
  3539. /* match_records() modifies func, and we need the original */
  3540. func = kstrdup(func_orig, GFP_KERNEL);
  3541. if (!func)
  3542. return -ENOMEM;
  3543. /*
  3544. * cmd == 'mod' because we only registered this func
  3545. * for the 'mod' ftrace_func_command.
  3546. * But if you register one func with multiple commands,
  3547. * you can tell which command was used by the cmd
  3548. * parameter.
  3549. */
  3550. ret = match_records(hash, func, strlen(func), module);
  3551. kfree(func);
  3552. if (!ret)
  3553. return cache_mod(tr, func_orig, module, enable);
  3554. if (ret < 0)
  3555. return ret;
  3556. return 0;
  3557. }
  3558. static struct ftrace_func_command ftrace_mod_cmd = {
  3559. .name = "mod",
  3560. .func = ftrace_mod_callback,
  3561. };
  3562. static int __init ftrace_mod_cmd_init(void)
  3563. {
  3564. return register_ftrace_command(&ftrace_mod_cmd);
  3565. }
  3566. core_initcall(ftrace_mod_cmd_init);
  3567. static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip,
  3568. struct ftrace_ops *op, struct pt_regs *pt_regs)
  3569. {
  3570. struct ftrace_probe_ops *probe_ops;
  3571. struct ftrace_func_probe *probe;
  3572. probe = container_of(op, struct ftrace_func_probe, ops);
  3573. probe_ops = probe->probe_ops;
  3574. /*
  3575. * Disable preemption for these calls to prevent a RCU grace
  3576. * period. This syncs the hash iteration and freeing of items
  3577. * on the hash. rcu_read_lock is too dangerous here.
  3578. */
  3579. preempt_disable_notrace();
  3580. probe_ops->func(ip, parent_ip, probe->tr, probe_ops, probe->data);
  3581. preempt_enable_notrace();
  3582. }
  3583. struct ftrace_func_map {
  3584. struct ftrace_func_entry entry;
  3585. void *data;
  3586. };
  3587. struct ftrace_func_mapper {
  3588. struct ftrace_hash hash;
  3589. };
  3590. /**
  3591. * allocate_ftrace_func_mapper - allocate a new ftrace_func_mapper
  3592. *
  3593. * Returns a ftrace_func_mapper descriptor that can be used to map ips to data.
  3594. */
  3595. struct ftrace_func_mapper *allocate_ftrace_func_mapper(void)
  3596. {
  3597. struct ftrace_hash *hash;
  3598. /*
  3599. * The mapper is simply a ftrace_hash, but since the entries
  3600. * in the hash are not ftrace_func_entry type, we define it
  3601. * as a separate structure.
  3602. */
  3603. hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
  3604. return (struct ftrace_func_mapper *)hash;
  3605. }
  3606. /**
  3607. * ftrace_func_mapper_find_ip - Find some data mapped to an ip
  3608. * @mapper: The mapper that has the ip maps
  3609. * @ip: the instruction pointer to find the data for
  3610. *
  3611. * Returns the data mapped to @ip if found otherwise NULL. The return
  3612. * is actually the address of the mapper data pointer. The address is
  3613. * returned for use cases where the data is no bigger than a long, and
  3614. * the user can use the data pointer as its data instead of having to
  3615. * allocate more memory for the reference.
  3616. */
  3617. void **ftrace_func_mapper_find_ip(struct ftrace_func_mapper *mapper,
  3618. unsigned long ip)
  3619. {
  3620. struct ftrace_func_entry *entry;
  3621. struct ftrace_func_map *map;
  3622. entry = ftrace_lookup_ip(&mapper->hash, ip);
  3623. if (!entry)
  3624. return NULL;
  3625. map = (struct ftrace_func_map *)entry;
  3626. return &map->data;
  3627. }
  3628. /**
  3629. * ftrace_func_mapper_add_ip - Map some data to an ip
  3630. * @mapper: The mapper that has the ip maps
  3631. * @ip: The instruction pointer address to map @data to
  3632. * @data: The data to map to @ip
  3633. *
  3634. * Returns 0 on success otherwise an error.
  3635. */
  3636. int ftrace_func_mapper_add_ip(struct ftrace_func_mapper *mapper,
  3637. unsigned long ip, void *data)
  3638. {
  3639. struct ftrace_func_entry *entry;
  3640. struct ftrace_func_map *map;
  3641. entry = ftrace_lookup_ip(&mapper->hash, ip);
  3642. if (entry)
  3643. return -EBUSY;
  3644. map = kmalloc(sizeof(*map), GFP_KERNEL);
  3645. if (!map)
  3646. return -ENOMEM;
  3647. map->entry.ip = ip;
  3648. map->data = data;
  3649. __add_hash_entry(&mapper->hash, &map->entry);
  3650. return 0;
  3651. }
  3652. /**
  3653. * ftrace_func_mapper_remove_ip - Remove an ip from the mapping
  3654. * @mapper: The mapper that has the ip maps
  3655. * @ip: The instruction pointer address to remove the data from
  3656. *
  3657. * Returns the data if it is found, otherwise NULL.
  3658. * Note, if the data pointer is used as the data itself, (see
  3659. * ftrace_func_mapper_find_ip(), then the return value may be meaningless,
  3660. * if the data pointer was set to zero.
  3661. */
  3662. void *ftrace_func_mapper_remove_ip(struct ftrace_func_mapper *mapper,
  3663. unsigned long ip)
  3664. {
  3665. struct ftrace_func_entry *entry;
  3666. struct ftrace_func_map *map;
  3667. void *data;
  3668. entry = ftrace_lookup_ip(&mapper->hash, ip);
  3669. if (!entry)
  3670. return NULL;
  3671. map = (struct ftrace_func_map *)entry;
  3672. data = map->data;
  3673. remove_hash_entry(&mapper->hash, entry);
  3674. kfree(entry);
  3675. return data;
  3676. }
  3677. /**
  3678. * free_ftrace_func_mapper - free a mapping of ips and data
  3679. * @mapper: The mapper that has the ip maps
  3680. * @free_func: A function to be called on each data item.
  3681. *
  3682. * This is used to free the function mapper. The @free_func is optional
  3683. * and can be used if the data needs to be freed as well.
  3684. */
  3685. void free_ftrace_func_mapper(struct ftrace_func_mapper *mapper,
  3686. ftrace_mapper_func free_func)
  3687. {
  3688. struct ftrace_func_entry *entry;
  3689. struct ftrace_func_map *map;
  3690. struct hlist_head *hhd;
  3691. int size, i;
  3692. if (!mapper)
  3693. return;
  3694. if (free_func && mapper->hash.count) {
  3695. size = 1 << mapper->hash.size_bits;
  3696. for (i = 0; i < size; i++) {
  3697. hhd = &mapper->hash.buckets[i];
  3698. hlist_for_each_entry(entry, hhd, hlist) {
  3699. map = (struct ftrace_func_map *)entry;
  3700. free_func(map);
  3701. }
  3702. }
  3703. }
  3704. free_ftrace_hash(&mapper->hash);
  3705. }
  3706. static void release_probe(struct ftrace_func_probe *probe)
  3707. {
  3708. struct ftrace_probe_ops *probe_ops;
  3709. mutex_lock(&ftrace_lock);
  3710. WARN_ON(probe->ref <= 0);
  3711. /* Subtract the ref that was used to protect this instance */
  3712. probe->ref--;
  3713. if (!probe->ref) {
  3714. probe_ops = probe->probe_ops;
  3715. /*
  3716. * Sending zero as ip tells probe_ops to free
  3717. * the probe->data itself
  3718. */
  3719. if (probe_ops->free)
  3720. probe_ops->free(probe_ops, probe->tr, 0, probe->data);
  3721. list_del(&probe->list);
  3722. kfree(probe);
  3723. }
  3724. mutex_unlock(&ftrace_lock);
  3725. }
  3726. static void acquire_probe_locked(struct ftrace_func_probe *probe)
  3727. {
  3728. /*
  3729. * Add one ref to keep it from being freed when releasing the
  3730. * ftrace_lock mutex.
  3731. */
  3732. probe->ref++;
  3733. }
  3734. int
  3735. register_ftrace_function_probe(char *glob, struct trace_array *tr,
  3736. struct ftrace_probe_ops *probe_ops,
  3737. void *data)
  3738. {
  3739. struct ftrace_func_entry *entry;
  3740. struct ftrace_func_probe *probe;
  3741. struct ftrace_hash **orig_hash;
  3742. struct ftrace_hash *old_hash;
  3743. struct ftrace_hash *hash;
  3744. int count = 0;
  3745. int size;
  3746. int ret;
  3747. int i;
  3748. if (WARN_ON(!tr))
  3749. return -EINVAL;
  3750. /* We do not support '!' for function probes */
  3751. if (WARN_ON(glob[0] == '!'))
  3752. return -EINVAL;
  3753. mutex_lock(&ftrace_lock);
  3754. /* Check if the probe_ops is already registered */
  3755. list_for_each_entry(probe, &tr->func_probes, list) {
  3756. if (probe->probe_ops == probe_ops)
  3757. break;
  3758. }
  3759. if (&probe->list == &tr->func_probes) {
  3760. probe = kzalloc(sizeof(*probe), GFP_KERNEL);
  3761. if (!probe) {
  3762. mutex_unlock(&ftrace_lock);
  3763. return -ENOMEM;
  3764. }
  3765. probe->probe_ops = probe_ops;
  3766. probe->ops.func = function_trace_probe_call;
  3767. probe->tr = tr;
  3768. ftrace_ops_init(&probe->ops);
  3769. list_add(&probe->list, &tr->func_probes);
  3770. }
  3771. acquire_probe_locked(probe);
  3772. mutex_unlock(&ftrace_lock);
  3773. /*
  3774. * Note, there's a small window here that the func_hash->filter_hash
  3775. * may be NULL or empty. Need to be careful when reading the loop.
  3776. */
  3777. mutex_lock(&probe->ops.func_hash->regex_lock);
  3778. orig_hash = &probe->ops.func_hash->filter_hash;
  3779. old_hash = *orig_hash;
  3780. hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash);
  3781. if (!hash) {
  3782. ret = -ENOMEM;
  3783. goto out;
  3784. }
  3785. ret = ftrace_match_records(hash, glob, strlen(glob));
  3786. /* Nothing found? */
  3787. if (!ret)
  3788. ret = -EINVAL;
  3789. if (ret < 0)
  3790. goto out;
  3791. size = 1 << hash->size_bits;
  3792. for (i = 0; i < size; i++) {
  3793. hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
  3794. if (ftrace_lookup_ip(old_hash, entry->ip))
  3795. continue;
  3796. /*
  3797. * The caller might want to do something special
  3798. * for each function we find. We call the callback
  3799. * to give the caller an opportunity to do so.
  3800. */
  3801. if (probe_ops->init) {
  3802. ret = probe_ops->init(probe_ops, tr,
  3803. entry->ip, data,
  3804. &probe->data);
  3805. if (ret < 0) {
  3806. if (probe_ops->free && count)
  3807. probe_ops->free(probe_ops, tr,
  3808. 0, probe->data);
  3809. probe->data = NULL;
  3810. goto out;
  3811. }
  3812. }
  3813. count++;
  3814. }
  3815. }
  3816. mutex_lock(&ftrace_lock);
  3817. if (!count) {
  3818. /* Nothing was added? */
  3819. ret = -EINVAL;
  3820. goto out_unlock;
  3821. }
  3822. ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash,
  3823. hash, 1);
  3824. if (ret < 0)
  3825. goto err_unlock;
  3826. /* One ref for each new function traced */
  3827. probe->ref += count;
  3828. if (!(probe->ops.flags & FTRACE_OPS_FL_ENABLED))
  3829. ret = ftrace_startup(&probe->ops, 0);
  3830. out_unlock:
  3831. mutex_unlock(&ftrace_lock);
  3832. if (!ret)
  3833. ret = count;
  3834. out:
  3835. mutex_unlock(&probe->ops.func_hash->regex_lock);
  3836. free_ftrace_hash(hash);
  3837. release_probe(probe);
  3838. return ret;
  3839. err_unlock:
  3840. if (!probe_ops->free || !count)
  3841. goto out_unlock;
  3842. /* Failed to do the move, need to call the free functions */
  3843. for (i = 0; i < size; i++) {
  3844. hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
  3845. if (ftrace_lookup_ip(old_hash, entry->ip))
  3846. continue;
  3847. probe_ops->free(probe_ops, tr, entry->ip, probe->data);
  3848. }
  3849. }
  3850. goto out_unlock;
  3851. }
  3852. int
  3853. unregister_ftrace_function_probe_func(char *glob, struct trace_array *tr,
  3854. struct ftrace_probe_ops *probe_ops)
  3855. {
  3856. struct ftrace_ops_hash old_hash_ops;
  3857. struct ftrace_func_entry *entry;
  3858. struct ftrace_func_probe *probe;
  3859. struct ftrace_glob func_g;
  3860. struct ftrace_hash **orig_hash;
  3861. struct ftrace_hash *old_hash;
  3862. struct ftrace_hash *hash = NULL;
  3863. struct hlist_node *tmp;
  3864. struct hlist_head hhd;
  3865. char str[KSYM_SYMBOL_LEN];
  3866. int count = 0;
  3867. int i, ret = -ENODEV;
  3868. int size;
  3869. if (!glob || !strlen(glob) || !strcmp(glob, "*"))
  3870. func_g.search = NULL;
  3871. else {
  3872. int not;
  3873. func_g.type = filter_parse_regex(glob, strlen(glob),
  3874. &func_g.search, &not);
  3875. func_g.len = strlen(func_g.search);
  3876. /* we do not support '!' for function probes */
  3877. if (WARN_ON(not))
  3878. return -EINVAL;
  3879. }
  3880. mutex_lock(&ftrace_lock);
  3881. /* Check if the probe_ops is already registered */
  3882. list_for_each_entry(probe, &tr->func_probes, list) {
  3883. if (probe->probe_ops == probe_ops)
  3884. break;
  3885. }
  3886. if (&probe->list == &tr->func_probes)
  3887. goto err_unlock_ftrace;
  3888. ret = -EINVAL;
  3889. if (!(probe->ops.flags & FTRACE_OPS_FL_INITIALIZED))
  3890. goto err_unlock_ftrace;
  3891. acquire_probe_locked(probe);
  3892. mutex_unlock(&ftrace_lock);
  3893. mutex_lock(&probe->ops.func_hash->regex_lock);
  3894. orig_hash = &probe->ops.func_hash->filter_hash;
  3895. old_hash = *orig_hash;
  3896. if (ftrace_hash_empty(old_hash))
  3897. goto out_unlock;
  3898. old_hash_ops.filter_hash = old_hash;
  3899. /* Probes only have filters */
  3900. old_hash_ops.notrace_hash = NULL;
  3901. ret = -ENOMEM;
  3902. hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash);
  3903. if (!hash)
  3904. goto out_unlock;
  3905. INIT_HLIST_HEAD(&hhd);
  3906. size = 1 << hash->size_bits;
  3907. for (i = 0; i < size; i++) {
  3908. hlist_for_each_entry_safe(entry, tmp, &hash->buckets[i], hlist) {
  3909. if (func_g.search) {
  3910. kallsyms_lookup(entry->ip, NULL, NULL,
  3911. NULL, str);
  3912. if (!ftrace_match(str, &func_g))
  3913. continue;
  3914. }
  3915. count++;
  3916. remove_hash_entry(hash, entry);
  3917. hlist_add_head(&entry->hlist, &hhd);
  3918. }
  3919. }
  3920. /* Nothing found? */
  3921. if (!count) {
  3922. ret = -EINVAL;
  3923. goto out_unlock;
  3924. }
  3925. mutex_lock(&ftrace_lock);
  3926. WARN_ON(probe->ref < count);
  3927. probe->ref -= count;
  3928. if (ftrace_hash_empty(hash))
  3929. ftrace_shutdown(&probe->ops, 0);
  3930. ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash,
  3931. hash, 1);
  3932. /* still need to update the function call sites */
  3933. if (ftrace_enabled && !ftrace_hash_empty(hash))
  3934. ftrace_run_modify_code(&probe->ops, FTRACE_UPDATE_CALLS,
  3935. &old_hash_ops);
  3936. synchronize_rcu();
  3937. hlist_for_each_entry_safe(entry, tmp, &hhd, hlist) {
  3938. hlist_del(&entry->hlist);
  3939. if (probe_ops->free)
  3940. probe_ops->free(probe_ops, tr, entry->ip, probe->data);
  3941. kfree(entry);
  3942. }
  3943. mutex_unlock(&ftrace_lock);
  3944. out_unlock:
  3945. mutex_unlock(&probe->ops.func_hash->regex_lock);
  3946. free_ftrace_hash(hash);
  3947. release_probe(probe);
  3948. return ret;
  3949. err_unlock_ftrace:
  3950. mutex_unlock(&ftrace_lock);
  3951. return ret;
  3952. }
  3953. void clear_ftrace_function_probes(struct trace_array *tr)
  3954. {
  3955. struct ftrace_func_probe *probe, *n;
  3956. list_for_each_entry_safe(probe, n, &tr->func_probes, list)
  3957. unregister_ftrace_function_probe_func(NULL, tr, probe->probe_ops);
  3958. }
  3959. static LIST_HEAD(ftrace_commands);
  3960. static DEFINE_MUTEX(ftrace_cmd_mutex);
  3961. /*
  3962. * Currently we only register ftrace commands from __init, so mark this
  3963. * __init too.
  3964. */
  3965. __init int register_ftrace_command(struct ftrace_func_command *cmd)
  3966. {
  3967. struct ftrace_func_command *p;
  3968. int ret = 0;
  3969. mutex_lock(&ftrace_cmd_mutex);
  3970. list_for_each_entry(p, &ftrace_commands, list) {
  3971. if (strcmp(cmd->name, p->name) == 0) {
  3972. ret = -EBUSY;
  3973. goto out_unlock;
  3974. }
  3975. }
  3976. list_add(&cmd->list, &ftrace_commands);
  3977. out_unlock:
  3978. mutex_unlock(&ftrace_cmd_mutex);
  3979. return ret;
  3980. }
  3981. /*
  3982. * Currently we only unregister ftrace commands from __init, so mark
  3983. * this __init too.
  3984. */
  3985. __init int unregister_ftrace_command(struct ftrace_func_command *cmd)
  3986. {
  3987. struct ftrace_func_command *p, *n;
  3988. int ret = -ENODEV;
  3989. mutex_lock(&ftrace_cmd_mutex);
  3990. list_for_each_entry_safe(p, n, &ftrace_commands, list) {
  3991. if (strcmp(cmd->name, p->name) == 0) {
  3992. ret = 0;
  3993. list_del_init(&p->list);
  3994. goto out_unlock;
  3995. }
  3996. }
  3997. out_unlock:
  3998. mutex_unlock(&ftrace_cmd_mutex);
  3999. return ret;
  4000. }
  4001. static int ftrace_process_regex(struct ftrace_iterator *iter,
  4002. char *buff, int len, int enable)
  4003. {
  4004. struct ftrace_hash *hash = iter->hash;
  4005. struct trace_array *tr = iter->ops->private;
  4006. char *func, *command, *next = buff;
  4007. struct ftrace_func_command *p;
  4008. int ret = -EINVAL;
  4009. func = strsep(&next, ":");
  4010. if (!next) {
  4011. ret = ftrace_match_records(hash, func, len);
  4012. if (!ret)
  4013. ret = -EINVAL;
  4014. if (ret < 0)
  4015. return ret;
  4016. return 0;
  4017. }
  4018. /* command found */
  4019. command = strsep(&next, ":");
  4020. mutex_lock(&ftrace_cmd_mutex);
  4021. list_for_each_entry(p, &ftrace_commands, list) {
  4022. if (strcmp(p->name, command) == 0) {
  4023. ret = p->func(tr, hash, func, command, next, enable);
  4024. goto out_unlock;
  4025. }
  4026. }
  4027. out_unlock:
  4028. mutex_unlock(&ftrace_cmd_mutex);
  4029. return ret;
  4030. }
  4031. static ssize_t
  4032. ftrace_regex_write(struct file *file, const char __user *ubuf,
  4033. size_t cnt, loff_t *ppos, int enable)
  4034. {
  4035. struct ftrace_iterator *iter;
  4036. struct trace_parser *parser;
  4037. ssize_t ret, read;
  4038. if (!cnt)
  4039. return 0;
  4040. if (file->f_mode & FMODE_READ) {
  4041. struct seq_file *m = file->private_data;
  4042. iter = m->private;
  4043. } else
  4044. iter = file->private_data;
  4045. if (unlikely(ftrace_disabled))
  4046. return -ENODEV;
  4047. /* iter->hash is a local copy, so we don't need regex_lock */
  4048. parser = &iter->parser;
  4049. read = trace_get_user(parser, ubuf, cnt, ppos);
  4050. if (read >= 0 && trace_parser_loaded(parser) &&
  4051. !trace_parser_cont(parser)) {
  4052. ret = ftrace_process_regex(iter, parser->buffer,
  4053. parser->idx, enable);
  4054. trace_parser_clear(parser);
  4055. if (ret < 0)
  4056. goto out;
  4057. }
  4058. ret = read;
  4059. out:
  4060. return ret;
  4061. }
  4062. ssize_t
  4063. ftrace_filter_write(struct file *file, const char __user *ubuf,
  4064. size_t cnt, loff_t *ppos)
  4065. {
  4066. return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
  4067. }
  4068. ssize_t
  4069. ftrace_notrace_write(struct file *file, const char __user *ubuf,
  4070. size_t cnt, loff_t *ppos)
  4071. {
  4072. return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
  4073. }
  4074. static int
  4075. ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove)
  4076. {
  4077. struct ftrace_func_entry *entry;
  4078. if (!ftrace_location(ip))
  4079. return -EINVAL;
  4080. if (remove) {
  4081. entry = ftrace_lookup_ip(hash, ip);
  4082. if (!entry)
  4083. return -ENOENT;
  4084. free_hash_entry(hash, entry);
  4085. return 0;
  4086. }
  4087. return add_hash_entry(hash, ip);
  4088. }
  4089. static int
  4090. ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len,
  4091. unsigned long ip, int remove, int reset, int enable)
  4092. {
  4093. struct ftrace_hash **orig_hash;
  4094. struct ftrace_hash *hash;
  4095. int ret;
  4096. if (unlikely(ftrace_disabled))
  4097. return -ENODEV;
  4098. mutex_lock(&ops->func_hash->regex_lock);
  4099. if (enable)
  4100. orig_hash = &ops->func_hash->filter_hash;
  4101. else
  4102. orig_hash = &ops->func_hash->notrace_hash;
  4103. if (reset)
  4104. hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
  4105. else
  4106. hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
  4107. if (!hash) {
  4108. ret = -ENOMEM;
  4109. goto out_regex_unlock;
  4110. }
  4111. if (buf && !ftrace_match_records(hash, buf, len)) {
  4112. ret = -EINVAL;
  4113. goto out_regex_unlock;
  4114. }
  4115. if (ip) {
  4116. ret = ftrace_match_addr(hash, ip, remove);
  4117. if (ret < 0)
  4118. goto out_regex_unlock;
  4119. }
  4120. mutex_lock(&ftrace_lock);
  4121. ret = ftrace_hash_move_and_update_ops(ops, orig_hash, hash, enable);
  4122. mutex_unlock(&ftrace_lock);
  4123. out_regex_unlock:
  4124. mutex_unlock(&ops->func_hash->regex_lock);
  4125. free_ftrace_hash(hash);
  4126. return ret;
  4127. }
  4128. static int
  4129. ftrace_set_addr(struct ftrace_ops *ops, unsigned long ip, int remove,
  4130. int reset, int enable)
  4131. {
  4132. return ftrace_set_hash(ops, NULL, 0, ip, remove, reset, enable);
  4133. }
  4134. #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
  4135. struct ftrace_direct_func {
  4136. struct list_head next;
  4137. unsigned long addr;
  4138. int count;
  4139. };
  4140. static LIST_HEAD(ftrace_direct_funcs);
  4141. /**
  4142. * ftrace_find_direct_func - test an address if it is a registered direct caller
  4143. * @addr: The address of a registered direct caller
  4144. *
  4145. * This searches to see if a ftrace direct caller has been registered
  4146. * at a specific address, and if so, it returns a descriptor for it.
  4147. *
  4148. * This can be used by architecture code to see if an address is
  4149. * a direct caller (trampoline) attached to a fentry/mcount location.
  4150. * This is useful for the function_graph tracer, as it may need to
  4151. * do adjustments if it traced a location that also has a direct
  4152. * trampoline attached to it.
  4153. */
  4154. struct ftrace_direct_func *ftrace_find_direct_func(unsigned long addr)
  4155. {
  4156. struct ftrace_direct_func *entry;
  4157. bool found = false;
  4158. /* May be called by fgraph trampoline (protected by rcu tasks) */
  4159. list_for_each_entry_rcu(entry, &ftrace_direct_funcs, next) {
  4160. if (entry->addr == addr) {
  4161. found = true;
  4162. break;
  4163. }
  4164. }
  4165. if (found)
  4166. return entry;
  4167. return NULL;
  4168. }
  4169. static struct ftrace_direct_func *ftrace_alloc_direct_func(unsigned long addr)
  4170. {
  4171. struct ftrace_direct_func *direct;
  4172. direct = kmalloc(sizeof(*direct), GFP_KERNEL);
  4173. if (!direct)
  4174. return NULL;
  4175. direct->addr = addr;
  4176. direct->count = 0;
  4177. list_add_rcu(&direct->next, &ftrace_direct_funcs);
  4178. ftrace_direct_func_count++;
  4179. return direct;
  4180. }
  4181. /**
  4182. * register_ftrace_direct - Call a custom trampoline directly
  4183. * @ip: The address of the nop at the beginning of a function
  4184. * @addr: The address of the trampoline to call at @ip
  4185. *
  4186. * This is used to connect a direct call from the nop location (@ip)
  4187. * at the start of ftrace traced functions. The location that it calls
  4188. * (@addr) must be able to handle a direct call, and save the parameters
  4189. * of the function being traced, and restore them (or inject new ones
  4190. * if needed), before returning.
  4191. *
  4192. * Returns:
  4193. * 0 on success
  4194. * -EBUSY - Another direct function is already attached (there can be only one)
  4195. * -ENODEV - @ip does not point to a ftrace nop location (or not supported)
  4196. * -ENOMEM - There was an allocation failure.
  4197. */
  4198. int register_ftrace_direct(unsigned long ip, unsigned long addr)
  4199. {
  4200. struct ftrace_direct_func *direct;
  4201. struct ftrace_func_entry *entry;
  4202. struct ftrace_hash *free_hash = NULL;
  4203. struct dyn_ftrace *rec;
  4204. int ret = -EBUSY;
  4205. mutex_lock(&direct_mutex);
  4206. /* See if there's a direct function at @ip already */
  4207. if (ftrace_find_rec_direct(ip))
  4208. goto out_unlock;
  4209. ret = -ENODEV;
  4210. rec = lookup_rec(ip, ip);
  4211. if (!rec)
  4212. goto out_unlock;
  4213. /*
  4214. * Check if the rec says it has a direct call but we didn't
  4215. * find one earlier?
  4216. */
  4217. if (WARN_ON(rec->flags & FTRACE_FL_DIRECT))
  4218. goto out_unlock;
  4219. /* Make sure the ip points to the exact record */
  4220. if (ip != rec->ip) {
  4221. ip = rec->ip;
  4222. /* Need to check this ip for a direct. */
  4223. if (ftrace_find_rec_direct(ip))
  4224. goto out_unlock;
  4225. }
  4226. ret = -ENOMEM;
  4227. if (ftrace_hash_empty(direct_functions) ||
  4228. direct_functions->count > 2 * (1 << direct_functions->size_bits)) {
  4229. struct ftrace_hash *new_hash;
  4230. int size = ftrace_hash_empty(direct_functions) ? 0 :
  4231. direct_functions->count + 1;
  4232. if (size < 32)
  4233. size = 32;
  4234. new_hash = dup_hash(direct_functions, size);
  4235. if (!new_hash)
  4236. goto out_unlock;
  4237. free_hash = direct_functions;
  4238. direct_functions = new_hash;
  4239. }
  4240. entry = kmalloc(sizeof(*entry), GFP_KERNEL);
  4241. if (!entry)
  4242. goto out_unlock;
  4243. direct = ftrace_find_direct_func(addr);
  4244. if (!direct) {
  4245. direct = ftrace_alloc_direct_func(addr);
  4246. if (!direct) {
  4247. kfree(entry);
  4248. goto out_unlock;
  4249. }
  4250. }
  4251. entry->ip = ip;
  4252. entry->direct = addr;
  4253. __add_hash_entry(direct_functions, entry);
  4254. ret = ftrace_set_filter_ip(&direct_ops, ip, 0, 0);
  4255. if (ret)
  4256. remove_hash_entry(direct_functions, entry);
  4257. if (!ret && !(direct_ops.flags & FTRACE_OPS_FL_ENABLED)) {
  4258. ret = register_ftrace_function(&direct_ops);
  4259. if (ret)
  4260. ftrace_set_filter_ip(&direct_ops, ip, 1, 0);
  4261. }
  4262. if (ret) {
  4263. kfree(entry);
  4264. if (!direct->count) {
  4265. list_del_rcu(&direct->next);
  4266. synchronize_rcu_tasks();
  4267. kfree(direct);
  4268. if (free_hash)
  4269. free_ftrace_hash(free_hash);
  4270. free_hash = NULL;
  4271. ftrace_direct_func_count--;
  4272. }
  4273. } else {
  4274. direct->count++;
  4275. }
  4276. out_unlock:
  4277. mutex_unlock(&direct_mutex);
  4278. if (free_hash) {
  4279. synchronize_rcu_tasks();
  4280. free_ftrace_hash(free_hash);
  4281. }
  4282. return ret;
  4283. }
  4284. EXPORT_SYMBOL_GPL(register_ftrace_direct);
  4285. static struct ftrace_func_entry *find_direct_entry(unsigned long *ip,
  4286. struct dyn_ftrace **recp)
  4287. {
  4288. struct ftrace_func_entry *entry;
  4289. struct dyn_ftrace *rec;
  4290. rec = lookup_rec(*ip, *ip);
  4291. if (!rec)
  4292. return NULL;
  4293. entry = __ftrace_lookup_ip(direct_functions, rec->ip);
  4294. if (!entry) {
  4295. WARN_ON(rec->flags & FTRACE_FL_DIRECT);
  4296. return NULL;
  4297. }
  4298. WARN_ON(!(rec->flags & FTRACE_FL_DIRECT));
  4299. /* Passed in ip just needs to be on the call site */
  4300. *ip = rec->ip;
  4301. if (recp)
  4302. *recp = rec;
  4303. return entry;
  4304. }
  4305. int unregister_ftrace_direct(unsigned long ip, unsigned long addr)
  4306. {
  4307. struct ftrace_direct_func *direct;
  4308. struct ftrace_func_entry *entry;
  4309. int ret = -ENODEV;
  4310. mutex_lock(&direct_mutex);
  4311. entry = find_direct_entry(&ip, NULL);
  4312. if (!entry)
  4313. goto out_unlock;
  4314. if (direct_functions->count == 1)
  4315. unregister_ftrace_function(&direct_ops);
  4316. ret = ftrace_set_filter_ip(&direct_ops, ip, 1, 0);
  4317. WARN_ON(ret);
  4318. remove_hash_entry(direct_functions, entry);
  4319. direct = ftrace_find_direct_func(addr);
  4320. if (!WARN_ON(!direct)) {
  4321. /* This is the good path (see the ! before WARN) */
  4322. direct->count--;
  4323. WARN_ON(direct->count < 0);
  4324. if (!direct->count) {
  4325. list_del_rcu(&direct->next);
  4326. synchronize_rcu_tasks();
  4327. kfree(direct);
  4328. kfree(entry);
  4329. ftrace_direct_func_count--;
  4330. }
  4331. }
  4332. out_unlock:
  4333. mutex_unlock(&direct_mutex);
  4334. return ret;
  4335. }
  4336. EXPORT_SYMBOL_GPL(unregister_ftrace_direct);
  4337. static struct ftrace_ops stub_ops = {
  4338. .func = ftrace_stub,
  4339. };
  4340. /**
  4341. * ftrace_modify_direct_caller - modify ftrace nop directly
  4342. * @entry: The ftrace hash entry of the direct helper for @rec
  4343. * @rec: The record representing the function site to patch
  4344. * @old_addr: The location that the site at @rec->ip currently calls
  4345. * @new_addr: The location that the site at @rec->ip should call
  4346. *
  4347. * An architecture may overwrite this function to optimize the
  4348. * changing of the direct callback on an ftrace nop location.
  4349. * This is called with the ftrace_lock mutex held, and no other
  4350. * ftrace callbacks are on the associated record (@rec). Thus,
  4351. * it is safe to modify the ftrace record, where it should be
  4352. * currently calling @old_addr directly, to call @new_addr.
  4353. *
  4354. * Safety checks should be made to make sure that the code at
  4355. * @rec->ip is currently calling @old_addr. And this must
  4356. * also update entry->direct to @new_addr.
  4357. */
  4358. int __weak ftrace_modify_direct_caller(struct ftrace_func_entry *entry,
  4359. struct dyn_ftrace *rec,
  4360. unsigned long old_addr,
  4361. unsigned long new_addr)
  4362. {
  4363. unsigned long ip = rec->ip;
  4364. int ret;
  4365. /*
  4366. * The ftrace_lock was used to determine if the record
  4367. * had more than one registered user to it. If it did,
  4368. * we needed to prevent that from changing to do the quick
  4369. * switch. But if it did not (only a direct caller was attached)
  4370. * then this function is called. But this function can deal
  4371. * with attached callers to the rec that we care about, and
  4372. * since this function uses standard ftrace calls that take
  4373. * the ftrace_lock mutex, we need to release it.
  4374. */
  4375. mutex_unlock(&ftrace_lock);
  4376. /*
  4377. * By setting a stub function at the same address, we force
  4378. * the code to call the iterator and the direct_ops helper.
  4379. * This means that @ip does not call the direct call, and
  4380. * we can simply modify it.
  4381. */
  4382. ret = ftrace_set_filter_ip(&stub_ops, ip, 0, 0);
  4383. if (ret)
  4384. goto out_lock;
  4385. ret = register_ftrace_function(&stub_ops);
  4386. if (ret) {
  4387. ftrace_set_filter_ip(&stub_ops, ip, 1, 0);
  4388. goto out_lock;
  4389. }
  4390. entry->direct = new_addr;
  4391. /*
  4392. * By removing the stub, we put back the direct call, calling
  4393. * the @new_addr.
  4394. */
  4395. unregister_ftrace_function(&stub_ops);
  4396. ftrace_set_filter_ip(&stub_ops, ip, 1, 0);
  4397. out_lock:
  4398. mutex_lock(&ftrace_lock);
  4399. return ret;
  4400. }
  4401. /**
  4402. * modify_ftrace_direct - Modify an existing direct call to call something else
  4403. * @ip: The instruction pointer to modify
  4404. * @old_addr: The address that the current @ip calls directly
  4405. * @new_addr: The address that the @ip should call
  4406. *
  4407. * This modifies a ftrace direct caller at an instruction pointer without
  4408. * having to disable it first. The direct call will switch over to the
  4409. * @new_addr without missing anything.
  4410. *
  4411. * Returns: zero on success. Non zero on error, which includes:
  4412. * -ENODEV : the @ip given has no direct caller attached
  4413. * -EINVAL : the @old_addr does not match the current direct caller
  4414. */
  4415. int modify_ftrace_direct(unsigned long ip,
  4416. unsigned long old_addr, unsigned long new_addr)
  4417. {
  4418. struct ftrace_direct_func *direct, *new_direct = NULL;
  4419. struct ftrace_func_entry *entry;
  4420. struct dyn_ftrace *rec;
  4421. int ret = -ENODEV;
  4422. mutex_lock(&direct_mutex);
  4423. mutex_lock(&ftrace_lock);
  4424. entry = find_direct_entry(&ip, &rec);
  4425. if (!entry)
  4426. goto out_unlock;
  4427. ret = -EINVAL;
  4428. if (entry->direct != old_addr)
  4429. goto out_unlock;
  4430. direct = ftrace_find_direct_func(old_addr);
  4431. if (WARN_ON(!direct))
  4432. goto out_unlock;
  4433. if (direct->count > 1) {
  4434. ret = -ENOMEM;
  4435. new_direct = ftrace_alloc_direct_func(new_addr);
  4436. if (!new_direct)
  4437. goto out_unlock;
  4438. direct->count--;
  4439. new_direct->count++;
  4440. } else {
  4441. direct->addr = new_addr;
  4442. }
  4443. /*
  4444. * If there's no other ftrace callback on the rec->ip location,
  4445. * then it can be changed directly by the architecture.
  4446. * If there is another caller, then we just need to change the
  4447. * direct caller helper to point to @new_addr.
  4448. */
  4449. if (ftrace_rec_count(rec) == 1) {
  4450. ret = ftrace_modify_direct_caller(entry, rec, old_addr, new_addr);
  4451. } else {
  4452. entry->direct = new_addr;
  4453. ret = 0;
  4454. }
  4455. if (unlikely(ret && new_direct)) {
  4456. direct->count++;
  4457. list_del_rcu(&new_direct->next);
  4458. synchronize_rcu_tasks();
  4459. kfree(new_direct);
  4460. ftrace_direct_func_count--;
  4461. }
  4462. out_unlock:
  4463. mutex_unlock(&ftrace_lock);
  4464. mutex_unlock(&direct_mutex);
  4465. return ret;
  4466. }
  4467. EXPORT_SYMBOL_GPL(modify_ftrace_direct);
  4468. #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
  4469. /**
  4470. * ftrace_set_filter_ip - set a function to filter on in ftrace by address
  4471. * @ops - the ops to set the filter with
  4472. * @ip - the address to add to or remove from the filter.
  4473. * @remove - non zero to remove the ip from the filter
  4474. * @reset - non zero to reset all filters before applying this filter.
  4475. *
  4476. * Filters denote which functions should be enabled when tracing is enabled
  4477. * If @ip is NULL, it failes to update filter.
  4478. */
  4479. int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip,
  4480. int remove, int reset)
  4481. {
  4482. ftrace_ops_init(ops);
  4483. return ftrace_set_addr(ops, ip, remove, reset, 1);
  4484. }
  4485. EXPORT_SYMBOL_GPL(ftrace_set_filter_ip);
  4486. /**
  4487. * ftrace_ops_set_global_filter - setup ops to use global filters
  4488. * @ops - the ops which will use the global filters
  4489. *
  4490. * ftrace users who need global function trace filtering should call this.
  4491. * It can set the global filter only if ops were not initialized before.
  4492. */
  4493. void ftrace_ops_set_global_filter(struct ftrace_ops *ops)
  4494. {
  4495. if (ops->flags & FTRACE_OPS_FL_INITIALIZED)
  4496. return;
  4497. ftrace_ops_init(ops);
  4498. ops->func_hash = &global_ops.local_hash;
  4499. }
  4500. EXPORT_SYMBOL_GPL(ftrace_ops_set_global_filter);
  4501. static int
  4502. ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
  4503. int reset, int enable)
  4504. {
  4505. return ftrace_set_hash(ops, buf, len, 0, 0, reset, enable);
  4506. }
  4507. /**
  4508. * ftrace_set_filter - set a function to filter on in ftrace
  4509. * @ops - the ops to set the filter with
  4510. * @buf - the string that holds the function filter text.
  4511. * @len - the length of the string.
  4512. * @reset - non zero to reset all filters before applying this filter.
  4513. *
  4514. * Filters denote which functions should be enabled when tracing is enabled.
  4515. * If @buf is NULL and reset is set, all functions will be enabled for tracing.
  4516. */
  4517. int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
  4518. int len, int reset)
  4519. {
  4520. ftrace_ops_init(ops);
  4521. return ftrace_set_regex(ops, buf, len, reset, 1);
  4522. }
  4523. EXPORT_SYMBOL_GPL(ftrace_set_filter);
  4524. /**
  4525. * ftrace_set_notrace - set a function to not trace in ftrace
  4526. * @ops - the ops to set the notrace filter with
  4527. * @buf - the string that holds the function notrace text.
  4528. * @len - the length of the string.
  4529. * @reset - non zero to reset all filters before applying this filter.
  4530. *
  4531. * Notrace Filters denote which functions should not be enabled when tracing
  4532. * is enabled. If @buf is NULL and reset is set, all functions will be enabled
  4533. * for tracing.
  4534. */
  4535. int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
  4536. int len, int reset)
  4537. {
  4538. ftrace_ops_init(ops);
  4539. return ftrace_set_regex(ops, buf, len, reset, 0);
  4540. }
  4541. EXPORT_SYMBOL_GPL(ftrace_set_notrace);
  4542. /**
  4543. * ftrace_set_global_filter - set a function to filter on with global tracers
  4544. * @buf - the string that holds the function filter text.
  4545. * @len - the length of the string.
  4546. * @reset - non zero to reset all filters before applying this filter.
  4547. *
  4548. * Filters denote which functions should be enabled when tracing is enabled.
  4549. * If @buf is NULL and reset is set, all functions will be enabled for tracing.
  4550. */
  4551. void ftrace_set_global_filter(unsigned char *buf, int len, int reset)
  4552. {
  4553. ftrace_set_regex(&global_ops, buf, len, reset, 1);
  4554. }
  4555. EXPORT_SYMBOL_GPL(ftrace_set_global_filter);
  4556. /**
  4557. * ftrace_set_global_notrace - set a function to not trace with global tracers
  4558. * @buf - the string that holds the function notrace text.
  4559. * @len - the length of the string.
  4560. * @reset - non zero to reset all filters before applying this filter.
  4561. *
  4562. * Notrace Filters denote which functions should not be enabled when tracing
  4563. * is enabled. If @buf is NULL and reset is set, all functions will be enabled
  4564. * for tracing.
  4565. */
  4566. void ftrace_set_global_notrace(unsigned char *buf, int len, int reset)
  4567. {
  4568. ftrace_set_regex(&global_ops, buf, len, reset, 0);
  4569. }
  4570. EXPORT_SYMBOL_GPL(ftrace_set_global_notrace);
  4571. /*
  4572. * command line interface to allow users to set filters on boot up.
  4573. */
  4574. #define FTRACE_FILTER_SIZE COMMAND_LINE_SIZE
  4575. static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
  4576. static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;
  4577. /* Used by function selftest to not test if filter is set */
  4578. bool ftrace_filter_param __initdata;
  4579. static int __init set_ftrace_notrace(char *str)
  4580. {
  4581. ftrace_filter_param = true;
  4582. strlcpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
  4583. return 1;
  4584. }
  4585. __setup("ftrace_notrace=", set_ftrace_notrace);
  4586. static int __init set_ftrace_filter(char *str)
  4587. {
  4588. ftrace_filter_param = true;
  4589. strlcpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
  4590. return 1;
  4591. }
  4592. __setup("ftrace_filter=", set_ftrace_filter);
  4593. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  4594. static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
  4595. static char ftrace_graph_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
  4596. static int ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer);
  4597. static int __init set_graph_function(char *str)
  4598. {
  4599. strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
  4600. return 1;
  4601. }
  4602. __setup("ftrace_graph_filter=", set_graph_function);
  4603. static int __init set_graph_notrace_function(char *str)
  4604. {
  4605. strlcpy(ftrace_graph_notrace_buf, str, FTRACE_FILTER_SIZE);
  4606. return 1;
  4607. }
  4608. __setup("ftrace_graph_notrace=", set_graph_notrace_function);
  4609. static int __init set_graph_max_depth_function(char *str)
  4610. {
  4611. if (!str)
  4612. return 0;
  4613. fgraph_max_depth = simple_strtoul(str, NULL, 0);
  4614. return 1;
  4615. }
  4616. __setup("ftrace_graph_max_depth=", set_graph_max_depth_function);
  4617. static void __init set_ftrace_early_graph(char *buf, int enable)
  4618. {
  4619. int ret;
  4620. char *func;
  4621. struct ftrace_hash *hash;
  4622. hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
  4623. if (MEM_FAIL(!hash, "Failed to allocate hash\n"))
  4624. return;
  4625. while (buf) {
  4626. func = strsep(&buf, ",");
  4627. /* we allow only one expression at a time */
  4628. ret = ftrace_graph_set_hash(hash, func);
  4629. if (ret)
  4630. printk(KERN_DEBUG "ftrace: function %s not "
  4631. "traceable\n", func);
  4632. }
  4633. if (enable)
  4634. ftrace_graph_hash = hash;
  4635. else
  4636. ftrace_graph_notrace_hash = hash;
  4637. }
  4638. #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
  4639. void __init
  4640. ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable)
  4641. {
  4642. char *func;
  4643. ftrace_ops_init(ops);
  4644. while (buf) {
  4645. func = strsep(&buf, ",");
  4646. ftrace_set_regex(ops, func, strlen(func), 0, enable);
  4647. }
  4648. }
  4649. static void __init set_ftrace_early_filters(void)
  4650. {
  4651. if (ftrace_filter_buf[0])
  4652. ftrace_set_early_filter(&global_ops, ftrace_filter_buf, 1);
  4653. if (ftrace_notrace_buf[0])
  4654. ftrace_set_early_filter(&global_ops, ftrace_notrace_buf, 0);
  4655. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  4656. if (ftrace_graph_buf[0])
  4657. set_ftrace_early_graph(ftrace_graph_buf, 1);
  4658. if (ftrace_graph_notrace_buf[0])
  4659. set_ftrace_early_graph(ftrace_graph_notrace_buf, 0);
  4660. #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
  4661. }
  4662. int ftrace_regex_release(struct inode *inode, struct file *file)
  4663. {
  4664. struct seq_file *m = (struct seq_file *)file->private_data;
  4665. struct ftrace_iterator *iter;
  4666. struct ftrace_hash **orig_hash;
  4667. struct trace_parser *parser;
  4668. int filter_hash;
  4669. int ret;
  4670. if (file->f_mode & FMODE_READ) {
  4671. iter = m->private;
  4672. seq_release(inode, file);
  4673. } else
  4674. iter = file->private_data;
  4675. parser = &iter->parser;
  4676. if (trace_parser_loaded(parser)) {
  4677. int enable = !(iter->flags & FTRACE_ITER_NOTRACE);
  4678. ftrace_process_regex(iter, parser->buffer,
  4679. parser->idx, enable);
  4680. }
  4681. trace_parser_put(parser);
  4682. mutex_lock(&iter->ops->func_hash->regex_lock);
  4683. if (file->f_mode & FMODE_WRITE) {
  4684. filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);
  4685. if (filter_hash) {
  4686. orig_hash = &iter->ops->func_hash->filter_hash;
  4687. if (iter->tr && !list_empty(&iter->tr->mod_trace))
  4688. iter->hash->flags |= FTRACE_HASH_FL_MOD;
  4689. } else
  4690. orig_hash = &iter->ops->func_hash->notrace_hash;
  4691. mutex_lock(&ftrace_lock);
  4692. ret = ftrace_hash_move_and_update_ops(iter->ops, orig_hash,
  4693. iter->hash, filter_hash);
  4694. mutex_unlock(&ftrace_lock);
  4695. } else {
  4696. /* For read only, the hash is the ops hash */
  4697. iter->hash = NULL;
  4698. }
  4699. mutex_unlock(&iter->ops->func_hash->regex_lock);
  4700. free_ftrace_hash(iter->hash);
  4701. if (iter->tr)
  4702. trace_array_put(iter->tr);
  4703. kfree(iter);
  4704. return 0;
  4705. }
  4706. static const struct file_operations ftrace_avail_fops = {
  4707. .open = ftrace_avail_open,
  4708. .read = seq_read,
  4709. .llseek = seq_lseek,
  4710. .release = seq_release_private,
  4711. };
  4712. static const struct file_operations ftrace_enabled_fops = {
  4713. .open = ftrace_enabled_open,
  4714. .read = seq_read,
  4715. .llseek = seq_lseek,
  4716. .release = seq_release_private,
  4717. };
  4718. static const struct file_operations ftrace_filter_fops = {
  4719. .open = ftrace_filter_open,
  4720. .read = seq_read,
  4721. .write = ftrace_filter_write,
  4722. .llseek = tracing_lseek,
  4723. .release = ftrace_regex_release,
  4724. };
  4725. static const struct file_operations ftrace_notrace_fops = {
  4726. .open = ftrace_notrace_open,
  4727. .read = seq_read,
  4728. .write = ftrace_notrace_write,
  4729. .llseek = tracing_lseek,
  4730. .release = ftrace_regex_release,
  4731. };
  4732. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  4733. static DEFINE_MUTEX(graph_lock);
  4734. struct ftrace_hash __rcu *ftrace_graph_hash = EMPTY_HASH;
  4735. struct ftrace_hash __rcu *ftrace_graph_notrace_hash = EMPTY_HASH;
  4736. enum graph_filter_type {
  4737. GRAPH_FILTER_NOTRACE = 0,
  4738. GRAPH_FILTER_FUNCTION,
  4739. };
  4740. #define FTRACE_GRAPH_EMPTY ((void *)1)
  4741. struct ftrace_graph_data {
  4742. struct ftrace_hash *hash;
  4743. struct ftrace_func_entry *entry;
  4744. int idx; /* for hash table iteration */
  4745. enum graph_filter_type type;
  4746. struct ftrace_hash *new_hash;
  4747. const struct seq_operations *seq_ops;
  4748. struct trace_parser parser;
  4749. };
  4750. static void *
  4751. __g_next(struct seq_file *m, loff_t *pos)
  4752. {
  4753. struct ftrace_graph_data *fgd = m->private;
  4754. struct ftrace_func_entry *entry = fgd->entry;
  4755. struct hlist_head *head;
  4756. int i, idx = fgd->idx;
  4757. if (*pos >= fgd->hash->count)
  4758. return NULL;
  4759. if (entry) {
  4760. hlist_for_each_entry_continue(entry, hlist) {
  4761. fgd->entry = entry;
  4762. return entry;
  4763. }
  4764. idx++;
  4765. }
  4766. for (i = idx; i < 1 << fgd->hash->size_bits; i++) {
  4767. head = &fgd->hash->buckets[i];
  4768. hlist_for_each_entry(entry, head, hlist) {
  4769. fgd->entry = entry;
  4770. fgd->idx = i;
  4771. return entry;
  4772. }
  4773. }
  4774. return NULL;
  4775. }
  4776. static void *
  4777. g_next(struct seq_file *m, void *v, loff_t *pos)
  4778. {
  4779. (*pos)++;
  4780. return __g_next(m, pos);
  4781. }
  4782. static void *g_start(struct seq_file *m, loff_t *pos)
  4783. {
  4784. struct ftrace_graph_data *fgd = m->private;
  4785. mutex_lock(&graph_lock);
  4786. if (fgd->type == GRAPH_FILTER_FUNCTION)
  4787. fgd->hash = rcu_dereference_protected(ftrace_graph_hash,
  4788. lockdep_is_held(&graph_lock));
  4789. else
  4790. fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
  4791. lockdep_is_held(&graph_lock));
  4792. /* Nothing, tell g_show to print all functions are enabled */
  4793. if (ftrace_hash_empty(fgd->hash) && !*pos)
  4794. return FTRACE_GRAPH_EMPTY;
  4795. fgd->idx = 0;
  4796. fgd->entry = NULL;
  4797. return __g_next(m, pos);
  4798. }
  4799. static void g_stop(struct seq_file *m, void *p)
  4800. {
  4801. mutex_unlock(&graph_lock);
  4802. }
  4803. static int g_show(struct seq_file *m, void *v)
  4804. {
  4805. struct ftrace_func_entry *entry = v;
  4806. if (!entry)
  4807. return 0;
  4808. if (entry == FTRACE_GRAPH_EMPTY) {
  4809. struct ftrace_graph_data *fgd = m->private;
  4810. if (fgd->type == GRAPH_FILTER_FUNCTION)
  4811. seq_puts(m, "#### all functions enabled ####\n");
  4812. else
  4813. seq_puts(m, "#### no functions disabled ####\n");
  4814. return 0;
  4815. }
  4816. seq_printf(m, "%ps\n", (void *)entry->ip);
  4817. return 0;
  4818. }
  4819. static const struct seq_operations ftrace_graph_seq_ops = {
  4820. .start = g_start,
  4821. .next = g_next,
  4822. .stop = g_stop,
  4823. .show = g_show,
  4824. };
  4825. static int
  4826. __ftrace_graph_open(struct inode *inode, struct file *file,
  4827. struct ftrace_graph_data *fgd)
  4828. {
  4829. int ret;
  4830. struct ftrace_hash *new_hash = NULL;
  4831. ret = security_locked_down(LOCKDOWN_TRACEFS);
  4832. if (ret)
  4833. return ret;
  4834. if (file->f_mode & FMODE_WRITE) {
  4835. const int size_bits = FTRACE_HASH_DEFAULT_BITS;
  4836. if (trace_parser_get_init(&fgd->parser, FTRACE_BUFF_MAX))
  4837. return -ENOMEM;
  4838. if (file->f_flags & O_TRUNC)
  4839. new_hash = alloc_ftrace_hash(size_bits);
  4840. else
  4841. new_hash = alloc_and_copy_ftrace_hash(size_bits,
  4842. fgd->hash);
  4843. if (!new_hash) {
  4844. ret = -ENOMEM;
  4845. goto out;
  4846. }
  4847. }
  4848. if (file->f_mode & FMODE_READ) {
  4849. ret = seq_open(file, &ftrace_graph_seq_ops);
  4850. if (!ret) {
  4851. struct seq_file *m = file->private_data;
  4852. m->private = fgd;
  4853. } else {
  4854. /* Failed */
  4855. free_ftrace_hash(new_hash);
  4856. new_hash = NULL;
  4857. }
  4858. } else
  4859. file->private_data = fgd;
  4860. out:
  4861. if (ret < 0 && file->f_mode & FMODE_WRITE)
  4862. trace_parser_put(&fgd->parser);
  4863. fgd->new_hash = new_hash;
  4864. /*
  4865. * All uses of fgd->hash must be taken with the graph_lock
  4866. * held. The graph_lock is going to be released, so force
  4867. * fgd->hash to be reinitialized when it is taken again.
  4868. */
  4869. fgd->hash = NULL;
  4870. return ret;
  4871. }
  4872. static int
  4873. ftrace_graph_open(struct inode *inode, struct file *file)
  4874. {
  4875. struct ftrace_graph_data *fgd;
  4876. int ret;
  4877. if (unlikely(ftrace_disabled))
  4878. return -ENODEV;
  4879. fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
  4880. if (fgd == NULL)
  4881. return -ENOMEM;
  4882. mutex_lock(&graph_lock);
  4883. fgd->hash = rcu_dereference_protected(ftrace_graph_hash,
  4884. lockdep_is_held(&graph_lock));
  4885. fgd->type = GRAPH_FILTER_FUNCTION;
  4886. fgd->seq_ops = &ftrace_graph_seq_ops;
  4887. ret = __ftrace_graph_open(inode, file, fgd);
  4888. if (ret < 0)
  4889. kfree(fgd);
  4890. mutex_unlock(&graph_lock);
  4891. return ret;
  4892. }
  4893. static int
  4894. ftrace_graph_notrace_open(struct inode *inode, struct file *file)
  4895. {
  4896. struct ftrace_graph_data *fgd;
  4897. int ret;
  4898. if (unlikely(ftrace_disabled))
  4899. return -ENODEV;
  4900. fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
  4901. if (fgd == NULL)
  4902. return -ENOMEM;
  4903. mutex_lock(&graph_lock);
  4904. fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
  4905. lockdep_is_held(&graph_lock));
  4906. fgd->type = GRAPH_FILTER_NOTRACE;
  4907. fgd->seq_ops = &ftrace_graph_seq_ops;
  4908. ret = __ftrace_graph_open(inode, file, fgd);
  4909. if (ret < 0)
  4910. kfree(fgd);
  4911. mutex_unlock(&graph_lock);
  4912. return ret;
  4913. }
  4914. static int
  4915. ftrace_graph_release(struct inode *inode, struct file *file)
  4916. {
  4917. struct ftrace_graph_data *fgd;
  4918. struct ftrace_hash *old_hash, *new_hash;
  4919. struct trace_parser *parser;
  4920. int ret = 0;
  4921. if (file->f_mode & FMODE_READ) {
  4922. struct seq_file *m = file->private_data;
  4923. fgd = m->private;
  4924. seq_release(inode, file);
  4925. } else {
  4926. fgd = file->private_data;
  4927. }
  4928. if (file->f_mode & FMODE_WRITE) {
  4929. parser = &fgd->parser;
  4930. if (trace_parser_loaded((parser))) {
  4931. ret = ftrace_graph_set_hash(fgd->new_hash,
  4932. parser->buffer);
  4933. }
  4934. trace_parser_put(parser);
  4935. new_hash = __ftrace_hash_move(fgd->new_hash);
  4936. if (!new_hash) {
  4937. ret = -ENOMEM;
  4938. goto out;
  4939. }
  4940. mutex_lock(&graph_lock);
  4941. if (fgd->type == GRAPH_FILTER_FUNCTION) {
  4942. old_hash = rcu_dereference_protected(ftrace_graph_hash,
  4943. lockdep_is_held(&graph_lock));
  4944. rcu_assign_pointer(ftrace_graph_hash, new_hash);
  4945. } else {
  4946. old_hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
  4947. lockdep_is_held(&graph_lock));
  4948. rcu_assign_pointer(ftrace_graph_notrace_hash, new_hash);
  4949. }
  4950. mutex_unlock(&graph_lock);
  4951. /*
  4952. * We need to do a hard force of sched synchronization.
  4953. * This is because we use preempt_disable() to do RCU, but
  4954. * the function tracers can be called where RCU is not watching
  4955. * (like before user_exit()). We can not rely on the RCU
  4956. * infrastructure to do the synchronization, thus we must do it
  4957. * ourselves.
  4958. */
  4959. synchronize_rcu_tasks_rude();
  4960. free_ftrace_hash(old_hash);
  4961. }
  4962. out:
  4963. free_ftrace_hash(fgd->new_hash);
  4964. kfree(fgd);
  4965. return ret;
  4966. }
  4967. static int
  4968. ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer)
  4969. {
  4970. struct ftrace_glob func_g;
  4971. struct dyn_ftrace *rec;
  4972. struct ftrace_page *pg;
  4973. struct ftrace_func_entry *entry;
  4974. int fail = 1;
  4975. int not;
  4976. /* decode regex */
  4977. func_g.type = filter_parse_regex(buffer, strlen(buffer),
  4978. &func_g.search, &not);
  4979. func_g.len = strlen(func_g.search);
  4980. mutex_lock(&ftrace_lock);
  4981. if (unlikely(ftrace_disabled)) {
  4982. mutex_unlock(&ftrace_lock);
  4983. return -ENODEV;
  4984. }
  4985. do_for_each_ftrace_rec(pg, rec) {
  4986. if (rec->flags & FTRACE_FL_DISABLED)
  4987. continue;
  4988. if (ftrace_match_record(rec, &func_g, NULL, 0)) {
  4989. entry = ftrace_lookup_ip(hash, rec->ip);
  4990. if (!not) {
  4991. fail = 0;
  4992. if (entry)
  4993. continue;
  4994. if (add_hash_entry(hash, rec->ip) < 0)
  4995. goto out;
  4996. } else {
  4997. if (entry) {
  4998. free_hash_entry(hash, entry);
  4999. fail = 0;
  5000. }
  5001. }
  5002. }
  5003. } while_for_each_ftrace_rec();
  5004. out:
  5005. mutex_unlock(&ftrace_lock);
  5006. if (fail)
  5007. return -EINVAL;
  5008. return 0;
  5009. }
  5010. static ssize_t
  5011. ftrace_graph_write(struct file *file, const char __user *ubuf,
  5012. size_t cnt, loff_t *ppos)
  5013. {
  5014. ssize_t read, ret = 0;
  5015. struct ftrace_graph_data *fgd = file->private_data;
  5016. struct trace_parser *parser;
  5017. if (!cnt)
  5018. return 0;
  5019. /* Read mode uses seq functions */
  5020. if (file->f_mode & FMODE_READ) {
  5021. struct seq_file *m = file->private_data;
  5022. fgd = m->private;
  5023. }
  5024. parser = &fgd->parser;
  5025. read = trace_get_user(parser, ubuf, cnt, ppos);
  5026. if (read >= 0 && trace_parser_loaded(parser) &&
  5027. !trace_parser_cont(parser)) {
  5028. ret = ftrace_graph_set_hash(fgd->new_hash,
  5029. parser->buffer);
  5030. trace_parser_clear(parser);
  5031. }
  5032. if (!ret)
  5033. ret = read;
  5034. return ret;
  5035. }
  5036. static const struct file_operations ftrace_graph_fops = {
  5037. .open = ftrace_graph_open,
  5038. .read = seq_read,
  5039. .write = ftrace_graph_write,
  5040. .llseek = tracing_lseek,
  5041. .release = ftrace_graph_release,
  5042. };
  5043. static const struct file_operations ftrace_graph_notrace_fops = {
  5044. .open = ftrace_graph_notrace_open,
  5045. .read = seq_read,
  5046. .write = ftrace_graph_write,
  5047. .llseek = tracing_lseek,
  5048. .release = ftrace_graph_release,
  5049. };
  5050. #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
  5051. void ftrace_create_filter_files(struct ftrace_ops *ops,
  5052. struct dentry *parent)
  5053. {
  5054. trace_create_file("set_ftrace_filter", 0644, parent,
  5055. ops, &ftrace_filter_fops);
  5056. trace_create_file("set_ftrace_notrace", 0644, parent,
  5057. ops, &ftrace_notrace_fops);
  5058. }
  5059. /*
  5060. * The name "destroy_filter_files" is really a misnomer. Although
  5061. * in the future, it may actually delete the files, but this is
  5062. * really intended to make sure the ops passed in are disabled
  5063. * and that when this function returns, the caller is free to
  5064. * free the ops.
  5065. *
  5066. * The "destroy" name is only to match the "create" name that this
  5067. * should be paired with.
  5068. */
  5069. void ftrace_destroy_filter_files(struct ftrace_ops *ops)
  5070. {
  5071. mutex_lock(&ftrace_lock);
  5072. if (ops->flags & FTRACE_OPS_FL_ENABLED)
  5073. ftrace_shutdown(ops, 0);
  5074. ops->flags |= FTRACE_OPS_FL_DELETED;
  5075. ftrace_free_filter(ops);
  5076. mutex_unlock(&ftrace_lock);
  5077. }
  5078. static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer)
  5079. {
  5080. trace_create_file("available_filter_functions", 0444,
  5081. d_tracer, NULL, &ftrace_avail_fops);
  5082. trace_create_file("enabled_functions", 0444,
  5083. d_tracer, NULL, &ftrace_enabled_fops);
  5084. ftrace_create_filter_files(&global_ops, d_tracer);
  5085. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  5086. trace_create_file("set_graph_function", 0644, d_tracer,
  5087. NULL,
  5088. &ftrace_graph_fops);
  5089. trace_create_file("set_graph_notrace", 0644, d_tracer,
  5090. NULL,
  5091. &ftrace_graph_notrace_fops);
  5092. #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
  5093. return 0;
  5094. }
  5095. static int ftrace_cmp_ips(const void *a, const void *b)
  5096. {
  5097. const unsigned long *ipa = a;
  5098. const unsigned long *ipb = b;
  5099. if (*ipa > *ipb)
  5100. return 1;
  5101. if (*ipa < *ipb)
  5102. return -1;
  5103. return 0;
  5104. }
  5105. static int ftrace_process_locs(struct module *mod,
  5106. unsigned long *start,
  5107. unsigned long *end)
  5108. {
  5109. struct ftrace_page *start_pg;
  5110. struct ftrace_page *pg;
  5111. struct dyn_ftrace *rec;
  5112. unsigned long count;
  5113. unsigned long *p;
  5114. unsigned long addr;
  5115. unsigned long flags = 0; /* Shut up gcc */
  5116. int ret = -ENOMEM;
  5117. count = end - start;
  5118. if (!count)
  5119. return 0;
  5120. sort(start, count, sizeof(*start),
  5121. ftrace_cmp_ips, NULL);
  5122. start_pg = ftrace_allocate_pages(count);
  5123. if (!start_pg)
  5124. return -ENOMEM;
  5125. mutex_lock(&ftrace_lock);
  5126. /*
  5127. * Core and each module needs their own pages, as
  5128. * modules will free them when they are removed.
  5129. * Force a new page to be allocated for modules.
  5130. */
  5131. if (!mod) {
  5132. WARN_ON(ftrace_pages || ftrace_pages_start);
  5133. /* First initialization */
  5134. ftrace_pages = ftrace_pages_start = start_pg;
  5135. } else {
  5136. if (!ftrace_pages)
  5137. goto out;
  5138. if (WARN_ON(ftrace_pages->next)) {
  5139. /* Hmm, we have free pages? */
  5140. while (ftrace_pages->next)
  5141. ftrace_pages = ftrace_pages->next;
  5142. }
  5143. ftrace_pages->next = start_pg;
  5144. }
  5145. p = start;
  5146. pg = start_pg;
  5147. while (p < end) {
  5148. addr = ftrace_call_adjust(*p++);
  5149. /*
  5150. * Some architecture linkers will pad between
  5151. * the different mcount_loc sections of different
  5152. * object files to satisfy alignments.
  5153. * Skip any NULL pointers.
  5154. */
  5155. if (!addr)
  5156. continue;
  5157. if (pg->index == pg->size) {
  5158. /* We should have allocated enough */
  5159. if (WARN_ON(!pg->next))
  5160. break;
  5161. pg = pg->next;
  5162. }
  5163. rec = &pg->records[pg->index++];
  5164. rec->ip = addr;
  5165. }
  5166. /* We should have used all pages */
  5167. WARN_ON(pg->next);
  5168. /* Assign the last page to ftrace_pages */
  5169. ftrace_pages = pg;
  5170. /*
  5171. * We only need to disable interrupts on start up
  5172. * because we are modifying code that an interrupt
  5173. * may execute, and the modification is not atomic.
  5174. * But for modules, nothing runs the code we modify
  5175. * until we are finished with it, and there's no
  5176. * reason to cause large interrupt latencies while we do it.
  5177. */
  5178. if (!mod)
  5179. local_irq_save(flags);
  5180. ftrace_update_code(mod, start_pg);
  5181. if (!mod)
  5182. local_irq_restore(flags);
  5183. ret = 0;
  5184. out:
  5185. mutex_unlock(&ftrace_lock);
  5186. return ret;
  5187. }
  5188. struct ftrace_mod_func {
  5189. struct list_head list;
  5190. char *name;
  5191. unsigned long ip;
  5192. unsigned int size;
  5193. };
  5194. struct ftrace_mod_map {
  5195. struct rcu_head rcu;
  5196. struct list_head list;
  5197. struct module *mod;
  5198. unsigned long start_addr;
  5199. unsigned long end_addr;
  5200. struct list_head funcs;
  5201. unsigned int num_funcs;
  5202. };
  5203. static int ftrace_get_trampoline_kallsym(unsigned int symnum,
  5204. unsigned long *value, char *type,
  5205. char *name, char *module_name,
  5206. int *exported)
  5207. {
  5208. struct ftrace_ops *op;
  5209. list_for_each_entry_rcu(op, &ftrace_ops_trampoline_list, list) {
  5210. if (!op->trampoline || symnum--)
  5211. continue;
  5212. *value = op->trampoline;
  5213. *type = 't';
  5214. strlcpy(name, FTRACE_TRAMPOLINE_SYM, KSYM_NAME_LEN);
  5215. strlcpy(module_name, FTRACE_TRAMPOLINE_MOD, MODULE_NAME_LEN);
  5216. *exported = 0;
  5217. return 0;
  5218. }
  5219. return -ERANGE;
  5220. }
  5221. #ifdef CONFIG_MODULES
  5222. #define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next)
  5223. static LIST_HEAD(ftrace_mod_maps);
  5224. static int referenced_filters(struct dyn_ftrace *rec)
  5225. {
  5226. struct ftrace_ops *ops;
  5227. int cnt = 0;
  5228. for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) {
  5229. if (ops_references_rec(ops, rec)) {
  5230. if (WARN_ON_ONCE(ops->flags & FTRACE_OPS_FL_DIRECT))
  5231. continue;
  5232. if (WARN_ON_ONCE(ops->flags & FTRACE_OPS_FL_IPMODIFY))
  5233. continue;
  5234. cnt++;
  5235. if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
  5236. rec->flags |= FTRACE_FL_REGS;
  5237. if (cnt == 1 && ops->trampoline)
  5238. rec->flags |= FTRACE_FL_TRAMP;
  5239. else
  5240. rec->flags &= ~FTRACE_FL_TRAMP;
  5241. }
  5242. }
  5243. return cnt;
  5244. }
  5245. static void
  5246. clear_mod_from_hash(struct ftrace_page *pg, struct ftrace_hash *hash)
  5247. {
  5248. struct ftrace_func_entry *entry;
  5249. struct dyn_ftrace *rec;
  5250. int i;
  5251. if (ftrace_hash_empty(hash))
  5252. return;
  5253. for (i = 0; i < pg->index; i++) {
  5254. rec = &pg->records[i];
  5255. entry = __ftrace_lookup_ip(hash, rec->ip);
  5256. /*
  5257. * Do not allow this rec to match again.
  5258. * Yeah, it may waste some memory, but will be removed
  5259. * if/when the hash is modified again.
  5260. */
  5261. if (entry)
  5262. entry->ip = 0;
  5263. }
  5264. }
  5265. /* Clear any records from hashs */
  5266. static void clear_mod_from_hashes(struct ftrace_page *pg)
  5267. {
  5268. struct trace_array *tr;
  5269. mutex_lock(&trace_types_lock);
  5270. list_for_each_entry(tr, &ftrace_trace_arrays, list) {
  5271. if (!tr->ops || !tr->ops->func_hash)
  5272. continue;
  5273. mutex_lock(&tr->ops->func_hash->regex_lock);
  5274. clear_mod_from_hash(pg, tr->ops->func_hash->filter_hash);
  5275. clear_mod_from_hash(pg, tr->ops->func_hash->notrace_hash);
  5276. mutex_unlock(&tr->ops->func_hash->regex_lock);
  5277. }
  5278. mutex_unlock(&trace_types_lock);
  5279. }
  5280. static void ftrace_free_mod_map(struct rcu_head *rcu)
  5281. {
  5282. struct ftrace_mod_map *mod_map = container_of(rcu, struct ftrace_mod_map, rcu);
  5283. struct ftrace_mod_func *mod_func;
  5284. struct ftrace_mod_func *n;
  5285. /* All the contents of mod_map are now not visible to readers */
  5286. list_for_each_entry_safe(mod_func, n, &mod_map->funcs, list) {
  5287. kfree(mod_func->name);
  5288. list_del(&mod_func->list);
  5289. kfree(mod_func);
  5290. }
  5291. kfree(mod_map);
  5292. }
  5293. void ftrace_release_mod(struct module *mod)
  5294. {
  5295. struct ftrace_mod_map *mod_map;
  5296. struct ftrace_mod_map *n;
  5297. struct dyn_ftrace *rec;
  5298. struct ftrace_page **last_pg;
  5299. struct ftrace_page *tmp_page = NULL;
  5300. struct ftrace_page *pg;
  5301. int order;
  5302. mutex_lock(&ftrace_lock);
  5303. if (ftrace_disabled)
  5304. goto out_unlock;
  5305. list_for_each_entry_safe(mod_map, n, &ftrace_mod_maps, list) {
  5306. if (mod_map->mod == mod) {
  5307. list_del_rcu(&mod_map->list);
  5308. call_rcu(&mod_map->rcu, ftrace_free_mod_map);
  5309. break;
  5310. }
  5311. }
  5312. /*
  5313. * Each module has its own ftrace_pages, remove
  5314. * them from the list.
  5315. */
  5316. last_pg = &ftrace_pages_start;
  5317. for (pg = ftrace_pages_start; pg; pg = *last_pg) {
  5318. rec = &pg->records[0];
  5319. if (within_module_core(rec->ip, mod) ||
  5320. within_module_init(rec->ip, mod)) {
  5321. /*
  5322. * As core pages are first, the first
  5323. * page should never be a module page.
  5324. */
  5325. if (WARN_ON(pg == ftrace_pages_start))
  5326. goto out_unlock;
  5327. /* Check if we are deleting the last page */
  5328. if (pg == ftrace_pages)
  5329. ftrace_pages = next_to_ftrace_page(last_pg);
  5330. ftrace_update_tot_cnt -= pg->index;
  5331. *last_pg = pg->next;
  5332. pg->next = tmp_page;
  5333. tmp_page = pg;
  5334. } else
  5335. last_pg = &pg->next;
  5336. }
  5337. out_unlock:
  5338. mutex_unlock(&ftrace_lock);
  5339. for (pg = tmp_page; pg; pg = tmp_page) {
  5340. /* Needs to be called outside of ftrace_lock */
  5341. clear_mod_from_hashes(pg);
  5342. order = get_count_order(pg->size / ENTRIES_PER_PAGE);
  5343. if (order >= 0)
  5344. free_pages((unsigned long)pg->records, order);
  5345. tmp_page = pg->next;
  5346. kfree(pg);
  5347. ftrace_number_of_pages -= 1 << order;
  5348. ftrace_number_of_groups--;
  5349. }
  5350. }
  5351. void ftrace_module_enable(struct module *mod)
  5352. {
  5353. struct dyn_ftrace *rec;
  5354. struct ftrace_page *pg;
  5355. mutex_lock(&ftrace_lock);
  5356. if (ftrace_disabled)
  5357. goto out_unlock;
  5358. /*
  5359. * If the tracing is enabled, go ahead and enable the record.
  5360. *
  5361. * The reason not to enable the record immediately is the
  5362. * inherent check of ftrace_make_nop/ftrace_make_call for
  5363. * correct previous instructions. Making first the NOP
  5364. * conversion puts the module to the correct state, thus
  5365. * passing the ftrace_make_call check.
  5366. *
  5367. * We also delay this to after the module code already set the
  5368. * text to read-only, as we now need to set it back to read-write
  5369. * so that we can modify the text.
  5370. */
  5371. if (ftrace_start_up)
  5372. ftrace_arch_code_modify_prepare();
  5373. do_for_each_ftrace_rec(pg, rec) {
  5374. int cnt;
  5375. /*
  5376. * do_for_each_ftrace_rec() is a double loop.
  5377. * module text shares the pg. If a record is
  5378. * not part of this module, then skip this pg,
  5379. * which the "break" will do.
  5380. */
  5381. if (!within_module_core(rec->ip, mod) &&
  5382. !within_module_init(rec->ip, mod))
  5383. break;
  5384. cnt = 0;
  5385. /*
  5386. * When adding a module, we need to check if tracers are
  5387. * currently enabled and if they are, and can trace this record,
  5388. * we need to enable the module functions as well as update the
  5389. * reference counts for those function records.
  5390. */
  5391. if (ftrace_start_up)
  5392. cnt += referenced_filters(rec);
  5393. rec->flags &= ~FTRACE_FL_DISABLED;
  5394. rec->flags += cnt;
  5395. if (ftrace_start_up && cnt) {
  5396. int failed = __ftrace_replace_code(rec, 1);
  5397. if (failed) {
  5398. ftrace_bug(failed, rec);
  5399. goto out_loop;
  5400. }
  5401. }
  5402. } while_for_each_ftrace_rec();
  5403. out_loop:
  5404. if (ftrace_start_up)
  5405. ftrace_arch_code_modify_post_process();
  5406. out_unlock:
  5407. mutex_unlock(&ftrace_lock);
  5408. process_cached_mods(mod->name);
  5409. }
  5410. void ftrace_module_init(struct module *mod)
  5411. {
  5412. if (ftrace_disabled || !mod->num_ftrace_callsites)
  5413. return;
  5414. ftrace_process_locs(mod, mod->ftrace_callsites,
  5415. mod->ftrace_callsites + mod->num_ftrace_callsites);
  5416. }
  5417. static void save_ftrace_mod_rec(struct ftrace_mod_map *mod_map,
  5418. struct dyn_ftrace *rec)
  5419. {
  5420. struct ftrace_mod_func *mod_func;
  5421. unsigned long symsize;
  5422. unsigned long offset;
  5423. char str[KSYM_SYMBOL_LEN];
  5424. char *modname;
  5425. const char *ret;
  5426. ret = kallsyms_lookup(rec->ip, &symsize, &offset, &modname, str);
  5427. if (!ret)
  5428. return;
  5429. mod_func = kmalloc(sizeof(*mod_func), GFP_KERNEL);
  5430. if (!mod_func)
  5431. return;
  5432. mod_func->name = kstrdup(str, GFP_KERNEL);
  5433. if (!mod_func->name) {
  5434. kfree(mod_func);
  5435. return;
  5436. }
  5437. mod_func->ip = rec->ip - offset;
  5438. mod_func->size = symsize;
  5439. mod_map->num_funcs++;
  5440. list_add_rcu(&mod_func->list, &mod_map->funcs);
  5441. }
  5442. static struct ftrace_mod_map *
  5443. allocate_ftrace_mod_map(struct module *mod,
  5444. unsigned long start, unsigned long end)
  5445. {
  5446. struct ftrace_mod_map *mod_map;
  5447. mod_map = kmalloc(sizeof(*mod_map), GFP_KERNEL);
  5448. if (!mod_map)
  5449. return NULL;
  5450. mod_map->mod = mod;
  5451. mod_map->start_addr = start;
  5452. mod_map->end_addr = end;
  5453. mod_map->num_funcs = 0;
  5454. INIT_LIST_HEAD_RCU(&mod_map->funcs);
  5455. list_add_rcu(&mod_map->list, &ftrace_mod_maps);
  5456. return mod_map;
  5457. }
  5458. static const char *
  5459. ftrace_func_address_lookup(struct ftrace_mod_map *mod_map,
  5460. unsigned long addr, unsigned long *size,
  5461. unsigned long *off, char *sym)
  5462. {
  5463. struct ftrace_mod_func *found_func = NULL;
  5464. struct ftrace_mod_func *mod_func;
  5465. list_for_each_entry_rcu(mod_func, &mod_map->funcs, list) {
  5466. if (addr >= mod_func->ip &&
  5467. addr < mod_func->ip + mod_func->size) {
  5468. found_func = mod_func;
  5469. break;
  5470. }
  5471. }
  5472. if (found_func) {
  5473. if (size)
  5474. *size = found_func->size;
  5475. if (off)
  5476. *off = addr - found_func->ip;
  5477. if (sym)
  5478. strlcpy(sym, found_func->name, KSYM_NAME_LEN);
  5479. return found_func->name;
  5480. }
  5481. return NULL;
  5482. }
  5483. const char *
  5484. ftrace_mod_address_lookup(unsigned long addr, unsigned long *size,
  5485. unsigned long *off, char **modname, char *sym)
  5486. {
  5487. struct ftrace_mod_map *mod_map;
  5488. const char *ret = NULL;
  5489. /* mod_map is freed via call_rcu() */
  5490. preempt_disable();
  5491. list_for_each_entry_rcu(mod_map, &ftrace_mod_maps, list) {
  5492. ret = ftrace_func_address_lookup(mod_map, addr, size, off, sym);
  5493. if (ret) {
  5494. if (modname)
  5495. *modname = mod_map->mod->name;
  5496. break;
  5497. }
  5498. }
  5499. preempt_enable();
  5500. return ret;
  5501. }
  5502. int ftrace_mod_get_kallsym(unsigned int symnum, unsigned long *value,
  5503. char *type, char *name,
  5504. char *module_name, int *exported)
  5505. {
  5506. struct ftrace_mod_map *mod_map;
  5507. struct ftrace_mod_func *mod_func;
  5508. int ret;
  5509. preempt_disable();
  5510. list_for_each_entry_rcu(mod_map, &ftrace_mod_maps, list) {
  5511. if (symnum >= mod_map->num_funcs) {
  5512. symnum -= mod_map->num_funcs;
  5513. continue;
  5514. }
  5515. list_for_each_entry_rcu(mod_func, &mod_map->funcs, list) {
  5516. if (symnum > 1) {
  5517. symnum--;
  5518. continue;
  5519. }
  5520. *value = mod_func->ip;
  5521. *type = 'T';
  5522. strlcpy(name, mod_func->name, KSYM_NAME_LEN);
  5523. strlcpy(module_name, mod_map->mod->name, MODULE_NAME_LEN);
  5524. *exported = 1;
  5525. preempt_enable();
  5526. return 0;
  5527. }
  5528. WARN_ON(1);
  5529. break;
  5530. }
  5531. ret = ftrace_get_trampoline_kallsym(symnum, value, type, name,
  5532. module_name, exported);
  5533. preempt_enable();
  5534. return ret;
  5535. }
  5536. #else
  5537. static void save_ftrace_mod_rec(struct ftrace_mod_map *mod_map,
  5538. struct dyn_ftrace *rec) { }
  5539. static inline struct ftrace_mod_map *
  5540. allocate_ftrace_mod_map(struct module *mod,
  5541. unsigned long start, unsigned long end)
  5542. {
  5543. return NULL;
  5544. }
  5545. int ftrace_mod_get_kallsym(unsigned int symnum, unsigned long *value,
  5546. char *type, char *name, char *module_name,
  5547. int *exported)
  5548. {
  5549. int ret;
  5550. preempt_disable();
  5551. ret = ftrace_get_trampoline_kallsym(symnum, value, type, name,
  5552. module_name, exported);
  5553. preempt_enable();
  5554. return ret;
  5555. }
  5556. #endif /* CONFIG_MODULES */
  5557. struct ftrace_init_func {
  5558. struct list_head list;
  5559. unsigned long ip;
  5560. };
  5561. /* Clear any init ips from hashes */
  5562. static void
  5563. clear_func_from_hash(struct ftrace_init_func *func, struct ftrace_hash *hash)
  5564. {
  5565. struct ftrace_func_entry *entry;
  5566. entry = ftrace_lookup_ip(hash, func->ip);
  5567. /*
  5568. * Do not allow this rec to match again.
  5569. * Yeah, it may waste some memory, but will be removed
  5570. * if/when the hash is modified again.
  5571. */
  5572. if (entry)
  5573. entry->ip = 0;
  5574. }
  5575. static void
  5576. clear_func_from_hashes(struct ftrace_init_func *func)
  5577. {
  5578. struct trace_array *tr;
  5579. mutex_lock(&trace_types_lock);
  5580. list_for_each_entry(tr, &ftrace_trace_arrays, list) {
  5581. if (!tr->ops || !tr->ops->func_hash)
  5582. continue;
  5583. mutex_lock(&tr->ops->func_hash->regex_lock);
  5584. clear_func_from_hash(func, tr->ops->func_hash->filter_hash);
  5585. clear_func_from_hash(func, tr->ops->func_hash->notrace_hash);
  5586. mutex_unlock(&tr->ops->func_hash->regex_lock);
  5587. }
  5588. mutex_unlock(&trace_types_lock);
  5589. }
  5590. static void add_to_clear_hash_list(struct list_head *clear_list,
  5591. struct dyn_ftrace *rec)
  5592. {
  5593. struct ftrace_init_func *func;
  5594. func = kmalloc(sizeof(*func), GFP_KERNEL);
  5595. if (!func) {
  5596. MEM_FAIL(1, "alloc failure, ftrace filter could be stale\n");
  5597. return;
  5598. }
  5599. func->ip = rec->ip;
  5600. list_add(&func->list, clear_list);
  5601. }
  5602. void ftrace_free_mem(struct module *mod, void *start_ptr, void *end_ptr)
  5603. {
  5604. unsigned long start = (unsigned long)(start_ptr);
  5605. unsigned long end = (unsigned long)(end_ptr);
  5606. struct ftrace_page **last_pg = &ftrace_pages_start;
  5607. struct ftrace_page *pg;
  5608. struct dyn_ftrace *rec;
  5609. struct dyn_ftrace key;
  5610. struct ftrace_mod_map *mod_map = NULL;
  5611. struct ftrace_init_func *func, *func_next;
  5612. struct list_head clear_hash;
  5613. int order;
  5614. INIT_LIST_HEAD(&clear_hash);
  5615. key.ip = start;
  5616. key.flags = end; /* overload flags, as it is unsigned long */
  5617. mutex_lock(&ftrace_lock);
  5618. /*
  5619. * If we are freeing module init memory, then check if
  5620. * any tracer is active. If so, we need to save a mapping of
  5621. * the module functions being freed with the address.
  5622. */
  5623. if (mod && ftrace_ops_list != &ftrace_list_end)
  5624. mod_map = allocate_ftrace_mod_map(mod, start, end);
  5625. for (pg = ftrace_pages_start; pg; last_pg = &pg->next, pg = *last_pg) {
  5626. if (end < pg->records[0].ip ||
  5627. start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
  5628. continue;
  5629. again:
  5630. rec = bsearch(&key, pg->records, pg->index,
  5631. sizeof(struct dyn_ftrace),
  5632. ftrace_cmp_recs);
  5633. if (!rec)
  5634. continue;
  5635. /* rec will be cleared from hashes after ftrace_lock unlock */
  5636. add_to_clear_hash_list(&clear_hash, rec);
  5637. if (mod_map)
  5638. save_ftrace_mod_rec(mod_map, rec);
  5639. pg->index--;
  5640. ftrace_update_tot_cnt--;
  5641. if (!pg->index) {
  5642. *last_pg = pg->next;
  5643. order = get_count_order(pg->size / ENTRIES_PER_PAGE);
  5644. if (order >= 0)
  5645. free_pages((unsigned long)pg->records, order);
  5646. ftrace_number_of_pages -= 1 << order;
  5647. ftrace_number_of_groups--;
  5648. kfree(pg);
  5649. pg = container_of(last_pg, struct ftrace_page, next);
  5650. if (!(*last_pg))
  5651. ftrace_pages = pg;
  5652. continue;
  5653. }
  5654. memmove(rec, rec + 1,
  5655. (pg->index - (rec - pg->records)) * sizeof(*rec));
  5656. /* More than one function may be in this block */
  5657. goto again;
  5658. }
  5659. mutex_unlock(&ftrace_lock);
  5660. list_for_each_entry_safe(func, func_next, &clear_hash, list) {
  5661. clear_func_from_hashes(func);
  5662. kfree(func);
  5663. }
  5664. }
  5665. void __init ftrace_free_init_mem(void)
  5666. {
  5667. void *start = (void *)(&__init_begin);
  5668. void *end = (void *)(&__init_end);
  5669. ftrace_free_mem(NULL, start, end);
  5670. }
  5671. void __init ftrace_init(void)
  5672. {
  5673. extern unsigned long __start_mcount_loc[];
  5674. extern unsigned long __stop_mcount_loc[];
  5675. unsigned long count, flags;
  5676. int ret;
  5677. local_irq_save(flags);
  5678. ret = ftrace_dyn_arch_init();
  5679. local_irq_restore(flags);
  5680. if (ret)
  5681. goto failed;
  5682. count = __stop_mcount_loc - __start_mcount_loc;
  5683. if (!count) {
  5684. pr_info("ftrace: No functions to be traced?\n");
  5685. goto failed;
  5686. }
  5687. pr_info("ftrace: allocating %ld entries in %ld pages\n",
  5688. count, count / ENTRIES_PER_PAGE + 1);
  5689. last_ftrace_enabled = ftrace_enabled = 1;
  5690. ret = ftrace_process_locs(NULL,
  5691. __start_mcount_loc,
  5692. __stop_mcount_loc);
  5693. pr_info("ftrace: allocated %ld pages with %ld groups\n",
  5694. ftrace_number_of_pages, ftrace_number_of_groups);
  5695. set_ftrace_early_filters();
  5696. return;
  5697. failed:
  5698. ftrace_disabled = 1;
  5699. }
  5700. /* Do nothing if arch does not support this */
  5701. void __weak arch_ftrace_update_trampoline(struct ftrace_ops *ops)
  5702. {
  5703. }
  5704. static void ftrace_update_trampoline(struct ftrace_ops *ops)
  5705. {
  5706. unsigned long trampoline = ops->trampoline;
  5707. arch_ftrace_update_trampoline(ops);
  5708. if (ops->trampoline && ops->trampoline != trampoline &&
  5709. (ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP)) {
  5710. /* Add to kallsyms before the perf events */
  5711. ftrace_add_trampoline_to_kallsyms(ops);
  5712. perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL,
  5713. ops->trampoline, ops->trampoline_size, false,
  5714. FTRACE_TRAMPOLINE_SYM);
  5715. /*
  5716. * Record the perf text poke event after the ksymbol register
  5717. * event.
  5718. */
  5719. perf_event_text_poke((void *)ops->trampoline, NULL, 0,
  5720. (void *)ops->trampoline,
  5721. ops->trampoline_size);
  5722. }
  5723. }
  5724. void ftrace_init_trace_array(struct trace_array *tr)
  5725. {
  5726. INIT_LIST_HEAD(&tr->func_probes);
  5727. INIT_LIST_HEAD(&tr->mod_trace);
  5728. INIT_LIST_HEAD(&tr->mod_notrace);
  5729. }
  5730. #else
  5731. struct ftrace_ops global_ops = {
  5732. .func = ftrace_stub,
  5733. .flags = FTRACE_OPS_FL_RECURSION_SAFE |
  5734. FTRACE_OPS_FL_INITIALIZED |
  5735. FTRACE_OPS_FL_PID,
  5736. };
  5737. static int __init ftrace_nodyn_init(void)
  5738. {
  5739. ftrace_enabled = 1;
  5740. return 0;
  5741. }
  5742. core_initcall(ftrace_nodyn_init);
  5743. static inline int ftrace_init_dyn_tracefs(struct dentry *d_tracer) { return 0; }
  5744. static inline void ftrace_startup_enable(int command) { }
  5745. static inline void ftrace_startup_all(int command) { }
  5746. # define ftrace_startup_sysctl() do { } while (0)
  5747. # define ftrace_shutdown_sysctl() do { } while (0)
  5748. static void ftrace_update_trampoline(struct ftrace_ops *ops)
  5749. {
  5750. }
  5751. #endif /* CONFIG_DYNAMIC_FTRACE */
  5752. __init void ftrace_init_global_array_ops(struct trace_array *tr)
  5753. {
  5754. tr->ops = &global_ops;
  5755. tr->ops->private = tr;
  5756. ftrace_init_trace_array(tr);
  5757. }
  5758. void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func)
  5759. {
  5760. /* If we filter on pids, update to use the pid function */
  5761. if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
  5762. if (WARN_ON(tr->ops->func != ftrace_stub))
  5763. printk("ftrace ops had %pS for function\n",
  5764. tr->ops->func);
  5765. }
  5766. tr->ops->func = func;
  5767. tr->ops->private = tr;
  5768. }
  5769. void ftrace_reset_array_ops(struct trace_array *tr)
  5770. {
  5771. tr->ops->func = ftrace_stub;
  5772. }
  5773. static nokprobe_inline void
  5774. __ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
  5775. struct ftrace_ops *ignored, struct pt_regs *regs)
  5776. {
  5777. struct ftrace_ops *op;
  5778. int bit;
  5779. bit = trace_test_and_set_recursion(TRACE_LIST_START);
  5780. if (bit < 0)
  5781. return;
  5782. /*
  5783. * Some of the ops may be dynamically allocated,
  5784. * they must be freed after a synchronize_rcu().
  5785. */
  5786. preempt_disable_notrace();
  5787. do_for_each_ftrace_op(op, ftrace_ops_list) {
  5788. /* Stub functions don't need to be called nor tested */
  5789. if (op->flags & FTRACE_OPS_FL_STUB)
  5790. continue;
  5791. /*
  5792. * Check the following for each ops before calling their func:
  5793. * if RCU flag is set, then rcu_is_watching() must be true
  5794. * if PER_CPU is set, then ftrace_function_local_disable()
  5795. * must be false
  5796. * Otherwise test if the ip matches the ops filter
  5797. *
  5798. * If any of the above fails then the op->func() is not executed.
  5799. */
  5800. if ((!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching()) &&
  5801. ftrace_ops_test(op, ip, regs)) {
  5802. if (FTRACE_WARN_ON(!op->func)) {
  5803. pr_warn("op=%p %pS\n", op, op);
  5804. goto out;
  5805. }
  5806. op->func(ip, parent_ip, op, regs);
  5807. }
  5808. } while_for_each_ftrace_op(op);
  5809. out:
  5810. preempt_enable_notrace();
  5811. trace_clear_recursion(bit);
  5812. }
  5813. /*
  5814. * Some archs only support passing ip and parent_ip. Even though
  5815. * the list function ignores the op parameter, we do not want any
  5816. * C side effects, where a function is called without the caller
  5817. * sending a third parameter.
  5818. * Archs are to support both the regs and ftrace_ops at the same time.
  5819. * If they support ftrace_ops, it is assumed they support regs.
  5820. * If call backs want to use regs, they must either check for regs
  5821. * being NULL, or CONFIG_DYNAMIC_FTRACE_WITH_REGS.
  5822. * Note, CONFIG_DYNAMIC_FTRACE_WITH_REGS expects a full regs to be saved.
  5823. * An architecture can pass partial regs with ftrace_ops and still
  5824. * set the ARCH_SUPPORTS_FTRACE_OPS.
  5825. */
  5826. #if ARCH_SUPPORTS_FTRACE_OPS
  5827. static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
  5828. struct ftrace_ops *op, struct pt_regs *regs)
  5829. {
  5830. __ftrace_ops_list_func(ip, parent_ip, NULL, regs);
  5831. }
  5832. NOKPROBE_SYMBOL(ftrace_ops_list_func);
  5833. #else
  5834. static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip)
  5835. {
  5836. __ftrace_ops_list_func(ip, parent_ip, NULL, NULL);
  5837. }
  5838. NOKPROBE_SYMBOL(ftrace_ops_no_ops);
  5839. #endif
  5840. /*
  5841. * If there's only one function registered but it does not support
  5842. * recursion, needs RCU protection and/or requires per cpu handling, then
  5843. * this function will be called by the mcount trampoline.
  5844. */
  5845. static void ftrace_ops_assist_func(unsigned long ip, unsigned long parent_ip,
  5846. struct ftrace_ops *op, struct pt_regs *regs)
  5847. {
  5848. int bit;
  5849. bit = trace_test_and_set_recursion(TRACE_LIST_START);
  5850. if (bit < 0)
  5851. return;
  5852. preempt_disable_notrace();
  5853. if (!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching())
  5854. op->func(ip, parent_ip, op, regs);
  5855. preempt_enable_notrace();
  5856. trace_clear_recursion(bit);
  5857. }
  5858. NOKPROBE_SYMBOL(ftrace_ops_assist_func);
  5859. /**
  5860. * ftrace_ops_get_func - get the function a trampoline should call
  5861. * @ops: the ops to get the function for
  5862. *
  5863. * Normally the mcount trampoline will call the ops->func, but there
  5864. * are times that it should not. For example, if the ops does not
  5865. * have its own recursion protection, then it should call the
  5866. * ftrace_ops_assist_func() instead.
  5867. *
  5868. * Returns the function that the trampoline should call for @ops.
  5869. */
  5870. ftrace_func_t ftrace_ops_get_func(struct ftrace_ops *ops)
  5871. {
  5872. /*
  5873. * If the function does not handle recursion, needs to be RCU safe,
  5874. * or does per cpu logic, then we need to call the assist handler.
  5875. */
  5876. if (!(ops->flags & FTRACE_OPS_FL_RECURSION_SAFE) ||
  5877. ops->flags & FTRACE_OPS_FL_RCU)
  5878. return ftrace_ops_assist_func;
  5879. return ops->func;
  5880. }
  5881. static void
  5882. ftrace_filter_pid_sched_switch_probe(void *data, bool preempt,
  5883. struct task_struct *prev, struct task_struct *next)
  5884. {
  5885. struct trace_array *tr = data;
  5886. struct trace_pid_list *pid_list;
  5887. struct trace_pid_list *no_pid_list;
  5888. pid_list = rcu_dereference_sched(tr->function_pids);
  5889. no_pid_list = rcu_dereference_sched(tr->function_no_pids);
  5890. if (trace_ignore_this_task(pid_list, no_pid_list, next))
  5891. this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid,
  5892. FTRACE_PID_IGNORE);
  5893. else
  5894. this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid,
  5895. next->pid);
  5896. }
  5897. static void
  5898. ftrace_pid_follow_sched_process_fork(void *data,
  5899. struct task_struct *self,
  5900. struct task_struct *task)
  5901. {
  5902. struct trace_pid_list *pid_list;
  5903. struct trace_array *tr = data;
  5904. pid_list = rcu_dereference_sched(tr->function_pids);
  5905. trace_filter_add_remove_task(pid_list, self, task);
  5906. pid_list = rcu_dereference_sched(tr->function_no_pids);
  5907. trace_filter_add_remove_task(pid_list, self, task);
  5908. }
  5909. static void
  5910. ftrace_pid_follow_sched_process_exit(void *data, struct task_struct *task)
  5911. {
  5912. struct trace_pid_list *pid_list;
  5913. struct trace_array *tr = data;
  5914. pid_list = rcu_dereference_sched(tr->function_pids);
  5915. trace_filter_add_remove_task(pid_list, NULL, task);
  5916. pid_list = rcu_dereference_sched(tr->function_no_pids);
  5917. trace_filter_add_remove_task(pid_list, NULL, task);
  5918. }
  5919. void ftrace_pid_follow_fork(struct trace_array *tr, bool enable)
  5920. {
  5921. if (enable) {
  5922. register_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork,
  5923. tr);
  5924. register_trace_sched_process_free(ftrace_pid_follow_sched_process_exit,
  5925. tr);
  5926. } else {
  5927. unregister_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork,
  5928. tr);
  5929. unregister_trace_sched_process_free(ftrace_pid_follow_sched_process_exit,
  5930. tr);
  5931. }
  5932. }
  5933. static void clear_ftrace_pids(struct trace_array *tr, int type)
  5934. {
  5935. struct trace_pid_list *pid_list;
  5936. struct trace_pid_list *no_pid_list;
  5937. int cpu;
  5938. pid_list = rcu_dereference_protected(tr->function_pids,
  5939. lockdep_is_held(&ftrace_lock));
  5940. no_pid_list = rcu_dereference_protected(tr->function_no_pids,
  5941. lockdep_is_held(&ftrace_lock));
  5942. /* Make sure there's something to do */
  5943. if (!pid_type_enabled(type, pid_list, no_pid_list))
  5944. return;
  5945. /* See if the pids still need to be checked after this */
  5946. if (!still_need_pid_events(type, pid_list, no_pid_list)) {
  5947. unregister_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);
  5948. for_each_possible_cpu(cpu)
  5949. per_cpu_ptr(tr->array_buffer.data, cpu)->ftrace_ignore_pid = FTRACE_PID_TRACE;
  5950. }
  5951. if (type & TRACE_PIDS)
  5952. rcu_assign_pointer(tr->function_pids, NULL);
  5953. if (type & TRACE_NO_PIDS)
  5954. rcu_assign_pointer(tr->function_no_pids, NULL);
  5955. /* Wait till all users are no longer using pid filtering */
  5956. synchronize_rcu();
  5957. if ((type & TRACE_PIDS) && pid_list)
  5958. trace_free_pid_list(pid_list);
  5959. if ((type & TRACE_NO_PIDS) && no_pid_list)
  5960. trace_free_pid_list(no_pid_list);
  5961. }
  5962. void ftrace_clear_pids(struct trace_array *tr)
  5963. {
  5964. mutex_lock(&ftrace_lock);
  5965. clear_ftrace_pids(tr, TRACE_PIDS | TRACE_NO_PIDS);
  5966. mutex_unlock(&ftrace_lock);
  5967. }
  5968. static void ftrace_pid_reset(struct trace_array *tr, int type)
  5969. {
  5970. mutex_lock(&ftrace_lock);
  5971. clear_ftrace_pids(tr, type);
  5972. ftrace_update_pid_func();
  5973. ftrace_startup_all(0);
  5974. mutex_unlock(&ftrace_lock);
  5975. }
  5976. /* Greater than any max PID */
  5977. #define FTRACE_NO_PIDS (void *)(PID_MAX_LIMIT + 1)
  5978. static void *fpid_start(struct seq_file *m, loff_t *pos)
  5979. __acquires(RCU)
  5980. {
  5981. struct trace_pid_list *pid_list;
  5982. struct trace_array *tr = m->private;
  5983. mutex_lock(&ftrace_lock);
  5984. rcu_read_lock_sched();
  5985. pid_list = rcu_dereference_sched(tr->function_pids);
  5986. if (!pid_list)
  5987. return !(*pos) ? FTRACE_NO_PIDS : NULL;
  5988. return trace_pid_start(pid_list, pos);
  5989. }
  5990. static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
  5991. {
  5992. struct trace_array *tr = m->private;
  5993. struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_pids);
  5994. if (v == FTRACE_NO_PIDS) {
  5995. (*pos)++;
  5996. return NULL;
  5997. }
  5998. return trace_pid_next(pid_list, v, pos);
  5999. }
  6000. static void fpid_stop(struct seq_file *m, void *p)
  6001. __releases(RCU)
  6002. {
  6003. rcu_read_unlock_sched();
  6004. mutex_unlock(&ftrace_lock);
  6005. }
  6006. static int fpid_show(struct seq_file *m, void *v)
  6007. {
  6008. if (v == FTRACE_NO_PIDS) {
  6009. seq_puts(m, "no pid\n");
  6010. return 0;
  6011. }
  6012. return trace_pid_show(m, v);
  6013. }
  6014. static const struct seq_operations ftrace_pid_sops = {
  6015. .start = fpid_start,
  6016. .next = fpid_next,
  6017. .stop = fpid_stop,
  6018. .show = fpid_show,
  6019. };
  6020. static void *fnpid_start(struct seq_file *m, loff_t *pos)
  6021. __acquires(RCU)
  6022. {
  6023. struct trace_pid_list *pid_list;
  6024. struct trace_array *tr = m->private;
  6025. mutex_lock(&ftrace_lock);
  6026. rcu_read_lock_sched();
  6027. pid_list = rcu_dereference_sched(tr->function_no_pids);
  6028. if (!pid_list)
  6029. return !(*pos) ? FTRACE_NO_PIDS : NULL;
  6030. return trace_pid_start(pid_list, pos);
  6031. }
  6032. static void *fnpid_next(struct seq_file *m, void *v, loff_t *pos)
  6033. {
  6034. struct trace_array *tr = m->private;
  6035. struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_no_pids);
  6036. if (v == FTRACE_NO_PIDS) {
  6037. (*pos)++;
  6038. return NULL;
  6039. }
  6040. return trace_pid_next(pid_list, v, pos);
  6041. }
  6042. static const struct seq_operations ftrace_no_pid_sops = {
  6043. .start = fnpid_start,
  6044. .next = fnpid_next,
  6045. .stop = fpid_stop,
  6046. .show = fpid_show,
  6047. };
  6048. static int pid_open(struct inode *inode, struct file *file, int type)
  6049. {
  6050. const struct seq_operations *seq_ops;
  6051. struct trace_array *tr = inode->i_private;
  6052. struct seq_file *m;
  6053. int ret = 0;
  6054. ret = tracing_check_open_get_tr(tr);
  6055. if (ret)
  6056. return ret;
  6057. if ((file->f_mode & FMODE_WRITE) &&
  6058. (file->f_flags & O_TRUNC))
  6059. ftrace_pid_reset(tr, type);
  6060. switch (type) {
  6061. case TRACE_PIDS:
  6062. seq_ops = &ftrace_pid_sops;
  6063. break;
  6064. case TRACE_NO_PIDS:
  6065. seq_ops = &ftrace_no_pid_sops;
  6066. break;
  6067. default:
  6068. trace_array_put(tr);
  6069. WARN_ON_ONCE(1);
  6070. return -EINVAL;
  6071. }
  6072. ret = seq_open(file, seq_ops);
  6073. if (ret < 0) {
  6074. trace_array_put(tr);
  6075. } else {
  6076. m = file->private_data;
  6077. /* copy tr over to seq ops */
  6078. m->private = tr;
  6079. }
  6080. return ret;
  6081. }
  6082. static int
  6083. ftrace_pid_open(struct inode *inode, struct file *file)
  6084. {
  6085. return pid_open(inode, file, TRACE_PIDS);
  6086. }
  6087. static int
  6088. ftrace_no_pid_open(struct inode *inode, struct file *file)
  6089. {
  6090. return pid_open(inode, file, TRACE_NO_PIDS);
  6091. }
  6092. static void ignore_task_cpu(void *data)
  6093. {
  6094. struct trace_array *tr = data;
  6095. struct trace_pid_list *pid_list;
  6096. struct trace_pid_list *no_pid_list;
  6097. /*
  6098. * This function is called by on_each_cpu() while the
  6099. * event_mutex is held.
  6100. */
  6101. pid_list = rcu_dereference_protected(tr->function_pids,
  6102. mutex_is_locked(&ftrace_lock));
  6103. no_pid_list = rcu_dereference_protected(tr->function_no_pids,
  6104. mutex_is_locked(&ftrace_lock));
  6105. if (trace_ignore_this_task(pid_list, no_pid_list, current))
  6106. this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid,
  6107. FTRACE_PID_IGNORE);
  6108. else
  6109. this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid,
  6110. current->pid);
  6111. }
  6112. static ssize_t
  6113. pid_write(struct file *filp, const char __user *ubuf,
  6114. size_t cnt, loff_t *ppos, int type)
  6115. {
  6116. struct seq_file *m = filp->private_data;
  6117. struct trace_array *tr = m->private;
  6118. struct trace_pid_list *filtered_pids;
  6119. struct trace_pid_list *other_pids;
  6120. struct trace_pid_list *pid_list;
  6121. ssize_t ret;
  6122. if (!cnt)
  6123. return 0;
  6124. mutex_lock(&ftrace_lock);
  6125. switch (type) {
  6126. case TRACE_PIDS:
  6127. filtered_pids = rcu_dereference_protected(tr->function_pids,
  6128. lockdep_is_held(&ftrace_lock));
  6129. other_pids = rcu_dereference_protected(tr->function_no_pids,
  6130. lockdep_is_held(&ftrace_lock));
  6131. break;
  6132. case TRACE_NO_PIDS:
  6133. filtered_pids = rcu_dereference_protected(tr->function_no_pids,
  6134. lockdep_is_held(&ftrace_lock));
  6135. other_pids = rcu_dereference_protected(tr->function_pids,
  6136. lockdep_is_held(&ftrace_lock));
  6137. break;
  6138. default:
  6139. ret = -EINVAL;
  6140. WARN_ON_ONCE(1);
  6141. goto out;
  6142. }
  6143. ret = trace_pid_write(filtered_pids, &pid_list, ubuf, cnt);
  6144. if (ret < 0)
  6145. goto out;
  6146. switch (type) {
  6147. case TRACE_PIDS:
  6148. rcu_assign_pointer(tr->function_pids, pid_list);
  6149. break;
  6150. case TRACE_NO_PIDS:
  6151. rcu_assign_pointer(tr->function_no_pids, pid_list);
  6152. break;
  6153. }
  6154. if (filtered_pids) {
  6155. synchronize_rcu();
  6156. trace_free_pid_list(filtered_pids);
  6157. } else if (pid_list && !other_pids) {
  6158. /* Register a probe to set whether to ignore the tracing of a task */
  6159. register_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);
  6160. }
  6161. /*
  6162. * Ignoring of pids is done at task switch. But we have to
  6163. * check for those tasks that are currently running.
  6164. * Always do this in case a pid was appended or removed.
  6165. */
  6166. on_each_cpu(ignore_task_cpu, tr, 1);
  6167. ftrace_update_pid_func();
  6168. ftrace_startup_all(0);
  6169. out:
  6170. mutex_unlock(&ftrace_lock);
  6171. if (ret > 0)
  6172. *ppos += ret;
  6173. return ret;
  6174. }
  6175. static ssize_t
  6176. ftrace_pid_write(struct file *filp, const char __user *ubuf,
  6177. size_t cnt, loff_t *ppos)
  6178. {
  6179. return pid_write(filp, ubuf, cnt, ppos, TRACE_PIDS);
  6180. }
  6181. static ssize_t
  6182. ftrace_no_pid_write(struct file *filp, const char __user *ubuf,
  6183. size_t cnt, loff_t *ppos)
  6184. {
  6185. return pid_write(filp, ubuf, cnt, ppos, TRACE_NO_PIDS);
  6186. }
  6187. static int
  6188. ftrace_pid_release(struct inode *inode, struct file *file)
  6189. {
  6190. struct trace_array *tr = inode->i_private;
  6191. trace_array_put(tr);
  6192. return seq_release(inode, file);
  6193. }
  6194. static const struct file_operations ftrace_pid_fops = {
  6195. .open = ftrace_pid_open,
  6196. .write = ftrace_pid_write,
  6197. .read = seq_read,
  6198. .llseek = tracing_lseek,
  6199. .release = ftrace_pid_release,
  6200. };
  6201. static const struct file_operations ftrace_no_pid_fops = {
  6202. .open = ftrace_no_pid_open,
  6203. .write = ftrace_no_pid_write,
  6204. .read = seq_read,
  6205. .llseek = tracing_lseek,
  6206. .release = ftrace_pid_release,
  6207. };
  6208. void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d_tracer)
  6209. {
  6210. trace_create_file("set_ftrace_pid", 0644, d_tracer,
  6211. tr, &ftrace_pid_fops);
  6212. trace_create_file("set_ftrace_notrace_pid", 0644, d_tracer,
  6213. tr, &ftrace_no_pid_fops);
  6214. }
  6215. void __init ftrace_init_tracefs_toplevel(struct trace_array *tr,
  6216. struct dentry *d_tracer)
  6217. {
  6218. /* Only the top level directory has the dyn_tracefs and profile */
  6219. WARN_ON(!(tr->flags & TRACE_ARRAY_FL_GLOBAL));
  6220. ftrace_init_dyn_tracefs(d_tracer);
  6221. ftrace_profile_tracefs(d_tracer);
  6222. }
  6223. /**
  6224. * ftrace_kill - kill ftrace
  6225. *
  6226. * This function should be used by panic code. It stops ftrace
  6227. * but in a not so nice way. If you need to simply kill ftrace
  6228. * from a non-atomic section, use ftrace_kill.
  6229. */
  6230. void ftrace_kill(void)
  6231. {
  6232. ftrace_disabled = 1;
  6233. ftrace_enabled = 0;
  6234. ftrace_trace_function = ftrace_stub;
  6235. }
  6236. /**
  6237. * Test if ftrace is dead or not.
  6238. */
  6239. int ftrace_is_dead(void)
  6240. {
  6241. return ftrace_disabled;
  6242. }
  6243. /**
  6244. * register_ftrace_function - register a function for profiling
  6245. * @ops - ops structure that holds the function for profiling.
  6246. *
  6247. * Register a function to be called by all functions in the
  6248. * kernel.
  6249. *
  6250. * Note: @ops->func and all the functions it calls must be labeled
  6251. * with "notrace", otherwise it will go into a
  6252. * recursive loop.
  6253. */
  6254. int register_ftrace_function(struct ftrace_ops *ops)
  6255. {
  6256. int ret = -1;
  6257. ftrace_ops_init(ops);
  6258. mutex_lock(&ftrace_lock);
  6259. ret = ftrace_startup(ops, 0);
  6260. mutex_unlock(&ftrace_lock);
  6261. return ret;
  6262. }
  6263. EXPORT_SYMBOL_GPL(register_ftrace_function);
  6264. /**
  6265. * unregister_ftrace_function - unregister a function for profiling.
  6266. * @ops - ops structure that holds the function to unregister
  6267. *
  6268. * Unregister a function that was added to be called by ftrace profiling.
  6269. */
  6270. int unregister_ftrace_function(struct ftrace_ops *ops)
  6271. {
  6272. int ret;
  6273. mutex_lock(&ftrace_lock);
  6274. ret = ftrace_shutdown(ops, 0);
  6275. mutex_unlock(&ftrace_lock);
  6276. return ret;
  6277. }
  6278. EXPORT_SYMBOL_GPL(unregister_ftrace_function);
  6279. static bool is_permanent_ops_registered(void)
  6280. {
  6281. struct ftrace_ops *op;
  6282. do_for_each_ftrace_op(op, ftrace_ops_list) {
  6283. if (op->flags & FTRACE_OPS_FL_PERMANENT)
  6284. return true;
  6285. } while_for_each_ftrace_op(op);
  6286. return false;
  6287. }
  6288. int
  6289. ftrace_enable_sysctl(struct ctl_table *table, int write,
  6290. void *buffer, size_t *lenp, loff_t *ppos)
  6291. {
  6292. int ret = -ENODEV;
  6293. mutex_lock(&ftrace_lock);
  6294. if (unlikely(ftrace_disabled))
  6295. goto out;
  6296. ret = proc_dointvec(table, write, buffer, lenp, ppos);
  6297. if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
  6298. goto out;
  6299. if (ftrace_enabled) {
  6300. /* we are starting ftrace again */
  6301. if (rcu_dereference_protected(ftrace_ops_list,
  6302. lockdep_is_held(&ftrace_lock)) != &ftrace_list_end)
  6303. update_ftrace_function();
  6304. ftrace_startup_sysctl();
  6305. } else {
  6306. if (is_permanent_ops_registered()) {
  6307. ftrace_enabled = true;
  6308. ret = -EBUSY;
  6309. goto out;
  6310. }
  6311. /* stopping ftrace calls (just send to ftrace_stub) */
  6312. ftrace_trace_function = ftrace_stub;
  6313. ftrace_shutdown_sysctl();
  6314. }
  6315. last_ftrace_enabled = !!ftrace_enabled;
  6316. out:
  6317. mutex_unlock(&ftrace_lock);
  6318. return ret;
  6319. }