machine.c 75 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124
  1. // SPDX-License-Identifier: GPL-2.0
  2. #include <dirent.h>
  3. #include <errno.h>
  4. #include <inttypes.h>
  5. #include <regex.h>
  6. #include <stdlib.h>
  7. #include "callchain.h"
  8. #include "debug.h"
  9. #include "dso.h"
  10. #include "env.h"
  11. #include "event.h"
  12. #include "evsel.h"
  13. #include "hist.h"
  14. #include "machine.h"
  15. #include "map.h"
  16. #include "map_symbol.h"
  17. #include "branch.h"
  18. #include "mem-events.h"
  19. #include "srcline.h"
  20. #include "symbol.h"
  21. #include "sort.h"
  22. #include "strlist.h"
  23. #include "target.h"
  24. #include "thread.h"
  25. #include "util.h"
  26. #include "vdso.h"
  27. #include <stdbool.h>
  28. #include <sys/types.h>
  29. #include <sys/stat.h>
  30. #include <unistd.h>
  31. #include "unwind.h"
  32. #include "linux/hash.h"
  33. #include "asm/bug.h"
  34. #include "bpf-event.h"
  35. #include <internal/lib.h> // page_size
  36. #include "cgroup.h"
  37. #include <linux/ctype.h>
  38. #include <symbol/kallsyms.h>
  39. #include <linux/mman.h>
  40. #include <linux/string.h>
  41. #include <linux/zalloc.h>
  42. static void __machine__remove_thread(struct machine *machine, struct thread *th, bool lock);
  43. static struct dso *machine__kernel_dso(struct machine *machine)
  44. {
  45. return machine->vmlinux_map->dso;
  46. }
  47. static void dsos__init(struct dsos *dsos)
  48. {
  49. INIT_LIST_HEAD(&dsos->head);
  50. dsos->root = RB_ROOT;
  51. init_rwsem(&dsos->lock);
  52. }
  53. static void machine__threads_init(struct machine *machine)
  54. {
  55. int i;
  56. for (i = 0; i < THREADS__TABLE_SIZE; i++) {
  57. struct threads *threads = &machine->threads[i];
  58. threads->entries = RB_ROOT_CACHED;
  59. init_rwsem(&threads->lock);
  60. threads->nr = 0;
  61. INIT_LIST_HEAD(&threads->dead);
  62. threads->last_match = NULL;
  63. }
  64. }
  65. static int machine__set_mmap_name(struct machine *machine)
  66. {
  67. if (machine__is_host(machine))
  68. machine->mmap_name = strdup("[kernel.kallsyms]");
  69. else if (machine__is_default_guest(machine))
  70. machine->mmap_name = strdup("[guest.kernel.kallsyms]");
  71. else if (asprintf(&machine->mmap_name, "[guest.kernel.kallsyms.%d]",
  72. machine->pid) < 0)
  73. machine->mmap_name = NULL;
  74. return machine->mmap_name ? 0 : -ENOMEM;
  75. }
  76. int machine__init(struct machine *machine, const char *root_dir, pid_t pid)
  77. {
  78. int err = -ENOMEM;
  79. memset(machine, 0, sizeof(*machine));
  80. maps__init(&machine->kmaps, machine);
  81. RB_CLEAR_NODE(&machine->rb_node);
  82. dsos__init(&machine->dsos);
  83. machine__threads_init(machine);
  84. machine->vdso_info = NULL;
  85. machine->env = NULL;
  86. machine->pid = pid;
  87. machine->id_hdr_size = 0;
  88. machine->kptr_restrict_warned = false;
  89. machine->comm_exec = false;
  90. machine->kernel_start = 0;
  91. machine->vmlinux_map = NULL;
  92. machine->root_dir = strdup(root_dir);
  93. if (machine->root_dir == NULL)
  94. return -ENOMEM;
  95. if (machine__set_mmap_name(machine))
  96. goto out;
  97. if (pid != HOST_KERNEL_ID) {
  98. struct thread *thread = machine__findnew_thread(machine, -1,
  99. pid);
  100. char comm[64];
  101. if (thread == NULL)
  102. goto out;
  103. snprintf(comm, sizeof(comm), "[guest/%d]", pid);
  104. thread__set_comm(thread, comm, 0);
  105. thread__put(thread);
  106. }
  107. machine->current_tid = NULL;
  108. err = 0;
  109. out:
  110. if (err) {
  111. zfree(&machine->root_dir);
  112. zfree(&machine->mmap_name);
  113. }
  114. return 0;
  115. }
  116. struct machine *machine__new_host(void)
  117. {
  118. struct machine *machine = malloc(sizeof(*machine));
  119. if (machine != NULL) {
  120. machine__init(machine, "", HOST_KERNEL_ID);
  121. if (machine__create_kernel_maps(machine) < 0)
  122. goto out_delete;
  123. }
  124. return machine;
  125. out_delete:
  126. free(machine);
  127. return NULL;
  128. }
  129. struct machine *machine__new_kallsyms(void)
  130. {
  131. struct machine *machine = machine__new_host();
  132. /*
  133. * FIXME:
  134. * 1) We should switch to machine__load_kallsyms(), i.e. not explicitly
  135. * ask for not using the kcore parsing code, once this one is fixed
  136. * to create a map per module.
  137. */
  138. if (machine && machine__load_kallsyms(machine, "/proc/kallsyms") <= 0) {
  139. machine__delete(machine);
  140. machine = NULL;
  141. }
  142. return machine;
  143. }
  144. static void dsos__purge(struct dsos *dsos)
  145. {
  146. struct dso *pos, *n;
  147. down_write(&dsos->lock);
  148. list_for_each_entry_safe(pos, n, &dsos->head, node) {
  149. RB_CLEAR_NODE(&pos->rb_node);
  150. pos->root = NULL;
  151. list_del_init(&pos->node);
  152. dso__put(pos);
  153. }
  154. up_write(&dsos->lock);
  155. }
  156. static void dsos__exit(struct dsos *dsos)
  157. {
  158. dsos__purge(dsos);
  159. exit_rwsem(&dsos->lock);
  160. }
  161. void machine__delete_threads(struct machine *machine)
  162. {
  163. struct rb_node *nd;
  164. int i;
  165. for (i = 0; i < THREADS__TABLE_SIZE; i++) {
  166. struct threads *threads = &machine->threads[i];
  167. down_write(&threads->lock);
  168. nd = rb_first_cached(&threads->entries);
  169. while (nd) {
  170. struct thread *t = rb_entry(nd, struct thread, rb_node);
  171. nd = rb_next(nd);
  172. __machine__remove_thread(machine, t, false);
  173. }
  174. up_write(&threads->lock);
  175. }
  176. }
  177. void machine__exit(struct machine *machine)
  178. {
  179. int i;
  180. if (machine == NULL)
  181. return;
  182. machine__destroy_kernel_maps(machine);
  183. maps__exit(&machine->kmaps);
  184. dsos__exit(&machine->dsos);
  185. machine__exit_vdso(machine);
  186. zfree(&machine->root_dir);
  187. zfree(&machine->mmap_name);
  188. zfree(&machine->current_tid);
  189. for (i = 0; i < THREADS__TABLE_SIZE; i++) {
  190. struct threads *threads = &machine->threads[i];
  191. struct thread *thread, *n;
  192. /*
  193. * Forget about the dead, at this point whatever threads were
  194. * left in the dead lists better have a reference count taken
  195. * by who is using them, and then, when they drop those references
  196. * and it finally hits zero, thread__put() will check and see that
  197. * its not in the dead threads list and will not try to remove it
  198. * from there, just calling thread__delete() straight away.
  199. */
  200. list_for_each_entry_safe(thread, n, &threads->dead, node)
  201. list_del_init(&thread->node);
  202. exit_rwsem(&threads->lock);
  203. }
  204. }
  205. void machine__delete(struct machine *machine)
  206. {
  207. if (machine) {
  208. machine__exit(machine);
  209. free(machine);
  210. }
  211. }
  212. void machines__init(struct machines *machines)
  213. {
  214. machine__init(&machines->host, "", HOST_KERNEL_ID);
  215. machines->guests = RB_ROOT_CACHED;
  216. }
  217. void machines__exit(struct machines *machines)
  218. {
  219. machine__exit(&machines->host);
  220. /* XXX exit guest */
  221. }
  222. struct machine *machines__add(struct machines *machines, pid_t pid,
  223. const char *root_dir)
  224. {
  225. struct rb_node **p = &machines->guests.rb_root.rb_node;
  226. struct rb_node *parent = NULL;
  227. struct machine *pos, *machine = malloc(sizeof(*machine));
  228. bool leftmost = true;
  229. if (machine == NULL)
  230. return NULL;
  231. if (machine__init(machine, root_dir, pid) != 0) {
  232. free(machine);
  233. return NULL;
  234. }
  235. while (*p != NULL) {
  236. parent = *p;
  237. pos = rb_entry(parent, struct machine, rb_node);
  238. if (pid < pos->pid)
  239. p = &(*p)->rb_left;
  240. else {
  241. p = &(*p)->rb_right;
  242. leftmost = false;
  243. }
  244. }
  245. rb_link_node(&machine->rb_node, parent, p);
  246. rb_insert_color_cached(&machine->rb_node, &machines->guests, leftmost);
  247. return machine;
  248. }
  249. void machines__set_comm_exec(struct machines *machines, bool comm_exec)
  250. {
  251. struct rb_node *nd;
  252. machines->host.comm_exec = comm_exec;
  253. for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
  254. struct machine *machine = rb_entry(nd, struct machine, rb_node);
  255. machine->comm_exec = comm_exec;
  256. }
  257. }
  258. struct machine *machines__find(struct machines *machines, pid_t pid)
  259. {
  260. struct rb_node **p = &machines->guests.rb_root.rb_node;
  261. struct rb_node *parent = NULL;
  262. struct machine *machine;
  263. struct machine *default_machine = NULL;
  264. if (pid == HOST_KERNEL_ID)
  265. return &machines->host;
  266. while (*p != NULL) {
  267. parent = *p;
  268. machine = rb_entry(parent, struct machine, rb_node);
  269. if (pid < machine->pid)
  270. p = &(*p)->rb_left;
  271. else if (pid > machine->pid)
  272. p = &(*p)->rb_right;
  273. else
  274. return machine;
  275. if (!machine->pid)
  276. default_machine = machine;
  277. }
  278. return default_machine;
  279. }
  280. struct machine *machines__findnew(struct machines *machines, pid_t pid)
  281. {
  282. char path[PATH_MAX];
  283. const char *root_dir = "";
  284. struct machine *machine = machines__find(machines, pid);
  285. if (machine && (machine->pid == pid))
  286. goto out;
  287. if ((pid != HOST_KERNEL_ID) &&
  288. (pid != DEFAULT_GUEST_KERNEL_ID) &&
  289. (symbol_conf.guestmount)) {
  290. sprintf(path, "%s/%d", symbol_conf.guestmount, pid);
  291. if (access(path, R_OK)) {
  292. static struct strlist *seen;
  293. if (!seen)
  294. seen = strlist__new(NULL, NULL);
  295. if (!strlist__has_entry(seen, path)) {
  296. pr_err("Can't access file %s\n", path);
  297. strlist__add(seen, path);
  298. }
  299. machine = NULL;
  300. goto out;
  301. }
  302. root_dir = path;
  303. }
  304. machine = machines__add(machines, pid, root_dir);
  305. out:
  306. return machine;
  307. }
  308. void machines__process_guests(struct machines *machines,
  309. machine__process_t process, void *data)
  310. {
  311. struct rb_node *nd;
  312. for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
  313. struct machine *pos = rb_entry(nd, struct machine, rb_node);
  314. process(pos, data);
  315. }
  316. }
  317. void machines__set_id_hdr_size(struct machines *machines, u16 id_hdr_size)
  318. {
  319. struct rb_node *node;
  320. struct machine *machine;
  321. machines->host.id_hdr_size = id_hdr_size;
  322. for (node = rb_first_cached(&machines->guests); node;
  323. node = rb_next(node)) {
  324. machine = rb_entry(node, struct machine, rb_node);
  325. machine->id_hdr_size = id_hdr_size;
  326. }
  327. return;
  328. }
  329. static void machine__update_thread_pid(struct machine *machine,
  330. struct thread *th, pid_t pid)
  331. {
  332. struct thread *leader;
  333. if (pid == th->pid_ || pid == -1 || th->pid_ != -1)
  334. return;
  335. th->pid_ = pid;
  336. if (th->pid_ == th->tid)
  337. return;
  338. leader = __machine__findnew_thread(machine, th->pid_, th->pid_);
  339. if (!leader)
  340. goto out_err;
  341. if (!leader->maps)
  342. leader->maps = maps__new(machine);
  343. if (!leader->maps)
  344. goto out_err;
  345. if (th->maps == leader->maps)
  346. return;
  347. if (th->maps) {
  348. /*
  349. * Maps are created from MMAP events which provide the pid and
  350. * tid. Consequently there never should be any maps on a thread
  351. * with an unknown pid. Just print an error if there are.
  352. */
  353. if (!maps__empty(th->maps))
  354. pr_err("Discarding thread maps for %d:%d\n",
  355. th->pid_, th->tid);
  356. maps__put(th->maps);
  357. }
  358. th->maps = maps__get(leader->maps);
  359. out_put:
  360. thread__put(leader);
  361. return;
  362. out_err:
  363. pr_err("Failed to join map groups for %d:%d\n", th->pid_, th->tid);
  364. goto out_put;
  365. }
  366. /*
  367. * Front-end cache - TID lookups come in blocks,
  368. * so most of the time we dont have to look up
  369. * the full rbtree:
  370. */
  371. static struct thread*
  372. __threads__get_last_match(struct threads *threads, struct machine *machine,
  373. int pid, int tid)
  374. {
  375. struct thread *th;
  376. th = threads->last_match;
  377. if (th != NULL) {
  378. if (th->tid == tid) {
  379. machine__update_thread_pid(machine, th, pid);
  380. return thread__get(th);
  381. }
  382. threads->last_match = NULL;
  383. }
  384. return NULL;
  385. }
  386. static struct thread*
  387. threads__get_last_match(struct threads *threads, struct machine *machine,
  388. int pid, int tid)
  389. {
  390. struct thread *th = NULL;
  391. if (perf_singlethreaded)
  392. th = __threads__get_last_match(threads, machine, pid, tid);
  393. return th;
  394. }
  395. static void
  396. __threads__set_last_match(struct threads *threads, struct thread *th)
  397. {
  398. threads->last_match = th;
  399. }
  400. static void
  401. threads__set_last_match(struct threads *threads, struct thread *th)
  402. {
  403. if (perf_singlethreaded)
  404. __threads__set_last_match(threads, th);
  405. }
  406. /*
  407. * Caller must eventually drop thread->refcnt returned with a successful
  408. * lookup/new thread inserted.
  409. */
  410. static struct thread *____machine__findnew_thread(struct machine *machine,
  411. struct threads *threads,
  412. pid_t pid, pid_t tid,
  413. bool create)
  414. {
  415. struct rb_node **p = &threads->entries.rb_root.rb_node;
  416. struct rb_node *parent = NULL;
  417. struct thread *th;
  418. bool leftmost = true;
  419. th = threads__get_last_match(threads, machine, pid, tid);
  420. if (th)
  421. return th;
  422. while (*p != NULL) {
  423. parent = *p;
  424. th = rb_entry(parent, struct thread, rb_node);
  425. if (th->tid == tid) {
  426. threads__set_last_match(threads, th);
  427. machine__update_thread_pid(machine, th, pid);
  428. return thread__get(th);
  429. }
  430. if (tid < th->tid)
  431. p = &(*p)->rb_left;
  432. else {
  433. p = &(*p)->rb_right;
  434. leftmost = false;
  435. }
  436. }
  437. if (!create)
  438. return NULL;
  439. th = thread__new(pid, tid);
  440. if (th != NULL) {
  441. rb_link_node(&th->rb_node, parent, p);
  442. rb_insert_color_cached(&th->rb_node, &threads->entries, leftmost);
  443. /*
  444. * We have to initialize maps separately after rb tree is updated.
  445. *
  446. * The reason is that we call machine__findnew_thread
  447. * within thread__init_maps to find the thread
  448. * leader and that would screwed the rb tree.
  449. */
  450. if (thread__init_maps(th, machine)) {
  451. rb_erase_cached(&th->rb_node, &threads->entries);
  452. RB_CLEAR_NODE(&th->rb_node);
  453. thread__put(th);
  454. return NULL;
  455. }
  456. /*
  457. * It is now in the rbtree, get a ref
  458. */
  459. thread__get(th);
  460. threads__set_last_match(threads, th);
  461. ++threads->nr;
  462. }
  463. return th;
  464. }
  465. struct thread *__machine__findnew_thread(struct machine *machine, pid_t pid, pid_t tid)
  466. {
  467. return ____machine__findnew_thread(machine, machine__threads(machine, tid), pid, tid, true);
  468. }
  469. struct thread *machine__findnew_thread(struct machine *machine, pid_t pid,
  470. pid_t tid)
  471. {
  472. struct threads *threads = machine__threads(machine, tid);
  473. struct thread *th;
  474. down_write(&threads->lock);
  475. th = __machine__findnew_thread(machine, pid, tid);
  476. up_write(&threads->lock);
  477. return th;
  478. }
  479. struct thread *machine__find_thread(struct machine *machine, pid_t pid,
  480. pid_t tid)
  481. {
  482. struct threads *threads = machine__threads(machine, tid);
  483. struct thread *th;
  484. down_read(&threads->lock);
  485. th = ____machine__findnew_thread(machine, threads, pid, tid, false);
  486. up_read(&threads->lock);
  487. return th;
  488. }
  489. struct comm *machine__thread_exec_comm(struct machine *machine,
  490. struct thread *thread)
  491. {
  492. if (machine->comm_exec)
  493. return thread__exec_comm(thread);
  494. else
  495. return thread__comm(thread);
  496. }
  497. int machine__process_comm_event(struct machine *machine, union perf_event *event,
  498. struct perf_sample *sample)
  499. {
  500. struct thread *thread = machine__findnew_thread(machine,
  501. event->comm.pid,
  502. event->comm.tid);
  503. bool exec = event->header.misc & PERF_RECORD_MISC_COMM_EXEC;
  504. int err = 0;
  505. if (exec)
  506. machine->comm_exec = true;
  507. if (dump_trace)
  508. perf_event__fprintf_comm(event, stdout);
  509. if (thread == NULL ||
  510. __thread__set_comm(thread, event->comm.comm, sample->time, exec)) {
  511. dump_printf("problem processing PERF_RECORD_COMM, skipping event.\n");
  512. err = -1;
  513. }
  514. thread__put(thread);
  515. return err;
  516. }
  517. int machine__process_namespaces_event(struct machine *machine __maybe_unused,
  518. union perf_event *event,
  519. struct perf_sample *sample __maybe_unused)
  520. {
  521. struct thread *thread = machine__findnew_thread(machine,
  522. event->namespaces.pid,
  523. event->namespaces.tid);
  524. int err = 0;
  525. WARN_ONCE(event->namespaces.nr_namespaces > NR_NAMESPACES,
  526. "\nWARNING: kernel seems to support more namespaces than perf"
  527. " tool.\nTry updating the perf tool..\n\n");
  528. WARN_ONCE(event->namespaces.nr_namespaces < NR_NAMESPACES,
  529. "\nWARNING: perf tool seems to support more namespaces than"
  530. " the kernel.\nTry updating the kernel..\n\n");
  531. if (dump_trace)
  532. perf_event__fprintf_namespaces(event, stdout);
  533. if (thread == NULL ||
  534. thread__set_namespaces(thread, sample->time, &event->namespaces)) {
  535. dump_printf("problem processing PERF_RECORD_NAMESPACES, skipping event.\n");
  536. err = -1;
  537. }
  538. thread__put(thread);
  539. return err;
  540. }
  541. int machine__process_cgroup_event(struct machine *machine,
  542. union perf_event *event,
  543. struct perf_sample *sample __maybe_unused)
  544. {
  545. struct cgroup *cgrp;
  546. if (dump_trace)
  547. perf_event__fprintf_cgroup(event, stdout);
  548. cgrp = cgroup__findnew(machine->env, event->cgroup.id, event->cgroup.path);
  549. if (cgrp == NULL)
  550. return -ENOMEM;
  551. return 0;
  552. }
  553. int machine__process_lost_event(struct machine *machine __maybe_unused,
  554. union perf_event *event, struct perf_sample *sample __maybe_unused)
  555. {
  556. dump_printf(": id:%" PRI_lu64 ": lost:%" PRI_lu64 "\n",
  557. event->lost.id, event->lost.lost);
  558. return 0;
  559. }
  560. int machine__process_lost_samples_event(struct machine *machine __maybe_unused,
  561. union perf_event *event, struct perf_sample *sample)
  562. {
  563. dump_printf(": id:%" PRIu64 ": lost samples :%" PRI_lu64 "\n",
  564. sample->id, event->lost_samples.lost);
  565. return 0;
  566. }
  567. static struct dso *machine__findnew_module_dso(struct machine *machine,
  568. struct kmod_path *m,
  569. const char *filename)
  570. {
  571. struct dso *dso;
  572. down_write(&machine->dsos.lock);
  573. dso = __dsos__find(&machine->dsos, m->name, true);
  574. if (!dso) {
  575. dso = __dsos__addnew(&machine->dsos, m->name);
  576. if (dso == NULL)
  577. goto out_unlock;
  578. dso__set_module_info(dso, m, machine);
  579. dso__set_long_name(dso, strdup(filename), true);
  580. dso->kernel = DSO_SPACE__KERNEL;
  581. }
  582. dso__get(dso);
  583. out_unlock:
  584. up_write(&machine->dsos.lock);
  585. return dso;
  586. }
  587. int machine__process_aux_event(struct machine *machine __maybe_unused,
  588. union perf_event *event)
  589. {
  590. if (dump_trace)
  591. perf_event__fprintf_aux(event, stdout);
  592. return 0;
  593. }
  594. int machine__process_itrace_start_event(struct machine *machine __maybe_unused,
  595. union perf_event *event)
  596. {
  597. if (dump_trace)
  598. perf_event__fprintf_itrace_start(event, stdout);
  599. return 0;
  600. }
  601. int machine__process_switch_event(struct machine *machine __maybe_unused,
  602. union perf_event *event)
  603. {
  604. if (dump_trace)
  605. perf_event__fprintf_switch(event, stdout);
  606. return 0;
  607. }
  608. static int machine__process_ksymbol_register(struct machine *machine,
  609. union perf_event *event,
  610. struct perf_sample *sample __maybe_unused)
  611. {
  612. struct symbol *sym;
  613. struct map *map = maps__find(&machine->kmaps, event->ksymbol.addr);
  614. if (!map) {
  615. struct dso *dso = dso__new(event->ksymbol.name);
  616. if (dso) {
  617. dso->kernel = DSO_SPACE__KERNEL;
  618. map = map__new2(0, dso);
  619. }
  620. if (!dso || !map) {
  621. dso__put(dso);
  622. return -ENOMEM;
  623. }
  624. if (event->ksymbol.ksym_type == PERF_RECORD_KSYMBOL_TYPE_OOL) {
  625. map->dso->binary_type = DSO_BINARY_TYPE__OOL;
  626. map->dso->data.file_size = event->ksymbol.len;
  627. dso__set_loaded(map->dso);
  628. }
  629. map->start = event->ksymbol.addr;
  630. map->end = map->start + event->ksymbol.len;
  631. maps__insert(&machine->kmaps, map);
  632. dso__set_loaded(dso);
  633. if (is_bpf_image(event->ksymbol.name)) {
  634. dso->binary_type = DSO_BINARY_TYPE__BPF_IMAGE;
  635. dso__set_long_name(dso, "", false);
  636. }
  637. }
  638. sym = symbol__new(map->map_ip(map, map->start),
  639. event->ksymbol.len,
  640. 0, 0, event->ksymbol.name);
  641. if (!sym)
  642. return -ENOMEM;
  643. dso__insert_symbol(map->dso, sym);
  644. return 0;
  645. }
  646. static int machine__process_ksymbol_unregister(struct machine *machine,
  647. union perf_event *event,
  648. struct perf_sample *sample __maybe_unused)
  649. {
  650. struct symbol *sym;
  651. struct map *map;
  652. map = maps__find(&machine->kmaps, event->ksymbol.addr);
  653. if (!map)
  654. return 0;
  655. if (map != machine->vmlinux_map)
  656. maps__remove(&machine->kmaps, map);
  657. else {
  658. sym = dso__find_symbol(map->dso, map->map_ip(map, map->start));
  659. if (sym)
  660. dso__delete_symbol(map->dso, sym);
  661. }
  662. return 0;
  663. }
  664. int machine__process_ksymbol(struct machine *machine __maybe_unused,
  665. union perf_event *event,
  666. struct perf_sample *sample)
  667. {
  668. if (dump_trace)
  669. perf_event__fprintf_ksymbol(event, stdout);
  670. if (event->ksymbol.flags & PERF_RECORD_KSYMBOL_FLAGS_UNREGISTER)
  671. return machine__process_ksymbol_unregister(machine, event,
  672. sample);
  673. return machine__process_ksymbol_register(machine, event, sample);
  674. }
  675. int machine__process_text_poke(struct machine *machine, union perf_event *event,
  676. struct perf_sample *sample __maybe_unused)
  677. {
  678. struct map *map = maps__find(&machine->kmaps, event->text_poke.addr);
  679. u8 cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK;
  680. if (dump_trace)
  681. perf_event__fprintf_text_poke(event, machine, stdout);
  682. if (!event->text_poke.new_len)
  683. return 0;
  684. if (cpumode != PERF_RECORD_MISC_KERNEL) {
  685. pr_debug("%s: unsupported cpumode - ignoring\n", __func__);
  686. return 0;
  687. }
  688. if (map && map->dso) {
  689. u8 *new_bytes = event->text_poke.bytes + event->text_poke.old_len;
  690. int ret;
  691. /*
  692. * Kernel maps might be changed when loading symbols so loading
  693. * must be done prior to using kernel maps.
  694. */
  695. map__load(map);
  696. ret = dso__data_write_cache_addr(map->dso, map, machine,
  697. event->text_poke.addr,
  698. new_bytes,
  699. event->text_poke.new_len);
  700. if (ret != event->text_poke.new_len)
  701. pr_debug("Failed to write kernel text poke at %#" PRI_lx64 "\n",
  702. event->text_poke.addr);
  703. } else {
  704. pr_debug("Failed to find kernel text poke address map for %#" PRI_lx64 "\n",
  705. event->text_poke.addr);
  706. }
  707. return 0;
  708. }
  709. static struct map *machine__addnew_module_map(struct machine *machine, u64 start,
  710. const char *filename)
  711. {
  712. struct map *map = NULL;
  713. struct kmod_path m;
  714. struct dso *dso;
  715. if (kmod_path__parse_name(&m, filename))
  716. return NULL;
  717. dso = machine__findnew_module_dso(machine, &m, filename);
  718. if (dso == NULL)
  719. goto out;
  720. map = map__new2(start, dso);
  721. if (map == NULL)
  722. goto out;
  723. maps__insert(&machine->kmaps, map);
  724. /* Put the map here because maps__insert alread got it */
  725. map__put(map);
  726. out:
  727. /* put the dso here, corresponding to machine__findnew_module_dso */
  728. dso__put(dso);
  729. zfree(&m.name);
  730. return map;
  731. }
  732. size_t machines__fprintf_dsos(struct machines *machines, FILE *fp)
  733. {
  734. struct rb_node *nd;
  735. size_t ret = __dsos__fprintf(&machines->host.dsos.head, fp);
  736. for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
  737. struct machine *pos = rb_entry(nd, struct machine, rb_node);
  738. ret += __dsos__fprintf(&pos->dsos.head, fp);
  739. }
  740. return ret;
  741. }
  742. size_t machine__fprintf_dsos_buildid(struct machine *m, FILE *fp,
  743. bool (skip)(struct dso *dso, int parm), int parm)
  744. {
  745. return __dsos__fprintf_buildid(&m->dsos.head, fp, skip, parm);
  746. }
  747. size_t machines__fprintf_dsos_buildid(struct machines *machines, FILE *fp,
  748. bool (skip)(struct dso *dso, int parm), int parm)
  749. {
  750. struct rb_node *nd;
  751. size_t ret = machine__fprintf_dsos_buildid(&machines->host, fp, skip, parm);
  752. for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
  753. struct machine *pos = rb_entry(nd, struct machine, rb_node);
  754. ret += machine__fprintf_dsos_buildid(pos, fp, skip, parm);
  755. }
  756. return ret;
  757. }
  758. size_t machine__fprintf_vmlinux_path(struct machine *machine, FILE *fp)
  759. {
  760. int i;
  761. size_t printed = 0;
  762. struct dso *kdso = machine__kernel_dso(machine);
  763. if (kdso->has_build_id) {
  764. char filename[PATH_MAX];
  765. if (dso__build_id_filename(kdso, filename, sizeof(filename),
  766. false))
  767. printed += fprintf(fp, "[0] %s\n", filename);
  768. }
  769. for (i = 0; i < vmlinux_path__nr_entries; ++i)
  770. printed += fprintf(fp, "[%d] %s\n",
  771. i + kdso->has_build_id, vmlinux_path[i]);
  772. return printed;
  773. }
  774. size_t machine__fprintf(struct machine *machine, FILE *fp)
  775. {
  776. struct rb_node *nd;
  777. size_t ret;
  778. int i;
  779. for (i = 0; i < THREADS__TABLE_SIZE; i++) {
  780. struct threads *threads = &machine->threads[i];
  781. down_read(&threads->lock);
  782. ret = fprintf(fp, "Threads: %u\n", threads->nr);
  783. for (nd = rb_first_cached(&threads->entries); nd;
  784. nd = rb_next(nd)) {
  785. struct thread *pos = rb_entry(nd, struct thread, rb_node);
  786. ret += thread__fprintf(pos, fp);
  787. }
  788. up_read(&threads->lock);
  789. }
  790. return ret;
  791. }
  792. static struct dso *machine__get_kernel(struct machine *machine)
  793. {
  794. const char *vmlinux_name = machine->mmap_name;
  795. struct dso *kernel;
  796. if (machine__is_host(machine)) {
  797. if (symbol_conf.vmlinux_name)
  798. vmlinux_name = symbol_conf.vmlinux_name;
  799. kernel = machine__findnew_kernel(machine, vmlinux_name,
  800. "[kernel]", DSO_SPACE__KERNEL);
  801. } else {
  802. if (symbol_conf.default_guest_vmlinux_name)
  803. vmlinux_name = symbol_conf.default_guest_vmlinux_name;
  804. kernel = machine__findnew_kernel(machine, vmlinux_name,
  805. "[guest.kernel]",
  806. DSO_SPACE__KERNEL_GUEST);
  807. }
  808. if (kernel != NULL && (!kernel->has_build_id))
  809. dso__read_running_kernel_build_id(kernel, machine);
  810. return kernel;
  811. }
  812. struct process_args {
  813. u64 start;
  814. };
  815. void machine__get_kallsyms_filename(struct machine *machine, char *buf,
  816. size_t bufsz)
  817. {
  818. if (machine__is_default_guest(machine))
  819. scnprintf(buf, bufsz, "%s", symbol_conf.default_guest_kallsyms);
  820. else
  821. scnprintf(buf, bufsz, "%s/proc/kallsyms", machine->root_dir);
  822. }
  823. const char *ref_reloc_sym_names[] = {"_text", "_stext", NULL};
  824. /* Figure out the start address of kernel map from /proc/kallsyms.
  825. * Returns the name of the start symbol in *symbol_name. Pass in NULL as
  826. * symbol_name if it's not that important.
  827. */
  828. static int machine__get_running_kernel_start(struct machine *machine,
  829. const char **symbol_name,
  830. u64 *start, u64 *end)
  831. {
  832. char filename[PATH_MAX];
  833. int i, err = -1;
  834. const char *name;
  835. u64 addr = 0;
  836. machine__get_kallsyms_filename(machine, filename, PATH_MAX);
  837. if (symbol__restricted_filename(filename, "/proc/kallsyms"))
  838. return 0;
  839. for (i = 0; (name = ref_reloc_sym_names[i]) != NULL; i++) {
  840. err = kallsyms__get_function_start(filename, name, &addr);
  841. if (!err)
  842. break;
  843. }
  844. if (err)
  845. return -1;
  846. if (symbol_name)
  847. *symbol_name = name;
  848. *start = addr;
  849. err = kallsyms__get_function_start(filename, "_etext", &addr);
  850. if (!err)
  851. *end = addr;
  852. return 0;
  853. }
  854. int machine__create_extra_kernel_map(struct machine *machine,
  855. struct dso *kernel,
  856. struct extra_kernel_map *xm)
  857. {
  858. struct kmap *kmap;
  859. struct map *map;
  860. map = map__new2(xm->start, kernel);
  861. if (!map)
  862. return -1;
  863. map->end = xm->end;
  864. map->pgoff = xm->pgoff;
  865. kmap = map__kmap(map);
  866. strlcpy(kmap->name, xm->name, KMAP_NAME_LEN);
  867. maps__insert(&machine->kmaps, map);
  868. pr_debug2("Added extra kernel map %s %" PRIx64 "-%" PRIx64 "\n",
  869. kmap->name, map->start, map->end);
  870. map__put(map);
  871. return 0;
  872. }
  873. static u64 find_entry_trampoline(struct dso *dso)
  874. {
  875. /* Duplicates are removed so lookup all aliases */
  876. const char *syms[] = {
  877. "_entry_trampoline",
  878. "__entry_trampoline_start",
  879. "entry_SYSCALL_64_trampoline",
  880. };
  881. struct symbol *sym = dso__first_symbol(dso);
  882. unsigned int i;
  883. for (; sym; sym = dso__next_symbol(sym)) {
  884. if (sym->binding != STB_GLOBAL)
  885. continue;
  886. for (i = 0; i < ARRAY_SIZE(syms); i++) {
  887. if (!strcmp(sym->name, syms[i]))
  888. return sym->start;
  889. }
  890. }
  891. return 0;
  892. }
  893. /*
  894. * These values can be used for kernels that do not have symbols for the entry
  895. * trampolines in kallsyms.
  896. */
  897. #define X86_64_CPU_ENTRY_AREA_PER_CPU 0xfffffe0000000000ULL
  898. #define X86_64_CPU_ENTRY_AREA_SIZE 0x2c000
  899. #define X86_64_ENTRY_TRAMPOLINE 0x6000
  900. /* Map x86_64 PTI entry trampolines */
  901. int machine__map_x86_64_entry_trampolines(struct machine *machine,
  902. struct dso *kernel)
  903. {
  904. struct maps *kmaps = &machine->kmaps;
  905. int nr_cpus_avail, cpu;
  906. bool found = false;
  907. struct map *map;
  908. u64 pgoff;
  909. /*
  910. * In the vmlinux case, pgoff is a virtual address which must now be
  911. * mapped to a vmlinux offset.
  912. */
  913. maps__for_each_entry(kmaps, map) {
  914. struct kmap *kmap = __map__kmap(map);
  915. struct map *dest_map;
  916. if (!kmap || !is_entry_trampoline(kmap->name))
  917. continue;
  918. dest_map = maps__find(kmaps, map->pgoff);
  919. if (dest_map != map)
  920. map->pgoff = dest_map->map_ip(dest_map, map->pgoff);
  921. found = true;
  922. }
  923. if (found || machine->trampolines_mapped)
  924. return 0;
  925. pgoff = find_entry_trampoline(kernel);
  926. if (!pgoff)
  927. return 0;
  928. nr_cpus_avail = machine__nr_cpus_avail(machine);
  929. /* Add a 1 page map for each CPU's entry trampoline */
  930. for (cpu = 0; cpu < nr_cpus_avail; cpu++) {
  931. u64 va = X86_64_CPU_ENTRY_AREA_PER_CPU +
  932. cpu * X86_64_CPU_ENTRY_AREA_SIZE +
  933. X86_64_ENTRY_TRAMPOLINE;
  934. struct extra_kernel_map xm = {
  935. .start = va,
  936. .end = va + page_size,
  937. .pgoff = pgoff,
  938. };
  939. strlcpy(xm.name, ENTRY_TRAMPOLINE_NAME, KMAP_NAME_LEN);
  940. if (machine__create_extra_kernel_map(machine, kernel, &xm) < 0)
  941. return -1;
  942. }
  943. machine->trampolines_mapped = nr_cpus_avail;
  944. return 0;
  945. }
  946. int __weak machine__create_extra_kernel_maps(struct machine *machine __maybe_unused,
  947. struct dso *kernel __maybe_unused)
  948. {
  949. return 0;
  950. }
  951. static int
  952. __machine__create_kernel_maps(struct machine *machine, struct dso *kernel)
  953. {
  954. /* In case of renewal the kernel map, destroy previous one */
  955. machine__destroy_kernel_maps(machine);
  956. machine->vmlinux_map = map__new2(0, kernel);
  957. if (machine->vmlinux_map == NULL)
  958. return -1;
  959. machine->vmlinux_map->map_ip = machine->vmlinux_map->unmap_ip = identity__map_ip;
  960. maps__insert(&machine->kmaps, machine->vmlinux_map);
  961. return 0;
  962. }
  963. void machine__destroy_kernel_maps(struct machine *machine)
  964. {
  965. struct kmap *kmap;
  966. struct map *map = machine__kernel_map(machine);
  967. if (map == NULL)
  968. return;
  969. kmap = map__kmap(map);
  970. maps__remove(&machine->kmaps, map);
  971. if (kmap && kmap->ref_reloc_sym) {
  972. zfree((char **)&kmap->ref_reloc_sym->name);
  973. zfree(&kmap->ref_reloc_sym);
  974. }
  975. map__zput(machine->vmlinux_map);
  976. }
  977. int machines__create_guest_kernel_maps(struct machines *machines)
  978. {
  979. int ret = 0;
  980. struct dirent **namelist = NULL;
  981. int i, items = 0;
  982. char path[PATH_MAX];
  983. pid_t pid;
  984. char *endp;
  985. if (symbol_conf.default_guest_vmlinux_name ||
  986. symbol_conf.default_guest_modules ||
  987. symbol_conf.default_guest_kallsyms) {
  988. machines__create_kernel_maps(machines, DEFAULT_GUEST_KERNEL_ID);
  989. }
  990. if (symbol_conf.guestmount) {
  991. items = scandir(symbol_conf.guestmount, &namelist, NULL, NULL);
  992. if (items <= 0)
  993. return -ENOENT;
  994. for (i = 0; i < items; i++) {
  995. if (!isdigit(namelist[i]->d_name[0])) {
  996. /* Filter out . and .. */
  997. continue;
  998. }
  999. pid = (pid_t)strtol(namelist[i]->d_name, &endp, 10);
  1000. if ((*endp != '\0') ||
  1001. (endp == namelist[i]->d_name) ||
  1002. (errno == ERANGE)) {
  1003. pr_debug("invalid directory (%s). Skipping.\n",
  1004. namelist[i]->d_name);
  1005. continue;
  1006. }
  1007. sprintf(path, "%s/%s/proc/kallsyms",
  1008. symbol_conf.guestmount,
  1009. namelist[i]->d_name);
  1010. ret = access(path, R_OK);
  1011. if (ret) {
  1012. pr_debug("Can't access file %s\n", path);
  1013. goto failure;
  1014. }
  1015. machines__create_kernel_maps(machines, pid);
  1016. }
  1017. failure:
  1018. free(namelist);
  1019. }
  1020. return ret;
  1021. }
  1022. void machines__destroy_kernel_maps(struct machines *machines)
  1023. {
  1024. struct rb_node *next = rb_first_cached(&machines->guests);
  1025. machine__destroy_kernel_maps(&machines->host);
  1026. while (next) {
  1027. struct machine *pos = rb_entry(next, struct machine, rb_node);
  1028. next = rb_next(&pos->rb_node);
  1029. rb_erase_cached(&pos->rb_node, &machines->guests);
  1030. machine__delete(pos);
  1031. }
  1032. }
  1033. int machines__create_kernel_maps(struct machines *machines, pid_t pid)
  1034. {
  1035. struct machine *machine = machines__findnew(machines, pid);
  1036. if (machine == NULL)
  1037. return -1;
  1038. return machine__create_kernel_maps(machine);
  1039. }
  1040. int machine__load_kallsyms(struct machine *machine, const char *filename)
  1041. {
  1042. struct map *map = machine__kernel_map(machine);
  1043. int ret = __dso__load_kallsyms(map->dso, filename, map, true);
  1044. if (ret > 0) {
  1045. dso__set_loaded(map->dso);
  1046. /*
  1047. * Since /proc/kallsyms will have multiple sessions for the
  1048. * kernel, with modules between them, fixup the end of all
  1049. * sections.
  1050. */
  1051. maps__fixup_end(&machine->kmaps);
  1052. }
  1053. return ret;
  1054. }
  1055. int machine__load_vmlinux_path(struct machine *machine)
  1056. {
  1057. struct map *map = machine__kernel_map(machine);
  1058. int ret = dso__load_vmlinux_path(map->dso, map);
  1059. if (ret > 0)
  1060. dso__set_loaded(map->dso);
  1061. return ret;
  1062. }
  1063. static char *get_kernel_version(const char *root_dir)
  1064. {
  1065. char version[PATH_MAX];
  1066. FILE *file;
  1067. char *name, *tmp;
  1068. const char *prefix = "Linux version ";
  1069. sprintf(version, "%s/proc/version", root_dir);
  1070. file = fopen(version, "r");
  1071. if (!file)
  1072. return NULL;
  1073. tmp = fgets(version, sizeof(version), file);
  1074. fclose(file);
  1075. if (!tmp)
  1076. return NULL;
  1077. name = strstr(version, prefix);
  1078. if (!name)
  1079. return NULL;
  1080. name += strlen(prefix);
  1081. tmp = strchr(name, ' ');
  1082. if (tmp)
  1083. *tmp = '\0';
  1084. return strdup(name);
  1085. }
  1086. static bool is_kmod_dso(struct dso *dso)
  1087. {
  1088. return dso->symtab_type == DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE ||
  1089. dso->symtab_type == DSO_BINARY_TYPE__GUEST_KMODULE;
  1090. }
  1091. static int maps__set_module_path(struct maps *maps, const char *path, struct kmod_path *m)
  1092. {
  1093. char *long_name;
  1094. struct map *map = maps__find_by_name(maps, m->name);
  1095. if (map == NULL)
  1096. return 0;
  1097. long_name = strdup(path);
  1098. if (long_name == NULL)
  1099. return -ENOMEM;
  1100. dso__set_long_name(map->dso, long_name, true);
  1101. dso__kernel_module_get_build_id(map->dso, "");
  1102. /*
  1103. * Full name could reveal us kmod compression, so
  1104. * we need to update the symtab_type if needed.
  1105. */
  1106. if (m->comp && is_kmod_dso(map->dso)) {
  1107. map->dso->symtab_type++;
  1108. map->dso->comp = m->comp;
  1109. }
  1110. return 0;
  1111. }
  1112. static int maps__set_modules_path_dir(struct maps *maps, const char *dir_name, int depth)
  1113. {
  1114. struct dirent *dent;
  1115. DIR *dir = opendir(dir_name);
  1116. int ret = 0;
  1117. if (!dir) {
  1118. pr_debug("%s: cannot open %s dir\n", __func__, dir_name);
  1119. return -1;
  1120. }
  1121. while ((dent = readdir(dir)) != NULL) {
  1122. char path[PATH_MAX];
  1123. struct stat st;
  1124. /*sshfs might return bad dent->d_type, so we have to stat*/
  1125. snprintf(path, sizeof(path), "%s/%s", dir_name, dent->d_name);
  1126. if (stat(path, &st))
  1127. continue;
  1128. if (S_ISDIR(st.st_mode)) {
  1129. if (!strcmp(dent->d_name, ".") ||
  1130. !strcmp(dent->d_name, ".."))
  1131. continue;
  1132. /* Do not follow top-level source and build symlinks */
  1133. if (depth == 0) {
  1134. if (!strcmp(dent->d_name, "source") ||
  1135. !strcmp(dent->d_name, "build"))
  1136. continue;
  1137. }
  1138. ret = maps__set_modules_path_dir(maps, path, depth + 1);
  1139. if (ret < 0)
  1140. goto out;
  1141. } else {
  1142. struct kmod_path m;
  1143. ret = kmod_path__parse_name(&m, dent->d_name);
  1144. if (ret)
  1145. goto out;
  1146. if (m.kmod)
  1147. ret = maps__set_module_path(maps, path, &m);
  1148. zfree(&m.name);
  1149. if (ret)
  1150. goto out;
  1151. }
  1152. }
  1153. out:
  1154. closedir(dir);
  1155. return ret;
  1156. }
  1157. static int machine__set_modules_path(struct machine *machine)
  1158. {
  1159. char *version;
  1160. char modules_path[PATH_MAX];
  1161. version = get_kernel_version(machine->root_dir);
  1162. if (!version)
  1163. return -1;
  1164. snprintf(modules_path, sizeof(modules_path), "%s/lib/modules/%s",
  1165. machine->root_dir, version);
  1166. free(version);
  1167. return maps__set_modules_path_dir(&machine->kmaps, modules_path, 0);
  1168. }
  1169. int __weak arch__fix_module_text_start(u64 *start __maybe_unused,
  1170. u64 *size __maybe_unused,
  1171. const char *name __maybe_unused)
  1172. {
  1173. return 0;
  1174. }
  1175. static int machine__create_module(void *arg, const char *name, u64 start,
  1176. u64 size)
  1177. {
  1178. struct machine *machine = arg;
  1179. struct map *map;
  1180. if (arch__fix_module_text_start(&start, &size, name) < 0)
  1181. return -1;
  1182. map = machine__addnew_module_map(machine, start, name);
  1183. if (map == NULL)
  1184. return -1;
  1185. map->end = start + size;
  1186. dso__kernel_module_get_build_id(map->dso, machine->root_dir);
  1187. return 0;
  1188. }
  1189. static int machine__create_modules(struct machine *machine)
  1190. {
  1191. const char *modules;
  1192. char path[PATH_MAX];
  1193. if (machine__is_default_guest(machine)) {
  1194. modules = symbol_conf.default_guest_modules;
  1195. } else {
  1196. snprintf(path, PATH_MAX, "%s/proc/modules", machine->root_dir);
  1197. modules = path;
  1198. }
  1199. if (symbol__restricted_filename(modules, "/proc/modules"))
  1200. return -1;
  1201. if (modules__parse(modules, machine, machine__create_module))
  1202. return -1;
  1203. if (!machine__set_modules_path(machine))
  1204. return 0;
  1205. pr_debug("Problems setting modules path maps, continuing anyway...\n");
  1206. return 0;
  1207. }
  1208. static void machine__set_kernel_mmap(struct machine *machine,
  1209. u64 start, u64 end)
  1210. {
  1211. machine->vmlinux_map->start = start;
  1212. machine->vmlinux_map->end = end;
  1213. /*
  1214. * Be a bit paranoid here, some perf.data file came with
  1215. * a zero sized synthesized MMAP event for the kernel.
  1216. */
  1217. if (start == 0 && end == 0)
  1218. machine->vmlinux_map->end = ~0ULL;
  1219. }
  1220. static void machine__update_kernel_mmap(struct machine *machine,
  1221. u64 start, u64 end)
  1222. {
  1223. struct map *map = machine__kernel_map(machine);
  1224. map__get(map);
  1225. maps__remove(&machine->kmaps, map);
  1226. machine__set_kernel_mmap(machine, start, end);
  1227. maps__insert(&machine->kmaps, map);
  1228. map__put(map);
  1229. }
  1230. int machine__create_kernel_maps(struct machine *machine)
  1231. {
  1232. struct dso *kernel = machine__get_kernel(machine);
  1233. const char *name = NULL;
  1234. struct map *map;
  1235. u64 start = 0, end = ~0ULL;
  1236. int ret;
  1237. if (kernel == NULL)
  1238. return -1;
  1239. ret = __machine__create_kernel_maps(machine, kernel);
  1240. if (ret < 0)
  1241. goto out_put;
  1242. if (symbol_conf.use_modules && machine__create_modules(machine) < 0) {
  1243. if (machine__is_host(machine))
  1244. pr_debug("Problems creating module maps, "
  1245. "continuing anyway...\n");
  1246. else
  1247. pr_debug("Problems creating module maps for guest %d, "
  1248. "continuing anyway...\n", machine->pid);
  1249. }
  1250. if (!machine__get_running_kernel_start(machine, &name, &start, &end)) {
  1251. if (name &&
  1252. map__set_kallsyms_ref_reloc_sym(machine->vmlinux_map, name, start)) {
  1253. machine__destroy_kernel_maps(machine);
  1254. ret = -1;
  1255. goto out_put;
  1256. }
  1257. /*
  1258. * we have a real start address now, so re-order the kmaps
  1259. * assume it's the last in the kmaps
  1260. */
  1261. machine__update_kernel_mmap(machine, start, end);
  1262. }
  1263. if (machine__create_extra_kernel_maps(machine, kernel))
  1264. pr_debug("Problems creating extra kernel maps, continuing anyway...\n");
  1265. if (end == ~0ULL) {
  1266. /* update end address of the kernel map using adjacent module address */
  1267. map = map__next(machine__kernel_map(machine));
  1268. if (map)
  1269. machine__set_kernel_mmap(machine, start, map->start);
  1270. }
  1271. out_put:
  1272. dso__put(kernel);
  1273. return ret;
  1274. }
  1275. static bool machine__uses_kcore(struct machine *machine)
  1276. {
  1277. struct dso *dso;
  1278. list_for_each_entry(dso, &machine->dsos.head, node) {
  1279. if (dso__is_kcore(dso))
  1280. return true;
  1281. }
  1282. return false;
  1283. }
  1284. static bool perf_event__is_extra_kernel_mmap(struct machine *machine,
  1285. union perf_event *event)
  1286. {
  1287. return machine__is(machine, "x86_64") &&
  1288. is_entry_trampoline(event->mmap.filename);
  1289. }
  1290. static int machine__process_extra_kernel_map(struct machine *machine,
  1291. union perf_event *event)
  1292. {
  1293. struct dso *kernel = machine__kernel_dso(machine);
  1294. struct extra_kernel_map xm = {
  1295. .start = event->mmap.start,
  1296. .end = event->mmap.start + event->mmap.len,
  1297. .pgoff = event->mmap.pgoff,
  1298. };
  1299. if (kernel == NULL)
  1300. return -1;
  1301. strlcpy(xm.name, event->mmap.filename, KMAP_NAME_LEN);
  1302. return machine__create_extra_kernel_map(machine, kernel, &xm);
  1303. }
  1304. static int machine__process_kernel_mmap_event(struct machine *machine,
  1305. union perf_event *event)
  1306. {
  1307. struct map *map;
  1308. enum dso_space_type dso_space;
  1309. bool is_kernel_mmap;
  1310. /* If we have maps from kcore then we do not need or want any others */
  1311. if (machine__uses_kcore(machine))
  1312. return 0;
  1313. if (machine__is_host(machine))
  1314. dso_space = DSO_SPACE__KERNEL;
  1315. else
  1316. dso_space = DSO_SPACE__KERNEL_GUEST;
  1317. is_kernel_mmap = memcmp(event->mmap.filename,
  1318. machine->mmap_name,
  1319. strlen(machine->mmap_name) - 1) == 0;
  1320. if (event->mmap.filename[0] == '/' ||
  1321. (!is_kernel_mmap && event->mmap.filename[0] == '[')) {
  1322. map = machine__addnew_module_map(machine, event->mmap.start,
  1323. event->mmap.filename);
  1324. if (map == NULL)
  1325. goto out_problem;
  1326. map->end = map->start + event->mmap.len;
  1327. } else if (is_kernel_mmap) {
  1328. const char *symbol_name = (event->mmap.filename +
  1329. strlen(machine->mmap_name));
  1330. /*
  1331. * Should be there already, from the build-id table in
  1332. * the header.
  1333. */
  1334. struct dso *kernel = NULL;
  1335. struct dso *dso;
  1336. down_read(&machine->dsos.lock);
  1337. list_for_each_entry(dso, &machine->dsos.head, node) {
  1338. /*
  1339. * The cpumode passed to is_kernel_module is not the
  1340. * cpumode of *this* event. If we insist on passing
  1341. * correct cpumode to is_kernel_module, we should
  1342. * record the cpumode when we adding this dso to the
  1343. * linked list.
  1344. *
  1345. * However we don't really need passing correct
  1346. * cpumode. We know the correct cpumode must be kernel
  1347. * mode (if not, we should not link it onto kernel_dsos
  1348. * list).
  1349. *
  1350. * Therefore, we pass PERF_RECORD_MISC_CPUMODE_UNKNOWN.
  1351. * is_kernel_module() treats it as a kernel cpumode.
  1352. */
  1353. if (!dso->kernel ||
  1354. is_kernel_module(dso->long_name,
  1355. PERF_RECORD_MISC_CPUMODE_UNKNOWN))
  1356. continue;
  1357. kernel = dso;
  1358. break;
  1359. }
  1360. up_read(&machine->dsos.lock);
  1361. if (kernel == NULL)
  1362. kernel = machine__findnew_dso(machine, machine->mmap_name);
  1363. if (kernel == NULL)
  1364. goto out_problem;
  1365. kernel->kernel = dso_space;
  1366. if (__machine__create_kernel_maps(machine, kernel) < 0) {
  1367. dso__put(kernel);
  1368. goto out_problem;
  1369. }
  1370. if (strstr(kernel->long_name, "vmlinux"))
  1371. dso__set_short_name(kernel, "[kernel.vmlinux]", false);
  1372. machine__update_kernel_mmap(machine, event->mmap.start,
  1373. event->mmap.start + event->mmap.len);
  1374. /*
  1375. * Avoid using a zero address (kptr_restrict) for the ref reloc
  1376. * symbol. Effectively having zero here means that at record
  1377. * time /proc/sys/kernel/kptr_restrict was non zero.
  1378. */
  1379. if (event->mmap.pgoff != 0) {
  1380. map__set_kallsyms_ref_reloc_sym(machine->vmlinux_map,
  1381. symbol_name,
  1382. event->mmap.pgoff);
  1383. }
  1384. if (machine__is_default_guest(machine)) {
  1385. /*
  1386. * preload dso of guest kernel and modules
  1387. */
  1388. dso__load(kernel, machine__kernel_map(machine));
  1389. }
  1390. } else if (perf_event__is_extra_kernel_mmap(machine, event)) {
  1391. return machine__process_extra_kernel_map(machine, event);
  1392. }
  1393. return 0;
  1394. out_problem:
  1395. return -1;
  1396. }
  1397. int machine__process_mmap2_event(struct machine *machine,
  1398. union perf_event *event,
  1399. struct perf_sample *sample)
  1400. {
  1401. struct thread *thread;
  1402. struct map *map;
  1403. struct dso_id dso_id = {
  1404. .maj = event->mmap2.maj,
  1405. .min = event->mmap2.min,
  1406. .ino = event->mmap2.ino,
  1407. .ino_generation = event->mmap2.ino_generation,
  1408. };
  1409. int ret = 0;
  1410. if (dump_trace)
  1411. perf_event__fprintf_mmap2(event, stdout);
  1412. if (sample->cpumode == PERF_RECORD_MISC_GUEST_KERNEL ||
  1413. sample->cpumode == PERF_RECORD_MISC_KERNEL) {
  1414. ret = machine__process_kernel_mmap_event(machine, event);
  1415. if (ret < 0)
  1416. goto out_problem;
  1417. return 0;
  1418. }
  1419. thread = machine__findnew_thread(machine, event->mmap2.pid,
  1420. event->mmap2.tid);
  1421. if (thread == NULL)
  1422. goto out_problem;
  1423. map = map__new(machine, event->mmap2.start,
  1424. event->mmap2.len, event->mmap2.pgoff,
  1425. &dso_id, event->mmap2.prot,
  1426. event->mmap2.flags,
  1427. event->mmap2.filename, thread);
  1428. if (map == NULL)
  1429. goto out_problem_map;
  1430. ret = thread__insert_map(thread, map);
  1431. if (ret)
  1432. goto out_problem_insert;
  1433. thread__put(thread);
  1434. map__put(map);
  1435. return 0;
  1436. out_problem_insert:
  1437. map__put(map);
  1438. out_problem_map:
  1439. thread__put(thread);
  1440. out_problem:
  1441. dump_printf("problem processing PERF_RECORD_MMAP2, skipping event.\n");
  1442. return 0;
  1443. }
  1444. int machine__process_mmap_event(struct machine *machine, union perf_event *event,
  1445. struct perf_sample *sample)
  1446. {
  1447. struct thread *thread;
  1448. struct map *map;
  1449. u32 prot = 0;
  1450. int ret = 0;
  1451. if (dump_trace)
  1452. perf_event__fprintf_mmap(event, stdout);
  1453. if (sample->cpumode == PERF_RECORD_MISC_GUEST_KERNEL ||
  1454. sample->cpumode == PERF_RECORD_MISC_KERNEL) {
  1455. ret = machine__process_kernel_mmap_event(machine, event);
  1456. if (ret < 0)
  1457. goto out_problem;
  1458. return 0;
  1459. }
  1460. thread = machine__findnew_thread(machine, event->mmap.pid,
  1461. event->mmap.tid);
  1462. if (thread == NULL)
  1463. goto out_problem;
  1464. if (!(event->header.misc & PERF_RECORD_MISC_MMAP_DATA))
  1465. prot = PROT_EXEC;
  1466. map = map__new(machine, event->mmap.start,
  1467. event->mmap.len, event->mmap.pgoff,
  1468. NULL, prot, 0, event->mmap.filename, thread);
  1469. if (map == NULL)
  1470. goto out_problem_map;
  1471. ret = thread__insert_map(thread, map);
  1472. if (ret)
  1473. goto out_problem_insert;
  1474. thread__put(thread);
  1475. map__put(map);
  1476. return 0;
  1477. out_problem_insert:
  1478. map__put(map);
  1479. out_problem_map:
  1480. thread__put(thread);
  1481. out_problem:
  1482. dump_printf("problem processing PERF_RECORD_MMAP, skipping event.\n");
  1483. return 0;
  1484. }
  1485. static void __machine__remove_thread(struct machine *machine, struct thread *th, bool lock)
  1486. {
  1487. struct threads *threads = machine__threads(machine, th->tid);
  1488. if (threads->last_match == th)
  1489. threads__set_last_match(threads, NULL);
  1490. if (lock)
  1491. down_write(&threads->lock);
  1492. BUG_ON(refcount_read(&th->refcnt) == 0);
  1493. rb_erase_cached(&th->rb_node, &threads->entries);
  1494. RB_CLEAR_NODE(&th->rb_node);
  1495. --threads->nr;
  1496. /*
  1497. * Move it first to the dead_threads list, then drop the reference,
  1498. * if this is the last reference, then the thread__delete destructor
  1499. * will be called and we will remove it from the dead_threads list.
  1500. */
  1501. list_add_tail(&th->node, &threads->dead);
  1502. /*
  1503. * We need to do the put here because if this is the last refcount,
  1504. * then we will be touching the threads->dead head when removing the
  1505. * thread.
  1506. */
  1507. thread__put(th);
  1508. if (lock)
  1509. up_write(&threads->lock);
  1510. }
  1511. void machine__remove_thread(struct machine *machine, struct thread *th)
  1512. {
  1513. return __machine__remove_thread(machine, th, true);
  1514. }
  1515. int machine__process_fork_event(struct machine *machine, union perf_event *event,
  1516. struct perf_sample *sample)
  1517. {
  1518. struct thread *thread = machine__find_thread(machine,
  1519. event->fork.pid,
  1520. event->fork.tid);
  1521. struct thread *parent = machine__findnew_thread(machine,
  1522. event->fork.ppid,
  1523. event->fork.ptid);
  1524. bool do_maps_clone = true;
  1525. int err = 0;
  1526. if (dump_trace)
  1527. perf_event__fprintf_task(event, stdout);
  1528. /*
  1529. * There may be an existing thread that is not actually the parent,
  1530. * either because we are processing events out of order, or because the
  1531. * (fork) event that would have removed the thread was lost. Assume the
  1532. * latter case and continue on as best we can.
  1533. */
  1534. if (parent->pid_ != (pid_t)event->fork.ppid) {
  1535. dump_printf("removing erroneous parent thread %d/%d\n",
  1536. parent->pid_, parent->tid);
  1537. machine__remove_thread(machine, parent);
  1538. thread__put(parent);
  1539. parent = machine__findnew_thread(machine, event->fork.ppid,
  1540. event->fork.ptid);
  1541. }
  1542. /* if a thread currently exists for the thread id remove it */
  1543. if (thread != NULL) {
  1544. machine__remove_thread(machine, thread);
  1545. thread__put(thread);
  1546. }
  1547. thread = machine__findnew_thread(machine, event->fork.pid,
  1548. event->fork.tid);
  1549. /*
  1550. * When synthesizing FORK events, we are trying to create thread
  1551. * objects for the already running tasks on the machine.
  1552. *
  1553. * Normally, for a kernel FORK event, we want to clone the parent's
  1554. * maps because that is what the kernel just did.
  1555. *
  1556. * But when synthesizing, this should not be done. If we do, we end up
  1557. * with overlapping maps as we process the sythesized MMAP2 events that
  1558. * get delivered shortly thereafter.
  1559. *
  1560. * Use the FORK event misc flags in an internal way to signal this
  1561. * situation, so we can elide the map clone when appropriate.
  1562. */
  1563. if (event->fork.header.misc & PERF_RECORD_MISC_FORK_EXEC)
  1564. do_maps_clone = false;
  1565. if (thread == NULL || parent == NULL ||
  1566. thread__fork(thread, parent, sample->time, do_maps_clone) < 0) {
  1567. dump_printf("problem processing PERF_RECORD_FORK, skipping event.\n");
  1568. err = -1;
  1569. }
  1570. thread__put(thread);
  1571. thread__put(parent);
  1572. return err;
  1573. }
  1574. int machine__process_exit_event(struct machine *machine, union perf_event *event,
  1575. struct perf_sample *sample __maybe_unused)
  1576. {
  1577. struct thread *thread = machine__find_thread(machine,
  1578. event->fork.pid,
  1579. event->fork.tid);
  1580. if (dump_trace)
  1581. perf_event__fprintf_task(event, stdout);
  1582. if (thread != NULL) {
  1583. thread__exited(thread);
  1584. thread__put(thread);
  1585. }
  1586. return 0;
  1587. }
  1588. int machine__process_event(struct machine *machine, union perf_event *event,
  1589. struct perf_sample *sample)
  1590. {
  1591. int ret;
  1592. switch (event->header.type) {
  1593. case PERF_RECORD_COMM:
  1594. ret = machine__process_comm_event(machine, event, sample); break;
  1595. case PERF_RECORD_MMAP:
  1596. ret = machine__process_mmap_event(machine, event, sample); break;
  1597. case PERF_RECORD_NAMESPACES:
  1598. ret = machine__process_namespaces_event(machine, event, sample); break;
  1599. case PERF_RECORD_CGROUP:
  1600. ret = machine__process_cgroup_event(machine, event, sample); break;
  1601. case PERF_RECORD_MMAP2:
  1602. ret = machine__process_mmap2_event(machine, event, sample); break;
  1603. case PERF_RECORD_FORK:
  1604. ret = machine__process_fork_event(machine, event, sample); break;
  1605. case PERF_RECORD_EXIT:
  1606. ret = machine__process_exit_event(machine, event, sample); break;
  1607. case PERF_RECORD_LOST:
  1608. ret = machine__process_lost_event(machine, event, sample); break;
  1609. case PERF_RECORD_AUX:
  1610. ret = machine__process_aux_event(machine, event); break;
  1611. case PERF_RECORD_ITRACE_START:
  1612. ret = machine__process_itrace_start_event(machine, event); break;
  1613. case PERF_RECORD_LOST_SAMPLES:
  1614. ret = machine__process_lost_samples_event(machine, event, sample); break;
  1615. case PERF_RECORD_SWITCH:
  1616. case PERF_RECORD_SWITCH_CPU_WIDE:
  1617. ret = machine__process_switch_event(machine, event); break;
  1618. case PERF_RECORD_KSYMBOL:
  1619. ret = machine__process_ksymbol(machine, event, sample); break;
  1620. case PERF_RECORD_BPF_EVENT:
  1621. ret = machine__process_bpf(machine, event, sample); break;
  1622. case PERF_RECORD_TEXT_POKE:
  1623. ret = machine__process_text_poke(machine, event, sample); break;
  1624. default:
  1625. ret = -1;
  1626. break;
  1627. }
  1628. return ret;
  1629. }
  1630. static bool symbol__match_regex(struct symbol *sym, regex_t *regex)
  1631. {
  1632. if (!regexec(regex, sym->name, 0, NULL, 0))
  1633. return 1;
  1634. return 0;
  1635. }
  1636. static void ip__resolve_ams(struct thread *thread,
  1637. struct addr_map_symbol *ams,
  1638. u64 ip)
  1639. {
  1640. struct addr_location al;
  1641. memset(&al, 0, sizeof(al));
  1642. /*
  1643. * We cannot use the header.misc hint to determine whether a
  1644. * branch stack address is user, kernel, guest, hypervisor.
  1645. * Branches may straddle the kernel/user/hypervisor boundaries.
  1646. * Thus, we have to try consecutively until we find a match
  1647. * or else, the symbol is unknown
  1648. */
  1649. thread__find_cpumode_addr_location(thread, ip, &al);
  1650. ams->addr = ip;
  1651. ams->al_addr = al.addr;
  1652. ams->ms.maps = al.maps;
  1653. ams->ms.sym = al.sym;
  1654. ams->ms.map = al.map;
  1655. ams->phys_addr = 0;
  1656. }
  1657. static void ip__resolve_data(struct thread *thread,
  1658. u8 m, struct addr_map_symbol *ams,
  1659. u64 addr, u64 phys_addr)
  1660. {
  1661. struct addr_location al;
  1662. memset(&al, 0, sizeof(al));
  1663. thread__find_symbol(thread, m, addr, &al);
  1664. ams->addr = addr;
  1665. ams->al_addr = al.addr;
  1666. ams->ms.maps = al.maps;
  1667. ams->ms.sym = al.sym;
  1668. ams->ms.map = al.map;
  1669. ams->phys_addr = phys_addr;
  1670. }
  1671. struct mem_info *sample__resolve_mem(struct perf_sample *sample,
  1672. struct addr_location *al)
  1673. {
  1674. struct mem_info *mi = mem_info__new();
  1675. if (!mi)
  1676. return NULL;
  1677. ip__resolve_ams(al->thread, &mi->iaddr, sample->ip);
  1678. ip__resolve_data(al->thread, al->cpumode, &mi->daddr,
  1679. sample->addr, sample->phys_addr);
  1680. mi->data_src.val = sample->data_src;
  1681. return mi;
  1682. }
  1683. static char *callchain_srcline(struct map_symbol *ms, u64 ip)
  1684. {
  1685. struct map *map = ms->map;
  1686. char *srcline = NULL;
  1687. if (!map || callchain_param.key == CCKEY_FUNCTION)
  1688. return srcline;
  1689. srcline = srcline__tree_find(&map->dso->srclines, ip);
  1690. if (!srcline) {
  1691. bool show_sym = false;
  1692. bool show_addr = callchain_param.key == CCKEY_ADDRESS;
  1693. srcline = get_srcline(map->dso, map__rip_2objdump(map, ip),
  1694. ms->sym, show_sym, show_addr, ip);
  1695. srcline__tree_insert(&map->dso->srclines, ip, srcline);
  1696. }
  1697. return srcline;
  1698. }
  1699. struct iterations {
  1700. int nr_loop_iter;
  1701. u64 cycles;
  1702. };
  1703. static int add_callchain_ip(struct thread *thread,
  1704. struct callchain_cursor *cursor,
  1705. struct symbol **parent,
  1706. struct addr_location *root_al,
  1707. u8 *cpumode,
  1708. u64 ip,
  1709. bool branch,
  1710. struct branch_flags *flags,
  1711. struct iterations *iter,
  1712. u64 branch_from)
  1713. {
  1714. struct map_symbol ms;
  1715. struct addr_location al;
  1716. int nr_loop_iter = 0;
  1717. u64 iter_cycles = 0;
  1718. const char *srcline = NULL;
  1719. al.filtered = 0;
  1720. al.sym = NULL;
  1721. al.srcline = NULL;
  1722. if (!cpumode) {
  1723. thread__find_cpumode_addr_location(thread, ip, &al);
  1724. } else {
  1725. if (ip >= PERF_CONTEXT_MAX) {
  1726. switch (ip) {
  1727. case PERF_CONTEXT_HV:
  1728. *cpumode = PERF_RECORD_MISC_HYPERVISOR;
  1729. break;
  1730. case PERF_CONTEXT_KERNEL:
  1731. *cpumode = PERF_RECORD_MISC_KERNEL;
  1732. break;
  1733. case PERF_CONTEXT_USER:
  1734. *cpumode = PERF_RECORD_MISC_USER;
  1735. break;
  1736. default:
  1737. pr_debug("invalid callchain context: "
  1738. "%"PRId64"\n", (s64) ip);
  1739. /*
  1740. * It seems the callchain is corrupted.
  1741. * Discard all.
  1742. */
  1743. callchain_cursor_reset(cursor);
  1744. return 1;
  1745. }
  1746. return 0;
  1747. }
  1748. thread__find_symbol(thread, *cpumode, ip, &al);
  1749. }
  1750. if (al.sym != NULL) {
  1751. if (perf_hpp_list.parent && !*parent &&
  1752. symbol__match_regex(al.sym, &parent_regex))
  1753. *parent = al.sym;
  1754. else if (have_ignore_callees && root_al &&
  1755. symbol__match_regex(al.sym, &ignore_callees_regex)) {
  1756. /* Treat this symbol as the root,
  1757. forgetting its callees. */
  1758. *root_al = al;
  1759. callchain_cursor_reset(cursor);
  1760. }
  1761. }
  1762. if (symbol_conf.hide_unresolved && al.sym == NULL)
  1763. return 0;
  1764. if (iter) {
  1765. nr_loop_iter = iter->nr_loop_iter;
  1766. iter_cycles = iter->cycles;
  1767. }
  1768. ms.maps = al.maps;
  1769. ms.map = al.map;
  1770. ms.sym = al.sym;
  1771. srcline = callchain_srcline(&ms, al.addr);
  1772. return callchain_cursor_append(cursor, ip, &ms,
  1773. branch, flags, nr_loop_iter,
  1774. iter_cycles, branch_from, srcline);
  1775. }
  1776. struct branch_info *sample__resolve_bstack(struct perf_sample *sample,
  1777. struct addr_location *al)
  1778. {
  1779. unsigned int i;
  1780. const struct branch_stack *bs = sample->branch_stack;
  1781. struct branch_entry *entries = perf_sample__branch_entries(sample);
  1782. struct branch_info *bi = calloc(bs->nr, sizeof(struct branch_info));
  1783. if (!bi)
  1784. return NULL;
  1785. for (i = 0; i < bs->nr; i++) {
  1786. ip__resolve_ams(al->thread, &bi[i].to, entries[i].to);
  1787. ip__resolve_ams(al->thread, &bi[i].from, entries[i].from);
  1788. bi[i].flags = entries[i].flags;
  1789. }
  1790. return bi;
  1791. }
  1792. static void save_iterations(struct iterations *iter,
  1793. struct branch_entry *be, int nr)
  1794. {
  1795. int i;
  1796. iter->nr_loop_iter++;
  1797. iter->cycles = 0;
  1798. for (i = 0; i < nr; i++)
  1799. iter->cycles += be[i].flags.cycles;
  1800. }
  1801. #define CHASHSZ 127
  1802. #define CHASHBITS 7
  1803. #define NO_ENTRY 0xff
  1804. #define PERF_MAX_BRANCH_DEPTH 127
  1805. /* Remove loops. */
  1806. static int remove_loops(struct branch_entry *l, int nr,
  1807. struct iterations *iter)
  1808. {
  1809. int i, j, off;
  1810. unsigned char chash[CHASHSZ];
  1811. memset(chash, NO_ENTRY, sizeof(chash));
  1812. BUG_ON(PERF_MAX_BRANCH_DEPTH > 255);
  1813. for (i = 0; i < nr; i++) {
  1814. int h = hash_64(l[i].from, CHASHBITS) % CHASHSZ;
  1815. /* no collision handling for now */
  1816. if (chash[h] == NO_ENTRY) {
  1817. chash[h] = i;
  1818. } else if (l[chash[h]].from == l[i].from) {
  1819. bool is_loop = true;
  1820. /* check if it is a real loop */
  1821. off = 0;
  1822. for (j = chash[h]; j < i && i + off < nr; j++, off++)
  1823. if (l[j].from != l[i + off].from) {
  1824. is_loop = false;
  1825. break;
  1826. }
  1827. if (is_loop) {
  1828. j = nr - (i + off);
  1829. if (j > 0) {
  1830. save_iterations(iter + i + off,
  1831. l + i, off);
  1832. memmove(iter + i, iter + i + off,
  1833. j * sizeof(*iter));
  1834. memmove(l + i, l + i + off,
  1835. j * sizeof(*l));
  1836. }
  1837. nr -= off;
  1838. }
  1839. }
  1840. }
  1841. return nr;
  1842. }
  1843. static int lbr_callchain_add_kernel_ip(struct thread *thread,
  1844. struct callchain_cursor *cursor,
  1845. struct perf_sample *sample,
  1846. struct symbol **parent,
  1847. struct addr_location *root_al,
  1848. u64 branch_from,
  1849. bool callee, int end)
  1850. {
  1851. struct ip_callchain *chain = sample->callchain;
  1852. u8 cpumode = PERF_RECORD_MISC_USER;
  1853. int err, i;
  1854. if (callee) {
  1855. for (i = 0; i < end + 1; i++) {
  1856. err = add_callchain_ip(thread, cursor, parent,
  1857. root_al, &cpumode, chain->ips[i],
  1858. false, NULL, NULL, branch_from);
  1859. if (err)
  1860. return err;
  1861. }
  1862. return 0;
  1863. }
  1864. for (i = end; i >= 0; i--) {
  1865. err = add_callchain_ip(thread, cursor, parent,
  1866. root_al, &cpumode, chain->ips[i],
  1867. false, NULL, NULL, branch_from);
  1868. if (err)
  1869. return err;
  1870. }
  1871. return 0;
  1872. }
  1873. static void save_lbr_cursor_node(struct thread *thread,
  1874. struct callchain_cursor *cursor,
  1875. int idx)
  1876. {
  1877. struct lbr_stitch *lbr_stitch = thread->lbr_stitch;
  1878. if (!lbr_stitch)
  1879. return;
  1880. if (cursor->pos == cursor->nr) {
  1881. lbr_stitch->prev_lbr_cursor[idx].valid = false;
  1882. return;
  1883. }
  1884. if (!cursor->curr)
  1885. cursor->curr = cursor->first;
  1886. else
  1887. cursor->curr = cursor->curr->next;
  1888. memcpy(&lbr_stitch->prev_lbr_cursor[idx], cursor->curr,
  1889. sizeof(struct callchain_cursor_node));
  1890. lbr_stitch->prev_lbr_cursor[idx].valid = true;
  1891. cursor->pos++;
  1892. }
  1893. static int lbr_callchain_add_lbr_ip(struct thread *thread,
  1894. struct callchain_cursor *cursor,
  1895. struct perf_sample *sample,
  1896. struct symbol **parent,
  1897. struct addr_location *root_al,
  1898. u64 *branch_from,
  1899. bool callee)
  1900. {
  1901. struct branch_stack *lbr_stack = sample->branch_stack;
  1902. struct branch_entry *entries = perf_sample__branch_entries(sample);
  1903. u8 cpumode = PERF_RECORD_MISC_USER;
  1904. int lbr_nr = lbr_stack->nr;
  1905. struct branch_flags *flags;
  1906. int err, i;
  1907. u64 ip;
  1908. /*
  1909. * The curr and pos are not used in writing session. They are cleared
  1910. * in callchain_cursor_commit() when the writing session is closed.
  1911. * Using curr and pos to track the current cursor node.
  1912. */
  1913. if (thread->lbr_stitch) {
  1914. cursor->curr = NULL;
  1915. cursor->pos = cursor->nr;
  1916. if (cursor->nr) {
  1917. cursor->curr = cursor->first;
  1918. for (i = 0; i < (int)(cursor->nr - 1); i++)
  1919. cursor->curr = cursor->curr->next;
  1920. }
  1921. }
  1922. if (callee) {
  1923. /* Add LBR ip from first entries.to */
  1924. ip = entries[0].to;
  1925. flags = &entries[0].flags;
  1926. *branch_from = entries[0].from;
  1927. err = add_callchain_ip(thread, cursor, parent,
  1928. root_al, &cpumode, ip,
  1929. true, flags, NULL,
  1930. *branch_from);
  1931. if (err)
  1932. return err;
  1933. /*
  1934. * The number of cursor node increases.
  1935. * Move the current cursor node.
  1936. * But does not need to save current cursor node for entry 0.
  1937. * It's impossible to stitch the whole LBRs of previous sample.
  1938. */
  1939. if (thread->lbr_stitch && (cursor->pos != cursor->nr)) {
  1940. if (!cursor->curr)
  1941. cursor->curr = cursor->first;
  1942. else
  1943. cursor->curr = cursor->curr->next;
  1944. cursor->pos++;
  1945. }
  1946. /* Add LBR ip from entries.from one by one. */
  1947. for (i = 0; i < lbr_nr; i++) {
  1948. ip = entries[i].from;
  1949. flags = &entries[i].flags;
  1950. err = add_callchain_ip(thread, cursor, parent,
  1951. root_al, &cpumode, ip,
  1952. true, flags, NULL,
  1953. *branch_from);
  1954. if (err)
  1955. return err;
  1956. save_lbr_cursor_node(thread, cursor, i);
  1957. }
  1958. return 0;
  1959. }
  1960. /* Add LBR ip from entries.from one by one. */
  1961. for (i = lbr_nr - 1; i >= 0; i--) {
  1962. ip = entries[i].from;
  1963. flags = &entries[i].flags;
  1964. err = add_callchain_ip(thread, cursor, parent,
  1965. root_al, &cpumode, ip,
  1966. true, flags, NULL,
  1967. *branch_from);
  1968. if (err)
  1969. return err;
  1970. save_lbr_cursor_node(thread, cursor, i);
  1971. }
  1972. /* Add LBR ip from first entries.to */
  1973. ip = entries[0].to;
  1974. flags = &entries[0].flags;
  1975. *branch_from = entries[0].from;
  1976. err = add_callchain_ip(thread, cursor, parent,
  1977. root_al, &cpumode, ip,
  1978. true, flags, NULL,
  1979. *branch_from);
  1980. if (err)
  1981. return err;
  1982. return 0;
  1983. }
  1984. static int lbr_callchain_add_stitched_lbr_ip(struct thread *thread,
  1985. struct callchain_cursor *cursor)
  1986. {
  1987. struct lbr_stitch *lbr_stitch = thread->lbr_stitch;
  1988. struct callchain_cursor_node *cnode;
  1989. struct stitch_list *stitch_node;
  1990. int err;
  1991. list_for_each_entry(stitch_node, &lbr_stitch->lists, node) {
  1992. cnode = &stitch_node->cursor;
  1993. err = callchain_cursor_append(cursor, cnode->ip,
  1994. &cnode->ms,
  1995. cnode->branch,
  1996. &cnode->branch_flags,
  1997. cnode->nr_loop_iter,
  1998. cnode->iter_cycles,
  1999. cnode->branch_from,
  2000. cnode->srcline);
  2001. if (err)
  2002. return err;
  2003. }
  2004. return 0;
  2005. }
  2006. static struct stitch_list *get_stitch_node(struct thread *thread)
  2007. {
  2008. struct lbr_stitch *lbr_stitch = thread->lbr_stitch;
  2009. struct stitch_list *stitch_node;
  2010. if (!list_empty(&lbr_stitch->free_lists)) {
  2011. stitch_node = list_first_entry(&lbr_stitch->free_lists,
  2012. struct stitch_list, node);
  2013. list_del(&stitch_node->node);
  2014. return stitch_node;
  2015. }
  2016. return malloc(sizeof(struct stitch_list));
  2017. }
  2018. static bool has_stitched_lbr(struct thread *thread,
  2019. struct perf_sample *cur,
  2020. struct perf_sample *prev,
  2021. unsigned int max_lbr,
  2022. bool callee)
  2023. {
  2024. struct branch_stack *cur_stack = cur->branch_stack;
  2025. struct branch_entry *cur_entries = perf_sample__branch_entries(cur);
  2026. struct branch_stack *prev_stack = prev->branch_stack;
  2027. struct branch_entry *prev_entries = perf_sample__branch_entries(prev);
  2028. struct lbr_stitch *lbr_stitch = thread->lbr_stitch;
  2029. int i, j, nr_identical_branches = 0;
  2030. struct stitch_list *stitch_node;
  2031. u64 cur_base, distance;
  2032. if (!cur_stack || !prev_stack)
  2033. return false;
  2034. /* Find the physical index of the base-of-stack for current sample. */
  2035. cur_base = max_lbr - cur_stack->nr + cur_stack->hw_idx + 1;
  2036. distance = (prev_stack->hw_idx > cur_base) ? (prev_stack->hw_idx - cur_base) :
  2037. (max_lbr + prev_stack->hw_idx - cur_base);
  2038. /* Previous sample has shorter stack. Nothing can be stitched. */
  2039. if (distance + 1 > prev_stack->nr)
  2040. return false;
  2041. /*
  2042. * Check if there are identical LBRs between two samples.
  2043. * Identicall LBRs must have same from, to and flags values. Also,
  2044. * they have to be saved in the same LBR registers (same physical
  2045. * index).
  2046. *
  2047. * Starts from the base-of-stack of current sample.
  2048. */
  2049. for (i = distance, j = cur_stack->nr - 1; (i >= 0) && (j >= 0); i--, j--) {
  2050. if ((prev_entries[i].from != cur_entries[j].from) ||
  2051. (prev_entries[i].to != cur_entries[j].to) ||
  2052. (prev_entries[i].flags.value != cur_entries[j].flags.value))
  2053. break;
  2054. nr_identical_branches++;
  2055. }
  2056. if (!nr_identical_branches)
  2057. return false;
  2058. /*
  2059. * Save the LBRs between the base-of-stack of previous sample
  2060. * and the base-of-stack of current sample into lbr_stitch->lists.
  2061. * These LBRs will be stitched later.
  2062. */
  2063. for (i = prev_stack->nr - 1; i > (int)distance; i--) {
  2064. if (!lbr_stitch->prev_lbr_cursor[i].valid)
  2065. continue;
  2066. stitch_node = get_stitch_node(thread);
  2067. if (!stitch_node)
  2068. return false;
  2069. memcpy(&stitch_node->cursor, &lbr_stitch->prev_lbr_cursor[i],
  2070. sizeof(struct callchain_cursor_node));
  2071. if (callee)
  2072. list_add(&stitch_node->node, &lbr_stitch->lists);
  2073. else
  2074. list_add_tail(&stitch_node->node, &lbr_stitch->lists);
  2075. }
  2076. return true;
  2077. }
  2078. static bool alloc_lbr_stitch(struct thread *thread, unsigned int max_lbr)
  2079. {
  2080. if (thread->lbr_stitch)
  2081. return true;
  2082. thread->lbr_stitch = zalloc(sizeof(*thread->lbr_stitch));
  2083. if (!thread->lbr_stitch)
  2084. goto err;
  2085. thread->lbr_stitch->prev_lbr_cursor = calloc(max_lbr + 1, sizeof(struct callchain_cursor_node));
  2086. if (!thread->lbr_stitch->prev_lbr_cursor)
  2087. goto free_lbr_stitch;
  2088. INIT_LIST_HEAD(&thread->lbr_stitch->lists);
  2089. INIT_LIST_HEAD(&thread->lbr_stitch->free_lists);
  2090. return true;
  2091. free_lbr_stitch:
  2092. zfree(&thread->lbr_stitch);
  2093. err:
  2094. pr_warning("Failed to allocate space for stitched LBRs. Disable LBR stitch\n");
  2095. thread->lbr_stitch_enable = false;
  2096. return false;
  2097. }
  2098. /*
  2099. * Recolve LBR callstack chain sample
  2100. * Return:
  2101. * 1 on success get LBR callchain information
  2102. * 0 no available LBR callchain information, should try fp
  2103. * negative error code on other errors.
  2104. */
  2105. static int resolve_lbr_callchain_sample(struct thread *thread,
  2106. struct callchain_cursor *cursor,
  2107. struct perf_sample *sample,
  2108. struct symbol **parent,
  2109. struct addr_location *root_al,
  2110. int max_stack,
  2111. unsigned int max_lbr)
  2112. {
  2113. bool callee = (callchain_param.order == ORDER_CALLEE);
  2114. struct ip_callchain *chain = sample->callchain;
  2115. int chain_nr = min(max_stack, (int)chain->nr), i;
  2116. struct lbr_stitch *lbr_stitch;
  2117. bool stitched_lbr = false;
  2118. u64 branch_from = 0;
  2119. int err;
  2120. for (i = 0; i < chain_nr; i++) {
  2121. if (chain->ips[i] == PERF_CONTEXT_USER)
  2122. break;
  2123. }
  2124. /* LBR only affects the user callchain */
  2125. if (i == chain_nr)
  2126. return 0;
  2127. if (thread->lbr_stitch_enable && !sample->no_hw_idx &&
  2128. (max_lbr > 0) && alloc_lbr_stitch(thread, max_lbr)) {
  2129. lbr_stitch = thread->lbr_stitch;
  2130. stitched_lbr = has_stitched_lbr(thread, sample,
  2131. &lbr_stitch->prev_sample,
  2132. max_lbr, callee);
  2133. if (!stitched_lbr && !list_empty(&lbr_stitch->lists)) {
  2134. list_replace_init(&lbr_stitch->lists,
  2135. &lbr_stitch->free_lists);
  2136. }
  2137. memcpy(&lbr_stitch->prev_sample, sample, sizeof(*sample));
  2138. }
  2139. if (callee) {
  2140. /* Add kernel ip */
  2141. err = lbr_callchain_add_kernel_ip(thread, cursor, sample,
  2142. parent, root_al, branch_from,
  2143. true, i);
  2144. if (err)
  2145. goto error;
  2146. err = lbr_callchain_add_lbr_ip(thread, cursor, sample, parent,
  2147. root_al, &branch_from, true);
  2148. if (err)
  2149. goto error;
  2150. if (stitched_lbr) {
  2151. err = lbr_callchain_add_stitched_lbr_ip(thread, cursor);
  2152. if (err)
  2153. goto error;
  2154. }
  2155. } else {
  2156. if (stitched_lbr) {
  2157. err = lbr_callchain_add_stitched_lbr_ip(thread, cursor);
  2158. if (err)
  2159. goto error;
  2160. }
  2161. err = lbr_callchain_add_lbr_ip(thread, cursor, sample, parent,
  2162. root_al, &branch_from, false);
  2163. if (err)
  2164. goto error;
  2165. /* Add kernel ip */
  2166. err = lbr_callchain_add_kernel_ip(thread, cursor, sample,
  2167. parent, root_al, branch_from,
  2168. false, i);
  2169. if (err)
  2170. goto error;
  2171. }
  2172. return 1;
  2173. error:
  2174. return (err < 0) ? err : 0;
  2175. }
  2176. static int find_prev_cpumode(struct ip_callchain *chain, struct thread *thread,
  2177. struct callchain_cursor *cursor,
  2178. struct symbol **parent,
  2179. struct addr_location *root_al,
  2180. u8 *cpumode, int ent)
  2181. {
  2182. int err = 0;
  2183. while (--ent >= 0) {
  2184. u64 ip = chain->ips[ent];
  2185. if (ip >= PERF_CONTEXT_MAX) {
  2186. err = add_callchain_ip(thread, cursor, parent,
  2187. root_al, cpumode, ip,
  2188. false, NULL, NULL, 0);
  2189. break;
  2190. }
  2191. }
  2192. return err;
  2193. }
  2194. static int thread__resolve_callchain_sample(struct thread *thread,
  2195. struct callchain_cursor *cursor,
  2196. struct evsel *evsel,
  2197. struct perf_sample *sample,
  2198. struct symbol **parent,
  2199. struct addr_location *root_al,
  2200. int max_stack)
  2201. {
  2202. struct branch_stack *branch = sample->branch_stack;
  2203. struct branch_entry *entries = perf_sample__branch_entries(sample);
  2204. struct ip_callchain *chain = sample->callchain;
  2205. int chain_nr = 0;
  2206. u8 cpumode = PERF_RECORD_MISC_USER;
  2207. int i, j, err, nr_entries;
  2208. int skip_idx = -1;
  2209. int first_call = 0;
  2210. if (chain)
  2211. chain_nr = chain->nr;
  2212. if (evsel__has_branch_callstack(evsel)) {
  2213. struct perf_env *env = evsel__env(evsel);
  2214. err = resolve_lbr_callchain_sample(thread, cursor, sample, parent,
  2215. root_al, max_stack,
  2216. !env ? 0 : env->max_branches);
  2217. if (err)
  2218. return (err < 0) ? err : 0;
  2219. }
  2220. /*
  2221. * Based on DWARF debug information, some architectures skip
  2222. * a callchain entry saved by the kernel.
  2223. */
  2224. skip_idx = arch_skip_callchain_idx(thread, chain);
  2225. /*
  2226. * Add branches to call stack for easier browsing. This gives
  2227. * more context for a sample than just the callers.
  2228. *
  2229. * This uses individual histograms of paths compared to the
  2230. * aggregated histograms the normal LBR mode uses.
  2231. *
  2232. * Limitations for now:
  2233. * - No extra filters
  2234. * - No annotations (should annotate somehow)
  2235. */
  2236. if (branch && callchain_param.branch_callstack) {
  2237. int nr = min(max_stack, (int)branch->nr);
  2238. struct branch_entry be[nr];
  2239. struct iterations iter[nr];
  2240. if (branch->nr > PERF_MAX_BRANCH_DEPTH) {
  2241. pr_warning("corrupted branch chain. skipping...\n");
  2242. goto check_calls;
  2243. }
  2244. for (i = 0; i < nr; i++) {
  2245. if (callchain_param.order == ORDER_CALLEE) {
  2246. be[i] = entries[i];
  2247. if (chain == NULL)
  2248. continue;
  2249. /*
  2250. * Check for overlap into the callchain.
  2251. * The return address is one off compared to
  2252. * the branch entry. To adjust for this
  2253. * assume the calling instruction is not longer
  2254. * than 8 bytes.
  2255. */
  2256. if (i == skip_idx ||
  2257. chain->ips[first_call] >= PERF_CONTEXT_MAX)
  2258. first_call++;
  2259. else if (be[i].from < chain->ips[first_call] &&
  2260. be[i].from >= chain->ips[first_call] - 8)
  2261. first_call++;
  2262. } else
  2263. be[i] = entries[branch->nr - i - 1];
  2264. }
  2265. memset(iter, 0, sizeof(struct iterations) * nr);
  2266. nr = remove_loops(be, nr, iter);
  2267. for (i = 0; i < nr; i++) {
  2268. err = add_callchain_ip(thread, cursor, parent,
  2269. root_al,
  2270. NULL, be[i].to,
  2271. true, &be[i].flags,
  2272. NULL, be[i].from);
  2273. if (!err)
  2274. err = add_callchain_ip(thread, cursor, parent, root_al,
  2275. NULL, be[i].from,
  2276. true, &be[i].flags,
  2277. &iter[i], 0);
  2278. if (err == -EINVAL)
  2279. break;
  2280. if (err)
  2281. return err;
  2282. }
  2283. if (chain_nr == 0)
  2284. return 0;
  2285. chain_nr -= nr;
  2286. }
  2287. check_calls:
  2288. if (chain && callchain_param.order != ORDER_CALLEE) {
  2289. err = find_prev_cpumode(chain, thread, cursor, parent, root_al,
  2290. &cpumode, chain->nr - first_call);
  2291. if (err)
  2292. return (err < 0) ? err : 0;
  2293. }
  2294. for (i = first_call, nr_entries = 0;
  2295. i < chain_nr && nr_entries < max_stack; i++) {
  2296. u64 ip;
  2297. if (callchain_param.order == ORDER_CALLEE)
  2298. j = i;
  2299. else
  2300. j = chain->nr - i - 1;
  2301. #ifdef HAVE_SKIP_CALLCHAIN_IDX
  2302. if (j == skip_idx)
  2303. continue;
  2304. #endif
  2305. ip = chain->ips[j];
  2306. if (ip < PERF_CONTEXT_MAX)
  2307. ++nr_entries;
  2308. else if (callchain_param.order != ORDER_CALLEE) {
  2309. err = find_prev_cpumode(chain, thread, cursor, parent,
  2310. root_al, &cpumode, j);
  2311. if (err)
  2312. return (err < 0) ? err : 0;
  2313. continue;
  2314. }
  2315. err = add_callchain_ip(thread, cursor, parent,
  2316. root_al, &cpumode, ip,
  2317. false, NULL, NULL, 0);
  2318. if (err)
  2319. return (err < 0) ? err : 0;
  2320. }
  2321. return 0;
  2322. }
  2323. static int append_inlines(struct callchain_cursor *cursor, struct map_symbol *ms, u64 ip)
  2324. {
  2325. struct symbol *sym = ms->sym;
  2326. struct map *map = ms->map;
  2327. struct inline_node *inline_node;
  2328. struct inline_list *ilist;
  2329. u64 addr;
  2330. int ret = 1;
  2331. if (!symbol_conf.inline_name || !map || !sym)
  2332. return ret;
  2333. addr = map__map_ip(map, ip);
  2334. addr = map__rip_2objdump(map, addr);
  2335. inline_node = inlines__tree_find(&map->dso->inlined_nodes, addr);
  2336. if (!inline_node) {
  2337. inline_node = dso__parse_addr_inlines(map->dso, addr, sym);
  2338. if (!inline_node)
  2339. return ret;
  2340. inlines__tree_insert(&map->dso->inlined_nodes, inline_node);
  2341. }
  2342. list_for_each_entry(ilist, &inline_node->val, list) {
  2343. struct map_symbol ilist_ms = {
  2344. .maps = ms->maps,
  2345. .map = map,
  2346. .sym = ilist->symbol,
  2347. };
  2348. ret = callchain_cursor_append(cursor, ip, &ilist_ms, false,
  2349. NULL, 0, 0, 0, ilist->srcline);
  2350. if (ret != 0)
  2351. return ret;
  2352. }
  2353. return ret;
  2354. }
  2355. static int unwind_entry(struct unwind_entry *entry, void *arg)
  2356. {
  2357. struct callchain_cursor *cursor = arg;
  2358. const char *srcline = NULL;
  2359. u64 addr = entry->ip;
  2360. if (symbol_conf.hide_unresolved && entry->ms.sym == NULL)
  2361. return 0;
  2362. if (append_inlines(cursor, &entry->ms, entry->ip) == 0)
  2363. return 0;
  2364. /*
  2365. * Convert entry->ip from a virtual address to an offset in
  2366. * its corresponding binary.
  2367. */
  2368. if (entry->ms.map)
  2369. addr = map__map_ip(entry->ms.map, entry->ip);
  2370. srcline = callchain_srcline(&entry->ms, addr);
  2371. return callchain_cursor_append(cursor, entry->ip, &entry->ms,
  2372. false, NULL, 0, 0, 0, srcline);
  2373. }
  2374. static int thread__resolve_callchain_unwind(struct thread *thread,
  2375. struct callchain_cursor *cursor,
  2376. struct evsel *evsel,
  2377. struct perf_sample *sample,
  2378. int max_stack)
  2379. {
  2380. /* Can we do dwarf post unwind? */
  2381. if (!((evsel->core.attr.sample_type & PERF_SAMPLE_REGS_USER) &&
  2382. (evsel->core.attr.sample_type & PERF_SAMPLE_STACK_USER)))
  2383. return 0;
  2384. /* Bail out if nothing was captured. */
  2385. if ((!sample->user_regs.regs) ||
  2386. (!sample->user_stack.size))
  2387. return 0;
  2388. return unwind__get_entries(unwind_entry, cursor,
  2389. thread, sample, max_stack);
  2390. }
  2391. int thread__resolve_callchain(struct thread *thread,
  2392. struct callchain_cursor *cursor,
  2393. struct evsel *evsel,
  2394. struct perf_sample *sample,
  2395. struct symbol **parent,
  2396. struct addr_location *root_al,
  2397. int max_stack)
  2398. {
  2399. int ret = 0;
  2400. callchain_cursor_reset(cursor);
  2401. if (callchain_param.order == ORDER_CALLEE) {
  2402. ret = thread__resolve_callchain_sample(thread, cursor,
  2403. evsel, sample,
  2404. parent, root_al,
  2405. max_stack);
  2406. if (ret)
  2407. return ret;
  2408. ret = thread__resolve_callchain_unwind(thread, cursor,
  2409. evsel, sample,
  2410. max_stack);
  2411. } else {
  2412. ret = thread__resolve_callchain_unwind(thread, cursor,
  2413. evsel, sample,
  2414. max_stack);
  2415. if (ret)
  2416. return ret;
  2417. ret = thread__resolve_callchain_sample(thread, cursor,
  2418. evsel, sample,
  2419. parent, root_al,
  2420. max_stack);
  2421. }
  2422. return ret;
  2423. }
  2424. int machine__for_each_thread(struct machine *machine,
  2425. int (*fn)(struct thread *thread, void *p),
  2426. void *priv)
  2427. {
  2428. struct threads *threads;
  2429. struct rb_node *nd;
  2430. struct thread *thread;
  2431. int rc = 0;
  2432. int i;
  2433. for (i = 0; i < THREADS__TABLE_SIZE; i++) {
  2434. threads = &machine->threads[i];
  2435. for (nd = rb_first_cached(&threads->entries); nd;
  2436. nd = rb_next(nd)) {
  2437. thread = rb_entry(nd, struct thread, rb_node);
  2438. rc = fn(thread, priv);
  2439. if (rc != 0)
  2440. return rc;
  2441. }
  2442. list_for_each_entry(thread, &threads->dead, node) {
  2443. rc = fn(thread, priv);
  2444. if (rc != 0)
  2445. return rc;
  2446. }
  2447. }
  2448. return rc;
  2449. }
  2450. int machines__for_each_thread(struct machines *machines,
  2451. int (*fn)(struct thread *thread, void *p),
  2452. void *priv)
  2453. {
  2454. struct rb_node *nd;
  2455. int rc = 0;
  2456. rc = machine__for_each_thread(&machines->host, fn, priv);
  2457. if (rc != 0)
  2458. return rc;
  2459. for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) {
  2460. struct machine *machine = rb_entry(nd, struct machine, rb_node);
  2461. rc = machine__for_each_thread(machine, fn, priv);
  2462. if (rc != 0)
  2463. return rc;
  2464. }
  2465. return rc;
  2466. }
  2467. pid_t machine__get_current_tid(struct machine *machine, int cpu)
  2468. {
  2469. int nr_cpus = min(machine->env->nr_cpus_avail, MAX_NR_CPUS);
  2470. if (cpu < 0 || cpu >= nr_cpus || !machine->current_tid)
  2471. return -1;
  2472. return machine->current_tid[cpu];
  2473. }
  2474. int machine__set_current_tid(struct machine *machine, int cpu, pid_t pid,
  2475. pid_t tid)
  2476. {
  2477. struct thread *thread;
  2478. int nr_cpus = min(machine->env->nr_cpus_avail, MAX_NR_CPUS);
  2479. if (cpu < 0)
  2480. return -EINVAL;
  2481. if (!machine->current_tid) {
  2482. int i;
  2483. machine->current_tid = calloc(nr_cpus, sizeof(pid_t));
  2484. if (!machine->current_tid)
  2485. return -ENOMEM;
  2486. for (i = 0; i < nr_cpus; i++)
  2487. machine->current_tid[i] = -1;
  2488. }
  2489. if (cpu >= nr_cpus) {
  2490. pr_err("Requested CPU %d too large. ", cpu);
  2491. pr_err("Consider raising MAX_NR_CPUS\n");
  2492. return -EINVAL;
  2493. }
  2494. machine->current_tid[cpu] = tid;
  2495. thread = machine__findnew_thread(machine, pid, tid);
  2496. if (!thread)
  2497. return -ENOMEM;
  2498. thread->cpu = cpu;
  2499. thread__put(thread);
  2500. return 0;
  2501. }
  2502. /*
  2503. * Compares the raw arch string. N.B. see instead perf_env__arch() if a
  2504. * normalized arch is needed.
  2505. */
  2506. bool machine__is(struct machine *machine, const char *arch)
  2507. {
  2508. return machine && !strcmp(perf_env__raw_arch(machine->env), arch);
  2509. }
  2510. int machine__nr_cpus_avail(struct machine *machine)
  2511. {
  2512. return machine ? perf_env__nr_cpus_avail(machine->env) : 0;
  2513. }
  2514. int machine__get_kernel_start(struct machine *machine)
  2515. {
  2516. struct map *map = machine__kernel_map(machine);
  2517. int err = 0;
  2518. /*
  2519. * The only addresses above 2^63 are kernel addresses of a 64-bit
  2520. * kernel. Note that addresses are unsigned so that on a 32-bit system
  2521. * all addresses including kernel addresses are less than 2^32. In
  2522. * that case (32-bit system), if the kernel mapping is unknown, all
  2523. * addresses will be assumed to be in user space - see
  2524. * machine__kernel_ip().
  2525. */
  2526. machine->kernel_start = 1ULL << 63;
  2527. if (map) {
  2528. err = map__load(map);
  2529. /*
  2530. * On x86_64, PTI entry trampolines are less than the
  2531. * start of kernel text, but still above 2^63. So leave
  2532. * kernel_start = 1ULL << 63 for x86_64.
  2533. */
  2534. if (!err && !machine__is(machine, "x86_64"))
  2535. machine->kernel_start = map->start;
  2536. }
  2537. return err;
  2538. }
  2539. u8 machine__addr_cpumode(struct machine *machine, u8 cpumode, u64 addr)
  2540. {
  2541. u8 addr_cpumode = cpumode;
  2542. bool kernel_ip;
  2543. if (!machine->single_address_space)
  2544. goto out;
  2545. kernel_ip = machine__kernel_ip(machine, addr);
  2546. switch (cpumode) {
  2547. case PERF_RECORD_MISC_KERNEL:
  2548. case PERF_RECORD_MISC_USER:
  2549. addr_cpumode = kernel_ip ? PERF_RECORD_MISC_KERNEL :
  2550. PERF_RECORD_MISC_USER;
  2551. break;
  2552. case PERF_RECORD_MISC_GUEST_KERNEL:
  2553. case PERF_RECORD_MISC_GUEST_USER:
  2554. addr_cpumode = kernel_ip ? PERF_RECORD_MISC_GUEST_KERNEL :
  2555. PERF_RECORD_MISC_GUEST_USER;
  2556. break;
  2557. default:
  2558. break;
  2559. }
  2560. out:
  2561. return addr_cpumode;
  2562. }
  2563. struct dso *machine__findnew_dso_id(struct machine *machine, const char *filename, struct dso_id *id)
  2564. {
  2565. return dsos__findnew_id(&machine->dsos, filename, id);
  2566. }
  2567. struct dso *machine__findnew_dso(struct machine *machine, const char *filename)
  2568. {
  2569. return machine__findnew_dso_id(machine, filename, NULL);
  2570. }
  2571. char *machine__resolve_kernel_addr(void *vmachine, unsigned long long *addrp, char **modp)
  2572. {
  2573. struct machine *machine = vmachine;
  2574. struct map *map;
  2575. struct symbol *sym = machine__find_kernel_symbol(machine, *addrp, &map);
  2576. if (sym == NULL)
  2577. return NULL;
  2578. *modp = __map__is_kmodule(map) ? (char *)map->dso->short_name : NULL;
  2579. *addrp = map->unmap_ip(map, sym->start);
  2580. return sym->name;
  2581. }
  2582. int machine__for_each_dso(struct machine *machine, machine__dso_t fn, void *priv)
  2583. {
  2584. struct dso *pos;
  2585. int err = 0;
  2586. list_for_each_entry(pos, &machine->dsos.head, node) {
  2587. if (fn(pos, machine, priv))
  2588. err = -1;
  2589. }
  2590. return err;
  2591. }