auxtrace.c 59 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * auxtrace.c: AUX area trace support
  4. * Copyright (c) 2013-2015, Intel Corporation.
  5. */
  6. #include <inttypes.h>
  7. #include <sys/types.h>
  8. #include <sys/mman.h>
  9. #include <stdbool.h>
  10. #include <string.h>
  11. #include <limits.h>
  12. #include <errno.h>
  13. #include <linux/kernel.h>
  14. #include <linux/perf_event.h>
  15. #include <linux/types.h>
  16. #include <linux/bitops.h>
  17. #include <linux/log2.h>
  18. #include <linux/string.h>
  19. #include <linux/time64.h>
  20. #include <sys/param.h>
  21. #include <stdlib.h>
  22. #include <stdio.h>
  23. #include <linux/list.h>
  24. #include <linux/zalloc.h>
  25. #include "evlist.h"
  26. #include "dso.h"
  27. #include "map.h"
  28. #include "pmu.h"
  29. #include "evsel.h"
  30. #include "evsel_config.h"
  31. #include "symbol.h"
  32. #include "util/perf_api_probe.h"
  33. #include "util/synthetic-events.h"
  34. #include "thread_map.h"
  35. #include "asm/bug.h"
  36. #include "auxtrace.h"
  37. #include <linux/hash.h>
  38. #include "event.h"
  39. #include "record.h"
  40. #include "session.h"
  41. #include "debug.h"
  42. #include <subcmd/parse-options.h>
  43. #include "cs-etm.h"
  44. #include "intel-pt.h"
  45. #include "intel-bts.h"
  46. #include "arm-spe.h"
  47. #include "s390-cpumsf.h"
  48. #include "util/mmap.h"
  49. #include <linux/ctype.h>
  50. #include "symbol/kallsyms.h"
  51. #include <internal/lib.h>
  52. /*
  53. * Make a group from 'leader' to 'last', requiring that the events were not
  54. * already grouped to a different leader.
  55. */
  56. static int perf_evlist__regroup(struct evlist *evlist,
  57. struct evsel *leader,
  58. struct evsel *last)
  59. {
  60. struct evsel *evsel;
  61. bool grp;
  62. if (!evsel__is_group_leader(leader))
  63. return -EINVAL;
  64. grp = false;
  65. evlist__for_each_entry(evlist, evsel) {
  66. if (grp) {
  67. if (!(evsel->leader == leader ||
  68. (evsel->leader == evsel &&
  69. evsel->core.nr_members <= 1)))
  70. return -EINVAL;
  71. } else if (evsel == leader) {
  72. grp = true;
  73. }
  74. if (evsel == last)
  75. break;
  76. }
  77. grp = false;
  78. evlist__for_each_entry(evlist, evsel) {
  79. if (grp) {
  80. if (evsel->leader != leader) {
  81. evsel->leader = leader;
  82. if (leader->core.nr_members < 1)
  83. leader->core.nr_members = 1;
  84. leader->core.nr_members += 1;
  85. }
  86. } else if (evsel == leader) {
  87. grp = true;
  88. }
  89. if (evsel == last)
  90. break;
  91. }
  92. return 0;
  93. }
  94. static bool auxtrace__dont_decode(struct perf_session *session)
  95. {
  96. return !session->itrace_synth_opts ||
  97. session->itrace_synth_opts->dont_decode;
  98. }
  99. int auxtrace_mmap__mmap(struct auxtrace_mmap *mm,
  100. struct auxtrace_mmap_params *mp,
  101. void *userpg, int fd)
  102. {
  103. struct perf_event_mmap_page *pc = userpg;
  104. WARN_ONCE(mm->base, "Uninitialized auxtrace_mmap\n");
  105. mm->userpg = userpg;
  106. mm->mask = mp->mask;
  107. mm->len = mp->len;
  108. mm->prev = 0;
  109. mm->idx = mp->idx;
  110. mm->tid = mp->tid;
  111. mm->cpu = mp->cpu;
  112. if (!mp->len) {
  113. mm->base = NULL;
  114. return 0;
  115. }
  116. #if BITS_PER_LONG != 64 && !defined(HAVE_SYNC_COMPARE_AND_SWAP_SUPPORT)
  117. pr_err("Cannot use AUX area tracing mmaps\n");
  118. return -1;
  119. #endif
  120. pc->aux_offset = mp->offset;
  121. pc->aux_size = mp->len;
  122. mm->base = mmap(NULL, mp->len, mp->prot, MAP_SHARED, fd, mp->offset);
  123. if (mm->base == MAP_FAILED) {
  124. pr_debug2("failed to mmap AUX area\n");
  125. mm->base = NULL;
  126. return -1;
  127. }
  128. return 0;
  129. }
  130. void auxtrace_mmap__munmap(struct auxtrace_mmap *mm)
  131. {
  132. if (mm->base) {
  133. munmap(mm->base, mm->len);
  134. mm->base = NULL;
  135. }
  136. }
  137. void auxtrace_mmap_params__init(struct auxtrace_mmap_params *mp,
  138. off_t auxtrace_offset,
  139. unsigned int auxtrace_pages,
  140. bool auxtrace_overwrite)
  141. {
  142. if (auxtrace_pages) {
  143. mp->offset = auxtrace_offset;
  144. mp->len = auxtrace_pages * (size_t)page_size;
  145. mp->mask = is_power_of_2(mp->len) ? mp->len - 1 : 0;
  146. mp->prot = PROT_READ | (auxtrace_overwrite ? 0 : PROT_WRITE);
  147. pr_debug2("AUX area mmap length %zu\n", mp->len);
  148. } else {
  149. mp->len = 0;
  150. }
  151. }
  152. void auxtrace_mmap_params__set_idx(struct auxtrace_mmap_params *mp,
  153. struct evlist *evlist, int idx,
  154. bool per_cpu)
  155. {
  156. mp->idx = idx;
  157. if (per_cpu) {
  158. mp->cpu = evlist->core.cpus->map[idx];
  159. if (evlist->core.threads)
  160. mp->tid = perf_thread_map__pid(evlist->core.threads, 0);
  161. else
  162. mp->tid = -1;
  163. } else {
  164. mp->cpu = -1;
  165. mp->tid = perf_thread_map__pid(evlist->core.threads, idx);
  166. }
  167. }
  168. #define AUXTRACE_INIT_NR_QUEUES 32
  169. static struct auxtrace_queue *auxtrace_alloc_queue_array(unsigned int nr_queues)
  170. {
  171. struct auxtrace_queue *queue_array;
  172. unsigned int max_nr_queues, i;
  173. max_nr_queues = UINT_MAX / sizeof(struct auxtrace_queue);
  174. if (nr_queues > max_nr_queues)
  175. return NULL;
  176. queue_array = calloc(nr_queues, sizeof(struct auxtrace_queue));
  177. if (!queue_array)
  178. return NULL;
  179. for (i = 0; i < nr_queues; i++) {
  180. INIT_LIST_HEAD(&queue_array[i].head);
  181. queue_array[i].priv = NULL;
  182. }
  183. return queue_array;
  184. }
  185. int auxtrace_queues__init(struct auxtrace_queues *queues)
  186. {
  187. queues->nr_queues = AUXTRACE_INIT_NR_QUEUES;
  188. queues->queue_array = auxtrace_alloc_queue_array(queues->nr_queues);
  189. if (!queues->queue_array)
  190. return -ENOMEM;
  191. return 0;
  192. }
  193. static int auxtrace_queues__grow(struct auxtrace_queues *queues,
  194. unsigned int new_nr_queues)
  195. {
  196. unsigned int nr_queues = queues->nr_queues;
  197. struct auxtrace_queue *queue_array;
  198. unsigned int i;
  199. if (!nr_queues)
  200. nr_queues = AUXTRACE_INIT_NR_QUEUES;
  201. while (nr_queues && nr_queues < new_nr_queues)
  202. nr_queues <<= 1;
  203. if (nr_queues < queues->nr_queues || nr_queues < new_nr_queues)
  204. return -EINVAL;
  205. queue_array = auxtrace_alloc_queue_array(nr_queues);
  206. if (!queue_array)
  207. return -ENOMEM;
  208. for (i = 0; i < queues->nr_queues; i++) {
  209. list_splice_tail(&queues->queue_array[i].head,
  210. &queue_array[i].head);
  211. queue_array[i].tid = queues->queue_array[i].tid;
  212. queue_array[i].cpu = queues->queue_array[i].cpu;
  213. queue_array[i].set = queues->queue_array[i].set;
  214. queue_array[i].priv = queues->queue_array[i].priv;
  215. }
  216. queues->nr_queues = nr_queues;
  217. queues->queue_array = queue_array;
  218. return 0;
  219. }
  220. static void *auxtrace_copy_data(u64 size, struct perf_session *session)
  221. {
  222. int fd = perf_data__fd(session->data);
  223. void *p;
  224. ssize_t ret;
  225. if (size > SSIZE_MAX)
  226. return NULL;
  227. p = malloc(size);
  228. if (!p)
  229. return NULL;
  230. ret = readn(fd, p, size);
  231. if (ret != (ssize_t)size) {
  232. free(p);
  233. return NULL;
  234. }
  235. return p;
  236. }
  237. static int auxtrace_queues__queue_buffer(struct auxtrace_queues *queues,
  238. unsigned int idx,
  239. struct auxtrace_buffer *buffer)
  240. {
  241. struct auxtrace_queue *queue;
  242. int err;
  243. if (idx >= queues->nr_queues) {
  244. err = auxtrace_queues__grow(queues, idx + 1);
  245. if (err)
  246. return err;
  247. }
  248. queue = &queues->queue_array[idx];
  249. if (!queue->set) {
  250. queue->set = true;
  251. queue->tid = buffer->tid;
  252. queue->cpu = buffer->cpu;
  253. }
  254. buffer->buffer_nr = queues->next_buffer_nr++;
  255. list_add_tail(&buffer->list, &queue->head);
  256. queues->new_data = true;
  257. queues->populated = true;
  258. return 0;
  259. }
  260. /* Limit buffers to 32MiB on 32-bit */
  261. #define BUFFER_LIMIT_FOR_32_BIT (32 * 1024 * 1024)
  262. static int auxtrace_queues__split_buffer(struct auxtrace_queues *queues,
  263. unsigned int idx,
  264. struct auxtrace_buffer *buffer)
  265. {
  266. u64 sz = buffer->size;
  267. bool consecutive = false;
  268. struct auxtrace_buffer *b;
  269. int err;
  270. while (sz > BUFFER_LIMIT_FOR_32_BIT) {
  271. b = memdup(buffer, sizeof(struct auxtrace_buffer));
  272. if (!b)
  273. return -ENOMEM;
  274. b->size = BUFFER_LIMIT_FOR_32_BIT;
  275. b->consecutive = consecutive;
  276. err = auxtrace_queues__queue_buffer(queues, idx, b);
  277. if (err) {
  278. auxtrace_buffer__free(b);
  279. return err;
  280. }
  281. buffer->data_offset += BUFFER_LIMIT_FOR_32_BIT;
  282. sz -= BUFFER_LIMIT_FOR_32_BIT;
  283. consecutive = true;
  284. }
  285. buffer->size = sz;
  286. buffer->consecutive = consecutive;
  287. return 0;
  288. }
  289. static bool filter_cpu(struct perf_session *session, int cpu)
  290. {
  291. unsigned long *cpu_bitmap = session->itrace_synth_opts->cpu_bitmap;
  292. return cpu_bitmap && cpu != -1 && !test_bit(cpu, cpu_bitmap);
  293. }
  294. static int auxtrace_queues__add_buffer(struct auxtrace_queues *queues,
  295. struct perf_session *session,
  296. unsigned int idx,
  297. struct auxtrace_buffer *buffer,
  298. struct auxtrace_buffer **buffer_ptr)
  299. {
  300. int err = -ENOMEM;
  301. if (filter_cpu(session, buffer->cpu))
  302. return 0;
  303. buffer = memdup(buffer, sizeof(*buffer));
  304. if (!buffer)
  305. return -ENOMEM;
  306. if (session->one_mmap) {
  307. buffer->data = buffer->data_offset - session->one_mmap_offset +
  308. session->one_mmap_addr;
  309. } else if (perf_data__is_pipe(session->data)) {
  310. buffer->data = auxtrace_copy_data(buffer->size, session);
  311. if (!buffer->data)
  312. goto out_free;
  313. buffer->data_needs_freeing = true;
  314. } else if (BITS_PER_LONG == 32 &&
  315. buffer->size > BUFFER_LIMIT_FOR_32_BIT) {
  316. err = auxtrace_queues__split_buffer(queues, idx, buffer);
  317. if (err)
  318. goto out_free;
  319. }
  320. err = auxtrace_queues__queue_buffer(queues, idx, buffer);
  321. if (err)
  322. goto out_free;
  323. /* FIXME: Doesn't work for split buffer */
  324. if (buffer_ptr)
  325. *buffer_ptr = buffer;
  326. return 0;
  327. out_free:
  328. auxtrace_buffer__free(buffer);
  329. return err;
  330. }
  331. int auxtrace_queues__add_event(struct auxtrace_queues *queues,
  332. struct perf_session *session,
  333. union perf_event *event, off_t data_offset,
  334. struct auxtrace_buffer **buffer_ptr)
  335. {
  336. struct auxtrace_buffer buffer = {
  337. .pid = -1,
  338. .tid = event->auxtrace.tid,
  339. .cpu = event->auxtrace.cpu,
  340. .data_offset = data_offset,
  341. .offset = event->auxtrace.offset,
  342. .reference = event->auxtrace.reference,
  343. .size = event->auxtrace.size,
  344. };
  345. unsigned int idx = event->auxtrace.idx;
  346. return auxtrace_queues__add_buffer(queues, session, idx, &buffer,
  347. buffer_ptr);
  348. }
  349. static int auxtrace_queues__add_indexed_event(struct auxtrace_queues *queues,
  350. struct perf_session *session,
  351. off_t file_offset, size_t sz)
  352. {
  353. union perf_event *event;
  354. int err;
  355. char buf[PERF_SAMPLE_MAX_SIZE];
  356. err = perf_session__peek_event(session, file_offset, buf,
  357. PERF_SAMPLE_MAX_SIZE, &event, NULL);
  358. if (err)
  359. return err;
  360. if (event->header.type == PERF_RECORD_AUXTRACE) {
  361. if (event->header.size < sizeof(struct perf_record_auxtrace) ||
  362. event->header.size != sz) {
  363. err = -EINVAL;
  364. goto out;
  365. }
  366. file_offset += event->header.size;
  367. err = auxtrace_queues__add_event(queues, session, event,
  368. file_offset, NULL);
  369. }
  370. out:
  371. return err;
  372. }
  373. void auxtrace_queues__free(struct auxtrace_queues *queues)
  374. {
  375. unsigned int i;
  376. for (i = 0; i < queues->nr_queues; i++) {
  377. while (!list_empty(&queues->queue_array[i].head)) {
  378. struct auxtrace_buffer *buffer;
  379. buffer = list_entry(queues->queue_array[i].head.next,
  380. struct auxtrace_buffer, list);
  381. list_del_init(&buffer->list);
  382. auxtrace_buffer__free(buffer);
  383. }
  384. }
  385. zfree(&queues->queue_array);
  386. queues->nr_queues = 0;
  387. }
  388. static void auxtrace_heapify(struct auxtrace_heap_item *heap_array,
  389. unsigned int pos, unsigned int queue_nr,
  390. u64 ordinal)
  391. {
  392. unsigned int parent;
  393. while (pos) {
  394. parent = (pos - 1) >> 1;
  395. if (heap_array[parent].ordinal <= ordinal)
  396. break;
  397. heap_array[pos] = heap_array[parent];
  398. pos = parent;
  399. }
  400. heap_array[pos].queue_nr = queue_nr;
  401. heap_array[pos].ordinal = ordinal;
  402. }
  403. int auxtrace_heap__add(struct auxtrace_heap *heap, unsigned int queue_nr,
  404. u64 ordinal)
  405. {
  406. struct auxtrace_heap_item *heap_array;
  407. if (queue_nr >= heap->heap_sz) {
  408. unsigned int heap_sz = AUXTRACE_INIT_NR_QUEUES;
  409. while (heap_sz <= queue_nr)
  410. heap_sz <<= 1;
  411. heap_array = realloc(heap->heap_array,
  412. heap_sz * sizeof(struct auxtrace_heap_item));
  413. if (!heap_array)
  414. return -ENOMEM;
  415. heap->heap_array = heap_array;
  416. heap->heap_sz = heap_sz;
  417. }
  418. auxtrace_heapify(heap->heap_array, heap->heap_cnt++, queue_nr, ordinal);
  419. return 0;
  420. }
  421. void auxtrace_heap__free(struct auxtrace_heap *heap)
  422. {
  423. zfree(&heap->heap_array);
  424. heap->heap_cnt = 0;
  425. heap->heap_sz = 0;
  426. }
  427. void auxtrace_heap__pop(struct auxtrace_heap *heap)
  428. {
  429. unsigned int pos, last, heap_cnt = heap->heap_cnt;
  430. struct auxtrace_heap_item *heap_array;
  431. if (!heap_cnt)
  432. return;
  433. heap->heap_cnt -= 1;
  434. heap_array = heap->heap_array;
  435. pos = 0;
  436. while (1) {
  437. unsigned int left, right;
  438. left = (pos << 1) + 1;
  439. if (left >= heap_cnt)
  440. break;
  441. right = left + 1;
  442. if (right >= heap_cnt) {
  443. heap_array[pos] = heap_array[left];
  444. return;
  445. }
  446. if (heap_array[left].ordinal < heap_array[right].ordinal) {
  447. heap_array[pos] = heap_array[left];
  448. pos = left;
  449. } else {
  450. heap_array[pos] = heap_array[right];
  451. pos = right;
  452. }
  453. }
  454. last = heap_cnt - 1;
  455. auxtrace_heapify(heap_array, pos, heap_array[last].queue_nr,
  456. heap_array[last].ordinal);
  457. }
  458. size_t auxtrace_record__info_priv_size(struct auxtrace_record *itr,
  459. struct evlist *evlist)
  460. {
  461. if (itr)
  462. return itr->info_priv_size(itr, evlist);
  463. return 0;
  464. }
  465. static int auxtrace_not_supported(void)
  466. {
  467. pr_err("AUX area tracing is not supported on this architecture\n");
  468. return -EINVAL;
  469. }
  470. int auxtrace_record__info_fill(struct auxtrace_record *itr,
  471. struct perf_session *session,
  472. struct perf_record_auxtrace_info *auxtrace_info,
  473. size_t priv_size)
  474. {
  475. if (itr)
  476. return itr->info_fill(itr, session, auxtrace_info, priv_size);
  477. return auxtrace_not_supported();
  478. }
  479. void auxtrace_record__free(struct auxtrace_record *itr)
  480. {
  481. if (itr)
  482. itr->free(itr);
  483. }
  484. int auxtrace_record__snapshot_start(struct auxtrace_record *itr)
  485. {
  486. if (itr && itr->snapshot_start)
  487. return itr->snapshot_start(itr);
  488. return 0;
  489. }
  490. int auxtrace_record__snapshot_finish(struct auxtrace_record *itr, bool on_exit)
  491. {
  492. if (!on_exit && itr && itr->snapshot_finish)
  493. return itr->snapshot_finish(itr);
  494. return 0;
  495. }
  496. int auxtrace_record__find_snapshot(struct auxtrace_record *itr, int idx,
  497. struct auxtrace_mmap *mm,
  498. unsigned char *data, u64 *head, u64 *old)
  499. {
  500. if (itr && itr->find_snapshot)
  501. return itr->find_snapshot(itr, idx, mm, data, head, old);
  502. return 0;
  503. }
  504. int auxtrace_record__options(struct auxtrace_record *itr,
  505. struct evlist *evlist,
  506. struct record_opts *opts)
  507. {
  508. if (itr) {
  509. itr->evlist = evlist;
  510. return itr->recording_options(itr, evlist, opts);
  511. }
  512. return 0;
  513. }
  514. u64 auxtrace_record__reference(struct auxtrace_record *itr)
  515. {
  516. if (itr)
  517. return itr->reference(itr);
  518. return 0;
  519. }
  520. int auxtrace_parse_snapshot_options(struct auxtrace_record *itr,
  521. struct record_opts *opts, const char *str)
  522. {
  523. if (!str)
  524. return 0;
  525. /* PMU-agnostic options */
  526. switch (*str) {
  527. case 'e':
  528. opts->auxtrace_snapshot_on_exit = true;
  529. str++;
  530. break;
  531. default:
  532. break;
  533. }
  534. if (itr && itr->parse_snapshot_options)
  535. return itr->parse_snapshot_options(itr, opts, str);
  536. pr_err("No AUX area tracing to snapshot\n");
  537. return -EINVAL;
  538. }
  539. int auxtrace_record__read_finish(struct auxtrace_record *itr, int idx)
  540. {
  541. struct evsel *evsel;
  542. if (!itr->evlist || !itr->pmu)
  543. return -EINVAL;
  544. evlist__for_each_entry(itr->evlist, evsel) {
  545. if (evsel->core.attr.type == itr->pmu->type) {
  546. if (evsel->disabled)
  547. return 0;
  548. return perf_evlist__enable_event_idx(itr->evlist, evsel,
  549. idx);
  550. }
  551. }
  552. return -EINVAL;
  553. }
  554. /*
  555. * Event record size is 16-bit which results in a maximum size of about 64KiB.
  556. * Allow about 4KiB for the rest of the sample record, to give a maximum
  557. * AUX area sample size of 60KiB.
  558. */
  559. #define MAX_AUX_SAMPLE_SIZE (60 * 1024)
  560. /* Arbitrary default size if no other default provided */
  561. #define DEFAULT_AUX_SAMPLE_SIZE (4 * 1024)
  562. static int auxtrace_validate_aux_sample_size(struct evlist *evlist,
  563. struct record_opts *opts)
  564. {
  565. struct evsel *evsel;
  566. bool has_aux_leader = false;
  567. u32 sz;
  568. evlist__for_each_entry(evlist, evsel) {
  569. sz = evsel->core.attr.aux_sample_size;
  570. if (evsel__is_group_leader(evsel)) {
  571. has_aux_leader = evsel__is_aux_event(evsel);
  572. if (sz) {
  573. if (has_aux_leader)
  574. pr_err("Cannot add AUX area sampling to an AUX area event\n");
  575. else
  576. pr_err("Cannot add AUX area sampling to a group leader\n");
  577. return -EINVAL;
  578. }
  579. }
  580. if (sz > MAX_AUX_SAMPLE_SIZE) {
  581. pr_err("AUX area sample size %u too big, max. %d\n",
  582. sz, MAX_AUX_SAMPLE_SIZE);
  583. return -EINVAL;
  584. }
  585. if (sz) {
  586. if (!has_aux_leader) {
  587. pr_err("Cannot add AUX area sampling because group leader is not an AUX area event\n");
  588. return -EINVAL;
  589. }
  590. evsel__set_sample_bit(evsel, AUX);
  591. opts->auxtrace_sample_mode = true;
  592. } else {
  593. evsel__reset_sample_bit(evsel, AUX);
  594. }
  595. }
  596. if (!opts->auxtrace_sample_mode) {
  597. pr_err("AUX area sampling requires an AUX area event group leader plus other events to which to add samples\n");
  598. return -EINVAL;
  599. }
  600. if (!perf_can_aux_sample()) {
  601. pr_err("AUX area sampling is not supported by kernel\n");
  602. return -EINVAL;
  603. }
  604. return 0;
  605. }
  606. int auxtrace_parse_sample_options(struct auxtrace_record *itr,
  607. struct evlist *evlist,
  608. struct record_opts *opts, const char *str)
  609. {
  610. struct evsel_config_term *term;
  611. struct evsel *aux_evsel;
  612. bool has_aux_sample_size = false;
  613. bool has_aux_leader = false;
  614. struct evsel *evsel;
  615. char *endptr;
  616. unsigned long sz;
  617. if (!str)
  618. goto no_opt;
  619. if (!itr) {
  620. pr_err("No AUX area event to sample\n");
  621. return -EINVAL;
  622. }
  623. sz = strtoul(str, &endptr, 0);
  624. if (*endptr || sz > UINT_MAX) {
  625. pr_err("Bad AUX area sampling option: '%s'\n", str);
  626. return -EINVAL;
  627. }
  628. if (!sz)
  629. sz = itr->default_aux_sample_size;
  630. if (!sz)
  631. sz = DEFAULT_AUX_SAMPLE_SIZE;
  632. /* Set aux_sample_size based on --aux-sample option */
  633. evlist__for_each_entry(evlist, evsel) {
  634. if (evsel__is_group_leader(evsel)) {
  635. has_aux_leader = evsel__is_aux_event(evsel);
  636. } else if (has_aux_leader) {
  637. evsel->core.attr.aux_sample_size = sz;
  638. }
  639. }
  640. no_opt:
  641. aux_evsel = NULL;
  642. /* Override with aux_sample_size from config term */
  643. evlist__for_each_entry(evlist, evsel) {
  644. if (evsel__is_aux_event(evsel))
  645. aux_evsel = evsel;
  646. term = evsel__get_config_term(evsel, AUX_SAMPLE_SIZE);
  647. if (term) {
  648. has_aux_sample_size = true;
  649. evsel->core.attr.aux_sample_size = term->val.aux_sample_size;
  650. /* If possible, group with the AUX event */
  651. if (aux_evsel && evsel->core.attr.aux_sample_size)
  652. perf_evlist__regroup(evlist, aux_evsel, evsel);
  653. }
  654. }
  655. if (!str && !has_aux_sample_size)
  656. return 0;
  657. if (!itr) {
  658. pr_err("No AUX area event to sample\n");
  659. return -EINVAL;
  660. }
  661. return auxtrace_validate_aux_sample_size(evlist, opts);
  662. }
  663. struct auxtrace_record *__weak
  664. auxtrace_record__init(struct evlist *evlist __maybe_unused, int *err)
  665. {
  666. *err = 0;
  667. return NULL;
  668. }
  669. static int auxtrace_index__alloc(struct list_head *head)
  670. {
  671. struct auxtrace_index *auxtrace_index;
  672. auxtrace_index = malloc(sizeof(struct auxtrace_index));
  673. if (!auxtrace_index)
  674. return -ENOMEM;
  675. auxtrace_index->nr = 0;
  676. INIT_LIST_HEAD(&auxtrace_index->list);
  677. list_add_tail(&auxtrace_index->list, head);
  678. return 0;
  679. }
  680. void auxtrace_index__free(struct list_head *head)
  681. {
  682. struct auxtrace_index *auxtrace_index, *n;
  683. list_for_each_entry_safe(auxtrace_index, n, head, list) {
  684. list_del_init(&auxtrace_index->list);
  685. free(auxtrace_index);
  686. }
  687. }
  688. static struct auxtrace_index *auxtrace_index__last(struct list_head *head)
  689. {
  690. struct auxtrace_index *auxtrace_index;
  691. int err;
  692. if (list_empty(head)) {
  693. err = auxtrace_index__alloc(head);
  694. if (err)
  695. return NULL;
  696. }
  697. auxtrace_index = list_entry(head->prev, struct auxtrace_index, list);
  698. if (auxtrace_index->nr >= PERF_AUXTRACE_INDEX_ENTRY_COUNT) {
  699. err = auxtrace_index__alloc(head);
  700. if (err)
  701. return NULL;
  702. auxtrace_index = list_entry(head->prev, struct auxtrace_index,
  703. list);
  704. }
  705. return auxtrace_index;
  706. }
  707. int auxtrace_index__auxtrace_event(struct list_head *head,
  708. union perf_event *event, off_t file_offset)
  709. {
  710. struct auxtrace_index *auxtrace_index;
  711. size_t nr;
  712. auxtrace_index = auxtrace_index__last(head);
  713. if (!auxtrace_index)
  714. return -ENOMEM;
  715. nr = auxtrace_index->nr;
  716. auxtrace_index->entries[nr].file_offset = file_offset;
  717. auxtrace_index->entries[nr].sz = event->header.size;
  718. auxtrace_index->nr += 1;
  719. return 0;
  720. }
  721. static int auxtrace_index__do_write(int fd,
  722. struct auxtrace_index *auxtrace_index)
  723. {
  724. struct auxtrace_index_entry ent;
  725. size_t i;
  726. for (i = 0; i < auxtrace_index->nr; i++) {
  727. ent.file_offset = auxtrace_index->entries[i].file_offset;
  728. ent.sz = auxtrace_index->entries[i].sz;
  729. if (writen(fd, &ent, sizeof(ent)) != sizeof(ent))
  730. return -errno;
  731. }
  732. return 0;
  733. }
  734. int auxtrace_index__write(int fd, struct list_head *head)
  735. {
  736. struct auxtrace_index *auxtrace_index;
  737. u64 total = 0;
  738. int err;
  739. list_for_each_entry(auxtrace_index, head, list)
  740. total += auxtrace_index->nr;
  741. if (writen(fd, &total, sizeof(total)) != sizeof(total))
  742. return -errno;
  743. list_for_each_entry(auxtrace_index, head, list) {
  744. err = auxtrace_index__do_write(fd, auxtrace_index);
  745. if (err)
  746. return err;
  747. }
  748. return 0;
  749. }
  750. static int auxtrace_index__process_entry(int fd, struct list_head *head,
  751. bool needs_swap)
  752. {
  753. struct auxtrace_index *auxtrace_index;
  754. struct auxtrace_index_entry ent;
  755. size_t nr;
  756. if (readn(fd, &ent, sizeof(ent)) != sizeof(ent))
  757. return -1;
  758. auxtrace_index = auxtrace_index__last(head);
  759. if (!auxtrace_index)
  760. return -1;
  761. nr = auxtrace_index->nr;
  762. if (needs_swap) {
  763. auxtrace_index->entries[nr].file_offset =
  764. bswap_64(ent.file_offset);
  765. auxtrace_index->entries[nr].sz = bswap_64(ent.sz);
  766. } else {
  767. auxtrace_index->entries[nr].file_offset = ent.file_offset;
  768. auxtrace_index->entries[nr].sz = ent.sz;
  769. }
  770. auxtrace_index->nr = nr + 1;
  771. return 0;
  772. }
  773. int auxtrace_index__process(int fd, u64 size, struct perf_session *session,
  774. bool needs_swap)
  775. {
  776. struct list_head *head = &session->auxtrace_index;
  777. u64 nr;
  778. if (readn(fd, &nr, sizeof(u64)) != sizeof(u64))
  779. return -1;
  780. if (needs_swap)
  781. nr = bswap_64(nr);
  782. if (sizeof(u64) + nr * sizeof(struct auxtrace_index_entry) > size)
  783. return -1;
  784. while (nr--) {
  785. int err;
  786. err = auxtrace_index__process_entry(fd, head, needs_swap);
  787. if (err)
  788. return -1;
  789. }
  790. return 0;
  791. }
  792. static int auxtrace_queues__process_index_entry(struct auxtrace_queues *queues,
  793. struct perf_session *session,
  794. struct auxtrace_index_entry *ent)
  795. {
  796. return auxtrace_queues__add_indexed_event(queues, session,
  797. ent->file_offset, ent->sz);
  798. }
  799. int auxtrace_queues__process_index(struct auxtrace_queues *queues,
  800. struct perf_session *session)
  801. {
  802. struct auxtrace_index *auxtrace_index;
  803. struct auxtrace_index_entry *ent;
  804. size_t i;
  805. int err;
  806. if (auxtrace__dont_decode(session))
  807. return 0;
  808. list_for_each_entry(auxtrace_index, &session->auxtrace_index, list) {
  809. for (i = 0; i < auxtrace_index->nr; i++) {
  810. ent = &auxtrace_index->entries[i];
  811. err = auxtrace_queues__process_index_entry(queues,
  812. session,
  813. ent);
  814. if (err)
  815. return err;
  816. }
  817. }
  818. return 0;
  819. }
  820. struct auxtrace_buffer *auxtrace_buffer__next(struct auxtrace_queue *queue,
  821. struct auxtrace_buffer *buffer)
  822. {
  823. if (buffer) {
  824. if (list_is_last(&buffer->list, &queue->head))
  825. return NULL;
  826. return list_entry(buffer->list.next, struct auxtrace_buffer,
  827. list);
  828. } else {
  829. if (list_empty(&queue->head))
  830. return NULL;
  831. return list_entry(queue->head.next, struct auxtrace_buffer,
  832. list);
  833. }
  834. }
  835. struct auxtrace_queue *auxtrace_queues__sample_queue(struct auxtrace_queues *queues,
  836. struct perf_sample *sample,
  837. struct perf_session *session)
  838. {
  839. struct perf_sample_id *sid;
  840. unsigned int idx;
  841. u64 id;
  842. id = sample->id;
  843. if (!id)
  844. return NULL;
  845. sid = perf_evlist__id2sid(session->evlist, id);
  846. if (!sid)
  847. return NULL;
  848. idx = sid->idx;
  849. if (idx >= queues->nr_queues)
  850. return NULL;
  851. return &queues->queue_array[idx];
  852. }
  853. int auxtrace_queues__add_sample(struct auxtrace_queues *queues,
  854. struct perf_session *session,
  855. struct perf_sample *sample, u64 data_offset,
  856. u64 reference)
  857. {
  858. struct auxtrace_buffer buffer = {
  859. .pid = -1,
  860. .data_offset = data_offset,
  861. .reference = reference,
  862. .size = sample->aux_sample.size,
  863. };
  864. struct perf_sample_id *sid;
  865. u64 id = sample->id;
  866. unsigned int idx;
  867. if (!id)
  868. return -EINVAL;
  869. sid = perf_evlist__id2sid(session->evlist, id);
  870. if (!sid)
  871. return -ENOENT;
  872. idx = sid->idx;
  873. buffer.tid = sid->tid;
  874. buffer.cpu = sid->cpu;
  875. return auxtrace_queues__add_buffer(queues, session, idx, &buffer, NULL);
  876. }
  877. struct queue_data {
  878. bool samples;
  879. bool events;
  880. };
  881. static int auxtrace_queue_data_cb(struct perf_session *session,
  882. union perf_event *event, u64 offset,
  883. void *data)
  884. {
  885. struct queue_data *qd = data;
  886. struct perf_sample sample;
  887. int err;
  888. if (qd->events && event->header.type == PERF_RECORD_AUXTRACE) {
  889. if (event->header.size < sizeof(struct perf_record_auxtrace))
  890. return -EINVAL;
  891. offset += event->header.size;
  892. return session->auxtrace->queue_data(session, NULL, event,
  893. offset);
  894. }
  895. if (!qd->samples || event->header.type != PERF_RECORD_SAMPLE)
  896. return 0;
  897. err = perf_evlist__parse_sample(session->evlist, event, &sample);
  898. if (err)
  899. return err;
  900. if (!sample.aux_sample.size)
  901. return 0;
  902. offset += sample.aux_sample.data - (void *)event;
  903. return session->auxtrace->queue_data(session, &sample, NULL, offset);
  904. }
  905. int auxtrace_queue_data(struct perf_session *session, bool samples, bool events)
  906. {
  907. struct queue_data qd = {
  908. .samples = samples,
  909. .events = events,
  910. };
  911. if (auxtrace__dont_decode(session))
  912. return 0;
  913. if (!session->auxtrace || !session->auxtrace->queue_data)
  914. return -EINVAL;
  915. return perf_session__peek_events(session, session->header.data_offset,
  916. session->header.data_size,
  917. auxtrace_queue_data_cb, &qd);
  918. }
  919. void *auxtrace_buffer__get_data(struct auxtrace_buffer *buffer, int fd)
  920. {
  921. size_t adj = buffer->data_offset & (page_size - 1);
  922. size_t size = buffer->size + adj;
  923. off_t file_offset = buffer->data_offset - adj;
  924. void *addr;
  925. if (buffer->data)
  926. return buffer->data;
  927. addr = mmap(NULL, size, PROT_READ, MAP_SHARED, fd, file_offset);
  928. if (addr == MAP_FAILED)
  929. return NULL;
  930. buffer->mmap_addr = addr;
  931. buffer->mmap_size = size;
  932. buffer->data = addr + adj;
  933. return buffer->data;
  934. }
  935. void auxtrace_buffer__put_data(struct auxtrace_buffer *buffer)
  936. {
  937. if (!buffer->data || !buffer->mmap_addr)
  938. return;
  939. munmap(buffer->mmap_addr, buffer->mmap_size);
  940. buffer->mmap_addr = NULL;
  941. buffer->mmap_size = 0;
  942. buffer->data = NULL;
  943. buffer->use_data = NULL;
  944. }
  945. void auxtrace_buffer__drop_data(struct auxtrace_buffer *buffer)
  946. {
  947. auxtrace_buffer__put_data(buffer);
  948. if (buffer->data_needs_freeing) {
  949. buffer->data_needs_freeing = false;
  950. zfree(&buffer->data);
  951. buffer->use_data = NULL;
  952. buffer->size = 0;
  953. }
  954. }
  955. void auxtrace_buffer__free(struct auxtrace_buffer *buffer)
  956. {
  957. auxtrace_buffer__drop_data(buffer);
  958. free(buffer);
  959. }
  960. void auxtrace_synth_error(struct perf_record_auxtrace_error *auxtrace_error, int type,
  961. int code, int cpu, pid_t pid, pid_t tid, u64 ip,
  962. const char *msg, u64 timestamp)
  963. {
  964. size_t size;
  965. memset(auxtrace_error, 0, sizeof(struct perf_record_auxtrace_error));
  966. auxtrace_error->header.type = PERF_RECORD_AUXTRACE_ERROR;
  967. auxtrace_error->type = type;
  968. auxtrace_error->code = code;
  969. auxtrace_error->cpu = cpu;
  970. auxtrace_error->pid = pid;
  971. auxtrace_error->tid = tid;
  972. auxtrace_error->fmt = 1;
  973. auxtrace_error->ip = ip;
  974. auxtrace_error->time = timestamp;
  975. strlcpy(auxtrace_error->msg, msg, MAX_AUXTRACE_ERROR_MSG);
  976. size = (void *)auxtrace_error->msg - (void *)auxtrace_error +
  977. strlen(auxtrace_error->msg) + 1;
  978. auxtrace_error->header.size = PERF_ALIGN(size, sizeof(u64));
  979. }
  980. int perf_event__synthesize_auxtrace_info(struct auxtrace_record *itr,
  981. struct perf_tool *tool,
  982. struct perf_session *session,
  983. perf_event__handler_t process)
  984. {
  985. union perf_event *ev;
  986. size_t priv_size;
  987. int err;
  988. pr_debug2("Synthesizing auxtrace information\n");
  989. priv_size = auxtrace_record__info_priv_size(itr, session->evlist);
  990. ev = zalloc(sizeof(struct perf_record_auxtrace_info) + priv_size);
  991. if (!ev)
  992. return -ENOMEM;
  993. ev->auxtrace_info.header.type = PERF_RECORD_AUXTRACE_INFO;
  994. ev->auxtrace_info.header.size = sizeof(struct perf_record_auxtrace_info) +
  995. priv_size;
  996. err = auxtrace_record__info_fill(itr, session, &ev->auxtrace_info,
  997. priv_size);
  998. if (err)
  999. goto out_free;
  1000. err = process(tool, ev, NULL, NULL);
  1001. out_free:
  1002. free(ev);
  1003. return err;
  1004. }
  1005. static void unleader_evsel(struct evlist *evlist, struct evsel *leader)
  1006. {
  1007. struct evsel *new_leader = NULL;
  1008. struct evsel *evsel;
  1009. /* Find new leader for the group */
  1010. evlist__for_each_entry(evlist, evsel) {
  1011. if (evsel->leader != leader || evsel == leader)
  1012. continue;
  1013. if (!new_leader)
  1014. new_leader = evsel;
  1015. evsel->leader = new_leader;
  1016. }
  1017. /* Update group information */
  1018. if (new_leader) {
  1019. zfree(&new_leader->group_name);
  1020. new_leader->group_name = leader->group_name;
  1021. leader->group_name = NULL;
  1022. new_leader->core.nr_members = leader->core.nr_members - 1;
  1023. leader->core.nr_members = 1;
  1024. }
  1025. }
  1026. static void unleader_auxtrace(struct perf_session *session)
  1027. {
  1028. struct evsel *evsel;
  1029. evlist__for_each_entry(session->evlist, evsel) {
  1030. if (auxtrace__evsel_is_auxtrace(session, evsel) &&
  1031. evsel__is_group_leader(evsel)) {
  1032. unleader_evsel(session->evlist, evsel);
  1033. }
  1034. }
  1035. }
  1036. int perf_event__process_auxtrace_info(struct perf_session *session,
  1037. union perf_event *event)
  1038. {
  1039. enum auxtrace_type type = event->auxtrace_info.type;
  1040. int err;
  1041. if (dump_trace)
  1042. fprintf(stdout, " type: %u\n", type);
  1043. switch (type) {
  1044. case PERF_AUXTRACE_INTEL_PT:
  1045. err = intel_pt_process_auxtrace_info(event, session);
  1046. break;
  1047. case PERF_AUXTRACE_INTEL_BTS:
  1048. err = intel_bts_process_auxtrace_info(event, session);
  1049. break;
  1050. case PERF_AUXTRACE_ARM_SPE:
  1051. err = arm_spe_process_auxtrace_info(event, session);
  1052. break;
  1053. case PERF_AUXTRACE_CS_ETM:
  1054. err = cs_etm__process_auxtrace_info(event, session);
  1055. break;
  1056. case PERF_AUXTRACE_S390_CPUMSF:
  1057. err = s390_cpumsf_process_auxtrace_info(event, session);
  1058. break;
  1059. case PERF_AUXTRACE_UNKNOWN:
  1060. default:
  1061. return -EINVAL;
  1062. }
  1063. if (err)
  1064. return err;
  1065. unleader_auxtrace(session);
  1066. return 0;
  1067. }
  1068. s64 perf_event__process_auxtrace(struct perf_session *session,
  1069. union perf_event *event)
  1070. {
  1071. s64 err;
  1072. if (dump_trace)
  1073. fprintf(stdout, " size: %#"PRI_lx64" offset: %#"PRI_lx64" ref: %#"PRI_lx64" idx: %u tid: %d cpu: %d\n",
  1074. event->auxtrace.size, event->auxtrace.offset,
  1075. event->auxtrace.reference, event->auxtrace.idx,
  1076. event->auxtrace.tid, event->auxtrace.cpu);
  1077. if (auxtrace__dont_decode(session))
  1078. return event->auxtrace.size;
  1079. if (!session->auxtrace || event->header.type != PERF_RECORD_AUXTRACE)
  1080. return -EINVAL;
  1081. err = session->auxtrace->process_auxtrace_event(session, event, session->tool);
  1082. if (err < 0)
  1083. return err;
  1084. return event->auxtrace.size;
  1085. }
  1086. #define PERF_ITRACE_DEFAULT_PERIOD_TYPE PERF_ITRACE_PERIOD_NANOSECS
  1087. #define PERF_ITRACE_DEFAULT_PERIOD 100000
  1088. #define PERF_ITRACE_DEFAULT_CALLCHAIN_SZ 16
  1089. #define PERF_ITRACE_MAX_CALLCHAIN_SZ 1024
  1090. #define PERF_ITRACE_DEFAULT_LAST_BRANCH_SZ 64
  1091. #define PERF_ITRACE_MAX_LAST_BRANCH_SZ 1024
  1092. void itrace_synth_opts__set_default(struct itrace_synth_opts *synth_opts,
  1093. bool no_sample)
  1094. {
  1095. synth_opts->branches = true;
  1096. synth_opts->transactions = true;
  1097. synth_opts->ptwrites = true;
  1098. synth_opts->pwr_events = true;
  1099. synth_opts->other_events = true;
  1100. synth_opts->errors = true;
  1101. synth_opts->flc = true;
  1102. synth_opts->llc = true;
  1103. synth_opts->tlb = true;
  1104. synth_opts->remote_access = true;
  1105. if (no_sample) {
  1106. synth_opts->period_type = PERF_ITRACE_PERIOD_INSTRUCTIONS;
  1107. synth_opts->period = 1;
  1108. synth_opts->calls = true;
  1109. } else {
  1110. synth_opts->instructions = true;
  1111. synth_opts->period_type = PERF_ITRACE_DEFAULT_PERIOD_TYPE;
  1112. synth_opts->period = PERF_ITRACE_DEFAULT_PERIOD;
  1113. }
  1114. synth_opts->callchain_sz = PERF_ITRACE_DEFAULT_CALLCHAIN_SZ;
  1115. synth_opts->last_branch_sz = PERF_ITRACE_DEFAULT_LAST_BRANCH_SZ;
  1116. synth_opts->initial_skip = 0;
  1117. }
  1118. static int get_flag(const char **ptr, unsigned int *flags)
  1119. {
  1120. while (1) {
  1121. char c = **ptr;
  1122. if (c >= 'a' && c <= 'z') {
  1123. *flags |= 1 << (c - 'a');
  1124. ++*ptr;
  1125. return 0;
  1126. } else if (c == ' ') {
  1127. ++*ptr;
  1128. continue;
  1129. } else {
  1130. return -1;
  1131. }
  1132. }
  1133. }
  1134. static int get_flags(const char **ptr, unsigned int *plus_flags, unsigned int *minus_flags)
  1135. {
  1136. while (1) {
  1137. switch (**ptr) {
  1138. case '+':
  1139. ++*ptr;
  1140. if (get_flag(ptr, plus_flags))
  1141. return -1;
  1142. break;
  1143. case '-':
  1144. ++*ptr;
  1145. if (get_flag(ptr, minus_flags))
  1146. return -1;
  1147. break;
  1148. case ' ':
  1149. ++*ptr;
  1150. break;
  1151. default:
  1152. return 0;
  1153. }
  1154. }
  1155. }
  1156. /*
  1157. * Please check tools/perf/Documentation/perf-script.txt for information
  1158. * about the options parsed here, which is introduced after this cset,
  1159. * when support in 'perf script' for these options is introduced.
  1160. */
  1161. int itrace_parse_synth_opts(const struct option *opt, const char *str,
  1162. int unset)
  1163. {
  1164. struct itrace_synth_opts *synth_opts = opt->value;
  1165. const char *p;
  1166. char *endptr;
  1167. bool period_type_set = false;
  1168. bool period_set = false;
  1169. synth_opts->set = true;
  1170. if (unset) {
  1171. synth_opts->dont_decode = true;
  1172. return 0;
  1173. }
  1174. if (!str) {
  1175. itrace_synth_opts__set_default(synth_opts,
  1176. synth_opts->default_no_sample);
  1177. return 0;
  1178. }
  1179. for (p = str; *p;) {
  1180. switch (*p++) {
  1181. case 'i':
  1182. synth_opts->instructions = true;
  1183. while (*p == ' ' || *p == ',')
  1184. p += 1;
  1185. if (isdigit(*p)) {
  1186. synth_opts->period = strtoull(p, &endptr, 10);
  1187. period_set = true;
  1188. p = endptr;
  1189. while (*p == ' ' || *p == ',')
  1190. p += 1;
  1191. switch (*p++) {
  1192. case 'i':
  1193. synth_opts->period_type =
  1194. PERF_ITRACE_PERIOD_INSTRUCTIONS;
  1195. period_type_set = true;
  1196. break;
  1197. case 't':
  1198. synth_opts->period_type =
  1199. PERF_ITRACE_PERIOD_TICKS;
  1200. period_type_set = true;
  1201. break;
  1202. case 'm':
  1203. synth_opts->period *= 1000;
  1204. /* Fall through */
  1205. case 'u':
  1206. synth_opts->period *= 1000;
  1207. /* Fall through */
  1208. case 'n':
  1209. if (*p++ != 's')
  1210. goto out_err;
  1211. synth_opts->period_type =
  1212. PERF_ITRACE_PERIOD_NANOSECS;
  1213. period_type_set = true;
  1214. break;
  1215. case '\0':
  1216. goto out;
  1217. default:
  1218. goto out_err;
  1219. }
  1220. }
  1221. break;
  1222. case 'b':
  1223. synth_opts->branches = true;
  1224. break;
  1225. case 'x':
  1226. synth_opts->transactions = true;
  1227. break;
  1228. case 'w':
  1229. synth_opts->ptwrites = true;
  1230. break;
  1231. case 'p':
  1232. synth_opts->pwr_events = true;
  1233. break;
  1234. case 'o':
  1235. synth_opts->other_events = true;
  1236. break;
  1237. case 'e':
  1238. synth_opts->errors = true;
  1239. if (get_flags(&p, &synth_opts->error_plus_flags,
  1240. &synth_opts->error_minus_flags))
  1241. goto out_err;
  1242. break;
  1243. case 'd':
  1244. synth_opts->log = true;
  1245. if (get_flags(&p, &synth_opts->log_plus_flags,
  1246. &synth_opts->log_minus_flags))
  1247. goto out_err;
  1248. break;
  1249. case 'c':
  1250. synth_opts->branches = true;
  1251. synth_opts->calls = true;
  1252. break;
  1253. case 'r':
  1254. synth_opts->branches = true;
  1255. synth_opts->returns = true;
  1256. break;
  1257. case 'G':
  1258. case 'g':
  1259. if (p[-1] == 'G')
  1260. synth_opts->add_callchain = true;
  1261. else
  1262. synth_opts->callchain = true;
  1263. synth_opts->callchain_sz =
  1264. PERF_ITRACE_DEFAULT_CALLCHAIN_SZ;
  1265. while (*p == ' ' || *p == ',')
  1266. p += 1;
  1267. if (isdigit(*p)) {
  1268. unsigned int val;
  1269. val = strtoul(p, &endptr, 10);
  1270. p = endptr;
  1271. if (!val || val > PERF_ITRACE_MAX_CALLCHAIN_SZ)
  1272. goto out_err;
  1273. synth_opts->callchain_sz = val;
  1274. }
  1275. break;
  1276. case 'L':
  1277. case 'l':
  1278. if (p[-1] == 'L')
  1279. synth_opts->add_last_branch = true;
  1280. else
  1281. synth_opts->last_branch = true;
  1282. synth_opts->last_branch_sz =
  1283. PERF_ITRACE_DEFAULT_LAST_BRANCH_SZ;
  1284. while (*p == ' ' || *p == ',')
  1285. p += 1;
  1286. if (isdigit(*p)) {
  1287. unsigned int val;
  1288. val = strtoul(p, &endptr, 10);
  1289. p = endptr;
  1290. if (!val ||
  1291. val > PERF_ITRACE_MAX_LAST_BRANCH_SZ)
  1292. goto out_err;
  1293. synth_opts->last_branch_sz = val;
  1294. }
  1295. break;
  1296. case 's':
  1297. synth_opts->initial_skip = strtoul(p, &endptr, 10);
  1298. if (p == endptr)
  1299. goto out_err;
  1300. p = endptr;
  1301. break;
  1302. case 'f':
  1303. synth_opts->flc = true;
  1304. break;
  1305. case 'm':
  1306. synth_opts->llc = true;
  1307. break;
  1308. case 't':
  1309. synth_opts->tlb = true;
  1310. break;
  1311. case 'a':
  1312. synth_opts->remote_access = true;
  1313. break;
  1314. case 'q':
  1315. synth_opts->quick += 1;
  1316. break;
  1317. case ' ':
  1318. case ',':
  1319. break;
  1320. default:
  1321. goto out_err;
  1322. }
  1323. }
  1324. out:
  1325. if (synth_opts->instructions) {
  1326. if (!period_type_set)
  1327. synth_opts->period_type =
  1328. PERF_ITRACE_DEFAULT_PERIOD_TYPE;
  1329. if (!period_set)
  1330. synth_opts->period = PERF_ITRACE_DEFAULT_PERIOD;
  1331. }
  1332. return 0;
  1333. out_err:
  1334. pr_err("Bad Instruction Tracing options '%s'\n", str);
  1335. return -EINVAL;
  1336. }
  1337. static const char * const auxtrace_error_type_name[] = {
  1338. [PERF_AUXTRACE_ERROR_ITRACE] = "instruction trace",
  1339. };
  1340. static const char *auxtrace_error_name(int type)
  1341. {
  1342. const char *error_type_name = NULL;
  1343. if (type < PERF_AUXTRACE_ERROR_MAX)
  1344. error_type_name = auxtrace_error_type_name[type];
  1345. if (!error_type_name)
  1346. error_type_name = "unknown AUX";
  1347. return error_type_name;
  1348. }
  1349. size_t perf_event__fprintf_auxtrace_error(union perf_event *event, FILE *fp)
  1350. {
  1351. struct perf_record_auxtrace_error *e = &event->auxtrace_error;
  1352. unsigned long long nsecs = e->time;
  1353. const char *msg = e->msg;
  1354. int ret;
  1355. ret = fprintf(fp, " %s error type %u",
  1356. auxtrace_error_name(e->type), e->type);
  1357. if (e->fmt && nsecs) {
  1358. unsigned long secs = nsecs / NSEC_PER_SEC;
  1359. nsecs -= secs * NSEC_PER_SEC;
  1360. ret += fprintf(fp, " time %lu.%09llu", secs, nsecs);
  1361. } else {
  1362. ret += fprintf(fp, " time 0");
  1363. }
  1364. if (!e->fmt)
  1365. msg = (const char *)&e->time;
  1366. ret += fprintf(fp, " cpu %d pid %d tid %d ip %#"PRI_lx64" code %u: %s\n",
  1367. e->cpu, e->pid, e->tid, e->ip, e->code, msg);
  1368. return ret;
  1369. }
  1370. void perf_session__auxtrace_error_inc(struct perf_session *session,
  1371. union perf_event *event)
  1372. {
  1373. struct perf_record_auxtrace_error *e = &event->auxtrace_error;
  1374. if (e->type < PERF_AUXTRACE_ERROR_MAX)
  1375. session->evlist->stats.nr_auxtrace_errors[e->type] += 1;
  1376. }
  1377. void events_stats__auxtrace_error_warn(const struct events_stats *stats)
  1378. {
  1379. int i;
  1380. for (i = 0; i < PERF_AUXTRACE_ERROR_MAX; i++) {
  1381. if (!stats->nr_auxtrace_errors[i])
  1382. continue;
  1383. ui__warning("%u %s errors\n",
  1384. stats->nr_auxtrace_errors[i],
  1385. auxtrace_error_name(i));
  1386. }
  1387. }
  1388. int perf_event__process_auxtrace_error(struct perf_session *session,
  1389. union perf_event *event)
  1390. {
  1391. if (auxtrace__dont_decode(session))
  1392. return 0;
  1393. perf_event__fprintf_auxtrace_error(event, stdout);
  1394. return 0;
  1395. }
  1396. static int __auxtrace_mmap__read(struct mmap *map,
  1397. struct auxtrace_record *itr,
  1398. struct perf_tool *tool, process_auxtrace_t fn,
  1399. bool snapshot, size_t snapshot_size)
  1400. {
  1401. struct auxtrace_mmap *mm = &map->auxtrace_mmap;
  1402. u64 head, old = mm->prev, offset, ref;
  1403. unsigned char *data = mm->base;
  1404. size_t size, head_off, old_off, len1, len2, padding;
  1405. union perf_event ev;
  1406. void *data1, *data2;
  1407. if (snapshot) {
  1408. head = auxtrace_mmap__read_snapshot_head(mm);
  1409. if (auxtrace_record__find_snapshot(itr, mm->idx, mm, data,
  1410. &head, &old))
  1411. return -1;
  1412. } else {
  1413. head = auxtrace_mmap__read_head(mm);
  1414. }
  1415. if (old == head)
  1416. return 0;
  1417. pr_debug3("auxtrace idx %d old %#"PRIx64" head %#"PRIx64" diff %#"PRIx64"\n",
  1418. mm->idx, old, head, head - old);
  1419. if (mm->mask) {
  1420. head_off = head & mm->mask;
  1421. old_off = old & mm->mask;
  1422. } else {
  1423. head_off = head % mm->len;
  1424. old_off = old % mm->len;
  1425. }
  1426. if (head_off > old_off)
  1427. size = head_off - old_off;
  1428. else
  1429. size = mm->len - (old_off - head_off);
  1430. if (snapshot && size > snapshot_size)
  1431. size = snapshot_size;
  1432. ref = auxtrace_record__reference(itr);
  1433. if (head > old || size <= head || mm->mask) {
  1434. offset = head - size;
  1435. } else {
  1436. /*
  1437. * When the buffer size is not a power of 2, 'head' wraps at the
  1438. * highest multiple of the buffer size, so we have to subtract
  1439. * the remainder here.
  1440. */
  1441. u64 rem = (0ULL - mm->len) % mm->len;
  1442. offset = head - size - rem;
  1443. }
  1444. if (size > head_off) {
  1445. len1 = size - head_off;
  1446. data1 = &data[mm->len - len1];
  1447. len2 = head_off;
  1448. data2 = &data[0];
  1449. } else {
  1450. len1 = size;
  1451. data1 = &data[head_off - len1];
  1452. len2 = 0;
  1453. data2 = NULL;
  1454. }
  1455. if (itr->alignment) {
  1456. unsigned int unwanted = len1 % itr->alignment;
  1457. len1 -= unwanted;
  1458. size -= unwanted;
  1459. }
  1460. /* padding must be written by fn() e.g. record__process_auxtrace() */
  1461. padding = size & (PERF_AUXTRACE_RECORD_ALIGNMENT - 1);
  1462. if (padding)
  1463. padding = PERF_AUXTRACE_RECORD_ALIGNMENT - padding;
  1464. memset(&ev, 0, sizeof(ev));
  1465. ev.auxtrace.header.type = PERF_RECORD_AUXTRACE;
  1466. ev.auxtrace.header.size = sizeof(ev.auxtrace);
  1467. ev.auxtrace.size = size + padding;
  1468. ev.auxtrace.offset = offset;
  1469. ev.auxtrace.reference = ref;
  1470. ev.auxtrace.idx = mm->idx;
  1471. ev.auxtrace.tid = mm->tid;
  1472. ev.auxtrace.cpu = mm->cpu;
  1473. if (fn(tool, map, &ev, data1, len1, data2, len2))
  1474. return -1;
  1475. mm->prev = head;
  1476. if (!snapshot) {
  1477. auxtrace_mmap__write_tail(mm, head);
  1478. if (itr->read_finish) {
  1479. int err;
  1480. err = itr->read_finish(itr, mm->idx);
  1481. if (err < 0)
  1482. return err;
  1483. }
  1484. }
  1485. return 1;
  1486. }
  1487. int auxtrace_mmap__read(struct mmap *map, struct auxtrace_record *itr,
  1488. struct perf_tool *tool, process_auxtrace_t fn)
  1489. {
  1490. return __auxtrace_mmap__read(map, itr, tool, fn, false, 0);
  1491. }
  1492. int auxtrace_mmap__read_snapshot(struct mmap *map,
  1493. struct auxtrace_record *itr,
  1494. struct perf_tool *tool, process_auxtrace_t fn,
  1495. size_t snapshot_size)
  1496. {
  1497. return __auxtrace_mmap__read(map, itr, tool, fn, true, snapshot_size);
  1498. }
  1499. /**
  1500. * struct auxtrace_cache - hash table to implement a cache
  1501. * @hashtable: the hashtable
  1502. * @sz: hashtable size (number of hlists)
  1503. * @entry_size: size of an entry
  1504. * @limit: limit the number of entries to this maximum, when reached the cache
  1505. * is dropped and caching begins again with an empty cache
  1506. * @cnt: current number of entries
  1507. * @bits: hashtable size (@sz = 2^@bits)
  1508. */
  1509. struct auxtrace_cache {
  1510. struct hlist_head *hashtable;
  1511. size_t sz;
  1512. size_t entry_size;
  1513. size_t limit;
  1514. size_t cnt;
  1515. unsigned int bits;
  1516. };
  1517. struct auxtrace_cache *auxtrace_cache__new(unsigned int bits, size_t entry_size,
  1518. unsigned int limit_percent)
  1519. {
  1520. struct auxtrace_cache *c;
  1521. struct hlist_head *ht;
  1522. size_t sz, i;
  1523. c = zalloc(sizeof(struct auxtrace_cache));
  1524. if (!c)
  1525. return NULL;
  1526. sz = 1UL << bits;
  1527. ht = calloc(sz, sizeof(struct hlist_head));
  1528. if (!ht)
  1529. goto out_free;
  1530. for (i = 0; i < sz; i++)
  1531. INIT_HLIST_HEAD(&ht[i]);
  1532. c->hashtable = ht;
  1533. c->sz = sz;
  1534. c->entry_size = entry_size;
  1535. c->limit = (c->sz * limit_percent) / 100;
  1536. c->bits = bits;
  1537. return c;
  1538. out_free:
  1539. free(c);
  1540. return NULL;
  1541. }
  1542. static void auxtrace_cache__drop(struct auxtrace_cache *c)
  1543. {
  1544. struct auxtrace_cache_entry *entry;
  1545. struct hlist_node *tmp;
  1546. size_t i;
  1547. if (!c)
  1548. return;
  1549. for (i = 0; i < c->sz; i++) {
  1550. hlist_for_each_entry_safe(entry, tmp, &c->hashtable[i], hash) {
  1551. hlist_del(&entry->hash);
  1552. auxtrace_cache__free_entry(c, entry);
  1553. }
  1554. }
  1555. c->cnt = 0;
  1556. }
  1557. void auxtrace_cache__free(struct auxtrace_cache *c)
  1558. {
  1559. if (!c)
  1560. return;
  1561. auxtrace_cache__drop(c);
  1562. zfree(&c->hashtable);
  1563. free(c);
  1564. }
  1565. void *auxtrace_cache__alloc_entry(struct auxtrace_cache *c)
  1566. {
  1567. return malloc(c->entry_size);
  1568. }
  1569. void auxtrace_cache__free_entry(struct auxtrace_cache *c __maybe_unused,
  1570. void *entry)
  1571. {
  1572. free(entry);
  1573. }
  1574. int auxtrace_cache__add(struct auxtrace_cache *c, u32 key,
  1575. struct auxtrace_cache_entry *entry)
  1576. {
  1577. if (c->limit && ++c->cnt > c->limit)
  1578. auxtrace_cache__drop(c);
  1579. entry->key = key;
  1580. hlist_add_head(&entry->hash, &c->hashtable[hash_32(key, c->bits)]);
  1581. return 0;
  1582. }
  1583. static struct auxtrace_cache_entry *auxtrace_cache__rm(struct auxtrace_cache *c,
  1584. u32 key)
  1585. {
  1586. struct auxtrace_cache_entry *entry;
  1587. struct hlist_head *hlist;
  1588. struct hlist_node *n;
  1589. if (!c)
  1590. return NULL;
  1591. hlist = &c->hashtable[hash_32(key, c->bits)];
  1592. hlist_for_each_entry_safe(entry, n, hlist, hash) {
  1593. if (entry->key == key) {
  1594. hlist_del(&entry->hash);
  1595. return entry;
  1596. }
  1597. }
  1598. return NULL;
  1599. }
  1600. void auxtrace_cache__remove(struct auxtrace_cache *c, u32 key)
  1601. {
  1602. struct auxtrace_cache_entry *entry = auxtrace_cache__rm(c, key);
  1603. auxtrace_cache__free_entry(c, entry);
  1604. }
  1605. void *auxtrace_cache__lookup(struct auxtrace_cache *c, u32 key)
  1606. {
  1607. struct auxtrace_cache_entry *entry;
  1608. struct hlist_head *hlist;
  1609. if (!c)
  1610. return NULL;
  1611. hlist = &c->hashtable[hash_32(key, c->bits)];
  1612. hlist_for_each_entry(entry, hlist, hash) {
  1613. if (entry->key == key)
  1614. return entry;
  1615. }
  1616. return NULL;
  1617. }
  1618. static void addr_filter__free_str(struct addr_filter *filt)
  1619. {
  1620. zfree(&filt->str);
  1621. filt->action = NULL;
  1622. filt->sym_from = NULL;
  1623. filt->sym_to = NULL;
  1624. filt->filename = NULL;
  1625. }
  1626. static struct addr_filter *addr_filter__new(void)
  1627. {
  1628. struct addr_filter *filt = zalloc(sizeof(*filt));
  1629. if (filt)
  1630. INIT_LIST_HEAD(&filt->list);
  1631. return filt;
  1632. }
  1633. static void addr_filter__free(struct addr_filter *filt)
  1634. {
  1635. if (filt)
  1636. addr_filter__free_str(filt);
  1637. free(filt);
  1638. }
  1639. static void addr_filters__add(struct addr_filters *filts,
  1640. struct addr_filter *filt)
  1641. {
  1642. list_add_tail(&filt->list, &filts->head);
  1643. filts->cnt += 1;
  1644. }
  1645. static void addr_filters__del(struct addr_filters *filts,
  1646. struct addr_filter *filt)
  1647. {
  1648. list_del_init(&filt->list);
  1649. filts->cnt -= 1;
  1650. }
  1651. void addr_filters__init(struct addr_filters *filts)
  1652. {
  1653. INIT_LIST_HEAD(&filts->head);
  1654. filts->cnt = 0;
  1655. }
  1656. void addr_filters__exit(struct addr_filters *filts)
  1657. {
  1658. struct addr_filter *filt, *n;
  1659. list_for_each_entry_safe(filt, n, &filts->head, list) {
  1660. addr_filters__del(filts, filt);
  1661. addr_filter__free(filt);
  1662. }
  1663. }
  1664. static int parse_num_or_str(char **inp, u64 *num, const char **str,
  1665. const char *str_delim)
  1666. {
  1667. *inp += strspn(*inp, " ");
  1668. if (isdigit(**inp)) {
  1669. char *endptr;
  1670. if (!num)
  1671. return -EINVAL;
  1672. errno = 0;
  1673. *num = strtoull(*inp, &endptr, 0);
  1674. if (errno)
  1675. return -errno;
  1676. if (endptr == *inp)
  1677. return -EINVAL;
  1678. *inp = endptr;
  1679. } else {
  1680. size_t n;
  1681. if (!str)
  1682. return -EINVAL;
  1683. *inp += strspn(*inp, " ");
  1684. *str = *inp;
  1685. n = strcspn(*inp, str_delim);
  1686. if (!n)
  1687. return -EINVAL;
  1688. *inp += n;
  1689. if (**inp) {
  1690. **inp = '\0';
  1691. *inp += 1;
  1692. }
  1693. }
  1694. return 0;
  1695. }
  1696. static int parse_action(struct addr_filter *filt)
  1697. {
  1698. if (!strcmp(filt->action, "filter")) {
  1699. filt->start = true;
  1700. filt->range = true;
  1701. } else if (!strcmp(filt->action, "start")) {
  1702. filt->start = true;
  1703. } else if (!strcmp(filt->action, "stop")) {
  1704. filt->start = false;
  1705. } else if (!strcmp(filt->action, "tracestop")) {
  1706. filt->start = false;
  1707. filt->range = true;
  1708. filt->action += 5; /* Change 'tracestop' to 'stop' */
  1709. } else {
  1710. return -EINVAL;
  1711. }
  1712. return 0;
  1713. }
  1714. static int parse_sym_idx(char **inp, int *idx)
  1715. {
  1716. *idx = -1;
  1717. *inp += strspn(*inp, " ");
  1718. if (**inp != '#')
  1719. return 0;
  1720. *inp += 1;
  1721. if (**inp == 'g' || **inp == 'G') {
  1722. *inp += 1;
  1723. *idx = 0;
  1724. } else {
  1725. unsigned long num;
  1726. char *endptr;
  1727. errno = 0;
  1728. num = strtoul(*inp, &endptr, 0);
  1729. if (errno)
  1730. return -errno;
  1731. if (endptr == *inp || num > INT_MAX)
  1732. return -EINVAL;
  1733. *inp = endptr;
  1734. *idx = num;
  1735. }
  1736. return 0;
  1737. }
  1738. static int parse_addr_size(char **inp, u64 *num, const char **str, int *idx)
  1739. {
  1740. int err = parse_num_or_str(inp, num, str, " ");
  1741. if (!err && *str)
  1742. err = parse_sym_idx(inp, idx);
  1743. return err;
  1744. }
  1745. static int parse_one_filter(struct addr_filter *filt, const char **filter_inp)
  1746. {
  1747. char *fstr;
  1748. int err;
  1749. filt->str = fstr = strdup(*filter_inp);
  1750. if (!fstr)
  1751. return -ENOMEM;
  1752. err = parse_num_or_str(&fstr, NULL, &filt->action, " ");
  1753. if (err)
  1754. goto out_err;
  1755. err = parse_action(filt);
  1756. if (err)
  1757. goto out_err;
  1758. err = parse_addr_size(&fstr, &filt->addr, &filt->sym_from,
  1759. &filt->sym_from_idx);
  1760. if (err)
  1761. goto out_err;
  1762. fstr += strspn(fstr, " ");
  1763. if (*fstr == '/') {
  1764. fstr += 1;
  1765. err = parse_addr_size(&fstr, &filt->size, &filt->sym_to,
  1766. &filt->sym_to_idx);
  1767. if (err)
  1768. goto out_err;
  1769. filt->range = true;
  1770. }
  1771. fstr += strspn(fstr, " ");
  1772. if (*fstr == '@') {
  1773. fstr += 1;
  1774. err = parse_num_or_str(&fstr, NULL, &filt->filename, " ,");
  1775. if (err)
  1776. goto out_err;
  1777. }
  1778. fstr += strspn(fstr, " ,");
  1779. *filter_inp += fstr - filt->str;
  1780. return 0;
  1781. out_err:
  1782. addr_filter__free_str(filt);
  1783. return err;
  1784. }
  1785. int addr_filters__parse_bare_filter(struct addr_filters *filts,
  1786. const char *filter)
  1787. {
  1788. struct addr_filter *filt;
  1789. const char *fstr = filter;
  1790. int err;
  1791. while (*fstr) {
  1792. filt = addr_filter__new();
  1793. err = parse_one_filter(filt, &fstr);
  1794. if (err) {
  1795. addr_filter__free(filt);
  1796. addr_filters__exit(filts);
  1797. return err;
  1798. }
  1799. addr_filters__add(filts, filt);
  1800. }
  1801. return 0;
  1802. }
  1803. struct sym_args {
  1804. const char *name;
  1805. u64 start;
  1806. u64 size;
  1807. int idx;
  1808. int cnt;
  1809. bool started;
  1810. bool global;
  1811. bool selected;
  1812. bool duplicate;
  1813. bool near;
  1814. };
  1815. static bool kern_sym_match(struct sym_args *args, const char *name, char type)
  1816. {
  1817. /* A function with the same name, and global or the n'th found or any */
  1818. return kallsyms__is_function(type) &&
  1819. !strcmp(name, args->name) &&
  1820. ((args->global && isupper(type)) ||
  1821. (args->selected && ++(args->cnt) == args->idx) ||
  1822. (!args->global && !args->selected));
  1823. }
  1824. static int find_kern_sym_cb(void *arg, const char *name, char type, u64 start)
  1825. {
  1826. struct sym_args *args = arg;
  1827. if (args->started) {
  1828. if (!args->size)
  1829. args->size = start - args->start;
  1830. if (args->selected) {
  1831. if (args->size)
  1832. return 1;
  1833. } else if (kern_sym_match(args, name, type)) {
  1834. args->duplicate = true;
  1835. return 1;
  1836. }
  1837. } else if (kern_sym_match(args, name, type)) {
  1838. args->started = true;
  1839. args->start = start;
  1840. }
  1841. return 0;
  1842. }
  1843. static int print_kern_sym_cb(void *arg, const char *name, char type, u64 start)
  1844. {
  1845. struct sym_args *args = arg;
  1846. if (kern_sym_match(args, name, type)) {
  1847. pr_err("#%d\t0x%"PRIx64"\t%c\t%s\n",
  1848. ++args->cnt, start, type, name);
  1849. args->near = true;
  1850. } else if (args->near) {
  1851. args->near = false;
  1852. pr_err("\t\twhich is near\t\t%s\n", name);
  1853. }
  1854. return 0;
  1855. }
  1856. static int sym_not_found_error(const char *sym_name, int idx)
  1857. {
  1858. if (idx > 0) {
  1859. pr_err("N'th occurrence (N=%d) of symbol '%s' not found.\n",
  1860. idx, sym_name);
  1861. } else if (!idx) {
  1862. pr_err("Global symbol '%s' not found.\n", sym_name);
  1863. } else {
  1864. pr_err("Symbol '%s' not found.\n", sym_name);
  1865. }
  1866. pr_err("Note that symbols must be functions.\n");
  1867. return -EINVAL;
  1868. }
  1869. static int find_kern_sym(const char *sym_name, u64 *start, u64 *size, int idx)
  1870. {
  1871. struct sym_args args = {
  1872. .name = sym_name,
  1873. .idx = idx,
  1874. .global = !idx,
  1875. .selected = idx > 0,
  1876. };
  1877. int err;
  1878. *start = 0;
  1879. *size = 0;
  1880. err = kallsyms__parse("/proc/kallsyms", &args, find_kern_sym_cb);
  1881. if (err < 0) {
  1882. pr_err("Failed to parse /proc/kallsyms\n");
  1883. return err;
  1884. }
  1885. if (args.duplicate) {
  1886. pr_err("Multiple kernel symbols with name '%s'\n", sym_name);
  1887. args.cnt = 0;
  1888. kallsyms__parse("/proc/kallsyms", &args, print_kern_sym_cb);
  1889. pr_err("Disambiguate symbol name by inserting #n after the name e.g. %s #2\n",
  1890. sym_name);
  1891. pr_err("Or select a global symbol by inserting #0 or #g or #G\n");
  1892. return -EINVAL;
  1893. }
  1894. if (!args.started) {
  1895. pr_err("Kernel symbol lookup: ");
  1896. return sym_not_found_error(sym_name, idx);
  1897. }
  1898. *start = args.start;
  1899. *size = args.size;
  1900. return 0;
  1901. }
  1902. static int find_entire_kern_cb(void *arg, const char *name __maybe_unused,
  1903. char type, u64 start)
  1904. {
  1905. struct sym_args *args = arg;
  1906. if (!kallsyms__is_function(type))
  1907. return 0;
  1908. if (!args->started) {
  1909. args->started = true;
  1910. args->start = start;
  1911. }
  1912. /* Don't know exactly where the kernel ends, so we add a page */
  1913. args->size = round_up(start, page_size) + page_size - args->start;
  1914. return 0;
  1915. }
  1916. static int addr_filter__entire_kernel(struct addr_filter *filt)
  1917. {
  1918. struct sym_args args = { .started = false };
  1919. int err;
  1920. err = kallsyms__parse("/proc/kallsyms", &args, find_entire_kern_cb);
  1921. if (err < 0 || !args.started) {
  1922. pr_err("Failed to parse /proc/kallsyms\n");
  1923. return err;
  1924. }
  1925. filt->addr = args.start;
  1926. filt->size = args.size;
  1927. return 0;
  1928. }
  1929. static int check_end_after_start(struct addr_filter *filt, u64 start, u64 size)
  1930. {
  1931. if (start + size >= filt->addr)
  1932. return 0;
  1933. if (filt->sym_from) {
  1934. pr_err("Symbol '%s' (0x%"PRIx64") comes before '%s' (0x%"PRIx64")\n",
  1935. filt->sym_to, start, filt->sym_from, filt->addr);
  1936. } else {
  1937. pr_err("Symbol '%s' (0x%"PRIx64") comes before address 0x%"PRIx64")\n",
  1938. filt->sym_to, start, filt->addr);
  1939. }
  1940. return -EINVAL;
  1941. }
  1942. static int addr_filter__resolve_kernel_syms(struct addr_filter *filt)
  1943. {
  1944. bool no_size = false;
  1945. u64 start, size;
  1946. int err;
  1947. if (symbol_conf.kptr_restrict) {
  1948. pr_err("Kernel addresses are restricted. Unable to resolve kernel symbols.\n");
  1949. return -EINVAL;
  1950. }
  1951. if (filt->sym_from && !strcmp(filt->sym_from, "*"))
  1952. return addr_filter__entire_kernel(filt);
  1953. if (filt->sym_from) {
  1954. err = find_kern_sym(filt->sym_from, &start, &size,
  1955. filt->sym_from_idx);
  1956. if (err)
  1957. return err;
  1958. filt->addr = start;
  1959. if (filt->range && !filt->size && !filt->sym_to) {
  1960. filt->size = size;
  1961. no_size = !size;
  1962. }
  1963. }
  1964. if (filt->sym_to) {
  1965. err = find_kern_sym(filt->sym_to, &start, &size,
  1966. filt->sym_to_idx);
  1967. if (err)
  1968. return err;
  1969. err = check_end_after_start(filt, start, size);
  1970. if (err)
  1971. return err;
  1972. filt->size = start + size - filt->addr;
  1973. no_size = !size;
  1974. }
  1975. /* The very last symbol in kallsyms does not imply a particular size */
  1976. if (no_size) {
  1977. pr_err("Cannot determine size of symbol '%s'\n",
  1978. filt->sym_to ? filt->sym_to : filt->sym_from);
  1979. return -EINVAL;
  1980. }
  1981. return 0;
  1982. }
  1983. static struct dso *load_dso(const char *name)
  1984. {
  1985. struct map *map;
  1986. struct dso *dso;
  1987. map = dso__new_map(name);
  1988. if (!map)
  1989. return NULL;
  1990. if (map__load(map) < 0)
  1991. pr_err("File '%s' not found or has no symbols.\n", name);
  1992. dso = dso__get(map->dso);
  1993. map__put(map);
  1994. return dso;
  1995. }
  1996. static bool dso_sym_match(struct symbol *sym, const char *name, int *cnt,
  1997. int idx)
  1998. {
  1999. /* Same name, and global or the n'th found or any */
  2000. return !arch__compare_symbol_names(name, sym->name) &&
  2001. ((!idx && sym->binding == STB_GLOBAL) ||
  2002. (idx > 0 && ++*cnt == idx) ||
  2003. idx < 0);
  2004. }
  2005. static void print_duplicate_syms(struct dso *dso, const char *sym_name)
  2006. {
  2007. struct symbol *sym;
  2008. bool near = false;
  2009. int cnt = 0;
  2010. pr_err("Multiple symbols with name '%s'\n", sym_name);
  2011. sym = dso__first_symbol(dso);
  2012. while (sym) {
  2013. if (dso_sym_match(sym, sym_name, &cnt, -1)) {
  2014. pr_err("#%d\t0x%"PRIx64"\t%c\t%s\n",
  2015. ++cnt, sym->start,
  2016. sym->binding == STB_GLOBAL ? 'g' :
  2017. sym->binding == STB_LOCAL ? 'l' : 'w',
  2018. sym->name);
  2019. near = true;
  2020. } else if (near) {
  2021. near = false;
  2022. pr_err("\t\twhich is near\t\t%s\n", sym->name);
  2023. }
  2024. sym = dso__next_symbol(sym);
  2025. }
  2026. pr_err("Disambiguate symbol name by inserting #n after the name e.g. %s #2\n",
  2027. sym_name);
  2028. pr_err("Or select a global symbol by inserting #0 or #g or #G\n");
  2029. }
  2030. static int find_dso_sym(struct dso *dso, const char *sym_name, u64 *start,
  2031. u64 *size, int idx)
  2032. {
  2033. struct symbol *sym;
  2034. int cnt = 0;
  2035. *start = 0;
  2036. *size = 0;
  2037. sym = dso__first_symbol(dso);
  2038. while (sym) {
  2039. if (*start) {
  2040. if (!*size)
  2041. *size = sym->start - *start;
  2042. if (idx > 0) {
  2043. if (*size)
  2044. return 1;
  2045. } else if (dso_sym_match(sym, sym_name, &cnt, idx)) {
  2046. print_duplicate_syms(dso, sym_name);
  2047. return -EINVAL;
  2048. }
  2049. } else if (dso_sym_match(sym, sym_name, &cnt, idx)) {
  2050. *start = sym->start;
  2051. *size = sym->end - sym->start;
  2052. }
  2053. sym = dso__next_symbol(sym);
  2054. }
  2055. if (!*start)
  2056. return sym_not_found_error(sym_name, idx);
  2057. return 0;
  2058. }
  2059. static int addr_filter__entire_dso(struct addr_filter *filt, struct dso *dso)
  2060. {
  2061. if (dso__data_file_size(dso, NULL)) {
  2062. pr_err("Failed to determine filter for %s\nCannot determine file size.\n",
  2063. filt->filename);
  2064. return -EINVAL;
  2065. }
  2066. filt->addr = 0;
  2067. filt->size = dso->data.file_size;
  2068. return 0;
  2069. }
  2070. static int addr_filter__resolve_syms(struct addr_filter *filt)
  2071. {
  2072. u64 start, size;
  2073. struct dso *dso;
  2074. int err = 0;
  2075. if (!filt->sym_from && !filt->sym_to)
  2076. return 0;
  2077. if (!filt->filename)
  2078. return addr_filter__resolve_kernel_syms(filt);
  2079. dso = load_dso(filt->filename);
  2080. if (!dso) {
  2081. pr_err("Failed to load symbols from: %s\n", filt->filename);
  2082. return -EINVAL;
  2083. }
  2084. if (filt->sym_from && !strcmp(filt->sym_from, "*")) {
  2085. err = addr_filter__entire_dso(filt, dso);
  2086. goto put_dso;
  2087. }
  2088. if (filt->sym_from) {
  2089. err = find_dso_sym(dso, filt->sym_from, &start, &size,
  2090. filt->sym_from_idx);
  2091. if (err)
  2092. goto put_dso;
  2093. filt->addr = start;
  2094. if (filt->range && !filt->size && !filt->sym_to)
  2095. filt->size = size;
  2096. }
  2097. if (filt->sym_to) {
  2098. err = find_dso_sym(dso, filt->sym_to, &start, &size,
  2099. filt->sym_to_idx);
  2100. if (err)
  2101. goto put_dso;
  2102. err = check_end_after_start(filt, start, size);
  2103. if (err)
  2104. return err;
  2105. filt->size = start + size - filt->addr;
  2106. }
  2107. put_dso:
  2108. dso__put(dso);
  2109. return err;
  2110. }
  2111. static char *addr_filter__to_str(struct addr_filter *filt)
  2112. {
  2113. char filename_buf[PATH_MAX];
  2114. const char *at = "";
  2115. const char *fn = "";
  2116. char *filter;
  2117. int err;
  2118. if (filt->filename) {
  2119. at = "@";
  2120. fn = realpath(filt->filename, filename_buf);
  2121. if (!fn)
  2122. return NULL;
  2123. }
  2124. if (filt->range) {
  2125. err = asprintf(&filter, "%s 0x%"PRIx64"/0x%"PRIx64"%s%s",
  2126. filt->action, filt->addr, filt->size, at, fn);
  2127. } else {
  2128. err = asprintf(&filter, "%s 0x%"PRIx64"%s%s",
  2129. filt->action, filt->addr, at, fn);
  2130. }
  2131. return err < 0 ? NULL : filter;
  2132. }
  2133. static int parse_addr_filter(struct evsel *evsel, const char *filter,
  2134. int max_nr)
  2135. {
  2136. struct addr_filters filts;
  2137. struct addr_filter *filt;
  2138. int err;
  2139. addr_filters__init(&filts);
  2140. err = addr_filters__parse_bare_filter(&filts, filter);
  2141. if (err)
  2142. goto out_exit;
  2143. if (filts.cnt > max_nr) {
  2144. pr_err("Error: number of address filters (%d) exceeds maximum (%d)\n",
  2145. filts.cnt, max_nr);
  2146. err = -EINVAL;
  2147. goto out_exit;
  2148. }
  2149. list_for_each_entry(filt, &filts.head, list) {
  2150. char *new_filter;
  2151. err = addr_filter__resolve_syms(filt);
  2152. if (err)
  2153. goto out_exit;
  2154. new_filter = addr_filter__to_str(filt);
  2155. if (!new_filter) {
  2156. err = -ENOMEM;
  2157. goto out_exit;
  2158. }
  2159. if (evsel__append_addr_filter(evsel, new_filter)) {
  2160. err = -ENOMEM;
  2161. goto out_exit;
  2162. }
  2163. }
  2164. out_exit:
  2165. addr_filters__exit(&filts);
  2166. if (err) {
  2167. pr_err("Failed to parse address filter: '%s'\n", filter);
  2168. pr_err("Filter format is: filter|start|stop|tracestop <start symbol or address> [/ <end symbol or size>] [@<file name>]\n");
  2169. pr_err("Where multiple filters are separated by space or comma.\n");
  2170. }
  2171. return err;
  2172. }
  2173. static int evsel__nr_addr_filter(struct evsel *evsel)
  2174. {
  2175. struct perf_pmu *pmu = evsel__find_pmu(evsel);
  2176. int nr_addr_filters = 0;
  2177. if (!pmu)
  2178. return 0;
  2179. perf_pmu__scan_file(pmu, "nr_addr_filters", "%d", &nr_addr_filters);
  2180. return nr_addr_filters;
  2181. }
  2182. int auxtrace_parse_filters(struct evlist *evlist)
  2183. {
  2184. struct evsel *evsel;
  2185. char *filter;
  2186. int err, max_nr;
  2187. evlist__for_each_entry(evlist, evsel) {
  2188. filter = evsel->filter;
  2189. max_nr = evsel__nr_addr_filter(evsel);
  2190. if (!filter || !max_nr)
  2191. continue;
  2192. evsel->filter = NULL;
  2193. err = parse_addr_filter(evsel, filter, max_nr);
  2194. free(filter);
  2195. if (err)
  2196. return err;
  2197. pr_debug("Address filter: %s\n", evsel->filter);
  2198. }
  2199. return 0;
  2200. }
  2201. int auxtrace__process_event(struct perf_session *session, union perf_event *event,
  2202. struct perf_sample *sample, struct perf_tool *tool)
  2203. {
  2204. if (!session->auxtrace)
  2205. return 0;
  2206. return session->auxtrace->process_event(session, event, sample, tool);
  2207. }
  2208. void auxtrace__dump_auxtrace_sample(struct perf_session *session,
  2209. struct perf_sample *sample)
  2210. {
  2211. if (!session->auxtrace || !session->auxtrace->dump_auxtrace_sample ||
  2212. auxtrace__dont_decode(session))
  2213. return;
  2214. session->auxtrace->dump_auxtrace_sample(session, sample);
  2215. }
  2216. int auxtrace__flush_events(struct perf_session *session, struct perf_tool *tool)
  2217. {
  2218. if (!session->auxtrace)
  2219. return 0;
  2220. return session->auxtrace->flush_events(session, tool);
  2221. }
  2222. void auxtrace__free_events(struct perf_session *session)
  2223. {
  2224. if (!session->auxtrace)
  2225. return;
  2226. return session->auxtrace->free_events(session);
  2227. }
  2228. void auxtrace__free(struct perf_session *session)
  2229. {
  2230. if (!session->auxtrace)
  2231. return;
  2232. return session->auxtrace->free(session);
  2233. }
  2234. bool auxtrace__evsel_is_auxtrace(struct perf_session *session,
  2235. struct evsel *evsel)
  2236. {
  2237. if (!session->auxtrace || !session->auxtrace->evsel_is_auxtrace)
  2238. return false;
  2239. return session->auxtrace->evsel_is_auxtrace(session, evsel);
  2240. }