bpf-event.c 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576
  1. // SPDX-License-Identifier: GPL-2.0
  2. #include <errno.h>
  3. #include <stdlib.h>
  4. #include <bpf/bpf.h>
  5. #include <bpf/btf.h>
  6. #include <bpf/libbpf.h>
  7. #include <linux/btf.h>
  8. #include <linux/err.h>
  9. #include <linux/string.h>
  10. #include <internal/lib.h>
  11. #include <symbol/kallsyms.h>
  12. #include "bpf-event.h"
  13. #include "debug.h"
  14. #include "dso.h"
  15. #include "symbol.h"
  16. #include "machine.h"
  17. #include "env.h"
  18. #include "session.h"
  19. #include "map.h"
  20. #include "evlist.h"
  21. #include "record.h"
  22. #include "util/synthetic-events.h"
  23. #define ptr_to_u64(ptr) ((__u64)(unsigned long)(ptr))
  24. static int snprintf_hex(char *buf, size_t size, unsigned char *data, size_t len)
  25. {
  26. int ret = 0;
  27. size_t i;
  28. for (i = 0; i < len; i++)
  29. ret += snprintf(buf + ret, size - ret, "%02x", data[i]);
  30. return ret;
  31. }
  32. static int machine__process_bpf_event_load(struct machine *machine,
  33. union perf_event *event,
  34. struct perf_sample *sample __maybe_unused)
  35. {
  36. struct bpf_prog_info_linear *info_linear;
  37. struct bpf_prog_info_node *info_node;
  38. struct perf_env *env = machine->env;
  39. int id = event->bpf.id;
  40. unsigned int i;
  41. /* perf-record, no need to handle bpf-event */
  42. if (env == NULL)
  43. return 0;
  44. info_node = perf_env__find_bpf_prog_info(env, id);
  45. if (!info_node)
  46. return 0;
  47. info_linear = info_node->info_linear;
  48. for (i = 0; i < info_linear->info.nr_jited_ksyms; i++) {
  49. u64 *addrs = (u64 *)(uintptr_t)(info_linear->info.jited_ksyms);
  50. u64 addr = addrs[i];
  51. struct map *map = maps__find(&machine->kmaps, addr);
  52. if (map) {
  53. map->dso->binary_type = DSO_BINARY_TYPE__BPF_PROG_INFO;
  54. map->dso->bpf_prog.id = id;
  55. map->dso->bpf_prog.sub_id = i;
  56. map->dso->bpf_prog.env = env;
  57. }
  58. }
  59. return 0;
  60. }
  61. int machine__process_bpf(struct machine *machine, union perf_event *event,
  62. struct perf_sample *sample)
  63. {
  64. if (dump_trace)
  65. perf_event__fprintf_bpf(event, stdout);
  66. switch (event->bpf.type) {
  67. case PERF_BPF_EVENT_PROG_LOAD:
  68. return machine__process_bpf_event_load(machine, event, sample);
  69. case PERF_BPF_EVENT_PROG_UNLOAD:
  70. /*
  71. * Do not free bpf_prog_info and btf of the program here,
  72. * as annotation still need them. They will be freed at
  73. * the end of the session.
  74. */
  75. break;
  76. default:
  77. pr_debug("unexpected bpf event type of %d\n", event->bpf.type);
  78. break;
  79. }
  80. return 0;
  81. }
  82. static int perf_env__fetch_btf(struct perf_env *env,
  83. u32 btf_id,
  84. struct btf *btf)
  85. {
  86. struct btf_node *node;
  87. u32 data_size;
  88. const void *data;
  89. data = btf__get_raw_data(btf, &data_size);
  90. node = malloc(data_size + sizeof(struct btf_node));
  91. if (!node)
  92. return -1;
  93. node->id = btf_id;
  94. node->data_size = data_size;
  95. memcpy(node->data, data, data_size);
  96. if (!perf_env__insert_btf(env, node)) {
  97. /* Insertion failed because of a duplicate. */
  98. free(node);
  99. return -1;
  100. }
  101. return 0;
  102. }
  103. static int synthesize_bpf_prog_name(char *buf, int size,
  104. struct bpf_prog_info *info,
  105. struct btf *btf,
  106. u32 sub_id)
  107. {
  108. u8 (*prog_tags)[BPF_TAG_SIZE] = (void *)(uintptr_t)(info->prog_tags);
  109. void *func_infos = (void *)(uintptr_t)(info->func_info);
  110. u32 sub_prog_cnt = info->nr_jited_ksyms;
  111. const struct bpf_func_info *finfo;
  112. const char *short_name = NULL;
  113. const struct btf_type *t;
  114. int name_len;
  115. name_len = snprintf(buf, size, "bpf_prog_");
  116. name_len += snprintf_hex(buf + name_len, size - name_len,
  117. prog_tags[sub_id], BPF_TAG_SIZE);
  118. if (btf) {
  119. finfo = func_infos + sub_id * info->func_info_rec_size;
  120. t = btf__type_by_id(btf, finfo->type_id);
  121. short_name = btf__name_by_offset(btf, t->name_off);
  122. } else if (sub_id == 0 && sub_prog_cnt == 1) {
  123. /* no subprog */
  124. if (info->name[0])
  125. short_name = info->name;
  126. } else
  127. short_name = "F";
  128. if (short_name)
  129. name_len += snprintf(buf + name_len, size - name_len,
  130. "_%s", short_name);
  131. return name_len;
  132. }
  133. /*
  134. * Synthesize PERF_RECORD_KSYMBOL and PERF_RECORD_BPF_EVENT for one bpf
  135. * program. One PERF_RECORD_BPF_EVENT is generated for the program. And
  136. * one PERF_RECORD_KSYMBOL is generated for each sub program.
  137. *
  138. * Returns:
  139. * 0 for success;
  140. * -1 for failures;
  141. * -2 for lack of kernel support.
  142. */
  143. static int perf_event__synthesize_one_bpf_prog(struct perf_session *session,
  144. perf_event__handler_t process,
  145. struct machine *machine,
  146. int fd,
  147. union perf_event *event,
  148. struct record_opts *opts)
  149. {
  150. struct perf_record_ksymbol *ksymbol_event = &event->ksymbol;
  151. struct perf_record_bpf_event *bpf_event = &event->bpf;
  152. struct bpf_prog_info_linear *info_linear;
  153. struct perf_tool *tool = session->tool;
  154. struct bpf_prog_info_node *info_node;
  155. struct bpf_prog_info *info;
  156. struct btf *btf = NULL;
  157. struct perf_env *env;
  158. u32 sub_prog_cnt, i;
  159. int err = 0;
  160. u64 arrays;
  161. /*
  162. * for perf-record and perf-report use header.env;
  163. * otherwise, use global perf_env.
  164. */
  165. env = session->data ? &session->header.env : &perf_env;
  166. arrays = 1UL << BPF_PROG_INFO_JITED_KSYMS;
  167. arrays |= 1UL << BPF_PROG_INFO_JITED_FUNC_LENS;
  168. arrays |= 1UL << BPF_PROG_INFO_FUNC_INFO;
  169. arrays |= 1UL << BPF_PROG_INFO_PROG_TAGS;
  170. arrays |= 1UL << BPF_PROG_INFO_JITED_INSNS;
  171. arrays |= 1UL << BPF_PROG_INFO_LINE_INFO;
  172. arrays |= 1UL << BPF_PROG_INFO_JITED_LINE_INFO;
  173. info_linear = bpf_program__get_prog_info_linear(fd, arrays);
  174. if (IS_ERR_OR_NULL(info_linear)) {
  175. info_linear = NULL;
  176. pr_debug("%s: failed to get BPF program info. aborting\n", __func__);
  177. return -1;
  178. }
  179. if (info_linear->info_len < offsetof(struct bpf_prog_info, prog_tags)) {
  180. pr_debug("%s: the kernel is too old, aborting\n", __func__);
  181. return -2;
  182. }
  183. info = &info_linear->info;
  184. /* number of ksyms, func_lengths, and tags should match */
  185. sub_prog_cnt = info->nr_jited_ksyms;
  186. if (sub_prog_cnt != info->nr_prog_tags ||
  187. sub_prog_cnt != info->nr_jited_func_lens)
  188. return -1;
  189. /* check BTF func info support */
  190. if (info->btf_id && info->nr_func_info && info->func_info_rec_size) {
  191. /* btf func info number should be same as sub_prog_cnt */
  192. if (sub_prog_cnt != info->nr_func_info) {
  193. pr_debug("%s: mismatch in BPF sub program count and BTF function info count, aborting\n", __func__);
  194. err = -1;
  195. goto out;
  196. }
  197. if (btf__get_from_id(info->btf_id, &btf)) {
  198. pr_debug("%s: failed to get BTF of id %u, aborting\n", __func__, info->btf_id);
  199. err = -1;
  200. btf = NULL;
  201. goto out;
  202. }
  203. perf_env__fetch_btf(env, info->btf_id, btf);
  204. }
  205. /* Synthesize PERF_RECORD_KSYMBOL */
  206. for (i = 0; i < sub_prog_cnt; i++) {
  207. __u32 *prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens);
  208. __u64 *prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms);
  209. int name_len;
  210. *ksymbol_event = (struct perf_record_ksymbol) {
  211. .header = {
  212. .type = PERF_RECORD_KSYMBOL,
  213. .size = offsetof(struct perf_record_ksymbol, name),
  214. },
  215. .addr = prog_addrs[i],
  216. .len = prog_lens[i],
  217. .ksym_type = PERF_RECORD_KSYMBOL_TYPE_BPF,
  218. .flags = 0,
  219. };
  220. name_len = synthesize_bpf_prog_name(ksymbol_event->name,
  221. KSYM_NAME_LEN, info, btf, i);
  222. ksymbol_event->header.size += PERF_ALIGN(name_len + 1,
  223. sizeof(u64));
  224. memset((void *)event + event->header.size, 0, machine->id_hdr_size);
  225. event->header.size += machine->id_hdr_size;
  226. err = perf_tool__process_synth_event(tool, event,
  227. machine, process);
  228. }
  229. if (!opts->no_bpf_event) {
  230. /* Synthesize PERF_RECORD_BPF_EVENT */
  231. *bpf_event = (struct perf_record_bpf_event) {
  232. .header = {
  233. .type = PERF_RECORD_BPF_EVENT,
  234. .size = sizeof(struct perf_record_bpf_event),
  235. },
  236. .type = PERF_BPF_EVENT_PROG_LOAD,
  237. .flags = 0,
  238. .id = info->id,
  239. };
  240. memcpy(bpf_event->tag, info->tag, BPF_TAG_SIZE);
  241. memset((void *)event + event->header.size, 0, machine->id_hdr_size);
  242. event->header.size += machine->id_hdr_size;
  243. /* save bpf_prog_info to env */
  244. info_node = malloc(sizeof(struct bpf_prog_info_node));
  245. if (!info_node) {
  246. err = -1;
  247. goto out;
  248. }
  249. info_node->info_linear = info_linear;
  250. perf_env__insert_bpf_prog_info(env, info_node);
  251. info_linear = NULL;
  252. /*
  253. * process after saving bpf_prog_info to env, so that
  254. * required information is ready for look up
  255. */
  256. err = perf_tool__process_synth_event(tool, event,
  257. machine, process);
  258. }
  259. out:
  260. free(info_linear);
  261. free(btf);
  262. return err ? -1 : 0;
  263. }
  264. struct kallsyms_parse {
  265. union perf_event *event;
  266. perf_event__handler_t process;
  267. struct machine *machine;
  268. struct perf_tool *tool;
  269. };
  270. static int
  271. process_bpf_image(char *name, u64 addr, struct kallsyms_parse *data)
  272. {
  273. struct machine *machine = data->machine;
  274. union perf_event *event = data->event;
  275. struct perf_record_ksymbol *ksymbol;
  276. int len;
  277. ksymbol = &event->ksymbol;
  278. *ksymbol = (struct perf_record_ksymbol) {
  279. .header = {
  280. .type = PERF_RECORD_KSYMBOL,
  281. .size = offsetof(struct perf_record_ksymbol, name),
  282. },
  283. .addr = addr,
  284. .len = page_size,
  285. .ksym_type = PERF_RECORD_KSYMBOL_TYPE_BPF,
  286. .flags = 0,
  287. };
  288. len = scnprintf(ksymbol->name, KSYM_NAME_LEN, "%s", name);
  289. ksymbol->header.size += PERF_ALIGN(len + 1, sizeof(u64));
  290. memset((void *) event + event->header.size, 0, machine->id_hdr_size);
  291. event->header.size += machine->id_hdr_size;
  292. return perf_tool__process_synth_event(data->tool, event, machine,
  293. data->process);
  294. }
  295. static int
  296. kallsyms_process_symbol(void *data, const char *_name,
  297. char type __maybe_unused, u64 start)
  298. {
  299. char disp[KSYM_NAME_LEN];
  300. char *module, *name;
  301. unsigned long id;
  302. int err = 0;
  303. module = strchr(_name, '\t');
  304. if (!module)
  305. return 0;
  306. /* We are going after [bpf] module ... */
  307. if (strcmp(module + 1, "[bpf]"))
  308. return 0;
  309. name = memdup(_name, (module - _name) + 1);
  310. if (!name)
  311. return -ENOMEM;
  312. name[module - _name] = 0;
  313. /* .. and only for trampolines and dispatchers */
  314. if ((sscanf(name, "bpf_trampoline_%lu", &id) == 1) ||
  315. (sscanf(name, "bpf_dispatcher_%s", disp) == 1))
  316. err = process_bpf_image(name, start, data);
  317. free(name);
  318. return err;
  319. }
  320. int perf_event__synthesize_bpf_events(struct perf_session *session,
  321. perf_event__handler_t process,
  322. struct machine *machine,
  323. struct record_opts *opts)
  324. {
  325. const char *kallsyms_filename = "/proc/kallsyms";
  326. struct kallsyms_parse arg;
  327. union perf_event *event;
  328. __u32 id = 0;
  329. int err;
  330. int fd;
  331. event = malloc(sizeof(event->bpf) + KSYM_NAME_LEN + machine->id_hdr_size);
  332. if (!event)
  333. return -1;
  334. /* Synthesize all the bpf programs in system. */
  335. while (true) {
  336. err = bpf_prog_get_next_id(id, &id);
  337. if (err) {
  338. if (errno == ENOENT) {
  339. err = 0;
  340. break;
  341. }
  342. pr_debug("%s: can't get next program: %s%s\n",
  343. __func__, strerror(errno),
  344. errno == EINVAL ? " -- kernel too old?" : "");
  345. /* don't report error on old kernel or EPERM */
  346. err = (errno == EINVAL || errno == EPERM) ? 0 : -1;
  347. break;
  348. }
  349. fd = bpf_prog_get_fd_by_id(id);
  350. if (fd < 0) {
  351. pr_debug("%s: failed to get fd for prog_id %u\n",
  352. __func__, id);
  353. continue;
  354. }
  355. err = perf_event__synthesize_one_bpf_prog(session, process,
  356. machine, fd,
  357. event, opts);
  358. close(fd);
  359. if (err) {
  360. /* do not return error for old kernel */
  361. if (err == -2)
  362. err = 0;
  363. break;
  364. }
  365. }
  366. /* Synthesize all the bpf images - trampolines/dispatchers. */
  367. if (symbol_conf.kallsyms_name != NULL)
  368. kallsyms_filename = symbol_conf.kallsyms_name;
  369. arg = (struct kallsyms_parse) {
  370. .event = event,
  371. .process = process,
  372. .machine = machine,
  373. .tool = session->tool,
  374. };
  375. if (kallsyms__parse(kallsyms_filename, &arg, kallsyms_process_symbol)) {
  376. pr_err("%s: failed to synthesize bpf images: %s\n",
  377. __func__, strerror(errno));
  378. }
  379. free(event);
  380. return err;
  381. }
  382. static void perf_env__add_bpf_info(struct perf_env *env, u32 id)
  383. {
  384. struct bpf_prog_info_linear *info_linear;
  385. struct bpf_prog_info_node *info_node;
  386. struct btf *btf = NULL;
  387. u64 arrays;
  388. u32 btf_id;
  389. int fd;
  390. fd = bpf_prog_get_fd_by_id(id);
  391. if (fd < 0)
  392. return;
  393. arrays = 1UL << BPF_PROG_INFO_JITED_KSYMS;
  394. arrays |= 1UL << BPF_PROG_INFO_JITED_FUNC_LENS;
  395. arrays |= 1UL << BPF_PROG_INFO_FUNC_INFO;
  396. arrays |= 1UL << BPF_PROG_INFO_PROG_TAGS;
  397. arrays |= 1UL << BPF_PROG_INFO_JITED_INSNS;
  398. arrays |= 1UL << BPF_PROG_INFO_LINE_INFO;
  399. arrays |= 1UL << BPF_PROG_INFO_JITED_LINE_INFO;
  400. info_linear = bpf_program__get_prog_info_linear(fd, arrays);
  401. if (IS_ERR_OR_NULL(info_linear)) {
  402. pr_debug("%s: failed to get BPF program info. aborting\n", __func__);
  403. goto out;
  404. }
  405. btf_id = info_linear->info.btf_id;
  406. info_node = malloc(sizeof(struct bpf_prog_info_node));
  407. if (info_node) {
  408. info_node->info_linear = info_linear;
  409. perf_env__insert_bpf_prog_info(env, info_node);
  410. } else
  411. free(info_linear);
  412. if (btf_id == 0)
  413. goto out;
  414. if (btf__get_from_id(btf_id, &btf)) {
  415. pr_debug("%s: failed to get BTF of id %u, aborting\n",
  416. __func__, btf_id);
  417. goto out;
  418. }
  419. perf_env__fetch_btf(env, btf_id, btf);
  420. out:
  421. free(btf);
  422. close(fd);
  423. }
  424. static int bpf_event__sb_cb(union perf_event *event, void *data)
  425. {
  426. struct perf_env *env = data;
  427. if (event->header.type != PERF_RECORD_BPF_EVENT)
  428. return -1;
  429. switch (event->bpf.type) {
  430. case PERF_BPF_EVENT_PROG_LOAD:
  431. perf_env__add_bpf_info(env, event->bpf.id);
  432. case PERF_BPF_EVENT_PROG_UNLOAD:
  433. /*
  434. * Do not free bpf_prog_info and btf of the program here,
  435. * as annotation still need them. They will be freed at
  436. * the end of the session.
  437. */
  438. break;
  439. default:
  440. pr_debug("unexpected bpf event type of %d\n", event->bpf.type);
  441. break;
  442. }
  443. return 0;
  444. }
  445. int evlist__add_bpf_sb_event(struct evlist *evlist, struct perf_env *env)
  446. {
  447. struct perf_event_attr attr = {
  448. .type = PERF_TYPE_SOFTWARE,
  449. .config = PERF_COUNT_SW_DUMMY,
  450. .sample_id_all = 1,
  451. .watermark = 1,
  452. .bpf_event = 1,
  453. .size = sizeof(attr), /* to capture ABI version */
  454. };
  455. /*
  456. * Older gcc versions don't support designated initializers, like above,
  457. * for unnamed union members, such as the following:
  458. */
  459. attr.wakeup_watermark = 1;
  460. return perf_evlist__add_sb_event(evlist, &attr, bpf_event__sb_cb, env);
  461. }
  462. void bpf_event__print_bpf_prog_info(struct bpf_prog_info *info,
  463. struct perf_env *env,
  464. FILE *fp)
  465. {
  466. __u32 *prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens);
  467. __u64 *prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms);
  468. char name[KSYM_NAME_LEN];
  469. struct btf *btf = NULL;
  470. u32 sub_prog_cnt, i;
  471. sub_prog_cnt = info->nr_jited_ksyms;
  472. if (sub_prog_cnt != info->nr_prog_tags ||
  473. sub_prog_cnt != info->nr_jited_func_lens)
  474. return;
  475. if (info->btf_id) {
  476. struct btf_node *node;
  477. node = perf_env__find_btf(env, info->btf_id);
  478. if (node)
  479. btf = btf__new((__u8 *)(node->data),
  480. node->data_size);
  481. }
  482. if (sub_prog_cnt == 1) {
  483. synthesize_bpf_prog_name(name, KSYM_NAME_LEN, info, btf, 0);
  484. fprintf(fp, "# bpf_prog_info %u: %s addr 0x%llx size %u\n",
  485. info->id, name, prog_addrs[0], prog_lens[0]);
  486. goto out;
  487. }
  488. fprintf(fp, "# bpf_prog_info %u:\n", info->id);
  489. for (i = 0; i < sub_prog_cnt; i++) {
  490. synthesize_bpf_prog_name(name, KSYM_NAME_LEN, info, btf, i);
  491. fprintf(fp, "# \tsub_prog %u: %s addr 0x%llx size %u\n",
  492. i, name, prog_addrs[i], prog_lens[i]);
  493. }
  494. out:
  495. btf__free(btf);
  496. }