builtin-kmem.c 46 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022
  1. // SPDX-License-Identifier: GPL-2.0
  2. #include "builtin.h"
  3. #include "perf.h"
  4. #include "util/dso.h"
  5. #include "util/evlist.h"
  6. #include "util/evsel.h"
  7. #include "util/config.h"
  8. #include "util/map.h"
  9. #include "util/symbol.h"
  10. #include "util/thread.h"
  11. #include "util/header.h"
  12. #include "util/session.h"
  13. #include "util/tool.h"
  14. #include "util/callchain.h"
  15. #include "util/time-utils.h"
  16. #include <linux/err.h>
  17. #include <subcmd/pager.h>
  18. #include <subcmd/parse-options.h>
  19. #include "util/trace-event.h"
  20. #include "util/data.h"
  21. #include "util/cpumap.h"
  22. #include "util/debug.h"
  23. #include "util/string2.h"
  24. #include <linux/kernel.h>
  25. #include <linux/rbtree.h>
  26. #include <linux/string.h>
  27. #include <linux/zalloc.h>
  28. #include <errno.h>
  29. #include <inttypes.h>
  30. #include <locale.h>
  31. #include <regex.h>
  32. #include <linux/ctype.h>
  33. static int kmem_slab;
  34. static int kmem_page;
  35. static long kmem_page_size;
  36. static enum {
  37. KMEM_SLAB,
  38. KMEM_PAGE,
  39. } kmem_default = KMEM_SLAB; /* for backward compatibility */
  40. struct alloc_stat;
  41. typedef int (*sort_fn_t)(void *, void *);
  42. static int alloc_flag;
  43. static int caller_flag;
  44. static int alloc_lines = -1;
  45. static int caller_lines = -1;
  46. static bool raw_ip;
  47. struct alloc_stat {
  48. u64 call_site;
  49. u64 ptr;
  50. u64 bytes_req;
  51. u64 bytes_alloc;
  52. u64 last_alloc;
  53. u32 hit;
  54. u32 pingpong;
  55. short alloc_cpu;
  56. struct rb_node node;
  57. };
  58. static struct rb_root root_alloc_stat;
  59. static struct rb_root root_alloc_sorted;
  60. static struct rb_root root_caller_stat;
  61. static struct rb_root root_caller_sorted;
  62. static unsigned long total_requested, total_allocated, total_freed;
  63. static unsigned long nr_allocs, nr_cross_allocs;
  64. /* filters for controlling start and stop of time of analysis */
  65. static struct perf_time_interval ptime;
  66. const char *time_str;
  67. static int insert_alloc_stat(unsigned long call_site, unsigned long ptr,
  68. int bytes_req, int bytes_alloc, int cpu)
  69. {
  70. struct rb_node **node = &root_alloc_stat.rb_node;
  71. struct rb_node *parent = NULL;
  72. struct alloc_stat *data = NULL;
  73. while (*node) {
  74. parent = *node;
  75. data = rb_entry(*node, struct alloc_stat, node);
  76. if (ptr > data->ptr)
  77. node = &(*node)->rb_right;
  78. else if (ptr < data->ptr)
  79. node = &(*node)->rb_left;
  80. else
  81. break;
  82. }
  83. if (data && data->ptr == ptr) {
  84. data->hit++;
  85. data->bytes_req += bytes_req;
  86. data->bytes_alloc += bytes_alloc;
  87. } else {
  88. data = malloc(sizeof(*data));
  89. if (!data) {
  90. pr_err("%s: malloc failed\n", __func__);
  91. return -1;
  92. }
  93. data->ptr = ptr;
  94. data->pingpong = 0;
  95. data->hit = 1;
  96. data->bytes_req = bytes_req;
  97. data->bytes_alloc = bytes_alloc;
  98. rb_link_node(&data->node, parent, node);
  99. rb_insert_color(&data->node, &root_alloc_stat);
  100. }
  101. data->call_site = call_site;
  102. data->alloc_cpu = cpu;
  103. data->last_alloc = bytes_alloc;
  104. return 0;
  105. }
  106. static int insert_caller_stat(unsigned long call_site,
  107. int bytes_req, int bytes_alloc)
  108. {
  109. struct rb_node **node = &root_caller_stat.rb_node;
  110. struct rb_node *parent = NULL;
  111. struct alloc_stat *data = NULL;
  112. while (*node) {
  113. parent = *node;
  114. data = rb_entry(*node, struct alloc_stat, node);
  115. if (call_site > data->call_site)
  116. node = &(*node)->rb_right;
  117. else if (call_site < data->call_site)
  118. node = &(*node)->rb_left;
  119. else
  120. break;
  121. }
  122. if (data && data->call_site == call_site) {
  123. data->hit++;
  124. data->bytes_req += bytes_req;
  125. data->bytes_alloc += bytes_alloc;
  126. } else {
  127. data = malloc(sizeof(*data));
  128. if (!data) {
  129. pr_err("%s: malloc failed\n", __func__);
  130. return -1;
  131. }
  132. data->call_site = call_site;
  133. data->pingpong = 0;
  134. data->hit = 1;
  135. data->bytes_req = bytes_req;
  136. data->bytes_alloc = bytes_alloc;
  137. rb_link_node(&data->node, parent, node);
  138. rb_insert_color(&data->node, &root_caller_stat);
  139. }
  140. return 0;
  141. }
  142. static int evsel__process_alloc_event(struct evsel *evsel, struct perf_sample *sample)
  143. {
  144. unsigned long ptr = evsel__intval(evsel, sample, "ptr"),
  145. call_site = evsel__intval(evsel, sample, "call_site");
  146. int bytes_req = evsel__intval(evsel, sample, "bytes_req"),
  147. bytes_alloc = evsel__intval(evsel, sample, "bytes_alloc");
  148. if (insert_alloc_stat(call_site, ptr, bytes_req, bytes_alloc, sample->cpu) ||
  149. insert_caller_stat(call_site, bytes_req, bytes_alloc))
  150. return -1;
  151. total_requested += bytes_req;
  152. total_allocated += bytes_alloc;
  153. nr_allocs++;
  154. return 0;
  155. }
  156. static int evsel__process_alloc_node_event(struct evsel *evsel, struct perf_sample *sample)
  157. {
  158. int ret = evsel__process_alloc_event(evsel, sample);
  159. if (!ret) {
  160. int node1 = cpu__get_node(sample->cpu),
  161. node2 = evsel__intval(evsel, sample, "node");
  162. if (node1 != node2)
  163. nr_cross_allocs++;
  164. }
  165. return ret;
  166. }
  167. static int ptr_cmp(void *, void *);
  168. static int slab_callsite_cmp(void *, void *);
  169. static struct alloc_stat *search_alloc_stat(unsigned long ptr,
  170. unsigned long call_site,
  171. struct rb_root *root,
  172. sort_fn_t sort_fn)
  173. {
  174. struct rb_node *node = root->rb_node;
  175. struct alloc_stat key = { .ptr = ptr, .call_site = call_site };
  176. while (node) {
  177. struct alloc_stat *data;
  178. int cmp;
  179. data = rb_entry(node, struct alloc_stat, node);
  180. cmp = sort_fn(&key, data);
  181. if (cmp < 0)
  182. node = node->rb_left;
  183. else if (cmp > 0)
  184. node = node->rb_right;
  185. else
  186. return data;
  187. }
  188. return NULL;
  189. }
  190. static int evsel__process_free_event(struct evsel *evsel, struct perf_sample *sample)
  191. {
  192. unsigned long ptr = evsel__intval(evsel, sample, "ptr");
  193. struct alloc_stat *s_alloc, *s_caller;
  194. s_alloc = search_alloc_stat(ptr, 0, &root_alloc_stat, ptr_cmp);
  195. if (!s_alloc)
  196. return 0;
  197. total_freed += s_alloc->last_alloc;
  198. if ((short)sample->cpu != s_alloc->alloc_cpu) {
  199. s_alloc->pingpong++;
  200. s_caller = search_alloc_stat(0, s_alloc->call_site,
  201. &root_caller_stat,
  202. slab_callsite_cmp);
  203. if (!s_caller)
  204. return -1;
  205. s_caller->pingpong++;
  206. }
  207. s_alloc->alloc_cpu = -1;
  208. return 0;
  209. }
  210. static u64 total_page_alloc_bytes;
  211. static u64 total_page_free_bytes;
  212. static u64 total_page_nomatch_bytes;
  213. static u64 total_page_fail_bytes;
  214. static unsigned long nr_page_allocs;
  215. static unsigned long nr_page_frees;
  216. static unsigned long nr_page_fails;
  217. static unsigned long nr_page_nomatch;
  218. static bool use_pfn;
  219. static bool live_page;
  220. static struct perf_session *kmem_session;
  221. #define MAX_MIGRATE_TYPES 6
  222. #define MAX_PAGE_ORDER 11
  223. static int order_stats[MAX_PAGE_ORDER][MAX_MIGRATE_TYPES];
  224. struct page_stat {
  225. struct rb_node node;
  226. u64 page;
  227. u64 callsite;
  228. int order;
  229. unsigned gfp_flags;
  230. unsigned migrate_type;
  231. u64 alloc_bytes;
  232. u64 free_bytes;
  233. int nr_alloc;
  234. int nr_free;
  235. };
  236. static struct rb_root page_live_tree;
  237. static struct rb_root page_alloc_tree;
  238. static struct rb_root page_alloc_sorted;
  239. static struct rb_root page_caller_tree;
  240. static struct rb_root page_caller_sorted;
  241. struct alloc_func {
  242. u64 start;
  243. u64 end;
  244. char *name;
  245. };
  246. static int nr_alloc_funcs;
  247. static struct alloc_func *alloc_func_list;
  248. static int funcmp(const void *a, const void *b)
  249. {
  250. const struct alloc_func *fa = a;
  251. const struct alloc_func *fb = b;
  252. if (fa->start > fb->start)
  253. return 1;
  254. else
  255. return -1;
  256. }
  257. static int callcmp(const void *a, const void *b)
  258. {
  259. const struct alloc_func *fa = a;
  260. const struct alloc_func *fb = b;
  261. if (fb->start <= fa->start && fa->end < fb->end)
  262. return 0;
  263. if (fa->start > fb->start)
  264. return 1;
  265. else
  266. return -1;
  267. }
  268. static int build_alloc_func_list(void)
  269. {
  270. int ret;
  271. struct map *kernel_map;
  272. struct symbol *sym;
  273. struct rb_node *node;
  274. struct alloc_func *func;
  275. struct machine *machine = &kmem_session->machines.host;
  276. regex_t alloc_func_regex;
  277. static const char pattern[] = "^_?_?(alloc|get_free|get_zeroed)_pages?";
  278. ret = regcomp(&alloc_func_regex, pattern, REG_EXTENDED);
  279. if (ret) {
  280. char err[BUFSIZ];
  281. regerror(ret, &alloc_func_regex, err, sizeof(err));
  282. pr_err("Invalid regex: %s\n%s", pattern, err);
  283. return -EINVAL;
  284. }
  285. kernel_map = machine__kernel_map(machine);
  286. if (map__load(kernel_map) < 0) {
  287. pr_err("cannot load kernel map\n");
  288. return -ENOENT;
  289. }
  290. map__for_each_symbol(kernel_map, sym, node) {
  291. if (regexec(&alloc_func_regex, sym->name, 0, NULL, 0))
  292. continue;
  293. func = realloc(alloc_func_list,
  294. (nr_alloc_funcs + 1) * sizeof(*func));
  295. if (func == NULL)
  296. return -ENOMEM;
  297. pr_debug("alloc func: %s\n", sym->name);
  298. func[nr_alloc_funcs].start = sym->start;
  299. func[nr_alloc_funcs].end = sym->end;
  300. func[nr_alloc_funcs].name = sym->name;
  301. alloc_func_list = func;
  302. nr_alloc_funcs++;
  303. }
  304. qsort(alloc_func_list, nr_alloc_funcs, sizeof(*func), funcmp);
  305. regfree(&alloc_func_regex);
  306. return 0;
  307. }
  308. /*
  309. * Find first non-memory allocation function from callchain.
  310. * The allocation functions are in the 'alloc_func_list'.
  311. */
  312. static u64 find_callsite(struct evsel *evsel, struct perf_sample *sample)
  313. {
  314. struct addr_location al;
  315. struct machine *machine = &kmem_session->machines.host;
  316. struct callchain_cursor_node *node;
  317. if (alloc_func_list == NULL) {
  318. if (build_alloc_func_list() < 0)
  319. goto out;
  320. }
  321. al.thread = machine__findnew_thread(machine, sample->pid, sample->tid);
  322. sample__resolve_callchain(sample, &callchain_cursor, NULL, evsel, &al, 16);
  323. callchain_cursor_commit(&callchain_cursor);
  324. while (true) {
  325. struct alloc_func key, *caller;
  326. u64 addr;
  327. node = callchain_cursor_current(&callchain_cursor);
  328. if (node == NULL)
  329. break;
  330. key.start = key.end = node->ip;
  331. caller = bsearch(&key, alloc_func_list, nr_alloc_funcs,
  332. sizeof(key), callcmp);
  333. if (!caller) {
  334. /* found */
  335. if (node->ms.map)
  336. addr = map__unmap_ip(node->ms.map, node->ip);
  337. else
  338. addr = node->ip;
  339. return addr;
  340. } else
  341. pr_debug3("skipping alloc function: %s\n", caller->name);
  342. callchain_cursor_advance(&callchain_cursor);
  343. }
  344. out:
  345. pr_debug2("unknown callsite: %"PRIx64 "\n", sample->ip);
  346. return sample->ip;
  347. }
  348. struct sort_dimension {
  349. const char name[20];
  350. sort_fn_t cmp;
  351. struct list_head list;
  352. };
  353. static LIST_HEAD(page_alloc_sort_input);
  354. static LIST_HEAD(page_caller_sort_input);
  355. static struct page_stat *
  356. __page_stat__findnew_page(struct page_stat *pstat, bool create)
  357. {
  358. struct rb_node **node = &page_live_tree.rb_node;
  359. struct rb_node *parent = NULL;
  360. struct page_stat *data;
  361. while (*node) {
  362. s64 cmp;
  363. parent = *node;
  364. data = rb_entry(*node, struct page_stat, node);
  365. cmp = data->page - pstat->page;
  366. if (cmp < 0)
  367. node = &parent->rb_left;
  368. else if (cmp > 0)
  369. node = &parent->rb_right;
  370. else
  371. return data;
  372. }
  373. if (!create)
  374. return NULL;
  375. data = zalloc(sizeof(*data));
  376. if (data != NULL) {
  377. data->page = pstat->page;
  378. data->order = pstat->order;
  379. data->gfp_flags = pstat->gfp_flags;
  380. data->migrate_type = pstat->migrate_type;
  381. rb_link_node(&data->node, parent, node);
  382. rb_insert_color(&data->node, &page_live_tree);
  383. }
  384. return data;
  385. }
  386. static struct page_stat *page_stat__find_page(struct page_stat *pstat)
  387. {
  388. return __page_stat__findnew_page(pstat, false);
  389. }
  390. static struct page_stat *page_stat__findnew_page(struct page_stat *pstat)
  391. {
  392. return __page_stat__findnew_page(pstat, true);
  393. }
  394. static struct page_stat *
  395. __page_stat__findnew_alloc(struct page_stat *pstat, bool create)
  396. {
  397. struct rb_node **node = &page_alloc_tree.rb_node;
  398. struct rb_node *parent = NULL;
  399. struct page_stat *data;
  400. struct sort_dimension *sort;
  401. while (*node) {
  402. int cmp = 0;
  403. parent = *node;
  404. data = rb_entry(*node, struct page_stat, node);
  405. list_for_each_entry(sort, &page_alloc_sort_input, list) {
  406. cmp = sort->cmp(pstat, data);
  407. if (cmp)
  408. break;
  409. }
  410. if (cmp < 0)
  411. node = &parent->rb_left;
  412. else if (cmp > 0)
  413. node = &parent->rb_right;
  414. else
  415. return data;
  416. }
  417. if (!create)
  418. return NULL;
  419. data = zalloc(sizeof(*data));
  420. if (data != NULL) {
  421. data->page = pstat->page;
  422. data->order = pstat->order;
  423. data->gfp_flags = pstat->gfp_flags;
  424. data->migrate_type = pstat->migrate_type;
  425. rb_link_node(&data->node, parent, node);
  426. rb_insert_color(&data->node, &page_alloc_tree);
  427. }
  428. return data;
  429. }
  430. static struct page_stat *page_stat__find_alloc(struct page_stat *pstat)
  431. {
  432. return __page_stat__findnew_alloc(pstat, false);
  433. }
  434. static struct page_stat *page_stat__findnew_alloc(struct page_stat *pstat)
  435. {
  436. return __page_stat__findnew_alloc(pstat, true);
  437. }
  438. static struct page_stat *
  439. __page_stat__findnew_caller(struct page_stat *pstat, bool create)
  440. {
  441. struct rb_node **node = &page_caller_tree.rb_node;
  442. struct rb_node *parent = NULL;
  443. struct page_stat *data;
  444. struct sort_dimension *sort;
  445. while (*node) {
  446. int cmp = 0;
  447. parent = *node;
  448. data = rb_entry(*node, struct page_stat, node);
  449. list_for_each_entry(sort, &page_caller_sort_input, list) {
  450. cmp = sort->cmp(pstat, data);
  451. if (cmp)
  452. break;
  453. }
  454. if (cmp < 0)
  455. node = &parent->rb_left;
  456. else if (cmp > 0)
  457. node = &parent->rb_right;
  458. else
  459. return data;
  460. }
  461. if (!create)
  462. return NULL;
  463. data = zalloc(sizeof(*data));
  464. if (data != NULL) {
  465. data->callsite = pstat->callsite;
  466. data->order = pstat->order;
  467. data->gfp_flags = pstat->gfp_flags;
  468. data->migrate_type = pstat->migrate_type;
  469. rb_link_node(&data->node, parent, node);
  470. rb_insert_color(&data->node, &page_caller_tree);
  471. }
  472. return data;
  473. }
  474. static struct page_stat *page_stat__find_caller(struct page_stat *pstat)
  475. {
  476. return __page_stat__findnew_caller(pstat, false);
  477. }
  478. static struct page_stat *page_stat__findnew_caller(struct page_stat *pstat)
  479. {
  480. return __page_stat__findnew_caller(pstat, true);
  481. }
  482. static bool valid_page(u64 pfn_or_page)
  483. {
  484. if (use_pfn && pfn_or_page == -1UL)
  485. return false;
  486. if (!use_pfn && pfn_or_page == 0)
  487. return false;
  488. return true;
  489. }
  490. struct gfp_flag {
  491. unsigned int flags;
  492. char *compact_str;
  493. char *human_readable;
  494. };
  495. static struct gfp_flag *gfps;
  496. static int nr_gfps;
  497. static int gfpcmp(const void *a, const void *b)
  498. {
  499. const struct gfp_flag *fa = a;
  500. const struct gfp_flag *fb = b;
  501. return fa->flags - fb->flags;
  502. }
  503. /* see include/trace/events/mmflags.h */
  504. static const struct {
  505. const char *original;
  506. const char *compact;
  507. } gfp_compact_table[] = {
  508. { "GFP_TRANSHUGE", "THP" },
  509. { "GFP_TRANSHUGE_LIGHT", "THL" },
  510. { "GFP_HIGHUSER_MOVABLE", "HUM" },
  511. { "GFP_HIGHUSER", "HU" },
  512. { "GFP_USER", "U" },
  513. { "GFP_KERNEL_ACCOUNT", "KAC" },
  514. { "GFP_KERNEL", "K" },
  515. { "GFP_NOFS", "NF" },
  516. { "GFP_ATOMIC", "A" },
  517. { "GFP_NOIO", "NI" },
  518. { "GFP_NOWAIT", "NW" },
  519. { "GFP_DMA", "D" },
  520. { "__GFP_HIGHMEM", "HM" },
  521. { "GFP_DMA32", "D32" },
  522. { "__GFP_HIGH", "H" },
  523. { "__GFP_ATOMIC", "_A" },
  524. { "__GFP_IO", "I" },
  525. { "__GFP_FS", "F" },
  526. { "__GFP_NOWARN", "NWR" },
  527. { "__GFP_RETRY_MAYFAIL", "R" },
  528. { "__GFP_NOFAIL", "NF" },
  529. { "__GFP_NORETRY", "NR" },
  530. { "__GFP_COMP", "C" },
  531. { "__GFP_ZERO", "Z" },
  532. { "__GFP_NOMEMALLOC", "NMA" },
  533. { "__GFP_MEMALLOC", "MA" },
  534. { "__GFP_HARDWALL", "HW" },
  535. { "__GFP_THISNODE", "TN" },
  536. { "__GFP_RECLAIMABLE", "RC" },
  537. { "__GFP_MOVABLE", "M" },
  538. { "__GFP_ACCOUNT", "AC" },
  539. { "__GFP_WRITE", "WR" },
  540. { "__GFP_RECLAIM", "R" },
  541. { "__GFP_DIRECT_RECLAIM", "DR" },
  542. { "__GFP_KSWAPD_RECLAIM", "KR" },
  543. };
  544. static size_t max_gfp_len;
  545. static char *compact_gfp_flags(char *gfp_flags)
  546. {
  547. char *orig_flags = strdup(gfp_flags);
  548. char *new_flags = NULL;
  549. char *str, *pos = NULL;
  550. size_t len = 0;
  551. if (orig_flags == NULL)
  552. return NULL;
  553. str = strtok_r(orig_flags, "|", &pos);
  554. while (str) {
  555. size_t i;
  556. char *new;
  557. const char *cpt;
  558. for (i = 0; i < ARRAY_SIZE(gfp_compact_table); i++) {
  559. if (strcmp(gfp_compact_table[i].original, str))
  560. continue;
  561. cpt = gfp_compact_table[i].compact;
  562. new = realloc(new_flags, len + strlen(cpt) + 2);
  563. if (new == NULL) {
  564. free(new_flags);
  565. free(orig_flags);
  566. return NULL;
  567. }
  568. new_flags = new;
  569. if (!len) {
  570. strcpy(new_flags, cpt);
  571. } else {
  572. strcat(new_flags, "|");
  573. strcat(new_flags, cpt);
  574. len++;
  575. }
  576. len += strlen(cpt);
  577. }
  578. str = strtok_r(NULL, "|", &pos);
  579. }
  580. if (max_gfp_len < len)
  581. max_gfp_len = len;
  582. free(orig_flags);
  583. return new_flags;
  584. }
  585. static char *compact_gfp_string(unsigned long gfp_flags)
  586. {
  587. struct gfp_flag key = {
  588. .flags = gfp_flags,
  589. };
  590. struct gfp_flag *gfp;
  591. gfp = bsearch(&key, gfps, nr_gfps, sizeof(*gfps), gfpcmp);
  592. if (gfp)
  593. return gfp->compact_str;
  594. return NULL;
  595. }
  596. static int parse_gfp_flags(struct evsel *evsel, struct perf_sample *sample,
  597. unsigned int gfp_flags)
  598. {
  599. struct tep_record record = {
  600. .cpu = sample->cpu,
  601. .data = sample->raw_data,
  602. .size = sample->raw_size,
  603. };
  604. struct trace_seq seq;
  605. char *str, *pos = NULL;
  606. if (nr_gfps) {
  607. struct gfp_flag key = {
  608. .flags = gfp_flags,
  609. };
  610. if (bsearch(&key, gfps, nr_gfps, sizeof(*gfps), gfpcmp))
  611. return 0;
  612. }
  613. trace_seq_init(&seq);
  614. tep_print_event(evsel->tp_format->tep,
  615. &seq, &record, "%s", TEP_PRINT_INFO);
  616. str = strtok_r(seq.buffer, " ", &pos);
  617. while (str) {
  618. if (!strncmp(str, "gfp_flags=", 10)) {
  619. struct gfp_flag *new;
  620. new = realloc(gfps, (nr_gfps + 1) * sizeof(*gfps));
  621. if (new == NULL)
  622. return -ENOMEM;
  623. gfps = new;
  624. new += nr_gfps++;
  625. new->flags = gfp_flags;
  626. new->human_readable = strdup(str + 10);
  627. new->compact_str = compact_gfp_flags(str + 10);
  628. if (!new->human_readable || !new->compact_str)
  629. return -ENOMEM;
  630. qsort(gfps, nr_gfps, sizeof(*gfps), gfpcmp);
  631. }
  632. str = strtok_r(NULL, " ", &pos);
  633. }
  634. trace_seq_destroy(&seq);
  635. return 0;
  636. }
  637. static int evsel__process_page_alloc_event(struct evsel *evsel, struct perf_sample *sample)
  638. {
  639. u64 page;
  640. unsigned int order = evsel__intval(evsel, sample, "order");
  641. unsigned int gfp_flags = evsel__intval(evsel, sample, "gfp_flags");
  642. unsigned int migrate_type = evsel__intval(evsel, sample,
  643. "migratetype");
  644. u64 bytes = kmem_page_size << order;
  645. u64 callsite;
  646. struct page_stat *pstat;
  647. struct page_stat this = {
  648. .order = order,
  649. .gfp_flags = gfp_flags,
  650. .migrate_type = migrate_type,
  651. };
  652. if (use_pfn)
  653. page = evsel__intval(evsel, sample, "pfn");
  654. else
  655. page = evsel__intval(evsel, sample, "page");
  656. nr_page_allocs++;
  657. total_page_alloc_bytes += bytes;
  658. if (!valid_page(page)) {
  659. nr_page_fails++;
  660. total_page_fail_bytes += bytes;
  661. return 0;
  662. }
  663. if (parse_gfp_flags(evsel, sample, gfp_flags) < 0)
  664. return -1;
  665. callsite = find_callsite(evsel, sample);
  666. /*
  667. * This is to find the current page (with correct gfp flags and
  668. * migrate type) at free event.
  669. */
  670. this.page = page;
  671. pstat = page_stat__findnew_page(&this);
  672. if (pstat == NULL)
  673. return -ENOMEM;
  674. pstat->nr_alloc++;
  675. pstat->alloc_bytes += bytes;
  676. pstat->callsite = callsite;
  677. if (!live_page) {
  678. pstat = page_stat__findnew_alloc(&this);
  679. if (pstat == NULL)
  680. return -ENOMEM;
  681. pstat->nr_alloc++;
  682. pstat->alloc_bytes += bytes;
  683. pstat->callsite = callsite;
  684. }
  685. this.callsite = callsite;
  686. pstat = page_stat__findnew_caller(&this);
  687. if (pstat == NULL)
  688. return -ENOMEM;
  689. pstat->nr_alloc++;
  690. pstat->alloc_bytes += bytes;
  691. order_stats[order][migrate_type]++;
  692. return 0;
  693. }
  694. static int evsel__process_page_free_event(struct evsel *evsel, struct perf_sample *sample)
  695. {
  696. u64 page;
  697. unsigned int order = evsel__intval(evsel, sample, "order");
  698. u64 bytes = kmem_page_size << order;
  699. struct page_stat *pstat;
  700. struct page_stat this = {
  701. .order = order,
  702. };
  703. if (use_pfn)
  704. page = evsel__intval(evsel, sample, "pfn");
  705. else
  706. page = evsel__intval(evsel, sample, "page");
  707. nr_page_frees++;
  708. total_page_free_bytes += bytes;
  709. this.page = page;
  710. pstat = page_stat__find_page(&this);
  711. if (pstat == NULL) {
  712. pr_debug2("missing free at page %"PRIx64" (order: %d)\n",
  713. page, order);
  714. nr_page_nomatch++;
  715. total_page_nomatch_bytes += bytes;
  716. return 0;
  717. }
  718. this.gfp_flags = pstat->gfp_flags;
  719. this.migrate_type = pstat->migrate_type;
  720. this.callsite = pstat->callsite;
  721. rb_erase(&pstat->node, &page_live_tree);
  722. free(pstat);
  723. if (live_page) {
  724. order_stats[this.order][this.migrate_type]--;
  725. } else {
  726. pstat = page_stat__find_alloc(&this);
  727. if (pstat == NULL)
  728. return -ENOMEM;
  729. pstat->nr_free++;
  730. pstat->free_bytes += bytes;
  731. }
  732. pstat = page_stat__find_caller(&this);
  733. if (pstat == NULL)
  734. return -ENOENT;
  735. pstat->nr_free++;
  736. pstat->free_bytes += bytes;
  737. if (live_page) {
  738. pstat->nr_alloc--;
  739. pstat->alloc_bytes -= bytes;
  740. if (pstat->nr_alloc == 0) {
  741. rb_erase(&pstat->node, &page_caller_tree);
  742. free(pstat);
  743. }
  744. }
  745. return 0;
  746. }
  747. static bool perf_kmem__skip_sample(struct perf_sample *sample)
  748. {
  749. /* skip sample based on time? */
  750. if (perf_time__skip_sample(&ptime, sample->time))
  751. return true;
  752. return false;
  753. }
  754. typedef int (*tracepoint_handler)(struct evsel *evsel,
  755. struct perf_sample *sample);
  756. static int process_sample_event(struct perf_tool *tool __maybe_unused,
  757. union perf_event *event,
  758. struct perf_sample *sample,
  759. struct evsel *evsel,
  760. struct machine *machine)
  761. {
  762. int err = 0;
  763. struct thread *thread = machine__findnew_thread(machine, sample->pid,
  764. sample->tid);
  765. if (thread == NULL) {
  766. pr_debug("problem processing %d event, skipping it.\n",
  767. event->header.type);
  768. return -1;
  769. }
  770. if (perf_kmem__skip_sample(sample))
  771. return 0;
  772. dump_printf(" ... thread: %s:%d\n", thread__comm_str(thread), thread->tid);
  773. if (evsel->handler != NULL) {
  774. tracepoint_handler f = evsel->handler;
  775. err = f(evsel, sample);
  776. }
  777. thread__put(thread);
  778. return err;
  779. }
  780. static struct perf_tool perf_kmem = {
  781. .sample = process_sample_event,
  782. .comm = perf_event__process_comm,
  783. .mmap = perf_event__process_mmap,
  784. .mmap2 = perf_event__process_mmap2,
  785. .namespaces = perf_event__process_namespaces,
  786. .ordered_events = true,
  787. };
  788. static double fragmentation(unsigned long n_req, unsigned long n_alloc)
  789. {
  790. if (n_alloc == 0)
  791. return 0.0;
  792. else
  793. return 100.0 - (100.0 * n_req / n_alloc);
  794. }
  795. static void __print_slab_result(struct rb_root *root,
  796. struct perf_session *session,
  797. int n_lines, int is_caller)
  798. {
  799. struct rb_node *next;
  800. struct machine *machine = &session->machines.host;
  801. printf("%.105s\n", graph_dotted_line);
  802. printf(" %-34s |", is_caller ? "Callsite": "Alloc Ptr");
  803. printf(" Total_alloc/Per | Total_req/Per | Hit | Ping-pong | Frag\n");
  804. printf("%.105s\n", graph_dotted_line);
  805. next = rb_first(root);
  806. while (next && n_lines--) {
  807. struct alloc_stat *data = rb_entry(next, struct alloc_stat,
  808. node);
  809. struct symbol *sym = NULL;
  810. struct map *map;
  811. char buf[BUFSIZ];
  812. u64 addr;
  813. if (is_caller) {
  814. addr = data->call_site;
  815. if (!raw_ip)
  816. sym = machine__find_kernel_symbol(machine, addr, &map);
  817. } else
  818. addr = data->ptr;
  819. if (sym != NULL)
  820. snprintf(buf, sizeof(buf), "%s+%" PRIx64 "", sym->name,
  821. addr - map->unmap_ip(map, sym->start));
  822. else
  823. snprintf(buf, sizeof(buf), "%#" PRIx64 "", addr);
  824. printf(" %-34s |", buf);
  825. printf(" %9llu/%-5lu | %9llu/%-5lu | %8lu | %9lu | %6.3f%%\n",
  826. (unsigned long long)data->bytes_alloc,
  827. (unsigned long)data->bytes_alloc / data->hit,
  828. (unsigned long long)data->bytes_req,
  829. (unsigned long)data->bytes_req / data->hit,
  830. (unsigned long)data->hit,
  831. (unsigned long)data->pingpong,
  832. fragmentation(data->bytes_req, data->bytes_alloc));
  833. next = rb_next(next);
  834. }
  835. if (n_lines == -1)
  836. printf(" ... | ... | ... | ... | ... | ... \n");
  837. printf("%.105s\n", graph_dotted_line);
  838. }
  839. static const char * const migrate_type_str[] = {
  840. "UNMOVABL",
  841. "RECLAIM",
  842. "MOVABLE",
  843. "RESERVED",
  844. "CMA/ISLT",
  845. "UNKNOWN",
  846. };
  847. static void __print_page_alloc_result(struct perf_session *session, int n_lines)
  848. {
  849. struct rb_node *next = rb_first(&page_alloc_sorted);
  850. struct machine *machine = &session->machines.host;
  851. const char *format;
  852. int gfp_len = max(strlen("GFP flags"), max_gfp_len);
  853. printf("\n%.105s\n", graph_dotted_line);
  854. printf(" %-16s | %5s alloc (KB) | Hits | Order | Mig.type | %-*s | Callsite\n",
  855. use_pfn ? "PFN" : "Page", live_page ? "Live" : "Total",
  856. gfp_len, "GFP flags");
  857. printf("%.105s\n", graph_dotted_line);
  858. if (use_pfn)
  859. format = " %16llu | %'16llu | %'9d | %5d | %8s | %-*s | %s\n";
  860. else
  861. format = " %016llx | %'16llu | %'9d | %5d | %8s | %-*s | %s\n";
  862. while (next && n_lines--) {
  863. struct page_stat *data;
  864. struct symbol *sym;
  865. struct map *map;
  866. char buf[32];
  867. char *caller = buf;
  868. data = rb_entry(next, struct page_stat, node);
  869. sym = machine__find_kernel_symbol(machine, data->callsite, &map);
  870. if (sym)
  871. caller = sym->name;
  872. else
  873. scnprintf(buf, sizeof(buf), "%"PRIx64, data->callsite);
  874. printf(format, (unsigned long long)data->page,
  875. (unsigned long long)data->alloc_bytes / 1024,
  876. data->nr_alloc, data->order,
  877. migrate_type_str[data->migrate_type],
  878. gfp_len, compact_gfp_string(data->gfp_flags), caller);
  879. next = rb_next(next);
  880. }
  881. if (n_lines == -1) {
  882. printf(" ... | ... | ... | ... | ... | %-*s | ...\n",
  883. gfp_len, "...");
  884. }
  885. printf("%.105s\n", graph_dotted_line);
  886. }
  887. static void __print_page_caller_result(struct perf_session *session, int n_lines)
  888. {
  889. struct rb_node *next = rb_first(&page_caller_sorted);
  890. struct machine *machine = &session->machines.host;
  891. int gfp_len = max(strlen("GFP flags"), max_gfp_len);
  892. printf("\n%.105s\n", graph_dotted_line);
  893. printf(" %5s alloc (KB) | Hits | Order | Mig.type | %-*s | Callsite\n",
  894. live_page ? "Live" : "Total", gfp_len, "GFP flags");
  895. printf("%.105s\n", graph_dotted_line);
  896. while (next && n_lines--) {
  897. struct page_stat *data;
  898. struct symbol *sym;
  899. struct map *map;
  900. char buf[32];
  901. char *caller = buf;
  902. data = rb_entry(next, struct page_stat, node);
  903. sym = machine__find_kernel_symbol(machine, data->callsite, &map);
  904. if (sym)
  905. caller = sym->name;
  906. else
  907. scnprintf(buf, sizeof(buf), "%"PRIx64, data->callsite);
  908. printf(" %'16llu | %'9d | %5d | %8s | %-*s | %s\n",
  909. (unsigned long long)data->alloc_bytes / 1024,
  910. data->nr_alloc, data->order,
  911. migrate_type_str[data->migrate_type],
  912. gfp_len, compact_gfp_string(data->gfp_flags), caller);
  913. next = rb_next(next);
  914. }
  915. if (n_lines == -1) {
  916. printf(" ... | ... | ... | ... | %-*s | ...\n",
  917. gfp_len, "...");
  918. }
  919. printf("%.105s\n", graph_dotted_line);
  920. }
  921. static void print_gfp_flags(void)
  922. {
  923. int i;
  924. printf("#\n");
  925. printf("# GFP flags\n");
  926. printf("# ---------\n");
  927. for (i = 0; i < nr_gfps; i++) {
  928. printf("# %08x: %*s: %s\n", gfps[i].flags,
  929. (int) max_gfp_len, gfps[i].compact_str,
  930. gfps[i].human_readable);
  931. }
  932. }
  933. static void print_slab_summary(void)
  934. {
  935. printf("\nSUMMARY (SLAB allocator)");
  936. printf("\n========================\n");
  937. printf("Total bytes requested: %'lu\n", total_requested);
  938. printf("Total bytes allocated: %'lu\n", total_allocated);
  939. printf("Total bytes freed: %'lu\n", total_freed);
  940. if (total_allocated > total_freed) {
  941. printf("Net total bytes allocated: %'lu\n",
  942. total_allocated - total_freed);
  943. }
  944. printf("Total bytes wasted on internal fragmentation: %'lu\n",
  945. total_allocated - total_requested);
  946. printf("Internal fragmentation: %f%%\n",
  947. fragmentation(total_requested, total_allocated));
  948. printf("Cross CPU allocations: %'lu/%'lu\n", nr_cross_allocs, nr_allocs);
  949. }
  950. static void print_page_summary(void)
  951. {
  952. int o, m;
  953. u64 nr_alloc_freed = nr_page_frees - nr_page_nomatch;
  954. u64 total_alloc_freed_bytes = total_page_free_bytes - total_page_nomatch_bytes;
  955. printf("\nSUMMARY (page allocator)");
  956. printf("\n========================\n");
  957. printf("%-30s: %'16lu [ %'16"PRIu64" KB ]\n", "Total allocation requests",
  958. nr_page_allocs, total_page_alloc_bytes / 1024);
  959. printf("%-30s: %'16lu [ %'16"PRIu64" KB ]\n", "Total free requests",
  960. nr_page_frees, total_page_free_bytes / 1024);
  961. printf("\n");
  962. printf("%-30s: %'16"PRIu64" [ %'16"PRIu64" KB ]\n", "Total alloc+freed requests",
  963. nr_alloc_freed, (total_alloc_freed_bytes) / 1024);
  964. printf("%-30s: %'16"PRIu64" [ %'16"PRIu64" KB ]\n", "Total alloc-only requests",
  965. nr_page_allocs - nr_alloc_freed,
  966. (total_page_alloc_bytes - total_alloc_freed_bytes) / 1024);
  967. printf("%-30s: %'16lu [ %'16"PRIu64" KB ]\n", "Total free-only requests",
  968. nr_page_nomatch, total_page_nomatch_bytes / 1024);
  969. printf("\n");
  970. printf("%-30s: %'16lu [ %'16"PRIu64" KB ]\n", "Total allocation failures",
  971. nr_page_fails, total_page_fail_bytes / 1024);
  972. printf("\n");
  973. printf("%5s %12s %12s %12s %12s %12s\n", "Order", "Unmovable",
  974. "Reclaimable", "Movable", "Reserved", "CMA/Isolated");
  975. printf("%.5s %.12s %.12s %.12s %.12s %.12s\n", graph_dotted_line,
  976. graph_dotted_line, graph_dotted_line, graph_dotted_line,
  977. graph_dotted_line, graph_dotted_line);
  978. for (o = 0; o < MAX_PAGE_ORDER; o++) {
  979. printf("%5d", o);
  980. for (m = 0; m < MAX_MIGRATE_TYPES - 1; m++) {
  981. if (order_stats[o][m])
  982. printf(" %'12d", order_stats[o][m]);
  983. else
  984. printf(" %12c", '.');
  985. }
  986. printf("\n");
  987. }
  988. }
  989. static void print_slab_result(struct perf_session *session)
  990. {
  991. if (caller_flag)
  992. __print_slab_result(&root_caller_sorted, session, caller_lines, 1);
  993. if (alloc_flag)
  994. __print_slab_result(&root_alloc_sorted, session, alloc_lines, 0);
  995. print_slab_summary();
  996. }
  997. static void print_page_result(struct perf_session *session)
  998. {
  999. if (caller_flag || alloc_flag)
  1000. print_gfp_flags();
  1001. if (caller_flag)
  1002. __print_page_caller_result(session, caller_lines);
  1003. if (alloc_flag)
  1004. __print_page_alloc_result(session, alloc_lines);
  1005. print_page_summary();
  1006. }
  1007. static void print_result(struct perf_session *session)
  1008. {
  1009. if (kmem_slab)
  1010. print_slab_result(session);
  1011. if (kmem_page)
  1012. print_page_result(session);
  1013. }
  1014. static LIST_HEAD(slab_caller_sort);
  1015. static LIST_HEAD(slab_alloc_sort);
  1016. static LIST_HEAD(page_caller_sort);
  1017. static LIST_HEAD(page_alloc_sort);
  1018. static void sort_slab_insert(struct rb_root *root, struct alloc_stat *data,
  1019. struct list_head *sort_list)
  1020. {
  1021. struct rb_node **new = &(root->rb_node);
  1022. struct rb_node *parent = NULL;
  1023. struct sort_dimension *sort;
  1024. while (*new) {
  1025. struct alloc_stat *this;
  1026. int cmp = 0;
  1027. this = rb_entry(*new, struct alloc_stat, node);
  1028. parent = *new;
  1029. list_for_each_entry(sort, sort_list, list) {
  1030. cmp = sort->cmp(data, this);
  1031. if (cmp)
  1032. break;
  1033. }
  1034. if (cmp > 0)
  1035. new = &((*new)->rb_left);
  1036. else
  1037. new = &((*new)->rb_right);
  1038. }
  1039. rb_link_node(&data->node, parent, new);
  1040. rb_insert_color(&data->node, root);
  1041. }
  1042. static void __sort_slab_result(struct rb_root *root, struct rb_root *root_sorted,
  1043. struct list_head *sort_list)
  1044. {
  1045. struct rb_node *node;
  1046. struct alloc_stat *data;
  1047. for (;;) {
  1048. node = rb_first(root);
  1049. if (!node)
  1050. break;
  1051. rb_erase(node, root);
  1052. data = rb_entry(node, struct alloc_stat, node);
  1053. sort_slab_insert(root_sorted, data, sort_list);
  1054. }
  1055. }
  1056. static void sort_page_insert(struct rb_root *root, struct page_stat *data,
  1057. struct list_head *sort_list)
  1058. {
  1059. struct rb_node **new = &root->rb_node;
  1060. struct rb_node *parent = NULL;
  1061. struct sort_dimension *sort;
  1062. while (*new) {
  1063. struct page_stat *this;
  1064. int cmp = 0;
  1065. this = rb_entry(*new, struct page_stat, node);
  1066. parent = *new;
  1067. list_for_each_entry(sort, sort_list, list) {
  1068. cmp = sort->cmp(data, this);
  1069. if (cmp)
  1070. break;
  1071. }
  1072. if (cmp > 0)
  1073. new = &parent->rb_left;
  1074. else
  1075. new = &parent->rb_right;
  1076. }
  1077. rb_link_node(&data->node, parent, new);
  1078. rb_insert_color(&data->node, root);
  1079. }
  1080. static void __sort_page_result(struct rb_root *root, struct rb_root *root_sorted,
  1081. struct list_head *sort_list)
  1082. {
  1083. struct rb_node *node;
  1084. struct page_stat *data;
  1085. for (;;) {
  1086. node = rb_first(root);
  1087. if (!node)
  1088. break;
  1089. rb_erase(node, root);
  1090. data = rb_entry(node, struct page_stat, node);
  1091. sort_page_insert(root_sorted, data, sort_list);
  1092. }
  1093. }
  1094. static void sort_result(void)
  1095. {
  1096. if (kmem_slab) {
  1097. __sort_slab_result(&root_alloc_stat, &root_alloc_sorted,
  1098. &slab_alloc_sort);
  1099. __sort_slab_result(&root_caller_stat, &root_caller_sorted,
  1100. &slab_caller_sort);
  1101. }
  1102. if (kmem_page) {
  1103. if (live_page)
  1104. __sort_page_result(&page_live_tree, &page_alloc_sorted,
  1105. &page_alloc_sort);
  1106. else
  1107. __sort_page_result(&page_alloc_tree, &page_alloc_sorted,
  1108. &page_alloc_sort);
  1109. __sort_page_result(&page_caller_tree, &page_caller_sorted,
  1110. &page_caller_sort);
  1111. }
  1112. }
  1113. static int __cmd_kmem(struct perf_session *session)
  1114. {
  1115. int err = -EINVAL;
  1116. struct evsel *evsel;
  1117. const struct evsel_str_handler kmem_tracepoints[] = {
  1118. /* slab allocator */
  1119. { "kmem:kmalloc", evsel__process_alloc_event, },
  1120. { "kmem:kmem_cache_alloc", evsel__process_alloc_event, },
  1121. { "kmem:kmalloc_node", evsel__process_alloc_node_event, },
  1122. { "kmem:kmem_cache_alloc_node", evsel__process_alloc_node_event, },
  1123. { "kmem:kfree", evsel__process_free_event, },
  1124. { "kmem:kmem_cache_free", evsel__process_free_event, },
  1125. /* page allocator */
  1126. { "kmem:mm_page_alloc", evsel__process_page_alloc_event, },
  1127. { "kmem:mm_page_free", evsel__process_page_free_event, },
  1128. };
  1129. if (!perf_session__has_traces(session, "kmem record"))
  1130. goto out;
  1131. if (perf_session__set_tracepoints_handlers(session, kmem_tracepoints)) {
  1132. pr_err("Initializing perf session tracepoint handlers failed\n");
  1133. goto out;
  1134. }
  1135. evlist__for_each_entry(session->evlist, evsel) {
  1136. if (!strcmp(evsel__name(evsel), "kmem:mm_page_alloc") &&
  1137. evsel__field(evsel, "pfn")) {
  1138. use_pfn = true;
  1139. break;
  1140. }
  1141. }
  1142. setup_pager();
  1143. err = perf_session__process_events(session);
  1144. if (err != 0) {
  1145. pr_err("error during process events: %d\n", err);
  1146. goto out;
  1147. }
  1148. sort_result();
  1149. print_result(session);
  1150. out:
  1151. return err;
  1152. }
  1153. /* slab sort keys */
  1154. static int ptr_cmp(void *a, void *b)
  1155. {
  1156. struct alloc_stat *l = a;
  1157. struct alloc_stat *r = b;
  1158. if (l->ptr < r->ptr)
  1159. return -1;
  1160. else if (l->ptr > r->ptr)
  1161. return 1;
  1162. return 0;
  1163. }
  1164. static struct sort_dimension ptr_sort_dimension = {
  1165. .name = "ptr",
  1166. .cmp = ptr_cmp,
  1167. };
  1168. static int slab_callsite_cmp(void *a, void *b)
  1169. {
  1170. struct alloc_stat *l = a;
  1171. struct alloc_stat *r = b;
  1172. if (l->call_site < r->call_site)
  1173. return -1;
  1174. else if (l->call_site > r->call_site)
  1175. return 1;
  1176. return 0;
  1177. }
  1178. static struct sort_dimension callsite_sort_dimension = {
  1179. .name = "callsite",
  1180. .cmp = slab_callsite_cmp,
  1181. };
  1182. static int hit_cmp(void *a, void *b)
  1183. {
  1184. struct alloc_stat *l = a;
  1185. struct alloc_stat *r = b;
  1186. if (l->hit < r->hit)
  1187. return -1;
  1188. else if (l->hit > r->hit)
  1189. return 1;
  1190. return 0;
  1191. }
  1192. static struct sort_dimension hit_sort_dimension = {
  1193. .name = "hit",
  1194. .cmp = hit_cmp,
  1195. };
  1196. static int bytes_cmp(void *a, void *b)
  1197. {
  1198. struct alloc_stat *l = a;
  1199. struct alloc_stat *r = b;
  1200. if (l->bytes_alloc < r->bytes_alloc)
  1201. return -1;
  1202. else if (l->bytes_alloc > r->bytes_alloc)
  1203. return 1;
  1204. return 0;
  1205. }
  1206. static struct sort_dimension bytes_sort_dimension = {
  1207. .name = "bytes",
  1208. .cmp = bytes_cmp,
  1209. };
  1210. static int frag_cmp(void *a, void *b)
  1211. {
  1212. double x, y;
  1213. struct alloc_stat *l = a;
  1214. struct alloc_stat *r = b;
  1215. x = fragmentation(l->bytes_req, l->bytes_alloc);
  1216. y = fragmentation(r->bytes_req, r->bytes_alloc);
  1217. if (x < y)
  1218. return -1;
  1219. else if (x > y)
  1220. return 1;
  1221. return 0;
  1222. }
  1223. static struct sort_dimension frag_sort_dimension = {
  1224. .name = "frag",
  1225. .cmp = frag_cmp,
  1226. };
  1227. static int pingpong_cmp(void *a, void *b)
  1228. {
  1229. struct alloc_stat *l = a;
  1230. struct alloc_stat *r = b;
  1231. if (l->pingpong < r->pingpong)
  1232. return -1;
  1233. else if (l->pingpong > r->pingpong)
  1234. return 1;
  1235. return 0;
  1236. }
  1237. static struct sort_dimension pingpong_sort_dimension = {
  1238. .name = "pingpong",
  1239. .cmp = pingpong_cmp,
  1240. };
  1241. /* page sort keys */
  1242. static int page_cmp(void *a, void *b)
  1243. {
  1244. struct page_stat *l = a;
  1245. struct page_stat *r = b;
  1246. if (l->page < r->page)
  1247. return -1;
  1248. else if (l->page > r->page)
  1249. return 1;
  1250. return 0;
  1251. }
  1252. static struct sort_dimension page_sort_dimension = {
  1253. .name = "page",
  1254. .cmp = page_cmp,
  1255. };
  1256. static int page_callsite_cmp(void *a, void *b)
  1257. {
  1258. struct page_stat *l = a;
  1259. struct page_stat *r = b;
  1260. if (l->callsite < r->callsite)
  1261. return -1;
  1262. else if (l->callsite > r->callsite)
  1263. return 1;
  1264. return 0;
  1265. }
  1266. static struct sort_dimension page_callsite_sort_dimension = {
  1267. .name = "callsite",
  1268. .cmp = page_callsite_cmp,
  1269. };
  1270. static int page_hit_cmp(void *a, void *b)
  1271. {
  1272. struct page_stat *l = a;
  1273. struct page_stat *r = b;
  1274. if (l->nr_alloc < r->nr_alloc)
  1275. return -1;
  1276. else if (l->nr_alloc > r->nr_alloc)
  1277. return 1;
  1278. return 0;
  1279. }
  1280. static struct sort_dimension page_hit_sort_dimension = {
  1281. .name = "hit",
  1282. .cmp = page_hit_cmp,
  1283. };
  1284. static int page_bytes_cmp(void *a, void *b)
  1285. {
  1286. struct page_stat *l = a;
  1287. struct page_stat *r = b;
  1288. if (l->alloc_bytes < r->alloc_bytes)
  1289. return -1;
  1290. else if (l->alloc_bytes > r->alloc_bytes)
  1291. return 1;
  1292. return 0;
  1293. }
  1294. static struct sort_dimension page_bytes_sort_dimension = {
  1295. .name = "bytes",
  1296. .cmp = page_bytes_cmp,
  1297. };
  1298. static int page_order_cmp(void *a, void *b)
  1299. {
  1300. struct page_stat *l = a;
  1301. struct page_stat *r = b;
  1302. if (l->order < r->order)
  1303. return -1;
  1304. else if (l->order > r->order)
  1305. return 1;
  1306. return 0;
  1307. }
  1308. static struct sort_dimension page_order_sort_dimension = {
  1309. .name = "order",
  1310. .cmp = page_order_cmp,
  1311. };
  1312. static int migrate_type_cmp(void *a, void *b)
  1313. {
  1314. struct page_stat *l = a;
  1315. struct page_stat *r = b;
  1316. /* for internal use to find free'd page */
  1317. if (l->migrate_type == -1U)
  1318. return 0;
  1319. if (l->migrate_type < r->migrate_type)
  1320. return -1;
  1321. else if (l->migrate_type > r->migrate_type)
  1322. return 1;
  1323. return 0;
  1324. }
  1325. static struct sort_dimension migrate_type_sort_dimension = {
  1326. .name = "migtype",
  1327. .cmp = migrate_type_cmp,
  1328. };
  1329. static int gfp_flags_cmp(void *a, void *b)
  1330. {
  1331. struct page_stat *l = a;
  1332. struct page_stat *r = b;
  1333. /* for internal use to find free'd page */
  1334. if (l->gfp_flags == -1U)
  1335. return 0;
  1336. if (l->gfp_flags < r->gfp_flags)
  1337. return -1;
  1338. else if (l->gfp_flags > r->gfp_flags)
  1339. return 1;
  1340. return 0;
  1341. }
  1342. static struct sort_dimension gfp_flags_sort_dimension = {
  1343. .name = "gfp",
  1344. .cmp = gfp_flags_cmp,
  1345. };
  1346. static struct sort_dimension *slab_sorts[] = {
  1347. &ptr_sort_dimension,
  1348. &callsite_sort_dimension,
  1349. &hit_sort_dimension,
  1350. &bytes_sort_dimension,
  1351. &frag_sort_dimension,
  1352. &pingpong_sort_dimension,
  1353. };
  1354. static struct sort_dimension *page_sorts[] = {
  1355. &page_sort_dimension,
  1356. &page_callsite_sort_dimension,
  1357. &page_hit_sort_dimension,
  1358. &page_bytes_sort_dimension,
  1359. &page_order_sort_dimension,
  1360. &migrate_type_sort_dimension,
  1361. &gfp_flags_sort_dimension,
  1362. };
  1363. static int slab_sort_dimension__add(const char *tok, struct list_head *list)
  1364. {
  1365. struct sort_dimension *sort;
  1366. int i;
  1367. for (i = 0; i < (int)ARRAY_SIZE(slab_sorts); i++) {
  1368. if (!strcmp(slab_sorts[i]->name, tok)) {
  1369. sort = memdup(slab_sorts[i], sizeof(*slab_sorts[i]));
  1370. if (!sort) {
  1371. pr_err("%s: memdup failed\n", __func__);
  1372. return -1;
  1373. }
  1374. list_add_tail(&sort->list, list);
  1375. return 0;
  1376. }
  1377. }
  1378. return -1;
  1379. }
  1380. static int page_sort_dimension__add(const char *tok, struct list_head *list)
  1381. {
  1382. struct sort_dimension *sort;
  1383. int i;
  1384. for (i = 0; i < (int)ARRAY_SIZE(page_sorts); i++) {
  1385. if (!strcmp(page_sorts[i]->name, tok)) {
  1386. sort = memdup(page_sorts[i], sizeof(*page_sorts[i]));
  1387. if (!sort) {
  1388. pr_err("%s: memdup failed\n", __func__);
  1389. return -1;
  1390. }
  1391. list_add_tail(&sort->list, list);
  1392. return 0;
  1393. }
  1394. }
  1395. return -1;
  1396. }
  1397. static int setup_slab_sorting(struct list_head *sort_list, const char *arg)
  1398. {
  1399. char *tok;
  1400. char *str = strdup(arg);
  1401. char *pos = str;
  1402. if (!str) {
  1403. pr_err("%s: strdup failed\n", __func__);
  1404. return -1;
  1405. }
  1406. while (true) {
  1407. tok = strsep(&pos, ",");
  1408. if (!tok)
  1409. break;
  1410. if (slab_sort_dimension__add(tok, sort_list) < 0) {
  1411. pr_err("Unknown slab --sort key: '%s'", tok);
  1412. free(str);
  1413. return -1;
  1414. }
  1415. }
  1416. free(str);
  1417. return 0;
  1418. }
  1419. static int setup_page_sorting(struct list_head *sort_list, const char *arg)
  1420. {
  1421. char *tok;
  1422. char *str = strdup(arg);
  1423. char *pos = str;
  1424. if (!str) {
  1425. pr_err("%s: strdup failed\n", __func__);
  1426. return -1;
  1427. }
  1428. while (true) {
  1429. tok = strsep(&pos, ",");
  1430. if (!tok)
  1431. break;
  1432. if (page_sort_dimension__add(tok, sort_list) < 0) {
  1433. pr_err("Unknown page --sort key: '%s'", tok);
  1434. free(str);
  1435. return -1;
  1436. }
  1437. }
  1438. free(str);
  1439. return 0;
  1440. }
  1441. static int parse_sort_opt(const struct option *opt __maybe_unused,
  1442. const char *arg, int unset __maybe_unused)
  1443. {
  1444. if (!arg)
  1445. return -1;
  1446. if (kmem_page > kmem_slab ||
  1447. (kmem_page == 0 && kmem_slab == 0 && kmem_default == KMEM_PAGE)) {
  1448. if (caller_flag > alloc_flag)
  1449. return setup_page_sorting(&page_caller_sort, arg);
  1450. else
  1451. return setup_page_sorting(&page_alloc_sort, arg);
  1452. } else {
  1453. if (caller_flag > alloc_flag)
  1454. return setup_slab_sorting(&slab_caller_sort, arg);
  1455. else
  1456. return setup_slab_sorting(&slab_alloc_sort, arg);
  1457. }
  1458. return 0;
  1459. }
  1460. static int parse_caller_opt(const struct option *opt __maybe_unused,
  1461. const char *arg __maybe_unused,
  1462. int unset __maybe_unused)
  1463. {
  1464. caller_flag = (alloc_flag + 1);
  1465. return 0;
  1466. }
  1467. static int parse_alloc_opt(const struct option *opt __maybe_unused,
  1468. const char *arg __maybe_unused,
  1469. int unset __maybe_unused)
  1470. {
  1471. alloc_flag = (caller_flag + 1);
  1472. return 0;
  1473. }
  1474. static int parse_slab_opt(const struct option *opt __maybe_unused,
  1475. const char *arg __maybe_unused,
  1476. int unset __maybe_unused)
  1477. {
  1478. kmem_slab = (kmem_page + 1);
  1479. return 0;
  1480. }
  1481. static int parse_page_opt(const struct option *opt __maybe_unused,
  1482. const char *arg __maybe_unused,
  1483. int unset __maybe_unused)
  1484. {
  1485. kmem_page = (kmem_slab + 1);
  1486. return 0;
  1487. }
  1488. static int parse_line_opt(const struct option *opt __maybe_unused,
  1489. const char *arg, int unset __maybe_unused)
  1490. {
  1491. int lines;
  1492. if (!arg)
  1493. return -1;
  1494. lines = strtoul(arg, NULL, 10);
  1495. if (caller_flag > alloc_flag)
  1496. caller_lines = lines;
  1497. else
  1498. alloc_lines = lines;
  1499. return 0;
  1500. }
  1501. static int __cmd_record(int argc, const char **argv)
  1502. {
  1503. const char * const record_args[] = {
  1504. "record", "-a", "-R", "-c", "1",
  1505. };
  1506. const char * const slab_events[] = {
  1507. "-e", "kmem:kmalloc",
  1508. "-e", "kmem:kmalloc_node",
  1509. "-e", "kmem:kfree",
  1510. "-e", "kmem:kmem_cache_alloc",
  1511. "-e", "kmem:kmem_cache_alloc_node",
  1512. "-e", "kmem:kmem_cache_free",
  1513. };
  1514. const char * const page_events[] = {
  1515. "-e", "kmem:mm_page_alloc",
  1516. "-e", "kmem:mm_page_free",
  1517. };
  1518. unsigned int rec_argc, i, j;
  1519. const char **rec_argv;
  1520. rec_argc = ARRAY_SIZE(record_args) + argc - 1;
  1521. if (kmem_slab)
  1522. rec_argc += ARRAY_SIZE(slab_events);
  1523. if (kmem_page)
  1524. rec_argc += ARRAY_SIZE(page_events) + 1; /* for -g */
  1525. rec_argv = calloc(rec_argc + 1, sizeof(char *));
  1526. if (rec_argv == NULL)
  1527. return -ENOMEM;
  1528. for (i = 0; i < ARRAY_SIZE(record_args); i++)
  1529. rec_argv[i] = strdup(record_args[i]);
  1530. if (kmem_slab) {
  1531. for (j = 0; j < ARRAY_SIZE(slab_events); j++, i++)
  1532. rec_argv[i] = strdup(slab_events[j]);
  1533. }
  1534. if (kmem_page) {
  1535. rec_argv[i++] = strdup("-g");
  1536. for (j = 0; j < ARRAY_SIZE(page_events); j++, i++)
  1537. rec_argv[i] = strdup(page_events[j]);
  1538. }
  1539. for (j = 1; j < (unsigned int)argc; j++, i++)
  1540. rec_argv[i] = argv[j];
  1541. return cmd_record(i, rec_argv);
  1542. }
  1543. static int kmem_config(const char *var, const char *value, void *cb __maybe_unused)
  1544. {
  1545. if (!strcmp(var, "kmem.default")) {
  1546. if (!strcmp(value, "slab"))
  1547. kmem_default = KMEM_SLAB;
  1548. else if (!strcmp(value, "page"))
  1549. kmem_default = KMEM_PAGE;
  1550. else
  1551. pr_err("invalid default value ('slab' or 'page' required): %s\n",
  1552. value);
  1553. return 0;
  1554. }
  1555. return 0;
  1556. }
  1557. int cmd_kmem(int argc, const char **argv)
  1558. {
  1559. const char * const default_slab_sort = "frag,hit,bytes";
  1560. const char * const default_page_sort = "bytes,hit";
  1561. struct perf_data data = {
  1562. .mode = PERF_DATA_MODE_READ,
  1563. };
  1564. const struct option kmem_options[] = {
  1565. OPT_STRING('i', "input", &input_name, "file", "input file name"),
  1566. OPT_INCR('v', "verbose", &verbose,
  1567. "be more verbose (show symbol address, etc)"),
  1568. OPT_CALLBACK_NOOPT(0, "caller", NULL, NULL,
  1569. "show per-callsite statistics", parse_caller_opt),
  1570. OPT_CALLBACK_NOOPT(0, "alloc", NULL, NULL,
  1571. "show per-allocation statistics", parse_alloc_opt),
  1572. OPT_CALLBACK('s', "sort", NULL, "key[,key2...]",
  1573. "sort by keys: ptr, callsite, bytes, hit, pingpong, frag, "
  1574. "page, order, migtype, gfp", parse_sort_opt),
  1575. OPT_CALLBACK('l', "line", NULL, "num", "show n lines", parse_line_opt),
  1576. OPT_BOOLEAN(0, "raw-ip", &raw_ip, "show raw ip instead of symbol"),
  1577. OPT_BOOLEAN('f', "force", &data.force, "don't complain, do it"),
  1578. OPT_CALLBACK_NOOPT(0, "slab", NULL, NULL, "Analyze slab allocator",
  1579. parse_slab_opt),
  1580. OPT_CALLBACK_NOOPT(0, "page", NULL, NULL, "Analyze page allocator",
  1581. parse_page_opt),
  1582. OPT_BOOLEAN(0, "live", &live_page, "Show live page stat"),
  1583. OPT_STRING(0, "time", &time_str, "str",
  1584. "Time span of interest (start,stop)"),
  1585. OPT_END()
  1586. };
  1587. const char *const kmem_subcommands[] = { "record", "stat", NULL };
  1588. const char *kmem_usage[] = {
  1589. NULL,
  1590. NULL
  1591. };
  1592. struct perf_session *session;
  1593. static const char errmsg[] = "No %s allocation events found. Have you run 'perf kmem record --%s'?\n";
  1594. int ret = perf_config(kmem_config, NULL);
  1595. if (ret)
  1596. return ret;
  1597. argc = parse_options_subcommand(argc, argv, kmem_options,
  1598. kmem_subcommands, kmem_usage,
  1599. PARSE_OPT_STOP_AT_NON_OPTION);
  1600. if (!argc)
  1601. usage_with_options(kmem_usage, kmem_options);
  1602. if (kmem_slab == 0 && kmem_page == 0) {
  1603. if (kmem_default == KMEM_SLAB)
  1604. kmem_slab = 1;
  1605. else
  1606. kmem_page = 1;
  1607. }
  1608. if (!strncmp(argv[0], "rec", 3)) {
  1609. symbol__init(NULL);
  1610. return __cmd_record(argc, argv);
  1611. }
  1612. data.path = input_name;
  1613. kmem_session = session = perf_session__new(&data, false, &perf_kmem);
  1614. if (IS_ERR(session))
  1615. return PTR_ERR(session);
  1616. ret = -1;
  1617. if (kmem_slab) {
  1618. if (!perf_evlist__find_tracepoint_by_name(session->evlist,
  1619. "kmem:kmalloc")) {
  1620. pr_err(errmsg, "slab", "slab");
  1621. goto out_delete;
  1622. }
  1623. }
  1624. if (kmem_page) {
  1625. struct evsel *evsel;
  1626. evsel = perf_evlist__find_tracepoint_by_name(session->evlist,
  1627. "kmem:mm_page_alloc");
  1628. if (evsel == NULL) {
  1629. pr_err(errmsg, "page", "page");
  1630. goto out_delete;
  1631. }
  1632. kmem_page_size = tep_get_page_size(evsel->tp_format->tep);
  1633. symbol_conf.use_callchain = true;
  1634. }
  1635. symbol__init(&session->header.env);
  1636. if (perf_time__parse_str(&ptime, time_str) != 0) {
  1637. pr_err("Invalid time string\n");
  1638. ret = -EINVAL;
  1639. goto out_delete;
  1640. }
  1641. if (!strcmp(argv[0], "stat")) {
  1642. setlocale(LC_ALL, "");
  1643. if (cpu__setup_cpunode_map())
  1644. goto out_delete;
  1645. if (list_empty(&slab_caller_sort))
  1646. setup_slab_sorting(&slab_caller_sort, default_slab_sort);
  1647. if (list_empty(&slab_alloc_sort))
  1648. setup_slab_sorting(&slab_alloc_sort, default_slab_sort);
  1649. if (list_empty(&page_caller_sort))
  1650. setup_page_sorting(&page_caller_sort, default_page_sort);
  1651. if (list_empty(&page_alloc_sort))
  1652. setup_page_sorting(&page_alloc_sort, default_page_sort);
  1653. if (kmem_page) {
  1654. setup_page_sorting(&page_alloc_sort_input,
  1655. "page,order,migtype,gfp");
  1656. setup_page_sorting(&page_caller_sort_input,
  1657. "callsite,order,migtype,gfp");
  1658. }
  1659. ret = __cmd_kmem(session);
  1660. } else
  1661. usage_with_options(kmem_usage, kmem_options);
  1662. out_delete:
  1663. perf_session__delete(session);
  1664. return ret;
  1665. }