builtin-sched.c 90 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610
  1. // SPDX-License-Identifier: GPL-2.0
  2. #include "builtin.h"
  3. #include "perf.h"
  4. #include "perf-sys.h"
  5. #include "util/cpumap.h"
  6. #include "util/evlist.h"
  7. #include "util/evsel.h"
  8. #include "util/evsel_fprintf.h"
  9. #include "util/symbol.h"
  10. #include "util/thread.h"
  11. #include "util/header.h"
  12. #include "util/session.h"
  13. #include "util/tool.h"
  14. #include "util/cloexec.h"
  15. #include "util/thread_map.h"
  16. #include "util/color.h"
  17. #include "util/stat.h"
  18. #include "util/string2.h"
  19. #include "util/callchain.h"
  20. #include "util/time-utils.h"
  21. #include <subcmd/pager.h>
  22. #include <subcmd/parse-options.h>
  23. #include "util/trace-event.h"
  24. #include "util/debug.h"
  25. #include "util/event.h"
  26. #include <linux/kernel.h>
  27. #include <linux/log2.h>
  28. #include <linux/zalloc.h>
  29. #include <sys/prctl.h>
  30. #include <sys/resource.h>
  31. #include <inttypes.h>
  32. #include <errno.h>
  33. #include <semaphore.h>
  34. #include <pthread.h>
  35. #include <math.h>
  36. #include <api/fs/fs.h>
  37. #include <perf/cpumap.h>
  38. #include <linux/time64.h>
  39. #include <linux/err.h>
  40. #include <linux/ctype.h>
  41. #define PR_SET_NAME 15 /* Set process name */
  42. #define MAX_CPUS 4096
  43. #define COMM_LEN 20
  44. #define SYM_LEN 129
  45. #define MAX_PID 1024000
  46. static const char *cpu_list;
  47. static DECLARE_BITMAP(cpu_bitmap, MAX_NR_CPUS);
  48. struct sched_atom;
  49. struct task_desc {
  50. unsigned long nr;
  51. unsigned long pid;
  52. char comm[COMM_LEN];
  53. unsigned long nr_events;
  54. unsigned long curr_event;
  55. struct sched_atom **atoms;
  56. pthread_t thread;
  57. sem_t sleep_sem;
  58. sem_t ready_for_work;
  59. sem_t work_done_sem;
  60. u64 cpu_usage;
  61. };
  62. enum sched_event_type {
  63. SCHED_EVENT_RUN,
  64. SCHED_EVENT_SLEEP,
  65. SCHED_EVENT_WAKEUP,
  66. SCHED_EVENT_MIGRATION,
  67. };
  68. struct sched_atom {
  69. enum sched_event_type type;
  70. int specific_wait;
  71. u64 timestamp;
  72. u64 duration;
  73. unsigned long nr;
  74. sem_t *wait_sem;
  75. struct task_desc *wakee;
  76. };
  77. #define TASK_STATE_TO_CHAR_STR "RSDTtZXxKWP"
  78. /* task state bitmask, copied from include/linux/sched.h */
  79. #define TASK_RUNNING 0
  80. #define TASK_INTERRUPTIBLE 1
  81. #define TASK_UNINTERRUPTIBLE 2
  82. #define __TASK_STOPPED 4
  83. #define __TASK_TRACED 8
  84. /* in tsk->exit_state */
  85. #define EXIT_DEAD 16
  86. #define EXIT_ZOMBIE 32
  87. #define EXIT_TRACE (EXIT_ZOMBIE | EXIT_DEAD)
  88. /* in tsk->state again */
  89. #define TASK_DEAD 64
  90. #define TASK_WAKEKILL 128
  91. #define TASK_WAKING 256
  92. #define TASK_PARKED 512
  93. enum thread_state {
  94. THREAD_SLEEPING = 0,
  95. THREAD_WAIT_CPU,
  96. THREAD_SCHED_IN,
  97. THREAD_IGNORE
  98. };
  99. struct work_atom {
  100. struct list_head list;
  101. enum thread_state state;
  102. u64 sched_out_time;
  103. u64 wake_up_time;
  104. u64 sched_in_time;
  105. u64 runtime;
  106. };
  107. struct work_atoms {
  108. struct list_head work_list;
  109. struct thread *thread;
  110. struct rb_node node;
  111. u64 max_lat;
  112. u64 max_lat_start;
  113. u64 max_lat_end;
  114. u64 total_lat;
  115. u64 nb_atoms;
  116. u64 total_runtime;
  117. int num_merged;
  118. };
  119. typedef int (*sort_fn_t)(struct work_atoms *, struct work_atoms *);
  120. struct perf_sched;
  121. struct trace_sched_handler {
  122. int (*switch_event)(struct perf_sched *sched, struct evsel *evsel,
  123. struct perf_sample *sample, struct machine *machine);
  124. int (*runtime_event)(struct perf_sched *sched, struct evsel *evsel,
  125. struct perf_sample *sample, struct machine *machine);
  126. int (*wakeup_event)(struct perf_sched *sched, struct evsel *evsel,
  127. struct perf_sample *sample, struct machine *machine);
  128. /* PERF_RECORD_FORK event, not sched_process_fork tracepoint */
  129. int (*fork_event)(struct perf_sched *sched, union perf_event *event,
  130. struct machine *machine);
  131. int (*migrate_task_event)(struct perf_sched *sched,
  132. struct evsel *evsel,
  133. struct perf_sample *sample,
  134. struct machine *machine);
  135. };
  136. #define COLOR_PIDS PERF_COLOR_BLUE
  137. #define COLOR_CPUS PERF_COLOR_BG_RED
  138. struct perf_sched_map {
  139. DECLARE_BITMAP(comp_cpus_mask, MAX_CPUS);
  140. int *comp_cpus;
  141. bool comp;
  142. struct perf_thread_map *color_pids;
  143. const char *color_pids_str;
  144. struct perf_cpu_map *color_cpus;
  145. const char *color_cpus_str;
  146. struct perf_cpu_map *cpus;
  147. const char *cpus_str;
  148. };
  149. struct perf_sched {
  150. struct perf_tool tool;
  151. const char *sort_order;
  152. unsigned long nr_tasks;
  153. struct task_desc **pid_to_task;
  154. struct task_desc **tasks;
  155. const struct trace_sched_handler *tp_handler;
  156. pthread_mutex_t start_work_mutex;
  157. pthread_mutex_t work_done_wait_mutex;
  158. int profile_cpu;
  159. /*
  160. * Track the current task - that way we can know whether there's any
  161. * weird events, such as a task being switched away that is not current.
  162. */
  163. int max_cpu;
  164. u32 curr_pid[MAX_CPUS];
  165. struct thread *curr_thread[MAX_CPUS];
  166. char next_shortname1;
  167. char next_shortname2;
  168. unsigned int replay_repeat;
  169. unsigned long nr_run_events;
  170. unsigned long nr_sleep_events;
  171. unsigned long nr_wakeup_events;
  172. unsigned long nr_sleep_corrections;
  173. unsigned long nr_run_events_optimized;
  174. unsigned long targetless_wakeups;
  175. unsigned long multitarget_wakeups;
  176. unsigned long nr_runs;
  177. unsigned long nr_timestamps;
  178. unsigned long nr_unordered_timestamps;
  179. unsigned long nr_context_switch_bugs;
  180. unsigned long nr_events;
  181. unsigned long nr_lost_chunks;
  182. unsigned long nr_lost_events;
  183. u64 run_measurement_overhead;
  184. u64 sleep_measurement_overhead;
  185. u64 start_time;
  186. u64 cpu_usage;
  187. u64 runavg_cpu_usage;
  188. u64 parent_cpu_usage;
  189. u64 runavg_parent_cpu_usage;
  190. u64 sum_runtime;
  191. u64 sum_fluct;
  192. u64 run_avg;
  193. u64 all_runtime;
  194. u64 all_count;
  195. u64 cpu_last_switched[MAX_CPUS];
  196. struct rb_root_cached atom_root, sorted_atom_root, merged_atom_root;
  197. struct list_head sort_list, cmp_pid;
  198. bool force;
  199. bool skip_merge;
  200. struct perf_sched_map map;
  201. /* options for timehist command */
  202. bool summary;
  203. bool summary_only;
  204. bool idle_hist;
  205. bool show_callchain;
  206. unsigned int max_stack;
  207. bool show_cpu_visual;
  208. bool show_wakeups;
  209. bool show_next;
  210. bool show_migrations;
  211. bool show_state;
  212. u64 skipped_samples;
  213. const char *time_str;
  214. struct perf_time_interval ptime;
  215. struct perf_time_interval hist_time;
  216. };
  217. /* per thread run time data */
  218. struct thread_runtime {
  219. u64 last_time; /* time of previous sched in/out event */
  220. u64 dt_run; /* run time */
  221. u64 dt_sleep; /* time between CPU access by sleep (off cpu) */
  222. u64 dt_iowait; /* time between CPU access by iowait (off cpu) */
  223. u64 dt_preempt; /* time between CPU access by preempt (off cpu) */
  224. u64 dt_delay; /* time between wakeup and sched-in */
  225. u64 ready_to_run; /* time of wakeup */
  226. struct stats run_stats;
  227. u64 total_run_time;
  228. u64 total_sleep_time;
  229. u64 total_iowait_time;
  230. u64 total_preempt_time;
  231. u64 total_delay_time;
  232. int last_state;
  233. char shortname[3];
  234. bool comm_changed;
  235. u64 migrations;
  236. };
  237. /* per event run time data */
  238. struct evsel_runtime {
  239. u64 *last_time; /* time this event was last seen per cpu */
  240. u32 ncpu; /* highest cpu slot allocated */
  241. };
  242. /* per cpu idle time data */
  243. struct idle_thread_runtime {
  244. struct thread_runtime tr;
  245. struct thread *last_thread;
  246. struct rb_root_cached sorted_root;
  247. struct callchain_root callchain;
  248. struct callchain_cursor cursor;
  249. };
  250. /* track idle times per cpu */
  251. static struct thread **idle_threads;
  252. static int idle_max_cpu;
  253. static char idle_comm[] = "<idle>";
  254. static u64 get_nsecs(void)
  255. {
  256. struct timespec ts;
  257. clock_gettime(CLOCK_MONOTONIC, &ts);
  258. return ts.tv_sec * NSEC_PER_SEC + ts.tv_nsec;
  259. }
  260. static void burn_nsecs(struct perf_sched *sched, u64 nsecs)
  261. {
  262. u64 T0 = get_nsecs(), T1;
  263. do {
  264. T1 = get_nsecs();
  265. } while (T1 + sched->run_measurement_overhead < T0 + nsecs);
  266. }
  267. static void sleep_nsecs(u64 nsecs)
  268. {
  269. struct timespec ts;
  270. ts.tv_nsec = nsecs % 999999999;
  271. ts.tv_sec = nsecs / 999999999;
  272. nanosleep(&ts, NULL);
  273. }
  274. static void calibrate_run_measurement_overhead(struct perf_sched *sched)
  275. {
  276. u64 T0, T1, delta, min_delta = NSEC_PER_SEC;
  277. int i;
  278. for (i = 0; i < 10; i++) {
  279. T0 = get_nsecs();
  280. burn_nsecs(sched, 0);
  281. T1 = get_nsecs();
  282. delta = T1-T0;
  283. min_delta = min(min_delta, delta);
  284. }
  285. sched->run_measurement_overhead = min_delta;
  286. printf("run measurement overhead: %" PRIu64 " nsecs\n", min_delta);
  287. }
  288. static void calibrate_sleep_measurement_overhead(struct perf_sched *sched)
  289. {
  290. u64 T0, T1, delta, min_delta = NSEC_PER_SEC;
  291. int i;
  292. for (i = 0; i < 10; i++) {
  293. T0 = get_nsecs();
  294. sleep_nsecs(10000);
  295. T1 = get_nsecs();
  296. delta = T1-T0;
  297. min_delta = min(min_delta, delta);
  298. }
  299. min_delta -= 10000;
  300. sched->sleep_measurement_overhead = min_delta;
  301. printf("sleep measurement overhead: %" PRIu64 " nsecs\n", min_delta);
  302. }
  303. static struct sched_atom *
  304. get_new_event(struct task_desc *task, u64 timestamp)
  305. {
  306. struct sched_atom *event = zalloc(sizeof(*event));
  307. unsigned long idx = task->nr_events;
  308. size_t size;
  309. event->timestamp = timestamp;
  310. event->nr = idx;
  311. task->nr_events++;
  312. size = sizeof(struct sched_atom *) * task->nr_events;
  313. task->atoms = realloc(task->atoms, size);
  314. BUG_ON(!task->atoms);
  315. task->atoms[idx] = event;
  316. return event;
  317. }
  318. static struct sched_atom *last_event(struct task_desc *task)
  319. {
  320. if (!task->nr_events)
  321. return NULL;
  322. return task->atoms[task->nr_events - 1];
  323. }
  324. static void add_sched_event_run(struct perf_sched *sched, struct task_desc *task,
  325. u64 timestamp, u64 duration)
  326. {
  327. struct sched_atom *event, *curr_event = last_event(task);
  328. /*
  329. * optimize an existing RUN event by merging this one
  330. * to it:
  331. */
  332. if (curr_event && curr_event->type == SCHED_EVENT_RUN) {
  333. sched->nr_run_events_optimized++;
  334. curr_event->duration += duration;
  335. return;
  336. }
  337. event = get_new_event(task, timestamp);
  338. event->type = SCHED_EVENT_RUN;
  339. event->duration = duration;
  340. sched->nr_run_events++;
  341. }
  342. static void add_sched_event_wakeup(struct perf_sched *sched, struct task_desc *task,
  343. u64 timestamp, struct task_desc *wakee)
  344. {
  345. struct sched_atom *event, *wakee_event;
  346. event = get_new_event(task, timestamp);
  347. event->type = SCHED_EVENT_WAKEUP;
  348. event->wakee = wakee;
  349. wakee_event = last_event(wakee);
  350. if (!wakee_event || wakee_event->type != SCHED_EVENT_SLEEP) {
  351. sched->targetless_wakeups++;
  352. return;
  353. }
  354. if (wakee_event->wait_sem) {
  355. sched->multitarget_wakeups++;
  356. return;
  357. }
  358. wakee_event->wait_sem = zalloc(sizeof(*wakee_event->wait_sem));
  359. sem_init(wakee_event->wait_sem, 0, 0);
  360. wakee_event->specific_wait = 1;
  361. event->wait_sem = wakee_event->wait_sem;
  362. sched->nr_wakeup_events++;
  363. }
  364. static void add_sched_event_sleep(struct perf_sched *sched, struct task_desc *task,
  365. u64 timestamp, u64 task_state __maybe_unused)
  366. {
  367. struct sched_atom *event = get_new_event(task, timestamp);
  368. event->type = SCHED_EVENT_SLEEP;
  369. sched->nr_sleep_events++;
  370. }
  371. static struct task_desc *register_pid(struct perf_sched *sched,
  372. unsigned long pid, const char *comm)
  373. {
  374. struct task_desc *task;
  375. static int pid_max;
  376. if (sched->pid_to_task == NULL) {
  377. if (sysctl__read_int("kernel/pid_max", &pid_max) < 0)
  378. pid_max = MAX_PID;
  379. BUG_ON((sched->pid_to_task = calloc(pid_max, sizeof(struct task_desc *))) == NULL);
  380. }
  381. if (pid >= (unsigned long)pid_max) {
  382. BUG_ON((sched->pid_to_task = realloc(sched->pid_to_task, (pid + 1) *
  383. sizeof(struct task_desc *))) == NULL);
  384. while (pid >= (unsigned long)pid_max)
  385. sched->pid_to_task[pid_max++] = NULL;
  386. }
  387. task = sched->pid_to_task[pid];
  388. if (task)
  389. return task;
  390. task = zalloc(sizeof(*task));
  391. task->pid = pid;
  392. task->nr = sched->nr_tasks;
  393. strcpy(task->comm, comm);
  394. /*
  395. * every task starts in sleeping state - this gets ignored
  396. * if there's no wakeup pointing to this sleep state:
  397. */
  398. add_sched_event_sleep(sched, task, 0, 0);
  399. sched->pid_to_task[pid] = task;
  400. sched->nr_tasks++;
  401. sched->tasks = realloc(sched->tasks, sched->nr_tasks * sizeof(struct task_desc *));
  402. BUG_ON(!sched->tasks);
  403. sched->tasks[task->nr] = task;
  404. if (verbose > 0)
  405. printf("registered task #%ld, PID %ld (%s)\n", sched->nr_tasks, pid, comm);
  406. return task;
  407. }
  408. static void print_task_traces(struct perf_sched *sched)
  409. {
  410. struct task_desc *task;
  411. unsigned long i;
  412. for (i = 0; i < sched->nr_tasks; i++) {
  413. task = sched->tasks[i];
  414. printf("task %6ld (%20s:%10ld), nr_events: %ld\n",
  415. task->nr, task->comm, task->pid, task->nr_events);
  416. }
  417. }
  418. static void add_cross_task_wakeups(struct perf_sched *sched)
  419. {
  420. struct task_desc *task1, *task2;
  421. unsigned long i, j;
  422. for (i = 0; i < sched->nr_tasks; i++) {
  423. task1 = sched->tasks[i];
  424. j = i + 1;
  425. if (j == sched->nr_tasks)
  426. j = 0;
  427. task2 = sched->tasks[j];
  428. add_sched_event_wakeup(sched, task1, 0, task2);
  429. }
  430. }
  431. static void perf_sched__process_event(struct perf_sched *sched,
  432. struct sched_atom *atom)
  433. {
  434. int ret = 0;
  435. switch (atom->type) {
  436. case SCHED_EVENT_RUN:
  437. burn_nsecs(sched, atom->duration);
  438. break;
  439. case SCHED_EVENT_SLEEP:
  440. if (atom->wait_sem)
  441. ret = sem_wait(atom->wait_sem);
  442. BUG_ON(ret);
  443. break;
  444. case SCHED_EVENT_WAKEUP:
  445. if (atom->wait_sem)
  446. ret = sem_post(atom->wait_sem);
  447. BUG_ON(ret);
  448. break;
  449. case SCHED_EVENT_MIGRATION:
  450. break;
  451. default:
  452. BUG_ON(1);
  453. }
  454. }
  455. static u64 get_cpu_usage_nsec_parent(void)
  456. {
  457. struct rusage ru;
  458. u64 sum;
  459. int err;
  460. err = getrusage(RUSAGE_SELF, &ru);
  461. BUG_ON(err);
  462. sum = ru.ru_utime.tv_sec * NSEC_PER_SEC + ru.ru_utime.tv_usec * NSEC_PER_USEC;
  463. sum += ru.ru_stime.tv_sec * NSEC_PER_SEC + ru.ru_stime.tv_usec * NSEC_PER_USEC;
  464. return sum;
  465. }
  466. static int self_open_counters(struct perf_sched *sched, unsigned long cur_task)
  467. {
  468. struct perf_event_attr attr;
  469. char sbuf[STRERR_BUFSIZE], info[STRERR_BUFSIZE];
  470. int fd;
  471. struct rlimit limit;
  472. bool need_privilege = false;
  473. memset(&attr, 0, sizeof(attr));
  474. attr.type = PERF_TYPE_SOFTWARE;
  475. attr.config = PERF_COUNT_SW_TASK_CLOCK;
  476. force_again:
  477. fd = sys_perf_event_open(&attr, 0, -1, -1,
  478. perf_event_open_cloexec_flag());
  479. if (fd < 0) {
  480. if (errno == EMFILE) {
  481. if (sched->force) {
  482. BUG_ON(getrlimit(RLIMIT_NOFILE, &limit) == -1);
  483. limit.rlim_cur += sched->nr_tasks - cur_task;
  484. if (limit.rlim_cur > limit.rlim_max) {
  485. limit.rlim_max = limit.rlim_cur;
  486. need_privilege = true;
  487. }
  488. if (setrlimit(RLIMIT_NOFILE, &limit) == -1) {
  489. if (need_privilege && errno == EPERM)
  490. strcpy(info, "Need privilege\n");
  491. } else
  492. goto force_again;
  493. } else
  494. strcpy(info, "Have a try with -f option\n");
  495. }
  496. pr_err("Error: sys_perf_event_open() syscall returned "
  497. "with %d (%s)\n%s", fd,
  498. str_error_r(errno, sbuf, sizeof(sbuf)), info);
  499. exit(EXIT_FAILURE);
  500. }
  501. return fd;
  502. }
  503. static u64 get_cpu_usage_nsec_self(int fd)
  504. {
  505. u64 runtime;
  506. int ret;
  507. ret = read(fd, &runtime, sizeof(runtime));
  508. BUG_ON(ret != sizeof(runtime));
  509. return runtime;
  510. }
  511. struct sched_thread_parms {
  512. struct task_desc *task;
  513. struct perf_sched *sched;
  514. int fd;
  515. };
  516. static void *thread_func(void *ctx)
  517. {
  518. struct sched_thread_parms *parms = ctx;
  519. struct task_desc *this_task = parms->task;
  520. struct perf_sched *sched = parms->sched;
  521. u64 cpu_usage_0, cpu_usage_1;
  522. unsigned long i, ret;
  523. char comm2[22];
  524. int fd = parms->fd;
  525. zfree(&parms);
  526. sprintf(comm2, ":%s", this_task->comm);
  527. prctl(PR_SET_NAME, comm2);
  528. if (fd < 0)
  529. return NULL;
  530. again:
  531. ret = sem_post(&this_task->ready_for_work);
  532. BUG_ON(ret);
  533. ret = pthread_mutex_lock(&sched->start_work_mutex);
  534. BUG_ON(ret);
  535. ret = pthread_mutex_unlock(&sched->start_work_mutex);
  536. BUG_ON(ret);
  537. cpu_usage_0 = get_cpu_usage_nsec_self(fd);
  538. for (i = 0; i < this_task->nr_events; i++) {
  539. this_task->curr_event = i;
  540. perf_sched__process_event(sched, this_task->atoms[i]);
  541. }
  542. cpu_usage_1 = get_cpu_usage_nsec_self(fd);
  543. this_task->cpu_usage = cpu_usage_1 - cpu_usage_0;
  544. ret = sem_post(&this_task->work_done_sem);
  545. BUG_ON(ret);
  546. ret = pthread_mutex_lock(&sched->work_done_wait_mutex);
  547. BUG_ON(ret);
  548. ret = pthread_mutex_unlock(&sched->work_done_wait_mutex);
  549. BUG_ON(ret);
  550. goto again;
  551. }
  552. static void create_tasks(struct perf_sched *sched)
  553. {
  554. struct task_desc *task;
  555. pthread_attr_t attr;
  556. unsigned long i;
  557. int err;
  558. err = pthread_attr_init(&attr);
  559. BUG_ON(err);
  560. err = pthread_attr_setstacksize(&attr,
  561. (size_t) max(16 * 1024, PTHREAD_STACK_MIN));
  562. BUG_ON(err);
  563. err = pthread_mutex_lock(&sched->start_work_mutex);
  564. BUG_ON(err);
  565. err = pthread_mutex_lock(&sched->work_done_wait_mutex);
  566. BUG_ON(err);
  567. for (i = 0; i < sched->nr_tasks; i++) {
  568. struct sched_thread_parms *parms = malloc(sizeof(*parms));
  569. BUG_ON(parms == NULL);
  570. parms->task = task = sched->tasks[i];
  571. parms->sched = sched;
  572. parms->fd = self_open_counters(sched, i);
  573. sem_init(&task->sleep_sem, 0, 0);
  574. sem_init(&task->ready_for_work, 0, 0);
  575. sem_init(&task->work_done_sem, 0, 0);
  576. task->curr_event = 0;
  577. err = pthread_create(&task->thread, &attr, thread_func, parms);
  578. BUG_ON(err);
  579. }
  580. }
  581. static void wait_for_tasks(struct perf_sched *sched)
  582. {
  583. u64 cpu_usage_0, cpu_usage_1;
  584. struct task_desc *task;
  585. unsigned long i, ret;
  586. sched->start_time = get_nsecs();
  587. sched->cpu_usage = 0;
  588. pthread_mutex_unlock(&sched->work_done_wait_mutex);
  589. for (i = 0; i < sched->nr_tasks; i++) {
  590. task = sched->tasks[i];
  591. ret = sem_wait(&task->ready_for_work);
  592. BUG_ON(ret);
  593. sem_init(&task->ready_for_work, 0, 0);
  594. }
  595. ret = pthread_mutex_lock(&sched->work_done_wait_mutex);
  596. BUG_ON(ret);
  597. cpu_usage_0 = get_cpu_usage_nsec_parent();
  598. pthread_mutex_unlock(&sched->start_work_mutex);
  599. for (i = 0; i < sched->nr_tasks; i++) {
  600. task = sched->tasks[i];
  601. ret = sem_wait(&task->work_done_sem);
  602. BUG_ON(ret);
  603. sem_init(&task->work_done_sem, 0, 0);
  604. sched->cpu_usage += task->cpu_usage;
  605. task->cpu_usage = 0;
  606. }
  607. cpu_usage_1 = get_cpu_usage_nsec_parent();
  608. if (!sched->runavg_cpu_usage)
  609. sched->runavg_cpu_usage = sched->cpu_usage;
  610. sched->runavg_cpu_usage = (sched->runavg_cpu_usage * (sched->replay_repeat - 1) + sched->cpu_usage) / sched->replay_repeat;
  611. sched->parent_cpu_usage = cpu_usage_1 - cpu_usage_0;
  612. if (!sched->runavg_parent_cpu_usage)
  613. sched->runavg_parent_cpu_usage = sched->parent_cpu_usage;
  614. sched->runavg_parent_cpu_usage = (sched->runavg_parent_cpu_usage * (sched->replay_repeat - 1) +
  615. sched->parent_cpu_usage)/sched->replay_repeat;
  616. ret = pthread_mutex_lock(&sched->start_work_mutex);
  617. BUG_ON(ret);
  618. for (i = 0; i < sched->nr_tasks; i++) {
  619. task = sched->tasks[i];
  620. sem_init(&task->sleep_sem, 0, 0);
  621. task->curr_event = 0;
  622. }
  623. }
  624. static void run_one_test(struct perf_sched *sched)
  625. {
  626. u64 T0, T1, delta, avg_delta, fluct;
  627. T0 = get_nsecs();
  628. wait_for_tasks(sched);
  629. T1 = get_nsecs();
  630. delta = T1 - T0;
  631. sched->sum_runtime += delta;
  632. sched->nr_runs++;
  633. avg_delta = sched->sum_runtime / sched->nr_runs;
  634. if (delta < avg_delta)
  635. fluct = avg_delta - delta;
  636. else
  637. fluct = delta - avg_delta;
  638. sched->sum_fluct += fluct;
  639. if (!sched->run_avg)
  640. sched->run_avg = delta;
  641. sched->run_avg = (sched->run_avg * (sched->replay_repeat - 1) + delta) / sched->replay_repeat;
  642. printf("#%-3ld: %0.3f, ", sched->nr_runs, (double)delta / NSEC_PER_MSEC);
  643. printf("ravg: %0.2f, ", (double)sched->run_avg / NSEC_PER_MSEC);
  644. printf("cpu: %0.2f / %0.2f",
  645. (double)sched->cpu_usage / NSEC_PER_MSEC, (double)sched->runavg_cpu_usage / NSEC_PER_MSEC);
  646. #if 0
  647. /*
  648. * rusage statistics done by the parent, these are less
  649. * accurate than the sched->sum_exec_runtime based statistics:
  650. */
  651. printf(" [%0.2f / %0.2f]",
  652. (double)sched->parent_cpu_usage / NSEC_PER_MSEC,
  653. (double)sched->runavg_parent_cpu_usage / NSEC_PER_MSEC);
  654. #endif
  655. printf("\n");
  656. if (sched->nr_sleep_corrections)
  657. printf(" (%ld sleep corrections)\n", sched->nr_sleep_corrections);
  658. sched->nr_sleep_corrections = 0;
  659. }
  660. static void test_calibrations(struct perf_sched *sched)
  661. {
  662. u64 T0, T1;
  663. T0 = get_nsecs();
  664. burn_nsecs(sched, NSEC_PER_MSEC);
  665. T1 = get_nsecs();
  666. printf("the run test took %" PRIu64 " nsecs\n", T1 - T0);
  667. T0 = get_nsecs();
  668. sleep_nsecs(NSEC_PER_MSEC);
  669. T1 = get_nsecs();
  670. printf("the sleep test took %" PRIu64 " nsecs\n", T1 - T0);
  671. }
  672. static int
  673. replay_wakeup_event(struct perf_sched *sched,
  674. struct evsel *evsel, struct perf_sample *sample,
  675. struct machine *machine __maybe_unused)
  676. {
  677. const char *comm = evsel__strval(evsel, sample, "comm");
  678. const u32 pid = evsel__intval(evsel, sample, "pid");
  679. struct task_desc *waker, *wakee;
  680. if (verbose > 0) {
  681. printf("sched_wakeup event %p\n", evsel);
  682. printf(" ... pid %d woke up %s/%d\n", sample->tid, comm, pid);
  683. }
  684. waker = register_pid(sched, sample->tid, "<unknown>");
  685. wakee = register_pid(sched, pid, comm);
  686. add_sched_event_wakeup(sched, waker, sample->time, wakee);
  687. return 0;
  688. }
  689. static int replay_switch_event(struct perf_sched *sched,
  690. struct evsel *evsel,
  691. struct perf_sample *sample,
  692. struct machine *machine __maybe_unused)
  693. {
  694. const char *prev_comm = evsel__strval(evsel, sample, "prev_comm"),
  695. *next_comm = evsel__strval(evsel, sample, "next_comm");
  696. const u32 prev_pid = evsel__intval(evsel, sample, "prev_pid"),
  697. next_pid = evsel__intval(evsel, sample, "next_pid");
  698. const u64 prev_state = evsel__intval(evsel, sample, "prev_state");
  699. struct task_desc *prev, __maybe_unused *next;
  700. u64 timestamp0, timestamp = sample->time;
  701. int cpu = sample->cpu;
  702. s64 delta;
  703. if (verbose > 0)
  704. printf("sched_switch event %p\n", evsel);
  705. if (cpu >= MAX_CPUS || cpu < 0)
  706. return 0;
  707. timestamp0 = sched->cpu_last_switched[cpu];
  708. if (timestamp0)
  709. delta = timestamp - timestamp0;
  710. else
  711. delta = 0;
  712. if (delta < 0) {
  713. pr_err("hm, delta: %" PRIu64 " < 0 ?\n", delta);
  714. return -1;
  715. }
  716. pr_debug(" ... switch from %s/%d to %s/%d [ran %" PRIu64 " nsecs]\n",
  717. prev_comm, prev_pid, next_comm, next_pid, delta);
  718. prev = register_pid(sched, prev_pid, prev_comm);
  719. next = register_pid(sched, next_pid, next_comm);
  720. sched->cpu_last_switched[cpu] = timestamp;
  721. add_sched_event_run(sched, prev, timestamp, delta);
  722. add_sched_event_sleep(sched, prev, timestamp, prev_state);
  723. return 0;
  724. }
  725. static int replay_fork_event(struct perf_sched *sched,
  726. union perf_event *event,
  727. struct machine *machine)
  728. {
  729. struct thread *child, *parent;
  730. child = machine__findnew_thread(machine, event->fork.pid,
  731. event->fork.tid);
  732. parent = machine__findnew_thread(machine, event->fork.ppid,
  733. event->fork.ptid);
  734. if (child == NULL || parent == NULL) {
  735. pr_debug("thread does not exist on fork event: child %p, parent %p\n",
  736. child, parent);
  737. goto out_put;
  738. }
  739. if (verbose > 0) {
  740. printf("fork event\n");
  741. printf("... parent: %s/%d\n", thread__comm_str(parent), parent->tid);
  742. printf("... child: %s/%d\n", thread__comm_str(child), child->tid);
  743. }
  744. register_pid(sched, parent->tid, thread__comm_str(parent));
  745. register_pid(sched, child->tid, thread__comm_str(child));
  746. out_put:
  747. thread__put(child);
  748. thread__put(parent);
  749. return 0;
  750. }
  751. struct sort_dimension {
  752. const char *name;
  753. sort_fn_t cmp;
  754. struct list_head list;
  755. };
  756. /*
  757. * handle runtime stats saved per thread
  758. */
  759. static struct thread_runtime *thread__init_runtime(struct thread *thread)
  760. {
  761. struct thread_runtime *r;
  762. r = zalloc(sizeof(struct thread_runtime));
  763. if (!r)
  764. return NULL;
  765. init_stats(&r->run_stats);
  766. thread__set_priv(thread, r);
  767. return r;
  768. }
  769. static struct thread_runtime *thread__get_runtime(struct thread *thread)
  770. {
  771. struct thread_runtime *tr;
  772. tr = thread__priv(thread);
  773. if (tr == NULL) {
  774. tr = thread__init_runtime(thread);
  775. if (tr == NULL)
  776. pr_debug("Failed to malloc memory for runtime data.\n");
  777. }
  778. return tr;
  779. }
  780. static int
  781. thread_lat_cmp(struct list_head *list, struct work_atoms *l, struct work_atoms *r)
  782. {
  783. struct sort_dimension *sort;
  784. int ret = 0;
  785. BUG_ON(list_empty(list));
  786. list_for_each_entry(sort, list, list) {
  787. ret = sort->cmp(l, r);
  788. if (ret)
  789. return ret;
  790. }
  791. return ret;
  792. }
  793. static struct work_atoms *
  794. thread_atoms_search(struct rb_root_cached *root, struct thread *thread,
  795. struct list_head *sort_list)
  796. {
  797. struct rb_node *node = root->rb_root.rb_node;
  798. struct work_atoms key = { .thread = thread };
  799. while (node) {
  800. struct work_atoms *atoms;
  801. int cmp;
  802. atoms = container_of(node, struct work_atoms, node);
  803. cmp = thread_lat_cmp(sort_list, &key, atoms);
  804. if (cmp > 0)
  805. node = node->rb_left;
  806. else if (cmp < 0)
  807. node = node->rb_right;
  808. else {
  809. BUG_ON(thread != atoms->thread);
  810. return atoms;
  811. }
  812. }
  813. return NULL;
  814. }
  815. static void
  816. __thread_latency_insert(struct rb_root_cached *root, struct work_atoms *data,
  817. struct list_head *sort_list)
  818. {
  819. struct rb_node **new = &(root->rb_root.rb_node), *parent = NULL;
  820. bool leftmost = true;
  821. while (*new) {
  822. struct work_atoms *this;
  823. int cmp;
  824. this = container_of(*new, struct work_atoms, node);
  825. parent = *new;
  826. cmp = thread_lat_cmp(sort_list, data, this);
  827. if (cmp > 0)
  828. new = &((*new)->rb_left);
  829. else {
  830. new = &((*new)->rb_right);
  831. leftmost = false;
  832. }
  833. }
  834. rb_link_node(&data->node, parent, new);
  835. rb_insert_color_cached(&data->node, root, leftmost);
  836. }
  837. static int thread_atoms_insert(struct perf_sched *sched, struct thread *thread)
  838. {
  839. struct work_atoms *atoms = zalloc(sizeof(*atoms));
  840. if (!atoms) {
  841. pr_err("No memory at %s\n", __func__);
  842. return -1;
  843. }
  844. atoms->thread = thread__get(thread);
  845. INIT_LIST_HEAD(&atoms->work_list);
  846. __thread_latency_insert(&sched->atom_root, atoms, &sched->cmp_pid);
  847. return 0;
  848. }
  849. static char sched_out_state(u64 prev_state)
  850. {
  851. const char *str = TASK_STATE_TO_CHAR_STR;
  852. return str[prev_state];
  853. }
  854. static int
  855. add_sched_out_event(struct work_atoms *atoms,
  856. char run_state,
  857. u64 timestamp)
  858. {
  859. struct work_atom *atom = zalloc(sizeof(*atom));
  860. if (!atom) {
  861. pr_err("Non memory at %s", __func__);
  862. return -1;
  863. }
  864. atom->sched_out_time = timestamp;
  865. if (run_state == 'R') {
  866. atom->state = THREAD_WAIT_CPU;
  867. atom->wake_up_time = atom->sched_out_time;
  868. }
  869. list_add_tail(&atom->list, &atoms->work_list);
  870. return 0;
  871. }
  872. static void
  873. add_runtime_event(struct work_atoms *atoms, u64 delta,
  874. u64 timestamp __maybe_unused)
  875. {
  876. struct work_atom *atom;
  877. BUG_ON(list_empty(&atoms->work_list));
  878. atom = list_entry(atoms->work_list.prev, struct work_atom, list);
  879. atom->runtime += delta;
  880. atoms->total_runtime += delta;
  881. }
  882. static void
  883. add_sched_in_event(struct work_atoms *atoms, u64 timestamp)
  884. {
  885. struct work_atom *atom;
  886. u64 delta;
  887. if (list_empty(&atoms->work_list))
  888. return;
  889. atom = list_entry(atoms->work_list.prev, struct work_atom, list);
  890. if (atom->state != THREAD_WAIT_CPU)
  891. return;
  892. if (timestamp < atom->wake_up_time) {
  893. atom->state = THREAD_IGNORE;
  894. return;
  895. }
  896. atom->state = THREAD_SCHED_IN;
  897. atom->sched_in_time = timestamp;
  898. delta = atom->sched_in_time - atom->wake_up_time;
  899. atoms->total_lat += delta;
  900. if (delta > atoms->max_lat) {
  901. atoms->max_lat = delta;
  902. atoms->max_lat_start = atom->wake_up_time;
  903. atoms->max_lat_end = timestamp;
  904. }
  905. atoms->nb_atoms++;
  906. }
  907. static int latency_switch_event(struct perf_sched *sched,
  908. struct evsel *evsel,
  909. struct perf_sample *sample,
  910. struct machine *machine)
  911. {
  912. const u32 prev_pid = evsel__intval(evsel, sample, "prev_pid"),
  913. next_pid = evsel__intval(evsel, sample, "next_pid");
  914. const u64 prev_state = evsel__intval(evsel, sample, "prev_state");
  915. struct work_atoms *out_events, *in_events;
  916. struct thread *sched_out, *sched_in;
  917. u64 timestamp0, timestamp = sample->time;
  918. int cpu = sample->cpu, err = -1;
  919. s64 delta;
  920. BUG_ON(cpu >= MAX_CPUS || cpu < 0);
  921. timestamp0 = sched->cpu_last_switched[cpu];
  922. sched->cpu_last_switched[cpu] = timestamp;
  923. if (timestamp0)
  924. delta = timestamp - timestamp0;
  925. else
  926. delta = 0;
  927. if (delta < 0) {
  928. pr_err("hm, delta: %" PRIu64 " < 0 ?\n", delta);
  929. return -1;
  930. }
  931. sched_out = machine__findnew_thread(machine, -1, prev_pid);
  932. sched_in = machine__findnew_thread(machine, -1, next_pid);
  933. if (sched_out == NULL || sched_in == NULL)
  934. goto out_put;
  935. out_events = thread_atoms_search(&sched->atom_root, sched_out, &sched->cmp_pid);
  936. if (!out_events) {
  937. if (thread_atoms_insert(sched, sched_out))
  938. goto out_put;
  939. out_events = thread_atoms_search(&sched->atom_root, sched_out, &sched->cmp_pid);
  940. if (!out_events) {
  941. pr_err("out-event: Internal tree error");
  942. goto out_put;
  943. }
  944. }
  945. if (add_sched_out_event(out_events, sched_out_state(prev_state), timestamp))
  946. return -1;
  947. in_events = thread_atoms_search(&sched->atom_root, sched_in, &sched->cmp_pid);
  948. if (!in_events) {
  949. if (thread_atoms_insert(sched, sched_in))
  950. goto out_put;
  951. in_events = thread_atoms_search(&sched->atom_root, sched_in, &sched->cmp_pid);
  952. if (!in_events) {
  953. pr_err("in-event: Internal tree error");
  954. goto out_put;
  955. }
  956. /*
  957. * Take came in we have not heard about yet,
  958. * add in an initial atom in runnable state:
  959. */
  960. if (add_sched_out_event(in_events, 'R', timestamp))
  961. goto out_put;
  962. }
  963. add_sched_in_event(in_events, timestamp);
  964. err = 0;
  965. out_put:
  966. thread__put(sched_out);
  967. thread__put(sched_in);
  968. return err;
  969. }
  970. static int latency_runtime_event(struct perf_sched *sched,
  971. struct evsel *evsel,
  972. struct perf_sample *sample,
  973. struct machine *machine)
  974. {
  975. const u32 pid = evsel__intval(evsel, sample, "pid");
  976. const u64 runtime = evsel__intval(evsel, sample, "runtime");
  977. struct thread *thread = machine__findnew_thread(machine, -1, pid);
  978. struct work_atoms *atoms = thread_atoms_search(&sched->atom_root, thread, &sched->cmp_pid);
  979. u64 timestamp = sample->time;
  980. int cpu = sample->cpu, err = -1;
  981. if (thread == NULL)
  982. return -1;
  983. BUG_ON(cpu >= MAX_CPUS || cpu < 0);
  984. if (!atoms) {
  985. if (thread_atoms_insert(sched, thread))
  986. goto out_put;
  987. atoms = thread_atoms_search(&sched->atom_root, thread, &sched->cmp_pid);
  988. if (!atoms) {
  989. pr_err("in-event: Internal tree error");
  990. goto out_put;
  991. }
  992. if (add_sched_out_event(atoms, 'R', timestamp))
  993. goto out_put;
  994. }
  995. add_runtime_event(atoms, runtime, timestamp);
  996. err = 0;
  997. out_put:
  998. thread__put(thread);
  999. return err;
  1000. }
  1001. static int latency_wakeup_event(struct perf_sched *sched,
  1002. struct evsel *evsel,
  1003. struct perf_sample *sample,
  1004. struct machine *machine)
  1005. {
  1006. const u32 pid = evsel__intval(evsel, sample, "pid");
  1007. struct work_atoms *atoms;
  1008. struct work_atom *atom;
  1009. struct thread *wakee;
  1010. u64 timestamp = sample->time;
  1011. int err = -1;
  1012. wakee = machine__findnew_thread(machine, -1, pid);
  1013. if (wakee == NULL)
  1014. return -1;
  1015. atoms = thread_atoms_search(&sched->atom_root, wakee, &sched->cmp_pid);
  1016. if (!atoms) {
  1017. if (thread_atoms_insert(sched, wakee))
  1018. goto out_put;
  1019. atoms = thread_atoms_search(&sched->atom_root, wakee, &sched->cmp_pid);
  1020. if (!atoms) {
  1021. pr_err("wakeup-event: Internal tree error");
  1022. goto out_put;
  1023. }
  1024. if (add_sched_out_event(atoms, 'S', timestamp))
  1025. goto out_put;
  1026. }
  1027. BUG_ON(list_empty(&atoms->work_list));
  1028. atom = list_entry(atoms->work_list.prev, struct work_atom, list);
  1029. /*
  1030. * As we do not guarantee the wakeup event happens when
  1031. * task is out of run queue, also may happen when task is
  1032. * on run queue and wakeup only change ->state to TASK_RUNNING,
  1033. * then we should not set the ->wake_up_time when wake up a
  1034. * task which is on run queue.
  1035. *
  1036. * You WILL be missing events if you've recorded only
  1037. * one CPU, or are only looking at only one, so don't
  1038. * skip in this case.
  1039. */
  1040. if (sched->profile_cpu == -1 && atom->state != THREAD_SLEEPING)
  1041. goto out_ok;
  1042. sched->nr_timestamps++;
  1043. if (atom->sched_out_time > timestamp) {
  1044. sched->nr_unordered_timestamps++;
  1045. goto out_ok;
  1046. }
  1047. atom->state = THREAD_WAIT_CPU;
  1048. atom->wake_up_time = timestamp;
  1049. out_ok:
  1050. err = 0;
  1051. out_put:
  1052. thread__put(wakee);
  1053. return err;
  1054. }
  1055. static int latency_migrate_task_event(struct perf_sched *sched,
  1056. struct evsel *evsel,
  1057. struct perf_sample *sample,
  1058. struct machine *machine)
  1059. {
  1060. const u32 pid = evsel__intval(evsel, sample, "pid");
  1061. u64 timestamp = sample->time;
  1062. struct work_atoms *atoms;
  1063. struct work_atom *atom;
  1064. struct thread *migrant;
  1065. int err = -1;
  1066. /*
  1067. * Only need to worry about migration when profiling one CPU.
  1068. */
  1069. if (sched->profile_cpu == -1)
  1070. return 0;
  1071. migrant = machine__findnew_thread(machine, -1, pid);
  1072. if (migrant == NULL)
  1073. return -1;
  1074. atoms = thread_atoms_search(&sched->atom_root, migrant, &sched->cmp_pid);
  1075. if (!atoms) {
  1076. if (thread_atoms_insert(sched, migrant))
  1077. goto out_put;
  1078. register_pid(sched, migrant->tid, thread__comm_str(migrant));
  1079. atoms = thread_atoms_search(&sched->atom_root, migrant, &sched->cmp_pid);
  1080. if (!atoms) {
  1081. pr_err("migration-event: Internal tree error");
  1082. goto out_put;
  1083. }
  1084. if (add_sched_out_event(atoms, 'R', timestamp))
  1085. goto out_put;
  1086. }
  1087. BUG_ON(list_empty(&atoms->work_list));
  1088. atom = list_entry(atoms->work_list.prev, struct work_atom, list);
  1089. atom->sched_in_time = atom->sched_out_time = atom->wake_up_time = timestamp;
  1090. sched->nr_timestamps++;
  1091. if (atom->sched_out_time > timestamp)
  1092. sched->nr_unordered_timestamps++;
  1093. err = 0;
  1094. out_put:
  1095. thread__put(migrant);
  1096. return err;
  1097. }
  1098. static void output_lat_thread(struct perf_sched *sched, struct work_atoms *work_list)
  1099. {
  1100. int i;
  1101. int ret;
  1102. u64 avg;
  1103. char max_lat_start[32], max_lat_end[32];
  1104. if (!work_list->nb_atoms)
  1105. return;
  1106. /*
  1107. * Ignore idle threads:
  1108. */
  1109. if (!strcmp(thread__comm_str(work_list->thread), "swapper"))
  1110. return;
  1111. sched->all_runtime += work_list->total_runtime;
  1112. sched->all_count += work_list->nb_atoms;
  1113. if (work_list->num_merged > 1)
  1114. ret = printf(" %s:(%d) ", thread__comm_str(work_list->thread), work_list->num_merged);
  1115. else
  1116. ret = printf(" %s:%d ", thread__comm_str(work_list->thread), work_list->thread->tid);
  1117. for (i = 0; i < 24 - ret; i++)
  1118. printf(" ");
  1119. avg = work_list->total_lat / work_list->nb_atoms;
  1120. timestamp__scnprintf_usec(work_list->max_lat_start, max_lat_start, sizeof(max_lat_start));
  1121. timestamp__scnprintf_usec(work_list->max_lat_end, max_lat_end, sizeof(max_lat_end));
  1122. printf("|%11.3f ms |%9" PRIu64 " | avg:%8.3f ms | max:%8.3f ms | max start: %12s s | max end: %12s s\n",
  1123. (double)work_list->total_runtime / NSEC_PER_MSEC,
  1124. work_list->nb_atoms, (double)avg / NSEC_PER_MSEC,
  1125. (double)work_list->max_lat / NSEC_PER_MSEC,
  1126. max_lat_start, max_lat_end);
  1127. }
  1128. static int pid_cmp(struct work_atoms *l, struct work_atoms *r)
  1129. {
  1130. if (l->thread == r->thread)
  1131. return 0;
  1132. if (l->thread->tid < r->thread->tid)
  1133. return -1;
  1134. if (l->thread->tid > r->thread->tid)
  1135. return 1;
  1136. return (int)(l->thread - r->thread);
  1137. }
  1138. static int avg_cmp(struct work_atoms *l, struct work_atoms *r)
  1139. {
  1140. u64 avgl, avgr;
  1141. if (!l->nb_atoms)
  1142. return -1;
  1143. if (!r->nb_atoms)
  1144. return 1;
  1145. avgl = l->total_lat / l->nb_atoms;
  1146. avgr = r->total_lat / r->nb_atoms;
  1147. if (avgl < avgr)
  1148. return -1;
  1149. if (avgl > avgr)
  1150. return 1;
  1151. return 0;
  1152. }
  1153. static int max_cmp(struct work_atoms *l, struct work_atoms *r)
  1154. {
  1155. if (l->max_lat < r->max_lat)
  1156. return -1;
  1157. if (l->max_lat > r->max_lat)
  1158. return 1;
  1159. return 0;
  1160. }
  1161. static int switch_cmp(struct work_atoms *l, struct work_atoms *r)
  1162. {
  1163. if (l->nb_atoms < r->nb_atoms)
  1164. return -1;
  1165. if (l->nb_atoms > r->nb_atoms)
  1166. return 1;
  1167. return 0;
  1168. }
  1169. static int runtime_cmp(struct work_atoms *l, struct work_atoms *r)
  1170. {
  1171. if (l->total_runtime < r->total_runtime)
  1172. return -1;
  1173. if (l->total_runtime > r->total_runtime)
  1174. return 1;
  1175. return 0;
  1176. }
  1177. static int sort_dimension__add(const char *tok, struct list_head *list)
  1178. {
  1179. size_t i;
  1180. static struct sort_dimension avg_sort_dimension = {
  1181. .name = "avg",
  1182. .cmp = avg_cmp,
  1183. };
  1184. static struct sort_dimension max_sort_dimension = {
  1185. .name = "max",
  1186. .cmp = max_cmp,
  1187. };
  1188. static struct sort_dimension pid_sort_dimension = {
  1189. .name = "pid",
  1190. .cmp = pid_cmp,
  1191. };
  1192. static struct sort_dimension runtime_sort_dimension = {
  1193. .name = "runtime",
  1194. .cmp = runtime_cmp,
  1195. };
  1196. static struct sort_dimension switch_sort_dimension = {
  1197. .name = "switch",
  1198. .cmp = switch_cmp,
  1199. };
  1200. struct sort_dimension *available_sorts[] = {
  1201. &pid_sort_dimension,
  1202. &avg_sort_dimension,
  1203. &max_sort_dimension,
  1204. &switch_sort_dimension,
  1205. &runtime_sort_dimension,
  1206. };
  1207. for (i = 0; i < ARRAY_SIZE(available_sorts); i++) {
  1208. if (!strcmp(available_sorts[i]->name, tok)) {
  1209. list_add_tail(&available_sorts[i]->list, list);
  1210. return 0;
  1211. }
  1212. }
  1213. return -1;
  1214. }
  1215. static void perf_sched__sort_lat(struct perf_sched *sched)
  1216. {
  1217. struct rb_node *node;
  1218. struct rb_root_cached *root = &sched->atom_root;
  1219. again:
  1220. for (;;) {
  1221. struct work_atoms *data;
  1222. node = rb_first_cached(root);
  1223. if (!node)
  1224. break;
  1225. rb_erase_cached(node, root);
  1226. data = rb_entry(node, struct work_atoms, node);
  1227. __thread_latency_insert(&sched->sorted_atom_root, data, &sched->sort_list);
  1228. }
  1229. if (root == &sched->atom_root) {
  1230. root = &sched->merged_atom_root;
  1231. goto again;
  1232. }
  1233. }
  1234. static int process_sched_wakeup_event(struct perf_tool *tool,
  1235. struct evsel *evsel,
  1236. struct perf_sample *sample,
  1237. struct machine *machine)
  1238. {
  1239. struct perf_sched *sched = container_of(tool, struct perf_sched, tool);
  1240. if (sched->tp_handler->wakeup_event)
  1241. return sched->tp_handler->wakeup_event(sched, evsel, sample, machine);
  1242. return 0;
  1243. }
  1244. union map_priv {
  1245. void *ptr;
  1246. bool color;
  1247. };
  1248. static bool thread__has_color(struct thread *thread)
  1249. {
  1250. union map_priv priv = {
  1251. .ptr = thread__priv(thread),
  1252. };
  1253. return priv.color;
  1254. }
  1255. static struct thread*
  1256. map__findnew_thread(struct perf_sched *sched, struct machine *machine, pid_t pid, pid_t tid)
  1257. {
  1258. struct thread *thread = machine__findnew_thread(machine, pid, tid);
  1259. union map_priv priv = {
  1260. .color = false,
  1261. };
  1262. if (!sched->map.color_pids || !thread || thread__priv(thread))
  1263. return thread;
  1264. if (thread_map__has(sched->map.color_pids, tid))
  1265. priv.color = true;
  1266. thread__set_priv(thread, priv.ptr);
  1267. return thread;
  1268. }
  1269. static int map_switch_event(struct perf_sched *sched, struct evsel *evsel,
  1270. struct perf_sample *sample, struct machine *machine)
  1271. {
  1272. const u32 next_pid = evsel__intval(evsel, sample, "next_pid");
  1273. struct thread *sched_in;
  1274. struct thread_runtime *tr;
  1275. int new_shortname;
  1276. u64 timestamp0, timestamp = sample->time;
  1277. s64 delta;
  1278. int i, this_cpu = sample->cpu;
  1279. int cpus_nr;
  1280. bool new_cpu = false;
  1281. const char *color = PERF_COLOR_NORMAL;
  1282. char stimestamp[32];
  1283. BUG_ON(this_cpu >= MAX_CPUS || this_cpu < 0);
  1284. if (this_cpu > sched->max_cpu)
  1285. sched->max_cpu = this_cpu;
  1286. if (sched->map.comp) {
  1287. cpus_nr = bitmap_weight(sched->map.comp_cpus_mask, MAX_CPUS);
  1288. if (!test_and_set_bit(this_cpu, sched->map.comp_cpus_mask)) {
  1289. sched->map.comp_cpus[cpus_nr++] = this_cpu;
  1290. new_cpu = true;
  1291. }
  1292. } else
  1293. cpus_nr = sched->max_cpu;
  1294. timestamp0 = sched->cpu_last_switched[this_cpu];
  1295. sched->cpu_last_switched[this_cpu] = timestamp;
  1296. if (timestamp0)
  1297. delta = timestamp - timestamp0;
  1298. else
  1299. delta = 0;
  1300. if (delta < 0) {
  1301. pr_err("hm, delta: %" PRIu64 " < 0 ?\n", delta);
  1302. return -1;
  1303. }
  1304. sched_in = map__findnew_thread(sched, machine, -1, next_pid);
  1305. if (sched_in == NULL)
  1306. return -1;
  1307. tr = thread__get_runtime(sched_in);
  1308. if (tr == NULL) {
  1309. thread__put(sched_in);
  1310. return -1;
  1311. }
  1312. sched->curr_thread[this_cpu] = thread__get(sched_in);
  1313. printf(" ");
  1314. new_shortname = 0;
  1315. if (!tr->shortname[0]) {
  1316. if (!strcmp(thread__comm_str(sched_in), "swapper")) {
  1317. /*
  1318. * Don't allocate a letter-number for swapper:0
  1319. * as a shortname. Instead, we use '.' for it.
  1320. */
  1321. tr->shortname[0] = '.';
  1322. tr->shortname[1] = ' ';
  1323. } else {
  1324. tr->shortname[0] = sched->next_shortname1;
  1325. tr->shortname[1] = sched->next_shortname2;
  1326. if (sched->next_shortname1 < 'Z') {
  1327. sched->next_shortname1++;
  1328. } else {
  1329. sched->next_shortname1 = 'A';
  1330. if (sched->next_shortname2 < '9')
  1331. sched->next_shortname2++;
  1332. else
  1333. sched->next_shortname2 = '0';
  1334. }
  1335. }
  1336. new_shortname = 1;
  1337. }
  1338. for (i = 0; i < cpus_nr; i++) {
  1339. int cpu = sched->map.comp ? sched->map.comp_cpus[i] : i;
  1340. struct thread *curr_thread = sched->curr_thread[cpu];
  1341. struct thread_runtime *curr_tr;
  1342. const char *pid_color = color;
  1343. const char *cpu_color = color;
  1344. if (curr_thread && thread__has_color(curr_thread))
  1345. pid_color = COLOR_PIDS;
  1346. if (sched->map.cpus && !cpu_map__has(sched->map.cpus, cpu))
  1347. continue;
  1348. if (sched->map.color_cpus && cpu_map__has(sched->map.color_cpus, cpu))
  1349. cpu_color = COLOR_CPUS;
  1350. if (cpu != this_cpu)
  1351. color_fprintf(stdout, color, " ");
  1352. else
  1353. color_fprintf(stdout, cpu_color, "*");
  1354. if (sched->curr_thread[cpu]) {
  1355. curr_tr = thread__get_runtime(sched->curr_thread[cpu]);
  1356. if (curr_tr == NULL) {
  1357. thread__put(sched_in);
  1358. return -1;
  1359. }
  1360. color_fprintf(stdout, pid_color, "%2s ", curr_tr->shortname);
  1361. } else
  1362. color_fprintf(stdout, color, " ");
  1363. }
  1364. if (sched->map.cpus && !cpu_map__has(sched->map.cpus, this_cpu))
  1365. goto out;
  1366. timestamp__scnprintf_usec(timestamp, stimestamp, sizeof(stimestamp));
  1367. color_fprintf(stdout, color, " %12s secs ", stimestamp);
  1368. if (new_shortname || tr->comm_changed || (verbose > 0 && sched_in->tid)) {
  1369. const char *pid_color = color;
  1370. if (thread__has_color(sched_in))
  1371. pid_color = COLOR_PIDS;
  1372. color_fprintf(stdout, pid_color, "%s => %s:%d",
  1373. tr->shortname, thread__comm_str(sched_in), sched_in->tid);
  1374. tr->comm_changed = false;
  1375. }
  1376. if (sched->map.comp && new_cpu)
  1377. color_fprintf(stdout, color, " (CPU %d)", this_cpu);
  1378. out:
  1379. color_fprintf(stdout, color, "\n");
  1380. thread__put(sched_in);
  1381. return 0;
  1382. }
  1383. static int process_sched_switch_event(struct perf_tool *tool,
  1384. struct evsel *evsel,
  1385. struct perf_sample *sample,
  1386. struct machine *machine)
  1387. {
  1388. struct perf_sched *sched = container_of(tool, struct perf_sched, tool);
  1389. int this_cpu = sample->cpu, err = 0;
  1390. u32 prev_pid = evsel__intval(evsel, sample, "prev_pid"),
  1391. next_pid = evsel__intval(evsel, sample, "next_pid");
  1392. if (sched->curr_pid[this_cpu] != (u32)-1) {
  1393. /*
  1394. * Are we trying to switch away a PID that is
  1395. * not current?
  1396. */
  1397. if (sched->curr_pid[this_cpu] != prev_pid)
  1398. sched->nr_context_switch_bugs++;
  1399. }
  1400. if (sched->tp_handler->switch_event)
  1401. err = sched->tp_handler->switch_event(sched, evsel, sample, machine);
  1402. sched->curr_pid[this_cpu] = next_pid;
  1403. return err;
  1404. }
  1405. static int process_sched_runtime_event(struct perf_tool *tool,
  1406. struct evsel *evsel,
  1407. struct perf_sample *sample,
  1408. struct machine *machine)
  1409. {
  1410. struct perf_sched *sched = container_of(tool, struct perf_sched, tool);
  1411. if (sched->tp_handler->runtime_event)
  1412. return sched->tp_handler->runtime_event(sched, evsel, sample, machine);
  1413. return 0;
  1414. }
  1415. static int perf_sched__process_fork_event(struct perf_tool *tool,
  1416. union perf_event *event,
  1417. struct perf_sample *sample,
  1418. struct machine *machine)
  1419. {
  1420. struct perf_sched *sched = container_of(tool, struct perf_sched, tool);
  1421. /* run the fork event through the perf machineruy */
  1422. perf_event__process_fork(tool, event, sample, machine);
  1423. /* and then run additional processing needed for this command */
  1424. if (sched->tp_handler->fork_event)
  1425. return sched->tp_handler->fork_event(sched, event, machine);
  1426. return 0;
  1427. }
  1428. static int process_sched_migrate_task_event(struct perf_tool *tool,
  1429. struct evsel *evsel,
  1430. struct perf_sample *sample,
  1431. struct machine *machine)
  1432. {
  1433. struct perf_sched *sched = container_of(tool, struct perf_sched, tool);
  1434. if (sched->tp_handler->migrate_task_event)
  1435. return sched->tp_handler->migrate_task_event(sched, evsel, sample, machine);
  1436. return 0;
  1437. }
  1438. typedef int (*tracepoint_handler)(struct perf_tool *tool,
  1439. struct evsel *evsel,
  1440. struct perf_sample *sample,
  1441. struct machine *machine);
  1442. static int perf_sched__process_tracepoint_sample(struct perf_tool *tool __maybe_unused,
  1443. union perf_event *event __maybe_unused,
  1444. struct perf_sample *sample,
  1445. struct evsel *evsel,
  1446. struct machine *machine)
  1447. {
  1448. int err = 0;
  1449. if (evsel->handler != NULL) {
  1450. tracepoint_handler f = evsel->handler;
  1451. err = f(tool, evsel, sample, machine);
  1452. }
  1453. return err;
  1454. }
  1455. static int perf_sched__process_comm(struct perf_tool *tool __maybe_unused,
  1456. union perf_event *event,
  1457. struct perf_sample *sample,
  1458. struct machine *machine)
  1459. {
  1460. struct thread *thread;
  1461. struct thread_runtime *tr;
  1462. int err;
  1463. err = perf_event__process_comm(tool, event, sample, machine);
  1464. if (err)
  1465. return err;
  1466. thread = machine__find_thread(machine, sample->pid, sample->tid);
  1467. if (!thread) {
  1468. pr_err("Internal error: can't find thread\n");
  1469. return -1;
  1470. }
  1471. tr = thread__get_runtime(thread);
  1472. if (tr == NULL) {
  1473. thread__put(thread);
  1474. return -1;
  1475. }
  1476. tr->comm_changed = true;
  1477. thread__put(thread);
  1478. return 0;
  1479. }
  1480. static int perf_sched__read_events(struct perf_sched *sched)
  1481. {
  1482. const struct evsel_str_handler handlers[] = {
  1483. { "sched:sched_switch", process_sched_switch_event, },
  1484. { "sched:sched_stat_runtime", process_sched_runtime_event, },
  1485. { "sched:sched_wakeup", process_sched_wakeup_event, },
  1486. { "sched:sched_wakeup_new", process_sched_wakeup_event, },
  1487. { "sched:sched_migrate_task", process_sched_migrate_task_event, },
  1488. };
  1489. struct perf_session *session;
  1490. struct perf_data data = {
  1491. .path = input_name,
  1492. .mode = PERF_DATA_MODE_READ,
  1493. .force = sched->force,
  1494. };
  1495. int rc = -1;
  1496. session = perf_session__new(&data, false, &sched->tool);
  1497. if (IS_ERR(session)) {
  1498. pr_debug("Error creating perf session");
  1499. return PTR_ERR(session);
  1500. }
  1501. symbol__init(&session->header.env);
  1502. if (perf_session__set_tracepoints_handlers(session, handlers))
  1503. goto out_delete;
  1504. if (perf_session__has_traces(session, "record -R")) {
  1505. int err = perf_session__process_events(session);
  1506. if (err) {
  1507. pr_err("Failed to process events, error %d", err);
  1508. goto out_delete;
  1509. }
  1510. sched->nr_events = session->evlist->stats.nr_events[0];
  1511. sched->nr_lost_events = session->evlist->stats.total_lost;
  1512. sched->nr_lost_chunks = session->evlist->stats.nr_events[PERF_RECORD_LOST];
  1513. }
  1514. rc = 0;
  1515. out_delete:
  1516. perf_session__delete(session);
  1517. return rc;
  1518. }
  1519. /*
  1520. * scheduling times are printed as msec.usec
  1521. */
  1522. static inline void print_sched_time(unsigned long long nsecs, int width)
  1523. {
  1524. unsigned long msecs;
  1525. unsigned long usecs;
  1526. msecs = nsecs / NSEC_PER_MSEC;
  1527. nsecs -= msecs * NSEC_PER_MSEC;
  1528. usecs = nsecs / NSEC_PER_USEC;
  1529. printf("%*lu.%03lu ", width, msecs, usecs);
  1530. }
  1531. /*
  1532. * returns runtime data for event, allocating memory for it the
  1533. * first time it is used.
  1534. */
  1535. static struct evsel_runtime *evsel__get_runtime(struct evsel *evsel)
  1536. {
  1537. struct evsel_runtime *r = evsel->priv;
  1538. if (r == NULL) {
  1539. r = zalloc(sizeof(struct evsel_runtime));
  1540. evsel->priv = r;
  1541. }
  1542. return r;
  1543. }
  1544. /*
  1545. * save last time event was seen per cpu
  1546. */
  1547. static void evsel__save_time(struct evsel *evsel, u64 timestamp, u32 cpu)
  1548. {
  1549. struct evsel_runtime *r = evsel__get_runtime(evsel);
  1550. if (r == NULL)
  1551. return;
  1552. if ((cpu >= r->ncpu) || (r->last_time == NULL)) {
  1553. int i, n = __roundup_pow_of_two(cpu+1);
  1554. void *p = r->last_time;
  1555. p = realloc(r->last_time, n * sizeof(u64));
  1556. if (!p)
  1557. return;
  1558. r->last_time = p;
  1559. for (i = r->ncpu; i < n; ++i)
  1560. r->last_time[i] = (u64) 0;
  1561. r->ncpu = n;
  1562. }
  1563. r->last_time[cpu] = timestamp;
  1564. }
  1565. /* returns last time this event was seen on the given cpu */
  1566. static u64 evsel__get_time(struct evsel *evsel, u32 cpu)
  1567. {
  1568. struct evsel_runtime *r = evsel__get_runtime(evsel);
  1569. if ((r == NULL) || (r->last_time == NULL) || (cpu >= r->ncpu))
  1570. return 0;
  1571. return r->last_time[cpu];
  1572. }
  1573. static int comm_width = 30;
  1574. static char *timehist_get_commstr(struct thread *thread)
  1575. {
  1576. static char str[32];
  1577. const char *comm = thread__comm_str(thread);
  1578. pid_t tid = thread->tid;
  1579. pid_t pid = thread->pid_;
  1580. int n;
  1581. if (pid == 0)
  1582. n = scnprintf(str, sizeof(str), "%s", comm);
  1583. else if (tid != pid)
  1584. n = scnprintf(str, sizeof(str), "%s[%d/%d]", comm, tid, pid);
  1585. else
  1586. n = scnprintf(str, sizeof(str), "%s[%d]", comm, tid);
  1587. if (n > comm_width)
  1588. comm_width = n;
  1589. return str;
  1590. }
  1591. static void timehist_header(struct perf_sched *sched)
  1592. {
  1593. u32 ncpus = sched->max_cpu + 1;
  1594. u32 i, j;
  1595. printf("%15s %6s ", "time", "cpu");
  1596. if (sched->show_cpu_visual) {
  1597. printf(" ");
  1598. for (i = 0, j = 0; i < ncpus; ++i) {
  1599. printf("%x", j++);
  1600. if (j > 15)
  1601. j = 0;
  1602. }
  1603. printf(" ");
  1604. }
  1605. printf(" %-*s %9s %9s %9s", comm_width,
  1606. "task name", "wait time", "sch delay", "run time");
  1607. if (sched->show_state)
  1608. printf(" %s", "state");
  1609. printf("\n");
  1610. /*
  1611. * units row
  1612. */
  1613. printf("%15s %-6s ", "", "");
  1614. if (sched->show_cpu_visual)
  1615. printf(" %*s ", ncpus, "");
  1616. printf(" %-*s %9s %9s %9s", comm_width,
  1617. "[tid/pid]", "(msec)", "(msec)", "(msec)");
  1618. if (sched->show_state)
  1619. printf(" %5s", "");
  1620. printf("\n");
  1621. /*
  1622. * separator
  1623. */
  1624. printf("%.15s %.6s ", graph_dotted_line, graph_dotted_line);
  1625. if (sched->show_cpu_visual)
  1626. printf(" %.*s ", ncpus, graph_dotted_line);
  1627. printf(" %.*s %.9s %.9s %.9s", comm_width,
  1628. graph_dotted_line, graph_dotted_line, graph_dotted_line,
  1629. graph_dotted_line);
  1630. if (sched->show_state)
  1631. printf(" %.5s", graph_dotted_line);
  1632. printf("\n");
  1633. }
  1634. static char task_state_char(struct thread *thread, int state)
  1635. {
  1636. static const char state_to_char[] = TASK_STATE_TO_CHAR_STR;
  1637. unsigned bit = state ? ffs(state) : 0;
  1638. /* 'I' for idle */
  1639. if (thread->tid == 0)
  1640. return 'I';
  1641. return bit < sizeof(state_to_char) - 1 ? state_to_char[bit] : '?';
  1642. }
  1643. static void timehist_print_sample(struct perf_sched *sched,
  1644. struct evsel *evsel,
  1645. struct perf_sample *sample,
  1646. struct addr_location *al,
  1647. struct thread *thread,
  1648. u64 t, int state)
  1649. {
  1650. struct thread_runtime *tr = thread__priv(thread);
  1651. const char *next_comm = evsel__strval(evsel, sample, "next_comm");
  1652. const u32 next_pid = evsel__intval(evsel, sample, "next_pid");
  1653. u32 max_cpus = sched->max_cpu + 1;
  1654. char tstr[64];
  1655. char nstr[30];
  1656. u64 wait_time;
  1657. if (cpu_list && !test_bit(sample->cpu, cpu_bitmap))
  1658. return;
  1659. timestamp__scnprintf_usec(t, tstr, sizeof(tstr));
  1660. printf("%15s [%04d] ", tstr, sample->cpu);
  1661. if (sched->show_cpu_visual) {
  1662. u32 i;
  1663. char c;
  1664. printf(" ");
  1665. for (i = 0; i < max_cpus; ++i) {
  1666. /* flag idle times with 'i'; others are sched events */
  1667. if (i == sample->cpu)
  1668. c = (thread->tid == 0) ? 'i' : 's';
  1669. else
  1670. c = ' ';
  1671. printf("%c", c);
  1672. }
  1673. printf(" ");
  1674. }
  1675. printf(" %-*s ", comm_width, timehist_get_commstr(thread));
  1676. wait_time = tr->dt_sleep + tr->dt_iowait + tr->dt_preempt;
  1677. print_sched_time(wait_time, 6);
  1678. print_sched_time(tr->dt_delay, 6);
  1679. print_sched_time(tr->dt_run, 6);
  1680. if (sched->show_state)
  1681. printf(" %5c ", task_state_char(thread, state));
  1682. if (sched->show_next) {
  1683. snprintf(nstr, sizeof(nstr), "next: %s[%d]", next_comm, next_pid);
  1684. printf(" %-*s", comm_width, nstr);
  1685. }
  1686. if (sched->show_wakeups && !sched->show_next)
  1687. printf(" %-*s", comm_width, "");
  1688. if (thread->tid == 0)
  1689. goto out;
  1690. if (sched->show_callchain)
  1691. printf(" ");
  1692. sample__fprintf_sym(sample, al, 0,
  1693. EVSEL__PRINT_SYM | EVSEL__PRINT_ONELINE |
  1694. EVSEL__PRINT_CALLCHAIN_ARROW |
  1695. EVSEL__PRINT_SKIP_IGNORED,
  1696. &callchain_cursor, symbol_conf.bt_stop_list, stdout);
  1697. out:
  1698. printf("\n");
  1699. }
  1700. /*
  1701. * Explanation of delta-time stats:
  1702. *
  1703. * t = time of current schedule out event
  1704. * tprev = time of previous sched out event
  1705. * also time of schedule-in event for current task
  1706. * last_time = time of last sched change event for current task
  1707. * (i.e, time process was last scheduled out)
  1708. * ready_to_run = time of wakeup for current task
  1709. *
  1710. * -----|------------|------------|------------|------
  1711. * last ready tprev t
  1712. * time to run
  1713. *
  1714. * |-------- dt_wait --------|
  1715. * |- dt_delay -|-- dt_run --|
  1716. *
  1717. * dt_run = run time of current task
  1718. * dt_wait = time between last schedule out event for task and tprev
  1719. * represents time spent off the cpu
  1720. * dt_delay = time between wakeup and schedule-in of task
  1721. */
  1722. static void timehist_update_runtime_stats(struct thread_runtime *r,
  1723. u64 t, u64 tprev)
  1724. {
  1725. r->dt_delay = 0;
  1726. r->dt_sleep = 0;
  1727. r->dt_iowait = 0;
  1728. r->dt_preempt = 0;
  1729. r->dt_run = 0;
  1730. if (tprev) {
  1731. r->dt_run = t - tprev;
  1732. if (r->ready_to_run) {
  1733. if (r->ready_to_run > tprev)
  1734. pr_debug("time travel: wakeup time for task > previous sched_switch event\n");
  1735. else
  1736. r->dt_delay = tprev - r->ready_to_run;
  1737. }
  1738. if (r->last_time > tprev)
  1739. pr_debug("time travel: last sched out time for task > previous sched_switch event\n");
  1740. else if (r->last_time) {
  1741. u64 dt_wait = tprev - r->last_time;
  1742. if (r->last_state == TASK_RUNNING)
  1743. r->dt_preempt = dt_wait;
  1744. else if (r->last_state == TASK_UNINTERRUPTIBLE)
  1745. r->dt_iowait = dt_wait;
  1746. else
  1747. r->dt_sleep = dt_wait;
  1748. }
  1749. }
  1750. update_stats(&r->run_stats, r->dt_run);
  1751. r->total_run_time += r->dt_run;
  1752. r->total_delay_time += r->dt_delay;
  1753. r->total_sleep_time += r->dt_sleep;
  1754. r->total_iowait_time += r->dt_iowait;
  1755. r->total_preempt_time += r->dt_preempt;
  1756. }
  1757. static bool is_idle_sample(struct perf_sample *sample,
  1758. struct evsel *evsel)
  1759. {
  1760. /* pid 0 == swapper == idle task */
  1761. if (strcmp(evsel__name(evsel), "sched:sched_switch") == 0)
  1762. return evsel__intval(evsel, sample, "prev_pid") == 0;
  1763. return sample->pid == 0;
  1764. }
  1765. static void save_task_callchain(struct perf_sched *sched,
  1766. struct perf_sample *sample,
  1767. struct evsel *evsel,
  1768. struct machine *machine)
  1769. {
  1770. struct callchain_cursor *cursor = &callchain_cursor;
  1771. struct thread *thread;
  1772. /* want main thread for process - has maps */
  1773. thread = machine__findnew_thread(machine, sample->pid, sample->pid);
  1774. if (thread == NULL) {
  1775. pr_debug("Failed to get thread for pid %d.\n", sample->pid);
  1776. return;
  1777. }
  1778. if (!sched->show_callchain || sample->callchain == NULL)
  1779. return;
  1780. if (thread__resolve_callchain(thread, cursor, evsel, sample,
  1781. NULL, NULL, sched->max_stack + 2) != 0) {
  1782. if (verbose > 0)
  1783. pr_err("Failed to resolve callchain. Skipping\n");
  1784. return;
  1785. }
  1786. callchain_cursor_commit(cursor);
  1787. while (true) {
  1788. struct callchain_cursor_node *node;
  1789. struct symbol *sym;
  1790. node = callchain_cursor_current(cursor);
  1791. if (node == NULL)
  1792. break;
  1793. sym = node->ms.sym;
  1794. if (sym) {
  1795. if (!strcmp(sym->name, "schedule") ||
  1796. !strcmp(sym->name, "__schedule") ||
  1797. !strcmp(sym->name, "preempt_schedule"))
  1798. sym->ignore = 1;
  1799. }
  1800. callchain_cursor_advance(cursor);
  1801. }
  1802. }
  1803. static int init_idle_thread(struct thread *thread)
  1804. {
  1805. struct idle_thread_runtime *itr;
  1806. thread__set_comm(thread, idle_comm, 0);
  1807. itr = zalloc(sizeof(*itr));
  1808. if (itr == NULL)
  1809. return -ENOMEM;
  1810. init_stats(&itr->tr.run_stats);
  1811. callchain_init(&itr->callchain);
  1812. callchain_cursor_reset(&itr->cursor);
  1813. thread__set_priv(thread, itr);
  1814. return 0;
  1815. }
  1816. /*
  1817. * Track idle stats per cpu by maintaining a local thread
  1818. * struct for the idle task on each cpu.
  1819. */
  1820. static int init_idle_threads(int ncpu)
  1821. {
  1822. int i, ret;
  1823. idle_threads = zalloc(ncpu * sizeof(struct thread *));
  1824. if (!idle_threads)
  1825. return -ENOMEM;
  1826. idle_max_cpu = ncpu;
  1827. /* allocate the actual thread struct if needed */
  1828. for (i = 0; i < ncpu; ++i) {
  1829. idle_threads[i] = thread__new(0, 0);
  1830. if (idle_threads[i] == NULL)
  1831. return -ENOMEM;
  1832. ret = init_idle_thread(idle_threads[i]);
  1833. if (ret < 0)
  1834. return ret;
  1835. }
  1836. return 0;
  1837. }
  1838. static void free_idle_threads(void)
  1839. {
  1840. int i;
  1841. if (idle_threads == NULL)
  1842. return;
  1843. for (i = 0; i < idle_max_cpu; ++i) {
  1844. if ((idle_threads[i]))
  1845. thread__delete(idle_threads[i]);
  1846. }
  1847. free(idle_threads);
  1848. }
  1849. static struct thread *get_idle_thread(int cpu)
  1850. {
  1851. /*
  1852. * expand/allocate array of pointers to local thread
  1853. * structs if needed
  1854. */
  1855. if ((cpu >= idle_max_cpu) || (idle_threads == NULL)) {
  1856. int i, j = __roundup_pow_of_two(cpu+1);
  1857. void *p;
  1858. p = realloc(idle_threads, j * sizeof(struct thread *));
  1859. if (!p)
  1860. return NULL;
  1861. idle_threads = (struct thread **) p;
  1862. for (i = idle_max_cpu; i < j; ++i)
  1863. idle_threads[i] = NULL;
  1864. idle_max_cpu = j;
  1865. }
  1866. /* allocate a new thread struct if needed */
  1867. if (idle_threads[cpu] == NULL) {
  1868. idle_threads[cpu] = thread__new(0, 0);
  1869. if (idle_threads[cpu]) {
  1870. if (init_idle_thread(idle_threads[cpu]) < 0)
  1871. return NULL;
  1872. }
  1873. }
  1874. return idle_threads[cpu];
  1875. }
  1876. static void save_idle_callchain(struct perf_sched *sched,
  1877. struct idle_thread_runtime *itr,
  1878. struct perf_sample *sample)
  1879. {
  1880. if (!sched->show_callchain || sample->callchain == NULL)
  1881. return;
  1882. callchain_cursor__copy(&itr->cursor, &callchain_cursor);
  1883. }
  1884. static struct thread *timehist_get_thread(struct perf_sched *sched,
  1885. struct perf_sample *sample,
  1886. struct machine *machine,
  1887. struct evsel *evsel)
  1888. {
  1889. struct thread *thread;
  1890. if (is_idle_sample(sample, evsel)) {
  1891. thread = get_idle_thread(sample->cpu);
  1892. if (thread == NULL)
  1893. pr_err("Failed to get idle thread for cpu %d.\n", sample->cpu);
  1894. } else {
  1895. /* there were samples with tid 0 but non-zero pid */
  1896. thread = machine__findnew_thread(machine, sample->pid,
  1897. sample->tid ?: sample->pid);
  1898. if (thread == NULL) {
  1899. pr_debug("Failed to get thread for tid %d. skipping sample.\n",
  1900. sample->tid);
  1901. }
  1902. save_task_callchain(sched, sample, evsel, machine);
  1903. if (sched->idle_hist) {
  1904. struct thread *idle;
  1905. struct idle_thread_runtime *itr;
  1906. idle = get_idle_thread(sample->cpu);
  1907. if (idle == NULL) {
  1908. pr_err("Failed to get idle thread for cpu %d.\n", sample->cpu);
  1909. return NULL;
  1910. }
  1911. itr = thread__priv(idle);
  1912. if (itr == NULL)
  1913. return NULL;
  1914. itr->last_thread = thread;
  1915. /* copy task callchain when entering to idle */
  1916. if (evsel__intval(evsel, sample, "next_pid") == 0)
  1917. save_idle_callchain(sched, itr, sample);
  1918. }
  1919. }
  1920. return thread;
  1921. }
  1922. static bool timehist_skip_sample(struct perf_sched *sched,
  1923. struct thread *thread,
  1924. struct evsel *evsel,
  1925. struct perf_sample *sample)
  1926. {
  1927. bool rc = false;
  1928. if (thread__is_filtered(thread)) {
  1929. rc = true;
  1930. sched->skipped_samples++;
  1931. }
  1932. if (sched->idle_hist) {
  1933. if (strcmp(evsel__name(evsel), "sched:sched_switch"))
  1934. rc = true;
  1935. else if (evsel__intval(evsel, sample, "prev_pid") != 0 &&
  1936. evsel__intval(evsel, sample, "next_pid") != 0)
  1937. rc = true;
  1938. }
  1939. return rc;
  1940. }
  1941. static void timehist_print_wakeup_event(struct perf_sched *sched,
  1942. struct evsel *evsel,
  1943. struct perf_sample *sample,
  1944. struct machine *machine,
  1945. struct thread *awakened)
  1946. {
  1947. struct thread *thread;
  1948. char tstr[64];
  1949. thread = machine__findnew_thread(machine, sample->pid, sample->tid);
  1950. if (thread == NULL)
  1951. return;
  1952. /* show wakeup unless both awakee and awaker are filtered */
  1953. if (timehist_skip_sample(sched, thread, evsel, sample) &&
  1954. timehist_skip_sample(sched, awakened, evsel, sample)) {
  1955. return;
  1956. }
  1957. timestamp__scnprintf_usec(sample->time, tstr, sizeof(tstr));
  1958. printf("%15s [%04d] ", tstr, sample->cpu);
  1959. if (sched->show_cpu_visual)
  1960. printf(" %*s ", sched->max_cpu + 1, "");
  1961. printf(" %-*s ", comm_width, timehist_get_commstr(thread));
  1962. /* dt spacer */
  1963. printf(" %9s %9s %9s ", "", "", "");
  1964. printf("awakened: %s", timehist_get_commstr(awakened));
  1965. printf("\n");
  1966. }
  1967. static int timehist_sched_wakeup_ignore(struct perf_tool *tool __maybe_unused,
  1968. union perf_event *event __maybe_unused,
  1969. struct evsel *evsel __maybe_unused,
  1970. struct perf_sample *sample __maybe_unused,
  1971. struct machine *machine __maybe_unused)
  1972. {
  1973. return 0;
  1974. }
  1975. static int timehist_sched_wakeup_event(struct perf_tool *tool,
  1976. union perf_event *event __maybe_unused,
  1977. struct evsel *evsel,
  1978. struct perf_sample *sample,
  1979. struct machine *machine)
  1980. {
  1981. struct perf_sched *sched = container_of(tool, struct perf_sched, tool);
  1982. struct thread *thread;
  1983. struct thread_runtime *tr = NULL;
  1984. /* want pid of awakened task not pid in sample */
  1985. const u32 pid = evsel__intval(evsel, sample, "pid");
  1986. thread = machine__findnew_thread(machine, 0, pid);
  1987. if (thread == NULL)
  1988. return -1;
  1989. tr = thread__get_runtime(thread);
  1990. if (tr == NULL)
  1991. return -1;
  1992. if (tr->ready_to_run == 0)
  1993. tr->ready_to_run = sample->time;
  1994. /* show wakeups if requested */
  1995. if (sched->show_wakeups &&
  1996. !perf_time__skip_sample(&sched->ptime, sample->time))
  1997. timehist_print_wakeup_event(sched, evsel, sample, machine, thread);
  1998. return 0;
  1999. }
  2000. static void timehist_print_migration_event(struct perf_sched *sched,
  2001. struct evsel *evsel,
  2002. struct perf_sample *sample,
  2003. struct machine *machine,
  2004. struct thread *migrated)
  2005. {
  2006. struct thread *thread;
  2007. char tstr[64];
  2008. u32 max_cpus = sched->max_cpu + 1;
  2009. u32 ocpu, dcpu;
  2010. if (sched->summary_only)
  2011. return;
  2012. max_cpus = sched->max_cpu + 1;
  2013. ocpu = evsel__intval(evsel, sample, "orig_cpu");
  2014. dcpu = evsel__intval(evsel, sample, "dest_cpu");
  2015. thread = machine__findnew_thread(machine, sample->pid, sample->tid);
  2016. if (thread == NULL)
  2017. return;
  2018. if (timehist_skip_sample(sched, thread, evsel, sample) &&
  2019. timehist_skip_sample(sched, migrated, evsel, sample)) {
  2020. return;
  2021. }
  2022. timestamp__scnprintf_usec(sample->time, tstr, sizeof(tstr));
  2023. printf("%15s [%04d] ", tstr, sample->cpu);
  2024. if (sched->show_cpu_visual) {
  2025. u32 i;
  2026. char c;
  2027. printf(" ");
  2028. for (i = 0; i < max_cpus; ++i) {
  2029. c = (i == sample->cpu) ? 'm' : ' ';
  2030. printf("%c", c);
  2031. }
  2032. printf(" ");
  2033. }
  2034. printf(" %-*s ", comm_width, timehist_get_commstr(thread));
  2035. /* dt spacer */
  2036. printf(" %9s %9s %9s ", "", "", "");
  2037. printf("migrated: %s", timehist_get_commstr(migrated));
  2038. printf(" cpu %d => %d", ocpu, dcpu);
  2039. printf("\n");
  2040. }
  2041. static int timehist_migrate_task_event(struct perf_tool *tool,
  2042. union perf_event *event __maybe_unused,
  2043. struct evsel *evsel,
  2044. struct perf_sample *sample,
  2045. struct machine *machine)
  2046. {
  2047. struct perf_sched *sched = container_of(tool, struct perf_sched, tool);
  2048. struct thread *thread;
  2049. struct thread_runtime *tr = NULL;
  2050. /* want pid of migrated task not pid in sample */
  2051. const u32 pid = evsel__intval(evsel, sample, "pid");
  2052. thread = machine__findnew_thread(machine, 0, pid);
  2053. if (thread == NULL)
  2054. return -1;
  2055. tr = thread__get_runtime(thread);
  2056. if (tr == NULL)
  2057. return -1;
  2058. tr->migrations++;
  2059. /* show migrations if requested */
  2060. timehist_print_migration_event(sched, evsel, sample, machine, thread);
  2061. return 0;
  2062. }
  2063. static int timehist_sched_change_event(struct perf_tool *tool,
  2064. union perf_event *event,
  2065. struct evsel *evsel,
  2066. struct perf_sample *sample,
  2067. struct machine *machine)
  2068. {
  2069. struct perf_sched *sched = container_of(tool, struct perf_sched, tool);
  2070. struct perf_time_interval *ptime = &sched->ptime;
  2071. struct addr_location al;
  2072. struct thread *thread;
  2073. struct thread_runtime *tr = NULL;
  2074. u64 tprev, t = sample->time;
  2075. int rc = 0;
  2076. int state = evsel__intval(evsel, sample, "prev_state");
  2077. if (machine__resolve(machine, &al, sample) < 0) {
  2078. pr_err("problem processing %d event. skipping it\n",
  2079. event->header.type);
  2080. rc = -1;
  2081. goto out;
  2082. }
  2083. thread = timehist_get_thread(sched, sample, machine, evsel);
  2084. if (thread == NULL) {
  2085. rc = -1;
  2086. goto out;
  2087. }
  2088. if (timehist_skip_sample(sched, thread, evsel, sample))
  2089. goto out;
  2090. tr = thread__get_runtime(thread);
  2091. if (tr == NULL) {
  2092. rc = -1;
  2093. goto out;
  2094. }
  2095. tprev = evsel__get_time(evsel, sample->cpu);
  2096. /*
  2097. * If start time given:
  2098. * - sample time is under window user cares about - skip sample
  2099. * - tprev is under window user cares about - reset to start of window
  2100. */
  2101. if (ptime->start && ptime->start > t)
  2102. goto out;
  2103. if (tprev && ptime->start > tprev)
  2104. tprev = ptime->start;
  2105. /*
  2106. * If end time given:
  2107. * - previous sched event is out of window - we are done
  2108. * - sample time is beyond window user cares about - reset it
  2109. * to close out stats for time window interest
  2110. */
  2111. if (ptime->end) {
  2112. if (tprev > ptime->end)
  2113. goto out;
  2114. if (t > ptime->end)
  2115. t = ptime->end;
  2116. }
  2117. if (!sched->idle_hist || thread->tid == 0) {
  2118. if (!cpu_list || test_bit(sample->cpu, cpu_bitmap))
  2119. timehist_update_runtime_stats(tr, t, tprev);
  2120. if (sched->idle_hist) {
  2121. struct idle_thread_runtime *itr = (void *)tr;
  2122. struct thread_runtime *last_tr;
  2123. BUG_ON(thread->tid != 0);
  2124. if (itr->last_thread == NULL)
  2125. goto out;
  2126. /* add current idle time as last thread's runtime */
  2127. last_tr = thread__get_runtime(itr->last_thread);
  2128. if (last_tr == NULL)
  2129. goto out;
  2130. timehist_update_runtime_stats(last_tr, t, tprev);
  2131. /*
  2132. * remove delta time of last thread as it's not updated
  2133. * and otherwise it will show an invalid value next
  2134. * time. we only care total run time and run stat.
  2135. */
  2136. last_tr->dt_run = 0;
  2137. last_tr->dt_delay = 0;
  2138. last_tr->dt_sleep = 0;
  2139. last_tr->dt_iowait = 0;
  2140. last_tr->dt_preempt = 0;
  2141. if (itr->cursor.nr)
  2142. callchain_append(&itr->callchain, &itr->cursor, t - tprev);
  2143. itr->last_thread = NULL;
  2144. }
  2145. }
  2146. if (!sched->summary_only)
  2147. timehist_print_sample(sched, evsel, sample, &al, thread, t, state);
  2148. out:
  2149. if (sched->hist_time.start == 0 && t >= ptime->start)
  2150. sched->hist_time.start = t;
  2151. if (ptime->end == 0 || t <= ptime->end)
  2152. sched->hist_time.end = t;
  2153. if (tr) {
  2154. /* time of this sched_switch event becomes last time task seen */
  2155. tr->last_time = sample->time;
  2156. /* last state is used to determine where to account wait time */
  2157. tr->last_state = state;
  2158. /* sched out event for task so reset ready to run time */
  2159. tr->ready_to_run = 0;
  2160. }
  2161. evsel__save_time(evsel, sample->time, sample->cpu);
  2162. return rc;
  2163. }
  2164. static int timehist_sched_switch_event(struct perf_tool *tool,
  2165. union perf_event *event,
  2166. struct evsel *evsel,
  2167. struct perf_sample *sample,
  2168. struct machine *machine __maybe_unused)
  2169. {
  2170. return timehist_sched_change_event(tool, event, evsel, sample, machine);
  2171. }
  2172. static int process_lost(struct perf_tool *tool __maybe_unused,
  2173. union perf_event *event,
  2174. struct perf_sample *sample,
  2175. struct machine *machine __maybe_unused)
  2176. {
  2177. char tstr[64];
  2178. timestamp__scnprintf_usec(sample->time, tstr, sizeof(tstr));
  2179. printf("%15s ", tstr);
  2180. printf("lost %" PRI_lu64 " events on cpu %d\n", event->lost.lost, sample->cpu);
  2181. return 0;
  2182. }
  2183. static void print_thread_runtime(struct thread *t,
  2184. struct thread_runtime *r)
  2185. {
  2186. double mean = avg_stats(&r->run_stats);
  2187. float stddev;
  2188. printf("%*s %5d %9" PRIu64 " ",
  2189. comm_width, timehist_get_commstr(t), t->ppid,
  2190. (u64) r->run_stats.n);
  2191. print_sched_time(r->total_run_time, 8);
  2192. stddev = rel_stddev_stats(stddev_stats(&r->run_stats), mean);
  2193. print_sched_time(r->run_stats.min, 6);
  2194. printf(" ");
  2195. print_sched_time((u64) mean, 6);
  2196. printf(" ");
  2197. print_sched_time(r->run_stats.max, 6);
  2198. printf(" ");
  2199. printf("%5.2f", stddev);
  2200. printf(" %5" PRIu64, r->migrations);
  2201. printf("\n");
  2202. }
  2203. static void print_thread_waittime(struct thread *t,
  2204. struct thread_runtime *r)
  2205. {
  2206. printf("%*s %5d %9" PRIu64 " ",
  2207. comm_width, timehist_get_commstr(t), t->ppid,
  2208. (u64) r->run_stats.n);
  2209. print_sched_time(r->total_run_time, 8);
  2210. print_sched_time(r->total_sleep_time, 6);
  2211. printf(" ");
  2212. print_sched_time(r->total_iowait_time, 6);
  2213. printf(" ");
  2214. print_sched_time(r->total_preempt_time, 6);
  2215. printf(" ");
  2216. print_sched_time(r->total_delay_time, 6);
  2217. printf("\n");
  2218. }
  2219. struct total_run_stats {
  2220. struct perf_sched *sched;
  2221. u64 sched_count;
  2222. u64 task_count;
  2223. u64 total_run_time;
  2224. };
  2225. static int __show_thread_runtime(struct thread *t, void *priv)
  2226. {
  2227. struct total_run_stats *stats = priv;
  2228. struct thread_runtime *r;
  2229. if (thread__is_filtered(t))
  2230. return 0;
  2231. r = thread__priv(t);
  2232. if (r && r->run_stats.n) {
  2233. stats->task_count++;
  2234. stats->sched_count += r->run_stats.n;
  2235. stats->total_run_time += r->total_run_time;
  2236. if (stats->sched->show_state)
  2237. print_thread_waittime(t, r);
  2238. else
  2239. print_thread_runtime(t, r);
  2240. }
  2241. return 0;
  2242. }
  2243. static int show_thread_runtime(struct thread *t, void *priv)
  2244. {
  2245. if (t->dead)
  2246. return 0;
  2247. return __show_thread_runtime(t, priv);
  2248. }
  2249. static int show_deadthread_runtime(struct thread *t, void *priv)
  2250. {
  2251. if (!t->dead)
  2252. return 0;
  2253. return __show_thread_runtime(t, priv);
  2254. }
  2255. static size_t callchain__fprintf_folded(FILE *fp, struct callchain_node *node)
  2256. {
  2257. const char *sep = " <- ";
  2258. struct callchain_list *chain;
  2259. size_t ret = 0;
  2260. char bf[1024];
  2261. bool first;
  2262. if (node == NULL)
  2263. return 0;
  2264. ret = callchain__fprintf_folded(fp, node->parent);
  2265. first = (ret == 0);
  2266. list_for_each_entry(chain, &node->val, list) {
  2267. if (chain->ip >= PERF_CONTEXT_MAX)
  2268. continue;
  2269. if (chain->ms.sym && chain->ms.sym->ignore)
  2270. continue;
  2271. ret += fprintf(fp, "%s%s", first ? "" : sep,
  2272. callchain_list__sym_name(chain, bf, sizeof(bf),
  2273. false));
  2274. first = false;
  2275. }
  2276. return ret;
  2277. }
  2278. static size_t timehist_print_idlehist_callchain(struct rb_root_cached *root)
  2279. {
  2280. size_t ret = 0;
  2281. FILE *fp = stdout;
  2282. struct callchain_node *chain;
  2283. struct rb_node *rb_node = rb_first_cached(root);
  2284. printf(" %16s %8s %s\n", "Idle time (msec)", "Count", "Callchains");
  2285. printf(" %.16s %.8s %.50s\n", graph_dotted_line, graph_dotted_line,
  2286. graph_dotted_line);
  2287. while (rb_node) {
  2288. chain = rb_entry(rb_node, struct callchain_node, rb_node);
  2289. rb_node = rb_next(rb_node);
  2290. ret += fprintf(fp, " ");
  2291. print_sched_time(chain->hit, 12);
  2292. ret += 16; /* print_sched_time returns 2nd arg + 4 */
  2293. ret += fprintf(fp, " %8d ", chain->count);
  2294. ret += callchain__fprintf_folded(fp, chain);
  2295. ret += fprintf(fp, "\n");
  2296. }
  2297. return ret;
  2298. }
  2299. static void timehist_print_summary(struct perf_sched *sched,
  2300. struct perf_session *session)
  2301. {
  2302. struct machine *m = &session->machines.host;
  2303. struct total_run_stats totals;
  2304. u64 task_count;
  2305. struct thread *t;
  2306. struct thread_runtime *r;
  2307. int i;
  2308. u64 hist_time = sched->hist_time.end - sched->hist_time.start;
  2309. memset(&totals, 0, sizeof(totals));
  2310. totals.sched = sched;
  2311. if (sched->idle_hist) {
  2312. printf("\nIdle-time summary\n");
  2313. printf("%*s parent sched-out ", comm_width, "comm");
  2314. printf(" idle-time min-idle avg-idle max-idle stddev migrations\n");
  2315. } else if (sched->show_state) {
  2316. printf("\nWait-time summary\n");
  2317. printf("%*s parent sched-in ", comm_width, "comm");
  2318. printf(" run-time sleep iowait preempt delay\n");
  2319. } else {
  2320. printf("\nRuntime summary\n");
  2321. printf("%*s parent sched-in ", comm_width, "comm");
  2322. printf(" run-time min-run avg-run max-run stddev migrations\n");
  2323. }
  2324. printf("%*s (count) ", comm_width, "");
  2325. printf(" (msec) (msec) (msec) (msec) %s\n",
  2326. sched->show_state ? "(msec)" : "%");
  2327. printf("%.117s\n", graph_dotted_line);
  2328. machine__for_each_thread(m, show_thread_runtime, &totals);
  2329. task_count = totals.task_count;
  2330. if (!task_count)
  2331. printf("<no still running tasks>\n");
  2332. printf("\nTerminated tasks:\n");
  2333. machine__for_each_thread(m, show_deadthread_runtime, &totals);
  2334. if (task_count == totals.task_count)
  2335. printf("<no terminated tasks>\n");
  2336. /* CPU idle stats not tracked when samples were skipped */
  2337. if (sched->skipped_samples && !sched->idle_hist)
  2338. return;
  2339. printf("\nIdle stats:\n");
  2340. for (i = 0; i < idle_max_cpu; ++i) {
  2341. if (cpu_list && !test_bit(i, cpu_bitmap))
  2342. continue;
  2343. t = idle_threads[i];
  2344. if (!t)
  2345. continue;
  2346. r = thread__priv(t);
  2347. if (r && r->run_stats.n) {
  2348. totals.sched_count += r->run_stats.n;
  2349. printf(" CPU %2d idle for ", i);
  2350. print_sched_time(r->total_run_time, 6);
  2351. printf(" msec (%6.2f%%)\n", 100.0 * r->total_run_time / hist_time);
  2352. } else
  2353. printf(" CPU %2d idle entire time window\n", i);
  2354. }
  2355. if (sched->idle_hist && sched->show_callchain) {
  2356. callchain_param.mode = CHAIN_FOLDED;
  2357. callchain_param.value = CCVAL_PERIOD;
  2358. callchain_register_param(&callchain_param);
  2359. printf("\nIdle stats by callchain:\n");
  2360. for (i = 0; i < idle_max_cpu; ++i) {
  2361. struct idle_thread_runtime *itr;
  2362. t = idle_threads[i];
  2363. if (!t)
  2364. continue;
  2365. itr = thread__priv(t);
  2366. if (itr == NULL)
  2367. continue;
  2368. callchain_param.sort(&itr->sorted_root.rb_root, &itr->callchain,
  2369. 0, &callchain_param);
  2370. printf(" CPU %2d:", i);
  2371. print_sched_time(itr->tr.total_run_time, 6);
  2372. printf(" msec\n");
  2373. timehist_print_idlehist_callchain(&itr->sorted_root);
  2374. printf("\n");
  2375. }
  2376. }
  2377. printf("\n"
  2378. " Total number of unique tasks: %" PRIu64 "\n"
  2379. "Total number of context switches: %" PRIu64 "\n",
  2380. totals.task_count, totals.sched_count);
  2381. printf(" Total run time (msec): ");
  2382. print_sched_time(totals.total_run_time, 2);
  2383. printf("\n");
  2384. printf(" Total scheduling time (msec): ");
  2385. print_sched_time(hist_time, 2);
  2386. printf(" (x %d)\n", sched->max_cpu);
  2387. }
  2388. typedef int (*sched_handler)(struct perf_tool *tool,
  2389. union perf_event *event,
  2390. struct evsel *evsel,
  2391. struct perf_sample *sample,
  2392. struct machine *machine);
  2393. static int perf_timehist__process_sample(struct perf_tool *tool,
  2394. union perf_event *event,
  2395. struct perf_sample *sample,
  2396. struct evsel *evsel,
  2397. struct machine *machine)
  2398. {
  2399. struct perf_sched *sched = container_of(tool, struct perf_sched, tool);
  2400. int err = 0;
  2401. int this_cpu = sample->cpu;
  2402. if (this_cpu > sched->max_cpu)
  2403. sched->max_cpu = this_cpu;
  2404. if (evsel->handler != NULL) {
  2405. sched_handler f = evsel->handler;
  2406. err = f(tool, event, evsel, sample, machine);
  2407. }
  2408. return err;
  2409. }
  2410. static int timehist_check_attr(struct perf_sched *sched,
  2411. struct evlist *evlist)
  2412. {
  2413. struct evsel *evsel;
  2414. struct evsel_runtime *er;
  2415. list_for_each_entry(evsel, &evlist->core.entries, core.node) {
  2416. er = evsel__get_runtime(evsel);
  2417. if (er == NULL) {
  2418. pr_err("Failed to allocate memory for evsel runtime data\n");
  2419. return -1;
  2420. }
  2421. if (sched->show_callchain && !evsel__has_callchain(evsel)) {
  2422. pr_info("Samples do not have callchains.\n");
  2423. sched->show_callchain = 0;
  2424. symbol_conf.use_callchain = 0;
  2425. }
  2426. }
  2427. return 0;
  2428. }
  2429. static int perf_sched__timehist(struct perf_sched *sched)
  2430. {
  2431. struct evsel_str_handler handlers[] = {
  2432. { "sched:sched_switch", timehist_sched_switch_event, },
  2433. { "sched:sched_wakeup", timehist_sched_wakeup_event, },
  2434. { "sched:sched_waking", timehist_sched_wakeup_event, },
  2435. { "sched:sched_wakeup_new", timehist_sched_wakeup_event, },
  2436. };
  2437. const struct evsel_str_handler migrate_handlers[] = {
  2438. { "sched:sched_migrate_task", timehist_migrate_task_event, },
  2439. };
  2440. struct perf_data data = {
  2441. .path = input_name,
  2442. .mode = PERF_DATA_MODE_READ,
  2443. .force = sched->force,
  2444. };
  2445. struct perf_session *session;
  2446. struct evlist *evlist;
  2447. int err = -1;
  2448. /*
  2449. * event handlers for timehist option
  2450. */
  2451. sched->tool.sample = perf_timehist__process_sample;
  2452. sched->tool.mmap = perf_event__process_mmap;
  2453. sched->tool.comm = perf_event__process_comm;
  2454. sched->tool.exit = perf_event__process_exit;
  2455. sched->tool.fork = perf_event__process_fork;
  2456. sched->tool.lost = process_lost;
  2457. sched->tool.attr = perf_event__process_attr;
  2458. sched->tool.tracing_data = perf_event__process_tracing_data;
  2459. sched->tool.build_id = perf_event__process_build_id;
  2460. sched->tool.ordered_events = true;
  2461. sched->tool.ordering_requires_timestamps = true;
  2462. symbol_conf.use_callchain = sched->show_callchain;
  2463. session = perf_session__new(&data, false, &sched->tool);
  2464. if (IS_ERR(session))
  2465. return PTR_ERR(session);
  2466. if (cpu_list) {
  2467. err = perf_session__cpu_bitmap(session, cpu_list, cpu_bitmap);
  2468. if (err < 0)
  2469. goto out;
  2470. }
  2471. evlist = session->evlist;
  2472. symbol__init(&session->header.env);
  2473. if (perf_time__parse_str(&sched->ptime, sched->time_str) != 0) {
  2474. pr_err("Invalid time string\n");
  2475. return -EINVAL;
  2476. }
  2477. if (timehist_check_attr(sched, evlist) != 0)
  2478. goto out;
  2479. setup_pager();
  2480. /* prefer sched_waking if it is captured */
  2481. if (perf_evlist__find_tracepoint_by_name(session->evlist,
  2482. "sched:sched_waking"))
  2483. handlers[1].handler = timehist_sched_wakeup_ignore;
  2484. /* setup per-evsel handlers */
  2485. if (perf_session__set_tracepoints_handlers(session, handlers))
  2486. goto out;
  2487. /* sched_switch event at a minimum needs to exist */
  2488. if (!perf_evlist__find_tracepoint_by_name(session->evlist,
  2489. "sched:sched_switch")) {
  2490. pr_err("No sched_switch events found. Have you run 'perf sched record'?\n");
  2491. goto out;
  2492. }
  2493. if (sched->show_migrations &&
  2494. perf_session__set_tracepoints_handlers(session, migrate_handlers))
  2495. goto out;
  2496. /* pre-allocate struct for per-CPU idle stats */
  2497. sched->max_cpu = session->header.env.nr_cpus_online;
  2498. if (sched->max_cpu == 0)
  2499. sched->max_cpu = 4;
  2500. if (init_idle_threads(sched->max_cpu))
  2501. goto out;
  2502. /* summary_only implies summary option, but don't overwrite summary if set */
  2503. if (sched->summary_only)
  2504. sched->summary = sched->summary_only;
  2505. if (!sched->summary_only)
  2506. timehist_header(sched);
  2507. err = perf_session__process_events(session);
  2508. if (err) {
  2509. pr_err("Failed to process events, error %d", err);
  2510. goto out;
  2511. }
  2512. sched->nr_events = evlist->stats.nr_events[0];
  2513. sched->nr_lost_events = evlist->stats.total_lost;
  2514. sched->nr_lost_chunks = evlist->stats.nr_events[PERF_RECORD_LOST];
  2515. if (sched->summary)
  2516. timehist_print_summary(sched, session);
  2517. out:
  2518. free_idle_threads();
  2519. perf_session__delete(session);
  2520. return err;
  2521. }
  2522. static void print_bad_events(struct perf_sched *sched)
  2523. {
  2524. if (sched->nr_unordered_timestamps && sched->nr_timestamps) {
  2525. printf(" INFO: %.3f%% unordered timestamps (%ld out of %ld)\n",
  2526. (double)sched->nr_unordered_timestamps/(double)sched->nr_timestamps*100.0,
  2527. sched->nr_unordered_timestamps, sched->nr_timestamps);
  2528. }
  2529. if (sched->nr_lost_events && sched->nr_events) {
  2530. printf(" INFO: %.3f%% lost events (%ld out of %ld, in %ld chunks)\n",
  2531. (double)sched->nr_lost_events/(double)sched->nr_events * 100.0,
  2532. sched->nr_lost_events, sched->nr_events, sched->nr_lost_chunks);
  2533. }
  2534. if (sched->nr_context_switch_bugs && sched->nr_timestamps) {
  2535. printf(" INFO: %.3f%% context switch bugs (%ld out of %ld)",
  2536. (double)sched->nr_context_switch_bugs/(double)sched->nr_timestamps*100.0,
  2537. sched->nr_context_switch_bugs, sched->nr_timestamps);
  2538. if (sched->nr_lost_events)
  2539. printf(" (due to lost events?)");
  2540. printf("\n");
  2541. }
  2542. }
  2543. static void __merge_work_atoms(struct rb_root_cached *root, struct work_atoms *data)
  2544. {
  2545. struct rb_node **new = &(root->rb_root.rb_node), *parent = NULL;
  2546. struct work_atoms *this;
  2547. const char *comm = thread__comm_str(data->thread), *this_comm;
  2548. bool leftmost = true;
  2549. while (*new) {
  2550. int cmp;
  2551. this = container_of(*new, struct work_atoms, node);
  2552. parent = *new;
  2553. this_comm = thread__comm_str(this->thread);
  2554. cmp = strcmp(comm, this_comm);
  2555. if (cmp > 0) {
  2556. new = &((*new)->rb_left);
  2557. } else if (cmp < 0) {
  2558. new = &((*new)->rb_right);
  2559. leftmost = false;
  2560. } else {
  2561. this->num_merged++;
  2562. this->total_runtime += data->total_runtime;
  2563. this->nb_atoms += data->nb_atoms;
  2564. this->total_lat += data->total_lat;
  2565. list_splice(&data->work_list, &this->work_list);
  2566. if (this->max_lat < data->max_lat) {
  2567. this->max_lat = data->max_lat;
  2568. this->max_lat_start = data->max_lat_start;
  2569. this->max_lat_end = data->max_lat_end;
  2570. }
  2571. zfree(&data);
  2572. return;
  2573. }
  2574. }
  2575. data->num_merged++;
  2576. rb_link_node(&data->node, parent, new);
  2577. rb_insert_color_cached(&data->node, root, leftmost);
  2578. }
  2579. static void perf_sched__merge_lat(struct perf_sched *sched)
  2580. {
  2581. struct work_atoms *data;
  2582. struct rb_node *node;
  2583. if (sched->skip_merge)
  2584. return;
  2585. while ((node = rb_first_cached(&sched->atom_root))) {
  2586. rb_erase_cached(node, &sched->atom_root);
  2587. data = rb_entry(node, struct work_atoms, node);
  2588. __merge_work_atoms(&sched->merged_atom_root, data);
  2589. }
  2590. }
  2591. static int perf_sched__lat(struct perf_sched *sched)
  2592. {
  2593. struct rb_node *next;
  2594. setup_pager();
  2595. if (perf_sched__read_events(sched))
  2596. return -1;
  2597. perf_sched__merge_lat(sched);
  2598. perf_sched__sort_lat(sched);
  2599. printf("\n -------------------------------------------------------------------------------------------------------------------------------------------\n");
  2600. printf(" Task | Runtime ms | Switches | Avg delay ms | Max delay ms | Max delay start | Max delay end |\n");
  2601. printf(" -------------------------------------------------------------------------------------------------------------------------------------------\n");
  2602. next = rb_first_cached(&sched->sorted_atom_root);
  2603. while (next) {
  2604. struct work_atoms *work_list;
  2605. work_list = rb_entry(next, struct work_atoms, node);
  2606. output_lat_thread(sched, work_list);
  2607. next = rb_next(next);
  2608. thread__zput(work_list->thread);
  2609. }
  2610. printf(" -----------------------------------------------------------------------------------------------------------------\n");
  2611. printf(" TOTAL: |%11.3f ms |%9" PRIu64 " |\n",
  2612. (double)sched->all_runtime / NSEC_PER_MSEC, sched->all_count);
  2613. printf(" ---------------------------------------------------\n");
  2614. print_bad_events(sched);
  2615. printf("\n");
  2616. return 0;
  2617. }
  2618. static int setup_map_cpus(struct perf_sched *sched)
  2619. {
  2620. struct perf_cpu_map *map;
  2621. sched->max_cpu = sysconf(_SC_NPROCESSORS_CONF);
  2622. if (sched->map.comp) {
  2623. sched->map.comp_cpus = zalloc(sched->max_cpu * sizeof(int));
  2624. if (!sched->map.comp_cpus)
  2625. return -1;
  2626. }
  2627. if (!sched->map.cpus_str)
  2628. return 0;
  2629. map = perf_cpu_map__new(sched->map.cpus_str);
  2630. if (!map) {
  2631. pr_err("failed to get cpus map from %s\n", sched->map.cpus_str);
  2632. return -1;
  2633. }
  2634. sched->map.cpus = map;
  2635. return 0;
  2636. }
  2637. static int setup_color_pids(struct perf_sched *sched)
  2638. {
  2639. struct perf_thread_map *map;
  2640. if (!sched->map.color_pids_str)
  2641. return 0;
  2642. map = thread_map__new_by_tid_str(sched->map.color_pids_str);
  2643. if (!map) {
  2644. pr_err("failed to get thread map from %s\n", sched->map.color_pids_str);
  2645. return -1;
  2646. }
  2647. sched->map.color_pids = map;
  2648. return 0;
  2649. }
  2650. static int setup_color_cpus(struct perf_sched *sched)
  2651. {
  2652. struct perf_cpu_map *map;
  2653. if (!sched->map.color_cpus_str)
  2654. return 0;
  2655. map = perf_cpu_map__new(sched->map.color_cpus_str);
  2656. if (!map) {
  2657. pr_err("failed to get thread map from %s\n", sched->map.color_cpus_str);
  2658. return -1;
  2659. }
  2660. sched->map.color_cpus = map;
  2661. return 0;
  2662. }
  2663. static int perf_sched__map(struct perf_sched *sched)
  2664. {
  2665. if (setup_map_cpus(sched))
  2666. return -1;
  2667. if (setup_color_pids(sched))
  2668. return -1;
  2669. if (setup_color_cpus(sched))
  2670. return -1;
  2671. setup_pager();
  2672. if (perf_sched__read_events(sched))
  2673. return -1;
  2674. print_bad_events(sched);
  2675. return 0;
  2676. }
  2677. static int perf_sched__replay(struct perf_sched *sched)
  2678. {
  2679. unsigned long i;
  2680. calibrate_run_measurement_overhead(sched);
  2681. calibrate_sleep_measurement_overhead(sched);
  2682. test_calibrations(sched);
  2683. if (perf_sched__read_events(sched))
  2684. return -1;
  2685. printf("nr_run_events: %ld\n", sched->nr_run_events);
  2686. printf("nr_sleep_events: %ld\n", sched->nr_sleep_events);
  2687. printf("nr_wakeup_events: %ld\n", sched->nr_wakeup_events);
  2688. if (sched->targetless_wakeups)
  2689. printf("target-less wakeups: %ld\n", sched->targetless_wakeups);
  2690. if (sched->multitarget_wakeups)
  2691. printf("multi-target wakeups: %ld\n", sched->multitarget_wakeups);
  2692. if (sched->nr_run_events_optimized)
  2693. printf("run atoms optimized: %ld\n",
  2694. sched->nr_run_events_optimized);
  2695. print_task_traces(sched);
  2696. add_cross_task_wakeups(sched);
  2697. create_tasks(sched);
  2698. printf("------------------------------------------------------------\n");
  2699. for (i = 0; i < sched->replay_repeat; i++)
  2700. run_one_test(sched);
  2701. return 0;
  2702. }
  2703. static void setup_sorting(struct perf_sched *sched, const struct option *options,
  2704. const char * const usage_msg[])
  2705. {
  2706. char *tmp, *tok, *str = strdup(sched->sort_order);
  2707. for (tok = strtok_r(str, ", ", &tmp);
  2708. tok; tok = strtok_r(NULL, ", ", &tmp)) {
  2709. if (sort_dimension__add(tok, &sched->sort_list) < 0) {
  2710. usage_with_options_msg(usage_msg, options,
  2711. "Unknown --sort key: `%s'", tok);
  2712. }
  2713. }
  2714. free(str);
  2715. sort_dimension__add("pid", &sched->cmp_pid);
  2716. }
  2717. static bool schedstat_events_exposed(void)
  2718. {
  2719. /*
  2720. * Select "sched:sched_stat_wait" event to check
  2721. * whether schedstat tracepoints are exposed.
  2722. */
  2723. return IS_ERR(trace_event__tp_format("sched", "sched_stat_wait")) ?
  2724. false : true;
  2725. }
  2726. static int __cmd_record(int argc, const char **argv)
  2727. {
  2728. unsigned int rec_argc, i, j;
  2729. const char **rec_argv;
  2730. const char * const record_args[] = {
  2731. "record",
  2732. "-a",
  2733. "-R",
  2734. "-m", "1024",
  2735. "-c", "1",
  2736. "-e", "sched:sched_switch",
  2737. "-e", "sched:sched_stat_runtime",
  2738. "-e", "sched:sched_process_fork",
  2739. "-e", "sched:sched_wakeup_new",
  2740. "-e", "sched:sched_migrate_task",
  2741. };
  2742. /*
  2743. * The tracepoints trace_sched_stat_{wait, sleep, iowait}
  2744. * are not exposed to user if CONFIG_SCHEDSTATS is not set,
  2745. * to prevent "perf sched record" execution failure, determine
  2746. * whether to record schedstat events according to actual situation.
  2747. */
  2748. const char * const schedstat_args[] = {
  2749. "-e", "sched:sched_stat_wait",
  2750. "-e", "sched:sched_stat_sleep",
  2751. "-e", "sched:sched_stat_iowait",
  2752. };
  2753. unsigned int schedstat_argc = schedstat_events_exposed() ?
  2754. ARRAY_SIZE(schedstat_args) : 0;
  2755. struct tep_event *waking_event;
  2756. /*
  2757. * +2 for either "-e", "sched:sched_wakeup" or
  2758. * "-e", "sched:sched_waking"
  2759. */
  2760. rec_argc = ARRAY_SIZE(record_args) + 2 + schedstat_argc + argc - 1;
  2761. rec_argv = calloc(rec_argc + 1, sizeof(char *));
  2762. if (rec_argv == NULL)
  2763. return -ENOMEM;
  2764. for (i = 0; i < ARRAY_SIZE(record_args); i++)
  2765. rec_argv[i] = strdup(record_args[i]);
  2766. rec_argv[i++] = "-e";
  2767. waking_event = trace_event__tp_format("sched", "sched_waking");
  2768. if (!IS_ERR(waking_event))
  2769. rec_argv[i++] = strdup("sched:sched_waking");
  2770. else
  2771. rec_argv[i++] = strdup("sched:sched_wakeup");
  2772. for (j = 0; j < schedstat_argc; j++)
  2773. rec_argv[i++] = strdup(schedstat_args[j]);
  2774. for (j = 1; j < (unsigned int)argc; j++, i++)
  2775. rec_argv[i] = argv[j];
  2776. BUG_ON(i != rec_argc);
  2777. return cmd_record(i, rec_argv);
  2778. }
  2779. int cmd_sched(int argc, const char **argv)
  2780. {
  2781. static const char default_sort_order[] = "avg, max, switch, runtime";
  2782. struct perf_sched sched = {
  2783. .tool = {
  2784. .sample = perf_sched__process_tracepoint_sample,
  2785. .comm = perf_sched__process_comm,
  2786. .namespaces = perf_event__process_namespaces,
  2787. .lost = perf_event__process_lost,
  2788. .fork = perf_sched__process_fork_event,
  2789. .ordered_events = true,
  2790. },
  2791. .cmp_pid = LIST_HEAD_INIT(sched.cmp_pid),
  2792. .sort_list = LIST_HEAD_INIT(sched.sort_list),
  2793. .start_work_mutex = PTHREAD_MUTEX_INITIALIZER,
  2794. .work_done_wait_mutex = PTHREAD_MUTEX_INITIALIZER,
  2795. .sort_order = default_sort_order,
  2796. .replay_repeat = 10,
  2797. .profile_cpu = -1,
  2798. .next_shortname1 = 'A',
  2799. .next_shortname2 = '0',
  2800. .skip_merge = 0,
  2801. .show_callchain = 1,
  2802. .max_stack = 5,
  2803. };
  2804. const struct option sched_options[] = {
  2805. OPT_STRING('i', "input", &input_name, "file",
  2806. "input file name"),
  2807. OPT_INCR('v', "verbose", &verbose,
  2808. "be more verbose (show symbol address, etc)"),
  2809. OPT_BOOLEAN('D', "dump-raw-trace", &dump_trace,
  2810. "dump raw trace in ASCII"),
  2811. OPT_BOOLEAN('f', "force", &sched.force, "don't complain, do it"),
  2812. OPT_END()
  2813. };
  2814. const struct option latency_options[] = {
  2815. OPT_STRING('s', "sort", &sched.sort_order, "key[,key2...]",
  2816. "sort by key(s): runtime, switch, avg, max"),
  2817. OPT_INTEGER('C', "CPU", &sched.profile_cpu,
  2818. "CPU to profile on"),
  2819. OPT_BOOLEAN('p', "pids", &sched.skip_merge,
  2820. "latency stats per pid instead of per comm"),
  2821. OPT_PARENT(sched_options)
  2822. };
  2823. const struct option replay_options[] = {
  2824. OPT_UINTEGER('r', "repeat", &sched.replay_repeat,
  2825. "repeat the workload replay N times (-1: infinite)"),
  2826. OPT_PARENT(sched_options)
  2827. };
  2828. const struct option map_options[] = {
  2829. OPT_BOOLEAN(0, "compact", &sched.map.comp,
  2830. "map output in compact mode"),
  2831. OPT_STRING(0, "color-pids", &sched.map.color_pids_str, "pids",
  2832. "highlight given pids in map"),
  2833. OPT_STRING(0, "color-cpus", &sched.map.color_cpus_str, "cpus",
  2834. "highlight given CPUs in map"),
  2835. OPT_STRING(0, "cpus", &sched.map.cpus_str, "cpus",
  2836. "display given CPUs in map"),
  2837. OPT_PARENT(sched_options)
  2838. };
  2839. const struct option timehist_options[] = {
  2840. OPT_STRING('k', "vmlinux", &symbol_conf.vmlinux_name,
  2841. "file", "vmlinux pathname"),
  2842. OPT_STRING(0, "kallsyms", &symbol_conf.kallsyms_name,
  2843. "file", "kallsyms pathname"),
  2844. OPT_BOOLEAN('g', "call-graph", &sched.show_callchain,
  2845. "Display call chains if present (default on)"),
  2846. OPT_UINTEGER(0, "max-stack", &sched.max_stack,
  2847. "Maximum number of functions to display backtrace."),
  2848. OPT_STRING(0, "symfs", &symbol_conf.symfs, "directory",
  2849. "Look for files with symbols relative to this directory"),
  2850. OPT_BOOLEAN('s', "summary", &sched.summary_only,
  2851. "Show only syscall summary with statistics"),
  2852. OPT_BOOLEAN('S', "with-summary", &sched.summary,
  2853. "Show all syscalls and summary with statistics"),
  2854. OPT_BOOLEAN('w', "wakeups", &sched.show_wakeups, "Show wakeup events"),
  2855. OPT_BOOLEAN('n', "next", &sched.show_next, "Show next task"),
  2856. OPT_BOOLEAN('M', "migrations", &sched.show_migrations, "Show migration events"),
  2857. OPT_BOOLEAN('V', "cpu-visual", &sched.show_cpu_visual, "Add CPU visual"),
  2858. OPT_BOOLEAN('I', "idle-hist", &sched.idle_hist, "Show idle events only"),
  2859. OPT_STRING(0, "time", &sched.time_str, "str",
  2860. "Time span for analysis (start,stop)"),
  2861. OPT_BOOLEAN(0, "state", &sched.show_state, "Show task state when sched-out"),
  2862. OPT_STRING('p', "pid", &symbol_conf.pid_list_str, "pid[,pid...]",
  2863. "analyze events only for given process id(s)"),
  2864. OPT_STRING('t', "tid", &symbol_conf.tid_list_str, "tid[,tid...]",
  2865. "analyze events only for given thread id(s)"),
  2866. OPT_STRING('C', "cpu", &cpu_list, "cpu", "list of cpus to profile"),
  2867. OPT_PARENT(sched_options)
  2868. };
  2869. const char * const latency_usage[] = {
  2870. "perf sched latency [<options>]",
  2871. NULL
  2872. };
  2873. const char * const replay_usage[] = {
  2874. "perf sched replay [<options>]",
  2875. NULL
  2876. };
  2877. const char * const map_usage[] = {
  2878. "perf sched map [<options>]",
  2879. NULL
  2880. };
  2881. const char * const timehist_usage[] = {
  2882. "perf sched timehist [<options>]",
  2883. NULL
  2884. };
  2885. const char *const sched_subcommands[] = { "record", "latency", "map",
  2886. "replay", "script",
  2887. "timehist", NULL };
  2888. const char *sched_usage[] = {
  2889. NULL,
  2890. NULL
  2891. };
  2892. struct trace_sched_handler lat_ops = {
  2893. .wakeup_event = latency_wakeup_event,
  2894. .switch_event = latency_switch_event,
  2895. .runtime_event = latency_runtime_event,
  2896. .migrate_task_event = latency_migrate_task_event,
  2897. };
  2898. struct trace_sched_handler map_ops = {
  2899. .switch_event = map_switch_event,
  2900. };
  2901. struct trace_sched_handler replay_ops = {
  2902. .wakeup_event = replay_wakeup_event,
  2903. .switch_event = replay_switch_event,
  2904. .fork_event = replay_fork_event,
  2905. };
  2906. unsigned int i;
  2907. for (i = 0; i < ARRAY_SIZE(sched.curr_pid); i++)
  2908. sched.curr_pid[i] = -1;
  2909. argc = parse_options_subcommand(argc, argv, sched_options, sched_subcommands,
  2910. sched_usage, PARSE_OPT_STOP_AT_NON_OPTION);
  2911. if (!argc)
  2912. usage_with_options(sched_usage, sched_options);
  2913. /*
  2914. * Aliased to 'perf script' for now:
  2915. */
  2916. if (!strcmp(argv[0], "script"))
  2917. return cmd_script(argc, argv);
  2918. if (!strncmp(argv[0], "rec", 3)) {
  2919. return __cmd_record(argc, argv);
  2920. } else if (!strncmp(argv[0], "lat", 3)) {
  2921. sched.tp_handler = &lat_ops;
  2922. if (argc > 1) {
  2923. argc = parse_options(argc, argv, latency_options, latency_usage, 0);
  2924. if (argc)
  2925. usage_with_options(latency_usage, latency_options);
  2926. }
  2927. setup_sorting(&sched, latency_options, latency_usage);
  2928. return perf_sched__lat(&sched);
  2929. } else if (!strcmp(argv[0], "map")) {
  2930. if (argc) {
  2931. argc = parse_options(argc, argv, map_options, map_usage, 0);
  2932. if (argc)
  2933. usage_with_options(map_usage, map_options);
  2934. }
  2935. sched.tp_handler = &map_ops;
  2936. setup_sorting(&sched, latency_options, latency_usage);
  2937. return perf_sched__map(&sched);
  2938. } else if (!strncmp(argv[0], "rep", 3)) {
  2939. sched.tp_handler = &replay_ops;
  2940. if (argc) {
  2941. argc = parse_options(argc, argv, replay_options, replay_usage, 0);
  2942. if (argc)
  2943. usage_with_options(replay_usage, replay_options);
  2944. }
  2945. return perf_sched__replay(&sched);
  2946. } else if (!strcmp(argv[0], "timehist")) {
  2947. if (argc) {
  2948. argc = parse_options(argc, argv, timehist_options,
  2949. timehist_usage, 0);
  2950. if (argc)
  2951. usage_with_options(timehist_usage, timehist_options);
  2952. }
  2953. if ((sched.show_wakeups || sched.show_next) &&
  2954. sched.summary_only) {
  2955. pr_err(" Error: -s and -[n|w] are mutually exclusive.\n");
  2956. parse_options_usage(timehist_usage, timehist_options, "s", true);
  2957. if (sched.show_wakeups)
  2958. parse_options_usage(NULL, timehist_options, "w", true);
  2959. if (sched.show_next)
  2960. parse_options_usage(NULL, timehist_options, "n", true);
  2961. return -EINVAL;
  2962. }
  2963. return perf_sched__timehist(&sched);
  2964. } else {
  2965. usage_with_options(sched_usage, sched_options);
  2966. }
  2967. return 0;
  2968. }