trace_sched_wakeup.c 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * trace task wakeup timings
  4. *
  5. * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
  6. * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
  7. *
  8. * Based on code from the latency_tracer, that is:
  9. *
  10. * Copyright (C) 2004-2006 Ingo Molnar
  11. * Copyright (C) 2004 Nadia Yvette Chambers
  12. */
  13. #include <linux/module.h>
  14. #include <linux/kallsyms.h>
  15. #include <linux/uaccess.h>
  16. #include <linux/ftrace.h>
  17. #include <linux/sched/rt.h>
  18. #include <linux/sched/deadline.h>
  19. #include <trace/events/sched.h>
  20. #include "trace.h"
  21. static struct trace_array *wakeup_trace;
  22. static int __read_mostly tracer_enabled;
  23. static struct task_struct *wakeup_task;
  24. static int wakeup_cpu;
  25. static int wakeup_current_cpu;
  26. static unsigned wakeup_prio = -1;
  27. static int wakeup_rt;
  28. static int wakeup_dl;
  29. static int tracing_dl = 0;
  30. static arch_spinlock_t wakeup_lock =
  31. (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
  32. static void wakeup_reset(struct trace_array *tr);
  33. static void __wakeup_reset(struct trace_array *tr);
  34. static int start_func_tracer(struct trace_array *tr, int graph);
  35. static void stop_func_tracer(struct trace_array *tr, int graph);
  36. static int save_flags;
  37. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  38. # define is_graph(tr) ((tr)->trace_flags & TRACE_ITER_DISPLAY_GRAPH)
  39. #else
  40. # define is_graph(tr) false
  41. #endif
  42. #ifdef CONFIG_FUNCTION_TRACER
  43. static bool function_enabled;
  44. /*
  45. * Prologue for the wakeup function tracers.
  46. *
  47. * Returns 1 if it is OK to continue, and preemption
  48. * is disabled and data->disabled is incremented.
  49. * 0 if the trace is to be ignored, and preemption
  50. * is not disabled and data->disabled is
  51. * kept the same.
  52. *
  53. * Note, this function is also used outside this ifdef but
  54. * inside the #ifdef of the function graph tracer below.
  55. * This is OK, since the function graph tracer is
  56. * dependent on the function tracer.
  57. */
  58. static int
  59. func_prolog_preempt_disable(struct trace_array *tr,
  60. struct trace_array_cpu **data,
  61. int *pc)
  62. {
  63. long disabled;
  64. int cpu;
  65. if (likely(!wakeup_task))
  66. return 0;
  67. *pc = preempt_count();
  68. preempt_disable_notrace();
  69. cpu = raw_smp_processor_id();
  70. if (cpu != wakeup_current_cpu)
  71. goto out_enable;
  72. *data = per_cpu_ptr(tr->array_buffer.data, cpu);
  73. disabled = atomic_inc_return(&(*data)->disabled);
  74. if (unlikely(disabled != 1))
  75. goto out;
  76. return 1;
  77. out:
  78. atomic_dec(&(*data)->disabled);
  79. out_enable:
  80. preempt_enable_notrace();
  81. return 0;
  82. }
  83. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  84. static int wakeup_display_graph(struct trace_array *tr, int set)
  85. {
  86. if (!(is_graph(tr) ^ set))
  87. return 0;
  88. stop_func_tracer(tr, !set);
  89. wakeup_reset(wakeup_trace);
  90. tr->max_latency = 0;
  91. return start_func_tracer(tr, set);
  92. }
  93. static int wakeup_graph_entry(struct ftrace_graph_ent *trace)
  94. {
  95. struct trace_array *tr = wakeup_trace;
  96. struct trace_array_cpu *data;
  97. unsigned long flags;
  98. int pc, ret = 0;
  99. if (ftrace_graph_ignore_func(trace))
  100. return 0;
  101. /*
  102. * Do not trace a function if it's filtered by set_graph_notrace.
  103. * Make the index of ret stack negative to indicate that it should
  104. * ignore further functions. But it needs its own ret stack entry
  105. * to recover the original index in order to continue tracing after
  106. * returning from the function.
  107. */
  108. if (ftrace_graph_notrace_addr(trace->func))
  109. return 1;
  110. if (!func_prolog_preempt_disable(tr, &data, &pc))
  111. return 0;
  112. local_save_flags(flags);
  113. ret = __trace_graph_entry(tr, trace, flags, pc);
  114. atomic_dec(&data->disabled);
  115. preempt_enable_notrace();
  116. return ret;
  117. }
  118. static void wakeup_graph_return(struct ftrace_graph_ret *trace)
  119. {
  120. struct trace_array *tr = wakeup_trace;
  121. struct trace_array_cpu *data;
  122. unsigned long flags;
  123. int pc;
  124. ftrace_graph_addr_finish(trace);
  125. if (!func_prolog_preempt_disable(tr, &data, &pc))
  126. return;
  127. local_save_flags(flags);
  128. __trace_graph_return(tr, trace, flags, pc);
  129. atomic_dec(&data->disabled);
  130. preempt_enable_notrace();
  131. return;
  132. }
  133. static struct fgraph_ops fgraph_wakeup_ops = {
  134. .entryfunc = &wakeup_graph_entry,
  135. .retfunc = &wakeup_graph_return,
  136. };
  137. static void wakeup_trace_open(struct trace_iterator *iter)
  138. {
  139. if (is_graph(iter->tr))
  140. graph_trace_open(iter);
  141. }
  142. static void wakeup_trace_close(struct trace_iterator *iter)
  143. {
  144. if (iter->private)
  145. graph_trace_close(iter);
  146. }
  147. #define GRAPH_TRACER_FLAGS (TRACE_GRAPH_PRINT_PROC | \
  148. TRACE_GRAPH_PRINT_CPU | \
  149. TRACE_GRAPH_PRINT_REL_TIME | \
  150. TRACE_GRAPH_PRINT_DURATION | \
  151. TRACE_GRAPH_PRINT_OVERHEAD | \
  152. TRACE_GRAPH_PRINT_IRQS)
  153. static enum print_line_t wakeup_print_line(struct trace_iterator *iter)
  154. {
  155. /*
  156. * In graph mode call the graph tracer output function,
  157. * otherwise go with the TRACE_FN event handler
  158. */
  159. if (is_graph(iter->tr))
  160. return print_graph_function_flags(iter, GRAPH_TRACER_FLAGS);
  161. return TRACE_TYPE_UNHANDLED;
  162. }
  163. static void wakeup_print_header(struct seq_file *s)
  164. {
  165. if (is_graph(wakeup_trace))
  166. print_graph_headers_flags(s, GRAPH_TRACER_FLAGS);
  167. else
  168. trace_default_header(s);
  169. }
  170. #endif /* else CONFIG_FUNCTION_GRAPH_TRACER */
  171. /*
  172. * wakeup uses its own tracer function to keep the overhead down:
  173. */
  174. static void
  175. wakeup_tracer_call(unsigned long ip, unsigned long parent_ip,
  176. struct ftrace_ops *op, struct pt_regs *pt_regs)
  177. {
  178. struct trace_array *tr = wakeup_trace;
  179. struct trace_array_cpu *data;
  180. unsigned long flags;
  181. int pc;
  182. if (!func_prolog_preempt_disable(tr, &data, &pc))
  183. return;
  184. local_irq_save(flags);
  185. trace_function(tr, ip, parent_ip, flags, pc);
  186. local_irq_restore(flags);
  187. atomic_dec(&data->disabled);
  188. preempt_enable_notrace();
  189. }
  190. static int register_wakeup_function(struct trace_array *tr, int graph, int set)
  191. {
  192. int ret;
  193. /* 'set' is set if TRACE_ITER_FUNCTION is about to be set */
  194. if (function_enabled || (!set && !(tr->trace_flags & TRACE_ITER_FUNCTION)))
  195. return 0;
  196. if (graph)
  197. ret = register_ftrace_graph(&fgraph_wakeup_ops);
  198. else
  199. ret = register_ftrace_function(tr->ops);
  200. if (!ret)
  201. function_enabled = true;
  202. return ret;
  203. }
  204. static void unregister_wakeup_function(struct trace_array *tr, int graph)
  205. {
  206. if (!function_enabled)
  207. return;
  208. if (graph)
  209. unregister_ftrace_graph(&fgraph_wakeup_ops);
  210. else
  211. unregister_ftrace_function(tr->ops);
  212. function_enabled = false;
  213. }
  214. static int wakeup_function_set(struct trace_array *tr, u32 mask, int set)
  215. {
  216. if (!(mask & TRACE_ITER_FUNCTION))
  217. return 0;
  218. if (set)
  219. register_wakeup_function(tr, is_graph(tr), 1);
  220. else
  221. unregister_wakeup_function(tr, is_graph(tr));
  222. return 1;
  223. }
  224. #else /* CONFIG_FUNCTION_TRACER */
  225. static int register_wakeup_function(struct trace_array *tr, int graph, int set)
  226. {
  227. return 0;
  228. }
  229. static void unregister_wakeup_function(struct trace_array *tr, int graph) { }
  230. static int wakeup_function_set(struct trace_array *tr, u32 mask, int set)
  231. {
  232. return 0;
  233. }
  234. #endif /* else CONFIG_FUNCTION_TRACER */
  235. #ifndef CONFIG_FUNCTION_GRAPH_TRACER
  236. static enum print_line_t wakeup_print_line(struct trace_iterator *iter)
  237. {
  238. return TRACE_TYPE_UNHANDLED;
  239. }
  240. static void wakeup_trace_open(struct trace_iterator *iter) { }
  241. static void wakeup_trace_close(struct trace_iterator *iter) { }
  242. static void wakeup_print_header(struct seq_file *s)
  243. {
  244. trace_default_header(s);
  245. }
  246. #endif /* !CONFIG_FUNCTION_GRAPH_TRACER */
  247. static void
  248. __trace_function(struct trace_array *tr,
  249. unsigned long ip, unsigned long parent_ip,
  250. unsigned long flags, int pc)
  251. {
  252. if (is_graph(tr))
  253. trace_graph_function(tr, ip, parent_ip, flags, pc);
  254. else
  255. trace_function(tr, ip, parent_ip, flags, pc);
  256. }
  257. static int wakeup_flag_changed(struct trace_array *tr, u32 mask, int set)
  258. {
  259. struct tracer *tracer = tr->current_trace;
  260. if (wakeup_function_set(tr, mask, set))
  261. return 0;
  262. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  263. if (mask & TRACE_ITER_DISPLAY_GRAPH)
  264. return wakeup_display_graph(tr, set);
  265. #endif
  266. return trace_keep_overwrite(tracer, mask, set);
  267. }
  268. static int start_func_tracer(struct trace_array *tr, int graph)
  269. {
  270. int ret;
  271. ret = register_wakeup_function(tr, graph, 0);
  272. if (!ret && tracing_is_enabled())
  273. tracer_enabled = 1;
  274. else
  275. tracer_enabled = 0;
  276. return ret;
  277. }
  278. static void stop_func_tracer(struct trace_array *tr, int graph)
  279. {
  280. tracer_enabled = 0;
  281. unregister_wakeup_function(tr, graph);
  282. }
  283. /*
  284. * Should this new latency be reported/recorded?
  285. */
  286. static bool report_latency(struct trace_array *tr, u64 delta)
  287. {
  288. if (tracing_thresh) {
  289. if (delta < tracing_thresh)
  290. return false;
  291. } else {
  292. if (delta <= tr->max_latency)
  293. return false;
  294. }
  295. return true;
  296. }
  297. static void
  298. probe_wakeup_migrate_task(void *ignore, struct task_struct *task, int cpu)
  299. {
  300. if (task != wakeup_task)
  301. return;
  302. wakeup_current_cpu = cpu;
  303. }
  304. static void
  305. tracing_sched_switch_trace(struct trace_array *tr,
  306. struct task_struct *prev,
  307. struct task_struct *next,
  308. unsigned long flags, int pc)
  309. {
  310. struct trace_event_call *call = &event_context_switch;
  311. struct trace_buffer *buffer = tr->array_buffer.buffer;
  312. struct ring_buffer_event *event;
  313. struct ctx_switch_entry *entry;
  314. event = trace_buffer_lock_reserve(buffer, TRACE_CTX,
  315. sizeof(*entry), flags, pc);
  316. if (!event)
  317. return;
  318. entry = ring_buffer_event_data(event);
  319. entry->prev_pid = prev->pid;
  320. entry->prev_prio = prev->prio;
  321. entry->prev_state = task_state_index(prev);
  322. entry->next_pid = next->pid;
  323. entry->next_prio = next->prio;
  324. entry->next_state = task_state_index(next);
  325. entry->next_cpu = task_cpu(next);
  326. if (!call_filter_check_discard(call, entry, buffer, event))
  327. trace_buffer_unlock_commit(tr, buffer, event, flags, pc);
  328. }
  329. static void
  330. tracing_sched_wakeup_trace(struct trace_array *tr,
  331. struct task_struct *wakee,
  332. struct task_struct *curr,
  333. unsigned long flags, int pc)
  334. {
  335. struct trace_event_call *call = &event_wakeup;
  336. struct ring_buffer_event *event;
  337. struct ctx_switch_entry *entry;
  338. struct trace_buffer *buffer = tr->array_buffer.buffer;
  339. event = trace_buffer_lock_reserve(buffer, TRACE_WAKE,
  340. sizeof(*entry), flags, pc);
  341. if (!event)
  342. return;
  343. entry = ring_buffer_event_data(event);
  344. entry->prev_pid = curr->pid;
  345. entry->prev_prio = curr->prio;
  346. entry->prev_state = task_state_index(curr);
  347. entry->next_pid = wakee->pid;
  348. entry->next_prio = wakee->prio;
  349. entry->next_state = task_state_index(wakee);
  350. entry->next_cpu = task_cpu(wakee);
  351. if (!call_filter_check_discard(call, entry, buffer, event))
  352. trace_buffer_unlock_commit(tr, buffer, event, flags, pc);
  353. }
  354. static void notrace
  355. probe_wakeup_sched_switch(void *ignore, bool preempt,
  356. struct task_struct *prev, struct task_struct *next)
  357. {
  358. struct trace_array_cpu *data;
  359. u64 T0, T1, delta;
  360. unsigned long flags;
  361. long disabled;
  362. int cpu;
  363. int pc;
  364. tracing_record_cmdline(prev);
  365. if (unlikely(!tracer_enabled))
  366. return;
  367. /*
  368. * When we start a new trace, we set wakeup_task to NULL
  369. * and then set tracer_enabled = 1. We want to make sure
  370. * that another CPU does not see the tracer_enabled = 1
  371. * and the wakeup_task with an older task, that might
  372. * actually be the same as next.
  373. */
  374. smp_rmb();
  375. if (next != wakeup_task)
  376. return;
  377. pc = preempt_count();
  378. /* disable local data, not wakeup_cpu data */
  379. cpu = raw_smp_processor_id();
  380. disabled = atomic_inc_return(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled);
  381. if (likely(disabled != 1))
  382. goto out;
  383. local_irq_save(flags);
  384. arch_spin_lock(&wakeup_lock);
  385. /* We could race with grabbing wakeup_lock */
  386. if (unlikely(!tracer_enabled || next != wakeup_task))
  387. goto out_unlock;
  388. /* The task we are waiting for is waking up */
  389. data = per_cpu_ptr(wakeup_trace->array_buffer.data, wakeup_cpu);
  390. __trace_function(wakeup_trace, CALLER_ADDR0, CALLER_ADDR1, flags, pc);
  391. tracing_sched_switch_trace(wakeup_trace, prev, next, flags, pc);
  392. __trace_stack(wakeup_trace, flags, 0, pc);
  393. T0 = data->preempt_timestamp;
  394. T1 = ftrace_now(cpu);
  395. delta = T1-T0;
  396. if (!report_latency(wakeup_trace, delta))
  397. goto out_unlock;
  398. if (likely(!is_tracing_stopped())) {
  399. wakeup_trace->max_latency = delta;
  400. update_max_tr(wakeup_trace, wakeup_task, wakeup_cpu, NULL);
  401. }
  402. out_unlock:
  403. __wakeup_reset(wakeup_trace);
  404. arch_spin_unlock(&wakeup_lock);
  405. local_irq_restore(flags);
  406. out:
  407. atomic_dec(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled);
  408. }
  409. static void __wakeup_reset(struct trace_array *tr)
  410. {
  411. wakeup_cpu = -1;
  412. wakeup_prio = -1;
  413. tracing_dl = 0;
  414. if (wakeup_task)
  415. put_task_struct(wakeup_task);
  416. wakeup_task = NULL;
  417. }
  418. static void wakeup_reset(struct trace_array *tr)
  419. {
  420. unsigned long flags;
  421. tracing_reset_online_cpus(&tr->array_buffer);
  422. local_irq_save(flags);
  423. arch_spin_lock(&wakeup_lock);
  424. __wakeup_reset(tr);
  425. arch_spin_unlock(&wakeup_lock);
  426. local_irq_restore(flags);
  427. }
  428. static void
  429. probe_wakeup(void *ignore, struct task_struct *p)
  430. {
  431. struct trace_array_cpu *data;
  432. int cpu = smp_processor_id();
  433. unsigned long flags;
  434. long disabled;
  435. int pc;
  436. if (likely(!tracer_enabled))
  437. return;
  438. tracing_record_cmdline(p);
  439. tracing_record_cmdline(current);
  440. /*
  441. * Semantic is like this:
  442. * - wakeup tracer handles all tasks in the system, independently
  443. * from their scheduling class;
  444. * - wakeup_rt tracer handles tasks belonging to sched_dl and
  445. * sched_rt class;
  446. * - wakeup_dl handles tasks belonging to sched_dl class only.
  447. */
  448. if (tracing_dl || (wakeup_dl && !dl_task(p)) ||
  449. (wakeup_rt && !dl_task(p) && !rt_task(p)) ||
  450. (!dl_task(p) && (p->prio >= wakeup_prio || p->prio >= current->prio)))
  451. return;
  452. pc = preempt_count();
  453. disabled = atomic_inc_return(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled);
  454. if (unlikely(disabled != 1))
  455. goto out;
  456. /* interrupts should be off from try_to_wake_up */
  457. arch_spin_lock(&wakeup_lock);
  458. /* check for races. */
  459. if (!tracer_enabled || tracing_dl ||
  460. (!dl_task(p) && p->prio >= wakeup_prio))
  461. goto out_locked;
  462. /* reset the trace */
  463. __wakeup_reset(wakeup_trace);
  464. wakeup_cpu = task_cpu(p);
  465. wakeup_current_cpu = wakeup_cpu;
  466. wakeup_prio = p->prio;
  467. /*
  468. * Once you start tracing a -deadline task, don't bother tracing
  469. * another task until the first one wakes up.
  470. */
  471. if (dl_task(p))
  472. tracing_dl = 1;
  473. else
  474. tracing_dl = 0;
  475. wakeup_task = get_task_struct(p);
  476. local_save_flags(flags);
  477. data = per_cpu_ptr(wakeup_trace->array_buffer.data, wakeup_cpu);
  478. data->preempt_timestamp = ftrace_now(cpu);
  479. tracing_sched_wakeup_trace(wakeup_trace, p, current, flags, pc);
  480. __trace_stack(wakeup_trace, flags, 0, pc);
  481. /*
  482. * We must be careful in using CALLER_ADDR2. But since wake_up
  483. * is not called by an assembly function (where as schedule is)
  484. * it should be safe to use it here.
  485. */
  486. __trace_function(wakeup_trace, CALLER_ADDR1, CALLER_ADDR2, flags, pc);
  487. out_locked:
  488. arch_spin_unlock(&wakeup_lock);
  489. out:
  490. atomic_dec(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled);
  491. }
  492. static void start_wakeup_tracer(struct trace_array *tr)
  493. {
  494. int ret;
  495. ret = register_trace_sched_wakeup(probe_wakeup, NULL);
  496. if (ret) {
  497. pr_info("wakeup trace: Couldn't activate tracepoint"
  498. " probe to kernel_sched_wakeup\n");
  499. return;
  500. }
  501. ret = register_trace_sched_wakeup_new(probe_wakeup, NULL);
  502. if (ret) {
  503. pr_info("wakeup trace: Couldn't activate tracepoint"
  504. " probe to kernel_sched_wakeup_new\n");
  505. goto fail_deprobe;
  506. }
  507. ret = register_trace_sched_switch(probe_wakeup_sched_switch, NULL);
  508. if (ret) {
  509. pr_info("sched trace: Couldn't activate tracepoint"
  510. " probe to kernel_sched_switch\n");
  511. goto fail_deprobe_wake_new;
  512. }
  513. ret = register_trace_sched_migrate_task(probe_wakeup_migrate_task, NULL);
  514. if (ret) {
  515. pr_info("wakeup trace: Couldn't activate tracepoint"
  516. " probe to kernel_sched_migrate_task\n");
  517. goto fail_deprobe_sched_switch;
  518. }
  519. wakeup_reset(tr);
  520. /*
  521. * Don't let the tracer_enabled = 1 show up before
  522. * the wakeup_task is reset. This may be overkill since
  523. * wakeup_reset does a spin_unlock after setting the
  524. * wakeup_task to NULL, but I want to be safe.
  525. * This is a slow path anyway.
  526. */
  527. smp_wmb();
  528. if (start_func_tracer(tr, is_graph(tr)))
  529. printk(KERN_ERR "failed to start wakeup tracer\n");
  530. return;
  531. fail_deprobe_sched_switch:
  532. unregister_trace_sched_switch(probe_wakeup_sched_switch, NULL);
  533. fail_deprobe_wake_new:
  534. unregister_trace_sched_wakeup_new(probe_wakeup, NULL);
  535. fail_deprobe:
  536. unregister_trace_sched_wakeup(probe_wakeup, NULL);
  537. }
  538. static void stop_wakeup_tracer(struct trace_array *tr)
  539. {
  540. tracer_enabled = 0;
  541. stop_func_tracer(tr, is_graph(tr));
  542. unregister_trace_sched_switch(probe_wakeup_sched_switch, NULL);
  543. unregister_trace_sched_wakeup_new(probe_wakeup, NULL);
  544. unregister_trace_sched_wakeup(probe_wakeup, NULL);
  545. unregister_trace_sched_migrate_task(probe_wakeup_migrate_task, NULL);
  546. }
  547. static bool wakeup_busy;
  548. static int __wakeup_tracer_init(struct trace_array *tr)
  549. {
  550. save_flags = tr->trace_flags;
  551. /* non overwrite screws up the latency tracers */
  552. set_tracer_flag(tr, TRACE_ITER_OVERWRITE, 1);
  553. set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, 1);
  554. tr->max_latency = 0;
  555. wakeup_trace = tr;
  556. ftrace_init_array_ops(tr, wakeup_tracer_call);
  557. start_wakeup_tracer(tr);
  558. wakeup_busy = true;
  559. return 0;
  560. }
  561. static int wakeup_tracer_init(struct trace_array *tr)
  562. {
  563. if (wakeup_busy)
  564. return -EBUSY;
  565. wakeup_dl = 0;
  566. wakeup_rt = 0;
  567. return __wakeup_tracer_init(tr);
  568. }
  569. static int wakeup_rt_tracer_init(struct trace_array *tr)
  570. {
  571. if (wakeup_busy)
  572. return -EBUSY;
  573. wakeup_dl = 0;
  574. wakeup_rt = 1;
  575. return __wakeup_tracer_init(tr);
  576. }
  577. static int wakeup_dl_tracer_init(struct trace_array *tr)
  578. {
  579. if (wakeup_busy)
  580. return -EBUSY;
  581. wakeup_dl = 1;
  582. wakeup_rt = 0;
  583. return __wakeup_tracer_init(tr);
  584. }
  585. static void wakeup_tracer_reset(struct trace_array *tr)
  586. {
  587. int lat_flag = save_flags & TRACE_ITER_LATENCY_FMT;
  588. int overwrite_flag = save_flags & TRACE_ITER_OVERWRITE;
  589. stop_wakeup_tracer(tr);
  590. /* make sure we put back any tasks we are tracing */
  591. wakeup_reset(tr);
  592. set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, lat_flag);
  593. set_tracer_flag(tr, TRACE_ITER_OVERWRITE, overwrite_flag);
  594. ftrace_reset_array_ops(tr);
  595. wakeup_busy = false;
  596. }
  597. static void wakeup_tracer_start(struct trace_array *tr)
  598. {
  599. wakeup_reset(tr);
  600. tracer_enabled = 1;
  601. }
  602. static void wakeup_tracer_stop(struct trace_array *tr)
  603. {
  604. tracer_enabled = 0;
  605. }
  606. static struct tracer wakeup_tracer __read_mostly =
  607. {
  608. .name = "wakeup",
  609. .init = wakeup_tracer_init,
  610. .reset = wakeup_tracer_reset,
  611. .start = wakeup_tracer_start,
  612. .stop = wakeup_tracer_stop,
  613. .print_max = true,
  614. .print_header = wakeup_print_header,
  615. .print_line = wakeup_print_line,
  616. .flag_changed = wakeup_flag_changed,
  617. #ifdef CONFIG_FTRACE_SELFTEST
  618. .selftest = trace_selftest_startup_wakeup,
  619. #endif
  620. .open = wakeup_trace_open,
  621. .close = wakeup_trace_close,
  622. .allow_instances = true,
  623. .use_max_tr = true,
  624. };
  625. static struct tracer wakeup_rt_tracer __read_mostly =
  626. {
  627. .name = "wakeup_rt",
  628. .init = wakeup_rt_tracer_init,
  629. .reset = wakeup_tracer_reset,
  630. .start = wakeup_tracer_start,
  631. .stop = wakeup_tracer_stop,
  632. .print_max = true,
  633. .print_header = wakeup_print_header,
  634. .print_line = wakeup_print_line,
  635. .flag_changed = wakeup_flag_changed,
  636. #ifdef CONFIG_FTRACE_SELFTEST
  637. .selftest = trace_selftest_startup_wakeup,
  638. #endif
  639. .open = wakeup_trace_open,
  640. .close = wakeup_trace_close,
  641. .allow_instances = true,
  642. .use_max_tr = true,
  643. };
  644. static struct tracer wakeup_dl_tracer __read_mostly =
  645. {
  646. .name = "wakeup_dl",
  647. .init = wakeup_dl_tracer_init,
  648. .reset = wakeup_tracer_reset,
  649. .start = wakeup_tracer_start,
  650. .stop = wakeup_tracer_stop,
  651. .print_max = true,
  652. .print_header = wakeup_print_header,
  653. .print_line = wakeup_print_line,
  654. .flag_changed = wakeup_flag_changed,
  655. #ifdef CONFIG_FTRACE_SELFTEST
  656. .selftest = trace_selftest_startup_wakeup,
  657. #endif
  658. .open = wakeup_trace_open,
  659. .close = wakeup_trace_close,
  660. .allow_instances = true,
  661. .use_max_tr = true,
  662. };
  663. __init static int init_wakeup_tracer(void)
  664. {
  665. int ret;
  666. ret = register_tracer(&wakeup_tracer);
  667. if (ret)
  668. return ret;
  669. ret = register_tracer(&wakeup_rt_tracer);
  670. if (ret)
  671. return ret;
  672. ret = register_tracer(&wakeup_dl_tracer);
  673. if (ret)
  674. return ret;
  675. return 0;
  676. }
  677. core_initcall(init_wakeup_tracer);