trace_preemptirq.c 4.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * preemptoff and irqoff tracepoints
  4. *
  5. * Copyright (C) Joel Fernandes (Google) <joel@joelfernandes.org>
  6. */
  7. #include <linux/kallsyms.h>
  8. #include <linux/uaccess.h>
  9. #include <linux/module.h>
  10. #include <linux/ftrace.h>
  11. #include <linux/kprobes.h>
  12. #include "trace.h"
  13. #define CREATE_TRACE_POINTS
  14. #include <trace/events/preemptirq.h>
  15. #undef CREATE_TRACE_POINTS
  16. #include <trace/hooks/preemptirq.h>
  17. #ifdef CONFIG_TRACE_IRQFLAGS
  18. /* Per-cpu variable to prevent redundant calls when IRQs already off */
  19. static DEFINE_PER_CPU(int, tracing_irq_cpu);
  20. /*
  21. * Like trace_hardirqs_on() but without the lockdep invocation. This is
  22. * used in the low level entry code where the ordering vs. RCU is important
  23. * and lockdep uses a staged approach which splits the lockdep hardirq
  24. * tracking into a RCU on and a RCU off section.
  25. */
  26. void trace_hardirqs_on_prepare(void)
  27. {
  28. if (this_cpu_read(tracing_irq_cpu)) {
  29. if (!in_nmi()) {
  30. trace_irq_enable(CALLER_ADDR0, CALLER_ADDR1);
  31. trace_android_rvh_irqs_enable(CALLER_ADDR0,
  32. CALLER_ADDR1);
  33. }
  34. tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1);
  35. this_cpu_write(tracing_irq_cpu, 0);
  36. }
  37. }
  38. EXPORT_SYMBOL(trace_hardirqs_on_prepare);
  39. NOKPROBE_SYMBOL(trace_hardirqs_on_prepare);
  40. void trace_hardirqs_on(void)
  41. {
  42. if (this_cpu_read(tracing_irq_cpu)) {
  43. if (!in_nmi()) {
  44. trace_irq_enable_rcuidle(CALLER_ADDR0, CALLER_ADDR1);
  45. trace_android_rvh_irqs_enable(CALLER_ADDR0,
  46. CALLER_ADDR1);
  47. }
  48. tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1);
  49. this_cpu_write(tracing_irq_cpu, 0);
  50. }
  51. lockdep_hardirqs_on_prepare(CALLER_ADDR0);
  52. lockdep_hardirqs_on(CALLER_ADDR0);
  53. }
  54. EXPORT_SYMBOL(trace_hardirqs_on);
  55. NOKPROBE_SYMBOL(trace_hardirqs_on);
  56. /*
  57. * Like trace_hardirqs_off() but without the lockdep invocation. This is
  58. * used in the low level entry code where the ordering vs. RCU is important
  59. * and lockdep uses a staged approach which splits the lockdep hardirq
  60. * tracking into a RCU on and a RCU off section.
  61. */
  62. void trace_hardirqs_off_finish(void)
  63. {
  64. if (!this_cpu_read(tracing_irq_cpu)) {
  65. this_cpu_write(tracing_irq_cpu, 1);
  66. tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1);
  67. if (!in_nmi()) {
  68. trace_irq_disable(CALLER_ADDR0, CALLER_ADDR1);
  69. trace_android_rvh_irqs_disable(CALLER_ADDR0,
  70. CALLER_ADDR1);
  71. }
  72. }
  73. }
  74. EXPORT_SYMBOL(trace_hardirqs_off_finish);
  75. NOKPROBE_SYMBOL(trace_hardirqs_off_finish);
  76. void trace_hardirqs_off(void)
  77. {
  78. lockdep_hardirqs_off(CALLER_ADDR0);
  79. if (!this_cpu_read(tracing_irq_cpu)) {
  80. this_cpu_write(tracing_irq_cpu, 1);
  81. tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1);
  82. if (!in_nmi()) {
  83. trace_irq_disable_rcuidle(CALLER_ADDR0, CALLER_ADDR1);
  84. trace_android_rvh_irqs_disable(CALLER_ADDR0,
  85. CALLER_ADDR1);
  86. }
  87. }
  88. }
  89. EXPORT_SYMBOL(trace_hardirqs_off);
  90. NOKPROBE_SYMBOL(trace_hardirqs_off);
  91. __visible void trace_hardirqs_on_caller(unsigned long caller_addr)
  92. {
  93. if (this_cpu_read(tracing_irq_cpu)) {
  94. if (!in_nmi()) {
  95. trace_irq_enable_rcuidle(CALLER_ADDR0, caller_addr);
  96. trace_android_rvh_irqs_enable(CALLER_ADDR0,
  97. caller_addr);
  98. }
  99. tracer_hardirqs_on(CALLER_ADDR0, caller_addr);
  100. this_cpu_write(tracing_irq_cpu, 0);
  101. }
  102. lockdep_hardirqs_on_prepare(CALLER_ADDR0);
  103. lockdep_hardirqs_on(CALLER_ADDR0);
  104. }
  105. EXPORT_SYMBOL(trace_hardirqs_on_caller);
  106. NOKPROBE_SYMBOL(trace_hardirqs_on_caller);
  107. __visible void trace_hardirqs_off_caller(unsigned long caller_addr)
  108. {
  109. lockdep_hardirqs_off(CALLER_ADDR0);
  110. if (!this_cpu_read(tracing_irq_cpu)) {
  111. this_cpu_write(tracing_irq_cpu, 1);
  112. tracer_hardirqs_off(CALLER_ADDR0, caller_addr);
  113. if (!in_nmi()) {
  114. trace_irq_disable_rcuidle(CALLER_ADDR0, caller_addr);
  115. trace_android_rvh_irqs_enable(CALLER_ADDR0,
  116. caller_addr);
  117. }
  118. }
  119. }
  120. EXPORT_SYMBOL(trace_hardirqs_off_caller);
  121. NOKPROBE_SYMBOL(trace_hardirqs_off_caller);
  122. #endif /* CONFIG_TRACE_IRQFLAGS */
  123. #ifdef CONFIG_TRACE_PREEMPT_TOGGLE
  124. void trace_preempt_on(unsigned long a0, unsigned long a1)
  125. {
  126. if (!in_nmi()) {
  127. trace_preempt_enable_rcuidle(a0, a1);
  128. trace_android_rvh_preempt_enable(a0, a1);
  129. }
  130. tracer_preempt_on(a0, a1);
  131. }
  132. void trace_preempt_off(unsigned long a0, unsigned long a1)
  133. {
  134. if (!in_nmi()) {
  135. trace_preempt_disable_rcuidle(a0, a1);
  136. trace_android_rvh_preempt_disable(a0, a1);
  137. }
  138. tracer_preempt_off(a0, a1);
  139. }
  140. #endif