hardirq.h 3.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. #ifndef LINUX_HARDIRQ_H
  3. #define LINUX_HARDIRQ_H
  4. #include <linux/context_tracking_state.h>
  5. #include <linux/preempt.h>
  6. #include <linux/lockdep.h>
  7. #include <linux/ftrace_irq.h>
  8. #include <linux/vtime.h>
  9. #include <asm/hardirq.h>
  10. extern void synchronize_irq(unsigned int irq);
  11. extern bool synchronize_hardirq(unsigned int irq);
  12. #ifdef CONFIG_NO_HZ_FULL
  13. void __rcu_irq_enter_check_tick(void);
  14. #else
  15. static inline void __rcu_irq_enter_check_tick(void) { }
  16. #endif
  17. static __always_inline void rcu_irq_enter_check_tick(void)
  18. {
  19. if (context_tracking_enabled())
  20. __rcu_irq_enter_check_tick();
  21. }
  22. /*
  23. * It is safe to do non-atomic ops on ->hardirq_context,
  24. * because NMI handlers may not preempt and the ops are
  25. * always balanced, so the interrupted value of ->hardirq_context
  26. * will always be restored.
  27. */
  28. #define __irq_enter() \
  29. do { \
  30. account_irq_enter_time(current); \
  31. preempt_count_add(HARDIRQ_OFFSET); \
  32. lockdep_hardirq_enter(); \
  33. } while (0)
  34. /*
  35. * Like __irq_enter() without time accounting for fast
  36. * interrupts, e.g. reschedule IPI where time accounting
  37. * is more expensive than the actual interrupt.
  38. */
  39. #define __irq_enter_raw() \
  40. do { \
  41. preempt_count_add(HARDIRQ_OFFSET); \
  42. lockdep_hardirq_enter(); \
  43. } while (0)
  44. /*
  45. * Enter irq context (on NO_HZ, update jiffies):
  46. */
  47. void irq_enter(void);
  48. /*
  49. * Like irq_enter(), but RCU is already watching.
  50. */
  51. void irq_enter_rcu(void);
  52. /*
  53. * Exit irq context without processing softirqs:
  54. */
  55. #define __irq_exit() \
  56. do { \
  57. lockdep_hardirq_exit(); \
  58. account_irq_exit_time(current); \
  59. preempt_count_sub(HARDIRQ_OFFSET); \
  60. } while (0)
  61. /*
  62. * Like __irq_exit() without time accounting
  63. */
  64. #define __irq_exit_raw() \
  65. do { \
  66. lockdep_hardirq_exit(); \
  67. preempt_count_sub(HARDIRQ_OFFSET); \
  68. } while (0)
  69. /*
  70. * Exit irq context and process softirqs if needed:
  71. */
  72. void irq_exit(void);
  73. /*
  74. * Like irq_exit(), but return with RCU watching.
  75. */
  76. void irq_exit_rcu(void);
  77. #ifndef arch_nmi_enter
  78. #define arch_nmi_enter() do { } while (0)
  79. #define arch_nmi_exit() do { } while (0)
  80. #endif
  81. #ifdef CONFIG_TINY_RCU
  82. static inline void rcu_nmi_enter(void) { }
  83. static inline void rcu_nmi_exit(void) { }
  84. #else
  85. extern void rcu_nmi_enter(void);
  86. extern void rcu_nmi_exit(void);
  87. #endif
  88. /*
  89. * NMI vs Tracing
  90. * --------------
  91. *
  92. * We must not land in a tracer until (or after) we've changed preempt_count
  93. * such that in_nmi() becomes true. To that effect all NMI C entry points must
  94. * be marked 'notrace' and call nmi_enter() as soon as possible.
  95. */
  96. /*
  97. * nmi_enter() can nest up to 15 times; see NMI_BITS.
  98. */
  99. #define __nmi_enter() \
  100. do { \
  101. lockdep_off(); \
  102. arch_nmi_enter(); \
  103. printk_nmi_enter(); \
  104. BUG_ON(in_nmi() == NMI_MASK); \
  105. __preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET); \
  106. } while (0)
  107. #define nmi_enter() \
  108. do { \
  109. __nmi_enter(); \
  110. lockdep_hardirq_enter(); \
  111. rcu_nmi_enter(); \
  112. instrumentation_begin(); \
  113. ftrace_nmi_enter(); \
  114. instrumentation_end(); \
  115. } while (0)
  116. #define __nmi_exit() \
  117. do { \
  118. BUG_ON(!in_nmi()); \
  119. __preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); \
  120. printk_nmi_exit(); \
  121. arch_nmi_exit(); \
  122. lockdep_on(); \
  123. } while (0)
  124. #define nmi_exit() \
  125. do { \
  126. instrumentation_begin(); \
  127. ftrace_nmi_exit(); \
  128. instrumentation_end(); \
  129. rcu_nmi_exit(); \
  130. lockdep_hardirq_exit(); \
  131. __nmi_exit(); \
  132. } while (0)
  133. #endif /* LINUX_HARDIRQ_H */