context_tracking.h 5.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. #ifndef _LINUX_CONTEXT_TRACKING_H
  3. #define _LINUX_CONTEXT_TRACKING_H
  4. #include <linux/sched.h>
  5. #include <linux/vtime.h>
  6. #include <linux/context_tracking_state.h>
  7. #include <linux/instrumentation.h>
  8. #include <asm/ptrace.h>
  9. #ifdef CONFIG_CONTEXT_TRACKING
  10. extern void context_tracking_cpu_set(int cpu);
  11. /* Called with interrupts disabled. */
  12. extern void __context_tracking_enter(enum ctx_state state);
  13. extern void __context_tracking_exit(enum ctx_state state);
  14. extern void context_tracking_enter(enum ctx_state state);
  15. extern void context_tracking_exit(enum ctx_state state);
  16. extern void context_tracking_user_enter(void);
  17. extern void context_tracking_user_exit(void);
  18. static inline void user_enter(void)
  19. {
  20. if (context_tracking_enabled())
  21. context_tracking_enter(CONTEXT_USER);
  22. }
  23. static inline void user_exit(void)
  24. {
  25. if (context_tracking_enabled())
  26. context_tracking_exit(CONTEXT_USER);
  27. }
  28. /* Called with interrupts disabled. */
  29. static __always_inline void user_enter_irqoff(void)
  30. {
  31. if (context_tracking_enabled())
  32. __context_tracking_enter(CONTEXT_USER);
  33. }
  34. static __always_inline void user_exit_irqoff(void)
  35. {
  36. if (context_tracking_enabled())
  37. __context_tracking_exit(CONTEXT_USER);
  38. }
  39. static inline enum ctx_state exception_enter(void)
  40. {
  41. enum ctx_state prev_ctx;
  42. if (!context_tracking_enabled())
  43. return 0;
  44. prev_ctx = this_cpu_read(context_tracking.state);
  45. if (prev_ctx != CONTEXT_KERNEL)
  46. context_tracking_exit(prev_ctx);
  47. return prev_ctx;
  48. }
  49. static inline void exception_exit(enum ctx_state prev_ctx)
  50. {
  51. if (context_tracking_enabled()) {
  52. if (prev_ctx != CONTEXT_KERNEL)
  53. context_tracking_enter(prev_ctx);
  54. }
  55. }
  56. /**
  57. * ct_state() - return the current context tracking state if known
  58. *
  59. * Returns the current cpu's context tracking state if context tracking
  60. * is enabled. If context tracking is disabled, returns
  61. * CONTEXT_DISABLED. This should be used primarily for debugging.
  62. */
  63. static __always_inline enum ctx_state ct_state(void)
  64. {
  65. return context_tracking_enabled() ?
  66. this_cpu_read(context_tracking.state) : CONTEXT_DISABLED;
  67. }
  68. #else
  69. static inline void user_enter(void) { }
  70. static inline void user_exit(void) { }
  71. static inline void user_enter_irqoff(void) { }
  72. static inline void user_exit_irqoff(void) { }
  73. static inline enum ctx_state exception_enter(void) { return 0; }
  74. static inline void exception_exit(enum ctx_state prev_ctx) { }
  75. static inline enum ctx_state ct_state(void) { return CONTEXT_DISABLED; }
  76. #endif /* !CONFIG_CONTEXT_TRACKING */
  77. #define CT_WARN_ON(cond) WARN_ON(context_tracking_enabled() && (cond))
  78. #ifdef CONFIG_CONTEXT_TRACKING_FORCE
  79. extern void context_tracking_init(void);
  80. #else
  81. static inline void context_tracking_init(void) { }
  82. #endif /* CONFIG_CONTEXT_TRACKING_FORCE */
  83. #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
  84. /* must be called with irqs disabled */
  85. static __always_inline void guest_enter_irqoff(void)
  86. {
  87. instrumentation_begin();
  88. if (vtime_accounting_enabled_this_cpu())
  89. vtime_guest_enter(current);
  90. else
  91. current->flags |= PF_VCPU;
  92. instrumentation_end();
  93. if (context_tracking_enabled())
  94. __context_tracking_enter(CONTEXT_GUEST);
  95. /* KVM does not hold any references to rcu protected data when it
  96. * switches CPU into a guest mode. In fact switching to a guest mode
  97. * is very similar to exiting to userspace from rcu point of view. In
  98. * addition CPU may stay in a guest mode for quite a long time (up to
  99. * one time slice). Lets treat guest mode as quiescent state, just like
  100. * we do with user-mode execution.
  101. */
  102. if (!context_tracking_enabled_this_cpu()) {
  103. instrumentation_begin();
  104. rcu_virt_note_context_switch(smp_processor_id());
  105. instrumentation_end();
  106. }
  107. }
  108. static __always_inline void context_tracking_guest_exit(void)
  109. {
  110. if (context_tracking_enabled())
  111. __context_tracking_exit(CONTEXT_GUEST);
  112. }
  113. static __always_inline void vtime_account_guest_exit(void)
  114. {
  115. if (vtime_accounting_enabled_this_cpu())
  116. vtime_guest_exit(current);
  117. else
  118. current->flags &= ~PF_VCPU;
  119. }
  120. static __always_inline void guest_exit_irqoff(void)
  121. {
  122. context_tracking_guest_exit();
  123. instrumentation_begin();
  124. vtime_account_guest_exit();
  125. instrumentation_end();
  126. }
  127. #else
  128. static __always_inline void guest_enter_irqoff(void)
  129. {
  130. /*
  131. * This is running in ioctl context so its safe
  132. * to assume that it's the stime pending cputime
  133. * to flush.
  134. */
  135. instrumentation_begin();
  136. vtime_account_kernel(current);
  137. current->flags |= PF_VCPU;
  138. rcu_virt_note_context_switch(smp_processor_id());
  139. instrumentation_end();
  140. }
  141. static __always_inline void context_tracking_guest_exit(void) { }
  142. static __always_inline void vtime_account_guest_exit(void)
  143. {
  144. vtime_account_kernel(current);
  145. current->flags &= ~PF_VCPU;
  146. }
  147. static __always_inline void guest_exit_irqoff(void)
  148. {
  149. instrumentation_begin();
  150. /* Flush the guest cputime we spent on the guest */
  151. vtime_account_guest_exit();
  152. instrumentation_end();
  153. }
  154. #endif /* CONFIG_VIRT_CPU_ACCOUNTING_GEN */
  155. static inline void guest_exit(void)
  156. {
  157. unsigned long flags;
  158. local_irq_save(flags);
  159. guest_exit_irqoff();
  160. local_irq_restore(flags);
  161. }
  162. #endif