context_tracking.c 6.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Context tracking: Probe on high level context boundaries such as kernel
  4. * and userspace. This includes syscalls and exceptions entry/exit.
  5. *
  6. * This is used by RCU to remove its dependency on the timer tick while a CPU
  7. * runs in userspace.
  8. *
  9. * Started by Frederic Weisbecker:
  10. *
  11. * Copyright (C) 2012 Red Hat, Inc., Frederic Weisbecker <fweisbec@redhat.com>
  12. *
  13. * Many thanks to Gilad Ben-Yossef, Paul McKenney, Ingo Molnar, Andrew Morton,
  14. * Steven Rostedt, Peter Zijlstra for suggestions and improvements.
  15. *
  16. */
  17. #include <linux/context_tracking.h>
  18. #include <linux/rcupdate.h>
  19. #include <linux/sched.h>
  20. #include <linux/hardirq.h>
  21. #include <linux/export.h>
  22. #include <linux/kprobes.h>
  23. #define CREATE_TRACE_POINTS
  24. #include <trace/events/context_tracking.h>
  25. DEFINE_STATIC_KEY_FALSE(context_tracking_key);
  26. EXPORT_SYMBOL_GPL(context_tracking_key);
  27. DEFINE_PER_CPU(struct context_tracking, context_tracking);
  28. EXPORT_SYMBOL_GPL(context_tracking);
  29. static noinstr bool context_tracking_recursion_enter(void)
  30. {
  31. int recursion;
  32. recursion = __this_cpu_inc_return(context_tracking.recursion);
  33. if (recursion == 1)
  34. return true;
  35. WARN_ONCE((recursion < 1), "Invalid context tracking recursion value %d\n", recursion);
  36. __this_cpu_dec(context_tracking.recursion);
  37. return false;
  38. }
  39. static __always_inline void context_tracking_recursion_exit(void)
  40. {
  41. __this_cpu_dec(context_tracking.recursion);
  42. }
  43. /**
  44. * context_tracking_enter - Inform the context tracking that the CPU is going
  45. * enter user or guest space mode.
  46. *
  47. * This function must be called right before we switch from the kernel
  48. * to user or guest space, when it's guaranteed the remaining kernel
  49. * instructions to execute won't use any RCU read side critical section
  50. * because this function sets RCU in extended quiescent state.
  51. */
  52. void noinstr __context_tracking_enter(enum ctx_state state)
  53. {
  54. /* Kernel threads aren't supposed to go to userspace */
  55. WARN_ON_ONCE(!current->mm);
  56. if (!context_tracking_recursion_enter())
  57. return;
  58. if ( __this_cpu_read(context_tracking.state) != state) {
  59. if (__this_cpu_read(context_tracking.active)) {
  60. /*
  61. * At this stage, only low level arch entry code remains and
  62. * then we'll run in userspace. We can assume there won't be
  63. * any RCU read-side critical section until the next call to
  64. * user_exit() or rcu_irq_enter(). Let's remove RCU's dependency
  65. * on the tick.
  66. */
  67. if (state == CONTEXT_USER) {
  68. instrumentation_begin();
  69. trace_user_enter(0);
  70. vtime_user_enter(current);
  71. instrumentation_end();
  72. }
  73. rcu_user_enter();
  74. }
  75. /*
  76. * Even if context tracking is disabled on this CPU, because it's outside
  77. * the full dynticks mask for example, we still have to keep track of the
  78. * context transitions and states to prevent inconsistency on those of
  79. * other CPUs.
  80. * If a task triggers an exception in userspace, sleep on the exception
  81. * handler and then migrate to another CPU, that new CPU must know where
  82. * the exception returns by the time we call exception_exit().
  83. * This information can only be provided by the previous CPU when it called
  84. * exception_enter().
  85. * OTOH we can spare the calls to vtime and RCU when context_tracking.active
  86. * is false because we know that CPU is not tickless.
  87. */
  88. __this_cpu_write(context_tracking.state, state);
  89. }
  90. context_tracking_recursion_exit();
  91. }
  92. EXPORT_SYMBOL_GPL(__context_tracking_enter);
  93. void context_tracking_enter(enum ctx_state state)
  94. {
  95. unsigned long flags;
  96. /*
  97. * Some contexts may involve an exception occuring in an irq,
  98. * leading to that nesting:
  99. * rcu_irq_enter() rcu_user_exit() rcu_user_exit() rcu_irq_exit()
  100. * This would mess up the dyntick_nesting count though. And rcu_irq_*()
  101. * helpers are enough to protect RCU uses inside the exception. So
  102. * just return immediately if we detect we are in an IRQ.
  103. */
  104. if (in_interrupt())
  105. return;
  106. local_irq_save(flags);
  107. __context_tracking_enter(state);
  108. local_irq_restore(flags);
  109. }
  110. NOKPROBE_SYMBOL(context_tracking_enter);
  111. EXPORT_SYMBOL_GPL(context_tracking_enter);
  112. void context_tracking_user_enter(void)
  113. {
  114. user_enter();
  115. }
  116. NOKPROBE_SYMBOL(context_tracking_user_enter);
  117. /**
  118. * context_tracking_exit - Inform the context tracking that the CPU is
  119. * exiting user or guest mode and entering the kernel.
  120. *
  121. * This function must be called after we entered the kernel from user or
  122. * guest space before any use of RCU read side critical section. This
  123. * potentially include any high level kernel code like syscalls, exceptions,
  124. * signal handling, etc...
  125. *
  126. * This call supports re-entrancy. This way it can be called from any exception
  127. * handler without needing to know if we came from userspace or not.
  128. */
  129. void noinstr __context_tracking_exit(enum ctx_state state)
  130. {
  131. if (!context_tracking_recursion_enter())
  132. return;
  133. if (__this_cpu_read(context_tracking.state) == state) {
  134. if (__this_cpu_read(context_tracking.active)) {
  135. /*
  136. * We are going to run code that may use RCU. Inform
  137. * RCU core about that (ie: we may need the tick again).
  138. */
  139. rcu_user_exit();
  140. if (state == CONTEXT_USER) {
  141. instrumentation_begin();
  142. vtime_user_exit(current);
  143. trace_user_exit(0);
  144. instrumentation_end();
  145. }
  146. }
  147. __this_cpu_write(context_tracking.state, CONTEXT_KERNEL);
  148. }
  149. context_tracking_recursion_exit();
  150. }
  151. EXPORT_SYMBOL_GPL(__context_tracking_exit);
  152. void context_tracking_exit(enum ctx_state state)
  153. {
  154. unsigned long flags;
  155. if (in_interrupt())
  156. return;
  157. local_irq_save(flags);
  158. __context_tracking_exit(state);
  159. local_irq_restore(flags);
  160. }
  161. NOKPROBE_SYMBOL(context_tracking_exit);
  162. EXPORT_SYMBOL_GPL(context_tracking_exit);
  163. void context_tracking_user_exit(void)
  164. {
  165. user_exit();
  166. }
  167. NOKPROBE_SYMBOL(context_tracking_user_exit);
  168. void __init context_tracking_cpu_set(int cpu)
  169. {
  170. static __initdata bool initialized = false;
  171. if (!per_cpu(context_tracking.active, cpu)) {
  172. per_cpu(context_tracking.active, cpu) = true;
  173. static_branch_inc(&context_tracking_key);
  174. }
  175. if (initialized)
  176. return;
  177. #ifdef CONFIG_HAVE_TIF_NOHZ
  178. /*
  179. * Set TIF_NOHZ to init/0 and let it propagate to all tasks through fork
  180. * This assumes that init is the only task at this early boot stage.
  181. */
  182. set_tsk_thread_flag(&init_task, TIF_NOHZ);
  183. #endif
  184. WARN_ON_ONCE(!tasklist_empty());
  185. initialized = true;
  186. }
  187. #ifdef CONFIG_CONTEXT_TRACKING_FORCE
  188. void __init context_tracking_init(void)
  189. {
  190. int cpu;
  191. for_each_possible_cpu(cpu)
  192. context_tracking_cpu_set(cpu);
  193. }
  194. #endif