trace_clock.c 4.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * tracing clocks
  4. *
  5. * Copyright (C) 2009 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
  6. *
  7. * Implements 3 trace clock variants, with differing scalability/precision
  8. * tradeoffs:
  9. *
  10. * - local: CPU-local trace clock
  11. * - medium: scalable global clock with some jitter
  12. * - global: globally monotonic, serialized clock
  13. *
  14. * Tracer plugins will chose a default from these clocks.
  15. */
  16. #include <linux/spinlock.h>
  17. #include <linux/irqflags.h>
  18. #include <linux/hardirq.h>
  19. #include <linux/module.h>
  20. #include <linux/percpu.h>
  21. #include <linux/sched.h>
  22. #include <linux/sched/clock.h>
  23. #include <linux/ktime.h>
  24. #include <linux/trace_clock.h>
  25. /*
  26. * trace_clock_local(): the simplest and least coherent tracing clock.
  27. *
  28. * Useful for tracing that does not cross to other CPUs nor
  29. * does it go through idle events.
  30. */
  31. u64 notrace trace_clock_local(void)
  32. {
  33. u64 clock;
  34. /*
  35. * sched_clock() is an architecture implemented, fast, scalable,
  36. * lockless clock. It is not guaranteed to be coherent across
  37. * CPUs, nor across CPU idle events.
  38. */
  39. preempt_disable_notrace();
  40. clock = sched_clock();
  41. preempt_enable_notrace();
  42. return clock;
  43. }
  44. EXPORT_SYMBOL_GPL(trace_clock_local);
  45. /*
  46. * trace_clock(): 'between' trace clock. Not completely serialized,
  47. * but not completely incorrect when crossing CPUs either.
  48. *
  49. * This is based on cpu_clock(), which will allow at most ~1 jiffy of
  50. * jitter between CPUs. So it's a pretty scalable clock, but there
  51. * can be offsets in the trace data.
  52. */
  53. u64 notrace trace_clock(void)
  54. {
  55. return local_clock();
  56. }
  57. EXPORT_SYMBOL_GPL(trace_clock);
  58. /*
  59. * trace_jiffy_clock(): Simply use jiffies as a clock counter.
  60. * Note that this use of jiffies_64 is not completely safe on
  61. * 32-bit systems. But the window is tiny, and the effect if
  62. * we are affected is that we will have an obviously bogus
  63. * timestamp on a trace event - i.e. not life threatening.
  64. */
  65. u64 notrace trace_clock_jiffies(void)
  66. {
  67. return jiffies_64_to_clock_t(jiffies_64 - INITIAL_JIFFIES);
  68. }
  69. EXPORT_SYMBOL_GPL(trace_clock_jiffies);
  70. /*
  71. * trace_clock_global(): special globally coherent trace clock
  72. *
  73. * It has higher overhead than the other trace clocks but is still
  74. * an order of magnitude faster than GTOD derived hardware clocks.
  75. *
  76. * Used by plugins that need globally coherent timestamps.
  77. */
  78. /* keep prev_time and lock in the same cacheline. */
  79. static struct {
  80. u64 prev_time;
  81. arch_spinlock_t lock;
  82. } trace_clock_struct ____cacheline_aligned_in_smp =
  83. {
  84. .lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED,
  85. };
  86. u64 notrace trace_clock_global(void)
  87. {
  88. unsigned long flags;
  89. int this_cpu;
  90. u64 now, prev_time;
  91. raw_local_irq_save(flags);
  92. this_cpu = raw_smp_processor_id();
  93. /*
  94. * The global clock "guarantees" that the events are ordered
  95. * between CPUs. But if two events on two different CPUS call
  96. * trace_clock_global at roughly the same time, it really does
  97. * not matter which one gets the earlier time. Just make sure
  98. * that the same CPU will always show a monotonic clock.
  99. *
  100. * Use a read memory barrier to get the latest written
  101. * time that was recorded.
  102. */
  103. smp_rmb();
  104. prev_time = READ_ONCE(trace_clock_struct.prev_time);
  105. now = sched_clock_cpu(this_cpu);
  106. /* Make sure that now is always greater than or equal to prev_time */
  107. if ((s64)(now - prev_time) < 0)
  108. now = prev_time;
  109. /*
  110. * If in an NMI context then dont risk lockups and simply return
  111. * the current time.
  112. */
  113. if (unlikely(in_nmi()))
  114. goto out;
  115. /* Tracing can cause strange recursion, always use a try lock */
  116. if (arch_spin_trylock(&trace_clock_struct.lock)) {
  117. /* Reread prev_time in case it was already updated */
  118. prev_time = READ_ONCE(trace_clock_struct.prev_time);
  119. if ((s64)(now - prev_time) < 0)
  120. now = prev_time;
  121. trace_clock_struct.prev_time = now;
  122. /* The unlock acts as the wmb for the above rmb */
  123. arch_spin_unlock(&trace_clock_struct.lock);
  124. }
  125. out:
  126. raw_local_irq_restore(flags);
  127. return now;
  128. }
  129. EXPORT_SYMBOL_GPL(trace_clock_global);
  130. static atomic64_t trace_counter;
  131. /*
  132. * trace_clock_counter(): simply an atomic counter.
  133. * Use the trace_counter "counter" for cases where you do not care
  134. * about timings, but are interested in strict ordering.
  135. */
  136. u64 notrace trace_clock_counter(void)
  137. {
  138. return atomic64_add_return(1, &trace_counter);
  139. }