hrtimer.h 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * hrtimers - High-resolution kernel timers
  4. *
  5. * Copyright(C) 2005, Thomas Gleixner <tglx@linutronix.de>
  6. * Copyright(C) 2005, Red Hat, Inc., Ingo Molnar
  7. *
  8. * data type definitions, declarations, prototypes
  9. *
  10. * Started by: Thomas Gleixner and Ingo Molnar
  11. */
  12. #ifndef _LINUX_HRTIMER_H
  13. #define _LINUX_HRTIMER_H
  14. #include <linux/hrtimer_defs.h>
  15. #include <linux/rbtree.h>
  16. #include <linux/init.h>
  17. #include <linux/list.h>
  18. #include <linux/percpu.h>
  19. #include <linux/seqlock.h>
  20. #include <linux/timer.h>
  21. #include <linux/timerqueue.h>
  22. #include <linux/android_kabi.h>
  23. struct hrtimer_clock_base;
  24. struct hrtimer_cpu_base;
  25. /*
  26. * Mode arguments of xxx_hrtimer functions:
  27. *
  28. * HRTIMER_MODE_ABS - Time value is absolute
  29. * HRTIMER_MODE_REL - Time value is relative to now
  30. * HRTIMER_MODE_PINNED - Timer is bound to CPU (is only considered
  31. * when starting the timer)
  32. * HRTIMER_MODE_SOFT - Timer callback function will be executed in
  33. * soft irq context
  34. * HRTIMER_MODE_HARD - Timer callback function will be executed in
  35. * hard irq context even on PREEMPT_RT.
  36. */
  37. enum hrtimer_mode {
  38. HRTIMER_MODE_ABS = 0x00,
  39. HRTIMER_MODE_REL = 0x01,
  40. HRTIMER_MODE_PINNED = 0x02,
  41. HRTIMER_MODE_SOFT = 0x04,
  42. HRTIMER_MODE_HARD = 0x08,
  43. HRTIMER_MODE_ABS_PINNED = HRTIMER_MODE_ABS | HRTIMER_MODE_PINNED,
  44. HRTIMER_MODE_REL_PINNED = HRTIMER_MODE_REL | HRTIMER_MODE_PINNED,
  45. HRTIMER_MODE_ABS_SOFT = HRTIMER_MODE_ABS | HRTIMER_MODE_SOFT,
  46. HRTIMER_MODE_REL_SOFT = HRTIMER_MODE_REL | HRTIMER_MODE_SOFT,
  47. HRTIMER_MODE_ABS_PINNED_SOFT = HRTIMER_MODE_ABS_PINNED | HRTIMER_MODE_SOFT,
  48. HRTIMER_MODE_REL_PINNED_SOFT = HRTIMER_MODE_REL_PINNED | HRTIMER_MODE_SOFT,
  49. HRTIMER_MODE_ABS_HARD = HRTIMER_MODE_ABS | HRTIMER_MODE_HARD,
  50. HRTIMER_MODE_REL_HARD = HRTIMER_MODE_REL | HRTIMER_MODE_HARD,
  51. HRTIMER_MODE_ABS_PINNED_HARD = HRTIMER_MODE_ABS_PINNED | HRTIMER_MODE_HARD,
  52. HRTIMER_MODE_REL_PINNED_HARD = HRTIMER_MODE_REL_PINNED | HRTIMER_MODE_HARD,
  53. };
  54. /*
  55. * Return values for the callback function
  56. */
  57. enum hrtimer_restart {
  58. HRTIMER_NORESTART, /* Timer is not restarted */
  59. HRTIMER_RESTART, /* Timer must be restarted */
  60. };
  61. /*
  62. * Values to track state of the timer
  63. *
  64. * Possible states:
  65. *
  66. * 0x00 inactive
  67. * 0x01 enqueued into rbtree
  68. *
  69. * The callback state is not part of the timer->state because clearing it would
  70. * mean touching the timer after the callback, this makes it impossible to free
  71. * the timer from the callback function.
  72. *
  73. * Therefore we track the callback state in:
  74. *
  75. * timer->base->cpu_base->running == timer
  76. *
  77. * On SMP it is possible to have a "callback function running and enqueued"
  78. * status. It happens for example when a posix timer expired and the callback
  79. * queued a signal. Between dropping the lock which protects the posix timer
  80. * and reacquiring the base lock of the hrtimer, another CPU can deliver the
  81. * signal and rearm the timer.
  82. *
  83. * All state transitions are protected by cpu_base->lock.
  84. */
  85. #define HRTIMER_STATE_INACTIVE 0x00
  86. #define HRTIMER_STATE_ENQUEUED 0x01
  87. /**
  88. * struct hrtimer - the basic hrtimer structure
  89. * @node: timerqueue node, which also manages node.expires,
  90. * the absolute expiry time in the hrtimers internal
  91. * representation. The time is related to the clock on
  92. * which the timer is based. Is setup by adding
  93. * slack to the _softexpires value. For non range timers
  94. * identical to _softexpires.
  95. * @_softexpires: the absolute earliest expiry time of the hrtimer.
  96. * The time which was given as expiry time when the timer
  97. * was armed.
  98. * @function: timer expiry callback function
  99. * @base: pointer to the timer base (per cpu and per clock)
  100. * @state: state information (See bit values above)
  101. * @is_rel: Set if the timer was armed relative
  102. * @is_soft: Set if hrtimer will be expired in soft interrupt context.
  103. * @is_hard: Set if hrtimer will be expired in hard interrupt context
  104. * even on RT.
  105. *
  106. * The hrtimer structure must be initialized by hrtimer_init()
  107. */
  108. struct hrtimer {
  109. struct timerqueue_node node;
  110. ktime_t _softexpires;
  111. enum hrtimer_restart (*function)(struct hrtimer *);
  112. struct hrtimer_clock_base *base;
  113. u8 state;
  114. u8 is_rel;
  115. u8 is_soft;
  116. u8 is_hard;
  117. ANDROID_KABI_RESERVE(1);
  118. };
  119. /**
  120. * struct hrtimer_sleeper - simple sleeper structure
  121. * @timer: embedded timer structure
  122. * @task: task to wake up
  123. *
  124. * task is set to NULL, when the timer expires.
  125. */
  126. struct hrtimer_sleeper {
  127. struct hrtimer timer;
  128. struct task_struct *task;
  129. };
  130. #ifdef CONFIG_64BIT
  131. # define __hrtimer_clock_base_align ____cacheline_aligned
  132. #else
  133. # define __hrtimer_clock_base_align
  134. #endif
  135. /**
  136. * struct hrtimer_clock_base - the timer base for a specific clock
  137. * @cpu_base: per cpu clock base
  138. * @index: clock type index for per_cpu support when moving a
  139. * timer to a base on another cpu.
  140. * @clockid: clock id for per_cpu support
  141. * @seq: seqcount around __run_hrtimer
  142. * @running: pointer to the currently running hrtimer
  143. * @active: red black tree root node for the active timers
  144. * @get_time: function to retrieve the current time of the clock
  145. * @offset: offset of this clock to the monotonic base
  146. */
  147. struct hrtimer_clock_base {
  148. struct hrtimer_cpu_base *cpu_base;
  149. unsigned int index;
  150. clockid_t clockid;
  151. seqcount_raw_spinlock_t seq;
  152. struct hrtimer *running;
  153. struct timerqueue_head active;
  154. ktime_t (*get_time)(void);
  155. ktime_t offset;
  156. } __hrtimer_clock_base_align;
  157. enum hrtimer_base_type {
  158. HRTIMER_BASE_MONOTONIC,
  159. HRTIMER_BASE_REALTIME,
  160. HRTIMER_BASE_BOOTTIME,
  161. HRTIMER_BASE_TAI,
  162. HRTIMER_BASE_MONOTONIC_SOFT,
  163. HRTIMER_BASE_REALTIME_SOFT,
  164. HRTIMER_BASE_BOOTTIME_SOFT,
  165. HRTIMER_BASE_TAI_SOFT,
  166. HRTIMER_MAX_CLOCK_BASES,
  167. };
  168. /**
  169. * struct hrtimer_cpu_base - the per cpu clock bases
  170. * @lock: lock protecting the base and associated clock bases
  171. * and timers
  172. * @cpu: cpu number
  173. * @active_bases: Bitfield to mark bases with active timers
  174. * @clock_was_set_seq: Sequence counter of clock was set events
  175. * @hres_active: State of high resolution mode
  176. * @in_hrtirq: hrtimer_interrupt() is currently executing
  177. * @hang_detected: The last hrtimer interrupt detected a hang
  178. * @softirq_activated: displays, if the softirq is raised - update of softirq
  179. * related settings is not required then.
  180. * @nr_events: Total number of hrtimer interrupt events
  181. * @nr_retries: Total number of hrtimer interrupt retries
  182. * @nr_hangs: Total number of hrtimer interrupt hangs
  183. * @max_hang_time: Maximum time spent in hrtimer_interrupt
  184. * @softirq_expiry_lock: Lock which is taken while softirq based hrtimer are
  185. * expired
  186. * @timer_waiters: A hrtimer_cancel() invocation waits for the timer
  187. * callback to finish.
  188. * @expires_next: absolute time of the next event, is required for remote
  189. * hrtimer enqueue; it is the total first expiry time (hard
  190. * and soft hrtimer are taken into account)
  191. * @next_timer: Pointer to the first expiring timer
  192. * @softirq_expires_next: Time to check, if soft queues needs also to be expired
  193. * @softirq_next_timer: Pointer to the first expiring softirq based timer
  194. * @clock_base: array of clock bases for this cpu
  195. *
  196. * Note: next_timer is just an optimization for __remove_hrtimer().
  197. * Do not dereference the pointer because it is not reliable on
  198. * cross cpu removals.
  199. */
  200. struct hrtimer_cpu_base {
  201. raw_spinlock_t lock;
  202. unsigned int cpu;
  203. unsigned int active_bases;
  204. unsigned int clock_was_set_seq;
  205. unsigned int hres_active : 1,
  206. in_hrtirq : 1,
  207. hang_detected : 1,
  208. softirq_activated : 1;
  209. #ifdef CONFIG_HIGH_RES_TIMERS
  210. unsigned int nr_events;
  211. unsigned short nr_retries;
  212. unsigned short nr_hangs;
  213. unsigned int max_hang_time;
  214. #endif
  215. #ifdef CONFIG_PREEMPT_RT
  216. spinlock_t softirq_expiry_lock;
  217. atomic_t timer_waiters;
  218. #endif
  219. ktime_t expires_next;
  220. struct hrtimer *next_timer;
  221. ktime_t softirq_expires_next;
  222. struct hrtimer *softirq_next_timer;
  223. struct hrtimer_clock_base clock_base[HRTIMER_MAX_CLOCK_BASES];
  224. } ____cacheline_aligned;
  225. static inline void hrtimer_set_expires(struct hrtimer *timer, ktime_t time)
  226. {
  227. timer->node.expires = time;
  228. timer->_softexpires = time;
  229. }
  230. static inline void hrtimer_set_expires_range(struct hrtimer *timer, ktime_t time, ktime_t delta)
  231. {
  232. timer->_softexpires = time;
  233. timer->node.expires = ktime_add_safe(time, delta);
  234. }
  235. static inline void hrtimer_set_expires_range_ns(struct hrtimer *timer, ktime_t time, u64 delta)
  236. {
  237. timer->_softexpires = time;
  238. timer->node.expires = ktime_add_safe(time, ns_to_ktime(delta));
  239. }
  240. static inline void hrtimer_set_expires_tv64(struct hrtimer *timer, s64 tv64)
  241. {
  242. timer->node.expires = tv64;
  243. timer->_softexpires = tv64;
  244. }
  245. static inline void hrtimer_add_expires(struct hrtimer *timer, ktime_t time)
  246. {
  247. timer->node.expires = ktime_add_safe(timer->node.expires, time);
  248. timer->_softexpires = ktime_add_safe(timer->_softexpires, time);
  249. }
  250. static inline void hrtimer_add_expires_ns(struct hrtimer *timer, u64 ns)
  251. {
  252. timer->node.expires = ktime_add_ns(timer->node.expires, ns);
  253. timer->_softexpires = ktime_add_ns(timer->_softexpires, ns);
  254. }
  255. static inline ktime_t hrtimer_get_expires(const struct hrtimer *timer)
  256. {
  257. return timer->node.expires;
  258. }
  259. static inline ktime_t hrtimer_get_softexpires(const struct hrtimer *timer)
  260. {
  261. return timer->_softexpires;
  262. }
  263. static inline s64 hrtimer_get_expires_tv64(const struct hrtimer *timer)
  264. {
  265. return timer->node.expires;
  266. }
  267. static inline s64 hrtimer_get_softexpires_tv64(const struct hrtimer *timer)
  268. {
  269. return timer->_softexpires;
  270. }
  271. static inline s64 hrtimer_get_expires_ns(const struct hrtimer *timer)
  272. {
  273. return ktime_to_ns(timer->node.expires);
  274. }
  275. static inline ktime_t hrtimer_expires_remaining(const struct hrtimer *timer)
  276. {
  277. return ktime_sub(timer->node.expires, timer->base->get_time());
  278. }
  279. static inline ktime_t hrtimer_cb_get_time(struct hrtimer *timer)
  280. {
  281. return timer->base->get_time();
  282. }
  283. static inline int hrtimer_is_hres_active(struct hrtimer *timer)
  284. {
  285. return IS_ENABLED(CONFIG_HIGH_RES_TIMERS) ?
  286. timer->base->cpu_base->hres_active : 0;
  287. }
  288. #ifdef CONFIG_HIGH_RES_TIMERS
  289. struct clock_event_device;
  290. extern void hrtimer_interrupt(struct clock_event_device *dev);
  291. extern unsigned int hrtimer_resolution;
  292. #else
  293. #define hrtimer_resolution (unsigned int)LOW_RES_NSEC
  294. #endif
  295. static inline ktime_t
  296. __hrtimer_expires_remaining_adjusted(const struct hrtimer *timer, ktime_t now)
  297. {
  298. ktime_t rem = ktime_sub(timer->node.expires, now);
  299. /*
  300. * Adjust relative timers for the extra we added in
  301. * hrtimer_start_range_ns() to prevent short timeouts.
  302. */
  303. if (IS_ENABLED(CONFIG_TIME_LOW_RES) && timer->is_rel)
  304. rem -= hrtimer_resolution;
  305. return rem;
  306. }
  307. static inline ktime_t
  308. hrtimer_expires_remaining_adjusted(const struct hrtimer *timer)
  309. {
  310. return __hrtimer_expires_remaining_adjusted(timer,
  311. timer->base->get_time());
  312. }
  313. #ifdef CONFIG_TIMERFD
  314. extern void timerfd_clock_was_set(void);
  315. #else
  316. static inline void timerfd_clock_was_set(void) { }
  317. #endif
  318. extern void hrtimers_resume(void);
  319. DECLARE_PER_CPU(struct tick_device, tick_cpu_device);
  320. #ifdef CONFIG_PREEMPT_RT
  321. void hrtimer_cancel_wait_running(const struct hrtimer *timer);
  322. #else
  323. static inline void hrtimer_cancel_wait_running(struct hrtimer *timer)
  324. {
  325. cpu_relax();
  326. }
  327. #endif
  328. /* Exported timer functions: */
  329. /* Initialize timers: */
  330. extern void hrtimer_init(struct hrtimer *timer, clockid_t which_clock,
  331. enum hrtimer_mode mode);
  332. extern void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, clockid_t clock_id,
  333. enum hrtimer_mode mode);
  334. #ifdef CONFIG_DEBUG_OBJECTS_TIMERS
  335. extern void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t which_clock,
  336. enum hrtimer_mode mode);
  337. extern void hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper *sl,
  338. clockid_t clock_id,
  339. enum hrtimer_mode mode);
  340. extern void destroy_hrtimer_on_stack(struct hrtimer *timer);
  341. #else
  342. static inline void hrtimer_init_on_stack(struct hrtimer *timer,
  343. clockid_t which_clock,
  344. enum hrtimer_mode mode)
  345. {
  346. hrtimer_init(timer, which_clock, mode);
  347. }
  348. static inline void hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper *sl,
  349. clockid_t clock_id,
  350. enum hrtimer_mode mode)
  351. {
  352. hrtimer_init_sleeper(sl, clock_id, mode);
  353. }
  354. static inline void destroy_hrtimer_on_stack(struct hrtimer *timer) { }
  355. #endif
  356. /* Basic timer operations: */
  357. extern void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
  358. u64 range_ns, const enum hrtimer_mode mode);
  359. /**
  360. * hrtimer_start - (re)start an hrtimer
  361. * @timer: the timer to be added
  362. * @tim: expiry time
  363. * @mode: timer mode: absolute (HRTIMER_MODE_ABS) or
  364. * relative (HRTIMER_MODE_REL), and pinned (HRTIMER_MODE_PINNED);
  365. * softirq based mode is considered for debug purpose only!
  366. */
  367. static inline void hrtimer_start(struct hrtimer *timer, ktime_t tim,
  368. const enum hrtimer_mode mode)
  369. {
  370. hrtimer_start_range_ns(timer, tim, 0, mode);
  371. }
  372. extern int hrtimer_cancel(struct hrtimer *timer);
  373. extern int hrtimer_try_to_cancel(struct hrtimer *timer);
  374. static inline void hrtimer_start_expires(struct hrtimer *timer,
  375. enum hrtimer_mode mode)
  376. {
  377. u64 delta;
  378. ktime_t soft, hard;
  379. soft = hrtimer_get_softexpires(timer);
  380. hard = hrtimer_get_expires(timer);
  381. delta = ktime_to_ns(ktime_sub(hard, soft));
  382. hrtimer_start_range_ns(timer, soft, delta, mode);
  383. }
  384. void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl,
  385. enum hrtimer_mode mode);
  386. static inline void hrtimer_restart(struct hrtimer *timer)
  387. {
  388. hrtimer_start_expires(timer, HRTIMER_MODE_ABS);
  389. }
  390. /* Query timers: */
  391. extern ktime_t __hrtimer_get_remaining(const struct hrtimer *timer, bool adjust);
  392. static inline ktime_t hrtimer_get_remaining(const struct hrtimer *timer)
  393. {
  394. return __hrtimer_get_remaining(timer, false);
  395. }
  396. extern u64 hrtimer_get_next_event(void);
  397. extern u64 hrtimer_next_event_without(const struct hrtimer *exclude);
  398. extern bool hrtimer_active(const struct hrtimer *timer);
  399. /**
  400. * hrtimer_is_queued = check, whether the timer is on one of the queues
  401. * @timer: Timer to check
  402. *
  403. * Returns: True if the timer is queued, false otherwise
  404. *
  405. * The function can be used lockless, but it gives only a current snapshot.
  406. */
  407. static inline bool hrtimer_is_queued(struct hrtimer *timer)
  408. {
  409. /* The READ_ONCE pairs with the update functions of timer->state */
  410. return !!(READ_ONCE(timer->state) & HRTIMER_STATE_ENQUEUED);
  411. }
  412. /*
  413. * Helper function to check, whether the timer is running the callback
  414. * function
  415. */
  416. static inline int hrtimer_callback_running(struct hrtimer *timer)
  417. {
  418. return timer->base->running == timer;
  419. }
  420. /* Forward a hrtimer so it expires after now: */
  421. extern u64
  422. hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval);
  423. /**
  424. * hrtimer_forward_now - forward the timer expiry so it expires after now
  425. * @timer: hrtimer to forward
  426. * @interval: the interval to forward
  427. *
  428. * Forward the timer expiry so it will expire after the current time
  429. * of the hrtimer clock base. Returns the number of overruns.
  430. *
  431. * Can be safely called from the callback function of @timer. If
  432. * called from other contexts @timer must neither be enqueued nor
  433. * running the callback and the caller needs to take care of
  434. * serialization.
  435. *
  436. * Note: This only updates the timer expiry value and does not requeue
  437. * the timer.
  438. */
  439. static inline u64 hrtimer_forward_now(struct hrtimer *timer,
  440. ktime_t interval)
  441. {
  442. return hrtimer_forward(timer, timer->base->get_time(), interval);
  443. }
  444. /* Precise sleep: */
  445. extern int nanosleep_copyout(struct restart_block *, struct timespec64 *);
  446. extern long hrtimer_nanosleep(ktime_t rqtp, const enum hrtimer_mode mode,
  447. const clockid_t clockid);
  448. extern int schedule_hrtimeout_range(ktime_t *expires, u64 delta,
  449. const enum hrtimer_mode mode);
  450. extern int schedule_hrtimeout_range_clock(ktime_t *expires,
  451. u64 delta,
  452. const enum hrtimer_mode mode,
  453. clockid_t clock_id);
  454. extern int schedule_hrtimeout(ktime_t *expires, const enum hrtimer_mode mode);
  455. /* Soft interrupt function to run the hrtimer queues: */
  456. extern void hrtimer_run_queues(void);
  457. /* Bootup initialization: */
  458. extern void __init hrtimers_init(void);
  459. /* Show pending timers: */
  460. extern void sysrq_timer_list_show(void);
  461. int hrtimers_prepare_cpu(unsigned int cpu);
  462. #ifdef CONFIG_HOTPLUG_CPU
  463. int hrtimers_dead_cpu(unsigned int cpu);
  464. #else
  465. #define hrtimers_dead_cpu NULL
  466. #endif
  467. #endif