tick-common.c 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * This file contains the base functions to manage periodic tick
  4. * related events.
  5. *
  6. * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
  7. * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
  8. * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
  9. */
  10. #include <linux/cpu.h>
  11. #include <linux/err.h>
  12. #include <linux/hrtimer.h>
  13. #include <linux/interrupt.h>
  14. #include <linux/nmi.h>
  15. #include <linux/percpu.h>
  16. #include <linux/profile.h>
  17. #include <linux/sched.h>
  18. #include <linux/module.h>
  19. #include <trace/events/power.h>
  20. #include <trace/hooks/sched.h>
  21. #include <asm/irq_regs.h>
  22. #include "tick-internal.h"
  23. /*
  24. * Tick devices
  25. */
  26. DEFINE_PER_CPU(struct tick_device, tick_cpu_device);
  27. /*
  28. * Tick next event: keeps track of the tick time
  29. */
  30. ktime_t tick_next_period;
  31. ktime_t tick_period;
  32. /*
  33. * tick_do_timer_cpu is a timer core internal variable which holds the CPU NR
  34. * which is responsible for calling do_timer(), i.e. the timekeeping stuff. This
  35. * variable has two functions:
  36. *
  37. * 1) Prevent a thundering herd issue of a gazillion of CPUs trying to grab the
  38. * timekeeping lock all at once. Only the CPU which is assigned to do the
  39. * update is handling it.
  40. *
  41. * 2) Hand off the duty in the NOHZ idle case by setting the value to
  42. * TICK_DO_TIMER_NONE, i.e. a non existing CPU. So the next cpu which looks
  43. * at it will take over and keep the time keeping alive. The handover
  44. * procedure also covers cpu hotplug.
  45. */
  46. int tick_do_timer_cpu __read_mostly = TICK_DO_TIMER_BOOT;
  47. #ifdef CONFIG_NO_HZ_FULL
  48. /*
  49. * tick_do_timer_boot_cpu indicates the boot CPU temporarily owns
  50. * tick_do_timer_cpu and it should be taken over by an eligible secondary
  51. * when one comes online.
  52. */
  53. static int tick_do_timer_boot_cpu __read_mostly = -1;
  54. #endif
  55. /*
  56. * Debugging: see timer_list.c
  57. */
  58. struct tick_device *tick_get_device(int cpu)
  59. {
  60. return &per_cpu(tick_cpu_device, cpu);
  61. }
  62. /**
  63. * tick_is_oneshot_available - check for a oneshot capable event device
  64. */
  65. int tick_is_oneshot_available(void)
  66. {
  67. struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
  68. if (!dev || !(dev->features & CLOCK_EVT_FEAT_ONESHOT))
  69. return 0;
  70. if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
  71. return 1;
  72. return tick_broadcast_oneshot_available();
  73. }
  74. /*
  75. * Periodic tick
  76. */
  77. static void tick_periodic(int cpu)
  78. {
  79. if (tick_do_timer_cpu == cpu) {
  80. raw_spin_lock(&jiffies_lock);
  81. write_seqcount_begin(&jiffies_seq);
  82. /* Keep track of the next tick event */
  83. tick_next_period = ktime_add(tick_next_period, tick_period);
  84. do_timer(1);
  85. write_seqcount_end(&jiffies_seq);
  86. raw_spin_unlock(&jiffies_lock);
  87. update_wall_time();
  88. trace_android_vh_jiffies_update(NULL);
  89. }
  90. update_process_times(user_mode(get_irq_regs()));
  91. profile_tick(CPU_PROFILING);
  92. }
  93. /*
  94. * Event handler for periodic ticks
  95. */
  96. void tick_handle_periodic(struct clock_event_device *dev)
  97. {
  98. int cpu = smp_processor_id();
  99. ktime_t next = dev->next_event;
  100. tick_periodic(cpu);
  101. #if defined(CONFIG_HIGH_RES_TIMERS) || defined(CONFIG_NO_HZ_COMMON)
  102. /*
  103. * The cpu might have transitioned to HIGHRES or NOHZ mode via
  104. * update_process_times() -> run_local_timers() ->
  105. * hrtimer_run_queues().
  106. */
  107. if (dev->event_handler != tick_handle_periodic)
  108. return;
  109. #endif
  110. if (!clockevent_state_oneshot(dev))
  111. return;
  112. for (;;) {
  113. /*
  114. * Setup the next period for devices, which do not have
  115. * periodic mode:
  116. */
  117. next = ktime_add(next, tick_period);
  118. if (!clockevents_program_event(dev, next, false))
  119. return;
  120. /*
  121. * Have to be careful here. If we're in oneshot mode,
  122. * before we call tick_periodic() in a loop, we need
  123. * to be sure we're using a real hardware clocksource.
  124. * Otherwise we could get trapped in an infinite
  125. * loop, as the tick_periodic() increments jiffies,
  126. * which then will increment time, possibly causing
  127. * the loop to trigger again and again.
  128. */
  129. if (timekeeping_valid_for_hres())
  130. tick_periodic(cpu);
  131. }
  132. }
  133. /*
  134. * Setup the device for a periodic tick
  135. */
  136. void tick_setup_periodic(struct clock_event_device *dev, int broadcast)
  137. {
  138. tick_set_periodic_handler(dev, broadcast);
  139. /* Broadcast setup ? */
  140. if (!tick_device_is_functional(dev))
  141. return;
  142. if ((dev->features & CLOCK_EVT_FEAT_PERIODIC) &&
  143. !tick_broadcast_oneshot_active()) {
  144. clockevents_switch_state(dev, CLOCK_EVT_STATE_PERIODIC);
  145. } else {
  146. unsigned int seq;
  147. ktime_t next;
  148. do {
  149. seq = read_seqcount_begin(&jiffies_seq);
  150. next = tick_next_period;
  151. } while (read_seqcount_retry(&jiffies_seq, seq));
  152. clockevents_switch_state(dev, CLOCK_EVT_STATE_ONESHOT);
  153. for (;;) {
  154. if (!clockevents_program_event(dev, next, false))
  155. return;
  156. next = ktime_add(next, tick_period);
  157. }
  158. }
  159. }
  160. #ifdef CONFIG_NO_HZ_FULL
  161. static void giveup_do_timer(void *info)
  162. {
  163. int cpu = *(unsigned int *)info;
  164. WARN_ON(tick_do_timer_cpu != smp_processor_id());
  165. tick_do_timer_cpu = cpu;
  166. }
  167. static void tick_take_do_timer_from_boot(void)
  168. {
  169. int cpu = smp_processor_id();
  170. int from = tick_do_timer_boot_cpu;
  171. if (from >= 0 && from != cpu)
  172. smp_call_function_single(from, giveup_do_timer, &cpu, 1);
  173. }
  174. #endif
  175. /*
  176. * Setup the tick device
  177. */
  178. static void tick_setup_device(struct tick_device *td,
  179. struct clock_event_device *newdev, int cpu,
  180. const struct cpumask *cpumask)
  181. {
  182. void (*handler)(struct clock_event_device *) = NULL;
  183. ktime_t next_event = 0;
  184. /*
  185. * First device setup ?
  186. */
  187. if (!td->evtdev) {
  188. /*
  189. * If no cpu took the do_timer update, assign it to
  190. * this cpu:
  191. */
  192. if (tick_do_timer_cpu == TICK_DO_TIMER_BOOT) {
  193. tick_do_timer_cpu = cpu;
  194. tick_next_period = ktime_get();
  195. tick_period = NSEC_PER_SEC / HZ;
  196. #ifdef CONFIG_NO_HZ_FULL
  197. /*
  198. * The boot CPU may be nohz_full, in which case set
  199. * tick_do_timer_boot_cpu so the first housekeeping
  200. * secondary that comes up will take do_timer from
  201. * us.
  202. */
  203. if (tick_nohz_full_cpu(cpu))
  204. tick_do_timer_boot_cpu = cpu;
  205. } else if (tick_do_timer_boot_cpu != -1 &&
  206. !tick_nohz_full_cpu(cpu)) {
  207. tick_take_do_timer_from_boot();
  208. tick_do_timer_boot_cpu = -1;
  209. WARN_ON(tick_do_timer_cpu != cpu);
  210. #endif
  211. }
  212. /*
  213. * Startup in periodic mode first.
  214. */
  215. td->mode = TICKDEV_MODE_PERIODIC;
  216. } else {
  217. handler = td->evtdev->event_handler;
  218. next_event = td->evtdev->next_event;
  219. td->evtdev->event_handler = clockevents_handle_noop;
  220. }
  221. td->evtdev = newdev;
  222. /*
  223. * When the device is not per cpu, pin the interrupt to the
  224. * current cpu:
  225. */
  226. if (!cpumask_equal(newdev->cpumask, cpumask))
  227. irq_set_affinity(newdev->irq, cpumask);
  228. /*
  229. * When global broadcasting is active, check if the current
  230. * device is registered as a placeholder for broadcast mode.
  231. * This allows us to handle this x86 misfeature in a generic
  232. * way. This function also returns !=0 when we keep the
  233. * current active broadcast state for this CPU.
  234. */
  235. if (tick_device_uses_broadcast(newdev, cpu))
  236. return;
  237. if (td->mode == TICKDEV_MODE_PERIODIC)
  238. tick_setup_periodic(newdev, 0);
  239. else
  240. tick_setup_oneshot(newdev, handler, next_event);
  241. }
  242. void tick_install_replacement(struct clock_event_device *newdev)
  243. {
  244. struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
  245. int cpu = smp_processor_id();
  246. clockevents_exchange_device(td->evtdev, newdev);
  247. tick_setup_device(td, newdev, cpu, cpumask_of(cpu));
  248. if (newdev->features & CLOCK_EVT_FEAT_ONESHOT)
  249. tick_oneshot_notify();
  250. }
  251. static bool tick_check_percpu(struct clock_event_device *curdev,
  252. struct clock_event_device *newdev, int cpu)
  253. {
  254. if (!cpumask_test_cpu(cpu, newdev->cpumask))
  255. return false;
  256. if (cpumask_equal(newdev->cpumask, cpumask_of(cpu)))
  257. return true;
  258. /* Check if irq affinity can be set */
  259. if (newdev->irq >= 0 && !irq_can_set_affinity(newdev->irq))
  260. return false;
  261. /* Prefer an existing cpu local device */
  262. if (curdev && cpumask_equal(curdev->cpumask, cpumask_of(cpu)))
  263. return false;
  264. return true;
  265. }
  266. static bool tick_check_preferred(struct clock_event_device *curdev,
  267. struct clock_event_device *newdev)
  268. {
  269. /* Prefer oneshot capable device */
  270. if (!(newdev->features & CLOCK_EVT_FEAT_ONESHOT)) {
  271. if (curdev && (curdev->features & CLOCK_EVT_FEAT_ONESHOT))
  272. return false;
  273. if (tick_oneshot_mode_active())
  274. return false;
  275. }
  276. /*
  277. * Use the higher rated one, but prefer a CPU local device with a lower
  278. * rating than a non-CPU local device
  279. */
  280. return !curdev ||
  281. newdev->rating > curdev->rating ||
  282. !cpumask_equal(curdev->cpumask, newdev->cpumask);
  283. }
  284. /*
  285. * Check whether the new device is a better fit than curdev. curdev
  286. * can be NULL !
  287. */
  288. bool tick_check_replacement(struct clock_event_device *curdev,
  289. struct clock_event_device *newdev)
  290. {
  291. if (!tick_check_percpu(curdev, newdev, smp_processor_id()))
  292. return false;
  293. return tick_check_preferred(curdev, newdev);
  294. }
  295. /*
  296. * Check, if the new registered device should be used. Called with
  297. * clockevents_lock held and interrupts disabled.
  298. */
  299. void tick_check_new_device(struct clock_event_device *newdev)
  300. {
  301. struct clock_event_device *curdev;
  302. struct tick_device *td;
  303. int cpu;
  304. cpu = smp_processor_id();
  305. td = &per_cpu(tick_cpu_device, cpu);
  306. curdev = td->evtdev;
  307. /* cpu local device ? */
  308. if (!tick_check_percpu(curdev, newdev, cpu))
  309. goto out_bc;
  310. /* Preference decision */
  311. if (!tick_check_preferred(curdev, newdev))
  312. goto out_bc;
  313. if (!try_module_get(newdev->owner))
  314. return;
  315. /*
  316. * Replace the eventually existing device by the new
  317. * device. If the current device is the broadcast device, do
  318. * not give it back to the clockevents layer !
  319. */
  320. if (tick_is_broadcast_device(curdev)) {
  321. clockevents_shutdown(curdev);
  322. curdev = NULL;
  323. }
  324. clockevents_exchange_device(curdev, newdev);
  325. tick_setup_device(td, newdev, cpu, cpumask_of(cpu));
  326. if (newdev->features & CLOCK_EVT_FEAT_ONESHOT)
  327. tick_oneshot_notify();
  328. return;
  329. out_bc:
  330. /*
  331. * Can the new device be used as a broadcast device ?
  332. */
  333. tick_install_broadcast_device(newdev, cpu);
  334. }
  335. /**
  336. * tick_broadcast_oneshot_control - Enter/exit broadcast oneshot mode
  337. * @state: The target state (enter/exit)
  338. *
  339. * The system enters/leaves a state, where affected devices might stop
  340. * Returns 0 on success, -EBUSY if the cpu is used to broadcast wakeups.
  341. *
  342. * Called with interrupts disabled, so clockevents_lock is not
  343. * required here because the local clock event device cannot go away
  344. * under us.
  345. */
  346. int tick_broadcast_oneshot_control(enum tick_broadcast_state state)
  347. {
  348. struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
  349. if (!(td->evtdev->features & CLOCK_EVT_FEAT_C3STOP))
  350. return 0;
  351. return __tick_broadcast_oneshot_control(state);
  352. }
  353. EXPORT_SYMBOL_GPL(tick_broadcast_oneshot_control);
  354. #ifdef CONFIG_HOTPLUG_CPU
  355. /*
  356. * Transfer the do_timer job away from a dying cpu.
  357. *
  358. * Called with interrupts disabled. Not locking required. If
  359. * tick_do_timer_cpu is owned by this cpu, nothing can change it.
  360. */
  361. void tick_handover_do_timer(void)
  362. {
  363. if (tick_do_timer_cpu == smp_processor_id()) {
  364. int cpu = cpumask_first(cpu_online_mask);
  365. tick_do_timer_cpu = (cpu < nr_cpu_ids) ? cpu :
  366. TICK_DO_TIMER_NONE;
  367. }
  368. }
  369. /*
  370. * Shutdown an event device on a given cpu:
  371. *
  372. * This is called on a life CPU, when a CPU is dead. So we cannot
  373. * access the hardware device itself.
  374. * We just set the mode and remove it from the lists.
  375. */
  376. void tick_shutdown(unsigned int cpu)
  377. {
  378. struct tick_device *td = &per_cpu(tick_cpu_device, cpu);
  379. struct clock_event_device *dev = td->evtdev;
  380. td->mode = TICKDEV_MODE_PERIODIC;
  381. if (dev) {
  382. /*
  383. * Prevent that the clock events layer tries to call
  384. * the set mode function!
  385. */
  386. clockevent_set_state(dev, CLOCK_EVT_STATE_DETACHED);
  387. clockevents_exchange_device(dev, NULL);
  388. dev->event_handler = clockevents_handle_noop;
  389. td->evtdev = NULL;
  390. }
  391. }
  392. #endif
  393. /**
  394. * tick_suspend_local - Suspend the local tick device
  395. *
  396. * Called from the local cpu for freeze with interrupts disabled.
  397. *
  398. * No locks required. Nothing can change the per cpu device.
  399. */
  400. void tick_suspend_local(void)
  401. {
  402. struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
  403. clockevents_shutdown(td->evtdev);
  404. }
  405. /**
  406. * tick_resume_local - Resume the local tick device
  407. *
  408. * Called from the local CPU for unfreeze or XEN resume magic.
  409. *
  410. * No locks required. Nothing can change the per cpu device.
  411. */
  412. void tick_resume_local(void)
  413. {
  414. struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
  415. bool broadcast = tick_resume_check_broadcast();
  416. clockevents_tick_resume(td->evtdev);
  417. if (!broadcast) {
  418. if (td->mode == TICKDEV_MODE_PERIODIC)
  419. tick_setup_periodic(td->evtdev, 0);
  420. else
  421. tick_resume_oneshot();
  422. }
  423. }
  424. /**
  425. * tick_suspend - Suspend the tick and the broadcast device
  426. *
  427. * Called from syscore_suspend() via timekeeping_suspend with only one
  428. * CPU online and interrupts disabled or from tick_unfreeze() under
  429. * tick_freeze_lock.
  430. *
  431. * No locks required. Nothing can change the per cpu device.
  432. */
  433. void tick_suspend(void)
  434. {
  435. tick_suspend_local();
  436. tick_suspend_broadcast();
  437. }
  438. /**
  439. * tick_resume - Resume the tick and the broadcast device
  440. *
  441. * Called from syscore_resume() via timekeeping_resume with only one
  442. * CPU online and interrupts disabled.
  443. *
  444. * No locks required. Nothing can change the per cpu device.
  445. */
  446. void tick_resume(void)
  447. {
  448. tick_resume_broadcast();
  449. tick_resume_local();
  450. }
  451. #ifdef CONFIG_SUSPEND
  452. static DEFINE_RAW_SPINLOCK(tick_freeze_lock);
  453. static unsigned int tick_freeze_depth;
  454. /**
  455. * tick_freeze - Suspend the local tick and (possibly) timekeeping.
  456. *
  457. * Check if this is the last online CPU executing the function and if so,
  458. * suspend timekeeping. Otherwise suspend the local tick.
  459. *
  460. * Call with interrupts disabled. Must be balanced with %tick_unfreeze().
  461. * Interrupts must not be enabled before the subsequent %tick_unfreeze().
  462. */
  463. void tick_freeze(void)
  464. {
  465. raw_spin_lock(&tick_freeze_lock);
  466. tick_freeze_depth++;
  467. if (tick_freeze_depth == num_online_cpus()) {
  468. trace_suspend_resume(TPS("timekeeping_freeze"),
  469. smp_processor_id(), true);
  470. system_state = SYSTEM_SUSPEND;
  471. sched_clock_suspend();
  472. timekeeping_suspend();
  473. } else {
  474. tick_suspend_local();
  475. }
  476. raw_spin_unlock(&tick_freeze_lock);
  477. }
  478. /**
  479. * tick_unfreeze - Resume the local tick and (possibly) timekeeping.
  480. *
  481. * Check if this is the first CPU executing the function and if so, resume
  482. * timekeeping. Otherwise resume the local tick.
  483. *
  484. * Call with interrupts disabled. Must be balanced with %tick_freeze().
  485. * Interrupts must not be enabled after the preceding %tick_freeze().
  486. */
  487. void tick_unfreeze(void)
  488. {
  489. raw_spin_lock(&tick_freeze_lock);
  490. if (tick_freeze_depth == num_online_cpus()) {
  491. timekeeping_resume();
  492. sched_clock_resume();
  493. system_state = SYSTEM_RUNNING;
  494. trace_suspend_resume(TPS("timekeeping_freeze"),
  495. smp_processor_id(), false);
  496. } else {
  497. touch_softlockup_watchdog();
  498. tick_resume_local();
  499. }
  500. tick_freeze_depth--;
  501. raw_spin_unlock(&tick_freeze_lock);
  502. }
  503. #endif /* CONFIG_SUSPEND */
  504. /**
  505. * tick_init - initialize the tick control
  506. */
  507. void __init tick_init(void)
  508. {
  509. tick_broadcast_init();
  510. tick_nohz_init();
  511. }