tick-sched.c 35 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
  4. * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
  5. * Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner
  6. *
  7. * No idle tick implementation for low and high resolution timers
  8. *
  9. * Started by: Thomas Gleixner and Ingo Molnar
  10. */
  11. #include <linux/cpu.h>
  12. #include <linux/err.h>
  13. #include <linux/hrtimer.h>
  14. #include <linux/interrupt.h>
  15. #include <linux/kernel_stat.h>
  16. #include <linux/percpu.h>
  17. #include <linux/nmi.h>
  18. #include <linux/profile.h>
  19. #include <linux/sched/signal.h>
  20. #include <linux/sched/clock.h>
  21. #include <linux/sched/stat.h>
  22. #include <linux/sched/nohz.h>
  23. #include <linux/module.h>
  24. #include <linux/irq_work.h>
  25. #include <linux/posix-timers.h>
  26. #include <linux/context_tracking.h>
  27. #include <linux/mm.h>
  28. #include <trace/hooks/sched.h>
  29. #include <asm/irq_regs.h>
  30. #include "tick-internal.h"
  31. #include <trace/events/timer.h>
  32. /*
  33. * Per-CPU nohz control structure
  34. */
  35. static DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched);
  36. struct tick_sched *tick_get_tick_sched(int cpu)
  37. {
  38. return &per_cpu(tick_cpu_sched, cpu);
  39. }
  40. #if defined(CONFIG_NO_HZ_COMMON) || defined(CONFIG_HIGH_RES_TIMERS)
  41. /*
  42. * The time, when the last jiffy update happened. Protected by jiffies_lock.
  43. */
  44. static ktime_t last_jiffies_update;
  45. /*
  46. * Must be called with interrupts disabled !
  47. */
  48. static void tick_do_update_jiffies64(ktime_t now)
  49. {
  50. unsigned long ticks = 0;
  51. ktime_t delta;
  52. /*
  53. * Do a quick check without holding jiffies_lock:
  54. * The READ_ONCE() pairs with two updates done later in this function.
  55. */
  56. delta = ktime_sub(now, READ_ONCE(last_jiffies_update));
  57. if (delta < tick_period)
  58. return;
  59. /* Reevaluate with jiffies_lock held */
  60. raw_spin_lock(&jiffies_lock);
  61. write_seqcount_begin(&jiffies_seq);
  62. delta = ktime_sub(now, last_jiffies_update);
  63. if (delta >= tick_period) {
  64. delta = ktime_sub(delta, tick_period);
  65. /* Pairs with the lockless read in this function. */
  66. WRITE_ONCE(last_jiffies_update,
  67. ktime_add(last_jiffies_update, tick_period));
  68. /* Slow path for long timeouts */
  69. if (unlikely(delta >= tick_period)) {
  70. s64 incr = ktime_to_ns(tick_period);
  71. ticks = ktime_divns(delta, incr);
  72. /* Pairs with the lockless read in this function. */
  73. WRITE_ONCE(last_jiffies_update,
  74. ktime_add_ns(last_jiffies_update,
  75. incr * ticks));
  76. }
  77. do_timer(++ticks);
  78. /* Keep the tick_next_period variable up to date */
  79. tick_next_period = ktime_add(last_jiffies_update, tick_period);
  80. } else {
  81. write_seqcount_end(&jiffies_seq);
  82. raw_spin_unlock(&jiffies_lock);
  83. return;
  84. }
  85. write_seqcount_end(&jiffies_seq);
  86. raw_spin_unlock(&jiffies_lock);
  87. update_wall_time();
  88. }
  89. /*
  90. * Initialize and return retrieve the jiffies update.
  91. */
  92. static ktime_t tick_init_jiffy_update(void)
  93. {
  94. ktime_t period;
  95. raw_spin_lock(&jiffies_lock);
  96. write_seqcount_begin(&jiffies_seq);
  97. /* Did we start the jiffies update yet ? */
  98. if (last_jiffies_update == 0)
  99. last_jiffies_update = tick_next_period;
  100. period = last_jiffies_update;
  101. write_seqcount_end(&jiffies_seq);
  102. raw_spin_unlock(&jiffies_lock);
  103. return period;
  104. }
  105. static void tick_sched_do_timer(struct tick_sched *ts, ktime_t now)
  106. {
  107. int cpu = smp_processor_id();
  108. #ifdef CONFIG_NO_HZ_COMMON
  109. /*
  110. * Check if the do_timer duty was dropped. We don't care about
  111. * concurrency: This happens only when the CPU in charge went
  112. * into a long sleep. If two CPUs happen to assign themselves to
  113. * this duty, then the jiffies update is still serialized by
  114. * jiffies_lock.
  115. *
  116. * If nohz_full is enabled, this should not happen because the
  117. * tick_do_timer_cpu never relinquishes.
  118. */
  119. if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE)) {
  120. #ifdef CONFIG_NO_HZ_FULL
  121. WARN_ON_ONCE(tick_nohz_full_running);
  122. #endif
  123. tick_do_timer_cpu = cpu;
  124. }
  125. #endif
  126. /* Check, if the jiffies need an update */
  127. if (tick_do_timer_cpu == cpu) {
  128. tick_do_update_jiffies64(now);
  129. trace_android_vh_jiffies_update(NULL);
  130. }
  131. if (ts->inidle)
  132. ts->got_idle_tick = 1;
  133. }
  134. static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs)
  135. {
  136. #ifdef CONFIG_NO_HZ_COMMON
  137. /*
  138. * When we are idle and the tick is stopped, we have to touch
  139. * the watchdog as we might not schedule for a really long
  140. * time. This happens on complete idle SMP systems while
  141. * waiting on the login prompt. We also increment the "start of
  142. * idle" jiffy stamp so the idle accounting adjustment we do
  143. * when we go busy again does not account too much ticks.
  144. */
  145. if (ts->tick_stopped) {
  146. touch_softlockup_watchdog_sched();
  147. if (is_idle_task(current))
  148. ts->idle_jiffies++;
  149. /*
  150. * In case the current tick fired too early past its expected
  151. * expiration, make sure we don't bypass the next clock reprogramming
  152. * to the same deadline.
  153. */
  154. ts->next_tick = 0;
  155. }
  156. #endif
  157. update_process_times(user_mode(regs));
  158. profile_tick(CPU_PROFILING);
  159. }
  160. #endif
  161. #ifdef CONFIG_NO_HZ_FULL
  162. cpumask_var_t tick_nohz_full_mask;
  163. bool tick_nohz_full_running;
  164. EXPORT_SYMBOL_GPL(tick_nohz_full_running);
  165. static atomic_t tick_dep_mask;
  166. static bool check_tick_dependency(atomic_t *dep)
  167. {
  168. int val = atomic_read(dep);
  169. if (val & TICK_DEP_MASK_POSIX_TIMER) {
  170. trace_tick_stop(0, TICK_DEP_MASK_POSIX_TIMER);
  171. return true;
  172. }
  173. if (val & TICK_DEP_MASK_PERF_EVENTS) {
  174. trace_tick_stop(0, TICK_DEP_MASK_PERF_EVENTS);
  175. return true;
  176. }
  177. if (val & TICK_DEP_MASK_SCHED) {
  178. trace_tick_stop(0, TICK_DEP_MASK_SCHED);
  179. return true;
  180. }
  181. if (val & TICK_DEP_MASK_CLOCK_UNSTABLE) {
  182. trace_tick_stop(0, TICK_DEP_MASK_CLOCK_UNSTABLE);
  183. return true;
  184. }
  185. if (val & TICK_DEP_MASK_RCU) {
  186. trace_tick_stop(0, TICK_DEP_MASK_RCU);
  187. return true;
  188. }
  189. return false;
  190. }
  191. static bool can_stop_full_tick(int cpu, struct tick_sched *ts)
  192. {
  193. lockdep_assert_irqs_disabled();
  194. if (unlikely(!cpu_online(cpu)))
  195. return false;
  196. if (check_tick_dependency(&tick_dep_mask))
  197. return false;
  198. if (check_tick_dependency(&ts->tick_dep_mask))
  199. return false;
  200. if (check_tick_dependency(&current->tick_dep_mask))
  201. return false;
  202. if (check_tick_dependency(&current->signal->tick_dep_mask))
  203. return false;
  204. return true;
  205. }
  206. static void nohz_full_kick_func(struct irq_work *work)
  207. {
  208. /* Empty, the tick restart happens on tick_nohz_irq_exit() */
  209. }
  210. static DEFINE_PER_CPU(struct irq_work, nohz_full_kick_work) = {
  211. .func = nohz_full_kick_func,
  212. .flags = ATOMIC_INIT(IRQ_WORK_HARD_IRQ),
  213. };
  214. /*
  215. * Kick this CPU if it's full dynticks in order to force it to
  216. * re-evaluate its dependency on the tick and restart it if necessary.
  217. * This kick, unlike tick_nohz_full_kick_cpu() and tick_nohz_full_kick_all(),
  218. * is NMI safe.
  219. */
  220. static void tick_nohz_full_kick(void)
  221. {
  222. if (!tick_nohz_full_cpu(smp_processor_id()))
  223. return;
  224. irq_work_queue(this_cpu_ptr(&nohz_full_kick_work));
  225. }
  226. /*
  227. * Kick the CPU if it's full dynticks in order to force it to
  228. * re-evaluate its dependency on the tick and restart it if necessary.
  229. */
  230. void tick_nohz_full_kick_cpu(int cpu)
  231. {
  232. if (!tick_nohz_full_cpu(cpu))
  233. return;
  234. irq_work_queue_on(&per_cpu(nohz_full_kick_work, cpu), cpu);
  235. }
  236. /*
  237. * Kick all full dynticks CPUs in order to force these to re-evaluate
  238. * their dependency on the tick and restart it if necessary.
  239. */
  240. static void tick_nohz_full_kick_all(void)
  241. {
  242. int cpu;
  243. if (!tick_nohz_full_running)
  244. return;
  245. preempt_disable();
  246. for_each_cpu_and(cpu, tick_nohz_full_mask, cpu_online_mask)
  247. tick_nohz_full_kick_cpu(cpu);
  248. preempt_enable();
  249. }
  250. static void tick_nohz_dep_set_all(atomic_t *dep,
  251. enum tick_dep_bits bit)
  252. {
  253. int prev;
  254. prev = atomic_fetch_or(BIT(bit), dep);
  255. if (!prev)
  256. tick_nohz_full_kick_all();
  257. }
  258. /*
  259. * Set a global tick dependency. Used by perf events that rely on freq and
  260. * by unstable clock.
  261. */
  262. void tick_nohz_dep_set(enum tick_dep_bits bit)
  263. {
  264. tick_nohz_dep_set_all(&tick_dep_mask, bit);
  265. }
  266. void tick_nohz_dep_clear(enum tick_dep_bits bit)
  267. {
  268. atomic_andnot(BIT(bit), &tick_dep_mask);
  269. }
  270. /*
  271. * Set per-CPU tick dependency. Used by scheduler and perf events in order to
  272. * manage events throttling.
  273. */
  274. void tick_nohz_dep_set_cpu(int cpu, enum tick_dep_bits bit)
  275. {
  276. int prev;
  277. struct tick_sched *ts;
  278. ts = per_cpu_ptr(&tick_cpu_sched, cpu);
  279. prev = atomic_fetch_or(BIT(bit), &ts->tick_dep_mask);
  280. if (!prev) {
  281. preempt_disable();
  282. /* Perf needs local kick that is NMI safe */
  283. if (cpu == smp_processor_id()) {
  284. tick_nohz_full_kick();
  285. } else {
  286. /* Remote irq work not NMI-safe */
  287. if (!WARN_ON_ONCE(in_nmi()))
  288. tick_nohz_full_kick_cpu(cpu);
  289. }
  290. preempt_enable();
  291. }
  292. }
  293. EXPORT_SYMBOL_GPL(tick_nohz_dep_set_cpu);
  294. void tick_nohz_dep_clear_cpu(int cpu, enum tick_dep_bits bit)
  295. {
  296. struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu);
  297. atomic_andnot(BIT(bit), &ts->tick_dep_mask);
  298. }
  299. EXPORT_SYMBOL_GPL(tick_nohz_dep_clear_cpu);
  300. /*
  301. * Set a per-task tick dependency. RCU need this. Also posix CPU timers
  302. * in order to elapse per task timers.
  303. */
  304. void tick_nohz_dep_set_task(struct task_struct *tsk, enum tick_dep_bits bit)
  305. {
  306. if (!atomic_fetch_or(BIT(bit), &tsk->tick_dep_mask)) {
  307. if (tsk == current) {
  308. preempt_disable();
  309. tick_nohz_full_kick();
  310. preempt_enable();
  311. } else {
  312. /*
  313. * Some future tick_nohz_full_kick_task()
  314. * should optimize this.
  315. */
  316. tick_nohz_full_kick_all();
  317. }
  318. }
  319. }
  320. EXPORT_SYMBOL_GPL(tick_nohz_dep_set_task);
  321. void tick_nohz_dep_clear_task(struct task_struct *tsk, enum tick_dep_bits bit)
  322. {
  323. atomic_andnot(BIT(bit), &tsk->tick_dep_mask);
  324. }
  325. EXPORT_SYMBOL_GPL(tick_nohz_dep_clear_task);
  326. /*
  327. * Set a per-taskgroup tick dependency. Posix CPU timers need this in order to elapse
  328. * per process timers.
  329. */
  330. void tick_nohz_dep_set_signal(struct signal_struct *sig, enum tick_dep_bits bit)
  331. {
  332. tick_nohz_dep_set_all(&sig->tick_dep_mask, bit);
  333. }
  334. void tick_nohz_dep_clear_signal(struct signal_struct *sig, enum tick_dep_bits bit)
  335. {
  336. atomic_andnot(BIT(bit), &sig->tick_dep_mask);
  337. }
  338. /*
  339. * Re-evaluate the need for the tick as we switch the current task.
  340. * It might need the tick due to per task/process properties:
  341. * perf events, posix CPU timers, ...
  342. */
  343. void __tick_nohz_task_switch(void)
  344. {
  345. unsigned long flags;
  346. struct tick_sched *ts;
  347. local_irq_save(flags);
  348. if (!tick_nohz_full_cpu(smp_processor_id()))
  349. goto out;
  350. ts = this_cpu_ptr(&tick_cpu_sched);
  351. if (ts->tick_stopped) {
  352. if (atomic_read(&current->tick_dep_mask) ||
  353. atomic_read(&current->signal->tick_dep_mask))
  354. tick_nohz_full_kick();
  355. }
  356. out:
  357. local_irq_restore(flags);
  358. }
  359. /* Get the boot-time nohz CPU list from the kernel parameters. */
  360. void __init tick_nohz_full_setup(cpumask_var_t cpumask)
  361. {
  362. alloc_bootmem_cpumask_var(&tick_nohz_full_mask);
  363. cpumask_copy(tick_nohz_full_mask, cpumask);
  364. tick_nohz_full_running = true;
  365. }
  366. EXPORT_SYMBOL_GPL(tick_nohz_full_setup);
  367. static int tick_nohz_cpu_down(unsigned int cpu)
  368. {
  369. /*
  370. * The tick_do_timer_cpu CPU handles housekeeping duty (unbound
  371. * timers, workqueues, timekeeping, ...) on behalf of full dynticks
  372. * CPUs. It must remain online when nohz full is enabled.
  373. */
  374. if (tick_nohz_full_running && tick_do_timer_cpu == cpu)
  375. return -EBUSY;
  376. return 0;
  377. }
  378. void __init tick_nohz_init(void)
  379. {
  380. int cpu, ret;
  381. if (!tick_nohz_full_running)
  382. return;
  383. /*
  384. * Full dynticks uses irq work to drive the tick rescheduling on safe
  385. * locking contexts. But then we need irq work to raise its own
  386. * interrupts to avoid circular dependency on the tick
  387. */
  388. if (!arch_irq_work_has_interrupt()) {
  389. pr_warn("NO_HZ: Can't run full dynticks because arch doesn't support irq work self-IPIs\n");
  390. cpumask_clear(tick_nohz_full_mask);
  391. tick_nohz_full_running = false;
  392. return;
  393. }
  394. if (IS_ENABLED(CONFIG_PM_SLEEP_SMP) &&
  395. !IS_ENABLED(CONFIG_PM_SLEEP_SMP_NONZERO_CPU)) {
  396. cpu = smp_processor_id();
  397. if (cpumask_test_cpu(cpu, tick_nohz_full_mask)) {
  398. pr_warn("NO_HZ: Clearing %d from nohz_full range "
  399. "for timekeeping\n", cpu);
  400. cpumask_clear_cpu(cpu, tick_nohz_full_mask);
  401. }
  402. }
  403. for_each_cpu(cpu, tick_nohz_full_mask)
  404. context_tracking_cpu_set(cpu);
  405. ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
  406. "kernel/nohz:predown", NULL,
  407. tick_nohz_cpu_down);
  408. WARN_ON(ret < 0);
  409. pr_info("NO_HZ: Full dynticks CPUs: %*pbl.\n",
  410. cpumask_pr_args(tick_nohz_full_mask));
  411. }
  412. #endif
  413. /*
  414. * NOHZ - aka dynamic tick functionality
  415. */
  416. #ifdef CONFIG_NO_HZ_COMMON
  417. /*
  418. * NO HZ enabled ?
  419. */
  420. bool tick_nohz_enabled __read_mostly = true;
  421. unsigned long tick_nohz_active __read_mostly;
  422. /*
  423. * Enable / Disable tickless mode
  424. */
  425. static int __init setup_tick_nohz(char *str)
  426. {
  427. return (kstrtobool(str, &tick_nohz_enabled) == 0);
  428. }
  429. __setup("nohz=", setup_tick_nohz);
  430. bool tick_nohz_tick_stopped(void)
  431. {
  432. struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
  433. return ts->tick_stopped;
  434. }
  435. bool tick_nohz_tick_stopped_cpu(int cpu)
  436. {
  437. struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu);
  438. return ts->tick_stopped;
  439. }
  440. /**
  441. * tick_nohz_update_jiffies - update jiffies when idle was interrupted
  442. *
  443. * Called from interrupt entry when the CPU was idle
  444. *
  445. * In case the sched_tick was stopped on this CPU, we have to check if jiffies
  446. * must be updated. Otherwise an interrupt handler could use a stale jiffy
  447. * value. We do this unconditionally on any CPU, as we don't know whether the
  448. * CPU, which has the update task assigned is in a long sleep.
  449. */
  450. static void tick_nohz_update_jiffies(ktime_t now)
  451. {
  452. unsigned long flags;
  453. __this_cpu_write(tick_cpu_sched.idle_waketime, now);
  454. local_irq_save(flags);
  455. tick_do_update_jiffies64(now);
  456. local_irq_restore(flags);
  457. touch_softlockup_watchdog_sched();
  458. }
  459. /*
  460. * Updates the per-CPU time idle statistics counters
  461. */
  462. static void
  463. update_ts_time_stats(int cpu, struct tick_sched *ts, ktime_t now, u64 *last_update_time)
  464. {
  465. ktime_t delta;
  466. if (ts->idle_active) {
  467. delta = ktime_sub(now, ts->idle_entrytime);
  468. if (nr_iowait_cpu(cpu) > 0)
  469. ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta);
  470. else
  471. ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);
  472. ts->idle_entrytime = now;
  473. }
  474. if (last_update_time)
  475. *last_update_time = ktime_to_us(now);
  476. }
  477. static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now)
  478. {
  479. update_ts_time_stats(smp_processor_id(), ts, now, NULL);
  480. ts->idle_active = 0;
  481. sched_clock_idle_wakeup_event();
  482. }
  483. static void tick_nohz_start_idle(struct tick_sched *ts)
  484. {
  485. ts->idle_entrytime = ktime_get();
  486. ts->idle_active = 1;
  487. sched_clock_idle_sleep_event();
  488. }
  489. /**
  490. * get_cpu_idle_time_us - get the total idle time of a CPU
  491. * @cpu: CPU number to query
  492. * @last_update_time: variable to store update time in. Do not update
  493. * counters if NULL.
  494. *
  495. * Return the cumulative idle time (since boot) for a given
  496. * CPU, in microseconds.
  497. *
  498. * This time is measured via accounting rather than sampling,
  499. * and is as accurate as ktime_get() is.
  500. *
  501. * This function returns -1 if NOHZ is not enabled.
  502. */
  503. u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
  504. {
  505. struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
  506. ktime_t now, idle;
  507. if (!tick_nohz_active)
  508. return -1;
  509. now = ktime_get();
  510. if (last_update_time) {
  511. update_ts_time_stats(cpu, ts, now, last_update_time);
  512. idle = ts->idle_sleeptime;
  513. } else {
  514. if (ts->idle_active && !nr_iowait_cpu(cpu)) {
  515. ktime_t delta = ktime_sub(now, ts->idle_entrytime);
  516. idle = ktime_add(ts->idle_sleeptime, delta);
  517. } else {
  518. idle = ts->idle_sleeptime;
  519. }
  520. }
  521. return ktime_to_us(idle);
  522. }
  523. EXPORT_SYMBOL_GPL(get_cpu_idle_time_us);
  524. /**
  525. * get_cpu_iowait_time_us - get the total iowait time of a CPU
  526. * @cpu: CPU number to query
  527. * @last_update_time: variable to store update time in. Do not update
  528. * counters if NULL.
  529. *
  530. * Return the cumulative iowait time (since boot) for a given
  531. * CPU, in microseconds.
  532. *
  533. * This time is measured via accounting rather than sampling,
  534. * and is as accurate as ktime_get() is.
  535. *
  536. * This function returns -1 if NOHZ is not enabled.
  537. */
  538. u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
  539. {
  540. struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
  541. ktime_t now, iowait;
  542. if (!tick_nohz_active)
  543. return -1;
  544. now = ktime_get();
  545. if (last_update_time) {
  546. update_ts_time_stats(cpu, ts, now, last_update_time);
  547. iowait = ts->iowait_sleeptime;
  548. } else {
  549. if (ts->idle_active && nr_iowait_cpu(cpu) > 0) {
  550. ktime_t delta = ktime_sub(now, ts->idle_entrytime);
  551. iowait = ktime_add(ts->iowait_sleeptime, delta);
  552. } else {
  553. iowait = ts->iowait_sleeptime;
  554. }
  555. }
  556. return ktime_to_us(iowait);
  557. }
  558. EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us);
  559. static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
  560. {
  561. hrtimer_cancel(&ts->sched_timer);
  562. hrtimer_set_expires(&ts->sched_timer, ts->last_tick);
  563. /* Forward the time to expire in the future */
  564. hrtimer_forward(&ts->sched_timer, now, tick_period);
  565. if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
  566. hrtimer_start_expires(&ts->sched_timer,
  567. HRTIMER_MODE_ABS_PINNED_HARD);
  568. } else {
  569. tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
  570. }
  571. /*
  572. * Reset to make sure next tick stop doesn't get fooled by past
  573. * cached clock deadline.
  574. */
  575. ts->next_tick = 0;
  576. }
  577. static inline bool local_timer_softirq_pending(void)
  578. {
  579. return local_softirq_pending() & BIT(TIMER_SOFTIRQ);
  580. }
  581. static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu)
  582. {
  583. u64 basemono, next_tick, next_tmr, next_rcu, delta, expires;
  584. unsigned long basejiff;
  585. unsigned int seq;
  586. /* Read jiffies and the time when jiffies were updated last */
  587. do {
  588. seq = read_seqcount_begin(&jiffies_seq);
  589. basemono = last_jiffies_update;
  590. basejiff = jiffies;
  591. } while (read_seqcount_retry(&jiffies_seq, seq));
  592. ts->last_jiffies = basejiff;
  593. ts->timer_expires_base = basemono;
  594. /*
  595. * Keep the periodic tick, when RCU, architecture or irq_work
  596. * requests it.
  597. * Aside of that check whether the local timer softirq is
  598. * pending. If so its a bad idea to call get_next_timer_interrupt()
  599. * because there is an already expired timer, so it will request
  600. * immeditate expiry, which rearms the hardware timer with a
  601. * minimal delta which brings us back to this place
  602. * immediately. Lather, rinse and repeat...
  603. */
  604. if (rcu_needs_cpu(basemono, &next_rcu) || arch_needs_cpu() ||
  605. irq_work_needs_cpu() || local_timer_softirq_pending()) {
  606. next_tick = basemono + TICK_NSEC;
  607. } else {
  608. /*
  609. * Get the next pending timer. If high resolution
  610. * timers are enabled this only takes the timer wheel
  611. * timers into account. If high resolution timers are
  612. * disabled this also looks at the next expiring
  613. * hrtimer.
  614. */
  615. next_tmr = get_next_timer_interrupt(basejiff, basemono);
  616. ts->next_timer = next_tmr;
  617. /* Take the next rcu event into account */
  618. next_tick = next_rcu < next_tmr ? next_rcu : next_tmr;
  619. }
  620. /*
  621. * If the tick is due in the next period, keep it ticking or
  622. * force prod the timer.
  623. */
  624. delta = next_tick - basemono;
  625. if (delta <= (u64)TICK_NSEC) {
  626. /*
  627. * Tell the timer code that the base is not idle, i.e. undo
  628. * the effect of get_next_timer_interrupt():
  629. */
  630. timer_clear_idle();
  631. /*
  632. * We've not stopped the tick yet, and there's a timer in the
  633. * next period, so no point in stopping it either, bail.
  634. */
  635. if (!ts->tick_stopped) {
  636. ts->timer_expires = 0;
  637. goto out;
  638. }
  639. }
  640. /*
  641. * If this CPU is the one which had the do_timer() duty last, we limit
  642. * the sleep time to the timekeeping max_deferment value.
  643. * Otherwise we can sleep as long as we want.
  644. */
  645. delta = timekeeping_max_deferment();
  646. if (cpu != tick_do_timer_cpu &&
  647. (tick_do_timer_cpu != TICK_DO_TIMER_NONE || !ts->do_timer_last))
  648. delta = KTIME_MAX;
  649. /* Calculate the next expiry time */
  650. if (delta < (KTIME_MAX - basemono))
  651. expires = basemono + delta;
  652. else
  653. expires = KTIME_MAX;
  654. ts->timer_expires = min_t(u64, expires, next_tick);
  655. out:
  656. return ts->timer_expires;
  657. }
  658. static void tick_nohz_stop_tick(struct tick_sched *ts, int cpu)
  659. {
  660. struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
  661. u64 basemono = ts->timer_expires_base;
  662. u64 expires = ts->timer_expires;
  663. ktime_t tick = expires;
  664. /* Make sure we won't be trying to stop it twice in a row. */
  665. ts->timer_expires_base = 0;
  666. /*
  667. * If this CPU is the one which updates jiffies, then give up
  668. * the assignment and let it be taken by the CPU which runs
  669. * the tick timer next, which might be this CPU as well. If we
  670. * don't drop this here the jiffies might be stale and
  671. * do_timer() never invoked. Keep track of the fact that it
  672. * was the one which had the do_timer() duty last.
  673. */
  674. if (cpu == tick_do_timer_cpu) {
  675. tick_do_timer_cpu = TICK_DO_TIMER_NONE;
  676. ts->do_timer_last = 1;
  677. } else if (tick_do_timer_cpu != TICK_DO_TIMER_NONE) {
  678. ts->do_timer_last = 0;
  679. }
  680. /* Skip reprogram of event if its not changed */
  681. if (ts->tick_stopped && (expires == ts->next_tick)) {
  682. /* Sanity check: make sure clockevent is actually programmed */
  683. if (tick == KTIME_MAX || ts->next_tick == hrtimer_get_expires(&ts->sched_timer))
  684. return;
  685. WARN_ON_ONCE(1);
  686. printk_once("basemono: %llu ts->next_tick: %llu dev->next_event: %llu timer->active: %d timer->expires: %llu\n",
  687. basemono, ts->next_tick, dev->next_event,
  688. hrtimer_active(&ts->sched_timer), hrtimer_get_expires(&ts->sched_timer));
  689. }
  690. /*
  691. * nohz_stop_sched_tick can be called several times before
  692. * the nohz_restart_sched_tick is called. This happens when
  693. * interrupts arrive which do not cause a reschedule. In the
  694. * first call we save the current tick time, so we can restart
  695. * the scheduler tick in nohz_restart_sched_tick.
  696. */
  697. if (!ts->tick_stopped) {
  698. calc_load_nohz_start();
  699. quiet_vmstat();
  700. ts->last_tick = hrtimer_get_expires(&ts->sched_timer);
  701. ts->tick_stopped = 1;
  702. trace_tick_stop(1, TICK_DEP_MASK_NONE);
  703. }
  704. ts->next_tick = tick;
  705. /*
  706. * If the expiration time == KTIME_MAX, then we simply stop
  707. * the tick timer.
  708. */
  709. if (unlikely(expires == KTIME_MAX)) {
  710. if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
  711. hrtimer_cancel(&ts->sched_timer);
  712. return;
  713. }
  714. if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
  715. hrtimer_start(&ts->sched_timer, tick,
  716. HRTIMER_MODE_ABS_PINNED_HARD);
  717. } else {
  718. hrtimer_set_expires(&ts->sched_timer, tick);
  719. tick_program_event(tick, 1);
  720. }
  721. }
  722. static void tick_nohz_retain_tick(struct tick_sched *ts)
  723. {
  724. ts->timer_expires_base = 0;
  725. }
  726. #ifdef CONFIG_NO_HZ_FULL
  727. static void tick_nohz_stop_sched_tick(struct tick_sched *ts, int cpu)
  728. {
  729. if (tick_nohz_next_event(ts, cpu))
  730. tick_nohz_stop_tick(ts, cpu);
  731. else
  732. tick_nohz_retain_tick(ts);
  733. }
  734. #endif /* CONFIG_NO_HZ_FULL */
  735. static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now)
  736. {
  737. /* Update jiffies first */
  738. tick_do_update_jiffies64(now);
  739. /*
  740. * Clear the timer idle flag, so we avoid IPIs on remote queueing and
  741. * the clock forward checks in the enqueue path:
  742. */
  743. timer_clear_idle();
  744. calc_load_nohz_stop();
  745. touch_softlockup_watchdog_sched();
  746. /*
  747. * Cancel the scheduled timer and restore the tick
  748. */
  749. ts->tick_stopped = 0;
  750. ts->idle_exittime = now;
  751. tick_nohz_restart(ts, now);
  752. }
  753. static void tick_nohz_full_update_tick(struct tick_sched *ts)
  754. {
  755. #ifdef CONFIG_NO_HZ_FULL
  756. int cpu = smp_processor_id();
  757. if (!tick_nohz_full_cpu(cpu))
  758. return;
  759. if (!ts->tick_stopped && ts->nohz_mode == NOHZ_MODE_INACTIVE)
  760. return;
  761. if (can_stop_full_tick(cpu, ts))
  762. tick_nohz_stop_sched_tick(ts, cpu);
  763. else if (ts->tick_stopped)
  764. tick_nohz_restart_sched_tick(ts, ktime_get());
  765. #endif
  766. }
  767. static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)
  768. {
  769. /*
  770. * If this CPU is offline and it is the one which updates
  771. * jiffies, then give up the assignment and let it be taken by
  772. * the CPU which runs the tick timer next. If we don't drop
  773. * this here the jiffies might be stale and do_timer() never
  774. * invoked.
  775. */
  776. if (unlikely(!cpu_online(cpu))) {
  777. if (cpu == tick_do_timer_cpu)
  778. tick_do_timer_cpu = TICK_DO_TIMER_NONE;
  779. /*
  780. * Make sure the CPU doesn't get fooled by obsolete tick
  781. * deadline if it comes back online later.
  782. */
  783. ts->next_tick = 0;
  784. return false;
  785. }
  786. if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE))
  787. return false;
  788. if (need_resched())
  789. return false;
  790. if (unlikely(local_softirq_pending())) {
  791. static int ratelimit;
  792. if (ratelimit < 10 &&
  793. (local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK)) {
  794. pr_warn("NOHZ tick-stop error: Non-RCU local softirq work is pending, handler #%02x!!!\n",
  795. (unsigned int) local_softirq_pending());
  796. ratelimit++;
  797. }
  798. return false;
  799. }
  800. if (tick_nohz_full_enabled()) {
  801. /*
  802. * Keep the tick alive to guarantee timekeeping progression
  803. * if there are full dynticks CPUs around
  804. */
  805. if (tick_do_timer_cpu == cpu)
  806. return false;
  807. /* Should not happen for nohz-full */
  808. if (WARN_ON_ONCE(tick_do_timer_cpu == TICK_DO_TIMER_NONE))
  809. return false;
  810. }
  811. return true;
  812. }
  813. static void __tick_nohz_idle_stop_tick(struct tick_sched *ts)
  814. {
  815. ktime_t expires;
  816. int cpu = smp_processor_id();
  817. /*
  818. * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the
  819. * tick timer expiration time is known already.
  820. */
  821. if (ts->timer_expires_base)
  822. expires = ts->timer_expires;
  823. else if (can_stop_idle_tick(cpu, ts))
  824. expires = tick_nohz_next_event(ts, cpu);
  825. else
  826. return;
  827. ts->idle_calls++;
  828. if (expires > 0LL) {
  829. int was_stopped = ts->tick_stopped;
  830. tick_nohz_stop_tick(ts, cpu);
  831. ts->idle_sleeps++;
  832. ts->idle_expires = expires;
  833. if (!was_stopped && ts->tick_stopped) {
  834. ts->idle_jiffies = ts->last_jiffies;
  835. nohz_balance_enter_idle(cpu);
  836. }
  837. } else {
  838. tick_nohz_retain_tick(ts);
  839. }
  840. }
  841. /**
  842. * tick_nohz_idle_stop_tick - stop the idle tick from the idle task
  843. *
  844. * When the next event is more than a tick into the future, stop the idle tick
  845. */
  846. void tick_nohz_idle_stop_tick(void)
  847. {
  848. __tick_nohz_idle_stop_tick(this_cpu_ptr(&tick_cpu_sched));
  849. }
  850. void tick_nohz_idle_retain_tick(void)
  851. {
  852. tick_nohz_retain_tick(this_cpu_ptr(&tick_cpu_sched));
  853. /*
  854. * Undo the effect of get_next_timer_interrupt() called from
  855. * tick_nohz_next_event().
  856. */
  857. timer_clear_idle();
  858. }
  859. /**
  860. * tick_nohz_idle_enter - prepare for entering idle on the current CPU
  861. *
  862. * Called when we start the idle loop.
  863. */
  864. void tick_nohz_idle_enter(void)
  865. {
  866. struct tick_sched *ts;
  867. lockdep_assert_irqs_enabled();
  868. local_irq_disable();
  869. ts = this_cpu_ptr(&tick_cpu_sched);
  870. WARN_ON_ONCE(ts->timer_expires_base);
  871. ts->inidle = 1;
  872. tick_nohz_start_idle(ts);
  873. local_irq_enable();
  874. }
  875. /**
  876. * tick_nohz_irq_exit - update next tick event from interrupt exit
  877. *
  878. * When an interrupt fires while we are idle and it doesn't cause
  879. * a reschedule, it may still add, modify or delete a timer, enqueue
  880. * an RCU callback, etc...
  881. * So we need to re-calculate and reprogram the next tick event.
  882. */
  883. void tick_nohz_irq_exit(void)
  884. {
  885. struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
  886. if (ts->inidle)
  887. tick_nohz_start_idle(ts);
  888. else
  889. tick_nohz_full_update_tick(ts);
  890. }
  891. /**
  892. * tick_nohz_idle_got_tick - Check whether or not the tick handler has run
  893. */
  894. bool tick_nohz_idle_got_tick(void)
  895. {
  896. struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
  897. if (ts->got_idle_tick) {
  898. ts->got_idle_tick = 0;
  899. return true;
  900. }
  901. return false;
  902. }
  903. /**
  904. * tick_nohz_get_next_hrtimer - return the next expiration time for the hrtimer
  905. * or the tick, whatever that expires first. Note that, if the tick has been
  906. * stopped, it returns the next hrtimer.
  907. *
  908. * Called from power state control code with interrupts disabled
  909. */
  910. ktime_t tick_nohz_get_next_hrtimer(void)
  911. {
  912. return __this_cpu_read(tick_cpu_device.evtdev)->next_event;
  913. }
  914. /**
  915. * tick_nohz_get_sleep_length - return the expected length of the current sleep
  916. * @delta_next: duration until the next event if the tick cannot be stopped
  917. *
  918. * Called from power state control code with interrupts disabled
  919. */
  920. ktime_t tick_nohz_get_sleep_length(ktime_t *delta_next)
  921. {
  922. struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
  923. struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
  924. int cpu = smp_processor_id();
  925. /*
  926. * The idle entry time is expected to be a sufficient approximation of
  927. * the current time at this point.
  928. */
  929. ktime_t now = ts->idle_entrytime;
  930. ktime_t next_event;
  931. WARN_ON_ONCE(!ts->inidle);
  932. *delta_next = ktime_sub(dev->next_event, now);
  933. if (!can_stop_idle_tick(cpu, ts))
  934. return *delta_next;
  935. next_event = tick_nohz_next_event(ts, cpu);
  936. if (!next_event)
  937. return *delta_next;
  938. /*
  939. * If the next highres timer to expire is earlier than next_event, the
  940. * idle governor needs to know that.
  941. */
  942. next_event = min_t(u64, next_event,
  943. hrtimer_next_event_without(&ts->sched_timer));
  944. return ktime_sub(next_event, now);
  945. }
  946. EXPORT_SYMBOL_GPL(tick_nohz_get_sleep_length);
  947. /**
  948. * tick_nohz_get_idle_calls_cpu - return the current idle calls counter value
  949. * for a particular CPU.
  950. *
  951. * Called from the schedutil frequency scaling governor in scheduler context.
  952. */
  953. unsigned long tick_nohz_get_idle_calls_cpu(int cpu)
  954. {
  955. struct tick_sched *ts = tick_get_tick_sched(cpu);
  956. return ts->idle_calls;
  957. }
  958. EXPORT_SYMBOL_GPL(tick_nohz_get_idle_calls_cpu);
  959. /**
  960. * tick_nohz_get_idle_calls - return the current idle calls counter value
  961. *
  962. * Called from the schedutil frequency scaling governor in scheduler context.
  963. */
  964. unsigned long tick_nohz_get_idle_calls(void)
  965. {
  966. struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
  967. return ts->idle_calls;
  968. }
  969. static void tick_nohz_account_idle_ticks(struct tick_sched *ts)
  970. {
  971. #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
  972. unsigned long ticks;
  973. if (vtime_accounting_enabled_this_cpu())
  974. return;
  975. /*
  976. * We stopped the tick in idle. Update process times would miss the
  977. * time we slept as update_process_times does only a 1 tick
  978. * accounting. Enforce that this is accounted to idle !
  979. */
  980. ticks = jiffies - ts->idle_jiffies;
  981. /*
  982. * We might be one off. Do not randomly account a huge number of ticks!
  983. */
  984. if (ticks && ticks < LONG_MAX)
  985. account_idle_ticks(ticks);
  986. #endif
  987. }
  988. static void __tick_nohz_idle_restart_tick(struct tick_sched *ts, ktime_t now)
  989. {
  990. tick_nohz_restart_sched_tick(ts, now);
  991. tick_nohz_account_idle_ticks(ts);
  992. }
  993. void tick_nohz_idle_restart_tick(void)
  994. {
  995. struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
  996. if (ts->tick_stopped)
  997. __tick_nohz_idle_restart_tick(ts, ktime_get());
  998. }
  999. /**
  1000. * tick_nohz_idle_exit - restart the idle tick from the idle task
  1001. *
  1002. * Restart the idle tick when the CPU is woken up from idle
  1003. * This also exit the RCU extended quiescent state. The CPU
  1004. * can use RCU again after this function is called.
  1005. */
  1006. void tick_nohz_idle_exit(void)
  1007. {
  1008. struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
  1009. bool idle_active, tick_stopped;
  1010. ktime_t now;
  1011. local_irq_disable();
  1012. WARN_ON_ONCE(!ts->inidle);
  1013. WARN_ON_ONCE(ts->timer_expires_base);
  1014. ts->inidle = 0;
  1015. idle_active = ts->idle_active;
  1016. tick_stopped = ts->tick_stopped;
  1017. if (idle_active || tick_stopped)
  1018. now = ktime_get();
  1019. if (idle_active)
  1020. tick_nohz_stop_idle(ts, now);
  1021. if (tick_stopped)
  1022. __tick_nohz_idle_restart_tick(ts, now);
  1023. local_irq_enable();
  1024. }
  1025. /*
  1026. * The nohz low res interrupt handler
  1027. */
  1028. static void tick_nohz_handler(struct clock_event_device *dev)
  1029. {
  1030. struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
  1031. struct pt_regs *regs = get_irq_regs();
  1032. ktime_t now = ktime_get();
  1033. dev->next_event = KTIME_MAX;
  1034. tick_sched_do_timer(ts, now);
  1035. tick_sched_handle(ts, regs);
  1036. /* No need to reprogram if we are running tickless */
  1037. if (unlikely(ts->tick_stopped))
  1038. return;
  1039. hrtimer_forward(&ts->sched_timer, now, tick_period);
  1040. tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
  1041. }
  1042. static inline void tick_nohz_activate(struct tick_sched *ts, int mode)
  1043. {
  1044. if (!tick_nohz_enabled)
  1045. return;
  1046. ts->nohz_mode = mode;
  1047. /* One update is enough */
  1048. if (!test_and_set_bit(0, &tick_nohz_active))
  1049. timers_update_nohz();
  1050. }
  1051. /**
  1052. * tick_nohz_switch_to_nohz - switch to nohz mode
  1053. */
  1054. static void tick_nohz_switch_to_nohz(void)
  1055. {
  1056. struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
  1057. ktime_t next;
  1058. if (!tick_nohz_enabled)
  1059. return;
  1060. if (tick_switch_to_oneshot(tick_nohz_handler))
  1061. return;
  1062. /*
  1063. * Recycle the hrtimer in ts, so we can share the
  1064. * hrtimer_forward with the highres code.
  1065. */
  1066. hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
  1067. /* Get the next period */
  1068. next = tick_init_jiffy_update();
  1069. hrtimer_set_expires(&ts->sched_timer, next);
  1070. hrtimer_forward_now(&ts->sched_timer, tick_period);
  1071. tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
  1072. tick_nohz_activate(ts, NOHZ_MODE_LOWRES);
  1073. }
  1074. static inline void tick_nohz_irq_enter(void)
  1075. {
  1076. struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
  1077. ktime_t now;
  1078. if (!ts->idle_active && !ts->tick_stopped)
  1079. return;
  1080. now = ktime_get();
  1081. if (ts->idle_active)
  1082. tick_nohz_stop_idle(ts, now);
  1083. if (ts->tick_stopped)
  1084. tick_nohz_update_jiffies(now);
  1085. }
  1086. #else
  1087. static inline void tick_nohz_switch_to_nohz(void) { }
  1088. static inline void tick_nohz_irq_enter(void) { }
  1089. static inline void tick_nohz_activate(struct tick_sched *ts, int mode) { }
  1090. #endif /* CONFIG_NO_HZ_COMMON */
  1091. /*
  1092. * Called from irq_enter to notify about the possible interruption of idle()
  1093. */
  1094. void tick_irq_enter(void)
  1095. {
  1096. tick_check_oneshot_broadcast_this_cpu();
  1097. tick_nohz_irq_enter();
  1098. }
  1099. /*
  1100. * High resolution timer specific code
  1101. */
  1102. #ifdef CONFIG_HIGH_RES_TIMERS
  1103. /*
  1104. * We rearm the timer until we get disabled by the idle code.
  1105. * Called with interrupts disabled.
  1106. */
  1107. static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)
  1108. {
  1109. struct tick_sched *ts =
  1110. container_of(timer, struct tick_sched, sched_timer);
  1111. struct pt_regs *regs = get_irq_regs();
  1112. ktime_t now = ktime_get();
  1113. tick_sched_do_timer(ts, now);
  1114. /*
  1115. * Do not call, when we are not in irq context and have
  1116. * no valid regs pointer
  1117. */
  1118. if (regs)
  1119. tick_sched_handle(ts, regs);
  1120. else
  1121. ts->next_tick = 0;
  1122. /* No need to reprogram if we are in idle or full dynticks mode */
  1123. if (unlikely(ts->tick_stopped))
  1124. return HRTIMER_NORESTART;
  1125. hrtimer_forward(timer, now, tick_period);
  1126. return HRTIMER_RESTART;
  1127. }
  1128. static int sched_skew_tick;
  1129. static int __init skew_tick(char *str)
  1130. {
  1131. get_option(&str, &sched_skew_tick);
  1132. return 0;
  1133. }
  1134. early_param("skew_tick", skew_tick);
  1135. /**
  1136. * tick_setup_sched_timer - setup the tick emulation timer
  1137. */
  1138. void tick_setup_sched_timer(void)
  1139. {
  1140. struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
  1141. ktime_t now = ktime_get();
  1142. /*
  1143. * Emulate tick processing via per-CPU hrtimers:
  1144. */
  1145. hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
  1146. ts->sched_timer.function = tick_sched_timer;
  1147. /* Get the next period (per-CPU) */
  1148. hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update());
  1149. /* Offset the tick to avert jiffies_lock contention. */
  1150. if (sched_skew_tick) {
  1151. u64 offset = ktime_to_ns(tick_period) >> 1;
  1152. do_div(offset, num_possible_cpus());
  1153. offset *= smp_processor_id();
  1154. hrtimer_add_expires_ns(&ts->sched_timer, offset);
  1155. }
  1156. hrtimer_forward(&ts->sched_timer, now, tick_period);
  1157. hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED_HARD);
  1158. tick_nohz_activate(ts, NOHZ_MODE_HIGHRES);
  1159. }
  1160. #endif /* HIGH_RES_TIMERS */
  1161. #if defined CONFIG_NO_HZ_COMMON || defined CONFIG_HIGH_RES_TIMERS
  1162. void tick_cancel_sched_timer(int cpu)
  1163. {
  1164. struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
  1165. # ifdef CONFIG_HIGH_RES_TIMERS
  1166. if (ts->sched_timer.base)
  1167. hrtimer_cancel(&ts->sched_timer);
  1168. # endif
  1169. memset(ts, 0, sizeof(*ts));
  1170. }
  1171. #endif
  1172. /**
  1173. * Async notification about clocksource changes
  1174. */
  1175. void tick_clock_notify(void)
  1176. {
  1177. int cpu;
  1178. for_each_possible_cpu(cpu)
  1179. set_bit(0, &per_cpu(tick_cpu_sched, cpu).check_clocks);
  1180. }
  1181. /*
  1182. * Async notification about clock event changes
  1183. */
  1184. void tick_oneshot_notify(void)
  1185. {
  1186. struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
  1187. set_bit(0, &ts->check_clocks);
  1188. }
  1189. /**
  1190. * Check, if a change happened, which makes oneshot possible.
  1191. *
  1192. * Called cyclic from the hrtimer softirq (driven by the timer
  1193. * softirq) allow_nohz signals, that we can switch into low-res nohz
  1194. * mode, because high resolution timers are disabled (either compile
  1195. * or runtime). Called with interrupts disabled.
  1196. */
  1197. int tick_check_oneshot_change(int allow_nohz)
  1198. {
  1199. struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
  1200. if (!test_and_clear_bit(0, &ts->check_clocks))
  1201. return 0;
  1202. if (ts->nohz_mode != NOHZ_MODE_INACTIVE)
  1203. return 0;
  1204. if (!timekeeping_valid_for_hres() || !tick_is_oneshot_available())
  1205. return 0;
  1206. if (!allow_nohz)
  1207. return 1;
  1208. tick_nohz_switch_to_nohz();
  1209. return 0;
  1210. }