hrtimer.c 35 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440
  1. /*
  2. * linux/kernel/hrtimer.c
  3. *
  4. * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
  5. * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
  6. * Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner
  7. *
  8. * High-resolution kernel timers
  9. *
  10. * In contrast to the low-resolution timeout API implemented in
  11. * kernel/timer.c, hrtimers provide finer resolution and accuracy
  12. * depending on system configuration and capabilities.
  13. *
  14. * These timers are currently used for:
  15. * - itimers
  16. * - POSIX timers
  17. * - nanosleep
  18. * - precise in-kernel timing
  19. *
  20. * Started by: Thomas Gleixner and Ingo Molnar
  21. *
  22. * Credits:
  23. * based on kernel/timer.c
  24. *
  25. * Help, testing, suggestions, bugfixes, improvements were
  26. * provided by:
  27. *
  28. * George Anzinger, Andrew Morton, Steven Rostedt, Roman Zippel
  29. * et. al.
  30. *
  31. * For licencing details see kernel-base/COPYING
  32. */
  33. #include <linux/cpu.h>
  34. #include <linux/irq.h>
  35. #include <linux/module.h>
  36. #include <linux/percpu.h>
  37. #include <linux/hrtimer.h>
  38. #include <linux/notifier.h>
  39. #include <linux/syscalls.h>
  40. #include <linux/kallsyms.h>
  41. #include <linux/interrupt.h>
  42. #include <linux/tick.h>
  43. #include <linux/seq_file.h>
  44. #include <linux/err.h>
  45. #include <asm/uaccess.h>
  46. /**
  47. * ktime_get - get the monotonic time in ktime_t format
  48. *
  49. * returns the time in ktime_t format
  50. */
  51. ktime_t ktime_get(void)
  52. {
  53. struct timespec now;
  54. ktime_get_ts(&now);
  55. return timespec_to_ktime(now);
  56. }
  57. /**
  58. * ktime_get_real - get the real (wall-) time in ktime_t format
  59. *
  60. * returns the time in ktime_t format
  61. */
  62. ktime_t ktime_get_real(void)
  63. {
  64. struct timespec now;
  65. getnstimeofday(&now);
  66. return timespec_to_ktime(now);
  67. }
  68. EXPORT_SYMBOL_GPL(ktime_get_real);
  69. /*
  70. * The timer bases:
  71. *
  72. * Note: If we want to add new timer bases, we have to skip the two
  73. * clock ids captured by the cpu-timers. We do this by holding empty
  74. * entries rather than doing math adjustment of the clock ids.
  75. * This ensures that we capture erroneous accesses to these clock ids
  76. * rather than moving them into the range of valid clock id's.
  77. */
  78. DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) =
  79. {
  80. .clock_base =
  81. {
  82. {
  83. .index = CLOCK_REALTIME,
  84. .get_time = &ktime_get_real,
  85. .resolution = KTIME_LOW_RES,
  86. },
  87. {
  88. .index = CLOCK_MONOTONIC,
  89. .get_time = &ktime_get,
  90. .resolution = KTIME_LOW_RES,
  91. },
  92. }
  93. };
  94. /**
  95. * ktime_get_ts - get the monotonic clock in timespec format
  96. * @ts: pointer to timespec variable
  97. *
  98. * The function calculates the monotonic clock from the realtime
  99. * clock and the wall_to_monotonic offset and stores the result
  100. * in normalized timespec format in the variable pointed to by @ts.
  101. */
  102. void ktime_get_ts(struct timespec *ts)
  103. {
  104. struct timespec tomono;
  105. unsigned long seq;
  106. do {
  107. seq = read_seqbegin(&xtime_lock);
  108. getnstimeofday(ts);
  109. tomono = wall_to_monotonic;
  110. } while (read_seqretry(&xtime_lock, seq));
  111. set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec,
  112. ts->tv_nsec + tomono.tv_nsec);
  113. }
  114. EXPORT_SYMBOL_GPL(ktime_get_ts);
  115. /*
  116. * Get the coarse grained time at the softirq based on xtime and
  117. * wall_to_monotonic.
  118. */
  119. static void hrtimer_get_softirq_time(struct hrtimer_cpu_base *base)
  120. {
  121. ktime_t xtim, tomono;
  122. struct timespec xts, tom;
  123. unsigned long seq;
  124. do {
  125. seq = read_seqbegin(&xtime_lock);
  126. #ifdef CONFIG_NO_HZ
  127. getnstimeofday(&xts);
  128. #else
  129. xts = xtime;
  130. #endif
  131. tom = wall_to_monotonic;
  132. } while (read_seqretry(&xtime_lock, seq));
  133. xtim = timespec_to_ktime(xts);
  134. tomono = timespec_to_ktime(tom);
  135. base->clock_base[CLOCK_REALTIME].softirq_time = xtim;
  136. base->clock_base[CLOCK_MONOTONIC].softirq_time =
  137. ktime_add(xtim, tomono);
  138. }
  139. /*
  140. * Helper function to check, whether the timer is running the callback
  141. * function
  142. */
  143. static inline int hrtimer_callback_running(struct hrtimer *timer)
  144. {
  145. return timer->state & HRTIMER_STATE_CALLBACK;
  146. }
  147. /*
  148. * Functions and macros which are different for UP/SMP systems are kept in a
  149. * single place
  150. */
  151. #ifdef CONFIG_SMP
  152. /*
  153. * We are using hashed locking: holding per_cpu(hrtimer_bases)[n].lock
  154. * means that all timers which are tied to this base via timer->base are
  155. * locked, and the base itself is locked too.
  156. *
  157. * So __run_timers/migrate_timers can safely modify all timers which could
  158. * be found on the lists/queues.
  159. *
  160. * When the timer's base is locked, and the timer removed from list, it is
  161. * possible to set timer->base = NULL and drop the lock: the timer remains
  162. * locked.
  163. */
  164. static
  165. struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer,
  166. unsigned long *flags)
  167. {
  168. struct hrtimer_clock_base *base;
  169. for (;;) {
  170. base = timer->base;
  171. if (likely(base != NULL)) {
  172. spin_lock_irqsave(&base->cpu_base->lock, *flags);
  173. if (likely(base == timer->base))
  174. return base;
  175. /* The timer has migrated to another CPU: */
  176. spin_unlock_irqrestore(&base->cpu_base->lock, *flags);
  177. }
  178. cpu_relax();
  179. }
  180. }
  181. /*
  182. * Switch the timer base to the current CPU when possible.
  183. */
  184. static inline struct hrtimer_clock_base *
  185. switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base)
  186. {
  187. struct hrtimer_clock_base *new_base;
  188. struct hrtimer_cpu_base *new_cpu_base;
  189. new_cpu_base = &__get_cpu_var(hrtimer_bases);
  190. new_base = &new_cpu_base->clock_base[base->index];
  191. if (base != new_base) {
  192. /*
  193. * We are trying to schedule the timer on the local CPU.
  194. * However we can't change timer's base while it is running,
  195. * so we keep it on the same CPU. No hassle vs. reprogramming
  196. * the event source in the high resolution case. The softirq
  197. * code will take care of this when the timer function has
  198. * completed. There is no conflict as we hold the lock until
  199. * the timer is enqueued.
  200. */
  201. if (unlikely(hrtimer_callback_running(timer)))
  202. return base;
  203. /* See the comment in lock_timer_base() */
  204. timer->base = NULL;
  205. spin_unlock(&base->cpu_base->lock);
  206. spin_lock(&new_base->cpu_base->lock);
  207. timer->base = new_base;
  208. }
  209. return new_base;
  210. }
  211. #else /* CONFIG_SMP */
  212. static inline struct hrtimer_clock_base *
  213. lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
  214. {
  215. struct hrtimer_clock_base *base = timer->base;
  216. spin_lock_irqsave(&base->cpu_base->lock, *flags);
  217. return base;
  218. }
  219. # define switch_hrtimer_base(t, b) (b)
  220. #endif /* !CONFIG_SMP */
  221. /*
  222. * Functions for the union type storage format of ktime_t which are
  223. * too large for inlining:
  224. */
  225. #if BITS_PER_LONG < 64
  226. # ifndef CONFIG_KTIME_SCALAR
  227. /**
  228. * ktime_add_ns - Add a scalar nanoseconds value to a ktime_t variable
  229. * @kt: addend
  230. * @nsec: the scalar nsec value to add
  231. *
  232. * Returns the sum of kt and nsec in ktime_t format
  233. */
  234. ktime_t ktime_add_ns(const ktime_t kt, u64 nsec)
  235. {
  236. ktime_t tmp;
  237. if (likely(nsec < NSEC_PER_SEC)) {
  238. tmp.tv64 = nsec;
  239. } else {
  240. unsigned long rem = do_div(nsec, NSEC_PER_SEC);
  241. tmp = ktime_set((long)nsec, rem);
  242. }
  243. return ktime_add(kt, tmp);
  244. }
  245. # endif /* !CONFIG_KTIME_SCALAR */
  246. /*
  247. * Divide a ktime value by a nanosecond value
  248. */
  249. unsigned long ktime_divns(const ktime_t kt, s64 div)
  250. {
  251. u64 dclc, inc, dns;
  252. int sft = 0;
  253. dclc = dns = ktime_to_ns(kt);
  254. inc = div;
  255. /* Make sure the divisor is less than 2^32: */
  256. while (div >> 32) {
  257. sft++;
  258. div >>= 1;
  259. }
  260. dclc >>= sft;
  261. do_div(dclc, (unsigned long) div);
  262. return (unsigned long) dclc;
  263. }
  264. #endif /* BITS_PER_LONG >= 64 */
  265. /* High resolution timer related functions */
  266. #ifdef CONFIG_HIGH_RES_TIMERS
  267. /*
  268. * High resolution timer enabled ?
  269. */
  270. static int hrtimer_hres_enabled __read_mostly = 1;
  271. /*
  272. * Enable / Disable high resolution mode
  273. */
  274. static int __init setup_hrtimer_hres(char *str)
  275. {
  276. if (!strcmp(str, "off"))
  277. hrtimer_hres_enabled = 0;
  278. else if (!strcmp(str, "on"))
  279. hrtimer_hres_enabled = 1;
  280. else
  281. return 0;
  282. return 1;
  283. }
  284. __setup("highres=", setup_hrtimer_hres);
  285. /*
  286. * hrtimer_high_res_enabled - query, if the highres mode is enabled
  287. */
  288. static inline int hrtimer_is_hres_enabled(void)
  289. {
  290. return hrtimer_hres_enabled;
  291. }
  292. /*
  293. * Is the high resolution mode active ?
  294. */
  295. static inline int hrtimer_hres_active(void)
  296. {
  297. return __get_cpu_var(hrtimer_bases).hres_active;
  298. }
  299. /*
  300. * Reprogram the event source with checking both queues for the
  301. * next event
  302. * Called with interrupts disabled and base->lock held
  303. */
  304. static void hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base)
  305. {
  306. int i;
  307. struct hrtimer_clock_base *base = cpu_base->clock_base;
  308. ktime_t expires;
  309. cpu_base->expires_next.tv64 = KTIME_MAX;
  310. for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++, base++) {
  311. struct hrtimer *timer;
  312. if (!base->first)
  313. continue;
  314. timer = rb_entry(base->first, struct hrtimer, node);
  315. expires = ktime_sub(timer->expires, base->offset);
  316. if (expires.tv64 < cpu_base->expires_next.tv64)
  317. cpu_base->expires_next = expires;
  318. }
  319. if (cpu_base->expires_next.tv64 != KTIME_MAX)
  320. tick_program_event(cpu_base->expires_next, 1);
  321. }
  322. /*
  323. * Shared reprogramming for clock_realtime and clock_monotonic
  324. *
  325. * When a timer is enqueued and expires earlier than the already enqueued
  326. * timers, we have to check, whether it expires earlier than the timer for
  327. * which the clock event device was armed.
  328. *
  329. * Called with interrupts disabled and base->cpu_base.lock held
  330. */
  331. static int hrtimer_reprogram(struct hrtimer *timer,
  332. struct hrtimer_clock_base *base)
  333. {
  334. ktime_t *expires_next = &__get_cpu_var(hrtimer_bases).expires_next;
  335. ktime_t expires = ktime_sub(timer->expires, base->offset);
  336. int res;
  337. /*
  338. * When the callback is running, we do not reprogram the clock event
  339. * device. The timer callback is either running on a different CPU or
  340. * the callback is executed in the hrtimer_interupt context. The
  341. * reprogramming is handled either by the softirq, which called the
  342. * callback or at the end of the hrtimer_interrupt.
  343. */
  344. if (hrtimer_callback_running(timer))
  345. return 0;
  346. if (expires.tv64 >= expires_next->tv64)
  347. return 0;
  348. /*
  349. * Clockevents returns -ETIME, when the event was in the past.
  350. */
  351. res = tick_program_event(expires, 0);
  352. if (!IS_ERR_VALUE(res))
  353. *expires_next = expires;
  354. return res;
  355. }
  356. /*
  357. * Retrigger next event is called after clock was set
  358. *
  359. * Called with interrupts disabled via on_each_cpu()
  360. */
  361. static void retrigger_next_event(void *arg)
  362. {
  363. struct hrtimer_cpu_base *base;
  364. struct timespec realtime_offset;
  365. unsigned long seq;
  366. if (!hrtimer_hres_active())
  367. return;
  368. do {
  369. seq = read_seqbegin(&xtime_lock);
  370. set_normalized_timespec(&realtime_offset,
  371. -wall_to_monotonic.tv_sec,
  372. -wall_to_monotonic.tv_nsec);
  373. } while (read_seqretry(&xtime_lock, seq));
  374. base = &__get_cpu_var(hrtimer_bases);
  375. /* Adjust CLOCK_REALTIME offset */
  376. spin_lock(&base->lock);
  377. base->clock_base[CLOCK_REALTIME].offset =
  378. timespec_to_ktime(realtime_offset);
  379. hrtimer_force_reprogram(base);
  380. spin_unlock(&base->lock);
  381. }
  382. /*
  383. * Clock realtime was set
  384. *
  385. * Change the offset of the realtime clock vs. the monotonic
  386. * clock.
  387. *
  388. * We might have to reprogram the high resolution timer interrupt. On
  389. * SMP we call the architecture specific code to retrigger _all_ high
  390. * resolution timer interrupts. On UP we just disable interrupts and
  391. * call the high resolution interrupt code.
  392. */
  393. void clock_was_set(void)
  394. {
  395. /* Retrigger the CPU local events everywhere */
  396. on_each_cpu(retrigger_next_event, NULL, 0, 1);
  397. }
  398. /*
  399. * During resume we might have to reprogram the high resolution timer
  400. * interrupt (on the local CPU):
  401. */
  402. void hres_timers_resume(void)
  403. {
  404. WARN_ON_ONCE(num_online_cpus() > 1);
  405. /* Retrigger the CPU local events: */
  406. retrigger_next_event(NULL);
  407. }
  408. /*
  409. * Check, whether the timer is on the callback pending list
  410. */
  411. static inline int hrtimer_cb_pending(const struct hrtimer *timer)
  412. {
  413. return timer->state & HRTIMER_STATE_PENDING;
  414. }
  415. /*
  416. * Remove a timer from the callback pending list
  417. */
  418. static inline void hrtimer_remove_cb_pending(struct hrtimer *timer)
  419. {
  420. list_del_init(&timer->cb_entry);
  421. }
  422. /*
  423. * Initialize the high resolution related parts of cpu_base
  424. */
  425. static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base)
  426. {
  427. base->expires_next.tv64 = KTIME_MAX;
  428. base->hres_active = 0;
  429. INIT_LIST_HEAD(&base->cb_pending);
  430. }
  431. /*
  432. * Initialize the high resolution related parts of a hrtimer
  433. */
  434. static inline void hrtimer_init_timer_hres(struct hrtimer *timer)
  435. {
  436. INIT_LIST_HEAD(&timer->cb_entry);
  437. }
  438. /*
  439. * When High resolution timers are active, try to reprogram. Note, that in case
  440. * the state has HRTIMER_STATE_CALLBACK set, no reprogramming and no expiry
  441. * check happens. The timer gets enqueued into the rbtree. The reprogramming
  442. * and expiry check is done in the hrtimer_interrupt or in the softirq.
  443. */
  444. static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer,
  445. struct hrtimer_clock_base *base)
  446. {
  447. if (base->cpu_base->hres_active && hrtimer_reprogram(timer, base)) {
  448. /* Timer is expired, act upon the callback mode */
  449. switch(timer->cb_mode) {
  450. case HRTIMER_CB_IRQSAFE_NO_RESTART:
  451. /*
  452. * We can call the callback from here. No restart
  453. * happens, so no danger of recursion
  454. */
  455. BUG_ON(timer->function(timer) != HRTIMER_NORESTART);
  456. return 1;
  457. case HRTIMER_CB_IRQSAFE_NO_SOFTIRQ:
  458. /*
  459. * This is solely for the sched tick emulation with
  460. * dynamic tick support to ensure that we do not
  461. * restart the tick right on the edge and end up with
  462. * the tick timer in the softirq ! The calling site
  463. * takes care of this.
  464. */
  465. return 1;
  466. case HRTIMER_CB_IRQSAFE:
  467. case HRTIMER_CB_SOFTIRQ:
  468. /*
  469. * Move everything else into the softirq pending list !
  470. */
  471. list_add_tail(&timer->cb_entry,
  472. &base->cpu_base->cb_pending);
  473. timer->state = HRTIMER_STATE_PENDING;
  474. raise_softirq(HRTIMER_SOFTIRQ);
  475. return 1;
  476. default:
  477. BUG();
  478. }
  479. }
  480. return 0;
  481. }
  482. /*
  483. * Switch to high resolution mode
  484. */
  485. static int hrtimer_switch_to_hres(void)
  486. {
  487. struct hrtimer_cpu_base *base = &__get_cpu_var(hrtimer_bases);
  488. unsigned long flags;
  489. if (base->hres_active)
  490. return 1;
  491. local_irq_save(flags);
  492. if (tick_init_highres()) {
  493. local_irq_restore(flags);
  494. return 0;
  495. }
  496. base->hres_active = 1;
  497. base->clock_base[CLOCK_REALTIME].resolution = KTIME_HIGH_RES;
  498. base->clock_base[CLOCK_MONOTONIC].resolution = KTIME_HIGH_RES;
  499. tick_setup_sched_timer();
  500. /* "Retrigger" the interrupt to get things going */
  501. retrigger_next_event(NULL);
  502. local_irq_restore(flags);
  503. printk(KERN_INFO "Switched to high resolution mode on CPU %d\n",
  504. smp_processor_id());
  505. return 1;
  506. }
  507. #else
  508. static inline int hrtimer_hres_active(void) { return 0; }
  509. static inline int hrtimer_is_hres_enabled(void) { return 0; }
  510. static inline int hrtimer_switch_to_hres(void) { return 0; }
  511. static inline void hrtimer_force_reprogram(struct hrtimer_cpu_base *base) { }
  512. static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer,
  513. struct hrtimer_clock_base *base)
  514. {
  515. return 0;
  516. }
  517. static inline int hrtimer_cb_pending(struct hrtimer *timer) { return 0; }
  518. static inline void hrtimer_remove_cb_pending(struct hrtimer *timer) { }
  519. static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) { }
  520. static inline void hrtimer_init_timer_hres(struct hrtimer *timer) { }
  521. #endif /* CONFIG_HIGH_RES_TIMERS */
  522. #ifdef CONFIG_TIMER_STATS
  523. void __timer_stats_hrtimer_set_start_info(struct hrtimer *timer, void *addr)
  524. {
  525. if (timer->start_site)
  526. return;
  527. timer->start_site = addr;
  528. memcpy(timer->start_comm, current->comm, TASK_COMM_LEN);
  529. timer->start_pid = current->pid;
  530. }
  531. #endif
  532. /*
  533. * Counterpart to lock_timer_base above:
  534. */
  535. static inline
  536. void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
  537. {
  538. spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);
  539. }
  540. /**
  541. * hrtimer_forward - forward the timer expiry
  542. * @timer: hrtimer to forward
  543. * @now: forward past this time
  544. * @interval: the interval to forward
  545. *
  546. * Forward the timer expiry so it will expire in the future.
  547. * Returns the number of overruns.
  548. */
  549. unsigned long
  550. hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval)
  551. {
  552. unsigned long orun = 1;
  553. ktime_t delta;
  554. delta = ktime_sub(now, timer->expires);
  555. if (delta.tv64 < 0)
  556. return 0;
  557. if (interval.tv64 < timer->base->resolution.tv64)
  558. interval.tv64 = timer->base->resolution.tv64;
  559. if (unlikely(delta.tv64 >= interval.tv64)) {
  560. s64 incr = ktime_to_ns(interval);
  561. orun = ktime_divns(delta, incr);
  562. timer->expires = ktime_add_ns(timer->expires, incr * orun);
  563. if (timer->expires.tv64 > now.tv64)
  564. return orun;
  565. /*
  566. * This (and the ktime_add() below) is the
  567. * correction for exact:
  568. */
  569. orun++;
  570. }
  571. timer->expires = ktime_add(timer->expires, interval);
  572. /*
  573. * Make sure, that the result did not wrap with a very large
  574. * interval.
  575. */
  576. if (timer->expires.tv64 < 0)
  577. timer->expires = ktime_set(KTIME_SEC_MAX, 0);
  578. return orun;
  579. }
  580. /*
  581. * enqueue_hrtimer - internal function to (re)start a timer
  582. *
  583. * The timer is inserted in expiry order. Insertion into the
  584. * red black tree is O(log(n)). Must hold the base lock.
  585. */
  586. static void enqueue_hrtimer(struct hrtimer *timer,
  587. struct hrtimer_clock_base *base, int reprogram)
  588. {
  589. struct rb_node **link = &base->active.rb_node;
  590. struct rb_node *parent = NULL;
  591. struct hrtimer *entry;
  592. /*
  593. * Find the right place in the rbtree:
  594. */
  595. while (*link) {
  596. parent = *link;
  597. entry = rb_entry(parent, struct hrtimer, node);
  598. /*
  599. * We dont care about collisions. Nodes with
  600. * the same expiry time stay together.
  601. */
  602. if (timer->expires.tv64 < entry->expires.tv64)
  603. link = &(*link)->rb_left;
  604. else
  605. link = &(*link)->rb_right;
  606. }
  607. /*
  608. * Insert the timer to the rbtree and check whether it
  609. * replaces the first pending timer
  610. */
  611. if (!base->first || timer->expires.tv64 <
  612. rb_entry(base->first, struct hrtimer, node)->expires.tv64) {
  613. /*
  614. * Reprogram the clock event device. When the timer is already
  615. * expired hrtimer_enqueue_reprogram has either called the
  616. * callback or added it to the pending list and raised the
  617. * softirq.
  618. *
  619. * This is a NOP for !HIGHRES
  620. */
  621. if (reprogram && hrtimer_enqueue_reprogram(timer, base))
  622. return;
  623. base->first = &timer->node;
  624. }
  625. rb_link_node(&timer->node, parent, link);
  626. rb_insert_color(&timer->node, &base->active);
  627. /*
  628. * HRTIMER_STATE_ENQUEUED is or'ed to the current state to preserve the
  629. * state of a possibly running callback.
  630. */
  631. timer->state |= HRTIMER_STATE_ENQUEUED;
  632. }
  633. /*
  634. * __remove_hrtimer - internal function to remove a timer
  635. *
  636. * Caller must hold the base lock.
  637. *
  638. * High resolution timer mode reprograms the clock event device when the
  639. * timer is the one which expires next. The caller can disable this by setting
  640. * reprogram to zero. This is useful, when the context does a reprogramming
  641. * anyway (e.g. timer interrupt)
  642. */
  643. static void __remove_hrtimer(struct hrtimer *timer,
  644. struct hrtimer_clock_base *base,
  645. unsigned long newstate, int reprogram)
  646. {
  647. /* High res. callback list. NOP for !HIGHRES */
  648. if (hrtimer_cb_pending(timer))
  649. hrtimer_remove_cb_pending(timer);
  650. else {
  651. /*
  652. * Remove the timer from the rbtree and replace the
  653. * first entry pointer if necessary.
  654. */
  655. if (base->first == &timer->node) {
  656. base->first = rb_next(&timer->node);
  657. /* Reprogram the clock event device. if enabled */
  658. if (reprogram && hrtimer_hres_active())
  659. hrtimer_force_reprogram(base->cpu_base);
  660. }
  661. rb_erase(&timer->node, &base->active);
  662. }
  663. timer->state = newstate;
  664. }
  665. /*
  666. * remove hrtimer, called with base lock held
  667. */
  668. static inline int
  669. remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base)
  670. {
  671. if (hrtimer_is_queued(timer)) {
  672. int reprogram;
  673. /*
  674. * Remove the timer and force reprogramming when high
  675. * resolution mode is active and the timer is on the current
  676. * CPU. If we remove a timer on another CPU, reprogramming is
  677. * skipped. The interrupt event on this CPU is fired and
  678. * reprogramming happens in the interrupt handler. This is a
  679. * rare case and less expensive than a smp call.
  680. */
  681. timer_stats_hrtimer_clear_start_info(timer);
  682. reprogram = base->cpu_base == &__get_cpu_var(hrtimer_bases);
  683. __remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE,
  684. reprogram);
  685. return 1;
  686. }
  687. return 0;
  688. }
  689. /**
  690. * hrtimer_start - (re)start an relative timer on the current CPU
  691. * @timer: the timer to be added
  692. * @tim: expiry time
  693. * @mode: expiry mode: absolute (HRTIMER_ABS) or relative (HRTIMER_REL)
  694. *
  695. * Returns:
  696. * 0 on success
  697. * 1 when the timer was active
  698. */
  699. int
  700. hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode)
  701. {
  702. struct hrtimer_clock_base *base, *new_base;
  703. unsigned long flags;
  704. int ret;
  705. base = lock_hrtimer_base(timer, &flags);
  706. /* Remove an active timer from the queue: */
  707. ret = remove_hrtimer(timer, base);
  708. /* Switch the timer base, if necessary: */
  709. new_base = switch_hrtimer_base(timer, base);
  710. if (mode == HRTIMER_MODE_REL) {
  711. tim = ktime_add(tim, new_base->get_time());
  712. /*
  713. * CONFIG_TIME_LOW_RES is a temporary way for architectures
  714. * to signal that they simply return xtime in
  715. * do_gettimeoffset(). In this case we want to round up by
  716. * resolution when starting a relative timer, to avoid short
  717. * timeouts. This will go away with the GTOD framework.
  718. */
  719. #ifdef CONFIG_TIME_LOW_RES
  720. tim = ktime_add(tim, base->resolution);
  721. #endif
  722. }
  723. timer->expires = tim;
  724. timer_stats_hrtimer_set_start_info(timer);
  725. /*
  726. * Only allow reprogramming if the new base is on this CPU.
  727. * (it might still be on another CPU if the timer was pending)
  728. */
  729. enqueue_hrtimer(timer, new_base,
  730. new_base->cpu_base == &__get_cpu_var(hrtimer_bases));
  731. unlock_hrtimer_base(timer, &flags);
  732. return ret;
  733. }
  734. EXPORT_SYMBOL_GPL(hrtimer_start);
  735. /**
  736. * hrtimer_try_to_cancel - try to deactivate a timer
  737. * @timer: hrtimer to stop
  738. *
  739. * Returns:
  740. * 0 when the timer was not active
  741. * 1 when the timer was active
  742. * -1 when the timer is currently excuting the callback function and
  743. * cannot be stopped
  744. */
  745. int hrtimer_try_to_cancel(struct hrtimer *timer)
  746. {
  747. struct hrtimer_clock_base *base;
  748. unsigned long flags;
  749. int ret = -1;
  750. base = lock_hrtimer_base(timer, &flags);
  751. if (!hrtimer_callback_running(timer))
  752. ret = remove_hrtimer(timer, base);
  753. unlock_hrtimer_base(timer, &flags);
  754. return ret;
  755. }
  756. EXPORT_SYMBOL_GPL(hrtimer_try_to_cancel);
  757. /**
  758. * hrtimer_cancel - cancel a timer and wait for the handler to finish.
  759. * @timer: the timer to be cancelled
  760. *
  761. * Returns:
  762. * 0 when the timer was not active
  763. * 1 when the timer was active
  764. */
  765. int hrtimer_cancel(struct hrtimer *timer)
  766. {
  767. for (;;) {
  768. int ret = hrtimer_try_to_cancel(timer);
  769. if (ret >= 0)
  770. return ret;
  771. cpu_relax();
  772. }
  773. }
  774. EXPORT_SYMBOL_GPL(hrtimer_cancel);
  775. /**
  776. * hrtimer_get_remaining - get remaining time for the timer
  777. * @timer: the timer to read
  778. */
  779. ktime_t hrtimer_get_remaining(const struct hrtimer *timer)
  780. {
  781. struct hrtimer_clock_base *base;
  782. unsigned long flags;
  783. ktime_t rem;
  784. base = lock_hrtimer_base(timer, &flags);
  785. rem = ktime_sub(timer->expires, base->get_time());
  786. unlock_hrtimer_base(timer, &flags);
  787. return rem;
  788. }
  789. EXPORT_SYMBOL_GPL(hrtimer_get_remaining);
  790. #if defined(CONFIG_NO_IDLE_HZ) || defined(CONFIG_NO_HZ)
  791. /**
  792. * hrtimer_get_next_event - get the time until next expiry event
  793. *
  794. * Returns the delta to the next expiry event or KTIME_MAX if no timer
  795. * is pending.
  796. */
  797. ktime_t hrtimer_get_next_event(void)
  798. {
  799. struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases);
  800. struct hrtimer_clock_base *base = cpu_base->clock_base;
  801. ktime_t delta, mindelta = { .tv64 = KTIME_MAX };
  802. unsigned long flags;
  803. int i;
  804. spin_lock_irqsave(&cpu_base->lock, flags);
  805. if (!hrtimer_hres_active()) {
  806. for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++, base++) {
  807. struct hrtimer *timer;
  808. if (!base->first)
  809. continue;
  810. timer = rb_entry(base->first, struct hrtimer, node);
  811. delta.tv64 = timer->expires.tv64;
  812. delta = ktime_sub(delta, base->get_time());
  813. if (delta.tv64 < mindelta.tv64)
  814. mindelta.tv64 = delta.tv64;
  815. }
  816. }
  817. spin_unlock_irqrestore(&cpu_base->lock, flags);
  818. if (mindelta.tv64 < 0)
  819. mindelta.tv64 = 0;
  820. return mindelta;
  821. }
  822. #endif
  823. /**
  824. * hrtimer_init - initialize a timer to the given clock
  825. * @timer: the timer to be initialized
  826. * @clock_id: the clock to be used
  827. * @mode: timer mode abs/rel
  828. */
  829. void hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
  830. enum hrtimer_mode mode)
  831. {
  832. struct hrtimer_cpu_base *cpu_base;
  833. memset(timer, 0, sizeof(struct hrtimer));
  834. cpu_base = &__raw_get_cpu_var(hrtimer_bases);
  835. if (clock_id == CLOCK_REALTIME && mode != HRTIMER_MODE_ABS)
  836. clock_id = CLOCK_MONOTONIC;
  837. timer->base = &cpu_base->clock_base[clock_id];
  838. hrtimer_init_timer_hres(timer);
  839. #ifdef CONFIG_TIMER_STATS
  840. timer->start_site = NULL;
  841. timer->start_pid = -1;
  842. memset(timer->start_comm, 0, TASK_COMM_LEN);
  843. #endif
  844. }
  845. EXPORT_SYMBOL_GPL(hrtimer_init);
  846. /**
  847. * hrtimer_get_res - get the timer resolution for a clock
  848. * @which_clock: which clock to query
  849. * @tp: pointer to timespec variable to store the resolution
  850. *
  851. * Store the resolution of the clock selected by @which_clock in the
  852. * variable pointed to by @tp.
  853. */
  854. int hrtimer_get_res(const clockid_t which_clock, struct timespec *tp)
  855. {
  856. struct hrtimer_cpu_base *cpu_base;
  857. cpu_base = &__raw_get_cpu_var(hrtimer_bases);
  858. *tp = ktime_to_timespec(cpu_base->clock_base[which_clock].resolution);
  859. return 0;
  860. }
  861. EXPORT_SYMBOL_GPL(hrtimer_get_res);
  862. #ifdef CONFIG_HIGH_RES_TIMERS
  863. /*
  864. * High resolution timer interrupt
  865. * Called with interrupts disabled
  866. */
  867. void hrtimer_interrupt(struct clock_event_device *dev)
  868. {
  869. struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases);
  870. struct hrtimer_clock_base *base;
  871. ktime_t expires_next, now;
  872. int i, raise = 0;
  873. BUG_ON(!cpu_base->hres_active);
  874. cpu_base->nr_events++;
  875. dev->next_event.tv64 = KTIME_MAX;
  876. retry:
  877. now = ktime_get();
  878. expires_next.tv64 = KTIME_MAX;
  879. base = cpu_base->clock_base;
  880. for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
  881. ktime_t basenow;
  882. struct rb_node *node;
  883. spin_lock(&cpu_base->lock);
  884. basenow = ktime_add(now, base->offset);
  885. while ((node = base->first)) {
  886. struct hrtimer *timer;
  887. timer = rb_entry(node, struct hrtimer, node);
  888. if (basenow.tv64 < timer->expires.tv64) {
  889. ktime_t expires;
  890. expires = ktime_sub(timer->expires,
  891. base->offset);
  892. if (expires.tv64 < expires_next.tv64)
  893. expires_next = expires;
  894. break;
  895. }
  896. /* Move softirq callbacks to the pending list */
  897. if (timer->cb_mode == HRTIMER_CB_SOFTIRQ) {
  898. __remove_hrtimer(timer, base,
  899. HRTIMER_STATE_PENDING, 0);
  900. list_add_tail(&timer->cb_entry,
  901. &base->cpu_base->cb_pending);
  902. raise = 1;
  903. continue;
  904. }
  905. __remove_hrtimer(timer, base,
  906. HRTIMER_STATE_CALLBACK, 0);
  907. timer_stats_account_hrtimer(timer);
  908. /*
  909. * Note: We clear the CALLBACK bit after
  910. * enqueue_hrtimer to avoid reprogramming of
  911. * the event hardware. This happens at the end
  912. * of this function anyway.
  913. */
  914. if (timer->function(timer) != HRTIMER_NORESTART) {
  915. BUG_ON(timer->state != HRTIMER_STATE_CALLBACK);
  916. enqueue_hrtimer(timer, base, 0);
  917. }
  918. timer->state &= ~HRTIMER_STATE_CALLBACK;
  919. }
  920. spin_unlock(&cpu_base->lock);
  921. base++;
  922. }
  923. cpu_base->expires_next = expires_next;
  924. /* Reprogramming necessary ? */
  925. if (expires_next.tv64 != KTIME_MAX) {
  926. if (tick_program_event(expires_next, 0))
  927. goto retry;
  928. }
  929. /* Raise softirq ? */
  930. if (raise)
  931. raise_softirq(HRTIMER_SOFTIRQ);
  932. }
  933. static void run_hrtimer_softirq(struct softirq_action *h)
  934. {
  935. struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases);
  936. spin_lock_irq(&cpu_base->lock);
  937. while (!list_empty(&cpu_base->cb_pending)) {
  938. enum hrtimer_restart (*fn)(struct hrtimer *);
  939. struct hrtimer *timer;
  940. int restart;
  941. timer = list_entry(cpu_base->cb_pending.next,
  942. struct hrtimer, cb_entry);
  943. timer_stats_account_hrtimer(timer);
  944. fn = timer->function;
  945. __remove_hrtimer(timer, timer->base, HRTIMER_STATE_CALLBACK, 0);
  946. spin_unlock_irq(&cpu_base->lock);
  947. restart = fn(timer);
  948. spin_lock_irq(&cpu_base->lock);
  949. timer->state &= ~HRTIMER_STATE_CALLBACK;
  950. if (restart == HRTIMER_RESTART) {
  951. BUG_ON(hrtimer_active(timer));
  952. /*
  953. * Enqueue the timer, allow reprogramming of the event
  954. * device
  955. */
  956. enqueue_hrtimer(timer, timer->base, 1);
  957. } else if (hrtimer_active(timer)) {
  958. /*
  959. * If the timer was rearmed on another CPU, reprogram
  960. * the event device.
  961. */
  962. if (timer->base->first == &timer->node)
  963. hrtimer_reprogram(timer, timer->base);
  964. }
  965. }
  966. spin_unlock_irq(&cpu_base->lock);
  967. }
  968. #endif /* CONFIG_HIGH_RES_TIMERS */
  969. /*
  970. * Expire the per base hrtimer-queue:
  971. */
  972. static inline void run_hrtimer_queue(struct hrtimer_cpu_base *cpu_base,
  973. int index)
  974. {
  975. struct rb_node *node;
  976. struct hrtimer_clock_base *base = &cpu_base->clock_base[index];
  977. if (!base->first)
  978. return;
  979. if (base->get_softirq_time)
  980. base->softirq_time = base->get_softirq_time();
  981. spin_lock_irq(&cpu_base->lock);
  982. while ((node = base->first)) {
  983. struct hrtimer *timer;
  984. enum hrtimer_restart (*fn)(struct hrtimer *);
  985. int restart;
  986. timer = rb_entry(node, struct hrtimer, node);
  987. if (base->softirq_time.tv64 <= timer->expires.tv64)
  988. break;
  989. #ifdef CONFIG_HIGH_RES_TIMERS
  990. WARN_ON_ONCE(timer->cb_mode == HRTIMER_CB_IRQSAFE_NO_SOFTIRQ);
  991. #endif
  992. timer_stats_account_hrtimer(timer);
  993. fn = timer->function;
  994. __remove_hrtimer(timer, base, HRTIMER_STATE_CALLBACK, 0);
  995. spin_unlock_irq(&cpu_base->lock);
  996. restart = fn(timer);
  997. spin_lock_irq(&cpu_base->lock);
  998. timer->state &= ~HRTIMER_STATE_CALLBACK;
  999. if (restart != HRTIMER_NORESTART) {
  1000. BUG_ON(hrtimer_active(timer));
  1001. enqueue_hrtimer(timer, base, 0);
  1002. }
  1003. }
  1004. spin_unlock_irq(&cpu_base->lock);
  1005. }
  1006. /*
  1007. * Called from timer softirq every jiffy, expire hrtimers:
  1008. *
  1009. * For HRT its the fall back code to run the softirq in the timer
  1010. * softirq context in case the hrtimer initialization failed or has
  1011. * not been done yet.
  1012. */
  1013. void hrtimer_run_queues(void)
  1014. {
  1015. struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases);
  1016. int i;
  1017. if (hrtimer_hres_active())
  1018. return;
  1019. /*
  1020. * This _is_ ugly: We have to check in the softirq context,
  1021. * whether we can switch to highres and / or nohz mode. The
  1022. * clocksource switch happens in the timer interrupt with
  1023. * xtime_lock held. Notification from there only sets the
  1024. * check bit in the tick_oneshot code, otherwise we might
  1025. * deadlock vs. xtime_lock.
  1026. */
  1027. if (tick_check_oneshot_change(!hrtimer_is_hres_enabled()))
  1028. if (hrtimer_switch_to_hres())
  1029. return;
  1030. hrtimer_get_softirq_time(cpu_base);
  1031. for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++)
  1032. run_hrtimer_queue(cpu_base, i);
  1033. }
  1034. /*
  1035. * Sleep related functions:
  1036. */
  1037. static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer)
  1038. {
  1039. struct hrtimer_sleeper *t =
  1040. container_of(timer, struct hrtimer_sleeper, timer);
  1041. struct task_struct *task = t->task;
  1042. t->task = NULL;
  1043. if (task)
  1044. wake_up_process(task);
  1045. return HRTIMER_NORESTART;
  1046. }
  1047. void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task)
  1048. {
  1049. sl->timer.function = hrtimer_wakeup;
  1050. sl->task = task;
  1051. #ifdef CONFIG_HIGH_RES_TIMERS
  1052. sl->timer.cb_mode = HRTIMER_CB_IRQSAFE_NO_RESTART;
  1053. #endif
  1054. }
  1055. static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode)
  1056. {
  1057. hrtimer_init_sleeper(t, current);
  1058. do {
  1059. set_current_state(TASK_INTERRUPTIBLE);
  1060. hrtimer_start(&t->timer, t->timer.expires, mode);
  1061. if (likely(t->task))
  1062. schedule();
  1063. hrtimer_cancel(&t->timer);
  1064. mode = HRTIMER_MODE_ABS;
  1065. } while (t->task && !signal_pending(current));
  1066. return t->task == NULL;
  1067. }
  1068. long __sched hrtimer_nanosleep_restart(struct restart_block *restart)
  1069. {
  1070. struct hrtimer_sleeper t;
  1071. struct timespec __user *rmtp;
  1072. struct timespec tu;
  1073. ktime_t time;
  1074. restart->fn = do_no_restart_syscall;
  1075. hrtimer_init(&t.timer, restart->arg0, HRTIMER_MODE_ABS);
  1076. t.timer.expires.tv64 = ((u64)restart->arg3 << 32) | (u64) restart->arg2;
  1077. if (do_nanosleep(&t, HRTIMER_MODE_ABS))
  1078. return 0;
  1079. rmtp = (struct timespec __user *) restart->arg1;
  1080. if (rmtp) {
  1081. time = ktime_sub(t.timer.expires, t.timer.base->get_time());
  1082. if (time.tv64 <= 0)
  1083. return 0;
  1084. tu = ktime_to_timespec(time);
  1085. if (copy_to_user(rmtp, &tu, sizeof(tu)))
  1086. return -EFAULT;
  1087. }
  1088. restart->fn = hrtimer_nanosleep_restart;
  1089. /* The other values in restart are already filled in */
  1090. return -ERESTART_RESTARTBLOCK;
  1091. }
  1092. long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp,
  1093. const enum hrtimer_mode mode, const clockid_t clockid)
  1094. {
  1095. struct restart_block *restart;
  1096. struct hrtimer_sleeper t;
  1097. struct timespec tu;
  1098. ktime_t rem;
  1099. hrtimer_init(&t.timer, clockid, mode);
  1100. t.timer.expires = timespec_to_ktime(*rqtp);
  1101. if (do_nanosleep(&t, mode))
  1102. return 0;
  1103. /* Absolute timers do not update the rmtp value and restart: */
  1104. if (mode == HRTIMER_MODE_ABS)
  1105. return -ERESTARTNOHAND;
  1106. if (rmtp) {
  1107. rem = ktime_sub(t.timer.expires, t.timer.base->get_time());
  1108. if (rem.tv64 <= 0)
  1109. return 0;
  1110. tu = ktime_to_timespec(rem);
  1111. if (copy_to_user(rmtp, &tu, sizeof(tu)))
  1112. return -EFAULT;
  1113. }
  1114. restart = &current_thread_info()->restart_block;
  1115. restart->fn = hrtimer_nanosleep_restart;
  1116. restart->arg0 = (unsigned long) t.timer.base->index;
  1117. restart->arg1 = (unsigned long) rmtp;
  1118. restart->arg2 = t.timer.expires.tv64 & 0xFFFFFFFF;
  1119. restart->arg3 = t.timer.expires.tv64 >> 32;
  1120. return -ERESTART_RESTARTBLOCK;
  1121. }
  1122. asmlinkage long
  1123. sys_nanosleep(struct timespec __user *rqtp, struct timespec __user *rmtp)
  1124. {
  1125. struct timespec tu;
  1126. if (copy_from_user(&tu, rqtp, sizeof(tu)))
  1127. return -EFAULT;
  1128. if (!timespec_valid(&tu))
  1129. return -EINVAL;
  1130. return hrtimer_nanosleep(&tu, rmtp, HRTIMER_MODE_REL, CLOCK_MONOTONIC);
  1131. }
  1132. /*
  1133. * Functions related to boot-time initialization:
  1134. */
  1135. static void __devinit init_hrtimers_cpu(int cpu)
  1136. {
  1137. struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu);
  1138. int i;
  1139. spin_lock_init(&cpu_base->lock);
  1140. lockdep_set_class(&cpu_base->lock, &cpu_base->lock_key);
  1141. for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++)
  1142. cpu_base->clock_base[i].cpu_base = cpu_base;
  1143. hrtimer_init_hres(cpu_base);
  1144. }
  1145. #ifdef CONFIG_HOTPLUG_CPU
  1146. static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base,
  1147. struct hrtimer_clock_base *new_base)
  1148. {
  1149. struct hrtimer *timer;
  1150. struct rb_node *node;
  1151. while ((node = rb_first(&old_base->active))) {
  1152. timer = rb_entry(node, struct hrtimer, node);
  1153. BUG_ON(hrtimer_callback_running(timer));
  1154. __remove_hrtimer(timer, old_base, HRTIMER_STATE_INACTIVE, 0);
  1155. timer->base = new_base;
  1156. /*
  1157. * Enqueue the timer. Allow reprogramming of the event device
  1158. */
  1159. enqueue_hrtimer(timer, new_base, 1);
  1160. }
  1161. }
  1162. static void migrate_hrtimers(int cpu)
  1163. {
  1164. struct hrtimer_cpu_base *old_base, *new_base;
  1165. int i;
  1166. BUG_ON(cpu_online(cpu));
  1167. old_base = &per_cpu(hrtimer_bases, cpu);
  1168. new_base = &get_cpu_var(hrtimer_bases);
  1169. tick_cancel_sched_timer(cpu);
  1170. local_irq_disable();
  1171. double_spin_lock(&new_base->lock, &old_base->lock,
  1172. smp_processor_id() < cpu);
  1173. for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
  1174. migrate_hrtimer_list(&old_base->clock_base[i],
  1175. &new_base->clock_base[i]);
  1176. }
  1177. double_spin_unlock(&new_base->lock, &old_base->lock,
  1178. smp_processor_id() < cpu);
  1179. local_irq_enable();
  1180. put_cpu_var(hrtimer_bases);
  1181. }
  1182. #endif /* CONFIG_HOTPLUG_CPU */
  1183. static int __cpuinit hrtimer_cpu_notify(struct notifier_block *self,
  1184. unsigned long action, void *hcpu)
  1185. {
  1186. long cpu = (long)hcpu;
  1187. switch (action) {
  1188. case CPU_UP_PREPARE:
  1189. init_hrtimers_cpu(cpu);
  1190. break;
  1191. #ifdef CONFIG_HOTPLUG_CPU
  1192. case CPU_DEAD:
  1193. clockevents_notify(CLOCK_EVT_NOTIFY_CPU_DEAD, &cpu);
  1194. migrate_hrtimers(cpu);
  1195. break;
  1196. #endif
  1197. default:
  1198. break;
  1199. }
  1200. return NOTIFY_OK;
  1201. }
  1202. static struct notifier_block __cpuinitdata hrtimers_nb = {
  1203. .notifier_call = hrtimer_cpu_notify,
  1204. };
  1205. void __init hrtimers_init(void)
  1206. {
  1207. hrtimer_cpu_notify(&hrtimers_nb, (unsigned long)CPU_UP_PREPARE,
  1208. (void *)(long)smp_processor_id());
  1209. register_cpu_notifier(&hrtimers_nb);
  1210. #ifdef CONFIG_HIGH_RES_TIMERS
  1211. open_softirq(HRTIMER_SOFTIRQ, run_hrtimer_softirq, NULL);
  1212. #endif
  1213. }