vtime.c 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577
  1. /*
  2. * arch/s390/kernel/vtime.c
  3. * Virtual cpu timer based timer functions.
  4. *
  5. * S390 version
  6. * Copyright (C) 2004 IBM Deutschland Entwicklung GmbH, IBM Corporation
  7. * Author(s): Jan Glauber <jan.glauber@de.ibm.com>
  8. */
  9. #include <linux/module.h>
  10. #include <linux/kernel.h>
  11. #include <linux/time.h>
  12. #include <linux/delay.h>
  13. #include <linux/init.h>
  14. #include <linux/smp.h>
  15. #include <linux/types.h>
  16. #include <linux/timex.h>
  17. #include <linux/notifier.h>
  18. #include <linux/kernel_stat.h>
  19. #include <linux/rcupdate.h>
  20. #include <linux/posix-timers.h>
  21. #include <asm/s390_ext.h>
  22. #include <asm/timer.h>
  23. #include <asm/irq_regs.h>
  24. static ext_int_info_t ext_int_info_timer;
  25. static DEFINE_PER_CPU(struct vtimer_queue, virt_cpu_timer);
  26. #ifdef CONFIG_VIRT_CPU_ACCOUNTING
  27. /*
  28. * Update process times based on virtual cpu times stored by entry.S
  29. * to the lowcore fields user_timer, system_timer & steal_clock.
  30. */
  31. void account_tick_vtime(struct task_struct *tsk)
  32. {
  33. cputime_t cputime;
  34. __u64 timer, clock;
  35. int rcu_user_flag;
  36. timer = S390_lowcore.last_update_timer;
  37. clock = S390_lowcore.last_update_clock;
  38. asm volatile (" STPT %0\n" /* Store current cpu timer value */
  39. " STCK %1" /* Store current tod clock value */
  40. : "=m" (S390_lowcore.last_update_timer),
  41. "=m" (S390_lowcore.last_update_clock) );
  42. S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer;
  43. S390_lowcore.steal_clock += S390_lowcore.last_update_clock - clock;
  44. cputime = S390_lowcore.user_timer >> 12;
  45. rcu_user_flag = cputime != 0;
  46. S390_lowcore.user_timer -= cputime << 12;
  47. S390_lowcore.steal_clock -= cputime << 12;
  48. account_user_time(tsk, cputime);
  49. cputime = S390_lowcore.system_timer >> 12;
  50. S390_lowcore.system_timer -= cputime << 12;
  51. S390_lowcore.steal_clock -= cputime << 12;
  52. account_system_time(tsk, HARDIRQ_OFFSET, cputime);
  53. cputime = S390_lowcore.steal_clock;
  54. if ((__s64) cputime > 0) {
  55. cputime >>= 12;
  56. S390_lowcore.steal_clock -= cputime << 12;
  57. account_steal_time(tsk, cputime);
  58. }
  59. run_local_timers();
  60. if (rcu_pending(smp_processor_id()))
  61. rcu_check_callbacks(smp_processor_id(), rcu_user_flag);
  62. scheduler_tick();
  63. run_posix_cpu_timers(tsk);
  64. }
  65. /*
  66. * Update process times based on virtual cpu times stored by entry.S
  67. * to the lowcore fields user_timer, system_timer & steal_clock.
  68. */
  69. void account_vtime(struct task_struct *tsk)
  70. {
  71. cputime_t cputime;
  72. __u64 timer;
  73. timer = S390_lowcore.last_update_timer;
  74. asm volatile (" STPT %0" /* Store current cpu timer value */
  75. : "=m" (S390_lowcore.last_update_timer) );
  76. S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer;
  77. cputime = S390_lowcore.user_timer >> 12;
  78. S390_lowcore.user_timer -= cputime << 12;
  79. S390_lowcore.steal_clock -= cputime << 12;
  80. account_user_time(tsk, cputime);
  81. cputime = S390_lowcore.system_timer >> 12;
  82. S390_lowcore.system_timer -= cputime << 12;
  83. S390_lowcore.steal_clock -= cputime << 12;
  84. account_system_time(tsk, 0, cputime);
  85. }
  86. /*
  87. * Update process times based on virtual cpu times stored by entry.S
  88. * to the lowcore fields user_timer, system_timer & steal_clock.
  89. */
  90. void account_system_vtime(struct task_struct *tsk)
  91. {
  92. cputime_t cputime;
  93. __u64 timer;
  94. timer = S390_lowcore.last_update_timer;
  95. asm volatile (" STPT %0" /* Store current cpu timer value */
  96. : "=m" (S390_lowcore.last_update_timer) );
  97. S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer;
  98. cputime = S390_lowcore.system_timer >> 12;
  99. S390_lowcore.system_timer -= cputime << 12;
  100. S390_lowcore.steal_clock -= cputime << 12;
  101. account_system_time(tsk, 0, cputime);
  102. }
  103. static inline void set_vtimer(__u64 expires)
  104. {
  105. __u64 timer;
  106. asm volatile (" STPT %0\n" /* Store current cpu timer value */
  107. " SPT %1" /* Set new value immediatly afterwards */
  108. : "=m" (timer) : "m" (expires) );
  109. S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer;
  110. S390_lowcore.last_update_timer = expires;
  111. /* store expire time for this CPU timer */
  112. per_cpu(virt_cpu_timer, smp_processor_id()).to_expire = expires;
  113. }
  114. #else
  115. static inline void set_vtimer(__u64 expires)
  116. {
  117. S390_lowcore.last_update_timer = expires;
  118. asm volatile ("SPT %0" : : "m" (S390_lowcore.last_update_timer));
  119. /* store expire time for this CPU timer */
  120. per_cpu(virt_cpu_timer, smp_processor_id()).to_expire = expires;
  121. }
  122. #endif
  123. static void start_cpu_timer(void)
  124. {
  125. struct vtimer_queue *vt_list;
  126. vt_list = &per_cpu(virt_cpu_timer, smp_processor_id());
  127. /* CPU timer interrupt is pending, don't reprogramm it */
  128. if (vt_list->idle & 1LL<<63)
  129. return;
  130. if (!list_empty(&vt_list->list))
  131. set_vtimer(vt_list->idle);
  132. }
  133. static void stop_cpu_timer(void)
  134. {
  135. struct vtimer_queue *vt_list;
  136. vt_list = &per_cpu(virt_cpu_timer, smp_processor_id());
  137. /* nothing to do */
  138. if (list_empty(&vt_list->list)) {
  139. vt_list->idle = VTIMER_MAX_SLICE;
  140. goto fire;
  141. }
  142. /* store the actual expire value */
  143. asm volatile ("STPT %0" : "=m" (vt_list->idle));
  144. /*
  145. * If the CPU timer is negative we don't reprogramm
  146. * it because we will get instantly an interrupt.
  147. */
  148. if (vt_list->idle & 1LL<<63)
  149. return;
  150. vt_list->offset += vt_list->to_expire - vt_list->idle;
  151. /*
  152. * We cannot halt the CPU timer, we just write a value that
  153. * nearly never expires (only after 71 years) and re-write
  154. * the stored expire value if we continue the timer
  155. */
  156. fire:
  157. set_vtimer(VTIMER_MAX_SLICE);
  158. }
  159. /*
  160. * Sorted add to a list. List is linear searched until first bigger
  161. * element is found.
  162. */
  163. static void list_add_sorted(struct vtimer_list *timer, struct list_head *head)
  164. {
  165. struct vtimer_list *event;
  166. list_for_each_entry(event, head, entry) {
  167. if (event->expires > timer->expires) {
  168. list_add_tail(&timer->entry, &event->entry);
  169. return;
  170. }
  171. }
  172. list_add_tail(&timer->entry, head);
  173. }
  174. /*
  175. * Do the callback functions of expired vtimer events.
  176. * Called from within the interrupt handler.
  177. */
  178. static void do_callbacks(struct list_head *cb_list)
  179. {
  180. struct vtimer_queue *vt_list;
  181. struct vtimer_list *event, *tmp;
  182. void (*fn)(unsigned long);
  183. unsigned long data;
  184. if (list_empty(cb_list))
  185. return;
  186. vt_list = &per_cpu(virt_cpu_timer, smp_processor_id());
  187. list_for_each_entry_safe(event, tmp, cb_list, entry) {
  188. fn = event->function;
  189. data = event->data;
  190. fn(data);
  191. if (!event->interval)
  192. /* delete one shot timer */
  193. list_del_init(&event->entry);
  194. else {
  195. /* move interval timer back to list */
  196. spin_lock(&vt_list->lock);
  197. list_del_init(&event->entry);
  198. list_add_sorted(event, &vt_list->list);
  199. spin_unlock(&vt_list->lock);
  200. }
  201. }
  202. }
  203. /*
  204. * Handler for the virtual CPU timer.
  205. */
  206. static void do_cpu_timer_interrupt(__u16 error_code)
  207. {
  208. int cpu;
  209. __u64 next, delta;
  210. struct vtimer_queue *vt_list;
  211. struct vtimer_list *event, *tmp;
  212. struct list_head *ptr;
  213. /* the callback queue */
  214. struct list_head cb_list;
  215. INIT_LIST_HEAD(&cb_list);
  216. cpu = smp_processor_id();
  217. vt_list = &per_cpu(virt_cpu_timer, cpu);
  218. /* walk timer list, fire all expired events */
  219. spin_lock(&vt_list->lock);
  220. if (vt_list->to_expire < VTIMER_MAX_SLICE)
  221. vt_list->offset += vt_list->to_expire;
  222. list_for_each_entry_safe(event, tmp, &vt_list->list, entry) {
  223. if (event->expires > vt_list->offset)
  224. /* found first unexpired event, leave */
  225. break;
  226. /* re-charge interval timer, we have to add the offset */
  227. if (event->interval)
  228. event->expires = event->interval + vt_list->offset;
  229. /* move expired timer to the callback queue */
  230. list_move_tail(&event->entry, &cb_list);
  231. }
  232. spin_unlock(&vt_list->lock);
  233. do_callbacks(&cb_list);
  234. /* next event is first in list */
  235. spin_lock(&vt_list->lock);
  236. if (!list_empty(&vt_list->list)) {
  237. ptr = vt_list->list.next;
  238. event = list_entry(ptr, struct vtimer_list, entry);
  239. next = event->expires - vt_list->offset;
  240. /* add the expired time from this interrupt handler
  241. * and the callback functions
  242. */
  243. asm volatile ("STPT %0" : "=m" (delta));
  244. delta = 0xffffffffffffffffLL - delta + 1;
  245. vt_list->offset += delta;
  246. next -= delta;
  247. } else {
  248. vt_list->offset = 0;
  249. next = VTIMER_MAX_SLICE;
  250. }
  251. spin_unlock(&vt_list->lock);
  252. set_vtimer(next);
  253. }
  254. void init_virt_timer(struct vtimer_list *timer)
  255. {
  256. timer->function = NULL;
  257. INIT_LIST_HEAD(&timer->entry);
  258. spin_lock_init(&timer->lock);
  259. }
  260. EXPORT_SYMBOL(init_virt_timer);
  261. static inline int vtimer_pending(struct vtimer_list *timer)
  262. {
  263. return (!list_empty(&timer->entry));
  264. }
  265. /*
  266. * this function should only run on the specified CPU
  267. */
  268. static void internal_add_vtimer(struct vtimer_list *timer)
  269. {
  270. unsigned long flags;
  271. __u64 done;
  272. struct vtimer_list *event;
  273. struct vtimer_queue *vt_list;
  274. vt_list = &per_cpu(virt_cpu_timer, timer->cpu);
  275. spin_lock_irqsave(&vt_list->lock, flags);
  276. if (timer->cpu != smp_processor_id())
  277. printk("internal_add_vtimer: BUG, running on wrong CPU");
  278. /* if list is empty we only have to set the timer */
  279. if (list_empty(&vt_list->list)) {
  280. /* reset the offset, this may happen if the last timer was
  281. * just deleted by mod_virt_timer and the interrupt
  282. * didn't happen until here
  283. */
  284. vt_list->offset = 0;
  285. goto fire;
  286. }
  287. /* save progress */
  288. asm volatile ("STPT %0" : "=m" (done));
  289. /* calculate completed work */
  290. done = vt_list->to_expire - done + vt_list->offset;
  291. vt_list->offset = 0;
  292. list_for_each_entry(event, &vt_list->list, entry)
  293. event->expires -= done;
  294. fire:
  295. list_add_sorted(timer, &vt_list->list);
  296. /* get first element, which is the next vtimer slice */
  297. event = list_entry(vt_list->list.next, struct vtimer_list, entry);
  298. set_vtimer(event->expires);
  299. spin_unlock_irqrestore(&vt_list->lock, flags);
  300. /* release CPU acquired in prepare_vtimer or mod_virt_timer() */
  301. put_cpu();
  302. }
  303. static inline int prepare_vtimer(struct vtimer_list *timer)
  304. {
  305. if (!timer->function) {
  306. printk("add_virt_timer: uninitialized timer\n");
  307. return -EINVAL;
  308. }
  309. if (!timer->expires || timer->expires > VTIMER_MAX_SLICE) {
  310. printk("add_virt_timer: invalid timer expire value!\n");
  311. return -EINVAL;
  312. }
  313. if (vtimer_pending(timer)) {
  314. printk("add_virt_timer: timer pending\n");
  315. return -EBUSY;
  316. }
  317. timer->cpu = get_cpu();
  318. return 0;
  319. }
  320. /*
  321. * add_virt_timer - add an oneshot virtual CPU timer
  322. */
  323. void add_virt_timer(void *new)
  324. {
  325. struct vtimer_list *timer;
  326. timer = (struct vtimer_list *)new;
  327. if (prepare_vtimer(timer) < 0)
  328. return;
  329. timer->interval = 0;
  330. internal_add_vtimer(timer);
  331. }
  332. EXPORT_SYMBOL(add_virt_timer);
  333. /*
  334. * add_virt_timer_int - add an interval virtual CPU timer
  335. */
  336. void add_virt_timer_periodic(void *new)
  337. {
  338. struct vtimer_list *timer;
  339. timer = (struct vtimer_list *)new;
  340. if (prepare_vtimer(timer) < 0)
  341. return;
  342. timer->interval = timer->expires;
  343. internal_add_vtimer(timer);
  344. }
  345. EXPORT_SYMBOL(add_virt_timer_periodic);
  346. /*
  347. * If we change a pending timer the function must be called on the CPU
  348. * where the timer is running on, e.g. by smp_call_function_on()
  349. *
  350. * The original mod_timer adds the timer if it is not pending. For compatibility
  351. * we do the same. The timer will be added on the current CPU as a oneshot timer.
  352. *
  353. * returns whether it has modified a pending timer (1) or not (0)
  354. */
  355. int mod_virt_timer(struct vtimer_list *timer, __u64 expires)
  356. {
  357. struct vtimer_queue *vt_list;
  358. unsigned long flags;
  359. int cpu;
  360. if (!timer->function) {
  361. printk("mod_virt_timer: uninitialized timer\n");
  362. return -EINVAL;
  363. }
  364. if (!expires || expires > VTIMER_MAX_SLICE) {
  365. printk("mod_virt_timer: invalid expire range\n");
  366. return -EINVAL;
  367. }
  368. /*
  369. * This is a common optimization triggered by the
  370. * networking code - if the timer is re-modified
  371. * to be the same thing then just return:
  372. */
  373. if (timer->expires == expires && vtimer_pending(timer))
  374. return 1;
  375. cpu = get_cpu();
  376. vt_list = &per_cpu(virt_cpu_timer, cpu);
  377. /* disable interrupts before test if timer is pending */
  378. spin_lock_irqsave(&vt_list->lock, flags);
  379. /* if timer isn't pending add it on the current CPU */
  380. if (!vtimer_pending(timer)) {
  381. spin_unlock_irqrestore(&vt_list->lock, flags);
  382. /* we do not activate an interval timer with mod_virt_timer */
  383. timer->interval = 0;
  384. timer->expires = expires;
  385. timer->cpu = cpu;
  386. internal_add_vtimer(timer);
  387. return 0;
  388. }
  389. /* check if we run on the right CPU */
  390. if (timer->cpu != cpu) {
  391. printk("mod_virt_timer: running on wrong CPU, check your code\n");
  392. spin_unlock_irqrestore(&vt_list->lock, flags);
  393. put_cpu();
  394. return -EINVAL;
  395. }
  396. list_del_init(&timer->entry);
  397. timer->expires = expires;
  398. /* also change the interval if we have an interval timer */
  399. if (timer->interval)
  400. timer->interval = expires;
  401. /* the timer can't expire anymore so we can release the lock */
  402. spin_unlock_irqrestore(&vt_list->lock, flags);
  403. internal_add_vtimer(timer);
  404. return 1;
  405. }
  406. EXPORT_SYMBOL(mod_virt_timer);
  407. /*
  408. * delete a virtual timer
  409. *
  410. * returns whether the deleted timer was pending (1) or not (0)
  411. */
  412. int del_virt_timer(struct vtimer_list *timer)
  413. {
  414. unsigned long flags;
  415. struct vtimer_queue *vt_list;
  416. /* check if timer is pending */
  417. if (!vtimer_pending(timer))
  418. return 0;
  419. vt_list = &per_cpu(virt_cpu_timer, timer->cpu);
  420. spin_lock_irqsave(&vt_list->lock, flags);
  421. /* we don't interrupt a running timer, just let it expire! */
  422. list_del_init(&timer->entry);
  423. /* last timer removed */
  424. if (list_empty(&vt_list->list)) {
  425. vt_list->to_expire = 0;
  426. vt_list->offset = 0;
  427. }
  428. spin_unlock_irqrestore(&vt_list->lock, flags);
  429. return 1;
  430. }
  431. EXPORT_SYMBOL(del_virt_timer);
  432. /*
  433. * Start the virtual CPU timer on the current CPU.
  434. */
  435. void init_cpu_vtimer(void)
  436. {
  437. struct vtimer_queue *vt_list;
  438. /* kick the virtual timer */
  439. S390_lowcore.exit_timer = VTIMER_MAX_SLICE;
  440. S390_lowcore.last_update_timer = VTIMER_MAX_SLICE;
  441. asm volatile ("SPT %0" : : "m" (S390_lowcore.last_update_timer));
  442. asm volatile ("STCK %0" : "=m" (S390_lowcore.last_update_clock));
  443. /* enable cpu timer interrupts */
  444. __ctl_set_bit(0,10);
  445. vt_list = &per_cpu(virt_cpu_timer, smp_processor_id());
  446. INIT_LIST_HEAD(&vt_list->list);
  447. spin_lock_init(&vt_list->lock);
  448. vt_list->to_expire = 0;
  449. vt_list->offset = 0;
  450. vt_list->idle = 0;
  451. }
  452. static int vtimer_idle_notify(struct notifier_block *self,
  453. unsigned long action, void *hcpu)
  454. {
  455. switch (action) {
  456. case CPU_IDLE:
  457. stop_cpu_timer();
  458. break;
  459. case CPU_NOT_IDLE:
  460. start_cpu_timer();
  461. break;
  462. }
  463. return NOTIFY_OK;
  464. }
  465. static struct notifier_block vtimer_idle_nb = {
  466. .notifier_call = vtimer_idle_notify,
  467. };
  468. void __init vtime_init(void)
  469. {
  470. /* request the cpu timer external interrupt */
  471. if (register_early_external_interrupt(0x1005, do_cpu_timer_interrupt,
  472. &ext_int_info_timer) != 0)
  473. panic("Couldn't request external interrupt 0x1005");
  474. if (register_idle_notifier(&vtimer_idle_nb))
  475. panic("Couldn't register idle notifier");
  476. /* Enable cpu timer interrupts on the boot cpu. */
  477. init_cpu_vtimer();
  478. }