2 * arch/s390/kernel/vtime.c
3 * Virtual cpu timer based timer functions.
6 * Copyright (C) 2004 IBM Deutschland Entwicklung GmbH, IBM Corporation
7 * Author(s): Jan Glauber <jan.glauber@de.ibm.com>
10 #include <linux/module.h>
11 #include <linux/kernel.h>
12 #include <linux/time.h>
13 #include <linux/delay.h>
14 #include <linux/init.h>
15 #include <linux/smp.h>
16 #include <linux/types.h>
17 #include <linux/timex.h>
18 #include <linux/notifier.h>
19 #include <linux/kernel_stat.h>
20 #include <linux/rcupdate.h>
21 #include <linux/posix-timers.h>
23 #include <asm/s390_ext.h>
24 #include <asm/timer.h>
25 #include <asm/irq_regs.h>
26 #include <asm/cputime.h>
28 static DEFINE_PER_CPU(struct vtimer_queue, virt_cpu_timer);
30 DEFINE_PER_CPU(struct s390_idle_data, s390_idle);
32 static inline __u64 get_vtimer(void)
36 asm volatile("STPT %0" : "=m" (timer));
40 static inline void set_vtimer(__u64 expires)
44 asm volatile (" STPT %0\n" /* Store current cpu timer value */
45 " SPT %1" /* Set new value immediatly afterwards */
46 : "=m" (timer) : "m" (expires) );
47 S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer;
48 S390_lowcore.last_update_timer = expires;
52 * Update process times based on virtual cpu times stored by entry.S
53 * to the lowcore fields user_timer, system_timer & steal_clock.
55 static void do_account_vtime(struct task_struct *tsk, int hardirq_offset)
57 struct thread_info *ti = task_thread_info(tsk);
58 __u64 timer, clock, user, system, steal;
60 timer = S390_lowcore.last_update_timer;
61 clock = S390_lowcore.last_update_clock;
62 asm volatile (" STPT %0\n" /* Store current cpu timer value */
63 " STCK %1" /* Store current tod clock value */
64 : "=m" (S390_lowcore.last_update_timer),
65 "=m" (S390_lowcore.last_update_clock) );
66 S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer;
67 S390_lowcore.steal_timer += S390_lowcore.last_update_clock - clock;
69 user = S390_lowcore.user_timer - ti->user_timer;
70 S390_lowcore.steal_timer -= user;
71 ti->user_timer = S390_lowcore.user_timer;
72 account_user_time(tsk, user, user);
74 system = S390_lowcore.system_timer - ti->system_timer;
75 S390_lowcore.steal_timer -= system;
76 ti->system_timer = S390_lowcore.system_timer;
77 account_system_time(tsk, hardirq_offset, system, system);
79 steal = S390_lowcore.steal_timer;
80 if ((s64) steal > 0) {
81 S390_lowcore.steal_timer = 0;
82 account_steal_time(steal);
86 void account_vtime(struct task_struct *prev, struct task_struct *next)
88 struct thread_info *ti;
90 do_account_vtime(prev, 0);
91 ti = task_thread_info(prev);
92 ti->user_timer = S390_lowcore.user_timer;
93 ti->system_timer = S390_lowcore.system_timer;
94 ti = task_thread_info(next);
95 S390_lowcore.user_timer = ti->user_timer;
96 S390_lowcore.system_timer = ti->system_timer;
99 void account_process_tick(struct task_struct *tsk, int user_tick)
101 do_account_vtime(tsk, HARDIRQ_OFFSET);
105 * Update process times based on virtual cpu times stored by entry.S
106 * to the lowcore fields user_timer, system_timer & steal_clock.
108 void account_system_vtime(struct task_struct *tsk)
110 struct thread_info *ti = task_thread_info(tsk);
113 timer = S390_lowcore.last_update_timer;
114 S390_lowcore.last_update_timer = get_vtimer();
115 S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer;
117 system = S390_lowcore.system_timer - ti->system_timer;
118 S390_lowcore.steal_timer -= system;
119 ti->system_timer = S390_lowcore.system_timer;
120 account_system_time(tsk, 0, system, system);
122 EXPORT_SYMBOL_GPL(account_system_vtime);
124 void vtime_start_cpu(__u64 int_clock, __u64 enter_timer)
126 struct s390_idle_data *idle = &__get_cpu_var(s390_idle);
127 struct vtimer_queue *vq = &__get_cpu_var(virt_cpu_timer);
128 __u64 idle_time, expires;
130 if (idle->idle_enter == 0ULL)
133 /* Account time spent with enabled wait psw loaded as idle time. */
134 idle_time = int_clock - idle->idle_enter;
135 account_idle_time(idle_time);
136 S390_lowcore.steal_timer +=
137 idle->idle_enter - S390_lowcore.last_update_clock;
138 S390_lowcore.last_update_clock = int_clock;
140 /* Account system time spent going idle. */
141 S390_lowcore.system_timer += S390_lowcore.last_update_timer - vq->idle;
142 S390_lowcore.last_update_timer = enter_timer;
144 /* Restart vtime CPU timer */
146 /* Program old expire value but first save progress. */
147 expires = vq->idle - enter_timer;
148 expires += get_vtimer();
151 /* Don't account the CPU timer delta while the cpu was idle. */
152 vq->elapsed -= vq->idle - enter_timer;
157 idle->idle_time += idle_time;
158 idle->idle_enter = 0ULL;
164 void vtime_stop_cpu(void)
166 struct s390_idle_data *idle = &__get_cpu_var(s390_idle);
167 struct vtimer_queue *vq = &__get_cpu_var(virt_cpu_timer);
170 /* Wait for external, I/O or machine check interrupt. */
171 psw.mask = psw_kernel_bits | PSW_MASK_WAIT | PSW_MASK_IO | PSW_MASK_EXT;
173 idle->nohz_delay = 0;
175 /* Check if the CPU timer needs to be reprogrammed. */
177 __u64 vmax = VTIMER_MAX_SLICE;
179 * The inline assembly is equivalent to
180 * vq->idle = get_cpu_timer();
181 * set_cpu_timer(VTIMER_MAX_SLICE);
182 * idle->idle_enter = get_clock();
183 * __load_psw_mask(psw_kernel_bits | PSW_MASK_WAIT |
184 * PSW_MASK_IO | PSW_MASK_EXT);
185 * The difference is that the inline assembly makes sure that
186 * the last three instruction are stpt, stck and lpsw in that
187 * order. This is done to increase the precision.
194 #else /* CONFIG_64BIT */
197 #endif /* CONFIG_64BIT */
203 #else /* CONFIG_64BIT */
205 #endif /* CONFIG_64BIT */
207 : "=m" (idle->idle_enter), "=m" (vq->idle)
208 : "a" (&psw), "a" (&idle->idle_enter),
209 "a" (&vq->idle), "a" (&vmax), "m" (vmax), "m" (psw)
210 : "memory", "cc", "1");
213 * The inline assembly is equivalent to
214 * vq->idle = get_cpu_timer();
215 * idle->idle_enter = get_clock();
216 * __load_psw_mask(psw_kernel_bits | PSW_MASK_WAIT |
217 * PSW_MASK_IO | PSW_MASK_EXT);
218 * The difference is that the inline assembly makes sure that
219 * the last three instruction are stpt, stck and lpsw in that
220 * order. This is done to increase the precision.
227 #else /* CONFIG_64BIT */
230 #endif /* CONFIG_64BIT */
235 #else /* CONFIG_64BIT */
237 #endif /* CONFIG_64BIT */
239 : "=m" (idle->idle_enter), "=m" (vq->idle)
240 : "a" (&psw), "a" (&idle->idle_enter),
241 "a" (&vq->idle), "m" (psw)
242 : "memory", "cc", "1");
246 cputime64_t s390_get_idle_time(int cpu)
248 struct s390_idle_data *idle;
249 unsigned long long now, idle_time, idle_enter;
250 unsigned int sequence;
252 idle = &per_cpu(s390_idle, cpu);
256 sequence = idle->sequence;
261 idle_enter = idle->idle_enter;
262 if (idle_enter != 0ULL && idle_enter < now)
263 idle_time = now - idle_enter;
265 if (idle->sequence != sequence)
271 * Sorted add to a list. List is linear searched until first bigger
274 static void list_add_sorted(struct vtimer_list *timer, struct list_head *head)
276 struct vtimer_list *event;
278 list_for_each_entry(event, head, entry) {
279 if (event->expires > timer->expires) {
280 list_add_tail(&timer->entry, &event->entry);
284 list_add_tail(&timer->entry, head);
288 * Do the callback functions of expired vtimer events.
289 * Called from within the interrupt handler.
291 static void do_callbacks(struct list_head *cb_list)
293 struct vtimer_queue *vq;
294 struct vtimer_list *event, *tmp;
296 if (list_empty(cb_list))
299 vq = &__get_cpu_var(virt_cpu_timer);
301 list_for_each_entry_safe(event, tmp, cb_list, entry) {
302 list_del_init(&event->entry);
303 (event->function)(event->data);
304 if (event->interval) {
305 /* Recharge interval timer */
306 event->expires = event->interval + vq->elapsed;
307 spin_lock(&vq->lock);
308 list_add_sorted(event, &vq->list);
309 spin_unlock(&vq->lock);
315 * Handler for the virtual CPU timer.
317 static void do_cpu_timer_interrupt(unsigned int ext_int_code,
318 unsigned int param32, unsigned long param64)
320 struct vtimer_queue *vq;
321 struct vtimer_list *event, *tmp;
322 struct list_head cb_list; /* the callback queue */
325 INIT_LIST_HEAD(&cb_list);
326 vq = &__get_cpu_var(virt_cpu_timer);
328 /* walk timer list, fire all expired events */
329 spin_lock(&vq->lock);
331 elapsed = vq->elapsed + (vq->timer - S390_lowcore.async_enter_timer);
332 BUG_ON((s64) elapsed < 0);
334 list_for_each_entry_safe(event, tmp, &vq->list, entry) {
335 if (event->expires < elapsed)
336 /* move expired timer to the callback queue */
337 list_move_tail(&event->entry, &cb_list);
339 event->expires -= elapsed;
341 spin_unlock(&vq->lock);
343 vq->do_spt = list_empty(&cb_list);
344 do_callbacks(&cb_list);
346 /* next event is first in list */
347 next = VTIMER_MAX_SLICE;
348 spin_lock(&vq->lock);
349 if (!list_empty(&vq->list)) {
350 event = list_first_entry(&vq->list, struct vtimer_list, entry);
351 next = event->expires;
354 spin_unlock(&vq->lock);
356 * To improve precision add the time spent by the
357 * interrupt handler to the elapsed time.
358 * Note: CPU timer counts down and we got an interrupt,
359 * the current content is negative
361 elapsed = S390_lowcore.async_enter_timer - get_vtimer();
362 set_vtimer(next - elapsed);
363 vq->timer = next - elapsed;
364 vq->elapsed = elapsed;
367 void init_virt_timer(struct vtimer_list *timer)
369 timer->function = NULL;
370 INIT_LIST_HEAD(&timer->entry);
372 EXPORT_SYMBOL(init_virt_timer);
374 static inline int vtimer_pending(struct vtimer_list *timer)
376 return (!list_empty(&timer->entry));
380 * this function should only run on the specified CPU
382 static void internal_add_vtimer(struct vtimer_list *timer)
384 struct vtimer_queue *vq;
388 vq = &per_cpu(virt_cpu_timer, timer->cpu);
389 spin_lock_irqsave(&vq->lock, flags);
391 BUG_ON(timer->cpu != smp_processor_id());
393 if (list_empty(&vq->list)) {
394 /* First timer on this cpu, just program it. */
395 list_add(&timer->entry, &vq->list);
396 set_vtimer(timer->expires);
397 vq->timer = timer->expires;
400 /* Check progress of old timers. */
401 expires = timer->expires;
403 if (likely((s64) expires < (s64) left)) {
404 /* The new timer expires before the current timer. */
406 vq->elapsed += vq->timer - left;
409 vq->elapsed += vq->timer - left;
412 /* Insert new timer into per cpu list. */
413 timer->expires += vq->elapsed;
414 list_add_sorted(timer, &vq->list);
417 spin_unlock_irqrestore(&vq->lock, flags);
418 /* release CPU acquired in prepare_vtimer or mod_virt_timer() */
422 static inline void prepare_vtimer(struct vtimer_list *timer)
424 BUG_ON(!timer->function);
425 BUG_ON(!timer->expires || timer->expires > VTIMER_MAX_SLICE);
426 BUG_ON(vtimer_pending(timer));
427 timer->cpu = get_cpu();
431 * add_virt_timer - add an oneshot virtual CPU timer
433 void add_virt_timer(void *new)
435 struct vtimer_list *timer;
437 timer = (struct vtimer_list *)new;
438 prepare_vtimer(timer);
440 internal_add_vtimer(timer);
442 EXPORT_SYMBOL(add_virt_timer);
445 * add_virt_timer_int - add an interval virtual CPU timer
447 void add_virt_timer_periodic(void *new)
449 struct vtimer_list *timer;
451 timer = (struct vtimer_list *)new;
452 prepare_vtimer(timer);
453 timer->interval = timer->expires;
454 internal_add_vtimer(timer);
456 EXPORT_SYMBOL(add_virt_timer_periodic);
458 int __mod_vtimer(struct vtimer_list *timer, __u64 expires, int periodic)
460 struct vtimer_queue *vq;
464 BUG_ON(!timer->function);
465 BUG_ON(!expires || expires > VTIMER_MAX_SLICE);
467 if (timer->expires == expires && vtimer_pending(timer))
471 vq = &per_cpu(virt_cpu_timer, cpu);
473 /* disable interrupts before test if timer is pending */
474 spin_lock_irqsave(&vq->lock, flags);
476 /* if timer isn't pending add it on the current CPU */
477 if (!vtimer_pending(timer)) {
478 spin_unlock_irqrestore(&vq->lock, flags);
481 timer->interval = expires;
484 timer->expires = expires;
486 internal_add_vtimer(timer);
490 /* check if we run on the right CPU */
491 BUG_ON(timer->cpu != cpu);
493 list_del_init(&timer->entry);
494 timer->expires = expires;
496 timer->interval = expires;
498 /* the timer can't expire anymore so we can release the lock */
499 spin_unlock_irqrestore(&vq->lock, flags);
500 internal_add_vtimer(timer);
505 * If we change a pending timer the function must be called on the CPU
506 * where the timer is running on.
508 * returns whether it has modified a pending timer (1) or not (0)
510 int mod_virt_timer(struct vtimer_list *timer, __u64 expires)
512 return __mod_vtimer(timer, expires, 0);
514 EXPORT_SYMBOL(mod_virt_timer);
517 * If we change a pending timer the function must be called on the CPU
518 * where the timer is running on.
520 * returns whether it has modified a pending timer (1) or not (0)
522 int mod_virt_timer_periodic(struct vtimer_list *timer, __u64 expires)
524 return __mod_vtimer(timer, expires, 1);
526 EXPORT_SYMBOL(mod_virt_timer_periodic);
529 * delete a virtual timer
531 * returns whether the deleted timer was pending (1) or not (0)
533 int del_virt_timer(struct vtimer_list *timer)
536 struct vtimer_queue *vq;
538 /* check if timer is pending */
539 if (!vtimer_pending(timer))
542 vq = &per_cpu(virt_cpu_timer, timer->cpu);
543 spin_lock_irqsave(&vq->lock, flags);
545 /* we don't interrupt a running timer, just let it expire! */
546 list_del_init(&timer->entry);
548 spin_unlock_irqrestore(&vq->lock, flags);
551 EXPORT_SYMBOL(del_virt_timer);
554 * Start the virtual CPU timer on the current CPU.
556 void init_cpu_vtimer(void)
558 struct vtimer_queue *vq;
560 /* initialize per cpu vtimer structure */
561 vq = &__get_cpu_var(virt_cpu_timer);
562 INIT_LIST_HEAD(&vq->list);
563 spin_lock_init(&vq->lock);
565 /* enable cpu timer interrupts */
569 void __init vtime_init(void)
571 /* request the cpu timer external interrupt */
572 if (register_external_interrupt(0x1005, do_cpu_timer_interrupt))
573 panic("Couldn't request external interrupt 0x1005");
575 /* Enable cpu timer interrupts on the boot cpu. */