Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * arch/s390/kernel/vtime.c | |
3 | * Virtual cpu timer based timer functions. | |
4 | * | |
5 | * S390 version | |
6 | * Copyright (C) 2004 IBM Deutschland Entwicklung GmbH, IBM Corporation | |
7 | * Author(s): Jan Glauber <jan.glauber@de.ibm.com> | |
8 | */ | |
9 | ||
1da177e4 LT |
10 | #include <linux/module.h> |
11 | #include <linux/kernel.h> | |
12 | #include <linux/time.h> | |
13 | #include <linux/delay.h> | |
14 | #include <linux/init.h> | |
15 | #include <linux/smp.h> | |
16 | #include <linux/types.h> | |
17 | #include <linux/timex.h> | |
18 | #include <linux/notifier.h> | |
19 | #include <linux/kernel_stat.h> | |
20 | #include <linux/rcupdate.h> | |
21 | #include <linux/posix-timers.h> | |
22 | ||
23 | #include <asm/s390_ext.h> | |
24 | #include <asm/timer.h> | |
5a489b98 | 25 | #include <asm/irq_regs.h> |
1da177e4 | 26 | |
1da177e4 | 27 | static ext_int_info_t ext_int_info_timer; |
2b67fc46 | 28 | static DEFINE_PER_CPU(struct vtimer_queue, virt_cpu_timer); |
1da177e4 | 29 | |
1da177e4 LT |
30 | /* |
31 | * Update process times based on virtual cpu times stored by entry.S | |
32 | * to the lowcore fields user_timer, system_timer & steal_clock. | |
33 | */ | |
aa5e97ce | 34 | static void do_account_vtime(struct task_struct *tsk, int hardirq_offset) |
1da177e4 | 35 | { |
aa5e97ce MS |
36 | struct thread_info *ti = task_thread_info(tsk); |
37 | __u64 timer, clock, user, system, steal; | |
1da177e4 LT |
38 | |
39 | timer = S390_lowcore.last_update_timer; | |
40 | clock = S390_lowcore.last_update_clock; | |
41 | asm volatile (" STPT %0\n" /* Store current cpu timer value */ | |
42 | " STCK %1" /* Store current tod clock value */ | |
43 | : "=m" (S390_lowcore.last_update_timer), | |
44 | "=m" (S390_lowcore.last_update_clock) ); | |
45 | S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer; | |
aa5e97ce | 46 | S390_lowcore.steal_timer += S390_lowcore.last_update_clock - clock; |
1da177e4 | 47 | |
aa5e97ce MS |
48 | user = S390_lowcore.user_timer - ti->user_timer; |
49 | S390_lowcore.steal_timer -= user; | |
50 | ti->user_timer = S390_lowcore.user_timer; | |
51 | account_user_time(tsk, user, user); | |
1da177e4 | 52 | |
aa5e97ce MS |
53 | system = S390_lowcore.system_timer - ti->system_timer; |
54 | S390_lowcore.steal_timer -= system; | |
55 | ti->system_timer = S390_lowcore.system_timer; | |
79741dd3 | 56 | if (idle_task(smp_processor_id()) != current) |
aa5e97ce | 57 | account_system_time(tsk, hardirq_offset, system, system); |
79741dd3 | 58 | else |
aa5e97ce | 59 | account_idle_time(system); |
1da177e4 | 60 | |
aa5e97ce MS |
61 | steal = S390_lowcore.steal_timer; |
62 | if ((s64) steal > 0) { | |
63 | S390_lowcore.steal_timer = 0; | |
79741dd3 | 64 | if (idle_task(smp_processor_id()) != current) |
aa5e97ce | 65 | account_steal_time(steal); |
79741dd3 | 66 | else |
aa5e97ce | 67 | account_idle_time(steal); |
1da177e4 | 68 | } |
1da177e4 LT |
69 | } |
70 | ||
aa5e97ce | 71 | void account_vtime(struct task_struct *prev, struct task_struct *next) |
1f1c12af | 72 | { |
aa5e97ce MS |
73 | struct thread_info *ti; |
74 | ||
75 | do_account_vtime(prev, 0); | |
76 | ti = task_thread_info(prev); | |
77 | ti->user_timer = S390_lowcore.user_timer; | |
78 | ti->system_timer = S390_lowcore.system_timer; | |
79 | ti = task_thread_info(next); | |
80 | S390_lowcore.user_timer = ti->user_timer; | |
81 | S390_lowcore.system_timer = ti->system_timer; | |
82 | } | |
1f1c12af | 83 | |
aa5e97ce MS |
84 | void account_process_tick(struct task_struct *tsk, int user_tick) |
85 | { | |
86 | do_account_vtime(tsk, HARDIRQ_OFFSET); | |
1f1c12af MS |
87 | } |
88 | ||
1da177e4 LT |
89 | /* |
90 | * Update process times based on virtual cpu times stored by entry.S | |
91 | * to the lowcore fields user_timer, system_timer & steal_clock. | |
92 | */ | |
93 | void account_system_vtime(struct task_struct *tsk) | |
94 | { | |
aa5e97ce MS |
95 | struct thread_info *ti = task_thread_info(tsk); |
96 | __u64 timer, system; | |
1da177e4 LT |
97 | |
98 | timer = S390_lowcore.last_update_timer; | |
99 | asm volatile (" STPT %0" /* Store current cpu timer value */ | |
100 | : "=m" (S390_lowcore.last_update_timer) ); | |
101 | S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer; | |
102 | ||
aa5e97ce MS |
103 | system = S390_lowcore.system_timer - ti->system_timer; |
104 | S390_lowcore.steal_timer -= system; | |
105 | ti->system_timer = S390_lowcore.system_timer; | |
79741dd3 | 106 | if (in_irq() || idle_task(smp_processor_id()) != current) |
aa5e97ce | 107 | account_system_time(tsk, 0, system, system); |
79741dd3 | 108 | else |
aa5e97ce | 109 | account_idle_time(system); |
1da177e4 | 110 | } |
b0c632db | 111 | EXPORT_SYMBOL_GPL(account_system_vtime); |
1da177e4 LT |
112 | |
113 | static inline void set_vtimer(__u64 expires) | |
114 | { | |
6f430924 | 115 | struct vtimer_queue *vq = &__get_cpu_var(virt_cpu_timer); |
1da177e4 LT |
116 | __u64 timer; |
117 | ||
118 | asm volatile (" STPT %0\n" /* Store current cpu timer value */ | |
119 | " SPT %1" /* Set new value immediatly afterwards */ | |
120 | : "=m" (timer) : "m" (expires) ); | |
121 | S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer; | |
122 | S390_lowcore.last_update_timer = expires; | |
123 | ||
124 | /* store expire time for this CPU timer */ | |
6f430924 | 125 | vq->to_expire = expires; |
1da177e4 | 126 | } |
1da177e4 | 127 | |
773922e1 | 128 | void vtime_start_cpu_timer(void) |
1da177e4 LT |
129 | { |
130 | struct vtimer_queue *vt_list; | |
131 | ||
db77aa5f | 132 | vt_list = &__get_cpu_var(virt_cpu_timer); |
4b7e0706 MS |
133 | |
134 | /* CPU timer interrupt is pending, don't reprogramm it */ | |
135 | if (vt_list->idle & 1LL<<63) | |
136 | return; | |
137 | ||
138 | if (!list_empty(&vt_list->list)) | |
139 | set_vtimer(vt_list->idle); | |
1da177e4 LT |
140 | } |
141 | ||
773922e1 | 142 | void vtime_stop_cpu_timer(void) |
1da177e4 | 143 | { |
1da177e4 LT |
144 | struct vtimer_queue *vt_list; |
145 | ||
db77aa5f | 146 | vt_list = &__get_cpu_var(virt_cpu_timer); |
1da177e4 LT |
147 | |
148 | /* nothing to do */ | |
149 | if (list_empty(&vt_list->list)) { | |
150 | vt_list->idle = VTIMER_MAX_SLICE; | |
151 | goto fire; | |
152 | } | |
153 | ||
4b7e0706 MS |
154 | /* store the actual expire value */ |
155 | asm volatile ("STPT %0" : "=m" (vt_list->idle)); | |
1da177e4 LT |
156 | |
157 | /* | |
4b7e0706 MS |
158 | * If the CPU timer is negative we don't reprogramm |
159 | * it because we will get instantly an interrupt. | |
1da177e4 | 160 | */ |
4b7e0706 | 161 | if (vt_list->idle & 1LL<<63) |
1da177e4 | 162 | return; |
1da177e4 | 163 | |
4b7e0706 | 164 | vt_list->offset += vt_list->to_expire - vt_list->idle; |
1da177e4 LT |
165 | |
166 | /* | |
167 | * We cannot halt the CPU timer, we just write a value that | |
168 | * nearly never expires (only after 71 years) and re-write | |
169 | * the stored expire value if we continue the timer | |
170 | */ | |
171 | fire: | |
172 | set_vtimer(VTIMER_MAX_SLICE); | |
173 | } | |
174 | ||
175 | /* | |
176 | * Sorted add to a list. List is linear searched until first bigger | |
177 | * element is found. | |
178 | */ | |
179 | static void list_add_sorted(struct vtimer_list *timer, struct list_head *head) | |
180 | { | |
181 | struct vtimer_list *event; | |
182 | ||
183 | list_for_each_entry(event, head, entry) { | |
184 | if (event->expires > timer->expires) { | |
185 | list_add_tail(&timer->entry, &event->entry); | |
186 | return; | |
187 | } | |
188 | } | |
189 | list_add_tail(&timer->entry, head); | |
190 | } | |
191 | ||
192 | /* | |
193 | * Do the callback functions of expired vtimer events. | |
194 | * Called from within the interrupt handler. | |
195 | */ | |
9d0a57cb | 196 | static void do_callbacks(struct list_head *cb_list) |
1da177e4 LT |
197 | { |
198 | struct vtimer_queue *vt_list; | |
199 | struct vtimer_list *event, *tmp; | |
9d0a57cb | 200 | void (*fn)(unsigned long); |
1da177e4 LT |
201 | unsigned long data; |
202 | ||
203 | if (list_empty(cb_list)) | |
204 | return; | |
205 | ||
db77aa5f | 206 | vt_list = &__get_cpu_var(virt_cpu_timer); |
1da177e4 LT |
207 | |
208 | list_for_each_entry_safe(event, tmp, cb_list, entry) { | |
209 | fn = event->function; | |
210 | data = event->data; | |
9d0a57cb | 211 | fn(data); |
1da177e4 LT |
212 | |
213 | if (!event->interval) | |
214 | /* delete one shot timer */ | |
215 | list_del_init(&event->entry); | |
216 | else { | |
217 | /* move interval timer back to list */ | |
218 | spin_lock(&vt_list->lock); | |
219 | list_del_init(&event->entry); | |
220 | list_add_sorted(event, &vt_list->list); | |
221 | spin_unlock(&vt_list->lock); | |
222 | } | |
223 | } | |
224 | } | |
225 | ||
226 | /* | |
227 | * Handler for the virtual CPU timer. | |
228 | */ | |
5a489b98 | 229 | static void do_cpu_timer_interrupt(__u16 error_code) |
1da177e4 | 230 | { |
1da177e4 LT |
231 | __u64 next, delta; |
232 | struct vtimer_queue *vt_list; | |
233 | struct vtimer_list *event, *tmp; | |
234 | struct list_head *ptr; | |
235 | /* the callback queue */ | |
236 | struct list_head cb_list; | |
237 | ||
238 | INIT_LIST_HEAD(&cb_list); | |
db77aa5f | 239 | vt_list = &__get_cpu_var(virt_cpu_timer); |
1da177e4 LT |
240 | |
241 | /* walk timer list, fire all expired events */ | |
242 | spin_lock(&vt_list->lock); | |
243 | ||
244 | if (vt_list->to_expire < VTIMER_MAX_SLICE) | |
245 | vt_list->offset += vt_list->to_expire; | |
246 | ||
247 | list_for_each_entry_safe(event, tmp, &vt_list->list, entry) { | |
248 | if (event->expires > vt_list->offset) | |
249 | /* found first unexpired event, leave */ | |
250 | break; | |
251 | ||
252 | /* re-charge interval timer, we have to add the offset */ | |
253 | if (event->interval) | |
254 | event->expires = event->interval + vt_list->offset; | |
255 | ||
256 | /* move expired timer to the callback queue */ | |
257 | list_move_tail(&event->entry, &cb_list); | |
258 | } | |
259 | spin_unlock(&vt_list->lock); | |
9d0a57cb | 260 | do_callbacks(&cb_list); |
1da177e4 LT |
261 | |
262 | /* next event is first in list */ | |
263 | spin_lock(&vt_list->lock); | |
264 | if (!list_empty(&vt_list->list)) { | |
265 | ptr = vt_list->list.next; | |
266 | event = list_entry(ptr, struct vtimer_list, entry); | |
267 | next = event->expires - vt_list->offset; | |
268 | ||
269 | /* add the expired time from this interrupt handler | |
270 | * and the callback functions | |
271 | */ | |
272 | asm volatile ("STPT %0" : "=m" (delta)); | |
273 | delta = 0xffffffffffffffffLL - delta + 1; | |
274 | vt_list->offset += delta; | |
275 | next -= delta; | |
276 | } else { | |
277 | vt_list->offset = 0; | |
278 | next = VTIMER_MAX_SLICE; | |
279 | } | |
280 | spin_unlock(&vt_list->lock); | |
281 | set_vtimer(next); | |
282 | } | |
283 | ||
284 | void init_virt_timer(struct vtimer_list *timer) | |
285 | { | |
1da177e4 LT |
286 | timer->function = NULL; |
287 | INIT_LIST_HEAD(&timer->entry); | |
288 | spin_lock_init(&timer->lock); | |
289 | } | |
290 | EXPORT_SYMBOL(init_virt_timer); | |
291 | ||
1da177e4 LT |
292 | static inline int vtimer_pending(struct vtimer_list *timer) |
293 | { | |
294 | return (!list_empty(&timer->entry)); | |
295 | } | |
296 | ||
297 | /* | |
298 | * this function should only run on the specified CPU | |
299 | */ | |
300 | static void internal_add_vtimer(struct vtimer_list *timer) | |
301 | { | |
302 | unsigned long flags; | |
303 | __u64 done; | |
304 | struct vtimer_list *event; | |
305 | struct vtimer_queue *vt_list; | |
306 | ||
307 | vt_list = &per_cpu(virt_cpu_timer, timer->cpu); | |
308 | spin_lock_irqsave(&vt_list->lock, flags); | |
309 | ||
ca366a32 | 310 | BUG_ON(timer->cpu != smp_processor_id()); |
1da177e4 LT |
311 | |
312 | /* if list is empty we only have to set the timer */ | |
313 | if (list_empty(&vt_list->list)) { | |
314 | /* reset the offset, this may happen if the last timer was | |
315 | * just deleted by mod_virt_timer and the interrupt | |
316 | * didn't happen until here | |
317 | */ | |
318 | vt_list->offset = 0; | |
319 | goto fire; | |
320 | } | |
321 | ||
322 | /* save progress */ | |
323 | asm volatile ("STPT %0" : "=m" (done)); | |
324 | ||
325 | /* calculate completed work */ | |
326 | done = vt_list->to_expire - done + vt_list->offset; | |
327 | vt_list->offset = 0; | |
328 | ||
329 | list_for_each_entry(event, &vt_list->list, entry) | |
330 | event->expires -= done; | |
331 | ||
332 | fire: | |
333 | list_add_sorted(timer, &vt_list->list); | |
334 | ||
335 | /* get first element, which is the next vtimer slice */ | |
336 | event = list_entry(vt_list->list.next, struct vtimer_list, entry); | |
337 | ||
338 | set_vtimer(event->expires); | |
339 | spin_unlock_irqrestore(&vt_list->lock, flags); | |
d6e05edc | 340 | /* release CPU acquired in prepare_vtimer or mod_virt_timer() */ |
1da177e4 LT |
341 | put_cpu(); |
342 | } | |
343 | ||
ca366a32 | 344 | static inline void prepare_vtimer(struct vtimer_list *timer) |
1da177e4 | 345 | { |
ca366a32 MS |
346 | BUG_ON(!timer->function); |
347 | BUG_ON(!timer->expires || timer->expires > VTIMER_MAX_SLICE); | |
348 | BUG_ON(vtimer_pending(timer)); | |
1da177e4 | 349 | timer->cpu = get_cpu(); |
1da177e4 LT |
350 | } |
351 | ||
352 | /* | |
353 | * add_virt_timer - add an oneshot virtual CPU timer | |
354 | */ | |
355 | void add_virt_timer(void *new) | |
356 | { | |
357 | struct vtimer_list *timer; | |
358 | ||
359 | timer = (struct vtimer_list *)new; | |
ca366a32 | 360 | prepare_vtimer(timer); |
1da177e4 LT |
361 | timer->interval = 0; |
362 | internal_add_vtimer(timer); | |
363 | } | |
364 | EXPORT_SYMBOL(add_virt_timer); | |
365 | ||
366 | /* | |
367 | * add_virt_timer_int - add an interval virtual CPU timer | |
368 | */ | |
369 | void add_virt_timer_periodic(void *new) | |
370 | { | |
371 | struct vtimer_list *timer; | |
372 | ||
373 | timer = (struct vtimer_list *)new; | |
ca366a32 | 374 | prepare_vtimer(timer); |
1da177e4 LT |
375 | timer->interval = timer->expires; |
376 | internal_add_vtimer(timer); | |
377 | } | |
378 | EXPORT_SYMBOL(add_virt_timer_periodic); | |
379 | ||
380 | /* | |
381 | * If we change a pending timer the function must be called on the CPU | |
3bb447fc | 382 | * where the timer is running on, e.g. by smp_call_function_single() |
1da177e4 LT |
383 | * |
384 | * The original mod_timer adds the timer if it is not pending. For compatibility | |
385 | * we do the same. The timer will be added on the current CPU as a oneshot timer. | |
386 | * | |
387 | * returns whether it has modified a pending timer (1) or not (0) | |
388 | */ | |
389 | int mod_virt_timer(struct vtimer_list *timer, __u64 expires) | |
390 | { | |
391 | struct vtimer_queue *vt_list; | |
392 | unsigned long flags; | |
393 | int cpu; | |
394 | ||
ca366a32 MS |
395 | BUG_ON(!timer->function); |
396 | BUG_ON(!expires || expires > VTIMER_MAX_SLICE); | |
1da177e4 LT |
397 | |
398 | /* | |
399 | * This is a common optimization triggered by the | |
400 | * networking code - if the timer is re-modified | |
401 | * to be the same thing then just return: | |
402 | */ | |
403 | if (timer->expires == expires && vtimer_pending(timer)) | |
404 | return 1; | |
405 | ||
406 | cpu = get_cpu(); | |
407 | vt_list = &per_cpu(virt_cpu_timer, cpu); | |
408 | ||
ca366a32 MS |
409 | /* check if we run on the right CPU */ |
410 | BUG_ON(timer->cpu != cpu); | |
411 | ||
1da177e4 LT |
412 | /* disable interrupts before test if timer is pending */ |
413 | spin_lock_irqsave(&vt_list->lock, flags); | |
414 | ||
415 | /* if timer isn't pending add it on the current CPU */ | |
416 | if (!vtimer_pending(timer)) { | |
417 | spin_unlock_irqrestore(&vt_list->lock, flags); | |
418 | /* we do not activate an interval timer with mod_virt_timer */ | |
419 | timer->interval = 0; | |
420 | timer->expires = expires; | |
421 | timer->cpu = cpu; | |
422 | internal_add_vtimer(timer); | |
423 | return 0; | |
424 | } | |
425 | ||
1da177e4 LT |
426 | list_del_init(&timer->entry); |
427 | timer->expires = expires; | |
428 | ||
429 | /* also change the interval if we have an interval timer */ | |
430 | if (timer->interval) | |
431 | timer->interval = expires; | |
432 | ||
433 | /* the timer can't expire anymore so we can release the lock */ | |
434 | spin_unlock_irqrestore(&vt_list->lock, flags); | |
435 | internal_add_vtimer(timer); | |
436 | return 1; | |
437 | } | |
438 | EXPORT_SYMBOL(mod_virt_timer); | |
439 | ||
440 | /* | |
441 | * delete a virtual timer | |
442 | * | |
443 | * returns whether the deleted timer was pending (1) or not (0) | |
444 | */ | |
445 | int del_virt_timer(struct vtimer_list *timer) | |
446 | { | |
447 | unsigned long flags; | |
448 | struct vtimer_queue *vt_list; | |
449 | ||
1da177e4 LT |
450 | /* check if timer is pending */ |
451 | if (!vtimer_pending(timer)) | |
452 | return 0; | |
453 | ||
454 | vt_list = &per_cpu(virt_cpu_timer, timer->cpu); | |
455 | spin_lock_irqsave(&vt_list->lock, flags); | |
456 | ||
457 | /* we don't interrupt a running timer, just let it expire! */ | |
458 | list_del_init(&timer->entry); | |
459 | ||
460 | /* last timer removed */ | |
461 | if (list_empty(&vt_list->list)) { | |
462 | vt_list->to_expire = 0; | |
463 | vt_list->offset = 0; | |
464 | } | |
465 | ||
466 | spin_unlock_irqrestore(&vt_list->lock, flags); | |
467 | return 1; | |
468 | } | |
469 | EXPORT_SYMBOL(del_virt_timer); | |
470 | ||
471 | /* | |
472 | * Start the virtual CPU timer on the current CPU. | |
473 | */ | |
474 | void init_cpu_vtimer(void) | |
475 | { | |
476 | struct vtimer_queue *vt_list; | |
1da177e4 LT |
477 | |
478 | /* kick the virtual timer */ | |
479 | S390_lowcore.exit_timer = VTIMER_MAX_SLICE; | |
480 | S390_lowcore.last_update_timer = VTIMER_MAX_SLICE; | |
1da177e4 | 481 | asm volatile ("STCK %0" : "=m" (S390_lowcore.last_update_clock)); |
aa5e97ce | 482 | asm volatile ("SPT %0" : : "m" (S390_lowcore.last_update_timer)); |
d54853ef MS |
483 | |
484 | /* enable cpu timer interrupts */ | |
485 | __ctl_set_bit(0,10); | |
1da177e4 | 486 | |
db77aa5f | 487 | vt_list = &__get_cpu_var(virt_cpu_timer); |
1da177e4 LT |
488 | INIT_LIST_HEAD(&vt_list->list); |
489 | spin_lock_init(&vt_list->lock); | |
490 | vt_list->to_expire = 0; | |
491 | vt_list->offset = 0; | |
492 | vt_list->idle = 0; | |
493 | ||
494 | } | |
495 | ||
1da177e4 LT |
496 | void __init vtime_init(void) |
497 | { | |
498 | /* request the cpu timer external interrupt */ | |
499 | if (register_early_external_interrupt(0x1005, do_cpu_timer_interrupt, | |
500 | &ext_int_info_timer) != 0) | |
501 | panic("Couldn't request external interrupt 0x1005"); | |
502 | ||
d54853ef | 503 | /* Enable cpu timer interrupts on the boot cpu. */ |
1da177e4 LT |
504 | init_cpu_vtimer(); |
505 | } | |
506 |