tracing: tracepoints for softirq entry/exit - add softirq-to-name array
[linux-2.6-block.git] / kernel / softirq.c
CommitLineData
1da177e4
LT
1/*
2 * linux/kernel/softirq.c
3 *
4 * Copyright (C) 1992 Linus Torvalds
5 *
b10db7f0
PM
6 * Distribute under GPLv2.
7 *
8 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
54514a70
DM
9 *
10 * Remote softirq infrastructure is by Jens Axboe.
1da177e4
LT
11 */
12
13#include <linux/module.h>
14#include <linux/kernel_stat.h>
15#include <linux/interrupt.h>
16#include <linux/init.h>
17#include <linux/mm.h>
18#include <linux/notifier.h>
19#include <linux/percpu.h>
20#include <linux/cpu.h>
83144186 21#include <linux/freezer.h>
1da177e4
LT
22#include <linux/kthread.h>
23#include <linux/rcupdate.h>
7e49fcce 24#include <linux/ftrace.h>
78eef01b 25#include <linux/smp.h>
79bf2bb3 26#include <linux/tick.h>
1da177e4
LT
27
28#include <asm/irq.h>
29/*
30 - No shared variables, all the data are CPU local.
31 - If a softirq needs serialization, let it serialize itself
32 by its own spinlocks.
33 - Even if softirq is serialized, only local cpu is marked for
34 execution. Hence, we get something sort of weak cpu binding.
35 Though it is still not clear, will it result in better locality
36 or will not.
37
38 Examples:
39 - NET RX softirq. It is multithreaded and does not require
40 any global serialization.
41 - NET TX softirq. It kicks software netdevice queues, hence
42 it is logically serialized per device, but this serialization
43 is invisible to common code.
44 - Tasklets: serialized wrt itself.
45 */
46
47#ifndef __ARCH_IRQ_STAT
48irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
49EXPORT_SYMBOL(irq_stat);
50#endif
51
978b0116 52static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
1da177e4
LT
53
54static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
55
5d592b44
JB
56char *softirq_to_name[NR_SOFTIRQS] = {
57 "HI_SOFTIRQ", "TIMER_SOFTIRQ", "NET_TX_SOFTIRQ", "NET_RX_SOFTIRQ",
58 "BLOCK_SOFTIRQ", "TASKLET_SOFTIRQ", "SCHED_SOFTIRQ", "HRTIMER_SOFTIRQ",
59 "RCU_SOFTIRQ"
60};
61
1da177e4
LT
62/*
63 * we cannot loop indefinitely here to avoid userspace starvation,
64 * but we also don't want to introduce a worst case 1/HZ latency
65 * to the pending events, so lets the scheduler to balance
66 * the softirq load for us.
67 */
68static inline void wakeup_softirqd(void)
69{
70 /* Interrupts are disabled: no need to stop preemption */
71 struct task_struct *tsk = __get_cpu_var(ksoftirqd);
72
73 if (tsk && tsk->state != TASK_RUNNING)
74 wake_up_process(tsk);
75}
76
de30a2b3
IM
77/*
78 * This one is for softirq.c-internal use,
79 * where hardirqs are disabled legitimately:
80 */
3c829c36 81#ifdef CONFIG_TRACE_IRQFLAGS
de30a2b3
IM
82static void __local_bh_disable(unsigned long ip)
83{
84 unsigned long flags;
85
86 WARN_ON_ONCE(in_irq());
87
88 raw_local_irq_save(flags);
7e49fcce
SR
89 /*
90 * The preempt tracer hooks into add_preempt_count and will break
91 * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
92 * is set and before current->softirq_enabled is cleared.
93 * We must manually increment preempt_count here and manually
94 * call the trace_preempt_off later.
95 */
96 preempt_count() += SOFTIRQ_OFFSET;
de30a2b3
IM
97 /*
98 * Were softirqs turned off above:
99 */
100 if (softirq_count() == SOFTIRQ_OFFSET)
101 trace_softirqs_off(ip);
102 raw_local_irq_restore(flags);
7e49fcce
SR
103
104 if (preempt_count() == SOFTIRQ_OFFSET)
105 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
de30a2b3 106}
3c829c36
TC
107#else /* !CONFIG_TRACE_IRQFLAGS */
108static inline void __local_bh_disable(unsigned long ip)
109{
110 add_preempt_count(SOFTIRQ_OFFSET);
111 barrier();
112}
113#endif /* CONFIG_TRACE_IRQFLAGS */
de30a2b3
IM
114
115void local_bh_disable(void)
116{
117 __local_bh_disable((unsigned long)__builtin_return_address(0));
118}
119
120EXPORT_SYMBOL(local_bh_disable);
121
de30a2b3
IM
122/*
123 * Special-case - softirqs can safely be enabled in
124 * cond_resched_softirq(), or by __do_softirq(),
125 * without processing still-pending softirqs:
126 */
127void _local_bh_enable(void)
128{
129 WARN_ON_ONCE(in_irq());
130 WARN_ON_ONCE(!irqs_disabled());
131
132 if (softirq_count() == SOFTIRQ_OFFSET)
133 trace_softirqs_on((unsigned long)__builtin_return_address(0));
134 sub_preempt_count(SOFTIRQ_OFFSET);
135}
136
137EXPORT_SYMBOL(_local_bh_enable);
138
0f476b6d 139static inline void _local_bh_enable_ip(unsigned long ip)
de30a2b3 140{
0f476b6d 141 WARN_ON_ONCE(in_irq() || irqs_disabled());
3c829c36 142#ifdef CONFIG_TRACE_IRQFLAGS
0f476b6d 143 local_irq_disable();
3c829c36 144#endif
de30a2b3
IM
145 /*
146 * Are softirqs going to be turned on now:
147 */
148 if (softirq_count() == SOFTIRQ_OFFSET)
0f476b6d 149 trace_softirqs_on(ip);
de30a2b3
IM
150 /*
151 * Keep preemption disabled until we are done with
152 * softirq processing:
153 */
154 sub_preempt_count(SOFTIRQ_OFFSET - 1);
155
156 if (unlikely(!in_interrupt() && local_softirq_pending()))
157 do_softirq();
158
159 dec_preempt_count();
3c829c36 160#ifdef CONFIG_TRACE_IRQFLAGS
0f476b6d 161 local_irq_enable();
3c829c36 162#endif
de30a2b3
IM
163 preempt_check_resched();
164}
0f476b6d
JB
165
166void local_bh_enable(void)
167{
168 _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
169}
de30a2b3
IM
170EXPORT_SYMBOL(local_bh_enable);
171
172void local_bh_enable_ip(unsigned long ip)
173{
0f476b6d 174 _local_bh_enable_ip(ip);
de30a2b3
IM
175}
176EXPORT_SYMBOL(local_bh_enable_ip);
177
1da177e4
LT
178/*
179 * We restart softirq processing MAX_SOFTIRQ_RESTART times,
180 * and we fall back to softirqd after that.
181 *
182 * This number has been established via experimentation.
183 * The two things to balance is latency against fairness -
184 * we want to handle softirqs as soon as possible, but they
185 * should not be able to lock up the box.
186 */
187#define MAX_SOFTIRQ_RESTART 10
188
189asmlinkage void __do_softirq(void)
190{
191 struct softirq_action *h;
192 __u32 pending;
193 int max_restart = MAX_SOFTIRQ_RESTART;
194 int cpu;
195
196 pending = local_softirq_pending();
829035fd
PM
197 account_system_vtime(current);
198
de30a2b3 199 __local_bh_disable((unsigned long)__builtin_return_address(0));
d820ac4c 200 lockdep_softirq_enter();
1da177e4 201
1da177e4
LT
202 cpu = smp_processor_id();
203restart:
204 /* Reset the pending bitmask before enabling irqs */
3f74478b 205 set_softirq_pending(0);
1da177e4 206
c70f5d66 207 local_irq_enable();
1da177e4
LT
208
209 h = softirq_vec;
210
211 do {
212 if (pending & 1) {
8e85b4b5
TG
213 int prev_count = preempt_count();
214
1da177e4 215 h->action(h);
8e85b4b5
TG
216
217 if (unlikely(prev_count != preempt_count())) {
5d592b44 218 printk(KERN_ERR "huh, entered softirq %td %s %p"
8e85b4b5
TG
219 "with preempt_count %08x,"
220 " exited with %08x?\n", h - softirq_vec,
5d592b44 221 softirq_to_name[h - softirq_vec],
8e85b4b5
TG
222 h->action, prev_count, preempt_count());
223 preempt_count() = prev_count;
224 }
225
1da177e4
LT
226 rcu_bh_qsctr_inc(cpu);
227 }
228 h++;
229 pending >>= 1;
230 } while (pending);
231
c70f5d66 232 local_irq_disable();
1da177e4
LT
233
234 pending = local_softirq_pending();
235 if (pending && --max_restart)
236 goto restart;
237
238 if (pending)
239 wakeup_softirqd();
240
d820ac4c 241 lockdep_softirq_exit();
829035fd
PM
242
243 account_system_vtime(current);
de30a2b3 244 _local_bh_enable();
1da177e4
LT
245}
246
247#ifndef __ARCH_HAS_DO_SOFTIRQ
248
249asmlinkage void do_softirq(void)
250{
251 __u32 pending;
252 unsigned long flags;
253
254 if (in_interrupt())
255 return;
256
257 local_irq_save(flags);
258
259 pending = local_softirq_pending();
260
261 if (pending)
262 __do_softirq();
263
264 local_irq_restore(flags);
265}
266
1da177e4
LT
267#endif
268
dde4b2b5
IM
269/*
270 * Enter an interrupt context.
271 */
272void irq_enter(void)
273{
6378ddb5 274 int cpu = smp_processor_id();
719254fa 275
64db4cff 276 rcu_irq_enter();
ee5f80a9
TG
277 if (idle_cpu(cpu) && !in_interrupt()) {
278 __irq_enter();
719254fa 279 tick_check_idle(cpu);
ee5f80a9
TG
280 } else
281 __irq_enter();
dde4b2b5
IM
282}
283
1da177e4
LT
284#ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
285# define invoke_softirq() __do_softirq()
286#else
287# define invoke_softirq() do_softirq()
288#endif
289
290/*
291 * Exit an interrupt context. Process softirqs if needed and possible:
292 */
293void irq_exit(void)
294{
295 account_system_vtime(current);
de30a2b3 296 trace_hardirq_exit();
1da177e4
LT
297 sub_preempt_count(IRQ_EXIT_OFFSET);
298 if (!in_interrupt() && local_softirq_pending())
299 invoke_softirq();
79bf2bb3
TG
300
301#ifdef CONFIG_NO_HZ
302 /* Make sure that timer wheel updates are propagated */
2232c2d8 303 rcu_irq_exit();
64db4cff
PM
304 if (idle_cpu(smp_processor_id()) && !in_interrupt() && !need_resched())
305 tick_nohz_stop_sched_tick(0);
79bf2bb3 306#endif
1da177e4
LT
307 preempt_enable_no_resched();
308}
309
310/*
311 * This function must run with irqs disabled!
312 */
7ad5b3a5 313inline void raise_softirq_irqoff(unsigned int nr)
1da177e4
LT
314{
315 __raise_softirq_irqoff(nr);
316
317 /*
318 * If we're in an interrupt or softirq, we're done
319 * (this also catches softirq-disabled code). We will
320 * actually run the softirq once we return from
321 * the irq or softirq.
322 *
323 * Otherwise we wake up ksoftirqd to make sure we
324 * schedule the softirq soon.
325 */
326 if (!in_interrupt())
327 wakeup_softirqd();
328}
329
7ad5b3a5 330void raise_softirq(unsigned int nr)
1da177e4
LT
331{
332 unsigned long flags;
333
334 local_irq_save(flags);
335 raise_softirq_irqoff(nr);
336 local_irq_restore(flags);
337}
338
962cf36c 339void open_softirq(int nr, void (*action)(struct softirq_action *))
1da177e4 340{
1da177e4
LT
341 softirq_vec[nr].action = action;
342}
343
1da177e4
LT
344/* Tasklets */
345struct tasklet_head
346{
48f20a9a
OJ
347 struct tasklet_struct *head;
348 struct tasklet_struct **tail;
1da177e4
LT
349};
350
4620b49f
VN
351static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
352static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
1da177e4 353
7ad5b3a5 354void __tasklet_schedule(struct tasklet_struct *t)
1da177e4
LT
355{
356 unsigned long flags;
357
358 local_irq_save(flags);
48f20a9a
OJ
359 t->next = NULL;
360 *__get_cpu_var(tasklet_vec).tail = t;
361 __get_cpu_var(tasklet_vec).tail = &(t->next);
1da177e4
LT
362 raise_softirq_irqoff(TASKLET_SOFTIRQ);
363 local_irq_restore(flags);
364}
365
366EXPORT_SYMBOL(__tasklet_schedule);
367
7ad5b3a5 368void __tasklet_hi_schedule(struct tasklet_struct *t)
1da177e4
LT
369{
370 unsigned long flags;
371
372 local_irq_save(flags);
48f20a9a
OJ
373 t->next = NULL;
374 *__get_cpu_var(tasklet_hi_vec).tail = t;
375 __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
1da177e4
LT
376 raise_softirq_irqoff(HI_SOFTIRQ);
377 local_irq_restore(flags);
378}
379
380EXPORT_SYMBOL(__tasklet_hi_schedule);
381
382static void tasklet_action(struct softirq_action *a)
383{
384 struct tasklet_struct *list;
385
386 local_irq_disable();
48f20a9a
OJ
387 list = __get_cpu_var(tasklet_vec).head;
388 __get_cpu_var(tasklet_vec).head = NULL;
389 __get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
1da177e4
LT
390 local_irq_enable();
391
392 while (list) {
393 struct tasklet_struct *t = list;
394
395 list = list->next;
396
397 if (tasklet_trylock(t)) {
398 if (!atomic_read(&t->count)) {
399 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
400 BUG();
401 t->func(t->data);
402 tasklet_unlock(t);
403 continue;
404 }
405 tasklet_unlock(t);
406 }
407
408 local_irq_disable();
48f20a9a
OJ
409 t->next = NULL;
410 *__get_cpu_var(tasklet_vec).tail = t;
411 __get_cpu_var(tasklet_vec).tail = &(t->next);
1da177e4
LT
412 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
413 local_irq_enable();
414 }
415}
416
417static void tasklet_hi_action(struct softirq_action *a)
418{
419 struct tasklet_struct *list;
420
421 local_irq_disable();
48f20a9a
OJ
422 list = __get_cpu_var(tasklet_hi_vec).head;
423 __get_cpu_var(tasklet_hi_vec).head = NULL;
424 __get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
1da177e4
LT
425 local_irq_enable();
426
427 while (list) {
428 struct tasklet_struct *t = list;
429
430 list = list->next;
431
432 if (tasklet_trylock(t)) {
433 if (!atomic_read(&t->count)) {
434 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
435 BUG();
436 t->func(t->data);
437 tasklet_unlock(t);
438 continue;
439 }
440 tasklet_unlock(t);
441 }
442
443 local_irq_disable();
48f20a9a
OJ
444 t->next = NULL;
445 *__get_cpu_var(tasklet_hi_vec).tail = t;
446 __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
1da177e4
LT
447 __raise_softirq_irqoff(HI_SOFTIRQ);
448 local_irq_enable();
449 }
450}
451
452
453void tasklet_init(struct tasklet_struct *t,
454 void (*func)(unsigned long), unsigned long data)
455{
456 t->next = NULL;
457 t->state = 0;
458 atomic_set(&t->count, 0);
459 t->func = func;
460 t->data = data;
461}
462
463EXPORT_SYMBOL(tasklet_init);
464
465void tasklet_kill(struct tasklet_struct *t)
466{
467 if (in_interrupt())
468 printk("Attempt to kill tasklet from interrupt\n");
469
470 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
471 do
472 yield();
473 while (test_bit(TASKLET_STATE_SCHED, &t->state));
474 }
475 tasklet_unlock_wait(t);
476 clear_bit(TASKLET_STATE_SCHED, &t->state);
477}
478
479EXPORT_SYMBOL(tasklet_kill);
480
54514a70
DM
481DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
482EXPORT_PER_CPU_SYMBOL(softirq_work_list);
483
484static void __local_trigger(struct call_single_data *cp, int softirq)
485{
486 struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
487
488 list_add_tail(&cp->list, head);
489
490 /* Trigger the softirq only if the list was previously empty. */
491 if (head->next == &cp->list)
492 raise_softirq_irqoff(softirq);
493}
494
495#ifdef CONFIG_USE_GENERIC_SMP_HELPERS
496static void remote_softirq_receive(void *data)
497{
498 struct call_single_data *cp = data;
499 unsigned long flags;
500 int softirq;
501
502 softirq = cp->priv;
503
504 local_irq_save(flags);
505 __local_trigger(cp, softirq);
506 local_irq_restore(flags);
507}
508
509static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
510{
511 if (cpu_online(cpu)) {
512 cp->func = remote_softirq_receive;
513 cp->info = cp;
514 cp->flags = 0;
515 cp->priv = softirq;
516
517 __smp_call_function_single(cpu, cp);
518 return 0;
519 }
520 return 1;
521}
522#else /* CONFIG_USE_GENERIC_SMP_HELPERS */
523static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
524{
525 return 1;
526}
527#endif
528
529/**
530 * __send_remote_softirq - try to schedule softirq work on a remote cpu
531 * @cp: private SMP call function data area
532 * @cpu: the remote cpu
533 * @this_cpu: the currently executing cpu
534 * @softirq: the softirq for the work
535 *
536 * Attempt to schedule softirq work on a remote cpu. If this cannot be
537 * done, the work is instead queued up on the local cpu.
538 *
539 * Interrupts must be disabled.
540 */
541void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
542{
543 if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
544 __local_trigger(cp, softirq);
545}
546EXPORT_SYMBOL(__send_remote_softirq);
547
548/**
549 * send_remote_softirq - try to schedule softirq work on a remote cpu
550 * @cp: private SMP call function data area
551 * @cpu: the remote cpu
552 * @softirq: the softirq for the work
553 *
554 * Like __send_remote_softirq except that disabling interrupts and
555 * computing the current cpu is done for the caller.
556 */
557void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
558{
559 unsigned long flags;
560 int this_cpu;
561
562 local_irq_save(flags);
563 this_cpu = smp_processor_id();
564 __send_remote_softirq(cp, cpu, this_cpu, softirq);
565 local_irq_restore(flags);
566}
567EXPORT_SYMBOL(send_remote_softirq);
568
569static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
570 unsigned long action, void *hcpu)
571{
572 /*
573 * If a CPU goes away, splice its entries to the current CPU
574 * and trigger a run of the softirq
575 */
576 if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
577 int cpu = (unsigned long) hcpu;
578 int i;
579
580 local_irq_disable();
581 for (i = 0; i < NR_SOFTIRQS; i++) {
582 struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
583 struct list_head *local_head;
584
585 if (list_empty(head))
586 continue;
587
588 local_head = &__get_cpu_var(softirq_work_list[i]);
589 list_splice_init(head, local_head);
590 raise_softirq_irqoff(i);
591 }
592 local_irq_enable();
593 }
594
595 return NOTIFY_OK;
596}
597
598static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
599 .notifier_call = remote_softirq_cpu_notify,
600};
601
1da177e4
LT
602void __init softirq_init(void)
603{
48f20a9a
OJ
604 int cpu;
605
606 for_each_possible_cpu(cpu) {
54514a70
DM
607 int i;
608
48f20a9a
OJ
609 per_cpu(tasklet_vec, cpu).tail =
610 &per_cpu(tasklet_vec, cpu).head;
611 per_cpu(tasklet_hi_vec, cpu).tail =
612 &per_cpu(tasklet_hi_vec, cpu).head;
54514a70
DM
613 for (i = 0; i < NR_SOFTIRQS; i++)
614 INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
48f20a9a
OJ
615 }
616
54514a70
DM
617 register_hotcpu_notifier(&remote_softirq_cpu_notifier);
618
962cf36c
CM
619 open_softirq(TASKLET_SOFTIRQ, tasklet_action);
620 open_softirq(HI_SOFTIRQ, tasklet_hi_action);
1da177e4
LT
621}
622
623static int ksoftirqd(void * __bind_cpu)
624{
1da177e4
LT
625 set_current_state(TASK_INTERRUPTIBLE);
626
627 while (!kthread_should_stop()) {
628 preempt_disable();
629 if (!local_softirq_pending()) {
630 preempt_enable_no_resched();
631 schedule();
632 preempt_disable();
633 }
634
635 __set_current_state(TASK_RUNNING);
636
637 while (local_softirq_pending()) {
638 /* Preempt disable stops cpu going offline.
639 If already offline, we'll be on wrong CPU:
640 don't process */
641 if (cpu_is_offline((long)__bind_cpu))
642 goto wait_to_die;
643 do_softirq();
644 preempt_enable_no_resched();
645 cond_resched();
646 preempt_disable();
64ca5ab9 647 rcu_qsctr_inc((long)__bind_cpu);
1da177e4
LT
648 }
649 preempt_enable();
650 set_current_state(TASK_INTERRUPTIBLE);
651 }
652 __set_current_state(TASK_RUNNING);
653 return 0;
654
655wait_to_die:
656 preempt_enable();
657 /* Wait for kthread_stop */
658 set_current_state(TASK_INTERRUPTIBLE);
659 while (!kthread_should_stop()) {
660 schedule();
661 set_current_state(TASK_INTERRUPTIBLE);
662 }
663 __set_current_state(TASK_RUNNING);
664 return 0;
665}
666
667#ifdef CONFIG_HOTPLUG_CPU
668/*
669 * tasklet_kill_immediate is called to remove a tasklet which can already be
670 * scheduled for execution on @cpu.
671 *
672 * Unlike tasklet_kill, this function removes the tasklet
673 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
674 *
675 * When this function is called, @cpu must be in the CPU_DEAD state.
676 */
677void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
678{
679 struct tasklet_struct **i;
680
681 BUG_ON(cpu_online(cpu));
682 BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
683
684 if (!test_bit(TASKLET_STATE_SCHED, &t->state))
685 return;
686
687 /* CPU is dead, so no lock needed. */
48f20a9a 688 for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
1da177e4
LT
689 if (*i == t) {
690 *i = t->next;
48f20a9a
OJ
691 /* If this was the tail element, move the tail ptr */
692 if (*i == NULL)
693 per_cpu(tasklet_vec, cpu).tail = i;
1da177e4
LT
694 return;
695 }
696 }
697 BUG();
698}
699
700static void takeover_tasklets(unsigned int cpu)
701{
1da177e4
LT
702 /* CPU is dead, so no lock needed. */
703 local_irq_disable();
704
705 /* Find end, append list for that CPU. */
e5e41723
CB
706 if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
707 *(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
708 __get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
709 per_cpu(tasklet_vec, cpu).head = NULL;
710 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
711 }
1da177e4
LT
712 raise_softirq_irqoff(TASKLET_SOFTIRQ);
713
e5e41723
CB
714 if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
715 *__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
716 __get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
717 per_cpu(tasklet_hi_vec, cpu).head = NULL;
718 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
719 }
1da177e4
LT
720 raise_softirq_irqoff(HI_SOFTIRQ);
721
722 local_irq_enable();
723}
724#endif /* CONFIG_HOTPLUG_CPU */
725
8c78f307 726static int __cpuinit cpu_callback(struct notifier_block *nfb,
1da177e4
LT
727 unsigned long action,
728 void *hcpu)
729{
730 int hotcpu = (unsigned long)hcpu;
731 struct task_struct *p;
732
733 switch (action) {
734 case CPU_UP_PREPARE:
8bb78442 735 case CPU_UP_PREPARE_FROZEN:
1da177e4
LT
736 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
737 if (IS_ERR(p)) {
738 printk("ksoftirqd for %i failed\n", hotcpu);
739 return NOTIFY_BAD;
740 }
741 kthread_bind(p, hotcpu);
742 per_cpu(ksoftirqd, hotcpu) = p;
743 break;
744 case CPU_ONLINE:
8bb78442 745 case CPU_ONLINE_FROZEN:
1da177e4
LT
746 wake_up_process(per_cpu(ksoftirqd, hotcpu));
747 break;
748#ifdef CONFIG_HOTPLUG_CPU
749 case CPU_UP_CANCELED:
8bb78442 750 case CPU_UP_CANCELED_FROZEN:
fc75cdfa
HC
751 if (!per_cpu(ksoftirqd, hotcpu))
752 break;
1da177e4 753 /* Unbind so it can run. Fall thru. */
a4c4af7c 754 kthread_bind(per_cpu(ksoftirqd, hotcpu),
f1fc057c 755 cpumask_any(cpu_online_mask));
1da177e4 756 case CPU_DEAD:
1c6b4aa9
ST
757 case CPU_DEAD_FROZEN: {
758 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
759
1da177e4
LT
760 p = per_cpu(ksoftirqd, hotcpu);
761 per_cpu(ksoftirqd, hotcpu) = NULL;
961ccddd 762 sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
1da177e4
LT
763 kthread_stop(p);
764 takeover_tasklets(hotcpu);
765 break;
1c6b4aa9 766 }
1da177e4
LT
767#endif /* CONFIG_HOTPLUG_CPU */
768 }
769 return NOTIFY_OK;
770}
771
8c78f307 772static struct notifier_block __cpuinitdata cpu_nfb = {
1da177e4
LT
773 .notifier_call = cpu_callback
774};
775
7babe8db 776static __init int spawn_ksoftirqd(void)
1da177e4
LT
777{
778 void *cpu = (void *)(long)smp_processor_id();
07dccf33
AM
779 int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
780
781 BUG_ON(err == NOTIFY_BAD);
1da177e4
LT
782 cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
783 register_cpu_notifier(&cpu_nfb);
784 return 0;
785}
7babe8db 786early_initcall(spawn_ksoftirqd);
78eef01b
AM
787
788#ifdef CONFIG_SMP
789/*
790 * Call a function on all processors
791 */
15c8b6c1 792int on_each_cpu(void (*func) (void *info), void *info, int wait)
78eef01b
AM
793{
794 int ret = 0;
795
796 preempt_disable();
8691e5a8 797 ret = smp_call_function(func, info, wait);
78eef01b
AM
798 local_irq_disable();
799 func(info);
800 local_irq_enable();
801 preempt_enable();
802 return ret;
803}
804EXPORT_SYMBOL(on_each_cpu);
805#endif
43a25632
YL
806
807/*
808 * [ These __weak aliases are kept in a separate compilation unit, so that
809 * GCC does not inline them incorrectly. ]
810 */
811
812int __init __weak early_irq_init(void)
813{
814 return 0;
815}
816
4a046d17
YL
817int __init __weak arch_probe_nr_irqs(void)
818{
819 return 0;
820}
821
43a25632
YL
822int __init __weak arch_early_irq_init(void)
823{
824 return 0;
825}
826
827int __weak arch_init_chip_data(struct irq_desc *desc, int cpu)
828{
829 return 0;
830}