kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  *      Distribute under GPLv2.
   7  *
   8  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   9  *
  10  *      Remote softirq infrastructure is by Jens Axboe.
  11  */
  12
  13 #include <linux/module.h>
  14 #include <linux/kernel_stat.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/init.h>
  17 #include <linux/mm.h>
  18 #include <linux/notifier.h>
  19 #include <linux/percpu.h>
  20 #include <linux/cpu.h>
  21 #include <linux/freezer.h>
  22 #include <linux/kthread.h>
  23 #include <linux/rcupdate.h>
  24 #include <linux/ftrace.h>
  25 #include <linux/smp.h>
  26 #include <linux/tick.h>
  27
  28 #include <asm/irq.h>
  29 /*
  30    - No shared variables, all the data are CPU local.
  31    - If a softirq needs serialization, let it serialize itself
  32      by its own spinlocks.
  33    - Even if softirq is serialized, only local cpu is marked for
  34      execution. Hence, we get something sort of weak cpu binding.
  35      Though it is still not clear, will it result in better locality
  36      or will not.
  37
  38    Examples:
  39    - NET RX softirq. It is multithreaded and does not require
  40      any global serialization.
  41    - NET TX softirq. It kicks software netdevice queues, hence
  42      it is logically serialized per device, but this serialization
  43      is invisible to common code.
  44    - Tasklets: serialized wrt itself.
  45  */
  46
  47 #ifndef __ARCH_IRQ_STAT
  48 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  49 EXPORT_SYMBOL(irq_stat);
  50 #endif
  51
  52 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
  53
  54 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  55
  56 /*
  57  * we cannot loop indefinitely here to avoid userspace starvation,
  58  * but we also don't want to introduce a worst case 1/HZ latency
  59  * to the pending events, so lets the scheduler to balance
  60  * the softirq load for us.
  61  */
  62 static inline void wakeup_softirqd(void)
  63 {
  64         /* Interrupts are disabled: no need to stop preemption */
  65         struct task_struct *tsk = __get_cpu_var(ksoftirqd);
  66
  67         if (tsk && tsk->state != TASK_RUNNING)
  68                 wake_up_process(tsk);
  69 }
  70
  71 /*
  72  * This one is for softirq.c-internal use,
  73  * where hardirqs are disabled legitimately:
  74  */
  75 #ifdef CONFIG_TRACE_IRQFLAGS
  76 static void __local_bh_disable(unsigned long ip)
  77 {
  78         unsigned long flags;
  79
  80         WARN_ON_ONCE(in_irq());
  81
  82         raw_local_irq_save(flags);
  83         /*
  84          * The preempt tracer hooks into add_preempt_count and will break
  85          * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
  86          * is set and before current->softirq_enabled is cleared.
  87          * We must manually increment preempt_count here and manually
  88          * call the trace_preempt_off later.
  89          */
  90         preempt_count() += SOFTIRQ_OFFSET;
  91         /*
  92          * Were softirqs turned off above:
  93          */
  94         if (softirq_count() == SOFTIRQ_OFFSET)
  95                 trace_softirqs_off(ip);
  96         raw_local_irq_restore(flags);
  97
  98         if (preempt_count() == SOFTIRQ_OFFSET)
  99                 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
 100 }
 101 #else /* !CONFIG_TRACE_IRQFLAGS */
 102 static inline void __local_bh_disable(unsigned long ip)
 103 {
 104         add_preempt_count(SOFTIRQ_OFFSET);
 105         barrier();
 106 }
 107 #endif /* CONFIG_TRACE_IRQFLAGS */
 108
 109 void local_bh_disable(void)
 110 {
 111         __local_bh_disable((unsigned long)__builtin_return_address(0));
 112 }
 113
 114 EXPORT_SYMBOL(local_bh_disable);
 115
 116 /*
 117  * Special-case - softirqs can safely be enabled in
 118  * cond_resched_softirq(), or by __do_softirq(),
 119  * without processing still-pending softirqs:
 120  */
 121 void _local_bh_enable(void)
 122 {
 123         WARN_ON_ONCE(in_irq());
 124         WARN_ON_ONCE(!irqs_disabled());
 125
 126         if (softirq_count() == SOFTIRQ_OFFSET)
 127                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
 128         sub_preempt_count(SOFTIRQ_OFFSET);
 129 }
 130
 131 EXPORT_SYMBOL(_local_bh_enable);
 132
 133 static inline void _local_bh_enable_ip(unsigned long ip)
 134 {
 135         WARN_ON_ONCE(in_irq() || irqs_disabled());
 136 #ifdef CONFIG_TRACE_IRQFLAGS
 137         local_irq_disable();
 138 #endif
 139         /*
 140          * Are softirqs going to be turned on now:
 141          */
 142         if (softirq_count() == SOFTIRQ_OFFSET)
 143                 trace_softirqs_on(ip);
 144         /*
 145          * Keep preemption disabled until we are done with
 146          * softirq processing:
 147          */
 148         sub_preempt_count(SOFTIRQ_OFFSET - 1);
 149
 150         if (unlikely(!in_interrupt() && local_softirq_pending()))
 151                 do_softirq();
 152
 153         dec_preempt_count();
 154 #ifdef CONFIG_TRACE_IRQFLAGS
 155         local_irq_enable();
 156 #endif
 157         preempt_check_resched();
 158 }
 159
 160 void local_bh_enable(void)
 161 {
 162         _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 163 }
 164 EXPORT_SYMBOL(local_bh_enable);
 165
 166 void local_bh_enable_ip(unsigned long ip)
 167 {
 168         _local_bh_enable_ip(ip);
 169 }
 170 EXPORT_SYMBOL(local_bh_enable_ip);
 171
 172 /*
 173  * We restart softirq processing MAX_SOFTIRQ_RESTART times,
 174  * and we fall back to softirqd after that.
 175  *
 176  * This number has been established via experimentation.
 177  * The two things to balance is latency against fairness -
 178  * we want to handle softirqs as soon as possible, but they
 179  * should not be able to lock up the box.
 180  */
 181 #define MAX_SOFTIRQ_RESTART 10
 182
 183 asmlinkage void __do_softirq(void)
 184 {
 185         struct softirq_action *h;
 186         __u32 pending;
 187         int max_restart = MAX_SOFTIRQ_RESTART;
 188         int cpu;
 189
 190         pending = local_softirq_pending();
 191         account_system_vtime(current);
 192
 193         __local_bh_disable((unsigned long)__builtin_return_address(0));
 194         trace_softirq_enter();
 195
 196         cpu = smp_processor_id();
 197 restart:
 198         /* Reset the pending bitmask before enabling irqs */
 199         set_softirq_pending(0);
 200
 201         local_irq_enable();
 202
 203         h = softirq_vec;
 204
 205         do {
 206                 if (pending & 1) {
 207                         int prev_count = preempt_count();
 208
 209                         h->action(h);
 210
 211                         if (unlikely(prev_count != preempt_count())) {
 212                                 printk(KERN_ERR "huh, entered softirq %td %p"
 213                                        "with preempt_count %08x,"
 214                                        " exited with %08x?\n", h - softirq_vec,
 215                                        h->action, prev_count, preempt_count());
 216                                 preempt_count() = prev_count;
 217                         }
 218
 219                         rcu_bh_qsctr_inc(cpu);
 220                 }
 221                 h++;
 222                 pending >>= 1;
 223         } while (pending);
 224
 225         local_irq_disable();
 226
 227         pending = local_softirq_pending();
 228         if (pending && --max_restart)
 229                 goto restart;
 230
 231         if (pending)
 232                 wakeup_softirqd();
 233
 234         trace_softirq_exit();
 235
 236         account_system_vtime(current);
 237         _local_bh_enable();
 238 }
 239
 240 #ifndef __ARCH_HAS_DO_SOFTIRQ
 241
 242 asmlinkage void do_softirq(void)
 243 {
 244         __u32 pending;
 245         unsigned long flags;
 246
 247         if (in_interrupt())
 248                 return;
 249
 250         local_irq_save(flags);
 251
 252         pending = local_softirq_pending();
 253
 254         if (pending)
 255                 __do_softirq();
 256
 257         local_irq_restore(flags);
 258 }
 259
 260 #endif
 261
 262 /*
 263  * Enter an interrupt context.
 264  */
 265 void irq_enter(void)
 266 {
 267         int cpu = smp_processor_id();
 268
 269         rcu_irq_enter();
 270         if (idle_cpu(cpu) && !in_interrupt()) {
 271                 __irq_enter();
 272                 tick_check_idle(cpu);
 273         } else
 274                 __irq_enter();
 275 }
 276
 277 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
 278 # define invoke_softirq()       __do_softirq()
 279 #else
 280 # define invoke_softirq()       do_softirq()
 281 #endif
 282
 283 /*
 284  * Exit an interrupt context. Process softirqs if needed and possible:
 285  */
 286 void irq_exit(void)
 287 {
 288         account_system_vtime(current);
 289         trace_hardirq_exit();
 290         sub_preempt_count(IRQ_EXIT_OFFSET);
 291         if (!in_interrupt() && local_softirq_pending())
 292                 invoke_softirq();
 293
 294 #ifdef CONFIG_NO_HZ
 295         /* Make sure that timer wheel updates are propagated */
 296         rcu_irq_exit();
 297         if (idle_cpu(smp_processor_id()) && !in_interrupt() && !need_resched())
 298                 tick_nohz_stop_sched_tick(0);
 299 #endif
 300         preempt_enable_no_resched();
 301 }
 302
 303 /*
 304  * This function must run with irqs disabled!
 305  */
 306 inline void raise_softirq_irqoff(unsigned int nr)
 307 {
 308         __raise_softirq_irqoff(nr);
 309
 310         /*
 311          * If we're in an interrupt or softirq, we're done
 312          * (this also catches softirq-disabled code). We will
 313          * actually run the softirq once we return from
 314          * the irq or softirq.
 315          *
 316          * Otherwise we wake up ksoftirqd to make sure we
 317          * schedule the softirq soon.
 318          */
 319         if (!in_interrupt())
 320                 wakeup_softirqd();
 321 }
 322
 323 void raise_softirq(unsigned int nr)
 324 {
 325         unsigned long flags;
 326
 327         local_irq_save(flags);
 328         raise_softirq_irqoff(nr);
 329         local_irq_restore(flags);
 330 }
 331
 332 void open_softirq(int nr, void (*action)(struct softirq_action *))
 333 {
 334         softirq_vec[nr].action = action;
 335 }
 336
 337 /* Tasklets */
 338 struct tasklet_head
 339 {
 340         struct tasklet_struct *head;
 341         struct tasklet_struct **tail;
 342 };
 343
 344 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 345 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 346
 347 void __tasklet_schedule(struct tasklet_struct *t)
 348 {
 349         unsigned long flags;
 350
 351         local_irq_save(flags);
 352         t->next = NULL;
 353         *__get_cpu_var(tasklet_vec).tail = t;
 354         __get_cpu_var(tasklet_vec).tail = &(t->next);
 355         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 356         local_irq_restore(flags);
 357 }
 358
 359 EXPORT_SYMBOL(__tasklet_schedule);
 360
 361 void __tasklet_hi_schedule(struct tasklet_struct *t)
 362 {
 363         unsigned long flags;
 364
 365         local_irq_save(flags);
 366         t->next = NULL;
 367         *__get_cpu_var(tasklet_hi_vec).tail = t;
 368         __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
 369         raise_softirq_irqoff(HI_SOFTIRQ);
 370         local_irq_restore(flags);
 371 }
 372
 373 EXPORT_SYMBOL(__tasklet_hi_schedule);
 374
 375 static void tasklet_action(struct softirq_action *a)
 376 {
 377         struct tasklet_struct *list;
 378
 379         local_irq_disable();
 380         list = __get_cpu_var(tasklet_vec).head;
 381         __get_cpu_var(tasklet_vec).head = NULL;
 382         __get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
 383         local_irq_enable();
 384
 385         while (list) {
 386                 struct tasklet_struct *t = list;
 387
 388                 list = list->next;
 389
 390                 if (tasklet_trylock(t)) {
 391                         if (!atomic_read(&t->count)) {
 392                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 393                                         BUG();
 394                                 t->func(t->data);
 395                                 tasklet_unlock(t);
 396                                 continue;
 397                         }
 398                         tasklet_unlock(t);
 399                 }
 400
 401                 local_irq_disable();
 402                 t->next = NULL;
 403                 *__get_cpu_var(tasklet_vec).tail = t;
 404                 __get_cpu_var(tasklet_vec).tail = &(t->next);
 405                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 406                 local_irq_enable();
 407         }
 408 }
 409
 410 static void tasklet_hi_action(struct softirq_action *a)
 411 {
 412         struct tasklet_struct *list;
 413
 414         local_irq_disable();
 415         list = __get_cpu_var(tasklet_hi_vec).head;
 416         __get_cpu_var(tasklet_hi_vec).head = NULL;
 417         __get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
 418         local_irq_enable();
 419
 420         while (list) {
 421                 struct tasklet_struct *t = list;
 422
 423                 list = list->next;
 424
 425                 if (tasklet_trylock(t)) {
 426                         if (!atomic_read(&t->count)) {
 427                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 428                                         BUG();
 429                                 t->func(t->data);
 430                                 tasklet_unlock(t);
 431                                 continue;
 432                         }
 433                         tasklet_unlock(t);
 434                 }
 435
 436                 local_irq_disable();
 437                 t->next = NULL;
 438                 *__get_cpu_var(tasklet_hi_vec).tail = t;
 439                 __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
 440                 __raise_softirq_irqoff(HI_SOFTIRQ);
 441                 local_irq_enable();
 442         }
 443 }
 444
 445
 446 void tasklet_init(struct tasklet_struct *t,
 447                   void (*func)(unsigned long), unsigned long data)
 448 {
 449         t->next = NULL;
 450         t->state = 0;
 451         atomic_set(&t->count, 0);
 452         t->func = func;
 453         t->data = data;
 454 }
 455
 456 EXPORT_SYMBOL(tasklet_init);
 457
 458 void tasklet_kill(struct tasklet_struct *t)
 459 {
 460         if (in_interrupt())
 461                 printk("Attempt to kill tasklet from interrupt\n");
 462
 463         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 464                 do
 465                         yield();
 466                 while (test_bit(TASKLET_STATE_SCHED, &t->state));
 467         }
 468         tasklet_unlock_wait(t);
 469         clear_bit(TASKLET_STATE_SCHED, &t->state);
 470 }
 471
 472 EXPORT_SYMBOL(tasklet_kill);
 473
 474 DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
 475 EXPORT_PER_CPU_SYMBOL(softirq_work_list);
 476
 477 static void __local_trigger(struct call_single_data *cp, int softirq)
 478 {
 479         struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
 480
 481         list_add_tail(&cp->list, head);
 482
 483         /* Trigger the softirq only if the list was previously empty.  */
 484         if (head->next == &cp->list)
 485                 raise_softirq_irqoff(softirq);
 486 }
 487
 488 #ifdef CONFIG_USE_GENERIC_SMP_HELPERS
 489 static void remote_softirq_receive(void *data)
 490 {
 491         struct call_single_data *cp = data;
 492         unsigned long flags;
 493         int softirq;
 494
 495         softirq = cp->priv;
 496
 497         local_irq_save(flags);
 498         __local_trigger(cp, softirq);
 499         local_irq_restore(flags);
 500 }
 501
 502 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 503 {
 504         if (cpu_online(cpu)) {
 505                 cp->func = remote_softirq_receive;
 506                 cp->info = cp;
 507                 cp->flags = 0;
 508                 cp->priv = softirq;
 509
 510                 __smp_call_function_single(cpu, cp);
 511                 return 0;
 512         }
 513         return 1;
 514 }
 515 #else /* CONFIG_USE_GENERIC_SMP_HELPERS */
 516 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 517 {
 518         return 1;
 519 }
 520 #endif
 521
 522 /**
 523  * __send_remote_softirq - try to schedule softirq work on a remote cpu
 524  * @cp: private SMP call function data area
 525  * @cpu: the remote cpu
 526  * @this_cpu: the currently executing cpu
 527  * @softirq: the softirq for the work
 528  *
 529  * Attempt to schedule softirq work on a remote cpu.  If this cannot be
 530  * done, the work is instead queued up on the local cpu.
 531  *
 532  * Interrupts must be disabled.
 533  */
 534 void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
 535 {
 536         if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
 537                 __local_trigger(cp, softirq);
 538 }
 539 EXPORT_SYMBOL(__send_remote_softirq);
 540
 541 /**
 542  * send_remote_softirq - try to schedule softirq work on a remote cpu
 543  * @cp: private SMP call function data area
 544  * @cpu: the remote cpu
 545  * @softirq: the softirq for the work
 546  *
 547  * Like __send_remote_softirq except that disabling interrupts and
 548  * computing the current cpu is done for the caller.
 549  */
 550 void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 551 {
 552         unsigned long flags;
 553         int this_cpu;
 554
 555         local_irq_save(flags);
 556         this_cpu = smp_processor_id();
 557         __send_remote_softirq(cp, cpu, this_cpu, softirq);
 558         local_irq_restore(flags);
 559 }
 560 EXPORT_SYMBOL(send_remote_softirq);
 561
 562 static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
 563                                                unsigned long action, void *hcpu)
 564 {
 565         /*
 566          * If a CPU goes away, splice its entries to the current CPU
 567          * and trigger a run of the softirq
 568          */
 569         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
 570                 int cpu = (unsigned long) hcpu;
 571                 int i;
 572
 573                 local_irq_disable();
 574                 for (i = 0; i < NR_SOFTIRQS; i++) {
 575                         struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
 576                         struct list_head *local_head;
 577
 578                         if (list_empty(head))
 579                                 continue;
 580
 581                         local_head = &__get_cpu_var(softirq_work_list[i]);
 582                         list_splice_init(head, local_head);
 583                         raise_softirq_irqoff(i);
 584                 }
 585                 local_irq_enable();
 586         }
 587
 588         return NOTIFY_OK;
 589 }
 590
 591 static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
 592         .notifier_call  = remote_softirq_cpu_notify,
 593 };
 594
 595 void __init softirq_init(void)
 596 {
 597         int cpu;
 598
 599         for_each_possible_cpu(cpu) {
 600                 int i;
 601
 602                 per_cpu(tasklet_vec, cpu).tail =
 603                         &per_cpu(tasklet_vec, cpu).head;
 604                 per_cpu(tasklet_hi_vec, cpu).tail =
 605                         &per_cpu(tasklet_hi_vec, cpu).head;
 606                 for (i = 0; i < NR_SOFTIRQS; i++)
 607                         INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
 608         }
 609
 610         register_hotcpu_notifier(&remote_softirq_cpu_notifier);
 611
 612         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
 613         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 614 }
 615
 616 static int ksoftirqd(void * __bind_cpu)
 617 {
 618         set_current_state(TASK_INTERRUPTIBLE);
 619
 620         while (!kthread_should_stop()) {
 621                 preempt_disable();
 622                 if (!local_softirq_pending()) {
 623                         preempt_enable_no_resched();
 624                         schedule();
 625                         preempt_disable();
 626                 }
 627
 628                 __set_current_state(TASK_RUNNING);
 629
 630                 while (local_softirq_pending()) {
 631                         /* Preempt disable stops cpu going offline.
 632                            If already offline, we'll be on wrong CPU:
 633                            don't process */
 634                         if (cpu_is_offline((long)__bind_cpu))
 635                                 goto wait_to_die;
 636                         do_softirq();
 637                         preempt_enable_no_resched();
 638                         cond_resched();
 639                         preempt_disable();
 640                 }
 641                 preempt_enable();
 642                 set_current_state(TASK_INTERRUPTIBLE);
 643         }
 644         __set_current_state(TASK_RUNNING);
 645         return 0;
 646
 647 wait_to_die:
 648         preempt_enable();
 649         /* Wait for kthread_stop */
 650         set_current_state(TASK_INTERRUPTIBLE);
 651         while (!kthread_should_stop()) {
 652                 schedule();
 653                 set_current_state(TASK_INTERRUPTIBLE);
 654         }
 655         __set_current_state(TASK_RUNNING);
 656         return 0;
 657 }
 658
 659 #ifdef CONFIG_HOTPLUG_CPU
 660 /*
 661  * tasklet_kill_immediate is called to remove a tasklet which can already be
 662  * scheduled for execution on @cpu.
 663  *
 664  * Unlike tasklet_kill, this function removes the tasklet
 665  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 666  *
 667  * When this function is called, @cpu must be in the CPU_DEAD state.
 668  */
 669 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 670 {
 671         struct tasklet_struct **i;
 672
 673         BUG_ON(cpu_online(cpu));
 674         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 675
 676         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 677                 return;
 678
 679         /* CPU is dead, so no lock needed. */
 680         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
 681                 if (*i == t) {
 682                         *i = t->next;
 683                         /* If this was the tail element, move the tail ptr */
 684                         if (*i == NULL)
 685                                 per_cpu(tasklet_vec, cpu).tail = i;
 686                         return;
 687                 }
 688         }
 689         BUG();
 690 }
 691
 692 static void takeover_tasklets(unsigned int cpu)
 693 {
 694         /* CPU is dead, so no lock needed. */
 695         local_irq_disable();
 696
 697         /* Find end, append list for that CPU. */
 698         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
 699                 *(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
 700                 __get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
 701                 per_cpu(tasklet_vec, cpu).head = NULL;
 702                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
 703         }
 704         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 705
 706         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
 707                 *__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
 708                 __get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
 709                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
 710                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
 711         }
 712         raise_softirq_irqoff(HI_SOFTIRQ);
 713
 714         local_irq_enable();
 715 }
 716 #endif /* CONFIG_HOTPLUG_CPU */
 717
 718 static int __cpuinit cpu_callback(struct notifier_block *nfb,
 719                                   unsigned long action,
 720                                   void *hcpu)
 721 {
 722         int hotcpu = (unsigned long)hcpu;
 723         struct task_struct *p;
 724
 725         switch (action) {
 726         case CPU_UP_PREPARE:
 727         case CPU_UP_PREPARE_FROZEN:
 728                 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
 729                 if (IS_ERR(p)) {
 730                         printk("ksoftirqd for %i failed\n", hotcpu);
 731                         return NOTIFY_BAD;
 732                 }
 733                 kthread_bind(p, hotcpu);
 734                 per_cpu(ksoftirqd, hotcpu) = p;
 735                 break;
 736         case CPU_ONLINE:
 737         case CPU_ONLINE_FROZEN:
 738                 wake_up_process(per_cpu(ksoftirqd, hotcpu));
 739                 break;
 740 #ifdef CONFIG_HOTPLUG_CPU
 741         case CPU_UP_CANCELED:
 742         case CPU_UP_CANCELED_FROZEN:
 743                 if (!per_cpu(ksoftirqd, hotcpu))
 744                         break;
 745                 /* Unbind so it can run.  Fall thru. */
 746                 kthread_bind(per_cpu(ksoftirqd, hotcpu),
 747                              cpumask_any(cpu_online_mask));
 748         case CPU_DEAD:
 749         case CPU_DEAD_FROZEN: {
 750                 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
 751
 752                 p = per_cpu(ksoftirqd, hotcpu);
 753                 per_cpu(ksoftirqd, hotcpu) = NULL;
 754                 sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
 755                 kthread_stop(p);
 756                 takeover_tasklets(hotcpu);
 757                 break;
 758         }
 759 #endif /* CONFIG_HOTPLUG_CPU */
 760         }
 761         return NOTIFY_OK;
 762 }
 763
 764 static struct notifier_block __cpuinitdata cpu_nfb = {
 765         .notifier_call = cpu_callback
 766 };
 767
 768 static __init int spawn_ksoftirqd(void)
 769 {
 770         void *cpu = (void *)(long)smp_processor_id();
 771         int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
 772
 773         BUG_ON(err == NOTIFY_BAD);
 774         cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
 775         register_cpu_notifier(&cpu_nfb);
 776         return 0;
 777 }
 778 early_initcall(spawn_ksoftirqd);
 779
 780 #ifdef CONFIG_SMP
 781 /*
 782  * Call a function on all processors
 783  */
 784 int on_each_cpu(void (*func) (void *info), void *info, int wait)
 785 {
 786         int ret = 0;
 787
 788         preempt_disable();
 789         ret = smp_call_function(func, info, wait);
 790         local_irq_disable();
 791         func(info);
 792         local_irq_enable();
 793         preempt_enable();
 794         return ret;
 795 }
 796 EXPORT_SYMBOL(on_each_cpu);
 797 #endif
 798
 799 /*
 800  * [ These __weak aliases are kept in a separate compilation unit, so that
 801  *   GCC does not inline them incorrectly. ]
 802  */
 803
 804 int __init __weak early_irq_init(void)
 805 {
 806         return 0;
 807 }
 808
 809 int __init __weak arch_early_irq_init(void)
 810 {
 811         return 0;
 812 }
 813
 814 int __weak arch_init_chip_data(struct irq_desc *desc, int cpu)
 815 {
 816         return 0;
 817 }