kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  *      Distribute under GPLv2.
   7  *
   8  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   9  *
  10  *      Remote softirq infrastructure is by Jens Axboe.
  11  */
  12
  13 #include <linux/module.h>
  14 #include <linux/kernel_stat.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/init.h>
  17 #include <linux/mm.h>
  18 #include <linux/notifier.h>
  19 #include <linux/percpu.h>
  20 #include <linux/cpu.h>
  21 #include <linux/freezer.h>
  22 #include <linux/kthread.h>
  23 #include <linux/rcupdate.h>
  24 #include <linux/smp.h>
  25 #include <linux/tick.h>
  26
  27 #include <asm/irq.h>
  28 /*
  29    - No shared variables, all the data are CPU local.
  30    - If a softirq needs serialization, let it serialize itself
  31      by its own spinlocks.
  32    - Even if softirq is serialized, only local cpu is marked for
  33      execution. Hence, we get something sort of weak cpu binding.
  34      Though it is still not clear, will it result in better locality
  35      or will not.
  36
  37    Examples:
  38    - NET RX softirq. It is multithreaded and does not require
  39      any global serialization.
  40    - NET TX softirq. It kicks software netdevice queues, hence
  41      it is logically serialized per device, but this serialization
  42      is invisible to common code.
  43    - Tasklets: serialized wrt itself.
  44  */
  45
  46 #ifndef __ARCH_IRQ_STAT
  47 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  48 EXPORT_SYMBOL(irq_stat);
  49 #endif
  50
  51 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
  52
  53 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  54
  55 /*
  56  * we cannot loop indefinitely here to avoid userspace starvation,
  57  * but we also don't want to introduce a worst case 1/HZ latency
  58  * to the pending events, so lets the scheduler to balance
  59  * the softirq load for us.
  60  */
  61 static inline void wakeup_softirqd(void)
  62 {
  63         /* Interrupts are disabled: no need to stop preemption */
  64         struct task_struct *tsk = __get_cpu_var(ksoftirqd);
  65
  66         if (tsk && tsk->state != TASK_RUNNING)
  67                 wake_up_process(tsk);
  68 }
  69
  70 /*
  71  * This one is for softirq.c-internal use,
  72  * where hardirqs are disabled legitimately:
  73  */
  74 #ifdef CONFIG_TRACE_IRQFLAGS
  75 static void __local_bh_disable(unsigned long ip)
  76 {
  77         unsigned long flags;
  78
  79         WARN_ON_ONCE(in_irq());
  80
  81         raw_local_irq_save(flags);
  82         add_preempt_count(SOFTIRQ_OFFSET);
  83         /*
  84          * Were softirqs turned off above:
  85          */
  86         if (softirq_count() == SOFTIRQ_OFFSET)
  87                 trace_softirqs_off(ip);
  88         raw_local_irq_restore(flags);
  89 }
  90 #else /* !CONFIG_TRACE_IRQFLAGS */
  91 static inline void __local_bh_disable(unsigned long ip)
  92 {
  93         add_preempt_count(SOFTIRQ_OFFSET);
  94         barrier();
  95 }
  96 #endif /* CONFIG_TRACE_IRQFLAGS */
  97
  98 void local_bh_disable(void)
  99 {
 100         __local_bh_disable((unsigned long)__builtin_return_address(0));
 101 }
 102
 103 EXPORT_SYMBOL(local_bh_disable);
 104
 105 void __local_bh_enable(void)
 106 {
 107         WARN_ON_ONCE(in_irq());
 108
 109         /*
 110          * softirqs should never be enabled by __local_bh_enable(),
 111          * it always nests inside local_bh_enable() sections:
 112          */
 113         WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
 114
 115         sub_preempt_count(SOFTIRQ_OFFSET);
 116 }
 117 EXPORT_SYMBOL_GPL(__local_bh_enable);
 118
 119 /*
 120  * Special-case - softirqs can safely be enabled in
 121  * cond_resched_softirq(), or by __do_softirq(),
 122  * without processing still-pending softirqs:
 123  */
 124 void _local_bh_enable(void)
 125 {
 126         WARN_ON_ONCE(in_irq());
 127         WARN_ON_ONCE(!irqs_disabled());
 128
 129         if (softirq_count() == SOFTIRQ_OFFSET)
 130                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
 131         sub_preempt_count(SOFTIRQ_OFFSET);
 132 }
 133
 134 EXPORT_SYMBOL(_local_bh_enable);
 135
 136 static inline void _local_bh_enable_ip(unsigned long ip)
 137 {
 138         WARN_ON_ONCE(in_irq() || irqs_disabled());
 139 #ifdef CONFIG_TRACE_IRQFLAGS
 140         local_irq_disable();
 141 #endif
 142         /*
 143          * Are softirqs going to be turned on now:
 144          */
 145         if (softirq_count() == SOFTIRQ_OFFSET)
 146                 trace_softirqs_on(ip);
 147         /*
 148          * Keep preemption disabled until we are done with
 149          * softirq processing:
 150          */
 151         sub_preempt_count(SOFTIRQ_OFFSET - 1);
 152
 153         if (unlikely(!in_interrupt() && local_softirq_pending()))
 154                 do_softirq();
 155
 156         dec_preempt_count();
 157 #ifdef CONFIG_TRACE_IRQFLAGS
 158         local_irq_enable();
 159 #endif
 160         preempt_check_resched();
 161 }
 162
 163 void local_bh_enable(void)
 164 {
 165         _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 166 }
 167 EXPORT_SYMBOL(local_bh_enable);
 168
 169 void local_bh_enable_ip(unsigned long ip)
 170 {
 171         _local_bh_enable_ip(ip);
 172 }
 173 EXPORT_SYMBOL(local_bh_enable_ip);
 174
 175 /*
 176  * We restart softirq processing MAX_SOFTIRQ_RESTART times,
 177  * and we fall back to softirqd after that.
 178  *
 179  * This number has been established via experimentation.
 180  * The two things to balance is latency against fairness -
 181  * we want to handle softirqs as soon as possible, but they
 182  * should not be able to lock up the box.
 183  */
 184 #define MAX_SOFTIRQ_RESTART 10
 185
 186 asmlinkage void __do_softirq(void)
 187 {
 188         struct softirq_action *h;
 189         __u32 pending;
 190         int max_restart = MAX_SOFTIRQ_RESTART;
 191         int cpu;
 192
 193         pending = local_softirq_pending();
 194         account_system_vtime(current);
 195
 196         __local_bh_disable((unsigned long)__builtin_return_address(0));
 197         trace_softirq_enter();
 198
 199         cpu = smp_processor_id();
 200 restart:
 201         /* Reset the pending bitmask before enabling irqs */
 202         set_softirq_pending(0);
 203
 204         local_irq_enable();
 205
 206         h = softirq_vec;
 207
 208         do {
 209                 if (pending & 1) {
 210                         int prev_count = preempt_count();
 211
 212                         h->action(h);
 213
 214                         if (unlikely(prev_count != preempt_count())) {
 215                                 printk(KERN_ERR "huh, entered softirq %td %p"
 216                                        "with preempt_count %08x,"
 217                                        " exited with %08x?\n", h - softirq_vec,
 218                                        h->action, prev_count, preempt_count());
 219                                 preempt_count() = prev_count;
 220                         }
 221
 222                         rcu_bh_qsctr_inc(cpu);
 223                 }
 224                 h++;
 225                 pending >>= 1;
 226         } while (pending);
 227
 228         local_irq_disable();
 229
 230         pending = local_softirq_pending();
 231         if (pending && --max_restart)
 232                 goto restart;
 233
 234         if (pending)
 235                 wakeup_softirqd();
 236
 237         trace_softirq_exit();
 238
 239         account_system_vtime(current);
 240         _local_bh_enable();
 241 }
 242
 243 #ifndef __ARCH_HAS_DO_SOFTIRQ
 244
 245 asmlinkage void do_softirq(void)
 246 {
 247         __u32 pending;
 248         unsigned long flags;
 249
 250         if (in_interrupt())
 251                 return;
 252
 253         local_irq_save(flags);
 254
 255         pending = local_softirq_pending();
 256
 257         if (pending)
 258                 __do_softirq();
 259
 260         local_irq_restore(flags);
 261 }
 262
 263 #endif
 264
 265 /*
 266  * Enter an interrupt context.
 267  */
 268 void irq_enter(void)
 269 {
 270         int cpu = smp_processor_id();
 271
 272         if (idle_cpu(cpu) && !in_interrupt())
 273                 tick_check_idle(cpu);
 274
 275         __irq_enter();
 276 }
 277
 278 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
 279 # define invoke_softirq()       __do_softirq()
 280 #else
 281 # define invoke_softirq()       do_softirq()
 282 #endif
 283
 284 /*
 285  * Exit an interrupt context. Process softirqs if needed and possible:
 286  */
 287 void irq_exit(void)
 288 {
 289         account_system_vtime(current);
 290         trace_hardirq_exit();
 291         sub_preempt_count(IRQ_EXIT_OFFSET);
 292         if (!in_interrupt() && local_softirq_pending())
 293                 invoke_softirq();
 294
 295 #ifdef CONFIG_NO_HZ
 296         /* Make sure that timer wheel updates are propagated */
 297         if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
 298                 tick_nohz_stop_sched_tick(0);
 299         rcu_irq_exit();
 300 #endif
 301         preempt_enable_no_resched();
 302 }
 303
 304 /*
 305  * This function must run with irqs disabled!
 306  */
 307 inline void raise_softirq_irqoff(unsigned int nr)
 308 {
 309         __raise_softirq_irqoff(nr);
 310
 311         /*
 312          * If we're in an interrupt or softirq, we're done
 313          * (this also catches softirq-disabled code). We will
 314          * actually run the softirq once we return from
 315          * the irq or softirq.
 316          *
 317          * Otherwise we wake up ksoftirqd to make sure we
 318          * schedule the softirq soon.
 319          */
 320         if (!in_interrupt())
 321                 wakeup_softirqd();
 322 }
 323
 324 void raise_softirq(unsigned int nr)
 325 {
 326         unsigned long flags;
 327
 328         local_irq_save(flags);
 329         raise_softirq_irqoff(nr);
 330         local_irq_restore(flags);
 331 }
 332
 333 void open_softirq(int nr, void (*action)(struct softirq_action *))
 334 {
 335         softirq_vec[nr].action = action;
 336 }
 337
 338 /* Tasklets */
 339 struct tasklet_head
 340 {
 341         struct tasklet_struct *head;
 342         struct tasklet_struct **tail;
 343 };
 344
 345 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 346 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 347
 348 void __tasklet_schedule(struct tasklet_struct *t)
 349 {
 350         unsigned long flags;
 351
 352         local_irq_save(flags);
 353         t->next = NULL;
 354         *__get_cpu_var(tasklet_vec).tail = t;
 355         __get_cpu_var(tasklet_vec).tail = &(t->next);
 356         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 357         local_irq_restore(flags);
 358 }
 359
 360 EXPORT_SYMBOL(__tasklet_schedule);
 361
 362 void __tasklet_hi_schedule(struct tasklet_struct *t)
 363 {
 364         unsigned long flags;
 365
 366         local_irq_save(flags);
 367         t->next = NULL;
 368         *__get_cpu_var(tasklet_hi_vec).tail = t;
 369         __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
 370         raise_softirq_irqoff(HI_SOFTIRQ);
 371         local_irq_restore(flags);
 372 }
 373
 374 EXPORT_SYMBOL(__tasklet_hi_schedule);
 375
 376 static void tasklet_action(struct softirq_action *a)
 377 {
 378         struct tasklet_struct *list;
 379
 380         local_irq_disable();
 381         list = __get_cpu_var(tasklet_vec).head;
 382         __get_cpu_var(tasklet_vec).head = NULL;
 383         __get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
 384         local_irq_enable();
 385
 386         while (list) {
 387                 struct tasklet_struct *t = list;
 388
 389                 list = list->next;
 390
 391                 if (tasklet_trylock(t)) {
 392                         if (!atomic_read(&t->count)) {
 393                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 394                                         BUG();
 395                                 t->func(t->data);
 396                                 tasklet_unlock(t);
 397                                 continue;
 398                         }
 399                         tasklet_unlock(t);
 400                 }
 401
 402                 local_irq_disable();
 403                 t->next = NULL;
 404                 *__get_cpu_var(tasklet_vec).tail = t;
 405                 __get_cpu_var(tasklet_vec).tail = &(t->next);
 406                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 407                 local_irq_enable();
 408         }
 409 }
 410
 411 static void tasklet_hi_action(struct softirq_action *a)
 412 {
 413         struct tasklet_struct *list;
 414
 415         local_irq_disable();
 416         list = __get_cpu_var(tasklet_hi_vec).head;
 417         __get_cpu_var(tasklet_hi_vec).head = NULL;
 418         __get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
 419         local_irq_enable();
 420
 421         while (list) {
 422                 struct tasklet_struct *t = list;
 423
 424                 list = list->next;
 425
 426                 if (tasklet_trylock(t)) {
 427                         if (!atomic_read(&t->count)) {
 428                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 429                                         BUG();
 430                                 t->func(t->data);
 431                                 tasklet_unlock(t);
 432                                 continue;
 433                         }
 434                         tasklet_unlock(t);
 435                 }
 436
 437                 local_irq_disable();
 438                 t->next = NULL;
 439                 *__get_cpu_var(tasklet_hi_vec).tail = t;
 440                 __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
 441                 __raise_softirq_irqoff(HI_SOFTIRQ);
 442                 local_irq_enable();
 443         }
 444 }
 445
 446
 447 void tasklet_init(struct tasklet_struct *t,
 448                   void (*func)(unsigned long), unsigned long data)
 449 {
 450         t->next = NULL;
 451         t->state = 0;
 452         atomic_set(&t->count, 0);
 453         t->func = func;
 454         t->data = data;
 455 }
 456
 457 EXPORT_SYMBOL(tasklet_init);
 458
 459 void tasklet_kill(struct tasklet_struct *t)
 460 {
 461         if (in_interrupt())
 462                 printk("Attempt to kill tasklet from interrupt\n");
 463
 464         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 465                 do
 466                         yield();
 467                 while (test_bit(TASKLET_STATE_SCHED, &t->state));
 468         }
 469         tasklet_unlock_wait(t);
 470         clear_bit(TASKLET_STATE_SCHED, &t->state);
 471 }
 472
 473 EXPORT_SYMBOL(tasklet_kill);
 474
 475 DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
 476 EXPORT_PER_CPU_SYMBOL(softirq_work_list);
 477
 478 static void __local_trigger(struct call_single_data *cp, int softirq)
 479 {
 480         struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
 481
 482         list_add_tail(&cp->list, head);
 483
 484         /* Trigger the softirq only if the list was previously empty.  */
 485         if (head->next == &cp->list)
 486                 raise_softirq_irqoff(softirq);
 487 }
 488
 489 #ifdef CONFIG_USE_GENERIC_SMP_HELPERS
 490 static void remote_softirq_receive(void *data)
 491 {
 492         struct call_single_data *cp = data;
 493         unsigned long flags;
 494         int softirq;
 495
 496         softirq = cp->priv;
 497
 498         local_irq_save(flags);
 499         __local_trigger(cp, softirq);
 500         local_irq_restore(flags);
 501 }
 502
 503 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 504 {
 505         if (cpu_online(cpu)) {
 506                 cp->func = remote_softirq_receive;
 507                 cp->info = cp;
 508                 cp->flags = 0;
 509                 cp->priv = softirq;
 510
 511                 __smp_call_function_single(cpu, cp);
 512                 return 0;
 513         }
 514         return 1;
 515 }
 516 #else /* CONFIG_USE_GENERIC_SMP_HELPERS */
 517 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 518 {
 519         return 1;
 520 }
 521 #endif
 522
 523 /**
 524  * __send_remote_softirq - try to schedule softirq work on a remote cpu
 525  * @cp: private SMP call function data area
 526  * @cpu: the remote cpu
 527  * @this_cpu: the currently executing cpu
 528  * @softirq: the softirq for the work
 529  *
 530  * Attempt to schedule softirq work on a remote cpu.  If this cannot be
 531  * done, the work is instead queued up on the local cpu.
 532  *
 533  * Interrupts must be disabled.
 534  */
 535 void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
 536 {
 537         if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
 538                 __local_trigger(cp, softirq);
 539 }
 540 EXPORT_SYMBOL(__send_remote_softirq);
 541
 542 /**
 543  * send_remote_softirq - try to schedule softirq work on a remote cpu
 544  * @cp: private SMP call function data area
 545  * @cpu: the remote cpu
 546  * @softirq: the softirq for the work
 547  *
 548  * Like __send_remote_softirq except that disabling interrupts and
 549  * computing the current cpu is done for the caller.
 550  */
 551 void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 552 {
 553         unsigned long flags;
 554         int this_cpu;
 555
 556         local_irq_save(flags);
 557         this_cpu = smp_processor_id();
 558         __send_remote_softirq(cp, cpu, this_cpu, softirq);
 559         local_irq_restore(flags);
 560 }
 561 EXPORT_SYMBOL(send_remote_softirq);
 562
 563 static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
 564                                                unsigned long action, void *hcpu)
 565 {
 566         /*
 567          * If a CPU goes away, splice its entries to the current CPU
 568          * and trigger a run of the softirq
 569          */
 570         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
 571                 int cpu = (unsigned long) hcpu;
 572                 int i;
 573
 574                 local_irq_disable();
 575                 for (i = 0; i < NR_SOFTIRQS; i++) {
 576                         struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
 577                         struct list_head *local_head;
 578
 579                         if (list_empty(head))
 580                                 continue;
 581
 582                         local_head = &__get_cpu_var(softirq_work_list[i]);
 583                         list_splice_init(head, local_head);
 584                         raise_softirq_irqoff(i);
 585                 }
 586                 local_irq_enable();
 587         }
 588
 589         return NOTIFY_OK;
 590 }
 591
 592 static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
 593         .notifier_call  = remote_softirq_cpu_notify,
 594 };
 595
 596 void __init softirq_init(void)
 597 {
 598         int cpu;
 599
 600         for_each_possible_cpu(cpu) {
 601                 int i;
 602
 603                 per_cpu(tasklet_vec, cpu).tail =
 604                         &per_cpu(tasklet_vec, cpu).head;
 605                 per_cpu(tasklet_hi_vec, cpu).tail =
 606                         &per_cpu(tasklet_hi_vec, cpu).head;
 607                 for (i = 0; i < NR_SOFTIRQS; i++)
 608                         INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
 609         }
 610
 611         register_hotcpu_notifier(&remote_softirq_cpu_notifier);
 612
 613         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
 614         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 615 }
 616
 617 static int ksoftirqd(void * __bind_cpu)
 618 {
 619         set_current_state(TASK_INTERRUPTIBLE);
 620
 621         while (!kthread_should_stop()) {
 622                 preempt_disable();
 623                 if (!local_softirq_pending()) {
 624                         preempt_enable_no_resched();
 625                         schedule();
 626                         preempt_disable();
 627                 }
 628
 629                 __set_current_state(TASK_RUNNING);
 630
 631                 while (local_softirq_pending()) {
 632                         /* Preempt disable stops cpu going offline.
 633                            If already offline, we'll be on wrong CPU:
 634                            don't process */
 635                         if (cpu_is_offline((long)__bind_cpu))
 636                                 goto wait_to_die;
 637                         do_softirq();
 638                         preempt_enable_no_resched();
 639                         cond_resched();
 640                         preempt_disable();
 641                 }
 642                 preempt_enable();
 643                 set_current_state(TASK_INTERRUPTIBLE);
 644         }
 645         __set_current_state(TASK_RUNNING);
 646         return 0;
 647
 648 wait_to_die:
 649         preempt_enable();
 650         /* Wait for kthread_stop */
 651         set_current_state(TASK_INTERRUPTIBLE);
 652         while (!kthread_should_stop()) {
 653                 schedule();
 654                 set_current_state(TASK_INTERRUPTIBLE);
 655         }
 656         __set_current_state(TASK_RUNNING);
 657         return 0;
 658 }
 659
 660 #ifdef CONFIG_HOTPLUG_CPU
 661 /*
 662  * tasklet_kill_immediate is called to remove a tasklet which can already be
 663  * scheduled for execution on @cpu.
 664  *
 665  * Unlike tasklet_kill, this function removes the tasklet
 666  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 667  *
 668  * When this function is called, @cpu must be in the CPU_DEAD state.
 669  */
 670 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 671 {
 672         struct tasklet_struct **i;
 673
 674         BUG_ON(cpu_online(cpu));
 675         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 676
 677         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 678                 return;
 679
 680         /* CPU is dead, so no lock needed. */
 681         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
 682                 if (*i == t) {
 683                         *i = t->next;
 684                         /* If this was the tail element, move the tail ptr */
 685                         if (*i == NULL)
 686                                 per_cpu(tasklet_vec, cpu).tail = i;
 687                         return;
 688                 }
 689         }
 690         BUG();
 691 }
 692
 693 static void takeover_tasklets(unsigned int cpu)
 694 {
 695         /* CPU is dead, so no lock needed. */
 696         local_irq_disable();
 697
 698         /* Find end, append list for that CPU. */
 699         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
 700                 *(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
 701                 __get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
 702                 per_cpu(tasklet_vec, cpu).head = NULL;
 703                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
 704         }
 705         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 706
 707         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
 708                 *__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
 709                 __get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
 710                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
 711                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
 712         }
 713         raise_softirq_irqoff(HI_SOFTIRQ);
 714
 715         local_irq_enable();
 716 }
 717 #endif /* CONFIG_HOTPLUG_CPU */
 718
 719 static int __cpuinit cpu_callback(struct notifier_block *nfb,
 720                                   unsigned long action,
 721                                   void *hcpu)
 722 {
 723         int hotcpu = (unsigned long)hcpu;
 724         struct task_struct *p;
 725
 726         switch (action) {
 727         case CPU_UP_PREPARE:
 728         case CPU_UP_PREPARE_FROZEN:
 729                 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
 730                 if (IS_ERR(p)) {
 731                         printk("ksoftirqd for %i failed\n", hotcpu);
 732                         return NOTIFY_BAD;
 733                 }
 734                 kthread_bind(p, hotcpu);
 735                 per_cpu(ksoftirqd, hotcpu) = p;
 736                 break;
 737         case CPU_ONLINE:
 738         case CPU_ONLINE_FROZEN:
 739                 wake_up_process(per_cpu(ksoftirqd, hotcpu));
 740                 break;
 741 #ifdef CONFIG_HOTPLUG_CPU
 742         case CPU_UP_CANCELED:
 743         case CPU_UP_CANCELED_FROZEN:
 744                 if (!per_cpu(ksoftirqd, hotcpu))
 745                         break;
 746                 /* Unbind so it can run.  Fall thru. */
 747                 kthread_bind(per_cpu(ksoftirqd, hotcpu),
 748                              any_online_cpu(cpu_online_map));
 749         case CPU_DEAD:
 750         case CPU_DEAD_FROZEN: {
 751                 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
 752
 753                 p = per_cpu(ksoftirqd, hotcpu);
 754                 per_cpu(ksoftirqd, hotcpu) = NULL;
 755                 sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
 756                 kthread_stop(p);
 757                 takeover_tasklets(hotcpu);
 758                 break;
 759         }
 760 #endif /* CONFIG_HOTPLUG_CPU */
 761         }
 762         return NOTIFY_OK;
 763 }
 764
 765 static struct notifier_block __cpuinitdata cpu_nfb = {
 766         .notifier_call = cpu_callback
 767 };
 768
 769 static __init int spawn_ksoftirqd(void)
 770 {
 771         void *cpu = (void *)(long)smp_processor_id();
 772         int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
 773
 774         BUG_ON(err == NOTIFY_BAD);
 775         cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
 776         register_cpu_notifier(&cpu_nfb);
 777         return 0;
 778 }
 779 early_initcall(spawn_ksoftirqd);
 780
 781 #ifdef CONFIG_SMP
 782 /*
 783  * Call a function on all processors
 784  */
 785 int on_each_cpu(void (*func) (void *info), void *info, int wait)
 786 {
 787         int ret = 0;
 788
 789         preempt_disable();
 790         ret = smp_call_function(func, info, wait);
 791         local_irq_disable();
 792         func(info);
 793         local_irq_enable();
 794         preempt_enable();
 795         return ret;
 796 }
 797 EXPORT_SYMBOL(on_each_cpu);
 798 #endif