kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  *      Distribute under GPLv2.
   7  *
   8  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   9  *
  10  *      Remote softirq infrastructure is by Jens Axboe.
  11  */
  12
  13 #include <linux/export.h>
  14 #include <linux/kernel_stat.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/init.h>
  17 #include <linux/mm.h>
  18 #include <linux/notifier.h>
  19 #include <linux/percpu.h>
  20 #include <linux/cpu.h>
  21 #include <linux/freezer.h>
  22 #include <linux/kthread.h>
  23 #include <linux/rcupdate.h>
  24 #include <linux/ftrace.h>
  25 #include <linux/smp.h>
  26 #include <linux/smpboot.h>
  27 #include <linux/tick.h>
  28
  29 #define CREATE_TRACE_POINTS
  30 #include <trace/events/irq.h>
  31
  32 #include <asm/irq.h>
  33 /*
  34    - No shared variables, all the data are CPU local.
  35    - If a softirq needs serialization, let it serialize itself
  36      by its own spinlocks.
  37    - Even if softirq is serialized, only local cpu is marked for
  38      execution. Hence, we get something sort of weak cpu binding.
  39      Though it is still not clear, will it result in better locality
  40      or will not.
  41
  42    Examples:
  43    - NET RX softirq. It is multithreaded and does not require
  44      any global serialization.
  45    - NET TX softirq. It kicks software netdevice queues, hence
  46      it is logically serialized per device, but this serialization
  47      is invisible to common code.
  48    - Tasklets: serialized wrt itself.
  49  */
  50
  51 #ifndef __ARCH_IRQ_STAT
  52 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  53 EXPORT_SYMBOL(irq_stat);
  54 #endif
  55
  56 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
  57
  58 DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  59
  60 char *softirq_to_name[NR_SOFTIRQS] = {
  61         "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
  62         "TASKLET", "SCHED", "HRTIMER", "RCU"
  63 };
  64
  65 /*
  66  * we cannot loop indefinitely here to avoid userspace starvation,
  67  * but we also don't want to introduce a worst case 1/HZ latency
  68  * to the pending events, so lets the scheduler to balance
  69  * the softirq load for us.
  70  */
  71 static void wakeup_softirqd(void)
  72 {
  73         /* Interrupts are disabled: no need to stop preemption */
  74         struct task_struct *tsk = __this_cpu_read(ksoftirqd);
  75
  76         if (tsk && tsk->state != TASK_RUNNING)
  77                 wake_up_process(tsk);
  78 }
  79
  80 /*
  81  * preempt_count and SOFTIRQ_OFFSET usage:
  82  * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
  83  *   softirq processing.
  84  * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
  85  *   on local_bh_disable or local_bh_enable.
  86  * This lets us distinguish between whether we are currently processing
  87  * softirq and whether we just have bh disabled.
  88  */
  89
  90 /*
  91  * This one is for softirq.c-internal use,
  92  * where hardirqs are disabled legitimately:
  93  */
  94 #ifdef CONFIG_TRACE_IRQFLAGS
  95 static void __local_bh_disable(unsigned long ip, unsigned int cnt)
  96 {
  97         unsigned long flags;
  98
  99         WARN_ON_ONCE(in_irq());
 100
 101         raw_local_irq_save(flags);
 102         /*
 103          * The preempt tracer hooks into add_preempt_count and will break
 104          * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
 105          * is set and before current->softirq_enabled is cleared.
 106          * We must manually increment preempt_count here and manually
 107          * call the trace_preempt_off later.
 108          */
 109         preempt_count() += cnt;
 110         /*
 111          * Were softirqs turned off above:
 112          */
 113         if (softirq_count() == cnt)
 114                 trace_softirqs_off(ip);
 115         raw_local_irq_restore(flags);
 116
 117         if (preempt_count() == cnt)
 118                 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
 119 }
 120 #else /* !CONFIG_TRACE_IRQFLAGS */
 121 static inline void __local_bh_disable(unsigned long ip, unsigned int cnt)
 122 {
 123         add_preempt_count(cnt);
 124         barrier();
 125 }
 126 #endif /* CONFIG_TRACE_IRQFLAGS */
 127
 128 void local_bh_disable(void)
 129 {
 130         __local_bh_disable((unsigned long)__builtin_return_address(0),
 131                                 SOFTIRQ_DISABLE_OFFSET);
 132 }
 133
 134 EXPORT_SYMBOL(local_bh_disable);
 135
 136 static void __local_bh_enable(unsigned int cnt)
 137 {
 138         WARN_ON_ONCE(in_irq());
 139         WARN_ON_ONCE(!irqs_disabled());
 140
 141         if (softirq_count() == cnt)
 142                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
 143         sub_preempt_count(cnt);
 144 }
 145
 146 /*
 147  * Special-case - softirqs can safely be enabled in
 148  * cond_resched_softirq(), or by __do_softirq(),
 149  * without processing still-pending softirqs:
 150  */
 151 void _local_bh_enable(void)
 152 {
 153         __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
 154 }
 155
 156 EXPORT_SYMBOL(_local_bh_enable);
 157
 158 static inline void _local_bh_enable_ip(unsigned long ip)
 159 {
 160         WARN_ON_ONCE(in_irq() || irqs_disabled());
 161 #ifdef CONFIG_TRACE_IRQFLAGS
 162         local_irq_disable();
 163 #endif
 164         /*
 165          * Are softirqs going to be turned on now:
 166          */
 167         if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
 168                 trace_softirqs_on(ip);
 169         /*
 170          * Keep preemption disabled until we are done with
 171          * softirq processing:
 172          */
 173         sub_preempt_count(SOFTIRQ_DISABLE_OFFSET - 1);
 174
 175         if (unlikely(!in_interrupt() && local_softirq_pending()))
 176                 do_softirq();
 177
 178         dec_preempt_count();
 179 #ifdef CONFIG_TRACE_IRQFLAGS
 180         local_irq_enable();
 181 #endif
 182         preempt_check_resched();
 183 }
 184
 185 void local_bh_enable(void)
 186 {
 187         _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 188 }
 189 EXPORT_SYMBOL(local_bh_enable);
 190
 191 void local_bh_enable_ip(unsigned long ip)
 192 {
 193         _local_bh_enable_ip(ip);
 194 }
 195 EXPORT_SYMBOL(local_bh_enable_ip);
 196
 197 /*
 198  * We restart softirq processing MAX_SOFTIRQ_RESTART times,
 199  * and we fall back to softirqd after that.
 200  *
 201  * This number has been established via experimentation.
 202  * The two things to balance is latency against fairness -
 203  * we want to handle softirqs as soon as possible, but they
 204  * should not be able to lock up the box.
 205  */
 206 #define MAX_SOFTIRQ_RESTART 10
 207
 208 asmlinkage void __do_softirq(void)
 209 {
 210         struct softirq_action *h;
 211         __u32 pending;
 212         int max_restart = MAX_SOFTIRQ_RESTART;
 213         int cpu;
 214         unsigned long old_flags = current->flags;
 215
 216         /*
 217          * Mask out PF_MEMALLOC s current task context is borrowed for the
 218          * softirq. A softirq handled such as network RX might set PF_MEMALLOC
 219          * again if the socket is related to swap
 220          */
 221         current->flags &= ~PF_MEMALLOC;
 222
 223         pending = local_softirq_pending();
 224         vtime_account_irq_enter(current);
 225
 226         __local_bh_disable((unsigned long)__builtin_return_address(0),
 227                                 SOFTIRQ_OFFSET);
 228         lockdep_softirq_enter();
 229
 230         cpu = smp_processor_id();
 231 restart:
 232         /* Reset the pending bitmask before enabling irqs */
 233         set_softirq_pending(0);
 234
 235         local_irq_enable();
 236
 237         h = softirq_vec;
 238
 239         do {
 240                 if (pending & 1) {
 241                         unsigned int vec_nr = h - softirq_vec;
 242                         int prev_count = preempt_count();
 243
 244                         kstat_incr_softirqs_this_cpu(vec_nr);
 245
 246                         trace_softirq_entry(vec_nr);
 247                         h->action(h);
 248                         trace_softirq_exit(vec_nr);
 249                         if (unlikely(prev_count != preempt_count())) {
 250                                 printk(KERN_ERR "huh, entered softirq %u %s %p"
 251                                        "with preempt_count %08x,"
 252                                        " exited with %08x?\n", vec_nr,
 253                                        softirq_to_name[vec_nr], h->action,
 254                                        prev_count, preempt_count());
 255                                 preempt_count() = prev_count;
 256                         }
 257
 258                         rcu_bh_qs(cpu);
 259                 }
 260                 h++;
 261                 pending >>= 1;
 262         } while (pending);
 263
 264         local_irq_disable();
 265
 266         pending = local_softirq_pending();
 267         if (pending && --max_restart)
 268                 goto restart;
 269
 270         if (pending)
 271                 wakeup_softirqd();
 272
 273         lockdep_softirq_exit();
 274
 275         vtime_account_irq_exit(current);
 276         __local_bh_enable(SOFTIRQ_OFFSET);
 277         tsk_restore_flags(current, old_flags, PF_MEMALLOC);
 278 }
 279
 280 #ifndef __ARCH_HAS_DO_SOFTIRQ
 281
 282 asmlinkage void do_softirq(void)
 283 {
 284         __u32 pending;
 285         unsigned long flags;
 286
 287         if (in_interrupt())
 288                 return;
 289
 290         local_irq_save(flags);
 291
 292         pending = local_softirq_pending();
 293
 294         if (pending)
 295                 __do_softirq();
 296
 297         local_irq_restore(flags);
 298 }
 299
 300 #endif
 301
 302 /*
 303  * Enter an interrupt context.
 304  */
 305 void irq_enter(void)
 306 {
 307         int cpu = smp_processor_id();
 308
 309         rcu_irq_enter();
 310         if (is_idle_task(current) && !in_interrupt()) {
 311                 /*
 312                  * Prevent raise_softirq from needlessly waking up ksoftirqd
 313                  * here, as softirq will be serviced on return from interrupt.
 314                  */
 315                 local_bh_disable();
 316                 tick_check_idle(cpu);
 317                 _local_bh_enable();
 318         }
 319
 320         __irq_enter();
 321 }
 322
 323 static inline void invoke_softirq(void)
 324 {
 325         if (!force_irqthreads) {
 326 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
 327                 __do_softirq();
 328 #else
 329                 do_softirq();
 330 #endif
 331         } else {
 332                 __local_bh_disable((unsigned long)__builtin_return_address(0),
 333                                 SOFTIRQ_OFFSET);
 334                 wakeup_softirqd();
 335                 __local_bh_enable(SOFTIRQ_OFFSET);
 336         }
 337 }
 338
 339 /*
 340  * Exit an interrupt context. Process softirqs if needed and possible:
 341  */
 342 void irq_exit(void)
 343 {
 344         vtime_account_irq_exit(current);
 345         trace_hardirq_exit();
 346         sub_preempt_count(IRQ_EXIT_OFFSET);
 347         if (!in_interrupt() && local_softirq_pending())
 348                 invoke_softirq();
 349
 350 #ifdef CONFIG_NO_HZ
 351         /* Make sure that timer wheel updates are propagated */
 352         if (idle_cpu(smp_processor_id()) && !in_interrupt() && !need_resched())
 353                 tick_nohz_irq_exit();
 354 #endif
 355         rcu_irq_exit();
 356         sched_preempt_enable_no_resched();
 357 }
 358
 359 /*
 360  * This function must run with irqs disabled!
 361  */
 362 inline void raise_softirq_irqoff(unsigned int nr)
 363 {
 364         __raise_softirq_irqoff(nr);
 365
 366         /*
 367          * If we're in an interrupt or softirq, we're done
 368          * (this also catches softirq-disabled code). We will
 369          * actually run the softirq once we return from
 370          * the irq or softirq.
 371          *
 372          * Otherwise we wake up ksoftirqd to make sure we
 373          * schedule the softirq soon.
 374          */
 375         if (!in_interrupt())
 376                 wakeup_softirqd();
 377 }
 378
 379 void raise_softirq(unsigned int nr)
 380 {
 381         unsigned long flags;
 382
 383         local_irq_save(flags);
 384         raise_softirq_irqoff(nr);
 385         local_irq_restore(flags);
 386 }
 387
 388 void __raise_softirq_irqoff(unsigned int nr)
 389 {
 390         trace_softirq_raise(nr);
 391         or_softirq_pending(1UL << nr);
 392 }
 393
 394 void open_softirq(int nr, void (*action)(struct softirq_action *))
 395 {
 396         softirq_vec[nr].action = action;
 397 }
 398
 399 /*
 400  * Tasklets
 401  */
 402 struct tasklet_head
 403 {
 404         struct tasklet_struct *head;
 405         struct tasklet_struct **tail;
 406 };
 407
 408 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 409 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 410
 411 void __tasklet_schedule(struct tasklet_struct *t)
 412 {
 413         unsigned long flags;
 414
 415         local_irq_save(flags);
 416         t->next = NULL;
 417         *__this_cpu_read(tasklet_vec.tail) = t;
 418         __this_cpu_write(tasklet_vec.tail, &(t->next));
 419         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 420         clear_bit(TASKLET_STATE_HI, &t->state);
 421         local_irq_restore(flags);
 422 }
 423
 424 EXPORT_SYMBOL(__tasklet_schedule);
 425
 426 void __tasklet_hi_schedule(struct tasklet_struct *t)
 427 {
 428         unsigned long flags;
 429
 430         local_irq_save(flags);
 431         t->next = NULL;
 432         *__this_cpu_read(tasklet_hi_vec.tail) = t;
 433         __this_cpu_write(tasklet_hi_vec.tail,  &(t->next));
 434         raise_softirq_irqoff(HI_SOFTIRQ);
 435         set_bit(TASKLET_STATE_HI, &t->state);
 436         local_irq_restore(flags);
 437 }
 438
 439 EXPORT_SYMBOL(__tasklet_hi_schedule);
 440
 441 void __tasklet_hi_schedule_first(struct tasklet_struct *t)
 442 {
 443         BUG_ON(!irqs_disabled());
 444
 445         t->next = __this_cpu_read(tasklet_hi_vec.head);
 446         __this_cpu_write(tasklet_hi_vec.head, t);
 447         set_bit(TASKLET_STATE_HI, &t->state);
 448         __raise_softirq_irqoff(HI_SOFTIRQ);
 449 }
 450
 451 EXPORT_SYMBOL(__tasklet_hi_schedule_first);
 452
 453 static void tasklet_action(struct softirq_action *a)
 454 {
 455         struct tasklet_struct *list;
 456
 457         local_irq_disable();
 458         list = __this_cpu_read(tasklet_vec.head);
 459         __this_cpu_write(tasklet_vec.head, NULL);
 460         __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
 461         local_irq_enable();
 462
 463         while (list) {
 464                 struct tasklet_struct *t = list;
 465
 466                 list = list->next;
 467
 468                 if (tasklet_trylock(t)) {
 469                         if (!atomic_read(&t->count)) {
 470                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 471                                         BUG();
 472                                 t->func(t->data);
 473                         }
 474                         tasklet_unlock(t);
 475                         continue;
 476                 }
 477
 478                 local_irq_disable();
 479                 t->next = NULL;
 480                 *__this_cpu_read(tasklet_vec.tail) = t;
 481                 __this_cpu_write(tasklet_vec.tail, &(t->next));
 482                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 483                 local_irq_enable();
 484         }
 485 }
 486
 487 static void tasklet_hi_action(struct softirq_action *a)
 488 {
 489         struct tasklet_struct *list;
 490
 491         local_irq_disable();
 492         list = __this_cpu_read(tasklet_hi_vec.head);
 493         __this_cpu_write(tasklet_hi_vec.head, NULL);
 494         __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
 495         local_irq_enable();
 496
 497         while (list) {
 498                 struct tasklet_struct *t = list;
 499
 500                 list = list->next;
 501
 502                 if (tasklet_trylock(t)) {
 503                         if (!atomic_read(&t->count)) {
 504                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 505                                         BUG();
 506                                 t->func(t->data);
 507                         }
 508                         tasklet_unlock(t);
 509                         continue;
 510                 }
 511
 512                 local_irq_disable();
 513                 t->next = NULL;
 514                 *__this_cpu_read(tasklet_hi_vec.tail) = t;
 515                 __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
 516                 __raise_softirq_irqoff(HI_SOFTIRQ);
 517                 local_irq_enable();
 518         }
 519 }
 520
 521
 522 void tasklet_init(struct tasklet_struct *t,
 523                   void (*func)(unsigned long), unsigned long data)
 524 {
 525         t->next = NULL;
 526         t->state = 0;
 527         atomic_set(&t->count, 0);
 528         t->func = func;
 529         t->data = data;
 530 }
 531
 532 EXPORT_SYMBOL(tasklet_init);
 533
 534 void tasklet_kill(struct tasklet_struct *t)
 535 {
 536         if (in_interrupt())
 537                 printk("Attempt to kill tasklet from interrupt\n");
 538
 539         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 540                 do {
 541                         yield();
 542                 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
 543         }
 544         tasklet_unlock_wait(t);
 545         clear_bit(TASKLET_STATE_SCHED, &t->state);
 546 }
 547
 548 EXPORT_SYMBOL(tasklet_kill);
 549
 550 /*
 551  * tasklet_hrtimer
 552  */
 553
 554 /*
 555  * The trampoline is called when the hrtimer expires. It schedules a tasklet
 556  * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
 557  * hrtimer callback, but from softirq context.
 558  */
 559 static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
 560 {
 561         struct tasklet_hrtimer *ttimer =
 562                 container_of(timer, struct tasklet_hrtimer, timer);
 563
 564         tasklet_hi_schedule(&ttimer->tasklet);
 565         return HRTIMER_NORESTART;
 566 }
 567
 568 /*
 569  * Helper function which calls the hrtimer callback from
 570  * tasklet/softirq context
 571  */
 572 static void __tasklet_hrtimer_trampoline(unsigned long data)
 573 {
 574         struct tasklet_hrtimer *ttimer = (void *)data;
 575         enum hrtimer_restart restart;
 576
 577         restart = ttimer->function(&ttimer->timer);
 578         if (restart != HRTIMER_NORESTART)
 579                 hrtimer_restart(&ttimer->timer);
 580 }
 581
 582 /**
 583  * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
 584  * @ttimer:      tasklet_hrtimer which is initialized
 585  * @function:    hrtimer callback function which gets called from softirq context
 586  * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
 587  * @mode:        hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
 588  */
 589 void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
 590                           enum hrtimer_restart (*function)(struct hrtimer *),
 591                           clockid_t which_clock, enum hrtimer_mode mode)
 592 {
 593         hrtimer_init(&ttimer->timer, which_clock, mode);
 594         ttimer->timer.function = __hrtimer_tasklet_trampoline;
 595         tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
 596                      (unsigned long)ttimer);
 597         ttimer->function = function;
 598 }
 599 EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
 600
 601 /*
 602  * Remote softirq bits
 603  */
 604
 605 DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
 606 EXPORT_PER_CPU_SYMBOL(softirq_work_list);
 607
 608 static void __local_trigger(struct call_single_data *cp, int softirq)
 609 {
 610         struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
 611
 612         list_add_tail(&cp->list, head);
 613
 614         /* Trigger the softirq only if the list was previously empty.  */
 615         if (head->next == &cp->list)
 616                 raise_softirq_irqoff(softirq);
 617 }
 618
 619 #ifdef CONFIG_USE_GENERIC_SMP_HELPERS
 620 static void remote_softirq_receive(void *data)
 621 {
 622         struct call_single_data *cp = data;
 623         unsigned long flags;
 624         int softirq;
 625
 626         softirq = cp->priv;
 627
 628         local_irq_save(flags);
 629         __local_trigger(cp, softirq);
 630         local_irq_restore(flags);
 631 }
 632
 633 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 634 {
 635         if (cpu_online(cpu)) {
 636                 cp->func = remote_softirq_receive;
 637                 cp->info = cp;
 638                 cp->flags = 0;
 639                 cp->priv = softirq;
 640
 641                 __smp_call_function_single(cpu, cp, 0);
 642                 return 0;
 643         }
 644         return 1;
 645 }
 646 #else /* CONFIG_USE_GENERIC_SMP_HELPERS */
 647 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 648 {
 649         return 1;
 650 }
 651 #endif
 652
 653 /**
 654  * __send_remote_softirq - try to schedule softirq work on a remote cpu
 655  * @cp: private SMP call function data area
 656  * @cpu: the remote cpu
 657  * @this_cpu: the currently executing cpu
 658  * @softirq: the softirq for the work
 659  *
 660  * Attempt to schedule softirq work on a remote cpu.  If this cannot be
 661  * done, the work is instead queued up on the local cpu.
 662  *
 663  * Interrupts must be disabled.
 664  */
 665 void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
 666 {
 667         if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
 668                 __local_trigger(cp, softirq);
 669 }
 670 EXPORT_SYMBOL(__send_remote_softirq);
 671
 672 /**
 673  * send_remote_softirq - try to schedule softirq work on a remote cpu
 674  * @cp: private SMP call function data area
 675  * @cpu: the remote cpu
 676  * @softirq: the softirq for the work
 677  *
 678  * Like __send_remote_softirq except that disabling interrupts and
 679  * computing the current cpu is done for the caller.
 680  */
 681 void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 682 {
 683         unsigned long flags;
 684         int this_cpu;
 685
 686         local_irq_save(flags);
 687         this_cpu = smp_processor_id();
 688         __send_remote_softirq(cp, cpu, this_cpu, softirq);
 689         local_irq_restore(flags);
 690 }
 691 EXPORT_SYMBOL(send_remote_softirq);
 692
 693 static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
 694                                                unsigned long action, void *hcpu)
 695 {
 696         /*
 697          * If a CPU goes away, splice its entries to the current CPU
 698          * and trigger a run of the softirq
 699          */
 700         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
 701                 int cpu = (unsigned long) hcpu;
 702                 int i;
 703
 704                 local_irq_disable();
 705                 for (i = 0; i < NR_SOFTIRQS; i++) {
 706                         struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
 707                         struct list_head *local_head;
 708
 709                         if (list_empty(head))
 710                                 continue;
 711
 712                         local_head = &__get_cpu_var(softirq_work_list[i]);
 713                         list_splice_init(head, local_head);
 714                         raise_softirq_irqoff(i);
 715                 }
 716                 local_irq_enable();
 717         }
 718
 719         return NOTIFY_OK;
 720 }
 721
 722 static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
 723         .notifier_call  = remote_softirq_cpu_notify,
 724 };
 725
 726 void __init softirq_init(void)
 727 {
 728         int cpu;
 729
 730         for_each_possible_cpu(cpu) {
 731                 int i;
 732
 733                 per_cpu(tasklet_vec, cpu).tail =
 734                         &per_cpu(tasklet_vec, cpu).head;
 735                 per_cpu(tasklet_hi_vec, cpu).tail =
 736                         &per_cpu(tasklet_hi_vec, cpu).head;
 737                 for (i = 0; i < NR_SOFTIRQS; i++)
 738                         INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
 739         }
 740
 741         register_hotcpu_notifier(&remote_softirq_cpu_notifier);
 742
 743         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
 744         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 745 }
 746
 747 static int ksoftirqd_should_run(unsigned int cpu)
 748 {
 749         return local_softirq_pending();
 750 }
 751
 752 static void run_ksoftirqd(unsigned int cpu)
 753 {
 754         local_irq_disable();
 755         if (local_softirq_pending()) {
 756                 __do_softirq();
 757                 rcu_note_context_switch(cpu);
 758                 local_irq_enable();
 759                 cond_resched();
 760                 return;
 761         }
 762         local_irq_enable();
 763 }
 764
 765 #ifdef CONFIG_HOTPLUG_CPU
 766 /*
 767  * tasklet_kill_immediate is called to remove a tasklet which can already be
 768  * scheduled for execution on @cpu.
 769  *
 770  * Unlike tasklet_kill, this function removes the tasklet
 771  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 772  *
 773  * When this function is called, @cpu must be in the CPU_DEAD state.
 774  */
 775 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 776 {
 777         struct tasklet_struct **i;
 778
 779         BUG_ON(cpu_online(cpu));
 780         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 781
 782         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 783                 return;
 784
 785         /* CPU is dead, so no lock needed. */
 786         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
 787                 if (*i == t) {
 788                         *i = t->next;
 789                         /* If this was the tail element, move the tail ptr */
 790                         if (*i == NULL)
 791                                 per_cpu(tasklet_vec, cpu).tail = i;
 792                         return;
 793                 }
 794         }
 795         BUG();
 796 }
 797
 798 static void takeover_tasklets(unsigned int cpu)
 799 {
 800         /* CPU is dead, so no lock needed. */
 801         local_irq_disable();
 802
 803         /* Find end, append list for that CPU. */
 804         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
 805                 *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
 806                 this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
 807                 per_cpu(tasklet_vec, cpu).head = NULL;
 808                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
 809         }
 810         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 811
 812         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
 813                 *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
 814                 __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
 815                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
 816                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
 817         }
 818         raise_softirq_irqoff(HI_SOFTIRQ);
 819
 820         local_irq_enable();
 821 }
 822 #endif /* CONFIG_HOTPLUG_CPU */
 823
 824 static int __cpuinit cpu_callback(struct notifier_block *nfb,
 825                                   unsigned long action,
 826                                   void *hcpu)
 827 {
 828         switch (action) {
 829 #ifdef CONFIG_HOTPLUG_CPU
 830         case CPU_DEAD:
 831         case CPU_DEAD_FROZEN:
 832                 takeover_tasklets((unsigned long)hcpu);
 833                 break;
 834 #endif /* CONFIG_HOTPLUG_CPU */
 835         }
 836         return NOTIFY_OK;
 837 }
 838
 839 static struct notifier_block __cpuinitdata cpu_nfb = {
 840         .notifier_call = cpu_callback
 841 };
 842
 843 static struct smp_hotplug_thread softirq_threads = {
 844         .store                  = &ksoftirqd,
 845         .thread_should_run      = ksoftirqd_should_run,
 846         .thread_fn              = run_ksoftirqd,
 847         .thread_comm            = "ksoftirqd/%u",
 848 };
 849
 850 static __init int spawn_ksoftirqd(void)
 851 {
 852         register_cpu_notifier(&cpu_nfb);
 853
 854         BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
 855
 856         return 0;
 857 }
 858 early_initcall(spawn_ksoftirqd);
 859
 860 /*
 861  * [ These __weak aliases are kept in a separate compilation unit, so that
 862  *   GCC does not inline them incorrectly. ]
 863  */
 864
 865 int __init __weak early_irq_init(void)
 866 {
 867         return 0;
 868 }
 869
 870 #ifdef CONFIG_GENERIC_HARDIRQS
 871 int __init __weak arch_probe_nr_irqs(void)
 872 {
 873         return NR_IRQS_LEGACY;
 874 }
 875
 876 int __init __weak arch_early_irq_init(void)
 877 {
 878         return 0;
 879 }
 880 #endif