2 * linux/kernel/softirq.c
4 * Copyright (C) 1992 Linus Torvalds
6 * Distribute under GPLv2.
8 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
11 #include <linux/module.h>
12 #include <linux/kernel_stat.h>
13 #include <linux/interrupt.h>
14 #include <linux/init.h>
16 #include <linux/notifier.h>
17 #include <linux/percpu.h>
18 #include <linux/cpu.h>
19 #include <linux/freezer.h>
20 #include <linux/kthread.h>
21 #include <linux/rcupdate.h>
22 #include <linux/smp.h>
23 #include <linux/tick.h>
27 - No shared variables, all the data are CPU local.
28 - If a softirq needs serialization, let it serialize itself
30 - Even if softirq is serialized, only local cpu is marked for
31 execution. Hence, we get something sort of weak cpu binding.
32 Though it is still not clear, will it result in better locality
36 - NET RX softirq. It is multithreaded and does not require
37 any global serialization.
38 - NET TX softirq. It kicks software netdevice queues, hence
39 it is logically serialized per device, but this serialization
40 is invisible to common code.
41 - Tasklets: serialized wrt itself.
44 #ifndef __ARCH_IRQ_STAT
45 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
46 EXPORT_SYMBOL(irq_stat);
49 static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;
51 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
54 * we cannot loop indefinitely here to avoid userspace starvation,
55 * but we also don't want to introduce a worst case 1/HZ latency
56 * to the pending events, so lets the scheduler to balance
57 * the softirq load for us.
59 static inline void wakeup_softirqd(void)
61 /* Interrupts are disabled: no need to stop preemption */
62 struct task_struct *tsk = __get_cpu_var(ksoftirqd);
64 if (tsk && tsk->state != TASK_RUNNING)
69 * This one is for softirq.c-internal use,
70 * where hardirqs are disabled legitimately:
72 #ifdef CONFIG_TRACE_IRQFLAGS
73 static void __local_bh_disable(unsigned long ip)
77 WARN_ON_ONCE(in_irq());
79 raw_local_irq_save(flags);
80 add_preempt_count(SOFTIRQ_OFFSET);
82 * Were softirqs turned off above:
84 if (softirq_count() == SOFTIRQ_OFFSET)
85 trace_softirqs_off(ip);
86 raw_local_irq_restore(flags);
88 #else /* !CONFIG_TRACE_IRQFLAGS */
89 static inline void __local_bh_disable(unsigned long ip)
91 add_preempt_count(SOFTIRQ_OFFSET);
94 #endif /* CONFIG_TRACE_IRQFLAGS */
96 void local_bh_disable(void)
98 __local_bh_disable((unsigned long)__builtin_return_address(0));
101 EXPORT_SYMBOL(local_bh_disable);
103 void __local_bh_enable(void)
105 WARN_ON_ONCE(in_irq());
108 * softirqs should never be enabled by __local_bh_enable(),
109 * it always nests inside local_bh_enable() sections:
111 WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
113 sub_preempt_count(SOFTIRQ_OFFSET);
115 EXPORT_SYMBOL_GPL(__local_bh_enable);
118 * Special-case - softirqs can safely be enabled in
119 * cond_resched_softirq(), or by __do_softirq(),
120 * without processing still-pending softirqs:
122 void _local_bh_enable(void)
124 WARN_ON_ONCE(in_irq());
125 WARN_ON_ONCE(!irqs_disabled());
127 if (softirq_count() == SOFTIRQ_OFFSET)
128 trace_softirqs_on((unsigned long)__builtin_return_address(0));
129 sub_preempt_count(SOFTIRQ_OFFSET);
132 EXPORT_SYMBOL(_local_bh_enable);
134 void local_bh_enable(void)
136 #ifdef CONFIG_TRACE_IRQFLAGS
139 WARN_ON_ONCE(in_irq());
141 WARN_ON_ONCE(irqs_disabled());
143 #ifdef CONFIG_TRACE_IRQFLAGS
144 local_irq_save(flags);
147 * Are softirqs going to be turned on now:
149 if (softirq_count() == SOFTIRQ_OFFSET)
150 trace_softirqs_on((unsigned long)__builtin_return_address(0));
152 * Keep preemption disabled until we are done with
153 * softirq processing:
155 sub_preempt_count(SOFTIRQ_OFFSET - 1);
157 if (unlikely(!in_interrupt() && local_softirq_pending()))
161 #ifdef CONFIG_TRACE_IRQFLAGS
162 local_irq_restore(flags);
164 preempt_check_resched();
166 EXPORT_SYMBOL(local_bh_enable);
168 void local_bh_enable_ip(unsigned long ip)
170 #ifdef CONFIG_TRACE_IRQFLAGS
173 WARN_ON_ONCE(in_irq());
175 local_irq_save(flags);
178 * Are softirqs going to be turned on now:
180 if (softirq_count() == SOFTIRQ_OFFSET)
181 trace_softirqs_on(ip);
183 * Keep preemption disabled until we are done with
184 * softirq processing:
186 sub_preempt_count(SOFTIRQ_OFFSET - 1);
188 if (unlikely(!in_interrupt() && local_softirq_pending()))
192 #ifdef CONFIG_TRACE_IRQFLAGS
193 local_irq_restore(flags);
195 preempt_check_resched();
197 EXPORT_SYMBOL(local_bh_enable_ip);
200 * We restart softirq processing MAX_SOFTIRQ_RESTART times,
201 * and we fall back to softirqd after that.
203 * This number has been established via experimentation.
204 * The two things to balance is latency against fairness -
205 * we want to handle softirqs as soon as possible, but they
206 * should not be able to lock up the box.
208 #define MAX_SOFTIRQ_RESTART 10
210 asmlinkage void __do_softirq(void)
212 struct softirq_action *h;
214 int max_restart = MAX_SOFTIRQ_RESTART;
217 pending = local_softirq_pending();
218 account_system_vtime(current);
220 __local_bh_disable((unsigned long)__builtin_return_address(0));
221 trace_softirq_enter();
223 cpu = smp_processor_id();
225 /* Reset the pending bitmask before enabling irqs */
226 set_softirq_pending(0);
235 rcu_bh_qsctr_inc(cpu);
243 pending = local_softirq_pending();
244 if (pending && --max_restart)
250 trace_softirq_exit();
252 account_system_vtime(current);
256 #ifndef __ARCH_HAS_DO_SOFTIRQ
258 asmlinkage void do_softirq(void)
266 local_irq_save(flags);
268 pending = local_softirq_pending();
273 local_irq_restore(flags);
279 * Enter an interrupt context.
284 int cpu = smp_processor_id();
285 if (idle_cpu(cpu) && !in_interrupt())
286 tick_nohz_stop_idle(cpu);
291 tick_nohz_update_jiffies();
295 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
296 # define invoke_softirq() __do_softirq()
298 # define invoke_softirq() do_softirq()
302 * Exit an interrupt context. Process softirqs if needed and possible:
306 account_system_vtime(current);
307 trace_hardirq_exit();
308 sub_preempt_count(IRQ_EXIT_OFFSET);
309 if (!in_interrupt() && local_softirq_pending())
313 /* Make sure that timer wheel updates are propagated */
314 if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
315 tick_nohz_stop_sched_tick();
318 preempt_enable_no_resched();
322 * This function must run with irqs disabled!
324 inline void raise_softirq_irqoff(unsigned int nr)
326 __raise_softirq_irqoff(nr);
329 * If we're in an interrupt or softirq, we're done
330 * (this also catches softirq-disabled code). We will
331 * actually run the softirq once we return from
332 * the irq or softirq.
334 * Otherwise we wake up ksoftirqd to make sure we
335 * schedule the softirq soon.
341 void raise_softirq(unsigned int nr)
345 local_irq_save(flags);
346 raise_softirq_irqoff(nr);
347 local_irq_restore(flags);
350 void open_softirq(int nr, void (*action)(struct softirq_action *))
352 softirq_vec[nr].action = action;
358 struct tasklet_struct *head;
359 struct tasklet_struct **tail;
362 /* Some compilers disobey section attribute on statics when not
364 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
365 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };
367 void __tasklet_schedule(struct tasklet_struct *t)
371 local_irq_save(flags);
373 *__get_cpu_var(tasklet_vec).tail = t;
374 __get_cpu_var(tasklet_vec).tail = &(t->next);
375 raise_softirq_irqoff(TASKLET_SOFTIRQ);
376 local_irq_restore(flags);
379 EXPORT_SYMBOL(__tasklet_schedule);
381 void __tasklet_hi_schedule(struct tasklet_struct *t)
385 local_irq_save(flags);
387 *__get_cpu_var(tasklet_hi_vec).tail = t;
388 __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
389 raise_softirq_irqoff(HI_SOFTIRQ);
390 local_irq_restore(flags);
393 EXPORT_SYMBOL(__tasklet_hi_schedule);
395 static void tasklet_action(struct softirq_action *a)
397 struct tasklet_struct *list;
400 list = __get_cpu_var(tasklet_vec).head;
401 __get_cpu_var(tasklet_vec).head = NULL;
402 __get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
406 struct tasklet_struct *t = list;
410 if (tasklet_trylock(t)) {
411 if (!atomic_read(&t->count)) {
412 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
423 *__get_cpu_var(tasklet_vec).tail = t;
424 __get_cpu_var(tasklet_vec).tail = &(t->next);
425 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
430 static void tasklet_hi_action(struct softirq_action *a)
432 struct tasklet_struct *list;
435 list = __get_cpu_var(tasklet_hi_vec).head;
436 __get_cpu_var(tasklet_hi_vec).head = NULL;
437 __get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
441 struct tasklet_struct *t = list;
445 if (tasklet_trylock(t)) {
446 if (!atomic_read(&t->count)) {
447 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
458 *__get_cpu_var(tasklet_hi_vec).tail = t;
459 __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
460 __raise_softirq_irqoff(HI_SOFTIRQ);
466 void tasklet_init(struct tasklet_struct *t,
467 void (*func)(unsigned long), unsigned long data)
471 atomic_set(&t->count, 0);
476 EXPORT_SYMBOL(tasklet_init);
478 void tasklet_kill(struct tasklet_struct *t)
481 printk("Attempt to kill tasklet from interrupt\n");
483 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
486 while (test_bit(TASKLET_STATE_SCHED, &t->state));
488 tasklet_unlock_wait(t);
489 clear_bit(TASKLET_STATE_SCHED, &t->state);
492 EXPORT_SYMBOL(tasklet_kill);
494 void __init softirq_init(void)
498 for_each_possible_cpu(cpu) {
499 per_cpu(tasklet_vec, cpu).tail =
500 &per_cpu(tasklet_vec, cpu).head;
501 per_cpu(tasklet_hi_vec, cpu).tail =
502 &per_cpu(tasklet_hi_vec, cpu).head;
505 open_softirq(TASKLET_SOFTIRQ, tasklet_action);
506 open_softirq(HI_SOFTIRQ, tasklet_hi_action);
509 static int ksoftirqd(void * __bind_cpu)
511 set_current_state(TASK_INTERRUPTIBLE);
513 while (!kthread_should_stop()) {
515 if (!local_softirq_pending()) {
516 preempt_enable_no_resched();
521 __set_current_state(TASK_RUNNING);
523 while (local_softirq_pending()) {
524 /* Preempt disable stops cpu going offline.
525 If already offline, we'll be on wrong CPU:
527 if (cpu_is_offline((long)__bind_cpu))
530 preempt_enable_no_resched();
535 set_current_state(TASK_INTERRUPTIBLE);
537 __set_current_state(TASK_RUNNING);
542 /* Wait for kthread_stop */
543 set_current_state(TASK_INTERRUPTIBLE);
544 while (!kthread_should_stop()) {
546 set_current_state(TASK_INTERRUPTIBLE);
548 __set_current_state(TASK_RUNNING);
552 #ifdef CONFIG_HOTPLUG_CPU
554 * tasklet_kill_immediate is called to remove a tasklet which can already be
555 * scheduled for execution on @cpu.
557 * Unlike tasklet_kill, this function removes the tasklet
558 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
560 * When this function is called, @cpu must be in the CPU_DEAD state.
562 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
564 struct tasklet_struct **i;
566 BUG_ON(cpu_online(cpu));
567 BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
569 if (!test_bit(TASKLET_STATE_SCHED, &t->state))
572 /* CPU is dead, so no lock needed. */
573 for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
576 /* If this was the tail element, move the tail ptr */
578 per_cpu(tasklet_vec, cpu).tail = i;
585 static void takeover_tasklets(unsigned int cpu)
587 /* CPU is dead, so no lock needed. */
590 /* Find end, append list for that CPU. */
591 if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
592 *(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
593 __get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
594 per_cpu(tasklet_vec, cpu).head = NULL;
595 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
597 raise_softirq_irqoff(TASKLET_SOFTIRQ);
599 if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
600 *__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
601 __get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
602 per_cpu(tasklet_hi_vec, cpu).head = NULL;
603 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
605 raise_softirq_irqoff(HI_SOFTIRQ);
609 #endif /* CONFIG_HOTPLUG_CPU */
611 static int __cpuinit cpu_callback(struct notifier_block *nfb,
612 unsigned long action,
615 int hotcpu = (unsigned long)hcpu;
616 struct task_struct *p;
620 case CPU_UP_PREPARE_FROZEN:
621 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
623 printk("ksoftirqd for %i failed\n", hotcpu);
626 kthread_bind(p, hotcpu);
627 per_cpu(ksoftirqd, hotcpu) = p;
630 case CPU_ONLINE_FROZEN:
631 wake_up_process(per_cpu(ksoftirqd, hotcpu));
633 #ifdef CONFIG_HOTPLUG_CPU
634 case CPU_UP_CANCELED:
635 case CPU_UP_CANCELED_FROZEN:
636 if (!per_cpu(ksoftirqd, hotcpu))
638 /* Unbind so it can run. Fall thru. */
639 kthread_bind(per_cpu(ksoftirqd, hotcpu),
640 any_online_cpu(cpu_online_map));
642 case CPU_DEAD_FROZEN: {
643 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
645 p = per_cpu(ksoftirqd, hotcpu);
646 per_cpu(ksoftirqd, hotcpu) = NULL;
647 sched_setscheduler(p, SCHED_FIFO, ¶m);
649 takeover_tasklets(hotcpu);
652 #endif /* CONFIG_HOTPLUG_CPU */
657 static struct notifier_block __cpuinitdata cpu_nfb = {
658 .notifier_call = cpu_callback
661 __init int spawn_ksoftirqd(void)
663 void *cpu = (void *)(long)smp_processor_id();
664 int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
666 BUG_ON(err == NOTIFY_BAD);
667 cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
668 register_cpu_notifier(&cpu_nfb);
674 * Call a function on all processors
676 int on_each_cpu(void (*func) (void *info), void *info, int retry, int wait)
681 ret = smp_call_function(func, info, retry, wait);
688 EXPORT_SYMBOL(on_each_cpu);