2 * linux/kernel/irq/manage.c
4 * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
5 * Copyright (C) 2005-2006 Thomas Gleixner
7 * This file contains driver APIs to the irq subsystem.
10 #define pr_fmt(fmt) "genirq: " fmt
12 #include <linux/irq.h>
13 #include <linux/kthread.h>
14 #include <linux/module.h>
15 #include <linux/random.h>
16 #include <linux/interrupt.h>
17 #include <linux/slab.h>
18 #include <linux/sched.h>
19 #include <linux/sched/rt.h>
20 #include <linux/sched/task.h>
21 #include <uapi/linux/sched/types.h>
22 #include <linux/task_work.h>
24 #include "internals.h"
26 #ifdef CONFIG_IRQ_FORCED_THREADING
27 __read_mostly bool force_irqthreads;
29 static int __init setup_forced_irqthreads(char *arg)
31 force_irqthreads = true;
34 early_param("threadirqs", setup_forced_irqthreads);
37 static void __synchronize_hardirq(struct irq_desc *desc)
45 * Wait until we're out of the critical section. This might
46 * give the wrong answer due to the lack of memory barriers.
48 while (irqd_irq_inprogress(&desc->irq_data))
51 /* Ok, that indicated we're done: double-check carefully. */
52 raw_spin_lock_irqsave(&desc->lock, flags);
53 inprogress = irqd_irq_inprogress(&desc->irq_data);
54 raw_spin_unlock_irqrestore(&desc->lock, flags);
56 /* Oops, that failed? */
61 * synchronize_hardirq - wait for pending hard IRQ handlers (on other CPUs)
62 * @irq: interrupt number to wait for
64 * This function waits for any pending hard IRQ handlers for this
65 * interrupt to complete before returning. If you use this
66 * function while holding a resource the IRQ handler may need you
67 * will deadlock. It does not take associated threaded handlers
70 * Do not use this for shutdown scenarios where you must be sure
71 * that all parts (hardirq and threaded handler) have completed.
73 * Returns: false if a threaded handler is active.
75 * This function may be called - with care - from IRQ context.
77 bool synchronize_hardirq(unsigned int irq)
79 struct irq_desc *desc = irq_to_desc(irq);
82 __synchronize_hardirq(desc);
83 return !atomic_read(&desc->threads_active);
88 EXPORT_SYMBOL(synchronize_hardirq);
91 * synchronize_irq - wait for pending IRQ handlers (on other CPUs)
92 * @irq: interrupt number to wait for
94 * This function waits for any pending IRQ handlers for this interrupt
95 * to complete before returning. If you use this function while
96 * holding a resource the IRQ handler may need you will deadlock.
98 * This function may be called - with care - from IRQ context.
100 void synchronize_irq(unsigned int irq)
102 struct irq_desc *desc = irq_to_desc(irq);
105 __synchronize_hardirq(desc);
107 * We made sure that no hardirq handler is
108 * running. Now verify that no threaded handlers are
111 wait_event(desc->wait_for_threads,
112 !atomic_read(&desc->threads_active));
115 EXPORT_SYMBOL(synchronize_irq);
118 cpumask_var_t irq_default_affinity;
120 static bool __irq_can_set_affinity(struct irq_desc *desc)
122 if (!desc || !irqd_can_balance(&desc->irq_data) ||
123 !desc->irq_data.chip || !desc->irq_data.chip->irq_set_affinity)
129 * irq_can_set_affinity - Check if the affinity of a given irq can be set
130 * @irq: Interrupt to check
133 int irq_can_set_affinity(unsigned int irq)
135 return __irq_can_set_affinity(irq_to_desc(irq));
139 * irq_can_set_affinity_usr - Check if affinity of a irq can be set from user space
140 * @irq: Interrupt to check
142 * Like irq_can_set_affinity() above, but additionally checks for the
143 * AFFINITY_MANAGED flag.
145 bool irq_can_set_affinity_usr(unsigned int irq)
147 struct irq_desc *desc = irq_to_desc(irq);
149 return __irq_can_set_affinity(desc) &&
150 !irqd_affinity_is_managed(&desc->irq_data);
154 * irq_set_thread_affinity - Notify irq threads to adjust affinity
155 * @desc: irq descriptor which has affitnity changed
157 * We just set IRQTF_AFFINITY and delegate the affinity setting
158 * to the interrupt thread itself. We can not call
159 * set_cpus_allowed_ptr() here as we hold desc->lock and this
160 * code can be called from hard interrupt context.
162 void irq_set_thread_affinity(struct irq_desc *desc)
164 struct irqaction *action;
166 for_each_action_of_desc(desc, action)
168 set_bit(IRQTF_AFFINITY, &action->thread_flags);
171 #ifdef CONFIG_GENERIC_PENDING_IRQ
172 static inline bool irq_can_move_pcntxt(struct irq_data *data)
174 return irqd_can_move_in_process_context(data);
176 static inline bool irq_move_pending(struct irq_data *data)
178 return irqd_is_setaffinity_pending(data);
181 irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask)
183 cpumask_copy(desc->pending_mask, mask);
186 irq_get_pending(struct cpumask *mask, struct irq_desc *desc)
188 cpumask_copy(mask, desc->pending_mask);
191 static inline bool irq_can_move_pcntxt(struct irq_data *data) { return true; }
192 static inline bool irq_move_pending(struct irq_data *data) { return false; }
194 irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask) { }
196 irq_get_pending(struct cpumask *mask, struct irq_desc *desc) { }
199 int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask,
202 struct irq_desc *desc = irq_data_to_desc(data);
203 struct irq_chip *chip = irq_data_get_irq_chip(data);
206 ret = chip->irq_set_affinity(data, mask, force);
208 case IRQ_SET_MASK_OK:
209 case IRQ_SET_MASK_OK_DONE:
210 cpumask_copy(desc->irq_common_data.affinity, mask);
211 case IRQ_SET_MASK_OK_NOCOPY:
212 irq_set_thread_affinity(desc);
219 int irq_set_affinity_locked(struct irq_data *data, const struct cpumask *mask,
222 struct irq_chip *chip = irq_data_get_irq_chip(data);
223 struct irq_desc *desc = irq_data_to_desc(data);
226 if (!chip || !chip->irq_set_affinity)
229 if (irq_can_move_pcntxt(data)) {
230 ret = irq_do_set_affinity(data, mask, force);
232 irqd_set_move_pending(data);
233 irq_copy_pending(desc, mask);
236 if (desc->affinity_notify) {
237 kref_get(&desc->affinity_notify->kref);
238 schedule_work(&desc->affinity_notify->work);
240 irqd_set(data, IRQD_AFFINITY_SET);
245 int __irq_set_affinity(unsigned int irq, const struct cpumask *mask, bool force)
247 struct irq_desc *desc = irq_to_desc(irq);
254 raw_spin_lock_irqsave(&desc->lock, flags);
255 ret = irq_set_affinity_locked(irq_desc_get_irq_data(desc), mask, force);
256 raw_spin_unlock_irqrestore(&desc->lock, flags);
260 int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m)
263 struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
267 desc->affinity_hint = m;
268 irq_put_desc_unlock(desc, flags);
269 /* set the initial affinity to prevent every interrupt being on CPU0 */
271 __irq_set_affinity(irq, m, false);
274 EXPORT_SYMBOL_GPL(irq_set_affinity_hint);
276 static void irq_affinity_notify(struct work_struct *work)
278 struct irq_affinity_notify *notify =
279 container_of(work, struct irq_affinity_notify, work);
280 struct irq_desc *desc = irq_to_desc(notify->irq);
281 cpumask_var_t cpumask;
284 if (!desc || !alloc_cpumask_var(&cpumask, GFP_KERNEL))
287 raw_spin_lock_irqsave(&desc->lock, flags);
288 if (irq_move_pending(&desc->irq_data))
289 irq_get_pending(cpumask, desc);
291 cpumask_copy(cpumask, desc->irq_common_data.affinity);
292 raw_spin_unlock_irqrestore(&desc->lock, flags);
294 notify->notify(notify, cpumask);
296 free_cpumask_var(cpumask);
298 kref_put(¬ify->kref, notify->release);
302 * irq_set_affinity_notifier - control notification of IRQ affinity changes
303 * @irq: Interrupt for which to enable/disable notification
304 * @notify: Context for notification, or %NULL to disable
305 * notification. Function pointers must be initialised;
306 * the other fields will be initialised by this function.
308 * Must be called in process context. Notification may only be enabled
309 * after the IRQ is allocated and must be disabled before the IRQ is
310 * freed using free_irq().
313 irq_set_affinity_notifier(unsigned int irq, struct irq_affinity_notify *notify)
315 struct irq_desc *desc = irq_to_desc(irq);
316 struct irq_affinity_notify *old_notify;
319 /* The release function is promised process context */
325 /* Complete initialisation of *notify */
328 kref_init(¬ify->kref);
329 INIT_WORK(¬ify->work, irq_affinity_notify);
332 raw_spin_lock_irqsave(&desc->lock, flags);
333 old_notify = desc->affinity_notify;
334 desc->affinity_notify = notify;
335 raw_spin_unlock_irqrestore(&desc->lock, flags);
338 kref_put(&old_notify->kref, old_notify->release);
342 EXPORT_SYMBOL_GPL(irq_set_affinity_notifier);
344 #ifndef CONFIG_AUTO_IRQ_AFFINITY
346 * Generic version of the affinity autoselector.
348 static int setup_affinity(struct irq_desc *desc, struct cpumask *mask)
350 struct cpumask *set = irq_default_affinity;
351 int node = irq_desc_get_node(desc);
353 /* Excludes PER_CPU and NO_BALANCE interrupts */
354 if (!__irq_can_set_affinity(desc))
358 * Preserve the managed affinity setting and a userspace affinity
359 * setup, but make sure that one of the targets is online.
361 if (irqd_affinity_is_managed(&desc->irq_data) ||
362 irqd_has_set(&desc->irq_data, IRQD_AFFINITY_SET)) {
363 if (cpumask_intersects(desc->irq_common_data.affinity,
365 set = desc->irq_common_data.affinity;
367 irqd_clear(&desc->irq_data, IRQD_AFFINITY_SET);
370 cpumask_and(mask, cpu_online_mask, set);
371 if (node != NUMA_NO_NODE) {
372 const struct cpumask *nodemask = cpumask_of_node(node);
374 /* make sure at least one of the cpus in nodemask is online */
375 if (cpumask_intersects(mask, nodemask))
376 cpumask_and(mask, mask, nodemask);
378 irq_do_set_affinity(&desc->irq_data, mask, false);
382 /* Wrapper for ALPHA specific affinity selector magic */
383 static inline int setup_affinity(struct irq_desc *d, struct cpumask *mask)
385 return irq_select_affinity(irq_desc_get_irq(d));
390 * Called when affinity is set via /proc/irq
392 int irq_select_affinity_usr(unsigned int irq, struct cpumask *mask)
394 struct irq_desc *desc = irq_to_desc(irq);
398 raw_spin_lock_irqsave(&desc->lock, flags);
399 ret = setup_affinity(desc, mask);
400 raw_spin_unlock_irqrestore(&desc->lock, flags);
406 setup_affinity(struct irq_desc *desc, struct cpumask *mask)
413 * irq_set_vcpu_affinity - Set vcpu affinity for the interrupt
414 * @irq: interrupt number to set affinity
415 * @vcpu_info: vCPU specific data
417 * This function uses the vCPU specific data to set the vCPU
418 * affinity for an irq. The vCPU specific data is passed from
419 * outside, such as KVM. One example code path is as below:
420 * KVM -> IOMMU -> irq_set_vcpu_affinity().
422 int irq_set_vcpu_affinity(unsigned int irq, void *vcpu_info)
425 struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
426 struct irq_data *data;
427 struct irq_chip *chip;
433 data = irq_desc_get_irq_data(desc);
434 chip = irq_data_get_irq_chip(data);
435 if (chip && chip->irq_set_vcpu_affinity)
436 ret = chip->irq_set_vcpu_affinity(data, vcpu_info);
437 irq_put_desc_unlock(desc, flags);
441 EXPORT_SYMBOL_GPL(irq_set_vcpu_affinity);
443 void __disable_irq(struct irq_desc *desc)
449 static int __disable_irq_nosync(unsigned int irq)
452 struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
457 irq_put_desc_busunlock(desc, flags);
462 * disable_irq_nosync - disable an irq without waiting
463 * @irq: Interrupt to disable
465 * Disable the selected interrupt line. Disables and Enables are
467 * Unlike disable_irq(), this function does not ensure existing
468 * instances of the IRQ handler have completed before returning.
470 * This function may be called from IRQ context.
472 void disable_irq_nosync(unsigned int irq)
474 __disable_irq_nosync(irq);
476 EXPORT_SYMBOL(disable_irq_nosync);
479 * disable_irq - disable an irq and wait for completion
480 * @irq: Interrupt to disable
482 * Disable the selected interrupt line. Enables and Disables are
484 * This function waits for any pending IRQ handlers for this interrupt
485 * to complete before returning. If you use this function while
486 * holding a resource the IRQ handler may need you will deadlock.
488 * This function may be called - with care - from IRQ context.
490 void disable_irq(unsigned int irq)
492 if (!__disable_irq_nosync(irq))
493 synchronize_irq(irq);
495 EXPORT_SYMBOL(disable_irq);
498 * disable_hardirq - disables an irq and waits for hardirq completion
499 * @irq: Interrupt to disable
501 * Disable the selected interrupt line. Enables and Disables are
503 * This function waits for any pending hard IRQ handlers for this
504 * interrupt to complete before returning. If you use this function while
505 * holding a resource the hard IRQ handler may need you will deadlock.
507 * When used to optimistically disable an interrupt from atomic context
508 * the return value must be checked.
510 * Returns: false if a threaded handler is active.
512 * This function may be called - with care - from IRQ context.
514 bool disable_hardirq(unsigned int irq)
516 if (!__disable_irq_nosync(irq))
517 return synchronize_hardirq(irq);
521 EXPORT_SYMBOL_GPL(disable_hardirq);
523 void __enable_irq(struct irq_desc *desc)
525 switch (desc->depth) {
528 WARN(1, KERN_WARNING "Unbalanced enable for IRQ %d\n",
529 irq_desc_get_irq(desc));
532 if (desc->istate & IRQS_SUSPENDED)
534 /* Prevent probing on this irq: */
535 irq_settings_set_noprobe(desc);
537 check_irq_resend(desc);
546 * enable_irq - enable handling of an irq
547 * @irq: Interrupt to enable
549 * Undoes the effect of one call to disable_irq(). If this
550 * matches the last disable, processing of interrupts on this
551 * IRQ line is re-enabled.
553 * This function may be called from IRQ context only when
554 * desc->irq_data.chip->bus_lock and desc->chip->bus_sync_unlock are NULL !
556 void enable_irq(unsigned int irq)
559 struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
563 if (WARN(!desc->irq_data.chip,
564 KERN_ERR "enable_irq before setup/request_irq: irq %u\n", irq))
569 irq_put_desc_busunlock(desc, flags);
571 EXPORT_SYMBOL(enable_irq);
573 static int set_irq_wake_real(unsigned int irq, unsigned int on)
575 struct irq_desc *desc = irq_to_desc(irq);
578 if (irq_desc_get_chip(desc)->flags & IRQCHIP_SKIP_SET_WAKE)
581 if (desc->irq_data.chip->irq_set_wake)
582 ret = desc->irq_data.chip->irq_set_wake(&desc->irq_data, on);
588 * irq_set_irq_wake - control irq power management wakeup
589 * @irq: interrupt to control
590 * @on: enable/disable power management wakeup
592 * Enable/disable power management wakeup mode, which is
593 * disabled by default. Enables and disables must match,
594 * just as they match for non-wakeup mode support.
596 * Wakeup mode lets this IRQ wake the system from sleep
597 * states like "suspend to RAM".
599 int irq_set_irq_wake(unsigned int irq, unsigned int on)
602 struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
608 /* wakeup-capable irqs can be shared between drivers that
609 * don't need to have the same sleep mode behaviors.
612 if (desc->wake_depth++ == 0) {
613 ret = set_irq_wake_real(irq, on);
615 desc->wake_depth = 0;
617 irqd_set(&desc->irq_data, IRQD_WAKEUP_STATE);
620 if (desc->wake_depth == 0) {
621 WARN(1, "Unbalanced IRQ %d wake disable\n", irq);
622 } else if (--desc->wake_depth == 0) {
623 ret = set_irq_wake_real(irq, on);
625 desc->wake_depth = 1;
627 irqd_clear(&desc->irq_data, IRQD_WAKEUP_STATE);
630 irq_put_desc_busunlock(desc, flags);
633 EXPORT_SYMBOL(irq_set_irq_wake);
636 * Internal function that tells the architecture code whether a
637 * particular irq has been exclusively allocated or is available
640 int can_request_irq(unsigned int irq, unsigned long irqflags)
643 struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
649 if (irq_settings_can_request(desc)) {
651 irqflags & desc->action->flags & IRQF_SHARED)
654 irq_put_desc_unlock(desc, flags);
658 int __irq_set_trigger(struct irq_desc *desc, unsigned long flags)
660 struct irq_chip *chip = desc->irq_data.chip;
663 if (!chip || !chip->irq_set_type) {
665 * IRQF_TRIGGER_* but the PIC does not support multiple
668 pr_debug("No set_type function for IRQ %d (%s)\n",
669 irq_desc_get_irq(desc),
670 chip ? (chip->name ? : "unknown") : "unknown");
674 if (chip->flags & IRQCHIP_SET_TYPE_MASKED) {
675 if (!irqd_irq_masked(&desc->irq_data))
677 if (!irqd_irq_disabled(&desc->irq_data))
681 /* Mask all flags except trigger mode */
682 flags &= IRQ_TYPE_SENSE_MASK;
683 ret = chip->irq_set_type(&desc->irq_data, flags);
686 case IRQ_SET_MASK_OK:
687 case IRQ_SET_MASK_OK_DONE:
688 irqd_clear(&desc->irq_data, IRQD_TRIGGER_MASK);
689 irqd_set(&desc->irq_data, flags);
691 case IRQ_SET_MASK_OK_NOCOPY:
692 flags = irqd_get_trigger_type(&desc->irq_data);
693 irq_settings_set_trigger_mask(desc, flags);
694 irqd_clear(&desc->irq_data, IRQD_LEVEL);
695 irq_settings_clr_level(desc);
696 if (flags & IRQ_TYPE_LEVEL_MASK) {
697 irq_settings_set_level(desc);
698 irqd_set(&desc->irq_data, IRQD_LEVEL);
704 pr_err("Setting trigger mode %lu for irq %u failed (%pF)\n",
705 flags, irq_desc_get_irq(desc), chip->irq_set_type);
712 #ifdef CONFIG_HARDIRQS_SW_RESEND
713 int irq_set_parent(int irq, int parent_irq)
716 struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
721 desc->parent_irq = parent_irq;
723 irq_put_desc_unlock(desc, flags);
726 EXPORT_SYMBOL_GPL(irq_set_parent);
730 * Default primary interrupt handler for threaded interrupts. Is
731 * assigned as primary handler when request_threaded_irq is called
732 * with handler == NULL. Useful for oneshot interrupts.
734 static irqreturn_t irq_default_primary_handler(int irq, void *dev_id)
736 return IRQ_WAKE_THREAD;
740 * Primary handler for nested threaded interrupts. Should never be
743 static irqreturn_t irq_nested_primary_handler(int irq, void *dev_id)
745 WARN(1, "Primary handler called for nested irq %d\n", irq);
749 static irqreturn_t irq_forced_secondary_handler(int irq, void *dev_id)
751 WARN(1, "Secondary action handler called for irq %d\n", irq);
755 static int irq_wait_for_interrupt(struct irqaction *action)
757 set_current_state(TASK_INTERRUPTIBLE);
759 while (!kthread_should_stop()) {
761 if (test_and_clear_bit(IRQTF_RUNTHREAD,
762 &action->thread_flags)) {
763 __set_current_state(TASK_RUNNING);
767 set_current_state(TASK_INTERRUPTIBLE);
769 __set_current_state(TASK_RUNNING);
774 * Oneshot interrupts keep the irq line masked until the threaded
775 * handler finished. unmask if the interrupt has not been disabled and
778 static void irq_finalize_oneshot(struct irq_desc *desc,
779 struct irqaction *action)
781 if (!(desc->istate & IRQS_ONESHOT) ||
782 action->handler == irq_forced_secondary_handler)
786 raw_spin_lock_irq(&desc->lock);
789 * Implausible though it may be we need to protect us against
790 * the following scenario:
792 * The thread is faster done than the hard interrupt handler
793 * on the other CPU. If we unmask the irq line then the
794 * interrupt can come in again and masks the line, leaves due
795 * to IRQS_INPROGRESS and the irq line is masked forever.
797 * This also serializes the state of shared oneshot handlers
798 * versus "desc->threads_onehsot |= action->thread_mask;" in
799 * irq_wake_thread(). See the comment there which explains the
802 if (unlikely(irqd_irq_inprogress(&desc->irq_data))) {
803 raw_spin_unlock_irq(&desc->lock);
804 chip_bus_sync_unlock(desc);
810 * Now check again, whether the thread should run. Otherwise
811 * we would clear the threads_oneshot bit of this thread which
814 if (test_bit(IRQTF_RUNTHREAD, &action->thread_flags))
817 desc->threads_oneshot &= ~action->thread_mask;
819 if (!desc->threads_oneshot && !irqd_irq_disabled(&desc->irq_data) &&
820 irqd_irq_masked(&desc->irq_data))
821 unmask_threaded_irq(desc);
824 raw_spin_unlock_irq(&desc->lock);
825 chip_bus_sync_unlock(desc);
830 * Check whether we need to change the affinity of the interrupt thread.
833 irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action)
838 if (!test_and_clear_bit(IRQTF_AFFINITY, &action->thread_flags))
842 * In case we are out of memory we set IRQTF_AFFINITY again and
843 * try again next time
845 if (!alloc_cpumask_var(&mask, GFP_KERNEL)) {
846 set_bit(IRQTF_AFFINITY, &action->thread_flags);
850 raw_spin_lock_irq(&desc->lock);
852 * This code is triggered unconditionally. Check the affinity
853 * mask pointer. For CPU_MASK_OFFSTACK=n this is optimized out.
855 if (cpumask_available(desc->irq_common_data.affinity))
856 cpumask_copy(mask, desc->irq_common_data.affinity);
859 raw_spin_unlock_irq(&desc->lock);
862 set_cpus_allowed_ptr(current, mask);
863 free_cpumask_var(mask);
867 irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action) { }
871 * Interrupts which are not explicitely requested as threaded
872 * interrupts rely on the implicit bh/preempt disable of the hard irq
873 * context. So we need to disable bh here to avoid deadlocks and other
877 irq_forced_thread_fn(struct irq_desc *desc, struct irqaction *action)
882 ret = action->thread_fn(action->irq, action->dev_id);
883 irq_finalize_oneshot(desc, action);
889 * Interrupts explicitly requested as threaded interrupts want to be
890 * preemtible - many of them need to sleep and wait for slow busses to
893 static irqreturn_t irq_thread_fn(struct irq_desc *desc,
894 struct irqaction *action)
898 ret = action->thread_fn(action->irq, action->dev_id);
899 irq_finalize_oneshot(desc, action);
903 static void wake_threads_waitq(struct irq_desc *desc)
905 if (atomic_dec_and_test(&desc->threads_active))
906 wake_up(&desc->wait_for_threads);
909 static void irq_thread_dtor(struct callback_head *unused)
911 struct task_struct *tsk = current;
912 struct irq_desc *desc;
913 struct irqaction *action;
915 if (WARN_ON_ONCE(!(current->flags & PF_EXITING)))
918 action = kthread_data(tsk);
920 pr_err("exiting task \"%s\" (%d) is an active IRQ thread (irq %d)\n",
921 tsk->comm, tsk->pid, action->irq);
924 desc = irq_to_desc(action->irq);
926 * If IRQTF_RUNTHREAD is set, we need to decrement
927 * desc->threads_active and wake possible waiters.
929 if (test_and_clear_bit(IRQTF_RUNTHREAD, &action->thread_flags))
930 wake_threads_waitq(desc);
932 /* Prevent a stale desc->threads_oneshot */
933 irq_finalize_oneshot(desc, action);
936 static void irq_wake_secondary(struct irq_desc *desc, struct irqaction *action)
938 struct irqaction *secondary = action->secondary;
940 if (WARN_ON_ONCE(!secondary))
943 raw_spin_lock_irq(&desc->lock);
944 __irq_wake_thread(desc, secondary);
945 raw_spin_unlock_irq(&desc->lock);
949 * Interrupt handler thread
951 static int irq_thread(void *data)
953 struct callback_head on_exit_work;
954 struct irqaction *action = data;
955 struct irq_desc *desc = irq_to_desc(action->irq);
956 irqreturn_t (*handler_fn)(struct irq_desc *desc,
957 struct irqaction *action);
959 if (force_irqthreads && test_bit(IRQTF_FORCED_THREAD,
960 &action->thread_flags))
961 handler_fn = irq_forced_thread_fn;
963 handler_fn = irq_thread_fn;
965 init_task_work(&on_exit_work, irq_thread_dtor);
966 task_work_add(current, &on_exit_work, false);
968 irq_thread_check_affinity(desc, action);
970 while (!irq_wait_for_interrupt(action)) {
971 irqreturn_t action_ret;
973 irq_thread_check_affinity(desc, action);
975 action_ret = handler_fn(desc, action);
976 if (action_ret == IRQ_HANDLED)
977 atomic_inc(&desc->threads_handled);
978 if (action_ret == IRQ_WAKE_THREAD)
979 irq_wake_secondary(desc, action);
981 wake_threads_waitq(desc);
985 * This is the regular exit path. __free_irq() is stopping the
986 * thread via kthread_stop() after calling
987 * synchronize_irq(). So neither IRQTF_RUNTHREAD nor the
988 * oneshot mask bit can be set. We cannot verify that as we
989 * cannot touch the oneshot mask at this point anymore as
990 * __setup_irq() might have given out currents thread_mask
993 task_work_cancel(current, irq_thread_dtor);
998 * irq_wake_thread - wake the irq thread for the action identified by dev_id
999 * @irq: Interrupt line
1000 * @dev_id: Device identity for which the thread should be woken
1003 void irq_wake_thread(unsigned int irq, void *dev_id)
1005 struct irq_desc *desc = irq_to_desc(irq);
1006 struct irqaction *action;
1007 unsigned long flags;
1009 if (!desc || WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1012 raw_spin_lock_irqsave(&desc->lock, flags);
1013 for_each_action_of_desc(desc, action) {
1014 if (action->dev_id == dev_id) {
1016 __irq_wake_thread(desc, action);
1020 raw_spin_unlock_irqrestore(&desc->lock, flags);
1022 EXPORT_SYMBOL_GPL(irq_wake_thread);
1024 static int irq_setup_forced_threading(struct irqaction *new)
1026 if (!force_irqthreads)
1028 if (new->flags & (IRQF_NO_THREAD | IRQF_PERCPU | IRQF_ONESHOT))
1031 new->flags |= IRQF_ONESHOT;
1034 * Handle the case where we have a real primary handler and a
1035 * thread handler. We force thread them as well by creating a
1038 if (new->handler != irq_default_primary_handler && new->thread_fn) {
1039 /* Allocate the secondary action */
1040 new->secondary = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
1041 if (!new->secondary)
1043 new->secondary->handler = irq_forced_secondary_handler;
1044 new->secondary->thread_fn = new->thread_fn;
1045 new->secondary->dev_id = new->dev_id;
1046 new->secondary->irq = new->irq;
1047 new->secondary->name = new->name;
1049 /* Deal with the primary handler */
1050 set_bit(IRQTF_FORCED_THREAD, &new->thread_flags);
1051 new->thread_fn = new->handler;
1052 new->handler = irq_default_primary_handler;
1056 static int irq_request_resources(struct irq_desc *desc)
1058 struct irq_data *d = &desc->irq_data;
1059 struct irq_chip *c = d->chip;
1061 return c->irq_request_resources ? c->irq_request_resources(d) : 0;
1064 static void irq_release_resources(struct irq_desc *desc)
1066 struct irq_data *d = &desc->irq_data;
1067 struct irq_chip *c = d->chip;
1069 if (c->irq_release_resources)
1070 c->irq_release_resources(d);
1074 setup_irq_thread(struct irqaction *new, unsigned int irq, bool secondary)
1076 struct task_struct *t;
1077 struct sched_param param = {
1078 .sched_priority = MAX_USER_RT_PRIO/2,
1082 t = kthread_create(irq_thread, new, "irq/%d-%s", irq,
1085 t = kthread_create(irq_thread, new, "irq/%d-s-%s", irq,
1087 param.sched_priority -= 1;
1093 sched_setscheduler_nocheck(t, SCHED_FIFO, ¶m);
1096 * We keep the reference to the task struct even if
1097 * the thread dies to avoid that the interrupt code
1098 * references an already freed task_struct.
1103 * Tell the thread to set its affinity. This is
1104 * important for shared interrupt handlers as we do
1105 * not invoke setup_affinity() for the secondary
1106 * handlers as everything is already set up. Even for
1107 * interrupts marked with IRQF_NO_BALANCE this is
1108 * correct as we want the thread to move to the cpu(s)
1109 * on which the requesting code placed the interrupt.
1111 set_bit(IRQTF_AFFINITY, &new->thread_flags);
1116 * Internal function to register an irqaction - typically used to
1117 * allocate special interrupts that are part of the architecture.
1120 __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
1122 struct irqaction *old, **old_ptr;
1123 unsigned long flags, thread_mask = 0;
1124 int ret, nested, shared = 0;
1130 if (desc->irq_data.chip == &no_irq_chip)
1132 if (!try_module_get(desc->owner))
1138 * If the trigger type is not specified by the caller,
1139 * then use the default for this interrupt.
1141 if (!(new->flags & IRQF_TRIGGER_MASK))
1142 new->flags |= irqd_get_trigger_type(&desc->irq_data);
1145 * Check whether the interrupt nests into another interrupt
1148 nested = irq_settings_is_nested_thread(desc);
1150 if (!new->thread_fn) {
1155 * Replace the primary handler which was provided from
1156 * the driver for non nested interrupt handling by the
1157 * dummy function which warns when called.
1159 new->handler = irq_nested_primary_handler;
1161 if (irq_settings_can_thread(desc)) {
1162 ret = irq_setup_forced_threading(new);
1169 * Create a handler thread when a thread function is supplied
1170 * and the interrupt does not nest into another interrupt
1173 if (new->thread_fn && !nested) {
1174 ret = setup_irq_thread(new, irq, false);
1177 if (new->secondary) {
1178 ret = setup_irq_thread(new->secondary, irq, true);
1184 if (!alloc_cpumask_var(&mask, GFP_KERNEL)) {
1190 * Drivers are often written to work w/o knowledge about the
1191 * underlying irq chip implementation, so a request for a
1192 * threaded irq without a primary hard irq context handler
1193 * requires the ONESHOT flag to be set. Some irq chips like
1194 * MSI based interrupts are per se one shot safe. Check the
1195 * chip flags, so we can avoid the unmask dance at the end of
1196 * the threaded handler for those.
1198 if (desc->irq_data.chip->flags & IRQCHIP_ONESHOT_SAFE)
1199 new->flags &= ~IRQF_ONESHOT;
1202 * The following block of code has to be executed atomically
1204 raw_spin_lock_irqsave(&desc->lock, flags);
1205 old_ptr = &desc->action;
1209 * Can't share interrupts unless both agree to and are
1210 * the same type (level, edge, polarity). So both flag
1211 * fields must have IRQF_SHARED set and the bits which
1212 * set the trigger type must match. Also all must
1215 unsigned int oldtype = irqd_get_trigger_type(&desc->irq_data);
1217 if (!((old->flags & new->flags) & IRQF_SHARED) ||
1218 (oldtype != (new->flags & IRQF_TRIGGER_MASK)) ||
1219 ((old->flags ^ new->flags) & IRQF_ONESHOT))
1222 /* All handlers must agree on per-cpuness */
1223 if ((old->flags & IRQF_PERCPU) !=
1224 (new->flags & IRQF_PERCPU))
1227 /* add new interrupt at end of irq queue */
1230 * Or all existing action->thread_mask bits,
1231 * so we can find the next zero bit for this
1234 thread_mask |= old->thread_mask;
1235 old_ptr = &old->next;
1242 * Setup the thread mask for this irqaction for ONESHOT. For
1243 * !ONESHOT irqs the thread mask is 0 so we can avoid a
1244 * conditional in irq_wake_thread().
1246 if (new->flags & IRQF_ONESHOT) {
1248 * Unlikely to have 32 resp 64 irqs sharing one line,
1251 if (thread_mask == ~0UL) {
1256 * The thread_mask for the action is or'ed to
1257 * desc->thread_active to indicate that the
1258 * IRQF_ONESHOT thread handler has been woken, but not
1259 * yet finished. The bit is cleared when a thread
1260 * completes. When all threads of a shared interrupt
1261 * line have completed desc->threads_active becomes
1262 * zero and the interrupt line is unmasked. See
1263 * handle.c:irq_wake_thread() for further information.
1265 * If no thread is woken by primary (hard irq context)
1266 * interrupt handlers, then desc->threads_active is
1267 * also checked for zero to unmask the irq line in the
1268 * affected hard irq flow handlers
1269 * (handle_[fasteoi|level]_irq).
1271 * The new action gets the first zero bit of
1272 * thread_mask assigned. See the loop above which or's
1273 * all existing action->thread_mask bits.
1275 new->thread_mask = 1 << ffz(thread_mask);
1277 } else if (new->handler == irq_default_primary_handler &&
1278 !(desc->irq_data.chip->flags & IRQCHIP_ONESHOT_SAFE)) {
1280 * The interrupt was requested with handler = NULL, so
1281 * we use the default primary handler for it. But it
1282 * does not have the oneshot flag set. In combination
1283 * with level interrupts this is deadly, because the
1284 * default primary handler just wakes the thread, then
1285 * the irq lines is reenabled, but the device still
1286 * has the level irq asserted. Rinse and repeat....
1288 * While this works for edge type interrupts, we play
1289 * it safe and reject unconditionally because we can't
1290 * say for sure which type this interrupt really
1291 * has. The type flags are unreliable as the
1292 * underlying chip implementation can override them.
1294 pr_err("Threaded irq requested with handler=NULL and !ONESHOT for irq %d\n",
1301 ret = irq_request_resources(desc);
1303 pr_err("Failed to request resources for %s (irq %d) on irqchip %s\n",
1304 new->name, irq, desc->irq_data.chip->name);
1308 init_waitqueue_head(&desc->wait_for_threads);
1310 /* Setup the type (level, edge polarity) if configured: */
1311 if (new->flags & IRQF_TRIGGER_MASK) {
1312 ret = __irq_set_trigger(desc,
1313 new->flags & IRQF_TRIGGER_MASK);
1319 desc->istate &= ~(IRQS_AUTODETECT | IRQS_SPURIOUS_DISABLED | \
1320 IRQS_ONESHOT | IRQS_WAITING);
1321 irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS);
1323 if (new->flags & IRQF_PERCPU) {
1324 irqd_set(&desc->irq_data, IRQD_PER_CPU);
1325 irq_settings_set_per_cpu(desc);
1328 if (new->flags & IRQF_ONESHOT)
1329 desc->istate |= IRQS_ONESHOT;
1331 if (irq_settings_can_autoenable(desc))
1332 irq_startup(desc, true);
1334 /* Undo nested disables: */
1337 /* Exclude IRQ from balancing if requested */
1338 if (new->flags & IRQF_NOBALANCING) {
1339 irq_settings_set_no_balancing(desc);
1340 irqd_set(&desc->irq_data, IRQD_NO_BALANCING);
1343 /* Set default affinity mask once everything is setup */
1344 setup_affinity(desc, mask);
1346 } else if (new->flags & IRQF_TRIGGER_MASK) {
1347 unsigned int nmsk = new->flags & IRQF_TRIGGER_MASK;
1348 unsigned int omsk = irqd_get_trigger_type(&desc->irq_data);
1351 /* hope the handler works with current trigger mode */
1352 pr_warn("irq %d uses trigger mode %u; requested %u\n",
1358 irq_pm_install_action(desc, new);
1360 /* Reset broken irq detection when installing new handler */
1361 desc->irq_count = 0;
1362 desc->irqs_unhandled = 0;
1365 * Check whether we disabled the irq via the spurious handler
1366 * before. Reenable it and give it another chance.
1368 if (shared && (desc->istate & IRQS_SPURIOUS_DISABLED)) {
1369 desc->istate &= ~IRQS_SPURIOUS_DISABLED;
1373 raw_spin_unlock_irqrestore(&desc->lock, flags);
1376 * Strictly no need to wake it up, but hung_task complains
1377 * when no hard interrupt wakes the thread up.
1380 wake_up_process(new->thread);
1382 wake_up_process(new->secondary->thread);
1384 register_irq_proc(irq, desc);
1386 register_handler_proc(irq, new);
1387 free_cpumask_var(mask);
1392 if (!(new->flags & IRQF_PROBE_SHARED)) {
1393 pr_err("Flags mismatch irq %d. %08x (%s) vs. %08x (%s)\n",
1394 irq, new->flags, new->name, old->flags, old->name);
1395 #ifdef CONFIG_DEBUG_SHIRQ
1402 raw_spin_unlock_irqrestore(&desc->lock, flags);
1403 free_cpumask_var(mask);
1407 struct task_struct *t = new->thread;
1413 if (new->secondary && new->secondary->thread) {
1414 struct task_struct *t = new->secondary->thread;
1416 new->secondary->thread = NULL;
1421 module_put(desc->owner);
1426 * setup_irq - setup an interrupt
1427 * @irq: Interrupt line to setup
1428 * @act: irqaction for the interrupt
1430 * Used to statically setup interrupts in the early boot process.
1432 int setup_irq(unsigned int irq, struct irqaction *act)
1435 struct irq_desc *desc = irq_to_desc(irq);
1437 if (!desc || WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1440 retval = irq_chip_pm_get(&desc->irq_data);
1444 chip_bus_lock(desc);
1445 retval = __setup_irq(irq, desc, act);
1446 chip_bus_sync_unlock(desc);
1449 irq_chip_pm_put(&desc->irq_data);
1453 EXPORT_SYMBOL_GPL(setup_irq);
1456 * Internal function to unregister an irqaction - used to free
1457 * regular and special interrupts that are part of the architecture.
1459 static struct irqaction *__free_irq(unsigned int irq, void *dev_id)
1461 struct irq_desc *desc = irq_to_desc(irq);
1462 struct irqaction *action, **action_ptr;
1463 unsigned long flags;
1465 WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq);
1470 chip_bus_lock(desc);
1471 raw_spin_lock_irqsave(&desc->lock, flags);
1474 * There can be multiple actions per IRQ descriptor, find the right
1475 * one based on the dev_id:
1477 action_ptr = &desc->action;
1479 action = *action_ptr;
1482 WARN(1, "Trying to free already-free IRQ %d\n", irq);
1483 raw_spin_unlock_irqrestore(&desc->lock, flags);
1484 chip_bus_sync_unlock(desc);
1488 if (action->dev_id == dev_id)
1490 action_ptr = &action->next;
1493 /* Found it - now remove it from the list of entries: */
1494 *action_ptr = action->next;
1496 irq_pm_remove_action(desc, action);
1498 /* If this was the last handler, shut down the IRQ line: */
1499 if (!desc->action) {
1500 irq_settings_clr_disable_unlazy(desc);
1502 irq_release_resources(desc);
1506 /* make sure affinity_hint is cleaned up */
1507 if (WARN_ON_ONCE(desc->affinity_hint))
1508 desc->affinity_hint = NULL;
1511 raw_spin_unlock_irqrestore(&desc->lock, flags);
1512 chip_bus_sync_unlock(desc);
1514 unregister_handler_proc(irq, action);
1516 /* Make sure it's not being used on another CPU: */
1517 synchronize_irq(irq);
1519 #ifdef CONFIG_DEBUG_SHIRQ
1521 * It's a shared IRQ -- the driver ought to be prepared for an IRQ
1522 * event to happen even now it's being freed, so let's make sure that
1523 * is so by doing an extra call to the handler ....
1525 * ( We do this after actually deregistering it, to make sure that a
1526 * 'real' IRQ doesn't run in * parallel with our fake. )
1528 if (action->flags & IRQF_SHARED) {
1529 local_irq_save(flags);
1530 action->handler(irq, dev_id);
1531 local_irq_restore(flags);
1535 if (action->thread) {
1536 kthread_stop(action->thread);
1537 put_task_struct(action->thread);
1538 if (action->secondary && action->secondary->thread) {
1539 kthread_stop(action->secondary->thread);
1540 put_task_struct(action->secondary->thread);
1544 irq_chip_pm_put(&desc->irq_data);
1545 module_put(desc->owner);
1546 kfree(action->secondary);
1551 * remove_irq - free an interrupt
1552 * @irq: Interrupt line to free
1553 * @act: irqaction for the interrupt
1555 * Used to remove interrupts statically setup by the early boot process.
1557 void remove_irq(unsigned int irq, struct irqaction *act)
1559 struct irq_desc *desc = irq_to_desc(irq);
1561 if (desc && !WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1562 __free_irq(irq, act->dev_id);
1564 EXPORT_SYMBOL_GPL(remove_irq);
1567 * free_irq - free an interrupt allocated with request_irq
1568 * @irq: Interrupt line to free
1569 * @dev_id: Device identity to free
1571 * Remove an interrupt handler. The handler is removed and if the
1572 * interrupt line is no longer in use by any driver it is disabled.
1573 * On a shared IRQ the caller must ensure the interrupt is disabled
1574 * on the card it drives before calling this function. The function
1575 * does not return until any executing interrupts for this IRQ
1578 * This function must not be called from interrupt context.
1580 * Returns the devname argument passed to request_irq.
1582 const void *free_irq(unsigned int irq, void *dev_id)
1584 struct irq_desc *desc = irq_to_desc(irq);
1585 struct irqaction *action;
1586 const char *devname;
1588 if (!desc || WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1592 if (WARN_ON(desc->affinity_notify))
1593 desc->affinity_notify = NULL;
1596 action = __free_irq(irq, dev_id);
1597 devname = action->name;
1601 EXPORT_SYMBOL(free_irq);
1604 * request_threaded_irq - allocate an interrupt line
1605 * @irq: Interrupt line to allocate
1606 * @handler: Function to be called when the IRQ occurs.
1607 * Primary handler for threaded interrupts
1608 * If NULL and thread_fn != NULL the default
1609 * primary handler is installed
1610 * @thread_fn: Function called from the irq handler thread
1611 * If NULL, no irq thread is created
1612 * @irqflags: Interrupt type flags
1613 * @devname: An ascii name for the claiming device
1614 * @dev_id: A cookie passed back to the handler function
1616 * This call allocates interrupt resources and enables the
1617 * interrupt line and IRQ handling. From the point this
1618 * call is made your handler function may be invoked. Since
1619 * your handler function must clear any interrupt the board
1620 * raises, you must take care both to initialise your hardware
1621 * and to set up the interrupt handler in the right order.
1623 * If you want to set up a threaded irq handler for your device
1624 * then you need to supply @handler and @thread_fn. @handler is
1625 * still called in hard interrupt context and has to check
1626 * whether the interrupt originates from the device. If yes it
1627 * needs to disable the interrupt on the device and return
1628 * IRQ_WAKE_THREAD which will wake up the handler thread and run
1629 * @thread_fn. This split handler design is necessary to support
1630 * shared interrupts.
1632 * Dev_id must be globally unique. Normally the address of the
1633 * device data structure is used as the cookie. Since the handler
1634 * receives this value it makes sense to use it.
1636 * If your interrupt is shared you must pass a non NULL dev_id
1637 * as this is required when freeing the interrupt.
1641 * IRQF_SHARED Interrupt is shared
1642 * IRQF_TRIGGER_* Specify active edge(s) or level
1645 int request_threaded_irq(unsigned int irq, irq_handler_t handler,
1646 irq_handler_t thread_fn, unsigned long irqflags,
1647 const char *devname, void *dev_id)
1649 struct irqaction *action;
1650 struct irq_desc *desc;
1653 if (irq == IRQ_NOTCONNECTED)
1657 * Sanity-check: shared interrupts must pass in a real dev-ID,
1658 * otherwise we'll have trouble later trying to figure out
1659 * which interrupt is which (messes up the interrupt freeing
1662 * Also IRQF_COND_SUSPEND only makes sense for shared interrupts and
1663 * it cannot be set along with IRQF_NO_SUSPEND.
1665 if (((irqflags & IRQF_SHARED) && !dev_id) ||
1666 (!(irqflags & IRQF_SHARED) && (irqflags & IRQF_COND_SUSPEND)) ||
1667 ((irqflags & IRQF_NO_SUSPEND) && (irqflags & IRQF_COND_SUSPEND)))
1670 desc = irq_to_desc(irq);
1674 if (!irq_settings_can_request(desc) ||
1675 WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1681 handler = irq_default_primary_handler;
1684 action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
1688 action->handler = handler;
1689 action->thread_fn = thread_fn;
1690 action->flags = irqflags;
1691 action->name = devname;
1692 action->dev_id = dev_id;
1694 retval = irq_chip_pm_get(&desc->irq_data);
1700 chip_bus_lock(desc);
1701 retval = __setup_irq(irq, desc, action);
1702 chip_bus_sync_unlock(desc);
1705 irq_chip_pm_put(&desc->irq_data);
1706 kfree(action->secondary);
1710 #ifdef CONFIG_DEBUG_SHIRQ_FIXME
1711 if (!retval && (irqflags & IRQF_SHARED)) {
1713 * It's a shared IRQ -- the driver ought to be prepared for it
1714 * to happen immediately, so let's make sure....
1715 * We disable the irq to make sure that a 'real' IRQ doesn't
1716 * run in parallel with our fake.
1718 unsigned long flags;
1721 local_irq_save(flags);
1723 handler(irq, dev_id);
1725 local_irq_restore(flags);
1731 EXPORT_SYMBOL(request_threaded_irq);
1734 * request_any_context_irq - allocate an interrupt line
1735 * @irq: Interrupt line to allocate
1736 * @handler: Function to be called when the IRQ occurs.
1737 * Threaded handler for threaded interrupts.
1738 * @flags: Interrupt type flags
1739 * @name: An ascii name for the claiming device
1740 * @dev_id: A cookie passed back to the handler function
1742 * This call allocates interrupt resources and enables the
1743 * interrupt line and IRQ handling. It selects either a
1744 * hardirq or threaded handling method depending on the
1747 * On failure, it returns a negative value. On success,
1748 * it returns either IRQC_IS_HARDIRQ or IRQC_IS_NESTED.
1750 int request_any_context_irq(unsigned int irq, irq_handler_t handler,
1751 unsigned long flags, const char *name, void *dev_id)
1753 struct irq_desc *desc;
1756 if (irq == IRQ_NOTCONNECTED)
1759 desc = irq_to_desc(irq);
1763 if (irq_settings_is_nested_thread(desc)) {
1764 ret = request_threaded_irq(irq, NULL, handler,
1765 flags, name, dev_id);
1766 return !ret ? IRQC_IS_NESTED : ret;
1769 ret = request_irq(irq, handler, flags, name, dev_id);
1770 return !ret ? IRQC_IS_HARDIRQ : ret;
1772 EXPORT_SYMBOL_GPL(request_any_context_irq);
1774 void enable_percpu_irq(unsigned int irq, unsigned int type)
1776 unsigned int cpu = smp_processor_id();
1777 unsigned long flags;
1778 struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_PERCPU);
1784 * If the trigger type is not specified by the caller, then
1785 * use the default for this interrupt.
1787 type &= IRQ_TYPE_SENSE_MASK;
1788 if (type == IRQ_TYPE_NONE)
1789 type = irqd_get_trigger_type(&desc->irq_data);
1791 if (type != IRQ_TYPE_NONE) {
1794 ret = __irq_set_trigger(desc, type);
1797 WARN(1, "failed to set type for IRQ%d\n", irq);
1802 irq_percpu_enable(desc, cpu);
1804 irq_put_desc_unlock(desc, flags);
1806 EXPORT_SYMBOL_GPL(enable_percpu_irq);
1809 * irq_percpu_is_enabled - Check whether the per cpu irq is enabled
1810 * @irq: Linux irq number to check for
1812 * Must be called from a non migratable context. Returns the enable
1813 * state of a per cpu interrupt on the current cpu.
1815 bool irq_percpu_is_enabled(unsigned int irq)
1817 unsigned int cpu = smp_processor_id();
1818 struct irq_desc *desc;
1819 unsigned long flags;
1822 desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_PERCPU);
1826 is_enabled = cpumask_test_cpu(cpu, desc->percpu_enabled);
1827 irq_put_desc_unlock(desc, flags);
1831 EXPORT_SYMBOL_GPL(irq_percpu_is_enabled);
1833 void disable_percpu_irq(unsigned int irq)
1835 unsigned int cpu = smp_processor_id();
1836 unsigned long flags;
1837 struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_PERCPU);
1842 irq_percpu_disable(desc, cpu);
1843 irq_put_desc_unlock(desc, flags);
1845 EXPORT_SYMBOL_GPL(disable_percpu_irq);
1848 * Internal function to unregister a percpu irqaction.
1850 static struct irqaction *__free_percpu_irq(unsigned int irq, void __percpu *dev_id)
1852 struct irq_desc *desc = irq_to_desc(irq);
1853 struct irqaction *action;
1854 unsigned long flags;
1856 WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq);
1861 raw_spin_lock_irqsave(&desc->lock, flags);
1863 action = desc->action;
1864 if (!action || action->percpu_dev_id != dev_id) {
1865 WARN(1, "Trying to free already-free IRQ %d\n", irq);
1869 if (!cpumask_empty(desc->percpu_enabled)) {
1870 WARN(1, "percpu IRQ %d still enabled on CPU%d!\n",
1871 irq, cpumask_first(desc->percpu_enabled));
1875 /* Found it - now remove it from the list of entries: */
1876 desc->action = NULL;
1878 raw_spin_unlock_irqrestore(&desc->lock, flags);
1880 unregister_handler_proc(irq, action);
1882 irq_chip_pm_put(&desc->irq_data);
1883 module_put(desc->owner);
1887 raw_spin_unlock_irqrestore(&desc->lock, flags);
1892 * remove_percpu_irq - free a per-cpu interrupt
1893 * @irq: Interrupt line to free
1894 * @act: irqaction for the interrupt
1896 * Used to remove interrupts statically setup by the early boot process.
1898 void remove_percpu_irq(unsigned int irq, struct irqaction *act)
1900 struct irq_desc *desc = irq_to_desc(irq);
1902 if (desc && irq_settings_is_per_cpu_devid(desc))
1903 __free_percpu_irq(irq, act->percpu_dev_id);
1907 * free_percpu_irq - free an interrupt allocated with request_percpu_irq
1908 * @irq: Interrupt line to free
1909 * @dev_id: Device identity to free
1911 * Remove a percpu interrupt handler. The handler is removed, but
1912 * the interrupt line is not disabled. This must be done on each
1913 * CPU before calling this function. The function does not return
1914 * until any executing interrupts for this IRQ have completed.
1916 * This function must not be called from interrupt context.
1918 void free_percpu_irq(unsigned int irq, void __percpu *dev_id)
1920 struct irq_desc *desc = irq_to_desc(irq);
1922 if (!desc || !irq_settings_is_per_cpu_devid(desc))
1925 chip_bus_lock(desc);
1926 kfree(__free_percpu_irq(irq, dev_id));
1927 chip_bus_sync_unlock(desc);
1929 EXPORT_SYMBOL_GPL(free_percpu_irq);
1932 * setup_percpu_irq - setup a per-cpu interrupt
1933 * @irq: Interrupt line to setup
1934 * @act: irqaction for the interrupt
1936 * Used to statically setup per-cpu interrupts in the early boot process.
1938 int setup_percpu_irq(unsigned int irq, struct irqaction *act)
1940 struct irq_desc *desc = irq_to_desc(irq);
1943 if (!desc || !irq_settings_is_per_cpu_devid(desc))
1946 retval = irq_chip_pm_get(&desc->irq_data);
1950 chip_bus_lock(desc);
1951 retval = __setup_irq(irq, desc, act);
1952 chip_bus_sync_unlock(desc);
1955 irq_chip_pm_put(&desc->irq_data);
1961 * request_percpu_irq - allocate a percpu interrupt line
1962 * @irq: Interrupt line to allocate
1963 * @handler: Function to be called when the IRQ occurs.
1964 * @devname: An ascii name for the claiming device
1965 * @dev_id: A percpu cookie passed back to the handler function
1967 * This call allocates interrupt resources and enables the
1968 * interrupt on the local CPU. If the interrupt is supposed to be
1969 * enabled on other CPUs, it has to be done on each CPU using
1970 * enable_percpu_irq().
1972 * Dev_id must be globally unique. It is a per-cpu variable, and
1973 * the handler gets called with the interrupted CPU's instance of
1976 int request_percpu_irq(unsigned int irq, irq_handler_t handler,
1977 const char *devname, void __percpu *dev_id)
1979 struct irqaction *action;
1980 struct irq_desc *desc;
1986 desc = irq_to_desc(irq);
1987 if (!desc || !irq_settings_can_request(desc) ||
1988 !irq_settings_is_per_cpu_devid(desc))
1991 action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
1995 action->handler = handler;
1996 action->flags = IRQF_PERCPU | IRQF_NO_SUSPEND;
1997 action->name = devname;
1998 action->percpu_dev_id = dev_id;
2000 retval = irq_chip_pm_get(&desc->irq_data);
2006 chip_bus_lock(desc);
2007 retval = __setup_irq(irq, desc, action);
2008 chip_bus_sync_unlock(desc);
2011 irq_chip_pm_put(&desc->irq_data);
2017 EXPORT_SYMBOL_GPL(request_percpu_irq);
2020 * irq_get_irqchip_state - returns the irqchip state of a interrupt.
2021 * @irq: Interrupt line that is forwarded to a VM
2022 * @which: One of IRQCHIP_STATE_* the caller wants to know about
2023 * @state: a pointer to a boolean where the state is to be storeed
2025 * This call snapshots the internal irqchip state of an
2026 * interrupt, returning into @state the bit corresponding to
2029 * This function should be called with preemption disabled if the
2030 * interrupt controller has per-cpu registers.
2032 int irq_get_irqchip_state(unsigned int irq, enum irqchip_irq_state which,
2035 struct irq_desc *desc;
2036 struct irq_data *data;
2037 struct irq_chip *chip;
2038 unsigned long flags;
2041 desc = irq_get_desc_buslock(irq, &flags, 0);
2045 data = irq_desc_get_irq_data(desc);
2048 chip = irq_data_get_irq_chip(data);
2049 if (chip->irq_get_irqchip_state)
2051 #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
2052 data = data->parent_data;
2059 err = chip->irq_get_irqchip_state(data, which, state);
2061 irq_put_desc_busunlock(desc, flags);
2064 EXPORT_SYMBOL_GPL(irq_get_irqchip_state);
2067 * irq_set_irqchip_state - set the state of a forwarded interrupt.
2068 * @irq: Interrupt line that is forwarded to a VM
2069 * @which: State to be restored (one of IRQCHIP_STATE_*)
2070 * @val: Value corresponding to @which
2072 * This call sets the internal irqchip state of an interrupt,
2073 * depending on the value of @which.
2075 * This function should be called with preemption disabled if the
2076 * interrupt controller has per-cpu registers.
2078 int irq_set_irqchip_state(unsigned int irq, enum irqchip_irq_state which,
2081 struct irq_desc *desc;
2082 struct irq_data *data;
2083 struct irq_chip *chip;
2084 unsigned long flags;
2087 desc = irq_get_desc_buslock(irq, &flags, 0);
2091 data = irq_desc_get_irq_data(desc);
2094 chip = irq_data_get_irq_chip(data);
2095 if (chip->irq_set_irqchip_state)
2097 #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
2098 data = data->parent_data;
2105 err = chip->irq_set_irqchip_state(data, which, val);
2107 irq_put_desc_busunlock(desc, flags);
2110 EXPORT_SYMBOL_GPL(irq_set_irqchip_state);