4 * Xen models interrupts with abstract event channels. Because each
5 * domain gets 1024 event channels, but NR_IRQ is not that large, we
6 * must dynamically map irqs<->event channels. The event channels
7 * interface with the rest of the kernel by defining a xen interrupt
8 * chip. When an event is recieved, it is mapped to an irq and sent
9 * through the normal interrupt processing path.
11 * There are four kinds of events which can be mapped to an event
14 * 1. Inter-domain notifications. This includes all the virtual
15 * device events, since they're driven by front-ends in another domain
17 * 2. VIRQs, typically used for timers. These are per-cpu events.
19 * 4. PIRQs - Hardware interrupts.
21 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
24 #include <linux/linkage.h>
25 #include <linux/interrupt.h>
26 #include <linux/irq.h>
27 #include <linux/module.h>
28 #include <linux/string.h>
29 #include <linux/bootmem.h>
30 #include <linux/slab.h>
31 #include <linux/irqnr.h>
32 #include <linux/pci.h>
35 #include <asm/ptrace.h>
38 #include <asm/io_apic.h>
39 #include <asm/sync_bitops.h>
40 #include <asm/xen/pci.h>
41 #include <asm/xen/hypercall.h>
42 #include <asm/xen/hypervisor.h>
46 #include <xen/xen-ops.h>
47 #include <xen/events.h>
48 #include <xen/interface/xen.h>
49 #include <xen/interface/event_channel.h>
50 #include <xen/interface/hvm/hvm_op.h>
51 #include <xen/interface/hvm/params.h>
54 * This lock protects updates to the following mapping and reference-count
55 * arrays. The lock does not need to be acquired to read the mapping tables.
57 static DEFINE_SPINLOCK(irq_mapping_update_lock);
59 /* IRQ <-> VIRQ mapping. */
60 static DEFINE_PER_CPU(int [NR_VIRQS], virq_to_irq) = {[0 ... NR_VIRQS-1] = -1};
62 /* IRQ <-> IPI mapping */
63 static DEFINE_PER_CPU(int [XEN_NR_IPIS], ipi_to_irq) = {[0 ... XEN_NR_IPIS-1] = -1};
65 /* Interrupt types. */
75 * Packed IRQ information:
76 * type - enum xen_irq_type
77 * event channel - irq->event channel mapping
78 * cpu - cpu this event channel is bound to
79 * index - type-specific information:
80 * PIRQ - vector, with MSB being "needs EIO", or physical IRQ of the HVM
81 * guest, or GSI (real passthrough IRQ) of the device.
88 enum xen_irq_type type; /* type */
89 unsigned short evtchn; /* event channel */
90 unsigned short cpu; /* cpu bound */
103 #define PIRQ_NEEDS_EOI (1 << 0)
104 #define PIRQ_SHAREABLE (1 << 1)
106 static struct irq_info *irq_info;
107 static int *pirq_to_irq;
109 static int *evtchn_to_irq;
111 static DEFINE_PER_CPU(unsigned long [NR_EVENT_CHANNELS/BITS_PER_LONG],
114 /* Xen will never allocate port zero for any purpose. */
115 #define VALID_EVTCHN(chn) ((chn) != 0)
117 static struct irq_chip xen_dynamic_chip;
118 static struct irq_chip xen_percpu_chip;
119 static struct irq_chip xen_pirq_chip;
121 /* Constructor for packed IRQ information. */
122 static struct irq_info mk_unbound_info(void)
124 return (struct irq_info) { .type = IRQT_UNBOUND };
127 static struct irq_info mk_evtchn_info(unsigned short evtchn)
129 return (struct irq_info) { .type = IRQT_EVTCHN, .evtchn = evtchn,
133 static struct irq_info mk_ipi_info(unsigned short evtchn, enum ipi_vector ipi)
135 return (struct irq_info) { .type = IRQT_IPI, .evtchn = evtchn,
136 .cpu = 0, .u.ipi = ipi };
139 static struct irq_info mk_virq_info(unsigned short evtchn, unsigned short virq)
141 return (struct irq_info) { .type = IRQT_VIRQ, .evtchn = evtchn,
142 .cpu = 0, .u.virq = virq };
145 static struct irq_info mk_pirq_info(unsigned short evtchn, unsigned short pirq,
146 unsigned short gsi, unsigned short vector)
148 return (struct irq_info) { .type = IRQT_PIRQ, .evtchn = evtchn,
150 .u.pirq = { .pirq = pirq, .gsi = gsi, .vector = vector } };
154 * Accessors for packed IRQ information.
156 static struct irq_info *info_for_irq(unsigned irq)
158 return &irq_info[irq];
161 static unsigned int evtchn_from_irq(unsigned irq)
163 if (unlikely(WARN(irq < 0 || irq >= nr_irqs, "Invalid irq %d!\n", irq)))
166 return info_for_irq(irq)->evtchn;
169 unsigned irq_from_evtchn(unsigned int evtchn)
171 return evtchn_to_irq[evtchn];
173 EXPORT_SYMBOL_GPL(irq_from_evtchn);
175 static enum ipi_vector ipi_from_irq(unsigned irq)
177 struct irq_info *info = info_for_irq(irq);
179 BUG_ON(info == NULL);
180 BUG_ON(info->type != IRQT_IPI);
185 static unsigned virq_from_irq(unsigned irq)
187 struct irq_info *info = info_for_irq(irq);
189 BUG_ON(info == NULL);
190 BUG_ON(info->type != IRQT_VIRQ);
195 static unsigned pirq_from_irq(unsigned irq)
197 struct irq_info *info = info_for_irq(irq);
199 BUG_ON(info == NULL);
200 BUG_ON(info->type != IRQT_PIRQ);
202 return info->u.pirq.pirq;
205 static unsigned gsi_from_irq(unsigned irq)
207 struct irq_info *info = info_for_irq(irq);
209 BUG_ON(info == NULL);
210 BUG_ON(info->type != IRQT_PIRQ);
212 return info->u.pirq.gsi;
215 static enum xen_irq_type type_from_irq(unsigned irq)
217 return info_for_irq(irq)->type;
220 static unsigned cpu_from_irq(unsigned irq)
222 return info_for_irq(irq)->cpu;
225 static unsigned int cpu_from_evtchn(unsigned int evtchn)
227 int irq = evtchn_to_irq[evtchn];
231 ret = cpu_from_irq(irq);
236 static bool pirq_needs_eoi(unsigned irq)
238 struct irq_info *info = info_for_irq(irq);
240 BUG_ON(info->type != IRQT_PIRQ);
242 return info->u.pirq.flags & PIRQ_NEEDS_EOI;
245 static inline unsigned long active_evtchns(unsigned int cpu,
246 struct shared_info *sh,
249 return (sh->evtchn_pending[idx] &
250 per_cpu(cpu_evtchn_mask, cpu)[idx] &
251 ~sh->evtchn_mask[idx]);
254 static void bind_evtchn_to_cpu(unsigned int chn, unsigned int cpu)
256 int irq = evtchn_to_irq[chn];
260 cpumask_copy(irq_to_desc(irq)->irq_data.affinity, cpumask_of(cpu));
263 clear_bit(chn, per_cpu(cpu_evtchn_mask, cpu_from_irq(irq)));
264 set_bit(chn, per_cpu(cpu_evtchn_mask, cpu));
266 irq_info[irq].cpu = cpu;
269 static void init_evtchn_cpu_bindings(void)
273 struct irq_desc *desc;
275 /* By default all event channels notify CPU#0. */
276 for_each_irq_desc(i, desc) {
277 cpumask_copy(desc->irq_data.affinity, cpumask_of(0));
281 for_each_possible_cpu(i)
282 memset(per_cpu(cpu_evtchn_mask, i),
283 (i == 0) ? ~0 : 0, sizeof(*per_cpu(cpu_evtchn_mask, i)));
287 static inline void clear_evtchn(int port)
289 struct shared_info *s = HYPERVISOR_shared_info;
290 sync_clear_bit(port, &s->evtchn_pending[0]);
293 static inline void set_evtchn(int port)
295 struct shared_info *s = HYPERVISOR_shared_info;
296 sync_set_bit(port, &s->evtchn_pending[0]);
299 static inline int test_evtchn(int port)
301 struct shared_info *s = HYPERVISOR_shared_info;
302 return sync_test_bit(port, &s->evtchn_pending[0]);
307 * notify_remote_via_irq - send event to remote end of event channel via irq
308 * @irq: irq of event channel to send event to
310 * Unlike notify_remote_via_evtchn(), this is safe to use across
311 * save/restore. Notifications on a broken connection are silently
314 void notify_remote_via_irq(int irq)
316 int evtchn = evtchn_from_irq(irq);
318 if (VALID_EVTCHN(evtchn))
319 notify_remote_via_evtchn(evtchn);
321 EXPORT_SYMBOL_GPL(notify_remote_via_irq);
323 static void mask_evtchn(int port)
325 struct shared_info *s = HYPERVISOR_shared_info;
326 sync_set_bit(port, &s->evtchn_mask[0]);
329 static void unmask_evtchn(int port)
331 struct shared_info *s = HYPERVISOR_shared_info;
332 unsigned int cpu = get_cpu();
334 BUG_ON(!irqs_disabled());
336 /* Slow path (hypercall) if this is a non-local port. */
337 if (unlikely(cpu != cpu_from_evtchn(port))) {
338 struct evtchn_unmask unmask = { .port = port };
339 (void)HYPERVISOR_event_channel_op(EVTCHNOP_unmask, &unmask);
341 struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu);
343 sync_clear_bit(port, &s->evtchn_mask[0]);
346 * The following is basically the equivalent of
347 * 'hw_resend_irq'. Just like a real IO-APIC we 'lose
348 * the interrupt edge' if the channel is masked.
350 if (sync_test_bit(port, &s->evtchn_pending[0]) &&
351 !sync_test_and_set_bit(port / BITS_PER_LONG,
352 &vcpu_info->evtchn_pending_sel))
353 vcpu_info->evtchn_upcall_pending = 1;
359 static int xen_allocate_irq_dynamic(void)
364 #ifdef CONFIG_X86_IO_APIC
366 * For an HVM guest or domain 0 which see "real" (emulated or
367 * actual repectively) GSIs we allocate dynamic IRQs
368 * e.g. those corresponding to event channels or MSIs
369 * etc. from the range above those "real" GSIs to avoid
372 if (xen_initial_domain() || xen_hvm_domain())
373 first = get_nr_irqs_gsi();
377 irq = irq_alloc_desc_from(first, -1);
379 if (irq == -ENOMEM && first > NR_IRQS_LEGACY) {
380 printk(KERN_ERR "Out of dynamic IRQ space and eating into GSI space. You should increase nr_irqs\n");
381 first = max(NR_IRQS_LEGACY, first - NR_IRQS_LEGACY);
386 panic("No available IRQ to bind to: increase nr_irqs!\n");
391 static int xen_allocate_irq_gsi(unsigned gsi)
396 * A PV guest has no concept of a GSI (since it has no ACPI
397 * nor access to/knowledge of the physical APICs). Therefore
398 * all IRQs are dynamically allocated from the entire IRQ
401 if (xen_pv_domain() && !xen_initial_domain())
402 return xen_allocate_irq_dynamic();
404 /* Legacy IRQ descriptors are already allocated by the arch. */
405 if (gsi < NR_IRQS_LEGACY)
408 irq = irq_alloc_desc_at(gsi, -1);
410 panic("Unable to allocate to IRQ%d (%d)\n", gsi, irq);
415 static void xen_free_irq(unsigned irq)
417 /* Legacy IRQ descriptors are managed by the arch. */
418 if (irq < NR_IRQS_LEGACY)
424 static void pirq_unmask_notify(int irq)
426 struct physdev_eoi eoi = { .irq = pirq_from_irq(irq) };
428 if (unlikely(pirq_needs_eoi(irq))) {
429 int rc = HYPERVISOR_physdev_op(PHYSDEVOP_eoi, &eoi);
434 static void pirq_query_unmask(int irq)
436 struct physdev_irq_status_query irq_status;
437 struct irq_info *info = info_for_irq(irq);
439 BUG_ON(info->type != IRQT_PIRQ);
441 irq_status.irq = pirq_from_irq(irq);
442 if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status))
443 irq_status.flags = 0;
445 info->u.pirq.flags &= ~PIRQ_NEEDS_EOI;
446 if (irq_status.flags & XENIRQSTAT_needs_eoi)
447 info->u.pirq.flags |= PIRQ_NEEDS_EOI;
450 static bool probing_irq(int irq)
452 struct irq_desc *desc = irq_to_desc(irq);
454 return desc && desc->action == NULL;
457 static unsigned int __startup_pirq(unsigned int irq)
459 struct evtchn_bind_pirq bind_pirq;
460 struct irq_info *info = info_for_irq(irq);
461 int evtchn = evtchn_from_irq(irq);
464 BUG_ON(info->type != IRQT_PIRQ);
466 if (VALID_EVTCHN(evtchn))
469 bind_pirq.pirq = pirq_from_irq(irq);
470 /* NB. We are happy to share unless we are probing. */
471 bind_pirq.flags = info->u.pirq.flags & PIRQ_SHAREABLE ?
472 BIND_PIRQ__WILL_SHARE : 0;
473 rc = HYPERVISOR_event_channel_op(EVTCHNOP_bind_pirq, &bind_pirq);
475 if (!probing_irq(irq))
476 printk(KERN_INFO "Failed to obtain physical IRQ %d\n",
480 evtchn = bind_pirq.port;
482 pirq_query_unmask(irq);
484 evtchn_to_irq[evtchn] = irq;
485 bind_evtchn_to_cpu(evtchn, 0);
486 info->evtchn = evtchn;
489 unmask_evtchn(evtchn);
490 pirq_unmask_notify(irq);
495 static unsigned int startup_pirq(struct irq_data *data)
497 return __startup_pirq(data->irq);
500 static void shutdown_pirq(struct irq_data *data)
502 struct evtchn_close close;
503 unsigned int irq = data->irq;
504 struct irq_info *info = info_for_irq(irq);
505 int evtchn = evtchn_from_irq(irq);
507 BUG_ON(info->type != IRQT_PIRQ);
509 if (!VALID_EVTCHN(evtchn))
515 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close) != 0)
518 bind_evtchn_to_cpu(evtchn, 0);
519 evtchn_to_irq[evtchn] = -1;
523 static void enable_pirq(struct irq_data *data)
528 static void disable_pirq(struct irq_data *data)
532 static void ack_pirq(struct irq_data *data)
534 int evtchn = evtchn_from_irq(data->irq);
536 move_native_irq(data->irq);
538 if (VALID_EVTCHN(evtchn)) {
540 clear_evtchn(evtchn);
544 static int find_irq_by_gsi(unsigned gsi)
548 for (irq = 0; irq < nr_irqs; irq++) {
549 struct irq_info *info = info_for_irq(irq);
551 if (info == NULL || info->type != IRQT_PIRQ)
554 if (gsi_from_irq(irq) == gsi)
561 int xen_allocate_pirq_gsi(unsigned gsi)
567 * Do not make any assumptions regarding the relationship between the
568 * IRQ number returned here and the Xen pirq argument.
570 * Note: We don't assign an event channel until the irq actually started
571 * up. Return an existing irq if we've already got one for the gsi.
573 int xen_bind_pirq_gsi_to_irq(unsigned gsi,
574 unsigned pirq, int shareable, char *name)
577 struct physdev_irq irq_op;
579 spin_lock(&irq_mapping_update_lock);
581 if ((pirq > nr_irqs) || (gsi > nr_irqs)) {
582 printk(KERN_WARNING "xen_map_pirq_gsi: %s %s is incorrect!\n",
583 pirq > nr_irqs ? "pirq" :"",
584 gsi > nr_irqs ? "gsi" : "");
588 irq = find_irq_by_gsi(gsi);
590 printk(KERN_INFO "xen_map_pirq_gsi: returning irq %d for gsi %u\n",
592 goto out; /* XXX need refcount? */
595 irq = xen_allocate_irq_gsi(gsi);
597 set_irq_chip_and_handler_name(irq, &xen_pirq_chip,
598 handle_level_irq, name);
603 /* Only the privileged domain can do this. For non-priv, the pcifront
604 * driver provides a PCI bus that does the call to do exactly
605 * this in the priv domain. */
606 if (xen_initial_domain() &&
607 HYPERVISOR_physdev_op(PHYSDEVOP_alloc_irq_vector, &irq_op)) {
613 irq_info[irq] = mk_pirq_info(0, pirq, gsi, irq_op.vector);
614 irq_info[irq].u.pirq.flags |= shareable ? PIRQ_SHAREABLE : 0;
615 pirq_to_irq[pirq] = irq;
618 spin_unlock(&irq_mapping_update_lock);
623 #ifdef CONFIG_PCI_MSI
624 int xen_allocate_pirq_msi(struct pci_dev *dev, struct msi_desc *msidesc)
627 struct physdev_get_free_pirq op_get_free_pirq;
629 op_get_free_pirq.type = MAP_PIRQ_TYPE_MSI;
630 rc = HYPERVISOR_physdev_op(PHYSDEVOP_get_free_pirq, &op_get_free_pirq);
632 WARN_ONCE(rc == -ENOSYS,
633 "hypervisor does not support the PHYSDEVOP_get_free_pirq interface\n");
635 return rc ? -1 : op_get_free_pirq.pirq;
638 int xen_bind_pirq_msi_to_irq(struct pci_dev *dev, struct msi_desc *msidesc,
639 int pirq, int vector, const char *name)
643 spin_lock(&irq_mapping_update_lock);
645 irq = xen_allocate_irq_dynamic();
649 set_irq_chip_and_handler_name(irq, &xen_pirq_chip,
650 handle_level_irq, name);
652 irq_info[irq] = mk_pirq_info(0, pirq, 0, vector);
653 pirq_to_irq[pirq] = irq;
654 ret = set_irq_msi(irq, msidesc);
658 spin_unlock(&irq_mapping_update_lock);
661 spin_unlock(&irq_mapping_update_lock);
667 int xen_destroy_irq(int irq)
669 struct irq_desc *desc;
670 struct physdev_unmap_pirq unmap_irq;
671 struct irq_info *info = info_for_irq(irq);
674 spin_lock(&irq_mapping_update_lock);
676 desc = irq_to_desc(irq);
680 if (xen_initial_domain()) {
681 unmap_irq.pirq = info->u.pirq.pirq;
682 unmap_irq.domid = DOMID_SELF;
683 rc = HYPERVISOR_physdev_op(PHYSDEVOP_unmap_pirq, &unmap_irq);
685 printk(KERN_WARNING "unmap irq failed %d\n", rc);
689 pirq_to_irq[info->u.pirq.pirq] = -1;
691 irq_info[irq] = mk_unbound_info();
696 spin_unlock(&irq_mapping_update_lock);
700 int xen_irq_from_pirq(unsigned pirq)
702 return pirq_to_irq[pirq];
705 int bind_evtchn_to_irq(unsigned int evtchn)
709 spin_lock(&irq_mapping_update_lock);
711 irq = evtchn_to_irq[evtchn];
714 irq = xen_allocate_irq_dynamic();
716 set_irq_chip_and_handler_name(irq, &xen_dynamic_chip,
717 handle_fasteoi_irq, "event");
719 evtchn_to_irq[evtchn] = irq;
720 irq_info[irq] = mk_evtchn_info(evtchn);
723 spin_unlock(&irq_mapping_update_lock);
727 EXPORT_SYMBOL_GPL(bind_evtchn_to_irq);
729 static int bind_ipi_to_irq(unsigned int ipi, unsigned int cpu)
731 struct evtchn_bind_ipi bind_ipi;
734 spin_lock(&irq_mapping_update_lock);
736 irq = per_cpu(ipi_to_irq, cpu)[ipi];
739 irq = xen_allocate_irq_dynamic();
743 set_irq_chip_and_handler_name(irq, &xen_percpu_chip,
744 handle_percpu_irq, "ipi");
747 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
750 evtchn = bind_ipi.port;
752 evtchn_to_irq[evtchn] = irq;
753 irq_info[irq] = mk_ipi_info(evtchn, ipi);
754 per_cpu(ipi_to_irq, cpu)[ipi] = irq;
756 bind_evtchn_to_cpu(evtchn, cpu);
760 spin_unlock(&irq_mapping_update_lock);
765 int bind_virq_to_irq(unsigned int virq, unsigned int cpu)
767 struct evtchn_bind_virq bind_virq;
770 spin_lock(&irq_mapping_update_lock);
772 irq = per_cpu(virq_to_irq, cpu)[virq];
775 irq = xen_allocate_irq_dynamic();
777 set_irq_chip_and_handler_name(irq, &xen_percpu_chip,
778 handle_percpu_irq, "virq");
780 bind_virq.virq = virq;
781 bind_virq.vcpu = cpu;
782 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
785 evtchn = bind_virq.port;
787 evtchn_to_irq[evtchn] = irq;
788 irq_info[irq] = mk_virq_info(evtchn, virq);
790 per_cpu(virq_to_irq, cpu)[virq] = irq;
792 bind_evtchn_to_cpu(evtchn, cpu);
795 spin_unlock(&irq_mapping_update_lock);
800 static void unbind_from_irq(unsigned int irq)
802 struct evtchn_close close;
803 int evtchn = evtchn_from_irq(irq);
805 spin_lock(&irq_mapping_update_lock);
807 if (VALID_EVTCHN(evtchn)) {
809 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close) != 0)
812 switch (type_from_irq(irq)) {
814 per_cpu(virq_to_irq, cpu_from_evtchn(evtchn))
815 [virq_from_irq(irq)] = -1;
818 per_cpu(ipi_to_irq, cpu_from_evtchn(evtchn))
819 [ipi_from_irq(irq)] = -1;
825 /* Closed ports are implicitly re-bound to VCPU0. */
826 bind_evtchn_to_cpu(evtchn, 0);
828 evtchn_to_irq[evtchn] = -1;
831 if (irq_info[irq].type != IRQT_UNBOUND) {
832 irq_info[irq] = mk_unbound_info();
837 spin_unlock(&irq_mapping_update_lock);
840 int bind_evtchn_to_irqhandler(unsigned int evtchn,
841 irq_handler_t handler,
842 unsigned long irqflags,
843 const char *devname, void *dev_id)
848 irq = bind_evtchn_to_irq(evtchn);
849 retval = request_irq(irq, handler, irqflags, devname, dev_id);
851 unbind_from_irq(irq);
857 EXPORT_SYMBOL_GPL(bind_evtchn_to_irqhandler);
859 int bind_virq_to_irqhandler(unsigned int virq, unsigned int cpu,
860 irq_handler_t handler,
861 unsigned long irqflags, const char *devname, void *dev_id)
866 irq = bind_virq_to_irq(virq, cpu);
867 retval = request_irq(irq, handler, irqflags, devname, dev_id);
869 unbind_from_irq(irq);
875 EXPORT_SYMBOL_GPL(bind_virq_to_irqhandler);
877 int bind_ipi_to_irqhandler(enum ipi_vector ipi,
879 irq_handler_t handler,
880 unsigned long irqflags,
886 irq = bind_ipi_to_irq(ipi, cpu);
890 irqflags |= IRQF_NO_SUSPEND | IRQF_FORCE_RESUME;
891 retval = request_irq(irq, handler, irqflags, devname, dev_id);
893 unbind_from_irq(irq);
900 void unbind_from_irqhandler(unsigned int irq, void *dev_id)
902 free_irq(irq, dev_id);
903 unbind_from_irq(irq);
905 EXPORT_SYMBOL_GPL(unbind_from_irqhandler);
907 void xen_send_IPI_one(unsigned int cpu, enum ipi_vector vector)
909 int irq = per_cpu(ipi_to_irq, cpu)[vector];
911 notify_remote_via_irq(irq);
914 irqreturn_t xen_debug_interrupt(int irq, void *dev_id)
916 struct shared_info *sh = HYPERVISOR_shared_info;
917 int cpu = smp_processor_id();
918 unsigned long *cpu_evtchn = per_cpu(cpu_evtchn_mask, cpu);
921 static DEFINE_SPINLOCK(debug_lock);
924 spin_lock_irqsave(&debug_lock, flags);
926 printk("\nvcpu %d\n ", cpu);
928 for_each_online_cpu(i) {
930 v = per_cpu(xen_vcpu, i);
931 pending = (get_irq_regs() && i == cpu)
932 ? xen_irqs_disabled(get_irq_regs())
933 : v->evtchn_upcall_mask;
934 printk("%d: masked=%d pending=%d event_sel %0*lx\n ", i,
935 pending, v->evtchn_upcall_pending,
936 (int)(sizeof(v->evtchn_pending_sel)*2),
937 v->evtchn_pending_sel);
939 v = per_cpu(xen_vcpu, cpu);
941 printk("\npending:\n ");
942 for (i = ARRAY_SIZE(sh->evtchn_pending)-1; i >= 0; i--)
943 printk("%0*lx%s", (int)sizeof(sh->evtchn_pending[0])*2,
944 sh->evtchn_pending[i],
945 i % 8 == 0 ? "\n " : " ");
946 printk("\nglobal mask:\n ");
947 for (i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--)
949 (int)(sizeof(sh->evtchn_mask[0])*2),
951 i % 8 == 0 ? "\n " : " ");
953 printk("\nglobally unmasked:\n ");
954 for (i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--)
955 printk("%0*lx%s", (int)(sizeof(sh->evtchn_mask[0])*2),
956 sh->evtchn_pending[i] & ~sh->evtchn_mask[i],
957 i % 8 == 0 ? "\n " : " ");
959 printk("\nlocal cpu%d mask:\n ", cpu);
960 for (i = (NR_EVENT_CHANNELS/BITS_PER_LONG)-1; i >= 0; i--)
961 printk("%0*lx%s", (int)(sizeof(cpu_evtchn[0])*2),
963 i % 8 == 0 ? "\n " : " ");
965 printk("\nlocally unmasked:\n ");
966 for (i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--) {
967 unsigned long pending = sh->evtchn_pending[i]
968 & ~sh->evtchn_mask[i]
970 printk("%0*lx%s", (int)(sizeof(sh->evtchn_mask[0])*2),
971 pending, i % 8 == 0 ? "\n " : " ");
974 printk("\npending list:\n");
975 for (i = 0; i < NR_EVENT_CHANNELS; i++) {
976 if (sync_test_bit(i, sh->evtchn_pending)) {
977 int word_idx = i / BITS_PER_LONG;
978 printk(" %d: event %d -> irq %d%s%s%s\n",
979 cpu_from_evtchn(i), i,
981 sync_test_bit(word_idx, &v->evtchn_pending_sel)
983 !sync_test_bit(i, sh->evtchn_mask)
984 ? "" : " globally-masked",
985 sync_test_bit(i, cpu_evtchn)
986 ? "" : " locally-masked");
990 spin_unlock_irqrestore(&debug_lock, flags);
995 static DEFINE_PER_CPU(unsigned, xed_nesting_count);
998 * Search the CPUs pending events bitmasks. For each one found, map
999 * the event number to an irq, and feed it into do_IRQ() for
1002 * Xen uses a two-level bitmap to speed searching. The first level is
1003 * a bitset of words which contain pending event bits. The second
1004 * level is a bitset of pending events themselves.
1006 static void __xen_evtchn_do_upcall(void)
1008 int cpu = get_cpu();
1009 struct shared_info *s = HYPERVISOR_shared_info;
1010 struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu);
1014 unsigned long pending_words;
1016 vcpu_info->evtchn_upcall_pending = 0;
1018 if (__this_cpu_inc_return(xed_nesting_count) - 1)
1021 #ifndef CONFIG_X86 /* No need for a barrier -- XCHG is a barrier on x86. */
1022 /* Clear master flag /before/ clearing selector flag. */
1025 pending_words = xchg(&vcpu_info->evtchn_pending_sel, 0);
1026 while (pending_words != 0) {
1027 unsigned long pending_bits;
1028 int word_idx = __ffs(pending_words);
1029 pending_words &= ~(1UL << word_idx);
1031 while ((pending_bits = active_evtchns(cpu, s, word_idx)) != 0) {
1032 int bit_idx = __ffs(pending_bits);
1033 int port = (word_idx * BITS_PER_LONG) + bit_idx;
1034 int irq = evtchn_to_irq[port];
1035 struct irq_desc *desc;
1041 desc = irq_to_desc(irq);
1043 generic_handle_irq_desc(irq, desc);
1048 BUG_ON(!irqs_disabled());
1050 count = __this_cpu_read(xed_nesting_count);
1051 __this_cpu_write(xed_nesting_count, 0);
1052 } while (count != 1 || vcpu_info->evtchn_upcall_pending);
1059 void xen_evtchn_do_upcall(struct pt_regs *regs)
1061 struct pt_regs *old_regs = set_irq_regs(regs);
1066 __xen_evtchn_do_upcall();
1069 set_irq_regs(old_regs);
1072 void xen_hvm_evtchn_do_upcall(void)
1074 __xen_evtchn_do_upcall();
1076 EXPORT_SYMBOL_GPL(xen_hvm_evtchn_do_upcall);
1078 /* Rebind a new event channel to an existing irq. */
1079 void rebind_evtchn_irq(int evtchn, int irq)
1081 struct irq_info *info = info_for_irq(irq);
1083 /* Make sure the irq is masked, since the new event channel
1084 will also be masked. */
1087 spin_lock(&irq_mapping_update_lock);
1089 /* After resume the irq<->evtchn mappings are all cleared out */
1090 BUG_ON(evtchn_to_irq[evtchn] != -1);
1091 /* Expect irq to have been bound before,
1092 so there should be a proper type */
1093 BUG_ON(info->type == IRQT_UNBOUND);
1095 evtchn_to_irq[evtchn] = irq;
1096 irq_info[irq] = mk_evtchn_info(evtchn);
1098 spin_unlock(&irq_mapping_update_lock);
1100 /* new event channels are always bound to cpu 0 */
1101 irq_set_affinity(irq, cpumask_of(0));
1103 /* Unmask the event channel. */
1107 /* Rebind an evtchn so that it gets delivered to a specific cpu */
1108 static int rebind_irq_to_cpu(unsigned irq, unsigned tcpu)
1110 struct evtchn_bind_vcpu bind_vcpu;
1111 int evtchn = evtchn_from_irq(irq);
1113 if (!VALID_EVTCHN(evtchn))
1117 * Events delivered via platform PCI interrupts are always
1118 * routed to vcpu 0 and hence cannot be rebound.
1120 if (xen_hvm_domain() && !xen_have_vector_callback)
1123 /* Send future instances of this interrupt to other vcpu. */
1124 bind_vcpu.port = evtchn;
1125 bind_vcpu.vcpu = tcpu;
1128 * If this fails, it usually just indicates that we're dealing with a
1129 * virq or IPI channel, which don't actually need to be rebound. Ignore
1130 * it, but don't do the xenlinux-level rebind in that case.
1132 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu) >= 0)
1133 bind_evtchn_to_cpu(evtchn, tcpu);
1138 static int set_affinity_irq(struct irq_data *data, const struct cpumask *dest,
1141 unsigned tcpu = cpumask_first(dest);
1143 return rebind_irq_to_cpu(data->irq, tcpu);
1146 int resend_irq_on_evtchn(unsigned int irq)
1148 int masked, evtchn = evtchn_from_irq(irq);
1149 struct shared_info *s = HYPERVISOR_shared_info;
1151 if (!VALID_EVTCHN(evtchn))
1154 masked = sync_test_and_set_bit(evtchn, s->evtchn_mask);
1155 sync_set_bit(evtchn, s->evtchn_pending);
1157 unmask_evtchn(evtchn);
1162 static void enable_dynirq(struct irq_data *data)
1164 int evtchn = evtchn_from_irq(data->irq);
1166 if (VALID_EVTCHN(evtchn))
1167 unmask_evtchn(evtchn);
1170 static void disable_dynirq(struct irq_data *data)
1172 int evtchn = evtchn_from_irq(data->irq);
1174 if (VALID_EVTCHN(evtchn))
1175 mask_evtchn(evtchn);
1178 static void ack_dynirq(struct irq_data *data)
1180 int evtchn = evtchn_from_irq(data->irq);
1182 move_masked_irq(data->irq);
1184 if (VALID_EVTCHN(evtchn))
1185 unmask_evtchn(evtchn);
1188 static int retrigger_dynirq(struct irq_data *data)
1190 int evtchn = evtchn_from_irq(data->irq);
1191 struct shared_info *sh = HYPERVISOR_shared_info;
1194 if (VALID_EVTCHN(evtchn)) {
1197 masked = sync_test_and_set_bit(evtchn, sh->evtchn_mask);
1198 sync_set_bit(evtchn, sh->evtchn_pending);
1200 unmask_evtchn(evtchn);
1207 static void restore_pirqs(void)
1209 int pirq, rc, irq, gsi;
1210 struct physdev_map_pirq map_irq;
1212 for (pirq = 0; pirq < nr_irqs; pirq++) {
1213 irq = pirq_to_irq[pirq];
1217 /* save/restore of PT devices doesn't work, so at this point the
1218 * only devices present are GSI based emulated devices */
1219 gsi = gsi_from_irq(irq);
1223 map_irq.domid = DOMID_SELF;
1224 map_irq.type = MAP_PIRQ_TYPE_GSI;
1225 map_irq.index = gsi;
1226 map_irq.pirq = pirq;
1228 rc = HYPERVISOR_physdev_op(PHYSDEVOP_map_pirq, &map_irq);
1230 printk(KERN_WARNING "xen map irq failed gsi=%d irq=%d pirq=%d rc=%d\n",
1231 gsi, irq, pirq, rc);
1232 irq_info[irq] = mk_unbound_info();
1233 pirq_to_irq[pirq] = -1;
1237 printk(KERN_DEBUG "xen: --> irq=%d, pirq=%d\n", irq, map_irq.pirq);
1239 __startup_pirq(irq);
1243 static void restore_cpu_virqs(unsigned int cpu)
1245 struct evtchn_bind_virq bind_virq;
1246 int virq, irq, evtchn;
1248 for (virq = 0; virq < NR_VIRQS; virq++) {
1249 if ((irq = per_cpu(virq_to_irq, cpu)[virq]) == -1)
1252 BUG_ON(virq_from_irq(irq) != virq);
1254 /* Get a new binding from Xen. */
1255 bind_virq.virq = virq;
1256 bind_virq.vcpu = cpu;
1257 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
1260 evtchn = bind_virq.port;
1262 /* Record the new mapping. */
1263 evtchn_to_irq[evtchn] = irq;
1264 irq_info[irq] = mk_virq_info(evtchn, virq);
1265 bind_evtchn_to_cpu(evtchn, cpu);
1269 static void restore_cpu_ipis(unsigned int cpu)
1271 struct evtchn_bind_ipi bind_ipi;
1272 int ipi, irq, evtchn;
1274 for (ipi = 0; ipi < XEN_NR_IPIS; ipi++) {
1275 if ((irq = per_cpu(ipi_to_irq, cpu)[ipi]) == -1)
1278 BUG_ON(ipi_from_irq(irq) != ipi);
1280 /* Get a new binding from Xen. */
1281 bind_ipi.vcpu = cpu;
1282 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
1285 evtchn = bind_ipi.port;
1287 /* Record the new mapping. */
1288 evtchn_to_irq[evtchn] = irq;
1289 irq_info[irq] = mk_ipi_info(evtchn, ipi);
1290 bind_evtchn_to_cpu(evtchn, cpu);
1294 /* Clear an irq's pending state, in preparation for polling on it */
1295 void xen_clear_irq_pending(int irq)
1297 int evtchn = evtchn_from_irq(irq);
1299 if (VALID_EVTCHN(evtchn))
1300 clear_evtchn(evtchn);
1302 EXPORT_SYMBOL(xen_clear_irq_pending);
1303 void xen_set_irq_pending(int irq)
1305 int evtchn = evtchn_from_irq(irq);
1307 if (VALID_EVTCHN(evtchn))
1311 bool xen_test_irq_pending(int irq)
1313 int evtchn = evtchn_from_irq(irq);
1316 if (VALID_EVTCHN(evtchn))
1317 ret = test_evtchn(evtchn);
1322 /* Poll waiting for an irq to become pending with timeout. In the usual case,
1323 * the irq will be disabled so it won't deliver an interrupt. */
1324 void xen_poll_irq_timeout(int irq, u64 timeout)
1326 evtchn_port_t evtchn = evtchn_from_irq(irq);
1328 if (VALID_EVTCHN(evtchn)) {
1329 struct sched_poll poll;
1332 poll.timeout = timeout;
1333 set_xen_guest_handle(poll.ports, &evtchn);
1335 if (HYPERVISOR_sched_op(SCHEDOP_poll, &poll) != 0)
1339 EXPORT_SYMBOL(xen_poll_irq_timeout);
1340 /* Poll waiting for an irq to become pending. In the usual case, the
1341 * irq will be disabled so it won't deliver an interrupt. */
1342 void xen_poll_irq(int irq)
1344 xen_poll_irq_timeout(irq, 0 /* no timeout */);
1347 void xen_irq_resume(void)
1349 unsigned int cpu, irq, evtchn;
1351 init_evtchn_cpu_bindings();
1353 /* New event-channel space is not 'live' yet. */
1354 for (evtchn = 0; evtchn < NR_EVENT_CHANNELS; evtchn++)
1355 mask_evtchn(evtchn);
1357 /* No IRQ <-> event-channel mappings. */
1358 for (irq = 0; irq < nr_irqs; irq++)
1359 irq_info[irq].evtchn = 0; /* zap event-channel binding */
1361 for (evtchn = 0; evtchn < NR_EVENT_CHANNELS; evtchn++)
1362 evtchn_to_irq[evtchn] = -1;
1364 for_each_possible_cpu(cpu) {
1365 restore_cpu_virqs(cpu);
1366 restore_cpu_ipis(cpu);
1372 static struct irq_chip xen_dynamic_chip __read_mostly = {
1375 .irq_disable = disable_dynirq,
1376 .irq_mask = disable_dynirq,
1377 .irq_unmask = enable_dynirq,
1379 .irq_eoi = ack_dynirq,
1380 .irq_set_affinity = set_affinity_irq,
1381 .irq_retrigger = retrigger_dynirq,
1384 static struct irq_chip xen_pirq_chip __read_mostly = {
1387 .irq_startup = startup_pirq,
1388 .irq_shutdown = shutdown_pirq,
1390 .irq_enable = enable_pirq,
1391 .irq_unmask = enable_pirq,
1393 .irq_disable = disable_pirq,
1394 .irq_mask = disable_pirq,
1396 .irq_ack = ack_pirq,
1398 .irq_set_affinity = set_affinity_irq,
1400 .irq_retrigger = retrigger_dynirq,
1403 static struct irq_chip xen_percpu_chip __read_mostly = {
1404 .name = "xen-percpu",
1406 .irq_disable = disable_dynirq,
1407 .irq_mask = disable_dynirq,
1408 .irq_unmask = enable_dynirq,
1410 .irq_ack = ack_dynirq,
1413 int xen_set_callback_via(uint64_t via)
1415 struct xen_hvm_param a;
1416 a.domid = DOMID_SELF;
1417 a.index = HVM_PARAM_CALLBACK_IRQ;
1419 return HYPERVISOR_hvm_op(HVMOP_set_param, &a);
1421 EXPORT_SYMBOL_GPL(xen_set_callback_via);
1423 #ifdef CONFIG_XEN_PVHVM
1424 /* Vector callbacks are better than PCI interrupts to receive event
1425 * channel notifications because we can receive vector callbacks on any
1426 * vcpu and we don't need PCI support or APIC interactions. */
1427 void xen_callback_vector(void)
1430 uint64_t callback_via;
1431 if (xen_have_vector_callback) {
1432 callback_via = HVM_CALLBACK_VECTOR(XEN_HVM_EVTCHN_CALLBACK);
1433 rc = xen_set_callback_via(callback_via);
1435 printk(KERN_ERR "Request for Xen HVM callback vector"
1437 xen_have_vector_callback = 0;
1440 printk(KERN_INFO "Xen HVM callback vector for event delivery is "
1442 /* in the restore case the vector has already been allocated */
1443 if (!test_bit(XEN_HVM_EVTCHN_CALLBACK, used_vectors))
1444 alloc_intr_gate(XEN_HVM_EVTCHN_CALLBACK, xen_hvm_callback_vector);
1448 void xen_callback_vector(void) {}
1451 void __init xen_init_IRQ(void)
1455 irq_info = kcalloc(nr_irqs, sizeof(*irq_info), GFP_KERNEL);
1457 /* We are using nr_irqs as the maximum number of pirq available but
1458 * that number is actually chosen by Xen and we don't know exactly
1459 * what it is. Be careful choosing high pirq numbers. */
1460 pirq_to_irq = kcalloc(nr_irqs, sizeof(*pirq_to_irq), GFP_KERNEL);
1461 for (i = 0; i < nr_irqs; i++)
1462 pirq_to_irq[i] = -1;
1464 evtchn_to_irq = kcalloc(NR_EVENT_CHANNELS, sizeof(*evtchn_to_irq),
1466 for (i = 0; i < NR_EVENT_CHANNELS; i++)
1467 evtchn_to_irq[i] = -1;
1469 init_evtchn_cpu_bindings();
1471 /* No event channels are 'live' right now. */
1472 for (i = 0; i < NR_EVENT_CHANNELS; i++)
1475 if (xen_hvm_domain()) {
1476 xen_callback_vector();
1478 /* pci_xen_hvm_init must be called after native_init_IRQ so that
1479 * __acpi_register_gsi can point at the right function */
1482 irq_ctx_init(smp_processor_id());
1483 if (xen_initial_domain())