2 * Copyright (C) 2012 Red Hat, Inc. All rights reserved.
3 * Author: Alex Williamson <alex.williamson@redhat.com>
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
9 * Derived from original vfio:
10 * Copyright 2010 Cisco Systems, Inc. All rights reserved.
11 * Author: Tom Lyon, pugs@cisco.com
14 #include <linux/device.h>
15 #include <linux/eventfd.h>
16 #include <linux/interrupt.h>
17 #include <linux/iommu.h>
18 #include <linux/module.h>
19 #include <linux/mutex.h>
20 #include <linux/notifier.h>
21 #include <linux/pci.h>
22 #include <linux/pm_runtime.h>
23 #include <linux/slab.h>
24 #include <linux/types.h>
25 #include <linux/uaccess.h>
26 #include <linux/vfio.h>
28 #include "vfio_pci_private.h"
30 #define DRIVER_VERSION "0.2"
31 #define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"
32 #define DRIVER_DESC "VFIO PCI - User Level meta-driver"
34 static bool nointxmask;
35 module_param_named(nointxmask, nointxmask, bool, S_IRUGO | S_IWUSR);
36 MODULE_PARM_DESC(nointxmask,
37 "Disable support for PCI 2.3 style INTx masking. If this resolves problems for specific devices, report lspci -vvvxxx to linux-pci@vger.kernel.org so the device can be fixed automatically via the broken_intx_masking flag.");
39 static int vfio_pci_enable(struct vfio_pci_device *vdev)
41 struct pci_dev *pdev = vdev->pdev;
46 ret = pci_enable_device(pdev);
50 vdev->reset_works = (pci_reset_function(pdev) == 0);
52 vdev->pci_saved_state = pci_store_saved_state(pdev);
53 if (!vdev->pci_saved_state)
54 pr_debug("%s: Couldn't store %s saved state\n",
55 __func__, dev_name(&pdev->dev));
57 ret = vfio_config_init(vdev);
59 pci_load_and_free_saved_state(pdev, &vdev->pci_saved_state);
60 pci_disable_device(pdev);
64 if (likely(!nointxmask))
65 vdev->pci_2_3 = pci_intx_mask_supported(pdev);
67 pci_read_config_word(pdev, PCI_COMMAND, &cmd);
68 if (vdev->pci_2_3 && (cmd & PCI_COMMAND_INTX_DISABLE)) {
69 cmd &= ~PCI_COMMAND_INTX_DISABLE;
70 pci_write_config_word(pdev, PCI_COMMAND, cmd);
73 msix_pos = pdev->msix_cap;
78 pci_read_config_word(pdev, msix_pos + PCI_MSIX_FLAGS, &flags);
79 pci_read_config_dword(pdev, msix_pos + PCI_MSIX_TABLE, &table);
81 vdev->msix_bar = table & PCI_MSIX_TABLE_BIR;
82 vdev->msix_offset = table & PCI_MSIX_TABLE_OFFSET;
83 vdev->msix_size = ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) * 16;
85 vdev->msix_bar = 0xFF;
87 #ifdef CONFIG_VFIO_PCI_VGA
88 if ((pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
95 static void vfio_pci_disable(struct vfio_pci_device *vdev)
97 struct pci_dev *pdev = vdev->pdev;
100 pci_disable_device(pdev);
102 vfio_pci_set_irqs_ioctl(vdev, VFIO_IRQ_SET_DATA_NONE |
103 VFIO_IRQ_SET_ACTION_TRIGGER,
104 vdev->irq_type, 0, 0, NULL);
106 vdev->virq_disabled = false;
108 vfio_config_free(vdev);
110 for (bar = PCI_STD_RESOURCES; bar <= PCI_STD_RESOURCE_END; bar++) {
111 if (!vdev->barmap[bar])
113 pci_iounmap(pdev, vdev->barmap[bar]);
114 pci_release_selected_regions(pdev, 1 << bar);
115 vdev->barmap[bar] = NULL;
119 * If we have saved state, restore it. If we can reset the device,
120 * even better. Resetting with current state seems better than
121 * nothing, but saving and restoring current state without reset
124 if (pci_load_and_free_saved_state(pdev, &vdev->pci_saved_state)) {
125 pr_info("%s: Couldn't reload %s saved state\n",
126 __func__, dev_name(&pdev->dev));
128 if (!vdev->reset_works)
131 pci_save_state(pdev);
135 * Disable INTx and MSI, presumably to avoid spurious interrupts
136 * during reset. Stolen from pci_reset_function()
138 pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE);
141 * Careful, device_lock may already be held. This is the case if
142 * a driver unbind is blocked. Try to get the locks ourselves to
143 * prevent a deadlock.
145 if (vdev->reset_works) {
146 bool reset_done = false;
148 if (pci_cfg_access_trylock(pdev)) {
149 if (device_trylock(&pdev->dev)) {
150 __pci_reset_function_locked(pdev);
152 device_unlock(&pdev->dev);
154 pci_cfg_access_unlock(pdev);
158 pr_warn("%s: Unable to acquire locks for reset of %s\n",
159 __func__, dev_name(&pdev->dev));
162 pci_restore_state(pdev);
165 static void vfio_pci_release(void *device_data)
167 struct vfio_pci_device *vdev = device_data;
169 if (atomic_dec_and_test(&vdev->refcnt))
170 vfio_pci_disable(vdev);
172 module_put(THIS_MODULE);
175 static int vfio_pci_open(void *device_data)
177 struct vfio_pci_device *vdev = device_data;
179 if (!try_module_get(THIS_MODULE))
182 if (atomic_inc_return(&vdev->refcnt) == 1) {
183 int ret = vfio_pci_enable(vdev);
185 module_put(THIS_MODULE);
193 static int vfio_pci_get_irq_count(struct vfio_pci_device *vdev, int irq_type)
195 if (irq_type == VFIO_PCI_INTX_IRQ_INDEX) {
197 pci_read_config_byte(vdev->pdev, PCI_INTERRUPT_PIN, &pin);
201 } else if (irq_type == VFIO_PCI_MSI_IRQ_INDEX) {
205 pos = vdev->pdev->msi_cap;
207 pci_read_config_word(vdev->pdev,
208 pos + PCI_MSI_FLAGS, &flags);
210 return 1 << (flags & PCI_MSI_FLAGS_QMASK);
212 } else if (irq_type == VFIO_PCI_MSIX_IRQ_INDEX) {
216 pos = vdev->pdev->msix_cap;
218 pci_read_config_word(vdev->pdev,
219 pos + PCI_MSIX_FLAGS, &flags);
221 return (flags & PCI_MSIX_FLAGS_QSIZE) + 1;
223 } else if (irq_type == VFIO_PCI_ERR_IRQ_INDEX)
224 if (pci_is_pcie(vdev->pdev))
230 static long vfio_pci_ioctl(void *device_data,
231 unsigned int cmd, unsigned long arg)
233 struct vfio_pci_device *vdev = device_data;
236 if (cmd == VFIO_DEVICE_GET_INFO) {
237 struct vfio_device_info info;
239 minsz = offsetofend(struct vfio_device_info, num_irqs);
241 if (copy_from_user(&info, (void __user *)arg, minsz))
244 if (info.argsz < minsz)
247 info.flags = VFIO_DEVICE_FLAGS_PCI;
249 if (vdev->reset_works)
250 info.flags |= VFIO_DEVICE_FLAGS_RESET;
252 info.num_regions = VFIO_PCI_NUM_REGIONS;
253 info.num_irqs = VFIO_PCI_NUM_IRQS;
255 return copy_to_user((void __user *)arg, &info, minsz);
257 } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
258 struct pci_dev *pdev = vdev->pdev;
259 struct vfio_region_info info;
261 minsz = offsetofend(struct vfio_region_info, offset);
263 if (copy_from_user(&info, (void __user *)arg, minsz))
266 if (info.argsz < minsz)
269 switch (info.index) {
270 case VFIO_PCI_CONFIG_REGION_INDEX:
271 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
272 info.size = pdev->cfg_size;
273 info.flags = VFIO_REGION_INFO_FLAG_READ |
274 VFIO_REGION_INFO_FLAG_WRITE;
276 case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
277 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
278 info.size = pci_resource_len(pdev, info.index);
284 info.flags = VFIO_REGION_INFO_FLAG_READ |
285 VFIO_REGION_INFO_FLAG_WRITE;
286 if (pci_resource_flags(pdev, info.index) &
287 IORESOURCE_MEM && info.size >= PAGE_SIZE)
288 info.flags |= VFIO_REGION_INFO_FLAG_MMAP;
290 case VFIO_PCI_ROM_REGION_INDEX:
295 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
298 /* Report the BAR size, not the ROM size */
299 info.size = pci_resource_len(pdev, info.index);
303 /* Is it really there? */
304 io = pci_map_rom(pdev, &size);
309 pci_unmap_rom(pdev, io);
311 info.flags = VFIO_REGION_INFO_FLAG_READ;
314 case VFIO_PCI_VGA_REGION_INDEX:
318 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
320 info.flags = VFIO_REGION_INFO_FLAG_READ |
321 VFIO_REGION_INFO_FLAG_WRITE;
328 return copy_to_user((void __user *)arg, &info, minsz);
330 } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
331 struct vfio_irq_info info;
333 minsz = offsetofend(struct vfio_irq_info, count);
335 if (copy_from_user(&info, (void __user *)arg, minsz))
338 if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS)
341 switch (info.index) {
342 case VFIO_PCI_INTX_IRQ_INDEX ... VFIO_PCI_MSIX_IRQ_INDEX:
344 case VFIO_PCI_ERR_IRQ_INDEX:
345 if (pci_is_pcie(vdev->pdev))
347 /* pass thru to return error */
352 info.flags = VFIO_IRQ_INFO_EVENTFD;
354 info.count = vfio_pci_get_irq_count(vdev, info.index);
356 if (info.index == VFIO_PCI_INTX_IRQ_INDEX)
357 info.flags |= (VFIO_IRQ_INFO_MASKABLE |
358 VFIO_IRQ_INFO_AUTOMASKED);
360 info.flags |= VFIO_IRQ_INFO_NORESIZE;
362 return copy_to_user((void __user *)arg, &info, minsz);
364 } else if (cmd == VFIO_DEVICE_SET_IRQS) {
365 struct vfio_irq_set hdr;
369 minsz = offsetofend(struct vfio_irq_set, count);
371 if (copy_from_user(&hdr, (void __user *)arg, minsz))
374 if (hdr.argsz < minsz || hdr.index >= VFIO_PCI_NUM_IRQS ||
375 hdr.flags & ~(VFIO_IRQ_SET_DATA_TYPE_MASK |
376 VFIO_IRQ_SET_ACTION_TYPE_MASK))
379 if (!(hdr.flags & VFIO_IRQ_SET_DATA_NONE)) {
381 int max = vfio_pci_get_irq_count(vdev, hdr.index);
383 if (hdr.flags & VFIO_IRQ_SET_DATA_BOOL)
384 size = sizeof(uint8_t);
385 else if (hdr.flags & VFIO_IRQ_SET_DATA_EVENTFD)
386 size = sizeof(int32_t);
390 if (hdr.argsz - minsz < hdr.count * size ||
391 hdr.start >= max || hdr.start + hdr.count > max)
394 data = memdup_user((void __user *)(arg + minsz),
397 return PTR_ERR(data);
400 mutex_lock(&vdev->igate);
402 ret = vfio_pci_set_irqs_ioctl(vdev, hdr.flags, hdr.index,
403 hdr.start, hdr.count, data);
405 mutex_unlock(&vdev->igate);
410 } else if (cmd == VFIO_DEVICE_RESET)
411 return vdev->reset_works ?
412 pci_reset_function(vdev->pdev) : -EINVAL;
417 static ssize_t vfio_pci_rw(void *device_data, char __user *buf,
418 size_t count, loff_t *ppos, bool iswrite)
420 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
421 struct vfio_pci_device *vdev = device_data;
423 if (index >= VFIO_PCI_NUM_REGIONS)
427 case VFIO_PCI_CONFIG_REGION_INDEX:
428 return vfio_pci_config_rw(vdev, buf, count, ppos, iswrite);
430 case VFIO_PCI_ROM_REGION_INDEX:
433 return vfio_pci_bar_rw(vdev, buf, count, ppos, false);
435 case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
436 return vfio_pci_bar_rw(vdev, buf, count, ppos, iswrite);
438 case VFIO_PCI_VGA_REGION_INDEX:
439 return vfio_pci_vga_rw(vdev, buf, count, ppos, iswrite);
445 static ssize_t vfio_pci_read(void *device_data, char __user *buf,
446 size_t count, loff_t *ppos)
451 return vfio_pci_rw(device_data, buf, count, ppos, false);
454 static ssize_t vfio_pci_write(void *device_data, const char __user *buf,
455 size_t count, loff_t *ppos)
460 return vfio_pci_rw(device_data, (char __user *)buf, count, ppos, true);
463 static int vfio_pci_mmap(void *device_data, struct vm_area_struct *vma)
465 struct vfio_pci_device *vdev = device_data;
466 struct pci_dev *pdev = vdev->pdev;
468 u64 phys_len, req_len, pgoff, req_start;
471 index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
473 if (vma->vm_end < vma->vm_start)
475 if ((vma->vm_flags & VM_SHARED) == 0)
477 if (index >= VFIO_PCI_ROM_REGION_INDEX)
479 if (!(pci_resource_flags(pdev, index) & IORESOURCE_MEM))
482 phys_len = pci_resource_len(pdev, index);
483 req_len = vma->vm_end - vma->vm_start;
484 pgoff = vma->vm_pgoff &
485 ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
486 req_start = pgoff << PAGE_SHIFT;
488 if (phys_len < PAGE_SIZE || req_start + req_len > phys_len)
491 if (index == vdev->msix_bar) {
493 * Disallow mmaps overlapping the MSI-X table; users don't
494 * get to touch this directly. We could find somewhere
495 * else to map the overlap, but page granularity is only
496 * a recommendation, not a requirement, so the user needs
497 * to know which bits are real. Requiring them to mmap
498 * around the table makes that clear.
501 /* If neither entirely above nor below, then it overlaps */
502 if (!(req_start >= vdev->msix_offset + vdev->msix_size ||
503 req_start + req_len <= vdev->msix_offset))
508 * Even though we don't make use of the barmap for the mmap,
509 * we need to request the region and the barmap tracks that.
511 if (!vdev->barmap[index]) {
512 ret = pci_request_selected_regions(pdev,
513 1 << index, "vfio-pci");
517 vdev->barmap[index] = pci_iomap(pdev, index, 0);
520 vma->vm_private_data = vdev;
521 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
522 vma->vm_pgoff = (pci_resource_start(pdev, index) >> PAGE_SHIFT) + pgoff;
524 return remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
525 req_len, vma->vm_page_prot);
528 static const struct vfio_device_ops vfio_pci_ops = {
530 .open = vfio_pci_open,
531 .release = vfio_pci_release,
532 .ioctl = vfio_pci_ioctl,
533 .read = vfio_pci_read,
534 .write = vfio_pci_write,
535 .mmap = vfio_pci_mmap,
538 static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
541 struct vfio_pci_device *vdev;
542 struct iommu_group *group;
545 pci_read_config_byte(pdev, PCI_HEADER_TYPE, &type);
546 if ((type & PCI_HEADER_TYPE) != PCI_HEADER_TYPE_NORMAL)
549 group = iommu_group_get(&pdev->dev);
553 vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);
555 iommu_group_put(group);
560 vdev->irq_type = VFIO_PCI_NUM_IRQS;
561 mutex_init(&vdev->igate);
562 spin_lock_init(&vdev->irqlock);
563 atomic_set(&vdev->refcnt, 0);
565 ret = vfio_add_group_dev(&pdev->dev, &vfio_pci_ops, vdev);
567 iommu_group_put(group);
574 static void vfio_pci_remove(struct pci_dev *pdev)
576 struct vfio_pci_device *vdev;
578 vdev = vfio_del_group_dev(&pdev->dev);
582 iommu_group_put(pdev->dev.iommu_group);
586 static pci_ers_result_t vfio_pci_aer_err_detected(struct pci_dev *pdev,
587 pci_channel_state_t state)
589 struct vfio_pci_device *vdev;
590 struct vfio_device *device;
592 device = vfio_device_get_from_dev(&pdev->dev);
594 return PCI_ERS_RESULT_DISCONNECT;
596 vdev = vfio_device_data(device);
598 vfio_device_put(device);
599 return PCI_ERS_RESULT_DISCONNECT;
602 if (vdev->err_trigger)
603 eventfd_signal(vdev->err_trigger, 1);
605 vfio_device_put(device);
607 return PCI_ERS_RESULT_CAN_RECOVER;
610 static struct pci_error_handlers vfio_err_handlers = {
611 .error_detected = vfio_pci_aer_err_detected,
614 static struct pci_driver vfio_pci_driver = {
616 .id_table = NULL, /* only dynamic ids */
617 .probe = vfio_pci_probe,
618 .remove = vfio_pci_remove,
619 .err_handler = &vfio_err_handlers,
622 static void __exit vfio_pci_cleanup(void)
624 pci_unregister_driver(&vfio_pci_driver);
625 vfio_pci_virqfd_exit();
626 vfio_pci_uninit_perm_bits();
629 static int __init vfio_pci_init(void)
633 /* Allocate shared config space permision data used by all devices */
634 ret = vfio_pci_init_perm_bits();
638 /* Start the virqfd cleanup handler */
639 ret = vfio_pci_virqfd_init();
643 /* Register and scan for devices */
644 ret = pci_register_driver(&vfio_pci_driver);
651 vfio_pci_virqfd_exit();
653 vfio_pci_uninit_perm_bits();
657 module_init(vfio_pci_init);
658 module_exit(vfio_pci_cleanup);
660 MODULE_VERSION(DRIVER_VERSION);
661 MODULE_LICENSE("GPL v2");
662 MODULE_AUTHOR(DRIVER_AUTHOR);
663 MODULE_DESCRIPTION(DRIVER_DESC);