#include <linux/idr.h>
#include <linux/notifier.h>
#include <linux/err.h>
+#include <linux/pci.h>
#include <trace/events/iommu.h>
static struct kset *iommu_group_kset;
static struct ida iommu_group_ida;
static struct mutex iommu_group_mutex;
+struct iommu_callback_data {
+ const struct iommu_ops *ops;
+};
+
struct iommu_group {
struct kobject kobj;
struct kobject *devices_kobj;
}
EXPORT_SYMBOL_GPL(iommu_group_id);
+/*
+ * To consider a PCI device isolated, we require ACS to support Source
+ * Validation, Request Redirection, Completer Redirection, and Upstream
+ * Forwarding. This effectively means that devices cannot spoof their
+ * requester ID, requests and completions cannot be redirected, and all
+ * transactions are forwarded upstream, even as it passes through a
+ * bridge where the target device is downstream.
+ */
+#define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
+
+struct group_for_pci_data {
+ struct pci_dev *pdev;
+ struct iommu_group *group;
+};
+
+/*
+ * DMA alias iterator callback, return the last seen device. Stop and return
+ * the IOMMU group if we find one along the way.
+ */
+static int get_pci_alias_or_group(struct pci_dev *pdev, u16 alias, void *opaque)
+{
+ struct group_for_pci_data *data = opaque;
+
+ data->pdev = pdev;
+ data->group = iommu_group_get(&pdev->dev);
+
+ return data->group != NULL;
+}
+
+/*
+ * Use standard PCI bus topology, isolation features, and DMA alias quirks
+ * to find or create an IOMMU group for a device.
+ */
+static struct iommu_group *iommu_group_get_for_pci_dev(struct pci_dev *pdev)
+{
+ struct group_for_pci_data data;
+ struct pci_bus *bus;
+ struct iommu_group *group = NULL;
+ struct pci_dev *tmp;
+
+ /*
+ * Find the upstream DMA alias for the device. A device must not
+ * be aliased due to topology in order to have its own IOMMU group.
+ * If we find an alias along the way that already belongs to a
+ * group, use it.
+ */
+ if (pci_for_each_dma_alias(pdev, get_pci_alias_or_group, &data))
+ return data.group;
+
+ pdev = data.pdev;
+
+ /*
+ * Continue upstream from the point of minimum IOMMU granularity
+ * due to aliases to the point where devices are protected from
+ * peer-to-peer DMA by PCI ACS. Again, if we find an existing
+ * group, use it.
+ */
+ for (bus = pdev->bus; !pci_is_root_bus(bus); bus = bus->parent) {
+ if (!bus->self)
+ continue;
+
+ if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS))
+ break;
+
+ pdev = bus->self;
+
+ group = iommu_group_get(&pdev->dev);
+ if (group)
+ return group;
+ }
+
+ /*
+ * Next we need to consider DMA alias quirks. If one device aliases
+ * to another, they should be grouped together. It's theoretically
+ * possible that aliases could create chains of devices where each
+ * device aliases another device. If we then factor in multifunction
+ * ACS grouping requirements, each alias could incorporate a new slot
+ * with multiple functions, each with aliases. This is all extremely
+ * unlikely as DMA alias quirks are typically only used for PCIe
+ * devices where we usually have a single slot per bus. Furthermore,
+ * the alias quirk is usually to another function within the slot
+ * (and ACS multifunction is not supported) or to a different slot
+ * that doesn't physically exist. The likely scenario is therefore
+ * that everything on the bus gets grouped together. To reduce the
+ * problem space, share the IOMMU group for all devices on the bus
+ * if a DMA alias quirk is present on the bus.
+ */
+ tmp = NULL;
+ for_each_pci_dev(tmp) {
+ if (tmp->bus != pdev->bus ||
+ !(tmp->dev_flags & PCI_DEV_FLAGS_DMA_ALIAS_DEVFN))
+ continue;
+
+ pci_dev_put(tmp);
+ tmp = NULL;
+
+ /* We have an alias quirk, search for an existing group */
+ for_each_pci_dev(tmp) {
+ struct iommu_group *group_tmp;
+
+ if (tmp->bus != pdev->bus)
+ continue;
+
+ group_tmp = iommu_group_get(&tmp->dev);
+ if (!group) {
+ group = group_tmp;
+ continue;
+ }
+
+ if (group_tmp) {
+ WARN_ON(group != group_tmp);
+ iommu_group_put(group_tmp);
+ }
+ }
+
+ return group ? group : iommu_group_alloc();
+ }
+
+ /*
+ * Non-multifunction devices or multifunction devices supporting
+ * ACS get their own group.
+ */
+ if (!pdev->multifunction || pci_acs_enabled(pdev, REQ_ACS_FLAGS))
+ return iommu_group_alloc();
+
+ /*
+ * Multifunction devices not supporting ACS share a group with other
+ * similar devices in the same slot.
+ */
+ tmp = NULL;
+ for_each_pci_dev(tmp) {
+ if (tmp == pdev || tmp->bus != pdev->bus ||
+ PCI_SLOT(tmp->devfn) != PCI_SLOT(pdev->devfn) ||
+ pci_acs_enabled(tmp, REQ_ACS_FLAGS))
+ continue;
+
+ group = iommu_group_get(&tmp->dev);
+ if (group) {
+ pci_dev_put(tmp);
+ return group;
+ }
+ }
+
+ /* No shared group found, allocate new */
+ return iommu_group_alloc();
+}
+
+/**
+ * iommu_group_get_for_dev - Find or create the IOMMU group for a device
+ * @dev: target device
+ *
+ * This function is intended to be called by IOMMU drivers and extended to
+ * support common, bus-defined algorithms when determining or creating the
+ * IOMMU group for a device. On success, the caller will hold a reference
+ * to the returned IOMMU group, which will already include the provided
+ * device. The reference should be released with iommu_group_put().
+ */
+struct iommu_group *iommu_group_get_for_dev(struct device *dev)
+{
+ struct iommu_group *group = ERR_PTR(-EIO);
+ int ret;
+
+ group = iommu_group_get(dev);
+ if (group)
+ return group;
+
+ if (dev_is_pci(dev))
+ group = iommu_group_get_for_pci_dev(to_pci_dev(dev));
+
+ if (IS_ERR(group))
+ return group;
+
+ ret = iommu_group_add_device(group, dev);
+ if (ret) {
+ iommu_group_put(group);
+ return ERR_PTR(ret);
+ }
+
+ return group;
+}
+
static int add_iommu_group(struct device *dev, void *data)
{
- struct iommu_ops *ops = data;
+ struct iommu_callback_data *cb = data;
+ const struct iommu_ops *ops = cb->ops;
if (!ops->add_device)
return -ENODEV;
unsigned long action, void *data)
{
struct device *dev = data;
- struct iommu_ops *ops = dev->bus->iommu_ops;
+ const struct iommu_ops *ops = dev->bus->iommu_ops;
struct iommu_group *group;
unsigned long group_action = 0;
.notifier_call = iommu_bus_notifier,
};
-static void iommu_bus_init(struct bus_type *bus, struct iommu_ops *ops)
+static void iommu_bus_init(struct bus_type *bus, const struct iommu_ops *ops)
{
+ struct iommu_callback_data cb = {
+ .ops = ops,
+ };
+
bus_register_notifier(bus, &iommu_bus_nb);
- bus_for_each_dev(bus, NULL, ops, add_iommu_group);
+ bus_for_each_dev(bus, NULL, &cb, add_iommu_group);
}
/**
* is set up. With this function the iommu-driver can set the iommu-ops
* afterwards.
*/
-int bus_set_iommu(struct bus_type *bus, struct iommu_ops *ops)
+int bus_set_iommu(struct bus_type *bus, const struct iommu_ops *ops)
{
if (bus->iommu_ops != NULL)
return -EBUSY;
projects / karo-tx-linux.git / blobdiff