diff options
Diffstat (limited to 'drivers/iommu')
| -rw-r--r-- | drivers/iommu/amd_iommu.c | 2 | ||||
| -rw-r--r-- | drivers/iommu/amd_iommu_init.c | 4 | ||||
| -rw-r--r-- | drivers/iommu/amd_iommu_v2.c | 4 | ||||
| -rw-r--r-- | drivers/iommu/arm-smmu-v3.c | 129 | ||||
| -rw-r--r-- | drivers/iommu/arm-smmu.c | 193 | ||||
| -rw-r--r-- | drivers/iommu/dma-iommu.c | 24 | ||||
| -rw-r--r-- | drivers/iommu/dmar.c | 4 | ||||
| -rw-r--r-- | drivers/iommu/exynos-iommu.c | 293 | ||||
| -rw-r--r-- | drivers/iommu/intel-iommu.c | 27 | ||||
| -rw-r--r-- | drivers/iommu/intel-svm.c | 28 | ||||
| -rw-r--r-- | drivers/iommu/io-pgtable-arm-v7s.c | 5 | ||||
| -rw-r--r-- | drivers/iommu/io-pgtable-arm.c | 7 | ||||
| -rw-r--r-- | drivers/iommu/iommu.c | 53 | ||||
| -rw-r--r-- | drivers/iommu/iova.c | 2 | ||||
| -rw-r--r-- | drivers/iommu/mtk_iommu.c | 2 | ||||
| -rw-r--r-- | drivers/iommu/mtk_iommu_v1.c | 2 | ||||
| -rw-r--r-- | drivers/iommu/of_iommu.c | 39 | ||||
| -rw-r--r-- | drivers/iommu/s390-iommu.c | 1 |
18 files changed, 531 insertions, 288 deletions
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 754595ee11b6..019e02707cd5 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -373,6 +373,8 @@ static struct iommu_group *acpihid_device_group(struct device *dev) if (!entry->group) entry->group = generic_device_group(dev); + else + iommu_group_ref_get(entry->group); return entry->group; } diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index 157e93421fb8..971154cbbb03 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -28,6 +28,7 @@ #include <linux/amd-iommu.h> #include <linux/export.h> #include <linux/iommu.h> +#include <linux/kmemleak.h> #include <asm/pci-direct.h> #include <asm/iommu.h> #include <asm/gart.h> @@ -2090,6 +2091,7 @@ static struct syscore_ops amd_iommu_syscore_ops = { static void __init free_on_init_error(void) { + kmemleak_free(irq_lookup_table); free_pages((unsigned long)irq_lookup_table, get_order(rlookup_table_size)); @@ -2321,6 +2323,8 @@ static int __init early_amd_iommu_init(void) irq_lookup_table = (void *)__get_free_pages( GFP_KERNEL | __GFP_ZERO, get_order(rlookup_table_size)); + kmemleak_alloc(irq_lookup_table, rlookup_table_size, + 1, GFP_KERNEL); if (!irq_lookup_table) goto out; } diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c index 594849a3a9be..f8ed8c95b685 100644 --- a/drivers/iommu/amd_iommu_v2.c +++ b/drivers/iommu/amd_iommu_v2.c @@ -805,8 +805,10 @@ int amd_iommu_init_device(struct pci_dev *pdev, int pasids) goto out_free_domain; group = iommu_group_get(&pdev->dev); - if (!group) + if (!group) { + ret = -EINVAL; goto out_free_domain; + } ret = iommu_attach_group(dev_state->domain, group); if (ret != 0) diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 15c01c3cd540..4d6ec444a9d6 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -20,6 +20,8 @@ * This driver is powered by bad coffee and bombay mix. */ +#include <linux/acpi.h> +#include <linux/acpi_iort.h> #include <linux/delay.h> #include <linux/dma-iommu.h> #include <linux/err.h> @@ -1358,7 +1360,7 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, } while (size -= granule); } -static struct iommu_gather_ops arm_smmu_gather_ops = { +static const struct iommu_gather_ops arm_smmu_gather_ops = { .tlb_flush_all = arm_smmu_tlb_inv_context, .tlb_add_flush = arm_smmu_tlb_inv_range_nosync, .tlb_sync = arm_smmu_tlb_sync, @@ -1723,13 +1725,14 @@ static struct platform_driver arm_smmu_driver; static int arm_smmu_match_node(struct device *dev, void *data) { - return dev->of_node == data; + return dev->fwnode == data; } -static struct arm_smmu_device *arm_smmu_get_by_node(struct device_node *np) +static +struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode) { struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL, - np, arm_smmu_match_node); + fwnode, arm_smmu_match_node); put_device(dev); return dev ? dev_get_drvdata(dev) : NULL; } @@ -1765,7 +1768,7 @@ static int arm_smmu_add_device(struct device *dev) master = fwspec->iommu_priv; smmu = master->smmu; } else { - smmu = arm_smmu_get_by_node(to_of_node(fwspec->iommu_fwnode)); + smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode); if (!smmu) return -ENODEV; master = kzalloc(sizeof(*master), GFP_KERNEL); @@ -2380,10 +2383,10 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass) return 0; } -static int arm_smmu_device_probe(struct arm_smmu_device *smmu) +static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) { u32 reg; - bool coherent; + bool coherent = smmu->features & ARM_SMMU_FEAT_COHERENCY; /* IDR0 */ reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0); @@ -2435,13 +2438,9 @@ static int arm_smmu_device_probe(struct arm_smmu_device *smmu) smmu->features |= ARM_SMMU_FEAT_HYP; /* - * The dma-coherent property is used in preference to the ID + * The coherency feature as set by FW is used in preference to the ID * register, but warn on mismatch. */ - coherent = of_dma_is_coherent(smmu->dev->of_node); - if (coherent) - smmu->features |= ARM_SMMU_FEAT_COHERENCY; - if (!!(reg & IDR0_COHACC) != coherent) dev_warn(smmu->dev, "IDR0.COHACC overridden by dma-coherent property (%s)\n", coherent ? "true" : "false"); @@ -2562,21 +2561,61 @@ static int arm_smmu_device_probe(struct arm_smmu_device *smmu) return 0; } -static int arm_smmu_device_dt_probe(struct platform_device *pdev) +#ifdef CONFIG_ACPI +static int arm_smmu_device_acpi_probe(struct platform_device *pdev, + struct arm_smmu_device *smmu) +{ + struct acpi_iort_smmu_v3 *iort_smmu; + struct device *dev = smmu->dev; + struct acpi_iort_node *node; + + node = *(struct acpi_iort_node **)dev_get_platdata(dev); + + /* Retrieve SMMUv3 specific data */ + iort_smmu = (struct acpi_iort_smmu_v3 *)node->node_data; + + if (iort_smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE) + smmu->features |= ARM_SMMU_FEAT_COHERENCY; + + return 0; +} +#else +static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev, + struct arm_smmu_device *smmu) +{ + return -ENODEV; +} +#endif + +static int arm_smmu_device_dt_probe(struct platform_device *pdev, + struct arm_smmu_device *smmu) { - int irq, ret; - struct resource *res; - struct arm_smmu_device *smmu; struct device *dev = &pdev->dev; - bool bypass = true; u32 cells; + int ret = -EINVAL; if (of_property_read_u32(dev->of_node, "#iommu-cells", &cells)) dev_err(dev, "missing #iommu-cells property\n"); else if (cells != 1) dev_err(dev, "invalid #iommu-cells value (%d)\n", cells); else - bypass = false; + ret = 0; + + parse_driver_options(smmu); + + if (of_dma_is_coherent(dev->of_node)) + smmu->features |= ARM_SMMU_FEAT_COHERENCY; + + return ret; +} + +static int arm_smmu_device_probe(struct platform_device *pdev) +{ + int irq, ret; + struct resource *res; + struct arm_smmu_device *smmu; + struct device *dev = &pdev->dev; + bool bypass; smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); if (!smmu) { @@ -2613,10 +2652,19 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) if (irq > 0) smmu->gerr_irq = irq; - parse_driver_options(smmu); + if (dev->of_node) { + ret = arm_smmu_device_dt_probe(pdev, smmu); + } else { + ret = arm_smmu_device_acpi_probe(pdev, smmu); + if (ret == -ENODEV) + return ret; + } + + /* Set bypass mode according to firmware probing result */ + bypass = !!ret; /* Probe the h/w */ - ret = arm_smmu_device_probe(smmu); + ret = arm_smmu_device_hw_probe(smmu); if (ret) return ret; @@ -2634,19 +2682,29 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) return ret; /* And we're up. Go go go! */ - of_iommu_set_ops(dev->of_node, &arm_smmu_ops); + iommu_register_instance(dev->fwnode, &arm_smmu_ops); + #ifdef CONFIG_PCI - pci_request_acs(); - ret = bus_set_iommu(&pci_bus_type, &arm_smmu_ops); - if (ret) - return ret; + if (pci_bus_type.iommu_ops != &arm_smmu_ops) { + pci_request_acs(); + ret = bus_set_iommu(&pci_bus_type, &arm_smmu_ops); + if (ret) + return ret; + } #endif #ifdef CONFIG_ARM_AMBA - ret = bus_set_iommu(&amba_bustype, &arm_smmu_ops); - if (ret) - return ret; + if (amba_bustype.iommu_ops != &arm_smmu_ops) { + ret = bus_set_iommu(&amba_bustype, &arm_smmu_ops); + if (ret) + return ret; + } #endif - return bus_set_iommu(&platform_bus_type, &arm_smmu_ops); + if (platform_bus_type.iommu_ops != &arm_smmu_ops) { + ret = bus_set_iommu(&platform_bus_type, &arm_smmu_ops); + if (ret) + return ret; + } + return 0; } static int arm_smmu_device_remove(struct platform_device *pdev) @@ -2668,7 +2726,7 @@ static struct platform_driver arm_smmu_driver = { .name = "arm-smmu-v3", .of_match_table = of_match_ptr(arm_smmu_of_match), }, - .probe = arm_smmu_device_dt_probe, + .probe = arm_smmu_device_probe, .remove = arm_smmu_device_remove, }; @@ -2706,6 +2764,17 @@ static int __init arm_smmu_of_init(struct device_node *np) } IOMMU_OF_DECLARE(arm_smmuv3, "arm,smmu-v3", arm_smmu_of_init); +#ifdef CONFIG_ACPI +static int __init acpi_smmu_v3_init(struct acpi_table_header *table) +{ + if (iort_node_match(ACPI_IORT_NODE_SMMU_V3)) + return arm_smmu_init(); + + return 0; +} +IORT_ACPI_DECLARE(arm_smmu_v3, ACPI_SIG_IORT, acpi_smmu_v3_init); +#endif + MODULE_DESCRIPTION("IOMMU API for ARM architected SMMUv3 implementations"); MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index c841eb7a1a74..a60cded8a6ed 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -28,6 +28,8 @@ #define pr_fmt(fmt) "arm-smmu: " fmt +#include <linux/acpi.h> +#include <linux/acpi_iort.h> #include <linux/atomic.h> #include <linux/delay.h> #include <linux/dma-iommu.h> @@ -247,6 +249,7 @@ enum arm_smmu_s2cr_privcfg { #define ARM_MMU500_ACTLR_CPRE (1 << 1) #define ARM_MMU500_ACR_CACHE_LOCK (1 << 26) +#define ARM_MMU500_ACR_SMTNMB_TLBEN (1 << 8) #define CB_PAR_F (1 << 0) @@ -324,8 +327,10 @@ struct arm_smmu_master_cfg { #define INVALID_SMENDX -1 #define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv) #define fwspec_smmu(fw) (__fwspec_cfg(fw)->smmu) +#define fwspec_smendx(fw, i) \ + (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i]) #define for_each_cfg_sme(fw, i, idx) \ - for (i = 0; idx = __fwspec_cfg(fw)->smendx[i], i < fw->num_ids; ++i) + for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i) struct arm_smmu_device { struct device *dev; @@ -640,7 +645,7 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, } } -static struct iommu_gather_ops arm_smmu_gather_ops = { +static const struct iommu_gather_ops arm_smmu_gather_ops = { .tlb_flush_all = arm_smmu_tlb_inv_context, .tlb_add_flush = arm_smmu_tlb_inv_range_nosync, .tlb_sync = arm_smmu_tlb_sync, @@ -1228,6 +1233,16 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) return -ENXIO; } + /* + * FIXME: The arch/arm DMA API code tries to attach devices to its own + * domains between of_xlate() and add_device() - we have no way to cope + * with that, so until ARM gets converted to rely on groups and default + * domains, just say no (but more politely than by dereferencing NULL). + * This should be at least a WARN_ON once that's sorted. + */ + if (!fwspec->iommu_priv) + return -ENODEV; + smmu = fwspec_smmu(fwspec); /* Ensure that the domain is finalised */ ret = arm_smmu_init_domain_context(domain, smmu); @@ -1367,13 +1382,14 @@ static bool arm_smmu_capable(enum iommu_cap cap) static int arm_smmu_match_node(struct device *dev, void *data) { - return dev->of_node == data; + return dev->fwnode == data; } -static struct arm_smmu_device *arm_smmu_get_by_node(struct device_node *np) +static +struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode) { struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL, - np, arm_smmu_match_node); + fwnode, arm_smmu_match_node); put_device(dev); return dev ? dev_get_drvdata(dev) : NULL; } @@ -1390,8 +1406,8 @@ static int arm_smmu_add_device(struct device *dev) fwspec = dev->iommu_fwspec; if (ret) goto out_free; - } else if (fwspec) { - smmu = arm_smmu_get_by_node(to_of_node(fwspec->iommu_fwnode)); + } else if (fwspec && fwspec->ops == &arm_smmu_ops) { + smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode); } else { return -ENODEV; } @@ -1466,7 +1482,7 @@ static struct iommu_group *arm_smmu_device_group(struct device *dev) } if (group) - return group; + return iommu_group_ref_get(group); if (dev_is_pci(dev)) group = pci_device_group(dev); @@ -1569,16 +1585,22 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu) for (i = 0; i < smmu->num_mapping_groups; ++i) arm_smmu_write_sme(smmu, i); - /* - * Before clearing ARM_MMU500_ACTLR_CPRE, need to - * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK - * bit is only present in MMU-500r2 onwards. - */ - reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7); - major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK; - if ((smmu->model == ARM_MMU500) && (major >= 2)) { + if (smmu->model == ARM_MMU500) { + /* + * Before clearing ARM_MMU500_ACTLR_CPRE, need to + * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK + * bit is only present in MMU-500r2 onwards. + */ + reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7); + major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK; reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR); - reg &= ~ARM_MMU500_ACR_CACHE_LOCK; + if (major >= 2) + reg &= ~ARM_MMU500_ACR_CACHE_LOCK; + /* + * Allow unmatched Stream IDs to allocate bypass + * TLB entries for reduced latency. + */ + reg |= ARM_MMU500_ACR_SMTNMB_TLBEN; writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR); } @@ -1655,7 +1677,7 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) unsigned long size; void __iomem *gr0_base = ARM_SMMU_GR0(smmu); u32 id; - bool cttw_dt, cttw_reg; + bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK; int i; dev_notice(smmu->dev, "probing hardware configuration...\n"); @@ -1700,20 +1722,17 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) /* * In order for DMA API calls to work properly, we must defer to what - * the DT says about coherency, regardless of what the hardware claims. + * the FW says about coherency, regardless of what the hardware claims. * Fortunately, this also opens up a workaround for systems where the * ID register value has ended up configured incorrectly. */ - cttw_dt = of_dma_is_coherent(smmu->dev->of_node); cttw_reg = !!(id & ID0_CTTW); - if (cttw_dt) - smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK; - if (cttw_dt || cttw_reg) + if (cttw_fw || cttw_reg) dev_notice(smmu->dev, "\t%scoherent table walk\n", - cttw_dt ? "" : "non-"); - if (cttw_dt != cttw_reg) + cttw_fw ? "" : "non-"); + if (cttw_fw != cttw_reg) dev_notice(smmu->dev, - "\t(IDR0.CTTW overridden by dma-coherent property)\n"); + "\t(IDR0.CTTW overridden by FW configuration)\n"); /* Max. number of entries we have for stream matching/indexing */ size = 1 << ((id >> ID0_NUMSIDB_SHIFT) & ID0_NUMSIDB_MASK); @@ -1894,15 +1913,83 @@ static const struct of_device_id arm_smmu_of_match[] = { }; MODULE_DEVICE_TABLE(of, arm_smmu_of_match); -static int arm_smmu_device_dt_probe(struct platform_device *pdev) +#ifdef CONFIG_ACPI +static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu) +{ + int ret = 0; + + switch (model) { + case ACPI_IORT_SMMU_V1: + case ACPI_IORT_SMMU_CORELINK_MMU400: + smmu->version = ARM_SMMU_V1; + smmu->model = GENERIC_SMMU; + break; + case ACPI_IORT_SMMU_V2: + smmu->version = ARM_SMMU_V2; + smmu->model = GENERIC_SMMU; + break; + case ACPI_IORT_SMMU_CORELINK_MMU500: + smmu->version = ARM_SMMU_V2; + smmu->model = ARM_MMU500; + break; + default: + ret = -ENODEV; + } + + return ret; +} + +static int arm_smmu_device_acpi_probe(struct platform_device *pdev, + struct arm_smmu_device *smmu) +{ + struct device *dev = smmu->dev; + struct acpi_iort_node *node = + *(struct acpi_iort_node **)dev_get_platdata(dev); + struct acpi_iort_smmu *iort_smmu; + int ret; + + /* Retrieve SMMU1/2 specific data */ + iort_smmu = (struct acpi_iort_smmu *)node->node_data; + + ret = acpi_smmu_get_data(iort_smmu->model, smmu); + if (ret < 0) + return ret; + + /* Ignore the configuration access interrupt */ + smmu->num_global_irqs = 1; + + if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK) + smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK; + + return 0; +} +#else +static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev, + struct arm_smmu_device *smmu) +{ + return -ENODEV; +} +#endif + +static int arm_smmu_device_dt_probe(struct platform_device *pdev, + struct arm_smmu_device *smmu) { const struct arm_smmu_match_data *data; - struct resource *res; - struct arm_smmu_device *smmu; struct device *dev = &pdev->dev; - int num_irqs, i, err; bool legacy_binding; + if (of_property_read_u32(dev->of_node, "#global-interrupts", + &smmu->num_global_irqs)) { + dev_err(dev, "missing #global-interrupts property\n"); + return -ENODEV; + } + + data = of_device_get_match_data(dev); + smmu->version = data->version; + smmu->model = data->model; + + parse_driver_options(smmu); + legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL); if (legacy_binding && !using_generic_binding) { if (!using_legacy_binding) @@ -1915,6 +2002,19 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) return -ENODEV; } + if (of_dma_is_coherent(dev->of_node)) + smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK; + + return 0; +} + +static int arm_smmu_device_probe(struct platform_device *pdev) +{ + struct resource *res; + struct arm_smmu_device *smmu; + struct device *dev = &pdev->dev; + int num_irqs, i, err; + smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); if (!smmu) { dev_err(dev, "failed to allocate arm_smmu_device\n"); @@ -1922,9 +2022,13 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) } smmu->dev = dev; - data = of_device_get_match_data(dev); - smmu->version = data->version; - smmu->model = data->model; + if (dev->of_node) + err = arm_smmu_device_dt_probe(pdev, smmu); + else + err = arm_smmu_device_acpi_probe(pdev, smmu); + + if (err) + return err; res = platform_get_resource(pdev, IORESOURCE_MEM, 0); smmu->base = devm_ioremap_resource(dev, res); @@ -1932,12 +2036,6 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) return PTR_ERR(smmu->base); smmu->size = resource_size(res); - if (of_property_read_u32(dev->of_node, "#global-interrupts", - &smmu->num_global_irqs)) { - dev_err(dev, "missing #global-interrupts property\n"); - return -ENODEV; - } - num_irqs = 0; while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) { num_irqs++; @@ -1972,8 +2070,6 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) if (err) return err; - parse_driver_options(smmu); - if (smmu->version == ARM_SMMU_V2 && smmu->num_context_banks != smmu->num_context_irqs) { dev_err(dev, @@ -1995,7 +2091,7 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) } } - of_iommu_set_ops(dev->of_node, &arm_smmu_ops); + iommu_register_instance(dev->fwnode, &arm_smmu_ops); platform_set_drvdata(pdev, smmu); arm_smmu_device_reset(smmu); @@ -2035,7 +2131,7 @@ static struct platform_driver arm_smmu_driver = { .name = "arm-smmu", .of_match_table = of_match_ptr(arm_smmu_of_match), }, - .probe = arm_smmu_device_dt_probe, + .probe = arm_smmu_device_probe, .remove = arm_smmu_device_remove, }; @@ -2078,6 +2174,17 @@ IOMMU_OF_DECLARE(arm_mmu401, "arm,mmu-401", arm_smmu_of_init); IOMMU_OF_DECLARE(arm_mmu500, "arm,mmu-500", arm_smmu_of_init); IOMMU_OF_DECLARE(cavium_smmuv2, "cavium,smmu-v2", arm_smmu_of_init); +#ifdef CONFIG_ACPI +static int __init arm_smmu_acpi_init(struct acpi_table_header *table) +{ + if (iort_node_match(ACPI_IORT_NODE_SMMU)) + return arm_smmu_init(); + + return 0; +} +IORT_ACPI_DECLARE(arm_smmu, ACPI_SIG_IORT, arm_smmu_acpi_init); +#endif + MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations"); MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index c5ab8667e6f2..2db0d641cf45 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -432,13 +432,12 @@ int iommu_dma_mmap(struct page **pages, size_t size, struct vm_area_struct *vma) return ret; } -dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, int prot) +static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys, + size_t size, int prot) { dma_addr_t dma_addr; struct iommu_domain *domain = iommu_get_domain_for_dev(dev); struct iova_domain *iovad = cookie_iovad(domain); - phys_addr_t phys = page_to_phys(page) + offset; size_t iova_off = iova_offset(iovad, phys); size_t len = iova_align(iovad, size + iova_off); struct iova *iova = __alloc_iova(domain, len, dma_get_mask(dev)); @@ -454,6 +453,12 @@ dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, return dma_addr + iova_off; } +dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, + unsigned long offset, size_t size, int prot) +{ + return __iommu_dma_map(dev, page_to_phys(page) + offset, size, prot); +} + void iommu_dma_unmap_page(struct device *dev, dma_addr_t handle, size_t size, enum dma_data_direction dir, unsigned long attrs) { @@ -624,6 +629,19 @@ void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, __iommu_dma_unmap(iommu_get_domain_for_dev(dev), sg_dma_address(sg)); } +dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys, + size_t size, enum dma_data_direction dir, unsigned long attrs) +{ + return __iommu_dma_map(dev, phys, size, + dma_direction_to_prot(dir, false) | IOMMU_MMIO); +} + +void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle, + size_t size, enum dma_data_direction dir, unsigned long attrs) +{ + __iommu_dma_unmap(iommu_get_domain_for_dev(dev), handle); +} + int iommu_dma_supported(struct device *dev, u64 mask) { /* diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index 58470f5ced04..8c53748a769d 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -338,7 +338,9 @@ static int dmar_pci_bus_notifier(struct notifier_block *nb, struct pci_dev *pdev = to_pci_dev(data); struct dmar_pci_notify_info *info; - /* Only care about add/remove events for physical functions */ + /* Only care about add/remove events for physical functions. + * For VFs we actually do the lookup based on the corresponding + * PF in device_to_iommu() anyway. */ if (pdev->is_virtfn) return NOTIFY_DONE; if (action != BUS_NOTIFY_ADD_DEVICE && diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 30808e91b775..57ba0d3091ea 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -70,6 +70,36 @@ static short PG_ENT_SHIFT = -1; #define SYSMMU_PG_ENT_SHIFT 0 #define SYSMMU_V5_PG_ENT_SHIFT 4 +static const sysmmu_pte_t *LV1_PROT; +static const sysmmu_pte_t SYSMMU_LV1_PROT[] = { + ((0 << 15) | (0 << 10)), /* no access */ + ((1 << 15) | (1 << 10)), /* IOMMU_READ only */ + ((0 << 15) | (1 << 10)), /* IOMMU_WRITE not supported, use read/write */ + ((0 << 15) | (1 << 10)), /* IOMMU_READ | IOMMU_WRITE */ +}; +static const sysmmu_pte_t SYSMMU_V5_LV1_PROT[] = { + (0 << 4), /* no access */ + (1 << 4), /* IOMMU_READ only */ + (2 << 4), /* IOMMU_WRITE only */ + (3 << 4), /* IOMMU_READ | IOMMU_WRITE */ +}; + +static const sysmmu_pte_t *LV2_PROT; +static const sysmmu_pte_t SYSMMU_LV2_PROT[] = { + ((0 << 9) | (0 << 4)), /* no access */ + ((1 << 9) | (1 << 4)), /* IOMMU_READ only */ + ((0 << 9) | (1 << 4)), /* IOMMU_WRITE not supported, use read/write */ + ((0 << 9) | (1 << 4)), /* IOMMU_READ | IOMMU_WRITE */ +}; +static const sysmmu_pte_t SYSMMU_V5_LV2_PROT[] = { + (0 << 2), /* no access */ + (1 << 2), /* IOMMU_READ only */ + (2 << 2), /* IOMMU_WRITE only */ + (3 << 2), /* IOMMU_READ | IOMMU_WRITE */ +}; + +#define SYSMMU_SUPPORTED_PROT_BITS (IOMMU_READ | IOMMU_WRITE) + #define sect_to_phys(ent) (((phys_addr_t) ent) << PG_ENT_SHIFT) #define section_phys(sent) (sect_to_phys(*(sent)) & SECT_MASK) #define section_offs(iova) (iova & (SECT_SIZE - 1)) @@ -97,16 +127,17 @@ static u32 lv2ent_offset(sysmmu_iova_t iova) #define SPAGES_PER_LPAGE (LPAGE_SIZE / SPAGE_SIZE) #define lv2table_base(sent) (sect_to_phys(*(sent) & 0xFFFFFFC0)) -#define mk_lv1ent_sect(pa) ((pa >> PG_ENT_SHIFT) | 2) +#define mk_lv1ent_sect(pa, prot) ((pa >> PG_ENT_SHIFT) | LV1_PROT[prot] | 2) #define mk_lv1ent_page(pa) ((pa >> PG_ENT_SHIFT) | 1) -#define mk_lv2ent_lpage(pa) ((pa >> PG_ENT_SHIFT) | 1) -#define mk_lv2ent_spage(pa) ((pa >> PG_ENT_SHIFT) | 2) +#define mk_lv2ent_lpage(pa, prot) ((pa >> PG_ENT_SHIFT) | LV2_PROT[prot] | 1) +#define mk_lv2ent_spage(pa, prot) ((pa >> PG_ENT_SHIFT) | LV2_PROT[prot] | 2) #define CTRL_ENABLE 0x5 #define CTRL_BLOCK 0x7 #define CTRL_DISABLE 0x0 #define CFG_LRU 0x1 +#define CFG_EAP (1 << 2) #define CFG_QOS(n) ((n & 0xF) << 7) #define CFG_ACGEN (1 << 24) /* System MMU 3.3 only */ #define CFG_SYSSEL (1 << 22) /* System MMU 3.2 only */ @@ -206,6 +237,7 @@ static const struct sysmmu_fault_info sysmmu_v5_faults[] = { struct exynos_iommu_owner { struct list_head controllers; /* list of sysmmu_drvdata.owner_node */ struct iommu_domain *domain; /* domain this device is attached */ + struct mutex rpm_lock; /* for runtime pm of all sysmmus */ }; /* @@ -237,8 +269,8 @@ struct sysmmu_drvdata { struct clk *aclk; /* SYSMMU's aclk clock */ struct clk *pclk; /* SYSMMU's pclk clock */ struct clk *clk_master; /* master's device clock */ - int activations; /* number of calls to sysmmu_enable */ spinlock_t lock; /* lock for modyfying state */ + bool active; /* current status */ struct exynos_iommu_domain *domain; /* domain we belong to */ struct list_head domain_node; /* node for domain clients list */ struct list_head owner_node; /* node for owner controllers list */ @@ -251,25 +283,6 @@ static struct exynos_iommu_domain *to_exynos_domain(struct iommu_domain *dom) return container_of(dom, struct exynos_iommu_domain, domain); } -static bool set_sysmmu_active(struct sysmmu_drvdata *data) -{ - /* return true if the System MMU was not active previously - and it needs to be initialized */ - return ++data->activations == 1; -} - -static bool set_sysmmu_inactive(struct sysmmu_drvdata *data) -{ - /* return true if the System MMU is needed to be disabled */ - BUG_ON(data->activations < 1); - return --data->activations == 0; -} - -static bool is_sysmmu_active(struct sysmmu_drvdata *data) -{ - return data->activations > 0; -} - static void sysmmu_unblock(struct sysmmu_drvdata *data) { writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL); @@ -388,7 +401,7 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) unsigned short reg_status, reg_clear; int ret = -ENOSYS; - WARN_ON(!is_sysmmu_active(data)); + WARN_ON(!data->active); if (MMU_MAJ_VER(data->version) < 5) { reg_status = REG_INT_STATUS; @@ -434,40 +447,19 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) return IRQ_HANDLED; } -static void __sysmmu_disable_nocount(struct sysmmu_drvdata *data) +static void __sysmmu_disable(struct sysmmu_drvdata *data) { + unsigned long flags; + clk_enable(data->clk_master); + spin_lock_irqsave(&data->lock, flags); writel(CTRL_DISABLE, data->sfrbase + REG_MMU_CTRL); writel(0, data->sfrbase + REG_MMU_CFG); - - __sysmmu_disable_clocks(data); -} - -static bool __sysmmu_disable(struct sysmmu_drvdata *data) -{ - bool disabled; - unsigned long flags; - - spin_lock_irqsave(&data->lock, flags); - - disabled = set_sysmmu_inactive(data); - - if (disabled) { - data->pgtable = 0; - data->domain = NULL; - - __sysmmu_disable_nocount(data); - - dev_dbg(data->sysmmu, "Disabled\n"); - } else { - dev_dbg(data->sysmmu, "%d times left to disable\n", - data->activations); - } - + data->active = false; spin_unlock_irqrestore(&data->lock, flags); - return disabled; + __sysmmu_disable_clocks(data); } static void __sysmmu_init_config(struct sysmmu_drvdata *data) @@ -481,20 +473,24 @@ static void __sysmmu_init_config(struct sysmmu_drvdata *data) else cfg = CFG_QOS(15) | CFG_FLPDCACHE | CFG_ACGEN; + cfg |= CFG_EAP; /* enable access protection bits check */ + writel(cfg, data->sfrbase + REG_MMU_CFG); } -static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data) +static void __sysmmu_enable(struct sysmmu_drvdata *data) { + unsigned long flags; + __sysmmu_enable_clocks(data); + spin_lock_irqsave(&data->lock, flags); writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL); - __sysmmu_init_config(data); - __sysmmu_set_ptbase(data, data->pgtable); - writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL); + data->active = true; + spin_unlock_irqrestore(&data->lock, flags); /* * SYSMMU driver keeps master's clock enabled only for the short @@ -505,48 +501,18 @@ static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data) clk_disable(data->clk_master); } -static int __sysmmu_enable(struct sysmmu_drvdata *data, phys_addr_t pgtable, - struct exynos_iommu_domain *domain) -{ - int ret = 0; - unsigned long flags; - - spin_lock_irqsave(&data->lock, flags); - if (set_sysmmu_active(data)) { - data->pgtable = pgtable; - data->domain = domain; - - __sysmmu_enable_nocount(data); - - dev_dbg(data->sysmmu, "Enabled\n"); - } else { - ret = (pgtable == data->pgtable) ? 1 : -EBUSY; - - dev_dbg(data->sysmmu, "already enabled\n"); - } - - if (WARN_ON(ret < 0)) - set_sysmmu_inactive(data); /* decrement count */ - - spin_unlock_irqrestore(&data->lock, flags); - - return ret; -} - static void sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data, sysmmu_iova_t iova) { unsigned long flags; - spin_lock_irqsave(&data->lock, flags); - if (is_sysmmu_active(data) && data->version >= MAKE_MMU_VER(3, 3)) { + if (data->active && data->version >= MAKE_MMU_VER(3, 3)) { clk_enable(data->clk_master); __sysmmu_tlb_invalidate_entry(data, iova, 1); clk_disable(data->clk_master); } spin_unlock_irqrestore(&data->lock, flags); - } static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, @@ -555,7 +521,7 @@ static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, unsigned long flags; spin_lock_irqsave(&data->lock, flags); - if (is_sysmmu_active(data)) { + if (data->active) { unsigned int num_inv = 1; clk_enable(data->clk_master); @@ -578,9 +544,6 @@ static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, sysmmu_unblock(data); } clk_disable(data->clk_master); - } else { - dev_dbg(data->master, - "disabled. Skipping TLB invalidation @ %#x\n", iova); } spin_unlock_irqrestore(&data->lock, flags); } @@ -652,10 +615,15 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev) __sysmmu_get_version(data); if (PG_ENT_SHIFT < 0) { - if (MMU_MAJ_VER(data->version) < 5) + if (MMU_MAJ_VER(data->version) < 5) { PG_ENT_SHIFT = SYSMMU_PG_ENT_SHIFT; - else + LV1_PROT = SYSMMU_LV1_PROT; + LV2_PROT = SYSMMU_LV2_PROT; + } else { PG_ENT_SHIFT = SYSMMU_V5_PG_ENT_SHIFT; + LV1_PROT = SYSMMU_V5_LV1_PROT; + LV2_PROT = SYSMMU_V5_LV2_PROT; + } } pm_runtime_enable(dev); @@ -665,34 +633,46 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev) return 0; } -#ifdef CONFIG_PM_SLEEP -static int exynos_sysmmu_suspend(struct device *dev) +static int __maybe_unused exynos_sysmmu_suspend(struct device *dev) { struct sysmmu_drvdata *data = dev_get_drvdata(dev); + struct device *master = data->master; + + if (master) { + struct exynos_iommu_owner *owner = master->archdata.iommu; - dev_dbg(dev, "suspend\n"); - if (is_sysmmu_active(data)) { - __sysmmu_disable_nocount(data); - pm_runtime_put(dev); + mutex_lock(&owner->rpm_lock); + if (data->domain) { + dev_dbg(data->sysmmu, "saving state\n"); + __sysmmu_disable(data); + } + mutex_unlock(&owner->rpm_lock); } return 0; } -static int exynos_sysmmu_resume(struct device *dev) +static int __maybe_unused exynos_sysmmu_resume(struct device *dev) { struct sysmmu_drvdata *data = dev_get_drvdata(dev); + struct device *master = data->master; + + if (master) { + struct exynos_iommu_owner *owner = master->archdata.iommu; - dev_dbg(dev, "resume\n"); - if (is_sysmmu_active(data)) { - pm_runtime_get_sync(dev); - __sysmmu_enable_nocount(data); + mutex_lock(&owner->rpm_lock); + if (data->domain) { + dev_dbg(data->sysmmu, "restoring state\n"); + __sysmmu_enable(data); + } + mutex_unlock(&owner->rpm_lock); } return 0; } -#endif static const struct dev_pm_ops sysmmu_pm_ops = { - SET_LATE_SYSTEM_SLEEP_PM_OPS(exynos_sysmmu_suspend, exynos_sysmmu_resume) + SET_RUNTIME_PM_OPS(exynos_sysmmu_suspend, exynos_sysmmu_resume, NULL) + SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, + pm_runtime_force_resume) }; static const struct of_device_id sysmmu_of_match[] __initconst = { @@ -796,9 +776,12 @@ static void exynos_iommu_domain_free(struct iommu_domain *iommu_domain) spin_lock_irqsave(&domain->lock, flags); list_for_each_entry_safe(data, next, &domain->clients, domain_node) { - if (__sysmmu_disable(data)) - data->master = NULL; + spin_lock(&data->lock); + __sysmmu_disable(data); + data->pgtable = 0; + data->domain = NULL; list_del_init(&data->domain_node); + spin_unlock(&data->lock); } spin_unlock_irqrestore(&domain->lock, flags); @@ -832,31 +815,34 @@ static void exynos_iommu_detach_device(struct iommu_domain *iommu_domain, phys_addr_t pagetable = virt_to_phys(domain->pgtable); struct sysmmu_drvdata *data, *next; unsigned long flags; - bool found = false; if (!has_sysmmu(dev) || owner->domain != iommu_domain) return; + mutex_lock(&owner->rpm_lock); + + list_for_each_entry(data, &owner->controllers, owner_node) { + pm_runtime_get_noresume(data->sysmmu); + if (pm_runtime_active(data->sysmmu)) + __sysmmu_disable(data); + pm_runtime_put(data->sysmmu); + } + spin_lock_irqsave(&domain->lock, flags); list_for_each_entry_safe(data, next, &domain->clients, domain_node) { - if (data->master == dev) { - if (__sysmmu_disable(data)) { - data->master = NULL; - list_del_init(&data->domain_node); - } - pm_runtime_put(data->sysmmu); - found = true; - } + spin_lock(&data->lock); + data->pgtable = 0; + data->domain = NULL; + list_del_init(&data->domain_node); + spin_unlock(&data->lock); } + owner->domain = NULL; spin_unlock_irqrestore(&domain->lock, flags); - owner->domain = NULL; + mutex_unlock(&owner->rpm_lock); - if (found) - dev_dbg(dev, "%s: Detached IOMMU with pgtable %pa\n", - __func__, &pagetable); - else - dev_err(dev, "%s: No IOMMU is attached\n", __func__); + dev_dbg(dev, "%s: Detached IOMMU with pgtable %pa\n", __func__, + &pagetable); } static int exynos_iommu_attach_device(struct iommu_domain *iommu_domain, @@ -867,7 +853,6 @@ static int exynos_iommu_attach_device(struct iommu_domain *iommu_domain, struct sysmmu_drvdata *data; phys_addr_t pagetable = virt_to_phys(domain->pgtable); unsigned long flags; - int ret = -ENODEV; if (!has_sysmmu(dev)) return -ENODEV; @@ -875,29 +860,32 @@ static int exynos_iommu_attach_device(struct iommu_domain *iommu_domain, if (owner->domain) exynos_iommu_detach_device(owner->domain, dev); + mutex_lock(&owner->rpm_lock); + + spin_lock_irqsave(&domain->lock, flags); list_for_each_entry(data, &owner->controllers, owner_node) { - pm_runtime_get_sync(data->sysmmu); - ret = __sysmmu_enable(data, pagetable, domain); - if (ret >= 0) { - data->master = dev; - - spin_lock_irqsave(&domain->lock, flags); - list_add_tail(&data->domain_node, &domain->clients); - spin_unlock_irqrestore(&domain->lock, flags); - } + spin_lock(&data->lock); + data->pgtable = pagetable; + data->domain = domain; + list_add_tail(&data->domain_node, &domain->clients); + spin_unlock(&data->lock); } + owner->domain = iommu_domain; + spin_unlock_irqrestore(&domain->lock, flags); - if (ret < 0) { - dev_err(dev, "%s: Failed to attach IOMMU with pgtable %pa\n", - __func__, &pagetable); - return ret; + list_for_each_entry(data, &owner->controllers, owner_node) { + pm_runtime_get_noresume(data->sysmmu); + if (pm_runtime_active(data->sysmmu)) + __sysmmu_enable(data); + pm_runtime_put(data->sysmmu); } - owner->domain = iommu_domain; - dev_dbg(dev, "%s: Attached IOMMU with pgtable %pa %s\n", - __func__, &pagetable, (ret == 0) ? "" : ", again"); + mutex_unlock(&owner->rpm_lock); - return ret; + dev_dbg(dev, "%s: Attached IOMMU with pgtable %pa\n", __func__, + &pagetable); + + return 0; } static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *domain, @@ -954,7 +942,7 @@ static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *domain, static int lv1set_section(struct exynos_iommu_domain *domain, sysmmu_pte_t *sent, sysmmu_iova_t iova, - phys_addr_t paddr, short *pgcnt) + phys_addr_t paddr, int prot, short *pgcnt) { if (lv1ent_section(sent)) { WARN(1, "Trying mapping on 1MiB@%#08x that is mapped", @@ -973,7 +961,7 @@ static int lv1set_section(struct exynos_iommu_domain *domain, *pgcnt = 0; } - update_pte(sent, mk_lv1ent_sect(paddr)); + update_pte(sent, mk_lv1ent_sect(paddr, prot)); spin_lock(&domain->lock); if (lv1ent_page_zero(sent)) { @@ -991,13 +979,13 @@ static int lv1set_section(struct exynos_iommu_domain *domain, } static int lv2set_page(sysmmu_pte_t *pent, phys_addr_t paddr, size_t size, - short *pgcnt) + int prot, short *pgcnt) { if (size == SPAGE_SIZE) { if (WARN_ON(!lv2ent_fault(pent))) return -EADDRINUSE; - update_pte(pent, mk_lv2ent_spage(paddr)); + update_pte(pent, mk_lv2ent_spage(paddr, prot)); *pgcnt -= 1; } else { /* size == LPAGE_SIZE */ int i; @@ -1013,7 +1001,7 @@ static int lv2set_page(sysmmu_pte_t *pent, phys_addr_t paddr, size_t size, return -EADDRINUSE; } - *pent = mk_lv2ent_lpage(paddr); + *pent = mk_lv2ent_lpage(paddr, prot); } dma_sync_single_for_device(dma_dev, pent_base, sizeof(*pent) * SPAGES_PER_LPAGE, @@ -1061,13 +1049,14 @@ static int exynos_iommu_map(struct iommu_domain *iommu_domain, int ret = -ENOMEM; BUG_ON(domain->pgtable == NULL); + prot &= SYSMMU_SUPPORTED_PROT_BITS; spin_lock_irqsave(&domain->pgtablelock, flags); entry = section_entry(domain->pgtable, iova); if (size == SECT_SIZE) { - ret = lv1set_section(domain, entry, iova, paddr, + ret = lv1set_section(domain, entry, iova, paddr, prot, &domain->lv2entcnt[lv1ent_offset(iova)]); } else { sysmmu_pte_t *pent; @@ -1078,7 +1067,7 @@ static int exynos_iommu_map(struct iommu_domain *iommu_domain, if (IS_ERR(pent)) ret = PTR_ERR(pent); else - ret = lv2set_page(pent, paddr, size, + ret = lv2set_page(pent, paddr, size, prot, &domain->lv2entcnt[lv1ent_offset(iova)]); } @@ -1268,10 +1257,20 @@ static int exynos_iommu_of_xlate(struct device *dev, return -ENOMEM; INIT_LIST_HEAD(&owner->controllers); + mutex_init(&owner->rpm_lock); dev->archdata.iommu = owner; } list_add_tail(&data->owner_node, &owner->controllers); + data->master = dev; + + /* + * SYSMMU will be runtime activated via device link (dependency) to its + * master device, so there are no direct calls to pm_runtime_get/put + * in this driver. + */ + device_link_add(dev, data->sysmmu, DL_FLAG_PM_RUNTIME); + return 0; } diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index a4407eabf0e6..d8376c2d18b3 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -892,7 +892,13 @@ static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devf return NULL; if (dev_is_pci(dev)) { + struct pci_dev *pf_pdev; + pdev = to_pci_dev(dev); + /* VFs aren't listed in scope tables; we need to look up + * the PF instead to find the IOMMU. */ + pf_pdev = pci_physfn(pdev); + dev = &pf_pdev->dev; segment = pci_domain_nr(pdev->bus); } else if (has_acpi_companion(dev)) dev = &ACPI_COMPANION(dev)->dev; @@ -905,6 +911,13 @@ static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devf for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, tmp) { if (tmp == dev) { + /* For a VF use its original BDF# not that of the PF + * which we used for the IOMMU lookup. Strictly speaking + * we could do this for all PCI devices; we only need to + * get the BDF# from the scope table for ACPI matches. */ + if (pdev->is_virtfn) + goto got_pdev; + *bus = drhd->devices[i].bus; *devfn = drhd->devices[i].devfn; goto out; @@ -1711,6 +1724,7 @@ static void disable_dmar_iommu(struct intel_iommu *iommu) if (!iommu->domains || !iommu->domain_ids) return; +again: spin_lock_irqsave(&device_domain_lock, flags); list_for_each_entry_safe(info, tmp, &device_domain_list, global) { struct dmar_domain *domain; @@ -1723,10 +1737,19 @@ static void disable_dmar_iommu(struct intel_iommu *iommu) domain = info->domain; - dmar_remove_one_dev_info(domain, info->dev); + __dmar_remove_one_dev_info(info); - if (!domain_type_is_vm_or_si(domain)) + if (!domain_type_is_vm_or_si(domain)) { + /* + * The domain_exit() function can't be called under + * device_domain_lock, as it takes this lock itself. + * So release the lock here and re-run the loop + * afterwards. + */ + spin_unlock_irqrestore(&device_domain_lock, flags); domain_exit(domain); + goto again; + } } spin_unlock_irqrestore(&device_domain_lock, flags); diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel-svm.c index 8ebb3530afa7..cb72e0011310 100644 --- a/drivers/iommu/intel-svm.c +++ b/drivers/iommu/intel-svm.c @@ -39,10 +39,18 @@ int intel_svm_alloc_pasid_tables(struct intel_iommu *iommu) struct page *pages; int order; - order = ecap_pss(iommu->ecap) + 7 - PAGE_SHIFT; - if (order < 0) - order = 0; - + /* Start at 2 because it's defined as 2^(1+PSS) */ + iommu->pasid_max = 2 << ecap_pss(iommu->ecap); + + /* Eventually I'm promised we will get a multi-level PASID table + * and it won't have to be physically contiguous. Until then, + * limit the size because 8MiB contiguous allocations can be hard + * to come by. The limit of 0x20000, which is 1MiB for each of + * the PASID and PASID-state tables, is somewhat arbitrary. */ + if (iommu->pasid_max > 0x20000) + iommu->pasid_max = 0x20000; + + order = get_order(sizeof(struct pasid_entry) * iommu->pasid_max); pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, order); if (!pages) { pr_warn("IOMMU: %s: Failed to allocate PASID table\n", @@ -53,6 +61,8 @@ int intel_svm_alloc_pasid_tables(struct intel_iommu *iommu) pr_info("%s: Allocated order %d PASID table.\n", iommu->name, order); if (ecap_dis(iommu->ecap)) { + /* Just making it explicit... */ + BUILD_BUG_ON(sizeof(struct pasid_entry) != sizeof(struct pasid_state_entry)); pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, order); if (pages) iommu->pasid_state_table = page_address(pages); @@ -68,11 +78,7 @@ int intel_svm_alloc_pasid_tables(struct intel_iommu *iommu) int intel_svm_free_pasid_tables(struct intel_iommu *iommu) { - int order; - - order = ecap_pss(iommu->ecap) + 7 - PAGE_SHIFT; - if (order < 0) - order = 0; + int order = get_order(sizeof(struct pasid_entry) * iommu->pasid_max); if (iommu->pasid_table) { free_pages((unsigned long)iommu->pasid_table, order); @@ -371,8 +377,8 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_ } svm->iommu = iommu; - if (pasid_max > 2 << ecap_pss(iommu->ecap)) - pasid_max = 2 << ecap_pss(iommu->ecap); + if (pasid_max > iommu->pasid_max) + pasid_max = iommu->pasid_max; /* Do not use PASID 0 in caching mode (virtualised IOMMU) */ ret = idr_alloc(&iommu->pasid_idr, svm, diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c index f50e51c1a9c8..0769276c0537 100644 --- a/drivers/iommu/io-pgtable-arm-v7s.c +++ b/drivers/iommu/io-pgtable-arm-v7s.c @@ -793,8 +793,7 @@ static int __init arm_v7s_do_selftests(void) * Distinct mappings of different granule sizes. */ iova = 0; - i = find_first_bit(&cfg.pgsize_bitmap, BITS_PER_LONG); - while (i != BITS_PER_LONG) { + for_each_set_bit(i, &cfg.pgsize_bitmap, BITS_PER_LONG) { size = 1UL << i; if (ops->map(ops, iova, iova, size, IOMMU_READ | IOMMU_WRITE | @@ -811,8 +810,6 @@ static int __init arm_v7s_do_selftests(void) return __FAIL(ops); iova += SZ_16M; - i++; - i = find_next_bit(&cfg.pgsize_bitmap, BITS_PER_LONG, i); loopnr++; } diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index f5c90e1366ce..a40ce3406fef 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -916,7 +916,7 @@ static void dummy_tlb_sync(void *cookie) WARN_ON(cookie != cfg_cookie); } -static struct iommu_gather_ops dummy_tlb_ops __initdata = { +static const struct iommu_gather_ops dummy_tlb_ops __initconst = { .tlb_flush_all = dummy_tlb_flush_all, .tlb_add_flush = dummy_tlb_add_flush, .tlb_sync = dummy_tlb_sync, @@ -980,8 +980,7 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg) * Distinct mappings of different granule sizes. */ iova = 0; - j = find_first_bit(&cfg->pgsize_bitmap, BITS_PER_LONG); - while (j != BITS_PER_LONG) { + for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) { size = 1UL << j; if (ops->map(ops, iova, iova, size, IOMMU_READ | @@ -999,8 +998,6 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg) return __FAIL(ops, i); iova += SZ_1G; - j++; - j = find_next_bit(&cfg->pgsize_bitmap, BITS_PER_LONG, j); } /* Partial unmap */ diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 9a2f1960873b..dbe7f653bb7c 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -552,6 +552,19 @@ struct iommu_group *iommu_group_get(struct device *dev) EXPORT_SYMBOL_GPL(iommu_group_get); /** + * iommu_group_ref_get - Increment reference on a group + * @group: the group to use, must not be NULL + * + * This function is called by iommu drivers to take additional references on an + * existing group. Returns the given group for convenience. + */ +struct iommu_group *iommu_group_ref_get(struct iommu_group *group) +{ + kobject_get(group->devices_kobj); + return group; +} + +/** * iommu_group_put - Decrement group reference * @group: the group to use * @@ -1615,6 +1628,46 @@ out: return ret; } +struct iommu_instance { + struct list_head list; + struct fwnode_handle *fwnode; + const struct iommu_ops *ops; +}; +static LIST_HEAD(iommu_instance_list); +static DEFINE_SPINLOCK(iommu_instance_lock); + +void iommu_register_instance(struct fwnode_handle *fwnode, + const struct iommu_ops *ops) +{ + struct iommu_instance *iommu = kzalloc(sizeof(*iommu), GFP_KERNEL); + + if (WARN_ON(!iommu)) + return; + + of_node_get(to_of_node(fwnode)); + INIT_LIST_HEAD(&iommu->list); + iommu->fwnode = fwnode; + iommu->ops = ops; + spin_lock(&iommu_instance_lock); + list_add_tail(&iommu->list, &iommu_instance_list); + spin_unlock(&iommu_instance_lock); +} + +const struct iommu_ops *iommu_get_instance(struct fwnode_handle *fwnode) +{ + struct iommu_instance *instance; + const struct iommu_ops *ops = NULL; + + spin_lock(&iommu_instance_lock); + list_for_each_entry(instance, &iommu_instance_list, list) + if (instance->fwnode == fwnode) { + ops = instance->ops; + break; + } + spin_unlock(&iommu_instance_lock); + return ops; +} + int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode, const struct iommu_ops *ops) { diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index e23001bfcfee..080beca0197d 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -56,7 +56,7 @@ EXPORT_SYMBOL_GPL(init_iova_domain); static struct rb_node * __get_cached_rbnode(struct iova_domain *iovad, unsigned long *limit_pfn) { - if ((*limit_pfn != iovad->dma_32bit_pfn) || + if ((*limit_pfn > iovad->dma_32bit_pfn) || (iovad->cached32_node == NULL)) return rb_last(&iovad->rbroot); else { diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 55b29620fd4f..1479c76ece9e 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -394,6 +394,8 @@ static struct iommu_group *mtk_iommu_device_group(struct device *dev) data->m4u_group = iommu_group_alloc(); if (IS_ERR(data->m4u_group)) dev_err(dev, "Failed to allocate M4U IOMMU group\n"); + } else { + iommu_group_ref_get(data->m4u_group); } return data->m4u_group; } diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c index af70040038e9..19e010083408 100644 --- a/drivers/iommu/mtk_iommu_v1.c +++ b/drivers/iommu/mtk_iommu_v1.c @@ -479,6 +479,8 @@ static struct iommu_group *mtk_iommu_device_group(struct device *dev) data->m4u_group = iommu_group_alloc(); if (IS_ERR(data->m4u_group)) dev_err(dev, "Failed to allocate M4U IOMMU group\n"); + } else { + iommu_group_ref_get(data->m4u_group); } return data->m4u_group; } diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c index 5b82862f571f..0f57ddc4ecc2 100644 --- a/drivers/iommu/of_iommu.c +++ b/drivers/iommu/of_iommu.c @@ -96,45 +96,6 @@ int of_get_dma_window(struct device_node *dn, const char *prefix, int index, } EXPORT_SYMBOL_GPL(of_get_dma_window); -struct of_iommu_node { - struct list_head list; - struct device_node *np; - const struct iommu_ops *ops; -}; -static LIST_HEAD(of_iommu_list); -static DEFINE_SPINLOCK(of_iommu_lock); - -void of_iommu_set_ops(struct device_node *np, const struct iommu_ops *ops) -{ - struct of_iommu_node *iommu = kzalloc(sizeof(*iommu), GFP_KERNEL); - - if (WARN_ON(!iommu)) - return; - - of_node_get(np); - INIT_LIST_HEAD(&iommu->list); - iommu->np = np; - iommu->ops = ops; - spin_lock(&of_iommu_lock); - list_add_tail(&iommu->list, &of_iommu_list); - spin_unlock(&of_iommu_lock); -} - -const struct iommu_ops *of_iommu_get_ops(struct device_node *np) -{ - struct of_iommu_node *node; - const struct iommu_ops *ops = NULL; - - spin_lock(&of_iommu_lock); - list_for_each_entry(node, &of_iommu_list, list) - if (node->np == np) { - ops = node->ops; - break; - } - spin_unlock(&of_iommu_lock); - return ops; -} - static int __get_pci_rid(struct pci_dev *pdev, u16 alias, void *data) { struct of_phandle_args *iommu_spec = data; diff --git a/drivers/iommu/s390-iommu.c b/drivers/iommu/s390-iommu.c index 3b44b1d82f3b..179e636a4d91 100644 --- a/drivers/iommu/s390-iommu.c +++ b/drivers/iommu/s390-iommu.c @@ -8,7 +8,6 @@ #include <linux/pci.h> #include <linux/iommu.h> #include <linux/iommu-helper.h> -#include <linux/pci.h> #include <linux/sizes.h> #include <asm/pci_dma.h> |