From 1b932ceddd19de1cdf2a86f1fca77c37dad758cc Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Tue, 10 Jan 2023 10:54:04 +0800 Subject: iommu: Remove detach_dev callbacks The iommu core calls the driver's detach_dev domain op callback only when a device is finished assigning to user space and iommu_group_release_dma_owner() is called to return the device to the kernel, where iommu core wants to set the default domain to the device but the driver didn't provide one. In other words, if any iommu driver provides default domain support, the .detach_dev callback will never be called. This removes the detach_dev callbacks in those IOMMU drivers that support default domain. Reviewed-by: Jason Gunthorpe Reviewed-by: Sven Peter # apple-dart Acked-by: Chunyan Zhang # sprd Reviewed-by: Vasant Hegde # amd Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20230110025408.667767-2-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/exynos-iommu.c | 1 - 1 file changed, 1 deletion(-) (limited to 'drivers/iommu/exynos-iommu.c') diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index b0cde2211987..29ec713e8a21 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -1404,7 +1404,6 @@ static const struct iommu_ops exynos_iommu_ops = { .of_xlate = exynos_iommu_of_xlate, .default_domain_ops = &(const struct iommu_domain_ops) { .attach_dev = exynos_iommu_attach_device, - .detach_dev = exynos_iommu_detach_device, .map = exynos_iommu_map, .unmap = exynos_iommu_unmap, .iova_to_phys = exynos_iommu_iova_to_phys, -- cgit v1.2.3 From 53719876e064643a6e58b5e6067a149a0fd191ec Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Wed, 4 Jan 2023 17:57:02 +0800 Subject: iommu/exynos: Fix error handling in exynos_iommu_init() If platform_driver_register() fails, it don't need unregister and call kmem_cache_free() to free the memory allocated before calling register. Fixes: bbc4d205d93f ("iommu/exynos: Fix driver initialization sequence") Signed-off-by: Yang Yingliang Reviewed-by: Sam Protsenko Link: https://lore.kernel.org/r/20230104095702.2591122-1-yangyingliang@huawei.com Signed-off-by: Joerg Roedel --- drivers/iommu/exynos-iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu/exynos-iommu.c') diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index b0cde2211987..c1d579c24740 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -1446,7 +1446,7 @@ static int __init exynos_iommu_init(void) return 0; err_reg_driver: - platform_driver_unregister(&exynos_sysmmu_driver); + kmem_cache_free(lv2table_kmem_cache, zero_lv2_table); err_zero_lv2: kmem_cache_destroy(lv2table_kmem_cache); return ret; -- cgit v1.2.3 From c64074bfe2abbe18a57eb19c95f686298e8692b5 Mon Sep 17 00:00:00 2001 From: Sam Protsenko Date: Tue, 26 Jul 2022 23:07:38 +0300 Subject: iommu/exynos: Abstract getting the fault info Fault info obtaining is implemented for SysMMU v1..v5 in a very hardware specific way, as it relies on: - interrupt bits being tied to read or write access - having separate registers for the fault address w.r.t. AR/AW ops Newer SysMMU versions (like SysMMU v7) have different way of providing the fault info via registers: - the transaction type (read or write) should be read from the register (instead of hard-coding it w.r.t. corresponding interrupt status bit) - there is only one single register for storing the fault address Because of that, it is not possible to add newer SysMMU support into existing paradigm. Also it's not very effective performance-wise: - checking SysMMU version in ISR each time is not necessary - performing linear search to find the fault info by interrupt bit can be replaced with a single lookup operation Pave the way for adding support for new SysMMU versions by abstracting the getting of fault info in ISR. While at it, do some related style cleanups as well. This is mostly a refactoring patch, but there are some minor functional changes: - fault message format is a bit different; now instead of AR/AW prefixes for the fault's name, the request direction is printed as [READ]/[WRITE]. It has to be done to prepare an abstraction for SysMMU v7 support - don't panic on unknown interrupts; print corresponding message and continue - if fault wasn't recovered, panic with some sane message instead of just doing BUG_ON() The whole fault message looks like this now: [READ] PAGE FAULT occurred at 0x12341000 Signed-off-by: Sam Protsenko Acked-by: Marek Szyprowski Link: https://lore.kernel.org/r/20220726200739.30017-2-semen.protsenko@linaro.org Signed-off-by: Joerg Roedel --- drivers/iommu/exynos-iommu.c | 162 ++++++++++++++++++++++++++----------------- 1 file changed, 100 insertions(+), 62 deletions(-) (limited to 'drivers/iommu/exynos-iommu.c') diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index c1d579c24740..75cade7382ab 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -185,38 +185,36 @@ static sysmmu_pte_t *page_entry(sysmmu_pte_t *sent, sysmmu_iova_t iova) lv2table_base(sent)) + lv2ent_offset(iova); } -/* - * IOMMU fault information register - */ -struct sysmmu_fault_info { - unsigned int bit; /* bit number in STATUS register */ - unsigned short addr_reg; /* register to read VA fault address */ +struct sysmmu_fault { + sysmmu_iova_t addr; /* IOVA address that caused fault */ + const char *name; /* human readable fault name */ + unsigned int type; /* fault type for report_iommu_fault() */ +}; + +struct sysmmu_v1_fault_info { + unsigned short addr_reg; /* register to read IOVA fault address */ const char *name; /* human readable fault name */ unsigned int type; /* fault type for report_iommu_fault */ }; -static const struct sysmmu_fault_info sysmmu_faults[] = { - { 0, REG_PAGE_FAULT_ADDR, "PAGE", IOMMU_FAULT_READ }, - { 1, REG_AR_FAULT_ADDR, "AR MULTI-HIT", IOMMU_FAULT_READ }, - { 2, REG_AW_FAULT_ADDR, "AW MULTI-HIT", IOMMU_FAULT_WRITE }, - { 3, REG_DEFAULT_SLAVE_ADDR, "BUS ERROR", IOMMU_FAULT_READ }, - { 4, REG_AR_FAULT_ADDR, "AR SECURITY PROTECTION", IOMMU_FAULT_READ }, - { 5, REG_AR_FAULT_ADDR, "AR ACCESS PROTECTION", IOMMU_FAULT_READ }, - { 6, REG_AW_FAULT_ADDR, "AW SECURITY PROTECTION", IOMMU_FAULT_WRITE }, - { 7, REG_AW_FAULT_ADDR, "AW ACCESS PROTECTION", IOMMU_FAULT_WRITE }, +static const struct sysmmu_v1_fault_info sysmmu_v1_faults[] = { + { REG_PAGE_FAULT_ADDR, "PAGE", IOMMU_FAULT_READ }, + { REG_AR_FAULT_ADDR, "MULTI-HIT", IOMMU_FAULT_READ }, + { REG_AW_FAULT_ADDR, "MULTI-HIT", IOMMU_FAULT_WRITE }, + { REG_DEFAULT_SLAVE_ADDR, "BUS ERROR", IOMMU_FAULT_READ }, + { REG_AR_FAULT_ADDR, "SECURITY PROTECTION", IOMMU_FAULT_READ }, + { REG_AR_FAULT_ADDR, "ACCESS PROTECTION", IOMMU_FAULT_READ }, + { REG_AW_FAULT_ADDR, "SECURITY PROTECTION", IOMMU_FAULT_WRITE }, + { REG_AW_FAULT_ADDR, "ACCESS PROTECTION", IOMMU_FAULT_WRITE }, }; -static const struct sysmmu_fault_info sysmmu_v5_faults[] = { - { 0, REG_V5_FAULT_AR_VA, "AR PTW", IOMMU_FAULT_READ }, - { 1, REG_V5_FAULT_AR_VA, "AR PAGE", IOMMU_FAULT_READ }, - { 2, REG_V5_FAULT_AR_VA, "AR MULTI-HIT", IOMMU_FAULT_READ }, - { 3, REG_V5_FAULT_AR_VA, "AR ACCESS PROTECTION", IOMMU_FAULT_READ }, - { 4, REG_V5_FAULT_AR_VA, "AR SECURITY PROTECTION", IOMMU_FAULT_READ }, - { 16, REG_V5_FAULT_AW_VA, "AW PTW", IOMMU_FAULT_WRITE }, - { 17, REG_V5_FAULT_AW_VA, "AW PAGE", IOMMU_FAULT_WRITE }, - { 18, REG_V5_FAULT_AW_VA, "AW MULTI-HIT", IOMMU_FAULT_WRITE }, - { 19, REG_V5_FAULT_AW_VA, "AW ACCESS PROTECTION", IOMMU_FAULT_WRITE }, - { 20, REG_V5_FAULT_AW_VA, "AW SECURITY PROTECTION", IOMMU_FAULT_WRITE }, +/* SysMMU v5 has the same faults for AR (0..4 bits) and AW (16..20 bits) */ +static const char * const sysmmu_v5_fault_names[] = { + "PTW", + "PAGE", + "MULTI-HIT", + "ACCESS PROTECTION", + "SECURITY PROTECTION" }; /* @@ -246,9 +244,12 @@ struct exynos_iommu_domain { struct iommu_domain domain; /* generic domain data structure */ }; +struct sysmmu_drvdata; + /* * SysMMU version specific data. Contains offsets for the registers which can * be found in different SysMMU variants, but have different offset values. + * Also contains version specific callbacks to abstract the hardware. */ struct sysmmu_variant { u32 pt_base; /* page table base address (physical) */ @@ -259,6 +260,9 @@ struct sysmmu_variant { u32 flush_end; /* end address of range invalidation */ u32 int_status; /* interrupt status information */ u32 int_clear; /* clear the interrupt */ + + int (*get_fault_info)(struct sysmmu_drvdata *data, unsigned int itype, + struct sysmmu_fault *fault); }; /* @@ -293,6 +297,46 @@ struct sysmmu_drvdata { #define SYSMMU_REG(data, reg) ((data)->sfrbase + (data)->variant->reg) +static int exynos_sysmmu_v1_get_fault_info(struct sysmmu_drvdata *data, + unsigned int itype, + struct sysmmu_fault *fault) +{ + const struct sysmmu_v1_fault_info *finfo; + + if (itype >= ARRAY_SIZE(sysmmu_v1_faults)) + return -ENXIO; + + finfo = &sysmmu_v1_faults[itype]; + fault->addr = readl(data->sfrbase + finfo->addr_reg); + fault->name = finfo->name; + fault->type = finfo->type; + + return 0; +} + +static int exynos_sysmmu_v5_get_fault_info(struct sysmmu_drvdata *data, + unsigned int itype, + struct sysmmu_fault *fault) +{ + unsigned int addr_reg; + + if (itype < ARRAY_SIZE(sysmmu_v5_fault_names)) { + fault->type = IOMMU_FAULT_READ; + addr_reg = REG_V5_FAULT_AR_VA; + } else if (itype >= 16 && itype <= 20) { + fault->type = IOMMU_FAULT_WRITE; + addr_reg = REG_V5_FAULT_AW_VA; + itype -= 16; + } else { + return -ENXIO; + } + + fault->name = sysmmu_v5_fault_names[itype]; + fault->addr = readl(data->sfrbase + addr_reg); + + return 0; +} + /* SysMMU v1..v3 */ static const struct sysmmu_variant sysmmu_v1_variant = { .flush_all = 0x0c, @@ -300,6 +344,8 @@ static const struct sysmmu_variant sysmmu_v1_variant = { .pt_base = 0x14, .int_status = 0x18, .int_clear = 0x1c, + + .get_fault_info = exynos_sysmmu_v1_get_fault_info, }; /* SysMMU v5 and v7 (non-VM capable) */ @@ -312,6 +358,8 @@ static const struct sysmmu_variant sysmmu_v5_variant = { .flush_end = 0x24, .int_status = 0x60, .int_clear = 0x64, + + .get_fault_info = exynos_sysmmu_v5_get_fault_info, }; /* SysMMU v7: VM capable register set */ @@ -324,6 +372,8 @@ static const struct sysmmu_variant sysmmu_v7_vm_variant = { .flush_end = 0x8024, .int_status = 0x60, .int_clear = 0x64, + + .get_fault_info = exynos_sysmmu_v5_get_fault_info, }; static struct exynos_iommu_domain *to_exynos_domain(struct iommu_domain *dom) @@ -453,68 +503,56 @@ static void __sysmmu_get_version(struct sysmmu_drvdata *data) } static void show_fault_information(struct sysmmu_drvdata *data, - const struct sysmmu_fault_info *finfo, - sysmmu_iova_t fault_addr) + const struct sysmmu_fault *fault) { sysmmu_pte_t *ent; - dev_err(data->sysmmu, "%s: %s FAULT occurred at %#x\n", - dev_name(data->master), finfo->name, fault_addr); + dev_err(data->sysmmu, "%s: [%s] %s FAULT occurred at %#x\n", + dev_name(data->master), + fault->type == IOMMU_FAULT_READ ? "READ" : "WRITE", + fault->name, fault->addr); dev_dbg(data->sysmmu, "Page table base: %pa\n", &data->pgtable); - ent = section_entry(phys_to_virt(data->pgtable), fault_addr); + ent = section_entry(phys_to_virt(data->pgtable), fault->addr); dev_dbg(data->sysmmu, "\tLv1 entry: %#x\n", *ent); if (lv1ent_page(ent)) { - ent = page_entry(ent, fault_addr); + ent = page_entry(ent, fault->addr); dev_dbg(data->sysmmu, "\t Lv2 entry: %#x\n", *ent); } } static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) { - /* SYSMMU is in blocked state when interrupt occurred. */ struct sysmmu_drvdata *data = dev_id; - const struct sysmmu_fault_info *finfo; - unsigned int i, n, itype; - sysmmu_iova_t fault_addr; + unsigned int itype; + struct sysmmu_fault fault; int ret = -ENOSYS; WARN_ON(!data->active); - if (MMU_MAJ_VER(data->version) < 5) { - finfo = sysmmu_faults; - n = ARRAY_SIZE(sysmmu_faults); - } else { - finfo = sysmmu_v5_faults; - n = ARRAY_SIZE(sysmmu_v5_faults); - } - spin_lock(&data->lock); - clk_enable(data->clk_master); itype = __ffs(readl(SYSMMU_REG(data, int_status))); - for (i = 0; i < n; i++, finfo++) - if (finfo->bit == itype) - break; - /* unknown/unsupported fault */ - BUG_ON(i == n); - - /* print debug message */ - fault_addr = readl(data->sfrbase + finfo->addr_reg); - show_fault_information(data, finfo, fault_addr); - - if (data->domain) - ret = report_iommu_fault(&data->domain->domain, - data->master, fault_addr, finfo->type); - /* fault is not recovered by fault handler */ - BUG_ON(ret != 0); + ret = data->variant->get_fault_info(data, itype, &fault); + if (ret) { + dev_err(data->sysmmu, "Unhandled interrupt bit %u\n", itype); + goto out; + } + show_fault_information(data, &fault); + if (data->domain) { + ret = report_iommu_fault(&data->domain->domain, data->master, + fault.addr, fault.type); + } + if (ret) + panic("Unrecoverable System MMU Fault!"); + +out: writel(1 << itype, SYSMMU_REG(data, int_clear)); + /* SysMMU is in blocked state when interrupt occurred */ sysmmu_unblock(data); - clk_disable(data->clk_master); - spin_unlock(&data->lock); return IRQ_HANDLED; -- cgit v1.2.3 From 2f599c3ff4e6f89dfd27fe2f4e1a4c040ba5f92f Mon Sep 17 00:00:00 2001 From: Sam Protsenko Date: Tue, 26 Jul 2022 23:07:39 +0300 Subject: iommu/exynos: Implement fault handling on SysMMU v7 SysMMU v7 has a bit different registers for getting the fault info: - there is one single register (MMU_FAULT_VA) to get the fault address - fault access type (R/W) can be read from MMU_FAULT_TRANS_INFO register now - interrupt status register has different bits w.r.t. previous SysMMU versions - VM and non-VM layouts have different register addresses Add correct fault handling implementation for SysMMU v7, according to all mentioned differences. Only VID #0 (default) is handled, as VM domains support is not implemented yet. Signed-off-by: Sam Protsenko Link: https://lore.kernel.org/r/20220726200739.30017-3-semen.protsenko@linaro.org Signed-off-by: Joerg Roedel --- drivers/iommu/exynos-iommu.c | 48 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 4 deletions(-) (limited to 'drivers/iommu/exynos-iommu.c') diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 75cade7382ab..69fd7fa3e49a 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -217,6 +217,13 @@ static const char * const sysmmu_v5_fault_names[] = { "SECURITY PROTECTION" }; +static const char * const sysmmu_v7_fault_names[] = { + "PTW", + "PAGE", + "ACCESS PROTECTION", + "RESERVED" +}; + /* * This structure is attached to dev->iommu->priv of the master device * on device add, contains a list of SYSMMU controllers defined by device tree, @@ -260,6 +267,8 @@ struct sysmmu_variant { u32 flush_end; /* end address of range invalidation */ u32 int_status; /* interrupt status information */ u32 int_clear; /* clear the interrupt */ + u32 fault_va; /* IOVA address that caused fault */ + u32 fault_info; /* fault transaction info */ int (*get_fault_info)(struct sysmmu_drvdata *data, unsigned int itype, struct sysmmu_fault *fault); @@ -337,6 +346,19 @@ static int exynos_sysmmu_v5_get_fault_info(struct sysmmu_drvdata *data, return 0; } +static int exynos_sysmmu_v7_get_fault_info(struct sysmmu_drvdata *data, + unsigned int itype, + struct sysmmu_fault *fault) +{ + u32 info = readl(SYSMMU_REG(data, fault_info)); + + fault->addr = readl(SYSMMU_REG(data, fault_va)); + fault->name = sysmmu_v7_fault_names[itype % 4]; + fault->type = (info & BIT(20)) ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ; + + return 0; +} + /* SysMMU v1..v3 */ static const struct sysmmu_variant sysmmu_v1_variant = { .flush_all = 0x0c, @@ -348,7 +370,7 @@ static const struct sysmmu_variant sysmmu_v1_variant = { .get_fault_info = exynos_sysmmu_v1_get_fault_info, }; -/* SysMMU v5 and v7 (non-VM capable) */ +/* SysMMU v5 */ static const struct sysmmu_variant sysmmu_v5_variant = { .pt_base = 0x0c, .flush_all = 0x10, @@ -362,7 +384,23 @@ static const struct sysmmu_variant sysmmu_v5_variant = { .get_fault_info = exynos_sysmmu_v5_get_fault_info, }; -/* SysMMU v7: VM capable register set */ +/* SysMMU v7: non-VM capable register layout */ +static const struct sysmmu_variant sysmmu_v7_variant = { + .pt_base = 0x0c, + .flush_all = 0x10, + .flush_entry = 0x14, + .flush_range = 0x18, + .flush_start = 0x20, + .flush_end = 0x24, + .int_status = 0x60, + .int_clear = 0x64, + .fault_va = 0x70, + .fault_info = 0x78, + + .get_fault_info = exynos_sysmmu_v7_get_fault_info, +}; + +/* SysMMU v7: VM capable register layout */ static const struct sysmmu_variant sysmmu_v7_vm_variant = { .pt_base = 0x800c, .flush_all = 0x8010, @@ -372,8 +410,10 @@ static const struct sysmmu_variant sysmmu_v7_vm_variant = { .flush_end = 0x8024, .int_status = 0x60, .int_clear = 0x64, + .fault_va = 0x1000, + .fault_info = 0x1004, - .get_fault_info = exynos_sysmmu_v5_get_fault_info, + .get_fault_info = exynos_sysmmu_v7_get_fault_info, }; static struct exynos_iommu_domain *to_exynos_domain(struct iommu_domain *dom) @@ -496,7 +536,7 @@ static void __sysmmu_get_version(struct sysmmu_drvdata *data) if (data->has_vcr) data->variant = &sysmmu_v7_vm_variant; else - data->variant = &sysmmu_v5_variant; + data->variant = &sysmmu_v7_variant; } __sysmmu_disable_clocks(data); -- cgit v1.2.3 From 189d496b48b1731f21f19ff3b7109ec724f5dcb2 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Mon, 23 Jan 2023 10:31:01 +0100 Subject: iommu/exynos: Add missing set_platform_dma_ops callback Add set_platform_dma_ops() required for proper driver operation on ARM 32bit arch after recent changes in the IOMMU framework (detach ops removal). Fixes: c1fe9119ee70 ("iommu: Add set_platform_dma_ops callbacks") Signed-off-by: Marek Szyprowski Link: https://lore.kernel.org/r/20230123093102.12392-1-m.szyprowski@samsung.com Signed-off-by: Joerg Roedel --- drivers/iommu/exynos-iommu.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'drivers/iommu/exynos-iommu.c') diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 29ec713e8a21..7eaa58ced4ae 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -1346,8 +1346,10 @@ static void exynos_iommu_release_device(struct device *dev) struct iommu_group *group = iommu_group_get(dev); if (group) { +#ifndef CONFIG_ARM WARN_ON(owner->domain != iommu_group_default_domain(group)); +#endif exynos_iommu_detach_device(owner->domain, dev); iommu_group_put(group); } @@ -1398,6 +1400,9 @@ static int exynos_iommu_of_xlate(struct device *dev, static const struct iommu_ops exynos_iommu_ops = { .domain_alloc = exynos_iommu_domain_alloc, .device_group = generic_device_group, +#ifdef CONFIG_ARM + .set_platform_dma_ops = exynos_iommu_release_device, +#endif .probe_device = exynos_iommu_probe_device, .release_device = exynos_iommu_release_device, .pgsize_bitmap = SECT_SIZE | LPAGE_SIZE | SPAGE_SIZE, -- cgit v1.2.3