From 1a7180ff81aadd21556a5b5df7d0a3280f7d744f Mon Sep 17 00:00:00 2001 From: Bjorn Andersson Date: Wed, 20 Jan 2021 17:40:05 -0800 Subject: iommu/arm-smmu-qcom: Add Qualcomm SC8180X impl The primary SMMU found in Qualcomm SC8180X platform needs to use the Qualcomm implementation, so add a specific compatible for this. Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20210121014005.1612382-2-bjorn.andersson@linaro.org Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c index bcda17012aee..82c7edc6e025 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c @@ -166,6 +166,7 @@ static const struct of_device_id qcom_smmu_client_of_match[] __maybe_unused = { { .compatible = "qcom,mdss" }, { .compatible = "qcom,sc7180-mdss" }, { .compatible = "qcom,sc7180-mss-pil" }, + { .compatible = "qcom,sc8180x-mdss" }, { .compatible = "qcom,sdm845-mdss" }, { .compatible = "qcom,sdm845-mss-pil" }, { } @@ -327,6 +328,7 @@ static struct arm_smmu_device *qcom_smmu_create(struct arm_smmu_device *smmu, static const struct of_device_id __maybe_unused qcom_smmu_impl_of_match[] = { { .compatible = "qcom,msm8998-smmu-v2" }, { .compatible = "qcom,sc7180-smmu-500" }, + { .compatible = "qcom,sc8180x-smmu-500" }, { .compatible = "qcom,sdm630-smmu-v2" }, { .compatible = "qcom,sdm845-smmu-500" }, { .compatible = "qcom,sm8150-smmu-500" }, -- cgit v1.2.3 From d8498b1e4ecc75d023c59acb30e15e3d8d8f07ea Mon Sep 17 00:00:00 2001 From: Vinod Koul Date: Fri, 15 Jan 2021 14:33:22 +0530 Subject: iommu: arm-smmu-impl: Add SM8350 qcom iommu implementation Add SM8350 qcom iommu implementation to the table of qcom_smmu_impl_of_match table which brings in iommu support for SM8350 SoC Signed-off-by: Vinod Koul Link: https://lore.kernel.org/r/20210115090322.2287538-2-vkoul@kernel.org Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c index 82c7edc6e025..3b6fdbb6c916 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c @@ -333,6 +333,7 @@ static const struct of_device_id __maybe_unused qcom_smmu_impl_of_match[] = { { .compatible = "qcom,sdm845-smmu-500" }, { .compatible = "qcom,sm8150-smmu-500" }, { .compatible = "qcom,sm8250-smmu-500" }, + { .compatible = "qcom,sm8350-smmu-500" }, { } }; -- cgit v1.2.3 From 86d2d9214880c1879cb0ff466cce2fcd9c853e44 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Thu, 21 Jan 2021 14:09:42 +0000 Subject: iommu/arm-smmu-v3: Remove the page 1 fixup Since we now keep track of page 1 via a separate pointer that already encapsulates aliasing to page 0 as necessary, we can remove the clunky fixup routine and simply use the relevant bases directly. The current architecture spec (IHI0070D.a) defines SMMU_{EVENTQ,PRIQ}_{PROD,CONS} as offsets relative to page 1, so the cleanup represents a little bit of convergence as well as just lines of code saved. Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/08d9bda570bb5681f11a2f250a31be9ef763b8c5.1611238182.git.robin.murphy@arm.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 42 +++++++++++------------------ drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 8 +++--- 2 files changed, 20 insertions(+), 30 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 8ca7415d785d..bca458c00e48 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -88,15 +88,6 @@ static struct arm_smmu_option_prop arm_smmu_options[] = { { 0, NULL}, }; -static inline void __iomem *arm_smmu_page1_fixup(unsigned long offset, - struct arm_smmu_device *smmu) -{ - if (offset > SZ_64K) - return smmu->page1 + offset - SZ_64K; - - return smmu->base + offset; -} - static void parse_driver_options(struct arm_smmu_device *smmu) { int i = 0; @@ -2611,6 +2602,7 @@ static struct iommu_ops arm_smmu_ops = { /* Probing and initialisation functions */ static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu, struct arm_smmu_queue *q, + void __iomem *page, unsigned long prod_off, unsigned long cons_off, size_t dwords, const char *name) @@ -2639,8 +2631,8 @@ static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu, 1 << q->llq.max_n_shift, name); } - q->prod_reg = arm_smmu_page1_fixup(prod_off, smmu); - q->cons_reg = arm_smmu_page1_fixup(cons_off, smmu); + q->prod_reg = page + prod_off; + q->cons_reg = page + cons_off; q->ent_dwords = dwords; q->q_base = Q_BASE_RWA; @@ -2684,9 +2676,9 @@ static int arm_smmu_init_queues(struct arm_smmu_device *smmu) int ret; /* cmdq */ - ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, ARM_SMMU_CMDQ_PROD, - ARM_SMMU_CMDQ_CONS, CMDQ_ENT_DWORDS, - "cmdq"); + ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, smmu->base, + ARM_SMMU_CMDQ_PROD, ARM_SMMU_CMDQ_CONS, + CMDQ_ENT_DWORDS, "cmdq"); if (ret) return ret; @@ -2695,9 +2687,9 @@ static int arm_smmu_init_queues(struct arm_smmu_device *smmu) return ret; /* evtq */ - ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, ARM_SMMU_EVTQ_PROD, - ARM_SMMU_EVTQ_CONS, EVTQ_ENT_DWORDS, - "evtq"); + ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, smmu->page1, + ARM_SMMU_EVTQ_PROD, ARM_SMMU_EVTQ_CONS, + EVTQ_ENT_DWORDS, "evtq"); if (ret) return ret; @@ -2705,9 +2697,9 @@ static int arm_smmu_init_queues(struct arm_smmu_device *smmu) if (!(smmu->features & ARM_SMMU_FEAT_PRI)) return 0; - return arm_smmu_init_one_queue(smmu, &smmu->priq.q, ARM_SMMU_PRIQ_PROD, - ARM_SMMU_PRIQ_CONS, PRIQ_ENT_DWORDS, - "priq"); + return arm_smmu_init_one_queue(smmu, &smmu->priq.q, smmu->page1, + ARM_SMMU_PRIQ_PROD, ARM_SMMU_PRIQ_CONS, + PRIQ_ENT_DWORDS, "priq"); } static int arm_smmu_init_l1_strtab(struct arm_smmu_device *smmu) @@ -3099,10 +3091,8 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass) /* Event queue */ writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE); - writel_relaxed(smmu->evtq.q.llq.prod, - arm_smmu_page1_fixup(ARM_SMMU_EVTQ_PROD, smmu)); - writel_relaxed(smmu->evtq.q.llq.cons, - arm_smmu_page1_fixup(ARM_SMMU_EVTQ_CONS, smmu)); + writel_relaxed(smmu->evtq.q.llq.prod, smmu->page1 + ARM_SMMU_EVTQ_PROD); + writel_relaxed(smmu->evtq.q.llq.cons, smmu->page1 + ARM_SMMU_EVTQ_CONS); enables |= CR0_EVTQEN; ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, @@ -3117,9 +3107,9 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass) writeq_relaxed(smmu->priq.q.q_base, smmu->base + ARM_SMMU_PRIQ_BASE); writel_relaxed(smmu->priq.q.llq.prod, - arm_smmu_page1_fixup(ARM_SMMU_PRIQ_PROD, smmu)); + smmu->page1 + ARM_SMMU_PRIQ_PROD); writel_relaxed(smmu->priq.q.llq.cons, - arm_smmu_page1_fixup(ARM_SMMU_PRIQ_CONS, smmu)); + smmu->page1 + ARM_SMMU_PRIQ_CONS); enables |= CR0_PRIQEN; ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 96c2e9565e00..da525f46dab4 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -139,15 +139,15 @@ #define ARM_SMMU_CMDQ_CONS 0x9c #define ARM_SMMU_EVTQ_BASE 0xa0 -#define ARM_SMMU_EVTQ_PROD 0x100a8 -#define ARM_SMMU_EVTQ_CONS 0x100ac +#define ARM_SMMU_EVTQ_PROD 0xa8 +#define ARM_SMMU_EVTQ_CONS 0xac #define ARM_SMMU_EVTQ_IRQ_CFG0 0xb0 #define ARM_SMMU_EVTQ_IRQ_CFG1 0xb8 #define ARM_SMMU_EVTQ_IRQ_CFG2 0xbc #define ARM_SMMU_PRIQ_BASE 0xc0 -#define ARM_SMMU_PRIQ_PROD 0x100c8 -#define ARM_SMMU_PRIQ_CONS 0x100cc +#define ARM_SMMU_PRIQ_PROD 0xc8 +#define ARM_SMMU_PRIQ_CONS 0xcc #define ARM_SMMU_PRIQ_IRQ_CFG0 0xd0 #define ARM_SMMU_PRIQ_IRQ_CFG1 0xd8 #define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc -- cgit v1.2.3 From 932bc8c7d742c30ce61f2add6cd1340795b0051c Mon Sep 17 00:00:00 2001 From: Zhen Lei Date: Fri, 22 Jan 2021 21:14:48 +0800 Subject: iommu/arm-smmu-v3: Use DEFINE_RES_MEM() to simplify code No functional change. Signed-off-by: Zhen Lei Link: https://lore.kernel.org/r/20210122131448.1167-1-thunder.leizhen@huawei.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index bca458c00e48..f04c55a7503c 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -3479,11 +3479,7 @@ err_reset_pci_ops: __maybe_unused; static void __iomem *arm_smmu_ioremap(struct device *dev, resource_size_t start, resource_size_t size) { - struct resource res = { - .flags = IORESOURCE_MEM, - .start = start, - .end = start + size - 1, - }; + struct resource res = DEFINE_RES_MEM(start, size); return devm_ioremap_resource(dev, &res); } -- cgit v1.2.3 From eba8d2f8f80315094b61eaf2bc6cd481741d6d93 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Fri, 22 Jan 2021 16:10:54 +0100 Subject: iommu/arm-smmu-v3: Split arm_smmu_tlb_inv_range() Extract some of the cmd initialization and the ATC invalidation from arm_smmu_tlb_inv_range(), to allow an MMU notifier to invalidate a VA range by ASID. Reviewed-by: Jonathan Cameron Signed-off-by: Jean-Philippe Brucker Link: https://lore.kernel.org/r/20210122151054.2833521-2-jean-philippe@linaro.org Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 63 ++++++++++++++++------------- 1 file changed, 36 insertions(+), 27 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index f04c55a7503c..86cbac77c941 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1658,40 +1658,28 @@ static void arm_smmu_tlb_inv_context(void *cookie) arm_smmu_atc_inv_domain(smmu_domain, 0, 0, 0); } -static void arm_smmu_tlb_inv_range(unsigned long iova, size_t size, - size_t granule, bool leaf, - struct arm_smmu_domain *smmu_domain) +static void __arm_smmu_tlb_inv_range(struct arm_smmu_cmdq_ent *cmd, + unsigned long iova, size_t size, + size_t granule, + struct arm_smmu_domain *smmu_domain) { struct arm_smmu_device *smmu = smmu_domain->smmu; - unsigned long start = iova, end = iova + size, num_pages = 0, tg = 0; + unsigned long end = iova + size, num_pages = 0, tg = 0; size_t inv_range = granule; struct arm_smmu_cmdq_batch cmds = {}; - struct arm_smmu_cmdq_ent cmd = { - .tlbi = { - .leaf = leaf, - }, - }; if (!size) return; - if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { - cmd.opcode = CMDQ_OP_TLBI_NH_VA; - cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid; - } else { - cmd.opcode = CMDQ_OP_TLBI_S2_IPA; - cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid; - } - if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) { /* Get the leaf page size */ tg = __ffs(smmu_domain->domain.pgsize_bitmap); /* Convert page size of 12,14,16 (log2) to 1,2,3 */ - cmd.tlbi.tg = (tg - 10) / 2; + cmd->tlbi.tg = (tg - 10) / 2; /* Determine what level the granule is at */ - cmd.tlbi.ttl = 4 - ((ilog2(granule) - 3) / (tg - 3)); + cmd->tlbi.ttl = 4 - ((ilog2(granule) - 3) / (tg - 3)); num_pages = size >> tg; } @@ -1709,11 +1697,11 @@ static void arm_smmu_tlb_inv_range(unsigned long iova, size_t size, /* Determine the power of 2 multiple number of pages */ scale = __ffs(num_pages); - cmd.tlbi.scale = scale; + cmd->tlbi.scale = scale; /* Determine how many chunks of 2^scale size we have */ num = (num_pages >> scale) & CMDQ_TLBI_RANGE_NUM_MAX; - cmd.tlbi.num = num - 1; + cmd->tlbi.num = num - 1; /* range is num * 2^scale * pgsize */ inv_range = num << (scale + tg); @@ -1722,17 +1710,37 @@ static void arm_smmu_tlb_inv_range(unsigned long iova, size_t size, num_pages -= num << scale; } - cmd.tlbi.addr = iova; - arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd); + cmd->tlbi.addr = iova; + arm_smmu_cmdq_batch_add(smmu, &cmds, cmd); iova += inv_range; } arm_smmu_cmdq_batch_submit(smmu, &cmds); +} + +static void arm_smmu_tlb_inv_range_domain(unsigned long iova, size_t size, + size_t granule, bool leaf, + struct arm_smmu_domain *smmu_domain) +{ + struct arm_smmu_cmdq_ent cmd = { + .tlbi = { + .leaf = leaf, + }, + }; + + if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { + cmd.opcode = CMDQ_OP_TLBI_NH_VA; + cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid; + } else { + cmd.opcode = CMDQ_OP_TLBI_S2_IPA; + cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid; + } + __arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain); /* * Unfortunately, this can't be leaf-only since we may have * zapped an entire table. */ - arm_smmu_atc_inv_domain(smmu_domain, 0, start, size); + arm_smmu_atc_inv_domain(smmu_domain, 0, iova, size); } static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather, @@ -1748,7 +1756,7 @@ static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather, static void arm_smmu_tlb_inv_walk(unsigned long iova, size_t size, size_t granule, void *cookie) { - arm_smmu_tlb_inv_range(iova, size, granule, false, cookie); + arm_smmu_tlb_inv_range_domain(iova, size, granule, false, cookie); } static const struct iommu_flush_ops arm_smmu_flush_ops = { @@ -2271,8 +2279,9 @@ static void arm_smmu_iotlb_sync(struct iommu_domain *domain, { struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - arm_smmu_tlb_inv_range(gather->start, gather->end - gather->start, - gather->pgsize, true, smmu_domain); + arm_smmu_tlb_inv_range_domain(gather->start, + gather->end - gather->start, + gather->pgsize, true, smmu_domain); } static phys_addr_t -- cgit v1.2.3 From 51d113c3be09ff5b916576b1109daf413549cacc Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Fri, 22 Jan 2021 16:10:55 +0100 Subject: iommu/arm-smmu-v3: Make BTM optional for SVA When BTM isn't supported by the SMMU, send invalidations on the command queue. Reviewed-by: Jonathan Cameron Signed-off-by: Jean-Philippe Brucker Link: https://lore.kernel.org/r/20210122151054.2833521-3-jean-philippe@linaro.org Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 10 +++++++--- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 15 +++++++++++++++ drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 3 +++ 3 files changed, 25 insertions(+), 3 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index e13b092e6004..bb251cab61f3 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -182,9 +182,13 @@ static void arm_smmu_mm_invalidate_range(struct mmu_notifier *mn, unsigned long start, unsigned long end) { struct arm_smmu_mmu_notifier *smmu_mn = mn_to_smmu(mn); + struct arm_smmu_domain *smmu_domain = smmu_mn->domain; + size_t size = end - start + 1; - arm_smmu_atc_inv_domain(smmu_mn->domain, mm->pasid, start, - end - start + 1); + if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_BTM)) + arm_smmu_tlb_inv_range_asid(start, size, smmu_mn->cd->asid, + PAGE_SIZE, false, smmu_domain); + arm_smmu_atc_inv_domain(smmu_domain, mm->pasid, start, size); } static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) @@ -391,7 +395,7 @@ bool arm_smmu_sva_supported(struct arm_smmu_device *smmu) unsigned long reg, fld; unsigned long oas; unsigned long asid_bits; - u32 feat_mask = ARM_SMMU_FEAT_BTM | ARM_SMMU_FEAT_COHERENCY; + u32 feat_mask = ARM_SMMU_FEAT_COHERENCY; if (vabits_actual == 52) feat_mask |= ARM_SMMU_FEAT_VAX; diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 86cbac77c941..111467888e88 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1743,6 +1743,21 @@ static void arm_smmu_tlb_inv_range_domain(unsigned long iova, size_t size, arm_smmu_atc_inv_domain(smmu_domain, 0, iova, size); } +void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid, + size_t granule, bool leaf, + struct arm_smmu_domain *smmu_domain) +{ + struct arm_smmu_cmdq_ent cmd = { + .opcode = CMDQ_OP_TLBI_NH_VA, + .tlbi = { + .asid = asid, + .leaf = leaf, + }, + }; + + __arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain); +} + static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather, unsigned long iova, size_t granule, void *cookie) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index da525f46dab4..56bc0c3d4f4a 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -694,6 +694,9 @@ extern struct arm_smmu_ctx_desc quiet_cd; int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain, int ssid, struct arm_smmu_ctx_desc *cd); void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid); +void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid, + size_t granule, bool leaf, + struct arm_smmu_domain *smmu_domain); bool arm_smmu_free_asid(struct arm_smmu_ctx_desc *cd); int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid, unsigned long iova, size_t size); -- cgit v1.2.3 From 9111aebf770d6a6c9fcfd6301da4a0025f2d610f Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Fri, 22 Jan 2021 16:10:56 +0100 Subject: iommu/arm-smmu-v3: Add support for VHE ARMv8.1 extensions added Virtualization Host Extensions (VHE), which allow to run a host kernel at EL2. When using normal DMA, Device and CPU address spaces are dissociated, and do not need to implement the same capabilities, so VHE hasn't been used in the SMMU until now. With shared address spaces however, ASIDs are shared between MMU and SMMU, and broadcast TLB invalidations issued by a CPU are taken into account by the SMMU. TLB entries on both sides need to have identical exception level in order to be cleared with a single invalidation. When the CPU is using VHE, enable VHE in the SMMU for all STEs. Normal DMA mappings will need to use TLBI_EL2 commands instead of TLBI_NH, but shouldn't be otherwise affected by this change. Acked-by: Will Deacon Reviewed-by: Jonathan Cameron Signed-off-by: Jean-Philippe Brucker Link: https://lore.kernel.org/r/20210122151054.2833521-4-jean-philippe@linaro.org Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 32 ++++++++++++++++++++++------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 3 +++ 2 files changed, 28 insertions(+), 7 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 111467888e88..baebaac34a83 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -263,9 +263,11 @@ static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent) cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31); break; case CMDQ_OP_TLBI_NH_VA: + cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid); + fallthrough; + case CMDQ_OP_TLBI_EL2_VA: cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num); cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale); - cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid); cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid); cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf); cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl); @@ -287,6 +289,9 @@ static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent) case CMDQ_OP_TLBI_S12_VMALL: cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid); break; + case CMDQ_OP_TLBI_EL2_ASID: + cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid); + break; case CMDQ_OP_ATC_INV: cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid); cmd[0] |= FIELD_PREP(CMDQ_ATC_0_GLOBAL, ent->atc.global); @@ -877,7 +882,8 @@ static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu, void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid) { struct arm_smmu_cmdq_ent cmd = { - .opcode = CMDQ_OP_TLBI_NH_ASID, + .opcode = smmu->features & ARM_SMMU_FEAT_E2H ? + CMDQ_OP_TLBI_EL2_ASID : CMDQ_OP_TLBI_NH_ASID, .tlbi.asid = asid, }; @@ -1260,13 +1266,16 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid, } if (s1_cfg) { + u64 strw = smmu->features & ARM_SMMU_FEAT_E2H ? + STRTAB_STE_1_STRW_EL2 : STRTAB_STE_1_STRW_NSEL1; + BUG_ON(ste_live); dst[1] = cpu_to_le64( FIELD_PREP(STRTAB_STE_1_S1DSS, STRTAB_STE_1_S1DSS_SSID0) | FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) | FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) | FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) | - FIELD_PREP(STRTAB_STE_1_STRW, STRTAB_STE_1_STRW_NSEL1)); + FIELD_PREP(STRTAB_STE_1_STRW, strw)); if (smmu->features & ARM_SMMU_FEAT_STALLS && !(smmu->features & ARM_SMMU_FEAT_STALL_FORCE)) @@ -1728,7 +1737,8 @@ static void arm_smmu_tlb_inv_range_domain(unsigned long iova, size_t size, }; if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { - cmd.opcode = CMDQ_OP_TLBI_NH_VA; + cmd.opcode = smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ? + CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA; cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid; } else { cmd.opcode = CMDQ_OP_TLBI_S2_IPA; @@ -1748,7 +1758,8 @@ void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid, struct arm_smmu_domain *smmu_domain) { struct arm_smmu_cmdq_ent cmd = { - .opcode = CMDQ_OP_TLBI_NH_VA, + .opcode = smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ? + CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA, .tlbi = { .asid = asid, .leaf = leaf, @@ -3076,7 +3087,11 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass) writel_relaxed(reg, smmu->base + ARM_SMMU_CR1); /* CR2 (random crap) */ - reg = CR2_PTM | CR2_RECINVSID | CR2_E2H; + reg = CR2_PTM | CR2_RECINVSID; + + if (smmu->features & ARM_SMMU_FEAT_E2H) + reg |= CR2_E2H; + writel_relaxed(reg, smmu->base + ARM_SMMU_CR2); /* Stream table */ @@ -3235,8 +3250,11 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) smmu->options |= ARM_SMMU_OPT_MSIPOLL; } - if (reg & IDR0_HYP) + if (reg & IDR0_HYP) { smmu->features |= ARM_SMMU_FEAT_HYP; + if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN)) + smmu->features |= ARM_SMMU_FEAT_E2H; + } /* * The coherency feature as set by FW is used in preference to the ID diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 56bc0c3d4f4a..f985817c967a 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -430,6 +430,8 @@ struct arm_smmu_cmdq_ent { #define CMDQ_OP_TLBI_NH_ASID 0x11 #define CMDQ_OP_TLBI_NH_VA 0x12 #define CMDQ_OP_TLBI_EL2_ALL 0x20 + #define CMDQ_OP_TLBI_EL2_ASID 0x21 + #define CMDQ_OP_TLBI_EL2_VA 0x22 #define CMDQ_OP_TLBI_S12_VMALL 0x28 #define CMDQ_OP_TLBI_S2_IPA 0x2a #define CMDQ_OP_TLBI_NSNH_ALL 0x30 @@ -604,6 +606,7 @@ struct arm_smmu_device { #define ARM_SMMU_FEAT_RANGE_INV (1 << 15) #define ARM_SMMU_FEAT_BTM (1 << 16) #define ARM_SMMU_FEAT_SVA (1 << 17) +#define ARM_SMMU_FEAT_E2H (1 << 18) u32 features; #define ARM_SMMU_OPT_SKIP_PREFETCH (1 << 0) -- cgit v1.2.3 From dead723e6f049e9fb6b05e5b93456982798ea961 Mon Sep 17 00:00:00 2001 From: "Isaac J. Manjarres" Date: Mon, 25 Jan 2021 13:52:25 -0800 Subject: iommu/arm-smmu-qcom: Fix mask extraction for bootloader programmed SMRs When extracting the mask for a SMR that was programmed by the bootloader, the SMR's valid bit is also extracted and is treated as part of the mask, which is not correct. Consider the scenario where an SMMU master whose context is determined by a bootloader programmed SMR is removed (omitting parts of device/driver core): ->iommu_release_device() -> arm_smmu_release_device() -> arm_smmu_master_free_smes() -> arm_smmu_free_sme() /* Assume that the SME is now free */ -> arm_smmu_write_sme() -> arm_smmu_write_smr() /* Construct SMR value using mask and SID */ Since the valid bit was considered as part of the mask, the SMR will be programmed as valid. Fix the SMR mask extraction step for bootloader programmed SMRs by masking out the valid bit when we know that we're already working with a valid SMR. Fixes: 07a7f2caaa5a ("iommu/arm-smmu-qcom: Read back stream mappings") Signed-off-by: Isaac J. Manjarres Cc: stable@vger.kernel.org Reviewed-by: Robin Murphy Link: https://lore.kernel.org/r/1611611545-19055-1-git-send-email-isaacm@codeaurora.org Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c index 3b6fdbb6c916..98b3a1c2a181 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c @@ -207,6 +207,8 @@ static int qcom_smmu_cfg_probe(struct arm_smmu_device *smmu) smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i)); if (FIELD_GET(ARM_SMMU_SMR_VALID, smr)) { + /* Ignore valid bit for SMR mask extraction. */ + smr &= ~ARM_SMMU_SMR_VALID; smmu->smrs[i].id = FIELD_GET(ARM_SMMU_SMR_ID, smr); smmu->smrs[i].mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr); smmu->smrs[i].valid = true; -- cgit v1.2.3 From 9cc0aaeb96e7f894d4735f069174948c1516fea7 Mon Sep 17 00:00:00 2001 From: John Garry Date: Wed, 6 Jan 2021 21:35:06 +0800 Subject: iova: Make has_iova_flush_queue() private Function has_iova_flush_queue() has no users outside iova.c, so make it private. Signed-off-by: John Garry Acked-by: Will Deacon Link: https://lore.kernel.org/r/1609940111-28563-2-git-send-email-john.garry@huawei.com Signed-off-by: Joerg Roedel --- drivers/iommu/iova.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index d20b8b333d30..5f2caa95c1f0 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -55,7 +55,7 @@ init_iova_domain(struct iova_domain *iovad, unsigned long granule, } EXPORT_SYMBOL_GPL(init_iova_domain); -bool has_iova_flush_queue(struct iova_domain *iovad) +static bool has_iova_flush_queue(struct iova_domain *iovad) { return !!iovad->fq; } -- cgit v1.2.3 From 622106190175dbac2b0b0ee7d4275c474e5fe051 Mon Sep 17 00:00:00 2001 From: John Garry Date: Wed, 6 Jan 2021 21:35:07 +0800 Subject: iova: Delete copy_reserved_iova() Since commit c588072bba6b ("iommu/vt-d: Convert intel iommu driver to the iommu ops"), function copy_reserved_iova() is not referenced, so delete it. Signed-off-by: John Garry Acked-by: Will Deacon Link: https://lore.kernel.org/r/1609940111-28563-3-git-send-email-john.garry@huawei.com Signed-off-by: Joerg Roedel --- drivers/iommu/iova.c | 30 ------------------------------ 1 file changed, 30 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 5f2caa95c1f0..9b114cd886d5 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -710,36 +710,6 @@ finish: } EXPORT_SYMBOL_GPL(reserve_iova); -/** - * copy_reserved_iova - copies the reserved between domains - * @from: - source domain from where to copy - * @to: - destination domin where to copy - * This function copies reserved iova's from one domain to - * other. - */ -void -copy_reserved_iova(struct iova_domain *from, struct iova_domain *to) -{ - unsigned long flags; - struct rb_node *node; - - spin_lock_irqsave(&from->iova_rbtree_lock, flags); - for (node = rb_first(&from->rbroot); node; node = rb_next(node)) { - struct iova *iova = rb_entry(node, struct iova, node); - struct iova *new_iova; - - if (iova->pfn_lo == IOVA_ANCHOR) - continue; - - new_iova = reserve_iova(to, iova->pfn_lo, iova->pfn_hi); - if (!new_iova) - pr_err("Reserve iova range %lx@%lx failed\n", - iova->pfn_lo, iova->pfn_lo); - } - spin_unlock_irqrestore(&from->iova_rbtree_lock, flags); -} -EXPORT_SYMBOL_GPL(copy_reserved_iova); - /* * Magazine caches for IOVA ranges. For an introduction to magazines, * see the USENIX 2001 paper "Magazines and Vmem: Extending the Slab -- cgit v1.2.3 From 2cf7dbff0a955f546a1d2c132b94f9d5b837b714 Mon Sep 17 00:00:00 2001 From: John Garry Date: Wed, 6 Jan 2021 21:35:08 +0800 Subject: iova: Stop exporting some more functions The following functions are not referenced outside dma-iommu.c (and iova.c), which can only be built-in: - init_iova_flush_queue() - free_iova_fast() - queue_iova() - alloc_iova_fast() So stop exporting them. Signed-off-by: John Garry Acked-by: Will Deacon Link: https://lore.kernel.org/r/1609940111-28563-4-git-send-email-john.garry@huawei.com Signed-off-by: Joerg Roedel --- drivers/iommu/iova.c | 3 --- 1 file changed, 3 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 9b114cd886d5..e6e2fa85271c 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -112,7 +112,6 @@ int init_iova_flush_queue(struct iova_domain *iovad, return 0; } -EXPORT_SYMBOL_GPL(init_iova_flush_queue); static struct rb_node * __get_cached_rbnode(struct iova_domain *iovad, unsigned long limit_pfn) @@ -451,7 +450,6 @@ retry: return new_iova->pfn_lo; } -EXPORT_SYMBOL_GPL(alloc_iova_fast); /** * free_iova_fast - free iova pfn range into rcache @@ -598,7 +596,6 @@ void queue_iova(struct iova_domain *iovad, mod_timer(&iovad->fq_timer, jiffies + msecs_to_jiffies(IOVA_FQ_TIMEOUT)); } -EXPORT_SYMBOL_GPL(queue_iova); /** * put_iova_domain - destroys the iova domain -- cgit v1.2.3 From b91910a83d041d87115068c773438575d8279534 Mon Sep 17 00:00:00 2001 From: John Garry Date: Wed, 6 Jan 2021 21:35:09 +0800 Subject: iommu: Stop exporting iommu_map_sg_atomic() Function iommu_map_sg_atomic() is only referenced in dma-iommu.c, which can only be built-in, so stop exporting. Signed-off-by: John Garry Acked-by: Will Deacon Link: https://lore.kernel.org/r/1609940111-28563-5-git-send-email-john.garry@huawei.com Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 1 - 1 file changed, 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index ffeebda8d6de..4f4edb087ce6 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2586,7 +2586,6 @@ size_t iommu_map_sg_atomic(struct iommu_domain *domain, unsigned long iova, { return __iommu_map_sg(domain, iova, sg, nents, prot, GFP_ATOMIC); } -EXPORT_SYMBOL_GPL(iommu_map_sg_atomic); int iommu_domain_window_enable(struct iommu_domain *domain, u32 wnd_nr, phys_addr_t paddr, u64 size, int prot) -- cgit v1.2.3 From ab0a7119ba67be9e377b195d2b9baa9fb8b3b53e Mon Sep 17 00:00:00 2001 From: John Garry Date: Wed, 6 Jan 2021 21:35:10 +0800 Subject: iommu: Delete iommu_domain_window_disable() Function iommu_domain_window_disable() is not referenced in the tree, so delete it. Signed-off-by: John Garry Acked-by: Will Deacon Link: https://lore.kernel.org/r/1609940111-28563-6-git-send-email-john.garry@huawei.com Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 9 --------- 1 file changed, 9 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 4f4edb087ce6..20201ef97b8f 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2598,15 +2598,6 @@ int iommu_domain_window_enable(struct iommu_domain *domain, u32 wnd_nr, } EXPORT_SYMBOL_GPL(iommu_domain_window_enable); -void iommu_domain_window_disable(struct iommu_domain *domain, u32 wnd_nr) -{ - if (unlikely(domain->ops->domain_window_disable == NULL)) - return; - - return domain->ops->domain_window_disable(domain, wnd_nr); -} -EXPORT_SYMBOL_GPL(iommu_domain_window_disable); - /** * report_iommu_fault() - report about an IOMMU fault to the IOMMU framework * @domain: the iommu domain where the fault has happened -- cgit v1.2.3 From 262948f8ba573dc9c61650df8f23eaea7d43bc61 Mon Sep 17 00:00:00 2001 From: John Garry Date: Wed, 6 Jan 2021 21:35:11 +0800 Subject: iommu: Delete iommu_dev_has_feature() Function iommu_dev_has_feature() has never been referenced in the tree, and there does not appear to be anything coming soon to use it, so delete it. Signed-off-by: John Garry Acked-by: Will Deacon Link: https://lore.kernel.org/r/1609940111-28563-7-git-send-email-john.garry@huawei.com Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 11 ----------- 1 file changed, 11 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 20201ef97b8f..91f7871f5a37 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2853,17 +2853,6 @@ EXPORT_SYMBOL_GPL(iommu_fwspec_add_ids); /* * Per device IOMMU features. */ -bool iommu_dev_has_feature(struct device *dev, enum iommu_dev_features feat) -{ - const struct iommu_ops *ops = dev->bus->iommu_ops; - - if (ops && ops->dev_has_feat) - return ops->dev_has_feat(dev, feat); - - return false; -} -EXPORT_SYMBOL_GPL(iommu_dev_has_feature); - int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features feat) { const struct iommu_ops *ops = dev->bus->iommu_ops; -- cgit v1.2.3 From f8993dc666f44e802c705d1053c4491981413f9e Mon Sep 17 00:00:00 2001 From: Adrian Huang Date: Thu, 10 Dec 2020 10:13:30 +0800 Subject: iommu/amd: Remove unnecessary assignment From: Adrian Huang The values of local variables are assigned after local variables are declared, so no need to assign the initial value during the variable declaration. And, no need to assign NULL for the local variable 'ivrs_base' after invoking acpi_put_table(). Signed-off-by: Adrian Huang Acked-by: Will Deacon Link: https://lore.kernel.org/r/20201210021330.2022-1-adrianhuang0701@gmail.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 6a1f7048dacc..f999d3424876 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -1911,7 +1911,7 @@ static void print_iommu_info(void) static int __init amd_iommu_init_pci(void) { struct amd_iommu *iommu; - int ret = 0; + int ret; for_each_iommu(iommu) { ret = iommu_init_pci(iommu); @@ -2637,8 +2637,8 @@ static void __init free_dma_resources(void) static int __init early_amd_iommu_init(void) { struct acpi_table_header *ivrs_base; + int i, remap_cache_sz, ret; acpi_status status; - int i, remap_cache_sz, ret = 0; u32 pci_id; if (!amd_iommu_detected) @@ -2780,7 +2780,6 @@ static int __init early_amd_iommu_init(void) out: /* Don't leak any ACPI memory */ acpi_put_table(ivrs_base); - ivrs_base = NULL; return ret; } -- cgit v1.2.3 From 106650f1e94527f15c0e99285b88ced84b47ab52 Mon Sep 17 00:00:00 2001 From: Zheng Yongjun Date: Mon, 28 Dec 2020 21:51:12 +0800 Subject: iommu/amd: Use DEFINE_SPINLOCK() for spinlock Spinlock can be initialized automatically with DEFINE_SPINLOCK() rather than explicitly calling spin_lock_init(). Signed-off-by: Zheng Yongjun Link: https://lore.kernel.org/r/20201228135112.28621-1-zhengyongjun3@huawei.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu_v2.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/iommu_v2.c b/drivers/iommu/amd/iommu_v2.c index 5ecc0bc608ec..f8d4ad421e07 100644 --- a/drivers/iommu/amd/iommu_v2.c +++ b/drivers/iommu/amd/iommu_v2.c @@ -77,7 +77,7 @@ struct fault { }; static LIST_HEAD(state_list); -static spinlock_t state_lock; +static DEFINE_SPINLOCK(state_lock); static struct workqueue_struct *iommu_wq; @@ -938,8 +938,6 @@ static int __init amd_iommu_v2_init(void) return 0; } - spin_lock_init(&state_lock); - ret = -ENOMEM; iommu_wq = alloc_workqueue("amd_iommu_v2", WQ_MEM_RECLAIM, 0); if (iommu_wq == NULL) -- cgit v1.2.3 From 3703c839e1cf23a3ccecebed73cbf3621238ac00 Mon Sep 17 00:00:00 2001 From: Tom Rix Date: Tue, 15 Dec 2020 13:30:21 -0800 Subject: iommu/amd: remove h from printk format specifier See Documentation/core-api/printk-formats.rst. commit cbacb5ab0aa0 ("docs: printk-formats: Stop encouraging use of unnecessary %h[xudi] and %hh[xudi]") Standard integer promotion is already done and %hx and %hhx is useless so do not encourage the use of %hh[xudi] or %h[xudi]. Signed-off-by: Tom Rix Link: https://lore.kernel.org/r/20201215213021.2090698-1-trix@redhat.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index f999d3424876..1804c3e3b6ac 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -1883,7 +1883,7 @@ static void print_iommu_info(void) struct pci_dev *pdev = iommu->dev; int i; - pci_info(pdev, "Found IOMMU cap 0x%hx\n", iommu->cap_ptr); + pci_info(pdev, "Found IOMMU cap 0x%x\n", iommu->cap_ptr); if (iommu->cap & (1 << IOMMU_CAP_EFR)) { pci_info(pdev, "Extended features (%#llx):", -- cgit v1.2.3 From 8c112a6b3dc6e292625117255ab7bd049639f286 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Tue, 15 Dec 2020 01:36:53 -0600 Subject: iommu/amd: Re-define amd_iommu_domain_encode_pgtable as inline Move the function to header file to allow inclusion in other files. Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20201215073705.123786-2-suravee.suthikulpanit@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 13 +++++++++++++ drivers/iommu/amd/iommu.c | 10 ---------- 2 files changed, 13 insertions(+), 10 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 6b8cbdf71714..0817bc732d1a 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -102,6 +102,19 @@ static inline void *iommu_phys_to_virt(unsigned long paddr) return phys_to_virt(__sme_clr(paddr)); } +static inline +void amd_iommu_domain_set_pt_root(struct protection_domain *domain, u64 root) +{ + atomic64_set(&domain->pt_root, root); +} + +static inline +void amd_iommu_domain_clr_pt_root(struct protection_domain *domain) +{ + amd_iommu_domain_set_pt_root(domain, 0); +} + + extern bool translation_pre_enabled(struct amd_iommu *iommu); extern bool amd_iommu_is_attach_deferred(struct iommu_domain *domain, struct device *dev); diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index f0adbc48fd17..3485f8b60d7a 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -160,16 +160,6 @@ static void amd_iommu_domain_get_pgtable(struct protection_domain *domain, pgtable->mode = pt_root & 7; /* lowest 3 bits encode pgtable mode */ } -static void amd_iommu_domain_set_pt_root(struct protection_domain *domain, u64 root) -{ - atomic64_set(&domain->pt_root, root); -} - -static void amd_iommu_domain_clr_pt_root(struct protection_domain *domain) -{ - amd_iommu_domain_set_pt_root(domain, 0); -} - static void amd_iommu_domain_set_pgtable(struct protection_domain *domain, u64 *root, int mode) { -- cgit v1.2.3 From d8c1df02ac7f2c802a9b2afc0f5c888c4217f1d5 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Thu, 7 Jan 2021 20:29:03 +0800 Subject: iommu: Move iotlb_sync_map out from __iommu_map In the end of __iommu_map, It alway call iotlb_sync_map. This patch moves iotlb_sync_map out from __iommu_map since it is unnecessary to call this for each sg segment especially iotlb_sync_map is flush tlb all currently. Add a little helper _iommu_map for this. Signed-off-by: Yong Wu Reviewed-by: Robin Murphy Acked-by: Will Deacon Link: https://lore.kernel.org/r/20210107122909.16317-2-yong.wu@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/iommu.c | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index ffeebda8d6de..c304a6a30d42 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2426,9 +2426,6 @@ static int __iommu_map(struct iommu_domain *domain, unsigned long iova, size -= pgsize; } - if (ops->iotlb_sync_map) - ops->iotlb_sync_map(domain); - /* unroll mapping in case something went wrong */ if (ret) iommu_unmap(domain, orig_iova, orig_size - size); @@ -2438,18 +2435,31 @@ static int __iommu_map(struct iommu_domain *domain, unsigned long iova, return ret; } +static int _iommu_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t paddr, size_t size, int prot, gfp_t gfp) +{ + const struct iommu_ops *ops = domain->ops; + int ret; + + ret = __iommu_map(domain, iova, paddr, size, prot, GFP_KERNEL); + if (ret == 0 && ops->iotlb_sync_map) + ops->iotlb_sync_map(domain); + + return ret; +} + int iommu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot) { might_sleep(); - return __iommu_map(domain, iova, paddr, size, prot, GFP_KERNEL); + return _iommu_map(domain, iova, paddr, size, prot, GFP_KERNEL); } EXPORT_SYMBOL_GPL(iommu_map); int iommu_map_atomic(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot) { - return __iommu_map(domain, iova, paddr, size, prot, GFP_ATOMIC); + return _iommu_map(domain, iova, paddr, size, prot, GFP_ATOMIC); } EXPORT_SYMBOL_GPL(iommu_map_atomic); @@ -2533,6 +2543,7 @@ static size_t __iommu_map_sg(struct iommu_domain *domain, unsigned long iova, struct scatterlist *sg, unsigned int nents, int prot, gfp_t gfp) { + const struct iommu_ops *ops = domain->ops; size_t len = 0, mapped = 0; phys_addr_t start; unsigned int i = 0; @@ -2563,6 +2574,8 @@ static size_t __iommu_map_sg(struct iommu_domain *domain, unsigned long iova, sg = sg_next(sg); } + if (ops->iotlb_sync_map) + ops->iotlb_sync_map(domain); return mapped; out_err: -- cgit v1.2.3 From 2ebbd25873cef06f739489fd8ff9f707a3dfa2fa Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Thu, 7 Jan 2021 20:29:04 +0800 Subject: iommu: Add iova and size as parameters in iotlb_sync_map iotlb_sync_map allow IOMMU drivers tlb sync after completing the whole mapping. This patch adds iova and size as the parameters in it. then the IOMMU driver could flush tlb with the whole range once after iova mapping to improve performance. Signed-off-by: Yong Wu Reviewed-by: Robin Murphy Acked-by: Will Deacon Link: https://lore.kernel.org/r/20210107122909.16317-3-yong.wu@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/iommu.c | 4 ++-- drivers/iommu/tegra-gart.c | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index c304a6a30d42..3d099a31ddca 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2443,7 +2443,7 @@ static int _iommu_map(struct iommu_domain *domain, unsigned long iova, ret = __iommu_map(domain, iova, paddr, size, prot, GFP_KERNEL); if (ret == 0 && ops->iotlb_sync_map) - ops->iotlb_sync_map(domain); + ops->iotlb_sync_map(domain, iova, size); return ret; } @@ -2575,7 +2575,7 @@ static size_t __iommu_map_sg(struct iommu_domain *domain, unsigned long iova, } if (ops->iotlb_sync_map) - ops->iotlb_sync_map(domain); + ops->iotlb_sync_map(domain, iova, mapped); return mapped; out_err: diff --git a/drivers/iommu/tegra-gart.c b/drivers/iommu/tegra-gart.c index fac720273889..05e8e19b8269 100644 --- a/drivers/iommu/tegra-gart.c +++ b/drivers/iommu/tegra-gart.c @@ -261,7 +261,8 @@ static int gart_iommu_of_xlate(struct device *dev, return 0; } -static void gart_iommu_sync_map(struct iommu_domain *domain) +static void gart_iommu_sync_map(struct iommu_domain *domain, unsigned long iova, + size_t size) { FLUSH_GART_REGS(gart_handle); } @@ -269,7 +270,9 @@ static void gart_iommu_sync_map(struct iommu_domain *domain) static void gart_iommu_sync(struct iommu_domain *domain, struct iommu_iotlb_gather *gather) { - gart_iommu_sync_map(domain); + size_t length = gather->end - gather->start; + + gart_iommu_sync_map(domain, gather->start, length); } static const struct iommu_ops gart_iommu_ops = { -- cgit v1.2.3 From 20143451eff044310520932fb372704c99658b33 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Thu, 7 Jan 2021 20:29:05 +0800 Subject: iommu/mediatek: Add iotlb_sync_map to sync whole the iova range Remove IO_PGTABLE_QUIRK_TLBI_ON_MAP to avoid tlb sync for each a small chunk memory, Use the new iotlb_sync_map to tlb_sync once for whole the iova range of iommu_map. Signed-off-by: Yong Wu Reviewed-by: Robin Murphy Acked-by: Will Deacon Link: https://lore.kernel.org/r/20210107122909.16317-4-yong.wu@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/mtk_iommu.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 8e56cec532e7..f579fc21971e 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -322,7 +322,6 @@ static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom) dom->cfg = (struct io_pgtable_cfg) { .quirks = IO_PGTABLE_QUIRK_ARM_NS | IO_PGTABLE_QUIRK_NO_PERMS | - IO_PGTABLE_QUIRK_TLBI_ON_MAP | IO_PGTABLE_QUIRK_ARM_MTK_EXT, .pgsize_bitmap = mtk_iommu_ops.pgsize_bitmap, .ias = 32, @@ -453,6 +452,14 @@ static void mtk_iommu_iotlb_sync(struct iommu_domain *domain, data); } +static void mtk_iommu_sync_map(struct iommu_domain *domain, unsigned long iova, + size_t size) +{ + struct mtk_iommu_data *data = mtk_iommu_get_m4u_data(); + + mtk_iommu_tlb_flush_range_sync(iova, size, size, data); +} + static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) { @@ -539,6 +546,7 @@ static const struct iommu_ops mtk_iommu_ops = { .unmap = mtk_iommu_unmap, .flush_iotlb_all = mtk_iommu_flush_iotlb_all, .iotlb_sync = mtk_iommu_iotlb_sync, + .iotlb_sync_map = mtk_iommu_sync_map, .iova_to_phys = mtk_iommu_iova_to_phys, .probe_device = mtk_iommu_probe_device, .release_device = mtk_iommu_release_device, -- cgit v1.2.3 From 862c3715de8f3e5350489240c951d697f04bd8c9 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Thu, 7 Jan 2021 20:29:06 +0800 Subject: iommu: Switch gather->end to the inclusive end Currently gather->end is "unsigned long" which may be overflow in arch32 in the corner case: 0xfff00000 + 0x100000(iova + size). Although it doesn't affect the size(end - start), it affects the checking "gather->end < end" This patch changes this "end" to the real end address (end = start + size - 1). Correspondingly, update the length to "end - start + 1". Fixes: a7d20dc19d9e ("iommu: Introduce struct iommu_iotlb_gather for batching TLB flushes") Signed-off-by: Yong Wu Reviewed-by: Robin Murphy Acked-by: Will Deacon Link: https://lore.kernel.org/r/20210107122909.16317-5-yong.wu@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 2 +- drivers/iommu/mtk_iommu.c | 2 +- drivers/iommu/tegra-gart.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 8ca7415d785d..c70d6e79f534 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2280,7 +2280,7 @@ static void arm_smmu_iotlb_sync(struct iommu_domain *domain, { struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - arm_smmu_tlb_inv_range(gather->start, gather->end - gather->start, + arm_smmu_tlb_inv_range(gather->start, gather->end - gather->start + 1, gather->pgsize, true, smmu_domain); } diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index f579fc21971e..66a00a2cb445 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -443,7 +443,7 @@ static void mtk_iommu_iotlb_sync(struct iommu_domain *domain, struct iommu_iotlb_gather *gather) { struct mtk_iommu_data *data = mtk_iommu_get_m4u_data(); - size_t length = gather->end - gather->start; + size_t length = gather->end - gather->start + 1; if (gather->start == ULONG_MAX) return; diff --git a/drivers/iommu/tegra-gart.c b/drivers/iommu/tegra-gart.c index 05e8e19b8269..6f130e51f072 100644 --- a/drivers/iommu/tegra-gart.c +++ b/drivers/iommu/tegra-gart.c @@ -270,7 +270,7 @@ static void gart_iommu_sync_map(struct iommu_domain *domain, unsigned long iova, static void gart_iommu_sync(struct iommu_domain *domain, struct iommu_iotlb_gather *gather) { - size_t length = gather->end - gather->start; + size_t length = gather->end - gather->start + 1; gart_iommu_sync_map(domain, gather->start, length); } -- cgit v1.2.3 From f21ae3b100840c1439fb326841f24641cf3de4a1 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Thu, 7 Jan 2021 20:29:08 +0800 Subject: iommu/mediatek: Gather iova in iommu_unmap to achieve tlb sync once In current iommu_unmap, this code is: iommu_iotlb_gather_init(&iotlb_gather); ret = __iommu_unmap(domain, iova, size, &iotlb_gather); iommu_iotlb_sync(domain, &iotlb_gather); We could gather the whole iova range in __iommu_unmap, and then do tlb synchronization in the iommu_iotlb_sync. This patch implement this, Gather the range in mtk_iommu_unmap. then iommu_iotlb_sync call tlb synchronization for the gathered iova range. we don't call iommu_iotlb_gather_add_page since our tlb synchronization could be regardless of granule size. In this way, gather->start is impossible ULONG_MAX, remove the checking. This patch aims to do tlb synchronization *once* in the iommu_unmap. Signed-off-by: Yong Wu Reviewed-by: Robin Murphy Acked-by: Will Deacon Link: https://lore.kernel.org/r/20210107122909.16317-7-yong.wu@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/mtk_iommu.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 66a00a2cb445..d3b8a1649093 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -430,7 +430,12 @@ static size_t mtk_iommu_unmap(struct iommu_domain *domain, struct iommu_iotlb_gather *gather) { struct mtk_iommu_domain *dom = to_mtk_domain(domain); + unsigned long end = iova + size - 1; + if (gather->start > iova) + gather->start = iova; + if (gather->end < end) + gather->end = end; return dom->iop->unmap(dom->iop, iova, size, gather); } @@ -445,9 +450,6 @@ static void mtk_iommu_iotlb_sync(struct iommu_domain *domain, struct mtk_iommu_data *data = mtk_iommu_get_m4u_data(); size_t length = gather->end - gather->start + 1; - if (gather->start == ULONG_MAX) - return; - mtk_iommu_tlb_flush_range_sync(gather->start, length, gather->pgsize, data); } -- cgit v1.2.3 From 0954d61a59e3c014e52b8d938bc12dc5a2e4949c Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Thu, 7 Jan 2021 20:29:09 +0800 Subject: iommu/mediatek: Remove the tlb-ops for v7s Until now, we have already used the tlb operations from iommu framework, then the tlb operations for v7s can be removed. Correspondingly, Switch the paramenter "cookie" to the internal structure. Signed-off-by: Yong Wu Reviewed-by: Robin Murphy Acked-by: Will Deacon Link: https://lore.kernel.org/r/20210107122909.16317-8-yong.wu@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/mtk_iommu.c | 27 ++++----------------------- 1 file changed, 4 insertions(+), 23 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index d3b8a1649093..86ab577c9520 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -182,10 +182,8 @@ static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom) return container_of(dom, struct mtk_iommu_domain, domain); } -static void mtk_iommu_tlb_flush_all(void *cookie) +static void mtk_iommu_tlb_flush_all(struct mtk_iommu_data *data) { - struct mtk_iommu_data *data = cookie; - for_each_m4u(data) { writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, data->base + data->plat_data->inv_sel_reg); @@ -195,9 +193,9 @@ static void mtk_iommu_tlb_flush_all(void *cookie) } static void mtk_iommu_tlb_flush_range_sync(unsigned long iova, size_t size, - size_t granule, void *cookie) + size_t granule, + struct mtk_iommu_data *data) { - struct mtk_iommu_data *data = cookie; unsigned long flags; int ret; u32 tmp; @@ -219,7 +217,7 @@ static void mtk_iommu_tlb_flush_range_sync(unsigned long iova, size_t size, if (ret) { dev_warn(data->dev, "Partial TLB flush timed out, falling back to full flush\n"); - mtk_iommu_tlb_flush_all(cookie); + mtk_iommu_tlb_flush_all(data); } /* Clear the CPE status */ writel_relaxed(0, data->base + REG_MMU_CPE_DONE); @@ -227,22 +225,6 @@ static void mtk_iommu_tlb_flush_range_sync(unsigned long iova, size_t size, } } -static void mtk_iommu_tlb_flush_page_nosync(struct iommu_iotlb_gather *gather, - unsigned long iova, size_t granule, - void *cookie) -{ - struct mtk_iommu_data *data = cookie; - struct iommu_domain *domain = &data->m4u_dom->domain; - - iommu_iotlb_gather_add_page(domain, gather, iova, granule); -} - -static const struct iommu_flush_ops mtk_iommu_flush_ops = { - .tlb_flush_all = mtk_iommu_tlb_flush_all, - .tlb_flush_walk = mtk_iommu_tlb_flush_range_sync, - .tlb_add_page = mtk_iommu_tlb_flush_page_nosync, -}; - static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) { struct mtk_iommu_data *data = dev_id; @@ -326,7 +308,6 @@ static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom) .pgsize_bitmap = mtk_iommu_ops.pgsize_bitmap, .ias = 32, .oas = 34, - .tlb = &mtk_iommu_flush_ops, .iommu_dev = data->dev, }; -- cgit v1.2.3 From 9872f9bd9dbd68f75e8db782717d71e8594f6a02 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Thu, 14 Jan 2021 16:50:21 +0800 Subject: iommu/vt-d: Consolidate duplicate cache invaliation code The pasid based IOTLB and devTLB invalidation code is duplicate in several places. Consolidate them by using the common helpers. Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20210114085021.717041-1-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/pasid.c | 18 ++------------- drivers/iommu/intel/svm.c | 55 ++++++++------------------------------------- 2 files changed, 11 insertions(+), 62 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index b92af83b79bd..f26cb6195b2c 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -456,20 +456,6 @@ pasid_cache_invalidation_with_pasid(struct intel_iommu *iommu, qi_submit_sync(iommu, &desc, 1, 0); } -static void -iotlb_invalidation_with_pasid(struct intel_iommu *iommu, u16 did, u32 pasid) -{ - struct qi_desc desc; - - desc.qw0 = QI_EIOTLB_PASID(pasid) | QI_EIOTLB_DID(did) | - QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) | QI_EIOTLB_TYPE; - desc.qw1 = 0; - desc.qw2 = 0; - desc.qw3 = 0; - - qi_submit_sync(iommu, &desc, 1, 0); -} - static void devtlb_invalidation_with_pasid(struct intel_iommu *iommu, struct device *dev, u32 pasid) @@ -514,7 +500,7 @@ void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev, clflush_cache_range(pte, sizeof(*pte)); pasid_cache_invalidation_with_pasid(iommu, did, pasid); - iotlb_invalidation_with_pasid(iommu, did, pasid); + qi_flush_piotlb(iommu, did, pasid, 0, -1, 0); /* Device IOTLB doesn't need to be flushed in caching mode. */ if (!cap_caching_mode(iommu->cap)) @@ -530,7 +516,7 @@ static void pasid_flush_caches(struct intel_iommu *iommu, if (cap_caching_mode(iommu->cap)) { pasid_cache_invalidation_with_pasid(iommu, did, pasid); - iotlb_invalidation_with_pasid(iommu, did, pasid); + qi_flush_piotlb(iommu, did, pasid, 0, -1, 0); } else { iommu_flush_write_buffer(iommu); } diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 18a9f05df407..033b25886e57 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -123,53 +123,16 @@ static void __flush_svm_range_dev(struct intel_svm *svm, unsigned long address, unsigned long pages, int ih) { - struct qi_desc desc; + struct device_domain_info *info = get_domain_info(sdev->dev); - if (pages == -1) { - desc.qw0 = QI_EIOTLB_PASID(svm->pasid) | - QI_EIOTLB_DID(sdev->did) | - QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) | - QI_EIOTLB_TYPE; - desc.qw1 = 0; - } else { - int mask = ilog2(__roundup_pow_of_two(pages)); - - desc.qw0 = QI_EIOTLB_PASID(svm->pasid) | - QI_EIOTLB_DID(sdev->did) | - QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) | - QI_EIOTLB_TYPE; - desc.qw1 = QI_EIOTLB_ADDR(address) | - QI_EIOTLB_IH(ih) | - QI_EIOTLB_AM(mask); - } - desc.qw2 = 0; - desc.qw3 = 0; - qi_submit_sync(sdev->iommu, &desc, 1, 0); - - if (sdev->dev_iotlb) { - desc.qw0 = QI_DEV_EIOTLB_PASID(svm->pasid) | - QI_DEV_EIOTLB_SID(sdev->sid) | - QI_DEV_EIOTLB_QDEP(sdev->qdep) | - QI_DEIOTLB_TYPE; - if (pages == -1) { - desc.qw1 = QI_DEV_EIOTLB_ADDR(-1ULL >> 1) | - QI_DEV_EIOTLB_SIZE; - } else if (pages > 1) { - /* The least significant zero bit indicates the size. So, - * for example, an "address" value of 0x12345f000 will - * flush from 0x123440000 to 0x12347ffff (256KiB). */ - unsigned long last = address + ((unsigned long)(pages - 1) << VTD_PAGE_SHIFT); - unsigned long mask = __rounddown_pow_of_two(address ^ last); - - desc.qw1 = QI_DEV_EIOTLB_ADDR((address & ~mask) | - (mask - 1)) | QI_DEV_EIOTLB_SIZE; - } else { - desc.qw1 = QI_DEV_EIOTLB_ADDR(address); - } - desc.qw2 = 0; - desc.qw3 = 0; - qi_submit_sync(sdev->iommu, &desc, 1, 0); - } + if (WARN_ON(!pages)) + return; + + qi_flush_piotlb(sdev->iommu, sdev->did, svm->pasid, address, pages, ih); + if (info->ats_enabled) + qi_flush_dev_iotlb_pasid(sdev->iommu, sdev->sid, info->pfsid, + svm->pasid, sdev->qdep, address, + order_base_2(pages)); } static void intel_flush_svm_range_dev(struct intel_svm *svm, -- cgit v1.2.3 From f2dd871799ba5d80f95f9bdbc0e60d390e1bcd22 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Thu, 14 Jan 2021 17:04:00 +0800 Subject: iommu/vt-d: Add qi_submit trace event This adds a new trace event to track the submissions of requests to the invalidation queue. This event will provide the information like: - IOMMU name - Invalidation type - Descriptor raw data A sample output like: | qi_submit: iotlb_inv dmar1: 0x100e2 0x0 0x0 0x0 | qi_submit: dev_tlb_inv dmar1: 0x1000000003 0x7ffffffffffff001 0x0 0x0 | qi_submit: iotlb_inv dmar2: 0x800f2 0xf9a00005 0x0 0x0 This will be helpful for queued invalidation related debugging. Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20210114090400.736104-1-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/dmar.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index 004feaed3c72..bd51f33642e0 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -31,6 +31,7 @@ #include #include #include +#include #include "../irq_remapping.h" @@ -1307,6 +1308,8 @@ restart: offset = ((index + i) % QI_LENGTH) << shift; memcpy(qi->desc + offset, &desc[i], 1 << shift); qi->desc_status[(index + i) % QI_LENGTH] = QI_IN_USE; + trace_qi_submit(iommu, desc[i].qw0, desc[i].qw1, + desc[i].qw2, desc[i].qw3); } qi->desc_status[wait_index] = QI_IN_USE; -- cgit v1.2.3 From a8ce9ebbecdfda3322bbcece6b3b25888217f8e3 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Fri, 15 Jan 2021 08:42:02 +0800 Subject: iommu/vt-d: Preset Access/Dirty bits for IOVA over FL The Access/Dirty bits in the first level page table entry will be set whenever a page table entry was used for address translation or write permission was successfully translated. This is always true when using the first-level page table for kernel IOVA. Instead of wasting hardware cycles to update the certain bits, it's better to set them up at the beginning. Suggested-by: Ashok Raj Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20210115004202.953965-1-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index f665322a0991..be85af612bc1 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -1017,8 +1017,11 @@ static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain, domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE); pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE; - if (domain_use_first_level(domain)) + if (domain_use_first_level(domain)) { pteval |= DMA_FL_PTE_XD | DMA_FL_PTE_US; + if (domain->domain.type == IOMMU_DOMAIN_DMA) + pteval |= DMA_FL_PTE_ACCESS; + } if (cmpxchg64(&pte->val, 0ULL, pteval)) /* Someone else set it while we were thinking; use theirs. */ free_pgtable_page(tmp_page); @@ -2310,9 +2313,16 @@ __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, return -EINVAL; attr = prot & (DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP); - if (domain_use_first_level(domain)) + if (domain_use_first_level(domain)) { attr |= DMA_FL_PTE_PRESENT | DMA_FL_PTE_XD | DMA_FL_PTE_US; + if (domain->domain.type == IOMMU_DOMAIN_DMA) { + attr |= DMA_FL_PTE_ACCESS; + if (prot & DMA_PTE_WRITE) + attr |= DMA_FL_PTE_DIRTY; + } + } + pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | attr; while (nr_pages > 0) { -- cgit v1.2.3 From a8e8af35c9f4f75f981a95488c7066d31bac4bef Mon Sep 17 00:00:00 2001 From: Lianbo Jiang Date: Tue, 26 Jan 2021 19:53:36 +0800 Subject: dma-iommu: use static-key to minimize the impact in the fast-path Let's move out the is_kdump_kernel() check from iommu_dma_deferred_attach() to iommu_dma_init(), and use the static-key in the fast-path to minimize the impact in the normal case. Co-developed-by: Robin Murphy Signed-off-by: Lianbo Jiang Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/20210126115337.20068-2-lijiang@redhat.com Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 4078358ed66e..c80056f6c9f9 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -51,6 +51,8 @@ struct iommu_dma_cookie { struct iommu_domain *fq_domain; }; +static DEFINE_STATIC_KEY_FALSE(iommu_deferred_attach_enabled); + void iommu_dma_free_cpu_cached_iovas(unsigned int cpu, struct iommu_domain *domain) { @@ -383,9 +385,6 @@ static int iommu_dma_deferred_attach(struct device *dev, { const struct iommu_ops *ops = domain->ops; - if (!is_kdump_kernel()) - return 0; - if (unlikely(ops->is_attach_deferred && ops->is_attach_deferred(domain, dev))) return iommu_attach_device(domain, dev); @@ -535,7 +534,8 @@ static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys, size_t iova_off = iova_offset(iovad, phys); dma_addr_t iova; - if (unlikely(iommu_dma_deferred_attach(dev, domain))) + if (static_branch_unlikely(&iommu_deferred_attach_enabled) && + iommu_dma_deferred_attach(dev, domain)) return DMA_MAPPING_ERROR; size = iova_align(iovad, size + iova_off); @@ -693,7 +693,8 @@ static void *iommu_dma_alloc_remap(struct device *dev, size_t size, *dma_handle = DMA_MAPPING_ERROR; - if (unlikely(iommu_dma_deferred_attach(dev, domain))) + if (static_branch_unlikely(&iommu_deferred_attach_enabled) && + iommu_dma_deferred_attach(dev, domain)) return NULL; min_size = alloc_sizes & -alloc_sizes; @@ -976,7 +977,8 @@ static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, unsigned long mask = dma_get_seg_boundary(dev); int i; - if (unlikely(iommu_dma_deferred_attach(dev, domain))) + if (static_branch_unlikely(&iommu_deferred_attach_enabled) && + iommu_dma_deferred_attach(dev, domain)) return 0; if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) @@ -1424,6 +1426,9 @@ void iommu_dma_compose_msi_msg(struct msi_desc *desc, static int iommu_dma_init(void) { + if (is_kdump_kernel()) + static_branch_enable(&iommu_deferred_attach_enabled); + return iova_cache_get(); } arch_initcall(iommu_dma_init); -- cgit v1.2.3 From 3ab657291638ea267654c3e4798161b2cee6ae01 Mon Sep 17 00:00:00 2001 From: Lianbo Jiang Date: Tue, 26 Jan 2021 19:53:37 +0800 Subject: iommu: use the __iommu_attach_device() directly for deferred attach Currently, because domain attach allows to be deferred from iommu driver to device driver, and when iommu initializes, the devices on the bus will be scanned and the default groups will be allocated. Due to the above changes, some devices could be added to the same group as below: [ 3.859417] pci 0000:01:00.0: Adding to iommu group 16 [ 3.864572] pci 0000:01:00.1: Adding to iommu group 16 [ 3.869738] pci 0000:02:00.0: Adding to iommu group 17 [ 3.874892] pci 0000:02:00.1: Adding to iommu group 17 But when attaching these devices, it doesn't allow that a group has more than one device, otherwise it will return an error. This conflicts with the deferred attaching. Unfortunately, it has two devices in the same group for my side, for example: [ 9.627014] iommu_group_device_count(): device name[0]:0000:01:00.0 [ 9.633545] iommu_group_device_count(): device name[1]:0000:01:00.1 ... [ 10.255609] iommu_group_device_count(): device name[0]:0000:02:00.0 [ 10.262144] iommu_group_device_count(): device name[1]:0000:02:00.1 Finally, which caused the failure of tg3 driver when tg3 driver calls the dma_alloc_coherent() to allocate coherent memory in the tg3_test_dma(). [ 9.660310] tg3 0000:01:00.0: DMA engine test failed, aborting [ 9.754085] tg3: probe of 0000:01:00.0 failed with error -12 [ 9.997512] tg3 0000:01:00.1: DMA engine test failed, aborting [ 10.043053] tg3: probe of 0000:01:00.1 failed with error -12 [ 10.288905] tg3 0000:02:00.0: DMA engine test failed, aborting [ 10.334070] tg3: probe of 0000:02:00.0 failed with error -12 [ 10.578303] tg3 0000:02:00.1: DMA engine test failed, aborting [ 10.622629] tg3: probe of 0000:02:00.1 failed with error -12 In addition, the similar situations also occur in other drivers such as the bnxt_en driver. That can be reproduced easily in kdump kernel when SME is active. Let's move the handling currently in iommu_dma_deferred_attach() into the iommu core code so that it can call the __iommu_attach_device() directly instead of the iommu_attach_device(). The external interface iommu_attach_device() is not suitable for handling this situation. Signed-off-by: Lianbo Jiang Reviewed-by: Robin Murphy Link: https://lore.kernel.org/r/20210126115337.20068-3-lijiang@redhat.com Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 18 +++--------------- drivers/iommu/iommu.c | 10 ++++++++++ 2 files changed, 13 insertions(+), 15 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index c80056f6c9f9..f659395e7959 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -380,18 +380,6 @@ static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, return iova_reserve_iommu_regions(dev, domain); } -static int iommu_dma_deferred_attach(struct device *dev, - struct iommu_domain *domain) -{ - const struct iommu_ops *ops = domain->ops; - - if (unlikely(ops->is_attach_deferred && - ops->is_attach_deferred(domain, dev))) - return iommu_attach_device(domain, dev); - - return 0; -} - /** * dma_info_to_prot - Translate DMA API directions and attributes to IOMMU API * page flags. @@ -535,7 +523,7 @@ static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys, dma_addr_t iova; if (static_branch_unlikely(&iommu_deferred_attach_enabled) && - iommu_dma_deferred_attach(dev, domain)) + iommu_deferred_attach(dev, domain)) return DMA_MAPPING_ERROR; size = iova_align(iovad, size + iova_off); @@ -694,7 +682,7 @@ static void *iommu_dma_alloc_remap(struct device *dev, size_t size, *dma_handle = DMA_MAPPING_ERROR; if (static_branch_unlikely(&iommu_deferred_attach_enabled) && - iommu_dma_deferred_attach(dev, domain)) + iommu_deferred_attach(dev, domain)) return NULL; min_size = alloc_sizes & -alloc_sizes; @@ -978,7 +966,7 @@ static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, int i; if (static_branch_unlikely(&iommu_deferred_attach_enabled) && - iommu_dma_deferred_attach(dev, domain)) + iommu_deferred_attach(dev, domain)) return 0; if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 91f7871f5a37..fd76e2f579fe 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1980,6 +1980,16 @@ out_unlock: } EXPORT_SYMBOL_GPL(iommu_attach_device); +int iommu_deferred_attach(struct device *dev, struct iommu_domain *domain) +{ + const struct iommu_ops *ops = domain->ops; + + if (ops->is_attach_deferred && ops->is_attach_deferred(domain, dev)) + return __iommu_attach_device(domain, dev); + + return 0; +} + /* * Check flags and other user provided data for valid combinations. We also * make sure no reserved fields or unused flags are set. This is to ensure -- cgit v1.2.3 From c9b258c6be09283663c6851725b322568d867c0b Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Tue, 15 Dec 2020 01:36:54 -0600 Subject: iommu/amd: Prepare for generic IO page table framework Add initial hook up code to implement generic IO page table framework. Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20201215073705.123786-3-suravee.suthikulpanit@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/Kconfig | 1 + drivers/iommu/amd/Makefile | 2 +- drivers/iommu/amd/amd_iommu_types.h | 35 +++++++++++++++++++ drivers/iommu/amd/io_pgtable.c | 69 +++++++++++++++++++++++++++++++++++++ drivers/iommu/amd/iommu.c | 10 ------ drivers/iommu/io-pgtable.c | 3 ++ 6 files changed, 109 insertions(+), 11 deletions(-) create mode 100644 drivers/iommu/amd/io_pgtable.c (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/Kconfig b/drivers/iommu/amd/Kconfig index 626b97d0dd21..a3cbafb603f5 100644 --- a/drivers/iommu/amd/Kconfig +++ b/drivers/iommu/amd/Kconfig @@ -10,6 +10,7 @@ config AMD_IOMMU select IOMMU_API select IOMMU_IOVA select IOMMU_DMA + select IOMMU_IO_PGTABLE depends on X86_64 && PCI && ACPI && HAVE_CMPXCHG_DOUBLE help With this option you can enable support for AMD IOMMU hardware in diff --git a/drivers/iommu/amd/Makefile b/drivers/iommu/amd/Makefile index dc5a2fa4fd37..a935f8f4b974 100644 --- a/drivers/iommu/amd/Makefile +++ b/drivers/iommu/amd/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_AMD_IOMMU) += iommu.o init.o quirks.o +obj-$(CONFIG_AMD_IOMMU) += iommu.o init.o quirks.o io_pgtable.o obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += debugfs.o obj-$(CONFIG_AMD_IOMMU_V2) += iommu_v2.o diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 553587827771..b2365924d898 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -15,6 +15,7 @@ #include #include #include +#include /* * Maximum number of IOMMUs supported @@ -252,6 +253,19 @@ #define GA_GUEST_NR 0x1 +#define IOMMU_IN_ADDR_BIT_SIZE 52 +#define IOMMU_OUT_ADDR_BIT_SIZE 52 + +/* + * This bitmap is used to advertise the page sizes our hardware support + * to the IOMMU core, which will then use this information to split + * physically contiguous memory regions it is mapping into page sizes + * that we support. + * + * 512GB Pages are not supported due to a hardware bug + */ +#define AMD_IOMMU_PGSIZES ((~0xFFFUL) & ~(2ULL << 38)) + /* Bit value definition for dte irq remapping fields*/ #define DTE_IRQ_PHYS_ADDR_MASK (((1ULL << 45)-1) << 6) #define DTE_IRQ_REMAP_INTCTL_MASK (0x3ULL << 60) @@ -466,6 +480,26 @@ struct amd_irte_ops; #define AMD_IOMMU_FLAG_TRANS_PRE_ENABLED (1 << 0) +#define io_pgtable_to_data(x) \ + container_of((x), struct amd_io_pgtable, iop) + +#define io_pgtable_ops_to_data(x) \ + io_pgtable_to_data(io_pgtable_ops_to_pgtable(x)) + +#define io_pgtable_ops_to_domain(x) \ + container_of(io_pgtable_ops_to_data(x), \ + struct protection_domain, iop) + +#define io_pgtable_cfg_to_data(x) \ + container_of((x), struct amd_io_pgtable, pgtbl_cfg) + +struct amd_io_pgtable { + struct io_pgtable_cfg pgtbl_cfg; + struct io_pgtable iop; + int mode; + u64 *root; +}; + /* * This structure contains generic data for IOMMU protection domains * independent of their use. @@ -474,6 +508,7 @@ struct protection_domain { struct list_head dev_list; /* List of all devices in this domain */ struct iommu_domain domain; /* generic domain handle used by iommu core code */ + struct amd_io_pgtable iop; spinlock_t lock; /* mostly used to lock the page table*/ u16 id; /* the domain id written to the device table */ atomic64_t pt_root; /* pgtable root and pgtable mode */ diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c new file mode 100644 index 000000000000..cf2cd2d2e43e --- /dev/null +++ b/drivers/iommu/amd/io_pgtable.c @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * CPU-agnostic AMD IO page table allocator. + * + * Copyright (C) 2020 Advanced Micro Devices, Inc. + * Author: Suravee Suthikulpanit + */ + +#define pr_fmt(fmt) "AMD-Vi: " fmt +#define dev_fmt(fmt) pr_fmt(fmt) + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "amd_iommu_types.h" +#include "amd_iommu.h" + +static void v1_tlb_flush_all(void *cookie) +{ +} + +static void v1_tlb_flush_walk(unsigned long iova, size_t size, + size_t granule, void *cookie) +{ +} + +static void v1_tlb_add_page(struct iommu_iotlb_gather *gather, + unsigned long iova, size_t granule, + void *cookie) +{ +} + +static const struct iommu_flush_ops v1_flush_ops = { + .tlb_flush_all = v1_tlb_flush_all, + .tlb_flush_walk = v1_tlb_flush_walk, + .tlb_add_page = v1_tlb_add_page, +}; + +/* + * ---------------------------------------------------- + */ +static void v1_free_pgtable(struct io_pgtable *iop) +{ +} + +static struct io_pgtable *v1_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) +{ + struct amd_io_pgtable *pgtable = io_pgtable_cfg_to_data(cfg); + + cfg->pgsize_bitmap = AMD_IOMMU_PGSIZES, + cfg->ias = IOMMU_IN_ADDR_BIT_SIZE, + cfg->oas = IOMMU_OUT_ADDR_BIT_SIZE, + cfg->tlb = &v1_flush_ops; + + return &pgtable->iop; +} + +struct io_pgtable_init_fns io_pgtable_amd_iommu_v1_init_fns = { + .alloc = v1_alloc_pgtable, + .free = v1_free_pgtable, +}; diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 3485f8b60d7a..18d8c97f681e 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -57,16 +57,6 @@ #define HT_RANGE_START (0xfd00000000ULL) #define HT_RANGE_END (0xffffffffffULL) -/* - * This bitmap is used to advertise the page sizes our hardware support - * to the IOMMU core, which will then use this information to split - * physically contiguous memory regions it is mapping into page sizes - * that we support. - * - * 512GB Pages are not supported due to a hardware bug - */ -#define AMD_IOMMU_PGSIZES ((~0xFFFUL) & ~(2ULL << 38)) - #define DEFAULT_PGTABLE_LEVEL PAGE_MODE_3_LEVEL static DEFINE_SPINLOCK(pd_bitmap_lock); diff --git a/drivers/iommu/io-pgtable.c b/drivers/iommu/io-pgtable.c index 94394c81468f..6e9917ce980f 100644 --- a/drivers/iommu/io-pgtable.c +++ b/drivers/iommu/io-pgtable.c @@ -24,6 +24,9 @@ io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = { #ifdef CONFIG_IOMMU_IO_PGTABLE_ARMV7S [ARM_V7S] = &io_pgtable_arm_v7s_init_fns, #endif +#ifdef CONFIG_AMD_IOMMU + [AMD_IOMMU_V1] = &io_pgtable_amd_iommu_v1_init_fns, +#endif }; struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt, -- cgit v1.2.3 From d2272ec7f946470e861b77572a2f31325faf59c6 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Tue, 15 Dec 2020 01:36:55 -0600 Subject: iommu/amd: Move pt_root to struct amd_io_pgtable To better organize the data structure since it contains IO page table related information. Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20201215073705.123786-4-suravee.suthikulpanit@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 2 +- drivers/iommu/amd/amd_iommu_types.h | 2 +- drivers/iommu/amd/iommu.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 0817bc732d1a..b8dae3941f0f 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -105,7 +105,7 @@ static inline void *iommu_phys_to_virt(unsigned long paddr) static inline void amd_iommu_domain_set_pt_root(struct protection_domain *domain, u64 root) { - atomic64_set(&domain->pt_root, root); + atomic64_set(&domain->iop.pt_root, root); } static inline diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index b2365924d898..a50f117879f2 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -498,6 +498,7 @@ struct amd_io_pgtable { struct io_pgtable iop; int mode; u64 *root; + atomic64_t pt_root; /* pgtable root and pgtable mode */ }; /* @@ -511,7 +512,6 @@ struct protection_domain { struct amd_io_pgtable iop; spinlock_t lock; /* mostly used to lock the page table*/ u16 id; /* the domain id written to the device table */ - atomic64_t pt_root; /* pgtable root and pgtable mode */ int glx; /* Number of levels for GCR3 table */ u64 *gcr3_tbl; /* Guest CR3 table */ unsigned long flags; /* flags to find out type of domain */ diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 18d8c97f681e..2a4730380462 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -144,7 +144,7 @@ static struct protection_domain *to_pdomain(struct iommu_domain *dom) static void amd_iommu_domain_get_pgtable(struct protection_domain *domain, struct domain_pgtable *pgtable) { - u64 pt_root = atomic64_read(&domain->pt_root); + u64 pt_root = atomic64_read(&domain->iop.pt_root); pgtable->root = (u64 *)(pt_root & PAGE_MASK); pgtable->mode = pt_root & 7; /* lowest 3 bits encode pgtable mode */ -- cgit v1.2.3 From 1f5855306676cf1d10785043d8fdc7a82bebd50b Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Tue, 15 Dec 2020 01:36:56 -0600 Subject: iommu/amd: Convert to using amd_io_pgtable Make use of the new struct amd_io_pgtable in preparation to remove the struct domain_pgtable. Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20201215073705.123786-5-suravee.suthikulpanit@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 1 + drivers/iommu/amd/iommu.c | 25 ++++++++++--------------- 2 files changed, 11 insertions(+), 15 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index b8dae3941f0f..bf9723b35e77 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -56,6 +56,7 @@ extern void amd_iommu_domain_direct_map(struct iommu_domain *dom); extern int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids); extern int amd_iommu_flush_page(struct iommu_domain *dom, u32 pasid, u64 address); +extern void amd_iommu_update_and_flush_device_table(struct protection_domain *domain); extern int amd_iommu_flush_tlb(struct iommu_domain *dom, u32 pasid); extern int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, u32 pasid, unsigned long cr3); diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 2a4730380462..763efee81524 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -88,8 +88,6 @@ struct kmem_cache *amd_iommu_irq_cache; static void update_domain(struct protection_domain *domain); static void detach_device(struct device *dev); -static void update_and_flush_device_table(struct protection_domain *domain, - struct domain_pgtable *pgtable); /**************************************************************************** * @@ -1501,7 +1499,7 @@ static bool increase_address_space(struct protection_domain *domain, pgtable.root = pte; pgtable.mode += 1; - update_and_flush_device_table(domain, &pgtable); + amd_iommu_update_and_flush_device_table(domain); domain_flush_complete(domain); /* @@ -1876,17 +1874,16 @@ static void free_gcr3_table(struct protection_domain *domain) } static void set_dte_entry(u16 devid, struct protection_domain *domain, - struct domain_pgtable *pgtable, bool ats, bool ppr) { u64 pte_root = 0; u64 flags = 0; u32 old_domid; - if (pgtable->mode != PAGE_MODE_NONE) - pte_root = iommu_virt_to_phys(pgtable->root); + if (domain->iop.mode != PAGE_MODE_NONE) + pte_root = iommu_virt_to_phys(domain->iop.root); - pte_root |= (pgtable->mode & DEV_ENTRY_MODE_MASK) + pte_root |= (domain->iop.mode & DEV_ENTRY_MODE_MASK) << DEV_ENTRY_MODE_SHIFT; pte_root |= DTE_FLAG_IR | DTE_FLAG_IW | DTE_FLAG_V | DTE_FLAG_TV; @@ -1976,7 +1973,7 @@ static void do_attach(struct iommu_dev_data *dev_data, /* Update device table */ amd_iommu_domain_get_pgtable(domain, &pgtable); - set_dte_entry(dev_data->devid, domain, &pgtable, + set_dte_entry(dev_data->devid, domain, ats, dev_data->iommu_v2); clone_aliases(dev_data->pdev); @@ -2283,22 +2280,20 @@ static int amd_iommu_domain_get_attr(struct iommu_domain *domain, * *****************************************************************************/ -static void update_device_table(struct protection_domain *domain, - struct domain_pgtable *pgtable) +static void update_device_table(struct protection_domain *domain) { struct iommu_dev_data *dev_data; list_for_each_entry(dev_data, &domain->dev_list, list) { - set_dte_entry(dev_data->devid, domain, pgtable, + set_dte_entry(dev_data->devid, domain, dev_data->ats.enabled, dev_data->iommu_v2); clone_aliases(dev_data->pdev); } } -static void update_and_flush_device_table(struct protection_domain *domain, - struct domain_pgtable *pgtable) +void amd_iommu_update_and_flush_device_table(struct protection_domain *domain) { - update_device_table(domain, pgtable); + update_device_table(domain); domain_flush_devices(domain); } @@ -2308,7 +2303,7 @@ static void update_domain(struct protection_domain *domain) /* Update device table */ amd_iommu_domain_get_pgtable(domain, &pgtable); - update_and_flush_device_table(domain, &pgtable); + amd_iommu_update_and_flush_device_table(domain); /* Flush domain TLB(s) and wait for completion */ domain_flush_tlb_pde(domain); -- cgit v1.2.3 From f9b4df790aa4372bfa11b7d212e537b763295429 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Tue, 15 Dec 2020 01:36:57 -0600 Subject: iommu/amd: Declare functions as extern And move declaration to header file so that they can be included across multiple files. There is no functional change. Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20201215073705.123786-6-suravee.suthikulpanit@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 3 +++ drivers/iommu/amd/iommu.c | 39 +++++++++++++++++++-------------------- 2 files changed, 22 insertions(+), 20 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index bf9723b35e77..bf29ab8c99f0 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -57,6 +57,9 @@ extern int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids); extern int amd_iommu_flush_page(struct iommu_domain *dom, u32 pasid, u64 address); extern void amd_iommu_update_and_flush_device_table(struct protection_domain *domain); +extern void amd_iommu_domain_update(struct protection_domain *domain); +extern void amd_iommu_domain_flush_complete(struct protection_domain *domain); +extern void amd_iommu_domain_flush_tlb_pde(struct protection_domain *domain); extern int amd_iommu_flush_tlb(struct iommu_domain *dom, u32 pasid); extern int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, u32 pasid, unsigned long cr3); diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 763efee81524..76e69c510db8 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -86,7 +86,6 @@ struct iommu_cmd { struct kmem_cache *amd_iommu_irq_cache; -static void update_domain(struct protection_domain *domain); static void detach_device(struct device *dev); /**************************************************************************** @@ -1313,12 +1312,12 @@ static void domain_flush_pages(struct protection_domain *domain, } /* Flush the whole IO/TLB for a given protection domain - including PDE */ -static void domain_flush_tlb_pde(struct protection_domain *domain) +void amd_iommu_domain_flush_tlb_pde(struct protection_domain *domain) { __domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 1); } -static void domain_flush_complete(struct protection_domain *domain) +void amd_iommu_domain_flush_complete(struct protection_domain *domain) { int i; @@ -1343,7 +1342,7 @@ static void domain_flush_np_cache(struct protection_domain *domain, spin_lock_irqsave(&domain->lock, flags); domain_flush_pages(domain, iova, size); - domain_flush_complete(domain); + amd_iommu_domain_flush_complete(domain); spin_unlock_irqrestore(&domain->lock, flags); } } @@ -1500,7 +1499,7 @@ static bool increase_address_space(struct protection_domain *domain, pgtable.root = pte; pgtable.mode += 1; amd_iommu_update_and_flush_device_table(domain); - domain_flush_complete(domain); + amd_iommu_domain_flush_complete(domain); /* * Device Table needs to be updated and flushed before the new root can @@ -1753,8 +1752,8 @@ out: * Updates and flushing already happened in * increase_address_space(). */ - domain_flush_tlb_pde(dom); - domain_flush_complete(dom); + amd_iommu_domain_flush_tlb_pde(dom); + amd_iommu_domain_flush_complete(dom); spin_unlock_irqrestore(&dom->lock, flags); } @@ -1997,10 +1996,10 @@ static void do_detach(struct iommu_dev_data *dev_data) device_flush_dte(dev_data); /* Flush IOTLB */ - domain_flush_tlb_pde(domain); + amd_iommu_domain_flush_tlb_pde(domain); /* Wait for the flushes to finish */ - domain_flush_complete(domain); + amd_iommu_domain_flush_complete(domain); /* decrease reference counters - needs to happen after the flushes */ domain->dev_iommu[iommu->index] -= 1; @@ -2133,9 +2132,9 @@ skip_ats_check: * left the caches in the IOMMU dirty. So we have to flush * here to evict all dirty stuff. */ - domain_flush_tlb_pde(domain); + amd_iommu_domain_flush_tlb_pde(domain); - domain_flush_complete(domain); + amd_iommu_domain_flush_complete(domain); out: spin_unlock(&dev_data->lock); @@ -2297,7 +2296,7 @@ void amd_iommu_update_and_flush_device_table(struct protection_domain *domain) domain_flush_devices(domain); } -static void update_domain(struct protection_domain *domain) +void amd_iommu_domain_update(struct protection_domain *domain) { struct domain_pgtable pgtable; @@ -2306,8 +2305,8 @@ static void update_domain(struct protection_domain *domain) amd_iommu_update_and_flush_device_table(domain); /* Flush domain TLB(s) and wait for completion */ - domain_flush_tlb_pde(domain); - domain_flush_complete(domain); + amd_iommu_domain_flush_tlb_pde(domain); + amd_iommu_domain_flush_complete(domain); } int __init amd_iommu_init_api(void) @@ -2695,8 +2694,8 @@ static void amd_iommu_flush_iotlb_all(struct iommu_domain *domain) unsigned long flags; spin_lock_irqsave(&dom->lock, flags); - domain_flush_tlb_pde(dom); - domain_flush_complete(dom); + amd_iommu_domain_flush_tlb_pde(dom); + amd_iommu_domain_flush_complete(dom); spin_unlock_irqrestore(&dom->lock, flags); } @@ -2786,7 +2785,7 @@ void amd_iommu_domain_direct_map(struct iommu_domain *dom) amd_iommu_domain_clr_pt_root(domain); /* Make changes visible to IOMMUs */ - update_domain(domain); + amd_iommu_domain_update(domain); /* Page-table is not visible to IOMMU anymore, so free it */ free_pagetable(&pgtable); @@ -2830,7 +2829,7 @@ int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids) domain->glx = levels; domain->flags |= PD_IOMMUV2_MASK; - update_domain(domain); + amd_iommu_domain_update(domain); ret = 0; @@ -2867,7 +2866,7 @@ static int __flush_pasid(struct protection_domain *domain, u32 pasid, } /* Wait until IOMMU TLB flushes are complete */ - domain_flush_complete(domain); + amd_iommu_domain_flush_complete(domain); /* Now flush device TLBs */ list_for_each_entry(dev_data, &domain->dev_list, list) { @@ -2893,7 +2892,7 @@ static int __flush_pasid(struct protection_domain *domain, u32 pasid, } /* Wait until all device TLBs are flushed */ - domain_flush_complete(domain); + amd_iommu_domain_flush_complete(domain); ret = 0; -- cgit v1.2.3 From 18954252a1d0b12e1b77087b55c37fb43b09e12a Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Tue, 15 Dec 2020 01:36:58 -0600 Subject: iommu/amd: Move IO page table related functions Preparing to migrate to use IO page table framework. There is no functional change. Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20201215073705.123786-7-suravee.suthikulpanit@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 18 ++ drivers/iommu/amd/io_pgtable.c | 473 ++++++++++++++++++++++++++++++++++++++++ drivers/iommu/amd/iommu.c | 476 +---------------------------------------- 3 files changed, 493 insertions(+), 474 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index bf29ab8c99f0..1bad42a3c73c 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -131,4 +131,22 @@ void amd_iommu_apply_ivrs_quirks(void); static inline void amd_iommu_apply_ivrs_quirks(void) { } #endif +/* TODO: These are temporary and will be removed once fully transition */ +extern void free_pagetable(struct domain_pgtable *pgtable); +extern int iommu_map_page(struct protection_domain *dom, + unsigned long bus_addr, + unsigned long phys_addr, + unsigned long page_size, + int prot, + gfp_t gfp); +extern unsigned long iommu_unmap_page(struct protection_domain *dom, + unsigned long bus_addr, + unsigned long page_size); +extern u64 *fetch_pte(struct protection_domain *domain, + unsigned long address, + unsigned long *page_size); +extern void amd_iommu_domain_get_pgtable(struct protection_domain *domain, + struct domain_pgtable *pgtable); +extern void amd_iommu_domain_set_pgtable(struct protection_domain *domain, + u64 *root, int mode); #endif diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c index cf2cd2d2e43e..dbc0ad775d5e 100644 --- a/drivers/iommu/amd/io_pgtable.c +++ b/drivers/iommu/amd/io_pgtable.c @@ -44,6 +44,479 @@ static const struct iommu_flush_ops v1_flush_ops = { .tlb_add_page = v1_tlb_add_page, }; +/* + * Helper function to get the first pte of a large mapping + */ +static u64 *first_pte_l7(u64 *pte, unsigned long *page_size, + unsigned long *count) +{ + unsigned long pte_mask, pg_size, cnt; + u64 *fpte; + + pg_size = PTE_PAGE_SIZE(*pte); + cnt = PAGE_SIZE_PTE_COUNT(pg_size); + pte_mask = ~((cnt << 3) - 1); + fpte = (u64 *)(((unsigned long)pte) & pte_mask); + + if (page_size) + *page_size = pg_size; + + if (count) + *count = cnt; + + return fpte; +} + +/**************************************************************************** + * + * The functions below are used the create the page table mappings for + * unity mapped regions. + * + ****************************************************************************/ + +static void free_page_list(struct page *freelist) +{ + while (freelist != NULL) { + unsigned long p = (unsigned long)page_address(freelist); + + freelist = freelist->freelist; + free_page(p); + } +} + +static struct page *free_pt_page(unsigned long pt, struct page *freelist) +{ + struct page *p = virt_to_page((void *)pt); + + p->freelist = freelist; + + return p; +} + +#define DEFINE_FREE_PT_FN(LVL, FN) \ +static struct page *free_pt_##LVL (unsigned long __pt, struct page *freelist) \ +{ \ + unsigned long p; \ + u64 *pt; \ + int i; \ + \ + pt = (u64 *)__pt; \ + \ + for (i = 0; i < 512; ++i) { \ + /* PTE present? */ \ + if (!IOMMU_PTE_PRESENT(pt[i])) \ + continue; \ + \ + /* Large PTE? */ \ + if (PM_PTE_LEVEL(pt[i]) == 0 || \ + PM_PTE_LEVEL(pt[i]) == 7) \ + continue; \ + \ + p = (unsigned long)IOMMU_PTE_PAGE(pt[i]); \ + freelist = FN(p, freelist); \ + } \ + \ + return free_pt_page((unsigned long)pt, freelist); \ +} + +DEFINE_FREE_PT_FN(l2, free_pt_page) +DEFINE_FREE_PT_FN(l3, free_pt_l2) +DEFINE_FREE_PT_FN(l4, free_pt_l3) +DEFINE_FREE_PT_FN(l5, free_pt_l4) +DEFINE_FREE_PT_FN(l6, free_pt_l5) + +static struct page *free_sub_pt(unsigned long root, int mode, + struct page *freelist) +{ + switch (mode) { + case PAGE_MODE_NONE: + case PAGE_MODE_7_LEVEL: + break; + case PAGE_MODE_1_LEVEL: + freelist = free_pt_page(root, freelist); + break; + case PAGE_MODE_2_LEVEL: + freelist = free_pt_l2(root, freelist); + break; + case PAGE_MODE_3_LEVEL: + freelist = free_pt_l3(root, freelist); + break; + case PAGE_MODE_4_LEVEL: + freelist = free_pt_l4(root, freelist); + break; + case PAGE_MODE_5_LEVEL: + freelist = free_pt_l5(root, freelist); + break; + case PAGE_MODE_6_LEVEL: + freelist = free_pt_l6(root, freelist); + break; + default: + BUG(); + } + + return freelist; +} + +void free_pagetable(struct domain_pgtable *pgtable) +{ + struct page *freelist = NULL; + unsigned long root; + + if (pgtable->mode == PAGE_MODE_NONE) + return; + + BUG_ON(pgtable->mode < PAGE_MODE_NONE || + pgtable->mode > PAGE_MODE_6_LEVEL); + + root = (unsigned long)pgtable->root; + freelist = free_sub_pt(root, pgtable->mode, freelist); + + free_page_list(freelist); +} + +void amd_iommu_domain_set_pgtable(struct protection_domain *domain, + u64 *root, int mode) +{ + u64 pt_root; + + /* lowest 3 bits encode pgtable mode */ + pt_root = mode & 7; + pt_root |= (u64)root; + + amd_iommu_domain_set_pt_root(domain, pt_root); +} + +/* + * This function is used to add another level to an IO page table. Adding + * another level increases the size of the address space by 9 bits to a size up + * to 64 bits. + */ +static bool increase_address_space(struct protection_domain *domain, + unsigned long address, + gfp_t gfp) +{ + struct domain_pgtable pgtable; + unsigned long flags; + bool ret = true; + u64 *pte; + + spin_lock_irqsave(&domain->lock, flags); + + amd_iommu_domain_get_pgtable(domain, &pgtable); + + if (address <= PM_LEVEL_SIZE(pgtable.mode)) + goto out; + + ret = false; + if (WARN_ON_ONCE(pgtable.mode == PAGE_MODE_6_LEVEL)) + goto out; + + pte = (void *)get_zeroed_page(gfp); + if (!pte) + goto out; + + *pte = PM_LEVEL_PDE(pgtable.mode, iommu_virt_to_phys(pgtable.root)); + + pgtable.root = pte; + pgtable.mode += 1; + amd_iommu_update_and_flush_device_table(domain); + amd_iommu_domain_flush_complete(domain); + + /* + * Device Table needs to be updated and flushed before the new root can + * be published. + */ + amd_iommu_domain_set_pgtable(domain, pte, pgtable.mode); + + ret = true; + +out: + spin_unlock_irqrestore(&domain->lock, flags); + + return ret; +} + +static u64 *alloc_pte(struct protection_domain *domain, + unsigned long address, + unsigned long page_size, + u64 **pte_page, + gfp_t gfp, + bool *updated) +{ + struct domain_pgtable pgtable; + int level, end_lvl; + u64 *pte, *page; + + BUG_ON(!is_power_of_2(page_size)); + + amd_iommu_domain_get_pgtable(domain, &pgtable); + + while (address > PM_LEVEL_SIZE(pgtable.mode)) { + /* + * Return an error if there is no memory to update the + * page-table. + */ + if (!increase_address_space(domain, address, gfp)) + return NULL; + + /* Read new values to check if update was successful */ + amd_iommu_domain_get_pgtable(domain, &pgtable); + } + + + level = pgtable.mode - 1; + pte = &pgtable.root[PM_LEVEL_INDEX(level, address)]; + address = PAGE_SIZE_ALIGN(address, page_size); + end_lvl = PAGE_SIZE_LEVEL(page_size); + + while (level > end_lvl) { + u64 __pte, __npte; + int pte_level; + + __pte = *pte; + pte_level = PM_PTE_LEVEL(__pte); + + /* + * If we replace a series of large PTEs, we need + * to tear down all of them. + */ + if (IOMMU_PTE_PRESENT(__pte) && + pte_level == PAGE_MODE_7_LEVEL) { + unsigned long count, i; + u64 *lpte; + + lpte = first_pte_l7(pte, NULL, &count); + + /* + * Unmap the replicated PTEs that still match the + * original large mapping + */ + for (i = 0; i < count; ++i) + cmpxchg64(&lpte[i], __pte, 0ULL); + + *updated = true; + continue; + } + + if (!IOMMU_PTE_PRESENT(__pte) || + pte_level == PAGE_MODE_NONE) { + page = (u64 *)get_zeroed_page(gfp); + + if (!page) + return NULL; + + __npte = PM_LEVEL_PDE(level, iommu_virt_to_phys(page)); + + /* pte could have been changed somewhere. */ + if (cmpxchg64(pte, __pte, __npte) != __pte) + free_page((unsigned long)page); + else if (IOMMU_PTE_PRESENT(__pte)) + *updated = true; + + continue; + } + + /* No level skipping support yet */ + if (pte_level != level) + return NULL; + + level -= 1; + + pte = IOMMU_PTE_PAGE(__pte); + + if (pte_page && level == end_lvl) + *pte_page = pte; + + pte = &pte[PM_LEVEL_INDEX(level, address)]; + } + + return pte; +} + +/* + * This function checks if there is a PTE for a given dma address. If + * there is one, it returns the pointer to it. + */ +u64 *fetch_pte(struct protection_domain *domain, + unsigned long address, + unsigned long *page_size) +{ + struct domain_pgtable pgtable; + int level; + u64 *pte; + + *page_size = 0; + + amd_iommu_domain_get_pgtable(domain, &pgtable); + + if (address > PM_LEVEL_SIZE(pgtable.mode)) + return NULL; + + level = pgtable.mode - 1; + pte = &pgtable.root[PM_LEVEL_INDEX(level, address)]; + *page_size = PTE_LEVEL_PAGE_SIZE(level); + + while (level > 0) { + + /* Not Present */ + if (!IOMMU_PTE_PRESENT(*pte)) + return NULL; + + /* Large PTE */ + if (PM_PTE_LEVEL(*pte) == 7 || + PM_PTE_LEVEL(*pte) == 0) + break; + + /* No level skipping support yet */ + if (PM_PTE_LEVEL(*pte) != level) + return NULL; + + level -= 1; + + /* Walk to the next level */ + pte = IOMMU_PTE_PAGE(*pte); + pte = &pte[PM_LEVEL_INDEX(level, address)]; + *page_size = PTE_LEVEL_PAGE_SIZE(level); + } + + /* + * If we have a series of large PTEs, make + * sure to return a pointer to the first one. + */ + if (PM_PTE_LEVEL(*pte) == PAGE_MODE_7_LEVEL) + pte = first_pte_l7(pte, page_size, NULL); + + return pte; +} + +static struct page *free_clear_pte(u64 *pte, u64 pteval, struct page *freelist) +{ + unsigned long pt; + int mode; + + while (cmpxchg64(pte, pteval, 0) != pteval) { + pr_warn("AMD-Vi: IOMMU pte changed since we read it\n"); + pteval = *pte; + } + + if (!IOMMU_PTE_PRESENT(pteval)) + return freelist; + + pt = (unsigned long)IOMMU_PTE_PAGE(pteval); + mode = IOMMU_PTE_MODE(pteval); + + return free_sub_pt(pt, mode, freelist); +} + +/* + * Generic mapping functions. It maps a physical address into a DMA + * address space. It allocates the page table pages if necessary. + * In the future it can be extended to a generic mapping function + * supporting all features of AMD IOMMU page tables like level skipping + * and full 64 bit address spaces. + */ +int iommu_map_page(struct protection_domain *dom, + unsigned long bus_addr, + unsigned long phys_addr, + unsigned long page_size, + int prot, + gfp_t gfp) +{ + struct page *freelist = NULL; + bool updated = false; + u64 __pte, *pte; + int ret, i, count; + + BUG_ON(!IS_ALIGNED(bus_addr, page_size)); + BUG_ON(!IS_ALIGNED(phys_addr, page_size)); + + ret = -EINVAL; + if (!(prot & IOMMU_PROT_MASK)) + goto out; + + count = PAGE_SIZE_PTE_COUNT(page_size); + pte = alloc_pte(dom, bus_addr, page_size, NULL, gfp, &updated); + + ret = -ENOMEM; + if (!pte) + goto out; + + for (i = 0; i < count; ++i) + freelist = free_clear_pte(&pte[i], pte[i], freelist); + + if (freelist != NULL) + updated = true; + + if (count > 1) { + __pte = PAGE_SIZE_PTE(__sme_set(phys_addr), page_size); + __pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_PR | IOMMU_PTE_FC; + } else + __pte = __sme_set(phys_addr) | IOMMU_PTE_PR | IOMMU_PTE_FC; + + if (prot & IOMMU_PROT_IR) + __pte |= IOMMU_PTE_IR; + if (prot & IOMMU_PROT_IW) + __pte |= IOMMU_PTE_IW; + + for (i = 0; i < count; ++i) + pte[i] = __pte; + + ret = 0; + +out: + if (updated) { + unsigned long flags; + + spin_lock_irqsave(&dom->lock, flags); + /* + * Flush domain TLB(s) and wait for completion. Any Device-Table + * Updates and flushing already happened in + * increase_address_space(). + */ + amd_iommu_domain_flush_tlb_pde(dom); + amd_iommu_domain_flush_complete(dom); + spin_unlock_irqrestore(&dom->lock, flags); + } + + /* Everything flushed out, free pages now */ + free_page_list(freelist); + + return ret; +} + +unsigned long iommu_unmap_page(struct protection_domain *dom, + unsigned long bus_addr, + unsigned long page_size) +{ + unsigned long long unmapped; + unsigned long unmap_size; + u64 *pte; + + BUG_ON(!is_power_of_2(page_size)); + + unmapped = 0; + + while (unmapped < page_size) { + + pte = fetch_pte(dom, bus_addr, &unmap_size); + + if (pte) { + int i, count; + + count = PAGE_SIZE_PTE_COUNT(unmap_size); + for (i = 0; i < count; i++) + pte[i] = 0ULL; + } + + bus_addr = (bus_addr & ~(unmap_size - 1)) + unmap_size; + unmapped += unmap_size; + } + + BUG_ON(unmapped && !is_power_of_2(unmapped)); + + return unmapped; +} + /* * ---------------------------------------------------- */ diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 76e69c510db8..eda8e196fb14 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -138,8 +138,8 @@ static struct protection_domain *to_pdomain(struct iommu_domain *dom) return container_of(dom, struct protection_domain, domain); } -static void amd_iommu_domain_get_pgtable(struct protection_domain *domain, - struct domain_pgtable *pgtable) +void amd_iommu_domain_get_pgtable(struct protection_domain *domain, + struct domain_pgtable *pgtable) { u64 pt_root = atomic64_read(&domain->iop.pt_root); @@ -147,18 +147,6 @@ static void amd_iommu_domain_get_pgtable(struct protection_domain *domain, pgtable->mode = pt_root & 7; /* lowest 3 bits encode pgtable mode */ } -static void amd_iommu_domain_set_pgtable(struct protection_domain *domain, - u64 *root, int mode) -{ - u64 pt_root; - - /* lowest 3 bits encode pgtable mode */ - pt_root = mode & 7; - pt_root |= (u64)root; - - amd_iommu_domain_set_pt_root(domain, pt_root); -} - static struct iommu_dev_data *alloc_dev_data(u16 devid) { struct iommu_dev_data *dev_data; @@ -414,29 +402,6 @@ static void amd_iommu_uninit_device(struct device *dev) */ } -/* - * Helper function to get the first pte of a large mapping - */ -static u64 *first_pte_l7(u64 *pte, unsigned long *page_size, - unsigned long *count) -{ - unsigned long pte_mask, pg_size, cnt; - u64 *fpte; - - pg_size = PTE_PAGE_SIZE(*pte); - cnt = PAGE_SIZE_PTE_COUNT(pg_size); - pte_mask = ~((cnt << 3) - 1); - fpte = (u64 *)(((unsigned long)pte) & pte_mask); - - if (page_size) - *page_size = pg_size; - - if (count) - *count = cnt; - - return fpte; -} - /**************************************************************************** * * Interrupt handling functions @@ -1359,443 +1324,6 @@ static void domain_flush_devices(struct protection_domain *domain) device_flush_dte(dev_data); } -/**************************************************************************** - * - * The functions below are used the create the page table mappings for - * unity mapped regions. - * - ****************************************************************************/ - -static void free_page_list(struct page *freelist) -{ - while (freelist != NULL) { - unsigned long p = (unsigned long)page_address(freelist); - freelist = freelist->freelist; - free_page(p); - } -} - -static struct page *free_pt_page(unsigned long pt, struct page *freelist) -{ - struct page *p = virt_to_page((void *)pt); - - p->freelist = freelist; - - return p; -} - -#define DEFINE_FREE_PT_FN(LVL, FN) \ -static struct page *free_pt_##LVL (unsigned long __pt, struct page *freelist) \ -{ \ - unsigned long p; \ - u64 *pt; \ - int i; \ - \ - pt = (u64 *)__pt; \ - \ - for (i = 0; i < 512; ++i) { \ - /* PTE present? */ \ - if (!IOMMU_PTE_PRESENT(pt[i])) \ - continue; \ - \ - /* Large PTE? */ \ - if (PM_PTE_LEVEL(pt[i]) == 0 || \ - PM_PTE_LEVEL(pt[i]) == 7) \ - continue; \ - \ - p = (unsigned long)IOMMU_PTE_PAGE(pt[i]); \ - freelist = FN(p, freelist); \ - } \ - \ - return free_pt_page((unsigned long)pt, freelist); \ -} - -DEFINE_FREE_PT_FN(l2, free_pt_page) -DEFINE_FREE_PT_FN(l3, free_pt_l2) -DEFINE_FREE_PT_FN(l4, free_pt_l3) -DEFINE_FREE_PT_FN(l5, free_pt_l4) -DEFINE_FREE_PT_FN(l6, free_pt_l5) - -static struct page *free_sub_pt(unsigned long root, int mode, - struct page *freelist) -{ - switch (mode) { - case PAGE_MODE_NONE: - case PAGE_MODE_7_LEVEL: - break; - case PAGE_MODE_1_LEVEL: - freelist = free_pt_page(root, freelist); - break; - case PAGE_MODE_2_LEVEL: - freelist = free_pt_l2(root, freelist); - break; - case PAGE_MODE_3_LEVEL: - freelist = free_pt_l3(root, freelist); - break; - case PAGE_MODE_4_LEVEL: - freelist = free_pt_l4(root, freelist); - break; - case PAGE_MODE_5_LEVEL: - freelist = free_pt_l5(root, freelist); - break; - case PAGE_MODE_6_LEVEL: - freelist = free_pt_l6(root, freelist); - break; - default: - BUG(); - } - - return freelist; -} - -static void free_pagetable(struct domain_pgtable *pgtable) -{ - struct page *freelist = NULL; - unsigned long root; - - if (pgtable->mode == PAGE_MODE_NONE) - return; - - BUG_ON(pgtable->mode < PAGE_MODE_NONE || - pgtable->mode > PAGE_MODE_6_LEVEL); - - root = (unsigned long)pgtable->root; - freelist = free_sub_pt(root, pgtable->mode, freelist); - - free_page_list(freelist); -} - -/* - * This function is used to add another level to an IO page table. Adding - * another level increases the size of the address space by 9 bits to a size up - * to 64 bits. - */ -static bool increase_address_space(struct protection_domain *domain, - unsigned long address, - gfp_t gfp) -{ - struct domain_pgtable pgtable; - unsigned long flags; - bool ret = true; - u64 *pte; - - spin_lock_irqsave(&domain->lock, flags); - - amd_iommu_domain_get_pgtable(domain, &pgtable); - - if (address <= PM_LEVEL_SIZE(pgtable.mode)) - goto out; - - ret = false; - if (WARN_ON_ONCE(pgtable.mode == PAGE_MODE_6_LEVEL)) - goto out; - - pte = (void *)get_zeroed_page(gfp); - if (!pte) - goto out; - - *pte = PM_LEVEL_PDE(pgtable.mode, iommu_virt_to_phys(pgtable.root)); - - pgtable.root = pte; - pgtable.mode += 1; - amd_iommu_update_and_flush_device_table(domain); - amd_iommu_domain_flush_complete(domain); - - /* - * Device Table needs to be updated and flushed before the new root can - * be published. - */ - amd_iommu_domain_set_pgtable(domain, pte, pgtable.mode); - - ret = true; - -out: - spin_unlock_irqrestore(&domain->lock, flags); - - return ret; -} - -static u64 *alloc_pte(struct protection_domain *domain, - unsigned long address, - unsigned long page_size, - u64 **pte_page, - gfp_t gfp, - bool *updated) -{ - struct domain_pgtable pgtable; - int level, end_lvl; - u64 *pte, *page; - - BUG_ON(!is_power_of_2(page_size)); - - amd_iommu_domain_get_pgtable(domain, &pgtable); - - while (address > PM_LEVEL_SIZE(pgtable.mode)) { - /* - * Return an error if there is no memory to update the - * page-table. - */ - if (!increase_address_space(domain, address, gfp)) - return NULL; - - /* Read new values to check if update was successful */ - amd_iommu_domain_get_pgtable(domain, &pgtable); - } - - - level = pgtable.mode - 1; - pte = &pgtable.root[PM_LEVEL_INDEX(level, address)]; - address = PAGE_SIZE_ALIGN(address, page_size); - end_lvl = PAGE_SIZE_LEVEL(page_size); - - while (level > end_lvl) { - u64 __pte, __npte; - int pte_level; - - __pte = *pte; - pte_level = PM_PTE_LEVEL(__pte); - - /* - * If we replace a series of large PTEs, we need - * to tear down all of them. - */ - if (IOMMU_PTE_PRESENT(__pte) && - pte_level == PAGE_MODE_7_LEVEL) { - unsigned long count, i; - u64 *lpte; - - lpte = first_pte_l7(pte, NULL, &count); - - /* - * Unmap the replicated PTEs that still match the - * original large mapping - */ - for (i = 0; i < count; ++i) - cmpxchg64(&lpte[i], __pte, 0ULL); - - *updated = true; - continue; - } - - if (!IOMMU_PTE_PRESENT(__pte) || - pte_level == PAGE_MODE_NONE) { - page = (u64 *)get_zeroed_page(gfp); - - if (!page) - return NULL; - - __npte = PM_LEVEL_PDE(level, iommu_virt_to_phys(page)); - - /* pte could have been changed somewhere. */ - if (cmpxchg64(pte, __pte, __npte) != __pte) - free_page((unsigned long)page); - else if (IOMMU_PTE_PRESENT(__pte)) - *updated = true; - - continue; - } - - /* No level skipping support yet */ - if (pte_level != level) - return NULL; - - level -= 1; - - pte = IOMMU_PTE_PAGE(__pte); - - if (pte_page && level == end_lvl) - *pte_page = pte; - - pte = &pte[PM_LEVEL_INDEX(level, address)]; - } - - return pte; -} - -/* - * This function checks if there is a PTE for a given dma address. If - * there is one, it returns the pointer to it. - */ -static u64 *fetch_pte(struct protection_domain *domain, - unsigned long address, - unsigned long *page_size) -{ - struct domain_pgtable pgtable; - int level; - u64 *pte; - - *page_size = 0; - - amd_iommu_domain_get_pgtable(domain, &pgtable); - - if (address > PM_LEVEL_SIZE(pgtable.mode)) - return NULL; - - level = pgtable.mode - 1; - pte = &pgtable.root[PM_LEVEL_INDEX(level, address)]; - *page_size = PTE_LEVEL_PAGE_SIZE(level); - - while (level > 0) { - - /* Not Present */ - if (!IOMMU_PTE_PRESENT(*pte)) - return NULL; - - /* Large PTE */ - if (PM_PTE_LEVEL(*pte) == 7 || - PM_PTE_LEVEL(*pte) == 0) - break; - - /* No level skipping support yet */ - if (PM_PTE_LEVEL(*pte) != level) - return NULL; - - level -= 1; - - /* Walk to the next level */ - pte = IOMMU_PTE_PAGE(*pte); - pte = &pte[PM_LEVEL_INDEX(level, address)]; - *page_size = PTE_LEVEL_PAGE_SIZE(level); - } - - /* - * If we have a series of large PTEs, make - * sure to return a pointer to the first one. - */ - if (PM_PTE_LEVEL(*pte) == PAGE_MODE_7_LEVEL) - pte = first_pte_l7(pte, page_size, NULL); - - return pte; -} - -static struct page *free_clear_pte(u64 *pte, u64 pteval, struct page *freelist) -{ - unsigned long pt; - int mode; - - while (cmpxchg64(pte, pteval, 0) != pteval) { - pr_warn("AMD-Vi: IOMMU pte changed since we read it\n"); - pteval = *pte; - } - - if (!IOMMU_PTE_PRESENT(pteval)) - return freelist; - - pt = (unsigned long)IOMMU_PTE_PAGE(pteval); - mode = IOMMU_PTE_MODE(pteval); - - return free_sub_pt(pt, mode, freelist); -} - -/* - * Generic mapping functions. It maps a physical address into a DMA - * address space. It allocates the page table pages if necessary. - * In the future it can be extended to a generic mapping function - * supporting all features of AMD IOMMU page tables like level skipping - * and full 64 bit address spaces. - */ -static int iommu_map_page(struct protection_domain *dom, - unsigned long bus_addr, - unsigned long phys_addr, - unsigned long page_size, - int prot, - gfp_t gfp) -{ - struct page *freelist = NULL; - bool updated = false; - u64 __pte, *pte; - int ret, i, count; - - BUG_ON(!IS_ALIGNED(bus_addr, page_size)); - BUG_ON(!IS_ALIGNED(phys_addr, page_size)); - - ret = -EINVAL; - if (!(prot & IOMMU_PROT_MASK)) - goto out; - - count = PAGE_SIZE_PTE_COUNT(page_size); - pte = alloc_pte(dom, bus_addr, page_size, NULL, gfp, &updated); - - ret = -ENOMEM; - if (!pte) - goto out; - - for (i = 0; i < count; ++i) - freelist = free_clear_pte(&pte[i], pte[i], freelist); - - if (freelist != NULL) - updated = true; - - if (count > 1) { - __pte = PAGE_SIZE_PTE(__sme_set(phys_addr), page_size); - __pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_PR | IOMMU_PTE_FC; - } else - __pte = __sme_set(phys_addr) | IOMMU_PTE_PR | IOMMU_PTE_FC; - - if (prot & IOMMU_PROT_IR) - __pte |= IOMMU_PTE_IR; - if (prot & IOMMU_PROT_IW) - __pte |= IOMMU_PTE_IW; - - for (i = 0; i < count; ++i) - pte[i] = __pte; - - ret = 0; - -out: - if (updated) { - unsigned long flags; - - spin_lock_irqsave(&dom->lock, flags); - /* - * Flush domain TLB(s) and wait for completion. Any Device-Table - * Updates and flushing already happened in - * increase_address_space(). - */ - amd_iommu_domain_flush_tlb_pde(dom); - amd_iommu_domain_flush_complete(dom); - spin_unlock_irqrestore(&dom->lock, flags); - } - - /* Everything flushed out, free pages now */ - free_page_list(freelist); - - return ret; -} - -static unsigned long iommu_unmap_page(struct protection_domain *dom, - unsigned long bus_addr, - unsigned long page_size) -{ - unsigned long long unmapped; - unsigned long unmap_size; - u64 *pte; - - BUG_ON(!is_power_of_2(page_size)); - - unmapped = 0; - - while (unmapped < page_size) { - - pte = fetch_pte(dom, bus_addr, &unmap_size); - - if (pte) { - int i, count; - - count = PAGE_SIZE_PTE_COUNT(unmap_size); - for (i = 0; i < count; i++) - pte[i] = 0ULL; - } - - bus_addr = (bus_addr & ~(unmap_size - 1)) + unmap_size; - unmapped += unmap_size; - } - - BUG_ON(unmapped && !is_power_of_2(unmapped)); - - return unmapped; -} - /**************************************************************************** * * The next functions belong to the domain allocation. A domain is -- cgit v1.2.3 From e42ba0633064ef23eb1c8c21edf96bac1541bd4b Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Tue, 15 Dec 2020 01:36:59 -0600 Subject: iommu/amd: Restructure code for freeing page table By consolidate logic into v1_free_pgtable helper function, which is called from IO page table framework. Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20201215073705.123786-8-suravee.suthikulpanit@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 1 - drivers/iommu/amd/io_pgtable.c | 41 ++++++++++++++++++++++++----------------- drivers/iommu/amd/iommu.c | 21 ++++----------------- 3 files changed, 28 insertions(+), 35 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 1bad42a3c73c..91d098003f12 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -132,7 +132,6 @@ static inline void amd_iommu_apply_ivrs_quirks(void) { } #endif /* TODO: These are temporary and will be removed once fully transition */ -extern void free_pagetable(struct domain_pgtable *pgtable); extern int iommu_map_page(struct protection_domain *dom, unsigned long bus_addr, unsigned long phys_addr, diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c index dbc0ad775d5e..a8fea06efb4f 100644 --- a/drivers/iommu/amd/io_pgtable.c +++ b/drivers/iommu/amd/io_pgtable.c @@ -157,23 +157,6 @@ static struct page *free_sub_pt(unsigned long root, int mode, return freelist; } -void free_pagetable(struct domain_pgtable *pgtable) -{ - struct page *freelist = NULL; - unsigned long root; - - if (pgtable->mode == PAGE_MODE_NONE) - return; - - BUG_ON(pgtable->mode < PAGE_MODE_NONE || - pgtable->mode > PAGE_MODE_6_LEVEL); - - root = (unsigned long)pgtable->root; - freelist = free_sub_pt(root, pgtable->mode, freelist); - - free_page_list(freelist); -} - void amd_iommu_domain_set_pgtable(struct protection_domain *domain, u64 *root, int mode) { @@ -522,6 +505,30 @@ unsigned long iommu_unmap_page(struct protection_domain *dom, */ static void v1_free_pgtable(struct io_pgtable *iop) { + struct amd_io_pgtable *pgtable = container_of(iop, struct amd_io_pgtable, iop); + struct protection_domain *dom; + struct page *freelist = NULL; + unsigned long root; + + if (pgtable->mode == PAGE_MODE_NONE) + return; + + dom = container_of(pgtable, struct protection_domain, iop); + + /* Update data structure */ + amd_iommu_domain_clr_pt_root(dom); + + /* Make changes visible to IOMMUs */ + amd_iommu_domain_update(dom); + + /* Page-table is not visible to IOMMU anymore, so free it */ + BUG_ON(pgtable->mode < PAGE_MODE_NONE || + pgtable->mode > PAGE_MODE_6_LEVEL); + + root = (unsigned long)pgtable->root; + freelist = free_sub_pt(root, pgtable->mode, freelist); + + free_page_list(freelist); } static struct io_pgtable *v1_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index eda8e196fb14..d02aeb17fd81 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1902,17 +1902,14 @@ static void cleanup_domain(struct protection_domain *domain) static void protection_domain_free(struct protection_domain *domain) { - struct domain_pgtable pgtable; - if (!domain) return; if (domain->id) domain_id_free(domain->id); - amd_iommu_domain_get_pgtable(domain, &pgtable); - amd_iommu_domain_clr_pt_root(domain); - free_pagetable(&pgtable); + if (domain->iop.pgtbl_cfg.tlb) + free_io_pgtable_ops(&domain->iop.iop.ops); kfree(domain); } @@ -2301,22 +2298,12 @@ EXPORT_SYMBOL(amd_iommu_unregister_ppr_notifier); void amd_iommu_domain_direct_map(struct iommu_domain *dom) { struct protection_domain *domain = to_pdomain(dom); - struct domain_pgtable pgtable; unsigned long flags; spin_lock_irqsave(&domain->lock, flags); - /* First save pgtable configuration*/ - amd_iommu_domain_get_pgtable(domain, &pgtable); - - /* Remove page-table from domain */ - amd_iommu_domain_clr_pt_root(domain); - - /* Make changes visible to IOMMUs */ - amd_iommu_domain_update(domain); - - /* Page-table is not visible to IOMMU anymore, so free it */ - free_pagetable(&pgtable); + if (domain->iop.pgtbl_cfg.tlb) + free_io_pgtable_ops(&domain->iop.iop.ops); spin_unlock_irqrestore(&domain->lock, flags); } -- cgit v1.2.3 From 6eedb59c18a3be2b670e0adc68e46d22ebd42823 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Tue, 15 Dec 2020 01:37:00 -0600 Subject: iommu/amd: Remove amd_iommu_domain_get_pgtable Since the IO page table root and mode parameters have been moved into the struct amd_io_pg, the function is no longer needed. Therefore, remove it along with the struct domain_pgtable. Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20201215073705.123786-9-suravee.suthikulpanit@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 4 ++-- drivers/iommu/amd/amd_iommu_types.h | 6 ------ drivers/iommu/amd/io_pgtable.c | 36 ++++++++++++------------------------ drivers/iommu/amd/iommu.c | 34 +++++----------------------------- 4 files changed, 19 insertions(+), 61 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 91d098003f12..76276d9e463c 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -110,6 +110,8 @@ static inline void amd_iommu_domain_set_pt_root(struct protection_domain *domain, u64 root) { atomic64_set(&domain->iop.pt_root, root); + domain->iop.root = (u64 *)(root & PAGE_MASK); + domain->iop.mode = root & 7; /* lowest 3 bits encode pgtable mode */ } static inline @@ -144,8 +146,6 @@ extern unsigned long iommu_unmap_page(struct protection_domain *dom, extern u64 *fetch_pte(struct protection_domain *domain, unsigned long address, unsigned long *page_size); -extern void amd_iommu_domain_get_pgtable(struct protection_domain *domain, - struct domain_pgtable *pgtable); extern void amd_iommu_domain_set_pgtable(struct protection_domain *domain, u64 *root, int mode); #endif diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index a50f117879f2..8915c832448f 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -519,12 +519,6 @@ struct protection_domain { unsigned dev_iommu[MAX_IOMMUS]; /* per-IOMMU reference count */ }; -/* For decocded pt_root */ -struct domain_pgtable { - int mode; - u64 *root; -}; - /* * Structure where we save information about one hardware AMD IOMMU in the * system. diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c index a8fea06efb4f..bd8c65a39b1c 100644 --- a/drivers/iommu/amd/io_pgtable.c +++ b/drivers/iommu/amd/io_pgtable.c @@ -178,30 +178,27 @@ static bool increase_address_space(struct protection_domain *domain, unsigned long address, gfp_t gfp) { - struct domain_pgtable pgtable; unsigned long flags; bool ret = true; u64 *pte; spin_lock_irqsave(&domain->lock, flags); - amd_iommu_domain_get_pgtable(domain, &pgtable); - - if (address <= PM_LEVEL_SIZE(pgtable.mode)) + if (address <= PM_LEVEL_SIZE(domain->iop.mode)) goto out; ret = false; - if (WARN_ON_ONCE(pgtable.mode == PAGE_MODE_6_LEVEL)) + if (WARN_ON_ONCE(domain->iop.mode == PAGE_MODE_6_LEVEL)) goto out; pte = (void *)get_zeroed_page(gfp); if (!pte) goto out; - *pte = PM_LEVEL_PDE(pgtable.mode, iommu_virt_to_phys(pgtable.root)); + *pte = PM_LEVEL_PDE(domain->iop.mode, iommu_virt_to_phys(domain->iop.root)); - pgtable.root = pte; - pgtable.mode += 1; + domain->iop.root = pte; + domain->iop.mode += 1; amd_iommu_update_and_flush_device_table(domain); amd_iommu_domain_flush_complete(domain); @@ -209,7 +206,7 @@ static bool increase_address_space(struct protection_domain *domain, * Device Table needs to be updated and flushed before the new root can * be published. */ - amd_iommu_domain_set_pgtable(domain, pte, pgtable.mode); + amd_iommu_domain_set_pgtable(domain, pte, domain->iop.mode); ret = true; @@ -226,29 +223,23 @@ static u64 *alloc_pte(struct protection_domain *domain, gfp_t gfp, bool *updated) { - struct domain_pgtable pgtable; int level, end_lvl; u64 *pte, *page; BUG_ON(!is_power_of_2(page_size)); - amd_iommu_domain_get_pgtable(domain, &pgtable); - - while (address > PM_LEVEL_SIZE(pgtable.mode)) { + while (address > PM_LEVEL_SIZE(domain->iop.mode)) { /* * Return an error if there is no memory to update the * page-table. */ if (!increase_address_space(domain, address, gfp)) return NULL; - - /* Read new values to check if update was successful */ - amd_iommu_domain_get_pgtable(domain, &pgtable); } - level = pgtable.mode - 1; - pte = &pgtable.root[PM_LEVEL_INDEX(level, address)]; + level = domain->iop.mode - 1; + pte = &domain->iop.root[PM_LEVEL_INDEX(level, address)]; address = PAGE_SIZE_ALIGN(address, page_size); end_lvl = PAGE_SIZE_LEVEL(page_size); @@ -324,19 +315,16 @@ u64 *fetch_pte(struct protection_domain *domain, unsigned long address, unsigned long *page_size) { - struct domain_pgtable pgtable; int level; u64 *pte; *page_size = 0; - amd_iommu_domain_get_pgtable(domain, &pgtable); - - if (address > PM_LEVEL_SIZE(pgtable.mode)) + if (address > PM_LEVEL_SIZE(domain->iop.mode)) return NULL; - level = pgtable.mode - 1; - pte = &pgtable.root[PM_LEVEL_INDEX(level, address)]; + level = domain->iop.mode - 1; + pte = &domain->iop.root[PM_LEVEL_INDEX(level, address)]; *page_size = PTE_LEVEL_PAGE_SIZE(level); while (level > 0) { diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index d02aeb17fd81..bba3d1802b50 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -138,15 +138,6 @@ static struct protection_domain *to_pdomain(struct iommu_domain *dom) return container_of(dom, struct protection_domain, domain); } -void amd_iommu_domain_get_pgtable(struct protection_domain *domain, - struct domain_pgtable *pgtable) -{ - u64 pt_root = atomic64_read(&domain->iop.pt_root); - - pgtable->root = (u64 *)(pt_root & PAGE_MASK); - pgtable->mode = pt_root & 7; /* lowest 3 bits encode pgtable mode */ -} - static struct iommu_dev_data *alloc_dev_data(u16 devid) { struct iommu_dev_data *dev_data; @@ -1483,7 +1474,6 @@ static void clear_dte_entry(u16 devid) static void do_attach(struct iommu_dev_data *dev_data, struct protection_domain *domain) { - struct domain_pgtable pgtable; struct amd_iommu *iommu; bool ats; @@ -1499,7 +1489,6 @@ static void do_attach(struct iommu_dev_data *dev_data, domain->dev_cnt += 1; /* Update device table */ - amd_iommu_domain_get_pgtable(domain, &pgtable); set_dte_entry(dev_data->devid, domain, ats, dev_data->iommu_v2); clone_aliases(dev_data->pdev); @@ -1826,10 +1815,7 @@ void amd_iommu_update_and_flush_device_table(struct protection_domain *domain) void amd_iommu_domain_update(struct protection_domain *domain) { - struct domain_pgtable pgtable; - /* Update device table */ - amd_iommu_domain_get_pgtable(domain, &pgtable); amd_iommu_update_and_flush_device_table(domain); /* Flush domain TLB(s) and wait for completion */ @@ -2079,12 +2065,10 @@ static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova, gfp_t gfp) { struct protection_domain *domain = to_pdomain(dom); - struct domain_pgtable pgtable; int prot = 0; int ret; - amd_iommu_domain_get_pgtable(domain, &pgtable); - if (pgtable.mode == PAGE_MODE_NONE) + if (domain->iop.mode == PAGE_MODE_NONE) return -EINVAL; if (iommu_prot & IOMMU_READ) @@ -2104,10 +2088,8 @@ static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova, struct iommu_iotlb_gather *gather) { struct protection_domain *domain = to_pdomain(dom); - struct domain_pgtable pgtable; - amd_iommu_domain_get_pgtable(domain, &pgtable); - if (pgtable.mode == PAGE_MODE_NONE) + if (domain->iop.mode == PAGE_MODE_NONE) return 0; return iommu_unmap_page(domain, iova, page_size); @@ -2118,11 +2100,9 @@ static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom, { struct protection_domain *domain = to_pdomain(dom); unsigned long offset_mask, pte_pgsize; - struct domain_pgtable pgtable; u64 *pte, __pte; - amd_iommu_domain_get_pgtable(domain, &pgtable); - if (pgtable.mode == PAGE_MODE_NONE) + if (domain->iop.mode == PAGE_MODE_NONE) return iova; pte = fetch_pte(domain, iova, &pte_pgsize); @@ -2492,11 +2472,9 @@ static u64 *__get_gcr3_pte(u64 *root, int level, u32 pasid, bool alloc) static int __set_gcr3(struct protection_domain *domain, u32 pasid, unsigned long cr3) { - struct domain_pgtable pgtable; u64 *pte; - amd_iommu_domain_get_pgtable(domain, &pgtable); - if (pgtable.mode != PAGE_MODE_NONE) + if (domain->iop.mode != PAGE_MODE_NONE) return -EINVAL; pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, true); @@ -2510,11 +2488,9 @@ static int __set_gcr3(struct protection_domain *domain, u32 pasid, static int __clear_gcr3(struct protection_domain *domain, u32 pasid) { - struct domain_pgtable pgtable; u64 *pte; - amd_iommu_domain_get_pgtable(domain, &pgtable); - if (pgtable.mode != PAGE_MODE_NONE) + if (domain->iop.mode != PAGE_MODE_NONE) return -EINVAL; pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, false); -- cgit v1.2.3 From 33aef9786046d9a5744cd1e8d5d0ce800d611fdc Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Tue, 15 Dec 2020 01:37:01 -0600 Subject: iommu/amd: Rename variables to be consistent with struct io_pgtable_ops There is no functional change. Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20201215073705.123786-10-suravee.suthikulpanit@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/io_pgtable.c | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c index bd8c65a39b1c..af6b7f11ebc3 100644 --- a/drivers/iommu/amd/io_pgtable.c +++ b/drivers/iommu/amd/io_pgtable.c @@ -387,9 +387,9 @@ static struct page *free_clear_pte(u64 *pte, u64 pteval, struct page *freelist) * and full 64 bit address spaces. */ int iommu_map_page(struct protection_domain *dom, - unsigned long bus_addr, - unsigned long phys_addr, - unsigned long page_size, + unsigned long iova, + unsigned long paddr, + unsigned long size, int prot, gfp_t gfp) { @@ -398,15 +398,15 @@ int iommu_map_page(struct protection_domain *dom, u64 __pte, *pte; int ret, i, count; - BUG_ON(!IS_ALIGNED(bus_addr, page_size)); - BUG_ON(!IS_ALIGNED(phys_addr, page_size)); + BUG_ON(!IS_ALIGNED(iova, size)); + BUG_ON(!IS_ALIGNED(paddr, size)); ret = -EINVAL; if (!(prot & IOMMU_PROT_MASK)) goto out; - count = PAGE_SIZE_PTE_COUNT(page_size); - pte = alloc_pte(dom, bus_addr, page_size, NULL, gfp, &updated); + count = PAGE_SIZE_PTE_COUNT(size); + pte = alloc_pte(dom, iova, size, NULL, gfp, &updated); ret = -ENOMEM; if (!pte) @@ -419,10 +419,10 @@ int iommu_map_page(struct protection_domain *dom, updated = true; if (count > 1) { - __pte = PAGE_SIZE_PTE(__sme_set(phys_addr), page_size); + __pte = PAGE_SIZE_PTE(__sme_set(paddr), size); __pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_PR | IOMMU_PTE_FC; } else - __pte = __sme_set(phys_addr) | IOMMU_PTE_PR | IOMMU_PTE_FC; + __pte = __sme_set(paddr) | IOMMU_PTE_PR | IOMMU_PTE_FC; if (prot & IOMMU_PROT_IR) __pte |= IOMMU_PTE_IR; @@ -456,20 +456,19 @@ out: } unsigned long iommu_unmap_page(struct protection_domain *dom, - unsigned long bus_addr, - unsigned long page_size) + unsigned long iova, + unsigned long size) { unsigned long long unmapped; unsigned long unmap_size; u64 *pte; - BUG_ON(!is_power_of_2(page_size)); + BUG_ON(!is_power_of_2(size)); unmapped = 0; - while (unmapped < page_size) { - - pte = fetch_pte(dom, bus_addr, &unmap_size); + while (unmapped < size) { + pte = fetch_pte(dom, iova, &unmap_size); if (pte) { int i, count; @@ -479,7 +478,7 @@ unsigned long iommu_unmap_page(struct protection_domain *dom, pte[i] = 0ULL; } - bus_addr = (bus_addr & ~(unmap_size - 1)) + unmap_size; + iova = (iova & ~(unmap_size - 1)) + unmap_size; unmapped += unmap_size; } -- cgit v1.2.3 From 0633bbcc1eedb7015554254d54e14602b1d8b989 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Tue, 15 Dec 2020 01:37:02 -0600 Subject: iommu/amd: Refactor fetch_pte to use struct amd_io_pgtable To simplify the fetch_pte function. There is no functional change. Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20201215073705.123786-11-suravee.suthikulpanit@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 2 +- drivers/iommu/amd/io_pgtable.c | 13 +++++++------ drivers/iommu/amd/iommu.c | 4 +++- 3 files changed, 11 insertions(+), 8 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 76276d9e463c..83ca822c5349 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -143,7 +143,7 @@ extern int iommu_map_page(struct protection_domain *dom, extern unsigned long iommu_unmap_page(struct protection_domain *dom, unsigned long bus_addr, unsigned long page_size); -extern u64 *fetch_pte(struct protection_domain *domain, +extern u64 *fetch_pte(struct amd_io_pgtable *pgtable, unsigned long address, unsigned long *page_size); extern void amd_iommu_domain_set_pgtable(struct protection_domain *domain, diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c index af6b7f11ebc3..d7924eb20178 100644 --- a/drivers/iommu/amd/io_pgtable.c +++ b/drivers/iommu/amd/io_pgtable.c @@ -311,7 +311,7 @@ static u64 *alloc_pte(struct protection_domain *domain, * This function checks if there is a PTE for a given dma address. If * there is one, it returns the pointer to it. */ -u64 *fetch_pte(struct protection_domain *domain, +u64 *fetch_pte(struct amd_io_pgtable *pgtable, unsigned long address, unsigned long *page_size) { @@ -320,11 +320,11 @@ u64 *fetch_pte(struct protection_domain *domain, *page_size = 0; - if (address > PM_LEVEL_SIZE(domain->iop.mode)) + if (address > PM_LEVEL_SIZE(pgtable->mode)) return NULL; - level = domain->iop.mode - 1; - pte = &domain->iop.root[PM_LEVEL_INDEX(level, address)]; + level = pgtable->mode - 1; + pte = &pgtable->root[PM_LEVEL_INDEX(level, address)]; *page_size = PTE_LEVEL_PAGE_SIZE(level); while (level > 0) { @@ -459,6 +459,8 @@ unsigned long iommu_unmap_page(struct protection_domain *dom, unsigned long iova, unsigned long size) { + struct io_pgtable_ops *ops = &dom->iop.iop.ops; + struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops); unsigned long long unmapped; unsigned long unmap_size; u64 *pte; @@ -468,8 +470,7 @@ unsigned long iommu_unmap_page(struct protection_domain *dom, unmapped = 0; while (unmapped < size) { - pte = fetch_pte(dom, iova, &unmap_size); - + pte = fetch_pte(pgtable, iova, &unmap_size); if (pte) { int i, count; diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index bba3d1802b50..f1a4f535eac8 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2099,13 +2099,15 @@ static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom, dma_addr_t iova) { struct protection_domain *domain = to_pdomain(dom); + struct io_pgtable_ops *ops = &domain->iop.iop.ops; + struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops); unsigned long offset_mask, pte_pgsize; u64 *pte, __pte; if (domain->iop.mode == PAGE_MODE_NONE) return iova; - pte = fetch_pte(domain, iova, &pte_pgsize); + pte = fetch_pte(pgtable, iova, &pte_pgsize); if (!pte || !IOMMU_PTE_PRESENT(*pte)) return 0; -- cgit v1.2.3 From 441555c63aca3300a0f6cd5948dbf5bd6e7760b3 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Tue, 15 Dec 2020 01:37:03 -0600 Subject: iommu/amd: Introduce iommu_v1_iova_to_phys This implements iova_to_phys for AMD IOMMU v1 pagetable, which will be used by the IO page table framework. Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20201215073705.123786-12-suravee.suthikulpanit@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/io_pgtable.c | 22 ++++++++++++++++++++++ drivers/iommu/amd/iommu.c | 16 +--------------- 2 files changed, 23 insertions(+), 15 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c index d7924eb20178..b70eb79d05fb 100644 --- a/drivers/iommu/amd/io_pgtable.c +++ b/drivers/iommu/amd/io_pgtable.c @@ -488,6 +488,26 @@ unsigned long iommu_unmap_page(struct protection_domain *dom, return unmapped; } +static phys_addr_t iommu_v1_iova_to_phys(struct io_pgtable_ops *ops, unsigned long iova) +{ + struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops); + unsigned long offset_mask, pte_pgsize; + u64 *pte, __pte; + + if (pgtable->mode == PAGE_MODE_NONE) + return iova; + + pte = fetch_pte(pgtable, iova, &pte_pgsize); + + if (!pte || !IOMMU_PTE_PRESENT(*pte)) + return 0; + + offset_mask = pte_pgsize - 1; + __pte = __sme_clr(*pte & PM_ADDR_MASK); + + return (__pte & ~offset_mask) | (iova & offset_mask); +} + /* * ---------------------------------------------------- */ @@ -528,6 +548,8 @@ static struct io_pgtable *v1_alloc_pgtable(struct io_pgtable_cfg *cfg, void *coo cfg->oas = IOMMU_OUT_ADDR_BIT_SIZE, cfg->tlb = &v1_flush_ops; + pgtable->iop.ops.iova_to_phys = iommu_v1_iova_to_phys; + return &pgtable->iop; } diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index f1a4f535eac8..144ea91a9ec9 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2100,22 +2100,8 @@ static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom, { struct protection_domain *domain = to_pdomain(dom); struct io_pgtable_ops *ops = &domain->iop.iop.ops; - struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops); - unsigned long offset_mask, pte_pgsize; - u64 *pte, __pte; - if (domain->iop.mode == PAGE_MODE_NONE) - return iova; - - pte = fetch_pte(pgtable, iova, &pte_pgsize); - - if (!pte || !IOMMU_PTE_PRESENT(*pte)) - return 0; - - offset_mask = pte_pgsize - 1; - __pte = __sme_clr(*pte & PM_ADDR_MASK); - - return (__pte & ~offset_mask) | (iova & offset_mask); + return ops->iova_to_phys(ops, iova); } static bool amd_iommu_capable(enum iommu_cap cap) -- cgit v1.2.3 From fd86c9501a9b3e5eb7e46b03f03ffa4bf892b523 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Tue, 15 Dec 2020 01:37:04 -0600 Subject: iommu/amd: Introduce iommu_v1_map_page and iommu_v1_unmap_page These implement map and unmap for AMD IOMMU v1 pagetable, which will be used by the IO pagetable framework. Also clean up unused extern function declarations. Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20201215073705.123786-13-suravee.suthikulpanit@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 13 ------------- drivers/iommu/amd/io_pgtable.c | 25 ++++++++++++------------- drivers/iommu/amd/iommu.c | 13 ++++++++----- 3 files changed, 20 insertions(+), 31 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 83ca822c5349..3770b1a4d51c 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -133,19 +133,6 @@ void amd_iommu_apply_ivrs_quirks(void); static inline void amd_iommu_apply_ivrs_quirks(void) { } #endif -/* TODO: These are temporary and will be removed once fully transition */ -extern int iommu_map_page(struct protection_domain *dom, - unsigned long bus_addr, - unsigned long phys_addr, - unsigned long page_size, - int prot, - gfp_t gfp); -extern unsigned long iommu_unmap_page(struct protection_domain *dom, - unsigned long bus_addr, - unsigned long page_size); -extern u64 *fetch_pte(struct amd_io_pgtable *pgtable, - unsigned long address, - unsigned long *page_size); extern void amd_iommu_domain_set_pgtable(struct protection_domain *domain, u64 *root, int mode); #endif diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c index b70eb79d05fb..1c4961e05c12 100644 --- a/drivers/iommu/amd/io_pgtable.c +++ b/drivers/iommu/amd/io_pgtable.c @@ -311,9 +311,9 @@ static u64 *alloc_pte(struct protection_domain *domain, * This function checks if there is a PTE for a given dma address. If * there is one, it returns the pointer to it. */ -u64 *fetch_pte(struct amd_io_pgtable *pgtable, - unsigned long address, - unsigned long *page_size) +static u64 *fetch_pte(struct amd_io_pgtable *pgtable, + unsigned long address, + unsigned long *page_size) { int level; u64 *pte; @@ -386,13 +386,10 @@ static struct page *free_clear_pte(u64 *pte, u64 pteval, struct page *freelist) * supporting all features of AMD IOMMU page tables like level skipping * and full 64 bit address spaces. */ -int iommu_map_page(struct protection_domain *dom, - unsigned long iova, - unsigned long paddr, - unsigned long size, - int prot, - gfp_t gfp) +static int iommu_v1_map_page(struct io_pgtable_ops *ops, unsigned long iova, + phys_addr_t paddr, size_t size, int prot, gfp_t gfp) { + struct protection_domain *dom = io_pgtable_ops_to_domain(ops); struct page *freelist = NULL; bool updated = false; u64 __pte, *pte; @@ -455,11 +452,11 @@ out: return ret; } -unsigned long iommu_unmap_page(struct protection_domain *dom, - unsigned long iova, - unsigned long size) +static unsigned long iommu_v1_unmap_page(struct io_pgtable_ops *ops, + unsigned long iova, + size_t size, + struct iommu_iotlb_gather *gather) { - struct io_pgtable_ops *ops = &dom->iop.iop.ops; struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops); unsigned long long unmapped; unsigned long unmap_size; @@ -548,6 +545,8 @@ static struct io_pgtable *v1_alloc_pgtable(struct io_pgtable_cfg *cfg, void *coo cfg->oas = IOMMU_OUT_ADDR_BIT_SIZE, cfg->tlb = &v1_flush_ops; + pgtable->iop.ops.map = iommu_v1_map_page; + pgtable->iop.ops.unmap = iommu_v1_unmap_page; pgtable->iop.ops.iova_to_phys = iommu_v1_iova_to_phys; return &pgtable->iop; diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 144ea91a9ec9..256d38ad6afb 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2065,8 +2065,9 @@ static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova, gfp_t gfp) { struct protection_domain *domain = to_pdomain(dom); + struct io_pgtable_ops *ops = &domain->iop.iop.ops; int prot = 0; - int ret; + int ret = -EINVAL; if (domain->iop.mode == PAGE_MODE_NONE) return -EINVAL; @@ -2076,9 +2077,10 @@ static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova, if (iommu_prot & IOMMU_WRITE) prot |= IOMMU_PROT_IW; - ret = iommu_map_page(domain, iova, paddr, page_size, prot, gfp); - - domain_flush_np_cache(domain, iova, page_size); + if (ops->map) { + ret = ops->map(ops, iova, paddr, page_size, prot, gfp); + domain_flush_np_cache(domain, iova, page_size); + } return ret; } @@ -2088,11 +2090,12 @@ static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova, struct iommu_iotlb_gather *gather) { struct protection_domain *domain = to_pdomain(dom); + struct io_pgtable_ops *ops = &domain->iop.iop.ops; if (domain->iop.mode == PAGE_MODE_NONE) return 0; - return iommu_unmap_page(domain, iova, page_size); + return (ops->unmap) ? ops->unmap(ops, iova, page_size, gather) : 0; } static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom, -- cgit v1.2.3 From 89c9a09cb9f6e0a7df77f9c9bafd3c96148bf0d5 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Tue, 15 Dec 2020 01:37:05 -0600 Subject: iommu/amd: Adopt IO page table framework for AMD IOMMU v1 page table Switch to using IO page table framework for AMD IOMMU v1 page table. Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20201215073705.123786-14-suravee.suthikulpanit@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 1 + drivers/iommu/amd/init.c | 2 ++ drivers/iommu/amd/iommu.c | 48 ++++++++++++++++++++++++++++++++----------- 3 files changed, 39 insertions(+), 12 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 3770b1a4d51c..91452e0ff072 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -36,6 +36,7 @@ extern void amd_iommu_disable(void); extern int amd_iommu_reenable(int); extern int amd_iommu_enable_faulting(void); extern int amd_iommu_guest_ir; +extern enum io_pgtable_fmt amd_iommu_pgtable; /* IOMMUv2 specific functions */ struct iommu_domain; diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 1804c3e3b6ac..5a6c511e91d1 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -147,6 +147,8 @@ struct ivmd_header { bool amd_iommu_dump; bool amd_iommu_irq_remap __read_mostly; +enum io_pgtable_fmt amd_iommu_pgtable = AMD_IOMMU_V1; + int amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC; static int amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE; diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 256d38ad6afb..a69a8b573e40 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -1900,7 +1901,7 @@ static void protection_domain_free(struct protection_domain *domain) kfree(domain); } -static int protection_domain_init(struct protection_domain *domain, int mode) +static int protection_domain_init_v1(struct protection_domain *domain, int mode) { u64 *pt_root = NULL; @@ -1923,34 +1924,55 @@ static int protection_domain_init(struct protection_domain *domain, int mode) return 0; } -static struct protection_domain *protection_domain_alloc(int mode) +static struct protection_domain *protection_domain_alloc(unsigned int type) { + struct io_pgtable_ops *pgtbl_ops; struct protection_domain *domain; + int pgtable = amd_iommu_pgtable; + int mode = DEFAULT_PGTABLE_LEVEL; + int ret; domain = kzalloc(sizeof(*domain), GFP_KERNEL); if (!domain) return NULL; - if (protection_domain_init(domain, mode)) + /* + * Force IOMMU v1 page table when iommu=pt and + * when allocating domain for pass-through devices. + */ + if (type == IOMMU_DOMAIN_IDENTITY) { + pgtable = AMD_IOMMU_V1; + mode = PAGE_MODE_NONE; + } else if (type == IOMMU_DOMAIN_UNMANAGED) { + pgtable = AMD_IOMMU_V1; + } + + switch (pgtable) { + case AMD_IOMMU_V1: + ret = protection_domain_init_v1(domain, mode); + break; + default: + ret = -EINVAL; + } + + if (ret) goto out_err; - return domain; + pgtbl_ops = alloc_io_pgtable_ops(pgtable, &domain->iop.pgtbl_cfg, domain); + if (!pgtbl_ops) + goto out_err; + return domain; out_err: kfree(domain); - return NULL; } static struct iommu_domain *amd_iommu_domain_alloc(unsigned type) { struct protection_domain *domain; - int mode = DEFAULT_PGTABLE_LEVEL; - - if (type == IOMMU_DOMAIN_IDENTITY) - mode = PAGE_MODE_NONE; - domain = protection_domain_alloc(mode); + domain = protection_domain_alloc(type); if (!domain) return NULL; @@ -2069,7 +2091,8 @@ static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova, int prot = 0; int ret = -EINVAL; - if (domain->iop.mode == PAGE_MODE_NONE) + if ((amd_iommu_pgtable == AMD_IOMMU_V1) && + (domain->iop.mode == PAGE_MODE_NONE)) return -EINVAL; if (iommu_prot & IOMMU_READ) @@ -2092,7 +2115,8 @@ static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova, struct protection_domain *domain = to_pdomain(dom); struct io_pgtable_ops *ops = &domain->iop.iop.ops; - if (domain->iop.mode == PAGE_MODE_NONE) + if ((amd_iommu_pgtable == AMD_IOMMU_V1) && + (domain->iop.mode == PAGE_MODE_NONE)) return 0; return (ops->unmap) ? ops->unmap(ops, iova, page_size, gather) : 0; -- cgit v1.2.3 From c867c78acae96fc359f2a2f375ce64b5f0871802 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 27 Jan 2021 16:29:28 +0000 Subject: iommu/msm: Hook up iotlb_sync_map The core API can now accommodate invalidate-on-map style behaviour in a single efficient call, so hook that up instead of having io-pgatble do it piecemeal. Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/e95223a0abf129230a0bec6743f837075f0a2fcb.1611764372.git.robin.murphy@arm.com Signed-off-by: Will Deacon --- drivers/iommu/msm_iommu.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c index 040e85f70861..f0ba6a09b434 100644 --- a/drivers/iommu/msm_iommu.c +++ b/drivers/iommu/msm_iommu.c @@ -343,7 +343,6 @@ static int msm_iommu_domain_config(struct msm_priv *priv) spin_lock_init(&priv->pgtlock); priv->cfg = (struct io_pgtable_cfg) { - .quirks = IO_PGTABLE_QUIRK_TLBI_ON_MAP, .pgsize_bitmap = msm_iommu_ops.pgsize_bitmap, .ias = 32, .oas = 32, @@ -490,6 +489,14 @@ static int msm_iommu_map(struct iommu_domain *domain, unsigned long iova, return ret; } +static void msm_iommu_sync_map(struct iommu_domain *domain, unsigned long iova, + size_t size) +{ + struct msm_priv *priv = to_msm_priv(domain); + + __flush_iotlb_range(iova, size, SZ_4K, false, priv); +} + static size_t msm_iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t len, struct iommu_iotlb_gather *gather) { @@ -680,6 +687,7 @@ static struct iommu_ops msm_iommu_ops = { * kick starting the other master. */ .iotlb_sync = NULL, + .iotlb_sync_map = msm_iommu_sync_map, .iova_to_phys = msm_iommu_iova_to_phys, .probe_device = msm_iommu_probe_device, .release_device = msm_iommu_release_device, -- cgit v1.2.3 From 3d5eab41451f8e28f3e45eef8f6b372bf56612fb Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 27 Jan 2021 16:29:29 +0000 Subject: iommu/io-pgtable: Remove TLBI_ON_MAP quirk IO_PGTABLE_QUIRK_TLBI_ON_MAP is now fully superseded by the core API's iotlb_sync_map callback. Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/5abb80bba3a7c371d5ffb7e59c05586deddb9a91.1611764372.git.robin.murphy@arm.com [will: Remove unused 'iop' local variable from arm_v7s_map()] Signed-off-by: Will Deacon --- drivers/iommu/io-pgtable-arm-v7s.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c index 1d92ac948db7..929cb1518db1 100644 --- a/drivers/iommu/io-pgtable-arm-v7s.c +++ b/drivers/iommu/io-pgtable-arm-v7s.c @@ -519,7 +519,6 @@ static int arm_v7s_map(struct io_pgtable_ops *ops, unsigned long iova, phys_addr_t paddr, size_t size, int prot, gfp_t gfp) { struct arm_v7s_io_pgtable *data = io_pgtable_ops_to_data(ops); - struct io_pgtable *iop = &data->iop; int ret; if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias) || @@ -535,12 +534,7 @@ static int arm_v7s_map(struct io_pgtable_ops *ops, unsigned long iova, * Synchronise all PTE updates for the new mapping before there's * a chance for anything to kick off a table walk for the new iova. */ - if (iop->cfg.quirks & IO_PGTABLE_QUIRK_TLBI_ON_MAP) { - io_pgtable_tlb_flush_walk(iop, iova, size, - ARM_V7S_BLOCK_SIZE(2)); - } else { - wmb(); - } + wmb(); return ret; } @@ -759,7 +753,6 @@ static struct io_pgtable *arm_v7s_alloc_pgtable(struct io_pgtable_cfg *cfg, if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS | IO_PGTABLE_QUIRK_NO_PERMS | - IO_PGTABLE_QUIRK_TLBI_ON_MAP | IO_PGTABLE_QUIRK_ARM_MTK_EXT | IO_PGTABLE_QUIRK_NON_STRICT)) return NULL; -- cgit v1.2.3 From 28a77185f1cd0650b664f54614143aaaa3a7a615 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Tue, 26 Jan 2021 16:07:29 +0800 Subject: iommu/vt-d: Clear PRQ overflow only when PRQ is empty It is incorrect to always clear PRO when it's set w/o first checking whether the overflow condition has been cleared. Current code assumes that if an overflow condition occurs it must have been cleared by earlier loop. However since the code runs in a threaded context, the overflow condition could occur even after setting the head to the tail under some extreme condition. To be sane, we should read both head/tail again when seeing a pending PRO and only clear PRO after all pending PRs have been handled. Suggested-by: Kevin Tian Signed-off-by: Lu Baolu Link: https://lore.kernel.org/linux-iommu/MWHPR11MB18862D2EA5BD432BF22D99A48CA09@MWHPR11MB1886.namprd11.prod.outlook.com/ Link: https://lore.kernel.org/r/20210126080730.2232859-2-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/svm.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 033b25886e57..d7c98c5fa4e7 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -1042,8 +1042,17 @@ prq_advance: * Clear the page request overflow bit and wake up all threads that * are waiting for the completion of this handling. */ - if (readl(iommu->reg + DMAR_PRS_REG) & DMA_PRS_PRO) - writel(DMA_PRS_PRO, iommu->reg + DMAR_PRS_REG); + if (readl(iommu->reg + DMAR_PRS_REG) & DMA_PRS_PRO) { + pr_info_ratelimited("IOMMU: %s: PRQ overflow detected\n", + iommu->name); + head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK; + tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK; + if (head == tail) { + writel(DMA_PRS_PRO, iommu->reg + DMAR_PRS_REG); + pr_info_ratelimited("IOMMU: %s: PRQ overflow cleared", + iommu->name); + } + } if (!completion_done(&iommu->prq_complete)) complete(&iommu->prq_complete); -- cgit v1.2.3 From 3aa7c62cb7d7986063f352d96e921ee2bf2d9749 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Tue, 26 Jan 2021 16:07:30 +0800 Subject: iommu/vt-d: Use INVALID response code instead of FAILURE The VT-d IOMMU response RESPONSE_FAILURE for a page request in below cases: - When it gets a Page_Request with no PASID; - When it gets a Page_Request with PASID that is not in use for this device. This is allowed by the spec, but IOMMU driver doesn't support such cases today. When the device receives RESPONSE_FAILURE, it sends the device state machine to HALT state. Now if we try to unload the driver, it hangs since the device doesn't send any outbound transactions to host when the driver is trying to clear things up. The only possible responses would be for invalidation requests. Let's use RESPONSE_INVALID instead for now, so that the device state machine doesn't enter HALT state. Suggested-by: Ashok Raj Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20210126080730.2232859-3-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/svm.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index d7c98c5fa4e7..574a7e657a9a 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -911,10 +911,8 @@ static irqreturn_t prq_event_thread(int irq, void *d) u64 address; handled = 1; - req = &iommu->prq[head / sizeof(*req)]; - - result = QI_RESP_FAILURE; + result = QI_RESP_INVALID; address = (u64)req->addr << VTD_PAGE_SHIFT; if (!req->pasid_present) { pr_err("%s: Page request without PASID: %08llx %08llx\n", @@ -952,7 +950,6 @@ static irqreturn_t prq_event_thread(int irq, void *d) rcu_read_unlock(); } - result = QI_RESP_INVALID; /* Since we're using init_mm.pgd directly, we should never take * any faults on kernel addresses. */ if (!svm->mm) -- cgit v1.2.3 From 815cdd86038b809ef9fc3d42aed1f248414df3ce Mon Sep 17 00:00:00 2001 From: Yoshihiro Shimoda Date: Thu, 28 Jan 2021 22:02:59 +0900 Subject: iommu/ipmmu-vmsa: Refactor ipmmu_of_xlate() Refactor ipmmu_of_xlate() to improve readability/scalability. Signed-off-by: Yoshihiro Shimoda Link: https://lore.kernel.org/r/1611838980-4940-2-git-send-email-yoshihiro.shimoda.uh@renesas.com Signed-off-by: Joerg Roedel --- drivers/iommu/ipmmu-vmsa.c | 49 +++++++++++++++++----------------------------- 1 file changed, 18 insertions(+), 31 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index d71f10257f15..96aee98889be 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -734,54 +734,41 @@ static int ipmmu_init_platform_device(struct device *dev, return 0; } -static const struct soc_device_attribute soc_rcar_gen3[] = { - { .soc_id = "r8a774a1", }, - { .soc_id = "r8a774b1", }, - { .soc_id = "r8a774c0", }, - { .soc_id = "r8a774e1", }, - { .soc_id = "r8a7795", }, - { .soc_id = "r8a77961", }, - { .soc_id = "r8a7796", }, - { .soc_id = "r8a77965", }, - { .soc_id = "r8a77970", }, - { .soc_id = "r8a77990", }, - { .soc_id = "r8a77995", }, +static const struct soc_device_attribute soc_needs_opt_in[] = { + { .family = "R-Car Gen3", }, + { .family = "RZ/G2", }, { /* sentinel */ } }; -static const struct soc_device_attribute soc_rcar_gen3_whitelist[] = { - { .soc_id = "r8a774b1", }, - { .soc_id = "r8a774c0", }, - { .soc_id = "r8a774e1", }, - { .soc_id = "r8a7795", .revision = "ES3.*" }, - { .soc_id = "r8a77961", }, - { .soc_id = "r8a77965", }, - { .soc_id = "r8a77990", }, - { .soc_id = "r8a77995", }, +static const struct soc_device_attribute soc_denylist[] = { + { .soc_id = "r8a774a1", }, + { .soc_id = "r8a7795", .revision = "ES1.*" }, + { .soc_id = "r8a7795", .revision = "ES2.*" }, + { .soc_id = "r8a7796", }, { /* sentinel */ } }; -static const char * const rcar_gen3_slave_whitelist[] = { +static const char * const devices_allowlist[] = { }; -static bool ipmmu_slave_whitelist(struct device *dev) +static bool ipmmu_device_is_allowed(struct device *dev) { unsigned int i; /* - * For R-Car Gen3 use a white list to opt-in slave devices. + * R-Car Gen3 and RZ/G2 use the allow list to opt-in devices. * For Other SoCs, this returns true anyway. */ - if (!soc_device_match(soc_rcar_gen3)) + if (!soc_device_match(soc_needs_opt_in)) return true; - /* Check whether this R-Car Gen3 can use the IPMMU correctly or not */ - if (!soc_device_match(soc_rcar_gen3_whitelist)) + /* Check whether this SoC can use the IPMMU correctly or not */ + if (soc_device_match(soc_denylist)) return false; - /* Check whether this slave device can work with the IPMMU */ - for (i = 0; i < ARRAY_SIZE(rcar_gen3_slave_whitelist); i++) { - if (!strcmp(dev_name(dev), rcar_gen3_slave_whitelist[i])) + /* Check whether this device can work with the IPMMU */ + for (i = 0; i < ARRAY_SIZE(devices_allowlist); i++) { + if (!strcmp(dev_name(dev), devices_allowlist[i])) return true; } @@ -792,7 +779,7 @@ static bool ipmmu_slave_whitelist(struct device *dev) static int ipmmu_of_xlate(struct device *dev, struct of_phandle_args *spec) { - if (!ipmmu_slave_whitelist(dev)) + if (!ipmmu_device_is_allowed(dev)) return -ENODEV; iommu_fwspec_add_ids(dev, spec->args, 1); -- cgit v1.2.3 From cec0813da56aa5fcca8da839c76f58c8273a4c66 Mon Sep 17 00:00:00 2001 From: Yoshihiro Shimoda Date: Thu, 28 Jan 2021 22:03:00 +0900 Subject: iommu/ipmmu-vmsa: Allow SDHI devices Add SDHI devices into devices_allowlist. Signed-off-by: Yoshihiro Shimoda Link: https://lore.kernel.org/r/1611838980-4940-3-git-send-email-yoshihiro.shimoda.uh@renesas.com Signed-off-by: Joerg Roedel --- drivers/iommu/ipmmu-vmsa.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 96aee98889be..eaaec0a55cc6 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -749,6 +749,10 @@ static const struct soc_device_attribute soc_denylist[] = { }; static const char * const devices_allowlist[] = { + "ee100000.mmc", + "ee120000.mmc", + "ee140000.mmc", + "ee160000.mmc" }; static bool ipmmu_device_is_allowed(struct device *dev) -- cgit v1.2.3 From 66a28915541549f0fe0f4c705dab77443f22b5d6 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Mon, 11 Jan 2021 19:18:49 +0800 Subject: iommu/mediatek: Use the common mtk-memory-port.h Use the common memory header(larb-port) in the source code. Signed-off-by: Yong Wu Acked-by: Krzysztof Kozlowski Reviewed-by: Tomasz Figa Link: https://lore.kernel.org/r/20210111111914.22211-9-yong.wu@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/mtk_iommu.c | 7 ------- drivers/iommu/mtk_iommu.h | 1 + 2 files changed, 1 insertion(+), 7 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 86ab577c9520..f594971dbeb2 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -103,13 +103,6 @@ #define MTK_PROTECT_PA_ALIGN 256 -/* - * Get the local arbiter ID and the portid within the larb arbiter - * from mtk_m4u_id which is defined by MTK_M4U_ID. - */ -#define MTK_M4U_TO_LARB(id) (((id) >> 5) & 0xf) -#define MTK_M4U_TO_PORT(id) ((id) & 0x1f) - #define HAS_4GB_MODE BIT(0) /* HW will use the EMI clock if there isn't the "bclk". */ #define HAS_BCLK BIT(1) diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h index df32b3e3408b..c1584dea66cb 100644 --- a/drivers/iommu/mtk_iommu.h +++ b/drivers/iommu/mtk_iommu.h @@ -17,6 +17,7 @@ #include #include #include +#include #define MTK_LARB_COM_MAX 8 #define MTK_LARB_SUBCOM_MAX 4 -- cgit v1.2.3 From 859da21112520829638a82f458ac2c8c2637e17d Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Mon, 11 Jan 2021 19:18:50 +0800 Subject: iommu/io-pgtable-arm-v7s: Use ias to check the valid iova in unmap Use the ias for the valid iova checking in arm_v7s_unmap. This is a preparing patch for supporting iova 34bit for MediaTek. Signed-off-by: Yong Wu Reviewed-by: Robin Murphy Reviewed-by: Tomasz Figa Link: https://lore.kernel.org/r/20210111111914.22211-10-yong.wu@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/io-pgtable-arm-v7s.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c index 929cb1518db1..03f35065acff 100644 --- a/drivers/iommu/io-pgtable-arm-v7s.c +++ b/drivers/iommu/io-pgtable-arm-v7s.c @@ -711,7 +711,7 @@ static size_t arm_v7s_unmap(struct io_pgtable_ops *ops, unsigned long iova, { struct arm_v7s_io_pgtable *data = io_pgtable_ops_to_data(ops); - if (WARN_ON(upper_32_bits(iova))) + if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias))) return 0; return __arm_v7s_unmap(data, gather, iova, size, 1, data->pgd); -- cgit v1.2.3 From 40596d2f2b6075f6c33180b2f55c814ff4885475 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Mon, 11 Jan 2021 19:18:51 +0800 Subject: iommu/io-pgtable-arm-v7s: Extend PA34 for MediaTek MediaTek extend the bit5 in lvl1 and lvl2 descriptor as PA34. Signed-off-by: Yong Wu Acked-by: Will Deacon Reviewed-by: Robin Murphy Reviewed-by: Tomasz Figa Link: https://lore.kernel.org/r/20210111111914.22211-11-yong.wu@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/io-pgtable-arm-v7s.c | 9 +++++++-- drivers/iommu/mtk_iommu.c | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c index 03f35065acff..b1b2ff069040 100644 --- a/drivers/iommu/io-pgtable-arm-v7s.c +++ b/drivers/iommu/io-pgtable-arm-v7s.c @@ -112,9 +112,10 @@ #define ARM_V7S_TEX_MASK 0x7 #define ARM_V7S_ATTR_TEX(val) (((val) & ARM_V7S_TEX_MASK) << ARM_V7S_TEX_SHIFT) -/* MediaTek extend the two bits for PA 32bit/33bit */ +/* MediaTek extend the bits below for PA 32bit/33bit/34bit */ #define ARM_V7S_ATTR_MTK_PA_BIT32 BIT(9) #define ARM_V7S_ATTR_MTK_PA_BIT33 BIT(4) +#define ARM_V7S_ATTR_MTK_PA_BIT34 BIT(5) /* *well, except for TEX on level 2 large pages, of course :( */ #define ARM_V7S_CONT_PAGE_TEX_SHIFT 6 @@ -194,6 +195,8 @@ static arm_v7s_iopte paddr_to_iopte(phys_addr_t paddr, int lvl, pte |= ARM_V7S_ATTR_MTK_PA_BIT32; if (paddr & BIT_ULL(33)) pte |= ARM_V7S_ATTR_MTK_PA_BIT33; + if (paddr & BIT_ULL(34)) + pte |= ARM_V7S_ATTR_MTK_PA_BIT34; return pte; } @@ -218,6 +221,8 @@ static phys_addr_t iopte_to_paddr(arm_v7s_iopte pte, int lvl, paddr |= BIT_ULL(32); if (pte & ARM_V7S_ATTR_MTK_PA_BIT33) paddr |= BIT_ULL(33); + if (pte & ARM_V7S_ATTR_MTK_PA_BIT34) + paddr |= BIT_ULL(34); return paddr; } @@ -748,7 +753,7 @@ static struct io_pgtable *arm_v7s_alloc_pgtable(struct io_pgtable_cfg *cfg, if (cfg->ias > ARM_V7S_ADDR_BITS) return NULL; - if (cfg->oas > (arm_v7s_is_mtk_enabled(cfg) ? 34 : ARM_V7S_ADDR_BITS)) + if (cfg->oas > (arm_v7s_is_mtk_enabled(cfg) ? 35 : ARM_V7S_ADDR_BITS)) return NULL; if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS | diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index f594971dbeb2..485f3b6d1a21 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -300,7 +300,7 @@ static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom) IO_PGTABLE_QUIRK_ARM_MTK_EXT, .pgsize_bitmap = mtk_iommu_ops.pgsize_bitmap, .ias = 32, - .oas = 34, + .oas = 35, .iommu_dev = data->dev, }; -- cgit v1.2.3 From 00ab6f2d61583e072a0cced0420ee1134d853a35 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Mon, 11 Jan 2021 19:18:52 +0800 Subject: iommu/io-pgtable-arm-v7s: Clarify LVL_SHIFT/BITS macro The current _ARM_V7S_LVL_BITS/ARM_V7S_LVL_SHIFT use a formula to calculate the corresponding value for level1 and level2 to pretend the code sane. Actually their level1 and level2 values are different from each other. This patch only clarify the two macro. No functional change. Suggested-by: Robin Murphy Signed-off-by: Yong Wu Reviewed-by: Robin Murphy Reviewed-by: Tomasz Figa Link: https://lore.kernel.org/r/20210111111914.22211-12-yong.wu@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/io-pgtable-arm-v7s.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c index b1b2ff069040..a129988cc1a6 100644 --- a/drivers/iommu/io-pgtable-arm-v7s.c +++ b/drivers/iommu/io-pgtable-arm-v7s.c @@ -44,13 +44,11 @@ /* * We have 32 bits total; 12 bits resolved at level 1, 8 bits at level 2, - * and 12 bits in a page. With some carefully-chosen coefficients we can - * hide the ugly inconsistencies behind these macros and at least let the - * rest of the code pretend to be somewhat sane. + * and 12 bits in a page. */ #define ARM_V7S_ADDR_BITS 32 -#define _ARM_V7S_LVL_BITS(lvl) (16 - (lvl) * 4) -#define ARM_V7S_LVL_SHIFT(lvl) (ARM_V7S_ADDR_BITS - (4 + 8 * (lvl))) +#define _ARM_V7S_LVL_BITS(lvl) ((lvl) == 1 ? 12 : 8) +#define ARM_V7S_LVL_SHIFT(lvl) ((lvl) == 1 ? 20 : 12) #define ARM_V7S_TABLE_SHIFT 10 #define ARM_V7S_PTES_PER_LVL(lvl) (1 << _ARM_V7S_LVL_BITS(lvl)) -- cgit v1.2.3 From 468ea0bfaecd097c223e13400d8e8fbae75e1f68 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Mon, 11 Jan 2021 19:18:53 +0800 Subject: iommu/io-pgtable-arm-v7s: Add cfg as a param in some macros Add "cfg" as a parameter for some macros. This is a preparing patch for mediatek extend the lvl1 pgtable. No functional change. Signed-off-by: Yong Wu Acked-by: Will Deacon Reviewed-by: Robin Murphy Reviewed-by: Tomasz Figa Link: https://lore.kernel.org/r/20210111111914.22211-13-yong.wu@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/io-pgtable-arm-v7s.c | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c index a129988cc1a6..2e161f0a65f0 100644 --- a/drivers/iommu/io-pgtable-arm-v7s.c +++ b/drivers/iommu/io-pgtable-arm-v7s.c @@ -47,21 +47,21 @@ * and 12 bits in a page. */ #define ARM_V7S_ADDR_BITS 32 -#define _ARM_V7S_LVL_BITS(lvl) ((lvl) == 1 ? 12 : 8) +#define _ARM_V7S_LVL_BITS(lvl, cfg) ((lvl) == 1 ? 12 : 8) #define ARM_V7S_LVL_SHIFT(lvl) ((lvl) == 1 ? 20 : 12) #define ARM_V7S_TABLE_SHIFT 10 -#define ARM_V7S_PTES_PER_LVL(lvl) (1 << _ARM_V7S_LVL_BITS(lvl)) -#define ARM_V7S_TABLE_SIZE(lvl) \ - (ARM_V7S_PTES_PER_LVL(lvl) * sizeof(arm_v7s_iopte)) +#define ARM_V7S_PTES_PER_LVL(lvl, cfg) (1 << _ARM_V7S_LVL_BITS(lvl, cfg)) +#define ARM_V7S_TABLE_SIZE(lvl, cfg) \ + (ARM_V7S_PTES_PER_LVL(lvl, cfg) * sizeof(arm_v7s_iopte)) #define ARM_V7S_BLOCK_SIZE(lvl) (1UL << ARM_V7S_LVL_SHIFT(lvl)) #define ARM_V7S_LVL_MASK(lvl) ((u32)(~0U << ARM_V7S_LVL_SHIFT(lvl))) #define ARM_V7S_TABLE_MASK ((u32)(~0U << ARM_V7S_TABLE_SHIFT)) -#define _ARM_V7S_IDX_MASK(lvl) (ARM_V7S_PTES_PER_LVL(lvl) - 1) -#define ARM_V7S_LVL_IDX(addr, lvl) ({ \ +#define _ARM_V7S_IDX_MASK(lvl, cfg) (ARM_V7S_PTES_PER_LVL(lvl, cfg) - 1) +#define ARM_V7S_LVL_IDX(addr, lvl, cfg) ({ \ int _l = lvl; \ - ((u32)(addr) >> ARM_V7S_LVL_SHIFT(_l)) & _ARM_V7S_IDX_MASK(_l); \ + ((u32)(addr) >> ARM_V7S_LVL_SHIFT(_l)) & _ARM_V7S_IDX_MASK(_l, cfg); \ }) /* @@ -237,7 +237,7 @@ static void *__arm_v7s_alloc_table(int lvl, gfp_t gfp, struct device *dev = cfg->iommu_dev; phys_addr_t phys; dma_addr_t dma; - size_t size = ARM_V7S_TABLE_SIZE(lvl); + size_t size = ARM_V7S_TABLE_SIZE(lvl, cfg); void *table = NULL; if (lvl == 1) @@ -283,7 +283,7 @@ static void __arm_v7s_free_table(void *table, int lvl, { struct io_pgtable_cfg *cfg = &data->iop.cfg; struct device *dev = cfg->iommu_dev; - size_t size = ARM_V7S_TABLE_SIZE(lvl); + size_t size = ARM_V7S_TABLE_SIZE(lvl, cfg); if (!cfg->coherent_walk) dma_unmap_single(dev, __arm_v7s_dma_addr(table), size, @@ -427,7 +427,7 @@ static int arm_v7s_init_pte(struct arm_v7s_io_pgtable *data, arm_v7s_iopte *tblp; size_t sz = ARM_V7S_BLOCK_SIZE(lvl); - tblp = ptep - ARM_V7S_LVL_IDX(iova, lvl); + tblp = ptep - ARM_V7S_LVL_IDX(iova, lvl, cfg); if (WARN_ON(__arm_v7s_unmap(data, NULL, iova + i * sz, sz, lvl, tblp) != sz)) return -EINVAL; @@ -480,7 +480,7 @@ static int __arm_v7s_map(struct arm_v7s_io_pgtable *data, unsigned long iova, int num_entries = size >> ARM_V7S_LVL_SHIFT(lvl); /* Find our entry at the current level */ - ptep += ARM_V7S_LVL_IDX(iova, lvl); + ptep += ARM_V7S_LVL_IDX(iova, lvl, cfg); /* If we can install a leaf entry at this level, then do so */ if (num_entries) @@ -547,7 +547,7 @@ static void arm_v7s_free_pgtable(struct io_pgtable *iop) struct arm_v7s_io_pgtable *data = io_pgtable_to_data(iop); int i; - for (i = 0; i < ARM_V7S_PTES_PER_LVL(1); i++) { + for (i = 0; i < ARM_V7S_PTES_PER_LVL(1, &data->iop.cfg); i++) { arm_v7s_iopte pte = data->pgd[i]; if (ARM_V7S_PTE_IS_TABLE(pte, 1)) @@ -599,9 +599,9 @@ static size_t arm_v7s_split_blk_unmap(struct arm_v7s_io_pgtable *data, if (!tablep) return 0; /* Bytes unmapped */ - num_ptes = ARM_V7S_PTES_PER_LVL(2); + num_ptes = ARM_V7S_PTES_PER_LVL(2, cfg); num_entries = size >> ARM_V7S_LVL_SHIFT(2); - unmap_idx = ARM_V7S_LVL_IDX(iova, 2); + unmap_idx = ARM_V7S_LVL_IDX(iova, 2, cfg); pte = arm_v7s_prot_to_pte(arm_v7s_pte_to_prot(blk_pte, 1), 2, cfg); if (num_entries > 1) @@ -643,7 +643,7 @@ static size_t __arm_v7s_unmap(struct arm_v7s_io_pgtable *data, if (WARN_ON(lvl > 2)) return 0; - idx = ARM_V7S_LVL_IDX(iova, lvl); + idx = ARM_V7S_LVL_IDX(iova, lvl, &iop->cfg); ptep += idx; do { pte[i] = READ_ONCE(ptep[i]); @@ -729,7 +729,7 @@ static phys_addr_t arm_v7s_iova_to_phys(struct io_pgtable_ops *ops, u32 mask; do { - ptep += ARM_V7S_LVL_IDX(iova, ++lvl); + ptep += ARM_V7S_LVL_IDX(iova, ++lvl, &data->iop.cfg); pte = READ_ONCE(*ptep); ptep = iopte_deref(pte, lvl, data); } while (ARM_V7S_PTE_IS_TABLE(pte, lvl)); @@ -771,8 +771,8 @@ static struct io_pgtable *arm_v7s_alloc_pgtable(struct io_pgtable_cfg *cfg, spin_lock_init(&data->split_lock); data->l2_tables = kmem_cache_create("io-pgtable_armv7s_l2", - ARM_V7S_TABLE_SIZE(2), - ARM_V7S_TABLE_SIZE(2), + ARM_V7S_TABLE_SIZE(2, cfg), + ARM_V7S_TABLE_SIZE(2, cfg), ARM_V7S_TABLE_SLAB_FLAGS, NULL); if (!data->l2_tables) goto out_free_data; -- cgit v1.2.3 From f3a8a46d714fd67b856f09e4c783363076ebac62 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Mon, 11 Jan 2021 19:18:54 +0800 Subject: iommu/io-pgtable-arm-v7s: Quad lvl1 pgtable for MediaTek The standard input iova bits is 32. MediaTek quad the lvl1 pagetable (4 * lvl1). No change for lvl2 pagetable. Then the iova bits can reach 34bit. Signed-off-by: Yong Wu Reviewed-by: Robin Murphy Reviewed-by: Tomasz Figa Link: https://lore.kernel.org/r/20210111111914.22211-14-yong.wu@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/io-pgtable-arm-v7s.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c index 2e161f0a65f0..d4004bcf333a 100644 --- a/drivers/iommu/io-pgtable-arm-v7s.c +++ b/drivers/iommu/io-pgtable-arm-v7s.c @@ -45,9 +45,10 @@ /* * We have 32 bits total; 12 bits resolved at level 1, 8 bits at level 2, * and 12 bits in a page. + * MediaTek extend 2 bits to reach 34bits, 14 bits at lvl1 and 8 bits at lvl2. */ #define ARM_V7S_ADDR_BITS 32 -#define _ARM_V7S_LVL_BITS(lvl, cfg) ((lvl) == 1 ? 12 : 8) +#define _ARM_V7S_LVL_BITS(lvl, cfg) ((lvl) == 1 ? ((cfg)->ias - 20) : 8) #define ARM_V7S_LVL_SHIFT(lvl) ((lvl) == 1 ? 20 : 12) #define ARM_V7S_TABLE_SHIFT 10 @@ -61,7 +62,7 @@ #define _ARM_V7S_IDX_MASK(lvl, cfg) (ARM_V7S_PTES_PER_LVL(lvl, cfg) - 1) #define ARM_V7S_LVL_IDX(addr, lvl, cfg) ({ \ int _l = lvl; \ - ((u32)(addr) >> ARM_V7S_LVL_SHIFT(_l)) & _ARM_V7S_IDX_MASK(_l, cfg); \ + ((addr) >> ARM_V7S_LVL_SHIFT(_l)) & _ARM_V7S_IDX_MASK(_l, cfg); \ }) /* @@ -748,7 +749,7 @@ static struct io_pgtable *arm_v7s_alloc_pgtable(struct io_pgtable_cfg *cfg, { struct arm_v7s_io_pgtable *data; - if (cfg->ias > ARM_V7S_ADDR_BITS) + if (cfg->ias > (arm_v7s_is_mtk_enabled(cfg) ? 34 : ARM_V7S_ADDR_BITS)) return NULL; if (cfg->oas > (arm_v7s_is_mtk_enabled(cfg) ? 35 : ARM_V7S_ADDR_BITS)) -- cgit v1.2.3 From 2f317da4332a145be76ccd49d0a38803b499c39c Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Mon, 11 Jan 2021 19:18:55 +0800 Subject: iommu/mediatek: Add a flag for iova 34bits case Add a HW flag for if the HW support 34bit IOVA. the previous SoC still use 32bit. normally the lvl1 pgtable size is 16KB when ias == 32. if ias == 34, lvl1 pgtable size is 16KB * 4. The purpose of this patch is to save 16KB*3 continuous memory for the previous SoC. Signed-off-by: Yong Wu Reviewed-by: Tomasz Figa Link: https://lore.kernel.org/r/20210111111914.22211-15-yong.wu@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/mtk_iommu.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 485f3b6d1a21..bf1277d58121 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -112,6 +112,7 @@ #define HAS_SUB_COMM BIT(5) #define WR_THROT_EN BIT(6) #define HAS_LEGACY_IVRP_PADDR BIT(7) +#define IOVA_34_EN BIT(8) #define MTK_IOMMU_HAS_FLAG(pdata, _x) \ ((((pdata)->flags) & (_x)) == (_x)) @@ -299,7 +300,7 @@ static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom) IO_PGTABLE_QUIRK_NO_PERMS | IO_PGTABLE_QUIRK_ARM_MTK_EXT, .pgsize_bitmap = mtk_iommu_ops.pgsize_bitmap, - .ias = 32, + .ias = MTK_IOMMU_HAS_FLAG(data->plat_data, IOVA_34_EN) ? 34 : 32, .oas = 35, .iommu_dev = data->dev, }; -- cgit v1.2.3 From 9bdfe4c175c8a9ea4959f7ad087e3d66d3917bc8 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Mon, 11 Jan 2021 19:18:56 +0800 Subject: iommu/mediatek: Update oas for v7s This patch only updates oas in different SoCs. If the SoC supports 4GB-mode and current dram size is 4GB, the oas is 33. otherwise, it's still 32. In the lastest SoC, the oas is 35bits. Signed-off-by: Yong Wu Reviewed-by: Tomasz Figa Link: https://lore.kernel.org/r/20210111111914.22211-16-yong.wu@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/mtk_iommu.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index bf1277d58121..1c4af574f5f7 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -301,10 +301,14 @@ static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom) IO_PGTABLE_QUIRK_ARM_MTK_EXT, .pgsize_bitmap = mtk_iommu_ops.pgsize_bitmap, .ias = MTK_IOMMU_HAS_FLAG(data->plat_data, IOVA_34_EN) ? 34 : 32, - .oas = 35, .iommu_dev = data->dev, }; + if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_4GB_MODE)) + dom->cfg.oas = data->enable_4GB ? 33 : 32; + else + dom->cfg.oas = 35; + dom->iop = alloc_io_pgtable_ops(ARM_V7S, &dom->cfg, data); if (!dom->iop) { dev_err(data->dev, "Failed to alloc io pgtable\n"); -- cgit v1.2.3 From 7f37a91dc85e37ed06422c6e460724375a86f40c Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Mon, 11 Jan 2021 19:18:57 +0800 Subject: iommu/mediatek: Move hw_init into attach_device In attach device, it will update the pagetable base address register. Move the hw_init function also here. Then it only need call pm_runtime_get/put one time here if m4u has power domain. Signed-off-by: Yong Wu Reviewed-by: Tomasz Figa Link: https://lore.kernel.org/r/20210111111914.22211-17-yong.wu@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/mtk_iommu.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 1c4af574f5f7..87c4626c9072 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -126,6 +126,8 @@ struct mtk_iommu_domain { static const struct iommu_ops mtk_iommu_ops; +static int mtk_iommu_hw_init(const struct mtk_iommu_data *data); + /* * In M4U 4GB mode, the physical address is remapped as below: * @@ -364,12 +366,15 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, { struct mtk_iommu_data *data = dev_iommu_priv_get(dev); struct mtk_iommu_domain *dom = to_mtk_domain(domain); + int ret; if (!data) return -ENODEV; - /* Update the pgtable base address register of the M4U HW */ - if (!data->m4u_dom) { + if (!data->m4u_dom) { /* Initialize the M4U HW */ + ret = mtk_iommu_hw_init(data); + if (ret) + return ret; data->m4u_dom = dom; writel(dom->cfg.arm_v7s_cfg.ttbr & MMU_PT_ADDR_MASK, data->base + REG_MMU_PT_BASE_ADDR); @@ -724,10 +729,6 @@ static int mtk_iommu_probe(struct platform_device *pdev) platform_set_drvdata(pdev, data); - ret = mtk_iommu_hw_init(data); - if (ret) - return ret; - ret = iommu_device_sysfs_add(&data->iommu, dev, NULL, "mtk-iommu.%pa", &ioaddr); if (ret) -- cgit v1.2.3 From 986d9ec5f176ff1a539e849d57b3d6ecc937c4a6 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Mon, 11 Jan 2021 19:18:58 +0800 Subject: iommu/mediatek: Add error handle for mtk_iommu_probe In the original code, we lack the error handle. This patch adds them. Signed-off-by: Yong Wu Reviewed-by: Tomasz Figa Link: https://lore.kernel.org/r/20210111111914.22211-18-yong.wu@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/mtk_iommu.c | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 87c4626c9072..189165e7a2ab 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -739,15 +739,30 @@ static int mtk_iommu_probe(struct platform_device *pdev) ret = iommu_device_register(&data->iommu); if (ret) - return ret; + goto out_sysfs_remove; spin_lock_init(&data->tlb_lock); list_add_tail(&data->list, &m4ulist); - if (!iommu_present(&platform_bus_type)) - bus_set_iommu(&platform_bus_type, &mtk_iommu_ops); + if (!iommu_present(&platform_bus_type)) { + ret = bus_set_iommu(&platform_bus_type, &mtk_iommu_ops); + if (ret) + goto out_list_del; + } - return component_master_add_with_match(dev, &mtk_iommu_com_ops, match); + ret = component_master_add_with_match(dev, &mtk_iommu_com_ops, match); + if (ret) + goto out_bus_set_null; + return ret; + +out_bus_set_null: + bus_set_iommu(&platform_bus_type, NULL); +out_list_del: + list_del(&data->list); + iommu_device_unregister(&data->iommu); +out_sysfs_remove: + iommu_device_sysfs_remove(&data->iommu); + return ret; } static int mtk_iommu_remove(struct platform_device *pdev) -- cgit v1.2.3 From baf94e6ebff9622f60c4a87ff59c85bf756b7e20 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Mon, 11 Jan 2021 19:18:59 +0800 Subject: iommu/mediatek: Add device link for smi-common and m4u In the lastest SoC, M4U has its special power domain. thus, If the engine begin to work, it should help enable the power for M4U firstly. Currently if the engine work, it always enable the power/clocks for smi-larbs/smi-common. This patch adds device_link for smi-common and M4U. then, if smi-common power is enabled, the M4U power also is powered on automatically. Normally M4U connect with several smi-larbs and their smi-common always are the same, In this patch it get smi-common dev from the last smi-larb device, then add the device_link. Signed-off-by: Yong Wu Reviewed-by: Tomasz Figa Link: https://lore.kernel.org/r/20210111111914.22211-19-yong.wu@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/mtk_iommu.c | 27 ++++++++++++++++++++++++--- drivers/iommu/mtk_iommu.h | 1 + 2 files changed, 25 insertions(+), 3 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 189165e7a2ab..0fe7c1617dc3 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -633,6 +634,9 @@ static int mtk_iommu_probe(struct platform_device *pdev) { struct mtk_iommu_data *data; struct device *dev = &pdev->dev; + struct device_node *larbnode, *smicomm_node; + struct platform_device *plarbdev; + struct device_link *link; struct resource *res; resource_size_t ioaddr; struct component_match *match = NULL; @@ -699,8 +703,6 @@ static int mtk_iommu_probe(struct platform_device *pdev) return larb_nr; for (i = 0; i < larb_nr; i++) { - struct device_node *larbnode; - struct platform_device *plarbdev; u32 id; larbnode = of_parse_phandle(dev->of_node, "mediatek,larbs", i); @@ -727,12 +729,28 @@ static int mtk_iommu_probe(struct platform_device *pdev) compare_of, larbnode); } + /* Get smi-common dev from the last larb. */ + smicomm_node = of_parse_phandle(larbnode, "mediatek,smi", 0); + if (!smicomm_node) + return -EINVAL; + + plarbdev = of_find_device_by_node(smicomm_node); + of_node_put(smicomm_node); + data->smicomm_dev = &plarbdev->dev; + + link = device_link_add(data->smicomm_dev, dev, + DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME); + if (!link) { + dev_err(dev, "Unable link %s.\n", dev_name(data->smicomm_dev)); + return -EINVAL; + } + platform_set_drvdata(pdev, data); ret = iommu_device_sysfs_add(&data->iommu, dev, NULL, "mtk-iommu.%pa", &ioaddr); if (ret) - return ret; + goto out_link_remove; iommu_device_set_ops(&data->iommu, &mtk_iommu_ops); iommu_device_set_fwnode(&data->iommu, &pdev->dev.of_node->fwnode); @@ -762,6 +780,8 @@ out_list_del: iommu_device_unregister(&data->iommu); out_sysfs_remove: iommu_device_sysfs_remove(&data->iommu); +out_link_remove: + device_link_remove(data->smicomm_dev, dev); return ret; } @@ -776,6 +796,7 @@ static int mtk_iommu_remove(struct platform_device *pdev) bus_set_iommu(&platform_bus_type, NULL); clk_disable_unprepare(data->bclk); + device_link_remove(data->smicomm_dev, &pdev->dev); devm_free_irq(&pdev->dev, data->irq, data); component_master_del(&pdev->dev, &mtk_iommu_com_ops); return 0; diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h index c1584dea66cb..a9b79e118f02 100644 --- a/drivers/iommu/mtk_iommu.h +++ b/drivers/iommu/mtk_iommu.h @@ -68,6 +68,7 @@ struct mtk_iommu_data { struct iommu_device iommu; const struct mtk_iommu_plat_data *plat_data; + struct device *smicomm_dev; struct dma_iommu_mapping *mapping; /* For mtk_iommu_v1.c */ -- cgit v1.2.3 From 34665c7929fc27351ee3f45554e8991a6fd6e284 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Mon, 11 Jan 2021 19:19:00 +0800 Subject: iommu/mediatek: Add pm runtime callback In pm runtime case, all the registers backup/restore and bclk are controlled in the pm_runtime callback, Rename the original suspend/resume to the runtime_suspend/resume. Use pm_runtime_force_suspend/resume as the normal suspend/resume. iommu should suspend after iommu consumer devices, thus use _LATE_. Signed-off-by: Yong Wu Reviewed-by: Tomasz Figa Link: https://lore.kernel.org/r/20210111111914.22211-20-yong.wu@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/mtk_iommu.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 0fe7c1617dc3..3682137b789a 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -802,7 +802,7 @@ static int mtk_iommu_remove(struct platform_device *pdev) return 0; } -static int __maybe_unused mtk_iommu_suspend(struct device *dev) +static int __maybe_unused mtk_iommu_runtime_suspend(struct device *dev) { struct mtk_iommu_data *data = dev_get_drvdata(dev); struct mtk_iommu_suspend_reg *reg = &data->reg; @@ -820,7 +820,7 @@ static int __maybe_unused mtk_iommu_suspend(struct device *dev) return 0; } -static int __maybe_unused mtk_iommu_resume(struct device *dev) +static int __maybe_unused mtk_iommu_runtime_resume(struct device *dev) { struct mtk_iommu_data *data = dev_get_drvdata(dev); struct mtk_iommu_suspend_reg *reg = &data->reg; @@ -848,7 +848,9 @@ static int __maybe_unused mtk_iommu_resume(struct device *dev) } static const struct dev_pm_ops mtk_iommu_pm_ops = { - SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(mtk_iommu_suspend, mtk_iommu_resume) + SET_RUNTIME_PM_OPS(mtk_iommu_runtime_suspend, mtk_iommu_runtime_resume, NULL) + SET_LATE_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, + pm_runtime_force_resume) }; static const struct mtk_iommu_plat_data mt2712_data = { -- cgit v1.2.3 From c0b57581b73be7b43f39e0dff201c93413f6a668 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Mon, 11 Jan 2021 19:19:01 +0800 Subject: iommu/mediatek: Add power-domain operation In the previous SoC, the M4U HW is in the EMI power domain which is always on. the latest M4U is in the display power domain which may be turned on/off, thus we have to add pm_runtime interface for it. When the engine work, the engine always enable the power and clocks for smi-larb/smi-common, then the M4U's power will always be powered on automatically via the device link with smi-common. Note: we don't enable the M4U power in iommu_map/unmap for tlb flush. If its power already is on, of course it is ok. if the power is off, the main tlb will be reset while M4U power on, thus the tlb flush while m4u power off is unnecessary, just skip it. Therefore, we increase the ref_count for pm when pm status is ACTIVE, otherwise, skip it. Meanwhile, the tlb_flush_range is called so often, thus, update pm ref_count while the SoC has power-domain to avoid touch the dev->power.lock. and the tlb_flush_all only is called when boot, so no need check if the SoC has power-domain to keep code clean. There will be one case that pm runctime status is not expected when tlb flush. After boot, the display may call dma_alloc_attrs before it call pm_runtime_get(disp-dev), then the m4u's pm status is not active inside the dma_alloc_attrs. Since it only happens after boot, the tlb is clean at that time, I also think this is ok. Signed-off-by: Yong Wu Reviewed-by: Tomasz Figa Link: https://lore.kernel.org/r/20210111111914.22211-21-yong.wu@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/mtk_iommu.c | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 3682137b789a..b9c63c8de33e 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -182,10 +182,15 @@ static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom) static void mtk_iommu_tlb_flush_all(struct mtk_iommu_data *data) { for_each_m4u(data) { + if (pm_runtime_get_if_in_use(data->dev) <= 0) + continue; + writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, data->base + data->plat_data->inv_sel_reg); writel_relaxed(F_ALL_INVLD, data->base + REG_MMU_INVALIDATE); wmb(); /* Make sure the tlb flush all done */ + + pm_runtime_put(data->dev); } } @@ -193,11 +198,17 @@ static void mtk_iommu_tlb_flush_range_sync(unsigned long iova, size_t size, size_t granule, struct mtk_iommu_data *data) { + bool has_pm = !!data->dev->pm_domain; unsigned long flags; int ret; u32 tmp; for_each_m4u(data) { + if (has_pm) { + if (pm_runtime_get_if_in_use(data->dev) <= 0) + continue; + } + spin_lock_irqsave(&data->tlb_lock, flags); writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, data->base + data->plat_data->inv_sel_reg); @@ -219,6 +230,9 @@ static void mtk_iommu_tlb_flush_range_sync(unsigned long iova, size_t size, /* Clear the CPE status */ writel_relaxed(0, data->base + REG_MMU_CPE_DONE); spin_unlock_irqrestore(&data->tlb_lock, flags); + + if (has_pm) + pm_runtime_put(data->dev); } } @@ -367,18 +381,27 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, { struct mtk_iommu_data *data = dev_iommu_priv_get(dev); struct mtk_iommu_domain *dom = to_mtk_domain(domain); + struct device *m4udev = data->dev; int ret; if (!data) return -ENODEV; if (!data->m4u_dom) { /* Initialize the M4U HW */ + ret = pm_runtime_resume_and_get(m4udev); + if (ret < 0) + return ret; + ret = mtk_iommu_hw_init(data); - if (ret) + if (ret) { + pm_runtime_put(m4udev); return ret; + } data->m4u_dom = dom; writel(dom->cfg.arm_v7s_cfg.ttbr & MMU_PT_ADDR_MASK, data->base + REG_MMU_PT_BASE_ADDR); + + pm_runtime_put(m4udev); } mtk_iommu_config(data, dev, true); @@ -738,11 +761,13 @@ static int mtk_iommu_probe(struct platform_device *pdev) of_node_put(smicomm_node); data->smicomm_dev = &plarbdev->dev; + pm_runtime_enable(dev); + link = device_link_add(data->smicomm_dev, dev, DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME); if (!link) { dev_err(dev, "Unable link %s.\n", dev_name(data->smicomm_dev)); - return -EINVAL; + goto out_runtime_disable; } platform_set_drvdata(pdev, data); @@ -782,6 +807,8 @@ out_sysfs_remove: iommu_device_sysfs_remove(&data->iommu); out_link_remove: device_link_remove(data->smicomm_dev, dev); +out_runtime_disable: + pm_runtime_disable(dev); return ret; } @@ -797,6 +824,7 @@ static int mtk_iommu_remove(struct platform_device *pdev) clk_disable_unprepare(data->bclk); device_link_remove(data->smicomm_dev, &pdev->dev); + pm_runtime_disable(&pdev->dev); devm_free_irq(&pdev->dev, data->irq, data); component_master_del(&pdev->dev, &mtk_iommu_com_ops); return 0; @@ -828,6 +856,9 @@ static int __maybe_unused mtk_iommu_runtime_resume(struct device *dev) void __iomem *base = data->base; int ret; + /* Avoid first resume to affect the default value of registers below. */ + if (!m4u_dom) + return 0; ret = clk_prepare_enable(data->bclk); if (ret) { dev_err(data->dev, "Failed to enable clk(%d) in resume\n", ret); @@ -841,9 +872,7 @@ static int __maybe_unused mtk_iommu_runtime_resume(struct device *dev) writel_relaxed(reg->int_main_control, base + REG_MMU_INT_MAIN_CONTROL); writel_relaxed(reg->ivrp_paddr, base + REG_MMU_IVRP_PADDR); writel_relaxed(reg->vld_pa_rng, base + REG_MMU_VLD_PA_RNG); - if (m4u_dom) - writel(m4u_dom->cfg.arm_v7s_cfg.ttbr & MMU_PT_ADDR_MASK, - base + REG_MMU_PT_BASE_ADDR); + writel(m4u_dom->cfg.arm_v7s_cfg.ttbr & MMU_PT_ADDR_MASK, base + REG_MMU_PT_BASE_ADDR); return 0; } -- cgit v1.2.3 From bfed873114c5fbb8982113abbab17cde7788dcba Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Mon, 11 Jan 2021 19:19:02 +0800 Subject: iommu/mediatek: Support up to 34bit iova in tlb flush If the iova is 34bit, the iova[32][33] is the bit0/1 in the tlb flush register. Add a new macro for this. In the macro, since (iova + size - 1) may be end with 0xfff, then the bit0/1 always is 1, thus add a mask. Signed-off-by: Yong Wu Reviewed-by: Tomasz Figa Link: https://lore.kernel.org/r/20210111111914.22211-22-yong.wu@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/mtk_iommu.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index b9c63c8de33e..468be7ca62e4 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -129,6 +129,11 @@ static const struct iommu_ops mtk_iommu_ops; static int mtk_iommu_hw_init(const struct mtk_iommu_data *data); +#define MTK_IOMMU_TLB_ADDR(iova) ({ \ + dma_addr_t _addr = iova; \ + ((lower_32_bits(_addr) & GENMASK(31, 12)) | upper_32_bits(_addr));\ +}) + /* * In M4U 4GB mode, the physical address is remapped as below: * @@ -213,8 +218,9 @@ static void mtk_iommu_tlb_flush_range_sync(unsigned long iova, size_t size, writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, data->base + data->plat_data->inv_sel_reg); - writel_relaxed(iova, data->base + REG_MMU_INVLD_START_A); - writel_relaxed(iova + size - 1, + writel_relaxed(MTK_IOMMU_TLB_ADDR(iova), + data->base + REG_MMU_INVLD_START_A); + writel_relaxed(MTK_IOMMU_TLB_ADDR(iova + size - 1), data->base + REG_MMU_INVLD_END_A); writel_relaxed(F_MMU_INV_RANGE, data->base + REG_MMU_INVALIDATE); -- cgit v1.2.3 From ef0f0986b670cb1627bff055102cab70a7d3852a Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Mon, 11 Jan 2021 19:19:03 +0800 Subject: iommu/mediatek: Support report iova 34bit translation fault in ISR If the iova is over 32bit, the fault status register bit is a little different. Signed-off-by: Yong Wu Reviewed-by: Tomasz Figa Link: https://lore.kernel.org/r/20210111111914.22211-23-yong.wu@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/mtk_iommu.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 468be7ca62e4..f3666b0d7577 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -3,6 +3,7 @@ * Copyright (c) 2015-2016 MediaTek Inc. * Author: Yong Wu */ +#include #include #include #include @@ -89,6 +90,9 @@ #define F_REG_MMU1_FAULT_MASK GENMASK(13, 7) #define REG_MMU0_FAULT_VA 0x13c +#define F_MMU_INVAL_VA_31_12_MASK GENMASK(31, 12) +#define F_MMU_INVAL_VA_34_32_MASK GENMASK(11, 9) +#define F_MMU_INVAL_PA_34_32_MASK GENMASK(8, 6) #define F_MMU_FAULT_VA_WRITE_BIT BIT(1) #define F_MMU_FAULT_VA_LAYER_BIT BIT(0) @@ -246,8 +250,9 @@ static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) { struct mtk_iommu_data *data = dev_id; struct mtk_iommu_domain *dom = data->m4u_dom; - u32 int_state, regval, fault_iova, fault_pa; unsigned int fault_larb, fault_port, sub_comm = 0; + u32 int_state, regval, va34_32, pa34_32; + u64 fault_iova, fault_pa; bool layer, write; /* Read error info from registers */ @@ -263,6 +268,14 @@ static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) } layer = fault_iova & F_MMU_FAULT_VA_LAYER_BIT; write = fault_iova & F_MMU_FAULT_VA_WRITE_BIT; + if (MTK_IOMMU_HAS_FLAG(data->plat_data, IOVA_34_EN)) { + va34_32 = FIELD_GET(F_MMU_INVAL_VA_34_32_MASK, fault_iova); + pa34_32 = FIELD_GET(F_MMU_INVAL_PA_34_32_MASK, fault_iova); + fault_iova = fault_iova & F_MMU_INVAL_VA_31_12_MASK; + fault_iova |= (u64)va34_32 << 32; + fault_pa |= (u64)pa34_32 << 32; + } + fault_port = F_MMU_INT_ID_PORT_ID(regval); if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_SUB_COMM)) { fault_larb = F_MMU_INT_ID_COMM_ID(regval); @@ -276,7 +289,7 @@ static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) write ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ)) { dev_err_ratelimited( data->dev, - "fault type=0x%x iova=0x%x pa=0x%x larb=%d port=%d layer=%d %s\n", + "fault type=0x%x iova=0x%llx pa=0x%llx larb=%d port=%d layer=%d %s\n", int_state, fault_iova, fault_pa, fault_larb, fault_port, layer, write ? "write" : "read"); } -- cgit v1.2.3 From 08500c43d4f70c1e8be6beda8a2aea0ecfb845d8 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Mon, 11 Jan 2021 19:19:04 +0800 Subject: iommu/mediatek: Adjust the structure Add "struct mtk_iommu_data *" in the "struct mtk_iommu_domain", reduce the call mtk_iommu_get_m4u_data(). No functional change. Signed-off-by: Yong Wu Reviewed-by: Tomasz Figa Link: https://lore.kernel.org/r/20210111111914.22211-24-yong.wu@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/mtk_iommu.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index f3666b0d7577..f1941608ccb7 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -126,6 +126,7 @@ struct mtk_iommu_domain { struct io_pgtable_cfg cfg; struct io_pgtable_ops *iop; + struct mtk_iommu_data *data; struct iommu_domain domain; }; @@ -351,6 +352,7 @@ static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom) return -EINVAL; } + dom->data = data; /* Update our support page sizes bitmap */ dom->domain.pgsize_bitmap = dom->cfg.pgsize_bitmap; return 0; @@ -442,10 +444,9 @@ static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot, gfp_t gfp) { struct mtk_iommu_domain *dom = to_mtk_domain(domain); - struct mtk_iommu_data *data = mtk_iommu_get_m4u_data(); /* The "4GB mode" M4U physically can not use the lower remap of Dram. */ - if (data->enable_4GB) + if (dom->data->enable_4GB) paddr |= BIT_ULL(32); /* Synchronize with the tlb_lock */ @@ -468,36 +469,37 @@ static size_t mtk_iommu_unmap(struct iommu_domain *domain, static void mtk_iommu_flush_iotlb_all(struct iommu_domain *domain) { - mtk_iommu_tlb_flush_all(mtk_iommu_get_m4u_data()); + struct mtk_iommu_domain *dom = to_mtk_domain(domain); + + mtk_iommu_tlb_flush_all(dom->data); } static void mtk_iommu_iotlb_sync(struct iommu_domain *domain, struct iommu_iotlb_gather *gather) { - struct mtk_iommu_data *data = mtk_iommu_get_m4u_data(); + struct mtk_iommu_domain *dom = to_mtk_domain(domain); size_t length = gather->end - gather->start + 1; mtk_iommu_tlb_flush_range_sync(gather->start, length, gather->pgsize, - data); + dom->data); } static void mtk_iommu_sync_map(struct iommu_domain *domain, unsigned long iova, size_t size) { - struct mtk_iommu_data *data = mtk_iommu_get_m4u_data(); + struct mtk_iommu_domain *dom = to_mtk_domain(domain); - mtk_iommu_tlb_flush_range_sync(iova, size, size, data); + mtk_iommu_tlb_flush_range_sync(iova, size, size, dom->data); } static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) { struct mtk_iommu_domain *dom = to_mtk_domain(domain); - struct mtk_iommu_data *data = mtk_iommu_get_m4u_data(); phys_addr_t pa; pa = dom->iop->iova_to_phys(dom->iop, iova); - if (data->enable_4GB && pa >= MTK_IOMMU_4GB_MODE_REMAP_BASE) + if (dom->data->enable_4GB && pa >= MTK_IOMMU_4GB_MODE_REMAP_BASE) pa &= ~BIT_ULL(32); return pa; -- cgit v1.2.3 From 4f956c97d26be65aea580e943f470ec70f7b0bb6 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Mon, 11 Jan 2021 19:19:05 +0800 Subject: iommu/mediatek: Move domain_finalise into attach_device Currently domain_alloc only has a parameter(type), We have no chance to input some special data. This patch moves the domain_finalise into attach_device which has the device information, then could update the domain's geometry.aperture ranges for each a device. Strictly, I should use the data from mtk_iommu_get_m4u_data as the parameter of mtk_iommu_domain_finalise in this patch. but dom->data only is used in tlb ops in which the data is get from the m4u_list, thus it is ok here. Signed-off-by: Yong Wu Reviewed-by: Tomasz Figa Link: https://lore.kernel.org/r/20210111111914.22211-25-yong.wu@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/mtk_iommu.c | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index f1941608ccb7..d321d09ac4c2 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -328,10 +328,9 @@ static void mtk_iommu_config(struct mtk_iommu_data *data, } } -static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom) +static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom, + struct mtk_iommu_data *data) { - struct mtk_iommu_data *data = mtk_iommu_get_m4u_data(); - dom->cfg = (struct io_pgtable_cfg) { .quirks = IO_PGTABLE_QUIRK_ARM_NS | IO_PGTABLE_QUIRK_NO_PERMS | @@ -352,7 +351,6 @@ static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom) return -EINVAL; } - dom->data = data; /* Update our support page sizes bitmap */ dom->domain.pgsize_bitmap = dom->cfg.pgsize_bitmap; return 0; @@ -369,30 +367,19 @@ static struct iommu_domain *mtk_iommu_domain_alloc(unsigned type) if (!dom) return NULL; - if (iommu_get_dma_cookie(&dom->domain)) - goto free_dom; - - if (mtk_iommu_domain_finalise(dom)) - goto put_dma_cookie; + if (iommu_get_dma_cookie(&dom->domain)) { + kfree(dom); + return NULL; + } dom->domain.geometry.aperture_start = 0; dom->domain.geometry.aperture_end = DMA_BIT_MASK(32); dom->domain.geometry.force_aperture = true; - return &dom->domain; - -put_dma_cookie: - iommu_put_dma_cookie(&dom->domain); -free_dom: - kfree(dom); - return NULL; } static void mtk_iommu_domain_free(struct iommu_domain *domain) { - struct mtk_iommu_domain *dom = to_mtk_domain(domain); - - free_io_pgtable_ops(dom->iop); iommu_put_dma_cookie(domain); kfree(to_mtk_domain(domain)); } @@ -408,6 +395,12 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, if (!data) return -ENODEV; + if (!dom->data) { + if (mtk_iommu_domain_finalise(dom, data)) + return -ENODEV; + dom->data = data; + } + if (!data->m4u_dom) { /* Initialize the M4U HW */ ret = pm_runtime_resume_and_get(m4udev); if (ret < 0) -- cgit v1.2.3 From b7875eb9458983ade0c780276b21cfbc0c22d70b Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Mon, 11 Jan 2021 19:19:06 +0800 Subject: iommu/mediatek: Move geometry.aperture updating into domain_finalise Move the domain geometry.aperture updating into domain_finalise. This is a preparing patch for updating the domain region. We know the detailed iova region in the attach_device. Signed-off-by: Yong Wu Reviewed-by: Tomasz Figa Link: https://lore.kernel.org/r/20210111111914.22211-26-yong.wu@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/mtk_iommu.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index d321d09ac4c2..309b06d5e1f9 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -353,6 +353,10 @@ static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom, /* Update our support page sizes bitmap */ dom->domain.pgsize_bitmap = dom->cfg.pgsize_bitmap; + + dom->domain.geometry.aperture_start = 0; + dom->domain.geometry.aperture_end = DMA_BIT_MASK(32); + dom->domain.geometry.force_aperture = true; return 0; } @@ -372,9 +376,6 @@ static struct iommu_domain *mtk_iommu_domain_alloc(unsigned type) return NULL; } - dom->domain.geometry.aperture_start = 0; - dom->domain.geometry.aperture_end = DMA_BIT_MASK(32); - dom->domain.geometry.force_aperture = true; return &dom->domain; } -- cgit v1.2.3 From 585e58f498a2855dc7a8d351df34dd645447f337 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Mon, 11 Jan 2021 19:19:07 +0800 Subject: iommu/mediatek: Add iova_region structure Add a new structure for the iova_region. Each a region will be a independent iommu domain. For the previous SoC, there is single iova region(0~4G). For the SoC that need support multi-domains, there will be several regions. Signed-off-by: Yong Wu Reviewed-by: Tomasz Figa Link: https://lore.kernel.org/r/20210111111914.22211-27-yong.wu@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/mtk_iommu.c | 19 +++++++++++++++++++ drivers/iommu/mtk_iommu.h | 5 +++++ 2 files changed, 24 insertions(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 309b06d5e1f9..6875ca1225f0 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -167,6 +167,15 @@ static LIST_HEAD(m4ulist); /* List all the M4U HWs */ #define for_each_m4u(data) list_for_each_entry(data, &m4ulist, list) +struct mtk_iommu_iova_region { + dma_addr_t iova_base; + unsigned long long size; +}; + +static const struct mtk_iommu_iova_region single_domain[] = { + {.iova_base = 0, .size = SZ_4G}, +}; + /* * There may be 1 or 2 M4U HWs, But we always expect they are in the same domain * for the performance. @@ -901,6 +910,8 @@ static const struct mtk_iommu_plat_data mt2712_data = { .m4u_plat = M4U_MT2712, .flags = HAS_4GB_MODE | HAS_BCLK | HAS_VLD_PA_RNG, .inv_sel_reg = REG_MMU_INV_SEL_GEN1, + .iova_region = single_domain, + .iova_region_nr = ARRAY_SIZE(single_domain), .larbid_remap = {{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}}, }; @@ -908,6 +919,8 @@ static const struct mtk_iommu_plat_data mt6779_data = { .m4u_plat = M4U_MT6779, .flags = HAS_SUB_COMM | OUT_ORDER_WR_EN | WR_THROT_EN, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, + .iova_region = single_domain, + .iova_region_nr = ARRAY_SIZE(single_domain), .larbid_remap = {{0}, {1}, {2}, {3}, {5}, {7, 8}, {10}, {9}}, }; @@ -915,6 +928,8 @@ static const struct mtk_iommu_plat_data mt8167_data = { .m4u_plat = M4U_MT8167, .flags = RESET_AXI | HAS_LEGACY_IVRP_PADDR, .inv_sel_reg = REG_MMU_INV_SEL_GEN1, + .iova_region = single_domain, + .iova_region_nr = ARRAY_SIZE(single_domain), .larbid_remap = {{0}, {1}, {2}}, /* Linear mapping. */ }; @@ -923,6 +938,8 @@ static const struct mtk_iommu_plat_data mt8173_data = { .flags = HAS_4GB_MODE | HAS_BCLK | RESET_AXI | HAS_LEGACY_IVRP_PADDR, .inv_sel_reg = REG_MMU_INV_SEL_GEN1, + .iova_region = single_domain, + .iova_region_nr = ARRAY_SIZE(single_domain), .larbid_remap = {{0}, {1}, {2}, {3}, {4}, {5}}, /* Linear mapping. */ }; @@ -930,6 +947,8 @@ static const struct mtk_iommu_plat_data mt8183_data = { .m4u_plat = M4U_MT8183, .flags = RESET_AXI, .inv_sel_reg = REG_MMU_INV_SEL_GEN1, + .iova_region = single_domain, + .iova_region_nr = ARRAY_SIZE(single_domain), .larbid_remap = {{0}, {4}, {5}, {6}, {7}, {2}, {3}, {1}}, }; diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h index a9b79e118f02..118170af1974 100644 --- a/drivers/iommu/mtk_iommu.h +++ b/drivers/iommu/mtk_iommu.h @@ -45,10 +45,15 @@ enum mtk_iommu_plat { M4U_MT8183, }; +struct mtk_iommu_iova_region; + struct mtk_iommu_plat_data { enum mtk_iommu_plat m4u_plat; u32 flags; u32 inv_sel_reg; + + unsigned int iova_region_nr; + const struct mtk_iommu_iova_region *iova_region; unsigned char larbid_remap[MTK_LARB_COM_MAX][MTK_LARB_SUBCOM_MAX]; }; -- cgit v1.2.3 From 803cf9e5a6aa64e94f9c554190bc0031929f6857 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Mon, 11 Jan 2021 19:19:08 +0800 Subject: iommu/mediatek: Add get_domain_id from dev->dma_range_map Add a new interface _get_domain_id from dev->dma_range_map, The iommu consumer device will use dma-ranges in dtsi node to indicate its dma address region requirement. In this iommu driver, we will get the requirement and decide which iova domain it should locate. In the lastest SoC, there will be several iova-regions(domains), we will compare and calculate which domain is right. If the start/end of device requirement equal some region. it is best fit of course. If it is inside some region, it is also ok. the iova requirement of a device should not be inside two or more regions. Signed-off-by: Yong Wu Reviewed-by: Tomasz Figa Link: https://lore.kernel.org/r/20210111111914.22211-28-yong.wu@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/mtk_iommu.c | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 6875ca1225f0..8fc17158bc28 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -314,6 +315,36 @@ static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) return IRQ_HANDLED; } +static int mtk_iommu_get_domain_id(struct device *dev, + const struct mtk_iommu_plat_data *plat_data) +{ + const struct mtk_iommu_iova_region *rgn = plat_data->iova_region; + const struct bus_dma_region *dma_rgn = dev->dma_range_map; + int i, candidate = -1; + dma_addr_t dma_end; + + if (!dma_rgn || plat_data->iova_region_nr == 1) + return 0; + + dma_end = dma_rgn->dma_start + dma_rgn->size - 1; + for (i = 0; i < plat_data->iova_region_nr; i++, rgn++) { + /* Best fit. */ + if (dma_rgn->dma_start == rgn->iova_base && + dma_end == rgn->iova_base + rgn->size - 1) + return i; + /* ok if it is inside this region. */ + if (dma_rgn->dma_start >= rgn->iova_base && + dma_end < rgn->iova_base + rgn->size) + candidate = i; + } + + if (candidate >= 0) + return candidate; + dev_err(dev, "Can NOT find the iommu domain id(%pad 0x%llx).\n", + &dma_rgn->dma_start, dma_rgn->size); + return -EINVAL; +} + static void mtk_iommu_config(struct mtk_iommu_data *data, struct device *dev, bool enable) { @@ -400,11 +431,15 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, struct mtk_iommu_data *data = dev_iommu_priv_get(dev); struct mtk_iommu_domain *dom = to_mtk_domain(domain); struct device *m4udev = data->dev; - int ret; + int ret, domid; if (!data) return -ENODEV; + domid = mtk_iommu_get_domain_id(dev, data->plat_data); + if (domid < 0) + return domid; + if (!dom->data) { if (mtk_iommu_domain_finalise(dom, data)) return -ENODEV; @@ -534,10 +569,15 @@ static void mtk_iommu_release_device(struct device *dev) static struct iommu_group *mtk_iommu_device_group(struct device *dev) { struct mtk_iommu_data *data = mtk_iommu_get_m4u_data(); + int domid; if (!data) return ERR_PTR(-ENODEV); + domid = mtk_iommu_get_domain_id(dev, data->plat_data); + if (domid < 0) + return ERR_PTR(domid); + /* All the client devices are in the same m4u iommu-group */ if (!data->m4u_group) { data->m4u_group = iommu_group_alloc(); -- cgit v1.2.3 From c3045f39244e90c4c45a404e35aa66403ca68815 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Mon, 11 Jan 2021 19:19:09 +0800 Subject: iommu/mediatek: Support for multi domains Some HW IP(ex: CCU) require the special iova range. That means the iova got from dma_alloc_attrs for that devices must locate in his special range. In this patch, we prepare a iommu group(domain) for each a iova range requirement. Meanwhile we still use one pagetable which support 16GB iova. After this patch, If the iova range of a master is over 4G, the master should: a) Declare its special dma-ranges in its dtsi node. For example, If we preassign the iova 4G-8G for vcodec, then the vcodec dtsi node should add this: /* * iova start at 0x1_0000_0000, pa still start at 0x4000_0000 * size is 0x1_0000_0000. */ dma-ranges = <0x1 0x0 0x0 0x40000000 0x1 0x0>; /* 4G ~ 8G */ Note: we don't have a actual bus concept here. the master doesn't have its special parent node, thus this dma-ranges can only be put in the master's node. b) Update the dma_mask: dma_set_mask_and_coherent(dev, DMA_BIT_MASK(33)); Signed-off-by: Yong Wu Reviewed-by: Tomasz Figa Link: https://lore.kernel.org/r/20210111111914.22211-29-yong.wu@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/mtk_iommu.c | 37 ++++++++++++++++++++++++++----------- drivers/iommu/mtk_iommu.h | 4 +++- 2 files changed, 29 insertions(+), 12 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 8fc17158bc28..b42fd2535b77 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -369,8 +369,19 @@ static void mtk_iommu_config(struct mtk_iommu_data *data, } static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom, - struct mtk_iommu_data *data) + struct mtk_iommu_data *data, + unsigned int domid) { + const struct mtk_iommu_iova_region *region; + + /* Use the exist domain as there is only one pgtable here. */ + if (data->m4u_dom) { + dom->iop = data->m4u_dom->iop; + dom->cfg = data->m4u_dom->cfg; + dom->domain.pgsize_bitmap = data->m4u_dom->cfg.pgsize_bitmap; + goto update_iova_region; + } + dom->cfg = (struct io_pgtable_cfg) { .quirks = IO_PGTABLE_QUIRK_ARM_NS | IO_PGTABLE_QUIRK_NO_PERMS | @@ -394,8 +405,11 @@ static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom, /* Update our support page sizes bitmap */ dom->domain.pgsize_bitmap = dom->cfg.pgsize_bitmap; - dom->domain.geometry.aperture_start = 0; - dom->domain.geometry.aperture_end = DMA_BIT_MASK(32); +update_iova_region: + /* Update the iova region for this domain */ + region = data->plat_data->iova_region + domid; + dom->domain.geometry.aperture_start = region->iova_base; + dom->domain.geometry.aperture_end = region->iova_base + region->size - 1; dom->domain.geometry.force_aperture = true; return 0; } @@ -441,7 +455,7 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, return domid; if (!dom->data) { - if (mtk_iommu_domain_finalise(dom, data)) + if (mtk_iommu_domain_finalise(dom, data, domid)) return -ENODEV; dom->data = data; } @@ -569,6 +583,7 @@ static void mtk_iommu_release_device(struct device *dev) static struct iommu_group *mtk_iommu_device_group(struct device *dev) { struct mtk_iommu_data *data = mtk_iommu_get_m4u_data(); + struct iommu_group *group; int domid; if (!data) @@ -578,15 +593,15 @@ static struct iommu_group *mtk_iommu_device_group(struct device *dev) if (domid < 0) return ERR_PTR(domid); - /* All the client devices are in the same m4u iommu-group */ - if (!data->m4u_group) { - data->m4u_group = iommu_group_alloc(); - if (IS_ERR(data->m4u_group)) - dev_err(dev, "Failed to allocate M4U IOMMU group\n"); + group = data->m4u_group[domid]; + if (!group) { + group = iommu_group_alloc(); + if (!IS_ERR(group)) + data->m4u_group[domid] = group; } else { - iommu_group_ref_get(data->m4u_group); + iommu_group_ref_get(group); } - return data->m4u_group; + return group; } static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h index 118170af1974..6f2168e3222d 100644 --- a/drivers/iommu/mtk_iommu.h +++ b/drivers/iommu/mtk_iommu.h @@ -22,6 +22,8 @@ #define MTK_LARB_COM_MAX 8 #define MTK_LARB_SUBCOM_MAX 4 +#define MTK_IOMMU_GROUP_MAX 8 + struct mtk_iommu_suspend_reg { union { u32 standard_axi_mode;/* v1 */ @@ -67,7 +69,7 @@ struct mtk_iommu_data { phys_addr_t protect_base; /* protect memory base */ struct mtk_iommu_suspend_reg reg; struct mtk_iommu_domain *m4u_dom; - struct iommu_group *m4u_group; + struct iommu_group *m4u_group[MTK_IOMMU_GROUP_MAX]; bool enable_4GB; spinlock_t tlb_lock; /* lock for tlb range flush */ -- cgit v1.2.3 From ab1d5281a62bafbd611fffdce7eab6eb9577aa0d Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Mon, 11 Jan 2021 19:19:10 +0800 Subject: iommu/mediatek: Add iova reserved function For multiple iommu_domains, we need to reserve some iova regions. Take a example, If the default iova region is 0 ~ 4G, but the 0x4000_0000 ~ 0x43ff_ffff is only for the special CCU0 domain. Thus we should exclude this region for the default iova region. Signed-off-by: Anan sun Signed-off-by: Chao Hao Signed-off-by: Yong Wu Reviewed-by: Tomasz Figa Link: https://lore.kernel.org/r/20210111111914.22211-30-yong.wu@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/mtk_iommu.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index b42fd2535b77..764dc0b93477 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -626,6 +626,35 @@ static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) return iommu_fwspec_add_ids(dev, args->args, 1); } +static void mtk_iommu_get_resv_regions(struct device *dev, + struct list_head *head) +{ + struct mtk_iommu_data *data = dev_iommu_priv_get(dev); + unsigned int domid = mtk_iommu_get_domain_id(dev, data->plat_data), i; + const struct mtk_iommu_iova_region *resv, *curdom; + struct iommu_resv_region *region; + int prot = IOMMU_WRITE | IOMMU_READ; + + if (domid < 0) + return; + curdom = data->plat_data->iova_region + domid; + for (i = 0; i < data->plat_data->iova_region_nr; i++) { + resv = data->plat_data->iova_region + i; + + /* Only reserve when the region is inside the current domain */ + if (resv->iova_base <= curdom->iova_base || + resv->iova_base + resv->size >= curdom->iova_base + curdom->size) + continue; + + region = iommu_alloc_resv_region(resv->iova_base, resv->size, + prot, IOMMU_RESV_RESERVED); + if (!region) + return; + + list_add_tail(®ion->list, head); + } +} + static const struct iommu_ops mtk_iommu_ops = { .domain_alloc = mtk_iommu_domain_alloc, .domain_free = mtk_iommu_domain_free, @@ -641,6 +670,8 @@ static const struct iommu_ops mtk_iommu_ops = { .release_device = mtk_iommu_release_device, .device_group = mtk_iommu_device_group, .of_xlate = mtk_iommu_of_xlate, + .get_resv_regions = mtk_iommu_get_resv_regions, + .put_resv_regions = generic_iommu_put_resv_regions, .pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M, }; -- cgit v1.2.3 From 8d2c749e5252466d8fc1917101f137966c61fd1c Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Mon, 11 Jan 2021 19:19:11 +0800 Subject: iommu/mediatek: Support master use iova over 32bit After extending v7s, our pagetable already support iova reach 16GB(34bit). the master got the iova via dma_alloc_attrs may reach 34bits, but its HW register still is 32bit. then how to set the bit32/bit33 iova? this depend on a SMI larb setting(bank_sel). we separate whole 16GB iova to four banks: bank: 0: 0~4G; 1: 4~8G; 2: 8-12G; 3: 12-16G; The bank number is (iova >> 32). We will preassign which bank the larbs belong to. currently we don't have a interface for master to adjust its bank number. Each a bank is a iova_region which is a independent iommu-domain. the iova range for each iommu-domain can't cross 4G. Signed-off-by: Yong Wu Acked-by: Krzysztof Kozlowski #for memory part Reviewed-by: Tomasz Figa Link: https://lore.kernel.org/r/20210111111914.22211-31-yong.wu@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/mtk_iommu.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 764dc0b93477..7403a7cb90ea 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -345,21 +345,27 @@ static int mtk_iommu_get_domain_id(struct device *dev, return -EINVAL; } -static void mtk_iommu_config(struct mtk_iommu_data *data, - struct device *dev, bool enable) +static void mtk_iommu_config(struct mtk_iommu_data *data, struct device *dev, + bool enable, unsigned int domid) { struct mtk_smi_larb_iommu *larb_mmu; unsigned int larbid, portid; struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + const struct mtk_iommu_iova_region *region; int i; for (i = 0; i < fwspec->num_ids; ++i) { larbid = MTK_M4U_TO_LARB(fwspec->ids[i]); portid = MTK_M4U_TO_PORT(fwspec->ids[i]); + larb_mmu = &data->larb_imu[larbid]; - dev_dbg(dev, "%s iommu port: %d\n", - enable ? "enable" : "disable", portid); + region = data->plat_data->iova_region + domid; + larb_mmu->bank[portid] = upper_32_bits(region->iova_base); + + dev_dbg(dev, "%s iommu for larb(%s) port %d dom %d bank %d.\n", + enable ? "enable" : "disable", dev_name(larb_mmu->dev), + portid, domid, larb_mmu->bank[portid]); if (enable) larb_mmu->mmu |= MTK_SMI_MMU_EN(portid); @@ -477,7 +483,7 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, pm_runtime_put(m4udev); } - mtk_iommu_config(data, dev, true); + mtk_iommu_config(data, dev, true, domid); return 0; } @@ -489,7 +495,7 @@ static void mtk_iommu_detach_device(struct iommu_domain *domain, if (!data) return; - mtk_iommu_config(data, dev, false); + mtk_iommu_config(data, dev, false, 0); } static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova, -- cgit v1.2.3 From 23357572bec6b7f900d855f6dda9c77bb6d1c2de Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Mon, 11 Jan 2021 19:19:12 +0800 Subject: iommu/mediatek: Remove unnecessary check in attach_device This priv_data is set in the of_xlate. if of_xlate failed, it should not enter attach_device. remove the unnecessary check. Signed-off-by: Yong Wu Reviewed-by: Tomasz Figa Link: https://lore.kernel.org/r/20210111111914.22211-32-yong.wu@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/mtk_iommu.c | 6 ------ 1 file changed, 6 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 7403a7cb90ea..084fb4394ffc 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -453,9 +453,6 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, struct device *m4udev = data->dev; int ret, domid; - if (!data) - return -ENODEV; - domid = mtk_iommu_get_domain_id(dev, data->plat_data); if (domid < 0) return domid; @@ -492,9 +489,6 @@ static void mtk_iommu_detach_device(struct iommu_domain *domain, { struct mtk_iommu_data *data = dev_iommu_priv_get(dev); - if (!data) - return; - mtk_iommu_config(data, dev, false, 0); } -- cgit v1.2.3 From 9e3489e06fb94a7908c3c8f098c191ff711a8182 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Mon, 11 Jan 2021 19:19:13 +0800 Subject: iommu/mediatek: Add mt8192 support Add mt8192 iommu support. Signed-off-by: Yong Wu Reviewed-by: Tomasz Figa Link: https://lore.kernel.org/r/20210111111914.22211-33-yong.wu@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/mtk_iommu.c | 22 ++++++++++++++++++++++ drivers/iommu/mtk_iommu.h | 1 + 2 files changed, 23 insertions(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 084fb4394ffc..0ad14a7604b1 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -177,6 +177,16 @@ static const struct mtk_iommu_iova_region single_domain[] = { {.iova_base = 0, .size = SZ_4G}, }; +static const struct mtk_iommu_iova_region mt8192_multi_dom[] = { + { .iova_base = 0x0, .size = SZ_4G}, /* disp: 0 ~ 4G */ + #if IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) + { .iova_base = SZ_4G, .size = SZ_4G}, /* vdec: 4G ~ 8G */ + { .iova_base = SZ_4G * 2, .size = SZ_4G}, /* CAM/MDP: 8G ~ 12G */ + { .iova_base = 0x240000000ULL, .size = 0x4000000}, /* CCU0 */ + { .iova_base = 0x244000000ULL, .size = 0x4000000}, /* CCU1 */ + #endif +}; + /* * There may be 1 or 2 M4U HWs, But we always expect they are in the same domain * for the performance. @@ -1038,12 +1048,24 @@ static const struct mtk_iommu_plat_data mt8183_data = { .larbid_remap = {{0}, {4}, {5}, {6}, {7}, {2}, {3}, {1}}, }; +static const struct mtk_iommu_plat_data mt8192_data = { + .m4u_plat = M4U_MT8192, + .flags = HAS_BCLK | HAS_SUB_COMM | OUT_ORDER_WR_EN | + WR_THROT_EN | IOVA_34_EN, + .inv_sel_reg = REG_MMU_INV_SEL_GEN2, + .iova_region = mt8192_multi_dom, + .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom), + .larbid_remap = {{0}, {1}, {4, 5}, {7}, {2}, {9, 11, 19, 20}, + {0, 14, 16}, {0, 13, 18, 17}}, +}; + static const struct of_device_id mtk_iommu_of_ids[] = { { .compatible = "mediatek,mt2712-m4u", .data = &mt2712_data}, { .compatible = "mediatek,mt6779-m4u", .data = &mt6779_data}, { .compatible = "mediatek,mt8167-m4u", .data = &mt8167_data}, { .compatible = "mediatek,mt8173-m4u", .data = &mt8173_data}, { .compatible = "mediatek,mt8183-m4u", .data = &mt8183_data}, + { .compatible = "mediatek,mt8192-m4u", .data = &mt8192_data}, {} }; diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h index 6f2168e3222d..f81fa8862ed0 100644 --- a/drivers/iommu/mtk_iommu.h +++ b/drivers/iommu/mtk_iommu.h @@ -45,6 +45,7 @@ enum mtk_iommu_plat { M4U_MT8167, M4U_MT8173, M4U_MT8183, + M4U_MT8192, }; struct mtk_iommu_iova_region; -- cgit v1.2.3 From e1ed66ac300373a8d7e8d95fd86ca522f36602d9 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Sat, 30 Jan 2021 23:19:07 +0800 Subject: iommu/vt-d: Fix compile error [-Werror=implicit-function-declaration] trace_qi_submit() could be used when interrupt remapping is supported, but DMA remapping is not. In this case, the following compile error occurs. ../drivers/iommu/intel/dmar.c: In function 'qi_submit_sync': ../drivers/iommu/intel/dmar.c:1311:3: error: implicit declaration of function 'trace_qi_submit'; did you mean 'ftrace_nmi_exit'? [-Werror=implicit-function-declaration] trace_qi_submit(iommu, desc[i].qw0, desc[i].qw1, ^~~~~~~~~~~~~~~ ftrace_nmi_exit Fixes: f2dd871799ba5 ("iommu/vt-d: Add qi_submit trace event") Reported-by: kernel test robot Reported-by: Randy Dunlap Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20210130151907.3929148-1-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/Makefile | 2 +- drivers/iommu/intel/iommu.c | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/Makefile b/drivers/iommu/intel/Makefile index fb8e1e8c8029..ae570810a35e 100644 --- a/drivers/iommu/intel/Makefile +++ b/drivers/iommu/intel/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DMAR_TABLE) += dmar.o obj-$(CONFIG_INTEL_IOMMU) += iommu.o pasid.o -obj-$(CONFIG_INTEL_IOMMU) += trace.o +obj-$(CONFIG_DMAR_TABLE) += trace.o obj-$(CONFIG_INTEL_IOMMU_DEBUGFS) += debugfs.o obj-$(CONFIG_INTEL_IOMMU_SVM) += svm.o obj-$(CONFIG_IRQ_REMAP) += irq_remapping.o diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index be85af612bc1..87d0ef79ba6a 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -44,7 +44,6 @@ #include #include #include -#include #include "../irq_remapping.h" #include "pasid.h" -- cgit v1.2.3 From b8437a3ef8c485903d05d1f261328aaf0c0a6cc2 Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Mon, 1 Feb 2021 17:06:23 -0800 Subject: iommu: Properly pass gfp_t in _iommu_map() to avoid atomic sleeping Sleeping while atomic = bad. Let's fix an obvious typo to try to avoid it. The warning that was seen (on a downstream kernel with the problematic patch backported): BUG: sleeping function called from invalid context at mm/page_alloc.c:4726 in_atomic(): 1, irqs_disabled(): 0, non_block: 0, pid: 9, name: ksoftirqd/0 CPU: 0 PID: 9 Comm: ksoftirqd/0 Not tainted 5.4.93-12508-gc10c93e28e39 #1 Call trace: dump_backtrace+0x0/0x154 show_stack+0x20/0x2c dump_stack+0xa0/0xfc ___might_sleep+0x11c/0x12c __might_sleep+0x50/0x84 __alloc_pages_nodemask+0xf8/0x2bc __arm_lpae_alloc_pages+0x48/0x1b4 __arm_lpae_map+0x124/0x274 __arm_lpae_map+0x1cc/0x274 arm_lpae_map+0x140/0x170 arm_smmu_map+0x78/0xbc __iommu_map+0xd4/0x210 _iommu_map+0x4c/0x84 iommu_map_atomic+0x44/0x58 __iommu_dma_map+0x8c/0xc4 iommu_dma_map_page+0xac/0xf0 Fixes: d8c1df02ac7f ("iommu: Move iotlb_sync_map out from __iommu_map") Signed-off-by: Douglas Anderson Reviewed-by: Yong Wu Acked-by: Will Deacon Link: https://lore.kernel.org/r/20210201170611.1.I64a7b62579287d668d7c89e105dcedf45d641063@changeid Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 3d099a31ddca..2b06b01850d5 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2441,7 +2441,7 @@ static int _iommu_map(struct iommu_domain *domain, unsigned long iova, const struct iommu_ops *ops = domain->ops; int ret; - ret = __iommu_map(domain, iova, paddr, size, prot, GFP_KERNEL); + ret = __iommu_map(domain, iova, paddr, size, prot, gfp); if (ret == 0 && ops->iotlb_sync_map) ops->iotlb_sync_map(domain, iova, size); -- cgit v1.2.3 From ad3d19029979b19378ece2011fc8ce07be98c905 Mon Sep 17 00:00:00 2001 From: Kyung Min Park Date: Thu, 4 Feb 2021 09:43:56 +0800 Subject: iommu/vt-d: Audit IOMMU Capabilities and add helper functions Audit IOMMU Capability/Extended Capability and check if the IOMMUs have the consistent value for features. Report out or scale to the lowest supported when IOMMU features have incompatibility among IOMMUs. Report out features when below features are mismatched: - First Level 5 Level Paging Support (FL5LP) - First Level 1 GByte Page Support (FL1GP) - Read Draining (DRD) - Write Draining (DWD) - Page Selective Invalidation (PSI) - Zero Length Read (ZLR) - Caching Mode (CM) - Protected High/Low-Memory Region (PHMR/PLMR) - Required Write-Buffer Flushing (RWBF) - Advanced Fault Logging (AFL) - RID-PASID Support (RPS) - Scalable Mode Page Walk Coherency (SMPWC) - First Level Translation Support (FLTS) - Second Level Translation Support (SLTS) - No Write Flag Support (NWFS) - Second Level Accessed/Dirty Support (SLADS) - Virtual Command Support (VCS) - Scalable Mode Translation Support (SMTS) - Device TLB Invalidation Throttle (DIT) - Page Drain Support (PDS) - Process Address Space ID Support (PASID) - Extended Accessed Flag Support (EAFS) - Supervisor Request Support (SRS) - Execute Request Support (ERS) - Page Request Support (PRS) - Nested Translation Support (NEST) - Snoop Control (SC) - Pass Through (PT) - Device TLB Support (DT) - Queued Invalidation (QI) - Page walk Coherency (C) Set capability to the lowest supported when below features are mismatched: - Maximum Address Mask Value (MAMV) - Number of Fault Recording Registers (NFR) - Second Level Large Page Support (SLLPS) - Fault Recording Offset (FRO) - Maximum Guest Address Width (MGAW) - Supported Adjusted Guest Address Width (SAGAW) - Number of Domains supported (NDOMS) - Pasid Size Supported (PSS) - Maximum Handle Mask Value (MHMV) - IOTLB Register Offset (IRO) Signed-off-by: Kyung Min Park Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20210130184452.31711-1-kyung.min.park@intel.com Link: https://lore.kernel.org/r/20210204014401.2846425-3-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/Makefile | 2 +- drivers/iommu/intel/cap_audit.c | 185 ++++++++++++++++++++++++++++++++++++ drivers/iommu/intel/cap_audit.h | 110 +++++++++++++++++++++ drivers/iommu/intel/iommu.c | 9 ++ drivers/iommu/intel/irq_remapping.c | 8 ++ 5 files changed, 313 insertions(+), 1 deletion(-) create mode 100644 drivers/iommu/intel/cap_audit.c create mode 100644 drivers/iommu/intel/cap_audit.h (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/Makefile b/drivers/iommu/intel/Makefile index ae570810a35e..ae236ec7d219 100644 --- a/drivers/iommu/intel/Makefile +++ b/drivers/iommu/intel/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DMAR_TABLE) += dmar.o obj-$(CONFIG_INTEL_IOMMU) += iommu.o pasid.o -obj-$(CONFIG_DMAR_TABLE) += trace.o +obj-$(CONFIG_DMAR_TABLE) += trace.o cap_audit.o obj-$(CONFIG_INTEL_IOMMU_DEBUGFS) += debugfs.o obj-$(CONFIG_INTEL_IOMMU_SVM) += svm.o obj-$(CONFIG_IRQ_REMAP) += irq_remapping.o diff --git a/drivers/iommu/intel/cap_audit.c b/drivers/iommu/intel/cap_audit.c new file mode 100644 index 000000000000..049bc0c76a00 --- /dev/null +++ b/drivers/iommu/intel/cap_audit.c @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * cap_audit.c - audit iommu capabilities for boot time and hot plug + * + * Copyright (C) 2021 Intel Corporation + * + * Author: Kyung Min Park + * Lu Baolu + */ + +#define pr_fmt(fmt) "DMAR: " fmt + +#include +#include "cap_audit.h" + +static u64 intel_iommu_cap_sanity; +static u64 intel_iommu_ecap_sanity; + +static inline void check_irq_capabilities(struct intel_iommu *a, + struct intel_iommu *b) +{ + CHECK_FEATURE_MISMATCH(a, b, cap, pi_support, CAP_PI_MASK); + CHECK_FEATURE_MISMATCH(a, b, ecap, eim_support, ECAP_EIM_MASK); +} + +static inline void check_dmar_capabilities(struct intel_iommu *a, + struct intel_iommu *b) +{ + MINIMAL_FEATURE_IOMMU(b, cap, CAP_MAMV_MASK); + MINIMAL_FEATURE_IOMMU(b, cap, CAP_NFR_MASK); + MINIMAL_FEATURE_IOMMU(b, cap, CAP_SLLPS_MASK); + MINIMAL_FEATURE_IOMMU(b, cap, CAP_FRO_MASK); + MINIMAL_FEATURE_IOMMU(b, cap, CAP_MGAW_MASK); + MINIMAL_FEATURE_IOMMU(b, cap, CAP_SAGAW_MASK); + MINIMAL_FEATURE_IOMMU(b, cap, CAP_NDOMS_MASK); + MINIMAL_FEATURE_IOMMU(b, ecap, ECAP_PSS_MASK); + MINIMAL_FEATURE_IOMMU(b, ecap, ECAP_MHMV_MASK); + MINIMAL_FEATURE_IOMMU(b, ecap, ECAP_IRO_MASK); + + CHECK_FEATURE_MISMATCH(a, b, cap, 5lp_support, CAP_FL5LP_MASK); + CHECK_FEATURE_MISMATCH(a, b, cap, fl1gp_support, CAP_FL1GP_MASK); + CHECK_FEATURE_MISMATCH(a, b, cap, read_drain, CAP_RD_MASK); + CHECK_FEATURE_MISMATCH(a, b, cap, write_drain, CAP_WD_MASK); + CHECK_FEATURE_MISMATCH(a, b, cap, pgsel_inv, CAP_PSI_MASK); + CHECK_FEATURE_MISMATCH(a, b, cap, zlr, CAP_ZLR_MASK); + CHECK_FEATURE_MISMATCH(a, b, cap, caching_mode, CAP_CM_MASK); + CHECK_FEATURE_MISMATCH(a, b, cap, phmr, CAP_PHMR_MASK); + CHECK_FEATURE_MISMATCH(a, b, cap, plmr, CAP_PLMR_MASK); + CHECK_FEATURE_MISMATCH(a, b, cap, rwbf, CAP_RWBF_MASK); + CHECK_FEATURE_MISMATCH(a, b, cap, afl, CAP_AFL_MASK); + CHECK_FEATURE_MISMATCH(a, b, ecap, rps, ECAP_RPS_MASK); + CHECK_FEATURE_MISMATCH(a, b, ecap, smpwc, ECAP_SMPWC_MASK); + CHECK_FEATURE_MISMATCH(a, b, ecap, flts, ECAP_FLTS_MASK); + CHECK_FEATURE_MISMATCH(a, b, ecap, slts, ECAP_SLTS_MASK); + CHECK_FEATURE_MISMATCH(a, b, ecap, nwfs, ECAP_NWFS_MASK); + CHECK_FEATURE_MISMATCH(a, b, ecap, slads, ECAP_SLADS_MASK); + CHECK_FEATURE_MISMATCH(a, b, ecap, vcs, ECAP_VCS_MASK); + CHECK_FEATURE_MISMATCH(a, b, ecap, smts, ECAP_SMTS_MASK); + CHECK_FEATURE_MISMATCH(a, b, ecap, pds, ECAP_PDS_MASK); + CHECK_FEATURE_MISMATCH(a, b, ecap, dit, ECAP_DIT_MASK); + CHECK_FEATURE_MISMATCH(a, b, ecap, pasid, ECAP_PASID_MASK); + CHECK_FEATURE_MISMATCH(a, b, ecap, eafs, ECAP_EAFS_MASK); + CHECK_FEATURE_MISMATCH(a, b, ecap, srs, ECAP_SRS_MASK); + CHECK_FEATURE_MISMATCH(a, b, ecap, ers, ECAP_ERS_MASK); + CHECK_FEATURE_MISMATCH(a, b, ecap, prs, ECAP_PRS_MASK); + CHECK_FEATURE_MISMATCH(a, b, ecap, nest, ECAP_NEST_MASK); + CHECK_FEATURE_MISMATCH(a, b, ecap, mts, ECAP_MTS_MASK); + CHECK_FEATURE_MISMATCH(a, b, ecap, sc_support, ECAP_SC_MASK); + CHECK_FEATURE_MISMATCH(a, b, ecap, pass_through, ECAP_PT_MASK); + CHECK_FEATURE_MISMATCH(a, b, ecap, dev_iotlb_support, ECAP_DT_MASK); + CHECK_FEATURE_MISMATCH(a, b, ecap, qis, ECAP_QI_MASK); + CHECK_FEATURE_MISMATCH(a, b, ecap, coherent, ECAP_C_MASK); +} + +static int cap_audit_hotplug(struct intel_iommu *iommu, enum cap_audit_type type) +{ + bool mismatch = false; + u64 old_cap = intel_iommu_cap_sanity; + u64 old_ecap = intel_iommu_ecap_sanity; + + if (type == CAP_AUDIT_HOTPLUG_IRQR) { + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, cap, pi_support, CAP_PI_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, ecap, eim_support, ECAP_EIM_MASK); + goto out; + } + + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, cap, 5lp_support, CAP_FL5LP_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, cap, fl1gp_support, CAP_FL1GP_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, cap, read_drain, CAP_RD_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, cap, write_drain, CAP_WD_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, cap, pgsel_inv, CAP_PSI_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, cap, zlr, CAP_ZLR_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, cap, caching_mode, CAP_CM_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, cap, phmr, CAP_PHMR_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, cap, plmr, CAP_PLMR_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, cap, rwbf, CAP_RWBF_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, cap, afl, CAP_AFL_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, ecap, rps, ECAP_RPS_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, ecap, smpwc, ECAP_SMPWC_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, ecap, flts, ECAP_FLTS_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, ecap, slts, ECAP_SLTS_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, ecap, nwfs, ECAP_NWFS_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, ecap, slads, ECAP_SLADS_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, ecap, vcs, ECAP_VCS_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, ecap, smts, ECAP_SMTS_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, ecap, pds, ECAP_PDS_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, ecap, dit, ECAP_DIT_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, ecap, pasid, ECAP_PASID_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, ecap, eafs, ECAP_EAFS_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, ecap, srs, ECAP_SRS_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, ecap, ers, ECAP_ERS_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, ecap, prs, ECAP_PRS_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, ecap, nest, ECAP_NEST_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, ecap, mts, ECAP_MTS_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, ecap, sc_support, ECAP_SC_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, ecap, pass_through, ECAP_PT_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, ecap, dev_iotlb_support, ECAP_DT_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, ecap, qis, ECAP_QI_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, ecap, coherent, ECAP_C_MASK); + + /* Abort hot plug if the hot plug iommu feature is smaller than global */ + MINIMAL_FEATURE_HOTPLUG(iommu, cap, max_amask_val, CAP_MAMV_MASK, mismatch); + MINIMAL_FEATURE_HOTPLUG(iommu, cap, num_fault_regs, CAP_NFR_MASK, mismatch); + MINIMAL_FEATURE_HOTPLUG(iommu, cap, super_page_val, CAP_SLLPS_MASK, mismatch); + MINIMAL_FEATURE_HOTPLUG(iommu, cap, fault_reg_offset, CAP_FRO_MASK, mismatch); + MINIMAL_FEATURE_HOTPLUG(iommu, cap, mgaw, CAP_MGAW_MASK, mismatch); + MINIMAL_FEATURE_HOTPLUG(iommu, cap, sagaw, CAP_SAGAW_MASK, mismatch); + MINIMAL_FEATURE_HOTPLUG(iommu, cap, ndoms, CAP_NDOMS_MASK, mismatch); + MINIMAL_FEATURE_HOTPLUG(iommu, ecap, pss, ECAP_PSS_MASK, mismatch); + MINIMAL_FEATURE_HOTPLUG(iommu, ecap, max_handle_mask, ECAP_MHMV_MASK, mismatch); + MINIMAL_FEATURE_HOTPLUG(iommu, ecap, iotlb_offset, ECAP_IRO_MASK, mismatch); + +out: + if (mismatch) { + intel_iommu_cap_sanity = old_cap; + intel_iommu_ecap_sanity = old_ecap; + return -EFAULT; + } + + return 0; +} + +static int cap_audit_static(struct intel_iommu *iommu, enum cap_audit_type type) +{ + struct dmar_drhd_unit *d; + struct intel_iommu *i; + + rcu_read_lock(); + if (list_empty(&dmar_drhd_units)) + goto out; + + for_each_active_iommu(i, d) { + if (!iommu) { + intel_iommu_ecap_sanity = i->ecap; + intel_iommu_cap_sanity = i->cap; + iommu = i; + continue; + } + + if (type == CAP_AUDIT_STATIC_DMAR) + check_dmar_capabilities(iommu, i); + else + check_irq_capabilities(iommu, i); + } + +out: + rcu_read_unlock(); + return 0; +} + +int intel_cap_audit(enum cap_audit_type type, struct intel_iommu *iommu) +{ + switch (type) { + case CAP_AUDIT_STATIC_DMAR: + case CAP_AUDIT_STATIC_IRQR: + return cap_audit_static(iommu, type); + case CAP_AUDIT_HOTPLUG_DMAR: + case CAP_AUDIT_HOTPLUG_IRQR: + return cap_audit_hotplug(iommu, type); + default: + break; + } + + return -EFAULT; +} diff --git a/drivers/iommu/intel/cap_audit.h b/drivers/iommu/intel/cap_audit.h new file mode 100644 index 000000000000..a6a1530441b7 --- /dev/null +++ b/drivers/iommu/intel/cap_audit.h @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * cap_audit.h - audit iommu capabilities header + * + * Copyright (C) 2021 Intel Corporation + * + * Author: Kyung Min Park + */ + +/* + * Capability Register Mask + */ +#define CAP_FL5LP_MASK BIT_ULL(60) +#define CAP_PI_MASK BIT_ULL(59) +#define CAP_FL1GP_MASK BIT_ULL(56) +#define CAP_RD_MASK BIT_ULL(55) +#define CAP_WD_MASK BIT_ULL(54) +#define CAP_MAMV_MASK GENMASK_ULL(53, 48) +#define CAP_NFR_MASK GENMASK_ULL(47, 40) +#define CAP_PSI_MASK BIT_ULL(39) +#define CAP_SLLPS_MASK GENMASK_ULL(37, 34) +#define CAP_FRO_MASK GENMASK_ULL(33, 24) +#define CAP_ZLR_MASK BIT_ULL(22) +#define CAP_MGAW_MASK GENMASK_ULL(21, 16) +#define CAP_SAGAW_MASK GENMASK_ULL(12, 8) +#define CAP_CM_MASK BIT_ULL(7) +#define CAP_PHMR_MASK BIT_ULL(6) +#define CAP_PLMR_MASK BIT_ULL(5) +#define CAP_RWBF_MASK BIT_ULL(4) +#define CAP_AFL_MASK BIT_ULL(3) +#define CAP_NDOMS_MASK GENMASK_ULL(2, 0) + +/* + * Extended Capability Register Mask + */ +#define ECAP_RPS_MASK BIT_ULL(49) +#define ECAP_SMPWC_MASK BIT_ULL(48) +#define ECAP_FLTS_MASK BIT_ULL(47) +#define ECAP_SLTS_MASK BIT_ULL(46) +#define ECAP_SLADS_MASK BIT_ULL(45) +#define ECAP_VCS_MASK BIT_ULL(44) +#define ECAP_SMTS_MASK BIT_ULL(43) +#define ECAP_PDS_MASK BIT_ULL(42) +#define ECAP_DIT_MASK BIT_ULL(41) +#define ECAP_PASID_MASK BIT_ULL(40) +#define ECAP_PSS_MASK GENMASK_ULL(39, 35) +#define ECAP_EAFS_MASK BIT_ULL(34) +#define ECAP_NWFS_MASK BIT_ULL(33) +#define ECAP_SRS_MASK BIT_ULL(31) +#define ECAP_ERS_MASK BIT_ULL(30) +#define ECAP_PRS_MASK BIT_ULL(29) +#define ECAP_NEST_MASK BIT_ULL(26) +#define ECAP_MTS_MASK BIT_ULL(25) +#define ECAP_MHMV_MASK GENMASK_ULL(23, 20) +#define ECAP_IRO_MASK GENMASK_ULL(17, 8) +#define ECAP_SC_MASK BIT_ULL(7) +#define ECAP_PT_MASK BIT_ULL(6) +#define ECAP_EIM_MASK BIT_ULL(4) +#define ECAP_DT_MASK BIT_ULL(2) +#define ECAP_QI_MASK BIT_ULL(1) +#define ECAP_C_MASK BIT_ULL(0) + +/* + * u64 intel_iommu_cap_sanity, intel_iommu_ecap_sanity will be adjusted as each + * IOMMU gets audited. + */ +#define DO_CHECK_FEATURE_MISMATCH(a, b, cap, feature, MASK) \ +do { \ + if (cap##_##feature(a) != cap##_##feature(b)) { \ + intel_iommu_##cap##_sanity &= ~(MASK); \ + pr_info("IOMMU feature %s inconsistent", #feature); \ + } \ +} while (0) + +#define CHECK_FEATURE_MISMATCH(a, b, cap, feature, MASK) \ + DO_CHECK_FEATURE_MISMATCH((a)->cap, (b)->cap, cap, feature, MASK) + +#define CHECK_FEATURE_MISMATCH_HOTPLUG(b, cap, feature, MASK) \ +do { \ + if (cap##_##feature(intel_iommu_##cap##_sanity)) \ + DO_CHECK_FEATURE_MISMATCH(intel_iommu_##cap##_sanity, \ + (b)->cap, cap, feature, MASK); \ +} while (0) + +#define MINIMAL_FEATURE_IOMMU(iommu, cap, MASK) \ +do { \ + u64 min_feature = intel_iommu_##cap##_sanity & (MASK); \ + min_feature = min_t(u64, min_feature, (iommu)->cap & (MASK)); \ + intel_iommu_##cap##_sanity = (intel_iommu_##cap##_sanity & ~(MASK)) | \ + min_feature; \ +} while (0) + +#define MINIMAL_FEATURE_HOTPLUG(iommu, cap, feature, MASK, mismatch) \ +do { \ + if ((intel_iommu_##cap##_sanity & (MASK)) > \ + (cap##_##feature((iommu)->cap))) \ + mismatch = true; \ + else \ + (iommu)->cap = ((iommu)->cap & ~(MASK)) | \ + (intel_iommu_##cap##_sanity & (MASK)); \ +} while (0) + +enum cap_audit_type { + CAP_AUDIT_STATIC_DMAR, + CAP_AUDIT_STATIC_IRQR, + CAP_AUDIT_HOTPLUG_DMAR, + CAP_AUDIT_HOTPLUG_IRQR, +}; + +int intel_cap_audit(enum cap_audit_type type, struct intel_iommu *iommu); diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 87d0ef79ba6a..de82d5da01f9 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -47,6 +47,7 @@ #include "../irq_remapping.h" #include "pasid.h" +#include "cap_audit.h" #define ROOT_SIZE VTD_PAGE_SIZE #define CONTEXT_SIZE VTD_PAGE_SIZE @@ -3206,6 +3207,10 @@ static int __init init_dmars(void) goto error; } + ret = intel_cap_audit(CAP_AUDIT_STATIC_DMAR, NULL); + if (ret) + goto free_iommu; + for_each_iommu(iommu, drhd) { if (drhd->ignored) { iommu_disable_translation(iommu); @@ -3757,6 +3762,10 @@ static int intel_iommu_add(struct dmar_drhd_unit *dmaru) if (g_iommus[iommu->seq_id]) return 0; + ret = intel_cap_audit(CAP_AUDIT_HOTPLUG_DMAR, iommu); + if (ret) + goto out; + if (hw_pass_through && !ecap_pass_through(iommu->ecap)) { pr_warn("%s: Doesn't support hardware pass through.\n", iommu->name); diff --git a/drivers/iommu/intel/irq_remapping.c b/drivers/iommu/intel/irq_remapping.c index 685200a5cff0..611ef5243cb6 100644 --- a/drivers/iommu/intel/irq_remapping.c +++ b/drivers/iommu/intel/irq_remapping.c @@ -22,6 +22,7 @@ #include #include "../irq_remapping.h" +#include "cap_audit.h" enum irq_mode { IRQ_REMAPPING, @@ -734,6 +735,9 @@ static int __init intel_prepare_irq_remapping(void) if (dmar_table_init() < 0) return -ENODEV; + if (intel_cap_audit(CAP_AUDIT_STATIC_IRQR, NULL)) + goto error; + if (!dmar_ir_support()) return -ENODEV; @@ -1439,6 +1443,10 @@ static int dmar_ir_add(struct dmar_drhd_unit *dmaru, struct intel_iommu *iommu) int ret; int eim = x2apic_enabled(); + ret = intel_cap_audit(CAP_AUDIT_HOTPLUG_IRQR, iommu); + if (ret) + return ret; + if (eim && !ecap_eim_support(iommu->ecap)) { pr_info("DRHD %Lx: EIM not supported by DRHD, ecap %Lx\n", iommu->reg_phys, iommu->ecap); -- cgit v1.2.3 From 010bf5659e01b0a169e8e6b9e6a8b7e62209470d Mon Sep 17 00:00:00 2001 From: Kyung Min Park Date: Thu, 4 Feb 2021 09:43:57 +0800 Subject: iommu/vt-d: Move capability check code to cap_audit files Move IOMMU capability check and sanity check code to cap_audit files. Also implement some helper functions for sanity checks. Signed-off-by: Kyung Min Park Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20210130184452.31711-1-kyung.min.park@intel.com Link: https://lore.kernel.org/r/20210204014401.2846425-4-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/cap_audit.c | 20 +++++++++++ drivers/iommu/intel/cap_audit.h | 20 +++++++++++ drivers/iommu/intel/iommu.c | 76 ++--------------------------------------- 3 files changed, 42 insertions(+), 74 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/cap_audit.c b/drivers/iommu/intel/cap_audit.c index 049bc0c76a00..b12e421a2f1a 100644 --- a/drivers/iommu/intel/cap_audit.c +++ b/drivers/iommu/intel/cap_audit.c @@ -183,3 +183,23 @@ int intel_cap_audit(enum cap_audit_type type, struct intel_iommu *iommu) return -EFAULT; } + +bool intel_cap_smts_sanity(void) +{ + return ecap_smts(intel_iommu_ecap_sanity); +} + +bool intel_cap_pasid_sanity(void) +{ + return ecap_pasid(intel_iommu_ecap_sanity); +} + +bool intel_cap_nest_sanity(void) +{ + return ecap_nest(intel_iommu_ecap_sanity); +} + +bool intel_cap_flts_sanity(void) +{ + return ecap_flts(intel_iommu_ecap_sanity); +} diff --git a/drivers/iommu/intel/cap_audit.h b/drivers/iommu/intel/cap_audit.h index a6a1530441b7..74cfccae0e81 100644 --- a/drivers/iommu/intel/cap_audit.h +++ b/drivers/iommu/intel/cap_audit.h @@ -107,4 +107,24 @@ enum cap_audit_type { CAP_AUDIT_HOTPLUG_IRQR, }; +bool intel_cap_smts_sanity(void); +bool intel_cap_pasid_sanity(void); +bool intel_cap_nest_sanity(void); +bool intel_cap_flts_sanity(void); + +static inline bool scalable_mode_support(void) +{ + return (intel_iommu_sm && intel_cap_smts_sanity()); +} + +static inline bool pasid_mode_support(void) +{ + return scalable_mode_support() && intel_cap_pasid_sanity(); +} + +static inline bool nested_mode_support(void) +{ + return scalable_mode_support() && intel_cap_nest_sanity(); +} + int intel_cap_audit(enum cap_audit_type type, struct intel_iommu *iommu); diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index de82d5da01f9..7dfe681acc3d 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -1864,25 +1864,7 @@ static void free_dmar_iommu(struct intel_iommu *iommu) */ static bool first_level_by_default(void) { - struct dmar_drhd_unit *drhd; - struct intel_iommu *iommu; - static int first_level_support = -1; - - if (likely(first_level_support != -1)) - return first_level_support; - - first_level_support = 1; - - rcu_read_lock(); - for_each_active_iommu(iommu, drhd) { - if (!sm_supported(iommu) || !ecap_flts(iommu->ecap)) { - first_level_support = 0; - break; - } - } - rcu_read_unlock(); - - return first_level_support; + return scalable_mode_support() && intel_cap_flts_sanity(); } static struct dmar_domain *alloc_domain(int flags) @@ -5058,60 +5040,6 @@ static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain, return phys; } -static inline bool scalable_mode_support(void) -{ - struct dmar_drhd_unit *drhd; - struct intel_iommu *iommu; - bool ret = true; - - rcu_read_lock(); - for_each_active_iommu(iommu, drhd) { - if (!sm_supported(iommu)) { - ret = false; - break; - } - } - rcu_read_unlock(); - - return ret; -} - -static inline bool iommu_pasid_support(void) -{ - struct dmar_drhd_unit *drhd; - struct intel_iommu *iommu; - bool ret = true; - - rcu_read_lock(); - for_each_active_iommu(iommu, drhd) { - if (!pasid_supported(iommu)) { - ret = false; - break; - } - } - rcu_read_unlock(); - - return ret; -} - -static inline bool nested_mode_support(void) -{ - struct dmar_drhd_unit *drhd; - struct intel_iommu *iommu; - bool ret = true; - - rcu_read_lock(); - for_each_active_iommu(iommu, drhd) { - if (!sm_supported(iommu) || !ecap_nest(iommu->ecap)) { - ret = false; - break; - } - } - rcu_read_unlock(); - - return ret; -} - static bool intel_iommu_capable(enum iommu_cap cap) { if (cap == IOMMU_CAP_CACHE_COHERENCY) @@ -5352,7 +5280,7 @@ intel_iommu_dev_has_feat(struct device *dev, enum iommu_dev_features feat) int ret; if (!dev_is_pci(dev) || dmar_disabled || - !scalable_mode_support() || !iommu_pasid_support()) + !scalable_mode_support() || !pasid_mode_support()) return false; ret = pci_pasid_features(to_pci_dev(dev)); -- cgit v1.2.3 From 933fcd01e97e2ba29880dd5f1239365e40094950 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Thu, 4 Feb 2021 09:43:58 +0800 Subject: iommu/vt-d: Add iotlb_sync_map callback Some Intel VT-d hardware implementations don't support memory coherency for page table walk (presented by the Page-Walk-coherency bit in the ecap register), so that software must flush the corresponding CPU cache lines explicitly after each page table entry update. The iommu_map_sg() code iterates through the given scatter-gather list and invokes iommu_map() for each element in the scatter-gather list, which calls into the vendor IOMMU driver through iommu_ops callback. As the result, a single sg mapping may lead to multiple cache line flushes, which leads to the degradation of I/O performance after the commit ("iommu/vt-d: Convert intel iommu driver to the iommu ops"). Fix this by adding iotlb_sync_map callback and centralizing the clflush operations after all sg mappings. Fixes: c588072bba6b5 ("iommu/vt-d: Convert intel iommu driver to the iommu ops") Reported-by: Chuck Lever Link: https://lore.kernel.org/linux-iommu/D81314ED-5673-44A6-B597-090E3CB83EB0@oracle.com/ Signed-off-by: Lu Baolu Cc: Robin Murphy [ cel: removed @first_pte, which is no longer used ] Signed-off-by: Chuck Lever Link: https://lore.kernel.org/linux-iommu/161177763962.1311.15577661784296014186.stgit@manet.1015granger.net Link: https://lore.kernel.org/r/20210204014401.2846425-5-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 90 ++++++++++++++++++++++++++++++--------------- 1 file changed, 60 insertions(+), 30 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 7dfe681acc3d..761c1b251c1e 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -2283,9 +2283,9 @@ static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, unsigned long phys_pfn, unsigned long nr_pages, int prot) { - struct dma_pte *first_pte = NULL, *pte = NULL; unsigned int largepage_lvl = 0; unsigned long lvl_pages = 0; + struct dma_pte *pte = NULL; phys_addr_t pteval; u64 attr; @@ -2314,7 +2314,7 @@ __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, nr_pages); - first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl); + pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl); if (!pte) return -ENOMEM; /* It is large page*/ @@ -2375,34 +2375,14 @@ __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, * recalculate 'pte' and switch back to smaller pages for the * end of the mapping, if the trailing size is not enough to * use another superpage (i.e. nr_pages < lvl_pages). + * + * We leave clflush for the leaf pte changes to iotlb_sync_map() + * callback. */ pte++; if (!nr_pages || first_pte_in_page(pte) || - (largepage_lvl > 1 && nr_pages < lvl_pages)) { - domain_flush_cache(domain, first_pte, - (void *)pte - (void *)first_pte); + (largepage_lvl > 1 && nr_pages < lvl_pages)) pte = NULL; - } - } - - return 0; -} - -static int -domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, - unsigned long phys_pfn, unsigned long nr_pages, int prot) -{ - int iommu_id, ret; - struct intel_iommu *iommu; - - /* Do the real mapping first */ - ret = __domain_mapping(domain, iov_pfn, phys_pfn, nr_pages, prot); - if (ret) - return ret; - - for_each_domain_iommu(iommu_id, domain) { - iommu = g_iommus[iommu_id]; - __mapping_notify_one(iommu, domain, iov_pfn, nr_pages); } return 0; @@ -4943,7 +4923,6 @@ static int intel_iommu_map(struct iommu_domain *domain, struct dmar_domain *dmar_domain = to_dmar_domain(domain); u64 max_addr; int prot = 0; - int ret; if (iommu_prot & IOMMU_READ) prot |= DMA_PTE_READ; @@ -4969,9 +4948,8 @@ static int intel_iommu_map(struct iommu_domain *domain, /* Round up size to next multiple of PAGE_SIZE, if it and the low bits of hpa would take us onto the next page */ size = aligned_nrpages(hpa, size); - ret = domain_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT, - hpa >> VTD_PAGE_SHIFT, size, prot); - return ret; + return __domain_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT, + hpa >> VTD_PAGE_SHIFT, size, prot); } static size_t intel_iommu_unmap(struct iommu_domain *domain, @@ -5424,6 +5402,57 @@ static bool risky_device(struct pci_dev *pdev) return false; } +static void clflush_sync_map(struct dmar_domain *domain, unsigned long clf_pfn, + unsigned long clf_pages) +{ + struct dma_pte *first_pte = NULL, *pte = NULL; + unsigned long lvl_pages = 0; + int level = 0; + + while (clf_pages > 0) { + if (!pte) { + level = 0; + pte = pfn_to_dma_pte(domain, clf_pfn, &level); + if (WARN_ON(!pte)) + return; + first_pte = pte; + lvl_pages = lvl_to_nr_pages(level); + } + + if (WARN_ON(!lvl_pages || clf_pages < lvl_pages)) + return; + + clf_pages -= lvl_pages; + clf_pfn += lvl_pages; + pte++; + + if (!clf_pages || first_pte_in_page(pte) || + (level > 1 && clf_pages < lvl_pages)) { + domain_flush_cache(domain, first_pte, + (void *)pte - (void *)first_pte); + pte = NULL; + } + } +} + +static void intel_iommu_iotlb_sync_map(struct iommu_domain *domain, + unsigned long iova, size_t size) +{ + struct dmar_domain *dmar_domain = to_dmar_domain(domain); + unsigned long pages = aligned_nrpages(iova, size); + unsigned long pfn = iova >> VTD_PAGE_SHIFT; + struct intel_iommu *iommu; + int iommu_id; + + if (!dmar_domain->iommu_coherency) + clflush_sync_map(dmar_domain, pfn, pages); + + for_each_domain_iommu(iommu_id, dmar_domain) { + iommu = g_iommus[iommu_id]; + __mapping_notify_one(iommu, dmar_domain, pfn, pages); + } +} + const struct iommu_ops intel_iommu_ops = { .capable = intel_iommu_capable, .domain_alloc = intel_iommu_domain_alloc, @@ -5436,6 +5465,7 @@ const struct iommu_ops intel_iommu_ops = { .aux_detach_dev = intel_iommu_aux_detach_device, .aux_get_pasid = intel_iommu_aux_get_pasid, .map = intel_iommu_map, + .iotlb_sync_map = intel_iommu_iotlb_sync_map, .unmap = intel_iommu_unmap, .flush_iotlb_all = intel_flush_iotlb_all, .iotlb_sync = intel_iommu_tlb_sync, -- cgit v1.2.3 From 31a75cbbb9274cf8185f402904bf11386917870b Mon Sep 17 00:00:00 2001 From: Yian Chen Date: Thu, 4 Feb 2021 09:44:00 +0800 Subject: iommu/vt-d: Parse SATC reporting structure Software should parse every SATC table and all devices in the tables reported by the BIOS and keep the information in kernel list for further reference. Signed-off-by: Yian Chen Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20210203093329.1617808-1-baolu.lu@linux.intel.com Link: https://lore.kernel.org/r/20210204014401.2846425-7-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/dmar.c | 8 ++++ drivers/iommu/intel/iommu.c | 89 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index bd51f33642e0..6b8fce1c85c4 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -526,6 +526,7 @@ dmar_table_print_dmar_entry(struct acpi_dmar_header *header) struct acpi_dmar_reserved_memory *rmrr; struct acpi_dmar_atsr *atsr; struct acpi_dmar_rhsa *rhsa; + struct acpi_dmar_satc *satc; switch (header->type) { case ACPI_DMAR_TYPE_HARDWARE_UNIT: @@ -555,6 +556,10 @@ dmar_table_print_dmar_entry(struct acpi_dmar_header *header) /* We don't print this here because we need to sanity-check it first. So print it in dmar_parse_one_andd() instead. */ break; + case ACPI_DMAR_TYPE_SATC: + satc = container_of(header, struct acpi_dmar_satc, header); + pr_info("SATC flags: 0x%x\n", satc->flags); + break; } } @@ -642,6 +647,7 @@ parse_dmar_table(void) .cb[ACPI_DMAR_TYPE_ROOT_ATS] = &dmar_parse_one_atsr, .cb[ACPI_DMAR_TYPE_HARDWARE_AFFINITY] = &dmar_parse_one_rhsa, .cb[ACPI_DMAR_TYPE_NAMESPACE] = &dmar_parse_one_andd, + .cb[ACPI_DMAR_TYPE_SATC] = &dmar_parse_one_satc, }; /* @@ -2077,6 +2083,7 @@ static guid_t dmar_hp_guid = #define DMAR_DSM_FUNC_DRHD 1 #define DMAR_DSM_FUNC_ATSR 2 #define DMAR_DSM_FUNC_RHSA 3 +#define DMAR_DSM_FUNC_SATC 4 static inline bool dmar_detect_dsm(acpi_handle handle, int func) { @@ -2094,6 +2101,7 @@ static int dmar_walk_dsm_resource(acpi_handle handle, int func, [DMAR_DSM_FUNC_DRHD] = ACPI_DMAR_TYPE_HARDWARE_UNIT, [DMAR_DSM_FUNC_ATSR] = ACPI_DMAR_TYPE_ROOT_ATS, [DMAR_DSM_FUNC_RHSA] = ACPI_DMAR_TYPE_HARDWARE_AFFINITY, + [DMAR_DSM_FUNC_SATC] = ACPI_DMAR_TYPE_SATC, }; if (!dmar_detect_dsm(handle, func)) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 761c1b251c1e..384cc546632a 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -316,8 +316,18 @@ struct dmar_atsr_unit { u8 include_all:1; /* include all ports */ }; +struct dmar_satc_unit { + struct list_head list; /* list of SATC units */ + struct acpi_dmar_header *hdr; /* ACPI header */ + struct dmar_dev_scope *devices; /* target devices */ + struct intel_iommu *iommu; /* the corresponding iommu */ + int devices_cnt; /* target device count */ + u8 atc_required:1; /* ATS is required */ +}; + static LIST_HEAD(dmar_atsr_units); static LIST_HEAD(dmar_rmrr_units); +static LIST_HEAD(dmar_satc_units); #define for_each_rmrr_units(rmrr) \ list_for_each_entry(rmrr, &dmar_rmrr_units, list) @@ -3716,6 +3726,57 @@ int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg) return 0; } +static struct dmar_satc_unit *dmar_find_satc(struct acpi_dmar_satc *satc) +{ + struct dmar_satc_unit *satcu; + struct acpi_dmar_satc *tmp; + + list_for_each_entry_rcu(satcu, &dmar_satc_units, list, + dmar_rcu_check()) { + tmp = (struct acpi_dmar_satc *)satcu->hdr; + if (satc->segment != tmp->segment) + continue; + if (satc->header.length != tmp->header.length) + continue; + if (memcmp(satc, tmp, satc->header.length) == 0) + return satcu; + } + + return NULL; +} + +int dmar_parse_one_satc(struct acpi_dmar_header *hdr, void *arg) +{ + struct acpi_dmar_satc *satc; + struct dmar_satc_unit *satcu; + + if (system_state >= SYSTEM_RUNNING && !intel_iommu_enabled) + return 0; + + satc = container_of(hdr, struct acpi_dmar_satc, header); + satcu = dmar_find_satc(satc); + if (satcu) + return 0; + + satcu = kzalloc(sizeof(*satcu) + hdr->length, GFP_KERNEL); + if (!satcu) + return -ENOMEM; + + satcu->hdr = (void *)(satcu + 1); + memcpy(satcu->hdr, hdr, hdr->length); + satcu->atc_required = satc->flags & 0x1; + satcu->devices = dmar_alloc_dev_scope((void *)(satc + 1), + (void *)satc + satc->header.length, + &satcu->devices_cnt); + if (satcu->devices_cnt && !satcu->devices) { + kfree(satcu); + return -ENOMEM; + } + list_add_rcu(&satcu->list, &dmar_satc_units); + + return 0; +} + static int intel_iommu_add(struct dmar_drhd_unit *dmaru) { int sp, ret; @@ -3823,6 +3884,7 @@ static void intel_iommu_free_dmars(void) { struct dmar_rmrr_unit *rmrru, *rmrr_n; struct dmar_atsr_unit *atsru, *atsr_n; + struct dmar_satc_unit *satcu, *satc_n; list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) { list_del(&rmrru->list); @@ -3834,6 +3896,11 @@ static void intel_iommu_free_dmars(void) list_del(&atsru->list); intel_iommu_free_atsr(atsru); } + list_for_each_entry_safe(satcu, satc_n, &dmar_satc_units, list) { + list_del(&satcu->list); + dmar_free_dev_scope(&satcu->devices, &satcu->devices_cnt); + kfree(satcu); + } } int dmar_find_matched_atsr_unit(struct pci_dev *dev) @@ -3885,8 +3952,10 @@ int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info) int ret; struct dmar_rmrr_unit *rmrru; struct dmar_atsr_unit *atsru; + struct dmar_satc_unit *satcu; struct acpi_dmar_atsr *atsr; struct acpi_dmar_reserved_memory *rmrr; + struct acpi_dmar_satc *satc; if (!intel_iommu_enabled && system_state >= SYSTEM_RUNNING) return 0; @@ -3927,6 +3996,23 @@ int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info) break; } } + list_for_each_entry(satcu, &dmar_satc_units, list) { + satc = container_of(satcu->hdr, struct acpi_dmar_satc, header); + if (info->event == BUS_NOTIFY_ADD_DEVICE) { + ret = dmar_insert_dev_scope(info, (void *)(satc + 1), + (void *)satc + satc->header.length, + satc->segment, satcu->devices, + satcu->devices_cnt); + if (ret > 0) + break; + else if (ret < 0) + return ret; + } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) { + if (dmar_remove_dev_scope(info, satc->segment, + satcu->devices, satcu->devices_cnt)) + break; + } + } return 0; } @@ -4270,6 +4356,9 @@ int __init intel_iommu_init(void) if (list_empty(&dmar_atsr_units)) pr_info("No ATSR found\n"); + if (list_empty(&dmar_satc_units)) + pr_info("No SATC found\n"); + if (dmar_map_gfx) intel_iommu_gfx_mapped = 1; -- cgit v1.2.3 From 7a5661739d4bf23cd356fb13bb1e0790e09df3c5 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Thu, 4 Feb 2021 15:00:01 +0000 Subject: iommu/mediatek: Fix unsigned domid comparison with less than zero Currently the check for domid < 0 is always false because domid is unsigned. Fix this by casting domid to an int before making the comparison. Signed-off-by: Colin Ian King Acked-by: Will Deacon Link: https://lore.kernel.org/r/20210204150001.102672-1-colin.king@canonical.com Addresses-Coverity: ("Unsigned comparison against 0") Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 0ad14a7604b1..1f262621ef19 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -645,7 +645,7 @@ static void mtk_iommu_get_resv_regions(struct device *dev, struct iommu_resv_region *region; int prot = IOMMU_WRITE | IOMMU_READ; - if (domid < 0) + if ((int)domid < 0) return; curdom = data->plat_data->iova_region + domid; for (i = 0; i < data->plat_data->iova_region_nr; i++) { -- cgit v1.2.3 From a92a90ac62d3284a8deaef43c6ea4dbd5dd878b0 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Fri, 5 Feb 2021 15:46:17 +0300 Subject: iommu/mediatek: Fix error code in probe() This error path is supposed to return -EINVAL. It used to return directly but we added some clean up and accidentally removed the error code. Also I fixed a typo in the error message. Fixes: c0b57581b73b ("iommu/mediatek: Add power-domain operation") Signed-off-by: Dan Carpenter Reviewed-by: Yong Wu Link: https://lore.kernel.org/r/YB0+GU5akSdu29Vu@mwanda Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 1f262621ef19..6ecc007f07cd 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -886,7 +886,8 @@ static int mtk_iommu_probe(struct platform_device *pdev) link = device_link_add(data->smicomm_dev, dev, DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME); if (!link) { - dev_err(dev, "Unable link %s.\n", dev_name(data->smicomm_dev)); + dev_err(dev, "Unable to link %s.\n", dev_name(data->smicomm_dev)); + ret = -EINVAL; goto out_runtime_disable; } -- cgit v1.2.3 From 6778ff5b21bd8e78c8bd547fd66437cf2657fd9b Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Mon, 8 Feb 2021 06:27:12 -0600 Subject: iommu/amd: Fix performance counter initialization Certain AMD platforms enable power gating feature for IOMMU PMC, which prevents the IOMMU driver from updating the counter while trying to validate the PMC functionality in the init_iommu_perf_ctr(). This results in disabling PMC support and the following error message: "AMD-Vi: Unable to read/write to IOMMU perf counter" To workaround this issue, disable power gating temporarily by programming the counter source to non-zero value while validating the counter, and restore the prior state afterward. Signed-off-by: Suravee Suthikulpanit Tested-by: Tj (Elloe Linux) Link: https://lore.kernel.org/r/20210208122712.5048-1-suravee.suthikulpanit@amd.com Bugzilla: https://bugzilla.kernel.org/show_bug.cgi?id=201753 Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 45 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 11 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 5a6c511e91d1..5a0762377de5 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -256,6 +257,8 @@ static enum iommu_init_state init_state = IOMMU_START_STATE; static int amd_iommu_enable_interrupts(void); static int __init iommu_go_to_state(enum iommu_init_state state); static void init_device_table_dma(void); +static int iommu_pc_get_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, + u8 fxn, u64 *value, bool is_write); static bool amd_iommu_pre_enabled = true; @@ -1697,13 +1700,11 @@ static int __init init_iommu_all(struct acpi_table_header *table) return 0; } -static int iommu_pc_get_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, - u8 fxn, u64 *value, bool is_write); - -static void init_iommu_perf_ctr(struct amd_iommu *iommu) +static void __init init_iommu_perf_ctr(struct amd_iommu *iommu) { + int retry; struct pci_dev *pdev = iommu->dev; - u64 val = 0xabcd, val2 = 0, save_reg = 0; + u64 val = 0xabcd, val2 = 0, save_reg, save_src; if (!iommu_feature(iommu, FEATURE_PC)) return; @@ -1711,17 +1712,39 @@ static void init_iommu_perf_ctr(struct amd_iommu *iommu) amd_iommu_pc_present = true; /* save the value to restore, if writable */ - if (iommu_pc_get_set_reg(iommu, 0, 0, 0, &save_reg, false)) + if (iommu_pc_get_set_reg(iommu, 0, 0, 0, &save_reg, false) || + iommu_pc_get_set_reg(iommu, 0, 0, 8, &save_src, false)) goto pc_false; - /* Check if the performance counters can be written to */ - if ((iommu_pc_get_set_reg(iommu, 0, 0, 0, &val, true)) || - (iommu_pc_get_set_reg(iommu, 0, 0, 0, &val2, false)) || - (val != val2)) + /* + * Disable power gating by programing the performance counter + * source to 20 (i.e. counts the reads and writes from/to IOMMU + * Reserved Register [MMIO Offset 1FF8h] that are ignored.), + * which never get incremented during this init phase. + * (Note: The event is also deprecated.) + */ + val = 20; + if (iommu_pc_get_set_reg(iommu, 0, 0, 8, &val, true)) goto pc_false; + /* Check if the performance counters can be written to */ + val = 0xabcd; + for (retry = 5; retry; retry--) { + if (iommu_pc_get_set_reg(iommu, 0, 0, 0, &val, true) || + iommu_pc_get_set_reg(iommu, 0, 0, 0, &val2, false) || + val2) + break; + + /* Wait about 20 msec for power gating to disable and retry. */ + msleep(20); + } + /* restore */ - if (iommu_pc_get_set_reg(iommu, 0, 0, 0, &save_reg, true)) + if (iommu_pc_get_set_reg(iommu, 0, 0, 0, &save_reg, true) || + iommu_pc_get_set_reg(iommu, 0, 0, 8, &save_src, true)) + goto pc_false; + + if (val != val2) goto pc_false; pci_info(pdev, "IOMMU performance counters supported\n"); -- cgit v1.2.3