From cae8d1f5e34e5b2604a52d705218a6b2d288365f Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Sat, 2 Apr 2022 12:08:38 +0200 Subject: iommu/fsl_pamu: Prepare cleanup of powerpc's asm/prom.h powerpc's asm/prom.h brings some headers that it doesn't need itself. In order to clean it up, first add missing headers in users of asm/prom.h Signed-off-by: Christophe Leroy Link: https://lore.kernel.org/r/06862cca930068e8fa4fdd0b20d74872d3b929d6.1648833431.git.christophe.leroy@csgroup.eu Signed-off-by: Joerg Roedel --- drivers/iommu/fsl_pamu.c | 3 +++ drivers/iommu/fsl_pamu_domain.c | 1 + 2 files changed, 4 insertions(+) (limited to 'drivers') diff --git a/drivers/iommu/fsl_pamu.c b/drivers/iommu/fsl_pamu.c index fc38b1fba7cf..0d03f837a5d4 100644 --- a/drivers/iommu/fsl_pamu.c +++ b/drivers/iommu/fsl_pamu.c @@ -11,6 +11,9 @@ #include #include #include +#include +#include +#include #include diff --git a/drivers/iommu/fsl_pamu_domain.c b/drivers/iommu/fsl_pamu_domain.c index 69a4a62dc3b9..94b4589dc67c 100644 --- a/drivers/iommu/fsl_pamu_domain.c +++ b/drivers/iommu/fsl_pamu_domain.c @@ -9,6 +9,7 @@ #include "fsl_pamu_domain.h" +#include #include /* -- cgit v1.2.3 From 121660bba631104154b7c15e88f208c48c8c3297 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 4 Apr 2022 15:47:22 -0500 Subject: iommu/amd: Enable swiotlb in all cases Previously the AMD IOMMU would only enable SWIOTLB in certain circumstances: * IOMMU in passthrough mode * SME enabled This logic however doesn't work when an untrusted device is plugged in that doesn't do page aligned DMA transactions. The expectation is that a bounce buffer is used for those transactions. This fails like this: swiotlb buffer is full (sz: 4096 bytes), total 0 (slots), used 0 (slots) That happens because the bounce buffers have been allocated, followed by freed during startup but the bounce buffering code expects that all IOMMUs have left it enabled. Remove the criteria to set up bounce buffers on AMD systems to ensure they're always available for supporting untrusted devices. Fixes: 82612d66d51d ("iommu: Allow the dma-iommu api to use bounce buffers") Suggested-by: Christoph Hellwig Signed-off-by: Mario Limonciello Reviewed-by: Robin Murphy Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20220404204723.9767-2-mario.limonciello@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 7 ------- 1 file changed, 7 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index a1ada7bff44e..079694f894b8 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1838,17 +1838,10 @@ void amd_iommu_domain_update(struct protection_domain *domain) amd_iommu_domain_flush_complete(domain); } -static void __init amd_iommu_init_dma_ops(void) -{ - swiotlb = (iommu_default_passthrough() || sme_me_mask) ? 1 : 0; -} - int __init amd_iommu_init_api(void) { int err; - amd_iommu_init_dma_ops(); - err = bus_set_iommu(&pci_bus_type, &amd_iommu_ops); if (err) return err; -- cgit v1.2.3 From f316ba0a8814f4c91e80a435da3421baf0ddd24c Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 4 Apr 2022 15:47:23 -0500 Subject: dma-iommu: Check that swiotlb is active before trying to use it If the IOMMU is in use and an untrusted device is connected to an external facing port but the address requested isn't page aligned will cause the kernel to attempt to use bounce buffers. If for some reason the bounce buffers have not been allocated this is a problem that should be made apparent to the user. Signed-off-by: Mario Limonciello Reviewed-by: Robin Murphy Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20220404204723.9767-3-mario.limonciello@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'drivers') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 09f6e1c0f9c0..1ca85d37eeab 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -971,6 +971,11 @@ static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, void *padding_start; size_t padding_size, aligned_size; + if (!is_swiotlb_active(dev)) { + dev_warn_once(dev, "DMA bounce buffers are inactive, unable to map unaligned transaction.\n"); + return DMA_MAPPING_ERROR; + } + aligned_size = iova_align(iovad, size); phys = swiotlb_tbl_map_single(dev, phys, size, aligned_size, iova_mask(iovad), dir, attrs); -- cgit v1.2.3 From ed36d04e8f8d7b00db451b0fa56a54e8e02ec43e Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 25 Apr 2022 13:42:02 +0100 Subject: iommu: Introduce device_iommu_capable() iommu_capable() only really works for systems where all IOMMU instances are completely homogeneous, and all devices are IOMMU-mapped. Implement the new variant which will be able to give a more accurate answer for whichever device the caller is actually interested in, and even more so once all the external users have been converted and we can reliably pass the device pointer through the internal driver interface too. Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/8407eb9586677995b7a9fd70d0fd82d85929a9bb.1650878781.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 23 +++++++++++++++++++++++ include/linux/iommu.h | 6 ++++++ 2 files changed, 29 insertions(+) (limited to 'drivers') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index f2c45b85b9fc..780c11734979 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1906,6 +1906,29 @@ bool iommu_present(struct bus_type *bus) } EXPORT_SYMBOL_GPL(iommu_present); +/** + * device_iommu_capable() - check for a general IOMMU capability + * @dev: device to which the capability would be relevant, if available + * @cap: IOMMU capability + * + * Return: true if an IOMMU is present and supports the given capability + * for the given device, otherwise false. + */ +bool device_iommu_capable(struct device *dev, enum iommu_cap cap) +{ + const struct iommu_ops *ops; + + if (!dev->iommu || !dev->iommu->iommu_dev) + return false; + + ops = dev_iommu_ops(dev); + if (!ops->capable) + return false; + + return ops->capable(cap); +} +EXPORT_SYMBOL_GPL(device_iommu_capable); + bool iommu_capable(struct bus_type *bus, enum iommu_cap cap) { if (!bus->iommu_ops || !bus->iommu_ops->capable) diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 9208eca4b0d1..e26cf84e5d82 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -417,6 +417,7 @@ static inline const struct iommu_ops *dev_iommu_ops(struct device *dev) extern int bus_set_iommu(struct bus_type *bus, const struct iommu_ops *ops); extern int bus_iommu_probe(struct bus_type *bus); extern bool iommu_present(struct bus_type *bus); +extern bool device_iommu_capable(struct device *dev, enum iommu_cap cap); extern bool iommu_capable(struct bus_type *bus, enum iommu_cap cap); extern struct iommu_domain *iommu_domain_alloc(struct bus_type *bus); extern struct iommu_group *iommu_group_get_by_id(int id); @@ -689,6 +690,11 @@ static inline bool iommu_present(struct bus_type *bus) return false; } +static inline bool device_iommu_capable(struct device *dev, enum iommu_cap cap) +{ + return false; +} + static inline bool iommu_capable(struct bus_type *bus, enum iommu_cap cap) { return false; -- cgit v1.2.3 From d0be55fbeb6ac694d15af5d1aad19cdec8cd64e5 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 25 Apr 2022 13:42:03 +0100 Subject: iommu: Add capability for pre-boot DMA protection VT-d's dmar_platform_optin() actually represents a combination of properties fairly well standardised by Microsoft as "Pre-boot DMA Protection" and "Kernel DMA Protection"[1]. As such, we can provide interested consumers with an abstracted capability rather than driver-specific interfaces that won't scale. We name it for the former aspect since that's what external callers are most likely to be interested in; the latter is for the IOMMU layer to handle itself. [1] https://docs.microsoft.com/en-us/windows-hardware/design/device-experiences/oem-kernel-dma-protection Suggested-by: Christoph Hellwig Reviewed-by: Christoph Hellwig Reviewed-by: Lu Baolu Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/d6218dff2702472da80db6aec2c9589010684551.1650878781.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 2 ++ include/linux/iommu.h | 2 ++ 2 files changed, 4 insertions(+) (limited to 'drivers') diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index df5c62ecf942..0edf6084dc14 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -4551,6 +4551,8 @@ static bool intel_iommu_capable(enum iommu_cap cap) return domain_update_iommu_snooping(NULL); if (cap == IOMMU_CAP_INTR_REMAP) return irq_remapping_enabled == 1; + if (cap == IOMMU_CAP_PRE_BOOT_PROTECTION) + return dmar_platform_optin(); return false; } diff --git a/include/linux/iommu.h b/include/linux/iommu.h index e26cf84e5d82..4123693ae319 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -107,6 +107,8 @@ enum iommu_cap { transactions */ IOMMU_CAP_INTR_REMAP, /* IOMMU supports interrupt isolation */ IOMMU_CAP_NOEXEC, /* IOMMU_NOEXEC flag */ + IOMMU_CAP_PRE_BOOT_PROTECTION, /* Firmware says it used the IOMMU for + DMA protection and we should too */ }; /* These are the possible reserved region types */ -- cgit v1.2.3 From 86eaf4a5b4312bea8676fb79399d9e08b53d8e71 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 25 Apr 2022 13:42:04 +0100 Subject: thunderbolt: Make iommu_dma_protection more accurate Between me trying to get rid of iommu_present() and Mario wanting to support the AMD equivalent of DMAR_PLATFORM_OPT_IN, scrutiny has shown that the iommu_dma_protection attribute is being far too optimistic. Even if an IOMMU might be present for some PCI segment in the system, that doesn't necessarily mean it provides translation for the device(s) we care about. Furthermore, all that DMAR_PLATFORM_OPT_IN really does is tell us that memory was protected before the kernel was loaded, and prevent the user from disabling the intel-iommu driver entirely. While that lets us assume kernel integrity, what matters for actual runtime DMA protection is whether we trust individual devices, based on the "external facing" property that we expect firmware to describe for Thunderbolt ports. It's proven challenging to determine the appropriate ports accurately given the variety of possible topologies, so while still not getting a perfect answer, by putting enough faith in firmware we can at least get a good bit closer. If we can see that any device near a Thunderbolt NHI has all the requisites for Kernel DMA Protection, chances are that it *is* a relevant port, but moreover that implies that firmware is playing the game overall, so we'll use that to assume that all Thunderbolt ports should be correctly marked and thus will end up fully protected. CC: Mario Limonciello Reviewed-by: Christoph Hellwig Acked-by: Mika Westerberg Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/b153f208bc9eafab5105bad0358b77366509d2d4.1650878781.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/thunderbolt/domain.c | 12 +++--------- drivers/thunderbolt/nhi.c | 44 ++++++++++++++++++++++++++++++++++++++++++++ include/linux/thunderbolt.h | 2 ++ 3 files changed, 49 insertions(+), 9 deletions(-) (limited to 'drivers') diff --git a/drivers/thunderbolt/domain.c b/drivers/thunderbolt/domain.c index 7018d959f775..2889a214dadc 100644 --- a/drivers/thunderbolt/domain.c +++ b/drivers/thunderbolt/domain.c @@ -7,9 +7,7 @@ */ #include -#include #include -#include #include #include #include @@ -257,13 +255,9 @@ static ssize_t iommu_dma_protection_show(struct device *dev, struct device_attribute *attr, char *buf) { - /* - * Kernel DMA protection is a feature where Thunderbolt security is - * handled natively using IOMMU. It is enabled when IOMMU is - * enabled and ACPI DMAR table has DMAR_PLATFORM_OPT_IN set. - */ - return sprintf(buf, "%d\n", - iommu_present(&pci_bus_type) && dmar_platform_optin()); + struct tb *tb = container_of(dev, struct tb, dev); + + return sysfs_emit(buf, "%d\n", tb->nhi->iommu_dma_protection); } static DEVICE_ATTR_RO(iommu_dma_protection); diff --git a/drivers/thunderbolt/nhi.c b/drivers/thunderbolt/nhi.c index 4a582183f675..4bc87b0f003a 100644 --- a/drivers/thunderbolt/nhi.c +++ b/drivers/thunderbolt/nhi.c @@ -15,9 +15,11 @@ #include #include #include +#include #include #include #include +#include #include "nhi.h" #include "nhi_regs.h" @@ -1103,6 +1105,47 @@ static void nhi_check_quirks(struct tb_nhi *nhi) nhi->quirks |= QUIRK_AUTO_CLEAR_INT; } +static int nhi_check_iommu_pdev(struct pci_dev *pdev, void *data) +{ + if (!pdev->external_facing || + !device_iommu_capable(&pdev->dev, IOMMU_CAP_PRE_BOOT_PROTECTION)) + return 0; + *(bool *)data = true; + return 1; /* Stop walking */ +} + +static void nhi_check_iommu(struct tb_nhi *nhi) +{ + struct pci_bus *bus = nhi->pdev->bus; + bool port_ok = false; + + /* + * Ideally what we'd do here is grab every PCI device that + * represents a tunnelling adapter for this NHI and check their + * status directly, but unfortunately USB4 seems to make it + * obnoxiously difficult to reliably make any correlation. + * + * So for now we'll have to bodge it... Hoping that the system + * is at least sane enough that an adapter is in the same PCI + * segment as its NHI, if we can find *something* on that segment + * which meets the requirements for Kernel DMA Protection, we'll + * take that to imply that firmware is aware and has (hopefully) + * done the right thing in general. We need to know that the PCI + * layer has seen the ExternalFacingPort property which will then + * inform the IOMMU layer to enforce the complete "untrusted DMA" + * flow, but also that the IOMMU driver itself can be trusted not + * to have been subverted by a pre-boot DMA attack. + */ + while (bus->parent) + bus = bus->parent; + + pci_walk_bus(bus, nhi_check_iommu_pdev, &port_ok); + + nhi->iommu_dma_protection = port_ok; + dev_dbg(&nhi->pdev->dev, "IOMMU DMA protection is %s\n", + str_enabled_disabled(port_ok)); +} + static int nhi_init_msi(struct tb_nhi *nhi) { struct pci_dev *pdev = nhi->pdev; @@ -1220,6 +1263,7 @@ static int nhi_probe(struct pci_dev *pdev, const struct pci_device_id *id) return -ENOMEM; nhi_check_quirks(nhi); + nhi_check_iommu(nhi); res = nhi_init_msi(nhi); if (res) { diff --git a/include/linux/thunderbolt.h b/include/linux/thunderbolt.h index 124e13cb1469..7a8ad984e651 100644 --- a/include/linux/thunderbolt.h +++ b/include/linux/thunderbolt.h @@ -465,6 +465,7 @@ static inline struct tb_xdomain *tb_service_parent(struct tb_service *svc) * @msix_ida: Used to allocate MSI-X vectors for rings * @going_away: The host controller device is about to disappear so when * this flag is set, avoid touching the hardware anymore. + * @iommu_dma_protection: An IOMMU will isolate external-facing ports. * @interrupt_work: Work scheduled to handle ring interrupt when no * MSI-X is used. * @hop_count: Number of rings (end point hops) supported by NHI. @@ -479,6 +480,7 @@ struct tb_nhi { struct tb_ring **rx_rings; struct ida msix_ida; bool going_away; + bool iommu_dma_protection; struct work_struct interrupt_work; u32 hop_count; unsigned long quirks; -- cgit v1.2.3 From f1ca70717bcb4525e29da422f3d280acbddb36fe Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 25 Apr 2022 13:42:05 +0100 Subject: iommu/amd: Indicate whether DMA remap support is enabled Bit 1 of the IVFS IVInfo field indicates that IOMMU has been used for pre-boot DMA protection. Export this capability to allow other places in the kernel to be able to check for it on AMD systems. Link: https://www.amd.com/system/files/TechDocs/48882_IOMMU.pdf Reviewed-by: Christoph Hellwig Signed-off-by: Mario Limonciello Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/ce7627fa1c596878ca6515dd9d4381a45b6ee38c.1650878781.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu_types.h | 4 ++++ drivers/iommu/amd/init.c | 3 +++ drivers/iommu/amd/iommu.c | 2 ++ 3 files changed, 9 insertions(+) (limited to 'drivers') diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 47108ed44fbb..72d0f5e2f651 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -407,6 +407,7 @@ /* IOMMU IVINFO */ #define IOMMU_IVINFO_OFFSET 36 #define IOMMU_IVINFO_EFRSUP BIT(0) +#define IOMMU_IVINFO_DMA_REMAP BIT(1) /* IOMMU Feature Reporting Field (for IVHD type 10h */ #define IOMMU_FEAT_GASUP_SHIFT 6 @@ -449,6 +450,9 @@ extern struct irq_remap_table **irq_lookup_table; /* Interrupt remapping feature used? */ extern bool amd_iommu_irq_remap; +/* IVRS indicates that pre-boot remapping was enabled */ +extern bool amdr_ivrs_remap_support; + /* kmem_cache to get tables with 128 byte alignement */ extern struct kmem_cache *amd_iommu_irq_cache; diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index b4a798c7b347..0467918bf7fd 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -182,6 +182,7 @@ u32 amd_iommu_max_pasid __read_mostly = ~0; bool amd_iommu_v2_present __read_mostly; static bool amd_iommu_pc_present __read_mostly; +bool amdr_ivrs_remap_support __read_mostly; bool amd_iommu_force_isolation __read_mostly; @@ -326,6 +327,8 @@ static void __init early_iommu_features_init(struct amd_iommu *iommu, { if (amd_iommu_ivinfo & IOMMU_IVINFO_EFRSUP) iommu->features = h->efr_reg; + if (amd_iommu_ivinfo & IOMMU_IVINFO_DMA_REMAP) + amdr_ivrs_remap_support = true; } /* Access to l1 and l2 indexed register spaces */ diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 079694f894b8..038e104b922c 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2155,6 +2155,8 @@ static bool amd_iommu_capable(enum iommu_cap cap) return (irq_remapping_enabled == 1); case IOMMU_CAP_NOEXEC: return false; + case IOMMU_CAP_PRE_BOOT_PROTECTION: + return amdr_ivrs_remap_support; default: break; } -- cgit v1.2.3 From bb5bdc5ab7f133ba6fd32657d2ac90039c561e48 Mon Sep 17 00:00:00 2001 From: Xiaoke Wang Date: Thu, 28 Apr 2022 16:52:39 +0800 Subject: iommu/msm: Add a check for the return of kzalloc() kzalloc() is a memory allocation function which can return NULL when some internal memory errors happen. So it is better to check it to prevent potential wrong memory access. Besides, to propagate the error to the caller, the type of insert_iommu_master() is changed to `int`. Several instructions related to it are also updated. Signed-off-by: Xiaoke Wang Link: https://lore.kernel.org/r/tencent_EDB94B1C7E14B4E1974A66FF4D2029CC6D08@qq.com Signed-off-by: Joerg Roedel --- drivers/iommu/msm_iommu.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c index 50f57624610f..3d9bd2043738 100644 --- a/drivers/iommu/msm_iommu.c +++ b/drivers/iommu/msm_iommu.c @@ -583,7 +583,7 @@ static void print_ctx_regs(void __iomem *base, int ctx) GET_SCTLR(base, ctx), GET_ACTLR(base, ctx)); } -static void insert_iommu_master(struct device *dev, +static int insert_iommu_master(struct device *dev, struct msm_iommu_dev **iommu, struct of_phandle_args *spec) { @@ -592,6 +592,10 @@ static void insert_iommu_master(struct device *dev, if (list_empty(&(*iommu)->ctx_list)) { master = kzalloc(sizeof(*master), GFP_ATOMIC); + if (!master) { + dev_err(dev, "Failed to allocate iommu_master\n"); + return -ENOMEM; + } master->of_node = dev->of_node; list_add(&master->list, &(*iommu)->ctx_list); dev_iommu_priv_set(dev, master); @@ -601,10 +605,11 @@ static void insert_iommu_master(struct device *dev, if (master->mids[sid] == spec->args[0]) { dev_warn(dev, "Stream ID 0x%hx repeated; ignoring\n", sid); - return; + return 0; } master->mids[master->num_mids++] = spec->args[0]; + return 0; } static int qcom_iommu_of_xlate(struct device *dev, @@ -624,7 +629,7 @@ static int qcom_iommu_of_xlate(struct device *dev, goto fail; } - insert_iommu_master(dev, &iommu, spec); + ret = insert_iommu_master(dev, &iommu, spec); fail: spin_unlock_irqrestore(&msm_iommu_lock, flags); -- cgit v1.2.3 From 1ea2a07a532b0e22aabe7e8483f935c672b9e7ed Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 18 Apr 2022 08:49:50 +0800 Subject: iommu: Add DMA ownership management interfaces Multiple devices may be placed in the same IOMMU group because they cannot be isolated from each other. These devices must either be entirely under kernel control or userspace control, never a mixture. This adds dma ownership management in iommu core and exposes several interfaces for the device drivers and the device userspace assignment framework (i.e. VFIO), so that any conflict between user and kernel controlled dma could be detected at the beginning. The device driver oriented interfaces are, int iommu_device_use_default_domain(struct device *dev); void iommu_device_unuse_default_domain(struct device *dev); By calling iommu_device_use_default_domain(), the device driver tells the iommu layer that the device dma is handled through the kernel DMA APIs. The iommu layer will manage the IOVA and use the default domain for DMA address translation. The device user-space assignment framework oriented interfaces are, int iommu_group_claim_dma_owner(struct iommu_group *group, void *owner); void iommu_group_release_dma_owner(struct iommu_group *group); bool iommu_group_dma_owner_claimed(struct iommu_group *group); The device userspace assignment must be disallowed if the DMA owner claiming interface returns failure. Signed-off-by: Jason Gunthorpe Signed-off-by: Kevin Tian Signed-off-by: Lu Baolu Reviewed-by: Robin Murphy Link: https://lore.kernel.org/r/20220418005000.897664-2-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 153 +++++++++++++++++++++++++++++++++++++++++++++++++- include/linux/iommu.h | 31 ++++++++++ 2 files changed, 181 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index f2c45b85b9fc..eba8e8ccf19d 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -48,6 +48,8 @@ struct iommu_group { struct iommu_domain *default_domain; struct iommu_domain *domain; struct list_head entry; + unsigned int owner_cnt; + void *owner; }; struct group_device { @@ -294,7 +296,11 @@ int iommu_probe_device(struct device *dev) mutex_lock(&group->mutex); iommu_alloc_default_domain(group, dev); - if (group->default_domain) { + /* + * If device joined an existing group which has been claimed, don't + * attach the default domain. + */ + if (group->default_domain && !group->owner) { ret = __iommu_attach_device(group->default_domain, dev); if (ret) { mutex_unlock(&group->mutex); @@ -2109,7 +2115,7 @@ static int __iommu_attach_group(struct iommu_domain *domain, { int ret; - if (group->default_domain && group->domain != group->default_domain) + if (group->domain && group->domain != group->default_domain) return -EBUSY; ret = __iommu_group_for_each_dev(group, domain, @@ -2146,7 +2152,11 @@ static void __iommu_detach_group(struct iommu_domain *domain, { int ret; - if (!group->default_domain) { + /* + * If the group has been claimed already, do not re-attach the default + * domain. + */ + if (!group->default_domain || group->owner) { __iommu_group_for_each_dev(group, domain, iommu_group_do_detach_device); group->domain = NULL; @@ -3095,3 +3105,140 @@ out: return ret; } + +/** + * iommu_device_use_default_domain() - Device driver wants to handle device + * DMA through the kernel DMA API. + * @dev: The device. + * + * The device driver about to bind @dev wants to do DMA through the kernel + * DMA API. Return 0 if it is allowed, otherwise an error. + */ +int iommu_device_use_default_domain(struct device *dev) +{ + struct iommu_group *group = iommu_group_get(dev); + int ret = 0; + + if (!group) + return 0; + + mutex_lock(&group->mutex); + if (group->owner_cnt) { + if (group->domain != group->default_domain || + group->owner) { + ret = -EBUSY; + goto unlock_out; + } + } + + group->owner_cnt++; + +unlock_out: + mutex_unlock(&group->mutex); + iommu_group_put(group); + + return ret; +} + +/** + * iommu_device_unuse_default_domain() - Device driver stops handling device + * DMA through the kernel DMA API. + * @dev: The device. + * + * The device driver doesn't want to do DMA through kernel DMA API anymore. + * It must be called after iommu_device_use_default_domain(). + */ +void iommu_device_unuse_default_domain(struct device *dev) +{ + struct iommu_group *group = iommu_group_get(dev); + + if (!group) + return; + + mutex_lock(&group->mutex); + if (!WARN_ON(!group->owner_cnt)) + group->owner_cnt--; + + mutex_unlock(&group->mutex); + iommu_group_put(group); +} + +/** + * iommu_group_claim_dma_owner() - Set DMA ownership of a group + * @group: The group. + * @owner: Caller specified pointer. Used for exclusive ownership. + * + * This is to support backward compatibility for vfio which manages + * the dma ownership in iommu_group level. New invocations on this + * interface should be prohibited. + */ +int iommu_group_claim_dma_owner(struct iommu_group *group, void *owner) +{ + int ret = 0; + + mutex_lock(&group->mutex); + if (group->owner_cnt) { + ret = -EPERM; + goto unlock_out; + } else { + if (group->domain && group->domain != group->default_domain) { + ret = -EBUSY; + goto unlock_out; + } + + group->owner = owner; + if (group->domain) + __iommu_detach_group(group->domain, group); + } + + group->owner_cnt++; +unlock_out: + mutex_unlock(&group->mutex); + + return ret; +} +EXPORT_SYMBOL_GPL(iommu_group_claim_dma_owner); + +/** + * iommu_group_release_dma_owner() - Release DMA ownership of a group + * @group: The group. + * + * Release the DMA ownership claimed by iommu_group_claim_dma_owner(). + */ +void iommu_group_release_dma_owner(struct iommu_group *group) +{ + mutex_lock(&group->mutex); + if (WARN_ON(!group->owner_cnt || !group->owner)) + goto unlock_out; + + group->owner_cnt = 0; + /* + * The UNMANAGED domain should be detached before all USER + * owners have been released. + */ + if (!WARN_ON(group->domain) && group->default_domain) + __iommu_attach_group(group->default_domain, group); + group->owner = NULL; +unlock_out: + mutex_unlock(&group->mutex); +} +EXPORT_SYMBOL_GPL(iommu_group_release_dma_owner); + +/** + * iommu_group_dma_owner_claimed() - Query group dma ownership status + * @group: The group. + * + * This provides status query on a given group. It is racy and only for + * non-binding status reporting. + */ +bool iommu_group_dma_owner_claimed(struct iommu_group *group) +{ + unsigned int user; + + mutex_lock(&group->mutex); + user = group->owner_cnt; + mutex_unlock(&group->mutex); + + return user; +} +EXPORT_SYMBOL_GPL(iommu_group_dma_owner_claimed); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 9208eca4b0d1..77972ef978b5 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -675,6 +675,13 @@ struct iommu_sva *iommu_sva_bind_device(struct device *dev, void iommu_sva_unbind_device(struct iommu_sva *handle); u32 iommu_sva_get_pasid(struct iommu_sva *handle); +int iommu_device_use_default_domain(struct device *dev); +void iommu_device_unuse_default_domain(struct device *dev); + +int iommu_group_claim_dma_owner(struct iommu_group *group, void *owner); +void iommu_group_release_dma_owner(struct iommu_group *group); +bool iommu_group_dma_owner_claimed(struct iommu_group *group); + #else /* CONFIG_IOMMU_API */ struct iommu_ops {}; @@ -1031,6 +1038,30 @@ static inline struct iommu_fwspec *dev_iommu_fwspec_get(struct device *dev) { return NULL; } + +static inline int iommu_device_use_default_domain(struct device *dev) +{ + return 0; +} + +static inline void iommu_device_unuse_default_domain(struct device *dev) +{ +} + +static inline int +iommu_group_claim_dma_owner(struct iommu_group *group, void *owner) +{ + return -ENODEV; +} + +static inline void iommu_group_release_dma_owner(struct iommu_group *group) +{ +} + +static inline bool iommu_group_dma_owner_claimed(struct iommu_group *group) +{ + return false; +} #endif /* CONFIG_IOMMU_API */ /** -- cgit v1.2.3 From 25f3bcfc54bcf7b0e45d140ec8bfbbf10ba11869 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 18 Apr 2022 08:49:51 +0800 Subject: driver core: Add dma_cleanup callback in bus_type The bus_type structure defines dma_configure() callback for bus drivers to configure DMA on the devices. This adds the paired dma_cleanup() callback and calls it during driver unbinding so that bus drivers can do some cleanup work. One use case for this paired DMA callbacks is for the bus driver to check for DMA ownership conflicts during driver binding, where multiple devices belonging to a same IOMMU group (the minimum granularity of isolation and protection) may be assigned to kernel drivers or user space respectively. Without this change, for example, the vfio driver has to listen to a bus BOUND_DRIVER event and then BUG_ON() in case of dma ownership conflict. This leads to bad user experience since careless driver binding operation may crash the system if the admin overlooks the group restriction. Aside from bad design, this leads to a security problem as a root user, even with lockdown=integrity, can force the kernel to BUG. With this change, the bus driver could check and set the DMA ownership in driver binding process and fail on ownership conflicts. The DMA ownership should be released during driver unbinding. Signed-off-by: Lu Baolu Reviewed-by: Greg Kroah-Hartman Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20220418005000.897664-3-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/base/dd.c | 5 +++++ include/linux/device/bus.h | 3 +++ 2 files changed, 8 insertions(+) (limited to 'drivers') diff --git a/drivers/base/dd.c b/drivers/base/dd.c index 3fc3b5940bb3..94b7ac9bf459 100644 --- a/drivers/base/dd.c +++ b/drivers/base/dd.c @@ -671,6 +671,8 @@ sysfs_failed: if (dev->bus) blocking_notifier_call_chain(&dev->bus->p->bus_notifier, BUS_NOTIFY_DRIVER_NOT_BOUND, dev); + if (dev->bus && dev->bus->dma_cleanup) + dev->bus->dma_cleanup(dev); pinctrl_bind_failed: device_links_no_driver(dev); device_unbind_cleanup(dev); @@ -1199,6 +1201,9 @@ static void __device_release_driver(struct device *dev, struct device *parent) device_remove(dev); + if (dev->bus && dev->bus->dma_cleanup) + dev->bus->dma_cleanup(dev); + device_links_driver_cleanup(dev); device_unbind_cleanup(dev); diff --git a/include/linux/device/bus.h b/include/linux/device/bus.h index a039ab809753..d8b29ccd07e5 100644 --- a/include/linux/device/bus.h +++ b/include/linux/device/bus.h @@ -59,6 +59,8 @@ struct fwnode_handle; * bus supports. * @dma_configure: Called to setup DMA configuration on a device on * this bus. + * @dma_cleanup: Called to cleanup DMA configuration on a device on + * this bus. * @pm: Power management operations of this bus, callback the specific * device driver's pm-ops. * @iommu_ops: IOMMU specific operations for this bus, used to attach IOMMU @@ -103,6 +105,7 @@ struct bus_type { int (*num_vf)(struct device *dev); int (*dma_configure)(struct device *dev); + void (*dma_cleanup)(struct device *dev); const struct dev_pm_ops *pm; -- cgit v1.2.3 From 4a6d9dd564d0e7339fc15ecc5ce66db4ad842be2 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 18 Apr 2022 08:49:52 +0800 Subject: amba: Stop sharing platform_dma_configure() Stop sharing platform_dma_configure() helper as they are about to have their own bus dma_configure callbacks. Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20220418005000.897664-4-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/amba/bus.c | 19 ++++++++++++++++++- drivers/base/platform.c | 3 +-- include/linux/platform_device.h | 2 -- 3 files changed, 19 insertions(+), 5 deletions(-) (limited to 'drivers') diff --git a/drivers/amba/bus.c b/drivers/amba/bus.c index d3bd14aaabf6..76b52bd2c2a4 100644 --- a/drivers/amba/bus.c +++ b/drivers/amba/bus.c @@ -20,6 +20,8 @@ #include #include #include +#include +#include #define to_amba_driver(d) container_of(d, struct amba_driver, drv) @@ -273,6 +275,21 @@ static void amba_shutdown(struct device *dev) drv->shutdown(to_amba_device(dev)); } +static int amba_dma_configure(struct device *dev) +{ + enum dev_dma_attr attr; + int ret = 0; + + if (dev->of_node) { + ret = of_dma_configure(dev, dev->of_node, true); + } else if (has_acpi_companion(dev)) { + attr = acpi_get_dma_attr(to_acpi_device_node(dev->fwnode)); + ret = acpi_dma_configure(dev, attr); + } + + return ret; +} + #ifdef CONFIG_PM /* * Hooks to provide runtime PM of the pclk (bus clock). It is safe to @@ -341,7 +358,7 @@ struct bus_type amba_bustype = { .probe = amba_probe, .remove = amba_remove, .shutdown = amba_shutdown, - .dma_configure = platform_dma_configure, + .dma_configure = amba_dma_configure, .pm = &amba_pm, }; EXPORT_SYMBOL_GPL(amba_bustype); diff --git a/drivers/base/platform.c b/drivers/base/platform.c index 8cc272fd5c99..d7915734d931 100644 --- a/drivers/base/platform.c +++ b/drivers/base/platform.c @@ -1454,8 +1454,7 @@ static void platform_shutdown(struct device *_dev) drv->shutdown(dev); } - -int platform_dma_configure(struct device *dev) +static int platform_dma_configure(struct device *dev) { enum dev_dma_attr attr; int ret = 0; diff --git a/include/linux/platform_device.h b/include/linux/platform_device.h index 7c96f169d274..17fde717df68 100644 --- a/include/linux/platform_device.h +++ b/include/linux/platform_device.h @@ -328,8 +328,6 @@ extern int platform_pm_restore(struct device *dev); #define platform_pm_restore NULL #endif -extern int platform_dma_configure(struct device *dev); - #ifdef CONFIG_PM_SLEEP #define USE_PLATFORM_PM_SLEEP_OPS \ .suspend = platform_pm_suspend, \ -- cgit v1.2.3 From 512881eacfa72c2136b27b9934b7b27504a9efc2 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 18 Apr 2022 08:49:53 +0800 Subject: bus: platform,amba,fsl-mc,PCI: Add device DMA ownership management The devices on platform/amba/fsl-mc/PCI buses could be bound to drivers with the device DMA managed by kernel drivers or user-space applications. Unfortunately, multiple devices may be placed in the same IOMMU group because they cannot be isolated from each other. The DMA on these devices must either be entirely under kernel control or userspace control, never a mixture. Otherwise the driver integrity is not guaranteed because they could access each other through the peer-to-peer accesses which by-pass the IOMMU protection. This checks and sets the default DMA mode during driver binding, and cleanups during driver unbinding. In the default mode, the device DMA is managed by the device driver which handles DMA operations through the kernel DMA APIs (see Documentation/core-api/dma-api.rst). For cases where the devices are assigned for userspace control through the userspace driver framework(i.e. VFIO), the drivers(for example, vfio_pci/ vfio_platfrom etc.) may set a new flag (driver_managed_dma) to skip this default setting in the assumption that the drivers know what they are doing with the device DMA. Calling iommu_device_use_default_domain() before {of,acpi}_dma_configure is currently a problem. As things stand, the IOMMU driver ignored the initial iommu_probe_device() call when the device was added, since at that point it had no fwspec yet. In this situation, {of,acpi}_iommu_configure() are retriggering iommu_probe_device() after the IOMMU driver has seen the firmware data via .of_xlate to learn that it actually responsible for the given device. As the result, before that gets fixed, iommu_use_default_domain() goes at the end, and calls arch_teardown_dma_ops() if it fails. Cc: Greg Kroah-Hartman Cc: Bjorn Helgaas Cc: Stuart Yoder Cc: Laurentiu Tudor Signed-off-by: Lu Baolu Reviewed-by: Greg Kroah-Hartman Reviewed-by: Jason Gunthorpe Reviewed-by: Robin Murphy Tested-by: Eric Auger Link: https://lore.kernel.org/r/20220418005000.897664-5-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/amba/bus.c | 18 ++++++++++++++++++ drivers/base/platform.c | 18 ++++++++++++++++++ drivers/bus/fsl-mc/fsl-mc-bus.c | 24 ++++++++++++++++++++++-- drivers/pci/pci-driver.c | 18 ++++++++++++++++++ include/linux/amba/bus.h | 8 ++++++++ include/linux/fsl/mc.h | 8 ++++++++ include/linux/pci.h | 8 ++++++++ include/linux/platform_device.h | 8 ++++++++ 8 files changed, 108 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/amba/bus.c b/drivers/amba/bus.c index 76b52bd2c2a4..a0ec61232b6c 100644 --- a/drivers/amba/bus.c +++ b/drivers/amba/bus.c @@ -22,6 +22,8 @@ #include #include #include +#include +#include #define to_amba_driver(d) container_of(d, struct amba_driver, drv) @@ -277,6 +279,7 @@ static void amba_shutdown(struct device *dev) static int amba_dma_configure(struct device *dev) { + struct amba_driver *drv = to_amba_driver(dev->driver); enum dev_dma_attr attr; int ret = 0; @@ -287,9 +290,23 @@ static int amba_dma_configure(struct device *dev) ret = acpi_dma_configure(dev, attr); } + if (!ret && !drv->driver_managed_dma) { + ret = iommu_device_use_default_domain(dev); + if (ret) + arch_teardown_dma_ops(dev); + } + return ret; } +static void amba_dma_cleanup(struct device *dev) +{ + struct amba_driver *drv = to_amba_driver(dev->driver); + + if (!drv->driver_managed_dma) + iommu_device_unuse_default_domain(dev); +} + #ifdef CONFIG_PM /* * Hooks to provide runtime PM of the pclk (bus clock). It is safe to @@ -359,6 +376,7 @@ struct bus_type amba_bustype = { .remove = amba_remove, .shutdown = amba_shutdown, .dma_configure = amba_dma_configure, + .dma_cleanup = amba_dma_cleanup, .pm = &amba_pm, }; EXPORT_SYMBOL_GPL(amba_bustype); diff --git a/drivers/base/platform.c b/drivers/base/platform.c index d7915734d931..70bc30cf575c 100644 --- a/drivers/base/platform.c +++ b/drivers/base/platform.c @@ -30,6 +30,8 @@ #include #include #include +#include +#include #include "base.h" #include "power/power.h" @@ -1456,6 +1458,7 @@ static void platform_shutdown(struct device *_dev) static int platform_dma_configure(struct device *dev) { + struct platform_driver *drv = to_platform_driver(dev->driver); enum dev_dma_attr attr; int ret = 0; @@ -1466,9 +1469,23 @@ static int platform_dma_configure(struct device *dev) ret = acpi_dma_configure(dev, attr); } + if (!ret && !drv->driver_managed_dma) { + ret = iommu_device_use_default_domain(dev); + if (ret) + arch_teardown_dma_ops(dev); + } + return ret; } +static void platform_dma_cleanup(struct device *dev) +{ + struct platform_driver *drv = to_platform_driver(dev->driver); + + if (!drv->driver_managed_dma) + iommu_device_unuse_default_domain(dev); +} + static const struct dev_pm_ops platform_dev_pm_ops = { SET_RUNTIME_PM_OPS(pm_generic_runtime_suspend, pm_generic_runtime_resume, NULL) USE_PLATFORM_PM_SLEEP_OPS @@ -1483,6 +1500,7 @@ struct bus_type platform_bus_type = { .remove = platform_remove, .shutdown = platform_shutdown, .dma_configure = platform_dma_configure, + .dma_cleanup = platform_dma_cleanup, .pm = &platform_dev_pm_ops, }; EXPORT_SYMBOL_GPL(platform_bus_type); diff --git a/drivers/bus/fsl-mc/fsl-mc-bus.c b/drivers/bus/fsl-mc/fsl-mc-bus.c index 8fd4a356a86e..76648c4fdaf4 100644 --- a/drivers/bus/fsl-mc/fsl-mc-bus.c +++ b/drivers/bus/fsl-mc/fsl-mc-bus.c @@ -21,6 +21,7 @@ #include #include #include +#include #include "fsl-mc-private.h" @@ -140,15 +141,33 @@ static int fsl_mc_dma_configure(struct device *dev) { struct device *dma_dev = dev; struct fsl_mc_device *mc_dev = to_fsl_mc_device(dev); + struct fsl_mc_driver *mc_drv = to_fsl_mc_driver(dev->driver); u32 input_id = mc_dev->icid; + int ret; while (dev_is_fsl_mc(dma_dev)) dma_dev = dma_dev->parent; if (dev_of_node(dma_dev)) - return of_dma_configure_id(dev, dma_dev->of_node, 0, &input_id); + ret = of_dma_configure_id(dev, dma_dev->of_node, 0, &input_id); + else + ret = acpi_dma_configure_id(dev, DEV_DMA_COHERENT, &input_id); + + if (!ret && !mc_drv->driver_managed_dma) { + ret = iommu_device_use_default_domain(dev); + if (ret) + arch_teardown_dma_ops(dev); + } + + return ret; +} + +static void fsl_mc_dma_cleanup(struct device *dev) +{ + struct fsl_mc_driver *mc_drv = to_fsl_mc_driver(dev->driver); - return acpi_dma_configure_id(dev, DEV_DMA_COHERENT, &input_id); + if (!mc_drv->driver_managed_dma) + iommu_device_unuse_default_domain(dev); } static ssize_t modalias_show(struct device *dev, struct device_attribute *attr, @@ -312,6 +331,7 @@ struct bus_type fsl_mc_bus_type = { .match = fsl_mc_bus_match, .uevent = fsl_mc_bus_uevent, .dma_configure = fsl_mc_dma_configure, + .dma_cleanup = fsl_mc_dma_cleanup, .dev_groups = fsl_mc_dev_groups, .bus_groups = fsl_mc_bus_groups, }; diff --git a/drivers/pci/pci-driver.c b/drivers/pci/pci-driver.c index 4ceeb75fc899..f83f7fbac68f 100644 --- a/drivers/pci/pci-driver.c +++ b/drivers/pci/pci-driver.c @@ -20,6 +20,7 @@ #include #include #include +#include #include "pci.h" #include "pcie/portdrv.h" @@ -1601,6 +1602,7 @@ static int pci_bus_num_vf(struct device *dev) */ static int pci_dma_configure(struct device *dev) { + struct pci_driver *driver = to_pci_driver(dev->driver); struct device *bridge; int ret = 0; @@ -1616,9 +1618,24 @@ static int pci_dma_configure(struct device *dev) } pci_put_host_bridge_device(bridge); + + if (!ret && !driver->driver_managed_dma) { + ret = iommu_device_use_default_domain(dev); + if (ret) + arch_teardown_dma_ops(dev); + } + return ret; } +static void pci_dma_cleanup(struct device *dev) +{ + struct pci_driver *driver = to_pci_driver(dev->driver); + + if (!driver->driver_managed_dma) + iommu_device_unuse_default_domain(dev); +} + struct bus_type pci_bus_type = { .name = "pci", .match = pci_bus_match, @@ -1632,6 +1649,7 @@ struct bus_type pci_bus_type = { .pm = PCI_PM_OPS_PTR, .num_vf = pci_bus_num_vf, .dma_configure = pci_dma_configure, + .dma_cleanup = pci_dma_cleanup, }; EXPORT_SYMBOL(pci_bus_type); diff --git a/include/linux/amba/bus.h b/include/linux/amba/bus.h index 6562f543c3e0..2ddce9bcd00e 100644 --- a/include/linux/amba/bus.h +++ b/include/linux/amba/bus.h @@ -79,6 +79,14 @@ struct amba_driver { void (*remove)(struct amba_device *); void (*shutdown)(struct amba_device *); const struct amba_id *id_table; + /* + * For most device drivers, no need to care about this flag as long as + * all DMAs are handled through the kernel DMA API. For some special + * ones, for example VFIO drivers, they know how to manage the DMA + * themselves and set this flag so that the IOMMU layer will allow them + * to setup and manage their own I/O address space. + */ + bool driver_managed_dma; }; /* diff --git a/include/linux/fsl/mc.h b/include/linux/fsl/mc.h index 7b6c42bfb660..27efef8affb1 100644 --- a/include/linux/fsl/mc.h +++ b/include/linux/fsl/mc.h @@ -32,6 +32,13 @@ struct fsl_mc_io; * @shutdown: Function called at shutdown time to quiesce the device * @suspend: Function called when a device is stopped * @resume: Function called when a device is resumed + * @driver_managed_dma: Device driver doesn't use kernel DMA API for DMA. + * For most device drivers, no need to care about this flag + * as long as all DMAs are handled through the kernel DMA API. + * For some special ones, for example VFIO drivers, they know + * how to manage the DMA themselves and set this flag so that + * the IOMMU layer will allow them to setup and manage their + * own I/O address space. * * Generic DPAA device driver object for device drivers that are registered * with a DPRC bus. This structure is to be embedded in each device-specific @@ -45,6 +52,7 @@ struct fsl_mc_driver { void (*shutdown)(struct fsl_mc_device *dev); int (*suspend)(struct fsl_mc_device *dev, pm_message_t state); int (*resume)(struct fsl_mc_device *dev); + bool driver_managed_dma; }; #define to_fsl_mc_driver(_drv) \ diff --git a/include/linux/pci.h b/include/linux/pci.h index 60adf42460ab..b933d2b08d4d 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -895,6 +895,13 @@ struct module; * created once it is bound to the driver. * @driver: Driver model structure. * @dynids: List of dynamically added device IDs. + * @driver_managed_dma: Device driver doesn't use kernel DMA API for DMA. + * For most device drivers, no need to care about this flag + * as long as all DMAs are handled through the kernel DMA API. + * For some special ones, for example VFIO drivers, they know + * how to manage the DMA themselves and set this flag so that + * the IOMMU layer will allow them to setup and manage their + * own I/O address space. */ struct pci_driver { struct list_head node; @@ -913,6 +920,7 @@ struct pci_driver { const struct attribute_group **dev_groups; struct device_driver driver; struct pci_dynids dynids; + bool driver_managed_dma; }; static inline struct pci_driver *to_pci_driver(struct device_driver *drv) diff --git a/include/linux/platform_device.h b/include/linux/platform_device.h index 17fde717df68..b3d9c744f1e5 100644 --- a/include/linux/platform_device.h +++ b/include/linux/platform_device.h @@ -210,6 +210,14 @@ struct platform_driver { struct device_driver driver; const struct platform_device_id *id_table; bool prevent_deferred_probe; + /* + * For most device drivers, no need to care about this flag as long as + * all DMAs are handled through the kernel DMA API. For some special + * ones, for example VFIO drivers, they know how to manage the DMA + * themselves and set this flag so that the IOMMU layer will allow them + * to setup and manage their own I/O address space. + */ + bool driver_managed_dma; }; #define to_platform_driver(drv) (container_of((drv), struct platform_driver, \ -- cgit v1.2.3 From 18c7a349d072a222ff80598373035820c194747b Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 18 Apr 2022 08:49:54 +0800 Subject: PCI: pci_stub: Set driver_managed_dma The current VFIO implementation allows pci-stub driver to be bound to a PCI device with other devices in the same IOMMU group being assigned to userspace. The pci-stub driver has no dependencies on DMA or the IOVA mapping of the device, but it does prevent the user from having direct access to the device, which is useful in some circumstances. The pci_dma_configure() marks the iommu_group as containing only devices with kernel drivers that manage DMA. For compatibility with the VFIO usage, avoid this default behavior for the pci_stub. This allows the pci_stub still able to be used by the admin to block driver binding after applying the DMA ownership to VFIO. Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Acked-by: Bjorn Helgaas Link: https://lore.kernel.org/r/20220418005000.897664-6-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/pci/pci-stub.c | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers') diff --git a/drivers/pci/pci-stub.c b/drivers/pci/pci-stub.c index e408099fea52..d1f4c1ce7bd1 100644 --- a/drivers/pci/pci-stub.c +++ b/drivers/pci/pci-stub.c @@ -36,6 +36,7 @@ static struct pci_driver stub_driver = { .name = "pci-stub", .id_table = NULL, /* only dynamic id's */ .probe = pci_stub_probe, + .driver_managed_dma = true, }; static int __init pci_stub_init(void) -- cgit v1.2.3 From c7d469849747ce380785536173d08b30a820a83c Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 18 Apr 2022 08:49:55 +0800 Subject: PCI: portdrv: Set driver_managed_dma If a switch lacks ACS P2P Request Redirect, a device below the switch can bypass the IOMMU and DMA directly to other devices below the switch, so all the downstream devices must be in the same IOMMU group as the switch itself. The existing VFIO framework allows the portdrv driver to be bound to the bridge while its downstream devices are assigned to user space. The pci_dma_configure() marks the IOMMU group as containing only devices with kernel drivers that manage DMA. Avoid this default behavior for the portdrv driver in order for compatibility with the current VFIO usage. We achieve this by setting ".driver_managed_dma = true" in pci_driver structure. It is safe because the portdrv driver meets below criteria: - This driver doesn't use DMA, as you can't find any related calls like pci_set_master() or any kernel DMA API (dma_map_*() and etc.). - It doesn't use MMIO as you can't find ioremap() or similar calls. It's tolerant to userspace possibly also touching the same MMIO registers via P2P DMA access. Suggested-by: Jason Gunthorpe Suggested-by: Kevin Tian Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Acked-by: Bjorn Helgaas Link: https://lore.kernel.org/r/20220418005000.897664-7-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/pci/pcie/portdrv_pci.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'drivers') diff --git a/drivers/pci/pcie/portdrv_pci.c b/drivers/pci/pcie/portdrv_pci.c index 4b8801656ffb..7f8788a970ae 100644 --- a/drivers/pci/pcie/portdrv_pci.c +++ b/drivers/pci/pcie/portdrv_pci.c @@ -202,6 +202,8 @@ static struct pci_driver pcie_portdriver = { .err_handler = &pcie_portdrv_err_handler, + .driver_managed_dma = true, + .driver.pm = PCIE_PORTDRV_PM_OPS, }; -- cgit v1.2.3 From 70693f470848f05fae04144675e6e41b105d45fe Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 18 Apr 2022 08:49:56 +0800 Subject: vfio: Set DMA ownership for VFIO devices Claim group dma ownership when an IOMMU group is set to a container, and release the dma ownership once the iommu group is unset from the container. This change disallows some unsafe bridge drivers to bind to non-ACS bridges while devices under them are assigned to user space. This is an intentional enhancement and possibly breaks some existing configurations. The recommendation to such an affected user would be that the previously allowed host bridge driver was unsafe for this use case and to continue to enable assignment of devices within that group, the driver should be unbound from the bridge device or replaced with the pci-stub driver. For any bridge driver, we consider it unsafe if it satisfies any of the following conditions: 1) The bridge driver uses DMA. Calling pci_set_master() or calling any kernel DMA API (dma_map_*() and etc.) is an indicate that the driver is doing DMA. 2) If the bridge driver uses MMIO, it should be tolerant to hostile userspace also touching the same MMIO registers via P2P DMA attacks. If the bridge driver turns out to be a safe one, it could be used as before by setting the driver's .driver_managed_dma field, just like what we have done in the pcieport driver. Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Acked-by: Alex Williamson Link: https://lore.kernel.org/r/20220418005000.897664-8-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/vfio/fsl-mc/vfio_fsl_mc.c | 1 + drivers/vfio/pci/vfio_pci.c | 1 + drivers/vfio/platform/vfio_amba.c | 1 + drivers/vfio/platform/vfio_platform.c | 1 + drivers/vfio/vfio.c | 10 +++++++++- 5 files changed, 13 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/vfio/fsl-mc/vfio_fsl_mc.c b/drivers/vfio/fsl-mc/vfio_fsl_mc.c index 6e2e62c6f47a..3feff729f3ce 100644 --- a/drivers/vfio/fsl-mc/vfio_fsl_mc.c +++ b/drivers/vfio/fsl-mc/vfio_fsl_mc.c @@ -588,6 +588,7 @@ static struct fsl_mc_driver vfio_fsl_mc_driver = { .name = "vfio-fsl-mc", .owner = THIS_MODULE, }, + .driver_managed_dma = true, }; static int __init vfio_fsl_mc_driver_init(void) diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c index 2b047469e02f..58839206d1ca 100644 --- a/drivers/vfio/pci/vfio_pci.c +++ b/drivers/vfio/pci/vfio_pci.c @@ -194,6 +194,7 @@ static struct pci_driver vfio_pci_driver = { .remove = vfio_pci_remove, .sriov_configure = vfio_pci_sriov_configure, .err_handler = &vfio_pci_core_err_handlers, + .driver_managed_dma = true, }; static void __init vfio_pci_fill_ids(void) diff --git a/drivers/vfio/platform/vfio_amba.c b/drivers/vfio/platform/vfio_amba.c index badfffea14fb..1aaa4f721bd2 100644 --- a/drivers/vfio/platform/vfio_amba.c +++ b/drivers/vfio/platform/vfio_amba.c @@ -95,6 +95,7 @@ static struct amba_driver vfio_amba_driver = { .name = "vfio-amba", .owner = THIS_MODULE, }, + .driver_managed_dma = true, }; module_amba_driver(vfio_amba_driver); diff --git a/drivers/vfio/platform/vfio_platform.c b/drivers/vfio/platform/vfio_platform.c index 68a1c87066d7..04f40c5acfd6 100644 --- a/drivers/vfio/platform/vfio_platform.c +++ b/drivers/vfio/platform/vfio_platform.c @@ -76,6 +76,7 @@ static struct platform_driver vfio_platform_driver = { .driver = { .name = "vfio-platform", }, + .driver_managed_dma = true, }; module_platform_driver(vfio_platform_driver); diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c index a4555014bd1e..56e741cbccce 100644 --- a/drivers/vfio/vfio.c +++ b/drivers/vfio/vfio.c @@ -1198,6 +1198,8 @@ static void __vfio_group_unset_container(struct vfio_group *group) driver->ops->detach_group(container->iommu_data, group->iommu_group); + iommu_group_release_dma_owner(group->iommu_group); + group->container = NULL; wake_up(&group->container_q); list_del(&group->container_next); @@ -1282,13 +1284,19 @@ static int vfio_group_set_container(struct vfio_group *group, int container_fd) goto unlock_out; } + ret = iommu_group_claim_dma_owner(group->iommu_group, f.file); + if (ret) + goto unlock_out; + driver = container->iommu_driver; if (driver) { ret = driver->ops->attach_group(container->iommu_data, group->iommu_group, group->type); - if (ret) + if (ret) { + iommu_group_release_dma_owner(group->iommu_group); goto unlock_out; + } } group->container = container; -- cgit v1.2.3 From 31076af0cb001f698138c8158c798ec9f4a8bbae Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 18 Apr 2022 08:49:57 +0800 Subject: vfio: Remove use of vfio_group_viable() As DMA ownership is claimed for the iommu group when a VFIO group is added to a VFIO container, the VFIO group viability is guaranteed as long as group->container_users > 0. Remove those unnecessary group viability checks which are only hit when group->container_users is not zero. The only remaining reference is in GROUP_GET_STATUS, which could be called at any time when group fd is valid. Here we just replace the vfio_group_viable() by directly calling IOMMU core to get viability status. Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Acked-by: Alex Williamson Link: https://lore.kernel.org/r/20220418005000.897664-9-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/vfio/vfio.c | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) (limited to 'drivers') diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c index 56e741cbccce..8a9347f732a5 100644 --- a/drivers/vfio/vfio.c +++ b/drivers/vfio/vfio.c @@ -1313,12 +1313,6 @@ unlock_out: return ret; } -static bool vfio_group_viable(struct vfio_group *group) -{ - return (iommu_group_for_each_dev(group->iommu_group, - group, vfio_dev_viable) == 0); -} - static int vfio_group_add_container_user(struct vfio_group *group) { if (!atomic_inc_not_zero(&group->container_users)) @@ -1328,7 +1322,7 @@ static int vfio_group_add_container_user(struct vfio_group *group) atomic_dec(&group->container_users); return -EPERM; } - if (!group->container->iommu_driver || !vfio_group_viable(group)) { + if (!group->container->iommu_driver) { atomic_dec(&group->container_users); return -EINVAL; } @@ -1346,7 +1340,7 @@ static int vfio_group_get_device_fd(struct vfio_group *group, char *buf) int ret = 0; if (0 == atomic_read(&group->container_users) || - !group->container->iommu_driver || !vfio_group_viable(group)) + !group->container->iommu_driver) return -EINVAL; if (group->type == VFIO_NO_IOMMU && !capable(CAP_SYS_RAWIO)) @@ -1438,11 +1432,11 @@ static long vfio_group_fops_unl_ioctl(struct file *filep, status.flags = 0; - if (vfio_group_viable(group)) - status.flags |= VFIO_GROUP_FLAGS_VIABLE; - if (group->container) - status.flags |= VFIO_GROUP_FLAGS_CONTAINER_SET; + status.flags |= VFIO_GROUP_FLAGS_CONTAINER_SET | + VFIO_GROUP_FLAGS_VIABLE; + else if (!iommu_group_dma_owner_claimed(group->iommu_group)) + status.flags |= VFIO_GROUP_FLAGS_VIABLE; if (copy_to_user((void __user *)arg, &status, minsz)) return -EFAULT; -- cgit v1.2.3 From 93219ea94388b5195d1eeadb0bda6ed1471fba56 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 18 Apr 2022 08:49:58 +0800 Subject: vfio: Delete the unbound_list commit 60720a0fc646 ("vfio: Add device tracking during unbind") added the unbound list to plug a problem with KVM where KVM_DEV_VFIO_GROUP_DEL relied on vfio_group_get_external_user() succeeding to return the vfio_group from a group file descriptor. The unbound list allowed vfio_group_get_external_user() to continue to succeed in edge cases. However commit 5d6dee80a1e9 ("vfio: New external user group/file match") deleted the call to vfio_group_get_external_user() during KVM_DEV_VFIO_GROUP_DEL. Instead vfio_external_group_match_file() is used to directly match the file descriptor to the group pointer. This in turn avoids the call down to vfio_dev_viable() during KVM_DEV_VFIO_GROUP_DEL and also avoids the trouble the first commit was trying to fix. There are no other users of vfio_dev_viable() that care about the time after vfio_unregister_group_dev() returns, so simply delete the unbound_list entirely. Reviewed-by: Chaitanya Kulkarni Signed-off-by: Jason Gunthorpe Signed-off-by: Lu Baolu Acked-by: Alex Williamson Link: https://lore.kernel.org/r/20220418005000.897664-10-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/vfio/vfio.c | 74 ++--------------------------------------------------- 1 file changed, 2 insertions(+), 72 deletions(-) (limited to 'drivers') diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c index 8a9347f732a5..b2f19d17d0c3 100644 --- a/drivers/vfio/vfio.c +++ b/drivers/vfio/vfio.c @@ -62,11 +62,6 @@ struct vfio_container { bool noiommu; }; -struct vfio_unbound_dev { - struct device *dev; - struct list_head unbound_next; -}; - struct vfio_group { struct device dev; struct cdev cdev; @@ -79,8 +74,6 @@ struct vfio_group { struct notifier_block nb; struct list_head vfio_next; struct list_head container_next; - struct list_head unbound_list; - struct mutex unbound_lock; atomic_t opened; wait_queue_head_t container_q; enum vfio_group_type type; @@ -340,16 +333,8 @@ vfio_group_get_from_iommu(struct iommu_group *iommu_group) static void vfio_group_release(struct device *dev) { struct vfio_group *group = container_of(dev, struct vfio_group, dev); - struct vfio_unbound_dev *unbound, *tmp; - - list_for_each_entry_safe(unbound, tmp, - &group->unbound_list, unbound_next) { - list_del(&unbound->unbound_next); - kfree(unbound); - } mutex_destroy(&group->device_lock); - mutex_destroy(&group->unbound_lock); iommu_group_put(group->iommu_group); ida_free(&vfio.group_ida, MINOR(group->dev.devt)); kfree(group); @@ -381,8 +366,6 @@ static struct vfio_group *vfio_group_alloc(struct iommu_group *iommu_group, refcount_set(&group->users, 1); INIT_LIST_HEAD(&group->device_list); mutex_init(&group->device_lock); - INIT_LIST_HEAD(&group->unbound_list); - mutex_init(&group->unbound_lock); init_waitqueue_head(&group->container_q); group->iommu_group = iommu_group; /* put in vfio_group_release() */ @@ -571,19 +554,8 @@ static int vfio_dev_viable(struct device *dev, void *data) struct vfio_group *group = data; struct vfio_device *device; struct device_driver *drv = READ_ONCE(dev->driver); - struct vfio_unbound_dev *unbound; - int ret = -EINVAL; - mutex_lock(&group->unbound_lock); - list_for_each_entry(unbound, &group->unbound_list, unbound_next) { - if (dev == unbound->dev) { - ret = 0; - break; - } - } - mutex_unlock(&group->unbound_lock); - - if (!ret || !drv || vfio_dev_driver_allowed(dev, drv)) + if (!drv || vfio_dev_driver_allowed(dev, drv)) return 0; device = vfio_group_get_device(group, dev); @@ -592,7 +564,7 @@ static int vfio_dev_viable(struct device *dev, void *data) return 0; } - return ret; + return -EINVAL; } /* @@ -634,7 +606,6 @@ static int vfio_iommu_group_notifier(struct notifier_block *nb, { struct vfio_group *group = container_of(nb, struct vfio_group, nb); struct device *dev = data; - struct vfio_unbound_dev *unbound; switch (action) { case IOMMU_GROUP_NOTIFY_ADD_DEVICE: @@ -663,28 +634,6 @@ static int vfio_iommu_group_notifier(struct notifier_block *nb, __func__, iommu_group_id(group->iommu_group), dev->driver->name); break; - case IOMMU_GROUP_NOTIFY_UNBOUND_DRIVER: - dev_dbg(dev, "%s: group %d unbound from driver\n", __func__, - iommu_group_id(group->iommu_group)); - /* - * XXX An unbound device in a live group is ok, but we'd - * really like to avoid the above BUG_ON by preventing other - * drivers from binding to it. Once that occurs, we have to - * stop the system to maintain isolation. At a minimum, we'd - * want a toggle to disable driver auto probe for this device. - */ - - mutex_lock(&group->unbound_lock); - list_for_each_entry(unbound, - &group->unbound_list, unbound_next) { - if (dev == unbound->dev) { - list_del(&unbound->unbound_next); - kfree(unbound); - break; - } - } - mutex_unlock(&group->unbound_lock); - break; } return NOTIFY_OK; } @@ -889,29 +838,10 @@ static struct vfio_device *vfio_device_get_from_name(struct vfio_group *group, void vfio_unregister_group_dev(struct vfio_device *device) { struct vfio_group *group = device->group; - struct vfio_unbound_dev *unbound; unsigned int i = 0; bool interrupted = false; long rc; - /* - * When the device is removed from the group, the group suddenly - * becomes non-viable; the device has a driver (until the unbind - * completes), but it's not present in the group. This is bad news - * for any external users that need to re-acquire a group reference - * in order to match and release their existing reference. To - * solve this, we track such devices on the unbound_list to bridge - * the gap until they're fully unbound. - */ - unbound = kzalloc(sizeof(*unbound), GFP_KERNEL); - if (unbound) { - unbound->dev = device->dev; - mutex_lock(&group->unbound_lock); - list_add(&unbound->unbound_next, &group->unbound_list); - mutex_unlock(&group->unbound_lock); - } - WARN_ON(!unbound); - vfio_device_put(device); rc = try_wait_for_completion(&device->comp); while (rc <= 0) { -- cgit v1.2.3 From 3b86f317c9c71c9fe17409db5743b6a27114f248 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 18 Apr 2022 08:49:59 +0800 Subject: vfio: Remove iommu group notifier The iommu core and driver core have been enhanced to avoid unsafe driver binding to a live group after iommu_group_set_dma_owner(PRIVATE_USER) has been called. There's no need to register iommu group notifier. This removes the iommu group notifer which contains BUG_ON() and WARN(). Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Acked-by: Alex Williamson Link: https://lore.kernel.org/r/20220418005000.897664-11-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/vfio/vfio.c | 147 ---------------------------------------------------- 1 file changed, 147 deletions(-) (limited to 'drivers') diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c index b2f19d17d0c3..0c766384cee0 100644 --- a/drivers/vfio/vfio.c +++ b/drivers/vfio/vfio.c @@ -71,7 +71,6 @@ struct vfio_group { struct vfio_container *container; struct list_head device_list; struct mutex device_lock; - struct notifier_block nb; struct list_head vfio_next; struct list_head container_next; atomic_t opened; @@ -274,8 +273,6 @@ void vfio_unregister_iommu_driver(const struct vfio_iommu_driver_ops *ops) } EXPORT_SYMBOL_GPL(vfio_unregister_iommu_driver); -static int vfio_iommu_group_notifier(struct notifier_block *nb, - unsigned long action, void *data); static void vfio_group_get(struct vfio_group *group); /* @@ -395,13 +392,6 @@ static struct vfio_group *vfio_create_group(struct iommu_group *iommu_group, goto err_put; } - group->nb.notifier_call = vfio_iommu_group_notifier; - err = iommu_group_register_notifier(iommu_group, &group->nb); - if (err) { - ret = ERR_PTR(err); - goto err_put; - } - mutex_lock(&vfio.group_lock); /* Did we race creating this group? */ @@ -422,7 +412,6 @@ static struct vfio_group *vfio_create_group(struct iommu_group *iommu_group, err_unlock: mutex_unlock(&vfio.group_lock); - iommu_group_unregister_notifier(group->iommu_group, &group->nb); err_put: put_device(&group->dev); return ret; @@ -447,7 +436,6 @@ static void vfio_group_put(struct vfio_group *group) cdev_device_del(&group->cdev, &group->dev); mutex_unlock(&vfio.group_lock); - iommu_group_unregister_notifier(group->iommu_group, &group->nb); put_device(&group->dev); } @@ -503,141 +491,6 @@ static struct vfio_device *vfio_group_get_device(struct vfio_group *group, return NULL; } -/* - * Some drivers, like pci-stub, are only used to prevent other drivers from - * claiming a device and are therefore perfectly legitimate for a user owned - * group. The pci-stub driver has no dependencies on DMA or the IOVA mapping - * of the device, but it does prevent the user from having direct access to - * the device, which is useful in some circumstances. - * - * We also assume that we can include PCI interconnect devices, ie. bridges. - * IOMMU grouping on PCI necessitates that if we lack isolation on a bridge - * then all of the downstream devices will be part of the same IOMMU group as - * the bridge. Thus, if placing the bridge into the user owned IOVA space - * breaks anything, it only does so for user owned devices downstream. Note - * that error notification via MSI can be affected for platforms that handle - * MSI within the same IOVA space as DMA. - */ -static const char * const vfio_driver_allowed[] = { "pci-stub" }; - -static bool vfio_dev_driver_allowed(struct device *dev, - struct device_driver *drv) -{ - if (dev_is_pci(dev)) { - struct pci_dev *pdev = to_pci_dev(dev); - - if (pdev->hdr_type != PCI_HEADER_TYPE_NORMAL) - return true; - } - - return match_string(vfio_driver_allowed, - ARRAY_SIZE(vfio_driver_allowed), - drv->name) >= 0; -} - -/* - * A vfio group is viable for use by userspace if all devices are in - * one of the following states: - * - driver-less - * - bound to a vfio driver - * - bound to an otherwise allowed driver - * - a PCI interconnect device - * - * We use two methods to determine whether a device is bound to a vfio - * driver. The first is to test whether the device exists in the vfio - * group. The second is to test if the device exists on the group - * unbound_list, indicating it's in the middle of transitioning from - * a vfio driver to driver-less. - */ -static int vfio_dev_viable(struct device *dev, void *data) -{ - struct vfio_group *group = data; - struct vfio_device *device; - struct device_driver *drv = READ_ONCE(dev->driver); - - if (!drv || vfio_dev_driver_allowed(dev, drv)) - return 0; - - device = vfio_group_get_device(group, dev); - if (device) { - vfio_device_put(device); - return 0; - } - - return -EINVAL; -} - -/* - * Async device support - */ -static int vfio_group_nb_add_dev(struct vfio_group *group, struct device *dev) -{ - struct vfio_device *device; - - /* Do we already know about it? We shouldn't */ - device = vfio_group_get_device(group, dev); - if (WARN_ON_ONCE(device)) { - vfio_device_put(device); - return 0; - } - - /* Nothing to do for idle groups */ - if (!atomic_read(&group->container_users)) - return 0; - - /* TODO Prevent device auto probing */ - dev_WARN(dev, "Device added to live group %d!\n", - iommu_group_id(group->iommu_group)); - - return 0; -} - -static int vfio_group_nb_verify(struct vfio_group *group, struct device *dev) -{ - /* We don't care what happens when the group isn't in use */ - if (!atomic_read(&group->container_users)) - return 0; - - return vfio_dev_viable(dev, group); -} - -static int vfio_iommu_group_notifier(struct notifier_block *nb, - unsigned long action, void *data) -{ - struct vfio_group *group = container_of(nb, struct vfio_group, nb); - struct device *dev = data; - - switch (action) { - case IOMMU_GROUP_NOTIFY_ADD_DEVICE: - vfio_group_nb_add_dev(group, dev); - break; - case IOMMU_GROUP_NOTIFY_DEL_DEVICE: - /* - * Nothing to do here. If the device is in use, then the - * vfio sub-driver should block the remove callback until - * it is unused. If the device is unused or attached to a - * stub driver, then it should be released and we don't - * care that it will be going away. - */ - break; - case IOMMU_GROUP_NOTIFY_BIND_DRIVER: - dev_dbg(dev, "%s: group %d binding to driver\n", __func__, - iommu_group_id(group->iommu_group)); - break; - case IOMMU_GROUP_NOTIFY_BOUND_DRIVER: - dev_dbg(dev, "%s: group %d bound to driver %s\n", __func__, - iommu_group_id(group->iommu_group), dev->driver->name); - BUG_ON(vfio_group_nb_verify(group, dev)); - break; - case IOMMU_GROUP_NOTIFY_UNBIND_DRIVER: - dev_dbg(dev, "%s: group %d unbinding from driver %s\n", - __func__, iommu_group_id(group->iommu_group), - dev->driver->name); - break; - } - return NOTIFY_OK; -} - /* * VFIO driver API */ -- cgit v1.2.3 From a5f1bd1afacd7b1e088f93f66af5453df0d8be9a Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 18 Apr 2022 08:50:00 +0800 Subject: iommu: Remove iommu group changes notifier The iommu group changes notifer is not referenced in the tree. Remove it to avoid dead code. Suggested-by: Christoph Hellwig Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20220418005000.897664-12-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 75 --------------------------------------------------- include/linux/iommu.h | 23 ---------------- 2 files changed, 98 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index eba8e8ccf19d..0c42ece25854 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -40,7 +39,6 @@ struct iommu_group { struct kobject *devices_kobj; struct list_head devices; struct mutex mutex; - struct blocking_notifier_head notifier; void *iommu_data; void (*iommu_data_release)(void *iommu_data); char *name; @@ -632,7 +630,6 @@ struct iommu_group *iommu_group_alloc(void) mutex_init(&group->mutex); INIT_LIST_HEAD(&group->devices); INIT_LIST_HEAD(&group->entry); - BLOCKING_INIT_NOTIFIER_HEAD(&group->notifier); ret = ida_simple_get(&iommu_group_ida, 0, 0, GFP_KERNEL); if (ret < 0) { @@ -905,10 +902,6 @@ rename: if (ret) goto err_put_group; - /* Notify any listeners about change to group. */ - blocking_notifier_call_chain(&group->notifier, - IOMMU_GROUP_NOTIFY_ADD_DEVICE, dev); - trace_add_device_to_group(group->id, dev); dev_info(dev, "Adding to iommu group %d\n", group->id); @@ -950,10 +943,6 @@ void iommu_group_remove_device(struct device *dev) dev_info(dev, "Removing from iommu group %d\n", group->id); - /* Pre-notify listeners that a device is being removed. */ - blocking_notifier_call_chain(&group->notifier, - IOMMU_GROUP_NOTIFY_DEL_DEVICE, dev); - mutex_lock(&group->mutex); list_for_each_entry(tmp_device, &group->devices, list) { if (tmp_device->dev == dev) { @@ -1075,36 +1064,6 @@ void iommu_group_put(struct iommu_group *group) } EXPORT_SYMBOL_GPL(iommu_group_put); -/** - * iommu_group_register_notifier - Register a notifier for group changes - * @group: the group to watch - * @nb: notifier block to signal - * - * This function allows iommu group users to track changes in a group. - * See include/linux/iommu.h for actions sent via this notifier. Caller - * should hold a reference to the group throughout notifier registration. - */ -int iommu_group_register_notifier(struct iommu_group *group, - struct notifier_block *nb) -{ - return blocking_notifier_chain_register(&group->notifier, nb); -} -EXPORT_SYMBOL_GPL(iommu_group_register_notifier); - -/** - * iommu_group_unregister_notifier - Unregister a notifier - * @group: the group to watch - * @nb: notifier block to signal - * - * Unregister a previously registered group notifier block. - */ -int iommu_group_unregister_notifier(struct iommu_group *group, - struct notifier_block *nb) -{ - return blocking_notifier_chain_unregister(&group->notifier, nb); -} -EXPORT_SYMBOL_GPL(iommu_group_unregister_notifier); - /** * iommu_register_device_fault_handler() - Register a device fault handler * @dev: the device @@ -1650,14 +1609,8 @@ static int remove_iommu_group(struct device *dev, void *data) static int iommu_bus_notifier(struct notifier_block *nb, unsigned long action, void *data) { - unsigned long group_action = 0; struct device *dev = data; - struct iommu_group *group; - /* - * ADD/DEL call into iommu driver ops if provided, which may - * result in ADD/DEL notifiers to group->notifier - */ if (action == BUS_NOTIFY_ADD_DEVICE) { int ret; @@ -1668,34 +1621,6 @@ static int iommu_bus_notifier(struct notifier_block *nb, return NOTIFY_OK; } - /* - * Remaining BUS_NOTIFYs get filtered and republished to the - * group, if anyone is listening - */ - group = iommu_group_get(dev); - if (!group) - return 0; - - switch (action) { - case BUS_NOTIFY_BIND_DRIVER: - group_action = IOMMU_GROUP_NOTIFY_BIND_DRIVER; - break; - case BUS_NOTIFY_BOUND_DRIVER: - group_action = IOMMU_GROUP_NOTIFY_BOUND_DRIVER; - break; - case BUS_NOTIFY_UNBIND_DRIVER: - group_action = IOMMU_GROUP_NOTIFY_UNBIND_DRIVER; - break; - case BUS_NOTIFY_UNBOUND_DRIVER: - group_action = IOMMU_GROUP_NOTIFY_UNBOUND_DRIVER; - break; - } - - if (group_action) - blocking_notifier_call_chain(&group->notifier, - group_action, dev); - - iommu_group_put(group); return 0; } diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 77972ef978b5..6ef2df258673 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -407,13 +407,6 @@ static inline const struct iommu_ops *dev_iommu_ops(struct device *dev) return dev->iommu->iommu_dev->ops; } -#define IOMMU_GROUP_NOTIFY_ADD_DEVICE 1 /* Device added */ -#define IOMMU_GROUP_NOTIFY_DEL_DEVICE 2 /* Pre Device removed */ -#define IOMMU_GROUP_NOTIFY_BIND_DRIVER 3 /* Pre Driver bind */ -#define IOMMU_GROUP_NOTIFY_BOUND_DRIVER 4 /* Post Driver bind */ -#define IOMMU_GROUP_NOTIFY_UNBIND_DRIVER 5 /* Pre Driver unbind */ -#define IOMMU_GROUP_NOTIFY_UNBOUND_DRIVER 6 /* Post Driver unbind */ - extern int bus_set_iommu(struct bus_type *bus, const struct iommu_ops *ops); extern int bus_iommu_probe(struct bus_type *bus); extern bool iommu_present(struct bus_type *bus); @@ -478,10 +471,6 @@ extern int iommu_group_for_each_dev(struct iommu_group *group, void *data, extern struct iommu_group *iommu_group_get(struct device *dev); extern struct iommu_group *iommu_group_ref_get(struct iommu_group *group); extern void iommu_group_put(struct iommu_group *group); -extern int iommu_group_register_notifier(struct iommu_group *group, - struct notifier_block *nb); -extern int iommu_group_unregister_notifier(struct iommu_group *group, - struct notifier_block *nb); extern int iommu_register_device_fault_handler(struct device *dev, iommu_dev_fault_handler_t handler, void *data); @@ -878,18 +867,6 @@ static inline void iommu_group_put(struct iommu_group *group) { } -static inline int iommu_group_register_notifier(struct iommu_group *group, - struct notifier_block *nb) -{ - return -ENODEV; -} - -static inline int iommu_group_unregister_notifier(struct iommu_group *group, - struct notifier_block *nb) -{ - return 0; -} - static inline int iommu_register_device_fault_handler(struct device *dev, iommu_dev_fault_handler_t handler, -- cgit v1.2.3 From 6043257b1de069fbb5a2a52d7211c0275bc8c0e0 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 11 Apr 2022 12:16:05 -0300 Subject: iommu: Introduce the domain op enforce_cache_coherency() This new mechanism will replace using IOMMU_CAP_CACHE_COHERENCY and IOMMU_CACHE to control the no-snoop blocking behavior of the IOMMU. Currently only Intel and AMD IOMMUs are known to support this feature. They both implement it as an IOPTE bit, that when set, will cause PCIe TLPs to that IOVA with the no-snoop bit set to be treated as though the no-snoop bit was clear. The new API is triggered by calling enforce_cache_coherency() before mapping any IOVA to the domain which globally switches on no-snoop blocking. This allows other implementations that might block no-snoop globally and outside the IOPTE - AMD also documents such a HW capability. Leave AMD out of sync with Intel and have it block no-snoop even for in-kernel users. This can be trivially resolved in a follow up patch. Only VFIO needs to call this API because it does not have detailed control over the device to avoid requesting no-snoop behavior at the device level. Other places using domains with real kernel drivers should simply avoid asking their devices to set the no-snoop bit. Reviewed-by: Lu Baolu Reviewed-by: Kevin Tian Acked-by: Robin Murphy Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/1-v3-2cf356649677+a32-intel_no_snoop_jgg@nvidia.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 7 +++++++ drivers/iommu/intel/iommu.c | 14 +++++++++++++- include/linux/intel-iommu.h | 1 + include/linux/iommu.h | 4 ++++ 4 files changed, 25 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 038e104b922c..840831d5d2ad 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2266,6 +2266,12 @@ static int amd_iommu_def_domain_type(struct device *dev) return 0; } +static bool amd_iommu_enforce_cache_coherency(struct iommu_domain *domain) +{ + /* IOMMU_PTE_FC is always set */ + return true; +} + const struct iommu_ops amd_iommu_ops = { .capable = amd_iommu_capable, .domain_alloc = amd_iommu_domain_alloc, @@ -2288,6 +2294,7 @@ const struct iommu_ops amd_iommu_ops = { .flush_iotlb_all = amd_iommu_flush_iotlb_all, .iotlb_sync = amd_iommu_iotlb_sync, .free = amd_iommu_domain_free, + .enforce_cache_coherency = amd_iommu_enforce_cache_coherency, } }; diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 0edf6084dc14..161199f62270 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -4422,7 +4422,8 @@ static int intel_iommu_map(struct iommu_domain *domain, prot |= DMA_PTE_READ; if (iommu_prot & IOMMU_WRITE) prot |= DMA_PTE_WRITE; - if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping) + if (((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping) || + dmar_domain->force_snooping) prot |= DMA_PTE_SNP; max_addr = iova + size; @@ -4545,6 +4546,16 @@ static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain, return phys; } +static bool intel_iommu_enforce_cache_coherency(struct iommu_domain *domain) +{ + struct dmar_domain *dmar_domain = to_dmar_domain(domain); + + if (!dmar_domain->iommu_snooping) + return false; + dmar_domain->force_snooping = true; + return true; +} + static bool intel_iommu_capable(enum iommu_cap cap) { if (cap == IOMMU_CAP_CACHE_COHERENCY) @@ -4900,6 +4911,7 @@ const struct iommu_ops intel_iommu_ops = { .iotlb_sync = intel_iommu_tlb_sync, .iova_to_phys = intel_iommu_iova_to_phys, .free = intel_iommu_domain_free, + .enforce_cache_coherency = intel_iommu_enforce_cache_coherency, } }; diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index 2f9891cb3d00..4c2baf2446c2 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -540,6 +540,7 @@ struct dmar_domain { u8 has_iotlb_device: 1; u8 iommu_coherency: 1; /* indicate coherency of iommu access */ u8 iommu_snooping: 1; /* indicate snooping control feature */ + u8 force_snooping : 1; /* Create IOPTEs with snoop control */ struct list_head devices; /* all devices' list */ struct iova_domain iovad; /* iova's that belong to this domain */ diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 4123693ae319..c7ad6b10e261 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -274,6 +274,9 @@ struct iommu_ops { * @iotlb_sync: Flush all queued ranges from the hardware TLBs and empty flush * queue * @iova_to_phys: translate iova to physical address + * @enforce_cache_coherency: Prevent any kind of DMA from bypassing IOMMU_CACHE, + * including no-snoop TLPs on PCIe or other platform + * specific mechanisms. * @enable_nesting: Enable nesting * @set_pgtable_quirks: Set io page table quirks (IO_PGTABLE_QUIRK_*) * @free: Release the domain after use. @@ -302,6 +305,7 @@ struct iommu_domain_ops { phys_addr_t (*iova_to_phys)(struct iommu_domain *domain, dma_addr_t iova); + bool (*enforce_cache_coherency)(struct iommu_domain *domain); int (*enable_nesting)(struct iommu_domain *domain); int (*set_pgtable_quirks)(struct iommu_domain *domain, unsigned long quirks); -- cgit v1.2.3 From 71cfafda9c9bd9812cdb62ddb94daf65a1af12c1 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 11 Apr 2022 12:16:06 -0300 Subject: vfio: Move the Intel no-snoop control off of IOMMU_CACHE IOMMU_CACHE means "normal DMA to this iommu_domain's IOVA should be cache coherent" and is used by the DMA API. The definition allows for special non-coherent DMA to exist - ie processing of the no-snoop flag in PCIe TLPs - so long as this behavior is opt-in by the device driver. The flag is mainly used by the DMA API to synchronize the IOMMU setting with the expected cache behavior of the DMA master. eg based on dev_is_dma_coherent() in some case. For Intel IOMMU IOMMU_CACHE was redefined to mean 'force all DMA to be cache coherent' which has the practical effect of causing the IOMMU to ignore the no-snoop bit in a PCIe TLP. x86 platforms are always IOMMU_CACHE, so Intel should ignore this flag. Instead use the new domain op enforce_cache_coherency() which causes every IOPTE created in the domain to have the no-snoop blocking behavior. Reconfigure VFIO to always use IOMMU_CACHE and call enforce_cache_coherency() to operate the special Intel behavior. Remove the IOMMU_CACHE test from Intel IOMMU. Ultimately VFIO plumbs the result of enforce_cache_coherency() back into the x86 platform code through kvm_arch_register_noncoherent_dma() which controls if the WBINVD instruction is available in the guest. No other archs implement kvm_arch_register_noncoherent_dma() nor are there any other known consumers of VFIO_DMA_CC_IOMMU that might be affected by the user visible result change on non-x86 archs. Reviewed-by: Kevin Tian Reviewed-by: Lu Baolu Acked-by: Alex Williamson Acked-by: Robin Murphy Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/2-v3-2cf356649677+a32-intel_no_snoop_jgg@nvidia.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 7 ++----- drivers/vfio/vfio_iommu_type1.c | 30 +++++++++++++++++++----------- include/linux/intel-iommu.h | 1 - 3 files changed, 21 insertions(+), 17 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 161199f62270..38441cb06c8c 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -641,7 +641,6 @@ static unsigned long domain_super_pgsize_bitmap(struct dmar_domain *domain) static void domain_update_iommu_cap(struct dmar_domain *domain) { domain_update_iommu_coherency(domain); - domain->iommu_snooping = domain_update_iommu_snooping(NULL); domain->iommu_superpage = domain_update_iommu_superpage(domain, NULL); /* @@ -4283,7 +4282,6 @@ static int md_domain_init(struct dmar_domain *domain, int guest_width) domain->agaw = width_to_agaw(adjust_width); domain->iommu_coherency = false; - domain->iommu_snooping = false; domain->iommu_superpage = 0; domain->max_addr = 0; @@ -4422,8 +4420,7 @@ static int intel_iommu_map(struct iommu_domain *domain, prot |= DMA_PTE_READ; if (iommu_prot & IOMMU_WRITE) prot |= DMA_PTE_WRITE; - if (((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping) || - dmar_domain->force_snooping) + if (dmar_domain->force_snooping) prot |= DMA_PTE_SNP; max_addr = iova + size; @@ -4550,7 +4547,7 @@ static bool intel_iommu_enforce_cache_coherency(struct iommu_domain *domain) { struct dmar_domain *dmar_domain = to_dmar_domain(domain); - if (!dmar_domain->iommu_snooping) + if (!domain_update_iommu_snooping(NULL)) return false; dmar_domain->force_snooping = true; return true; diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 9394aa9444c1..c13b9290e357 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -84,8 +84,8 @@ struct vfio_domain { struct iommu_domain *domain; struct list_head next; struct list_head group_list; - int prot; /* IOMMU_CACHE */ - bool fgsp; /* Fine-grained super pages */ + bool fgsp : 1; /* Fine-grained super pages */ + bool enforce_cache_coherency : 1; }; struct vfio_dma { @@ -1461,7 +1461,7 @@ static int vfio_iommu_map(struct vfio_iommu *iommu, dma_addr_t iova, list_for_each_entry(d, &iommu->domain_list, next) { ret = iommu_map(d->domain, iova, (phys_addr_t)pfn << PAGE_SHIFT, - npage << PAGE_SHIFT, prot | d->prot); + npage << PAGE_SHIFT, prot | IOMMU_CACHE); if (ret) goto unwind; @@ -1771,7 +1771,7 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, } ret = iommu_map(domain->domain, iova, phys, - size, dma->prot | domain->prot); + size, dma->prot | IOMMU_CACHE); if (ret) { if (!dma->iommu_mapped) { vfio_unpin_pages_remote(dma, iova, @@ -1859,7 +1859,7 @@ static void vfio_test_domain_fgsp(struct vfio_domain *domain) return; ret = iommu_map(domain->domain, 0, page_to_phys(pages), PAGE_SIZE * 2, - IOMMU_READ | IOMMU_WRITE | domain->prot); + IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE); if (!ret) { size_t unmapped = iommu_unmap(domain->domain, 0, PAGE_SIZE); @@ -2267,8 +2267,15 @@ static int vfio_iommu_type1_attach_group(void *iommu_data, goto out_detach; } - if (iommu_capable(bus, IOMMU_CAP_CACHE_COHERENCY)) - domain->prot |= IOMMU_CACHE; + /* + * If the IOMMU can block non-coherent operations (ie PCIe TLPs with + * no-snoop set) then VFIO always turns this feature on because on Intel + * platforms it optimizes KVM to disable wbinvd emulation. + */ + if (domain->domain->ops->enforce_cache_coherency) + domain->enforce_cache_coherency = + domain->domain->ops->enforce_cache_coherency( + domain->domain); /* * Try to match an existing compatible domain. We don't want to @@ -2279,7 +2286,8 @@ static int vfio_iommu_type1_attach_group(void *iommu_data, */ list_for_each_entry(d, &iommu->domain_list, next) { if (d->domain->ops == domain->domain->ops && - d->prot == domain->prot) { + d->enforce_cache_coherency == + domain->enforce_cache_coherency) { iommu_detach_group(domain->domain, group->iommu_group); if (!iommu_attach_group(d->domain, group->iommu_group)) { @@ -2611,14 +2619,14 @@ static void vfio_iommu_type1_release(void *iommu_data) kfree(iommu); } -static int vfio_domains_have_iommu_cache(struct vfio_iommu *iommu) +static int vfio_domains_have_enforce_cache_coherency(struct vfio_iommu *iommu) { struct vfio_domain *domain; int ret = 1; mutex_lock(&iommu->lock); list_for_each_entry(domain, &iommu->domain_list, next) { - if (!(domain->prot & IOMMU_CACHE)) { + if (!(domain->enforce_cache_coherency)) { ret = 0; break; } @@ -2641,7 +2649,7 @@ static int vfio_iommu_type1_check_extension(struct vfio_iommu *iommu, case VFIO_DMA_CC_IOMMU: if (!iommu) return 0; - return vfio_domains_have_iommu_cache(iommu); + return vfio_domains_have_enforce_cache_coherency(iommu); default: return 0; } diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index 4c2baf2446c2..72e5d7900e71 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -539,7 +539,6 @@ struct dmar_domain { u8 has_iotlb_device: 1; u8 iommu_coherency: 1; /* indicate coherency of iommu access */ - u8 iommu_snooping: 1; /* indicate snooping control feature */ u8 force_snooping : 1; /* Create IOPTEs with snoop control */ struct list_head devices; /* all devices' list */ -- cgit v1.2.3 From f78dc1dad829e505d83e33dc0879887f074c52e1 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 11 Apr 2022 12:16:07 -0300 Subject: iommu: Redefine IOMMU_CAP_CACHE_COHERENCY as the cap flag for IOMMU_CACHE While the comment was correct that this flag was intended to convey the block no-snoop support in the IOMMU, it has become widely implemented and used to mean the IOMMU supports IOMMU_CACHE as a map flag. Only the Intel driver was different. Now that the Intel driver is using enforce_cache_coherency() update the comment to make it clear that IOMMU_CAP_CACHE_COHERENCY is only about IOMMU_CACHE. Fix the Intel driver to return true since IOMMU_CACHE always works. The two places that test this flag, usnic and vdpa, are both assigning userspace pages to a driver controlled iommu_domain and require IOMMU_CACHE behavior as they offer no way for userspace to synchronize caches. Reviewed-by: Kevin Tian Reviewed-by: Lu Baolu Acked-by: Robin Murphy Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/3-v3-2cf356649677+a32-intel_no_snoop_jgg@nvidia.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 2 +- include/linux/iommu.h | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 38441cb06c8c..efcecfa5952a 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -4556,7 +4556,7 @@ static bool intel_iommu_enforce_cache_coherency(struct iommu_domain *domain) static bool intel_iommu_capable(enum iommu_cap cap) { if (cap == IOMMU_CAP_CACHE_COHERENCY) - return domain_update_iommu_snooping(NULL); + return true; if (cap == IOMMU_CAP_INTR_REMAP) return irq_remapping_enabled == 1; if (cap == IOMMU_CAP_PRE_BOOT_PROTECTION) diff --git a/include/linux/iommu.h b/include/linux/iommu.h index c7ad6b10e261..575ab27ede5b 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -103,8 +103,7 @@ static inline bool iommu_is_dma_domain(struct iommu_domain *domain) } enum iommu_cap { - IOMMU_CAP_CACHE_COHERENCY, /* IOMMU can enforce cache coherent DMA - transactions */ + IOMMU_CAP_CACHE_COHERENCY, /* IOMMU_CACHE is supported */ IOMMU_CAP_INTR_REMAP, /* IOMMU supports interrupt isolation */ IOMMU_CAP_NOEXEC, /* IOMMU_NOEXEC flag */ IOMMU_CAP_PRE_BOOT_PROTECTION, /* Firmware says it used the IOMMU for -- cgit v1.2.3 From e8ae0e140c05319fa47bb3bf615c3a585f404b9d Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 11 Apr 2022 12:16:08 -0300 Subject: vfio: Require that devices support DMA cache coherence IOMMU_CACHE means that normal DMAs do not require any additional coherency mechanism and is the basic uAPI that VFIO exposes to userspace. For instance VFIO applications like DPDK will not work if additional coherency operations are required. Therefore check IOMMU_CAP_CACHE_COHERENCY like vdpa & usnic do before allowing an IOMMU backed VFIO device to be created. Reviewed-by: Kevin Tian Acked-by: Alex Williamson Acked-by: Robin Murphy Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/4-v3-2cf356649677+a32-intel_no_snoop_jgg@nvidia.com Signed-off-by: Joerg Roedel --- drivers/vfio/vfio.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'drivers') diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c index a4555014bd1e..9edad767cfda 100644 --- a/drivers/vfio/vfio.c +++ b/drivers/vfio/vfio.c @@ -815,6 +815,13 @@ static int __vfio_register_dev(struct vfio_device *device, int vfio_register_group_dev(struct vfio_device *device) { + /* + * VFIO always sets IOMMU_CACHE because we offer no way for userspace to + * restore cache coherency. + */ + if (!iommu_capable(device->dev->bus, IOMMU_CAP_CACHE_COHERENCY)) + return -EINVAL; + return __vfio_register_dev(device, vfio_group_find_or_alloc(device->dev)); } -- cgit v1.2.3 From 5edde870d3283edeaa27ab62ac4fac5ee8cae35a Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Sun, 13 Mar 2022 21:43:21 -0500 Subject: iommu/amd: Do not call sleep while holding spinlock Smatch static checker warns: drivers/iommu/amd/iommu_v2.c:133 free_device_state() warn: sleeping in atomic context Fixes by storing the list of struct device_state in a temporary list, and then free the memory after releasing the spinlock. Reported-by: Dan Carpenter Fixes: 9f968fc70d85 ("iommu/amd: Improve amd_iommu_v2_exit()") Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20220314024321.37411-1-suravee.suthikulpanit@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu_v2.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/iommu/amd/iommu_v2.c b/drivers/iommu/amd/iommu_v2.c index e56b137ceabd..afb3efd565b7 100644 --- a/drivers/iommu/amd/iommu_v2.c +++ b/drivers/iommu/amd/iommu_v2.c @@ -956,6 +956,7 @@ static void __exit amd_iommu_v2_exit(void) { struct device_state *dev_state, *next; unsigned long flags; + LIST_HEAD(freelist); if (!amd_iommu_v2_supported()) return; @@ -975,11 +976,20 @@ static void __exit amd_iommu_v2_exit(void) put_device_state(dev_state); list_del(&dev_state->list); - free_device_state(dev_state); + list_add_tail(&dev_state->list, &freelist); } spin_unlock_irqrestore(&state_lock, flags); + /* + * Since free_device_state waits on the count to be zero, + * we need to free dev_state outside the spinlock. + */ + list_for_each_entry_safe(dev_state, next, &freelist, list) { + list_del(&dev_state->list); + free_device_state(dev_state); + } + destroy_workqueue(iommu_wq); } -- cgit v1.2.3 From 9ed1d7f510336e283c0bb2d20fd771e9d8fa085b Mon Sep 17 00:00:00 2001 From: Vasant Hegde via iommu Date: Mon, 14 Mar 2022 12:32:26 +0530 Subject: iommu/amd: Remove redundant check smatch static checker warning: drivers/iommu/amd/init.c:1989 amd_iommu_init_pci() warn: duplicate check 'ret' (previous on line 1978) Reported-by: Dan Carpenter Fixes: 06687a03805e ("iommu/amd: Improve error handling for amd_iommu_init_pci") Signed-off-by: Vasant Hegde Link: https://lore.kernel.org/r/20220314070226.40641-1-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index b4a798c7b347..648d6b94ba8c 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -1986,8 +1986,7 @@ static int __init amd_iommu_init_pci(void) for_each_iommu(iommu) iommu_flush_all_caches(iommu); - if (!ret) - print_iommu_info(); + print_iommu_info(); out: return ret; -- cgit v1.2.3 From 645b87c190c959e9bb4f216b8c4add4ee880451a Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:13:55 +0800 Subject: iommu/mediatek: Fix 2 HW sharing pgtable issue In the commit 4f956c97d26b ("iommu/mediatek: Move domain_finalise into attach_device"), I overlooked the sharing pgtable case. After that commit, the "data" in the mtk_iommu_domain_finalise always is the data of the current IOMMU HW. Fix this for the sharing pgtable case. Only affect mt2712 which is the only SoC that share pgtable currently. Fixes: 4f956c97d26b ("iommu/mediatek: Move domain_finalise into attach_device") Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-5-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 6fd75a60abd6..95c82b8bcc35 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -446,7 +446,7 @@ static void mtk_iommu_domain_free(struct iommu_domain *domain) static int mtk_iommu_attach_device(struct iommu_domain *domain, struct device *dev) { - struct mtk_iommu_data *data = dev_iommu_priv_get(dev); + struct mtk_iommu_data *data = dev_iommu_priv_get(dev), *frstdata; struct mtk_iommu_domain *dom = to_mtk_domain(domain); struct device *m4udev = data->dev; int ret, domid; @@ -456,7 +456,10 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, return domid; if (!dom->data) { - if (mtk_iommu_domain_finalise(dom, data, domid)) + /* Data is in the frstdata in sharing pgtable case. */ + frstdata = mtk_iommu_get_m4u_data(); + + if (mtk_iommu_domain_finalise(dom, frstdata, domid)) return -ENODEV; dom->data = data; } -- cgit v1.2.3 From ee55f75e4bcade81d253163641b63bef3e76cac4 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:13:56 +0800 Subject: iommu/mediatek: Add list_del in mtk_iommu_remove Lack the list_del in the mtk_iommu_remove, and remove bus_set_iommu(*, NULL) since there may be several iommu HWs. we can not bus_set_iommu null when one iommu driver unbind. This could be a fix for mt2712 which support 2 M4U HW and list them. Fixes: 7c3a2ec02806 ("iommu/mediatek: Merge 2 M4U HWs into one iommu domain") Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-6-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 95c82b8bcc35..e4b4ebbcb73f 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -955,8 +955,7 @@ static int mtk_iommu_remove(struct platform_device *pdev) iommu_device_sysfs_remove(&data->iommu); iommu_device_unregister(&data->iommu); - if (iommu_present(&platform_bus_type)) - bus_set_iommu(&platform_bus_type, NULL); + list_del(&data->list); clk_disable_unprepare(data->bclk); device_link_remove(data->smicomm_dev, &pdev->dev); -- cgit v1.2.3 From 98df772bdd1c4ce717a26289efea15cbbe4b64ed Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:13:57 +0800 Subject: iommu/mediatek: Remove clk_disable in mtk_iommu_remove After the commit b34ea31fe013 ("iommu/mediatek: Always enable the clk on resume"), the iommu clock is controlled by the runtime callback. thus remove the clk control in the mtk_iommu_remove. Otherwise, it will warning like: echo 14018000.iommu > /sys/bus/platform/drivers/mtk-iommu/unbind [ 51.413044] ------------[ cut here ]------------ [ 51.413648] vpp0_smi_iommu already disabled [ 51.414233] WARNING: CPU: 2 PID: 157 at */v5.15-rc1/kernel/mediatek/ drivers/clk/clk.c:952 clk_core_disable+0xb0/0xb8 [ 51.417174] Hardware name: MT8195V/C(ENG) (DT) [ 51.418635] pc : clk_core_disable+0xb0/0xb8 [ 51.419177] lr : clk_core_disable+0xb0/0xb8 ... [ 51.429375] Call trace: [ 51.429694] clk_core_disable+0xb0/0xb8 [ 51.430193] clk_core_disable_lock+0x24/0x40 [ 51.430745] clk_disable+0x20/0x30 [ 51.431189] mtk_iommu_remove+0x58/0x118 [ 51.431705] platform_remove+0x28/0x60 [ 51.432197] device_release_driver_internal+0x110/0x1f0 [ 51.432873] device_driver_detach+0x18/0x28 [ 51.433418] unbind_store+0xd4/0x108 [ 51.433886] drv_attr_store+0x24/0x38 [ 51.434363] sysfs_kf_write+0x40/0x58 [ 51.434843] kernfs_fop_write_iter+0x164/0x1e0 Fixes: b34ea31fe013 ("iommu/mediatek: Always enable the clk on resume") Reported-by: Hsin-Yi Wang Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-7-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 1 - 1 file changed, 1 deletion(-) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index e4b4ebbcb73f..81b8db450eac 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -957,7 +957,6 @@ static int mtk_iommu_remove(struct platform_device *pdev) list_del(&data->list); - clk_disable_unprepare(data->bclk); device_link_remove(data->smicomm_dev, &pdev->dev); pm_runtime_disable(&pdev->dev); devm_free_irq(&pdev->dev, data->irq, data); -- cgit v1.2.3 From 0e5a3f2e630b28e88e018655548212ef8eb4dfcb Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:13:58 +0800 Subject: iommu/mediatek: Add mutex for m4u_group and m4u_dom in data Add a mutex to protect the data in the structure mtk_iommu_data, like ->"m4u_group" ->"m4u_dom". For the internal data, we should protect it in ourselves driver. Add a mutex for this. This could be a fix for the multi-groups support. Fixes: c3045f39244e ("iommu/mediatek: Support for multi domains") Signed-off-by: Yunfei Wang Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-8-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 13 +++++++++++-- drivers/iommu/mtk_iommu.h | 2 ++ 2 files changed, 13 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 81b8db450eac..3413cc98e57e 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -464,15 +464,16 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, dom->data = data; } + mutex_lock(&data->mutex); if (!data->m4u_dom) { /* Initialize the M4U HW */ ret = pm_runtime_resume_and_get(m4udev); if (ret < 0) - return ret; + goto err_unlock; ret = mtk_iommu_hw_init(data); if (ret) { pm_runtime_put(m4udev); - return ret; + goto err_unlock; } data->m4u_dom = dom; writel(dom->cfg.arm_v7s_cfg.ttbr & MMU_PT_ADDR_MASK, @@ -480,9 +481,14 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, pm_runtime_put(m4udev); } + mutex_unlock(&data->mutex); mtk_iommu_config(data, dev, true, domid); return 0; + +err_unlock: + mutex_unlock(&data->mutex); + return ret; } static void mtk_iommu_detach_device(struct iommu_domain *domain, @@ -622,6 +628,7 @@ static struct iommu_group *mtk_iommu_device_group(struct device *dev) if (domid < 0) return ERR_PTR(domid); + mutex_lock(&data->mutex); group = data->m4u_group[domid]; if (!group) { group = iommu_group_alloc(); @@ -630,6 +637,7 @@ static struct iommu_group *mtk_iommu_device_group(struct device *dev) } else { iommu_group_ref_get(group); } + mutex_unlock(&data->mutex); return group; } @@ -910,6 +918,7 @@ static int mtk_iommu_probe(struct platform_device *pdev) } platform_set_drvdata(pdev, data); + mutex_init(&data->mutex); ret = iommu_device_sysfs_add(&data->iommu, dev, NULL, "mtk-iommu.%pa", &ioaddr); diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h index b742432220c5..5e8da947affc 100644 --- a/drivers/iommu/mtk_iommu.h +++ b/drivers/iommu/mtk_iommu.h @@ -80,6 +80,8 @@ struct mtk_iommu_data { struct dma_iommu_mapping *mapping; /* For mtk_iommu_v1.c */ + struct mutex mutex; /* Protect m4u_group/m4u_dom above */ + struct list_head list; struct mtk_smi_larb_iommu larb_imu[MTK_LARB_NR_MAX]; }; -- cgit v1.2.3 From ddf67a87bebf2a8750d86f67cb298313ee93db31 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:13:59 +0800 Subject: iommu/mediatek: Add mutex for data in the mtk_iommu_domain Same with the previous patch, add a mutex for the "data" in the mtk_iommu_domain. Just improve the safety for multi devices enter attach_device at the same time. We don't get the real issue for this. Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-9-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 3413cc98e57e..ecdce5d3e8cf 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -128,6 +128,8 @@ struct mtk_iommu_domain { struct mtk_iommu_data *data; struct iommu_domain domain; + + struct mutex mutex; /* Protect "data" in this structure */ }; static const struct iommu_ops mtk_iommu_ops; @@ -434,6 +436,7 @@ static struct iommu_domain *mtk_iommu_domain_alloc(unsigned type) dom = kzalloc(sizeof(*dom), GFP_KERNEL); if (!dom) return NULL; + mutex_init(&dom->mutex); return &dom->domain; } @@ -455,14 +458,19 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, if (domid < 0) return domid; + mutex_lock(&dom->mutex); if (!dom->data) { /* Data is in the frstdata in sharing pgtable case. */ frstdata = mtk_iommu_get_m4u_data(); - if (mtk_iommu_domain_finalise(dom, frstdata, domid)) + ret = mtk_iommu_domain_finalise(dom, frstdata, domid); + if (ret) { + mutex_unlock(&dom->mutex); return -ENODEV; + } dom->data = data; } + mutex_unlock(&dom->mutex); mutex_lock(&data->mutex); if (!data->m4u_dom) { /* Initialize the M4U HW */ -- cgit v1.2.3 From 9e3a2a64365318a743e3c0b028952d2cdbaf2b0c Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:14:00 +0800 Subject: iommu/mediatek: Adapt sharing and non-sharing pgtable case In previous mt2712, Both IOMMUs are MM IOMMU, and they will share pgtable. However in the latest SoC, another is infra IOMMU, there is no reason to share pgtable between MM with INFRA IOMMU. This patch manage to implement the two case(sharing and non-sharing pgtable). Currently we use for_each_m4u to loop the 2 HWs. Add the list_head into this macro. In the sharing pgtable case, the list_head is the global "m4ulist". In the non-sharing pgtable case, the list_head is hw_list_head which is a variable in the "data". then for_each_m4u will only loop itself. Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-10-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 43 +++++++++++++++++++++++-------------------- drivers/iommu/mtk_iommu.h | 7 +++++++ 2 files changed, 30 insertions(+), 20 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index ecdce5d3e8cf..1b8e4405dd0f 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -118,6 +118,7 @@ #define WR_THROT_EN BIT(6) #define HAS_LEGACY_IVRP_PADDR BIT(7) #define IOVA_34_EN BIT(8) +#define SHARE_PGTABLE BIT(9) /* 2 HW share pgtable */ #define MTK_IOMMU_HAS_FLAG(pdata, _x) \ ((((pdata)->flags) & (_x)) == (_x)) @@ -167,7 +168,7 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data); static LIST_HEAD(m4ulist); /* List all the M4U HWs */ -#define for_each_m4u(data) list_for_each_entry(data, &m4ulist, list) +#define for_each_m4u(data, head) list_for_each_entry(data, head, list) struct mtk_iommu_iova_region { dma_addr_t iova_base; @@ -188,21 +189,10 @@ static const struct mtk_iommu_iova_region mt8192_multi_dom[] = { #endif }; -/* - * There may be 1 or 2 M4U HWs, But we always expect they are in the same domain - * for the performance. - * - * Here always return the mtk_iommu_data of the first probed M4U where the - * iommu domain information is recorded. - */ -static struct mtk_iommu_data *mtk_iommu_get_m4u_data(void) +/* If 2 M4U share a domain(use the same hwlist), Put the corresponding info in first data.*/ +static struct mtk_iommu_data *mtk_iommu_get_frst_data(struct list_head *hwlist) { - struct mtk_iommu_data *data; - - for_each_m4u(data) - return data; - - return NULL; + return list_first_entry(hwlist, struct mtk_iommu_data, list); } static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom) @@ -226,11 +216,12 @@ static void mtk_iommu_tlb_flush_range_sync(unsigned long iova, size_t size, size_t granule, struct mtk_iommu_data *data) { + struct list_head *head = data->hw_list; unsigned long flags; int ret; u32 tmp; - for_each_m4u(data) { + for_each_m4u(data, head) { if (pm_runtime_get_if_in_use(data->dev) <= 0) continue; @@ -451,6 +442,7 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, { struct mtk_iommu_data *data = dev_iommu_priv_get(dev), *frstdata; struct mtk_iommu_domain *dom = to_mtk_domain(domain); + struct list_head *hw_list = data->hw_list; struct device *m4udev = data->dev; int ret, domid; @@ -461,7 +453,7 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, mutex_lock(&dom->mutex); if (!dom->data) { /* Data is in the frstdata in sharing pgtable case. */ - frstdata = mtk_iommu_get_m4u_data(); + frstdata = mtk_iommu_get_frst_data(hw_list); ret = mtk_iommu_domain_finalise(dom, frstdata, domid); if (ret) { @@ -625,10 +617,12 @@ static void mtk_iommu_release_device(struct device *dev) static struct iommu_group *mtk_iommu_device_group(struct device *dev) { - struct mtk_iommu_data *data = mtk_iommu_get_m4u_data(); + struct mtk_iommu_data *c_data = dev_iommu_priv_get(dev), *data; + struct list_head *hw_list = c_data->hw_list; struct iommu_group *group; int domid; + data = mtk_iommu_get_frst_data(hw_list); if (!data) return ERR_PTR(-ENODEV); @@ -938,7 +932,15 @@ static int mtk_iommu_probe(struct platform_device *pdev) goto out_sysfs_remove; spin_lock_init(&data->tlb_lock); - list_add_tail(&data->list, &m4ulist); + + if (MTK_IOMMU_HAS_FLAG(data->plat_data, SHARE_PGTABLE)) { + list_add_tail(&data->list, data->plat_data->hw_list); + data->hw_list = data->plat_data->hw_list; + } else { + INIT_LIST_HEAD(&data->hw_list_head); + list_add_tail(&data->list, &data->hw_list_head); + data->hw_list = &data->hw_list_head; + } if (!iommu_present(&platform_bus_type)) { ret = bus_set_iommu(&platform_bus_type, &mtk_iommu_ops); @@ -1047,7 +1049,8 @@ static const struct dev_pm_ops mtk_iommu_pm_ops = { static const struct mtk_iommu_plat_data mt2712_data = { .m4u_plat = M4U_MT2712, - .flags = HAS_4GB_MODE | HAS_BCLK | HAS_VLD_PA_RNG, + .flags = HAS_4GB_MODE | HAS_BCLK | HAS_VLD_PA_RNG | SHARE_PGTABLE, + .hw_list = &m4ulist, .inv_sel_reg = REG_MMU_INV_SEL_GEN1, .iova_region = single_domain, .iova_region_nr = ARRAY_SIZE(single_domain), diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h index 5e8da947affc..dc868fce0d2a 100644 --- a/drivers/iommu/mtk_iommu.h +++ b/drivers/iommu/mtk_iommu.h @@ -55,6 +55,7 @@ struct mtk_iommu_plat_data { u32 flags; u32 inv_sel_reg; + struct list_head *hw_list; unsigned int iova_region_nr; const struct mtk_iommu_iova_region *iova_region; unsigned char larbid_remap[MTK_LARB_COM_MAX][MTK_LARB_SUBCOM_MAX]; @@ -82,6 +83,12 @@ struct mtk_iommu_data { struct mutex mutex; /* Protect m4u_group/m4u_dom above */ + /* + * In the sharing pgtable case, list data->list to the global list like m4ulist. + * In the non-sharing pgtable case, list data->list to the itself hw_list_head. + */ + struct list_head *hw_list; + struct list_head hw_list_head; struct list_head list; struct mtk_smi_larb_iommu larb_imu[MTK_LARB_NR_MAX]; }; -- cgit v1.2.3 From 129a3b885879c42494044ded94d2dd77e90004b0 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:14:01 +0800 Subject: iommu/mediatek: Add 12G~16G support for multi domains In mt8192, we preassign 0-4G; 4G-8G; 8G-12G for different multimedia engines. This depends on the "dma-ranges=" in the iommu consumer's dtsi node. Adds 12G-16G region here. and reword the previous comment. we don't limit which master locate in which region. CCU still is 8G-12G. Don't change it here. Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-11-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 1b8e4405dd0f..d91a0c138536 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -180,10 +180,12 @@ static const struct mtk_iommu_iova_region single_domain[] = { }; static const struct mtk_iommu_iova_region mt8192_multi_dom[] = { - { .iova_base = 0x0, .size = SZ_4G}, /* disp: 0 ~ 4G */ + { .iova_base = 0x0, .size = SZ_4G}, /* 0 ~ 4G */ #if IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) - { .iova_base = SZ_4G, .size = SZ_4G}, /* vdec: 4G ~ 8G */ - { .iova_base = SZ_4G * 2, .size = SZ_4G}, /* CAM/MDP: 8G ~ 12G */ + { .iova_base = SZ_4G, .size = SZ_4G}, /* 4G ~ 8G */ + { .iova_base = SZ_4G * 2, .size = SZ_4G}, /* 8G ~ 12G */ + { .iova_base = SZ_4G * 3, .size = SZ_4G}, /* 12G ~ 16G */ + { .iova_base = 0x240000000ULL, .size = 0x4000000}, /* CCU0 */ { .iova_base = 0x244000000ULL, .size = 0x4000000}, /* CCU1 */ #endif -- cgit v1.2.3 From 9a87005ed9c2a2135e15e03d97de849577d29437 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:14:02 +0800 Subject: iommu/mediatek: Add a flag DCM_DISABLE In the infra iommu, we should disable DCM. add a new flag for this. Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-12-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index d91a0c138536..92f172a772d1 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -51,6 +51,8 @@ #define F_MMU_STANDARD_AXI_MODE_MASK (BIT(3) | BIT(19)) #define REG_MMU_DCM_DIS 0x050 +#define F_MMU_DCM BIT(8) + #define REG_MMU_WR_LEN_CTRL 0x054 #define F_MMU_WR_THROT_DIS_MASK (BIT(5) | BIT(21)) @@ -119,6 +121,7 @@ #define HAS_LEGACY_IVRP_PADDR BIT(7) #define IOVA_34_EN BIT(8) #define SHARE_PGTABLE BIT(9) /* 2 HW share pgtable */ +#define DCM_DISABLE BIT(10) #define MTK_IOMMU_HAS_FLAG(pdata, _x) \ ((((pdata)->flags) & (_x)) == (_x)) @@ -765,7 +768,11 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) regval = F_MMU_VLD_PA_RNG(7, 4); writel_relaxed(regval, data->base + REG_MMU_VLD_PA_RNG); } - writel_relaxed(0, data->base + REG_MMU_DCM_DIS); + if (MTK_IOMMU_HAS_FLAG(data->plat_data, DCM_DISABLE)) + writel_relaxed(F_MMU_DCM, data->base + REG_MMU_DCM_DIS); + else + writel_relaxed(0, data->base + REG_MMU_DCM_DIS); + if (MTK_IOMMU_HAS_FLAG(data->plat_data, WR_THROT_EN)) { /* write command throttling mode */ regval = readl_relaxed(data->base + REG_MMU_WR_LEN_CTRL); -- cgit v1.2.3 From d265a4addc9c118d2467177b77a78a9956ddb2d5 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:14:03 +0800 Subject: iommu/mediatek: Add a flag STD_AXI_MODE Add a new flag STD_AXI_MODE which is prepared for infra and apu iommu which use the standard axi mode. All the current SoC don't use this flag. Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-13-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 92f172a772d1..22e2b104e3ee 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -122,6 +122,7 @@ #define IOVA_34_EN BIT(8) #define SHARE_PGTABLE BIT(9) /* 2 HW share pgtable */ #define DCM_DISABLE BIT(10) +#define STD_AXI_MODE BIT(11) /* For non MM iommu */ #define MTK_IOMMU_HAS_FLAG(pdata, _x) \ ((((pdata)->flags) & (_x)) == (_x)) @@ -785,7 +786,8 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) regval = 0; } else { regval = readl_relaxed(data->base + REG_MMU_MISC_CTRL); - regval &= ~F_MMU_STANDARD_AXI_MODE_MASK; + if (!MTK_IOMMU_HAS_FLAG(data->plat_data, STD_AXI_MODE)) + regval &= ~F_MMU_STANDARD_AXI_MODE_MASK; if (MTK_IOMMU_HAS_FLAG(data->plat_data, OUT_ORDER_WR_EN)) regval &= ~F_MMU_IN_ORDER_WR_EN_MASK; } -- cgit v1.2.3 From e6d25e7daaf28341b3be4b8d4ea0bdee6265b1c9 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:14:04 +0800 Subject: iommu/mediatek: Remove the granule in the tlb flush The MediaTek IOMMU doesn't care about granule when tlb flushing. Remove this variable. Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-14-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 22e2b104e3ee..ca77e7f1ce5d 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -219,7 +219,6 @@ static void mtk_iommu_tlb_flush_all(struct mtk_iommu_data *data) } static void mtk_iommu_tlb_flush_range_sync(unsigned long iova, size_t size, - size_t granule, struct mtk_iommu_data *data) { struct list_head *head = data->hw_list; @@ -541,8 +540,7 @@ static void mtk_iommu_iotlb_sync(struct iommu_domain *domain, struct mtk_iommu_domain *dom = to_mtk_domain(domain); size_t length = gather->end - gather->start + 1; - mtk_iommu_tlb_flush_range_sync(gather->start, length, gather->pgsize, - dom->data); + mtk_iommu_tlb_flush_range_sync(gather->start, length, dom->data); } static void mtk_iommu_sync_map(struct iommu_domain *domain, unsigned long iova, @@ -550,7 +548,7 @@ static void mtk_iommu_sync_map(struct iommu_domain *domain, unsigned long iova, { struct mtk_iommu_domain *dom = to_mtk_domain(domain); - mtk_iommu_tlb_flush_range_sync(iova, size, size, dom->data); + mtk_iommu_tlb_flush_range_sync(iova, size, dom->data); } static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain, -- cgit v1.2.3 From 82e51771bff9ec900891a51629278a54a866a6ba Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:14:05 +0800 Subject: iommu/mediatek: Always enable output PA over 32bits in isr Currently the output PA[32:33] is contained by the flag IOVA_34. This is not right. the iova_34 has no relation with pa[32:33], the 32bits iova still could map to pa[32:33]. Move it out from the flag. No need fix tag since currently only mt8192 use the calulation and it always has this IOVA_34 flag. Prepare for the IOMMU that still use IOVA 32bits but its dram size may be over 4GB. Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-15-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index ca77e7f1ce5d..d9689e041336 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -283,11 +283,11 @@ static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) write = fault_iova & F_MMU_FAULT_VA_WRITE_BIT; if (MTK_IOMMU_HAS_FLAG(data->plat_data, IOVA_34_EN)) { va34_32 = FIELD_GET(F_MMU_INVAL_VA_34_32_MASK, fault_iova); - pa34_32 = FIELD_GET(F_MMU_INVAL_PA_34_32_MASK, fault_iova); fault_iova = fault_iova & F_MMU_INVAL_VA_31_12_MASK; fault_iova |= (u64)va34_32 << 32; - fault_pa |= (u64)pa34_32 << 32; } + pa34_32 = FIELD_GET(F_MMU_INVAL_PA_34_32_MASK, fault_iova); + fault_pa |= (u64)pa34_32 << 32; fault_port = F_MMU_INT_ID_PORT_ID(regval); if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_SUB_COMM)) { -- cgit v1.2.3 From 9ec30c09547d8dcf5491bb1334a5099a3412538b Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:14:06 +0800 Subject: iommu/mediatek: Add SUB_COMMON_3BITS flag In prevous SoC, the sub common id occupy 2 bits. the mt8195's sub common id has 3bits. Add a new flag for this. and rename the previous flag to _2BITS. For readable, I put these two flags together, then move the other flags. no functional change. Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-16-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 26 ++++++++++++++++---------- drivers/iommu/mtk_iommu.h | 2 +- 2 files changed, 17 insertions(+), 11 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index d9689e041336..937478cd8966 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -105,6 +105,8 @@ #define REG_MMU1_INT_ID 0x154 #define F_MMU_INT_ID_COMM_ID(a) (((a) >> 9) & 0x7) #define F_MMU_INT_ID_SUB_COMM_ID(a) (((a) >> 7) & 0x3) +#define F_MMU_INT_ID_COMM_ID_EXT(a) (((a) >> 10) & 0x7) +#define F_MMU_INT_ID_SUB_COMM_ID_EXT(a) (((a) >> 7) & 0x7) #define F_MMU_INT_ID_LARB_ID(a) (((a) >> 7) & 0x7) #define F_MMU_INT_ID_PORT_ID(a) (((a) >> 2) & 0x1f) @@ -116,13 +118,14 @@ #define HAS_VLD_PA_RNG BIT(2) #define RESET_AXI BIT(3) #define OUT_ORDER_WR_EN BIT(4) -#define HAS_SUB_COMM BIT(5) -#define WR_THROT_EN BIT(6) -#define HAS_LEGACY_IVRP_PADDR BIT(7) -#define IOVA_34_EN BIT(8) -#define SHARE_PGTABLE BIT(9) /* 2 HW share pgtable */ -#define DCM_DISABLE BIT(10) -#define STD_AXI_MODE BIT(11) /* For non MM iommu */ +#define HAS_SUB_COMM_2BITS BIT(5) +#define HAS_SUB_COMM_3BITS BIT(6) +#define WR_THROT_EN BIT(7) +#define HAS_LEGACY_IVRP_PADDR BIT(8) +#define IOVA_34_EN BIT(9) +#define SHARE_PGTABLE BIT(10) /* 2 HW share pgtable */ +#define DCM_DISABLE BIT(11) +#define STD_AXI_MODE BIT(12) /* For non MM iommu */ #define MTK_IOMMU_HAS_FLAG(pdata, _x) \ ((((pdata)->flags) & (_x)) == (_x)) @@ -290,9 +293,12 @@ static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) fault_pa |= (u64)pa34_32 << 32; fault_port = F_MMU_INT_ID_PORT_ID(regval); - if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_SUB_COMM)) { + if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_SUB_COMM_2BITS)) { fault_larb = F_MMU_INT_ID_COMM_ID(regval); sub_comm = F_MMU_INT_ID_SUB_COMM_ID(regval); + } else if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_SUB_COMM_3BITS)) { + fault_larb = F_MMU_INT_ID_COMM_ID_EXT(regval); + sub_comm = F_MMU_INT_ID_SUB_COMM_ID_EXT(regval); } else { fault_larb = F_MMU_INT_ID_LARB_ID(regval); } @@ -1068,7 +1074,7 @@ static const struct mtk_iommu_plat_data mt2712_data = { static const struct mtk_iommu_plat_data mt6779_data = { .m4u_plat = M4U_MT6779, - .flags = HAS_SUB_COMM | OUT_ORDER_WR_EN | WR_THROT_EN, + .flags = HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN | WR_THROT_EN, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, .iova_region = single_domain, .iova_region_nr = ARRAY_SIZE(single_domain), @@ -1105,7 +1111,7 @@ static const struct mtk_iommu_plat_data mt8183_data = { static const struct mtk_iommu_plat_data mt8192_data = { .m4u_plat = M4U_MT8192, - .flags = HAS_BCLK | HAS_SUB_COMM | OUT_ORDER_WR_EN | + .flags = HAS_BCLK | HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN | WR_THROT_EN | IOVA_34_EN, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, .iova_region = mt8192_multi_dom, diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h index dc868fce0d2a..f41e32252056 100644 --- a/drivers/iommu/mtk_iommu.h +++ b/drivers/iommu/mtk_iommu.h @@ -20,7 +20,7 @@ #include #define MTK_LARB_COM_MAX 8 -#define MTK_LARB_SUBCOM_MAX 4 +#define MTK_LARB_SUBCOM_MAX 8 #define MTK_IOMMU_GROUP_MAX 8 -- cgit v1.2.3 From 8cd1e619e79b5bdcd6fc346324a89e25c3325fde Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:14:07 +0800 Subject: iommu/mediatek: Add IOMMU_TYPE flag Add IOMMU_TYPE definition. In the mt8195, we have another IOMMU_TYPE: infra iommu, also there will be another APU_IOMMU, thus, use 2bits for the IOMMU_TYPE. Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-17-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 937478cd8966..b5b95d9a372e 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -126,9 +126,17 @@ #define SHARE_PGTABLE BIT(10) /* 2 HW share pgtable */ #define DCM_DISABLE BIT(11) #define STD_AXI_MODE BIT(12) /* For non MM iommu */ +/* 2 bits: iommu type */ +#define MTK_IOMMU_TYPE_MM (0x0 << 13) +#define MTK_IOMMU_TYPE_INFRA (0x1 << 13) +#define MTK_IOMMU_TYPE_MASK (0x3 << 13) -#define MTK_IOMMU_HAS_FLAG(pdata, _x) \ - ((((pdata)->flags) & (_x)) == (_x)) +#define MTK_IOMMU_HAS_FLAG_MASK(pdata, _x, mask) \ + ((((pdata)->flags) & (mask)) == (_x)) + +#define MTK_IOMMU_HAS_FLAG(pdata, _x) MTK_IOMMU_HAS_FLAG_MASK(pdata, _x, _x) +#define MTK_IOMMU_IS_TYPE(pdata, _x) MTK_IOMMU_HAS_FLAG_MASK(pdata, _x,\ + MTK_IOMMU_TYPE_MASK) struct mtk_iommu_domain { struct io_pgtable_cfg cfg; -- cgit v1.2.3 From d2e9a1102cfc22c08450875faa667a60f0b1b1f6 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:14:08 +0800 Subject: iommu/mediatek: Contain MM IOMMU flow with the MM TYPE Prepare for supporting INFRA_IOMMU, and APU_IOMMU later. For Infra IOMMU/APU IOMMU, it doesn't have the "larb""port". thus, Use the MM flag contain the MM_IOMMU special flow, Also, it moves a big chunk code about parsing the mediatek,larbs into a function, this is only needed for MM IOMMU. and all the current SoC are MM_IOMMU. The device link between iommu consumer device and smi-larb device only is needed in MM iommu case. Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-18-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 213 ++++++++++++++++++++++++++-------------------- 1 file changed, 122 insertions(+), 91 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index b5b95d9a372e..7d4bc562b2a3 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -138,6 +138,8 @@ #define MTK_IOMMU_IS_TYPE(pdata, _x) MTK_IOMMU_HAS_FLAG_MASK(pdata, _x,\ MTK_IOMMU_TYPE_MASK) +#define MTK_INVALID_LARBID MTK_LARB_NR_MAX + struct mtk_iommu_domain { struct io_pgtable_cfg cfg; struct io_pgtable_ops *iop; @@ -274,7 +276,7 @@ static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) { struct mtk_iommu_data *data = dev_id; struct mtk_iommu_domain *dom = data->m4u_dom; - unsigned int fault_larb, fault_port, sub_comm = 0; + unsigned int fault_larb = MTK_INVALID_LARBID, fault_port = 0, sub_comm = 0; u32 int_state, regval, va34_32, pa34_32; u64 fault_iova, fault_pa; bool layer, write; @@ -300,17 +302,19 @@ static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) pa34_32 = FIELD_GET(F_MMU_INVAL_PA_34_32_MASK, fault_iova); fault_pa |= (u64)pa34_32 << 32; - fault_port = F_MMU_INT_ID_PORT_ID(regval); - if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_SUB_COMM_2BITS)) { - fault_larb = F_MMU_INT_ID_COMM_ID(regval); - sub_comm = F_MMU_INT_ID_SUB_COMM_ID(regval); - } else if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_SUB_COMM_3BITS)) { - fault_larb = F_MMU_INT_ID_COMM_ID_EXT(regval); - sub_comm = F_MMU_INT_ID_SUB_COMM_ID_EXT(regval); - } else { - fault_larb = F_MMU_INT_ID_LARB_ID(regval); + if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { + fault_port = F_MMU_INT_ID_PORT_ID(regval); + if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_SUB_COMM_2BITS)) { + fault_larb = F_MMU_INT_ID_COMM_ID(regval); + sub_comm = F_MMU_INT_ID_SUB_COMM_ID(regval); + } else if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_SUB_COMM_3BITS)) { + fault_larb = F_MMU_INT_ID_COMM_ID_EXT(regval); + sub_comm = F_MMU_INT_ID_SUB_COMM_ID_EXT(regval); + } else { + fault_larb = F_MMU_INT_ID_LARB_ID(regval); + } + fault_larb = data->plat_data->larbid_remap[fault_larb][sub_comm]; } - fault_larb = data->plat_data->larbid_remap[fault_larb][sub_comm]; if (report_iommu_fault(&dom->domain, data->dev, fault_iova, write ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ)) { @@ -374,19 +378,21 @@ static void mtk_iommu_config(struct mtk_iommu_data *data, struct device *dev, larbid = MTK_M4U_TO_LARB(fwspec->ids[i]); portid = MTK_M4U_TO_PORT(fwspec->ids[i]); - larb_mmu = &data->larb_imu[larbid]; + if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { + larb_mmu = &data->larb_imu[larbid]; - region = data->plat_data->iova_region + domid; - larb_mmu->bank[portid] = upper_32_bits(region->iova_base); + region = data->plat_data->iova_region + domid; + larb_mmu->bank[portid] = upper_32_bits(region->iova_base); - dev_dbg(dev, "%s iommu for larb(%s) port %d dom %d bank %d.\n", - enable ? "enable" : "disable", dev_name(larb_mmu->dev), - portid, domid, larb_mmu->bank[portid]); + dev_dbg(dev, "%s iommu for larb(%s) port %d dom %d bank %d.\n", + enable ? "enable" : "disable", dev_name(larb_mmu->dev), + portid, domid, larb_mmu->bank[portid]); - if (enable) - larb_mmu->mmu |= MTK_SMI_MMU_EN(portid); - else - larb_mmu->mmu &= ~MTK_SMI_MMU_EN(portid); + if (enable) + larb_mmu->mmu |= MTK_SMI_MMU_EN(portid); + else + larb_mmu->mmu &= ~MTK_SMI_MMU_EN(portid); + } } } @@ -593,6 +599,9 @@ static struct iommu_device *mtk_iommu_probe_device(struct device *dev) data = dev_iommu_priv_get(dev); + if (!MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) + return &data->iommu; + /* * Link the consumer device with the smi-larb device(supplier). * The device that connects with each a larb is a independent HW. @@ -626,9 +635,11 @@ static void mtk_iommu_release_device(struct device *dev) return; data = dev_iommu_priv_get(dev); - larbid = MTK_M4U_TO_LARB(fwspec->ids[0]); - larbdev = data->larb_imu[larbid].dev; - device_link_remove(dev, larbdev); + if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { + larbid = MTK_M4U_TO_LARB(fwspec->ids[0]); + larbdev = data->larb_imu[larbid].dev; + device_link_remove(dev, larbdev); + } iommu_fwspec_free(dev); } @@ -820,19 +831,77 @@ static const struct component_master_ops mtk_iommu_com_ops = { .unbind = mtk_iommu_unbind, }; +static int mtk_iommu_mm_dts_parse(struct device *dev, struct component_match **match, + struct mtk_iommu_data *data) +{ + struct device_node *larbnode, *smicomm_node; + struct platform_device *plarbdev; + struct device_link *link; + int i, larb_nr, ret; + + larb_nr = of_count_phandle_with_args(dev->of_node, "mediatek,larbs", NULL); + if (larb_nr < 0) + return larb_nr; + + for (i = 0; i < larb_nr; i++) { + u32 id; + + larbnode = of_parse_phandle(dev->of_node, "mediatek,larbs", i); + if (!larbnode) + return -EINVAL; + + if (!of_device_is_available(larbnode)) { + of_node_put(larbnode); + continue; + } + + ret = of_property_read_u32(larbnode, "mediatek,larb-id", &id); + if (ret)/* The id is consecutive if there is no this property */ + id = i; + + plarbdev = of_find_device_by_node(larbnode); + if (!plarbdev) { + of_node_put(larbnode); + return -ENODEV; + } + if (!plarbdev->dev.driver) { + of_node_put(larbnode); + return -EPROBE_DEFER; + } + data->larb_imu[id].dev = &plarbdev->dev; + + component_match_add_release(dev, match, component_release_of, + component_compare_of, larbnode); + } + + /* Get smi-common dev from the last larb. */ + smicomm_node = of_parse_phandle(larbnode, "mediatek,smi", 0); + if (!smicomm_node) + return -EINVAL; + + plarbdev = of_find_device_by_node(smicomm_node); + of_node_put(smicomm_node); + data->smicomm_dev = &plarbdev->dev; + + link = device_link_add(data->smicomm_dev, dev, + DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME); + if (!link) { + dev_err(dev, "Unable to link %s.\n", dev_name(data->smicomm_dev)); + return -EINVAL; + } + return 0; +} + static int mtk_iommu_probe(struct platform_device *pdev) { struct mtk_iommu_data *data; struct device *dev = &pdev->dev; - struct device_node *larbnode, *smicomm_node; - struct platform_device *plarbdev; - struct device_link *link; struct resource *res; resource_size_t ioaddr; struct component_match *match = NULL; struct regmap *infracfg; void *protect; - int i, larb_nr, ret; + int ret; u32 val; char *p; @@ -887,59 +956,14 @@ static int mtk_iommu_probe(struct platform_device *pdev) return PTR_ERR(data->bclk); } - larb_nr = of_count_phandle_with_args(dev->of_node, - "mediatek,larbs", NULL); - if (larb_nr < 0) - return larb_nr; - - for (i = 0; i < larb_nr; i++) { - u32 id; - - larbnode = of_parse_phandle(dev->of_node, "mediatek,larbs", i); - if (!larbnode) - return -EINVAL; - - if (!of_device_is_available(larbnode)) { - of_node_put(larbnode); - continue; - } - - ret = of_property_read_u32(larbnode, "mediatek,larb-id", &id); - if (ret)/* The id is consecutive if there is no this property */ - id = i; - - plarbdev = of_find_device_by_node(larbnode); - if (!plarbdev) { - of_node_put(larbnode); - return -ENODEV; - } - if (!plarbdev->dev.driver) { - of_node_put(larbnode); - return -EPROBE_DEFER; - } - data->larb_imu[id].dev = &plarbdev->dev; - - component_match_add_release(dev, &match, component_release_of, - component_compare_of, larbnode); - } - - /* Get smi-common dev from the last larb. */ - smicomm_node = of_parse_phandle(larbnode, "mediatek,smi", 0); - if (!smicomm_node) - return -EINVAL; - - plarbdev = of_find_device_by_node(smicomm_node); - of_node_put(smicomm_node); - data->smicomm_dev = &plarbdev->dev; - pm_runtime_enable(dev); - link = device_link_add(data->smicomm_dev, dev, - DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME); - if (!link) { - dev_err(dev, "Unable to link %s.\n", dev_name(data->smicomm_dev)); - ret = -EINVAL; - goto out_runtime_disable; + if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { + ret = mtk_iommu_mm_dts_parse(dev, &match, data); + if (ret) { + dev_err(dev, "mm dts parse fail(%d).", ret); + goto out_runtime_disable; + } } platform_set_drvdata(pdev, data); @@ -971,9 +995,11 @@ static int mtk_iommu_probe(struct platform_device *pdev) goto out_list_del; } - ret = component_master_add_with_match(dev, &mtk_iommu_com_ops, match); - if (ret) - goto out_bus_set_null; + if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { + ret = component_master_add_with_match(dev, &mtk_iommu_com_ops, match); + if (ret) + goto out_bus_set_null; + } return ret; out_bus_set_null: @@ -984,7 +1010,8 @@ out_list_del: out_sysfs_remove: iommu_device_sysfs_remove(&data->iommu); out_link_remove: - device_link_remove(data->smicomm_dev, dev); + if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) + device_link_remove(data->smicomm_dev, dev); out_runtime_disable: pm_runtime_disable(dev); return ret; @@ -999,10 +1026,12 @@ static int mtk_iommu_remove(struct platform_device *pdev) list_del(&data->list); - device_link_remove(data->smicomm_dev, &pdev->dev); + if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { + device_link_remove(data->smicomm_dev, &pdev->dev); + component_master_del(&pdev->dev, &mtk_iommu_com_ops); + } pm_runtime_disable(&pdev->dev); devm_free_irq(&pdev->dev, data->irq, data); - component_master_del(&pdev->dev, &mtk_iommu_com_ops); return 0; } @@ -1072,7 +1101,8 @@ static const struct dev_pm_ops mtk_iommu_pm_ops = { static const struct mtk_iommu_plat_data mt2712_data = { .m4u_plat = M4U_MT2712, - .flags = HAS_4GB_MODE | HAS_BCLK | HAS_VLD_PA_RNG | SHARE_PGTABLE, + .flags = HAS_4GB_MODE | HAS_BCLK | HAS_VLD_PA_RNG | SHARE_PGTABLE | + MTK_IOMMU_TYPE_MM, .hw_list = &m4ulist, .inv_sel_reg = REG_MMU_INV_SEL_GEN1, .iova_region = single_domain, @@ -1082,7 +1112,8 @@ static const struct mtk_iommu_plat_data mt2712_data = { static const struct mtk_iommu_plat_data mt6779_data = { .m4u_plat = M4U_MT6779, - .flags = HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN | WR_THROT_EN, + .flags = HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN | WR_THROT_EN | + MTK_IOMMU_TYPE_MM, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, .iova_region = single_domain, .iova_region_nr = ARRAY_SIZE(single_domain), @@ -1091,7 +1122,7 @@ static const struct mtk_iommu_plat_data mt6779_data = { static const struct mtk_iommu_plat_data mt8167_data = { .m4u_plat = M4U_MT8167, - .flags = RESET_AXI | HAS_LEGACY_IVRP_PADDR, + .flags = RESET_AXI | HAS_LEGACY_IVRP_PADDR | MTK_IOMMU_TYPE_MM, .inv_sel_reg = REG_MMU_INV_SEL_GEN1, .iova_region = single_domain, .iova_region_nr = ARRAY_SIZE(single_domain), @@ -1101,7 +1132,7 @@ static const struct mtk_iommu_plat_data mt8167_data = { static const struct mtk_iommu_plat_data mt8173_data = { .m4u_plat = M4U_MT8173, .flags = HAS_4GB_MODE | HAS_BCLK | RESET_AXI | - HAS_LEGACY_IVRP_PADDR, + HAS_LEGACY_IVRP_PADDR | MTK_IOMMU_TYPE_MM, .inv_sel_reg = REG_MMU_INV_SEL_GEN1, .iova_region = single_domain, .iova_region_nr = ARRAY_SIZE(single_domain), @@ -1110,7 +1141,7 @@ static const struct mtk_iommu_plat_data mt8173_data = { static const struct mtk_iommu_plat_data mt8183_data = { .m4u_plat = M4U_MT8183, - .flags = RESET_AXI, + .flags = RESET_AXI | MTK_IOMMU_TYPE_MM, .inv_sel_reg = REG_MMU_INV_SEL_GEN1, .iova_region = single_domain, .iova_region_nr = ARRAY_SIZE(single_domain), @@ -1120,7 +1151,7 @@ static const struct mtk_iommu_plat_data mt8183_data = { static const struct mtk_iommu_plat_data mt8192_data = { .m4u_plat = M4U_MT8192, .flags = HAS_BCLK | HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN | - WR_THROT_EN | IOVA_34_EN, + WR_THROT_EN | IOVA_34_EN | MTK_IOMMU_TYPE_MM, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, .iova_region = mt8192_multi_dom, .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom), -- cgit v1.2.3 From f7b71d0d3ee7f83439f82af2d7e595ecd9b9b42c Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:14:09 +0800 Subject: iommu/mediatek: Adjust device link when it is sub-common For MM IOMMU, We always add device link between smi-common and IOMMU HW. In mt8195, we add smi-sub-common. Thus, if the node is sub-common, we still need find again to get smi-common, then do device link. Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-19-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 7d4bc562b2a3..bcdc932cd473 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -834,7 +834,7 @@ static const struct component_master_ops mtk_iommu_com_ops = { static int mtk_iommu_mm_dts_parse(struct device *dev, struct component_match **match, struct mtk_iommu_data *data) { - struct device_node *larbnode, *smicomm_node; + struct device_node *larbnode, *smicomm_node, *smi_subcomm_node; struct platform_device *plarbdev; struct device_link *link; int i, larb_nr, ret; @@ -874,11 +874,21 @@ static int mtk_iommu_mm_dts_parse(struct device *dev, struct component_match **m component_compare_of, larbnode); } - /* Get smi-common dev from the last larb. */ - smicomm_node = of_parse_phandle(larbnode, "mediatek,smi", 0); - if (!smicomm_node) + /* Get smi-(sub)-common dev from the last larb. */ + smi_subcomm_node = of_parse_phandle(larbnode, "mediatek,smi", 0); + if (!smi_subcomm_node) return -EINVAL; + /* + * It may have two level smi-common. the node is smi-sub-common if it + * has a new mediatek,smi property. otherwise it is smi-commmon. + */ + smicomm_node = of_parse_phandle(smi_subcomm_node, "mediatek,smi", 0); + if (smicomm_node) + of_node_put(smi_subcomm_node); + else + smicomm_node = smi_subcomm_node; + plarbdev = of_find_device_by_node(smicomm_node); of_node_put(smicomm_node); data->smicomm_dev = &plarbdev->dev; -- cgit v1.2.3 From 32e1cccf68123bcd357c0115567d2ce3453cd3ed Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:14:10 +0800 Subject: iommu/mediatek: Allow IOMMU_DOMAIN_UNMANAGED for PCIe VFIO Allow the type IOMMU_DOMAIN_UNMANAGED since vfio_iommu_type1.c always call iommu_domain_alloc. The PCIe EP works ok when going through vfio. Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-20-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index bcdc932cd473..0121d71c315f 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -446,7 +446,7 @@ static struct iommu_domain *mtk_iommu_domain_alloc(unsigned type) { struct mtk_iommu_domain *dom; - if (type != IOMMU_DOMAIN_DMA) + if (type != IOMMU_DOMAIN_DMA && type != IOMMU_DOMAIN_UNMANAGED) return NULL; dom = kzalloc(sizeof(*dom), GFP_KERNEL); -- cgit v1.2.3 From 6077c7e5d22f2e0ffdb1dd97bf37cba66f317627 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:14:11 +0800 Subject: iommu/mediatek: Add a PM_CLK_AO flag for infra iommu The power/clock of infra iommu is always on, and it doesn't have the device link with the master devices, then the infra iommu device's PM status is not active, thus we add A PM_CLK_AO flag for infra iommu. The tlb operation is a bit not clear here, there are 2 special cases. Comment them in the code. Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-21-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 0121d71c315f..cd89c109e8c4 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -130,6 +130,8 @@ #define MTK_IOMMU_TYPE_MM (0x0 << 13) #define MTK_IOMMU_TYPE_INFRA (0x1 << 13) #define MTK_IOMMU_TYPE_MASK (0x3 << 13) +/* PM and clock always on. e.g. infra iommu */ +#define PM_CLK_AO BIT(15) #define MTK_IOMMU_HAS_FLAG_MASK(pdata, _x, mask) \ ((((pdata)->flags) & (mask)) == (_x)) @@ -235,13 +237,33 @@ static void mtk_iommu_tlb_flush_range_sync(unsigned long iova, size_t size, struct mtk_iommu_data *data) { struct list_head *head = data->hw_list; + bool check_pm_status; unsigned long flags; int ret; u32 tmp; for_each_m4u(data, head) { - if (pm_runtime_get_if_in_use(data->dev) <= 0) - continue; + /* + * To avoid resume the iommu device frequently when the iommu device + * is not active, it doesn't always call pm_runtime_get here, then tlb + * flush depends on the tlb flush all in the runtime resume. + * + * There are 2 special cases: + * + * Case1: The iommu dev doesn't have power domain but has bclk. This case + * should also avoid the tlb flush while the dev is not active to mute + * the tlb timeout log. like mt8173. + * + * Case2: The power/clock of infra iommu is always on, and it doesn't + * have the device link with the master devices. This case should avoid + * the PM status check. + */ + check_pm_status = !MTK_IOMMU_HAS_FLAG(data->plat_data, PM_CLK_AO); + + if (check_pm_status) { + if (pm_runtime_get_if_in_use(data->dev) <= 0) + continue; + } spin_lock_irqsave(&data->tlb_lock, flags); writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, @@ -268,7 +290,8 @@ static void mtk_iommu_tlb_flush_range_sync(unsigned long iova, size_t size, mtk_iommu_tlb_flush_all(data); } - pm_runtime_put(data->dev); + if (check_pm_status) + pm_runtime_put(data->dev); } } -- cgit v1.2.3 From f9b8c9b2194327c26eecfc82dde9541e80c24c80 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:14:12 +0800 Subject: iommu/mediatek: Add infra iommu support The infra iommu enable bits in mt8195 is in the pericfg register segment, use regmap to update it. If infra iommu master translation fault, It doesn't have the larbid/portid, thus print out the whole register value. Since regmap_update_bits may fail, add return value for mtk_iommu_config. Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-22-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 36 +++++++++++++++++++++++++++++------- drivers/iommu/mtk_iommu.h | 2 ++ 2 files changed, 31 insertions(+), 7 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index cd89c109e8c4..ff48506b480c 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -112,6 +112,8 @@ #define MTK_PROTECT_PA_ALIGN 256 +#define PERICFG_IOMMU_1 0x714 + #define HAS_4GB_MODE BIT(0) /* HW will use the EMI clock if there isn't the "bclk". */ #define HAS_BCLK BIT(1) @@ -343,8 +345,8 @@ static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) write ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ)) { dev_err_ratelimited( data->dev, - "fault type=0x%x iova=0x%llx pa=0x%llx larb=%d port=%d layer=%d %s\n", - int_state, fault_iova, fault_pa, fault_larb, fault_port, + "fault type=0x%x iova=0x%llx pa=0x%llx master=0x%x(larb=%d port=%d) layer=%d %s\n", + int_state, fault_iova, fault_pa, regval, fault_larb, fault_port, layer, write ? "write" : "read"); } @@ -388,14 +390,15 @@ static int mtk_iommu_get_domain_id(struct device *dev, return -EINVAL; } -static void mtk_iommu_config(struct mtk_iommu_data *data, struct device *dev, - bool enable, unsigned int domid) +static int mtk_iommu_config(struct mtk_iommu_data *data, struct device *dev, + bool enable, unsigned int domid) { struct mtk_smi_larb_iommu *larb_mmu; unsigned int larbid, portid; struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); const struct mtk_iommu_iova_region *region; - int i; + u32 peri_mmuen, peri_mmuen_msk; + int i, ret = 0; for (i = 0; i < fwspec->num_ids; ++i) { larbid = MTK_M4U_TO_LARB(fwspec->ids[i]); @@ -415,8 +418,19 @@ static void mtk_iommu_config(struct mtk_iommu_data *data, struct device *dev, larb_mmu->mmu |= MTK_SMI_MMU_EN(portid); else larb_mmu->mmu &= ~MTK_SMI_MMU_EN(portid); + } else if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_INFRA)) { + peri_mmuen_msk = BIT(portid); + peri_mmuen = enable ? peri_mmuen_msk : 0; + + ret = regmap_update_bits(data->pericfg, PERICFG_IOMMU_1, + peri_mmuen_msk, peri_mmuen); + if (ret) + dev_err(dev, "%s iommu(%s) inframaster 0x%x fail(%d).\n", + enable ? "enable" : "disable", + dev_name(data->dev), peri_mmuen_msk, ret); } } + return ret; } static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom, @@ -531,8 +545,7 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, } mutex_unlock(&data->mutex); - mtk_iommu_config(data, dev, true, domid); - return 0; + return mtk_iommu_config(data, dev, true, domid); err_unlock: mutex_unlock(&data->mutex); @@ -997,6 +1010,15 @@ static int mtk_iommu_probe(struct platform_device *pdev) dev_err(dev, "mm dts parse fail(%d).", ret); goto out_runtime_disable; } + } else if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_INFRA) && + data->plat_data->pericfg_comp_str) { + infracfg = syscon_regmap_lookup_by_compatible(data->plat_data->pericfg_comp_str); + if (IS_ERR(infracfg)) { + ret = PTR_ERR(infracfg); + goto out_runtime_disable; + } + + data->pericfg = infracfg; } platform_set_drvdata(pdev, data); diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h index f41e32252056..56838fad8c73 100644 --- a/drivers/iommu/mtk_iommu.h +++ b/drivers/iommu/mtk_iommu.h @@ -55,6 +55,7 @@ struct mtk_iommu_plat_data { u32 flags; u32 inv_sel_reg; + char *pericfg_comp_str; struct list_head *hw_list; unsigned int iova_region_nr; const struct mtk_iommu_iova_region *iova_region; @@ -80,6 +81,7 @@ struct mtk_iommu_data { struct device *smicomm_dev; struct dma_iommu_mapping *mapping; /* For mtk_iommu_v1.c */ + struct regmap *pericfg; struct mutex mutex; /* Protect m4u_group/m4u_dom above */ -- cgit v1.2.3 From e76290702570bb0ef825050ce4a529b10dab2dba Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:14:13 +0800 Subject: iommu/mediatek: Add PCIe support Currently the code for of_iommu_configure_dev_id is like this: static int of_iommu_configure_dev_id(struct device_node *master_np, struct device *dev, const u32 *id) { struct of_phandle_args iommu_spec = { .args_count = 1 }; err = of_map_id(master_np, *id, "iommu-map", "iommu-map-mask", &iommu_spec.np, iommu_spec.args); ... } It supports only one id output. BUT our PCIe HW has two ID(one is for writing, the other is for reading). I'm not sure if we should change of_map_id to support output MAX_PHANDLE_ARGS. Here add the solution in ourselve drivers. If it's pcie case, enable one more bit. Not all infra iommu support PCIe, thus add a PCIe support flag here. Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-23-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index ff48506b480c..d63fe28c1403 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -134,6 +135,7 @@ #define MTK_IOMMU_TYPE_MASK (0x3 << 13) /* PM and clock always on. e.g. infra iommu */ #define PM_CLK_AO BIT(15) +#define IFA_IOMMU_PCIE_SUPPORT BIT(16) #define MTK_IOMMU_HAS_FLAG_MASK(pdata, _x, mask) \ ((((pdata)->flags) & (mask)) == (_x)) @@ -420,8 +422,11 @@ static int mtk_iommu_config(struct mtk_iommu_data *data, struct device *dev, larb_mmu->mmu &= ~MTK_SMI_MMU_EN(portid); } else if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_INFRA)) { peri_mmuen_msk = BIT(portid); - peri_mmuen = enable ? peri_mmuen_msk : 0; + /* PCI dev has only one output id, enable the next writing bit for PCIe */ + if (dev_is_pci(dev)) + peri_mmuen_msk |= BIT(portid + 1); + peri_mmuen = enable ? peri_mmuen_msk : 0; ret = regmap_update_bits(data->pericfg, PERICFG_IOMMU_1, peri_mmuen_msk, peri_mmuen); if (ret) @@ -1054,6 +1059,15 @@ static int mtk_iommu_probe(struct platform_device *pdev) ret = component_master_add_with_match(dev, &mtk_iommu_com_ops, match); if (ret) goto out_bus_set_null; + } else if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_INFRA) && + MTK_IOMMU_HAS_FLAG(data->plat_data, IFA_IOMMU_PCIE_SUPPORT)) { +#ifdef CONFIG_PCI + if (!iommu_present(&pci_bus_type)) { + ret = bus_set_iommu(&pci_bus_type, &mtk_iommu_ops); + if (ret) /* PCIe fail don't affect platform_bus. */ + goto out_list_del; + } +#endif } return ret; @@ -1084,6 +1098,11 @@ static int mtk_iommu_remove(struct platform_device *pdev) if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { device_link_remove(data->smicomm_dev, &pdev->dev); component_master_del(&pdev->dev, &mtk_iommu_com_ops); + } else if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_INFRA) && + MTK_IOMMU_HAS_FLAG(data->plat_data, IFA_IOMMU_PCIE_SUPPORT)) { +#ifdef CONFIG_PCI + bus_set_iommu(&pci_bus_type, NULL); +#endif } pm_runtime_disable(&pdev->dev); devm_free_irq(&pdev->dev, data->irq, data); -- cgit v1.2.3 From ef68a193c719436a2203c2d59442e0f32775385c Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:14:14 +0800 Subject: iommu/mediatek: Add mt8195 support mt8195 has 3 IOMMU, containing 2 MM IOMMUs, one is for vdo, the other is for vpp. and 1 INFRA IOMMU. Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-24-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 41 +++++++++++++++++++++++++++++++++++++++++ drivers/iommu/mtk_iommu.h | 1 + 2 files changed, 42 insertions(+) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index d63fe28c1403..47355010502e 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -1233,6 +1233,44 @@ static const struct mtk_iommu_plat_data mt8192_data = { {0, 14, 16}, {0, 13, 18, 17}}, }; +static const struct mtk_iommu_plat_data mt8195_data_infra = { + .m4u_plat = M4U_MT8195, + .flags = WR_THROT_EN | DCM_DISABLE | STD_AXI_MODE | PM_CLK_AO | + MTK_IOMMU_TYPE_INFRA | IFA_IOMMU_PCIE_SUPPORT, + .pericfg_comp_str = "mediatek,mt8195-pericfg_ao", + .inv_sel_reg = REG_MMU_INV_SEL_GEN2, + .iova_region = single_domain, + .iova_region_nr = ARRAY_SIZE(single_domain), +}; + +static const struct mtk_iommu_plat_data mt8195_data_vdo = { + .m4u_plat = M4U_MT8195, + .flags = HAS_BCLK | HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN | + WR_THROT_EN | IOVA_34_EN | SHARE_PGTABLE | MTK_IOMMU_TYPE_MM, + .hw_list = &m4ulist, + .inv_sel_reg = REG_MMU_INV_SEL_GEN2, + .iova_region = mt8192_multi_dom, + .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom), + .larbid_remap = {{2, 0}, {21}, {24}, {7}, {19}, {9, 10, 11}, + {13, 17, 15/* 17b */, 25}, {5}}, +}; + +static const struct mtk_iommu_plat_data mt8195_data_vpp = { + .m4u_plat = M4U_MT8195, + .flags = HAS_BCLK | HAS_SUB_COMM_3BITS | OUT_ORDER_WR_EN | + WR_THROT_EN | IOVA_34_EN | SHARE_PGTABLE | MTK_IOMMU_TYPE_MM, + .hw_list = &m4ulist, + .inv_sel_reg = REG_MMU_INV_SEL_GEN2, + .iova_region = mt8192_multi_dom, + .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom), + .larbid_remap = {{1}, {3}, + {22, MTK_INVALID_LARBID, MTK_INVALID_LARBID, MTK_INVALID_LARBID, 23}, + {8}, {20}, {12}, + /* 16: 16a; 29: 16b; 30: CCUtop0; 31: CCUtop1 */ + {14, 16, 29, 26, 30, 31, 18}, + {4, MTK_INVALID_LARBID, MTK_INVALID_LARBID, MTK_INVALID_LARBID, 6}}, +}; + static const struct of_device_id mtk_iommu_of_ids[] = { { .compatible = "mediatek,mt2712-m4u", .data = &mt2712_data}, { .compatible = "mediatek,mt6779-m4u", .data = &mt6779_data}, @@ -1240,6 +1278,9 @@ static const struct of_device_id mtk_iommu_of_ids[] = { { .compatible = "mediatek,mt8173-m4u", .data = &mt8173_data}, { .compatible = "mediatek,mt8183-m4u", .data = &mt8183_data}, { .compatible = "mediatek,mt8192-m4u", .data = &mt8192_data}, + { .compatible = "mediatek,mt8195-iommu-infra", .data = &mt8195_data_infra}, + { .compatible = "mediatek,mt8195-iommu-vdo", .data = &mt8195_data_vdo}, + { .compatible = "mediatek,mt8195-iommu-vpp", .data = &mt8195_data_vpp}, {} }; diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h index 56838fad8c73..f2ee11cd254a 100644 --- a/drivers/iommu/mtk_iommu.h +++ b/drivers/iommu/mtk_iommu.h @@ -46,6 +46,7 @@ enum mtk_iommu_plat { M4U_MT8173, M4U_MT8183, M4U_MT8192, + M4U_MT8195, }; struct mtk_iommu_iova_region; -- cgit v1.2.3 From 887cf6a74a2f7225300186ff0896aa93ce42f969 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:14:15 +0800 Subject: iommu/mediatek: Only adjust code about register base No functional change. Use "base" instead of the data->base. This is avoid to touch too many lines in the next patches. Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-25-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 51 +++++++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 24 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 47355010502e..de448d056135 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -227,12 +227,12 @@ static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom) static void mtk_iommu_tlb_flush_all(struct mtk_iommu_data *data) { + void __iomem *base = data->base; unsigned long flags; spin_lock_irqsave(&data->tlb_lock, flags); - writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, - data->base + data->plat_data->inv_sel_reg); - writel_relaxed(F_ALL_INVLD, data->base + REG_MMU_INVALIDATE); + writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, base + data->plat_data->inv_sel_reg); + writel_relaxed(F_ALL_INVLD, base + REG_MMU_INVALIDATE); wmb(); /* Make sure the tlb flush all done */ spin_unlock_irqrestore(&data->tlb_lock, flags); } @@ -243,6 +243,7 @@ static void mtk_iommu_tlb_flush_range_sync(unsigned long iova, size_t size, struct list_head *head = data->hw_list; bool check_pm_status; unsigned long flags; + void __iomem *base; int ret; u32 tmp; @@ -269,23 +270,23 @@ static void mtk_iommu_tlb_flush_range_sync(unsigned long iova, size_t size, continue; } + base = data->base; + spin_lock_irqsave(&data->tlb_lock, flags); writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, - data->base + data->plat_data->inv_sel_reg); + base + data->plat_data->inv_sel_reg); - writel_relaxed(MTK_IOMMU_TLB_ADDR(iova), - data->base + REG_MMU_INVLD_START_A); + writel_relaxed(MTK_IOMMU_TLB_ADDR(iova), base + REG_MMU_INVLD_START_A); writel_relaxed(MTK_IOMMU_TLB_ADDR(iova + size - 1), - data->base + REG_MMU_INVLD_END_A); - writel_relaxed(F_MMU_INV_RANGE, - data->base + REG_MMU_INVALIDATE); + base + REG_MMU_INVLD_END_A); + writel_relaxed(F_MMU_INV_RANGE, base + REG_MMU_INVALIDATE); /* tlb sync */ - ret = readl_poll_timeout_atomic(data->base + REG_MMU_CPE_DONE, + ret = readl_poll_timeout_atomic(base + REG_MMU_CPE_DONE, tmp, tmp != 0, 10, 1000); /* Clear the CPE status */ - writel_relaxed(0, data->base + REG_MMU_CPE_DONE); + writel_relaxed(0, base + REG_MMU_CPE_DONE); spin_unlock_irqrestore(&data->tlb_lock, flags); if (ret) { @@ -305,23 +306,25 @@ static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) struct mtk_iommu_domain *dom = data->m4u_dom; unsigned int fault_larb = MTK_INVALID_LARBID, fault_port = 0, sub_comm = 0; u32 int_state, regval, va34_32, pa34_32; + const struct mtk_iommu_plat_data *plat_data = data->plat_data; + void __iomem *base = data->base; u64 fault_iova, fault_pa; bool layer, write; /* Read error info from registers */ - int_state = readl_relaxed(data->base + REG_MMU_FAULT_ST1); + int_state = readl_relaxed(base + REG_MMU_FAULT_ST1); if (int_state & F_REG_MMU0_FAULT_MASK) { - regval = readl_relaxed(data->base + REG_MMU0_INT_ID); - fault_iova = readl_relaxed(data->base + REG_MMU0_FAULT_VA); - fault_pa = readl_relaxed(data->base + REG_MMU0_INVLD_PA); + regval = readl_relaxed(base + REG_MMU0_INT_ID); + fault_iova = readl_relaxed(base + REG_MMU0_FAULT_VA); + fault_pa = readl_relaxed(base + REG_MMU0_INVLD_PA); } else { - regval = readl_relaxed(data->base + REG_MMU1_INT_ID); - fault_iova = readl_relaxed(data->base + REG_MMU1_FAULT_VA); - fault_pa = readl_relaxed(data->base + REG_MMU1_INVLD_PA); + regval = readl_relaxed(base + REG_MMU1_INT_ID); + fault_iova = readl_relaxed(base + REG_MMU1_FAULT_VA); + fault_pa = readl_relaxed(base + REG_MMU1_INVLD_PA); } layer = fault_iova & F_MMU_FAULT_VA_LAYER_BIT; write = fault_iova & F_MMU_FAULT_VA_WRITE_BIT; - if (MTK_IOMMU_HAS_FLAG(data->plat_data, IOVA_34_EN)) { + if (MTK_IOMMU_HAS_FLAG(plat_data, IOVA_34_EN)) { va34_32 = FIELD_GET(F_MMU_INVAL_VA_34_32_MASK, fault_iova); fault_iova = fault_iova & F_MMU_INVAL_VA_31_12_MASK; fault_iova |= (u64)va34_32 << 32; @@ -329,12 +332,12 @@ static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) pa34_32 = FIELD_GET(F_MMU_INVAL_PA_34_32_MASK, fault_iova); fault_pa |= (u64)pa34_32 << 32; - if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { + if (MTK_IOMMU_IS_TYPE(plat_data, MTK_IOMMU_TYPE_MM)) { fault_port = F_MMU_INT_ID_PORT_ID(regval); - if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_SUB_COMM_2BITS)) { + if (MTK_IOMMU_HAS_FLAG(plat_data, HAS_SUB_COMM_2BITS)) { fault_larb = F_MMU_INT_ID_COMM_ID(regval); sub_comm = F_MMU_INT_ID_SUB_COMM_ID(regval); - } else if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_SUB_COMM_3BITS)) { + } else if (MTK_IOMMU_HAS_FLAG(plat_data, HAS_SUB_COMM_3BITS)) { fault_larb = F_MMU_INT_ID_COMM_ID_EXT(regval); sub_comm = F_MMU_INT_ID_SUB_COMM_ID_EXT(regval); } else { @@ -353,9 +356,9 @@ static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) } /* Interrupt clear */ - regval = readl_relaxed(data->base + REG_MMU_INT_CONTROL0); + regval = readl_relaxed(base + REG_MMU_INT_CONTROL0); regval |= F_INT_CLR_BIT; - writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL0); + writel_relaxed(regval, base + REG_MMU_INT_CONTROL0); mtk_iommu_tlb_flush_all(data); -- cgit v1.2.3 From 634f57df1f3b61cc1b1441762718a14da28c27f8 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:14:16 +0800 Subject: iommu/mediatek: Just move code position in hw_init No functional change too, prepare for mt8195 IOMMU support bank functions. Some global control settings are in bank0 while the other banks have their bank independent setting. Here only move the global control settings and the independent registers together. Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-26-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 48 +++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index de448d056135..ac1681858af8 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -803,30 +803,6 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) } writel_relaxed(regval, data->base + REG_MMU_CTRL_REG); - regval = F_L2_MULIT_HIT_EN | - F_TABLE_WALK_FAULT_INT_EN | - F_PREETCH_FIFO_OVERFLOW_INT_EN | - F_MISS_FIFO_OVERFLOW_INT_EN | - F_PREFETCH_FIFO_ERR_INT_EN | - F_MISS_FIFO_ERR_INT_EN; - writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL0); - - regval = F_INT_TRANSLATION_FAULT | - F_INT_MAIN_MULTI_HIT_FAULT | - F_INT_INVALID_PA_FAULT | - F_INT_ENTRY_REPLACEMENT_FAULT | - F_INT_TLB_MISS_FAULT | - F_INT_MISS_TRANSACTION_FIFO_FAULT | - F_INT_PRETETCH_TRANSATION_FIFO_FAULT; - writel_relaxed(regval, data->base + REG_MMU_INT_MAIN_CONTROL); - - if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_LEGACY_IVRP_PADDR)) - regval = (data->protect_base >> 1) | (data->enable_4GB << 31); - else - regval = lower_32_bits(data->protect_base) | - upper_32_bits(data->protect_base); - writel_relaxed(regval, data->base + REG_MMU_IVRP_PADDR); - if (data->enable_4GB && MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_VLD_PA_RNG)) { /* @@ -860,6 +836,30 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) } writel_relaxed(regval, data->base + REG_MMU_MISC_CTRL); + regval = F_L2_MULIT_HIT_EN | + F_TABLE_WALK_FAULT_INT_EN | + F_PREETCH_FIFO_OVERFLOW_INT_EN | + F_MISS_FIFO_OVERFLOW_INT_EN | + F_PREFETCH_FIFO_ERR_INT_EN | + F_MISS_FIFO_ERR_INT_EN; + writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL0); + + regval = F_INT_TRANSLATION_FAULT | + F_INT_MAIN_MULTI_HIT_FAULT | + F_INT_INVALID_PA_FAULT | + F_INT_ENTRY_REPLACEMENT_FAULT | + F_INT_TLB_MISS_FAULT | + F_INT_MISS_TRANSACTION_FIFO_FAULT | + F_INT_PRETETCH_TRANSATION_FIFO_FAULT; + writel_relaxed(regval, data->base + REG_MMU_INT_MAIN_CONTROL); + + if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_LEGACY_IVRP_PADDR)) + regval = (data->protect_base >> 1) | (data->enable_4GB << 31); + else + regval = lower_32_bits(data->protect_base) | + upper_32_bits(data->protect_base); + writel_relaxed(regval, data->base + REG_MMU_IVRP_PADDR); + if (devm_request_irq(data->dev, data->irq, mtk_iommu_isr, 0, dev_name(data->dev), (void *)data)) { writel_relaxed(0, data->base + REG_MMU_PT_BASE_ADDR); -- cgit v1.2.3 From 9485a04a5bb97e2b9749277005e1f7c23b9ec8be Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:14:17 +0800 Subject: iommu/mediatek: Separate mtk_iommu_data for v1 and v2 Prepare for adding the structure "mtk_iommu_bank_data". No functional change. The mtk_iommu_domain in v1 and v2 are different, we could not add current data as bank[0] in v1 simplistically. Currently we have no plan to add new SoC for v1, in order to avoid affect v1 when we add many new features for v2, I totally separate v1 and v2 in this patch, there are many structures only for v2. Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-27-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 82 +++++++++++++++++++++++++++++++++++++++++--- drivers/iommu/mtk_iommu.h | 81 ------------------------------------------- drivers/iommu/mtk_iommu_v1.c | 29 ++++++++++++++++ 3 files changed, 106 insertions(+), 86 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index ac1681858af8..7383a5df6021 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -146,6 +146,69 @@ #define MTK_INVALID_LARBID MTK_LARB_NR_MAX +#define MTK_LARB_COM_MAX 8 +#define MTK_LARB_SUBCOM_MAX 8 + +#define MTK_IOMMU_GROUP_MAX 8 + +enum mtk_iommu_plat { + M4U_MT2712, + M4U_MT6779, + M4U_MT8167, + M4U_MT8173, + M4U_MT8183, + M4U_MT8192, + M4U_MT8195, +}; + +struct mtk_iommu_iova_region { + dma_addr_t iova_base; + unsigned long long size; +}; + +struct mtk_iommu_plat_data { + enum mtk_iommu_plat m4u_plat; + u32 flags; + u32 inv_sel_reg; + + char *pericfg_comp_str; + struct list_head *hw_list; + unsigned int iova_region_nr; + const struct mtk_iommu_iova_region *iova_region; + unsigned char larbid_remap[MTK_LARB_COM_MAX][MTK_LARB_SUBCOM_MAX]; +}; + +struct mtk_iommu_data { + void __iomem *base; + int irq; + struct device *dev; + struct clk *bclk; + phys_addr_t protect_base; /* protect memory base */ + struct mtk_iommu_suspend_reg reg; + struct mtk_iommu_domain *m4u_dom; + struct iommu_group *m4u_group[MTK_IOMMU_GROUP_MAX]; + bool enable_4GB; + spinlock_t tlb_lock; /* lock for tlb range flush */ + + struct iommu_device iommu; + const struct mtk_iommu_plat_data *plat_data; + struct device *smicomm_dev; + + struct dma_iommu_mapping *mapping; /* For mtk_iommu_v1.c */ + struct regmap *pericfg; + + struct mutex mutex; /* Protect m4u_group/m4u_dom above */ + + /* + * In the sharing pgtable case, list data->list to the global list like m4ulist. + * In the non-sharing pgtable case, list data->list to the itself hw_list_head. + */ + struct list_head *hw_list; + struct list_head hw_list_head; + struct list_head list; + struct mtk_smi_larb_iommu larb_imu[MTK_LARB_NR_MAX]; +}; + struct mtk_iommu_domain { struct io_pgtable_cfg cfg; struct io_pgtable_ops *iop; @@ -156,6 +219,20 @@ struct mtk_iommu_domain { struct mutex mutex; /* Protect "data" in this structure */ }; +static int mtk_iommu_bind(struct device *dev) +{ + struct mtk_iommu_data *data = dev_get_drvdata(dev); + + return component_bind_all(dev, &data->larb_imu); +} + +static void mtk_iommu_unbind(struct device *dev) +{ + struct mtk_iommu_data *data = dev_get_drvdata(dev); + + component_unbind_all(dev, &data->larb_imu); +} + static const struct iommu_ops mtk_iommu_ops; static int mtk_iommu_hw_init(const struct mtk_iommu_data *data); @@ -193,11 +270,6 @@ static LIST_HEAD(m4ulist); /* List all the M4U HWs */ #define for_each_m4u(data, head) list_for_each_entry(data, head, list) -struct mtk_iommu_iova_region { - dma_addr_t iova_base; - unsigned long long size; -}; - static const struct mtk_iommu_iova_region single_domain[] = { {.iova_base = 0, .size = SZ_4G}, }; diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h index f2ee11cd254a..305243e18aa9 100644 --- a/drivers/iommu/mtk_iommu.h +++ b/drivers/iommu/mtk_iommu.h @@ -7,23 +7,14 @@ #ifndef _MTK_IOMMU_H_ #define _MTK_IOMMU_H_ -#include -#include #include #include #include #include -#include #include -#include #include #include -#define MTK_LARB_COM_MAX 8 -#define MTK_LARB_SUBCOM_MAX 8 - -#define MTK_IOMMU_GROUP_MAX 8 - struct mtk_iommu_suspend_reg { union { u32 standard_axi_mode;/* v1 */ @@ -38,76 +29,4 @@ struct mtk_iommu_suspend_reg { u32 wr_len_ctrl; }; -enum mtk_iommu_plat { - M4U_MT2701, - M4U_MT2712, - M4U_MT6779, - M4U_MT8167, - M4U_MT8173, - M4U_MT8183, - M4U_MT8192, - M4U_MT8195, -}; - -struct mtk_iommu_iova_region; - -struct mtk_iommu_plat_data { - enum mtk_iommu_plat m4u_plat; - u32 flags; - u32 inv_sel_reg; - - char *pericfg_comp_str; - struct list_head *hw_list; - unsigned int iova_region_nr; - const struct mtk_iommu_iova_region *iova_region; - unsigned char larbid_remap[MTK_LARB_COM_MAX][MTK_LARB_SUBCOM_MAX]; -}; - -struct mtk_iommu_domain; - -struct mtk_iommu_data { - void __iomem *base; - int irq; - struct device *dev; - struct clk *bclk; - phys_addr_t protect_base; /* protect memory base */ - struct mtk_iommu_suspend_reg reg; - struct mtk_iommu_domain *m4u_dom; - struct iommu_group *m4u_group[MTK_IOMMU_GROUP_MAX]; - bool enable_4GB; - spinlock_t tlb_lock; /* lock for tlb range flush */ - - struct iommu_device iommu; - const struct mtk_iommu_plat_data *plat_data; - struct device *smicomm_dev; - - struct dma_iommu_mapping *mapping; /* For mtk_iommu_v1.c */ - struct regmap *pericfg; - - struct mutex mutex; /* Protect m4u_group/m4u_dom above */ - - /* - * In the sharing pgtable case, list data->list to the global list like m4ulist. - * In the non-sharing pgtable case, list data->list to the itself hw_list_head. - */ - struct list_head *hw_list; - struct list_head hw_list_head; - struct list_head list; - struct mtk_smi_larb_iommu larb_imu[MTK_LARB_NR_MAX]; -}; - -static inline int mtk_iommu_bind(struct device *dev) -{ - struct mtk_iommu_data *data = dev_get_drvdata(dev); - - return component_bind_all(dev, &data->larb_imu); -} - -static inline void mtk_iommu_unbind(struct device *dev) -{ - struct mtk_iommu_data *data = dev_get_drvdata(dev); - - component_unbind_all(dev, &data->larb_imu); -} - #endif diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c index ecff800656e6..6d1c09c91e1f 100644 --- a/drivers/iommu/mtk_iommu_v1.c +++ b/drivers/iommu/mtk_iommu_v1.c @@ -87,6 +87,21 @@ */ #define M2701_IOMMU_PGT_SIZE SZ_4M +struct mtk_iommu_data { + void __iomem *base; + int irq; + struct device *dev; + struct clk *bclk; + phys_addr_t protect_base; /* protect memory base */ + struct mtk_iommu_domain *m4u_dom; + + struct iommu_device iommu; + struct dma_iommu_mapping *mapping; + struct mtk_smi_larb_iommu larb_imu[MTK_LARB_NR_MAX]; + + struct mtk_iommu_suspend_reg reg; +}; + struct mtk_iommu_domain { spinlock_t pgtlock; /* lock for page table */ struct iommu_domain domain; @@ -95,6 +110,20 @@ struct mtk_iommu_domain { struct mtk_iommu_data *data; }; +static int mtk_iommu_bind(struct device *dev) +{ + struct mtk_iommu_data *data = dev_get_drvdata(dev); + + return component_bind_all(dev, &data->larb_imu); +} + +static void mtk_iommu_unbind(struct device *dev) +{ + struct mtk_iommu_data *data = dev_get_drvdata(dev); + + component_unbind_all(dev, &data->larb_imu); +} + static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom) { return container_of(dom, struct mtk_iommu_domain, domain); -- cgit v1.2.3 From 6a513de3efe099976fc5924082e48b79c467664f Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:14:18 +0800 Subject: iommu/mediatek: Remove mtk_iommu.h Currently there is a suspend structure in the header file. It's no need to keep a header file only for this. Move these into the c file and rm this header file. Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-28-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 14 +++++++++++++- drivers/iommu/mtk_iommu.h | 32 -------------------------------- drivers/iommu/mtk_iommu_v1.c | 11 ++++++++--- 3 files changed, 21 insertions(+), 36 deletions(-) delete mode 100644 drivers/iommu/mtk_iommu.h (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 7383a5df6021..349640bcbd01 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -30,7 +31,7 @@ #include #include -#include "mtk_iommu.h" +#include #define REG_MMU_PT_BASE_ADDR 0x000 #define MMU_PT_ADDR_MASK GENMASK(31, 7) @@ -166,6 +167,17 @@ struct mtk_iommu_iova_region { unsigned long long size; }; +struct mtk_iommu_suspend_reg { + u32 misc_ctrl; + u32 dcm_dis; + u32 ctrl_reg; + u32 int_control0; + u32 int_main_control; + u32 ivrp_paddr; + u32 vld_pa_rng; + u32 wr_len_ctrl; +}; + struct mtk_iommu_plat_data { enum mtk_iommu_plat m4u_plat; u32 flags; diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h deleted file mode 100644 index 305243e18aa9..000000000000 --- a/drivers/iommu/mtk_iommu.h +++ /dev/null @@ -1,32 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (c) 2015-2016 MediaTek Inc. - * Author: Honghui Zhang - */ - -#ifndef _MTK_IOMMU_H_ -#define _MTK_IOMMU_H_ - -#include -#include -#include -#include -#include -#include -#include - -struct mtk_iommu_suspend_reg { - union { - u32 standard_axi_mode;/* v1 */ - u32 misc_ctrl;/* v2 */ - }; - u32 dcm_dis; - u32 ctrl_reg; - u32 int_control0; - u32 int_main_control; - u32 ivrp_paddr; - u32 vld_pa_rng; - u32 wr_len_ctrl; -}; - -#endif diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c index 6d1c09c91e1f..3d1f0897d1cc 100644 --- a/drivers/iommu/mtk_iommu_v1.c +++ b/drivers/iommu/mtk_iommu_v1.c @@ -7,7 +7,6 @@ * * Based on driver/iommu/mtk_iommu.c */ -#include #include #include #include @@ -28,10 +27,9 @@ #include #include #include -#include +#include #include #include -#include "mtk_iommu.h" #define REG_MMU_PT_BASE_ADDR 0x000 @@ -87,6 +85,13 @@ */ #define M2701_IOMMU_PGT_SIZE SZ_4M +struct mtk_iommu_suspend_reg { + u32 standard_axi_mode; + u32 dcm_dis; + u32 ctrl_reg; + u32 int_control0; +}; + struct mtk_iommu_data { void __iomem *base; int irq; -- cgit v1.2.3 From ad9b10e533f6fb3fa74afdc2341774b1305c176f Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:14:19 +0800 Subject: iommu/mediatek-v1: Just rename mtk_iommu to mtk_iommu_v1 No functional change. Just rename this for readable. Differentiate this from mtk_iommu.c Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-29-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu_v1.c | 211 +++++++++++++++++++++---------------------- 1 file changed, 103 insertions(+), 108 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c index 3d1f0897d1cc..62669e60991f 100644 --- a/drivers/iommu/mtk_iommu_v1.c +++ b/drivers/iommu/mtk_iommu_v1.c @@ -85,53 +85,53 @@ */ #define M2701_IOMMU_PGT_SIZE SZ_4M -struct mtk_iommu_suspend_reg { +struct mtk_iommu_v1_suspend_reg { u32 standard_axi_mode; u32 dcm_dis; u32 ctrl_reg; u32 int_control0; }; -struct mtk_iommu_data { +struct mtk_iommu_v1_data { void __iomem *base; int irq; struct device *dev; struct clk *bclk; phys_addr_t protect_base; /* protect memory base */ - struct mtk_iommu_domain *m4u_dom; + struct mtk_iommu_v1_domain *m4u_dom; struct iommu_device iommu; struct dma_iommu_mapping *mapping; struct mtk_smi_larb_iommu larb_imu[MTK_LARB_NR_MAX]; - struct mtk_iommu_suspend_reg reg; + struct mtk_iommu_v1_suspend_reg reg; }; -struct mtk_iommu_domain { +struct mtk_iommu_v1_domain { spinlock_t pgtlock; /* lock for page table */ struct iommu_domain domain; u32 *pgt_va; dma_addr_t pgt_pa; - struct mtk_iommu_data *data; + struct mtk_iommu_v1_data *data; }; -static int mtk_iommu_bind(struct device *dev) +static int mtk_iommu_v1_bind(struct device *dev) { - struct mtk_iommu_data *data = dev_get_drvdata(dev); + struct mtk_iommu_v1_data *data = dev_get_drvdata(dev); return component_bind_all(dev, &data->larb_imu); } -static void mtk_iommu_unbind(struct device *dev) +static void mtk_iommu_v1_unbind(struct device *dev) { - struct mtk_iommu_data *data = dev_get_drvdata(dev); + struct mtk_iommu_v1_data *data = dev_get_drvdata(dev); component_unbind_all(dev, &data->larb_imu); } -static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom) +static struct mtk_iommu_v1_domain *to_mtk_domain(struct iommu_domain *dom) { - return container_of(dom, struct mtk_iommu_domain, domain); + return container_of(dom, struct mtk_iommu_v1_domain, domain); } static const int mt2701_m4u_in_larb[] = { @@ -157,7 +157,7 @@ static inline int mt2701_m4u_to_port(int id) return id - mt2701_m4u_in_larb[larb]; } -static void mtk_iommu_tlb_flush_all(struct mtk_iommu_data *data) +static void mtk_iommu_v1_tlb_flush_all(struct mtk_iommu_v1_data *data) { writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, data->base + REG_MMU_INV_SEL); @@ -165,8 +165,8 @@ static void mtk_iommu_tlb_flush_all(struct mtk_iommu_data *data) wmb(); /* Make sure the tlb flush all done */ } -static void mtk_iommu_tlb_flush_range(struct mtk_iommu_data *data, - unsigned long iova, size_t size) +static void mtk_iommu_v1_tlb_flush_range(struct mtk_iommu_v1_data *data, + unsigned long iova, size_t size) { int ret; u32 tmp; @@ -184,16 +184,16 @@ static void mtk_iommu_tlb_flush_range(struct mtk_iommu_data *data, if (ret) { dev_warn(data->dev, "Partial TLB flush timed out, falling back to full flush\n"); - mtk_iommu_tlb_flush_all(data); + mtk_iommu_v1_tlb_flush_all(data); } /* Clear the CPE status */ writel_relaxed(0, data->base + REG_MMU_CPE_DONE); } -static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) +static irqreturn_t mtk_iommu_v1_isr(int irq, void *dev_id) { - struct mtk_iommu_data *data = dev_id; - struct mtk_iommu_domain *dom = data->m4u_dom; + struct mtk_iommu_v1_data *data = dev_id; + struct mtk_iommu_v1_domain *dom = data->m4u_dom; u32 int_state, regval, fault_iova, fault_pa; unsigned int fault_larb, fault_port; @@ -223,13 +223,13 @@ static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) regval |= F_INT_CLR_BIT; writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL); - mtk_iommu_tlb_flush_all(data); + mtk_iommu_v1_tlb_flush_all(data); return IRQ_HANDLED; } -static void mtk_iommu_config(struct mtk_iommu_data *data, - struct device *dev, bool enable) +static void mtk_iommu_v1_config(struct mtk_iommu_v1_data *data, + struct device *dev, bool enable) { struct mtk_smi_larb_iommu *larb_mmu; unsigned int larbid, portid; @@ -251,9 +251,9 @@ static void mtk_iommu_config(struct mtk_iommu_data *data, } } -static int mtk_iommu_domain_finalise(struct mtk_iommu_data *data) +static int mtk_iommu_v1_domain_finalise(struct mtk_iommu_v1_data *data) { - struct mtk_iommu_domain *dom = data->m4u_dom; + struct mtk_iommu_v1_domain *dom = data->m4u_dom; spin_lock_init(&dom->pgtlock); @@ -269,9 +269,9 @@ static int mtk_iommu_domain_finalise(struct mtk_iommu_data *data) return 0; } -static struct iommu_domain *mtk_iommu_domain_alloc(unsigned type) +static struct iommu_domain *mtk_iommu_v1_domain_alloc(unsigned type) { - struct mtk_iommu_domain *dom; + struct mtk_iommu_v1_domain *dom; if (type != IOMMU_DOMAIN_UNMANAGED) return NULL; @@ -283,21 +283,20 @@ static struct iommu_domain *mtk_iommu_domain_alloc(unsigned type) return &dom->domain; } -static void mtk_iommu_domain_free(struct iommu_domain *domain) +static void mtk_iommu_v1_domain_free(struct iommu_domain *domain) { - struct mtk_iommu_domain *dom = to_mtk_domain(domain); - struct mtk_iommu_data *data = dom->data; + struct mtk_iommu_v1_domain *dom = to_mtk_domain(domain); + struct mtk_iommu_v1_data *data = dom->data; dma_free_coherent(data->dev, M2701_IOMMU_PGT_SIZE, dom->pgt_va, dom->pgt_pa); kfree(to_mtk_domain(domain)); } -static int mtk_iommu_attach_device(struct iommu_domain *domain, - struct device *dev) +static int mtk_iommu_v1_attach_device(struct iommu_domain *domain, struct device *dev) { - struct mtk_iommu_data *data = dev_iommu_priv_get(dev); - struct mtk_iommu_domain *dom = to_mtk_domain(domain); + struct mtk_iommu_v1_data *data = dev_iommu_priv_get(dev); + struct mtk_iommu_v1_domain *dom = to_mtk_domain(domain); struct dma_iommu_mapping *mtk_mapping; int ret; @@ -308,29 +307,28 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, if (!data->m4u_dom) { data->m4u_dom = dom; - ret = mtk_iommu_domain_finalise(data); + ret = mtk_iommu_v1_domain_finalise(data); if (ret) { data->m4u_dom = NULL; return ret; } } - mtk_iommu_config(data, dev, true); + mtk_iommu_v1_config(data, dev, true); return 0; } -static void mtk_iommu_detach_device(struct iommu_domain *domain, - struct device *dev) +static void mtk_iommu_v1_detach_device(struct iommu_domain *domain, struct device *dev) { - struct mtk_iommu_data *data = dev_iommu_priv_get(dev); + struct mtk_iommu_v1_data *data = dev_iommu_priv_get(dev); - mtk_iommu_config(data, dev, false); + mtk_iommu_v1_config(data, dev, false); } -static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova, - phys_addr_t paddr, size_t size, int prot, gfp_t gfp) +static int mtk_iommu_v1_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t paddr, size_t size, int prot, gfp_t gfp) { - struct mtk_iommu_domain *dom = to_mtk_domain(domain); + struct mtk_iommu_v1_domain *dom = to_mtk_domain(domain); unsigned int page_num = size >> MT2701_IOMMU_PAGE_SHIFT; unsigned long flags; unsigned int i; @@ -351,16 +349,15 @@ static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova, spin_unlock_irqrestore(&dom->pgtlock, flags); - mtk_iommu_tlb_flush_range(dom->data, iova, size); + mtk_iommu_v1_tlb_flush_range(dom->data, iova, size); return map_size == size ? 0 : -EEXIST; } -static size_t mtk_iommu_unmap(struct iommu_domain *domain, - unsigned long iova, size_t size, - struct iommu_iotlb_gather *gather) +static size_t mtk_iommu_v1_unmap(struct iommu_domain *domain, unsigned long iova, + size_t size, struct iommu_iotlb_gather *gather) { - struct mtk_iommu_domain *dom = to_mtk_domain(domain); + struct mtk_iommu_v1_domain *dom = to_mtk_domain(domain); unsigned long flags; u32 *pgt_base_iova = dom->pgt_va + (iova >> MT2701_IOMMU_PAGE_SHIFT); unsigned int page_num = size >> MT2701_IOMMU_PAGE_SHIFT; @@ -369,15 +366,14 @@ static size_t mtk_iommu_unmap(struct iommu_domain *domain, memset(pgt_base_iova, 0, page_num * sizeof(u32)); spin_unlock_irqrestore(&dom->pgtlock, flags); - mtk_iommu_tlb_flush_range(dom->data, iova, size); + mtk_iommu_v1_tlb_flush_range(dom->data, iova, size); return size; } -static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain, - dma_addr_t iova) +static phys_addr_t mtk_iommu_v1_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) { - struct mtk_iommu_domain *dom = to_mtk_domain(domain); + struct mtk_iommu_v1_domain *dom = to_mtk_domain(domain); unsigned long flags; phys_addr_t pa; @@ -389,17 +385,16 @@ static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain, return pa; } -static const struct iommu_ops mtk_iommu_ops; +static const struct iommu_ops mtk_iommu_v1_ops; /* * MTK generation one iommu HW only support one iommu domain, and all the client * sharing the same iova address space. */ -static int mtk_iommu_create_mapping(struct device *dev, - struct of_phandle_args *args) +static int mtk_iommu_v1_create_mapping(struct device *dev, struct of_phandle_args *args) { struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); - struct mtk_iommu_data *data; + struct mtk_iommu_v1_data *data; struct platform_device *m4updev; struct dma_iommu_mapping *mtk_mapping; int ret; @@ -411,11 +406,11 @@ static int mtk_iommu_create_mapping(struct device *dev, } if (!fwspec) { - ret = iommu_fwspec_init(dev, &args->np->fwnode, &mtk_iommu_ops); + ret = iommu_fwspec_init(dev, &args->np->fwnode, &mtk_iommu_v1_ops); if (ret) return ret; fwspec = dev_iommu_fwspec_get(dev); - } else if (dev_iommu_fwspec_get(dev)->ops != &mtk_iommu_ops) { + } else if (dev_iommu_fwspec_get(dev)->ops != &mtk_iommu_v1_ops) { return -EINVAL; } @@ -447,16 +442,16 @@ static int mtk_iommu_create_mapping(struct device *dev, return 0; } -static int mtk_iommu_def_domain_type(struct device *dev) +static int mtk_iommu_v1_def_domain_type(struct device *dev) { return IOMMU_DOMAIN_UNMANAGED; } -static struct iommu_device *mtk_iommu_probe_device(struct device *dev) +static struct iommu_device *mtk_iommu_v1_probe_device(struct device *dev) { struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct of_phandle_args iommu_spec; - struct mtk_iommu_data *data; + struct mtk_iommu_v1_data *data; int err, idx = 0, larbid, larbidx; struct device_link *link; struct device *larbdev; @@ -474,7 +469,7 @@ static struct iommu_device *mtk_iommu_probe_device(struct device *dev) "#iommu-cells", idx, &iommu_spec)) { - err = mtk_iommu_create_mapping(dev, &iommu_spec); + err = mtk_iommu_v1_create_mapping(dev, &iommu_spec); of_node_put(iommu_spec.np); if (err) return ERR_PTR(err); @@ -484,7 +479,7 @@ static struct iommu_device *mtk_iommu_probe_device(struct device *dev) idx++; } - if (!fwspec || fwspec->ops != &mtk_iommu_ops) + if (!fwspec || fwspec->ops != &mtk_iommu_v1_ops) return ERR_PTR(-ENODEV); /* Not a iommu client device */ data = dev_iommu_priv_get(dev); @@ -509,10 +504,10 @@ static struct iommu_device *mtk_iommu_probe_device(struct device *dev) return &data->iommu; } -static void mtk_iommu_probe_finalize(struct device *dev) +static void mtk_iommu_v1_probe_finalize(struct device *dev) { struct dma_iommu_mapping *mtk_mapping; - struct mtk_iommu_data *data; + struct mtk_iommu_v1_data *data; int err; data = dev_iommu_priv_get(dev); @@ -523,14 +518,14 @@ static void mtk_iommu_probe_finalize(struct device *dev) dev_err(dev, "Can't create IOMMU mapping - DMA-OPS will not work\n"); } -static void mtk_iommu_release_device(struct device *dev) +static void mtk_iommu_v1_release_device(struct device *dev) { struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); - struct mtk_iommu_data *data; + struct mtk_iommu_v1_data *data; struct device *larbdev; unsigned int larbid; - if (!fwspec || fwspec->ops != &mtk_iommu_ops) + if (!fwspec || fwspec->ops != &mtk_iommu_v1_ops) return; data = dev_iommu_priv_get(dev); @@ -541,7 +536,7 @@ static void mtk_iommu_release_device(struct device *dev) iommu_fwspec_free(dev); } -static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) +static int mtk_iommu_v1_hw_init(const struct mtk_iommu_v1_data *data) { u32 regval; int ret; @@ -571,7 +566,7 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) writel_relaxed(F_MMU_DCM_ON, data->base + REG_MMU_DCM); - if (devm_request_irq(data->dev, data->irq, mtk_iommu_isr, 0, + if (devm_request_irq(data->dev, data->irq, mtk_iommu_v1_isr, 0, dev_name(data->dev), (void *)data)) { writel_relaxed(0, data->base + REG_MMU_PT_BASE_ADDR); clk_disable_unprepare(data->bclk); @@ -582,39 +577,39 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) return 0; } -static const struct iommu_ops mtk_iommu_ops = { - .domain_alloc = mtk_iommu_domain_alloc, - .probe_device = mtk_iommu_probe_device, - .probe_finalize = mtk_iommu_probe_finalize, - .release_device = mtk_iommu_release_device, - .def_domain_type = mtk_iommu_def_domain_type, +static const struct iommu_ops mtk_iommu_v1_ops = { + .domain_alloc = mtk_iommu_v1_domain_alloc, + .probe_device = mtk_iommu_v1_probe_device, + .probe_finalize = mtk_iommu_v1_probe_finalize, + .release_device = mtk_iommu_v1_release_device, + .def_domain_type = mtk_iommu_v1_def_domain_type, .device_group = generic_device_group, .pgsize_bitmap = ~0UL << MT2701_IOMMU_PAGE_SHIFT, .owner = THIS_MODULE, .default_domain_ops = &(const struct iommu_domain_ops) { - .attach_dev = mtk_iommu_attach_device, - .detach_dev = mtk_iommu_detach_device, - .map = mtk_iommu_map, - .unmap = mtk_iommu_unmap, - .iova_to_phys = mtk_iommu_iova_to_phys, - .free = mtk_iommu_domain_free, + .attach_dev = mtk_iommu_v1_attach_device, + .detach_dev = mtk_iommu_v1_detach_device, + .map = mtk_iommu_v1_map, + .unmap = mtk_iommu_v1_unmap, + .iova_to_phys = mtk_iommu_v1_iova_to_phys, + .free = mtk_iommu_v1_domain_free, } }; -static const struct of_device_id mtk_iommu_of_ids[] = { +static const struct of_device_id mtk_iommu_v1_of_ids[] = { { .compatible = "mediatek,mt2701-m4u", }, {} }; -static const struct component_master_ops mtk_iommu_com_ops = { - .bind = mtk_iommu_bind, - .unbind = mtk_iommu_unbind, +static const struct component_master_ops mtk_iommu_v1_com_ops = { + .bind = mtk_iommu_v1_bind, + .unbind = mtk_iommu_v1_unbind, }; -static int mtk_iommu_probe(struct platform_device *pdev) +static int mtk_iommu_v1_probe(struct platform_device *pdev) { - struct mtk_iommu_data *data; struct device *dev = &pdev->dev; + struct mtk_iommu_v1_data *data; struct resource *res; struct component_match *match = NULL; void *protect; @@ -681,7 +676,7 @@ static int mtk_iommu_probe(struct platform_device *pdev) platform_set_drvdata(pdev, data); - ret = mtk_iommu_hw_init(data); + ret = mtk_iommu_v1_hw_init(data); if (ret) return ret; @@ -690,17 +685,17 @@ static int mtk_iommu_probe(struct platform_device *pdev) if (ret) return ret; - ret = iommu_device_register(&data->iommu, &mtk_iommu_ops, dev); + ret = iommu_device_register(&data->iommu, &mtk_iommu_v1_ops, dev); if (ret) goto out_sysfs_remove; if (!iommu_present(&platform_bus_type)) { - ret = bus_set_iommu(&platform_bus_type, &mtk_iommu_ops); + ret = bus_set_iommu(&platform_bus_type, &mtk_iommu_v1_ops); if (ret) goto out_dev_unreg; } - ret = component_master_add_with_match(dev, &mtk_iommu_com_ops, match); + ret = component_master_add_with_match(dev, &mtk_iommu_v1_com_ops, match); if (ret) goto out_bus_set_null; return ret; @@ -714,9 +709,9 @@ out_sysfs_remove: return ret; } -static int mtk_iommu_remove(struct platform_device *pdev) +static int mtk_iommu_v1_remove(struct platform_device *pdev) { - struct mtk_iommu_data *data = platform_get_drvdata(pdev); + struct mtk_iommu_v1_data *data = platform_get_drvdata(pdev); iommu_device_sysfs_remove(&data->iommu); iommu_device_unregister(&data->iommu); @@ -726,14 +721,14 @@ static int mtk_iommu_remove(struct platform_device *pdev) clk_disable_unprepare(data->bclk); devm_free_irq(&pdev->dev, data->irq, data); - component_master_del(&pdev->dev, &mtk_iommu_com_ops); + component_master_del(&pdev->dev, &mtk_iommu_v1_com_ops); return 0; } -static int __maybe_unused mtk_iommu_suspend(struct device *dev) +static int __maybe_unused mtk_iommu_v1_suspend(struct device *dev) { - struct mtk_iommu_data *data = dev_get_drvdata(dev); - struct mtk_iommu_suspend_reg *reg = &data->reg; + struct mtk_iommu_v1_data *data = dev_get_drvdata(dev); + struct mtk_iommu_v1_suspend_reg *reg = &data->reg; void __iomem *base = data->base; reg->standard_axi_mode = readl_relaxed(base + @@ -744,10 +739,10 @@ static int __maybe_unused mtk_iommu_suspend(struct device *dev) return 0; } -static int __maybe_unused mtk_iommu_resume(struct device *dev) +static int __maybe_unused mtk_iommu_v1_resume(struct device *dev) { - struct mtk_iommu_data *data = dev_get_drvdata(dev); - struct mtk_iommu_suspend_reg *reg = &data->reg; + struct mtk_iommu_v1_data *data = dev_get_drvdata(dev); + struct mtk_iommu_v1_suspend_reg *reg = &data->reg; void __iomem *base = data->base; writel_relaxed(data->m4u_dom->pgt_pa, base + REG_MMU_PT_BASE_ADDR); @@ -760,20 +755,20 @@ static int __maybe_unused mtk_iommu_resume(struct device *dev) return 0; } -static const struct dev_pm_ops mtk_iommu_pm_ops = { - SET_SYSTEM_SLEEP_PM_OPS(mtk_iommu_suspend, mtk_iommu_resume) +static const struct dev_pm_ops mtk_iommu_v1_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(mtk_iommu_v1_suspend, mtk_iommu_v1_resume) }; -static struct platform_driver mtk_iommu_driver = { - .probe = mtk_iommu_probe, - .remove = mtk_iommu_remove, +static struct platform_driver mtk_iommu_v1_driver = { + .probe = mtk_iommu_v1_probe, + .remove = mtk_iommu_v1_remove, .driver = { .name = "mtk-iommu-v1", - .of_match_table = mtk_iommu_of_ids, - .pm = &mtk_iommu_pm_ops, + .of_match_table = mtk_iommu_v1_of_ids, + .pm = &mtk_iommu_v1_pm_ops, } }; -module_platform_driver(mtk_iommu_driver); +module_platform_driver(mtk_iommu_v1_driver); MODULE_DESCRIPTION("IOMMU API for MediaTek M4U v1 implementations"); MODULE_LICENSE("GPL v2"); -- cgit v1.2.3 From 99ca02281332957e0ee7e3702c4713dfae4a6046 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:14:20 +0800 Subject: iommu/mediatek: Add mtk_iommu_bank_data structure Prepare for supporting multi-banks for the IOMMU HW, No functional change. Add a new structure(mtk_iommu_bank_data) for each a bank. Each a bank have the independent HW base/IRQ/tlb-range ops, and each a bank has its special iommu-domain(independent pgtable), thus, also move the domain information into it. In previous SoC, we have only one bank which could be treated as bank0( bankid always is 0 for the previous SoC). After adding this structure, the tlb operations and irq could use bank_data as parameter. Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-30-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 179 ++++++++++++++++++++++++++++++---------------- 1 file changed, 117 insertions(+), 62 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 349640bcbd01..968e1486e6fd 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -151,6 +151,7 @@ #define MTK_LARB_SUBCOM_MAX 8 #define MTK_IOMMU_GROUP_MAX 8 +#define MTK_IOMMU_BANK_MAX 5 enum mtk_iommu_plat { M4U_MT2712, @@ -187,25 +188,36 @@ struct mtk_iommu_plat_data { struct list_head *hw_list; unsigned int iova_region_nr; const struct mtk_iommu_iova_region *iova_region; + + u8 banks_num; + bool banks_enable[MTK_IOMMU_BANK_MAX]; unsigned char larbid_remap[MTK_LARB_COM_MAX][MTK_LARB_SUBCOM_MAX]; }; -struct mtk_iommu_data { +struct mtk_iommu_bank_data { void __iomem *base; int irq; + u8 id; + struct device *parent_dev; + struct mtk_iommu_data *parent_data; + spinlock_t tlb_lock; /* lock for tlb range flush */ + struct mtk_iommu_domain *m4u_dom; /* Each bank has a domain */ +}; + +struct mtk_iommu_data { struct device *dev; struct clk *bclk; phys_addr_t protect_base; /* protect memory base */ struct mtk_iommu_suspend_reg reg; - struct mtk_iommu_domain *m4u_dom; struct iommu_group *m4u_group[MTK_IOMMU_GROUP_MAX]; bool enable_4GB; - spinlock_t tlb_lock; /* lock for tlb range flush */ struct iommu_device iommu; const struct mtk_iommu_plat_data *plat_data; struct device *smicomm_dev; + struct mtk_iommu_bank_data *bank; + struct dma_iommu_mapping *mapping; /* For mtk_iommu_v1.c */ struct regmap *pericfg; @@ -225,7 +237,7 @@ struct mtk_iommu_domain { struct io_pgtable_cfg cfg; struct io_pgtable_ops *iop; - struct mtk_iommu_data *data; + struct mtk_iommu_bank_data *bank; struct iommu_domain domain; struct mutex mutex; /* Protect "data" in this structure */ @@ -311,20 +323,24 @@ static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom) static void mtk_iommu_tlb_flush_all(struct mtk_iommu_data *data) { - void __iomem *base = data->base; + /* Tlb flush all always is in bank0. */ + struct mtk_iommu_bank_data *bank = &data->bank[0]; + void __iomem *base = bank->base; unsigned long flags; - spin_lock_irqsave(&data->tlb_lock, flags); + spin_lock_irqsave(&bank->tlb_lock, flags); writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, base + data->plat_data->inv_sel_reg); writel_relaxed(F_ALL_INVLD, base + REG_MMU_INVALIDATE); wmb(); /* Make sure the tlb flush all done */ - spin_unlock_irqrestore(&data->tlb_lock, flags); + spin_unlock_irqrestore(&bank->tlb_lock, flags); } static void mtk_iommu_tlb_flush_range_sync(unsigned long iova, size_t size, - struct mtk_iommu_data *data) + struct mtk_iommu_bank_data *bank) { - struct list_head *head = data->hw_list; + struct list_head *head = bank->parent_data->hw_list; + struct mtk_iommu_bank_data *curbank; + struct mtk_iommu_data *data; bool check_pm_status; unsigned long flags; void __iomem *base; @@ -354,9 +370,10 @@ static void mtk_iommu_tlb_flush_range_sync(unsigned long iova, size_t size, continue; } - base = data->base; + curbank = &data->bank[bank->id]; + base = curbank->base; - spin_lock_irqsave(&data->tlb_lock, flags); + spin_lock_irqsave(&curbank->tlb_lock, flags); writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, base + data->plat_data->inv_sel_reg); @@ -371,7 +388,7 @@ static void mtk_iommu_tlb_flush_range_sync(unsigned long iova, size_t size, /* Clear the CPE status */ writel_relaxed(0, base + REG_MMU_CPE_DONE); - spin_unlock_irqrestore(&data->tlb_lock, flags); + spin_unlock_irqrestore(&curbank->tlb_lock, flags); if (ret) { dev_warn(data->dev, @@ -386,12 +403,13 @@ static void mtk_iommu_tlb_flush_range_sync(unsigned long iova, size_t size, static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) { - struct mtk_iommu_data *data = dev_id; - struct mtk_iommu_domain *dom = data->m4u_dom; + struct mtk_iommu_bank_data *bank = dev_id; + struct mtk_iommu_data *data = bank->parent_data; + struct mtk_iommu_domain *dom = bank->m4u_dom; unsigned int fault_larb = MTK_INVALID_LARBID, fault_port = 0, sub_comm = 0; u32 int_state, regval, va34_32, pa34_32; const struct mtk_iommu_plat_data *plat_data = data->plat_data; - void __iomem *base = data->base; + void __iomem *base = bank->base; u64 fault_iova, fault_pa; bool layer, write; @@ -430,10 +448,10 @@ static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) fault_larb = data->plat_data->larbid_remap[fault_larb][sub_comm]; } - if (report_iommu_fault(&dom->domain, data->dev, fault_iova, + if (report_iommu_fault(&dom->domain, bank->parent_dev, fault_iova, write ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ)) { dev_err_ratelimited( - data->dev, + bank->parent_dev, "fault type=0x%x iova=0x%llx pa=0x%llx master=0x%x(larb=%d port=%d) layer=%d %s\n", int_state, fault_iova, fault_pa, regval, fault_larb, fault_port, layer, write ? "write" : "read"); @@ -530,12 +548,14 @@ static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom, unsigned int domid) { const struct mtk_iommu_iova_region *region; - - /* Use the exist domain as there is only one pgtable here. */ - if (data->m4u_dom) { - dom->iop = data->m4u_dom->iop; - dom->cfg = data->m4u_dom->cfg; - dom->domain.pgsize_bitmap = data->m4u_dom->cfg.pgsize_bitmap; + struct mtk_iommu_domain *m4u_dom; + + /* Always use bank0 in sharing pgtable case */ + m4u_dom = data->bank[0].m4u_dom; + if (m4u_dom) { + dom->iop = m4u_dom->iop; + dom->cfg = m4u_dom->cfg; + dom->domain.pgsize_bitmap = m4u_dom->cfg.pgsize_bitmap; goto update_iova_region; } @@ -598,6 +618,8 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, struct mtk_iommu_domain *dom = to_mtk_domain(domain); struct list_head *hw_list = data->hw_list; struct device *m4udev = data->dev; + struct mtk_iommu_bank_data *bank; + unsigned int bankid = 0; int ret, domid; domid = mtk_iommu_get_domain_id(dev, data->plat_data); @@ -605,7 +627,7 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, return domid; mutex_lock(&dom->mutex); - if (!dom->data) { + if (!dom->bank) { /* Data is in the frstdata in sharing pgtable case. */ frstdata = mtk_iommu_get_frst_data(hw_list); @@ -614,12 +636,13 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, mutex_unlock(&dom->mutex); return -ENODEV; } - dom->data = data; + dom->bank = &data->bank[bankid]; } mutex_unlock(&dom->mutex); mutex_lock(&data->mutex); - if (!data->m4u_dom) { /* Initialize the M4U HW */ + bank = &data->bank[bankid]; + if (!bank->m4u_dom) { /* Initialize the M4U HW */ ret = pm_runtime_resume_and_get(m4udev); if (ret < 0) goto err_unlock; @@ -629,9 +652,9 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, pm_runtime_put(m4udev); goto err_unlock; } - data->m4u_dom = dom; + bank->m4u_dom = dom; writel(dom->cfg.arm_v7s_cfg.ttbr & MMU_PT_ADDR_MASK, - data->base + REG_MMU_PT_BASE_ADDR); + bank->base + REG_MMU_PT_BASE_ADDR); pm_runtime_put(m4udev); } @@ -658,7 +681,7 @@ static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova, struct mtk_iommu_domain *dom = to_mtk_domain(domain); /* The "4GB mode" M4U physically can not use the lower remap of Dram. */ - if (dom->data->enable_4GB) + if (dom->bank->parent_data->enable_4GB) paddr |= BIT_ULL(32); /* Synchronize with the tlb_lock */ @@ -679,7 +702,7 @@ static void mtk_iommu_flush_iotlb_all(struct iommu_domain *domain) { struct mtk_iommu_domain *dom = to_mtk_domain(domain); - mtk_iommu_tlb_flush_all(dom->data); + mtk_iommu_tlb_flush_all(dom->bank->parent_data); } static void mtk_iommu_iotlb_sync(struct iommu_domain *domain, @@ -688,7 +711,7 @@ static void mtk_iommu_iotlb_sync(struct iommu_domain *domain, struct mtk_iommu_domain *dom = to_mtk_domain(domain); size_t length = gather->end - gather->start + 1; - mtk_iommu_tlb_flush_range_sync(gather->start, length, dom->data); + mtk_iommu_tlb_flush_range_sync(gather->start, length, dom->bank); } static void mtk_iommu_sync_map(struct iommu_domain *domain, unsigned long iova, @@ -696,7 +719,7 @@ static void mtk_iommu_sync_map(struct iommu_domain *domain, unsigned long iova, { struct mtk_iommu_domain *dom = to_mtk_domain(domain); - mtk_iommu_tlb_flush_range_sync(iova, size, dom->data); + mtk_iommu_tlb_flush_range_sync(iova, size, dom->bank); } static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain, @@ -707,7 +730,7 @@ static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain, pa = dom->iop->iova_to_phys(dom->iop, iova); if (IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT) && - dom->data->enable_4GB && + dom->bank->parent_data->enable_4GB && pa >= MTK_IOMMU_4GB_MODE_REMAP_BASE) pa &= ~BIT_ULL(32); @@ -876,16 +899,17 @@ static const struct iommu_ops mtk_iommu_ops = { static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) { + const struct mtk_iommu_bank_data *bank0 = &data->bank[0]; u32 regval; if (data->plat_data->m4u_plat == M4U_MT8173) { regval = F_MMU_PREFETCH_RT_REPLACE_MOD | F_MMU_TF_PROT_TO_PROGRAM_ADDR_MT8173; } else { - regval = readl_relaxed(data->base + REG_MMU_CTRL_REG); + regval = readl_relaxed(bank0->base + REG_MMU_CTRL_REG); regval |= F_MMU_TF_PROT_TO_PROGRAM_ADDR; } - writel_relaxed(regval, data->base + REG_MMU_CTRL_REG); + writel_relaxed(regval, bank0->base + REG_MMU_CTRL_REG); if (data->enable_4GB && MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_VLD_PA_RNG)) { @@ -894,31 +918,31 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) * 0x1_0000_0000 to 0x1_ffff_ffff. here record bit[32:30]. */ regval = F_MMU_VLD_PA_RNG(7, 4); - writel_relaxed(regval, data->base + REG_MMU_VLD_PA_RNG); + writel_relaxed(regval, bank0->base + REG_MMU_VLD_PA_RNG); } if (MTK_IOMMU_HAS_FLAG(data->plat_data, DCM_DISABLE)) - writel_relaxed(F_MMU_DCM, data->base + REG_MMU_DCM_DIS); + writel_relaxed(F_MMU_DCM, bank0->base + REG_MMU_DCM_DIS); else - writel_relaxed(0, data->base + REG_MMU_DCM_DIS); + writel_relaxed(0, bank0->base + REG_MMU_DCM_DIS); if (MTK_IOMMU_HAS_FLAG(data->plat_data, WR_THROT_EN)) { /* write command throttling mode */ - regval = readl_relaxed(data->base + REG_MMU_WR_LEN_CTRL); + regval = readl_relaxed(bank0->base + REG_MMU_WR_LEN_CTRL); regval &= ~F_MMU_WR_THROT_DIS_MASK; - writel_relaxed(regval, data->base + REG_MMU_WR_LEN_CTRL); + writel_relaxed(regval, bank0->base + REG_MMU_WR_LEN_CTRL); } if (MTK_IOMMU_HAS_FLAG(data->plat_data, RESET_AXI)) { /* The register is called STANDARD_AXI_MODE in this case */ regval = 0; } else { - regval = readl_relaxed(data->base + REG_MMU_MISC_CTRL); + regval = readl_relaxed(bank0->base + REG_MMU_MISC_CTRL); if (!MTK_IOMMU_HAS_FLAG(data->plat_data, STD_AXI_MODE)) regval &= ~F_MMU_STANDARD_AXI_MODE_MASK; if (MTK_IOMMU_HAS_FLAG(data->plat_data, OUT_ORDER_WR_EN)) regval &= ~F_MMU_IN_ORDER_WR_EN_MASK; } - writel_relaxed(regval, data->base + REG_MMU_MISC_CTRL); + writel_relaxed(regval, bank0->base + REG_MMU_MISC_CTRL); regval = F_L2_MULIT_HIT_EN | F_TABLE_WALK_FAULT_INT_EN | @@ -926,7 +950,7 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) F_MISS_FIFO_OVERFLOW_INT_EN | F_PREFETCH_FIFO_ERR_INT_EN | F_MISS_FIFO_ERR_INT_EN; - writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL0); + writel_relaxed(regval, bank0->base + REG_MMU_INT_CONTROL0); regval = F_INT_TRANSLATION_FAULT | F_INT_MAIN_MULTI_HIT_FAULT | @@ -935,19 +959,19 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) F_INT_TLB_MISS_FAULT | F_INT_MISS_TRANSACTION_FIFO_FAULT | F_INT_PRETETCH_TRANSATION_FIFO_FAULT; - writel_relaxed(regval, data->base + REG_MMU_INT_MAIN_CONTROL); + writel_relaxed(regval, bank0->base + REG_MMU_INT_MAIN_CONTROL); if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_LEGACY_IVRP_PADDR)) regval = (data->protect_base >> 1) | (data->enable_4GB << 31); else regval = lower_32_bits(data->protect_base) | upper_32_bits(data->protect_base); - writel_relaxed(regval, data->base + REG_MMU_IVRP_PADDR); + writel_relaxed(regval, bank0->base + REG_MMU_IVRP_PADDR); - if (devm_request_irq(data->dev, data->irq, mtk_iommu_isr, 0, - dev_name(data->dev), (void *)data)) { - writel_relaxed(0, data->base + REG_MMU_PT_BASE_ADDR); - dev_err(data->dev, "Failed @ IRQ-%d Request\n", data->irq); + if (devm_request_irq(bank0->parent_dev, bank0->irq, mtk_iommu_isr, 0, + dev_name(bank0->parent_dev), (void *)bank0)) { + writel_relaxed(0, bank0->base + REG_MMU_PT_BASE_ADDR); + dev_err(bank0->parent_dev, "Failed @ IRQ-%d Request\n", bank0->irq); return -ENODEV; } @@ -1039,9 +1063,11 @@ static int mtk_iommu_probe(struct platform_device *pdev) struct component_match *match = NULL; struct regmap *infracfg; void *protect; - int ret; + int ret, banks_num; u32 val; char *p; + struct mtk_iommu_bank_data *bank; + void __iomem *base; data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); if (!data) @@ -1079,14 +1105,26 @@ static int mtk_iommu_probe(struct platform_device *pdev) } res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - data->base = devm_ioremap_resource(dev, res); - if (IS_ERR(data->base)) - return PTR_ERR(data->base); + base = devm_ioremap_resource(dev, res); + if (IS_ERR(base)) + return PTR_ERR(base); ioaddr = res->start; - data->irq = platform_get_irq(pdev, 0); - if (data->irq < 0) - return data->irq; + banks_num = data->plat_data->banks_num; + data->bank = devm_kmalloc(dev, banks_num * sizeof(*data->bank), GFP_KERNEL); + if (!data->bank) + return -ENOMEM; + + bank = &data->bank[0]; + bank->id = 0; + bank->base = base; + bank->m4u_dom = NULL; + bank->irq = platform_get_irq(pdev, 0); + if (bank->irq < 0) + return bank->irq; + bank->parent_dev = dev; + bank->parent_data = data; + spin_lock_init(&bank->tlb_lock); if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_BCLK)) { data->bclk = devm_clk_get(dev, "bclk"); @@ -1125,8 +1163,6 @@ static int mtk_iommu_probe(struct platform_device *pdev) if (ret) goto out_sysfs_remove; - spin_lock_init(&data->tlb_lock); - if (MTK_IOMMU_HAS_FLAG(data->plat_data, SHARE_PGTABLE)) { list_add_tail(&data->list, data->plat_data->hw_list); data->hw_list = data->plat_data->hw_list; @@ -1176,6 +1212,7 @@ out_runtime_disable: static int mtk_iommu_remove(struct platform_device *pdev) { struct mtk_iommu_data *data = platform_get_drvdata(pdev); + struct mtk_iommu_bank_data *bank = &data->bank[0]; iommu_device_sysfs_remove(&data->iommu); iommu_device_unregister(&data->iommu); @@ -1192,7 +1229,7 @@ static int mtk_iommu_remove(struct platform_device *pdev) #endif } pm_runtime_disable(&pdev->dev); - devm_free_irq(&pdev->dev, data->irq, data); + devm_free_irq(&pdev->dev, bank->irq, bank); return 0; } @@ -1200,7 +1237,7 @@ static int __maybe_unused mtk_iommu_runtime_suspend(struct device *dev) { struct mtk_iommu_data *data = dev_get_drvdata(dev); struct mtk_iommu_suspend_reg *reg = &data->reg; - void __iomem *base = data->base; + void __iomem *base = data->bank[0].base; reg->wr_len_ctrl = readl_relaxed(base + REG_MMU_WR_LEN_CTRL); reg->misc_ctrl = readl_relaxed(base + REG_MMU_MISC_CTRL); @@ -1218,8 +1255,8 @@ static int __maybe_unused mtk_iommu_runtime_resume(struct device *dev) { struct mtk_iommu_data *data = dev_get_drvdata(dev); struct mtk_iommu_suspend_reg *reg = &data->reg; - struct mtk_iommu_domain *m4u_dom = data->m4u_dom; - void __iomem *base = data->base; + struct mtk_iommu_domain *m4u_dom = data->bank[0].m4u_dom; + void __iomem *base = data->bank[0].base; int ret; ret = clk_prepare_enable(data->bclk); @@ -1267,6 +1304,8 @@ static const struct mtk_iommu_plat_data mt2712_data = { .hw_list = &m4ulist, .inv_sel_reg = REG_MMU_INV_SEL_GEN1, .iova_region = single_domain, + .banks_num = 1, + .banks_enable = {true}, .iova_region_nr = ARRAY_SIZE(single_domain), .larbid_remap = {{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}}, }; @@ -1276,6 +1315,8 @@ static const struct mtk_iommu_plat_data mt6779_data = { .flags = HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN | WR_THROT_EN | MTK_IOMMU_TYPE_MM, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, + .banks_num = 1, + .banks_enable = {true}, .iova_region = single_domain, .iova_region_nr = ARRAY_SIZE(single_domain), .larbid_remap = {{0}, {1}, {2}, {3}, {5}, {7, 8}, {10}, {9}}, @@ -1285,6 +1326,8 @@ static const struct mtk_iommu_plat_data mt8167_data = { .m4u_plat = M4U_MT8167, .flags = RESET_AXI | HAS_LEGACY_IVRP_PADDR | MTK_IOMMU_TYPE_MM, .inv_sel_reg = REG_MMU_INV_SEL_GEN1, + .banks_num = 1, + .banks_enable = {true}, .iova_region = single_domain, .iova_region_nr = ARRAY_SIZE(single_domain), .larbid_remap = {{0}, {1}, {2}}, /* Linear mapping. */ @@ -1295,6 +1338,8 @@ static const struct mtk_iommu_plat_data mt8173_data = { .flags = HAS_4GB_MODE | HAS_BCLK | RESET_AXI | HAS_LEGACY_IVRP_PADDR | MTK_IOMMU_TYPE_MM, .inv_sel_reg = REG_MMU_INV_SEL_GEN1, + .banks_num = 1, + .banks_enable = {true}, .iova_region = single_domain, .iova_region_nr = ARRAY_SIZE(single_domain), .larbid_remap = {{0}, {1}, {2}, {3}, {4}, {5}}, /* Linear mapping. */ @@ -1304,6 +1349,8 @@ static const struct mtk_iommu_plat_data mt8183_data = { .m4u_plat = M4U_MT8183, .flags = RESET_AXI | MTK_IOMMU_TYPE_MM, .inv_sel_reg = REG_MMU_INV_SEL_GEN1, + .banks_num = 1, + .banks_enable = {true}, .iova_region = single_domain, .iova_region_nr = ARRAY_SIZE(single_domain), .larbid_remap = {{0}, {4}, {5}, {6}, {7}, {2}, {3}, {1}}, @@ -1314,6 +1361,8 @@ static const struct mtk_iommu_plat_data mt8192_data = { .flags = HAS_BCLK | HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN | WR_THROT_EN | IOVA_34_EN | MTK_IOMMU_TYPE_MM, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, + .banks_num = 1, + .banks_enable = {true}, .iova_region = mt8192_multi_dom, .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom), .larbid_remap = {{0}, {1}, {4, 5}, {7}, {2}, {9, 11, 19, 20}, @@ -1326,6 +1375,8 @@ static const struct mtk_iommu_plat_data mt8195_data_infra = { MTK_IOMMU_TYPE_INFRA | IFA_IOMMU_PCIE_SUPPORT, .pericfg_comp_str = "mediatek,mt8195-pericfg_ao", .inv_sel_reg = REG_MMU_INV_SEL_GEN2, + .banks_num = 1, + .banks_enable = {true}, .iova_region = single_domain, .iova_region_nr = ARRAY_SIZE(single_domain), }; @@ -1336,6 +1387,8 @@ static const struct mtk_iommu_plat_data mt8195_data_vdo = { WR_THROT_EN | IOVA_34_EN | SHARE_PGTABLE | MTK_IOMMU_TYPE_MM, .hw_list = &m4ulist, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, + .banks_num = 1, + .banks_enable = {true}, .iova_region = mt8192_multi_dom, .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom), .larbid_remap = {{2, 0}, {21}, {24}, {7}, {19}, {9, 10, 11}, @@ -1348,6 +1401,8 @@ static const struct mtk_iommu_plat_data mt8195_data_vpp = { WR_THROT_EN | IOVA_34_EN | SHARE_PGTABLE | MTK_IOMMU_TYPE_MM, .hw_list = &m4ulist, .inv_sel_reg = REG_MMU_INV_SEL_GEN2, + .banks_num = 1, + .banks_enable = {true}, .iova_region = mt8192_multi_dom, .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom), .larbid_remap = {{1}, {3}, -- cgit v1.2.3 From e24453e165bc774b53f826e86e8f4e9931ffcfba Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:14:21 +0800 Subject: iommu/mediatek: Initialise bank HW for each a bank The mt8195 IOMMU HW max support 5 banks, and regarding the banks' registers, it looks like: ---------------------------------------- |bank0 | bank1 | bank2 | bank3 | bank4| ---------------------------------------- |global | |control| null |regs | ----------------------------------------- |bank |bank |bank |bank |bank | |regs |regs |regs |regs |regs | | | | | | | ----------------------------------------- Each bank has some special bank registers and it share bank0's global control registers. this patch initialise the bank hw with the bankid. In the hw_init, we always initialise bank0's control register since we don't know if the bank0 is initialised. Additionally, About each bank's register base, always delta 0x1000. like bank[x + 1] = bank[x] + 0x1000. Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-31-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 968e1486e6fd..5f298cf6aac3 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -259,7 +259,7 @@ static void mtk_iommu_unbind(struct device *dev) static const struct iommu_ops mtk_iommu_ops; -static int mtk_iommu_hw_init(const struct mtk_iommu_data *data); +static int mtk_iommu_hw_init(const struct mtk_iommu_data *data, unsigned int bankid); #define MTK_IOMMU_TLB_ADDR(iova) ({ \ dma_addr_t _addr = iova; \ @@ -642,12 +642,14 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, mutex_lock(&data->mutex); bank = &data->bank[bankid]; - if (!bank->m4u_dom) { /* Initialize the M4U HW */ + if (!bank->m4u_dom) { /* Initialize the M4U HW for each a BANK */ ret = pm_runtime_resume_and_get(m4udev); - if (ret < 0) + if (ret < 0) { + dev_err(m4udev, "pm get fail(%d) in attach.\n", ret); goto err_unlock; + } - ret = mtk_iommu_hw_init(data); + ret = mtk_iommu_hw_init(data, bankid); if (ret) { pm_runtime_put(m4udev); goto err_unlock; @@ -897,11 +899,16 @@ static const struct iommu_ops mtk_iommu_ops = { } }; -static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) +static int mtk_iommu_hw_init(const struct mtk_iommu_data *data, unsigned int bankid) { + const struct mtk_iommu_bank_data *bankx = &data->bank[bankid]; const struct mtk_iommu_bank_data *bank0 = &data->bank[0]; u32 regval; + /* + * Global control settings are in bank0. May re-init these global registers + * since no sure if there is bank0 consumers. + */ if (data->plat_data->m4u_plat == M4U_MT8173) { regval = F_MMU_PREFETCH_RT_REPLACE_MOD | F_MMU_TF_PROT_TO_PROGRAM_ADDR_MT8173; @@ -944,13 +951,14 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) } writel_relaxed(regval, bank0->base + REG_MMU_MISC_CTRL); + /* Independent settings for each bank */ regval = F_L2_MULIT_HIT_EN | F_TABLE_WALK_FAULT_INT_EN | F_PREETCH_FIFO_OVERFLOW_INT_EN | F_MISS_FIFO_OVERFLOW_INT_EN | F_PREFETCH_FIFO_ERR_INT_EN | F_MISS_FIFO_ERR_INT_EN; - writel_relaxed(regval, bank0->base + REG_MMU_INT_CONTROL0); + writel_relaxed(regval, bankx->base + REG_MMU_INT_CONTROL0); regval = F_INT_TRANSLATION_FAULT | F_INT_MAIN_MULTI_HIT_FAULT | @@ -959,19 +967,19 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) F_INT_TLB_MISS_FAULT | F_INT_MISS_TRANSACTION_FIFO_FAULT | F_INT_PRETETCH_TRANSATION_FIFO_FAULT; - writel_relaxed(regval, bank0->base + REG_MMU_INT_MAIN_CONTROL); + writel_relaxed(regval, bankx->base + REG_MMU_INT_MAIN_CONTROL); if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_LEGACY_IVRP_PADDR)) regval = (data->protect_base >> 1) | (data->enable_4GB << 31); else regval = lower_32_bits(data->protect_base) | upper_32_bits(data->protect_base); - writel_relaxed(regval, bank0->base + REG_MMU_IVRP_PADDR); + writel_relaxed(regval, bankx->base + REG_MMU_IVRP_PADDR); - if (devm_request_irq(bank0->parent_dev, bank0->irq, mtk_iommu_isr, 0, - dev_name(bank0->parent_dev), (void *)bank0)) { - writel_relaxed(0, bank0->base + REG_MMU_PT_BASE_ADDR); - dev_err(bank0->parent_dev, "Failed @ IRQ-%d Request\n", bank0->irq); + if (devm_request_irq(bankx->parent_dev, bankx->irq, mtk_iommu_isr, 0, + dev_name(bankx->parent_dev), (void *)bankx)) { + writel_relaxed(0, bankx->base + REG_MMU_PT_BASE_ADDR); + dev_err(bankx->parent_dev, "Failed @ IRQ-%d Request\n", bankx->irq); return -ENODEV; } -- cgit v1.2.3 From d72e0ff5424e74b551c73ed2d34c73b70b921733 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:14:22 +0800 Subject: iommu/mediatek: Change the domid to iova_region_id Prepare for adding bankid, also no functional change. In the previous SoC, each a iova_region is a domain; In the multi-banks case, each a bank is a domain, then the original function name "mtk_iommu_get_domain_id" is not proper. Use "iova_region_id" instead of "domain_id". Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-32-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 5f298cf6aac3..023bb7d3ffb2 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -467,8 +467,8 @@ static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) return IRQ_HANDLED; } -static int mtk_iommu_get_domain_id(struct device *dev, - const struct mtk_iommu_plat_data *plat_data) +static int mtk_iommu_get_iova_region_id(struct device *dev, + const struct mtk_iommu_plat_data *plat_data) { const struct mtk_iommu_iova_region *rgn = plat_data->iova_region; const struct bus_dma_region *dma_rgn = dev->dma_range_map; @@ -498,7 +498,7 @@ static int mtk_iommu_get_domain_id(struct device *dev, } static int mtk_iommu_config(struct mtk_iommu_data *data, struct device *dev, - bool enable, unsigned int domid) + bool enable, unsigned int regionid) { struct mtk_smi_larb_iommu *larb_mmu; unsigned int larbid, portid; @@ -514,12 +514,12 @@ static int mtk_iommu_config(struct mtk_iommu_data *data, struct device *dev, if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { larb_mmu = &data->larb_imu[larbid]; - region = data->plat_data->iova_region + domid; + region = data->plat_data->iova_region + regionid; larb_mmu->bank[portid] = upper_32_bits(region->iova_base); - dev_dbg(dev, "%s iommu for larb(%s) port %d dom %d bank %d.\n", + dev_dbg(dev, "%s iommu for larb(%s) port %d region %d rgn-bank %d.\n", enable ? "enable" : "disable", dev_name(larb_mmu->dev), - portid, domid, larb_mmu->bank[portid]); + portid, regionid, larb_mmu->bank[portid]); if (enable) larb_mmu->mmu |= MTK_SMI_MMU_EN(portid); @@ -545,7 +545,7 @@ static int mtk_iommu_config(struct mtk_iommu_data *data, struct device *dev, static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom, struct mtk_iommu_data *data, - unsigned int domid) + unsigned int region_id) { const struct mtk_iommu_iova_region *region; struct mtk_iommu_domain *m4u_dom; @@ -584,7 +584,7 @@ static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom, update_iova_region: /* Update the iova region for this domain */ - region = data->plat_data->iova_region + domid; + region = data->plat_data->iova_region + region_id; dom->domain.geometry.aperture_start = region->iova_base; dom->domain.geometry.aperture_end = region->iova_base + region->size - 1; dom->domain.geometry.force_aperture = true; @@ -620,18 +620,18 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, struct device *m4udev = data->dev; struct mtk_iommu_bank_data *bank; unsigned int bankid = 0; - int ret, domid; + int ret, region_id; - domid = mtk_iommu_get_domain_id(dev, data->plat_data); - if (domid < 0) - return domid; + region_id = mtk_iommu_get_iova_region_id(dev, data->plat_data); + if (region_id < 0) + return region_id; mutex_lock(&dom->mutex); if (!dom->bank) { /* Data is in the frstdata in sharing pgtable case. */ frstdata = mtk_iommu_get_frst_data(hw_list); - ret = mtk_iommu_domain_finalise(dom, frstdata, domid); + ret = mtk_iommu_domain_finalise(dom, frstdata, region_id); if (ret) { mutex_unlock(&dom->mutex); return -ENODEV; @@ -662,7 +662,7 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, } mutex_unlock(&data->mutex); - return mtk_iommu_config(data, dev, true, domid); + return mtk_iommu_config(data, dev, true, region_id); err_unlock: mutex_unlock(&data->mutex); @@ -802,22 +802,22 @@ static struct iommu_group *mtk_iommu_device_group(struct device *dev) struct mtk_iommu_data *c_data = dev_iommu_priv_get(dev), *data; struct list_head *hw_list = c_data->hw_list; struct iommu_group *group; - int domid; + int regionid; data = mtk_iommu_get_frst_data(hw_list); if (!data) return ERR_PTR(-ENODEV); - domid = mtk_iommu_get_domain_id(dev, data->plat_data); - if (domid < 0) - return ERR_PTR(domid); + regionid = mtk_iommu_get_iova_region_id(dev, data->plat_data); + if (regionid < 0) + return ERR_PTR(regionid); mutex_lock(&data->mutex); - group = data->m4u_group[domid]; + group = data->m4u_group[regionid]; if (!group) { group = iommu_group_alloc(); if (!IS_ERR(group)) - data->m4u_group[domid] = group; + data->m4u_group[regionid] = group; } else { iommu_group_ref_get(group); } @@ -851,14 +851,14 @@ static void mtk_iommu_get_resv_regions(struct device *dev, struct list_head *head) { struct mtk_iommu_data *data = dev_iommu_priv_get(dev); - unsigned int domid = mtk_iommu_get_domain_id(dev, data->plat_data), i; + unsigned int regionid = mtk_iommu_get_iova_region_id(dev, data->plat_data), i; const struct mtk_iommu_iova_region *resv, *curdom; struct iommu_resv_region *region; int prot = IOMMU_WRITE | IOMMU_READ; - if ((int)domid < 0) + if ((int)regionid < 0) return; - curdom = data->plat_data->iova_region + domid; + curdom = data->plat_data->iova_region + regionid; for (i = 0; i < data->plat_data->iova_region_nr; i++) { resv = data->plat_data->iova_region + i; -- cgit v1.2.3 From 57fb481f9008997563a8de96aa49726fd51d6e3e Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:14:23 +0800 Subject: iommu/mediatek: Get the proper bankid for multi banks We preassign some ports in a special bank via the new defined banks_portmsk. Put it in the plat_data means it is not expected to be adjusted dynamically. If the iommu id in the iommu consumer's dtsi node is inside this banks_portmsk, then we switch it to this special iommu bank, and initialise the IOMMU bank HW. Each bank has the independent pgtable(4GB iova range). Each bank is a independent iommu domain/group. Currently we don't separate different iova ranges inside a bank. Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-33-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 55 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 7 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 023bb7d3ffb2..5f3e88c44514 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -191,6 +191,7 @@ struct mtk_iommu_plat_data { u8 banks_num; bool banks_enable[MTK_IOMMU_BANK_MAX]; + unsigned int banks_portmsk[MTK_IOMMU_BANK_MAX]; unsigned char larbid_remap[MTK_LARB_COM_MAX][MTK_LARB_SUBCOM_MAX]; }; @@ -467,6 +468,30 @@ static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) return IRQ_HANDLED; } +static unsigned int mtk_iommu_get_bank_id(struct device *dev, + const struct mtk_iommu_plat_data *plat_data) +{ + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + unsigned int i, portmsk = 0, bankid = 0; + + if (plat_data->banks_num == 1) + return bankid; + + for (i = 0; i < fwspec->num_ids; i++) + portmsk |= BIT(MTK_M4U_TO_PORT(fwspec->ids[i])); + + for (i = 0; i < plat_data->banks_num && i < MTK_IOMMU_BANK_MAX; i++) { + if (!plat_data->banks_enable[i]) + continue; + + if (portmsk & plat_data->banks_portmsk[i]) { + bankid = i; + break; + } + } + return bankid; /* default is 0 */ +} + static int mtk_iommu_get_iova_region_id(struct device *dev, const struct mtk_iommu_plat_data *plat_data) { @@ -619,13 +644,14 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, struct list_head *hw_list = data->hw_list; struct device *m4udev = data->dev; struct mtk_iommu_bank_data *bank; - unsigned int bankid = 0; + unsigned int bankid; int ret, region_id; region_id = mtk_iommu_get_iova_region_id(dev, data->plat_data); if (region_id < 0) return region_id; + bankid = mtk_iommu_get_bank_id(dev, data->plat_data); mutex_lock(&dom->mutex); if (!dom->bank) { /* Data is in the frstdata in sharing pgtable case. */ @@ -797,27 +823,42 @@ static void mtk_iommu_release_device(struct device *dev) iommu_fwspec_free(dev); } +static int mtk_iommu_get_group_id(struct device *dev, const struct mtk_iommu_plat_data *plat_data) +{ + unsigned int bankid; + + /* + * If the bank function is enabled, each bank is a iommu group/domain. + * Otherwise, each iova region is a iommu group/domain. + */ + bankid = mtk_iommu_get_bank_id(dev, plat_data); + if (bankid) + return bankid; + + return mtk_iommu_get_iova_region_id(dev, plat_data); +} + static struct iommu_group *mtk_iommu_device_group(struct device *dev) { struct mtk_iommu_data *c_data = dev_iommu_priv_get(dev), *data; struct list_head *hw_list = c_data->hw_list; struct iommu_group *group; - int regionid; + int groupid; data = mtk_iommu_get_frst_data(hw_list); if (!data) return ERR_PTR(-ENODEV); - regionid = mtk_iommu_get_iova_region_id(dev, data->plat_data); - if (regionid < 0) - return ERR_PTR(regionid); + groupid = mtk_iommu_get_group_id(dev, data->plat_data); + if (groupid < 0) + return ERR_PTR(groupid); mutex_lock(&data->mutex); - group = data->m4u_group[regionid]; + group = data->m4u_group[groupid]; if (!group) { group = iommu_group_alloc(); if (!IS_ERR(group)) - data->m4u_group[regionid] = group; + data->m4u_group[groupid] = group; } else { iommu_group_ref_get(group); } -- cgit v1.2.3 From 42d57fc58aebc5801804424082028f43bad1b73c Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:14:24 +0800 Subject: iommu/mediatek: Initialise/Remove for multi bank dev The registers for each bank of the IOMMU base are in order, delta is 0x1000. Initialise the base for each bank. For all the previous SoC, we only have bank0. thus use "do {} while()" to allow bank0 always go. When removing the device, Not always all the banks are initialised, it depend on if there is masters for that bank. Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-34-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 44 ++++++++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 14 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 5f3e88c44514..400dea33aea1 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -113,6 +113,7 @@ #define F_MMU_INT_ID_PORT_ID(a) (((a) >> 2) & 0x1f) #define MTK_PROTECT_PA_ALIGN 256 +#define MTK_IOMMU_BANK_SZ 0x1000 #define PERICFG_IOMMU_1 0x714 @@ -1112,7 +1113,7 @@ static int mtk_iommu_probe(struct platform_device *pdev) struct component_match *match = NULL; struct regmap *infracfg; void *protect; - int ret, banks_num; + int ret, banks_num, i = 0; u32 val; char *p; struct mtk_iommu_bank_data *bank; @@ -1153,27 +1154,36 @@ static int mtk_iommu_probe(struct platform_device *pdev) data->enable_4GB = !!(val & F_DDR_4GB_SUPPORT_EN); } + banks_num = data->plat_data->banks_num; res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (resource_size(res) < banks_num * MTK_IOMMU_BANK_SZ) { + dev_err(dev, "banknr %d. res %pR is not enough.\n", banks_num, res); + return -EINVAL; + } base = devm_ioremap_resource(dev, res); if (IS_ERR(base)) return PTR_ERR(base); ioaddr = res->start; - banks_num = data->plat_data->banks_num; data->bank = devm_kmalloc(dev, banks_num * sizeof(*data->bank), GFP_KERNEL); if (!data->bank) return -ENOMEM; - bank = &data->bank[0]; - bank->id = 0; - bank->base = base; - bank->m4u_dom = NULL; - bank->irq = platform_get_irq(pdev, 0); - if (bank->irq < 0) - return bank->irq; - bank->parent_dev = dev; - bank->parent_data = data; - spin_lock_init(&bank->tlb_lock); + do { + if (!data->plat_data->banks_enable[i]) + continue; + bank = &data->bank[i]; + bank->id = i; + bank->base = base + i * MTK_IOMMU_BANK_SZ; + bank->m4u_dom = NULL; + + bank->irq = platform_get_irq(pdev, i); + if (bank->irq < 0) + return bank->irq; + bank->parent_dev = dev; + bank->parent_data = data; + spin_lock_init(&bank->tlb_lock); + } while (++i < banks_num); if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_BCLK)) { data->bclk = devm_clk_get(dev, "bclk"); @@ -1261,7 +1271,8 @@ out_runtime_disable: static int mtk_iommu_remove(struct platform_device *pdev) { struct mtk_iommu_data *data = platform_get_drvdata(pdev); - struct mtk_iommu_bank_data *bank = &data->bank[0]; + struct mtk_iommu_bank_data *bank; + int i; iommu_device_sysfs_remove(&data->iommu); iommu_device_unregister(&data->iommu); @@ -1278,7 +1289,12 @@ static int mtk_iommu_remove(struct platform_device *pdev) #endif } pm_runtime_disable(&pdev->dev); - devm_free_irq(&pdev->dev, bank->irq, bank); + for (i = 0; i < data->plat_data->banks_num; i++) { + bank = &data->bank[i]; + if (!bank->m4u_dom) + continue; + devm_free_irq(&pdev->dev, bank->irq, bank); + } return 0; } -- cgit v1.2.3 From d7127de15eba3f96f9ed6854d3275a4e5919f025 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:14:25 +0800 Subject: iommu/mediatek: Backup/restore regsiters for multi banks Each bank has some independent registers. thus backup/restore them for each a bank when suspend and resume. Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-35-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 46 +++++++++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 15 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 400dea33aea1..d3e8773b4c47 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -173,11 +173,12 @@ struct mtk_iommu_suspend_reg { u32 misc_ctrl; u32 dcm_dis; u32 ctrl_reg; - u32 int_control0; - u32 int_main_control; - u32 ivrp_paddr; u32 vld_pa_rng; u32 wr_len_ctrl; + + u32 int_control[MTK_IOMMU_BANK_MAX]; + u32 int_main_control[MTK_IOMMU_BANK_MAX]; + u32 ivrp_paddr[MTK_IOMMU_BANK_MAX]; }; struct mtk_iommu_plat_data { @@ -1302,16 +1303,23 @@ static int __maybe_unused mtk_iommu_runtime_suspend(struct device *dev) { struct mtk_iommu_data *data = dev_get_drvdata(dev); struct mtk_iommu_suspend_reg *reg = &data->reg; - void __iomem *base = data->bank[0].base; + void __iomem *base; + int i = 0; + base = data->bank[i].base; reg->wr_len_ctrl = readl_relaxed(base + REG_MMU_WR_LEN_CTRL); reg->misc_ctrl = readl_relaxed(base + REG_MMU_MISC_CTRL); reg->dcm_dis = readl_relaxed(base + REG_MMU_DCM_DIS); reg->ctrl_reg = readl_relaxed(base + REG_MMU_CTRL_REG); - reg->int_control0 = readl_relaxed(base + REG_MMU_INT_CONTROL0); - reg->int_main_control = readl_relaxed(base + REG_MMU_INT_MAIN_CONTROL); - reg->ivrp_paddr = readl_relaxed(base + REG_MMU_IVRP_PADDR); reg->vld_pa_rng = readl_relaxed(base + REG_MMU_VLD_PA_RNG); + do { + if (!data->plat_data->banks_enable[i]) + continue; + base = data->bank[i].base; + reg->int_control[i] = readl_relaxed(base + REG_MMU_INT_CONTROL0); + reg->int_main_control[i] = readl_relaxed(base + REG_MMU_INT_MAIN_CONTROL); + reg->ivrp_paddr[i] = readl_relaxed(base + REG_MMU_IVRP_PADDR); + } while (++i < data->plat_data->banks_num); clk_disable_unprepare(data->bclk); return 0; } @@ -1320,9 +1328,9 @@ static int __maybe_unused mtk_iommu_runtime_resume(struct device *dev) { struct mtk_iommu_data *data = dev_get_drvdata(dev); struct mtk_iommu_suspend_reg *reg = &data->reg; - struct mtk_iommu_domain *m4u_dom = data->bank[0].m4u_dom; - void __iomem *base = data->bank[0].base; - int ret; + struct mtk_iommu_domain *m4u_dom; + void __iomem *base; + int ret, i = 0; ret = clk_prepare_enable(data->bclk); if (ret) { @@ -1334,18 +1342,26 @@ static int __maybe_unused mtk_iommu_runtime_resume(struct device *dev) * Uppon first resume, only enable the clk and return, since the values of the * registers are not yet set. */ - if (!m4u_dom) + if (!reg->wr_len_ctrl) return 0; + base = data->bank[i].base; writel_relaxed(reg->wr_len_ctrl, base + REG_MMU_WR_LEN_CTRL); writel_relaxed(reg->misc_ctrl, base + REG_MMU_MISC_CTRL); writel_relaxed(reg->dcm_dis, base + REG_MMU_DCM_DIS); writel_relaxed(reg->ctrl_reg, base + REG_MMU_CTRL_REG); - writel_relaxed(reg->int_control0, base + REG_MMU_INT_CONTROL0); - writel_relaxed(reg->int_main_control, base + REG_MMU_INT_MAIN_CONTROL); - writel_relaxed(reg->ivrp_paddr, base + REG_MMU_IVRP_PADDR); writel_relaxed(reg->vld_pa_rng, base + REG_MMU_VLD_PA_RNG); - writel(m4u_dom->cfg.arm_v7s_cfg.ttbr & MMU_PT_ADDR_MASK, base + REG_MMU_PT_BASE_ADDR); + do { + m4u_dom = data->bank[i].m4u_dom; + if (!data->plat_data->banks_enable[i] || !m4u_dom) + continue; + base = data->bank[i].base; + writel_relaxed(reg->int_control[i], base + REG_MMU_INT_CONTROL0); + writel_relaxed(reg->int_main_control[i], base + REG_MMU_INT_MAIN_CONTROL); + writel_relaxed(reg->ivrp_paddr[i], base + REG_MMU_IVRP_PADDR); + writel(m4u_dom->cfg.arm_v7s_cfg.ttbr & MMU_PT_ADDR_MASK, + base + REG_MMU_PT_BASE_ADDR); + } while (++i < data->plat_data->banks_num); /* * Users may allocate dma buffer before they call pm_runtime_get, -- cgit v1.2.3 From 7597e3c5618d40986b23ddfd65d55d741536bf85 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:14:26 +0800 Subject: iommu/mediatek: mt8195: Enable multi banks for infra iommu Enable the multi-bank functions for infra-iommu. We put PCIE in bank0 and USB in the last bank(bank4). and we don't use the other banks currently, disable them. Signed-off-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20220503071427.2285-36-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index d3e8773b4c47..0fa1d5240ac6 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -1456,8 +1456,11 @@ static const struct mtk_iommu_plat_data mt8195_data_infra = { MTK_IOMMU_TYPE_INFRA | IFA_IOMMU_PCIE_SUPPORT, .pericfg_comp_str = "mediatek,mt8195-pericfg_ao", .inv_sel_reg = REG_MMU_INV_SEL_GEN2, - .banks_num = 1, - .banks_enable = {true}, + .banks_num = 5, + .banks_enable = {true, false, false, false, true}, + .banks_portmsk = {[0] = GENMASK(19, 16), /* PCIe */ + [4] = GENMASK(31, 20), /* USB */ + }, .iova_region = single_domain, .iova_region_nr = ARRAY_SIZE(single_domain), }; -- cgit v1.2.3 From e8d7ccaa3fb7575fda703ee6e05640c1c830fa06 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 3 May 2022 15:14:27 +0800 Subject: iommu/mediatek: Add mt8186 iommu support Add mt8186 iommu supports. Signed-off-by: Anan Sun Signed-off-by: Yong Wu Reviewed-by: Matthias Brugger Reviewed-by: AngeloGioacchino Del Regno Link: https://lore.kernel.org/r/20220503071427.2285-37-yong.wu@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 0fa1d5240ac6..71b2ace74cd6 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -160,6 +160,7 @@ enum mtk_iommu_plat { M4U_MT8167, M4U_MT8173, M4U_MT8183, + M4U_MT8186, M4U_MT8192, M4U_MT8195, }; @@ -1437,6 +1438,20 @@ static const struct mtk_iommu_plat_data mt8183_data = { .larbid_remap = {{0}, {4}, {5}, {6}, {7}, {2}, {3}, {1}}, }; +static const struct mtk_iommu_plat_data mt8186_data_mm = { + .m4u_plat = M4U_MT8186, + .flags = HAS_BCLK | HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN | + WR_THROT_EN | IOVA_34_EN | MTK_IOMMU_TYPE_MM, + .larbid_remap = {{0}, {1, MTK_INVALID_LARBID, 8}, {4}, {7}, {2}, {9, 11, 19, 20}, + {MTK_INVALID_LARBID, 14, 16}, + {MTK_INVALID_LARBID, 13, MTK_INVALID_LARBID, 17}}, + .inv_sel_reg = REG_MMU_INV_SEL_GEN2, + .banks_num = 1, + .banks_enable = {true}, + .iova_region = mt8192_multi_dom, + .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom), +}; + static const struct mtk_iommu_plat_data mt8192_data = { .m4u_plat = M4U_MT8192, .flags = HAS_BCLK | HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN | @@ -1503,6 +1518,7 @@ static const struct of_device_id mtk_iommu_of_ids[] = { { .compatible = "mediatek,mt8167-m4u", .data = &mt8167_data}, { .compatible = "mediatek,mt8173-m4u", .data = &mt8173_data}, { .compatible = "mediatek,mt8183-m4u", .data = &mt8183_data}, + { .compatible = "mediatek,mt8186-iommu-mm", .data = &mt8186_data_mm}, /* mm: m4u */ { .compatible = "mediatek,mt8192-m4u", .data = &mt8192_data}, { .compatible = "mediatek,mt8195-iommu-infra", .data = &mt8195_data_infra}, { .compatible = "mediatek,mt8195-iommu-vdo", .data = &mt8195_data_vdo}, -- cgit v1.2.3 From 8b9ad480bd1dd25f4ff4854af5685fa334a2f57a Mon Sep 17 00:00:00 2001 From: Xiaomeng Tong Date: Sun, 1 May 2022 21:28:23 +0800 Subject: iommu/msm: Fix an incorrect NULL check on list iterator The bug is here: if (!iommu || iommu->dev->of_node != spec->np) { The list iterator value 'iommu' will *always* be set and non-NULL by list_for_each_entry(), so it is incorrect to assume that the iterator value will be NULL if the list is empty or no element is found (in fact, it will point to a invalid structure object containing HEAD). To fix the bug, use a new value 'iter' as the list iterator, while use the old value 'iommu' as a dedicated variable to point to the found one, and remove the unneeded check for 'iommu->dev->of_node != spec->np' outside the loop. Cc: stable@vger.kernel.org Fixes: f78ebca8ff3d6 ("iommu/msm: Add support for generic master bindings") Signed-off-by: Xiaomeng Tong Link: https://lore.kernel.org/r/20220501132823.12714-1-xiam0nd.tong@gmail.com Signed-off-by: Joerg Roedel --- drivers/iommu/msm_iommu.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c index 3d9bd2043738..f09aedfdd462 100644 --- a/drivers/iommu/msm_iommu.c +++ b/drivers/iommu/msm_iommu.c @@ -615,16 +615,19 @@ static int insert_iommu_master(struct device *dev, static int qcom_iommu_of_xlate(struct device *dev, struct of_phandle_args *spec) { - struct msm_iommu_dev *iommu; + struct msm_iommu_dev *iommu = NULL, *iter; unsigned long flags; int ret = 0; spin_lock_irqsave(&msm_iommu_lock, flags); - list_for_each_entry(iommu, &qcom_iommu_devices, dev_node) - if (iommu->dev->of_node == spec->np) + list_for_each_entry(iter, &qcom_iommu_devices, dev_node) { + if (iter->dev->of_node == spec->np) { + iommu = iter; break; + } + } - if (!iommu || iommu->dev->of_node != spec->np) { + if (!iommu) { ret = -ENODEV; goto fail; } -- cgit v1.2.3 From d9ed8af1dee37f181096631fb03729ece98ba816 Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Mon, 25 Apr 2022 19:41:36 +0800 Subject: iommu/arm-smmu: fix possible null-ptr-deref in arm_smmu_device_probe() It will cause null-ptr-deref when using 'res', if platform_get_resource() returns NULL, so move using 'res' after devm_ioremap_resource() that will check it to avoid null-ptr-deref. And use devm_platform_get_and_ioremap_resource() to simplify code. Signed-off-by: Yang Yingliang Link: https://lore.kernel.org/r/20220425114136.2649310-1-yangyingliang@huawei.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu/arm-smmu.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c b/drivers/iommu/arm/arm-smmu/arm-smmu.c index 568cce590ccc..52b71f6aee3f 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c @@ -2092,11 +2092,10 @@ static int arm_smmu_device_probe(struct platform_device *pdev) if (err) return err; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - ioaddr = res->start; - smmu->base = devm_ioremap_resource(dev, res); + smmu->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); if (IS_ERR(smmu->base)) return PTR_ERR(smmu->base); + ioaddr = res->start; /* * The resource size should effectively match the value of SMMU_TOP; * stash that temporarily until we know PAGESIZE to validate it with. -- cgit v1.2.3 From b131fa8c1d2afd05d0b7598621114674289c2fbb Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Mon, 25 Apr 2022 19:45:25 +0800 Subject: iommu/arm-smmu-v3: check return value after calling platform_get_resource() It will cause null-ptr-deref if platform_get_resource() returns NULL, we need check the return value. Signed-off-by: Yang Yingliang Link: https://lore.kernel.org/r/20220425114525.2651143-1-yangyingliang@huawei.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'drivers') diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 627a3ed5ee8f..88817a3376ef 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -3770,6 +3770,8 @@ static int arm_smmu_device_probe(struct platform_device *pdev) /* Base address */ res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) + return -EINVAL; if (resource_size(res) < arm_smmu_resource_size(smmu)) { dev_err(dev, "MMIO region too small (%pr)\n", res); return -EINVAL; -- cgit v1.2.3 From cbd23144f7662b00bcde32a938c4a4057e476d68 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Tue, 26 Apr 2022 14:04:45 +0100 Subject: iommu/arm-smmu-v3-sva: Fix mm use-after-free We currently call arm64_mm_context_put() without holding a reference to the mm, which can result in use-after-free. Call mmgrab()/mmdrop() to ensure the mm only gets freed after we unpinned the ASID. Fixes: 32784a9562fb ("iommu/arm-smmu-v3: Implement iommu_sva_bind/unbind()") Signed-off-by: Jean-Philippe Brucker Tested-by: Zhangfei Gao Link: https://lore.kernel.org/r/20220426130444.300556-1-jean-philippe@linaro.org Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index c623dae1e115..1ef7bbb4acf3 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include "arm-smmu-v3.h" @@ -96,9 +97,14 @@ static struct arm_smmu_ctx_desc *arm_smmu_alloc_shared_cd(struct mm_struct *mm) struct arm_smmu_ctx_desc *cd; struct arm_smmu_ctx_desc *ret = NULL; + /* Don't free the mm until we release the ASID */ + mmgrab(mm); + asid = arm64_mm_context_get(mm); - if (!asid) - return ERR_PTR(-ESRCH); + if (!asid) { + err = -ESRCH; + goto out_drop_mm; + } cd = kzalloc(sizeof(*cd), GFP_KERNEL); if (!cd) { @@ -165,6 +171,8 @@ out_free_cd: kfree(cd); out_put_context: arm64_mm_context_put(mm); +out_drop_mm: + mmdrop(mm); return err < 0 ? ERR_PTR(err) : ret; } @@ -173,6 +181,7 @@ static void arm_smmu_free_shared_cd(struct arm_smmu_ctx_desc *cd) if (arm_smmu_free_asid(cd)) { /* Unpin ASID */ arm64_mm_context_put(cd->mm); + mmdrop(cd->mm); kfree(cd); } } -- cgit v1.2.3 From d044023e219d415d2af87e5e48e9e03488ad145c Mon Sep 17 00:00:00 2001 From: Bjorn Andersson Date: Tue, 3 May 2022 09:34:29 -0700 Subject: iommu/arm-smmu-qcom: Add SC8280XP support Add the Qualcomm SC8280XP platform to the list of compatible for which the Qualcomm-impl of the ARM SMMU should apply. Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20220503163429.960998-3-bjorn.andersson@linaro.org Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers') diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c index ba6298c7140e..7820711c4560 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c @@ -408,6 +408,7 @@ static const struct of_device_id __maybe_unused qcom_smmu_impl_of_match[] = { { .compatible = "qcom,sc7180-smmu-500" }, { .compatible = "qcom,sc7280-smmu-500" }, { .compatible = "qcom,sc8180x-smmu-500" }, + { .compatible = "qcom,sc8280xp-smmu-500" }, { .compatible = "qcom,sdm630-smmu-v2" }, { .compatible = "qcom,sdm845-smmu-500" }, { .compatible = "qcom,sm6125-smmu-500" }, -- cgit v1.2.3 From 5ca216155b5ead6e2d64738644afed9ac17f1fe3 Mon Sep 17 00:00:00 2001 From: Thierry Reding Date: Fri, 29 Apr 2022 10:22:43 +0200 Subject: iommu/arm-smmu: Support Tegra234 SMMU Allow the NVIDIA-specific ARM SMMU implementation to bind to the SMMU instances found on Tegra234. Acked-by: Robin Murphy Acked-by: Will Deacon Signed-off-by: Thierry Reding Link: https://lore.kernel.org/r/20220429082243.496000-4-thierry.reding@gmail.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu/arm-smmu-impl.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-impl.c b/drivers/iommu/arm/arm-smmu/arm-smmu-impl.c index 2c25cce38060..658f3cc83278 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-impl.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-impl.c @@ -211,7 +211,8 @@ struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu) if (of_property_read_bool(np, "calxeda,smmu-secure-config-access")) smmu->impl = &calxeda_impl; - if (of_device_is_compatible(np, "nvidia,tegra194-smmu") || + if (of_device_is_compatible(np, "nvidia,tegra234-smmu") || + of_device_is_compatible(np, "nvidia,tegra194-smmu") || of_device_is_compatible(np, "nvidia,tegra186-smmu")) return nvidia_smmu_impl_init(smmu); -- cgit v1.2.3 From 628bf55b620497a105f4963ee8fb84769f7e6bb4 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Tue, 10 May 2022 09:38:58 +0100 Subject: iommu/arm-smmu: Force identity domains for legacy binding When using the legacy "mmu-masters" DT binding, we reject DMA domains since we have no guarantee of driver probe order and thus can't rely on client drivers getting the correct DMA ops. However, we can do better than fall back to the old no-default-domain behaviour now, by forcing an identity default domain instead. This also means that detaching from a VFIO domain can actually work - that looks to have been broken for over 6 years, so clearly isn't something that legacy binding users care about, but we may as well make the driver code make sense anyway. Suggested-by: Jason Gunthorpe Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/9805e4c492cb972bdcdd57999d2d001a2d8b5aab.1652171938.git.robin.murphy@arm.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu/arm-smmu.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'drivers') diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c b/drivers/iommu/arm/arm-smmu/arm-smmu.c index 52b71f6aee3f..2ed3594f384e 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c @@ -1574,6 +1574,9 @@ static int arm_smmu_def_domain_type(struct device *dev) struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); const struct arm_smmu_impl *impl = cfg->smmu->impl; + if (using_legacy_binding) + return IOMMU_DOMAIN_IDENTITY; + if (impl && impl->def_domain_type) return impl->def_domain_type(dev); -- cgit v1.2.3 From 0286300e60455534b23f4b86ce79247829ceddb8 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe via iommu Date: Mon, 9 May 2022 13:19:19 -0300 Subject: iommu: iommu_group_claim_dma_owner() must always assign a domain Once the group enters 'owned' mode it can never be assigned back to the default_domain or to a NULL domain. It must always be actively assigned to a current domain. If the caller hasn't provided a domain then the core must provide an explicit DMA blocking domain that has no DMA map. Lazily create a group-global blocking DMA domain when iommu_group_claim_dma_owner is first called and immediately assign the group to it. This ensures that DMA is immediately fully isolated on all IOMMU drivers. If the user attaches/detaches while owned then detach will set the group back to the blocking domain. Slightly reorganize the call chains so that __iommu_group_set_core_domain() is the function that removes any caller configured domain and sets the domains back a core owned domain with an appropriate lifetime. __iommu_group_set_domain() is the worker function that can change the domain assigned to a group to any target domain, including NULL. Add comments clarifying how the NULL vs detach_dev vs default_domain works based on Robin's remarks. This fixes an oops with VFIO and SMMUv3 because VFIO will call iommu_detach_group() and then immediately iommu_domain_free(), but SMMUv3 has no way to know that the domain it is holding a pointer to has been freed. Now the iommu_detach_group() will assign the blocking domain and SMMUv3 will no longer hold a stale domain reference. Fixes: 1ea2a07a532b ("iommu: Add DMA ownership management interfaces") Reported-by: Qian Cai Tested-by: Baolu Lu Tested-by: Nicolin Chen Co-developed-by: Robin Murphy Signed-off-by: Robin Murphy Signed-off-by: Jason Gunthorpe -- Just minor polishing as discussed v3: - Change names to __iommu_group_set_domain() / __iommu_group_set_core_domain() - Clarify comments - Call __iommu_group_set_domain() directly in iommu_group_release_dma_owner() since we know it is always selecting the default_domain - Remove redundant detach_dev ops check in __iommu_detach_device and make the added WARN_ON fail instead - Check for blocking_domain in __iommu_attach_group() so VFIO can actually attach a new group - Update comments and spelling - Fix missed change to new_domain in iommu_group_do_detach_device() v2: https://lore.kernel.org/r/0-v2-f62259511ac0+6-iommu_dma_block_jgg@nvidia.com v1: https://lore.kernel.org/r/0-v1-6e9d2d0a759d+11b-iommu_dma_block_jgg@nvidia.com Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/0-v3-db7f0785022b+149-iommu_dma_block_jgg@nvidia.com Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 127 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 91 insertions(+), 36 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 0c42ece25854..0b22e51e90f4 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -44,6 +44,7 @@ struct iommu_group { char *name; int id; struct iommu_domain *default_domain; + struct iommu_domain *blocking_domain; struct iommu_domain *domain; struct list_head entry; unsigned int owner_cnt; @@ -82,8 +83,8 @@ static int __iommu_attach_device(struct iommu_domain *domain, struct device *dev); static int __iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group); -static void __iommu_detach_group(struct iommu_domain *domain, - struct iommu_group *group); +static int __iommu_group_set_domain(struct iommu_group *group, + struct iommu_domain *new_domain); static int iommu_create_device_direct_mappings(struct iommu_group *group, struct device *dev); static struct iommu_group *iommu_group_get_for_dev(struct device *dev); @@ -596,6 +597,8 @@ static void iommu_group_release(struct kobject *kobj) if (group->default_domain) iommu_domain_free(group->default_domain); + if (group->blocking_domain) + iommu_domain_free(group->blocking_domain); kfree(group->name); kfree(group); @@ -1907,6 +1910,24 @@ void iommu_domain_free(struct iommu_domain *domain) } EXPORT_SYMBOL_GPL(iommu_domain_free); +/* + * Put the group's domain back to the appropriate core-owned domain - either the + * standard kernel-mode DMA configuration or an all-DMA-blocked domain. + */ +static void __iommu_group_set_core_domain(struct iommu_group *group) +{ + struct iommu_domain *new_domain; + int ret; + + if (group->owner) + new_domain = group->blocking_domain; + else + new_domain = group->default_domain; + + ret = __iommu_group_set_domain(group, new_domain); + WARN(ret, "iommu driver failed to attach the default/blocking domain"); +} + static int __iommu_attach_device(struct iommu_domain *domain, struct device *dev) { @@ -1963,9 +1984,6 @@ static void __iommu_detach_device(struct iommu_domain *domain, if (iommu_is_attach_deferred(dev)) return; - if (unlikely(domain->ops->detach_dev == NULL)) - return; - domain->ops->detach_dev(domain, dev); trace_detach_device_from_domain(dev); } @@ -1979,12 +1997,10 @@ void iommu_detach_device(struct iommu_domain *domain, struct device *dev) return; mutex_lock(&group->mutex); - if (iommu_group_device_count(group) != 1) { - WARN_ON(1); + if (WARN_ON(domain != group->domain) || + WARN_ON(iommu_group_device_count(group) != 1)) goto out_unlock; - } - - __iommu_detach_group(domain, group); + __iommu_group_set_core_domain(group); out_unlock: mutex_unlock(&group->mutex); @@ -2040,7 +2056,8 @@ static int __iommu_attach_group(struct iommu_domain *domain, { int ret; - if (group->domain && group->domain != group->default_domain) + if (group->domain && group->domain != group->default_domain && + group->domain != group->blocking_domain) return -EBUSY; ret = __iommu_group_for_each_dev(group, domain, @@ -2072,38 +2089,49 @@ static int iommu_group_do_detach_device(struct device *dev, void *data) return 0; } -static void __iommu_detach_group(struct iommu_domain *domain, - struct iommu_group *group) +static int __iommu_group_set_domain(struct iommu_group *group, + struct iommu_domain *new_domain) { int ret; + if (group->domain == new_domain) + return 0; + /* - * If the group has been claimed already, do not re-attach the default - * domain. + * New drivers should support default domains and so the detach_dev() op + * will never be called. Otherwise the NULL domain represents some + * platform specific behavior. */ - if (!group->default_domain || group->owner) { - __iommu_group_for_each_dev(group, domain, + if (!new_domain) { + if (WARN_ON(!group->domain->ops->detach_dev)) + return -EINVAL; + __iommu_group_for_each_dev(group, group->domain, iommu_group_do_detach_device); group->domain = NULL; - return; + return 0; } - if (group->domain == group->default_domain) - return; - - /* Detach by re-attaching to the default domain */ - ret = __iommu_group_for_each_dev(group, group->default_domain, + /* + * Changing the domain is done by calling attach_dev() on the new + * domain. This switch does not have to be atomic and DMA can be + * discarded during the transition. DMA must only be able to access + * either new_domain or group->domain, never something else. + * + * Note that this is called in error unwind paths, attaching to a + * domain that has already been attached cannot fail. + */ + ret = __iommu_group_for_each_dev(group, new_domain, iommu_group_do_attach_device); - if (ret != 0) - WARN_ON(1); - else - group->domain = group->default_domain; + if (ret) + return ret; + group->domain = new_domain; + return 0; } void iommu_detach_group(struct iommu_domain *domain, struct iommu_group *group) { mutex_lock(&group->mutex); - __iommu_detach_group(domain, group); + __iommu_group_set_core_domain(group); mutex_unlock(&group->mutex); } EXPORT_SYMBOL_GPL(iommu_detach_group); @@ -3088,6 +3116,29 @@ void iommu_device_unuse_default_domain(struct device *dev) iommu_group_put(group); } +static int __iommu_group_alloc_blocking_domain(struct iommu_group *group) +{ + struct group_device *dev = + list_first_entry(&group->devices, struct group_device, list); + + if (group->blocking_domain) + return 0; + + group->blocking_domain = + __iommu_domain_alloc(dev->dev->bus, IOMMU_DOMAIN_BLOCKED); + if (!group->blocking_domain) { + /* + * For drivers that do not yet understand IOMMU_DOMAIN_BLOCKED + * create an empty domain instead. + */ + group->blocking_domain = __iommu_domain_alloc( + dev->dev->bus, IOMMU_DOMAIN_UNMANAGED); + if (!group->blocking_domain) + return -EINVAL; + } + return 0; +} + /** * iommu_group_claim_dma_owner() - Set DMA ownership of a group * @group: The group. @@ -3111,9 +3162,14 @@ int iommu_group_claim_dma_owner(struct iommu_group *group, void *owner) goto unlock_out; } + ret = __iommu_group_alloc_blocking_domain(group); + if (ret) + goto unlock_out; + + ret = __iommu_group_set_domain(group, group->blocking_domain); + if (ret) + goto unlock_out; group->owner = owner; - if (group->domain) - __iommu_detach_group(group->domain, group); } group->owner_cnt++; @@ -3132,18 +3188,17 @@ EXPORT_SYMBOL_GPL(iommu_group_claim_dma_owner); */ void iommu_group_release_dma_owner(struct iommu_group *group) { + int ret; + mutex_lock(&group->mutex); if (WARN_ON(!group->owner_cnt || !group->owner)) goto unlock_out; group->owner_cnt = 0; - /* - * The UNMANAGED domain should be detached before all USER - * owners have been released. - */ - if (!WARN_ON(group->domain) && group->default_domain) - __iommu_attach_group(group->default_domain, group); group->owner = NULL; + ret = __iommu_group_set_domain(group, group->default_domain); + WARN(ret, "iommu driver failed to attach the default domain"); + unlock_out: mutex_unlock(&group->mutex); } -- cgit v1.2.3 From de78657e16f41417da9332f09c2d67d100096939 Mon Sep 17 00:00:00 2001 From: Miles Chen Date: Thu, 5 May 2022 21:27:30 +0800 Subject: iommu/mediatek: Fix NULL pointer dereference when printing dev_name When larbdev is NULL (in the case I hit, the node is incorrectly set iommus = <&iommu NUM>), it will cause device_link_add() fail and kernel crashes when we try to print dev_name(larbdev). Let's fail the probe if a larbdev is NULL to avoid invalid inputs from dts. It should work for normal correct setting and avoid the crash caused by my incorrect setting. Error log: [ 18.189042][ T301] Unable to handle kernel NULL pointer dereference at virtual address 0000000000000050 ... [ 18.344519][ T301] pstate: a0400005 (NzCv daif +PAN -UAO) [ 18.345213][ T301] pc : mtk_iommu_probe_device+0xf8/0x118 [mtk_iommu] [ 18.346050][ T301] lr : mtk_iommu_probe_device+0xd0/0x118 [mtk_iommu] [ 18.346884][ T301] sp : ffffffc00a5635e0 [ 18.347392][ T301] x29: ffffffc00a5635e0 x28: ffffffd44a46c1d8 [ 18.348156][ T301] x27: ffffff80c39a8000 x26: ffffffd44a80cc38 [ 18.348917][ T301] x25: 0000000000000000 x24: ffffffd44a80cc38 [ 18.349677][ T301] x23: ffffffd44e4da4c6 x22: ffffffd44a80cc38 [ 18.350438][ T301] x21: ffffff80cecd1880 x20: 0000000000000000 [ 18.351198][ T301] x19: ffffff80c439f010 x18: ffffffc00a50d0c0 [ 18.351959][ T301] x17: ffffffffffffffff x16: 0000000000000004 [ 18.352719][ T301] x15: 0000000000000004 x14: ffffffd44eb5d420 [ 18.353480][ T301] x13: 0000000000000ad2 x12: 0000000000000003 [ 18.354241][ T301] x11: 00000000fffffad2 x10: c0000000fffffad2 [ 18.355003][ T301] x9 : a0d288d8d7142d00 x8 : a0d288d8d7142d00 [ 18.355763][ T301] x7 : ffffffd44c2bc640 x6 : 0000000000000000 [ 18.356524][ T301] x5 : 0000000000000080 x4 : 0000000000000001 [ 18.357284][ T301] x3 : 0000000000000000 x2 : 0000000000000005 [ 18.358045][ T301] x1 : 0000000000000000 x0 : 0000000000000000 [ 18.360208][ T301] Hardware name: MT6873 (DT) [ 18.360771][ T301] Call trace: [ 18.361168][ T301] dump_backtrace+0xf8/0x1f0 [ 18.361737][ T301] dump_stack_lvl+0xa8/0x11c [ 18.362305][ T301] dump_stack+0x1c/0x2c [ 18.362816][ T301] mrdump_common_die+0x184/0x40c [mrdump] [ 18.363575][ T301] ipanic_die+0x24/0x38 [mrdump] [ 18.364230][ T301] atomic_notifier_call_chain+0x128/0x2b8 [ 18.364937][ T301] die+0x16c/0x568 [ 18.365394][ T301] __do_kernel_fault+0x1e8/0x214 [ 18.365402][ T301] do_page_fault+0xb8/0x678 [ 18.366934][ T301] do_translation_fault+0x48/0x64 [ 18.368645][ T301] do_mem_abort+0x68/0x148 [ 18.368652][ T301] el1_abort+0x40/0x64 [ 18.368660][ T301] el1h_64_sync_handler+0x54/0x88 [ 18.368668][ T301] el1h_64_sync+0x68/0x6c [ 18.368673][ T301] mtk_iommu_probe_device+0xf8/0x118 [mtk_iommu] ... Cc: Robin Murphy Cc: Yong Wu Reported-by: kernel test robot Fixes: 635319a4a744 ("media: iommu/mediatek: Add device_link between the consumer and the larb devices") Signed-off-by: Miles Chen Reviewed-by: Yong Wu Reviewed-by: AngeloGioacchino Del Regno Link: https://lore.kernel.org/r/20220505132731.21628-1-miles.chen@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 6 ++++++ drivers/iommu/mtk_iommu_v1.c | 7 +++++++ 2 files changed, 13 insertions(+) (limited to 'drivers') diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 71b2ace74cd6..bb9dd92c9898 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -790,6 +790,9 @@ static struct iommu_device *mtk_iommu_probe_device(struct device *dev) * All the ports in each a device should be in the same larbs. */ larbid = MTK_M4U_TO_LARB(fwspec->ids[0]); + if (larbid >= MTK_LARB_NR_MAX) + return ERR_PTR(-EINVAL); + for (i = 1; i < fwspec->num_ids; i++) { larbidx = MTK_M4U_TO_LARB(fwspec->ids[i]); if (larbid != larbidx) { @@ -799,6 +802,9 @@ static struct iommu_device *mtk_iommu_probe_device(struct device *dev) } } larbdev = data->larb_imu[larbid].dev; + if (!larbdev) + return ERR_PTR(-EINVAL); + link = device_link_add(dev, larbdev, DL_FLAG_PM_RUNTIME | DL_FLAG_STATELESS); if (!link) diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c index 62669e60991f..e1cb51b9866c 100644 --- a/drivers/iommu/mtk_iommu_v1.c +++ b/drivers/iommu/mtk_iommu_v1.c @@ -78,6 +78,7 @@ /* MTK generation one iommu HW only support 4K size mapping */ #define MT2701_IOMMU_PAGE_SHIFT 12 #define MT2701_IOMMU_PAGE_SIZE (1UL << MT2701_IOMMU_PAGE_SHIFT) +#define MT2701_LARB_NR_MAX 3 /* * MTK m4u support 4GB iova address space, and only support 4K page @@ -486,6 +487,9 @@ static struct iommu_device *mtk_iommu_v1_probe_device(struct device *dev) /* Link the consumer device with the smi-larb device(supplier) */ larbid = mt2701_m4u_to_larb(fwspec->ids[0]); + if (larbid >= MT2701_LARB_NR_MAX) + return ERR_PTR(-EINVAL); + for (idx = 1; idx < fwspec->num_ids; idx++) { larbidx = mt2701_m4u_to_larb(fwspec->ids[idx]); if (larbid != larbidx) { @@ -496,6 +500,9 @@ static struct iommu_device *mtk_iommu_v1_probe_device(struct device *dev) } larbdev = data->larb_imu[larbid].dev; + if (!larbdev) + return ERR_PTR(-EINVAL); + link = device_link_add(dev, larbdev, DL_FLAG_PM_RUNTIME | DL_FLAG_STATELESS); if (!link) -- cgit v1.2.3 From a3884774d731f03d3a3dd4fb70ec2d9341ceb39d Mon Sep 17 00:00:00 2001 From: Yunfei Wang Date: Sat, 7 May 2022 16:52:03 +0800 Subject: iommu/dma: Fix iova map result check bug The data type of the return value of the iommu_map_sg_atomic is ssize_t, but the data type of iova size is size_t, e.g. one is int while the other is unsigned int. When iommu_map_sg_atomic return value is compared with iova size, it will force the signed int to be converted to unsigned int, if iova map fails and iommu_map_sg_atomic return error code is less than 0, then (ret < iova_len) is false, which will to cause not do free iova, and the master can still successfully get the iova of map fail, which is not expected. Therefore, we need to check the return value of iommu_map_sg_atomic in two cases according to whether it is less than 0. Fixes: ad8f36e4b6b1 ("iommu: return full error code from iommu_map_sg[_atomic]()") Signed-off-by: Yunfei Wang Cc: # 5.15.* Reviewed-by: Robin Murphy Reviewed-by: Miles Chen Link: https://lore.kernel.org/r/20220507085204.16914-1-yf.wang@mediatek.com Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 1ca85d37eeab..8e0ed400a439 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -776,6 +776,7 @@ static struct page **__iommu_dma_alloc_noncontiguous(struct device *dev, unsigned int count, min_size, alloc_sizes = domain->pgsize_bitmap; struct page **pages; dma_addr_t iova; + ssize_t ret; if (static_branch_unlikely(&iommu_deferred_attach_enabled) && iommu_deferred_attach(dev, domain)) @@ -813,8 +814,8 @@ static struct page **__iommu_dma_alloc_noncontiguous(struct device *dev, arch_dma_prep_coherent(sg_page(sg), sg->length); } - if (iommu_map_sg_atomic(domain, iova, sgt->sgl, sgt->orig_nents, ioprot) - < size) + ret = iommu_map_sg_atomic(domain, iova, sgt->sgl, sgt->orig_nents, ioprot); + if (ret < 0 || ret < size) goto out_free_sg; sgt->sgl->dma_address = iova; @@ -1214,7 +1215,7 @@ static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, * implementation - it knows better than we do. */ ret = iommu_map_sg_atomic(domain, iova, sg, nents, prot); - if (ret < iova_len) + if (ret < 0 || ret < iova_len) goto out_free_iova; return __finalise_sg(dev, sg, nents, iova); -- cgit v1.2.3 From b8397a8f4ebc0b84eefd990dc08995ba2ae9015c Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 9 May 2022 11:16:08 +0100 Subject: iommu/dma: Explicitly sort PCI DMA windows Originally, creating the dma_ranges resource list in pre-sorted fashion was the simplest and most efficient way to enforce the order required by iova_reserve_pci_windows(). However since then at least one PCI host driver is now re-sorting the list for its own probe-time processing, which doesn't seem entirely unreasonable, so that basic assumption no longer holds. Make iommu-dma robust and get the sort order it needs by explicitly sorting, which means we can also save the effort at creation time and just build the list in whatever natural order the DT had. Signed-off-by: Robin Murphy Reviewed-by: Rob Herring Link: https://lore.kernel.org/r/35661036a7e4160850895f9b37f35408b6a29f2f.1652091160.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 13 ++++++++++++- drivers/pci/of.c | 8 +------- 2 files changed, 13 insertions(+), 8 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 8e0ed400a439..f90251572a5d 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -414,6 +415,15 @@ static int cookie_init_hw_msi_region(struct iommu_dma_cookie *cookie, return 0; } +static int iommu_dma_ranges_sort(void *priv, const struct list_head *a, + const struct list_head *b) +{ + struct resource_entry *res_a = list_entry(a, typeof(*res_a), node); + struct resource_entry *res_b = list_entry(b, typeof(*res_b), node); + + return res_a->res->start > res_b->res->start; +} + static int iova_reserve_pci_windows(struct pci_dev *dev, struct iova_domain *iovad) { @@ -432,6 +442,7 @@ static int iova_reserve_pci_windows(struct pci_dev *dev, } /* Get reserved DMA windows from host bridge */ + list_sort(NULL, &bridge->dma_ranges, iommu_dma_ranges_sort); resource_list_for_each_entry(window, &bridge->dma_ranges) { end = window->res->start - window->offset; resv_iova: @@ -440,7 +451,7 @@ resv_iova: hi = iova_pfn(iovad, end); reserve_iova(iovad, lo, hi); } else if (end < start) { - /* dma_ranges list should be sorted */ + /* DMA ranges should be non-overlapping */ dev_err(&dev->dev, "Failed to reserve IOVA [%pa-%pa]\n", &start, &end); diff --git a/drivers/pci/of.c b/drivers/pci/of.c index cb2e8351c2cc..8f0ebaf9ae85 100644 --- a/drivers/pci/of.c +++ b/drivers/pci/of.c @@ -369,7 +369,6 @@ static int devm_of_pci_get_host_bridge_resources(struct device *dev, dev_dbg(dev, "Parsing dma-ranges property...\n"); for_each_of_pci_range(&parser, &range) { - struct resource_entry *entry; /* * If we failed translation or got a zero-sized region * then skip this range @@ -393,12 +392,7 @@ static int devm_of_pci_get_host_bridge_resources(struct device *dev, goto failed; } - /* Keep the resource list sorted */ - resource_list_for_each_entry(entry, ib_resources) - if (entry->res->start > res->start) - break; - - pci_add_resource_offset(&entry->node, res, + pci_add_resource_offset(ib_resources, res, res->start - range.pci_addr); } -- cgit v1.2.3 From cd901e9284c7893eebee6857ebe30b691ea91224 Mon Sep 17 00:00:00 2001 From: Muhammad Usama Anjum Date: Tue, 10 May 2022 10:34:00 +0800 Subject: iommu/vt-d: Remove unneeded validity check on dev dev_iommu_priv_get() is being used at the top of this function which dereferences dev. Dev cannot be NULL after this. Remove the validity check on dev and simplify the code. Signed-off-by: Muhammad Usama Anjum Link: https://lore.kernel.org/r/20220313150337.593650-1-usama.anjum@collabora.com Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20220510023407.2759143-2-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index efcecfa5952a..b5caea9a7e78 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -2501,7 +2501,7 @@ static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu, } } - if (dev && domain_context_mapping(domain, dev)) { + if (domain_context_mapping(domain, dev)) { dev_err(dev, "Domain context map failed\n"); dmar_remove_one_dev_info(dev); return NULL; -- cgit v1.2.3 From e19c3992b9f8bef9c8c4aa59fa30a83e7e35eccc Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Tue, 10 May 2022 10:34:01 +0800 Subject: iommu/vt-d: Change return type of dmar_insert_one_dev_info() The dmar_insert_one_dev_info() returns the pass-in domain on success and NULL on failure. This doesn't make much sense. Change it to an integer. Signed-off-by: Lu Baolu Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20220416120423.879552-1-baolu.lu@linux.intel.com Link: https://lore.kernel.org/r/20220510023407.2759143-3-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index b5caea9a7e78..6bb229dbd0d9 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -2452,10 +2452,9 @@ static bool dev_is_real_dma_subdevice(struct device *dev) pci_real_dma_dev(to_pci_dev(dev)) != to_pci_dev(dev); } -static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu, - int bus, int devfn, - struct device *dev, - struct dmar_domain *domain) +static int dmar_insert_one_dev_info(struct intel_iommu *iommu, int bus, + int devfn, struct device *dev, + struct dmar_domain *domain) { struct device_domain_info *info = dev_iommu_priv_get(dev); unsigned long flags; @@ -2468,7 +2467,7 @@ static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu, spin_unlock(&iommu->lock); if (ret) { spin_unlock_irqrestore(&device_domain_lock, flags); - return NULL; + return ret; } list_add(&info->link, &domain->devices); spin_unlock_irqrestore(&device_domain_lock, flags); @@ -2479,7 +2478,7 @@ static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu, if (ret) { dev_err(dev, "PASID table allocation failed\n"); dmar_remove_one_dev_info(dev); - return NULL; + return ret; } /* Setup the PASID entry for requests without PASID: */ @@ -2497,17 +2496,18 @@ static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu, if (ret) { dev_err(dev, "Setup RID2PASID failed\n"); dmar_remove_one_dev_info(dev); - return NULL; + return ret; } } - if (domain_context_mapping(domain, dev)) { + ret = domain_context_mapping(domain, dev); + if (ret) { dev_err(dev, "Domain context map failed\n"); dmar_remove_one_dev_info(dev); - return NULL; + return ret; } - return domain; + return 0; } static int iommu_domain_identity_map(struct dmar_domain *domain, @@ -2585,7 +2585,6 @@ static int __init si_domain_init(int hw) static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev) { - struct dmar_domain *ndomain; struct intel_iommu *iommu; u8 bus, devfn; @@ -2593,11 +2592,7 @@ static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev) if (!iommu) return -ENODEV; - ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain); - if (ndomain != domain) - return -EBUSY; - - return 0; + return dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain); } static bool device_has_rmrr(struct device *dev) -- cgit v1.2.3 From bac4e778d67a5f0fd8cc49029c1fff127b86c874 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Tue, 10 May 2022 10:34:02 +0800 Subject: iommu/vt-d: Fold dmar_insert_one_dev_info() into its caller Fold dmar_insert_one_dev_info() into domain_add_dev_info() which is its only caller. No intentional functional impact. Suggested-by: Christoph Hellwig Signed-off-by: Lu Baolu Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20220416120423.879552-1-baolu.lu@linux.intel.com Link: https://lore.kernel.org/r/20220510023407.2759143-4-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 110 ++++++++++++++++++++------------------------ 1 file changed, 51 insertions(+), 59 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 6bb229dbd0d9..f200af2233d0 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -2452,64 +2452,6 @@ static bool dev_is_real_dma_subdevice(struct device *dev) pci_real_dma_dev(to_pci_dev(dev)) != to_pci_dev(dev); } -static int dmar_insert_one_dev_info(struct intel_iommu *iommu, int bus, - int devfn, struct device *dev, - struct dmar_domain *domain) -{ - struct device_domain_info *info = dev_iommu_priv_get(dev); - unsigned long flags; - int ret; - - spin_lock_irqsave(&device_domain_lock, flags); - info->domain = domain; - spin_lock(&iommu->lock); - ret = domain_attach_iommu(domain, iommu); - spin_unlock(&iommu->lock); - if (ret) { - spin_unlock_irqrestore(&device_domain_lock, flags); - return ret; - } - list_add(&info->link, &domain->devices); - spin_unlock_irqrestore(&device_domain_lock, flags); - - /* PASID table is mandatory for a PCI device in scalable mode. */ - if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev)) { - ret = intel_pasid_alloc_table(dev); - if (ret) { - dev_err(dev, "PASID table allocation failed\n"); - dmar_remove_one_dev_info(dev); - return ret; - } - - /* Setup the PASID entry for requests without PASID: */ - spin_lock_irqsave(&iommu->lock, flags); - if (hw_pass_through && domain_type_is_si(domain)) - ret = intel_pasid_setup_pass_through(iommu, domain, - dev, PASID_RID2PASID); - else if (domain_use_first_level(domain)) - ret = domain_setup_first_level(iommu, domain, dev, - PASID_RID2PASID); - else - ret = intel_pasid_setup_second_level(iommu, domain, - dev, PASID_RID2PASID); - spin_unlock_irqrestore(&iommu->lock, flags); - if (ret) { - dev_err(dev, "Setup RID2PASID failed\n"); - dmar_remove_one_dev_info(dev); - return ret; - } - } - - ret = domain_context_mapping(domain, dev); - if (ret) { - dev_err(dev, "Domain context map failed\n"); - dmar_remove_one_dev_info(dev); - return ret; - } - - return 0; -} - static int iommu_domain_identity_map(struct dmar_domain *domain, unsigned long first_vpfn, unsigned long last_vpfn) @@ -2585,14 +2527,64 @@ static int __init si_domain_init(int hw) static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev) { + struct device_domain_info *info = dev_iommu_priv_get(dev); struct intel_iommu *iommu; + unsigned long flags; u8 bus, devfn; + int ret; iommu = device_to_iommu(dev, &bus, &devfn); if (!iommu) return -ENODEV; - return dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain); + spin_lock_irqsave(&device_domain_lock, flags); + info->domain = domain; + spin_lock(&iommu->lock); + ret = domain_attach_iommu(domain, iommu); + spin_unlock(&iommu->lock); + if (ret) { + spin_unlock_irqrestore(&device_domain_lock, flags); + return ret; + } + list_add(&info->link, &domain->devices); + spin_unlock_irqrestore(&device_domain_lock, flags); + + /* PASID table is mandatory for a PCI device in scalable mode. */ + if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev)) { + ret = intel_pasid_alloc_table(dev); + if (ret) { + dev_err(dev, "PASID table allocation failed\n"); + dmar_remove_one_dev_info(dev); + return ret; + } + + /* Setup the PASID entry for requests without PASID: */ + spin_lock_irqsave(&iommu->lock, flags); + if (hw_pass_through && domain_type_is_si(domain)) + ret = intel_pasid_setup_pass_through(iommu, domain, + dev, PASID_RID2PASID); + else if (domain_use_first_level(domain)) + ret = domain_setup_first_level(iommu, domain, dev, + PASID_RID2PASID); + else + ret = intel_pasid_setup_second_level(iommu, domain, + dev, PASID_RID2PASID); + spin_unlock_irqrestore(&iommu->lock, flags); + if (ret) { + dev_err(dev, "Setup RID2PASID failed\n"); + dmar_remove_one_dev_info(dev); + return ret; + } + } + + ret = domain_context_mapping(domain, dev); + if (ret) { + dev_err(dev, "Domain context map failed\n"); + dmar_remove_one_dev_info(dev); + return ret; + } + + return 0; } static bool device_has_rmrr(struct device *dev) -- cgit v1.2.3 From 9d6ab26a75f47332da9949a0112727e10f2d6e51 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Tue, 10 May 2022 10:34:04 +0800 Subject: iommu/vt-d: Block force-snoop domain attaching if no SC support In the attach_dev callback of the default domain ops, if the domain has been set force_snooping, but the iommu hardware of the device does not support SC(Snoop Control) capability, the callback should block it and return a corresponding error code. Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20220508123525.1973626-1-baolu.lu@linux.intel.com Link: https://lore.kernel.org/r/20220510023407.2759143-6-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'drivers') diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index f200af2233d0..23985cb5c8ee 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -4333,6 +4333,9 @@ static int prepare_domain_attach_device(struct iommu_domain *domain, if (!iommu) return -ENODEV; + if (dmar_domain->force_snooping && !ecap_sc_support(iommu->ecap)) + return -EOPNOTSUPP; + /* check if this iommu agaw is sufficient for max mapped address */ addr_width = agaw_to_width(iommu->agaw); if (addr_width > cap_mgaw(iommu->cap)) -- cgit v1.2.3 From fc0051cb95909ab56bd8c929f24d48c9870c3e3a Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Tue, 10 May 2022 10:34:05 +0800 Subject: iommu/vt-d: Check domain force_snooping against attached devices As domain->force_snooping only impacts the devices attached with the domain, there's no need to check against all IOMMU units. On the other hand, force_snooping could be set on a domain no matter whether it has been attached or not, and once set it is an immutable flag. If no device attached, the operation always succeeds. Then this empty domain can be only attached to a device of which the IOMMU supports snoop control. Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20220508123525.1973626-1-baolu.lu@linux.intel.com Link: https://lore.kernel.org/r/20220510023407.2759143-7-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 53 ++++++++++++++++++++++++++++++++++++++++++--- drivers/iommu/intel/pasid.c | 42 +++++++++++++++++++++++++++++++++++ drivers/iommu/intel/pasid.h | 2 ++ include/linux/intel-iommu.h | 1 + 4 files changed, 95 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 23985cb5c8ee..f366e8c49636 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -2438,7 +2438,7 @@ static int domain_setup_first_level(struct intel_iommu *iommu, if (level == 5) flags |= PASID_FLAG_FL5LP; - if (domain->domain.type == IOMMU_DOMAIN_UNMANAGED) + if (domain->force_snooping) flags |= PASID_FLAG_PAGE_SNOOP; return intel_pasid_setup_first_level(iommu, dev, (pgd_t *)pgd, pasid, @@ -4410,7 +4410,7 @@ static int intel_iommu_map(struct iommu_domain *domain, prot |= DMA_PTE_READ; if (iommu_prot & IOMMU_WRITE) prot |= DMA_PTE_WRITE; - if (dmar_domain->force_snooping) + if (dmar_domain->set_pte_snp) prot |= DMA_PTE_SNP; max_addr = iova + size; @@ -4533,13 +4533,60 @@ static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain, return phys; } +static bool domain_support_force_snooping(struct dmar_domain *domain) +{ + struct device_domain_info *info; + bool support = true; + + assert_spin_locked(&device_domain_lock); + list_for_each_entry(info, &domain->devices, link) { + if (!ecap_sc_support(info->iommu->ecap)) { + support = false; + break; + } + } + + return support; +} + +static void domain_set_force_snooping(struct dmar_domain *domain) +{ + struct device_domain_info *info; + + assert_spin_locked(&device_domain_lock); + + /* + * Second level page table supports per-PTE snoop control. The + * iommu_map() interface will handle this by setting SNP bit. + */ + if (!domain_use_first_level(domain)) { + domain->set_pte_snp = true; + return; + } + + list_for_each_entry(info, &domain->devices, link) + intel_pasid_setup_page_snoop_control(info->iommu, info->dev, + PASID_RID2PASID); +} + static bool intel_iommu_enforce_cache_coherency(struct iommu_domain *domain) { struct dmar_domain *dmar_domain = to_dmar_domain(domain); + unsigned long flags; - if (!domain_update_iommu_snooping(NULL)) + if (dmar_domain->force_snooping) + return true; + + spin_lock_irqsave(&device_domain_lock, flags); + if (!domain_support_force_snooping(dmar_domain)) { + spin_unlock_irqrestore(&device_domain_lock, flags); return false; + } + + domain_set_force_snooping(dmar_domain); dmar_domain->force_snooping = true; + spin_unlock_irqrestore(&device_domain_lock, flags); + return true; } diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index f8d215d85695..d19dd66a670c 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -762,3 +762,45 @@ int intel_pasid_setup_pass_through(struct intel_iommu *iommu, return 0; } + +/* + * Set the page snoop control for a pasid entry which has been set up. + */ +void intel_pasid_setup_page_snoop_control(struct intel_iommu *iommu, + struct device *dev, u32 pasid) +{ + struct pasid_entry *pte; + u16 did; + + spin_lock(&iommu->lock); + pte = intel_pasid_get_entry(dev, pasid); + if (WARN_ON(!pte || !pasid_pte_is_present(pte))) { + spin_unlock(&iommu->lock); + return; + } + + pasid_set_pgsnp(pte); + did = pasid_get_domain_id(pte); + spin_unlock(&iommu->lock); + + if (!ecap_coherent(iommu->ecap)) + clflush_cache_range(pte, sizeof(*pte)); + + /* + * VT-d spec 3.4 table23 states guides for cache invalidation: + * + * - PASID-selective-within-Domain PASID-cache invalidation + * - PASID-selective PASID-based IOTLB invalidation + * - If (pasid is RID_PASID) + * - Global Device-TLB invalidation to affected functions + * Else + * - PASID-based Device-TLB invalidation (with S=1 and + * Addr[63:12]=0x7FFFFFFF_FFFFF) to affected functions + */ + pasid_cache_invalidation_with_pasid(iommu, did, pasid); + qi_flush_piotlb(iommu, did, pasid, 0, -1, 0); + + /* Device IOTLB doesn't need to be flushed in caching mode. */ + if (!cap_caching_mode(iommu->cap)) + devtlb_invalidation_with_pasid(iommu, dev, pasid); +} diff --git a/drivers/iommu/intel/pasid.h b/drivers/iommu/intel/pasid.h index ab4408c824a5..583ea67fc783 100644 --- a/drivers/iommu/intel/pasid.h +++ b/drivers/iommu/intel/pasid.h @@ -123,4 +123,6 @@ void intel_pasid_tear_down_entry(struct intel_iommu *iommu, bool fault_ignore); int vcmd_alloc_pasid(struct intel_iommu *iommu, u32 *pasid); void vcmd_free_pasid(struct intel_iommu *iommu, u32 pasid); +void intel_pasid_setup_page_snoop_control(struct intel_iommu *iommu, + struct device *dev, u32 pasid); #endif /* __INTEL_PASID_H */ diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index 72e5d7900e71..4f29139bbfc3 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -540,6 +540,7 @@ struct dmar_domain { u8 has_iotlb_device: 1; u8 iommu_coherency: 1; /* indicate coherency of iommu access */ u8 force_snooping : 1; /* Create IOPTEs with snoop control */ + u8 set_pte_snp:1; struct list_head devices; /* all devices' list */ struct iova_domain iovad; /* iova's that belong to this domain */ -- cgit v1.2.3 From e80552267b63df211c805e8e8c534bfe5909b7e7 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Tue, 10 May 2022 10:34:06 +0800 Subject: iommu/vt-d: Remove domain_update_iommu_snooping() The IOMMU force snooping capability is not required to be consistent among all the IOMMUs anymore. Remove force snooping capability check in the IOMMU hot-add path and domain_update_iommu_snooping() becomes a dead code now. Signed-off-by: Lu Baolu Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20220508123525.1973626-1-baolu.lu@linux.intel.com Link: https://lore.kernel.org/r/20220510023407.2759143-8-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 34 +--------------------------------- 1 file changed, 1 insertion(+), 33 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index f366e8c49636..985a83e24244 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -533,33 +533,6 @@ static void domain_update_iommu_coherency(struct dmar_domain *domain) rcu_read_unlock(); } -static bool domain_update_iommu_snooping(struct intel_iommu *skip) -{ - struct dmar_drhd_unit *drhd; - struct intel_iommu *iommu; - bool ret = true; - - rcu_read_lock(); - for_each_active_iommu(iommu, drhd) { - if (iommu != skip) { - /* - * If the hardware is operating in the scalable mode, - * the snooping control is always supported since we - * always set PASID-table-entry.PGSNP bit if the domain - * is managed outside (UNMANAGED). - */ - if (!sm_supported(iommu) && - !ecap_sc_support(iommu->ecap)) { - ret = false; - break; - } - } - } - rcu_read_unlock(); - - return ret; -} - static int domain_update_iommu_superpage(struct dmar_domain *domain, struct intel_iommu *skip) { @@ -3572,12 +3545,7 @@ static int intel_iommu_add(struct dmar_drhd_unit *dmaru) iommu->name); return -ENXIO; } - if (!ecap_sc_support(iommu->ecap) && - domain_update_iommu_snooping(iommu)) { - pr_warn("%s: Doesn't support snooping.\n", - iommu->name); - return -ENXIO; - } + sp = domain_update_iommu_superpage(NULL, iommu) - 1; if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) { pr_warn("%s: Doesn't support large page.\n", -- cgit v1.2.3 From 0d647b33e74f7cb98b7c74d638922d6c03bfdb94 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Tue, 10 May 2022 10:34:07 +0800 Subject: iommu/vt-d: Remove hard coding PGSNP bit in PASID entries As enforce_cache_coherency has been introduced into the iommu_domain_ops, the kernel component which owns the iommu domain is able to opt-in its requirement for force snooping support. The iommu driver has no need to hard code the page snoop control bit in the PASID table entries anymore. Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20220508123525.1973626-1-baolu.lu@linux.intel.com Link: https://lore.kernel.org/r/20220510023407.2759143-9-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/pasid.c | 3 --- 1 file changed, 3 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index d19dd66a670c..cb4c1d0cf25c 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -710,9 +710,6 @@ int intel_pasid_setup_second_level(struct intel_iommu *iommu, pasid_set_fault_enable(pte); pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap)); - if (domain->domain.type == IOMMU_DOMAIN_UNMANAGED) - pasid_set_pgsnp(pte); - /* * Since it is a second level only translation setup, we should * set SRE bit as well (addresses are expected to be GPAs). -- cgit v1.2.3 From fa7e9ecc5e1c1a1e8aa7014b2749b22edc801dd2 Mon Sep 17 00:00:00 2001 From: Matthew Rosato Date: Thu, 19 May 2022 14:29:29 -0400 Subject: iommu/s390: Tolerate repeat attach_dev calls Since commit 0286300e6045 ("iommu: iommu_group_claim_dma_owner() must always assign a domain") s390-iommu will get called to allocate multiple unmanaged iommu domains for a vfio-pci device -- however the current s390-iommu logic tolerates only one. Recognize that multiple domains can be allocated and handle switching between DMA or different iommu domain tables during attach_dev. Signed-off-by: Matthew Rosato Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/20220519182929.581898-1-mjrosato@linux.ibm.com Signed-off-by: Joerg Roedel --- drivers/iommu/s390-iommu.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/s390-iommu.c b/drivers/iommu/s390-iommu.c index 3833e86c6e7b..c898bcbbce11 100644 --- a/drivers/iommu/s390-iommu.c +++ b/drivers/iommu/s390-iommu.c @@ -99,7 +99,7 @@ static int s390_iommu_attach_device(struct iommu_domain *domain, if (!domain_device) return -ENOMEM; - if (zdev->dma_table) { + if (zdev->dma_table && !zdev->s390_domain) { cc = zpci_dma_exit_device(zdev); if (cc) { rc = -EIO; @@ -107,6 +107,9 @@ static int s390_iommu_attach_device(struct iommu_domain *domain, } } + if (zdev->s390_domain) + zpci_unregister_ioat(zdev, 0); + zdev->dma_table = s390_domain->dma_table; cc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma, virt_to_phys(zdev->dma_table)); @@ -136,7 +139,13 @@ static int s390_iommu_attach_device(struct iommu_domain *domain, return 0; out_restore: - zpci_dma_init_device(zdev); + if (!zdev->s390_domain) { + zpci_dma_init_device(zdev); + } else { + zdev->dma_table = zdev->s390_domain->dma_table; + zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma, + virt_to_phys(zdev->dma_table)); + } out_free: kfree(domain_device); @@ -167,7 +176,7 @@ static void s390_iommu_detach_device(struct iommu_domain *domain, } spin_unlock_irqrestore(&s390_domain->list_lock, flags); - if (found) { + if (found && (zdev->s390_domain == s390_domain)) { zdev->s390_domain = NULL; zpci_unregister_ioat(zdev, 0); zpci_dma_init_device(zdev); -- cgit v1.2.3 From 42bb5aa043382f09bef2cc33b8431be867c70f8e Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 20 May 2022 12:22:14 +0200 Subject: iommu/amd: Increase timeout waiting for GA log enablement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On some systems it can take a long time for the hardware to enable the GA log of the AMD IOMMU. The current wait time is only 0.1ms, but testing showed that it can take up to 14ms for the GA log to enter running state after it has been enabled. Sometimes the long delay happens when booting the system, sometimes only on resume. Adjust the timeout accordingly to not print a warning when hardware takes a longer than usual. There has already been an attempt to fix this with commit 9b45a7738eec ("iommu/amd: Fix loop timeout issue in iommu_ga_log_enable()") But that commit was based on some wrong math and did not fix the issue in all cases. Cc: "D. Ziegfeld" Cc: Jörg-Volker Peetz Fixes: 8bda0cfbdc1a ("iommu/amd: Detect and initialize guest vAPIC log") Signed-off-by: Joerg Roedel Link: https://lore.kernel.org/r/20220520102214.12563-1-joro@8bytes.org --- drivers/iommu/amd/init.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 648d6b94ba8c..b69f6ab440dc 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -84,7 +84,7 @@ #define ACPI_DEVFLAG_LINT1 0x80 #define ACPI_DEVFLAG_ATSDIS 0x10000000 -#define LOOP_TIMEOUT 100000 +#define LOOP_TIMEOUT 2000000 /* * ACPI table definitions * -- cgit v1.2.3