iommu: SVA and IOPF refactoring
Subsystems (9)
Lu Baolu
maintainerv14 cover iommu: SVA and IOPF refactoring
Hi Joerg,
This is a resent version of v14 which was posted here:
https://lore.kernel.org/linux-iommu/[email protected]/
No further comments since the last post. So I rebased it on the top of v6.1-rc3 so that you can easily take it for v6.2 if you have interest.
It's purely a rebased version, no functional change. Please refer to the original post for series description and change history.
This series is also available on github:
https://github.com/LuBaolu/intel-iommu/commits/iommu-sva-refactoring-v14-resend
Please consider it for your iommu tree.
Best regards,
baolu
Lu Baolu (13):
iommu: Add max_pasids field in struct iommu_device
iommu: Add max_pasids field in struct dev_iommu
iommu: Remove SVM_FLAG_SUPERVISOR_MODE support
PCI: Enable PASID only when ACS RR & UF enabled on upstream path
iommu: Add attach/detach_dev_pasid iommu interfaces
iommu: Add IOMMU SVA domain support
iommu/vt-d: Add SVA domain support
arm-smmu-v3/sva: Add SVA domain support
iommu/sva: Refactoring iommu_sva_bind/unbind_device()
iommu: Remove SVA related callbacks from iommu ops
iommu: Prepare IOMMU domain for IOPF
iommu: Per-domain I/O page fault handling
iommu: Rename iommu-sva-lib.{c,h}
include/linux/intel-svm.h | 13 -
include/linux/iommu.h | 116 ++++++--
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 23 +-
drivers/iommu/intel/iommu.h | 18 +-
.../iommu/{iommu-sva-lib.h => iommu-sva.h} | 14 +-
drivers/dma/idxd/cdev.c | 3 +-
drivers/dma/idxd/init.c | 25 +-
.../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 104 ++++---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 21 +-
drivers/iommu/intel/dmar.c | 7 +
drivers/iommu/intel/iommu.c | 30 +-
drivers/iommu/intel/svm.c | 145 ++++-----
drivers/iommu/io-pgfault.c | 77 ++---
drivers/iommu/iommu-sva-lib.c | 71 -----
drivers/iommu/iommu-sva.c | 240 +++++++++++++++
drivers/iommu/iommu.c | 277 ++++++++++++------
drivers/misc/uacce/uacce.c | 2 +-
drivers/pci/ats.c | 3 +
drivers/iommu/Makefile | 2 +-
19 files changed, 732 insertions(+), 459 deletions(-)
rename drivers/iommu/{iommu-sva-lib.h => iommu-sva.h} (83%)
delete mode 100644 drivers/iommu/iommu-sva-lib.c
create mode 100644 drivers/iommu/iommu-sva.cSignature · 1 line
2.34.1
Lu Baolu
maintainerPATCH RESEND v14 1/13 iommu: Add max_pasids field in struct iommu_device
Use this field to keep the number of supported PASIDs that an IOMMU hardware is able to support. This is a generic attribute of an IOMMU and lifting it into the per-IOMMU device structure makes it possible to allocate a PASID for device without calls into the IOMMU drivers. Any iommu driver that supports PASID related features should set this field before enabling them on the devices.
In the Intel IOMMU driver, intel_iommu_sm is moved to CONFIG_INTEL_IOMMU enclave so that the pasid_supported() helper could be used in dmar.c without compilation errors.
--- include/linux/iommu.h | 2 ++ drivers/iommu/intel/iommu.h | 4 ++-- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 1 + drivers/iommu/intel/dmar.c | 7 +++++++ 4 files changed, 12 insertions(+), 2 deletions(-)
include/linux/iommu.h +2 -0
@@ -322,12 +322,14 @@ struct iommu_domain_ops {* @list: Used by the iommu-core to keep a list of registered iommus * @ops: iommu-ops for talking to this iommu * @dev: struct device for sysfs handling +* @max_pasids: number of supported PASIDs */ struct iommu_device {struct list_head list;const struct iommu_ops *ops;struct fwnode_handle *fwnode;struct device *dev; +u32 max_pasids; };/**
drivers/iommu/intel/iommu.h +2 -2
@@ -480,8 +480,6 @@ enum {#define VTD_FLAG_IRQ_REMAP_PRE_ENABLED (1 << 1)#define VTD_FLAG_SVM_CAPABLE (1 << 2) -extern int intel_iommu_sm; -#define sm_supported (iommu) (intel_iommu_sm && ecap_smts((iommu)->ecap))#define pasid_supported (iommu) (sm_supported(iommu) && \ ecap_pasid((iommu)->ecap))
@@ -795,6 +793,7 @@ struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus,extern const struct iommu_ops intel_iommu_ops;#ifdef CONFIG_INTEL_IOMMU +extern int intel_iommu_sm;extern int iommu_calculate_agaw (struct intel_iommu *iommu);extern int iommu_calculate_max_sagaw (struct intel_iommu *iommu);extern int dmar_disabled;
@@ -810,6 +809,7 @@ static inline int iommu_calculate_max_sagaw(struct intel_iommu *iommu) }#define dmar_disabled (1)#define intel_iommu_enabled (0) +#define intel_iommu_sm (0)#endif static inline const char *decode_prq_descriptor(char *str, size_t size,
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +1 -0
@@ -3543,6 +3543,7 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu)/* SID/SSID sizes */ smmu->ssid_bits =FIELD_GET (IDR1_SSIDSIZE , reg); smmu->sid_bits =FIELD_GET (IDR1_SIDSIZE , reg); + smmu->iommu.max_pasids =1UL << smmu->ssid_bits;/* * If the SMMU supports fewer bits than would fill a single L2 stream
drivers/iommu/intel/dmar.c +7 -0
@@ -1104,6 +1104,13 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd)raw_spin_lock_init (&iommu->register_lock); +/* +* A value of N in PSS field of eCap register indicates hardware +* supports PASID field of N+1 bits. +*/ +if (pasid_supported (iommu)) + iommu->iommu.max_pasids =2UL <<ecap_pss (iommu->ecap); +/* * This is only for hotplug; at boot time intel_iommu_enabled won't * be set yet. When intel_iommu_init() runs, it registers the units
Signature · 1 line
2.34.1
Lu Baolu
maintainerPATCH RESEND v14 2/13 iommu: Add max_pasids field in struct dev_iommu
In reply to Lu BaoluUse this field to save the number of PASIDs that a device is able to consume. It is a generic attribute of a device and lifting it into the per-device dev_iommu struct could help to avoid the boilerplate code in various IOMMU drivers.
--- include/linux/iommu.h | 2 ++ drivers/iommu/iommu.c | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+)
include/linux/iommu.h +2 -0
@@ -368,6 +368,7 @@ struct iommu_fault_param {* @fwspec: IOMMU fwspec data * @iommu_dev: IOMMU device this device is linked to * @priv: IOMMU Driver private data +* @max_pasids: number of PASIDs this device can consume * * TODO: migrate other per device data pointers under iommu_dev_data, e.g. * struct iommu_group *iommu_group;
@@ -379,6 +380,7 @@ struct dev_iommu {struct iommu_fwspec *fwspec;struct iommu_device *iommu_dev;void *priv; +u32 max_pasids; }; int iommu_device_register(struct iommu_device *iommu,
drivers/iommu/iommu.c +20 -0
@@ -21,6 +21,7 @@#include <linux/idr.h> #include <linux/err.h> #include <linux/pci.h> +#include <linux/pci-ats.h> #include <linux/bitops.h> #include <linux/platform_device.h> #include <linux/property.h>
@@ -278,6 +279,24 @@ static void dev_iommu_free(struct device *dev)kfree (param); } +static u32 dev_iommu_get_max_pasids (struct device *dev) +{ +u32 max_pasids =0 , bits =0 ; +int ret; + +if (dev_is_pci (dev)) { + ret =pci_max_pasids (to_pci_dev (dev)); +if (ret >0 ) + max_pasids = ret; + }else { + ret =device_property_read_u32 (dev,"pasid-num-bits" , &bits); +if (!ret) + max_pasids =1UL << bits; + } + +return min_t (u32, max_pasids, dev->iommu->iommu_dev->max_pasids); +} +static int __iommu_probe_device (struct device *dev,struct list_head *group_list) {const struct iommu_ops *ops = dev->bus->iommu_ops;
@@ -303,6 +322,7 @@ static int __iommu_probe_device(struct device *dev, struct list_head *group_list } dev->iommu->iommu_dev = iommu_dev; + dev->iommu->max_pasids =dev_iommu_get_max_pasids (dev); group =iommu_group_get_for_dev (dev);if (IS_ERR (group)) {
Signature · 1 line
2.34.1
Lu Baolu
maintainerPATCH RESEND v14 3/13 iommu: Remove SVM_FLAG_SUPERVISOR_MODE support
In reply to Lu BaoluThe current kernel DMA with PASID support is based on the SVA with a flag SVM_FLAG_SUPERVISOR_MODE. The IOMMU driver binds the kernel memory address space to a PASID of the device. The device driver programs the device with kernel virtual address (KVA) for DMA access. There have been security and functional issues with this approach:
- The lack of IOTLB synchronization upon kernel page table updates. (vmalloc, module/BPF loading, CONFIG_DEBUG_PAGEALLOC etc.)
- Other than slight more protection, using kernel virtual address (KVA) has little advantage over physical address. There are also no use cases yet where DMA engines need kernel virtual addresses for in-kernel DMA.
This removes SVM_FLAG_SUPERVISOR_MODE support from the IOMMU interface. The device drivers are suggested to handle kernel DMA with PASID through the kernel DMA APIs.
The drvdata parameter in iommu_sva_bind_device() and all callbacks is not needed anymore. Cleanup them as well.
--- include/linux/intel-svm.h | 13 ----- include/linux/iommu.h | 8 +-- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 5 +- drivers/iommu/intel/iommu.h | 3 +- drivers/dma/idxd/cdev.c | 3 +- drivers/dma/idxd/init.c | 25 +-------- .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 3 +- drivers/iommu/intel/svm.c | 55 +++++-------------- drivers/iommu/iommu.c | 5 +- drivers/misc/uacce/uacce.c | 2 +- 10 files changed, 25 insertions(+), 97 deletions(-)
include/linux/intel-svm.h +0 -13
@@ -13,17 +13,4 @@#define PRQ_RING_MASK ((0x1000 << PRQ_ORDER) - 0x20)#define PRQ_DEPTH ((0x1000 << PRQ_ORDER) >> 5) -/* -* The SVM_FLAG_SUPERVISOR_MODE flag requests a PASID which can be used only -* for access to kernel addresses. No IOTLB flushes are automatically done -* for kernel mappings; it is valid only for access to the kernel's static -* 1:1 mapping of physical memory — not to vmalloc or even module mappings. -* A future API addition may permit the use of such ranges, by means of an -* explicit IOTLB flush call (akin to the DMA API's unmap method). -* -* It is unlikely that we will ever hook into flush_tlb_kernel_range() to -* do such IOTLB flushes automatically. -*/ -#define SVM_FLAG_SUPERVISOR_MODE BIT(0) -#endif /* __INTEL_SVM_H__ */
include/linux/iommu.h +3 -5
@@ -247,8 +247,7 @@ struct iommu_ops {int (*dev_enable_feat)(struct device *dev,enum iommu_dev_features f);int (*dev_disable_feat)(struct device *dev,enum iommu_dev_features f); -struct iommu_sva *(*sva_bind)(struct device *dev,struct mm_struct *mm, -void *drvdata); +struct iommu_sva *(*sva_bind)(struct device *dev,struct mm_struct *mm);void (*sva_unbind)(struct iommu_sva *handle);u32 (*sva_get_pasid)(struct iommu_sva *handle);
@@ -668,8 +667,7 @@ int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features f);int iommu_dev_disable_feature (struct device *dev,enum iommu_dev_features f);struct iommu_sva *iommu_sva_bind_device (struct device *dev, -struct mm_struct *mm, -void *drvdata); +struct mm_struct *mm);void iommu_sva_unbind_device (struct iommu_sva *handle);u32 iommu_sva_get_pasid (struct iommu_sva *handle);
@@ -1000,7 +998,7 @@ iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat) }static inline struct iommu_sva * -iommu_sva_bind_device (struct device *dev,struct mm_struct *mm,void *drvdata) +iommu_sva_bind_device (struct device *dev,struct mm_struct *mm) {return NULL ; }
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +2 -3
@@ -754,8 +754,7 @@ bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master);int arm_smmu_master_enable_sva (struct arm_smmu_master *master);int arm_smmu_master_disable_sva (struct arm_smmu_master *master);bool arm_smmu_master_iopf_supported (struct arm_smmu_master *master); -struct iommu_sva *arm_smmu_sva_bind (struct device *dev,struct mm_struct *mm, -void *drvdata); +struct iommu_sva *arm_smmu_sva_bind (struct device *dev,struct mm_struct *mm);void arm_smmu_sva_unbind (struct iommu_sva *handle);u32 arm_smmu_sva_get_pasid (struct iommu_sva *handle);void arm_smmu_sva_notifier_synchronize (void );
@@ -791,7 +790,7 @@ static inline bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master }static inline struct iommu_sva * -arm_smmu_sva_bind (struct device *dev,struct mm_struct *mm,void *drvdata) +arm_smmu_sva_bind (struct device *dev,struct mm_struct *mm) {return ERR_PTR (-ENODEV ); }
drivers/iommu/intel/iommu.h +1 -2
@@ -748,8 +748,7 @@ struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn);extern void intel_svm_check (struct intel_iommu *iommu);extern int intel_svm_enable_prq (struct intel_iommu *iommu);extern int intel_svm_finish_prq (struct intel_iommu *iommu); -struct iommu_sva *intel_svm_bind (struct device *dev,struct mm_struct *mm, -void *drvdata); +struct iommu_sva *intel_svm_bind (struct device *dev,struct mm_struct *mm);void intel_svm_unbind (struct iommu_sva *handle);u32 intel_svm_get_pasid (struct iommu_sva *handle); int intel_svm_page_response(struct device *dev, struct iommu_fault_event *evt,
drivers/dma/idxd/cdev.c +1 -2
@@ -6,7 +6,6 @@#include <linux/pci.h> #include <linux/device.h> #include <linux/sched/task.h> -#include <linux/intel-svm.h> #include <linux/io-64-nonatomic-lo-hi.h> #include <linux/cdev.h> #include <linux/fs.h>
@@ -100,7 +99,7 @@ static int idxd_cdev_open(struct inode *inode, struct file *filp) filp->private_data = ctx;if (device_user_pasid_enabled (idxd)) { - sva =iommu_sva_bind_device (dev, current->mm,NULL ); + sva =iommu_sva_bind_device (dev, current->mm);if (IS_ERR (sva)) { rc =PTR_ERR (sva);dev_err (dev,"pasid allocation failed: %d\n" , rc);
drivers/dma/idxd/init.c +1 -24
@@ -14,7 +14,6 @@#include <linux/io-64-nonatomic-lo-hi.h> #include <linux/device.h> #include <linux/idr.h> -#include <linux/intel-svm.h> #include <linux/iommu.h> #include <uapi/linux/idxd.h> #include <linux/dmaengine.h>
@@ -502,29 +501,7 @@ static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_dstatic int idxd_enable_system_pasid (struct idxd_device *idxd) { -int flags; -unsigned int pasid; -struct iommu_sva *sva; - - flags =SVM_FLAG_SUPERVISOR_MODE ; - - sva =iommu_sva_bind_device (&idxd->pdev->dev,NULL , &flags); -if (IS_ERR (sva)) { -dev_warn (&idxd->pdev->dev, -"iommu sva bind failed: %ld\n" ,PTR_ERR (sva)); -return PTR_ERR (sva); - } - - pasid =iommu_sva_get_pasid (sva); -if (pasid ==IOMMU_PASID_INVALID ) { -iommu_sva_unbind_device (sva); -return -ENODEV ; - } - - idxd->sva = sva; - idxd->pasid = pasid; -dev_dbg (&idxd->pdev->dev,"system pasid: %u\n" , pasid); -return 0 ; +return -EOPNOTSUPP ; }static void idxd_disable_system_pasid (struct idxd_device *idxd)
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +1 -2
@@ -367,8 +367,7 @@ __arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm)return ERR_PTR (ret); } -struct iommu_sva * -arm_smmu_sva_bind (struct device *dev,struct mm_struct *mm,void *drvdata) +struct iommu_sva *arm_smmu_sva_bind (struct device *dev,struct mm_struct *mm) {struct iommu_sva *handle;struct iommu_domain *domain =iommu_get_domain_for_dev (dev);
drivers/iommu/intel/svm.c +13 -42
108 lines not on this page · raw message
drivers/iommu/iommu.c +2 -3
@@ -2750,7 +2750,6 @@ EXPORT_SYMBOL_GPL(iommu_dev_disable_feature);* iommu_sva_bind_device() - Bind a process address space to a device * @dev: the device * @mm: the mm to bind, caller must hold a reference to it -* @drvdata: opaque data pointer to pass to bind callback * * Create a bond between device and address space, allowing the device to access * the mm using the returned PASID. If a bond already exists between @device and
@@ -2763,7 +2762,7 @@ EXPORT_SYMBOL_GPL(iommu_dev_disable_feature);* On error, returns an ERR_PTR value. */ struct iommu_sva * -iommu_sva_bind_device (struct device *dev,struct mm_struct *mm,void *drvdata) +iommu_sva_bind_device (struct device *dev,struct mm_struct *mm) {struct iommu_group *group;struct iommu_sva *handle =ERR_PTR (-EINVAL );
@@ -2788,7 +2787,7 @@ iommu_sva_bind_device(struct device *dev, struct mm_struct *mm, void *drvdata)if (iommu_group_device_count (group) !=1 ) goto out_unlock; - handle = ops->sva_bind (dev, mm, drvdata); + handle = ops->sva_bind (dev, mm); out_unlock:mutex_unlock (&group->mutex);
drivers/misc/uacce/uacce.c +1 -1
@@ -108,7 +108,7 @@ static int uacce_bind_queue(struct uacce_device *uacce, struct uacce_queue *q)if (!(uacce->flags &UACCE_DEV_SVA ))return 0 ; - handle =iommu_sva_bind_device (uacce->parent, current->mm,NULL ); + handle =iommu_sva_bind_device (uacce->parent, current->mm);if (IS_ERR (handle))return PTR_ERR (handle);
Signature · 1 line
2.34.1
Lu Baolu
maintainerPATCH RESEND v14 4/13 PCI: Enable PASID only when ACS RR & UF enabled on upstream path
In reply to Lu BaoluThe Requester ID/Process Address Space ID (PASID) combination identifies an address space distinct from the PCI bus address space, e.g., an address space defined by an IOMMU.
But the PCIe fabric routes Memory Requests based on the TLP address, ignoring any PASID (PCIe r6.0, sec 2.2.10.4), so a TLP with PASID that SHOULD go upstream to the IOMMU may instead be routed as a P2P Request if its address falls in a bridge window.
To ensure that all Memory Requests with PASID are routed upstream, only enable PASID if ACS P2P Request Redirect and Upstream Forwarding are enabled for the path leading to the device.
--- drivers/pci/ats.c | 3 +++ 1 file changed, 3 insertions(+)
drivers/pci/ats.c +3 -0
@@ -382,6 +382,9 @@ int pci_enable_pasid(struct pci_dev *pdev, int features)if (!pasid)return -EINVAL ; +if (!pci_acs_path_enabled (pdev,NULL ,PCI_ACS_RR |PCI_ACS_UF )) +return -EINVAL ; +pci_read_config_word (pdev, pasid +PCI_PASID_CAP , &supported); supported &=PCI_PASID_CAP_EXEC |PCI_PASID_CAP_PRIV ;
Signature · 1 line
2.34.1
Lu Baolu
maintainerPATCH RESEND v14 5/13 iommu: Add attach/detach_dev_pasid iommu interfaces
In reply to Lu BaoluAttaching an IOMMU domain to a PASID of a device is a generic operation for modern IOMMU drivers which support PASID-granular DMA address translation. Currently visible usage scenarios include (but not limited):
- SVA (Shared Virtual Address)
- kernel DMA with PASID
- hardware-assist mediated device
This adds the set_dev_pasid domain ops for setting the domain onto a PASID of a device and remove_dev_pasid iommu ops for removing any setup on a PASID of device. This also adds interfaces for device drivers to attach/detach/retrieve a domain for a PASID of a device.
If multiple devices share a single group, it's fine as long the fabric always routes every TLP marked with a PASID to the host bridge and only the host bridge. For example, ACS achieves this universally and has been checked when pci_enable_pasid() is called. As we can't reliably tell the source apart in a group, all the devices in a group have to be considered as the same source, and mapped to the same PASID table.
The DMA ownership is about the whole device (more precisely, iommu group), including the RID and PASIDs. When the ownership is converted, the pasid array must be empty. This also adds necessary checks in the DMA ownership interfaces.
--- include/linux/iommu.h | 32 ++++++++++ drivers/iommu/iommu.c | 141 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 169 insertions(+), 4 deletions(-)
include/linux/iommu.h +32 -0
@@ -223,6 +223,9 @@ struct iommu_iotlb_gather {* - IOMMU_DOMAIN_DMA: must use a dma domain * - 0: use the default setting * @default_domain_ops: the default ops for domains +* @remove_dev_pasid: Remove any translation configurations of a specific +* pasid, so that any DMA transactions with this pasid +* will be blocked by the hardware. * @pgsize_bitmap: bitmap of all possible supported page sizes * @owner: Driver module providing these ops */
@@ -256,6 +259,7 @@ struct iommu_ops { struct iommu_page_response *msg);int (*def_domain_type)(struct device *dev); +void (*remove_dev_pasid)(struct device *dev,ioasid_t pasid);const struct iommu_domain_ops *default_domain_ops;unsigned long pgsize_bitmap;
@@ -266,6 +270,7 @@ struct iommu_ops {* struct iommu_domain_ops - domain specific operations * @attach_dev: attach an iommu domain to a device * @detach_dev: detach an iommu domain from a device +* @set_dev_pasid: set an iommu domain to a pasid of device * @map: map a physically contiguous memory region to an iommu domain * @map_pages: map a physically contiguous set of pages of the same size to * an iommu domain.
@@ -286,6 +291,8 @@ struct iommu_ops {struct iommu_domain_ops {int (*attach_dev)(struct iommu_domain *domain,struct device *dev);void (*detach_dev)(struct iommu_domain *domain,struct device *dev); +int (*set_dev_pasid)(struct iommu_domain *domain,struct device *dev, +ioasid_t pasid);int (*map)(struct iommu_domain *domain,unsigned long iova,phys_addr_t paddr,size_t size,int prot,gfp_t gfp);
@@ -678,6 +685,13 @@ int iommu_group_claim_dma_owner(struct iommu_group *group, void *owner);void iommu_group_release_dma_owner (struct iommu_group *group);bool iommu_group_dma_owner_claimed (struct iommu_group *group); +int iommu_attach_device_pasid (struct iommu_domain *domain, +struct device *dev,ioasid_t pasid); +void iommu_detach_device_pasid (struct iommu_domain *domain, +struct device *dev,ioasid_t pasid); +struct iommu_domain * +iommu_get_domain_for_dev_pasid (struct device *dev,ioasid_t pasid, +unsigned int type);#else /* CONFIG_IOMMU_API */ struct iommu_ops {};
@@ -1040,6 +1054,24 @@ static inline bool iommu_group_dma_owner_claimed(struct iommu_group *group) {return false; } + +static inline int iommu_attach_device_pasid (struct iommu_domain *domain, +struct device *dev,ioasid_t pasid) +{ +return -ENODEV ; +} + +static inline void iommu_detach_device_pasid (struct iommu_domain *domain, +struct device *dev,ioasid_t pasid) +{ +} + +static inline struct iommu_domain * +iommu_get_domain_for_dev_pasid (struct device *dev,ioasid_t pasid, +unsigned int type) +{ +return NULL ; +}#endif /* CONFIG_IOMMU_API */ /**
drivers/iommu/iommu.c +137 -4
180 lines not on this page · raw message
Signature · 1 line
2.34.1
Lu Baolu
maintainerPATCH RESEND v14 6/13 iommu: Add IOMMU SVA domain support
In reply to Lu BaoluThe SVA iommu_domain represents a hardware pagetable that the IOMMU hardware could use for SVA translation. This adds some infrastructures to support SVA domain in the iommu core. It includes:
- Extend the iommu_domain to support a new IOMMU_DOMAIN_SVA domain type. The IOMMU drivers that support allocation of the SVA domain should provide its own SVA domain specific iommu_domain_ops.
- Add a helper to allocate an SVA domain. The iommu_domain_free() is still used to free an SVA domain.
The report_iommu_fault() should be replaced by the new iommu_report_device_fault(). Leave the existing fault handler with the existing users and the newly added SVA members excludes it.
--- include/linux/iommu.h | 25 +++++++++++++++++++++++-- drivers/iommu/iommu.c | 20 ++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-)
include/linux/iommu.h +23 -2
@@ -64,6 +64,8 @@ struct iommu_domain_geometry {#define __IOMMU_DOMAIN_PT (1U << 2)/* Domain is identity mapped */ #define __IOMMU_DOMAIN_DMA_FQ (1U << 3)/* DMA-API uses flush queue */ +#define __IOMMU_DOMAIN_SVA (1U << 4)/* Shared process address space */ +/* * This are the possible domain-types *
@@ -77,6 +79,8 @@ struct iommu_domain_geometry {* certain optimizations for these domains * IOMMU_DOMAIN_DMA_FQ - As above, but definitely using batched TLB * invalidation. +* IOMMU_DOMAIN_SVA - DMA addresses are shared process addresses +* represented by mm_struct's. */ #define IOMMU_DOMAIN_BLOCKED (0U)#define IOMMU_DOMAIN_IDENTITY (__IOMMU_DOMAIN_PT)
@@ -86,15 +90,24 @@ struct iommu_domain_geometry {#define IOMMU_DOMAIN_DMA_FQ (__IOMMU_DOMAIN_PAGING | \ __IOMMU_DOMAIN_DMA_API | \ __IOMMU_DOMAIN_DMA_FQ) +#define IOMMU_DOMAIN_SVA (__IOMMU_DOMAIN_SVA)struct iommu_domain {unsigned type;const struct iommu_domain_ops *ops;unsigned long pgsize_bitmap;/* Bitmap of page sizes in use */ -iommu_fault_handler_t handler; -void *handler_token;struct iommu_domain_geometry geometry;struct iommu_dma_cookie *iova_cookie; +union { +struct { +iommu_fault_handler_t handler; +void *handler_token; + }; +struct {/* IOMMU_DOMAIN_SVA */ +struct mm_struct *mm; +int users; + }; + }; };static inline bool iommu_is_dma_domain (struct iommu_domain *domain)
@@ -685,6 +698,8 @@ int iommu_group_claim_dma_owner(struct iommu_group *group, void *owner);void iommu_group_release_dma_owner (struct iommu_group *group);bool iommu_group_dma_owner_claimed (struct iommu_group *group); +struct iommu_domain *iommu_sva_domain_alloc (struct device *dev, +struct mm_struct *mm);int iommu_attach_device_pasid (struct iommu_domain *domain,struct device *dev,ioasid_t pasid); void iommu_detach_device_pasid(struct iommu_domain *domain,
@@ -1055,6 +1070,12 @@ static inline bool iommu_group_dma_owner_claimed(struct iommu_group *group)return false; } +static inline struct iommu_domain * +iommu_sva_domain_alloc (struct device *dev,struct mm_struct *mm) +{ +return NULL ; +} +static inline int iommu_attach_device_pasid (struct iommu_domain *domain,struct device *dev,ioasid_t pasid) {
drivers/iommu/iommu.c +20 -0
@@ -29,6 +29,7 @@#include <linux/module.h> #include <linux/cc_platform.h> #include <trace/events/iommu.h> +#include <linux/sched/mm.h> #include "dma-iommu.h"
@@ -1934,6 +1935,8 @@ EXPORT_SYMBOL_GPL(iommu_domain_alloc);void iommu_domain_free (struct iommu_domain *domain) { +if (domain->type ==IOMMU_DOMAIN_SVA ) +mmdrop (domain->mm);iommu_put_dma_cookie (domain); domain->ops->free (domain); }
@@ -3383,3 +3386,20 @@ struct iommu_domain *iommu_get_domain_for_dev_pasid(struct device *dev,return domain; }EXPORT_SYMBOL_GPL (iommu_get_domain_for_dev_pasid); + +struct iommu_domain *iommu_sva_domain_alloc (struct device *dev, +struct mm_struct *mm) +{ +const struct iommu_ops *ops =dev_iommu_ops (dev); +struct iommu_domain *domain; + + domain = ops->domain_alloc (IOMMU_DOMAIN_SVA ); +if (!domain) +return NULL ; + + domain->type =IOMMU_DOMAIN_SVA ; +mmgrab (mm); + domain->mm = mm; + +return domain; +}
Signature · 1 line
2.34.1
Lu Baolu
maintainerPATCH RESEND v14 7/13 iommu/vt-d: Add SVA domain support
In reply to Lu BaoluAdd support for SVA domain allocation and provide an SVA-specific iommu_domain_ops. This implementation is based on the existing SVA code. Possible cleanup and refactoring are left for incremental changes later.
The VT-d driver will also need to support setting a DMA domain to a PASID of device. Current SVA implementation uses different data structures to track the domain and device PASID relationship. That's the reason why we need to check the domain type in remove_dev_pasid callback. Eventually we'll consolidate the data structures and remove the need of domain type check.
--- drivers/iommu/intel/iommu.h | 10 ++++++++ drivers/iommu/intel/iommu.c | 25 ++++++++++++++++++++ drivers/iommu/intel/svm.c | 47 +++++++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+)
drivers/iommu/intel/iommu.h +10 -0
@@ -753,6 +753,8 @@ void intel_svm_unbind(struct iommu_sva *handle);u32 intel_svm_get_pasid (struct iommu_sva *handle);int intel_svm_page_response (struct device *dev,struct iommu_fault_event *evt,struct iommu_page_response *msg); +struct iommu_domain *intel_svm_domain_alloc (void ); +void intel_svm_remove_dev_pasid (struct device *dev,ioasid_t pasid);struct intel_svm_dev {struct list_head list;
@@ -777,6 +779,14 @@ struct intel_svm { };#else static inline void intel_svm_check (struct intel_iommu *iommu) {} +static inline struct iommu_domain *intel_svm_domain_alloc (void ) +{ +return NULL ; +} + +static inline void intel_svm_remove_dev_pasid (struct device *dev,ioasid_t pasid) +{ +}#endif #ifdef CONFIG_INTEL_IOMMU_DEBUGFS
drivers/iommu/intel/iommu.c +25 -0
@@ -4169,6 +4169,8 @@ static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)return domain;case IOMMU_DOMAIN_IDENTITY :return &si_domain->domain; +case IOMMU_DOMAIN_SVA : +return intel_svm_domain_alloc ();default :return NULL ; }
@@ -4712,6 +4714,28 @@ static void intel_iommu_iotlb_sync_map(struct iommu_domain *domain,__mapping_notify_one (info->iommu, dmar_domain, pfn, pages); } +static void intel_iommu_remove_dev_pasid (struct device *dev,ioasid_t pasid) +{ +struct intel_iommu *iommu =device_to_iommu (dev,NULL ,NULL ); +struct iommu_domain *domain; + +/* Domain type specific cleanup: */ + domain =iommu_get_domain_for_dev_pasid (dev, pasid,0 ); +if (domain) { +switch (domain->type) { +case IOMMU_DOMAIN_SVA : +intel_svm_remove_dev_pasid (dev, pasid); +break ; +default : +/* should never reach here */ +WARN_ON (1 ); +break ; + } + } + +intel_pasid_tear_down_entry (iommu, dev, pasid, false); +} +const struct iommu_ops intel_iommu_ops = { .capable = intel_iommu_capable, .domain_alloc = intel_iommu_domain_alloc,
@@ -4724,6 +4748,7 @@ const struct iommu_ops intel_iommu_ops = { .dev_disable_feat = intel_iommu_dev_disable_feat, .is_attach_deferred = intel_iommu_is_attach_deferred, .def_domain_type = device_def_domain_type, + .remove_dev_pasid = intel_iommu_remove_dev_pasid, .pgsize_bitmap =SZ_4K , #ifdef CONFIG_INTEL_IOMMU_SVM .sva_bind = intel_svm_bind,
drivers/iommu/intel/svm.c +47 -0
50 lines not on this page · raw message
Signature · 1 line
2.34.1
Lu Baolu
maintainerPATCH RESEND v14 8/13 arm-smmu-v3/sva: Add SVA domain support
In reply to Lu BaoluAdd support for SVA domain allocation and provide an SVA-specific iommu_domain_ops. This implementation is based on the existing SVA code. Possible cleanup and refactoring are left for incremental changes later.
--- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 14 +++++ .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 61 +++++++++++++++++++ drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 15 +++++ 3 files changed, 90 insertions(+)
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +14 -0
@@ -758,6 +758,9 @@ struct iommu_sva *arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm);void arm_smmu_sva_unbind (struct iommu_sva *handle);u32 arm_smmu_sva_get_pasid (struct iommu_sva *handle);void arm_smmu_sva_notifier_synchronize (void ); +struct iommu_domain *arm_smmu_sva_domain_alloc (void ); +void arm_smmu_sva_remove_dev_pasid (struct iommu_domain *domain, +struct device *dev,ioasid_t id);#else /* CONFIG_ARM_SMMU_V3_SVA */ static inline bool arm_smmu_sva_supported (struct arm_smmu_device *smmu) {
@@ -803,5 +806,16 @@ static inline u32 arm_smmu_sva_get_pasid(struct iommu_sva *handle) }static inline void arm_smmu_sva_notifier_synchronize (void ) {} + +static inline struct iommu_domain *arm_smmu_sva_domain_alloc (void ) +{ +return NULL ; +} + +static inline void arm_smmu_sva_remove_dev_pasid (struct iommu_domain *domain, +struct device *dev, +ioasid_t id) +{ +}#endif /* CONFIG_ARM_SMMU_V3_SVA */ #endif /* _ARM_SMMU_V3_H */
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +61 -0
64 lines not on this page · raw message
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +15 -0
@@ -2009,6 +2009,9 @@ static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) {struct arm_smmu_domain *smmu_domain; +if (type ==IOMMU_DOMAIN_SVA ) +return arm_smmu_sva_domain_alloc (); + if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA && type != IOMMU_DOMAIN_DMA_FQ &&
@@ -2838,6 +2841,17 @@ static int arm_smmu_def_domain_type(struct device *dev)return 0 ; } +static void arm_smmu_remove_dev_pasid (struct device *dev,ioasid_t pasid) +{ +struct iommu_domain *domain; + + domain =iommu_get_domain_for_dev_pasid (dev, pasid,IOMMU_DOMAIN_SVA ); +if (WARN_ON (IS_ERR (domain)) || !domain) +return ; + +arm_smmu_sva_remove_dev_pasid (domain, dev, pasid); +} +static struct iommu_ops arm_smmu_ops = { .capable = arm_smmu_capable, .domain_alloc = arm_smmu_domain_alloc,
@@ -2846,6 +2860,7 @@ static struct iommu_ops arm_smmu_ops = { .device_group = arm_smmu_device_group, .of_xlate = arm_smmu_of_xlate, .get_resv_regions = arm_smmu_get_resv_regions, + .remove_dev_pasid = arm_smmu_remove_dev_pasid, .dev_enable_feat = arm_smmu_dev_enable_feature, .dev_disable_feat = arm_smmu_dev_disable_feature, .sva_bind = arm_smmu_sva_bind,
Signature · 1 line
2.34.1
Lu Baolu
maintainerPATCH RESEND v14 9/13 iommu/sva: Refactoring iommu_sva_bind/unbind_device()
In reply to Lu BaoluThe existing iommu SVA interfaces are implemented by calling the SVA specific iommu ops provided by the IOMMU drivers. There's no need for any SVA specific ops in iommu_ops vector anymore as we can achieve this through the generic attach/detach_dev_pasid domain ops.
This refactors the IOMMU SVA interfaces implementation by using the iommu_attach/detach_device_pasid interfaces and align them with the concept of the SVA iommu domain. Put the new SVA code in the SVA related file in order to make it self-contained.
--- include/linux/iommu.h | 43 +++++++------ drivers/iommu/iommu-sva-lib.c | 111 ++++++++++++++++++++++++++++++++++ drivers/iommu/iommu.c | 91 ---------------------------- 3 files changed, 134 insertions(+), 111 deletions(-)
include/linux/iommu.h +23 -20
@@ -645,6 +645,7 @@ struct iommu_fwspec {*/ struct iommu_sva {struct device *dev; +struct iommu_domain *domain; }; int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode,
@@ -686,11 +687,6 @@ void iommu_release_device(struct device *dev);int iommu_dev_enable_feature (struct device *dev,enum iommu_dev_features f);int iommu_dev_disable_feature (struct device *dev,enum iommu_dev_features f); -struct iommu_sva *iommu_sva_bind_device (struct device *dev, -struct mm_struct *mm); -void iommu_sva_unbind_device (struct iommu_sva *handle); -u32 iommu_sva_get_pasid (struct iommu_sva *handle); -int iommu_device_use_default_domain (struct device *dev);void iommu_device_unuse_default_domain (struct device *dev);
@@ -1026,21 +1022,6 @@ iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat)return -ENODEV ; } -static inline struct iommu_sva * -iommu_sva_bind_device (struct device *dev,struct mm_struct *mm) -{ -return NULL ; -} - -static inline void iommu_sva_unbind_device (struct iommu_sva *handle) -{ -} - -static inline u32 iommu_sva_get_pasid (struct iommu_sva *handle) -{ -return IOMMU_PASID_INVALID ; -} -static inline struct iommu_fwspec *dev_iommu_fwspec_get (struct device *dev) {return NULL ;
@@ -1154,4 +1135,26 @@ static inline void iommu_dma_compose_msi_msg(struct msi_desc *desc, struct msi_m#endif /* CONFIG_IOMMU_DMA */ +#ifdef CONFIG_IOMMU_SVA +struct iommu_sva *iommu_sva_bind_device (struct device *dev, +struct mm_struct *mm); +void iommu_sva_unbind_device (struct iommu_sva *handle); +u32 iommu_sva_get_pasid (struct iommu_sva *handle); +#else +static inline struct iommu_sva * +iommu_sva_bind_device (struct device *dev,struct mm_struct *mm) +{ +return NULL ; +} + +static inline void iommu_sva_unbind_device (struct iommu_sva *handle) +{ +} + +static inline u32 iommu_sva_get_pasid (struct iommu_sva *handle) +{ +return IOMMU_PASID_INVALID ; +} +#endif /* CONFIG_IOMMU_SVA */ +#endif /* __LINUX_IOMMU_H */
drivers/iommu/iommu-sva-lib.c +111 -0
120 lines not on this page · raw message
drivers/iommu/iommu.c +0 -91
97 lines not on this page · raw message
Signature · 1 line
2.34.1
Lu Baolu
maintainerPATCH RESEND v14 10/13 iommu: Remove SVA related callbacks from iommu ops
In reply to Lu BaoluThese ops'es have been deprecated. There's no need for them anymore. Remove them to avoid dead code.
--- include/linux/iommu.h | 7 --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 16 ------ drivers/iommu/intel/iommu.h | 3 -- .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 40 --------------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 3 -- drivers/iommu/intel/iommu.c | 3 -- drivers/iommu/intel/svm.c | 49 ------------------- 7 files changed, 121 deletions(-)
include/linux/iommu.h +0 -7
@@ -227,9 +227,6 @@ struct iommu_iotlb_gather {* driver init to device driver init (default no) * @dev_enable/disable_feat: per device entries to enable/disable * iommu specific features. -* @sva_bind: Bind process address space to device -* @sva_unbind: Unbind process address space from device -* @sva_get_pasid: Get PASID associated to a SVA handle * @page_response: handle page request response * @def_domain_type: device default domain type, return value: * - IOMMU_DOMAIN_IDENTITY: must use an identity domain
@@ -263,10 +260,6 @@ struct iommu_ops {int (*dev_enable_feat)(struct device *dev,enum iommu_dev_features f);int (*dev_disable_feat)(struct device *dev,enum iommu_dev_features f); -struct iommu_sva *(*sva_bind)(struct device *dev,struct mm_struct *mm); -void (*sva_unbind)(struct iommu_sva *handle); -u32 (*sva_get_pasid)(struct iommu_sva *handle); -int (*page_response)(struct device *dev,struct iommu_fault_event *evt,struct iommu_page_response *msg);
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +0 -16
@@ -754,9 +754,6 @@ bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master);int arm_smmu_master_enable_sva (struct arm_smmu_master *master);int arm_smmu_master_disable_sva (struct arm_smmu_master *master);bool arm_smmu_master_iopf_supported (struct arm_smmu_master *master); -struct iommu_sva *arm_smmu_sva_bind (struct device *dev,struct mm_struct *mm); -void arm_smmu_sva_unbind (struct iommu_sva *handle); -u32 arm_smmu_sva_get_pasid (struct iommu_sva *handle);void arm_smmu_sva_notifier_synchronize (void );struct iommu_domain *arm_smmu_sva_domain_alloc (void ); void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain,
@@ -792,19 +789,6 @@ static inline bool arm_smmu_master_iopf_supported(struct arm_smmu_master *masterreturn false; } -static inline struct iommu_sva * -arm_smmu_sva_bind (struct device *dev,struct mm_struct *mm) -{ -return ERR_PTR (-ENODEV ); -} - -static inline void arm_smmu_sva_unbind (struct iommu_sva *handle) {} - -static inline u32 arm_smmu_sva_get_pasid (struct iommu_sva *handle) -{ -return IOMMU_PASID_INVALID ; -} -static inline void arm_smmu_sva_notifier_synchronize (void ) {}static inline struct iommu_domain *arm_smmu_sva_domain_alloc (void )
drivers/iommu/intel/iommu.h +0 -3
@@ -748,9 +748,6 @@ struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn);extern void intel_svm_check (struct intel_iommu *iommu);extern int intel_svm_enable_prq (struct intel_iommu *iommu);extern int intel_svm_finish_prq (struct intel_iommu *iommu); -struct iommu_sva *intel_svm_bind (struct device *dev,struct mm_struct *mm); -void intel_svm_unbind (struct iommu_sva *handle); -u32 intel_svm_get_pasid (struct iommu_sva *handle);int intel_svm_page_response (struct device *dev,struct iommu_fault_event *evt,struct iommu_page_response *msg);struct iommu_domain *intel_svm_domain_alloc (void );
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +0 -40
@@ -344,11 +344,6 @@ __arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm)if (!bond)return ERR_PTR (-ENOMEM ); -/* Allocate a PASID for this mm if necessary */ - ret =iommu_sva_alloc_pasid (mm,1 , (1U << master->ssid_bits) -1 ); -if (ret) - goto err_free_bond; - bond->mm = mm; bond->sva.dev = dev;refcount_set (&bond->refs,1 );
@@ -367,41 +362,6 @@ __arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm)return ERR_PTR (ret); } -struct iommu_sva *arm_smmu_sva_bind (struct device *dev,struct mm_struct *mm) -{ -struct iommu_sva *handle; -struct iommu_domain *domain =iommu_get_domain_for_dev (dev); -struct arm_smmu_domain *smmu_domain =to_smmu_domain (domain); - -if (smmu_domain->stage !=ARM_SMMU_DOMAIN_S1 ) -return ERR_PTR (-EINVAL ); - -mutex_lock (&sva_lock); - handle =__arm_smmu_sva_bind (dev, mm); -mutex_unlock (&sva_lock); -return handle; -} - -void arm_smmu_sva_unbind (struct iommu_sva *handle) -{ -struct arm_smmu_bond *bond =sva_to_bond (handle); - -mutex_lock (&sva_lock); -if (refcount_dec_and_test (&bond->refs)) { -list_del (&bond->list); -arm_smmu_mmu_notifier_put (bond->smmu_mn); -kfree (bond); - } -mutex_unlock (&sva_lock); -} - -u32 arm_smmu_sva_get_pasid (struct iommu_sva *handle) -{ -struct arm_smmu_bond *bond =sva_to_bond (handle); - -return bond->mm->pasid; -} -bool arm_smmu_sva_supported (struct arm_smmu_device *smmu) {unsigned long reg, fld;
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +0 -3
@@ -2863,9 +2863,6 @@ static struct iommu_ops arm_smmu_ops = { .remove_dev_pasid = arm_smmu_remove_dev_pasid, .dev_enable_feat = arm_smmu_dev_enable_feature, .dev_disable_feat = arm_smmu_dev_disable_feature, - .sva_bind = arm_smmu_sva_bind, - .sva_unbind = arm_smmu_sva_unbind, - .sva_get_pasid = arm_smmu_sva_get_pasid, .page_response = arm_smmu_page_response, .def_domain_type = arm_smmu_def_domain_type, .pgsize_bitmap = -1UL, /* Restricted during device attach */
drivers/iommu/intel/iommu.c +0 -3
@@ -4751,9 +4751,6 @@ const struct iommu_ops intel_iommu_ops = { .remove_dev_pasid = intel_iommu_remove_dev_pasid, .pgsize_bitmap =SZ_4K , #ifdef CONFIG_INTEL_IOMMU_SVM - .sva_bind = intel_svm_bind, - .sva_unbind = intel_svm_unbind, - .sva_get_pasid = intel_svm_get_pasid, .page_response = intel_svm_page_response, #endif .default_domain_ops = &(const struct iommu_domain_ops ) {
drivers/iommu/intel/svm.c +0 -49
61 lines not on this page · raw message
Signature · 1 line
2.34.1
Lu Baolu
maintainerPATCH RESEND v14 11/13 iommu: Prepare IOMMU domain for IOPF
In reply to Lu BaoluThis adds some mechanisms around the iommu_domain so that the I/O page fault handling framework could route a page fault to the domain and call the fault handler from it.
Add pointers to the page fault handler and its private data in struct iommu_domain. The fault handler will be called with the private data as a parameter once a page fault is routed to the domain. Any kernel component which owns an iommu domain could install handler and its private parameter so that the page fault could be further routed and handled.
This also prepares the SVA implementation to be the first consumer of the per-domain page fault handling model. The I/O page fault handler for SVA is copied to the SVA file with mmget_not_zero() added before mmap_read_lock().
--- include/linux/iommu.h | 3 ++ drivers/iommu/iommu-sva-lib.h | 8 +++++ drivers/iommu/io-pgfault.c | 7 +++++ drivers/iommu/iommu-sva-lib.c | 58 +++++++++++++++++++++++++++++++++++ drivers/iommu/iommu.c | 4 +++ 5 files changed, 80 insertions(+)
include/linux/iommu.h +3 -0
@@ -98,6 +98,9 @@ struct iommu_domain {unsigned long pgsize_bitmap;/* Bitmap of page sizes in use */ struct iommu_domain_geometry geometry;struct iommu_dma_cookie *iova_cookie; +enum iommu_page_response_code (*iopf_handler)(struct iommu_fault *fault, +void *data); +void *fault_data; union { struct { iommu_fault_handler_t handler;
drivers/iommu/iommu-sva-lib.h +8 -0
@@ -26,6 +26,8 @@ int iopf_queue_flush_dev(struct device *dev);struct iopf_queue *iopf_queue_alloc (const char *name);void iopf_queue_free (struct iopf_queue *queue);int iopf_queue_discard_partial (struct iopf_queue *queue); +enum iommu_page_response_code +iommu_sva_handle_iopf (struct iommu_fault *fault,void *data);#else /* CONFIG_IOMMU_SVA */ static inline int iommu_queue_iopf (struct iommu_fault *fault,void *cookie)
@@ -63,5 +65,11 @@ static inline int iopf_queue_discard_partial(struct iopf_queue *queue) {return -ENODEV ; } + +static inline enum iommu_page_response_code +iommu_sva_handle_iopf (struct iommu_fault *fault,void *data) +{ +return IOMMU_PAGE_RESP_INVALID ; +}#endif /* CONFIG_IOMMU_SVA */ #endif /* _IOMMU_SVA_LIB_H */
drivers/iommu/io-pgfault.c +7 -0
@@ -181,6 +181,13 @@ static void iopf_handle_group(struct work_struct *work)* request completes, outstanding faults will have been dealt with by the time * the PASID is freed. * +* Any valid page fault will be eventually routed to an iommu domain and the +* page fault handler installed there will get called. The users of this +* handling framework should guarantee that the iommu domain could only be +* freed after the device has stopped generating page faults (or the iommu +* hardware has been set to block the page faults) and the pending page faults +* have been flushed. +* * Return: 0 on success and <0 on error. */ int iommu_queue_iopf (struct iommu_fault *fault,void *cookie)
drivers/iommu/iommu-sva-lib.c +58 -0
61 lines not on this page · raw message
drivers/iommu/iommu.c +4 -0
@@ -33,6 +33,8 @@#include "dma-iommu.h" +#include "iommu-sva-lib.h" +static struct kset *iommu_group_kset;static DEFINE_IDA (iommu_group_ida);
@@ -3309,6 +3311,8 @@ struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, domain->type =IOMMU_DOMAIN_SVA ;mmgrab (mm); domain->mm = mm; + domain->iopf_handler = iommu_sva_handle_iopf; + domain->fault_data = mm;return domain; }
Signature · 1 line
2.34.1
Lu Baolu
maintainerPATCH RESEND v14 12/13 iommu: Per-domain I/O page fault handling
In reply to Lu BaoluTweak the I/O page fault handling framework to route the page faults to the domain and call the page fault handler retrieved from the domain. This makes the I/O page fault handling framework possible to serve more usage scenarios as long as they have an IOMMU domain and install a page fault handler in it. Some unused functions are also removed to avoid dead code.
The iommu_get_domain_for_dev_pasid() which retrieves attached domain for a {device, PASID} pair is used. It will be used by the page fault handling framework which knows {device, PASID} reported from the iommu driver. We have a guarantee that the SVA domain doesn't go away during IOPF handling, because unbind() won't free the domain until all the pending page requests have been flushed from the pipeline. The drivers either call iopf_queue_flush_dev() explicitly, or in stall case, the device driver is required to flush all DMAs including stalled transactions before calling unbind().
This also renames iopf_handle_group() to iopf_handler() to avoid confusing.
--- drivers/iommu/io-pgfault.c | 68 +++++--------------------------------- 1 file changed, 9 insertions(+), 59 deletions(-)
drivers/iommu/io-pgfault.c +9 -59
@@ -69,69 +69,18 @@ static int iopf_complete_group(struct device *dev, struct iopf_fault *iopf,return iommu_page_response (dev, &resp); } -static enum iommu_page_response_code -iopf_handle_single (struct iopf_fault *iopf) -{ -vm_fault_t ret; -struct mm_struct *mm; -struct vm_area_struct *vma; -unsigned int access_flags =0 ; -unsigned int fault_flags =FAULT_FLAG_REMOTE ; -struct iommu_fault_page_request *prm = &iopf->fault.prm; -enum iommu_page_response_code status =IOMMU_PAGE_RESP_INVALID ; - -if (!(prm->flags &IOMMU_FAULT_PAGE_REQUEST_PASID_VALID )) -return status; - - mm =iommu_sva_find (prm->pasid); -if (IS_ERR_OR_NULL (mm)) -return status; - -mmap_read_lock (mm); - - vma =find_extend_vma (mm, prm->addr); -if (!vma) -/* Unmapped area */ - goto out_put_mm; - -if (prm->perm &IOMMU_FAULT_PERM_READ ) - access_flags |=VM_READ ; - -if (prm->perm &IOMMU_FAULT_PERM_WRITE ) { - access_flags |=VM_WRITE ; - fault_flags |=FAULT_FLAG_WRITE ; - } - -if (prm->perm &IOMMU_FAULT_PERM_EXEC ) { - access_flags |=VM_EXEC ; - fault_flags |=FAULT_FLAG_INSTRUCTION ; - } - -if (!(prm->perm &IOMMU_FAULT_PERM_PRIV )) - fault_flags |=FAULT_FLAG_USER ; - -if (access_flags & ~vma->vm_flags) -/* Access fault */ - goto out_put_mm; - - ret =handle_mm_fault (vma, prm->addr, fault_flags,NULL ); - status = ret &VM_FAULT_ERROR ?IOMMU_PAGE_RESP_INVALID : -IOMMU_PAGE_RESP_SUCCESS ; - -out_put_mm: -mmap_read_unlock (mm); -mmput (mm); - -return status; -} - -static void iopf_handle_group (struct work_struct *work) +static void iopf_handler (struct work_struct *work) {struct iopf_group *group; +struct iommu_domain *domain;struct iopf_fault *iopf, *next;enum iommu_page_response_code status =IOMMU_PAGE_RESP_SUCCESS ; group = container_of(work, struct iopf_group, work); + domain =iommu_get_domain_for_dev_pasid (group->dev, + group->last_fault.fault.prm.pasid,0 ); +if (!domain || !domain->iopf_handler) + status =IOMMU_PAGE_RESP_INVALID ;list_for_each_entry_safe (iopf, next, &group->faults, list) {/*
@@ -139,7 +88,8 @@ static void iopf_handle_group(struct work_struct *work)* faults in the group if there is an error. */ if (status ==IOMMU_PAGE_RESP_SUCCESS ) - status =iopf_handle_single (iopf); + status = domain->iopf_handler (&iopf->fault, + domain->fault_data);if (!(iopf->fault.prm.flags &IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE ))
@@ -242,7 +192,7 @@ int iommu_queue_iopf(struct iommu_fault *fault, void *cookie) group->last_fault.fault = *fault;INIT_LIST_HEAD (&group->faults);list_add (&group->last_fault.list, &group->faults); -INIT_WORK (&group->work, iopf_handle_group); +INIT_WORK (&group->work, iopf_handler);/* See if we have partial faults for this group */ list_for_each_entry_safe (iopf, next, &iopf_param->partial, list) {
Signature · 1 line
2.34.1
Lu Baolu
maintainerPATCH RESEND v14 13/13 iommu: Rename iommu-sva-lib.{c,h}
In reply to Lu BaoluRename iommu-sva-lib.c[h] to iommu-sva.c[h] as it contains all code for SVA implementation in iommu core.
---
drivers/iommu/{iommu-sva-lib.h => iommu-sva.h} | 6 +++---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 2 +-
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 2 +-
drivers/iommu/intel/iommu.c | 2 +-
drivers/iommu/intel/svm.c | 2 +-
drivers/iommu/io-pgfault.c | 2 +-
drivers/iommu/{iommu-sva-lib.c => iommu-sva.c} | 2 +-
drivers/iommu/iommu.c | 2 +-
drivers/iommu/Makefile | 2 +-
9 files changed, 11 insertions(+), 11 deletions(-)
rename drivers/iommu/{iommu-sva-lib.h => iommu-sva.h} (95%)
rename drivers/iommu/{iommu-sva-lib.c => iommu-sva.c} (99%)drivers/iommu/iommu-sva.h renamed from drivers/iommu/iommu-sva-lib.h +3 -3
@@ -2,8 +2,8 @@/* * SVA library for IOMMU drivers */ -#ifndef _IOMMU_SVA_LIB_H -#define _IOMMU_SVA_LIB_H +#ifndef _IOMMU_SVA_H +#define _IOMMU_SVA_H#include <linux/ioasid.h> #include <linux/mm_types.h>
@@ -72,4 +72,4 @@ iommu_sva_handle_iopf(struct iommu_fault *fault, void *data)return IOMMU_PAGE_RESP_INVALID ; }#endif /* CONFIG_IOMMU_SVA */ -#endif /* _IOMMU_SVA_LIB_H */ +#endif /* _IOMMU_SVA_H */
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +1 -1
@@ -10,7 +10,7 @@#include <linux/slab.h> #include "arm-smmu-v3.h" -#include "../../iommu-sva-lib.h" +#include "../../iommu-sva.h" #include "../../io-pgtable-arm.h" struct arm_smmu_mmu_notifier {
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +1 -1
@@ -29,7 +29,7 @@#include "arm-smmu-v3.h" #include "../../dma-iommu.h" -#include "../../iommu-sva-lib.h" +#include "../../iommu-sva.h" static bool disable_bypass = true;module_param (disable_bypass, bool,0444 );
drivers/iommu/intel/iommu.c +1 -1
@@ -27,7 +27,7 @@#include "iommu.h" #include "../dma-iommu.h" #include "../irq_remapping.h" -#include "../iommu-sva-lib.h" +#include "../iommu-sva.h" #include "pasid.h" #include "cap_audit.h"
drivers/iommu/intel/svm.c +1 -1
@@ -24,7 +24,7 @@#include "iommu.h" #include "pasid.h" #include "perf.h" -#include "../iommu-sva-lib.h" +#include "../iommu-sva.h" #include "trace.h" static irqreturn_t prq_event_thread (int irq,void *d);
drivers/iommu/io-pgfault.c +1 -1
@@ -11,7 +11,7 @@#include <linux/slab.h> #include <linux/workqueue.h> -#include "iommu-sva-lib.h" +#include "iommu-sva.h" /** * struct iopf_queue - IO Page Fault queue
drivers/iommu/iommu-sva.c renamed from drivers/iommu/iommu-sva-lib.c +1 -1
@@ -6,7 +6,7 @@#include <linux/sched/mm.h> #include <linux/iommu.h> -#include "iommu-sva-lib.h" +#include "iommu-sva.h" static DEFINE_MUTEX (iommu_sva_lock);static DECLARE_IOASID_SET (iommu_sva_pasid);
drivers/iommu/iommu.c +1 -1
@@ -33,7 +33,7 @@#include "dma-iommu.h" -#include "iommu-sva-lib.h" +#include "iommu-sva.h" static struct kset *iommu_group_kset;static DEFINE_IDA (iommu_group_ida);
drivers/iommu/Makefile +1 -1
@@ -28,6 +28,6 @@ obj-$(CONFIG_FSL_PAMU) += fsl_pamu.o fsl_pamu_domain.o obj-$(CONFIG_S390_IOMMU) += s390-iommu.o obj-$(CONFIG_HYPERV_IOMMU) += hyperv-iommu.o obj-$(CONFIG_VIRTIO_IOMMU) += virtio-iommu.o -obj-$(CONFIG_IOMMU_SVA) += iommu-sva-lib.o io-pgfault.o +obj-$(CONFIG_IOMMU_SVA) += iommu-sva.o io-pgfault.o obj-$(CONFIG_SPRD_IOMMU) += sprd-iommu.o obj-$(CONFIG_APPLE_DART) += apple-dart.o
Signature · 1 line
2.34.1
Quoted 6 lines · Lu Baolu
Lu Baolu (13):
iommu: Addmax_pasidsfield in structiommu_device
iommu: Addmax_pasidsfield in structdev_iommu
iommu: RemoveSVM_FLAG_SUPERVISOR_MODEsupport
PCI: Enable PASID only when ACS RR & UF enabled on upstream path
iommu: Add attach/detach_dev_pasid iommu interfaces
Quoted 8 lines
iommu: Add IOMMU SVA domain support iommu/vt-d: Add SVA domain support arm-smmu-v3/sva: Add SVA domain support iommu/sva: Refactoring iommu_sva_bind/unbind_device()
iommu: Remove SVA related callbacks from iommu ops
iommu: Prepare IOMMU domain for IOPF
iommu: Per-domain I/O page fault handling
iommu: Rename iommu-sva-lib.{c,h}
Applied, thanks everyone.