iommu/amd: SVA Support (Part 4) - SVA and IOPF
Subsystems (2)
Vasant Hegde
maintainerv4 cover iommu/amd: SVA Support (Part 4) - SVA and IOPF
This is part 4 of the 4-part series to introduce Share Virtual Address (SVA) support for devices, which can support PCI ATS, PASID and PRI capabilities. These devices are referred to as SVA-capable devices in this series.
It contains the following enhancements:
* Patch 1 - 3:
Rename, add support to enable/disable features, update DTE etc.
* Patch 4 - 10:
Add IOPF support
* Patch 11 - 14:
Introduce SVA support
This patch series is based on top of SVA Part 3 v4.
https://lore.kernel.org/linux-iommu/[email protected]/T/#t
Also depends on :
1) Baolu's IOPF improvement
https://lore.kernel.org/linux-iommu/[email protected]/
This is also available at github :
https://github.com/AMDESE/linux/tree/iommu_sva_part4_v4_v6.7_rc5
Thanks everyone who reviewed previous version and provided valuable feedbacks.
Changes from v3 -> v4:
- Moved amd_iommu_dev_update_dte() after set/clear_dte() so that we can avoid forward declaration
- Dropped "iommu/amd: Do not override PASID entry in GCR3 table"
- Added patch to fix PPR interrupt processing logic
- Renamed enable_iommus_v2() -> enable_iommus_ppr()
- Added ops->domain_alloc_sva()
- Added domain_alloc_sva() support and reorganize SVA patches
- In error path iommu_call_iopf_notifier() calls amd_iommu_complete_ppr() instead of amd_iommu_page_response()
v3: https://lore.kernel.org/linux-iommu/[email protected]/T/#t
Changes from v2 -> v3:
- Rename sva.c -> pasid.c
- Changed amd_iommu_sva_supported() -> amd_iommu_pasid_supported()
- Added patch to update/flush DTE
- Rework part of SVA support
- Move IOPF enablement to PASID bind time
v2: https://lore.kernel.org/linux-iommu/[email protected]/T/#u
Changes from v1 -> v2:
- Added new patch to fix PASID override issue in GCR3 table
- Complete rework of SVA code on top of Tina's SVA series.
- Rework SVA enable code
- Reworked IOPF handler code on top of Baolu's IOPF improvement series.
v1: https://lore.kernel.org/linux-iommu/[email protected]/T/#t
Thank you,
Vasant / Suravee
Jason Gunthorpe (1):
iommu: Add ops->domain_alloc_sva()
Suravee Suthikulpanit (5):
iommu/amd: Move PPR-related functions into ppr.c iommu/amd: Define per-IOMMU iopf_queue iommu/amd: Add support for page response iommu/amd: Introduce logic to enable/disable IOPF iommu/amd: Add GCR3 [un]initialization function
Vasant Hegde (5):
iommu/amd: Rename amd_iommu_v2_supported() as amd_iommu_pasid_supported() iommu/amd: Introduce per device DTE update function iommu/amd: Fix PPR interrupt processing logic
iommu/amd: Initial SVA support for AMD IOMMU
iommu/amd: Add SVA domain support
Wei Huang (3):
iommu/amd: Add support for enabling/disabling IOMMU features iommu/amd: Add support for add/remove device for IOPF iommu/amd: Add IO page fault notifier handler
drivers/iommu/amd/Kconfig | 2 + drivers/iommu/amd/Makefile | 2 +- drivers/iommu/amd/amd_iommu.h | 51 +++- drivers/iommu/amd/amd_iommu_types.h | 27 +++ drivers/iommu/amd/init.c | 79 ++---- drivers/iommu/amd/iommu.c | 182 ++++++++------ drivers/iommu/amd/pasid.c | 277 +++++++++++++++++++++ drivers/iommu/amd/ppr.c | 362 ++++++++++++++++++++++++++++ drivers/iommu/iommu-sva.c | 17 +- include/linux/iommu.h | 3 + 10 files changed, 862 insertions(+), 140 deletions(-) create mode 100644 drivers/iommu/amd/pasid.c create mode 100644 drivers/iommu/amd/ppr.c
Signature · 1 line
2.31.1
Vasant Hegde
maintainerPATCH v4 1/14 iommu/amd: Rename amd_iommu_v2_supported() as amd_iommu_pasid_supported()
To reflect its usage. No functional changes intended.
--- drivers/iommu/amd/amd_iommu.h | 2 +- drivers/iommu/amd/init.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)
drivers/iommu/amd/amd_iommu.h +1 -1
@@ -38,7 +38,7 @@ extern int amd_iommu_guest_ir;extern enum io_pgtable_fmt amd_iommu_pgtable;extern int amd_iommu_gpt_level; -bool amd_iommu_v2_supported (void ); +bool amd_iommu_pasid_supported (void );struct amd_iommu *get_amd_iommu (unsigned int idx);u8 amd_iommu_pc_get_max_banks (unsigned int idx);bool amd_iommu_pc_supported (void );
drivers/iommu/amd/init.c +1 -1
@@ -3652,7 +3652,7 @@ __setup("ivrs_ioapic", parse_ivrs_ioapic);__setup ("ivrs_hpet" , parse_ivrs_hpet);__setup ("ivrs_acpihid" , parse_ivrs_acpihid); -bool amd_iommu_v2_supported (void ) +bool amd_iommu_pasid_supported (void ) {/* CPU page table size should match IOMMU guest page table size */ if (cpu_feature_enabled(X86_FEATURE_LA57) &&
Signature · 1 line
2.31.1
Vasant Hegde
maintainerPATCH v4 2/14 iommu/amd: Introduce per device DTE update function
In reply to Vasant HegdeConsolidate per device update and flush logic into separate function. Also make it as global function as it will be used in subsequent series to update the DTE.
--- drivers/iommu/amd/amd_iommu.h | 1 + drivers/iommu/amd/iommu.c | 26 ++++++++++++++++++-------- 2 files changed, 19 insertions(+), 8 deletions(-)
drivers/iommu/amd/amd_iommu.h +1 -0
@@ -64,6 +64,7 @@ int amd_iommu_clear_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid);void amd_iommu_flush_all_caches (struct amd_iommu *iommu);void amd_iommu_update_and_flush_device_table (struct protection_domain *domain);void amd_iommu_domain_update (struct protection_domain *domain); +void amd_iommu_dev_update_dte (struct iommu_dev_data *dev_data,bool set);void amd_iommu_domain_flush_complete (struct protection_domain *domain);void amd_iommu_domain_flush_pages (struct protection_domain *domain,u64 address,size_t size);
drivers/iommu/amd/iommu.c +18 -8
@@ -2025,6 +2025,21 @@ static void clear_dte_entry(struct amd_iommu *iommu, u16 devid)amd_iommu_apply_erratum_63 (iommu, devid); } +/* Update and flush DTE for the given device */ +void amd_iommu_dev_update_dte (struct iommu_dev_data *dev_data,bool set) +{ +struct amd_iommu *iommu =get_amd_iommu_from_dev (dev_data->dev); + +if (set) +set_dte_entry (iommu, dev_data); +else +clear_dte_entry (iommu, dev_data->devid); + +clone_aliases (iommu, dev_data->dev); + +device_flush_dte (dev_data); +} +static int do_attach (struct iommu_dev_data *dev_data,struct protection_domain *domain) {
@@ -2068,10 +2083,7 @@ static int do_attach(struct iommu_dev_data *dev_data, }/* Update device table */ -set_dte_entry (iommu, dev_data); -clone_aliases (iommu, dev_data->dev); - -device_flush_dte (dev_data); +amd_iommu_dev_update_dte (dev_data, true);return ret; }
@@ -2092,11 +2104,9 @@ static void do_detach(struct iommu_dev_data *dev_data)/* Update data structures */ dev_data->domain =NULL ;list_del (&dev_data->list); -clear_dte_entry (iommu, dev_data->devid); -clone_aliases (iommu, dev_data->dev); -/* Flush the DTE entry */ -device_flush_dte (dev_data); +/* Clear DTE and flush the entry */ +amd_iommu_dev_update_dte (dev_data, false);/* Flush IOTLB and wait for the flushes to finish */ amd_iommu_domain_flush_all (domain);
Signature · 1 line
2.31.1
Vasant Hegde
maintainerPATCH v4 3/14 iommu/amd: Add support for enabling/disabling IOMMU features
In reply to Vasant HegdeFrom: Wei Huang <[email protected]>
Add support for struct iommu_ops.dev_{enable/disable}_feat. Please note that the empty feature switches will be populated by subsequent patches.
--- drivers/iommu/amd/iommu.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+)
drivers/iommu/amd/iommu.c +28 -0
@@ -2830,6 +2830,32 @@ static const struct iommu_dirty_ops amd_dirty_ops = { .read_and_clear_dirty = amd_iommu_read_and_clear_dirty, }; +static int amd_iommu_dev_enable_feature (struct device *dev, +enum iommu_dev_features feat) +{ +int ret; + +switch (feat) { +default : + ret = -EINVAL ; +break ; + } +return ret; +} + +static int amd_iommu_dev_disable_feature (struct device *dev, +enum iommu_dev_features feat) +{ +int ret; + +switch (feat) { +default : + ret = -EINVAL ; +break ; + } +return ret; +} +const struct iommu_ops amd_iommu_ops = { .capable = amd_iommu_capable, .domain_alloc = amd_iommu_domain_alloc,
@@ -2842,6 +2868,8 @@ const struct iommu_ops amd_iommu_ops = { .is_attach_deferred = amd_iommu_is_attach_deferred, .pgsize_bitmap = AMD_IOMMU_PGSIZES, .def_domain_type = amd_iommu_def_domain_type, + .dev_enable_feat = amd_iommu_dev_enable_feature, + .dev_disable_feat = amd_iommu_dev_disable_feature, .default_domain_ops = &(const struct iommu_domain_ops ) { .attach_dev = amd_iommu_attach_device, .map_pages = amd_iommu_map_pages,
Signature · 1 line
2.31.1
Vasant Hegde
maintainerPATCH v4 4/14 iommu/amd: Move PPR-related functions into ppr.c
In reply to Vasant HegdeFrom: Suravee Suthikulpanit <[email protected]>
In preparation to subsequent PPR-related patches, and also remove static declaration for certain helper functions so that it can be reused in other files.
Also rename below functions:
alloc_ppr_log -> amd_iommu_alloc_ppr_log
iommu_enable_ppr_log -> amd_iommu_enable_ppr_log
free_ppr_log -> amd_iommu_free_ppr_log
iommu_poll_ppr_log -> amd_iommu_poll_ppr_log
--- drivers/iommu/amd/Makefile | 2 +- drivers/iommu/amd/amd_iommu.h | 17 +++++- drivers/iommu/amd/init.c | 65 +++----------------- drivers/iommu/amd/iommu.c | 55 +---------------- drivers/iommu/amd/ppr.c | 112 ++++++++++++++++++++++++++++++++++ 5 files changed, 137 insertions(+), 114 deletions(-) create mode 100644 drivers/iommu/amd/ppr.c
drivers/iommu/amd/Makefile +1 -1
@@ -1,3 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_AMD_IOMMU) += iommu.o init.o quirks.o io_pgtable.o io_pgtable_v2.o +obj-$(CONFIG_AMD_IOMMU) += iommu.o init.o quirks.o io_pgtable.o io_pgtable_v2.o ppr.o obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += debugfs.o
drivers/iommu/amd/amd_iommu.h +14 -3
@@ -17,10 +17,16 @@ irqreturn_t amd_iommu_int_thread_pprlog(int irq, void *data);irqreturn_t amd_iommu_int_thread_galog (int irq,void *data);irqreturn_t amd_iommu_int_handler (int irq,void *data);void amd_iommu_apply_erratum_63 (struct amd_iommu *iommu,u16 devid); +void amd_iommu_restart_log (struct amd_iommu *iommu,const char *evt_type, +u8 cntrl_intr,u8 cntrl_log, +u32 status_run_mask,u32 status_overflow_mask);void amd_iommu_restart_event_logging (struct amd_iommu *iommu);void amd_iommu_restart_ga_log (struct amd_iommu *iommu);void amd_iommu_restart_ppr_log (struct amd_iommu *iommu);void amd_iommu_set_rlookup_table (struct amd_iommu *iommu,u16 devid); +void iommu_feature_enable (struct amd_iommu *iommu,u8 bit); +void *__init iommu_alloc_4k_pages (struct amd_iommu *iommu, +gfp_t gfp,size_t size);#ifdef CONFIG_AMD_IOMMU_DEBUGFS void amd_iommu_debugfs_setup (struct amd_iommu *iommu);
@@ -57,6 +63,14 @@ int amd_iommu_set_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid, unsigned long gcr3);int amd_iommu_clear_gcr3 (struct iommu_dev_data *dev_data,ioasid_t pasid); +/* PPR */ +int __init amd_iommu_alloc_ppr_log (struct amd_iommu *iommu); +void __init amd_iommu_free_ppr_log (struct amd_iommu *iommu); +void amd_iommu_enable_ppr_log (struct amd_iommu *iommu); +void amd_iommu_poll_ppr_log (struct amd_iommu *iommu); +int amd_iommu_complete_ppr (struct pci_dev *pdev,u32 pasid, +int status,int tag); +/* * This function flushes all internal caches of * the IOMMU used by this driver.
@@ -86,9 +100,6 @@ static inline int amd_iommu_create_irq_domain(struct amd_iommu *iommu)#define PPR_INVALID 0x1#define PPR_FAILURE 0xf -int amd_iommu_complete_ppr (struct pci_dev *pdev,u32 pasid, -int status,int tag); -static inline bool is_rd890_iommu (struct pci_dev *pdev) { return (pdev->vendor == PCI_VENDOR_ID_ATI) &&
drivers/iommu/amd/init.c +9 -56
113 lines not on this page · raw message
drivers/iommu/amd/iommu.c +1 -54
67 lines not on this page · raw message
drivers/iommu/amd/ppr.c new file +112 -0
112 lines not on this page · raw message
Signature · 1 line
2.31.1
Vasant Hegde
maintainerPATCH v4 5/14 iommu/amd: Fix PPR interrupt processing logic
In reply to Vasant Hegde* Do not re-read ppr head pointer as its just updated by the driver.
* Do not read PPR buffer tail pointer inside while loop. If IOMMU generates PPR events continuously then completing interrupt processing takes long time. In worst case it may cause infinite loop.
--- drivers/iommu/amd/ppr.c | 4 ---- 1 file changed, 4 deletions(-)
drivers/iommu/amd/ppr.c +0 -4
@@ -104,9 +104,5 @@ void amd_iommu_poll_ppr_log(struct amd_iommu *iommu)writel (head, iommu->mmio_base +MMIO_PPR_HEAD_OFFSET );/* TODO: PPR Handler will be added when we add IOPF support */ - -/* Refresh ring-buffer information */ - head =readl (iommu->mmio_base +MMIO_PPR_HEAD_OFFSET ); - tail =readl (iommu->mmio_base +MMIO_PPR_TAIL_OFFSET ); } }
Signature · 1 line
2.31.1
Vasant Hegde
maintainerPATCH v4 6/14 iommu/amd: Define per-IOMMU iopf_queue
In reply to Vasant HegdeFrom: Suravee Suthikulpanit <[email protected]>
AMD IOMMU hardware supports PCI Peripheral Paging Request (PPR) using a PPR log, which is a circular buffer containing requests from downstream end-point devices.
There is one PPR log per IOMMU instance. Therefore, allocate an iopf_queue per IOMMU instance during driver initialization, and free the queue during driver deinitialization.
Also rename enable_iommus_v2() -> enable_iommus_ppr() to reflect its usage.
--- drivers/iommu/amd/amd_iommu.h | 5 ++++ drivers/iommu/amd/amd_iommu_types.h | 4 +++ drivers/iommu/amd/init.c | 12 ++++++--- drivers/iommu/amd/ppr.c | 42 +++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 3 deletions(-)
drivers/iommu/amd/amd_iommu.h +5 -0
@@ -45,6 +45,11 @@ extern enum io_pgtable_fmt amd_iommu_pgtable;extern int amd_iommu_gpt_level;bool amd_iommu_pasid_supported (void ); + +/* IOPF */ +int amd_iommu_iopf_init (struct amd_iommu *iommu); +void amd_iommu_iopf_uninit (struct amd_iommu *iommu); +struct amd_iommu *get_amd_iommu (unsigned int idx);u8 amd_iommu_pc_get_max_banks (unsigned int idx);bool amd_iommu_pc_supported (void );
drivers/iommu/amd/amd_iommu_types.h +4 -0
@@ -769,6 +769,10 @@ struct amd_iommu {/* DebugFS Info */ struct dentry *debugfs;#endif + +/* IOPF support */ +struct iopf_queue *iopf_queue; +unsigned char iopfq_name[32 ]; };static inline struct amd_iommu *dev_to_amd_iommu (struct device *dev)
drivers/iommu/amd/init.c +9 -3
@@ -1638,6 +1638,7 @@ static void __init free_iommu_one(struct amd_iommu *iommu)amd_iommu_free_ppr_log (iommu);free_ga_log (iommu);iommu_unmap_mmio_space (iommu); +amd_iommu_iopf_uninit (iommu); }static void __init free_iommu_all (void )
@@ -2789,12 +2790,17 @@ static void early_enable_iommus(void) } } -static void enable_iommus_v2 (void ) +static void enable_iommus_ppr (void ) {struct amd_iommu *iommu; -for_each_iommu (iommu) +if (!amd_iommu_gt_ppr_supported ()) +return ; + +for_each_iommu (iommu) {amd_iommu_enable_ppr_log (iommu); +amd_iommu_iopf_init (iommu); + } }static void enable_iommus_vapic (void )
@@ -3130,7 +3136,7 @@ static int amd_iommu_enable_interrupts(void)* PPR and GA log interrupt for all IOMMUs. */ enable_iommus_vapic (); -enable_iommus_v2 (); +enable_iommus_ppr (); out:return ret;
drivers/iommu/amd/ppr.c +42 -0
45 lines not on this page · raw message
Signature · 1 line
2.31.1
Vasant Hegde
maintainerPATCH v4 7/14 iommu/amd: Add support for page response
In reply to Vasant HegdeFrom: Suravee Suthikulpanit <[email protected]>
This generates AMD IOMMU COMPLETE_PPR_REQUEST for the specified device with the specified PRI Response Code.
--- drivers/iommu/amd/amd_iommu.h | 3 +++ drivers/iommu/amd/iommu.c | 1 + drivers/iommu/amd/ppr.c | 15 +++++++++++++++ 3 files changed, 19 insertions(+)
drivers/iommu/amd/amd_iommu.h +3 -0
@@ -49,6 +49,9 @@ bool amd_iommu_pasid_supported(void);/* IOPF */ int amd_iommu_iopf_init (struct amd_iommu *iommu);void amd_iommu_iopf_uninit (struct amd_iommu *iommu); +int amd_iommu_page_response (struct device *dev, +struct iopf_fault *evt, +struct iommu_page_response *resp);struct amd_iommu *get_amd_iommu (unsigned int idx);u8 amd_iommu_pc_get_max_banks (unsigned int idx);
drivers/iommu/amd/iommu.c +1 -0
@@ -2817,6 +2817,7 @@ const struct iommu_ops amd_iommu_ops = { .def_domain_type = amd_iommu_def_domain_type, .dev_enable_feat = amd_iommu_dev_enable_feature, .dev_disable_feat = amd_iommu_dev_disable_feature, + .page_response = amd_iommu_page_response, .default_domain_ops = &(const struct iommu_domain_ops ) { .attach_dev = amd_iommu_attach_device, .map_pages = amd_iommu_map_pages,
drivers/iommu/amd/ppr.c +15 -0
@@ -148,3 +148,18 @@ void amd_iommu_iopf_uninit(struct amd_iommu *iommu)raw_spin_unlock_irqrestore (&iommu->lock, flags); } + +int amd_iommu_page_response (struct device *dev, +struct iopf_fault *evt, +struct iommu_page_response *resp) +{ +struct pci_dev *pdev; + +if (!dev || !dev_is_pci (dev)) +return -ENODEV ; + + pdev =to_pci_dev (dev); + +return amd_iommu_complete_ppr (pdev, resp->pasid, resp->code, + resp->grpid); +}
Signature · 1 line
2.31.1
Vasant Hegde
maintainerPATCH v4 8/14 iommu/amd: Add support for add/remove device for IOPF
In reply to Vasant HegdeFrom: Wei Huang <[email protected]>
Which adds/removes the device to the corresponding per-IOMMU iopf_queue. It also registers device fault handler. These interfaces are called from IOPF feature enable/disable path.
--- drivers/iommu/amd/amd_iommu.h | 2 ++ drivers/iommu/amd/ppr.c | 38 +++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+)
drivers/iommu/amd/amd_iommu.h +2 -0
@@ -49,6 +49,8 @@ bool amd_iommu_pasid_supported(void);/* IOPF */ int amd_iommu_iopf_init (struct amd_iommu *iommu);void amd_iommu_iopf_uninit (struct amd_iommu *iommu); +int amd_iommu_iopf_add_device (struct amd_iommu *iommu,struct device *dev); +int amd_iommu_iopf_remove_device (struct amd_iommu *iommu,struct device *dev);int amd_iommu_page_response (struct device *dev,struct iopf_fault *evt,struct iommu_page_response *resp);
drivers/iommu/amd/ppr.c +38 -0
@@ -163,3 +163,41 @@ int amd_iommu_page_response(struct device *dev,return amd_iommu_complete_ppr (pdev, resp->pasid, resp->code, resp->grpid); } + +int amd_iommu_iopf_add_device (struct amd_iommu *iommu,struct device *dev) +{ +unsigned long flags; +int ret = -EINVAL ; + +raw_spin_lock_irqsave (&iommu->lock, flags); + +if (!iommu->iopf_queue) + goto out; + + ret =iopf_queue_add_device (iommu->iopf_queue, dev); + +out: +raw_spin_unlock_irqrestore (&iommu->lock, flags); +return ret; +} + +int amd_iommu_iopf_remove_device (struct amd_iommu *iommu,struct device *dev) +{ +unsigned long flags; +int ret = -EINVAL ; + +raw_spin_lock_irqsave (&iommu->lock, flags); + +if (!iommu->iopf_queue) + goto out; + + ret =iopf_queue_remove_device (iommu->iopf_queue, dev); +if (ret) { +pr_warn ("Failed to remove device (0x%x) from iopf queue\n" , + dev->id); + } + +out: +raw_spin_unlock_irqrestore (&iommu->lock, flags); +return ret; +}
Signature · 1 line
2.31.1
Vasant Hegde
maintainerPATCH v4 9/14 iommu/amd: Add IO page fault notifier handler
In reply to Vasant HegdeFrom: Wei Huang <[email protected]>
Whenever there is a page fault IOMMU logs entry to ppr log and sends interrupt to host. We have to handle the page fault and respond to IOMMU.
Add support to validate page fault request and hook it to core iommu page fault handler.
--- drivers/iommu/amd/amd_iommu_types.h | 8 +++ drivers/iommu/amd/ppr.c | 100 +++++++++++++++++++++++++++- 2 files changed, 107 insertions(+), 1 deletion(-)
drivers/iommu/amd/amd_iommu_types.h +8 -0
@@ -251,6 +251,14 @@#define PPR_ENTRY_SIZE 16#define PPR_LOG_SIZE (PPR_ENTRY_SIZE * PPR_LOG_ENTRIES) +/* PAGE_SERVICE_REQUEST PPR Log Buffer Entry flags */ +#define PPR_FLAG_EXEC 0x002/* Execute permission requested */ +#define PPR_FLAG_READ 0x004/* Read permission requested */ +#define PPR_FLAG_WRITE 0x020/* Write permission requested */ +#define PPR_FLAG_US 0x040/* 1: User, 0: Supervisor */ +#define PPR_FLAG_RVSD 0x080/* Reserved bit not zero */ +#define PPR_FLAG_GN 0x100/* GVA and PASID is valid */ +#define PPR_REQ_TYPE (x) (((x) >> 60) & 0xfULL)#define PPR_FLAGS (x) (((x) >> 48) & 0xfffULL)#define PPR_DEVID (x) ((x) & 0xffffULL)
drivers/iommu/amd/ppr.c +99 -1
112 lines not on this page · raw message
Signature · 1 line
2.31.1
Vasant Hegde
maintainerPATCH v4 10/14 iommu/amd: Introduce logic to enable/disable IOPF
In reply to Vasant HegdeFrom: Suravee Suthikulpanit <[email protected]>
Add support to enable/disable IOPF feature. So that PRI capable device can make use of page fault handler in SVA mode.
Note that enable_feature(IOPF) just returns success, as this interface is going away. IOPF handler will be setup in PASID bind path. Following patch will add this support.
Also add IOMMU_IOPF as dependency to AMD_IOMMU driver.
--- drivers/iommu/amd/Kconfig | 1 + drivers/iommu/amd/amd_iommu.h | 2 ++ drivers/iommu/amd/iommu.c | 8 +++-- drivers/iommu/amd/ppr.c | 61 +++++++++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 2 deletions(-)
drivers/iommu/amd/Kconfig +1 -0
@@ -10,6 +10,7 @@ config AMD_IOMMU select IOMMU_API select IOMMU_IOVA select IOMMU_IO_PGTABLE + select IOMMU_IOPF select IOMMUFD_DRIVER if IOMMUFD depends on X86_64 && PCI && ACPI && HAVE_CMPXCHG_DOUBLE help
drivers/iommu/amd/amd_iommu.h +2 -0
@@ -54,6 +54,8 @@ int amd_iommu_iopf_remove_device(struct amd_iommu *iommu, struct device *dev);int amd_iommu_page_response (struct device *dev,struct iopf_fault *evt,struct iommu_page_response *resp); +int amd_iommu_iopf_enable_device (struct device *dev); +int amd_iommu_iopf_disable_device (struct device *dev);struct amd_iommu *get_amd_iommu (unsigned int idx);u8 amd_iommu_pc_get_max_banks (unsigned int idx);
drivers/iommu/amd/iommu.c +6 -2
@@ -2780,9 +2780,11 @@ static const struct iommu_dirty_ops amd_dirty_ops = {static int amd_iommu_dev_enable_feature (struct device *dev,enum iommu_dev_features feat) { -int ret; +int ret =0 ;switch (feat) { +case IOMMU_DEV_FEAT_IOPF : +break ;default : ret = -EINVAL ;break ;
@@ -2793,9 +2795,11 @@ static int amd_iommu_dev_enable_feature(struct device *dev,static int amd_iommu_dev_disable_feature (struct device *dev,enum iommu_dev_features feat) { -int ret; +int ret =0 ;switch (feat) { +case IOMMU_DEV_FEAT_IOPF : +break ;default : ret = -EINVAL ;break ;
drivers/iommu/amd/ppr.c +61 -0
64 lines not on this page · raw message
Signature · 1 line
2.31.1
Vasant Hegde
maintainerPATCH v4 11/14 iommu/amd: Add GCR3 [un]initialization function
In reply to Vasant HegdeFrom: Suravee Suthikulpanit <[email protected]>
These functions are used in PASID to device bind path. In this path it checks whether device GCR3 table is setup or not. If not it will setup GCR3 table.
--- drivers/iommu/amd/amd_iommu.h | 3 +++ drivers/iommu/amd/iommu.c | 44 +++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+)
drivers/iommu/amd/amd_iommu.h +3 -0
@@ -44,7 +44,10 @@ extern int amd_iommu_guest_ir;extern enum io_pgtable_fmt amd_iommu_pgtable;extern int amd_iommu_gpt_level; +/* SVA/PASID */ bool amd_iommu_pasid_supported (void ); +int amd_iommu_gcr3_init (struct iommu_dev_data *dev_data,ioasid_t pasids); +void amd_iommu_gcr3_uninit (struct iommu_dev_data *dev_data);/* IOPF */ int amd_iommu_iopf_init (struct amd_iommu *iommu);
drivers/iommu/amd/iommu.c +44 -0
56 lines not on this page · raw message
Signature · 1 line
2.31.1
Vasant Hegde
maintainerPATCH v4 12/14 iommu/amd: Initial SVA support for AMD IOMMU
In reply to Vasant HegdeThis includes :
- Add data structure to track per protection domain dev/pasid binding details
- Add support to check PASID is supported or not. Also setup gcr3 table. Also if PRI is supported then enable IOPF.
- Move 'to_pdomain()' to header file
- Add iommu_sva_set_dev_pasid(). It will check whether PASID is supported or not. Also adds PASID to SVA protection domain list as well as to device GCR3 table.
- Add iommu_ops.remove_dev_pasid support. It will unbind PASID from device. Also remove pasid data from protection domain.
- Add IOMMU_SVA as dependency to AMD_IOMMU driver
For a given PASID, iommu_set_dev_pasid() will bind all devices to same SVA protection domain (1 PASID : 1 SVA protection domain : N devices). This protection domain is different from device protection domain (one that's mapped in attach_device() path). IOMMU uses domain ID for caching, invalidation, etc. In SVA mode it will use per-device-domain-ID. Hence in invalidation path we retrieve domain ID from iommu_dev_data structure and use that for invalidation.
--- drivers/iommu/amd/Kconfig | 1 + drivers/iommu/amd/Makefile | 2 +- drivers/iommu/amd/amd_iommu.h | 11 ++ drivers/iommu/amd/amd_iommu_types.h | 14 ++ drivers/iommu/amd/iommu.c | 14 +- drivers/iommu/amd/pasid.c | 193 ++++++++++++++++++++++++++++ 6 files changed, 229 insertions(+), 6 deletions(-) create mode 100644 drivers/iommu/amd/pasid.c
drivers/iommu/amd/Kconfig +1 -0
@@ -10,6 +10,7 @@ config AMD_IOMMU select IOMMU_API select IOMMU_IOVA select IOMMU_IO_PGTABLE + select IOMMU_SVA select IOMMU_IOPF select IOMMUFD_DRIVER if IOMMUFD depends on X86_64 && PCI && ACPI && HAVE_CMPXCHG_DOUBLE
drivers/iommu/amd/Makefile +1 -1
@@ -1,3 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_AMD_IOMMU) += iommu.o init.o quirks.o io_pgtable.o io_pgtable_v2.o ppr.o +obj-$(CONFIG_AMD_IOMMU) += iommu.o init.o quirks.o io_pgtable.o io_pgtable_v2.o ppr.o pasid.o obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += debugfs.o
drivers/iommu/amd/amd_iommu.h +11 -0
@@ -44,6 +44,11 @@ extern int amd_iommu_guest_ir;extern enum io_pgtable_fmt amd_iommu_pgtable;extern int amd_iommu_gpt_level; +/* Protection domain ops */ +int iommu_sva_set_dev_pasid (struct iommu_domain *domain, +struct device *dev,ioasid_t pasid); +void amd_iommu_remove_dev_pasid (struct device *dev,ioasid_t pasid); +/* SVA/PASID */ bool amd_iommu_pasid_supported (void );int amd_iommu_gcr3_init (struct iommu_dev_data *dev_data,ioasid_t pasids);
@@ -72,6 +77,7 @@ int amd_iommu_pc_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr,/* Device capabilities */ int amd_iommu_pdev_enable_cap_pri (struct pci_dev *pdev);void amd_iommu_pdev_disable_cap_pri (struct pci_dev *pdev); +bool amd_iommu_pdev_pri_supported (struct pci_dev *pdev);/* GCR3 setup */ int amd_iommu_set_gcr3(struct iommu_dev_data *dev_data,
@@ -192,6 +198,11 @@ static inline struct amd_iommu *get_amd_iommu_from_dev(struct device *dev) return container_of(iommu, struct amd_iommu, iommu); } +static inline struct protection_domain *to_pdomain (struct iommu_domain *dom) +{ + return container_of(dom, struct protection_domain, domain); +} +bool translation_pre_enabled (struct amd_iommu *iommu);bool amd_iommu_is_attach_deferred (struct device *dev);int __init add_special_device (u8 type,u8 id,u32 *devid,bool cmd_line);
drivers/iommu/amd/amd_iommu_types.h +14 -0
@@ -8,7 +8,9 @@#ifndef _ASM_X86_AMD_IOMMU_TYPES_H#define _ASM_X86_AMD_IOMMU_TYPES_H +#include <linux/iommu.h> #include <linux/types.h> +#include <linux/mmu_notifier.h> #include <linux/mutex.h> #include <linux/msi.h> #include <linux/list.h>
@@ -559,6 +561,16 @@ enum protection_domain_mode { PD_MODE_V2, }; +/* Track PASID list for the protection domain */ +struct pdom_pasid_data { +/* PASID attached to the protection domain */ +ioasid_t pasid; +/* Points to attached device data */ +struct iommu_dev_data *dev_data; +/* Link to protection domain */ +struct list_head pdom_link; +}; +/* * This structure contains generic data for IOMMU protection domains * independent of their use.
@@ -575,6 +587,8 @@ struct protection_domain {bool dirty_tracking;/* dirty tracking is enabled in the domain */ unsigned dev_cnt;/* devices assigned to this domain */ unsigned dev_iommu[MAX_IOMMUS ];/* per-IOMMU reference count */ + +struct list_head pasid_list;/* List of pdom_pasid_data */ };/*
drivers/iommu/amd/iommu.c +9 -5
@@ -196,11 +196,6 @@ static struct amd_iommu *rlookup_amd_iommu(struct device *dev)return __rlookup_amd_iommu (seg,PCI_SBDF_TO_DEVID (devid)); } -static struct protection_domain *to_pdomain (struct iommu_domain *dom) -{ - return container_of(dom, struct protection_domain, domain); -} -static struct iommu_dev_data *alloc_dev_data (struct amd_iommu *iommu,u16 devid) {struct iommu_dev_data *dev_data;
@@ -346,6 +341,13 @@ static inline bool pdev_pasid_supported(struct iommu_dev_data *dev_data)return (dev_data->flags &AMD_IOMMU_DEVICE_FLAG_PASID_SUP ); } +inline bool amd_iommu_pdev_pri_supported (struct pci_dev *pdev) +{ +struct iommu_dev_data *dev_data =dev_iommu_priv_get (&pdev->dev); + +return (dev_data->flags &AMD_IOMMU_DEVICE_FLAG_PRI_SUP ); +} +static u32 pdev_get_caps (struct pci_dev *pdev) {int features;
@@ -2336,6 +2338,7 @@ static struct protection_domain *protection_domain_alloc(unsigned int type)spin_lock_init (&domain->lock);INIT_LIST_HEAD (&domain->dev_list); +INIT_LIST_HEAD (&domain->pasid_list); domain->nid =NUMA_NO_NODE ;switch (type) {
@@ -2865,6 +2868,7 @@ const struct iommu_ops amd_iommu_ops = { .def_domain_type = amd_iommu_def_domain_type, .dev_enable_feat = amd_iommu_dev_enable_feature, .dev_disable_feat = amd_iommu_dev_disable_feature, + .remove_dev_pasid = amd_iommu_remove_dev_pasid, .page_response = amd_iommu_page_response, .default_domain_ops = &(const struct iommu_domain_ops ) { .attach_dev = amd_iommu_attach_device,
drivers/iommu/amd/pasid.c new file +193 -0
193 lines not on this page · raw message
Signature · 1 line
2.31.1
Vasant Hegde
maintainerPATCH v4 13/14 iommu: Add ops->domain_alloc_sva()
In reply to Vasant HegdeFrom: Jason Gunthorpe <[email protected]>
Make a new op that receives the device and the mm_struct that the SVA domain should be created for. Unlike domain_alloc_paging() the dev argument is never NULL here.
This allows drivers to fully initialize the SVA domain and allocate the mmu_notifier during allocation. It allows the notifier lifetime to follow the lifetime of the iommu_domain.
Since we have only one call site, upgrade the new op to return ERR_PTR instead of NULL.
--- drivers/iommu/iommu-sva.c | 17 ++++++++++++----- include/linux/iommu.h | 3 +++ 2 files changed, 15 insertions(+), 5 deletions(-)
drivers/iommu/iommu-sva.c +12 -5
@@ -99,8 +99,8 @@ struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm/* Allocate a new domain and set it on device pasid. */ domain =iommu_sva_domain_alloc (dev, mm); -if (!domain) { - ret = -ENOMEM ; +if (IS_ERR (domain)) { + ret =PTR_ERR (domain); goto out_free_handle; }
@@ -270,11 +270,18 @@ struct iommu_domain *iommu_sva_domain_alloc(struct device *dev,const struct iommu_ops *ops =dev_iommu_ops (dev);struct iommu_domain *domain; - domain = ops->domain_alloc (IOMMU_DOMAIN_SVA ); -if (!domain) -return NULL ; +if (ops->domain_alloc_sva) { + domain = ops->domain_alloc_sva (dev, mm); +if (IS_ERR (domain)) +return domain; + }else { + domain = ops->domain_alloc (IOMMU_DOMAIN_SVA ); +if (!domain) +return ERR_PTR (-ENOMEM ); + } domain->type =IOMMU_DOMAIN_SVA ; + domain->owner = ops;mmgrab (mm); domain->mm = mm; domain->iopf_handler = iommu_sva_iopf_handler;
include/linux/iommu.h +3 -0
@@ -450,6 +450,7 @@ static inline int __iommu_copy_struct_from_user(* Upon failure, ERR_PTR must be returned. * @domain_alloc_paging: Allocate an iommu_domain that can be used for * UNMANAGED, DMA, and DMA_FQ domain types. +* @domain_alloc_sva: Allocate an iommu_domain for Shared Virtual Addressing. * @probe_device: Add device to iommu driver handling * @release_device: Remove device from iommu driver handling * @probe_finalize: Do final setup work after the device is added to an IOMMU
@@ -490,6 +491,8 @@ struct iommu_ops { struct device *dev, u32 flags, struct iommu_domain *parent, const struct iommu_user_data *user_data);struct iommu_domain *(*domain_alloc_paging)(struct device *dev); +struct iommu_domain *(*domain_alloc_sva)(struct device *dev, +struct mm_struct *mm);struct iommu_device *(*probe_device)(struct device *dev);void (*release_device)(struct device *dev);
Signature · 1 line
2.31.1
Re: [PATCH v4 13/14] iommu: Add ops->domain_alloc_sva()
Quoted 10 lines · Vasant Hegde
From: Jason Gunthorpe <[email protected]>
Make a new op that receives the device and the
mm_structthat the SVA domain should be created for. Unlikedomain_alloc_paging()the dev argument is never NULL here.This allows drivers to fully initialize the SVA domain and allocate the
mmu_notifierduring allocation. It allows the notifier lifetime to follow the lifetime of theiommu_domain.
Quoted 6 lines
Since we have only one call site, upgrade the new op to return
ERR_PTRinstead of NULL.Signed-off-by Jason Gunthorpe <[email protected]>[Updated domain owner and removed smmu3 related changes - Vasant]Signed-off-by Vasant Hegde <[email protected]>
Regards,
-Tina
Quoted 63 lines
--- drivers/iommu/iommu-sva.c | 17 ++++++++++++----- include/linux/iommu.h | 3 +++ 2 files changed, 15 insertions(+), 5 deletions(-)drivers/iommu/iommu-sva.c +12
-5@@ -99,8 +99,8 @@ struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm/* Allocate a new domain and set it on device pasid. */ domain =iommu_sva_domain_alloc (dev, mm); -if (!domain) { - ret = -ENOMEM ; +if (IS_ERR (domain)) { + ret =PTR_ERR (domain); goto out_free_handle; }@@ -270,11 +270,18 @@ struct iommu_domain *iommu_sva_domain_alloc(struct device *dev,const struct iommu_ops *ops =dev_iommu_ops (dev);struct iommu_domain *domain; - domain = ops->domain_alloc (IOMMU_DOMAIN_SVA ); -if (!domain) -return NULL ; +if (ops->domain_alloc_sva) { + domain = ops->domain_alloc_sva (dev, mm); +if (IS_ERR (domain)) +return domain; + }else { + domain = ops->domain_alloc (IOMMU_DOMAIN_SVA ); +if (!domain) +return ERR_PTR (-ENOMEM ); + } domain->type =IOMMU_DOMAIN_SVA ; + domain->owner = ops;mmgrab (mm); domain->mm = mm; domain->iopf_handler = iommu_sva_iopf_handler;include/linux/iommu.h +3
-0@@ -450,6 +450,7 @@ static inline int __iommu_copy_struct_from_user(* Upon failure, ERR_PTR must be returned. * @domain_alloc_paging: Allocate an iommu_domain that can be used for * UNMANAGED, DMA, and DMA_FQ domain types. +* @domain_alloc_sva: Allocate an iommu_domain for Shared Virtual Addressing. * @probe_device: Add device to iommu driver handling * @release_device: Remove device from iommu driver handling * @probe_finalize: Do final setup work after the device is added to an IOMMU @@ -490,6 +491,8 @@ struct iommu_ops { struct device *dev, u32 flags, struct iommu_domain *parent, const struct iommu_user_data *user_data);struct iommu_domain *(*domain_alloc_paging)(struct device *dev); +struct iommu_domain *(*domain_alloc_sva)(struct device *dev, +struct mm_struct *mm);struct iommu_device *(*probe_device)(struct device *dev);void (*release_device)(struct device *dev);
Re: [PATCH v4 13/14] iommu: Add ops->domain_alloc_sva()
In reply to Vasant HegdeQuoted 12 lines · Vasant Hegde
drivers/iommu/iommu-sva.c +5
-3@@ -270,11 +270,18 @@ struct iommu_domain *iommu_sva_domain_alloc(struct device *dev,const struct iommu_ops *ops =dev_iommu_ops (dev);struct iommu_domain *domain; - domain = ops->domain_alloc (IOMMU_DOMAIN_SVA ); -if (!domain) -return NULL ; +if (ops->domain_alloc_sva) { + domain = ops->domain_alloc_sva (dev, mm); +if (IS_ERR (domain)) +return domain; + }else {
drivers/iommu/iommu-sva.c struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, · line 287
Quoted 10 lines
drivers/iommu/iommu-sva.c +5
-0+ domain = ops->domain_alloc (IOMMU_DOMAIN_SVA ); +if (!domain) +return ERR_PTR (-ENOMEM ); + } domain->type =IOMMU_DOMAIN_SVA ; + domain->owner = ops;mmgrab (mm); domain->mm = mm; domain->iopf_handler = iommu_sva_iopf_handler;
This hunk is in commit 7be423336ecc ("iommu: Set owner token to SVA domain") which is already merged to Joerg's tree. Make sure to drop it from this patch when you rebase
Thanks,
Jason
Vasant Hegde
maintainerRe: [PATCH v4 13/14] iommu: Add ops->domain_alloc_sva()
Quoted 18 lines · Jason Gunthorpe
Quoted 17 lines · Vasant Hegde
drivers/iommu/iommu-sva.c +9
-3@@ -270,11 +270,18 @@ struct iommu_domain *iommu_sva_domain_alloc(struct device *dev,const struct iommu_ops *ops =dev_iommu_ops (dev);struct iommu_domain *domain; - domain = ops->domain_alloc (IOMMU_DOMAIN_SVA ); -if (!domain) -return NULL ; +if (ops->domain_alloc_sva) { + domain = ops->domain_alloc_sva (dev, mm); +if (IS_ERR (domain)) +return domain; + }else { + domain = ops->domain_alloc (IOMMU_DOMAIN_SVA ); +if (!domain) +return ERR_PTR (-ENOMEM ); + }
Quoted 9 lines
drivers/iommu/iommu-sva.c struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, · line 287
Quoted 5 lines
drivers/iommu/iommu-sva.c +1
-0domain->type =IOMMU_DOMAIN_SVA ; + domain->owner = ops;mmgrab (mm); domain->mm = mm; domain->iopf_handler = iommu_sva_iopf_handler;This hunk is in commit
7be423336ecc("iommu: Set owner token to SVA domain") which is already merged to Joerg's tree. Make sure to drop it from this patch when you rebase
Right. I will rebase and drop this hunk in next iteration.
-Vasant
Vasant Hegde
maintainerPATCH v4 14/14 iommu/amd: Add SVA domain support
In reply to Vasant Hegde- Allocate SVA domain and setup mmu notifier. In free path unregister mmu notifier and free protection domain.
- Add mmu notifier callback function. It will retrieve SVA protection domain and invalidates IO/TLB.
--- drivers/iommu/amd/amd_iommu.h | 5 ++ drivers/iommu/amd/amd_iommu_types.h | 1 + drivers/iommu/amd/iommu.c | 10 ++-- drivers/iommu/amd/pasid.c | 84 +++++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 3 deletions(-)
drivers/iommu/amd/amd_iommu.h +5 -0
@@ -45,6 +45,11 @@ extern enum io_pgtable_fmt amd_iommu_pgtable;extern int amd_iommu_gpt_level;/* Protection domain ops */ +struct protection_domain *protection_domain_alloc (unsigned int type); +void protection_domain_free (struct protection_domain *domain); +struct iommu_domain *amd_iommu_domain_alloc_sva (struct device *dev, +struct mm_struct *mm); +void amd_iommu_domain_free (struct iommu_domain *dom);int iommu_sva_set_dev_pasid (struct iommu_domain *domain,struct device *dev,ioasid_t pasid);void amd_iommu_remove_dev_pasid (struct device *dev,ioasid_t pasid);
drivers/iommu/amd/amd_iommu_types.h +1 -0
@@ -588,6 +588,7 @@ struct protection_domain {unsigned dev_cnt;/* devices assigned to this domain */ unsigned dev_iommu[MAX_IOMMUS ];/* per-IOMMU reference count */ +struct mmu_notifier mn;/* mmu notifier for the SVA domain */ struct list_head pasid_list;/* List of pdom_pasid_data */ };
drivers/iommu/amd/iommu.c +7 -3
@@ -2278,7 +2278,7 @@ static void cleanup_domain(struct protection_domain *domain)WARN_ON (domain->dev_cnt !=0 ); } -static void protection_domain_free (struct protection_domain *domain) +void protection_domain_free (struct protection_domain *domain) {if (!domain)return ;
@@ -2321,7 +2321,7 @@ static int protection_domain_init_v2(struct protection_domain *pdom)return 0 ; } -static struct protection_domain *protection_domain_alloc (unsigned int type) +struct protection_domain *protection_domain_alloc (unsigned int type) {struct io_pgtable_ops *pgtbl_ops;struct protection_domain *domain;
@@ -2344,6 +2344,7 @@ static struct protection_domain *protection_domain_alloc(unsigned int type)switch (type) {/* No need to allocate io pgtable ops in passthrough mode */ case IOMMU_DOMAIN_IDENTITY : +case IOMMU_DOMAIN_SVA :return domain;case IOMMU_DOMAIN_DMA : pgtable = amd_iommu_pgtable;
@@ -2463,7 +2464,7 @@ amd_iommu_domain_alloc_user(struct device *dev, u32 flags,return do_iommu_domain_alloc (type, dev, flags); } -static void amd_iommu_domain_free (struct iommu_domain *dom) +void amd_iommu_domain_free (struct iommu_domain *dom) {struct protection_domain *domain;unsigned long flags;
@@ -2831,6 +2832,7 @@ static int amd_iommu_dev_enable_feature(struct device *dev,switch (feat) {case IOMMU_DEV_FEAT_IOPF : +case IOMMU_DEV_FEAT_SVA :break ;default : ret = -EINVAL ;
@@ -2846,6 +2848,7 @@ static int amd_iommu_dev_disable_feature(struct device *dev,switch (feat) {case IOMMU_DEV_FEAT_IOPF : +case IOMMU_DEV_FEAT_SVA :break ;default : ret = -EINVAL ;
@@ -2858,6 +2861,7 @@ const struct iommu_ops amd_iommu_ops = { .capable = amd_iommu_capable, .domain_alloc = amd_iommu_domain_alloc, .domain_alloc_user = amd_iommu_domain_alloc_user, + .domain_alloc_sva = amd_iommu_domain_alloc_sva, .probe_device = amd_iommu_probe_device, .release_device = amd_iommu_release_device, .probe_finalize = amd_iommu_probe_finalize,
drivers/iommu/amd/pasid.c +84 -0
93 lines not on this page · raw message
Signature · 1 line
2.31.1