Convert Intel IOMMU to use sva-lib helpers
Subsystems (3)
Lu Baolu
maintainercover Convert Intel IOMMU to use sva-lib helpers
This patch series converts the Intel IOMMU to use the generic sva helpers defined in iommu-sva-lib.c and io-pgfault.c. It includes the SVA pasid management and IO page fault handling.
This series could be divided into below parts:
PATCH[1~3]: Use iommu_sva_alloc/free_pasid() to manage the SVA pasid. PATCH[4-6]: Use the generic sva io page fault handler iommu_queue_iopf. PATCH[7-11]: Add ftrace and debugfs support for io page fault handling.
Please help to review.
Best regards,
baolu
Lu Baolu (11):
iommu/vt-d: Add pasid private data helpers
iommu/vt-d: Use iommu_sva_alloc(free)_pasid() helpers iommu/vt-d: Use common helper to lookup svm devices iommu/vt-d: Refactor prq_event_thread()
iommu/vt-d: Allocate/register iopf queue for sva devices iommu/vt-d: Report prq to io-pgfault framework iommu/vt-d: Add prq_report trace event
iommu/vt-d: Add common code for dmar latency performance monitors iommu/vt-d: Expose latency monitor data through debugfs iommu/vt-d: Add cache invalidation latency sampling iommu/vt-d: Add PRQ handling latency sampling
include/linux/intel-iommu.h | 33 +- drivers/iommu/intel/perf.h | 73 ++++ include/trace/events/intel_iommu.h | 37 ++ drivers/iommu/intel/debugfs.c | 111 +++++ drivers/iommu/intel/dmar.c | 31 ++ drivers/iommu/intel/iommu.c | 73 +++- drivers/iommu/intel/perf.c | 166 ++++++++ drivers/iommu/intel/svm.c | 627 ++++++++++++++--------------- drivers/iommu/intel/Kconfig | 5 + drivers/iommu/intel/Makefile | 1 + 10 files changed, 819 insertions(+), 338 deletions(-) create mode 100644 drivers/iommu/intel/perf.h create mode 100644 drivers/iommu/intel/perf.c
Signature · 1 line
2.25.1
Lu Baolu
maintainerPATCH 1/11 iommu/vt-d: Add pasid private data helpers
We are about to use iommu_sva_alloc/free_pasid() helpers in iommu core. That means the pasid life cycle will be managed by iommu core. Use a local array to save the per pasid private data instead of attaching it the real pasid.
--- drivers/iommu/intel/svm.c | 62 ++++++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 21 deletions(-)
drivers/iommu/intel/svm.c +41 -21
@@ -17,6 +17,7 @@#include <linux/dmar.h> #include <linux/interrupt.h> #include <linux/mm_types.h> +#include <linux/xarray.h> #include <linux/ioasid.h> #include <asm/page.h> #include <asm/fpu/api.h>
@@ -28,6 +29,23 @@ static void intel_svm_drain_prq(struct device *dev, u32 pasid);#define PRQ_ORDER 0 +static DEFINE_XARRAY_ALLOC (pasid_private_array); +static int pasid_private_add (ioasid_t pasid,void *priv) +{ +return xa_alloc (&pasid_private_array, &pasid, priv, +XA_LIMIT (pasid, pasid),GFP_ATOMIC ); +} + +static void pasid_private_remove (ioasid_t pasid) +{ +xa_erase (&pasid_private_array, pasid); +} + +static void *pasid_private_find (ioasid_t pasid) +{ +return xa_load (&pasid_private_array, pasid); +} +int intel_svm_enable_prq (struct intel_iommu *iommu) {struct page *pages;
@@ -224,7 +242,7 @@ static int pasid_to_svm_sdev(struct device *dev, unsigned int pasid,if (pasid ==INVALID_IOASID || pasid >=PASID_MAX )return -EINVAL ; - svm =ioasid_find (NULL , pasid,NULL ); + svm =pasid_private_find (pasid);if (IS_ERR (svm))return PTR_ERR (svm);
@@ -334,7 +352,7 @@ int intel_svm_bind_gpasid(struct iommu_domain *domain, struct device *dev, svm->gpasid = data->gpasid; svm->flags |=SVM_FLAG_GUEST_PASID ; } -ioasid_set_data (data->hpasid, svm); +pasid_private_add (data->hpasid, svm);INIT_LIST_HEAD_RCU (&svm->devs);mmput (svm->mm); }
@@ -388,7 +406,7 @@ int intel_svm_bind_gpasid(struct iommu_domain *domain, struct device *dev,list_add_rcu (&sdev->list, &svm->devs); out:if (!IS_ERR_OR_NULL (svm) &&list_empty (&svm->devs)) { -ioasid_set_data (data->hpasid,NULL ); +pasid_private_remove (data->hpasid);kfree (svm); }
@@ -431,7 +449,7 @@ int intel_svm_unbind_gpasid(struct device *dev, u32 pasid)* the unbind, IOMMU driver will get notified * and perform cleanup. */ -ioasid_set_data (pasid,NULL ); +pasid_private_remove (pasid);kfree (svm); } }
@@ -547,8 +565,7 @@ intel_svm_bind_mm(struct device *dev, unsigned int flags, svm =kzalloc (sizeof (*svm),GFP_KERNEL );if (!svm) { ret = -ENOMEM ; -kfree (sdev); - goto out; + goto sdev_err; }if (pasid_max > intel_pasid_max_id)
@@ -556,13 +573,16 @@ intel_svm_bind_mm(struct device *dev, unsigned int flags,/* Do not use PASID 0, reserved for RID to PASID */ svm->pasid =ioasid_alloc (NULL ,PASID_MIN , - pasid_max -1 , svm); + pasid_max -1 ,NULL );if (svm->pasid ==INVALID_IOASID ) { -kfree (svm); -kfree (sdev); ret = -ENOSPC ; - goto out; + goto svm_err; } + + ret =pasid_private_add (svm->pasid, svm); +if (ret) + goto pasid_err; + svm->notifier.ops = &intel_mmuops; svm->mm = mm; svm->flags = flags;
@@ -571,12 +591,8 @@ intel_svm_bind_mm(struct device *dev, unsigned int flags, ret = -ENOMEM ;if (mm) { ret =mmu_notifier_register (&svm->notifier, mm); -if (ret) { -ioasid_put (svm->pasid); -kfree (svm); -kfree (sdev); - goto out; - } +if (ret) + goto priv_err; }spin_lock_irqsave (&iommu->lock, iflags);
@@ -590,8 +606,13 @@ intel_svm_bind_mm(struct device *dev, unsigned int flags,if (ret) {if (mm)mmu_notifier_unregister (&svm->notifier, mm); +priv_err: +pasid_private_remove (svm->pasid); +pasid_err:ioasid_put (svm->pasid); +svm_err:kfree (svm); +sdev_err:kfree (sdev); goto out; }
@@ -614,10 +635,8 @@ intel_svm_bind_mm(struct device *dev, unsigned int flags, (cpu_feature_enabled (X86_FEATURE_LA57 ) ? PASID_FLAG_FL5LP : 0));spin_unlock_irqrestore (&iommu->lock, iflags); -if (ret) { -kfree (sdev); - goto out; - } +if (ret) + goto sdev_err; }list_add_rcu (&sdev->list, &svm->devs); success:
@@ -670,6 +689,7 @@ static int intel_svm_unbind_mm(struct device *dev, u32 pasid)load_pasid (svm->mm,PASID_DISABLED ); }list_del (&svm->list); +pasid_private_remove (svm->pasid);/* We mandate that no page faults may be outstanding * for the PASID when intel_svm_unbind_mm() is called. * If that is not obeyed, subtle errors will happen.
@@ -924,7 +944,7 @@ static irqreturn_t prq_event_thread(int irq, void *d) }if (!svm || svm->pasid != req->pasid) {rcu_read_lock (); - svm =ioasid_find (NULL , req->pasid,NULL ); + svm =pasid_private_find (req->pasid);/* It *can't* go away, because the driver is not permitted * to unbind the mm while any page faults are outstanding. * So we only need RCU to protect the internal idr code. */
Signature · 1 line
2.25.1
Lu Baolu
maintainerPATCH 2/11 iommu/vt-d: Use iommu_sva_alloc(free)_pasid() helpers
In reply to Lu BaoluAlign the pasid alloc/free code with the generic helpers defined in the iommu core. This also refactored the SVA binding code to improve the readability.
--- include/linux/intel-iommu.h | 1 - drivers/iommu/intel/iommu.c | 3 + drivers/iommu/intel/svm.c | 278 +++++++++++++++--------------------- drivers/iommu/intel/Kconfig | 1 + 4 files changed, 120 insertions(+), 163 deletions(-)
include/linux/intel-iommu.h +0 -1
@@ -791,7 +791,6 @@ struct intel_svm {u32 pasid;int gpasid;/* In case that guest PASID is different from host PASID */ struct list_head devs; - struct list_head list; };#else static inline void intel_svm_check (struct intel_iommu *iommu) {}
drivers/iommu/intel/iommu.c +3 -0
@@ -5411,6 +5411,9 @@ intel_iommu_dev_enable_feat(struct device *dev, enum iommu_dev_features feat)if (!info)return -EINVAL ; +if (intel_iommu_enable_pasid (info->iommu, dev)) +return -ENODEV ; +if (!info->pasid_enabled || !info->pri_enabled || !info->ats_enabled)return -EINVAL ;
drivers/iommu/intel/svm.c +116 -162
373 lines not on this page · raw message
drivers/iommu/intel/Kconfig +1 -0
@@ -42,6 +42,7 @@ config INTEL_IOMMU_SVM select PCI_PRI select MMU_NOTIFIER select IOASID + select IOMMU_SVA_LIB help Shared Virtual Memory (SVM) provides a facility for devices to access DMA resources through process address space by
Signature · 6 lines
2.25.1 _______________________________________________ iommu mailing list [email protected] https://lists.linuxfoundation.org/mailman/listinfo/iommu
Lu Baolu
maintainerPATCH 3/11 iommu/vt-d: Use common helper to lookup svm devices
In reply to Lu BaoluIt's common to iterate the svm device list and find a matched device. Add common helpers to do this and consolidate the code.
--- drivers/iommu/intel/svm.c | 68 +++++++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 28 deletions(-)
drivers/iommu/intel/svm.c +40 -28
@@ -48,6 +48,40 @@ static void *pasid_private_find(ioasid_t pasid)return xa_load (&pasid_private_array, pasid); } +static struct intel_svm_dev * +svm_lookup_device_by_sid (struct intel_svm *svm,u16 sid) +{ +struct intel_svm_dev *sdev =NULL , *t; + +rcu_read_lock (); +list_for_each_entry_rcu (t, &svm->devs, list) { +if (t->sid == sid) { + sdev = t; +break ; + } + } +rcu_read_unlock (); + +return sdev; +} + +static struct intel_svm_dev * +svm_lookup_device_by_dev (struct intel_svm *svm,struct device *dev) +{ +struct intel_svm_dev *sdev =NULL , *t; + +rcu_read_lock (); +list_for_each_entry_rcu (t, &svm->devs, list) { +if (t->dev == dev) { + sdev = t; +break ; + } + } +rcu_read_unlock (); + +return sdev; +} +int intel_svm_enable_prq (struct intel_iommu *iommu) {struct page *pages;
@@ -225,15 +259,11 @@ static const struct mmu_notifier_ops intel_mmuops = {static DEFINE_MUTEX (pasid_mutex); -#define for_each_svm_dev (sdev, svm, d) \ - list_for_each_entry((sdev), &(svm)->devs, list) \ - if ((d) != (sdev)->dev) {} else -static int pasid_to_svm_sdev (struct device *dev,unsigned int pasid,struct intel_svm **rsvm,struct intel_svm_dev **rsdev) { -struct intel_svm_dev *d, *sdev =NULL ; +struct intel_svm_dev *sdev =NULL ;struct intel_svm *svm;/* The caller should hold the pasid_mutex lock */
@@ -256,15 +286,7 @@ static int pasid_to_svm_sdev(struct device *dev, unsigned int pasid,*/ if (WARN_ON (list_empty (&svm->devs)))return -EINVAL ; - -rcu_read_lock (); -list_for_each_entry_rcu (d, &svm->devs, list) { -if (d->dev == dev) { - sdev = d; -break ; - } - } -rcu_read_unlock (); + sdev =svm_lookup_device_by_dev (svm, dev); out: *rsvm = svm;
@@ -533,7 +555,8 @@ static struct iommu_sva *intel_svm_bind_mm(struct intel_iommu *iommu, }/* Find the matching device in svm list */ -for_each_svm_dev (sdev, svm, dev) { + sdev =svm_lookup_device_by_dev (svm, dev); +if (sdev) { sdev->users++; goto success; }
@@ -901,19 +924,8 @@ static irqreturn_t prq_event_thread(int irq, void *d) } } -if (!sdev || sdev->sid != req->rid) { -struct intel_svm_dev *t; - - sdev =NULL ; -rcu_read_lock (); -list_for_each_entry_rcu (t, &svm->devs, list) { -if (t->sid == req->rid) { - sdev = t; -break ; - } - } -rcu_read_unlock (); - } +if (!sdev || sdev->sid != req->rid) + sdev =svm_lookup_device_by_sid (svm, req->rid);/* Since we're using init_mm.pgd directly, we should never take * any faults on kernel addresses. */
Signature · 6 lines
2.25.1 _______________________________________________ iommu mailing list [email protected] https://lists.linuxfoundation.org/mailman/listinfo/iommu
Lu Baolu
maintainerPATCH 4/11 iommu/vt-d: Refactor prq_event_thread()
In reply to Lu BaoluRefactor prq_event_thread() by moving handling single prq event out of the main loop.
--- drivers/iommu/intel/svm.c | 239 ++++++++++++++++++++++---------------- 1 file changed, 136 insertions(+), 103 deletions(-)
drivers/iommu/intel/svm.c +136 -103
@@ -866,141 +866,174 @@ intel_svm_prq_report(struct device *dev, struct page_req_dsc *desc)return iommu_report_device_fault (dev, &event); } +static void handle_bad_prq_event (struct intel_iommu *iommu, +struct page_req_dsc *req,int result) +{ +struct qi_desc desc; + +pr_err ("%s: Invalid page request: %08llx %08llx\n" , + iommu->name, ((unsigned long long *)req)[0 ], + ((unsigned long long *)req)[1 ]); + +/* +* Per VT-d spec. v3.0 ch7.7, system software must +* respond with page group response if private data +* is present (PDP) or last page in group (LPIG) bit +* is set. This is an additional VT-d feature beyond +* PCI ATS spec. +*/ +if (!req->lpig && !req->priv_data_present) +return ; + + desc.qw0 =QI_PGRP_PASID (req->pasid) | +QI_PGRP_DID (req->rid) | +QI_PGRP_PASID_P (req->pasid_present) | +QI_PGRP_PDP (req->priv_data_present) | +QI_PGRP_RESP_CODE (result) | +QI_PGRP_RESP_TYPE ; + desc.qw1 =QI_PGRP_IDX (req->prg_index) | +QI_PGRP_LPIG (req->lpig); + desc.qw2 =0 ; + desc.qw3 =0 ; + +if (req->priv_data_present) +memcpy (&desc.qw2, req->priv_data,sizeof (req->priv_data)); +qi_submit_sync (iommu, &desc,1 ,0 ); +} + +static void handle_single_prq_event (struct intel_iommu *iommu, +struct mm_struct *mm, +struct page_req_dsc *req) +{ +u64 address = (u64 )req->addr <<VTD_PAGE_SHIFT ; +int result =QI_RESP_INVALID ; +struct vm_area_struct *vma; +struct qi_desc desc; +unsigned int flags; +vm_fault_t ret; + +/* If the mm is already defunct, don't handle faults. */ +if (!mmget_not_zero (mm)) + goto response; + +mmap_read_lock (mm); + vma =find_extend_vma (mm, address); +if (!vma || address < vma->vm_start) + goto invalid; + +if (access_error (vma, req)) + goto invalid; + + flags =FAULT_FLAG_USER |FAULT_FLAG_REMOTE ; +if (req->wr_req) + flags |=FAULT_FLAG_WRITE ; + + ret =handle_mm_fault (vma, address, flags,NULL ); +if (!(ret &VM_FAULT_ERROR )) + result =QI_RESP_SUCCESS ; +invalid: +mmap_read_unlock (mm); +mmput (mm); + +response: +if (!(req->lpig || req->priv_data_present)) +return ; + + desc.qw0 =QI_PGRP_PASID (req->pasid) | +QI_PGRP_DID (req->rid) | +QI_PGRP_PASID_P (req->pasid_present) | +QI_PGRP_PDP (req->priv_data_present) | +QI_PGRP_RESP_CODE (result) | +QI_PGRP_RESP_TYPE ; + desc.qw1 =QI_PGRP_IDX (req->prg_index) | +QI_PGRP_LPIG (req->lpig); + desc.qw2 =0 ; + desc.qw3 =0 ; + +if (req->priv_data_present) +memcpy (&desc.qw2, req->priv_data,sizeof (req->priv_data)); + +qi_submit_sync (iommu, &desc,1 ,0 ); +} +static irqreturn_t prq_event_thread (int irq,void *d) {struct intel_svm_dev *sdev =NULL ;struct intel_iommu *iommu = d;struct intel_svm *svm =NULL ; -int head, tail, handled =0 ; -unsigned int flags =0 ; +struct page_req_dsc *req; +int head, tail, handled; +u64 address; -/* Clear PPR bit before reading head/tail registers, to -* ensure that we get a new interrupt if needed. */ +/* +* Clear PPR bit before reading head/tail registers, to ensure that +* we get a new interrupt if needed. +*/ writel (DMA_PRS_PPR , iommu->reg +DMAR_PRS_REG ); tail =dmar_readq (iommu->reg +DMAR_PQT_REG ) &PRQ_RING_MASK ; head =dmar_readq (iommu->reg +DMAR_PQH_REG ) &PRQ_RING_MASK ; + handled = (head != tail);while (head != tail) { -struct vm_area_struct *vma; -struct page_req_dsc *req; -struct qi_desc resp; -int result; -vm_fault_t ret; -u64 address; - - handled =1 ; req = &iommu->prq[head /sizeof (*req)]; - result =QI_RESP_INVALID ; address = (u64 )req->addr <<VTD_PAGE_SHIFT ; -if (!req->pasid_present) { -pr_err ("%s: Page request without PASID: %08llx %08llx\n" , - iommu->name, ((unsigned long long *)req)[0 ], - ((unsigned long long *)req)[1 ]); - goto no_pasid; + +if (unlikely (!req->pasid_present)) { +pr_err ("IOMMU: %s: Page request without PASID\n" , + iommu->name); +bad_req: + svm =NULL ; + sdev =NULL ; +handle_bad_prq_event (iommu, req,QI_RESP_INVALID ); + goto prq_advance; } -/* We shall not receive page request for supervisor SVM */ -if (req->pm_req && (req->rd_req | req->wr_req)) { -pr_err ("Unexpected page request in Privilege Mode" ); -/* No need to find the matching sdev as for bad_req */ - goto no_pasid; + +if (unlikely (!is_canonical_address (address))) { +pr_err ("IOMMU: %s: Address is not canonical\n" , + iommu->name); + goto bad_req; + } + +if (unlikely (req->pm_req && (req->rd_req | req->wr_req))) { +pr_err ("IOMMU: %s: Page request in Privilege Mode\n" , + iommu->name); + goto bad_req; } -/* DMA read with exec requeset is not supported. */ -if (req->exe_req && req->rd_req) { -pr_err ("Execution request not supported\n" ); - goto no_pasid; + +if (unlikely (req->exe_req && req->rd_req)) { +pr_err ("IOMMU: %s: Execution request not supported\n" , + iommu->name); + goto bad_req; } +if (!svm || svm->pasid != req->pasid) { -rcu_read_lock (); - svm =pasid_private_find (req->pasid); -/* It *can't* go away, because the driver is not permitted +/* +* It can't go away, because the driver is not permitted * to unbind the mm while any page faults are outstanding. -* So we only need RCU to protect the internal idr code. */ -rcu_read_unlock (); -if (IS_ERR_OR_NULL (svm)) { -pr_err ("%s: Page request for invalid PASID %d: %08llx %08llx\n" , - iommu->name, req->pasid, ((unsigned long long *)req)[0 ], - ((unsigned long long *)req)[1 ]); - goto no_pasid; - } +*/ + svm =pasid_private_find (req->pasid); +if (IS_ERR_OR_NULL (svm) || (svm->flags &SVM_FLAG_SUPERVISOR_MODE )) + goto bad_req; } -if (!sdev || sdev->sid != req->rid) +if (!sdev || sdev->sid != req->rid) { sdev =svm_lookup_device_by_sid (svm, req->rid); - -/* Since we're using init_mm.pgd directly, we should never take -* any faults on kernel addresses. */ -if (!svm->mm) - goto bad_req; - -/* If address is not canonical, return invalid response */ -if (!is_canonical_address (address)) - goto bad_req; +if (!sdev) + goto bad_req; + }/* * If prq is to be handled outside iommu driver via receiver of * the fault notifiers, we skip the page response here. */ if (svm->flags &SVM_FLAG_GUEST_MODE ) { -if (sdev && !intel_svm_prq_report (sdev->dev, req)) +if (!intel_svm_prq_report (sdev->dev, req)) goto prq_advance;else goto bad_req; } -/* If the mm is already defunct, don't handle faults. */ -if (!mmget_not_zero (svm->mm)) - goto bad_req; - -mmap_read_lock (svm->mm); - vma =find_extend_vma (svm->mm, address); -if (!vma || address < vma->vm_start) - goto invalid; - -if (access_error (vma, req)) - goto invalid; - - flags =FAULT_FLAG_USER |FAULT_FLAG_REMOTE ; -if (req->wr_req) - flags |=FAULT_FLAG_WRITE ; - - ret =handle_mm_fault (vma, address, flags,NULL ); -if (ret &VM_FAULT_ERROR ) - goto invalid; - - result =QI_RESP_SUCCESS ; -invalid: -mmap_read_unlock (svm->mm); -mmput (svm->mm); -bad_req: -/* We get here in the error case where the PASID lookup failed, -and these can be NULL. Do not use them below this point! */ - sdev =NULL ; - svm =NULL ; -no_pasid: -if (req->lpig || req->priv_data_present) { -/* -* Per VT-d spec. v3.0 ch7.7, system software must -* respond with page group response if private data -* is present (PDP) or last page in group (LPIG) bit -* is set. This is an additional VT-d feature beyond -* PCI ATS spec. -*/ - resp.qw0 =QI_PGRP_PASID (req->pasid) | -QI_PGRP_DID (req->rid) | -QI_PGRP_PASID_P (req->pasid_present) | -QI_PGRP_PDP (req->priv_data_present) | -QI_PGRP_RESP_CODE (result) | -QI_PGRP_RESP_TYPE ; - resp.qw1 =QI_PGRP_IDX (req->prg_index) | -QI_PGRP_LPIG (req->lpig); - resp.qw2 =0 ; - resp.qw3 =0 ; - -if (req->priv_data_present) -memcpy (&resp.qw2, req->priv_data, -sizeof (req->priv_data)); -qi_submit_sync (iommu, &resp,1 ,0 ); - } +handle_single_prq_event (iommu, svm->mm, req); prq_advance: head = (head +sizeof (*req)) &PRQ_RING_MASK ; }
Signature · 6 lines
2.25.1 _______________________________________________ iommu mailing list [email protected] https://lists.linuxfoundation.org/mailman/listinfo/iommu
Lu Baolu
maintainerPATCH 5/11 iommu/vt-d: Allocate/register iopf queue for sva devices
In reply to Lu BaoluThis allocates and registers the iopf queue infrastructure for devices which want to support IO page fault for SVA.
--- include/linux/intel-iommu.h | 2 ++ drivers/iommu/intel/iommu.c | 66 ++++++++++++++++++++++++++----------- drivers/iommu/intel/svm.c | 37 +++++++++++++++++---- 3 files changed, 79 insertions(+), 26 deletions(-)
include/linux/intel-iommu.h +2 -0
@@ -606,6 +606,8 @@ struct intel_iommu {struct completion prq_complete;struct ioasid_allocator_ops pasid_allocator;/* Custom allocator for PASIDs */ #endif +struct iopf_queue *iopf_queue; +unsigned char iopfq_name[16 ];struct q_inval *qi;/* Queued invalidation info */ u32 *iommu_state;/* Store iommu states between suspend and resume.*/
drivers/iommu/intel/iommu.c +47 -19
97 lines not on this page · raw message
drivers/iommu/intel/svm.c +30 -7
@@ -84,6 +84,7 @@ svm_lookup_device_by_dev(struct intel_svm *svm, struct device *dev)int intel_svm_enable_prq (struct intel_iommu *iommu) { +struct iopf_queue *iopfq;struct page *pages;int irq, ret;
@@ -100,13 +101,20 @@ int intel_svm_enable_prq(struct intel_iommu *iommu)pr_err ("IOMMU: %s: Failed to create IRQ vector for page request queue\n" , iommu->name); ret = -EINVAL ; - err: -free_pages ((unsigned long )iommu->prq,PRQ_ORDER ); - iommu->prq =NULL ; -return ret; + goto free_prq; } iommu->pr_irq = irq; +snprintf (iommu->iopfq_name,sizeof (iommu->iopfq_name), +"dmar%d-iopfq" , iommu->seq_id); + iopfq =iopf_queue_alloc (iommu->iopfq_name); +if (!iopfq) { +pr_err ("IOMMU: %s: Failed to allocate iopf queue\n" , iommu->name); + ret = -ENOMEM ; + goto free_hwirq; + } + iommu->iopf_queue = iopfq; +snprintf (iommu->prq_name,sizeof (iommu->prq_name),"dmar%d-prq" , iommu->seq_id); ret = request_threaded_irq(irq, NULL, prq_event_thread, IRQF_ONESHOT,
@@ -114,9 +122,7 @@ int intel_svm_enable_prq(struct intel_iommu *iommu)if (ret) {pr_err ("IOMMU: %s: Failed to request IRQ for page request queue\n" , iommu->name); -dmar_free_hwirq (irq); - iommu->pr_irq =0 ; - goto err; + goto free_iopfq; }dmar_writeq (iommu->reg +DMAR_PQH_REG ,0ULL );dmar_writeq (iommu->reg +DMAR_PQT_REG ,0ULL );
@@ -125,6 +131,18 @@ int intel_svm_enable_prq(struct intel_iommu *iommu)init_completion (&iommu->prq_complete);return 0 ; + +free_iopfq: +iopf_queue_free (iommu->iopf_queue); + iommu->iopf_queue =NULL ; +free_hwirq: +dmar_free_hwirq (irq); + iommu->pr_irq =0 ; +free_prq: +free_pages ((unsigned long )iommu->prq,PRQ_ORDER ); + iommu->prq =NULL ; + +return ret; }int intel_svm_finish_prq (struct intel_iommu *iommu)
@@ -139,6 +157,11 @@ int intel_svm_finish_prq(struct intel_iommu *iommu) iommu->pr_irq =0 ; } +if (iommu->iopf_queue) { +iopf_queue_free (iommu->iopf_queue); + iommu->iopf_queue =NULL ; + } +free_pages ((unsigned long )iommu->prq,PRQ_ORDER ); iommu->prq =NULL ;
Signature · 6 lines
2.25.1 _______________________________________________ iommu mailing list [email protected] https://lists.linuxfoundation.org/mailman/listinfo/iommu
Lu Baolu
maintainerPATCH 6/11 iommu/vt-d: Report prq to io-pgfault framework
In reply to Lu BaoluLet the IO page fault requests get handled through the io-pgfault framework.
--- drivers/iommu/intel/iommu.c | 14 ++++++- drivers/iommu/intel/svm.c | 84 +++---------------------------------- 2 files changed, 17 insertions(+), 81 deletions(-)
drivers/iommu/intel/iommu.c +12 -2
@@ -5343,6 +5343,7 @@ static int intel_iommu_enable_sva(struct device *dev) {struct device_domain_info *info =get_domain_info (dev);struct intel_iommu *iommu = info->iommu; +int ret;if (!info || !iommu || dmar_disabled)return -EINVAL ;
@@ -5356,15 +5357,24 @@ static int intel_iommu_enable_sva(struct device *dev)if (intel_iommu_enable_pasid (iommu, dev))return -ENODEV ; -return iopf_queue_add_device (iommu->iopf_queue, dev); + ret =iopf_queue_add_device (iommu->iopf_queue, dev); +if (!ret) + ret =iommu_register_device_fault_handler (dev, iommu_queue_iopf, dev); + +return ret; }static int intel_iommu_disable_sva (struct device *dev) {struct device_domain_info *info =get_domain_info (dev);struct intel_iommu *iommu = info->iommu; +int ret; + + ret =iommu_unregister_device_fault_handler (dev); +if (!ret) + ret =iopf_queue_remove_device (iommu->iopf_queue, dev); -return iopf_queue_remove_device (iommu->iopf_queue, dev); +return ret; }/*
drivers/iommu/intel/svm.c +5 -79
114 lines not on this page · raw message
Signature · 6 lines
2.25.1 _______________________________________________ iommu mailing list [email protected] https://lists.linuxfoundation.org/mailman/listinfo/iommu
Lu Baolu
maintainerPATCH 7/11 iommu/vt-d: Add prq_report trace event
In reply to Lu BaoluThis adds a new trace event to track the page fault request report. This event will provide almost all information defined in a page request descriptor.
A sample output:
| prq_report: dmar0/0000:00:0a.0 seq# 1: rid=0x50 addr=0x559ef6f97 r---- pasid=0x2 index=0x1 | prq_report: dmar0/0000:00:0a.0 seq# 2: rid=0x50 addr=0x559ef6f9c rw--l pasid=0x2 index=0x1 | prq_report: dmar0/0000:00:0a.0 seq# 3: rid=0x50 addr=0x559ef6f98 r---- pasid=0x2 index=0x1 | prq_report: dmar0/0000:00:0a.0 seq# 4: rid=0x50 addr=0x559ef6f9d rw--l pasid=0x2 index=0x1 | prq_report: dmar0/0000:00:0a.0 seq# 5: rid=0x50 addr=0x559ef6f99 r---- pasid=0x2 index=0x1 | prq_report: dmar0/0000:00:0a.0 seq# 6: rid=0x50 addr=0x559ef6f9e rw--l pasid=0x2 index=0x1 | prq_report: dmar0/0000:00:0a.0 seq# 7: rid=0x50 addr=0x559ef6f9a r---- pasid=0x2 index=0x1 | prq_report: dmar0/0000:00:0a.0 seq# 8: rid=0x50 addr=0x559ef6f9f rw--l pasid=0x2 index=0x1
This will be helpful for I/O page fault related debugging.
--- include/linux/intel-iommu.h | 29 +++++++++++++++++++++++ include/trace/events/intel_iommu.h | 37 ++++++++++++++++++++++++++++++ drivers/iommu/intel/svm.c | 7 ++++++ 3 files changed, 73 insertions(+)
include/linux/intel-iommu.h +29 -0
@@ -778,6 +778,7 @@ struct intel_svm_dev {struct device *dev;struct intel_iommu *iommu;struct iommu_sva sva; +unsigned long prq_seq_number;u32 pasid;int users;u16 did;
@@ -828,4 +829,32 @@ static inline int iommu_calculate_max_sagaw(struct intel_iommu *iommu)#define intel_iommu_enabled (0)#endif +static inline const char *decode_prq_descriptor (char *str,size_t size, +u64 dw0,u64 dw1,u64 dw2,u64 dw3) +{ +char *buf = str; +int bytes; + + bytes =snprintf (buf, size, +"rid=0x%llx addr=0x%llx %c%c%c%c%c pasid=0x%llx index=0x%llx" , +FIELD_GET (GENMASK_ULL (31 ,16 ), dw0), +FIELD_GET (GENMASK_ULL (63 ,12 ), dw1), + dw1 &BIT_ULL (0 ) ?'r' :'-' , + dw1 &BIT_ULL (1 ) ?'w' :'-' , + dw0 &BIT_ULL (52 ) ?'x' :'-' , + dw0 &BIT_ULL (53 ) ?'p' :'-' , + dw1 &BIT_ULL (2 ) ?'l' :'-' , +FIELD_GET (GENMASK_ULL (51 ,32 ), dw0), +FIELD_GET (GENMASK_ULL (11 ,3 ), dw1)); + +/* Private Data */ +if (dw0 &BIT_ULL (9 )) { + size -= bytes; + buf += bytes; +snprintf (buf, size," private=0x%llx/0x%llx\n" , dw2, dw3); + } + +return str; +} +#endif
include/trace/events/intel_iommu.h +37 -0
@@ -15,6 +15,8 @@#include <linux/tracepoint.h> #include <linux/intel-iommu.h> +#define MSG_MAX 256 + TRACE_EVENT(qi_submit, TP_PROTO(struct intel_iommu *iommu, u64 qw0, u64 qw1, u64 qw2, u64 qw3),
@@ -51,6 +53,41 @@ TRACE_EVENT(qi_submit, __entry->qw0, __entry->qw1, __entry->qw2, __entry->qw3 ) ); + +TRACE_EVENT (prq_report, + TP_PROTO(struct intel_iommu *iommu, struct device *dev, + u64 dw0, u64 dw1, u64 dw2, u64 dw3, + unsigned long seq), + +TP_ARGS (iommu, dev, dw0, dw1, dw2, dw3, seq), + +TP_STRUCT__entry ( + __field(u64, dw0) + __field(u64, dw1) + __field(u64, dw2) + __field(u64, dw3) + __field(unsigned long, seq) + __string(iommu, iommu->name) + __string(dev, dev_name(dev)) +__dynamic_array (char, buff,MSG_MAX ) + ), + +TP_fast_assign ( + __entry->dw0 = dw0; + __entry->dw1 = dw1; + __entry->dw2 = dw2; + __entry->dw3 = dw3; + __entry->seq = seq; + __assign_str(iommu, iommu->name); + __assign_str(dev, dev_name(dev)); + ), + +TP_printk ("%s/%s seq# %ld: %s" , +__get_str (iommu),__get_str (dev), __entry->seq, +decode_prq_descriptor (__get_str (buff),MSG_MAX , __entry->dw0, + __entry->dw1, __entry->dw2, __entry->dw3) + ) +);#endif /* _TRACE_INTEL_IOMMU_H */ /* This part must be outside protection */
drivers/iommu/intel/svm.c +7 -0
@@ -21,6 +21,7 @@#include <linux/ioasid.h> #include <asm/page.h> #include <asm/fpu/api.h> +#include <trace/events/intel_iommu.h> #include "pasid.h" #include "../iommu-sva-lib.h"
@@ -976,12 +977,18 @@ static irqreturn_t prq_event_thread(int irq, void *d) goto bad_req; } + sdev->prq_seq_number++; +/* * If prq is to be handled outside iommu driver via receiver of * the fault notifiers, we skip the page response here. */ if (intel_svm_prq_report (sdev->dev, req))handle_bad_prq_event (iommu, req,QI_RESP_INVALID ); + +trace_prq_report (iommu, sdev->dev, req->qw_0, req->qw_1, + req->priv_data[0 ], req->priv_data[1 ], + sdev->prq_seq_number); prq_advance: head = (head +sizeof (*req)) &PRQ_RING_MASK ; }
Signature · 6 lines
2.25.1 _______________________________________________ iommu mailing list [email protected] https://lists.linuxfoundation.org/mailman/listinfo/iommu
Lu Baolu
maintainerPATCH 8/11 iommu/vt-d: Add common code for dmar latency performance monitors
In reply to Lu BaoluThe execution time of some operations is very performance critical, such as cache invalidation and PRQ processing time. This adds some common code to monitor the execution time range of those operations. The interfaces include enabling/disabling, checking status, updating sampling data and providing a common string format for users.
--- include/linux/intel-iommu.h | 1 + drivers/iommu/intel/perf.h | 73 +++++++++++++++ drivers/iommu/intel/perf.c | 166 +++++++++++++++++++++++++++++++++++ drivers/iommu/intel/Kconfig | 3 + drivers/iommu/intel/Makefile | 1 + 5 files changed, 244 insertions(+) create mode 100644 drivers/iommu/intel/perf.h create mode 100644 drivers/iommu/intel/perf.c
include/linux/intel-iommu.h +1 -0
@@ -621,6 +621,7 @@ struct intel_iommu {u32 flags;/* Software defined flags */ struct dmar_drhd_unit *drhd; +void *perf_statistic; };/* Per subdevice private data */
drivers/iommu/intel/perf.h new file +73 -0
73 lines not on this page · raw message
drivers/iommu/intel/perf.c new file +166 -0
166 lines not on this page · raw message
drivers/iommu/intel/Kconfig +3 -0
@@ -3,6 +3,9 @@ config DMAR_TABLE bool +config DMAR_PERF + bool + config INTEL_IOMMU bool "Support for Intel IOMMU using DMA Remapping Devices" depends on PCI_MSI && ACPI && (X86 || IA64)
drivers/iommu/intel/Makefile +1 -0
@@ -2,6 +2,7 @@ obj-$(CONFIG_DMAR_TABLE) += dmar.o obj-$(CONFIG_INTEL_IOMMU) += iommu.o pasid.o obj-$(CONFIG_DMAR_TABLE) += trace.o cap_audit.o +obj-$(CONFIG_DMAR_PERF) += perf.o obj-$(CONFIG_INTEL_IOMMU_DEBUGFS) += debugfs.o obj-$(CONFIG_INTEL_IOMMU_SVM) += svm.o obj-$(CONFIG_IRQ_REMAP) += irq_remapping.o
Signature · 6 lines
2.25.1 _______________________________________________ iommu mailing list [email protected] https://lists.linuxfoundation.org/mailman/listinfo/iommu
Lu Baolu
maintainerPATCH 9/11 iommu/vt-d: Expose latency monitor data through debugfs
In reply to Lu BaoluA debugfs interface /sys/kernel/debug/iommu/intel/dmar_perf_latency is created to control and show counts of execution time ranges for various types per DMAR. The interface may help debug any potential performance issue.
By default, the interface is disabled.
Possible write value of /sys/kernel/debug/iommu/intel/dmar_perf_latency
0 - disable sampling all latency data
1 - enable sampling IOTLB invalidation latency data
2 - enable sampling devTLB invalidation latency data
3 - enable sampling intr entry cache invalidation latency data 4 - enable sampling prq handling latency data
Read /sys/kernel/debug/iommu/intel/dmar_perf_latency gives a snapshot of sampling result of all enabled monitors.
--- drivers/iommu/intel/debugfs.c | 111 ++++++++++++++++++++++++++++++++++ drivers/iommu/intel/Kconfig | 1 + 2 files changed, 112 insertions(+)
drivers/iommu/intel/debugfs.c +111 -0
@@ -16,6 +16,7 @@#include <asm/irq_remapping.h> #include "pasid.h" +#include "perf.h" struct tbl_walk {u16 bus;
@@ -31,6 +32,9 @@ struct iommu_regset {const char *regs; }; +#define DEBUG_BUFFER_SIZE 1024 +static char debug_buf[DEBUG_BUFFER_SIZE ]; +#define IOMMU_REGSET_ENTRY (_reg_) \ { DMAR_##_reg_##_REG, __stringify(_reg_) }
@@ -538,6 +542,111 @@ static int ir_translation_struct_show(struct seq_file *m, void *unused)DEFINE_SHOW_ATTRIBUTE (ir_translation_struct);#endif +static void latency_show_one (struct seq_file *m,struct intel_iommu *iommu, +struct dmar_drhd_unit *drhd) +{ +int ret; + +seq_printf (m,"IOMMU: %s Register Base Address: %llx\n" , + iommu->name, drhd->reg_base_addr); + + ret =dmar_latency_snapshot (iommu, debug_buf,DEBUG_BUFFER_SIZE ); +if (ret <0 ) +seq_puts (m,"Failed to get latency snapshot" ); +else +seq_puts (m, debug_buf); +seq_puts (m,"\n" ); +} + +static int latency_show (struct seq_file *m,void *v) +{ +struct dmar_drhd_unit *drhd; +struct intel_iommu *iommu; + +rcu_read_lock (); +for_each_active_iommu (iommu, drhd) +latency_show_one (m, iommu, drhd); +rcu_read_unlock (); + +return 0 ; +} + +static int dmar_perf_latency_open (struct inode *inode,struct file *filp) +{ +return single_open (filp, latency_show,NULL ); +} + +static ssize_t dmar_perf_latency_write (struct file *filp, +const char __user *ubuf, +size_t cnt,loff_t *ppos) +{ +struct dmar_drhd_unit *drhd; +struct intel_iommu *iommu; +int counting; +char buf[64 ]; + +if (cnt >63 ) + cnt =63 ; + +if (copy_from_user (&buf, ubuf, cnt)) +return -EFAULT ; + + buf[cnt] =0 ; + +if (kstrtoint (buf,0 , &counting)) +return -EINVAL ; + +switch (counting) { +case 0 : +rcu_read_lock (); +for_each_active_iommu (iommu, drhd) { +dmar_latency_disable (iommu,DMAR_LATENCY_INV_IOTLB ); +dmar_latency_disable (iommu,DMAR_LATENCY_INV_DEVTLB ); +dmar_latency_disable (iommu,DMAR_LATENCY_INV_IEC ); +dmar_latency_disable (iommu,DMAR_LATENCY_PRQ ); + } +rcu_read_unlock (); +break ; +case 1 : +rcu_read_lock (); +for_each_active_iommu (iommu, drhd) +dmar_latency_enable (iommu,DMAR_LATENCY_INV_IOTLB ); +rcu_read_unlock (); +break ; +case 2 : +rcu_read_lock (); +for_each_active_iommu (iommu, drhd) +dmar_latency_enable (iommu,DMAR_LATENCY_INV_DEVTLB ); +rcu_read_unlock (); +break ; +case 3 : +rcu_read_lock (); +for_each_active_iommu (iommu, drhd) +dmar_latency_enable (iommu,DMAR_LATENCY_INV_IEC ); +rcu_read_unlock (); +break ; +case 4 : +rcu_read_lock (); +for_each_active_iommu (iommu, drhd) +dmar_latency_enable (iommu,DMAR_LATENCY_PRQ ); +rcu_read_unlock (); +break ; +default : +return -EINVAL ; + } + + *ppos += cnt; +return cnt; +} + +static const struct file_operations dmar_perf_latency_fops = { + .open = dmar_perf_latency_open, + .write = dmar_perf_latency_write, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; +void __init intel_iommu_debugfs_init (void ) { struct dentry *intel_iommu_debug = debugfs_create_dir("intel",
@@ -556,4 +665,6 @@ void __init intel_iommu_debugfs_init(void)debugfs_create_file ("ir_translation_struct" ,0444 , intel_iommu_debug,NULL , &ir_translation_struct_fops);#endif +debugfs_create_file ("dmar_perf_latency" ,0644 , intel_iommu_debug, +NULL , &dmar_perf_latency_fops); }
drivers/iommu/intel/Kconfig +1 -0
@@ -28,6 +28,7 @@ config INTEL_IOMMU config INTEL_IOMMU_DEBUGFS bool "Export Intel IOMMU internals in Debugfs" depends on INTEL_IOMMU && IOMMU_DEBUGFS + select DMAR_PERF help !!!WARNING!!!
Signature · 6 lines
2.25.1 _______________________________________________ iommu mailing list [email protected] https://lists.linuxfoundation.org/mailman/listinfo/iommu
Lu Baolu
maintainerPATCH 10/11 iommu/vt-d: Add cache invalidation latency sampling
In reply to Lu BaoluQueued invalidation execution time is performance critical and needs to be monitored. This adds code to sample the execution time of IOTLB/ devTLB/ICE cache invalidation.
--- drivers/iommu/intel/dmar.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+)
drivers/iommu/intel/dmar.c +31 -0
@@ -34,6 +34,7 @@#include <trace/events/intel_iommu.h> #include "../irq_remapping.h" +#include "perf.h" typedef int (*dmar_res_handler_t )(struct acpi_dmar_header *,void *);struct dmar_res_callback {
@@ -1340,15 +1341,33 @@ int qi_submit_sync(struct intel_iommu *iommu, struct qi_desc *desc, unsigned int count, unsigned long options) {struct q_inval *qi = iommu->qi; +s64 devtlb_start_ktime =0 ; +s64 iotlb_start_ktime =0 ; +s64 iec_start_ktime =0 ;struct qi_desc wait_desc;int wait_index, index;unsigned long flags;int offset, shift;int rc, i; +u64 type;if (!qi)return 0 ; + type = desc->qw0 &GENMASK_ULL (3 ,0 ); + +if ((type ==QI_IOTLB_TYPE || type ==QI_EIOTLB_TYPE ) && +dmar_latency_enabled (iommu,DMAR_LATENCY_INV_IOTLB )) + iotlb_start_ktime =ktime_to_ns (ktime_get ()); + +if ((type ==QI_DIOTLB_TYPE || type ==QI_DEIOTLB_TYPE ) && +dmar_latency_enabled (iommu,DMAR_LATENCY_INV_DEVTLB )) + devtlb_start_ktime =ktime_to_ns (ktime_get ()); + +if (type ==QI_IEC_TYPE && +dmar_latency_enabled (iommu,DMAR_LATENCY_INV_IEC )) + iec_start_ktime =ktime_to_ns (ktime_get ()); + restart: rc =0 ;
@@ -1423,6 +1442,18 @@ int qi_submit_sync(struct intel_iommu *iommu, struct qi_desc *desc,if (rc == -EAGAIN ) goto restart; +if (iotlb_start_ktime) +dmar_latency_update (iommu,DMAR_LATENCY_INV_IOTLB , +ktime_to_ns (ktime_get ()) - iotlb_start_ktime); + +if (devtlb_start_ktime) +dmar_latency_update (iommu,DMAR_LATENCY_INV_DEVTLB , +ktime_to_ns (ktime_get ()) - devtlb_start_ktime); + +if (iec_start_ktime) +dmar_latency_update (iommu,DMAR_LATENCY_INV_IEC , +ktime_to_ns (ktime_get ()) - iec_start_ktime); +return rc; }
Signature · 6 lines
2.25.1 _______________________________________________ iommu mailing list [email protected] https://lists.linuxfoundation.org/mailman/listinfo/iommu
Lu Baolu
maintainerPATCH 11/11 iommu/vt-d: Add PRQ handling latency sampling
In reply to Lu BaoluThe execution time for page fault request handling is performance critical and needs to be monitored. This adds code to sample the execution time of page fault request handling.
--- drivers/iommu/intel/svm.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-)
drivers/iommu/intel/svm.c +13 -3
@@ -24,6 +24,7 @@#include <trace/events/intel_iommu.h> #include "pasid.h" +#include "perf.h" #include "../iommu-sva-lib.h" static irqreturn_t prq_event_thread (int irq,void *d);
@@ -838,8 +839,8 @@ static int prq_to_iommu_prot(struct page_req_dsc *req)return prot; } -static int -intel_svm_prq_report (struct device *dev,struct page_req_dsc *desc) +static int intel_svm_prq_report (struct intel_iommu *iommu,struct device *dev, +struct page_req_dsc *desc) {struct iommu_fault_event event;
@@ -871,6 +872,12 @@ intel_svm_prq_report(struct device *dev, struct page_req_dsc *desc) event.fault.prm.flags |=IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA ;memcpy (event.fault.prm.private_data, desc->priv_data,sizeof (desc->priv_data)); + }else if (dmar_latency_enabled (iommu,DMAR_LATENCY_PRQ )) { +/* +* If the private data fields are not used by hardware, use it +* to monitor the prq handle latency. +*/ + event.fault.prm.private_data[0 ] =ktime_to_ns (ktime_get ()); }return iommu_report_device_fault (dev, &event);
@@ -983,7 +990,7 @@ static irqreturn_t prq_event_thread(int irq, void *d)* If prq is to be handled outside iommu driver via receiver of * the fault notifiers, we skip the page response here. */ -if (intel_svm_prq_report (sdev->dev, req)) +if (intel_svm_prq_report (iommu, sdev->dev, req))handle_bad_prq_event (iommu, req,QI_RESP_INVALID ); trace_prq_report(iommu, sdev->dev, req->qw_0, req->qw_1,
@@ -1172,6 +1179,9 @@ int intel_svm_page_response(struct device *dev,if (private_present)memcpy (&desc.qw2, prm->private_data,sizeof (prm->private_data)); +else if (prm->private_data[0 ]) +dmar_latency_update (iommu,DMAR_LATENCY_PRQ , +ktime_to_ns (ktime_get ()) - prm->private_data[0 ]);qi_submit_sync (iommu, &desc,1 ,0 ); }
Signature · 6 lines
2.25.1 _______________________________________________ iommu mailing list [email protected] https://lists.linuxfoundation.org/mailman/listinfo/iommu
Re: [PATCH 01/11] iommu/vt-d: Add pasid private data helpers
In reply to Lu BaoluHi BaoLu,
Quoted 5 lines · Lu Baolu
We are about to use iommu_sva_alloc/free_pasid() helpers in iommu core. That means the pasid life cycle will be managed by iommu core. Use a local array to save the per pasid private data instead of attaching it the real pasid.
I feel a little awkward to have a separate xarray for storing per IOASID data. Seems duplicated.
Jason suggested in another thread that we can make ioasid_data public and embeded in struct intel_svm, then we can get rid of the private data pointer. ioasid_find will return the ioasid_data, then we can retrieve the private data with container_of.
roughly,
struct intel_svm {
...
struct ioasid_data;
};struct ioasid_data {
ioasid_t id;
refcount_t refs;
struct mm_struct *mm;
};This can be a separate patch/effort if it make sense to you.
Quoted 169 lines
Signed-off-by Lu Baolu <[email protected]>--- drivers/iommu/intel/svm.c | 62 ++++++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 21 deletions(-)drivers/iommu/intel/svm.c +21
-3@@ -17,6 +17,7 @@#include <linux/dmar.h> #include <linux/interrupt.h> #include <linux/mm_types.h> +#include <linux/xarray.h> #include <linux/ioasid.h> #include <asm/page.h> #include <asm/fpu/api.h> @@ -28,6 +29,23 @@ static void intel_svm_drain_prq(struct device *dev, u32 pasid);#define PRQ_ORDER 0 +static DEFINE_XARRAY_ALLOC (pasid_private_array); +static int pasid_private_add (ioasid_t pasid,void *priv) +{ +return xa_alloc (&pasid_private_array, &pasid, priv, +XA_LIMIT (pasid, pasid),GFP_ATOMIC ); +} + +static void pasid_private_remove (ioasid_t pasid) +{ +xa_erase (&pasid_private_array, pasid); +} + +static void *pasid_private_find (ioasid_t pasid) +{ +return xa_load (&pasid_private_array, pasid); +} +int intel_svm_enable_prq (struct intel_iommu *iommu) {struct page *pages;@@ -224,7 +242,7 @@ static int pasid_to_svm_sdev(struct device *dev, unsigned int pasid, if (pasid == INVALID_IOASID || pasid >= PASID_MAX) return -EINVAL; - svm =ioasid_find (NULL , pasid,NULL ); + svm =pasid_private_find (pasid);if (IS_ERR (svm))return PTR_ERR (svm);@@ -334,7 +352,7 @@ int intel_svm_bind_gpasid(struct iommu_domain *domain, struct device *dev, svm->gpasid = data->gpasid; svm->flags |=SVM_FLAG_GUEST_PASID ; } -ioasid_set_data (data->hpasid, svm); +pasid_private_add (data->hpasid, svm);INIT_LIST_HEAD_RCU (&svm->devs);mmput (svm->mm); }@@ -388,7 +406,7 @@ int intel_svm_bind_gpasid(struct iommu_domain *domain, struct device *dev, list_add_rcu(&sdev->list, &svm->devs); out:if (!IS_ERR_OR_NULL (svm) &&list_empty (&svm->devs)) { -ioasid_set_data (data->hpasid,NULL ); +pasid_private_remove (data->hpasid);kfree (svm); }@@ -431,7 +449,7 @@ int intel_svm_unbind_gpasid(struct device *dev, u32 pasid) * the unbind, IOMMU driver will getnotified
drivers/iommu/intel/svm.c +11
-13* and perform cleanup. */ -ioasid_set_data (pasid,NULL ); +pasid_private_remove (pasid);kfree (svm); } }@@ -547,8 +565,7 @@ intel_svm_bind_mm(struct device *dev, unsigned int flags, svm =kzalloc (sizeof (*svm),GFP_KERNEL );if (!svm) { ret = -ENOMEM ; -kfree (sdev); - goto out; + goto sdev_err; }if (pasid_max > intel_pasid_max_id)@@ -556,13 +573,16 @@ intel_svm_bind_mm(struct device *dev, unsigned int flags,/* Do not use PASID 0, reserved for RID to PASID */ svm->pasid =ioasid_alloc (NULL ,PASID_MIN , - pasid_max -1 , svm); + pasid_max -1 ,NULL );if (svm->pasid ==INVALID_IOASID ) { -kfree (svm); -kfree (sdev); ret = -ENOSPC ; - goto out; + goto svm_err; } + + ret =pasid_private_add (svm->pasid, svm); +if (ret) + goto pasid_err; + svm->notifier.ops = &intel_mmuops; svm->mm = mm; svm->flags = flags;@@ -571,12 +591,8 @@ intel_svm_bind_mm(struct device *dev, unsigned int flags, ret = -ENOMEM ;if (mm) { ret =mmu_notifier_register (&svm->notifier, mm); -if (ret) { -ioasid_put (svm->pasid); -kfree (svm); -kfree (sdev); - goto out; - } +if (ret) + goto priv_err; }spin_lock_irqsave (&iommu->lock, iflags);@@ -590,8 +606,13 @@ intel_svm_bind_mm(struct device *dev, unsigned int flags, if (ret) {if (mm) mmu_notifier_unregister(&svm->notifier,mm); +priv_err:
drivers/iommu/intel/svm.c +4
-0+pasid_private_remove (svm->pasid); +pasid_err:ioasid_put (svm->pasid); +svm_err:kfree (svm); +sdev_err:kfree (sdev); goto out; }@@ -614,10 +635,8 @@ intel_svm_bind_mm(struct device *dev, unsigned intflags, (
cpu_feature_enabled(X86_FEATURE_LA57)?drivers/iommu/intel/svm.c +3
-4PASID_FLAG_FL5LP : 0));spin_unlock_irqrestore (&iommu->lock, iflags); -if (ret) { -kfree (sdev); - goto out; - } +if (ret) + goto sdev_err; }list_add_rcu (&sdev->list, &svm->devs); success:@@ -670,6 +689,7 @@ static int intel_svm_unbind_mm(struct device *dev, u32 pasid) load_pasid(svm->mm, PASID_DISABLED); }list_del (&svm->list); +pasid_private_remove (svm->pasid);/* We mandate that no page faults may be outstanding
* for the PASID whenintel_svm_unbind_mm()is called.
* If that is not obeyed, subtle errors
will happen. @@ -924,7 +944,7 @@ staticirqreturn_tprq_event_thread(int irq, void *d)}drivers/iommu/intel/svm.c +1
-1if (!svm || svm->pasid != req->pasid) {rcu_read_lock (); - svm =ioasid_find (NULL , req->pasid,NULL ); + svm =pasid_private_find (req->pasid);/* It *can't* go away, because the driver is not
Quoted 5 lines
permitted
* to unbind the mm while any page faults are
outstanding.
* So we only need RCU to protect the internal
idr code. */
Thanks,
Jacob
_______________________________________________
iommu mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/iommu
Lu Baolu
maintainerRe: [PATCH 01/11] iommu/vt-d: Add pasid private data helpers
Hi Jacob,
Thanks for reviewing my patch.
Quoted 10 lines · Jacob Pan
Hi BaoLu,
Quoted 5 lines · Lu Baolu
We are about to use iommu_sva_alloc/free_pasid() helpers in iommu core. That means the pasid life cycle will be managed by iommu core. Use a local array to save the per pasid private data instead of attaching it the real pasid.
Quoted 6 lines
I feel a little awkward to have a separate xarray for storing per IOASID data. Seems duplicated.
Jason suggested in another thread that we can makeioasid_datapublic and embeded in structintel_svm, then we can get rid of the private data pointer.ioasid_findwill return theioasid_data, then we can retrieve the private data withcontainer_of.
The problem that this patch wants to solve is that the iommu_sva_alloc_pasid() will attach the mm pointer to the sva pasid.
pasid = ioasid_alloc(&iommu_sva_pasid, min, max, mm);
Assuming that each sva pasid can have only a single private data pointer, the vendor iommu driver shouldn't set the private data again.
Quoted 8 lines
roughly,
struct intel_svm { ... struct ioasid_data; };
Quoted 7 lines
struct ioasid_data { ioasid_t id; refcount_t refs; struct mm_struct *mm; };This can be a separate patch/effort if it make sense to you.
Yes if we have a better solution.
Best regards,
baolu
Re: [PATCH 01/11] iommu/vt-d: Add pasid private data helpers
Hi Lu,
Quoted 22 lines · Lu Baolu
Hi Jacob,
Thanks for reviewing my patch.
Quoted 10 lines · Jacob Pan
Hi BaoLu,
Quoted 5 lines · Lu Baolu
We are about to use iommu_sva_alloc/free_pasid() helpers in iommu core. That means the pasid life cycle will be managed by iommu core. Use a local array to save the per pasid private data instead of attaching it the real pasid.
Quoted 6 lines
I feel a little awkward to have a separate xarray for storing per IOASID data. Seems duplicated.
Jason suggested in another thread that we can makeioasid_datapublic and embeded in structintel_svm, then we can get rid of the private data pointer.ioasid_findwill return theioasid_data, then we can retrieve the private data withcontainer_of.
Quoted 8 lines
The problem that this patch wants to solve is that the
iommu_sva_alloc_pasid()will attach the mm pointer to the sva pasid.pasid = ioasid_alloc(&iommu_sva_pasid, min, max, mm);Assuming that each sva pasid can have only a single private data pointer, the vendor iommu driver shouldn't set the private data again.
You are right. I got confused with vSVM, the guest will have the private data assigned after the bind.
Quoted 14 lines
Quoted 14 lines
roughly,
struct intel_svm { ... struct ioasid_data; };struct ioasid_data { ioasid_t id; refcount_t refs; struct mm_struct *mm; };
Quoted 4 lines
Quoted 1 line
This can be a separate patch/effort if it make sense to you.
Yes if we have a better solution.
Will be part of the IOASID core change.
Thanks,
Quoted 2 lines
Best regards,
baolu
Thanks,
Jacob