iommu/vt-d: Add a fix for devices need extra dtlb flush
Subsystems (2)
PATCH 1/1 iommu/vt-d: Add a fix for devices need extra dtlb flush
QAT devices on Intel Sapphire Rapids and Emerald Rapids has a defect in address translation service (ATS). These devices may inadvertently issue ATS invalidation completion before posted writes initiated with translated address that utilized translations matching the invalidation address range, violating the invalidation completion ordering.
An additional device TLB invalidation is needed to ensure no more posted writes with translated address following the invalidation completion.
Without this fix, DMA writes with translated address can happen after device TLB invalidation is completed. Random data corruption can occur as the DMA buffer gets reused after invalidation.
This patch adds a flag to mark the need for an extra flush based on given device IDs, this flag is set during initialization when ATS support is enabled.
At runtime, this flag is used to control the extra dTLB flush. The overhead of checking this unlikely true flag is smaller than checking PCI device IDs every time.
Device TLBs are invalidated under the following six conditions:
1. Device driver does DMA API unmap IOVA
2. Device driver unbind a PASID from a process, sva_unbind_device()
3. PASID is torn down, after PASID cache is flushed. e.g. process exit_mmap() due to crash
4. Under SVA usage, called by mmu_notifier.invalidate_range() where VM has to free pages that were unmapped
5. userspace driver unmaps a DMA buffer
6. Cache invalidation in vSVA usage (upcoming)
For #1 and #2, device drivers are responsible for stopping DMA traffic before unmap/unbind. For #3, iommu driver gets mmu_notifier to invalidate TLB the same way as normal user unmap which will do an extra invalidation. The dTLB invalidation after PASID cache flush does not need an extra invalidation.
Therefore, we only need to deal with #4 and #5 in this patch. #1 is also covered by this patch due to common code path with #5.
--- drivers/iommu/intel/iommu.c | 100 +++++++++++++++++++++++++++++------- drivers/iommu/intel/iommu.h | 3 ++ drivers/iommu/intel/svm.c | 4 +- 3 files changed, 87 insertions(+), 20 deletions(-)
drivers/iommu/intel/iommu.c +81 -19
@@ -1396,6 +1396,39 @@ static void domain_update_iotlb(struct dmar_domain *domain)spin_unlock_irqrestore (&domain->lock, flags); } +/* +* Check that the device does not live on an external facing PCI port that is +* marked as untrusted. Such devices should not be able to apply quirks and +* thus not be able to bypass the IOMMU restrictions. +*/ +static bool risky_device (struct pci_dev *pdev) +{ +if (pdev->untrusted) { +pci_info (pdev, +"Skipping IOMMU quirk for dev [%04X:%04X] on untrusted PCI link\n" , + pdev->vendor, pdev->device); +pci_info (pdev,"Please check with your BIOS/Platform vendor about this\n" ); +return true; + } +return false; +} + +/* Impacted QAT device IDs ranging from 0x4940 to 0x4943 */ +#define BUGGY_QAT_DEVID_MASK 0x494c +static bool dev_needs_extra_dtlb_flush (struct pci_dev *pdev) +{ +if (pdev->vendor !=PCI_VENDOR_ID_INTEL ) +return false; + +if ((pdev->device &0xfffc ) !=BUGGY_QAT_DEVID_MASK ) +return false; + +if (risky_device (pdev)) +return false; + +return true; +} +static void iommu_enable_pci_caps (struct device_domain_info *info) {struct pci_dev *pdev;
@@ -1478,6 +1511,7 @@ static void __iommu_flush_dev_iotlb(struct device_domain_info *info, qdep = info->ats_qdep;qi_flush_dev_iotlb (info->iommu, sid, info->pfsid, qdep, addr, mask); +quirk_dev_tlb (info, addr, mask,PASID_RID2PASID , qdep); } static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
@@ -4490,9 +4524,10 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev)if (dev_is_pci (dev)) {if (ecap_dev_iotlb_support (iommu->ecap) &&pci_ats_supported (pdev) && -dmar_ats_supported (pdev, iommu)) +dmar_ats_supported (pdev, iommu)) { info->ats_supported =1 ; - + info->dtlb_extra_inval =dev_needs_extra_dtlb_flush (pdev); + }if (sm_supported (iommu)) {if (pasid_supported (iommu)) {int features =pci_pasid_features (pdev);
@@ -4680,23 +4715,6 @@ static bool intel_iommu_is_attach_deferred(struct device *dev)return translation_pre_enabled (info->iommu) && !info->domain; } -/* -* Check that the device does not live on an external facing PCI port that is -* marked as untrusted. Such devices should not be able to apply quirks and -* thus not be able to bypass the IOMMU restrictions. -*/ -static bool risky_device (struct pci_dev *pdev) -{ -if (pdev->untrusted) { -pci_info (pdev, -"Skipping IOMMU quirk for dev [%04X:%04X] on untrusted PCI link\n" , - pdev->vendor, pdev->device); -pci_info (pdev,"Please check with your BIOS/Platform vendor about this\n" ); -return true; - } -return false; -} -static void intel_iommu_iotlb_sync_map (struct iommu_domain *domain,unsigned long iova,size_t size) {
@@ -4931,3 +4949,47 @@ static void __init check_tylersburg_isoch(void)pr_warn ("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n" , vtisochctrl); } + +/* +* Here we deal with a device TLB defect where device may inadvertently issue ATS +* invalidation completion before posted writes initiated with translated address +* that utilized translations matching the invalidation address range, violating +* the invalidation completion ordering. +* Therefore, any use cases that cannot guarantee DMA is stopped before unmap is +* vulnerable to this defect. In other words, any dTLB invalidation initiated not +* under the control of the trusted/privileged host device driver must use this +* quirk. +* Device TLBs are invalidated under the following six conditions: +* 1. Device driver does DMA API unmap IOVA +* 2. Device driver unbind a PASID from a process, sva_unbind_device() +* 3. PASID is torn down, after PASID cache is flushed. e.g. process +* exit_mmap() due to crash +* 4. Under SVA usage, called by mmu_notifier.invalidate_range() where +* VM has to free pages that were unmapped +* 5. Userspace driver unmaps a DMA buffer +* 6. Cache invalidation in vSVA usage (upcoming) +* +* For #1 and #2, device drivers are responsible for stopping DMA traffic +* before unmap/unbind. For #3, iommu driver gets mmu_notifier to +* invalidate TLB the same way as normal user unmap which will use this quirk. +* The dTLB invalidation after PASID cache flush does not need this quirk. +* +* As a reminder, #6 will *NEED* this quirk as we enable nested translation. +*/ +void quirk_dev_tlb (struct device_domain_info *info,unsigned long address, +unsigned long mask,u32 pasid,u16 qdep) +{ +u16 sid; + +if (likely (!info->dtlb_extra_inval)) +return ; + + sid =PCI_DEVID (info->bus, info->devfn); +if (pasid ==PASID_RID2PASID ) { +qi_flush_dev_iotlb (info->iommu, sid, info->pfsid, + qdep, address, mask); + }else { +qi_flush_dev_iotlb_pasid (info->iommu, sid, info->pfsid, + pasid, qdep, address, mask); + } +}
drivers/iommu/intel/iommu.h +3 -0
@@ -623,6 +623,7 @@ struct device_domain_info { u8 pri_enabled:1; u8 ats_supported:1; u8 ats_enabled:1; + u8 dtlb_extra_inval:1; /* Quirk for devices need extra flush */u8 ats_qdep;struct device *dev;/* it's NULL for PCIe-to-PCI bridge */ struct intel_iommu *iommu;/* IOMMU used by this device */
@@ -728,6 +729,8 @@ void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,void qi_flush_dev_iotlb_pasid (struct intel_iommu *iommu,u16 sid,u16 pfsid,u32 pasid,u16 qdep,u64 addr,unsigned int size_order); +void quirk_dev_tlb (struct device_domain_info *info,unsigned long address, +unsigned long pages,u32 pasid,u16 qdep);void qi_flush_pasid_cache (struct intel_iommu *iommu,u16 did,u64 granu,u32 pasid);
drivers/iommu/intel/svm.c +3 -1
@@ -184,10 +184,12 @@ static void __flush_svm_range_dev(struct intel_svm *svm,return ;qi_flush_piotlb (sdev->iommu, sdev->did, svm->pasid, address, pages, ih); -if (info->ats_enabled) +if (info->ats_enabled) {qi_flush_dev_iotlb_pasid (sdev->iommu, sdev->sid, info->pfsid, svm->pasid, sdev->qdep, address,order_base_2 (pages)); +quirk_dev_tlb (info, address,order_base_2 (pages), svm->pasid, sdev->qdep); + } } static void intel_flush_svm_range_dev(struct intel_svm *svm,
Signature · 1 line
2.25.1
Quoted 1 line · Jacob Pan
QAT devices on Intel Sapphire Rapids and Emerald Rapids has a defect in
s/has/have
Quoted 4 lines
address translation service (ATS). These devices may inadvertently issue ATS invalidation completion before posted writes initiated with translated address that utilized translations matching the invalidation address range, violating the invalidation completion ordering.
Above is a bit hard to read, can you see if the rephrasing works?
These devices send the invalidation response ahead of servicing any posted writes that could be using the same address range. This can lead to memory corruption.
Quoted 7 lines
An additional device TLB invalidation is needed to ensure no more posted writes with translated address following the invalidation completion.
Without this fix, DMA writes with translated address can happen after device TLB invalidation is completed. Random data corruption can occur as the DMA buffer gets reused after invalidation.
Maybe drop both para above and replace with.
An additional dTLB invalidation ensures the ordering is preserved and prevents any data-corruption.
Quoted 8 lines
This patch adds a flag to mark the need for an extra flush based on given device IDs, this flag is set during initialization when ATS support is enabled.
At runtime, this flag is used to control the extra dTLB flush. The overhead of checking this unlikely true flag is smaller than checking PCI device IDs every time.
The above 2 para's can be dropped? Since that's what the code does and probably not needed in the commit log.
Quoted 11 lines
Device TLBs are invalidated under the following six conditions:
1. Device driver does DMA API unmap IOVA
2. Device driver unbind a PASID from a process,sva_unbind_device()
3. PASID is torn down, after PASID cache is flushed. e.g. processexit_mmap()due to crash
4. Under SVA usage, called bymmu_notifier.invalidate_range()where VM has to free pages that were unmapped
5. userspace driver unmaps a DMA buffer
6. Cache invalidation in vSVA usage (upcoming)
Quoted 8 lines
For #1 and #2, device drivers are responsible for stopping DMA traffic before unmap/unbind. For #3, iommu driver gets
mmu_notifierto invalidate TLB the same way as normal user unmap which will do an extra invalidation. The dTLB invalidation after PASID cache flush does not need an extra invalidation.Therefore, we only need to deal with #4 and #5 in this patch. #1 is also covered by this patch due to common code path with #5.
I was told to add an imperative statement to tell what the patch does, without the code.
Maybe something like:
Add an extra device TLB invalidation for affected devices.
Quoted 4 lines
Tested-by Yuzhang Luo <[email protected]>Signed-off-by Jacob Pan <[email protected]>---
Quoted 185 lines
drivers/iommu/intel/iommu.c | 100 +++++++++++++++++++++++++++++------- drivers/iommu/intel/iommu.h | 3 ++ drivers/iommu/intel/svm.c | 4 +- 3 files changed, 87 insertions(+), 20 deletions(-)drivers/iommu/intel/iommu.c +81
-19@@ -1396,6 +1396,39 @@ static void domain_update_iotlb(struct dmar_domain *domain)spin_unlock_irqrestore (&domain->lock, flags); } +/* +* Check that the device does not live on an external facing PCI port that is +* marked as untrusted. Such devices should not be able to apply quirks and +* thus not be able to bypass the IOMMU restrictions. +*/ +static bool risky_device (struct pci_dev *pdev) +{ +if (pdev->untrusted) { +pci_info (pdev, +"Skipping IOMMU quirk for dev [%04X:%04X] on untrusted PCI link\n" , + pdev->vendor, pdev->device); +pci_info (pdev,"Please check with your BIOS/Platform vendor about this\n" ); +return true; + } +return false; +} + +/* Impacted QAT device IDs ranging from 0x4940 to 0x4943 */ +#define BUGGY_QAT_DEVID_MASK 0x494c +static bool dev_needs_extra_dtlb_flush (struct pci_dev *pdev) +{ +if (pdev->vendor !=PCI_VENDOR_ID_INTEL ) +return false; + +if ((pdev->device &0xfffc ) !=BUGGY_QAT_DEVID_MASK ) +return false; + +if (risky_device (pdev)) +return false; + +return true; +} +static void iommu_enable_pci_caps (struct device_domain_info *info) {struct pci_dev *pdev;@@ -1478,6 +1511,7 @@ static void __iommu_flush_dev_iotlb(struct device_domain_info *info, qdep = info->ats_qdep;qi_flush_dev_iotlb (info->iommu, sid, info->pfsid, qdep, addr, mask); +quirk_dev_tlb (info, addr, mask,PASID_RID2PASID , qdep); } static void iommu_flush_dev_iotlb(struct dmar_domain *domain,@@ -4490,9 +4524,10 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev)if (dev_is_pci (dev)) {if (ecap_dev_iotlb_support (iommu->ecap) &&pci_ats_supported (pdev) && -dmar_ats_supported (pdev, iommu)) +dmar_ats_supported (pdev, iommu)) { info->ats_supported =1 ; - + info->dtlb_extra_inval =dev_needs_extra_dtlb_flush (pdev); + }if (sm_supported (iommu)) {if (pasid_supported (iommu)) {int features =pci_pasid_features (pdev);@@ -4680,23 +4715,6 @@ static bool intel_iommu_is_attach_deferred(struct device *dev)return translation_pre_enabled (info->iommu) && !info->domain; } -/* -* Check that the device does not live on an external facing PCI port that is -* marked as untrusted. Such devices should not be able to apply quirks and -* thus not be able to bypass the IOMMU restrictions. -*/ -static bool risky_device (struct pci_dev *pdev) -{ -if (pdev->untrusted) { -pci_info (pdev, -"Skipping IOMMU quirk for dev [%04X:%04X] on untrusted PCI link\n" , - pdev->vendor, pdev->device); -pci_info (pdev,"Please check with your BIOS/Platform vendor about this\n" ); -return true; - } -return false; -} -static void intel_iommu_iotlb_sync_map (struct iommu_domain *domain,unsigned long iova,size_t size) {@@ -4931,3 +4949,47 @@ static void __init check_tylersburg_isoch(void)pr_warn ("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n" , vtisochctrl); } + +/* +* Here we deal with a device TLB defect where device may inadvertently issue ATS +* invalidation completion before posted writes initiated with translated address +* that utilized translations matching the invalidation address range, violating +* the invalidation completion ordering. +* Therefore, any use cases that cannot guarantee DMA is stopped before unmap is +* vulnerable to this defect. In other words, any dTLB invalidation initiated not +* under the control of the trusted/privileged host device driver must use this +* quirk. +* Device TLBs are invalidated under the following six conditions: +* 1. Device driver does DMA API unmap IOVA +* 2. Device driver unbind a PASID from a process, sva_unbind_device() +* 3. PASID is torn down, after PASID cache is flushed. e.g. process +* exit_mmap() due to crash +* 4. Under SVA usage, called by mmu_notifier.invalidate_range() where +* VM has to free pages that were unmapped +* 5. Userspace driver unmaps a DMA buffer +* 6. Cache invalidation in vSVA usage (upcoming) +* +* For #1 and #2, device drivers are responsible for stopping DMA traffic +* before unmap/unbind. For #3, iommu driver gets mmu_notifier to +* invalidate TLB the same way as normal user unmap which will use this quirk. +* The dTLB invalidation after PASID cache flush does not need this quirk. +* +* As a reminder, #6 will *NEED* this quirk as we enable nested translation. +*/ +void quirk_dev_tlb (struct device_domain_info *info,unsigned long address, +unsigned long mask,u32 pasid,u16 qdep) +{ +u16 sid; + +if (likely (!info->dtlb_extra_inval)) +return ; + + sid =PCI_DEVID (info->bus, info->devfn); +if (pasid ==PASID_RID2PASID ) { +qi_flush_dev_iotlb (info->iommu, sid, info->pfsid, + qdep, address, mask); + }else { +qi_flush_dev_iotlb_pasid (info->iommu, sid, info->pfsid, + pasid, qdep, address, mask); + } +}drivers/iommu/intel/iommu.h +3
-0@@ -623,6 +623,7 @@ struct device_domain_info { u8 pri_enabled:1; u8 ats_supported:1; u8 ats_enabled:1; + u8 dtlb_extra_inval:1; /* Quirk for devices need extra flush */u8 ats_qdep;struct device *dev;/* it's NULL for PCIe-to-PCI bridge */ struct intel_iommu *iommu;/* IOMMU used by this device */ @@ -728,6 +729,8 @@ void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,void qi_flush_dev_iotlb_pasid (struct intel_iommu *iommu,u16 sid,u16 pfsid,u32 pasid,u16 qdep,u64 addr,unsigned int size_order); +void quirk_dev_tlb (struct device_domain_info *info,unsigned long address, +unsigned long pages,u32 pasid,u16 qdep);void qi_flush_pasid_cache (struct intel_iommu *iommu,u16 did,u64 granu,u32 pasid);drivers/iommu/intel/svm.c +3
-1@@ -184,10 +184,12 @@ static void __flush_svm_range_dev(struct intel_svm *svm,return ;qi_flush_piotlb (sdev->iommu, sdev->did, svm->pasid, address, pages, ih); -if (info->ats_enabled) +if (info->ats_enabled) {qi_flush_dev_iotlb_pasid (sdev->iommu, sdev->sid, info->pfsid, svm->pasid, sdev->qdep, address,order_base_2 (pages)); +quirk_dev_tlb (info, address,order_base_2 (pages), svm->pasid, sdev->qdep); + } } static void intel_flush_svm_range_dev(struct intel_svm *svm,Signature · 1 line
2.25.1
Robin Murphy
reviewerQuoted 75 lines · Jacob Pan
QAT devices on Intel Sapphire Rapids and Emerald Rapids has a defect in address translation service (ATS). These devices may inadvertently issue ATS invalidation completion before posted writes initiated with translated address that utilized translations matching the invalidation address range, violating the invalidation completion ordering.
An additional device TLB invalidation is needed to ensure no more posted writes with translated address following the invalidation completion.
Without this fix, DMA writes with translated address can happen after device TLB invalidation is completed. Random data corruption can occur as the DMA buffer gets reused after invalidation.
This patch adds a flag to mark the need for an extra flush based on given device IDs, this flag is set during initialization when ATS support is enabled.
At runtime, this flag is used to control the extra dTLB flush. The overhead of checking this unlikely true flag is smaller than checking PCI device IDs every time.
Device TLBs are invalidated under the following six conditions:
1. Device driver does DMA API unmap IOVA
2. Device driver unbind a PASID from a process,sva_unbind_device()
3. PASID is torn down, after PASID cache is flushed. e.g. processexit_mmap()due to crash
4. Under SVA usage, called bymmu_notifier.invalidate_range()where VM has to free pages that were unmapped
5. userspace driver unmaps a DMA buffer
6. Cache invalidation in vSVA usage (upcoming)For #1 and #2, device drivers are responsible for stopping DMA traffic before unmap/unbind. For #3, iommu driver gets
mmu_notifierto invalidate TLB the same way as normal user unmap which will do an extra invalidation. The dTLB invalidation after PASID cache flush does not need an extra invalidation.Therefore, we only need to deal with #4 and #5 in this patch. #1 is also covered by this patch due to common code path with #5.
Tested-by Yuzhang Luo <[email protected]>Signed-off-by Jacob Pan <[email protected]>--- drivers/iommu/intel/iommu.c | 100 +++++++++++++++++++++++++++++------- drivers/iommu/intel/iommu.h | 3 ++ drivers/iommu/intel/svm.c | 4 +- 3 files changed, 87 insertions(+), 20 deletions(-)drivers/iommu/intel/iommu.c +19
-0@@ -1396,6 +1396,39 @@ static void domain_update_iotlb(struct dmar_domain *domain)spin_unlock_irqrestore (&domain->lock, flags); } +/* +* Check that the device does not live on an external facing PCI port that is +* marked as untrusted. Such devices should not be able to apply quirks and +* thus not be able to bypass the IOMMU restrictions. +*/ +static bool risky_device (struct pci_dev *pdev) +{ +if (pdev->untrusted) { +pci_info (pdev, +"Skipping IOMMU quirk for dev [%04X:%04X] on untrusted PCI link\n" , + pdev->vendor, pdev->device); +pci_info (pdev,"Please check with your BIOS/Platform vendor about this\n" ); +return true; + } +return false; +} + +/* Impacted QAT device IDs ranging from 0x4940 to 0x4943 */ +#define BUGGY_QAT_DEVID_MASK 0x494c
drivers/iommu/intel/iommu.c static void domain_update_iotlb(struct dmar_domain *domain) · line 1427
Quoted 10 lines
drivers/iommu/intel/iommu.c +10
-0+static bool dev_needs_extra_dtlb_flush (struct pci_dev *pdev) +{ +if (pdev->vendor !=PCI_VENDOR_ID_INTEL ) +return false; + +if ((pdev->device &0xfffc ) !=BUGGY_QAT_DEVID_MASK ) +return false; + +if (risky_device (pdev)) +return false;
Hmm, I'm not sure that that makes much sense to me - what privilege can the device gain from being told to invalidate things twice? Why would we want to implicitly *allow* a device to potentially keep using a stale translation if for some bizarre reason firmware has marked it as external, surely that's worse?
Robin.
Quoted 139 lines
drivers/iommu/intel/iommu.c +52
-19+ +return true; +} +static void iommu_enable_pci_caps (struct device_domain_info *info) {struct pci_dev *pdev;@@ -1478,6 +1511,7 @@ static void __iommu_flush_dev_iotlb(struct device_domain_info *info, qdep = info->ats_qdep;qi_flush_dev_iotlb (info->iommu, sid, info->pfsid, qdep, addr, mask); +quirk_dev_tlb (info, addr, mask,PASID_RID2PASID , qdep); } static void iommu_flush_dev_iotlb(struct dmar_domain *domain,@@ -4490,9 +4524,10 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev)if (dev_is_pci (dev)) {if (ecap_dev_iotlb_support (iommu->ecap) &&pci_ats_supported (pdev) && -dmar_ats_supported (pdev, iommu)) +dmar_ats_supported (pdev, iommu)) { info->ats_supported =1 ; - + info->dtlb_extra_inval =dev_needs_extra_dtlb_flush (pdev); + }if (sm_supported (iommu)) {if (pasid_supported (iommu)) {int features =pci_pasid_features (pdev);@@ -4680,23 +4715,6 @@ static bool intel_iommu_is_attach_deferred(struct device *dev)return translation_pre_enabled (info->iommu) && !info->domain; } -/* -* Check that the device does not live on an external facing PCI port that is -* marked as untrusted. Such devices should not be able to apply quirks and -* thus not be able to bypass the IOMMU restrictions. -*/ -static bool risky_device (struct pci_dev *pdev) -{ -if (pdev->untrusted) { -pci_info (pdev, -"Skipping IOMMU quirk for dev [%04X:%04X] on untrusted PCI link\n" , - pdev->vendor, pdev->device); -pci_info (pdev,"Please check with your BIOS/Platform vendor about this\n" ); -return true; - } -return false; -} -static void intel_iommu_iotlb_sync_map (struct iommu_domain *domain,unsigned long iova,size_t size) {@@ -4931,3 +4949,47 @@ static void __init check_tylersburg_isoch(void)pr_warn ("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n" , vtisochctrl); } + +/* +* Here we deal with a device TLB defect where device may inadvertently issue ATS +* invalidation completion before posted writes initiated with translated address +* that utilized translations matching the invalidation address range, violating +* the invalidation completion ordering. +* Therefore, any use cases that cannot guarantee DMA is stopped before unmap is +* vulnerable to this defect. In other words, any dTLB invalidation initiated not +* under the control of the trusted/privileged host device driver must use this +* quirk. +* Device TLBs are invalidated under the following six conditions: +* 1. Device driver does DMA API unmap IOVA +* 2. Device driver unbind a PASID from a process, sva_unbind_device() +* 3. PASID is torn down, after PASID cache is flushed. e.g. process +* exit_mmap() due to crash +* 4. Under SVA usage, called by mmu_notifier.invalidate_range() where +* VM has to free pages that were unmapped +* 5. Userspace driver unmaps a DMA buffer +* 6. Cache invalidation in vSVA usage (upcoming) +* +* For #1 and #2, device drivers are responsible for stopping DMA traffic +* before unmap/unbind. For #3, iommu driver gets mmu_notifier to +* invalidate TLB the same way as normal user unmap which will use this quirk. +* The dTLB invalidation after PASID cache flush does not need this quirk. +* +* As a reminder, #6 will *NEED* this quirk as we enable nested translation. +*/ +void quirk_dev_tlb (struct device_domain_info *info,unsigned long address, +unsigned long mask,u32 pasid,u16 qdep) +{ +u16 sid; + +if (likely (!info->dtlb_extra_inval)) +return ; + + sid =PCI_DEVID (info->bus, info->devfn); +if (pasid ==PASID_RID2PASID ) { +qi_flush_dev_iotlb (info->iommu, sid, info->pfsid, + qdep, address, mask); + }else { +qi_flush_dev_iotlb_pasid (info->iommu, sid, info->pfsid, + pasid, qdep, address, mask); + } +}drivers/iommu/intel/iommu.h +3
-0@@ -623,6 +623,7 @@ struct device_domain_info { u8 pri_enabled:1; u8 ats_supported:1; u8 ats_enabled:1; + u8 dtlb_extra_inval:1; /* Quirk for devices need extra flush */u8 ats_qdep;struct device *dev;/* it's NULL for PCIe-to-PCI bridge */ struct intel_iommu *iommu;/* IOMMU used by this device */ @@ -728,6 +729,8 @@ void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,void qi_flush_dev_iotlb_pasid (struct intel_iommu *iommu,u16 sid,u16 pfsid,u32 pasid,u16 qdep,u64 addr,unsigned int size_order); +void quirk_dev_tlb (struct device_domain_info *info,unsigned long address, +unsigned long pages,u32 pasid,u16 qdep);void qi_flush_pasid_cache (struct intel_iommu *iommu,u16 did,u64 granu,u32 pasid);drivers/iommu/intel/svm.c +3
-1@@ -184,10 +184,12 @@ static void __flush_svm_range_dev(struct intel_svm *svm,return ;qi_flush_piotlb (sdev->iommu, sdev->did, svm->pasid, address, pages, ih); -if (info->ats_enabled) +if (info->ats_enabled) {qi_flush_dev_iotlb_pasid (sdev->iommu, sdev->sid, info->pfsid, svm->pasid, sdev->qdep, address,order_base_2 (pages)); +quirk_dev_tlb (info, address,order_base_2 (pages), svm->pasid, sdev->qdep); + } } static void intel_flush_svm_range_dev(struct intel_svm *svm,
Quoted 14 lines
From: Robin Murphy <[email protected]>
Sent: Wednesday, November 23, 2022 1:49 AMQuoted 11 lines
drivers/iommu/intel/iommu.c +11
-0+ +/* Impacted QAT device IDs ranging from 0x4940 to 0x4943 */ +#define BUGGY_QAT_DEVID_MASK 0x494c +static bool dev_needs_extra_dtlb_flush (struct pci_dev *pdev) +{ +if (pdev->vendor !=PCI_VENDOR_ID_INTEL ) +return false; + +if ((pdev->device &0xfffc ) !=BUGGY_QAT_DEVID_MASK ) +return false; +
Quoted 9 lines
drivers/iommu/intel/iommu.c static void domain_update_iotlb(struct dmar_domain *domain) · line 1427
Quoted 2 lines
drivers/iommu/intel/iommu.c +2
-0+if (risky_device (pdev)) +return false;Hmm, I'm not sure that that makes much sense to me - what privilege can the device gain from being told to invalidate things twice? Why would we want to implicitly *allow* a device to potentially keep using a stale translation if for some bizarre reason firmware has marked it as external, surely that's worse?
ATS is disabled for such device hence no dtlb at all.
bool pci_ats_supported(struct pci_dev *dev)
{
if (!dev->ats_cap)
return false;
return (dev->untrusted == 0);
}So above check doesn't make things worse. It's kind of meaningless but according to Baolu he wants that check in every quirk...
Baolu Lu
maintainerQuoted 14 lines · Kevin
Quoted 14 lines
From: Robin Murphy <[email protected]>
Sent: Wednesday, November 23, 2022 1:49 AMQuoted 11 lines
drivers/iommu/intel/iommu.c +11
-0+ +/* Impacted QAT device IDs ranging from 0x4940 to 0x4943 */ +#define BUGGY_QAT_DEVID_MASK 0x494c +static bool dev_needs_extra_dtlb_flush (struct pci_dev *pdev) +{ +if (pdev->vendor !=PCI_VENDOR_ID_INTEL ) +return false; + +if ((pdev->device &0xfffc ) !=BUGGY_QAT_DEVID_MASK ) +return false; +
Quoted 8 lines
Quoted 8 lines
drivers/iommu/intel/iommu.c static void domain_update_iotlb(struct dmar_domain *domain) · line 1427
Quoted 2 lines
drivers/iommu/intel/iommu.c +2
-0+if (risky_device (pdev)) +return false;Hmm, I'm not sure that that makes much sense to me - what privilege can the device gain from being told to invalidate things twice? Why would we want to implicitly *allow* a device to potentially keep using a stale translation if for some bizarre reason firmware has marked it as external, surely that's worse?
From the perspective of IOMMU, any quirk is only applicable to trusted devices. If the IOMMU driver detects that a quirk is being applied to an untrusted device, it is already buggy or malicious. The IOMMU driver should let the users know by:
pci_info(pdev, "Skipping IOMMU quirk for dev [%04X:%04X] on untrusted PCI link\n", pdev->vendor, pdev->device); pci_info(pdev, "Please check with your BIOS/Platform vendor about this\n");
and stop applying any quirk.
Quoted 12 lines
Quoted 1 line
ATS is disabled for such device hence no dtlb at all.
bool
pci_ats_supported(struct pci_dev *dev){ if (!dev->ats_cap) return false; return (dev->untrusted == 0); }
Quoted 2 lines
So above check doesn't make things worse. It's kind of meaningless but according to Baolu he wants that check in every quirk...
At some time in the future, the hardware may support kind of enhanced ATS (or Secure ATS). At that time, above condition may be changed.
Best regards,
baolu
Quoted 32 lines
From: Baolu Lu <[email protected]>
Sent: Wednesday, November 23, 2022 1:04 PMQuoted 14 lines · Kevin
Quoted 14 lines
From: Robin Murphy <[email protected]>
Sent: Wednesday, November 23, 2022 1:49 AMQuoted 11 lines
drivers/iommu/intel/iommu.c +11
-0+ +/* Impacted QAT device IDs ranging from 0x4940 to 0x4943 */ +#define BUGGY_QAT_DEVID_MASK 0x494c +static bool dev_needs_extra_dtlb_flush (struct pci_dev *pdev) +{ +if (pdev->vendor !=PCI_VENDOR_ID_INTEL ) +return false; + +if ((pdev->device &0xfffc ) !=BUGGY_QAT_DEVID_MASK ) +return false; +Quoted 8 lines
Quoted 8 lines
drivers/iommu/intel/iommu.c static void domain_update_iotlb(struct dmar_domain *domain) · line 1427
Quoted 2 lines
drivers/iommu/intel/iommu.c +2
-0+if (risky_device (pdev)) +return false;Hmm, I'm not sure that that makes much sense to me - what privilege can the device gain from being told to invalidate things twice? Why would we want to implicitly *allow* a device to potentially keep using a stale translation if for some bizarre reason firmware has marked it as external, surely that's worse?
From the perspective of IOMMU, any quirk is only applicable to trusted
devices. If the IOMMU driver detects that a quirk is being applied to an untrusted device, it is already buggy or malicious. The IOMMU driver should let the users know by:
Quoted 9 lines
pci_info(pdev,
"Skipping IOMMU quirk for dev [%04X:%04X] on untrusted
PCI link\n",pdev->vendor, pdev->device); pci_info(pdev, "Please check with your BIOS/Platform vendor aboutthis\n");
and stop applying any quirk.
A quirk usually relaxes something then you want it only on trusted devices.
but the quirk in this patch is trying to fix a vulnerability. In concept it's weird to skip it on untrusted devices. This iiuc was the part causing confusion to Robin.
Robin Murphy
reviewerQuoted 39 lines · Kevin
Quoted 39 lines
From: Baolu Lu <[email protected]>
Sent: Wednesday, November 23, 2022 1:04 PMQuoted 14 lines · Kevin
Quoted 14 lines
From: Robin Murphy <[email protected]>
Sent: Wednesday, November 23, 2022 1:49 AMQuoted 11 lines
drivers/iommu/intel/iommu.c +11
-0+ +/* Impacted QAT device IDs ranging from 0x4940 to 0x4943 */ +#define BUGGY_QAT_DEVID_MASK 0x494c +static bool dev_needs_extra_dtlb_flush (struct pci_dev *pdev) +{ +if (pdev->vendor !=PCI_VENDOR_ID_INTEL ) +return false; + +if ((pdev->device &0xfffc ) !=BUGGY_QAT_DEVID_MASK ) +return false; +Quoted 8 lines
Quoted 8 lines
drivers/iommu/intel/iommu.c static void domain_update_iotlb(struct dmar_domain *domain) · line 1427
Quoted 2 lines
drivers/iommu/intel/iommu.c +2
-0+if (risky_device (pdev)) +return false;Hmm, I'm not sure that that makes much sense to me - what privilege can the device gain from being told to invalidate things twice? Why would we want to implicitly *allow* a device to potentially keep using a stale translation if for some bizarre reason firmware has marked it as external, surely that's worse?
From the perspective of IOMMU, any quirk is only applicable to trusted
devices. If the IOMMU driver detects that a quirk is being applied to an untrusted device, it is already buggy or malicious. The IOMMU driver should let the users know by:pci_info(pdev,
"Skipping IOMMU quirk for dev [%04X:%04X] on untrusted
PCI link\n",pdev->vendor, pdev->device); pci_info(pdev, "Please check with your BIOS/Platform vendor aboutthis\n");
Quoted 8 lines
Quoted 2 lines
and stop applying any quirk.
A quirk usually relaxes something then you want it only on trusted devices.
but the quirk in this patch is trying to fix a vulnerability. In concept it's weird to skip it on untrusted devices. This iiuc was the part causing confusion to Robin.
Right, it's that reasoning in general that seems bogus to me. Clearly any quirk that effectively grants additional privileges, like an identity mapping quirk, should not be applied to untrusted external devices which may be spoofing an affected VID/DID to gain that privilege, but not all quirks imply privilege. If, say, a WiFI controller needs something innocuous like a DMA alias or address width quirk to function correctly, it will still need that regardless of whether it's soldered to a motherboard or to a removable expansion card, and it would do nobody any good to deny correct functionality based on that unnecessary distinction. Yes, I appreciate that in practice many of those kind of quirks will be applied in other layers anyway, but I still think it's wrong to make a sweeping assumption that all IOMMU-level quirks are precious treasure not to be shared with outsiders, rather than assess their impact individually. The detriment in this case is small (just needless code churn), but even that's still not nothing.
Thanks,
Robin.
Baolu Lu
maintainerQuoted 43 lines · Robin Murphy
Quoted 42 lines · Kevin
Quoted 41 lines
From: Baolu Lu <[email protected]>
Sent: Wednesday, November 23, 2022 1:04 PMQuoted 14 lines · Kevin
Quoted 14 lines
From: Robin Murphy <[email protected]>
Sent: Wednesday, November 23, 2022 1:49 AMQuoted 11 lines
drivers/iommu/intel/iommu.c +11
-0+ +/* Impacted QAT device IDs ranging from 0x4940 to 0x4943 */ +#define BUGGY_QAT_DEVID_MASK 0x494c +static bool dev_needs_extra_dtlb_flush (struct pci_dev *pdev) +{ + if (pdev->vendor != PCI_VENDOR_ID_INTEL) + return false; + + if ((pdev->device & 0xfffc) != BUGGY_QAT_DEVID_MASK) + return false; +Quoted 8 lines
Quoted 8 lines
Quoted 2 lines
+ if (
risky_device(pdev))
+ return false;Hmm, I'm not sure that that makes much sense to me - what privilege can the device gain from being told to invalidate things twice? Why would we want to implicitly *allow* a device to potentially keep using a stale translation if for some bizarre reason firmware has marked it as external, surely that's worse?
From the perspective of IOMMU, any quirk is only applicable to trusted devices. If the IOMMU driver detects that a quirk is being applied to an untrusted device, it is already buggy or malicious. The IOMMU driver should let the users know by:
pci_info(pdev,
"Skipping IOMMU quirk for dev [%04X:%04X] on untrusted PCI link\n",
pdev->vendor,pdev->device);
pci_info(pdev, "Please check with your BIOS/Platform vendor about this\n");and stop applying any quirk.
Quoted 7 lines
Quoted 5 lines
A quirk usually relaxes something then you want it only on trusted devices.
but the quirk in this patch is trying to fix a vulnerability. In concept it's weird to skip it on untrusted devices. This iiuc was the part causing confusion to Robin.
Right, it's that reasoning in general that seems bogus to me. Clearly any quirk that effectively grants additional privileges, like an identity mapping quirk, should not be applied to untrusted external devices which may be spoofing an affected VID/DID to gain that privilege, but not all quirks imply privilege. If, say, a WiFI controller needs something innocuous like a DMA alias or address width quirk to function correctly, it will still need that regardless of whether it's soldered to a motherboard or to a removable expansion card, and it would do nobody any good to deny correct functionality based on that unnecessary distinction. Yes, I appreciate that in practice many of those kind of quirks will be applied in other layers anyway, but I still think it's wrong to make a sweeping assumption that all IOMMU-level quirks are precious treasure not to be shared with outsiders, rather than assess their impact individually. The detriment in this case is small (just needless code churn), but even that's still not nothing.
Fair enough. I agreed here.
Can we put some comments here so that people can still easily read the discussion here after a long time?
Best regards,
baolu
Hi Baolu,
Quoted 72 lines · Baolu Lu
Quoted 54 lines · Robin Murphy
Quoted 44 lines · Kevin
Quoted 34 lines
From: Baolu Lu <[email protected]>
Sent: Wednesday, November 23, 2022 1:04 PMQuoted 16 lines · Kevin
Quoted 16 lines
From: Robin Murphy <[email protected]>
Sent: Wednesday, November 23, 2022 1:49 AMQuoted 13 lines
(quoted diff) +13
-0+ +/* Impacted QAT device IDs ranging from 0x4940 to 0x4943 */ +#define BUGGY_QAT_DEVID_MASK 0x494c +static bool dev_needs_extra_dtlb_flush(struct pci_dev *pdev) +{ + if (pdev->vendor != PCI_VENDOR_ID_INTEL) + return false; + + if ((pdev->device & 0xfffc) != BUGGY_QAT_DEVID_MASK) + return false; + + if (risky_device(pdev)) + return false;Quoted 8 lines
Quoted 8 lines
Hmm, I'm not sure that that makes much sense to me - what privilege can
the device gain from being told to invalidate things twice? Why would we
want to implicitly *allow* a device to potentially keep using a stale translation if for some bizarre reason firmware has marked it as external, surely that's worse?From the perspective of IOMMU, any quirk is only applicable to trusted devices. If the IOMMU driver detects that a quirk is being applied to an untrusted device, it is already buggy or malicious. The IOMMU driver should let the users know by:
Quoted 9 lines
pci_info(pdev,
"Skipping IOMMU quirk for dev [%04X:%04X] on untrusted PCI link\n",
pdev->vendor,pdev->device);
pci_info(pdev, "Please check with your BIOS/Platform vendor about this\n");and stop applying any quirk.
Quoted 8 lines
A quirk usually relaxes something then you want it only on trusted devices.
but the quirk in this patch is trying to fix a vulnerability. In concept it's
weird to skip it on untrusted devices. This iiuc was the part causing confusion
to Robin.Quoted 16 lines
Right, it's that reasoning in general that seems bogus to me. Clearly any quirk that effectively grants additional privileges, like an identity mapping quirk, should not be applied to untrusted external devices which may be spoofing an affected VID/DID to gain that privilege, but not all quirks imply privilege. If, say, a WiFI controller needs something innocuous like a DMA alias or address width quirk to function correctly, it will still need that regardless of whether it's soldered to a motherboard or to a removable expansion card, and it would do nobody any good to deny correct functionality based on that unnecessary distinction. Yes, I appreciate that in practice many of those kind of quirks will be applied in other layers anyway, but I still think it's wrong to make a sweeping assumption that all IOMMU-level quirks are precious treasure not to be shared with outsiders, rather than assess their impact individually. The detriment in this case is small (just needless code churn), but even that's still not nothing.
Quoted 4 lines
Fair enough. I agreed here.
Can we put some comments here so that people can still easily read the discussion here after a long time?
Sure, I will remove risky_device(pdev) check and add a comment explaining the exemption.
Thanks,
Jacob
Hi Ashok,
Quoted 5 lines · Ashok Raj
Quoted 2 lines · Jacob Pan
QAT devices on Intel Sapphire Rapids and Emerald Rapids has a defect in
s/has/have
will do
Quoted 11 lines
Quoted 4 lines
address translation service (ATS). These devices may inadvertently issue ATS invalidation completion before posted writes initiated with translated address that utilized translations matching the invalidation address range, violating the invalidation completion ordering.
Above is a bit hard to read, can you see if the rephrasing works?
These devices send the invalidation response ahead of servicing any posted writes that could be using the same address range. This can lead to memory corruption.
It is a little long but these are taken from the errata draft. I think it is better to keep the wording consistent?
Quoted 4 lines
Quoted 4 lines
An additional device TLB invalidation is needed to ensure no more posted writes with translated address following the invalidation completion.
Quoted 8 lines
Quoted 3 lines
Without this fix, DMA writes with translated address can happen after device TLB invalidation is completed. Random data corruption can occur as the DMA buffer gets reused after invalidation.
Maybe drop both para above and replace with.
An additional dTLB invalidation ensures the ordering is preserved and prevents any data-corruption.
I can add this but I am not seeing any gains here.
Quoted 11 lines
Quoted 8 lines
This patch adds a flag to mark the need for an extra flush based on given device IDs, this flag is set during initialization when ATS support is enabled.
At runtime, this flag is used to control the extra dTLB flush. The overhead of checking this unlikely true flag is smaller than checking PCI device IDs every time.
The above 2 para's can be dropped? Since that's what the code does and probably not needed in the commit log.
sounds good.
Quoted 20 lines
Quoted 11 lines
Device TLBs are invalidated under the following six conditions:
1. Device driver does DMA API unmap IOVA
2. Device driver unbind a PASID from a process,sva_unbind_device()
3. PASID is torn down, after PASID cache is flushed. e.g. processexit_mmap()due to crash
4. Under SVA usage, called bymmu_notifier.invalidate_range()where VM has to free pages that were unmapped
5. userspace driver unmaps a DMA buffer
6. Cache invalidation in vSVA usage (upcoming)Quoted 8 lines
For #1 and #2, device drivers are responsible for stopping DMA traffic before unmap/unbind. For #3, iommu driver gets
mmu_notifierto invalidate TLB the same way as normal user unmap which will do an extra invalidation. The dTLB invalidation after PASID cache flush does not need an extra invalidation.Therefore, we only need to deal with #4 and #5 in this patch. #1 is also covered by this patch due to common code path with #5.
Quoted 7 lines
I was told to add an imperative statement to tell what the patch does, without the code.
Maybe something like:
Add an extra device TLB invalidation for affected devices.
Sounds good,I will merge this with the statement above.
Quoted 210 lines
Quoted 4 lines
Tested-by Yuzhang Luo <[email protected]>Signed-off-by Jacob Pan <[email protected]>---
Reviewed-by Ashok Raj <[email protected]>Quoted 203 lines
drivers/iommu/intel/iommu.c | 100 +++++++++++++++++++++++++++++------- drivers/iommu/intel/iommu.h | 3 ++ drivers/iommu/intel/svm.c | 4 +- 3 files changed, 87 insertions(+), 20 deletions(-)drivers/iommu/intel/iommu.c +2
-0@@ -1396,6 +1396,39 @@ static void domain_update_iotlb(struct dmar_domain *domain) spin_unlock_irqrestore(&domain->lock, flags); } +/* +* Check that the device does not live on an external facing PCI port that is
+ * marked as untrusted. Such devices should not be able to apply quirks and
+ * thus not be able to bypass the IOMMU restrictions.
+ */drivers/iommu/intel/iommu.c +5
-0+static bool risky_device (struct pci_dev *pdev) +{ +if (pdev->untrusted) { + pci_info(pdev, + "Skipping IOMMU quirk for dev [%04X:%04X] onuntrusted PCI link\n",
drivers/iommu/intel/iommu.c +2
-0+ pdev->vendor, pdev->device); + pci_info(pdev, "Please check with your BIOS/Platformvendor about this\n");
drivers/iommu/intel/iommu.c +25
-2+return true; + } +return false; +} + +/* Impacted QAT device IDs ranging from 0x4940 to 0x4943 */ +#define BUGGY_QAT_DEVID_MASK 0x494c +static bool dev_needs_extra_dtlb_flush (struct pci_dev *pdev) +{ +if (pdev->vendor !=PCI_VENDOR_ID_INTEL ) +return false; + +if ((pdev->device &0xfffc ) !=BUGGY_QAT_DEVID_MASK ) +return false; + +if (risky_device (pdev)) +return false; + +return true; +} +static void iommu_enable_pci_caps (struct device_domain_info *info) {struct pci_dev *pdev;@@ -1478,6 +1511,7 @@ static void __iommu_flush_dev_iotlb(structdevice_domain_info *info, qdep = info->ats_qdep;qi_flush_dev_iotlb (info->iommu, sid, info->pfsid, qdep, addr, mask); +quirk_dev_tlb (info, addr, mask,PASID_RID2PASID , qdep); } static void iommu_flush_dev_iotlb(struct dmar_domain *domain,@@ -4490,9 +4524,10 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev) if (dev_is_pci(dev)) {if (ecap_dev_iotlb_support (iommu->ecap) &&pci_ats_supported (pdev) && -dmar_ats_supported (pdev, iommu)) +dmar_ats_supported (pdev, iommu)) { info->ats_supported =1 ; - + info->dtlb_extra_inval =dev_needs_extra_dtlb_flush (pdev); + }if (sm_supported (iommu)) {if (pasid_supported (iommu)) {int features =
pci_pasid_features(pdev); @@ -4680,23 +4715,6 @@ static boolintel_iommu_is_attach_deferred(struct device *dev)returntranslation_pre_enabled(info->iommu)&& !info->domain; }-/*
- * Check that the device does not live on an external facing PCI port that is
- * marked as untrusted. Such devices should not be able to apply quirks and
- * thus not be able to bypass the IOMMU restrictions.
- */drivers/iommu/intel/iommu.c +0
-5-static bool risky_device (struct pci_dev *pdev) -{ -if (pdev->untrusted) { - pci_info(pdev, - "Skipping IOMMU quirk for dev [%04X:%04X] onuntrusted PCI link\n",
drivers/iommu/intel/iommu.c +0
-2- pdev->vendor, pdev->device); - pci_info(pdev, "Please check with your BIOS/Platformvendor about this\n");
drivers/iommu/intel/iommu.c +3
-5-return true; - } -return false; -} -static void intel_iommu_iotlb_sync_map (struct iommu_domain *domain,unsigned long iova,size_t size) {@@ -4931,3 +4949,47 @@ static void __init check_tylersburg_isoch(void) pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n", vtisochctrl); } + +/* + * Here we deal with a device TLB defect where device mayinadvertently issue ATS
+ * invalidation completion before posted writes initiated with translated address
+ * that utilized translations matching the invalidation address range, violating
+ * the invalidation completion ordering.
+ * Therefore, any use cases that cannot guarantee DMA is stopped before unmap is
+ * vulnerable to this defect. In other words, any dTLB invalidation initiated not
+ * under the control of the trusted/privileged host device driver must use this
+ * quirk.
+ * Device TLBs are invalidated under the following six conditions:
+ * 1. Device driver does DMA API unmap IOVA
+ * 2. Device driver unbind a PASID from a process,sva_unbind_device()
+ * 3. PASID is torn down, after PASID cache is flushed. e.g. process
+ *exit_mmap()due to crash
+ * 4. Under SVA usage, called bymmu_notifier.invalidate_range()where
+ * VM has to free pages that were unmapped
+ * 5. Userspace driver unmaps a DMA buffer
+ * 6. Cache invalidation in vSVA usage (upcoming)
+ *
+ * For #1 and #2, device drivers are responsible for stopping DMA traffic
+ * before unmap/unbind. For #3, iommu driver getsmmu_notifierto
+ * invalidate TLB the same way as normal user unmap which will use this quirk.
+ * The dTLB invalidation after PASID cache flush does not need this quirk.
+ *
+ * As a reminder, #6 will *NEED* this quirk as we enable nested translation.
+ */
+void quirk_dev_tlb(structdevice_domain_info*info, unsigned long address,drivers/iommu/intel/iommu.c +16
-0+ unsigned long mask, u32 pasid, u16 qdep) +{ + u16 sid; + +if (likely (!info->dtlb_extra_inval)) +return ; + + sid =PCI_DEVID (info->bus, info->devfn); +if (pasid ==PASID_RID2PASID ) { +qi_flush_dev_iotlb (info->iommu, sid, info->pfsid, + qdep, address, mask); + }else { +qi_flush_dev_iotlb_pasid (info->iommu, sid, info->pfsid, + pasid, qdep, address, mask); + } +}drivers/iommu/intel/iommu.h +1
-0@@ -623,6 +623,7 @@ struct device_domain_info { u8 pri_enabled:1; u8 ats_supported:1; u8 ats_enabled:1; + u8 dtlb_extra_inval:1; /* Quirk for devices need extraflush */ u8 ats_qdep;struct device *dev;/* it's NULL for PCIe-to-PCI bridge */ struct intel_iommu *iommu;/* IOMMU used by this device */ @@ -728,6 +729,8 @@ void qi_flush_piotlb(struct intel_iommu *iommu, u16did, u32 pasid, u64 addr, void qi_flush_dev_iotlb_pasid(struct
intel_iommu*iommu, u16 sid, u16 pfsid, u32 pasid, u16 qdep, u64 addr,drivers/iommu/intel/iommu.h +1
-0unsigned int size_order); +void quirk_dev_tlb(struct device_domain_info *info, unsigned longaddress,
drivers/iommu/intel/iommu.h +1
-0+ unsigned long pages, u32 pasid, u16 qdep); void qi_flush_pasid_cache(struct intel_iommu *iommu, u16 did, u64granu, u32 pasid);
drivers/iommu/intel/svm.c +3
-1@@ -184,10 +184,12 @@ static void __flush_svm_range_dev(structintel_svm *svm, return;qi_flush_piotlb (sdev->iommu, sdev->did, svm->pasid, address, pages, ih); -if (info->ats_enabled) +if (info->ats_enabled) {qi_flush_dev_iotlb_pasid (sdev->iommu, sdev->sid, info->pfsid, svm->pasid, sdev->qdep, address,order_base_2 (pages)); +quirk_dev_tlb (info, address,order_base_2 (pages), svm->pasid, sdev->qdep); + } }
Quoted 5 lines
Quoted 5 lines
static void intel_flush_svm_range_dev(struct
intel_svm*svm,Signature · 1 line
2.25.1
Thanks,
Jacob