x86 / iommu: set up a scratch page in the quarantine domain
PATCH 1/1 x86 / iommu: set up a scratch page in the quarantine domain
This patch introduces a new iommu_op to facilitate a per-implementation quarantine set up, and then further code for x86 implementations (amd and vtd) to set up a read/wrote scratch page to serve as the source/ target for all DMA whilst a device is assigned to dom_io.
The reason for doing this is that some hardware may continue to re-try DMA, despite FLR, in the event of an error. Having a scratch page mapped will allow pending DMA to drain and thus quiesce such buggy hardware.
---
Cc: Jan Beulich <[email protected]>
Cc: Andrew Cooper <[email protected]>
Cc: Kevin Tian <[email protected]>
Cc: Wei Liu <[email protected]>
Cc: "Roger Pau Monné" <[email protected]>
--- xen/drivers/passthrough/amd/iommu_map.c | 57 +++++++++++++++ xen/drivers/passthrough/amd/pci_amd_iommu.c | 9 +-- xen/drivers/passthrough/iommu.c | 25 ++++++- xen/drivers/passthrough/vtd/iommu.c | 71 +++++++++++++++---- xen/include/asm-x86/hvm/svm/amd-iommu-proto.h | 2 + xen/include/xen/iommu.h | 1 + 6 files changed, 143 insertions(+), 22 deletions(-)
xen/drivers/passthrough/amd/iommu_map.c +57 -0
@@ -560,6 +560,63 @@ int amd_iommu_reserve_domain_unity_map(struct domain *domain,return rt; } +int amd_iommu_quarantine_init (struct domain *d) +{ +struct domain_iommu *hd =dom_iommu (d); +unsigned int level; +struct amd_iommu_pte *table; + +if ( hd->arch.root_table ) + { +ASSERT_UNREACHABLE (); +return 0 ; + } + +spin_lock (&hd->arch.mapping_lock); + + level = hd->arch.paging_mode; + + hd->arch.root_table =alloc_amd_iommu_pgtable (); +if ( !hd->arch.root_table ) + goto out; + + table =__map_domain_page (hd->arch.root_table); +while ( level ) + { +struct page_info *pg; +unsigned int i; + +/* +* The pgtable allocator is fine for the leaf page, as well as +* page table pages. +*/ + pg =alloc_amd_iommu_pgtable (); +if ( !pg ) +break ; + +for ( i =0 ; i <PTE_PER_TABLE_SIZE ; i++ ) + { +struct amd_iommu_pte *pde = &table[i]; + +set_iommu_pde_present (pde,mfn_x (page_to_mfn (pg)), level -1 , + true, true); + } + +unmap_domain_page (table); + table =__map_domain_page (pg); + level--; + } +unmap_domain_page (table); + + out: +spin_unlock (&hd->arch.mapping_lock); + +amd_iommu_flush_all_pages (d); + +/* Pages leaked in failure case */ +return level ? -ENOMEM :0 ; +} +/* * Local variables: * mode: C
xen/drivers/passthrough/amd/pci_amd_iommu.c +1 -8
@@ -95,10 +95,6 @@ static void amd_iommu_setup_domain_device(u8 bus = pdev->bus;const struct domain_iommu *hd =dom_iommu (domain); -/* dom_io is used as a sentinel for quarantined devices */ -if ( domain == dom_io ) -return ; -BUG_ON ( !hd->arch.root_table || !hd->arch.paging_mode || !iommu->dev_table.buffer );
@@ -290,10 +286,6 @@ static void amd_iommu_disable_domain_device(const struct domain *domain,int req_id;u8 bus = pdev->bus; -/* dom_io is used as a sentinel for quarantined devices */ -if ( domain == dom_io ) -return ; -BUG_ON ( iommu->dev_table.buffer ==NULL ); req_id =get_dma_requestor_id (iommu->seg,PCI_BDF2 (bus, devfn)); table = iommu->dev_table.buffer;
@@ -632,6 +624,7 @@ static void amd_dump_p2m_table(struct domain *d) static const struct iommu_ops __initconstrel _iommu_ops = { .init = amd_iommu_domain_init, .hwdom_init = amd_iommu_hwdom_init, + .quarantine_init = amd_iommu_quarantine_init, .add_device = amd_iommu_add_device, .remove_device = amd_iommu_remove_device, .assign_device = amd_iommu_assign_device,
xen/drivers/passthrough/iommu.c +23 -2
@@ -440,6 +440,28 @@ int iommu_iotlb_flush_all(struct domain *d, unsigned int flush_flags)return rc; } +static int __init iommu_quarantine_init (void ) +{ +const struct domain_iommu *hd =dom_iommu (dom_io); +int rc; + + dom_io->options |= XEN_DOMCTL_CDF_iommu; + + rc =iommu_domain_init (dom_io,0 ); +if ( rc ) +return rc; + +if ( !hd->platform_ops->quarantine_init ) +return 0 ; + + rc = hd->platform_ops->quarantine_init (dom_io); + +if ( !rc ) +printk ("Quarantine initialized\n" ); + +return rc; +} +int __init iommu_setup (void ) {int rc = -ENODEV ;
@@ -473,8 +495,7 @@ int __init iommu_setup(void) }else { - dom_io->options |= XEN_DOMCTL_CDF_iommu; -if (iommu_domain_init (dom_io,0 ) ) +if (iommu_quarantine_init () )panic ("Could not set up quarantine\n" ); printk(" - Dom0 mode: %s\n",
xen/drivers/passthrough/vtd/iommu.c +59 -12
98 lines not on this page · raw message
xen/include/asm-x86/hvm/svm/amd-iommu-proto.h +2 -0
@@ -54,6 +54,8 @@ int amd_iommu_init_late(void);int amd_iommu_update_ivrs_mapping_acpi (void );int iov_adjust_irq_affinities (void ); +int amd_iommu_quarantine_init (struct domain *d); +/* mapping functions */ int __must_check amd_iommu_map_page(struct domain *d, dfn_t dfn, mfn_t mfn, unsigned int flags,
xen/include/xen/iommu.h +1 -0
@@ -211,6 +211,7 @@ typedef int iommu_grdm_t(xen_pfn_t start, xen_ulong_t nr, u32 id, void *ctxt);struct iommu_ops {int (*init)(struct domain *d);void (*hwdom_init)(struct domain *d); +int (*quarantine_init)(struct domain *d);int (*add_device)(u8 devfn,device_t *dev);int (*enable_device)(device_t *dev);int (*remove_device)(u8 devfn,device_t *dev);
Signature · 7 lines
2.20.1 _______________________________________________ Xen-devel mailing list [email protected] https://lists.xenproject.org/mailman/listinfo/xen-devel
Quoted 4 lines · Paul Durrant
This patch introduces a new
iommu_opto facilitate a per-implementation quarantine set up, and then further code for x86 implementations (amd and vtd) to set up a read/wrote scratch page to serve as the source/ target for all DMA whilst a device is assigned todom_io.
A single page in the system won't do, I'm afraid. If one guest's (prior) device is retrying reads with data containing secrets of that guest, another guest's (prior) device could end up writing this data to e.g. storage where after a guest restart it is then available to the wrong guest.
Also nit: s/wrote/write/ .
Quoted 3 lines
The reason for doing this is that some hardware may continue to re-try DMA, despite FLR, in the event of an error. Having a scratch page mapped will allow pending DMA to drain and thus quiesce such buggy hardware.
Without a "sink" page mapped, this would result in IOMMU faults aiui. What's the problem with having these faults surface and get handled, eventually leading to the device getting bus-mastering disabled? Is it that devices continue DMAing even when bus-mastering is off? If so, is it even safe to pass through any such device? In any event the description needs to be extended here.
Quoted 1 line
Signed-off-by Paul Durrant <[email protected]>
What about Arm? Can devices which Arm allows to assign to guests also "babble" like this after de-assignment? If not, this should be said in the description. If so, obviously that side would also want fixing.
xen/drivers/passthrough/amd/iommu_map.c int amd_iommu_reserve_domain_unity_map(struct domain *domain, · line 563
Quoted 7 lines
xen/drivers/passthrough/amd/iommu_map.c +1
-0@@ -560,6 +560,63 @@ int amd_iommu_reserve_domain_unity_map(struct domain *domain,return rt; } +int amd_iommu_quarantine_init (struct domain *d)
__init
Quoted 4 lines
xen/drivers/passthrough/amd/iommu_map.c +4
-0+{ +struct domain_iommu *hd =dom_iommu (d); +unsigned int level; +struct amd_iommu_pte *table;
xen/drivers/passthrough/amd/iommu_map.c int amd_iommu_reserve_domain_unity_map(struct domain *domain, · line 577
Quoted 10 lines
xen/drivers/passthrough/amd/iommu_map.c +10
-0+ +if ( hd->arch.root_table ) + { +ASSERT_UNREACHABLE (); +return 0 ; + } + +spin_lock (&hd->arch.mapping_lock); + + level = hd->arch.paging_mode;
With DomIO being PV in principle, this is going to be the fixed value PV domains get set, merely depending on RAM size at boot time (i.e. not accounting for memory hotplug). This could be easily too little for HVM guests, which are free to extend their GFN (and hence DFN) space. Therefore I think you need to set the maximum possible level here.
Quoted 14 lines
xen/drivers/passthrough/amd/iommu_map.c +14
-0+ hd->arch.root_table =alloc_amd_iommu_pgtable (); +if ( !hd->arch.root_table ) + goto out; + + table =__map_domain_page (hd->arch.root_table); +while ( level ) + { +struct page_info *pg; +unsigned int i; + +/* +* The pgtable allocator is fine for the leaf page, as well as +* page table pages. +*/
xen/drivers/passthrough/amd/iommu_map.c int amd_iommu_reserve_domain_unity_map(struct domain *domain, · line 602
Quoted 10 lines
xen/drivers/passthrough/amd/iommu_map.c +10
-0+ pg =alloc_amd_iommu_pgtable (); +if ( !pg ) +break ; + +for ( i =0 ; i <PTE_PER_TABLE_SIZE ; i++ ) + { +struct amd_iommu_pte *pde = &table[i]; + +set_iommu_pde_present (pde,mfn_x (page_to_mfn (pg)), level -1 , + true, true);
This would also benefit from a comment indicating that it's fine for the leaf level as well, despite the "pde" in the name (and its sibling set_iommu_pte_present() actually existing). Of course you could as well extend the comment a few lines up.
What you do need to do though is pre-fill the leaf page ...
xen/drivers/passthrough/amd/iommu_map.c int amd_iommu_reserve_domain_unity_map(struct domain *domain, · line 608
Quoted 6 lines
xen/drivers/passthrough/amd/iommu_map.c +6
-0+ } + +unmap_domain_page (table); + table =__map_domain_page (pg); + level--; + }
... here, such that possible left over secrets can't end up getting written to e.g. persistent storage or over a network.
xen/drivers/passthrough/amd/iommu_map.c static void vtd_dump_p2m_table(struct domain *d) · line 2674
Quoted 5 lines
xen/drivers/passthrough/amd/iommu_map.c +1
-0@@ -2683,9 +2671,68 @@ static void vtd_dump_p2m_table(struct domain *d)vtd_dump_p2m_table_level (hd->arch.pgd_maddr,agaw_to_level (hd->arch.agaw),0 ,0 ); } +static int intel_iommu_quarantine_init (struct domain *d)
__init again.
xen/drivers/passthrough/amd/iommu_map.c static void vtd_dump_p2m_table(struct domain *d) · line 2678
Quoted 4 lines
xen/drivers/passthrough/amd/iommu_map.c +4
-0+{ +struct domain_iommu *hd =dom_iommu (d); +struct dma_pte *parent; +unsigned int level =agaw_to_level (hd->arch.agaw);
Other than for AMD this is not a problem here, as all domains (currently) get the same AGAW. I wonder though whether precautions would be possible here against the "normal" domain setting getting adjusted without recalling the need to come back here.
Quoted 5 lines
xen/drivers/passthrough/amd/iommu_map.c +5
-0+int rc; + +if ( hd->arch.pgd_maddr ) + { +ASSERT_UNREACHABLE ();
xen/drivers/passthrough/amd/iommu_map.c static void vtd_dump_p2m_table(struct domain *d) · line 2693
Quoted 10 lines
xen/drivers/passthrough/amd/iommu_map.c +10
-0+return 0 ; + } + +spin_lock (&hd->arch.mapping_lock); + + hd->arch.pgd_maddr =alloc_pgtable_maddr (1 , hd->node); +if ( !hd->arch.pgd_maddr ) + goto out; + + parent = (struct dma_pte *)map_vtd_domain_page (hd->arch.pgd_maddr);
Unnecessary cast; funnily enough you don't have one further down.
Quoted 17 lines
xen/drivers/passthrough/amd/iommu_map.c +17
-0+while ( level ) + { +uint64_t maddr; +unsigned int offset; + +/* +* The pgtable allocator is fine for the leaf page, as well as +* page table pages. +*/ + maddr =alloc_pgtable_maddr (1 , hd->node); +if ( !maddr ) +break ; + +for ( offset =0 ; offset <PTE_NUM ; offset++ ) + { +struct dma_pte *pte = &parent[offset]; +
xen/drivers/passthrough/amd/iommu_map.c static void vtd_dump_p2m_table(struct domain *d) · line 2720
Quoted 10 lines
xen/drivers/passthrough/amd/iommu_map.c +10
-0+dma_set_pte_addr (*pte, maddr); +dma_set_pte_readable (*pte); +dma_set_pte_writable (*pte); + } +iommu_flush_cache_page (parent,1 ); + +unmap_vtd_domain_page (parent); + parent =map_vtd_domain_page (maddr); + level--; + }
The leaf page wants scrubbing here as well.
Jan
_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel
Quoted 11 lines
From: Paul Durrant [mailto:[email protected]]
Sent: Wednesday, November 20, 2019 8:09 PMThis patch introduces a new
iommu_opto facilitate a per-implementation quarantine set up, and then further code for x86 implementations (amd and vtd) to set up a read/wrote scratch page to serve as the source/ target for all DMA whilst a device is assigned todom_io.The reason for doing this is that some hardware may continue to re-try DMA, despite FLR, in the event of an error. Having a scratch page mapped will allow pending DMA to drain and thus quiesce such buggy hardware.
then there is no diagnostics at all since all faults are quiescent now... why do we want to support such buggy hardware? Is it better to make it an default-off option since buggy is supposed to niche case?
Quoted 307 lines
Signed-off-by Paul Durrant <[email protected]>---
Cc: Jan Beulich <[email protected]>
Cc: Andrew Cooper <[email protected]>
Cc: Kevin Tian <[email protected]>
Cc: Wei Liu <[email protected]>
Cc: "Roger Pau Monné" <[email protected]>--- xen/drivers/passthrough/amd/iommu_map.c | 57 +++++++++++++++ xen/drivers/passthrough/amd/pci_amd_iommu.c | 9 +-- xen/drivers/passthrough/iommu.c | 25 ++++++- xen/drivers/passthrough/vtd/iommu.c | 71 +++++++++++++++---- xen/include/asm-x86/hvm/svm/amd-iommu-proto.h | 2 + xen/include/xen/iommu.h | 1 + 6 files changed, 143 insertions(+), 22 deletions(-)xen/drivers/passthrough/amd/iommu_map.c +57
-0@@ -560,6 +560,63 @@ int amd_iommu_reserve_domain_unity_map(struct domain *domain, return rt; } +int amd_iommu_quarantine_init (struct domain *d) +{ +struct domain_iommu *hd =dom_iommu (d); +unsigned int level; +struct amd_iommu_pte *table; + +if ( hd->arch.root_table ) + { +ASSERT_UNREACHABLE (); +return 0 ; + } + +spin_lock (&hd->arch.mapping_lock); + + level = hd->arch.paging_mode; + + hd->arch.root_table =alloc_amd_iommu_pgtable (); +if ( !hd->arch.root_table ) + goto out; + + table =__map_domain_page (hd->arch.root_table); +while ( level ) + { +struct page_info *pg; +unsigned int i; + +/* +* The pgtable allocator is fine for the leaf page, as well as +* page table pages. +*/ + pg =alloc_amd_iommu_pgtable (); +if ( !pg ) +break ; + +for ( i =0 ; i <PTE_PER_TABLE_SIZE ; i++ ) + { +struct amd_iommu_pte *pde = &table[i]; + +set_iommu_pde_present (pde,mfn_x (page_to_mfn (pg)), level -1 , + true, true); + } + +unmap_domain_page (table); + table =__map_domain_page (pg); + level--; + } +unmap_domain_page (table); + + out: +spin_unlock (&hd->arch.mapping_lock); + +amd_iommu_flush_all_pages (d); + +/* Pages leaked in failure case */ +return level ? -ENOMEM :0 ; +} +/* * Local variables: * mode: C
xen/drivers/passthrough/amd/pci_amd_iommu.c +0
-8@@ -95,10 +95,6 @@ static void amd_iommu_setup_domain_device(u8 bus = pdev->bus;const struct domain_iommu *hd =dom_iommu (domain); -/* dom_io is used as a sentinel for quarantined devices */ -if ( domain == dom_io ) -return ; -BUG_ON ( !hd->arch.root_table || !hd->arch.paging_mode || !iommu->dev_table.buffer );@@ -290,10 +286,6 @@ static void amd_iommu_disable_domain_device(const struct domain *domain, int req_id;u8 bus = pdev->bus; -/* dom_io is used as a sentinel for quarantined devices */ -if ( domain == dom_io ) -return ; -BUG_ON ( iommu->dev_table.buffer ==NULL ); req_id =get_dma_requestor_id (iommu->seg,PCI_BDF2 (bus, devfn));table = iommu->dev_table.buffer;xen/drivers/passthrough/amd/pci_amd_iommu.c +1
-0@@ -632,6 +624,7 @@ static void amd_dump_p2m_table(struct domain *d) static const struct iommu_ops __initconstrel _iommu_ops = { .init = amd_iommu_domain_init, .hwdom_init = amd_iommu_hwdom_init, + .quarantine_init = amd_iommu_quarantine_init, .add_device = amd_iommu_add_device, .remove_device = amd_iommu_remove_device,.assign_device =
amd_iommu_assign_device,xen/drivers/passthrough/iommu.c +22
-0@@ -440,6 +440,28 @@ int iommu_iotlb_flush_all(struct domain *d, unsigned int flush_flags)return rc; } +static int __init iommu_quarantine_init (void ) +{ +const struct domain_iommu *hd =dom_iommu (dom_io); +int rc; + + dom_io->options |= XEN_DOMCTL_CDF_iommu; + + rc =iommu_domain_init (dom_io,0 ); +if ( rc ) +return rc; + +if ( !hd->platform_ops->quarantine_init ) +return 0 ; + + rc = hd->platform_ops->quarantine_init (dom_io); + +if ( !rc ) +printk ("Quarantine initialized\n" ); + +return rc; +} +int __init iommu_setup (void ) {int rc = -ENODEV;xen/drivers/passthrough/iommu.c +1
-2@@ -473,8 +495,7 @@ int __init iommu_setup(void) }else { - dom_io->options |= XEN_DOMCTL_CDF_iommu; -if (iommu_domain_init (dom_io,0 ) ) +if (iommu_quarantine_init () )panic ("Could not set up quarantine\n" ); printk(" - Dom0 mode: %s\n",xen/drivers/passthrough/vtd/iommu.c +0
-12@@ -1291,10 +1291,6 @@ int domain_context_mapping_one(int agaw, rc, ret;bool_t flush_dev_iotlb; -/* dom_io is used as a sentinel for quarantined devices */ -if ( domain == dom_io ) -return 0 ; -ASSERT (pcidevs_locked ());spin_lock (&iommu->lock); maddr =bus_to_context_maddr (iommu, bus);@@ -1541,10 +1537,6 @@ int domain_context_unmap_one(int iommu_domid, rc, ret;bool_t flush_dev_iotlb; -/* dom_io is used as a sentinel for quarantined devices */ -if ( domain == dom_io ) -return 0 ; -ASSERT (pcidevs_locked ());spin_lock (&iommu->lock);@@ -1677,10 +1669,6 @@ static int domain_context_unmap(struct domain *domain, u8 devfn, goto out; } -/* dom_io is used as a sentinel for quarantined devices */ -if ( domain == dom_io ) - goto out; -/* * if no other devices under the same iommu owned by this domain, * clear iommu in
iommu_bitmapand cleardomain_idindomid_bitmpxen/drivers/passthrough/vtd/iommu.c +59
-0@@ -2683,9 +2671,68 @@ static void vtd_dump_p2m_table(struct domain *d)vtd_dump_p2m_table_level (hd->arch.pgd_maddr,agaw_to_level (hd- >arch.agaw), 0, 0); } +static int intel_iommu_quarantine_init (struct domain *d) +{ +struct domain_iommu *hd =dom_iommu (d); +struct dma_pte *parent; +unsigned int level =agaw_to_level (hd->arch.agaw); +int rc; + +if ( hd->arch.pgd_maddr ) + { +ASSERT_UNREACHABLE (); +return 0 ; + } + +spin_lock (&hd->arch.mapping_lock); + + hd->arch.pgd_maddr =alloc_pgtable_maddr (1 , hd->node); +if ( !hd->arch.pgd_maddr ) + goto out; + + parent = (struct dma_pte *)map_vtd_domain_page (hd- >arch.pgd_maddr); +while ( level ) + { +uint64_t maddr; +unsigned int offset; + +/* +* The pgtable allocator is fine for the leaf page, as well as +* page table pages. +*/ + maddr =alloc_pgtable_maddr (1 , hd->node); +if ( !maddr ) +break ; + +for ( offset =0 ; offset <PTE_NUM ; offset++ ) + { +struct dma_pte *pte = &parent[offset]; + +dma_set_pte_addr (*pte, maddr); +dma_set_pte_readable (*pte); +dma_set_pte_writable (*pte); + } +iommu_flush_cache_page (parent,1 ); + +unmap_vtd_domain_page (parent); + parent =map_vtd_domain_page (maddr); + level--; + } +unmap_vtd_domain_page (parent); + + out: +spin_unlock (&hd->arch.mapping_lock); + + rc =iommu_flush_iotlb_all (d); + +/* Pages leaked in failure case */ +return level ? -ENOMEM : rc; +} + const struct iommu_ops __initconstrel intel_iommu_ops = { .init = intel_iommu_domain_init, .hwdom_init = intel_iommu_hwdom_init, + .quarantine_init = intel_iommu_quarantine_init,.add_device =
intel_iommu_add_device,
.enable_device =intel_iommu_enable_device,
.remove_device =intel_iommu_remove_device,xen/include/asm-x86/hvm/svm/amd-iommu-proto.h +2
-0@@ -54,6 +54,8 @@ int amd_iommu_init_late(void);int amd_iommu_update_ivrs_mapping_acpi (void );int iov_adjust_irq_affinities (void ); +int amd_iommu_quarantine_init (struct domain *d); +/* mapping functions */ int __must_check amd_iommu_map_page(struct domain *d, dfn_t dfn, mfn_t mfn, unsigned int flags,xen/include/xen/iommu.h +1
-0@@ -211,6 +211,7 @@ typedef int iommu_grdm_t(xen_pfn_t start, xen_ulong_t nr, u32 id, void *ctxt);struct iommu_ops {int (*init)(struct domain *d);void (*hwdom_init)(struct domain *d); +int (*quarantine_init)(struct domain *d);int (*add_device)(u8 devfn,device_t *dev);int (*enable_device)(device_t *dev);int (*remove_device)(u8 devfn, device_t *dev);
Quoted 2 lines
--
2.20.1
_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel
Quoted 19 lines
-----Original Message-----
From: Tian, Kevin <[email protected]>
Sent: 25 November 2019 08:22
To: Durrant, Paul <[email protected]>; [email protected]
Cc: Jan Beulich <[email protected]>; Andrew Cooper <[email protected]>; Wei Liu <[email protected]>; Roger Pau Monné <[email protected]>
Subject: RE: [PATCH] x86 / iommu: set up a scratch page in the quarantine domainQuoted 6 lines
From: Paul Durrant [mailto:[email protected]]
Sent: Wednesday, November 20, 2019 8:09 PMThis patch introduces a new
iommu_opto facilitate a per-implementation quarantine set up, and then further code for x86 implementations (amd and vtd) to set up a read/wrote scratch page to serve as thesource/
Quoted 2 lines
target for all DMA whilst a device is assigned to
dom_io.
Quoted 7 lines
Quoted 3 lines
The reason for doing this is that some hardware may continue to re-try DMA, despite FLR, in the event of an error. Having a scratch page mapped will allow pending DMA to drain and thus quiesce such buggy hardware.
then there is no diagnostics at all since all faults are quiescent now... why do we want to support such buggy hardware? Is it better to make it an default-off option since buggy is supposed to niche case?
I guess it could be a command line option... perhaps making the new 'iommu=quarantine' boolean into something more complex, but I'm not sure it's really worth it. Perhaps a compile time option would be better?
Paul
_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel
Quoted 23 lines · Paul
Quoted 23 lines
-----Original Message-----
From: Tian, Kevin <[email protected]>
Sent: 25 November 2019 08:22
To: Durrant, Paul <[email protected]>; [email protected]
Cc: Jan Beulich <[email protected]>; Andrew Cooper <[email protected]>; Wei Liu <[email protected]>; Roger Pau Monné <[email protected]>
Subject: RE: [PATCH] x86 / iommu: set up a scratch page in the quarantine domainQuoted 6 lines
From: Paul Durrant [mailto:[email protected]]
Sent: Wednesday, November 20, 2019 8:09 PMThis patch introduces a new
iommu_opto facilitate a per-implementation quarantine set up, and then further code for x86 implementations (amd and vtd) to set up a read/wrote scratch page to serve as thesource/
Quoted 5 lines
target for all DMA whilst a device is assigned to
dom_io.The reason for doing this is that some hardware may continue to re-try DMA, despite FLR, in the event of an error. Having a scratch page mapped will allow pending DMA to drain and thus quiesce such buggy hardware.
Quoted 8 lines
Quoted 3 lines
then there is no diagnostics at all since all faults are quiescent now... why do we want to support such buggy hardware? Is it better to make it an default-off option since buggy is supposed to niche case?
I guess it could be a command line option... perhaps making the new 'iommu=quarantine' boolean into something more complex, but I'm not sure it's really worth it. Perhaps a compile time option would be better?
Yet another option: How about installing the scratch page mappings only after a (handful of) IOMMU faults? But of course there was the related earlier question of whether indeed our turning off of bus mastering doesn't already help silencing the faults.
Jan
_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel
Quoted 15 lines
-----Original Message-----
From: Xen-devel <[email protected]> On Behalf Of Jan Beulich
Sent: 20 November 2019 13:52
To: Durrant, Paul <[email protected]>
Cc: [email protected]; Kevin Tian <[email protected]>; Roger Pau Monné <[email protected]>; Wei Liu <[email protected]>; Andrew Cooper <[email protected]>
Subject: Re: [Xen-devel] [PATCH] x86 / iommu: set up a scratch page in the quarantine domainQuoted 3 lines · Paul Durrant
This patch introduces a new
iommu_opto facilitate a per-implementation quarantine set up, and then further code for x86 implementations (amd and vtd) to set up a read/wrote scratch page to serve as the
Quoted 9 lines
source/
Quoted 1 line
target for all DMA whilst a device is assigned to
dom_io.A single page in the system won't do, I'm afraid. If one guest's (prior) device is retrying reads with data containing secrets of that guest, another guest's (prior) device could end up writing this data to e.g. storage where after a guest restart it is then available to the wrong guest.
True. I was unsure whether this was a concern in the scenarios we had to deal with but I'm informed it is, and in the general case it is too.
Quoted 2 lines
Also nit: s/wrote/write/ .
Yep. Will fix.
Quoted 11 lines
Quoted 3 lines
The reason for doing this is that some hardware may continue to re-try DMA, despite FLR, in the event of an error. Having a scratch page mapped will allow pending DMA to drain and thus quiesce such buggy hardware.
Without a "sink" page mapped, this would result in IOMMU faults aiui. What's the problem with having these faults surface and get handled, eventually leading to the device getting bus-mastering disabled? Is it that devices continue DMAing even when bus-mastering is off? If so, is it even safe to pass through any such device? In any event the description needs to be extended here.
The devices in question ignore both FLR and BME and some IOMMU faults are fatal. I believe, however, write faults are not and so I think a single read-only 'source' page will be sufficient.
Quoted 11 lines
Quoted 1 line
Signed-off-by Paul Durrant <[email protected]>What about Arm? Can devices which Arm allows to assign to guests also "babble" like this after de-assignment? If not, this should be said in the description. If so, obviously that side would also want fixing.
xen/drivers/passthrough/amd/iommu_map.c int amd_iommu_reserve_domain_unity_map(struct
Quoted 3 lines
xen/drivers/passthrough/amd/iommu_map.c +0
-0@@ -560,6 +560,63 @@ int amd_iommu_reserve_domain_unity_map(structdomain *domain,
Quoted 7 lines
xen/drivers/passthrough/amd/iommu_map.c int amd_iommu_reserve_domain_unity_map(struct domain *domain, · line 563
Quoted 4 lines
xen/drivers/passthrough/amd/iommu_map.c +1
-0return rt; } +int amd_iommu_quarantine_init (struct domain *d)
__init
Ok.
Quoted 13 lines
Quoted 13 lines
xen/drivers/passthrough/amd/iommu_map.c +13
-0+{ +struct domain_iommu *hd =dom_iommu (d); +unsigned int level; +struct amd_iommu_pte *table; + +if ( hd->arch.root_table ) + { +ASSERT_UNREACHABLE (); +return 0 ; + } + +spin_lock (&hd->arch.mapping_lock); +
Quoted 9 lines
xen/drivers/passthrough/amd/iommu_map.c int amd_iommu_reserve_domain_unity_map(struct domain *domain, · line 577
Quoted 1 line
xen/drivers/passthrough/amd/iommu_map.c +1
-0+ level = hd->arch.paging_mode;With DomIO being PV in principle, this is going to be the fixed value PV domains get set, merely depending on RAM size at boot time (i.e. not accounting for memory hotplug). This could be easily too little for HVM guests, which are free to extend their GFN (and hence DFN) space. Therefore I think you need to set the maximum possible level here.
Ok. I'd not considered memory hotplug. I'll use a static maximum value instead, as VT-d does in general.
Quoted 24 lines
Quoted 13 lines
xen/drivers/passthrough/amd/iommu_map.c +13
-0+ hd->arch.root_table =alloc_amd_iommu_pgtable (); +if ( !hd->arch.root_table ) + goto out; + + table =__map_domain_page (hd->arch.root_table); +while ( level ) + { +struct page_info *pg; +unsigned int i; + +/* +* The pgtable allocator is fine for the leaf page, as well as +* page table pages. xen/drivers/passthrough/amd/iommu_map.c int amd_iommu_reserve_domain_unity_map(struct domain *domain,
Quoted 10 lines
xen/drivers/passthrough/amd/iommu_map.c +10
-0+*/ + pg =alloc_amd_iommu_pgtable (); +if ( !pg ) +break ; + +for ( i =0 ; i <PTE_PER_TABLE_SIZE ; i++ ) + { +struct amd_iommu_pte *pde = &table[i]; + + set_iommu_pde_present(pde, mfn_x(page_to_mfn(pg)), level -1,
Quoted 6 lines
xen/drivers/passthrough/amd/iommu_map.c int amd_iommu_reserve_domain_unity_map(struct domain *domain, · line 602
Quoted 1 line
xen/drivers/passthrough/amd/iommu_map.c +1
-0+ true, true);This would also benefit from a comment indicating that it's fine for the leaf level as well, despite the "pde" in the name (and its sibling
set_iommu_pte_present()actually existing). Of course you could as well extend the comment a few lines up.
The AMD IOMMU conflates PDE and PTE all over the place but I'll add a comment here to that effect.
Quoted 10 lines
What you do need to do though is pre-fill the leaf page ...
xen/drivers/passthrough/amd/iommu_map.c int amd_iommu_reserve_domain_unity_map(struct domain *domain, · line 608
Quoted 6 lines
xen/drivers/passthrough/amd/iommu_map.c +6
-0+ } + +unmap_domain_page (table); + table =__map_domain_page (pg); + level--; + }
Quoted 3 lines
... here, such that possible left over secrets can't end up getting written to e.g. persistent storage or over a network.
That's actually one reason for using the pgtable allocator... It already does that.
Quoted 9 lines
xen/drivers/passthrough/amd/iommu_map.c static void vtd_dump_p2m_table(struct domain *d) · line 2675
Quoted 6 lines
xen/drivers/passthrough/amd/iommu_map.c +1
-0@@ -2683,9 +2671,68 @@ static void vtd_dump_p2m_table(struct domain *d)vtd_dump_p2m_table_level (hd->arch.pgd_maddr,agaw_to_level (hd- arch.agaw),0 ,0 ); } +static int intel_iommu_quarantine_init (struct domain *d)
__initagain.
Ok.
Quoted 10 lines
xen/drivers/passthrough/amd/iommu_map.c static void vtd_dump_p2m_table(struct domain *d) · line 2679
Quoted 4 lines
xen/drivers/passthrough/amd/iommu_map.c +4
-0+{ +struct domain_iommu *hd =dom_iommu (d); +struct dma_pte *parent; +unsigned int level =agaw_to_level (hd->arch.agaw);Other than for AMD this is not a problem here, as all domains (currently) get the same AGAW. I wonder though whether precautions would be possible here against the "normal" domain setting getting adjusted without recalling the need to come back here.
I could just go for a hardcoded width value here too.
Quoted 9 lines
Quoted 9 lines
xen/drivers/passthrough/amd/iommu_map.c +9
-0+int rc; + +if ( hd->arch.pgd_maddr ) + { +ASSERT_UNREACHABLE (); +return 0 ; + } + +spin_lock (&hd->arch.mapping_lock);
Quoted 9 lines
xen/drivers/passthrough/amd/iommu_map.c static void vtd_dump_p2m_table(struct domain *d) · line 2694
Quoted 6 lines
xen/drivers/passthrough/amd/iommu_map.c +6
-0+ + hd->arch.pgd_maddr =alloc_pgtable_maddr (1 , hd->node); +if ( !hd->arch.pgd_maddr ) + goto out; + + parent = (struct dma_pte *)map_vtd_domain_page (hd->arch.pgd_maddr);Unnecessary cast; funnily enough you don't have one further down.
Sorry. Cut'n'paste.
Quoted 21 lines
Quoted 21 lines
xen/drivers/passthrough/amd/iommu_map.c +21
-0+while ( level ) + { +uint64_t maddr; +unsigned int offset; + +/* +* The pgtable allocator is fine for the leaf page, as well as +* page table pages. +*/ + maddr =alloc_pgtable_maddr (1 , hd->node); +if ( !maddr ) +break ; + +for ( offset =0 ; offset <PTE_NUM ; offset++ ) + { +struct dma_pte *pte = &parent[offset]; + +dma_set_pte_addr (*pte, maddr); +dma_set_pte_readable (*pte); +dma_set_pte_writable (*pte); + }
Quoted 9 lines
xen/drivers/passthrough/amd/iommu_map.c static void vtd_dump_p2m_table(struct domain *d) · line 2721
Quoted 6 lines
xen/drivers/passthrough/amd/iommu_map.c +6
-0+iommu_flush_cache_page (parent,1 ); + +unmap_vtd_domain_page (parent); + parent =map_vtd_domain_page (maddr); + level--; + }The leaf page wants scrubbing here as well.
Already done by alloc_pgtable_maddr(). I'll note in the comment in this hunk and the AMD one that the pages are zeroed.
Paul
Quoted 6 lines
Jan
_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel
_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel
Quoted 47 lines
-----Original Message-----
From: Jan Beulich <[email protected]>
Sent: 27 November 2019 15:26
To: Durrant, Paul <[email protected]>
Cc: Kevin Tian <[email protected]>; [email protected]; Andrew Cooper <[email protected]>; Roger Pau Monné <[email protected]>; Wei Liu <[email protected]>
Subject: Re: [Xen-devel] [PATCH] x86 / iommu: set up a scratch page in the quarantine domainQuoted 8 lines · Paul
Quoted 8 lines
-----Original Message-----
From: Tian, Kevin <[email protected]>
Sent: 25 November 2019 08:22
To: Durrant, Paul <[email protected]>; [email protected]
Cc: Jan Beulich <[email protected]>; Andrew Cooper <[email protected]>; Wei Liu <[email protected]>; Roger Pau Monné <[email protected]>
Subject: RE: [PATCH] x86 / iommu: set up a scratch page in thequarantine
Quoted 6 lines
Quoted 6 lines
domain
Quoted 4 lines
From: Paul Durrant [mailto:[email protected]]
Sent: Wednesday, November 20, 2019 8:09 PMThis patch introduces a new
iommu_opto facilitate a per-implementation
Quoted 7 lines
Quoted 7 lines
Quoted 2 lines
quarantine set up, and then further code for x86 implementations (amd and vtd) to set up a read/wrote scratch page to serve as the
source/
Quoted 4 lines
target for all DMA whilst a device is assigned to
dom_io.The reason for doing this is that some hardware may continue to re-try DMA, despite FLR, in the event of an error. Having a scratch page
mapped
Quoted 3 lines
Quoted 3 lines
Quoted 1 line
will allow pending DMA to drain and thus quiesce such buggy hardware.
then there is no diagnostics at all since all faults are quiescent
now...
Quoted 7 lines
Quoted 2 lines
why do we want to support such buggy hardware? Is it better to make it an default-off option since buggy is supposed to niche case?
I guess it could be a command line option... perhaps making the new 'iommu=quarantine' boolean into something more complex, but I'm not sure it's really worth it. Perhaps a compile time option would be better?
Quoted 4 lines
Yet another option: How about installing the scratch page mappings only after a (handful of) IOMMU faults? But of course there was the related earlier question of whether indeed our turning off of bus mastering doesn't already help silencing the faults.
No. Unfortunately the h/w has zero tolerance for some faults.
Paul
Quoted 2 lines
Jan
_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel