Passthrough support for AMD-IOMMU (AMD-Vi)
cover Passthrough support for AMD-IOMMU (AMD-Vi)
Hi,
this series of patches implement iommu=pt support for AMD IOMMU (AMD-Vi). If this option is passed on the kernel command line, the AMD-Vi driver will not register itself as the dma_ops backend and allows all devices unlimited access to main memory as long as no other kernel part (currently only KVM will do so) assigns the device to another domain using the IOMMU-API.
Joerg
Diffstat:
arch/x86/include/asm/amd_iommu.h | 1 + arch/x86/include/asm/amd_iommu_types.h | 4 + arch/x86/kernel/amd_iommu.c | 140 +++++++++++++++++++++++++++---- arch/x86/kernel/amd_iommu_init.c | 8 ++- arch/x86/kernel/pci-dma.c | 9 ++- 5 files changed, 142 insertions(+), 20 deletions(-)
Shortlog:
Joerg Roedel (8):
x86/dma: Mark iommu_pass_through as __read_mostly x86/amd-iommu: Add core functions for pd allocation/freeing x86/amd-iommu: Add passthrough mode initialization functions x86/amd-iommu: Fix device table write order x86/amd-iommu: Align locking between attach_device and detach_device x86/amd-iommu: Make sure a device is assigned in passthrough mode x86/amd-iommu: Don't detach device from pt domain on driver unbind x86/amd-iommu: Initialize passthrough mode when requested
PATCH 1/8 x86/dma: Mark iommu_pass_through as __read_mostly
This variable is read most of the time. This patch marks it as such. It also documents the meaning the this variable while at it.
--- arch/x86/kernel/pci-dma.c | 9 ++++++++- 1 files changed, 8 insertions(+), 1 deletions(-)
arch/x86/kernel/pci-dma.c +8 -1
@@ -32,7 +32,14 @@ int no_iommu __read_mostly;/* Set this to 1 if there is a HW IOMMU in the system */ int iommu_detected__read_mostly =0 ; -int iommu_pass_through; +/* +* This variable becomes 1 if iommu=pt is passed on the kernel command line. +* If this variable is 1, IOMMU implementations do no DMA ranslation for +* devices and allow every device to access to whole physical memory. This is +* useful if a user want to use an IOMMU only for KVM device assignment to +* guests and not for driver dma translation. +*/ +int iommu_pass_through__read_mostly ;dma_addr_t bad_dma_address__read_mostly =0 ;EXPORT_SYMBOL (bad_dma_address);
Signature · 1 line
1.6.3.3
PATCH 2/8 x86/amd-iommu: Add core functions for pd allocation/freeing
In reply to Joerg RoedelThis patch factors some code of protection domain allocation into seperate functions. This way the logic can be used to allocate the passthrough domain later. As a side effect this patch fixes an unlikely domain id leakage bug.
--- arch/x86/kernel/amd_iommu.c | 36 ++++++++++++++++++++++++++++++++---- 1 files changed, 32 insertions(+), 4 deletions(-)
arch/x86/kernel/amd_iommu.c +32 -4
@@ -1988,19 +1988,47 @@ static void cleanup_domain(struct protection_domain *domain)write_unlock_irqrestore (&amd_iommu_devtable_lock, flags); } -static int amd_iommu_domain_init (struct iommu_domain *dom) +static void protection_domain_free (struct protection_domain *domain) +{ +if (!domain) +return ; + +if (domain->id) +domain_id_free (domain->id); + +kfree (domain); +} + +static struct protection_domain *protection_domain_alloc (void ) {struct protection_domain *domain; domain =kzalloc (sizeof (*domain),GFP_KERNEL );if (!domain) -return -ENOMEM ; +return NULL ;spin_lock_init (&domain->lock); - domain->mode =PAGE_MODE_3_LEVEL ; domain->id =domain_id_alloc ();if (!domain->id) + goto out_err; + +return domain; + +out_err: +kfree (domain); + +return NULL ; +} + +static int amd_iommu_domain_init (struct iommu_domain *dom) +{ +struct protection_domain *domain; + + domain =protection_domain_alloc (); +if (!domain) goto out_free; + + domain->mode =PAGE_MODE_3_LEVEL ; domain->pt_root = (void *)get_zeroed_page (GFP_KERNEL );if (!domain->pt_root) goto out_free;
@@ -2010,7 +2038,7 @@ static int amd_iommu_domain_init(struct iommu_domain *dom)return 0 ; out_free: -kfree (domain); +protection_domain_free (domain);return -ENOMEM ; }
Signature · 1 line
1.6.3.3
PATCH 3/8 x86/amd-iommu: Add passthrough mode initialization functions
In reply to Joerg RoedelWhen iommu=pt is passed on kernel command line the devices should run untranslated. This requires the allocation of a special domain for that purpose. This patch implements the allocation and initialization path for iommu=pt.
--- arch/x86/include/asm/amd_iommu.h | 1 + arch/x86/include/asm/amd_iommu_types.h | 4 ++ arch/x86/kernel/amd_iommu.c | 72 ++++++++++++++++++++++++++++---- 3 files changed, 69 insertions(+), 8 deletions(-)
arch/x86/include/asm/amd_iommu.h +1 -0
@@ -25,6 +25,7 @@#ifdef CONFIG_AMD_IOMMU extern int amd_iommu_init (void );extern int amd_iommu_init_dma_ops (void ); +extern int amd_iommu_init_passthrough (void );extern void amd_iommu_detect (void );extern irqreturn_t amd_iommu_int_handler (int irq,void *data);extern void amd_iommu_flush_all_domains (void );
arch/x86/include/asm/amd_iommu_types.h +4 -0
@@ -143,6 +143,7 @@#define EVT_BUFFER_SIZE 8192/* 512 entries */ #define EVT_LEN_MASK (0x9ULL << 56) +#define PAGE_MODE_NONE 0x00#define PAGE_MODE_1_LEVEL 0x01#define PAGE_MODE_2_LEVEL 0x02#define PAGE_MODE_3_LEVEL 0x03
@@ -194,6 +195,9 @@#define PD_DMA_OPS_MASK (1UL << 0)/* domain used for dma_ops */ #define PD_DEFAULT_MASK (1UL << 1)/* domain is a default dma_ops domain for an IOMMU */ +#define PD_PASSTHROUGH_MASK (1UL << 2)/* domain has no page +translation */ +extern bool amd_iommu_dump; #define DUMP_printk(format, arg...) \ do { \
arch/x86/kernel/amd_iommu.c +64 -8
94 lines not on this page · raw message
Signature · 1 line
1.6.3.3
PATCH 4/8 x86/amd-iommu: Fix device table write order
In reply to Joerg RoedelThe V bit of the device table entry has to be set after the rest of the entry is written to not confuse the hardware.
--- arch/x86/kernel/amd_iommu.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-)
arch/x86/kernel/amd_iommu.c +2 -2
@@ -1087,9 +1087,9 @@ static void __attach_device(struct amd_iommu *iommu, pte_root |=IOMMU_PTE_IR |IOMMU_PTE_IW |IOMMU_PTE_P |IOMMU_PTE_TV ;write_lock_irqsave (&amd_iommu_devtable_lock, flags); - amd_iommu_dev_table[devid].data[0 ] =lower_32_bits (pte_root); - amd_iommu_dev_table[devid].data[1 ] =upper_32_bits (pte_root); amd_iommu_dev_table[devid].data[2 ] = domain->id; + amd_iommu_dev_table[devid].data[1 ] =upper_32_bits (pte_root); + amd_iommu_dev_table[devid].data[0 ] =lower_32_bits (pte_root); amd_iommu_pd_table[devid] = domain;write_unlock_irqrestore (&amd_iommu_devtable_lock, flags);
Signature · 1 line
1.6.3.3
PATCH 5/8 x86/amd-iommu: Align locking between attach_device and detach_device
In reply to Joerg RoedelThis patch makes the locking behavior between the functions attach_device and __attach_device consistent with the locking behavior between detach_device and __detach_device.
--- arch/x86/kernel/amd_iommu.c | 19 ++++++++++++++----- 1 files changed, 14 insertions(+), 5 deletions(-)
arch/x86/kernel/amd_iommu.c +14 -5
@@ -1077,29 +1077,38 @@ static void __attach_device(struct amd_iommu *iommu, struct protection_domain *domain, u16 devid) { -unsigned long flags; -u64 pte_root =virt_to_phys (domain->pt_root); +u64 pte_root; - domain->dev_cnt +=1 ; +/* lock domain */ +spin_lock (&domain->lock); + + pte_root =virt_to_phys (domain->pt_root); pte_root |= (domain->mode &DEV_ENTRY_MODE_MASK ) <<DEV_ENTRY_MODE_SHIFT ; pte_root |=IOMMU_PTE_IR |IOMMU_PTE_IW |IOMMU_PTE_P |IOMMU_PTE_TV ; -write_lock_irqsave (&amd_iommu_devtable_lock, flags); amd_iommu_dev_table[devid].data[2 ] = domain->id; amd_iommu_dev_table[devid].data[1 ] =upper_32_bits (pte_root); amd_iommu_dev_table[devid].data[0 ] =lower_32_bits (pte_root); amd_iommu_pd_table[devid] = domain; -write_unlock_irqrestore (&amd_iommu_devtable_lock, flags); + + domain->dev_cnt +=1 ; + +/* ready */ +spin_unlock (&domain->lock); }static void attach_device (struct amd_iommu *iommu,struct protection_domain *domain,u16 devid) { +unsigned long flags; + +write_lock_irqsave (&amd_iommu_devtable_lock, flags);__attach_device (iommu, domain, devid); +write_unlock_irqrestore (&amd_iommu_devtable_lock, flags);/* * We might boot into a crash-kernel here. The crashed kernel
Signature · 1 line
1.6.3.3
PATCH 6/8 x86/amd-iommu: Make sure a device is assigned in passthrough mode
In reply to Joerg RoedelWhen the IOMMU driver runs in passthrough mode it has to make sure that every device not assigned to an IOMMU-API domain must be put into the passthrough domain instead of keeping it unassigned.
--- arch/x86/kernel/amd_iommu.c | 9 +++++++++ 1 files changed, 9 insertions(+), 0 deletions(-)
arch/x86/kernel/amd_iommu.c +9 -0
@@ -1141,6 +1141,15 @@ static void __detach_device(struct protection_domain *domain, u16 devid)/* ready */ spin_unlock (&domain->lock); + +/* +* If we run in passthrough mode the device must be assigned to the +* passthrough domain if it is detached from any other domain +*/ +if (iommu_pass_through) { +struct amd_iommu *iommu = amd_iommu_rlookup_table[devid]; +__attach_device (iommu, pt_domain, devid); + } }/*
Signature · 1 line
1.6.3.3
PATCH 7/8 x86/amd-iommu: Don't detach device from pt domain on driver unbind
In reply to Joerg RoedelThis patch makes sure a device is not detached from the passthrough domain when the device driver is unloaded or does otherwise release the device.
--- arch/x86/kernel/amd_iommu.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-)
arch/x86/kernel/amd_iommu.c +2 -0
@@ -1195,6 +1195,8 @@ static int device_change_notifier(struct notifier_block *nb,case BUS_NOTIFY_UNBOUND_DRIVER :if (!domain) goto out; +if (iommu_pass_through) +break ;detach_device (domain, devid);break ;case BUS_NOTIFY_ADD_DEVICE :
Signature · 1 line
1.6.3.3
PATCH 8/8 x86/amd-iommu: Initialize passthrough mode when requested
In reply to Joerg RoedelThis patch enables the passthrough mode for AMD IOMMU by running the initialization function when iommu=pt is passed on the kernel command line.
--- arch/x86/kernel/amd_iommu_init.c | 8 +++++++- 1 files changed, 7 insertions(+), 1 deletions(-)
arch/x86/kernel/amd_iommu_init.c +7 -1
@@ -1242,12 +1242,18 @@ int __init amd_iommu_init(void)if (ret) goto free; - ret =amd_iommu_init_dma_ops (); +if (iommu_pass_through) + ret =amd_iommu_init_passthrough (); +else + ret =amd_iommu_init_dma_ops ();if (ret) goto free;enable_iommus (); +if (iommu_pass_through) + goto out; +printk (KERN_INFO "AMD IOMMU: device isolation " );if (amd_iommu_isolate)printk ("enabled\n" );
Signature · 1 line
1.6.3.3
Quoted 6 lines · Joerg Roedel
this series of patches implement iommu=pt support for AMD IOMMU (AMD-Vi). If this option is passed on the kernel command line, the AMD-Vi driver will not register itself as the
dma_opsbackend and allows all devices unlimited access to main memory as long as no other kernel part (currently only KVM will do so) assigns the device to another domain using the IOMMU-API.
The Intel IOMMU used to behave like that in passthrough mode, but I've now changed it. Now, it will still register itself as the dma_ops backend, and it won't actually bypass the IOMMU for devices with a limited dma_mask. So you don't end up having to use swiotlb for those (which wasn't working anyway, iirc).
The devices for which you actually care about performance are still going to end up with the passthrough mapping as you desire, anyway.
Signature · 2 lines
David Woodhouse Open Source Technology Centre [email protected] Intel Corporation