powerpc/powernv: Add iommu DMA bypass support for IODA2
PATCH 1/1 powerpc/powernv: Add iommu DMA bypass support for IODA2
this patch adds the support for to create a direct iommu "bypass" window on IODA2 bridges (such as Power8) allowing to bypass iommu page translation completely for 64-bit DMA capable devices, thus significantly improving DMA performances.
Additionally, this adds a hook to the struct iommu_table so that the IOMMU API / VFIO can disable the bypass when external ownership is requested, since in that case, the device will be used by an environment such as userspace or a KVM guest which must not be allowed to bypass translations.
---
This is pretty much identical to the old version I posted a while ago, except that it does have the iommu calls to enable/disable which I had forgotten to git add before posting the previous one.
arch/powerpc/include/asm/dma-mapping.h | 1 + arch/powerpc/include/asm/iommu.h | 1 + arch/powerpc/kernel/dma.c | 10 ++-- arch/powerpc/kernel/iommu.c | 12 +++++ arch/powerpc/platforms/powernv/pci-ioda.c | 84 +++++++++++++++++++++++++++++++ arch/powerpc/platforms/powernv/pci.c | 10 ++++ arch/powerpc/platforms/powernv/pci.h | 6 ++- arch/powerpc/platforms/powernv/powernv.h | 4 ++ arch/powerpc/platforms/powernv/setup.c | 9 ++++ 9 files changed, 133 insertions(+), 4 deletions(-)
arch/powerpc/include/asm/dma-mapping.h +1 -0
@@ -134,6 +134,7 @@ static inline int dma_supported(struct device *dev, u64 mask) }extern int dma_set_mask (struct device *dev,u64 dma_mask); +extern int __dma_set_mask (struct device *dev,u64 dma_mask);#define dma_alloc_coherent (d,s,h,f) dma_alloc_attrs(d,s,h,f,NULL)
arch/powerpc/include/asm/iommu.h +1 -0
@@ -77,6 +77,7 @@ struct iommu_table {#ifdef CONFIG_IOMMU_API struct iommu_group *it_group;#endif +void (*set_bypass)(struct iommu_table *tbl,bool enable); };/* Pure 2^n version of get_order */
arch/powerpc/kernel/dma.c +7 -3
@@ -191,12 +191,10 @@ EXPORT_SYMBOL(dma_direct_ops);#define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16) -int dma_set_mask (struct device *dev,u64 dma_mask) +int __dma_set_mask (struct device *dev,u64 dma_mask) {struct dma_map_ops *dma_ops =get_dma_ops (dev); -if (ppc_md.dma_set_mask) -return ppc_md.dma_set_mask (dev, dma_mask);if ((dma_ops !=NULL ) && (dma_ops->set_dma_mask !=NULL ))return dma_ops->set_dma_mask (dev, dma_mask);if (!dev->dma_mask || !dma_supported (dev, dma_mask))
@@ -204,6 +202,12 @@ int dma_set_mask(struct device *dev, u64 dma_mask) *dev->dma_mask = dma_mask;return 0 ; } +int dma_set_mask (struct device *dev,u64 dma_mask) +{ +if (ppc_md.dma_set_mask) +return ppc_md.dma_set_mask (dev, dma_mask); +return __dma_set_mask (dev, dma_mask); +}EXPORT_SYMBOL (dma_set_mask);u64 dma_get_required_mask (struct device *dev)
arch/powerpc/kernel/iommu.c +12 -0
@@ -1088,6 +1088,14 @@ int iommu_take_ownership(struct iommu_table *tbl)memset (tbl->it_map,0xff , sz);iommu_clear_tces_and_put_pages (tbl, tbl->it_offset, tbl->it_size); +/* +* Disable iommu bypass, otherwise the user can DMA to all of +* our physical memory via the bypass window instead of just +* the pages that has been explicitly mapped into the iommu +*/ +if (tbl->set_bypass) + tbl->set_bypass (tbl, false); +return 0 ; }EXPORT_SYMBOL_GPL (iommu_take_ownership);
@@ -1102,6 +1110,10 @@ void iommu_release_ownership(struct iommu_table *tbl)/* Restore bit#0 set by iommu_init_table() */ if (tbl->it_offset ==0 )set_bit (0 , tbl->it_map); + +/* The kernel owns the device now, we can restore the iommu bypass */ +if (tbl->set_bypass) + tbl->set_bypass (tbl, true); }EXPORT_SYMBOL_GPL (iommu_release_ownership);
arch/powerpc/platforms/powernv/pci-ioda.c +84 -0
117 lines not on this page · raw message
arch/powerpc/platforms/powernv/pci.c +10 -0
@@ -634,6 +634,16 @@ static void pnv_pci_dma_dev_setup(struct pci_dev *pdev)pnv_pci_dma_fallback_setup (hose, pdev); } +int pnv_pci_dma_set_mask (struct pci_dev *pdev,u64 dma_mask) +{ +struct pci_controller *hose =pci_bus_to_host (pdev->bus); +struct pnv_phb *phb = hose->private_data; + +if (phb && phb->dma_set_mask) +return phb->dma_set_mask (phb, pdev, dma_mask); +return __dma_set_mask (&pdev->dev, dma_mask); +} +void pnv_pci_shutdown (void ) {struct pci_controller *hose;
arch/powerpc/platforms/powernv/pci.h +5 -1
@@ -54,7 +54,9 @@ struct pnv_ioda_pe {struct iommu_table tce32_table;phys_addr_t tce_inval_reg_phys; -/* XXX TODO: Add support for additional 64-bit iommus */ +/* 64-bit TCE bypass region */ +bool tce_bypass_enabled; +uint64_t tce_bypass_base;/* MSIs. MVE index is identical for for 32 and 64 bit MSI * and -1 if not supported. (It's actually identical to the
@@ -113,6 +115,8 @@ struct pnv_phb { unsigned int hwirq, unsigned int virq, unsigned int is_64, struct msi_msg *msg);void (*dma_dev_setup)(struct pnv_phb *phb,struct pci_dev *pdev); +int (*dma_set_mask)(struct pnv_phb *phb,struct pci_dev *pdev, +u64 dma_mask);void (*fixup_phb)(struct pci_controller *hose);u32 (*bdfn_to_pe)(struct pnv_phb *phb,struct pci_bus *bus,u32 devfn);void (*shutdown)(struct pnv_phb *phb);
arch/powerpc/platforms/powernv/powernv.h +4 -0
@@ -7,12 +7,16 @@ extern void pnv_smp_init(void);static inline void pnv_smp_init (void ) { }#endif +struct pci_dev ; +#ifdef CONFIG_PCI extern void pnv_pci_init (void );extern void pnv_pci_shutdown (void ); +extern int pnv_pci_dma_set_mask (struct pci_dev *pdev,u64 dma_mask);#else static inline void pnv_pci_init (void ) { }static inline void pnv_pci_shutdown (void ) { } +static inline int pnv_pci_dma_set_mask (struct pci_dev *pdev,u64 dma_mask) { }#endif extern void pnv_lpc_init (void );
arch/powerpc/platforms/powernv/setup.c +9 -0
@@ -27,6 +27,7 @@#include <linux/interrupt.h> #include <linux/bug.h> #include <linux/cpuidle.h> +#include <linux/pci.h> #include <asm/machdep.h> #include <asm/firmware.h>
@@ -141,6 +142,13 @@ static void pnv_progress(char *s, unsigned short hex) { } +static int pnv_dma_set_mask (struct device *dev,u64 dma_mask) +{ +if (dev_is_pci (dev)) +return pnv_pci_dma_set_mask (to_pci_dev (dev), dma_mask); +return __dma_set_mask (dev, dma_mask); +} +static void pnv_shutdown (void ) {/* Let the PCI code clear up IODA tables */
@@ -238,6 +246,7 @@ define_machine(powernv) { .machine_shutdown = pnv_shutdown, .power_save = powernv_idle, .calibrate_decr = generic_calibrate_decr, + .dma_set_mask = pnv_dma_set_mask,#ifdef CONFIG_KEXEC .kexec_cpu_down = pnv_kexec_cpu_down,#endif