use generic dma-direct and swiotlb code for x86
Subsystems (15)
Hi all,
this series switches the x86 code the the dma-direct implementation for direct (non-iommu) dma and the generic swiotlb ops. This includes getting rid of the special ops for the AMD memory encryption case and the STA2x11 SOC. The generic implementations are based on the x86 code, so they provide the same functionality.
PATCH 1/13 x86: remove X86_PPRO_FENCE
There were only a few Pentium Pro multiprocessors systems where this errata applied. They are more than 20 years old now, and we've slowly dropped places where put the workarounds in and discouraged anyone from enabling the workaround.
Get rid of it for good.
--- arch/x86/Kconfig.cpu | 13 ------------- arch/x86/entry/vdso/vdso32/vclock_gettime.c | 2 -- arch/x86/include/asm/barrier.h | 30 ----------------------------- arch/x86/include/asm/io.h | 15 --------------- arch/x86/kernel/pci-nommu.c | 19 ------------------ arch/x86/um/asm/barrier.h | 4 ---- 6 files changed, 83 deletions(-)
arch/x86/Kconfig.cpu +0 -13
@@ -315,19 +315,6 @@ config X86_L1_CACHE_SHIFT default "4" if MELAN || M486 || MGEODEGX1 default "5" if MWINCHIP3D || MWINCHIPC6 || MCRUSOE || MEFFICEON || MCYRIXIII || MK6 || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || M586 || MVIAC3_2 || MGEODE_LX -config X86_PPRO_FENCE - bool "PentiumPro memory ordering errata workaround" - depends on M686 || M586MMX || M586TSC || M586 || M486 || MGEODEGX1 - ---help--- - Old PentiumPro multiprocessor systems had errata that could cause - memory operations to violate the x86 ordering standard in rare cases. - Enabling this option will attempt to work around some (but not all) - occurrences of this problem, at the cost of much heavier spinlock and - memory barrier operations. - - If unsure, say n here. Even distro kernels should think twice before - enabling this: there are few systems, and an unlikely bug. - config X86_F00F_BUG def_bool y depends on M586MMX || M586TSC || M586 || M486
arch/x86/entry/vdso/vdso32/vclock_gettime.c +0 -2
@@ -5,8 +5,6 @@#undef CONFIG_OPTIMIZE_INLINING#endif -#undef CONFIG_X86_PPRO_FENCE -#ifdef CONFIG_X86_64 /*
arch/x86/include/asm/barrier.h +0 -30
@@ -52,11 +52,7 @@ static inline unsigned long array_index_mask_nospec(unsigned long index,#define barrier_nospec () alternative_2("", "mfence", X86_FEATURE_MFENCE_RDTSC, \ "lfence", X86_FEATURE_LFENCE_RDTSC) -#ifdef CONFIG_X86_PPRO_FENCE -#define dma_rmb () rmb() -#else #define dma_rmb () barrier() -#endif #define dma_wmb () barrier()#ifdef CONFIG_X86_32
@@ -68,30 +64,6 @@ static inline unsigned long array_index_mask_nospec(unsigned long index,#define __smp_wmb () barrier()#define __smp_store_mb (var, value) do { (void)xchg(&var, value); } while (0) -#if defined(CONFIG_X86_PPRO_FENCE ) - -/* -* For this option x86 doesn't have a strong TSO memory -* model and we should fall back to full barriers. -*/ - -#define __smp_store_release (p, v) \ -do { \ - compiletime_assert_atomic_type(*p); \ - __smp_mb(); \ - WRITE_ONCE(*p, v); \ -} while (0) - -#define __smp_load_acquire (p) \ -({ \ - typeof(*p) ___p1 = READ_ONCE(*p); \ - compiletime_assert_atomic_type(*p); \ - __smp_mb(); \ - ___p1; \ -}) - -#else /* regular x86 TSO memory ordering */ -#define __smp_store_release (p, v) \ do { \ compiletime_assert_atomic_type(*p); \
@@ -107,8 +79,6 @@ do { \ ___p1; \ }) -#endif -/* Atomic operations are already serializing on x86 */ #define __smp_mb__before_atomic () barrier()#define __smp_mb__after_atomic () barrier()
arch/x86/include/asm/io.h +0 -15
@@ -232,21 +232,6 @@ extern void set_iounmap_nonlazy(void);*/ #define __ISA_IO_base ((char __iomem *)(PAGE_OFFSET)) -/* -* Cache management -* -* This needed for two cases -* 1. Out of order aware processors -* 2. Accidentally out of order processors (PPro errata #51) -*/ - -static inline void flush_write_buffers (void ) -{ -#if defined(CONFIG_X86_PPRO_FENCE ) - asmvolatile ("lock; addl $0,0(%%esp)" : : :"memory" ); -#endif -} -#endif /* __KERNEL__ */ extern void native_io_delay (void );
arch/x86/kernel/pci-nommu.c +0 -19
@@ -37,7 +37,6 @@ static dma_addr_t nommu_map_page(struct device *dev, struct page *page,WARN_ON (size ==0 );if (!check_addr ("map_single" , dev, bus, size))return NOMMU_MAPPING_ERROR ; -flush_write_buffers ();return bus; }
@@ -72,25 +71,9 @@ static int nommu_map_sg(struct device *hwdev, struct scatterlist *sg,return 0 ; s->dma_length = s->length; } -flush_write_buffers ();return nents; } -static void nommu_sync_single_for_device (struct device *dev, -dma_addr_t addr,size_t size, -enum dma_data_direction dir) -{ -flush_write_buffers (); -} - - -static void nommu_sync_sg_for_device (struct device *dev, -struct scatterlist *sg,int nelems, -enum dma_data_direction dir) -{ -flush_write_buffers (); -} -static int nommu_mapping_error (struct device *dev,dma_addr_t dma_addr) {return dma_addr ==NOMMU_MAPPING_ERROR ;
@@ -101,8 +84,6 @@ const struct dma_map_ops nommu_dma_ops = { .free = dma_generic_free_coherent, .map_sg = nommu_map_sg, .map_page = nommu_map_page, - .sync_single_for_device = nommu_sync_single_for_device, - .sync_sg_for_device = nommu_sync_sg_for_device, .is_phys = 1, .mapping_error = nommu_mapping_error, .dma_supported = x86_dma_supported,
arch/x86/um/asm/barrier.h +0 -4
@@ -30,11 +30,7 @@#endif /* CONFIG_X86_32 */ -#ifdef CONFIG_X86_PPRO_FENCE -#define dma_rmb () rmb() -#else /* CONFIG_X86_PPRO_FENCE */ #define dma_rmb () barrier() -#endif /* CONFIG_X86_PPRO_FENCE */ #define dma_wmb () barrier()#include <asm-generic/barrier.h>
Signature · 1 line
2.14.2
Re: [PATCH 01/13] x86: remove X86_PPRO_FENCE
Quoted 7 lines · Christoph Hellwig
There were only a few Pentium Pro multiprocessors systems where this errata applied. They are more than 20 years old now, and we've slowly dropped places where put the workarounds in and discouraged anyone from enabling the workaround.
Get rid of it for good.
RIP.
Quoted 1 line
Signed-off-by Christoph Hellwig <[email protected]>
Thank you!
PATCH 2/13 x86: remove dma_alloc_coherent_mask
In reply to Christoph HellwigThese days all devices (including the ISA fallback device) have a coherent DMA mask set, so remove the workaround.
--- arch/x86/include/asm/dma-mapping.h | 18 ++---------------- arch/x86/kernel/pci-dma.c | 10 ++++------ arch/x86/mm/mem_encrypt.c | 4 +--- drivers/xen/swiotlb-xen.c | 16 +--------------- 4 files changed, 8 insertions(+), 40 deletions(-)
arch/x86/include/asm/dma-mapping.h +2 -16
@@ -44,26 +44,12 @@ extern void dma_generic_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_addr, unsigned long attrs); -static inline unsigned long dma_alloc_coherent_mask (struct device *dev, -gfp_t gfp) -{ -unsigned long dma_mask =0 ; - - dma_mask = dev->coherent_dma_mask; -if (!dma_mask) - dma_mask = (gfp &GFP_DMA ) ?DMA_BIT_MASK (24 ) :DMA_BIT_MASK (32 ); - -return dma_mask; -} -static inline gfp_t dma_alloc_coherent_gfp_flags (struct device *dev,gfp_t gfp) { -unsigned long dma_mask =dma_alloc_coherent_mask (dev, gfp); - -if (dma_mask <=DMA_BIT_MASK (24 )) +if (dev->coherent_dma_mask <=DMA_BIT_MASK (24 )) gfp |=GFP_DMA ;#ifdef CONFIG_X86_64 -if (dma_mask <=DMA_BIT_MASK (32 ) && !(gfp &GFP_DMA )) +if (dev->coherent_dma_mask <=DMA_BIT_MASK (32 ) && !(gfp &GFP_DMA )) gfp |=GFP_DMA32 ;#endif return gfp;
arch/x86/kernel/pci-dma.c +4 -6
@@ -80,13 +80,10 @@ void *dma_generic_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr, gfp_t flag, unsigned long attrs) { - unsigned long dma_mask; struct page *page;unsigned int count =PAGE_ALIGN (size) >>PAGE_SHIFT ;dma_addr_t addr; - dma_mask =dma_alloc_coherent_mask (dev, flag); - again: page =NULL ;/* CMA can be used only in the context which permits sleeping */
@@ -95,7 +92,7 @@ void *dma_generic_alloc_coherent(struct device *dev, size_t size, flag);if (page) { addr =phys_to_dma (dev,page_to_phys (page)); -if (addr + size > dma_mask) { +if (addr + size > dev->coherent_dma_mask) {dma_release_from_contiguous (dev, page, count); page =NULL ; }
@@ -108,10 +105,11 @@ void *dma_generic_alloc_coherent(struct device *dev, size_t size,return NULL ; addr =phys_to_dma (dev,page_to_phys (page)); -if (addr + size > dma_mask) { +if (addr + size > dev->coherent_dma_mask) {__free_pages (page,get_order (size)); -if (dma_mask <DMA_BIT_MASK (32 ) && !(flag &GFP_DMA )) { +if (dev->coherent_dma_mask <DMA_BIT_MASK (32 ) && + !(flag &GFP_DMA )) { flag = (flag & ~GFP_DMA32 ) |GFP_DMA ; goto again; }
arch/x86/mm/mem_encrypt.c +1 -3
@@ -203,12 +203,10 @@ void __init sme_early_init(void)static void *sev_alloc (struct device *dev,size_t size,dma_addr_t *dma_handle,gfp_t gfp,unsigned long attrs) { -unsigned long dma_mask;unsigned int order;struct page *page;void *vaddr =NULL ; - dma_mask =dma_alloc_coherent_mask (dev, gfp); order =get_order (size);/*
@@ -226,7 +224,7 @@ static void *sev_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,* mask with it already cleared. */ addr =__sme_clr (phys_to_dma (dev,page_to_phys (page))); -if ((addr + size) > dma_mask) { +if ((addr + size) > dev->coherent_dma_mask) {__free_pages (page,get_order (size)); }else { vaddr =page_address (page);
drivers/xen/swiotlb-xen.c +1 -15
@@ -53,20 +53,6 @@* API. */ -#ifndef CONFIG_X86 -static unsigned long dma_alloc_coherent_mask (struct device *dev, -gfp_t gfp) -{ -unsigned long dma_mask =0 ; - - dma_mask = dev->coherent_dma_mask; -if (!dma_mask) - dma_mask = (gfp &GFP_DMA ) ?DMA_BIT_MASK (24 ) :DMA_BIT_MASK (32 ); - -return dma_mask; -} -#endif -#define XEN_SWIOTLB_ERROR_CODE (~(dma_addr_t)0x0)static char *xen_io_tlb_start, *xen_io_tlb_end;
@@ -328,7 +314,7 @@ xen_swiotlb_alloc_coherent(struct device *hwdev, size_t size,return ret;if (hwdev && hwdev->coherent_dma_mask) - dma_mask =dma_alloc_coherent_mask (hwdev, flags); + dma_mask = hwdev->coherent_dma_mask;/* At this point dma_handle is the physical address, next we are * going to set it to the machine address.
Signature · 1 line
2.14.2
Re: [PATCH 02/13] x86: remove dma_alloc_coherent_mask
Quoted 4 lines · Christoph Hellwig
These days all devices (including the ISA fallback device) have a coherent DMA mask set, so remove the workaround.
Signed-off-by Christoph Hellwig <[email protected]>
Thank you!
PATCH 3/13 x86: use dma-direct
In reply to Christoph HellwigThe generic dma-direct implementation is now functionally equivalent to the x86 nommu dma_map implementation, so switch over to using it.
Note that the various iommu drivers are switched from x86_dma_supported to dma_direct_supported to provide identical functionality, although the checks looks fairly questionable for at least some of them.
--- arch/x86/Kconfig | 1 + arch/x86/include/asm/dma-mapping.h | 8 ----- arch/x86/include/asm/iommu.h | 3 -- arch/x86/kernel/Makefile | 2 +- arch/x86/kernel/amd_gart_64.c | 7 ++-- arch/x86/kernel/pci-calgary_64.c | 3 +- arch/x86/kernel/pci-dma.c | 66 +------------------------------------- arch/x86/kernel/pci-swiotlb.c | 5 ++- arch/x86/pci/sta2x11-fixup.c | 2 +- drivers/iommu/amd_iommu.c | 7 ++-- drivers/iommu/intel-iommu.c | 3 +- 11 files changed, 17 insertions(+), 90 deletions(-)
arch/x86/Kconfig +1 -0
@@ -83,6 +83,7 @@ config X86 select CLOCKSOURCE_VALIDATE_LAST_CYCLE select CLOCKSOURCE_WATCHDOG select DCACHE_WORD_ACCESS + select DMA_DIRECT_OPS select EDAC_ATOMIC_SCRUB select EDAC_SUPPORT select GENERIC_CLOCKEVENTS
arch/x86/include/asm/dma-mapping.h +0 -8
@@ -36,14 +36,6 @@ int arch_dma_supported(struct device *dev, u64 mask);bool arch_dma_alloc_attrs (struct device **dev,gfp_t *gfp);#define arch_dma_alloc_attrs arch_dma_alloc_attrs -extern void *dma_generic_alloc_coherent (struct device *dev,size_t size, -dma_addr_t *dma_addr,gfp_t flag, -unsigned long attrs); - -extern void dma_generic_free_coherent (struct device *dev,size_t size, -void *vaddr,dma_addr_t dma_addr, -unsigned long attrs); -static inline gfp_t dma_alloc_coherent_gfp_flags (struct device *dev,gfp_t gfp) {if (dev->coherent_dma_mask <=DMA_BIT_MASK (24 ))
arch/x86/include/asm/iommu.h +0 -3
@@ -2,13 +2,10 @@#ifndef _ASM_X86_IOMMU_H#define _ASM_X86_IOMMU_H -extern const struct dma_map_ops nommu_dma_ops;extern int force_iommu, no_iommu;extern int iommu_detected;extern int iommu_pass_through; -int x86_dma_supported (struct device *dev,u64 mask); -/* 10 seconds */ #define DMAR_OPERATION_TIMEOUT ((cycles_t) tsc_khz*10*1000)
arch/x86/kernel/Makefile +1 -1
@@ -57,7 +57,7 @@ obj-$(CONFIG_X86_ESPFIX64) += espfix_64.o obj-$(CONFIG_SYSFS) += ksysfs.o obj-y += bootflag.o e820.o obj-y += pci-dma.o quirks.o topology.o kdebugfs.o -obj-y += alternative.o i8253.o pci-nommu.o hw_breakpoint.o +obj-y += alternative.o i8253.o hw_breakpoint.o obj-y += tsc.o tsc_msr.o io_delay.o rtc.o obj-y += pci-iommu_table.o obj-y += resource.o
arch/x86/kernel/amd_gart_64.c +3 -4
@@ -501,8 +501,7 @@ gart_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr, }__free_pages (page,get_order (size)); }else -return dma_generic_alloc_coherent (dev, size, dma_addr, flag, - attrs); +return dma_direct_alloc (dev, size, dma_addr, flag, attrs);return NULL ; }
@@ -513,7 +512,7 @@ gart_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_addr, unsigned long attrs) { gart_unmap_page(dev, dma_addr, size, DMA_BIDIRECTIONAL, 0); -dma_generic_free_coherent (dev, size, vaddr, dma_addr, attrs); +dma_direct_free (dev, size, vaddr, dma_addr, attrs); }static int gart_mapping_error (struct device *dev,dma_addr_t dma_addr)
@@ -705,7 +704,7 @@ static const struct dma_map_ops gart_dma_ops = { .alloc = gart_alloc_coherent, .free = gart_free_coherent, .mapping_error = gart_mapping_error, - .dma_supported = x86_dma_supported, + .dma_supported = dma_direct_supported, };static void gart_iommu_shutdown (void )
arch/x86/kernel/pci-calgary_64.c +2 -1
@@ -33,6 +33,7 @@#include <linux/string.h> #include <linux/crash_dump.h> #include <linux/dma-mapping.h> +#include <linux/dma-direct.h> #include <linux/bitmap.h> #include <linux/pci_ids.h> #include <linux/pci.h>
@@ -493,7 +494,7 @@ static const struct dma_map_ops calgary_dma_ops = { .map_page = calgary_map_page, .unmap_page = calgary_unmap_page, .mapping_error = calgary_mapping_error, - .dma_supported = x86_dma_supported, + .dma_supported = dma_direct_supported, };static inline void __iomem *busno_to_bbar (unsigned char num)
arch/x86/kernel/pci-dma.c +1 -65
84 lines not on this page · raw message
arch/x86/kernel/pci-swiotlb.c +2 -3
@@ -30,8 +30,7 @@ void *x86_swiotlb_alloc_coherent(struct device *hwdev, size_t size,*/ flags |= __GFP_NOWARN; - vaddr =dma_generic_alloc_coherent (hwdev, size, dma_handle, flags, - attrs); + vaddr =dma_direct_alloc (hwdev, size, dma_handle, flags, attrs);if (vaddr)return vaddr;
@@ -45,7 +44,7 @@ void x86_swiotlb_free_coherent(struct device *dev, size_t size,if (is_swiotlb_buffer (dma_to_phys (dev, dma_addr)))swiotlb_free_coherent (dev, size, vaddr, dma_addr);else -dma_generic_free_coherent (dev, size, vaddr, dma_addr, attrs); +dma_direct_free (dev, size, vaddr, dma_addr, attrs); }static const struct dma_map_ops x86_swiotlb_dma_ops = {
arch/x86/pci/sta2x11-fixup.c +1 -1
@@ -193,7 +193,7 @@ static const struct dma_map_ops sta2x11_dma_ops = { .sync_sg_for_cpu = swiotlb_sync_sg_for_cpu, .sync_sg_for_device = swiotlb_sync_sg_for_device, .mapping_error = swiotlb_dma_mapping_error, - .dma_supported = x86_dma_supported, + .dma_supported = dma_direct_supported, };/* At setup time, we use our own ops if the device is a ConneXt one */
drivers/iommu/amd_iommu.c +4 -3
@@ -28,6 +28,7 @@#include <linux/debugfs.h> #include <linux/scatterlist.h> #include <linux/dma-mapping.h> +#include <linux/dma-direct.h> #include <linux/iommu-helper.h> #include <linux/iommu.h> #include <linux/delay.h>
@@ -2193,7 +2194,7 @@ static int amd_iommu_add_device(struct device *dev) dev_name(dev));iommu_ignore_device (dev); - dev->dma_ops = &nommu_dma_ops; + dev->dma_ops = &dma_direct_ops; goto out; }init_iommu_group (dev);
@@ -2680,7 +2681,7 @@ static void free_coherent(struct device *dev, size_t size,*/ static int amd_iommu_dma_supported (struct device *dev,u64 mask) { -if (!x86_dma_supported (dev, mask)) +if (!dma_direct_supported (dev, mask))return 0 ;return check_device (dev); }
@@ -2794,7 +2795,7 @@ int __init amd_iommu_init_dma_ops(void)* continue to be SWIOTLB. */ if (!swiotlb) - dma_ops = &nommu_dma_ops; + dma_ops = &dma_direct_ops;if (amd_iommu_unmap_flush)pr_info ("AMD-Vi: IO/TLB flush on unmap enabled\n" );
drivers/iommu/intel-iommu.c +2 -1
@@ -45,6 +45,7 @@#include <linux/pci-ats.h> #include <linux/memblock.h> #include <linux/dma-contiguous.h> +#include <linux/dma-direct.h> #include <linux/crash_dump.h> #include <asm/irq_remapping.h> #include <asm/cacheflush.h>
@@ -3871,7 +3872,7 @@ const struct dma_map_ops intel_dma_ops = { .unmap_page = intel_unmap_page, .mapping_error = intel_mapping_error, #ifdef CONFIG_X86 - .dma_supported = x86_dma_supported, + .dma_supported = dma_direct_supported,#endif };
Signature · 1 line
2.14.2
PATCH 4/13 x86: use generic swiotlb_ops
In reply to Christoph HellwigThe generic swiotlb dma ops were based on the x86 ones and provide equivalent functionality, so use them.
Also fix the sta2x11 case. For that SOC the dma map ops need an additional physical to dma address translations. For swiotlb buffers that is done throught the phys_to_dma helper, but the sta2x11_dma_ops also added an additional translation on the return value from x86_swiotlb_alloc_coherent, which is only correct if that functions returns a direct allocation and not a swiotlb buffer. With the generic swiotlb and dma-direct code phys_to_dma is not always used and the separate sta2x11_dma_ops can be replaced with a simple bit that marks if the additional physical to dma address translation is needed.
--- arch/x86/include/asm/device.h | 3 +++ arch/x86/include/asm/swiotlb.h | 8 ------- arch/x86/kernel/pci-swiotlb.c | 47 +----------------------------------------- arch/x86/pci/sta2x11-fixup.c | 46 +++++------------------------------------ 4 files changed, 9 insertions(+), 95 deletions(-)
arch/x86/include/asm/device.h +3 -0
@@ -6,6 +6,9 @@ struct dev_archdata {#if defined(CONFIG_INTEL_IOMMU ) || defined(CONFIG_AMD_IOMMU )void *iommu;/* hook for IOMMU specific extension */ #endif +#ifdef CONFIG_STA2X11 + bool is_sta2x11 : 1; +#endif };#if defined(CONFIG_X86_DEV_DMA_OPS ) && defined(CONFIG_PCI_DOMAINS )
arch/x86/include/asm/swiotlb.h +0 -8
@@ -27,12 +27,4 @@ static inline void pci_swiotlb_late_init(void) { }#endif - -extern void *x86_swiotlb_alloc_coherent (struct device *hwdev,size_t size, -dma_addr_t *dma_handle,gfp_t flags, -unsigned long attrs); -extern void x86_swiotlb_free_coherent (struct device *dev,size_t size, -void *vaddr,dma_addr_t dma_addr, -unsigned long attrs); -#endif /* _ASM_X86_SWIOTLB_H */
arch/x86/kernel/pci-swiotlb.c +1 -46
59 lines not on this page · raw message
arch/x86/pci/sta2x11-fixup.c +5 -41
76 lines not on this page · raw message
Signature · 1 line
2.14.2
Robin Murphy
reviewerRe: [PATCH 04/13] x86: use generic swiotlb_ops
Hi Christoph,
Quoted 13 lines · Christoph Hellwig
The generic swiotlb dma ops were based on the x86 ones and provide equivalent functionality, so use them.
Also fix the sta2x11 case. For that SOC the dma map ops need an additional physical to dma address translations. For swiotlb buffers that is done throught the
phys_to_dmahelper, but thesta2x11_dma_opsalso added an additional translation on the return value fromx86_swiotlb_alloc_coherent, which is only correct if that functions returns a direct allocation and not a swiotlb buffer. With the generic swiotlb and dma-direct codephys_to_dmais not always used and the separatesta2x11_dma_opscan be replaced with a simple bit that marks if the additional physical to dma address translation is needed.
FWIW, last time I looked I got the impression that STA2x11 could just use dma_pfn_offset - the comments and a2p/p2a logic in sta2x11-fixup.c certainly imply that the underlying hardware situation is pretty much exactly that for which dma_pfn_offset exists.
Robin.
Quoted 191 lines
Signed-off-by Christoph Hellwig <[email protected]>--- arch/x86/include/asm/device.h | 3 +++ arch/x86/include/asm/swiotlb.h | 8 ------- arch/x86/kernel/pci-swiotlb.c | 47 +----------------------------------------- arch/x86/pci/sta2x11-fixup.c | 46 +++++------------------------------------ 4 files changed, 9 insertions(+), 95 deletions(-)arch/x86/include/asm/device.h +3
-0@@ -6,6 +6,9 @@ struct dev_archdata {#if defined(CONFIG_INTEL_IOMMU ) || defined(CONFIG_AMD_IOMMU )void *iommu;/* hook for IOMMU specific extension */ #endif +#ifdef CONFIG_STA2X11 + bool is_sta2x11 : 1; +#endif };#if defined(CONFIG_X86_DEV_DMA_OPS ) && defined(CONFIG_PCI_DOMAINS )arch/x86/include/asm/swiotlb.h +0
-8@@ -27,12 +27,4 @@ static inline void pci_swiotlb_late_init(void) { }#endif - -extern void *x86_swiotlb_alloc_coherent (struct device *hwdev,size_t size, -dma_addr_t *dma_handle,gfp_t flags, -unsigned long attrs); -extern void x86_swiotlb_free_coherent (struct device *dev,size_t size, -void *vaddr,dma_addr_t dma_addr, -unsigned long attrs); -#endif /* _ASM_X86_SWIOTLB_H */ arch/x86/kernel/pci-swiotlb.c +1
-46@@ -17,51 +17,6 @@int swiotlb__read_mostly ; -void *x86_swiotlb_alloc_coherent (struct device *hwdev,size_t size, -dma_addr_t *dma_handle,gfp_t flags, -unsigned long attrs) -{ -void *vaddr; - -/* -* Don't print a warning when the first allocation attempt fails. -* swiotlb_alloc_coherent() will print a warning when the DMA -* memory allocation ultimately failed. -*/ - flags |= __GFP_NOWARN; - - vaddr =dma_direct_alloc (hwdev, size, dma_handle, flags, attrs); -if (vaddr) -return vaddr; - -return swiotlb_alloc_coherent (hwdev, size, dma_handle, flags); -} - -void x86_swiotlb_free_coherent (struct device *dev,size_t size, -void *vaddr,dma_addr_t dma_addr, -unsigned long attrs) -{ -if (is_swiotlb_buffer (dma_to_phys (dev, dma_addr))) -swiotlb_free_coherent (dev, size, vaddr, dma_addr); -else -dma_direct_free (dev, size, vaddr, dma_addr, attrs); -} - -static const struct dma_map_ops x86_swiotlb_dma_ops = { - .mapping_error = swiotlb_dma_mapping_error, - .alloc = x86_swiotlb_alloc_coherent, - .free = x86_swiotlb_free_coherent, - .sync_single_for_cpu = swiotlb_sync_single_for_cpu, - .sync_single_for_device = swiotlb_sync_single_for_device, - .sync_sg_for_cpu = swiotlb_sync_sg_for_cpu, - .sync_sg_for_device = swiotlb_sync_sg_for_device, - .map_sg = swiotlb_map_sg_attrs, - .unmap_sg = swiotlb_unmap_sg_attrs, - .map_page = swiotlb_map_page, - .unmap_page = swiotlb_unmap_page, - .dma_supported =NULL , -}; -/* * pci_swiotlb_detect_override - set swiotlb to 1 if necessary * @@ -111,7 +66,7 @@ void __init pci_swiotlb_init(void) {if (swiotlb) {swiotlb_init (0 ); - dma_ops = &x86_swiotlb_dma_ops; + dma_ops = &swiotlb_dma_ops; } }arch/x86/pci/sta2x11-fixup.c +5
-41@@ -159,43 +159,6 @@ static dma_addr_t a2p(dma_addr_t a, struct pci_dev *pdev)return p; } -/** -* sta2x11_swiotlb_alloc_coherent - Allocate swiotlb bounce buffers -* returns virtual address. This is the only "special" function here. -* @dev: PCI device -* @size: Size of the buffer -* @dma_handle: DMA address -* @flags: memory flags -*/ -static void *sta2x11_swiotlb_alloc_coherent (struct device *dev, -size_t size, -dma_addr_t *dma_handle, -gfp_t flags, -unsigned long attrs) -{ -void *vaddr; - - vaddr =x86_swiotlb_alloc_coherent (dev, size, dma_handle, flags, attrs); - *dma_handle =p2a (*dma_handle,to_pci_dev (dev)); -return vaddr; -} - -/* We have our own dma_ops: the same as swiotlb but from alloc (above) */ -static const struct dma_map_ops sta2x11_dma_ops = { - .alloc = sta2x11_swiotlb_alloc_coherent, - .free = x86_swiotlb_free_coherent, - .map_page = swiotlb_map_page, - .unmap_page = swiotlb_unmap_page, - .map_sg = swiotlb_map_sg_attrs, - .unmap_sg = swiotlb_unmap_sg_attrs, - .sync_single_for_cpu = swiotlb_sync_single_for_cpu, - .sync_single_for_device = swiotlb_sync_single_for_device, - .sync_sg_for_cpu = swiotlb_sync_sg_for_cpu, - .sync_sg_for_device = swiotlb_sync_sg_for_device, - .mapping_error = swiotlb_dma_mapping_error, - .dma_supported = dma_direct_supported, -}; -/* At setup time, we use our own ops if the device is a ConneXt one */ static void sta2x11_setup_pdev (struct pci_dev *pdev) {@@ -205,7 +168,8 @@ static void sta2x11_setup_pdev(struct pci_dev *pdev)return ;pci_set_consistent_dma_mask (pdev,STA2X11_AMBA_SIZE -1 );pci_set_dma_mask (pdev,STA2X11_AMBA_SIZE -1 ); - pdev->dev.dma_ops = &sta2x11_dma_ops; + pdev->dev.dma_ops = &swiotlb_dma_ops; + pdev->dev.archdata.is_sta2x11 = true;/* We must enable all devices as master, for audio DMA to work */ pci_set_master (pdev);@@ -225,7 +189,7 @@ bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) {struct sta2x11_mapping *map; -if (dev->dma_ops != &sta2x11_dma_ops) { +if (!dev->archdata.is_sta2x11) {if (!dev->dma_mask)return false;return addr + size -1 <= *dev->dma_mask;@@ -249,7 +213,7 @@ bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)*/ dma_addr_t phys_to_dma (struct device *dev,phys_addr_t paddr) { -if (dev->dma_ops != &sta2x11_dma_ops) +if (!dev->archdata.is_sta2x11)return paddr;return p2a (paddr,to_pci_dev (dev)); }@@ -261,7 +225,7 @@ dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)*/ phys_addr_t dma_to_phys (struct device *dev,dma_addr_t daddr) { -if (dev->dma_ops != &sta2x11_dma_ops) +if (!dev->archdata.is_sta2x11)return daddr;return a2p (daddr,to_pci_dev (dev)); }
Re: [PATCH 04/13] x86: use generic swiotlb_ops
Quoted 11 lines · Robin Murphy
Quoted 10 lines
Also fix the sta2x11 case. For that SOC the dma map ops need an additional physical to dma address translations. For swiotlb buffers that is done throught the
phys_to_dmahelper, but thesta2x11_dma_opsalso added an additional translation on the return value fromx86_swiotlb_alloc_coherent, which is only correct if that functions returns a direct allocation and not a swiotlb buffer. With the generic swiotlb and dma-direct codephys_to_dmais not always used and the separatesta2x11_dma_opscan be replaced with a simple bit that marks if the additional physical to dma address translation is needed.
Quoted 4 lines
FWIW, last time I looked I got the impression that STA2x11 could just use
dma_pfn_offset- the comments and a2p/p2a logic insta2x11-fixup.ccertainly imply that the underlying hardware situation is pretty much exactly that for whichdma_pfn_offsetexists.
That probably is the case. But without access to the hardware I don't feel like doing this deeper surgery. And even without that this case provides a great simplification.
PATCH 5/13 x86/amd_gart: look at coherent_dma_mask instead of GFP_DMA
In reply to Christoph HellwigWe want to phase out looking at the magic GFP_DMA flag in the dma mapping routines, so switch the gart driver to use the coherent_dma_mask instead, which is used to select the GFP_DMA flag in the caller.
--- arch/x86/kernel/amd_gart_64.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/kernel/amd_gart_64.c +1 -1
@@ -484,7 +484,7 @@ gart_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr,unsigned long align_mask;struct page *page; -if (force_iommu && !(flag &GFP_DMA )) { +if (force_iommu && dev->coherent_dma_mask >DMA_BIT_MASK (24 )) { flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32); page =alloc_pages (flag | __GFP_ZERO,get_order (size));if (!page)
Signature · 1 line
2.14.2
PATCH 6/13 x86/amd_gart: use dma_direct_{alloc,free}
In reply to Christoph HellwigThis gains support for CMA allocations for the force_iommu case, and cleans up the code a bit.
--- arch/x86/kernel/amd_gart_64.c | 36 ++++++++++++++---------------------- 1 file changed, 14 insertions(+), 22 deletions(-)
arch/x86/kernel/amd_gart_64.c +14 -22
@@ -480,29 +480,21 @@ static void * gart_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr, gfp_t flag, unsigned long attrs) { - dma_addr_t paddr; -unsigned long align_mask; -struct page *page; - -if (force_iommu && dev->coherent_dma_mask >DMA_BIT_MASK (24 )) { - flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32); - page =alloc_pages (flag | __GFP_ZERO,get_order (size)); -if (!page) -return NULL ; - - align_mask = (1UL <<get_order (size)) -1 ; - paddr =dma_map_area (dev,page_to_phys (page), size, -DMA_BIDIRECTIONAL , align_mask); - -flush_gart (); -if (paddr != bad_dma_addr) { - *dma_addr = paddr; -return page_address (page); - } -__free_pages (page,get_order (size)); - }else -return dma_direct_alloc (dev, size, dma_addr, flag, attrs); + void *vaddr; + + vaddr =dma_direct_alloc (dev, size, dma_addr, flag, attrs); +if (!vaddr || + !force_iommu || dev->coherent_dma_mask <=DMA_BIT_MASK (24 )) +return vaddr; + *dma_addr =dma_map_area (dev,virt_to_phys (vaddr), size, +DMA_BIDIRECTIONAL , (1UL <<get_order (size)) -1 ); +flush_gart (); +if (unlikely (*dma_addr == bad_dma_addr)) + goto out_free; +return vaddr; +out_free: +dma_direct_free (dev, size, vaddr, *dma_addr, attrs);return NULL ; }
Signature · 1 line
2.14.2
PATCH 7/13 iommu/amd_iommu: use dma_direct_{alloc,free}
In reply to Christoph HellwigThis cleans up the code a lot by removing duplicate logic.
--- drivers/iommu/Kconfig | 1 + drivers/iommu/amd_iommu.c | 68 +++++++++++++++-------------------------------- 2 files changed, 22 insertions(+), 47 deletions(-)
drivers/iommu/Kconfig +1 -0
@@ -107,6 +107,7 @@ config IOMMU_PGTABLES_L2 # AMD IOMMU support config AMD_IOMMU bool "AMD IOMMU support" + select DMA_DIRECT_OPS select SWIOTLB select PCI_MSI select PCI_ATS
drivers/iommu/amd_iommu.c +21 -47
96 lines not on this page · raw message
Signature · 1 line
2.14.2
PATCH 8/13 iommu/intel-iommu: cleanup intel_{alloc,free}_coherent
In reply to Christoph HellwigUse the dma_direct_* helpers and cleanup the code flow.
--- drivers/iommu/Kconfig | 1 + drivers/iommu/intel-iommu.c | 62 ++++++++++++--------------------------------- 2 files changed, 17 insertions(+), 46 deletions(-)
drivers/iommu/Kconfig +1 -0
@@ -143,6 +143,7 @@ config DMAR_TABLE config INTEL_IOMMU bool "Support for Intel IOMMU using DMA Remapping Devices" depends on PCI_MSI && ACPI && (X86 || IA64_GENERIC) + select DMA_DIRECT_OPS select IOMMU_API select IOMMU_IOVA select DMAR_TABLE
drivers/iommu/intel-iommu.c +16 -46
83 lines not on this page · raw message
Signature · 1 line
2.14.2
PATCH 9/13 x86: remove dma_alloc_coherent_gfp_flags
In reply to Christoph HellwigAll dma_ops implementations used on x86 now take care of setting their own required GFP_ masks for the allocation. And given that the common code now clears harmful flags itself that means we can stop the flags in all the iommu implementations as well.
--- arch/x86/include/asm/dma-mapping.h | 11 ----------- arch/x86/kernel/pci-calgary_64.c | 2 -- arch/x86/kernel/pci-dma.c | 2 -- arch/x86/mm/mem_encrypt.c | 7 ------- 4 files changed, 22 deletions(-)
arch/x86/include/asm/dma-mapping.h +0 -11
@@ -36,15 +36,4 @@ int arch_dma_supported(struct device *dev, u64 mask);bool arch_dma_alloc_attrs (struct device **dev,gfp_t *gfp);#define arch_dma_alloc_attrs arch_dma_alloc_attrs -static inline gfp_t dma_alloc_coherent_gfp_flags (struct device *dev,gfp_t gfp) -{ -if (dev->coherent_dma_mask <=DMA_BIT_MASK (24 )) - gfp |=GFP_DMA ; -#ifdef CONFIG_X86_64 -if (dev->coherent_dma_mask <=DMA_BIT_MASK (32 ) && !(gfp &GFP_DMA )) - gfp |=GFP_DMA32 ; -#endif -return gfp; -} -#endif
arch/x86/kernel/pci-calgary_64.c +0 -2
@@ -446,8 +446,6 @@ static void* calgary_alloc_coherent(struct device *dev, size_t size, npages = size >>PAGE_SHIFT ; order =get_order (size); - flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32); -/* alloc enough pages (and possibly more) */ ret = (void *)__get_free_pages (flag, order);if (!ret)
arch/x86/kernel/pci-dma.c +0 -2
@@ -82,8 +82,6 @@ bool arch_dma_alloc_attrs(struct device **dev, gfp_t *gfp)if (!*dev) *dev = &x86_dma_fallback_dev; - *gfp =dma_alloc_coherent_gfp_flags (*dev, *gfp); -if (!is_device_dma_capable (*dev))return false;return true;
arch/x86/mm/mem_encrypt.c +0 -7
@@ -208,13 +208,6 @@ static void *sev_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,void *vaddr =NULL ; order =get_order (size); - -/* -* Memory will be memset to zero after marking decrypted, so don't -* bother clearing it before. -*/ - gfp &= ~__GFP_ZERO; - page =alloc_pages_node (dev_to_node (dev), gfp, order);if (page) {dma_addr_t addr;
Signature · 1 line
2.14.2
PATCH 10/13 set_memory.h: provide set_memory_{en,de}crypted stubs
In reply to Christoph Hellwig--- include/linux/set_memory.h | 12 ++++++++++++ 1 file changed, 12 insertions(+)
include/linux/set_memory.h +12 -0
@@ -17,4 +17,16 @@ static inline int set_memory_x(unsigned long addr, int numpages) { return 0; }static inline int set_memory_nx (unsigned long addr,int numpages) {return 0 ; }#endif +#ifndef CONFIG_ARCH_HAS_MEM_ENCRYPT +static inline int set_memory_encrypted (unsigned long addr,int numpages) +{ +return 0 ; +} + +static inline int set_memory_decrypted (unsigned long addr,int numpages) +{ +return 0 ; +} +#endif /* CONFIG_ARCH_HAS_MEM_ENCRYPT */ +#endif /* _LINUX_SET_MEMORY_H_ */
Signature · 1 line
2.14.2
PATCH 11/13 dma-direct: handle the memory encryption bit in common code
In reply to Christoph HellwigGive the basic phys_to_dma and dma_to_phys helpers a __-prefix and add the memory encryption mask to the non-prefixed versions. Use the __-prefixed versions directly instead of clearing the mask again in various places.
With that in place the generic dma-direct routines can be used to allocate non-encrypted bounce buffers, and the x86 SEV case can use the generic swiotlb ops.
--- arch/arm/include/asm/dma-direct.h | 4 +- arch/mips/cavium-octeon/dma-octeon.c | 10 +-- .../include/asm/mach-cavium-octeon/dma-coherence.h | 4 +- .../include/asm/mach-loongson64/dma-coherence.h | 10 +-- arch/mips/loongson64/common/dma-swiotlb.c | 4 +- arch/powerpc/include/asm/dma-direct.h | 4 +- arch/x86/Kconfig | 2 +- arch/x86/include/asm/dma-direct.h | 25 +------- arch/x86/mm/mem_encrypt.c | 73 +--------------------- arch/x86/pci/sta2x11-fixup.c | 6 +- include/linux/dma-direct.h | 21 ++++++- lib/dma-direct.c | 21 +++++-- lib/swiotlb.c | 25 +++----- 13 files changed, 70 insertions(+), 139 deletions(-)
arch/arm/include/asm/dma-direct.h +2 -2
@@ -2,13 +2,13 @@#ifndef ASM_ARM_DMA_DIRECT_H #define ASM_ARM_DMA_DIRECT_H 1 -static inline dma_addr_t phys_to_dma (struct device *dev,phys_addr_t paddr) +static inline dma_addr_t __phys_to_dma (struct device *dev,phys_addr_t paddr) {unsigned int offset = paddr & ~PAGE_MASK ;return pfn_to_dma (dev,__phys_to_pfn (paddr)) + offset; } -static inline phys_addr_t dma_to_phys (struct device *dev,dma_addr_t dev_addr) +static inline phys_addr_t __dma_to_phys (struct device *dev,dma_addr_t dev_addr) {unsigned int offset = dev_addr & ~PAGE_MASK ;return __pfn_to_phys (dma_to_pfn (dev, dev_addr)) + offset;
arch/mips/cavium-octeon/dma-octeon.c +5 -5
@@ -10,7 +10,7 @@* IP32 changes by Ilya. * Copyright (C) 2010 Cavium Networks, Inc. */ -#include <linux/dma-mapping.h> +#include <linux/dma-direct.h> #include <linux/scatterlist.h> #include <linux/bootmem.h> #include <linux/export.h>
@@ -182,7 +182,7 @@ struct octeon_dma_map_ops {phys_addr_t (*dma_to_phys)(struct device *dev,dma_addr_t daddr); }; -dma_addr_t phys_to_dma (struct device *dev,phys_addr_t paddr) +dma_addr_t __phys_to_dma (struct device *dev,phys_addr_t paddr) { struct octeon_dma_map_ops *ops = container_of(get_dma_ops(dev), struct octeon_dma_map_ops,
@@ -190,9 +190,9 @@ dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)return ops->phys_to_dma (dev, paddr); } -EXPORT_SYMBOL (phys_to_dma); +EXPORT_SYMBOL (__phys_to_dma); -phys_addr_t dma_to_phys (struct device *dev,dma_addr_t daddr) +phys_addr_t __dma_to_phys (struct device *dev,dma_addr_t daddr) { struct octeon_dma_map_ops *ops = container_of(get_dma_ops(dev), struct octeon_dma_map_ops,
@@ -200,7 +200,7 @@ phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)return ops->dma_to_phys (dev, daddr); } -EXPORT_SYMBOL (dma_to_phys); +EXPORT_SYMBOL (__dma_to_phys);static struct octeon_dma_map_ops octeon_linear_dma_map_ops = { .dma_map_ops = {
arch/mips/include/asm/mach-cavium-octeon/dma-coherence.h +2 -2
@@ -69,8 +69,8 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)return addr + size -1 <= *dev->dma_mask; } -dma_addr_t phys_to_dma (struct device *dev,phys_addr_t paddr); -phys_addr_t dma_to_phys (struct device *dev,dma_addr_t daddr); +dma_addr_t __phys_to_dma (struct device *dev,phys_addr_t paddr); +phys_addr_t __dma_to_phys (struct device *dev,dma_addr_t daddr);struct dma_map_ops ;extern const struct dma_map_ops *octeon_pci_dma_map_ops;
arch/mips/include/asm/mach-loongson64/dma-coherence.h +5 -5
@@ -25,13 +25,13 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)return addr + size -1 <= *dev->dma_mask; } -extern dma_addr_t phys_to_dma (struct device *dev,phys_addr_t paddr); -extern phys_addr_t dma_to_phys (struct device *dev,dma_addr_t daddr); +extern dma_addr_t __phys_to_dma (struct device *dev,phys_addr_t paddr); +extern phys_addr_t __dma_to_phys (struct device *dev,dma_addr_t daddr);static inline dma_addr_t plat_map_dma_mem (struct device *dev,void *addr,size_t size) {#ifdef CONFIG_CPU_LOONGSON3 -return phys_to_dma (dev,virt_to_phys (addr)); +return __phys_to_dma (dev,virt_to_phys (addr));#else return virt_to_phys (addr) |0x80000000 ;#endif
@@ -41,7 +41,7 @@ static inline dma_addr_t plat_map_dma_mem_page(struct device *dev, struct page *page) {#ifdef CONFIG_CPU_LOONGSON3 -return phys_to_dma (dev,page_to_phys (page)); +return __phys_to_dma (dev,page_to_phys (page));#else return page_to_phys (page) |0x80000000 ;#endif
@@ -51,7 +51,7 @@ static inline unsigned long plat_dma_addr_to_phys(struct device *dev, dma_addr_t dma_addr) {#if defined(CONFIG_CPU_LOONGSON3 ) && defined(CONFIG_64BIT ) -return dma_to_phys (dev, dma_addr); +return __dma_to_phys (dev, dma_addr);#elif defined(CONFIG_CPU_LOONGSON2F ) && defined(CONFIG_64BIT )return (dma_addr >0x8fffffff ) ? dma_addr : (dma_addr &0x0fffffff );#else
arch/mips/loongson64/common/dma-swiotlb.c +2 -2
@@ -63,7 +63,7 @@ static int loongson_dma_supported(struct device *dev, u64 mask)return swiotlb_dma_supported (dev, mask); } -dma_addr_t phys_to_dma (struct device *dev,phys_addr_t paddr) +dma_addr_t __phys_to_dma (struct device *dev,phys_addr_t paddr) {long nid;#ifdef CONFIG_PHYS48_TO_HT40
@@ -75,7 +75,7 @@ dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)return paddr; } -phys_addr_t dma_to_phys (struct device *dev,dma_addr_t daddr) +phys_addr_t __dma_to_phys (struct device *dev,dma_addr_t daddr) {long nid;#ifdef CONFIG_PHYS48_TO_HT40
arch/powerpc/include/asm/dma-direct.h +2 -2
@@ -17,12 +17,12 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)return addr + size -1 <= *dev->dma_mask; } -static inline dma_addr_t phys_to_dma (struct device *dev,phys_addr_t paddr) +static inline dma_addr_t __phys_to_dma (struct device *dev,phys_addr_t paddr) {return paddr +get_dma_offset (dev); } -static inline phys_addr_t dma_to_phys (struct device *dev,dma_addr_t daddr) +static inline phys_addr_t __dma_to_phys (struct device *dev,dma_addr_t daddr) {return daddr -get_dma_offset (dev); }
arch/x86/Kconfig +1 -1
@@ -54,7 +54,6 @@ config X86 select ARCH_HAS_FORTIFY_SOURCE select ARCH_HAS_GCOV_PROFILE_ALL select ARCH_HAS_KCOV if X86_64 - select ARCH_HAS_PHYS_TO_DMA select ARCH_HAS_MEMBARRIER_SYNC_CORE select ARCH_HAS_PMEM_API if X86_64 select ARCH_HAS_REFCOUNT
@@ -691,6 +690,7 @@ config X86_SUPPORTS_MEMORY_FAILURE config STA2X11 bool "STA2X11 Companion Chip Support" depends on X86_32_NON_STANDARD && PCI + select ARCH_HAS_PHYS_TO_DMA select X86_DEV_DMA_OPS select X86_DMA_REMAP select SWIOTLB
arch/x86/include/asm/dma-direct.h +2 -23
@@ -2,29 +2,8 @@#ifndef ASM_X86_DMA_DIRECT_H #define ASM_X86_DMA_DIRECT_H 1 -#include <linux/mem_encrypt.h> - -#ifdef CONFIG_X86_DMA_REMAP /* Platform code defines bridge-specific code */ bool dma_capable (struct device *dev,dma_addr_t addr,size_t size); -dma_addr_t phys_to_dma (struct device *dev,phys_addr_t paddr); -phys_addr_t dma_to_phys (struct device *dev,dma_addr_t daddr); -#else -static inline bool dma_capable (struct device *dev,dma_addr_t addr,size_t size) -{ -if (!dev->dma_mask) -return 0 ; - -return addr + size -1 <= *dev->dma_mask; -} - -static inline dma_addr_t phys_to_dma (struct device *dev,phys_addr_t paddr) -{ -return __sme_set (paddr); -} +dma_addr_t __phys_to_dma (struct device *dev,phys_addr_t paddr); +phys_addr_t __dma_to_phys (struct device *dev,dma_addr_t daddr); -static inline phys_addr_t dma_to_phys (struct device *dev,dma_addr_t daddr) -{ -return __sme_clr (daddr); -} -#endif /* CONFIG_X86_DMA_REMAP */ #endif /* ASM_X86_DMA_DIRECT_H */
arch/x86/mm/mem_encrypt.c +3 -70
93 lines not on this page · raw message
arch/x86/pci/sta2x11-fixup.c +3 -3
@@ -207,11 +207,11 @@ bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) }/** -* phys_to_dma - Return the DMA AMBA address used for this STA2x11 device +* __phys_to_dma - Return the DMA AMBA address used for this STA2x11 device * @dev: device for a PCI device * @paddr: Physical address */ -dma_addr_t phys_to_dma (struct device *dev,phys_addr_t paddr) +dma_addr_t __phys_to_dma (struct device *dev,phys_addr_t paddr) {if (!dev->archdata.is_sta2x11)return paddr;
@@ -223,7 +223,7 @@ dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)* @dev: device for a PCI device * @daddr: STA2x11 AMBA DMA address */ -phys_addr_t dma_to_phys (struct device *dev,dma_addr_t daddr) +phys_addr_t __dma_to_phys (struct device *dev,dma_addr_t daddr) {if (!dev->archdata.is_sta2x11)return daddr;
include/linux/dma-direct.h +19 -2
@@ -3,18 +3,19 @@#define _LINUX_DMA_DIRECT_H 1#include <linux/dma-mapping.h> +#include <linux/mem_encrypt.h> #ifdef CONFIG_ARCH_HAS_PHYS_TO_DMA #include <asm/dma-direct.h> #else -static inline dma_addr_t phys_to_dma (struct device *dev,phys_addr_t paddr) +static inline dma_addr_t __phys_to_dma (struct device *dev,phys_addr_t paddr) {dma_addr_t dev_addr = (dma_addr_t )paddr;return dev_addr - ((dma_addr_t )dev->dma_pfn_offset <<PAGE_SHIFT ); } -static inline phys_addr_t dma_to_phys (struct device *dev,dma_addr_t dev_addr) +static inline phys_addr_t __dma_to_phys (struct device *dev,dma_addr_t dev_addr) {phys_addr_t paddr = (phys_addr_t )dev_addr;
@@ -30,6 +31,22 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) }#endif /* !CONFIG_ARCH_HAS_PHYS_TO_DMA */ +/* +* If memory encryption is supported, phys_to_dma will set the memory encryption +* bit in the DMA address, and dma_to_phys will clear it. The raw __phys_to_dma +* and __dma_to_phys versions should only be used on non-encrypted memory for +* special occasions like DMA coherent buffers. +*/ +static inline dma_addr_t phys_to_dma (struct device *dev,phys_addr_t paddr) +{ +return __sme_set (__phys_to_dma (dev, paddr)); +} + +static inline phys_addr_t dma_to_phys (struct device *dev,dma_addr_t daddr) +{ +return __sme_clr (__dma_to_phys (dev, daddr)); +} +#ifdef CONFIG_ARCH_HAS_DMA_MARK_CLEAN void dma_mark_clean (void *addr,size_t size);#else
lib/dma-direct.c +15 -6
@@ -9,6 +9,7 @@#include <linux/scatterlist.h> #include <linux/dma-contiguous.h> #include <linux/pfn.h> +#include <linux/set_memory.h> #define DIRECT_MAPPING_ERROR 0
@@ -35,9 +36,13 @@ check_addr(struct device *dev, dma_addr_t dma_addr, size_t size,return true; } +/* +* Since we will be clearing the encryption bit, check the mask with it already +* cleared. +*/ static bool dma_coherent_ok (struct device *dev,phys_addr_t phys,size_t size) { -return phys_to_dma (dev, phys) + size -1 <= dev->coherent_dma_mask; +return __phys_to_dma (dev, phys) + size -1 <= dev->coherent_dma_mask; } void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
@@ -46,6 +51,7 @@ void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,unsigned int count =PAGE_ALIGN (size) >>PAGE_SHIFT ;int page_order =get_order (size);struct page *page =NULL ; +void *ret;/* GFP_DMA32 and GFP_DMA are no ops without the corresponding zones: */ if (dev->coherent_dma_mask <=DMA_BIT_MASK (ARCH_ZONE_DMA_BITS ))
@@ -78,10 +84,11 @@ void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,if (!page)return NULL ; - - *dma_handle =phys_to_dma (dev,page_to_phys (page)); -memset (page_address (page),0 , size); -return page_address (page); + *dma_handle =__phys_to_dma (dev,page_to_phys (page)); + ret =page_address (page); +set_memory_decrypted ((unsigned long )ret, page_order); +memset (ret,0 , size); +return ret; }/*
@@ -92,9 +99,11 @@ void dma_direct_free(struct device *dev, size_t size, void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs) {unsigned int count =PAGE_ALIGN (size) >>PAGE_SHIFT ; +unsigned int page_order =get_order (size); +set_memory_encrypted ((unsigned long )cpu_addr,1 << page_order);if (!dma_release_from_contiguous (dev,virt_to_page (cpu_addr), count)) -free_pages ((unsigned long )cpu_addr,get_order (size)); +free_pages ((unsigned long )cpu_addr, page_order); } static dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,
lib/swiotlb.c +9 -16
@@ -158,13 +158,6 @@ unsigned long swiotlb_size_or_default(void)void __weak swiotlb_set_mem_attributes (void *vaddr,unsigned long size) { } -/* For swiotlb, clear memory encryption mask from dma addresses */ -static dma_addr_t swiotlb_phys_to_dma (struct device *hwdev, -phys_addr_t address) -{ -return __sme_clr (phys_to_dma (hwdev, address)); -} -/* Note that this doesn't work with highmem page */ static dma_addr_t swiotlb_virt_to_bus (struct device *hwdev,volatile void *address)
@@ -622,7 +615,7 @@ map_single(struct device *hwdev, phys_addr_t phys, size_t size,return SWIOTLB_MAP_ERROR ; } - start_dma_addr =swiotlb_phys_to_dma (hwdev, io_tlb_start); + start_dma_addr =__phys_to_dma (hwdev, io_tlb_start);return swiotlb_tbl_map_single (hwdev, start_dma_addr, phys, size, dir, attrs); }
@@ -726,12 +719,12 @@ swiotlb_alloc_buffer(struct device *dev, size_t size, dma_addr_t *dma_handle, goto out_warn; phys_addr =swiotlb_tbl_map_single (dev, -swiotlb_phys_to_dma (dev, io_tlb_start), +__phys_to_dma (dev, io_tlb_start),0 , size,DMA_FROM_DEVICE ,0 );if (phys_addr ==SWIOTLB_MAP_ERROR ) goto out_warn; - *dma_handle =swiotlb_phys_to_dma (dev, phys_addr); + *dma_handle =__phys_to_dma (dev, phys_addr);if (dma_coherent_ok (dev, *dma_handle, size)) goto out_unmap;
@@ -867,10 +860,10 @@ dma_addr_t swiotlb_map_page(struct device *dev, struct page *page, map =map_single (dev, phys, size, dir, attrs);if (map ==SWIOTLB_MAP_ERROR ) {swiotlb_full (dev, size, dir,1 ); -return swiotlb_phys_to_dma (dev, io_tlb_overflow_buffer); +return __phys_to_dma (dev, io_tlb_overflow_buffer); } - dev_addr =swiotlb_phys_to_dma (dev, map); + dev_addr =__phys_to_dma (dev, map);/* Ensure that the address returned is DMA'ble */ if (dma_capable (dev, dev_addr, size))
@@ -879,7 +872,7 @@ dma_addr_t swiotlb_map_page(struct device *dev, struct page *page, attrs |=DMA_ATTR_SKIP_CPU_SYNC ;swiotlb_tbl_unmap_single (dev, map, size, dir, attrs); -return swiotlb_phys_to_dma (dev, io_tlb_overflow_buffer); +return __phys_to_dma (dev, io_tlb_overflow_buffer); }/*
@@ -1009,7 +1002,7 @@ swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems,sg_dma_len (sgl) =0 ;return 0 ; } - sg->dma_address =swiotlb_phys_to_dma (hwdev, map); + sg->dma_address =__phys_to_dma (hwdev, map); }else sg->dma_address = dev_addr;sg_dma_len (sg) = sg->length;
@@ -1073,7 +1066,7 @@ swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,int swiotlb_dma_mapping_error (struct device *hwdev,dma_addr_t dma_addr) { -return (dma_addr ==swiotlb_phys_to_dma (hwdev, io_tlb_overflow_buffer)); +return (dma_addr ==__phys_to_dma (hwdev, io_tlb_overflow_buffer)); }/*
@@ -1085,7 +1078,7 @@ swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr)int swiotlb_dma_supported (struct device *hwdev,u64 mask) { -return swiotlb_phys_to_dma (hwdev, io_tlb_end -1 ) <= mask; +return __phys_to_dma (hwdev, io_tlb_end -1 ) <= mask; }#ifdef CONFIG_DMA_DIRECT_OPS
Signature · 1 line
2.14.2
Re: [PATCH 11/13] dma-direct: handle the memory encryption bit in common code
Quoted 10 lines · Christoph Hellwig
Give the basic
phys_to_dmaanddma_to_physhelpers a __-prefix and add the memory encryption mask to the non-prefixed versions. Use the __-prefixed versions directly instead of clearing the mask again in various places.With that in place the generic dma-direct routines can be used to allocate non-encrypted bounce buffers, and the x86 SEV case can use the generic swiotlb ops.
Signed-off-by Christoph Hellwig <[email protected]>
So this patch results in my system failing to boot when SME is active. I'm investigating further to see why. I'll follow up with more details as I find them.
Additionally, when running with SME (not SEV), this is forcing all DMA coherent allocations to be decrypted, when that isn't required with SME (as long as the device can perform 48-bit or greater DMA). So it may be worth looking at only doing the decrypted allocations for SEV.
Thanks,
Tom
Quoted 562 lines
--- arch/arm/include/asm/dma-direct.h | 4 +- arch/mips/cavium-octeon/dma-octeon.c | 10 +-- .../include/asm/mach-cavium-octeon/dma-coherence.h | 4 +- .../include/asm/mach-loongson64/dma-coherence.h | 10 +-- arch/mips/loongson64/common/dma-swiotlb.c | 4 +- arch/powerpc/include/asm/dma-direct.h | 4 +- arch/x86/Kconfig | 2 +- arch/x86/include/asm/dma-direct.h | 25 +------- arch/x86/mm/mem_encrypt.c | 73 +--------------------- arch/x86/pci/sta2x11-fixup.c | 6 +- include/linux/dma-direct.h | 21 ++++++- lib/dma-direct.c | 21 +++++-- lib/swiotlb.c | 25 +++----- 13 files changed, 70 insertions(+), 139 deletions(-)arch/arm/include/asm/dma-direct.h +2
-2@@ -2,13 +2,13 @@#ifndef ASM_ARM_DMA_DIRECT_H #define ASM_ARM_DMA_DIRECT_H 1 -static inline dma_addr_t phys_to_dma (struct device *dev,phys_addr_t paddr) +static inline dma_addr_t __phys_to_dma (struct device *dev,phys_addr_t paddr) {unsigned int offset = paddr & ~PAGE_MASK ;return pfn_to_dma (dev,__phys_to_pfn (paddr)) + offset; } -static inline phys_addr_t dma_to_phys (struct device *dev,dma_addr_t dev_addr) +static inline phys_addr_t __dma_to_phys (struct device *dev,dma_addr_t dev_addr) {unsigned int offset = dev_addr & ~PAGE_MASK ;return __pfn_to_phys (dma_to_pfn (dev, dev_addr)) + offset;arch/mips/cavium-octeon/dma-octeon.c +5
-5@@ -10,7 +10,7 @@* IP32 changes by Ilya. * Copyright (C) 2010 Cavium Networks, Inc. */ -#include <linux/dma-mapping.h> +#include <linux/dma-direct.h> #include <linux/scatterlist.h> #include <linux/bootmem.h> #include <linux/export.h> @@ -182,7 +182,7 @@ struct octeon_dma_map_ops {phys_addr_t (*dma_to_phys)(struct device *dev,dma_addr_t daddr); }; -dma_addr_t phys_to_dma (struct device *dev,phys_addr_t paddr) +dma_addr_t __phys_to_dma (struct device *dev,phys_addr_t paddr) { struct octeon_dma_map_ops *ops = container_of(get_dma_ops(dev), struct octeon_dma_map_ops,@@ -190,9 +190,9 @@ dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)return ops->phys_to_dma (dev, paddr); } -EXPORT_SYMBOL (phys_to_dma); +EXPORT_SYMBOL (__phys_to_dma); -phys_addr_t dma_to_phys (struct device *dev,dma_addr_t daddr) +phys_addr_t __dma_to_phys (struct device *dev,dma_addr_t daddr) { struct octeon_dma_map_ops *ops = container_of(get_dma_ops(dev), struct octeon_dma_map_ops,@@ -200,7 +200,7 @@ phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)return ops->dma_to_phys (dev, daddr); } -EXPORT_SYMBOL (dma_to_phys); +EXPORT_SYMBOL (__dma_to_phys);static struct octeon_dma_map_ops octeon_linear_dma_map_ops = { .dma_map_ops = {arch/mips/include/asm/mach-cavium-octeon/dma-coherence.h +2
-2@@ -69,8 +69,8 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)return addr + size -1 <= *dev->dma_mask; } -dma_addr_t phys_to_dma (struct device *dev,phys_addr_t paddr); -phys_addr_t dma_to_phys (struct device *dev,dma_addr_t daddr); +dma_addr_t __phys_to_dma (struct device *dev,phys_addr_t paddr); +phys_addr_t __dma_to_phys (struct device *dev,dma_addr_t daddr);struct dma_map_ops ;extern const struct dma_map_ops *octeon_pci_dma_map_ops;arch/mips/include/asm/mach-loongson64/dma-coherence.h +5
-5@@ -25,13 +25,13 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)return addr + size -1 <= *dev->dma_mask; } -extern dma_addr_t phys_to_dma (struct device *dev,phys_addr_t paddr); -extern phys_addr_t dma_to_phys (struct device *dev,dma_addr_t daddr); +extern dma_addr_t __phys_to_dma (struct device *dev,phys_addr_t paddr); +extern phys_addr_t __dma_to_phys (struct device *dev,dma_addr_t daddr);static inline dma_addr_t plat_map_dma_mem (struct device *dev,void *addr,size_t size) {#ifdef CONFIG_CPU_LOONGSON3 -return phys_to_dma (dev,virt_to_phys (addr)); +return __phys_to_dma (dev,virt_to_phys (addr));#else return virt_to_phys (addr) |0x80000000 ;#endif @@ -41,7 +41,7 @@ static inline dma_addr_t plat_map_dma_mem_page(struct device *dev, struct page *page) {#ifdef CONFIG_CPU_LOONGSON3 -return phys_to_dma (dev,page_to_phys (page)); +return __phys_to_dma (dev,page_to_phys (page));#else return page_to_phys (page) |0x80000000 ;#endif @@ -51,7 +51,7 @@ static inline unsigned long plat_dma_addr_to_phys(struct device *dev, dma_addr_t dma_addr) {#if defined(CONFIG_CPU_LOONGSON3 ) && defined(CONFIG_64BIT ) -return dma_to_phys (dev, dma_addr); +return __dma_to_phys (dev, dma_addr);#elif defined(CONFIG_CPU_LOONGSON2F ) && defined(CONFIG_64BIT )return (dma_addr >0x8fffffff ) ? dma_addr : (dma_addr &0x0fffffff );#else arch/mips/loongson64/common/dma-swiotlb.c +2
-2@@ -63,7 +63,7 @@ static int loongson_dma_supported(struct device *dev, u64 mask)return swiotlb_dma_supported (dev, mask); } -dma_addr_t phys_to_dma (struct device *dev,phys_addr_t paddr) +dma_addr_t __phys_to_dma (struct device *dev,phys_addr_t paddr) {long nid;#ifdef CONFIG_PHYS48_TO_HT40 @@ -75,7 +75,7 @@ dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)return paddr; } -phys_addr_t dma_to_phys (struct device *dev,dma_addr_t daddr) +phys_addr_t __dma_to_phys (struct device *dev,dma_addr_t daddr) {long nid;#ifdef CONFIG_PHYS48_TO_HT40 arch/powerpc/include/asm/dma-direct.h +2
-2@@ -17,12 +17,12 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)return addr + size -1 <= *dev->dma_mask; } -static inline dma_addr_t phys_to_dma (struct device *dev,phys_addr_t paddr) +static inline dma_addr_t __phys_to_dma (struct device *dev,phys_addr_t paddr) {return paddr +get_dma_offset (dev); } -static inline phys_addr_t dma_to_phys (struct device *dev,dma_addr_t daddr) +static inline phys_addr_t __dma_to_phys (struct device *dev,dma_addr_t daddr) {return daddr -get_dma_offset (dev); }arch/x86/Kconfig +1
-1@@ -54,7 +54,6 @@ config X86 select ARCH_HAS_FORTIFY_SOURCE select ARCH_HAS_GCOV_PROFILE_ALL select ARCH_HAS_KCOV if X86_64 - select ARCH_HAS_PHYS_TO_DMA select ARCH_HAS_MEMBARRIER_SYNC_CORE select ARCH_HAS_PMEM_API if X86_64 select ARCH_HAS_REFCOUNT@@ -691,6 +690,7 @@ config X86_SUPPORTS_MEMORY_FAILURE config STA2X11 bool "STA2X11 Companion Chip Support" depends on X86_32_NON_STANDARD && PCI + select ARCH_HAS_PHYS_TO_DMA select X86_DEV_DMA_OPS select X86_DMA_REMAP select SWIOTLBarch/x86/include/asm/dma-direct.h +2
-23@@ -2,29 +2,8 @@#ifndef ASM_X86_DMA_DIRECT_H #define ASM_X86_DMA_DIRECT_H 1 -#include <linux/mem_encrypt.h> - -#ifdef CONFIG_X86_DMA_REMAP /* Platform code defines bridge-specific code */ bool dma_capable (struct device *dev,dma_addr_t addr,size_t size); -dma_addr_t phys_to_dma (struct device *dev,phys_addr_t paddr); -phys_addr_t dma_to_phys (struct device *dev,dma_addr_t daddr); -#else -static inline bool dma_capable (struct device *dev,dma_addr_t addr,size_t size) -{ -if (!dev->dma_mask) -return 0 ; - -return addr + size -1 <= *dev->dma_mask; -} - -static inline dma_addr_t phys_to_dma (struct device *dev,phys_addr_t paddr) -{ -return __sme_set (paddr); -} +dma_addr_t __phys_to_dma (struct device *dev,phys_addr_t paddr); +phys_addr_t __dma_to_phys (struct device *dev,dma_addr_t daddr); -static inline phys_addr_t dma_to_phys (struct device *dev,dma_addr_t daddr) -{ -return __sme_clr (daddr); -} -#endif /* CONFIG_X86_DMA_REMAP */ #endif /* ASM_X86_DMA_DIRECT_H */ arch/x86/mm/mem_encrypt.c +3
-70@@ -200,58 +200,6 @@ void __init sme_early_init(void) swiotlb_force =SWIOTLB_FORCE ; } -static void *sev_alloc (struct device *dev,size_t size,dma_addr_t *dma_handle, -gfp_t gfp,unsigned long attrs) -{ -unsigned int order; -struct page *page; -void *vaddr =NULL ; - - order =get_order (size); - page =alloc_pages_node (dev_to_node (dev), gfp, order); -if (page) { -dma_addr_t addr; - -/* -* Since we will be clearing the encryption bit, check the -* mask with it already cleared. -*/ - addr =__sme_clr (phys_to_dma (dev,page_to_phys (page))); -if ((addr + size) > dev->coherent_dma_mask) { -__free_pages (page,get_order (size)); - }else { - vaddr =page_address (page); - *dma_handle = addr; - } - } - -if (!vaddr) - vaddr =swiotlb_alloc_coherent (dev, size, dma_handle, gfp); - -if (!vaddr) -return NULL ; - -/* Clear the SME encryption bit for DMA use if not swiotlb area */ -if (!is_swiotlb_buffer (dma_to_phys (dev, *dma_handle))) { -set_memory_decrypted ((unsigned long )vaddr,1 << order); -memset (vaddr,0 ,PAGE_SIZE << order); - *dma_handle =__sme_clr (*dma_handle); - } - -return vaddr; -} - -static void sev_free (struct device *dev,size_t size,void *vaddr, -dma_addr_t dma_handle,unsigned long attrs) -{ -/* Set the SME encryption bit for re-use if not swiotlb area */ -if (!is_swiotlb_buffer (dma_to_phys (dev, dma_handle))) -set_memory_encrypted ((unsigned long )vaddr, -1 <<get_order (size)); - -swiotlb_free_coherent (dev, size, vaddr, dma_handle); -} -static void __init __set_clr_pte_enc (pte_t *kpte,int level,bool enc) {pgprot_t old_prot, new_prot;@@ -404,20 +352,6 @@ bool sev_active(void) }EXPORT_SYMBOL (sev_active); -static const struct dma_map_ops sev_dma_ops = { - .alloc = sev_alloc, - .free = sev_free, - .map_page = swiotlb_map_page, - .unmap_page = swiotlb_unmap_page, - .map_sg = swiotlb_map_sg_attrs, - .unmap_sg = swiotlb_unmap_sg_attrs, - .sync_single_for_cpu = swiotlb_sync_single_for_cpu, - .sync_single_for_device = swiotlb_sync_single_for_device, - .sync_sg_for_cpu = swiotlb_sync_sg_for_cpu, - .sync_sg_for_device = swiotlb_sync_sg_for_device, - .mapping_error = swiotlb_dma_mapping_error, -}; -/* Architecture __weak replacement functions */ void __init mem_encrypt_init (void ) {@@ -428,12 +362,11 @@ void __init mem_encrypt_init(void)swiotlb_update_mem_attributes ();/* -* With SEV, DMA operations cannot use encryption. New DMA ops -* are required in order to mark the DMA areas as decrypted or -* to use bounce buffers. +* With SEV, DMA operations cannot use encryption, we need to use +* SWIOTLB to bounce buffer DMA operation. */ if (sev_active ()) - dma_ops = &sev_dma_ops; + dma_ops = &swiotlb_dma_ops;/* * With SEV, we need to unroll the rep string I/O instructions. arch/x86/pci/sta2x11-fixup.c +3
-3@@ -207,11 +207,11 @@ bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) }/** -* phys_to_dma - Return the DMA AMBA address used for this STA2x11 device +* __phys_to_dma - Return the DMA AMBA address used for this STA2x11 device * @dev: device for a PCI device * @paddr: Physical address */ -dma_addr_t phys_to_dma (struct device *dev,phys_addr_t paddr) +dma_addr_t __phys_to_dma (struct device *dev,phys_addr_t paddr) {if (!dev->archdata.is_sta2x11)return paddr;@@ -223,7 +223,7 @@ dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)* @dev: device for a PCI device * @daddr: STA2x11 AMBA DMA address */ -phys_addr_t dma_to_phys (struct device *dev,dma_addr_t daddr) +phys_addr_t __dma_to_phys (struct device *dev,dma_addr_t daddr) {if (!dev->archdata.is_sta2x11)return daddr;include/linux/dma-direct.h +19
-2@@ -3,18 +3,19 @@#define _LINUX_DMA_DIRECT_H 1#include <linux/dma-mapping.h> +#include <linux/mem_encrypt.h> #ifdef CONFIG_ARCH_HAS_PHYS_TO_DMA #include <asm/dma-direct.h> #else -static inline dma_addr_t phys_to_dma (struct device *dev,phys_addr_t paddr) +static inline dma_addr_t __phys_to_dma (struct device *dev,phys_addr_t paddr) {dma_addr_t dev_addr = (dma_addr_t )paddr;return dev_addr - ((dma_addr_t )dev->dma_pfn_offset <<PAGE_SHIFT ); } -static inline phys_addr_t dma_to_phys (struct device *dev,dma_addr_t dev_addr) +static inline phys_addr_t __dma_to_phys (struct device *dev,dma_addr_t dev_addr) {phys_addr_t paddr = (phys_addr_t )dev_addr;@@ -30,6 +31,22 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) }#endif /* !CONFIG_ARCH_HAS_PHYS_TO_DMA */ +/* +* If memory encryption is supported, phys_to_dma will set the memory encryption +* bit in the DMA address, and dma_to_phys will clear it. The raw __phys_to_dma +* and __dma_to_phys versions should only be used on non-encrypted memory for +* special occasions like DMA coherent buffers. +*/ +static inline dma_addr_t phys_to_dma (struct device *dev,phys_addr_t paddr) +{ +return __sme_set (__phys_to_dma (dev, paddr)); +} + +static inline phys_addr_t dma_to_phys (struct device *dev,dma_addr_t daddr) +{ +return __sme_clr (__dma_to_phys (dev, daddr)); +} +#ifdef CONFIG_ARCH_HAS_DMA_MARK_CLEAN void dma_mark_clean (void *addr,size_t size);#else lib/dma-direct.c +15
-6@@ -9,6 +9,7 @@#include <linux/scatterlist.h> #include <linux/dma-contiguous.h> #include <linux/pfn.h> +#include <linux/set_memory.h> #define DIRECT_MAPPING_ERROR 0@@ -35,9 +36,13 @@ check_addr(struct device *dev, dma_addr_t dma_addr, size_t size,return true; } +/* +* Since we will be clearing the encryption bit, check the mask with it already +* cleared. +*/ static bool dma_coherent_ok (struct device *dev,phys_addr_t phys,size_t size) { -return phys_to_dma (dev, phys) + size -1 <= dev->coherent_dma_mask; +return __phys_to_dma (dev, phys) + size -1 <= dev->coherent_dma_mask; } void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,@@ -46,6 +51,7 @@ void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,unsigned int count =PAGE_ALIGN (size) >>PAGE_SHIFT ;int page_order =get_order (size);struct page *page =NULL ; +void *ret;/* GFP_DMA32 and GFP_DMA are no ops without the corresponding zones: */ if (dev->coherent_dma_mask <=DMA_BIT_MASK (ARCH_ZONE_DMA_BITS ))@@ -78,10 +84,11 @@ void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,if (!page)return NULL ; - - *dma_handle =phys_to_dma (dev,page_to_phys (page)); -memset (page_address (page),0 , size); -return page_address (page); + *dma_handle =__phys_to_dma (dev,page_to_phys (page)); + ret =page_address (page); +set_memory_decrypted ((unsigned long )ret, page_order); +memset (ret,0 , size); +return ret; }/* @@ -92,9 +99,11 @@ void dma_direct_free(struct device *dev, size_t size, void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs) {unsigned int count =PAGE_ALIGN (size) >>PAGE_SHIFT ; +unsigned int page_order =get_order (size); +set_memory_encrypted ((unsigned long )cpu_addr,1 << page_order);if (!dma_release_from_contiguous (dev,virt_to_page (cpu_addr), count)) -free_pages ((unsigned long )cpu_addr,get_order (size)); +free_pages ((unsigned long )cpu_addr, page_order); } static dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,lib/swiotlb.c +9
-16@@ -158,13 +158,6 @@ unsigned long swiotlb_size_or_default(void)void __weak swiotlb_set_mem_attributes (void *vaddr,unsigned long size) { } -/* For swiotlb, clear memory encryption mask from dma addresses */ -static dma_addr_t swiotlb_phys_to_dma (struct device *hwdev, -phys_addr_t address) -{ -return __sme_clr (phys_to_dma (hwdev, address)); -} -/* Note that this doesn't work with highmem page */ static dma_addr_t swiotlb_virt_to_bus (struct device *hwdev,volatile void *address)@@ -622,7 +615,7 @@ map_single(struct device *hwdev, phys_addr_t phys, size_t size,return SWIOTLB_MAP_ERROR ; } - start_dma_addr =swiotlb_phys_to_dma (hwdev, io_tlb_start); + start_dma_addr =__phys_to_dma (hwdev, io_tlb_start);return swiotlb_tbl_map_single (hwdev, start_dma_addr, phys, size, dir, attrs); }@@ -726,12 +719,12 @@ swiotlb_alloc_buffer(struct device *dev, size_t size, dma_addr_t *dma_handle, goto out_warn; phys_addr =swiotlb_tbl_map_single (dev, -swiotlb_phys_to_dma (dev, io_tlb_start), +__phys_to_dma (dev, io_tlb_start),0 , size,DMA_FROM_DEVICE ,0 );if (phys_addr ==SWIOTLB_MAP_ERROR ) goto out_warn; - *dma_handle =swiotlb_phys_to_dma (dev, phys_addr); + *dma_handle =__phys_to_dma (dev, phys_addr);if (dma_coherent_ok (dev, *dma_handle, size)) goto out_unmap;@@ -867,10 +860,10 @@ dma_addr_t swiotlb_map_page(struct device *dev, struct page *page, map =map_single (dev, phys, size, dir, attrs);if (map ==SWIOTLB_MAP_ERROR ) {swiotlb_full (dev, size, dir,1 ); -return swiotlb_phys_to_dma (dev, io_tlb_overflow_buffer); +return __phys_to_dma (dev, io_tlb_overflow_buffer); } - dev_addr =swiotlb_phys_to_dma (dev, map); + dev_addr =__phys_to_dma (dev, map);/* Ensure that the address returned is DMA'ble */ if (dma_capable (dev, dev_addr, size))@@ -879,7 +872,7 @@ dma_addr_t swiotlb_map_page(struct device *dev, struct page *page, attrs |=DMA_ATTR_SKIP_CPU_SYNC ;swiotlb_tbl_unmap_single (dev, map, size, dir, attrs); -return swiotlb_phys_to_dma (dev, io_tlb_overflow_buffer); +return __phys_to_dma (dev, io_tlb_overflow_buffer); }/* @@ -1009,7 +1002,7 @@ swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems,sg_dma_len (sgl) =0 ;return 0 ; } - sg->dma_address =swiotlb_phys_to_dma (hwdev, map); + sg->dma_address =__phys_to_dma (hwdev, map); }else sg->dma_address = dev_addr;sg_dma_len (sg) = sg->length;@@ -1073,7 +1066,7 @@ swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,int swiotlb_dma_mapping_error (struct device *hwdev,dma_addr_t dma_addr) { -return (dma_addr ==swiotlb_phys_to_dma (hwdev, io_tlb_overflow_buffer)); +return (dma_addr ==__phys_to_dma (hwdev, io_tlb_overflow_buffer)); }/* @@ -1085,7 +1078,7 @@ swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr)int swiotlb_dma_supported (struct device *hwdev,u64 mask) { -return swiotlb_phys_to_dma (hwdev, io_tlb_end -1 ) <= mask; +return __phys_to_dma (hwdev, io_tlb_end -1 ) <= mask; }#ifdef CONFIG_DMA_DIRECT_OPS
Re: [PATCH 11/13] dma-direct: handle the memory encryption bit in common code
Quoted 10 lines · Tom Lendacky
Quoted 9 lines · Christoph Hellwig
Give the basic
phys_to_dmaanddma_to_physhelpers a __-prefix and add the memory encryption mask to the non-prefixed versions. Use the __-prefixed versions directly instead of clearing the mask again in various places.With that in place the generic dma-direct routines can be used to allocate non-encrypted bounce buffers, and the x86 SEV case can use the generic swiotlb ops.
Quoted 5 lines
Quoted 1 line
Signed-off-by Christoph Hellwig <[email protected]>So this patch results in my system failing to boot when SME is active. I'm investigating further to see why. I'll follow up with more details as I find them.
Ok, I found one issue that allows this to work when the IOMMU isn't enabled (see below).
But the bigger issue is when the IOMMU is enabled. The IOMMU code uses a common mapping routine to create the I/O page tables. This routine assumes that all memory being mapped is encrypted and therefore sets the encryption bit in the I/O page tables. With this patch, the call to dma_alloc_direct() now returns un-encrypted memory which results in an encryption mis-match. I think keeping dma_alloc_direct() as it was prior to this patch is the way to go. It allows SME DMA allocations to remain encrypted and avoids added complexity in the amd_iommu.c file. This would mean that SEV would still have special DMA operations (so that the alloc/free can change the memory to un-encrypted).
What do you think?
Quoted 25 lines
Additionally, when running with SME (not SEV), this is forcing all DMA coherent allocations to be decrypted, when that isn't required with SME (as long as the device can perform 48-bit or greater DMA). So it may be worth looking at only doing the decrypted allocations for SEV.
Thanks,
TomQuoted 16 lines
--- arch/arm/include/asm/dma-direct.h | 4 +- arch/mips/cavium-octeon/dma-octeon.c | 10 +-- .../include/asm/mach-cavium-octeon/dma-coherence.h | 4 +- .../include/asm/mach-loongson64/dma-coherence.h | 10 +-- arch/mips/loongson64/common/dma-swiotlb.c | 4 +- arch/powerpc/include/asm/dma-direct.h | 4 +- arch/x86/Kconfig | 2 +- arch/x86/include/asm/dma-direct.h | 25 +------- arch/x86/mm/mem_encrypt.c | 73 +--------------------- arch/x86/pci/sta2x11-fixup.c | 6 +- include/linux/dma-direct.h | 21 ++++++- lib/dma-direct.c | 21 +++++-- lib/swiotlb.c | 25 +++----- 13 files changed, 70 insertions(+), 139 deletions(-)
< ... SNIP ... >
Quoted 37 lines
Quoted 37 lines
lib/dma-direct.c +7
-1@@ -9,6 +9,7 @@#include <linux/scatterlist.h> #include <linux/dma-contiguous.h> #include <linux/pfn.h> +#include <linux/set_memory.h> #define DIRECT_MAPPING_ERROR 0@@ -35,9 +36,13 @@ check_addr(struct device *dev, dma_addr_t dma_addr, size_t size,return true; } +/* +* Since we will be clearing the encryption bit, check the mask with it already +* cleared. +*/ static bool dma_coherent_ok (struct device *dev,phys_addr_t phys,size_t size) { -return phys_to_dma (dev, phys) + size -1 <= dev->coherent_dma_mask; +return __phys_to_dma (dev, phys) + size -1 <= dev->coherent_dma_mask; } void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,@@ -46,6 +51,7 @@ void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,unsigned int count =PAGE_ALIGN (size) >>PAGE_SHIFT ;int page_order =get_order (size);struct page *page =NULL ; +void *ret;/* GFP_DMA32 and GFP_DMA are no ops without the corresponding zones: */ if (dev->coherent_dma_mask <=DMA_BIT_MASK (ARCH_ZONE_DMA_BITS ))@@ -78,10 +84,11 @@ void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
Quoted 9 lines
lib/dma-direct.c void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, · line 89
Quoted 9 lines
lib/dma-direct.c +3
-4if (!page)return NULL ; - - *dma_handle =phys_to_dma (dev,page_to_phys (page)); -memset (page_address (page),0 , size); -return page_address (page); + *dma_handle =__phys_to_dma (dev,page_to_phys (page)); + ret =page_address (page); +set_memory_decrypted ((unsigned long )ret, page_order);
The second parameter should be 1 << page_order to get the number of pages.
Thanks,
Tom
Quoted 110 lines
Quoted 110 lines
lib/dma-direct.c +5
-1+memset (ret,0 , size); +return ret; }/* @@ -92,9 +99,11 @@ void dma_direct_free(struct device *dev, size_t size, void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs) {unsigned int count =PAGE_ALIGN (size) >>PAGE_SHIFT ; +unsigned int page_order =get_order (size); +set_memory_encrypted ((unsigned long )cpu_addr,1 << page_order);if (!dma_release_from_contiguous (dev,virt_to_page (cpu_addr), count)) -free_pages ((unsigned long )cpu_addr,get_order (size)); +free_pages ((unsigned long )cpu_addr, page_order); } static dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,lib/swiotlb.c +9
-16@@ -158,13 +158,6 @@ unsigned long swiotlb_size_or_default(void)void __weak swiotlb_set_mem_attributes (void *vaddr,unsigned long size) { } -/* For swiotlb, clear memory encryption mask from dma addresses */ -static dma_addr_t swiotlb_phys_to_dma (struct device *hwdev, -phys_addr_t address) -{ -return __sme_clr (phys_to_dma (hwdev, address)); -} -/* Note that this doesn't work with highmem page */ static dma_addr_t swiotlb_virt_to_bus (struct device *hwdev,volatile void *address)@@ -622,7 +615,7 @@ map_single(struct device *hwdev, phys_addr_t phys, size_t size,return SWIOTLB_MAP_ERROR ; } - start_dma_addr =swiotlb_phys_to_dma (hwdev, io_tlb_start); + start_dma_addr =__phys_to_dma (hwdev, io_tlb_start);return swiotlb_tbl_map_single (hwdev, start_dma_addr, phys, size, dir, attrs); }@@ -726,12 +719,12 @@ swiotlb_alloc_buffer(struct device *dev, size_t size, dma_addr_t *dma_handle, goto out_warn; phys_addr =swiotlb_tbl_map_single (dev, -swiotlb_phys_to_dma (dev, io_tlb_start), +__phys_to_dma (dev, io_tlb_start),0 , size,DMA_FROM_DEVICE ,0 );if (phys_addr ==SWIOTLB_MAP_ERROR ) goto out_warn; - *dma_handle =swiotlb_phys_to_dma (dev, phys_addr); + *dma_handle =__phys_to_dma (dev, phys_addr);if (dma_coherent_ok (dev, *dma_handle, size)) goto out_unmap;@@ -867,10 +860,10 @@ dma_addr_t swiotlb_map_page(struct device *dev, struct page *page, map =map_single (dev, phys, size, dir, attrs);if (map ==SWIOTLB_MAP_ERROR ) {swiotlb_full (dev, size, dir,1 ); -return swiotlb_phys_to_dma (dev, io_tlb_overflow_buffer); +return __phys_to_dma (dev, io_tlb_overflow_buffer); } - dev_addr =swiotlb_phys_to_dma (dev, map); + dev_addr =__phys_to_dma (dev, map);/* Ensure that the address returned is DMA'ble */ if (dma_capable (dev, dev_addr, size))@@ -879,7 +872,7 @@ dma_addr_t swiotlb_map_page(struct device *dev, struct page *page, attrs |=DMA_ATTR_SKIP_CPU_SYNC ;swiotlb_tbl_unmap_single (dev, map, size, dir, attrs); -return swiotlb_phys_to_dma (dev, io_tlb_overflow_buffer); +return __phys_to_dma (dev, io_tlb_overflow_buffer); }/* @@ -1009,7 +1002,7 @@ swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems,sg_dma_len (sgl) =0 ;return 0 ; } - sg->dma_address =swiotlb_phys_to_dma (hwdev, map); + sg->dma_address =__phys_to_dma (hwdev, map); }else sg->dma_address = dev_addr;sg_dma_len (sg) = sg->length;@@ -1073,7 +1066,7 @@ swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,int swiotlb_dma_mapping_error (struct device *hwdev,dma_addr_t dma_addr) { -return (dma_addr ==swiotlb_phys_to_dma (hwdev, io_tlb_overflow_buffer)); +return (dma_addr ==__phys_to_dma (hwdev, io_tlb_overflow_buffer)); }/* @@ -1085,7 +1078,7 @@ swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr)int swiotlb_dma_supported (struct device *hwdev,u64 mask) { -return swiotlb_phys_to_dma (hwdev, io_tlb_end -1 ) <= mask; +return __phys_to_dma (hwdev, io_tlb_end -1 ) <= mask; }#ifdef CONFIG_DMA_DIRECT_OPS
Re: [PATCH 11/13] dma-direct: handle the memory encryption bit in common code
Quoted 2 lines · Tom Lendacky
Ok, I found one issue that allows this to work when the IOMMU isn't enabled (see below).
Thanks, folded!
Quoted 11 lines
But the bigger issue is when the IOMMU is enabled. The IOMMU code uses a common mapping routine to create the I/O page tables. This routine assumes that all memory being mapped is encrypted and therefore sets the encryption bit in the I/O page tables. With this patch, the call to
dma_alloc_direct()now returns un-encrypted memory which results in an encryption mis-match. I think keepingdma_alloc_direct()as it was prior to this patch is the way to go. It allows SME DMA allocations to remain encrypted and avoids added complexity in theamd_iommu.cfile. This would mean that SEV would still have special DMA operations (so that the alloc/free can change the memory to un-encrypted).
Quoted 1 line
What do you think?
In terms of logic you are right. I still don't like keeping a just slightly tweaked version of dma_alloc_direct around just for this, it will be perpetually out of sync in terms of features and bug fixes.
What do you think about this version that does the decision at runtime:
http://git.infradead.org/users/hch/misc.git/commitdiff/b89f24dc856595dc7610d672bf077195ab0dabf4
The full tree is available here for testing:
git://git.infradead.org/users/hch/misc.git dma-direct-x86
Re: [PATCH 11/13] dma-direct: handle the memory encryption bit in common code
Quoted 23 lines · Christoph Hellwig
Quoted 2 lines · Tom Lendacky
Ok, I found one issue that allows this to work when the IOMMU isn't enabled (see below).
Thanks, folded!
Quoted 11 lines
But the bigger issue is when the IOMMU is enabled. The IOMMU code uses a common mapping routine to create the I/O page tables. This routine assumes that all memory being mapped is encrypted and therefore sets the encryption bit in the I/O page tables. With this patch, the call to
dma_alloc_direct()now returns un-encrypted memory which results in an encryption mis-match. I think keepingdma_alloc_direct()as it was prior to this patch is the way to go. It allows SME DMA allocations to remain encrypted and avoids added complexity in theamd_iommu.cfile. This would mean that SEV would still have special DMA operations (so that the alloc/free can change the memory to un-encrypted).Quoted 1 line
What do you think?
In terms of logic you are right. I still don't like keeping a just slightly tweaked version of
dma_alloc_directaround just for this, it will be perpetually out of sync in terms of features and bug fixes.
Quoted 8 lines
What do you think about this version that does the decision at runtime:
http://git.infradead.org/users/hch/misc.git/commitdiff/b89f24dc856595dc7610d672bf077195ab0dabf4The full tree is available here for testing:
git://git.infradead.org/users/hch/misc.gitdma-direct-x86
Thanks for the pointer to the tree. I did find one bug in the allocation routine, that once fixed (see below), worked with SME for IOMMU on and off and worked with an SEV guest.
I understand the comment about using sev_active() in the dma-direct code, maybe we can up with something later to address that.
Thanks,
Tom
lib/dma-direct.c +0 -0
@@ -82,10 +82,12 @@ void *dma_direct_alloc(struct device *dev, size_t
size, dma_addr_t *dma_handle,
if (!page)
return NULL;
- *dma_handle = __phys_to_dma(dev, page_to_phys(page));
+ *dma_handle = phys_to_dma(dev, page_to_phys(page));
ret = page_address(page);
- if (sev_active())
+ if (sev_active()) {
+ *dma_handle = __phys_to_dma(dev, page_to_phys(page));
set_memory_decrypted((unsigned long)ret, 1 << page_order);
+ }
memset(ret, 0, size);
return ret;
}
Re: [PATCH 11/13] dma-direct: handle the memory encryption bit in common code
Can you test and review the V3 of the series I just sent out? We reall should get it into linux-next ASAP.
Re: [PATCH 11/13] dma-direct: handle the memory encryption bit in common code
Quoted 2 lines · Christoph Hellwig
Can you test and review the V3 of the series I just sent out? We reall should get it into linux-next ASAP.
Can do. I'll get back to you on V3 thread with the results.
Thanks,
Tom
Quoted 1 line
PATCH 12/13 swiotlb: remove swiotlb_set_mem_attributes
In reply to Christoph HellwigNow that set_memory_decrypted is always available we can just call it directly.
--- arch/x86/include/asm/mem_encrypt.h | 2 -- arch/x86/mm/mem_encrypt.c | 9 --------- lib/swiotlb.c | 12 ++++++------ 3 files changed, 6 insertions(+), 17 deletions(-)
arch/x86/include/asm/mem_encrypt.h +0 -2
@@ -48,8 +48,6 @@ int __init early_set_memory_encrypted(unsigned long vaddr, unsigned long size);/* Architecture __weak replacement functions */ void __init mem_encrypt_init (void ); -void swiotlb_set_mem_attributes (void *vaddr,unsigned long size); -bool sme_active (void );bool sev_active (void );
arch/x86/mm/mem_encrypt.c +0 -9
@@ -379,15 +379,6 @@ void __init mem_encrypt_init(void) : "Secure Memory Encryption (SME)"); } -void swiotlb_set_mem_attributes (void *vaddr,unsigned long size) -{ -WARN (PAGE_ALIGN (size) != size, -"size is not page-aligned (%#lx)\n" , size); - -/* Make the SWIOTLB buffer area decrypted */ -set_memory_decrypted ((unsigned long )vaddr, size >>PAGE_SHIFT ); -} -struct sme_populate_pgd_data {void *pgtable_area;pgd_t *pgd;
lib/swiotlb.c +6 -6
@@ -31,6 +31,7 @@#include <linux/gfp.h> #include <linux/scatterlist.h> #include <linux/mem_encrypt.h> +#include <linux/set_memory.h> #include <asm/io.h> #include <asm/dma.h>
@@ -156,8 +157,6 @@ unsigned long swiotlb_size_or_default(void)return size ? size : (IO_TLB_DEFAULT_SIZE ); } -void __weak swiotlb_set_mem_attributes (void *vaddr,unsigned long size) { } -/* Note that this doesn't work with highmem page */ static dma_addr_t swiotlb_virt_to_bus (struct device *hwdev,volatile void *address)
@@ -202,12 +201,12 @@ void __init swiotlb_update_mem_attributes(void) vaddr =phys_to_virt (io_tlb_start); bytes =PAGE_ALIGN (io_tlb_nslabs <<IO_TLB_SHIFT ); -swiotlb_set_mem_attributes (vaddr, bytes); +set_memory_decrypted ((unsigned long )vaddr, bytes >>PAGE_SHIFT );memset (vaddr,0 , bytes); vaddr =phys_to_virt (io_tlb_overflow_buffer); bytes =PAGE_ALIGN (io_tlb_overflow); -swiotlb_set_mem_attributes (vaddr, bytes); +set_memory_decrypted ((unsigned long )vaddr, bytes >>PAGE_SHIFT );memset (vaddr,0 , bytes); }
@@ -348,7 +347,7 @@ swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs) io_tlb_start =virt_to_phys (tlb); io_tlb_end = io_tlb_start + bytes; -swiotlb_set_mem_attributes (tlb, bytes); +set_memory_decrypted ((unsigned long )tlb, bytes >>PAGE_SHIFT );memset (tlb,0 , bytes);/*
@@ -359,7 +358,8 @@ swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs)if (!v_overflow_buffer) goto cleanup2; -swiotlb_set_mem_attributes (v_overflow_buffer, io_tlb_overflow); +set_memory_decrypted ((unsigned long )v_overflow_buffer, + io_tlb_overflow >>PAGE_SHIFT );memset (v_overflow_buffer,0 , io_tlb_overflow); io_tlb_overflow_buffer =virt_to_phys (v_overflow_buffer);
Signature · 1 line
2.14.2
PATCH 13/13 swiotlb: remove swiotlb_{alloc,free}_coherent
In reply to Christoph HellwigUnused now that everyone uses swiotlb_{alloc,free}.
--- include/linux/swiotlb.h | 8 -------- lib/swiotlb.c | 38 -------------------------------------- 2 files changed, 46 deletions(-)
include/linux/swiotlb.h +0 -8
@@ -72,14 +72,6 @@ void *swiotlb_alloc(struct device *hwdev, size_t size, dma_addr_t *dma_handle,void swiotlb_free (struct device *dev,size_t size,void *vaddr,dma_addr_t dma_addr,unsigned long attrs); -extern void -*swiotlb_alloc_coherent (struct device *hwdev,size_t size, -dma_addr_t *dma_handle,gfp_t flags); - -extern void -swiotlb_free_coherent (struct device *hwdev,size_t size, -void *vaddr,dma_addr_t dma_handle); - extern dma_addr_t swiotlb_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir,
lib/swiotlb.c +0 -38
@@ -157,13 +157,6 @@ unsigned long swiotlb_size_or_default(void)return size ? size : (IO_TLB_DEFAULT_SIZE ); } -/* Note that this doesn't work with highmem page */ -static dma_addr_t swiotlb_virt_to_bus (struct device *hwdev, -volatile void *address) -{ -return phys_to_dma (hwdev,virt_to_phys (address)); -} -static bool no_iotlb_memory;void swiotlb_print_info (void )
@@ -752,28 +745,6 @@ swiotlb_alloc_buffer(struct device *dev, size_t size, dma_addr_t *dma_handle,return NULL ; } -void * -swiotlb_alloc_coherent (struct device *hwdev,size_t size, -dma_addr_t *dma_handle,gfp_t flags) -{ -int order =get_order (size); -unsigned long attrs = (flags & __GFP_NOWARN) ?DMA_ATTR_NO_WARN :0 ; -void *ret; - - ret = (void *)__get_free_pages (flags, order); -if (ret) { - *dma_handle =swiotlb_virt_to_bus (hwdev, ret); -if (dma_coherent_ok (hwdev, *dma_handle, size)) { -memset (ret,0 , size); -return ret; - } -free_pages ((unsigned long )ret, order); - } - -return swiotlb_alloc_buffer (hwdev, size, dma_handle, attrs); -} -EXPORT_SYMBOL (swiotlb_alloc_coherent); -static bool swiotlb_free_buffer (struct device *dev,size_t size,dma_addr_t dma_addr) {
@@ -793,15 +764,6 @@ static bool swiotlb_free_buffer(struct device *dev, size_t size,return true; } -void -swiotlb_free_coherent (struct device *hwdev,size_t size,void *vaddr, -dma_addr_t dev_addr) -{ -if (!swiotlb_free_buffer (hwdev, size, dev_addr)) -free_pages ((unsigned long )vaddr,get_order (size)); -} -EXPORT_SYMBOL (swiotlb_free_coherent); -static void swiotlb_full (struct device *dev,size_t size,enum dma_data_direction dir,int do_panic)
Signature · 1 line
2.14.2