iommu/tegra-smmu: Add missing locks around mapping operations
Subsystems (3)
PATCH 1/1 iommu/tegra-smmu: Add missing locks around mapping operations
The mapping operations of the Tegra SMMU driver are subjected to a race condition issues because SMMU Address Space isn't allocated and freed atomically, while it should be. This patch makes the mapping operations atomic, it fixes an accidentally released Host1x Address Space problem which happens while running multiple graphics tests in parallel on Tegra30, i.e. by having multiple threads racing with each other in the Host1x's submission and completion code paths, performing IOVA mappings and unmappings in parallel.
--- drivers/iommu/tegra-smmu.c | 43 +++++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 5 deletions(-)
drivers/iommu/tegra-smmu.c +38 -5
@@ -12,6 +12,7 @@#include <linux/of_device.h> #include <linux/platform_device.h> #include <linux/slab.h> +#include <linux/spinlock.h> #include <linux/dma-mapping.h> #include <soc/tegra/ahb.h>
@@ -49,6 +50,7 @@ struct tegra_smmu_as {struct iommu_domain domain;struct tegra_smmu *smmu;unsigned int use_count; +spinlock_t lock;u32 *count;struct page **pts;struct page *pd;
@@ -308,6 +310,8 @@ static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type)return NULL ; } +spin_lock_init (&as->lock); +/* setup aperture */ as->domain.geometry.aperture_start =0 ; as->domain.geometry.aperture_end =0xffffffff ;
@@ -578,7 +582,7 @@ static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova,struct page *page;dma_addr_t dma; - page =alloc_page (GFP_KERNEL | __GFP_DMA | __GFP_ZERO); + page =alloc_page (GFP_ATOMIC | __GFP_DMA | __GFP_ZERO);if (!page)return NULL ;
@@ -655,8 +659,9 @@ static void tegra_smmu_set_pte(struct tegra_smmu_as *as, unsigned long iova,smmu_flush (smmu); } -static int tegra_smmu_map (struct iommu_domain *domain,unsigned long iova, -phys_addr_t paddr,size_t size,int prot,gfp_t gfp) +static int +tegra_smmu_map_locked (struct iommu_domain *domain,unsigned long iova, +phys_addr_t paddr,size_t size,int prot,gfp_t gfp) {struct tegra_smmu_as *as =to_smmu_as (domain);dma_addr_t pte_dma;
@@ -685,8 +690,9 @@ static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova,return 0 ; } -static size_t tegra_smmu_unmap (struct iommu_domain *domain,unsigned long iova, -size_t size,struct iommu_iotlb_gather *gather) +static size_t +tegra_smmu_unmap_locked (struct iommu_domain *domain,unsigned long iova, +size_t size,struct iommu_iotlb_gather *gather) {struct tegra_smmu_as *as =to_smmu_as (domain);dma_addr_t pte_dma;
@@ -702,6 +708,33 @@ static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova,return size; } +static int tegra_smmu_map (struct iommu_domain *domain,unsigned long iova, +phys_addr_t paddr,size_t size,int prot,gfp_t gfp) +{ +struct tegra_smmu_as *as =to_smmu_as (domain); +unsigned long flags; +int ret; + +spin_lock_irqsave (&as->lock, flags); + ret =tegra_smmu_map_locked (domain, iova, paddr, size, prot, gfp); +spin_unlock_irqrestore (&as->lock, flags); + +return ret; +} + +static size_t tegra_smmu_unmap (struct iommu_domain *domain,unsigned long iova, +size_t size,struct iommu_iotlb_gather *gather) +{ +struct tegra_smmu_as *as =to_smmu_as (domain); +unsigned long flags; + +spin_lock_irqsave (&as->lock, flags); + size =tegra_smmu_unmap_locked (domain, iova, size, gather); +spin_unlock_irqrestore (&as->lock, flags); + +return size; +} +static phys_addr_t tegra_smmu_iova_to_phys (struct iommu_domain *domain,dma_addr_t iova) {
Signature · 1 line
2.26.0
Quoted 45 lines · Dmitry Osipenko
The mapping operations of the Tegra SMMU driver are subjected to a race condition issues because SMMU Address Space isn't allocated and freed atomically, while it should be. This patch makes the mapping operations atomic, it fixes an accidentally released Host1x Address Space problem which happens while running multiple graphics tests in parallel on Tegra30, i.e. by having multiple threads racing with each other in the Host1x's submission and completion code paths, performing IOVA mappings and unmappings in parallel.
Cc <[email protected]>Signed-off-by Dmitry Osipenko <[email protected]>--- drivers/iommu/tegra-smmu.c | 43 +++++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 5 deletions(-)drivers/iommu/tegra-smmu.c +4
-0@@ -12,6 +12,7 @@#include <linux/of_device.h> #include <linux/platform_device.h> #include <linux/slab.h> +#include <linux/spinlock.h> #include <linux/dma-mapping.h> #include <soc/tegra/ahb.h> @@ -49,6 +50,7 @@ struct tegra_smmu_as {struct iommu_domain domain;struct tegra_smmu *smmu;unsigned int use_count; +spinlock_t lock;u32 *count;struct page **pts;struct page *pd;@@ -308,6 +310,8 @@ static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type)return NULL ; } +spin_lock_init (&as->lock); +/* setup aperture */ as->domain.geometry.aperture_start =0 ; as->domain.geometry.aperture_end =0xffffffff ;
drivers/iommu/tegra-smmu.c static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, · line 585
Quoted 5 lines
drivers/iommu/tegra-smmu.c +1
-1@@ -578,7 +582,7 @@ static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova,struct page *page;dma_addr_t dma; - page =alloc_page (GFP_KERNEL | __GFP_DMA | __GFP_ZERO); + page =alloc_page (GFP_ATOMIC | __GFP_DMA | __GFP_ZERO);
I'm not sure this is a good idea. My recollection is that GFP_ATOMIC will allocate from a special reserved region of memory, which may be easily exhausted.
Is there any reason why we need the spinlock? Can't we use a mutex instead?
drivers/iommu/tegra-smmu.c static void tegra_smmu_set_pte(struct tegra_smmu_as *as, unsigned long iova, · line 664
Quoted 12 lines
drivers/iommu/tegra-smmu.c +3
-2if (!page)return NULL ;@@ -655,8 +659,9 @@ static void tegra_smmu_set_pte(struct tegra_smmu_as *as, unsigned long iova,smmu_flush (smmu); } -static int tegra_smmu_map (struct iommu_domain *domain,unsigned long iova, -phys_addr_t paddr,size_t size,int prot,gfp_t gfp) +static int +tegra_smmu_map_locked (struct iommu_domain *domain,unsigned long iova, +phys_addr_t paddr,size_t size,int prot,gfp_t gfp)
I think it's more typical to use the _unlocked suffix for functions that don't take a lock themselves.
Quoted 35 lines
drivers/iommu/tegra-smmu.c +19
-2{struct tegra_smmu_as *as =to_smmu_as (domain);dma_addr_t pte_dma;@@ -685,8 +690,9 @@ static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova,return 0 ; } -static size_t tegra_smmu_unmap (struct iommu_domain *domain,unsigned long iova, -size_t size,struct iommu_iotlb_gather *gather) +static size_t +tegra_smmu_unmap_locked (struct iommu_domain *domain,unsigned long iova, +size_t size,struct iommu_iotlb_gather *gather) {struct tegra_smmu_as *as =to_smmu_as (domain);dma_addr_t pte_dma;@@ -702,6 +708,33 @@ static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova,return size; } +static int tegra_smmu_map (struct iommu_domain *domain,unsigned long iova, +phys_addr_t paddr,size_t size,int prot,gfp_t gfp) +{ +struct tegra_smmu_as *as =to_smmu_as (domain); +unsigned long flags; +int ret; + +spin_lock_irqsave (&as->lock, flags); + ret =tegra_smmu_map_locked (domain, iova, paddr, size, prot, gfp); +spin_unlock_irqrestore (&as->lock, flags); + +return ret; +} + +static size_t tegra_smmu_unmap (struct iommu_domain *domain,unsigned long iova, +size_t size,struct iommu_iotlb_gather *gather)
drivers/iommu/tegra-smmu.c static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova, · line 736
Quoted 10 lines
drivers/iommu/tegra-smmu.c +10
-0+{ +struct tegra_smmu_as *as =to_smmu_as (domain); +unsigned long flags; + +spin_lock_irqsave (&as->lock, flags); + size =tegra_smmu_unmap_locked (domain, iova, size, gather); +spin_unlock_irqrestore (&as->lock, flags); + +return size; +}
Why the extra functions here? We never call locked vs. unlocked variants in the driver and the IOMMU framework only has a single callback, so I think the locking can just move into the main implementation.
Thierry
25.05.2020 11:35, Thierry Reding пишет:
Quoted 49 lines · Dmitry Osipenko
Quoted 48 lines
The mapping operations of the Tegra SMMU driver are subjected to a race condition issues because SMMU Address Space isn't allocated and freed atomically, while it should be. This patch makes the mapping operations atomic, it fixes an accidentally released Host1x Address Space problem which happens while running multiple graphics tests in parallel on Tegra30, i.e. by having multiple threads racing with each other in the Host1x's submission and completion code paths, performing IOVA mappings and unmappings in parallel.
Cc <[email protected]>Signed-off-by Dmitry Osipenko <[email protected]>--- drivers/iommu/tegra-smmu.c | 43 +++++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 5 deletions(-)drivers/iommu/tegra-smmu.c +4
-0@@ -12,6 +12,7 @@#include <linux/of_device.h> #include <linux/platform_device.h> #include <linux/slab.h> +#include <linux/spinlock.h> #include <linux/dma-mapping.h> #include <soc/tegra/ahb.h> @@ -49,6 +50,7 @@ struct tegra_smmu_as {struct iommu_domain domain;struct tegra_smmu *smmu;unsigned int use_count; +spinlock_t lock;u32 *count;struct page **pts;struct page *pd;@@ -308,6 +310,8 @@ static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type)return NULL ; } +spin_lock_init (&as->lock); +/* setup aperture */ as->domain.geometry.aperture_start =0 ; as->domain.geometry.aperture_end =0xffffffff ;@@ -578,7 +582,7 @@ static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova,struct page *page;dma_addr_t dma;
Quoted 6 lines
drivers/iommu/tegra-smmu.c static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, · line 585
Quoted 2 lines
drivers/iommu/tegra-smmu.c +1
-1- page =alloc_page (GFP_KERNEL | __GFP_DMA | __GFP_ZERO); + page =alloc_page (GFP_ATOMIC | __GFP_DMA | __GFP_ZERO);I'm not sure this is a good idea. My recollection is that
GFP_ATOMICwill allocate from a special reserved region of memory, which may be easily exhausted.
So far I haven't noticed any problems. Will be great if you could provide more details about the pool size and how this exhaustion problem could be reproduced in practice.
Quoted 2 lines
Is there any reason why we need the spinlock? Can't we use a mutex instead?
This is what other IOMMU drivers do. I guess mutex might be too expensive, it may create a noticeable contention which you don't want to have in a case of a GPU submission code path.
I also suspect that drivers of other platforms are using IOMMU API in interrupt context, although today this is not needed for Tegra.
Quoted 7 lines
Quoted 7 lines
drivers/iommu/tegra-smmu.c +0
-0if (!page)return NULL ;@@ -655,8 +659,9 @@ static void tegra_smmu_set_pte(struct tegra_smmu_as *as, unsigned long iova,smmu_flush (smmu); }
Quoted 8 lines
drivers/iommu/tegra-smmu.c static void tegra_smmu_set_pte(struct tegra_smmu_as *as, unsigned long iova, · line 664
Quoted 5 lines
drivers/iommu/tegra-smmu.c +3
-2-static int tegra_smmu_map (struct iommu_domain *domain,unsigned long iova, -phys_addr_t paddr,size_t size,int prot,gfp_t gfp) +static int +tegra_smmu_map_locked (struct iommu_domain *domain,unsigned long iova, +phys_addr_t paddr,size_t size,int prot,gfp_t gfp)I think it's more typical to use the
_unlockedsuffix for functions that don't take a lock themselves.
Personally I can't feel the difference. Both variants are good to me. I can replace the literal postfix with a __tegra_smmu prefix, similarly to what we have in the GART driver, to avoid bikeshedding.
Quoted 41 lines
Quoted 41 lines
drivers/iommu/tegra-smmu.c +25
-2{struct tegra_smmu_as *as =to_smmu_as (domain);dma_addr_t pte_dma;@@ -685,8 +690,9 @@ static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova,return 0 ; } -static size_t tegra_smmu_unmap (struct iommu_domain *domain,unsigned long iova, -size_t size,struct iommu_iotlb_gather *gather) +static size_t +tegra_smmu_unmap_locked (struct iommu_domain *domain,unsigned long iova, +size_t size,struct iommu_iotlb_gather *gather) {struct tegra_smmu_as *as =to_smmu_as (domain);dma_addr_t pte_dma;@@ -702,6 +708,33 @@ static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova,return size; } +static int tegra_smmu_map (struct iommu_domain *domain,unsigned long iova, +phys_addr_t paddr,size_t size,int prot,gfp_t gfp) +{ +struct tegra_smmu_as *as =to_smmu_as (domain); +unsigned long flags; +int ret; + +spin_lock_irqsave (&as->lock, flags); + ret =tegra_smmu_map_locked (domain, iova, paddr, size, prot, gfp); +spin_unlock_irqrestore (&as->lock, flags); + +return ret; +} + +static size_t tegra_smmu_unmap (struct iommu_domain *domain,unsigned long iova, +size_t size,struct iommu_iotlb_gather *gather) +{ +struct tegra_smmu_as *as =to_smmu_as (domain); +unsigned long flags; + +spin_lock_irqsave (&as->lock, flags); + size =tegra_smmu_unmap_locked (domain, iova, size, gather);
Quoted 8 lines
drivers/iommu/tegra-smmu.c static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova, · line 736
Quoted 4 lines
drivers/iommu/tegra-smmu.c +4
-0+spin_unlock_irqrestore (&as->lock, flags); + +return size; +}Why the extra functions here? We never call locked vs. unlocked variants in the driver and the IOMMU framework only has a single callback, so I think the locking can just move into the main implementation.
Because this makes code cleaner, easier to read and follow. You don't need to care about handling error code paths. This is the same what we do in the GART driver. Pretty much every other IOMMU driver use this code pattern as well.
Quoted 57 lines · Dmitry Osipenko
25.05.2020 11:35, Thierry Reding пишет:
Quoted 49 lines · Dmitry Osipenko
Quoted 48 lines
The mapping operations of the Tegra SMMU driver are subjected to a race condition issues because SMMU Address Space isn't allocated and freed atomically, while it should be. This patch makes the mapping operations atomic, it fixes an accidentally released Host1x Address Space problem which happens while running multiple graphics tests in parallel on Tegra30, i.e. by having multiple threads racing with each other in the Host1x's submission and completion code paths, performing IOVA mappings and unmappings in parallel.
Cc <[email protected]>Signed-off-by Dmitry Osipenko <[email protected]>--- drivers/iommu/tegra-smmu.c | 43 +++++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 5 deletions(-)drivers/iommu/tegra-smmu.c +4
-0@@ -12,6 +12,7 @@#include <linux/of_device.h> #include <linux/platform_device.h> #include <linux/slab.h> +#include <linux/spinlock.h> #include <linux/dma-mapping.h> #include <soc/tegra/ahb.h> @@ -49,6 +50,7 @@ struct tegra_smmu_as {struct iommu_domain domain;struct tegra_smmu *smmu;unsigned int use_count; +spinlock_t lock;u32 *count;struct page **pts;struct page *pd;@@ -308,6 +310,8 @@ static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type)return NULL ; } +spin_lock_init (&as->lock); +/* setup aperture */ as->domain.geometry.aperture_start =0 ; as->domain.geometry.aperture_end =0xffffffff ;@@ -578,7 +582,7 @@ static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova,struct page *page;dma_addr_t dma;Quoted 6 lines
drivers/iommu/tegra-smmu.c static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, · line 585
Quoted 2 lines
drivers/iommu/tegra-smmu.c +1
-1- page =alloc_page (GFP_KERNEL | __GFP_DMA | __GFP_ZERO); + page =alloc_page (GFP_ATOMIC | __GFP_DMA | __GFP_ZERO);I'm not sure this is a good idea. My recollection is that
GFP_ATOMICwill allocate from a special reserved region of memory, which may be easily exhausted.
Quoted 3 lines
So far I haven't noticed any problems. Will be great if you could provide more details about the pool size and how this exhaustion problem could be reproduced in practice.
I can't exactly pinpoint where that pool is created nor what its size is, but just from looking at:
Documentation/core-api/memory-allocation.rst
and searching for GFP_ATOMIC it says:
* If you think that accessing memory reserves is justified and the kernel will be stressed unless allocation succeeds, you may use ``GFP_ATOMIC``.
That doesn't sound like the case that we're running into here. It sounds to me like GFP_ATOMIC should really only be used in cases where interrupts are being processed or where allocations really need to be successful to ensure the system keeps operational (i.e. during handling of severe errors, ...). Failure to allocate a page directory is hardly very critical.
Quoted 6 lines
Quoted 2 lines
Is there any reason why we need the spinlock? Can't we use a mutex instead?
This is what other IOMMU drivers do. I guess mutex might be too expensive, it may create a noticeable contention which you don't want to have in a case of a GPU submission code path.
I see indeed that many other drivers seem to heavily make use of GFP_ATOMIC. But that doesn't necessarily mean that it's the right thing to do on Tegra (or for the other drivers for that matter).
Do we have a good way to find out how bad exactly the contention would be when using a mutex?
Quoted 19 lines
I also suspect that drivers of other platforms are using IOMMU API in interrupt context, although today this is not needed for Tegra.
Quoted 7 lines
Quoted 7 lines
drivers/iommu/tegra-smmu.c +0
-0if (!page)return NULL ;@@ -655,8 +659,9 @@ static void tegra_smmu_set_pte(struct tegra_smmu_as *as, unsigned long iova,smmu_flush (smmu); }Quoted 8 lines
drivers/iommu/tegra-smmu.c static void tegra_smmu_set_pte(struct tegra_smmu_as *as, unsigned long iova, · line 664
Quoted 5 lines
drivers/iommu/tegra-smmu.c +3
-2-static int tegra_smmu_map (struct iommu_domain *domain,unsigned long iova, -phys_addr_t paddr,size_t size,int prot,gfp_t gfp) +static int +tegra_smmu_map_locked (struct iommu_domain *domain,unsigned long iova, +phys_addr_t paddr,size_t size,int prot,gfp_t gfp)I think it's more typical to use the
_unlockedsuffix for functions that don't take a lock themselves.
Quoted 3 lines
Personally I can't feel the difference. Both variants are good to me. I can replace the literal postfix with a
__tegra_smmuprefix, similarly to what we have in the GART driver, to avoid bikeshedding.
In my opinion a function name is most useful when it describes what the function does. To me, an _unlocked suffix indicates that the function itself doesn't lock, and therefore it needs to be called with some specific lock already held.
Conversely, a _locked suffix indicates that the function will itself take that specific lock and hence it can be used without taking any extra care about locking.
With that interpretation the code below (taking a lock and then calling a "_locked" function) seems like a bug on the surface.
Admittedly that's somewhat subjective. The __ prefix seems like a good way to avoid some of the ambiguity, since it's probably an even more common theme.
Quoted 51 lines
Quoted 42 lines
Quoted 42 lines
drivers/iommu/tegra-smmu.c +25
-2{struct tegra_smmu_as *as =to_smmu_as (domain);dma_addr_t pte_dma;@@ -685,8 +690,9 @@ static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova,return 0 ; } -static size_t tegra_smmu_unmap (struct iommu_domain *domain,unsigned long iova, -size_t size,struct iommu_iotlb_gather *gather) +static size_t +tegra_smmu_unmap_locked (struct iommu_domain *domain,unsigned long iova, +size_t size,struct iommu_iotlb_gather *gather) {struct tegra_smmu_as *as =to_smmu_as (domain);dma_addr_t pte_dma;@@ -702,6 +708,33 @@ static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova,return size; } +static int tegra_smmu_map (struct iommu_domain *domain,unsigned long iova, +phys_addr_t paddr,size_t size,int prot,gfp_t gfp) +{ +struct tegra_smmu_as *as =to_smmu_as (domain); +unsigned long flags; +int ret; + +spin_lock_irqsave (&as->lock, flags); + ret =tegra_smmu_map_locked (domain, iova, paddr, size, prot, gfp); +spin_unlock_irqrestore (&as->lock, flags); + +return ret; +} + +static size_t tegra_smmu_unmap (struct iommu_domain *domain,unsigned long iova, +size_t size,struct iommu_iotlb_gather *gather) +{ +struct tegra_smmu_as *as =to_smmu_as (domain); +unsigned long flags; + +spin_lock_irqsave (&as->lock, flags); + size =tegra_smmu_unmap_locked (domain, iova, size, gather);Quoted 8 lines
drivers/iommu/tegra-smmu.c static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova, · line 736
Quoted 4 lines
drivers/iommu/tegra-smmu.c +4
-0+spin_unlock_irqrestore (&as->lock, flags); + +return size; +}Why the extra functions here? We never call locked vs. unlocked variants in the driver and the IOMMU framework only has a single callback, so I think the locking can just move into the main implementation.
Quoted 4 lines
Because this makes code cleaner, easier to read and follow. You don't need to care about handling error code paths. This is the same what we do in the GART driver. Pretty much every other IOMMU driver use this code pattern as well.
Alright, I don't mind very much. It just seemed odd to have that extra function that doesn't really do anything.
Thierry
25.05.2020 15:20, Thierry Reding пишет:
...
Quoted 2 lines
Do we have a good way to find out how bad exactly the contention would be when using a mutex?
I'm now having a second thought about it. We don't need to care about that scenario at all because it's a software-design defect of the upstream Host1x driver that it maps gathers dynamically. The defect can be fixed and then the potential problem won't exist at all.
Neither of the drivers that are using Tegra SMMU need the IOMMU mapping operations to be performed under spinlock, so mutex will be good a variant. I'll make a v2 with a mutex, thank you for the suggestion.