iommu/dma: Use NUMA aware memory allocations in __iommu_dma_alloc_pages()
Subsystems (2)
PATCH v2 1/1 iommu/dma: Use NUMA aware memory allocations in __iommu_dma_alloc_pages()
From: Ganapatrao Kulkarni <[email protected]>
Change function __iommu_dma_alloc_pages() to allocate memory/pages for DMA from respective device NUMA node.
---
Difference v1->v2:
- Add Ganapatrao's tag and change author
This patch was originally posted by Ganapatrao in [1].
However, after initial review, it was never reposted (due to lack of cycles, I think). In addition, the functionality in its sibling patches were merged through patches, as mentioned in [2]; this also refers to a discussion on device local allocations vs CPU local allocations for DMA pool, and which is better [3].
However, as mentioned in [3], dma_alloc_coherent() uses the locality information from the device - as in direct DMA - so this patch is just applying this same policy.
[1] https://lore.kernel.org/patchwork/patch/833004/
[2] https://lkml.org/lkml/2018/8/22/391
[3] https://www.mail-archive.com/[email protected]/msg1692998.html
drivers/iommu/dma-iommu.c +10 -10
@@ -449,20 +449,17 @@ static void __iommu_dma_free_pages(struct page **pages, int count)kvfree (pages); } -static struct page **__iommu_dma_alloc_pages (unsigned int count, -unsigned long order_mask,gfp_t gfp) +static struct page **__iommu_dma_alloc_pages (struct device *dev, +unsigned int count,unsigned long order_mask,gfp_t gfp) {struct page **pages; -unsigned int i =0 , array_size = count *sizeof (*pages); +unsigned int i =0 , nid =dev_to_node (dev); order_mask &= (2U <<MAX_ORDER ) -1 ;if (!order_mask)return NULL ; -if (array_size <=PAGE_SIZE ) - pages =kzalloc (array_size,GFP_KERNEL ); -else - pages =vzalloc (array_size); + pages =kvzalloc_node (count *sizeof (*pages),GFP_KERNEL , nid);if (!pages)return NULL ;
@@ -483,8 +480,10 @@ static struct page **__iommu_dma_alloc_pages(unsigned int count,unsigned int order =__fls (order_mask); order_size =1U << order; - page =alloc_pages ((order_mask - order_size) ? - gfp | __GFP_NORETRY : gfp, order); + page =alloc_pages_node (nid, + (order_mask - order_size) ? + gfp | __GFP_NORETRY : gfp, + order);if (!page)continue ;if (!order)
@@ -569,7 +568,8 @@ struct page **iommu_dma_alloc(struct device *dev, size_t size, gfp_t gfp, alloc_sizes = min_size; count =PAGE_ALIGN (size) >>PAGE_SHIFT ; - pages =__iommu_dma_alloc_pages (count, alloc_sizes >>PAGE_SHIFT , gfp); + pages =__iommu_dma_alloc_pages (dev, count, alloc_sizes >>PAGE_SHIFT , + gfp);if (!pages)return NULL ;
Signature · 1 line
1.9.1
Robin Murphy
maintainerQuoted 45 lines · John Garry
From: Ganapatrao Kulkarni <[email protected]>
Change function
__iommu_dma_alloc_pages()to allocate memory/pages for DMA from respective device NUMA node.Signed-off-by Ganapatrao Kulkarni <[email protected]>[JPG: Modifed to use kvzalloc() and fixed indentation]Signed-off-by John Garry <[email protected]>---
Differencev1->v2:
- Add Ganapatrao's tag and change authorThis patch was originally posted by Ganapatrao in [1].
However, after initial review, it was never reposted (due to lack of cycles, I think). In addition, the functionality in its sibling patches were merged through patches, as mentioned in [2]; this also refers to a discussion on device local allocations vs CPU local allocations for DMA pool, and which is better [3].
However, as mentioned in [3],
dma_alloc_coherent()uses the locality information from the device - as in direct DMA - so this patch is just applying this same policy.[1] https://lore.kernel.org/patchwork/patch/833004/
[2] https://lkml.org/lkml/2018/8/22/391
[3] https://www.mail-archive.com/[email protected]/msg1692998.htmldrivers/iommu/dma-iommu.c +3
-3@@ -449,20 +449,17 @@ static void __iommu_dma_free_pages(struct page **pages, int count)kvfree (pages); } -static struct page **__iommu_dma_alloc_pages (unsigned int count, -unsigned long order_mask,gfp_t gfp) +static struct page **__iommu_dma_alloc_pages (struct device *dev, +unsigned int count,unsigned long order_mask,gfp_t gfp) {struct page **pages; -unsigned int i =0 , array_size = count *sizeof (*pages); +unsigned int i =0 , nid =dev_to_node (dev);
drivers/iommu/dma-iommu.c static void __iommu_dma_free_pages(struct page **pages, int count) · line 462
Quoted 10 lines
drivers/iommu/dma-iommu.c +1
-4order_mask &= (2U <<MAX_ORDER ) -1 ;if (!order_mask)return NULL ; -if (array_size <=PAGE_SIZE ) - pages =kzalloc (array_size,GFP_KERNEL ); -else - pages =vzalloc (array_size); + pages =kvzalloc_node (count *sizeof (*pages),GFP_KERNEL , nid);
The pages array is only accessed by the CPU servicing the iommu_dma_alloc() call, and is usually freed again before that call even returns. It's certainly never touched by the device, so forcing it to a potentially non-local node doesn't make a great deal of sense.
drivers/iommu/dma-iommu.c static struct page **__iommu_dma_alloc_pages(unsigned int count, · line 486
Quoted 13 lines
drivers/iommu/dma-iommu.c +4
-2if (!pages)return NULL ;@@ -483,8 +480,10 @@ static struct page **__iommu_dma_alloc_pages(unsigned int count,unsigned int order =__fls (order_mask); order_size =1U << order; - page =alloc_pages ((order_mask - order_size) ? - gfp | __GFP_NORETRY : gfp, order); + page =alloc_pages_node (nid, + (order_mask - order_size) ? + gfp | __GFP_NORETRY : gfp, + order);
If we're touching this, can we sort out that horrendous ternary? FWIW I found I have a local version of the original patch which I tweaked at the time, and apparently I reworked this hunk as below, which does seem somewhat nicer for the same diffstat.
Robin.
@@ -446,10 +443,12 @@ static struct page **__iommu_dma_alloc_pages(unsigned int count,
for (order_mask &= (2U << __fls(count)) - 1;
order_mask; order_mask &= ~order_size) {
unsigned int order = __fls(order_mask);+ gfp_t alloc_flags = gfp;
order_size = 1U << order;
- page = alloc_pages((order_mask - order_size) ?
- gfp | __GFP_NORETRY : gfp, order);
+ if (order_size < order_mask)
+ alloc_flags |= __GFP_NORETRY;
+ page = alloc_pages_node(nid, alloc_flags, order);
if (!page)
continue;
if (!order)Quoted 52 lines · Robin Murphy
Quoted 51 lines · John Garry
From: Ganapatrao Kulkarni <ganapatrao.kulkarni-YGCgFSpz5w/[email protected]>
Change function
__iommu_dma_alloc_pages()to allocate memory/pages for DMA from respective device NUMA node.[JPG: Modifed to use kvzalloc() and fixed indentation]Signed-off-by John Garry <[email protected]>---
Differencev1->v2:
- Add Ganapatrao's tag and change authorThis patch was originally posted by Ganapatrao in [1].
However, after initial review, it was never reposted (due to lack of cycles, I think). In addition, the functionality in its sibling patches were merged through patches, as mentioned in [2]; this also refers to a discussion on device local allocations vs CPU local allocations for DMA pool, and which is better [3].
However, as mentioned in [3],
dma_alloc_coherent()uses the locality information from the device - as in direct DMA - so this patch is just applying this same policy.[1] https://lore.kernel.org/patchwork/patch/833004/
[2] https://lkml.org/lkml/2018/8/22/391
[3]
https://www.mail-archive.com/[email protected]/msg1692998.htmldrivers/iommu/dma-iommu.c +3
-3@@ -449,20 +449,17 @@ static void __iommu_dma_free_pages(struct page **pages, int count)kvfree (pages); } -static struct page **__iommu_dma_alloc_pages(unsigned int count, -unsigned long order_mask,gfp_t gfp) +static struct page **__iommu_dma_alloc_pages(struct device *dev, + unsigned int count, unsigned long order_mask, gfp_t gfp) { struct page **pages; -unsigned int i =0 , array_size = count *sizeof (*pages); + unsigned int i = 0, nid = dev_to_node(dev); order_mask &= (2U << MAX_ORDER) - 1; if (!order_mask) return NULL; - if (array_size <= PAGE_SIZE) - pages = kzalloc(array_size, GFP_KERNEL);
Quoted 8 lines
drivers/iommu/dma-iommu.c static void __iommu_dma_free_pages(struct page **pages, int count) · line 462
Quoted 3 lines
drivers/iommu/dma-iommu.c +1
-2-else - pages =vzalloc (array_size); + pages =kvzalloc_node (count *sizeof (*pages),GFP_KERNEL , nid);The pages array is only accessed by the CPU servicing the
iommu_dma_alloc()call, and is usually freed again before that call even returns. It's certainly never touched by the device, so forcing it to a potentially non-local node doesn't make a great deal of sense.
Right, it seems sensible to not make this allocation include the device-locality requirement, so can leave as is. However modifying to use kvzalloc() would seem ok.
Quoted 25 lines
drivers/iommu/dma-iommu.c static struct page **__iommu_dma_alloc_pages(unsigned int count, · line 486
Quoted 13 lines
drivers/iommu/dma-iommu.c +0
-0if (!pages)return NULL ; @@ -483,8 +480,10 @@ static struct page**__iommu_dma_alloc_pages(unsigned int count,
unsigned int order = __fls(order_mask); order_size = 1U << order;drivers/iommu/dma-iommu.c +4
-2- page =alloc_pages ((order_mask - order_size) ? - gfp | __GFP_NORETRY : gfp, order); + page =alloc_pages_node (nid, + (order_mask - order_size) ? + gfp | __GFP_NORETRY : gfp, + order);If we're touching this, can we sort out that horrendous ternary? FWIW I found I have a local version of the original patch which I tweaked at the time, and apparently I reworked this hunk as below, which does seem somewhat nicer for the same diffstat.
Robin.
(quoted diff) +0
-0@@ -446,10 +443,12 @@ static struct page **__iommu_dma_alloc_pages(unsigned int count, for (order_mask &= (2U << __fls(count)) - 1;
Review static struct page · line 452
Quoted 10 lines
(quoted diff) +3
-2order_mask; order_mask &= ~order_size) { unsigned int order = __fls(order_mask); + gfp_t alloc_flags = gfp; order_size = 1U << order; - page = alloc_pages((order_mask - order_size) ? - gfp | __GFP_NORETRY : gfp, order); + if (order_size < order_mask) + alloc_flags |= __GFP_NORETRY;
Sure, this can be included
Quoted 7 lines
(quoted diff) +1
-0+ page = alloc_pages_node(nid, alloc_flags, order); if (!page)continue; if (!order).
Cheers,
John