Skip to contentSkip to conversation

add DMA-sync-for-device capability to page_pool API

PATCH Superseded Lorenzo Bianconi 3 participants 9 messages Nov 15, 2019 – Nov 16, 2019

Subsystems (4)
Networking Network drivers Marvell mvneta Ethernet driver Page pool

v3 of 5 Superseded Latest is v5 →

lore
Share & tools
lore

Lorenzo Bianconi

author
#0 lore raw

v3 cover add DMA-sync-for-device capability to page_pool API

Introduce the possibility to sync DMA memory for device in the page_pool API. This feature allows to sync proper DMA size and not always full buffer (dma_sync_single_for_device can be very costly).
Please note DMA-sync-for-CPU is still device driver responsibility. Relying on page_pool DMA sync mvneta driver improves XDP_DROP pps of about 180Kpps:

- XDP_DROP DMA sync managed by mvneta driver: ~420Kpps
- XDP_DROP DMA sync managed by page_pool API: ~595Kpps

Changes since v2:
- rely on PP_FLAG_DMA_SYNC_DEV flag instead of dma_sync

Changes since v1:
- rename sync in dma_sync
- set dma_sync_size to 0xFFFFFFFF in page_pool_recycle_direct and page_pool_put_page routines
- Improve documentation

Lorenzo Bianconi (3):
net: mvneta: rely on page_pool_recycle_direct in mvneta_run_xdp
net: page_pool: add the possibility to sync DMA memory for device
net: mvneta: get rid of huge dma sync in mvneta_rx_refill

 drivers/net/ethernet/marvell/mvneta.c | 24 +++++++++------
 include/net/page_pool.h               | 21 ++++++++++----
 net/core/page_pool.c                  | 42 +++++++++++++++++++++++----
 3 files changed, 66 insertions(+), 21 deletions(-)
Signature · 1 line
2.21.0
author
#1 lore raw

PATCH v3 1/3 net: mvneta: rely on page_pool_recycle_direct in mvneta_run_xdp

Rely on page_pool_recycle_direct and not on xdp_return_buff in mvneta_run_xdp. This is a preliminary patch to limit the dma sync len to the one strictly necessary

Signed-off-by Lorenzo Bianconi <[email protected]>
---
 drivers/net/ethernet/marvell/mvneta.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)
drivers/net/ethernet/marvell/mvneta.c +4 -2
index 12e03b15f0ab..f7713c2c68e1 100644
@@ -2097,7 +2097,8 @@ mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq,
 		err = xdp_do_redirect(pp->dev, xdp, prog);
 		if (err) {
 			ret = MVNETA_XDP_DROPPED;
-			xdp_return_buff(xdp);
+			page_pool_recycle_direct(rxq->page_pool,
+						 virt_to_head_page(xdp->data));
 		} else {
 			ret = MVNETA_XDP_REDIR;
 		}

@@ -2106,7 +2107,8 @@ mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq,
 	case XDP_TX:
 		ret = mvneta_xdp_xmit_back(pp, xdp);
 		if (ret != MVNETA_XDP_TX)
-			xdp_return_buff(xdp);
+			page_pool_recycle_direct(rxq->page_pool,
+						 virt_to_head_page(xdp->data));
 		break;
 	default:
 		bpf_warn_invalid_xdp_action(act);
Signature · 1 line
2.21.0
author
#2 lore raw

PATCH v3 2/3 net: page_pool: add the possibility to sync DMA memory for device

In reply to Lorenzo Bianconi

Introduce the following parameters in order to add the possibility to sync DMA memory for device before putting allocated pages in the page_pool caches:
- PP_FLAG_DMA_SYNC_DEV: if set in page_pool_params flags, all pages that the driver gets from page_pool will be DMA-synced-for-device according to the length provided by the device driver. Please note DMA-sync-for-CPU is still device driver responsibility
- offset: DMA address offset where the DMA engine starts copying rx data
- max_len: maximum DMA memory size page_pool is allowed to flush. This is currently used in __page_pool_alloc_pages_slow routine when pages are allocated from page allocator
These parameters are supposed to be set by device drivers.

This optimization reduces the length of the DMA-sync-for-device. The optimization is valid because pages are initially DMA-synced-for-device as defined via max_len. At RX time, the driver will perform a DMA-sync-for-CPU on the memory for the packet length. What is important is the memory occupied by packet payload, because this is the area CPU is allowed to read and modify. As we don't track cache-lines written into by the CPU, simply use the packet payload length as dma_sync_size at page_pool recycle time. This also take into account any tail-extend.

Tested-by Matteo Croce <[email protected]>
Signed-off-by Lorenzo Bianconi <[email protected]>
---
 include/net/page_pool.h | 21 +++++++++++++++------
 net/core/page_pool.c    | 42 +++++++++++++++++++++++++++++++++++------
 2 files changed, 51 insertions(+), 12 deletions(-)
include/net/page_pool.h +15 -6
index 2cbcdbdec254..8856d2c815d0 100644
@@ -34,8 +34,15 @@
 #include <linux/ptr_ring.h>
 #include <linux/dma-direction.h>
 
-#define PP_FLAG_DMA_MAP 1 /* Should page_pool do the DMA map/unmap */
-#define PP_FLAG_ALL	PP_FLAG_DMA_MAP
+#define PP_FLAG_DMA_MAP		1 /* Should page_pool do the DMA map/unmap */
+#define PP_FLAG_DMA_SYNC_DEV	2 /* if set all pages that the driver gets
+				   * from page_pool will be
+				   * DMA-synced-for-device according to the
+				   * length provided by the device driver.
+				   * Please note DMA-sync-for-CPU is still
+				   * device driver responsibility
+				   */
+#define PP_FLAG_ALL		(PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV)
 
 /*
  * Fast allocation side cache array/stack

@@ -65,6 +72,8 @@ struct page_pool_params {
 	int		nid;  /* Numa node id to allocate from pages from */
 	struct device	*dev; /* device, for DMA pre-mapping purposes */
 	enum dma_data_direction dma_dir; /* DMA mapping direction */
+	unsigned int	max_len; /* max DMA sync memory size */
+	unsigned int	offset;  /* DMA addr offset */
 };
 
 struct page_pool {

@@ -150,8 +159,8 @@ static inline void page_pool_destroy(struct page_pool *pool)
 }
 
 /* Never call this directly, use helpers below */
-void __page_pool_put_page(struct page_pool *pool,
-			  struct page *page, bool allow_direct);
+void __page_pool_put_page(struct page_pool *pool, struct page *page,
+			  unsigned int dma_sync_size, bool allow_direct);
 
 static inline void page_pool_put_page(struct page_pool *pool,
 				      struct page *page, bool allow_direct)

@@ -160,14 +169,14 @@ static inline void page_pool_put_page(struct page_pool *pool,
 	 * allow registering MEM_TYPE_PAGE_POOL, but shield linker.
 	 */
 #ifdef CONFIG_PAGE_POOL
-	__page_pool_put_page(pool, page, allow_direct);
+	__page_pool_put_page(pool, page, -1, allow_direct);
 #endif
 }
 /* Very limited use-cases allow recycle direct */
 static inline void page_pool_recycle_direct(struct page_pool *pool,
 					    struct page *page)
 {
-	__page_pool_put_page(pool, page, true);
+	__page_pool_put_page(pool, page, -1, true);
 }
 
 /* API user MUST have disconnected alloc-side (not allowed to call
net/core/page_pool.c +36 -6

95 lines not on this page · raw message

Signature · 1 line
2.21.0
Acked-by on v3 2/3
#3 lore raw

Re: [PATCH v3 net-next 2/3] net: page_pool: add the possibility to sync DMA memory for device

Quoted 23 lines · Lorenzo Bianconi

Introduce the following parameters in order to add the possibility to sync DMA memory for device before putting allocated pages in the page_pool caches:
- PP_FLAG_DMA_SYNC_DEV: if set in page_pool_params flags, all pages that the driver gets from page_pool will be DMA-synced-for-device according to the length provided by the device driver. Please note DMA-sync-for-CPU is still device driver responsibility
- offset: DMA address offset where the DMA engine starts copying rx data
- max_len: maximum DMA memory size page_pool is allowed to flush. This is currently used in __page_pool_alloc_pages_slow routine when pages are allocated from page allocator
These parameters are supposed to be set by device drivers.

This optimization reduces the length of the DMA-sync-for-device. The optimization is valid because pages are initially DMA-synced-for-device as defined via max_len. At RX time, the driver will perform a DMA-sync-for-CPU on the memory for the packet length. What is important is the memory occupied by packet payload, because this is the area CPU is allowed to read and modify. As we don't track cache-lines written into by the CPU, simply use the packet payload length as dma_sync_size at page_pool recycle time. This also take into account any tail-extend.

Quoted 2 lines
Tested-by Matteo Croce <[email protected]>
Signed-off-by Lorenzo Bianconi <[email protected]>
Acked-by Jonathan Lemon <[email protected]>
#4 lore raw

Re: [PATCH v3 net-next 2/3] net: page_pool: add the possibility to sync DMA memory for device

In reply to Lorenzo Bianconi
Quoted 8 lines · On Fri, 15 Nov 2019 21:01:38 +0200 Lorenzo Bianconi <[email protected]> wrote:

static bool __page_pool_recycle_into_ring(struct page_pool *pool,

net/core/page_pool.c +2 -1
-				   struct page *page)
+					  struct page *page,
+					  unsigned int dma_sync_size)
 {
 	int ret;
 	/* BH protection not needed if current is serving softirq */

net/core/page_pool.c static bool __page_pool_recycle_into_ring(struct page_pool *pool, · line 292

Quoted 8 lines
net/core/page_pool.c +3 -0
@@ -264,6 +285,9 @@ static bool __page_pool_recycle_into_ring(struct page_pool *pool,
 	else
 		ret = ptr_ring_produce_bh(&pool->ring, page);
 
+	if (ret == 0 && (pool->p.flags & PP_FLAG_DMA_SYNC_DEV))
+		page_pool_dma_sync_for_device(pool, page, dma_sync_size);
+
 	return (ret == 0) ? true : false;
 }

I do wonder if we should DMA-sync-for-device BEFORE putting page into ptr_ring, as this is a channel between several concurrent CPUs.

Quoted 8 lines
net/core/page_pool.c +2 -1
@@ -273,18 +297,22 @@ static bool __page_pool_recycle_into_ring(struct page_pool *pool,
  * Caller must provide appropriate safe context.
  */
 static bool __page_pool_recycle_direct(struct page *page,
-				       struct page_pool *pool)
+				       struct page_pool *pool,
+				       unsigned int dma_sync_size)
 {

net/core/page_pool.c static bool __page_pool_recycle_into_ring(struct page_pool *pool, · line 312

Quoted 10 lines
net/core/page_pool.c +3 -0
 	if (unlikely(pool->alloc.count == PP_ALLOC_CACHE_SIZE))
 		return false;
 
 	/* Caller MUST have verified/know (page_ref_count(page) == 1) */
 	pool->alloc.cache[pool->alloc.count++] = page;
+
+	if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV)
+		page_pool_dma_sync_for_device(pool, page, dma_sync_size);
 	return true;
 }

We know __page_pool_recycle_direct() is concurrency safe, and only a single (NAPI processing) CPU can enter. (So, the DMA-sync order is not wrong here, but it could be swapped)

Signature · 4 lines
Best regards,
  Jesper Dangaard Brouer
  MSc.CS, Principal Kernel Engineer at Red Hat
  LinkedIn: http://www.linkedin.com/in/brouer
author
#5 lore raw

Re: [PATCH v3 net-next 2/3] net: page_pool: add the possibility to sync DMA memory for device

Quoted 14 lines
Quoted 11 lines · On Fri, 15 Nov 2019 21:01:38 +0200 Lorenzo Bianconi <[email protected]> wrote:

static bool __page_pool_recycle_into_ring(struct page_pool *pool,

net/core/page_pool.c +2 -1
-				   struct page *page)
+					  struct page *page,
+					  unsigned int dma_sync_size)
 {
 	int ret;
 	/* BH protection not needed if current is serving softirq */

@@ -264,6 +285,9 @@ static bool __page_pool_recycle_into_ring(struct page_pool *pool,
 	else
 		ret = ptr_ring_produce_bh(&pool->ring, page);
Quoted 9 lines

net/core/page_pool.c static bool __page_pool_recycle_into_ring(struct page_pool *pool, · line 292

Quoted 5 lines
net/core/page_pool.c +3 -0
+	if (ret == 0 && (pool->p.flags & PP_FLAG_DMA_SYNC_DEV))
+		page_pool_dma_sync_for_device(pool, page, dma_sync_size);
+
 	return (ret == 0) ? true : false;
 }

I do wonder if we should DMA-sync-for-device BEFORE putting page into ptr_ring, as this is a channel between several concurrent CPUs.

Hi Jesper,

in this way we can end up syncing the DMA page even if it is unmapped in __page_pool_clean_page (e.g. if the ptr_ring is full), right?

Quoted 14 lines
Quoted 14 lines
net/core/page_pool.c +2 -1
@@ -273,18 +297,22 @@ static bool __page_pool_recycle_into_ring(struct page_pool *pool,
  * Caller must provide appropriate safe context.
  */
 static bool __page_pool_recycle_direct(struct page *page,
-				       struct page_pool *pool)
+				       struct page_pool *pool,
+				       unsigned int dma_sync_size)
 {
 	if (unlikely(pool->alloc.count == PP_ALLOC_CACHE_SIZE))
 		return false;
Quoted 11 lines

net/core/page_pool.c static bool __page_pool_recycle_into_ring(struct page_pool *pool, · line 312

Quoted 7 lines
net/core/page_pool.c +3 -0
 	/* Caller MUST have verified/know (page_ref_count(page) == 1) */
 	pool->alloc.cache[pool->alloc.count++] = page;
+
+	if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV)
+		page_pool_dma_sync_for_device(pool, page, dma_sync_size);
 	return true;
 }

We know __page_pool_recycle_direct() is concurrency safe, and only a single (NAPI processing) CPU can enter. (So, the DMA-sync order is not wrong here, but it could be swapped)

do you mean move it before putting the page in the cache?

pool->alloc.cache[pool->alloc.count++] = page;

Regards,
Lorenzo

Quoted 7 lines
Signature · 5 lines
-- 
Best regards,
  Jesper Dangaard Brouer
  MSc.CS, Principal Kernel Engineer at Red Hat
  LinkedIn: http://www.linkedin.com/in/brouer
#6 lore raw

Re: [PATCH v3 net-next 2/3] net: page_pool: add the possibility to sync DMA memory for device

Quoted 24 lines · On Sat, 16 Nov 2019 13:36:30 +0200 Lorenzo Bianconi <[email protected]> wrote:
Quoted 14 lines
Quoted 11 lines · On Fri, 15 Nov 2019 21:01:38 +0200 Lorenzo Bianconi <[email protected]> wrote:

static bool __page_pool_recycle_into_ring(struct page_pool *pool,

net/core/page_pool.c +2 -1
-				   struct page *page)
+					  struct page *page,
+					  unsigned int dma_sync_size)
 {
 	int ret;
 	/* BH protection not needed if current is serving softirq */

@@ -264,6 +285,9 @@ static bool __page_pool_recycle_into_ring(struct page_pool *pool,
 	else
 		ret = ptr_ring_produce_bh(&pool->ring, page);
Quoted 9 lines

net/core/page_pool.c static bool __page_pool_recycle_into_ring(struct page_pool *pool, · line 292

Quoted 5 lines
net/core/page_pool.c +3 -0
+	if (ret == 0 && (pool->p.flags & PP_FLAG_DMA_SYNC_DEV))
+		page_pool_dma_sync_for_device(pool, page, dma_sync_size);
+
 	return (ret == 0) ? true : false;
 }

I do wonder if we should DMA-sync-for-device BEFORE putting page into ptr_ring, as this is a channel between several concurrent CPUs.

Quoted 4 lines

Hi Jesper,

in this way we can end up syncing the DMA page even if it is unmapped in __page_pool_clean_page (e.g. if the ptr_ring is full), right?

Yes. The call __page_pool_clean_page() will do a dma_unmap_page, so it
should still be safe/correct. I can see, that it is not optimal
performance wise, in-case the ptr_ring is full, as DMA-sync-for-device is wasted work.

I don't know if you can find an argument, that proves that it cannot happen, that a remote CPU can dequeue/consume the page from ptr_ring and give it to the device, while you (the CPU the enqueued) are still doing the DMA-sync-for-device.

Quoted 23 lines
Quoted 11 lines
Quoted 11 lines
net/core/page_pool.c +2 -1
@@ -273,18 +297,22 @@ static bool __page_pool_recycle_into_ring(struct page_pool *pool,
  * Caller must provide appropriate safe context.
  */
 static bool __page_pool_recycle_direct(struct page *page,
-				       struct page_pool *pool)
+				       struct page_pool *pool,
+				       unsigned int dma_sync_size)
 {
 	if (unlikely(pool->alloc.count == PP_ALLOC_CACHE_SIZE))
 		return false;
Quoted 11 lines

net/core/page_pool.c static bool __page_pool_recycle_into_ring(struct page_pool *pool, · line 312

Quoted 7 lines
net/core/page_pool.c +3 -0
 	/* Caller MUST have verified/know (page_ref_count(page) == 1) */
 	pool->alloc.cache[pool->alloc.count++] = page;
+
+	if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV)
+		page_pool_dma_sync_for_device(pool, page, dma_sync_size);
 	return true;
 }

We know __page_pool_recycle_direct() is concurrency safe, and only a single (NAPI processing) CPU can enter. (So, the DMA-sync order is not wrong here, but it could be swapped)

Quoted 3 lines

do you mean move it before putting the page in the cache?

pool->alloc.cache[pool->alloc.count++] = page;

Yes, but here the order doesn't matter.

If you choose to do the DMA-sync-for-device earlier/before, then look at the code, and see of it makes sense to do it in __page_pool_put_page() ? (I've not checked the details)

Signature · 4 lines
Best regards,
  Jesper Dangaard Brouer
  MSc.CS, Principal Kernel Engineer at Red Hat
  LinkedIn: http://www.linkedin.com/in/brouer
author
#7 lore raw

Re: [PATCH v3 net-next 2/3] net: page_pool: add the possibility to sync DMA memory for device

Quoted 37 lines
Quoted 24 lines · On Sat, 16 Nov 2019 13:36:30 +0200 Lorenzo Bianconi <[email protected]> wrote:
Quoted 14 lines
Quoted 11 lines · On Fri, 15 Nov 2019 21:01:38 +0200 Lorenzo Bianconi <[email protected]> wrote:

static bool __page_pool_recycle_into_ring(struct page_pool *pool,

net/core/page_pool.c +2 -1
-				   struct page *page)
+					  struct page *page,
+					  unsigned int dma_sync_size)
 {
 	int ret;
 	/* BH protection not needed if current is serving softirq */

@@ -264,6 +285,9 @@ static bool __page_pool_recycle_into_ring(struct page_pool *pool,
 	else
 		ret = ptr_ring_produce_bh(&pool->ring, page);
Quoted 9 lines

net/core/page_pool.c static bool __page_pool_recycle_into_ring(struct page_pool *pool, · line 292

Quoted 5 lines
net/core/page_pool.c +3 -0
+	if (ret == 0 && (pool->p.flags & PP_FLAG_DMA_SYNC_DEV))
+		page_pool_dma_sync_for_device(pool, page, dma_sync_size);
+
 	return (ret == 0) ? true : false;
 }

I do wonder if we should DMA-sync-for-device BEFORE putting page into ptr_ring, as this is a channel between several concurrent CPUs.

Quoted 4 lines

Hi Jesper,

in this way we can end up syncing the DMA page even if it is unmapped in __page_pool_clean_page (e.g. if the ptr_ring is full), right?

Yes. The call __page_pool_clean_page() will do a dma_unmap_page, so it
should still be safe/correct. I can see, that it is not optimal
performance wise, in-case the ptr_ring is full, as DMA-sync-for-device is wasted work.

Quoted 4 lines

I don't know if you can find an argument, that proves that it cannot happen, that a remote CPU can dequeue/consume the page from ptr_ring and give it to the device, while you (the CPU the enqueued) are still doing the DMA-sync-for-device.

right, I can see it now :)

Quoted 27 lines
Quoted 27 lines
Quoted 13 lines
Quoted 13 lines
net/core/page_pool.c +2 -1
@@ -273,18 +297,22 @@ static bool __page_pool_recycle_into_ring(struct page_pool *pool,
  * Caller must provide appropriate safe context.
  */
 static bool __page_pool_recycle_direct(struct page *page,
-				       struct page_pool *pool)
+				       struct page_pool *pool,
+				       unsigned int dma_sync_size)
 {
 	if (unlikely(pool->alloc.count == PP_ALLOC_CACHE_SIZE))
 		return false;
Quoted 11 lines

net/core/page_pool.c static bool __page_pool_recycle_into_ring(struct page_pool *pool, · line 312

Quoted 7 lines
net/core/page_pool.c +3 -0
 	/* Caller MUST have verified/know (page_ref_count(page) == 1) */
 	pool->alloc.cache[pool->alloc.count++] = page;
+
+	if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV)
+		page_pool_dma_sync_for_device(pool, page, dma_sync_size);
 	return true;
 }

We know __page_pool_recycle_direct() is concurrency safe, and only a single (NAPI processing) CPU can enter. (So, the DMA-sync order is not wrong here, but it could be swapped)

do you mean move it before putting the page in the cache?

Quoted 7 lines
Quoted 1 line

pool->alloc.cache[pool->alloc.count++] = page;

Yes, but here the order doesn't matter.

If you choose to do the DMA-sync-for-device earlier/before, then look at the code, and see of it makes sense to do it in __page_pool_put_page() ? (I've not checked the details)

I guess we can move page_pool_dma_sync_for_device() before __page_pool_recycle_direct and __page_pool_recycle_into_ring since even if __page_pool_put_page is not running in NAPI context or if alloc.cache is full we will end up calling page_pool_dma_sync_for_device as first task in __page_pool_recycle_into_ring. I will fix in v4.

Regards,
Lorenzo

Quoted 7 lines
Signature · 5 lines
-- 
Best regards,
  Jesper Dangaard Brouer
  MSc.CS, Principal Kernel Engineer at Red Hat
  LinkedIn: http://www.linkedin.com/in/brouer
author
#8 lore raw

PATCH v3 3/3 net: mvneta: get rid of huge dma sync in mvneta_rx_refill

In reply to Lorenzo Bianconi

Get rid of costly dma_sync_single_for_device in mvneta_rx_refill since now the driver can let page_pool API to manage needed DMA sync with a proper size.

- XDP_DROP DMA sync managed by mvneta driver: ~420Kpps
- XDP_DROP DMA sync managed by page_pool API: ~595Kpps

Tested-by Matteo Croce <[email protected]>
Signed-off-by Lorenzo Bianconi <[email protected]>
---
 drivers/net/ethernet/marvell/mvneta.c | 26 +++++++++++++++-----------
 1 file changed, 15 insertions(+), 11 deletions(-)
drivers/net/ethernet/marvell/mvneta.c +15 -11
index f7713c2c68e1..a06d109c9e80 100644
@@ -1846,7 +1846,6 @@ static int mvneta_rx_refill(struct mvneta_port *pp,
 			    struct mvneta_rx_queue *rxq,
 			    gfp_t gfp_mask)
 {
-	enum dma_data_direction dma_dir;
 	dma_addr_t phys_addr;
 	struct page *page;
 

@@ -1856,9 +1855,6 @@ static int mvneta_rx_refill(struct mvneta_port *pp,
 		return -ENOMEM;
 
 	phys_addr = page_pool_get_dma_addr(page) + pp->rx_offset_correction;
-	dma_dir = page_pool_get_dma_dir(rxq->page_pool);
-	dma_sync_single_for_device(pp->dev->dev.parent, phys_addr,
-				   MVNETA_MAX_RX_BUF_SIZE, dma_dir);
 	mvneta_rx_desc_fill(rx_desc, phys_addr, page, rxq);
 
 	return 0;

@@ -2097,8 +2093,10 @@ mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq,
 		err = xdp_do_redirect(pp->dev, xdp, prog);
 		if (err) {
 			ret = MVNETA_XDP_DROPPED;
-			page_pool_recycle_direct(rxq->page_pool,
-						 virt_to_head_page(xdp->data));
+			__page_pool_put_page(rxq->page_pool,
+					virt_to_head_page(xdp->data),
+					xdp->data_end - xdp->data_hard_start,
+					true);
 		} else {
 			ret = MVNETA_XDP_REDIR;
 		}

@@ -2107,8 +2105,10 @@ mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq,
 	case XDP_TX:
 		ret = mvneta_xdp_xmit_back(pp, xdp);
 		if (ret != MVNETA_XDP_TX)
-			page_pool_recycle_direct(rxq->page_pool,
-						 virt_to_head_page(xdp->data));
+			__page_pool_put_page(rxq->page_pool,
+					virt_to_head_page(xdp->data),
+					xdp->data_end - xdp->data_hard_start,
+					true);
 		break;
 	default:
 		bpf_warn_invalid_xdp_action(act);

@@ -2117,8 +2117,10 @@ mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq,
 		trace_xdp_exception(pp->dev, prog, act);
 		/* fall through */
 	case XDP_DROP:
-		page_pool_recycle_direct(rxq->page_pool,
-					 virt_to_head_page(xdp->data));
+		__page_pool_put_page(rxq->page_pool,
+				     virt_to_head_page(xdp->data),
+				     xdp->data_end - xdp->data_hard_start,
+				     true);
 		ret = MVNETA_XDP_DROPPED;
 		break;
 	}

@@ -3067,11 +3069,13 @@ static int mvneta_create_page_pool(struct mvneta_port *pp,
 	struct bpf_prog *xdp_prog = READ_ONCE(pp->xdp_prog);
 	struct page_pool_params pp_params = {
 		.order = 0,
-		.flags = PP_FLAG_DMA_MAP,
+		.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
 		.pool_size = size,
 		.nid = cpu_to_node(0),
 		.dev = pp->dev->dev.parent,
 		.dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE,
+		.offset = pp->rx_offset_correction,
+		.max_len = MVNETA_MAX_RX_BUF_SIZE,
 	};
 	int err;
Signature · 1 line
2.21.0