xdp: introduce bulking for page_pool tx return path

index 54b0bf574c05..af33cc62ed4c 100644
@@ -663,6 +663,8 @@ struct mvneta_tx_queue {
 
 	/* Affinity mask for CPUs*/
 	cpumask_t affinity_mask;
+
+	struct xdp_frame_bulk bq;
 };
 
 struct mvneta_rx_queue {

@@ -1854,12 +1856,10 @@ static void mvneta_txq_bufs_free(struct mvneta_port *pp,
 			dev_kfree_skb_any(buf->skb);
 		} else if (buf->type == MVNETA_TYPE_XDP_TX ||
 			   buf->type == MVNETA_TYPE_XDP_NDO) {
-			if (napi && buf->type == MVNETA_TYPE_XDP_TX)
-				xdp_return_frame_rx_napi(buf->xdpf);
-			else
-				xdp_return_frame(buf->xdpf);
+			xdp_return_frame_bulk(buf->xdpf, &txq->bq, napi);
 		}
 	}
+	xdp_flush_frame_bulk(&txq->bq, napi);
 
 	netdev_tx_completed_queue(nq, pkts_compl, bytes_compl);
 }

index 3814fb631d52..4b79d50afe36 100644
@@ -104,6 +104,12 @@ struct xdp_frame {
 	struct net_device *dev_rx; /* used by cpumap */
 };
 
+#define XDP_BULK_QUEUE_SIZE	16
+struct xdp_frame_bulk {
+	void *q[XDP_BULK_QUEUE_SIZE];
+	int count;
+	void *xa;
+};
 
 static inline struct skb_shared_info *
 xdp_get_shared_info_from_frame(struct xdp_frame *frame)

@@ -194,6 +200,11 @@ struct xdp_frame *xdp_convert_buff_to_frame(struct xdp_buff *xdp)
 void xdp_return_frame(struct xdp_frame *xdpf);
 void xdp_return_frame_rx_napi(struct xdp_frame *xdpf);
 void xdp_return_buff(struct xdp_buff *xdp);
+void xdp_flush_frame_bulk(struct xdp_frame_bulk *bq,
+			  bool napi_direct);
+void xdp_return_frame_bulk(struct xdp_frame *xdpf,
+			   struct xdp_frame_bulk *bq,
+			   bool napi_direct);
 
 /* When sending xdp_frame into the network stack, then there is no
  * return point callback, which is needed to release e.g. DMA-mapping

56 lines not on this page · raw message

index 81d7773f96cd..1330419efec7 100644
@@ -169,6 +169,8 @@ static inline void page_pool_release_page(struct page_pool *pool,
 
 void page_pool_put_page(struct page_pool *pool, struct page *page,
 			unsigned int dma_sync_size, bool allow_direct);
+void page_pool_put_page_bulk(struct page_pool *pool, void **data, int count,
+			     bool allow_direct);
 
 /* Same as above but will try to sync the entire area pool->max_len */
 static inline void page_pool_put_full_page(struct page_pool *pool,

@@ -215,4 +217,23 @@ static inline void page_pool_nid_changed(struct page_pool *pool, int new_nid)
 	if (unlikely(pool->p.nid != new_nid))
 		page_pool_update_nid(pool, new_nid);
 }
+
+static inline void page_pool_ring_lock(struct page_pool *pool)
+	__acquires(&pool->ring.producer_lock)
+{
+	if (in_serving_softirq())
+		spin_lock(&pool->ring.producer_lock);
+	else
+		spin_lock_bh(&pool->ring.producer_lock);
+}
+
+static inline void page_pool_ring_unlock(struct page_pool *pool)
+	__releases(&pool->ring.producer_lock)
+{
+	if (in_serving_softirq())
+		spin_unlock(&pool->ring.producer_lock);
+	else
+		spin_unlock_bh(&pool->ring.producer_lock);
+}
+
 #endif /* _NET_PAGE_POOL_H */

index ef98372facf6..03c3a92c9179 100644
@@ -11,6 +11,8 @@
 #include <linux/device.h>
 
 #include <net/page_pool.h>
+#include <net/xdp.h>
+
 #include <linux/dma-direction.h>
 #include <linux/dma-mapping.h>
 #include <linux/page-flags.h>

@@ -408,6 +410,41 @@ void page_pool_put_page(struct page_pool *pool, struct page *page,
 }
 EXPORT_SYMBOL(page_pool_put_page);
 
+void page_pool_put_page_bulk(struct page_pool *pool, void **data, int count,
+			     bool allow_direct)
+{
+	struct page *page_ring[XDP_BULK_QUEUE_SIZE];
+	int i, len = 0;
+
+	for (i = 0; i < count; i++) {
+		struct page *page = virt_to_head_page(data[i]);
+
+		if (unlikely(page_ref_count(page) != 1 ||
+			     !pool_page_reusable(pool, page))) {
+			page_pool_release_page(pool, page);
+			put_page(page);
+			continue;
+		}
+
+		if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV)
+			page_pool_dma_sync_for_device(pool, page, -1);
+
+		if (allow_direct && in_serving_softirq() &&
+		    page_pool_recycle_in_cache(page, pool))
+			continue;
+
+		page_ring[len++] = page;
+	}
+
+	page_pool_ring_lock(pool);
+	for (i = 0; i < len; i++) {
+		if (__ptr_ring_produce(&pool->ring, page_ring[i]))
+			page_pool_return_page(pool, page_ring[i]);
+	}
+	page_pool_ring_unlock(pool);
+}
+EXPORT_SYMBOL(page_pool_put_page_bulk);
+
 static void page_pool_empty_ring(struct page_pool *pool)
 {
 	struct page *page;

index b05467a916b4..7ebe159e3835 100644
@@ -384,14 +384,13 @@ void xdp_flush_frame_bulk(struct xdp_frame_bulk *bq,
 			  bool napi_direct)
 {
 	struct xdp_mem_allocator *xa = bq->xa;
-	int i;
 
-	for (i = 0; i < bq->count; i++) {
-		napi_direct &= !xdp_return_frame_no_direct();
-		page_pool_put_full_page(xa->page_pool,
-					virt_to_head_page(bq->q[i]),
-					napi_direct);
-	}
+	if (unlikely(!bq->count))
+		return;
+
+	napi_direct &= !xdp_return_frame_no_direct();
+	page_pool_put_page_bulk(xa->page_pool, bq->q, bq->count,
+				napi_direct);
 	bq->count = 0;
 }
 EXPORT_SYMBOL_GPL(xdp_flush_frame_bulk);