add multi-buff support for xdp running in generic mode
Subsystems (5)
v8 cover add multi-buff support for xdp running in generic mode
Introduce multi-buffer support for xdp running in generic mode not always linearizing the skb in netif_receive_generic_xdp routine. Introduce generic percpu page_pools allocator.
Changes since v7:
- fix sparse warnings
Changes since v6:
- remove patch 4/5 'net: page_pool: make stats available just for global pools'
- rename netif_skb_segment_for_xdp() in skb_cow_data_for_xdp()/skb_pp_cow_data()
- rename net_page_pool_alloc() in net_page_pool_create()
- rename page_pool percpu pointer in system_page_pool
- set percpu page_pool memory size
Changes since v5:
- move percpu page_pool pointer out of softnet_data in a dedicated variable
- make page_pool stats available just for global pools
- rely on netif_skb_segment_for_xdp utility routine in veth driver Changes since v4:
- fix compilation error if page_pools are not enabled
Changes since v3:
- introduce page_pool in softnet_data structure
- rely on page_pools for xdp_generic code
Changes since v2:
- rely on napi_alloc_frag() and napi_build_skb() to build the new skb Changes since v1:
- explicitly keep the skb segmented in netif_skb_check_for_generic_xdp() and do not rely on pskb_expand_head()
Lorenzo Bianconi (4):
net: add generic percpu page_pool allocator
xdp: rely on skb pointer reference in do_xdp_generic and
netif_receive_generic_xdp
xdp: add multi-buff support for xdp running in generic mode
veth: rely on skb_cow_data_for_xdp utility routine
drivers/net/tun.c | 4 +- drivers/net/veth.c | 79 ++------------------ include/linux/netdevice.h | 2 +- include/linux/skbuff.h | 2 + include/net/page_pool/types.h | 3 + net/core/dev.c | 131 +++++++++++++++++++++++++++------- net/core/page_pool.c | 23 ++++-- net/core/skbuff.c | 96 ++++++++++++++++++++++++- 8 files changed, 231 insertions(+), 109 deletions(-)
Signature · 1 line
2.43.0
PATCH v8 1/4 net: add generic percpu page_pool allocator
Introduce generic percpu page_pools allocator.
Moreover add page_pool_create_percpu() and cpuid filed in page_pool struct in order to recycle the page in the page_pool "hot" cache if napi_pp_put_page() is running on the same cpu.
This is a preliminary patch to add xdp multi-buff support for xdp running in generic mode.
--- include/net/page_pool/types.h | 3 +++ net/core/dev.c | 45 +++++++++++++++++++++++++++++++++++ net/core/page_pool.c | 23 ++++++++++++++---- net/core/skbuff.c | 5 ++-- 4 files changed, 70 insertions(+), 6 deletions(-)
include/net/page_pool/types.h +3 -0
@@ -128,6 +128,7 @@ struct page_pool_stats {struct page_pool {struct page_pool_params_fast p; +int cpuid;bool has_init_callback;long frag_users;
@@ -203,6 +204,8 @@ struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp);struct page *page_pool_alloc_frag (struct page_pool *pool,unsigned int *offset,unsigned int size,gfp_t gfp);struct page_pool *page_pool_create (const struct page_pool_params *params); +struct page_pool *page_pool_create_percpu (const struct page_pool_params *params, +int cpuid);struct xdp_mem_info ;
net/core/dev.c +45 -0
75 lines not on this page · raw message
net/core/page_pool.c +19 -4
@@ -171,13 +171,16 @@ static void page_pool_producer_unlock(struct page_pool *pool, }static int page_pool_init (struct page_pool *pool, -const struct page_pool_params *params) +const struct page_pool_params *params, +int cpuid) {unsigned int ring_qsize =1024 ;/* Default */ memcpy (&pool->p, ¶ms->fast,sizeof (pool->p));memcpy (&pool->slow, ¶ms->slow,sizeof (pool->slow)); + pool->cpuid = cpuid; +/* Validate only known flags were used */ if (pool->p.flags & ~(PP_FLAG_ALL ))return -EINVAL ;
@@ -253,10 +256,12 @@ static void page_pool_uninit(struct page_pool *pool) }/** -* page_pool_create() - create a page pool. +* page_pool_create_percpu() - create a page pool for a given cpu. * @params: parameters, see struct page_pool_params +* @cpuid: cpu identifier */ -struct page_pool *page_pool_create (const struct page_pool_params *params) +struct page_pool * +page_pool_create_percpu (const struct page_pool_params *params,int cpuid) {struct page_pool *pool;int err;
@@ -265,7 +270,7 @@ struct page_pool *page_pool_create(const struct page_pool_params *params)if (!pool)return ERR_PTR (-ENOMEM ); - err =page_pool_init (pool, params); + err =page_pool_init (pool, params, cpuid);if (err <0 ) goto err_free;
@@ -282,6 +287,16 @@ struct page_pool *page_pool_create(const struct page_pool_params *params)kfree (pool);return ERR_PTR (err); } +EXPORT_SYMBOL (page_pool_create_percpu); + +/** +* page_pool_create() - create a page pool +* @params: parameters, see struct page_pool_params +*/ +struct page_pool *page_pool_create (const struct page_pool_params *params) +{ +return page_pool_create_percpu (params,-1 ); +}EXPORT_SYMBOL (page_pool_create);static void page_pool_return_page (struct page_pool *pool,struct page *page);
net/core/skbuff.c +3 -2
@@ -923,9 +923,10 @@ bool napi_pp_put_page(struct page *page, bool napi_safe)*/ if (napi_safe ||in_softirq ()) {const struct napi_struct *napi =READ_ONCE (pp->p.napi); +unsigned int cpuid =smp_processor_id (); - allow_direct = napi && -READ_ONCE (napi->list_owner) ==smp_processor_id (); + allow_direct = napi &&READ_ONCE (napi->list_owner) == cpuid; + allow_direct |= (pp->cpuid == cpuid); }/* Driver set this to memory recycling info. Reset it on recycle.
Signature · 1 line
2.43.0
Re: [PATCH v8 net-next 1/4] net: add generic percpu page_pool allocator
Quoted 204 lines · Lorenzo Bianconi
Introduce generic percpu
page_poolsallocator.
Moreover addpage_pool_create_percpu()and cpuid filed inpage_poolstruct in order to recycle the page in thepage_pool"hot" cache ifnapi_pp_put_page()is running on the same cpu.
This is a preliminary patch to add xdp multi-buff support for xdp running in generic mode.Reviewed-by Toke Hoiland-Jorgensen <[email protected]>Signed-off-by Lorenzo Bianconi <[email protected]>--- include/net/page_pool/types.h | 3 +++ net/core/dev.c | 45 +++++++++++++++++++++++++++++++++++ net/core/page_pool.c | 23 ++++++++++++++---- net/core/skbuff.c | 5 ++-- 4 files changed, 70 insertions(+), 6 deletions(-)include/net/page_pool/types.h +3
-0@@ -128,6 +128,7 @@ struct page_pool_stats {struct page_pool {struct page_pool_params_fast p; +int cpuid;bool has_init_callback;long frag_users;@@ -203,6 +204,8 @@ struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp);struct page *page_pool_alloc_frag (struct page_pool *pool,unsigned int *offset,unsigned int size,gfp_t gfp);struct page_pool *page_pool_create (const struct page_pool_params *params); +struct page_pool *page_pool_create_percpu (const struct page_pool_params *params, +int cpuid);struct xdp_mem_info ;net/core/dev.c +45
-0@@ -153,6 +153,8 @@#include <linux/prandom.h> #include <linux/once_lite.h> #include <net/netdev_rx_queue.h> +#include <net/page_pool/types.h> +#include <net/page_pool/helpers.h> #include "dev.h" #include "net-sysfs.h" @@ -450,6 +452,12 @@ static RAW_NOTIFIER_HEAD(netdev_chain); DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);EXPORT_PER_CPU_SYMBOL (softnet_data); +/* Page_pool has a lockless array/stack to alloc/recycle pages. +* PP consumers must pay attention to run APIs in the appropriate context +* (e.g. NAPI context). +*/ +static DEFINE_PER_CPU_ALIGNED(struct page_pool *, system_page_pool); +#ifdef CONFIG_LOCKDEP /* * register_netdevice() inits txq->_xmit_lock and sets lockdep class @@ -11697,6 +11705,27 @@ static void __init net_dev_struct_check(void)* */ +/* We allocate 256 pages for each CPU if PAGE_SHIFT is 12 */ +#define SYSTEM_PERCPU_PAGE_POOL_SIZE ((1 << 20) / PAGE_SIZE) + +static int net_page_pool_create (int cpuid) +{ +#if IS_ENABLED (CONFIG_PAGE_POOL ) +struct page_pool_params page_pool_params = { + .pool_size =SYSTEM_PERCPU_PAGE_POOL_SIZE , + .nid =NUMA_NO_NODE , + }; +struct page_pool *pp_ptr; + + pp_ptr =page_pool_create_percpu (&page_pool_params, cpuid); +if (IS_ERR (pp_ptr)) +return -ENOMEM ; + +per_cpu (system_page_pool, cpuid) = pp_ptr; +#endif +return 0 ; +} +/* * This is called single threaded during boot, so no need * to take the rtnl semaphore. @@ -11749,6 +11778,9 @@ static int __init net_dev_init(void)init_gro_hash (&sd->backlog); sd->backlog.poll = process_backlog; sd->backlog.weight = weight_p; + +if (net_page_pool_create (i)) + goto out; } dev_boot_phase =0 ;@@ -11776,6 +11808,19 @@ static int __init net_dev_init(void)WARN_ON (rc <0 ); rc =0 ; out: +if (rc <0 ) { +for_each_possible_cpu (i) { +struct page_pool *pp_ptr; + + pp_ptr =per_cpu (system_page_pool, i); +if (!pp_ptr) +continue ; + +page_pool_destroy (pp_ptr); +per_cpu (system_page_pool, i) =NULL ; + } + } +return rc; }net/core/page_pool.c +19
-4@@ -171,13 +171,16 @@ static void page_pool_producer_unlock(struct page_pool *pool, }static int page_pool_init (struct page_pool *pool, -const struct page_pool_params *params) +const struct page_pool_params *params, +int cpuid) {unsigned int ring_qsize =1024 ;/* Default */ memcpy (&pool->p, ¶ms->fast,sizeof (pool->p));memcpy (&pool->slow, ¶ms->slow,sizeof (pool->slow)); + pool->cpuid = cpuid; +/* Validate only known flags were used */ if (pool->p.flags & ~(PP_FLAG_ALL ))return -EINVAL ;@@ -253,10 +256,12 @@ static void page_pool_uninit(struct page_pool *pool) }/** -* page_pool_create() - create a page pool. +* page_pool_create_percpu() - create a page pool for a given cpu. * @params: parameters, see struct page_pool_params +* @cpuid: cpu identifier */ -struct page_pool *page_pool_create (const struct page_pool_params *params) +struct page_pool * +page_pool_create_percpu (const struct page_pool_params *params,int cpuid) {struct page_pool *pool;int err;@@ -265,7 +270,7 @@ struct page_pool *page_pool_create(const struct page_pool_params *params)if (!pool)return ERR_PTR (-ENOMEM ); - err =page_pool_init (pool, params); + err =page_pool_init (pool, params, cpuid);if (err <0 ) goto err_free;@@ -282,6 +287,16 @@ struct page_pool *page_pool_create(const struct page_pool_params *params)kfree (pool);return ERR_PTR (err); } +EXPORT_SYMBOL (page_pool_create_percpu); + +/** +* page_pool_create() - create a page pool +* @params: parameters, see struct page_pool_params +*/ +struct page_pool *page_pool_create (const struct page_pool_params *params) +{ +return page_pool_create_percpu (params,-1 ); +}EXPORT_SYMBOL (page_pool_create);static void page_pool_return_page (struct page_pool *pool,struct page *page);net/core/skbuff.c +3
-3@@ -923,9 +923,10 @@ bool napi_pp_put_page(struct page *page, bool napi_safe)*/ if (napi_safe ||in_softirq ()) {const struct napi_struct *napi =READ_ONCE (pp->p.napi); +unsigned int cpuid =smp_processor_id (); - allow_direct = napi && -READ_ONCE (napi->list_owner) ==smp_processor_id (); + allow_direct = napi &&READ_ONCE (napi->list_owner) == cpuid; + allow_direct |= (pp->cpuid == cpuid); }/* Driver set this to memory recycling info. Reset it on recycle. --
Quoted 2 lines
2.43.0
PATCH v8 2/4 xdp: rely on skb pointer reference in do_xdp_generic and netif_receive_generic_xdp
In reply to Lorenzo BianconiRely on skb pointer reference instead of the skb pointer in do_xdp_generic and netif_receive_generic_xdp routine signatures.
This is a preliminary patch to add multi-buff support for xdp running in generic mode where we will need to reallocate the skb to avoid linearization and we will need to make it visible to do_xdp_generic() caller.
--- drivers/net/tun.c | 4 ++-- include/linux/netdevice.h | 2 +- net/core/dev.c | 16 +++++++++------- 3 files changed, 12 insertions(+), 10 deletions(-)
drivers/net/tun.c +2 -2
@@ -1926,7 +1926,7 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,rcu_read_lock (); xdp_prog =rcu_dereference (tun->xdp_prog);if (xdp_prog) { - ret =do_xdp_generic (xdp_prog, skb); + ret =do_xdp_generic (xdp_prog, &skb);if (ret !=XDP_PASS ) {rcu_read_unlock ();local_bh_enable ();
@@ -2516,7 +2516,7 @@ static int tun_xdp_one(struct tun_struct *tun,skb_record_rx_queue (skb, tfile->queue_index);if (skb_xdp) { - ret =do_xdp_generic (xdp_prog, skb); + ret =do_xdp_generic (xdp_prog, &skb);if (ret !=XDP_PASS ) { ret =0 ; goto out;
include/linux/netdevice.h +1 -1
@@ -3958,7 +3958,7 @@ static inline void dev_consume_skb_any(struct sk_buff *skb)u32 bpf_prog_run_generic_xdp (struct sk_buff *skb,struct xdp_buff *xdp,struct bpf_prog *xdp_prog);void generic_xdp_tx (struct sk_buff *skb,struct bpf_prog *xdp_prog); -int do_xdp_generic (struct bpf_prog *xdp_prog,struct sk_buff *skb); +int do_xdp_generic (struct bpf_prog *xdp_prog,struct sk_buff **pskb);int netif_rx (struct sk_buff *skb);int __netif_rx (struct sk_buff *skb);
net/core/dev.c +9 -7
@@ -4936,10 +4936,11 @@ u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp,return act; } -static u32 netif_receive_generic_xdp (struct sk_buff *skb, +static u32 netif_receive_generic_xdp (struct sk_buff **pskb,struct xdp_buff *xdp,struct bpf_prog *xdp_prog) { +struct sk_buff *skb = *pskb;u32 act =XDP_DROP ;/* Reinjected packets coming from act_mirred or similar should
@@ -5020,24 +5021,24 @@ void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog)static DEFINE_STATIC_KEY_FALSE (generic_xdp_needed_key); -int do_xdp_generic (struct bpf_prog *xdp_prog,struct sk_buff *skb) +int do_xdp_generic (struct bpf_prog *xdp_prog,struct sk_buff **pskb) {if (xdp_prog) {struct xdp_buff xdp;u32 act;int err; - act =netif_receive_generic_xdp (skb, &xdp, xdp_prog); + act =netif_receive_generic_xdp (pskb, &xdp, xdp_prog);if (act !=XDP_PASS ) {switch (act) {case XDP_REDIRECT : - err =xdp_do_generic_redirect (skb->dev, skb, + err =xdp_do_generic_redirect ((*pskb)->dev, *pskb, &xdp, xdp_prog);if (err) goto out_redir;break ;case XDP_TX : -generic_xdp_tx (skb, xdp_prog); +generic_xdp_tx (*pskb, xdp_prog);break ; }return XDP_DROP ;
@@ -5045,7 +5046,7 @@ int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb) }return XDP_PASS ; out_redir: -kfree_skb_reason (skb,SKB_DROP_REASON_XDP ); +kfree_skb_reason (*pskb,SKB_DROP_REASON_XDP );return XDP_DROP ; }EXPORT_SYMBOL_GPL (do_xdp_generic);
@@ -5368,7 +5369,8 @@ static int __netif_receive_skb_core(struct sk_buff **pskb, bool pfmemalloc,int ret2;migrate_disable (); - ret2 =do_xdp_generic (rcu_dereference (skb->dev->xdp_prog), skb); + ret2 =do_xdp_generic (rcu_dereference (skb->dev->xdp_prog), + &skb);migrate_enable ();if (ret2 !=XDP_PASS ) {
Signature · 1 line
2.43.0
PATCH v8 3/4 xdp: add multi-buff support for xdp running in generic mode
In reply to Lorenzo BianconiSimilar to native xdp, do not always linearize the skb in netif_receive_generic_xdp routine but create a non-linear xdp_buff to be processed by the eBPF program. This allow to add multi-buffer support for xdp running in generic mode.
--- include/linux/skbuff.h | 2 + net/core/dev.c | 70 +++++++++++++++++++++++--------- net/core/skbuff.c | 91 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+), 19 deletions(-)
include/linux/skbuff.h +2 -0
@@ -3446,6 +3446,8 @@ static inline void skb_frag_ref(struct sk_buff *skb, int f)__skb_frag_ref (&skb_shinfo (skb)->frags[f]); } +int skb_cow_data_for_xdp (struct page_pool *pool,struct sk_buff **pskb, +struct bpf_prog *prog);bool napi_pp_put_page (struct page *page,bool napi_safe); static inline void
net/core/dev.c +51 -19
116 lines not on this page · raw message
net/core/skbuff.c +91 -0
97 lines not on this page · raw message
Signature · 1 line
2.43.0
PATCH v8 4/4 veth: rely on skb_cow_data_for_xdp utility routine
In reply to Lorenzo BianconiRely on skb_cow_data_for_xdp utility routine and remove duplicated code.
--- drivers/net/veth.c | 79 +++------------------------------------------- 1 file changed, 5 insertions(+), 74 deletions(-)
drivers/net/veth.c +5 -74
@@ -721,7 +721,8 @@ static void veth_xdp_get(struct xdp_buff *xdp)static int veth_convert_skb_to_xdp_buff (struct veth_rq *rq,struct xdp_buff *xdp, -struct sk_buff **pskb) +struct sk_buff **pskb, +struct bpf_prog *prog) {struct sk_buff *skb = *pskb;u32 frame_sz;
@@ -729,80 +730,10 @@ static int veth_convert_skb_to_xdp_buff(struct veth_rq *rq,if (skb_shared (skb) ||skb_head_is_locked (skb) ||skb_shinfo (skb)->nr_frags ||skb_headroom (skb) <XDP_PACKET_HEADROOM ) { -u32 size, len, max_head_size, off, truesize, page_offset; -struct sk_buff *nskb; -struct page *page; -int i, head_off; -void *va; - -/* We need a private copy of the skb and data buffers since -* the ebpf program can modify it. We segment the original skb -* into order-0 pages without linearize it. -* -* Make sure we have enough space for linear and paged area -*/ - max_head_size =SKB_WITH_OVERHEAD (PAGE_SIZE - -VETH_XDP_HEADROOM ); -if (skb->len >PAGE_SIZE *MAX_SKB_FRAGS + max_head_size) - goto drop; - - size =min_t (u32, skb->len, max_head_size); - truesize =SKB_HEAD_ALIGN (size) +VETH_XDP_HEADROOM ; - -/* Allocate skb head */ - va =page_pool_dev_alloc_va (rq->page_pool, &truesize); -if (!va) - goto drop; - - nskb =napi_build_skb (va, truesize); -if (!nskb) { -page_pool_free_va (rq->page_pool, va, true); +if (skb_cow_data_for_xdp (rq->page_pool, pskb, prog)) goto drop; - } - -skb_reserve (nskb,VETH_XDP_HEADROOM ); -skb_copy_header (nskb, skb); -skb_mark_for_recycle (nskb); - -if (skb_copy_bits (skb,0 , nskb->data, size)) { -consume_skb (nskb); - goto drop; - } -skb_put (nskb, size); - head_off =skb_headroom (nskb) -skb_headroom (skb); -skb_headers_offset_update (nskb, head_off); - -/* Allocate paged area of new skb */ - off = size; - len = skb->len - off; - -for (i =0 ; i <MAX_SKB_FRAGS && off < skb->len; i++) { - size =min_t (u32, len,PAGE_SIZE ); - truesize = size; - - page =page_pool_dev_alloc (rq->page_pool, &page_offset, - &truesize); -if (!page) { -consume_skb (nskb); - goto drop; - } - -skb_add_rx_frag (nskb, i, page, page_offset, size, - truesize); -if (skb_copy_bits (skb, off, -page_address (page) + page_offset, - size)) { -consume_skb (nskb); - goto drop; - } - - len -= size; - off += size; - } - -consume_skb (skb); - skb = nskb; + skb = *pskb; }/* SKB "head" area always have tailroom for skb_shared_info */
@@ -850,7 +781,7 @@ static struct sk_buff *veth_xdp_rcv_skb(struct veth_rq *rq, }__skb_push (skb, skb->data -skb_mac_header (skb)); -if (veth_convert_skb_to_xdp_buff (rq, xdp, &skb)) +if (veth_convert_skb_to_xdp_buff (rq, xdp, &skb, xdp_prog)) goto drop; vxbuf.skb = skb;
Signature · 1 line
2.43.0
Jakub Kicinski
maintainerQuoted 3 lines · On Mon, 5 Feb 2024 12:35:11 +0100 Lorenzo Bianconi wrote:
Introduce multi-buffer support for xdp running in generic mode not always linearizing the skb in
netif_receive_generic_xdproutine. Introduce generic percpupage_poolsallocator.
breaks the veth test, apparently:
https://netdev-3.bots.linux.dev/vmksft-net/results/458181/60-veth-sh/stdout
could be that the test needs fixing not the code.
But either way we need a respin :(
Signature · 1 line
pw-bot: cr
Quoted 5 lines · On Mon, 5 Feb 2024 12:35:11 +0100 Lorenzo Bianconi wrote:
Quoted 3 lines
Introduce multi-buffer support for xdp running in generic mode not always linearizing the skb in
netif_receive_generic_xdproutine. Introduce generic percpupage_poolsallocator.
Quoted 8 lines
breaks the veth test, apparently:
https://netdev-3.bots.linux.dev/vmksft-net/results/458181/60-veth-sh/stdout
could be that the test needs fixing not the code.
But either way we need a respin :(Signature · 1 line
pw-bot: cr
Hi Jakub,
Ack, thx for reporting the problem. The issue is we should use skb_pp_cow_data() instead of skb_cow_data_for_xdp() in veth_convert_skb_to_xdp_buff() since we do not have any requirement for the attached bpf_prog there. I will fix it in v9.
Regards,
Lorenzo