page_pool: allow caching from safely localized NAPI
Subsystems (5)
Jakub Kicinski
maintainercover page_pool: allow caching from safely localized NAPI
I went back to the explicit "are we in NAPI method", mostly because I don't like having both around :( (even tho I maintain that in_softirq() && !in_hardirq() is as safe, as softirqs do not nest).
Still returning the skbs to a CPU, tho, not to the NAPI instance. I reckon we could create a small refcounted struct per NAPI instance which would allow sockets and other users so hold a persisent and safe reference. But that's a bigger change, and I get 90+% recycling thru the cache with just these patches (for RR and streaming tests with 100% CPU use it's almost 100%).
Some numbers for streaming test with 100% CPU use (from previous version, but really they perform the same):
HW-GRO page=page before after before after
recycle:
cached: 0 138669686 0 150197505cache_full: 0 223391 0 74582
ring: 138551933 9997191 149299454 0ring_full: 0 488 3154 127590released_refcnt: 0 0 0 0
alloc:
fast: 136491361 148615710 146969587 150322859
slow: 1772 1799 144 105slow_high_order: 0 0 0 0
empty: 1772 1799 144 105
refill: 2165245 156302 2332880 2128
waive: 0 0 0 0
v1:
- rename the arg in_normal_napi -> napi_safe
- also allow recycling in __kfree_skb_defer()
rfcv2: https://lore.kernel.org/all/[email protected]/
Jakub Kicinski (3):
net: skb: plumb napi state thru skb freeing paths
page_pool: allow caching from safely localized NAPI
bnxt: hook NAPIs to page pools
Documentation/networking/page_pool.rst | 1 + drivers/net/ethernet/broadcom/bnxt/bnxt.c | 1 + include/linux/netdevice.h | 3 ++ include/linux/skbuff.h | 20 +++++++---- include/net/page_pool.h | 3 +- net/core/dev.c | 3 ++ net/core/page_pool.c | 15 ++++++-- net/core/skbuff.c | 42 ++++++++++++----------- 8 files changed, 58 insertions(+), 30 deletions(-)
Signature · 1 line
2.39.2
Jakub Kicinski
maintainerPATCH 1/3 net: skb: plumb napi state thru skb freeing paths
We maintain a NAPI-local cache of skbs which is fed by napi_consume_skb(). Going forward we will also try to cache head and data pages. Plumb the "are we in a normal NAPI context" information thru deeper into the freeing path, up to skb_release_data() and skb_free_head()/skb_pp_recycle().
Use "bool in_normal_napi" rather than bare "int budget", the further we get from NAPI the more confusing the budget argument may seem (particularly whether 0 or MAX is the correct value to pass in when not in NAPI).
---
v1:
- feed the cache in __kfree_skb_defer(), it's in NAPI
- s/in_normal_napi/napi_safe/
--- net/core/skbuff.c | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-)
net/core/skbuff.c +20 -18
@@ -839,7 +839,7 @@ static void skb_clone_fraglist(struct sk_buff *skb)skb_get (list); } -static bool skb_pp_recycle (struct sk_buff *skb,void *data) +static bool skb_pp_recycle (struct sk_buff *skb,void *data,bool napi_safe) {if (!IS_ENABLED (CONFIG_PAGE_POOL ) || !skb->pp_recycle)return false;
@@ -856,12 +856,12 @@ static void skb_kfree_head(void *head, unsigned int end_offset)kfree (head); } -static void skb_free_head (struct sk_buff *skb) +static void skb_free_head (struct sk_buff *skb,bool napi_safe) {unsigned char *head = skb->head;if (skb->head_frag) { -if (skb_pp_recycle (skb, head)) +if (skb_pp_recycle (skb, head, napi_safe))return ;skb_free_frag (head); }else {
@@ -869,7 +869,8 @@ static void skb_free_head(struct sk_buff *skb) } } -static void skb_release_data (struct sk_buff *skb,enum skb_drop_reason reason) +static void skb_release_data (struct sk_buff *skb,enum skb_drop_reason reason, +bool napi_safe) {struct skb_shared_info *shinfo =skb_shinfo (skb);int i;
@@ -894,7 +895,7 @@ static void skb_release_data(struct sk_buff *skb, enum skb_drop_reason reason)if (shinfo->frag_list)kfree_skb_list_reason (shinfo->frag_list, reason); -skb_free_head (skb); +skb_free_head (skb, napi_safe); exit:/* When we clone an SKB we copy the reycling bit. The pp_recycle * bit is only set on the head though, so in order to avoid races
@@ -955,11 +956,12 @@ void skb_release_head_state(struct sk_buff *skb) }/* Free everything but the sk_buff shell. */ -static void skb_release_all (struct sk_buff *skb,enum skb_drop_reason reason) +static void skb_release_all (struct sk_buff *skb,enum skb_drop_reason reason, +bool napi_safe) {skb_release_head_state (skb);if (likely (skb->head)) -skb_release_data (skb, reason); +skb_release_data (skb, reason, napi_safe); }/**
@@ -973,7 +975,7 @@ static void skb_release_all(struct sk_buff *skb, enum skb_drop_reason reason)void __kfree_skb (struct sk_buff *skb) { -skb_release_all (skb,SKB_DROP_REASON_NOT_SPECIFIED ); +skb_release_all (skb,SKB_DROP_REASON_NOT_SPECIFIED , false);kfree_skbmem (skb); }EXPORT_SYMBOL (__kfree_skb);
@@ -1027,7 +1029,7 @@ static void kfree_skb_add_bulk(struct sk_buff *skb,return ; } -skb_release_all (skb, reason); +skb_release_all (skb, reason, false); sa->skb_array[sa->skb_count++] = skb;if (unlikely (sa->skb_count ==KFREE_SKB_BULK_SIZE )) {
@@ -1201,7 +1203,7 @@ EXPORT_SYMBOL(consume_skb);void __consume_stateless_skb (struct sk_buff *skb) {trace_consume_skb (skb,__builtin_return_address (0 )); -skb_release_data (skb,SKB_CONSUMED ); +skb_release_data (skb,SKB_CONSUMED , false);kfree_skbmem (skb); }
@@ -1226,7 +1228,7 @@ static void napi_skb_cache_put(struct sk_buff *skb)void __kfree_skb_defer (struct sk_buff *skb) { -skb_release_all (skb,SKB_DROP_REASON_NOT_SPECIFIED ); +skb_release_all (skb,SKB_DROP_REASON_NOT_SPECIFIED , true);napi_skb_cache_put (skb); }
@@ -1264,7 +1266,7 @@ void napi_consume_skb(struct sk_buff *skb, int budget)return ; } -skb_release_all (skb,SKB_CONSUMED ); +skb_release_all (skb,SKB_CONSUMED , !!budget);napi_skb_cache_put (skb); }EXPORT_SYMBOL (napi_consume_skb);
@@ -1395,7 +1397,7 @@ EXPORT_SYMBOL_GPL(alloc_skb_for_msg);*/ struct sk_buff *skb_morph (struct sk_buff *dst,struct sk_buff *src) { -skb_release_all (dst,SKB_CONSUMED ); +skb_release_all (dst,SKB_CONSUMED , false);return __skb_clone (dst, src); }EXPORT_SYMBOL_GPL (skb_morph);
@@ -2018,9 +2020,9 @@ int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,if (skb_has_frag_list (skb))skb_clone_fraglist (skb); -skb_release_data (skb,SKB_CONSUMED ); +skb_release_data (skb,SKB_CONSUMED , false); }else { -skb_free_head (skb); +skb_free_head (skb, false); } off = (data + nhead) - skb->head;
@@ -6389,12 +6391,12 @@ static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off,skb_frag_ref (skb, i);if (skb_has_frag_list (skb))skb_clone_fraglist (skb); -skb_release_data (skb,SKB_CONSUMED ); +skb_release_data (skb,SKB_CONSUMED , false); }else {/* we can reuse existing recount- all we did was * relocate values */ -skb_free_head (skb); +skb_free_head (skb, false); } skb->head = data;
@@ -6529,7 +6531,7 @@ static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off,skb_kfree_head (data, size);return -ENOMEM ; } -skb_release_data (skb,SKB_CONSUMED ); +skb_release_data (skb,SKB_CONSUMED , false); skb->head = data; skb->head_frag =0 ;
Signature · 1 line
2.39.2
Jakub Kicinski
maintainerPATCH 3/3 bnxt: hook NAPIs to page pools
In reply to Jakub Kicinskibnxt has 1:1 mapping of page pools and NAPIs, so it's safe to hoook them up together.
---
CC: [email protected]
--- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 1 + 1 file changed, 1 insertion(+)
drivers/net/ethernet/broadcom/bnxt/bnxt.c +1 -0
@@ -3211,6 +3211,7 @@ static int bnxt_alloc_rx_page_pool(struct bnxt *bp, pp.pool_size = bp->rx_ring_size; pp.nid =dev_to_node (&bp->pdev->dev); + pp.napi = &rxr->bnapi->napi; pp.dev = &bp->pdev->dev; pp.dma_dir =DMA_BIDIRECTIONAL ;
Signature · 1 line
2.39.2
Jakub Kicinski
maintainerPATCH 2/3 page_pool: allow caching from safely localized NAPI
In reply to Jakub KicinskiRecent patches to mlx5 mentioned a regression when moving from driver local page pool to only using the generic page pool code. Page pool has two recycling paths (1) direct one, which runs in safe NAPI context (basically consumer context, so producing can be lockless); and (2) via a ptr_ring, which takes a spin lock because the freeing can happen from any CPU; producer and consumer may run concurrently.
Since the page pool code was added, Eric introduced a revised version of deferred skb freeing. TCP skbs are now usually returned to the CPU which allocated them, and freed in softirq context. This places the freeing (producing of pages back to the pool) enticingly close to the allocation (consumer).
If we can prove that we're freeing in the same softirq context in which the consumer NAPI will run - lockless use of the cache is perfectly fine, no need for the lock.
Let drivers link the page pool to a NAPI instance. If the NAPI instance is scheduled on the same CPU on which we're freeing - place the pages in the direct cache.
With that and patched bnxt (XDP enabled to engage the page pool, sigh, bnxt really needs page pool work :() I see a 2.6% perf boost with a TCP stream test (app on a different physical core than softirq).
The CPU use of relevant functions decreases as expected:
page_pool_refill_alloc_cache 1.17% -> 0%
_raw_spin_lock 2.41% -> 0.98%
Only consider lockless path to be safe when NAPI is scheduled
- in practice this should cover majority if not all of steady state workloads. It's usually the NAPI kicking in that causes the skb flush.
The main case we'll miss out on is when application runs on the same CPU as NAPI. In that case we don't use the deferred skb free path.
---
v1:
- s/in_napi/napi_safe/
rfc v2: https://lore.kernel.org/all/[email protected]/
- plumb thru "are we in NAPI" bool rather than guessing based on softirq && !hardirq
--- Documentation/networking/page_pool.rst | 1 + include/linux/netdevice.h | 3 +++ include/linux/skbuff.h | 20 +++++++++++++------- include/net/page_pool.h | 3 ++- net/core/dev.c | 3 +++ net/core/page_pool.c | 15 +++++++++++++-- net/core/skbuff.c | 4 ++-- 7 files changed, 37 insertions(+), 12 deletions(-)
Documentation/networking/page_pool.rst +1 -0
@@ -165,6 +165,7 @@ Registration pp_params.pool_size = DESC_NUM; pp_params.nid = NUMA_NO_NODE; pp_params.dev = priv->dev; + pp_params.napi = napi; /* only if locking is tied to NAPI */ pp_params.dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; page_pool = page_pool_create(&pp_params);
include/linux/netdevice.h +3 -0
@@ -360,8 +360,11 @@ struct napi_struct {unsigned long gro_bitmask;int (*poll)(struct napi_struct *,int );#ifdef CONFIG_NETPOLL +/* CPU actively polling if netpoll is configured */ int poll_owner;#endif +/* CPU on which NAPI has been scheduled for processing */ +int list_owner;struct net_device *dev;struct gro_list gro_hash[GRO_HASH_BUCKETS ];struct sk_buff *skb;
include/linux/skbuff.h +13 -7
@@ -3386,6 +3386,18 @@ static inline void skb_frag_ref(struct sk_buff *skb, int f)__skb_frag_ref (&skb_shinfo (skb)->frags[f]); } +static inline void +napi_frag_unref (skb_frag_t *frag,bool recycle,bool napi_safe) +{ +struct page *page =skb_frag_page (frag); + +#ifdef CONFIG_PAGE_POOL +if (recycle &&page_pool_return_skb_page (page, napi_safe)) +return ; +#endif +put_page (page); +} +/** * __skb_frag_unref - release a reference on a paged fragment. * @frag: the paged fragment
@@ -3396,13 +3408,7 @@ static inline void skb_frag_ref(struct sk_buff *skb, int f)*/ static inline void __skb_frag_unref (skb_frag_t *frag,bool recycle) { -struct page *page =skb_frag_page (frag); - -#ifdef CONFIG_PAGE_POOL -if (recycle &&page_pool_return_skb_page (page)) -return ; -#endif -put_page (page); +napi_frag_unref (frag, recycle, false); }/**
include/net/page_pool.h +2 -1
@@ -77,6 +77,7 @@ struct page_pool_params {unsigned int pool_size;int nid;/* Numa node id to allocate from pages from */ struct device *dev;/* device, for DMA pre-mapping purposes */ +struct napi_struct *napi;/* Sole consumer of pages, otherwise NULL */ enum dma_data_direction dma_dir;/* DMA mapping direction */ unsigned int max_len;/* max DMA sync memory size */ unsigned int offset;/* DMA addr offset */
@@ -239,7 +240,7 @@ inline enum dma_data_direction page_pool_get_dma_dir(struct page_pool *pool)return pool->p.dma_dir; } -bool page_pool_return_skb_page (struct page *page); +bool page_pool_return_skb_page (struct page *page,bool napi_safe);struct page_pool *page_pool_create (const struct page_pool_params *params);
net/core/dev.c +3 -0
@@ -4358,6 +4358,7 @@ static inline void ____napi_schedule(struct softnet_data *sd, }list_add_tail (&napi->poll_list, &sd->poll_list); +WRITE_ONCE (napi->list_owner,smp_processor_id ());/* If not called from net_rx_action() * we have to raise NET_RX_SOFTIRQ. */
@@ -6068,6 +6069,7 @@ bool napi_complete_done(struct napi_struct *n, int work_done)list_del_init (&n->poll_list);local_irq_restore (flags); } +WRITE_ONCE (n->list_owner,-1 ); val =READ_ONCE (n->state); do {
@@ -6383,6 +6385,7 @@ void netif_napi_add_weight(struct net_device *dev, struct napi_struct *napi,#ifdef CONFIG_NETPOLL napi->poll_owner =-1 ;#endif + napi->list_owner =-1 ;set_bit (NAPI_STATE_SCHED , &napi->state);set_bit (NAPI_STATE_NPSVC , &napi->state);list_add_rcu (&napi->dev_list, &dev->napi_list);
net/core/page_pool.c +13 -2
@@ -19,6 +19,7 @@#include <linux/mm.h> /* for put_page() */ #include <linux/poison.h> #include <linux/ethtool.h> +#include <linux/netdevice.h> #include <trace/events/page_pool.h>
@@ -874,9 +875,11 @@ void page_pool_update_nid(struct page_pool *pool, int new_nid) }EXPORT_SYMBOL (page_pool_update_nid); -bool page_pool_return_skb_page (struct page *page) +bool page_pool_return_skb_page (struct page *page,bool napi_safe) { +struct napi_struct *napi;struct page_pool *pp; +bool allow_direct; page =compound_head (page);
@@ -892,12 +895,20 @@ bool page_pool_return_skb_page(struct page *page) pp = page->pp; +/* Allow direct recycle if we have reasons to believe that we are +* in the same context as the consumer would run, so there's +* no possible race. +*/ + napi = pp->p.napi; + allow_direct = napi_safe && napi && +READ_ONCE (napi->list_owner) ==smp_processor_id (); +/* Driver set this to memory recycling info. Reset it on recycle. * This will *not* work for NIC using a split-page memory model. * The page will be returned to the pool here regardless of the * 'flipped' fragment being in use or not. */ -page_pool_put_full_page (pp, page, false); +page_pool_put_full_page (pp, page, allow_direct);return true; }
net/core/skbuff.c +2 -2
@@ -843,7 +843,7 @@ static bool skb_pp_recycle(struct sk_buff *skb, void *data, bool napi_safe) {if (!IS_ENABLED (CONFIG_PAGE_POOL ) || !skb->pp_recycle)return false; -return page_pool_return_skb_page (virt_to_page (data)); +return page_pool_return_skb_page (virt_to_page (data), napi_safe); }static void skb_kfree_head (void *head,unsigned int end_offset)
@@ -889,7 +889,7 @@ static void skb_release_data(struct sk_buff *skb, enum skb_drop_reason reason, }for (i =0 ; i < shinfo->nr_frags; i++) -__skb_frag_unref (&shinfo->frags[i], skb->pp_recycle); +napi_frag_unref (&shinfo->frags[i], skb->pp_recycle, napi_safe); free_head:if (shinfo->frag_list)
Signature · 1 line
2.39.2
Hi,
Happy holidays! I was traveling and couldn't review the RFCs.
Dragos and I had this task in our plans after our recent page pool work in mlx5e. We're so glad to see this patchset! :)
Quoted 15 lines · Jakub Kicinski
I went back to the explicit "are we in NAPI method", mostly because I don't like having both around :( (even tho I maintain that
in_softirq()&& !in_hardirq() is as safe, as softirqs do not nest).Still returning the skbs to a CPU, tho, not to the NAPI instance. I reckon we could create a small refcounted struct per NAPI instance which would allow sockets and other users so hold a persisent and safe reference. But that's a bigger change, and I get 90+% recycling thru the cache with just these patches (for RR and streaming tests with 100% CPU use it's almost 100%).
Some numbers for streaming test with 100% CPU use (from previous version, but really they perform the same):
Quoted 9 lines
HW-GRO page=page before after before afterrecycle:
cached: 0 138669686 0 150197505cache_full: 0 223391 0 74582
ring: 138551933 9997191 149299454 0ring_full: 0 488 3154 127590released_refcnt: 0 0 0 0
Impressive.
Dragos tested your RFC v1.
He can test this one as well, expecting the same effect.
Quoted 8 lines
alloc:
fast: 136491361 148615710 146969587 150322859
slow: 1772 1799 144 105slow_high_order: 0 0 0 0
empty: 1772 1799 144 105
refill: 2165245 156302 2332880 2128
waive: 0 0 0 0
General note:
For fragmented page-pool pages, the decision to whether go though the cache or the ring depends only on the last releasing thread and its context.
Our in-driver deferred release trick is in many cases beneficial even in the presence of this idea. The sets of cases improved by each idea intersect, but are not completely identical. That's why we decided to go with both solutions working together, and not only one.
Quoted 20 lines
v1:
- rename the argin_normal_napi->napi_safe
- also allow recycling in__kfree_skb_defer()
rfcv2: https://lore.kernel.org/all/[email protected]/Jakub Kicinski (3):
net: skb: plumb napi state thru skb freeing paths
page_pool: allow caching from safely localized NAPI
bnxt: hook NAPIs to page poolsDocumentation/networking/page_pool.rst | 1 + drivers/net/ethernet/broadcom/bnxt/bnxt.c | 1 + include/linux/netdevice.h | 3 ++ include/linux/skbuff.h | 20 +++++++---- include/net/page_pool.h | 3 +- net/core/dev.c | 3 ++ net/core/page_pool.c | 15 ++++++-- net/core/skbuff.c | 42 ++++++++++++----------- 8 files changed, 58 insertions(+), 30 deletions(-)
Tariq Toukan
Reviewed-byQuoted 46 lines · Tariq Toukan
Hi,
Happy holidays! I was traveling and couldn't review the RFCs.
Dragos and I had this task in our plans after our recent page pool work in mlx5e. We're so glad to see this patchset! :)
Quoted 15 lines · Jakub Kicinski
I went back to the explicit "are we in NAPI method", mostly because I don't like having both around :( (even tho I maintain that
in_softirq()&& !in_hardirq() is as safe, as softirqs do not nest).Still returning the skbs to a CPU, tho, not to the NAPI instance. I reckon we could create a small refcounted struct per NAPI instance which would allow sockets and other users so hold a persisent and safe reference. But that's a bigger change, and I get 90+% recycling thru the cache with just these patches (for RR and streaming tests with 100% CPU use it's almost 100%).
Some numbers for streaming test with 100% CPU use (from previous version, but really they perform the same):
Quoted 9 lines
HW-GRO page=page
before after before after recycle:
cached: 0 138669686 0 150197505 cache_full: 0 223391 0 74582 ring: 138551933 9997191 149299454 0 ring_full: 0 488 3154 127590 released_refcnt: 0 0 0 0Impressive.
Dragos tested your RFC v1.
He can test this one as well, expecting the same effect.Quoted 8 lines
alloc:
fast: 136491361 148615710 146969587 150322859 slow: 1772 1799 144 105 slow_high_order: 0 0 0 0 empty: 1772 1799 144 105 refill: 2165245 156302 2332880 2128 waive: 0 0 0 0
Quoted 27 lines
General note:
For fragmented page-pool pages, the decision to whether go though the cache or the ring depends only on the last releasing thread and its context.Our in-driver deferred release trick is in many cases beneficial even in the presence of this idea. The sets of cases improved by each idea intersect, but are not completely identical. That's why we decided to go with both solutions working together, and not only one.
Quoted 20 lines
v1:
- rename the argin_normal_napi->napi_safe
- also allow recycling in__kfree_skb_defer()
rfcv2: https://lore.kernel.org/all/[email protected]/Jakub Kicinski (3):
net: skb: plumb napi state thru skb freeing paths
page_pool: allow caching from safely localized NAPI
bnxt: hook NAPIs to page poolsDocumentation/networking/page_pool.rst | 1 +
drivers/net/ethernet/broadcom/bnxt/bnxt.c| 1 + include/linux/netdevice.h | 3 ++ include/linux/skbuff.h | 20 +++++++---- include/net/page_pool.h | 3 +- net/core/dev.c | 3 ++ net/core/page_pool.c | 15 ++++++-- net/core/skbuff.c | 42 ++++++++++++-----------8 files changed, 58 insertions(+), 30 deletions(-)
We'll push a patch to enable it in mlx5e immediately once this is accepted.
Re: [PATCH net-next 1/3] net: skb: plumb napi state thru skb freeing paths
In reply to Jakub KicinskiQuoted 7 lines · Jakub Kicinski
We maintain a NAPI-local cache of skbs which is fed by
napi_consume_skb(). Going forward we will also try to cache head and data pages. Plumb the "are we in a normal NAPI context" information thru deeper into the freeing path, up toskb_release_data()and skb_free_head()/skb_pp_recycle().Use "bool
in_normal_napi" rather than bare "int budget",
The code was changed to "napi_safe", the desc should reflect this ;-)
Quoted 3 lines
the further we get from NAPI the more confusing the budget argument may seem (particularly whether 0 or MAX is the correct value to pass in when not in NAPI).
I do like the code cleanup.
It is worth explaining/mentioning that where budget==0 comes from?
(Cc. Alex, please correct me.)
My understanding is that this is caused by netconsole/netpoll (see net/core/netpoll.c func poll_one_napi()), which is a kernel (net)console debugging facility sending UDP packets via using only TX side of napi_poll. Thus, we are really trying to protect against doing these recycle tricks for when netpoll/netconcole is running (which I guess makes sense as we are likely printing/sending an OOPS msg over UDP).
Quoted 10 lines
Signed-off-by Jakub Kicinski <[email protected]>---
v1:
- feed the cache in__kfree_skb_defer(), it's in NAPI
- s/in_normal_napi/napi_safe/--- net/core/skbuff.c | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-)
(no more comments below, but didn't cut code for Alex to see)
Quoted 152 lines
net/core/skbuff.c +20
-18@@ -839,7 +839,7 @@ static void skb_clone_fraglist(struct sk_buff *skb)skb_get (list); } -static bool skb_pp_recycle (struct sk_buff *skb,void *data) +static bool skb_pp_recycle (struct sk_buff *skb,void *data,bool napi_safe) {if (!IS_ENABLED (CONFIG_PAGE_POOL ) || !skb->pp_recycle)return false;@@ -856,12 +856,12 @@ static void skb_kfree_head(void *head, unsigned int end_offset)kfree (head); } -static void skb_free_head (struct sk_buff *skb) +static void skb_free_head (struct sk_buff *skb,bool napi_safe) {unsigned char *head = skb->head;if (skb->head_frag) { -if (skb_pp_recycle (skb, head)) +if (skb_pp_recycle (skb, head, napi_safe))return ;skb_free_frag (head); }else {@@ -869,7 +869,8 @@ static void skb_free_head(struct sk_buff *skb) } } -static void skb_release_data (struct sk_buff *skb,enum skb_drop_reason reason) +static void skb_release_data (struct sk_buff *skb,enum skb_drop_reason reason, +bool napi_safe) {struct skb_shared_info *shinfo =skb_shinfo (skb);int i;@@ -894,7 +895,7 @@ static void skb_release_data(struct sk_buff *skb, enum skb_drop_reason reason)if (shinfo->frag_list)kfree_skb_list_reason (shinfo->frag_list, reason); -skb_free_head (skb); +skb_free_head (skb, napi_safe); exit:/* When we clone an SKB we copy the reycling bit. The pp_recycle * bit is only set on the head though, so in order to avoid races @@ -955,11 +956,12 @@ void skb_release_head_state(struct sk_buff *skb) }/* Free everything but the sk_buff shell. */ -static void skb_release_all (struct sk_buff *skb,enum skb_drop_reason reason) +static void skb_release_all (struct sk_buff *skb,enum skb_drop_reason reason, +bool napi_safe) {skb_release_head_state (skb);if (likely (skb->head)) -skb_release_data (skb, reason); +skb_release_data (skb, reason, napi_safe); }/** @@ -973,7 +975,7 @@ static void skb_release_all(struct sk_buff *skb, enum skb_drop_reason reason)void __kfree_skb (struct sk_buff *skb) { -skb_release_all (skb,SKB_DROP_REASON_NOT_SPECIFIED ); +skb_release_all (skb,SKB_DROP_REASON_NOT_SPECIFIED , false);kfree_skbmem (skb); }EXPORT_SYMBOL (__kfree_skb);@@ -1027,7 +1029,7 @@ static void kfree_skb_add_bulk(struct sk_buff *skb,return ; } -skb_release_all (skb, reason); +skb_release_all (skb, reason, false); sa->skb_array[sa->skb_count++] = skb;if (unlikely (sa->skb_count ==KFREE_SKB_BULK_SIZE )) {@@ -1201,7 +1203,7 @@ EXPORT_SYMBOL(consume_skb);void __consume_stateless_skb (struct sk_buff *skb) {trace_consume_skb (skb,__builtin_return_address (0 )); -skb_release_data (skb,SKB_CONSUMED ); +skb_release_data (skb,SKB_CONSUMED , false);kfree_skbmem (skb); }@@ -1226,7 +1228,7 @@ static void napi_skb_cache_put(struct sk_buff *skb)void __kfree_skb_defer (struct sk_buff *skb) { -skb_release_all (skb,SKB_DROP_REASON_NOT_SPECIFIED ); +skb_release_all (skb,SKB_DROP_REASON_NOT_SPECIFIED , true);napi_skb_cache_put (skb); }@@ -1264,7 +1266,7 @@ void napi_consume_skb(struct sk_buff *skb, int budget)return ; } -skb_release_all (skb,SKB_CONSUMED ); +skb_release_all (skb,SKB_CONSUMED , !!budget);napi_skb_cache_put (skb); }EXPORT_SYMBOL (napi_consume_skb);@@ -1395,7 +1397,7 @@ EXPORT_SYMBOL_GPL(alloc_skb_for_msg);*/ struct sk_buff *skb_morph (struct sk_buff *dst,struct sk_buff *src) { -skb_release_all (dst,SKB_CONSUMED ); +skb_release_all (dst,SKB_CONSUMED , false);return __skb_clone (dst, src); }EXPORT_SYMBOL_GPL (skb_morph);@@ -2018,9 +2020,9 @@ int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,if (skb_has_frag_list (skb))skb_clone_fraglist (skb); -skb_release_data (skb,SKB_CONSUMED ); +skb_release_data (skb,SKB_CONSUMED , false); }else { -skb_free_head (skb); +skb_free_head (skb, false); } off = (data + nhead) - skb->head;@@ -6389,12 +6391,12 @@ static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off,skb_frag_ref (skb, i);if (skb_has_frag_list (skb))skb_clone_fraglist (skb); -skb_release_data (skb,SKB_CONSUMED ); +skb_release_data (skb,SKB_CONSUMED , false); }else {/* we can reuse existing recount- all we did was * relocate values */ -skb_free_head (skb); +skb_free_head (skb, false); } skb->head = data;@@ -6529,7 +6531,7 @@ static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off,skb_kfree_head (data, size);return -ENOMEM ; } -skb_release_data (skb,SKB_CONSUMED ); +skb_release_data (skb,SKB_CONSUMED , false); skb->head = data; skb->head_frag =0 ;
Jakub Kicinski
maintainerQuoted 4 lines · On Wed, 12 Apr 2023 09:41:16 +0300 Tariq Toukan wrote:
Happy holidays! I was traveling and couldn't review the RFCs.
Dragos and I had this task in our plans after our recent page pool work in mlx5e. We're so glad to see this patchset! :)
I may end up sending some hacks for mlx4 depending on what results I get, I wonder if the reaction will be as enthusiastic :)
Jakub Kicinski
maintainerRe: [PATCH net-next 1/3] net: skb: plumb napi state thru skb freeing paths
In reply to Jesper Dangaard BrouerQuoted 10 lines · On Wed, 12 Apr 2023 11:29:08 +0200 Jesper Dangaard Brouer wrote:
Quoted 7 lines · Jakub Kicinski
We maintain a NAPI-local cache of skbs which is fed by
napi_consume_skb(). Going forward we will also try to cache head and data pages. Plumb the "are we in a normal NAPI context" information thru deeper into the freeing path, up toskb_release_data()and skb_free_head()/skb_pp_recycle().Use "bool
in_normal_napi" rather than bare "int budget",The code was changed to "
napi_safe", the desc should reflect this ;-)
Ah, sed would have been a better idea after all :)
Quoted 7 lines
Quoted 3 lines
the further we get from NAPI the more confusing the budget argument may seem (particularly whether 0 or MAX is the correct value to pass in when not in NAPI).
I do like the code cleanup.
It is worth explaining/mentioning that where budget==0 comes from?
Quoted 7 lines
(Cc. Alex, please correct me.)
My understanding is that this is caused by netconsole/netpoll (seenet/core/netpoll.cfuncpoll_one_napi()), which is a kernel (net)console debugging facility sending UDP packets via using only TX side ofnapi_poll. Thus, we are really trying to protect against doing these recycle tricks for when netpoll/netconcole is running (which I guess makes sense as we are likely printing/sending an OOPS msg over UDP).
Yup, that's correct. I'll add a sentence or two to that effect.