net/packet: defer vmalloc TX_RING free until skbs finish
Subsystems (4)
PATCH v2 1/1 net/packet: defer vmalloc TX_RING free until skbs finish
AF_PACKET TX_RING skbs keep a raw pointer to their ring frame. The skb page references preserve page-backed ring blocks after pg_vec is freed, but they do not preserve a vmalloc mapping.
tpacket_destruct_skb() currently drops the pending reference before writing the timestamp and TP_STATUS_AVAILABLE to the frame. Move the decrement after those stores. The smp_wmb() in __packet_set_status() orders the frame stores before the decrement.
Also recheck pending TX frames under pg_vec_lock before non-closing ring replacement, so a racing send cannot add a pending skb between the initial check and the ring swap.
Ring allocation can produce a mixture of page-backed and vmalloc-backed blocks. Allocate deferred-work storage during TX ring setup when the first vmalloc-backed block is encountered, and keep its pointer in the pg_vec allocation header. If allocation fails, return -ENOMEM from ring setup. On socket close, a non-NULL pointer identifies a vmalloc-backed vector without a scan. If TX skbs remain, defer the whole vector to system_long_wq.
After pg_vec is detached, a late destructor can skip the pending decrement. Use socket write-memory accounting as the deferred lifetime gate instead: an skb remains charged through its final sock_wfree(), after all ring-frame accesses. The delayed work retains a socket reference and reschedules itself until no TX skbs remain.
Move pending_refcnt release to packet_sock_destruct() so late skb destructors and deferred cleanup can safely use it after packet_release(). Page-backed teardown remains synchronous, and no lock is added to the TX completion hot path.
---
Changes in v2:
- Allocate cleanup work during setup, only for vmalloc-backed TX rings.
- Keep its pointer in the pg_vec allocation header; remove the teardown detection scan, allocation, and synchronous fallback.
- Use socket write-memory accounting for both the close-time decision and the deferred worker.
- Recheck pending TX frames under pg_vec_lock before non-closing swaps.
- Rebase onto f5bbbfec59b4.
net/packet/af_packet.c | 96 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 87 insertions(+), 9 deletions(-)
net/packet/af_packet.c +87 -9
@@ -88,6 +88,7 @@#include <linux/errqueue.h> #include <linux/net_tstamp.h> #include <linux/percpu.h> +#include <linux/workqueue.h> #ifdef CONFIG_INET #include <net/inet_common.h> #endif
@@ -1341,6 +1342,8 @@ static void packet_sock_destruct(struct sock *sk)WARN_ON (atomic_read (&sk->sk_rmem_alloc));WARN_ON (refcount_read (&sk->sk_wmem_alloc)); +packet_free_pending (pkt_sk (sk)); +if (!sock_flag (sk,SOCK_DEAD )) {pr_err ("Attempt to release alive packet socket: %p\n" , sk);return ;
@@ -2534,11 +2537,11 @@ static void tpacket_destruct_skb(struct sk_buff *skb)__u32 ts; ph =skb_zcopy_get_nouarg (skb); -packet_dec_pending (&po->tx_ring); ts =__packet_set_timestamp (po, ph, skb);__packet_set_status (po, ph,TP_STATUS_AVAILABLE | ts); +packet_dec_pending (&po->tx_ring);complete (&po->skb_completion); }
@@ -3204,7 +3207,6 @@ static int packet_release(struct socket *sock)/* Purge queues */ skb_queue_purge (&sk->sk_receive_queue); -packet_free_pending (po);sock_put (sk);return 0 ;
@@ -4367,11 +4369,26 @@ static const struct vm_operations_struct packet_mmap_ops = { .close = packet_mm_close, }; +struct packet_pg_vec { +struct packet_pg_vec_free *deferred; +unsigned int order; +unsigned int len; + struct pgv pg_vec[] __counted_by(len); +}; + +struct packet_pg_vec_free { +struct delayed_work work; +struct sock *sk; +struct packet_pg_vec *vec; +}; +static void free_pg_vec (struct pgv *pg_vec,unsigned int order,unsigned int len) { +struct packet_pg_vec *vec;int i; + vec = container_of_const(pg_vec, struct packet_pg_vec, pg_vec[0]);for (i =0 ; i < len; i++) {if (likely (pg_vec[i].buffer)) {if (is_vmalloc_addr (pg_vec[i].buffer))
@@ -4382,7 +4399,46 @@ static void free_pg_vec(struct pgv *pg_vec, unsigned int order, pg_vec[i].buffer =NULL ; } } -kfree (pg_vec); +kfree (vec->deferred); +kfree (vec); +} + +static void packet_free_pg_vec_work (struct work_struct *work) +{ +struct packet_pg_vec_free *deferred; +struct packet_pg_vec *vec; +struct sock *sk; + + deferred =container_of_const (to_delayed_work (work), + struct packet_pg_vec_free, work); + vec = deferred->vec; + sk = deferred->sk; +if (sk_wmem_alloc_get (sk)) { +queue_delayed_work (system_long_wq, &deferred->work,1 ); +return ; + } + +free_pg_vec (vec->pg_vec, vec->order, vec->len); +sock_put (sk); +} + +static void packet_free_tx_ring (struct sock *sk,struct pgv *pg_vec, +unsigned int order,unsigned int len) +{ +struct packet_pg_vec_free *deferred; +struct packet_pg_vec *vec; + + vec = container_of_const(pg_vec, struct packet_pg_vec, pg_vec[0]); + deferred = vec->deferred; +if (!deferred || !sk_wmem_alloc_get (sk)) { +free_pg_vec (pg_vec, order, len); +return ; + } + +/* A detached ring's pending count can miss late skb destructors. */ + deferred->sk = sk; +sock_hold (sk); +queue_delayed_work (system_long_wq, &deferred->work,0 ); }static char *alloc_one_pg_vec_page (unsigned long order)
@@ -4410,20 +4466,35 @@ static char *alloc_one_pg_vec_page(unsigned long order)return NULL ; } -static struct pgv *alloc_pg_vec (struct tpacket_req *req,int order) +static struct pgv *alloc_pg_vec (struct tpacket_req *req,int order,bool tx_ring) {unsigned int block_nr = req->tp_block_nr; +struct packet_pg_vec *vec;struct pgv *pg_vec;int i; - pg_vec = kzalloc_objs(struct pgv, block_nr, GFP_KERNEL | __GFP_NOWARN); -if (unlikely (!pg_vec)) - goto out; + vec =kzalloc_flex (*vec, pg_vec, block_nr,GFP_KERNEL | __GFP_NOWARN); +if (unlikely (!vec)) +return NULL ; + vec->order = order; + vec->len = block_nr; + pg_vec = vec->pg_vec;for (i =0 ; i < block_nr; i++) { pg_vec[i].buffer =alloc_one_pg_vec_page (order);if (unlikely (!pg_vec[i].buffer)) goto out_free_pgvec; + +if (tx_ring && !vec->deferred && +is_vmalloc_addr (pg_vec[i].buffer)) { + vec->deferred =kzalloc_obj (*vec->deferred, +GFP_KERNEL | __GFP_NOWARN); +if (!vec->deferred) + goto out_free_pgvec; + vec->deferred->vec = vec; +INIT_DELAYED_WORK (&vec->deferred->work, + packet_free_pg_vec_work); + } } out:
@@ -4506,7 +4577,7 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u, err = -ENOMEM ; order =get_order (req->tp_block_size); - pg_vec =alloc_pg_vec (req, order); + pg_vec =alloc_pg_vec (req, order, tx_ring);if (unlikely (!pg_vec)) goto out;switch (po->tp_version) {
@@ -4558,6 +4629,9 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u, err = -EBUSY ;mutex_lock (&po->pg_vec_lock);if (closing ||atomic_long_read (&po->mapped) ==0 ) { +if (tx_ring && !closing &&packet_read_pending (rb)) + goto out_unlock; + err =0 ;spin_lock_bh (&rb_queue->lock);swap (rb->pg_vec, pg_vec);
@@ -4579,6 +4653,7 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,pr_err ("packet_mmap: vma is busy: %ld\n" ,atomic_long_read (&po->mapped)); } +out_unlock:mutex_unlock (&po->pg_vec_lock);spin_lock (&po->bind_lock);
@@ -4600,7 +4675,10 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u, out_free_pg_vec:if (pg_vec) {bitmap_free (rx_owner_map); -free_pg_vec (pg_vec, order, req->tp_block_nr); +if (tx_ring && closing) +packet_free_tx_ring (sk, pg_vec, order, req->tp_block_nr); +else +free_pg_vec (pg_vec, order, req->tp_block_nr); } out:return err;
base-commit: f5bbbfec59b4e2fb7520a91de3df8a6174325d6a
Signature · 1 line
2.55.0.openai.347.g2b0ba75e4a61
Willem de Bruijn
maintainerQuoted 3 lines · Kyle Zeng
AF_PACKETTX_RINGskbs keep a raw pointer to their ring frame. The skb page references preserve page-backed ring blocks afterpg_vecis freed, but they do not preserve a vmalloc mapping.
Claude shows an interesting case where this page-backed statement does not hold: if the entire skb is linear. Not for this patch, but a similar case.
Quoted 23 lines
tpacket_destruct_skb()currently drops the pending reference before writing the timestamp andTP_STATUS_AVAILABLEto the frame. Move the decrement after those stores. Thesmp_wmb()in__packet_set_status()orders the frame stores before the decrement.Also recheck pending TX frames under
pg_vec_lockbefore non-closing ring replacement, so a racing send cannot add a pending skb between the initial check and the ring swap.Ring allocation can produce a mixture of page-backed and vmalloc-backed blocks. Allocate deferred-work storage during TX ring setup when the first vmalloc-backed block is encountered, and keep its pointer in the
pg_vecallocation header. If allocation fails, return-ENOMEMfrom ring setup. On socket close, a non-NULL pointer identifies a vmalloc-backed vector without a scan. If TX skbs remain, defer the whole vector tosystem_long_wq.After
pg_vecis detached, a late destructor can skip the pending decrement. Use socket write-memory accounting as the deferred lifetime gate instead: an skb remains charged through its finalsock_wfree(), after all ring-frame accesses. The delayed work retains a socket reference and reschedules itself until no TX skbs remain.
Quoted 4 lines
Move
pending_refcntrelease topacket_sock_destruct()so late skb destructors and deferred cleanup can safely use it afterpacket_release(). Page-backed teardown remains synchronous, and no lock is added to the TX completion hot path.
This one patch combines multiple fixes. If and only if a respin is needed, it may be good to break it up to help understanding.
Quoted 23 lines
Fixes b013840810c2 ("packet: use percpu mmap tx frame pending refcount")Suggested-by Eric Dumazet <[email protected]>Suggested-by Willem de Bruijn <[email protected]>Assisted-by Codex:gpt-5.6-solnet/packet/af_packet.c +14
-0+struct packet_pg_vec { +struct packet_pg_vec_free *deferred; +unsigned int order; +unsigned int len; + struct pgv pg_vec[] __counted_by(len); +}; + +struct packet_pg_vec_free { +struct delayed_work work; +struct sock *sk; +struct packet_pg_vec *vec; +};@@ -4382,7 +4399,46 @@ static void free_pg_vec(struct pgv *pg_vec, unsigned int order, +static void packet_free_pg_vec_work (struct work_struct *work) +{
net/packet/af_packet.c static void free_pg_vec(struct pgv *pg_vec, unsigned int order, · line 4410
Quoted 10 lines
net/packet/af_packet.c +10
-0+struct packet_pg_vec_free *deferred; +struct packet_pg_vec *vec; +struct sock *sk; + + deferred =container_of_const (to_delayed_work (work), + struct packet_pg_vec_free, work); + vec = deferred->vec; + sk = deferred->sk; +if (sk_wmem_alloc_get (sk)) { +queue_delayed_work (system_long_wq, &deferred->work,1 );
Can this keep requeueing itself? Claude suggests using pending ring count as gate.
Quoted 44 lines
net/packet/af_packet.c +25
-4+static void packet_free_tx_ring (struct sock *sk,struct pgv *pg_vec, +unsigned int order,unsigned int len) +{ +struct packet_pg_vec_free *deferred; +struct packet_pg_vec *vec; + + vec = container_of_const(pg_vec, struct packet_pg_vec, pg_vec[0]); + deferred = vec->deferred; +if (!deferred || !sk_wmem_alloc_get (sk)) { +free_pg_vec (pg_vec, order, len); +return ; + } + +/* A detached ring's pending count can miss late skb destructors. */ + deferred->sk = sk; +sock_hold (sk); +queue_delayed_work (system_long_wq, &deferred->work,0 ); }static char *alloc_one_pg_vec_page (unsigned long order)@@ -4410,20 +4466,35 @@ static char *alloc_one_pg_vec_page(unsigned long order)return NULL ; } -static struct pgv *alloc_pg_vec (struct tpacket_req *req,int order) +static struct pgv *alloc_pg_vec (struct tpacket_req *req,int order,bool tx_ring) {unsigned int block_nr = req->tp_block_nr; +struct packet_pg_vec *vec;struct pgv *pg_vec;int i; - pg_vec = kzalloc_objs(struct pgv, block_nr, GFP_KERNEL | __GFP_NOWARN); -if (unlikely (!pg_vec)) - goto out; + vec =kzalloc_flex (*vec, pg_vec, block_nr,GFP_KERNEL | __GFP_NOWARN); +if (unlikely (!vec)) +return NULL ; + vec->order = order; + vec->len = block_nr; + pg_vec = vec->pg_vec;for (i =0 ; i < block_nr; i++) { pg_vec[i].buffer =alloc_one_pg_vec_page (order);
net/packet/af_packet.c static char *alloc_one_pg_vec_page(unsigned long order) · line 4494
Quoted 10 lines
net/packet/af_packet.c +8
-0if (unlikely (!pg_vec[i].buffer)) goto out_free_pgvec; + +if (tx_ring && !vec->deferred && +is_vmalloc_addr (pg_vec[i].buffer)) { + vec->deferred =kzalloc_obj (*vec->deferred, +GFP_KERNEL | __GFP_NOWARN); +if (!vec->deferred) + goto out_free_pgvec; + vec->deferred->vec = vec;
The nested structures are fairly complex.
Would it make sense to avoid the separate packet_pg_vec_free, fold that into packet_pg_vec and use a different field to identify whether vmalloc backed pages are used. Maybe sk, or even a new boolean field has_vmalloc, for readability
Quoted 134 lines · Willem de Bruijn
Quoted 3 lines · Kyle Zeng
AF_PACKETTX_RINGskbs keep a raw pointer to their ring frame. The skb page references preserve page-backed ring blocks afterpg_vecis freed, but they do not preserve a vmalloc mapping.Claude shows an interesting case where this page-backed statement does not hold: if the entire skb is linear. Not for this patch, but a similar case.
Quoted 23 lines
tpacket_destruct_skb()currently drops the pending reference before writing the timestamp andTP_STATUS_AVAILABLEto the frame. Move the decrement after those stores. Thesmp_wmb()in__packet_set_status()orders the frame stores before the decrement.Also recheck pending TX frames under
pg_vec_lockbefore non-closing ring replacement, so a racing send cannot add a pending skb between the initial check and the ring swap.Ring allocation can produce a mixture of page-backed and vmalloc-backed blocks. Allocate deferred-work storage during TX ring setup when the first vmalloc-backed block is encountered, and keep its pointer in the
pg_vecallocation header. If allocation fails, return-ENOMEMfrom ring setup. On socket close, a non-NULL pointer identifies a vmalloc-backed vector without a scan. If TX skbs remain, defer the whole vector tosystem_long_wq.After
pg_vecis detached, a late destructor can skip the pending decrement. Use socket write-memory accounting as the deferred lifetime gate instead: an skb remains charged through its finalsock_wfree(), after all ring-frame accesses. The delayed work retains a socket reference and reschedules itself until no TX skbs remain.Quoted 4 lines
Move
pending_refcntrelease topacket_sock_destruct()so late skb destructors and deferred cleanup can safely use it afterpacket_release(). Page-backed teardown remains synchronous, and no lock is added to the TX completion hot path.This one patch combines multiple fixes. If and only if a respin is needed, it may be good to break it up to help understanding.
Quoted 23 lines
Fixes b013840810c2 ("packet: use percpu mmap tx frame pending refcount")Suggested-by Eric Dumazet <[email protected]>Suggested-by Willem de Bruijn <[email protected]>Assisted-by Codex:gpt-5.6-solnet/packet/af_packet.c +14
-0+struct packet_pg_vec { +struct packet_pg_vec_free *deferred; +unsigned int order; +unsigned int len; + struct pgv pg_vec[] __counted_by(len); +}; + +struct packet_pg_vec_free { +struct delayed_work work; +struct sock *sk; +struct packet_pg_vec *vec; +};@@ -4382,7 +4399,46 @@ static void free_pg_vec(struct pgv *pg_vec, unsigned int order, +static void packet_free_pg_vec_work (struct work_struct *work) +{net/packet/af_packet.c static void free_pg_vec(struct pgv *pg_vec, unsigned int order, · line 4410
Quoted 10 lines
net/packet/af_packet.c +10
-0+struct packet_pg_vec_free *deferred; +struct packet_pg_vec *vec; +struct sock *sk; + + deferred =container_of_const (to_delayed_work (work), + struct packet_pg_vec_free, work); + vec = deferred->vec; + sk = deferred->sk; +if (sk_wmem_alloc_get (sk)) { +queue_delayed_work (system_long_wq, &deferred->work,1 );Can this keep requeueing itself? Claude suggests using pending ring count as gate.
Quoted 44 lines
net/packet/af_packet.c +25
-4+static void packet_free_tx_ring (struct sock *sk,struct pgv *pg_vec, +unsigned int order,unsigned int len) +{ +struct packet_pg_vec_free *deferred; +struct packet_pg_vec *vec; + + vec = container_of_const(pg_vec, struct packet_pg_vec, pg_vec[0]); + deferred = vec->deferred; +if (!deferred || !sk_wmem_alloc_get (sk)) { +free_pg_vec (pg_vec, order, len); +return ; + } + +/* A detached ring's pending count can miss late skb destructors. */ + deferred->sk = sk; +sock_hold (sk); +queue_delayed_work (system_long_wq, &deferred->work,0 ); }static char *alloc_one_pg_vec_page (unsigned long order)@@ -4410,20 +4466,35 @@ static char *alloc_one_pg_vec_page(unsigned long order)return NULL ; } -static struct pgv *alloc_pg_vec (struct tpacket_req *req,int order) +static struct pgv *alloc_pg_vec (struct tpacket_req *req,int order,bool tx_ring) {unsigned int block_nr = req->tp_block_nr; +struct packet_pg_vec *vec;struct pgv *pg_vec;int i; - pg_vec = kzalloc_objs(struct pgv, block_nr, GFP_KERNEL | __GFP_NOWARN); -if (unlikely (!pg_vec)) - goto out; + vec =kzalloc_flex (*vec, pg_vec, block_nr,GFP_KERNEL | __GFP_NOWARN); +if (unlikely (!vec)) +return NULL ; + vec->order = order; + vec->len = block_nr; + pg_vec = vec->pg_vec;for (i =0 ; i < block_nr; i++) { pg_vec[i].buffer =alloc_one_pg_vec_page (order);net/packet/af_packet.c static char *alloc_one_pg_vec_page(unsigned long order) · line 4494
Quoted 10 lines
net/packet/af_packet.c +8
-0if (unlikely (!pg_vec[i].buffer)) goto out_free_pgvec; + +if (tx_ring && !vec->deferred && +is_vmalloc_addr (pg_vec[i].buffer)) { + vec->deferred =kzalloc_obj (*vec->deferred, +GFP_KERNEL | __GFP_NOWARN); +if (!vec->deferred) + goto out_free_pgvec; + vec->deferred->vec = vec;The nested structures are fairly complex.
Quoted 4 lines
Would it make sense to avoid the separate
packet_pg_vec_free, fold that intopacket_pg_vecand use a different field to identify whether vmalloc backed pages are used. Maybe sk, or even a new boolean fieldhas_vmalloc, for readability
To be clear, these are questions, not blocking issues.
Jakub Kicinski
maintainerQuoted 6 lines · On Sun, 16 Aug 2026 16:56:46 -0700 Kyle Zeng wrote:
Fixes b013840810c2 ("packet: use percpu mmap tx frame pending refcount")Suggested-by Eric Dumazet <[email protected]>Suggested-by Willem de Bruijn <[email protected]>Assisted-by Codex:gpt-5.6-sol
Kyle, we're missing your sign-off tag here.
Quoted 9 lines · Jakub Kicinski
Quoted 6 lines · On Sun, 16 Aug 2026 16:56:46 -0700 Kyle Zeng wrote:
Fixes b013840810c2 ("packet: use percpu mmap tx frame pending refcount")Suggested-by Eric Dumazet <[email protected]>Suggested-by Willem de Bruijn <[email protected]>Assisted-by Codex:gpt-5.6-solKyle, we're missing your sign-off tag here.
Apologies, here is the tag:
This patch was applied to netdev/net.git (main) by Jakub Kicinski <[email protected]>:
Hello:
This patch was applied to netdev/net.git (main)
by Jakub Kicinski <[email protected]>:
Quoted 10 lines · On Sun, 16 Aug 2026 16:56:46 -0700 you wrote:
AF_PACKETTX_RINGskbs keep a raw pointer to their ring frame. The skb page references preserve page-backed ring blocks afterpg_vecis freed, but they do not preserve a vmalloc mapping.
tpacket_destruct_skb()currently drops the pending reference before writing the timestamp andTP_STATUS_AVAILABLEto the frame. Move the decrement after those stores. Thesmp_wmb()in__packet_set_status()orders the frame stores before the decrement.[...]
Here is the summary with links:
- [net,v2] net/packet: defer vmalloc TX_RING free until skbs finish
https://git.kernel.org/netdev/net/c/992cc9f94ca9
You are awesome, thank you!
Signature · 2 lines
Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/patchwork/pwbot.html