xsk: fix NULL pointer dereference in __xsk_rcv()
Subsystems (4)
PATCH v2 1/1 xsk: fix NULL pointer dereference in __xsk_rcv()
In the __xsk_rcv() multi-buffer path, xsk_buff_alloc() is called in a loop without checking its return value. xsk_buff_can_alloc() only counts fill queue entries without validating their addresses, so it can succeed while xsk_buff_alloc() rejects all remaining entries and returns NULL.
Oops: general protection fault, probably for non-canonical address 0xdffffc0000000000 KASAN: null-ptr-deref in range [0x0000000000000000-0x0000000000000007] RIP: 0010:__xsk_rcv+0x426/0xc20 (net/xdp/xsk.c:350) Call Trace: xsk_generic_rcv+0x26d/0x5f0 xdp_do_generic_redirect+0x3c5/0xcf0 do_xdp_generic+0x92f/0xe70 __netif_receive_skb_core.constprop.0+0xf7e/0x2b30
Fix this with a two-stage transaction. First allocate and stage all buffers required for the packet, recycling all staged buffers with xsk_buff_free() if any allocation fails. Only after this stage succeeds, copy the data, reserve the RX descriptors, and release the buffers in an error-free loop.
---
v2:
- Allocate all packet buffers before reserving RX descriptors.
- Recycle partially allocated buffers instead of only cancelling the RX producer reservations.
net/xdp/xsk.c | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-)
net/xdp/xsk.c +26 -3
@@ -298,9 +298,11 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)u32 frame_size =__xsk_pool_get_rx_frame_size (xs->pool);void *copy_from =xsk_copy_xdp_start (xdp), *copy_to;u32 from_len, meta_len, rem, num_desc; -struct xdp_buff_xsk *xskb; +struct xdp_buff_xsk *xskb, *tmp;struct xdp_buff *xsk_xdp; +LIST_HEAD (xsk_buffs);skb_frag_t *frag; +u32 i; from_len = xdp->data_end - copy_from; meta_len = xdp->data - copy_from;
@@ -343,23 +345,44 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) frag = &sinfo->frags[0 ]; } +for (i =0 ; i < num_desc; i++) { + xsk_xdp =xsk_buff_alloc (xs->pool); +if (!xsk_xdp) + goto err_alloc; + + xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp); +if (unlikely (!list_empty (&xskb->list_node))) + goto err_alloc; +list_add_tail (&xskb->list_node, &xsk_buffs); + } +do {u32 to_len = frame_size + meta_len;u32 copied; - xsk_xdp =xsk_buff_alloc (xs->pool); + xskb = list_first_entry(&xsk_buffs, struct xdp_buff_xsk, + list_node); +list_del_init (&xskb->list_node); + xsk_xdp = &xskb->xdp; copy_to = xsk_xdp->data - meta_len; copied =xsk_copy_xdp (copy_to, ©_from, to_len, &from_len, &frag, rem); rem -= copied; - xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);__xsk_rcv_zc_safe (xs, xskb, copied - meta_len, rem ?XDP_PKT_CONTD :0 ); meta_len =0 ; }while (rem);return 0 ; + +err_alloc: +list_for_each_entry_safe (xskb, tmp, &xsk_buffs, list_node) { +list_del_init (&xskb->list_node); +xsk_buff_free (&xskb->xdp); + } + xs->rx_dropped++; +return -ENOMEM ; }static bool xsk_tx_writeable (struct xdp_sock *xs)
Signature · 1 line
2.53.0
Quoted 24 lines · Cen Zhang (Microsoft)
In the
__xsk_rcv()multi-buffer path,xsk_buff_alloc()is called in a loop without checking its return value.xsk_buff_can_alloc()only counts fill queue entries without validating their addresses, so it can succeed whilexsk_buff_alloc()rejects all remaining entries and returns NULL.Oops: general protection fault, probably for non-canonical address 0xdffffc0000000000 KASAN: null-ptr-deref in range [0x0000000000000000-0x0000000000000007] RIP: 0010:__xsk_rcv+0x426/0xc20 (net/xdp/xsk.c:350) Call Trace: xsk_generic_rcv+0x26d/0x5f0 xdp_do_generic_redirect+0x3c5/0xcf0 do_xdp_generic+0x92f/0xe70 __netif_receive_skb_core.constprop.0+0xf7e/0x2b30Fix this with a two-stage transaction. First allocate and stage all buffers required for the packet, recycling all staged buffers with
xsk_buff_free()if any allocation fails. Only after this stage succeeds, copy the data, reserve the RX descriptors, and release the buffers in an error-free loop.
Quoted 3 lines
Fixes 804627751b42 ("xsk: add support for AF_XDP multi-buffer on Rx path")Reported-by [email protected]Signed-off-by Cen Zhang (Microsoft) <[email protected]>
You sent two patches within 24 hour, which obeys the basic rule of the netdev community.
Please wait.
Thanks,
Jason
Quoted 78 lines
---
v2:
- Allocate all packet buffers before reserving RX descriptors.
- Recycle partially allocated buffers instead of only cancelling the RX producer reservations.net/xdp/xsk.c | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-)net/xdp/xsk.c +26
-4@@ -298,9 +298,11 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)u32 frame_size =__xsk_pool_get_rx_frame_size (xs->pool);void *copy_from =xsk_copy_xdp_start (xdp), *copy_to;u32 from_len, meta_len, rem, num_desc; -struct xdp_buff_xsk *xskb; +struct xdp_buff_xsk *xskb, *tmp;struct xdp_buff *xsk_xdp; +LIST_HEAD (xsk_buffs);skb_frag_t *frag; +u32 i; from_len = xdp->data_end - copy_from; meta_len = xdp->data - copy_from;@@ -343,23 +345,44 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) frag = &sinfo->frags[0 ]; } +for (i =0 ; i < num_desc; i++) { + xsk_xdp =xsk_buff_alloc (xs->pool); +if (!xsk_xdp) + goto err_alloc; + + xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp); +if (unlikely (!list_empty (&xskb->list_node))) + goto err_alloc; +list_add_tail (&xskb->list_node, &xsk_buffs); + } +do {u32 to_len = frame_size + meta_len;u32 copied; - xsk_xdp =xsk_buff_alloc (xs->pool); + xskb = list_first_entry(&xsk_buffs, struct xdp_buff_xsk, + list_node); +list_del_init (&xskb->list_node); + xsk_xdp = &xskb->xdp; copy_to = xsk_xdp->data - meta_len; copied =xsk_copy_xdp (copy_to, ©_from, to_len, &from_len, &frag, rem); rem -= copied; - xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);__xsk_rcv_zc_safe (xs, xskb, copied - meta_len, rem ?XDP_PKT_CONTD :0 ); meta_len =0 ; }while (rem);return 0 ; + +err_alloc: +list_for_each_entry_safe (xskb, tmp, &xsk_buffs, list_node) { +list_del_init (&xskb->list_node); +xsk_buff_free (&xskb->xdp); + } + xs->rx_dropped++; +return -ENOMEM ; }static bool xsk_tx_writeable (struct xdp_sock *xs) --2.53.0
Maciej Fijalkowski
maintainerQuoted 56 lines · Cen Zhang (Microsoft)
In the
__xsk_rcv()multi-buffer path,xsk_buff_alloc()is called in a loop without checking its return value.xsk_buff_can_alloc()only counts fill queue entries without validating their addresses, so it can succeed whilexsk_buff_alloc()rejects all remaining entries and returns NULL.Oops: general protection fault, probably for non-canonical address 0xdffffc0000000000 KASAN: null-ptr-deref in range [0x0000000000000000-0x0000000000000007] RIP: 0010:__xsk_rcv+0x426/0xc20 (net/xdp/xsk.c:350) Call Trace: xsk_generic_rcv+0x26d/0x5f0 xdp_do_generic_redirect+0x3c5/0xcf0 do_xdp_generic+0x92f/0xe70 __netif_receive_skb_core.constprop.0+0xf7e/0x2b30Fix this with a two-stage transaction. First allocate and stage all buffers required for the packet, recycling all staged buffers with
xsk_buff_free()if any allocation fails. Only after this stage succeeds, copy the data, reserve the RX descriptors, and release the buffers in an error-free loop.Fixes 804627751b42 ("xsk: add support for AF_XDP multi-buffer on Rx path")Reported-by [email protected]Signed-off-by Cen Zhang (Microsoft) <[email protected]>---
v2:
- Allocate all packet buffers before reserving RX descriptors.
- Recycle partially allocated buffers instead of only cancelling the RX producer reservations.net/xdp/xsk.c | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-)net/xdp/xsk.c +3
-1@@ -298,9 +298,11 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)u32 frame_size =__xsk_pool_get_rx_frame_size (xs->pool);void *copy_from =xsk_copy_xdp_start (xdp), *copy_to;u32 from_len, meta_len, rem, num_desc; -struct xdp_buff_xsk *xskb; +struct xdp_buff_xsk *xskb, *tmp;struct xdp_buff *xsk_xdp; +LIST_HEAD (xsk_buffs);skb_frag_t *frag; +u32 i; from_len = xdp->data_end - copy_from; meta_len = xdp->data - copy_from;@@ -343,23 +345,44 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) frag = &sinfo->frags[0 ]; }
net/xdp/xsk.c static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) · line 356
Quoted 10 lines
net/xdp/xsk.c +9
-0+for (i =0 ; i < num_desc; i++) { + xsk_xdp =xsk_buff_alloc (xs->pool); +if (!xsk_xdp) + goto err_alloc; + + xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp); +if (unlikely (!list_empty (&xskb->list_node))) + goto err_alloc; +list_add_tail (&xskb->list_node, &xsk_buffs);
could we use existing xsk_buff_add_frag() ?
then I presume xsk_buff_free() would understand list and walk through xdp_buff's and free it ?
I believe we could reuse pool's xskb_list instead of fabricating the on-stack variant here.
Quoted 37 lines
net/xdp/xsk.c +14
-2+ } +do {u32 to_len = frame_size + meta_len;u32 copied; - xsk_xdp =xsk_buff_alloc (xs->pool); + xskb = list_first_entry(&xsk_buffs, struct xdp_buff_xsk, + list_node); +list_del_init (&xskb->list_node); + xsk_xdp = &xskb->xdp; copy_to = xsk_xdp->data - meta_len; copied =xsk_copy_xdp (copy_to, ©_from, to_len, &from_len, &frag, rem); rem -= copied; - xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);__xsk_rcv_zc_safe (xs, xskb, copied - meta_len, rem ?XDP_PKT_CONTD :0 ); meta_len =0 ; }while (rem);return 0 ; + +err_alloc: +list_for_each_entry_safe (xskb, tmp, &xsk_buffs, list_node) { +list_del_init (&xskb->list_node); +xsk_buff_free (&xskb->xdp); + } + xs->rx_dropped++; +return -ENOMEM ; }static bool xsk_tx_writeable (struct xdp_sock *xs)Signature · 1 line
2.53.0
Jason Xing
Reviewed-byHi Maciej,
Quoted 25 lines · Maciej Fijalkowski
Quoted 23 lines · Cen Zhang (Microsoft)
In the
__xsk_rcv()multi-buffer path,xsk_buff_alloc()is called in a loop without checking its return value.xsk_buff_can_alloc()only counts fill queue entries without validating their addresses, so it can succeed whilexsk_buff_alloc()rejects all remaining entries and returns NULL.Oops: general protection fault, probably for non-canonical address 0xdffffc0000000000 KASAN: null-ptr-deref in range [0x0000000000000000-0x0000000000000007] RIP: 0010:__xsk_rcv+0x426/0xc20 (net/xdp/xsk.c:350) Call Trace: xsk_generic_rcv+0x26d/0x5f0 xdp_do_generic_redirect+0x3c5/0xcf0 do_xdp_generic+0x92f/0xe70 __netif_receive_skb_core.constprop.0+0xf7e/0x2b30Fix this with a two-stage transaction. First allocate and stage all buffers required for the packet, recycling all staged buffers with
xsk_buff_free()if any allocation fails. Only after this stage succeeds, copy the data, reserve the RX descriptors, and release the buffers in an error-free loop.
Quoted 3 lines
Quoted 3 lines
Fixes 804627751b42 ("xsk: add support for AF_XDP multi-buffer on Rx path")Reported-by [email protected]Signed-off-by Cen Zhang (Microsoft) <[email protected]>
Quoted 31 lines
Quoted 31 lines
---
v2:
- Allocate all packet buffers before reserving RX descriptors.
- Recycle partially allocated buffers instead of only cancelling the RX producer reservations.net/xdp/xsk.c | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-)net/xdp/xsk.c +3
-1@@ -298,9 +298,11 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)u32 frame_size =__xsk_pool_get_rx_frame_size (xs->pool);void *copy_from =xsk_copy_xdp_start (xdp), *copy_to;u32 from_len, meta_len, rem, num_desc; -struct xdp_buff_xsk *xskb; +struct xdp_buff_xsk *xskb, *tmp;struct xdp_buff *xsk_xdp; +LIST_HEAD (xsk_buffs);skb_frag_t *frag; +u32 i; from_len = xdp->data_end - copy_from; meta_len = xdp->data - copy_from;@@ -343,23 +345,44 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) frag = &sinfo->frags[0 ]; }
Quoted 13 lines
net/xdp/xsk.c static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) · line 356
Quoted 9 lines
net/xdp/xsk.c +9
-0+for (i =0 ; i < num_desc; i++) { + xsk_xdp =xsk_buff_alloc (xs->pool); +if (!xsk_xdp) + goto err_alloc; + + xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp); +if (unlikely (!list_empty (&xskb->list_node))) + goto err_alloc; +list_add_tail (&xskb->list_node, &xsk_buffs);could we use existing
xsk_buff_add_frag()?
then I presumexsk_buff_free()would understand list and walk through xdp_buff's and free it ?
Good suggestion, but xsk_buff_add_frag() will add more irrelevant stuff like nr_frags, xdp_frags_size, xdp_frags_truesize... They are all happening in the softirq context. Refactoring the helper would bring more work here.
Honestly, I like the local array which seems simpler/cleaner to understand :)
Thanks,
Jason
Quoted 42 lines
I believe we could reuse pool's
xskb_listinstead of fabricating the on-stack variant here.Quoted 37 lines
net/xdp/xsk.c +14
-3+ } +do {u32 to_len = frame_size + meta_len;u32 copied; - xsk_xdp =xsk_buff_alloc (xs->pool); + xskb = list_first_entry(&xsk_buffs, struct xdp_buff_xsk, + list_node); +list_del_init (&xskb->list_node); + xsk_xdp = &xskb->xdp; copy_to = xsk_xdp->data - meta_len; copied =xsk_copy_xdp (copy_to, ©_from, to_len, &from_len, &frag, rem); rem -= copied; - xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);__xsk_rcv_zc_safe (xs, xskb, copied - meta_len, rem ?XDP_PKT_CONTD :0 ); meta_len =0 ; }while (rem);return 0 ; + +err_alloc: +list_for_each_entry_safe (xskb, tmp, &xsk_buffs, list_node) { +list_del_init (&xskb->list_node); +xsk_buff_free (&xskb->xdp); + } + xs->rx_dropped++; +return -ENOMEM ; }static bool xsk_tx_writeable (struct xdp_sock *xs) --2.53.0
Maciej Fijalkowski
maintainerQuoted 80 lines · Jason Xing
Hi Maciej,
Quoted 25 lines · Maciej Fijalkowski
Quoted 23 lines · Cen Zhang (Microsoft)
In the
__xsk_rcv()multi-buffer path,xsk_buff_alloc()is called in a loop without checking its return value.xsk_buff_can_alloc()only counts fill queue entries without validating their addresses, so it can succeed whilexsk_buff_alloc()rejects all remaining entries and returns NULL.Oops: general protection fault, probably for non-canonical address 0xdffffc0000000000 KASAN: null-ptr-deref in range [0x0000000000000000-0x0000000000000007] RIP: 0010:__xsk_rcv+0x426/0xc20 (net/xdp/xsk.c:350) Call Trace: xsk_generic_rcv+0x26d/0x5f0 xdp_do_generic_redirect+0x3c5/0xcf0 do_xdp_generic+0x92f/0xe70 __netif_receive_skb_core.constprop.0+0xf7e/0x2b30Fix this with a two-stage transaction. First allocate and stage all buffers required for the packet, recycling all staged buffers with
xsk_buff_free()if any allocation fails. Only after this stage succeeds, copy the data, reserve the RX descriptors, and release the buffers in an error-free loop.Quoted 3 lines
Quoted 3 lines
Fixes 804627751b42 ("xsk: add support for AF_XDP multi-buffer on Rx path")Reported-by [email protected]Signed-off-by Cen Zhang (Microsoft) <[email protected]>Reviewed-by Jason Xing <[email protected]>Quoted 31 lines
Quoted 31 lines
---
v2:
- Allocate all packet buffers before reserving RX descriptors.
- Recycle partially allocated buffers instead of only cancelling the RX producer reservations.net/xdp/xsk.c | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-)net/xdp/xsk.c +3
-1@@ -298,9 +298,11 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)u32 frame_size =__xsk_pool_get_rx_frame_size (xs->pool);void *copy_from =xsk_copy_xdp_start (xdp), *copy_to;u32 from_len, meta_len, rem, num_desc; -struct xdp_buff_xsk *xskb; +struct xdp_buff_xsk *xskb, *tmp;struct xdp_buff *xsk_xdp; +LIST_HEAD (xsk_buffs);skb_frag_t *frag; +u32 i; from_len = xdp->data_end - copy_from; meta_len = xdp->data - copy_from;@@ -343,23 +345,44 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) frag = &sinfo->frags[0 ]; }Quoted 13 lines
net/xdp/xsk.c static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) · line 356
Quoted 9 lines
net/xdp/xsk.c +9
-0+for (i =0 ; i < num_desc; i++) { + xsk_xdp =xsk_buff_alloc (xs->pool); +if (!xsk_xdp) + goto err_alloc; + + xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp); +if (unlikely (!list_empty (&xskb->list_node))) + goto err_alloc; +list_add_tail (&xskb->list_node, &xsk_buffs);could we use existing
xsk_buff_add_frag()?
then I presumexsk_buff_free()would understand list and walk through xdp_buff's and free it ?
Quoted 4 lines
Good suggestion, but
xsk_buff_add_frag()will add more irrelevant stuff likenr_frags,xdp_frags_size,xdp_frags_truesize... They are all happening in the softirq context. Refactoring the helper would bring more work here.
Fair enough, how about we meet in the halfway?
Don't use xxx_add_frag but reuse pool's list. Reason I'm pushing for it is xsk_buff_free() has been thought to consume multi-buffer xskb's, so all the list-walking would be hidden and error path would only consist of a single free() call.
then the do/while loop while iterating could be using xsk_buff_add_frag()
Quoted 48 lines
Honestly, I like the local array which seems simpler/cleaner to understand :)
Thanks,
JasonQuoted 42 lines
I believe we could reuse pool's
xskb_listinstead of fabricating the on-stack variant here.Quoted 37 lines
net/xdp/xsk.c +14
-3+ } +do {u32 to_len = frame_size + meta_len;u32 copied; - xsk_xdp =xsk_buff_alloc (xs->pool); + xskb = list_first_entry(&xsk_buffs, struct xdp_buff_xsk, + list_node); +list_del_init (&xskb->list_node); + xsk_xdp = &xskb->xdp; copy_to = xsk_xdp->data - meta_len; copied =xsk_copy_xdp (copy_to, ©_from, to_len, &from_len, &frag, rem); rem -= copied; - xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);__xsk_rcv_zc_safe (xs, xskb, copied - meta_len, rem ?XDP_PKT_CONTD :0 ); meta_len =0 ; }while (rem);return 0 ; + +err_alloc: +list_for_each_entry_safe (xskb, tmp, &xsk_buffs, list_node) { +list_del_init (&xskb->list_node); +xsk_buff_free (&xskb->xdp); + } + xs->rx_dropped++; +return -ENOMEM ; }static bool xsk_tx_writeable (struct xdp_sock *xs) --2.53.0
Quoted 87 lines · Maciej Fijalkowski
Quoted 80 lines · Jason Xing
Hi Maciej,
Quoted 25 lines · Maciej Fijalkowski
Quoted 23 lines · Cen Zhang (Microsoft)
In the
__xsk_rcv()multi-buffer path,xsk_buff_alloc()is called in a loop without checking its return value.xsk_buff_can_alloc()only counts fill queue entries without validating their addresses, so it can succeed whilexsk_buff_alloc()rejects all remaining entries and returns NULL.Oops: general protection fault, probably for non-canonical address 0xdffffc0000000000 KASAN: null-ptr-deref in range [0x0000000000000000-0x0000000000000007] RIP: 0010:__xsk_rcv+0x426/0xc20 (net/xdp/xsk.c:350) Call Trace: xsk_generic_rcv+0x26d/0x5f0 xdp_do_generic_redirect+0x3c5/0xcf0 do_xdp_generic+0x92f/0xe70 __netif_receive_skb_core.constprop.0+0xf7e/0x2b30Fix this with a two-stage transaction. First allocate and stage all buffers required for the packet, recycling all staged buffers with
xsk_buff_free()if any allocation fails. Only after this stage succeeds, copy the data, reserve the RX descriptors, and release the buffers in an error-free loop.Quoted 3 lines
Quoted 3 lines
Fixes 804627751b42 ("xsk: add support for AF_XDP multi-buffer on Rx path")Reported-by [email protected]Signed-off-by Cen Zhang (Microsoft) <[email protected]>Reviewed-by Jason Xing <[email protected]>Quoted 31 lines
Quoted 31 lines
---
v2:
- Allocate all packet buffers before reserving RX descriptors.
- Recycle partially allocated buffers instead of only cancelling the RX producer reservations.net/xdp/xsk.c | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-)net/xdp/xsk.c +3
-1@@ -298,9 +298,11 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)u32 frame_size =__xsk_pool_get_rx_frame_size (xs->pool);void *copy_from =xsk_copy_xdp_start (xdp), *copy_to;u32 from_len, meta_len, rem, num_desc; -struct xdp_buff_xsk *xskb; +struct xdp_buff_xsk *xskb, *tmp;struct xdp_buff *xsk_xdp; +LIST_HEAD (xsk_buffs);skb_frag_t *frag; +u32 i; from_len = xdp->data_end - copy_from; meta_len = xdp->data - copy_from;@@ -343,23 +345,44 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) frag = &sinfo->frags[0 ]; }Quoted 13 lines
net/xdp/xsk.c static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) · line 356
Quoted 9 lines
net/xdp/xsk.c +9
-0+for (i =0 ; i < num_desc; i++) { + xsk_xdp =xsk_buff_alloc (xs->pool); +if (!xsk_xdp) + goto err_alloc; + + xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp); +if (unlikely (!list_empty (&xskb->list_node))) + goto err_alloc; +list_add_tail (&xskb->list_node, &xsk_buffs);could we use existing
xsk_buff_add_frag()?
then I presumexsk_buff_free()would understand list and walk through xdp_buff's and free it ?Quoted 4 lines
Good suggestion, but
xsk_buff_add_frag()will add more irrelevant stuff likenr_frags,xdp_frags_size,xdp_frags_truesize... They are all happening in the softirq context. Refactoring the helper would bring more work here.
Quoted 6 lines
Fair enough, how about we meet in the halfway?
Don't use
xxx_add_fragbut reuse pool's list. Reason I'm pushing for it isxsk_buff_free()has been thought to consume multi-buffer xskb's, so all the list-walking would be hidden and error path would only consist of a singlefree()call.
It does make sense (reusing/developing common helpers is always a good way to go). Side note: maybe Cen needs to handle xdp_buff_set_frags_flag() to allow xsk_buff_free() freeing a list of nodes.
Quoted 2 lines
then the do/while loop while iterating could be using
xsk_buff_add_frag()
But I don't get it. Sorry. Are you saying we still need to call xsk_buff_add_frag()?
Thanks,
Jason
Quoted 49 lines
Quoted 49 lines
Honestly, I like the local array which seems simpler/cleaner to understand :)
Thanks,
JasonQuoted 42 lines
I believe we could reuse pool's
xskb_listinstead of fabricating the on-stack variant here.Quoted 37 lines
net/xdp/xsk.c +14
-3+ } +do {u32 to_len = frame_size + meta_len;u32 copied; - xsk_xdp =xsk_buff_alloc (xs->pool); + xskb = list_first_entry(&xsk_buffs, struct xdp_buff_xsk, + list_node); +list_del_init (&xskb->list_node); + xsk_xdp = &xskb->xdp; copy_to = xsk_xdp->data - meta_len; copied =xsk_copy_xdp (copy_to, ©_from, to_len, &from_len, &frag, rem); rem -= copied; - xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);__xsk_rcv_zc_safe (xs, xskb, copied - meta_len, rem ?XDP_PKT_CONTD :0 ); meta_len =0 ; }while (rem);return 0 ; + +err_alloc: +list_for_each_entry_safe (xskb, tmp, &xsk_buffs, list_node) { +list_del_init (&xskb->list_node); +xsk_buff_free (&xskb->xdp); + } + xs->rx_dropped++; +return -ENOMEM ; }static bool xsk_tx_writeable (struct xdp_sock *xs) --2.53.0
Maciej Fijalkowski
maintainerQuoted 96 lines · Jason Xing
Quoted 87 lines · Maciej Fijalkowski
Quoted 80 lines · Jason Xing
Hi Maciej,
Quoted 25 lines · Maciej Fijalkowski
Quoted 23 lines · Cen Zhang (Microsoft)
In the
__xsk_rcv()multi-buffer path,xsk_buff_alloc()is called in a loop without checking its return value.xsk_buff_can_alloc()only counts fill queue entries without validating their addresses, so it can succeed whilexsk_buff_alloc()rejects all remaining entries and returns NULL.Oops: general protection fault, probably for non-canonical address 0xdffffc0000000000 KASAN: null-ptr-deref in range [0x0000000000000000-0x0000000000000007] RIP: 0010:__xsk_rcv+0x426/0xc20 (net/xdp/xsk.c:350) Call Trace: xsk_generic_rcv+0x26d/0x5f0 xdp_do_generic_redirect+0x3c5/0xcf0 do_xdp_generic+0x92f/0xe70 __netif_receive_skb_core.constprop.0+0xf7e/0x2b30Fix this with a two-stage transaction. First allocate and stage all buffers required for the packet, recycling all staged buffers with
xsk_buff_free()if any allocation fails. Only after this stage succeeds, copy the data, reserve the RX descriptors, and release the buffers in an error-free loop.Quoted 3 lines
Quoted 3 lines
Fixes 804627751b42 ("xsk: add support for AF_XDP multi-buffer on Rx path")Reported-by [email protected]Signed-off-by Cen Zhang (Microsoft) <[email protected]>Reviewed-by Jason Xing <[email protected]>Quoted 31 lines
Quoted 31 lines
---
v2:
- Allocate all packet buffers before reserving RX descriptors.
- Recycle partially allocated buffers instead of only cancelling the RX producer reservations.net/xdp/xsk.c | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-)net/xdp/xsk.c +3
-1@@ -298,9 +298,11 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)u32 frame_size =__xsk_pool_get_rx_frame_size (xs->pool);void *copy_from =xsk_copy_xdp_start (xdp), *copy_to;u32 from_len, meta_len, rem, num_desc; -struct xdp_buff_xsk *xskb; +struct xdp_buff_xsk *xskb, *tmp;struct xdp_buff *xsk_xdp; +LIST_HEAD (xsk_buffs);skb_frag_t *frag; +u32 i; from_len = xdp->data_end - copy_from; meta_len = xdp->data - copy_from;@@ -343,23 +345,44 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) frag = &sinfo->frags[0 ]; }Quoted 13 lines
net/xdp/xsk.c static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) · line 356
Quoted 9 lines
net/xdp/xsk.c +9
-0+for (i =0 ; i < num_desc; i++) { + xsk_xdp =xsk_buff_alloc (xs->pool); +if (!xsk_xdp) + goto err_alloc; + + xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp); +if (unlikely (!list_empty (&xskb->list_node))) + goto err_alloc; +list_add_tail (&xskb->list_node, &xsk_buffs);could we use existing
xsk_buff_add_frag()?
then I presumexsk_buff_free()would understand list and walk through xdp_buff's and free it ?Quoted 4 lines
Good suggestion, but
xsk_buff_add_frag()will add more irrelevant stuff likenr_frags,xdp_frags_size,xdp_frags_truesize... They are all happening in the softirq context. Refactoring the helper would bring more work here.Quoted 6 lines
Fair enough, how about we meet in the halfway?
Don't use
xxx_add_fragbut reuse pool's list. Reason I'm pushing for it isxsk_buff_free()has been thought to consume multi-buffer xskb's, so all the list-walking would be hidden and error path would only consist of a singlefree()call.
Quoted 4 lines
It does make sense (reusing/developing common helpers is always a good way to go). Side note: maybe Cen needs to handle
xdp_buff_set_frags_flag()to allowxsk_buff_free()freeing a list of nodes.
Yeah exactly this needs to be done on a 'head' xskb.
Quoted 6 lines
Quoted 3 lines
then the do/while loop while iterating could be using
xsk_buff_add_frag()But I don't get it. Sorry. Are you saying we still need to call
xsk_buff_add_frag()?
Sorry i meant xsk_buff_get_frag() which would be peeeling off xskbs from pool's xskb_list.
Quoted 53 lines
Thanks,
JasonQuoted 49 lines
Quoted 49 lines
Honestly, I like the local array which seems simpler/cleaner to understand :)
Thanks,
JasonQuoted 42 lines
I believe we could reuse pool's
xskb_listinstead of fabricating the on-stack variant here.Quoted 37 lines
net/xdp/xsk.c +14
-3+ } +do {u32 to_len = frame_size + meta_len;u32 copied; - xsk_xdp =xsk_buff_alloc (xs->pool); + xskb = list_first_entry(&xsk_buffs, struct xdp_buff_xsk, + list_node); +list_del_init (&xskb->list_node); + xsk_xdp = &xskb->xdp; copy_to = xsk_xdp->data - meta_len; copied =xsk_copy_xdp (copy_to, ©_from, to_len, &from_len, &frag, rem); rem -= copied; - xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);__xsk_rcv_zc_safe (xs, xskb, copied - meta_len, rem ?XDP_PKT_CONTD :0 ); meta_len =0 ; }while (rem);return 0 ; + +err_alloc: +list_for_each_entry_safe (xskb, tmp, &xsk_buffs, list_node) { +list_del_init (&xskb->list_node); +xsk_buff_free (&xskb->xdp); + } + xs->rx_dropped++; +return -ENOMEM ; }static bool xsk_tx_writeable (struct xdp_sock *xs) --2.53.0
Thanks for the suggestions. I am back and will prepare and send v3 in the next few days.
Quoted 56 lines · Cen Zhang (Microsoft)
In the
__xsk_rcv()multi-buffer path,xsk_buff_alloc()is called in a loop without checking its return value.xsk_buff_can_alloc()only counts fill queue entries without validating their addresses, so it can succeed whilexsk_buff_alloc()rejects all remaining entries and returns NULL.Oops: general protection fault, probably for non-canonical address 0xdffffc0000000000 KASAN: null-ptr-deref in range [0x0000000000000000-0x0000000000000007] RIP: 0010:__xsk_rcv+0x426/0xc20 (net/xdp/xsk.c:350) Call Trace: xsk_generic_rcv+0x26d/0x5f0 xdp_do_generic_redirect+0x3c5/0xcf0 do_xdp_generic+0x92f/0xe70 __netif_receive_skb_core.constprop.0+0xf7e/0x2b30Fix this with a two-stage transaction. First allocate and stage all buffers required for the packet, recycling all staged buffers with
xsk_buff_free()if any allocation fails. Only after this stage succeeds, copy the data, reserve the RX descriptors, and release the buffers in an error-free loop.Fixes 804627751b42 ("xsk: add support for AF_XDP multi-buffer on Rx path")Reported-by [email protected]Signed-off-by Cen Zhang (Microsoft) <[email protected]>---
v2:
- Allocate all packet buffers before reserving RX descriptors.
- Recycle partially allocated buffers instead of only cancelling the RX producer reservations.net/xdp/xsk.c | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-)net/xdp/xsk.c +3
-1@@ -298,9 +298,11 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)u32 frame_size =__xsk_pool_get_rx_frame_size (xs->pool);void *copy_from =xsk_copy_xdp_start (xdp), *copy_to;u32 from_len, meta_len, rem, num_desc; -struct xdp_buff_xsk *xskb; +struct xdp_buff_xsk *xskb, *tmp;struct xdp_buff *xsk_xdp; +LIST_HEAD (xsk_buffs);skb_frag_t *frag; +u32 i; from_len = xdp->data_end - copy_from; meta_len = xdp->data - copy_from;@@ -343,23 +345,44 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) frag = &sinfo->frags[0 ];
net/xdp/xsk.c static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) · line 355
Quoted 10 lines
net/xdp/xsk.c +8
-0} +for (i =0 ; i < num_desc; i++) { + xsk_xdp =xsk_buff_alloc (xs->pool); +if (!xsk_xdp) + goto err_alloc; + + xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp); +if (unlikely (!list_empty (&xskb->list_node))) + goto err_alloc;
It will cause a memory leak because the current xsk_xdp that is not added to the local list will miss the chance to get freed? And the empty list_node cannot be easily freed by xp_free()...
Thanks,
Jason
Quoted 39 lines
net/xdp/xsk.c +15
-3+list_add_tail (&xskb->list_node, &xsk_buffs); + } +do {u32 to_len = frame_size + meta_len;u32 copied; - xsk_xdp =xsk_buff_alloc (xs->pool); + xskb = list_first_entry(&xsk_buffs, struct xdp_buff_xsk, + list_node); +list_del_init (&xskb->list_node); + xsk_xdp = &xskb->xdp; copy_to = xsk_xdp->data - meta_len; copied =xsk_copy_xdp (copy_to, ©_from, to_len, &from_len, &frag, rem); rem -= copied; - xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);__xsk_rcv_zc_safe (xs, xskb, copied - meta_len, rem ?XDP_PKT_CONTD :0 ); meta_len =0 ; }while (rem);return 0 ; + +err_alloc: +list_for_each_entry_safe (xskb, tmp, &xsk_buffs, list_node) { +list_del_init (&xskb->list_node); +xsk_buff_free (&xskb->xdp); + } + xs->rx_dropped++; +return -ENOMEM ; }static bool xsk_tx_writeable (struct xdp_sock *xs) --2.53.0