Skip to contentSkip to conversation

xsk: fix NULL pointer dereference in __xsk_rcv()

PATCH Superseded Cen Zhang (Microsoft) 3 participants 9 messages Jul 25 – Aug 3

Subsystems (4)

v2 of 4 Superseded Latest is v4 →

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 xsk: fix NULL pointer dereference in __xsk_rcv()

In the __xsk_rcv() multi-buffer path, xsk_buff_alloc() is called in a loop without checking its return value. xsk_buff_can_alloc() only counts fill queue entries without validating their addresses, so it can succeed while xsk_buff_alloc() rejects all remaining entries and returns NULL.

  Oops: general protection fault, probably for non-canonical address
   0xdffffc0000000000
  KASAN: null-ptr-deref in range
   [0x0000000000000000-0x0000000000000007]
  RIP: 0010:__xsk_rcv+0x426/0xc20 (net/xdp/xsk.c:350)
  Call Trace:
   xsk_generic_rcv+0x26d/0x5f0
   xdp_do_generic_redirect+0x3c5/0xcf0
   do_xdp_generic+0x92f/0xe70
   __netif_receive_skb_core.constprop.0+0xf7e/0x2b30

Fix this with a two-stage transaction. First allocate and stage all buffers required for the packet, recycling all staged buffers with xsk_buff_free() if any allocation fails. Only after this stage succeeds, copy the data, reserve the RX descriptors, and release the buffers in an error-free loop.

Fixes 804627751b42 ("xsk: add support for AF_XDP multi-buffer on Rx path")
Reported-by [email protected]

---
v2:
- Allocate all packet buffers before reserving RX descriptors.
- Recycle partially allocated buffers instead of only cancelling the RX producer reservations.

 net/xdp/xsk.c | 29 ++++++++++++++++++++++++++---
 1 file changed, 26 insertions(+), 3 deletions(-)
net/xdp/xsk.c +26 -3
index f906d51b6699..383fc2b1de48 100644
@@ -298,9 +298,11 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
 	u32 frame_size = __xsk_pool_get_rx_frame_size(xs->pool);
 	void *copy_from = xsk_copy_xdp_start(xdp), *copy_to;
 	u32 from_len, meta_len, rem, num_desc;
-	struct xdp_buff_xsk *xskb;
+	struct xdp_buff_xsk *xskb, *tmp;
 	struct xdp_buff *xsk_xdp;
+	LIST_HEAD(xsk_buffs);
 	skb_frag_t *frag;
+	u32 i;
 
 	from_len = xdp->data_end - copy_from;
 	meta_len = xdp->data - copy_from;
@@ -343,23 +345,44 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
 		frag =  &sinfo->frags[0];
 	}
 
+	for (i = 0; i < num_desc; i++) {
+		xsk_xdp = xsk_buff_alloc(xs->pool);
+		if (!xsk_xdp)
+			goto err_alloc;
+
+		xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);
+		if (unlikely(!list_empty(&xskb->list_node)))
+			goto err_alloc;
+		list_add_tail(&xskb->list_node, &xsk_buffs);
+	}
+
 	do {
 		u32 to_len = frame_size + meta_len;
 		u32 copied;
 
-		xsk_xdp = xsk_buff_alloc(xs->pool);
+		xskb = list_first_entry(&xsk_buffs, struct xdp_buff_xsk,
+					list_node);
+		list_del_init(&xskb->list_node);
+		xsk_xdp = &xskb->xdp;
 		copy_to = xsk_xdp->data - meta_len;
 
 		copied = xsk_copy_xdp(copy_to, &copy_from, to_len, &from_len, &frag, rem);
 		rem -= copied;
 
-		xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);
 		__xsk_rcv_zc_safe(xs, xskb, copied - meta_len,
 				  rem ? XDP_PKT_CONTD : 0);
 		meta_len = 0;
 	} while (rem);
 
 	return 0;
+
+err_alloc:
+	list_for_each_entry_safe(xskb, tmp, &xsk_buffs, list_node) {
+		list_del_init(&xskb->list_node);
+		xsk_buff_free(&xskb->xdp);
+	}
+	xs->rx_dropped++;
+	return -ENOMEM;
 }
 
 static bool xsk_tx_writeable(struct xdp_sock *xs)
Signature · 1 line
2.53.0
#1 lore raw
Quoted 24 lines · Cen Zhang (Microsoft)

In the __xsk_rcv() multi-buffer path, xsk_buff_alloc() is called in a loop without checking its return value. xsk_buff_can_alloc() only counts fill queue entries without validating their addresses, so it can succeed while xsk_buff_alloc() rejects all remaining entries and returns NULL.

  Oops: general protection fault, probably for non-canonical address
   0xdffffc0000000000
  KASAN: null-ptr-deref in range
   [0x0000000000000000-0x0000000000000007]
  RIP: 0010:__xsk_rcv+0x426/0xc20 (net/xdp/xsk.c:350)
  Call Trace:
   xsk_generic_rcv+0x26d/0x5f0
   xdp_do_generic_redirect+0x3c5/0xcf0
   do_xdp_generic+0x92f/0xe70
   __netif_receive_skb_core.constprop.0+0xf7e/0x2b30

Fix this with a two-stage transaction. First allocate and stage all buffers required for the packet, recycling all staged buffers with xsk_buff_free() if any allocation fails. Only after this stage succeeds, copy the data, reserve the RX descriptors, and release the buffers in an error-free loop.

Quoted 3 lines
Fixes 804627751b42 ("xsk: add support for AF_XDP multi-buffer on Rx path")
Reported-by [email protected]

You sent two patches within 24 hour, which obeys the basic rule of the netdev community.

Please wait.

Thanks,
Jason

Quoted 78 lines

---
v2:
- Allocate all packet buffers before reserving RX descriptors.
- Recycle partially allocated buffers instead of only cancelling the RX producer reservations.

 net/xdp/xsk.c | 29 ++++++++++++++++++++++++++---
 1 file changed, 26 insertions(+), 3 deletions(-)
net/xdp/xsk.c +26 -4
index f906d51b6699..383fc2b1de48 100644
@@ -298,9 +298,11 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
        u32 frame_size = __xsk_pool_get_rx_frame_size(xs->pool);
        void *copy_from = xsk_copy_xdp_start(xdp), *copy_to;
        u32 from_len, meta_len, rem, num_desc;
-       struct xdp_buff_xsk *xskb;
+       struct xdp_buff_xsk *xskb, *tmp;
        struct xdp_buff *xsk_xdp;
+       LIST_HEAD(xsk_buffs);
        skb_frag_t *frag;
+       u32 i;
 
        from_len = xdp->data_end - copy_from;
        meta_len = xdp->data - copy_from;
@@ -343,23 +345,44 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
                frag =  &sinfo->frags[0];
        }
 
+       for (i = 0; i < num_desc; i++) {
+               xsk_xdp = xsk_buff_alloc(xs->pool);
+               if (!xsk_xdp)
+                       goto err_alloc;
+
+               xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);
+               if (unlikely(!list_empty(&xskb->list_node)))
+                       goto err_alloc;
+               list_add_tail(&xskb->list_node, &xsk_buffs);
+       }
+
        do {
                u32 to_len = frame_size + meta_len;
                u32 copied;
 
-               xsk_xdp = xsk_buff_alloc(xs->pool);
+               xskb = list_first_entry(&xsk_buffs, struct xdp_buff_xsk,
+                                       list_node);
+               list_del_init(&xskb->list_node);
+               xsk_xdp = &xskb->xdp;
                copy_to = xsk_xdp->data - meta_len;
 
                copied = xsk_copy_xdp(copy_to, &copy_from, to_len, &from_len, &frag, rem);
                rem -= copied;
 
-               xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);
                __xsk_rcv_zc_safe(xs, xskb, copied - meta_len,
                                  rem ? XDP_PKT_CONTD : 0);
                meta_len = 0;
        } while (rem);
 
        return 0;
+
+err_alloc:
+       list_for_each_entry_safe(xskb, tmp, &xsk_buffs, list_node) {
+               list_del_init(&xskb->list_node);
+               xsk_buff_free(&xskb->xdp);
+       }
+       xs->rx_dropped++;
+       return -ENOMEM;
 }
 
 static bool xsk_tx_writeable(struct xdp_sock *xs)
--

2.53.0

#2 lore raw
In reply to Cen Zhang (Microsoft)
Quoted 56 lines · Cen Zhang (Microsoft)

In the __xsk_rcv() multi-buffer path, xsk_buff_alloc() is called in a loop without checking its return value. xsk_buff_can_alloc() only counts fill queue entries without validating their addresses, so it can succeed while xsk_buff_alloc() rejects all remaining entries and returns NULL.

  Oops: general protection fault, probably for non-canonical address
   0xdffffc0000000000
  KASAN: null-ptr-deref in range
   [0x0000000000000000-0x0000000000000007]
  RIP: 0010:__xsk_rcv+0x426/0xc20 (net/xdp/xsk.c:350)
  Call Trace:
   xsk_generic_rcv+0x26d/0x5f0
   xdp_do_generic_redirect+0x3c5/0xcf0
   do_xdp_generic+0x92f/0xe70
   __netif_receive_skb_core.constprop.0+0xf7e/0x2b30

Fix this with a two-stage transaction. First allocate and stage all buffers required for the packet, recycling all staged buffers with xsk_buff_free() if any allocation fails. Only after this stage succeeds, copy the data, reserve the RX descriptors, and release the buffers in an error-free loop.

Fixes 804627751b42 ("xsk: add support for AF_XDP multi-buffer on Rx path")
Reported-by [email protected]

---
v2:
- Allocate all packet buffers before reserving RX descriptors.
- Recycle partially allocated buffers instead of only cancelling the RX producer reservations.

 net/xdp/xsk.c | 29 ++++++++++++++++++++++++++---
 1 file changed, 26 insertions(+), 3 deletions(-)
net/xdp/xsk.c +3 -1
index f906d51b6699..383fc2b1de48 100644
@@ -298,9 +298,11 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
 	u32 frame_size = __xsk_pool_get_rx_frame_size(xs->pool);
 	void *copy_from = xsk_copy_xdp_start(xdp), *copy_to;
 	u32 from_len, meta_len, rem, num_desc;
-	struct xdp_buff_xsk *xskb;
+	struct xdp_buff_xsk *xskb, *tmp;
 	struct xdp_buff *xsk_xdp;
+	LIST_HEAD(xsk_buffs);
 	skb_frag_t *frag;
+	u32 i;
 
 	from_len = xdp->data_end - copy_from;
 	meta_len = xdp->data - copy_from;
@@ -343,23 +345,44 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
 		frag =  &sinfo->frags[0];
 	}

net/xdp/xsk.c static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) · line 356

Quoted 10 lines
net/xdp/xsk.c +9 -0
 
+	for (i = 0; i < num_desc; i++) {
+		xsk_xdp = xsk_buff_alloc(xs->pool);
+		if (!xsk_xdp)
+			goto err_alloc;
+
+		xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);
+		if (unlikely(!list_empty(&xskb->list_node)))
+			goto err_alloc;
+		list_add_tail(&xskb->list_node, &xsk_buffs);

could we use existing xsk_buff_add_frag() ?
then I presume xsk_buff_free() would understand list and walk through xdp_buff's and free it ?

I believe we could reuse pool's xskb_list instead of fabricating the on-stack variant here.

Quoted 37 lines
net/xdp/xsk.c +14 -2
+	}
+
 	do {
 		u32 to_len = frame_size + meta_len;
 		u32 copied;
 
-		xsk_xdp = xsk_buff_alloc(xs->pool);
+		xskb = list_first_entry(&xsk_buffs, struct xdp_buff_xsk,
+					list_node);
+		list_del_init(&xskb->list_node);
+		xsk_xdp = &xskb->xdp;
 		copy_to = xsk_xdp->data - meta_len;
 
 		copied = xsk_copy_xdp(copy_to, &copy_from, to_len, &from_len, &frag, rem);
 		rem -= copied;
 
-		xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);
 		__xsk_rcv_zc_safe(xs, xskb, copied - meta_len,
 				  rem ? XDP_PKT_CONTD : 0);
 		meta_len = 0;
 	} while (rem);
 
 	return 0;
+
+err_alloc:
+	list_for_each_entry_safe(xskb, tmp, &xsk_buffs, list_node) {
+		list_del_init(&xskb->list_node);
+		xsk_buff_free(&xskb->xdp);
+	}
+	xs->rx_dropped++;
+	return -ENOMEM;
 }
 
 static bool xsk_tx_writeable(struct xdp_sock *xs)
Signature · 1 line
2.53.0
Reviewed-by
#3 lore raw

Hi Maciej,

Quoted 25 lines · Maciej Fijalkowski
Quoted 23 lines · Cen Zhang (Microsoft)

In the __xsk_rcv() multi-buffer path, xsk_buff_alloc() is called in a loop without checking its return value. xsk_buff_can_alloc() only counts fill queue entries without validating their addresses, so it can succeed while xsk_buff_alloc() rejects all remaining entries and returns NULL.

  Oops: general protection fault, probably for non-canonical address
   0xdffffc0000000000
  KASAN: null-ptr-deref in range
   [0x0000000000000000-0x0000000000000007]
  RIP: 0010:__xsk_rcv+0x426/0xc20 (net/xdp/xsk.c:350)
  Call Trace:
   xsk_generic_rcv+0x26d/0x5f0
   xdp_do_generic_redirect+0x3c5/0xcf0
   do_xdp_generic+0x92f/0xe70
   __netif_receive_skb_core.constprop.0+0xf7e/0x2b30

Fix this with a two-stage transaction. First allocate and stage all buffers required for the packet, recycling all staged buffers with xsk_buff_free() if any allocation fails. Only after this stage succeeds, copy the data, reserve the RX descriptors, and release the buffers in an error-free loop.

Quoted 3 lines
Quoted 3 lines
Fixes 804627751b42 ("xsk: add support for AF_XDP multi-buffer on Rx path")
Reported-by [email protected]
Quoted 31 lines
Quoted 31 lines

---
v2:
- Allocate all packet buffers before reserving RX descriptors.
- Recycle partially allocated buffers instead of only cancelling the RX producer reservations.

 net/xdp/xsk.c | 29 ++++++++++++++++++++++++++---
 1 file changed, 26 insertions(+), 3 deletions(-)
net/xdp/xsk.c +3 -1
index f906d51b6699..383fc2b1de48 100644
@@ -298,9 +298,11 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
      u32 frame_size = __xsk_pool_get_rx_frame_size(xs->pool);
      void *copy_from = xsk_copy_xdp_start(xdp), *copy_to;
      u32 from_len, meta_len, rem, num_desc;
-     struct xdp_buff_xsk *xskb;
+     struct xdp_buff_xsk *xskb, *tmp;
      struct xdp_buff *xsk_xdp;
+     LIST_HEAD(xsk_buffs);
      skb_frag_t *frag;
+     u32 i;
 
      from_len = xdp->data_end - copy_from;
      meta_len = xdp->data - copy_from;
@@ -343,23 +345,44 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
              frag =  &sinfo->frags[0];
      }
 
Quoted 13 lines

net/xdp/xsk.c static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) · line 356

Quoted 9 lines
net/xdp/xsk.c +9 -0
+     for (i = 0; i < num_desc; i++) {
+             xsk_xdp = xsk_buff_alloc(xs->pool);
+             if (!xsk_xdp)
+                     goto err_alloc;
+
+             xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);
+             if (unlikely(!list_empty(&xskb->list_node)))
+                     goto err_alloc;
+             list_add_tail(&xskb->list_node, &xsk_buffs);

could we use existing xsk_buff_add_frag() ?
then I presume xsk_buff_free() would understand list and walk through xdp_buff's and free it ?

Good suggestion, but xsk_buff_add_frag() will add more irrelevant stuff like nr_frags, xdp_frags_size, xdp_frags_truesize... They are all happening in the softirq context. Refactoring the helper would bring more work here.

Honestly, I like the local array which seems simpler/cleaner to understand :)

Thanks,
Jason

Quoted 42 lines

I believe we could reuse pool's xskb_list instead of fabricating the on-stack variant here.

Quoted 37 lines
net/xdp/xsk.c +14 -3
+     }
+
      do {
              u32 to_len = frame_size + meta_len;
              u32 copied;
 
-             xsk_xdp = xsk_buff_alloc(xs->pool);
+             xskb = list_first_entry(&xsk_buffs, struct xdp_buff_xsk,
+                                     list_node);
+             list_del_init(&xskb->list_node);
+             xsk_xdp = &xskb->xdp;
              copy_to = xsk_xdp->data - meta_len;
 
              copied = xsk_copy_xdp(copy_to, &copy_from, to_len, &from_len, &frag, rem);
              rem -= copied;
 
-             xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);
              __xsk_rcv_zc_safe(xs, xskb, copied - meta_len,
                                rem ? XDP_PKT_CONTD : 0);
              meta_len = 0;
      } while (rem);
 
      return 0;
+
+err_alloc:
+     list_for_each_entry_safe(xskb, tmp, &xsk_buffs, list_node) {
+             list_del_init(&xskb->list_node);
+             xsk_buff_free(&xskb->xdp);
+     }
+     xs->rx_dropped++;
+     return -ENOMEM;
 }
 
 static bool xsk_tx_writeable(struct xdp_sock *xs)
--

2.53.0

#8 lore raw
In reply to Cen Zhang (Microsoft)
Quoted 56 lines · Cen Zhang (Microsoft)

In the __xsk_rcv() multi-buffer path, xsk_buff_alloc() is called in a loop without checking its return value. xsk_buff_can_alloc() only counts fill queue entries without validating their addresses, so it can succeed while xsk_buff_alloc() rejects all remaining entries and returns NULL.

  Oops: general protection fault, probably for non-canonical address
   0xdffffc0000000000
  KASAN: null-ptr-deref in range
   [0x0000000000000000-0x0000000000000007]
  RIP: 0010:__xsk_rcv+0x426/0xc20 (net/xdp/xsk.c:350)
  Call Trace:
   xsk_generic_rcv+0x26d/0x5f0
   xdp_do_generic_redirect+0x3c5/0xcf0
   do_xdp_generic+0x92f/0xe70
   __netif_receive_skb_core.constprop.0+0xf7e/0x2b30

Fix this with a two-stage transaction. First allocate and stage all buffers required for the packet, recycling all staged buffers with xsk_buff_free() if any allocation fails. Only after this stage succeeds, copy the data, reserve the RX descriptors, and release the buffers in an error-free loop.

Fixes 804627751b42 ("xsk: add support for AF_XDP multi-buffer on Rx path")
Reported-by [email protected]

---
v2:
- Allocate all packet buffers before reserving RX descriptors.
- Recycle partially allocated buffers instead of only cancelling the RX producer reservations.

 net/xdp/xsk.c | 29 ++++++++++++++++++++++++++---
 1 file changed, 26 insertions(+), 3 deletions(-)
net/xdp/xsk.c +3 -1
index f906d51b6699..383fc2b1de48 100644
@@ -298,9 +298,11 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
        u32 frame_size = __xsk_pool_get_rx_frame_size(xs->pool);
        void *copy_from = xsk_copy_xdp_start(xdp), *copy_to;
        u32 from_len, meta_len, rem, num_desc;
-       struct xdp_buff_xsk *xskb;
+       struct xdp_buff_xsk *xskb, *tmp;
        struct xdp_buff *xsk_xdp;
+       LIST_HEAD(xsk_buffs);
        skb_frag_t *frag;
+       u32 i;
 
        from_len = xdp->data_end - copy_from;
        meta_len = xdp->data - copy_from;
@@ -343,23 +345,44 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
                frag =  &sinfo->frags[0];

net/xdp/xsk.c static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) · line 355

Quoted 10 lines
net/xdp/xsk.c +8 -0
        }
 
+       for (i = 0; i < num_desc; i++) {
+               xsk_xdp = xsk_buff_alloc(xs->pool);
+               if (!xsk_xdp)
+                       goto err_alloc;
+
+               xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);
+               if (unlikely(!list_empty(&xskb->list_node)))
+                       goto err_alloc;

It will cause a memory leak because the current xsk_xdp that is not added to the local list will miss the chance to get freed? And the empty list_node cannot be easily freed by xp_free()...

Thanks,
Jason

Quoted 39 lines
net/xdp/xsk.c +15 -3
+               list_add_tail(&xskb->list_node, &xsk_buffs);
+       }
+
        do {
                u32 to_len = frame_size + meta_len;
                u32 copied;
 
-               xsk_xdp = xsk_buff_alloc(xs->pool);
+               xskb = list_first_entry(&xsk_buffs, struct xdp_buff_xsk,
+                                       list_node);
+               list_del_init(&xskb->list_node);
+               xsk_xdp = &xskb->xdp;
                copy_to = xsk_xdp->data - meta_len;
 
                copied = xsk_copy_xdp(copy_to, &copy_from, to_len, &from_len, &frag, rem);
                rem -= copied;
 
-               xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);
                __xsk_rcv_zc_safe(xs, xskb, copied - meta_len,
                                  rem ? XDP_PKT_CONTD : 0);
                meta_len = 0;
        } while (rem);
 
        return 0;
+
+err_alloc:
+       list_for_each_entry_safe(xskb, tmp, &xsk_buffs, list_node) {
+               list_del_init(&xskb->list_node);
+               xsk_buff_free(&xskb->xdp);
+       }
+       xs->rx_dropped++;
+       return -ENOMEM;
 }
 
 static bool xsk_tx_writeable(struct xdp_sock *xs)
--

2.53.0

#4 lore raw
In reply to Jason Xing
Quoted 80 lines · Jason Xing

Hi Maciej,

Quoted 25 lines · Maciej Fijalkowski
Quoted 23 lines · Cen Zhang (Microsoft)

In the __xsk_rcv() multi-buffer path, xsk_buff_alloc() is called in a loop without checking its return value. xsk_buff_can_alloc() only counts fill queue entries without validating their addresses, so it can succeed while xsk_buff_alloc() rejects all remaining entries and returns NULL.

  Oops: general protection fault, probably for non-canonical address
   0xdffffc0000000000
  KASAN: null-ptr-deref in range
   [0x0000000000000000-0x0000000000000007]
  RIP: 0010:__xsk_rcv+0x426/0xc20 (net/xdp/xsk.c:350)
  Call Trace:
   xsk_generic_rcv+0x26d/0x5f0
   xdp_do_generic_redirect+0x3c5/0xcf0
   do_xdp_generic+0x92f/0xe70
   __netif_receive_skb_core.constprop.0+0xf7e/0x2b30

Fix this with a two-stage transaction. First allocate and stage all buffers required for the packet, recycling all staged buffers with xsk_buff_free() if any allocation fails. Only after this stage succeeds, copy the data, reserve the RX descriptors, and release the buffers in an error-free loop.

Quoted 3 lines
Quoted 3 lines
Fixes 804627751b42 ("xsk: add support for AF_XDP multi-buffer on Rx path")
Reported-by [email protected]
Quoted 31 lines
Quoted 31 lines

---
v2:
- Allocate all packet buffers before reserving RX descriptors.
- Recycle partially allocated buffers instead of only cancelling the RX producer reservations.

 net/xdp/xsk.c | 29 ++++++++++++++++++++++++++---
 1 file changed, 26 insertions(+), 3 deletions(-)
net/xdp/xsk.c +3 -1
index f906d51b6699..383fc2b1de48 100644
@@ -298,9 +298,11 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
      u32 frame_size = __xsk_pool_get_rx_frame_size(xs->pool);
      void *copy_from = xsk_copy_xdp_start(xdp), *copy_to;
      u32 from_len, meta_len, rem, num_desc;
-     struct xdp_buff_xsk *xskb;
+     struct xdp_buff_xsk *xskb, *tmp;
      struct xdp_buff *xsk_xdp;
+     LIST_HEAD(xsk_buffs);
      skb_frag_t *frag;
+     u32 i;
 
      from_len = xdp->data_end - copy_from;
      meta_len = xdp->data - copy_from;
@@ -343,23 +345,44 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
              frag =  &sinfo->frags[0];
      }
 
Quoted 13 lines

net/xdp/xsk.c static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) · line 356

Quoted 9 lines
net/xdp/xsk.c +9 -0
+     for (i = 0; i < num_desc; i++) {
+             xsk_xdp = xsk_buff_alloc(xs->pool);
+             if (!xsk_xdp)
+                     goto err_alloc;
+
+             xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);
+             if (unlikely(!list_empty(&xskb->list_node)))
+                     goto err_alloc;
+             list_add_tail(&xskb->list_node, &xsk_buffs);

could we use existing xsk_buff_add_frag() ?
then I presume xsk_buff_free() would understand list and walk through xdp_buff's and free it ?

Quoted 4 lines

Good suggestion, but xsk_buff_add_frag() will add more irrelevant stuff like nr_frags, xdp_frags_size, xdp_frags_truesize... They are all happening in the softirq context. Refactoring the helper would bring more work here.

Fair enough, how about we meet in the halfway?

Don't use xxx_add_frag but reuse pool's list. Reason I'm pushing for it is xsk_buff_free() has been thought to consume multi-buffer xskb's, so all the list-walking would be hidden and error path would only consist of a single free() call.

then the do/while loop while iterating could be using xsk_buff_add_frag()

Quoted 48 lines

Honestly, I like the local array which seems simpler/cleaner to understand :)

Thanks,
Jason

Quoted 42 lines

I believe we could reuse pool's xskb_list instead of fabricating the on-stack variant here.

Quoted 37 lines
net/xdp/xsk.c +14 -3
+     }
+
      do {
              u32 to_len = frame_size + meta_len;
              u32 copied;
 
-             xsk_xdp = xsk_buff_alloc(xs->pool);
+             xskb = list_first_entry(&xsk_buffs, struct xdp_buff_xsk,
+                                     list_node);
+             list_del_init(&xskb->list_node);
+             xsk_xdp = &xskb->xdp;
              copy_to = xsk_xdp->data - meta_len;
 
              copied = xsk_copy_xdp(copy_to, &copy_from, to_len, &from_len, &frag, rem);
              rem -= copied;
 
-             xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);
              __xsk_rcv_zc_safe(xs, xskb, copied - meta_len,
                                rem ? XDP_PKT_CONTD : 0);
              meta_len = 0;
      } while (rem);
 
      return 0;
+
+err_alloc:
+     list_for_each_entry_safe(xskb, tmp, &xsk_buffs, list_node) {
+             list_del_init(&xskb->list_node);
+             xsk_buff_free(&xskb->xdp);
+     }
+     xs->rx_dropped++;
+     return -ENOMEM;
 }
 
 static bool xsk_tx_writeable(struct xdp_sock *xs)
--

2.53.0

#5 lore raw
Quoted 87 lines · Maciej Fijalkowski
Quoted 80 lines · Jason Xing

Hi Maciej,

Quoted 25 lines · Maciej Fijalkowski
Quoted 23 lines · Cen Zhang (Microsoft)

In the __xsk_rcv() multi-buffer path, xsk_buff_alloc() is called in a loop without checking its return value. xsk_buff_can_alloc() only counts fill queue entries without validating their addresses, so it can succeed while xsk_buff_alloc() rejects all remaining entries and returns NULL.

  Oops: general protection fault, probably for non-canonical address
   0xdffffc0000000000
  KASAN: null-ptr-deref in range
   [0x0000000000000000-0x0000000000000007]
  RIP: 0010:__xsk_rcv+0x426/0xc20 (net/xdp/xsk.c:350)
  Call Trace:
   xsk_generic_rcv+0x26d/0x5f0
   xdp_do_generic_redirect+0x3c5/0xcf0
   do_xdp_generic+0x92f/0xe70
   __netif_receive_skb_core.constprop.0+0xf7e/0x2b30

Fix this with a two-stage transaction. First allocate and stage all buffers required for the packet, recycling all staged buffers with xsk_buff_free() if any allocation fails. Only after this stage succeeds, copy the data, reserve the RX descriptors, and release the buffers in an error-free loop.

Quoted 3 lines
Quoted 3 lines
Fixes 804627751b42 ("xsk: add support for AF_XDP multi-buffer on Rx path")
Reported-by [email protected]
Quoted 31 lines
Quoted 31 lines

---
v2:
- Allocate all packet buffers before reserving RX descriptors.
- Recycle partially allocated buffers instead of only cancelling the RX producer reservations.

 net/xdp/xsk.c | 29 ++++++++++++++++++++++++++---
 1 file changed, 26 insertions(+), 3 deletions(-)
net/xdp/xsk.c +3 -1
index f906d51b6699..383fc2b1de48 100644
@@ -298,9 +298,11 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
      u32 frame_size = __xsk_pool_get_rx_frame_size(xs->pool);
      void *copy_from = xsk_copy_xdp_start(xdp), *copy_to;
      u32 from_len, meta_len, rem, num_desc;
-     struct xdp_buff_xsk *xskb;
+     struct xdp_buff_xsk *xskb, *tmp;
      struct xdp_buff *xsk_xdp;
+     LIST_HEAD(xsk_buffs);
      skb_frag_t *frag;
+     u32 i;
 
      from_len = xdp->data_end - copy_from;
      meta_len = xdp->data - copy_from;
@@ -343,23 +345,44 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
              frag =  &sinfo->frags[0];
      }
 
Quoted 13 lines

net/xdp/xsk.c static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) · line 356

Quoted 9 lines
net/xdp/xsk.c +9 -0
+     for (i = 0; i < num_desc; i++) {
+             xsk_xdp = xsk_buff_alloc(xs->pool);
+             if (!xsk_xdp)
+                     goto err_alloc;
+
+             xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);
+             if (unlikely(!list_empty(&xskb->list_node)))
+                     goto err_alloc;
+             list_add_tail(&xskb->list_node, &xsk_buffs);

could we use existing xsk_buff_add_frag() ?
then I presume xsk_buff_free() would understand list and walk through xdp_buff's and free it ?

Quoted 4 lines

Good suggestion, but xsk_buff_add_frag() will add more irrelevant stuff like nr_frags, xdp_frags_size, xdp_frags_truesize... They are all happening in the softirq context. Refactoring the helper would bring more work here.

Quoted 6 lines

Fair enough, how about we meet in the halfway?

Don't use xxx_add_frag but reuse pool's list. Reason I'm pushing for it is xsk_buff_free() has been thought to consume multi-buffer xskb's, so all the list-walking would be hidden and error path would only consist of a single free() call.

It does make sense (reusing/developing common helpers is always a good way to go). Side note: maybe Cen needs to handle xdp_buff_set_frags_flag() to allow xsk_buff_free() freeing a list of nodes.

Quoted 2 lines

then the do/while loop while iterating could be using xsk_buff_add_frag()

But I don't get it. Sorry. Are you saying we still need to call xsk_buff_add_frag()?

Thanks,
Jason

Quoted 49 lines
Quoted 49 lines

Honestly, I like the local array which seems simpler/cleaner to understand :)

Thanks,
Jason

Quoted 42 lines

I believe we could reuse pool's xskb_list instead of fabricating the on-stack variant here.

Quoted 37 lines
net/xdp/xsk.c +14 -3
+     }
+
      do {
              u32 to_len = frame_size + meta_len;
              u32 copied;
 
-             xsk_xdp = xsk_buff_alloc(xs->pool);
+             xskb = list_first_entry(&xsk_buffs, struct xdp_buff_xsk,
+                                     list_node);
+             list_del_init(&xskb->list_node);
+             xsk_xdp = &xskb->xdp;
              copy_to = xsk_xdp->data - meta_len;
 
              copied = xsk_copy_xdp(copy_to, &copy_from, to_len, &from_len, &frag, rem);
              rem -= copied;
 
-             xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);
              __xsk_rcv_zc_safe(xs, xskb, copied - meta_len,
                                rem ? XDP_PKT_CONTD : 0);
              meta_len = 0;
      } while (rem);
 
      return 0;
+
+err_alloc:
+     list_for_each_entry_safe(xskb, tmp, &xsk_buffs, list_node) {
+             list_del_init(&xskb->list_node);
+             xsk_buff_free(&xskb->xdp);
+     }
+     xs->rx_dropped++;
+     return -ENOMEM;
 }
 
 static bool xsk_tx_writeable(struct xdp_sock *xs)
--

2.53.0

#6 lore raw
Quoted 96 lines · Jason Xing
Quoted 87 lines · Maciej Fijalkowski
Quoted 80 lines · Jason Xing

Hi Maciej,

Quoted 25 lines · Maciej Fijalkowski
Quoted 23 lines · Cen Zhang (Microsoft)

In the __xsk_rcv() multi-buffer path, xsk_buff_alloc() is called in a loop without checking its return value. xsk_buff_can_alloc() only counts fill queue entries without validating their addresses, so it can succeed while xsk_buff_alloc() rejects all remaining entries and returns NULL.

  Oops: general protection fault, probably for non-canonical address
   0xdffffc0000000000
  KASAN: null-ptr-deref in range
   [0x0000000000000000-0x0000000000000007]
  RIP: 0010:__xsk_rcv+0x426/0xc20 (net/xdp/xsk.c:350)
  Call Trace:
   xsk_generic_rcv+0x26d/0x5f0
   xdp_do_generic_redirect+0x3c5/0xcf0
   do_xdp_generic+0x92f/0xe70
   __netif_receive_skb_core.constprop.0+0xf7e/0x2b30

Fix this with a two-stage transaction. First allocate and stage all buffers required for the packet, recycling all staged buffers with xsk_buff_free() if any allocation fails. Only after this stage succeeds, copy the data, reserve the RX descriptors, and release the buffers in an error-free loop.

Quoted 3 lines
Quoted 3 lines
Fixes 804627751b42 ("xsk: add support for AF_XDP multi-buffer on Rx path")
Reported-by [email protected]
Quoted 31 lines
Quoted 31 lines

---
v2:
- Allocate all packet buffers before reserving RX descriptors.
- Recycle partially allocated buffers instead of only cancelling the RX producer reservations.

 net/xdp/xsk.c | 29 ++++++++++++++++++++++++++---
 1 file changed, 26 insertions(+), 3 deletions(-)
net/xdp/xsk.c +3 -1
index f906d51b6699..383fc2b1de48 100644
@@ -298,9 +298,11 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
      u32 frame_size = __xsk_pool_get_rx_frame_size(xs->pool);
      void *copy_from = xsk_copy_xdp_start(xdp), *copy_to;
      u32 from_len, meta_len, rem, num_desc;
-     struct xdp_buff_xsk *xskb;
+     struct xdp_buff_xsk *xskb, *tmp;
      struct xdp_buff *xsk_xdp;
+     LIST_HEAD(xsk_buffs);
      skb_frag_t *frag;
+     u32 i;
 
      from_len = xdp->data_end - copy_from;
      meta_len = xdp->data - copy_from;
@@ -343,23 +345,44 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
              frag =  &sinfo->frags[0];
      }
 
Quoted 13 lines

net/xdp/xsk.c static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) · line 356

Quoted 9 lines
net/xdp/xsk.c +9 -0
+     for (i = 0; i < num_desc; i++) {
+             xsk_xdp = xsk_buff_alloc(xs->pool);
+             if (!xsk_xdp)
+                     goto err_alloc;
+
+             xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);
+             if (unlikely(!list_empty(&xskb->list_node)))
+                     goto err_alloc;
+             list_add_tail(&xskb->list_node, &xsk_buffs);

could we use existing xsk_buff_add_frag() ?
then I presume xsk_buff_free() would understand list and walk through xdp_buff's and free it ?

Quoted 4 lines

Good suggestion, but xsk_buff_add_frag() will add more irrelevant stuff like nr_frags, xdp_frags_size, xdp_frags_truesize... They are all happening in the softirq context. Refactoring the helper would bring more work here.

Quoted 6 lines

Fair enough, how about we meet in the halfway?

Don't use xxx_add_frag but reuse pool's list. Reason I'm pushing for it is xsk_buff_free() has been thought to consume multi-buffer xskb's, so all the list-walking would be hidden and error path would only consist of a single free() call.

Quoted 4 lines

It does make sense (reusing/developing common helpers is always a good way to go). Side note: maybe Cen needs to handle xdp_buff_set_frags_flag() to allow xsk_buff_free() freeing a list of nodes.

Yeah exactly this needs to be done on a 'head' xskb.

Quoted 6 lines
Quoted 3 lines

then the do/while loop while iterating could be using xsk_buff_add_frag()

But I don't get it. Sorry. Are you saying we still need to call xsk_buff_add_frag()?

Sorry i meant xsk_buff_get_frag() which would be peeeling off xskbs from pool's xskb_list.

Quoted 53 lines

Thanks,
Jason

Quoted 49 lines
Quoted 49 lines

Honestly, I like the local array which seems simpler/cleaner to understand :)

Thanks,
Jason

Quoted 42 lines

I believe we could reuse pool's xskb_list instead of fabricating the on-stack variant here.

Quoted 37 lines
net/xdp/xsk.c +14 -3
+     }
+
      do {
              u32 to_len = frame_size + meta_len;
              u32 copied;
 
-             xsk_xdp = xsk_buff_alloc(xs->pool);
+             xskb = list_first_entry(&xsk_buffs, struct xdp_buff_xsk,
+                                     list_node);
+             list_del_init(&xskb->list_node);
+             xsk_xdp = &xskb->xdp;
              copy_to = xsk_xdp->data - meta_len;
 
              copied = xsk_copy_xdp(copy_to, &copy_from, to_len, &from_len, &frag, rem);
              rem -= copied;
 
-             xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);
              __xsk_rcv_zc_safe(xs, xskb, copied - meta_len,
                                rem ? XDP_PKT_CONTD : 0);
              meta_len = 0;
      } while (rem);
 
      return 0;
+
+err_alloc:
+     list_for_each_entry_safe(xskb, tmp, &xsk_buffs, list_node) {
+             list_del_init(&xskb->list_node);
+             xsk_buff_free(&xskb->xdp);
+     }
+     xs->rx_dropped++;
+     return -ENOMEM;
 }
 
 static bool xsk_tx_writeable(struct xdp_sock *xs)
--

2.53.0

#7 lore raw

Thanks for the suggestions. I am back and will prepare and send v3 in the next few days.