virtio_net: add XDP meta data support
Subsystems (4)
PATCH 1/1 virtio_net: add XDP meta data support
This adds XDP meta data support to both receive_small() and receive_mergeable().
--- drivers/net/virtio_net.c | 40 +++++++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 11 deletions(-)
drivers/net/virtio_net.c +29 -11
@@ -371,7 +371,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,struct receive_queue *rq, struct page *page, unsigned int offset, unsigned int len, unsigned int truesize, - bool hdr_valid) + bool hdr_valid, unsigned int metasize) { struct sk_buff *skb;struct virtio_net_hdr_mrg_rxbuf *hdr;
@@ -393,17 +393,25 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,else hdr_padded_len =sizeof (struct padded_vnet_hdr ); -if (hdr_valid) +if (hdr_valid && !metasize)memcpy (hdr, p, hdr_len); len -= hdr_len; offset += hdr_padded_len; p += hdr_padded_len; - copy = len; + copy = len + metasize;if (copy >skb_tailroom (skb)) copy =skb_tailroom (skb); -skb_put_data (skb, p, copy); + +if (metasize) { +skb_put_data (skb, p - metasize, copy); +__skb_pull (skb, metasize); +skb_metadata_set (skb, metasize); + copy -= metasize; + }else { +skb_put_data (skb, p, copy); + } len -= copy; offset += copy;
@@ -644,6 +652,7 @@ static struct sk_buff *receive_small(struct net_device *dev,unsigned int delta =0 ;struct page *xdp_page;int err; +unsigned int metasize =0 ; len -= vi->hdr_len; stats->bytes += len;
@@ -683,8 +692,8 @@ static struct sk_buff *receive_small(struct net_device *dev, xdp.data_hard_start = buf +VIRTNET_RX_PAD + vi->hdr_len; xdp.data = xdp.data_hard_start + xdp_headroom; -xdp_set_data_meta_invalid (&xdp); xdp.data_end = xdp.data + len; + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; orig_data = xdp.data; act =bpf_prog_run_xdp (xdp_prog, &xdp);
@@ -695,9 +704,11 @@ static struct sk_buff *receive_small(struct net_device *dev,/* Recalculate length in case bpf program changed it */ delta = orig_data - xdp.data; len = xdp.data_end - xdp.data; + metasize = xdp.data - xdp.data_meta;break ;case XDP_TX : stats->xdp_tx++; + xdp.data_meta = xdp.data; xdpf =convert_to_xdp_frame (&xdp);if (unlikely (!xdpf)) goto err_xdp;
@@ -735,11 +746,14 @@ static struct sk_buff *receive_small(struct net_device *dev, }skb_reserve (skb, headroom - delta);skb_put (skb, len); -if (!delta) { +if (!delta && !metasize) { buf += header_offset;memcpy (skb_vnet_hdr (skb), buf, vi->hdr_len); }/* keep zeroed vnet hdr since packet was changed by bpf */ +if (metasize) +skb_metadata_set (skb, metasize); + err:return skb;
@@ -761,7 +775,7 @@ static struct sk_buff *receive_big(struct net_device *dev, {struct page *page = buf;struct sk_buff *skb =page_to_skb (vi, rq, page,0 , len, -PAGE_SIZE , true); +PAGE_SIZE , true,0 ); stats->bytes += len - vi->hdr_len;if (unlikely (!skb))
@@ -793,6 +807,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,unsigned int truesize;unsigned int headroom =mergeable_ctx_to_headroom (ctx);int err; +unsigned int metasize =0 ; head_skb =NULL ; stats->bytes += len - vi->hdr_len;
@@ -839,8 +854,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, data =page_address (xdp_page) + offset; xdp.data_hard_start = data -VIRTIO_XDP_HEADROOM + vi->hdr_len; xdp.data = data + vi->hdr_len; -xdp_set_data_meta_invalid (&xdp); xdp.data_end = xdp.data + (len - vi->hdr_len); + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; act =bpf_prog_run_xdp (xdp_prog, &xdp);
@@ -859,18 +874,20 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,* adjusted */ len = xdp.data_end - xdp.data + vi->hdr_len; + metasize = xdp.data - xdp.data_meta;/* We can only create skb based on xdp_page. */ if (unlikely (xdp_page != page)) {rcu_read_unlock ();put_page (page); head_skb =page_to_skb (vi, rq, xdp_page, - offset, len, -PAGE_SIZE , false); + offset, len, +PAGE_SIZE , false, metasize);return head_skb; }break ;case XDP_TX : stats->xdp_tx++; + xdp.data_meta = xdp.data; xdpf =convert_to_xdp_frame (&xdp);if (unlikely (!xdpf)) goto err_xdp;
@@ -921,7 +938,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, goto err_skb; } - head_skb =page_to_skb (vi, rq, page, offset, len, truesize, !xdp_prog); + head_skb =page_to_skb (vi, rq, page, offset, len, truesize, !xdp_prog, + metasize); curr_skb = head_skb;if (unlikely (!curr_skb))
Signature · 1 line
2.20.1
Quoted 32 lines · Yuya Kusakabe
This adds XDP meta data support to both
receive_small()andreceive_mergeable().Fixes de8f3a83b0a0 ("bpf: add meta pointer for direct access")Signed-off-by Yuya Kusakabe <[email protected]>--- drivers/net/virtio_net.c | 40 +++++++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 11 deletions(-)drivers/net/virtio_net.c +2
-2@@ -371,7 +371,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,struct receive_queue *rq, struct page *page, unsigned int offset, unsigned int len, unsigned int truesize, - bool hdr_valid) + bool hdr_valid, unsigned int metasize) { struct sk_buff *skb;struct virtio_net_hdr_mrg_rxbuf *hdr;@@ -393,17 +393,25 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,else hdr_padded_len =sizeof (struct padded_vnet_hdr ); -if (hdr_valid) +if (hdr_valid && !metasize)memcpy (hdr, p, hdr_len); len -= hdr_len; offset += hdr_padded_len;
drivers/net/virtio_net.c static struct sk_buff *page_to_skb(struct virtnet_info *vi, · line 408
Quoted 10 lines
drivers/net/virtio_net.c +4
-2p += hdr_padded_len; - copy = len; + copy = len + metasize;if (copy >skb_tailroom (skb)) copy =skb_tailroom (skb); -skb_put_data (skb, p, copy); + +if (metasize) { +skb_put_data (skb, p - metasize, copy);
I would rather keep copy untouched above, and use copy + metasize here, then you can save the following decrement as well. Or tweak the caller the count the meta in to offset, then we need only deal with skb_pull() and skb_metadata_set() here.
Quoted 28 lines
drivers/net/virtio_net.c +8
-1+__skb_pull (skb, metasize); +skb_metadata_set (skb, metasize); + copy -= metasize; + }else { +skb_put_data (skb, p, copy); + } len -= copy; offset += copy;@@ -644,6 +652,7 @@ static struct sk_buff *receive_small(struct net_device *dev,unsigned int delta =0 ;struct page *xdp_page;int err; +unsigned int metasize =0 ; len -= vi->hdr_len; stats->bytes += len;@@ -683,8 +692,8 @@ static struct sk_buff *receive_small(struct net_device *dev, xdp.data_hard_start = buf +VIRTNET_RX_PAD + vi->hdr_len; xdp.data = xdp.data_hard_start + xdp_headroom; -xdp_set_data_meta_invalid (&xdp); xdp.data_end = xdp.data + len; + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; orig_data = xdp.data; act =bpf_prog_run_xdp (xdp_prog, &xdp);
drivers/net/virtio_net.c static struct sk_buff *receive_small(struct net_device *dev, · line 711
Quoted 8 lines
drivers/net/virtio_net.c +2
-0@@ -695,9 +704,11 @@ static struct sk_buff *receive_small(struct net_device *dev,/* Recalculate length in case bpf program changed it */ delta = orig_data - xdp.data; len = xdp.data_end - xdp.data; + metasize = xdp.data - xdp.data_meta;break ;case XDP_TX : stats->xdp_tx++; + xdp.data_meta = xdp.data;
Why need this?
drivers/net/virtio_net.c static struct sk_buff *receive_small(struct net_device *dev, · line 752
Quoted 12 lines
drivers/net/virtio_net.c +1
-1xdpf =convert_to_xdp_frame (&xdp);if (unlikely (!xdpf)) goto err_xdp;@@ -735,11 +746,14 @@ static struct sk_buff *receive_small(struct net_device *dev, }skb_reserve (skb, headroom - delta);skb_put (skb, len); -if (!delta) { +if (!delta && !metasize) { buf += header_offset;memcpy (skb_vnet_hdr (skb), buf, vi->hdr_len); }/* keep zeroed vnet hdr since packet was changed by bpf */
Is there any method to preserve the vnet header here? We probably don't want to lose it for XDP_PASS when packet is not modified.
Quoted 38 lines
drivers/net/virtio_net.c +6
-2+if (metasize) +skb_metadata_set (skb, metasize); + err:return skb;@@ -761,7 +775,7 @@ static struct sk_buff *receive_big(struct net_device *dev, {struct page *page = buf;struct sk_buff *skb =page_to_skb (vi, rq, page,0 , len, -PAGE_SIZE , true); +PAGE_SIZE , true,0 ); stats->bytes += len - vi->hdr_len;if (unlikely (!skb))@@ -793,6 +807,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,unsigned int truesize;unsigned int headroom =mergeable_ctx_to_headroom (ctx);int err; +unsigned int metasize =0 ; head_skb =NULL ; stats->bytes += len - vi->hdr_len;@@ -839,8 +854,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, data =page_address (xdp_page) + offset; xdp.data_hard_start = data -VIRTIO_XDP_HEADROOM + vi->hdr_len; xdp.data = data + vi->hdr_len; -xdp_set_data_meta_invalid (&xdp); xdp.data_end = xdp.data + (len - vi->hdr_len); + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; act =bpf_prog_run_xdp (xdp_prog, &xdp);@@ -859,18 +874,20 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,* adjusted */ len = xdp.data_end - xdp.data + vi->hdr_len;
drivers/net/virtio_net.c static struct sk_buff *receive_mergeable(struct net_device *dev, · line 884
Quoted 10 lines
drivers/net/virtio_net.c +3
-2+ metasize = xdp.data - xdp.data_meta;/* We can only create skb based on xdp_page. */ if (unlikely (xdp_page != page)) {rcu_read_unlock ();put_page (page); head_skb =page_to_skb (vi, rq, xdp_page, - offset, len, -PAGE_SIZE , false); + offset, len, +PAGE_SIZE , false, metasize);
Indentation is wired.
Thanks
Quoted 19 lines
drivers/net/virtio_net.c +3
-1return head_skb; }break ;case XDP_TX : stats->xdp_tx++; + xdp.data_meta = xdp.data; xdpf =convert_to_xdp_frame (&xdp);if (unlikely (!xdpf)) goto err_xdp;@@ -921,7 +938,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, goto err_skb; } - head_skb =page_to_skb (vi, rq, page, offset, len, truesize, !xdp_prog); + head_skb =page_to_skb (vi, rq, page, offset, len, truesize, !xdp_prog, + metasize); curr_skb = head_skb;if (unlikely (!curr_skb))
Quoted 36 lines · Jason Wang
Quoted 34 lines · Yuya Kusakabe
This adds XDP meta data support to both
receive_small()andreceive_mergeable().Fixes de8f3a83b0a0 ("bpf: add meta pointer for direct access")Signed-off-by Yuya Kusakabe <[email protected]>---
drivers/net/virtio_net.c| 40 +++++++++++++++++++++++++++++-----------1 file changed, 29 insertions(+), 11 deletions(-)drivers/net/virtio_net.c +1
-1@@ -371,7 +371,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, struct receive_queue *rq, struct page *page, unsigned int offset, unsigned int len, unsigned int truesize, - bool hdr_valid) + bool hdr_valid, unsigned int metasize) { struct sk_buff *skb; struct virtio_net_hdr_mrg_rxbuf *hdr;@@ -393,17 +393,25 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,else
hdr_padded_len=sizeof(struct padded_vnet_hdr); - if (hdr_valid)
+ if (hdr_valid&& !metasize)
memcpy(hdr, p, hdr_len);
len -=hdr_len;
offset +=hdr_padded_len;
p +=hdr_padded_len;
- copy = len;
+ copy = len + metasize;
if (copy >skb_tailroom(skb))
Quoted 8 lines
Quoted 5 lines
copy =
skb_tailroom(skb);
-skb_put_data(skb, p, copy);
+
+ if (metasize) {
+skb_put_data(skb, p - metasize, copy);I would rather keep copy untouched above, and use copy + metasize here, then you can save the following decrement as well. Or tweak the caller the count the meta in to offset, then we need only deal with
skb_pull()andskb_metadata_set()here.
I think the latter is better, because copy + metasize must be smaller than skb tailroom size.
Quoted 29 lines
Quoted 29 lines
+
__skb_pull(skb, metasize);
+skb_metadata_set(skb, metasize);
+ copy -= metasize;
+ } else {
+skb_put_data(skb, p, copy);
+ }
len -= copy;
offset += copy;drivers/net/virtio_net.c +2
-1@@ -644,6 +652,7 @@ static struct sk_buff *receive_small(struct net_device *dev, unsigned int delta = 0; struct page *xdp_page; int err; + unsigned int metasize = 0; len -= vi->hdr_len; stats->bytes += len;@@ -683,8 +692,8 @@ static struct sk_buff *receive_small(struct net_device *dev, xdp.data_hard_start = buf + VIRTNET_RX_PAD + vi->hdr_len; xdp.data = xdp.data_hard_start + xdp_headroom; - xdp_set_data_meta_invalid(&xdp); xdp.data_end = xdp.data + len; + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; orig_data = xdp.data; act = bpf_prog_run_xdp(xdp_prog, &xdp);@@ -695,9 +704,11 @@ static struct sk_buff *receive_small(struct net_device *dev, /* Recalculate length in case bpf program changed it */ delta = orig_data - xdp.data; len = xdp.data_end - xdp.data;
Quoted 8 lines
Quoted 5 lines
drivers/net/virtio_net.c +1
-0+ metasize = xdp.data - xdp.data_meta; break;case
XDP_TX:
stats->xdp_tx++;
+ xdp.data_meta = xdp.data;Why need this?
Because virtnet_xdp_xmit() doesn't support XDP meta data as below. And I suppose that we don't need to support XDP metadata for XDP_TX.
static int __virtnet_xdp_xmit_one(struct virtnet_info *vi,
struct send_queue *sq, struct xdp_frame *xdpf)
{
struct virtio_net_hdr_mrg_rxbuf *hdr;
int err;
/* virtqueue want to use data area in-front of packet */
if (unlikely(xdpf->metasize > 0))
return -EOPNOTSUPP;Quoted 16 lines
drivers/net/virtio_net.c static struct sk_buff *receive_small(struct net_device *dev, · line 752
Quoted 13 lines
xdpf =
convert_to_xdp_frame(&xdp);
if (unlikely(!xdpf))
gotoerr_xdp;drivers/net/virtio_net.c +1
-1@@ -735,11 +746,14 @@ static struct sk_buff *receive_small(struct net_device *dev, } skb_reserve(skb, headroom - delta); skb_put(skb, len); - if (!delta) { + if (!delta && !metasize) { buf += header_offset; memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len); } /* keep zeroed vnet hdr since packet was changed by bpf */Is there any method to preserve the vnet header here? We probably don't want to lose it for
XDP_PASSwhen packet is not modified.
I'll try to keep the vnet header with moving the meta data to the front of the vnet header.
Quoted 39 lines
Quoted 39 lines
drivers/net/virtio_net.c +2
-0+ if (metasize) + skb_metadata_set(skb, metasize); +err:
return skb;
@@ -761,7 +775,7 @@ static structsk_buff*receive_big(structnet_device*dev, {
struct page *page = buf;
structsk_buff*skb = page_to_skb(vi, rq, page, 0, len,
-PAGE_SIZE, true);
+PAGE_SIZE, true, 0);
stats->bytes+= len -vi->hdr_len;
if (unlikely(!skb))drivers/net/virtio_net.c +3
-1@@ -793,6 +807,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, unsigned int truesize; unsigned int headroom = mergeable_ctx_to_headroom(ctx); int err; + unsigned int metasize = 0; head_skb = NULL; stats->bytes += len - vi->hdr_len;@@ -839,8 +854,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, data = page_address(xdp_page) + offset; xdp.data_hard_start = data - VIRTIO_XDP_HEADROOM + vi->hdr_len; xdp.data = data + vi->hdr_len; - xdp_set_data_meta_invalid(&xdp); xdp.data_end = xdp.data + (len - vi->hdr_len); + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; act = bpf_prog_run_xdp(xdp_prog, &xdp);@@ -859,18 +874,20 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,* adjusted */ len = xdp.data_end - xdp.data + vi->hdr_len; + metasize = xdp.data - xdp.data_meta; /* We can only create skb based on xdp_page. */ if (unlikely(xdp_page != page)) { rcu_read_unlock(); put_page(page);
Quoted 8 lines
drivers/net/virtio_net.c static struct sk_buff *receive_mergeable(struct net_device *dev, · line 884
Quoted 5 lines
drivers/net/virtio_net.c +2
-2head_skb = page_to_skb(vi, rq, xdp_page, - offset, len, - PAGE_SIZE, false); + offset, len, + PAGE_SIZE, false, metasize);Indentation is wired.
Sorry. I'll fix it.
Quoted 21 lines
Thanks
Quoted 17 lines
drivers/net/virtio_net.c +0
-0return head_skb; } break;case
XDP_TX:
stats->xdp_tx++;
+ xdp.data_meta = xdp.data;
xdpf =convert_to_xdp_frame(&xdp);
if (unlikely(!xdpf))
gotoerr_xdp;drivers/net/virtio_net.c +2
-0@@ -921,7 +938,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, goto err_skb; } - head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog); + head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog, + metasize); curr_skb = head_skb; if (unlikely(!curr_skb))
PATCH v2 1/1 virtio_net: add XDP meta data support
This adds XDP meta data support to both receive_small() and receive_mergeable().
---
v2:
- keep copy untouched in page_to_skb().
- preserve the vnet header in receive_small().
- fix indentation.
--- drivers/net/virtio_net.c | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-)
drivers/net/virtio_net.c +27 -12
@@ -371,7 +371,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,struct receive_queue *rq, struct page *page, unsigned int offset, unsigned int len, unsigned int truesize, - bool hdr_valid) + bool hdr_valid, unsigned int metasize) { struct sk_buff *skb;struct virtio_net_hdr_mrg_rxbuf *hdr;
@@ -393,7 +393,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,else hdr_padded_len =sizeof (struct padded_vnet_hdr ); -if (hdr_valid) +if (hdr_valid && !metasize)memcpy (hdr, p, hdr_len); len -= hdr_len;
@@ -405,6 +405,11 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, copy =skb_tailroom (skb);skb_put_data (skb, p, copy); +if (metasize) { +__skb_pull (skb, metasize); +skb_metadata_set (skb, metasize); + } + len -= copy; offset += copy;
@@ -644,6 +649,7 @@ static struct sk_buff *receive_small(struct net_device *dev,unsigned int delta =0 ;struct page *xdp_page;int err; +unsigned int metasize =0 ; len -= vi->hdr_len; stats->bytes += len;
@@ -683,8 +689,8 @@ static struct sk_buff *receive_small(struct net_device *dev, xdp.data_hard_start = buf +VIRTNET_RX_PAD + vi->hdr_len; xdp.data = xdp.data_hard_start + xdp_headroom; -xdp_set_data_meta_invalid (&xdp); xdp.data_end = xdp.data + len; + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; orig_data = xdp.data; act =bpf_prog_run_xdp (xdp_prog, &xdp);
@@ -695,9 +701,11 @@ static struct sk_buff *receive_small(struct net_device *dev,/* Recalculate length in case bpf program changed it */ delta = orig_data - xdp.data; len = xdp.data_end - xdp.data; + metasize = xdp.data - xdp.data_meta;break ;case XDP_TX : stats->xdp_tx++; + xdp.data_meta = xdp.data; xdpf =convert_to_xdp_frame (&xdp);if (unlikely (!xdpf)) goto err_xdp;
@@ -740,6 +748,9 @@ static struct sk_buff *receive_small(struct net_device *dev,memcpy (skb_vnet_hdr (skb), buf, vi->hdr_len); }/* keep zeroed vnet hdr since packet was changed by bpf */ +if (metasize) +skb_metadata_set (skb, metasize); + err:return skb;
@@ -760,8 +771,8 @@ static struct sk_buff *receive_big(struct net_device *dev, struct virtnet_rq_stats *stats) {struct page *page = buf; -struct sk_buff *skb =page_to_skb (vi, rq, page,0 , len, -PAGE_SIZE , true); +struct sk_buff *skb = +page_to_skb (vi, rq, page,0 , len,PAGE_SIZE , true,0 ); stats->bytes += len - vi->hdr_len;if (unlikely (!skb))
@@ -793,6 +804,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,unsigned int truesize;unsigned int headroom =mergeable_ctx_to_headroom (ctx);int err; +unsigned int metasize =0 ; head_skb =NULL ; stats->bytes += len - vi->hdr_len;
@@ -839,8 +851,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, data =page_address (xdp_page) + offset; xdp.data_hard_start = data -VIRTIO_XDP_HEADROOM + vi->hdr_len; xdp.data = data + vi->hdr_len; -xdp_set_data_meta_invalid (&xdp); xdp.data_end = xdp.data + (len - vi->hdr_len); + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; act =bpf_prog_run_xdp (xdp_prog, &xdp);
@@ -852,8 +864,9 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,* adjustments. Note other cases do not build an * skb and avoid using offset */ - offset = xdp.data - -page_address (xdp_page) - vi->hdr_len; + metasize = xdp.data - xdp.data_meta; + offset = xdp.data -page_address (xdp_page) - + vi->hdr_len - metasize;/* recalculate len if xdp.data or xdp.data_end were * adjusted
@@ -863,14 +876,15 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,if (unlikely (xdp_page != page)) {rcu_read_unlock ();put_page (page); - head_skb =page_to_skb (vi, rq, xdp_page, - offset, len, -PAGE_SIZE , false); + head_skb =page_to_skb (vi, rq, xdp_page, offset, + len,PAGE_SIZE , false, + metasize);return head_skb; }break ;case XDP_TX : stats->xdp_tx++; + xdp.data_meta = xdp.data; xdpf =convert_to_xdp_frame (&xdp);if (unlikely (!xdpf)) goto err_xdp;
@@ -921,7 +935,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, goto err_skb; } - head_skb =page_to_skb (vi, rq, page, offset, len, truesize, !xdp_prog); + head_skb =page_to_skb (vi, rq, page, offset, len, truesize, !xdp_prog, + metasize); curr_skb = head_skb;if (unlikely (!curr_skb))
Signature · 1 line
2.20.1
Quoted 79 lines · Yuya Kusakabe
This adds XDP meta data support to both
receive_small()andreceive_mergeable().Fixes de8f3a83b0a0 ("bpf: add meta pointer for direct access")Signed-off-by Yuya Kusakabe <[email protected]>---
v2:
- keep copy untouched inpage_to_skb().
- preserve the vnet header inreceive_small().
- fix indentation.--- drivers/net/virtio_net.c | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-)drivers/net/virtio_net.c +11
-3@@ -371,7 +371,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,struct receive_queue *rq, struct page *page, unsigned int offset, unsigned int len, unsigned int truesize, - bool hdr_valid) + bool hdr_valid, unsigned int metasize) { struct sk_buff *skb;struct virtio_net_hdr_mrg_rxbuf *hdr;@@ -393,7 +393,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,else hdr_padded_len =sizeof (struct padded_vnet_hdr ); -if (hdr_valid) +if (hdr_valid && !metasize)memcpy (hdr, p, hdr_len); len -= hdr_len;@@ -405,6 +405,11 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, copy =skb_tailroom (skb);skb_put_data (skb, p, copy); +if (metasize) { +__skb_pull (skb, metasize); +skb_metadata_set (skb, metasize); + } + len -= copy; offset += copy;@@ -644,6 +649,7 @@ static struct sk_buff *receive_small(struct net_device *dev,unsigned int delta =0 ;struct page *xdp_page;int err; +unsigned int metasize =0 ; len -= vi->hdr_len; stats->bytes += len;@@ -683,8 +689,8 @@ static struct sk_buff *receive_small(struct net_device *dev, xdp.data_hard_start = buf +VIRTNET_RX_PAD + vi->hdr_len; xdp.data = xdp.data_hard_start + xdp_headroom; -xdp_set_data_meta_invalid (&xdp); xdp.data_end = xdp.data + len; + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; orig_data = xdp.data; act =bpf_prog_run_xdp (xdp_prog, &xdp);@@ -695,9 +701,11 @@ static struct sk_buff *receive_small(struct net_device *dev,/* Recalculate length in case bpf program changed it */ delta = orig_data - xdp.data; len = xdp.data_end - xdp.data; + metasize = xdp.data - xdp.data_meta;break ;case XDP_TX : stats->xdp_tx++; + xdp.data_meta = xdp.data; xdpf =convert_to_xdp_frame (&xdp);if (unlikely (!xdpf)) goto err_xdp;
drivers/net/virtio_net.c static struct sk_buff *receive_small(struct net_device *dev, · line 749
Quoted 2 lines
drivers/net/virtio_net.c +0
-0@@ -740,6 +748,9 @@ static struct sk_buff *receive_small(struct net_device *dev,memcpy (skb_vnet_hdr (skb), buf, vi->hdr_len); }/* keep zeroed vnet hdr since packet was changed by bpf */
I wonder whether or not it's as simple as this. Consider bpf may adjust meta, it looks to me that the vnet header will be overwrite here? If yes, we probably need to have a device specific value then bpf can move the device metadata like vnet header for us?
Thanks
Quoted 77 lines
drivers/net/virtio_net.c +16
-9+if (metasize) +skb_metadata_set (skb, metasize); + err:return skb;@@ -760,8 +771,8 @@ static struct sk_buff *receive_big(struct net_device *dev, struct virtnet_rq_stats *stats) {struct page *page = buf; -struct sk_buff *skb =page_to_skb (vi, rq, page,0 , len, -PAGE_SIZE , true); +struct sk_buff *skb = +page_to_skb (vi, rq, page,0 , len,PAGE_SIZE , true,0 ); stats->bytes += len - vi->hdr_len;if (unlikely (!skb))@@ -793,6 +804,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,unsigned int truesize;unsigned int headroom =mergeable_ctx_to_headroom (ctx);int err; +unsigned int metasize =0 ; head_skb =NULL ; stats->bytes += len - vi->hdr_len;@@ -839,8 +851,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, data =page_address (xdp_page) + offset; xdp.data_hard_start = data -VIRTIO_XDP_HEADROOM + vi->hdr_len; xdp.data = data + vi->hdr_len; -xdp_set_data_meta_invalid (&xdp); xdp.data_end = xdp.data + (len - vi->hdr_len); + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; act =bpf_prog_run_xdp (xdp_prog, &xdp);@@ -852,8 +864,9 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,* adjustments. Note other cases do not build an * skb and avoid using offset */ - offset = xdp.data - -page_address (xdp_page) - vi->hdr_len; + metasize = xdp.data - xdp.data_meta; + offset = xdp.data -page_address (xdp_page) - + vi->hdr_len - metasize;/* recalculate len if xdp.data or xdp.data_end were * adjusted @@ -863,14 +876,15 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,if (unlikely (xdp_page != page)) {rcu_read_unlock ();put_page (page); - head_skb =page_to_skb (vi, rq, xdp_page, - offset, len, -PAGE_SIZE , false); + head_skb =page_to_skb (vi, rq, xdp_page, offset, + len,PAGE_SIZE , false, + metasize);return head_skb; }break ;case XDP_TX : stats->xdp_tx++; + xdp.data_meta = xdp.data; xdpf =convert_to_xdp_frame (&xdp);if (unlikely (!xdpf)) goto err_xdp;@@ -921,7 +935,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, goto err_skb; } - head_skb =page_to_skb (vi, rq, page, offset, len, truesize, !xdp_prog); + head_skb =page_to_skb (vi, rq, page, offset, len, truesize, !xdp_prog, + metasize); curr_skb = head_skb;if (unlikely (!curr_skb))
Quoted 73 lines · Jason Wang
Quoted 71 lines · Yuya Kusakabe
This adds XDP meta data support to both
receive_small()andreceive_mergeable().Fixes de8f3a83b0a0 ("bpf: add meta pointer for direct access")Signed-off-by Yuya Kusakabe <[email protected]>---
v2:
- keep copy untouched inpage_to_skb().
- preserve the vnet header inreceive_small().
- fix indentation.
---
drivers/net/virtio_net.c| 39 +++++++++++++++++++++++++++------------1 file changed, 27 insertions(+), 12 deletions(-)drivers/net/virtio_net.c +1
-1@@ -371,7 +371,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, struct receive_queue *rq, struct page *page, unsigned int offset, unsigned int len, unsigned int truesize, - bool hdr_valid) + bool hdr_valid, unsigned int metasize) { struct sk_buff *skb; struct virtio_net_hdr_mrg_rxbuf *hdr;@@ -393,7 +393,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,else
hdr_padded_len=sizeof(struct padded_vnet_hdr); - if (hdr_valid)
+ if (hdr_valid&& !metasize)
memcpy(hdr, p, hdr_len);
len -=hdr_len;drivers/net/virtio_net.c +4
-0@@ -405,6 +405,11 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, copy = skb_tailroom(skb); skb_put_data(skb, p, copy); + if (metasize) { + __skb_pull(skb, metasize); + skb_metadata_set(skb, metasize); + } + len -= copy; offset += copy; @@ -644,6 +649,7 @@ static struct sk_buff *receive_small(struct net_device *dev,unsigned int delta = 0;
struct page*xdp_page;
int err;
+ unsigned int metasize = 0;
len -=vi->hdr_len;
stats->bytes+= len;drivers/net/virtio_net.c +2
-1@@ -683,8 +689,8 @@ static struct sk_buff *receive_small(struct net_device *dev, xdp.data_hard_start = buf + VIRTNET_RX_PAD + vi->hdr_len; xdp.data = xdp.data_hard_start + xdp_headroom; - xdp_set_data_meta_invalid(&xdp); xdp.data_end = xdp.data + len; + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; orig_data = xdp.data; act = bpf_prog_run_xdp(xdp_prog, &xdp);@@ -695,9 +701,11 @@ static struct sk_buff *receive_small(struct net_device *dev, /* Recalculate length in case bpf program changed it */ delta = orig_data - xdp.data; len = xdp.data_end - xdp.data; + metasize = xdp.data - xdp.data_meta; break;case
XDP_TX:
stats->xdp_tx++;
+ xdp.data_meta = xdp.data;
xdpf =convert_to_xdp_frame(&xdp);
if (unlikely(!xdpf))
Quoted 8 lines
drivers/net/virtio_net.c static struct sk_buff *receive_small(struct net_device *dev, · line 749
Quoted 4 lines
goto
err_xdp;drivers/net/virtio_net.c +0
-0@@ -740,6 +748,9 @@ static struct sk_buff *receive_small(struct net_device *dev, memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len); } /* keep zeroed vnet hdr since packet was changed by bpf */I wonder whether or not it's as simple as this. Consider bpf may adjust meta, it looks to me that the vnet header will be overwrite here? If yes, we probably need to have a device specific value then bpf can move the device metadata like vnet header for us?> Thanks
Yes, the vnet header is overwrite by xdp metadata. I'll fix it.
Thanks.
Quoted 71 lines
Quoted 71 lines
drivers/net/virtio_net.c +2
-0+ if (metasize) + skb_metadata_set(skb, metasize); +err:
return skb;
@@ -760,8 +771,8 @@ static structsk_buff*receive_big(structnet_device*dev, structvirtnet_rq_stats*stats)
{
struct page *page = buf;
- structsk_buff*skb = page_to_skb(vi, rq, page, 0, len,
-PAGE_SIZE, true);
+ structsk_buff*skb =
+page_to_skb(vi, rq, page, 0, len, PAGE_SIZE, true, 0);stats->bytes+= len -vi->hdr_len;
if (unlikely(!skb))drivers/net/virtio_net.c +8
-6@@ -793,6 +804,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, unsigned int truesize; unsigned int headroom = mergeable_ctx_to_headroom(ctx); int err; + unsigned int metasize = 0; head_skb = NULL; stats->bytes += len - vi->hdr_len;@@ -839,8 +851,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, data = page_address(xdp_page) + offset; xdp.data_hard_start = data - VIRTIO_XDP_HEADROOM + vi->hdr_len; xdp.data = data + vi->hdr_len; - xdp_set_data_meta_invalid(&xdp); xdp.data_end = xdp.data + (len - vi->hdr_len); + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; act = bpf_prog_run_xdp(xdp_prog, &xdp);@@ -852,8 +864,9 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,* adjustments. Note other cases do not build an * skb and avoid using offset */ - offset = xdp.data - - page_address(xdp_page) - vi->hdr_len; + metasize = xdp.data - xdp.data_meta; + offset = xdp.data - page_address(xdp_page) - + vi->hdr_len - metasize; /* recalculate len if xdp.data or xdp.data_end were* adjusted @@ -863,14 +876,15 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, if (unlikely(xdp_page != page)) { rcu_read_unlock(); put_page(page); - head_skb = page_to_skb(vi, rq, xdp_page, - offset, len, - PAGE_SIZE, false); + head_skb = page_to_skb(vi, rq, xdp_page, offset, + len, PAGE_SIZE, false, + metasize); return head_skb; } break;case
XDP_TX:
stats->xdp_tx++;
+ xdp.data_meta = xdp.data;
xdpf =convert_to_xdp_frame(&xdp);
if (unlikely(!xdpf))
gotoerr_xdp;drivers/net/virtio_net.c +2
-0@@ -921,7 +935,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, goto err_skb; } - head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog); + head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog, + metasize); curr_skb = head_skb; if (unlikely(!curr_skb))
PATCH v3 1/1 virtio_net: add XDP meta data support
This adds XDP meta data support to both receive_small() and receive_mergeable().
---
v3:
- fix preserve the vnet header in receive_small().
v2:
- keep copy untouched in page_to_skb().
- preserve the vnet header in receive_small().
- fix indentation.
--- drivers/net/virtio_net.c | 45 +++++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 14 deletions(-)
drivers/net/virtio_net.c +31 -14
@@ -371,7 +371,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,struct receive_queue *rq, struct page *page, unsigned int offset, unsigned int len, unsigned int truesize, - bool hdr_valid) + bool hdr_valid, unsigned int metasize) { struct sk_buff *skb;struct virtio_net_hdr_mrg_rxbuf *hdr;
@@ -393,7 +393,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,else hdr_padded_len =sizeof (struct padded_vnet_hdr ); -if (hdr_valid) +if (hdr_valid && !metasize)memcpy (hdr, p, hdr_len); len -= hdr_len;
@@ -405,6 +405,11 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, copy =skb_tailroom (skb);skb_put_data (skb, p, copy); +if (metasize) { +__skb_pull (skb, metasize); +skb_metadata_set (skb, metasize); + } + len -= copy; offset += copy;
@@ -644,6 +649,7 @@ static struct sk_buff *receive_small(struct net_device *dev,unsigned int delta =0 ;struct page *xdp_page;int err; +unsigned int metasize =0 ; len -= vi->hdr_len; stats->bytes += len;
@@ -683,10 +689,13 @@ static struct sk_buff *receive_small(struct net_device *dev, xdp.data_hard_start = buf +VIRTNET_RX_PAD + vi->hdr_len; xdp.data = xdp.data_hard_start + xdp_headroom; -xdp_set_data_meta_invalid (&xdp); xdp.data_end = xdp.data + len; + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; orig_data = xdp.data; +/* Copy the vnet header to the front of data_hard_start to avoid +* overwriting by XDP meta data */ +memcpy (xdp.data_hard_start - vi->hdr_len, xdp.data - vi->hdr_len, vi->hdr_len); act =bpf_prog_run_xdp (xdp_prog, &xdp); stats->xdp_packets++;
@@ -695,9 +704,11 @@ static struct sk_buff *receive_small(struct net_device *dev,/* Recalculate length in case bpf program changed it */ delta = orig_data - xdp.data; len = xdp.data_end - xdp.data; + metasize = xdp.data - xdp.data_meta;break ;case XDP_TX : stats->xdp_tx++; + xdp.data_meta = xdp.data; xdpf =convert_to_xdp_frame (&xdp);if (unlikely (!xdpf)) goto err_xdp;
@@ -736,10 +747,12 @@ static struct sk_buff *receive_small(struct net_device *dev,skb_reserve (skb, headroom - delta);skb_put (skb, len);if (!delta) { - buf += header_offset; -memcpy (skb_vnet_hdr (skb), buf, vi->hdr_len); +memcpy (skb_vnet_hdr (skb), buf +VIRTNET_RX_PAD , vi->hdr_len); }/* keep zeroed vnet hdr since packet was changed by bpf */ +if (metasize) +skb_metadata_set (skb, metasize); + err:return skb;
@@ -760,8 +773,8 @@ static struct sk_buff *receive_big(struct net_device *dev, struct virtnet_rq_stats *stats) {struct page *page = buf; -struct sk_buff *skb =page_to_skb (vi, rq, page,0 , len, -PAGE_SIZE , true); +struct sk_buff *skb = +page_to_skb (vi, rq, page,0 , len,PAGE_SIZE , true,0 ); stats->bytes += len - vi->hdr_len;if (unlikely (!skb))
@@ -793,6 +806,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,unsigned int truesize;unsigned int headroom =mergeable_ctx_to_headroom (ctx);int err; +unsigned int metasize =0 ; head_skb =NULL ; stats->bytes += len - vi->hdr_len;
@@ -839,8 +853,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, data =page_address (xdp_page) + offset; xdp.data_hard_start = data -VIRTIO_XDP_HEADROOM + vi->hdr_len; xdp.data = data + vi->hdr_len; -xdp_set_data_meta_invalid (&xdp); xdp.data_end = xdp.data + (len - vi->hdr_len); + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; act =bpf_prog_run_xdp (xdp_prog, &xdp);
@@ -852,8 +866,9 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,* adjustments. Note other cases do not build an * skb and avoid using offset */ - offset = xdp.data - -page_address (xdp_page) - vi->hdr_len; + metasize = xdp.data - xdp.data_meta; + offset = xdp.data -page_address (xdp_page) - + vi->hdr_len - metasize;/* recalculate len if xdp.data or xdp.data_end were * adjusted
@@ -863,14 +878,15 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,if (unlikely (xdp_page != page)) {rcu_read_unlock ();put_page (page); - head_skb =page_to_skb (vi, rq, xdp_page, - offset, len, -PAGE_SIZE , false); + head_skb =page_to_skb (vi, rq, xdp_page, offset, + len,PAGE_SIZE , false, + metasize);return head_skb; }break ;case XDP_TX : stats->xdp_tx++; + xdp.data_meta = xdp.data; xdpf =convert_to_xdp_frame (&xdp);if (unlikely (!xdpf)) goto err_xdp;
@@ -921,7 +937,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, goto err_skb; } - head_skb =page_to_skb (vi, rq, page, offset, len, truesize, !xdp_prog); + head_skb =page_to_skb (vi, rq, page, offset, len, truesize, !xdp_prog, + metasize); curr_skb = head_skb;if (unlikely (!curr_skb))
Signature · 1 line
2.20.1
Quoted 60 lines · Yuya Kusakabe
This adds XDP meta data support to both
receive_small()andreceive_mergeable().Fixes de8f3a83b0a0 ("bpf: add meta pointer for direct access")Signed-off-by Yuya Kusakabe <[email protected]>---
v3:
- fix preserve the vnet header inreceive_small().
v2:
- keep copy untouched inpage_to_skb().
- preserve the vnet header inreceive_small().
- fix indentation.--- drivers/net/virtio_net.c | 45 +++++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 14 deletions(-)drivers/net/virtio_net.c +8
-2@@ -371,7 +371,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,struct receive_queue *rq, struct page *page, unsigned int offset, unsigned int len, unsigned int truesize, - bool hdr_valid) + bool hdr_valid, unsigned int metasize) { struct sk_buff *skb;struct virtio_net_hdr_mrg_rxbuf *hdr;@@ -393,7 +393,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,else hdr_padded_len =sizeof (struct padded_vnet_hdr ); -if (hdr_valid) +if (hdr_valid && !metasize)memcpy (hdr, p, hdr_len); len -= hdr_len;@@ -405,6 +405,11 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, copy =skb_tailroom (skb);skb_put_data (skb, p, copy); +if (metasize) { +__skb_pull (skb, metasize); +skb_metadata_set (skb, metasize); + } + len -= copy; offset += copy;@@ -644,6 +649,7 @@ static struct sk_buff *receive_small(struct net_device *dev,unsigned int delta =0 ;struct page *xdp_page;int err; +unsigned int metasize =0 ; len -= vi->hdr_len; stats->bytes += len;@@ -683,10 +689,13 @@ static struct sk_buff *receive_small(struct net_device *dev,
drivers/net/virtio_net.c static struct sk_buff *receive_small(struct net_device *dev, · line 698
Quoted 10 lines
drivers/net/virtio_net.c +4
-1xdp.data_hard_start = buf +VIRTNET_RX_PAD + vi->hdr_len; xdp.data = xdp.data_hard_start + xdp_headroom; -xdp_set_data_meta_invalid (&xdp); xdp.data_end = xdp.data + len; + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; orig_data = xdp.data; +/* Copy the vnet header to the front of data_hard_start to avoid +* overwriting by XDP meta data */ +memcpy (xdp.data_hard_start - vi->hdr_len, xdp.data - vi->hdr_len, vi->hdr_len);
What happens if we have a large metadata that occupies all headroom here?
Thanks
Quoted 100 lines
drivers/net/virtio_net.c +19
-11act =bpf_prog_run_xdp (xdp_prog, &xdp); stats->xdp_packets++;@@ -695,9 +704,11 @@ static struct sk_buff *receive_small(struct net_device *dev,/* Recalculate length in case bpf program changed it */ delta = orig_data - xdp.data; len = xdp.data_end - xdp.data; + metasize = xdp.data - xdp.data_meta;break ;case XDP_TX : stats->xdp_tx++; + xdp.data_meta = xdp.data; xdpf =convert_to_xdp_frame (&xdp);if (unlikely (!xdpf)) goto err_xdp;@@ -736,10 +747,12 @@ static struct sk_buff *receive_small(struct net_device *dev,skb_reserve (skb, headroom - delta);skb_put (skb, len);if (!delta) { - buf += header_offset; -memcpy (skb_vnet_hdr (skb), buf, vi->hdr_len); +memcpy (skb_vnet_hdr (skb), buf +VIRTNET_RX_PAD , vi->hdr_len); }/* keep zeroed vnet hdr since packet was changed by bpf */ +if (metasize) +skb_metadata_set (skb, metasize); + err:return skb;@@ -760,8 +773,8 @@ static struct sk_buff *receive_big(struct net_device *dev, struct virtnet_rq_stats *stats) {struct page *page = buf; -struct sk_buff *skb =page_to_skb (vi, rq, page,0 , len, -PAGE_SIZE , true); +struct sk_buff *skb = +page_to_skb (vi, rq, page,0 , len,PAGE_SIZE , true,0 ); stats->bytes += len - vi->hdr_len;if (unlikely (!skb))@@ -793,6 +806,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,unsigned int truesize;unsigned int headroom =mergeable_ctx_to_headroom (ctx);int err; +unsigned int metasize =0 ; head_skb =NULL ; stats->bytes += len - vi->hdr_len;@@ -839,8 +853,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, data =page_address (xdp_page) + offset; xdp.data_hard_start = data -VIRTIO_XDP_HEADROOM + vi->hdr_len; xdp.data = data + vi->hdr_len; -xdp_set_data_meta_invalid (&xdp); xdp.data_end = xdp.data + (len - vi->hdr_len); + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; act =bpf_prog_run_xdp (xdp_prog, &xdp);@@ -852,8 +866,9 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,* adjustments. Note other cases do not build an * skb and avoid using offset */ - offset = xdp.data - -page_address (xdp_page) - vi->hdr_len; + metasize = xdp.data - xdp.data_meta; + offset = xdp.data -page_address (xdp_page) - + vi->hdr_len - metasize;/* recalculate len if xdp.data or xdp.data_end were * adjusted @@ -863,14 +878,15 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,if (unlikely (xdp_page != page)) {rcu_read_unlock ();put_page (page); - head_skb =page_to_skb (vi, rq, xdp_page, - offset, len, -PAGE_SIZE , false); + head_skb =page_to_skb (vi, rq, xdp_page, offset, + len,PAGE_SIZE , false, + metasize);return head_skb; }break ;case XDP_TX : stats->xdp_tx++; + xdp.data_meta = xdp.data; xdpf =convert_to_xdp_frame (&xdp);if (unlikely (!xdpf)) goto err_xdp;@@ -921,7 +937,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, goto err_skb; } - head_skb =page_to_skb (vi, rq, page, offset, len, truesize, !xdp_prog); + head_skb =page_to_skb (vi, rq, page, offset, len, truesize, !xdp_prog, + metasize); curr_skb = head_skb;if (unlikely (!curr_skb))
Quoted 70 lines · Jason Wang
Quoted 54 lines · Yuya Kusakabe
This adds XDP meta data support to both
receive_small()andreceive_mergeable().Fixes de8f3a83b0a0 ("bpf: add meta pointer for direct access")Signed-off-by Yuya Kusakabe <[email protected]>---
v3:
- fix preserve the vnet header inreceive_small().
v2:
- keep copy untouched inpage_to_skb().
- preserve the vnet header inreceive_small().
- fix indentation.
---
drivers/net/virtio_net.c| 45 +++++++++++++++++++++++++++-------------1 file changed, 31 insertions(+), 14 deletions(-)drivers/net/virtio_net.c +1
-1@@ -371,7 +371,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, struct receive_queue *rq, struct page *page, unsigned int offset, unsigned int len, unsigned int truesize, - bool hdr_valid) + bool hdr_valid, unsigned int metasize) { struct sk_buff *skb; struct virtio_net_hdr_mrg_rxbuf *hdr;@@ -393,7 +393,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,else
hdr_padded_len=sizeof(struct padded_vnet_hdr); - if (hdr_valid)
+ if (hdr_valid&& !metasize)
memcpy(hdr, p, hdr_len);
len -=hdr_len;drivers/net/virtio_net.c +4
-0@@ -405,6 +405,11 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, copy = skb_tailroom(skb); skb_put_data(skb, p, copy); + if (metasize) { + __skb_pull(skb, metasize); + skb_metadata_set(skb, metasize); + } + len -= copy; offset += copy; @@ -644,6 +649,7 @@ static struct sk_buff *receive_small(struct net_device *dev,unsigned int delta = 0;
struct page*xdp_page;
int err;
+ unsigned int metasize = 0;
len -=vi->hdr_len;
stats->bytes+= len;drivers/net/virtio_net.c static struct sk_buff *receive_small(struct net_device *dev, · line 697
Quoted 10 lines
drivers/net/virtio_net.c +4
-1@@ -683,10 +689,13 @@ static struct sk_buff *receive_small(struct net_device *dev, xdp.data_hard_start = buf + VIRTNET_RX_PAD + vi->hdr_len; xdp.data = xdp.data_hard_start + xdp_headroom; - xdp_set_data_meta_invalid(&xdp); xdp.data_end = xdp.data + len; + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; orig_data = xdp.data; + /* Copy the vnet header to the front of data_hard_start to avoid + * overwriting by XDP meta data */ + memcpy(xdp.data_hard_start - vi->hdr_len, xdp.data - vi->hdr_len, vi->hdr_len);What happens if we have a large metadata that occupies all headroom here?
Quoted 1 line
Thanks
Do you mean a large "XDP" metadata? If a large metadata is a large "XDP" metadata, I think we can not use a metadata that occupies all headroom. The size of metadata limited by bpf_xdp_adjust_meta() as below. bpf_xdp_adjust_meta() in net/core/filter.c:
if (unlikely((metalen & (sizeof(__u32) - 1)) || (metalen > 32))) return -EACCES;
Thanks.
Quoted 93 lines
Quoted 93 lines
drivers/net/virtio_net.c +0
-0act = bpf_prog_run_xdp(xdp_prog, &xdp); stats->xdp_packets++; @@ -695,9 +704,11 @@ static struct sk_buff *receive_small(struct net_device *dev, /* Recalculate length in case bpf program changed it */delta =
orig_data- xdp.data;
len = xdp.data_end - xdp.data;
+ metasize = xdp.data - xdp.data_meta;
break;
caseXDP_TX:
stats->xdp_tx++;
+ xdp.data_meta = xdp.data;
xdpf =convert_to_xdp_frame(&xdp);
if (unlikely(!xdpf))
gotoerr_xdp;drivers/net/virtio_net.c +3
-2@@ -736,10 +747,12 @@ static struct sk_buff *receive_small(struct net_device *dev, skb_reserve(skb, headroom - delta); skb_put(skb, len); if (!delta) { - buf += header_offset; - memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len); + memcpy(skb_vnet_hdr(skb), buf + VIRTNET_RX_PAD, vi->hdr_len); } /* keep zeroed vnet hdr since packet was changed by bpf */ + if (metasize) + skb_metadata_set(skb, metasize); +err:
return skb;
@@ -760,8 +773,8 @@ static structsk_buff*receive_big(structnet_device*dev, structvirtnet_rq_stats*stats)
{
struct page *page = buf;
- structsk_buff*skb = page_to_skb(vi, rq, page, 0, len,
-PAGE_SIZE, true);
+ structsk_buff*skb =
+page_to_skb(vi, rq, page, 0, len, PAGE_SIZE, true, 0);stats->bytes+= len -vi->hdr_len;
if (unlikely(!skb))drivers/net/virtio_net.c +8
-6@@ -793,6 +806,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, unsigned int truesize; unsigned int headroom = mergeable_ctx_to_headroom(ctx); int err; + unsigned int metasize = 0; head_skb = NULL; stats->bytes += len - vi->hdr_len;@@ -839,8 +853,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, data = page_address(xdp_page) + offset; xdp.data_hard_start = data - VIRTIO_XDP_HEADROOM + vi->hdr_len; xdp.data = data + vi->hdr_len; - xdp_set_data_meta_invalid(&xdp); xdp.data_end = xdp.data + (len - vi->hdr_len); + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; act = bpf_prog_run_xdp(xdp_prog, &xdp);@@ -852,8 +866,9 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,* adjustments. Note other cases do not build an * skb and avoid using offset */ - offset = xdp.data - - page_address(xdp_page) - vi->hdr_len; + metasize = xdp.data - xdp.data_meta; + offset = xdp.data - page_address(xdp_page) - + vi->hdr_len - metasize; /* recalculate len if xdp.data or xdp.data_end were* adjusted @@ -863,14 +878,15 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, if (unlikely(xdp_page != page)) { rcu_read_unlock(); put_page(page); - head_skb = page_to_skb(vi, rq, xdp_page, - offset, len, - PAGE_SIZE, false); + head_skb = page_to_skb(vi, rq, xdp_page, offset, + len, PAGE_SIZE, false, + metasize); return head_skb; } break;case
XDP_TX:
stats->xdp_tx++;
+ xdp.data_meta = xdp.data;
xdpf =convert_to_xdp_frame(&xdp);
if (unlikely(!xdpf))
gotoerr_xdp;drivers/net/virtio_net.c +2
-0@@ -921,7 +937,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, goto err_skb; } - head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog); + head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog, + metasize); curr_skb = head_skb; if (unlikely(!curr_skb))
Daniel Borkmann
maintainerQuoted 58 lines · Yuya Kusakabe
Quoted 57 lines · Jason Wang
Quoted 56 lines · Yuya Kusakabe
This adds XDP meta data support to both
receive_small()andreceive_mergeable().Fixes de8f3a83b0a0 ("bpf: add meta pointer for direct access")Signed-off-by Yuya Kusakabe <[email protected]>---
v3:
- fix preserve the vnet header inreceive_small().
v2:
- keep copy untouched inpage_to_skb().
- preserve the vnet header inreceive_small().
- fix indentation.
---
drivers/net/virtio_net.c| 45 +++++++++++++++++++++++++++-------------1 file changed, 31 insertions(+), 14 deletions(-)drivers/net/virtio_net.c +1
-1@@ -371,7 +371,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, struct receive_queue *rq, struct page *page, unsigned int offset, unsigned int len, unsigned int truesize, - bool hdr_valid) + bool hdr_valid, unsigned int metasize) { struct sk_buff *skb; struct virtio_net_hdr_mrg_rxbuf *hdr;@@ -393,7 +393,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,else
hdr_padded_len=sizeof(struct padded_vnet_hdr); - if (hdr_valid)
+ if (hdr_valid&& !metasize)
memcpy(hdr, p, hdr_len);
len -=hdr_len;drivers/net/virtio_net.c +4
-0@@ -405,6 +405,11 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, copy = skb_tailroom(skb); skb_put_data(skb, p, copy); + if (metasize) { + __skb_pull(skb, metasize); + skb_metadata_set(skb, metasize); + } + len -= copy; offset += copy; @@ -644,6 +649,7 @@ static struct sk_buff *receive_small(struct net_device *dev,unsigned int delta = 0;
struct page*xdp_page;
int err;
+ unsigned int metasize = 0;
len -=vi->hdr_len;
stats->bytes+= len;drivers/net/virtio_net.c +0
-0@@ -683,10 +689,13 @@ static struct sk_buff *receive_small(struct net_device *dev, xdp.data_hard_start = buf + VIRTNET_RX_PAD + vi->hdr_len; xdp.data = xdp.data_hard_start + xdp_headroom;
Quoted 8 lines
Quoted 8 lines
drivers/net/virtio_net.c static struct sk_buff *receive_small(struct net_device *dev, · line 698
Quoted 8 lines
drivers/net/virtio_net.c +4
-1- xdp_set_data_meta_invalid(&xdp); xdp.data_end = xdp.data + len; + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; orig_data = xdp.data; + /* Copy the vnet header to the front of data_hard_start to avoid + * overwriting by XDP meta data */ + memcpy(xdp.data_hard_start - vi->hdr_len, xdp.data - vi->hdr_len, vi->hdr_len);
I'm not fully sure if I'm following this one correctly, probably just missing something. Isn't the vnet header based on how we set up xdp.data_hard_start earlier already in front of it? Wouldn't we copy invalid data from xdp.data - vi->hdr_len into the vnet header at that point (given there can be up to 256 bytes of headroom between the two)? If it's relative to xdp.data and headroom is >0, then BPF prog could otherwise mangle this; something doesn't add up to me here. Could you clarify? Thx
Quoted 105 lines
Quoted 3 lines
What happens if we have a large metadata that occupies all headroom here?
Thanks
Do you mean a large "XDP" metadata? If a large metadata is a large "XDP" metadata, I think we can not use a metadata that occupies all headroom. The size of metadata limited by
bpf_xdp_adjust_meta()as below.bpf_xdp_adjust_meta()innet/core/filter.c:if (unlikely((metalen & (sizeof(__u32) - 1)) || (metalen > 32))) return -EACCES;Thanks.
Quoted 93 lines
Quoted 93 lines
drivers/net/virtio_net.c +0
-0act = bpf_prog_run_xdp(xdp_prog, &xdp); stats->xdp_packets++; @@ -695,9 +704,11 @@ static struct sk_buff *receive_small(struct net_device *dev, /* Recalculate length in case bpf program changed it */delta =
orig_data- xdp.data;
len = xdp.data_end - xdp.data;
+ metasize = xdp.data - xdp.data_meta;
break;
caseXDP_TX:
stats->xdp_tx++;
+ xdp.data_meta = xdp.data;
xdpf =convert_to_xdp_frame(&xdp);
if (unlikely(!xdpf))
gotoerr_xdp;drivers/net/virtio_net.c +3
-2@@ -736,10 +747,12 @@ static struct sk_buff *receive_small(struct net_device *dev, skb_reserve(skb, headroom - delta); skb_put(skb, len); if (!delta) { - buf += header_offset; - memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len); + memcpy(skb_vnet_hdr(skb), buf + VIRTNET_RX_PAD, vi->hdr_len); } /* keep zeroed vnet hdr since packet was changed by bpf */ + if (metasize) + skb_metadata_set(skb, metasize); +err:
return skb;
@@ -760,8 +773,8 @@ static structsk_buff*receive_big(structnet_device*dev, structvirtnet_rq_stats*stats)
{
struct page *page = buf;
- structsk_buff*skb = page_to_skb(vi, rq, page, 0, len,
-PAGE_SIZE, true);
+ structsk_buff*skb =
+page_to_skb(vi, rq, page, 0, len, PAGE_SIZE, true, 0);stats->bytes+= len -vi->hdr_len;
if (unlikely(!skb))drivers/net/virtio_net.c +8
-6@@ -793,6 +806,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, unsigned int truesize; unsigned int headroom = mergeable_ctx_to_headroom(ctx); int err; + unsigned int metasize = 0; head_skb = NULL; stats->bytes += len - vi->hdr_len;@@ -839,8 +853,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, data = page_address(xdp_page) + offset; xdp.data_hard_start = data - VIRTIO_XDP_HEADROOM + vi->hdr_len; xdp.data = data + vi->hdr_len; - xdp_set_data_meta_invalid(&xdp); xdp.data_end = xdp.data + (len - vi->hdr_len); + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; act = bpf_prog_run_xdp(xdp_prog, &xdp);@@ -852,8 +866,9 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,* adjustments. Note other cases do not build an * skb and avoid using offset */ - offset = xdp.data - - page_address(xdp_page) - vi->hdr_len; + metasize = xdp.data - xdp.data_meta; + offset = xdp.data - page_address(xdp_page) - + vi->hdr_len - metasize; /* recalculate len if xdp.data or xdp.data_end were* adjusted @@ -863,14 +878,15 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, if (unlikely(xdp_page != page)) { rcu_read_unlock(); put_page(page); - head_skb = page_to_skb(vi, rq, xdp_page, - offset, len, - PAGE_SIZE, false); + head_skb = page_to_skb(vi, rq, xdp_page, offset, + len, PAGE_SIZE, false, + metasize); return head_skb; } break;case
XDP_TX:
stats->xdp_tx++;
+ xdp.data_meta = xdp.data;
xdpf =convert_to_xdp_frame(&xdp);
if (unlikely(!xdpf))
gotoerr_xdp;drivers/net/virtio_net.c +2
-0@@ -921,7 +937,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, goto err_skb; } - head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog); + head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog, + metasize); curr_skb = head_skb; if (unlikely(!curr_skb))
Quoted 66 lines · Daniel Borkmann
Quoted 65 lines · Yuya Kusakabe
Quoted 64 lines · Jason Wang
Quoted 63 lines · Yuya Kusakabe
This adds XDP meta data support to both
receive_small()andreceive_mergeable().Fixes de8f3a83b0a0 ("bpf: add meta pointer for direct access")Signed-off-by Yuya Kusakabe <[email protected]>---
v3:
- fix preserve the vnet header inreceive_small().
v2:
- keep copy untouched inpage_to_skb().
- preserve the vnet header inreceive_small().
- fix indentation.--- drivers/net/virtio_net.c | 45 +++++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 14 deletions(-)drivers/net/virtio_net.c +10
-2@@ -371,7 +371,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, struct receive_queue *rq, struct page *page, unsigned int offset, unsigned int len, unsigned int truesize, - bool hdr_valid) + bool hdr_valid, unsigned int metasize) { struct sk_buff *skb; struct virtio_net_hdr_mrg_rxbuf *hdr;@@ -393,7 +393,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, else hdr_padded_len = sizeof(struct padded_vnet_hdr); - if (hdr_valid) + if (hdr_valid && !metasize) memcpy(hdr, p, hdr_len); len -= hdr_len;@@ -405,6 +405,11 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, copy = skb_tailroom(skb); skb_put_data(skb, p, copy); + if (metasize) { + __skb_pull(skb, metasize); + skb_metadata_set(skb, metasize); + } + len -= copy; offset += copy; @@ -644,6 +649,7 @@ static struct sk_buff *receive_small(struct net_device *dev, unsigned int delta = 0; struct page *xdp_page; int err; + unsigned int metasize = 0; len -= vi->hdr_len; stats->bytes += len;@@ -683,10 +689,13 @@ static struct sk_buff *receive_small(struct net_device *dev, xdp.data_hard_start = buf + VIRTNET_RX_PAD + vi->hdr_len; xdp.data = xdp.data_hard_start + xdp_headroom; - xdp_set_data_meta_invalid(&xdp); xdp.data_end = xdp.data + len; + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; orig_data = xdp.data; + /* Copy the vnet header to the front of data_hard_start to avoid +* overwriting by XDP meta data */
Quoted 8 lines
Quoted 1 line
Quoted 1 line
drivers/net/virtio_net.c static struct sk_buff *receive_small(struct net_device *dev, · line 698
Quoted 1 line
drivers/net/virtio_net.c +1
-0+ memcpy(xdp.data_hard_start - vi->hdr_len, xdp.data - vi->hdr_len, vi->hdr_len);I'm not fully sure if I'm following this one correctly, probably just missing something. Isn't the vnet header based on how we set up xdp.data_hard_start earlier already in front of it? Wouldn't we copy invalid data from xdp.data -
vi->hdr_leninto the vnet header at that point (given there can be up to 256 bytes of headroom between the two)? If it's relative to xdp.data and headroom is >0, then BPF prog could otherwise mangle this; something doesn't add up to me here. Could you clarify? Thx
Vnet headr sits just in front of xdp.data not xdp.data_hard_start. So it could be overwrote by metadata, that's why we need a copy here.
Thanks
Quoted 104 lines
Quoted 104 lines
Quoted 4 lines
What happens if we have a large metadata that occupies all headroom here?
Thanks
Do you mean a large "XDP" metadata? If a large metadata is a large "XDP" metadata, I think we can not use a metadata that occupies all headroom. The size of metadata limited by
bpf_xdp_adjust_meta()as below.bpf_xdp_adjust_meta()innet/core/filter.c:if (unlikely((metalen & (sizeof(__u32) - 1)) || (metalen > 32))) return -EACCES;Thanks.
Quoted 92 lines
Quoted 92 lines
drivers/net/virtio_net.c +0
-0act = bpf_prog_run_xdp(xdp_prog, &xdp); stats->xdp_packets++; @@ -695,9 +704,11 @@ static struct sk_buff *receive_small(struct net_device *dev, /* Recalculate length in case bpf program changed it */delta =
orig_data- xdp.data;
len = xdp.data_end - xdp.data;
+ metasize = xdp.data - xdp.data_meta;
break;
caseXDP_TX:
stats->xdp_tx++;
+ xdp.data_meta = xdp.data;
xdpf =convert_to_xdp_frame(&xdp);
if (unlikely(!xdpf))
gotoerr_xdp;drivers/net/virtio_net.c +16
-10@@ -736,10 +747,12 @@ static struct sk_buff *receive_small(struct net_device *dev, skb_reserve(skb, headroom - delta); skb_put(skb, len); if (!delta) { - buf += header_offset; - memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len); + memcpy(skb_vnet_hdr(skb), buf + VIRTNET_RX_PAD, vi->hdr_len); } /* keep zeroed vnet hdr since packet was changed by bpf */ + if (metasize) + skb_metadata_set(skb, metasize); + err: return skb; @@ -760,8 +773,8 @@ static struct sk_buff *receive_big(struct net_device *dev, struct virtnet_rq_stats *stats) { struct page *page = buf; - struct sk_buff *skb = page_to_skb(vi, rq, page, 0, len, - PAGE_SIZE, true); + struct sk_buff *skb = + page_to_skb(vi, rq, page, 0, len, PAGE_SIZE, true, 0); stats->bytes += len - vi->hdr_len; if (unlikely(!skb))@@ -793,6 +806,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, unsigned int truesize; unsigned int headroom = mergeable_ctx_to_headroom(ctx); int err; + unsigned int metasize = 0; head_skb = NULL; stats->bytes += len - vi->hdr_len;@@ -839,8 +853,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, data = page_address(xdp_page) + offset; xdp.data_hard_start = data - VIRTIO_XDP_HEADROOM + vi->hdr_len; xdp.data = data + vi->hdr_len; - xdp_set_data_meta_invalid(&xdp); xdp.data_end = xdp.data + (len - vi->hdr_len); + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; act = bpf_prog_run_xdp(xdp_prog, &xdp);@@ -852,8 +866,9 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,* adjustments. Note other cases do not build an * skb and avoid using offset */ - offset = xdp.data - - page_address(xdp_page) - vi->hdr_len; + metasize = xdp.data - xdp.data_meta; + offset = xdp.data - page_address(xdp_page) - + vi->hdr_len - metasize; /* recalculate len if xdp.data or xdp.data_end were* adjusted @@ -863,14 +878,15 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, if (unlikely(xdp_page != page)) { rcu_read_unlock(); put_page(page); - head_skb = page_to_skb(vi, rq, xdp_page, - offset, len, - PAGE_SIZE, false); + head_skb = page_to_skb(vi, rq, xdp_page, offset, + len, PAGE_SIZE, false, + metasize); return head_skb; } break; case XDP_TX: stats->xdp_tx++; + xdp.data_meta = xdp.data; xdpf = convert_to_xdp_frame(&xdp); if (unlikely(!xdpf)) goto err_xdp;@@ -921,7 +937,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, goto err_skb; } - head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog); + head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog, + metasize); curr_skb = head_skb; if (unlikely(!curr_skb))
Daniel Borkmann
maintainerQuoted 76 lines · Jason Wang
Quoted 66 lines · Daniel Borkmann
Quoted 65 lines · Yuya Kusakabe
Quoted 64 lines · Jason Wang
Quoted 63 lines · Yuya Kusakabe
This adds XDP meta data support to both
receive_small()andreceive_mergeable().Fixes de8f3a83b0a0 ("bpf: add meta pointer for direct access")Signed-off-by Yuya Kusakabe <[email protected]>---
v3:
- fix preserve the vnet header inreceive_small().
v2:
- keep copy untouched inpage_to_skb().
- preserve the vnet header inreceive_small().
- fix indentation.
---
drivers/net/virtio_net.c| 45 +++++++++++++++++++++++++++-------------1 file changed, 31 insertions(+), 14 deletions(-)drivers/net/virtio_net.c +1
-1@@ -371,7 +371,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, struct receive_queue *rq, struct page *page, unsigned int offset, unsigned int len, unsigned int truesize, - bool hdr_valid) + bool hdr_valid, unsigned int metasize) { struct sk_buff *skb; struct virtio_net_hdr_mrg_rxbuf *hdr;@@ -393,7 +393,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,else
hdr_padded_len=sizeof(struct padded_vnet_hdr); - if (hdr_valid)
+ if (hdr_valid&& !metasize)
memcpy(hdr, p, hdr_len);
len -=hdr_len;drivers/net/virtio_net.c +4
-0@@ -405,6 +405,11 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, copy = skb_tailroom(skb); skb_put_data(skb, p, copy); + if (metasize) { + __skb_pull(skb, metasize); + skb_metadata_set(skb, metasize); + } + len -= copy; offset += copy; @@ -644,6 +649,7 @@ static struct sk_buff *receive_small(struct net_device *dev,unsigned int delta = 0;
struct page*xdp_page;
int err;
+ unsigned int metasize = 0;
len -=vi->hdr_len;
stats->bytes+= len;drivers/net/virtio_net.c +3
-1@@ -683,10 +689,13 @@ static struct sk_buff *receive_small(struct net_device *dev, xdp.data_hard_start = buf + VIRTNET_RX_PAD + vi->hdr_len; xdp.data = xdp.data_hard_start + xdp_headroom; - xdp_set_data_meta_invalid(&xdp); xdp.data_end = xdp.data + len; + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; orig_data = xdp.data; + /* Copy the vnet header to the front of data_hard_start to avoid +* overwriting by XDP meta data */ Quoted 8 lines
Quoted 1 line
Quoted 1 line
drivers/net/virtio_net.c static struct sk_buff *receive_small(struct net_device *dev, · line 698
Quoted 1 line
drivers/net/virtio_net.c +1
-0+ memcpy(xdp.data_hard_start - vi->hdr_len, xdp.data - vi->hdr_len, vi->hdr_len);I'm not fully sure if I'm following this one correctly, probably just missing something. Isn't the vnet header based on how we set up xdp.data_hard_start earlier already in front of it? Wouldn't we copy invalid data from xdp.data -
vi->hdr_leninto the vnet header at that point (given there can be up to 256 bytes of headroom between the two)? If it's relative to xdp.data and headroom is >0, then BPF prog could otherwise mangle this; something doesn't add up to me here. Could you clarify? Thx
Quoted 1 line
Vnet headr sits just in front of xdp.data not xdp.data_hard_start. So it could be overwrote by metadata, that's why we need a copy here.
For the current code, you can adjust the xdp.data with a positive/negative offset already via bpf_xdp_adjust_head() helper. If vnet headr sits just in front of xdp.data, couldn't this be overridden today as well then? Anyway, just wondering how this is handled differently?
Thanks,
Daniel
Quoted 76 lines · Daniel Borkmann
Quoted 75 lines · Jason Wang
Quoted 74 lines · Daniel Borkmann
Quoted 58 lines · Yuya Kusakabe
Quoted 57 lines · Jason Wang
Quoted 56 lines · Yuya Kusakabe
This adds XDP meta data support to both
receive_small()andreceive_mergeable().Fixes de8f3a83b0a0 ("bpf: add meta pointer for direct access")Signed-off-by Yuya Kusakabe <[email protected]>---
v3:
- fix preserve the vnet header inreceive_small().
v2:
- keep copy untouched inpage_to_skb().
- preserve the vnet header inreceive_small().
- fix indentation.--- drivers/net/virtio_net.c | 45 +++++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 14 deletions(-)drivers/net/virtio_net.c +7
-1@@ -371,7 +371,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, struct receive_queue *rq, struct page *page, unsigned int offset, unsigned int len, unsigned int truesize, - bool hdr_valid) + bool hdr_valid, unsigned int metasize) { struct sk_buff *skb; struct virtio_net_hdr_mrg_rxbuf *hdr;@@ -393,7 +393,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, else hdr_padded_len = sizeof(struct padded_vnet_hdr); - if (hdr_valid) + if (hdr_valid && !metasize) memcpy(hdr, p, hdr_len); len -= hdr_len;@@ -405,6 +405,11 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, copy = skb_tailroom(skb); skb_put_data(skb, p, copy); + if (metasize) { + __skb_pull(skb, metasize); + skb_metadata_set(skb, metasize); + } + len -= copy; offset += copy; @@ -644,6 +649,7 @@ static struct sk_buff *receive_small(struct net_device *dev, unsigned int delta = 0; struct page *xdp_page; int err; + unsigned int metasize = 0; len -= vi->hdr_len; stats->bytes += len;@@ -683,10 +689,13 @@ static struct sk_buff *receive_small(struct net_device *dev, xdp.data_hard_start = buf + VIRTNET_RX_PAD + vi->hdr_len; xdp.data = xdp.data_hard_start + xdp_headroom;Quoted 8 lines
Quoted 8 lines
drivers/net/virtio_net.c static struct sk_buff *receive_small(struct net_device *dev, · line 698
Quoted 8 lines
drivers/net/virtio_net.c +4
-1- xdp_set_data_meta_invalid(&xdp); xdp.data_end = xdp.data + len; + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; orig_data = xdp.data; + /* Copy the vnet header to the front of data_hard_start to avoid + * overwriting by XDP meta data */ + memcpy(xdp.data_hard_start - vi->hdr_len, xdp.data - vi->hdr_len, vi->hdr_len);I'm not fully sure if I'm following this one correctly, probably just missing something. Isn't the vnet header based on how we set up xdp.data_hard_start earlier already in front of it? Wouldn't we copy invalid data from xdp.data -
vi->hdr_leninto the vnet header at that point (given there can be up to 256 bytes of headroom between the two)? If it's relative to xdp.data and headroom is >0, then BPF prog could otherwise mangle this; something doesn't add up to me here. Could you clarify? Thx
Quoted 5 lines
Quoted 1 line
Vnet headr sits just in front of xdp.data not xdp.data_hard_start. So it could be overwrote by metadata, that's why we need a copy here.
For the current code, you can adjust the xdp.data with a positive/negative offset already via
bpf_xdp_adjust_head()helper. If vnet headr sits just in front of xdp.data, couldn't this be overridden today as well then? Anyway, just wondering how this is handled differently?
We will invalidate the vnet header in this case. But for the case of metadata adjustment without header adjustment, we want to seek a way to preserve that.
Thanks
Quoted 3 lines
Thanks,
Daniel
I'm sorry for the late reply.
I saw the status of this patch is "Awaiting Upstream" on
https://patchwork.ozlabs.org/patch/1126046/. What is "Awaiting Upstream"? Is there anything that I can do?
Thank you,
Yuya
Quoted 96 lines · Jason Wang
Quoted 76 lines · Daniel Borkmann
Quoted 75 lines · Jason Wang
Quoted 74 lines · Daniel Borkmann
Quoted 58 lines · Yuya Kusakabe
Quoted 57 lines · Jason Wang
Quoted 56 lines · Yuya Kusakabe
This adds XDP meta data support to both
receive_small()andreceive_mergeable().Fixes de8f3a83b0a0 ("bpf: add meta pointer for direct access")Signed-off-by Yuya Kusakabe <[email protected]>---
v3:
- fix preserve the vnet header inreceive_small().
v2:
- keep copy untouched inpage_to_skb().
- preserve the vnet header inreceive_small().
- fix indentation.--- drivers/net/virtio_net.c | 45 +++++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 14 deletions(-)drivers/net/virtio_net.c +7
-1@@ -371,7 +371,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,struct receive_queue *rq, struct page *page, unsigned int offset, unsigned int len, unsigned int truesize, - bool hdr_valid) + bool hdr_valid, unsigned int metasize) { struct sk_buff *skb;struct virtio_net_hdr_mrg_rxbuf *hdr;@@ -393,7 +393,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,else hdr_padded_len =sizeof (struct padded_vnet_hdr ); - if (hdr_valid) +if (hdr_valid && !metasize)memcpy (hdr, p, hdr_len); len -= hdr_len;@@ -405,6 +405,11 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, copy =skb_tailroom (skb);skb_put_data (skb, p, copy); + if (metasize) { +__skb_pull (skb, metasize); +skb_metadata_set (skb, metasize); + } + len -= copy; offset += copy; @@ -644,6 +649,7 @@ static struct sk_buff *receive_small(struct net_device *dev, unsigned int delta = 0;struct page *xdp_page;int err; +unsigned int metasize =0 ; len -= vi->hdr_len; stats->bytes += len;@@ -683,10 +689,13 @@ static struct sk_buff *receive_small(struct net_device *dev, xdp.data_hard_start = buf +VIRTNET_RX_PAD + vi->hdr_len; xdp.data = xdp.data_hard_start + xdp_headroom;Quoted 8 lines
Quoted 8 lines
drivers/net/virtio_net.c static struct sk_buff *receive_small(struct net_device *dev, · line 697
Quoted 8 lines
drivers/net/virtio_net.c +4
-1-xdp_set_data_meta_invalid (&xdp); xdp.data_end = xdp.data + len; + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; orig_data = xdp.data; +/* Copy the vnet header to the front of data_hard_start to avoid +* overwriting by XDP meta data */ +memcpy (xdp.data_hard_start - vi->hdr_len, xdp.data - vi->hdr_len, vi->hdr_len);I'm not fully sure if I'm following this one correctly, probably just missing something. Isn't the vnet header based on how we set up xdp.data_hard_start earlier already in front of it? Wouldn't we copy invalid data from xdp.data -
vi->hdr_leninto the vnet header at that point (given there can be up to 256 bytes of headroom between the two)? If it's relative to xdp.data and headroom is >0, then BPF prog could otherwise mangle this; something doesn't add up to me here. Could you clarify? ThxQuoted 5 lines
Quoted 1 line
Vnet headr sits just in front of xdp.data not xdp.data_hard_start. So it could be overwrote by metadata, that's why we need a copy here.
For the current code, you can adjust the xdp.data with a positive/negative offset already via
bpf_xdp_adjust_head()helper. If vnet headr sits just in front of xdp.data, couldn't this be overridden today as well then? Anyway, just wondering how this is handled differently?We will invalidate the vnet header in this case. But for the case of metadata adjustment without header adjustment, we want to seek a way to preserve that.
Thanks
Quoted 3 lines
Thanks,
Daniel
Signature · 1 line
Yuya Kusakabe
Quoted 5 lines · Yuya Kusakabe
I'm sorry for the late reply.
I saw the status of this patch is "Awaiting Upstream" on
https://patchwork.ozlabs.org/patch/1126046/. What is "Awaiting Upstream"? Is there anything that I can do?
You can post a new version I think.
Thanks
Quoted 100 lines
Thank you,
YuyaQuoted 93 lines · Jason Wang
Quoted 76 lines · Daniel Borkmann
Quoted 75 lines · Jason Wang
Quoted 74 lines · Daniel Borkmann
Quoted 58 lines · Yuya Kusakabe
Quoted 57 lines · Jason Wang
Quoted 56 lines · Yuya Kusakabe
This adds XDP meta data support to both
receive_small()andreceive_mergeable().Fixes de8f3a83b0a0 ("bpf: add meta pointer for direct access")Signed-off-by Yuya Kusakabe <[email protected]>---
v3:
- fix preserve the vnet header inreceive_small().
v2:
- keep copy untouched inpage_to_skb().
- preserve the vnet header inreceive_small().
- fix indentation.--- drivers/net/virtio_net.c | 45 +++++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 14 deletions(-)drivers/net/virtio_net.c +7
-1@@ -371,7 +371,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,struct receive_queue *rq, struct page *page, unsigned int offset, unsigned int len, unsigned int truesize, - bool hdr_valid) + bool hdr_valid, unsigned int metasize) { struct sk_buff *skb;struct virtio_net_hdr_mrg_rxbuf *hdr;@@ -393,7 +393,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,else hdr_padded_len =sizeof (struct padded_vnet_hdr ); - if (hdr_valid) +if (hdr_valid && !metasize)memcpy (hdr, p, hdr_len); len -= hdr_len;@@ -405,6 +405,11 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, copy =skb_tailroom (skb);skb_put_data (skb, p, copy); + if (metasize) { +__skb_pull (skb, metasize); +skb_metadata_set (skb, metasize); + } + len -= copy; offset += copy; @@ -644,6 +649,7 @@ static struct sk_buff *receive_small(struct net_device *dev, unsigned int delta = 0;struct page *xdp_page;int err; +unsigned int metasize =0 ; len -= vi->hdr_len; stats->bytes += len;@@ -683,10 +689,13 @@ static struct sk_buff *receive_small(struct net_device *dev, xdp.data_hard_start = buf +VIRTNET_RX_PAD + vi->hdr_len; xdp.data = xdp.data_hard_start + xdp_headroom;Quoted 8 lines
Quoted 8 lines
drivers/net/virtio_net.c static struct sk_buff *receive_small(struct net_device *dev, · line 697
Quoted 8 lines
drivers/net/virtio_net.c +4
-1-xdp_set_data_meta_invalid (&xdp); xdp.data_end = xdp.data + len; + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; orig_data = xdp.data; +/* Copy the vnet header to the front of data_hard_start to avoid +* overwriting by XDP meta data */ +memcpy (xdp.data_hard_start - vi->hdr_len, xdp.data - vi->hdr_len, vi->hdr_len);I'm not fully sure if I'm following this one correctly, probably just missing something. Isn't the vnet header based on how we set up xdp.data_hard_start earlier already in front of it? Wouldn't we copy invalid data from xdp.data -
vi->hdr_leninto the vnet header at that point (given there can be up to 256 bytes of headroom between the two)? If it's relative to xdp.data and headroom is >0, then BPF prog could otherwise mangle this; something doesn't add up to me here. Could you clarify? ThxQuoted 5 lines
Quoted 1 line
Vnet headr sits just in front of xdp.data not xdp.data_hard_start. So it could be overwrote by metadata, that's why we need a copy here.
For the current code, you can adjust the xdp.data with a positive/negative offset already via
bpf_xdp_adjust_head()helper. If vnet headr sits just in front of xdp.data, couldn't this be overridden today as well then? Anyway, just wondering how this is handled differently?We will invalidate the vnet header in this case. But for the case of metadata adjustment without header adjustment, we want to seek a way to preserve that.
Thanks
Quoted 2 lines
Thanks,
Daniel
PATCH v4 1/1 virtio_net: add XDP meta data support
Implement support for transferring XDP meta data into skb for virtio_net driver; before calling into the program, xdp.data_meta points to xdp.data and copy vnet header to the front of xdp.data_hard_start to avoid overwriting it, where on program return with pass verdict, we call into skb_metadata_set().
--- drivers/net/virtio_net.c | 47 ++++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 14 deletions(-)
drivers/net/virtio_net.c +33 -14
@@ -371,7 +371,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,struct receive_queue *rq, struct page *page, unsigned int offset, unsigned int len, unsigned int truesize, - bool hdr_valid) + bool hdr_valid, unsigned int metasize) { struct sk_buff *skb;struct virtio_net_hdr_mrg_rxbuf *hdr;
@@ -393,7 +393,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,else hdr_padded_len =sizeof (struct padded_vnet_hdr ); -if (hdr_valid) +if (hdr_valid && !metasize)memcpy (hdr, p, hdr_len); len -= hdr_len;
@@ -405,6 +405,11 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, copy =skb_tailroom (skb);skb_put_data (skb, p, copy); +if (metasize) { +__skb_pull (skb, metasize); +skb_metadata_set (skb, metasize); + } + len -= copy; offset += copy;
@@ -644,6 +649,7 @@ static struct sk_buff *receive_small(struct net_device *dev,unsigned int delta =0 ;struct page *xdp_page;int err; +unsigned int metasize =0 ; len -= vi->hdr_len; stats->bytes += len;
@@ -683,10 +689,15 @@ static struct sk_buff *receive_small(struct net_device *dev, xdp.data_hard_start = buf +VIRTNET_RX_PAD + vi->hdr_len; xdp.data = xdp.data_hard_start + xdp_headroom; -xdp_set_data_meta_invalid (&xdp); xdp.data_end = xdp.data + len; + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; orig_data = xdp.data; +/* Copy the vnet header to the front of data_hard_start to avoid +* overwriting it by XDP meta data. +*/ +memcpy (xdp.data_hard_start - vi->hdr_len, + xdp.data - vi->hdr_len, vi->hdr_len); act =bpf_prog_run_xdp (xdp_prog, &xdp); stats->xdp_packets++;
@@ -695,9 +706,11 @@ static struct sk_buff *receive_small(struct net_device *dev,/* Recalculate length in case bpf program changed it */ delta = orig_data - xdp.data; len = xdp.data_end - xdp.data; + metasize = xdp.data - xdp.data_meta;break ;case XDP_TX : stats->xdp_tx++; + xdp.data_meta = xdp.data; xdpf =convert_to_xdp_frame (&xdp);if (unlikely (!xdpf)) goto err_xdp;
@@ -736,10 +749,12 @@ static struct sk_buff *receive_small(struct net_device *dev,skb_reserve (skb, headroom - delta);skb_put (skb, len);if (!delta) { - buf += header_offset; -memcpy (skb_vnet_hdr (skb), buf, vi->hdr_len); +memcpy (skb_vnet_hdr (skb), buf +VIRTNET_RX_PAD , vi->hdr_len); }/* keep zeroed vnet hdr since packet was changed by bpf */ +if (metasize) +skb_metadata_set (skb, metasize); + err:return skb;
@@ -760,8 +775,8 @@ static struct sk_buff *receive_big(struct net_device *dev, struct virtnet_rq_stats *stats) {struct page *page = buf; -struct sk_buff *skb =page_to_skb (vi, rq, page,0 , len, -PAGE_SIZE , true); +struct sk_buff *skb = +page_to_skb (vi, rq, page,0 , len,PAGE_SIZE , true,0 ); stats->bytes += len - vi->hdr_len;if (unlikely (!skb))
@@ -793,6 +808,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,unsigned int truesize;unsigned int headroom =mergeable_ctx_to_headroom (ctx);int err; +unsigned int metasize =0 ; head_skb =NULL ; stats->bytes += len - vi->hdr_len;
@@ -839,8 +855,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, data =page_address (xdp_page) + offset; xdp.data_hard_start = data -VIRTIO_XDP_HEADROOM + vi->hdr_len; xdp.data = data + vi->hdr_len; -xdp_set_data_meta_invalid (&xdp); xdp.data_end = xdp.data + (len - vi->hdr_len); + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; act =bpf_prog_run_xdp (xdp_prog, &xdp);
@@ -852,8 +868,9 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,* adjustments. Note other cases do not build an * skb and avoid using offset */ - offset = xdp.data - -page_address (xdp_page) - vi->hdr_len; + metasize = xdp.data - xdp.data_meta; + offset = xdp.data -page_address (xdp_page) - + vi->hdr_len - metasize;/* recalculate len if xdp.data or xdp.data_end were * adjusted
@@ -863,14 +880,15 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,if (unlikely (xdp_page != page)) {rcu_read_unlock ();put_page (page); - head_skb =page_to_skb (vi, rq, xdp_page, - offset, len, -PAGE_SIZE , false); + head_skb =page_to_skb (vi, rq, xdp_page, offset, + len,PAGE_SIZE , false, + metasize);return head_skb; }break ;case XDP_TX : stats->xdp_tx++; + xdp.data_meta = xdp.data; xdpf =convert_to_xdp_frame (&xdp);if (unlikely (!xdpf)) goto err_xdp;
@@ -921,7 +939,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, goto err_skb; } - head_skb =page_to_skb (vi, rq, page, offset, len, truesize, !xdp_prog); + head_skb =page_to_skb (vi, rq, page, offset, len, truesize, !xdp_prog, + metasize); curr_skb = head_skb;if (unlikely (!curr_skb))
Signature · 1 line
2.24.1
Quoted 26 lines · Yuya Kusakabe
Implement support for transferring XDP meta data into skb for
virtio_netdriver; before calling into the program, xdp.data_meta points to xdp.data and copy vnet header to the front of xdp.data_hard_start to avoid overwriting it, where on program return with pass verdict, we call intoskb_metadata_set().Fixes de8f3a83b0a0 ("bpf: add meta pointer for direct access")Signed-off-by Yuya Kusakabe <[email protected]>--- drivers/net/virtio_net.c | 47 ++++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 14 deletions(-)drivers/net/virtio_net.c +1
-1@@ -371,7 +371,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,struct receive_queue *rq, struct page *page, unsigned int offset, unsigned int len, unsigned int truesize, - bool hdr_valid) + bool hdr_valid, unsigned int metasize) { struct sk_buff *skb;struct virtio_net_hdr_mrg_rxbuf *hdr;
drivers/net/virtio_net.c static struct sk_buff *page_to_skb(struct virtnet_info *vi, · line 396
Quoted 5 lines
drivers/net/virtio_net.c +1
-1@@ -393,7 +393,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,else hdr_padded_len =sizeof (struct padded_vnet_hdr ); -if (hdr_valid) +if (hdr_valid && !metasize)
hdr_valid means no XDP, so I think we can remove the check for metasize here and add a comment instead?
Quoted 27 lines
drivers/net/virtio_net.c +6
-0memcpy (hdr, p, hdr_len); len -= hdr_len;@@ -405,6 +405,11 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, copy =skb_tailroom (skb);skb_put_data (skb, p, copy); +if (metasize) { +__skb_pull (skb, metasize); +skb_metadata_set (skb, metasize); + } + len -= copy; offset += copy;@@ -644,6 +649,7 @@ static struct sk_buff *receive_small(struct net_device *dev,unsigned int delta =0 ;struct page *xdp_page;int err; +unsigned int metasize =0 ; len -= vi->hdr_len; stats->bytes += len;@@ -683,10 +689,15 @@ static struct sk_buff *receive_small(struct net_device *dev, xdp.data_hard_start = buf +VIRTNET_RX_PAD + vi->hdr_len; xdp.data = xdp.data_hard_start + xdp_headroom;
drivers/net/virtio_net.c static struct sk_buff *receive_small(struct net_device *dev, · line 700
Quoted 10 lines
drivers/net/virtio_net.c +6
-1-xdp_set_data_meta_invalid (&xdp); xdp.data_end = xdp.data + len; + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; orig_data = xdp.data; +/* Copy the vnet header to the front of data_hard_start to avoid +* overwriting it by XDP meta data. +*/ +memcpy (xdp.data_hard_start - vi->hdr_len, + xdp.data - vi->hdr_len, vi->hdr_len);
I think we don't need this. And it looks to me there's a bug in the current code.
Commit 436c9453a1ac0 ("virtio-net: keep vnet header zeroed after processing XDP") leave the a corner case for receive_small() which still use:
if (!delta) {
buf += header_offset; memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len); } /* keep zeroed vnet hdr since packet was changed by bpf */
Which seems wrong, we need check xdp_prog instead of delta.
With this fixed, there's no need to care about the vnet header here since we don't know whether or not packet is modified by XDP.
drivers/net/virtio_net.c static struct sk_buff *receive_small(struct net_device *dev, · line 713
Quoted 12 lines
drivers/net/virtio_net.c +2
-0act =bpf_prog_run_xdp (xdp_prog, &xdp); stats->xdp_packets++;@@ -695,9 +706,11 @@ static struct sk_buff *receive_small(struct net_device *dev,/* Recalculate length in case bpf program changed it */ delta = orig_data - xdp.data; len = xdp.data_end - xdp.data; + metasize = xdp.data - xdp.data_meta;break ;case XDP_TX : stats->xdp_tx++; + xdp.data_meta = xdp.data;
I think we should remove the xdp_set_data_meta_invalid() at least? And move this initialization just after xdp.data is initialized.
Testing receive_small() requires to disable mrg_rxbuf, guest_tso4, guest_tso6 and guest_ufo from qemu command line.
drivers/net/virtio_net.c static struct sk_buff *receive_small(struct net_device *dev, · line 751
Quoted 7 lines
drivers/net/virtio_net.c +0
-0xdpf =convert_to_xdp_frame (&xdp);if (unlikely (!xdpf)) goto err_xdp;@@ -736,10 +749,12 @@ static struct sk_buff *receive_small(struct net_device *dev,skb_reserve (skb, headroom - delta);skb_put (skb, len);if (!delta) {
Need to check xdp_prog (need another patch).
Quoted 58 lines
drivers/net/virtio_net.c +11
-9- buf += header_offset; -memcpy (skb_vnet_hdr (skb), buf, vi->hdr_len); +memcpy (skb_vnet_hdr (skb), buf +VIRTNET_RX_PAD , vi->hdr_len); }/* keep zeroed vnet hdr since packet was changed by bpf */ +if (metasize) +skb_metadata_set (skb, metasize); + err:return skb;@@ -760,8 +775,8 @@ static struct sk_buff *receive_big(struct net_device *dev, struct virtnet_rq_stats *stats) {struct page *page = buf; -struct sk_buff *skb =page_to_skb (vi, rq, page,0 , len, -PAGE_SIZE , true); +struct sk_buff *skb = +page_to_skb (vi, rq, page,0 , len,PAGE_SIZE , true,0 ); stats->bytes += len - vi->hdr_len;if (unlikely (!skb))@@ -793,6 +808,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,unsigned int truesize;unsigned int headroom =mergeable_ctx_to_headroom (ctx);int err; +unsigned int metasize =0 ; head_skb =NULL ; stats->bytes += len - vi->hdr_len;@@ -839,8 +855,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, data =page_address (xdp_page) + offset; xdp.data_hard_start = data -VIRTIO_XDP_HEADROOM + vi->hdr_len; xdp.data = data + vi->hdr_len; -xdp_set_data_meta_invalid (&xdp); xdp.data_end = xdp.data + (len - vi->hdr_len); + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; act =bpf_prog_run_xdp (xdp_prog, &xdp);@@ -852,8 +868,9 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,* adjustments. Note other cases do not build an * skb and avoid using offset */ - offset = xdp.data - -page_address (xdp_page) - vi->hdr_len; + metasize = xdp.data - xdp.data_meta; + offset = xdp.data -page_address (xdp_page) - + vi->hdr_len - metasize;/* recalculate len if xdp.data or xdp.data_end were * adjusted @@ -863,14 +880,15 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,if (unlikely (xdp_page != page)) {rcu_read_unlock ();put_page (page); - head_skb = page_to_skb(vi, rq, xdp_page, - offset, len,
drivers/net/virtio_net.c static struct sk_buff *receive_mergeable(struct net_device *dev, · line 891
Quoted 10 lines
drivers/net/virtio_net.c +4
-1- PAGE_SIZE, false); + head_skb =page_to_skb (vi, rq, xdp_page, offset, + len,PAGE_SIZE , false, + metasize);return head_skb; }break ;case XDP_TX : stats->xdp_tx++; + xdp.data_meta = xdp.data;
Any reason for doing this?
Thanks
Quoted 13 lines
drivers/net/virtio_net.c +2
-1xdpf =convert_to_xdp_frame (&xdp);if (unlikely (!xdpf)) goto err_xdp;@@ -921,7 +939,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, goto err_skb; } - head_skb =page_to_skb (vi, rq, page, offset, len, truesize, !xdp_prog); + head_skb =page_to_skb (vi, rq, page, offset, len, truesize, !xdp_prog, + metasize); curr_skb = head_skb;if (unlikely (!curr_skb))
Quoted 27 lines · Jason Wang
Quoted 25 lines · Yuya Kusakabe
Implement support for transferring XDP meta data into skb for
virtio_netdriver; before calling into the program, xdp.data_meta points to xdp.data and copy vnet header to the front of xdp.data_hard_start to avoid overwriting it, where on program return with pass verdict, we call intoskb_metadata_set().Fixes de8f3a83b0a0 ("bpf: add meta pointer for direct access")Signed-off-by Yuya Kusakabe <[email protected]>---
drivers/net/virtio_net.c| 47 ++++++++++++++++++++++++++++------------1 file changed, 33 insertions(+), 14 deletions(-)drivers/net/virtio_net.c +1
-1@@ -371,7 +371,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, struct receive_queue *rq, struct page *page, unsigned int offset, unsigned int len, unsigned int truesize, - bool hdr_valid) + bool hdr_valid, unsigned int metasize) { struct sk_buff *skb; struct virtio_net_hdr_mrg_rxbuf *hdr;
Quoted 8 lines
Quoted 5 lines
drivers/net/virtio_net.c +0
-0@@ -393,7 +393,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,else
hdr_padded_len=sizeof(struct padded_vnet_hdr); - if (hdr_valid)
+ if (hdr_valid&& !metasize)
hdr_validmeans no XDP, so I think we can remove the check for metasize here and add a comment instead?
I will fix it on next patch.
Quoted 39 lines
Quoted 23 lines
memcpy(hdr, p, hdr_len);
len -=hdr_len;drivers/net/virtio_net.c +4
-0@@ -405,6 +405,11 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, copy = skb_tailroom(skb); skb_put_data(skb, p, copy); + if (metasize) { + __skb_pull(skb, metasize); + skb_metadata_set(skb, metasize); + } + len -= copy; offset += copy; @@ -644,6 +649,7 @@ static struct sk_buff *receive_small(struct net_device *dev,unsigned int delta = 0;
struct page*xdp_page;
int err;
+ unsigned int metasize = 0;
len -=vi->hdr_len;
stats->bytes+= len;drivers/net/virtio_net.c +0
-0@@ -683,10 +689,15 @@ static struct sk_buff *receive_small(struct net_device *dev, xdp.data_hard_start = buf + VIRTNET_RX_PAD + vi->hdr_len; xdp.data = xdp.data_hard_start + xdp_headroom;drivers/net/virtio_net.c static struct sk_buff *receive_small(struct net_device *dev, · line 699
Quoted 10 lines
drivers/net/virtio_net.c +6
-1- xdp_set_data_meta_invalid(&xdp); xdp.data_end = xdp.data + len; + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; orig_data = xdp.data; + /* Copy the vnet header to the front of data_hard_start to avoid + * overwriting it by XDP meta data. + */ + memcpy(xdp.data_hard_start - vi->hdr_len, + xdp.data - vi->hdr_len, vi->hdr_len);I think we don't need this. And it looks to me there's a bug in the current code.
Commit
436c9453a1ac0("virtio-net: keep vnet header zeroed after processing XDP") leave the a corner case forreceive_small()which still use:
Quoted 8 lines
if (!delta) {
buf +=header_offset;memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len); } /* keep zeroed vnet hdr since packet was changed by bpf */Which seems wrong, we need check
xdp_proginstead of delta.With this fixed, there's no need to care about the vnet header here since we don't know whether or not packet is modified by XDP.
I missed this commit. I understand this is the reason for "Awaiting Upstream".
Quoted 19 lines
Quoted 5 lines
drivers/net/virtio_net.c +0
-0act = bpf_prog_run_xdp(xdp_prog, &xdp); stats->xdp_packets++; @@ -695,9 +706,11 @@ static struct sk_buff *receive_small(struct net_device *dev, /* Recalculate length in case bpf program changed it */Quoted 7 lines
delta =
orig_data- xdp.data;
len = xdp.data_end - xdp.data;
+ metasize = xdp.data - xdp.data_meta;
break;
caseXDP_TX:
stats->xdp_tx++;
+ xdp.data_meta = xdp.data;I think we should remove the
xdp_set_data_meta_invalid()at least? And move this initialization just after xdp.data is initialized.Testing
receive_small()requires to disablemrg_rxbuf,guest_tso4,guest_tso6andguest_ufofrom qemu command line.
Quoted 10 lines
drivers/net/virtio_net.c static struct sk_buff *receive_small(struct net_device *dev, · line 751
Quoted 7 lines
xdpf =
convert_to_xdp_frame(&xdp);
if (unlikely(!xdpf))
gotoerr_xdp;drivers/net/virtio_net.c +0
-0@@ -736,10 +749,12 @@ static struct sk_buff *receive_small(struct net_device *dev, skb_reserve(skb, headroom - delta); skb_put(skb, len); if (!delta) {Need to check
xdp_prog(need another patch).
I will fix it on next patch.
Quoted 59 lines
Quoted 59 lines
drivers/net/virtio_net.c +3
-2- buf += header_offset; - memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len); + memcpy(skb_vnet_hdr(skb), buf + VIRTNET_RX_PAD, vi->hdr_len); } /* keep zeroed vnet hdr since packet was changed by bpf */ + if (metasize) + skb_metadata_set(skb, metasize); +err:
return skb;
@@ -760,8 +775,8 @@ static structsk_buff*receive_big(structnet_device*dev, structvirtnet_rq_stats*stats)
{
struct page *page = buf;
- structsk_buff*skb = page_to_skb(vi, rq, page, 0, len,
-PAGE_SIZE, true);
+ structsk_buff*skb =
+page_to_skb(vi, rq, page, 0, len, PAGE_SIZE, true, 0);stats->bytes+= len -vi->hdr_len;
if (unlikely(!skb))drivers/net/virtio_net.c +8
-6@@ -793,6 +808,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, unsigned int truesize; unsigned int headroom = mergeable_ctx_to_headroom(ctx); int err; + unsigned int metasize = 0; head_skb = NULL; stats->bytes += len - vi->hdr_len;@@ -839,8 +855,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, data = page_address(xdp_page) + offset; xdp.data_hard_start = data - VIRTIO_XDP_HEADROOM + vi->hdr_len; xdp.data = data + vi->hdr_len; - xdp_set_data_meta_invalid(&xdp); xdp.data_end = xdp.data + (len - vi->hdr_len); + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; act = bpf_prog_run_xdp(xdp_prog, &xdp);@@ -852,8 +868,9 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,* adjustments. Note other cases do not build an * skb and avoid using offset */ - offset = xdp.data - - page_address(xdp_page) - vi->hdr_len; + metasize = xdp.data - xdp.data_meta; + offset = xdp.data - page_address(xdp_page) - + vi->hdr_len - metasize; /* recalculate len if xdp.data or xdp.data_end were* adjusted @@ -863,14 +880,15 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, if (unlikely(xdp_page != page)) { rcu_read_unlock(); put_page(page); - head_skb = page_to_skb(vi, rq, xdp_page, - offset, len, - PAGE_SIZE, false); + head_skb = page_to_skb(vi, rq, xdp_page, offset, + len, PAGE_SIZE, false, + metasize); return head_skb;
Quoted 8 lines
Quoted 5 lines
drivers/net/virtio_net.c +0
-0} break;case
XDP_TX:
stats->xdp_tx++;
+ xdp.data_meta = xdp.data;Any reason for doing this?
XDP_TX can not support metadata for now, because if metasize > 0, __virtnet_xdp_xmit_one() returns EOPNOTSUPP.
static int __virtnet_xdp_xmit_one(struct virtnet_info *vi,
struct send_queue *sq, struct xdp_frame *xdpf)
{
struct virtio_net_hdr_mrg_rxbuf *hdr;
int err;
/* virtqueue want to use data area in-front of packet */
if (unlikely(xdpf->metasize > 0))
return -EOPNOTSUPP;Quoted 4 lines
Thanks
Quoted 12 lines
drivers/net/virtio_net.c static struct sk_buff *receive_mergeable(struct net_device *dev, · line 945
Quoted 11 lines
xdpf =
convert_to_xdp_frame(&xdp);
if (unlikely(!xdpf))
gotoerr_xdp;drivers/net/virtio_net.c +2
-0@@ -921,7 +939,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, goto err_skb; } - head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog); + head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog, + metasize); curr_skb = head_skb; if (unlikely(!curr_skb))
Thank you for your kind review.
Quoted 9 lines · Yuya Kusakabe
Quoted 4 lines
Quoted 3 lines
case
XDP_TX:
stats->xdp_tx++;
+ xdp.data_meta = xdp.data;Any reason for doing this?
XDP_TXcan not support metadata for now, because if metasize > 0,__virtnet_xdp_xmit_one()returns EOPNOTSUPP.static int __virtnet_xdp_xmit_one(struct
virtnet_info*vi,
structsend_queue*sq, structxdp_frame*xdpf)
Quoted 9 lines
{
struct virtio_net_hdr_mrg_rxbuf *hdr;
int err;
/* virtqueue want to use data area in-front of packet */
if (unlikely(xdpf->metasize > 0))
return -EOPNOTSUPP;I see.
Then I think it's better to fix __virtnet_xdp_xmit_one() instead.
Thanks
PATCH v5 1/1 virtio_net: add XDP meta data support
Implement support for transferring XDP meta data into skb for virtio_net driver; before calling into the program, xdp.data_meta points to xdp.data, where on program return with pass verdict, we call into skb_metadata_set().
Tested with the script at
https://github.com/higebu/virtio_net-xdp-metadata-test.
---
v5:
- page_to_skb(): copy vnet header if hdr_valid without checking metasize.
- receive_small(): do not copy vnet header if xdp_prog is availavle.
- __virtnet_xdp_xmit_one(): remove the xdp_set_data_meta_invalid().
- improve comments.
v4:
- improve commit message
v3:
- fix preserve the vnet header in receive_small().
v2:
- keep copy untouched in page_to_skb().
- preserve the vnet header in receive_small().
- fix indentation.
--- drivers/net/virtio_net.c | 54 ++++++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 21 deletions(-)
drivers/net/virtio_net.c +33 -21
@@ -371,7 +371,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,struct receive_queue *rq, struct page *page, unsigned int offset, unsigned int len, unsigned int truesize, - bool hdr_valid) + bool hdr_valid, unsigned int metasize) { struct sk_buff *skb;struct virtio_net_hdr_mrg_rxbuf *hdr;
@@ -393,6 +393,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,else hdr_padded_len =sizeof (struct padded_vnet_hdr ); +/* hdr_valid means no XDP, so we can copy the vnet header */ if (hdr_valid)memcpy (hdr, p, hdr_len);
@@ -405,6 +406,11 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, copy =skb_tailroom (skb);skb_put_data (skb, p, copy); +if (metasize) { +__skb_pull (skb, metasize); +skb_metadata_set (skb, metasize); + } + len -= copy; offset += copy;
@@ -450,10 +456,6 @@ static int __virtnet_xdp_xmit_one(struct virtnet_info *vi,struct virtio_net_hdr_mrg_rxbuf *hdr;int err; -/* virtqueue want to use data area in-front of packet */ -if (unlikely (xdpf->metasize >0 )) -return -EOPNOTSUPP ; -if (unlikely (xdpf->headroom < vi->hdr_len))return -EOVERFLOW ;
@@ -644,6 +646,7 @@ static struct sk_buff *receive_small(struct net_device *dev,unsigned int delta =0 ;struct page *xdp_page;int err; +unsigned int metasize =0 ; len -= vi->hdr_len; stats->bytes += len;
@@ -683,8 +686,8 @@ static struct sk_buff *receive_small(struct net_device *dev, xdp.data_hard_start = buf +VIRTNET_RX_PAD + vi->hdr_len; xdp.data = xdp.data_hard_start + xdp_headroom; -xdp_set_data_meta_invalid (&xdp); xdp.data_end = xdp.data + len; + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; orig_data = xdp.data; act =bpf_prog_run_xdp (xdp_prog, &xdp);
@@ -695,6 +698,7 @@ static struct sk_buff *receive_small(struct net_device *dev,/* Recalculate length in case bpf program changed it */ delta = orig_data - xdp.data; len = xdp.data_end - xdp.data; + metasize = xdp.data - xdp.data_meta;break ;case XDP_TX : stats->xdp_tx++;
@@ -735,11 +739,14 @@ static struct sk_buff *receive_small(struct net_device *dev, }skb_reserve (skb, headroom - delta);skb_put (skb, len); -if (!delta) { +if (!xdp_prog) { buf += header_offset;memcpy (skb_vnet_hdr (skb), buf, vi->hdr_len); }/* keep zeroed vnet hdr since packet was changed by bpf */ +if (metasize) +skb_metadata_set (skb, metasize); + err:return skb;
@@ -760,8 +767,8 @@ static struct sk_buff *receive_big(struct net_device *dev, struct virtnet_rq_stats *stats) {struct page *page = buf; -struct sk_buff *skb =page_to_skb (vi, rq, page,0 , len, -PAGE_SIZE , true); +struct sk_buff *skb = +page_to_skb (vi, rq, page,0 , len,PAGE_SIZE , true,0 ); stats->bytes += len - vi->hdr_len;if (unlikely (!skb))
@@ -793,6 +800,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,unsigned int truesize;unsigned int headroom =mergeable_ctx_to_headroom (ctx);int err; +unsigned int metasize =0 ; head_skb =NULL ; stats->bytes += len - vi->hdr_len;
@@ -839,8 +847,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, data =page_address (xdp_page) + offset; xdp.data_hard_start = data -VIRTIO_XDP_HEADROOM + vi->hdr_len; xdp.data = data + vi->hdr_len; -xdp_set_data_meta_invalid (&xdp); xdp.data_end = xdp.data + (len - vi->hdr_len); + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; act =bpf_prog_run_xdp (xdp_prog, &xdp);
@@ -848,24 +856,27 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,switch (act) {case XDP_PASS : + metasize = xdp.data - xdp.data_meta; +/* recalculate offset to account for any header -* adjustments. Note other cases do not build an -* skb and avoid using offset +* adjustments and minus the metasize to copy the +* metadata in page_to_skb(). Note other cases do not +* build an skb and avoid using offset */ - offset = xdp.data - -page_address (xdp_page) - vi->hdr_len; + offset = xdp.data -page_address (xdp_page) - + vi->hdr_len - metasize; -/* recalculate len if xdp.data or xdp.data_end were -* adjusted +/* recalculate len if xdp.data, xdp.data_end or +* xdp.data_meta were adjusted */ - len = xdp.data_end - xdp.data + vi->hdr_len; + len = xdp.data_end - xdp.data + vi->hdr_len + metasize;/* We can only create skb based on xdp_page. */ if (unlikely (xdp_page != page)) {rcu_read_unlock ();put_page (page); - head_skb =page_to_skb (vi, rq, xdp_page, - offset, len, -PAGE_SIZE , false); + head_skb =page_to_skb (vi, rq, xdp_page, offset, + len,PAGE_SIZE , false, + metasize);return head_skb; }break ;
@@ -921,7 +932,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, goto err_skb; } - head_skb =page_to_skb (vi, rq, page, offset, len, truesize, !xdp_prog); + head_skb =page_to_skb (vi, rq, page, offset, len, truesize, !xdp_prog, + metasize); curr_skb = head_skb;if (unlikely (!curr_skb))
Signature · 1 line
2.24.1
Quoted 9 lines · Yuya Kusakabe
Implement support for transferring XDP meta data into skb for
virtio_netdriver; before calling into the program, xdp.data_meta points to xdp.data, where on program return with pass verdict, we call intoskb_metadata_set().Tested with the script at
https://github.com/higebu/virtio_net-xdp-metadata-test.Fixes de8f3a83b0a0 ("bpf: add meta pointer for direct access")
I'm not sure this is correct since virtio-net claims to not support metadata by calling xdp_set_data_meta_invalid()?
Quoted 90 lines
Signed-off-by Yuya Kusakabe <[email protected]>---
v5:
-page_to_skb(): copy vnet header ifhdr_validwithout checking metasize.
-receive_small(): do not copy vnet header ifxdp_progis availavle.
-__virtnet_xdp_xmit_one(): remove thexdp_set_data_meta_invalid().
- improve comments.
v4:
- improve commit message
v3:
- fix preserve the vnet header inreceive_small().
v2:
- keep copy untouched inpage_to_skb().
- preserve the vnet header inreceive_small().
- fix indentation.--- drivers/net/virtio_net.c | 54 ++++++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 21 deletions(-)drivers/net/virtio_net.c +10
-6@@ -371,7 +371,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,struct receive_queue *rq, struct page *page, unsigned int offset, unsigned int len, unsigned int truesize, - bool hdr_valid) + bool hdr_valid, unsigned int metasize) { struct sk_buff *skb;struct virtio_net_hdr_mrg_rxbuf *hdr;@@ -393,6 +393,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,else hdr_padded_len =sizeof (struct padded_vnet_hdr ); +/* hdr_valid means no XDP, so we can copy the vnet header */ if (hdr_valid)memcpy (hdr, p, hdr_len);@@ -405,6 +406,11 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, copy =skb_tailroom (skb);skb_put_data (skb, p, copy); +if (metasize) { +__skb_pull (skb, metasize); +skb_metadata_set (skb, metasize); + } + len -= copy; offset += copy;@@ -450,10 +456,6 @@ static int __virtnet_xdp_xmit_one(struct virtnet_info *vi,struct virtio_net_hdr_mrg_rxbuf *hdr;int err; -/* virtqueue want to use data area in-front of packet */ -if (unlikely (xdpf->metasize >0 )) -return -EOPNOTSUPP ; -if (unlikely (xdpf->headroom < vi->hdr_len))return -EOVERFLOW ;@@ -644,6 +646,7 @@ static struct sk_buff *receive_small(struct net_device *dev,unsigned int delta =0 ;struct page *xdp_page;int err; +unsigned int metasize =0 ; len -= vi->hdr_len; stats->bytes += len;@@ -683,8 +686,8 @@ static struct sk_buff *receive_small(struct net_device *dev, xdp.data_hard_start = buf +VIRTNET_RX_PAD + vi->hdr_len; xdp.data = xdp.data_hard_start + xdp_headroom; -xdp_set_data_meta_invalid (&xdp); xdp.data_end = xdp.data + len; + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; orig_data = xdp.data; act =bpf_prog_run_xdp (xdp_prog, &xdp);@@ -695,6 +698,7 @@ static struct sk_buff *receive_small(struct net_device *dev,/* Recalculate length in case bpf program changed it */ delta = orig_data - xdp.data; len = xdp.data_end - xdp.data; + metasize = xdp.data - xdp.data_meta;break ;case XDP_TX : stats->xdp_tx++;
drivers/net/virtio_net.c static struct sk_buff *receive_small(struct net_device *dev, · line 745
Quoted 8 lines
drivers/net/virtio_net.c +1
-1@@ -735,11 +739,14 @@ static struct sk_buff *receive_small(struct net_device *dev, }skb_reserve (skb, headroom - delta);skb_put (skb, len); -if (!delta) { +if (!xdp_prog) { buf += header_offset;memcpy (skb_vnet_hdr (skb), buf, vi->hdr_len); }/* keep zeroed vnet hdr since packet was changed by bpf */
I prefer to make this an independent patch and cc stable.
Other looks good.
Thanks
Quoted 84 lines
drivers/net/virtio_net.c +22
-14+if (metasize) +skb_metadata_set (skb, metasize); + err:return skb;@@ -760,8 +767,8 @@ static struct sk_buff *receive_big(struct net_device *dev, struct virtnet_rq_stats *stats) {struct page *page = buf; -struct sk_buff *skb =page_to_skb (vi, rq, page,0 , len, -PAGE_SIZE , true); +struct sk_buff *skb = +page_to_skb (vi, rq, page,0 , len,PAGE_SIZE , true,0 ); stats->bytes += len - vi->hdr_len;if (unlikely (!skb))@@ -793,6 +800,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,unsigned int truesize;unsigned int headroom =mergeable_ctx_to_headroom (ctx);int err; +unsigned int metasize =0 ; head_skb =NULL ; stats->bytes += len - vi->hdr_len;@@ -839,8 +847,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, data =page_address (xdp_page) + offset; xdp.data_hard_start = data -VIRTIO_XDP_HEADROOM + vi->hdr_len; xdp.data = data + vi->hdr_len; -xdp_set_data_meta_invalid (&xdp); xdp.data_end = xdp.data + (len - vi->hdr_len); + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; act =bpf_prog_run_xdp (xdp_prog, &xdp);@@ -848,24 +856,27 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,switch (act) {case XDP_PASS : + metasize = xdp.data - xdp.data_meta; +/* recalculate offset to account for any header -* adjustments. Note other cases do not build an -* skb and avoid using offset +* adjustments and minus the metasize to copy the +* metadata in page_to_skb(). Note other cases do not +* build an skb and avoid using offset */ - offset = xdp.data - -page_address (xdp_page) - vi->hdr_len; + offset = xdp.data -page_address (xdp_page) - + vi->hdr_len - metasize; -/* recalculate len if xdp.data or xdp.data_end were -* adjusted +/* recalculate len if xdp.data, xdp.data_end or +* xdp.data_meta were adjusted */ - len = xdp.data_end - xdp.data + vi->hdr_len; + len = xdp.data_end - xdp.data + vi->hdr_len + metasize;/* We can only create skb based on xdp_page. */ if (unlikely (xdp_page != page)) {rcu_read_unlock ();put_page (page); - head_skb =page_to_skb (vi, rq, xdp_page, - offset, len, -PAGE_SIZE , false); + head_skb =page_to_skb (vi, rq, xdp_page, offset, + len,PAGE_SIZE , false, + metasize);return head_skb; }break ;@@ -921,7 +932,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, goto err_skb; } - head_skb =page_to_skb (vi, rq, page, offset, len, truesize, !xdp_prog); + head_skb =page_to_skb (vi, rq, page, offset, len, truesize, !xdp_prog, + metasize); curr_skb = head_skb;if (unlikely (!curr_skb))
Quoted 7 lines · Jason Wang
Quoted 5 lines · Yuya Kusakabe
Implement support for transferring XDP meta data into skb for
virtio_netdriver; before calling into the program, xdp.data_meta points to xdp.data, where on program return with pass verdict, we call intoskb_metadata_set().
Quoted 7 lines
Quoted 4 lines
Tested with the script at
https://github.com/higebu/virtio_net-xdp-metadata-test.Fixes de8f3a83b0a0 ("bpf: add meta pointer for direct access")I'm not sure this is correct since virtio-net claims to not support metadata by calling
xdp_set_data_meta_invalid()?
virtio_net doesn't support by calling xdp_set_data_meta_invalid() for now.
https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/tree/drivers/net/virtio_net.c?id=e42da4c62abb547d9c9138e0e7fcd1f36057b5e8#n686
https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/tree/drivers/net/virtio_net.c?id=e42da4c62abb547d9c9138e0e7fcd1f36057b5e8#n842
And xdp_set_data_meta_invalid() are added by de8f3a83b0a0.
$ git blame ./drivers/net/virtio_net.c | grep xdp_set_data_meta_invalid de8f3a83b0a0f (Daniel Borkmann 2017-09-25 02:25:51 +0200 686) xdp_set_data_meta_invalid(&xdp); de8f3a83b0a0f (Daniel Borkmann 2017-09-25 02:25:51 +0200 842) xdp_set_data_meta_invalid(&xdp);
So I added Fixes: de8f3a83b0a0 ("bpf: add meta pointer for direct access") to the comment.
Quoted 96 lines
Quoted 84 lines
Signed-off-by Yuya Kusakabe <[email protected]>---
v5:
-page_to_skb(): copy vnet header ifhdr_validwithout checking metasize.
-receive_small(): do not copy vnet header ifxdp_progis availavle.
-__virtnet_xdp_xmit_one(): remove thexdp_set_data_meta_invalid().
- improve comments.
v4:
- improve commit message
v3:
- fix preserve the vnet header inreceive_small().
v2:
- keep copy untouched inpage_to_skb().
- preserve the vnet header inreceive_small().
- fix indentation.
---
drivers/net/virtio_net.c| 54 ++++++++++++++++++++++++----------------1 file changed, 33 insertions(+), 21 deletions(-)drivers/net/virtio_net.c +1
-1@@ -371,7 +371,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, struct receive_queue *rq, struct page *page, unsigned int offset, unsigned int len, unsigned int truesize, - bool hdr_valid) + bool hdr_valid, unsigned int metasize) { struct sk_buff *skb; struct virtio_net_hdr_mrg_rxbuf *hdr;@@ -393,6 +393,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,else
hdr_padded_len=sizeof(struct padded_vnet_hdr);
+ /*hdr_validmeans no XDP, so we can copy the vnet header */ if (hdr_valid)
memcpy(hdr, p, hdr_len);
@@ -405,6 +406,11 @@ static structsk_buff*page_to_skb(structvirtnet_info*vi, copy =skb_tailroom(skb);
skb_put_data(skb, p, copy);
+ if (metasize) {
+__skb_pull(skb, metasize);
+skb_metadata_set(skb, metasize);
+ }
+
len -= copy;
offset += copy;
@@ -450,10 +456,6 @@ static int __virtnet_xdp_xmit_one(structvirtnet_info*vi, structvirtio_net_hdr_mrg_rxbuf*hdr;
int err;
- /* virtqueue want to use data area in-front of packet */
- if (unlikely(xdpf->metasize > 0))
- return-EOPNOTSUPP;
-
if (unlikely(xdpf->headroom < vi->hdr_len))
return-EOVERFLOW;
@@ -644,6 +646,7 @@ static structsk_buff*receive_small(structnet_device*dev, unsigned int delta = 0;
struct page*xdp_page;
int err;
+ unsigned int metasize = 0;
len -=vi->hdr_len;
stats->bytes+= len;drivers/net/virtio_net.c +2
-1@@ -683,8 +686,8 @@ static struct sk_buff *receive_small(struct net_device *dev, xdp.data_hard_start = buf + VIRTNET_RX_PAD + vi->hdr_len; xdp.data = xdp.data_hard_start + xdp_headroom; - xdp_set_data_meta_invalid(&xdp); xdp.data_end = xdp.data + len; + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; orig_data = xdp.data; act = bpf_prog_run_xdp(xdp_prog, &xdp);@@ -695,6 +698,7 @@ static struct sk_buff *receive_small(struct net_device *dev, /* Recalculate length in case bpf program changed it */ delta = orig_data - xdp.data; len = xdp.data_end - xdp.data; + metasize = xdp.data - xdp.data_meta; break;case
XDP_TX:
stats->xdp_tx++;drivers/net/virtio_net.c static struct sk_buff *receive_small(struct net_device *dev, · line 745
Quoted 8 lines
drivers/net/virtio_net.c +1
-1@@ -735,11 +739,14 @@ static struct sk_buff *receive_small(struct net_device *dev, } skb_reserve(skb, headroom - delta); skb_put(skb, len); - if (!delta) { + if (!xdp_prog) { buf += header_offset; memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len); } /* keep zeroed vnet hdr since packet was changed by bpf */I prefer to make this an independent patch and cc stable.
Quoted 3 lines
Other looks good.
Thanks
I see. So I need to revert to delta from xdp_prog?
Thank you.
Quoted 77 lines
Quoted 76 lines
drivers/net/virtio_net.c +2
-0+ if (metasize) + skb_metadata_set(skb, metasize); +err:
return skb;
@@ -760,8 +767,8 @@ static structsk_buff*receive_big(structnet_device*dev, structvirtnet_rq_stats*stats)
{
struct page *page = buf;
- structsk_buff*skb = page_to_skb(vi, rq, page, 0, len,
-PAGE_SIZE, true);
+ structsk_buff*skb =
+page_to_skb(vi, rq, page, 0, len, PAGE_SIZE, true, 0);stats->bytes+= len -vi->hdr_len;
if (unlikely(!skb))drivers/net/virtio_net.c +2
-1@@ -793,6 +800,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, unsigned int truesize; unsigned int headroom = mergeable_ctx_to_headroom(ctx); int err; + unsigned int metasize = 0; head_skb = NULL; stats->bytes += len - vi->hdr_len;@@ -839,8 +847,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, data = page_address(xdp_page) + offset; xdp.data_hard_start = data - VIRTIO_XDP_HEADROOM + vi->hdr_len; xdp.data = data + vi->hdr_len; - xdp_set_data_meta_invalid(&xdp); xdp.data_end = xdp.data + (len - vi->hdr_len); + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; act = bpf_prog_run_xdp(xdp_prog, &xdp);@@ -848,24 +856,27 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, switch (act) {case
XDP_PASS:
+ metasize = xdp.data - xdp.data_meta;
+
/* recalculate offset to account for any header - * adjustments. Note other cases do not build an - * skb and avoid using offset
+ * adjustments and minus the metasize to copy the + * metadata inpage_to_skb(). Note other cases do not + * build an skb and avoid using offset
*/
- offset = xdp.data -
-page_address(xdp_page)-vi->hdr_len; + offset = xdp.data -page_address(xdp_page)- +vi->hdr_len- metasize;
- /* recalculate len if xdp.data or xdp.data_end were - * adjusted
+ /* recalculate len if xdp.data, xdp.data_end or + * xdp.data_meta were adjusted
*/
- len = xdp.data_end - xdp.data +vi->hdr_len; + len = xdp.data_end - xdp.data +vi->hdr_len+ metasize; /* We can only create skb based onxdp_page. */ if (unlikely(xdp_page != page)) {rcu_read_unlock();
put_page(page);
-head_skb= page_to_skb(vi, rq,xdp_page, - offset, len,
-PAGE_SIZE, false);
+head_skb= page_to_skb(vi, rq,xdp_page, offset, + len,PAGE_SIZE, false,
+ metasize);
returnhead_skb;
}
break;drivers/net/virtio_net.c +2
-0@@ -921,7 +932,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, goto err_skb; } - head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog); + head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog, + metasize); curr_skb = head_skb; if (unlikely(!curr_skb))
Michael S. Tsirkin
maintainerQuoted 21 lines · Yuya Kusakabe
Quoted 7 lines · Jason Wang
Quoted 5 lines · Yuya Kusakabe
Implement support for transferring XDP meta data into skb for
virtio_netdriver; before calling into the program, xdp.data_meta points to xdp.data, where on program return with pass verdict, we call intoskb_metadata_set().Quoted 7 lines
Quoted 4 lines
Tested with the script at
https://github.com/higebu/virtio_net-xdp-metadata-test.Fixes de8f3a83b0a0 ("bpf: add meta pointer for direct access")I'm not sure this is correct since virtio-net claims to not support metadata by calling
xdp_set_data_meta_invalid()?
virtio_netdoesn't support by callingxdp_set_data_meta_invalid()for now.https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/tree/drivers/net/virtio_net.c?id=e42da4c62abb547d9c9138e0e7fcd1f36057b5e8#n686
https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/tree/drivers/net/virtio_net.c?id=e42da4c62abb547d9c9138e0e7fcd1f36057b5e8#n842
Quoted 7 lines
And
xdp_set_data_meta_invalid()are added byde8f3a83b0a0.$ git blame ./drivers/net/virtio_net.c | grep xdp_set_data_meta_invalid de8f3a83b0a0f (Daniel Borkmann 2017-09-25 02:25:51 +0200 686) xdp_set_data_meta_invalid(&xdp); de8f3a83b0a0f (Daniel Borkmann 2017-09-25 02:25:51 +0200 842) xdp_set_data_meta_invalid(&xdp);So I added
Fixes: de8f3a83b0a0 ("bpf: add meta pointer for direct access")to the comment.
Fixes basically means "must be backported to any kernel that has de8f3a83b0a0 in order to fix a bug". This looks more like a feature than a bug though, so I'm not sure Fixes
is approproate. Correct me if I'm wrong.
Quoted 181 lines
Quoted 96 lines
Quoted 84 lines
Signed-off-by Yuya Kusakabe <[email protected]>---
v5:
-page_to_skb(): copy vnet header ifhdr_validwithout checking metasize.
-receive_small(): do not copy vnet header ifxdp_progis availavle.
-__virtnet_xdp_xmit_one(): remove thexdp_set_data_meta_invalid().
- improve comments.
v4:
- improve commit message
v3:
- fix preserve the vnet header inreceive_small().
v2:
- keep copy untouched inpage_to_skb().
- preserve the vnet header inreceive_small().
- fix indentation.
---
drivers/net/virtio_net.c| 54 ++++++++++++++++++++++++----------------1 file changed, 33 insertions(+), 21 deletions(-)drivers/net/virtio_net.c +1
-1@@ -371,7 +371,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, struct receive_queue *rq, struct page *page, unsigned int offset, unsigned int len, unsigned int truesize, - bool hdr_valid) + bool hdr_valid, unsigned int metasize) { struct sk_buff *skb; struct virtio_net_hdr_mrg_rxbuf *hdr;@@ -393,6 +393,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,else
hdr_padded_len=sizeof(struct padded_vnet_hdr);
+ /*hdr_validmeans no XDP, so we can copy the vnet header */ if (hdr_valid)
memcpy(hdr, p, hdr_len);
@@ -405,6 +406,11 @@ static structsk_buff*page_to_skb(structvirtnet_info*vi, copy =skb_tailroom(skb);
skb_put_data(skb, p, copy);
+ if (metasize) {
+__skb_pull(skb, metasize);
+skb_metadata_set(skb, metasize);
+ }
+
len -= copy;
offset += copy;
@@ -450,10 +456,6 @@ static int __virtnet_xdp_xmit_one(structvirtnet_info*vi, structvirtio_net_hdr_mrg_rxbuf*hdr;
int err;
- /* virtqueue want to use data area in-front of packet */
- if (unlikely(xdpf->metasize > 0))
- return-EOPNOTSUPP;
-
if (unlikely(xdpf->headroom < vi->hdr_len))
return-EOVERFLOW;
@@ -644,6 +646,7 @@ static structsk_buff*receive_small(structnet_device*dev, unsigned int delta = 0;
struct page*xdp_page;
int err;
+ unsigned int metasize = 0;
len -=vi->hdr_len;
stats->bytes+= len;drivers/net/virtio_net.c +2
-1@@ -683,8 +686,8 @@ static struct sk_buff *receive_small(struct net_device *dev, xdp.data_hard_start = buf + VIRTNET_RX_PAD + vi->hdr_len; xdp.data = xdp.data_hard_start + xdp_headroom; - xdp_set_data_meta_invalid(&xdp); xdp.data_end = xdp.data + len; + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; orig_data = xdp.data; act = bpf_prog_run_xdp(xdp_prog, &xdp);@@ -695,6 +698,7 @@ static struct sk_buff *receive_small(struct net_device *dev, /* Recalculate length in case bpf program changed it */ delta = orig_data - xdp.data; len = xdp.data_end - xdp.data; + metasize = xdp.data - xdp.data_meta; break;case
XDP_TX:
stats->xdp_tx++;drivers/net/virtio_net.c static struct sk_buff *receive_small(struct net_device *dev, · line 745
Quoted 8 lines
drivers/net/virtio_net.c +1
-1@@ -735,11 +739,14 @@ static struct sk_buff *receive_small(struct net_device *dev, } skb_reserve(skb, headroom - delta); skb_put(skb, len); - if (!delta) { + if (!xdp_prog) { buf += header_offset; memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len); } /* keep zeroed vnet hdr since packet was changed by bpf */I prefer to make this an independent patch and cc stable.
Quoted 3 lines
Other looks good.
Thanks
I see. So I need to revert to delta from
xdp_prog?Thank you.
Quoted 77 lines
Quoted 76 lines
drivers/net/virtio_net.c +2
-0+ if (metasize) + skb_metadata_set(skb, metasize); +err:
return skb;
@@ -760,8 +767,8 @@ static structsk_buff*receive_big(structnet_device*dev, structvirtnet_rq_stats*stats)
{
struct page *page = buf;
- structsk_buff*skb = page_to_skb(vi, rq, page, 0, len,
-PAGE_SIZE, true);
+ structsk_buff*skb =
+page_to_skb(vi, rq, page, 0, len, PAGE_SIZE, true, 0);stats->bytes+= len -vi->hdr_len;
if (unlikely(!skb))drivers/net/virtio_net.c +2
-1@@ -793,6 +800,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, unsigned int truesize; unsigned int headroom = mergeable_ctx_to_headroom(ctx); int err; + unsigned int metasize = 0; head_skb = NULL; stats->bytes += len - vi->hdr_len;@@ -839,8 +847,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, data = page_address(xdp_page) + offset; xdp.data_hard_start = data - VIRTIO_XDP_HEADROOM + vi->hdr_len; xdp.data = data + vi->hdr_len; - xdp_set_data_meta_invalid(&xdp); xdp.data_end = xdp.data + (len - vi->hdr_len); + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; act = bpf_prog_run_xdp(xdp_prog, &xdp);@@ -848,24 +856,27 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, switch (act) {case
XDP_PASS:
+ metasize = xdp.data - xdp.data_meta;
+
/* recalculate offset to account for any header - * adjustments. Note other cases do not build an - * skb and avoid using offset
+ * adjustments and minus the metasize to copy the + * metadata inpage_to_skb(). Note other cases do not + * build an skb and avoid using offset
*/
- offset = xdp.data -
-page_address(xdp_page)-vi->hdr_len; + offset = xdp.data -page_address(xdp_page)- +vi->hdr_len- metasize;
- /* recalculate len if xdp.data or xdp.data_end were - * adjusted
+ /* recalculate len if xdp.data, xdp.data_end or + * xdp.data_meta were adjusted
*/
- len = xdp.data_end - xdp.data +vi->hdr_len; + len = xdp.data_end - xdp.data +vi->hdr_len+ metasize; /* We can only create skb based onxdp_page. */ if (unlikely(xdp_page != page)) {rcu_read_unlock();
put_page(page);
-head_skb= page_to_skb(vi, rq,xdp_page, - offset, len,
-PAGE_SIZE, false);
+head_skb= page_to_skb(vi, rq,xdp_page, offset, + len,PAGE_SIZE, false,
+ metasize);
returnhead_skb;
}
break;drivers/net/virtio_net.c +2
-0@@ -921,7 +932,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, goto err_skb; } - head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog); + head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog, + metasize); curr_skb = head_skb; if (unlikely(!curr_skb))
Michael S. Tsirkin
maintainerQuoted 125 lines · Yuya Kusakabe
Quoted 7 lines · Jason Wang
Quoted 5 lines · Yuya Kusakabe
Implement support for transferring XDP meta data into skb for
virtio_netdriver; before calling into the program, xdp.data_meta points to xdp.data, where on program return with pass verdict, we call intoskb_metadata_set().Quoted 7 lines
Quoted 4 lines
Tested with the script at
https://github.com/higebu/virtio_net-xdp-metadata-test.Fixes de8f3a83b0a0 ("bpf: add meta pointer for direct access")I'm not sure this is correct since virtio-net claims to not support metadata by calling
xdp_set_data_meta_invalid()?
virtio_netdoesn't support by callingxdp_set_data_meta_invalid()for now.https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/tree/drivers/net/virtio_net.c?id=e42da4c62abb547d9c9138e0e7fcd1f36057b5e8#n686
https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/tree/drivers/net/virtio_net.c?id=e42da4c62abb547d9c9138e0e7fcd1f36057b5e8#n842And
xdp_set_data_meta_invalid()are added byde8f3a83b0a0.$ git blame ./drivers/net/virtio_net.c | grep xdp_set_data_meta_invalid de8f3a83b0a0f (Daniel Borkmann 2017-09-25 02:25:51 +0200 686) xdp_set_data_meta_invalid(&xdp); de8f3a83b0a0f (Daniel Borkmann 2017-09-25 02:25:51 +0200 842) xdp_set_data_meta_invalid(&xdp);So I added
Fixes: de8f3a83b0a0 ("bpf: add meta pointer for direct access")to the comment.Quoted 96 lines
Quoted 84 lines
Signed-off-by Yuya Kusakabe <[email protected]>---
v5:
-page_to_skb(): copy vnet header ifhdr_validwithout checking metasize.
-receive_small(): do not copy vnet header ifxdp_progis availavle.
-__virtnet_xdp_xmit_one(): remove thexdp_set_data_meta_invalid().
- improve comments.
v4:
- improve commit message
v3:
- fix preserve the vnet header inreceive_small().
v2:
- keep copy untouched inpage_to_skb().
- preserve the vnet header inreceive_small().
- fix indentation.
---
drivers/net/virtio_net.c| 54 ++++++++++++++++++++++++----------------1 file changed, 33 insertions(+), 21 deletions(-)drivers/net/virtio_net.c +1
-1@@ -371,7 +371,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, struct receive_queue *rq, struct page *page, unsigned int offset, unsigned int len, unsigned int truesize, - bool hdr_valid) + bool hdr_valid, unsigned int metasize) { struct sk_buff *skb; struct virtio_net_hdr_mrg_rxbuf *hdr;@@ -393,6 +393,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,else
hdr_padded_len=sizeof(struct padded_vnet_hdr);
+ /*hdr_validmeans no XDP, so we can copy the vnet header */ if (hdr_valid)
memcpy(hdr, p, hdr_len);
@@ -405,6 +406,11 @@ static structsk_buff*page_to_skb(structvirtnet_info*vi, copy =skb_tailroom(skb);
skb_put_data(skb, p, copy);
+ if (metasize) {
+__skb_pull(skb, metasize);
+skb_metadata_set(skb, metasize);
+ }
+
len -= copy;
offset += copy;
@@ -450,10 +456,6 @@ static int __virtnet_xdp_xmit_one(structvirtnet_info*vi, structvirtio_net_hdr_mrg_rxbuf*hdr;
int err;
- /* virtqueue want to use data area in-front of packet */
- if (unlikely(xdpf->metasize > 0))
- return-EOPNOTSUPP;
-
if (unlikely(xdpf->headroom < vi->hdr_len))
return-EOVERFLOW;
@@ -644,6 +646,7 @@ static structsk_buff*receive_small(structnet_device*dev, unsigned int delta = 0;
struct page*xdp_page;
int err;
+ unsigned int metasize = 0;
len -=vi->hdr_len;
stats->bytes+= len;drivers/net/virtio_net.c +2
-1@@ -683,8 +686,8 @@ static struct sk_buff *receive_small(struct net_device *dev, xdp.data_hard_start = buf + VIRTNET_RX_PAD + vi->hdr_len; xdp.data = xdp.data_hard_start + xdp_headroom; - xdp_set_data_meta_invalid(&xdp); xdp.data_end = xdp.data + len; + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; orig_data = xdp.data; act = bpf_prog_run_xdp(xdp_prog, &xdp);@@ -695,6 +698,7 @@ static struct sk_buff *receive_small(struct net_device *dev, /* Recalculate length in case bpf program changed it */ delta = orig_data - xdp.data; len = xdp.data_end - xdp.data; + metasize = xdp.data - xdp.data_meta; break;case
XDP_TX:
stats->xdp_tx++;drivers/net/virtio_net.c static struct sk_buff *receive_small(struct net_device *dev, · line 745
Quoted 8 lines
drivers/net/virtio_net.c +1
-1@@ -735,11 +739,14 @@ static struct sk_buff *receive_small(struct net_device *dev, } skb_reserve(skb, headroom - delta); skb_put(skb, len); - if (!delta) { + if (!xdp_prog) { buf += header_offset; memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len); } /* keep zeroed vnet hdr since packet was changed by bpf */I prefer to make this an independent patch and cc stable.
Quoted 7 lines
Quoted 3 lines
Other looks good.
Thanks
I see. So I need to revert to delta from
xdp_prog?Thank you.
So maybe send a 2 patch series: 1/2 is this chunk with the appropriate description. Actually for netdev David prefers that people do not cc stable directly, just include Fixes tag and mention in the commit log it's also needed for stable. Patch 2/2 is the rest handling metadata.
Quoted 77 lines
Quoted 77 lines
Quoted 76 lines
drivers/net/virtio_net.c +2
-0+ if (metasize) + skb_metadata_set(skb, metasize); +err:
return skb;
@@ -760,8 +767,8 @@ static structsk_buff*receive_big(structnet_device*dev, structvirtnet_rq_stats*stats)
{
struct page *page = buf;
- structsk_buff*skb = page_to_skb(vi, rq, page, 0, len,
-PAGE_SIZE, true);
+ structsk_buff*skb =
+page_to_skb(vi, rq, page, 0, len, PAGE_SIZE, true, 0);stats->bytes+= len -vi->hdr_len;
if (unlikely(!skb))drivers/net/virtio_net.c +2
-1@@ -793,6 +800,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, unsigned int truesize; unsigned int headroom = mergeable_ctx_to_headroom(ctx); int err; + unsigned int metasize = 0; head_skb = NULL; stats->bytes += len - vi->hdr_len;@@ -839,8 +847,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, data = page_address(xdp_page) + offset; xdp.data_hard_start = data - VIRTIO_XDP_HEADROOM + vi->hdr_len; xdp.data = data + vi->hdr_len; - xdp_set_data_meta_invalid(&xdp); xdp.data_end = xdp.data + (len - vi->hdr_len); + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; act = bpf_prog_run_xdp(xdp_prog, &xdp);@@ -848,24 +856,27 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, switch (act) {case
XDP_PASS:
+ metasize = xdp.data - xdp.data_meta;
+
/* recalculate offset to account for any header - * adjustments. Note other cases do not build an - * skb and avoid using offset
+ * adjustments and minus the metasize to copy the + * metadata inpage_to_skb(). Note other cases do not + * build an skb and avoid using offset
*/
- offset = xdp.data -
-page_address(xdp_page)-vi->hdr_len; + offset = xdp.data -page_address(xdp_page)- +vi->hdr_len- metasize;
- /* recalculate len if xdp.data or xdp.data_end were - * adjusted
+ /* recalculate len if xdp.data, xdp.data_end or + * xdp.data_meta were adjusted
*/
- len = xdp.data_end - xdp.data +vi->hdr_len; + len = xdp.data_end - xdp.data +vi->hdr_len+ metasize; /* We can only create skb based onxdp_page. */ if (unlikely(xdp_page != page)) {rcu_read_unlock();
put_page(page);
-head_skb= page_to_skb(vi, rq,xdp_page, - offset, len,
-PAGE_SIZE, false);
+head_skb= page_to_skb(vi, rq,xdp_page, offset, + len,PAGE_SIZE, false,
+ metasize);
returnhead_skb;
}
break;drivers/net/virtio_net.c +2
-0@@ -921,7 +932,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, goto err_skb; } - head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog); + head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog, + metasize); curr_skb = head_skb; if (unlikely(!curr_skb))
Quoted 121 lines · Michael S. Tsirkin
Quoted 120 lines · Yuya Kusakabe
Quoted 11 lines · Jason Wang
Quoted 9 lines · Yuya Kusakabe
Implement support for transferring XDP meta data into skb for
virtio_netdriver; before calling into the program, xdp.data_meta points to xdp.data, where on program return with pass verdict, we call intoskb_metadata_set().Tested with the script at
https://github.com/higebu/virtio_net-xdp-metadata-test.Fixes de8f3a83b0a0 ("bpf: add meta pointer for direct access")I'm not sure this is correct since virtio-net claims to not support metadata by calling
xdp_set_data_meta_invalid()?
virtio_netdoesn't support by callingxdp_set_data_meta_invalid()for now.https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/tree/drivers/net/virtio_net.c?id=e42da4c62abb547d9c9138e0e7fcd1f36057b5e8#n686
https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/tree/drivers/net/virtio_net.c?id=e42da4c62abb547d9c9138e0e7fcd1f36057b5e8#n842And
xdp_set_data_meta_invalid()are added byde8f3a83b0a0.$ git blame ./drivers/net/virtio_net.c | grep xdp_set_data_meta_invalid de8f3a83b0a0f (Daniel Borkmann 2017-09-25 02:25:51 +0200 686) xdp_set_data_meta_invalid(&xdp); de8f3a83b0a0f (Daniel Borkmann 2017-09-25 02:25:51 +0200 842) xdp_set_data_meta_invalid(&xdp);So I added
Fixes: de8f3a83b0a0 ("bpf: add meta pointer for direct access")to the comment.Quoted 95 lines
Quoted 82 lines
Signed-off-by Yuya Kusakabe<[email protected]>---
v5:
-page_to_skb(): copy vnet header ifhdr_validwithout checking metasize.
-receive_small(): do not copy vnet header ifxdp_progis availavle.
-__virtnet_xdp_xmit_one(): remove thexdp_set_data_meta_invalid().
- improve comments.
v4:
- improve commit message
v3:
- fix preserve the vnet header inreceive_small().
v2:
- keep copy untouched inpage_to_skb().
- preserve the vnet header inreceive_small().
- fix indentation.--- drivers/net/virtio_net.c | 54 ++++++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 21 deletions(-)drivers/net/virtio_net.c +8
-5@@ -371,7 +371,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, struct receive_queue *rq, struct page *page, unsigned int offset, unsigned int len, unsigned int truesize, - bool hdr_valid) + bool hdr_valid, unsigned int metasize) { struct sk_buff *skb; struct virtio_net_hdr_mrg_rxbuf *hdr;@@ -393,6 +393,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, else hdr_padded_len = sizeof(struct padded_vnet_hdr); + /* hdr_valid means no XDP, so we can copy the vnet header */ if (hdr_valid) memcpy(hdr, p, hdr_len); @@ -405,6 +406,11 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, copy = skb_tailroom(skb); skb_put_data(skb, p, copy); + if (metasize) { + __skb_pull(skb, metasize); + skb_metadata_set(skb, metasize); + } + len -= copy; offset += copy; @@ -450,10 +456,6 @@ static int __virtnet_xdp_xmit_one(struct virtnet_info *vi, struct virtio_net_hdr_mrg_rxbuf *hdr; int err; - /* virtqueue want to use data area in-front of packet */ - if (unlikely(xdpf->metasize > 0)) - return -EOPNOTSUPP; - if (unlikely(xdpf->headroom < vi->hdr_len)) return -EOVERFLOW; @@ -644,6 +646,7 @@ static struct sk_buff *receive_small(struct net_device *dev, unsigned int delta = 0; struct page *xdp_page; int err; + unsigned int metasize = 0; len -= vi->hdr_len; stats->bytes += len;@@ -683,8 +686,8 @@ static struct sk_buff *receive_small(struct net_device *dev, xdp.data_hard_start = buf + VIRTNET_RX_PAD + vi->hdr_len; xdp.data = xdp.data_hard_start + xdp_headroom; - xdp_set_data_meta_invalid(&xdp); xdp.data_end = xdp.data + len; + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; orig_data = xdp.data; act = bpf_prog_run_xdp(xdp_prog, &xdp);@@ -695,6 +698,7 @@ static struct sk_buff *receive_small(struct net_device *dev, /* Recalculate length in case bpf program changed it */ delta = orig_data - xdp.data; len = xdp.data_end - xdp.data; + metasize = xdp.data - xdp.data_meta; break; case XDP_TX: stats->xdp_tx++;drivers/net/virtio_net.c static struct sk_buff *receive_small(struct net_device *dev, · line 745
Quoted 8 lines
drivers/net/virtio_net.c +1
-1@@ -735,11 +739,14 @@ static struct sk_buff *receive_small(struct net_device *dev, } skb_reserve(skb, headroom - delta); skb_put(skb, len); - if (!delta) { + if (!xdp_prog) { buf += header_offset; memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len); } /* keep zeroed vnet hdr since packet was changed by bpf */I prefer to make this an independent patch and cc stable.
Other looks good.
Thanks
Quoted 8 lines
Quoted 3 lines
I see. So I need to revert to delta from
xdp_prog?Thank you.
So maybe send a 2 patch series: 1/2 is this chunk with the appropriate description. Actually for netdev David prefers that people do not cc stable directly, just include Fixes tag and mention in the commit log it's also needed for stable. Patch 2/2 is the rest handling metadata.
+1
Thanks
Quoted 133 lines · Jason Wang
Quoted 121 lines · Michael S. Tsirkin
Quoted 120 lines · Yuya Kusakabe
Quoted 11 lines · Jason Wang
Quoted 9 lines · Yuya Kusakabe
Implement support for transferring XDP meta data into skb for
virtio_netdriver; before calling into the program, xdp.data_meta points to xdp.data, where on program return with pass verdict, we call intoskb_metadata_set().Tested with the script at
https://github.com/higebu/virtio_net-xdp-metadata-test.Fixes de8f3a83b0a0 ("bpf: add meta pointer for direct access")I'm not sure this is correct since virtio-net claims to not support metadata by calling
xdp_set_data_meta_invalid()?
virtio_netdoesn't support by callingxdp_set_data_meta_invalid()for now.https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/tree/drivers/net/virtio_net.c?id=e42da4c62abb547d9c9138e0e7fcd1f36057b5e8#n686
https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/tree/drivers/net/virtio_net.c?id=e42da4c62abb547d9c9138e0e7fcd1f36057b5e8#n842And
xdp_set_data_meta_invalid()are added byde8f3a83b0a0.$ git blame ./drivers/net/virtio_net.c | grep xdp_set_data_meta_invalid de8f3a83b0a0f (Daniel Borkmann 2017-09-25 02:25:51 +0200 686) xdp_set_data_meta_invalid(&xdp); de8f3a83b0a0f (Daniel Borkmann 2017-09-25 02:25:51 +0200 842) xdp_set_data_meta_invalid(&xdp);So I added
Fixes: de8f3a83b0a0 ("bpf: add meta pointer for direct access")to the comment.Quoted 95 lines
Quoted 82 lines
Signed-off-by Yuya Kusakabe<[email protected]>---
v5:
-page_to_skb(): copy vnet header ifhdr_validwithout checking metasize.
-receive_small(): do not copy vnet header ifxdp_progis availavle.
-__virtnet_xdp_xmit_one(): remove thexdp_set_data_meta_invalid().
- improve comments.
v4:
- improve commit message
v3:
- fix preserve the vnet header inreceive_small().
v2:
- keep copy untouched inpage_to_skb().
- preserve the vnet header inreceive_small().
- fix indentation.
---
drivers/net/virtio_net.c| 54 ++++++++++++++++++++++++----------------1 file changed, 33 insertions(+), 21 deletions(-)drivers/net/virtio_net.c +1
-1@@ -371,7 +371,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, struct receive_queue *rq, struct page *page, unsigned int offset, unsigned int len, unsigned int truesize, - bool hdr_valid) + bool hdr_valid, unsigned int metasize) { struct sk_buff *skb; struct virtio_net_hdr_mrg_rxbuf *hdr;@@ -393,6 +393,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,else
hdr_padded_len=sizeof(struct padded_vnet_hdr);
+ /*hdr_validmeans no XDP, so we can copy the vnet header */ if (hdr_valid)
memcpy(hdr, p, hdr_len);
@@ -405,6 +406,11 @@ static structsk_buff*page_to_skb(structvirtnet_info*vi, copy =skb_tailroom(skb);
skb_put_data(skb, p, copy);
+ if (metasize) {
+__skb_pull(skb, metasize);
+skb_metadata_set(skb, metasize);
+ }
+
len -= copy;
offset += copy;
@@ -450,10 +456,6 @@ static int __virtnet_xdp_xmit_one(structvirtnet_info*vi, structvirtio_net_hdr_mrg_rxbuf*hdr;
int err;
- /* virtqueue want to use data area in-front of packet */
- if (unlikely(xdpf->metasize > 0))
- return-EOPNOTSUPP;
-
if (unlikely(xdpf->headroom < vi->hdr_len))
return-EOVERFLOW;
@@ -644,6 +646,7 @@ static structsk_buff*receive_small(structnet_device*dev, unsigned int delta = 0;
struct page*xdp_page;
int err;
+ unsigned int metasize = 0;
len -=vi->hdr_len;
stats->bytes+= len;drivers/net/virtio_net.c +2
-1@@ -683,8 +686,8 @@ static struct sk_buff *receive_small(struct net_device *dev, xdp.data_hard_start = buf + VIRTNET_RX_PAD + vi->hdr_len; xdp.data = xdp.data_hard_start + xdp_headroom; - xdp_set_data_meta_invalid(&xdp); xdp.data_end = xdp.data + len; + xdp.data_meta = xdp.data; xdp.rxq = &rq->xdp_rxq; orig_data = xdp.data; act = bpf_prog_run_xdp(xdp_prog, &xdp);@@ -695,6 +698,7 @@ static struct sk_buff *receive_small(struct net_device *dev, /* Recalculate length in case bpf program changed it */ delta = orig_data - xdp.data; len = xdp.data_end - xdp.data; + metasize = xdp.data - xdp.data_meta; break;case
XDP_TX:
stats->xdp_tx++;drivers/net/virtio_net.c static struct sk_buff *receive_small(struct net_device *dev, · line 745
Quoted 8 lines
drivers/net/virtio_net.c +1
-1@@ -735,11 +739,14 @@ static struct sk_buff *receive_small(struct net_device *dev, } skb_reserve(skb, headroom - delta); skb_put(skb, len); - if (!delta) { + if (!xdp_prog) { buf += header_offset; memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len); } /* keep zeroed vnet hdr since packet was changed by bpf */I prefer to make this an independent patch and cc stable.
Other looks good.
Thanks
Quoted 8 lines
Quoted 3 lines
I see. So I need to revert to delta from
xdp_prog?Thank you.
So maybe send a 2 patch series: 1/2 is this chunk with the appropriate description. Actually for netdev David prefers that people do not cc stable directly, just include Fixes tag and mention in the commit log it's also needed for stable. Patch 2/2 is the rest handling metadata.
Quoted 5 lines
+1
Thanks
Thank you for the detailed explanation. I will make a 2 patch series.
Michael S. Tsirkin
maintainerdrivers/net/virtio_net.c static struct sk_buff *receive_mergeable(struct net_device *dev, · line 876
Quoted 12 lines · Yuya Kusakabe
drivers/net/virtio_net.c +3
-2@@ -852,8 +868,9 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,* adjustments. Note other cases do not build an * skb and avoid using offset */ - offset = xdp.data - -page_address (xdp_page) - vi->hdr_len; + metasize = xdp.data - xdp.data_meta; + offset = xdp.data -page_address (xdp_page) - + vi->hdr_len - metasize;/* recalculate len if xdp.data or xdp.data_end were * adjusted
Tricky to get one's head around.
Can you pls update the comment above to document the new math?
Signature · 1 line
MST
Quoted 11 lines · Michael S. Tsirkin
Quoted 10 lines · Yuya Kusakabe
drivers/net/virtio_net.c +3
-2@@ -852,8 +868,9 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,* adjustments. Note other cases do not build an * skb and avoid using offset */ - offset = xdp.data - -page_address (xdp_page) - vi->hdr_len; + metasize = xdp.data - xdp.data_meta; + offset = xdp.data -page_address (xdp_page) - + vi->hdr_len - metasize;
Quoted 6 lines
drivers/net/virtio_net.c static struct sk_buff *receive_mergeable(struct net_device *dev, · line 876
Quoted 2 lines
drivers/net/virtio_net.c +0
-0/* recalculate len if xdp.data or xdp.data_end were * adjusted Tricky to get one's head around.
Can you pls update the comment above to document the new math?
Thank you for your review.
I will update the comment on next patch.