xen networking: add basic XDP support for xen-netfront
PATCH v8 1/3 xen networking: add basic XDP support for xen-netfront
The patch adds a basic XDP processing to xen-netfront driver.
We ran an XDP program for an RX response received from netback driver. Also we request xen-netback to adjust data offset for bpf_xdp_adjust_head() header space for custom headers.
synchronization between frontend and backend parts is done by using xenbus state switching:
Reconfiguring -> Reconfigured- > Connected
UDP packets drop rate using xdp program is around 310 kpps using ./pktgen_sample04_many_flows.sh and 160 kpps without the patch.
v8:
- add PAGE_POOL config dependency
- keep the state of XDP processing in netfront_xdp_enabled
- fixed allocator type in xdp_rxq_info_reg_mem_model()
- minor cleanups in xen-netback
v7:
- use page_pool_dev_alloc_pages() on page allocation
- remove the leftover break statement from netback_changed
v6:
- added the missing SOB line
- fixed subject
v5:
- split netfront/netback changes
- added a sync point between backend/frontend on switching to XDP
- added pagepool API
v4:
- added verbose patch descriprion
- don't expose the XDP headroom offset to the domU guest
- add a modparam to netback to toggle XDP offset
- don't process jumbo frames for now
v3:
- added XDP_TX support (tested with xdping echoserver)
- added XDP_REDIRECT support (tested with modified xdp_redirect_kern)
- moved xdp negotiation to xen-netback
v2:
- avoid data copying while passing to XDP
- tell xen-netback that we need the headroom space
--- drivers/net/xen-netfront.c | 311 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 307 insertions(+), 4 deletions(-)
drivers/net/xen-netfront.c +307 -4
@@ -44,6 +44,9 @@#include <linux/mm.h> #include <linux/slab.h> #include <net/ip.h> +#include <linux/bpf.h> +#include <net/page_pool.h> +#include <linux/bpf_trace.h> #include <xen/xen.h> #include <xen/xenbus.h>
@@ -102,6 +105,8 @@ struct netfront_queue {char name[QUEUE_NAME_SIZE ];/* DEVNAME-qN */ struct netfront_info *info; +struct bpf_prog __rcu *xdp_prog; +struct napi_struct napi;/* Split event channels support, tx_* == rx_* when using
@@ -144,6 +149,9 @@ struct netfront_queue {struct sk_buff *rx_skbs[NET_RX_RING_SIZE ];grant_ref_t gref_rx_head;grant_ref_t grant_rx_ref[NET_RX_RING_SIZE ]; + +struct page_pool *page_pool; +struct xdp_rxq_info xdp_rxq; };struct netfront_info {
@@ -159,6 +167,10 @@ struct netfront_info {struct netfront_stats __percpu *rx_stats;struct netfront_stats __percpu *tx_stats; +/* XDP state */ +bool netback_has_xdp_headroom; +bool netfront_xdp_enabled; +atomic_t rx_gso_checksum_fixup; };
@@ -167,6 +179,9 @@ struct netfront_rx_info {struct xen_netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX -1 ]; }; +static int xennet_xdp_xmit (struct net_device *dev,int n, +struct xdp_frame **frames,u32 flags); +static void skb_entry_set_link (union skb_entry *list,unsigned short id) { list->link = id;
@@ -265,8 +280,8 @@ static struct sk_buff *xennet_alloc_one_rx_buffer(struct netfront_queue *queue)if (unlikely (!skb))return NULL ; - page =alloc_page (GFP_ATOMIC | __GFP_NOWARN); -if (!page) { + page =page_pool_dev_alloc_pages (queue->page_pool); +if (unlikely (!page)) {kfree_skb (skb);return NULL ; }
@@ -778,6 +793,53 @@ static int xennet_get_extras(struct netfront_queue *queue,return err; } +u32 xennet_run_xdp (struct netfront_queue *queue,struct page *pdata, +struct xen_netif_rx_response *rx,struct bpf_prog *prog, +struct xdp_buff *xdp) +{ +struct xdp_frame *xdpf; +u32 len = rx->status; +u32 act =XDP_PASS ; +int err; + + xdp->data_hard_start =page_address (pdata); + xdp->data = xdp->data_hard_start +XDP_PACKET_HEADROOM ; +xdp_set_data_meta_invalid (xdp); + xdp->data_end = xdp->data + len; + xdp->rxq = &queue->xdp_rxq; + xdp->handle =0 ; + + act =bpf_prog_run_xdp (prog, xdp); +switch (act) { +case XDP_TX : +get_page (pdata); + xdpf =convert_to_xdp_frame (xdp); + err =xennet_xdp_xmit (queue->info->netdev,1 , &xdpf,0 ); +if (unlikely (err <0 )) +trace_xdp_exception (queue->info->netdev, prog, act); +break ; +case XDP_REDIRECT : +get_page (pdata); + err =xdp_do_redirect (queue->info->netdev, xdp, prog); +if (unlikely (err)) +trace_xdp_exception (queue->info->netdev, prog, act); +xdp_do_flush (); +break ; +case XDP_PASS : +case XDP_DROP : +break ; + +case XDP_ABORTED : +trace_xdp_exception (queue->info->netdev, prog, act); +break ; + +default : +bpf_warn_invalid_xdp_action (act); + } + +return act; +} +static int xennet_get_responses (struct netfront_queue *queue,struct netfront_rx_info *rinfo,RING_IDX rp,struct sk_buff_head *list)
@@ -792,6 +854,9 @@ static int xennet_get_responses(struct netfront_queue *queue,int slots =1 ;int err =0 ;unsigned long ret; +struct bpf_prog *xdp_prog; +struct xdp_buff xdp; +u32 verdict;if (rx->flags & XEN_NETRXF_extra_info) { err =xennet_get_extras (queue, extras, rp);
@@ -827,9 +892,20 @@ static int xennet_get_responses(struct netfront_queue *queue,gnttab_release_grant_reference (&queue->gref_rx_head, ref); -__skb_queue_tail (list, skb); - +rcu_read_lock (); + xdp_prog =rcu_dereference (queue->xdp_prog); +if (xdp_prog && !(rx->flags & XEN_NETRXF_more_data)) { +/* currently only a single page contains data */ +WARN_ON_ONCE (skb_shinfo (skb)->nr_frags !=1 ); + verdict =xennet_run_xdp (queue, +skb_frag_page (&skb_shinfo (skb)->frags[0 ]), + rx, xdp_prog, &xdp); +if (verdict !=XDP_PASS ) + err = -EINVAL ; + } +rcu_read_unlock (); next: +__skb_queue_tail (list, skb);if (!(rx->flags & XEN_NETRXF_more_data))break ;
@@ -997,6 +1073,7 @@ static int xennet_poll(struct napi_struct *napi, int budget)struct sk_buff_head rxq;struct sk_buff_head errq;struct sk_buff_head tmpq; +struct bpf_prog *xdp_prog;int err;spin_lock (&queue->rx_lock);
@@ -1014,6 +1091,12 @@ static int xennet_poll(struct napi_struct *napi, int budget)memcpy (rx,RING_GET_RESPONSE (&queue->rx, i),sizeof (*rx));memset (extras,0 ,sizeof (rinfo.extras)); +rcu_read_lock (); + xdp_prog =rcu_dereference (queue->xdp_prog); +if (xdp_prog) + rx->offset =XDP_PACKET_HEADROOM ; +rcu_read_unlock (); + err =xennet_get_responses (queue, &rinfo, rp, &tmpq);if (unlikely (err)) {
@@ -1261,6 +1344,157 @@ static void xennet_poll_controller(struct net_device *dev) }#endif +#define NETBACK_XDP_HEADROOM_DISABLE 0 +#define NETBACK_XDP_HEADROOM_ENABLE 1 + +static int talk_to_netback_xdp (struct netfront_info *np,int xdp) +{ +int err; + + err =xenbus_printf (XBT_NIL , np->xbdev->nodename, +"feature-xdp" ,"%u" , xdp); +if (err) +pr_debug ("Error writing feature-xdp\n" ); + +return err; +} + +static int xennet_xdp_set (struct net_device *dev,struct bpf_prog *prog, +struct netlink_ext_ack *extack) +{ +struct netfront_info *np =netdev_priv (dev); +struct bpf_prog *old_prog; +unsigned int i, err; +unsigned long int max_mtu =XEN_PAGE_SIZE -XDP_PACKET_HEADROOM ; + +if (dev->mtu > max_mtu) { +netdev_warn (dev,"XDP requires MTU less than %lu\n" , max_mtu); +return -EINVAL ; + } + +if (!np->netback_has_xdp_headroom) +return 0 ; + + old_prog =rtnl_dereference (np->queues[0 ].xdp_prog); +if (!old_prog && !prog) +return 0 ; + +if (prog) +bpf_prog_add (prog, dev->real_num_tx_queues); + +for (i =0 ; i < dev->real_num_tx_queues; ++i) +rcu_assign_pointer (np->queues[i].xdp_prog, prog); + +if (old_prog) +for (i =0 ; i < dev->real_num_tx_queues; ++i) +bpf_prog_put (old_prog); + +xenbus_switch_state (np->xbdev, XenbusStateReconfiguring); + + err =talk_to_netback_xdp (np, prog ?NETBACK_XDP_HEADROOM_ENABLE : +NETBACK_XDP_HEADROOM_DISABLE ); +if (err) +return err; + +/* avoid race with XDP headroom adjustment */ +wait_event (module_wq, +xenbus_read_driver_state (np->xbdev->otherend) == + XenbusStateReconfigured); + np->netfront_xdp_enabled = true; +xenbus_switch_state (np->xbdev, XenbusStateConnected); + +return 0 ; +} + +static u32 xennet_xdp_query (struct net_device *dev) +{ +struct netfront_info *np =netdev_priv (dev); +unsigned int num_queues = dev->real_num_tx_queues; +unsigned int i; +struct netfront_queue *queue; +const struct bpf_prog *xdp_prog; + +for (i =0 ; i < num_queues; ++i) { + queue = &np->queues[i]; + xdp_prog =rtnl_dereference (queue->xdp_prog); +if (xdp_prog) +return xdp_prog->aux->id; + } + +return 0 ; +} + +static int xennet_xdp (struct net_device *dev,struct netdev_bpf *xdp) +{ +switch (xdp->command) { +case XDP_SETUP_PROG : +return xennet_xdp_set (dev, xdp->prog, xdp->extack); +case XDP_QUERY_PROG : + xdp->prog_id =xennet_xdp_query (dev); +return 0 ; +default : +return -EINVAL ; + } +} + +static int xennet_xdp_xmit_one (struct net_device *dev,struct xdp_frame *xdpf) +{ +struct netfront_info *np =netdev_priv (dev); +struct netfront_stats *tx_stats =this_cpu_ptr (np->tx_stats); +struct netfront_queue *queue =NULL ; +unsigned int num_queues = dev->real_num_tx_queues; +unsigned long flags; +int notify; +struct xen_netif_tx_request *tx; + + queue = &np->queues[smp_processor_id () % num_queues]; + +spin_lock_irqsave (&queue->tx_lock, flags); + + tx =xennet_make_first_txreq (queue,NULL , +virt_to_page (xdpf->data), +offset_in_page (xdpf->data), + xdpf->len); + +RING_PUSH_REQUESTS_AND_CHECK_NOTIFY (&queue->tx, notify); +if (notify) +notify_remote_via_irq (queue->tx_irq); + +u64_stats_update_begin (&tx_stats->syncp); + tx_stats->bytes += xdpf->len; + tx_stats->packets++; +u64_stats_update_end (&tx_stats->syncp); + +xennet_tx_buf_gc (queue); + +spin_unlock_irqrestore (&queue->tx_lock, flags); +return 0 ; +} + +static int xennet_xdp_xmit (struct net_device *dev,int n, +struct xdp_frame **frames,u32 flags) +{ +int drops =0 ; +int i, err; + +if (unlikely (flags & ~XDP_XMIT_FLAGS_MASK )) +return -EINVAL ; + +for (i =0 ; i < n; i++) { +struct xdp_frame *xdpf = frames[i]; + +if (!xdpf) +continue ; + err =xennet_xdp_xmit_one (dev, xdpf); +if (err) { +xdp_return_frame_rx_napi (xdpf); + drops++; + } + } + +return n - drops; +} +static const struct net_device_ops xennet_netdev_ops = { .ndo_open = xennet_open, .ndo_stop = xennet_close,
@@ -1272,6 +1506,8 @@ static void xennet_poll_controller(struct net_device *dev) .ndo_fix_features = xennet_fix_features, .ndo_set_features = xennet_set_features, .ndo_select_queue = xennet_select_queue, + .ndo_bpf = xennet_xdp, + .ndo_xdp_xmit = xennet_xdp_xmit,#ifdef CONFIG_NET_POLL_CONTROLLER .ndo_poll_controller = xennet_poll_controller,#endif
@@ -1331,6 +1567,7 @@ static struct net_device *xennet_create_dev(struct xenbus_device *dev)SET_NETDEV_DEV (netdev, &dev->dev); np->netdev = netdev; + np->netfront_xdp_enabled = false;netif_carrier_off (netdev);
@@ -1419,6 +1656,8 @@ static void xennet_disconnect_backend(struct netfront_info *info) queue->rx_ring_ref =GRANT_INVALID_REF ; queue->tx.sring =NULL ; queue->rx.sring =NULL ; + +page_pool_destroy (queue->page_pool); } }
@@ -1754,6 +1993,51 @@ static void xennet_destroy_queues(struct netfront_info *info) info->queues =NULL ; } + + +static int xennet_create_page_pool (struct netfront_queue *queue) +{ +int err; +struct page_pool_params pp_params = { + .order =0 , + .flags =0 , + .pool_size =NET_RX_RING_SIZE , + .nid =NUMA_NO_NODE , + .dev = &queue->info->netdev->dev, + .offset =XDP_PACKET_HEADROOM , + .max_len =XEN_PAGE_SIZE -XDP_PACKET_HEADROOM , + }; + + queue->page_pool =page_pool_create (&pp_params); +if (IS_ERR (queue->page_pool)) { + err =PTR_ERR (queue->page_pool); + queue->page_pool =NULL ; +return err; + } + + err =xdp_rxq_info_reg (&queue->xdp_rxq, queue->info->netdev, + queue->id); +if (err) { +netdev_err (queue->info->netdev,"xdp_rxq_info_reg failed\n" ); + goto err_free_pp; + } + + err =xdp_rxq_info_reg_mem_model (&queue->xdp_rxq, +MEM_TYPE_PAGE_POOL , queue->page_pool); +if (err) { +netdev_err (queue->info->netdev,"xdp_rxq_info_reg_mem_model failed\n" ); + goto err_unregister_rxq; + } +return 0 ; + +err_unregister_rxq: +xdp_rxq_info_unreg (&queue->xdp_rxq); +err_free_pp: +page_pool_destroy (queue->page_pool); + queue->page_pool =NULL ; +return err; +} +static int xennet_create_queues (struct netfront_info *info,unsigned int *num_queues) {
@@ -1779,6 +2063,14 @@ static int xennet_create_queues(struct netfront_info *info,break ; } +/* use page pool recycling instead of buddy allocator */ + ret =xennet_create_page_pool (queue); +if (ret <0 ) { +dev_err (&info->xbdev->dev,"can't allocate page pool\n" ); + *num_queues = i; +return ret; + } +netif_napi_add (queue->info->netdev, &queue->napi, xennet_poll,64 );if (netif_running (info->netdev))
@@ -1825,6 +2117,15 @@ static int talk_to_netback(struct xenbus_device *dev, goto out_unlocked; } + info->netback_has_xdp_headroom =xenbus_read_unsigned (info->xbdev->otherend, +"feature-xdp-headroom" ,0 ); +/* set the current xen-netfront xdp state */ + err =talk_to_netback_xdp (info, info->netfront_xdp_enabled ? +NETBACK_XDP_HEADROOM_ENABLE : +NETBACK_XDP_HEADROOM_DISABLE ); +if (err) + goto out_unlocked; +rtnl_lock ();if (info->queues)xennet_destroy_queues (info);
@@ -1959,6 +2260,8 @@ static int xennet_connect(struct net_device *dev) err =talk_to_netback (np->xbdev, np);if (err)return err; +if (np->netback_has_xdp_headroom) +pr_info ("backend supports XDP headroom\n" );/* talk_to_netback() sets the correct number of queues */ num_queues = dev->real_num_tx_queues;
Signature · 1 line
1.8.3.1
PATCH v8 2/3 xen networking: add XDP offset adjustment to xen-netback
the patch basically adds the offset adjustment and netfront state reading to make XDP work on netfront side.
--- drivers/net/xen-netback/common.h | 2 ++ drivers/net/xen-netback/netback.c | 7 +++++++ drivers/net/xen-netback/rx.c | 7 ++++++- drivers/net/xen-netback/xenbus.c | 28 ++++++++++++++++++++++++++++ 4 files changed, 43 insertions(+), 1 deletion(-)
drivers/net/xen-netback/common.h +2 -0
@@ -280,6 +280,7 @@ struct xenvif { u8 ip_csum:1; u8 ipv6_csum:1; u8 multicast_control:1; + u8 xdp_enabled:1;/* Is this interface disabled? True when backend discovers * frontend is rogue.
@@ -395,6 +396,7 @@ static inline pending_ring_idx_t nr_pending_reqs(struct xenvif_queue *queue)irqreturn_t xenvif_interrupt (int irq,void *dev_id);extern bool separate_tx_rx_irq; +extern bool provides_xdp_headroom;extern unsigned int rx_drain_timeout_msecs;extern unsigned int rx_stall_timeout_msecs;
drivers/net/xen-netback/netback.c +7 -0
@@ -96,6 +96,13 @@module_param_named (hash_cache_size, xenvif_hash_cache_size, uint,0644 );MODULE_PARM_DESC (hash_cache_size,"Number of flows in the hash cache" ); +/* The module parameter tells that we have to put data +* for xen-netfront with the XDP_PACKET_HEADROOM offset +* needed for XDP processing +*/ +bool provides_xdp_headroom = true; +module_param (provides_xdp_headroom, bool,0644 ); +static void xenvif_idx_release (struct xenvif_queue *queue,u16 pending_idx,u8 status);
drivers/net/xen-netback/rx.c +6 -1
@@ -33,6 +33,11 @@#include <xen/xen.h> #include <xen/events.h> +static inline int xenvif_rx_xdp_offset (struct xenvif *vif) +{ +return vif->xdp_enabled ?XDP_PACKET_HEADROOM :0 ; +} +static bool xenvif_rx_ring_slots_available (struct xenvif_queue *queue) {RING_IDX prod, cons;
@@ -356,7 +361,7 @@ static void xenvif_rx_data_slot(struct xenvif_queue *queue,struct xen_netif_rx_request *req, struct xen_netif_rx_response *rsp) { - unsigned int offset = 0; + unsigned int offset = xenvif_rx_xdp_offset(queue->vif);unsigned int flags; do {
drivers/net/xen-netback/xenbus.c +28 -0
@@ -393,6 +393,20 @@ static void set_backend_state(struct backend_info *be, } } +static void read_xenbus_frontend_xdp (struct backend_info *be, +struct xenbus_device *dev) +{ +struct xenvif *vif = be->vif; +unsigned int val; +int err; + + err =xenbus_scanf (XBT_NIL , dev->otherend, +"feature-xdp" ,"%u" , &val); +if (err !=1 ) +return ; + vif->xdp_enabled = val; +} +/** * Callback received when the frontend's state changes. */
@@ -417,6 +431,11 @@ static void frontend_changed(struct xenbus_device *dev,set_backend_state (be, XenbusStateConnected);break ; +case XenbusStateReconfiguring: +read_xenbus_frontend_xdp (be, dev); +xenbus_switch_state (dev, XenbusStateReconfigured); +break ; +case XenbusStateClosing:set_backend_state (be, XenbusStateClosing);break ;
@@ -1036,6 +1055,15 @@ static int netback_probe(struct xenbus_device *dev, goto abort_transaction; } +/* we can adjust a headroom for netfront XDP processing */ + err =xenbus_printf (xbt, dev->nodename, +"feature-xdp-headroom" ,"%d" , + provides_xdp_headroom); +if (err) { + message ="writing feature-xdp-headroom" ; + goto abort_transaction; + } +/* We don't support rx-flip path (except old guests who * don't grok this feature flag). */
Signature · 1 line
1.8.3.1
Jakub Kicinski
maintainerRe: [PATCH net-next v8 2/3] xen networking: add XDP offset adjustment to xen-netback
Quoted 1 line · On Thu, 7 May 2020 15:40:40 +0300 Denis Kirjanov wrote:
+static inline int
xenvif_rx_xdp_offset(struct xenvif *vif)
Please don't use static inlines in C files, compiler will know when to inline and when not.
Quoted 3 lines
drivers/net/xen-netback/rx.c +3
-0+{ +return vif->xdp_enabled ?XDP_PACKET_HEADROOM :0 ; +}
PATCH v8 3/3 net: xen: select PAGE_POOL for xen-netfront
In reply to Denis Kirjanovxen-netfront uses page pool API so select it in Kconfig
--- drivers/net/Kconfig | 1 + 1 file changed, 1 insertion(+)
drivers/net/Kconfig +1 -0
@@ -479,6 +479,7 @@ config XEN_NETDEV_FRONTEND tristate "Xen network device frontend driver" depends on XEN select XEN_XENBUS_FRONTEND + select PAGE_POOL default y help This driver provides support for Xen paravirtual network
Signature · 1 line
1.8.3.1
Jakub Kicinski
maintainerRe: [PATCH net-next v8 3/3] net: xen: select PAGE_POOL for xen-netfront
Quoted 3 lines · On Thu, 7 May 2020 15:40:41 +0300 Denis Kirjanov wrote:
xen-netfront uses page pool API so select it in
KconfigSigned-off-by Denis Kirjanov <[email protected]>
This needs to be a part of the patch that uses page pool otherwise build may be broken between the two patches.
Jakub Kicinski
maintainerQuoted 6 lines · On Thu, 7 May 2020 15:40:39 +0300 Denis Kirjanov wrote:
The patch adds a basic XDP processing to xen-netfront driver.
We ran an XDP program for an RX response received from netback driver. Also we request xen-netback to adjust data offset for
bpf_xdp_adjust_head()header space for custom headers.
Quoted 6 lines
synchronization between frontend and backend parts is done by using xenbus state switching:
Reconfiguring -> Reconfigured- > ConnectedUDP packets drop rate using xdp program is around 310 kpps using ./pktgen_sample04_many_flows.sh and 160 kpps without the patch.
Please provide a cover letter for the submission.
drivers/net/xen-netfront.c struct netfront_rx_info { · line 183
Quoted 6 lines
drivers/net/xen-netfront.c +2
-0@@ -167,6 +179,9 @@ struct netfront_rx_info {struct xen_netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX -1 ]; }; +static int xennet_xdp_xmit (struct net_device *dev,int n, +struct xdp_frame **frames,u32 flags);
Is it possible to put the function here and avoid the forward declaration?
Quoted 15 lines
drivers/net/xen-netfront.c +2
-2static void skb_entry_set_link (union skb_entry *list,unsigned short id) { list->link = id;@@ -265,8 +280,8 @@ static struct sk_buff *xennet_alloc_one_rx_buffer(struct netfront_queue *queue)if (unlikely (!skb))return NULL ; - page =alloc_page (GFP_ATOMIC | __GFP_NOWARN); -if (!page) { + page =page_pool_dev_alloc_pages (queue->page_pool); +if (unlikely (!page)) {kfree_skb (skb);return NULL ; }
drivers/net/xen-netfront.c static int xennet_get_extras(struct netfront_queue *queue, · line 796
Quoted 4 lines
drivers/net/xen-netfront.c +1
-0@@ -778,6 +793,53 @@ static int xennet_get_extras(struct netfront_queue *queue,return err; } +u32 xennet_run_xdp(struct netfront_queue *queue, struct page *pdata,
static
Quoted 4 lines
drivers/net/xen-netfront.c +4
-0+ struct xen_netif_rx_response *rx, struct bpf_prog *prog, + struct xdp_buff *xdp) +{ + struct xdp_frame *xdpf;
drivers/net/xen-netfront.c static int xennet_get_extras(struct netfront_queue *queue, · line 810
Quoted 10 lines
drivers/net/xen-netfront.c +10
-0+u32 len = rx->status; +u32 act =XDP_PASS ; +int err; + + xdp->data_hard_start =page_address (pdata); + xdp->data = xdp->data_hard_start +XDP_PACKET_HEADROOM ; +xdp_set_data_meta_invalid (xdp); + xdp->data_end = xdp->data + len; + xdp->rxq = &queue->xdp_rxq; + xdp->handle =0 ;
No need to clear this.
Quoted 5 lines
drivers/net/xen-netfront.c +5
-0+ act =bpf_prog_run_xdp (prog, xdp); +switch (act) { +case XDP_TX : +get_page (pdata); + xdpf =convert_to_xdp_frame (xdp);
drivers/net/xen-netfront.c static int xennet_get_extras(struct netfront_queue *queue, · line 826
Quoted 10 lines
drivers/net/xen-netfront.c +10
-0+ err =xennet_xdp_xmit (queue->info->netdev,1 , &xdpf,0 ); +if (unlikely (err <0 )) +trace_xdp_exception (queue->info->netdev, prog, act); +break ; +case XDP_REDIRECT : +get_page (pdata); + err =xdp_do_redirect (queue->info->netdev, xdp, prog); +if (unlikely (err)) +trace_xdp_exception (queue->info->netdev, prog, act); +xdp_do_flush ();
Can you call flush after the RX loop? (xennet_poll()?) This is supposed to batch all redirections in the NAPI cycle.
Quoted 30 lines
drivers/net/xen-netfront.c +19
-0+break ; +case XDP_PASS : +case XDP_DROP : +break ; + +case XDP_ABORTED : +trace_xdp_exception (queue->info->netdev, prog, act); +break ; + +default : +bpf_warn_invalid_xdp_action (act); + } + +return act; +} +static int xennet_get_responses (struct netfront_queue *queue,struct netfront_rx_info *rinfo,RING_IDX rp,struct sk_buff_head *list)@@ -792,6 +854,9 @@ static int xennet_get_responses(struct netfront_queue *queue,int slots =1 ;int err =0 ;unsigned long ret; +struct bpf_prog *xdp_prog; +struct xdp_buff xdp; +u32 verdict;if (rx->flags & XEN_NETRXF_extra_info) { err =xennet_get_extras (queue, extras, rp);
drivers/net/xen-netfront.c static int xennet_get_responses(struct netfront_queue *queue, · line 897
Quoted 8 lines
drivers/net/xen-netfront.c +3
-2@@ -827,9 +892,20 @@ static int xennet_get_responses(struct netfront_queue *queue,gnttab_release_grant_reference (&queue->gref_rx_head, ref); -__skb_queue_tail (list, skb); - +rcu_read_lock (); + xdp_prog =rcu_dereference (queue->xdp_prog); +if (xdp_prog && !(rx->flags & XEN_NETRXF_more_data)) {
What is XEN_NETRXF_more_data?
If you have a mis-formatted frame it has to be dropped, nothing can escape the XDP program and go to the stack.
Quoted 23 lines
(quoted diff) +18
-0+ /* currently only a single page contains data */ + WARN_ON_ONCE(skb_shinfo(skb)->nr_frags != 1); + verdict = xennet_run_xdp(queue, + skb_frag_page(&skb_shinfo(skb)->frags[0]), + rx, xdp_prog, &xdp); + if (verdict != XDP_PASS) + err = -EINVAL; + } + rcu_read_unlock(); next: + __skb_queue_tail(list, skb); if (!(rx->flags & XEN_NETRXF_more_data)) break; +static int xennet_xdp_set(struct net_device *dev, struct bpf_prog *prog, + struct netlink_ext_ack *extack) +{ + struct netfront_info *np = netdev_priv(dev); + struct bpf_prog *old_prog; + unsigned int i, err; + unsigned long int max_mtu = XEN_PAGE_SIZE - XDP_PACKET_HEADROOM; +
Review static int xennet_get_responses(struct netfront_queue *queue,
Quoted 10 lines
(quoted diff) +10
-0+ if (dev->mtu > max_mtu) { + netdev_warn(dev, "XDP requires MTU less than %lu\n", max_mtu); + return -EINVAL; + } + + if (!np->netback_has_xdp_headroom) + return 0; + + old_prog = rtnl_dereference(np->queues[0].xdp_prog); + if (!old_prog && !prog)
You don't have to check this, the core will not call drivers any more if there is no program and request is to disable XDP.
drivers/net/xen-netfront.c static void xennet_poll_controller(struct net_device *dev) · line 1386
Quoted 7 lines
drivers/net/xen-netfront.c +7
-0+return 0 ; + +if (prog) +bpf_prog_add (prog, dev->real_num_tx_queues); + +for (i =0 ; i < dev->real_num_tx_queues; ++i) +rcu_assign_pointer (np->queues[i].xdp_prog, prog);
I'm I reading this right the moment you assign the program the RX function will start assuming there is a headroom, even if you haven't reconfigured the backend, yet..
Quoted 23 lines
drivers/net/xen-netfront.c +21
-0+if (old_prog) +for (i =0 ; i < dev->real_num_tx_queues; ++i) +bpf_prog_put (old_prog); + +xenbus_switch_state (np->xbdev, XenbusStateReconfiguring); + + err =talk_to_netback_xdp (np, prog ?NETBACK_XDP_HEADROOM_ENABLE : +NETBACK_XDP_HEADROOM_DISABLE ); +if (err) +return err; + +/* avoid race with XDP headroom adjustment */ +wait_event (module_wq, +xenbus_read_driver_state (np->xbdev->otherend) == + XenbusStateReconfigured); + np->netfront_xdp_enabled = true; +xenbus_switch_state (np->xbdev, XenbusStateConnected); + +return 0 ; +} +
drivers/net/xen-netfront.c static void xennet_destroy_queues(struct netfront_info *info) · line 1998
Quoted 6 lines
drivers/net/xen-netfront.c +3
-0@@ -1754,6 +1993,51 @@ static void xennet_destroy_queues(struct netfront_info *info) info->queues =NULL ; } + + +static int xennet_create_page_pool (struct netfront_queue *queue)
Please run checkpatch --strict on your patches.
Quoted 20 lines · Jakub Kicinski
Quoted 6 lines · On Thu, 7 May 2020 15:40:39 +0300 Denis Kirjanov wrote:
The patch adds a basic XDP processing to xen-netfront driver.
We ran an XDP program for an RX response received from netback driver. Also we request xen-netback to adjust data offset for
bpf_xdp_adjust_head()header space for custom headers.Quoted 6 lines
synchronization between frontend and backend parts is done by using xenbus state switching:
Reconfiguring -> Reconfigured- > ConnectedUDP packets drop rate using xdp program is around 310 kpps using ./pktgen_sample04_many_flows.sh and 160 kpps without the patch.
Please provide a cover letter for the submission.
Quoted 4 lines
drivers/net/xen-netfront.c +0
-0@@ -167,6 +179,9 @@ struct netfront_rx_info {struct xen_netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX -1 ]; };
Quoted 5 lines
drivers/net/xen-netfront.c struct netfront_rx_info { · line 183
Quoted 2 lines
drivers/net/xen-netfront.c +2
-0+static int xennet_xdp_xmit (struct net_device *dev,int n, +struct xdp_frame **frames,u32 flags);Is it possible to put the function here and avoid the forward declaration?
Sure
Quoted 52 lines
Quoted 17 lines
drivers/net/xen-netfront.c +2
-2static void skb_entry_set_link (union skb_entry *list,unsigned short id) { list->link = id;@@ -265,8 +280,8 @@ static struct sk_buff *xennet_alloc_one_rx_buffer(struct netfront_queue *queue)if (unlikely (!skb))return NULL ; - page =alloc_page (GFP_ATOMIC | __GFP_NOWARN); -if (!page) { + page =page_pool_dev_alloc_pages (queue->page_pool); +if (unlikely (!page)) {kfree_skb (skb);return NULL ;drivers/net/xen-netfront.c +0
-0}drivers/net/xen-netfront.c static int xennet_get_extras(struct netfront_queue · line 797
Quoted 5 lines
drivers/net/xen-netfront.c +1
-0@@ -778,6 +793,53 @@ static int xennet_get_extras(struct netfront_queue *queue, return err; } +u32 xennet_run_xdp(struct netfront_queue *queue, struct page *pdata,static
Quoted 4 lines
drivers/net/xen-netfront.c +4
-0+ struct xen_netif_rx_response *rx, struct bpf_prog *prog, + struct xdp_buff *xdp) +{ + struct xdp_frame *xdpf;drivers/net/xen-netfront.c static int xennet_get_extras(struct netfront_queue *queue, · line 810
Quoted 10 lines
drivers/net/xen-netfront.c +10
-0+u32 len = rx->status; +u32 act =XDP_PASS ; +int err; + + xdp->data_hard_start =page_address (pdata); + xdp->data = xdp->data_hard_start +XDP_PACKET_HEADROOM ; +xdp_set_data_meta_invalid (xdp); + xdp->data_end = xdp->data + len; + xdp->rxq = &queue->xdp_rxq; + xdp->handle =0 ;No need to clear this.
Quoted 10 lines
drivers/net/xen-netfront.c +10
-0+ act =bpf_prog_run_xdp (prog, xdp); +switch (act) { +case XDP_TX : +get_page (pdata); + xdpf =convert_to_xdp_frame (xdp); + err =xennet_xdp_xmit (queue->info->netdev,1 , &xdpf,0 ); +if (unlikely (err <0 )) +trace_xdp_exception (queue->info->netdev, prog, act); +break ; +case XDP_REDIRECT :
Quoted 8 lines
drivers/net/xen-netfront.c static int xennet_get_extras(struct netfront_queue *queue, · line 826
Quoted 5 lines
drivers/net/xen-netfront.c +5
-0+get_page (pdata); + err =xdp_do_redirect (queue->info->netdev, xdp, prog); +if (unlikely (err)) +trace_xdp_exception (queue->info->netdev, prog, act); +xdp_do_flush ();Can you call flush after the RX loop? (
xennet_poll()?) This is supposed to batch all redirections in the NAPI cycle.
Ok, will do
Quoted 44 lines
Quoted 32 lines
drivers/net/xen-netfront.c +19
-0+break ; +case XDP_PASS : +case XDP_DROP : +break ; + +case XDP_ABORTED : +trace_xdp_exception (queue->info->netdev, prog, act); +break ; + +default : +bpf_warn_invalid_xdp_action (act); + } + +return act; +} +static int xennet_get_responses (struct netfront_queue *queue,struct netfront_rx_info *rinfo,RING_IDX rp,struct sk_buff_head *list)@@ -792,6 +854,9 @@ static int xennet_get_responses(struct netfront_queue *queue, int slots = 1;int err =0 ;unsigned long ret; +struct bpf_prog *xdp_prog; +struct xdp_buff xdp; +u32 verdict;if (rx->flags & XEN_NETRXF_extra_info) {drivers/net/xen-netfront.c +0
-0err =xennet_get_extras (queue, extras, rp);drivers/net/xen-netfront.c static int xennet_get_responses(struct netfront_queue · line 898
Quoted 9 lines
drivers/net/xen-netfront.c +3
-2@@ -827,9 +892,20 @@ static int xennet_get_responses(struct netfront_queue *queue,gnttab_release_grant_reference (&queue->gref_rx_head, ref); -__skb_queue_tail (list, skb); - +rcu_read_lock (); + xdp_prog =rcu_dereference (queue->xdp_prog); +if (xdp_prog && !(rx->flags & XEN_NETRXF_more_data)) {What is
XEN_NETRXF_more_data?
Quoted 2 lines
If you have a mis-formatted frame it has to be dropped, nothing can escape the XDP program and go to the stack.
I see, thanks.
Quoted 38 lines
Quoted 24 lines
(quoted diff) +18
-0+ /* currently only a single page contains data */ + WARN_ON_ONCE(skb_shinfo(skb)->nr_frags != 1); + verdict = xennet_run_xdp(queue, + skb_frag_page(&skb_shinfo(skb)->frags[0]), + rx, xdp_prog, &xdp); + if (verdict != XDP_PASS) + err = -EINVAL; + } + rcu_read_unlock(); next: + __skb_queue_tail(list, skb); if (!(rx->flags & XEN_NETRXF_more_data)) break; +static int xennet_xdp_set(struct net_device *dev, struct bpf_prog *prog, + struct netlink_ext_ack *extack) +{ + struct netfront_info *np = netdev_priv(dev); + struct bpf_prog *old_prog; + unsigned int i, err; + unsigned long int max_mtu = XEN_PAGE_SIZE - XDP_PACKET_HEADROOM; +Review static int xennet_get_responses(struct netfront_queue
Quoted 10 lines
(quoted diff) +10
-0+ if (dev->mtu > max_mtu) { + netdev_warn(dev, "XDP requires MTU less than %lu\n", max_mtu); + return -EINVAL; + } + + if (!np->netback_has_xdp_headroom) + return 0; + + old_prog = rtnl_dereference(np->queues[0].xdp_prog); + if (!old_prog && !prog)You don't have to check this, the core will not call drivers any more if there is no program and request is to disable XDP.
Quoted 11 lines
drivers/net/xen-netfront.c static void xennet_poll_controller(struct net_device *dev) · line 1386
Quoted 7 lines
drivers/net/xen-netfront.c +7
-0+return 0 ; + +if (prog) +bpf_prog_add (prog, dev->real_num_tx_queues); + +for (i =0 ; i < dev->real_num_tx_queues; ++i) +rcu_assign_pointer (np->queues[i].xdp_prog, prog);I'm I reading this right the moment you assign the program the RX function will start assuming there is a headroom, even if you haven't reconfigured the backend, yet..
That's a good catch. Thank you.
Quoted 34 lines
Quoted 24 lines
drivers/net/xen-netfront.c +21
-0+if (old_prog) +for (i =0 ; i < dev->real_num_tx_queues; ++i) +bpf_prog_put (old_prog); + +xenbus_switch_state (np->xbdev, XenbusStateReconfiguring); + + err =talk_to_netback_xdp (np, prog ?NETBACK_XDP_HEADROOM_ENABLE : +NETBACK_XDP_HEADROOM_DISABLE ); +if (err) +return err; + +/* avoid race with XDP headroom adjustment */ +wait_event (module_wq, +xenbus_read_driver_state (np->xbdev->otherend) == + XenbusStateReconfigured); + np->netfront_xdp_enabled = true; +xenbus_switch_state (np->xbdev, XenbusStateConnected); + +return 0 ; +} +drivers/net/xen-netfront.c static void xennet_destroy_queues(struct · line 1999
Quoted 7 lines
drivers/net/xen-netfront.c +3
-0@@ -1754,6 +1993,51 @@ static void xennet_destroy_queues(struct netfront_info *info) info->queues = NULL; } + + +static int xennet_create_page_pool (struct netfront_queue *queue)Please run checkpatch --strict on your patches.