Introduce XDP to ena
Subsystems (3)
v2 cover Introduce XDP to ena
From: Sameeh Jubran <[email protected]>
This patchset includes 3 patches:
* XDP_DROP implementation
* XDP_TX implementation
* A fix for an issue which might occur due to the XDP_TX patch. I see fit to place it as a standalone patch for clarity.
Difference from RFC v1 (XDP_DROP patch):
* Initialized xdp.rxq pointer
* Updated max_mtu on attachment of xdp and removed the check from ena_change_mtu()
* Moved the xdp execution from ena_rx_skb() to ena_clean_rx_irq()
* Moved xdp buff (struct xdp_buff) from rx_ring to the local stack
* Started using netlink's extack mechanism to deliver error messages to the user
Sameeh Jubran (3):
net: ena: implement XDP drop support
net: ena: Implement XDP_TX action
net: ena: Add first_interrupt field to napi struct
drivers/net/ethernet/amazon/ena/ena_ethtool.c | 4 +- drivers/net/ethernet/amazon/ena/ena_netdev.c | 839 ++++++++++++++++-- drivers/net/ethernet/amazon/ena/ena_netdev.h | 70 ++ 3 files changed, 815 insertions(+), 98 deletions(-)
Signature · 1 line
2.17.1
RFC v2 1/3 net: ena: implement XDP drop support
From: Sameeh Jubran <[email protected]>
This commit implements the basic functionality of drop/pass logic in the ena driver.
--- drivers/net/ethernet/amazon/ena/ena_netdev.c | 132 +++++++++++++++++-- drivers/net/ethernet/amazon/ena/ena_netdev.h | 29 ++++ 2 files changed, 152 insertions(+), 9 deletions(-)
drivers/net/ethernet/amazon/ena/ena_netdev.c +123 -9
@@ -33,10 +33,10 @@#define pr_fmt (fmt) KBUILD_MODNAME ": " fmt#ifdef CONFIG_RFS_ACCEL +#include <linux/bpf_trace.h> #include <linux/cpu_rmap.h> #endif /* CONFIG_RFS_ACCEL */ #include <linux/ethtool.h> -#include <linux/if_vlan.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/numa.h>
@@ -105,6 +105,79 @@ static void update_rx_ring_mtu(struct ena_adapter *adapter, int mtu) adapter->rx_ring[i].mtu = mtu; } +static int ena_xdp_execute (struct ena_ring *rx_ring,struct xdp_buff *xdp) +{ +struct bpf_prog *xdp_prog = rx_ring->xdp_bpf_prog; +u32 verdict =XDP_PASS ; + +rcu_read_lock (); + +if (!xdp_prog) + goto out; + + verdict =bpf_prog_run_xdp (xdp_prog, xdp); + +if (unlikely (verdict ==XDP_ABORTED )) +trace_xdp_exception (rx_ring->netdev, xdp_prog, verdict); +else if (unlikely (verdict >=XDP_TX )) +bpf_warn_invalid_xdp_action (verdict); +out: +rcu_read_unlock (); +return verdict; +} + +static int ena_xdp_set (struct net_device *netdev,struct netdev_bpf *bpf) +{ +struct ena_adapter *adapter =netdev_priv (netdev); +struct bpf_prog *prog = bpf->prog; +struct bpf_prog *old_bpf_prog; +int i, prev_mtu; + +if (ena_xdp_allowed (adapter)) { + old_bpf_prog =xchg (&adapter->xdp_bpf_prog, prog); + +for (i =0 ; i < adapter->num_io_queues; i++) +xchg (&adapter->rx_ring[i].xdp_bpf_prog, prog); + +if (old_bpf_prog) +bpf_prog_put (old_bpf_prog); + + prev_mtu = netdev->max_mtu; + netdev->max_mtu = prog ?ENA_XDP_MAX_MTU : adapter->max_mtu; +netif_info (adapter, drv, adapter->netdev,"xdp program set, changging the max_mtu from %d to %d" , + prev_mtu, netdev->max_mtu); + + }else { +netif_err (adapter, drv, adapter->netdev,"Failed to set xdp program, the current MTU (%d) is larger than the maximal allowed MTU (%lu) while xdp is on" , + netdev->mtu,ENA_XDP_MAX_MTU ); +NL_SET_ERR_MSG_MOD (bpf->extack,"Failed to set xdp program, the current MTU is larger than the maximal allowed MTU. Check the dmesg for more info" ); +return -EINVAL ; + } + +return 0 ; +} + +/* This is the main xdp callback, it's used by the kernel to set/unset the xdp +* program as well as to query the current xdp program id. +*/ +static int ena_xdp (struct net_device *netdev,struct netdev_bpf *bpf) +{ +struct ena_adapter *adapter =netdev_priv (netdev); + +switch (bpf->command) { +case XDP_SETUP_PROG : +return ena_xdp_set (netdev, bpf); +case XDP_QUERY_PROG : + bpf->prog_id = adapter->xdp_bpf_prog ? + adapter->xdp_bpf_prog->aux->id :0 ; +break ; +default : +NL_SET_ERR_MSG_MOD (bpf->extack,"Unsupported XDP command" ); +return -EINVAL ; + } +return 0 ; +} +static int ena_change_mtu (struct net_device *dev,int new_mtu) {struct ena_adapter *adapter =netdev_priv (dev);
@@ -417,6 +490,9 @@ static void ena_free_rx_resources(struct ena_adapter *adapter,vfree (rx_ring->free_ids); rx_ring->free_ids =NULL ; + +xdp_rxq_info_unreg_mem_model (&rx_ring->xdp_rxq); +xdp_rxq_info_unreg (&rx_ring->xdp_rxq); }/* ena_setup_all_rx_resources - allocate I/O Rx queues resources for all queues
@@ -1051,13 +1127,16 @@ static int ena_clean_rx_irq(struct ena_ring *rx_ring, struct napi_struct *napi,u32 res_budget, work_done;struct ena_com_rx_ctx ena_rx_ctx; +struct ena_rx_buffer *rx_info;struct ena_adapter *adapter; -struct sk_buff *skb; +int xdp_verdict =XDP_PASS ; +struct sk_buff *skb =NULL ;int refill_required;int refill_threshold; -int rc =0 ; -int total_len =0 ; +struct xdp_buff xdp;int rx_copybreak_pkt =0 ; +int total_len =0 ; +int rc =0 ;int i; netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
@@ -1082,12 +1161,24 @@ static int ena_clean_rx_irq(struct ena_ring *rx_ring, struct napi_struct *napi, rx_ring->qid, ena_rx_ctx.descs, ena_rx_ctx.l3_proto, ena_rx_ctx.l4_proto, ena_rx_ctx.hash); +if (ena_xdp_present_ring (rx_ring)) { + rx_info = &rx_ring->rx_buffer_info[rx_ring->ena_bufs[0 ].req_id]; + xdp.data =page_address (rx_info->page) + + rx_info->page_offset; + xdp.data_meta = xdp.data; + xdp.data_hard_start = xdp.data - + rx_info->page_offset; + xdp.data_end = xdp.data + rx_ring->ena_bufs[0 ].len; + xdp_verdict =ena_xdp_execute (rx_ring, &xdp); + } +/* allocate skb and fill it */ - skb =ena_rx_skb (rx_ring, rx_ring->ena_bufs, ena_rx_ctx.descs, - &next_to_clean); +if (xdp_verdict ==XDP_PASS ) + skb =ena_rx_skb (rx_ring, rx_ring->ena_bufs, + ena_rx_ctx.descs, &next_to_clean);/* exit if we failed to retrieve a buffer */ -if (unlikely (!skb)) { +if (!skb) {for (i =0 ; i < ena_rx_ctx.descs; i++) { rx_ring->free_ids[next_to_clean] = rx_ring->ena_bufs[i].req_id;
@@ -1727,12 +1818,34 @@ static int ena_create_io_rx_queue(struct ena_adapter *adapter, int qid)netif_err (adapter, ifup, adapter->netdev,"Failed to get RX queue handlers. RX queue num %d rc: %d\n" , qid, rc); -ena_com_destroy_io_queue (ena_dev, ena_qid); -return rc; + goto err; }ena_com_update_numa_node (rx_ring->ena_com_io_cq, ctx.numa_node); + rc =xdp_rxq_info_reg (&rx_ring->xdp_rxq, adapter->netdev, qid); + +if (rc) { +netif_err (adapter, ifup, adapter->netdev, +"Failed to register xdp rx queue info. RX queue num %d rc: %d\n" , + qid, rc); + goto err; + } + + rc =xdp_rxq_info_reg_mem_model (&rx_ring->xdp_rxq,MEM_TYPE_PAGE_SHARED , +NULL ); + +if (rc) { +netif_err (adapter, ifup, adapter->netdev, +"Failed to register xdp rx queue info memory model. RX queue num %d rc: %d\n" , + qid, rc); +xdp_rxq_info_unreg (&rx_ring->xdp_rxq); + goto err; + } + +return rc; +err: +ena_com_destroy_io_queue (ena_dev, ena_qid);return rc; }
@@ -2568,6 +2681,7 @@ static const struct net_device_ops ena_netdev_ops = { .ndo_change_mtu = ena_change_mtu, .ndo_set_mac_address = NULL, .ndo_validate_addr = eth_validate_addr, + .ndo_bpf = ena_xdp, }; static int ena_device_validate_params(struct ena_adapter *adapter,
drivers/net/ethernet/amazon/ena/ena_netdev.h +29 -0
@@ -36,6 +36,7 @@#include <linux/bitops.h> #include <linux/dim.h> #include <linux/etherdevice.h> +#include <linux/if_vlan.h> #include <linux/inetdevice.h> #include <linux/interrupt.h> #include <linux/netdevice.h>
@@ -142,6 +143,14 @@#define ENA_MMIO_DISABLE_REG_READ BIT(0) +/* The max MTU size is configured to be the ethernet frame without the overhead +* of the ethernet header, which can have VLAN header, and the frame check +* sequence (FCS). +* The buffer sizes we share with the device are defined to be ENA_PAGE_SIZE +*/ +#define ENA_XDP_MAX_MTU (ENA_PAGE_SIZE - ETH_HLEN - ETH_FCS_LEN - \ + VLAN_HLEN - XDP_PACKET_HEADROOM) +struct ena_irq {irq_handler_t handler;void *data;
@@ -291,6 +300,8 @@ struct ena_ring {u8 *push_buf_intermediate_buf;int empty_rx_queue; +struct bpf_prog *xdp_bpf_prog; +struct xdp_rxq_info xdp_rxq; }____cacheline_aligned ;struct ena_stats_dev {
@@ -379,6 +390,9 @@ struct ena_adapter {u32 last_monitored_tx_qid;enum ena_regs_reset_reason_types reset_reason; + +/* XDP structures */ +struct bpf_prog *xdp_bpf_prog; };void ena_set_ethtool_ops (struct net_device *netdev);
@@ -394,4 +408,19 @@ int ena_update_queue_count(struct ena_adapter *adapter, u32 new_channel_count);int ena_get_sset_count (struct net_device *netdev,int sset); +static inline bool ena_xdp_present (struct ena_adapter *adapter) +{ +return !!adapter->xdp_bpf_prog; +} + +static inline bool ena_xdp_present_ring (struct ena_ring *ring) +{ +return !!ring->xdp_bpf_prog; +} + +static inline bool ena_xdp_allowed (struct ena_adapter *adapter) +{ +return adapter->netdev->mtu <=ENA_XDP_MAX_MTU ; +} +#endif /* !(ENA_H) */
Signature · 1 line
2.17.1
RFC v2 2/3 net: ena: Implement XDP_TX action
In reply to [email protected]From: Sameeh Jubran <[email protected]>
This commit implements the XDP_TX action in the ena driver. We allocate separate tx queues for the XDP_TX. We currently allow xdp only when there is enough queues to allocate for xdp.
--- drivers/net/ethernet/amazon/ena/ena_ethtool.c | 4 +- drivers/net/ethernet/amazon/ena/ena_netdev.c | 743 +++++++++++++++--- drivers/net/ethernet/amazon/ena/ena_netdev.h | 44 +- 3 files changed, 681 insertions(+), 110 deletions(-)
drivers/net/ethernet/amazon/ena/ena_ethtool.c +3 -1
@@ -744,7 +744,9 @@ static int ena_set_channels(struct net_device *netdev,struct ena_adapter *adapter =netdev_priv (netdev);u32 count = channels->combined_count;/* The check for max value is already done in ethtool */ -if (count <ENA_MIN_NUM_IO_QUEUES ) +if (count <ENA_MIN_NUM_IO_QUEUES || + (ena_xdp_present (adapter) && + !ena_xdp_legal_queue_count (adapter, channels->combined_count)))return -EINVAL ;return ena_update_queue_count (adapter, count);
drivers/net/ethernet/amazon/ena/ena_netdev.c +636 -107
1,106 lines not on this page · raw message
drivers/net/ethernet/amazon/ena/ena_netdev.h +42 -2
81 lines not on this page · raw message
Signature · 1 line
2.17.1
RFC v2 3/3 net: ena: Add first_interrupt field to napi struct
In reply to [email protected]From: Sameeh Jubran <[email protected]>
The first_interrupt field is accessed in ena_intr_msix_io() upon receiving an interrupt.The rx_ring and tx_ring fields of napi can be NULL when receiving interrupt for xdp queues. This patch fixes the issue by moving the field to the ena_napi struct.
--- drivers/net/ethernet/amazon/ena/ena_netdev.c | 6 ++++-- drivers/net/ethernet/amazon/ena/ena_netdev.h | 1 + 2 files changed, 5 insertions(+), 2 deletions(-)
drivers/net/ethernet/amazon/ena/ena_netdev.c +4 -2
@@ -1809,6 +1809,9 @@ static int ena_io_poll(struct napi_struct *napi, int budget) tx_ring = ena_napi->tx_ring; rx_ring = ena_napi->rx_ring; + tx_ring->first_interrupt = ena_napi->first_interrupt; + rx_ring->first_interrupt = ena_napi->first_interrupt; + tx_budget = tx_ring->ring_size /ENA_TX_POLL_BUDGET_DIVIDER ; if (!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) ||
@@ -1880,8 +1883,7 @@ static irqreturn_t ena_intr_msix_io(int irq, void *data) {struct ena_napi *ena_napi = data; - ena_napi->tx_ring->first_interrupt = true; - ena_napi->rx_ring->first_interrupt = true; + ena_napi->first_interrupt = true;napi_schedule_irqoff (&ena_napi->napi);
drivers/net/ethernet/amazon/ena/ena_netdev.h +1 -0
@@ -168,6 +168,7 @@ struct ena_napi {struct ena_ring *tx_ring;struct ena_ring *rx_ring;struct ena_ring *xdp_ring; +bool first_interrupt;u32 qid;struct dim dim; };
Signature · 1 line
2.17.1