ionic: add XDP support
Subsystems (3)
cover ionic: add XDP support
This is an RFC patchset of new support in ionic for XDP processing, including basic XDP on Rx packets, TX and REDIRECT, and frags for jumbo frames. We present this here for your comments and our education, and are hoping to get it merged into net-next in this current development cycle.
Since ionic has not yet been converted to use the page_pool APIs, this uses the simple MEM_TYPE_PAGE_ORDER0 buffering. There are plans to convert the driver in the future, but not yet scheduled.
Shannon Nelson (9):
ionic: set adminq irq affinity
ionic: add helpers for accessing buffer info
ionic: use dma range APIs
ionic: add initial framework for XDP support
ionic: Add XDP packet headroom
ionic: Add XDP_TX support
ionic: Add XDP_REDIRECT support
ionic: add ndo_xdp_xmit
ionic: implement xdp frags support
.../net/ethernet/pensando/ionic/ionic_dev.h | 11 + .../ethernet/pensando/ionic/ionic_ethtool.c | 5 + .../net/ethernet/pensando/ionic/ionic_lif.c | 190 +++++++- .../net/ethernet/pensando/ionic/ionic_lif.h | 13 + .../net/ethernet/pensando/ionic/ionic_stats.c | 18 + .../net/ethernet/pensando/ionic/ionic_txrx.c | 454 ++++++++++++++++-- .../net/ethernet/pensando/ionic/ionic_txrx.h | 1 + 7 files changed, 654 insertions(+), 38 deletions(-)
Signature · 1 line
2.17.1
RFC PATCH 1/9 ionic: set adminq irq affinity
We claim to have the AdminQ on our irq0 and thus cpu id 0, but we need to be sure we set the affinity hint to try to keep it there.
--- drivers/net/ethernet/pensando/ionic/ionic_lif.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)
drivers/net/ethernet/pensando/ionic/ionic_lif.c +4 -1
@@ -3391,9 +3391,12 @@ static int ionic_lif_adminq_init(struct ionic_lif *lif)napi_enable (&qcq->napi); -if (qcq->flags &IONIC_QCQ_F_INTR ) +if (qcq->flags &IONIC_QCQ_F_INTR ) { +irq_set_affinity_hint (qcq->intr.vector, + &qcq->intr.affinity_mask);ionic_intr_mask (idev->intr_ctrl, qcq->intr.index,IONIC_INTR_MASK_CLEAR ); + } qcq->flags |=IONIC_QCQ_F_INITED ;
Signature · 1 line
2.17.1
RFC PATCH 2/9 ionic: add helpers for accessing buffer info
In reply to Shannon NelsonThese helpers clean up some of the code around DMA mapping and other buffer references, and will be used in the next few patches for the XDP support.
--- .../net/ethernet/pensando/ionic/ionic_txrx.c | 37 ++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-)
drivers/net/ethernet/pensando/ionic/ionic_txrx.c +24 -13
@@ -88,6 +88,21 @@ static inline struct netdev_queue *q_to_ndq(struct ionic_queue *q)return netdev_get_tx_queue (q->lif->netdev, q->index); } +static inline void *ionic_rx_buf_va (struct ionic_buf_info *buf_info) +{ +return page_address (buf_info->page) + buf_info->page_offset; +} + +static inline dma_addr_t ionic_rx_buf_pa (struct ionic_buf_info *buf_info) +{ +return buf_info->dma_addr + buf_info->page_offset; +} + +static inline unsigned int ionic_rx_buf_size (struct ionic_buf_info *buf_info) +{ +return min_t (u32,IONIC_MAX_BUF_LEN ,IONIC_PAGE_SIZE - buf_info->page_offset); +} +static int ionic_rx_page_alloc (struct ionic_queue *q,struct ionic_buf_info *buf_info) {
@@ -207,12 +222,11 @@ static struct sk_buff *ionic_rx_frags(struct ionic_queue *q,return NULL ; } - frag_len =min_t (u16, len,min_t (u32,IONIC_MAX_BUF_LEN , -IONIC_PAGE_SIZE - buf_info->page_offset)); + frag_len =min_t (u16, len,ionic_rx_buf_size (buf_info)); len -= frag_len;dma_sync_single_for_cpu (dev, - buf_info->dma_addr + buf_info->page_offset, +ionic_rx_buf_pa (buf_info), frag_len,DMA_FROM_DEVICE ); skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
@@ -262,10 +276,10 @@ static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q,return NULL ; } -dma_sync_single_for_cpu (dev, buf_info->dma_addr + buf_info->page_offset, +dma_sync_single_for_cpu (dev,ionic_rx_buf_pa (buf_info), len,DMA_FROM_DEVICE ); -skb_copy_to_linear_data (skb,page_address (buf_info->page) + buf_info->page_offset, len); -dma_sync_single_for_device (dev, buf_info->dma_addr + buf_info->page_offset, +skb_copy_to_linear_data (skb,ionic_rx_buf_va (buf_info), len); +dma_sync_single_for_device (dev,ionic_rx_buf_pa (buf_info), len,DMA_FROM_DEVICE );skb_put (skb, len);
@@ -452,9 +466,8 @@ void ionic_rx_fill(struct ionic_queue *q) }/* fill main descriptor - buf[0] */ - desc->addr =cpu_to_le64 (buf_info->dma_addr + buf_info->page_offset); - frag_len =min_t (u16, len,min_t (u32,IONIC_MAX_BUF_LEN , -IONIC_PAGE_SIZE - buf_info->page_offset)); + desc->addr =cpu_to_le64 (ionic_rx_buf_pa (buf_info)); + frag_len =min_t (u16, len,ionic_rx_buf_size (buf_info)); desc->len =cpu_to_le16 (frag_len); remain_len -= frag_len; buf_info++;
@@ -472,10 +485,8 @@ void ionic_rx_fill(struct ionic_queue *q) } } - sg_elem->addr =cpu_to_le64 (buf_info->dma_addr + buf_info->page_offset); - frag_len =min_t (u16, remain_len,min_t (u32,IONIC_MAX_BUF_LEN , -IONIC_PAGE_SIZE - - buf_info->page_offset)); + sg_elem->addr =cpu_to_le64 (ionic_rx_buf_pa (buf_info)); + frag_len =min_t (u16, remain_len,ionic_rx_buf_size (buf_info)); sg_elem->len =cpu_to_le16 (frag_len); remain_len -= frag_len; buf_info++;
Signature · 1 line
2.17.1
RFC PATCH 3/9 ionic: use dma range APIs
In reply to Shannon NelsonConvert Rx datapath handling to use the DMA range APIs in preparation for adding XDP handling.
--- drivers/net/ethernet/pensando/ionic/ionic_txrx.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-)
drivers/net/ethernet/pensando/ionic/ionic_txrx.c +6 -7
@@ -225,9 +225,8 @@ static struct sk_buff *ionic_rx_frags(struct ionic_queue *q, frag_len =min_t (u16, len,ionic_rx_buf_size (buf_info)); len -= frag_len; -dma_sync_single_for_cpu (dev, -ionic_rx_buf_pa (buf_info), - frag_len,DMA_FROM_DEVICE ); +dma_sync_single_range_for_cpu (dev,ionic_rx_buf_pa (buf_info), +0 , frag_len,DMA_FROM_DEVICE ); skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, buf_info->page, buf_info->page_offset, frag_len,
@@ -276,11 +275,11 @@ static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q,return NULL ; } -dma_sync_single_for_cpu (dev,ionic_rx_buf_pa (buf_info), - len,DMA_FROM_DEVICE ); +dma_sync_single_range_for_cpu (dev,ionic_rx_buf_pa (buf_info), +0 , len,DMA_FROM_DEVICE );skb_copy_to_linear_data (skb,ionic_rx_buf_va (buf_info), len); -dma_sync_single_for_device (dev,ionic_rx_buf_pa (buf_info), - len,DMA_FROM_DEVICE ); +dma_sync_single_range_for_device (dev,ionic_rx_buf_pa (buf_info), +0 , len,DMA_FROM_DEVICE );skb_put (skb, len); skb->protocol =eth_type_trans (skb, q->lif->netdev);
Signature · 1 line
2.17.1
RFC PATCH 4/9 ionic: add initial framework for XDP support
In reply to Shannon NelsonSet up the basics for running Rx packets through XDP programs. Add new queue setup and teardown steps for adding/removing an XDP program, and add the call to run the XDP on a packet.
The XDP frame size needs to be the MTU plus standard ethernet header, plus head room for XDP scribblings and tail room for a struct skb_shared_info. Also, at this point, we don't support XDP frags, only a single contiguous Rx buffer. This means that our page splitting is not very useful, so when XDP is in use we need to use the full Rx buffer size and not do sharing.
--- .../net/ethernet/pensando/ionic/ionic_dev.h | 7 + .../ethernet/pensando/ionic/ionic_ethtool.c | 5 + .../net/ethernet/pensando/ionic/ionic_lif.c | 172 +++++++++++++++++- .../net/ethernet/pensando/ionic/ionic_lif.h | 7 + .../net/ethernet/pensando/ionic/ionic_stats.c | 9 + .../net/ethernet/pensando/ionic/ionic_txrx.c | 61 ++++++- 6 files changed, 255 insertions(+), 6 deletions(-)
drivers/net/ethernet/pensando/ionic/ionic_dev.h +7 -0
@@ -8,6 +8,7 @@#include <linux/mutex.h> #include <linux/workqueue.h> #include <linux/skbuff.h> +#include <linux/bpf_trace.h> #include "ionic_if.h" #include "ionic_regs.h"
@@ -195,6 +196,11 @@ typedef void (*ionic_desc_cb)(struct ionic_queue *q,#define IONIC_PAGE_GFP_MASK (GFP_ATOMIC | __GFP_NOWARN |\ __GFP_COMP | __GFP_MEMALLOC) +#define IONIC_XDP_MAX_LINEAR_MTU (IONIC_PAGE_SIZE - \ + (VLAN_ETH_HLEN + \ + XDP_PACKET_HEADROOM + \ + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))) +struct ionic_buf_info {struct page *page;dma_addr_t dma_addr;
@@ -256,6 +262,7 @@ struct ionic_queue {struct ionic_txq_sg_desc *txq_sgl;struct ionic_rxq_sg_desc *rxq_sgl; }; +struct xdp_rxq_info *xdp_rxq_info;dma_addr_t base_pa;dma_addr_t cmb_base_pa;dma_addr_t sg_base_pa;
drivers/net/ethernet/pensando/ionic/ionic_ethtool.c +5 -0
@@ -721,6 +721,11 @@ static int ionic_set_channels(struct net_device *netdev,ionic_init_queue_params (lif, &qparam); +if ((ch->rx_count || ch->tx_count) && lif->xdp_prog) { +netdev_info (lif->netdev,"Split Tx/Rx interrupts not available when using XDP\n" ); +return -EOPNOTSUPP ; + } +if (ch->rx_count != ch->tx_count) {netdev_info (netdev,"The rx and tx count must be equal\n" );return -EINVAL ;
drivers/net/ethernet/pensando/ionic/ionic_lif.c +170 -2
245 lines not on this page · raw message
drivers/net/ethernet/pensando/ionic/ionic_lif.h +7 -0
@@ -51,6 +51,9 @@ struct ionic_rx_stats {u64 alloc_err;u64 hwstamp_valid;u64 hwstamp_invalid; +u64 xdp_drop; +u64 xdp_aborted; +u64 xdp_pass; };#define IONIC_QCQ_F_INITED BIT(0)
@@ -135,6 +138,9 @@ struct ionic_lif_sw_stats {u64 hw_rx_over_errors;u64 hw_rx_missed_errors;u64 hw_tx_aborted_errors; +u64 xdp_drop; +u64 xdp_aborted; +u64 xdp_pass; };enum ionic_lif_state_flags {
@@ -230,6 +236,7 @@ struct ionic_lif {struct ionic_phc *phc;struct dentry *dentry; +struct bpf_prog *xdp_prog; };struct ionic_phc {
drivers/net/ethernet/pensando/ionic/ionic_stats.c +9 -0
@@ -27,6 +27,9 @@ static const struct ionic_stat_desc ionic_lif_stats_desc[] = {IONIC_LIF_STAT_DESC (hw_rx_over_errors),IONIC_LIF_STAT_DESC (hw_rx_missed_errors ),IONIC_LIF_STAT_DESC (hw_tx_aborted_errors ), +IONIC_LIF_STAT_DESC (xdp_drop ), +IONIC_LIF_STAT_DESC (xdp_aborted ), + IONIC_LIF_STAT_DESC(xdp_pass), };static const struct ionic_stat_desc ionic_port_stats_desc[] = {
@@ -149,6 +152,9 @@ static const struct ionic_stat_desc ionic_rx_stats_desc[] = {IONIC_RX_STAT_DESC (hwstamp_invalid),IONIC_RX_STAT_DESC (dropped),IONIC_RX_STAT_DESC (vlan_stripped), +IONIC_RX_STAT_DESC (xdp_drop), +IONIC_RX_STAT_DESC (xdp_aborted), + IONIC_RX_STAT_DESC(xdp_pass), };#define IONIC_NUM_LIF_STATS ARRAY_SIZE(ionic_lif_stats_desc)
@@ -185,6 +191,9 @@ static void ionic_add_lif_rxq_stats(struct ionic_lif *lif, int q_num, stats->rx_csum_error += rxstats->csum_error; stats->rx_hwstamp_valid += rxstats->hwstamp_valid; stats->rx_hwstamp_invalid += rxstats->hwstamp_invalid; + stats->xdp_drop += rxstats->xdp_drop; + stats->xdp_aborted += rxstats->xdp_aborted; + stats->xdp_pass += rxstats->xdp_pass; } static void ionic_get_lif_stats(struct ionic_lif *lif,
drivers/net/ethernet/pensando/ionic/ionic_txrx.c +57 -4
93 lines not on this page · raw message
Signature · 1 line
2.17.1
RFC PATCH 5/9 ionic: Add XDP packet headroom
In reply to Shannon NelsonIf an xdp program is loaded, add headroom at the beginning of the frame to allow for editing and insertions that an XDP program might need room for, and tailroom used later for XDP frame tracking. These are only needed in the first Rx buffer in a packet, not for any trailing frags.
--- .../net/ethernet/pensando/ionic/ionic_txrx.c | 67 ++++++++++++------- 1 file changed, 44 insertions(+), 23 deletions(-)
drivers/net/ethernet/pensando/ionic/ionic_txrx.c +44 -23
@@ -189,7 +189,9 @@ static bool ionic_rx_buf_recycle(struct ionic_queue *q,static struct sk_buff *ionic_rx_frags (struct ionic_queue *q,struct ionic_desc_info *desc_info, -struct ionic_rxq_comp *comp) +unsigned int headroom, +unsigned int len, +unsigned int num_sg_elems) {struct net_device *netdev = q->lif->netdev;struct ionic_buf_info *buf_info;
@@ -198,12 +200,10 @@ static struct sk_buff *ionic_rx_frags(struct ionic_queue *q,struct sk_buff *skb;unsigned int i;u16 frag_len; -u16 len; stats =q_to_rx_stats (q); buf_info = &desc_info->bufs[0 ]; - len =le16_to_cpu (comp->len);prefetchw (buf_info->page);
@@ -215,22 +215,25 @@ static struct sk_buff *ionic_rx_frags(struct ionic_queue *q,return NULL ; } - i = comp->num_sg_elems +1 ; + i = num_sg_elems +1 ; do { if (unlikely(!buf_info->page)) { dev_kfree_skb(skb); return NULL; } - frag_len = min_t(u16, len, ionic_rx_buf_size(buf_info)); + if (headroom) + frag_len = min_t(u16, len, IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN); + else + frag_len = min_t(u16, len, ionic_rx_buf_size(buf_info)); len -= frag_len; dma_sync_single_range_for_cpu(dev, ionic_rx_buf_pa(buf_info), - 0, frag_len, DMA_FROM_DEVICE); + headroom, frag_len, DMA_FROM_DEVICE); skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, - buf_info->page, buf_info->page_offset, frag_len, - IONIC_PAGE_SIZE); + buf_info->page, buf_info->page_offset + headroom, + frag_len, IONIC_PAGE_SIZE); if (!ionic_rx_buf_recycle(q, buf_info, frag_len)) { dma_unmap_page(dev, buf_info->dma_addr,
@@ -238,6 +241,10 @@ static struct sk_buff *ionic_rx_frags(struct ionic_queue *q, buf_info->page =NULL ; } +/* only needed on the first buffer */ +if (headroom) + headroom =0 ; + buf_info++; i--;
@@ -248,19 +255,18 @@ static struct sk_buff *ionic_rx_frags(struct ionic_queue *q,static struct sk_buff *ionic_rx_copybreak (struct ionic_queue *q,struct ionic_desc_info *desc_info, -struct ionic_rxq_comp *comp) +unsigned int headroom, +unsigned int len) {struct net_device *netdev = q->lif->netdev;struct ionic_buf_info *buf_info;struct ionic_rx_stats *stats;struct device *dev = q->dev;struct sk_buff *skb; -u16 len; stats =q_to_rx_stats (q); buf_info = &desc_info->bufs[0 ]; - len =le16_to_cpu (comp->len); skb =napi_alloc_skb (&q_to_qcq (q)->napi, len);if (unlikely (!skb)) {
@@ -276,10 +282,10 @@ static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q, }dma_sync_single_range_for_cpu (dev,ionic_rx_buf_pa (buf_info), -0 , len,DMA_FROM_DEVICE ); -skb_copy_to_linear_data (skb,ionic_rx_buf_va (buf_info), len); + headroom, len,DMA_FROM_DEVICE ); +skb_copy_to_linear_data (skb,ionic_rx_buf_va (buf_info) + headroom, len);dma_sync_single_range_for_device (dev,ionic_rx_buf_pa (buf_info), -0 , len,DMA_FROM_DEVICE ); + headroom, len,DMA_FROM_DEVICE );skb_put (skb, len); skb->protocol =eth_type_trans (skb, q->lif->netdev);
@@ -303,10 +309,10 @@ static bool ionic_run_xdp(struct ionic_rx_stats *stats,xdp_init_buff (&xdp_buf,IONIC_PAGE_SIZE , rxq->xdp_rxq_info);xdp_prepare_buff (&xdp_buf,ionic_rx_buf_va (buf_info), -0 , len, false); +XDP_PACKET_HEADROOM , len, false);dma_sync_single_range_for_cpu (rxq->dev,ionic_rx_buf_pa (buf_info), -0 , len, +XDP_PACKET_HEADROOM , len,DMA_FROM_DEVICE );prefetchw (&xdp_buf.data_hard_start);
@@ -344,6 +350,7 @@ static void ionic_rx_clean(struct ionic_queue *q,struct ionic_qcq *qcq =q_to_qcq (q);struct ionic_rx_stats *stats;struct ionic_rxq_comp *comp; +unsigned int headroom;struct sk_buff *skb;u16 len;
@@ -363,10 +370,11 @@ static void ionic_rx_clean(struct ionic_queue *q,if (ionic_run_xdp (stats, netdev, q, desc_info->bufs, len))return ; + headroom = q->xdp_rxq_info ?XDP_PACKET_HEADROOM :0 ;if (len <= q->lif->rx_copybreak) - skb =ionic_rx_copybreak (q, desc_info, comp); + skb =ionic_rx_copybreak (q, desc_info, headroom, len);else - skb =ionic_rx_frags (q, desc_info, comp); + skb =ionic_rx_frags (q, desc_info, headroom, len, comp->num_sg_elems);if (unlikely (!skb)) { stats->dropped++;
@@ -490,8 +498,9 @@ void ionic_rx_fill(struct ionic_queue *q)unsigned int frag_len;unsigned int nfrags;unsigned int n_fill; -unsigned int i, j;unsigned int len; +unsigned int i; +unsigned int j; n_fill =ionic_q_space_avail (q);
@@ -500,9 +509,12 @@ void ionic_rx_fill(struct ionic_queue *q)if (n_fill < fill_threshold)return ; - len = netdev->mtu +ETH_HLEN +VLAN_HLEN ; + len = netdev->mtu +VLAN_ETH_HLEN ;for (i = n_fill; i; i--) { +unsigned int headroom; +unsigned int buf_len; + nfrags =0 ; remain_len = len; desc_info = &q->info[q->head_idx];
@@ -517,9 +529,18 @@ void ionic_rx_fill(struct ionic_queue *q) } } -/* fill main descriptor - buf[0] */ - desc->addr =cpu_to_le64 (ionic_rx_buf_pa (buf_info)); - frag_len =min_t (u16, len,ionic_rx_buf_size (buf_info)); +/* fill main descriptor - buf[0] +* XDP uses space in the first buffer, so account for +* head room, tail room, and ip header in the first frag size. +*/ + headroom = q->xdp_rxq_info ?XDP_PACKET_HEADROOM :0 ; +if (q->xdp_rxq_info) + buf_len =IONIC_XDP_MAX_LINEAR_MTU +VLAN_ETH_HLEN ; +else + buf_len =ionic_rx_buf_size (buf_info); + frag_len =min_t (u16, len, buf_len); + + desc->addr =cpu_to_le64 (ionic_rx_buf_pa (buf_info) + headroom); desc->len =cpu_to_le16 (frag_len); remain_len -= frag_len; buf_info++;
Signature · 1 line
2.17.1
RFC PATCH 6/9 ionic: Add XDP_TX support
In reply to Shannon NelsonThe XDP_TX packets get fed back into the Rx queue's partnered Tx queue as an xdp_frame.
--- .../net/ethernet/pensando/ionic/ionic_dev.h | 3 + .../net/ethernet/pensando/ionic/ionic_lif.c | 4 + .../net/ethernet/pensando/ionic/ionic_lif.h | 4 + .../net/ethernet/pensando/ionic/ionic_stats.c | 6 + .../net/ethernet/pensando/ionic/ionic_txrx.c | 137 +++++++++++++++++- 5 files changed, 151 insertions(+), 3 deletions(-)
drivers/net/ethernet/pensando/ionic/ionic_dev.h +3 -0
@@ -228,6 +228,8 @@ struct ionic_desc_info {struct ionic_buf_info bufs[MAX_SKB_FRAGS +1 ];ionic_desc_cb cb;void *cb_arg; +struct xdp_frame *xdpf; +enum xdp_action act; };#define IONIC_QUEUE_NAME_MAX_SZ 16
@@ -263,6 +265,7 @@ struct ionic_queue {struct ionic_rxq_sg_desc *rxq_sgl; };struct xdp_rxq_info *xdp_rxq_info; +struct ionic_queue *partner;dma_addr_t base_pa;dma_addr_t cmb_base_pa;dma_addr_t sg_base_pa;
drivers/net/ethernet/pensando/ionic/ionic_lif.c +4 -0
@@ -878,6 +878,9 @@ static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq) };int err; + q->partner = &lif->txqcqs[q->index]->q; + q->partner->partner = q; +if (!lif->xdp_prog) ctx.cmd.q_init.flags |=cpu_to_le16 (IONIC_QINIT_F_SG );
@@ -2923,6 +2926,7 @@ static void ionic_swap_queues(struct ionic_qcq *a, struct ionic_qcq *b)swap (a->q.base_pa, b->q.base_pa);swap (a->q.info, b->q.info);swap (a->q.xdp_rxq_info, b->q.xdp_rxq_info); +swap (a->q.partner, b->q.partner);swap (a->q_base, b->q_base);swap (a->q_base_pa, b->q_base_pa);swap (a->q_size, b->q_size);
drivers/net/ethernet/pensando/ionic/ionic_lif.h +4 -0
@@ -37,6 +37,7 @@ struct ionic_tx_stats {u64 dma_map_err;u64 hwstamp_valid;u64 hwstamp_invalid; +u64 xdp_frames; };struct ionic_rx_stats {
@@ -54,6 +55,7 @@ struct ionic_rx_stats {u64 xdp_drop;u64 xdp_aborted;u64 xdp_pass; +u64 xdp_tx; };#define IONIC_QCQ_F_INITED BIT(0)
@@ -141,6 +143,8 @@ struct ionic_lif_sw_stats {u64 xdp_drop;u64 xdp_aborted;u64 xdp_pass; +u64 xdp_tx; +u64 xdp_frames; };enum ionic_lif_state_flags {
drivers/net/ethernet/pensando/ionic/ionic_stats.c +6 -0
@@ -30,6 +30,8 @@ static const struct ionic_stat_desc ionic_lif_stats_desc[] = {IONIC_LIF_STAT_DESC (xdp_drop),IONIC_LIF_STAT_DESC (xdp_aborted ),IONIC_LIF_STAT_DESC (xdp_pass ), +IONIC_LIF_STAT_DESC (xdp_tx ), + IONIC_LIF_STAT_DESC(xdp_frames), };static const struct ionic_stat_desc ionic_port_stats_desc[] = {
@@ -138,6 +140,7 @@ static const struct ionic_stat_desc ionic_tx_stats_desc[] = {IONIC_TX_STAT_DESC (csum_none),IONIC_TX_STAT_DESC (csum ),IONIC_TX_STAT_DESC (vlan_inserted ), + IONIC_TX_STAT_DESC(xdp_frames), };static const struct ionic_stat_desc ionic_rx_stats_desc[] = {
@@ -155,6 +158,7 @@ static const struct ionic_stat_desc ionic_rx_stats_desc[] = {IONIC_RX_STAT_DESC (xdp_drop),IONIC_RX_STAT_DESC (xdp_aborted),IONIC_RX_STAT_DESC (xdp_pass), + IONIC_RX_STAT_DESC(xdp_tx), };#define IONIC_NUM_LIF_STATS ARRAY_SIZE(ionic_lif_stats_desc)
@@ -177,6 +181,7 @@ static void ionic_add_lif_txq_stats(struct ionic_lif *lif, int q_num, stats->tx_csum += txstats->csum; stats->tx_hwstamp_valid += txstats->hwstamp_valid; stats->tx_hwstamp_invalid += txstats->hwstamp_invalid; + stats->xdp_frames += txstats->xdp_frames; } static void ionic_add_lif_rxq_stats(struct ionic_lif *lif, int q_num,
@@ -194,6 +199,7 @@ static void ionic_add_lif_rxq_stats(struct ionic_lif *lif, int q_num, stats->xdp_drop += rxstats->xdp_drop; stats->xdp_aborted += rxstats->xdp_aborted; stats->xdp_pass += rxstats->xdp_pass; + stats->xdp_tx += rxstats->xdp_tx; } static void ionic_get_lif_stats(struct ionic_lif *lif,
drivers/net/ethernet/pensando/ionic/ionic_txrx.c +134 -3
172 lines not on this page · raw message
Signature · 1 line
2.17.1
RFC PATCH 7/9 ionic: Add XDP_REDIRECT support
In reply to Shannon NelsonThe XDP_REDIRECT packets are given to the XDP stack and we drop the use of the related page: it will get freed by the driver that ends up doing the Tx. Because we have some hardware configurations with limited queue resources, we use the existing datapath Tx queues rather than creating and managing a separate set of xdp_tx queues.
--- .../net/ethernet/pensando/ionic/ionic_dev.h | 1 + .../net/ethernet/pensando/ionic/ionic_lif.c | 3 ++- .../net/ethernet/pensando/ionic/ionic_lif.h | 2 ++ .../net/ethernet/pensando/ionic/ionic_stats.c | 3 +++ .../net/ethernet/pensando/ionic/ionic_txrx.c | 24 ++++++++++++++++++- 5 files changed, 31 insertions(+), 2 deletions(-)
drivers/net/ethernet/pensando/ionic/ionic_dev.h +1 -0
@@ -266,6 +266,7 @@ struct ionic_queue { };struct xdp_rxq_info *xdp_rxq_info;struct ionic_queue *partner; +bool xdp_flush;dma_addr_t base_pa;dma_addr_t cmb_base_pa;dma_addr_t sg_base_pa;
drivers/net/ethernet/pensando/ionic/ionic_lif.c +2 -1
@@ -1649,7 +1649,8 @@ static int ionic_init_nic_features(struct ionic_lif *lif) netdev->priv_flags |=IFF_UNICAST_FLT |IFF_LIVE_ADDR_CHANGE ; - netdev->xdp_features =NETDEV_XDP_ACT_BASIC ; + netdev->xdp_features =NETDEV_XDP_ACT_BASIC | +NETDEV_XDP_ACT_REDIRECT ;return 0 ; }
drivers/net/ethernet/pensando/ionic/ionic_lif.h +2 -0
@@ -56,6 +56,7 @@ struct ionic_rx_stats {u64 xdp_aborted;u64 xdp_pass;u64 xdp_tx; +u64 xdp_redirect; };#define IONIC_QCQ_F_INITED BIT(0)
@@ -144,6 +145,7 @@ struct ionic_lif_sw_stats {u64 xdp_aborted;u64 xdp_pass;u64 xdp_tx; +u64 xdp_redirect;u64 xdp_frames; };
drivers/net/ethernet/pensando/ionic/ionic_stats.c +3 -0
@@ -31,6 +31,7 @@ static const struct ionic_stat_desc ionic_lif_stats_desc[] = {IONIC_LIF_STAT_DESC (xdp_aborted),IONIC_LIF_STAT_DESC (xdp_pass),IONIC_LIF_STAT_DESC (xdp_tx), +IONIC_LIF_STAT_DESC (xdp_redirect), IONIC_LIF_STAT_DESC(xdp_frames), };
@@ -159,6 +160,7 @@ static const struct ionic_stat_desc ionic_rx_stats_desc[] = {IONIC_RX_STAT_DESC (xdp_aborted),IONIC_RX_STAT_DESC (xdp_pass),IONIC_RX_STAT_DESC (xdp_tx), + IONIC_RX_STAT_DESC(xdp_redirect), };#define IONIC_NUM_LIF_STATS ARRAY_SIZE(ionic_lif_stats_desc)
@@ -200,6 +202,7 @@ static void ionic_add_lif_rxq_stats(struct ionic_lif *lif, int q_num, stats->xdp_aborted += rxstats->xdp_aborted; stats->xdp_pass += rxstats->xdp_pass; stats->xdp_tx += rxstats->xdp_tx; + stats->xdp_redirect += rxstats->xdp_redirect; } static void ionic_get_lif_stats(struct ionic_lif *lif,
drivers/net/ethernet/pensando/ionic/ionic_txrx.c +23 -1
@@ -413,7 +413,19 @@ static bool ionic_run_xdp(struct ionic_rx_stats *stats,break ;case XDP_REDIRECT : - goto out_xdp_abort; +/* unmap the pages before handing them to a different device */ +dma_unmap_page (rxq->dev, buf_info->dma_addr, +IONIC_PAGE_SIZE ,DMA_FROM_DEVICE ); + + err =xdp_do_redirect (netdev, &xdp_buf, xdp_prog); +if (err) { +netdev_dbg (netdev,"xdp_do_redirect err %d\n" , err); + goto out_xdp_abort; + } + buf_info->page =NULL ; + rxq->xdp_flush = true; + stats->xdp_redirect++; +break ;case XDP_TX : xdpf =xdp_convert_buff_to_frame (&xdp_buf);
@@ -803,6 +815,14 @@ int ionic_tx_napi(struct napi_struct *napi, int budget)return work_done; } +static void ionic_xdp_do_flush (struct ionic_cq *cq) +{ +if (cq->bound_q->xdp_flush) { +xdp_do_flush (); + cq->bound_q->xdp_flush = false; + } +} +int ionic_rx_napi (struct napi_struct *napi,int budget) {struct ionic_qcq *qcq =napi_to_qcq (napi);
@@ -820,6 +840,7 @@ int ionic_rx_napi(struct napi_struct *napi, int budget)ionic_rx_fill (cq->bound_q); +ionic_xdp_do_flush (cq);if (work_done < budget &&napi_complete_done (napi, work_done)) {ionic_dim_update (qcq,IONIC_LIF_F_RX_DIM_INTR ); flags |=IONIC_INTR_CRED_UNMASK ;
@@ -866,6 +887,7 @@ int ionic_txrx_napi(struct napi_struct *napi, int budget)ionic_rx_fill (rxcq->bound_q); +ionic_xdp_do_flush (rxcq);if (rx_work_done < budget &&napi_complete_done (napi, rx_work_done)) {ionic_dim_update (rxqcq,0 ); flags |=IONIC_INTR_CRED_UNMASK ;
Signature · 1 line
2.17.1
RFC PATCH 8/9 ionic: add ndo_xdp_xmit
In reply to Shannon NelsonWhen our ndo_xdp_xmit is called we mark the buffer with XDP_REDIRECT so we know to return it to the XDP stack for cleaning.
--- .../net/ethernet/pensando/ionic/ionic_lif.c | 4 +- .../net/ethernet/pensando/ionic/ionic_txrx.c | 62 ++++++++++++++++++- .../net/ethernet/pensando/ionic/ionic_txrx.h | 1 + 3 files changed, 65 insertions(+), 2 deletions(-)
drivers/net/ethernet/pensando/ionic/ionic_lif.c +3 -1
@@ -1650,7 +1650,8 @@ static int ionic_init_nic_features(struct ionic_lif *lif)IFF_LIVE_ADDR_CHANGE ; netdev->xdp_features =NETDEV_XDP_ACT_BASIC | -NETDEV_XDP_ACT_REDIRECT ; +NETDEV_XDP_ACT_REDIRECT | +NETDEV_XDP_ACT_NDO_XMIT ;return 0 ; }
@@ -2844,6 +2845,7 @@ static const struct net_device_ops ionic_netdev_ops = { .ndo_eth_ioctl = ionic_eth_ioctl, .ndo_start_xmit = ionic_start_xmit, .ndo_bpf = ionic_xdp, + .ndo_xdp_xmit = ionic_xdp_xmit, .ndo_get_stats64 = ionic_get_stats64, .ndo_set_rx_mode = ionic_ndo_set_rx_mode, .ndo_set_features = ionic_set_features,
drivers/net/ethernet/pensando/ionic/ionic_txrx.c +61 -1
76 lines not on this page · raw message
drivers/net/ethernet/pensando/ionic/ionic_txrx.h +1 -0
@@ -17,4 +17,5 @@ netdev_tx_t ionic_start_xmit(struct sk_buff *skb, struct net_device *netdev);bool ionic_rx_service (struct ionic_cq *cq,struct ionic_cq_info *cq_info);bool ionic_tx_service (struct ionic_cq *cq,struct ionic_cq_info *cq_info); +int ionic_xdp_xmit (struct net_device *netdev,int n,struct xdp_frame **xdp,u32 flags);#endif /* _IONIC_TXRX_H_ */
Signature · 1 line
2.17.1
RFC PATCH 9/9 ionic: implement xdp frags support
In reply to Shannon NelsonAdd support for using scatter-gather / frags in XDP in both Rx and Tx paths.
--- .../net/ethernet/pensando/ionic/ionic_lif.c | 12 ++- .../net/ethernet/pensando/ionic/ionic_txrx.c | 91 ++++++++++++++++++- 2 files changed, 98 insertions(+), 5 deletions(-)
drivers/net/ethernet/pensando/ionic/ionic_lif.c +9 -3
@@ -881,7 +881,8 @@ static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq) q->partner = &lif->txqcqs[q->index]->q; q->partner->partner = q; -if (!lif->xdp_prog) +if (!lif->xdp_prog || + (lif->xdp_prog->aux && lif->xdp_prog->aux->xdp_has_frags)) ctx.cmd.q_init.flags |=cpu_to_le16 (IONIC_QINIT_F_SG );if (qcq->flags &IONIC_QCQ_F_CMB_RINGS ) {
@@ -1651,7 +1652,9 @@ static int ionic_init_nic_features(struct ionic_lif *lif) netdev->xdp_features =NETDEV_XDP_ACT_BASIC |NETDEV_XDP_ACT_REDIRECT | -NETDEV_XDP_ACT_NDO_XMIT ; +NETDEV_XDP_ACT_RX_SG | +NETDEV_XDP_ACT_NDO_XMIT | +NETDEV_XDP_ACT_NDO_XMIT_SG ;return 0 ; }
@@ -1799,6 +1802,9 @@ static bool ionic_xdp_is_valid_mtu(struct ionic_lif *lif, u32 mtu,if (mtu <=IONIC_XDP_MAX_LINEAR_MTU )return true; +if (xdp_prog->aux && xdp_prog->aux->xdp_has_frags) +return true; +return false; }
@@ -2809,7 +2815,7 @@ static int ionic_xdp_config(struct net_device *netdev, struct netdev_bpf *bpf) } maxfs =__le32_to_cpu (lif->identity->eth.max_frame_size) -VLAN_ETH_HLEN ; -if (bpf->prog) +if (bpf->prog && !(bpf->prog->aux && bpf->prog->aux->xdp_has_frags)) maxfs =min_t (u32, maxfs,IONIC_XDP_MAX_LINEAR_MTU ); netdev->max_mtu = maxfs;
drivers/net/ethernet/pensando/ionic/ionic_txrx.c +89 -2
135 lines not on this page · raw message
Signature · 1 line
2.17.1