bnxt: Support XDP multi buffer
Subsystems (5)
Michael Chan
maintainerv3 cover bnxt: Support XDP multi buffer
This series adds XDP multi buffer support, allowing MTU to go beyond the page size limit.
v3: Simplify page mode buffer size calculation
Check to make sure XDP program supports multipage packets
v2: Fix uninitialized variable warnings in patch 1 and 10.
Andy Gospodarek (11):
bnxt: refactor bnxt_rx_xdp to separate xdp_init_buff/xdp_prepare_buff
bnxt: add flag to denote that an xdp program is currently attached
bnxt: refactor bnxt_rx_pages operate on skb_shared_info
bnxt: rename bnxt_rx_pages to bnxt_rx_agg_pages_skb
bnxt: adding bnxt_rx_agg_pages_xdp for aggregated xdp
bnxt: set xdp_buff pfmemalloc flag if needed
bnxt: change receive ring space parameters
bnxt: add page_pool support for aggregation ring when using xdp
bnxt: adding bnxt_xdp_build_skb to build skb from multibuffer xdp_buff
bnxt: support transmit and free of aggregation buffers
bnxt: XDP multibuffer enablement
drivers/net/ethernet/broadcom/bnxt/bnxt.c | 304 +++++++++++++----- drivers/net/ethernet/broadcom/bnxt/bnxt.h | 16 +- .../net/ethernet/broadcom/bnxt/bnxt_ethtool.c | 2 +- drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c | 191 +++++++++-- drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h | 16 +- 5 files changed, 416 insertions(+), 113 deletions(-)
Signature · 1 line
2.18.1
Michael Chan
maintainerPATCH v3 1/11 bnxt: refactor bnxt_rx_xdp to separate xdp_init_buff/xdp_prepare_buff
From: Andy Gospodarek <[email protected]>
Move initialization of xdp_buff outside of bnxt_rx_xdp to prepare for allowing bnxt_rx_xdp to operate on multibuffer xdp_buffs.
v2: Fix uninitalized variables warning in bnxt_xdp.c.
v3: Add new define BNXT_PAGE_MODE_BUF_SIZE
--- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 11 +++-- drivers/net/ethernet/broadcom/bnxt/bnxt.h | 8 ++-- drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c | 46 ++++++++++++++----- drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h | 7 ++- 4 files changed, 53 insertions(+), 19 deletions(-)
drivers/net/ethernet/broadcom/bnxt/bnxt.c +7 -4
@@ -1731,6 +1731,7 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,u8 *data_ptr, agg_bufs, cmp_type;dma_addr_t dma_addr;struct sk_buff *skb; +struct xdp_buff xdp;u32 flags, misc;void *data;int rc =0 ;
@@ -1839,11 +1840,13 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, len = flags >>RX_CMP_LEN_SHIFT ; dma_addr = rx_buf->mapping; -if (bnxt_rx_xdp (bp, rxr, cons, data, &data_ptr, &len, event)) { - rc =1 ; - goto next_rx; +if (bnxt_xdp_attached (bp, rxr)) { +bnxt_xdp_buff_init (bp, rxr, cons, &data_ptr, &len, &xdp); +if (bnxt_rx_xdp (bp, rxr, cons, xdp, data, &len, event)) { + rc =1 ; + goto next_rx; + } } -if (len <= bp->rx_copy_thresh) { skb =bnxt_copy_skb (bnapi, data_ptr, len, dma_addr);bnxt_reuse_rx_data (rxr, cons, data);
drivers/net/ethernet/broadcom/bnxt/bnxt.h +5 -3
@@ -591,10 +591,12 @@ struct nqe_cn {#define BNXT_RX_PAGE_SIZE (1 << BNXT_RX_PAGE_SHIFT)#define BNXT_MAX_MTU 9500 -#define BNXT_MAX_PAGE_MODE_MTU \ +#define BNXT_PAGE_MODE_BUF_SIZE \ ((unsigned int)PAGE_SIZE - VLAN_ETH_HLEN - NET_IP_ALIGN - \ - XDP_PACKET_HEADROOM - \ - SKB_DATA_ALIGN((unsigned int)sizeof(struct skb_shared_info))) + XDP_PACKET_HEADROOM) +#define BNXT_MAX_PAGE_MODE_MTU \ + BNXT_PAGE_MODE_BUF_SIZE - \ + SKB_DATA_ALIGN((unsigned int)sizeof(struct skb_shared_info))#define BNXT_MIN_PKT_SIZE 52
drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c +35 -11
90 lines not on this page · raw message
drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h +6 -1
@@ -17,10 +17,15 @@ struct bnxt_sw_tx_bd *bnxt_xmit_bd(struct bnxt *bp, dma_addr_t mapping, u32 len);void bnxt_tx_int_xdp (struct bnxt *bp,struct bnxt_napi *bnapi,int nr_pkts);bool bnxt_rx_xdp (struct bnxt *bp,struct bnxt_rx_ring_info *rxr,u16 cons, -struct page *page,u8 **data_ptr,unsigned int *len, +struct xdp_buff xdp,struct page *page,unsigned int *len,u8 *event);int bnxt_xdp (struct net_device *dev,struct netdev_bpf *xdp);int bnxt_xdp_xmit (struct net_device *dev,int num_frames,struct xdp_frame **frames,u32 flags); +bool bnxt_xdp_attached (struct bnxt *bp,struct bnxt_rx_ring_info *rxr); + +void bnxt_xdp_buff_init (struct bnxt *bp,struct bnxt_rx_ring_info *rxr, +u16 cons,u8 **data_ptr,unsigned int *len, +struct xdp_buff *xdp);#endif
Signature · 1 line
2.18.1
Michael Chan
maintainerPATCH v3 2/11 bnxt: add flag to denote that an xdp program is currently attached
In reply to Michael ChanFrom: Andy Gospodarek <[email protected]>
This will be used to determine if bnxt_rx_xdp should be called rather than calling it every time.
--- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 7 +++++++ 1 file changed, 7 insertions(+)
drivers/net/ethernet/broadcom/bnxt/bnxt.c +7 -0
@@ -1729,6 +1729,7 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,struct bnxt_sw_rx_bd *rx_buf;unsigned int len;u8 *data_ptr, agg_bufs, cmp_type; +bool xdp_active = false;dma_addr_t dma_addr;struct sk_buff *skb;struct xdp_buff xdp;
@@ -1842,11 +1843,17 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,if (bnxt_xdp_attached (bp, rxr)) {bnxt_xdp_buff_init (bp, rxr, cons, &data_ptr, &len, &xdp); + xdp_active = true; + } + +/* skip running XDP prog if there are aggregation bufs */ +if (!agg_bufs && xdp_active) {if (bnxt_rx_xdp (bp, rxr, cons, xdp, data, &len, event)) { rc =1 ; goto next_rx; } } +if (len <= bp->rx_copy_thresh) { skb =bnxt_copy_skb (bnapi, data_ptr, len, dma_addr);bnxt_reuse_rx_data (rxr, cons, data);
Signature · 1 line
2.18.1
Michael Chan
maintainerPATCH v3 3/11 bnxt: refactor bnxt_rx_pages operate on skb_shared_info
In reply to Michael ChanFrom: Andy Gospodarek <[email protected]>
Rather than operating on an sk_buff, add frags from the aggregation ring into the frags of an skb_shared_info. This will allow the caller to use either an sk_buff or xdp_buff.
--- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 50 +++++++++++++++-------- 1 file changed, 33 insertions(+), 17 deletions(-)
drivers/net/ethernet/broadcom/bnxt/bnxt.c +33 -17
@@ -1038,22 +1038,23 @@ static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,return skb; } -static struct sk_buff *bnxt_rx_pages (struct bnxt *bp, -struct bnxt_cp_ring_info *cpr, -struct sk_buff *skb,u16 idx, -u32 agg_bufs,bool tpa) +static u32 __bnxt_rx_pages (struct bnxt *bp, +struct bnxt_cp_ring_info *cpr, +struct skb_shared_info *shinfo, +u16 idx,u32 agg_bufs,bool tpa) {struct bnxt_napi *bnapi = cpr->bnapi;struct pci_dev *pdev = bp->pdev;struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;u16 prod = rxr->rx_agg_prod; +u32 i, total_frag_len =0 ;bool p5_tpa = false; -u32 i;if ((bp->flags &BNXT_FLAG_CHIP_P5 ) && tpa) p5_tpa = true;for (i =0 ; i < agg_bufs; i++) { +skb_frag_t *frag = &shinfo->frags[i];u16 cons, frag_len;struct rx_agg_cmp *agg;struct bnxt_sw_rx_agg_bd *cons_rx_buf;
@@ -1069,8 +1070,10 @@ static struct sk_buff *bnxt_rx_pages(struct bnxt *bp, RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT; cons_rx_buf = &rxr->rx_agg_ring[cons]; -skb_fill_page_desc (skb, i, cons_rx_buf->page, - cons_rx_buf->offset, frag_len); +skb_frag_off_set (frag, cons_rx_buf->offset); +skb_frag_size_set (frag, frag_len); +__skb_frag_set_page (frag, cons_rx_buf->page); + shinfo->nr_frags = i +1 ;__clear_bit (cons, rxr->rx_agg_bmap);/* It is possible for bnxt_alloc_rx_page() to allocate
@@ -1082,15 +1085,10 @@ static struct sk_buff *bnxt_rx_pages(struct bnxt *bp, cons_rx_buf->page =NULL ;if (bnxt_alloc_rx_page (bp, rxr, prod,GFP_ATOMIC ) !=0 ) { -struct skb_shared_info *shinfo;unsigned int nr_frags; - shinfo =skb_shinfo (skb); nr_frags = --shinfo->nr_frags;__skb_frag_set_page (&shinfo->frags[nr_frags],NULL ); - -dev_kfree_skb (skb); - cons_rx_buf->page = page;/* Update prod since possibly some pages have been
@@ -1098,20 +1096,38 @@ static struct sk_buff *bnxt_rx_pages(struct bnxt *bp,*/ rxr->rx_agg_prod = prod;bnxt_reuse_rx_agg_bufs (cpr, idx, i, agg_bufs - i, tpa); -return NULL ; +return 0 ; }dma_unmap_page_attrs (&pdev->dev, mapping,BNXT_RX_PAGE_SIZE ,DMA_FROM_DEVICE ,DMA_ATTR_WEAK_ORDERING ); - skb->data_len += frag_len; - skb->len += frag_len; - skb->truesize +=PAGE_SIZE ; - + total_frag_len += frag_len; prod =NEXT_RX_AGG (prod); } rxr->rx_agg_prod = prod; +return total_frag_len; +} + +static struct sk_buff *bnxt_rx_pages (struct bnxt *bp, +struct bnxt_cp_ring_info *cpr, +struct sk_buff *skb,u16 idx, +u32 agg_bufs,bool tpa) +{ +struct skb_shared_info *shinfo =skb_shinfo (skb); +u32 total_frag_len =0 ; + + total_frag_len =__bnxt_rx_pages (bp, cpr, shinfo, idx, agg_bufs, tpa); + +if (!total_frag_len) { +dev_kfree_skb (skb); +return NULL ; + } + + skb->data_len += total_frag_len; + skb->len += total_frag_len; + skb->truesize +=PAGE_SIZE * agg_bufs;return skb; }
Signature · 1 line
2.18.1
Michael Chan
maintainerPATCH v3 4/11 bnxt: rename bnxt_rx_pages to bnxt_rx_agg_pages_skb
In reply to Michael ChanFrom: Andy Gospodarek <[email protected]>
Clarify that this is reading buffers from the aggregation ring.
--- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-)
drivers/net/ethernet/broadcom/bnxt/bnxt.c +11 -11
@@ -1038,10 +1038,10 @@ static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,return skb; } -static u32 __bnxt_rx_pages (struct bnxt *bp, -struct bnxt_cp_ring_info *cpr, -struct skb_shared_info *shinfo, -u16 idx,u32 agg_bufs,bool tpa) +static u32 __bnxt_rx_agg_pages (struct bnxt *bp, +struct bnxt_cp_ring_info *cpr, +struct skb_shared_info *shinfo, +u16 idx,u32 agg_bufs,bool tpa) {struct bnxt_napi *bnapi = cpr->bnapi;struct pci_dev *pdev = bp->pdev;
@@ -1110,15 +1110,15 @@ static u32 __bnxt_rx_pages(struct bnxt *bp,return total_frag_len; } -static struct sk_buff *bnxt_rx_pages (struct bnxt *bp, -struct bnxt_cp_ring_info *cpr, -struct sk_buff *skb,u16 idx, -u32 agg_bufs,bool tpa) +static struct sk_buff *bnxt_rx_agg_pages_skb (struct bnxt *bp, +struct bnxt_cp_ring_info *cpr, +struct sk_buff *skb,u16 idx, +u32 agg_bufs,bool tpa) {struct skb_shared_info *shinfo =skb_shinfo (skb);u32 total_frag_len =0 ; - total_frag_len =__bnxt_rx_pages (bp, cpr, shinfo, idx, agg_bufs, tpa); + total_frag_len =__bnxt_rx_agg_pages (bp, cpr, shinfo, idx, agg_bufs, tpa);if (!total_frag_len) {dev_kfree_skb (skb);
@@ -1660,7 +1660,7 @@ static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp, }if (agg_bufs) { - skb =bnxt_rx_pages (bp, cpr, skb, idx, agg_bufs, true); + skb =bnxt_rx_agg_pages_skb (bp, cpr, skb, idx, agg_bufs, true);if (!skb) {/* Page reuse already handled by bnxt_rx_pages(). */ cpr->sw_stats.rx.rx_oom_discards +=1 ;
@@ -1898,7 +1898,7 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, }if (agg_bufs) { - skb =bnxt_rx_pages (bp, cpr, skb, cp_cons, agg_bufs, false); + skb =bnxt_rx_agg_pages_skb (bp, cpr, skb, cp_cons, agg_bufs, false);if (!skb) { cpr->sw_stats.rx.rx_oom_discards +=1 ; rc = -ENOMEM ;
Signature · 1 line
2.18.1
Michael Chan
maintainerPATCH v3 5/11 bnxt: adding bnxt_rx_agg_pages_xdp for aggregated xdp
In reply to Michael ChanFrom: Andy Gospodarek <[email protected]>
This patch adds a new function that will read pages from the aggregation ring and create an xdp_buff with frags based on the entries in the aggregation ring.
--- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 31 +++++++++++++++++++++++ 1 file changed, 31 insertions(+)
drivers/net/ethernet/broadcom/bnxt/bnxt.c +31 -0
@@ -1131,6 +1131,27 @@ static struct sk_buff *bnxt_rx_agg_pages_skb(struct bnxt *bp,return skb; } +static u32 bnxt_rx_agg_pages_xdp (struct bnxt *bp, +struct bnxt_cp_ring_info *cpr, +struct xdp_buff *xdp,u16 idx, +u32 agg_bufs,bool tpa) +{ +struct skb_shared_info *shinfo =xdp_get_shared_info_from_buff (xdp); +u32 total_frag_len =0 ; + +if (!xdp_buff_has_frags (xdp)) + shinfo->nr_frags =0 ; + + total_frag_len =__bnxt_rx_agg_pages (bp, cpr, shinfo, idx, agg_bufs, tpa); + +if (total_frag_len) { +xdp_buff_set_frags_flag (xdp); + shinfo->nr_frags = agg_bufs; + shinfo->xdp_frags_size = total_frag_len; + } +return total_frag_len; +} +static int bnxt_agg_bufs_valid (struct bnxt *bp,struct bnxt_cp_ring_info *cpr,u8 agg_bufs,u32 *raw_cons) {
@@ -1859,6 +1880,16 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,if (bnxt_xdp_attached (bp, rxr)) {bnxt_xdp_buff_init (bp, rxr, cons, &data_ptr, &len, &xdp); +if (agg_bufs) { +u32 frag_len =bnxt_rx_agg_pages_xdp (bp, cpr, &xdp, + cp_cons, agg_bufs, + false); +if (!frag_len) { + cpr->sw_stats.rx.rx_oom_discards +=1 ; + rc = -ENOMEM ; + goto next_rx; + } + } xdp_active = true; }
Signature · 1 line
2.18.1
Michael Chan
maintainerPATCH v3 6/11 bnxt: set xdp_buff pfmemalloc flag if needed
In reply to Michael ChanFrom: Andy Gospodarek <[email protected]>
Set the pfmemaloc flag in the xdp buff so that this can be copied to the skb if needed for an XDP_PASS action.
--- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-)
drivers/net/ethernet/broadcom/bnxt/bnxt.c +9 -5
@@ -1041,7 +1041,8 @@ static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,static u32 __bnxt_rx_agg_pages (struct bnxt *bp,struct bnxt_cp_ring_info *cpr,struct skb_shared_info *shinfo, -u16 idx,u32 agg_bufs,bool tpa) +u16 idx,u32 agg_bufs,bool tpa, +struct xdp_buff *xdp) {struct bnxt_napi *bnapi = cpr->bnapi;struct pci_dev *pdev = bp->pdev;
@@ -1084,6 +1085,9 @@ static u32 __bnxt_rx_agg_pages(struct bnxt *bp, page = cons_rx_buf->page; cons_rx_buf->page =NULL ; +if (xdp &&page_is_pfmemalloc (page)) +xdp_buff_set_frag_pfmemalloc (xdp); +if (bnxt_alloc_rx_page (bp, rxr, prod,GFP_ATOMIC ) !=0 ) {unsigned int nr_frags;
@@ -1118,8 +1122,8 @@ static struct sk_buff *bnxt_rx_agg_pages_skb(struct bnxt *bp,struct skb_shared_info *shinfo =skb_shinfo (skb);u32 total_frag_len =0 ; - total_frag_len =__bnxt_rx_agg_pages (bp, cpr, shinfo, idx, agg_bufs, tpa); - + total_frag_len =__bnxt_rx_agg_pages (bp, cpr, shinfo, idx, + agg_bufs, tpa,NULL );if (!total_frag_len) {dev_kfree_skb (skb);return NULL ;
@@ -1142,8 +1146,8 @@ static u32 bnxt_rx_agg_pages_xdp(struct bnxt *bp,if (!xdp_buff_has_frags (xdp)) shinfo->nr_frags =0 ; - total_frag_len =__bnxt_rx_agg_pages (bp, cpr, shinfo, idx, agg_bufs, tpa); - + total_frag_len =__bnxt_rx_agg_pages (bp, cpr, shinfo, + idx, agg_bufs, tpa, xdp);if (total_frag_len) {xdp_buff_set_frags_flag (xdp); shinfo->nr_frags = agg_bufs;
Signature · 1 line
2.18.1
David Miller
maintainerFrom: Michael Chan <[email protected]>
Date: Tue, 5 Apr 2022 22:53:42 -0400
Quoted 6 lines
This series adds XDP multi buffer support, allowing MTU to go beyond the page size limit.
v3: Simplify page mode buffer size calculation
Check to make sure XDP program supports multipage packets
v2: Fix uninitialized variable warnings in patch 1 and 10.
Not all the patches made it into patchwork, please re-post.
Thank you.
Michael Chan
maintainerQuoted 4 lines · David Miller
From: Michael Chan <[email protected]>
Date: Tue, 5 Apr 2022 22:53:42 -0400
Quoted 9 lines
Quoted 6 lines
This series adds XDP multi buffer support, allowing MTU to go beyond the page size limit.
v3: Simplify page mode buffer size calculation
Check to make sure XDP program supports multipage packets
v2: Fix uninitialized variable warnings in patch 1 and 10.Not all the patches made it into patchwork, please re-post.
I will repost after net merges into net-next to avoid merge conflicts. Dave, will you be doing a merge soon?
Michael Chan
maintainerPATCH v3 7/11 bnxt: change receive ring space parameters
In reply to Michael ChanFrom: Andy Gospodarek <[email protected]>
Modify ring header data split and jumbo parameters to account for the fact that the design for XDP multibuffer puts close to the first 4k of data in a page and the remaining portions of the packet go in the aggregation ring.
v3: Simplified code around initial buffer size calculation
--- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 42 +++++++++++++++-------- drivers/net/ethernet/broadcom/bnxt/bnxt.h | 1 + 2 files changed, 28 insertions(+), 15 deletions(-)
drivers/net/ethernet/broadcom/bnxt/bnxt.c +27 -15
@@ -56,6 +56,7 @@#include <linux/hwmon.h> #include <linux/hwmon-sysfs.h> #include <net/page_pool.h> +#include <linux/align.h> #include "bnxt_hsi.h" #include "bnxt.h"
@@ -1933,11 +1934,13 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, }if (agg_bufs) { - skb =bnxt_rx_agg_pages_skb (bp, cpr, skb, cp_cons, agg_bufs, false); -if (!skb) { - cpr->sw_stats.rx.rx_oom_discards +=1 ; - rc = -ENOMEM ; - goto next_rx; +if (!xdp_active) { + skb =bnxt_rx_agg_pages_skb (bp, cpr, skb, cp_cons, agg_bufs, false); +if (!skb) { + cpr->sw_stats.rx.rx_oom_discards +=1 ; + rc = -ENOMEM ; + goto next_rx; + } } }
@@ -3854,7 +3857,7 @@ void bnxt_set_ring_params(struct bnxt *bp)/* 8 for CRC and VLAN */ rx_size =SKB_DATA_ALIGN (bp->dev->mtu +ETH_HLEN +NET_IP_ALIGN +8 ); - rx_space = rx_size +NET_SKB_PAD + + rx_space = rx_size +ALIGN (max (NET_SKB_PAD ,XDP_PACKET_HEADROOM ),8 ) +SKB_DATA_ALIGN (sizeof (struct skb_shared_info )); bp->rx_copy_thresh =BNXT_RX_COPY_THRESH ;
@@ -3895,9 +3898,15 @@ void bnxt_set_ring_params(struct bnxt *bp) } bp->rx_agg_ring_size = agg_ring_size; bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages *RX_DESC_CNT ) -1 ; - rx_size =SKB_DATA_ALIGN (BNXT_RX_COPY_THRESH +NET_IP_ALIGN ); - rx_space = rx_size +NET_SKB_PAD + -SKB_DATA_ALIGN (sizeof (struct skb_shared_info )); + +if (BNXT_RX_PAGE_MODE (bp)) { + rx_space =BNXT_PAGE_MODE_BUF_SIZE ; + rx_size =BNXT_MAX_PAGE_MODE_MTU ; + }else { + rx_size =SKB_DATA_ALIGN (BNXT_RX_COPY_THRESH +NET_IP_ALIGN ); + rx_space = rx_size +NET_SKB_PAD + +SKB_DATA_ALIGN (sizeof (struct skb_shared_info )); + } } bp->rx_buf_use_size = rx_size;
@@ -5287,12 +5296,15 @@ static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)if (rc)return rc; - req->flags =cpu_to_le32 (VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT | -VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 | -VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6 ); - req->enables = -cpu_to_le32 (VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID | -VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID ); + req->flags =cpu_to_le32 (VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT ); + req->enables =cpu_to_le32 (VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID ); + +if (BNXT_RX_PAGE_MODE (bp) && !BNXT_RX_JUMBO_MODE (bp)) { + req->flags |=cpu_to_le32 (VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 | +VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6 ); + req->enables |= +cpu_to_le32 (VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID ); + }/* thresholds not implemented in firmware yet */ req->jumbo_thresh =cpu_to_le16 (bp->rx_copy_thresh); req->hds_threshold =cpu_to_le16 (bp->rx_copy_thresh);
drivers/net/ethernet/broadcom/bnxt/bnxt.h +1 -0
@@ -1819,6 +1819,7 @@ struct bnxt {#define BNXT_SUPPORTS_TPA (bp) (!BNXT_CHIP_TYPE_NITRO_A0(bp) && \ (!((bp)->flags & BNXT_FLAG_CHIP_P5) || \ (bp)->max_tpa_v2) && !is_kdump_kernel()) +#define BNXT_RX_JUMBO_MODE (bp) ((bp)->flags & BNXT_FLAG_JUMBO)#define BNXT_CHIP_SR2 (bp) \ ((bp)->chip_num == CHIP_NUM_58818)
Signature · 1 line
2.18.1
Michael Chan
maintainerPATCH v3 8/11 bnxt: add page_pool support for aggregation ring when using xdp
In reply to Michael ChanFrom: Andy Gospodarek <[email protected]>
If we are using aggregation rings with XDP enabled, allocate page buffers for the aggregation rings from the page_pool.
--- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 77 ++++++++++++++--------- 1 file changed, 47 insertions(+), 30 deletions(-)
drivers/net/ethernet/broadcom/bnxt/bnxt.c +47 -30
@@ -739,7 +739,6 @@ static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,page_pool_recycle_direct (rxr->page_pool, page);return NULL ; } - *mapping += bp->rx_dma_offset;return page; }
@@ -781,6 +780,7 @@ int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,if (!page)return -ENOMEM ; + mapping += bp->rx_dma_offset; rx_buf->data = page; rx_buf->data_ptr =page_address (page) + bp->rx_offset; }else {
@@ -841,33 +841,41 @@ static inline int bnxt_alloc_rx_page(struct bnxt *bp,u16 sw_prod = rxr->rx_sw_agg_prod;unsigned int offset =0 ; -if (PAGE_SIZE >BNXT_RX_PAGE_SIZE ) { - page = rxr->rx_page; -if (!page) { +if (BNXT_RX_PAGE_MODE (bp)) { + page =__bnxt_alloc_rx_page (bp, &mapping, rxr, gfp); + +if (!page) +return -ENOMEM ; + + }else { +if (PAGE_SIZE >BNXT_RX_PAGE_SIZE ) { + page = rxr->rx_page; +if (!page) { + page =alloc_page (gfp); +if (!page) +return -ENOMEM ; + rxr->rx_page = page; + rxr->rx_page_offset =0 ; + } + offset = rxr->rx_page_offset; + rxr->rx_page_offset +=BNXT_RX_PAGE_SIZE ; +if (rxr->rx_page_offset ==PAGE_SIZE ) + rxr->rx_page =NULL ; +else +get_page (page); + }else { page =alloc_page (gfp);if (!page)return -ENOMEM ; - rxr->rx_page = page; - rxr->rx_page_offset =0 ; } - offset = rxr->rx_page_offset; - rxr->rx_page_offset +=BNXT_RX_PAGE_SIZE ; -if (rxr->rx_page_offset ==PAGE_SIZE ) - rxr->rx_page =NULL ; -else -get_page (page); - }else { - page =alloc_page (gfp); -if (!page) -return -ENOMEM ; - } - mapping =dma_map_page_attrs (&pdev->dev, page, offset, -BNXT_RX_PAGE_SIZE ,DMA_FROM_DEVICE , -DMA_ATTR_WEAK_ORDERING ); -if (dma_mapping_error (&pdev->dev, mapping)) { -__free_page (page); -return -EIO ; + mapping =dma_map_page_attrs (&pdev->dev, page, offset, +BNXT_RX_PAGE_SIZE ,DMA_FROM_DEVICE , +DMA_ATTR_WEAK_ORDERING ); +if (dma_mapping_error (&pdev->dev, mapping)) { +__free_page (page); +return -EIO ; + } }if (unlikely (test_bit (sw_prod, rxr->rx_agg_bmap)))
@@ -1105,7 +1113,7 @@ static u32 __bnxt_rx_agg_pages(struct bnxt *bp, }dma_unmap_page_attrs (&pdev->dev, mapping,BNXT_RX_PAGE_SIZE , -DMA_FROM_DEVICE , + bp->rx_dir,DMA_ATTR_WEAK_ORDERING ); total_frag_len += frag_len;
@@ -2936,14 +2944,23 @@ static void bnxt_free_one_rx_ring_skbs(struct bnxt *bp, int ring_nr)if (!page)continue ; -dma_unmap_page_attrs (&pdev->dev, rx_agg_buf->mapping, -BNXT_RX_PAGE_SIZE ,DMA_FROM_DEVICE , -DMA_ATTR_WEAK_ORDERING ); +if (BNXT_RX_PAGE_MODE (bp)) { +dma_unmap_page_attrs (&pdev->dev, rx_agg_buf->mapping, +BNXT_RX_PAGE_SIZE , bp->rx_dir, +DMA_ATTR_WEAK_ORDERING ); + rx_agg_buf->page =NULL ; +__clear_bit (i, rxr->rx_agg_bmap); - rx_agg_buf->page =NULL ; -__clear_bit (i, rxr->rx_agg_bmap); +page_pool_recycle_direct (rxr->page_pool, page); + }else { +dma_unmap_page_attrs (&pdev->dev, rx_agg_buf->mapping, +BNXT_RX_PAGE_SIZE ,DMA_FROM_DEVICE , +DMA_ATTR_WEAK_ORDERING ); + rx_agg_buf->page =NULL ; +__clear_bit (i, rxr->rx_agg_bmap); -__free_page (page); +__free_page (page); + } } skip_rx_agg_free:
Signature · 1 line
2.18.1
Michael Chan
maintainerPATCH v3 9/11 bnxt: adding bnxt_xdp_build_skb to build skb from multibuffer xdp_buff
In reply to Michael ChanFrom: Andy Gospodarek <[email protected]>
Since we have an xdp_buff with frags there needs to be a way to convert that into a valid sk_buff in the event that XDP_PASS is the resulting operation. This adds a new rx_skb_func when the netdev has an MTU that prevents the packets from sitting in a single page.
This also make sure that GRO/LRO stay disabled even when using the aggregation ring for large buffers.
v3: Use BNXT_PAGE_MODE_BUF_SIZE for build_skb
--- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 65 +++++++++++++++++-- drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c | 23 +++++++ drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h | 4 ++ 3 files changed, 85 insertions(+), 7 deletions(-)
drivers/net/ethernet/broadcom/bnxt/bnxt.c +58 -7
@@ -971,6 +971,39 @@ static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx, rxr->rx_sw_agg_prod = sw_prod; } +static struct sk_buff *bnxt_rx_multi_page_skb (struct bnxt *bp, +struct bnxt_rx_ring_info *rxr, +u16 cons,void *data,u8 *data_ptr, +dma_addr_t dma_addr, +unsigned int offset_and_len) +{ +unsigned int len = offset_and_len &0xffff ; +struct page *page = data; +u16 prod = rxr->rx_prod; +struct sk_buff *skb; +int err; + + err =bnxt_alloc_rx_data (bp, rxr, prod,GFP_ATOMIC ); +if (unlikely (err)) { +bnxt_reuse_rx_data (rxr, cons, data); +return NULL ; + } + dma_addr -= bp->rx_dma_offset; +dma_unmap_page_attrs (&bp->pdev->dev, dma_addr,PAGE_SIZE , bp->rx_dir, +DMA_ATTR_WEAK_ORDERING ); + skb =build_skb (page_address (page),BNXT_PAGE_MODE_BUF_SIZE + + bp->rx_dma_offset); +if (!skb) { +__free_page (page); +return NULL ; + } +skb_mark_for_recycle (skb); +skb_reserve (skb, bp->rx_dma_offset); +__skb_put (skb, len); + +return skb; +} + static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons, void *data, u8 *data_ptr,
@@ -993,7 +1026,6 @@ static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp, dma_addr -= bp->rx_dma_offset;dma_unmap_page_attrs (&bp->pdev->dev, dma_addr,PAGE_SIZE , bp->rx_dir,DMA_ATTR_WEAK_ORDERING ); -page_pool_release_page (rxr->page_pool, page);if (unlikely (!payload)) payload =eth_get_headlen (bp->dev, data_ptr, len);
@@ -1004,6 +1036,7 @@ static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,return NULL ; } +skb_mark_for_recycle (skb); off = (void *)data_ptr -page_address (page);skb_add_rx_frag (skb,0 , page, off, len,PAGE_SIZE ); memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
@@ -1949,6 +1982,14 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, rc = -ENOMEM ; goto next_rx; } + }else { + skb =bnxt_xdp_build_skb (bp, skb, agg_bufs, rxr->page_pool, &xdp, rxcmp1); +if (!skb) { +/* we should be able to free the old skb here */ + cpr->sw_stats.rx.rx_oom_discards +=1 ; + rc = -ENOMEM ; + goto next_rx; + } } }
@@ -3964,14 +4005,21 @@ void bnxt_set_ring_params(struct bnxt *bp)int bnxt_set_rx_skb_mode (struct bnxt *bp,bool page_mode) {if (page_mode) { -if (bp->dev->mtu >BNXT_MAX_PAGE_MODE_MTU ) -return -EOPNOTSUPP ; - bp->dev->max_mtu = -min_t (u16, bp->max_mtu,BNXT_MAX_PAGE_MODE_MTU ); bp->flags &= ~BNXT_FLAG_AGG_RINGS ; - bp->flags |=BNXT_FLAG_NO_AGG_RINGS |BNXT_FLAG_RX_PAGE_MODE ; + bp->flags |=BNXT_FLAG_RX_PAGE_MODE ; + +if (bp->dev->mtu >BNXT_MAX_PAGE_MODE_MTU ) { + bp->flags |=BNXT_FLAG_JUMBO ; + bp->rx_skb_func = bnxt_rx_multi_page_skb; + bp->dev->max_mtu = +min_t (u16, bp->max_mtu,BNXT_MAX_MTU ); + }else { + bp->flags |=BNXT_FLAG_NO_AGG_RINGS ; + bp->rx_skb_func = bnxt_rx_page_skb; + bp->dev->max_mtu = +min_t (u16, bp->max_mtu,BNXT_MAX_PAGE_MODE_MTU ); + } bp->rx_dir =DMA_BIDIRECTIONAL ; - bp->rx_skb_func = bnxt_rx_page_skb;/* Disable LRO or GRO_HW */ netdev_update_features (bp->dev); }else {
@@ -11121,6 +11169,9 @@ static netdev_features_t bnxt_fix_features(struct net_device *dev,if (bp->flags &BNXT_FLAG_NO_AGG_RINGS ) features &= ~(NETIF_F_LRO |NETIF_F_GRO_HW ); +if (!(bp->flags &BNXT_FLAG_TPA )) + features &= ~(NETIF_F_LRO |NETIF_F_GRO_HW ); +if (!(features &NETIF_F_GRO )) features &= ~NETIF_F_GRO_HW ;
drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c +23 -0
@@ -361,3 +361,26 @@ int bnxt_xdp(struct net_device *dev, struct netdev_bpf *xdp) }return rc; } + +struct sk_buff * +bnxt_xdp_build_skb (struct bnxt *bp,struct sk_buff *skb,u8 num_frags, +struct page_pool *pool,struct xdp_buff *xdp, +struct rx_cmp_ext *rxcmp1) +{ +struct skb_shared_info *sinfo =xdp_get_shared_info_from_buff (xdp); + +if (!skb) +return NULL ; +skb_checksum_none_assert (skb); +if (RX_CMP_L4_CS_OK (rxcmp1)) { +if (bp->dev->features &NETIF_F_RXCSUM ) { + skb->ip_summed =CHECKSUM_UNNECESSARY ; + skb->csum_level =RX_CMP_ENCAP (rxcmp1); + } + } +xdp_update_skb_shared_info (skb, num_frags, + sinfo->xdp_frags_size, +PAGE_SIZE * sinfo->nr_frags, +xdp_buff_is_frag_pfmemalloc (xdp)); +return skb; +}
drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h +4 -0
@@ -28,4 +28,8 @@ bool bnxt_xdp_attached(struct bnxt *bp, struct bnxt_rx_ring_info *rxr);void bnxt_xdp_buff_init (struct bnxt *bp,struct bnxt_rx_ring_info *rxr,u16 cons,u8 **data_ptr,unsigned int *len,struct xdp_buff *xdp); +struct sk_buff *bnxt_xdp_build_skb (struct bnxt *bp,struct sk_buff *skb, +u8 num_frags,struct page_pool *pool, +struct xdp_buff *xdp, +struct rx_cmp_ext *rxcmp1);#endif
Signature · 1 line
2.18.1
Michael Chan
maintainerPATCH v3 10/11 bnxt: support transmit and free of aggregation buffers
In reply to Michael ChanFrom: Andy Gospodarek <[email protected]>
This patch adds the following features:
- Support for XDP_TX and XDP_DROP action when using xdp_buff with frags
- Support for freeing all frags attached to an xdp_buff
- Cleanup of TX ring buffers after transmits complete
- Slight change in definition of bnxt_sw_tx_bd since nr_frags and RX producer may both need to be used
- Clear out skb_shared_info at the end of the buffer
v2: Fix uninitialized variable warning in bnxt_xdp_buff_frags_free().
--- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 18 ++- drivers/net/ethernet/broadcom/bnxt/bnxt.h | 7 +- .../net/ethernet/broadcom/bnxt/bnxt_ethtool.c | 2 +- drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c | 117 ++++++++++++++++-- drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h | 5 +- 5 files changed, 126 insertions(+), 23 deletions(-)
drivers/net/ethernet/broadcom/bnxt/bnxt.c +13 -5
@@ -1951,9 +1951,13 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, skb =bnxt_copy_skb (bnapi, data_ptr, len, dma_addr);bnxt_reuse_rx_data (rxr, cons, data);if (!skb) { -if (agg_bufs) -bnxt_reuse_rx_agg_bufs (cpr, cp_cons,0 , - agg_bufs, false); +if (agg_bufs) { +if (!xdp_active) +bnxt_reuse_rx_agg_bufs (cpr, cp_cons,0 , + agg_bufs, false); +else +bnxt_xdp_buff_frags_free (rxr, &xdp); + } cpr->sw_stats.rx.rx_oom_discards +=1 ; rc = -ENOMEM ; goto next_rx;
@@ -1986,6 +1990,7 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, skb =bnxt_xdp_build_skb (bp, skb, agg_bufs, rxr->page_pool, &xdp, rxcmp1);if (!skb) {/* we should be able to free the old skb here */ +bnxt_xdp_buff_frags_free (rxr, &xdp); cpr->sw_stats.rx.rx_oom_discards +=1 ; rc = -ENOMEM ; goto next_rx;
@@ -2605,10 +2610,13 @@ static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi)if ((bnapi->events &BNXT_RX_EVENT ) && !(bnapi->in_reset)) {struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; -if (bnapi->events &BNXT_AGG_EVENT ) -bnxt_db_write (bp, &rxr->rx_agg_db, rxr->rx_agg_prod);bnxt_db_write (bp, &rxr->rx_db, rxr->rx_prod); } +if (bnapi->events &BNXT_AGG_EVENT ) { +struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; + +bnxt_db_write (bp, &rxr->rx_agg_db, rxr->rx_agg_prod); + } bnapi->events =0 ; }
drivers/net/ethernet/broadcom/bnxt/bnxt.h +3 -4
@@ -701,13 +701,12 @@ struct bnxt_sw_tx_bd { };DEFINE_DMA_UNMAP_ADDR (mapping);DEFINE_DMA_UNMAP_LEN (len); +struct page *page;u8 is_gso;u8 is_push;u8 action; -union { -unsigned short nr_frags; -u16 rx_prod; - }; +unsigned short nr_frags; +u16 rx_prod; };struct bnxt_sw_rx_bd {
drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c +1 -1
@@ -3491,7 +3491,7 @@ static int bnxt_run_loopback(struct bnxt *bp)dev_kfree_skb (skb);return -EIO ; } -bnxt_xmit_bd (bp, txr, map, pkt_size); +bnxt_xmit_bd (bp, txr, map, pkt_size,NULL );/* Sync BD data before updating doorbell */ wmb ();
drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c +105 -12
209 lines not on this page · raw message
drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h +4 -1
@@ -14,7 +14,8 @@ DECLARE_STATIC_KEY_FALSE(bnxt_xdp_locking_key);struct bnxt_sw_tx_bd *bnxt_xmit_bd (struct bnxt *bp,struct bnxt_tx_ring_info *txr, -dma_addr_t mapping,u32 len); +dma_addr_t mapping,u32 len, +struct xdp_buff *xdp);void bnxt_tx_int_xdp (struct bnxt *bp,struct bnxt_napi *bnapi,int nr_pkts); bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons, struct xdp_buff xdp, struct page *page, unsigned int *len,
@@ -28,6 +29,8 @@ bool bnxt_xdp_attached(struct bnxt *bp, struct bnxt_rx_ring_info *rxr);void bnxt_xdp_buff_init (struct bnxt *bp,struct bnxt_rx_ring_info *rxr,u16 cons,u8 **data_ptr,unsigned int *len,struct xdp_buff *xdp); +void bnxt_xdp_buff_frags_free (struct bnxt_rx_ring_info *rxr, +struct xdp_buff *xdp); struct sk_buff *bnxt_xdp_build_skb(struct bnxt *bp, struct sk_buff *skb, u8 num_frags, struct page_pool *pool, struct xdp_buff *xdp,
Signature · 1 line
2.18.1
Michael Chan
maintainerPATCH v3 11/11 bnxt: XDP multibuffer enablement
In reply to Michael ChanFrom: Andy Gospodarek <[email protected]>
Allow aggregation buffers to be in place in the receive path and allow XDP programs to be attached when using a larger than 4k MTU.
v3: Add a check to sure XDP program supports multipage packets.
--- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 3 +-- drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-)
drivers/net/ethernet/broadcom/bnxt/bnxt.c +1 -2
@@ -1939,8 +1939,7 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, xdp_active = true; } -/* skip running XDP prog if there are aggregation bufs */ -if (!agg_bufs && xdp_active) { +if (xdp_active) {if (bnxt_rx_xdp (bp, rxr, cons, xdp, data, &len, event)) { rc =1 ; goto next_rx;
drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c +3 -2
@@ -387,8 +387,9 @@ static int bnxt_xdp_set(struct bnxt *bp, struct bpf_prog *prog)int tx_xdp =0 , rc, tc;struct bpf_prog *old; -if (prog && bp->dev->mtu >BNXT_MAX_PAGE_MODE_MTU ) { -netdev_warn (dev,"MTU %d larger than largest XDP supported MTU %d.\n" , +if (prog && !prog->aux->xdp_has_frags && + bp->dev->mtu >BNXT_MAX_PAGE_MODE_MTU ) { +netdev_warn (dev,"MTU %d larger than %d without XDP frag support.\n" , bp->dev->mtu,BNXT_MAX_PAGE_MODE_MTU );return -EOPNOTSUPP ; }
Signature · 1 line
2.18.1