i40e: Support for XDP
Subsystems (2)
v3 cover i40e: Support for XDP
From: Bj?rn T?pel <[email protected]>
This series adds XDP support for i40e-based NICs.
The first patch prepares i40e_fetch_rx_buffer() for upcoming changes, followed by XDP_RX support, the third adds XDP_TX support and the last patch validates bpf_xdp_adjust_head() support.
Thanks to Alex, Daniel and John for all the feedback!
v3:
* Rebased patch set on Jeff's dev-queue branch
* MSI-X is no longer a prerequisite for XDP
* RCU locking for the XDP program and XDP_RX support is introduced in the same patch
* Rx bytes is now bumped for XDP
* Removed pointer-to-pointer clunkiness
* Added comments to XDP preconditions in ndo_xdp
* When a non-EOF is received, log once, and drop the frame
v2:
* Fixed kbuild error for PAGE_SIZE >= 8192.
* Renamed i40e_try_flip_rx_page to i40e_can_reuse_rx_page, which is more in line to the other Intel Ethernet drivers (igb/fm10k).
* Validate xdp_adjust_head support in ndo_xdp/XDP_SETUP_PROG.
Bj?rn
Bj?rn T?pel (4):
i40e: Sync DMA region prior skbuff allocation
i40e: Initial support for XDP
i40e: Add XDP_TX support
i40e: Validate xdp_adjust_head support
drivers/net/ethernet/intel/i40e/i40e.h | 18 ++ drivers/net/ethernet/intel/i40e/i40e_ethtool.c | 4 + drivers/net/ethernet/intel/i40e/i40e_main.c | 380 ++++++++++++++++++++---- drivers/net/ethernet/intel/i40e/i40e_txrx.c | 391 ++++++++++++++++++++++--- drivers/net/ethernet/intel/i40e/i40e_txrx.h | 7 + 5 files changed, 711 insertions(+), 89 deletions(-)
Signature · 1 line
2.9.3
PATCH v3 1/4 i40e: Sync DMA region prior skbuff allocation
From: Bj?rn T?pel <[email protected]>
This patch prepares i40e_fetch_rx_buffer() for upcoming XDP support, where there's a need to access the device buffers prior skbuff allocation.
--- drivers/net/ethernet/intel/i40e/i40e_txrx.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-)
drivers/net/ethernet/intel/i40e/i40e_txrx.c +7 -7
@@ -1646,6 +1646,13 @@ struct sk_buff *i40e_fetch_rx_buffer(struct i40e_ring *rx_ring, page = rx_buffer->page;prefetchw (page); +/* we are reusing so sync this buffer for CPU use */ +dma_sync_single_range_for_cpu (rx_ring->dev, + rx_buffer->dma, + rx_buffer->page_offset, + size, +DMA_FROM_DEVICE ); +if (likely (!skb)) {void *page_addr =page_address (page) + rx_buffer->page_offset;
@@ -1671,13 +1678,6 @@ struct sk_buff *i40e_fetch_rx_buffer(struct i40e_ring *rx_ring,prefetchw (skb->data); } -/* we are reusing so sync this buffer for CPU use */ -dma_sync_single_range_for_cpu (rx_ring->dev, - rx_buffer->dma, - rx_buffer->page_offset, - size, -DMA_FROM_DEVICE ); -/* pull page into skb */ if (i40e_add_rx_frag (rx_ring, rx_buffer, size, skb)) {/* hand second half of page back to the ring */
Signature · 1 line
2.9.3
PATCH v3 2/4 i40e: Initial support for XDP
In reply to Björn TöpelFrom: Bj?rn T?pel <[email protected]>
This commit adds basic XDP support for i40e derived NICs. All XDP actions will end up in XDP_DROP.
Only the default/main VSI has support for enabling XDP.
--- drivers/net/ethernet/intel/i40e/i40e.h | 13 +++ drivers/net/ethernet/intel/i40e/i40e_ethtool.c | 4 + drivers/net/ethernet/intel/i40e/i40e_main.c | 83 +++++++++++++++++ drivers/net/ethernet/intel/i40e/i40e_txrx.c | 123 +++++++++++++++++++++++-- drivers/net/ethernet/intel/i40e/i40e_txrx.h | 2 + 5 files changed, 219 insertions(+), 6 deletions(-)
drivers/net/ethernet/intel/i40e/i40e.h +13 -0
@@ -587,6 +587,8 @@ struct i40e_vsi {struct i40e_ring **rx_rings;struct i40e_ring **tx_rings; +bool xdp_enabled; +u32 active_filters;u32 promisc_threshold;
@@ -946,4 +948,15 @@ i40e_status i40e_get_npar_bw_setting(struct i40e_pf *pf);i40e_status i40e_set_npar_bw_setting (struct i40e_pf *pf);i40e_status i40e_commit_npar_bw_setting (struct i40e_pf *pf);void i40e_print_link_message (struct i40e_vsi *vsi,bool isup); + +/** +* i40e_enabled_xdp_vsi - Check if VSI has XDP enabled +* @vsi: pointer to a vsi +* +* Returns true if the VSI has XDP enabled. +**/ +static inline bool i40e_enabled_xdp_vsi (const struct i40e_vsi *vsi) +{ +return !!vsi->xdp_enabled; +}#endif /* _I40E_H_ */
drivers/net/ethernet/intel/i40e/i40e_ethtool.c +4 -0
@@ -1257,6 +1257,10 @@ static int i40e_set_ringparam(struct net_device *netdev,if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending))return -EINVAL ; +/* Don't allow any change while XDP is enabled. */ +if (i40e_enabled_xdp_vsi (vsi)) +return -EINVAL ; + if (ring->tx_pending > I40E_MAX_NUM_DESCRIPTORS || ring->tx_pending < I40E_MIN_NUM_DESCRIPTORS || ring->rx_pending > I40E_MAX_NUM_DESCRIPTORS ||
drivers/net/ethernet/intel/i40e/i40e_main.c +83 -0
114 lines not on this page · raw message
drivers/net/ethernet/intel/i40e/i40e_txrx.c +117 -6
185 lines not on this page · raw message
drivers/net/ethernet/intel/i40e/i40e_txrx.h +2 -0
@@ -361,6 +361,8 @@ struct i40e_ring {* i40e_clean_rx_ring_irq() is called * for this ring. */ + +struct bpf_prog __rcu *xdp_prog; }____cacheline_internodealigned_in_smp ;enum i40e_latency_range {
Signature · 1 line
2.9.3
PATCH v3 2/4 i40e: Initial support for XDP
Quoted 9 lines · Bj?rn T?pel
From: Bj?rn T?pel <[email protected]>
This commit adds basic XDP support for i40e derived NICs. All XDP actions will end up in
XDP_DROP.Only the default/main VSI has support for enabling XDP.
Signed-off-by Bj?rn T?pel <[email protected]>---
[...]
Quoted 20 lines
/**
+ *i40e_run_xdp- Runs an XDP program for an Rx ring
+ * @rx_ring: Rx ring used for XDP
+ * @rx_buffer: current Rx buffer
+ * @rx_desc: current Rx descriptor
+ * @size: buffer size
+ * @xdp_prog: the XDP program to run
+ *
+ * Returns true if the XDP program consumed the incoming frame. False
+ * means pass the frame to the good old stack.
+ **/drivers/net/ethernet/intel/i40e/i40e_txrx.c +8
-0+static bool i40e_run_xdp (struct i40e_ring *rx_ring, +struct i40e_rx_buffer *rx_buffer, +union i40e_rx_desc *rx_desc, +unsigned int size, +struct bpf_prog *xdp_prog) +{ +struct xdp_buff xdp; +u32 xdp_action;
drivers/net/ethernet/intel/i40e/i40e_txrx.c static bool i40e_add_rx_frag(struct i40e_ring *rx_ring, · line 1657
Quoted 10 lines
drivers/net/ethernet/intel/i40e/i40e_txrx.c +10
-0+ +if (unlikely (!i40e_test_staterr (rx_desc, +BIT (I40E_RX_DESC_STATUS_EOF_SHIFT )))) { +dev_warn_once (&rx_ring->vsi->back->pdev->dev, +"Received unexpected RXD_EOF!\n" ); + goto do_drop; + } + + xdp.data =page_address (rx_buffer->page) + rx_buffer->page_offset; + xdp.data_end = xdp.data + size;
We need to set the xdp.data_hard_start as well here.
xdp.data_hard_start = xdp.data
should be good enough for now.
Quoted 20 lines
drivers/net/ethernet/intel/i40e/i40e_txrx.c +20
-0+ xdp_action =bpf_prog_run_xdp (xdp_prog, &xdp); + +switch (xdp_action) { +case XDP_PASS : +return false; +default : +bpf_warn_invalid_xdp_action (xdp_action); +case XDP_ABORTED : +case XDP_TX : +case XDP_DROP : +do_drop: +if (likely (i40e_page_is_reusable (rx_buffer->page))) { +i40e_reuse_rx_page (rx_ring, rx_buffer); + rx_ring->rx_stats.page_reuse_count++; +break ; + } + +/* we are not reusing the buffer so unmap it */ +dma_unmap_page (rx_ring->dev, rx_buffer->dma,PAGE_SIZE , +DMA_FROM_DEVICE );
drivers/net/ethernet/intel/i40e/i40e_txrx.c static bool i40e_add_rx_frag(struct i40e_ring *rx_ring, · line 1687
Quoted 10 lines
drivers/net/ethernet/intel/i40e/i40e_txrx.c +9
-0+__free_pages (rx_buffer->page,0 ); + } + +/* clear contents of buffer_info */ + rx_buffer->page =NULL ; +return true;/* Swallowed by XDP */ +} + +/** * i40e_fetch_rx_buffer - Allocate skb and populate it
Thanks,
John
PATCH v3 2/4 i40e: Initial support for XDP
2016-12-16 19:07 GMT+01:00 John Fastabend <[email protected]>:
[...]
Quoted 33 lines
Quoted 20 lines
/**
+ *i40e_run_xdp- Runs an XDP program for an Rx ring
+ * @rx_ring: Rx ring used for XDP
+ * @rx_buffer: current Rx buffer
+ * @rx_desc: current Rx descriptor
+ * @size: buffer size
+ * @xdp_prog: the XDP program to run
+ *
+ * Returns true if the XDP program consumed the incoming frame. False
+ * means pass the frame to the good old stack.
+ **/drivers/net/ethernet/intel/i40e/i40e_txrx.c +8
-0+static bool i40e_run_xdp (struct i40e_ring *rx_ring, +struct i40e_rx_buffer *rx_buffer, +union i40e_rx_desc *rx_desc, +unsigned int size, +struct bpf_prog *xdp_prog) +{ +struct xdp_buff xdp; +u32 xdp_action;drivers/net/ethernet/intel/i40e/i40e_txrx.c static bool i40e_add_rx_frag(struct i40e_ring *rx_ring, · line 1657
Quoted 10 lines
drivers/net/ethernet/intel/i40e/i40e_txrx.c +10
-0+ +if (unlikely (!i40e_test_staterr (rx_desc, +BIT (I40E_RX_DESC_STATUS_EOF_SHIFT )))) { +dev_warn_once (&rx_ring->vsi->back->pdev->dev, +"Received unexpected RXD_EOF!\n" ); + goto do_drop; + } + + xdp.data =page_address (rx_buffer->page) + rx_buffer->page_offset; + xdp.data_end = xdp.data + size;We need to set the xdp.data_hard_start as well here.
Quoted 3 lines
xdp.data_hard_start = xdp.data
should be good enough for now.
OK! Thanks for pointing that out. I'll address that in v4!
Bj?rn
PATCH v3 3/4 i40e: Add XDP_TX support
In reply to Björn TöpelFrom: Bj?rn T?pel <[email protected]>
This patch adds proper XDP_TX support.
--- drivers/net/ethernet/intel/i40e/i40e.h | 5 + drivers/net/ethernet/intel/i40e/i40e_main.c | 294 +++++++++++++++++++++++----- drivers/net/ethernet/intel/i40e/i40e_txrx.c | 264 ++++++++++++++++++++++--- drivers/net/ethernet/intel/i40e/i40e_txrx.h | 5 + 4 files changed, 487 insertions(+), 81 deletions(-)
drivers/net/ethernet/intel/i40e/i40e.h +5 -0
@@ -587,6 +587,10 @@ struct i40e_vsi {struct i40e_ring **rx_rings;struct i40e_ring **tx_rings; +/* The XDP rings are Tx only, and follows the count of the +* regular rings, i.e. alloc_queue_pairs/num_queue_pairs +*/ +struct i40e_ring **xdp_rings;bool xdp_enabled;u32 active_filters;
@@ -664,6 +668,7 @@ struct i40e_q_vector {struct i40e_ring_container rx;struct i40e_ring_container tx; +struct i40e_ring_container xdp;u8 num_ringpairs;/* total number of ring pairs in vector */
drivers/net/ethernet/intel/i40e/i40e_main.c +241 -53
483 lines not on this page · raw message
drivers/net/ethernet/intel/i40e/i40e_txrx.c +236 -28
330 lines not on this page · raw message
drivers/net/ethernet/intel/i40e/i40e_txrx.h +5 -0
@@ -233,6 +233,7 @@ static inline unsigned int i40e_txd_use_count(unsigned int size)#define I40E_TX_FLAGS_TSYN BIT(8)#define I40E_TX_FLAGS_FD_SB BIT(9)#define I40E_TX_FLAGS_UDP_TUNNEL BIT(10) +#define I40E_TX_FLAGS_XDP BIT(11)#define I40E_TX_FLAGS_VLAN_MASK 0xffff0000#define I40E_TX_FLAGS_VLAN_PRIO_MASK 0xe0000000#define I40E_TX_FLAGS_VLAN_PRIO_SHIFT 29
@@ -243,6 +244,7 @@ struct i40e_tx_buffer {union {struct sk_buff *skb;void *raw_buf; +struct page *page; };unsigned int bytecount;unsigned short gso_segs;
@@ -363,6 +365,9 @@ struct i40e_ring {*/ struct bpf_prog __rcu *xdp_prog; +struct i40e_ring *xdp_sibling;/* rx to xdp, and xdp to rx */ +bool xdp_needs_tail_bump; +u16 curr_in_use; }____cacheline_internodealigned_in_smp ;enum i40e_latency_range {
Signature · 1 line
2.9.3
PATCH v3 3/4 i40e: Add XDP_TX support
2016-12-13 14:40 GMT+01:00 Bj?rn T?pel <[email protected]>:
[...]
drivers/net/ethernet/intel/i40e/i40e_txrx.c u32 i40e_get_tx_pending(struct i40e_ring *ring, bool in_sw) · line 636
Quoted 11 lines
drivers/net/ethernet/intel/i40e/i40e_txrx.c +7
-0@@ -625,6 +627,15 @@ u32 i40e_get_tx_pending(struct i40e_ring *ring, bool in_sw)#define WB_STRIDE 4/** +* i40e_page_is_reserved - check if reuse is possible +* @page: page struct to check +*/ +static inline bool i40e_page_is_reserved (struct page *page) +{ +return (page_to_nid (page) !=numa_mem_id ()) ||page_is_pfmemalloc (page); +}
I started to use use Scott's i40e_page_is_reusable(), but missed to get rid if this
leftover function.
So this one is not used, and should be removed.
[...]
Bj?rn
PATCH v3 4/4 i40e: Validate xdp_adjust_head support
In reply to Björn TöpelFrom: Bj?rn T?pel <[email protected]>
This patch will tell the user that bpf_xdp_adjust_head() is currently not supported.
--- drivers/net/ethernet/intel/i40e/i40e_main.c | 3 +++ 1 file changed, 3 insertions(+)
drivers/net/ethernet/intel/i40e/i40e_main.c +3 -0
@@ -9532,6 +9532,9 @@ static int i40e_xdp_setup(struct i40e_vsi *vsi,bool need_reset;struct bpf_prog *old_prog; +if (prog && prog->xdp_adjust_head) +return -EOPNOTSUPP ; +/* The Rx frame has to fit in 2k */ if (frame_size >I40E_RXBUFFER_2048 )return -EINVAL ;
Signature · 1 line
2.9.3