Skip to contentSkip to conversation

i40e: Support for XDP

PATCH Superseded Björn Töpel 2 participants 8 messages Dec 13, 2016 – Dec 16, 2016

Subsystems (2)

v3 of 7 Superseded Latest is v7 →

lore
Share & tools
lore
#0 lore raw

v3 cover i40e: Support for XDP

From: Bj?rn T?pel <[email protected]>

This series adds XDP support for i40e-based NICs.

The first patch prepares i40e_fetch_rx_buffer() for upcoming changes, followed by XDP_RX support, the third adds XDP_TX support and the last patch validates bpf_xdp_adjust_head() support.

Thanks to Alex, Daniel and John for all the feedback!

v3:
* Rebased patch set on Jeff's dev-queue branch
* MSI-X is no longer a prerequisite for XDP
* RCU locking for the XDP program and XDP_RX support is introduced in the same patch
* Rx bytes is now bumped for XDP
* Removed pointer-to-pointer clunkiness
* Added comments to XDP preconditions in ndo_xdp
* When a non-EOF is received, log once, and drop the frame

v2:
* Fixed kbuild error for PAGE_SIZE >= 8192.
* Renamed i40e_try_flip_rx_page to i40e_can_reuse_rx_page, which is more in line to the other Intel Ethernet drivers (igb/fm10k).
* Validate xdp_adjust_head support in ndo_xdp/XDP_SETUP_PROG.

Bj?rn

Bj?rn T?pel (4):
i40e: Sync DMA region prior skbuff allocation
i40e: Initial support for XDP
i40e: Add XDP_TX support
i40e: Validate xdp_adjust_head support

 drivers/net/ethernet/intel/i40e/i40e.h         |  18 ++
 drivers/net/ethernet/intel/i40e/i40e_ethtool.c |   4 +
 drivers/net/ethernet/intel/i40e/i40e_main.c    | 380 ++++++++++++++++++++----
 drivers/net/ethernet/intel/i40e/i40e_txrx.c    | 391 ++++++++++++++++++++++---
 drivers/net/ethernet/intel/i40e/i40e_txrx.h    |   7 +
 5 files changed, 711 insertions(+), 89 deletions(-)
Signature · 1 line
2.9.3
#1 lore raw

PATCH v3 1/4 i40e: Sync DMA region prior skbuff allocation

From: Bj?rn T?pel <[email protected]>

This patch prepares i40e_fetch_rx_buffer() for upcoming XDP support, where there's a need to access the device buffers prior skbuff allocation.

---
 drivers/net/ethernet/intel/i40e/i40e_txrx.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)
drivers/net/ethernet/intel/i40e/i40e_txrx.c +7 -7
index e88e335c34ef..7dacde6608d1 100644
@@ -1646,6 +1646,13 @@ struct sk_buff *i40e_fetch_rx_buffer(struct i40e_ring *rx_ring,
 	page = rx_buffer->page;
 	prefetchw(page);
 
+	/* we are reusing so sync this buffer for CPU use */
+	dma_sync_single_range_for_cpu(rx_ring->dev,
+				      rx_buffer->dma,
+				      rx_buffer->page_offset,
+				      size,
+				      DMA_FROM_DEVICE);
+
 	if (likely(!skb)) {
 		void *page_addr = page_address(page) + rx_buffer->page_offset;
 
@@ -1671,13 +1678,6 @@ struct sk_buff *i40e_fetch_rx_buffer(struct i40e_ring *rx_ring,
 		prefetchw(skb->data);
 	}
 
-	/* we are reusing so sync this buffer for CPU use */
-	dma_sync_single_range_for_cpu(rx_ring->dev,
-				      rx_buffer->dma,
-				      rx_buffer->page_offset,
-				      size,
-				      DMA_FROM_DEVICE);
-
 	/* pull page into skb */
 	if (i40e_add_rx_frag(rx_ring, rx_buffer, size, skb)) {
 		/* hand second half of page back to the ring */
Signature · 1 line
2.9.3
#2 lore raw

PATCH v3 2/4 i40e: Initial support for XDP

In reply to Björn Töpel

From: Bj?rn T?pel <[email protected]>

This commit adds basic XDP support for i40e derived NICs. All XDP actions will end up in XDP_DROP.

Only the default/main VSI has support for enabling XDP.

---
 drivers/net/ethernet/intel/i40e/i40e.h         |  13 +++
 drivers/net/ethernet/intel/i40e/i40e_ethtool.c |   4 +
 drivers/net/ethernet/intel/i40e/i40e_main.c    |  83 +++++++++++++++++
 drivers/net/ethernet/intel/i40e/i40e_txrx.c    | 123 +++++++++++++++++++++++--
 drivers/net/ethernet/intel/i40e/i40e_txrx.h    |   2 +
 5 files changed, 219 insertions(+), 6 deletions(-)
drivers/net/ethernet/intel/i40e/i40e.h +13 -0
index 0f2bf241b24e..be26986ed25f 100644
@@ -587,6 +587,8 @@ struct i40e_vsi {
 	struct i40e_ring **rx_rings;
 	struct i40e_ring **tx_rings;
 
+	bool xdp_enabled;
+
 	u32  active_filters;
 	u32  promisc_threshold;
 
@@ -946,4 +948,15 @@ i40e_status i40e_get_npar_bw_setting(struct i40e_pf *pf);
 i40e_status i40e_set_npar_bw_setting(struct i40e_pf *pf);
 i40e_status i40e_commit_npar_bw_setting(struct i40e_pf *pf);
 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup);
+
+/**
+ * i40e_enabled_xdp_vsi - Check if VSI has XDP enabled
+ * @vsi: pointer to a vsi
+ *
+ * Returns true if the VSI has XDP enabled.
+ **/
+static inline bool i40e_enabled_xdp_vsi(const struct i40e_vsi *vsi)
+{
+	return !!vsi->xdp_enabled;
+}
 #endif /* _I40E_H_ */
drivers/net/ethernet/intel/i40e/i40e_ethtool.c +4 -0
index dece0d676482..ccb3b77405d7 100644
@@ -1257,6 +1257,10 @@ static int i40e_set_ringparam(struct net_device *netdev,
 	if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending))
 		return -EINVAL;
 
+	/* Don't allow any change while XDP is enabled. */
+	if (i40e_enabled_xdp_vsi(vsi))
+		return -EINVAL;
+
 	if (ring->tx_pending > I40E_MAX_NUM_DESCRIPTORS ||
 	    ring->tx_pending < I40E_MIN_NUM_DESCRIPTORS ||
 	    ring->rx_pending > I40E_MAX_NUM_DESCRIPTORS ||
drivers/net/ethernet/intel/i40e/i40e_main.c +83 -0

114 lines not on this page · raw message

drivers/net/ethernet/intel/i40e/i40e_txrx.c +117 -6

185 lines not on this page · raw message

drivers/net/ethernet/intel/i40e/i40e_txrx.h +2 -0
index f80979025c01..78d0aa0468f1 100644
@@ -361,6 +361,8 @@ struct i40e_ring {
 					 * i40e_clean_rx_ring_irq() is called
 					 * for this ring.
 					 */
+
+	struct bpf_prog __rcu *xdp_prog;
 } ____cacheline_internodealigned_in_smp;
 
 enum i40e_latency_range {
Signature · 1 line
2.9.3
#3 lore raw

PATCH v3 2/4 i40e: Initial support for XDP

Quoted 9 lines · Bj?rn T?pel

From: Bj?rn T?pel <[email protected]>

This commit adds basic XDP support for i40e derived NICs. All XDP actions will end up in XDP_DROP.

Only the default/main VSI has support for enabling XDP.

---

[...]

Quoted 20 lines

/**
+ * i40e_run_xdp - Runs an XDP program for an Rx ring
+ * @rx_ring: Rx ring used for XDP
+ * @rx_buffer: current Rx buffer
+ * @rx_desc: current Rx descriptor
+ * @size: buffer size
+ * @xdp_prog: the XDP program to run
+ *
+ * Returns true if the XDP program consumed the incoming frame. False
+ * means pass the frame to the good old stack.
+ **/

drivers/net/ethernet/intel/i40e/i40e_txrx.c +8 -0
+static bool i40e_run_xdp(struct i40e_ring *rx_ring,
+			 struct i40e_rx_buffer *rx_buffer,
+			 union i40e_rx_desc *rx_desc,
+			 unsigned int size,
+			 struct bpf_prog *xdp_prog)
+{
+	struct xdp_buff xdp;
+	u32 xdp_action;

drivers/net/ethernet/intel/i40e/i40e_txrx.c static bool i40e_add_rx_frag(struct i40e_ring *rx_ring, · line 1657

Quoted 10 lines
drivers/net/ethernet/intel/i40e/i40e_txrx.c +10 -0
+
+	if (unlikely(!i40e_test_staterr(rx_desc,
+					BIT(I40E_RX_DESC_STATUS_EOF_SHIFT)))) {
+		dev_warn_once(&rx_ring->vsi->back->pdev->dev,
+			      "Received unexpected RXD_EOF!\n");
+		goto do_drop;
+	}
+
+	xdp.data = page_address(rx_buffer->page) + rx_buffer->page_offset;
+	xdp.data_end = xdp.data + size;

We need to set the xdp.data_hard_start as well here.

xdp.data_hard_start = xdp.data

should be good enough for now.

Quoted 20 lines
drivers/net/ethernet/intel/i40e/i40e_txrx.c +20 -0
+	xdp_action = bpf_prog_run_xdp(xdp_prog, &xdp);
+
+	switch (xdp_action) {
+	case XDP_PASS:
+		return false;
+	default:
+		bpf_warn_invalid_xdp_action(xdp_action);
+	case XDP_ABORTED:
+	case XDP_TX:
+	case XDP_DROP:
+do_drop:
+		if (likely(i40e_page_is_reusable(rx_buffer->page))) {
+			i40e_reuse_rx_page(rx_ring, rx_buffer);
+			rx_ring->rx_stats.page_reuse_count++;
+			break;
+		}
+
+		/* we are not reusing the buffer so unmap it */
+		dma_unmap_page(rx_ring->dev, rx_buffer->dma, PAGE_SIZE,
+			       DMA_FROM_DEVICE);

drivers/net/ethernet/intel/i40e/i40e_txrx.c static bool i40e_add_rx_frag(struct i40e_ring *rx_ring, · line 1687

Quoted 10 lines
drivers/net/ethernet/intel/i40e/i40e_txrx.c +9 -0
+		__free_pages(rx_buffer->page, 0);
+	}
+
+	/* clear contents of buffer_info */
+	rx_buffer->page = NULL;
+	return true; /* Swallowed by XDP */
+}
+
+/**
  * i40e_fetch_rx_buffer - Allocate skb and populate it

Thanks,
John

#4 lore raw

PATCH v3 2/4 i40e: Initial support for XDP

2016-12-16 19:07 GMT+01:00 John Fastabend <[email protected]>:
[...]

Quoted 33 lines
Quoted 20 lines

/**
+ * i40e_run_xdp - Runs an XDP program for an Rx ring
+ * @rx_ring: Rx ring used for XDP
+ * @rx_buffer: current Rx buffer
+ * @rx_desc: current Rx descriptor
+ * @size: buffer size
+ * @xdp_prog: the XDP program to run
+ *
+ * Returns true if the XDP program consumed the incoming frame. False
+ * means pass the frame to the good old stack.
+ **/

drivers/net/ethernet/intel/i40e/i40e_txrx.c +8 -0
+static bool i40e_run_xdp(struct i40e_ring *rx_ring,
+                      struct i40e_rx_buffer *rx_buffer,
+                      union i40e_rx_desc *rx_desc,
+                      unsigned int size,
+                      struct bpf_prog *xdp_prog)
+{
+     struct xdp_buff xdp;
+     u32 xdp_action;

drivers/net/ethernet/intel/i40e/i40e_txrx.c static bool i40e_add_rx_frag(struct i40e_ring *rx_ring, · line 1657

Quoted 10 lines
drivers/net/ethernet/intel/i40e/i40e_txrx.c +10 -0
+
+     if (unlikely(!i40e_test_staterr(rx_desc,
+                                     BIT(I40E_RX_DESC_STATUS_EOF_SHIFT)))) {
+             dev_warn_once(&rx_ring->vsi->back->pdev->dev,
+                           "Received unexpected RXD_EOF!\n");
+             goto do_drop;
+     }
+
+     xdp.data = page_address(rx_buffer->page) + rx_buffer->page_offset;
+     xdp.data_end = xdp.data + size;

We need to set the xdp.data_hard_start as well here.

Quoted 3 lines

xdp.data_hard_start = xdp.data

should be good enough for now.

OK! Thanks for pointing that out. I'll address that in v4!

Bj?rn

#5 lore raw

PATCH v3 3/4 i40e: Add XDP_TX support

In reply to Björn Töpel

From: Bj?rn T?pel <[email protected]>

This patch adds proper XDP_TX support.

---
 drivers/net/ethernet/intel/i40e/i40e.h      |   5 +
 drivers/net/ethernet/intel/i40e/i40e_main.c | 294 +++++++++++++++++++++++-----
 drivers/net/ethernet/intel/i40e/i40e_txrx.c | 264 ++++++++++++++++++++++---
 drivers/net/ethernet/intel/i40e/i40e_txrx.h |   5 +
 4 files changed, 487 insertions(+), 81 deletions(-)
drivers/net/ethernet/intel/i40e/i40e.h +5 -0
index be26986ed25f..a91c8a258fbf 100644
@@ -587,6 +587,10 @@ struct i40e_vsi {
 	struct i40e_ring **rx_rings;
 	struct i40e_ring **tx_rings;
 
+	/* The XDP rings are Tx only, and follows the count of the
+	 * regular rings, i.e. alloc_queue_pairs/num_queue_pairs
+	 */
+	struct i40e_ring **xdp_rings;
 	bool xdp_enabled;
 
 	u32  active_filters;
@@ -664,6 +668,7 @@ struct i40e_q_vector {
 
 	struct i40e_ring_container rx;
 	struct i40e_ring_container tx;
+	struct i40e_ring_container xdp;
 
 	u8 num_ringpairs;	/* total number of ring pairs in vector */
 
drivers/net/ethernet/intel/i40e/i40e_main.c +241 -53

483 lines not on this page · raw message

drivers/net/ethernet/intel/i40e/i40e_txrx.c +236 -28

330 lines not on this page · raw message

drivers/net/ethernet/intel/i40e/i40e_txrx.h +5 -0
index 78d0aa0468f1..3250be70271d 100644
@@ -233,6 +233,7 @@ static inline unsigned int i40e_txd_use_count(unsigned int size)
 #define I40E_TX_FLAGS_TSYN		BIT(8)
 #define I40E_TX_FLAGS_FD_SB		BIT(9)
 #define I40E_TX_FLAGS_UDP_TUNNEL	BIT(10)
+#define I40E_TX_FLAGS_XDP		BIT(11)
 #define I40E_TX_FLAGS_VLAN_MASK		0xffff0000
 #define I40E_TX_FLAGS_VLAN_PRIO_MASK	0xe0000000
 #define I40E_TX_FLAGS_VLAN_PRIO_SHIFT	29
@@ -243,6 +244,7 @@ struct i40e_tx_buffer {
 	union {
 		struct sk_buff *skb;
 		void *raw_buf;
+		struct page *page;
 	};
 	unsigned int bytecount;
 	unsigned short gso_segs;
@@ -363,6 +365,9 @@ struct i40e_ring {
 					 */
 
 	struct bpf_prog __rcu *xdp_prog;
+	struct i40e_ring *xdp_sibling;  /* rx to xdp, and xdp to rx */
+	bool xdp_needs_tail_bump;
+	u16 curr_in_use;
 } ____cacheline_internodealigned_in_smp;
 
 enum i40e_latency_range {
Signature · 1 line
2.9.3
#6 lore raw

PATCH v3 3/4 i40e: Add XDP_TX support

2016-12-13 14:40 GMT+01:00 Bj?rn T?pel <[email protected]>:
[...]

drivers/net/ethernet/intel/i40e/i40e_txrx.c u32 i40e_get_tx_pending(struct i40e_ring *ring, bool in_sw) · line 636

Quoted 11 lines
drivers/net/ethernet/intel/i40e/i40e_txrx.c +7 -0
@@ -625,6 +627,15 @@ u32 i40e_get_tx_pending(struct i40e_ring *ring, bool in_sw)
 #define WB_STRIDE 4
 
 /**
+ * i40e_page_is_reserved - check if reuse is possible
+ * @page: page struct to check
+ */
+static inline bool i40e_page_is_reserved(struct page *page)
+{
+       return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page);
+}

I started to use use Scott's i40e_page_is_reusable(), but missed to get rid if this
leftover function.

So this one is not used, and should be removed.

[...]

Bj?rn

#7 lore raw

PATCH v3 4/4 i40e: Validate xdp_adjust_head support

In reply to Björn Töpel

From: Bj?rn T?pel <[email protected]>

This patch will tell the user that bpf_xdp_adjust_head() is currently not supported.

---
 drivers/net/ethernet/intel/i40e/i40e_main.c | 3 +++
 1 file changed, 3 insertions(+)
drivers/net/ethernet/intel/i40e/i40e_main.c +3 -0
index f7e627a540cc..f22cfa385b75 100644
@@ -9532,6 +9532,9 @@ static int i40e_xdp_setup(struct i40e_vsi *vsi,
 	bool need_reset;
 	struct bpf_prog *old_prog;
 
+	if (prog && prog->xdp_adjust_head)
+		return -EOPNOTSUPP;
+
 	/* The Rx frame has to fit in 2k */
 	if (frame_size > I40E_RXBUFFER_2048)
 		return -EINVAL;
Signature · 1 line
2.9.3