Skip to contentSkip to conversation

ice: Add support for XDP

PATCH New Tony Nguyen 2 participants 6 messages Oct 31, 2019 – Nov 4, 2019

Subsystems (2)
lore
Share & tools
lore

Tony Nguyen

author maintainer
#0 lore raw

PATCH v4 3/9 ice: Add support for XDP

From: Maciej Fijalkowski <[email protected]>

Add support for XDP. Implement ndo_bpf and ndo_xdp_xmit. Upon load of an XDP program, allocate additional Tx rings for dedicated XDP use. The following actions are supported: XDP_TX, XDP_DROP, XDP_REDIRECT, XDP_PASS, and XDP_ABORTED.

---
v4:
- return -EOPNOTSUPP instead of ENOTSUPP for too large MTU which makes it impossible to attach XDP prog
- don't check for case when there's no XDP prog currently on interface and ice_xdp() is called with NULL bpf_prog; this happens when user does "ip link set eth0 xdp off" and no prog is present on VSI; no need for that as it is handled by higher layer
- drop the extack message for unknown xdp->command
- use the smp_processor_id() for accessing the XDP Tx ring for XDP_TX action
- don't leave the interface in downed state in case of any failure during the XDP Tx resources handling
- undo rename of ice_build_ctob

---
 drivers/net/ethernet/intel/ice/ice.h         |  24 +-
 drivers/net/ethernet/intel/ice/ice_base.c    |  28 +-
 drivers/net/ethernet/intel/ice/ice_ethtool.c |  50 ++-
 drivers/net/ethernet/intel/ice/ice_lib.c     |  68 +++-
 drivers/net/ethernet/intel/ice/ice_lib.h     |   6 +
 drivers/net/ethernet/intel/ice/ice_main.c    | 326 +++++++++++++++++
 drivers/net/ethernet/intel/ice/ice_txrx.c    | 346 ++++++++++++++++---
 drivers/net/ethernet/intel/ice/ice_txrx.h    |  34 +-
 8 files changed, 825 insertions(+), 57 deletions(-)
drivers/net/ethernet/intel/ice/ice.h +22 -2
index 42eaae2f356d..0a90de35ca96 100644
@@ -29,8 +29,10 @@
 #include <linux/ip.h>
 #include <linux/sctp.h>
 #include <linux/ipv6.h>
+#include <linux/pkt_sched.h>
 #include <linux/if_bridge.h>
 #include <linux/ctype.h>
+#include <linux/bpf.h>
 #include <linux/avf/virtchnl.h>
 #include <linux/mfd/core.h>
 #include <net/ipv6.h>
@@ -81,8 +83,7 @@ extern const char ice_drv_ver[];
 
 #define ICE_DFLT_NETIF_M (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
 
-#define ICE_MAX_MTU	(ICE_AQ_SET_MAC_FRAME_SIZE_MAX - \
-			(ETH_HLEN + ETH_FCS_LEN + (VLAN_HLEN * 2)))
+#define ICE_MAX_MTU	(ICE_AQ_SET_MAC_FRAME_SIZE_MAX - ICE_ETH_PKT_HDR_PAD)
 
 #define ICE_UP_TABLE_TRANSLATE(val, i) \
 		(((val) << ICE_AQ_VSI_UP_TABLE_UP##i##_S) & \
@@ -286,6 +287,10 @@ struct ice_vsi {
 	u16 num_tx_desc;
 	u16 qset_handle[ICE_MAX_TRAFFIC_CLASS];
 	struct ice_tc_cfg tc_cfg;
+	struct bpf_prog *xdp_prog;
+	struct ice_ring **xdp_rings;	 /* XDP ring array */
+	u16 num_xdp_txq;		 /* Used XDP queues */
+	u8 xdp_mapping_mode;		 /* ICE_MAP_MODE_[CONTIG|SCATTER] */
 } ____cacheline_internodealigned_in_smp;
 
 /* struct that defines an interrupt vector */
@@ -438,6 +443,16 @@ static inline struct ice_pf *ice_netdev_to_pf(struct net_device *netdev)
 	return np->vsi->back;
 }
 
+static inline bool ice_is_xdp_ena_vsi(struct ice_vsi *vsi)
+{
+	return !!vsi->xdp_prog;
+}
+
+static inline void ice_set_ring_xdp(struct ice_ring *ring)
+{
+	ring->flags |= ICE_TX_FLAGS_RING_XDP;
+}
+
 /**
  * ice_get_main_vsi - Get the PF VSI
  * @pf: PF instance
@@ -464,6 +479,11 @@ int ice_up(struct ice_vsi *vsi);
 int ice_down(struct ice_vsi *vsi);
 int ice_vsi_cfg(struct ice_vsi *vsi);
 struct ice_vsi *ice_lb_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi);
+int ice_prepare_xdp_rings(struct ice_vsi *vsi, struct bpf_prog *prog);
+int ice_destroy_xdp_rings(struct ice_vsi *vsi);
+int
+ice_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames,
+	     u32 flags);
 int ice_set_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size);
 int ice_get_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size);
 void ice_fill_rss_lut(u8 *lut, u16 rss_table_size, u16 rss_size);
drivers/net/ethernet/intel/ice/ice_base.c +27 -1
index 1dd9c18ecd8c..e335256d483d 100644
@@ -198,6 +198,9 @@ static void ice_cfg_itr_gran(struct ice_hw *hw)
  */
 static u16 ice_calc_q_handle(struct ice_vsi *vsi, struct ice_ring *ring, u8 tc)
 {
+	WARN_ONCE(ice_ring_is_xdp(ring) && tc,
+		  "XDP ring can't belong to TC other than 0");
+
 	/* idea here for calculation is that we subtract the number of queue
 	 * count from TC that ring belongs to from it's absolute queue index
 	 * and as a result we get the queue's index within TC.
@@ -287,6 +290,22 @@ int ice_setup_rx_ctx(struct ice_ring *ring)
 	/* clear the context structure first */
 	memset(&rlan_ctx, 0, sizeof(rlan_ctx));
 
+	ring->rx_buf_len = vsi->rx_buf_len;
+
+	if (ring->vsi->type == ICE_VSI_PF) {
+		if (!xdp_rxq_info_is_reg(&ring->xdp_rxq))
+			xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev,
+					 ring->q_index);
+
+		err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
+						 MEM_TYPE_PAGE_SHARED, NULL);
+		if (err)
+			return err;
+	}
+	/* Receive Queue Base Address.
+	 * Indicates the starting address of the descriptor queue defined in
+	 * 128 Byte units.
+	 */
 	rlan_ctx.base = ring->dma >> 7;
 
 	rlan_ctx.qlen = ring->count;
@@ -294,7 +313,7 @@ int ice_setup_rx_ctx(struct ice_ring *ring)
 	/* Receive Packet Data Buffer Size.
 	 * The Packet Data Buffer Size is defined in 128 byte units.
 	 */
-	rlan_ctx.dbuf = vsi->rx_buf_len >> ICE_RLAN_CTX_DBUF_S;
+	rlan_ctx.dbuf = ring->rx_buf_len >> ICE_RLAN_CTX_DBUF_S;
 
 	/* use 32 byte descriptors */
 	rlan_ctx.dsize = 1;
@@ -657,6 +676,13 @@ ice_cfg_txq_interrupt(struct ice_vsi *vsi, u16 txq, u16 msix_idx, u16 itr_idx)
 	      ((msix_idx << QINT_TQCTL_MSIX_INDX_S) & QINT_TQCTL_MSIX_INDX_M);
 
 	wr32(hw, QINT_TQCTL(vsi->txq_map[txq]), val);
+	if (ice_is_xdp_ena_vsi(vsi)) {
+		u32 xdp_txq = txq + vsi->num_xdp_txq;
+
+		wr32(hw, QINT_TQCTL(vsi->txq_map[xdp_txq]),
+		     val);
+	}
+	ice_flush(hw);
 }
 
 /**
drivers/net/ethernet/intel/ice/ice_ethtool.c +47 -3

80 lines not on this page · raw message

drivers/net/ethernet/intel/ice/ice_lib.c +59 -9

124 lines not on this page · raw message

drivers/net/ethernet/intel/ice/ice_lib.h +6 -0
index 9919d3f810e3..53758ec71d3a 100644
@@ -38,6 +38,10 @@ int
 ice_vsi_stop_lan_tx_rings(struct ice_vsi *vsi, enum ice_disq_rst_src rst_src,
 			  u16 rel_vmvf_num);
 
+int ice_vsi_cfg_xdp_txqs(struct ice_vsi *vsi);
+
+int ice_vsi_stop_xdp_tx_rings(struct ice_vsi *vsi);
+
 int ice_cfg_vlan_pruning(struct ice_vsi *vsi, bool ena, bool vlan_promisc);
 
 void ice_cfg_sw_lldp(struct ice_vsi *vsi, bool tx, bool create);
@@ -81,6 +85,8 @@ void ice_vsi_free_tx_rings(struct ice_vsi *vsi);
 
 int ice_vsi_manage_rss_lut(struct ice_vsi *vsi, bool ena);
 
+void ice_vsi_cfg_frame_size(struct ice_vsi *vsi);
+
 u32 ice_intrl_usec_to_reg(u8 intrl, u8 gran);
 
 char *ice_nvm_version_str(struct ice_hw *hw);
drivers/net/ethernet/intel/ice/ice_main.c +326 -0

360 lines not on this page · raw message

drivers/net/ethernet/intel/ice/ice_txrx.c +305 -41

481 lines not on this page · raw message

drivers/net/ethernet/intel/ice/ice_txrx.h +33 -1
index a914e603b2ed..e40b4cb54ce3 100644
@@ -22,6 +22,16 @@
 #define ICE_RX_BUF_WRITE	16	/* Must be power of 2 */
 #define ICE_MAX_TXQ_PER_TXQG	128
 
+static inline __le64
+build_ctob(u64 td_cmd, u64 td_offset, unsigned int size, u64 td_tag)
+{
+	return cpu_to_le64(ICE_TX_DESC_DTYPE_DATA |
+			   (td_cmd    << ICE_TXD_QW1_CMD_S) |
+			   (td_offset << ICE_TXD_QW1_OFFSET_S) |
+			   ((u64)size << ICE_TXD_QW1_TX_BUF_SZ_S) |
+			   (td_tag    << ICE_TXD_QW1_L2TAG1_S));
+}
+
 /* We are assuming that the cache line is always 64 Bytes here for ice.
  * In order to make sure that is a correct assumption there is a check in probe
  * to print a warning if the read from GLPCI_CNF2 tells us that the cache line
@@ -49,12 +59,24 @@
 #define ICE_TX_FLAGS_VLAN_PR_S	29
 #define ICE_TX_FLAGS_VLAN_S	16
 
+#define ICE_XDP_PASS		0
+#define ICE_XDP_CONSUMED	BIT(0)
+#define ICE_XDP_TX		BIT(1)
+#define ICE_XDP_REDIR		BIT(2)
+
 #define ICE_RX_DMA_ATTR \
 	(DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)
 
+#define ICE_ETH_PKT_HDR_PAD	(ETH_HLEN + ETH_FCS_LEN + (VLAN_HLEN * 2))
+
+#define ICE_TXD_LAST_DESC_CMD (ICE_TX_DESC_CMD_EOP | ICE_TX_DESC_CMD_RS)
+
 struct ice_tx_buf {
 	struct ice_tx_desc *next_to_watch;
-	struct sk_buff *skb;
+	union {
+		struct sk_buff *skb;
+		void *raw_buf; /* used for XDP */
+	};
 	unsigned int bytecount;
 	unsigned short gso_segs;
 	u32 tx_flags;
@@ -198,9 +220,14 @@ struct ice_ring {
 	};
 
 	struct rcu_head rcu;		/* to avoid race on free */
+	struct bpf_prog *xdp_prog;
+	/* CL3 - 3rd cacheline starts here */
+	struct xdp_rxq_info xdp_rxq;
 	/* CLX - the below items are only accessed infrequently and should be
 	 * in their own cache line if possible
 	 */
+#define ICE_TX_FLAGS_RING_XDP		BIT(0)
+	u8 flags;
 	dma_addr_t dma;			/* physical address of ring */
 	unsigned int size;		/* length of descriptor ring in bytes */
 	u32 txq_teid;			/* Added Tx queue TEID */
@@ -208,6 +235,11 @@ struct ice_ring {
 	u8 dcb_tc;			/* Traffic class of ring */
 } ____cacheline_internodealigned_in_smp;
 
+static inline bool ice_ring_is_xdp(struct ice_ring *ring)
+{
+	return !!(ring->flags & ICE_TX_FLAGS_RING_XDP);
+}
+
 struct ice_ring_container {
 	/* head of linked-list of rings */
 	struct ice_ring *ring;
Signature · 1 line
2.20.1
author maintainer
#1 lore raw

PATCH v4 4/9 ice: Move common functions to ice_txrx_lib.c

From: Krzysztof Kazimierczak <[email protected]>

In preparation of AF XDP, move functions that will be used both by skb and zero-copy paths to a new file called ice_txrx_lib.c. This allows us to avoid using ifdefs to control the staticness of said functions.

Move other functions (ice_rx_csum, ice_rx_hash and ice_ptype_to_htype) called only by the moved ones to the new file as well.

---
v4:
- Change ice_build_ctob() call to build_ctob()
v2:
- Move ice_build_ctob() to ice_txrx_lib.h

---
 drivers/net/ethernet/intel/ice/Makefile       |   1 +
 drivers/net/ethernet/intel/ice/ice_txrx.c     | 303 +-----------------
 drivers/net/ethernet/intel/ice/ice_txrx.h     |  10 -
 drivers/net/ethernet/intel/ice/ice_txrx_lib.c | 273 ++++++++++++++++
 drivers/net/ethernet/intel/ice/ice_txrx_lib.h |  59 ++++
 5 files changed, 334 insertions(+), 312 deletions(-)
 create mode 100644 drivers/net/ethernet/intel/ice/ice_txrx_lib.c
 create mode 100644 drivers/net/ethernet/intel/ice/ice_txrx_lib.h
drivers/net/ethernet/intel/ice/Makefile +1 -0
index ff737f880c12..94fe430fdfdd 100644
@@ -15,6 +15,7 @@ ice-y := ice_main.o	\
 	 ice_sched.o	\
 	 ice_base.o	\
 	 ice_lib.o	\
+	 ice_txrx_lib.o	\
 	 ice_txrx.o	\
 	 ice_flex_pipe.o	\
 	 ice_idc.o	\
drivers/net/ethernet/intel/ice/ice_txrx.c +1 -302

339 lines not on this page · raw message

drivers/net/ethernet/intel/ice/ice_txrx.h +0 -10
index e40b4cb54ce3..a07101b13226 100644
@@ -22,16 +22,6 @@
 #define ICE_RX_BUF_WRITE	16	/* Must be power of 2 */
 #define ICE_MAX_TXQ_PER_TXQG	128
 
-static inline __le64
-build_ctob(u64 td_cmd, u64 td_offset, unsigned int size, u64 td_tag)
-{
-	return cpu_to_le64(ICE_TX_DESC_DTYPE_DATA |
-			   (td_cmd    << ICE_TXD_QW1_CMD_S) |
-			   (td_offset << ICE_TXD_QW1_OFFSET_S) |
-			   ((u64)size << ICE_TXD_QW1_TX_BUF_SZ_S) |
-			   (td_tag    << ICE_TXD_QW1_L2TAG1_S));
-}
-
 /* We are assuming that the cache line is always 64 Bytes here for ice.
  * In order to make sure that is a correct assumption there is a check in probe
  * to print a warning if the read from GLPCI_CNF2 tells us that the cache line
drivers/net/ethernet/intel/ice/ice_txrx_lib.c new file +273 -0

273 lines not on this page · raw message

drivers/net/ethernet/intel/ice/ice_txrx_lib.h new file +59 -0

59 lines not on this page · raw message

Signature · 1 line
2.20.1
#2 lore raw

PATCH v4 4/9 ice: Move common functions to ice_txrx_lib.c

Quoted 24 lines

-----Original Message-----
From: Intel-wired-lan [mailto:intel-wired-lan-bounces at osuosl.org] On Behalf Of Tony Nguyen
Sent: Thursday, October 31, 2019 7:30 AM
To: intel-wired-lan at lists.osuosl.org
Subject: [Intel-wired-lan] [PATCH S30 v4 4/9] ice: Move common functions to ice_txrx_lib.c

From: Krzysztof Kazimierczak <[email protected]>

In preparation of AF XDP, move functions that will be used both by skb and zero-copy paths to a new file called ice_txrx_lib.c. This allows us to avoid using ifdefs to control the staticness of said functions.

Move other functions (ice_rx_csum, ice_rx_hash and ice_ptype_to_htype) called only by the moved ones to the new file as well.

---
v4:
- Change ice_build_ctob() call to build_ctob()
v2:
- Move ice_build_ctob() to ice_txrx_lib.h

Quoted 9 lines
---
 drivers/net/ethernet/intel/ice/Makefile       |   1 +
 drivers/net/ethernet/intel/ice/ice_txrx.c     | 303 +-----------------
 drivers/net/ethernet/intel/ice/ice_txrx.h     |  10 -
 drivers/net/ethernet/intel/ice/ice_txrx_lib.c | 273 ++++++++++++++++
drivers/net/ethernet/intel/ice/ice_txrx_lib.h |  59 ++++
 5 files changed, 334 insertions(+), 312 deletions(-)  create mode 100644
drivers/net/ethernet/intel/ice/ice_txrx_lib.c
 create mode 100644 drivers/net/ethernet/intel/ice/ice_txrx_lib.h
author maintainer
#3 lore raw

PATCH v4 5/9 ice: Add support for AF_XDP

In reply to Tony Nguyen

From: Krzysztof Kazimierczak <[email protected]>

Add zero copy AF_XDP support. This patch adds zero copy support for Tx and Rx; code for zero copy is added to ice_xsk.h and ice_xsk.c.

For Tx, implement ndo_xsk_wakeup. As with other drivers, reuse existing XDP Tx queues for this task, since XDP_REDIRECT guarantees mutual exclusion between different NAPI contexts based on CPU ID. In turn, a netdev can XDP_REDIRECT to another netdev with a different NAPI context, since the operation is bound to a specific core and each core has its own hardware ring.

For Rx, allocate frames as MEM_TYPE_ZERO_COPY on queues that AF_XDP is enabled.

---
v4:
- Change ice_build_ctob() call to build_ctob()
v3:
- Remove unused prototype ice_xsk_umem_query()

---
 drivers/net/ethernet/intel/ice/Makefile      |    1 +
 drivers/net/ethernet/intel/ice/ice.h         |   26 +
 drivers/net/ethernet/intel/ice/ice_base.c    |   53 +-
 drivers/net/ethernet/intel/ice/ice_ethtool.c |    7 +
 drivers/net/ethernet/intel/ice/ice_lib.c     |   57 +-
 drivers/net/ethernet/intel/ice/ice_lib.h     |    4 +
 drivers/net/ethernet/intel/ice/ice_main.c    |   16 +
 drivers/net/ethernet/intel/ice/ice_txrx.c    |   46 +-
 drivers/net/ethernet/intel/ice/ice_txrx.h    |   20 +-
 drivers/net/ethernet/intel/ice/ice_xsk.c     | 1181 ++++++++++++++++++
 drivers/net/ethernet/intel/ice/ice_xsk.h     |   72 ++
 11 files changed, 1456 insertions(+), 27 deletions(-)
 create mode 100644 drivers/net/ethernet/intel/ice/ice_xsk.c
 create mode 100644 drivers/net/ethernet/intel/ice/ice_xsk.h
drivers/net/ethernet/intel/ice/Makefile +1 -0
index 94fe430fdfdd..774aad3ce237 100644
@@ -22,3 +22,4 @@ ice-y := ice_main.o	\
 	 ice_ethtool.o
 ice-$(CONFIG_PCI_IOV) += ice_virtchnl_pf.o ice_sriov.o
 ice-$(CONFIG_DCB) += ice_dcb.o ice_dcb_lib.o
+ice-$(CONFIG_XDP_SOCKETS) += ice_xsk.o
drivers/net/ethernet/intel/ice/ice.h +26 -0
index 0a90de35ca96..61e2030283aa 100644
@@ -36,6 +36,7 @@
 #include <linux/avf/virtchnl.h>
 #include <linux/mfd/core.h>
 #include <net/ipv6.h>
+#include <net/xdp_sock.h>
 #include "ice_devids.h"
 #include "ice_type.h"
 #include "ice_txrx.h"
@@ -46,6 +47,7 @@
 #include "ice_idc_int.h"
 #include "ice_virtchnl_pf.h"
 #include "ice_sriov.h"
+#include "ice_xsk.h"
 
 extern const char ice_drv_ver[];
 #define ICE_BAR0		0
@@ -291,6 +293,9 @@ struct ice_vsi {
 	struct ice_ring **xdp_rings;	 /* XDP ring array */
 	u16 num_xdp_txq;		 /* Used XDP queues */
 	u8 xdp_mapping_mode;		 /* ICE_MAP_MODE_[CONTIG|SCATTER] */
+	struct xdp_umem **xsk_umems;
+	u16 num_xsk_umems_used;
+	u16 num_xsk_umems;
 } ____cacheline_internodealigned_in_smp;
 
 /* struct that defines an interrupt vector */
@@ -453,6 +458,27 @@ static inline void ice_set_ring_xdp(struct ice_ring *ring)
 	ring->flags |= ICE_TX_FLAGS_RING_XDP;
 }
 
+/**
+ * ice_xsk_umem - get XDP UMEM bound to a ring
+ * @ring - ring to use
+ *
+ * Returns a pointer to xdp_umem structure if there is an UMEM present,
+ * NULL otherwise.
+ */
+static inline struct xdp_umem *ice_xsk_umem(struct ice_ring *ring)
+{
+	struct xdp_umem **umems = ring->vsi->xsk_umems;
+	int qid = ring->q_index;
+
+	if (ice_ring_is_xdp(ring))
+		qid -= ring->vsi->num_xdp_txq;
+
+	if (!umems || !umems[qid] || !ice_is_xdp_ena_vsi(ring->vsi))
+		return NULL;
+
+	return umems[qid];
+}
+
 /**
  * ice_get_main_vsi - Get the PF VSI
  * @pf: PF instance
drivers/net/ethernet/intel/ice/ice_base.c +46 -7

84 lines not on this page · raw message

drivers/net/ethernet/intel/ice/ice_ethtool.c +7 -0
index 6cee99b5865b..42b032620f66 100644
@@ -2612,6 +2612,13 @@ ice_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring)
 		return 0;
 	}
 
+	/* If there is a AF_XDP UMEM attached to any of Rx rings,
+	 * disallow changing the number of descriptors -- regardless
+	 * if the netdev is running or not.
+	 */
+	if (ice_xsk_any_rx_ring_ena(vsi))
+		return -EBUSY;
+
 	while (test_and_set_bit(__ICE_CFG_BUSY, pf->state)) {
 		timeout--;
 		if (!timeout)
drivers/net/ethernet/intel/ice/ice_lib.c +56 -1

69 lines not on this page · raw message

drivers/net/ethernet/intel/ice/ice_lib.h +4 -0
index 53758ec71d3a..12cb89e28631 100644
@@ -85,6 +85,10 @@ void ice_vsi_free_tx_rings(struct ice_vsi *vsi);
 
 int ice_vsi_manage_rss_lut(struct ice_vsi *vsi, bool ena);
 
+void ice_update_tx_ring_stats(struct ice_ring *ring, u64 pkts, u64 bytes);
+
+void ice_update_rx_ring_stats(struct ice_ring *ring, u64 pkts, u64 bytes);
+
 void ice_vsi_cfg_frame_size(struct ice_vsi *vsi);
 
 u32 ice_intrl_usec_to_reg(u8 intrl, u8 gran);
drivers/net/ethernet/intel/ice/ice_main.c +16 -0
index fff7e5567910..5b270fe0e349 100644
@@ -1734,6 +1734,7 @@ static int ice_xdp_alloc_setup_rings(struct ice_vsi *vsi)
 		if (ice_setup_tx_ring(xdp_ring))
 			goto free_xdp_rings;
 		ice_set_ring_xdp(xdp_ring);
+		xdp_ring->xsk_umem = ice_xsk_umem(xdp_ring);
 	}
 
 	return 0;
@@ -1976,6 +1977,17 @@ ice_xdp_setup_prog(struct ice_vsi *vsi, struct bpf_prog *prog,
 	if (if_running)
 		ret = ice_up(vsi);
 
+	if (!ret && prog && vsi->xsk_umems) {
+		int i;
+
+		ice_for_each_rxq(vsi, i) {
+			struct ice_ring *rx_ring = vsi->rx_rings[i];
+
+			if (rx_ring->xsk_umem)
+				napi_schedule(&rx_ring->q_vector->napi);
+		}
+	}
+
 	return (ret || xdp_ring_err) ? -ENOMEM : 0;
 }
 
@@ -2001,6 +2013,9 @@ static int ice_xdp(struct net_device *dev, struct netdev_bpf *xdp)
 	case XDP_QUERY_PROG:
 		xdp->prog_id = vsi->xdp_prog ? vsi->xdp_prog->aux->id : 0;
 		return 0;
+	case XDP_SETUP_XSK_UMEM:
+		return ice_xsk_umem_setup(vsi, xdp->xsk.umem,
+					  xdp->xsk.queue_id);
 	default:
 		return -EINVAL;
 	}
@@ -5334,4 +5349,5 @@ static const struct net_device_ops ice_netdev_ops = {
 	.ndo_tx_timeout = ice_tx_timeout,
 	.ndo_bpf = ice_xdp,
 	.ndo_xdp_xmit = ice_xdp_xmit,
+	.ndo_xsk_wakeup = ice_xsk_wakeup,
 };
drivers/net/ethernet/intel/ice/ice_txrx.c +30 -16

101 lines not on this page · raw message

drivers/net/ethernet/intel/ice/ice_txrx.h +17 -3
index a07101b13226..d5d243b8e69f 100644
@@ -4,6 +4,8 @@
 #ifndef _ICE_TXRX_H_
 #define _ICE_TXRX_H_
 
+#include "ice_type.h"
+
 #define ICE_DFLT_IRQ_WORK	256
 #define ICE_RXBUF_2048		2048
 #define ICE_MAX_CHAINED_RX_BUFS	5
@@ -88,9 +90,17 @@ struct ice_tx_offload_params {
 struct ice_rx_buf {
 	struct sk_buff *skb;
 	dma_addr_t dma;
-	struct page *page;
-	unsigned int page_offset;
-	u16 pagecnt_bias;
+	union {
+		struct {
+			struct page *page;
+			unsigned int page_offset;
+			u16 pagecnt_bias;
+		};
+		struct {
+			void *addr;
+			u64 handle;
+		};
+	};
 };
 
 struct ice_q_stats {
@@ -211,6 +221,8 @@ struct ice_ring {
 
 	struct rcu_head rcu;		/* to avoid race on free */
 	struct bpf_prog *xdp_prog;
+	struct xdp_umem *xsk_umem;
+	struct zero_copy_allocator zca;
 	/* CL3 - 3rd cacheline starts here */
 	struct xdp_rxq_info xdp_rxq;
 	/* CLX - the below items are only accessed infrequently and should be
@@ -250,6 +262,8 @@ struct ice_ring_container {
 #define ice_for_each_ring(pos, head) \
 	for (pos = (head).ring; pos; pos = pos->next)
 
+union ice_32b_rx_flex_desc;
+
 bool ice_alloc_rx_bufs(struct ice_ring *rxr, u16 cleaned_count);
 netdev_tx_t ice_start_xmit(struct sk_buff *skb, struct net_device *netdev);
 void ice_clean_tx_ring(struct ice_ring *tx_ring);
drivers/net/ethernet/intel/ice/ice_xsk.c new file +1181 -0

1,181 lines not on this page · raw message

drivers/net/ethernet/intel/ice/ice_xsk.h new file +72 -0

72 lines not on this page · raw message

Signature · 1 line
2.20.1
#4 lore raw

PATCH v4 5/9 ice: Add support for AF_XDP

Quoted 31 lines

-----Original Message-----
From: Intel-wired-lan [mailto:intel-wired-lan-bounces at osuosl.org] On Behalf Of Tony Nguyen
Sent: Thursday, October 31, 2019 7:30 AM
To: intel-wired-lan at lists.osuosl.org
Subject: [Intel-wired-lan] [PATCH S30 v4 5/9] ice: Add support for AF_XDP

From: Krzysztof Kazimierczak <[email protected]>

Add zero copy AF_XDP support. This patch adds zero copy support for Tx and Rx; code for zero copy is added to ice_xsk.h and ice_xsk.c.

For Tx, implement ndo_xsk_wakeup. As with other drivers, reuse existing XDP Tx queues for this task, since XDP_REDIRECT guarantees mutual exclusion between different NAPI contexts based on CPU ID. In turn, a netdev can XDP_REDIRECT to another netdev with a different NAPI context, since the operation is bound to a specific core and each core has its own hardware ring.

For Rx, allocate frames as MEM_TYPE_ZERO_COPY on queues that AF_XDP is enabled.

---
v4:
- Change ice_build_ctob() call to build_ctob()
v3:
- Remove unused prototype ice_xsk_umem_query()

Quoted 15 lines
---
 drivers/net/ethernet/intel/ice/Makefile      |    1 +
 drivers/net/ethernet/intel/ice/ice.h         |   26 +
 drivers/net/ethernet/intel/ice/ice_base.c    |   53 +-
 drivers/net/ethernet/intel/ice/ice_ethtool.c |    7 +
 drivers/net/ethernet/intel/ice/ice_lib.c     |   57 +-
 drivers/net/ethernet/intel/ice/ice_lib.h     |    4 +
 drivers/net/ethernet/intel/ice/ice_main.c    |   16 +
 drivers/net/ethernet/intel/ice/ice_txrx.c    |   46 +-
 drivers/net/ethernet/intel/ice/ice_txrx.h    |   20 +-
 drivers/net/ethernet/intel/ice/ice_xsk.c     | 1181 ++++++++++++++++++
 drivers/net/ethernet/intel/ice/ice_xsk.h     |   72 ++
 11 files changed, 1456 insertions(+), 27 deletions(-)  create mode 100644
drivers/net/ethernet/intel/ice/ice_xsk.c
 create mode 100644 drivers/net/ethernet/intel/ice/ice_xsk.h
#5 lore raw

PATCH v4 3/9 ice: Add support for XDP

In reply to Tony Nguyen
Quoted 30 lines

-----Original Message-----
From: Intel-wired-lan [mailto:intel-wired-lan-bounces at osuosl.org] On Behalf Of Tony Nguyen
Sent: Thursday, October 31, 2019 7:30 AM
To: intel-wired-lan at lists.osuosl.org
Subject: [Intel-wired-lan] [PATCH S30 v4 3/9] ice: Add support for XDP

From: Maciej Fijalkowski <[email protected]>

Add support for XDP. Implement ndo_bpf and ndo_xdp_xmit. Upon load of an XDP program, allocate additional Tx rings for dedicated XDP use. The following actions are supported: XDP_TX, XDP_DROP, XDP_REDIRECT, XDP_PASS, and XDP_ABORTED.

---
v4:
- return -EOPNOTSUPP instead of ENOTSUPP for too large MTU which makes it impossible to attach XDP prog
- don't check for case when there's no XDP prog currently on interface and ice_xdp() is called with NULL bpf_prog; this happens when user does "ip link set eth0 xdp off" and no prog is present on VSI; no need for that as it is handled by higher layer
- drop the extack message for unknown xdp->command
- use the smp_processor_id() for accessing the XDP Tx ring for XDP_TX action
- don't leave the interface in downed state in case of any failure during the XDP Tx resources handling
- undo rename of ice_build_ctob

Quoted 10 lines
---
 drivers/net/ethernet/intel/ice/ice.h         |  24 +-
 drivers/net/ethernet/intel/ice/ice_base.c    |  28 +-
 drivers/net/ethernet/intel/ice/ice_ethtool.c |  50 ++-
 drivers/net/ethernet/intel/ice/ice_lib.c     |  68 +++-
 drivers/net/ethernet/intel/ice/ice_lib.h     |   6 +
 drivers/net/ethernet/intel/ice/ice_main.c    | 326 +++++++++++++++++
 drivers/net/ethernet/intel/ice/ice_txrx.c    | 346 ++++++++++++++++---
 drivers/net/ethernet/intel/ice/ice_txrx.h    |  34 +-
 8 files changed, 825 insertions(+), 57 deletions(-)