Skip to contentSkip to conversation

ice: Add support for XDP

PATCH Superseded Tony Nguyen 1 participant 2 messages Jun 26, 2019

Subsystems (2)

v2 of 4 Superseded Latest is v4 →

lore
Share & tools
lore

Tony Nguyen

author maintainer
#0 lore raw

PATCH v2 1/2 ice: Add support for XDP

From: Maciej Fijalkowski <[email protected]>

Add support for XDP. Implement ndo_bpf and ndo_xdp_xmit. Upon load of an XDP program, allocate additional Tx rings for dedicated XDP use. The following actions are supported: XDP_TX, XDP_DROP, XDP_REDIRECT, XDP_PASS, and XDP_ABORTED.

Move build_ctob() up so that no forward declaration is needed and rename it to ice_build_ctob() since it's an ice function.

---
 drivers/net/ethernet/intel/ice/ice.h          |  21 ++
 drivers/net/ethernet/intel/ice/ice_ethtool.c  |  53 ++-
 drivers/net/ethernet/intel/ice/ice_lib.c      |  84 ++++-
 drivers/net/ethernet/intel/ice/ice_lib.h      |   6 +
 drivers/net/ethernet/intel/ice/ice_main.c     | 319 ++++++++++++++++
 drivers/net/ethernet/intel/ice/ice_txrx.c     | 343 +++++++++++++++---
 drivers/net/ethernet/intel/ice/ice_txrx.h     |  29 +-
 .../net/ethernet/intel/ice/ice_virtchnl_pf.c  |   1 +
 8 files changed, 799 insertions(+), 57 deletions(-)
drivers/net/ethernet/intel/ice/ice.h +21 -0
index 9ee6b55553c0..53adb93c6b61 100644
@@ -28,7 +28,10 @@
 #include <linux/ip.h>
 #include <linux/sctp.h>
 #include <linux/ipv6.h>
+#include <linux/pkt_sched.h>
 #include <linux/if_bridge.h>
+#include <linux/ctype.h>
+#include <linux/bpf.h>
 #include <linux/avf/virtchnl.h>
 #include <net/ipv6.h>
 #include "ice_devids.h"
@@ -301,6 +304,10 @@ struct ice_vsi {
 	u16 num_rx_desc;
 	u16 num_tx_desc;
 	struct ice_tc_cfg tc_cfg;
+	struct bpf_prog *xdp_prog;
+	struct ice_ring **xdp_rings;	 /* XDP ring array */
+	u16 num_xdp_txq;		 /* Used XDP queues */
+	u8 xdp_mapping_mode;		 /* ICE_MAP_MODE_[CONTIG|SCATTER] */
 } ____cacheline_internodealigned_in_smp;
 
 /* struct that defines an interrupt vector */
@@ -432,6 +439,16 @@ ice_irq_dynamic_ena(struct ice_hw *hw, struct ice_vsi *vsi,
 	wr32(hw, GLINT_DYN_CTL(vector), val);
 }
 
+static inline bool ice_is_xdp_ena_vsi(struct ice_vsi *vsi)
+{
+	return !!vsi->xdp_prog;
+}
+
+static inline void ice_set_ring_xdp(struct ice_ring *ring)
+{
+	ring->tx_buf[0].tx_flags |= ICE_TX_FLAGS_RING_XDP;
+}
+
 /**
  * ice_find_vsi_by_type - Find and return VSI of a given type
  * @pf: PF to search for VSI
@@ -459,6 +476,10 @@ int ice_up(struct ice_vsi *vsi);
 int ice_down(struct ice_vsi *vsi);
 int ice_vsi_cfg(struct ice_vsi *vsi);
 struct ice_vsi *ice_lb_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi);
+int ice_prepare_xdp_rings(struct ice_vsi *vsi);
+int ice_destroy_xdp_rings(struct ice_vsi *vsi);
+int ice_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames,
+		 u32 flags);
 int ice_set_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size);
 int ice_get_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size);
 void ice_fill_rss_lut(u8 *lut, u16 rss_table_size, u16 rss_size);
drivers/net/ethernet/intel/ice/ice_ethtool.c +50 -3

83 lines not on this page · raw message

drivers/net/ethernet/intel/ice/ice_lib.c +75 -9

173 lines not on this page · raw message

drivers/net/ethernet/intel/ice/ice_lib.h +6 -0
index 6e43ef03bfc3..c4c6eca05757 100644
@@ -43,6 +43,10 @@ int
 ice_vsi_stop_lan_tx_rings(struct ice_vsi *vsi, enum ice_disq_rst_src rst_src,
 			  u16 rel_vmvf_num);
 
+int ice_vsi_cfg_xdp_txqs(struct ice_vsi *vsi);
+
+int ice_vsi_stop_xdp_tx_rings(struct ice_vsi *vsi);
+
 int ice_cfg_vlan_pruning(struct ice_vsi *vsi, bool ena, bool vlan_promisc);
 
 void ice_cfg_sw_lldp(struct ice_vsi *vsi, bool tx, bool create);
@@ -76,6 +80,8 @@ bool ice_is_reset_in_progress(unsigned long *state);
 
 void ice_vsi_free_q_vectors(struct ice_vsi *vsi);
 
+int __ice_vsi_get_qs(struct ice_qs_cfg *qs_cfg);
+
 void ice_trigger_sw_intr(struct ice_hw *hw, struct ice_q_vector *q_vector);
 
 void ice_vsi_put_qs(struct ice_vsi *vsi);
drivers/net/ethernet/intel/ice/ice_main.c +319 -0

347 lines not on this page · raw message

drivers/net/ethernet/intel/ice/ice_txrx.c +299 -44

496 lines not on this page · raw message

drivers/net/ethernet/intel/ice/ice_txrx.h +28 -1
index ec76aba347b9..355834b4abda 100644
@@ -44,17 +44,33 @@
 #define ICE_TX_FLAGS_TSO	BIT(0)
 #define ICE_TX_FLAGS_HW_VLAN	BIT(1)
 #define ICE_TX_FLAGS_SW_VLAN	BIT(2)
+/* ICE_TX_FLAGS_RING_XDP is used to indicate that whole ring is dedicated for
+ * XDP purposes; at this point struct ice_ring doesn't have an appropriate
+ * field that could be used for setting this flag, so let's use the tx_flags
+ * field of the first ice_tx_buf from ice_ring
+ */
+#define ICE_TX_FLAGS_RING_XDP	BIT(8)
 #define ICE_TX_FLAGS_VLAN_M	0xffff0000
 #define ICE_TX_FLAGS_VLAN_PR_M	0xe0000000
 #define ICE_TX_FLAGS_VLAN_PR_S	29
 #define ICE_TX_FLAGS_VLAN_S	16
 
+#define ICE_XDP_PASS		0
+#define ICE_XDP_CONSUMED	BIT(0)
+#define ICE_XDP_TX		BIT(1)
+#define ICE_XDP_REDIR		BIT(2)
+
 #define ICE_RX_DMA_ATTR \
 	(DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)
 
+#define ICE_TXD_CMD (ICE_TX_DESC_CMD_EOP | ICE_TX_DESC_CMD_RS)
+
 struct ice_tx_buf {
 	struct ice_tx_desc *next_to_watch;
-	struct sk_buff *skb;
+	union {
+		struct sk_buff *skb;
+		void *raw_buf; /* used for XDP */
+	};
 	unsigned int bytecount;
 	unsigned short gso_segs;
 	u32 tx_flags;
@@ -185,6 +201,9 @@ struct ice_ring {
 	};
 
 	struct rcu_head rcu;		/* to avoid race on free */
+	struct bpf_prog *xdp_prog;
+	/* CL3 - 3rd cacheline starts here */
+	struct xdp_rxq_info xdp_rxq;
 	/* CLX - the below items are only accessed infrequently and should be
 	 * in their own cache line if possible
 	 */
@@ -197,6 +216,14 @@ struct ice_ring {
 #endif /* CONFIG_DCB */
 } ____cacheline_internodealigned_in_smp;
 
+static inline bool ice_ring_is_xdp(struct ice_ring *ring)
+{
+	if (!ring->tx_buf)
+		return false;
+
+	return !!(ring->tx_buf[0].tx_flags & ICE_TX_FLAGS_RING_XDP);
+}
+
 struct ice_ring_container {
 	/* head of linked-list of rings */
 	struct ice_ring *ring;
drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c +1 -0
index 5d24b539648f..f5eaf3059063 100644
@@ -2102,6 +2102,7 @@ static int ice_vc_cfg_qs_msg(struct ice_vf *vf, u8 *msg)
 			goto error_param;
 		}
 		vsi->rx_buf_len = qpi->rxq.databuffer_size;
+		vsi->rx_rings[i]->rx_buf_len = vsi->rx_buf_len;
 		if (qpi->rxq.max_pkt_size >= (16 * 1024) ||
 		    qpi->rxq.max_pkt_size < 64) {
 			v_ret = VIRTCHNL_STATUS_ERR_PARAM;
Signature · 1 line
2.20.1
author maintainer
#1 lore raw

PATCH v2 2/2 ice: Add support for AF_XDP

From: Krzysztof Kazimierczak <[email protected]>

Add zero copy AF_XDP support. This patch adds zero copy support for Tx and Rx; code for zero copy is added to ice_xsk.h and ice_xsk.c.

For Tx, implement ndo_xsk_async_xmit. As with other drivers, reuse existing XDP Tx queues for this task, since XDP_REDIRECT guarantees mutual exclusion between different NAPI contexts based on CPU ID. In turn, a netdev can XDP_REDIRECT to another netdev with a different NAPI context, since the operation is bound to a specific core and each core has its own hardware ring.

For Rx, allocate frames as MEM_TYPE_ZERO_COPY on queues that AF_XDP is enabled.

---
v2: Fix build warning for 32 bit systems
Remove unneeded casts to u8*

 drivers/net/ethernet/intel/ice/Makefile      |   1 +
 drivers/net/ethernet/intel/ice/ice.h         |  26 +
 drivers/net/ethernet/intel/ice/ice_ethtool.c |   7 +
 drivers/net/ethernet/intel/ice/ice_lib.c     |  65 +-
 drivers/net/ethernet/intel/ice/ice_main.c    |  13 +
 drivers/net/ethernet/intel/ice/ice_txrx.c    |  84 +-
 drivers/net/ethernet/intel/ice/ice_txrx.h    |  35 +-
 drivers/net/ethernet/intel/ice/ice_xsk.c     | 941 +++++++++++++++++++
 drivers/net/ethernet/intel/ice/ice_xsk.h     |  81 ++
 9 files changed, 1223 insertions(+), 30 deletions(-)
 create mode 100644 drivers/net/ethernet/intel/ice/ice_xsk.c
 create mode 100644 drivers/net/ethernet/intel/ice/ice_xsk.h
drivers/net/ethernet/intel/ice/Makefile +1 -0
index 2d140ba83781..54bad11e4736 100644
@@ -18,3 +18,4 @@ ice-y := ice_main.o	\
 	 ice_ethtool.o
 ice-$(CONFIG_PCI_IOV) += ice_virtchnl_pf.o ice_sriov.o
 ice-$(CONFIG_DCB) += ice_dcb.o ice_dcb_lib.o
+ice-$(CONFIG_XDP_SOCKETS) += ice_xsk.o
drivers/net/ethernet/intel/ice/ice.h +26 -0
index 53adb93c6b61..3639cf8c75ac 100644
@@ -34,6 +34,8 @@
 #include <linux/bpf.h>
 #include <linux/avf/virtchnl.h>
 #include <net/ipv6.h>
+#include <net/xdp_sock.h>
+#include "ice_xsk.h"
 #include "ice_devids.h"
 #include "ice_type.h"
 #include "ice_txrx.h"
@@ -308,6 +310,9 @@ struct ice_vsi {
 	struct ice_ring **xdp_rings;	 /* XDP ring array */
 	u16 num_xdp_txq;		 /* Used XDP queues */
 	u8 xdp_mapping_mode;		 /* ICE_MAP_MODE_[CONTIG|SCATTER] */
+	struct xdp_umem **xsk_umems;
+	u16 num_xsk_umems_used;
+	u16 num_xsk_umems;
 } ____cacheline_internodealigned_in_smp;
 
 /* struct that defines an interrupt vector */
@@ -449,6 +454,27 @@ static inline void ice_set_ring_xdp(struct ice_ring *ring)
 	ring->tx_buf[0].tx_flags |= ICE_TX_FLAGS_RING_XDP;
 }
 
+/**
+ * ice_xsk_umem - get XDP UMEM bound to a ring
+ * @ring - ring to use
+ *
+ * Returns a pointer to xdp_umem structure if there is an UMEM present,
+ * NULL otherwise.
+ */
+static inline struct xdp_umem *ice_xsk_umem(struct ice_ring *ring)
+{
+	struct xdp_umem **umems = ring->vsi->xsk_umems;
+	int qid = ring->q_index;
+
+	if (ice_ring_is_xdp(ring))
+		qid -= ring->vsi->num_xdp_txq;
+
+	if (!umems || !umems[qid] || !ice_is_xdp_ena_vsi(ring->vsi))
+		return NULL;
+
+	return umems[qid];
+}
+
 /**
  * ice_find_vsi_by_type - Find and return VSI of a given type
  * @pf: PF to search for VSI
drivers/net/ethernet/intel/ice/ice_ethtool.c +7 -0
index 2d9c184a2333..5c9f7a8bfd16 100644
@@ -2593,6 +2593,13 @@ ice_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring)
 		return 0;
 	}
 
+	/* If there is a AF_XDP UMEM attached to any of Rx rings,
+	 * disallow changing the number of descriptors -- regardless
+	 * if the netdev is running or not.
+	 */
+	if (ice_xsk_any_rx_ring_ena(vsi))
+		return -EBUSY;
+
 	while (test_and_set_bit(__ICE_CFG_BUSY, pf->state)) {
 		timeout--;
 		if (!timeout)
drivers/net/ethernet/intel/ice/ice_lib.c +57 -8

102 lines not on this page · raw message

drivers/net/ethernet/intel/ice/ice_main.c +13 -0
index 5d14627a6ab6..5f45548f2940 100644
@@ -1520,6 +1520,7 @@ static int ice_xdp_alloc_setup_rings(struct ice_vsi *vsi)
 		if (ice_setup_tx_ring(xdp_ring))
 			goto free_xdp_rings;
 		ice_set_ring_xdp(xdp_ring);
+		xdp_ring->xsk_umem = ice_xsk_umem(xdp_ring);
 	}
 
 	return 0;
@@ -1755,6 +1756,14 @@ ice_xdp_setup_prog(struct ice_vsi *vsi, struct bpf_prog *prog,
 	if (if_running)
 		ret = ice_up(vsi);
 
+	if (!ret && prog && vsi->xsk_umems)
+		ice_for_each_rxq(vsi, i) {
+			struct ice_ring *rx_ring = vsi->rx_rings[i];
+
+			if (rx_ring->xsk_umem)
+				napi_schedule(&rx_ring->q_vector->napi);
+		}
+
 skip_setting_prog:
 	return ret;
 }
@@ -1781,6 +1790,9 @@ static int ice_xdp(struct net_device *dev, struct netdev_bpf *xdp)
 	case XDP_QUERY_PROG:
 		xdp->prog_id = vsi->xdp_prog ? vsi->xdp_prog->aux->id : 0;
 		return 0;
+	case XDP_SETUP_XSK_UMEM:
+		return ice_xsk_umem_setup(vsi, xdp->xsk.umem,
+					  xdp->xsk.queue_id);
 	default:
 		NL_SET_ERR_MSG_MOD(xdp->extack, "Unknown XDP command");
 		return -EINVAL;
@@ -4731,4 +4743,5 @@ static const struct net_device_ops ice_netdev_ops = {
 	.ndo_tx_timeout = ice_tx_timeout,
 	.ndo_bpf = ice_xdp,
 	.ndo_xdp_xmit = ice_xdp_xmit,
+	.ndo_xsk_async_xmit = ice_xsk_async_xmit,
 };
drivers/net/ethernet/intel/ice/ice_txrx.c +66 -18

193 lines not on this page · raw message

drivers/net/ethernet/intel/ice/ice_txrx.h +31 -4
index 355834b4abda..8f857e423739 100644
@@ -92,9 +92,17 @@ struct ice_tx_offload_params {
 struct ice_rx_buf {
 	struct sk_buff *skb;
 	dma_addr_t dma;
-	struct page *page;
-	unsigned int page_offset;
-	u16 pagecnt_bias;
+	union {
+		struct {
+			struct page *page;
+			unsigned int page_offset;
+			u16 pagecnt_bias;
+		};
+		struct {
+			void *addr;
+			u64 handle;
+		};
+	};
 };
 
 struct ice_q_stats {
@@ -202,6 +210,8 @@ struct ice_ring {
 
 	struct rcu_head rcu;		/* to avoid race on free */
 	struct bpf_prog *xdp_prog;
+	struct xdp_umem *xsk_umem;
+	struct zero_copy_allocator zca;
 	/* CL3 - 3rd cacheline starts here */
 	struct xdp_rxq_info xdp_rxq;
 	/* CLX - the below items are only accessed infrequently and should be
@@ -244,6 +254,8 @@ struct ice_ring_container {
 #define ice_for_each_ring(pos, head) \
 	for (pos = (head).ring; pos; pos = pos->next)
 
+union ice_32b_rx_flex_desc;
+
 bool ice_alloc_rx_bufs(struct ice_ring *rxr, u16 cleaned_count);
 netdev_tx_t ice_start_xmit(struct sk_buff *skb, struct net_device *netdev);
 void ice_clean_tx_ring(struct ice_ring *tx_ring);
@@ -253,5 +265,20 @@ int ice_setup_rx_ring(struct ice_ring *rx_ring);
 void ice_free_tx_ring(struct ice_ring *tx_ring);
 void ice_free_rx_ring(struct ice_ring *rx_ring);
 int ice_napi_poll(struct napi_struct *napi, int budget);
-
+#ifdef CONFIG_XDP_SOCKETS
+void ice_finalize_xdp_rx(struct ice_ring *rx_ring, unsigned int xdp_res);
+void ice_release_rx_desc(struct ice_ring *rx_ring, u32 val);
+bool
+ice_test_staterr(union ice_32b_rx_flex_desc *rx_desc, const u16 stat_err_bits);
+void
+ice_process_skb_fields(struct ice_ring *rx_ring,
+		       union ice_32b_rx_flex_desc *rx_desc,
+		       struct sk_buff *skb, u8 ptype);
+void
+ice_receive_skb(struct ice_ring *rx_ring, struct sk_buff *skb, u16 vlan_tag);
+bool ice_cleanup_hdrs(struct sk_buff *skb);
+void ice_xdp_ring_update_tail(struct ice_ring *xdp_ring);
+__le64 ice_build_ctob(u64 td_cmd, u64 td_offset, unsigned int size, u64 td_tag);
+int ice_xmit_xdp_ring(void *data, u16 size, struct ice_ring *xdp_ring);
+#endif /* CONFIG_XDP_SOCKETS */
 #endif /* _ICE_TXRX_H_ */
drivers/net/ethernet/intel/ice/ice_xsk.c new file +941 -0

941 lines not on this page · raw message

drivers/net/ethernet/intel/ice/ice_xsk.h new file +81 -0

81 lines not on this page · raw message

Signature · 1 line
2.20.1