ice: Add support for XDP
Subsystems (2)
Tony Nguyen
maintainerPATCH v2 1/2 ice: Add support for XDP
From: Maciej Fijalkowski <[email protected]>
Add support for XDP. Implement ndo_bpf and ndo_xdp_xmit. Upon load of an XDP program, allocate additional Tx rings for dedicated XDP use. The following actions are supported: XDP_TX, XDP_DROP, XDP_REDIRECT, XDP_PASS, and XDP_ABORTED.
Move build_ctob() up so that no forward declaration is needed and rename it to ice_build_ctob() since it's an ice function.
--- drivers/net/ethernet/intel/ice/ice.h | 21 ++ drivers/net/ethernet/intel/ice/ice_ethtool.c | 53 ++- drivers/net/ethernet/intel/ice/ice_lib.c | 84 ++++- drivers/net/ethernet/intel/ice/ice_lib.h | 6 + drivers/net/ethernet/intel/ice/ice_main.c | 319 ++++++++++++++++ drivers/net/ethernet/intel/ice/ice_txrx.c | 343 +++++++++++++++--- drivers/net/ethernet/intel/ice/ice_txrx.h | 29 +- .../net/ethernet/intel/ice/ice_virtchnl_pf.c | 1 + 8 files changed, 799 insertions(+), 57 deletions(-)
drivers/net/ethernet/intel/ice/ice.h +21 -0
@@ -28,7 +28,10 @@#include <linux/ip.h> #include <linux/sctp.h> #include <linux/ipv6.h> +#include <linux/pkt_sched.h> #include <linux/if_bridge.h> +#include <linux/ctype.h> +#include <linux/bpf.h> #include <linux/avf/virtchnl.h> #include <net/ipv6.h> #include "ice_devids.h"
@@ -301,6 +304,10 @@ struct ice_vsi {u16 num_rx_desc;u16 num_tx_desc;struct ice_tc_cfg tc_cfg; +struct bpf_prog *xdp_prog; +struct ice_ring **xdp_rings;/* XDP ring array */ +u16 num_xdp_txq;/* Used XDP queues */ +u8 xdp_mapping_mode;/* ICE_MAP_MODE_[CONTIG|SCATTER] */ }____cacheline_internodealigned_in_smp ;/* struct that defines an interrupt vector */
@@ -432,6 +439,16 @@ ice_irq_dynamic_ena(struct ice_hw *hw, struct ice_vsi *vsi,wr32 (hw,GLINT_DYN_CTL (vector), val); } +static inline bool ice_is_xdp_ena_vsi (struct ice_vsi *vsi) +{ +return !!vsi->xdp_prog; +} + +static inline void ice_set_ring_xdp (struct ice_ring *ring) +{ + ring->tx_buf[0 ].tx_flags |=ICE_TX_FLAGS_RING_XDP ; +} +/** * ice_find_vsi_by_type - Find and return VSI of a given type * @pf: PF to search for VSI
@@ -459,6 +476,10 @@ int ice_up(struct ice_vsi *vsi);int ice_down (struct ice_vsi *vsi);int ice_vsi_cfg (struct ice_vsi *vsi);struct ice_vsi *ice_lb_vsi_setup (struct ice_pf *pf,struct ice_port_info *pi); +int ice_prepare_xdp_rings (struct ice_vsi *vsi); +int ice_destroy_xdp_rings (struct ice_vsi *vsi); +int ice_xdp_xmit (struct net_device *dev,int n,struct xdp_frame **frames, +u32 flags);int ice_set_rss (struct ice_vsi *vsi,u8 *seed,u8 *lut,u16 lut_size);int ice_get_rss (struct ice_vsi *vsi,u8 *seed,u8 *lut,u16 lut_size);void ice_fill_rss_lut (u8 *lut,u16 rss_table_size,u16 rss_size);
drivers/net/ethernet/intel/ice/ice_ethtool.c +50 -3
83 lines not on this page · raw message
drivers/net/ethernet/intel/ice/ice_lib.c +75 -9
173 lines not on this page · raw message
drivers/net/ethernet/intel/ice/ice_lib.h +6 -0
@@ -43,6 +43,10 @@ int ice_vsi_stop_lan_tx_rings(struct ice_vsi *vsi, enum ice_disq_rst_src rst_src, u16 rel_vmvf_num); +int ice_vsi_cfg_xdp_txqs (struct ice_vsi *vsi); + +int ice_vsi_stop_xdp_tx_rings (struct ice_vsi *vsi); +int ice_cfg_vlan_pruning (struct ice_vsi *vsi,bool ena,bool vlan_promisc);void ice_cfg_sw_lldp (struct ice_vsi *vsi,bool tx,bool create);
@@ -76,6 +80,8 @@ bool ice_is_reset_in_progress(unsigned long *state);void ice_vsi_free_q_vectors (struct ice_vsi *vsi); +int __ice_vsi_get_qs (struct ice_qs_cfg *qs_cfg); +void ice_trigger_sw_intr (struct ice_hw *hw,struct ice_q_vector *q_vector);void ice_vsi_put_qs (struct ice_vsi *vsi);
drivers/net/ethernet/intel/ice/ice_main.c +319 -0
347 lines not on this page · raw message
drivers/net/ethernet/intel/ice/ice_txrx.c +299 -44
496 lines not on this page · raw message
drivers/net/ethernet/intel/ice/ice_txrx.h +28 -1
@@ -44,17 +44,33 @@#define ICE_TX_FLAGS_TSO BIT(0)#define ICE_TX_FLAGS_HW_VLAN BIT(1)#define ICE_TX_FLAGS_SW_VLAN BIT(2) +/* ICE_TX_FLAGS_RING_XDP is used to indicate that whole ring is dedicated for +* XDP purposes; at this point struct ice_ring doesn't have an appropriate +* field that could be used for setting this flag, so let's use the tx_flags +* field of the first ice_tx_buf from ice_ring +*/ +#define ICE_TX_FLAGS_RING_XDP BIT(8)#define ICE_TX_FLAGS_VLAN_M 0xffff0000#define ICE_TX_FLAGS_VLAN_PR_M 0xe0000000#define ICE_TX_FLAGS_VLAN_PR_S 29#define ICE_TX_FLAGS_VLAN_S 16 +#define ICE_XDP_PASS 0 +#define ICE_XDP_CONSUMED BIT(0) +#define ICE_XDP_TX BIT(1) +#define ICE_XDP_REDIR BIT(2) +#define ICE_RX_DMA_ATTR \ (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING) +#define ICE_TXD_CMD (ICE_TX_DESC_CMD_EOP | ICE_TX_DESC_CMD_RS) +struct ice_tx_buf {struct ice_tx_desc *next_to_watch; -struct sk_buff *skb; +union { +struct sk_buff *skb; +void *raw_buf;/* used for XDP */ + };unsigned int bytecount;unsigned short gso_segs;u32 tx_flags;
@@ -185,6 +201,9 @@ struct ice_ring { };struct rcu_head rcu;/* to avoid race on free */ +struct bpf_prog *xdp_prog; +/* CL3 - 3rd cacheline starts here */ +struct xdp_rxq_info xdp_rxq;/* CLX - the below items are only accessed infrequently and should be * in their own cache line if possible */
@@ -197,6 +216,14 @@ struct ice_ring {#endif /* CONFIG_DCB */ }____cacheline_internodealigned_in_smp ; +static inline bool ice_ring_is_xdp (struct ice_ring *ring) +{ +if (!ring->tx_buf) +return false; + +return !!(ring->tx_buf[0 ].tx_flags &ICE_TX_FLAGS_RING_XDP ); +} +struct ice_ring_container {/* head of linked-list of rings */ struct ice_ring *ring;
drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c +1 -0
@@ -2102,6 +2102,7 @@ static int ice_vc_cfg_qs_msg(struct ice_vf *vf, u8 *msg) goto error_param; } vsi->rx_buf_len = qpi->rxq.databuffer_size; + vsi->rx_rings[i]->rx_buf_len = vsi->rx_buf_len;if (qpi->rxq.max_pkt_size >= (16 *1024 ) || qpi->rxq.max_pkt_size <64 ) { v_ret =VIRTCHNL_STATUS_ERR_PARAM ;
Signature · 1 line
2.20.1
Tony Nguyen
maintainerPATCH v2 2/2 ice: Add support for AF_XDP
From: Krzysztof Kazimierczak <[email protected]>
Add zero copy AF_XDP support. This patch adds zero copy support for Tx and Rx; code for zero copy is added to ice_xsk.h and ice_xsk.c.
For Tx, implement ndo_xsk_async_xmit. As with other drivers, reuse existing XDP Tx queues for this task, since XDP_REDIRECT guarantees mutual exclusion between different NAPI contexts based on CPU ID. In turn, a netdev can XDP_REDIRECT to another netdev with a different NAPI context, since the operation is bound to a specific core and each core has its own hardware ring.
For Rx, allocate frames as MEM_TYPE_ZERO_COPY on queues that AF_XDP is enabled.
---
v2: Fix build warning for 32 bit systems
Remove unneeded casts to u8*
drivers/net/ethernet/intel/ice/Makefile | 1 + drivers/net/ethernet/intel/ice/ice.h | 26 + drivers/net/ethernet/intel/ice/ice_ethtool.c | 7 + drivers/net/ethernet/intel/ice/ice_lib.c | 65 +- drivers/net/ethernet/intel/ice/ice_main.c | 13 + drivers/net/ethernet/intel/ice/ice_txrx.c | 84 +- drivers/net/ethernet/intel/ice/ice_txrx.h | 35 +- drivers/net/ethernet/intel/ice/ice_xsk.c | 941 +++++++++++++++++++ drivers/net/ethernet/intel/ice/ice_xsk.h | 81 ++ 9 files changed, 1223 insertions(+), 30 deletions(-) create mode 100644 drivers/net/ethernet/intel/ice/ice_xsk.c create mode 100644 drivers/net/ethernet/intel/ice/ice_xsk.h
drivers/net/ethernet/intel/ice/Makefile +1 -0
@@ -18,3 +18,4 @@ ice-y := ice_main.o \ ice_ethtool.o ice-$(CONFIG_PCI_IOV) += ice_virtchnl_pf.o ice_sriov.o ice-$(CONFIG_DCB) += ice_dcb.o ice_dcb_lib.o +ice-$(CONFIG_XDP_SOCKETS) += ice_xsk.o
drivers/net/ethernet/intel/ice/ice.h +26 -0
@@ -34,6 +34,8 @@#include <linux/bpf.h> #include <linux/avf/virtchnl.h> #include <net/ipv6.h> +#include <net/xdp_sock.h> +#include "ice_xsk.h" #include "ice_devids.h" #include "ice_type.h" #include "ice_txrx.h"
@@ -308,6 +310,9 @@ struct ice_vsi {struct ice_ring **xdp_rings;/* XDP ring array */ u16 num_xdp_txq;/* Used XDP queues */ u8 xdp_mapping_mode;/* ICE_MAP_MODE_[CONTIG|SCATTER] */ +struct xdp_umem **xsk_umems; +u16 num_xsk_umems_used; +u16 num_xsk_umems; }____cacheline_internodealigned_in_smp ;/* struct that defines an interrupt vector */
@@ -449,6 +454,27 @@ static inline void ice_set_ring_xdp(struct ice_ring *ring) ring->tx_buf[0 ].tx_flags |=ICE_TX_FLAGS_RING_XDP ; } +/** +* ice_xsk_umem - get XDP UMEM bound to a ring +* @ring - ring to use +* +* Returns a pointer to xdp_umem structure if there is an UMEM present, +* NULL otherwise. +*/ +static inline struct xdp_umem *ice_xsk_umem (struct ice_ring *ring) +{ +struct xdp_umem **umems = ring->vsi->xsk_umems; +int qid = ring->q_index; + +if (ice_ring_is_xdp (ring)) + qid -= ring->vsi->num_xdp_txq; + +if (!umems || !umems[qid] || !ice_is_xdp_ena_vsi (ring->vsi)) +return NULL ; + +return umems[qid]; +} +/** * ice_find_vsi_by_type - Find and return VSI of a given type * @pf: PF to search for VSI
drivers/net/ethernet/intel/ice/ice_ethtool.c +7 -0
@@ -2593,6 +2593,13 @@ ice_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring)return 0 ; } +/* If there is a AF_XDP UMEM attached to any of Rx rings, +* disallow changing the number of descriptors -- regardless +* if the netdev is running or not. +*/ +if (ice_xsk_any_rx_ring_ena (vsi)) +return -EBUSY ; +while (test_and_set_bit (__ICE_CFG_BUSY, pf->state)) { timeout--;if (!timeout)
drivers/net/ethernet/intel/ice/ice_lib.c +57 -8
102 lines not on this page · raw message
drivers/net/ethernet/intel/ice/ice_main.c +13 -0
@@ -1520,6 +1520,7 @@ static int ice_xdp_alloc_setup_rings(struct ice_vsi *vsi)if (ice_setup_tx_ring (xdp_ring)) goto free_xdp_rings;ice_set_ring_xdp (xdp_ring); + xdp_ring->xsk_umem =ice_xsk_umem (xdp_ring); }return 0 ;
@@ -1755,6 +1756,14 @@ ice_xdp_setup_prog(struct ice_vsi *vsi, struct bpf_prog *prog,if (if_running) ret =ice_up (vsi); +if (!ret && prog && vsi->xsk_umems) +ice_for_each_rxq (vsi, i) { +struct ice_ring *rx_ring = vsi->rx_rings[i]; + +if (rx_ring->xsk_umem) +napi_schedule (&rx_ring->q_vector->napi); + } + skip_setting_prog:return ret; }
@@ -1781,6 +1790,9 @@ static int ice_xdp(struct net_device *dev, struct netdev_bpf *xdp)case XDP_QUERY_PROG : xdp->prog_id = vsi->xdp_prog ? vsi->xdp_prog->aux->id :0 ;return 0 ; +case XDP_SETUP_XSK_UMEM : +return ice_xsk_umem_setup (vsi, xdp->xsk.umem, + xdp->xsk.queue_id);default :NL_SET_ERR_MSG_MOD (xdp->extack,"Unknown XDP command" );return -EINVAL ;
@@ -4731,4 +4743,5 @@ static const struct net_device_ops ice_netdev_ops = { .ndo_tx_timeout = ice_tx_timeout, .ndo_bpf = ice_xdp, .ndo_xdp_xmit = ice_xdp_xmit, + .ndo_xsk_async_xmit = ice_xsk_async_xmit, };
drivers/net/ethernet/intel/ice/ice_txrx.c +66 -18
193 lines not on this page · raw message
drivers/net/ethernet/intel/ice/ice_txrx.h +31 -4
@@ -92,9 +92,17 @@ struct ice_tx_offload_params {struct ice_rx_buf {struct sk_buff *skb;dma_addr_t dma; -struct page *page; -unsigned int page_offset; -u16 pagecnt_bias; +union { +struct { +struct page *page; +unsigned int page_offset; +u16 pagecnt_bias; + }; +struct { +void *addr; +u64 handle; + }; + }; };struct ice_q_stats {
@@ -202,6 +210,8 @@ struct ice_ring {struct rcu_head rcu;/* to avoid race on free */ struct bpf_prog *xdp_prog; +struct xdp_umem *xsk_umem; +struct zero_copy_allocator zca;/* CL3 - 3rd cacheline starts here */ struct xdp_rxq_info xdp_rxq;/* CLX - the below items are only accessed infrequently and should be
@@ -244,6 +254,8 @@ struct ice_ring_container {#define ice_for_each_ring (pos, head) \ for (pos = (head).ring; pos; pos = pos->next) +union ice_32b_rx_flex_desc ; +bool ice_alloc_rx_bufs (struct ice_ring *rxr,u16 cleaned_count);netdev_tx_t ice_start_xmit (struct sk_buff *skb,struct net_device *netdev);void ice_clean_tx_ring (struct ice_ring *tx_ring);
@@ -253,5 +265,20 @@ int ice_setup_rx_ring(struct ice_ring *rx_ring);void ice_free_tx_ring (struct ice_ring *tx_ring);void ice_free_rx_ring (struct ice_ring *rx_ring);int ice_napi_poll (struct napi_struct *napi,int budget); - +#ifdef CONFIG_XDP_SOCKETS +void ice_finalize_xdp_rx (struct ice_ring *rx_ring,unsigned int xdp_res); +void ice_release_rx_desc (struct ice_ring *rx_ring,u32 val); +bool +ice_test_staterr (union ice_32b_rx_flex_desc *rx_desc,const u16 stat_err_bits); +void +ice_process_skb_fields (struct ice_ring *rx_ring, +union ice_32b_rx_flex_desc *rx_desc, +struct sk_buff *skb,u8 ptype); +void +ice_receive_skb (struct ice_ring *rx_ring,struct sk_buff *skb,u16 vlan_tag); +bool ice_cleanup_hdrs (struct sk_buff *skb); +void ice_xdp_ring_update_tail (struct ice_ring *xdp_ring); +__le64 ice_build_ctob (u64 td_cmd,u64 td_offset,unsigned int size,u64 td_tag); +int ice_xmit_xdp_ring (void *data,u16 size,struct ice_ring *xdp_ring); +#endif /* CONFIG_XDP_SOCKETS */ #endif /* _ICE_TXRX_H_ */
drivers/net/ethernet/intel/ice/ice_xsk.c new file +941 -0
941 lines not on this page · raw message
drivers/net/ethernet/intel/ice/ice_xsk.h new file +81 -0
81 lines not on this page · raw message
Signature · 1 line
2.20.1