dpaa_eth: add XDP support
Subsystems (3)
v3 cover dpaa_eth: add XDP support
Enable XDP support for the QorIQ DPAA1 platforms.
Implement all the current actions (DROP, ABORTED, PASS, TX, REDIRECT). No Tx batching is added at this time.
Additional XDP_PACKET_HEADROOM bytes are reserved in each frame's headroom.
After transmit, a reference to the xdp_frame is saved in the buffer for clean-up on confirmation in a newly created structure for software annotations.
Changes in v2:
- warn only once if extracting the timestamp from a received frame fails in 2/7
Changes in v3:
- drop received S/G frames when XDP is enabled in 2/7
Camelia Groza (7):
dpaa_eth: add struct for software backpointers
dpaa_eth: add basic XDP support
dpaa_eth: limit the possible MTU range when XDP is enabled
dpaa_eth: add XDP_TX support
dpaa_eth: add XDP_REDIRECT support
dpaa_eth: rename current skb A050385 erratum workaround
dpaa_eth: implement the A050385 erratum workaround for XDP
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c | 457 +++++++++++++++++++++++-- drivers/net/ethernet/freescale/dpaa/dpaa_eth.h | 13 + 2 files changed, 440 insertions(+), 30 deletions(-)
--
1.9.1
PATCH v3 1/7 dpaa_eth: add struct for software backpointers
We maintain an skb backpointer in the software annotations area of Tx frames. Introduce a structure for explicit handling.
--- drivers/net/ethernet/freescale/dpaa/dpaa_eth.c | 16 +++++++++------- drivers/net/ethernet/freescale/dpaa/dpaa_eth.h | 8 ++++++++ 2 files changed, 17 insertions(+), 7 deletions(-)
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +9 -7
@@ -1633,6 +1633,7 @@ static struct sk_buff *dpaa_cleanup_tx_fd(const struct dpaa_priv *priv,dma_addr_t addr =qm_fd_addr (fd);void *vaddr =phys_to_virt (addr);const struct qm_sg_entry *sgt; +struct dpaa_eth_swbp *swbp;struct sk_buff *skb;u64 ns;int i;
@@ -1665,7 +1666,8 @@ static struct sk_buff *dpaa_cleanup_tx_fd(const struct dpaa_priv *priv, dma_dir); } - skb = *(struct sk_buff **)vaddr; + swbp = (struct dpaa_eth_swbp *)vaddr; + skb = swbp->skb;/* DMA unmapping is required before accessing the HW provided info */ if (ts && priv->tx_tstamp &&
@@ -1879,8 +1881,8 @@ static int skb_to_contig_fd(struct dpaa_priv *priv, {struct net_device *net_dev = priv->net_dev;enum dma_data_direction dma_dir; +struct dpaa_eth_swbp *swbp;unsigned char *buff_start; -struct sk_buff **skbh;dma_addr_t addr;int err;
@@ -1891,8 +1893,8 @@ static int skb_to_contig_fd(struct dpaa_priv *priv, buff_start = skb->data - priv->tx_headroom; dma_dir =DMA_TO_DEVICE ; - skbh = (struct sk_buff **)buff_start; - *skbh = skb; + swbp = (struct dpaa_eth_swbp *)buff_start; + swbp->skb = skb;/* Enable L3/L4 hardware checksum computation. *
@@ -1931,8 +1933,8 @@ static int skb_to_sg_fd(struct dpaa_priv *priv,const enum dma_data_direction dma_dir =DMA_TO_DEVICE ;const int nr_frags =skb_shinfo (skb)->nr_frags;struct net_device *net_dev = priv->net_dev; +struct dpaa_eth_swbp *swbp;struct qm_sg_entry *sgt; -struct sk_buff **skbh;void *buff_start;skb_frag_t *frag;dma_addr_t addr;
@@ -2005,8 +2007,8 @@ static int skb_to_sg_fd(struct dpaa_priv *priv,qm_fd_set_sg (fd, priv->tx_headroom, skb->len);/* DMA map the SGT page */ - skbh = (struct sk_buff **)buff_start; - *skbh = skb; + swbp = (struct dpaa_eth_swbp *)buff_start; + swbp->skb = skb; addr =dma_map_page (priv->tx_dma_dev, p,0 , priv->tx_headroom +DPAA_SGT_SIZE , dma_dir);
drivers/net/ethernet/freescale/dpaa/dpaa_eth.h +8 -0
@@ -144,6 +144,14 @@ struct dpaa_buffer_layout {u16 priv_data_size; }; +/* Information to be used on the Tx confirmation path. Stored just +* before the start of the transmit buffer. Maximum size allowed +* is DPAA_TX_PRIV_DATA_SIZE bytes. +*/ +struct dpaa_eth_swbp { +struct sk_buff *skb; +}; +struct dpaa_priv {struct dpaa_percpu_priv __percpu *percpu_priv;struct dpaa_bp *dpaa_bp;
Signature · 1 line
1.9.1
PATCH v3 2/7 dpaa_eth: add basic XDP support
In reply to Camelia GrozaImplement the XDP_DROP and XDP_PASS actions.
Avoid draining and reconfiguring the buffer pool at each XDP setup/teardown by increasing the frame headroom and reserving XDP_PACKET_HEADROOM bytes from the start. Since we always reserve an entire page per buffer, this change only impacts Jumbo frame scenarios where the maximum linear frame size is reduced by 256 bytes. Multi buffer Scatter/Gather frames are now used instead in these scenarios.
Allow XDP programs to access the entire buffer.
The data in the received frame's headroom can be overwritten by the XDP program. Extract the relevant fields from the headroom while they are still available, before running the XDP program.
Since the headroom might be resized before the frame is passed up to the stack, remove the check for a fixed headroom value when building an skb.
Allow the meta data to be updated and pass the information up the stack.
Scatter/Gather frames are dropped when XDP is enabled.
---
Changes in v2:
- warn only once if extracting the timestamp from a received frame fails
Changes in v3:
- drop received S/G frames when XDP is enabled
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c | 158 ++++++++++++++++++++++--- drivers/net/ethernet/freescale/dpaa/dpaa_eth.h | 2 + 2 files changed, 144 insertions(+), 16 deletions(-)
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +142 -16
@@ -53,6 +53,8 @@#include <linux/dma-mapping.h> #include <linux/sort.h> #include <linux/phy_fixed.h> +#include <linux/bpf.h> +#include <linux/bpf_trace.h> #include <soc/fsl/bman.h> #include <soc/fsl/qman.h> #include "fman.h"
@@ -177,7 +179,7 @@#define DPAA_HWA_SIZE (DPAA_PARSE_RESULTS_SIZE + DPAA_TIME_STAMP_SIZE \ + DPAA_HASH_RESULTS_SIZE)#define DPAA_RX_PRIV_DATA_DEFAULT_SIZE (DPAA_TX_PRIV_DATA_SIZE + \ - dpaa_rx_extra_headroom) + XDP_PACKET_HEADROOM - DPAA_HWA_SIZE)#ifdef CONFIG_DPAA_ERRATUM_A050385 #define DPAA_RX_PRIV_DATA_A050385_SIZE (DPAA_A050385_ALIGN - DPAA_HWA_SIZE)#define DPAA_RX_PRIV_DATA_SIZE (fman_has_errata_a050385() ? \
@@ -1733,7 +1735,6 @@ static struct sk_buff *contig_fd_to_skb(const struct dpaa_priv *priv, SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));if (WARN_ONCE (!skb,"Build skb failure on Rx\n" )) goto free_buffer; -WARN_ON (fd_off != priv->rx_headroom);skb_reserve (skb, fd_off);skb_put (skb,qm_fd_get_length (fd));
@@ -2349,12 +2350,62 @@ static enum qman_cb_dqrr_result rx_error_dqrr(struct qman_portal *portal,return qman_cb_dqrr_consume; } +static u32 dpaa_run_xdp (struct dpaa_priv *priv,struct qm_fd *fd,void *vaddr, +unsigned int *xdp_meta_len) +{ +ssize_t fd_off =qm_fd_get_offset (fd); +struct bpf_prog *xdp_prog; +struct xdp_buff xdp; +u32 xdp_act; + +rcu_read_lock (); + + xdp_prog =READ_ONCE (priv->xdp_prog); +if (!xdp_prog) { +rcu_read_unlock (); +return XDP_PASS ; + } + + xdp.data = vaddr + fd_off; + xdp.data_meta = xdp.data; + xdp.data_hard_start = xdp.data -XDP_PACKET_HEADROOM ; + xdp.data_end = xdp.data +qm_fd_get_length (fd); + xdp.frame_sz =DPAA_BP_RAW_SIZE ; + + xdp_act =bpf_prog_run_xdp (xdp_prog, &xdp); + +/* Update the length and the offset of the FD */ +qm_fd_set_contig (fd, xdp.data - vaddr, xdp.data_end - xdp.data); + +switch (xdp_act) { +case XDP_PASS : + *xdp_meta_len = xdp.data - xdp.data_meta; +break ; +default : +bpf_warn_invalid_xdp_action (xdp_act); + fallthrough; +case XDP_ABORTED : +trace_xdp_exception (priv->net_dev, xdp_prog, xdp_act); + fallthrough; +case XDP_DROP : +/* Free the buffer */ +free_pages ((unsigned long )vaddr,0 ); +break ; + } + +rcu_read_unlock (); + +return xdp_act; +} +static enum qman_cb_dqrr_result rx_default_dqrr (struct qman_portal *portal,struct qman_fq *fq,const struct qm_dqrr_entry *dq,bool sched_napi) { +bool ts_valid = false, hash_valid = false;struct skb_shared_hwtstamps *shhwtstamps; +unsigned int skb_len, xdp_meta_len =0 ;struct rtnl_link_stats64 *percpu_stats;struct dpaa_percpu_priv *percpu_priv;const struct qm_fd *fd = &dq->fd;
@@ -2362,12 +2413,14 @@ static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,enum qm_fd_format fd_format;struct net_device *net_dev;u32 fd_status, hash_offset; +struct qm_sg_entry *sgt;struct dpaa_bp *dpaa_bp;struct dpaa_priv *priv; -unsigned int skb_len;struct sk_buff *skb;int *count_ptr; +u32 xdp_act;void *vaddr; +u32 hash;u64 ns; fd_status =be32_to_cpu (fd->status);
@@ -2423,35 +2476,67 @@ static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal, count_ptr =this_cpu_ptr (dpaa_bp->percpu_count); (*count_ptr)--; -if (likely (fd_format == qm_fd_contig)) +/* Extract the timestamp stored in the headroom before running XDP */ +if (priv->rx_tstamp) { +if (!fman_port_get_tstamp (priv->mac_dev->port[RX ], vaddr, &ns)) + ts_valid = true; +else +WARN_ONCE (1 ,"fman_port_get_tstamp failed!\n" ); + } + +/* Extract the hash stored in the headroom before running XDP */ +if (net_dev->features &NETIF_F_RXHASH && priv->keygen_in_use && + !fman_port_get_hash_result_offset (priv->mac_dev->port[RX ], + &hash_offset)) { + hash =be32_to_cpu (*(u32 *)(vaddr + hash_offset)); + hash_valid = true; + } + +if (likely (fd_format == qm_fd_contig)) { + xdp_act =dpaa_run_xdp (priv, (struct qm_fd *)fd, vaddr, + &xdp_meta_len); +if (xdp_act !=XDP_PASS ) { + percpu_stats->rx_packets++; + percpu_stats->rx_bytes +=qm_fd_get_length (fd); +return qman_cb_dqrr_consume; + } skb =contig_fd_to_skb (priv, fd); -else + }else { +/* XDP doesn't support S/G frames. Return the fragments to the +* buffer pool and release the SGT. +*/ +if (READ_ONCE (priv->xdp_prog)) { +WARN_ONCE (1 ,"S/G frames not supported under XDP\n" ); + sgt = vaddr +qm_fd_get_offset (fd); +dpaa_release_sgt_members (sgt); +free_pages ((unsigned long )vaddr,0 ); +return qman_cb_dqrr_consume; + } skb =sg_fd_to_skb (priv, fd); + }if (!skb)return qman_cb_dqrr_consume; -if (priv->rx_tstamp) { +if (xdp_meta_len) +skb_metadata_set (skb, xdp_meta_len); + +/* Set the previously extracted timestamp */ +if (ts_valid) { shhwtstamps =skb_hwtstamps (skb);memset (shhwtstamps,0 ,sizeof (*shhwtstamps)); - -if (!fman_port_get_tstamp (priv->mac_dev->port[RX ], vaddr, &ns)) - shhwtstamps->hwtstamp =ns_to_ktime (ns); -else -dev_warn (net_dev->dev.parent,"fman_port_get_tstamp failed!\n" ); + shhwtstamps->hwtstamp =ns_to_ktime (ns); } skb->protocol =eth_type_trans (skb, net_dev); -if (net_dev->features &NETIF_F_RXHASH && priv->keygen_in_use && - !fman_port_get_hash_result_offset (priv->mac_dev->port[RX ], - &hash_offset)) { +/* Set the previously extracted hash */ +if (hash_valid) {enum pkt_hash_types type;/* if L4 exists, it was used in the hash generation */ type =be32_to_cpu (fd->status) &FM_FD_STAT_L4CV ?PKT_HASH_TYPE_L4 :PKT_HASH_TYPE_L3 ; -skb_set_hash (skb,be32_to_cpu (*(u32 *)(vaddr + hash_offset)), - type); +skb_set_hash (skb, hash, type); } skb_len = skb->len;
@@ -2671,6 +2756,46 @@ static int dpaa_eth_stop(struct net_device *net_dev)return err; } +static int dpaa_setup_xdp (struct net_device *net_dev,struct bpf_prog *prog) +{ +struct dpaa_priv *priv =netdev_priv (net_dev); +struct bpf_prog *old_prog; +bool up; +int err; + +/* S/G fragments are not supported in XDP-mode */ +if (prog && (priv->dpaa_bp->raw_size < + net_dev->mtu +VLAN_ETH_HLEN +ETH_FCS_LEN )) +return -EINVAL ; + + up =netif_running (net_dev); + +if (up) +dpaa_eth_stop (net_dev); + + old_prog =xchg (&priv->xdp_prog, prog); +if (old_prog) +bpf_prog_put (old_prog); + +if (up) { + err =dpaa_open (net_dev); +if (err) +return err; + } + +return 0 ; +} + +static int dpaa_xdp (struct net_device *net_dev,struct netdev_bpf *xdp) +{ +switch (xdp->command) { +case XDP_SETUP_PROG : +return dpaa_setup_xdp (net_dev, xdp->prog); +default : +return -EINVAL ; + } +} +static int dpaa_ts_ioctl (struct net_device *dev,struct ifreq *rq,int cmd) {struct dpaa_priv *priv =netdev_priv (dev);
@@ -2737,6 +2862,7 @@ static int dpaa_ioctl(struct net_device *net_dev, struct ifreq *rq, int cmd) .ndo_set_rx_mode = dpaa_set_rx_mode, .ndo_do_ioctl = dpaa_ioctl, .ndo_setup_tc = dpaa_setup_tc, + .ndo_bpf = dpaa_xdp, };static int dpaa_napi_add (struct net_device *net_dev)
drivers/net/ethernet/freescale/dpaa/dpaa_eth.h +2 -0
@@ -196,6 +196,8 @@ struct dpaa_priv {bool tx_tstamp;/* Tx timestamping enabled */ bool rx_tstamp;/* Rx timestamping enabled */ + +struct bpf_prog *xdp_prog; };/* from dpaa_ethtool.c */
--
1.9.1
Re: [PATCH net-next v3 2/7] dpaa_eth: add basic XDP support
Quoted 76 lines · Camelia Groza
Implement the
XDP_DROPandXDP_PASSactions.Avoid draining and reconfiguring the buffer pool at each XDP setup/teardown by increasing the frame headroom and reserving
XDP_PACKET_HEADROOMbytes from the start. Since we always reserve an entire page per buffer, this change only impacts Jumbo frame scenarios where the maximum linear frame size is reduced by 256 bytes. Multi buffer Scatter/Gather frames are now used instead in these scenarios.Allow XDP programs to access the entire buffer.
The data in the received frame's headroom can be overwritten by the XDP program. Extract the relevant fields from the headroom while they are still available, before running the XDP program.
Since the headroom might be resized before the frame is passed up to the stack, remove the check for a fixed headroom value when building an skb.
Allow the meta data to be updated and pass the information up the stack.
Scatter/Gather frames are dropped when XDP is enabled.
Acked-by Madalin Bucur <[email protected]>Signed-off-by Camelia Groza <[email protected]>---
Changes in v2:
- warn only once if extracting the timestamp from a received frame failsChanges in v3:
- drop received S/G frames when XDP is enableddrivers/net/ethernet/freescale/dpaa/dpaa_eth.c | 158 ++++++++++++++++++++++--- drivers/net/ethernet/freescale/dpaa/dpaa_eth.h | 2 + 2 files changed, 144 insertions(+), 16 deletions(-)drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +10
-2@@ -53,6 +53,8 @@#include <linux/dma-mapping.h> #include <linux/sort.h> #include <linux/phy_fixed.h> +#include <linux/bpf.h> +#include <linux/bpf_trace.h> #include <soc/fsl/bman.h> #include <soc/fsl/qman.h> #include "fman.h" @@ -177,7 +179,7 @@#define DPAA_HWA_SIZE (DPAA_PARSE_RESULTS_SIZE + DPAA_TIME_STAMP_SIZE \ + DPAA_HASH_RESULTS_SIZE)#define DPAA_RX_PRIV_DATA_DEFAULT_SIZE (DPAA_TX_PRIV_DATA_SIZE + \ - dpaa_rx_extra_headroom) + XDP_PACKET_HEADROOM - DPAA_HWA_SIZE)#ifdef CONFIG_DPAA_ERRATUM_A050385 #define DPAA_RX_PRIV_DATA_A050385_SIZE (DPAA_A050385_ALIGN - DPAA_HWA_SIZE)#define DPAA_RX_PRIV_DATA_SIZE (fman_has_errata_a050385() ? \@@ -1733,7 +1735,6 @@ static struct sk_buff *contig_fd_to_skb(const struct dpaa_priv *priv, SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));if (WARN_ONCE (!skb,"Build skb failure on Rx\n" )) goto free_buffer; -WARN_ON (fd_off != priv->rx_headroom);skb_reserve (skb, fd_off);skb_put (skb,qm_fd_get_length (fd));@@ -2349,12 +2350,62 @@ static enum qman_cb_dqrr_result rx_error_dqrr(struct qman_portal *portal,return qman_cb_dqrr_consume; } +static u32 dpaa_run_xdp (struct dpaa_priv *priv,struct qm_fd *fd,void *vaddr, +unsigned int *xdp_meta_len) +{ +ssize_t fd_off =qm_fd_get_offset (fd); +struct bpf_prog *xdp_prog; +struct xdp_buff xdp; +u32 xdp_act;
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_error_dqrr(struct qman_portal *portal, · line 2369
Quoted 10 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +10
-0+ +rcu_read_lock (); + + xdp_prog =READ_ONCE (priv->xdp_prog); +if (!xdp_prog) { +rcu_read_unlock (); +return XDP_PASS ; + } + + xdp.data = vaddr + fd_off;
I feel like a little drawing of xdp_buff layout would help me with understanding what is going on over here :)
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_error_dqrr(struct qman_portal *portal, · line 2373
Quoted 4 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +4
-0+ xdp.data_meta = xdp.data; + xdp.data_hard_start = xdp.data -XDP_PACKET_HEADROOM ; + xdp.data_end = xdp.data +qm_fd_get_length (fd); + xdp.frame_sz =DPAA_BP_RAW_SIZE ;
Maybe you could fill xdp_buff outside of this function so that later on you could set xdp.rxq once per napi?
Quoted 94 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +65
-3+ + xdp_act =bpf_prog_run_xdp (xdp_prog, &xdp); + +/* Update the length and the offset of the FD */ +qm_fd_set_contig (fd, xdp.data - vaddr, xdp.data_end - xdp.data); + +switch (xdp_act) { +case XDP_PASS : + *xdp_meta_len = xdp.data - xdp.data_meta; +break ; +default : +bpf_warn_invalid_xdp_action (xdp_act); + fallthrough; +case XDP_ABORTED : +trace_xdp_exception (priv->net_dev, xdp_prog, xdp_act); + fallthrough; +case XDP_DROP : +/* Free the buffer */ +free_pages ((unsigned long )vaddr,0 ); +break ; + } + +rcu_read_unlock (); + +return xdp_act; +} +static enum qman_cb_dqrr_result rx_default_dqrr (struct qman_portal *portal,struct qman_fq *fq,const struct qm_dqrr_entry *dq,bool sched_napi) { +bool ts_valid = false, hash_valid = false;struct skb_shared_hwtstamps *shhwtstamps; +unsigned int skb_len, xdp_meta_len =0 ;struct rtnl_link_stats64 *percpu_stats;struct dpaa_percpu_priv *percpu_priv;const struct qm_fd *fd = &dq->fd;@@ -2362,12 +2413,14 @@ static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,enum qm_fd_format fd_format;struct net_device *net_dev;u32 fd_status, hash_offset; +struct qm_sg_entry *sgt;struct dpaa_bp *dpaa_bp;struct dpaa_priv *priv; -unsigned int skb_len;struct sk_buff *skb;int *count_ptr; +u32 xdp_act;void *vaddr; +u32 hash;u64 ns; fd_status =be32_to_cpu (fd->status);@@ -2423,35 +2476,67 @@ static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal, count_ptr =this_cpu_ptr (dpaa_bp->percpu_count); (*count_ptr)--; -if (likely (fd_format == qm_fd_contig)) +/* Extract the timestamp stored in the headroom before running XDP */ +if (priv->rx_tstamp) { +if (!fman_port_get_tstamp (priv->mac_dev->port[RX ], vaddr, &ns)) + ts_valid = true; +else +WARN_ONCE (1 ,"fman_port_get_tstamp failed!\n" ); + } + +/* Extract the hash stored in the headroom before running XDP */ +if (net_dev->features &NETIF_F_RXHASH && priv->keygen_in_use && + !fman_port_get_hash_result_offset (priv->mac_dev->port[RX ], + &hash_offset)) { + hash =be32_to_cpu (*(u32 *)(vaddr + hash_offset)); + hash_valid = true; + } + +if (likely (fd_format == qm_fd_contig)) { + xdp_act =dpaa_run_xdp (priv, (struct qm_fd *)fd, vaddr, + &xdp_meta_len); +if (xdp_act !=XDP_PASS ) { + percpu_stats->rx_packets++; + percpu_stats->rx_bytes +=qm_fd_get_length (fd); +return qman_cb_dqrr_consume; + } skb =contig_fd_to_skb (priv, fd); -else + }else { +/* XDP doesn't support S/G frames. Return the fragments to the +* buffer pool and release the SGT. +*/ +if (READ_ONCE (priv->xdp_prog)) { +WARN_ONCE (1 ,"S/G frames not supported under XDP\n" ); + sgt = vaddr +qm_fd_get_offset (fd); +dpaa_release_sgt_members (sgt); +free_pages ((unsigned long )vaddr,0 );
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal, · line 2521
Quoted 10 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +5
-1+return qman_cb_dqrr_consume; + } skb =sg_fd_to_skb (priv, fd); + }if (!skb)return qman_cb_dqrr_consume; -if (priv->rx_tstamp) { +if (xdp_meta_len) +skb_metadata_set (skb, xdp_meta_len);
This is working on a single buffer, right? So there's no need to clear xdp_meta_len?
Quoted 36 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +8
-10+ +/* Set the previously extracted timestamp */ +if (ts_valid) { shhwtstamps =skb_hwtstamps (skb);memset (shhwtstamps,0 ,sizeof (*shhwtstamps)); - -if (!fman_port_get_tstamp (priv->mac_dev->port[RX ], vaddr, &ns)) - shhwtstamps->hwtstamp =ns_to_ktime (ns); -else -dev_warn (net_dev->dev.parent,"fman_port_get_tstamp failed!\n" ); + shhwtstamps->hwtstamp =ns_to_ktime (ns); } skb->protocol =eth_type_trans (skb, net_dev); -if (net_dev->features &NETIF_F_RXHASH && priv->keygen_in_use && - !fman_port_get_hash_result_offset (priv->mac_dev->port[RX ], - &hash_offset)) { +/* Set the previously extracted hash */ +if (hash_valid) {enum pkt_hash_types type;/* if L4 exists, it was used in the hash generation */ type =be32_to_cpu (fd->status) &FM_FD_STAT_L4CV ?PKT_HASH_TYPE_L4 :PKT_HASH_TYPE_L3 ; -skb_set_hash (skb,be32_to_cpu (*(u32 *)(vaddr + hash_offset)), - type); +skb_set_hash (skb, hash, type); } skb_len = skb->len;@@ -2671,6 +2756,46 @@ static int dpaa_eth_stop(struct net_device *net_dev)return err; } +static int dpaa_setup_xdp (struct net_device *net_dev,struct bpf_prog *prog)
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static int dpaa_eth_stop(struct net_device *net_dev) · line 2769
Quoted 10 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +10
-0+{ +struct dpaa_priv *priv =netdev_priv (net_dev); +struct bpf_prog *old_prog; +bool up; +int err; + +/* S/G fragments are not supported in XDP-mode */ +if (prog && (priv->dpaa_bp->raw_size < + net_dev->mtu +VLAN_ETH_HLEN +ETH_FCS_LEN )) +return -EINVAL ;
Don't you want to include extack message here?
Quoted 4 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +4
-0+ + up =netif_running (net_dev); + +if (up)
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static int dpaa_eth_stop(struct net_device *net_dev) · line 2783
Quoted 10 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +10
-0+dpaa_eth_stop (net_dev); + + old_prog =xchg (&priv->xdp_prog, prog); +if (old_prog) +bpf_prog_put (old_prog); + +if (up) { + err =dpaa_open (net_dev); +if (err) +return err;
Out of curiosity, what should user do for that case? Would he be aware of the weird state of interface?
Quoted 42 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +16
-0+ } + +return 0 ; +} + +static int dpaa_xdp (struct net_device *net_dev,struct netdev_bpf *xdp) +{ +switch (xdp->command) { +case XDP_SETUP_PROG : +return dpaa_setup_xdp (net_dev, xdp->prog); +default : +return -EINVAL ; + } +} +static int dpaa_ts_ioctl (struct net_device *dev,struct ifreq *rq,int cmd) {struct dpaa_priv *priv =netdev_priv (dev);@@ -2737,6 +2862,7 @@ static int dpaa_ioctl(struct net_device *net_dev, struct ifreq *rq, int cmd) .ndo_set_rx_mode = dpaa_set_rx_mode, .ndo_do_ioctl = dpaa_ioctl, .ndo_setup_tc = dpaa_setup_tc, + .ndo_bpf = dpaa_xdp, };static int dpaa_napi_add (struct net_device *net_dev)drivers/net/ethernet/freescale/dpaa/dpaa_eth.h +2
-1@@ -196,6 +196,8 @@ struct dpaa_priv {bool tx_tstamp;/* Tx timestamping enabled */ bool rx_tstamp;/* Rx timestamping enabled */ + +struct bpf_prog *xdp_prog; };/* from dpaa_ethtool.c */ --1.9.1
RE: [PATCH net-next v3 2/7] dpaa_eth: add basic XDP support
Quoted 102 lines
-----Original Message-----
From: Maciej Fijalkowski <[email protected]>
Sent: Friday, November 20, 2020 02:19
To: Camelia Alexandra Groza <[email protected]>
Cc: [email protected]; [email protected]; [email protected]; [email protected]; Madalin Bucur (OSS) <[email protected]>; Ioana Ciornei <[email protected]>; [email protected]
Subject: Re: [PATCH net-next v3 2/7]dpaa_eth: add basic XDP supportQuoted 5 lines · Camelia Groza
Implement the
XDP_DROPandXDP_PASSactions.Avoid draining and reconfiguring the buffer pool at each XDP setup/teardown by increasing the frame headroom and reserving
XDP_PACKET_HEADROOMbytes from the start. Since we always reservean
Quoted 19 lines
entire page per buffer, this change only impacts Jumbo frame scenarios where the maximum linear frame size is reduced by 256 bytes. Multi buffer Scatter/Gather frames are now used instead in these scenarios.
Allow XDP programs to access the entire buffer.
The data in the received frame's headroom can be overwritten by the XDP program. Extract the relevant fields from the headroom while they are still available, before running the XDP program.
Since the headroom might be resized before the frame is passed up to the stack, remove the check for a fixed headroom value when building an skb.
Allow the meta data to be updated and pass the information up the stack.
Scatter/Gather frames are dropped when XDP is enabled.
Acked-by Madalin Bucur <[email protected]>Signed-off-by Camelia Groza <[email protected]>Quoted 8 lines
---
Changes in v2:
- warn only once if extracting the timestamp from a received frame failsChanges in v3:
- drop received S/G frames when XDP is enableddrivers/net/ethernet/freescale/dpaa/dpaa_eth.c | 158++++++++++++++++++++++---
Review
Quoted 4 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.h | 2 + 2 files changed, 144 insertions(+), 16 deletions(-)(quoted diff) +0
-0
b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.cQuoted 13 lines
index 88533a2..102023c 100644
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +2
-0@@ -53,6 +53,8 @@#include <linux/dma-mapping.h> #include <linux/sort.h> #include <linux/phy_fixed.h> +#include <linux/bpf.h> +#include <linux/bpf_trace.h> #include <soc/fsl/bman.h> #include <soc/fsl/qman.h> #include "fman.h" drivers/net/ethernet/freescale/dpaa/dpaa_eth.c line 179
Quoted 1 line
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +0
-0@@ -177,7 +179,7 @@#define DPAA_HWA_SIZE (DPAA_PARSE_RESULTS_SIZE +
DPAA_TIME_STAMP_SIZE\drivers/net/ethernet/freescale/dpaa/dpaa_eth.c line 181
Quoted 2 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +0
-0+ DPAA_HASH_RESULTS_SIZE)#define DPAA_RX_PRIV_DATA_DEFAULT_SIZE (
DPAA_TX_PRIV_DATA_SIZE+ \drivers/net/ethernet/freescale/dpaa/dpaa_eth.c line 182
Quoted 2 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +1
-1- dpaa_rx_extra_headroom) + XDP_PACKET_HEADROOM -
DPAA_HWA_SIZE)drivers/net/ethernet/freescale/dpaa/dpaa_eth.c line 184
Quoted 2 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +0
-0#ifdef CONFIG_DPAA_ERRATUM_A050385 #define DPAA_RX_PRIV_DATA_A050385_SIZE (DPAA_A050385_ALIGN -
DPAA_HWA_SIZE)drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static struct sk_buff *contig_fd_to_skb(const
Quoted 2 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +0
-0#define DPAA_RX_PRIV_DATA_SIZE (fman_has_errata_a050385() ? \@@ -1733,7 +1735,6 @@ static struct sk_buff *contig_fd_to_skb(conststruct
dpaa_priv*priv,drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result
Quoted 8 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +0
-1SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));if (WARN_ONCE (!skb,"Build skb failure on Rx\n" )) goto free_buffer; -WARN_ON (fd_off != priv->rx_headroom);skb_reserve (skb, fd_off);skb_put (skb,qm_fd_get_length (fd));@@ -2349,12 +2350,62 @@ static enum qman_cb_dqrr_resultrx_error_dqrr(struct
qman_portal*portal,drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_error_dqrr(struct qman_portal *portal,
Quoted 4 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +1
-0return qman_cb_dqrr_consume; } +static u32 dpaa_run_xdp(struct dpaa_priv *priv, struct qm_fd *fd, void*vaddr,
Quoted 11 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +11
-0+ unsigned int *xdp_meta_len) +{ +ssize_t fd_off =qm_fd_get_offset (fd); +struct bpf_prog *xdp_prog; +struct xdp_buff xdp; +u32 xdp_act; + +rcu_read_lock (); + + xdp_prog =READ_ONCE (priv->xdp_prog); +if (!xdp_prog) {
Quoted 8 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_error_dqrr(struct qman_portal *portal, · line 2369
Quoted 5 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +5
-0+rcu_read_unlock (); +return XDP_PASS ; + } + + xdp.data = vaddr + fd_off;I feel like a little drawing of
xdp_bufflayout would help me with understanding what is going on over here :)
A region at the start of the buffer is reserved for storing hardware annotations and room for the XDP_PACKET_HEADROOM, before the actual data starts. So vaddr points to the start of the buffer, while fd offset provides the offset of the data inside the buffer. I don't feel that we are filling the xdp_buff in a majorly different way from other drivers, so please mention what is unclear here and I can provide more details.
Quoted 7 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_error_dqrr(struct qman_portal *portal, · line 2373
Quoted 4 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +4
-0+ xdp.data_meta = xdp.data; + xdp.data_hard_start = xdp.data -XDP_PACKET_HEADROOM ; + xdp.data_end = xdp.data +qm_fd_get_length (fd); + xdp.frame_sz =DPAA_BP_RAW_SIZE ;Maybe you could fill
xdp_buffoutside of this function so that later on you could set xdp.rxq once per napi?
I admit I haven't looked into exactly how much performance we would gain from this, but I don't think it would be enough to justify the code churn. We don't have a clean loop for processing the received frames like I see the Intel and ENA drivers have.
Quoted 110 lines
Quoted 18 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +18
-0+ + xdp_act =bpf_prog_run_xdp (xdp_prog, &xdp); + +/* Update the length and the offset of the FD */ +qm_fd_set_contig (fd, xdp.data - vaddr, xdp.data_end - xdp.data); + +switch (xdp_act) { +case XDP_PASS : + *xdp_meta_len = xdp.data - xdp.data_meta; +break ; +default : +bpf_warn_invalid_xdp_action (xdp_act); + fallthrough; +case XDP_ABORTED : +trace_xdp_exception (priv->net_dev, xdp_prog, xdp_act); + fallthrough; +case XDP_DROP : +/* Free the buffer */ drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_error_dqrr(struct qman_portal *portal,
Quoted 10 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +9
-0+free_pages ((unsigned long )vaddr,0 ); +break ; + } + +rcu_read_unlock (); + +return xdp_act; +} + static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal*portal,
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result · line 2403
Quoted 2 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +0
-0struct qman_fq *fq, const struct qm_dqrr_entry*dq,
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result
Quoted 9 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +2
-0bool sched_napi) { +bool ts_valid = false, hash_valid = false;struct skb_shared_hwtstamps *shhwtstamps; +unsigned int skb_len, xdp_meta_len =0 ;struct rtnl_link_stats64 *percpu_stats;struct dpaa_percpu_priv *percpu_priv;const struct qm_fd *fd = &dq->fd;@@ -2362,12 +2413,14 @@ static enum qman_cb_dqrr_resultrx_default_dqrr(struct
qman_portal*portal,drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result
Quoted 16 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +3
-1enum qm_fd_format fd_format;struct net_device *net_dev;u32 fd_status, hash_offset; +struct qm_sg_entry *sgt;struct dpaa_bp *dpaa_bp;struct dpaa_priv *priv; -unsigned int skb_len;struct sk_buff *skb;int *count_ptr; +u32 xdp_act;void *vaddr; +u32 hash;u64 ns; fd_status =be32_to_cpu (fd->status);@@ -2423,35 +2476,67 @@ static enum qman_cb_dqrr_resultrx_default_dqrr(struct
qman_portal*portal,drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
Quoted 5 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +1
-1count_ptr =this_cpu_ptr (dpaa_bp->percpu_count); (*count_ptr)--; -if (likely (fd_format == qm_fd_contig)) +/* Extract the timestamp stored in the headroom before running XDP */
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result · line 2481
Quoted 2 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +2
-0+ if (priv->rx_tstamp) { + if (!fman_port_get_tstamp(priv->mac_dev->port[RX], vaddr,&ns))
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
Quoted 7 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +7
-0+ ts_valid = true; + else + WARN_ONCE(1, "fman_port_get_tstamp failed!\n"); + } + +/* Extract the hash stored in the headroom before running XDP */ + if (net_dev->features & NETIF_F_RXHASH && priv->keygen_in_use&&
Quoted 8 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +8
-0+ !fman_port_get_hash_result_offset(priv->mac_dev->port[RX], + &hash_offset)) { + hash =be32_to_cpu (*(u32 *)(vaddr + hash_offset)); + hash_valid = true; + } + +if (likely (fd_format == qm_fd_contig)) { + xdp_act = dpaa_run_xdp(priv, (struct qm_fd *)fd, vaddr,drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
Quoted 10 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +8
-1+ &xdp_meta_len); +if (xdp_act !=XDP_PASS ) { + percpu_stats->rx_packets++; + percpu_stats->rx_bytes +=qm_fd_get_length (fd); +return qman_cb_dqrr_consume; + } skb =contig_fd_to_skb (priv, fd); - else + }else { +/* XDP doesn't support S/G frames. Return the fragments to the
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
Quoted 4 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +4
-0+* buffer pool and release the SGT. +*/ +if (READ_ONCE (priv->xdp_prog)) { + WARN_ONCE(1, "S/G frames not supported underXDP\n");
Quoted 10 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +6
-0+ sgt = vaddr +qm_fd_get_offset (fd); +dpaa_release_sgt_members (sgt); +free_pages ((unsigned long )vaddr,0 ); +return qman_cb_dqrr_consume; + } skb =sg_fd_to_skb (priv, fd); + }if (!skb)return qman_cb_dqrr_consume;
Quoted 6 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal, · line 2521
Quoted 3 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +2
-1-if (priv->rx_tstamp) { +if (xdp_meta_len) +skb_metadata_set (skb, xdp_meta_len);This is working on a single buffer, right? So there's no need to clear
xdp_meta_len?
I don't think I understand what you mean. Are you saying I shouldn't be initializing xdp_meta_len to 0? This receive path is used when XDP is disabled as well, in which case we don't propagate the metadata.
Quoted 42 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
Quoted 7 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +3
-2+ +/* Set the previously extracted timestamp */ +if (ts_valid) { shhwtstamps =skb_hwtstamps (skb);memset (shhwtstamps,0 ,sizeof (*shhwtstamps)); - - if (!fman_port_get_tstamp(priv->mac_dev->port[RX], vaddr,&ns))
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
Quoted 3 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +0
-3- shhwtstamps->hwtstamp =ns_to_ktime (ns); - else - dev_warn(net_dev->dev.parent,"
fman_port_get_tstampfailed!\n");drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
Quoted 6 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +1
-1+ shhwtstamps->hwtstamp =ns_to_ktime (ns); } skb->protocol =eth_type_trans (skb, net_dev); - if (net_dev->features & NETIF_F_RXHASH && priv->keygen_in_use&&
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
Quoted 10 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +2
-3- !fman_port_get_hash_result_offset(priv->mac_dev->port[RX], - &hash_offset)) { +/* Set the previously extracted hash */ +if (hash_valid) {enum pkt_hash_types type;/* if L4 exists, it was used in the hash generation */ type =be32_to_cpu (fd->status) &FM_FD_STAT_L4CV ?PKT_HASH_TYPE_L4 :PKT_HASH_TYPE_L3 ; - skb_set_hash(skb, be32_to_cpu(*(u32 *)(vaddr +
hash_offset)),drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static int dpaa_eth_stop(struct net_device
Quoted 6 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +1
-1- type); +skb_set_hash (skb, hash, type); } skb_len = skb->len;@@ -2671,6 +2756,46 @@ static int dpaa_eth_stop(struct net_device
*net_dev)drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static int dpaa_eth_stop(struct net_device *net_dev)
Quoted 4 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +1
-0return err; } +static int dpaa_setup_xdp(struct net_device *net_dev, struct bpf_prog*prog)
Quoted 12 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static int dpaa_eth_stop(struct net_device *net_dev) · line 2769
Quoted 10 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +10
-0+{ +struct dpaa_priv *priv =netdev_priv (net_dev); +struct bpf_prog *old_prog; +bool up; +int err; + +/* S/G fragments are not supported in XDP-mode */ +if (prog && (priv->dpaa_bp->raw_size < + net_dev->mtu +VLAN_ETH_HLEN +ETH_FCS_LEN )) +return -EINVAL ;Don't you want to include extack message here?
This code is moved in the third patch and a dev_warn() is added. I should have added the warning from the start. I'll update it in the next version.
Quoted 9 lines
Quoted 9 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +9
-0+ + up =netif_running (net_dev); + +if (up) +dpaa_eth_stop (net_dev); + + old_prog =xchg (&priv->xdp_prog, prog); +if (old_prog) +bpf_prog_put (old_prog);
Quoted 8 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static int dpaa_eth_stop(struct net_device *net_dev) · line 2783
Quoted 5 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +5
-0+ +if (up) { + err =dpaa_open (net_dev); +if (err) +return err;Out of curiosity, what should user do for that case? Would he be aware of the weird state of interface?
This is an improbable state to reach. An error message would be useful. I'll add it in the next version. Thanks.
Quoted 44 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static int dpaa_ioctl(struct net_device *net_dev,
Quoted 19 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +15
-0+ } + +return 0 ; +} + +static int dpaa_xdp (struct net_device *net_dev,struct netdev_bpf *xdp) +{ +switch (xdp->command) { +case XDP_SETUP_PROG : +return dpaa_setup_xdp (net_dev, xdp->prog); +default : +return -EINVAL ; + } +} +static int dpaa_ts_ioctl (struct net_device *dev,struct ifreq *rq,int cmd) {struct dpaa_priv *priv =netdev_priv (dev);@@ -2737,6 +2862,7 @@ static int dpaa_ioctl(struct net_device *net_dev,struct ifreq *rq, int cmd)
Review
Quoted 8 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +1
-0.ndo_set_rx_mode = dpaa_set_rx_mode, .ndo_do_ioctl = dpaa_ioctl, .ndo_setup_tc = dpaa_setup_tc, + .ndo_bpf = dpaa_xdp, };static int dpaa_napi_add (struct net_device *net_dev)(quoted diff) +0
-0
b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.hQuoted 15 lines
index da30e5d..94e8613 100644
drivers/net/ethernet/freescale/dpaa/dpaa_eth.h +2
-1@@ -196,6 +196,8 @@ struct dpaa_priv {bool tx_tstamp;/* Tx timestamping enabled */ bool rx_tstamp;/* Rx timestamping enabled */ + +struct bpf_prog *xdp_prog; };/* from dpaa_ethtool.c */ --1.9.1
Re: [PATCH net-next v3 2/7] dpaa_eth: add basic XDP support
Quoted 111 lines · Camelia Alexandra Groza
Quoted 102 lines
-----Original Message-----
From: Maciej Fijalkowski <[email protected]>
Sent: Friday, November 20, 2020 02:19
To: Camelia Alexandra Groza <[email protected]>
Cc: [email protected]; [email protected]; [email protected]; [email protected]; Madalin Bucur (OSS) <[email protected]>; Ioana Ciornei <[email protected]>; [email protected]
Subject: Re: [PATCH net-next v3 2/7]dpaa_eth: add basic XDP supportQuoted 5 lines · Camelia Groza
Implement the
XDP_DROPandXDP_PASSactions.Avoid draining and reconfiguring the buffer pool at each XDP setup/teardown by increasing the frame headroom and reserving
XDP_PACKET_HEADROOMbytes from the start. Since we always reservean
Quoted 19 lines
entire page per buffer, this change only impacts Jumbo frame scenarios where the maximum linear frame size is reduced by 256 bytes. Multi buffer Scatter/Gather frames are now used instead in these scenarios.
Allow XDP programs to access the entire buffer.
The data in the received frame's headroom can be overwritten by the XDP program. Extract the relevant fields from the headroom while they are still available, before running the XDP program.
Since the headroom might be resized before the frame is passed up to the stack, remove the check for a fixed headroom value when building an skb.
Allow the meta data to be updated and pass the information up the stack.
Scatter/Gather frames are dropped when XDP is enabled.
Acked-by Madalin Bucur <[email protected]>Signed-off-by Camelia Groza <[email protected]>Quoted 8 lines
---
Changes in v2:
- warn only once if extracting the timestamp from a received frame failsChanges in v3:
- drop received S/G frames when XDP is enableddrivers/net/ethernet/freescale/dpaa/dpaa_eth.c | 158++++++++++++++++++++++---
Review
Quoted 4 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.h | 2 + 2 files changed, 144 insertions(+), 16 deletions(-)(quoted diff) +0
-0
b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.cQuoted 13 lines
index 88533a2..102023c 100644
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +2
-0@@ -53,6 +53,8 @@#include <linux/dma-mapping.h> #include <linux/sort.h> #include <linux/phy_fixed.h> +#include <linux/bpf.h> +#include <linux/bpf_trace.h> #include <soc/fsl/bman.h> #include <soc/fsl/qman.h> #include "fman.h" drivers/net/ethernet/freescale/dpaa/dpaa_eth.c line 179
Quoted 1 line
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +0
-0@@ -177,7 +179,7 @@#define DPAA_HWA_SIZE (DPAA_PARSE_RESULTS_SIZE +
DPAA_TIME_STAMP_SIZE\drivers/net/ethernet/freescale/dpaa/dpaa_eth.c line 181
Quoted 2 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +0
-0+ DPAA_HASH_RESULTS_SIZE)#define DPAA_RX_PRIV_DATA_DEFAULT_SIZE (
DPAA_TX_PRIV_DATA_SIZE+ \drivers/net/ethernet/freescale/dpaa/dpaa_eth.c line 182
Quoted 2 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +1
-1- dpaa_rx_extra_headroom) + XDP_PACKET_HEADROOM -
DPAA_HWA_SIZE)drivers/net/ethernet/freescale/dpaa/dpaa_eth.c line 184
Quoted 2 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +0
-0#ifdef CONFIG_DPAA_ERRATUM_A050385 #define DPAA_RX_PRIV_DATA_A050385_SIZE (DPAA_A050385_ALIGN -
DPAA_HWA_SIZE)drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static struct sk_buff *contig_fd_to_skb(const
Quoted 2 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +0
-0#define DPAA_RX_PRIV_DATA_SIZE (fman_has_errata_a050385() ? \@@ -1733,7 +1735,6 @@ static struct sk_buff *contig_fd_to_skb(conststruct
dpaa_priv*priv,drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result
Quoted 8 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +0
-1SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));if (WARN_ONCE (!skb,"Build skb failure on Rx\n" )) goto free_buffer; -WARN_ON (fd_off != priv->rx_headroom);skb_reserve (skb, fd_off);skb_put (skb,qm_fd_get_length (fd));@@ -2349,12 +2350,62 @@ static enum qman_cb_dqrr_resultrx_error_dqrr(struct
qman_portal*portal,drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_error_dqrr(struct qman_portal *portal,
Quoted 4 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +1
-0return qman_cb_dqrr_consume; } +static u32 dpaa_run_xdp(struct dpaa_priv *priv, struct qm_fd *fd, void*vaddr,
Quoted 11 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +11
-0+ unsigned int *xdp_meta_len) +{ +ssize_t fd_off =qm_fd_get_offset (fd); +struct bpf_prog *xdp_prog; +struct xdp_buff xdp; +u32 xdp_act; + +rcu_read_lock (); + + xdp_prog =READ_ONCE (priv->xdp_prog); +if (!xdp_prog) {Quoted 8 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_error_dqrr(struct qman_portal *portal, · line 2369
Quoted 5 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +5
-0+rcu_read_unlock (); +return XDP_PASS ; + } + + xdp.data = vaddr + fd_off;I feel like a little drawing of
xdp_bufflayout would help me with understanding what is going on over here :)
Quoted 1 line
A region at the start of the buffer is reserved for storing hardware annotations and room for the
XDP_PACKET_HEADROOM, before the actual data starts. So vaddr points to the start of the buffer, while fd offset provides the offset of the data inside the buffer. I don't feel that we are filling thexdp_buffin a majorly different way from other drivers, so please mention what is unclear here and I can provide more details.
Okay, so fd_off tells me where the frame starts, from vaddr to vaddr + fd_off there might be some HW provided data, so you extract it and then you are free to go with setting the data_hard_start?
Quoted 121 lines
Quoted 8 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_error_dqrr(struct qman_portal *portal, · line 2373
Quoted 5 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +4
-0+ xdp.data_meta = xdp.data; + xdp.data_hard_start = xdp.data -XDP_PACKET_HEADROOM ; + xdp.data_end = xdp.data +qm_fd_get_length (fd); + xdp.frame_sz =DPAA_BP_RAW_SIZE ;Maybe you could fill
xdp_buffoutside of this function so that later on you could set xdp.rxq once per napi?I admit I haven't looked into exactly how much performance we would gain from this, but I don't think it would be enough to justify the code churn. We don't have a clean loop for processing the received frames like I see the Intel and ENA drivers have.
Quoted 110 lines
Quoted 18 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +18
-0+ + xdp_act =bpf_prog_run_xdp (xdp_prog, &xdp); + +/* Update the length and the offset of the FD */ +qm_fd_set_contig (fd, xdp.data - vaddr, xdp.data_end - xdp.data); + +switch (xdp_act) { +case XDP_PASS : + *xdp_meta_len = xdp.data - xdp.data_meta; +break ; +default : +bpf_warn_invalid_xdp_action (xdp_act); + fallthrough; +case XDP_ABORTED : +trace_xdp_exception (priv->net_dev, xdp_prog, xdp_act); + fallthrough; +case XDP_DROP : +/* Free the buffer */ drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_error_dqrr(struct qman_portal *portal,
Quoted 10 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +9
-0+free_pages ((unsigned long )vaddr,0 ); +break ; + } + +rcu_read_unlock (); + +return xdp_act; +} + static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal*portal,
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result · line 2403
Quoted 2 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +0
-0struct qman_fq *fq, const struct qm_dqrr_entry*dq,
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result
Quoted 9 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +2
-0bool sched_napi) { +bool ts_valid = false, hash_valid = false;struct skb_shared_hwtstamps *shhwtstamps; +unsigned int skb_len, xdp_meta_len =0 ;struct rtnl_link_stats64 *percpu_stats;struct dpaa_percpu_priv *percpu_priv;const struct qm_fd *fd = &dq->fd;@@ -2362,12 +2413,14 @@ static enum qman_cb_dqrr_resultrx_default_dqrr(struct
qman_portal*portal,drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result
Quoted 16 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +3
-1enum qm_fd_format fd_format;struct net_device *net_dev;u32 fd_status, hash_offset; +struct qm_sg_entry *sgt;struct dpaa_bp *dpaa_bp;struct dpaa_priv *priv; -unsigned int skb_len;struct sk_buff *skb;int *count_ptr; +u32 xdp_act;void *vaddr; +u32 hash;u64 ns; fd_status =be32_to_cpu (fd->status);@@ -2423,35 +2476,67 @@ static enum qman_cb_dqrr_resultrx_default_dqrr(struct
qman_portal*portal,drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
Quoted 5 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +1
-1count_ptr =this_cpu_ptr (dpaa_bp->percpu_count); (*count_ptr)--; -if (likely (fd_format == qm_fd_contig)) +/* Extract the timestamp stored in the headroom before running XDP */
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result · line 2481
Quoted 2 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +2
-0+ if (priv->rx_tstamp) { + if (!fman_port_get_tstamp(priv->mac_dev->port[RX], vaddr,&ns))
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
Quoted 7 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +7
-0+ ts_valid = true; + else + WARN_ONCE(1, "fman_port_get_tstamp failed!\n"); + } + +/* Extract the hash stored in the headroom before running XDP */ + if (net_dev->features & NETIF_F_RXHASH && priv->keygen_in_use&&
Quoted 8 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +8
-0+ !fman_port_get_hash_result_offset(priv->mac_dev->port[RX], + &hash_offset)) { + hash =be32_to_cpu (*(u32 *)(vaddr + hash_offset)); + hash_valid = true; + } + +if (likely (fd_format == qm_fd_contig)) { + xdp_act = dpaa_run_xdp(priv, (struct qm_fd *)fd, vaddr,drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
Quoted 10 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +8
-1+ &xdp_meta_len); +if (xdp_act !=XDP_PASS ) { + percpu_stats->rx_packets++; + percpu_stats->rx_bytes +=qm_fd_get_length (fd); +return qman_cb_dqrr_consume; + } skb =contig_fd_to_skb (priv, fd); - else + }else { +/* XDP doesn't support S/G frames. Return the fragments to the
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
Quoted 4 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +4
-0+* buffer pool and release the SGT. +*/ +if (READ_ONCE (priv->xdp_prog)) { + WARN_ONCE(1, "S/G frames not supported underXDP\n");
Quoted 10 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +6
-0+ sgt = vaddr +qm_fd_get_offset (fd); +dpaa_release_sgt_members (sgt); +free_pages ((unsigned long )vaddr,0 ); +return qman_cb_dqrr_consume; + } skb =sg_fd_to_skb (priv, fd); + }if (!skb)return qman_cb_dqrr_consume;
Quoted 8 lines
Quoted 6 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal, · line 2521
Quoted 3 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +2
-1-if (priv->rx_tstamp) { +if (xdp_meta_len) +skb_metadata_set (skb, xdp_meta_len);This is working on a single buffer, right? So there's no need to clear
xdp_meta_len?I don't think I understand what you mean. Are you saying I shouldn't be initializing
xdp_meta_lento 0? This receive path is used when XDP is disabled as well, in which case we don't propagate the metadata.
What I meant was that if this function would operate on many buffers then we would have to clear the xdp_meta_len, so that next buffers wouldn't get the value from previous bufs, but I suppose that this rx_default_dqrr callback is called once per each buffer.
Quoted 40 lines
Quoted 40 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
Quoted 8 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +3
-2+ +/* Set the previously extracted timestamp */ +if (ts_valid) { shhwtstamps =skb_hwtstamps (skb);memset (shhwtstamps,0 ,sizeof (*shhwtstamps)); - - if (!fman_port_get_tstamp(priv->mac_dev->port[RX], vaddr,&ns))
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
Quoted 3 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +0
-3- shhwtstamps->hwtstamp =ns_to_ktime (ns); - else - dev_warn(net_dev->dev.parent,"
fman_port_get_tstampfailed!\n");drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
Quoted 6 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +1
-1+ shhwtstamps->hwtstamp =ns_to_ktime (ns); } skb->protocol =eth_type_trans (skb, net_dev); - if (net_dev->features & NETIF_F_RXHASH && priv->keygen_in_use&&
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
Quoted 10 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +2
-3- !fman_port_get_hash_result_offset(priv->mac_dev->port[RX], - &hash_offset)) { +/* Set the previously extracted hash */ +if (hash_valid) {enum pkt_hash_types type;/* if L4 exists, it was used in the hash generation */ type =be32_to_cpu (fd->status) &FM_FD_STAT_L4CV ?PKT_HASH_TYPE_L4 :PKT_HASH_TYPE_L3 ; - skb_set_hash(skb, be32_to_cpu(*(u32 *)(vaddr +
hash_offset)),drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static int dpaa_eth_stop(struct net_device
Quoted 6 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +1
-1- type); +skb_set_hash (skb, hash, type); } skb_len = skb->len;@@ -2671,6 +2756,46 @@ static int dpaa_eth_stop(struct net_device
*net_dev)Quoted 2 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +0
-0return err; }
RE: [PATCH net-next v3 2/7] dpaa_eth: add basic XDP support
Quoted 136 lines
-----Original Message-----
From: Maciej Fijalkowski <[email protected]>
Sent: Tuesday, November 24, 2020 00:08
To: Camelia Alexandra Groza <[email protected]>
Cc: [email protected]; [email protected]; [email protected]; [email protected]; Madalin Bucur (OSS) <[email protected]>; Ioana Ciornei <[email protected]>; [email protected]
Subject: Re: [PATCH net-next v3 2/7]dpaa_eth: add basic XDP supportQuoted 10 lines · Camelia Alexandra Groza
Quoted 10 lines
-----Original Message-----
From: Maciej Fijalkowski <[email protected]>
Sent: Friday, November 20, 2020 02:19
To: Camelia Alexandra Groza <[email protected]>
Cc: [email protected]; [email protected]; [email protected]; [email protected]; Madalin Bucur (OSS) <[email protected]>; Ioana Ciornei <[email protected]>; [email protected]
Subject: Re: [PATCH net-next v3 2/7]dpaa_eth: add basic XDP supportQuoted 6 lines
Quoted 6 lines · Camelia Groza
Quoted 5 lines
Implement the
XDP_DROPandXDP_PASSactions.Avoid draining and reconfiguring the buffer pool at each XDP setup/teardown by increasing the frame headroom and reserving
XDP_PACKET_HEADROOMbytes from the start. Since we alwaysreserve
Quoted 8 lines
Quoted 8 lines
an
Quoted 7 lines
entire page per buffer, this change only impacts Jumbo frame scenarios where the maximum linear frame size is reduced by 256 bytes. Multi buffer Scatter/Gather frames are now used instead in these scenarios.
Allow XDP programs to access the entire buffer.
The data in the received frame's headroom can be overwritten by the
XDP
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
program. Extract the relevant fields from the headroom while they are still available, before running the XDP program.
Since the headroom might be resized before the frame is passed up to
the
Quoted 1 line
Quoted 1 line
Quoted 1 line
stack, remove the check for a fixed headroom value when building an
skb.
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Allow the meta data to be updated and pass the information up the
stack.
Quoted 34 lines
Quoted 34 lines
Quoted 5 lines
Scatter/Gather frames are dropped when XDP is enabled.
Acked-by Madalin Bucur <[email protected]>Signed-off-by Camelia Groza <[email protected]>Quoted 8 lines
---
Changes in v2:
- warn only once if extracting the timestamp from a received frame failsChanges in v3:
- drop received S/G frames when XDP is enableddrivers/net/ethernet/freescale/dpaa/dpaa_eth.c | 158++++++++++++++++++++++---
Review
Quoted 4 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.h | 2 + 2 files changed, 144 insertions(+), 16 deletions(-)(quoted diff) +0
-0
b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.cQuoted 13 lines
index 88533a2..102023c 100644
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +2
-0@@ -53,6 +53,8 @@#include <linux/dma-mapping.h> #include <linux/sort.h> #include <linux/phy_fixed.h> +#include <linux/bpf.h> +#include <linux/bpf_trace.h> #include <soc/fsl/bman.h> #include <soc/fsl/qman.h> #include "fman.h" drivers/net/ethernet/freescale/dpaa/dpaa_eth.c line 179
Quoted 1 line
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +0
-0@@ -177,7 +179,7 @@#define DPAA_HWA_SIZE (DPAA_PARSE_RESULTS_SIZE +
DPAA_TIME_STAMP_SIZE\Quoted 8 lines
Quoted 8 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c line 181
Quoted 2 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +0
-0+ DPAA_HASH_RESULTS_SIZE)#define DPAA_RX_PRIV_DATA_DEFAULT_SIZE (
DPAA_TX_PRIV_DATA_SIZE+ \drivers/net/ethernet/freescale/dpaa/dpaa_eth.c line 182
Quoted 2 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +1
-1- dpaa_rx_extra_headroom) + XDP_PACKET_HEADROOM -
DPAA_HWA_SIZE)Quoted 2 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +0
-0#ifdef CONFIG_DPAA_ERRATUM_A050385 #define DPAA_RX_PRIV_DATA_A050385_SIZE (DPAA_A050385_ALIGN-
Quoted 11 lines
Quoted 11 lines
DPAA_HWA_SIZE)drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static struct sk_buff *contig_fd_to_skb(const
Quoted 2 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +0
-0#define DPAA_RX_PRIV_DATA_SIZE (fman_has_errata_a050385() ? \@@ -1733,7 +1735,6 @@ static struct sk_buff *contig_fd_to_skb(conststruct
dpaa_priv*priv,Quoted 7 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +0
-1SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));if (WARN_ONCE (!skb,"Build skb failure on Rx\n" )) goto free_buffer; -WARN_ON (fd_off != priv->rx_headroom);skb_reserve (skb, fd_off);skb_put (skb,qm_fd_get_length (fd));Quoted 6 lines
Quoted 6 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result
Quoted 1 line
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +0
-0@@ -2349,12 +2350,62 @@ static enum qman_cb_dqrr_resultrx_error_dqrr(struct
qman_portal*portal,Quoted 4 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +1
-0return qman_cb_dqrr_consume; } +static u32 dpaa_run_xdp(struct dpaa_priv *priv, struct qm_fd *fd,void
Quoted 21 lines
Quoted 12 lines
*vaddr,
Quoted 11 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +11
-0+ unsigned int *xdp_meta_len) +{ +ssize_t fd_off =qm_fd_get_offset (fd); +struct bpf_prog *xdp_prog; +struct xdp_buff xdp; +u32 xdp_act; + +rcu_read_lock (); + + xdp_prog =READ_ONCE (priv->xdp_prog); +if (!xdp_prog) {Quoted 8 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_error_dqrr(struct qman_portal *portal, · line 2369
Quoted 5 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +5
-0+rcu_read_unlock (); +return XDP_PASS ; + } + + xdp.data = vaddr + fd_off;I feel like a little drawing of
xdp_bufflayout would help me with understanding what is going on over here :)Quoted 1 line
A region at the start of the buffer is reserved for storing hardware
annotations and room for the
XDP_PACKET_HEADROOM, before the actual data starts. So vaddr points to the start of the buffer, while fd offset provides the offset of the data inside the buffer. I don't feel that we are filling thexdp_buffin a majorly different way from other drivers, so please mention what is unclear here and I can provide more details.
Quoted 3 lines
Okay, so
fd_offtells me where the frame starts, from vaddr to vaddr +fd_offthere might be some HW provided data, so you extract it and then you are free to go with setting thedata_hard_start?
Yes, that's right.
Quoted 131 lines
Quoted 10 lines
Quoted 8 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_error_dqrr(struct qman_portal *portal, · line 2373
Quoted 5 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +4
-0+ xdp.data_meta = xdp.data; + xdp.data_hard_start = xdp.data -XDP_PACKET_HEADROOM ; + xdp.data_end = xdp.data +qm_fd_get_length (fd); + xdp.frame_sz =DPAA_BP_RAW_SIZE ;Maybe you could fill
xdp_buffoutside of this function so that later on you could set xdp.rxq once per napi?I admit I haven't looked into exactly how much performance we would gain
from this, but I don't think it would be enough to justify the code churn. We don't have a clean loop for processing the received frames like I see the Intel and ENA drivers have.
Quoted 118 lines
Quoted 117 lines
Quoted 19 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +18
-0+ + xdp_act =bpf_prog_run_xdp (xdp_prog, &xdp); + +/* Update the length and the offset of the FD */ +qm_fd_set_contig (fd, xdp.data - vaddr, xdp.data_end - xdp.data); + +switch (xdp_act) { +case XDP_PASS : + *xdp_meta_len = xdp.data - xdp.data_meta; +break ; +default : +bpf_warn_invalid_xdp_action (xdp_act); + fallthrough; +case XDP_ABORTED : +trace_xdp_exception (priv->net_dev, xdp_prog, xdp_act); + fallthrough; +case XDP_DROP : +/* Free the buffer */ drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_error_dqrr(struct qman_portal *portal,
Quoted 10 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +9
-0+free_pages ((unsigned long )vaddr,0 ); +break ; + } + +rcu_read_unlock (); + +return xdp_act; +} + static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal*portal,
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result · line 2403
Quoted 2 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +0
-0struct qman_fq *fq, const struct qm_dqrr_entry*dq,
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result
Quoted 9 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +2
-0bool sched_napi) { +bool ts_valid = false, hash_valid = false;struct skb_shared_hwtstamps *shhwtstamps; +unsigned int skb_len, xdp_meta_len =0 ;struct rtnl_link_stats64 *percpu_stats;struct dpaa_percpu_priv *percpu_priv;const struct qm_fd *fd = &dq->fd;@@ -2362,12 +2413,14 @@ static enum qman_cb_dqrr_resultrx_default_dqrr(struct
qman_portal*portal,drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result
Quoted 16 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +3
-1enum qm_fd_format fd_format;struct net_device *net_dev;u32 fd_status, hash_offset; +struct qm_sg_entry *sgt;struct dpaa_bp *dpaa_bp;struct dpaa_priv *priv; -unsigned int skb_len;struct sk_buff *skb;int *count_ptr; +u32 xdp_act;void *vaddr; +u32 hash;u64 ns; fd_status =be32_to_cpu (fd->status);@@ -2423,35 +2476,67 @@ static enum qman_cb_dqrr_resultrx_default_dqrr(struct
qman_portal*portal,drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
Quoted 5 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +1
-1count_ptr =this_cpu_ptr (dpaa_bp->percpu_count); (*count_ptr)--; -if (likely (fd_format == qm_fd_contig)) +/* Extract the timestamp stored in the headroom before running XDP */
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result · line 2481
Quoted 2 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +2
-0+ if (priv->rx_tstamp) { + if (!fman_port_get_tstamp(priv->mac_dev->port[RX], vaddr,&ns))
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
Quoted 7 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +7
-0+ ts_valid = true; + else + WARN_ONCE(1, "fman_port_get_tstamp failed!\n"); + } + +/* Extract the hash stored in the headroom before running XDP */ + if (net_dev->features & NETIF_F_RXHASH && priv->keygen_in_use&&
Quoted 8 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +8
-0+ !fman_port_get_hash_result_offset(priv->mac_dev->port[RX], + &hash_offset)) { + hash =be32_to_cpu (*(u32 *)(vaddr + hash_offset)); + hash_valid = true; + } + +if (likely (fd_format == qm_fd_contig)) { + xdp_act = dpaa_run_xdp(priv, (struct qm_fd *)fd, vaddr,drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
Quoted 10 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +8
-1+ &xdp_meta_len); +if (xdp_act !=XDP_PASS ) { + percpu_stats->rx_packets++; + percpu_stats->rx_bytes +=qm_fd_get_length (fd); +return qman_cb_dqrr_consume; + } skb =contig_fd_to_skb (priv, fd); - else + }else { +/* XDP doesn't support S/G frames. Return the fragments to the
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
Quoted 4 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +4
-0+* buffer pool and release the SGT. +*/ +if (READ_ONCE (priv->xdp_prog)) { + WARN_ONCE(1, "S/G frames not supported underXDP\n");
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal, · line 2521
Quoted 13 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +8
-1+ sgt = vaddr +qm_fd_get_offset (fd); +dpaa_release_sgt_members (sgt); +free_pages ((unsigned long )vaddr,0 ); +return qman_cb_dqrr_consume; + } skb =sg_fd_to_skb (priv, fd); + }if (!skb)return qman_cb_dqrr_consume; -if (priv->rx_tstamp) { +if (xdp_meta_len) +skb_metadata_set (skb, xdp_meta_len);This is working on a single buffer, right? So there's no need to clear
xdp_meta_len?
Quoted 8 lines
Quoted 1 line
I don't think I understand what you mean. Are you saying I shouldn't be
initializing
xdp_meta_lento 0? This receive path is used when XDP is disabled as well, in which case we don't propagate the metadata.What I meant was that if this function would operate on many buffers then we would have to clear the
xdp_meta_len, so that next buffers wouldn't get the value from previous bufs, but I suppose that thisrx_default_dqrrcallback is called once per each buffer.
Yes, rx_default_dqrr is processing only one buffer.
Quoted 40 lines
Quoted 40 lines
Quoted 40 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
Quoted 8 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +3
-2+ +/* Set the previously extracted timestamp */ +if (ts_valid) { shhwtstamps =skb_hwtstamps (skb);memset (shhwtstamps,0 ,sizeof (*shhwtstamps)); - - if (!fman_port_get_tstamp(priv->mac_dev->port[RX], vaddr,&ns))
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
Quoted 3 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +0
-3- shhwtstamps->hwtstamp =ns_to_ktime (ns); - else - dev_warn(net_dev->dev.parent,"
fman_port_get_tstampfailed!\n");drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
Quoted 6 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +1
-1+ shhwtstamps->hwtstamp =ns_to_ktime (ns); } skb->protocol =eth_type_trans (skb, net_dev); - if (net_dev->features & NETIF_F_RXHASH && priv->keygen_in_use&&
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
Quoted 10 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +2
-3- !fman_port_get_hash_result_offset(priv->mac_dev->port[RX], - &hash_offset)) { +/* Set the previously extracted hash */ +if (hash_valid) {enum pkt_hash_types type;/* if L4 exists, it was used in the hash generation */ type =be32_to_cpu (fd->status) &FM_FD_STAT_L4CV ?PKT_HASH_TYPE_L4 :PKT_HASH_TYPE_L3 ; - skb_set_hash(skb, be32_to_cpu(*(u32 *)(vaddr +
hash_offset)),drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static int dpaa_eth_stop(struct net_device
Quoted 6 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +1
-1- type); +skb_set_hash (skb, hash, type); } skb_len = skb->len;@@ -2671,6 +2756,46 @@ static int dpaa_eth_stop(struct net_device
*net_dev)Quoted 2 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +0
-0return err; }
PATCH v3 3/7 dpaa_eth: limit the possible MTU range when XDP is enabled
In reply to Camelia GrozaImplement the ndo_change_mtu callback to prevent users from setting an MTU that would permit processing of S/G frames. The maximum MTU size is dependent on the buffer size.
--- drivers/net/ethernet/freescale/dpaa/dpaa_eth.c | 32 ++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-)
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +30 -2
@@ -2756,6 +2756,34 @@ static int dpaa_eth_stop(struct net_device *net_dev)return err; } +static bool xdp_validate_mtu (struct dpaa_priv *priv,int mtu) +{ +int max_contig_data = priv->dpaa_bp->size - priv->rx_headroom; + +/* We do not support S/G fragments when XDP is enabled. +* Limit the MTU in relation to the buffer size. +*/ +if (mtu +VLAN_ETH_HLEN +ETH_FCS_LEN > max_contig_data) { +dev_warn (priv->net_dev->dev.parent, +"The maximum MTU for XDP is %d\n" , + max_contig_data -VLAN_ETH_HLEN -ETH_FCS_LEN ); +return false; + } + +return true; +} + +static int dpaa_change_mtu (struct net_device *net_dev,int new_mtu) +{ +struct dpaa_priv *priv =netdev_priv (net_dev); + +if (priv->xdp_prog && !xdp_validate_mtu (priv, new_mtu)) +return -EINVAL ; + + net_dev->mtu = new_mtu; +return 0 ; +} +static int dpaa_setup_xdp (struct net_device *net_dev,struct bpf_prog *prog) {struct dpaa_priv *priv =netdev_priv (net_dev);
@@ -2764,8 +2792,7 @@ static int dpaa_setup_xdp(struct net_device *net_dev, struct bpf_prog *prog)int err;/* S/G fragments are not supported in XDP-mode */ -if (prog && (priv->dpaa_bp->raw_size < - net_dev->mtu +VLAN_ETH_HLEN +ETH_FCS_LEN )) +if (prog && !xdp_validate_mtu (priv, net_dev->mtu))return -EINVAL ; up =netif_running (net_dev);
@@ -2862,6 +2889,7 @@ static int dpaa_ioctl(struct net_device *net_dev, struct ifreq *rq, int cmd) .ndo_set_rx_mode = dpaa_set_rx_mode, .ndo_do_ioctl = dpaa_ioctl, .ndo_setup_tc = dpaa_setup_tc, + .ndo_change_mtu = dpaa_change_mtu, .ndo_bpf = dpaa_xdp, };
Signature · 1 line
1.9.1
PATCH v3 4/7 dpaa_eth: add XDP_TX support
In reply to Camelia GrozaUse an xdp_frame structure for managing the frame. Store a backpointer to the structure at the start of the buffer before enqueueing. Use the XDP API for freeing the buffer when it returns to the driver on the TX confirmation path.
This approach will be reused for XDP REDIRECT.
--- drivers/net/ethernet/freescale/dpaa/dpaa_eth.c | 129 ++++++++++++++++++++++++- drivers/net/ethernet/freescale/dpaa/dpaa_eth.h | 2 + 2 files changed, 126 insertions(+), 5 deletions(-)
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +124 -5
@@ -1130,6 +1130,24 @@ static int dpaa_fq_init(struct dpaa_fq *dpaa_fq, bool td_enable) dpaa_fq->fqid =qman_fq_fqid (fq); +if (dpaa_fq->fq_type ==FQ_TYPE_RX_DEFAULT || + dpaa_fq->fq_type ==FQ_TYPE_RX_PCD ) { + err =xdp_rxq_info_reg (&dpaa_fq->xdp_rxq, dpaa_fq->net_dev, + dpaa_fq->fqid); +if (err) { +dev_err (dev,"xdp_rxq_info_reg failed\n" ); +return err; + } + + err =xdp_rxq_info_reg_mem_model (&dpaa_fq->xdp_rxq, +MEM_TYPE_PAGE_ORDER0 ,NULL ); +if (err) { +dev_err (dev,"xdp_rxq_info_reg_mem_model failed\n" ); +xdp_rxq_info_unreg (&dpaa_fq->xdp_rxq); +return err; + } + } +return 0 ; }
@@ -1159,6 +1177,10 @@ static int dpaa_fq_free_entry(struct device *dev, struct qman_fq *fq) } } +if (dpaa_fq->fq_type ==FQ_TYPE_RX_DEFAULT || + dpaa_fq->fq_type ==FQ_TYPE_RX_PCD ) +xdp_rxq_info_unreg (&dpaa_fq->xdp_rxq); +qman_destroy_fq (fq);list_del (&dpaa_fq->list);
@@ -1625,6 +1647,9 @@ static int dpaa_eth_refill_bpools(struct dpaa_priv *priv)* * Return the skb backpointer, since for S/G frames the buffer containing it * gets freed here. +* +* No skb backpointer is set when transmitting XDP frames. Cleanup the buffer +* and return NULL in this case. */ static struct sk_buff *dpaa_cleanup_tx_fd (const struct dpaa_priv *priv,const struct qm_fd *fd,bool ts)
@@ -1636,6 +1661,7 @@ static struct sk_buff *dpaa_cleanup_tx_fd(const struct dpaa_priv *priv,void *vaddr =phys_to_virt (addr);const struct qm_sg_entry *sgt;struct dpaa_eth_swbp *swbp; +struct xdp_frame *xdpf;struct sk_buff *skb;u64 ns;int i;
@@ -1664,13 +1690,22 @@ static struct sk_buff *dpaa_cleanup_tx_fd(const struct dpaa_priv *priv, } }else {dma_unmap_single (priv->tx_dma_dev, addr, - priv->tx_headroom +qm_fd_get_length (fd), +qm_fd_get_offset (fd) +qm_fd_get_length (fd), dma_dir); } swbp = (struct dpaa_eth_swbp *)vaddr; skb = swbp->skb; +/* No skb backpointer is set when running XDP. An xdp_frame +* backpointer is saved instead. +*/ +if (!skb) { + xdpf = swbp->xdpf; +xdp_return_frame (xdpf); +return NULL ; + } +/* DMA unmapping is required before accessing the HW provided info */ if (ts && priv->tx_tstamp &&skb_shinfo (skb)->tx_flags &SKBTX_HW_TSTAMP ) {
@@ -2350,11 +2385,76 @@ static enum qman_cb_dqrr_result rx_error_dqrr(struct qman_portal *portal,return qman_cb_dqrr_consume; } +static int dpaa_xdp_xmit_frame (struct net_device *net_dev, +struct xdp_frame *xdpf) +{ +struct dpaa_priv *priv =netdev_priv (net_dev); +struct rtnl_link_stats64 *percpu_stats; +struct dpaa_percpu_priv *percpu_priv; +struct dpaa_eth_swbp *swbp; +struct netdev_queue *txq; +void *buff_start; +struct qm_fd fd; +dma_addr_t addr; +int err; + + percpu_priv =this_cpu_ptr (priv->percpu_priv); + percpu_stats = &percpu_priv->stats; + +if (xdpf->headroom <DPAA_TX_PRIV_DATA_SIZE ) { + err = -EINVAL ; + goto out_error; + } + + buff_start = xdpf->data - xdpf->headroom; + +/* Leave empty the skb backpointer at the start of the buffer. +* Save the XDP frame for easy cleanup on confirmation. +*/ + swbp = (struct dpaa_eth_swbp *)buff_start; + swbp->skb =NULL ; + swbp->xdpf = xdpf; + +qm_fd_clear_fd (&fd); + fd.bpid =FSL_DPAA_BPID_INV ; + fd.cmd |=cpu_to_be32 (FM_FD_CMD_FCO ); +qm_fd_set_contig (&fd, xdpf->headroom, xdpf->len); + + addr =dma_map_single (priv->tx_dma_dev, buff_start, + xdpf->headroom + xdpf->len, +DMA_TO_DEVICE ); +if (unlikely (dma_mapping_error (priv->tx_dma_dev, addr))) { + err = -EINVAL ; + goto out_error; + } + +qm_fd_addr_set64 (&fd, addr); + +/* Bump the trans_start */ + txq =netdev_get_tx_queue (net_dev,smp_processor_id ()); + txq->trans_start = jiffies; + + err =dpaa_xmit (priv, percpu_stats,smp_processor_id (), &fd); +if (err) { +dma_unmap_single (priv->tx_dma_dev, addr, +qm_fd_get_offset (&fd) +qm_fd_get_length (&fd), +DMA_TO_DEVICE ); + goto out_error; + } + +return 0 ; + +out_error: + percpu_stats->tx_errors++; +return err; +} +static u32 dpaa_run_xdp (struct dpaa_priv *priv,struct qm_fd *fd,void *vaddr, -unsigned int *xdp_meta_len) +struct dpaa_fq *dpaa_fq,unsigned int *xdp_meta_len) {ssize_t fd_off =qm_fd_get_offset (fd);struct bpf_prog *xdp_prog; +struct xdp_frame *xdpf;struct xdp_buff xdp;u32 xdp_act;
@@ -2370,7 +2470,8 @@ static u32 dpaa_run_xdp(struct dpaa_priv *priv, struct qm_fd *fd, void *vaddr, xdp.data_meta = xdp.data; xdp.data_hard_start = xdp.data -XDP_PACKET_HEADROOM ; xdp.data_end = xdp.data +qm_fd_get_length (fd); - xdp.frame_sz =DPAA_BP_RAW_SIZE ; + xdp.frame_sz =DPAA_BP_RAW_SIZE -DPAA_TX_PRIV_DATA_SIZE ; + xdp.rxq = &dpaa_fq->xdp_rxq; xdp_act =bpf_prog_run_xdp (xdp_prog, &xdp);
@@ -2381,6 +2482,22 @@ static u32 dpaa_run_xdp(struct dpaa_priv *priv, struct qm_fd *fd, void *vaddr,case XDP_PASS : *xdp_meta_len = xdp.data - xdp.data_meta;break ; +case XDP_TX : +/* We can access the full headroom when sending the frame +* back out +*/ + xdp.data_hard_start = vaddr; + xdp.frame_sz =DPAA_BP_RAW_SIZE ; + xdpf =xdp_convert_buff_to_frame (&xdp); +if (unlikely (!xdpf)) { +free_pages ((unsigned long )vaddr,0 ); +break ; + } + +if (dpaa_xdp_xmit_frame (priv->net_dev, xdpf)) +xdp_return_frame_rx_napi (xdpf); + +break ;default :bpf_warn_invalid_xdp_action (xdp_act); fallthrough;
@@ -2415,6 +2532,7 @@ static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,u32 fd_status, hash_offset;struct qm_sg_entry *sgt;struct dpaa_bp *dpaa_bp; +struct dpaa_fq *dpaa_fq;struct dpaa_priv *priv;struct sk_buff *skb;int *count_ptr;
@@ -2423,9 +2541,10 @@ static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,u32 hash;u64 ns; + dpaa_fq = container_of(fq, struct dpaa_fq, fq_base); fd_status =be32_to_cpu (fd->status); fd_format =qm_fd_get_format (fd); - net_dev = ((struct dpaa_fq *)fq)->net_dev; + net_dev = dpaa_fq->net_dev; priv =netdev_priv (net_dev); dpaa_bp =dpaa_bpid2pool (dq->fd.bpid);if (!dpaa_bp)
@@ -2494,7 +2613,7 @@ static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,if (likely (fd_format == qm_fd_contig)) { xdp_act =dpaa_run_xdp (priv, (struct qm_fd *)fd, vaddr, - &xdp_meta_len); + dpaa_fq, &xdp_meta_len);if (xdp_act !=XDP_PASS ) { percpu_stats->rx_packets++; percpu_stats->rx_bytes +=qm_fd_get_length (fd);
drivers/net/ethernet/freescale/dpaa/dpaa_eth.h +2 -0
@@ -68,6 +68,7 @@ struct dpaa_fq {u16 channel;u8 wq;enum dpaa_fq_type fq_type; +struct xdp_rxq_info xdp_rxq; };struct dpaa_fq_cbs {
@@ -150,6 +151,7 @@ struct dpaa_buffer_layout {*/ struct dpaa_eth_swbp {struct sk_buff *skb; +struct xdp_frame *xdpf; };struct dpaa_priv {
Signature · 1 line
1.9.1
Re: [PATCH net-next v3 4/7] dpaa_eth: add XDP_TX support
Quoted 4 lines · Camelia Groza
Use an
xdp_framestructure for managing the frame. Store a backpointer to the structure at the start of the buffer before enqueueing. Use the XDP API for freeing the buffer when it returns to the driver on the TX confirmation path.
Completion path?
Quoted 15 lines
This approach will be reused for XDP REDIRECT.
Acked-by Madalin Bucur <[email protected]>Signed-off-by Camelia Groza <[email protected]>--- drivers/net/ethernet/freescale/dpaa/dpaa_eth.c | 129 ++++++++++++++++++++++++- drivers/net/ethernet/freescale/dpaa/dpaa_eth.h | 2 + 2 files changed, 126 insertions(+), 5 deletions(-)drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +0
-0
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static int dpaa_fq_init(struct dpaa_fq *dpaa_fq, bool td_enable) · line 1138
Quoted 9 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +6
-0@@ -1130,6 +1130,24 @@ static int dpaa_fq_init(struct dpaa_fq *dpaa_fq, bool td_enable) dpaa_fq->fqid =qman_fq_fqid (fq); +if (dpaa_fq->fq_type ==FQ_TYPE_RX_DEFAULT || + dpaa_fq->fq_type ==FQ_TYPE_RX_PCD ) { + err =xdp_rxq_info_reg (&dpaa_fq->xdp_rxq, dpaa_fq->net_dev, + dpaa_fq->fqid); +if (err) { +dev_err (dev,"xdp_rxq_info_reg failed\n" );
Print out the err?
Also, shouldn't you call qman_destroy_fq() for these error paths?
Quoted 16 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +12
-0+return err; + } + + err =xdp_rxq_info_reg_mem_model (&dpaa_fq->xdp_rxq, +MEM_TYPE_PAGE_ORDER0 ,NULL ); +if (err) { +dev_err (dev,"xdp_rxq_info_reg_mem_model failed\n" ); +xdp_rxq_info_unreg (&dpaa_fq->xdp_rxq); +return err; + } + } +return 0 ; }
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static int dpaa_fq_free_entry(struct device *dev, struct qman_fq *fq) · line 1181
Quoted 5 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +2
-0@@ -1159,6 +1177,10 @@ static int dpaa_fq_free_entry(struct device *dev, struct qman_fq *fq) } } +if (dpaa_fq->fq_type ==FQ_TYPE_RX_DEFAULT || + dpaa_fq->fq_type ==FQ_TYPE_RX_PCD )
You should call xdp_rxq_info_is_reg() before the unregister below.
Quoted 16 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +5
-0+xdp_rxq_info_unreg (&dpaa_fq->xdp_rxq); +qman_destroy_fq (fq);list_del (&dpaa_fq->list);@@ -1625,6 +1647,9 @@ static int dpaa_eth_refill_bpools(struct dpaa_priv *priv)* * Return the skb backpointer, since for S/G frames the buffer containing it * gets freed here. +* +* No skb backpointer is set when transmitting XDP frames. Cleanup the buffer +* and return NULL in this case. */ static struct sk_buff *dpaa_cleanup_tx_fd (const struct dpaa_priv *priv,const struct qm_fd *fd,bool ts)
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static struct sk_buff *dpaa_cleanup_tx_fd(const struct dpaa_priv *priv, · line 1664
Quoted 4 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +1
-0@@ -1636,6 +1661,7 @@ static struct sk_buff *dpaa_cleanup_tx_fd(const struct dpaa_priv *priv,void *vaddr =phys_to_virt (addr);const struct qm_sg_entry *sgt;struct dpaa_eth_swbp *swbp; +struct xdp_frame *xdpf;
This local variable feels a bit unnecessary.
Quoted 38 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +17
-1struct sk_buff *skb;u64 ns;int i;@@ -1664,13 +1690,22 @@ static struct sk_buff *dpaa_cleanup_tx_fd(const struct dpaa_priv *priv, } }else {dma_unmap_single (priv->tx_dma_dev, addr, - priv->tx_headroom +qm_fd_get_length (fd), +qm_fd_get_offset (fd) +qm_fd_get_length (fd), dma_dir); } swbp = (struct dpaa_eth_swbp *)vaddr; skb = swbp->skb; +/* No skb backpointer is set when running XDP. An xdp_frame +* backpointer is saved instead. +*/ +if (!skb) { + xdpf = swbp->xdpf; +xdp_return_frame (xdpf); +return NULL ; + } +/* DMA unmapping is required before accessing the HW provided info */ if (ts && priv->tx_tstamp &&skb_shinfo (skb)->tx_flags &SKBTX_HW_TSTAMP ) {@@ -2350,11 +2385,76 @@ static enum qman_cb_dqrr_result rx_error_dqrr(struct qman_portal *portal,return qman_cb_dqrr_consume; } +static int dpaa_xdp_xmit_frame (struct net_device *net_dev, +struct xdp_frame *xdpf) +{ +struct dpaa_priv *priv =netdev_priv (net_dev); +struct rtnl_link_stats64 *percpu_stats; +struct dpaa_percpu_priv *percpu_priv; +struct dpaa_eth_swbp *swbp;
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_error_dqrr(struct qman_portal *portal, · line 2404
Quoted 10 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +10
-0+struct netdev_queue *txq; +void *buff_start; +struct qm_fd fd; +dma_addr_t addr; +int err; + + percpu_priv =this_cpu_ptr (priv->percpu_priv); + percpu_stats = &percpu_priv->stats; + +if (xdpf->headroom <DPAA_TX_PRIV_DATA_SIZE ) {
Could you shed some light on DPAA_TX_PRIV_DATA_SIZE usage?
Quoted 23 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +23
-0+ err = -EINVAL ; + goto out_error; + } + + buff_start = xdpf->data - xdpf->headroom; + +/* Leave empty the skb backpointer at the start of the buffer. +* Save the XDP frame for easy cleanup on confirmation. +*/ + swbp = (struct dpaa_eth_swbp *)buff_start; + swbp->skb =NULL ; + swbp->xdpf = xdpf; + +qm_fd_clear_fd (&fd); + fd.bpid =FSL_DPAA_BPID_INV ; + fd.cmd |=cpu_to_be32 (FM_FD_CMD_FCO ); +qm_fd_set_contig (&fd, xdpf->headroom, xdpf->len); + + addr =dma_map_single (priv->tx_dma_dev, buff_start, + xdpf->headroom + xdpf->len, +DMA_TO_DEVICE ); +if (unlikely (dma_mapping_error (priv->tx_dma_dev, addr))) { + err = -EINVAL ;
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_error_dqrr(struct qman_portal *portal, · line 2437
Quoted 10 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +10
-0+ goto out_error; + } + +qm_fd_addr_set64 (&fd, addr); + +/* Bump the trans_start */ + txq =netdev_get_tx_queue (net_dev,smp_processor_id ()); + txq->trans_start = jiffies; + + err =dpaa_xmit (priv, percpu_stats,smp_processor_id (), &fd);
So it looks like you don't provide the XDP Tx resources and you share the netstack's Tx queues with XDP, if I'm reading this right.
Please mention it/explain in the cover letter or commit message of this patch. Furthermore, I don't see any locking happenning over here?
Quoted 35 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +18
-2+if (err) { +dma_unmap_single (priv->tx_dma_dev, addr, +qm_fd_get_offset (&fd) +qm_fd_get_length (&fd), +DMA_TO_DEVICE ); + goto out_error; + } + +return 0 ; + +out_error: + percpu_stats->tx_errors++; +return err; +} +static u32 dpaa_run_xdp (struct dpaa_priv *priv,struct qm_fd *fd,void *vaddr, -unsigned int *xdp_meta_len) +struct dpaa_fq *dpaa_fq,unsigned int *xdp_meta_len) {ssize_t fd_off =qm_fd_get_offset (fd);struct bpf_prog *xdp_prog; +struct xdp_frame *xdpf;struct xdp_buff xdp;u32 xdp_act;@@ -2370,7 +2470,8 @@ static u32 dpaa_run_xdp(struct dpaa_priv *priv, struct qm_fd *fd, void *vaddr, xdp.data_meta = xdp.data; xdp.data_hard_start = xdp.data -XDP_PACKET_HEADROOM ; xdp.data_end = xdp.data +qm_fd_get_length (fd); - xdp.frame_sz =DPAA_BP_RAW_SIZE ; + xdp.frame_sz =DPAA_BP_RAW_SIZE -DPAA_TX_PRIV_DATA_SIZE ; + xdp.rxq = &dpaa_fq->xdp_rxq; xdp_act =bpf_prog_run_xdp (xdp_prog, &xdp);
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static u32 dpaa_run_xdp(struct dpaa_priv *priv, struct qm_fd *fd, void *vaddr, · line 2487
Quoted 6 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +3
-0@@ -2381,6 +2482,22 @@ static u32 dpaa_run_xdp(struct dpaa_priv *priv, struct qm_fd *fd, void *vaddr,case XDP_PASS : *xdp_meta_len = xdp.data - xdp.data_meta;break ; +case XDP_TX : +/* We can access the full headroom when sending the frame +* back out
And normally why a piece of headroom is taken away? I probably should have started from the basic XDP support patch, but if you don't mind, please explain it a bit.
Quoted 68 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +17
-2+*/ + xdp.data_hard_start = vaddr; + xdp.frame_sz =DPAA_BP_RAW_SIZE ; + xdpf =xdp_convert_buff_to_frame (&xdp); +if (unlikely (!xdpf)) { +free_pages ((unsigned long )vaddr,0 ); +break ; + } + +if (dpaa_xdp_xmit_frame (priv->net_dev, xdpf)) +xdp_return_frame_rx_napi (xdpf); + +break ;default :bpf_warn_invalid_xdp_action (xdp_act); fallthrough;@@ -2415,6 +2532,7 @@ static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,u32 fd_status, hash_offset;struct qm_sg_entry *sgt;struct dpaa_bp *dpaa_bp; +struct dpaa_fq *dpaa_fq;struct dpaa_priv *priv;struct sk_buff *skb;int *count_ptr;@@ -2423,9 +2541,10 @@ static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,u32 hash;u64 ns; + dpaa_fq = container_of(fq, struct dpaa_fq, fq_base); fd_status =be32_to_cpu (fd->status); fd_format =qm_fd_get_format (fd); - net_dev = ((struct dpaa_fq *)fq)->net_dev; + net_dev = dpaa_fq->net_dev; priv =netdev_priv (net_dev); dpaa_bp =dpaa_bpid2pool (dq->fd.bpid);if (!dpaa_bp)@@ -2494,7 +2613,7 @@ static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,if (likely (fd_format == qm_fd_contig)) { xdp_act =dpaa_run_xdp (priv, (struct qm_fd *)fd, vaddr, - &xdp_meta_len); + dpaa_fq, &xdp_meta_len);if (xdp_act !=XDP_PASS ) { percpu_stats->rx_packets++; percpu_stats->rx_bytes +=qm_fd_get_length (fd);drivers/net/ethernet/freescale/dpaa/dpaa_eth.h +2
-0@@ -68,6 +68,7 @@ struct dpaa_fq {u16 channel;u8 wq;enum dpaa_fq_type fq_type; +struct xdp_rxq_info xdp_rxq; };struct dpaa_fq_cbs {@@ -150,6 +151,7 @@ struct dpaa_buffer_layout {*/ struct dpaa_eth_swbp {struct sk_buff *skb; +struct xdp_frame *xdpf; };struct dpaa_priv {Signature · 1 line
1.9.1
RE: [PATCH net-next v3 4/7] dpaa_eth: add XDP_TX support
Quoted 10 lines
-----Original Message-----
From: Maciej Fijalkowski <[email protected]>
Sent: Friday, November 20, 2020 01:51
To: Camelia Alexandra Groza <[email protected]>
Cc: [email protected]; [email protected]; [email protected]; [email protected]; Madalin Bucur (OSS) <[email protected]>; Ioana Ciornei <[email protected]>; [email protected]
Subject: Re: [PATCH net-next v3 4/7]dpaa_eth: addXDP_TXsupport
Quoted 8 lines
Quoted 1 line · Camelia Groza
Use an
xdp_framestructure for managing the frame. Store a backpointerto
Quoted 3 lines
the structure at the start of the buffer before enqueueing. Use the XDP API for freeing the buffer when it returns to the driver on the TX confirmation path.
Completion path?
The DPAA terminology uses confirmation instead of completion, but the meaning is similar. The dpaa.rst documentation has more details.
Quoted 21 lines
Quoted 7 lines
This approach will be reused for XDP REDIRECT.
Acked-by Madalin Bucur <[email protected]>Signed-off-by Camelia Groza <[email protected]>--- drivers/net/ethernet/freescale/dpaa/dpaa_eth.c | 129++++++++++++++++++++++++-
Review
Quoted 4 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.h | 2 + 2 files changed, 126 insertions(+), 5 deletions(-)(quoted diff) +0
-0
b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.cdrivers/net/ethernet/freescale/dpaa/dpaa_eth.c static int dpaa_fq_init(struct dpaa_fq *dpaa_fq,
Quoted 4 lines
index 242ed45..cd5f4f6 100644
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +0
-0@@ -1130,6 +1130,24 @@ static int dpaa_fq_init(struct dpaa_fq *dpaa_fq,bool
td_enable)Quoted 3 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +0
-0dpaa_fq->fqid =qman_fq_fqid (fq);
Quoted 9 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static int dpaa_fq_init(struct dpaa_fq *dpaa_fq, bool td_enable) · line 1138
Quoted 7 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +6
-0+if (dpaa_fq->fq_type ==FQ_TYPE_RX_DEFAULT || + dpaa_fq->fq_type ==FQ_TYPE_RX_PCD ) { + err =xdp_rxq_info_reg (&dpaa_fq->xdp_rxq, dpaa_fq- net_dev, + dpaa_fq->fqid); +if (err) { +dev_err (dev,"xdp_rxq_info_reg failed\n" );Print out the err?
I'll add the err in the next version.
Quoted 1 line
Also, shouldn't you call
qman_destroy_fq()for these error paths?
Yes, for the point of undoing all configurations on the queue. The entire probe is aborted if things fail at this point, so everything is cleaned either way. I'll fix this in the next version.
Quoted 19 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static int dpaa_fq_init(struct dpaa_fq *dpaa_fq, bool td_enable)
Quoted 5 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +5
-0+return err; + } + + err = xdp_rxq_info_reg_mem_model(&dpaa_fq->xdp_rxq, + MEM_TYPE_PAGE_ORDER0,NULL);
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static int dpaa_fq_init(struct dpaa_fq *dpaa_fq, · line 1145
Quoted 2 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +2
-0+if (err) { + dev_err(dev, "xdp_rxq_info_reg_mem_modelfailed\n");
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static int dpaa_fq_free_entry(struct device
Quoted 9 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +5
-0+xdp_rxq_info_unreg (&dpaa_fq->xdp_rxq); +return err; + } + } +return 0 ; }@@ -1159,6 +1177,10 @@ static int dpaa_fq_free_entry(struct device*dev, struct
qman_fq*fq)
Quoted 7 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static int dpaa_fq_free_entry(struct device *dev, struct qman_fq *fq) · line 1181
Quoted 5 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +2
-0} } +if (dpaa_fq->fq_type ==FQ_TYPE_RX_DEFAULT || + dpaa_fq->fq_type ==FQ_TYPE_RX_PCD )You should call
xdp_rxq_info_is_reg()before the unregister below.
I'll add it.
Quoted 19 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static int dpaa_eth_refill_bpools(struct dpaa_priv
Quoted 6 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +2
-0+xdp_rxq_info_unreg (&dpaa_fq->xdp_rxq); +qman_destroy_fq (fq);list_del (&dpaa_fq->list);@@ -1625,6 +1647,9 @@ static int dpaa_eth_refill_bpools(struct dpaa_priv*priv)
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static int dpaa_eth_refill_bpools(struct dpaa_priv *priv)
Quoted 5 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +2
-0* * Return the skb backpointer, since for S/G frames the buffer containing it * gets freed here. +* +* No skb backpointer is set when transmitting XDP frames. Cleanup the buffer
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static struct sk_buff *dpaa_cleanup_tx_fd(const
Quoted 5 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +1
-0+* and return NULL in this case. */ static struct sk_buff *dpaa_cleanup_tx_fd (const struct dpaa_priv *priv,const struct qm_fd *fd,bool ts)@@ -1636,6 +1661,7 @@ static struct sk_buff *dpaa_cleanup_tx_fd(conststruct
dpaa_priv*priv,
Quoted 6 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static struct sk_buff *dpaa_cleanup_tx_fd(const struct dpaa_priv *priv, · line 1664
Quoted 4 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +1
-0void *vaddr =phys_to_virt (addr);const struct qm_sg_entry *sgt;struct dpaa_eth_swbp *swbp; +struct xdp_frame *xdpf;This local variable feels a bit unnecessary.
Sure, I'll remove it.
Quoted 47 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static struct sk_buff
Quoted 4 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +0
-0struct sk_buff *skb;u64 ns;int i;@@ -1664,13 +1690,22 @@ static struct sk_buff*dpaa_cleanup_tx_fd(const struct
dpaa_priv*priv,drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static struct sk_buff *dpaa_cleanup_tx_fd(const struct dpaa_priv *priv,
Quoted 4 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +0
-1} }else { dma_unmap_single(priv->tx_dma_dev, addr, - priv->tx_headroom +
qm_fd_get_length(fd),drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static struct sk_buff · line 1693
Quoted 1 line
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +1
-0+ qm_fd_get_offset(fd) +
qm_fd_get_length(fd),Quoted 6 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +0
-0dma_dir); } swbp = (struct dpaa_eth_swbp *)vaddr; skb = swbp->skb;drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static struct sk_buff *dpaa_cleanup_tx_fd(const struct dpaa_priv *priv,
Quoted 10 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +9
-0+/* No skb backpointer is set when running XDP. An xdp_frame +* backpointer is saved instead. +*/ +if (!skb) { + xdpf = swbp->xdpf; +xdp_return_frame (xdpf); +return NULL ; + } +/* DMA unmapping is required before accessing the HW provided info */
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result
Quoted 3 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +0
-0if (ts && priv->tx_tstamp &&skb_shinfo (skb)->tx_flags &SKBTX_HW_TSTAMP ) {@@ -2350,11 +2385,76 @@ static enum qman_cb_dqrr_resultrx_error_dqrr(struct
qman_portal*portal,Quoted 14 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +11
-0return qman_cb_dqrr_consume; } +static int dpaa_xdp_xmit_frame (struct net_device *net_dev, +struct xdp_frame *xdpf) +{ +struct dpaa_priv *priv =netdev_priv (net_dev); +struct rtnl_link_stats64 *percpu_stats; +struct dpaa_percpu_priv *percpu_priv; +struct dpaa_eth_swbp *swbp; +struct netdev_queue *txq; +void *buff_start; +struct qm_fd fd; +dma_addr_t addr;
Quoted 8 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_error_dqrr(struct qman_portal *portal, · line 2404
Quoted 6 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +6
-0+int err; + + percpu_priv =this_cpu_ptr (priv->percpu_priv); + percpu_stats = &percpu_priv->stats; + +if (xdpf->headroom <DPAA_TX_PRIV_DATA_SIZE ) {Could you shed some light on
DPAA_TX_PRIV_DATA_SIZEusage?
We store information at the start of egress buffers that we use on the confirmation path for cleanup. DPAA_TX_PRIV_DATA_SIZE is the size of this reserved area. We require buffers to have at least this mush space available at the start before transmitting them.
Quoted 37 lines
Quoted 23 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +23
-0+ err = -EINVAL ; + goto out_error; + } + + buff_start = xdpf->data - xdpf->headroom; + +/* Leave empty the skb backpointer at the start of the buffer. +* Save the XDP frame for easy cleanup on confirmation. +*/ + swbp = (struct dpaa_eth_swbp *)buff_start; + swbp->skb =NULL ; + swbp->xdpf = xdpf; + +qm_fd_clear_fd (&fd); + fd.bpid =FSL_DPAA_BPID_INV ; + fd.cmd |=cpu_to_be32 (FM_FD_CMD_FCO ); +qm_fd_set_contig (&fd, xdpf->headroom, xdpf->len); + + addr =dma_map_single (priv->tx_dma_dev, buff_start, + xdpf->headroom + xdpf->len, +DMA_TO_DEVICE ); +if (unlikely (dma_mapping_error (priv->tx_dma_dev, addr))) { + err = -EINVAL ;drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_error_dqrr(struct qman_portal *portal, · line 2437
Quoted 10 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +10
-0+ goto out_error; + } + +qm_fd_addr_set64 (&fd, addr); + +/* Bump the trans_start */ + txq =netdev_get_tx_queue (net_dev,smp_processor_id ()); + txq->trans_start = jiffies; + + err =dpaa_xmit (priv, percpu_stats,smp_processor_id (), &fd);So it looks like you don't provide the XDP Tx resources and you share the netstack's Tx queues with XDP, if I'm reading this right.
Quoted 2 lines
Please mention it/explain in the cover letter or commit message of this patch. Furthermore, I don't see any locking happenning over here?
Yes, that's correct, we don't have dedicated Tx queues for XDP, we share them with the netstack.
We don't need locking at this level. We are a LLTX driver so we don't take the netif_tx_lock. I'll mention this in the cover letter.
Quoted 40 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_error_dqrr(struct qman_portal *portal,
Quoted 3 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +3
-0+if (err) { + dma_unmap_single(priv->tx_dma_dev, addr, + qm_fd_get_offset(&fd) +
qm_fd_get_length(&fd),drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result rx_error_dqrr(struct qman_portal *portal,
Quoted 12 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +11
-0+ DMA_TO_DEVICE); + goto out_error; + } + +return 0 ; + +out_error: + percpu_stats->tx_errors++; +return err; +} + static u32 dpaa_run_xdp(struct dpaa_priv *priv, struct qm_fd *fd, void*vaddr,
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result · line 2453
Quoted 2 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +1
-1- unsigned int *xdp_meta_len) + struct dpaa_fq *dpaa_fq, unsigned int
*xdp_meta_len)drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static u32 dpaa_run_xdp(struct dpaa_priv *priv,
Quoted 8 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +1
-0{ssize_t fd_off =qm_fd_get_offset (fd);struct bpf_prog *xdp_prog; +struct xdp_frame *xdpf;struct xdp_buff xdp;u32 xdp_act;@@ -2370,7 +2470,8 @@ static u32 dpaa_run_xdp(struct dpaa_priv *priv,struct
qm_fd*fd, void *vaddr,drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static u32 dpaa_run_xdp(struct dpaa_priv *priv,
Quoted 10 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +2
-1xdp.data_meta = xdp.data; xdp.data_hard_start = xdp.data -XDP_PACKET_HEADROOM ; xdp.data_end = xdp.data +qm_fd_get_length (fd); - xdp.frame_sz =DPAA_BP_RAW_SIZE ; + xdp.frame_sz =DPAA_BP_RAW_SIZE -DPAA_TX_PRIV_DATA_SIZE ; + xdp.rxq = &dpaa_fq->xdp_rxq; xdp_act =bpf_prog_run_xdp (xdp_prog, &xdp);@@ -2381,6 +2482,22 @@ static u32 dpaa_run_xdp(struct dpaa_priv *priv,struct
qm_fd*fd, void *vaddr,
Quoted 11 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static u32 dpaa_run_xdp(struct dpaa_priv *priv, struct qm_fd *fd, void *vaddr, · line 2487
Quoted 6 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +3
-0case XDP_PASS : *xdp_meta_len = xdp.data - xdp.data_meta;break ; +case XDP_TX : +/* We can access the full headroom when sending the frame +* back out And normally why a piece of headroom is taken away? I probably should have
started from the basic XDP support patch, but if you don't mind, please explain it a bit.
I mentioned we require DPAA_TX_PRIV_DATA_SIZE bytes at the start of the buffer in order to make sure we have enough space for our private info.
When setting up the xdp_buff, this area is reserved from the frame size exposed to the XDP program.
- xdp.frame_sz = DPAA_BP_RAW_SIZE;
+ xdp.frame_sz = DPAA_BP_RAW_SIZE - DPAA_TX_PRIV_DATA_SIZE;
After the XDP_TX verdict, we're sure that DPAA_TX_PRIV_DATA_SIZE bytes at the start of the buffer are free and we can use the full headroom how it suits us, hence the increase of the frame size back to DPAA_BP_RAW_SIZE.
Thanks for all your feedback.
Quoted 72 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result
Quoted 17 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +13
-0+*/ + xdp.data_hard_start = vaddr; + xdp.frame_sz =DPAA_BP_RAW_SIZE ; + xdpf =xdp_convert_buff_to_frame (&xdp); +if (unlikely (!xdpf)) { +free_pages ((unsigned long )vaddr,0 ); +break ; + } + +if (dpaa_xdp_xmit_frame (priv->net_dev, xdpf)) +xdp_return_frame_rx_napi (xdpf); + +break ;default :bpf_warn_invalid_xdp_action (xdp_act); fallthrough;@@ -2415,6 +2532,7 @@ static enum qman_cb_dqrr_resultrx_default_dqrr(struct
qman_portal*portal,drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result
Quoted 8 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +1
-0u32 fd_status, hash_offset;struct qm_sg_entry *sgt;struct dpaa_bp *dpaa_bp; +struct dpaa_fq *dpaa_fq;struct dpaa_priv *priv;struct sk_buff *skb;int *count_ptr;@@ -2423,9 +2541,10 @@ static enum qman_cb_dqrr_resultrx_default_dqrr(struct
qman_portal*portal,drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static enum qman_cb_dqrr_result
Quoted 12 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +2
-1u32 hash;u64 ns; + dpaa_fq = container_of(fq, struct dpaa_fq, fq_base); fd_status =be32_to_cpu (fd->status); fd_format =qm_fd_get_format (fd); - net_dev = ((struct dpaa_fq *)fq)->net_dev; + net_dev = dpaa_fq->net_dev; priv =netdev_priv (net_dev); dpaa_bp =dpaa_bpid2pool (dq->fd.bpid);if (!dpaa_bp)@@ -2494,7 +2613,7 @@ static enum qman_cb_dqrr_resultrx_default_dqrr(struct
qman_portal*portal,Review
Quoted 9 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +1
-1if (likely (fd_format == qm_fd_contig)) { xdp_act =dpaa_run_xdp (priv, (struct qm_fd *)fd, vaddr, - &xdp_meta_len); + dpaa_fq, &xdp_meta_len);if (xdp_act !=XDP_PASS ) { percpu_stats->rx_packets++; percpu_stats->rx_bytes +=qm_fd_get_length (fd);(quoted diff) +0
-0
b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.hQuoted 22 lines
index 94e8613..5c8d52a 100644
drivers/net/ethernet/freescale/dpaa/dpaa_eth.h +2
-1@@ -68,6 +68,7 @@ struct dpaa_fq {u16 channel;u8 wq;enum dpaa_fq_type fq_type; +struct xdp_rxq_info xdp_rxq; };struct dpaa_fq_cbs {@@ -150,6 +151,7 @@ struct dpaa_buffer_layout {*/ struct dpaa_eth_swbp {struct sk_buff *skb; +struct xdp_frame *xdpf; };struct dpaa_priv { --1.9.1
Re: [PATCH net-next v3 4/7] dpaa_eth: add XDP_TX support
Quoted 17 lines · Camelia Alexandra Groza
Quoted 17 lines
-----Original Message-----
From: Maciej Fijalkowski <[email protected]>
Sent: Friday, November 20, 2020 01:51
To: Camelia Alexandra Groza <[email protected]>
Cc: [email protected]; [email protected]; [email protected]; [email protected]; Madalin Bucur (OSS) <[email protected]>; Ioana Ciornei <[email protected]>; [email protected]
Subject: Re: [PATCH net-next v3 4/7]dpaa_eth: addXDP_TXsupportQuoted 1 line · Camelia Groza
Use an
xdp_framestructure for managing the frame. Store a backpointerto
Quoted 3 lines
the structure at the start of the buffer before enqueueing. Use the XDP API for freeing the buffer when it returns to the driver on the TX confirmation path.
[...]
Quoted 37 lines
Quoted 25 lines
Quoted 1 line
static u32 dpaa_run_xdp(struct
dpaa_priv*priv, structqm_fd*fd, void*vaddr,
Review
Quoted 2 lines
(quoted diff) +1
-1- unsigned int *xdp_meta_len) + struct dpaa_fq *dpaa_fq, unsigned int
*xdp_meta_len)drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static u32 dpaa_run_xdp(struct dpaa_priv *priv,
Quoted 8 lines
{
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +1
-0ssize_t fd_off =qm_fd_get_offset (fd);struct bpf_prog *xdp_prog; +struct xdp_frame *xdpf;struct xdp_buff xdp;u32 xdp_act;@@ -2370,7 +2470,8 @@ static u32 dpaa_run_xdp(struct dpaa_priv *priv,struct
qm_fd*fd, void *vaddr,drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static u32 dpaa_run_xdp(struct dpaa_priv *priv,
Quoted 10 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +2
-1xdp.data_meta = xdp.data; xdp.data_hard_start = xdp.data -XDP_PACKET_HEADROOM ; xdp.data_end = xdp.data +qm_fd_get_length (fd); - xdp.frame_sz =DPAA_BP_RAW_SIZE ; + xdp.frame_sz =DPAA_BP_RAW_SIZE -DPAA_TX_PRIV_DATA_SIZE ; + xdp.rxq = &dpaa_fq->xdp_rxq; xdp_act =bpf_prog_run_xdp (xdp_prog, &xdp);@@ -2381,6 +2482,22 @@ static u32 dpaa_run_xdp(struct dpaa_priv *priv,struct
qm_fd*fd, void *vaddr,Quoted 11 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static u32 dpaa_run_xdp(struct dpaa_priv *priv, struct qm_fd *fd, void *vaddr, · line 2487
Quoted 6 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +3
-0case XDP_PASS : *xdp_meta_len = xdp.data - xdp.data_meta;break ; +case XDP_TX : +/* We can access the full headroom when sending the frame +* back out And normally why a piece of headroom is taken away? I probably should have
started from the basic XDP support patch, but if you don't mind, please explain it a bit.
Quoted 1 line
I mentioned we require
DPAA_TX_PRIV_DATA_SIZEbytes at the start of the buffer in order to make sure we have enough space for our private info.
What is your private info?
Quoted 6 lines
When setting up the
xdp_buff, this area is reserved from the frame size exposed to the XDP program.
- xdp.frame_sz =DPAA_BP_RAW_SIZE;
+ xdp.frame_sz =DPAA_BP_RAW_SIZE-DPAA_TX_PRIV_DATA_SIZE;After the
XDP_TXverdict, we're sure thatDPAA_TX_PRIV_DATA_SIZEbytes at the start of the buffer are free and we can use the full headroom how it suits us, hence the increase of the frame size back toDPAA_BP_RAW_SIZE.
Not at the *end* of the buffer?
Quoted 20 lines
Thanks for all your feedback.
Quoted 17 lines
Quoted 17 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +13
-0+*/ + xdp.data_hard_start = vaddr; + xdp.frame_sz =DPAA_BP_RAW_SIZE ; + xdpf =xdp_convert_buff_to_frame (&xdp); +if (unlikely (!xdpf)) { +free_pages ((unsigned long )vaddr,0 ); +break ; + } + +if (dpaa_xdp_xmit_frame (priv->net_dev, xdpf)) +xdp_return_frame_rx_napi (xdpf); + +break ;default :bpf_warn_invalid_xdp_action (xdp_act); fallthrough;@@ -2415,6 +2532,7 @@ static enum qman_cb_dqrr_result
RE: [PATCH net-next v3 4/7] dpaa_eth: add XDP_TX support
Quoted 71 lines
-----Original Message-----
From: Maciej Fijalkowski <[email protected]>
Sent: Tuesday, November 24, 2020 00:18
To: Camelia Alexandra Groza <[email protected]>
Cc: [email protected]; [email protected]; [email protected]; [email protected]; Madalin Bucur (OSS) <[email protected]>; Ioana Ciornei <[email protected]>; [email protected]
Subject: Re: [PATCH net-next v3 4/7]dpaa_eth: addXDP_TXsupportQuoted 10 lines · Camelia Alexandra Groza
Quoted 10 lines
-----Original Message-----
From: Maciej Fijalkowski <[email protected]>
Sent: Friday, November 20, 2020 01:51
To: Camelia Alexandra Groza <[email protected]>
Cc: [email protected]; [email protected]; [email protected]; [email protected]; Madalin Bucur (OSS) <[email protected]>; Ioana Ciornei <[email protected]>; [email protected]
Subject: Re: [PATCH net-next v3 4/7]dpaa_eth: addXDP_TXsupportQuoted 2 lines
Quoted 2 lines · Camelia Groza
Quoted 1 line
Use an
xdp_framestructure for managing the frame. Store abackpointer
Quoted 5 lines
Quoted 5 lines
to
Quoted 3 lines
the structure at the start of the buffer before enqueueing. Use the XDP API for freeing the buffer when it returns to the driver on the TX confirmation path.
[...]
Quoted 5 lines
Quoted 5 lines
Quoted 1 line
static u32 dpaa_run_xdp(struct
dpaa_priv*priv, structqm_fd*fd, void*vaddr,
Review
Quoted 2 lines
(quoted diff) +1
-1- unsigned int *xdp_meta_len) + struct dpaa_fq *dpaa_fq, unsigned int
*xdp_meta_len)Quoted 8 lines
Quoted 8 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static u32 dpaa_run_xdp(struct dpaa_priv
Quoted 8 lines
{
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +1
-0ssize_t fd_off =qm_fd_get_offset (fd);struct bpf_prog *xdp_prog; +struct xdp_frame *xdpf;struct xdp_buff xdp;u32 xdp_act;@@ -2370,7 +2470,8 @@ static u32 dpaa_run_xdp(struct dpaa_priv*priv,
Quoted 11 lines
Quoted 11 lines
struct
qm_fd*fd, void *vaddr,drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static u32 dpaa_run_xdp(struct dpaa_priv
Quoted 10 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +2
-1xdp.data_meta = xdp.data; xdp.data_hard_start = xdp.data -XDP_PACKET_HEADROOM ; xdp.data_end = xdp.data +qm_fd_get_length (fd); - xdp.frame_sz =DPAA_BP_RAW_SIZE ; + xdp.frame_sz =DPAA_BP_RAW_SIZE -DPAA_TX_PRIV_DATA_SIZE ; + xdp.rxq = &dpaa_fq->xdp_rxq; xdp_act =bpf_prog_run_xdp (xdp_prog, &xdp);@@ -2381,6 +2482,22 @@ static u32 dpaa_run_xdp(struct dpaa_priv*priv,
Quoted 13 lines
Quoted 12 lines
struct
qm_fd*fd, void *vaddr,drivers/net/ethernet/freescale/dpaa/dpaa_eth.c static u32 dpaa_run_xdp(struct dpaa_priv *priv, struct qm_fd *fd, void *vaddr, · line 2487
Quoted 6 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +3
-0case XDP_PASS : *xdp_meta_len = xdp.data - xdp.data_meta;break ; +case XDP_TX : +/* We can access the full headroom when sending the frame +* back out And normally why a piece of headroom is taken away? I probably should have
started from the basic XDP support patch, but if you don't mind, please explain it a bit.
Quoted 4 lines
Quoted 1 line
I mentioned we require
DPAA_TX_PRIV_DATA_SIZEbytes at the start ofthe buffer in order to make sure we have enough space for our private info.
What is your private info?
The dpaa_eth_swbp struct from the first patch. It's the xdp_frame reference mentioned in the patch description, stored for cleanup on confirmation. We also store a skb reference for non-XDP use cases.
Quoted 6 lines
Quoted 2 lines
When setting up the
xdp_buff, this area is reserved from the frame sizeexposed to the XDP program.
Quoted 3 lines
- xdp.frame_sz =
DPAA_BP_RAW_SIZE;
+ xdp.frame_sz =DPAA_BP_RAW_SIZE-DPAA_TX_PRIV_DATA_SIZE;
Quoted 6 lines
Quoted 1 line
After the
XDP_TXverdict, we're sure thatDPAA_TX_PRIV_DATA_SIZEbytes at the start of the buffer are free and we can use the full headroom how it suits us, hence the increase of the frame size back to
DPAA_BP_RAW_SIZE.Not at the *end* of the buffer?
No, we store this information at the start of the buffer, before the XDP_PACKET_HEADROOM.
Quoted 20 lines
Quoted 20 lines
Thanks for all your feedback.
Quoted 17 lines
Quoted 17 lines
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +13
-0+*/ + xdp.data_hard_start = vaddr; + xdp.frame_sz =DPAA_BP_RAW_SIZE ; + xdpf =xdp_convert_buff_to_frame (&xdp); +if (unlikely (!xdpf)) { +free_pages ((unsigned long )vaddr,0 ); +break ; + } + +if (dpaa_xdp_xmit_frame (priv->net_dev, xdpf)) +xdp_return_frame_rx_napi (xdpf); + +break ;default :bpf_warn_invalid_xdp_action (xdp_act); fallthrough;@@ -2415,6 +2532,7 @@ static enum qman_cb_dqrr_result
PATCH v3 5/7 dpaa_eth: add XDP_REDIRECT support
In reply to Camelia GrozaAfter transmission, the frame is returned on confirmation queues for cleanup. For this, store a backpointer to the xdp_frame in the private reserved area at the start of the TX buffer.
No TX batching support is implemented at this time.
--- drivers/net/ethernet/freescale/dpaa/dpaa_eth.c | 48 +++++++++++++++++++++++++- drivers/net/ethernet/freescale/dpaa/dpaa_eth.h | 1 + 2 files changed, 48 insertions(+), 1 deletion(-)
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +47 -1
@@ -2305,8 +2305,11 @@ static int dpaa_eth_poll(struct napi_struct *napi, int budget) {struct dpaa_napi_portal *np = container_of(napi, struct dpaa_napi_portal, napi); +int cleaned; -int cleaned =qman_p_poll_dqrr (np->p, budget); + np->xdp_act =0 ; + + cleaned =qman_p_poll_dqrr (np->p, budget);if (cleaned < budget) {napi_complete_done (napi, cleaned);
@@ -2315,6 +2318,9 @@ static int dpaa_eth_poll(struct napi_struct *napi, int budget)qman_p_irqsource_add (np->p,QM_PIRQ_DQRI ); } +if (np->xdp_act &XDP_REDIRECT ) +xdp_do_flush (); +return cleaned; }
@@ -2457,6 +2463,7 @@ static u32 dpaa_run_xdp(struct dpaa_priv *priv, struct qm_fd *fd, void *vaddr,struct xdp_frame *xdpf;struct xdp_buff xdp;u32 xdp_act; +int err;rcu_read_lock ();
@@ -2498,6 +2505,17 @@ static u32 dpaa_run_xdp(struct dpaa_priv *priv, struct qm_fd *fd, void *vaddr,xdp_return_frame_rx_napi (xdpf);break ; +case XDP_REDIRECT : +/* Allow redirect to use the full headroom */ + xdp.data_hard_start = vaddr; + xdp.frame_sz =DPAA_BP_RAW_SIZE ; + + err =xdp_do_redirect (priv->net_dev, &xdp, xdp_prog); +if (err) { +trace_xdp_exception (priv->net_dev, xdp_prog, xdp_act); +free_pages ((unsigned long )vaddr,0 ); + } +break ;default :bpf_warn_invalid_xdp_action (xdp_act); fallthrough;
@@ -2527,6 +2545,7 @@ static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,struct dpaa_percpu_priv *percpu_priv;const struct qm_fd *fd = &dq->fd;dma_addr_t addr =qm_fd_addr (fd); +struct dpaa_napi_portal *np;enum qm_fd_format fd_format;struct net_device *net_dev;u32 fd_status, hash_offset;
@@ -2541,6 +2560,7 @@ static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,u32 hash;u64 ns; + np = container_of(&portal, struct dpaa_napi_portal, p); dpaa_fq = container_of(fq, struct dpaa_fq, fq_base); fd_status =be32_to_cpu (fd->status); fd_format =qm_fd_get_format (fd);
@@ -2614,6 +2634,7 @@ static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,if (likely (fd_format == qm_fd_contig)) { xdp_act =dpaa_run_xdp (priv, (struct qm_fd *)fd, vaddr, dpaa_fq, &xdp_meta_len); + np->xdp_act |= xdp_act;if (xdp_act !=XDP_PASS ) { percpu_stats->rx_packets++; percpu_stats->rx_bytes +=qm_fd_get_length (fd);
@@ -2942,6 +2963,30 @@ static int dpaa_xdp(struct net_device *net_dev, struct netdev_bpf *xdp) } } +static int dpaa_xdp_xmit (struct net_device *net_dev,int n, +struct xdp_frame **frames,u32 flags) +{ +struct xdp_frame *xdpf; +int i, err, drops =0 ; + +if (unlikely (flags & ~XDP_XMIT_FLAGS_MASK )) +return -EINVAL ; + +if (!netif_running (net_dev)) +return -ENETDOWN ; + +for (i =0 ; i < n; i++) { + xdpf = frames[i]; + err =dpaa_xdp_xmit_frame (net_dev, xdpf); +if (err) { +xdp_return_frame_rx_napi (xdpf); + drops++; + } + } + +return n - drops; +} +static int dpaa_ts_ioctl (struct net_device *dev,struct ifreq *rq,int cmd) {struct dpaa_priv *priv =netdev_priv (dev);
@@ -3010,6 +3055,7 @@ static int dpaa_ioctl(struct net_device *net_dev, struct ifreq *rq, int cmd) .ndo_setup_tc = dpaa_setup_tc, .ndo_change_mtu = dpaa_change_mtu, .ndo_bpf = dpaa_xdp, + .ndo_xdp_xmit = dpaa_xdp_xmit, };static int dpaa_napi_add (struct net_device *net_dev)
drivers/net/ethernet/freescale/dpaa/dpaa_eth.h +1 -0
@@ -127,6 +127,7 @@ struct dpaa_napi_portal {struct napi_struct napi;struct qman_portal *p;bool down; +int xdp_act; };struct dpaa_percpu_priv {
Signature · 1 line
1.9.1
PATCH v3 6/7 dpaa_eth: rename current skb A050385 erratum workaround
In reply to Camelia GrozaExplicitly point that the current workaround addresses skbs. This change is in preparation for adding a workaround for XDP scenarios.
--- drivers/net/ethernet/freescale/dpaa/dpaa_eth.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +2 -2
@@ -2105,7 +2105,7 @@ static inline int dpaa_xmit(struct dpaa_priv *priv, }#ifdef CONFIG_DPAA_ERRATUM_A050385 -static int dpaa_a050385_wa (struct net_device *net_dev,struct sk_buff **s) +static int dpaa_a050385_wa_skb (struct net_device *net_dev,struct sk_buff **s) {struct dpaa_priv *priv =netdev_priv (net_dev);struct sk_buff *new_skb, *skb = *s;
@@ -2221,7 +2221,7 @@ static int dpaa_a050385_wa(struct net_device *net_dev, struct sk_buff **s)#ifdef CONFIG_DPAA_ERRATUM_A050385 if (unlikely (fman_has_errata_a050385 ())) { -if (dpaa_a050385_wa (net_dev, &skb)) +if (dpaa_a050385_wa_skb (net_dev, &skb)) goto enomem; nonlinear =skb_is_nonlinear (skb); }
Signature · 1 line
1.9.1
PATCH v3 7/7 dpaa_eth: implement the A050385 erratum workaround for XDP
In reply to Camelia GrozaFor XDP TX, even tough we start out with correctly aligned buffers, the XDP program might change the data's alignment. For REDIRECT, we have no control over the alignment either.
Create a new workaround for xdp_frame structures to verify the erratum conditions and move the data to a fresh buffer if necessary. Create a new xdp_frame for managing the new buffer and free the old one using the XDP API.
Due to alignment constraints, all frames have a 256 byte headroom that is offered fully to XDP under the erratum. If the XDP program uses all of it, the data needs to be move to make room for the xdpf backpointer.
Disable the metadata support since the information can be lost.
--- drivers/net/ethernet/freescale/dpaa/dpaa_eth.c | 82 +++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 3 deletions(-)
drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +79 -3
@@ -2171,6 +2171,52 @@ static int dpaa_a050385_wa_skb(struct net_device *net_dev, struct sk_buff **s)return 0 ; } + +static int dpaa_a050385_wa_xdpf (struct dpaa_priv *priv, +struct xdp_frame **init_xdpf) +{ +struct xdp_frame *new_xdpf, *xdpf = *init_xdpf; +void *new_buff; +struct page *p; + +/* Check the data alignment and make sure the headroom is large +* enough to store the xdpf backpointer. Use an aligned headroom +* value. +* +* Due to alignment constraints, we give XDP access to the full 256 +* byte frame headroom. If the XDP program uses all of it, copy the +* data to a new buffer and make room for storing the backpointer. +*/ +if (PTR_IS_ALIGNED (xdpf->data,DPAA_A050385_ALIGN ) && + xdpf->headroom >= priv->tx_headroom) { + xdpf->headroom = priv->tx_headroom; +return 0 ; + } + + p =dev_alloc_pages (0 ); +if (unlikely (!p)) +return -ENOMEM ; + +/* Copy the data to the new buffer at a properly aligned offset */ + new_buff =page_address (p); +memcpy (new_buff + priv->tx_headroom, xdpf->data, xdpf->len); + +/* Create an XDP frame around the new buffer in a similar fashion +* to xdp_convert_buff_to_frame. +*/ + new_xdpf = new_buff; + new_xdpf->data = new_buff + priv->tx_headroom; + new_xdpf->len = xdpf->len; + new_xdpf->headroom = priv->tx_headroom; + new_xdpf->frame_sz =DPAA_BP_RAW_SIZE ; + new_xdpf->mem.type =MEM_TYPE_PAGE_ORDER0 ; + +/* Release the initial buffer */ +xdp_return_frame_rx_napi (xdpf); + + *init_xdpf = new_xdpf; +return 0 ; +}#endif static netdev_tx_t
@@ -2407,6 +2453,15 @@ static int dpaa_xdp_xmit_frame(struct net_device *net_dev, percpu_priv =this_cpu_ptr (priv->percpu_priv); percpu_stats = &percpu_priv->stats; +#ifdef CONFIG_DPAA_ERRATUM_A050385 +if (unlikely (fman_has_errata_a050385 ())) { +if (dpaa_a050385_wa_xdpf (priv, &xdpf)) { + err = -ENOMEM ; + goto out_error; + } + } +#endif +if (xdpf->headroom <DPAA_TX_PRIV_DATA_SIZE ) { err = -EINVAL ; goto out_error;
@@ -2480,6 +2535,20 @@ static u32 dpaa_run_xdp(struct dpaa_priv *priv, struct qm_fd *fd, void *vaddr, xdp.frame_sz =DPAA_BP_RAW_SIZE -DPAA_TX_PRIV_DATA_SIZE ; xdp.rxq = &dpaa_fq->xdp_rxq; +/* We reserve a fixed headroom of 256 bytes under the erratum and we +* offer it all to XDP programs to use. If no room is left for the +* xdpf backpointer on TX, we will need to copy the data. +* Disable metadata support since data realignments might be required +* and the information can be lost. +*/ +#ifdef CONFIG_DPAA_ERRATUM_A050385 +if (unlikely (fman_has_errata_a050385 ())) { +xdp_set_data_meta_invalid (&xdp); + xdp.data_hard_start = vaddr; + xdp.frame_sz =DPAA_BP_RAW_SIZE ; + } +#endif + xdp_act =bpf_prog_run_xdp (xdp_prog, &xdp);/* Update the length and the offset of the FD */
@@ -2487,7 +2556,8 @@ static u32 dpaa_run_xdp(struct dpaa_priv *priv, struct qm_fd *fd, void *vaddr,switch (xdp_act) {case XDP_PASS : - *xdp_meta_len = xdp.data - xdp.data_meta; + *xdp_meta_len =xdp_data_meta_unsupported (&xdp) ?0 : + xdp.data - xdp.data_meta;break ;case XDP_TX :/* We can access the full headroom when sending the frame
@@ -3187,10 +3257,16 @@ static u16 dpaa_get_headroom(struct dpaa_buffer_layout *bl,*/ headroom = (u16)(bl[port].priv_data_size +DPAA_HWA_SIZE ); -if (port ==RX ) +if (port ==RX ) { +#ifdef CONFIG_DPAA_ERRATUM_A050385 +if (unlikely (fman_has_errata_a050385 ())) + headroom =XDP_PACKET_HEADROOM ; +#endif +return ALIGN (headroom,DPAA_FD_RX_DATA_ALIGNMENT ); -else + }else {return ALIGN (headroom,DPAA_FD_DATA_ALIGNMENT ); + } }static int dpaa_eth_probe (struct platform_device *pdev)
Signature · 1 line
1.9.1