gve: XDP TX and redirect support for DQ RDA
Subsystems (4)
Harshitha Ramamurthy
maintainercover gve: XDP TX and redirect support for DQ RDA
From: Joshua Washington <[email protected]>
A previous patch series[1] introduced the ability to process XDP buffers to the DQ RDA queue format. This is a follow-up patch series to introduce XDP_TX and XDP_REDIRECT support and expose XDP support to the kernel.
Joshua Washington (3):
gve: rename gve_xdp_xmit to gve_xdp_xmit_gqi
gve: refactor DQO TX methods to be more generic for XDP
gve: add XDP_TX and XDP_REDIRECT support for DQ RDA
drivers/net/ethernet/google/gve/gve.h | 27 ++- drivers/net/ethernet/google/gve/gve_dqo.h | 2 + drivers/net/ethernet/google/gve/gve_main.c | 42 +++- drivers/net/ethernet/google/gve/gve_rx_dqo.c | 77 +++++- drivers/net/ethernet/google/gve/gve_tx.c | 4 +- drivers/net/ethernet/google/gve/gve_tx_dqo.c | 236 +++++++++++++++---- 6 files changed, 314 insertions(+), 74 deletions(-)
Signature · 1 line
2.49.0.1101.gccaa498523-goog
Harshitha Ramamurthy
maintainerPATCH 1/3 gve: rename gve_xdp_xmit to gve_xdp_xmit_gqi
From: Joshua Washington <[email protected]>
In preparation for XDP DQ support, the gve_xdp_xmit callback needs to be generalized for all queue formats. This patch renames the GQ-specific function to gve_xdp_xmit_gqi, and introduces a new gve_xdp_xmit callback which branches on queue format.
--- drivers/net/ethernet/google/gve/gve.h | 4 ++-- drivers/net/ethernet/google/gve/gve_main.c | 10 ++++++++++ drivers/net/ethernet/google/gve/gve_tx.c | 4 ++-- 3 files changed, 14 insertions(+), 4 deletions(-)
drivers/net/ethernet/google/gve/gve.h +2 -2
@@ -1178,8 +1178,8 @@ void gve_free_queue_page_list(struct gve_priv *priv, u32 id);/* tx handling */ netdev_tx_t gve_tx (struct sk_buff *skb,struct net_device *dev); -int gve_xdp_xmit (struct net_device *dev,int n,struct xdp_frame **frames, -u32 flags); +int gve_xdp_xmit_gqi (struct net_device *dev,int n,struct xdp_frame **frames, +u32 flags);int gve_xdp_xmit_one (struct gve_priv *priv,struct gve_tx_ring *tx,void *data,int len,void *frame_p);void gve_xdp_tx_flush (struct gve_priv *priv,u32 xdp_qid);
drivers/net/ethernet/google/gve/gve_main.c +10 -0
@@ -1516,6 +1516,16 @@ static int gve_set_xdp(struct gve_priv *priv, struct bpf_prog *prog,return err; } +static int gve_xdp_xmit (struct net_device *dev,int n, +struct xdp_frame **frames,u32 flags) +{ +struct gve_priv *priv =netdev_priv (dev); + +if (gve_is_gqi (priv)) +return gve_xdp_xmit_gqi (dev, n, frames, flags); +return -EOPNOTSUPP ; +} +static int gve_xsk_pool_enable (struct net_device *dev,struct xsk_buff_pool *pool,u16 qid)
drivers/net/ethernet/google/gve/gve_tx.c +2 -2
@@ -823,8 +823,8 @@ static int gve_tx_fill_xdp(struct gve_priv *priv, struct gve_tx_ring *tx,return ndescs; } -int gve_xdp_xmit (struct net_device *dev,int n,struct xdp_frame **frames, -u32 flags) +int gve_xdp_xmit_gqi (struct net_device *dev,int n,struct xdp_frame **frames, +u32 flags) {struct gve_priv *priv =netdev_priv (dev);struct gve_tx_ring *tx;
Signature · 1 line
2.50.0.rc2.761.g2dc52ea45b-goog
Harshitha Ramamurthy
maintainerPATCH 2/3 gve: refactor DQO TX methods to be more generic for XDP
In reply to Harshitha RamamurthyFrom: Joshua Washington <[email protected]>
This patch performs various minor DQO TX datapath refactors in preparation for adding XDP_TX and XDP_REDIRECT support. The following refactors are performed:
1) gve_tx_fill_pkt_desc_dqo() relies on a SKB pointer to get whether checksum offloading should be enabled. This won't work for the XDP case, which does not have a SKB. This patch updates the method to use a boolean representing whether checksum offloading should be enabled directly.
2) gve_maybe_stop_dqo() contains some synchronization between the true TX head and the cached value, a synchronization which is common for XDP queues and normal netdev queues. However, that method is reserved for netdev TX queues. To avoid duplicate code, this logic is factored out into a new method, gve_has_tx_slots_available().
3) gve_tx_update_tail() is added to update the TX tail, a functionality that will be common between normal TX and XDP TX codepaths.
--- drivers/net/ethernet/google/gve/gve_tx_dqo.c | 85 +++++++++++--------- 1 file changed, 47 insertions(+), 38 deletions(-)
drivers/net/ethernet/google/gve/gve_tx_dqo.c +47 -38
@@ -439,12 +439,28 @@ static u32 num_avail_tx_slots(const struct gve_tx_ring *tx)return tx->mask - num_used; } +/* Checks if the requested number of slots are available in the ring */ +static bool gve_has_tx_slots_available (struct gve_tx_ring *tx,u32 slots_req) +{ +u32 num_avail =num_avail_tx_slots (tx); + + slots_req +=GVE_TX_MIN_DESC_PREVENT_CACHE_OVERLAP ; + +if (num_avail >= slots_req) +return true; + +/* Update cached TX head pointer */ + tx->dqo_tx.head =atomic_read_acquire (&tx->dqo_compl.hw_tx_head); + +return num_avail_tx_slots (tx) >= slots_req; +} +static bool gve_has_avail_slots_tx_dqo (struct gve_tx_ring *tx,int desc_count,int buf_count) {return gve_has_pending_packet (tx) && -num_avail_tx_slots (tx) >= desc_count && -gve_has_free_tx_qpl_bufs (tx, buf_count); +gve_has_tx_slots_available (tx, desc_count) && +gve_has_free_tx_qpl_bufs (tx, buf_count); }/* Stops the queue if available descriptors is less than 'count'.
@@ -453,12 +469,6 @@ static bool gve_has_avail_slots_tx_dqo(struct gve_tx_ring *tx,static int gve_maybe_stop_tx_dqo (struct gve_tx_ring *tx,int desc_count,int buf_count) { -if (likely (gve_has_avail_slots_tx_dqo (tx, desc_count, buf_count))) -return 0 ; - -/* Update cached TX head pointer */ - tx->dqo_tx.head =atomic_read_acquire (&tx->dqo_compl.hw_tx_head); -if (likely (gve_has_avail_slots_tx_dqo (tx, desc_count, buf_count)))return 0 ;
@@ -472,8 +482,6 @@ static int gve_maybe_stop_tx_dqo(struct gve_tx_ring *tx,/* After stopping queue, check if we can transmit again in order to * avoid TOCTOU bug. */ - tx->dqo_tx.head =atomic_read_acquire (&tx->dqo_compl.hw_tx_head); -if (likely (!gve_has_avail_slots_tx_dqo (tx, desc_count, buf_count)))return -EBUSY ;
@@ -500,11 +508,9 @@ static void gve_extract_tx_metadata_dqo(const struct sk_buff *skb, }static void gve_tx_fill_pkt_desc_dqo (struct gve_tx_ring *tx,u32 *desc_idx, -struct sk_buff *skb,u32 len,u64 addr, +bool enable_csum,u32 len,u64 addr,s16 compl_tag,bool eop,bool is_gso) { -const bool checksum_offload_en = skb->ip_summed ==CHECKSUM_PARTIAL ; -while (len >0 ) {struct gve_tx_pkt_desc_dqo *desc = &tx->dqo.tx_ring[*desc_idx].pkt;
@@ -515,7 +521,7 @@ static void gve_tx_fill_pkt_desc_dqo(struct gve_tx_ring *tx, u32 *desc_idx, .buf_addr = cpu_to_le64(addr), .dtype = GVE_TX_PKT_DESC_DTYPE_DQO, .end_of_packet = cur_eop, - .checksum_offload_enable = checksum_offload_en, + .checksum_offload_enable = enable_csum, .compl_tag = cpu_to_le16(compl_tag), .buf_size = cur_len, };
@@ -612,6 +618,25 @@ gve_tx_fill_general_ctx_desc(struct gve_tx_general_context_desc_dqo *desc, }; } +static void gve_tx_update_tail (struct gve_tx_ring *tx,u32 desc_idx) +{ +u32 last_desc_idx = (desc_idx -1 ) & tx->mask; +u32 last_report_event_interval = + (last_desc_idx - tx->dqo_tx.last_re_idx) & tx->mask; + +/* Commit the changes to our state */ + tx->dqo_tx.tail = desc_idx; + +/* Request a descriptor completion on the last descriptor of the +* packet if we are allowed to by the HW enforced interval. +*/ + +if (unlikely (last_report_event_interval >=GVE_TX_MIN_RE_INTERVAL )) { + tx->dqo.tx_ring[last_desc_idx].pkt.report_event = true; + tx->dqo_tx.last_re_idx = last_desc_idx; + } +} + static int gve_tx_add_skb_no_copy_dqo(struct gve_tx_ring *tx, struct sk_buff *skb, struct gve_tx_pending_packet_dqo *pkt,
@@ -619,6 +644,7 @@ static int gve_tx_add_skb_no_copy_dqo(struct gve_tx_ring *tx,u32 *desc_idx, bool is_gso) { + bool enable_csum = skb->ip_summed == CHECKSUM_PARTIAL;const struct skb_shared_info *shinfo =skb_shinfo (skb);int i;
@@ -644,7 +670,7 @@ static int gve_tx_add_skb_no_copy_dqo(struct gve_tx_ring *tx,dma_unmap_addr_set (pkt, dma[pkt->num_bufs], addr); ++pkt->num_bufs; -gve_tx_fill_pkt_desc_dqo (tx, desc_idx, skb, len, addr, +gve_tx_fill_pkt_desc_dqo (tx, desc_idx, enable_csum, len, addr, completion_tag,/*eop=*/ shinfo->nr_frags ==0 , is_gso); }
@@ -664,7 +690,7 @@ static int gve_tx_add_skb_no_copy_dqo(struct gve_tx_ring *tx, dma[pkt->num_bufs], addr); ++pkt->num_bufs; -gve_tx_fill_pkt_desc_dqo (tx, desc_idx, skb, len, addr, +gve_tx_fill_pkt_desc_dqo (tx, desc_idx, enable_csum, len, addr, completion_tag, is_eop, is_gso); }
@@ -709,6 +735,7 @@ static int gve_tx_add_skb_copy_dqo(struct gve_tx_ring *tx,u32 *desc_idx, bool is_gso) { + bool enable_csum = skb->ip_summed == CHECKSUM_PARTIAL;u32 copy_offset =0 ;dma_addr_t dma_addr;u32 copy_len;
@@ -730,7 +757,7 @@ static int gve_tx_add_skb_copy_dqo(struct gve_tx_ring *tx, copy_offset += copy_len;dma_sync_single_for_device (tx->dev, dma_addr, copy_len,DMA_TO_DEVICE ); - gve_tx_fill_pkt_desc_dqo(tx, desc_idx, skb, + gve_tx_fill_pkt_desc_dqo(tx, desc_idx, enable_csum, copy_len, dma_addr, completion_tag,
@@ -800,24 +827,7 @@ static int gve_tx_add_skb_dqo(struct gve_tx_ring *tx, tx->dqo_tx.posted_packet_desc_cnt += pkt->num_bufs; -/* Commit the changes to our state */ - tx->dqo_tx.tail = desc_idx; - -/* Request a descriptor completion on the last descriptor of the -* packet if we are allowed to by the HW enforced interval. -*/ - { -u32 last_desc_idx = (desc_idx -1 ) & tx->mask; -u32 last_report_event_interval = - (last_desc_idx - tx->dqo_tx.last_re_idx) & tx->mask; - -if (unlikely (last_report_event_interval >= -GVE_TX_MIN_RE_INTERVAL )) { - tx->dqo.tx_ring[last_desc_idx].pkt.report_event = true; - tx->dqo_tx.last_re_idx = last_desc_idx; - } - } - +gve_tx_update_tail (tx, desc_idx);return 0 ; err:
@@ -951,9 +961,8 @@ static int gve_try_tx_skb(struct gve_priv *priv, struct gve_tx_ring *tx,/* Metadata + (optional TSO) + data descriptors. */ total_num_descs =1 +skb_is_gso (skb) + num_buffer_descs; -if (unlikely (gve_maybe_stop_tx_dqo (tx, total_num_descs + -GVE_TX_MIN_DESC_PREVENT_CACHE_OVERLAP , - num_buffer_descs))) { +if (unlikely (gve_maybe_stop_tx_dqo (tx, total_num_descs, + num_buffer_descs))) {return -1 ; }
Signature · 1 line
2.50.0.rc2.761.g2dc52ea45b-goog
Harshitha Ramamurthy
maintainerPATCH 3/3 gve: add XDP_TX and XDP_REDIRECT support for DQ RDA
In reply to Harshitha RamamurthyFrom: Joshua Washington <[email protected]>
This patch adds support for XDP_TX and XDP_REDIRECT for the DQ RDA queue format. To appropriately support transmission of XDP frames, a new pending packet type GVE_TX_PENDING_PACKET_DQO_XDP_FRAME is introduced for completion handling, as there was a previous assumption that completed packets would be SKBs.
XDP_TX handling completes the basic XDP actions, so the feature is recorded accordingly. This patch also enables the ndo_xdp_xmit callback allowing DQ to handle XDP_REDIRECT packets originating from another interface.
The XDP spinlock is moved to common TX ring fields so that it can be used in both GQ and DQ. Originally, it was in a section which was mutually exclusive for GQ and DQ.
In summary, 3 XDP features are exposed for the DQ RDA queue format:
1) NETDEV_XDP_ACT_BASIC
2) NETDEV_XDP_ACT_NDO_XMIT
3) NETDEV_XDP_ACT_REDIRECT
Note that XDP and header-data split are mutually exclusive for the time being due to lack of multi-buffer XDP support.
This patch does not add support for the DQ QPL format. That is to come in a future patch series.
--- drivers/net/ethernet/google/gve/gve.h | 23 ++- drivers/net/ethernet/google/gve/gve_dqo.h | 2 + drivers/net/ethernet/google/gve/gve_main.c | 34 ++++- drivers/net/ethernet/google/gve/gve_rx_dqo.c | 77 ++++++++-- drivers/net/ethernet/google/gve/gve_tx_dqo.c | 151 +++++++++++++++++-- 5 files changed, 254 insertions(+), 33 deletions(-)
drivers/net/ethernet/google/gve/gve.h +21 -2
@@ -402,8 +402,16 @@ enum gve_packet_state { GVE_PACKET_STATE_TIMED_OUT_COMPL, }; +enum gve_tx_pending_packet_dqo_type { +GVE_TX_PENDING_PACKET_DQO_SKB , +GVE_TX_PENDING_PACKET_DQO_XDP_FRAME +}; +struct gve_tx_pending_packet_dqo { -struct sk_buff *skb;/* skb for this packet */ +union { +struct sk_buff *skb; +struct xdp_frame *xdpf; + };/* 0th element corresponds to the linear portion of `skb`, should be * unmapped with `dma_unmap_single`.
@@ -433,7 +441,10 @@ struct gve_tx_pending_packet_dqo {/* Identifies the current state of the packet as defined in * `enum gve_packet_state`. */ -u8 state; + u8 state : 2; + +/* gve_tx_pending_packet_dqo_type */ + u8 type : 1;/* If packet is an outstanding miss completion, then the packet is * freed if the corresponding re-injection completion is not received
@@ -455,6 +466,9 @@ struct gve_tx_ring {/* DQO fields. */ struct { +/* Spinlock for XDP tx traffic */ +spinlock_t xdp_lock; +/* Linked list of gve_tx_pending_packet_dqo. Index into * pending_packets, or -1 if empty. *
@@ -1155,6 +1169,7 @@ static inline bool gve_supports_xdp_xmit(struct gve_priv *priv) {switch (priv->queue_format) {case GVE_GQI_QPL_FORMAT : +case GVE_DQO_RDA_FORMAT :return true;default :return false;
@@ -1180,9 +1195,13 @@ void gve_free_queue_page_list(struct gve_priv *priv,netdev_tx_t gve_tx (struct sk_buff *skb,struct net_device *dev);int gve_xdp_xmit_gqi (struct net_device *dev,int n,struct xdp_frame **frames,u32 flags); +int gve_xdp_xmit_dqo (struct net_device *dev,int n,struct xdp_frame **frames, +u32 flags);int gve_xdp_xmit_one (struct gve_priv *priv,struct gve_tx_ring *tx,void *data,int len,void *frame_p);void gve_xdp_tx_flush (struct gve_priv *priv,u32 xdp_qid); +int gve_xdp_xmit_one_dqo (struct gve_priv *priv,struct gve_tx_ring *tx, +struct xdp_frame *xdpf);bool gve_tx_poll (struct gve_notify_block *block,int budget);bool gve_xdp_poll (struct gve_notify_block *block,int budget);int gve_xsk_tx_poll (struct gve_notify_block *block,int budget);
drivers/net/ethernet/google/gve/gve_dqo.h +2 -0
@@ -37,6 +37,7 @@ netdev_features_t gve_features_check_dqo(struct sk_buff *skb,struct net_device *dev, netdev_features_t features);bool gve_tx_poll_dqo (struct gve_notify_block *block,bool do_clean); +bool gve_xdp_poll_dqo (struct gve_notify_block *block);int gve_rx_poll_dqo (struct gve_notify_block *block,int budget);int gve_tx_alloc_rings_dqo (struct gve_priv *priv,struct gve_tx_alloc_rings_cfg *cfg);
@@ -60,6 +61,7 @@ int gve_clean_tx_done_dqo(struct gve_priv *priv, struct gve_tx_ring *tx, struct napi_struct *napi);void gve_rx_post_buffers_dqo (struct gve_rx_ring *rx);void gve_rx_write_doorbell_dqo (const struct gve_priv *priv,int queue_idx); +void gve_xdp_tx_flush_dqo (struct gve_priv *priv,u32 xdp_qid); static inline void gve_tx_put_doorbell_dqo(const struct gve_priv *priv,
drivers/net/ethernet/google/gve/gve_main.c +26 -8
@@ -414,8 +414,12 @@ int gve_napi_poll_dqo(struct napi_struct *napi, int budget)bool reschedule = false;int work_done =0 ; -if (block->tx) - reschedule |=gve_tx_poll_dqo (block,/*do_clean=*/ true); +if (block->tx) { +if (block->tx->q_num < priv->tx_cfg.num_queues) + reschedule |=gve_tx_poll_dqo (block,/*do_clean=*/ true); +else + reschedule |=gve_xdp_poll_dqo (block); + }if (!budget)return 0 ;
@@ -1521,8 +1525,11 @@ static int gve_xdp_xmit(struct net_device *dev, int n, {struct gve_priv *priv =netdev_priv (dev); -if (gve_is_gqi (priv)) +if (priv->queue_format ==GVE_GQI_QPL_FORMAT )return gve_xdp_xmit_gqi (dev, n, frames, flags); +else if (priv->queue_format ==GVE_DQO_RDA_FORMAT ) +return gve_xdp_xmit_dqo (dev, n, frames, flags); +return -EOPNOTSUPP ; }
@@ -1661,9 +1668,8 @@ static int verify_xdp_configuration(struct net_device *dev)return -EOPNOTSUPP ; } -if (priv->queue_format !=GVE_GQI_QPL_FORMAT ) { -netdev_warn (dev,"XDP is not supported in mode %d.\n" , - priv->queue_format); +if (priv->header_split_enabled) { +netdev_warn (dev,"XDP is not supported when header-data split is enabled.\n" );return -EOPNOTSUPP ; }
@@ -1987,10 +1993,13 @@ u16 gve_get_pkt_buf_size(const struct gve_priv *priv, bool enable_hsplit)return GVE_DEFAULT_RX_BUFFER_SIZE ; } -/* header-split is not supported on non-DQO_RDA yet even if device advertises it */ +/* Header split is only supported on DQ RDA queue format. If XDP is enabled, +* header split is not allowed. +*/ bool gve_header_split_supported (const struct gve_priv *priv) { -return priv->header_buf_size && priv->queue_format ==GVE_DQO_RDA_FORMAT ; +return priv->header_buf_size && + priv->queue_format ==GVE_DQO_RDA_FORMAT && !priv->xdp_prog; }int gve_set_hsplit_config (struct gve_priv *priv,u8 tcp_data_split)
@@ -2039,6 +2048,12 @@ static int gve_set_features(struct net_device *netdev,if ((netdev->features &NETIF_F_LRO ) != (features &NETIF_F_LRO )) { netdev->features ^=NETIF_F_LRO ; +if (priv->xdp_prog && (netdev->features &NETIF_F_LRO )) { +netdev_warn (netdev, +"XDP is not supported when LRO is on.\n" ); + err = -EOPNOTSUPP ; + goto revert_features; + }if (netif_running (netdev)) { err =gve_adjust_config (priv, &tx_alloc_cfg, &rx_alloc_cfg);if (err)
@@ -2240,6 +2255,9 @@ static void gve_set_netdev_xdp_features(struct gve_priv *priv) xdp_features =NETDEV_XDP_ACT_BASIC ; xdp_features |=NETDEV_XDP_ACT_REDIRECT ; xdp_features |=NETDEV_XDP_ACT_XSK_ZEROCOPY ; + }else if (priv->queue_format ==GVE_DQO_RDA_FORMAT ) { + xdp_features =NETDEV_XDP_ACT_BASIC ; + xdp_features |=NETDEV_XDP_ACT_REDIRECT ; }else { xdp_features =0 ; }
drivers/net/ethernet/google/gve/gve_rx_dqo.c +67 -10
122 lines not on this page · raw message
drivers/net/ethernet/google/gve/gve_tx_dqo.c +138 -13
204 lines not on this page · raw message
Signature · 1 line
2.50.0.rc2.761.g2dc52ea45b-goog
This series was applied to netdev/net-next.git (main) by David S. Miller <[email protected]>:
Hello:
This series was applied to netdev/net-next.git (main) by David S. Miller <[email protected]>:
Quoted 8 lines · On Wed, 18 Jun 2025 20:56:10 +0000 you wrote:
From: Joshua Washington <[email protected]>
A previous patch series[1] introduced the ability to process XDP buffers to the DQ RDA queue format. This is a follow-up patch series to introduce
XDP_TXandXDP_REDIRECTsupport and expose XDP support to the kernel.[...]
Here is the summary with links:
- [net-next,1/3] gve: rename gve_xdp_xmit to gve_xdp_xmit_gqi
https://git.kernel.org/netdev/net-next/c/d05ebf7cc3c5 - [net-next,2/3] gve: refactor DQO TX methods to be more generic for XDP https://git.kernel.org/netdev/net-next/c/cb711b3d197a - [net-next,3/3] gve: add XDP_TX and XDP_REDIRECT support for DQ RDA https://git.kernel.org/netdev/net-next/c/d8a8ca14c937
You are awesome, thank you!
Signature · 2 lines
Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/patchwork/pwbot.html