Skip to contentSkip to conversation

Stability fixes for GVE

PATCH Superseded Joshua Washington 5 participants 39 messages Jul 3 – Jul 10 Network drivers

lore
Share & tools
lore
#0 lore raw

v2 cover Stability fixes for GVE

This patch series consists of mostly unrelated fixes in the GVE driver.

Joshua Washington (9):
net/gve: clear out shared memory region for stats report net/gve: delay adding mbuf head to software ring net/gve: copy data to QPL buffer when mbuf read does not net/gve: validate buf ID before processing Rx packet net/gve: set mbuf to null in software ring after use net/gve: free ctx mbuf if packet dropped after first segment net/gve: increase range of DMA memzone ids to 64 bits net/gve: don't reset ring size bounds to default on reset net/gve: restrict max ring size in GQ QPL to 2K

 drivers/net/gve/base/gve_adminq.c | 12 ++++++--
 drivers/net/gve/base/gve_osdep.h  |  4 +--
 drivers/net/gve/gve_ethdev.c      |  8 ++++--
 drivers/net/gve/gve_ethdev.h      |  1 +
 drivers/net/gve/gve_rx.c          |  3 ++
 drivers/net/gve/gve_rx_dqo.c      |  6 ++++
 drivers/net/gve/gve_tx.c          | 46 ++++++++++++++++++-------------
 7 files changed, 53 insertions(+), 27 deletions(-)

---
v2:
* Remove unused definition

--
2.55.0.rc0.799.gd6f94ed593-goog

#1 lore raw

PATCH v2 1/9 net/gve: clear out shared memory region for stats report

The stats report memzone is allocated from hugepage memory which could possibly have had sensitive data from a previous DPDK invocation.

Clear out the buffer before sharing the memory region with the virtual device to protect guest memory.

Fixes 458b53dec01e ("net/gve: enable imissed stats for GQ format")
---
 drivers/net/gve/gve_ethdev.c | 2 ++
 1 file changed, 2 insertions(+)
drivers/net/gve/gve_ethdev.c +2 -0
index 0b02dcb3ad..f73784a109 100644
@@ -306,6 +306,8 @@ gve_alloc_stats_report(struct gve_priv *priv,
 	if (!priv->stats_report_mem)
 		return -ENOMEM;
 
+	memset(priv->stats_report_mem->addr, 0, priv->stats_report_mem->len);
+
 	/* offset by skipping stats written by gve. */
 	priv->stats_start_idx = (GVE_TX_STATS_REPORT_NUM * nb_tx_queues) +
 		(GVE_RX_STATS_REPORT_NUM * nb_rx_queues);
Signature · 1 line
2.55.0.rc0.799.gd6f94ed593-goog
#2 lore raw

PATCH v2 2/9 net/gve: delay adding mbuf head to software ring

In reply to Joshua Washington

The GQ TX datapath was set up to write the mbuf head into the sw_ring before writing the descriptors. This poses a problem because it's possible for the packet to be dropped due to lacking the FIFO space to do a proper TX. In such a case, the packet won't be sent, and will lead to leaked mbufs in the subsequent segments.

There is also no real reason that the head mbuf must be set in the sw_ring separately from the others; the mbuf chain is not actually walked as part of GQ TX.

Fixes a46583cf43c8 ("net/gve: support Rx/Tx")
---
 drivers/net/gve/gve_tx.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)
drivers/net/gve/gve_tx.c +4 -4
index 5c73c21b8d..59c82b04ed 100644
@@ -301,7 +301,6 @@ gve_tx_burst_qpl(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 			(uint32_t)(tx_offload.l2_len + tx_offload.l3_len + tx_offload.l4_len) :
 			tx_pkt->pkt_len;
 
-		sw_ring[sw_id] = tx_pkt;
 		if (!is_fifo_avail(txq, hlen)) {
 			gve_tx_clean(txq);
 			if (!is_fifo_avail(txq, hlen))
@@ -344,13 +343,14 @@ gve_tx_burst_qpl(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 		}
 
 		/* record mbuf in sw_ring for free */
-		for (i = 1; i < first->nb_segs; i++) {
+		for (i = 0; i < first->nb_segs; i++) {
+			if (!tx_pkt)
+				break;
+			sw_ring[sw_id] = tx_pkt;
 			sw_id = (sw_id + 1) & mask;
 			tx_pkt = tx_pkt->next;
-			sw_ring[sw_id] = tx_pkt;
 		}
 
-		sw_id = (sw_id + 1) & mask;
 		tx_id = (tx_id + 1) & mask;
 
 		txq->nb_free -= nb_used;
Signature · 1 line
2.55.0.rc0.799.gd6f94ed593-goog
#3 lore raw

PATCH v2 3/9 net/gve: copy data to QPL buffer when mbuf read does not

In reply to Joshua Washington

The rte_pktmbuf_read method does not guarantee that data will be copied from an mbuf. If the requested data is all contiguous, the method will instead return a pointer to the memory location within the buffer that should be read from, leaving the destination buffer empty.

This is problematic for TSO/multi-segment TX packets which only make use of two mbufs. If all data in the second mbuf is contiguous, the data will not be read to QPL memory.

Update the QPL copy logic to copy if the rte_pktmbuf_read does not.

Fixes a46583cf43c8 ("net/gve: support Rx/Tx")

---
v2:
Removed unused declaration and definition of addr

 drivers/net/gve/gve_tx.c | 38 +++++++++++++++++++++++---------------
 1 file changed, 23 insertions(+), 15 deletions(-)
drivers/net/gve/gve_tx.c +23 -15
index 59c82b04ed..d220d5167a 100644
@@ -255,10 +255,10 @@ gve_tx_burst_qpl(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 	struct rte_mbuf **sw_ring = txq->sw_ring;
 	uint16_t mask = txq->nb_tx_desc - 1;
 	uint16_t tx_id = txq->tx_tail & mask;
-	uint64_t ol_flags, addr, fifo_addr;
 	uint32_t tx_tail = txq->tx_tail;
 	struct rte_mbuf *tx_pkt, *first;
 	uint16_t sw_id = txq->sw_tail;
+	uint64_t ol_flags, fifo_addr;
 	uint16_t nb_used, i;
 	uint64_t bytes = 0;
 	uint16_t nb_tx = 0;
@@ -273,6 +273,9 @@ gve_tx_burst_qpl(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 		gve_tx_clean_swr_qpl(txq);
 
 	for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
+		const void *mbuf_header_addr;
+		void *qpl_write_addr;
+
 		tx_pkt = *tx_pkts++;
 		ol_flags = tx_pkt->ol_flags;
 
@@ -306,7 +309,6 @@ gve_tx_burst_qpl(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 			if (!is_fifo_avail(txq, hlen))
 				goto end_of_tx;
 		}
-		addr = (uint64_t)(tx_pkt->buf_addr) + tx_pkt->data_off;
 		fifo_addr = gve_tx_alloc_from_fifo(txq, tx_id, hlen);
 
 		/* For TSO, check if there's enough fifo space for data first */
@@ -317,26 +319,32 @@ gve_tx_burst_qpl(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 					goto end_of_tx;
 			}
 		}
-		if (tx_pkt->nb_segs == 1 || ol_flags & RTE_MBUF_F_TX_TCP_SEG)
-			rte_memcpy((void *)(size_t)(fifo_addr + txq->fifo_base),
-				   (void *)(size_t)addr, hlen);
-		else
-			rte_pktmbuf_read(tx_pkt, 0, hlen,
-					 (void *)(size_t)(fifo_addr + txq->fifo_base));
+
+		qpl_write_addr = (void *)(size_t)(fifo_addr + txq->fifo_base);
+		mbuf_header_addr = rte_pktmbuf_read(tx_pkt, 0, hlen, qpl_write_addr);
+
+		/* Header data is linear in the mbuf head. Copy directly. */
+		if (mbuf_header_addr != qpl_write_addr)
+			rte_memcpy(qpl_write_addr, mbuf_header_addr, hlen);
+
 		gve_tx_fill_pkt_desc(txd, tx_pkt, nb_used, hlen, fifo_addr);
 
 		if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
+			const void *mbuf_payload_addr;
+
 			tx_id = (tx_id + 1) & mask;
 			txd = &txr[tx_id];
-			addr = (uint64_t)(tx_pkt->buf_addr) + tx_pkt->data_off + hlen;
 			fifo_addr = gve_tx_alloc_from_fifo(txq, tx_id, tx_pkt->pkt_len - hlen);
-			if (tx_pkt->nb_segs == 1)
-				rte_memcpy((void *)(size_t)(fifo_addr + txq->fifo_base),
-					   (void *)(size_t)addr,
+			qpl_write_addr = (void *)(txq->fifo_base + fifo_addr);
+			mbuf_payload_addr = rte_pktmbuf_read(tx_pkt, hlen, tx_pkt->pkt_len - hlen,
+							     qpl_write_addr);
+
+			/* Payload data is contiguous. Take the offset from the
+			 * read request and copy from there.
+			 */
+			if (mbuf_payload_addr != qpl_write_addr)
+				rte_memcpy(qpl_write_addr, mbuf_payload_addr,
 					   tx_pkt->pkt_len - hlen);
-			else
-				rte_pktmbuf_read(tx_pkt, hlen, tx_pkt->pkt_len - hlen,
-						 (void *)(size_t)(fifo_addr + txq->fifo_base));
 
 			gve_tx_fill_seg_desc(txd, ol_flags, tx_offload,
 					     tx_pkt->pkt_len - hlen, fifo_addr);

--
2.55.0.rc0.799.gd6f94ed593-goog

#4 lore raw

PATCH v2 4/9 net/gve: validate buf ID before processing Rx packet

In reply to Joshua Washington

The buffer id is part of the RX completion descriptor for packets in the DQ format. This value can technically go up to 64K, while the max RX ring size is 4K, meaning that there could similarly be an expected 4K RX buffer IDs. Validate that the RX buffer ID is valid before attempting to access it in the sw_ring to prevent a potential out of bounds in the event of a hardware error.

Fixes 1aed73b23ac0 ("net/gve: support out-of-order completions on DQ Rx")
---
 drivers/net/gve/gve_rx_dqo.c | 5 +++++
 1 file changed, 5 insertions(+)
drivers/net/gve/gve_rx_dqo.c +5 -0
index cc343f3fd8..c4e2d32067 100644
@@ -200,6 +200,11 @@ gve_rx_burst_dqo(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 		}
 
 		rx_buf_id = rte_le_to_cpu_16(rx_desc->buf_id);
+		if (unlikely(rx_buf_id >= rxq->nb_rx_desc)) {
+			PMD_DRV_DP_LOG(ERR, "Invalid buf_id %d", rx_buf_id);
+			continue;
+		}
+
 		rxm = rxq->sw_ring[rx_buf_id];
 		gve_completed_buf_list_push(rxq, rx_buf_id);
 
Signature · 1 line
2.55.0.rc0.799.gd6f94ed593-goog
#5 lore raw

PATCH v2 5/9 net/gve: set mbuf to null in software ring after use

In reply to Joshua Washington

Currently, it is possible for mbufs to be uncleared in the sw_ring after being returned to the application. This causes an erroneous dual-ownership over the buffer until GVE cleans the buffer queue and posts new mbufs, overwriting the older pointers. It is possible in such a case for a double free to occur while tearing down rings, as both the application and the driver could attempt to free the same mbuf. Release ownership of the mbuf from the sw_ring as soon as appropriate to avoid such a scenario.

Fixes a46583cf43c8 ("net/gve: support Rx/Tx")
Fixes 45da16b5b181 ("net/gve: support basic Rx data path for DQO")
---
 drivers/net/gve/gve_rx.c     | 1 +
 drivers/net/gve/gve_rx_dqo.c | 1 +
 2 files changed, 2 insertions(+)
drivers/net/gve/gve_rx.c +1 -0
index 625649cdcf..cda87af294 100644
@@ -152,6 +152,7 @@ gve_rx(struct gve_rx_queue *rxq, volatile struct gve_rx_desc *rxd, uint16_t rx_i
 
 	rxe = rxq->sw_ring[rx_id];
 	gve_rx_mbuf(rxq, rxe, frag_size, rx_id);
+	rxq->sw_ring[rx_id] = NULL;
 	rxq->stats.bytes += frag_size;
 
 	if (is_first_frag) {
drivers/net/gve/gve_rx_dqo.c +1 -0
index c4e2d32067..3665d9e4cd 100644
@@ -207,6 +207,7 @@ gve_rx_burst_dqo(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 
 		rxm = rxq->sw_ring[rx_buf_id];
 		gve_completed_buf_list_push(rxq, rx_buf_id);
+		rxq->sw_ring[rx_buf_id] = NULL;
 
 		/* Free buffer and report error. */
 		if (unlikely(rx_desc->rx_error)) {
Signature · 1 line
2.55.0.rc0.799.gd6f94ed593-goog
#6 lore raw

PATCH v2 6/9 net/gve: free ctx mbuf if packet dropped after first segment

In reply to Joshua Washington

GVE GQ has support for multi-descriptor RX. It is possible for a packet to be dropped after the first descriptor has been processed an mbuf has been added to the context. In such a case, the mbuf head should be freed before clearing the context so that mbufs aren't leaked.

In addition, clear mbuf from sw_ring after adding the packet to the context to avoid double-freeing buffers that have not been reposted, but have been reported to the application.

Fixes 496d4d2c8b54 ("net/gve: support jumbo frame for GQI")
---
 drivers/net/gve/gve_rx.c | 2 ++
 1 file changed, 2 insertions(+)
drivers/net/gve/gve_rx.c +2 -0
index cda87af294..567b82d020 100644
@@ -205,6 +205,8 @@ gve_rx_burst(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 		if (gve_rx(rxq, rxd, rx_id)) {
 			if (!ctx->drop_pkt)
 				rx_pkts[nb_rx++] = ctx->mbuf_head;
+			else if (ctx->mbuf_head != NULL)
+				rte_pktmbuf_free(ctx->mbuf_head);
 			rxq->nb_avail += ctx->total_frags;
 			gve_rx_ctx_clear(ctx);
 		}
Signature · 1 line
2.55.0.rc0.799.gd6f94ed593-goog
#7 lore raw

PATCH v2 7/9 net/gve: increase range of DMA memzone ids to 64 bits

In reply to Joshua Washington

Long running programs can very easily eclipse this 16-bit range, leading to name collisions and failed DMA region allocations despite there being plenty of available memory.

Fixes c9ba2caf6302 ("net/gve/base: add OS-specific implementation")
---
 drivers/net/gve/base/gve_osdep.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
drivers/net/gve/base/gve_osdep.h +2 -2
index c47ce4da85..55629a0e1a 100644
@@ -175,14 +175,14 @@ struct gve_dma_mem {
 static inline void *
 gve_alloc_dma_mem(struct gve_dma_mem *mem, u64 size)
 {
-	static RTE_ATOMIC(uint16_t) gve_dma_memzone_id;
+	static RTE_ATOMIC(uint64_t) gve_dma_memzone_id;
 	const struct rte_memzone *mz = NULL;
 	char z_name[RTE_MEMZONE_NAMESIZE];
 
 	if (!mem)
 		return NULL;
 
-	snprintf(z_name, sizeof(z_name), "gve_dma_%u",
+	snprintf(z_name, sizeof(z_name), "gve_dma_%" PRIu64,
 		 rte_atomic_fetch_add_explicit(&gve_dma_memzone_id, 1, rte_memory_order_relaxed));
 	mz = rte_memzone_reserve_aligned(z_name, size, SOCKET_ID_ANY,
 					 RTE_MEMZONE_IOVA_CONTIG,
Signature · 1 line
2.55.0.rc0.799.gd6f94ed593-goog
#8 lore raw

PATCH v2 8/9 net/gve: don't reset ring size bounds to default on reset

In reply to Joshua Washington

On device reset, GVE skips describe_device functionality, as the device is not expected to change on a reset. However, before skipping the describe_device functionality, GVE still sets the ring sizes to their default values. This effectively removes the ability to create queues with higher-than-default descriptor counts after a reset.

Fix this behavior by only setting the default ring size bounds is describe_device is being executed.

Fixes 1bf64edce3c4 ("net/gve: add reset path")
---
 drivers/net/gve/gve_ethdev.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
drivers/net/gve/gve_ethdev.c +3 -3
index f73784a109..c990920a4d 100644
@@ -1579,12 +1579,12 @@ gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
 		goto free_adminq;
 	}
 
-	/* Set default descriptor counts */
-	gve_set_default_ring_size_bounds(priv);
-
 	if (skip_describe_device)
 		goto setup_device;
 
+	/* Set default descriptor counts */
+	gve_set_default_ring_size_bounds(priv);
+
 	/* Get the initial information we need from the device */
 	err = gve_adminq_describe_device(priv);
 	if (err) {
Signature · 1 line
2.55.0.rc0.799.gd6f94ed593-goog
#9 lore raw

PATCH v2 9/9 net/gve: restrict max ring size in GQ QPL to 2K

In reply to Joshua Washington

The GQ QPL queue format has a maximum supported ring size of 2k. However, it is possible in some cases for the device to pass a larger ring size as the max ring size. Restrict the ring size in the driver to ensure that rings with invalid queue depths are not created.

Fixes cde01d164f8f ("net/gve: support modifying ring size in GQ format")
---
 drivers/net/gve/base/gve_adminq.c | 12 +++++++++---
 drivers/net/gve/gve_ethdev.h      |  1 +
 2 files changed, 10 insertions(+), 3 deletions(-)
drivers/net/gve/base/gve_adminq.c +9 -3
index 2b25c7f390..315e2456fd 100644
@@ -6,6 +6,7 @@
 #include "../gve_ethdev.h"
 #include "gve_adminq.h"
 #include "gve_register.h"
+#include "rte_common.h"
 
 #define GVE_MAX_ADMINQ_RELEASE_CHECK	500
 #define GVE_ADMINQ_SLEEP_LEN		20
@@ -946,15 +947,20 @@ static void
 gve_set_max_desc_cnt(struct gve_priv *priv,
 	const struct gve_device_option_modify_ring *modify_ring)
 {
+	priv->max_rx_desc_cnt = be16_to_cpu(modify_ring->max_ring_size.rx);
+	priv->max_tx_desc_cnt = be16_to_cpu(modify_ring->max_ring_size.tx);
+
 	if (priv->queue_format == GVE_DQO_RDA_FORMAT) {
 		PMD_DRV_LOG(DEBUG, "Overriding max ring size from device for DQ "
 			    "queue format to 4096.");
 		priv->max_rx_desc_cnt = GVE_MAX_QUEUE_SIZE_DQO;
 		priv->max_tx_desc_cnt = GVE_MAX_QUEUE_SIZE_DQO;
-		return;
+	} else if (priv->queue_format == GVE_GQI_QPL_FORMAT) {
+		priv->max_rx_desc_cnt = RTE_MIN(priv->max_rx_desc_cnt,
+						GVE_MAX_RING_SIZE_GQ_QPL);
+		priv->max_tx_desc_cnt = RTE_MIN(priv->max_tx_desc_cnt,
+						GVE_MAX_RING_SIZE_GQ_QPL);
 	}
-	priv->max_rx_desc_cnt = be16_to_cpu(modify_ring->max_ring_size.rx);
-	priv->max_tx_desc_cnt = be16_to_cpu(modify_ring->max_ring_size.tx);
 }
 
 static void gve_enable_supported_features(struct gve_priv *priv,
drivers/net/gve/gve_ethdev.h +1 -0
index 4a7e5ecdf3..c9a176ff17 100644
@@ -21,6 +21,7 @@
 #define DQO_TX_MULTIPLIER 4
 
 #define GVE_DEFAULT_MAX_RING_SIZE	1024
+#define GVE_MAX_RING_SIZE_GQ_QPL	2048
 #define GVE_DEFAULT_MIN_RX_RING_SIZE	512
 #define GVE_DEFAULT_MIN_TX_RING_SIZE	256
 
Signature · 1 line
2.55.0.rc0.799.gd6f94ed593-goog
#10 lore raw
In reply to Joshua Washington
Quoted 22 lines · On Fri, 3 Jul 2026 06:12:57 -0700 Joshua Washington <[email protected]>…

This patch series consists of mostly unrelated fixes in the GVE driver.

Joshua Washington (9):
net/gve: clear out shared memory region for stats report net/gve: delay adding mbuf head to software ring net/gve: copy data to QPL buffer when mbuf read does not net/gve: validate buf ID before processing Rx packet net/gve: set mbuf to null in software ring after use net/gve: free ctx mbuf if packet dropped after first segment net/gve: increase range of DMA memzone ids to 64 bits net/gve: don't reset ring size bounds to default on reset net/gve: restrict max ring size in GQ QPL to 2K

 drivers/net/gve/base/gve_adminq.c | 12 ++++++--
 drivers/net/gve/base/gve_osdep.h  |  4 +--
 drivers/net/gve/gve_ethdev.c      |  8 ++++--
 drivers/net/gve/gve_ethdev.h      |  1 +
 drivers/net/gve/gve_rx.c          |  3 ++
 drivers/net/gve/gve_rx_dqo.c      |  6 ++++
 drivers/net/gve/gve_tx.c          | 46 ++++++++++++++++++-------------
 7 files changed, 53 insertions(+), 27 deletions(-)
Quoted 7 lines

---
v2:
* Remove unused definition

--
2.55.0.rc0.799.gd6f94ed593-goog

Better but fails on 32 bit build during post-merge testing DPDK 26.07.0-rc2

User defined options

    Cross files           : /home/shemminger/DPDK/next-net/devtools/../config/x86/cross-32bit-debian.ini
    buildtype             : debugoptimized
    default_library       : shared

enable_deprecated_libs: *
examples : all
werror : true

Found ninja-1.13.2 at /usr/bin/ninja
ninja: Entering directory `./build-32b'
[937/2812] Compiling C object drivers/libtmp_rte_net_gve.a.p/net_gve_gve_tx.c.o
FAILED: [code=1] drivers/libtmp_rte_net_gve.a.p/net_gve_gve_tx.c.o
cc -Idrivers/libtmp_rte_net_gve.a.p -Idrivers -I../drivers -Idrivers/net/gve -I../drivers/net/gve -I../drivers/net/gve/base -Ilib/ethdev -I../lib/ethdev -Ilib/eal/common -I../lib/eal/common -I. -I.. -Iconfig -I../config -Ilib/eal/include -I../lib/eal/include -Ilib/eal/linux/include -I../lib/eal/linux/include -Ilib/eal/x86/include -I../lib/eal/x86/include -I../kernel/linux -Ilib/eal -I../lib/eal -Ilib/kvargs -I../lib/kvargs -Ilib/log -I../lib/log -Ilib/metrics -I../lib/metrics -Ilib/telemetry -I../lib/telemetry -Ilib/argparse -I../lib/argparse -Ilib/net -I../lib/net -Ilib/mbuf -I../lib/mbuf -Ilib/mempool -I../lib/mempool -Ilib/ring -I../lib/ring -Ilib/meter -I../lib/meter -Idrivers/bus/pci -I../drivers/bus/pci -I../drivers/bus/pci/linux -Ilib/pci -I../lib/pci -Idrivers/bus/vdev -I../drivers/bus/vdev -fdiagnostics-color=always -D_FILE_OFFSET_BITS=64 -Wall -Winvalid-pch -Wextra -Werror -std=c11 -O2 -g -include rte_config.h -Wvla -Wcast-qual -Wdeprecated -Wformat -Wformat-nonliteral -Wformat-security -Wmissing-declarations -Wmissing-prototypes -Wnested-externs -Wold-style-definition -Wpointer-arith -Wshadow -Wsign-compare -Wstrict-prototypes -Wundef -Wwrite-strings -Wno-packed-not-aligned -Wno-missing-field-initializers -fzero-init-padding-bits=all -Wno-pointer-to-int-cast -D_GNU_SOURCE -m32 -fPIC -march=native -mrtm -DALLOW_EXPERIMENTAL_API -DALLOW_INTERNAL_API -Wno-format-truncation -Wno-address-of-packed-member -Wno-vla -DRTE_COMPONENT_CLASS=pmd_net -DRTE_COMPONENT_NAME=gve -DRTE_LOG_DEFAULT_LOGTYPE=pmd.net.gve -MD -MQ drivers/libtmp_rte_net_gve.a.p/net_gve_gve_tx.c.o -MF drivers/libtmp_rte_net_gve.a.p/net_gve_gve_tx.c.o.d -o drivers/libtmp_rte_net_gve.a.p/net_gve_gve_tx.c.o -c ../drivers/net/gve/gve_tx.c ../drivers/net/gve/gve_tx.c: In function ‘gve_tx_burst_qpl’:
../drivers/net/gve/gve_tx.c:338:42: error: cast to pointer from integer of different size [-Werror=int-to-pointer-cast]
338 | qpl_write_addr = (void *)(txq->fifo_base + fifo_addr);
| ^
cc1: all warnings being treated as errors

#11 lore raw
Quoted 3 lines

Better but fails on 32 bit build during post-merge testing DPDK 26.07.0-rc2

Will fix in v3. Is there a way I can test 32-bit compile using test-meson-builds.sh? It doesn't seem to be covered there by default.

#12 lore raw
Quoted 8 lines · Joshua Washington
Quoted 4 lines

Better but fails on 32 bit build during post-merge testing DPDK 26.07.0-rc2

Will fix in v3. Is there a way I can test 32-bit compile using test-meson-builds.sh? It doesn't seem to be covered there by default.

You'll need support in your toolchain, but this script does support 32-bit compilation.

On debian for example, you'll need (at least):
dpkg --add-architecture i386
apt install -y gcc-multilib g++-multilib libnuma-dev:i386

If you have a personal github account, it is also possible to try your changes after enabling github actions in your dpdk fork.

Signature · 1 line
David Marchand
#13 lore raw
Quoted 11 lines · David Marchand
Quoted 8 lines · Joshua Washington
Quoted 4 lines

Better but fails on 32 bit build during post-merge testing DPDK 26.07.0-rc2

Will fix in v3. Is there a way I can test 32-bit compile using test-meson-builds.sh? It doesn't seem to be covered there by default.

Quoted 6 lines

You'll need support in your toolchain, but this script does support 32-bit compilation.

On debian for example, you'll need (at least):
dpkg --add-architecture i386
apt install -y gcc-multilib g++-multilib libnuma-dev:i386

Ah, thanks. I installed the required dependencies to build 32-bit manually, and the script picked it up automatically as expected.

Quoted 8 lines

If you have a personal github account, it is also possible to try your changes after enabling github actions in your dpdk fork.

--
David Marchand

#14 lore raw
In reply to Joshua Washington
Quoted 6 lines · Joshua Washington
Quoted 3 lines

Better but fails on 32 bit build during post-merge testing DPDK 26.07.0-rc2

Will fix in v3. Is there a way I can test 32-bit compile using test-meson-builds.sh? It doesn't seem to be covered there by default.

It is covered if the necessary 32-bit multilib packages are available. I verified using my ubuntu install here, and after installing gcc-multilib package I had 32-bit builds from test-meson-builds.sh. Unfortunately, the multilib package for 32-bit conflicts with the packages for arm and PPC building, which is why I didnt have them installed myself before now.

/Bruce

#15 lore raw

v3 cover Stability fixes for GVE

In reply to Joshua Washington

This patch series consists of mostly unrelated fixes in the GVE driver.

Joshua Washington (9):
net/gve: clear out shared memory region for stats report net/gve: delay adding mbuf head to software ring net/gve: copy data to QPL buffer when mbuf read does not net/gve: validate buf ID before processing Rx packet net/gve: set mbuf to null in software ring after use net/gve: free ctx mbuf if packet dropped after first segment net/gve: increase range of DMA memzone ids to 64 bits net/gve: don't reset ring size bounds to default on reset net/gve: restrict max ring size in GQ QPL to 2K

 drivers/net/gve/base/gve_adminq.c | 12 ++++++--
 drivers/net/gve/base/gve_osdep.h  |  4 +--
 drivers/net/gve/gve_ethdev.c      |  8 ++++--
 drivers/net/gve/gve_ethdev.h      |  1 +
 drivers/net/gve/gve_rx.c          |  3 ++
 drivers/net/gve/gve_rx_dqo.c      |  6 ++++
 drivers/net/gve/gve_tx.c          | 46 ++++++++++++++++++-------------
 7 files changed, 53 insertions(+), 27 deletions(-)

---
v3:
* Fix 32-bit complication issue
v2:
* Remove unused definition

Signature · 1 line
2.55.0.rc2.803.g1fd1e6609c-goog
#16 lore raw

PATCH v3 1/9 net/gve: clear out shared memory region for stats report

The stats report memzone is allocated from hugepage memory which could possibly have had sensitive data from a previous DPDK invocation.

Clear out the buffer before sharing the memory region with the virtual device to protect guest memory.

Fixes 458b53dec01e ("net/gve: enable imissed stats for GQ format")
---
 drivers/net/gve/gve_ethdev.c | 2 ++
 1 file changed, 2 insertions(+)
drivers/net/gve/gve_ethdev.c +2 -0
index 0b02dcb3ad..f73784a109 100644
@@ -306,6 +306,8 @@ gve_alloc_stats_report(struct gve_priv *priv,
 	if (!priv->stats_report_mem)
 		return -ENOMEM;
 
+	memset(priv->stats_report_mem->addr, 0, priv->stats_report_mem->len);
+
 	/* offset by skipping stats written by gve. */
 	priv->stats_start_idx = (GVE_TX_STATS_REPORT_NUM * nb_tx_queues) +
 		(GVE_RX_STATS_REPORT_NUM * nb_rx_queues);
Signature · 1 line
2.55.0.rc2.803.g1fd1e6609c-goog
#17 lore raw

PATCH v3 2/9 net/gve: delay adding mbuf head to software ring

In reply to Joshua Washington

The GQ TX datapath was set up to write the mbuf head into the sw_ring before writing the descriptors. This poses a problem because it's possible for the packet to be dropped due to lacking the FIFO space to do a proper TX. In such a case, the packet won't be sent, and will lead to leaked mbufs in the subsequent segments.

There is also no real reason that the head mbuf must be set in the sw_ring separately from the others; the mbuf chain is not actually walked as part of GQ TX.

Fixes a46583cf43c8 ("net/gve: support Rx/Tx")
---
 drivers/net/gve/gve_tx.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)
drivers/net/gve/gve_tx.c +4 -4
index 5c73c21b8d..59c82b04ed 100644
@@ -301,7 +301,6 @@ gve_tx_burst_qpl(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 			(uint32_t)(tx_offload.l2_len + tx_offload.l3_len + tx_offload.l4_len) :
 			tx_pkt->pkt_len;
 
-		sw_ring[sw_id] = tx_pkt;
 		if (!is_fifo_avail(txq, hlen)) {
 			gve_tx_clean(txq);
 			if (!is_fifo_avail(txq, hlen))
@@ -344,13 +343,14 @@ gve_tx_burst_qpl(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 		}
 
 		/* record mbuf in sw_ring for free */
-		for (i = 1; i < first->nb_segs; i++) {
+		for (i = 0; i < first->nb_segs; i++) {
+			if (!tx_pkt)
+				break;
+			sw_ring[sw_id] = tx_pkt;
 			sw_id = (sw_id + 1) & mask;
 			tx_pkt = tx_pkt->next;
-			sw_ring[sw_id] = tx_pkt;
 		}
 
-		sw_id = (sw_id + 1) & mask;
 		tx_id = (tx_id + 1) & mask;
 
 		txq->nb_free -= nb_used;
Signature · 1 line
2.55.0.rc2.803.g1fd1e6609c-goog
#18 lore raw

PATCH v3 3/9 net/gve: copy data to QPL buffer when mbuf read does not

In reply to Joshua Washington

The rte_pktmbuf_read method does not guarantee that data will be copied from an mbuf. If the requested data is all contiguous, the method will instead return a pointer to the memory location within the buffer that should be read from, leaving the destination buffer empty.

This is problematic for TSO/multi-segment TX packets which only make use of two mbufs. If all data in the second mbuf is contiguous, the data will not be read to QPL memory.

Update the QPL copy logic to copy if the rte_pktmbuf_read does not.

Fixes a46583cf43c8 ("net/gve: support Rx/Tx")

---
v3:
Fix 32-bit compile issue
v2:
Removed unused declaration and definition of addr

 drivers/net/gve/gve_tx.c | 38 +++++++++++++++++++++++---------------
 1 file changed, 23 insertions(+), 15 deletions(-)
drivers/net/gve/gve_tx.c +23 -15
index 59c82b04ed..c0400b07bf 100644
@@ -255,10 +255,10 @@ gve_tx_burst_qpl(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 	struct rte_mbuf **sw_ring = txq->sw_ring;
 	uint16_t mask = txq->nb_tx_desc - 1;
 	uint16_t tx_id = txq->tx_tail & mask;
-	uint64_t ol_flags, addr, fifo_addr;
 	uint32_t tx_tail = txq->tx_tail;
 	struct rte_mbuf *tx_pkt, *first;
 	uint16_t sw_id = txq->sw_tail;
+	uint64_t ol_flags, fifo_addr;
 	uint16_t nb_used, i;
 	uint64_t bytes = 0;
 	uint16_t nb_tx = 0;
@@ -273,6 +273,9 @@ gve_tx_burst_qpl(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 		gve_tx_clean_swr_qpl(txq);
 
 	for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
+		const void *mbuf_header_addr;
+		void *qpl_write_addr;
+
 		tx_pkt = *tx_pkts++;
 		ol_flags = tx_pkt->ol_flags;
 
@@ -306,7 +309,6 @@ gve_tx_burst_qpl(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 			if (!is_fifo_avail(txq, hlen))
 				goto end_of_tx;
 		}
-		addr = (uint64_t)(tx_pkt->buf_addr) + tx_pkt->data_off;
 		fifo_addr = gve_tx_alloc_from_fifo(txq, tx_id, hlen);
 
 		/* For TSO, check if there's enough fifo space for data first */
@@ -317,26 +319,32 @@ gve_tx_burst_qpl(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 					goto end_of_tx;
 			}
 		}
-		if (tx_pkt->nb_segs == 1 || ol_flags & RTE_MBUF_F_TX_TCP_SEG)
-			rte_memcpy((void *)(size_t)(fifo_addr + txq->fifo_base),
-				   (void *)(size_t)addr, hlen);
-		else
-			rte_pktmbuf_read(tx_pkt, 0, hlen,
-					 (void *)(size_t)(fifo_addr + txq->fifo_base));
+
+		qpl_write_addr = (void *)(size_t)(fifo_addr + txq->fifo_base);
+		mbuf_header_addr = rte_pktmbuf_read(tx_pkt, 0, hlen, qpl_write_addr);
+
+		/* Header data is linear in the mbuf head. Copy directly. */
+		if (mbuf_header_addr != qpl_write_addr)
+			rte_memcpy(qpl_write_addr, mbuf_header_addr, hlen);
+
 		gve_tx_fill_pkt_desc(txd, tx_pkt, nb_used, hlen, fifo_addr);
 
 		if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
+			const void *mbuf_payload_addr;
+
 			tx_id = (tx_id + 1) & mask;
 			txd = &txr[tx_id];
-			addr = (uint64_t)(tx_pkt->buf_addr) + tx_pkt->data_off + hlen;
 			fifo_addr = gve_tx_alloc_from_fifo(txq, tx_id, tx_pkt->pkt_len - hlen);
-			if (tx_pkt->nb_segs == 1)
-				rte_memcpy((void *)(size_t)(fifo_addr + txq->fifo_base),
-					   (void *)(size_t)addr,
+			qpl_write_addr = (void *)(size_t)(txq->fifo_base + fifo_addr);
+			mbuf_payload_addr = rte_pktmbuf_read(tx_pkt, hlen, tx_pkt->pkt_len - hlen,
+							     qpl_write_addr);
+
+			/* Payload data is contiguous. Take the offset from the
+			 * read request and copy from there.
+			 */
+			if (mbuf_payload_addr != qpl_write_addr)
+				rte_memcpy(qpl_write_addr, mbuf_payload_addr,
 					   tx_pkt->pkt_len - hlen);
-			else
-				rte_pktmbuf_read(tx_pkt, hlen, tx_pkt->pkt_len - hlen,
-						 (void *)(size_t)(fifo_addr + txq->fifo_base));
 
 			gve_tx_fill_seg_desc(txd, ol_flags, tx_offload,
 					     tx_pkt->pkt_len - hlen, fifo_addr);
Signature · 1 line
2.55.0.rc2.803.g1fd1e6609c-goog
#19 lore raw

PATCH v3 4/9 net/gve: validate buf ID before processing Rx packet

In reply to Joshua Washington

The buffer id is part of the RX completion descriptor for packets in the DQ format. This value can technically go up to 64K, while the max RX ring size is 4K, meaning that there could similarly be an expected 4K RX buffer IDs. Validate that the RX buffer ID is valid before attempting to access it in the sw_ring to prevent a potential out of bounds in the event of a hardware error.

Fixes 1aed73b23ac0 ("net/gve: support out-of-order completions on DQ Rx")
---
 drivers/net/gve/gve_rx_dqo.c | 5 +++++
 1 file changed, 5 insertions(+)
drivers/net/gve/gve_rx_dqo.c +5 -0
index cc343f3fd8..c4e2d32067 100644
@@ -200,6 +200,11 @@ gve_rx_burst_dqo(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 		}
 
 		rx_buf_id = rte_le_to_cpu_16(rx_desc->buf_id);
+		if (unlikely(rx_buf_id >= rxq->nb_rx_desc)) {
+			PMD_DRV_DP_LOG(ERR, "Invalid buf_id %d", rx_buf_id);
+			continue;
+		}
+
 		rxm = rxq->sw_ring[rx_buf_id];
 		gve_completed_buf_list_push(rxq, rx_buf_id);
 
Signature · 1 line
2.55.0.rc2.803.g1fd1e6609c-goog
#20 lore raw

PATCH v3 5/9 net/gve: set mbuf to null in software ring after use

In reply to Joshua Washington

Currently, it is possible for mbufs to be uncleared in the sw_ring after being returned to the application. This causes an erroneous dual-ownership over the buffer until GVE cleans the buffer queue and posts new mbufs, overwriting the older pointers. It is possible in such a case for a double free to occur while tearing down rings, as both the application and the driver could attempt to free the same mbuf. Release ownership of the mbuf from the sw_ring as soon as appropriate to avoid such a scenario.

Fixes a46583cf43c8 ("net/gve: support Rx/Tx")
Fixes 45da16b5b181 ("net/gve: support basic Rx data path for DQO")
---
 drivers/net/gve/gve_rx.c     | 1 +
 drivers/net/gve/gve_rx_dqo.c | 1 +
 2 files changed, 2 insertions(+)
drivers/net/gve/gve_rx.c +1 -0
index 625649cdcf..cda87af294 100644
@@ -152,6 +152,7 @@ gve_rx(struct gve_rx_queue *rxq, volatile struct gve_rx_desc *rxd, uint16_t rx_i
 
 	rxe = rxq->sw_ring[rx_id];
 	gve_rx_mbuf(rxq, rxe, frag_size, rx_id);
+	rxq->sw_ring[rx_id] = NULL;
 	rxq->stats.bytes += frag_size;
 
 	if (is_first_frag) {
drivers/net/gve/gve_rx_dqo.c +1 -0
index c4e2d32067..3665d9e4cd 100644
@@ -207,6 +207,7 @@ gve_rx_burst_dqo(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 
 		rxm = rxq->sw_ring[rx_buf_id];
 		gve_completed_buf_list_push(rxq, rx_buf_id);
+		rxq->sw_ring[rx_buf_id] = NULL;
 
 		/* Free buffer and report error. */
 		if (unlikely(rx_desc->rx_error)) {
Signature · 1 line
2.55.0.rc2.803.g1fd1e6609c-goog
#21 lore raw

PATCH v3 6/9 net/gve: free ctx mbuf if packet dropped after first segment

In reply to Joshua Washington

GVE GQ has support for multi-descriptor RX. It is possible for a packet to be dropped after the first descriptor has been processed an mbuf has been added to the context. In such a case, the mbuf head should be freed before clearing the context so that mbufs aren't leaked.

In addition, clear mbuf from sw_ring after adding the packet to the context to avoid double-freeing buffers that have not been reposted, but have been reported to the application.

Fixes 496d4d2c8b54 ("net/gve: support jumbo frame for GQI")
---
 drivers/net/gve/gve_rx.c | 2 ++
 1 file changed, 2 insertions(+)
drivers/net/gve/gve_rx.c +2 -0
index cda87af294..567b82d020 100644
@@ -205,6 +205,8 @@ gve_rx_burst(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 		if (gve_rx(rxq, rxd, rx_id)) {
 			if (!ctx->drop_pkt)
 				rx_pkts[nb_rx++] = ctx->mbuf_head;
+			else if (ctx->mbuf_head != NULL)
+				rte_pktmbuf_free(ctx->mbuf_head);
 			rxq->nb_avail += ctx->total_frags;
 			gve_rx_ctx_clear(ctx);
 		}
Signature · 1 line
2.55.0.rc2.803.g1fd1e6609c-goog
#22 lore raw

PATCH v3 7/9 net/gve: increase range of DMA memzone ids to 64 bits

In reply to Joshua Washington

Long running programs can very easily eclipse this 16-bit range, leading to name collisions and failed DMA region allocations despite there being plenty of available memory.

Fixes c9ba2caf6302 ("net/gve/base: add OS-specific implementation")
---
 drivers/net/gve/base/gve_osdep.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
drivers/net/gve/base/gve_osdep.h +2 -2
index c47ce4da85..55629a0e1a 100644
@@ -175,14 +175,14 @@ struct gve_dma_mem {
 static inline void *
 gve_alloc_dma_mem(struct gve_dma_mem *mem, u64 size)
 {
-	static RTE_ATOMIC(uint16_t) gve_dma_memzone_id;
+	static RTE_ATOMIC(uint64_t) gve_dma_memzone_id;
 	const struct rte_memzone *mz = NULL;
 	char z_name[RTE_MEMZONE_NAMESIZE];
 
 	if (!mem)
 		return NULL;
 
-	snprintf(z_name, sizeof(z_name), "gve_dma_%u",
+	snprintf(z_name, sizeof(z_name), "gve_dma_%" PRIu64,
 		 rte_atomic_fetch_add_explicit(&gve_dma_memzone_id, 1, rte_memory_order_relaxed));
 	mz = rte_memzone_reserve_aligned(z_name, size, SOCKET_ID_ANY,
 					 RTE_MEMZONE_IOVA_CONTIG,
Signature · 1 line
2.55.0.rc2.803.g1fd1e6609c-goog
#23 lore raw

PATCH v3 8/9 net/gve: don't reset ring size bounds to default on reset

In reply to Joshua Washington

On device reset, GVE skips describe_device functionality, as the device is not expected to change on a reset. However, before skipping the describe_device functionality, GVE still sets the ring sizes to their default values. This effectively removes the ability to create queues with higher-than-default descriptor counts after a reset.

Fix this behavior by only setting the default ring size bounds is describe_device is being executed.

Fixes 1bf64edce3c4 ("net/gve: add reset path")
---
 drivers/net/gve/gve_ethdev.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
drivers/net/gve/gve_ethdev.c +3 -3
index f73784a109..c990920a4d 100644
@@ -1579,12 +1579,12 @@ gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
 		goto free_adminq;
 	}
 
-	/* Set default descriptor counts */
-	gve_set_default_ring_size_bounds(priv);
-
 	if (skip_describe_device)
 		goto setup_device;
 
+	/* Set default descriptor counts */
+	gve_set_default_ring_size_bounds(priv);
+
 	/* Get the initial information we need from the device */
 	err = gve_adminq_describe_device(priv);
 	if (err) {
Signature · 1 line
2.55.0.rc2.803.g1fd1e6609c-goog
#24 lore raw

PATCH v3 9/9 net/gve: restrict max ring size in GQ QPL to 2K

In reply to Joshua Washington

The GQ QPL queue format has a maximum supported ring size of 2k. However, it is possible in some cases for the device to pass a larger ring size as the max ring size. Restrict the ring size in the driver to ensure that rings with invalid queue depths are not created.

Fixes cde01d164f8f ("net/gve: support modifying ring size in GQ format")
---
 drivers/net/gve/base/gve_adminq.c | 12 +++++++++---
 drivers/net/gve/gve_ethdev.h      |  1 +
 2 files changed, 10 insertions(+), 3 deletions(-)
drivers/net/gve/base/gve_adminq.c +9 -3
index 2b25c7f390..315e2456fd 100644
@@ -6,6 +6,7 @@
 #include "../gve_ethdev.h"
 #include "gve_adminq.h"
 #include "gve_register.h"
+#include "rte_common.h"
 
 #define GVE_MAX_ADMINQ_RELEASE_CHECK	500
 #define GVE_ADMINQ_SLEEP_LEN		20
@@ -946,15 +947,20 @@ static void
 gve_set_max_desc_cnt(struct gve_priv *priv,
 	const struct gve_device_option_modify_ring *modify_ring)
 {
+	priv->max_rx_desc_cnt = be16_to_cpu(modify_ring->max_ring_size.rx);
+	priv->max_tx_desc_cnt = be16_to_cpu(modify_ring->max_ring_size.tx);
+
 	if (priv->queue_format == GVE_DQO_RDA_FORMAT) {
 		PMD_DRV_LOG(DEBUG, "Overriding max ring size from device for DQ "
 			    "queue format to 4096.");
 		priv->max_rx_desc_cnt = GVE_MAX_QUEUE_SIZE_DQO;
 		priv->max_tx_desc_cnt = GVE_MAX_QUEUE_SIZE_DQO;
-		return;
+	} else if (priv->queue_format == GVE_GQI_QPL_FORMAT) {
+		priv->max_rx_desc_cnt = RTE_MIN(priv->max_rx_desc_cnt,
+						GVE_MAX_RING_SIZE_GQ_QPL);
+		priv->max_tx_desc_cnt = RTE_MIN(priv->max_tx_desc_cnt,
+						GVE_MAX_RING_SIZE_GQ_QPL);
 	}
-	priv->max_rx_desc_cnt = be16_to_cpu(modify_ring->max_ring_size.rx);
-	priv->max_tx_desc_cnt = be16_to_cpu(modify_ring->max_ring_size.tx);
 }
 
 static void gve_enable_supported_features(struct gve_priv *priv,
drivers/net/gve/gve_ethdev.h +1 -0
index 4a7e5ecdf3..c9a176ff17 100644
@@ -21,6 +21,7 @@
 #define DQO_TX_MULTIPLIER 4
 
 #define GVE_DEFAULT_MAX_RING_SIZE	1024
+#define GVE_MAX_RING_SIZE_GQ_QPL	2048
 #define GVE_DEFAULT_MIN_RX_RING_SIZE	512
 #define GVE_DEFAULT_MIN_TX_RING_SIZE	256
 
Signature · 1 line
2.55.0.rc2.803.g1fd1e6609c-goog
#25 lore raw

Re: [PATCH v3 9/9] net/gve: restrict max ring size in GQ QPL to 2K

Quoted 20 lines · Joshua Washington

The GQ QPL queue format has a maximum supported ring size of 2k. However, it is possible in some cases for the device to pass a larger ring size as the max ring size. Restrict the ring size in the driver to ensure that rings with invalid queue depths are not created.

Fixes cde01d164f8f ("net/gve: support modifying ring size in GQ format")
---
 drivers/net/gve/base/gve_adminq.c | 12 +++++++++---
 drivers/net/gve/gve_ethdev.h      |  1 +
 2 files changed, 10 insertions(+), 3 deletions(-)
drivers/net/gve/base/gve_adminq.c +0 -0
index 2b25c7f390..315e2456fd 100644

drivers/net/gve/base/gve_adminq.c line 9

Quoted 4 lines
drivers/net/gve/base/gve_adminq.c +1 -0
@@ -6,6 +6,7 @@
 #include "../gve_ethdev.h"
 #include "gve_adminq.h"
 #include "gve_register.h"
+#include "rte_common.h"

Looks like I forgot to fix the include formatting here. I will fix in the next revision.

Quoted 42 lines
drivers/net/gve/base/gve_adminq.c +8 -3
 #define GVE_MAX_ADMINQ_RELEASE_CHECK   500
 #define GVE_ADMINQ_SLEEP_LEN           20
@@ -946,15 +947,20 @@ static void
 gve_set_max_desc_cnt(struct gve_priv *priv,
        const struct gve_device_option_modify_ring *modify_ring)
 {
+       priv->max_rx_desc_cnt = be16_to_cpu(modify_ring->max_ring_size.rx);
+       priv->max_tx_desc_cnt = be16_to_cpu(modify_ring->max_ring_size.tx);
+
        if (priv->queue_format == GVE_DQO_RDA_FORMAT) {
                PMD_DRV_LOG(DEBUG, "Overriding max ring size from device for DQ "
                            "queue format to 4096.");
                priv->max_rx_desc_cnt = GVE_MAX_QUEUE_SIZE_DQO;
                priv->max_tx_desc_cnt = GVE_MAX_QUEUE_SIZE_DQO;
-               return;
+       } else if (priv->queue_format == GVE_GQI_QPL_FORMAT) {
+               priv->max_rx_desc_cnt = RTE_MIN(priv->max_rx_desc_cnt,
+                                               GVE_MAX_RING_SIZE_GQ_QPL);
+               priv->max_tx_desc_cnt = RTE_MIN(priv->max_tx_desc_cnt,
+                                               GVE_MAX_RING_SIZE_GQ_QPL);
        }
-       priv->max_rx_desc_cnt = be16_to_cpu(modify_ring->max_ring_size.rx);
-       priv->max_tx_desc_cnt = be16_to_cpu(modify_ring->max_ring_size.tx);
 }
 
 static void gve_enable_supported_features(struct gve_priv *priv,
drivers/net/gve/gve_ethdev.h +1 -1
index 4a7e5ecdf3..c9a176ff17 100644
@@ -21,6 +21,7 @@
 #define DQO_TX_MULTIPLIER 4
 
 #define GVE_DEFAULT_MAX_RING_SIZE      1024
+#define GVE_MAX_RING_SIZE_GQ_QPL       2048
 #define GVE_DEFAULT_MIN_RX_RING_SIZE   512
 #define GVE_DEFAULT_MIN_TX_RING_SIZE   256
 
--

2.55.0.rc2.803.g1fd1e6609c-goog

Signature · 1 line
Joshua Washington | Software Engineer | [email protected] | (414) 366-4423
#26 lore raw

v4 cover Stability fixes for GVE

In reply to Joshua Washington

This patch series consists of mostly unrelated fixes in the GVE driver.

v4:
- Remove unncessary include
v3:
- Fix 32-bit compilation issue
v2:
- Remove unused definition

Joshua Washington (9):
net/gve: clear out shared memory region for stats report net/gve: delay adding mbuf head to software ring net/gve: copy data to QPL buffer when mbuf read does not net/gve: validate buf ID before processing Rx packet net/gve: set mbuf to null in software ring after use net/gve: free ctx mbuf if packet dropped after first segment net/gve: increase range of DMA memzone ids to 64 bits net/gve: don't reset ring size bounds to default on reset net/gve: restrict max ring size in GQ QPL to 2K

 drivers/net/gve/base/gve_adminq.c | 11 ++++++--
 drivers/net/gve/base/gve_osdep.h  |  4 +--
 drivers/net/gve/gve_ethdev.c      |  8 ++++--
 drivers/net/gve/gve_ethdev.h      |  1 +
 drivers/net/gve/gve_rx.c          |  3 ++
 drivers/net/gve/gve_rx_dqo.c      |  6 ++++
 drivers/net/gve/gve_tx.c          | 46 ++++++++++++++++++-------------
 7 files changed, 52 insertions(+), 27 deletions(-)
Signature · 1 line
2.55.0.795.g602f6c329a-goog
#27 lore raw

PATCH v4 1/9 net/gve: clear out shared memory region for stats report

The stats report memzone is allocated from hugepage memory which could possibly have had sensitive data from a previous DPDK invocation.

Clear out the buffer before sharing the memory region with the virtual device to protect guest memory.

Fixes 458b53dec01e ("net/gve: enable imissed stats for GQ format")
---
 drivers/net/gve/gve_ethdev.c | 2 ++
 1 file changed, 2 insertions(+)
drivers/net/gve/gve_ethdev.c +2 -0
index 0b02dcb3ad..f73784a109 100644
@@ -306,6 +306,8 @@ gve_alloc_stats_report(struct gve_priv *priv,
 	if (!priv->stats_report_mem)
 		return -ENOMEM;
 
+	memset(priv->stats_report_mem->addr, 0, priv->stats_report_mem->len);
+
 	/* offset by skipping stats written by gve. */
 	priv->stats_start_idx = (GVE_TX_STATS_REPORT_NUM * nb_tx_queues) +
 		(GVE_RX_STATS_REPORT_NUM * nb_rx_queues);
Signature · 1 line
2.55.0.795.g602f6c329a-goog
#28 lore raw

PATCH v4 2/9 net/gve: delay adding mbuf head to software ring

In reply to Joshua Washington

The GQ TX datapath was set up to write the mbuf head into the sw_ring before writing the descriptors. This poses a problem because it's possible for the packet to be dropped due to lacking the FIFO space to do a proper TX. In such a case, the packet won't be sent, and will lead to leaked mbufs in the subsequent segments.

There is also no real reason that the head mbuf must be set in the sw_ring separately from the others; the mbuf chain is not actually walked as part of GQ TX.

Fixes a46583cf43c8 ("net/gve: support Rx/Tx")
---
 drivers/net/gve/gve_tx.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)
drivers/net/gve/gve_tx.c +4 -4
index 5c73c21b8d..59c82b04ed 100644
@@ -301,7 +301,6 @@ gve_tx_burst_qpl(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 			(uint32_t)(tx_offload.l2_len + tx_offload.l3_len + tx_offload.l4_len) :
 			tx_pkt->pkt_len;
 
-		sw_ring[sw_id] = tx_pkt;
 		if (!is_fifo_avail(txq, hlen)) {
 			gve_tx_clean(txq);
 			if (!is_fifo_avail(txq, hlen))
@@ -344,13 +343,14 @@ gve_tx_burst_qpl(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 		}
 
 		/* record mbuf in sw_ring for free */
-		for (i = 1; i < first->nb_segs; i++) {
+		for (i = 0; i < first->nb_segs; i++) {
+			if (!tx_pkt)
+				break;
+			sw_ring[sw_id] = tx_pkt;
 			sw_id = (sw_id + 1) & mask;
 			tx_pkt = tx_pkt->next;
-			sw_ring[sw_id] = tx_pkt;
 		}
 
-		sw_id = (sw_id + 1) & mask;
 		tx_id = (tx_id + 1) & mask;
 
 		txq->nb_free -= nb_used;
Signature · 1 line
2.55.0.795.g602f6c329a-goog
#29 lore raw

Re: [PATCH v4 2/9] net/gve: delay adding mbuf head to software ring

09/07/2026 06:07, Joshua Washington:

Quoted 6 lines

The GQ TX datapath was set up to write the mbuf head into the sw_ring before writing the descriptors. This poses a problem because it's possible for the packet to be dropped due to lacking the FIFO space to do a proper TX. In such a case, the packet won't be sent, and will lead to leaked mbufs in the subsequent segments.

Quoted 8 lines

There is also no real reason that the head mbuf must be set in the sw_ring separately from the others; the mbuf chain is not actually walked as part of GQ TX.

Fixes a46583cf43c8 ("net/gve: support Rx/Tx")

AI found this issue:

**Error:** Resource leak on early return from gve_tx_burst_qpl().

The patch modifies the loop to store mbufs in sw_ring only after checking FIFO availability, but introduces a leak when goto end_of_tx is taken after the first segment has been processed. At line 306-307:

```c
if (!is_fifo_avail(txq, hlen)) {
    gve_tx_clean(txq);
    if (!is_fifo_avail(txq, hlen))
        goto end_of_tx;
}
```

If the second is_fifo_avail() check fails and we goto end_of_tx, the current tx_pkt (which is the mbuf chain head) has not been added to sw_ring yet (that happens later in the modified loop at lines 346-351). The function will return without freeing tx_pkt, and the mbuf chain is lost.

**Suggested fix:** Free the current packet before goto end_of_tx:

```c
if (!is_fifo_avail(txq, hlen)) {
    gve_tx_clean(txq);
    if (!is_fifo_avail(txq, hlen)) {
        rte_pktmbuf_free(tx_pkt);
        goto end_of_tx;
    }
}
```

Similarly, for the TSO path at lines 314-316, if the FIFO space check fails, tx_pkt should be freed before the goto.

#30 lore raw

Re: [PATCH v4 2/9] net/gve: delay adding mbuf head to software ring

Quoted 45 lines · Thomas Monjalon

09/07/2026 06:07, Joshua Washington:

Quoted 6 lines

The GQ TX datapath was set up to write the mbuf head into the sw_ring before writing the descriptors. This poses a problem because it's possible for the packet to be dropped due to lacking the FIFO space to do a proper TX. In such a case, the packet won't be sent, and will lead to leaked mbufs in the subsequent segments.

Quoted 8 lines

There is also no real reason that the head mbuf must be set in the sw_ring separately from the others; the mbuf chain is not actually walked as part of GQ TX.

Fixes a46583cf43c8 ("net/gve: support Rx/Tx")

AI found this issue:

**Error:** Resource leak on early return from gve_tx_burst_qpl().

The patch modifies the loop to store mbufs in sw_ring only after checking FIFO availability, but introduces a leak when goto end_of_tx is taken after the first segment has been processed. At line 306-307:

```c
if (!is_fifo_avail(txq, hlen)) {
    gve_tx_clean(txq);
    if (!is_fifo_avail(txq, hlen))
        goto end_of_tx;
}
```

If the second is_fifo_avail() check fails and we goto end_of_tx, the current tx_pkt (which is the mbuf chain head) has not been added to sw_ring yet (that happens later in the modified loop at lines 346-351). The function will return without freeing tx_pkt, and the mbuf chain is lost.

**Suggested fix:** Free the current packet before goto end_of_tx:

```c
if (!is_fifo_avail(txq, hlen)) {
    gve_tx_clean(txq);
    if (!is_fifo_avail(txq, hlen)) {
        rte_pktmbuf_free(tx_pkt);
        goto end_of_tx;
    }
}
```
Quoted 1 line

Similarly, for the TSO path at lines 314-316, if the FIFO space check fails, tx_pkt should be freed before the goto.

I believe this is a hallucination. The driver does not free the TX packet in the case of a failed send because it does not count that packet in the packets sent to the application. It ultimately did not take ownership of the mbuf for the failing packet from the application, so it should not free that mbuf. The tx_id pointer is not incremented until after it is sure that the packet data can be copied to the DMA area (FIFO buffer), so there is no unintended descriptor write either, as far as I can tell.

-- Josh

#31 lore raw

PATCH v4 3/9 net/gve: copy data to QPL buffer when mbuf read does not

In reply to Joshua Washington

The rte_pktmbuf_read method does not guarantee that data will be copied from an mbuf. If the requested data is all contiguous, the method will instead return a pointer to the memory location within the buffer that should be read from, leaving the destination buffer empty.

This is problematic for TSO/multi-segment TX packets which only make use of two mbufs. If all data in the second mbuf is contiguous, the data will not be read to QPL memory.

Update the QPL copy logic to copy if the rte_pktmbuf_read does not.

Fixes a46583cf43c8 ("net/gve: support Rx/Tx")

---
v3:
- Fix 32-bit compile issue.
v2:
- Remove unused declaration and definition of addr

---
 drivers/net/gve/gve_tx.c | 38 +++++++++++++++++++++++---------------
 1 file changed, 23 insertions(+), 15 deletions(-)
drivers/net/gve/gve_tx.c +23 -15
index 59c82b04ed..c0400b07bf 100644
@@ -255,10 +255,10 @@ gve_tx_burst_qpl(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 	struct rte_mbuf **sw_ring = txq->sw_ring;
 	uint16_t mask = txq->nb_tx_desc - 1;
 	uint16_t tx_id = txq->tx_tail & mask;
-	uint64_t ol_flags, addr, fifo_addr;
 	uint32_t tx_tail = txq->tx_tail;
 	struct rte_mbuf *tx_pkt, *first;
 	uint16_t sw_id = txq->sw_tail;
+	uint64_t ol_flags, fifo_addr;
 	uint16_t nb_used, i;
 	uint64_t bytes = 0;
 	uint16_t nb_tx = 0;
@@ -273,6 +273,9 @@ gve_tx_burst_qpl(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 		gve_tx_clean_swr_qpl(txq);
 
 	for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
+		const void *mbuf_header_addr;
+		void *qpl_write_addr;
+
 		tx_pkt = *tx_pkts++;
 		ol_flags = tx_pkt->ol_flags;
 
@@ -306,7 +309,6 @@ gve_tx_burst_qpl(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 			if (!is_fifo_avail(txq, hlen))
 				goto end_of_tx;
 		}
-		addr = (uint64_t)(tx_pkt->buf_addr) + tx_pkt->data_off;
 		fifo_addr = gve_tx_alloc_from_fifo(txq, tx_id, hlen);
 
 		/* For TSO, check if there's enough fifo space for data first */
@@ -317,26 +319,32 @@ gve_tx_burst_qpl(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 					goto end_of_tx;
 			}
 		}
-		if (tx_pkt->nb_segs == 1 || ol_flags & RTE_MBUF_F_TX_TCP_SEG)
-			rte_memcpy((void *)(size_t)(fifo_addr + txq->fifo_base),
-				   (void *)(size_t)addr, hlen);
-		else
-			rte_pktmbuf_read(tx_pkt, 0, hlen,
-					 (void *)(size_t)(fifo_addr + txq->fifo_base));
+
+		qpl_write_addr = (void *)(size_t)(fifo_addr + txq->fifo_base);
+		mbuf_header_addr = rte_pktmbuf_read(tx_pkt, 0, hlen, qpl_write_addr);
+
+		/* Header data is linear in the mbuf head. Copy directly. */
+		if (mbuf_header_addr != qpl_write_addr)
+			rte_memcpy(qpl_write_addr, mbuf_header_addr, hlen);
+
 		gve_tx_fill_pkt_desc(txd, tx_pkt, nb_used, hlen, fifo_addr);
 
 		if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
+			const void *mbuf_payload_addr;
+
 			tx_id = (tx_id + 1) & mask;
 			txd = &txr[tx_id];
-			addr = (uint64_t)(tx_pkt->buf_addr) + tx_pkt->data_off + hlen;
 			fifo_addr = gve_tx_alloc_from_fifo(txq, tx_id, tx_pkt->pkt_len - hlen);
-			if (tx_pkt->nb_segs == 1)
-				rte_memcpy((void *)(size_t)(fifo_addr + txq->fifo_base),
-					   (void *)(size_t)addr,
+			qpl_write_addr = (void *)(size_t)(txq->fifo_base + fifo_addr);
+			mbuf_payload_addr = rte_pktmbuf_read(tx_pkt, hlen, tx_pkt->pkt_len - hlen,
+							     qpl_write_addr);
+
+			/* Payload data is contiguous. Take the offset from the
+			 * read request and copy from there.
+			 */
+			if (mbuf_payload_addr != qpl_write_addr)
+				rte_memcpy(qpl_write_addr, mbuf_payload_addr,
 					   tx_pkt->pkt_len - hlen);
-			else
-				rte_pktmbuf_read(tx_pkt, hlen, tx_pkt->pkt_len - hlen,
-						 (void *)(size_t)(fifo_addr + txq->fifo_base));
 
 			gve_tx_fill_seg_desc(txd, ol_flags, tx_offload,
 					     tx_pkt->pkt_len - hlen, fifo_addr);
Signature · 1 line
2.55.0.795.g602f6c329a-goog
#32 lore raw

PATCH v4 4/9 net/gve: validate buf ID before processing Rx packet

In reply to Joshua Washington

The buffer id is part of the RX completion descriptor for packets in the DQ format. This value can technically go up to 64K, while the max RX ring size is 4K, meaning that there could similarly be an expected 4K RX buffer IDs. Validate that the RX buffer ID is valid before attempting to access it in the sw_ring to prevent a potential out of bounds in the event of a hardware error.

Fixes 1aed73b23ac0 ("net/gve: support out-of-order completions on DQ Rx")
---
 drivers/net/gve/gve_rx_dqo.c | 5 +++++
 1 file changed, 5 insertions(+)
drivers/net/gve/gve_rx_dqo.c +5 -0
index cc343f3fd8..c4e2d32067 100644
@@ -200,6 +200,11 @@ gve_rx_burst_dqo(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 		}
 
 		rx_buf_id = rte_le_to_cpu_16(rx_desc->buf_id);
+		if (unlikely(rx_buf_id >= rxq->nb_rx_desc)) {
+			PMD_DRV_DP_LOG(ERR, "Invalid buf_id %d", rx_buf_id);
+			continue;
+		}
+
 		rxm = rxq->sw_ring[rx_buf_id];
 		gve_completed_buf_list_push(rxq, rx_buf_id);
 
Signature · 1 line
2.55.0.795.g602f6c329a-goog
#33 lore raw

PATCH v4 5/9 net/gve: set mbuf to null in software ring after use

In reply to Joshua Washington

Currently, it is possible for mbufs to be uncleared in the sw_ring after being returned to the application. This causes an erroneous dual-ownership over the buffer until GVE cleans the buffer queue and posts new mbufs, overwriting the older pointers. It is possible in such a case for a double free to occur while tearing down rings, as both the application and the driver could attempt to free the same mbuf. Release ownership of the mbuf from the sw_ring as soon as appropriate to avoid such a scenario.

Fixes a46583cf43c8 ("net/gve: support Rx/Tx")
Fixes 45da16b5b181 ("net/gve: support basic Rx data path for DQO")
---
 drivers/net/gve/gve_rx.c     | 1 +
 drivers/net/gve/gve_rx_dqo.c | 1 +
 2 files changed, 2 insertions(+)
drivers/net/gve/gve_rx.c +1 -0
index 625649cdcf..cda87af294 100644
@@ -152,6 +152,7 @@ gve_rx(struct gve_rx_queue *rxq, volatile struct gve_rx_desc *rxd, uint16_t rx_i
 
 	rxe = rxq->sw_ring[rx_id];
 	gve_rx_mbuf(rxq, rxe, frag_size, rx_id);
+	rxq->sw_ring[rx_id] = NULL;
 	rxq->stats.bytes += frag_size;
 
 	if (is_first_frag) {
drivers/net/gve/gve_rx_dqo.c +1 -0
index c4e2d32067..3665d9e4cd 100644
@@ -207,6 +207,7 @@ gve_rx_burst_dqo(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 
 		rxm = rxq->sw_ring[rx_buf_id];
 		gve_completed_buf_list_push(rxq, rx_buf_id);
+		rxq->sw_ring[rx_buf_id] = NULL;
 
 		/* Free buffer and report error. */
 		if (unlikely(rx_desc->rx_error)) {
Signature · 1 line
2.55.0.795.g602f6c329a-goog
#34 lore raw

PATCH v4 6/9 net/gve: free ctx mbuf if packet dropped after first segment

In reply to Joshua Washington

GVE GQ has support for multi-descriptor RX. It is possible for a packet to be dropped after the first descriptor has been processed an mbuf has been added to the context. In such a case, the mbuf head should be freed before clearing the context so that mbufs aren't leaked.

In addition, clear mbuf from sw_ring after adding the packet to the context to avoid double-freeing buffers that have not been reposted, but have been reported to the application.

Fixes 496d4d2c8b54 ("net/gve: support jumbo frame for GQI")
---
 drivers/net/gve/gve_rx.c | 2 ++
 1 file changed, 2 insertions(+)
drivers/net/gve/gve_rx.c +2 -0
index cda87af294..567b82d020 100644
@@ -205,6 +205,8 @@ gve_rx_burst(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 		if (gve_rx(rxq, rxd, rx_id)) {
 			if (!ctx->drop_pkt)
 				rx_pkts[nb_rx++] = ctx->mbuf_head;
+			else if (ctx->mbuf_head != NULL)
+				rte_pktmbuf_free(ctx->mbuf_head);
 			rxq->nb_avail += ctx->total_frags;
 			gve_rx_ctx_clear(ctx);
 		}
Signature · 1 line
2.55.0.795.g602f6c329a-goog
#35 lore raw

PATCH v4 7/9 net/gve: increase range of DMA memzone ids to 64 bits

In reply to Joshua Washington

Long running programs can very easily eclipse this 16-bit range, leading to name collisions and failed DMA region allocations despite there being plenty of available memory.

Fixes c9ba2caf6302 ("net/gve/base: add OS-specific implementation")
---
 drivers/net/gve/base/gve_osdep.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
drivers/net/gve/base/gve_osdep.h +2 -2
index c47ce4da85..55629a0e1a 100644
@@ -175,14 +175,14 @@ struct gve_dma_mem {
 static inline void *
 gve_alloc_dma_mem(struct gve_dma_mem *mem, u64 size)
 {
-	static RTE_ATOMIC(uint16_t) gve_dma_memzone_id;
+	static RTE_ATOMIC(uint64_t) gve_dma_memzone_id;
 	const struct rte_memzone *mz = NULL;
 	char z_name[RTE_MEMZONE_NAMESIZE];
 
 	if (!mem)
 		return NULL;
 
-	snprintf(z_name, sizeof(z_name), "gve_dma_%u",
+	snprintf(z_name, sizeof(z_name), "gve_dma_%" PRIu64,
 		 rte_atomic_fetch_add_explicit(&gve_dma_memzone_id, 1, rte_memory_order_relaxed));
 	mz = rte_memzone_reserve_aligned(z_name, size, SOCKET_ID_ANY,
 					 RTE_MEMZONE_IOVA_CONTIG,
Signature · 1 line
2.55.0.795.g602f6c329a-goog
#36 lore raw

PATCH v4 8/9 net/gve: don't reset ring size bounds to default on reset

In reply to Joshua Washington

On device reset, GVE skips describe_device functionality, as the device is not expected to change on a reset. However, before skipping the describe_device functionality, GVE still sets the ring sizes to their default values. This effectively removes the ability to create queues with higher-than-default descriptor counts after a reset.

Fix this behavior by only setting the default ring size bounds is describe_device is being executed.

Fixes 1bf64edce3c4 ("net/gve: add reset path")
---
 drivers/net/gve/gve_ethdev.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
drivers/net/gve/gve_ethdev.c +3 -3
index f73784a109..c990920a4d 100644
@@ -1579,12 +1579,12 @@ gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
 		goto free_adminq;
 	}
 
-	/* Set default descriptor counts */
-	gve_set_default_ring_size_bounds(priv);
-
 	if (skip_describe_device)
 		goto setup_device;
 
+	/* Set default descriptor counts */
+	gve_set_default_ring_size_bounds(priv);
+
 	/* Get the initial information we need from the device */
 	err = gve_adminq_describe_device(priv);
 	if (err) {
Signature · 1 line
2.55.0.795.g602f6c329a-goog
#37 lore raw

PATCH v4 9/9 net/gve: restrict max ring size in GQ QPL to 2K

In reply to Joshua Washington

The GQ QPL queue format has a maximum supported ring size of 2k. However, it is possible in some cases for the device to pass a larger ring size as the max ring size. Restrict the ring size in the driver to ensure that rings with invalid queue depths are not created.

Fixes cde01d164f8f ("net/gve: support modifying ring size in GQ format")

---
v4
- Remove unnecesary include

---
 drivers/net/gve/base/gve_adminq.c | 11 ++++++++---
 drivers/net/gve/gve_ethdev.h      |  1 +
 2 files changed, 9 insertions(+), 3 deletions(-)
drivers/net/gve/base/gve_adminq.c +8 -3
index 2b25c7f390..89bf669c24 100644
@@ -946,15 +946,20 @@ static void
 gve_set_max_desc_cnt(struct gve_priv *priv,
 	const struct gve_device_option_modify_ring *modify_ring)
 {
+	priv->max_rx_desc_cnt = be16_to_cpu(modify_ring->max_ring_size.rx);
+	priv->max_tx_desc_cnt = be16_to_cpu(modify_ring->max_ring_size.tx);
+
 	if (priv->queue_format == GVE_DQO_RDA_FORMAT) {
 		PMD_DRV_LOG(DEBUG, "Overriding max ring size from device for DQ "
 			    "queue format to 4096.");
 		priv->max_rx_desc_cnt = GVE_MAX_QUEUE_SIZE_DQO;
 		priv->max_tx_desc_cnt = GVE_MAX_QUEUE_SIZE_DQO;
-		return;
+	} else if (priv->queue_format == GVE_GQI_QPL_FORMAT) {
+		priv->max_rx_desc_cnt = RTE_MIN(priv->max_rx_desc_cnt,
+						GVE_MAX_RING_SIZE_GQ_QPL);
+		priv->max_tx_desc_cnt = RTE_MIN(priv->max_tx_desc_cnt,
+						GVE_MAX_RING_SIZE_GQ_QPL);
 	}
-	priv->max_rx_desc_cnt = be16_to_cpu(modify_ring->max_ring_size.rx);
-	priv->max_tx_desc_cnt = be16_to_cpu(modify_ring->max_ring_size.tx);
 }
 
 static void gve_enable_supported_features(struct gve_priv *priv,
drivers/net/gve/gve_ethdev.h +1 -0
index 4a7e5ecdf3..c9a176ff17 100644
@@ -21,6 +21,7 @@
 #define DQO_TX_MULTIPLIER 4
 
 #define GVE_DEFAULT_MAX_RING_SIZE	1024
+#define GVE_MAX_RING_SIZE_GQ_QPL	2048
 #define GVE_DEFAULT_MIN_RX_RING_SIZE	512
 #define GVE_DEFAULT_MIN_TX_RING_SIZE	256
 
Signature · 1 line
2.55.0.795.g602f6c329a-goog
#38 lore raw
In reply to Joshua Washington
Quoted 10 lines

Joshua Washington (9):
net/gve: clear out shared memory region for stats report net/gve: delay adding mbuf head to software ring net/gve: copy data to QPL buffer when mbuf read does not net/gve: validate buf ID before processing Rx packet net/gve: set mbuf to null in software ring after use net/gve: free ctx mbuf if packet dropped after first segment net/gve: increase range of DMA memzone ids to 64 bits net/gve: don't reset ring size bounds to default on reset net/gve: restrict max ring size in GQ QPL to 2K

Applied, thanks.