Skip to contentSkip to conversation

net/gve: change QPLs to be queue resources

PATCH Superseded Joshua Washington 2 participants 4 messages Jun 13, 2024 – Jul 5, 2024 Network drivers

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 net/gve: change QPLs to be queue resources

Prior to this change, queue page lists (QPLs) were kept as device resources, being stored in the gve_priv struct. This does not make sense because each QPL inherently belongs to a single queue.

This change moves all QPL resources into the queues themselves, and couples QPL allocation/registration and de-registration/deallocation with the queue creation and destruction processes, respectively. Before this change, QPL structs part of gve_priv were allocated as part of driver initialization, which similarly does not make sense.

Change-Id I3e2de1facef7449cd581d7bfa0dc2a9ca80bd86a
---
 .mailmap                     |   2 +-
 drivers/net/gve/gve_ethdev.c | 159 ++++++++++++++++++++---------------
 drivers/net/gve/gve_ethdev.h |   9 +-
 drivers/net/gve/gve_rx.c     |  22 +++--
 drivers/net/gve/gve_tx.c     |  21 +++--
 5 files changed, 127 insertions(+), 86 deletions(-)
.mailmap +1 -1
index 6b396107d0..48af16edf5 100644
@@ -503,7 +503,7 @@ Harold Huang <[email protected]>
 Harrison McCullough <[email protected]>
 Harry van Haaren <[email protected]>
 Harshad Narayane <[email protected]>
-Harshitha Ramamurthy <[email protected]>
+Harshitha Ramamurthy <[email protected]>
 Hasan Alayli <[email protected]>
 Hayato Momma <[email protected]>
 Heinrich Kuhn <[email protected]> <[email protected]>
drivers/net/gve/gve_ethdev.c +90 -69

213 lines not on this page · raw message

drivers/net/gve/gve_ethdev.h +7 -2
index 9b19fc55e3..ca8c6404fd 100644
@@ -288,8 +288,6 @@ struct gve_priv {
 	uint16_t max_mtu;
 	struct rte_ether_addr dev_addr; /* mac address */
 
-	struct gve_queue_page_list *qpl;
-
 	struct gve_tx_queue **txqs;
 	struct gve_rx_queue **rxqs;
 
@@ -416,6 +414,13 @@ gve_set_rx_function(struct rte_eth_dev *dev);
 void
 gve_set_tx_function(struct rte_eth_dev *dev);
 
+struct gve_queue_page_list *
+gve_setup_queue_page_list(struct gve_priv *priv, uint16_t queue_id, bool is_rx,
+	uint32_t num_pages);
+int
+gve_teardown_queue_page_list(struct gve_priv *priv,
+	struct gve_queue_page_list *qpl);
+
 /* Below functions are used for DQO */
 
 int
drivers/net/gve/gve_rx.c +15 -7
index 36a1b73c65..41987ec870 100644
@@ -279,7 +279,7 @@ gve_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
 		return;
 
 	if (q->is_gqi_qpl) {
-		gve_adminq_unregister_page_list(q->hw, q->qpl->id);
+		gve_teardown_queue_page_list(q->hw, q->qpl);
 		q->qpl = NULL;
 	}
 
@@ -382,11 +382,15 @@ gve_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
 	}
 	rxq->rx_data_ring = (union gve_rx_data_slot *)mz->addr;
 	rxq->data_mz = mz;
+
+	/* Allocate and register QPL for the queue. */
 	if (rxq->is_gqi_qpl) {
-		rxq->qpl = &hw->qpl[rxq->ntfy_id];
-		err = gve_adminq_register_page_list(hw, rxq->qpl);
-		if (err != 0) {
-			PMD_DRV_LOG(ERR, "Failed to register qpl %u", queue_id);
+		rxq->qpl = gve_setup_queue_page_list(hw, queue_id, true,
+						     hw->rx_data_slot_cnt);
+		if (!rxq->qpl) {
+			PMD_DRV_LOG(ERR,
+				    "Failed to alloc rx qpl for queue %hu.",
+				    queue_id);
 			goto err_data_ring;
 		}
 	}
@@ -397,7 +401,7 @@ gve_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
 	if (mz == NULL) {
 		PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for RX resource");
 		err = -ENOMEM;
-		goto err_data_ring;
+		goto err_qpl;
 	}
 	rxq->qres = (struct gve_queue_resources *)mz->addr;
 	rxq->qres_mz = mz;
@@ -407,7 +411,11 @@ gve_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
 	dev->data->rx_queues[queue_id] = rxq;
 
 	return 0;
-
+err_qpl:
+	if (rxq->is_gqi_qpl) {
+		gve_teardown_queue_page_list(hw, rxq->qpl);
+		rxq->qpl = NULL;
+	}
 err_data_ring:
 	rte_memzone_free(rxq->data_mz);
 err_rx_ring:
drivers/net/gve/gve_tx.c +14 -7
index 2e0d001109..70d3ef060c 100644
@@ -536,7 +536,7 @@ gve_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
 		return;
 
 	if (q->is_gqi_qpl) {
-		gve_adminq_unregister_page_list(q->hw, q->qpl->id);
+		gve_teardown_queue_page_list(q->hw, q->qpl);
 		rte_free(q->iov_ring);
 		q->qpl = NULL;
 	}
@@ -619,6 +619,7 @@ gve_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id, uint16_t nb_desc,
 	txq->tx_ring_phys_addr = mz->iova;
 	txq->mz = mz;
 
+	/* QPL-specific allocations. */
 	if (txq->is_gqi_qpl) {
 		txq->iov_ring = rte_zmalloc_socket("gve tx iov ring",
 						   sizeof(struct gve_tx_iovec) * nb_desc,
@@ -628,10 +629,12 @@ gve_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id, uint16_t nb_desc,
 			err = -ENOMEM;
 			goto err_tx_ring;
 		}
-		txq->qpl = &hw->qpl[queue_id];
-		err = gve_adminq_register_page_list(hw, txq->qpl);
-		if (err != 0) {
-			PMD_DRV_LOG(ERR, "Failed to register qpl %u", queue_id);
+
+		txq->qpl = gve_setup_queue_page_list(hw, queue_id, false,
+						     hw->tx_pages_per_qpl);
+		if (!txq->qpl) {
+			PMD_DRV_LOG(ERR, "Failed to alloc tx qpl for queue %hu.",
+				    queue_id);
 			goto err_iov_ring;
 		}
 	}
@@ -641,7 +644,7 @@ gve_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id, uint16_t nb_desc,
 	if (mz == NULL) {
 		PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for TX resource");
 		err = -ENOMEM;
-		goto err_iov_ring;
+		goto err_qpl;
 	}
 	txq->qres = (struct gve_queue_resources *)mz->addr;
 	txq->qres_mz = mz;
@@ -651,7 +654,11 @@ gve_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id, uint16_t nb_desc,
 	dev->data->tx_queues[queue_id] = txq;
 
 	return 0;
-
+err_qpl:
+	if (txq->is_gqi_qpl) {
+		gve_teardown_queue_page_list(hw, txq->qpl);
+		txq->qpl = NULL;
+	}
 err_iov_ring:
 	if (txq->is_gqi_qpl)
 		rte_free(txq->iov_ring);
Signature · 1 line
2.45.2.627.g7a2c4fd464-goog
#1 lore raw

PATCH v2 1/1 net/gve: change QPLs to be queue resources

Prior to this change, queue page lists (QPLs) were kept as device resources, being stored in the gve_priv struct. This does not make sense because each QPL inherently belongs to a single queue.

This change moves all QPL resources into the queues themselves, and couples QPL allocation/registration and de-registration/deallocation with the queue creation and destruction processes, respectively. Before this change, QPL structs part of gve_priv were allocated as part of driver initialization, which similarly does not make sense.

Change-Id I7098eae9d2469463c4f8dc76e770795df9b0ca91
---
 .mailmap                     |   2 +-
 drivers/net/gve/gve_ethdev.c | 159 ++++++++++++++++++++---------------
 drivers/net/gve/gve_ethdev.h |   9 +-
 drivers/net/gve/gve_rx.c     |  22 +++--
 drivers/net/gve/gve_tx.c     |  21 +++--
 5 files changed, 127 insertions(+), 86 deletions(-)
.mailmap +1 -1
index 6b396107d0..48af16edf5 100644
@@ -503,7 +503,7 @@ Harold Huang <[email protected]>
 Harrison McCullough <[email protected]>
 Harry van Haaren <[email protected]>
 Harshad Narayane <[email protected]>
-Harshitha Ramamurthy <[email protected]>
+Harshitha Ramamurthy <[email protected]>
 Hasan Alayli <[email protected]>
 Hayato Momma <[email protected]>
 Heinrich Kuhn <[email protected]> <[email protected]>
drivers/net/gve/gve_ethdev.c +90 -69

213 lines not on this page · raw message

drivers/net/gve/gve_ethdev.h +7 -2
index 9b19fc55e3..ca8c6404fd 100644
@@ -288,8 +288,6 @@ struct gve_priv {
 	uint16_t max_mtu;
 	struct rte_ether_addr dev_addr; /* mac address */
 
-	struct gve_queue_page_list *qpl;
-
 	struct gve_tx_queue **txqs;
 	struct gve_rx_queue **rxqs;
 
@@ -416,6 +414,13 @@ gve_set_rx_function(struct rte_eth_dev *dev);
 void
 gve_set_tx_function(struct rte_eth_dev *dev);
 
+struct gve_queue_page_list *
+gve_setup_queue_page_list(struct gve_priv *priv, uint16_t queue_id, bool is_rx,
+	uint32_t num_pages);
+int
+gve_teardown_queue_page_list(struct gve_priv *priv,
+	struct gve_queue_page_list *qpl);
+
 /* Below functions are used for DQO */
 
 int
drivers/net/gve/gve_rx.c +15 -7
index 36a1b73c65..41987ec870 100644
@@ -279,7 +279,7 @@ gve_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
 		return;
 
 	if (q->is_gqi_qpl) {
-		gve_adminq_unregister_page_list(q->hw, q->qpl->id);
+		gve_teardown_queue_page_list(q->hw, q->qpl);
 		q->qpl = NULL;
 	}
 
@@ -382,11 +382,15 @@ gve_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
 	}
 	rxq->rx_data_ring = (union gve_rx_data_slot *)mz->addr;
 	rxq->data_mz = mz;
+
+	/* Allocate and register QPL for the queue. */
 	if (rxq->is_gqi_qpl) {
-		rxq->qpl = &hw->qpl[rxq->ntfy_id];
-		err = gve_adminq_register_page_list(hw, rxq->qpl);
-		if (err != 0) {
-			PMD_DRV_LOG(ERR, "Failed to register qpl %u", queue_id);
+		rxq->qpl = gve_setup_queue_page_list(hw, queue_id, true,
+						     hw->rx_data_slot_cnt);
+		if (!rxq->qpl) {
+			PMD_DRV_LOG(ERR,
+				    "Failed to alloc rx qpl for queue %hu.",
+				    queue_id);
 			goto err_data_ring;
 		}
 	}
@@ -397,7 +401,7 @@ gve_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
 	if (mz == NULL) {
 		PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for RX resource");
 		err = -ENOMEM;
-		goto err_data_ring;
+		goto err_qpl;
 	}
 	rxq->qres = (struct gve_queue_resources *)mz->addr;
 	rxq->qres_mz = mz;
@@ -407,7 +411,11 @@ gve_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
 	dev->data->rx_queues[queue_id] = rxq;
 
 	return 0;
-
+err_qpl:
+	if (rxq->is_gqi_qpl) {
+		gve_teardown_queue_page_list(hw, rxq->qpl);
+		rxq->qpl = NULL;
+	}
 err_data_ring:
 	rte_memzone_free(rxq->data_mz);
 err_rx_ring:
drivers/net/gve/gve_tx.c +14 -7
index 2e0d001109..70d3ef060c 100644
@@ -536,7 +536,7 @@ gve_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
 		return;
 
 	if (q->is_gqi_qpl) {
-		gve_adminq_unregister_page_list(q->hw, q->qpl->id);
+		gve_teardown_queue_page_list(q->hw, q->qpl);
 		rte_free(q->iov_ring);
 		q->qpl = NULL;
 	}
@@ -619,6 +619,7 @@ gve_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id, uint16_t nb_desc,
 	txq->tx_ring_phys_addr = mz->iova;
 	txq->mz = mz;
 
+	/* QPL-specific allocations. */
 	if (txq->is_gqi_qpl) {
 		txq->iov_ring = rte_zmalloc_socket("gve tx iov ring",
 						   sizeof(struct gve_tx_iovec) * nb_desc,
@@ -628,10 +629,12 @@ gve_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id, uint16_t nb_desc,
 			err = -ENOMEM;
 			goto err_tx_ring;
 		}
-		txq->qpl = &hw->qpl[queue_id];
-		err = gve_adminq_register_page_list(hw, txq->qpl);
-		if (err != 0) {
-			PMD_DRV_LOG(ERR, "Failed to register qpl %u", queue_id);
+
+		txq->qpl = gve_setup_queue_page_list(hw, queue_id, false,
+						     hw->tx_pages_per_qpl);
+		if (!txq->qpl) {
+			PMD_DRV_LOG(ERR, "Failed to alloc tx qpl for queue %hu.",
+				    queue_id);
 			goto err_iov_ring;
 		}
 	}
@@ -641,7 +644,7 @@ gve_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id, uint16_t nb_desc,
 	if (mz == NULL) {
 		PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for TX resource");
 		err = -ENOMEM;
-		goto err_iov_ring;
+		goto err_qpl;
 	}
 	txq->qres = (struct gve_queue_resources *)mz->addr;
 	txq->qres_mz = mz;
@@ -651,7 +654,11 @@ gve_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id, uint16_t nb_desc,
 	dev->data->tx_queues[queue_id] = txq;
 
 	return 0;
-
+err_qpl:
+	if (txq->is_gqi_qpl) {
+		gve_teardown_queue_page_list(hw, txq->qpl);
+		txq->qpl = NULL;
+	}
 err_iov_ring:
 	if (txq->is_gqi_qpl)
 		rte_free(txq->iov_ring);
Signature · 1 line
2.45.2.627.g7a2c4fd464-goog
#2 lore raw

PATCH v2 1/1 net/gve: change QPLs to be queue resources

In reply to Joshua Washington

Prior to this change, queue page lists (QPLs) were kept as device resources, being stored in the gve_priv struct. This does not make sense because each QPL inherently belongs to a single queue.

This change moves all QPL resources into the queues themselves, and couples QPL allocation/registration and de-registration/deallocation with the queue creation and destruction processes, respectively. Before this change, QPL structs part of gve_priv were allocated as part of driver initialization, which similarly does not make sense.

---
 .mailmap                     |   2 +-
 drivers/net/gve/gve_ethdev.c | 159 ++++++++++++++++++++---------------
 drivers/net/gve/gve_ethdev.h |   9 +-
 drivers/net/gve/gve_rx.c     |  22 +++--
 drivers/net/gve/gve_tx.c     |  21 +++--
 5 files changed, 127 insertions(+), 86 deletions(-)
.mailmap +1 -1
index 6b396107d0..48af16edf5 100644
@@ -503,7 +503,7 @@ Harold Huang <[email protected]>
 Harrison McCullough <[email protected]>
 Harry van Haaren <[email protected]>
 Harshad Narayane <[email protected]>
-Harshitha Ramamurthy <[email protected]>
+Harshitha Ramamurthy <[email protected]>
 Hasan Alayli <[email protected]>
 Hayato Momma <[email protected]>
 Heinrich Kuhn <[email protected]> <[email protected]>
drivers/net/gve/gve_ethdev.c +90 -69

213 lines not on this page · raw message

drivers/net/gve/gve_ethdev.h +7 -2
index 9b19fc55e3..ca8c6404fd 100644
@@ -288,8 +288,6 @@ struct gve_priv {
 	uint16_t max_mtu;
 	struct rte_ether_addr dev_addr; /* mac address */
 
-	struct gve_queue_page_list *qpl;
-
 	struct gve_tx_queue **txqs;
 	struct gve_rx_queue **rxqs;
 
@@ -416,6 +414,13 @@ gve_set_rx_function(struct rte_eth_dev *dev);
 void
 gve_set_tx_function(struct rte_eth_dev *dev);
 
+struct gve_queue_page_list *
+gve_setup_queue_page_list(struct gve_priv *priv, uint16_t queue_id, bool is_rx,
+	uint32_t num_pages);
+int
+gve_teardown_queue_page_list(struct gve_priv *priv,
+	struct gve_queue_page_list *qpl);
+
 /* Below functions are used for DQO */
 
 int
drivers/net/gve/gve_rx.c +15 -7
index 36a1b73c65..41987ec870 100644
@@ -279,7 +279,7 @@ gve_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
 		return;
 
 	if (q->is_gqi_qpl) {
-		gve_adminq_unregister_page_list(q->hw, q->qpl->id);
+		gve_teardown_queue_page_list(q->hw, q->qpl);
 		q->qpl = NULL;
 	}
 
@@ -382,11 +382,15 @@ gve_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
 	}
 	rxq->rx_data_ring = (union gve_rx_data_slot *)mz->addr;
 	rxq->data_mz = mz;
+
+	/* Allocate and register QPL for the queue. */
 	if (rxq->is_gqi_qpl) {
-		rxq->qpl = &hw->qpl[rxq->ntfy_id];
-		err = gve_adminq_register_page_list(hw, rxq->qpl);
-		if (err != 0) {
-			PMD_DRV_LOG(ERR, "Failed to register qpl %u", queue_id);
+		rxq->qpl = gve_setup_queue_page_list(hw, queue_id, true,
+						     hw->rx_data_slot_cnt);
+		if (!rxq->qpl) {
+			PMD_DRV_LOG(ERR,
+				    "Failed to alloc rx qpl for queue %hu.",
+				    queue_id);
 			goto err_data_ring;
 		}
 	}
@@ -397,7 +401,7 @@ gve_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
 	if (mz == NULL) {
 		PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for RX resource");
 		err = -ENOMEM;
-		goto err_data_ring;
+		goto err_qpl;
 	}
 	rxq->qres = (struct gve_queue_resources *)mz->addr;
 	rxq->qres_mz = mz;
@@ -407,7 +411,11 @@ gve_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
 	dev->data->rx_queues[queue_id] = rxq;
 
 	return 0;
-
+err_qpl:
+	if (rxq->is_gqi_qpl) {
+		gve_teardown_queue_page_list(hw, rxq->qpl);
+		rxq->qpl = NULL;
+	}
 err_data_ring:
 	rte_memzone_free(rxq->data_mz);
 err_rx_ring:
drivers/net/gve/gve_tx.c +14 -7
index 2e0d001109..70d3ef060c 100644
@@ -536,7 +536,7 @@ gve_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
 		return;
 
 	if (q->is_gqi_qpl) {
-		gve_adminq_unregister_page_list(q->hw, q->qpl->id);
+		gve_teardown_queue_page_list(q->hw, q->qpl);
 		rte_free(q->iov_ring);
 		q->qpl = NULL;
 	}
@@ -619,6 +619,7 @@ gve_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id, uint16_t nb_desc,
 	txq->tx_ring_phys_addr = mz->iova;
 	txq->mz = mz;
 
+	/* QPL-specific allocations. */
 	if (txq->is_gqi_qpl) {
 		txq->iov_ring = rte_zmalloc_socket("gve tx iov ring",
 						   sizeof(struct gve_tx_iovec) * nb_desc,
@@ -628,10 +629,12 @@ gve_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id, uint16_t nb_desc,
 			err = -ENOMEM;
 			goto err_tx_ring;
 		}
-		txq->qpl = &hw->qpl[queue_id];
-		err = gve_adminq_register_page_list(hw, txq->qpl);
-		if (err != 0) {
-			PMD_DRV_LOG(ERR, "Failed to register qpl %u", queue_id);
+
+		txq->qpl = gve_setup_queue_page_list(hw, queue_id, false,
+						     hw->tx_pages_per_qpl);
+		if (!txq->qpl) {
+			PMD_DRV_LOG(ERR, "Failed to alloc tx qpl for queue %hu.",
+				    queue_id);
 			goto err_iov_ring;
 		}
 	}
@@ -641,7 +644,7 @@ gve_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id, uint16_t nb_desc,
 	if (mz == NULL) {
 		PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for TX resource");
 		err = -ENOMEM;
-		goto err_iov_ring;
+		goto err_qpl;
 	}
 	txq->qres = (struct gve_queue_resources *)mz->addr;
 	txq->qres_mz = mz;
@@ -651,7 +654,11 @@ gve_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id, uint16_t nb_desc,
 	dev->data->tx_queues[queue_id] = txq;
 
 	return 0;
-
+err_qpl:
+	if (txq->is_gqi_qpl) {
+		gve_teardown_queue_page_list(hw, txq->qpl);
+		txq->qpl = NULL;
+	}
 err_iov_ring:
 	if (txq->is_gqi_qpl)
 		rte_free(txq->iov_ring);
Signature · 1 line
2.45.2.627.g7a2c4fd464-goog
Applied
#3 lore raw
Quoted 10 lines · Joshua Washington

Prior to this change, queue page lists (QPLs) were kept as device resources, being stored in the gve_priv struct. This does not make sense because each QPL inherently belongs to a single queue.

This change moves all QPL resources into the queues themselves, and couples QPL allocation/registration and de-registration/deallocation with the queue creation and destruction processes, respectively. Before this change, QPL structs part of gve_priv were allocated as part of driver initialization, which similarly does not make sense.

Quoted 4 lines

Applied to dpdk-next-net/main, thanks.

(fixed .mailmap while fixing, it should list all emails in the git history, first one being primary one.)