Skip to contentSkip to conversation

net/gve: add support for basic stats

PATCH Applied Junfeng Guo 5 participants 17 messages Nov 24, 2022 – Feb 3, 2023 Network drivers

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 net/gve: add support for basic stats

Add support for dev_ops of stats_get and stats_reset.

Queue stats update will be moved into xstat [1], but the basic stats items may still be required. So we just keep the remaining ones and will implement the queue stats via xstats in the coming release.

[1]
https://elixir.bootlin.com/dpdk/v22.07/ \
source/doc/guides/rel_notes/deprecation.rst#L118

---
 doc/guides/nics/features/gve.ini |  1 +
 drivers/net/gve/gve_ethdev.c     | 60 ++++++++++++++++++++++++++++++++
 drivers/net/gve/gve_ethdev.h     | 11 ++++++
 drivers/net/gve/gve_rx.c         | 15 ++++++--
 drivers/net/gve/gve_tx.c         | 13 +++++++
 5 files changed, 98 insertions(+), 2 deletions(-)
doc/guides/nics/features/gve.ini +1 -0
index cdc46b08a3..838edd456a 100644
@@ -10,6 +10,7 @@ MTU update           = Y
 TSO                  = Y
 RSS hash             = Y
 L4 checksum offload  = Y
+Basic stats          = Y
 Linux                = Y
 x86-32               = Y
 x86-64               = Y
drivers/net/gve/gve_ethdev.c +60 -0

72 lines not on this page · raw message

drivers/net/gve/gve_ethdev.h +11 -0
index 235e55899e..64e571bcae 100644
@@ -92,6 +92,11 @@ struct gve_tx_queue {
 	struct gve_queue_page_list *qpl;
 	struct gve_tx_iovec *iov_ring;
 
+	/* stats items */
+	uint64_t packets;
+	uint64_t bytes;
+	uint64_t errors;
+
 	uint16_t port_id;
 	uint16_t queue_id;
 
@@ -130,6 +135,12 @@ struct gve_rx_queue {
 	/* only valid for GQI_QPL queue format */
 	struct gve_queue_page_list *qpl;
 
+	/* stats items */
+	uint64_t packets;
+	uint64_t bytes;
+	uint64_t errors;
+	uint64_t no_mbufs;
+
 	struct gve_priv *hw;
 	const struct rte_memzone *qres_mz;
 	struct gve_queue_resources *qres;
drivers/net/gve/gve_rx.c +13 -2
index 518c9d109c..66fbcf3930 100644
@@ -26,8 +26,10 @@ gve_rx_refill(struct gve_rx_queue *rxq)
 					break;
 				rxq->sw_ring[idx + i] = nmb;
 			}
-			if (i != nb_alloc)
+			if (i != nb_alloc) {
+				rxq->no_mbufs += nb_alloc - i;
 				nb_alloc = i;
+			}
 		}
 		rxq->nb_avail -= nb_alloc;
 		next_avail += nb_alloc;
@@ -88,6 +90,7 @@ gve_rx_burst(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 	uint16_t rx_id = rxq->rx_tail;
 	struct rte_mbuf *rxe;
 	uint16_t nb_rx, len;
+	uint64_t bytes = 0;
 	uint64_t addr;
 	uint16_t i;
 
@@ -99,8 +102,10 @@ gve_rx_burst(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 		if (GVE_SEQNO(rxd->flags_seq) != rxq->expected_seqno)
 			break;
 
-		if (rxd->flags_seq & GVE_RXF_ERR)
+		if (rxd->flags_seq & GVE_RXF_ERR) {
+			rxq->errors++;
 			continue;
+		}
 
 		len = rte_be_to_cpu_16(rxd->len) - GVE_RX_PAD;
 		rxe = rxq->sw_ring[rx_id];
@@ -135,6 +140,7 @@ gve_rx_burst(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 			rx_id = 0;
 
 		rx_pkts[nb_rx] = rxe;
+		bytes += len;
 		nb_rx++;
 	}
 
@@ -144,6 +150,11 @@ gve_rx_burst(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 	if (rxq->nb_avail > rxq->free_thresh)
 		gve_rx_refill(rxq);
 
+	if (nb_rx) {
+		rxq->packets += nb_rx;
+		rxq->bytes += bytes;
+	}
+
 	return nb_rx;
 }
 
drivers/net/gve/gve_tx.c +13 -0
index bf4e8fea2c..9b41c59358 100644
@@ -260,6 +260,7 @@ gve_tx_burst_qpl(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 	struct rte_mbuf *tx_pkt, *first;
 	uint16_t sw_id = txq->sw_tail;
 	uint16_t nb_used, i;
+	uint64_t bytes = 0;
 	uint16_t nb_tx = 0;
 	uint32_t hlen;
 
@@ -355,6 +356,8 @@ gve_tx_burst_qpl(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 		txq->nb_free -= nb_used;
 		txq->sw_nb_free -= first->nb_segs;
 		tx_tail += nb_used;
+
+		bytes += first->pkt_len;
 	}
 
 end_of_tx:
@@ -362,6 +365,10 @@ gve_tx_burst_qpl(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 		rte_write32(rte_cpu_to_be_32(tx_tail), txq->qtx_tail);
 		txq->tx_tail = tx_tail;
 		txq->sw_tail = sw_id;
+
+		txq->packets += nb_tx;
+		txq->bytes += bytes;
+		txq->errors += nb_pkts - nb_tx;
 	}
 
 	return nb_tx;
@@ -380,6 +387,7 @@ gve_tx_burst_ra(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 	struct rte_mbuf *tx_pkt, *first;
 	uint16_t nb_used, hlen, i;
 	uint64_t ol_flags, addr;
+	uint64_t bytes = 0;
 	uint16_t nb_tx = 0;
 
 	txr = txq->tx_desc_ring;
@@ -438,12 +446,17 @@ gve_tx_burst_ra(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 
 		txq->nb_free -= nb_used;
 		tx_tail += nb_used;
+
+		bytes += first->pkt_len;
 	}
 
 end_of_tx:
 	if (nb_tx) {
 		rte_write32(rte_cpu_to_be_32(tx_tail), txq->qtx_tail);
 		txq->tx_tail = tx_tail;
+
+		txq->packets += nb_tx;
+		txq->bytes += bytes;
 	}
 
 	return nb_tx;
Signature · 1 line
2.34.1
#1 lore raw
Quoted 19 lines · On Thu, 24 Nov 2022 15:33:35 +0800 Junfeng Guo <[email protected]> wrote:
drivers/net/gve/gve_ethdev.c +19 -0
+static int
+gve_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
+{
+	uint16_t i;
+
+	for (i = 0; i < dev->data->nb_tx_queues; i++) {
+		struct gve_tx_queue *txq = dev->data->tx_queues[i];
+		if (txq == NULL)
+			continue;
+
+		stats->opackets += txq->packets;
+		stats->obytes += txq->bytes;
+		stats->oerrors += txq->errors;
+	}
+
+	for (i = 0; i < dev->data->nb_rx_queues; i++) {
+		struct gve_rx_queue *rxq = dev->data->rx_queues[i];
+		if (rxq == NULL)
+			continue;

drivers/net/gve/gve_ethdev.c gve_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) · line 350

Quoted 10 lines
drivers/net/gve/gve_ethdev.c +10 -0
+
+		stats->ipackets += rxq->packets;
+		stats->ibytes += rxq->bytes;
+		stats->ierrors += rxq->errors;
+		stats->rx_nombuf += rxq->no_mbufs;
+	}
+
+	return 0;
+}
+

The driver should be filling in the per-queue stats as well. q_ipackets, q_opackets, q_ibytes, q_obytes, q_errors

#2 lore raw
Quoted 27 lines · Stephen Hemminger
Quoted 24 lines · On Thu, 24 Nov 2022 15:33:35 +0800 Junfeng Guo <[email protected]> wrote:
drivers/net/gve/gve_ethdev.c +24 -0
+static int
+gve_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
+{
+	uint16_t i;
+
+	for (i = 0; i < dev->data->nb_tx_queues; i++) {
+		struct gve_tx_queue *txq = dev->data->tx_queues[i];
+		if (txq == NULL)
+			continue;
+
+		stats->opackets += txq->packets;
+		stats->obytes += txq->bytes;
+		stats->oerrors += txq->errors;
+	}
+
+	for (i = 0; i < dev->data->nb_rx_queues; i++) {
+		struct gve_rx_queue *rxq = dev->data->rx_queues[i];
+		if (rxq == NULL)
+			continue;
+
+		stats->ipackets += rxq->packets;
+		stats->ibytes += rxq->bytes;
+		stats->ierrors += rxq->errors;
+		stats->rx_nombuf += rxq->no_mbufs;
Quoted 8 lines

drivers/net/gve/gve_ethdev.c gve_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) · line 350

Quoted 5 lines
drivers/net/gve/gve_ethdev.c +5 -0
+	}
+
+	return 0;
+}
+

The driver should be filling in the per-queue stats as well. q_ipackets, q_opackets, q_ibytes, q_obytes, q_errors

Hi Stephen,

Queue stats moved to xstats, and there is a long term goal to move all queue stats from basic stats to xstats for all PMDs, and remove interim 'RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS' flag.

That is why request to new PMDs is to not introduce queue stats in basic stats, but in xstats.

#3 lore raw

Makes sense.

Quoted 47 lines · Ferruh Yigit
Quoted 27 lines · Stephen Hemminger
Quoted 24 lines · On Thu, 24 Nov 2022 15:33:35 +0800 Junfeng Guo <[email protected]> wrote:
drivers/net/gve/gve_ethdev.c +24 -0
+static int
+gve_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
+{
+    uint16_t i;
+
+    for (i = 0; i < dev->data->nb_tx_queues; i++) {
+            struct gve_tx_queue *txq = dev->data->tx_queues[i];
+            if (txq == NULL)
+                    continue;
+
+            stats->opackets += txq->packets;
+            stats->obytes += txq->bytes;
+            stats->oerrors += txq->errors;
+    }
+
+    for (i = 0; i < dev->data->nb_rx_queues; i++) {
+            struct gve_rx_queue *rxq = dev->data->rx_queues[i];
+            if (rxq == NULL)
+                    continue;
+
+            stats->ipackets += rxq->packets;
+            stats->ibytes += rxq->bytes;
+            stats->ierrors += rxq->errors;
+            stats->rx_nombuf += rxq->no_mbufs;
Quoted 8 lines

drivers/net/gve/gve_ethdev.c gve_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) · line 350

Quoted 5 lines
drivers/net/gve/gve_ethdev.c +5 -0
+    }
+
+    return 0;
+}
+

The driver should be filling in the per-queue stats as well. q_ipackets, q_opackets, q_ibytes, q_obytes, q_errors

Hi Stephen,

Queue stats moved to xstats, and there is a long term goal to move all queue stats from basic stats to xstats for all PMDs, and remove interim 'RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS' flag.

That is why request to new PMDs is to not introduce queue stats in basic stats, but in xstats.

#4 lore raw
Quoted 5 lines · On Fri, 25 Nov 2022 19:16:00 -0800 Rushil Gupta <[email protected]> wrote:
Quoted 5 lines
Quoted 3 lines

The driver should be filling in the per-queue stats as well. q_ipackets, q_opackets, q_ibytes, q_obytes, q_errors

Quoted 8 lines
Quoted 8 lines

Hi Stephen,

Queue stats moved to xstats, and there is a long term goal to move all queue stats from basic stats to xstats for all PMDs, and remove interim 'RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS' flag.

That is why request to new PMDs is to not introduce queue stats in basic stats, but in xstats.

Agree that xstats are better but:

* the current checked in version of GVE does not have driver op for xstats

* if driver fills in the q_XXX[] stats then eth_dev_get_xstats_basic will be able to use it.

The per queue stats are limited to 256 queues and bloats the info structure, but until an API breaking change is made to remove them, drivers should support it.

#5 lore raw
Quoted 17 lines · Stephen Hemminger
Quoted 5 lines · On Fri, 25 Nov 2022 19:16:00 -0800 Rushil Gupta <[email protected]> wrote:
Quoted 5 lines
Quoted 3 lines

The driver should be filling in the per-queue stats as well. q_ipackets, q_opackets, q_ibytes, q_obytes, q_errors

Quoted 8 lines
Quoted 8 lines

Hi Stephen,

Queue stats moved to xstats, and there is a long term goal to move all queue stats from basic stats to xstats for all PMDs, and remove interim 'RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS' flag.

That is why request to new PMDs is to not introduce queue stats in basic stats, but in xstats.

Quoted 7 lines

Agree that xstats are better but:

* the current checked in version of GVE does not have driver op for xstats

* if driver fills in the q_XXX[] stats then eth_dev_get_xstats_basic will be able to use it.

That is an option, but it will require driver to add 'RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS' flag what we want to get rid of in long term.
That is why we are requesting new driver to add the xstats implementation instead of adding queue support in basic stats. It is up to PMD to provide xstats implementation if they want queue stats.

Quoted 3 lines

The per queue stats are limited to 256 queues and bloats the info structure, but until an API breaking change is made to remove them, drivers should support it.

#6 lore raw
Quoted 26 lines · On Mon, 28 Nov 2022 11:12:38 +0000 Ferruh Yigit <[email protected]> wrote:
Quoted 17 lines · Stephen Hemminger
Quoted 5 lines · On Fri, 25 Nov 2022 19:16:00 -0800 Rushil Gupta <[email protected]> wrote:
Quoted 5 lines
Quoted 3 lines

The driver should be filling in the per-queue stats as well. q_ipackets, q_opackets, q_ibytes, q_obytes, q_errors

Quoted 8 lines
Quoted 8 lines

Hi Stephen,

Queue stats moved to xstats, and there is a long term goal to move all queue stats from basic stats to xstats for all PMDs, and remove interim 'RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS' flag.

That is why request to new PMDs is to not introduce queue stats in basic stats, but in xstats.

Quoted 7 lines

Agree that xstats are better but:

* the current checked in version of GVE does not have driver op for xstats

* if driver fills in the q_XXX[] stats then eth_dev_get_xstats_basic will be able to use it.

Quoted 6 lines

That is an option, but it will require driver to add 'RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS' flag what we want to get rid of in long term.
That is why we are requesting new driver to add the xstats implementation instead of adding queue support in basic stats. It is up to PMD to provide xstats implementation if they want queue stats.

Agreed, having xstats in driver is the best way. Doing what virtio does now would be a good example.

#7 lore raw
Quoted 49 lines

-----Original Message-----
From: Stephen Hemminger <[email protected]>
Sent: Tuesday, November 29, 2022 01:24
To: Ferruh Yigit <[email protected]>
Cc: Rushil Gupta <[email protected]>; Guo, Junfeng <[email protected]>; Zhang, Qi Z <[email protected]>; Wu, Jingjing <[email protected]>; Xing, Beilei <[email protected]>; [email protected]; [email protected]; [email protected]; Li, Xiaoyun <[email protected]>
Subject: Re: [PATCH] net/gve: add support for basic stats

Quoted 5 lines · On Mon, 28 Nov 2022 11:12:38 +0000 Ferruh Yigit <[email protected]> wrote:
Quoted 4 lines · Stephen Hemminger
Quoted 1 line · On Fri, 25 Nov 2022 19:16:00 -0800 Rushil Gupta <[email protected]> wrote:
Quoted 1 line
Quoted 1 line
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 2 lines

The driver should be filling in the per-queue stats as well. q_ipackets, q_opackets, q_ibytes, q_obytes, q_errors

Hi Stephen,

Queue stats moved to xstats, and there is a long term goal to move

all

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

queue stats from basic stats to xstats for all PMDs, and remove

interim

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

'RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS' flag.

That is why request to new PMDs is to not introduce queue stats in

basic

Quoted 5 lines
Quoted 5 lines
Quoted 1 line
Quoted 1 line

stats, but in xstats.

Agree that xstats are better but:

* the current checked in version of GVE does not have driver op for

xstats

Quoted 2 lines
Quoted 2 lines

* if driver fills in the q_XXX[] stats then eth_dev_get_xstats_basic will

be

Quoted 5 lines
Quoted 2 lines

able to use it.

That is an option, but it will require driver to add 'RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS' flag what we want to get

Quoted 8 lines

rid of in

Quoted 4 lines

long term.
That is why we are requesting new driver to add the xstats implementation instead of adding queue support in basic stats. It is up to PMD to provide xstats implementation if they want queue stats.

Agreed, having xstats in driver is the best way. Doing what virtio does now would be a good example.

Sure, the xstats implementation is planed to be added in the coming releases. Currently the gve PMD only have some basic features supported, and will have the rest features enabled gradually (might co-dev with Google team). Thanks Stephen and Ferruh for talking about this feature and giving the detailed explanations and concerns.

Quoted 1 line
#8 lore raw
In reply to Junfeng Guo
Quoted 12 lines · Junfeng Guo

Add support for dev_ops of stats_get and stats_reset.

Queue stats update will be moved into xstat [1], but the basic stats items may still be required. So we just keep the remaining ones and will implement the queue stats via xstats in the coming release.

[1]
https://elixir.bootlin.com/dpdk/v22.07/ \
source/doc/guides/rel_notes/deprecation.rst#L118

<...>

drivers/net/gve/gve_ethdev.c gve_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) · line 334

Quoted 13 lines
drivers/net/gve/gve_ethdev.c +13 -0
+static int
+gve_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
+{
+	uint16_t i;
+
+	for (i = 0; i < dev->data->nb_tx_queues; i++) {
+		struct gve_tx_queue *txq = dev->data->tx_queues[i];
+		if (txq == NULL)
+			continue;
+
+		stats->opackets += txq->packets;
+		stats->obytes += txq->bytes;
+		stats->oerrors += txq->errors;

Hi Junfeng, Qi, Jingjing, Beilei,

Above logic looks wrong to me, did you test it?

If the 'gve_dev_stats_get()' called multiple times (without stats reset in between), same values will be keep added to stats.
Some hw based implementations does this, because reading from stats registers automatically reset those registers but this shouldn't be case for this driver.

I expect it to be something like:

local_stats = 0
foreach queue

	local_stats += queue->stats

stats = local_stats

#9 lore raw
Quoted 40 lines · On Wed, 7 Dec 2022 15:09:08 +0000 Ferruh Yigit <[email protected]> wrote:
Quoted 12 lines · Junfeng Guo

Add support for dev_ops of stats_get and stats_reset.

Queue stats update will be moved into xstat [1], but the basic stats items may still be required. So we just keep the remaining ones and will implement the queue stats via xstats in the coming release.

[1]
https://elixir.bootlin.com/dpdk/v22.07/ \
source/doc/guides/rel_notes/deprecation.rst#L118

<...>

drivers/net/gve/gve_ethdev.c gve_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) · line 334

Quoted 13 lines
drivers/net/gve/gve_ethdev.c +13 -0
+static int
+gve_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
+{
+	uint16_t i;
+
+	for (i = 0; i < dev->data->nb_tx_queues; i++) {
+		struct gve_tx_queue *txq = dev->data->tx_queues[i];
+		if (txq == NULL)
+			continue;
+
+		stats->opackets += txq->packets;
+		stats->obytes += txq->bytes;
+		stats->oerrors += txq->errors;  

Hi Junfeng, Qi, Jingjing, Beilei,

Above logic looks wrong to me, did you test it?

If the 'gve_dev_stats_get()' called multiple times (without stats reset in between), same values will be keep added to stats.
Some hw based implementations does this, because reading from stats registers automatically reset those registers but this shouldn't be case for this driver.

Quoted 6 lines

I expect it to be something like:

local_stats = 0
foreach queue

	local_stats += queue->stats

stats = local_stats

The zero of local_stats is unnecessary.
The only caller of the PMD stats_get is rte_ethdev_stats_get and it zeros the stats structure before calling the PMD.

int
rte_eth_stats_get(uint16_t port_id, struct rte_eth_stats *stats)

{
	struct rte_eth_dev *dev;

	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
	dev = &rte_eth_devices[port_id];

	memset(stats, 0, sizeof(*stats));
...
	stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed;
	return eth_err(port_id, (*dev->dev_ops->stats_get)(dev, stats));

If any PMD has extra memset in their stats get that could be removed.

#10 lore raw

Hi all
Josh just found out some inconsistencies in the Tx/Rx statistics sum for all ports. Not sure if we can screenshot here but it goes like this:
Tx-dropped for port0: 447034
Tx-dropped for port1: 0
Accumulated forward statistics for all ports: 807630

Please note that this issue is only with Tx-dropped (not Tx-packets/Tx-total).

Quoted 70 lines · Stephen Hemminger
Quoted 40 lines · On Wed, 7 Dec 2022 15:09:08 +0000 Ferruh Yigit <[email protected]> wrote:
Quoted 12 lines · Junfeng Guo

Add support for dev_ops of stats_get and stats_reset.

Queue stats update will be moved into xstat [1], but the basic stats items may still be required. So we just keep the remaining ones and will implement the queue stats via xstats in the coming release.

[1]
https://elixir.bootlin.com/dpdk/v22.07/ \
source/doc/guides/rel_notes/deprecation.rst#L118

<...>

drivers/net/gve/gve_ethdev.c gve_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) · line 334

Quoted 13 lines
drivers/net/gve/gve_ethdev.c +13 -0
+static int
+gve_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
+{
+   uint16_t i;
+
+   for (i = 0; i < dev->data->nb_tx_queues; i++) {
+           struct gve_tx_queue *txq = dev->data->tx_queues[i];
+           if (txq == NULL)
+                   continue;
+
+           stats->opackets += txq->packets;
+           stats->obytes += txq->bytes;
+           stats->oerrors += txq->errors;

Hi Junfeng, Qi, Jingjing, Beilei,

Above logic looks wrong to me, did you test it?

If the 'gve_dev_stats_get()' called multiple times (without stats reset in between), same values will be keep added to stats.
Some hw based implementations does this, because reading from stats registers automatically reset those registers but this shouldn't be case for this driver.

Quoted 6 lines

I expect it to be something like:

local_stats = 0
foreach queue

      local_stats += queue->stats

stats = local_stats

The zero of local_stats is unnecessary.
The only caller of the PMD stats_get is rte_ethdev_stats_get and it zeros the stats structure before calling the PMD.

int
rte_eth_stats_get(uint16_t port_id, struct rte_eth_stats *stats)

{
        struct rte_eth_dev *dev;

        RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
        dev = &rte_eth_devices[port_id];

        memset(stats, 0, sizeof(*stats));
...
        stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed;
        return eth_err(port_id, (*dev->dev_ops->stats_get)(dev, stats));

If any PMD has extra memset in their stats get that could be removed.

#11 lore raw

Hello,

As it turns out, this error actually propagates to the "total" stats as well, which I assume is just calculated by adding TX-packets and TX-dropped. Here are the full stats from the example that Rushil mentioned:

---------------------- Forward statistics for port 0
----------------------

  RX-packets: 2453802        RX-dropped: 0             RX-total: 2453802
  TX-packets: 34266881       TX-dropped: 447034        TX-total: 34713915

----------------------------------------------------------------------------

---------------------- Forward statistics for port 1
----------------------

  RX-packets: 34713915       RX-dropped: 0             RX-total: 34713915
  TX-packets: 2453802        TX-dropped: 0             TX-total: 2453802

----------------------------------------------------------------------------

+++++++++++++++ Accumulated forward statistics for all
ports+++++++++++++++

  RX-packets: 37167717       RX-dropped: 0             RX-total: 37167717
  TX-packets: 36720683       TX-dropped: 807630        TX-total: 37528313

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

It can be seen that the stats for the individual ports are consistent, but the TX-total and TX-dropped are not consistent with the stats for the individual ports, as I believe that the TX-total and RX-total accumulated stats should be equal.

Quoted 87 lines · Rushil Gupta

Hi all
Josh just found out some inconsistencies in the Tx/Rx statistics sum for all ports. Not sure if we can screenshot here but it goes like this:
Tx-dropped for port0: 447034
Tx-dropped for port1: 0
Accumulated forward statistics for all ports: 807630

Please note that this issue is only with Tx-dropped (not Tx-packets/Tx-total).

Quoted 15 lines · Stephen Hemminger
Quoted 11 lines · On Wed, 7 Dec 2022 15:09:08 +0000 Ferruh Yigit <[email protected]> wrote:
Quoted 10 lines · Junfeng Guo

Add support for dev_ops of stats_get and stats_reset.

Queue stats update will be moved into xstat [1], but the basic stats items may still be required. So we just keep the remaining ones and will implement the queue stats via xstats in the coming release.

[1]
https://elixir.bootlin.com/dpdk/v22.07/ \
source/doc/guides/rel_notes/deprecation.rst#L118

Quoted 7 lines
Quoted 7 lines
Quoted 2 lines

<...>

Review

Quoted 2 lines
(quoted diff) +2 -0
+static int
+gve_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats

*stats)

Quoted 14 lines
Quoted 14 lines

drivers/net/gve/gve_ethdev.c gve_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) · line 334

Quoted 11 lines
drivers/net/gve/gve_ethdev.c +11 -0
+{
+   uint16_t i;
+
+   for (i = 0; i < dev->data->nb_tx_queues; i++) {
+           struct gve_tx_queue *txq = dev->data->tx_queues[i];
+           if (txq == NULL)
+                   continue;
+
+           stats->opackets += txq->packets;
+           stats->obytes += txq->bytes;
+           stats->oerrors += txq->errors;

Hi Junfeng, Qi, Jingjing, Beilei,

Quoted 6 lines
Quoted 6 lines

Above logic looks wrong to me, did you test it?

If the 'gve_dev_stats_get()' called multiple times (without stats reset in between), same values will be keep added to stats.
Some hw based implementations does this, because reading from stats registers automatically reset those registers but this shouldn't be

case

Quoted 28 lines
Quoted 8 lines

for this driver.

I expect it to be something like:

local_stats = 0
foreach queue

      local_stats += queue->stats

stats = local_stats

The zero of local_stats is unnecessary.
The only caller of the PMD stats_get is rte_ethdev_stats_get and it zeros the stats structure before calling the PMD.

int
rte_eth_stats_get(uint16_t port_id, struct rte_eth_stats *stats)

{
        struct rte_eth_dev *dev;

        RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
        dev = &rte_eth_devices[port_id];

        memset(stats, 0, sizeof(*stats));
...
        stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed;
        return eth_err(port_id, (*dev->dev_ops->stats_get)(dev, stats));

If any PMD has extra memset in their stats get that could be removed.

Signature · 1 line
Joshua Washington | Software Engineer | [email protected] | (414) 366-4423
#12 lore raw
Quoted 27 lines · Joshua Washington

Hello,

As it turns out, this error actually propagates to the "total" stats as well, which I assume is just calculated by adding TX-packets and TX-dropped. Here are the full stats from the example that Rushil mentioned:

  ---------------------- Forward statistics for port 0
 ----------------------
  RX-packets: 2453802        RX-dropped: 0             RX-total: 2453802
  TX-packets: 34266881       TX-dropped: 447034        TX-total: 34713915
----------------------------------------------------------------------------

  ---------------------- Forward statistics for port 1
 ----------------------
  RX-packets: 34713915       RX-dropped: 0             RX-total: 34713915
  TX-packets: 2453802        TX-dropped: 0             TX-total: 2453802
----------------------------------------------------------------------------

  +++++++++++++++ Accumulated forward statistics for all ports+++++++++++++++
  RX-packets: 37167717       RX-dropped: 0             RX-total: 37167717
  TX-packets: 36720683       TX-dropped: 807630        TX-total: 37528313
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

Quoted 5 lines

It can be seen that the stats for the individual ports are consistent, but the TX-total and TX-dropped are not consistent with the stats for the individual ports, as I believe that the TX-total and RX-total accumulated stats should be equal.

Hi Joshua, Rushil,

As I checked for it, issue may be related to testpmd stats display,

While displaying per port TX-dropped value, it only takes 'ports_stats[pt_id].tx_dropped' into account,
but for accumulated TX-dropped results it takes both 'ports_stats[pt_id].tx_dropped' & 'stats.oerrors' into account.

If you can reproduce it easily, can you please test with following change:

diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c index 134d79a55547..49322d07d7d6 100644
--- a/app/test-pmd/testpmd.c
+++ b/app/test-pmd/testpmd.c
@@ -2056,6 +2056,8 @@ fwd_stats_display(void)

                         fwd_cycles += fs->core_cycles;
         }
         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
 +               uint64_t tx_dropped = 0;
 +
                 pt_id = fwd_ports_ids[i];
                 port = &ports[pt_id];

@@ -2077,8 +2079,9 @@ fwd_stats_display(void)

                 total_recv += stats.ipackets;
                 total_xmit += stats.opackets;
                 total_rx_dropped += stats.imissed;
 -               total_tx_dropped += ports_stats[pt_id].tx_dropped;
 -               total_tx_dropped += stats.oerrors;
 +               tx_dropped += ports_stats[pt_id].tx_dropped;
 +               tx_dropped += stats.oerrors;
 +               total_tx_dropped += tx_dropped;

total_rx_nombuf += stats.rx_nombuf;

printf("\n %s Forward statistics for port %-2d %s\n",
@@ -2105,8 +2108,8 @@ fwd_stats_display(void)

                 printf("  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64
                        "TX-total: %-"PRIu64"\n",
 -                      stats.opackets, ports_stats[pt_id].tx_dropped,
 -                      stats.opackets + ports_stats[pt_id].tx_dropped);
 +                      stats.opackets, tx_dropped,
 +                      stats.opackets + tx_dropped);
                 if (record_burst_stats) {
                         if (ports_stats[pt_id].rx_stream)
Quoted 106 lines

On Mon, Dec 19, 2022 at 11:17 AM Rushil Gupta <[email protected] <mailto:[email protected]>> wrote:

    Hi all
    Josh just found out some inconsistencies in the Tx/Rx statistics sum
    for all ports. Not sure if we can screenshot here but it goes like
    this:
    Tx-dropped for port0: 447034
    Tx-dropped for port1: 0
    Accumulated forward statistics for all ports: 807630

    Please note that this issue is only with Tx-dropped (not
    Tx-packets/Tx-total).


    On Wed, Dec 7, 2022 at 8:39 AM Stephen Hemminger
    <[email protected] <mailto:[email protected]>> wrote:
    >
    > On Wed, 7 Dec 2022 15:09:08 +0000
    > Ferruh Yigit <[email protected] <mailto:[email protected]>>
    wrote:
    >
    > > On 11/24/2022 7:33 AM, Junfeng Guo wrote:
    > > > Add support for dev_ops of stats_get and stats_reset.
    > > >
    > > > Queue stats update will be moved into xstat [1], but the basic
    stats
    > > > items may still be required. So we just keep the remaining
    ones and
    > > > will implement the queue stats via xstats in the coming release.
    > > >
    > > > [1]
    > > > https://elixir.bootlin.com/dpdk/v22.07/
    <https://elixir.bootlin.com/dpdk/v22.07/> \
    > > >     source/doc/guides/rel_notes/deprecation.rst#L118
    > > >
    > > > Signed-off-by: Xiaoyun Li <[email protected]
    <mailto:[email protected]>>
    > > > Signed-off-by: Junfeng Guo <[email protected]
    <mailto:[email protected]>>
    > >
    > > <...>
    > >
    > > > +static int
    > > > +gve_dev_stats_get(struct rte_eth_dev *dev, struct
    rte_eth_stats *stats)
    > > > +{
    > > > +   uint16_t i;
    > > > +
    > > > +   for (i = 0; i < dev->data->nb_tx_queues; i++) {
    > > > +           struct gve_tx_queue *txq = dev->data->tx_queues[i];
    > > > +           if (txq == NULL)
    > > > +                   continue;
    > > > +
    > > > +           stats->opackets += txq->packets;
    > > > +           stats->obytes += txq->bytes;
    > > > +           stats->oerrors += txq->errors;
    > >
    > > Hi Junfeng, Qi, Jingjing, Beilei,
    > >
    > > Above logic looks wrong to me, did you test it?
    > >
    > > If the 'gve_dev_stats_get()' called multiple times (without
    stats reset
    > > in between), same values will be keep added to stats.
    > > Some hw based implementations does this, because reading from stats
    > > registers automatically reset those registers but this shouldn't
    be case
    > > for this driver.
    > >
    > > I expect it to be something like:
    > >
    > > local_stats = 0
    > > foreach queue
    > >       local_stats += queue->stats
    > > stats = local_stats
    >
    > The zero of local_stats is unnecessary.
    > The only caller of the PMD stats_get is rte_ethdev_stats_get
    > and it zeros the stats structure before calling the PMD.
    >
    >
    > int
    > rte_eth_stats_get(uint16_t port_id, struct rte_eth_stats *stats)
    > {
    >         struct rte_eth_dev *dev;
    >
    >         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
    >         dev = &rte_eth_devices[port_id];
    >
    >         memset(stats, 0, sizeof(*stats));
    > ...
    >         stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed;
    >         return eth_err(port_id, (*dev->dev_ops->stats_get)(dev,
    stats));
    >
    > If any PMD has extra memset in their stats get that could be removed.
Signature · 3 lines
Joshua Washington | Software Engineer | [email protected]
<mailto:[email protected]> | (414) 366-4423
 
#13 lore raw

Hello,

I tested it out, and the updates to testpmd seem to work.

Before applying the second patch:
---------------------- Forward statistics for port 0
----------------------

  RX-packets: 0              RX-dropped: 0             RX-total: 0
  TX-packets: 43769238       TX-dropped: 62634         TX-total: 43831872

----------------------------------------------------------------------------

---------------------- Forward statistics for port 1
----------------------

  RX-packets: 0              RX-dropped: 0             RX-total: 0
  TX-packets: 43761119       TX-dropped: 70753         TX-total: 43831872

----------------------------------------------------------------------------

+++++++++++++++ Accumulated forward statistics for all
ports+++++++++++++++

  RX-packets: 0              RX-dropped: 0             RX-total: 0
  TX-packets: 87530357       TX-dropped: 157302        TX-total: 87687659

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

62634 + 70753 = 133387 != 157302

After applying the second patch:
---------------------- Forward statistics for port 0
----------------------

  RX-packets: 0              RX-dropped: 0             RX-total: 0
  TX-packets: 12590721       TX-dropped: 36638         TX-total: 12627359

----------------------------------------------------------------------------

---------------------- Forward statistics for port 1
----------------------

  RX-packets: 0              RX-dropped: 0             RX-total: 0
  TX-packets: 12596255       TX-dropped: 31746         TX-total: 12628001

----------------------------------------------------------------------------

+++++++++++++++ Accumulated forward statistics for all
ports+++++++++++++++

  RX-packets: 0              RX-dropped: 0             RX-total: 0
  TX-packets: 25186976       TX-dropped: 68384         TX-total: 25255360

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

Thanks,
Josh

Quoted 199 lines · Ferruh Yigit
Quoted 5 lines · Joshua Washington

Hello,

As it turns out, this error actually propagates to the "total" stats as well, which I assume is just calculated by adding TX-packets and TX-dropped. Here are the full stats from the example that Rushil

mentioned:

Quoted 7 lines

---------------------- Forward statistics for port 0
----------------------

  RX-packets: 2453802        RX-dropped: 0             RX-total: 2453802
  TX-packets: 34266881       TX-dropped: 447034        TX-total: 34713915

----------------------------------------------------------------------------

Quoted 7 lines

---------------------- Forward statistics for port 1
----------------------

  RX-packets: 34713915       RX-dropped: 0             RX-total: 34713915
  TX-packets: 2453802        TX-dropped: 0             TX-total: 2453802

----------------------------------------------------------------------------

Quoted 7 lines

+++++++++++++++ Accumulated forward statistics for all
ports+++++++++++++++

  RX-packets: 37167717       RX-dropped: 0             RX-total: 37167717
  TX-packets: 36720683       TX-dropped: 807630        TX-total: 37528313

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

Quoted 6 lines

It can be seen that the stats for the individual ports are consistent, but the TX-total and TX-dropped are not consistent with the stats for the individual ports, as I believe that the TX-total and RX-total accumulated stats should be equal.

Hi Joshua, Rushil,

As I checked for it, issue may be related to testpmd stats display,

While displaying per port TX-dropped value, it only takes 'ports_stats[pt_id].tx_dropped' into account,
but for accumulated TX-dropped results it takes both 'ports_stats[pt_id].tx_dropped' & 'stats.oerrors' into account.

If you can reproduce it easily, can you please test with following change:

diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c index 134d79a55547..49322d07d7d6 100644
--- a/app/test-pmd/testpmd.c
+++ b/app/test-pmd/testpmd.c
@@ -2056,6 +2056,8 @@ fwd_stats_display(void)

                         fwd_cycles += fs->core_cycles;
         }
         for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
 +               uint64_t tx_dropped = 0;
 +
                 pt_id = fwd_ports_ids[i];
                 port = &ports[pt_id];

@@ -2077,8 +2079,9 @@ fwd_stats_display(void)

                 total_recv += stats.ipackets;
                 total_xmit += stats.opackets;
                 total_rx_dropped += stats.imissed;
 -               total_tx_dropped += ports_stats[pt_id].tx_dropped;
 -               total_tx_dropped += stats.oerrors;
 +               tx_dropped += ports_stats[pt_id].tx_dropped;
 +               tx_dropped += stats.oerrors;
 +               total_tx_dropped += tx_dropped;

total_rx_nombuf += stats.rx_nombuf;

printf("\n %s Forward statistics for port %-2d %s\n",
@@ -2105,8 +2108,8 @@ fwd_stats_display(void)

                 printf("  TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64
                        "TX-total: %-"PRIu64"\n",
 -                      stats.opackets, ports_stats[pt_id].tx_dropped,
 -                      stats.opackets + ports_stats[pt_id].tx_dropped);
 +                      stats.opackets, tx_dropped,
 +                      stats.opackets + tx_dropped);
                 if (record_burst_stats) {
                         if (ports_stats[pt_id].rx_stream)
Quoted 18 lines

On Mon, Dec 19, 2022 at 11:17 AM Rushil Gupta <[email protected] <mailto:[email protected]>> wrote:

Hi all
Josh just found out some inconsistencies in the Tx/Rx statistics sum for all ports. Not sure if we can screenshot here but it goes like this:
Tx-dropped for port0: 447034
Tx-dropped for port1: 0
Accumulated forward statistics for all ports: 807630

Please note that this issue is only with Tx-dropped (not Tx-packets/Tx-total).

On Wed, Dec 7, 2022 at 8:39 AM Stephen Hemminger <[email protected] <mailto:[email protected]>>

Quoted 5 lines · wrote:
Quoted 3 lines

On Wed, 7 Dec 2022 15:09:08 +0000
Ferruh Yigit <[email protected] <mailto:[email protected]>>

wrote:
>

Quoted 8 lines

> > On 11/24/2022 7:33 AM, Junfeng Guo wrote:
> > > Add support for dev_ops of stats_get and stats_reset. > > >
> > > Queue stats update will be moved into xstat [1], but the basic stats
> > > items may still be required. So we just keep the remaining ones and
> > > will implement the queue stats via xstats in the coming

release.

Quoted 28 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

[1]
https://elixir.bootlin.com/dpdk/v22.07/

    <https://elixir.bootlin.com/dpdk/v22.07/> \
    > > >     source/doc/guides/rel_notes/deprecation.rst#L118
    > > >
    > > > Signed-off-by: Xiaoyun Li <[email protected]
    <mailto:[email protected]>>
    > > > Signed-off-by: Junfeng Guo <[email protected]
    <mailto:[email protected]>>
    > >
    > > <...>
    > >
    > > > +static int
    > > > +gve_dev_stats_get(struct rte_eth_dev *dev, struct
    rte_eth_stats *stats)
    > > > +{
    > > > +   uint16_t i;
    > > > +
    > > > +   for (i = 0; i < dev->data->nb_tx_queues; i++) {
    > > > +           struct gve_tx_queue *txq = dev->data->tx_queues[i];
    > > > +           if (txq == NULL)
    > > > +                   continue;
    > > > +
    > > > +           stats->opackets += txq->packets;
    > > > +           stats->obytes += txq->bytes;
    > > > +           stats->oerrors += txq->errors;
    > >
Quoted 8 lines
    > > Hi Junfeng, Qi, Jingjing, Beilei,
    > >
    > > Above logic looks wrong to me, did you test it?
    > >
    > > If the 'gve_dev_stats_get()' called multiple times (without
    stats reset
    > > in between), same values will be keep added to stats.
    > > Some hw based implementations does this, because reading from

stats

Quoted 23 lines
Quoted 1 line
Quoted 1 line

registers automatically reset those registers but this shouldn't

    be case
    > > for this driver.
    > >
    > > I expect it to be something like:
    > >
    > > local_stats = 0
    > > foreach queue
    > >       local_stats += queue->stats
    > > stats = local_stats
    >
    > The zero of local_stats is unnecessary.
    > The only caller of the PMD stats_get is rte_ethdev_stats_get
    > and it zeros the stats structure before calling the PMD.
    >
    >
    > int
    > rte_eth_stats_get(uint16_t port_id, struct rte_eth_stats *stats)
    > {
    >         struct rte_eth_dev *dev;
    >
    >         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
    >         dev = &rte_eth_devices[port_id];
Quoted 8 lines
    >
    >         memset(stats, 0, sizeof(*stats));
    > ...
    >         stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed;
    >         return eth_err(port_id, (*dev->dev_ops->stats_get)(dev,
    stats));
    >
    > If any PMD has extra memset in their stats get that could be

removed.

Quoted 8 lines

--

Joshua Washington | Software Engineer | [email protected] <mailto:[email protected]> | (414) 366-4423

Signature · 1 line
Joshua Washington | Software Engineer | [email protected] | (414) 366-4423
#14 lore raw
Quoted 4 lines · Joshua Washington

Hello,

I tested it out, and the updates to testpmd seem to work.

Hi Joshua,

Thanks for testing, I will send a patch soon.

But this was testpmd issue, do you have any objection with the net/gve patch, if not can you please record this with ack/review/tested-by tags, so I can proceed with it.

Quoted 281 lines

Before applying the second patch:
  ---------------------- Forward statistics for port 0
 ----------------------
  RX-packets: 0              RX-dropped: 0             RX-total: 0
  TX-packets: 43769238       TX-dropped: 62634         TX-total: 43831872
----------------------------------------------------------------------------

  ---------------------- Forward statistics for port 1
 ----------------------
  RX-packets: 0              RX-dropped: 0             RX-total: 0
  TX-packets: 43761119       TX-dropped: 70753         TX-total: 43831872
----------------------------------------------------------------------------

  +++++++++++++++ Accumulated forward statistics for all ports+++++++++++++++
  RX-packets: 0              RX-dropped: 0             RX-total: 0
  TX-packets: 87530357       TX-dropped: 157302        TX-total: 87687659
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

62634 + 70753 = 133387 != 157302

After applying the second patch:
  ---------------------- Forward statistics for port 0
 ----------------------
  RX-packets: 0              RX-dropped: 0             RX-total: 0
  TX-packets: 12590721       TX-dropped: 36638         TX-total: 12627359
----------------------------------------------------------------------------

  ---------------------- Forward statistics for port 1
 ----------------------
  RX-packets: 0              RX-dropped: 0             RX-total: 0
  TX-packets: 12596255       TX-dropped: 31746         TX-total: 12628001
----------------------------------------------------------------------------

  +++++++++++++++ Accumulated forward statistics for all ports+++++++++++++++
  RX-packets: 0              RX-dropped: 0             RX-total: 0
  TX-packets: 25186976       TX-dropped: 68384         TX-total: 25255360
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

Thanks,
Josh

On Wed, Jan 18, 2023 at 8:22 AM Ferruh Yigit <[email protected] <mailto:[email protected]>> wrote:

    On 12/19/2022 7:38 PM, Joshua Washington wrote:
    > Hello,
    >
    > As it turns out, this error actually propagates to the "total"
    stats as
    > well, which I assume is just calculated by adding TX-packets and
    > TX-dropped. Here are the full stats from the example that Rushil
    mentioned:
    >
    >   ---------------------- Forward statistics for port 0
    >  ----------------------
    >   RX-packets: 2453802        RX-dropped: 0             RX-total:
    2453802
    >   TX-packets: 34266881       TX-dropped: 447034        TX-total:
    34713915
    >  
    >
    ----------------------------------------------------------------------------
    >
    >   ---------------------- Forward statistics for port 1
    >  ----------------------
    >   RX-packets: 34713915       RX-dropped: 0             RX-total:
    34713915
    >   TX-packets: 2453802        TX-dropped: 0             TX-total:
    2453802
    >  
    >
    ----------------------------------------------------------------------------
    >
    >   +++++++++++++++ Accumulated forward statistics for all
    > ports+++++++++++++++
    >   RX-packets: 37167717       RX-dropped: 0             RX-total:
    37167717
    >   TX-packets: 36720683       TX-dropped: 807630        TX-total:
    37528313
    >  
    >
    ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
    >
    > It can be seen that the stats for the individual ports are consistent,
    > but the TX-total and TX-dropped are not consistent with the stats for
    > the individual ports, as I believe that the TX-total and RX-total
    > accumulated stats should be equal.
    >

    Hi Joshua, Rushil,

    As I checked for it, issue may be related to testpmd stats display,

    While displaying per port TX-dropped value, it only takes
    'ports_stats[pt_id].tx_dropped' into account,
    but for accumulated TX-dropped results it takes both
    'ports_stats[pt_id].tx_dropped' & 'stats.oerrors' into account.

    If you can reproduce it easily, can you please test with following
    change:

     diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
     index 134d79a55547..49322d07d7d6 100644
     --- a/app/test-pmd/testpmd.c
     +++ b/app/test-pmd/testpmd.c
     @@ -2056,6 +2056,8 @@ fwd_stats_display(void)
                             fwd_cycles += fs->core_cycles;
             }
             for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
     +               uint64_t tx_dropped = 0;
     +
                     pt_id = fwd_ports_ids[i];
                     port = &ports[pt_id];

     @@ -2077,8 +2079,9 @@ fwd_stats_display(void)
                     total_recv += stats.ipackets;
                     total_xmit += stats.opackets;
                     total_rx_dropped += stats.imissed;
     -               total_tx_dropped += ports_stats[pt_id].tx_dropped;
     -               total_tx_dropped += stats.oerrors;
     +               tx_dropped += ports_stats[pt_id].tx_dropped;
     +               tx_dropped += stats.oerrors;
     +               total_tx_dropped += tx_dropped;
                     total_rx_nombuf  += stats.rx_nombuf;

                     printf("\n  %s Forward statistics for port %-2d %s\n",
     @@ -2105,8 +2108,8 @@ fwd_stats_display(void)

                     printf("  TX-packets: %-14"PRIu64" TX-dropped:
    %-14"PRIu64
                            "TX-total: %-"PRIu64"\n",
     -                      stats.opackets, ports_stats[pt_id].tx_dropped,
     -                      stats.opackets + ports_stats[pt_id].tx_dropped);
     +                      stats.opackets, tx_dropped,
     +                      stats.opackets + tx_dropped);

                     if (record_burst_stats) {
                             if (ports_stats[pt_id].rx_stream)


    >
    > On Mon, Dec 19, 2022 at 11:17 AM Rushil Gupta <[email protected]
    <mailto:[email protected]>
    > <mailto:[email protected] <mailto:[email protected]>>> wrote:
    >
    >     Hi all
    >     Josh just found out some inconsistencies in the Tx/Rx
    statistics sum
    >     for all ports. Not sure if we can screenshot here but it goes like
    >     this:
    >     Tx-dropped for port0: 447034
    >     Tx-dropped for port1: 0
    >     Accumulated forward statistics for all ports: 807630
    >
    >     Please note that this issue is only with Tx-dropped (not
    >     Tx-packets/Tx-total).
    >
    >
    >     On Wed, Dec 7, 2022 at 8:39 AM Stephen Hemminger
    >     <[email protected]
    <mailto:[email protected]>
    <mailto:[email protected]
    <mailto:[email protected]>>> wrote:
    >     >
    >     > On Wed, 7 Dec 2022 15:09:08 +0000
    >     > Ferruh Yigit <[email protected]
    <mailto:[email protected]> <mailto:[email protected]
    <mailto:[email protected]>>>
    >     wrote:
    >     >
    >     > > On 11/24/2022 7:33 AM, Junfeng Guo wrote:
    >     > > > Add support for dev_ops of stats_get and stats_reset.
    >     > > >
    >     > > > Queue stats update will be moved into xstat [1], but the
    basic
    >     stats
    >     > > > items may still be required. So we just keep the remaining
    >     ones and
    >     > > > will implement the queue stats via xstats in the coming
    release.
    >     > > >
    >     > > > [1]
    >     > > > https://elixir.bootlin.com/dpdk/v22.07/
    <https://elixir.bootlin.com/dpdk/v22.07/>
    >     <https://elixir.bootlin.com/dpdk/v22.07/
    <https://elixir.bootlin.com/dpdk/v22.07/>> \
    >     > > >     source/doc/guides/rel_notes/deprecation.rst#L118
    >     > > >
    >     > > > Signed-off-by: Xiaoyun Li <[email protected]
    <mailto:[email protected]>
    >     <mailto:[email protected] <mailto:[email protected]>>>
    >     > > > Signed-off-by: Junfeng Guo <[email protected]
    <mailto:[email protected]>
    >     <mailto:[email protected] <mailto:[email protected]>>>
    >     > >
    >     > > <...>
    >     > >
    >     > > > +static int
    >     > > > +gve_dev_stats_get(struct rte_eth_dev *dev, struct
    >     rte_eth_stats *stats)
    >     > > > +{
    >     > > > +   uint16_t i;
    >     > > > +
    >     > > > +   for (i = 0; i < dev->data->nb_tx_queues; i++) {
    >     > > > +           struct gve_tx_queue *txq =
    dev->data->tx_queues[i];
    >     > > > +           if (txq == NULL)
    >     > > > +                   continue;
    >     > > > +
    >     > > > +           stats->opackets += txq->packets;
    >     > > > +           stats->obytes += txq->bytes;
    >     > > > +           stats->oerrors += txq->errors;
    >     > >
    >     > > Hi Junfeng, Qi, Jingjing, Beilei,
    >     > >
    >     > > Above logic looks wrong to me, did you test it?
    >     > >
    >     > > If the 'gve_dev_stats_get()' called multiple times (without
    >     stats reset
    >     > > in between), same values will be keep added to stats.
    >     > > Some hw based implementations does this, because reading
    from stats
    >     > > registers automatically reset those registers but this
    shouldn't
    >     be case
    >     > > for this driver.
    >     > >
    >     > > I expect it to be something like:
    >     > >
    >     > > local_stats = 0
    >     > > foreach queue
    >     > >       local_stats += queue->stats
    >     > > stats = local_stats
    >     >
    >     > The zero of local_stats is unnecessary.
    >     > The only caller of the PMD stats_get is rte_ethdev_stats_get
    >     > and it zeros the stats structure before calling the PMD.
    >     >
    >     >
    >     > int
    >     > rte_eth_stats_get(uint16_t port_id, struct rte_eth_stats *stats)
    >     > {
    >     >         struct rte_eth_dev *dev;
    >     >
    >     >         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
    >     >         dev = &rte_eth_devices[port_id];
    >     >
    >     >         memset(stats, 0, sizeof(*stats));
    >     > ...
    >     >         stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed;
    >     >         return eth_err(port_id, (*dev->dev_ops->stats_get)(dev,
    >     stats));
    >     >
    >     > If any PMD has extra memset in their stats get that could be
    removed.
    >
    >
    >
    > --
    >
    > Joshua Washington | Software Engineer | [email protected]
    <mailto:[email protected]>
    > <mailto:[email protected] <mailto:[email protected]>> | (414)
    366-4423 <tel:(414)%20366-4423>
    >  
Signature · 3 lines
Joshua Washington | Software Engineer | [email protected]
<mailto:[email protected]> | (414) 366-4423
 
#15 lore raw
Quoted 315 lines · Ferruh Yigit
Quoted 4 lines · Joshua Washington

Hello,

I tested it out, and the updates to testpmd seem to work.

Hi Joshua,

Thanks for testing, I will send a patch soon.

But this was testpmd issue, do you have any objection with the net/gve patch, if not can you please record this with ack/review/tested-by tags, so I can proceed with it.

Quoted 7 lines

Before applying the second patch:
---------------------- Forward statistics for port 0
----------------------

  RX-packets: 0              RX-dropped: 0             RX-total: 0
  TX-packets: 43769238       TX-dropped: 62634         TX-total: 43831872

----------------------------------------------------------------------------

Quoted 7 lines

---------------------- Forward statistics for port 1
----------------------

  RX-packets: 0              RX-dropped: 0             RX-total: 0
  TX-packets: 43761119       TX-dropped: 70753         TX-total: 43831872

----------------------------------------------------------------------------

Quoted 7 lines

+++++++++++++++ Accumulated forward statistics for all
ports+++++++++++++++

  RX-packets: 0              RX-dropped: 0             RX-total: 0
  TX-packets: 87530357       TX-dropped: 157302        TX-total: 87687659

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

Quoted 10 lines

62634 + 70753 = 133387 != 157302

After applying the second patch:
---------------------- Forward statistics for port 0
----------------------

  RX-packets: 0              RX-dropped: 0             RX-total: 0
  TX-packets: 12590721       TX-dropped: 36638         TX-total: 12627359

----------------------------------------------------------------------------

Quoted 7 lines

---------------------- Forward statistics for port 1
----------------------

  RX-packets: 0              RX-dropped: 0             RX-total: 0
  TX-packets: 12596255       TX-dropped: 31746         TX-total: 12628001

----------------------------------------------------------------------------

Quoted 7 lines

+++++++++++++++ Accumulated forward statistics for all
ports+++++++++++++++

  RX-packets: 0              RX-dropped: 0             RX-total: 0
  TX-packets: 25186976       TX-dropped: 68384         TX-total: 25255360

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

Quoted 16 lines

Thanks,
Josh

On Wed, Jan 18, 2023 at 8:22 AM Ferruh Yigit <[email protected] <mailto:[email protected]>> wrote:

    On 12/19/2022 7:38 PM, Joshua Washington wrote:
    > Hello,
    >
    > As it turns out, this error actually propagates to the "total"
    stats as
    > well, which I assume is just calculated by adding TX-packets and
    > TX-dropped. Here are the full stats from the example that Rushil
    mentioned:
    >
Quoted 9 lines
    >   ---------------------- Forward statistics for port 0
    >  ----------------------
    >   RX-packets: 2453802        RX-dropped: 0             RX-total:
    2453802
    >   TX-packets: 34266881       TX-dropped: 447034        TX-total:
    34713915
    >
    >

----------------------------------------------------------------------------

Quoted 10 lines
Quoted 4 lines

---------------------- Forward statistics for port 1
----------------------
RX-packets: 34713915 RX-dropped: 0 RX-total:

    34713915
    >   TX-packets: 2453802        TX-dropped: 0             TX-total:
    2453802
    >
    >

----------------------------------------------------------------------------

Quoted 10 lines
Quoted 4 lines

+++++++++++++++ Accumulated forward statistics for all
ports+++++++++++++++
RX-packets: 37167717 RX-dropped: 0 RX-total:

    37167717
    >   TX-packets: 36720683       TX-dropped: 807630        TX-total:
    37528313
    >
    >

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

Quoted 2 lines
Quoted 2 lines

It can be seen that the stats for the individual ports are

consistent,

Quoted 1 line
Quoted 1 line

but the TX-total and TX-dropped are not consistent with the stats

for

Quoted 33 lines
Quoted 3 lines

the individual ports, as I believe that the TX-total and RX-total accumulated stats should be equal.

    Hi Joshua, Rushil,

    As I checked for it, issue may be related to testpmd stats display,

    While displaying per port TX-dropped value, it only takes
    'ports_stats[pt_id].tx_dropped' into account,
    but for accumulated TX-dropped results it takes both
    'ports_stats[pt_id].tx_dropped' & 'stats.oerrors' into account.

    If you can reproduce it easily, can you please test with following
    change:

     diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
     index 134d79a55547..49322d07d7d6 100644
     --- a/app/test-pmd/testpmd.c
     +++ b/app/test-pmd/testpmd.c
     @@ -2056,6 +2056,8 @@ fwd_stats_display(void)
                             fwd_cycles += fs->core_cycles;
             }
             for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
     +               uint64_t tx_dropped = 0;
     +
                     pt_id = fwd_ports_ids[i];
                     port = &ports[pt_id];

     @@ -2077,8 +2079,9 @@ fwd_stats_display(void)
                     total_recv += stats.ipackets;
                     total_xmit += stats.opackets;
                     total_rx_dropped += stats.imissed;
Quoted 8 lines
     -               total_tx_dropped += ports_stats[pt_id].tx_dropped;
     -               total_tx_dropped += stats.oerrors;
     +               tx_dropped += ports_stats[pt_id].tx_dropped;
     +               tx_dropped += stats.oerrors;
     +               total_tx_dropped += tx_dropped;
                     total_rx_nombuf  += stats.rx_nombuf;

                     printf("\n  %s Forward statistics for port %-2d

%s\n",

Quoted 6 lines
     @@ -2105,8 +2108,8 @@ fwd_stats_display(void)

                     printf("  TX-packets: %-14"PRIu64" TX-dropped:
    %-14"PRIu64
                            "TX-total: %-"PRIu64"\n",
     -                      stats.opackets,

ports_stats[pt_id].tx_dropped,

Quoted 1 line

- stats.opackets +

ports_stats[pt_id].tx_dropped);

Quoted 8 lines

+ stats.opackets, tx_dropped,
+ stats.opackets + tx_dropped);

if (record_burst_stats) {
if (ports_stats[pt_id].rx_stream)

>

Quoted 8 lines

> On Mon, Dec 19, 2022 at 11:17 AM Rushil Gupta <[email protected] <mailto:[email protected]>
> <mailto:[email protected] <mailto:[email protected]>>> wrote:
>
> Hi all
> Josh just found out some inconsistencies in the Tx/Rx
statistics sum
> for all ports. Not sure if we can screenshot here but it goes

like

Quoted 20 lines
Quoted 11 lines

this:
Tx-dropped for port0: 447034
Tx-dropped for port1: 0
Accumulated forward statistics for all ports: 807630

Please note that this issue is only with Tx-dropped (not Tx-packets/Tx-total).

On Wed, Dec 7, 2022 at 8:39 AM Stephen Hemminger <[email protected]

    <mailto:[email protected]>
    <mailto:[email protected]
    <mailto:[email protected]>>> wrote:
    >     >
    >     > On Wed, 7 Dec 2022 15:09:08 +0000
    >     > Ferruh Yigit <[email protected]
    <mailto:[email protected]> <mailto:[email protected]
    <mailto:[email protected]>>>
    >     wrote:
Quoted 8 lines
    >     >
    >     > > On 11/24/2022 7:33 AM, Junfeng Guo wrote:
    >     > > > Add support for dev_ops of stats_get and stats_reset.
    >     > > >
    >     > > > Queue stats update will be moved into xstat [1], but the
    basic
    >     stats
    >     > > > items may still be required. So we just keep the

remaining

Quoted 56 lines
Quoted 2 lines

ones and
> > > will implement the queue stats via xstats in the coming

    release.
    >     > > >
    >     > > > [1]
    >     > > > https://elixir.bootlin.com/dpdk/v22.07/
    <https://elixir.bootlin.com/dpdk/v22.07/>
    >     <https://elixir.bootlin.com/dpdk/v22.07/
    <https://elixir.bootlin.com/dpdk/v22.07/>> \
    >     > > >     source/doc/guides/rel_notes/deprecation.rst#L118
    >     > > >
    >     > > > Signed-off-by: Xiaoyun Li <[email protected]
    <mailto:[email protected]>
    >     <mailto:[email protected] <mailto:[email protected]>>>
    >     > > > Signed-off-by: Junfeng Guo <[email protected]
    <mailto:[email protected]>
    >     <mailto:[email protected] <mailto:[email protected]>>>
    >     > >
    >     > > <...>
    >     > >
    >     > > > +static int
    >     > > > +gve_dev_stats_get(struct rte_eth_dev *dev, struct
    >     rte_eth_stats *stats)
    >     > > > +{
    >     > > > +   uint16_t i;
    >     > > > +
    >     > > > +   for (i = 0; i < dev->data->nb_tx_queues; i++) {
    >     > > > +           struct gve_tx_queue *txq =
    dev->data->tx_queues[i];
    >     > > > +           if (txq == NULL)
    >     > > > +                   continue;
    >     > > > +
    >     > > > +           stats->opackets += txq->packets;
    >     > > > +           stats->obytes += txq->bytes;
    >     > > > +           stats->oerrors += txq->errors;
    >     > >
    >     > > Hi Junfeng, Qi, Jingjing, Beilei,
    >     > >
    >     > > Above logic looks wrong to me, did you test it?
    >     > >
    >     > > If the 'gve_dev_stats_get()' called multiple times (without
    >     stats reset
    >     > > in between), same values will be keep added to stats.
    >     > > Some hw based implementations does this, because reading
    from stats
    >     > > registers automatically reset those registers but this
    shouldn't
    >     be case
    >     > > for this driver.
    >     > >
    >     > > I expect it to be something like:
    >     > >
    >     > > local_stats = 0
    >     > > foreach queue
    >     > >       local_stats += queue->stats
    >     > > stats = local_stats
Quoted 8 lines
    >     >
    >     > The zero of local_stats is unnecessary.
    >     > The only caller of the PMD stats_get is rte_ethdev_stats_get
    >     > and it zeros the stats structure before calling the PMD.
    >     >
    >     >
    >     > int
    >     > rte_eth_stats_get(uint16_t port_id, struct rte_eth_stats

*stats)

Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
{
        struct rte_eth_dev *dev;

        RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
        dev = &rte_eth_devices[port_id];

        memset(stats, 0, sizeof(*stats));
...
        stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed;
        return eth_err(port_id,

(*dev->dev_ops->stats_get)(dev,

Quoted 22 lines
Quoted 3 lines
    stats));
    >
    > If any PMD has extra memset in their stats get that could be
    removed.
    >
    >
    >
    > --
    >
    > Joshua Washington | Software Engineer | [email protected]
    <mailto:[email protected]>
    > <mailto:[email protected] <mailto:[email protected]>> | (414)
    366-4423 <tel:(414)%20366-4423>
    >

--

Joshua Washington | Software Engineer | [email protected] <mailto:[email protected]> | (414) 366-4423

Signature · 1 line
Joshua Washington | Software Engineer | [email protected] | (414) 366-4423
Applied
#16 lore raw
In reply to Junfeng Guo
Quoted 6 lines · Junfeng Guo

Add support for dev_ops of stats_get and stats_reset.

Queue stats update will be moved into xstat [1], but the basic stats items may still be required. So we just keep the remaining ones and will implement the queue stats via xstats in the coming release.

Quoted 9 lines

[1]
https://elixir.bootlin.com/dpdk/v22.07/ \
source/doc/guides/rel_notes/deprecation.rst#L118

Updated commit to add Joshua to .mailmap.

Applied to dpdk-next-net/main, thanks.