Skip to contentSkip to conversation

virtio_net: Add TX stop and wake counters

PATCH In review Daniel Jurgens 6 participants 31 messages Jan 30, 2024 – Jun 11, 2024

Subsystems (4)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 virtio_net: Add TX stop and wake counters

Add a tx queue stop and wake counters, they are useful for debugging.

	$ ethtool -S ens5f2 | grep 'tx_stop\|tx_wake'
	...
	tx_queue_1_tx_stop: 16726
	tx_queue_1_tx_wake: 16726
	...
	tx_queue_8_tx_stop: 1500110
	tx_queue_8_tx_wake: 1500110
---
 drivers/net/virtio_net.c | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)
drivers/net/virtio_net.c +24 -2
index 3cb8aa193884..7e3c31ceaf7e 100644
@@ -88,6 +88,8 @@ struct virtnet_sq_stats {
 	u64_stats_t xdp_tx_drops;
 	u64_stats_t kicks;
 	u64_stats_t tx_timeouts;
+	u64_stats_t tx_stop;
+	u64_stats_t tx_wake;
 };
 
 struct virtnet_rq_stats {
@@ -112,6 +114,8 @@ static const struct virtnet_stat_desc virtnet_sq_stats_desc[] = {
 	{ "xdp_tx_drops",	VIRTNET_SQ_STAT(xdp_tx_drops) },
 	{ "kicks",		VIRTNET_SQ_STAT(kicks) },
 	{ "tx_timeouts",	VIRTNET_SQ_STAT(tx_timeouts) },
+	{ "tx_stop",		VIRTNET_SQ_STAT(tx_stop) },
+	{ "tx_wake",		VIRTNET_SQ_STAT(tx_wake) },
 };
 
 static const struct virtnet_stat_desc virtnet_rq_stats_desc[] = {
@@ -843,6 +847,9 @@ static void check_sq_full_and_disable(struct virtnet_info *vi,
 	 */
 	if (sq->vq->num_free < 2+MAX_SKB_FRAGS) {
 		netif_stop_subqueue(dev, qnum);
+		u64_stats_update_begin(&sq->stats.syncp);
+		u64_stats_inc(&sq->stats.tx_stop);
+		u64_stats_update_end(&sq->stats.syncp);
 		if (use_napi) {
 			if (unlikely(!virtqueue_enable_cb_delayed(sq->vq)))
 				virtqueue_napi_schedule(&sq->napi, sq->vq);
@@ -851,6 +858,9 @@ static void check_sq_full_and_disable(struct virtnet_info *vi,
 			free_old_xmit_skbs(sq, false);
 			if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) {
 				netif_start_subqueue(dev, qnum);
+				u64_stats_update_begin(&sq->stats.syncp);
+				u64_stats_inc(&sq->stats.tx_wake);
+				u64_stats_update_end(&sq->stats.syncp);
 				virtqueue_disable_cb(sq->vq);
 			}
 		}
@@ -2163,8 +2173,14 @@ static void virtnet_poll_cleantx(struct receive_queue *rq)
 			free_old_xmit_skbs(sq, true);
 		} while (unlikely(!virtqueue_enable_cb_delayed(sq->vq)));
 
-		if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
+		if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) {
+			if (netif_tx_queue_stopped(txq)) {
+				u64_stats_update_begin(&sq->stats.syncp);
+				u64_stats_inc(&sq->stats.tx_wake);
+				u64_stats_update_end(&sq->stats.syncp);
+			}
 			netif_tx_wake_queue(txq);
+		}
 
 		__netif_tx_unlock(txq);
 	}
@@ -2310,8 +2326,14 @@ static int virtnet_poll_tx(struct napi_struct *napi, int budget)
 	virtqueue_disable_cb(sq->vq);
 	free_old_xmit_skbs(sq, true);
 
-	if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
+	if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) {
+		if (netif_tx_queue_stopped(txq)) {
+			u64_stats_update_begin(&sq->stats.syncp);
+			u64_stats_inc(&sq->stats.tx_wake);
+			u64_stats_update_end(&sq->stats.syncp);
+		}
 		netif_tx_wake_queue(txq);
+	}
 
 	opaque = virtqueue_enable_cb_prepare(sq->vq);
 
Signature · 1 line
2.42.0
#1 lore raw
Quoted 10 lines · Daniel Jurgens

Add a tx queue stop and wake counters, they are useful for debugging.

	$ ethtool -S ens5f2 | grep 'tx_stop\|tx_wake'
	...
	tx_queue_1_tx_stop: 16726
	tx_queue_1_tx_wake: 16726
	...
	tx_queue_8_tx_stop: 1500110
	tx_queue_8_tx_wake: 1500110
Quoted 2 lines

Hmm isn't one always same as the other, except when queue is stopped? And when it is stopped you can see that in the status?
So how is having two useful?

Quoted 80 lines
---
 drivers/net/virtio_net.c | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)
drivers/net/virtio_net.c +24 -2
index 3cb8aa193884..7e3c31ceaf7e 100644
@@ -88,6 +88,8 @@ struct virtnet_sq_stats {
 	u64_stats_t xdp_tx_drops;
 	u64_stats_t kicks;
 	u64_stats_t tx_timeouts;
+	u64_stats_t tx_stop;
+	u64_stats_t tx_wake;
 };
 
 struct virtnet_rq_stats {
@@ -112,6 +114,8 @@ static const struct virtnet_stat_desc virtnet_sq_stats_desc[] = {
 	{ "xdp_tx_drops",	VIRTNET_SQ_STAT(xdp_tx_drops) },
 	{ "kicks",		VIRTNET_SQ_STAT(kicks) },
 	{ "tx_timeouts",	VIRTNET_SQ_STAT(tx_timeouts) },
+	{ "tx_stop",		VIRTNET_SQ_STAT(tx_stop) },
+	{ "tx_wake",		VIRTNET_SQ_STAT(tx_wake) },
 };
 
 static const struct virtnet_stat_desc virtnet_rq_stats_desc[] = {
@@ -843,6 +847,9 @@ static void check_sq_full_and_disable(struct virtnet_info *vi,
 	 */
 	if (sq->vq->num_free < 2+MAX_SKB_FRAGS) {
 		netif_stop_subqueue(dev, qnum);
+		u64_stats_update_begin(&sq->stats.syncp);
+		u64_stats_inc(&sq->stats.tx_stop);
+		u64_stats_update_end(&sq->stats.syncp);
 		if (use_napi) {
 			if (unlikely(!virtqueue_enable_cb_delayed(sq->vq)))
 				virtqueue_napi_schedule(&sq->napi, sq->vq);
@@ -851,6 +858,9 @@ static void check_sq_full_and_disable(struct virtnet_info *vi,
 			free_old_xmit_skbs(sq, false);
 			if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) {
 				netif_start_subqueue(dev, qnum);
+				u64_stats_update_begin(&sq->stats.syncp);
+				u64_stats_inc(&sq->stats.tx_wake);
+				u64_stats_update_end(&sq->stats.syncp);
 				virtqueue_disable_cb(sq->vq);
 			}
 		}
@@ -2163,8 +2173,14 @@ static void virtnet_poll_cleantx(struct receive_queue *rq)
 			free_old_xmit_skbs(sq, true);
 		} while (unlikely(!virtqueue_enable_cb_delayed(sq->vq)));
 
-		if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
+		if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) {
+			if (netif_tx_queue_stopped(txq)) {
+				u64_stats_update_begin(&sq->stats.syncp);
+				u64_stats_inc(&sq->stats.tx_wake);
+				u64_stats_update_end(&sq->stats.syncp);
+			}
 			netif_tx_wake_queue(txq);
+		}
 
 		__netif_tx_unlock(txq);
 	}
@@ -2310,8 +2326,14 @@ static int virtnet_poll_tx(struct napi_struct *napi, int budget)
 	virtqueue_disable_cb(sq->vq);
 	free_old_xmit_skbs(sq, true);
 
-	if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
+	if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) {
+		if (netif_tx_queue_stopped(txq)) {
+			u64_stats_update_begin(&sq->stats.syncp);
+			u64_stats_inc(&sq->stats.tx_wake);
+			u64_stats_update_end(&sq->stats.syncp);
+		}
 		netif_tx_wake_queue(txq);
+	}
 
 	opaque = virtqueue_enable_cb_prepare(sq->vq);
 
Signature · 1 line
2.42.0
#28 lore raw
In reply to Daniel Jurgens
Quoted 21 lines · Daniel Jurgens

Add a tx queue stop and wake counters, they are useful for debugging.

	$ ethtool -S ens5f2 | grep 'tx_stop\|tx_wake'
	...
	tx_queue_1_tx_stop: 16726
	tx_queue_1_tx_wake: 16726
	...
	tx_queue_8_tx_stop: 1500110
	tx_queue_8_tx_wake: 1500110
---
  drivers/net/virtio_net.c | 26 ++++++++++++++++++++++++--
  1 file changed, 24 insertions(+), 2 deletions(-)
drivers/net/virtio_net.c +0 -0
index 3cb8aa193884..7e3c31ceaf7e 100644

drivers/net/virtio_net.c struct virtnet_sq_stats { · line 93

Quoted 6 lines
drivers/net/virtio_net.c +2 -0
@@ -88,6 +88,8 @@ struct virtnet_sq_stats {
  	u64_stats_t xdp_tx_drops;
  	u64_stats_t kicks;
  	u64_stats_t tx_timeouts;
+	u64_stats_t tx_stop;
+	u64_stats_t tx_wake;
  };

Hi Daniel!

tx_stop/wake only counts the status in the I/O path.
Do the status of virtnet_config_changed_work and virtnet_tx_resize need to be counted?

Thanks,
Heng

Quoted 63 lines
drivers/net/virtio_net.c +22 -2
  struct virtnet_rq_stats {
@@ -112,6 +114,8 @@ static const struct virtnet_stat_desc virtnet_sq_stats_desc[] = {
  	{ "xdp_tx_drops",	VIRTNET_SQ_STAT(xdp_tx_drops) },
  	{ "kicks",		VIRTNET_SQ_STAT(kicks) },
  	{ "tx_timeouts",	VIRTNET_SQ_STAT(tx_timeouts) },
+	{ "tx_stop",		VIRTNET_SQ_STAT(tx_stop) },
+	{ "tx_wake",		VIRTNET_SQ_STAT(tx_wake) },
  };
 
  static const struct virtnet_stat_desc virtnet_rq_stats_desc[] = {
@@ -843,6 +847,9 @@ static void check_sq_full_and_disable(struct virtnet_info *vi,
  	 */
  	if (sq->vq->num_free < 2+MAX_SKB_FRAGS) {
  		netif_stop_subqueue(dev, qnum);
+		u64_stats_update_begin(&sq->stats.syncp);
+		u64_stats_inc(&sq->stats.tx_stop);
+		u64_stats_update_end(&sq->stats.syncp);
  		if (use_napi) {
  			if (unlikely(!virtqueue_enable_cb_delayed(sq->vq)))
  				virtqueue_napi_schedule(&sq->napi, sq->vq);
@@ -851,6 +858,9 @@ static void check_sq_full_and_disable(struct virtnet_info *vi,
  			free_old_xmit_skbs(sq, false);
  			if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) {
  				netif_start_subqueue(dev, qnum);
+				u64_stats_update_begin(&sq->stats.syncp);
+				u64_stats_inc(&sq->stats.tx_wake);
+				u64_stats_update_end(&sq->stats.syncp);
  				virtqueue_disable_cb(sq->vq);
  			}
  		}
@@ -2163,8 +2173,14 @@ static void virtnet_poll_cleantx(struct receive_queue *rq)
  			free_old_xmit_skbs(sq, true);
  		} while (unlikely(!virtqueue_enable_cb_delayed(sq->vq)));
 
-		if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
+		if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) {
+			if (netif_tx_queue_stopped(txq)) {
+				u64_stats_update_begin(&sq->stats.syncp);
+				u64_stats_inc(&sq->stats.tx_wake);
+				u64_stats_update_end(&sq->stats.syncp);
+			}
  			netif_tx_wake_queue(txq);
+		}
 
  		__netif_tx_unlock(txq);
  	}
@@ -2310,8 +2326,14 @@ static int virtnet_poll_tx(struct napi_struct *napi, int budget)
  	virtqueue_disable_cb(sq->vq);
  	free_old_xmit_skbs(sq, true);
 
-	if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
+	if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) {
+		if (netif_tx_queue_stopped(txq)) {
+			u64_stats_update_begin(&sq->stats.syncp);
+			u64_stats_inc(&sq->stats.tx_wake);
+			u64_stats_update_end(&sq->stats.syncp);
+		}
  		netif_tx_wake_queue(txq);
+	}
 
  	opaque = virtqueue_enable_cb_prepare(sq->vq);
 
#2 lore raw
In reply to Michael S. Tsirkin
Quoted 14 lines

From: Michael S. Tsirkin <[email protected]>
Sent: Tuesday, January 30, 2024 8:58 AM

Quoted 10 lines · Daniel Jurgens

Add a tx queue stop and wake counters, they are useful for debugging.

	$ ethtool -S ens5f2 | grep 'tx_stop\|tx_wake'
	...
	tx_queue_1_tx_stop: 16726
	tx_queue_1_tx_wake: 16726
	...
	tx_queue_8_tx_stop: 1500110
	tx_queue_8_tx_wake: 1500110
Quoted 6 lines
Quoted 2 lines

Hmm isn't one always same as the other, except when queue is stopped? And when it is stopped you can see that in the status?
So how is having two useful?

At idle the counters will be the same, unless a tx_timeout occurs. But under load they can be monitored to see which queues are stopped and get an idea of how long they are stopped.

Other net drivers (not all), also have the wake counter. In my opinion it makes the stop counter more useful, at little cost.

Quoted 87 lines

drivers/net/virtio_net.c static const struct virtnet_stat_desc

Quoted 20 lines

---
 drivers/net/virtio_net.c | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)
drivers/net/virtio_net.c index renamed from drivers/net/virtio_net.c +0 -0

3cb8aa193884..7e3c31ceaf7e 100644

drivers/net/virtio_net.c +2 -0
@@ -88,6 +88,8 @@ struct virtnet_sq_stats {
 	u64_stats_t xdp_tx_drops;
 	u64_stats_t kicks;
 	u64_stats_t tx_timeouts;
+	u64_stats_t tx_stop;
+	u64_stats_t tx_wake;
 };
 
 struct virtnet_rq_stats {
@@ -112,6 +114,8 @@ static const struct virtnet_stat_desc

virtnet_sq_stats_desc[] = {

drivers/net/virtio_net.c static const struct virtnet_stat_desc virtnet_sq_stats_desc[] = {

Quoted 9 lines
drivers/net/virtio_net.c +2 -1
 	{ "xdp_tx_drops",	VIRTNET_SQ_STAT(xdp_tx_drops) },
 	{ "kicks",		VIRTNET_SQ_STAT(kicks) },
 	{ "tx_timeouts",	VIRTNET_SQ_STAT(tx_timeouts) },
+	{ "tx_stop",		VIRTNET_SQ_STAT(tx_stop) },
+	{ "tx_wake",		VIRTNET_SQ_STAT(tx_wake) },
 };
 
 static const struct virtnet_stat_desc virtnet_rq_stats_desc[] = { @@
-843,6 +847,9 @@ static void check_sq_full_and_disable(struct virtnet_info

*vi,

drivers/net/virtio_net.c static void virtnet_poll_cleantx(struct

Quoted 21 lines
drivers/net/virtio_net.c +3 -0
 	 */
 	if (sq->vq->num_free < 2+MAX_SKB_FRAGS) {
 		netif_stop_subqueue(dev, qnum);
+		u64_stats_update_begin(&sq->stats.syncp);
+		u64_stats_inc(&sq->stats.tx_stop);
+		u64_stats_update_end(&sq->stats.syncp);
 		if (use_napi) {
 			if (unlikely(!virtqueue_enable_cb_delayed(sq->vq)))
 				virtqueue_napi_schedule(&sq->napi, sq-

vq); @@ -851,6 +858,9 @@
static void check_sq_full_and_disable(struct virtnet_info *vi,

drivers/net/virtio_net.c +3 -0
 			free_old_xmit_skbs(sq, false);
 			if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) {
 				netif_start_subqueue(dev, qnum);
+				u64_stats_update_begin(&sq->stats.syncp);
+				u64_stats_inc(&sq->stats.tx_wake);
+				u64_stats_update_end(&sq->stats.syncp);
 				virtqueue_disable_cb(sq->vq);
 			}
 		}
@@ -2163,8 +2173,14 @@ static void virtnet_poll_cleantx(struct

receive_queue *rq)

drivers/net/virtio_net.c static int virtnet_poll_tx(struct napi_struct

Quoted 16 lines
drivers/net/virtio_net.c +7 -1
 			free_old_xmit_skbs(sq, true);
 		} while (unlikely(!virtqueue_enable_cb_delayed(sq->vq)));
 
-		if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
+		if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) {
+			if (netif_tx_queue_stopped(txq)) {
+				u64_stats_update_begin(&sq->stats.syncp);
+				u64_stats_inc(&sq->stats.tx_wake);
+				u64_stats_update_end(&sq->stats.syncp);
+			}
 			netif_tx_wake_queue(txq);
+		}
 
 		__netif_tx_unlock(txq);
 	}
@@ -2310,8 +2326,14 @@ static int virtnet_poll_tx(struct napi_struct

*napi, int budget)

Quoted 17 lines
drivers/net/virtio_net.c +7 -2
 	virtqueue_disable_cb(sq->vq);
 	free_old_xmit_skbs(sq, true);
 
-	if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
+	if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) {
+		if (netif_tx_queue_stopped(txq)) {
+			u64_stats_update_begin(&sq->stats.syncp);
+			u64_stats_inc(&sq->stats.tx_wake);
+			u64_stats_update_end(&sq->stats.syncp);
+		}
 		netif_tx_wake_queue(txq);
+	}
 
 	opaque = virtqueue_enable_cb_prepare(sq->vq);
 
--

2.42.0

#3 lore raw
Quoted 14 lines · Daniel Jurgens
Quoted 14 lines

From: Michael S. Tsirkin <[email protected]>
Sent: Tuesday, January 30, 2024 8:58 AM

Quoted 10 lines · Daniel Jurgens

Add a tx queue stop and wake counters, they are useful for debugging.

	$ ethtool -S ens5f2 | grep 'tx_stop\|tx_wake'
	...
	tx_queue_1_tx_stop: 16726
	tx_queue_1_tx_wake: 16726
	...
	tx_queue_8_tx_stop: 1500110
	tx_queue_8_tx_wake: 1500110
Quoted 8 lines
Quoted 6 lines
Quoted 2 lines

Hmm isn't one always same as the other, except when queue is stopped? And when it is stopped you can see that in the status?
So how is having two useful?

At idle the counters will be the same, unless a tx_timeout occurs. But under load they can be monitored to see which queues are stopped and get an idea of how long they are stopped.

how does it give you the idea of how long they are stopped?

Quoted 1 line

Other net drivers (not all), also have the wake counter.

Examples?

Quoted 89 lines

In my opinion it makes the stop counter more useful, at little cost.

Quoted 87 lines

drivers/net/virtio_net.c static const struct virtnet_stat_desc

Quoted 20 lines

---
 drivers/net/virtio_net.c | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)
drivers/net/virtio_net.c index renamed from drivers/net/virtio_net.c +0 -0

3cb8aa193884..7e3c31ceaf7e 100644

drivers/net/virtio_net.c +2 -0
@@ -88,6 +88,8 @@ struct virtnet_sq_stats {
 	u64_stats_t xdp_tx_drops;
 	u64_stats_t kicks;
 	u64_stats_t tx_timeouts;
+	u64_stats_t tx_stop;
+	u64_stats_t tx_wake;
 };
 
 struct virtnet_rq_stats {
@@ -112,6 +114,8 @@ static const struct virtnet_stat_desc

virtnet_sq_stats_desc[] = {

drivers/net/virtio_net.c static const struct virtnet_stat_desc virtnet_sq_stats_desc[] = {

Quoted 9 lines
drivers/net/virtio_net.c +2 -1
 	{ "xdp_tx_drops",	VIRTNET_SQ_STAT(xdp_tx_drops) },
 	{ "kicks",		VIRTNET_SQ_STAT(kicks) },
 	{ "tx_timeouts",	VIRTNET_SQ_STAT(tx_timeouts) },
+	{ "tx_stop",		VIRTNET_SQ_STAT(tx_stop) },
+	{ "tx_wake",		VIRTNET_SQ_STAT(tx_wake) },
 };
 
 static const struct virtnet_stat_desc virtnet_rq_stats_desc[] = { @@
-843,6 +847,9 @@ static void check_sq_full_and_disable(struct virtnet_info

*vi,

drivers/net/virtio_net.c static void virtnet_poll_cleantx(struct

Quoted 21 lines
drivers/net/virtio_net.c +3 -0
 	 */
 	if (sq->vq->num_free < 2+MAX_SKB_FRAGS) {
 		netif_stop_subqueue(dev, qnum);
+		u64_stats_update_begin(&sq->stats.syncp);
+		u64_stats_inc(&sq->stats.tx_stop);
+		u64_stats_update_end(&sq->stats.syncp);
 		if (use_napi) {
 			if (unlikely(!virtqueue_enable_cb_delayed(sq->vq)))
 				virtqueue_napi_schedule(&sq->napi, sq-

vq); @@ -851,6 +858,9 @@
static void check_sq_full_and_disable(struct virtnet_info *vi,

drivers/net/virtio_net.c +3 -0
 			free_old_xmit_skbs(sq, false);
 			if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) {
 				netif_start_subqueue(dev, qnum);
+				u64_stats_update_begin(&sq->stats.syncp);
+				u64_stats_inc(&sq->stats.tx_wake);
+				u64_stats_update_end(&sq->stats.syncp);
 				virtqueue_disable_cb(sq->vq);
 			}
 		}
@@ -2163,8 +2173,14 @@ static void virtnet_poll_cleantx(struct

receive_queue *rq)

drivers/net/virtio_net.c static int virtnet_poll_tx(struct napi_struct

Quoted 16 lines
drivers/net/virtio_net.c +7 -1
 			free_old_xmit_skbs(sq, true);
 		} while (unlikely(!virtqueue_enable_cb_delayed(sq->vq)));
 
-		if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
+		if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) {
+			if (netif_tx_queue_stopped(txq)) {
+				u64_stats_update_begin(&sq->stats.syncp);
+				u64_stats_inc(&sq->stats.tx_wake);
+				u64_stats_update_end(&sq->stats.syncp);
+			}
 			netif_tx_wake_queue(txq);
+		}
 
 		__netif_tx_unlock(txq);
 	}
@@ -2310,8 +2326,14 @@ static int virtnet_poll_tx(struct napi_struct

*napi, int budget)

Quoted 17 lines
drivers/net/virtio_net.c +7 -2
 	virtqueue_disable_cb(sq->vq);
 	free_old_xmit_skbs(sq, true);
 
-	if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
+	if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) {
+		if (netif_tx_queue_stopped(txq)) {
+			u64_stats_update_begin(&sq->stats.syncp);
+			u64_stats_inc(&sq->stats.tx_wake);
+			u64_stats_update_end(&sq->stats.syncp);
+		}
 		netif_tx_wake_queue(txq);
+	}
 
 	opaque = virtqueue_enable_cb_prepare(sq->vq);
 
--

2.42.0

#29 lore raw
In reply to Heng Qi
Quoted 33 lines

From: Heng Qi <[email protected]>
Sent: Tuesday, January 30, 2024 9:17 AM

Quoted 21 lines · Daniel Jurgens

Add a tx queue stop and wake counters, they are useful for debugging.

	$ ethtool -S ens5f2 | grep 'tx_stop\|tx_wake'
	...
	tx_queue_1_tx_stop: 16726
	tx_queue_1_tx_wake: 16726
	...
	tx_queue_8_tx_stop: 1500110
	tx_queue_8_tx_wake: 1500110
---
  drivers/net/virtio_net.c | 26 ++++++++++++++++++++++++--
  1 file changed, 24 insertions(+), 2 deletions(-)
drivers/net/virtio_net.c index renamed from drivers/net/virtio_net.c +0 -0

3cb8aa193884..7e3c31ceaf7e 100644

drivers/net/virtio_net.c struct virtnet_sq_stats { · line 93

Quoted 6 lines
drivers/net/virtio_net.c +2 -0
@@ -88,6 +88,8 @@ struct virtnet_sq_stats {
  	u64_stats_t xdp_tx_drops;
  	u64_stats_t kicks;
  	u64_stats_t tx_timeouts;
+	u64_stats_t tx_stop;
+	u64_stats_t tx_wake;
  };

Hi Daniel!

Quoted 4 lines

tx_stop/wake only counts the status in the I/O path.
Do the status of virtnet_config_changed_work and virtnet_tx_resize need to be counted?

My motivation for the counter is detecting full TX queues. I don't think counting them in the control path is useful, but it can be done if you disagree.

Quoted 71 lines

Thanks,
Heng

drivers/net/virtio_net.c static const struct virtnet_stat_desc

Quoted 3 lines
drivers/net/virtio_net.c +0 -0
  struct virtnet_rq_stats {
@@ -112,6 +114,8 @@ static const struct virtnet_stat_desc

virtnet_sq_stats_desc[] = {

drivers/net/virtio_net.c static const struct virtnet_stat_desc virtnet_sq_stats_desc[] = {

Quoted 9 lines
drivers/net/virtio_net.c +2 -1
  	{ "xdp_tx_drops",	VIRTNET_SQ_STAT(xdp_tx_drops) },
  	{ "kicks",		VIRTNET_SQ_STAT(kicks) },
  	{ "tx_timeouts",	VIRTNET_SQ_STAT(tx_timeouts) },
+	{ "tx_stop",		VIRTNET_SQ_STAT(tx_stop) },
+	{ "tx_wake",		VIRTNET_SQ_STAT(tx_wake) },
  };
 
  static const struct virtnet_stat_desc virtnet_rq_stats_desc[] = { @@
-843,6 +847,9 @@ static void check_sq_full_and_disable(struct virtnet_info

*vi,

drivers/net/virtio_net.c static void virtnet_poll_cleantx(struct

Quoted 21 lines
  	 */
  	if (sq->vq->num_free < 2+MAX_SKB_FRAGS) {
  		netif_stop_subqueue(dev, qnum);
drivers/net/virtio_net.c +3 -0
+		u64_stats_update_begin(&sq->stats.syncp);
+		u64_stats_inc(&sq->stats.tx_stop);
+		u64_stats_update_end(&sq->stats.syncp);
  		if (use_napi) {
  			if (unlikely(!virtqueue_enable_cb_delayed(sq->vq)))
  				virtqueue_napi_schedule(&sq->napi, sq-

vq); @@ -851,6 +858,9 @@
static void check_sq_full_and_disable(struct virtnet_info *vi,
free_old_xmit_skbs(sq, false); if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) {
netif_start_subqueue(dev, qnum);

drivers/net/virtio_net.c +3 -0
+				u64_stats_update_begin(&sq->stats.syncp);
+				u64_stats_inc(&sq->stats.tx_wake);
+				u64_stats_update_end(&sq->stats.syncp);
  				virtqueue_disable_cb(sq->vq);
  			}
  		}
@@ -2163,8 +2173,14 @@ static void virtnet_poll_cleantx(struct

receive_queue *rq)

drivers/net/virtio_net.c static int virtnet_poll_tx(struct napi_struct

Quoted 16 lines
drivers/net/virtio_net.c +7 -1
  			free_old_xmit_skbs(sq, true);
  		} while (unlikely(!virtqueue_enable_cb_delayed(sq->vq)));
 
-		if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
+		if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) {
+			if (netif_tx_queue_stopped(txq)) {
+				u64_stats_update_begin(&sq->stats.syncp);
+				u64_stats_inc(&sq->stats.tx_wake);
+				u64_stats_update_end(&sq->stats.syncp);
+			}
  			netif_tx_wake_queue(txq);
+		}
 
  		__netif_tx_unlock(txq);
  	}
@@ -2310,8 +2326,14 @@ static int virtnet_poll_tx(struct napi_struct

*napi, int budget)

Quoted 15 lines
drivers/net/virtio_net.c +7 -1
  	virtqueue_disable_cb(sq->vq);
  	free_old_xmit_skbs(sq, true);
 
-	if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
+	if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) {
+		if (netif_tx_queue_stopped(txq)) {
+			u64_stats_update_begin(&sq->stats.syncp);
+			u64_stats_inc(&sq->stats.tx_wake);
+			u64_stats_update_end(&sq->stats.syncp);
+		}
  		netif_tx_wake_queue(txq);
+	}
 
  	opaque = virtqueue_enable_cb_prepare(sq->vq);
 
#4 lore raw
In reply to Michael S. Tsirkin
Quoted 24 lines

From: Michael S. Tsirkin <[email protected]>
Sent: Tuesday, January 30, 2024 9:42 AM

Quoted 21 lines · Daniel Jurgens
Quoted 20 lines

From: Michael S. Tsirkin <[email protected]>
Sent: Tuesday, January 30, 2024 8:58 AM

Quoted 10 lines · Daniel Jurgens

Add a tx queue stop and wake counters, they are useful for debugging.

	$ ethtool -S ens5f2 | grep 'tx_stop\|tx_wake'
	...
	tx_queue_1_tx_stop: 16726
	tx_queue_1_tx_wake: 16726
	...
	tx_queue_8_tx_stop: 1500110
	tx_queue_8_tx_wake: 1500110
Quoted 2 lines

Hmm isn't one always same as the other, except when queue is stopped? And when it is stopped you can see that in the status?
So how is having two useful?

Quoted 5 lines
Quoted 1 line

At idle the counters will be the same, unless a tx_timeout occurs. But

under load they can be monitored to see which queues are stopped and get an idea of how long they are stopped.

how does it give you the idea of how long they are stopped?

By serially monitoring the counter you can see stops that persist long intervals that are less than the tx_timeout time.

Quoted 4 lines
Quoted 2 lines

Other net drivers (not all), also have the wake counter.

Examples?

[danielj@sw-mtx-051 upstream]$ ethtool -i ens2f1np1
driver: mlx5_core
version: 6.7.0+
...
[danielj@sw-mtx-051 upstream]$ ethtool -S ens2f1np1 | grep wake

     tx_queue_wake: 0
     tx0_wake: 0
Quoted 93 lines
Quoted 36 lines

In my opinion it makes the stop counter more useful, at little cost.

Quoted 33 lines

drivers/net/virtio_net.c static const struct virtnet_stat_desc

Quoted 20 lines

---
 drivers/net/virtio_net.c | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)
drivers/net/virtio_net.c +2 -0
index 3cb8aa193884..7e3c31ceaf7e 100644
@@ -88,6 +88,8 @@ struct virtnet_sq_stats {
 	u64_stats_t xdp_tx_drops;
 	u64_stats_t kicks;
 	u64_stats_t tx_timeouts;
+	u64_stats_t tx_stop;
+	u64_stats_t tx_wake;
 };
 
 struct virtnet_rq_stats {
@@ -112,6 +114,8 @@ static const struct virtnet_stat_desc

virtnet_sq_stats_desc[] = {

Quoted 11 lines
drivers/net/virtio_net.c +2 -0
 	{ "xdp_tx_drops",	VIRTNET_SQ_STAT(xdp_tx_drops) },
 	{ "kicks",		VIRTNET_SQ_STAT(kicks) },
 	{ "tx_timeouts",	VIRTNET_SQ_STAT(tx_timeouts) },
+	{ "tx_stop",		VIRTNET_SQ_STAT(tx_stop) },
+	{ "tx_wake",		VIRTNET_SQ_STAT(tx_wake) },
 };
 
 static const struct virtnet_stat_desc virtnet_rq_stats_desc[] = {

@@
-843,6 +847,9 @@ static void check_sq_full_and_disable(struct virtnet_info

*vi,

Quoted 9 lines
Quoted 9 lines

drivers/net/virtio_net.c static void check_sq_full_and_disable(struct virtnet_info *vi,

Quoted 9 lines
drivers/net/virtio_net.c +3 -0
 	 */
 	if (sq->vq->num_free < 2+MAX_SKB_FRAGS) {
 		netif_stop_subqueue(dev, qnum);
+		u64_stats_update_begin(&sq->stats.syncp);
+		u64_stats_inc(&sq->stats.tx_stop);
+		u64_stats_update_end(&sq->stats.syncp);
 		if (use_napi) {
 			if (unlikely(!virtqueue_enable_cb_delayed(sq->vq)))
 				virtqueue_napi_schedule(&sq->napi, sq- vq);

@@ -851,6 +858,9

Quoted 47 lines
Quoted 47 lines

drivers/net/virtio_net.c static void virtnet_poll_cleantx(struct

Quoted 11 lines

@@ static void check_sq_full_and_disable(struct virtnet_info *vi,

drivers/net/virtio_net.c +3 -0
 			free_old_xmit_skbs(sq, false);
 			if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) {
 				netif_start_subqueue(dev, qnum);
+				u64_stats_update_begin(&sq->stats.syncp);
+				u64_stats_inc(&sq->stats.tx_wake);
+				u64_stats_update_end(&sq->stats.syncp);
 				virtqueue_disable_cb(sq->vq);
 			}
 		}
@@ -2163,8 +2173,14 @@ static void virtnet_poll_cleantx(struct

receive_queue *rq)

Quoted 16 lines
drivers/net/virtio_net.c +7 -1
 			free_old_xmit_skbs(sq, true);
 		} while (unlikely(!virtqueue_enable_cb_delayed(sq->vq)));
 
-		if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
+		if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) {
+			if (netif_tx_queue_stopped(txq)) {
+				u64_stats_update_begin(&sq->stats.syncp);
+				u64_stats_inc(&sq->stats.tx_wake);
+				u64_stats_update_end(&sq->stats.syncp);
+			}
 			netif_tx_wake_queue(txq);
+		}
 
 		__netif_tx_unlock(txq);
 	}
@@ -2310,8 +2326,14 @@ static int virtnet_poll_tx(struct
Quoted 1 line

napi_struct

*napi, int budget)

Quoted 17 lines
drivers/net/virtio_net.c +7 -2
 	virtqueue_disable_cb(sq->vq);
 	free_old_xmit_skbs(sq, true);
 
-	if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
+	if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) {
+		if (netif_tx_queue_stopped(txq)) {
+			u64_stats_update_begin(&sq->stats.syncp);
+			u64_stats_inc(&sq->stats.tx_wake);
+			u64_stats_update_end(&sq->stats.syncp);
+		}
 		netif_tx_wake_queue(txq);
+	}
 
 	opaque = virtqueue_enable_cb_prepare(sq->vq);
 
--

2.42.0

#30 lore raw
In reply to Daniel Jurgens
Quoted 33 lines · Daniel Jurgens
Quoted 33 lines

From: Heng Qi <[email protected]>
Sent: Tuesday, January 30, 2024 9:17 AM

Quoted 21 lines · Daniel Jurgens

Add a tx queue stop and wake counters, they are useful for debugging.

	$ ethtool -S ens5f2 | grep 'tx_stop\|tx_wake'
	...
	tx_queue_1_tx_stop: 16726
	tx_queue_1_tx_wake: 16726
	...
	tx_queue_8_tx_stop: 1500110
	tx_queue_8_tx_wake: 1500110
---
  drivers/net/virtio_net.c | 26 ++++++++++++++++++++++++--
  1 file changed, 24 insertions(+), 2 deletions(-)
drivers/net/virtio_net.c index renamed from drivers/net/virtio_net.c +0 -0

3cb8aa193884..7e3c31ceaf7e 100644

drivers/net/virtio_net.c struct virtnet_sq_stats { · line 93

Quoted 6 lines
drivers/net/virtio_net.c +2 -0
@@ -88,6 +88,8 @@ struct virtnet_sq_stats {
  	u64_stats_t xdp_tx_drops;
  	u64_stats_t kicks;
  	u64_stats_t tx_timeouts;
+	u64_stats_t tx_stop;
+	u64_stats_t tx_wake;
  };

Hi Daniel!

Quoted 6 lines
Quoted 4 lines

tx_stop/wake only counts the status in the I/O path.
Do the status of virtnet_config_changed_work and virtnet_tx_resize need to be counted?

My motivation for the counter is detecting full TX queues. I don't think counting them in the control path is useful, but it can be done if you disagree.

Do we then just want "tx full" counter?

Quoted 72 lines
Quoted 71 lines

Thanks,
Heng

drivers/net/virtio_net.c static const struct virtnet_stat_desc

Quoted 3 lines
drivers/net/virtio_net.c +0 -0
  struct virtnet_rq_stats {
@@ -112,6 +114,8 @@ static const struct virtnet_stat_desc

virtnet_sq_stats_desc[] = {

drivers/net/virtio_net.c static const struct virtnet_stat_desc virtnet_sq_stats_desc[] = {

Quoted 9 lines
drivers/net/virtio_net.c +2 -1
  	{ "xdp_tx_drops",	VIRTNET_SQ_STAT(xdp_tx_drops) },
  	{ "kicks",		VIRTNET_SQ_STAT(kicks) },
  	{ "tx_timeouts",	VIRTNET_SQ_STAT(tx_timeouts) },
+	{ "tx_stop",		VIRTNET_SQ_STAT(tx_stop) },
+	{ "tx_wake",		VIRTNET_SQ_STAT(tx_wake) },
  };
 
  static const struct virtnet_stat_desc virtnet_rq_stats_desc[] = { @@
-843,6 +847,9 @@ static void check_sq_full_and_disable(struct virtnet_info

*vi,

drivers/net/virtio_net.c static void virtnet_poll_cleantx(struct

Quoted 21 lines
  	 */
  	if (sq->vq->num_free < 2+MAX_SKB_FRAGS) {
  		netif_stop_subqueue(dev, qnum);
drivers/net/virtio_net.c +3 -0
+		u64_stats_update_begin(&sq->stats.syncp);
+		u64_stats_inc(&sq->stats.tx_stop);
+		u64_stats_update_end(&sq->stats.syncp);
  		if (use_napi) {
  			if (unlikely(!virtqueue_enable_cb_delayed(sq->vq)))
  				virtqueue_napi_schedule(&sq->napi, sq-

vq); @@ -851,6 +858,9 @@
static void check_sq_full_and_disable(struct virtnet_info *vi,
free_old_xmit_skbs(sq, false); if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) {
netif_start_subqueue(dev, qnum);

drivers/net/virtio_net.c +3 -0
+				u64_stats_update_begin(&sq->stats.syncp);
+				u64_stats_inc(&sq->stats.tx_wake);
+				u64_stats_update_end(&sq->stats.syncp);
  				virtqueue_disable_cb(sq->vq);
  			}
  		}
@@ -2163,8 +2173,14 @@ static void virtnet_poll_cleantx(struct

receive_queue *rq)

drivers/net/virtio_net.c static int virtnet_poll_tx(struct napi_struct

Quoted 16 lines
drivers/net/virtio_net.c +7 -1
  			free_old_xmit_skbs(sq, true);
  		} while (unlikely(!virtqueue_enable_cb_delayed(sq->vq)));
 
-		if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
+		if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) {
+			if (netif_tx_queue_stopped(txq)) {
+				u64_stats_update_begin(&sq->stats.syncp);
+				u64_stats_inc(&sq->stats.tx_wake);
+				u64_stats_update_end(&sq->stats.syncp);
+			}
  			netif_tx_wake_queue(txq);
+		}
 
  		__netif_tx_unlock(txq);
  	}
@@ -2310,8 +2326,14 @@ static int virtnet_poll_tx(struct napi_struct

*napi, int budget)

Quoted 15 lines
drivers/net/virtio_net.c +7 -1
  	virtqueue_disable_cb(sq->vq);
  	free_old_xmit_skbs(sq, true);
 
-	if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
+	if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) {
+		if (netif_tx_queue_stopped(txq)) {
+			u64_stats_update_begin(&sq->stats.syncp);
+			u64_stats_inc(&sq->stats.tx_wake);
+			u64_stats_update_end(&sq->stats.syncp);
+		}
  		netif_tx_wake_queue(txq);
+	}
 
  	opaque = virtqueue_enable_cb_prepare(sq->vq);
 
#5 lore raw
In reply to Daniel Jurgens
Quoted 24 lines · Daniel Jurgens
Quoted 24 lines

From: Michael S. Tsirkin <[email protected]>
Sent: Tuesday, January 30, 2024 9:42 AM

Quoted 21 lines · Daniel Jurgens
Quoted 20 lines

From: Michael S. Tsirkin <[email protected]>
Sent: Tuesday, January 30, 2024 8:58 AM

Quoted 10 lines · Daniel Jurgens

Add a tx queue stop and wake counters, they are useful for debugging.

	$ ethtool -S ens5f2 | grep 'tx_stop\|tx_wake'
	...
	tx_queue_1_tx_stop: 16726
	tx_queue_1_tx_wake: 16726
	...
	tx_queue_8_tx_stop: 1500110
	tx_queue_8_tx_wake: 1500110
Quoted 2 lines

Hmm isn't one always same as the other, except when queue is stopped? And when it is stopped you can see that in the status?
So how is having two useful?

Quoted 7 lines
Quoted 5 lines
Quoted 1 line

At idle the counters will be the same, unless a tx_timeout occurs. But

under load they can be monitored to see which queues are stopped and get an idea of how long they are stopped.

how does it give you the idea of how long they are stopped?

By serially monitoring the counter you can see stops that persist long intervals that are less than the tx_timeout time.

Why don't you monitor queue status directly?

Quoted 5 lines
Quoted 4 lines
Quoted 2 lines

Other net drivers (not all), also have the wake counter.

Examples?

Quoted 7 lines

[danielj@sw-mtx-051 upstream]$ ethtool -i ens2f1np1
driver: mlx5_core
version: 6.7.0+
...
[danielj@sw-mtx-051 upstream]$ ethtool -S ens2f1np1 | grep wake

     tx_queue_wake: 0
     tx0_wake: 0

Do they have a stop counter too?

Quoted 93 lines
Quoted 93 lines
Quoted 36 lines

In my opinion it makes the stop counter more useful, at little cost.

Quoted 33 lines

drivers/net/virtio_net.c static const struct virtnet_stat_desc

Quoted 20 lines

---
 drivers/net/virtio_net.c | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)
drivers/net/virtio_net.c +2 -0
index 3cb8aa193884..7e3c31ceaf7e 100644
@@ -88,6 +88,8 @@ struct virtnet_sq_stats {
 	u64_stats_t xdp_tx_drops;
 	u64_stats_t kicks;
 	u64_stats_t tx_timeouts;
+	u64_stats_t tx_stop;
+	u64_stats_t tx_wake;
 };
 
 struct virtnet_rq_stats {
@@ -112,6 +114,8 @@ static const struct virtnet_stat_desc

virtnet_sq_stats_desc[] = {

Quoted 11 lines
drivers/net/virtio_net.c +2 -0
 	{ "xdp_tx_drops",	VIRTNET_SQ_STAT(xdp_tx_drops) },
 	{ "kicks",		VIRTNET_SQ_STAT(kicks) },
 	{ "tx_timeouts",	VIRTNET_SQ_STAT(tx_timeouts) },
+	{ "tx_stop",		VIRTNET_SQ_STAT(tx_stop) },
+	{ "tx_wake",		VIRTNET_SQ_STAT(tx_wake) },
 };
 
 static const struct virtnet_stat_desc virtnet_rq_stats_desc[] = {

@@
-843,6 +847,9 @@ static void check_sq_full_and_disable(struct virtnet_info

*vi,

Quoted 9 lines
Quoted 9 lines

drivers/net/virtio_net.c static void check_sq_full_and_disable(struct virtnet_info *vi,

Quoted 9 lines
drivers/net/virtio_net.c +3 -0
 	 */
 	if (sq->vq->num_free < 2+MAX_SKB_FRAGS) {
 		netif_stop_subqueue(dev, qnum);
+		u64_stats_update_begin(&sq->stats.syncp);
+		u64_stats_inc(&sq->stats.tx_stop);
+		u64_stats_update_end(&sq->stats.syncp);
 		if (use_napi) {
 			if (unlikely(!virtqueue_enable_cb_delayed(sq->vq)))
 				virtqueue_napi_schedule(&sq->napi, sq- vq);

@@ -851,6 +858,9

Quoted 47 lines
Quoted 47 lines

drivers/net/virtio_net.c static void virtnet_poll_cleantx(struct

Quoted 11 lines

@@ static void check_sq_full_and_disable(struct virtnet_info *vi,

drivers/net/virtio_net.c +3 -0
 			free_old_xmit_skbs(sq, false);
 			if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) {
 				netif_start_subqueue(dev, qnum);
+				u64_stats_update_begin(&sq->stats.syncp);
+				u64_stats_inc(&sq->stats.tx_wake);
+				u64_stats_update_end(&sq->stats.syncp);
 				virtqueue_disable_cb(sq->vq);
 			}
 		}
@@ -2163,8 +2173,14 @@ static void virtnet_poll_cleantx(struct

receive_queue *rq)

Quoted 16 lines
drivers/net/virtio_net.c +7 -1
 			free_old_xmit_skbs(sq, true);
 		} while (unlikely(!virtqueue_enable_cb_delayed(sq->vq)));
 
-		if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
+		if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) {
+			if (netif_tx_queue_stopped(txq)) {
+				u64_stats_update_begin(&sq->stats.syncp);
+				u64_stats_inc(&sq->stats.tx_wake);
+				u64_stats_update_end(&sq->stats.syncp);
+			}
 			netif_tx_wake_queue(txq);
+		}
 
 		__netif_tx_unlock(txq);
 	}
@@ -2310,8 +2326,14 @@ static int virtnet_poll_tx(struct
Quoted 1 line

napi_struct

*napi, int budget)

Quoted 17 lines
drivers/net/virtio_net.c +7 -2
 	virtqueue_disable_cb(sq->vq);
 	free_old_xmit_skbs(sq, true);
 
-	if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
+	if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) {
+		if (netif_tx_queue_stopped(txq)) {
+			u64_stats_update_begin(&sq->stats.syncp);
+			u64_stats_inc(&sq->stats.tx_wake);
+			u64_stats_update_end(&sq->stats.syncp);
+		}
 		netif_tx_wake_queue(txq);
+	}
 
 	opaque = virtqueue_enable_cb_prepare(sq->vq);
 
--

2.42.0

#6 lore raw
Quoted 34 lines

From: Michael S. Tsirkin <[email protected]>
Sent: Tuesday, January 30, 2024 9:53 AM

Quoted 8 lines · Daniel Jurgens
Quoted 8 lines

From: Michael S. Tsirkin <[email protected]>
Sent: Tuesday, January 30, 2024 9:42 AM On Tue, Jan 30, 2024 at

Quoted 5 lines · Daniel Jurgens
Quoted 5 lines

From: Michael S. Tsirkin <[email protected]>
Sent: Tuesday, January 30, 2024 8:58 AM

Quoted 1 line · Daniel Jurgens

Add a tx queue stop and wake counters, they are useful for

debugging.

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
	$ ethtool -S ens5f2 | grep 'tx_stop\|tx_wake'
	...
	tx_queue_1_tx_stop: 16726
	tx_queue_1_tx_wake: 16726
	...
	tx_queue_8_tx_stop: 1500110
	tx_queue_8_tx_wake: 1500110
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 2 lines

Hmm isn't one always same as the other, except when queue is

stopped?

Quoted 8 lines
Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

And when it is stopped you can see that in the status? So how is having two useful?

At idle the counters will be the same, unless a tx_timeout occurs. But

under load they can be monitored to see which queues are stopped and get an idea of how long they are stopped.

Quoted 6 lines
Quoted 3 lines
Quoted 1 line

how does it give you the idea of how long they are stopped?

By serially monitoring the counter you can see stops that persist long

intervals that are less than the tx_timeout time.

Why don't you monitor queue status directly?

How? I don't know of any interface to check if a queue is stopped.

Quoted 14 lines
Quoted 6 lines
Quoted 5 lines
Quoted 3 lines

Other net drivers (not all), also have the wake counter.

Examples?

Quoted 7 lines

[danielj@sw-mtx-051 upstream]$ ethtool -i ens2f1np1
driver: mlx5_core
version: 6.7.0+
...
[danielj@sw-mtx-051 upstream]$ ethtool -S ens2f1np1 | grep wake

     tx_queue_wake: 0
     tx0_wake: 0
Quoted 1 line

Do they have a stop counter too?

Yes:
[danielj@sw-mtx-051 upstream]$ ethtool -S ens2f1np1 | grep 'stop\|wake'

     tx_queue_stopped: 0
     tx_queue_wake: 0
     tx0_stopped: 0
     tx0_wake: 0
     ....
Quoted 103 lines
Quoted 38 lines
Quoted 38 lines
Quoted 38 lines

In my opinion it makes the stop counter more useful, at little cost.

Quoted 34 lines

drivers/net/virtio_net.c static const struct virtnet_stat_desc

Quoted 21 lines

---
 drivers/net/virtio_net.c | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)
(quoted diff) +0 -0

b/drivers/net/virtio_net.c index 3cb8aa193884..7e3c31ceaf7e 100644

drivers/net/virtio_net.c +2 -0
@@ -88,6 +88,8 @@ struct virtnet_sq_stats {
 	u64_stats_t xdp_tx_drops;
 	u64_stats_t kicks;
 	u64_stats_t tx_timeouts;
+	u64_stats_t tx_stop;
+	u64_stats_t tx_wake;
 };
 
 struct virtnet_rq_stats {
@@ -112,6 +114,8 @@ static const struct virtnet_stat_desc

virtnet_sq_stats_desc[] = {

Quoted 11 lines
drivers/net/virtio_net.c +2 -0
 	{ "xdp_tx_drops",	VIRTNET_SQ_STAT(xdp_tx_drops) },
 	{ "kicks",		VIRTNET_SQ_STAT(kicks) },
 	{ "tx_timeouts",	VIRTNET_SQ_STAT(tx_timeouts) },
+	{ "tx_stop",		VIRTNET_SQ_STAT(tx_stop) },
+	{ "tx_wake",		VIRTNET_SQ_STAT(tx_wake) },
 };
 
 static const struct virtnet_stat_desc virtnet_rq_stats_desc[]

= { @@
-843,6 +847,9 @@ static void check_sq_full_and_disable(struct virtnet_info

*vi,

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

drivers/net/virtio_net.c static void check_sq_full_and_disable(struct virtnet_info *vi,

Quoted 8 lines
drivers/net/virtio_net.c +3 -0
 	 */
 	if (sq->vq->num_free < 2+MAX_SKB_FRAGS) {
 		netif_stop_subqueue(dev, qnum);
+		u64_stats_update_begin(&sq->stats.syncp);
+		u64_stats_inc(&sq->stats.tx_stop);
+		u64_stats_update_end(&sq->stats.syncp);
 		if (use_napi) {
 			if

(unlikely(!virtqueue_enable_cb_delayed(sq->vq)))

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

virtqueue_napi_schedule(&sq->napi,

sq- vq);

Quoted 4 lines
Quoted 4 lines

@@ -851,6 +858,9

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

@@ static void check_sq_full_and_disable(struct virtnet_info *vi,

drivers/net/virtio_net.c +0 -0
 			free_old_xmit_skbs(sq, false);
 			if (sq->vq->num_free >=

2+MAX_SKB_FRAGS) {

Quoted 49 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

drivers/net/virtio_net.c

Quoted 2 lines
drivers/net/virtio_net.c +1 -0
 				netif_start_subqueue(dev, qnum);
+				u64_stats_update_begin(&sq-

stats.syncp);

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

drivers/net/virtio_net.c

Quoted 2 lines
drivers/net/virtio_net.c +2 -0
+				u64_stats_inc(&sq->stats.tx_wake);
+				u64_stats_update_end(&sq-

stats.syncp);

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

drivers/net/virtio_net.c static void virtnet_poll_cleantx(struct

Quoted 4 lines
drivers/net/virtio_net.c +0 -0
 				virtqueue_disable_cb(sq->vq);
 			}
 		}
@@ -2163,8 +2173,14 @@ static void virtnet_poll_cleantx(struct

receive_queue *rq)

Quoted 2 lines
drivers/net/virtio_net.c +0 -0
 			free_old_xmit_skbs(sq, true);
 		} while (unlikely(!virtqueue_enable_cb_delayed(sq-

vq)));

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

drivers/net/virtio_net.c static void virtnet_poll_cleantx(struct receive_queue *rq)

Quoted 5 lines
drivers/net/virtio_net.c +3 -1
-		if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
+		if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) {
+			if (netif_tx_queue_stopped(txq)) {
+				u64_stats_update_begin(&sq-

stats.syncp);

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

drivers/net/virtio_net.c static void virtnet_poll_cleantx(struct · line 2179

Quoted 2 lines
drivers/net/virtio_net.c +2 -0
+				u64_stats_inc(&sq->stats.tx_wake);
+				u64_stats_update_end(&sq-

stats.syncp);

Quoted 26 lines
Quoted 26 lines
Quoted 26 lines
Quoted 8 lines
drivers/net/virtio_net.c +2 -0
+			}
 			netif_tx_wake_queue(txq);
+		}
 
 		__netif_tx_unlock(txq);
 	}
@@ -2310,8 +2326,14 @@ static int virtnet_poll_tx(struct

napi_struct

*napi, int budget)

Quoted 17 lines
drivers/net/virtio_net.c +7 -2
 	virtqueue_disable_cb(sq->vq);
 	free_old_xmit_skbs(sq, true);
 
-	if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
+	if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) {
+		if (netif_tx_queue_stopped(txq)) {
+			u64_stats_update_begin(&sq->stats.syncp);
+			u64_stats_inc(&sq->stats.tx_wake);
+			u64_stats_update_end(&sq->stats.syncp);
+		}
 		netif_tx_wake_queue(txq);
+	}
 
 	opaque = virtqueue_enable_cb_prepare(sq->vq);
 
--

2.42.0

#7 lore raw
Quoted 58 lines · Daniel Jurgens
Quoted 35 lines

From: Michael S. Tsirkin <[email protected]>
Sent: Tuesday, January 30, 2024 9:53 AM

Quoted 8 lines · Daniel Jurgens
Quoted 8 lines

From: Michael S. Tsirkin <[email protected]>
Sent: Tuesday, January 30, 2024 9:42 AM On Tue, Jan 30, 2024 at

Quoted 5 lines · Daniel Jurgens
Quoted 5 lines

From: Michael S. Tsirkin <[email protected]>
Sent: Tuesday, January 30, 2024 8:58 AM

Quoted 1 line · Daniel Jurgens

Add a tx queue stop and wake counters, they are useful for

debugging.

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
    $ ethtool -S ens5f2 | grep 'tx_stop\|tx_wake'
    ...
    tx_queue_1_tx_stop: 16726
    tx_queue_1_tx_wake: 16726
    ...
    tx_queue_8_tx_stop: 1500110
    tx_queue_8_tx_wake: 1500110
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 2 lines

Hmm isn't one always same as the other, except when queue is

stopped?

Quoted 8 lines
Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

And when it is stopped you can see that in the status? So how is having two useful?

At idle the counters will be the same, unless a tx_timeout occurs. But

under load they can be monitored to see which queues are stopped and get an idea of how long they are stopped.

Quoted 6 lines
Quoted 3 lines
Quoted 1 line

how does it give you the idea of how long they are stopped?

By serially monitoring the counter you can see stops that persist long

intervals that are less than the tx_timeout time.

Why don't you monitor queue status directly?

How? I don't know of any interface to check if a queue is stopped.

Quoted 14 lines
Quoted 13 lines
Quoted 5 lines
Quoted 3 lines

Other net drivers (not all), also have the wake counter.

Examples?

[danielj@sw-mtx-051 upstream]$ ethtool -i ens2f1np1
driver: mlx5_core
version: 6.7.0+
...
[danielj@sw-mtx-051 upstream]$ ethtool -S ens2f1np1 | grep wake

     tx_queue_wake: 0
     tx0_wake: 0

[...]

Quoted 9 lines
Quoted 1 line

Do they have a stop counter too?

Yes:
[danielj@sw-mtx-051 upstream]$ ethtool -S ens2f1np1 | grep 'stop\|wake'

     tx_queue_stopped: 0
     tx_queue_wake: 0
     tx0_stopped: 0
     tx0_wake: 0
     ....

Yes, that's it! What I know is that only mlx drivers have those two counters, but they are very useful when debugging some issues or tracking some historical changes if we want to.

Thanks,
Jason

Quoted 106 lines
Quoted 104 lines
Quoted 39 lines
Quoted 39 lines
Quoted 39 lines

In my opinion it makes the stop counter more useful, at little cost.

Quoted 34 lines

drivers/net/virtio_net.c static const struct virtnet_stat_desc

Quoted 21 lines

---
 drivers/net/virtio_net.c | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)
(quoted diff) +0 -0

b/drivers/net/virtio_net.c index 3cb8aa193884..7e3c31ceaf7e 100644

drivers/net/virtio_net.c +2 -0
@@ -88,6 +88,8 @@ struct virtnet_sq_stats {
    u64_stats_t xdp_tx_drops;
    u64_stats_t kicks;
    u64_stats_t tx_timeouts;
+   u64_stats_t tx_stop;
+   u64_stats_t tx_wake;
 };
 
 struct virtnet_rq_stats {
@@ -112,6 +114,8 @@ static const struct virtnet_stat_desc

virtnet_sq_stats_desc[] = {

Quoted 11 lines
drivers/net/virtio_net.c +2 -0
    { "xdp_tx_drops",       VIRTNET_SQ_STAT(xdp_tx_drops) },
    { "kicks",              VIRTNET_SQ_STAT(kicks) },
    { "tx_timeouts",        VIRTNET_SQ_STAT(tx_timeouts) },
+   { "tx_stop",            VIRTNET_SQ_STAT(tx_stop) },
+   { "tx_wake",            VIRTNET_SQ_STAT(tx_wake) },
 };
 
 static const struct virtnet_stat_desc virtnet_rq_stats_desc[]

= { @@
-843,6 +847,9 @@ static void check_sq_full_and_disable(struct virtnet_info

*vi,

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

drivers/net/virtio_net.c

Quoted 8 lines
     */
    if (sq->vq->num_free < 2+MAX_SKB_FRAGS) {
            netif_stop_subqueue(dev, qnum);
drivers/net/virtio_net.c +3 -0
+           u64_stats_update_begin(&sq->stats.syncp);
+           u64_stats_inc(&sq->stats.tx_stop);
+           u64_stats_update_end(&sq->stats.syncp);
            if (use_napi) {
                    if

(unlikely(!virtqueue_enable_cb_delayed(sq->vq)))

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
                            virtqueue_napi_schedule(&sq->napi,

sq- vq);

Quoted 4 lines
Quoted 4 lines

@@ -851,6 +858,9

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

@@ static void check_sq_full_and_disable(struct virtnet_info *vi,

                    free_old_xmit_skbs(sq, false);
                    if (sq->vq->num_free >=

2+MAX_SKB_FRAGS) {

Quoted 49 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
                            netif_start_subqueue(dev, qnum);

+ u64_stats_update_begin(&sq-

stats.syncp);

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

drivers/net/virtio_net.c

Quoted 2 lines
drivers/net/virtio_net.c +2 -0
+                           u64_stats_inc(&sq->stats.tx_wake);
+                           u64_stats_update_end(&sq-

stats.syncp);

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

drivers/net/virtio_net.c static void virtnet_poll_cleantx(struct

Quoted 4 lines
                            virtqueue_disable_cb(sq->vq);
                    }
            }
drivers/net/virtio_net.c +0 -0
@@ -2163,8 +2173,14 @@ static void virtnet_poll_cleantx(struct

receive_queue *rq)

Quoted 2 lines
drivers/net/virtio_net.c +0 -0
                    free_old_xmit_skbs(sq, true);
            } while (unlikely(!virtqueue_enable_cb_delayed(sq-

vq)));

Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

drivers/net/virtio_net.c static void virtnet_poll_cleantx(struct · line 2177

Quoted 5 lines
drivers/net/virtio_net.c +3 -1
-           if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
+           if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) {
+                   if (netif_tx_queue_stopped(txq)) {
+                           u64_stats_update_begin(&sq-

stats.syncp);

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

drivers/net/virtio_net.c static void virtnet_poll_cleantx(struct · line 2179

Quoted 2 lines
drivers/net/virtio_net.c +2 -0
+                           u64_stats_inc(&sq->stats.tx_wake);
+                           u64_stats_update_end(&sq-

stats.syncp);

Quoted 26 lines
Quoted 26 lines
Quoted 26 lines
Quoted 8 lines
drivers/net/virtio_net.c +2 -0
+                   }
                    netif_tx_wake_queue(txq);
+           }
 
            __netif_tx_unlock(txq);
    }
@@ -2310,8 +2326,14 @@ static int virtnet_poll_tx(struct

napi_struct

*napi, int budget)

Quoted 17 lines
    virtqueue_disable_cb(sq->vq);
    free_old_xmit_skbs(sq, true);
drivers/net/virtio_net.c +7 -2
-   if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
+   if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) {
+           if (netif_tx_queue_stopped(txq)) {
+                   u64_stats_update_begin(&sq->stats.syncp);
+                   u64_stats_inc(&sq->stats.tx_wake);
+                   u64_stats_update_end(&sq->stats.syncp);
+           }
            netif_tx_wake_queue(txq);
+   }
 
    opaque = virtqueue_enable_cb_prepare(sq->vq);
 
--

2.42.0

maintainer
#8 lore raw
Quoted 10 lines · On Wed, 31 Jan 2024 10:54:33 +0800 Jason Xing wrote:
Quoted 6 lines

[danielj@sw-mtx-051 upstream]$ ethtool -S ens2f1np1 | grep 'stop\|wake'

     tx_queue_stopped: 0
     tx_queue_wake: 0
     tx0_stopped: 0
     tx0_wake: 0
     ....  

Yes, that's it! What I know is that only mlx drivers have those two counters, but they are very useful when debugging some issues or tracking some historical changes if we want to.

Can you say more? I'm curious what's your use case.

#9 lore raw
Quoted 9 lines · Jakub Kicinski
Quoted 7 lines · On Wed, 31 Jan 2024 10:54:33 +0800 Jason Xing wrote:
Quoted 6 lines

[danielj@sw-mtx-051 upstream]$ ethtool -S ens2f1np1 | grep 'stop\|wake'

     tx_queue_stopped: 0
     tx_queue_wake: 0
     tx0_stopped: 0
     tx0_wake: 0
     ....
Quoted 5 lines
Quoted 3 lines

Yes, that's it! What I know is that only mlx drivers have those two counters, but they are very useful when debugging some issues or tracking some historical changes if we want to.

Can you say more? I'm curious what's your use case.

I'm not working at Nvidia, so my point of view may differ from theirs. From what I can tell is that those two counters help me narrow down the range if I have to diagnose/debug some issues.
1) I sometimes notice that if some irq is held too long (say, one simple case: output of printk printed to the console), those two counters can reflect the issue.
2) Similarly in virtio net, recently I traced such counters the current kernel does not have and it turned out that one of the output queues in the backend behaves badly.
...

Stop/wake queue counters may not show directly the root cause of the issue, but help us 'guess' to some extent.

Thanks,
Jason

maintainer
#10 lore raw
Quoted 5 lines · On Fri, 2 Feb 2024 14:52:59 +0800 Jason Xing wrote:
Quoted 1 line

Can you say more? I'm curious what's your use case.

I'm not working at Nvidia, so my point of view may differ from theirs. From what I can tell is that those two counters help me narrow down the range if I have to diagnose/debug some issues.

right, i'm asking to collect useful debugging tricks, nothing against the patch itself :)

Quoted 10 lines

1) I sometimes notice that if some irq is held too long (say, one simple case: output of printk printed to the console), those two counters can reflect the issue.
2) Similarly in virtio net, recently I traced such counters the current kernel does not have and it turned out that one of the output queues in the backend behaves badly.
...

Stop/wake queue counters may not show directly the root cause of the issue, but help us 'guess' to some extent.

I'm surprised you say you can detect stall-related issues with this. I guess virtio doesn't have BQL support, which makes it special. Normal HW drivers with BQL almost never stop the queue by themselves. I mean - if they do, and BQL is active, then the system is probably misconfigured (queue is too short). This is what we use at Meta to detect stalls in drivers with BQL:

https://lore.kernel.org/all/[email protected]/

Daniel, I think this may be a good enough excuse to add per-queue stats to the netdev genl family, if you're up for that. LMK if you want more info, otherwise I guess ethtool -S is fine for now.

#11 lore raw
Quoted 32 lines

From: Jakub Kicinski <[email protected]>
Sent: Friday, February 2, 2024 10:01 AM
Subject: Re: [PATCH net-next] virtio_net: Add TX stop and wake counters

Quoted 5 lines · On Fri, 2 Feb 2024 14:52:59 +0800 Jason Xing wrote:
Quoted 1 line

Can you say more? I'm curious what's your use case.

I'm not working at Nvidia, so my point of view may differ from theirs. From what I can tell is that those two counters help me narrow down the range if I have to diagnose/debug some issues.

right, i'm asking to collect useful debugging tricks, nothing against the patch itself :)

Quoted 10 lines

1) I sometimes notice that if some irq is held too long (say, one simple case: output of printk printed to the console), those two counters can reflect the issue.
2) Similarly in virtio net, recently I traced such counters the current kernel does not have and it turned out that one of the output queues in the backend behaves badly.
...

Stop/wake queue counters may not show directly the root cause of the issue, but help us 'guess' to some extent.

I'm surprised you say you can detect stall-related issues with this. I guess virtio doesn't have BQL support, which makes it special. Normal HW drivers with BQL almost never stop the queue by themselves. I mean - if they do, and BQL is active, then the system is probably misconfigured (queue is too short). This is what we use at Meta to detect stalls in drivers with BQL:

Quoted 5 lines

https://lore.kernel.org/all/[email protected]/

Daniel, I think this may be a good enough excuse to add per-queue stats to the netdev genl family, if you're up for that. LMK if you want more info, otherwise I guess ethtool -S is fine for now.

For now, I would like to go with ethtool -S. But I can take on the netdev level as a background task. Are you suggesting adding them to rtnl_link_stats64?

maintainer
#12 lore raw
Quoted 7 lines · On Fri, 2 Feb 2024 16:46:28 +0000 Daniel Jurgens wrote:
Quoted 3 lines

Daniel, I think this may be a good enough excuse to add per-queue stats to the netdev genl family, if you're up for that. LMK if you want more info, otherwise I guess ethtool -S is fine for now.

For now, I would like to go with ethtool -S. But I can take on the netdev level as a background task. Are you suggesting adding them to rtnl_link_stats64?

More like page pool stats. See d49010adae7376384

#18 lore raw
In reply to Jakub Kicinski
Quoted 19 lines · Jakub Kicinski
Quoted 5 lines · On Fri, 2 Feb 2024 14:52:59 +0800 Jason Xing wrote:
Quoted 1 line

Can you say more? I'm curious what's your use case.

I'm not working at Nvidia, so my point of view may differ from theirs. From what I can tell is that those two counters help me narrow down the range if I have to diagnose/debug some issues.

right, i'm asking to collect useful debugging tricks, nothing against the patch itself :)

Quoted 8 lines

1) I sometimes notice that if some irq is held too long (say, one simple case: output of printk printed to the console), those two counters can reflect the issue.
2) Similarly in virtio net, recently I traced such counters the current kernel does not have and it turned out that one of the output queues in the backend behaves badly.
...

Quoted 5 lines
Quoted 2 lines

Stop/wake queue counters may not show directly the root cause of the issue, but help us 'guess' to some extent.

I'm surprised you say you can detect stall-related issues with this. I guess virtio doesn't have BQL support, which makes it special.

Yes, virtio-net has a legacy orphan mode, this is something that needs to be dropped in the future. This would make BQL much more easier to be implemented.

Quoted 11 lines

Normal HW drivers with BQL almost never stop the queue by themselves. I mean - if they do, and BQL is active, then the system is probably misconfigured (queue is too short). This is what we use at Meta to detect stalls in drivers with BQL:

https://lore.kernel.org/all/[email protected]/

Daniel, I think this may be a good enough excuse to add per-queue stats to the netdev genl family, if you're up for that. LMK if you want more info, otherwise I guess ethtool -S is fine for now.

Thanks

#19 lore raw
Quoted 23 lines · Jason Wang
Quoted 22 lines · Jakub Kicinski
Quoted 5 lines · On Fri, 2 Feb 2024 14:52:59 +0800 Jason Xing wrote:
Quoted 1 line

Can you say more? I'm curious what's your use case.

I'm not working at Nvidia, so my point of view may differ from theirs. From what I can tell is that those two counters help me narrow down the range if I have to diagnose/debug some issues.

right, i'm asking to collect useful debugging tricks, nothing against the patch itself :)

Quoted 10 lines

1) I sometimes notice that if some irq is held too long (say, one simple case: output of printk printed to the console), those two counters can reflect the issue.
2) Similarly in virtio net, recently I traced such counters the current kernel does not have and it turned out that one of the output queues in the backend behaves badly.
...

Stop/wake queue counters may not show directly the root cause of the issue, but help us 'guess' to some extent.

Quoted 6 lines
Quoted 2 lines

I'm surprised you say you can detect stall-related issues with this. I guess virtio doesn't have BQL support, which makes it special.

Yes, virtio-net has a legacy orphan mode, this is something that needs to be dropped in the future. This would make BQL much more easier to be implemented.

It's not that we can't implement BQL, it's that it does not seem to be benefitial - has been discussed many times.

Quoted 13 lines
Quoted 11 lines

Normal HW drivers with BQL almost never stop the queue by themselves. I mean - if they do, and BQL is active, then the system is probably misconfigured (queue is too short). This is what we use at Meta to detect stalls in drivers with BQL:

https://lore.kernel.org/all/[email protected]/

Daniel, I think this may be a good enough excuse to add per-queue stats to the netdev genl family, if you're up for that. LMK if you want more info, otherwise I guess ethtool -S is fine for now.

Thanks

#20 lore raw
Quoted 28 lines · Michael S. Tsirkin
Quoted 26 lines · Jason Wang
Quoted 24 lines · Jakub Kicinski
Quoted 5 lines · On Fri, 2 Feb 2024 14:52:59 +0800 Jason Xing wrote:
Quoted 1 line

Can you say more? I'm curious what's your use case.

I'm not working at Nvidia, so my point of view may differ from theirs. From what I can tell is that those two counters help me narrow down the range if I have to diagnose/debug some issues.

right, i'm asking to collect useful debugging tricks, nothing against the patch itself :)

Quoted 10 lines

1) I sometimes notice that if some irq is held too long (say, one simple case: output of printk printed to the console), those two counters can reflect the issue.
2) Similarly in virtio net, recently I traced such counters the current kernel does not have and it turned out that one of the output queues in the backend behaves badly.
...

Stop/wake queue counters may not show directly the root cause of the issue, but help us 'guess' to some extent.

I'm surprised you say you can detect stall-related issues with this. I guess virtio doesn't have BQL support, which makes it special.

Quoted 6 lines
Quoted 3 lines

Yes, virtio-net has a legacy orphan mode, this is something that needs to be dropped in the future. This would make BQL much more easier to be implemented.

It's not that we can't implement BQL,

Well, I don't say we can't, I say it's not easy as we need to deal with the switching between two modes[1]. If we just have one mode like TX interrupt, we don't need to care about that.

Quoted 2 lines

it's that it does not seem to
be benefitial - has been discussed many times.

Virtio doesn't differ from other NIC too much, for example gve supports bql.

1) There's no numbers in [1]
2) We only benchmark vhost-net but not others, for example, vhost-user and hardware implementations
3) We don't have interrupt coalescing in 2018 but now we have with DIM

Thanks

[1] https://lore.kernel.org/netdev/[email protected]/

Quoted 15 lines
Quoted 14 lines
Quoted 6 lines

Normal HW drivers with BQL almost never stop the queue by themselves. I mean - if they do, and BQL is active, then the system is probably misconfigured (queue is too short). This is what we use at Meta to detect stalls in drivers with BQL:

Quoted 6 lines

https://lore.kernel.org/all/[email protected]/

Daniel, I think this may be a good enough excuse to add per-queue stats to the netdev genl family, if you're up for that. LMK if you want more info, otherwise I guess ethtool -S is fine for now.

Thanks

#22 lore raw
In reply to Michael S. Tsirkin
Quoted 38 lines

From: Michael S. Tsirkin <[email protected]>
Sent: Sunday, February 4, 2024 6:40 AM
To: Jason Wang <[email protected]>
Cc: Jakub Kicinski <[email protected]>; Jason Xing <[email protected]>; Daniel Jurgens <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; Parav Pandit <[email protected]>
Subject: Re: [PATCH net-next] virtio_net: Add TX stop and wake counters

Quoted 26 lines · Jason Wang
Quoted 24 lines · Jakub Kicinski
Quoted 5 lines · On Fri, 2 Feb 2024 14:52:59 +0800 Jason Xing wrote:
Quoted 1 line

Can you say more? I'm curious what's your use case.

I'm not working at Nvidia, so my point of view may differ from theirs. From what I can tell is that those two counters help me narrow down the range if I have to diagnose/debug some issues.

right, i'm asking to collect useful debugging tricks, nothing against the patch itself :)

Quoted 10 lines

1) I sometimes notice that if some irq is held too long (say, one simple case: output of printk printed to the console), those two counters can reflect the issue.
2) Similarly in virtio net, recently I traced such counters the current kernel does not have and it turned out that one of the output queues in the backend behaves badly.
...

Stop/wake queue counters may not show directly the root cause of the issue, but help us 'guess' to some extent.

I'm surprised you say you can detect stall-related issues with this. I guess virtio doesn't have BQL support, which makes it special.

Quoted 22 lines
Quoted 3 lines

Yes, virtio-net has a legacy orphan mode, this is something that needs to be dropped in the future. This would make BQL much more easier to be implemented.

It's not that we can't implement BQL, it's that it does not seem to be benefitial - has been discussed many times.

Quoted 14 lines
Quoted 5 lines

Normal HW drivers with BQL almost never stop the queue by themselves. I mean - if they do, and BQL is active, then the system is probably misconfigured (queue is too short). This is what we use at Meta to detect stalls in drivers with BQL:

Quoted 7 lines

https://lore.kernel.org/all/[email protected]
g/

Daniel, I think this may be a good enough excuse to add per-queue stats to the netdev genl family, if you're up for that. LMK if you want more info, otherwise I guess ethtool -S is fine for now.

Thanks

Michael,
Are you OK with this patch? Unless I missed it I didn't see a response from you in our conversation the day I sent it.

#23 lore raw
Quoted 59 lines · Daniel Jurgens
Quoted 59 lines

From: Michael S. Tsirkin <[email protected]>
Sent: Sunday, February 4, 2024 6:40 AM
To: Jason Wang <[email protected]>
Cc: Jakub Kicinski <[email protected]>; Jason Xing <[email protected]>; Daniel Jurgens <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; Parav Pandit <[email protected]>
Subject: Re: [PATCH net-next] virtio_net: Add TX stop and wake counters

Quoted 23 lines · Jason Wang
Quoted 22 lines · Jakub Kicinski
Quoted 5 lines · On Fri, 2 Feb 2024 14:52:59 +0800 Jason Xing wrote:
Quoted 1 line

Can you say more? I'm curious what's your use case.

I'm not working at Nvidia, so my point of view may differ from theirs. From what I can tell is that those two counters help me narrow down the range if I have to diagnose/debug some issues.

right, i'm asking to collect useful debugging tricks, nothing against the patch itself :)

Quoted 10 lines

1) I sometimes notice that if some irq is held too long (say, one simple case: output of printk printed to the console), those two counters can reflect the issue.
2) Similarly in virtio net, recently I traced such counters the current kernel does not have and it turned out that one of the output queues in the backend behaves badly.
...

Stop/wake queue counters may not show directly the root cause of the issue, but help us 'guess' to some extent.

Quoted 6 lines
Quoted 2 lines

I'm surprised you say you can detect stall-related issues with this. I guess virtio doesn't have BQL support, which makes it special.

Yes, virtio-net has a legacy orphan mode, this is something that needs to be dropped in the future. This would make BQL much more easier to be implemented.

It's not that we can't implement BQL, it's that it does not seem to be benefitial - has been discussed many times.

Quoted 13 lines
Quoted 12 lines

Normal HW drivers with BQL almost never stop the queue by themselves. I mean - if they do, and BQL is active, then the system is probably misconfigured (queue is too short). This is what we use at Meta to detect stalls in drivers with BQL:

https://lore.kernel.org/all/[email protected]
g/

Daniel, I think this may be a good enough excuse to add per-queue stats to the netdev genl family, if you're up for that. LMK if you want more info, otherwise I guess ethtool -S is fine for now.

Quoted 5 lines
Quoted 1 line
Quoted 1 line

Thanks

Michael,
Are you OK with this patch? Unless I missed it I didn't see a response from you in our conversation the day I sent it.

I thought what is proposed is adding some support for these stats to core? Did I misunderstood?

Signature · 1 line
MST
#21 lore raw
In reply to Jason Wang
Quoted 43 lines · Jason Wang
Quoted 28 lines · Michael S. Tsirkin
Quoted 26 lines · Jason Wang
Quoted 24 lines · Jakub Kicinski
Quoted 5 lines · On Fri, 2 Feb 2024 14:52:59 +0800 Jason Xing wrote:
Quoted 1 line

Can you say more? I'm curious what's your use case.

I'm not working at Nvidia, so my point of view may differ from theirs. From what I can tell is that those two counters help me narrow down the range if I have to diagnose/debug some issues.

right, i'm asking to collect useful debugging tricks, nothing against the patch itself :)

Quoted 10 lines

1) I sometimes notice that if some irq is held too long (say, one simple case: output of printk printed to the console), those two counters can reflect the issue.
2) Similarly in virtio net, recently I traced such counters the current kernel does not have and it turned out that one of the output queues in the backend behaves badly.
...

Stop/wake queue counters may not show directly the root cause of the issue, but help us 'guess' to some extent.

I'm surprised you say you can detect stall-related issues with this. I guess virtio doesn't have BQL support, which makes it special.

Quoted 6 lines
Quoted 3 lines

Yes, virtio-net has a legacy orphan mode, this is something that needs to be dropped in the future. This would make BQL much more easier to be implemented.

It's not that we can't implement BQL,

Well, I don't say we can't, I say it's not easy as we need to deal with the switching between two modes[1]. If we just have one mode like TX interrupt, we don't need to care about that.

Quoted 2 lines

it's that it does not seem to
be benefitial - has been discussed many times.

Quoted 6 lines

Virtio doesn't differ from other NIC too much, for example gve supports bql.

1) There's no numbers in [1]
2) We only benchmark vhost-net but not others, for example, vhost-user and hardware implementations
3) We don't have interrupt coalescing in 2018 but now we have with DIM

Only works well with hardware virtio cards though.

Quoted 4 lines

Thanks

[1] https://lore.kernel.org/netdev/[email protected]/

So fundamentally, someone needs to show benefit and no serious regressions to add BQL at this point. I doubt it's easily practical. Hacks like wireless does to boost buffer sizes might be necessary.

Quoted 15 lines
Quoted 15 lines
Quoted 14 lines
Quoted 6 lines

Normal HW drivers with BQL almost never stop the queue by themselves. I mean - if they do, and BQL is active, then the system is probably misconfigured (queue is too short). This is what we use at Meta to detect stalls in drivers with BQL:

Quoted 6 lines

https://lore.kernel.org/all/[email protected]/

Daniel, I think this may be a good enough excuse to add per-queue stats to the netdev genl family, if you're up for that. LMK if you want more info, otherwise I guess ethtool -S is fine for now.

Thanks

#24 lore raw
In reply to Michael S. Tsirkin
Quoted 74 lines

From: Michael S. Tsirkin <[email protected]>
Sent: Wednesday, February 7, 2024 2:19 PM
To: Daniel Jurgens <[email protected]>
Subject: Re: [PATCH net-next] virtio_net: Add TX stop and wake counters

Quoted 14 lines · Daniel Jurgens
Quoted 14 lines

From: Michael S. Tsirkin <[email protected]>
Sent: Sunday, February 4, 2024 6:40 AM
To: Jason Wang <[email protected]>
Cc: Jakub Kicinski <[email protected]>; Jason Xing <[email protected]>; Daniel Jurgens <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; Parav Pandit <[email protected]>
Subject: Re: [PATCH net-next] virtio_net: Add TX stop and wake counters

Quoted 1 line · Jason Wang

On Sat, Feb 3, 2024 at 12:01 AM Jakub Kicinski <[email protected]>

Quoted 5 lines · wrote:
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines · On Fri, 2 Feb 2024 14:52:59 +0800 Jason Xing wrote:
Quoted 3 lines
Quoted 1 line

Can you say more? I'm curious what's your use case.

I'm not working at Nvidia, so my point of view may differ from

theirs.

Quoted 25 lines
Quoted 25 lines
Quoted 23 lines
Quoted 14 lines
Quoted 2 lines

From what I can tell is that those two counters help me narrow down the range if I have to diagnose/debug some issues.

right, i'm asking to collect useful debugging tricks, nothing against the patch itself :)

Quoted 8 lines

1) I sometimes notice that if some irq is held too long (say, one simple case: output of printk printed to the console), those two counters can reflect the issue.
2) Similarly in virtio net, recently I traced such counters the current kernel does not have and it turned out that one of the output queues in the backend behaves badly.
...

Quoted 5 lines
Quoted 2 lines

Stop/wake queue counters may not show directly the root cause of the issue, but help us 'guess' to some extent.

I'm surprised you say you can detect stall-related issues with this. I guess virtio doesn't have BQL support, which makes it special.

Yes, virtio-net has a legacy orphan mode, this is something that needs to be dropped in the future. This would make BQL much more easier to be implemented.

Quoted 4 lines
Quoted 4 lines

It's not that we can't implement BQL, it's that it does not seem to be benefitial - has been discussed many times.

Quoted 1 line
Quoted 1 line

Normal HW drivers with BQL almost never stop the queue by

themselves.

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines

I mean - if they do, and BQL is active, then the system is probably misconfigured (queue is too short). This is what we use at Meta to detect stalls in drivers with BQL:

https://lore.kernel.org/all/20240131102150.728960-3-leitao@debia
n.or
g/

Daniel, I think this may be a good enough excuse to add per-queue stats to the netdev genl family, if you're up for that. LMK if you want more info, otherwise I guess ethtool -S is fine

for now.

Quoted 4 lines
Quoted 3 lines
Quoted 3 lines
Quoted 1 line

Thanks

Quoted 8 lines
Quoted 2 lines

Michael,
Are you OK with this patch? Unless I missed it I didn't see a response

from you in our conversation the day I sent it.

Quoted 1 line

I thought what is proposed is adding some support for these stats to core? Did I misunderstood?

That's a much bigger change and going that route I think still need to count them at the driver level. I said I could potentially take that on as a background project. But would prefer to go with ethtool -S for now.

Quoted 2 lines

--
MST

#25 lore raw
Quoted 95 lines

From: Daniel Jurgens <[email protected]>
Sent: Wednesday, February 7, 2024 2:59 PM
To: Michael S. Tsirkin <[email protected]>
Cc: Jason Wang <[email protected]>; Jakub Kicinski <[email protected]>; Jason Xing <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; Parav Pandit <[email protected]>
Subject: RE: [PATCH net-next] virtio_net: Add TX stop and wake counters

Quoted 83 lines

From: Michael S. Tsirkin <[email protected]>
Sent: Wednesday, February 7, 2024 2:19 PM
To: Daniel Jurgens <[email protected]>
Subject: Re: [PATCH net-next] virtio_net: Add TX stop and wake counters

Quoted 14 lines · Daniel Jurgens
Quoted 14 lines

From: Michael S. Tsirkin <[email protected]>
Sent: Sunday, February 4, 2024 6:40 AM
To: Jason Wang <[email protected]>
Cc: Jakub Kicinski <[email protected]>; Jason Xing <[email protected]>; Daniel Jurgens <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; Parav Pandit <[email protected]>
Subject: Re: [PATCH net-next] virtio_net: Add TX stop and wake counters

Quoted 1 line · Jason Wang

On Sat, Feb 3, 2024 at 12:01 AM Jakub Kicinski <[email protected]>

Quoted 6 lines · wrote:
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines · On Fri, 2 Feb 2024 14:52:59 +0800 Jason Xing wrote:
Quoted 4 lines
Quoted 1 line

Can you say more? I'm curious what's your use case.

I'm not working at Nvidia, so my point of view may differ from

theirs.

Quoted 25 lines
Quoted 25 lines
Quoted 23 lines
Quoted 14 lines
Quoted 2 lines

From what I can tell is that those two counters help me narrow down the range if I have to diagnose/debug some issues.

right, i'm asking to collect useful debugging tricks, nothing against the patch itself :)

Quoted 8 lines

1) I sometimes notice that if some irq is held too long (say, one simple case: output of printk printed to the console), those two counters can reflect the issue.
2) Similarly in virtio net, recently I traced such counters the current kernel does not have and it turned out that one of the output queues in the backend behaves badly.
...

Quoted 5 lines
Quoted 2 lines

Stop/wake queue counters may not show directly the root cause of the issue, but help us 'guess' to some extent.

I'm surprised you say you can detect stall-related issues with this. I guess virtio doesn't have BQL support, which makes it special.

Yes, virtio-net has a legacy orphan mode, this is something that needs to be dropped in the future. This would make BQL much more easier to be implemented.

Quoted 4 lines
Quoted 4 lines

It's not that we can't implement BQL, it's that it does not seem to be benefitial - has been discussed many times.

Quoted 1 line
Quoted 1 line

Normal HW drivers with BQL almost never stop the queue by

themselves.

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines

I mean - if they do, and BQL is active, then the system is probably misconfigured (queue is too short). This is what we use at Meta to detect stalls in drivers with BQL:

https://lore.kernel.org/all/20240131102150.728960-3-leitao@deb
ia
n.or
g/

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

Daniel, I think this may be a good enough excuse to add per-queue stats to the netdev genl family, if you're up for that. LMK if you want more info, otherwise I guess ethtool -S is fine

for now.

Quoted 7 lines
Quoted 3 lines
Quoted 3 lines
Quoted 1 line

Thanks

Michael,
Are you OK with this patch? Unless I missed it I didn't see a
response

from you in our conversation the day I sent it.

Quoted 1 line
Quoted 7 lines
Quoted 3 lines

I thought what is proposed is adding some support for these stats to core? Did I misunderstood?

That's a much bigger change and going that route I think still need to count them at the driver level. I said I could potentially take that on as a background project. But would prefer to go with ethtool -S for now.

Michael, are you a NACK on this? Jakub seemed OK with it, Jason also thinks it's useful, and it's low risk.

Quoted 3 lines
Quoted 3 lines

--
MST

#26 lore raw
Quoted 99 lines · Dan Jurgens
Quoted 99 lines

From: Daniel Jurgens <[email protected]>
Sent: Wednesday, February 7, 2024 2:59 PM
To: Michael S. Tsirkin <[email protected]>
Cc: Jason Wang <[email protected]>; Jakub Kicinski <[email protected]>; Jason Xing <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; Parav Pandit <[email protected]>
Subject: RE: [PATCH net-next] virtio_net: Add TX stop and wake counters

Quoted 86 lines

From: Michael S. Tsirkin <[email protected]>
Sent: Wednesday, February 7, 2024 2:19 PM
To: Daniel Jurgens <[email protected]>
Subject: Re: [PATCH net-next] virtio_net: Add TX stop and wake counters

Quoted 14 lines · Daniel Jurgens
Quoted 14 lines

From: Michael S. Tsirkin <[email protected]>
Sent: Sunday, February 4, 2024 6:40 AM
To: Jason Wang <[email protected]>
Cc: Jakub Kicinski <[email protected]>; Jason Xing <[email protected]>; Daniel Jurgens <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; Parav Pandit <[email protected]>
Subject: Re: [PATCH net-next] virtio_net: Add TX stop and wake counters

Quoted 1 line · Jason Wang

On Sat, Feb 3, 2024 at 12:01 AM Jakub Kicinski <[email protected]>

Quoted 6 lines · wrote:
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines · On Fri, 2 Feb 2024 14:52:59 +0800 Jason Xing wrote:
Quoted 4 lines
Quoted 1 line

Can you say more? I'm curious what's your use case.

I'm not working at Nvidia, so my point of view may differ from

theirs.

Quoted 25 lines
Quoted 25 lines
Quoted 23 lines
Quoted 14 lines
Quoted 2 lines

From what I can tell is that those two counters help me narrow down the range if I have to diagnose/debug some issues.

right, i'm asking to collect useful debugging tricks, nothing against the patch itself :)

Quoted 8 lines

1) I sometimes notice that if some irq is held too long (say, one simple case: output of printk printed to the console), those two counters can reflect the issue.
2) Similarly in virtio net, recently I traced such counters the current kernel does not have and it turned out that one of the output queues in the backend behaves badly.
...

Quoted 5 lines
Quoted 2 lines

Stop/wake queue counters may not show directly the root cause of the issue, but help us 'guess' to some extent.

I'm surprised you say you can detect stall-related issues with this. I guess virtio doesn't have BQL support, which makes it special.

Yes, virtio-net has a legacy orphan mode, this is something that needs to be dropped in the future. This would make BQL much more easier to be implemented.

Quoted 4 lines
Quoted 4 lines

It's not that we can't implement BQL, it's that it does not seem to be benefitial - has been discussed many times.

Quoted 1 line
Quoted 1 line

Normal HW drivers with BQL almost never stop the queue by

themselves.

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines

I mean - if they do, and BQL is active, then the system is probably misconfigured (queue is too short). This is what we use at Meta to detect stalls in drivers with BQL:

https://lore.kernel.org/all/20240131102150.728960-3-leitao@deb
ia
n.or
g/

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

Daniel, I think this may be a good enough excuse to add per-queue stats to the netdev genl family, if you're up for that. LMK if you want more info, otherwise I guess ethtool -S is fine

for now.

Quoted 7 lines
Quoted 3 lines
Quoted 3 lines
Quoted 1 line

Thanks

Michael,
Are you OK with this patch? Unless I missed it I didn't see a
response

from you in our conversation the day I sent it.

Quoted 1 line

I thought what is proposed is adding some support for these stats to core? Did I misunderstood?

Quoted 5 lines
Quoted 3 lines

That's a much bigger change and going that route I think still need to count them at the driver level. I said I could potentially take that on as a background project. But would prefer to go with ethtool -S for now.

Michael, are you a NACK on this? Jakub seemed OK with it, Jason also thinks it's useful, and it's low risk.

Not too bad ... Jakub can you confirm though?

Quoted 4 lines
Quoted 3 lines
Quoted 3 lines

--
MST

maintainer
#27 lore raw
Quoted 4 lines · On Tue, 20 Feb 2024 13:05:46 -0500 Michael S. Tsirkin wrote:
Quoted 2 lines

Michael, are you a NACK on this? Jakub seemed OK with it, Jason also thinks it's useful, and it's low risk.

Not too bad ... Jakub can you confirm though?

Ughhm. Let me implement the generic API myself. I should have a few hours tomorrow, knock wood. I guess the initial set of stats requires some "know what you're doing", and it will be smooth sailing from there..

#13 lore raw
In reply to Daniel Jurgens
Quoted 33 lines · Daniel Jurgens
Quoted 33 lines

From: Jakub Kicinski <[email protected]>
Sent: Friday, February 2, 2024 10:01 AM
Subject: Re: [PATCH net-next] virtio_net: Add TX stop and wake counters

Quoted 5 lines · On Fri, 2 Feb 2024 14:52:59 +0800 Jason Xing wrote:
Quoted 1 line

Can you say more? I'm curious what's your use case.

I'm not working at Nvidia, so my point of view may differ from theirs. From what I can tell is that those two counters help me narrow down the range if I have to diagnose/debug some issues.

right, i'm asking to collect useful debugging tricks, nothing against the patch itself :)

Quoted 10 lines

1) I sometimes notice that if some irq is held too long (say, one simple case: output of printk printed to the console), those two counters can reflect the issue.
2) Similarly in virtio net, recently I traced such counters the current kernel does not have and it turned out that one of the output queues in the backend behaves badly.
...

Stop/wake queue counters may not show directly the root cause of the issue, but help us 'guess' to some extent.

I'm surprised you say you can detect stall-related issues with this. I guess virtio doesn't have BQL support, which makes it special. Normal HW drivers with BQL almost never stop the queue by themselves. I mean - if they do, and BQL is active, then the system is probably misconfigured (queue is too short). This is what we use at Meta to detect stalls in drivers with BQL:

Quoted 7 lines
Quoted 5 lines

https://lore.kernel.org/all/[email protected]/

Daniel, I think this may be a good enough excuse to add per-queue stats to the netdev genl family, if you're up for that. LMK if you want more info, otherwise I guess ethtool -S is fine for now.

For now, I would like to go with ethtool -S. But I can take on the netdev level as a background task. Are you suggesting adding them to rtnl_link_stats64?

Hello Daniel, Jakub,

Sorry to revive this thread. I wonder why not use this patch like mlnx driver does instead of adding statistics into the yaml file? Are we gradually using or adding more fields into the yaml file to replace the 'ethtool -S' command?

Thanks,
Jason

#14 lore raw
Quoted 53 lines

From: Jason Xing <[email protected]>
Sent: Friday, June 7, 2024 4:16 AM
To: Dan Jurgens <[email protected]>
Cc: Jakub Kicinski <[email protected]>; Michael S. Tsirkin <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; Parav Pandit <[email protected]>
Subject: Re: [PATCH net-next] virtio_net: Add TX stop and wake counters

Quoted 34 lines · Daniel Jurgens
Quoted 34 lines

From: Jakub Kicinski <[email protected]>
Sent: Friday, February 2, 2024 10:01 AM
Subject: Re: [PATCH net-next] virtio_net: Add TX stop and wake counters

Quoted 5 lines · On Fri, 2 Feb 2024 14:52:59 +0800 Jason Xing wrote:
Quoted 1 line

Can you say more? I'm curious what's your use case.

I'm not working at Nvidia, so my point of view may differ from theirs. From what I can tell is that those two counters help me narrow down the range if I have to diagnose/debug some issues.

right, i'm asking to collect useful debugging tricks, nothing against the patch itself :)

Quoted 10 lines

1) I sometimes notice that if some irq is held too long (say, one simple case: output of printk printed to the console), those two counters can reflect the issue.
2) Similarly in virtio net, recently I traced such counters the current kernel does not have and it turned out that one of the output queues in the backend behaves badly.
...

Stop/wake queue counters may not show directly the root cause of the issue, but help us 'guess' to some extent.

I'm surprised you say you can detect stall-related issues with this. I guess virtio doesn't have BQL support, which makes it special. Normal HW drivers with BQL almost never stop the queue by themselves. I mean - if they do, and BQL is active, then the system is probably misconfigured (queue is too short). This is what we use at Meta to detect stalls in drivers with BQL:

Quoted 8 lines
Quoted 6 lines

https://lore.kernel.org/all/[email protected]
g/

Daniel, I think this may be a good enough excuse to add per-queue stats to the netdev genl family, if you're up for that. LMK if you want more info, otherwise I guess ethtool -S is fine for now.

For now, I would like to go with ethtool -S. But I can take on the netdev

Quoted 9 lines

level as a background task. Are you suggesting adding them to rtnl_link_stats64?

Hello Daniel, Jakub,

Sorry to revive this thread. I wonder why not use this patch like mlnx driver does instead of adding statistics into the yaml file? Are we gradually using or adding more fields into the yaml file to replace the 'ethtool -S' command?

It's trivial to have the stats in ethtool as well. But I noticed the stats series intentionally removed some stats from ethtool. So I didn't put it both places.

Quoted 2 lines

Thanks,
Jason

#15 lore raw
Quoted 57 lines · Dan Jurgens
Quoted 57 lines

From: Jason Xing <[email protected]>
Sent: Friday, June 7, 2024 4:16 AM
To: Dan Jurgens <[email protected]>
Cc: Jakub Kicinski <[email protected]>; Michael S. Tsirkin <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; Parav Pandit <[email protected]>
Subject: Re: [PATCH net-next] virtio_net: Add TX stop and wake counters

Quoted 34 lines · Daniel Jurgens
Quoted 34 lines

From: Jakub Kicinski <[email protected]>
Sent: Friday, February 2, 2024 10:01 AM
Subject: Re: [PATCH net-next] virtio_net: Add TX stop and wake counters

Quoted 5 lines · On Fri, 2 Feb 2024 14:52:59 +0800 Jason Xing wrote:
Quoted 1 line

Can you say more? I'm curious what's your use case.

I'm not working at Nvidia, so my point of view may differ from theirs. From what I can tell is that those two counters help me narrow down the range if I have to diagnose/debug some issues.

right, i'm asking to collect useful debugging tricks, nothing against the patch itself :)

Quoted 10 lines

1) I sometimes notice that if some irq is held too long (say, one simple case: output of printk printed to the console), those two counters can reflect the issue.
2) Similarly in virtio net, recently I traced such counters the current kernel does not have and it turned out that one of the output queues in the backend behaves badly.
...

Stop/wake queue counters may not show directly the root cause of the issue, but help us 'guess' to some extent.

I'm surprised you say you can detect stall-related issues with this. I guess virtio doesn't have BQL support, which makes it special. Normal HW drivers with BQL almost never stop the queue by themselves. I mean - if they do, and BQL is active, then the system is probably misconfigured (queue is too short). This is what we use at Meta to detect stalls in drivers with BQL:

Quoted 8 lines
Quoted 6 lines

https://lore.kernel.org/all/[email protected]
g/

Daniel, I think this may be a good enough excuse to add per-queue stats to the netdev genl family, if you're up for that. LMK if you want more info, otherwise I guess ethtool -S is fine for now.

For now, I would like to go with ethtool -S. But I can take on the netdev

level as a background task. Are you suggesting adding them to rtnl_link_stats64?

Quoted 8 lines
Quoted 6 lines

Hello Daniel, Jakub,

Sorry to revive this thread. I wonder why not use this patch like mlnx driver does instead of adding statistics into the yaml file? Are we gradually using or adding more fields into the yaml file to replace the 'ethtool -S' command?

It's trivial to have the stats in ethtool as well. But I noticed the stats series intentionally removed some stats from ethtool. So I didn't put it both places.

Thank you for the reply. I thought there was some particular reason :-)

maintainer
#16 lore raw
Quoted 11 lines · On Sat, 8 Jun 2024 08:41:44 +0800 Jason Xing wrote:
Quoted 8 lines
Quoted 4 lines

Sorry to revive this thread. I wonder why not use this patch like mlnx driver does instead of adding statistics into the yaml file? Are we gradually using or adding more fields into the yaml file to replace the 'ethtool -S' command?

It's trivial to have the stats in ethtool as well. But I noticed the stats series intentionally removed some stats from ethtool. So I didn't put it both places.

Thank you for the reply. I thought there was some particular reason :-)

Yes, we don't want duplication. We have a long standing (and documented) policy against duplicating normal stats in custom stat APIs, otherwise vendors pile everything into the custom stats.

#17 lore raw
Quoted 11 lines · Jakub Kicinski
Quoted 9 lines · On Sat, 8 Jun 2024 08:41:44 +0800 Jason Xing wrote:
Quoted 8 lines
Quoted 4 lines

Sorry to revive this thread. I wonder why not use this patch like mlnx driver does instead of adding statistics into the yaml file? Are we gradually using or adding more fields into the yaml file to replace the 'ethtool -S' command?

It's trivial to have the stats in ethtool as well. But I noticed the stats series intentionally removed some stats from ethtool. So I didn't put it both places.

Quoted 6 lines
Quoted 2 lines

Thank you for the reply. I thought there was some particular reason :-)

Yes, we don't want duplication. We have a long standing (and documented) policy against duplicating normal stats in custom stat APIs, otherwise vendors pile everything into the custom stats.

Thanks, Jakub. I see :)