Skip to contentSkip to conversation

forcedeth: minor fixes for stats, rmmod, sparse

PATCH Superseded David Decotigny 3 participants 14 messages Nov 4, 2011 – Nov 5, 2011

Subsystems (3)

v2 of 5 Superseded Latest is v5 →

lore
Share & tools
lore
#0 lore raw

v2 cover forcedeth: minor fixes for stats, rmmod, sparse

Here is the v2 of a forcedeth patch series I sent a few months ago (... May).

Changes since v1:
- rebased on top of netdev tip
- do not repeat name of device in netdev_dbg
- do not completely mute TX timeout messages when debug_tx_timeout is not set
- make debug_tx_timeout writable in /sys/module

I am re-submitting "expose module parameters in /sys/module" as it can be useful in production and I was assured it doesn't add much memory overhead by the sysfs maintainers.

Tested:
16-way x86_64 SMP, dual forcedeth

############################################
# Patch Set Summary:

David Decotigny (2):
forcedeth: expose module parameters in /sys/module
forcedeth: fix a few sparse warnings (variable shadowing)

Mandeep Baines (1):
forcedeth: Improve stats counters

Mike Ditto (2):
forcedeth: Acknowledge only interrupts that are being processed
forcedeth: Add messages to indicate using MSI or MSI-X

Salman Qazi (1):
forcedeth: Fix a race during rmmod of forcedeth

Sameer Nanda (2):
forcedeth: new ethtool stat "tx_timeout" to account for tx_timeouts
forcedeth: allow to silence tx_timeout debug messages

 drivers/net/ethernet/nvidia/forcedeth.c |  182 +++++++++++++++++++------------
 1 files changed, 110 insertions(+), 72 deletions(-)
Signature · 1 line
1.7.3.1
#1 lore raw

PATCH v2 1/8 forcedeth: Improve stats counters

From: Mandeep Baines <[email protected]>

Rx byte count was off; instead use the hardware's count. Tx packet count was counting pre-TSO packets; instead count on-the-wire packets. Report hardware dropped frame count as rx_fifo_errors.

- The count of transmitted packets reported by the forcedeth driver reports pre-TSO (TCP Segmentation Offload) packet counts and not the count of the number of packets sent on the wire. This change fixes the forcedeth driver to report the correct count. Fixed the code by copying the count stored in the NIC H/W to the value reported by the driver.

- Count rx_drop_frame errors as rx_fifo_errors:
We see a lot of rx_drop_frame errors if we disable the rx bottom-halves for too long. Normally, rx_fifo_errors would be counted in this case. The rx_drop_frame error count is private to forcedeth and is not reported by ifconfig or sysfs. The rx_fifo_errors count is currently unused in the forcedeth driver. It is reported by ifconfig as overruns. This change reports rx_drop_frame errors as rx_fifo_errors.

---
 drivers/net/ethernet/nvidia/forcedeth.c |    4 ++++
 1 files changed, 4 insertions(+), 0 deletions(-)
drivers/net/ethernet/nvidia/forcedeth.c +4 -0
index 1e37eb9..64b1c7c 100644
@@ -1682,6 +1682,7 @@ static void nv_get_hw_stats(struct net_device *dev)
 		np->estats.tx_pause += readl(base + NvRegTxPause);
 		np->estats.rx_pause += readl(base + NvRegRxPause);
 		np->estats.rx_drop_frame += readl(base + NvRegRxDropFrame);
+		np->estats.rx_errors_total += np->estats.rx_drop_frame;
 	}
 
 	if (np->driver_data & DEV_HAS_STATISTICS_V3) {
@@ -1706,11 +1707,14 @@ static struct net_device_stats *nv_get_stats(struct net_device *dev)
 		nv_get_hw_stats(dev);
 
 		/* copy to net_device stats */
+		dev->stats.tx_packets = np->estats.tx_packets;
+		dev->stats.rx_bytes = np->estats.rx_bytes;
 		dev->stats.tx_bytes = np->estats.tx_bytes;
 		dev->stats.tx_fifo_errors = np->estats.tx_fifo_errors;
 		dev->stats.tx_carrier_errors = np->estats.tx_carrier_errors;
 		dev->stats.rx_crc_errors = np->estats.rx_crc_errors;
 		dev->stats.rx_over_errors = np->estats.rx_over_errors;
+		dev->stats.rx_fifo_errors = np->estats.rx_drop_frame;
 		dev->stats.rx_errors = np->estats.rx_errors_total;
 		dev->stats.tx_errors = np->estats.tx_errors_total;
 	}
Signature · 1 line
1.7.3.1
#2 lore raw

Re: [PATCH net v2 1/8] forcedeth: Improve stats counters

Quoted 45 lines · On Thu, 3 Nov 2011 18:41:16 -0700 David Decotigny…

From: Mandeep Baines <[email protected]>

Rx byte count was off; instead use the hardware's count. Tx packet count was counting pre-TSO packets; instead count on-the-wire packets. Report hardware dropped frame count as rx_fifo_errors.

- The count of transmitted packets reported by the forcedeth driver reports pre-TSO (TCP Segmentation Offload) packet counts and not the count of the number of packets sent on the wire. This change fixes the forcedeth driver to report the correct count. Fixed the code by copying the count stored in the NIC H/W to the value reported by the driver.

- Count rx_drop_frame errors as rx_fifo_errors:
We see a lot of rx_drop_frame errors if we disable the rx bottom-halves for too long. Normally, rx_fifo_errors would be counted in this case. The rx_drop_frame error count is private to forcedeth and is not reported by ifconfig or sysfs. The rx_fifo_errors count is currently unused in the forcedeth driver. It is reported by ifconfig as overruns. This change reports rx_drop_frame errors as rx_fifo_errors.

---
 drivers/net/ethernet/nvidia/forcedeth.c |    4 ++++
 1 files changed, 4 insertions(+), 0 deletions(-)
drivers/net/ethernet/nvidia/forcedeth.c +2 -0
index 1e37eb9..64b1c7c 100644
@@ -1682,6 +1682,7 @@ static void nv_get_hw_stats(struct net_device *dev)
 		np->estats.tx_pause += readl(base + NvRegTxPause);
 		np->estats.rx_pause += readl(base + NvRegRxPause);
 		np->estats.rx_drop_frame += readl(base + NvRegRxDropFrame);
+		np->estats.rx_errors_total += np->estats.rx_drop_frame;
 	}
 
 	if (np->driver_data & DEV_HAS_STATISTICS_V3) {
@@ -1706,11 +1707,14 @@ static struct net_device_stats *nv_get_stats(struct net_device *dev)
 		nv_get_hw_stats(dev);
 
 		/* copy to net_device stats */
+		dev->stats.tx_packets = np->estats.tx_packets;

drivers/net/ethernet/nvidia/forcedeth.c static struct net_device_stats *nv_get_stats(struct net_device *dev) · line 1720

Quoted 10 lines
drivers/net/ethernet/nvidia/forcedeth.c +2 -0
+		dev->stats.rx_bytes = np->estats.rx_bytes;
 		dev->stats.tx_bytes = np->estats.tx_bytes;
 		dev->stats.tx_fifo_errors = np->estats.tx_fifo_errors;
 		dev->stats.tx_carrier_errors = np->estats.tx_carrier_errors;
 		dev->stats.rx_crc_errors = np->estats.rx_crc_errors;
 		dev->stats.rx_over_errors = np->estats.rx_over_errors;
+		dev->stats.rx_fifo_errors = np->estats.rx_drop_frame;
 		dev->stats.rx_errors = np->estats.rx_errors_total;
 		dev->stats.tx_errors = np->estats.tx_errors_total;
 	}

Why not convert it to 64 bit stats as well.

#3 lore raw

Re: [PATCH net v2 1/8] forcedeth: Improve stats counters

Hi all,

Quoted 1 line · Stephen Hemminger

Why not convert it to 64 bit stats as well.

Doing it for next version of patch series. So please hang on...

Regards,

#4 lore raw

PATCH v2 2/8 forcedeth: new ethtool stat "tx_timeout" to account for tx_timeouts

In reply to David Decotigny

From: Sameer Nanda <[email protected]>

This change publishes a new ethtool stats: tx_timeout that counts the number of times the tx_timeout callback was triggered.

---
 drivers/net/ethernet/nvidia/forcedeth.c |    4 ++++
 1 files changed, 4 insertions(+), 0 deletions(-)
drivers/net/ethernet/nvidia/forcedeth.c +4 -0
index 64b1c7c..c82a3c7 100644
@@ -633,6 +633,7 @@ static const struct nv_ethtool_str nv_estats_str[] = {
 	{ "rx_packets" },
 	{ "rx_errors_total" },
 	{ "tx_errors_total" },
+	{ "tx_timeout" },
 
 	/* version 2 stats */
 	{ "tx_deferral" },
@@ -673,6 +674,7 @@ struct nv_ethtool_stats {
 	u64 rx_packets;
 	u64 rx_errors_total;
 	u64 tx_errors_total;
+	u64 tx_timeout;
 
 	/* version 2 stats */
 	u64 tx_deferral;
@@ -2534,6 +2536,8 @@ static void nv_tx_timeout(struct net_device *dev)
 
 	spin_lock_irq(&np->lock);
 
+	np->estats.tx_timeout++;
+
 	/* 1) stop tx engine */
 	nv_stop_tx(dev);
 
Signature · 1 line
1.7.3.1
#5 lore raw

PATCH v2 3/8 forcedeth: allow to silence tx_timeout debug messages

In reply to David Decotigny

From: Sameer Nanda <[email protected]>

This adds a new module parameter "debug_tx_timeout" to silence most debug messages in case of TX timeout. These messages don't provide a signal/noise ratio high enough for production systems and, with ~30kB logged each time, they tend to add to a cascade effect if the system is already under stress (memory pressure, disk, etc.).

By default, the parameter is clear, meaning the debug messages are not displayed.

---
 drivers/net/ethernet/nvidia/forcedeth.c |   98 ++++++++++++++++++-------------
 1 files changed, 57 insertions(+), 41 deletions(-)
drivers/net/ethernet/nvidia/forcedeth.c +57 -41
index c82a3c7..375b03b 100644
@@ -894,6 +894,11 @@ enum {
 static int dma_64bit = NV_DMA_64BIT_ENABLED;
 
 /*
+ * Debug output control for tx_timeout
+ */
+static bool debug_tx_timeout = false;
+
+/*
  * Crossover Detection
  * Realtek 8201 phy + some OEM boards do not work properly.
  */
@@ -2481,56 +2486,64 @@ static void nv_tx_timeout(struct net_device *dev)
 	u32 status;
 	union ring_type put_tx;
 	int saved_tx_limit;
-	int i;
 
 	if (np->msi_flags & NV_MSI_X_ENABLED)
 		status = readl(base + NvRegMSIXIrqStatus) & NVREG_IRQSTAT_MASK;
 	else
 		status = readl(base + NvRegIrqStatus) & NVREG_IRQSTAT_MASK;
 
-	netdev_info(dev, "Got tx_timeout. irq: %08x\n", status);
+	netdev_warn(dev, "Got tx_timeout. irq status: %08x\n", status);
 
-	netdev_info(dev, "Ring at %lx\n", (unsigned long)np->ring_addr);
-	netdev_info(dev, "Dumping tx registers\n");
-	for (i = 0; i <= np->register_size; i += 32) {
-		netdev_info(dev,
-			    "%3x: %08x %08x %08x %08x %08x %08x %08x %08x\n",
-			    i,
-			    readl(base + i + 0), readl(base + i + 4),
-			    readl(base + i + 8), readl(base + i + 12),
-			    readl(base + i + 16), readl(base + i + 20),
-			    readl(base + i + 24), readl(base + i + 28));
-	}
-	netdev_info(dev, "Dumping tx ring\n");
-	for (i = 0; i < np->tx_ring_size; i += 4) {
-		if (!nv_optimized(np)) {
-			netdev_info(dev,
-				    "%03x: %08x %08x // %08x %08x // %08x %08x // %08x %08x\n",
-				    i,
-				    le32_to_cpu(np->tx_ring.orig[i].buf),
-				    le32_to_cpu(np->tx_ring.orig[i].flaglen),
-				    le32_to_cpu(np->tx_ring.orig[i+1].buf),
-				    le32_to_cpu(np->tx_ring.orig[i+1].flaglen),
-				    le32_to_cpu(np->tx_ring.orig[i+2].buf),
-				    le32_to_cpu(np->tx_ring.orig[i+2].flaglen),
-				    le32_to_cpu(np->tx_ring.orig[i+3].buf),
-				    le32_to_cpu(np->tx_ring.orig[i+3].flaglen));
-		} else {
+	if (unlikely(debug_tx_timeout)) {
+		int i;
+
+		netdev_info(dev, "Ring at %lx\n", (unsigned long)np->ring_addr);
+		netdev_info(dev, "Dumping tx registers\n");
+		for (i = 0; i <= np->register_size; i += 32) {
 			netdev_info(dev,
-				    "%03x: %08x %08x %08x // %08x %08x %08x // %08x %08x %08x // %08x %08x %08x\n",
+				    "%3x: %08x %08x %08x %08x "
+				    "%08x %08x %08x %08x\n",
 				    i,
-				    le32_to_cpu(np->tx_ring.ex[i].bufhigh),
-				    le32_to_cpu(np->tx_ring.ex[i].buflow),
-				    le32_to_cpu(np->tx_ring.ex[i].flaglen),
-				    le32_to_cpu(np->tx_ring.ex[i+1].bufhigh),
-				    le32_to_cpu(np->tx_ring.ex[i+1].buflow),
-				    le32_to_cpu(np->tx_ring.ex[i+1].flaglen),
-				    le32_to_cpu(np->tx_ring.ex[i+2].bufhigh),
-				    le32_to_cpu(np->tx_ring.ex[i+2].buflow),
-				    le32_to_cpu(np->tx_ring.ex[i+2].flaglen),
-				    le32_to_cpu(np->tx_ring.ex[i+3].bufhigh),
-				    le32_to_cpu(np->tx_ring.ex[i+3].buflow),
-				    le32_to_cpu(np->tx_ring.ex[i+3].flaglen));
+				    readl(base + i + 0), readl(base + i + 4),
+				    readl(base + i + 8), readl(base + i + 12),
+				    readl(base + i + 16), readl(base + i + 20),
+				    readl(base + i + 24), readl(base + i + 28));
+		}
+		netdev_info(dev, "Dumping tx ring\n");
+		for (i = 0; i < np->tx_ring_size; i += 4) {
+			if (!nv_optimized(np)) {
+				netdev_info(dev,
+					    "%03x: %08x %08x // %08x %08x "
+					    "// %08x %08x // %08x %08x\n",
+					    i,
+					    le32_to_cpu(np->tx_ring.orig[i].buf),
+					    le32_to_cpu(np->tx_ring.orig[i].flaglen),
+					    le32_to_cpu(np->tx_ring.orig[i+1].buf),
+					    le32_to_cpu(np->tx_ring.orig[i+1].flaglen),
+					    le32_to_cpu(np->tx_ring.orig[i+2].buf),
+					    le32_to_cpu(np->tx_ring.orig[i+2].flaglen),
+					    le32_to_cpu(np->tx_ring.orig[i+3].buf),
+					    le32_to_cpu(np->tx_ring.orig[i+3].flaglen));
+			} else {
+				netdev_info(dev,
+					    "%03x: %08x %08x %08x "
+					    "// %08x %08x %08x "
+					    "// %08x %08x %08x "
+					    "// %08x %08x %08x\n",
+					    i,
+					    le32_to_cpu(np->tx_ring.ex[i].bufhigh),
+					    le32_to_cpu(np->tx_ring.ex[i].buflow),
+					    le32_to_cpu(np->tx_ring.ex[i].flaglen),
+					    le32_to_cpu(np->tx_ring.ex[i+1].bufhigh),
+					    le32_to_cpu(np->tx_ring.ex[i+1].buflow),
+					    le32_to_cpu(np->tx_ring.ex[i+1].flaglen),
+					    le32_to_cpu(np->tx_ring.ex[i+2].bufhigh),
+					    le32_to_cpu(np->tx_ring.ex[i+2].buflow),
+					    le32_to_cpu(np->tx_ring.ex[i+2].flaglen),
+					    le32_to_cpu(np->tx_ring.ex[i+3].bufhigh),
+					    le32_to_cpu(np->tx_ring.ex[i+3].buflow),
+					    le32_to_cpu(np->tx_ring.ex[i+3].flaglen));
+			}
 		}
 	}
 
@@ -6016,6 +6029,9 @@ module_param(phy_cross, int, 0);
 MODULE_PARM_DESC(phy_cross, "Phy crossover detection for Realtek 8201 phy is enabled by setting to 1 and disabled by setting to 0.");
 module_param(phy_power_down, int, 0);
 MODULE_PARM_DESC(phy_power_down, "Power down phy and disable link when interface is down (1), or leave phy powered up (0).");
+module_param(debug_tx_timeout, bool, 0);
+MODULE_PARM_DESC(debug_tx_timeout,
+		 "Dump tx related registers and ring when tx_timeout happens");
 
 MODULE_AUTHOR("Manfred Spraul <[email protected]>");
 MODULE_DESCRIPTION("Reverse Engineered nForce ethernet driver");
Signature · 1 line
1.7.3.1
#6 lore raw

PATCH v2 4/8 forcedeth: Acknowledge only interrupts that are being processed

In reply to David Decotigny

From: Mike Ditto <[email protected]>

This is to avoid a race, accidentally acknowledging an interrupt that we didn't notice and won't immediately process. This is based solely on code inspection; it is not known if there was an actual bug here.

---
 drivers/net/ethernet/nvidia/forcedeth.c |   13 ++++++++-----
 1 files changed, 8 insertions(+), 5 deletions(-)
drivers/net/ethernet/nvidia/forcedeth.c +8 -5
index 375b03b..4a45a46 100644
@@ -3419,7 +3419,8 @@ static irqreturn_t nv_nic_irq_tx(int foo, void *data)
 
 	for (i = 0;; i++) {
 		events = readl(base + NvRegMSIXIrqStatus) & NVREG_IRQ_TX_ALL;
-		writel(NVREG_IRQ_TX_ALL, base + NvRegMSIXIrqStatus);
+		writel(events, base + NvRegMSIXIrqStatus);
+		netdev_dbg(dev, "tx irq events: %08x\n", events);
 		if (!(events & np->irqmask))
 			break;
 
@@ -3530,7 +3531,8 @@ static irqreturn_t nv_nic_irq_rx(int foo, void *data)
 
 	for (i = 0;; i++) {
 		events = readl(base + NvRegMSIXIrqStatus) & NVREG_IRQ_RX_ALL;
-		writel(NVREG_IRQ_RX_ALL, base + NvRegMSIXIrqStatus);
+		writel(events, base + NvRegMSIXIrqStatus);
+		netdev_dbg(dev, "rx irq events: %08x\n", events);
 		if (!(events & np->irqmask))
 			break;
 
@@ -3574,7 +3576,8 @@ static irqreturn_t nv_nic_irq_other(int foo, void *data)
 
 	for (i = 0;; i++) {
 		events = readl(base + NvRegMSIXIrqStatus) & NVREG_IRQ_OTHER;
-		writel(NVREG_IRQ_OTHER, base + NvRegMSIXIrqStatus);
+		writel(events, base + NvRegMSIXIrqStatus);
+		netdev_dbg(dev, "irq events: %08x\n", events);
 		if (!(events & np->irqmask))
 			break;
 
@@ -3638,10 +3641,10 @@ static irqreturn_t nv_nic_irq_test(int foo, void *data)
 
 	if (!(np->msi_flags & NV_MSI_X_ENABLED)) {
 		events = readl(base + NvRegIrqStatus) & NVREG_IRQSTAT_MASK;
-		writel(NVREG_IRQ_TIMER, base + NvRegIrqStatus);
+		writel(events & NVREG_IRQ_TIMER, base + NvRegIrqStatus);
 	} else {
 		events = readl(base + NvRegMSIXIrqStatus) & NVREG_IRQSTAT_MASK;
-		writel(NVREG_IRQ_TIMER, base + NvRegMSIXIrqStatus);
+		writel(events & NVREG_IRQ_TIMER, base + NvRegMSIXIrqStatus);
 	}
 	pci_push(base);
 	if (!(events & NVREG_IRQ_TIMER))
Signature · 1 line
1.7.3.1
#7 lore raw

PATCH v2 5/8 forcedeth: Add messages to indicate using MSI or MSI-X

In reply to David Decotigny

From: Mike Ditto <[email protected]>

This adds a few debug messages to indicate whether PCIe interrupts are signaled with MSI or MSI-X.

---
 drivers/net/ethernet/nvidia/forcedeth.c |    2 ++
 1 files changed, 2 insertions(+), 0 deletions(-)
drivers/net/ethernet/nvidia/forcedeth.c +2 -0
index 4a45a46..0af12a8 100644
@@ -3761,6 +3761,7 @@ static int nv_request_irq(struct net_device *dev, int intr_test)
 				writel(0, base + NvRegMSIXMap0);
 				writel(0, base + NvRegMSIXMap1);
 			}
+			netdev_info(dev, "MSI-X enabled\n");
 		}
 	}
 	if (ret != 0 && np->msi_flags & NV_MSI_CAPABLE) {
@@ -3782,6 +3783,7 @@ static int nv_request_irq(struct net_device *dev, int intr_test)
 			writel(0, base + NvRegMSIMap1);
 			/* enable msi vector 0 */
 			writel(NVREG_MSI_VECTOR_0_ENABLED, base + NvRegMSIIrqMask);
+			netdev_info(dev, "MSI enabled\n");
 		}
 	}
 	if (ret != 0) {
Signature · 1 line
1.7.3.1
#8 lore raw

PATCH v2 6/8 forcedeth: Fix a race during rmmod of forcedeth

In reply to David Decotigny

From: Salman Qazi <[email protected]>

The race was between del_timer_sync and nv_do_stats_poll called through nv_get_ethtool_stats. To prevent this, we have to introduce mutual exclusion between nv_get_ethtool_stats and del_timer_sync. Notice that we don't put the mutual exclusion in nv_do_stats_poll. That's because doing so would result in a deadlock, since it is a timer callback and hence already waited for by timer deletion.

---
 drivers/net/ethernet/nvidia/forcedeth.c |   11 ++++++++++-
 1 files changed, 10 insertions(+), 1 deletions(-)
drivers/net/ethernet/nvidia/forcedeth.c +10 -1
index 0af12a8..7996782 100644
@@ -3937,6 +3937,10 @@ static void nv_poll_controller(struct net_device *dev)
 }
 #endif
 
+/* No locking is needed as long as this is in the timer
+ * callback.  However, any other callers must call this
+ * function with np->lock held.
+ */
 static void nv_do_stats_poll(unsigned long data)
 {
 	struct net_device *dev = (struct net_device *) data;
@@ -4589,12 +4593,17 @@ static int nv_get_sset_count(struct net_device *dev, int sset)
 
 static void nv_get_ethtool_stats(struct net_device *dev, struct ethtool_stats *estats, u64 *buffer)
 {
+	unsigned long flags;
 	struct fe_priv *np = netdev_priv(dev);
 
+	spin_lock_irqsave(&np->lock, flags);
+
 	/* update stats */
 	nv_do_stats_poll((unsigned long)dev);
 
 	memcpy(buffer, &np->estats, nv_get_sset_count(dev, ETH_SS_STATS)*sizeof(u64));
+
+	spin_unlock_irqrestore(&np->lock, flags);
 }
 
 static int nv_link_test(struct net_device *dev)
@@ -5189,13 +5198,13 @@ static int nv_close(struct net_device *dev)
 
 	spin_lock_irq(&np->lock);
 	np->in_shutdown = 1;
+	del_timer_sync(&np->stats_poll);
 	spin_unlock_irq(&np->lock);
 	nv_napi_disable(dev);
 	synchronize_irq(np->pci_dev->irq);
 
 	del_timer_sync(&np->oom_kick);
 	del_timer_sync(&np->nic_poll);
-	del_timer_sync(&np->stats_poll);
 
 	netif_stop_queue(dev);
 	spin_lock_irq(&np->lock);
Signature · 1 line
1.7.3.1
#9 lore raw

Re: [PATCH net v2 6/8] forcedeth: Fix a race during rmmod of forcedeth

Quoted 4 lines · David Decotigny

From: Salman Qazi <[email protected]>

The race was between del_timer_sync and nv_do_stats_poll called through nv_get_ethtool_stats.

I don't think so. nv_close() and nv_get_ethtool_stats() are both called with RTNL held.

Calling the timer function from nv_get_ethtool_stats is very likely part of the problem though, so why don't you stop doing that?

[...]

drivers/net/ethernet/nvidia/forcedeth.c static void nv_poll_controller(struct net_device *dev) · line 3943

Quoted 12 lines
drivers/net/ethernet/nvidia/forcedeth.c +4 -0
index 0af12a8..7996782 100644
@@ -3937,6 +3937,10 @@ static void nv_poll_controller(struct net_device *dev)
 }
 #endif
 
+/* No locking is needed as long as this is in the timer
+ * callback.  However, any other callers must call this
+ * function with np->lock held.
+ */

So long as this function is used by all of (1) the timer function (2) the ndo_get_stats implementation (3) the ethtool get_stats implementation, it can most certainly be called concurrently on multiple processors.

You could have (2) and (3) return the last polled stats and not poll the hardware themselves, but you would need to use the functions from <linux/u64_stats_sync.h> to avoid word-tearing on 32-bit architectures.

Quoted 8 lines
drivers/net/ethernet/nvidia/forcedeth.c +1 -0
 static void nv_do_stats_poll(unsigned long data)
 {
        struct net_device *dev = (struct net_device *) data;
@@ -4589,12 +4593,17 @@ static int nv_get_sset_count(struct net_device *dev, int sset)
 
 static void nv_get_ethtool_stats(struct net_device *dev, struct ethtool_stats *estats, u64 *buffer)
 {
+       unsigned long flags;

drivers/net/ethernet/nvidia/forcedeth.c static int nv_get_sset_count(struct net_device *dev, int sset) · line 4606

Quoted 10 lines
drivers/net/ethernet/nvidia/forcedeth.c +4 -0
        struct fe_priv *np = netdev_priv(dev);
 
+       spin_lock_irqsave(&np->lock, flags);
+
        /* update stats */
        nv_do_stats_poll((unsigned long)dev);
 
        memcpy(buffer, &np->estats, nv_get_sset_count(dev, ETH_SS_STATS)*sizeof(u64));
+
+       spin_unlock_irqrestore(&np->lock, flags);

This function is not called from interrupt context.

Quoted 8 lines
drivers/net/ethernet/nvidia/forcedeth.c +1 -0
 }
 
 static int nv_link_test(struct net_device *dev)
@@ -5189,13 +5198,13 @@ static int nv_close(struct net_device *dev)
 
        spin_lock_irq(&np->lock);
        np->in_shutdown = 1;
+       del_timer_sync(&np->stats_poll);

drivers/net/ethernet/nvidia/forcedeth.c static int nv_close(struct net_device *dev) · line 5210

Quoted 10 lines
drivers/net/ethernet/nvidia/forcedeth.c +0 -1
        spin_unlock_irq(&np->lock);
        nv_napi_disable(dev);
        synchronize_irq(np->pci_dev->irq);
 
        del_timer_sync(&np->oom_kick);
        del_timer_sync(&np->nic_poll);
-       del_timer_sync(&np->stats_poll);
 
        netif_stop_queue(dev);
        spin_lock_irq(&np->lock);

I don't believe this code movement is helpful.

Ben.

Signature · 3 lines
Ben Hutchings, Staff Engineer, Solarflare
Not speaking for my employer; that's the marketing department's job.
They asked us to note that Solarflare product names are trademarked.
#10 lore raw

Re: [PATCH net v2 6/8] forcedeth: Fix a race during rmmod of forcedeth

Ben,

Thank you for your comments. I understand this patch needs more work. So I am going to remove it from this series for now and work on it in isolation.

Regards,

Quoted 85 lines · Ben Hutchings
Quoted 4 lines · David Decotigny

From: Salman Qazi <[email protected]>

The race was between del_timer_sync and nv_do_stats_poll called through nv_get_ethtool_stats.

I don't think so.  nv_close() and nv_get_ethtool_stats() are both called with RTNL held.

Calling the timer function from nv_get_ethtool_stats is very likely part of the problem though, so why don't you stop doing that?

[...]

drivers/net/ethernet/nvidia/forcedeth.c static void nv_poll_controller(struct net_device *dev) · line 3943

Quoted 12 lines
drivers/net/ethernet/nvidia/forcedeth.c +4 -0
index 0af12a8..7996782 100644
@@ -3937,6 +3937,10 @@ static void nv_poll_controller(struct net_device *dev)
  }
  #endif
 
+/* No locking is needed as long as this is in the timer
+ * callback.  However, any other callers must call this
+ * function with np->lock held.
+ */

So long as this function is used by all of (1) the timer function (2) the ndo_get_stats implementation (3) the ethtool get_stats implementation, it can most certainly be called concurrently on multiple processors.

You could have (2) and (3) return the last polled stats and not poll the hardware themselves, but you would need to use the functions from <linux/u64_stats_sync.h> to avoid word-tearing on 32-bit architectures.

Quoted 8 lines
drivers/net/ethernet/nvidia/forcedeth.c +1 -0
  static void nv_do_stats_poll(unsigned long data)
  {
         struct net_device *dev = (struct net_device *) data;
@@ -4589,12 +4593,17 @@ static int nv_get_sset_count(struct net_device *dev, int sset)
 
  static void nv_get_ethtool_stats(struct net_device *dev, struct ethtool_stats *estats, u64 *buffer)
  {
+       unsigned long flags;

drivers/net/ethernet/nvidia/forcedeth.c static int nv_get_sset_count(struct net_device *dev, int sset) · line 4606

Quoted 10 lines
drivers/net/ethernet/nvidia/forcedeth.c +4 -0
         struct fe_priv *np = netdev_priv(dev);
 
+       spin_lock_irqsave(&np->lock, flags);
+
         /* update stats */
         nv_do_stats_poll((unsigned long)dev);
 
         memcpy(buffer, &np->estats, nv_get_sset_count(dev, ETH_SS_STATS)*sizeof(u64));
+
+       spin_unlock_irqrestore(&np->lock, flags);

This function is not called from interrupt context.

Quoted 8 lines
drivers/net/ethernet/nvidia/forcedeth.c +1 -0
  }
 
  static int nv_link_test(struct net_device *dev)
@@ -5189,13 +5198,13 @@ static int nv_close(struct net_device *dev)
 
         spin_lock_irq(&np->lock);
         np->in_shutdown = 1;
+       del_timer_sync(&np->stats_poll);

drivers/net/ethernet/nvidia/forcedeth.c static int nv_close(struct net_device *dev) · line 5210

Quoted 10 lines
drivers/net/ethernet/nvidia/forcedeth.c +0 -1
         spin_unlock_irq(&np->lock);
         nv_napi_disable(dev);
         synchronize_irq(np->pci_dev->irq);
 
         del_timer_sync(&np->oom_kick);
         del_timer_sync(&np->nic_poll);
-       del_timer_sync(&np->stats_poll);
 
         netif_stop_queue(dev);
         spin_lock_irq(&np->lock);

I don't believe this code movement is helpful.

Ben.

--
Ben Hutchings, Staff Engineer, Solarflare
Not speaking for my employer; that's the marketing department's job. They asked us to note that Solarflare product names are trademarked.

#11 lore raw

Re: [PATCH net v2 6/8] forcedeth: Fix a race during rmmod of forcedeth

In reply to Ben Hutchings

Hello,

Thank you for your feedback, Ben. I looked at this patch more carefully:

Quoted 11 lines · Ben Hutchings
Quoted 4 lines · David Decotigny

From: Salman Qazi <[email protected]>

The race was between del_timer_sync and nv_do_stats_poll called through nv_get_ethtool_stats.

I don't think so.  nv_close() and nv_get_ethtool_stats() are both called with RTNL held.

Calling the timer function from nv_get_ethtool_stats is very likely part of the problem though, so why don't you stop doing that?

Right. As the initial author noted, the problem is presumably that mod_timer was called after del_timer_sync, from a non-timer path (which can only be via nv_get_ethtool_stats in our case). As you noted, it's enough to ensure this path doesn't exist, which is easy to do here and doesn't require synchro. I'll send an interim patch for that to netdev (it should fix the race but will have the same shortcomings as current code wrt 64b-correctness on 32b hosts).

When switching to the ndo_get_stats64 api, I will make sure u64_stats_sync.h is used. This is for another patch series scheduled later for net-next.

Quoted 9 lines
Quoted 9 lines
drivers/net/ethernet/nvidia/forcedeth.c +1 -0
@@ -5189,13 +5198,13 @@ static int nv_close(struct net_device *dev)
 
         spin_lock_irq(&np->lock);
         np->in_shutdown = 1;
+       del_timer_sync(&np->stats_poll);
         spin_unlock_irq(&np->lock);
         nv_napi_disable(dev);
         synchronize_irq(np->pci_dev->irq);
 
Quoted 8 lines

drivers/net/ethernet/nvidia/forcedeth.c static int nv_close(struct net_device *dev) · line 5210

Quoted 6 lines
drivers/net/ethernet/nvidia/forcedeth.c +0 -1
         del_timer_sync(&np->oom_kick);
         del_timer_sync(&np->nic_poll);
-       del_timer_sync(&np->stats_poll);
 
         netif_stop_queue(dev);
         spin_lock_irq(&np->lock);

I don't believe this code movement is helpful.

I agree.

Regards,

#12 lore raw

PATCH v2 7/8 forcedeth: expose module parameters in /sys/module

In reply to David Decotigny

In particular, debug_tx_timeout can be updated at runtime.

---
 drivers/net/ethernet/nvidia/forcedeth.c |   18 +++++++++---------
 1 files changed, 9 insertions(+), 9 deletions(-)
drivers/net/ethernet/nvidia/forcedeth.c +9 -9
index 7996782..5442638 100644
@@ -6027,23 +6027,23 @@ static void __exit exit_nic(void)
 	pci_unregister_driver(&driver);
 }
 
-module_param(max_interrupt_work, int, 0);
+module_param(max_interrupt_work, int, S_IRUGO);
 MODULE_PARM_DESC(max_interrupt_work, "forcedeth maximum events handled per interrupt");
-module_param(optimization_mode, int, 0);
+module_param(optimization_mode, int, S_IRUGO);
 MODULE_PARM_DESC(optimization_mode, "In throughput mode (0), every tx & rx packet will generate an interrupt. In CPU mode (1), interrupts are controlled by a timer. In dynamic mode (2), the mode toggles between throughput and CPU mode based on network load.");
-module_param(poll_interval, int, 0);
+module_param(poll_interval, int, S_IRUGO);
 MODULE_PARM_DESC(poll_interval, "Interval determines how frequent timer interrupt is generated by [(time_in_micro_secs * 100) / (2^10)]. Min is 0 and Max is 65535.");
-module_param(msi, int, 0);
+module_param(msi, int, S_IRUGO);
 MODULE_PARM_DESC(msi, "MSI interrupts are enabled by setting to 1 and disabled by setting to 0.");
-module_param(msix, int, 0);
+module_param(msix, int, S_IRUGO);
 MODULE_PARM_DESC(msix, "MSIX interrupts are enabled by setting to 1 and disabled by setting to 0.");
-module_param(dma_64bit, int, 0);
+module_param(dma_64bit, int, S_IRUGO);
 MODULE_PARM_DESC(dma_64bit, "High DMA is enabled by setting to 1 and disabled by setting to 0.");
-module_param(phy_cross, int, 0);
+module_param(phy_cross, int, S_IRUGO);
 MODULE_PARM_DESC(phy_cross, "Phy crossover detection for Realtek 8201 phy is enabled by setting to 1 and disabled by setting to 0.");
-module_param(phy_power_down, int, 0);
+module_param(phy_power_down, int, S_IRUGO);
 MODULE_PARM_DESC(phy_power_down, "Power down phy and disable link when interface is down (1), or leave phy powered up (0).");
-module_param(debug_tx_timeout, bool, 0);
+module_param(debug_tx_timeout, bool, S_IRUGO|S_IWUSR);
 MODULE_PARM_DESC(debug_tx_timeout,
 		 "Dump tx related registers and ring when tx_timeout happens");
 
Signature · 1 line
1.7.3.1
#13 lore raw

PATCH v2 8/8 forcedeth: fix a few sparse warnings (variable shadowing)

In reply to David Decotigny

This fixes the following sparse warnings:
drivers/net/ethernet/nvidia/forcedeth.c:2113:7: warning: symbol 'size' shadows an earlier one drivers/net/ethernet/nvidia/forcedeth.c:2102:6: originally declared here drivers/net/ethernet/nvidia/forcedeth.c:2155:7: warning: symbol 'size' shadows an earlier one drivers/net/ethernet/nvidia/forcedeth.c:2102:6: originally declared here drivers/net/ethernet/nvidia/forcedeth.c:2227:7: warning: symbol 'size' shadows an earlier one drivers/net/ethernet/nvidia/forcedeth.c:2215:6: originally declared here drivers/net/ethernet/nvidia/forcedeth.c:2271:7: warning: symbol 'size' shadows an earlier one drivers/net/ethernet/nvidia/forcedeth.c:2215:6: originally declared here drivers/net/ethernet/nvidia/forcedeth.c:2986:20: warning: symbol 'addr' shadows an earlier one drivers/net/ethernet/nvidia/forcedeth.c:2963:6: originally declared here

---
 drivers/net/ethernet/nvidia/forcedeth.c |   34 +++++++++++++++---------------
 1 files changed, 17 insertions(+), 17 deletions(-)
drivers/net/ethernet/nvidia/forcedeth.c +17 -17
index 5442638..913f054 100644
@@ -2110,10 +2110,10 @@ static netdev_tx_t nv_start_xmit(struct sk_buff *skb, struct net_device *dev)
 
 	/* add fragments to entries count */
 	for (i = 0; i < fragments; i++) {
-		u32 size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
+		u32 frag_size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
 
-		entries += (size >> NV_TX2_TSO_MAX_SHIFT) +
-			   ((size & (NV_TX2_TSO_MAX_SIZE-1)) ? 1 : 0);
+		entries += (frag_size >> NV_TX2_TSO_MAX_SHIFT) +
+			   ((frag_size & (NV_TX2_TSO_MAX_SIZE-1)) ? 1 : 0);
 	}
 
 	spin_lock_irqsave(&np->lock, flags);
@@ -2152,13 +2152,13 @@ static netdev_tx_t nv_start_xmit(struct sk_buff *skb, struct net_device *dev)
 	/* setup the fragments */
 	for (i = 0; i < fragments; i++) {
 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
-		u32 size = skb_frag_size(frag);
+		u32 frag_size = skb_frag_size(frag);
 		offset = 0;
 
 		do {
 			prev_tx = put_tx;
 			prev_tx_ctx = np->put_tx_ctx;
-			bcnt = (size > NV_TX2_TSO_MAX_SIZE) ? NV_TX2_TSO_MAX_SIZE : size;
+			bcnt = (frag_size > NV_TX2_TSO_MAX_SIZE) ? NV_TX2_TSO_MAX_SIZE : frag_size;
 			np->put_tx_ctx->dma = skb_frag_dma_map(
 							&np->pci_dev->dev,
 							frag, offset,
@@ -2170,12 +2170,12 @@ static netdev_tx_t nv_start_xmit(struct sk_buff *skb, struct net_device *dev)
 			put_tx->flaglen = cpu_to_le32((bcnt-1) | tx_flags);
 
 			offset += bcnt;
-			size -= bcnt;
+			frag_size -= bcnt;
 			if (unlikely(put_tx++ == np->last_tx.orig))
 				put_tx = np->first_tx.orig;
 			if (unlikely(np->put_tx_ctx++ == np->last_tx_ctx))
 				np->put_tx_ctx = np->first_tx_ctx;
-		} while (size);
+		} while (frag_size);
 	}
 
 	/* set last fragment flag  */
@@ -2224,10 +2224,10 @@ static netdev_tx_t nv_start_xmit_optimized(struct sk_buff *skb,
 
 	/* add fragments to entries count */
 	for (i = 0; i < fragments; i++) {
-		u32 size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
+		u32 frag_size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
 
-		entries += (size >> NV_TX2_TSO_MAX_SHIFT) +
-			   ((size & (NV_TX2_TSO_MAX_SIZE-1)) ? 1 : 0);
+		entries += (frag_size >> NV_TX2_TSO_MAX_SHIFT) +
+			   ((frag_size & (NV_TX2_TSO_MAX_SIZE-1)) ? 1 : 0);
 	}
 
 	spin_lock_irqsave(&np->lock, flags);
@@ -2268,13 +2268,13 @@ static netdev_tx_t nv_start_xmit_optimized(struct sk_buff *skb,
 	/* setup the fragments */
 	for (i = 0; i < fragments; i++) {
 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
-		u32 size = skb_frag_size(frag);
+		u32 frag_size = skb_frag_size(frag);
 		offset = 0;
 
 		do {
 			prev_tx = put_tx;
 			prev_tx_ctx = np->put_tx_ctx;
-			bcnt = (size > NV_TX2_TSO_MAX_SIZE) ? NV_TX2_TSO_MAX_SIZE : size;
+			bcnt = (frag_size > NV_TX2_TSO_MAX_SIZE) ? NV_TX2_TSO_MAX_SIZE : frag_size;
 			np->put_tx_ctx->dma = skb_frag_dma_map(
 							&np->pci_dev->dev,
 							frag, offset,
@@ -2287,12 +2287,12 @@ static netdev_tx_t nv_start_xmit_optimized(struct sk_buff *skb,
 			put_tx->flaglen = cpu_to_le32((bcnt-1) | tx_flags);
 
 			offset += bcnt;
-			size -= bcnt;
+			frag_size -= bcnt;
 			if (unlikely(put_tx++ == np->last_tx.ex))
 				put_tx = np->first_tx.ex;
 			if (unlikely(np->put_tx_ctx++ == np->last_tx_ctx))
 				np->put_tx_ctx = np->first_tx_ctx;
-		} while (size);
+		} while (frag_size);
 	}
 
 	/* set last fragment flag  */
@@ -2983,11 +2983,11 @@ static void nv_set_multicast(struct net_device *dev)
 				struct netdev_hw_addr *ha;
 
 				netdev_for_each_mc_addr(ha, dev) {
-					unsigned char *addr = ha->addr;
+					unsigned char *hw_addr = ha->addr;
 					u32 a, b;
 
-					a = le32_to_cpu(*(__le32 *) addr);
-					b = le16_to_cpu(*(__le16 *) (&addr[4]));
+					a = le32_to_cpu(*(__le32 *) hw_addr);
+					b = le16_to_cpu(*(__le16 *) (&hw_addr[4]));
 					alwaysOn[0] &= a;
 					alwaysOff[0] &= ~a;
 					alwaysOn[1] &= b;
Signature · 1 line
1.7.3.1