Skip to contentSkip to conversation

sfc: I2C adapter initialisation fixes

PATCH In review Ben Hutchings 3 participants 10 messages Jul 18, 2008 – Jul 22, 2008

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/4 sfc: I2C adapter initialisation fixes

As recommended by Jean Delvare:
- Increase timeout to 50 ms
- Leave adapter class clear so that unwanted drivers do not probe our bus
- Use strlcpy() for name initialisation

---
Jeff,

These are exactly the differences between net-2.6 and the version of sfc I intended to submit to you before today. If this doesn't apply then I'm confused as to what you have.

Ben.

drivers/net/sfc/falcon.c +3 -9
index 630406e..9138ee5 100644
@@ -223,13 +223,8 @@ static struct i2c_algo_bit_data falcon_i2c_bit_operations = {
 	.getsda		= falcon_getsda,
 	.getscl		= falcon_getscl,
 	.udelay		= 5,
-	/*
-	 * This is the number of system clock ticks after which
-	 * i2c-algo-bit gives up waiting for SCL to become high.
-	 * It must be at least 2 since the first tick can happen
-	 * immediately after it starts waiting.
-	 */
-	.timeout	= 2,
+	/* Wait up to 50 ms for slave to let us pull SCL high */
+	.timeout	= DIV_ROUND_UP(HZ, 20),
 };
 
 /**************************************************************************
@@ -2479,12 +2474,11 @@ int falcon_probe_nic(struct efx_nic *efx)
 
 	/* Initialise I2C adapter */
  	efx->i2c_adap.owner = THIS_MODULE;
- 	efx->i2c_adap.class = I2C_CLASS_HWMON;
 	nic_data->i2c_data = falcon_i2c_bit_operations;
 	nic_data->i2c_data.data = efx;
  	efx->i2c_adap.algo_data = &nic_data->i2c_data;
 	efx->i2c_adap.dev.parent = &efx->pci_dev->dev;
-	strcpy(efx->i2c_adap.name, "SFC4000 GPIO");
+	strlcpy(efx->i2c_adap.name, "SFC4000 GPIO", sizeof(efx->i2c_adap.name));
 	rc = i2c_bit_add_bus(&efx->i2c_adap);
 	if (rc)
 		goto fail5;
Signature · 2 lines
Ben Hutchings, Senior Software Engineer, Solarflare Communications
Not speaking for my employer; that's the marketing department's job.
#1 lore raw

PATCH 2/4 sfc: Use a separate workqueue for resets

This avoids deadlock in case a reset is triggered during self-test.

---
 drivers/net/sfc/efx.c        |   20 +++++++++++++++++---
 drivers/net/sfc/net_driver.h |    5 ++++-
 2 files changed, 21 insertions(+), 4 deletions(-)
drivers/net/sfc/efx.c +17 -3
index 0e5fecf..4e89c89 100644
@@ -2046,7 +2046,7 @@ void efx_schedule_reset(struct efx_nic *efx, enum reset_type type)
 
 	efx->reset_pending = method;
 
-	queue_work(efx->workqueue, &efx->reset_work);
+	queue_work(efx->reset_workqueue, &efx->reset_work);
 }
 
 /**************************************************************************
@@ -2204,14 +2204,28 @@ static int efx_init_struct(struct efx_nic *efx, struct efx_nic_type *type,
 		goto fail1;
 	}
 
+	efx->reset_workqueue = create_singlethread_workqueue("sfc_reset");
+	if (!efx->reset_workqueue) {
+		rc = -ENOMEM;
+		goto fail2;
+	}
+
 	return 0;
 
+ fail2:
+	destroy_workqueue(efx->workqueue);
+	efx->workqueue = NULL;
+
  fail1:
 	return rc;
 }
 
 static void efx_fini_struct(struct efx_nic *efx)
 {
+	if (efx->reset_workqueue) {
+		destroy_workqueue(efx->reset_workqueue);
+		efx->reset_workqueue = NULL;
+	}
 	if (efx->workqueue) {
 		destroy_workqueue(efx->workqueue);
 		efx->workqueue = NULL;
@@ -2281,7 +2295,7 @@ static void efx_pci_remove(struct pci_dev *pci_dev)
 	 * scheduled from this point because efx_stop_all() has been
 	 * called, we are no longer registered with driverlink, and
 	 * the net_device's have been removed. */
-	flush_workqueue(efx->workqueue);
+	flush_workqueue(efx->reset_workqueue);
 
 	efx_pci_remove_main(efx);
 
@@ -2418,7 +2432,7 @@ static int __devinit efx_pci_probe(struct pci_dev *pci_dev,
 		 * scheduled since efx_stop_all() has been called, and we
 		 * have not and never have been registered with either
 		 * the rtnetlink or driverlink layers. */
-		cancel_work_sync(&efx->reset_work);
+		flush_workqueue(efx->reset_workqueue);
 
 		/* Retry if a recoverably reset event has been scheduled */
 		if ((efx->reset_pending != RESET_TYPE_INVISIBLE) &&
drivers/net/sfc/net_driver.h +4 -1
index 535807b..c9ce460 100644
@@ -733,7 +733,9 @@ struct efx_nic_errors {
  * @pci_dev: The PCI device
  * @type: Controller type attributes
  * @legacy_irq: IRQ number
- * @workqueue: Workqueue for resets, port reconfigures and the HW monitor
+ * @workqueue: Workqueue for port reconfigures and the HW monitor.
+ *	Work items do not hold and must not acquire RTNL.
+ * @reset_workqueue: Workqueue for resets.  Work item will acquire RTNL.
  * @reset_work: Scheduled reset workitem
  * @monitor_work: Hardware monitor workitem
  * @membase_phys: Memory BAR value as physical address
@@ -821,6 +823,7 @@ struct efx_nic {
 	const struct efx_nic_type *type;
 	int legacy_irq;
 	struct workqueue_struct *workqueue;
+	struct workqueue_struct *reset_workqueue;
 	struct work_struct reset_work;
 	struct delayed_work monitor_work;
 	unsigned long membase_phys;
Signature · 2 lines
Ben Hutchings, Senior Software Engineer, Solarflare Communications
Not speaking for my employer; that's the marketing department's job.
#2 lore raw

PATCH 3/4 sfc: resolve tx multiqueue bug

In reply to Ben Hutchings

With the recent changes to tx multiqueue, sfc was not calling netif_start_queue() before calling netif_wake_queue(). This causes an oops when opening a device.

---
 drivers/net/sfc/tx.c |    1 +
 1 files changed, 1 insertions(+), 0 deletions(-)
drivers/net/sfc/tx.c +1 -0
index 5cdd082..798b06b 100644
@@ -53,6 +53,7 @@ inline void efx_wake_queue(struct efx_nic *efx)
 	if (atomic_dec_and_lock(&efx->netif_stop_count,
 				&efx->netif_stop_lock)) {
 		EFX_TRACE(efx, "waking TX queue\n");
+		netif_start_queue(efx->net_dev);
 		netif_wake_queue(efx->net_dev);
 		spin_unlock(&efx->netif_stop_lock);
 	}
Signature · 2 lines
Ben Hutchings, Senior Software Engineer, Solarflare Communications
Not speaking for my employer; that's the marketing department's job.
#3 lore raw

Re: [PATCH 3/4] sfc: resolve tx multiqueue bug

Quoted 14 lines · Ben Hutchings

With the recent changes to tx multiqueue, sfc was not calling netif_start_queue() before calling netif_wake_queue(). This causes an oops when opening a device.

---
 drivers/net/sfc/tx.c |    1 +
 1 files changed, 1 insertions(+), 0 deletions(-)
drivers/net/sfc/tx.c +0 -0
index 5cdd082..798b06b 100644

drivers/net/sfc/tx.c inline void efx_wake_queue(struct efx_nic *efx) · line 59

Quoted 7 lines
drivers/net/sfc/tx.c +1 -0
@@ -53,6 +53,7 @@ inline void efx_wake_queue(struct efx_nic *efx)
 	if (atomic_dec_and_lock(&efx->netif_stop_count,
 				&efx->netif_stop_lock)) {
 		EFX_TRACE(efx, "waking TX queue\n");
+		netif_start_queue(efx->net_dev);
 		netif_wake_queue(efx->net_dev);
 		spin_unlock(&efx->netif_stop_lock);
 	}

Ahem... ignore this, it's obviously not right.

Ben.

Signature · 2 lines
Ben Hutchings, Senior Software Engineer, Solarflare Communications
Not speaking for my employer; that's the marketing department's job.
#4 lore raw

Re: [PATCH 3/4] sfc: resolve tx multiqueue bug

The more I look at this issue of start vs wake, the more it seems like the distinction should not be visible to drivers.

So long as a queue is only woken in response to TX completions, the current arrangement is fine. However, sfc needs to tear down and restart hardware TX queues as part of some reconfiguration, self-test and recovery code, and I doubt it's the only such driver.

I think the corresponding kernel queue must be stopped and then started when we do this (otherwise we could return NETDEV_TX_BUSY in hard_start_xmit, but I understand that to be deprecated). So we must also wake the kernel queue if and only if it's considered active - but we don't know whether that's the case because deactivation is done asynchronously.

Ben.

Signature · 2 lines
Ben Hutchings, Senior Software Engineer, Solarflare Communications
Not speaking for my employer; that's the marketing department's job.
maintainer
#5 lore raw

Re: [PATCH 3/4] sfc: resolve tx multiqueue bug

From: Ben Hutchings <[email protected]>
Date: Tue, 22 Jul 2008 19:41:19 +0100

Quoted 4 lines

So long as a queue is only woken in response to TX completions, the current arrangement is fine. However, sfc needs to tear down and restart hardware TX queues as part of some reconfiguration, self-test and recovery code, and I doubt it's the only such driver.

Mark the carrier off and packets will stop flowing to the driver.

Or, alternatively, take the necessary TX spinlocks around the reconfiguration.

Quoted 5 lines

I think the corresponding kernel queue must be stopped and then started when we do this (otherwise we could return NETDEV_TX_BUSY in hard_start_xmit, but I understand that to be deprecated). So we must also wake the kernel queue if and only if it's considered active - but we don't know whether that's the case because deactivation is done asynchronously.

See above.

#6 lore raw

Re: [PATCH 3/4] sfc: resolve tx multiqueue bug

Quoted 9 lines · David Miller

From: Ben Hutchings <[email protected]>
Date: Tue, 22 Jul 2008 19:41:19 +0100

Quoted 4 lines

So long as a queue is only woken in response to TX completions, the current arrangement is fine. However, sfc needs to tear down and restart hardware TX queues as part of some reconfiguration, self-test and recovery code, and I doubt it's the only such driver.

Mark the carrier off and packets will stop flowing to the driver.

Yep.

Though based on looking at a lot of existing driver code, I think this is sometimes unclear to driver writers -- when to manage carrier (netif_carrier_on/off) and when to manage queue flow (netif_queue_start/stop/wake).

Jeff

maintainer
#7 lore raw

Re: [PATCH 3/4] sfc: resolve tx multiqueue bug

From: Jeff Garzik <[email protected]>
Date: Tue, 22 Jul 2008 17:29:59 -0400

Quoted 9 lines
Quoted 8 lines · David Miller

From: Ben Hutchings <[email protected]>
Date: Tue, 22 Jul 2008 19:41:19 +0100

Quoted 4 lines

So long as a queue is only woken in response to TX completions, the current arrangement is fine. However, sfc needs to tear down and restart hardware TX queues as part of some reconfiguration, self-test and recovery code, and I doubt it's the only such driver.

Quoted 8 lines
Quoted 1 line

Mark the carrier off and packets will stop flowing to the driver.

Yep.

Though based on looking at a lot of existing driver code, I think this is sometimes unclear to driver writers -- when to manage carrier (netif_carrier_on/off) and when to manage queue flow (netif_queue_start/stop/wake).

This is true.

My current plan is to fix up as many drivers as possible, and assuming the reality that we can't expect to fix them all by 2.6.27 we will remove the WARN_ON_ONCE from __netif_schedule() right before 2.6.27-final then reinstate it for the 2.6.28 merge window.

#8 lore raw

PATCH 4/4 sfc: Create one RX queue and interrupt per CPU package by default

In reply to Ben Hutchings

Using multiple cores in the same package to handle received traffic does not appear to provide a performance benefit. Therefore use CPU topology information to count CPU packages and use that as the default number of RX queues and interrupts. We rely on interrupt balancing to spread the interrupts across packages.

---
 drivers/net/sfc/efx.c |   19 ++++++++++++++++++-
 1 files changed, 18 insertions(+), 1 deletions(-)
drivers/net/sfc/efx.c +18 -1
index 7b2a818..45c72ee 100644
@@ -19,6 +19,7 @@
 #include <linux/in.h>
 #include <linux/crc32.h>
 #include <linux/ethtool.h>
+#include <linux/topology.h>
 #include "net_driver.h"
 #include "gmii.h"
 #include "ethtool.h"
@@ -832,7 +833,23 @@ static void efx_probe_interrupts(struct efx_nic *efx)
 	if (efx->interrupt_mode == EFX_INT_MODE_MSIX) {
 		BUG_ON(!pci_find_capability(efx->pci_dev, PCI_CAP_ID_MSIX));
 
-		efx->rss_queues = rss_cpus ? rss_cpus : num_online_cpus();
+		if (rss_cpus == 0) {
+			cpumask_t core_mask;
+			int cpu;
+
+			cpus_clear(core_mask);
+			efx->rss_queues = 0;
+			for_each_online_cpu(cpu) {
+				if (!cpu_isset(cpu, core_mask)) {
+					++efx->rss_queues;
+					cpus_or(core_mask, core_mask,
+						topology_core_siblings(cpu));
+				}
+			}
+		} else {
+			efx->rss_queues = rss_cpus;
+		}
+
 		efx->rss_queues = min(efx->rss_queues, max_channel + 1);
 		efx->rss_queues = min(efx->rss_queues, EFX_MAX_CHANNELS);
 
Signature · 2 lines
Ben Hutchings, Senior Software Engineer, Solarflare Communications
Not speaking for my employer; that's the marketing department's job.
#9 lore raw
In reply to Ben Hutchings
Quoted 5 lines · Ben Hutchings

As recommended by Jean Delvare:
- Increase timeout to 50 ms
- Leave adapter class clear so that unwanted drivers do not probe our bus
- Use strlcpy() for name initialisation

Quoted 7 lines

---
Jeff,

These are exactly the differences between net-2.6 and the version of sfc I intended to submit to you before today. If this doesn't apply then I'm confused as to what you have.

applied patches 1, 2, 4