Skip to contentSkip to conversation

Multiple transmit/receive queue kernel

PATCH In review Kok, Auke 5 participants 17 messages Feb 8, 2007 – Mar 12, 2007

Subsystems (3)
lore
Share & tools
lore

Kok, Auke

author
#0 lore raw

cover Multiple transmit/receive queue kernel

Hi,

The following two patches are submitted for review. Please provide feedback and comments as usual.

This set of patches implements a generic API for multiqueue-capable devices to have network stack support to assign multiple flows into each queue on the NIC. It provides an interface for non-multiqueue devices to coexist in a machine running with both types of devices, as well as providing a modified queuing discipline to assign multiple flows in the stack to individual queues on the NIC. This is necessary to alleviate more head-of-line-blocking issues with different priority flows, where today QoS only prevents HOLB within the stack, but not in the device.

Some Intel PRO/1000 adapters support this hardware feature, so this series includes a patch to the e1000 driver to enable this hardware feature for those devices. You will need to setup the queue's using tc. Limited queue statistics are available through ethtool for devices that support multiple queues.

Regards,

Auke Kok
Peter Waskiewicz Jr.

---
These patches apply against commit b892afd1e60132a981b963929e352eabf3306ba2 of branch 'master' from torvalds/linux-2.6.

You can pull these patches through git:

git-pull git://lost.foo-projects.org/~ahkok/git/netdev-2.6 multiqueue

---
Summary:
Peter Waskiewicz Jr <[email protected]>:
NET: Multiple queue network device support
e1000: Implement the new kernel API for multiqueue TX support.

---
 drivers/net/Kconfig               |   13 +
 drivers/net/e1000/e1000.h         |   23 +++
 drivers/net/e1000/e1000_ethtool.c |   43 ++++++
 drivers/net/e1000/e1000_main.c    |  269 +++++++++++++++++++++++++++++++-------
 include/linux/netdevice.h         |   73 ++++++++++
 include/net/sch_generic.h         |    3 
 net/Kconfig                       |   20 ++
 net/core/dev.c                    |   51 +++++++
 net/sched/sch_generic.c           |  117 ++++++++++++++++
 net/sched/sch_prio.c              |  106 ++++++++++++++
 10 files changed, 668 insertions(+), 50 deletions(-)
author
#1 lore raw

PATCH 1/2 NET: Multiple queue network device support

From: Peter Waskiewicz Jr <[email protected]>

Added an API and associated supporting routines for multiqueue network devices. This allows network devices supporting multiple TX queues to configure each queue within the netdevice and manage each queue independantly. Changes to the PRIO Qdisc also allow a user to map multiple flows to individual TX queues, taking advantage of each queue on the device.

---

 include/linux/netdevice.h |   73 ++++++++++++++++++++++++++++
 include/net/sch_generic.h |    3 +
 net/Kconfig               |   20 ++++++++
 net/core/dev.c            |   51 ++++++++++++++++++++
 net/sched/sch_generic.c   |  117 +++++++++++++++++++++++++++++++++++++++++++++
 net/sched/sch_prio.c      |  106 ++++++++++++++++++++++++++++++++++++++---
 6 files changed, 364 insertions(+), 6 deletions(-)
include/linux/netdevice.h +73 -0
index 2e37f50..c7f94a8 100644
@@ -106,6 +106,16 @@ struct netpoll_info;
 #define MAX_HEADER (LL_MAX_HEADER + 48)
 #endif
 
+#ifdef CONFIG_NET_MULTI_QUEUE_DEVICE
+
+struct net_device_subqueue
+{
+	/* Give a lock and a flow control state for each queue */
+	unsigned long	state;
+	spinlock_t	queue_lock ____cacheline_aligned_in_smp;
+};
+#endif
+
 /*
  *	Network device statistics. Akin to the 2.0 ether stats but
  *	with byte counters.
@@ -339,6 +349,10 @@ struct net_device
 #define NETIF_F_GEN_CSUM	(NETIF_F_NO_CSUM | NETIF_F_HW_CSUM)
 #define NETIF_F_ALL_CSUM	(NETIF_F_IP_CSUM | NETIF_F_GEN_CSUM)
 
+#ifdef CONFIG_NET_MULTI_QUEUE_DEVICE
+#define NETIF_F_MULTI_QUEUE	8192	/* Has multiple TX/RX queues */
+#endif
+
 	struct net_device	*next_sched;
 
 	/* Interface index. Unique device identifier	*/
@@ -532,6 +546,19 @@ struct net_device
 	struct device		dev;
 	/* space for optional statistics and wireless sysfs groups */
 	struct attribute_group  *sysfs_groups[3];
+
+#ifdef CONFIG_NET_MULTI_QUEUE_DEVICE
+	/* To retrieve statistics per subqueue - FOR FUTURE USE */
+	struct net_device_stats* (*get_subqueue_stats)(struct net_device *dev,
+							int queue_index);
+
+	/* The TX queue control structures */
+	struct net_device_subqueue	*egress_subqueue;
+	int				egress_subqueue_count;
+	int			(*hard_start_subqueue_xmit)(struct sk_buff *skb,
+							struct net_device *dev,
+							int queue_index);
+#endif /* CONFIG_NET_MULTI_QUEUE_DEVICE */
 };
 #define to_net_dev(d) container_of(d, struct net_device, dev)
 
@@ -673,6 +700,52 @@ static inline int netif_running(const struct net_device *dev)
 	return test_bit(__LINK_STATE_START, &dev->state);
 }
 
+#ifdef CONFIG_NET_MULTI_QUEUE_DEVICE
+
+/*
+ * Routines to manage the subqueues on a device.  We only need start
+ * stop, and a check if it's stopped.  All other device management is
+ * done at the overall netdevice level.
+ * Also test the netif state if we're multi-queue at all.
+ */
+static inline void netif_start_subqueue(struct net_device *dev, int queue_index)
+{
+	clear_bit(__LINK_STATE_XOFF, &dev->egress_subqueue[queue_index].state);
+}
+
+static inline void netif_stop_subqueue(struct net_device *dev, int queue_index)
+{
+#ifdef CONFIG_NETPOLL_TRAP
+	if (netpoll_trap())
+		return;
+#endif
+	set_bit(__LINK_STATE_XOFF, &dev->egress_subqueue[queue_index].state);
+}
+
+static inline int netif_subqueue_stopped(const struct net_device *dev,
+                                         int queue_index)
+{
+	return test_bit(__LINK_STATE_XOFF,
+	                &dev->egress_subqueue[queue_index].state);
+}
+
+static inline void netif_wake_subqueue(struct net_device *dev, int queue_index)
+{
+#ifdef CONFIG_NETPOLL_TRAP
+	if (netpoll_trap())
+		return;
+#endif
+	if (test_and_clear_bit(__LINK_STATE_XOFF,
+	                       &dev->egress_subqueue[queue_index].state))
+		__netif_schedule(dev);
+}
+
+static inline int netif_is_multiqueue(const struct net_device *dev)
+{
+	return (!!(NETIF_F_MULTI_QUEUE & dev->features));
+}
+#endif /* CONFIG_NET_MULTI_QUEUE_DEVICE */
+
 
 /* Use this variant when it is known for sure that it
  * is executing from interrupt context.
include/net/sch_generic.h +3 -0
index 8208639..8751ba6 100644
@@ -102,6 +102,9 @@ struct Qdisc_ops
 
 	int			(*dump)(struct Qdisc *, struct sk_buff *);
 	int			(*dump_stats)(struct Qdisc *, struct gnet_dump *);
+#ifdef CONFIG_NET_MULTI_QUEUE_DEVICE
+	int			(*map_queue)(struct sk_buff *, struct Qdisc *);
+#endif
 
 	struct module		*owner;
 };
net/Kconfig +20 -0
index 7dfc949..796edb3 100644
@@ -38,6 +38,26 @@ source "net/packet/Kconfig"
 source "net/unix/Kconfig"
 source "net/xfrm/Kconfig"
 
+config NET_MULTI_QUEUE_DEVICE
+	bool "Multiple queue network device support (EXPERIMENTAL)"
+	depends on NET_SCHED && EXPERIMENTAL
+	help
+	  Saying Y here will add support for network devices that have more than
+	  one physical TX queue and/or RX queue.
+
+	  Multiple queue devices will require qdiscs that understand how to
+	  queue to multiple targets.  The default packet scheduler will default
+	  to the first queue in a device.  In other words, if you need the
+	  ability to spread traffic across queues, your queueing discipline
+	  needs to know how to do that.
+
+	  Note that saying Y here will give preferential treatment to multiple
+	  queue devices in the network stack.  A slight drop in single-queue
+	  device performance may be seen.
+
+	  Say N here unless you know your network device supports multiple
+	  TX and/or RX queues.
+
 config INET
 	bool "TCP/IP networking"
 	---help---
net/core/dev.c +51 -0

75 lines not on this page · raw message

net/sched/sch_generic.c +117 -0

196 lines not on this page · raw message

net/sched/sch_prio.c +100 -6

187 lines not on this page · raw message

---
Auke Kok <[email protected]>

maintainer
#2 lore raw

Re: [PATCH 1/2] NET: Multiple queue network device support

From: "Kok, Auke" <[email protected]>
Date: Thu, 08 Feb 2007 16:09:50 -0800

Quoted 11 lines

From: Peter Waskiewicz Jr <[email protected]>

Added an API and associated supporting routines for multiqueue network devices. This allows network devices supporting multiple TX queues to configure each queue within the netdevice and manage each queue independantly. Changes to the PRIO Qdisc also allow a user to map multiple flows to individual TX queues, taking advantage of each queue on the device.

Thanks for posting this.

I was wonding if it would be better to have the ->enqueue() perform the mapping operation, store the queue index selected inside of the skb, and just use the normal ->hard_start_xmit() to send the packet out?

The only thing to take care of is the per-queue locking, but that should be easily doable.

We might be able to do this even without making sk_buff any larger. For example, I suppose skb->priority might be deducable down to a u16. After a quick audit I really cannot see any legitimate use of anything larger than 16-bit values, but a more thorough audit would be necessary to validate this.

#3 lore raw

RE: [PATCH 1/2] NET: Multiple queue network device support

Dave,
Thanks for the feedback. Please see below for my comments /
questions.

PJ Waskiewicz
Intel Corporation

Quoted 15 lines
Quoted 3 lines

From: Peter Waskiewicz Jr <[email protected]>

Added an API and associated supporting routines for

multiqueue network

Quoted 1 line

devices. This allows network devices supporting multiple

TX queues to

Quoted 3 lines

configure each queue within the netdevice and manage each queue independantly. Changes to the PRIO Qdisc also allow a user to map multiple flows to individual TX queues, taking advantage of

each queue

Quoted 4 lines

on the device.

Quoted 6 lines

Thanks for posting this.

I was wonding if it would be better to have the ->enqueue() perform the mapping operation, store the queue index selected inside of the skb, and just use the normal ->hard_start_xmit() to send the packet out?

One of the reasons I chose to add more functions is for the queue management itself. Specifically, I needed the ability to lock queues, stop queues (netif_stop_queue), and wake them up, independent of the global queue_lock in the netdevice. At first I did think about storing the queue index in the skb, but that won't help with the queue locking, since only netdevice is passed to them outside of the context of the skb. As for adding the additional multiqueue version of hard_start_subqueue_xmit(), I did that so non-MQ devices would have a clean, untouched path in dev_queue_xmit(), and only MQ devices would touch a new codepath. This also removes the need for the device driver to inspect the queue index in the skb for which Tx ring to place the packet in, which is good for performance.

For having ->enqueue() take care of the mapping, I thought for awhile about this. Originally, I had ->enqueue() doing just that, until I realized I was not locking the individual subqueues correctly. The thought process was once I receive an skb for Tx, I need to lock a queue, enqueue the skb, call qdisc_run(), then unlock. That grew into lock a queue, enqueue, unlock the queue, have dequeue decide which queue to pull an skb from (since the queue an skb comes from can be different than the one just enqueued to), lock it, dequeue, unlock, and return the skb. The whole issue came down to needing to know what queue to lock before enqueuing. If I called enqueue, I would be going in unlocked. I suppose this would be ok if the lock is done inside the ->enqueue(), and unlock is done before exiting; let me look at that possibility.

Quoted 9 lines

The only thing to take care of is the per-queue locking, but that should be easily doable.

We might be able to do this even without making sk_buff any larger. For example, I suppose skb->priority might be deducable down to a u16. After a quick audit I really cannot see any legitimate use of anything larger than 16-bit values, but a more thorough audit would be necessary to validate this.

The skb->priority field right now is used to determine the PRIO / pfifo_fast band to place the skb in on enqueue. I still need to think about if storing the queue index in the skb would help, due to the queue management functions outside of the skb context.

Again, thanks for the feedback. I'll respond once I've looked at moving locks into ->enqueue() and removing ->map_queue(). Please note that the PRIO band to queue mapping in this patch has a problem when the number of bands is <= the number of Tx queues on the NIC. I have a fix, and will include that in a repost.

#4 lore raw

RE: [PATCH 1/2] NET: Multiple queue network device support

In reply to David Miller

Dave,
I did some research based on your feedback. Specifically, I
looked at removing ->map_queue() and allowing ->enqueue() to take care of mapping and locking of the queues (and ->dequeue()). I found that it can be done either way, but I'm not sure I like taking the locking out of dev_queue_xmit(), and relying on the qdisc ->enqueue() to make sure locking is completed correctly. Having a map routine also allows various parts of the stack to query the skb where it belongs.
I also looked into storing the mapped queue value in the skb,
namely in skb->priority. If I were to use this value, I'd lose filtering capabilities in sch_prio.c, where if no tc filter exists for the skb in question, it relies on the value of skb->priority to enqueue to a band. The value of skb->priority could also be used in a base driver for further filtering (some of the MAC-based QoS on wireless devices comes to mind), so I'm not sure I feel comfortable using that field to store the queue. I also envision some qdiscs in the future that could change the mapping of bands to queues without reloading the qdisc, so storing the queue information in the skb could lead to out-of-order packets into queues (stale information in the skb).
What do you think?

Thanks,

PJ Waskiewicz
Software Engineer
LAD - LAN Access Division, New Technologies
Intel Corporation
[email protected]

Quoted 43 lines

-----Original Message-----
From: David Miller [mailto:[email protected]]
Sent: Monday, February 26, 2007 5:03 PM
To: Kok, Auke-jan H
Cc: [email protected]; [email protected]; [email protected]; Waskiewicz Jr, Peter P; Brandeburg, Jesse; [email protected]; Ronciak, John
Subject: Re: [PATCH 1/2] NET: Multiple queue network device support

From: "Kok, Auke" <[email protected]>
Date: Thu, 08 Feb 2007 16:09:50 -0800

Quoted 3 lines

From: Peter Waskiewicz Jr <[email protected]>

Added an API and associated supporting routines for

multiqueue network

Quoted 1 line

devices. This allows network devices supporting multiple

TX queues to

Quoted 3 lines

configure each queue within the netdevice and manage each queue independantly. Changes to the PRIO Qdisc also allow a user to map multiple flows to individual TX queues, taking advantage of

each queue

Quoted 4 lines

on the device.

Thanks for posting this.

I was wonding if it would be better to have the ->enqueue() perform the mapping operation, store the queue index selected inside of the skb, and just use the normal ->hard_start_xmit() to send the packet out?

The only thing to take care of is the per-queue locking, but that should be easily doable.

We might be able to do this even without making sk_buff any larger. For example, I suppose skb->priority might be deducable down to a u16. After a quick audit I really cannot see any legitimate use of anything larger than 16-bit values, but a more thorough audit would be necessary to validate this.

maintainer
#5 lore raw

Re: [PATCH 1/2] NET: Multiple queue network device support

I didn't say to use skb->priority, I said to shrink skb->priority down to a u16 and then make another u16 which will store your queue mapping value.

#6 lore raw

Re: [PATCH 1/2] NET: Multiple queue network device support

Quoted 3 lines · David Miller

I didn't say to use skb->priority, I said to shrink skb->priority down to a u16 and then make another u16 which will store your queue mapping value.

Peter is right: this is fully used by schedulers (prio, CBQ, HTB, HFSC...) and would break users' scripts, so I wouldn't recommend, too.

Regards,
Jarek P.

#7 lore raw

Re: [PATCH 1/2] NET: Multiple queue network device support

In reply to Kok, Auke

* Kok, Auke <[email protected]> 2007-02-08 16:09

Quoted 15 lines
net/core/dev.c +7 -0
index 455d589..42b635c 100644
@@ -1477,6 +1477,49 @@ gso:
 	skb->tc_verd = SET_TC_AT(skb->tc_verd,AT_EGRESS);
 #endif
 	if (q->enqueue) {
+#ifdef CONFIG_NET_MULTI_QUEUE_DEVICE
+		int queue_index;
+		/* If we're a multi-queue device, get a queue index to lock */
+		if (netif_is_multiqueue(dev))
+		{
+			/* Get the queue index and lock it. */
+			if (likely(q->ops->map_queue)) {

net/core/dev.c gso: · line 1496

Quoted 10 lines
net/core/dev.c +10 -0
+				queue_index = q->ops->map_queue(skb, q);
+				spin_lock(&dev->egress_subqueue[queue_index].queue_lock);
+				rc = q->enqueue(skb, q);
+				/*
+				 * Unlock because the underlying qdisc
+				 * may queue and send a packet from a
+				 * different queue.
+				 */
+				spin_unlock(&dev->egress_subqueue[queue_index].queue_lock);
+				qdisc_run(dev);

I really dislike this asymmetric locking logic, while enqueue() is called with queue_lock held dequeue() is repsonsible to acquire the lock for qdisc_restart().

net/core/dev.c gso: · line 1505

Quoted 9 lines
net/core/dev.c +9 -0
+				rc = rc == NET_XMIT_BYPASS
+				           ? NET_XMIT_SUCCESS : rc;
+				goto out;
+			} else {
+				printk(KERN_CRIT "Device %s is "
+					"multiqueue, but map_queue is "
+					"undefined in the qdisc!!\n",
+					dev->name);
+				kfree_skb(skb);

Move this check to tc_modify_qdisc(), it's useless to check this for every packet being delivered.

Quoted 4 lines
net/core/dev.c +4 -0
+			}
+		} else {
+			/* We're not a multi-queue device. */
+			spin_lock(&dev->queue_lock);

net/core/dev.c gso: · line 1519

Quoted 10 lines
net/core/dev.c +10 -0
+			q = dev->qdisc;
+			if (q->enqueue) {
+				rc = q->enqueue(skb, q);
+				qdisc_run(dev);
+				spin_unlock(&dev->queue_lock);
+				rc = rc == NET_XMIT_BYPASS
+				           ? NET_XMIT_SUCCESS : rc;
+				goto out;
+			}
+			spin_unlock(&dev->queue_lock);

Please don't duplicate already existing code.

Quoted 5 lines
net/core/dev.c +1 -0
@@ -130,6 +140,72 @@ static inline int qdisc_restart(struct net_device *dev)
 		}
 
 		{
+#ifdef CONFIG_NET_MULTI_QUEUE_DEVICE

net/core/dev.c static inline int qdisc_restart(struct net_device *dev) · line 153

Quoted 10 lines
net/core/dev.c +10 -0
+			if (netif_is_multiqueue(dev)) {
+				if (likely(q->ops->map_queue)) {
+					queue_index = q->ops->map_queue(skb, q);
+				} else {
+					printk(KERN_CRIT "Device %s is "
+						"multiqueue, but map_queue is "
+						"undefined in the qdisc!!\n",
+						dev->name);
+					goto requeue;
+				}

Yet another condition completely useless for every transmission.

Quoted 14 lines
net/core/dev.c +14 -0
+				spin_unlock(&dev->egress_subqueue[queue_index].queue_lock);
+				/* Check top level device, and any sub-device */
+				if ((!netif_queue_stopped(dev)) &&
+				  (!netif_subqueue_stopped(dev, queue_index))) {
+					int ret;
+					ret = dev->hard_start_subqueue_xmit(skb, dev, queue_index);
+					if (ret == NETDEV_TX_OK) {
+						if (!nolock) {
+							netif_tx_unlock(dev);
+						}
+						return -1;
+					}
+					if (ret == NETDEV_TX_LOCKED && nolock) {
+						spin_lock(&dev->egress_subqueue[queue_index].queue_lock);

net/core/dev.c static inline int qdisc_restart(struct net_device *dev) · line 177

Quoted 10 lines
net/core/dev.c +10 -0
+						goto collision;
+					}
+				}
+				/* NETDEV_TX_BUSY - we need to requeue */
+				/* Release the driver */
+				if (!nolock) {
+					netif_tx_unlock(dev);
+				}
+				spin_lock(&dev->egress_subqueue[queue_index].queue_lock);
+				q = dev->qdisc;

This is identical to the existing logic except for the different lock, the additional to check subqueue state and a different hard_start_xmit call. Share the logic.

Quoted 19 lines
net/core/dev.c +19 -0
+			}
+			else
+			{
+				/* We're a single-queue device */
+				/* And release queue */
+				spin_unlock(&dev->queue_lock);
+				if (!netif_queue_stopped(dev)) {
+					int ret;
+
+					ret = dev->hard_start_xmit(skb, dev);
+					if (ret == NETDEV_TX_OK) {
+						if (!nolock) {
+							netif_tx_unlock(dev);
+						}
+						spin_lock(&dev->queue_lock);
+						return -1;
+					}
+					if (ret == NETDEV_TX_LOCKED && nolock) {
+						spin_lock(&dev->queue_lock);

net/core/dev.c static inline int qdisc_restart(struct net_device *dev) · line 206

Quoted 10 lines
net/core/dev.c +10 -0
+						goto collision;
+					}
+				}
+				/* NETDEV_TX_BUSY - we need to requeue */
+				/* Release the driver */
+				if (!nolock) {
+					netif_tx_unlock(dev);
+				}
+				spin_lock(&dev->queue_lock);
+				q = dev->qdisc;

Again, you just copied the existing code into a separate branch, fix the branching logic!

Quoted 6 lines
net/core/dev.c +2 -0
@@ -356,10 +454,18 @@ static struct sk_buff *pfifo_fast_dequeue(struct Qdisc* qdisc)
 	struct sk_buff_head *list = qdisc_priv(qdisc);
 
 	for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) {
+#ifdef CONFIG_NET_MULTI_QUEUE_DEVICE
+		if (netif_is_multiqueue(qdisc->dev))

net/core/dev.c static struct sk_buff *pfifo_fast_dequeue(struct Qdisc* qdisc) · line 468

Quoted 10 lines
net/core/dev.c +6 -0
+			spin_lock(&qdisc->dev->egress_subqueue[0].queue_lock);
+#endif
 		if (!skb_queue_empty(list + prio)) {
 			qdisc->q.qlen--;
 			return __qdisc_dequeue_head(qdisc, list + prio);
 		}
+#ifdef CONFIG_NET_MULTI_QUEUE_DEVICE
+		if (netif_is_multiqueue(qdisc->dev))
+			spin_unlock(&qdisc->dev->egress_subqueue[0].queue_lock);
+#endif

This lock/unlock for every band definitely screws performance.

Quoted 24 lines
net/core/dev.c +14 -0
 	}
 
 	return NULL;
@@ -141,18 +174,53 @@ prio_dequeue(struct Qdisc* sch)
 	struct sk_buff *skb;
 	struct prio_sched_data *q = qdisc_priv(sch);
 	int prio;
+#ifdef CONFIG_NET_MULTI_QUEUE_DEVICE
+	int queue;
+#endif
 	struct Qdisc *qdisc;
 
+	/*
+	 * If we're multiqueue, the basic approach is try the lock on each
+	 * queue.  If it's locked, either we're already dequeuing, or enqueue
+	 * is doing something.  Go to the next band if we're locked.  Once we
+	 * have a packet, unlock the queue.  NOTE: the underlying qdisc CANNOT
+	 * be a PRIO qdisc, otherwise we will deadlock.  FIXME
+	 */
 	for (prio = 0; prio < q->bands; prio++) {
+#ifdef CONFIG_NET_MULTI_QUEUE_DEVICE
+		if (netif_is_multiqueue(sch->dev)) {
+			queue = q->band2queue[prio];
+			if (spin_trylock(&sch->dev->egress_subqueue[queue].queue_lock)) {

net/core/dev.c prio_dequeue(struct Qdisc* sch) · line 203

Quoted 10 lines
net/core/dev.c +10 -0
+				qdisc = q->queues[prio];
+				skb = qdisc->dequeue(qdisc);
+				if (skb) {
+					sch->q.qlen--;
+					skb->priority = prio;
+					spin_unlock(&sch->dev->egress_subqueue[queue].queue_lock);
+					return skb;
+				}
+				spin_unlock(&sch->dev->egress_subqueue[queue].queue_lock);
+			}

Your modified qdisc_restart() expects the queue_lock to be locked, how can this work?

Quoted 24 lines
net/core/dev.c +12 -1
+		} else {
+			qdisc = q->queues[prio];
+			skb = qdisc->dequeue(qdisc);
+			if (skb) {
+				sch->q.qlen--;
+				skb->priority = prio;
+				return skb;
+			}
+		}
+#else
 		qdisc = q->queues[prio];
 		skb = qdisc->dequeue(qdisc);
 		if (skb) {
 			sch->q.qlen--;
+			skb->priority = prio;
 			return skb;
 		}
+#endif
 	}
 	return NULL;
-
 }
 
 static unsigned int prio_drop(struct Qdisc* sch)
#8 lore raw

RE: [PATCH 1/2] NET: Multiple queue network device support

Quoted 37 lines

-----Original Message-----
From: Thomas Graf [mailto:[email protected]]
Sent: Friday, March 09, 2007 5:41 AM
To: Kok, Auke-jan H
Cc: David Miller; Garzik, Jeff; [email protected]; [email protected]; Waskiewicz Jr, Peter P; Brandeburg, Jesse; Kok, Auke; Ronciak, John
Subject: Re: [PATCH 1/2] NET: Multiple queue network device support

* Kok, Auke <[email protected]> 2007-02-08 16:09

net/core/dev.c gso: · line 1482

Quoted 11 lines
net/core/dev.c index 455d589..42b635c renamed from net/core/dev.c +0 -0

100644

net/core/dev.c +3 -0
@@ -1477,6 +1477,49 @@ gso:
 	skb->tc_verd = SET_TC_AT(skb->tc_verd,AT_EGRESS);
 #endif
 	if (q->enqueue) {
+#ifdef CONFIG_NET_MULTI_QUEUE_DEVICE
+		int queue_index;
+		/* If we're a multi-queue device, get a queue 

index to lock */

net/core/dev.c gso: · line 1522

Quoted 6 lines
net/core/dev.c +6 -0
+		if (netif_is_multiqueue(dev))
+		{
+			/* Get the queue index and lock it. */
+			if (likely(q->ops->map_queue)) {
+				queue_index = q->ops->map_queue(skb, q);
+				

spin_lock(&dev->egress_subqueue[queue_index].queue_lock);

net/core/dev.c gso: · line 1522

Quoted 7 lines
net/core/dev.c +7 -0
+				rc = q->enqueue(skb, q);
+				/*
+				 * Unlock because the underlying qdisc
+				 * may queue and send a packet from a
+				 * different queue.
+				 */
+				

spin_unlock(&dev->egress_subqueue[queue_index].queue_lock);

Quoted 5 lines

net/core/dev.c gso: · line 1496

Quoted 1 line
net/core/dev.c +1 -0
+				qdisc_run(dev);

I really dislike this asymmetric locking logic, while enqueue() is called with queue_lock held dequeue() is repsonsible to acquire the lock for qdisc_restart().

I don't see how I can make this symmetrical when dealing with more than one queue. If an skb is classified to band 2 which maps to queue 1, we lock queue 1, and enqueue the packet. That queue will not be the first queue checked in dequeue for an skb, so I cannot rely on that lock protecting queue 0 in dequeue. Therefore I need to map the skb, lock, enqueue, unlock, then go into dequeue and lock the correct queue there. If I were to make this symmetrical, then the process of enqueuing and dequeuing would be serialized completely, and we'd be back to where we started without multiqueue.

Also, in the multiqueue code path, dev->queue_lock isn't held anywhere, it's the subqueue lock.

Quoted 13 lines

net/core/dev.c gso: · line 1505

Quoted 10 lines
net/core/dev.c +9 -0
+				rc = rc == NET_XMIT_BYPASS
+				           ? NET_XMIT_SUCCESS : rc;
+				goto out;
+			} else {
+				printk(KERN_CRIT "Device %s is "
+					"multiqueue, but map_queue is "
+					"undefined in the qdisc!!\n",
+					dev->name);
+				kfree_skb(skb);

Move this check to tc_modify_qdisc(), it's useless to check this for every packet being delivered.

The patches I sent yesterday modified the non-multiqueue qdiscs to not load if the device is multiqueue, so this check can be removed altogether.

Quoted 17 lines
Quoted 5 lines
net/core/dev.c +4 -0
+			}
+		} else {
+			/* We're not a multi-queue device. */
+			spin_lock(&dev->queue_lock);

net/core/dev.c gso: · line 1519

Quoted 10 lines
net/core/dev.c +10 -0
+			q = dev->qdisc;
+			if (q->enqueue) {
+				rc = q->enqueue(skb, q);
+				qdisc_run(dev);
+				spin_unlock(&dev->queue_lock);
+				rc = rc == NET_XMIT_BYPASS
+				           ? NET_XMIT_SUCCESS : rc;
+				goto out;
+			}
+			spin_unlock(&dev->queue_lock);

Please don't duplicate already existing code.

I don't want to mix multiqueue and non-multiqueue code in the transmit path. This was an attempt to allow MQ and non-MQ devices to coexist in a machine having separate code paths. Are you suggesting to combine them? That would get very messy trying to determine what type of lock to grab (subqueue lock or dev->queue_lock), not to mention grabbing the dev->queue_lock would block multiqueue devices in that same codepath.

Quoted 15 lines

net/core/dev.c static inline int qdisc_restart(struct

Quoted 2 lines
net/core/dev.c +0 -0
@@ -130,6 +140,72 @@ static inline int qdisc_restart(struct 

net_device *dev)

net/core/dev.c static inline int qdisc_restart(struct · line 146

Quoted 7 lines
net/core/dev.c +4 -0
 		}
 
 		{
+#ifdef CONFIG_NET_MULTI_QUEUE_DEVICE
+			if (netif_is_multiqueue(dev)) {
+				if (likely(q->ops->map_queue)) {
+					queue_index = 

q->ops->map_queue(skb, q);

net/core/dev.c gso:

Quoted 3 lines
net/core/dev.c +3 -0
+				} else {
+					printk(KERN_CRIT "Device %s is "
+						"multiqueue, 

but map_queue is "

Quoted 7 lines

net/core/dev.c static inline int qdisc_restart(struct · line 150

Quoted 1 line
net/core/dev.c +1 -0
+						"undefined in 

the qdisc!!\n",

net/core/dev.c static inline int qdisc_restart(struct · line 153

Quoted 3 lines
net/core/dev.c +3 -0
+						dev->name);
+					goto requeue;
+				}

Yet another condition completely useless for every transmission.

Yep, this will be removed. Thanks.

Quoted 33 lines

net/core/dev.c static inline int qdisc_restart(struct · line 154

Quoted 2 lines
net/core/dev.c +1 -0
+				

spin_unlock(&dev->egress_subqueue[queue_index].queue_lock);

net/core/dev.c static inline int qdisc_restart(struct · line 155

Quoted 1 line
net/core/dev.c +1 -0
+				/* Check top level device, and 

any sub-device */

net/core/dev.c static inline int qdisc_restart(struct · line 157

Quoted 2 lines
net/core/dev.c +2 -0
+				if ((!netif_queue_stopped(dev)) &&
+				  (!netif_subqueue_stopped(dev, 

queue_index))) {

net/core/dev.c static inline int qdisc_restart(struct · line 159

Quoted 2 lines
net/core/dev.c +2 -0
+					int ret;
+					ret = 

dev->hard_start_subqueue_xmit(skb, dev, queue_index);

net/core/dev.c static inline int qdisc_restart(struct · line 162

Quoted 3 lines
net/core/dev.c +3 -0
+					if (ret == NETDEV_TX_OK) {
+						if (!nolock) {
+							

netif_tx_unlock(dev);

net/core/dev.c static inline int qdisc_restart(struct · line 166

Quoted 4 lines
net/core/dev.c +4 -0
+						}
+						return -1;
+					}
+					if (ret == 

NETDEV_TX_LOCKED && nolock) {

net/core/dev.c static inline int qdisc_restart(struct · line 167

Quoted 1 line
net/core/dev.c +1 -0
+						

spin_lock(&dev->egress_subqueue[queue_index].queue_lock);

net/core/dev.c static inline int qdisc_restart(struct · line 171

Quoted 4 lines
net/core/dev.c +4 -0
+						goto collision;
+					}
+				}
+				/* NETDEV_TX_BUSY - we need to 

requeue */

net/core/dev.c static inline int qdisc_restart(struct · line 176

Quoted 5 lines
net/core/dev.c +5 -0
+				/* Release the driver */
+				if (!nolock) {
+					netif_tx_unlock(dev);
+				}
+				

spin_lock(&dev->egress_subqueue[queue_index].queue_lock);

Quoted 5 lines

net/core/dev.c static inline int qdisc_restart(struct · line 177

Quoted 1 line
net/core/dev.c +1 -0
+				q = dev->qdisc;

This is identical to the existing logic except for the different lock, the additional to check subqueue state and a different hard_start_xmit call. Share the logic.

Not necessarily. How we got here and how the locks are working are logically different due to the different queues. I'm working on storing the queue index in the skb now from Dave's suggestion, so there will be one hard_start_xmit, but the logic is different since the queue lock we're grabbing will be different than non-multiqueue. Perhaps I'm missing something here.

Quoted 30 lines

net/core/dev.c static inline int qdisc_restart(struct · line 187

Quoted 11 lines
net/core/dev.c +10 -0
+			}
+			else
+			{
+				/* We're a single-queue device */
+				/* And release queue */
+				spin_unlock(&dev->queue_lock);
+				if (!netif_queue_stopped(dev)) {
+					int ret;
+
+					ret = 

dev->hard_start_xmit(skb, dev);

net/core/dev.c static inline int qdisc_restart(struct · line 190

Quoted 3 lines
net/core/dev.c +3 -0
+					if (ret == NETDEV_TX_OK) {
+						if (!nolock) {
+							

netif_tx_unlock(dev);

net/core/dev.c static inline int qdisc_restart(struct · line 192

Quoted 2 lines
net/core/dev.c +2 -0
+						}
+						

spin_lock(&dev->queue_lock);

net/core/dev.c static inline int qdisc_restart(struct · line 195

Quoted 3 lines
net/core/dev.c +3 -0
+						return -1;
+					}
+					if (ret == 

NETDEV_TX_LOCKED && nolock) {

net/core/dev.c static inline int qdisc_restart(struct · line 196

Quoted 1 line
net/core/dev.c +1 -0
+						

spin_lock(&dev->queue_lock);

net/core/dev.c static inline int qdisc_restart(struct · line 200

Quoted 4 lines
net/core/dev.c +4 -0
+						goto collision;
+					}
+				}
+				/* NETDEV_TX_BUSY - we need to 

requeue */

Quoted 9 lines

net/core/dev.c static inline int qdisc_restart(struct · line 206

Quoted 6 lines
net/core/dev.c +6 -0
+				/* Release the driver */
+				if (!nolock) {
+					netif_tx_unlock(dev);
+				}
+				spin_lock(&dev->queue_lock);
+				q = dev->qdisc;

Again, you just copied the existing code into a separate branch, fix the branching logic!

This is another attempt to keep the two codepaths separate. The only way I see of combining them is to check netif_is_multiqueue() everytime I need to grab a lock, which I think would be excessive.

Quoted 19 lines

net/core/dev.c static struct sk_buff

Quoted 2 lines
net/core/dev.c +0 -0
@@ -356,10 +454,18 @@ static struct sk_buff 

*pfifo_fast_dequeue(struct Qdisc* qdisc)

net/core/dev.c static struct sk_buff · line 459

Quoted 6 lines
net/core/dev.c +3 -0
 	struct sk_buff_head *list = qdisc_priv(qdisc);
 
 	for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) {
+#ifdef CONFIG_NET_MULTI_QUEUE_DEVICE
+		if (netif_is_multiqueue(qdisc->dev))
+			

spin_lock(&qdisc->dev->egress_subqueue[0].queue_lock);

net/core/dev.c static struct sk_buff · line 467

Quoted 8 lines
net/core/dev.c +4 -0
+#endif
 		if (!skb_queue_empty(list + prio)) {
 			qdisc->q.qlen--;
 			return __qdisc_dequeue_head(qdisc, list + prio);
 		}
+#ifdef CONFIG_NET_MULTI_QUEUE_DEVICE
+		if (netif_is_multiqueue(qdisc->dev))
+			

spin_unlock(&qdisc->dev->egress_subqueue[0].queue_lock);

Quoted 4 lines

net/core/dev.c static struct sk_buff · line 468

Quoted 1 line
net/core/dev.c +1 -0
+#endif

This lock/unlock for every band definitely screws performance.

I will move the lock out of the for() loop since every band in pfifo_fast maps to queue 0. Thanks.

Quoted 36 lines
Quoted 4 lines
net/core/dev.c +0 -0
 	}
 
 	return NULL;

net/core/dev.c prio_dequeue(struct Qdisc* sch) · line 183

Quoted 10 lines
net/core/dev.c +5 -0
@@ -141,18 +174,53 @@ prio_dequeue(struct Qdisc* sch)
 	struct sk_buff *skb;
 	struct prio_sched_data *q = qdisc_priv(sch);
 	int prio;
+#ifdef CONFIG_NET_MULTI_QUEUE_DEVICE
+	int queue;
+#endif
 	struct Qdisc *qdisc;
 
+	/*
+	 * If we're multiqueue, the basic approach is try the 

lock on each

net/core/dev.c prio_dequeue(struct Qdisc* sch) · line 184

Quoted 1 line
net/core/dev.c +1 -0
+	 * queue.  If it's locked, either we're already 

dequeuing, or enqueue

net/core/dev.c prio_dequeue(struct Qdisc* sch) · line 185

Quoted 1 line
net/core/dev.c +1 -0
+	 * is doing something.  Go to the next band if we're 

locked. Once we

net/core/dev.c prio_dequeue(struct Qdisc* sch) · line 186

Quoted 1 line
net/core/dev.c +1 -0
+	 * have a packet, unlock the queue.  NOTE: the 

underlying qdisc CANNOT

net/core/dev.c prio_dequeue(struct Qdisc* sch) · line 193

Quoted 7 lines
net/core/dev.c +6 -0
+	 * be a PRIO qdisc, otherwise we will deadlock.  FIXME
+	 */
 	for (prio = 0; prio < q->bands; prio++) {
+#ifdef CONFIG_NET_MULTI_QUEUE_DEVICE
+		if (netif_is_multiqueue(sch->dev)) {
+			queue = q->band2queue[prio];
+			if 

(spin_trylock(&sch->dev->egress_subqueue[queue].queue_lock)) {

net/core/dev.c prio_dequeue(struct Qdisc* sch) · line 199

Quoted 6 lines
net/core/dev.c +6 -0
+				qdisc = q->queues[prio];
+				skb = qdisc->dequeue(qdisc);
+				if (skb) {
+					sch->q.qlen--;
+					skb->priority = prio;
+					

spin_unlock(&sch->dev->egress_subqueue[queue].queue_lock);

Quoted 8 lines

net/core/dev.c prio_dequeue(struct Qdisc* sch) · line 202

Quoted 3 lines
net/core/dev.c +3 -0
+					return skb;
+				}
+				

spin_unlock(&sch->dev->egress_subqueue[queue].queue_lock);

net/core/dev.c prio_dequeue(struct Qdisc* sch) · line 203

Quoted 1 line
net/core/dev.c +1 -0
+			}

Your modified qdisc_restart() expects the queue_lock to be locked, how can this work?

No, it doesn't expect the lock to be held. Because of the multiple queues, enqueueing and dequeueing are now asynchronous, since I can enqueue to queue 0 while dequeuing from queue 1. dev->queue_lock isn't held, so this can happen. Therefore the spin_trylock() is used in this dequeue because I don't want to wait for someone to finish with that queue in question (e.g. enqueue working), since that will block all other bands/queues after the band in question. So if the lock isn't available to grab, we move to the next band. If I were to wait for the lock, I'd serialize the enqueue/dequeue completely, and block other traffic flows in other queues waiting for the lock.

Quoted 26 lines
Quoted 25 lines
net/core/dev.c +12 -1
+		} else {
+			qdisc = q->queues[prio];
+			skb = qdisc->dequeue(qdisc);
+			if (skb) {
+				sch->q.qlen--;
+				skb->priority = prio;
+				return skb;
+			}
+		}
+#else
 		qdisc = q->queues[prio];
 		skb = qdisc->dequeue(qdisc);
 		if (skb) {
 			sch->q.qlen--;
+			skb->priority = prio;
 			return skb;
 		}
+#endif
 	}
 	return NULL;
-
 }
 
 static unsigned int prio_drop(struct Qdisc* sch)
#9 lore raw

Re: [PATCH 1/2] NET: Multiple queue network device support

* Waskiewicz Jr, Peter P <[email protected]> 2007-03-09 11:25

Quoted 17 lines
Quoted 8 lines
Quoted 8 lines
net/core/dev.c +8 -0
+			}
+		} else {
+			/* We're not a multi-queue device. */
+			spin_lock(&dev->queue_lock);
+			q = dev->qdisc;
+			if (q->enqueue) {
+				rc = q->enqueue(skb, q);
+				qdisc_run(dev);
Quoted 8 lines

net/core/dev.c gso: · line 1519

Quoted 6 lines
net/core/dev.c +6 -0
+				spin_unlock(&dev->queue_lock);
+				rc = rc == NET_XMIT_BYPASS
+				           ? NET_XMIT_SUCCESS : rc;
+				goto out;
+			}
+			spin_unlock(&dev->queue_lock);

Please don't duplicate already existing code.

Quoted 6 lines

I don't want to mix multiqueue and non-multiqueue code in the transmit path. This was an attempt to allow MQ and non-MQ devices to coexist in a machine having separate code paths. Are you suggesting to combine them? That would get very messy trying to determine what type of lock to grab (subqueue lock or dev->queue_lock), not to mention grabbing the dev->queue_lock would block multiqueue devices in that same codepath.

The piece of code I quoted above is the branch executed if multi queue is not enabled. The code you added is 100% identical to the already existing enqueue logic. Just execute the existing branch if multi queue is not enabled.

Quoted 3 lines

This is another attempt to keep the two codepaths separate. The only way I see of combining them is to check netif_is_multiqueue() everytime I need to grab a lock, which I think would be excessive.

The code added is 100% identical to the existing code, just be a little smarter on how you do the ifdefs.

Quoted 45 lines
Quoted 36 lines
Quoted 4 lines
net/sched/sch_generic.c +0 -0
 	}
 
 	return NULL;

net/sched/sch_generic.c prio_dequeue(struct Qdisc* sch) · line 183

Quoted 10 lines
net/sched/sch_generic.c +5 -0
@@ -141,18 +174,53 @@ prio_dequeue(struct Qdisc* sch)
 	struct sk_buff *skb;
 	struct prio_sched_data *q = qdisc_priv(sch);
 	int prio;
+#ifdef CONFIG_NET_MULTI_QUEUE_DEVICE
+	int queue;
+#endif
 	struct Qdisc *qdisc;
 
+	/*
+	 * If we're multiqueue, the basic approach is try the 

lock on each

Review prio_dequeue(struct Qdisc* sch) · line 184

Quoted 1 line
(quoted diff) +1 -0
+	 * queue.  If it's locked, either we're already 

dequeuing, or enqueue

Review prio_dequeue(struct Qdisc* sch) · line 185

Quoted 1 line
(quoted diff) +1 -0
+	 * is doing something.  Go to the next band if we're 

locked. Once we

Review prio_dequeue(struct Qdisc* sch) · line 186

Quoted 1 line
(quoted diff) +1 -0
+	 * have a packet, unlock the queue.  NOTE: the 

underlying qdisc CANNOT

net/sched/sch_prio.c prio_dequeue(struct Qdisc* sch)

Quoted 7 lines
net/sched/sch_prio.c +6 -0
+	 * be a PRIO qdisc, otherwise we will deadlock.  FIXME
+	 */
 	for (prio = 0; prio < q->bands; prio++) {
+#ifdef CONFIG_NET_MULTI_QUEUE_DEVICE
+		if (netif_is_multiqueue(sch->dev)) {
+			queue = q->band2queue[prio];
+			if 

(spin_trylock(&sch->dev->egress_subqueue[queue].queue_lock)) {

net/sched/sch_prio.c prio_dequeue(struct Qdisc* sch)

Quoted 6 lines
net/sched/sch_prio.c +6 -0
+				qdisc = q->queues[prio];
+				skb = qdisc->dequeue(qdisc);
+				if (skb) {
+					sch->q.qlen--;
+					skb->priority = prio;
+					

spin_unlock(&sch->dev->egress_subqueue[queue].queue_lock);

Quoted 8 lines

net/sched/sch_prio.c prio_dequeue(struct Qdisc* sch)

Quoted 3 lines
net/sched/sch_prio.c +3 -0
+					return skb;
+				}
+				

spin_unlock(&sch->dev->egress_subqueue[queue].queue_lock);

Review prio_dequeue(struct Qdisc* sch) · line 203

Quoted 1 line
(quoted diff) +1 -0
+			}

Your modified qdisc_restart() expects the queue_lock to be locked, how can this work?

Quoted 10 lines

No, it doesn't expect the lock to be held. Because of the multiple queues, enqueueing and dequeueing are now asynchronous, since I can enqueue to queue 0 while dequeuing from queue 1. dev->queue_lock isn't held, so this can happen. Therefore the spin_trylock() is used in this dequeue because I don't want to wait for someone to finish with that queue in question (e.g. enqueue working), since that will block all other bands/queues after the band in question. So if the lock isn't available to grab, we move to the next band. If I were to wait for the lock, I'd serialize the enqueue/dequeue completely, and block other traffic flows in other queues waiting for the lock.

The first thing you do in qdisc_restart() after dequeue()ing is unlock the sub queue lock. You explicitely unlock it before calling qdisc_run() so I assume dequeue() is expected to keep it locked. Something doesn't add up.

BTW, which lock serializes your write access to qdisc->q.qlen? It used to be dev->queue_lock but that is apparently not true for multi queue.

#10 lore raw

RE: [PATCH 1/2] NET: Multiple queue network device support

Quoted 30 lines

* Waskiewicz Jr, Peter P <[email protected]> 2007-03-09 11:25

Quoted 11 lines
Quoted 11 lines

net/core/dev.c gso:

Quoted 11 lines
net/core/dev.c +11 -0
+			}
+		} else {
+			/* We're not a multi-queue device. */
+			spin_lock(&dev->queue_lock);
+			q = dev->qdisc;
+			if (q->enqueue) {
+				rc = q->enqueue(skb, q);
+				qdisc_run(dev);
+				spin_unlock(&dev->queue_lock);
+				rc = rc == NET_XMIT_BYPASS
+				           ? 

NET_XMIT_SUCCESS : rc;

Quoted 7 lines
Quoted 5 lines

net/core/dev.c gso: · line 1514

Quoted 3 lines
net/core/dev.c +3 -0
+				goto out;
+			}
+			spin_unlock(&dev->queue_lock);

Please don't duplicate already existing code.

I don't want to mix multiqueue and non-multiqueue code in

the transmit

Quoted 1 line

path. This was an attempt to allow MQ and non-MQ devices

to coexist

Quoted 2 lines

in a machine having separate code paths. Are you suggesting to combine them? That would get very messy trying to

determine what type

Quoted 3 lines

of lock to grab (subqueue lock or dev->queue_lock), not to mention grabbing the
dev->queue_lock would block multiqueue devices in that same

Quoted 7 lines

codepath.

The piece of code I quoted above is the branch executed if multi queue is not enabled. The code you added is 100% identical to the already existing enqueue logic. Just execute the existing branch if multi queue is not enabled.

I totally agree this is identical code if multiqueue isn't enabled. However, when multiqueue is enabled, I don't want to make all network drivers implement the subqueue API just to be able to lock the queues. So the first thing I check is netif_is_multiqueue(dev), and if it *isn't* multiqueue, it will run the existing code. This way both non-multiqueue devices don't have to have any knowledge of the MQ API.

Quoted 7 lines
Quoted 1 line

This is another attempt to keep the two codepaths separate.

The only

Quoted 2 lines

way I see of combining them is to check netif_is_multiqueue() everytime I need to grab a lock, which I think would be excessive.

The code added is 100% identical to the existing code, just be a little smarter on how you do the ifdefs.

An example could be an on-board adapter isn't multiqueue, and an expansion card you have in your system is. If I handle multiqueue being on with just ifdef's, then I could use the expansion card, but not the on-board adapter as-is. The on-board driver would need to be updated to have 1 queue in the multiqueue context, which is what I tried to avoid.

Quoted 14 lines
Quoted 1 line
Quoted 1 line

Your modified qdisc_restart() expects the queue_lock to

be locked,

Quoted 6 lines
Quoted 1 line

how can this work?

No, it doesn't expect the lock to be held. Because of the multiple queues, enqueueing and dequeueing are now asynchronous, since I can enqueue to queue 0 while dequeuing from queue 1. dev->queue_lock isn't held, so this can happen. Therefore the

spin_trylock() is used

Quoted 4 lines

in this dequeue because I don't want to wait for someone to finish with that queue in question (e.g. enqueue working), since that will block all other bands/queues after the band in question. So if the lock isn't available to grab, we move to the next band. If

I were to

Quoted 8 lines
Quoted 1 line

wait for the lock, I'd serialize the enqueue/dequeue

completely, and

Quoted 1 line

block other traffic flows in other queues waiting for the lock.

The first thing you do in qdisc_restart() after dequeue()ing is unlock the sub queue lock. You explicitely unlock it before calling qdisc_run() so I assume dequeue() is expected to keep it locked. Something doesn't add up.

That's the entire point of this extra locking. enqueue() is going to put an skb into a band somewhere that maps to some queue, and there is no way to guarantee the skb I retrieve from dequeue() is headed for the same queue. Therefore, I need to unlock the queue after I finish enqueuing, since having that lock makes little sense to dequeue(). dequeue() will then grab *a* lock on a queue; it may be the same one we had during enqueue(), but it may not be. And the placement of the unlock of that queue is exactly where it happens in non-multiqueue, which is right before the hard_start_xmit().

I concede that the locking model is complex and seems really nasty, but to truly separate queue functionality from one another, I see no other feasible option than to run locking like this. I am very open to suggestions.

Quoted 5 lines

BTW, which lock serializes your write access to qdisc->q.qlen? It used to be dev->queue_lock but that is apparently not true for multi queue.

This is a very good catch, because it isn't being protected on the entire qdisc right now for PRIO. However, Chris Leech pointed out the LINK_STATE_QDISC_RUNNING bit is serializing things at the qdisc_run() level, so that's probably protecting this counter right now. I'm looking at pushing that down into the qdisc, but at the same time I can think of something to serialize these stats containers for the qdisc.

#11 lore raw

Re: [PATCH 1/2] NET: Multiple queue network device support

* Waskiewicz Jr, Peter P <[email protected]> 2007-03-09 15:27

Quoted 9 lines

That's the entire point of this extra locking. enqueue() is going to put an skb into a band somewhere that maps to some queue, and there is no way to guarantee the skb I retrieve from dequeue() is headed for the same queue. Therefore, I need to unlock the queue after I finish enqueuing, since having that lock makes little sense to dequeue(). dequeue() will then grab *a* lock on a queue; it may be the same one we had during enqueue(), but it may not be. And the placement of the unlock of that queue is exactly where it happens in non-multiqueue, which is right before the hard_start_xmit().

The lock is already unlocked after dequeue, from your prio_dequeue():

       if (netif_is_multiqueue(sch->dev)) {
               queue = q->band2queue[prio];
               if (spin_trylock(&sch->dev->egress_subqueue[queue].queue_lock)) {
                       qdisc = q->queues[prio];
                       skb = qdisc->dequeue(qdisc);
                       if (skb) {
                               sch->q.qlen--;
                               skb->priority = prio;
                               spin_unlock(&sch->dev->egress_subqueue[queue].queue_lock);
                               return skb;
                       }
                       spin_unlock(&sch->dev->egress_subqueue[queue].queue_lock);
       }
#12 lore raw

RE: [PATCH 1/2] NET: Multiple queue network device support

Quoted 38 lines

-----Original Message-----
From: Thomas Graf [mailto:[email protected]]
Sent: Friday, March 09, 2007 6:35 PM
To: Waskiewicz Jr, Peter P
Cc: Kok, Auke-jan H; David Miller; Garzik, Jeff; [email protected]; [email protected]; Brandeburg, Jesse; Kok, Auke; Ronciak, John
Subject: Re: [PATCH 1/2] NET: Multiple queue network device support

* Waskiewicz Jr, Peter P <[email protected]> 2007-03-09 15:27

Quoted 1 line

That's the entire point of this extra locking. enqueue()

is going to

Quoted 1 line

put an skb into a band somewhere that maps to some queue,

and there is

Quoted 2 lines

no way to guarantee the skb I retrieve from dequeue() is headed for the same queue. Therefore, I need to unlock the queue

after I finish

Quoted 2 lines

enqueuing, since having that lock makes little sense to dequeue(). dequeue() will then grab *a* lock on a queue; it may be the

same one

Quoted 1 line

we had during enqueue(), but it may not be. And the

placement of the

Quoted 2 lines

unlock of that queue is exactly where it happens in non-multiqueue, which is right before the hard_start_xmit().

The lock is already unlocked after dequeue, from your prio_dequeue():

       if (netif_is_multiqueue(sch->dev)) {
               queue = q->band2queue[prio];
               if 
(spin_trylock(&sch->dev->egress_subqueue[queue].queue_lock)) {
                       qdisc = q->queues[prio];
                       skb = qdisc->dequeue(qdisc);
                       if (skb) {
                               sch->q.qlen--;
                               skb->priority = prio;
Quoted 6 lines
spin_unlock(&sch->dev->egress_subqueue[queue].queue_lock);
                               return skb;
                       }
spin_unlock(&sch->dev->egress_subqueue[queue].queue_lock);
       }

Ok, now I see what's wrong. Taking Dave M.'s recommendation to store the queue mapping in the skb will let me unlock the queue when dequeue() returns. I'll fix this locking issue; thanks for the feedback and persistent drilling into my thick head.

-PJ Waskiewicz
[email protected]

#13 lore raw

Re: [PATCH 1/2] NET: Multiple queue network device support

In reply to Thomas Graf
Quoted 11 lines · Thomas Graf

* Kok, Auke <[email protected]> 2007-02-08 16:09

Quoted 10 lines
net/core/dev.c +2 -0
index 455d589..42b635c 100644
@@ -1477,6 +1477,49 @@ gso:
 	skb->tc_verd = SET_TC_AT(skb->tc_verd,AT_EGRESS);
 #endif
 	if (q->enqueue) {
+#ifdef CONFIG_NET_MULTI_QUEUE_DEVICE
+		int queue_index;
Quoted 8 lines

net/core/dev.c gso: · line 1489

Quoted 8 lines
net/core/dev.c +8 -0
+		/* If we're a multi-queue device, get a queue index to lock */
+		if (netif_is_multiqueue(dev))
+		{
+			/* Get the queue index and lock it. */
+			if (likely(q->ops->map_queue)) {
+				queue_index = q->ops->map_queue(skb, q);
+				spin_lock(&dev->egress_subqueue[queue_index].queue_lock);
+				rc = q->enqueue(skb, q);

I'm not sure Dave Miller thought about this place, when he proposed to save the mapping, but I think this could be not enough. This place is racy: ->map_queue() is called 2 times and with some filters (and policies/actions) results could differ. And of course the subqueue lock doesn't prevent any filter from a config change in the meantime.

After second reading of this patch I have doubts it's the proper way to solve the problem: there are many subqueues but we need a top queue (prio here) to mange them, anyway. So, why not to build this functionality directly into the queue? There is no difference for a device if skbs are going from the subqueue or a class, it is only interested in the mapping result and a possibility to stop and start a subqueue and to query its status. All this could be done by adding the callbacks directly to any classful scheduler or, if not enough, to write some specialized qdisc based on prio. The possibility to lock only a subqueue instead of a queue could be analized independently - current proposal doesn't solve this anyway.

Regards,
Jarek P.

#14 lore raw

RE: [PATCH 1/2] NET: Multiple queue network device support

Quoted 56 lines

-----Original Message-----
From: Jarek Poplawski [mailto:[email protected]]
Sent: Monday, March 12, 2007 1:58 AM
To: Thomas Graf
Cc: Kok, Auke-jan H; David Miller; Garzik, Jeff; [email protected]; [email protected]; Waskiewicz Jr, Peter P; Brandeburg, Jesse; Kok, Auke; Ronciak, John
Subject: Re: [PATCH 1/2] NET: Multiple queue network device support

Quoted 2 lines · Thomas Graf

* Kok, Auke <[email protected]> 2007-02-08 16:09

net/core/dev.c index

Quoted 1 line
net/core/dev.c index renamed from net/core/dev.c +0 -0

455d589..42b635c

Quoted 10 lines

net/core/dev.c gso: · line 1482

Quoted 10 lines

100644

net/core/dev.c +3 -0
@@ -1477,6 +1477,49 @@ gso:
 	skb->tc_verd = SET_TC_AT(skb->tc_verd,AT_EGRESS);
 #endif
 	if (q->enqueue) {
+#ifdef CONFIG_NET_MULTI_QUEUE_DEVICE
+		int queue_index;
+		/* If we're a multi-queue device, get a queue 

index to lock */

Quoted 6 lines

net/core/dev.c gso: · line 1522

Quoted 6 lines
net/core/dev.c +6 -0
+		if (netif_is_multiqueue(dev))
+		{
+			/* Get the queue index and lock it. */
+			if (likely(q->ops->map_queue)) {
+				queue_index = q->ops->map_queue(skb, q);
+				

spin_lock(&dev->egress_subqueue[queue_index].queue_lock);

Quoted 1 line

net/core/dev.c gso: · line 1489

Quoted 1 line
net/core/dev.c +1 -0
+				rc = q->enqueue(skb, q);

I'm not sure Dave Miller thought about this place, when he proposed to save the mapping, but I think this could be not enough. This place is racy: ->map_queue() is called 2 times and with some filters (and
policies/actions) results could differ. And of course the subqueue lock doesn't prevent any filter from a config change in the meantime.

After second reading of this patch I have doubts it's the proper way to solve the problem: there are many subqueues but we need a top queue (prio here) to mange them, anyway. So, why not to build this functionality directly into the queue? There is no difference for a device if skbs are going from the subqueue or a class, it is only interested in the mapping result and a possibility to stop and start a subqueue and to query its status. All this could be done by adding the callbacks directly to any classful scheduler or, if not enough, to write some specialized qdisc based on prio. The possibility to lock only a subqueue instead of a queue could be analized independently - current proposal doesn't solve this anyway.

Quoted 3 lines

Regards,
Jarek P.

Thanks again for the feedback. Given some discussions I had last week in the office and the general feedback here, I'm going to remove the new per-queue locking and leave the start/stop functions for each queue and combine entry points for hard_start_xmit(). I'll get this out asap for review once it's been tested here. If we see issues in the future with lock contention on the queues, we can revisit the per-queue locking.

Cheers,
-PJ Waskiewicz

author
#15 lore raw

PATCH 2/2 e1000: Implement the new kernel API for multiqueue TX support.

In reply to Kok, Auke

From: Peter Waskiewicz Jr. <[email protected]>

Several newer e1000 chipsets support multiple RX and TX queues. Most commonly, 82571's and ESB2LAN support 2 rx and 2 rx queues.

---

 drivers/net/Kconfig               |   13 ++
 drivers/net/e1000/e1000.h         |   23 +++
 drivers/net/e1000/e1000_ethtool.c |   43 ++++++
 drivers/net/e1000/e1000_main.c    |  269 +++++++++++++++++++++++++++++++------
 4 files changed, 304 insertions(+), 44 deletions(-)
drivers/net/Kconfig +13 -0
index ad92b6a..2d758ab 100644
@@ -1988,6 +1988,19 @@ config E1000_DISABLE_PACKET_SPLIT
 
 	  If in doubt, say N.
 
+config E1000_MQ
+	bool "Enable Tx/Rx Multiqueue Support (EXPERIMENTAL)"
+	depends on E1000 && NET_MULTI_QUEUE_DEVICE && EXPERIMENTAL
+	help
+	  Say Y here if you want to enable multiqueue support for supported
+	  e1000 devices.  This will enable both transmit and receive queues
+	  on devices that support them.
+
+	  In order to fully utilize the Tx queue support, use the SCH_PRIO
+	  queueing discipline.
+
+	  If in doubt, say N.
+
 source "drivers/net/ixp2000/Kconfig"
 
 config MYRI_SBUS
drivers/net/e1000/e1000.h +23 -0
index 689f158..cfcdc9d 100644
@@ -108,6 +108,10 @@ struct e1000_adapter;
 #define E1000_MIN_RXD                       80
 #define E1000_MAX_82544_RXD               4096
 
+#ifdef CONFIG_E1000_MQ
+#define E1000_MAX_TX_QUEUES                  4
+#endif
+
 /* this is the size past which hardware will drop packets when setting LPE=0 */
 #define MAXIMUM_ETHERNET_VLAN_SIZE 1522
 
@@ -168,6 +172,12 @@ struct e1000_buffer {
 	uint16_t next_to_watch;
 };
 
+#ifdef CONFIG_E1000_MQ
+struct e1000_queue_stats {
+	uint64_t packets;
+	uint64_t bytes;
+};
+#endif
 
 struct e1000_ps_page { struct page *ps_page[PS_PAGE_BUFFERS]; };
 struct e1000_ps_page_dma { uint64_t ps_page_dma[PS_PAGE_BUFFERS]; };
@@ -188,9 +198,16 @@ struct e1000_tx_ring {
 	/* array of buffer information structs */
 	struct e1000_buffer *buffer_info;
 
+#ifdef CONFIG_E1000_MQ
+	/* for tx ring cleanup - needed for multiqueue */
+	spinlock_t tx_queue_lock;
+#endif
 	spinlock_t tx_lock;
 	uint16_t tdh;
 	uint16_t tdt;
+#ifdef CONFIG_E1000_MQ
+	struct e1000_queue_stats tx_stats;
+#endif
 	boolean_t last_tx_tso;
 };
 
@@ -218,6 +235,9 @@ struct e1000_rx_ring {
 
 	uint16_t rdh;
 	uint16_t rdt;
+#ifdef CONFIG_E1000_MQ
+	struct e1000_queue_stats rx_stats;
+#endif
 };
 
 #define E1000_DESC_UNUSED(R) \
@@ -271,6 +291,9 @@ struct e1000_adapter {
 
 	/* TX */
 	struct e1000_tx_ring *tx_ring;      /* One per active queue */
+#ifdef CONFIG_E1000_MQ
+	struct e1000_tx_ring **cpu_tx_ring; /* per-cpu */
+#endif
 	unsigned int restart_queue;
 	unsigned long tx_queue_len;
 	uint32_t txd_cmd;
drivers/net/e1000/e1000_ethtool.c +43 -0

74 lines not on this page · raw message

drivers/net/e1000/e1000_main.c +225 -44

436 lines not on this page · raw message

---
Auke Kok <[email protected]>

#16 lore raw

Re: [PATCH 2/2] e1000: Implement the new kernel API for multiqueue TX support.

* Kok, Auke <[email protected]> 2007-02-08 16:09

Quoted 21 lines

From: Peter Waskiewicz Jr. <[email protected]>

Several newer e1000 chipsets support multiple RX and TX queues. Most commonly, 82571's and ESB2LAN support 2 rx and 2 rx queues.

---

 drivers/net/Kconfig               |   13 ++
 drivers/net/e1000/e1000.h         |   23 +++
 drivers/net/e1000/e1000_ethtool.c |   43 ++++++
 drivers/net/e1000/e1000_main.c    |  269 +++++++++++++++++++++++++++++++------
 4 files changed, 304 insertions(+), 44 deletions(-)
drivers/net/Kconfig +0 -0
index ad92b6a..2d758ab 100644

drivers/net/Kconfig config E1000_DISABLE_PACKET_SPLIT · line 1993

Quoted 6 lines
drivers/net/Kconfig +3 -0
@@ -1988,6 +1988,19 @@ config E1000_DISABLE_PACKET_SPLIT
 
 	  If in doubt, say N.
 
+config E1000_MQ
+	bool "Enable Tx/Rx Multiqueue Support (EXPERIMENTAL)"
+	depends on E1000 && NET_MULTI_QUEUE_DEVICE && EXPERIMENTAL

Would be better to just select NET_MULTI_QUEUE_DEVICE instead of depending on it.