ixgbe: Introduce MSI-X queue vector code

PATCH In review Auke Kok 2 participants 6 messages Mar 3, 2008 – Mar 5, 2008

Subsystems (2)

Networking Network drivers

lore

Share & tools

lore

Auke Kok

author

2008-03-03 23:03 UTC #0 lore raw

PATCH 1/5 ixgbe: Introduce MSI-X queue vector code

From: Ayyappan Veeraiyan <[email protected]>

This code abstracts the per-queue MSI-X interrupt vector into a queue vector layer. This abstraction is needed since there can be many more queues than available MSI-X vectors in a machine.

The MSI-X irq vectors are remapped to a shared queue vector which can point to several (both RX and TX) hardware queues. The NAPI algorithm then cleans the appropriate ring/queues on interrupt or poll.

The remapping is a delicate and complex calculation to make sure that we're not unbalancing the irq load, and spreads the irqs as much as possible, and may combine RX and TX flows onto the same queue vector.

This effectively enables receive flow hashing across vectors and helps irq load balance across CPUs.

Signed-off-by Ayyappan Veeraiyan <[email protected]>

Signed-off-by Auke Kok <[email protected]>

Acked-by Jesse Brandeburg <[email protected]>

Acked-by Waskiewicz Jr, Peter P <[email protected]>

---

 drivers/net/ixgbe/ixgbe.h      |   69 ++
 drivers/net/ixgbe/ixgbe_main.c | 1107 ++++++++++++++++++++++++++++------------
 2 files changed, 826 insertions(+), 350 deletions(-)

drivers/net/ixgbe/ixgbe.h +58 ~~-11~~

index d0bf206..2077477 100644
@@ -120,7 +120,6 @@ struct ixgbe_queue_stats {
 };
 
 struct ixgbe_ring {
-	struct ixgbe_adapter *adapter;	/* backlink */
 	void *desc;			/* descriptor ring memory */
 	dma_addr_t dma;			/* phys. address of descriptor ring */
 	unsigned int size;		/* length in bytes */

@@ -128,6 +127,7 @@ struct ixgbe_ring {
 	unsigned int next_to_use;
 	unsigned int next_to_clean;
 
+	int queue_index; /* needed for multiqueue queue management */
 	union {
 		struct ixgbe_tx_buffer *tx_buffer_info;
 		struct ixgbe_rx_buffer *rx_buffer_info;

@@ -137,7 +137,13 @@ struct ixgbe_ring {
 	u16 tail;
 
 
+	u16 reg_idx; /* holds the special value that gets the hardware register
+		      * offset associated with this ring, which is different
+		      * for DCE and RSS modes */
 	struct ixgbe_queue_stats stats;
+	u8 v_idx; /* maps directly to the index for this ring in the hardware
+		   * vector array, can also be used for finding the bit in EICR
+		   * and friends that represents the vector for this ring */
 
 	u32 eims_value;
 	u16 itr_register;

@@ -146,6 +152,31 @@ struct ixgbe_ring {
 	u16 work_limit;                /* max work per interrupt */
 };
 
+#define RING_F_VMDQ 1
+#define RING_F_RSS  2
+#define IXGBE_MAX_RSS_INDICES  16
+#define IXGBE_MAX_VMDQ_INDICES 16
+struct ixgbe_ring_feature {
+	int indices;
+	int mask;
+};
+
+#define MAX_RX_QUEUES 64
+#define MAX_TX_QUEUES 32
+
+/* MAX_MSIX_Q_VECTORS of these are allocated,
+ * but we only use one per queue-specific vector.
+ */
+struct ixgbe_q_vector {
+	struct ixgbe_adapter *adapter;
+	struct napi_struct napi;
+	DECLARE_BITMAP(rxr_idx, MAX_RX_QUEUES); /* Rx ring indices */
+	DECLARE_BITMAP(txr_idx, MAX_TX_QUEUES); /* Tx ring indices */
+	u8 rxr_count;     /* Rx ring count assigned to this vector */
+	u8 txr_count;     /* Tx ring count assigned to this vector */
+	u32 eitr;
+};
+
 /* Helper macros to switch between ints/sec and what the register uses.
  * And yes, it's the same math going both ways.
  */

@@ -166,6 +197,14 @@ struct ixgbe_ring {
 
 #define IXGBE_MAX_JUMBO_FRAME_SIZE        16128
 
+#define OTHER_VECTOR 1
+#define NON_Q_VECTORS (OTHER_VECTOR)
+
+#define MAX_MSIX_Q_VECTORS 16
+#define MIN_MSIX_Q_VECTORS 2
+#define MAX_MSIX_COUNT (MAX_MSIX_Q_VECTORS + NON_Q_VECTORS)
+#define MIN_MSIX_COUNT (MIN_MSIX_Q_VECTORS + NON_Q_VECTORS)
+
 /* board specific private data structure */
 struct ixgbe_adapter {
 	struct timer_list watchdog_timer;

@@ -173,10 +212,11 @@ struct ixgbe_adapter {
 	u16 bd_number;
 	u16 rx_buf_len;
 	struct work_struct reset_task;
+	struct ixgbe_q_vector q_vector[MAX_MSIX_Q_VECTORS];
+	char name[MAX_MSIX_COUNT][IFNAMSIZ + 5];
 
 	/* TX */
 	struct ixgbe_ring *tx_ring;	/* One per active queue */
-	struct napi_struct napi;
 	u64 restart_queue;
 	u64 lsc_int;
 	u64 hw_tso_ctxt;

@@ -192,22 +232,26 @@ struct ixgbe_adapter {
 	u64 non_eop_descs;
 	int num_tx_queues;
 	int num_rx_queues;
+	int num_msix_vectors;
+	struct ixgbe_ring_feature ring_feature[3];
 	struct msix_entry *msix_entries;
 
 	u64 rx_hdr_split;
 	u32 alloc_rx_page_failed;
 	u32 alloc_rx_buff_failed;
 
+	/* Some features need tri-state capability,
+	 * thus the additional *_CAPABLE flags.
+	 */
 	u32 flags;
-#define IXGBE_FLAG_RX_CSUM_ENABLED              (u32)(1)
+#define IXGBE_FLAG_RX_CSUM_ENABLED              (u32)(1 << 0)
 #define IXGBE_FLAG_MSI_ENABLED                  (u32)(1 << 1)
-#define IXGBE_FLAG_MSIX_ENABLED			(u32)(1 << 2)
-#define IXGBE_FLAG_RX_PS_ENABLED		(u32)(1 << 3)
-#define IXGBE_FLAG_IN_NETPOLL			(u32)(1 << 4)
-
-	/* Interrupt Throttle Rate */
-	u32 rx_eitr;
-	u32 tx_eitr;
+#define IXGBE_FLAG_MSIX_ENABLED                 (u32)(1 << 2)
+#define IXGBE_FLAG_RX_PS_ENABLED                (u32)(1 << 3)
+#define IXGBE_FLAG_IN_NETPOLL                   (u32)(1 << 4)
+#define IXGBE_FLAG_IMIR_ENABLED                 (u32)(1 << 5)
+#define IXGBE_FLAG_RSS_ENABLED                  (u32)(1 << 6)
+#define IXGBE_FLAG_VMDQ_ENABLED                 (u32)(1 << 7)
 
 	/* OS defined structs */
 	struct net_device *netdev;

@@ -218,7 +262,10 @@ struct ixgbe_adapter {
 	struct ixgbe_hw hw;
 	u16 msg_enable;
 	struct ixgbe_hw_stats stats;
-	char lsc_name[IFNAMSIZ + 5];
+
+	/* Interrupt Throttle Rate */
+	u32 rx_eitr;
+	u32 tx_eitr;
 
 	unsigned long state;
 	u64 tx_busy;

drivers/net/ixgbe/ixgbe_main.c +768 ~~-339~~

1,557 lines not on this page · raw message

-------------------------------------------------------------------------
This SF.net email is sponsored by: Microsoft Defy all challenges. Microsoft(R) Visual Studio 2008.
http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/

Auke Kok

author

2008-03-03 23:03 UTC #1 lore raw

PATCH 2/5 ixgbe: Introduce Multiqueue TX

From: Ayyappan Veeraiyan <[email protected]>

Now that the irq vector code is in place, we can add the conditional multiqueue TX code in the driver. This requires the optional CONFIG_NETDEVICES_MULTIQUEUE=y and will not be enabled without it.

Signed-off-by Ayyappan Veeraiyan <[email protected]>

Signed-off-by Auke Kok <[email protected]>

Acked-by Waskiewicz Jr, Peter P <[email protected]>

---

 drivers/net/ixgbe/ixgbe_ethtool.c |   15 +++++++++-
 drivers/net/ixgbe/ixgbe_main.c    |   56 ++++++++++++++++++++++++++++++++++---
 2 files changed, 66 insertions(+), 5 deletions(-)

drivers/net/ixgbe/ixgbe_ethtool.c +14 -1

index a119cbd..85b7d15 100644
@@ -246,13 +246,26 @@ static int ixgbe_set_tx_csum(struct net_device *netdev, u32 data)
 
 static int ixgbe_set_tso(struct net_device *netdev, u32 data)
 {
-
 	if (data) {
 		netdev->features |= NETIF_F_TSO;
 		netdev->features |= NETIF_F_TSO6;
 	} else {
+#ifdef CONFIG_NETDEVICES_MULTIQUEUE
+		struct ixgbe_adapter *adapter = netdev_priv(netdev);
+		int i;
+#endif
+		netif_stop_queue(netdev);
+#ifdef CONFIG_NETDEVICES_MULTIQUEUE
+		for (i = 0; i < adapter->num_tx_queues; i++)
+			netif_stop_subqueue(netdev, i);
+#endif
 		netdev->features &= ~NETIF_F_TSO;
 		netdev->features &= ~NETIF_F_TSO6;
+#ifdef CONFIG_NETDEVICES_MULTIQUEUE
+		for (i = 0; i < adapter->num_tx_queues; i++)
+			netif_start_subqueue(netdev, i);
+#endif
+		netif_start_queue(netdev);
 	}
 	return 0;
 }

drivers/net/ixgbe/ixgbe_main.c +52 -4

148 lines not on this page · raw message

Auke Kok

author

2008-03-03 23:06 UTC #2 lore raw

PATCH 3/5 ixgbe: Introduce adaptive interrupt moderation

In reply to Auke Kok

From: Ayyappan Veeraiyan <[email protected]>

82598 can produce a formidable interrupt rate, and is largely unusable without some form of moderation. The default behaviour before this patch is to limit irq's to a reasonable number. However, just like our other drivers we can reduce latency for small packet-type traffic considerably by allowing the irq rate to go up dynamically.

This patch introduces a simple irq moderation algorithm based on traffic analysis. The driver will use more CPU to service small packets quicker but will perform the same on bulk traffic as the old code.

Signed-off-by Ayyappan Veeraiyan <[email protected]>

Signed-off-by Auke Kok <[email protected]>

---

 drivers/net/ixgbe/ixgbe.h         |    9 ++
 drivers/net/ixgbe/ixgbe_ethtool.c |   24 ++--
 drivers/net/ixgbe/ixgbe_main.c    |  211 +++++++++++++++++++++++++++++++++++++
 3 files changed, 232 insertions(+), 12 deletions(-)

drivers/net/ixgbe/ixgbe.h +9 -0

index 2077477..79f5519 100644
@@ -136,6 +136,8 @@ struct ixgbe_ring {
 	u16 head;
 	u16 tail;
 
+	unsigned int total_bytes;
+	unsigned int total_packets;
 
 	u16 reg_idx; /* holds the special value that gets the hardware register
 		      * offset associated with this ring, which is different

@@ -174,6 +176,8 @@ struct ixgbe_q_vector {
 	DECLARE_BITMAP(txr_idx, MAX_TX_QUEUES); /* Tx ring indices */
 	u8 rxr_count;     /* Rx ring count assigned to this vector */
 	u8 txr_count;     /* Tx ring count assigned to this vector */
+	u8 tx_eitr;
+	u8 rx_eitr;
 	u32 eitr;
 };

@@ -215,6 +219,11 @@ struct ixgbe_adapter {
 	struct ixgbe_q_vector q_vector[MAX_MSIX_Q_VECTORS];
 	char name[MAX_MSIX_COUNT][IFNAMSIZ + 5];
 
+	/* Interrupt Throttle Rate */
+	u32 itr_setting;
+	u16 eitr_low;
+	u16 eitr_high;
+
 	/* TX */
 	struct ixgbe_ring *tx_ring;	/* One per active queue */
 	u64 restart_queue;

drivers/net/ixgbe/ixgbe_ethtool.c +14 ~~-10~~

index 85b7d15..4e46377 100644
@@ -886,13 +886,13 @@ static int ixgbe_get_coalesce(struct net_device *netdev,
 {
 	struct ixgbe_adapter *adapter = netdev_priv(netdev);
 
-	if (adapter->rx_eitr == 0)
-		ec->rx_coalesce_usecs = 0;
+	if (adapter->rx_eitr < IXGBE_MIN_ITR_USECS)
+		ec->rx_coalesce_usecs = adapter->rx_eitr;
 	else
 		ec->rx_coalesce_usecs = 1000000 / adapter->rx_eitr;
 
-	if (adapter->tx_eitr == 0)
-		ec->tx_coalesce_usecs = 0;
+	if (adapter->tx_eitr < IXGBE_MIN_ITR_USECS)
+		ec->tx_coalesce_usecs = adapter->tx_eitr;
 	else
 		ec->tx_coalesce_usecs = 1000000 / adapter->tx_eitr;

@@ -906,22 +906,26 @@ static int ixgbe_set_coalesce(struct net_device *netdev,
 	struct ixgbe_adapter *adapter = netdev_priv(netdev);
 
 	if ((ec->rx_coalesce_usecs > IXGBE_MAX_ITR_USECS) ||
-	    ((ec->rx_coalesce_usecs > 0) &&
+	    ((ec->rx_coalesce_usecs != 0) &&
+	     (ec->rx_coalesce_usecs != 1) &&
+	     (ec->rx_coalesce_usecs != 3) &&
 	     (ec->rx_coalesce_usecs < IXGBE_MIN_ITR_USECS)))
 		return -EINVAL;
 	if ((ec->tx_coalesce_usecs > IXGBE_MAX_ITR_USECS) ||
-	    ((ec->tx_coalesce_usecs > 0) &&
+	    ((ec->tx_coalesce_usecs != 0) &&
+	     (ec->tx_coalesce_usecs != 1) &&
+	     (ec->tx_coalesce_usecs != 3) &&
 	     (ec->tx_coalesce_usecs < IXGBE_MIN_ITR_USECS)))
 		return -EINVAL;
 
 	/* convert to rate of irq's per second */
-	if (ec->rx_coalesce_usecs == 0)
-		adapter->rx_eitr = 0;
+	if (ec->rx_coalesce_usecs < IXGBE_MIN_ITR_USECS)
+		adapter->rx_eitr = ec->rx_coalesce_usecs;
 	else
 		adapter->rx_eitr = (1000000 / ec->rx_coalesce_usecs);
 
-	if (ec->tx_coalesce_usecs == 0)
-		adapter->tx_eitr = 0;
+	if (ec->tx_coalesce_usecs < IXGBE_MIN_ITR_USECS)
+		adapter->tx_eitr = ec->rx_coalesce_usecs;
 	else
 		adapter->tx_eitr = (1000000 / ec->tx_coalesce_usecs);

drivers/net/ixgbe/ixgbe_main.c +209 -2

282 lines not on this page · raw message

Auke Kok

author

2008-03-03 23:04 UTC #3 lore raw

PATCH 4/5 ixgbe: Add optional DCA infrastructure

In reply to Auke Kok

From: Jeb Cramer <[email protected]>

82598 cards and up support DCA, which enables the chipset to warm up the caches for upcoming payload data. This code makes the driver plug into the CONFIG_DCA infrastructure that was merged earlier.

Signed-off-by Jeb Cramer <[email protected]>

Signed-off-by Auke Kok <[email protected]>

---

 drivers/net/ixgbe/ixgbe.h      |    9 ++
 drivers/net/ixgbe/ixgbe_main.c |  148 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 157 insertions(+), 0 deletions(-)

drivers/net/ixgbe/ixgbe.h +9 -0

index 79f5519..d981134 100644
@@ -36,6 +36,9 @@
 #include "ixgbe_type.h"
 #include "ixgbe_common.h"
 
+#ifdef CONFIG_DCA
+#include <linux/dca.h>
+#endif
 
 #define IXGBE_ERR(args...) printk(KERN_ERR "ixgbe: " args)

@@ -142,6 +145,11 @@ struct ixgbe_ring {
 	u16 reg_idx; /* holds the special value that gets the hardware register
 		      * offset associated with this ring, which is different
 		      * for DCE and RSS modes */
+
+#ifdef CONFIG_DCA
+	/* cpu for tx queue */
+	int cpu;
+#endif
 	struct ixgbe_queue_stats stats;
 	u8 v_idx; /* maps directly to the index for this ring in the hardware
 		   * vector array, can also be used for finding the bit in EICR

@@ -261,6 +269,7 @@ struct ixgbe_adapter {
 #define IXGBE_FLAG_IMIR_ENABLED                 (u32)(1 << 5)
 #define IXGBE_FLAG_RSS_ENABLED                  (u32)(1 << 6)
 #define IXGBE_FLAG_VMDQ_ENABLED                 (u32)(1 << 7)
+#define IXGBE_FLAG_DCA_ENABLED                  (u32)(1 << 8)
 
 	/* OS defined structs */
 	struct net_device *netdev;

drivers/net/ixgbe/ixgbe_main.c +148 -0

204 lines not on this page · raw message

Auke Kok

author

2008-03-03 23:05 UTC #4 lore raw

PATCH 5/5 ixgbe: Increment version

In reply to Auke Kok

Signed-off-by Auke Kok <[email protected]>

---

 drivers/net/ixgbe/ixgbe_main.c |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

drivers/net/ixgbe/ixgbe_main.c +1 -1

index da8becf..7ad2993 100644
@@ -48,7 +48,7 @@ char ixgbe_driver_name[] = "ixgbe";
 static const char ixgbe_driver_string[] =
 	"Intel(R) 10 Gigabit PCI Express Network Driver";
 
-#define DRV_VERSION "1.1.18"
+#define DRV_VERSION "1.3.18-k2"
 const char ixgbe_driver_version[] = DRV_VERSION;
 static const char ixgbe_copyright[] =
 	 "Copyright (c) 1999-2007 Intel Corporation.";

Jeff Garzik

2008-03-05 11:30 UTC #5 lore raw

In reply to Auke Kok

Quoted 16 lines · Auke Kok

From: Ayyappan Veeraiyan <[email protected]>
This code abstracts the per-queue MSI-X interrupt vector into a queue vector layer. This abstraction is needed since there can be many more queues than available MSI-X vectors in a machine.
The MSI-X irq vectors are remapped to a shared queue vector which can point to several (both RX and TX) hardware queues. The NAPI algorithm then cleans the appropriate ring/queues on interrupt or poll.
The remapping is a delicate and complex calculation to make sure that we're not unbalancing the irq load, and spreads the irqs as much as possible, and may combine RX and TX flows onto the same queue vector.

Quoted 7 lines

This effectively enables receive flow hashing across vectors and helps irq load balance across CPUs.
Signed-off-by Ayyappan Veeraiyan <[email protected]>
Signed-off-by Auke Kok <[email protected]>
Acked-by Jesse Brandeburg <[email protected]>
Acked-by Waskiewicz Jr, Peter P <[email protected]>

applied 1-5 to #upstream (2.6.26)