Skip to contentSkip to conversation

chelsio driver changes

PATCH Applied Stephen Hemminger 5 participants 19 messages Dec 15, 2006 – Jan 19, 2007 1 message not in the archive

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

cover chelsio driver changes

One error path bug fix, and a couple of changes to improve receive performance. The performance is now up to 5+ Gbits/sec which is about the limit of the memory/bus on the test systems.

#1 lore raw

PATCH 1/3 chelsio: fix error path

Fix handling of allocation failure.

drivers/net/chelsio/my3126.c +4 -3
@@ -170,13 +170,14 @@ static struct cphy *my3126_phy_create(ad
 {
 	struct cphy *cphy = kzalloc(sizeof (*cphy), GFP_KERNEL);
 
-	if (cphy)
-		cphy_init(cphy, adapter, phy_addr, &my3126_ops, mdio_ops);
+	if (!cphy)
+		return NULL;
 
+	cphy_init(cphy, adapter, phy_addr, &my3126_ops, mdio_ops);
 	INIT_DELAYED_WORK(&cphy->phy_update, my3216_poll);
 	cphy->bmsr = 0;
 
-	return (cphy);
+	return cphy;
 }
 
 /* Chip Reset */
#2 lore raw

PATCH 2/3 chelsio: NAPI speed improvement

In reply to Stephen Hemminger

Speedup and cleanup the receive processing by eliminating the mmio read and a lock round trip.

---
 drivers/net/chelsio/sge.c |   82 ++++++++++++++++++++--------------------------
 1 file changed, 37 insertions(+), 45 deletions(-)
drivers/net/chelsio/sge.c +37 -45
@@ -1576,6 +1576,14 @@ static int process_responses(struct adap
 	return budget;
 }
 
+static inline int responses_pending(const struct adapter *adapter)
+{
+	const struct respQ *Q = &adapter->sge->respQ;
+	const struct respQ_e *e = &Q->entries[Q->cidx];
+
+	return (e->GenerationBit == Q->genbit);
+}
+
 #ifdef CONFIG_CHELSIO_T1_NAPI
 /*
  * A simpler version of process_responses() that handles only pure (i.e.,
@@ -1585,13 +1593,16 @@ static int process_responses(struct adap
  * which the caller must ensure is a valid pure response.  Returns 1 if it
  * encounters a valid data-carrying response, 0 otherwise.
  */
-static int process_pure_responses(struct adapter *adapter, struct respQ_e *e)
+static int process_pure_responses(struct adapter *adapter)
 {
 	struct sge *sge = adapter->sge;
 	struct respQ *q = &sge->respQ;
+	struct respQ_e *e = &q->entries[q->cidx];
 	unsigned int flags = 0;
 	unsigned int cmdq_processed[SGE_CMDQ_N] = {0, 0};
 
+	if (e->DataValid)
+		return 1;
 	do {
 		flags |= e->Qsleeping;
 
@@ -1627,22 +1638,17 @@ static int process_pure_responses(struct
 int t1_poll(struct net_device *dev, int *budget)
 {
 	struct adapter *adapter = dev->priv;
-	int effective_budget = min(*budget, dev->quota);
-	int work_done = process_responses(adapter, effective_budget);
+	int work_done;
 
+	work_done = process_responses(adapter, min(*budget, dev->quota));
 	*budget -= work_done;
 	dev->quota -= work_done;
 
-	if (work_done >= effective_budget)
+	if (unlikely(responses_pending(adapter)))
 		return 1;
 
- 	spin_lock_irq(&adapter->async_lock);
-	__netif_rx_complete(dev);
+	netif_rx_complete(dev);
 	writel(adapter->sge->respQ.cidx, adapter->regs + A_SG_SLEEPING);
-	writel(adapter->slow_intr_mask | F_PL_INTR_SGE_DATA,
-	       adapter->regs + A_PL_ENABLE);
- 	spin_unlock_irq(&adapter->async_lock);
-
 	return 0;
 }
 
@@ -1652,44 +1658,34 @@ int t1_poll(struct net_device *dev, int 
 irqreturn_t t1_interrupt(int irq, void *data)
 {
 	struct adapter *adapter = data;
- 	struct net_device *dev = adapter->sge->netdev;
 	struct sge *sge = adapter->sge;
- 	u32 cause;
-	int handled = 0;
+	int handled;
 
-	cause = readl(adapter->regs + A_PL_CAUSE);
-	if (cause == 0 || cause == ~0)
-		return IRQ_NONE;
+	if (likely(responses_pending(adapter))) {
+		struct net_device *dev = sge->netdev;
 
-	spin_lock(&adapter->async_lock);
- 	if (cause & F_PL_INTR_SGE_DATA) {
-		struct respQ *q = &adapter->sge->respQ;
-		struct respQ_e *e = &q->entries[q->cidx];
-
- 		handled = 1;
- 		writel(F_PL_INTR_SGE_DATA, adapter->regs + A_PL_CAUSE);
-
-		if (e->GenerationBit == q->genbit &&
-		    __netif_rx_schedule_prep(dev)) {
-			if (e->DataValid || process_pure_responses(adapter, e)) {
-				/* mask off data IRQ */
-				writel(adapter->slow_intr_mask,
-				       adapter->regs + A_PL_ENABLE);
-				__netif_rx_schedule(sge->netdev);
-				goto unlock;
-			}
-			/* no data, no NAPI needed */
-			netif_poll_enable(dev);
+		writel(F_PL_INTR_SGE_DATA, adapter->regs + A_PL_CAUSE);
 
+		if (__netif_rx_schedule_prep(dev)) {
+			if (process_pure_responses(adapter)) {
+				__netif_rx_schedule(dev);
+			} else {
+				/* no data, no NAPI needed */
+				writel(sge->respQ.cidx,
+				       adapter->regs + A_SG_SLEEPING);
+				netif_poll_enable(dev);	/* undo schedule_prep */
+			}
 		}
-		writel(q->cidx, adapter->regs + A_SG_SLEEPING);
-	} else
-		handled = t1_slow_intr_handler(adapter);
+		return IRQ_HANDLED;
+	}
+
+	spin_lock(&adapter->async_lock);
+	handled = t1_slow_intr_handler(adapter);
+	spin_unlock(&adapter->async_lock);
 
 	if (!handled)
 		sge->stats.unhandled_irqs++;
-unlock:
-	spin_unlock(&adapter->async_lock);
+
 	return IRQ_RETVAL(handled != 0);
 }
 
@@ -1712,17 +1708,13 @@ unlock:
 irqreturn_t t1_interrupt(int irq, void *cookie)
 {
 	int work_done;
-	struct respQ_e *e;
 	struct adapter *adapter = cookie;
-	struct respQ *Q = &adapter->sge->respQ;
 
 	spin_lock(&adapter->async_lock);
-	e = &Q->entries[Q->cidx];
-	prefetch(e);
 
 	writel(F_PL_INTR_SGE_DATA, adapter->regs + A_PL_CAUSE);
 
-	if (likely(e->GenerationBit == Q->genbit))
+	if (likely(responses_pending(adapter))
 		work_done = process_responses(adapter, -1);
 	else
 		work_done = t1_slow_intr_handler(adapter);
#3 lore raw

PATCH 3/3 chelsio: more receive cleanup

In reply to Stephen Hemminger

Cleanup receive processing some more:
* do the reserve padding of skb during setup
* don't pass constants to get_packet
* do smart prefetch of skb
* make copybreak a module parameter

---
 drivers/net/chelsio/sge.c |   81 +++++++++++++++++++++++-----------------------
 1 file changed, 42 insertions(+), 39 deletions(-)
drivers/net/chelsio/sge.c +42 -39
@@ -71,12 +71,9 @@
 #define SGE_FREEL_REFILL_THRESH	16
 #define SGE_RESPQ_E_N		1024
 #define SGE_INTRTIMER_NRES	1000
-#define SGE_RX_COPY_THRES	256
 #define SGE_RX_SM_BUF_SIZE	1536
 #define SGE_TX_DESC_MAX_PLEN	16384
 
-# define SGE_RX_DROP_THRES 2
-
 #define SGE_RESPQ_REPLENISH_THRES (SGE_RESPQ_E_N / 4)
 
 /*
@@ -862,6 +859,8 @@ static void refill_free_list(struct sge 
 		skb_reserve(skb, q->dma_offset);
 		mapping = pci_map_single(pdev, skb->data, dma_len,
 					 PCI_DMA_FROMDEVICE);
+		skb_reserve(skb, sge->rx_pkt_pad);
+
 		ce->skb = skb;
 		pci_unmap_addr_set(ce, dma_addr, mapping);
 		pci_unmap_len_set(ce, dma_len, dma_len);
@@ -1041,6 +1040,10 @@ static void recycle_fl_buf(struct freelQ
 	}
 }
 
+static int copybreak __read_mostly = 256;
+module_param(copybreak, int, 0);
+MODULE_PARM_DESC(copybreak, "Receive copy threshold");
+
 /**
  *	get_packet - return the next ingress packet buffer
  *	@pdev: the PCI device that received the packet
@@ -1059,37 +1062,33 @@ static void recycle_fl_buf(struct freelQ
  *	threshold and the packet is too big to copy, or (b) the packet should
  *	be copied but there is no memory for the copy.
  */
-static inline struct sk_buff *get_packet(struct pci_dev *pdev,
-					 struct freelQ *fl, unsigned int len,
-					 int dma_pad, int skb_pad,
-					 unsigned int copy_thres,
-					 unsigned int drop_thres)
+static inline struct sk_buff *get_packet(struct pci_dev *pdev, struct freelQ *fl,
+					 unsigned int len)
 {
 	struct sk_buff *skb;
-	struct freelQ_ce *ce = &fl->centries[fl->cidx];
+	const struct freelQ_ce *ce = &fl->centries[fl->cidx];
+
+	if (len < copybreak) {
+		skb = alloc_skb(len + 2, GFP_ATOMIC);
+		if (!skb)
+			goto use_orig_buf;
 
-	if (len < copy_thres) {
-		skb = alloc_skb(len + skb_pad, GFP_ATOMIC);
-		if (likely(skb != NULL)) {
-			skb_reserve(skb, skb_pad);
-			skb_put(skb, len);
-			pci_dma_sync_single_for_cpu(pdev,
+		skb_reserve(skb, 2);	/* align IP header */
+		skb_put(skb, len);
+		pci_dma_sync_single_for_cpu(pdev,
 					    pci_unmap_addr(ce, dma_addr),
  					    pci_unmap_len(ce, dma_len),
 					    PCI_DMA_FROMDEVICE);
-			memcpy(skb->data, ce->skb->data + dma_pad, len);
-			pci_dma_sync_single_for_device(pdev,
+		memcpy(skb->data, ce->skb->data, len);
+		pci_dma_sync_single_for_device(pdev,
 					    pci_unmap_addr(ce, dma_addr),
  					    pci_unmap_len(ce, dma_len),
 					    PCI_DMA_FROMDEVICE);
-		} else if (!drop_thres)
-			goto use_orig_buf;
-
 		recycle_fl_buf(fl, fl->cidx);
 		return skb;
 	}
 
-	if (fl->credits < drop_thres) {
+	if (fl->credits < 2) {
 		recycle_fl_buf(fl, fl->cidx);
 		return NULL;
 	}
@@ -1098,7 +1097,8 @@ use_orig_buf:
 	pci_unmap_single(pdev, pci_unmap_addr(ce, dma_addr),
 			 pci_unmap_len(ce, dma_len), PCI_DMA_FROMDEVICE);
 	skb = ce->skb;
-	skb_reserve(skb, dma_pad);
+	prefetch(skb->data);
+
 	skb_put(skb, len);
 	return skb;
 }
@@ -1375,27 +1375,25 @@ static void restart_sched(unsigned long 
  *
  *	Process an ingress ethernet pakcet and deliver it to the stack.
  */
-static int sge_rx(struct sge *sge, struct freelQ *fl, unsigned int len)
+static void sge_rx(struct sge *sge, struct freelQ *fl, unsigned int len)
 {
 	struct sk_buff *skb;
-	struct cpl_rx_pkt *p;
+	const struct cpl_rx_pkt *p;
 	struct adapter *adapter = sge->adapter;
 	struct sge_port_stats *st;
 
-	skb = get_packet(adapter->pdev, fl, len - sge->rx_pkt_pad,
-			 sge->rx_pkt_pad, 2, SGE_RX_COPY_THRES,
-			 SGE_RX_DROP_THRES);
+	skb = get_packet(adapter->pdev, fl, len - sge->rx_pkt_pad);
 	if (unlikely(!skb)) {
 		sge->stats.rx_drops++;
-		return 0;
+		return;
 	}
 
-	p = (struct cpl_rx_pkt *)skb->data;
-	skb_pull(skb, sizeof(*p));
+	p = (const struct cpl_rx_pkt *) skb->data;
 	if (p->iff >= adapter->params.nports) {
 		kfree_skb(skb);
-		return 0;
+		return;
 	}
+	__skb_pull(skb, sizeof(*p));
 
 	skb->dev = adapter->port[p->iff].dev;
 	skb->dev->last_rx = jiffies;
@@ -1427,7 +1425,6 @@ static int sge_rx(struct sge *sge, struc
 		netif_rx(skb);
 #endif
 	}
-	return 0;
 }
 
 /*
@@ -1510,12 +1507,11 @@ static int process_responses(struct adap
 	struct sge *sge = adapter->sge;
 	struct respQ *q = &sge->respQ;
 	struct respQ_e *e = &q->entries[q->cidx];
-	int budget_left = budget;
+	int done = 0;
 	unsigned int flags = 0;
 	unsigned int cmdq_processed[SGE_CMDQ_N] = {0, 0};
 	
-
-	while (likely(budget_left && e->GenerationBit == q->genbit)) {
+	while (done < budget && e->GenerationBit == q->genbit) {
 		flags |= e->Qsleeping;
 		
 		cmdq_processed[0] += e->Cmdq0CreditReturn;
@@ -1525,14 +1521,16 @@ static int process_responses(struct adap
 		 * ping-pong of TX state information on MP where the sender
 		 * might run on a different CPU than this function...
 		 */
-		if (unlikely(flags & F_CMDQ0_ENABLE || cmdq_processed[0] > 64)) {
+		if (unlikely((flags & F_CMDQ0_ENABLE) || cmdq_processed[0] > 64)) {
 			flags = update_tx_info(adapter, flags, cmdq_processed[0]);
 			cmdq_processed[0] = 0;
 		}
+
 		if (unlikely(cmdq_processed[1] > 16)) {
 			sge->cmdQ[1].processed += cmdq_processed[1];
 			cmdq_processed[1] = 0;
 		}
+
 		if (likely(e->DataValid)) {
 			struct freelQ *fl = &sge->freelQ[e->FreelistQid];
 
@@ -1542,12 +1540,16 @@ static int process_responses(struct adap
 			else
 				sge_rx(sge, fl, e->BufferLength);
 
+			++done;
+
 			/*
 			 * Note: this depends on each packet consuming a
 			 * single free-list buffer; cf. the BUG above.
 			 */
 			if (++fl->cidx == fl->size)
 				fl->cidx = 0;
+			prefetch(fl->centries[fl->cidx].skb);
+
 			if (unlikely(--fl->credits <
 				     fl->size - SGE_FREEL_REFILL_THRESH))
 				refill_free_list(sge, fl);
@@ -1566,14 +1568,12 @@ static int process_responses(struct adap
 			writel(q->credits, adapter->regs + A_SG_RSPQUEUECREDIT);
 			q->credits = 0;
 		}
-		--budget_left;
 	}
 
 	flags = update_tx_info(adapter, flags, cmdq_processed[0]); 
 	sge->cmdQ[1].processed += cmdq_processed[1];
 
-	budget -= budget_left;
-	return budget;
+	return done;
 }
 
 static inline int responses_pending(const struct adapter *adapter)
@@ -1598,11 +1598,14 @@ static int process_pure_responses(struct
 	struct sge *sge = adapter->sge;
 	struct respQ *q = &sge->respQ;
 	struct respQ_e *e = &q->entries[q->cidx];
+	const struct freelQ *fl = &sge->freelQ[e->FreelistQid];
 	unsigned int flags = 0;
 	unsigned int cmdq_processed[SGE_CMDQ_N] = {0, 0};
 
+	prefetch(fl->centries[fl->cidx].skb);
 	if (e->DataValid)
 		return 1;
+
 	do {
 		flags |= e->Qsleeping;
 
#4 lore raw

Re: [PATCH 3/3] chelsio: more receive cleanup

Stephen Hemminger <[email protected]> :
[...]

Quoted 2 lines
drivers/net/chelsio/sge.c +0 -0

[...]

Quoted 7 lines
drivers/net/chelsio/sge.c +0 -3
@@ -1059,37 +1062,33 @@ static void recycle_fl_buf(struct freelQ
  *	threshold and the packet is too big to copy, or (b) the packet should
  *	be copied but there is no memory for the copy.
  */
-static inline struct sk_buff *get_packet(struct pci_dev *pdev,
-					 struct freelQ *fl, unsigned int len,
-					 int dma_pad, int skb_pad,

drivers/net/chelsio/sge.c static void recycle_fl_buf(struct freelQ · line 1071

Quoted 10 lines
drivers/net/chelsio/sge.c +5 -3
-					 unsigned int copy_thres,
-					 unsigned int drop_thres)
+static inline struct sk_buff *get_packet(struct pci_dev *pdev, struct freelQ *fl,
+					 unsigned int len)
 {
 	struct sk_buff *skb;
-	struct freelQ_ce *ce = &fl->centries[fl->cidx];
+	const struct freelQ_ce *ce = &fl->centries[fl->cidx];
+
+	if (len < copybreak) {

If you are into cleanups, maybe add likely/unlikely (and remove the inline) ?

drivers/net/chelsio/sge.c static void recycle_fl_buf(struct freelQ · line 1076

Quoted 11 lines
drivers/net/chelsio/sge.c +4 -6
+		skb = alloc_skb(len + 2, GFP_ATOMIC);
+		if (!skb)
+			goto use_orig_buf;
 
-	if (len < copy_thres) {
-		skb = alloc_skb(len + skb_pad, GFP_ATOMIC);
-		if (likely(skb != NULL)) {
-			skb_reserve(skb, skb_pad);
-			skb_put(skb, len);
-			pci_dma_sync_single_for_cpu(pdev,
+		skb_reserve(skb, 2);	/* align IP header */

s/2/NET_IP_ALIGN/

Btw, since the driver supports netpoll, its blind enabling of interrupts in t1_poll() seems old-fashoined (see thread starting at
http://lkml.org/lkml/2006/12/12/86)

Signature · 1 line
Ueimor
#5 lore raw

Re: [PATCH 3/3] chelsio: more receive cleanup

Quoted 16 lines · On Fri, 15 Dec 2006 21:10:19 +0100 Francois Romieu <[email protected]> wrote:

Stephen Hemminger <[email protected]> :
[...]

Quoted 2 lines
drivers/net/chelsio/sge.c +0 -0

[...]

Quoted 11 lines
drivers/net/chelsio/sge.c +2 -5
@@ -1059,37 +1062,33 @@ static void recycle_fl_buf(struct freelQ
  *	threshold and the packet is too big to copy, or (b) the packet should
  *	be copied but there is no memory for the copy.
  */
-static inline struct sk_buff *get_packet(struct pci_dev *pdev,
-					 struct freelQ *fl, unsigned int len,
-					 int dma_pad, int skb_pad,
-					 unsigned int copy_thres,
-					 unsigned int drop_thres)
+static inline struct sk_buff *get_packet(struct pci_dev *pdev, struct freelQ *fl,
+					 unsigned int len)
Quoted 8 lines

drivers/net/chelsio/sge.c static void recycle_fl_buf(struct freelQ · line 1071

Quoted 6 lines
drivers/net/chelsio/sge.c +3 -1
 {
 	struct sk_buff *skb;
-	struct freelQ_ce *ce = &fl->centries[fl->cidx];
+	const struct freelQ_ce *ce = &fl->centries[fl->cidx];
+
+	if (len < copybreak) {

If you are into cleanups, maybe add likely/unlikely (and remove the inline) ?

Why, you can't predict copybreak on off

Quoted 5 lines
Quoted 5 lines
drivers/net/chelsio/sge.c +3 -0
+		skb = alloc_skb(len + 2, GFP_ATOMIC);
+		if (!skb)
+			goto use_orig_buf;
 
Quoted 9 lines

drivers/net/chelsio/sge.c static void recycle_fl_buf(struct freelQ · line 1076

Quoted 7 lines
drivers/net/chelsio/sge.c +1 -6
-	if (len < copy_thres) {
-		skb = alloc_skb(len + skb_pad, GFP_ATOMIC);
-		if (likely(skb != NULL)) {
-			skb_reserve(skb, skb_pad);
-			skb_put(skb, len);
-			pci_dma_sync_single_for_cpu(pdev,
+		skb_reserve(skb, 2);	/* align IP header */

s/2/NET_IP_ALIGN/

This is wrong, NET_IP_ALIGN is for DMA buffers only. It is defined as 0 for those platforms where DMA alignment is important. In this case we are copying, so we want to force 2.

Quoted 4 lines

Btw, since the driver supports netpoll, its blind enabling of interrupts in t1_poll() seems old-fashoined (see thread starting at
http://lkml.org/lkml/2006/12/12/86)

Signature · 1 line
Stephen Hemminger <[email protected]>
#6 lore raw

Re: [PATCH 3/3] chelsio: more receive cleanup

Stephen Hemminger <[email protected]> :

Quoted 1 line

Francois Romieu <[email protected]> wrote:

[...]

Quoted 1 line

Why, you can't predict copybreak on off

Plain gut feeling. I'd rather spend extra cycles on small packets (whose allocation is already optimized) and keep the normal packets less expensive on a 10Gb/s network card.

[...]

Quoted 3 lines

This is wrong, NET_IP_ALIGN is for DMA buffers only. It is defined as 0 for those platforms where DMA alignment is important. In this case we are copying, so we want to force 2.

Ok, ok.

Signature · 1 line
Ueimor
#7 lore raw

Re: [PATCH 3/3] chelsio: more receive cleanup

Quoted 9 lines · On Sat, 16 Dec 2006 01:26:04 +0100 Francois Romieu <[email protected]> wrote:

Stephen Hemminger <[email protected]> :

Quoted 1 line

Francois Romieu <[email protected]> wrote:

[...]

Quoted 1 line

Why, you can't predict copybreak on off

Plain gut feeling. I'd rather spend extra cycles on small packets (whose allocation is already optimized) and keep the normal packets less expensive on a 10Gb/s network card.

Likely/unlikely is an optimization that doesn't make a big difference and I would rather the compiler choose unless it is something obvious like an error path or external interrupt (like PHY).

There was some discussion on LKML, which I mostly forgot, that hinted that on some architectures using likely/unlikely incorrectly could have a major hit.

Signature · 1 line
Stephen Hemminger <[email protected]>
#8 lore raw
In reply to Stephen Hemminger
Quoted 3 lines · Stephen Hemminger

One error path bug fix, and a couple of changes to improve receive performance. The performance is now up to 5+ Gbits/sec which is about the limit of the memory/bus on the test systems.

I request a rediff+resend against netdev-2.6.git#upstream, giving sufficient time [hours :(] for it to mirror out.

Jeff

#9 lore raw

PATCH 1/3 chelsio: error path fix

Fix handling of allocation failure.

drivers/net/chelsio/my3126.c +3 -2
@@ -170,9 +170,10 @@ static struct cphy *my3126_phy_create(ad
 {
 	struct cphy *cphy = kzalloc(sizeof (*cphy), GFP_KERNEL);
 
-	if (cphy)
-		cphy_init(cphy, adapter, phy_addr, &my3126_ops, mdio_ops);
+	if (!cphy)
+		return NULL;
 
+	cphy_init(cphy, adapter, phy_addr, &my3126_ops, mdio_ops);
 	INIT_DELAYED_WORK(&cphy->phy_update, my3216_poll);
 	cphy->bmsr = 0;
 

A message that is not in the archive (sent off-list or not yet received): [email protected]

#11 lore raw

PATCH 3/3 chelsio: more rx speedup

Cleanup receive processing some more:
* do the reserve padding of skb during setup
* don't pass constants to get_packet
* do smart prefetch of skb
* make copybreak a module parameter

---
 drivers/net/chelsio/sge.c |   87 +++++++++++++++++++++++-----------------------
 1 file changed, 45 insertions(+), 42 deletions(-)
drivers/net/chelsio/sge.c +45 -42
@@ -71,12 +71,9 @@
 #define SGE_FREEL_REFILL_THRESH	16
 #define SGE_RESPQ_E_N		1024
 #define SGE_INTRTIMER_NRES	1000
-#define SGE_RX_COPY_THRES	256
 #define SGE_RX_SM_BUF_SIZE	1536
 #define SGE_TX_DESC_MAX_PLEN	16384
 
-# define SGE_RX_DROP_THRES 2
-
 #define SGE_RESPQ_REPLENISH_THRES (SGE_RESPQ_E_N / 4)
 
 /*
@@ -846,6 +843,8 @@ static void refill_free_list(struct sge 
 		skb_reserve(skb, q->dma_offset);
 		mapping = pci_map_single(pdev, skb->data, dma_len,
 					 PCI_DMA_FROMDEVICE);
+		skb_reserve(skb, sge->rx_pkt_pad);
+
 		ce->skb = skb;
 		pci_unmap_addr_set(ce, dma_addr, mapping);
 		pci_unmap_len_set(ce, dma_len, dma_len);
@@ -1024,6 +1023,10 @@ static void recycle_fl_buf(struct freelQ
 	}
 }
 
+static int copybreak __read_mostly = 256;
+module_param(copybreak, int, 0);
+MODULE_PARM_DESC(copybreak, "Receive copy threshold");
+
 /**
  *	get_packet - return the next ingress packet buffer
  *	@pdev: the PCI device that received the packet
@@ -1043,45 +1046,42 @@ static void recycle_fl_buf(struct freelQ
  *	be copied but there is no memory for the copy.
  */
 static inline struct sk_buff *get_packet(struct pci_dev *pdev,
-					 struct freelQ *fl, unsigned int len,
-					 int dma_pad, int skb_pad,
-					 unsigned int copy_thres,
-					 unsigned int drop_thres)
+					 struct freelQ *fl, unsigned int len)
 {
 	struct sk_buff *skb;
-	struct freelQ_ce *ce = &fl->centries[fl->cidx];
+	const struct freelQ_ce *ce = &fl->centries[fl->cidx];
 
-	if (len < copy_thres) {
-		skb = alloc_skb(len + skb_pad, GFP_ATOMIC);
-		if (likely(skb != NULL)) {
-			skb_reserve(skb, skb_pad);
-			skb_put(skb, len);
-			pci_dma_sync_single_for_cpu(pdev,
-					    pci_unmap_addr(ce, dma_addr),
-					    pci_unmap_len(ce, dma_len),
-					    PCI_DMA_FROMDEVICE);
-			memcpy(skb->data, ce->skb->data + dma_pad, len);
-			pci_dma_sync_single_for_device(pdev,
+	if (len < copybreak) {
+		skb = alloc_skb(len + 2, GFP_ATOMIC);
+		if (!skb)
+			goto use_orig_buf;
+
+		skb_reserve(skb, 2);	/* align IP header */
+		skb_put(skb, len);
+		pci_dma_sync_single_for_cpu(pdev,
 					    pci_unmap_addr(ce, dma_addr),
 					    pci_unmap_len(ce, dma_len),
 					    PCI_DMA_FROMDEVICE);
-		} else if (!drop_thres)
-			goto use_orig_buf;
-
+		memcpy(skb->data, ce->skb->data, len);
+		pci_dma_sync_single_for_device(pdev,
+					       pci_unmap_addr(ce, dma_addr),
+					       pci_unmap_len(ce, dma_len),
+					       PCI_DMA_FROMDEVICE);
 		recycle_fl_buf(fl, fl->cidx);
 		return skb;
 	}
 
-	if (fl->credits < drop_thres) {
+use_orig_buf:
+	if (fl->credits < 2) {
 		recycle_fl_buf(fl, fl->cidx);
 		return NULL;
 	}
 
-use_orig_buf:
 	pci_unmap_single(pdev, pci_unmap_addr(ce, dma_addr),
 			 pci_unmap_len(ce, dma_len), PCI_DMA_FROMDEVICE);
 	skb = ce->skb;
-	skb_reserve(skb, dma_pad);
+	prefetch(skb->data);
+
 	skb_put(skb, len);
 	return skb;
 }
@@ -1359,27 +1359,25 @@ static void restart_sched(unsigned long 
  *
  *	Process an ingress ethernet pakcet and deliver it to the stack.
  */
-static int sge_rx(struct sge *sge, struct freelQ *fl, unsigned int len)
+static void sge_rx(struct sge *sge, struct freelQ *fl, unsigned int len)
 {
 	struct sk_buff *skb;
-	struct cpl_rx_pkt *p;
+	const struct cpl_rx_pkt *p;
 	struct adapter *adapter = sge->adapter;
 	struct sge_port_stats *st;
 
-	skb = get_packet(adapter->pdev, fl, len - sge->rx_pkt_pad,
-			 sge->rx_pkt_pad, 2, SGE_RX_COPY_THRES,
-			 SGE_RX_DROP_THRES);
+	skb = get_packet(adapter->pdev, fl, len - sge->rx_pkt_pad);
 	if (unlikely(!skb)) {
 		sge->stats.rx_drops++;
-		return 0;
+		return;
 	}
 
-	p = (struct cpl_rx_pkt *)skb->data;
-	skb_pull(skb, sizeof(*p));
+	p = (const struct cpl_rx_pkt *) skb->data;
 	if (p->iff >= adapter->params.nports) {
 		kfree_skb(skb);
-		return 0;
+		return;
 	}
+	__skb_pull(skb, sizeof(*p));
 
 	skb->dev = adapter->port[p->iff].dev;
 	skb->dev->last_rx = jiffies;
@@ -1411,7 +1409,6 @@ static int sge_rx(struct sge *sge, struc
 		netif_rx(skb);
 #endif
 	}
-	return 0;
 }
 
 /*
@@ -1493,12 +1490,11 @@ static int process_responses(struct adap
 	struct sge *sge = adapter->sge;
 	struct respQ *q = &sge->respQ;
 	struct respQ_e *e = &q->entries[q->cidx];
-	int budget_left = budget;
+	int done = 0;
 	unsigned int flags = 0;
 	unsigned int cmdq_processed[SGE_CMDQ_N] = {0, 0};
 
-
-	while (likely(budget_left && e->GenerationBit == q->genbit)) {
+	while (done < budget && e->GenerationBit == q->genbit) {
 		flags |= e->Qsleeping;
 
 		cmdq_processed[0] += e->Cmdq0CreditReturn;
@@ -1508,14 +1504,16 @@ static int process_responses(struct adap
 		 * ping-pong of TX state information on MP where the sender
 		 * might run on a different CPU than this function...
 		 */
-		if (unlikely(flags & F_CMDQ0_ENABLE || cmdq_processed[0] > 64)) {
+		if (unlikely((flags & F_CMDQ0_ENABLE) || cmdq_processed[0] > 64)) {
 			flags = update_tx_info(adapter, flags, cmdq_processed[0]);
 			cmdq_processed[0] = 0;
 		}
+
 		if (unlikely(cmdq_processed[1] > 16)) {
 			sge->cmdQ[1].processed += cmdq_processed[1];
 			cmdq_processed[1] = 0;
 		}
+
 		if (likely(e->DataValid)) {
 			struct freelQ *fl = &sge->freelQ[e->FreelistQid];
 
@@ -1525,12 +1523,16 @@ static int process_responses(struct adap
 			else
 				sge_rx(sge, fl, e->BufferLength);
 
+			++done;
+
 			/*
 			 * Note: this depends on each packet consuming a
 			 * single free-list buffer; cf. the BUG above.
 			 */
 			if (++fl->cidx == fl->size)
 				fl->cidx = 0;
+			prefetch(fl->centries[fl->cidx].skb);
+
 			if (unlikely(--fl->credits <
 				     fl->size - SGE_FREEL_REFILL_THRESH))
 				refill_free_list(sge, fl);
@@ -1549,14 +1551,12 @@ static int process_responses(struct adap
 			writel(q->credits, adapter->regs + A_SG_RSPQUEUECREDIT);
 			q->credits = 0;
 		}
-		--budget_left;
 	}
 
 	flags = update_tx_info(adapter, flags, cmdq_processed[0]);
 	sge->cmdQ[1].processed += cmdq_processed[1];
 
-	budget -= budget_left;
-	return budget;
+	return done;
 }
 
 static inline int responses_pending(const struct adapter *adapter)
@@ -1581,11 +1581,14 @@ static int process_pure_responses(struct
 	struct sge *sge = adapter->sge;
 	struct respQ *q = &sge->respQ;
 	struct respQ_e *e = &q->entries[q->cidx];
+	const struct freelQ *fl = &sge->freelQ[e->FreelistQid];
 	unsigned int flags = 0;
 	unsigned int cmdq_processed[SGE_CMDQ_N] = {0, 0};
 
+	prefetch(fl->centries[fl->cidx].skb);
 	if (e->DataValid)
 		return 1;
+
 	do {
 		flags |= e->Qsleeping;
 
#12 lore raw

Re: [PATCH 3/3] chelsio: more rx speedup

Hi Stephen,

Quoted 2 lines · Stephen Hemminger
drivers/net/chelsio/sge.c +0 -0

[...]

Quoted 21 lines
drivers/net/chelsio/sge.c +2 -12
@@ -1043,45 +1046,42 @@ static void recycle_fl_buf(struct freelQ
  *	be copied but there is no memory for the copy.
  */
 static inline struct sk_buff *get_packet(struct pci_dev *pdev,
-					 struct freelQ *fl, unsigned int len,
-					 int dma_pad, int skb_pad,
-					 unsigned int copy_thres,
-					 unsigned int drop_thres)
+					 struct freelQ *fl, unsigned int len)
 {
 	struct sk_buff *skb;
-	struct freelQ_ce *ce = &fl->centries[fl->cidx];
+	const struct freelQ_ce *ce = &fl->centries[fl->cidx];
 
-	if (len < copy_thres) {
-		skb = alloc_skb(len + skb_pad, GFP_ATOMIC);
-		if (likely(skb != NULL)) {
-			skb_reserve(skb, skb_pad);
-			skb_put(skb, len);
-			pci_dma_sync_single_for_cpu(pdev,
-					    pci_unmap_addr(ce, dma_addr),

drivers/net/chelsio/sge.c static void recycle_fl_buf(struct freelQ · line 1059

Quoted 10 lines
drivers/net/chelsio/sge.c +6 -4
-					    pci_unmap_len(ce, dma_len),
-					    PCI_DMA_FROMDEVICE);
-			memcpy(skb->data, ce->skb->data + dma_pad, len);
-			pci_dma_sync_single_for_device(pdev,
+	if (len < copybreak) {
+		skb = alloc_skb(len + 2, GFP_ATOMIC);
+		if (!skb)
+			goto use_orig_buf;
+
+		skb_reserve(skb, 2);	/* align IP header */

Please use NET_IP_ALIGN here:

+ skb = alloc_skb(len + NET_IP_ALIGN, GFP_ATOMIC);
+ if (!skb)
+ goto use_orig_buf;
+
+ skb_reserve(skb, NET_IP_ALIGN);

Quoted 10 lines
drivers/net/chelsio/sge.c +4 -3
+		skb_put(skb, len);
+		pci_dma_sync_single_for_cpu(pdev,
 					    pci_unmap_addr(ce, dma_addr),
 					    pci_unmap_len(ce, dma_len),
 					    PCI_DMA_FROMDEVICE);
-		} else if (!drop_thres)
-			goto use_orig_buf;
-
+		memcpy(skb->data, ce->skb->data, len);
+		pci_dma_sync_single_for_device(pdev,

drivers/net/chelsio/sge.c static void recycle_fl_buf(struct freelQ · line 1075

Quoted 10 lines
drivers/net/chelsio/sge.c +5 -1
+					       pci_unmap_addr(ce, dma_addr),
+					       pci_unmap_len(ce, dma_len),
+					       PCI_DMA_FROMDEVICE);
 		recycle_fl_buf(fl, fl->cidx);
 		return skb;
 	}
 
-	if (fl->credits < drop_thres) {
+use_orig_buf:
+	if (fl->credits < 2) {

Why 2? What does this magic number mean?

Regards

Ingo Oeser

#13 lore raw

Re: [PATCH 3/3] chelsio: more rx speedup

Quoted 31 lines · On Tue, 9 Jan 2007 09:42:03 +0100 Ingo Oeser <[email protected]> wrote:

Hi Stephen,

Quoted 2 lines · Stephen Hemminger
drivers/net/chelsio/sge.c +0 -0

[...]

Quoted 25 lines
drivers/net/chelsio/sge.c +2 -16
@@ -1043,45 +1046,42 @@ static void recycle_fl_buf(struct freelQ
  *	be copied but there is no memory for the copy.
  */
 static inline struct sk_buff *get_packet(struct pci_dev *pdev,
-					 struct freelQ *fl, unsigned int len,
-					 int dma_pad, int skb_pad,
-					 unsigned int copy_thres,
-					 unsigned int drop_thres)
+					 struct freelQ *fl, unsigned int len)
 {
 	struct sk_buff *skb;
-	struct freelQ_ce *ce = &fl->centries[fl->cidx];
+	const struct freelQ_ce *ce = &fl->centries[fl->cidx];
 
-	if (len < copy_thres) {
-		skb = alloc_skb(len + skb_pad, GFP_ATOMIC);
-		if (likely(skb != NULL)) {
-			skb_reserve(skb, skb_pad);
-			skb_put(skb, len);
-			pci_dma_sync_single_for_cpu(pdev,
-					    pci_unmap_addr(ce, dma_addr),
-					    pci_unmap_len(ce, dma_len),
-					    PCI_DMA_FROMDEVICE);
-			memcpy(skb->data, ce->skb->data + dma_pad, len);
-			pci_dma_sync_single_for_device(pdev,
Quoted 8 lines

drivers/net/chelsio/sge.c static void recycle_fl_buf(struct freelQ · line 1059

Quoted 6 lines
drivers/net/chelsio/sge.c +6 -0
+	if (len < copybreak) {
+		skb = alloc_skb(len + 2, GFP_ATOMIC);
+		if (!skb)
+			goto use_orig_buf;
+
+		skb_reserve(skb, 2);	/* align IP header */

Please use NET_IP_ALIGN here:

Wrong, NET_IP_ALIGN is intended to deal with platforms where alignment of DMA is more important of alignment of structures. Therefore if data is copied, it should always be 2.

Quoted 24 lines
drivers/net/chelsio/sge.c +5 -0
+		skb = alloc_skb(len + NET_IP_ALIGN, GFP_ATOMIC);
+		if (!skb)
+			goto use_orig_buf;
+
+		skb_reserve(skb, NET_IP_ALIGN);
Quoted 18 lines
drivers/net/chelsio/sge.c +7 -3
+		skb_put(skb, len);
+		pci_dma_sync_single_for_cpu(pdev,
 					    pci_unmap_addr(ce, dma_addr),
 					    pci_unmap_len(ce, dma_len),
 					    PCI_DMA_FROMDEVICE);
-		} else if (!drop_thres)
-			goto use_orig_buf;
-
+		memcpy(skb->data, ce->skb->data, len);
+		pci_dma_sync_single_for_device(pdev,
+					       pci_unmap_addr(ce, dma_addr),
+					       pci_unmap_len(ce, dma_len),
+					       PCI_DMA_FROMDEVICE);
 		recycle_fl_buf(fl, fl->cidx);
 		return skb;
 	}
 
Quoted 5 lines

drivers/net/chelsio/sge.c static void recycle_fl_buf(struct freelQ · line 1075

Quoted 3 lines
drivers/net/chelsio/sge.c +2 -1
-	if (fl->credits < drop_thres) {
+use_orig_buf:
+	if (fl->credits < 2) {

Why 2? What does this magic number mean?

No idea, it was there in the original. (as a parameter).

Signature · 1 line
Stephen Hemminger <[email protected]>
#14 lore raw

Re: [PATCH 3/3] chelsio: more rx speedup

Quoted 77 lines · Stephen Hemminger
Quoted 34 lines · On Tue, 9 Jan 2007 09:42:03 +0100 Ingo Oeser <[email protected]> wrote:

Hi Stephen,

Quoted 3 lines · Stephen Hemminger
drivers/net/chelsio/sge.c +0 -0

[...]

Quoted 25 lines
drivers/net/chelsio/sge.c +2 -16
@@ -1043,45 +1046,42 @@ static void recycle_fl_buf(struct freelQ
  *	be copied but there is no memory for the copy.
  */
 static inline struct sk_buff *get_packet(struct pci_dev *pdev,
-					 struct freelQ *fl, unsigned int len,
-					 int dma_pad, int skb_pad,
-					 unsigned int copy_thres,
-					 unsigned int drop_thres)
+					 struct freelQ *fl, unsigned int len)
 {
 	struct sk_buff *skb;
-	struct freelQ_ce *ce = &fl->centries[fl->cidx];
+	const struct freelQ_ce *ce = &fl->centries[fl->cidx];
 
-	if (len < copy_thres) {
-		skb = alloc_skb(len + skb_pad, GFP_ATOMIC);
-		if (likely(skb != NULL)) {
-			skb_reserve(skb, skb_pad);
-			skb_put(skb, len);
-			pci_dma_sync_single_for_cpu(pdev,
-					    pci_unmap_addr(ce, dma_addr),
-					    pci_unmap_len(ce, dma_len),
-					    PCI_DMA_FROMDEVICE);
-			memcpy(skb->data, ce->skb->data + dma_pad, len);
-			pci_dma_sync_single_for_device(pdev,
Quoted 9 lines

drivers/net/chelsio/sge.c static void recycle_fl_buf(struct freelQ · line 1073

Quoted 7 lines
drivers/net/chelsio/sge.c +6 -0
+	if (len < copybreak) {
+		skb = alloc_skb(len + 2, GFP_ATOMIC);
+		if (!skb)
+			goto use_orig_buf;
+
+		skb_reserve(skb, 2);	/* align IP header */
 

Please use NET_IP_ALIGN here:

Wrong, NET_IP_ALIGN is intended to deal with platforms where alignment of DMA is more important of alignment of structures. Therefore if data is copied, it should always be 2.

Quoted 24 lines
drivers/net/chelsio/sge.c +5 -0
+		skb = alloc_skb(len + NET_IP_ALIGN, GFP_ATOMIC);
+		if (!skb)
+			goto use_orig_buf;
+
+		skb_reserve(skb, NET_IP_ALIGN);
Quoted 19 lines
drivers/net/chelsio/sge.c +7 -3
+		skb_put(skb, len);
+		pci_dma_sync_single_for_cpu(pdev,
 					    pci_unmap_addr(ce, dma_addr),
 					    pci_unmap_len(ce, dma_len),
 					    PCI_DMA_FROMDEVICE);
-		} else if (!drop_thres)
-			goto use_orig_buf;
-
+		memcpy(skb->data, ce->skb->data, len);
+		pci_dma_sync_single_for_device(pdev,
+					       pci_unmap_addr(ce, dma_addr),
+					       pci_unmap_len(ce, dma_len),
+					       PCI_DMA_FROMDEVICE);
 		recycle_fl_buf(fl, fl->cidx);
 		return skb;
 	}
 
Quoted 10 lines
Quoted 6 lines

drivers/net/chelsio/sge.c static void recycle_fl_buf(struct freelQ · line 1079

Quoted 4 lines
drivers/net/chelsio/sge.c +2 -1
-	if (fl->credits < drop_thres) {
+use_orig_buf:
+	if (fl->credits < 2) {
 

Why 2? What does this magic number mean?

No idea, it was there in the original. (as a parameter).

The T2 HW behaves nicely when it is guaranteed to have 2 available entries in the rx free list.

Cheers,
Divy

#15 lore raw

Re: [PATCH 3/3] chelsio: more rx speedup

Quoted 4 lines · Divy Le Ray
Quoted 3 lines · Stephen Hemminger
Quoted 1 line · On Tue, 9 Jan 2007 09:42:03 +0100 Ingo Oeser <[email protected]> wrote:

Stephen Hemminger schrieb:

Quoted 8 lines
Quoted 6 lines
Quoted 5 lines

drivers/net/chelsio/sge.c static void recycle_fl_buf(struct freelQ · line 1075

Quoted 4 lines
drivers/net/chelsio/sge.c +2 -1
-	if (fl->credits < drop_thres) {
+use_orig_buf:
+	if (fl->credits < 2) {

Why 2? What does this magic number mean?

No idea, it was there in the original. (as a parameter).

The T2 HW behaves nicely when it is guaranteed to have 2 available entries in the rx free list.

Can we have a #define for this?

That would help people understand this issue more.
And don't be shy about errata, all hardware and software out there has them like the humans that made them :-)

Regards

Ingo Oeser

#16 lore raw

Re: [PATCH 3/3] chelsio: more rx speedup

In reply to Stephen Hemminger
Quoted 10 lines · Stephen Hemminger
Quoted 4 lines · On Tue, 9 Jan 2007 09:42:03 +0100 Ingo Oeser <[email protected]> wrote:

drivers/net/chelsio/sge.c

Quoted 2 lines · Stephen Hemminger
drivers/net/chelsio/sge.c +0 -0

Please use NET_IP_ALIGN here:

Wrong, NET_IP_ALIGN is intended to deal with platforms where alignment of DMA is more important of alignment of structures. Therefore if data is copied, it should always be 2.

Ah! Thanks for clearing this up. These magic numbers always make my head spin so I tried to come up with sth. useful :-)

If this pattern is present in many drivers, we might have a define or helper for this.

Regards

Ingo Oeser

#18 lore raw

PATCH 2/3 chelsio: NAPI speed improvement

In reply to Stephen Hemminger

Speedup and cleanup the receive processing by eliminating the mmio read and a lock round trip.

---
 drivers/net/chelsio/sge.c |   77 ++++++++++++++++++++--------------------------
 1 file changed, 35 insertions(+), 42 deletions(-)
drivers/net/chelsio/sge.c +35 -42
@@ -1559,6 +1559,14 @@ static int process_responses(struct adap
 	return budget;
 }
 
+static inline int responses_pending(const struct adapter *adapter)
+{
+	const struct respQ *Q = &adapter->sge->respQ;
+	const struct respQ_e *e = &Q->entries[Q->cidx];
+
+	return (e->GenerationBit == Q->genbit);
+}
+
 #ifdef CONFIG_CHELSIO_T1_NAPI
 /*
  * A simpler version of process_responses() that handles only pure (i.e.,
@@ -1568,13 +1576,16 @@ static int process_responses(struct adap
  * which the caller must ensure is a valid pure response.  Returns 1 if it
  * encounters a valid data-carrying response, 0 otherwise.
  */
-static int process_pure_responses(struct adapter *adapter, struct respQ_e *e)
+static int process_pure_responses(struct adapter *adapter)
 {
 	struct sge *sge = adapter->sge;
 	struct respQ *q = &sge->respQ;
+	struct respQ_e *e = &q->entries[q->cidx];
 	unsigned int flags = 0;
 	unsigned int cmdq_processed[SGE_CMDQ_N] = {0, 0};
 
+	if (e->DataValid)
+		return 1;
 	do {
 		flags |= e->Qsleeping;
 
@@ -1610,23 +1621,20 @@ static int process_pure_responses(struct
 int t1_poll(struct net_device *dev, int *budget)
 {
 	struct adapter *adapter = dev->priv;
-	int effective_budget = min(*budget, dev->quota);
-	int work_done = process_responses(adapter, effective_budget);
+	int work_done;
 
+	work_done = process_responses(adapter, min(*budget, dev->quota));
 	*budget -= work_done;
 	dev->quota -= work_done;
 
-	if (work_done >= effective_budget)
+	if (unlikely(responses_pending(adapter)))
 		return 1;
 
-	spin_lock_irq(&adapter->async_lock);
-	__netif_rx_complete(dev);
+	netif_rx_complete(dev);
 	writel(adapter->sge->respQ.cidx, adapter->regs + A_SG_SLEEPING);
-	writel(adapter->slow_intr_mask | F_PL_INTR_SGE_DATA,
-	       adapter->regs + A_PL_ENABLE);
-	spin_unlock_irq(&adapter->async_lock);
 
 	return 0;
+
 }
 
 /*
@@ -1635,44 +1643,33 @@ int t1_poll(struct net_device *dev, int 
 irqreturn_t t1_interrupt(int irq, void *data)
 {
 	struct adapter *adapter = data;
-	struct net_device *dev = adapter->sge->netdev;
 	struct sge *sge = adapter->sge;
-	u32 cause;
-	int handled = 0;
+	int handled;
 
-	cause = readl(adapter->regs + A_PL_CAUSE);
-	if (cause == 0 || cause == ~0)
-		return IRQ_NONE;
-
-	spin_lock(&adapter->async_lock);
-	if (cause & F_PL_INTR_SGE_DATA) {
-		struct respQ *q = &adapter->sge->respQ;
-		struct respQ_e *e = &q->entries[q->cidx];
+	if (likely(responses_pending(adapter))) {
+		struct net_device *dev = sge->netdev;
 
-		handled = 1;
 		writel(F_PL_INTR_SGE_DATA, adapter->regs + A_PL_CAUSE);
 
-		if (e->GenerationBit == q->genbit &&
-		    __netif_rx_schedule_prep(dev)) {
-			if (e->DataValid || process_pure_responses(adapter, e)) {
-				/* mask off data IRQ */
-				writel(adapter->slow_intr_mask,
-				       adapter->regs + A_PL_ENABLE);
-				__netif_rx_schedule(sge->netdev);
-				goto unlock;
+		if (__netif_rx_schedule_prep(dev)) {
+			if (process_pure_responses(adapter))
+				__netif_rx_schedule(dev);
+			else {
+				/* no data, no NAPI needed */
+				writel(sge->respQ.cidx, adapter->regs + A_SG_SLEEPING);
+				netif_poll_enable(dev);	/* undo schedule_prep */
 			}
-			/* no data, no NAPI needed */
-			netif_poll_enable(dev);
-
 		}
-		writel(q->cidx, adapter->regs + A_SG_SLEEPING);
-	} else
-		handled = t1_slow_intr_handler(adapter);
+		return IRQ_HANDLED;
+	}
+
+	spin_lock(&adapter->async_lock);
+	handled = t1_slow_intr_handler(adapter);
+	spin_unlock(&adapter->async_lock);
 
 	if (!handled)
 		sge->stats.unhandled_irqs++;
-unlock:
-	spin_unlock(&adapter->async_lock);
+
 	return IRQ_RETVAL(handled != 0);
 }
 
@@ -1695,17 +1692,13 @@ unlock:
 irqreturn_t t1_interrupt(int irq, void *cookie)
 {
 	int work_done;
-	struct respQ_e *e;
 	struct adapter *adapter = cookie;
-	struct respQ *Q = &adapter->sge->respQ;
 
 	spin_lock(&adapter->async_lock);
-	e = &Q->entries[Q->cidx];
-	prefetch(e);
 
 	writel(F_PL_INTR_SGE_DATA, adapter->regs + A_PL_CAUSE);
 
-	if (likely(e->GenerationBit == Q->genbit))
+	if (likely(responses_pending(adapter))
 		work_done = process_responses(adapter, -1);
 	else
 		work_done = t1_slow_intr_handler(adapter);