chelsio driver changes
Subsystems (2)
cover chelsio driver changes
One error path bug fix, and a couple of changes to improve receive performance. The performance is now up to 5+ Gbits/sec which is about the limit of the memory/bus on the test systems.
PATCH 1/3 chelsio: fix error path
Fix handling of allocation failure.
drivers/net/chelsio/my3126.c +4 -3
@@ -170,13 +170,14 @@ static struct cphy *my3126_phy_create(ad {struct cphy *cphy =kzalloc (sizeof (*cphy),GFP_KERNEL ); -if (cphy) -cphy_init (cphy, adapter, phy_addr, &my3126_ops, mdio_ops); +if (!cphy) +return NULL ; +cphy_init (cphy, adapter, phy_addr, &my3126_ops, mdio_ops);INIT_DELAYED_WORK (&cphy->phy_update, my3216_poll); cphy->bmsr =0 ; -return (cphy); +return cphy; }/* Chip Reset */
PATCH 2/3 chelsio: NAPI speed improvement
In reply to Stephen HemmingerSpeedup and cleanup the receive processing by eliminating the mmio read and a lock round trip.
--- drivers/net/chelsio/sge.c | 82 ++++++++++++++++++++-------------------------- 1 file changed, 37 insertions(+), 45 deletions(-)
drivers/net/chelsio/sge.c +37 -45
@@ -1576,6 +1576,14 @@ static int process_responses(struct adapreturn budget; } +static inline int responses_pending (const struct adapter *adapter) +{ +const struct respQ *Q = &adapter->sge->respQ; +const struct respQ_e *e = &Q ->entries[Q ->cidx]; + +return (e->GenerationBit ==Q ->genbit); +} +#ifdef CONFIG_CHELSIO_T1_NAPI /* * A simpler version of process_responses() that handles only pure (i.e.,
@@ -1585,13 +1593,16 @@ static int process_responses(struct adap* which the caller must ensure is a valid pure response. Returns 1 if it * encounters a valid data-carrying response, 0 otherwise. */ -static int process_pure_responses (struct adapter *adapter,struct respQ_e *e) +static int process_pure_responses (struct adapter *adapter) {struct sge *sge = adapter->sge;struct respQ *q = &sge->respQ; +struct respQ_e *e = &q->entries[q->cidx];unsigned int flags =0 ;unsigned int cmdq_processed[SGE_CMDQ_N ] = {0 ,0 }; +if (e->DataValid) +return 1 ; do { flags |= e->Qsleeping;
@@ -1627,22 +1638,17 @@ static int process_pure_responses(structint t1_poll (struct net_device *dev,int *budget) {struct adapter *adapter = dev->priv; -int effective_budget =min (*budget, dev->quota); -int work_done =process_responses (adapter, effective_budget); +int work_done; + work_done =process_responses (adapter,min (*budget, dev->quota)); *budget -= work_done; dev->quota -= work_done; -if (work_done >= effective_budget) +if (unlikely (responses_pending (adapter)))return 1 ; -spin_lock_irq (&adapter->async_lock); -__netif_rx_complete (dev); +netif_rx_complete (dev);writel (adapter->sge->respQ.cidx, adapter->regs +A_SG_SLEEPING ); -writel (adapter->slow_intr_mask |F_PL_INTR_SGE_DATA , - adapter->regs +A_PL_ENABLE ); -spin_unlock_irq (&adapter->async_lock); -return 0 ; }
@@ -1652,44 +1658,34 @@ int t1_poll(struct net_device *dev, intirqreturn_t t1_interrupt (int irq,void *data) {struct adapter *adapter = data; -struct net_device *dev = adapter->sge->netdev;struct sge *sge = adapter->sge; -u32 cause; -int handled =0 ; +int handled; - cause =readl (adapter->regs +A_PL_CAUSE ); -if (cause ==0 || cause == ~0 ) -return IRQ_NONE ; +if (likely (responses_pending (adapter))) { +struct net_device *dev = sge->netdev; -spin_lock (&adapter->async_lock); -if (cause &F_PL_INTR_SGE_DATA ) { -struct respQ *q = &adapter->sge->respQ; -struct respQ_e *e = &q->entries[q->cidx]; - - handled =1 ; -writel (F_PL_INTR_SGE_DATA , adapter->regs +A_PL_CAUSE ); - -if (e->GenerationBit == q->genbit && -__netif_rx_schedule_prep (dev)) { -if (e->DataValid ||process_pure_responses (adapter, e)) { -/* mask off data IRQ */ -writel (adapter->slow_intr_mask, - adapter->regs +A_PL_ENABLE ); -__netif_rx_schedule (sge->netdev); - goto unlock; - } -/* no data, no NAPI needed */ -netif_poll_enable (dev); +writel (F_PL_INTR_SGE_DATA , adapter->regs +A_PL_CAUSE ); +if (__netif_rx_schedule_prep (dev)) { +if (process_pure_responses (adapter)) { +__netif_rx_schedule (dev); + }else { +/* no data, no NAPI needed */ +writel (sge->respQ.cidx, + adapter->regs +A_SG_SLEEPING ); +netif_poll_enable (dev);/* undo schedule_prep */ + } } -writel (q->cidx, adapter->regs +A_SG_SLEEPING ); - }else - handled =t1_slow_intr_handler (adapter); +return IRQ_HANDLED ; + } + +spin_lock (&adapter->async_lock); + handled =t1_slow_intr_handler (adapter); +spin_unlock (&adapter->async_lock);if (!handled) sge->stats.unhandled_irqs++; -unlock: -spin_unlock (&adapter->async_lock); +return IRQ_RETVAL (handled !=0 ); }
@@ -1712,17 +1708,13 @@ unlock:irqreturn_t t1_interrupt (int irq,void *cookie) {int work_done; -struct respQ_e *e;struct adapter *adapter = cookie; -struct respQ *Q = &adapter->sge->respQ;spin_lock (&adapter->async_lock); - e = &Q ->entries[Q ->cidx]; -prefetch (e);writel (F_PL_INTR_SGE_DATA , adapter->regs +A_PL_CAUSE ); -if (likely (e->GenerationBit ==Q ->genbit)) + if (likely(responses_pending(adapter)) work_done =process_responses (adapter,-1 );else work_done =t1_slow_intr_handler (adapter);
PATCH 3/3 chelsio: more receive cleanup
In reply to Stephen HemmingerCleanup receive processing some more:
* do the reserve padding of skb during setup
* don't pass constants to get_packet
* do smart prefetch of skb
* make copybreak a module parameter
--- drivers/net/chelsio/sge.c | 81 +++++++++++++++++++++++----------------------- 1 file changed, 42 insertions(+), 39 deletions(-)
drivers/net/chelsio/sge.c +42 -39
@@ -71,12 +71,9 @@#define SGE_FREEL_REFILL_THRESH 16#define SGE_RESPQ_E_N 1024#define SGE_INTRTIMER_NRES 1000 -#define SGE_RX_COPY_THRES 256#define SGE_RX_SM_BUF_SIZE 1536#define SGE_TX_DESC_MAX_PLEN 16384 -# define SGE_RX_DROP_THRES 2 -#define SGE_RESPQ_REPLENISH_THRES (SGE_RESPQ_E_N / 4)/*
@@ -862,6 +859,8 @@ static void refill_free_list(struct sgeskb_reserve (skb, q->dma_offset); mapping =pci_map_single (pdev, skb->data, dma_len,PCI_DMA_FROMDEVICE ); +skb_reserve (skb, sge->rx_pkt_pad); + ce->skb = skb;pci_unmap_addr_set (ce, dma_addr, mapping);pci_unmap_len_set (ce, dma_len, dma_len);
@@ -1041,6 +1040,10 @@ static void recycle_fl_buf(struct freelQ } } +static int copybreak__read_mostly =256 ; +module_param (copybreak, int,0 ); +MODULE_PARM_DESC (copybreak,"Receive copy threshold" ); +/** * get_packet - return the next ingress packet buffer * @pdev: the PCI device that received the packet
@@ -1059,37 +1062,33 @@ static void recycle_fl_buf(struct freelQ* threshold and the packet is too big to copy, or (b) the packet should * be copied but there is no memory for the copy. */ -static inline struct sk_buff *get_packet (struct pci_dev *pdev, -struct freelQ *fl,unsigned int len, -int dma_pad,int skb_pad, -unsigned int copy_thres, -unsigned int drop_thres) +static inline struct sk_buff *get_packet (struct pci_dev *pdev,struct freelQ *fl, +unsigned int len) {struct sk_buff *skb; -struct freelQ_ce *ce = &fl->centries[fl->cidx]; +const struct freelQ_ce *ce = &fl->centries[fl->cidx]; + +if (len < copybreak) { + skb =alloc_skb (len +2 ,GFP_ATOMIC ); +if (!skb) + goto use_orig_buf; -if (len < copy_thres) { - skb =alloc_skb (len + skb_pad,GFP_ATOMIC ); -if (likely (skb !=NULL )) { -skb_reserve (skb, skb_pad); -skb_put (skb, len); -pci_dma_sync_single_for_cpu (pdev, +skb_reserve (skb,2 );/* align IP header */ +skb_put (skb, len); +pci_dma_sync_single_for_cpu (pdev,pci_unmap_addr (ce, dma_addr),pci_unmap_len (ce, dma_len),PCI_DMA_FROMDEVICE ); -memcpy (skb->data, ce->skb->data + dma_pad, len); -pci_dma_sync_single_for_device (pdev, +memcpy (skb->data, ce->skb->data, len); +pci_dma_sync_single_for_device (pdev,pci_unmap_addr (ce, dma_addr),pci_unmap_len (ce, dma_len),PCI_DMA_FROMDEVICE ); - }else if (!drop_thres) - goto use_orig_buf; -recycle_fl_buf (fl, fl->cidx);return skb; } -if (fl->credits < drop_thres) { +if (fl->credits <2 ) {recycle_fl_buf (fl, fl->cidx);return NULL ; }
@@ -1098,7 +1097,8 @@ use_orig_buf:pci_unmap_single (pdev,pci_unmap_addr (ce, dma_addr),pci_unmap_len (ce, dma_len),PCI_DMA_FROMDEVICE ); skb = ce->skb; -skb_reserve (skb, dma_pad); +prefetch (skb->data); +skb_put (skb, len);return skb; }
@@ -1375,27 +1375,25 @@ static void restart_sched(unsigned long* * Process an ingress ethernet pakcet and deliver it to the stack. */ -static int sge_rx (struct sge *sge,struct freelQ *fl,unsigned int len) +static void sge_rx (struct sge *sge,struct freelQ *fl,unsigned int len) {struct sk_buff *skb; -struct cpl_rx_pkt *p; +const struct cpl_rx_pkt *p;struct adapter *adapter = sge->adapter;struct sge_port_stats *st; - skb =get_packet (adapter->pdev, fl, len - sge->rx_pkt_pad, - sge->rx_pkt_pad,2 ,SGE_RX_COPY_THRES , -SGE_RX_DROP_THRES ); + skb =get_packet (adapter->pdev, fl, len - sge->rx_pkt_pad);if (unlikely (!skb)) { sge->stats.rx_drops++; -return 0 ; +return ; } - p = (struct cpl_rx_pkt *)skb->data; -skb_pull (skb,sizeof (*p)); + p = (const struct cpl_rx_pkt *) skb->data;if (p->iff >= adapter->params.nports) {kfree_skb (skb); -return 0 ; +return ; } +__skb_pull (skb,sizeof (*p)); skb->dev = adapter->port[p->iff].dev; skb->dev->last_rx = jiffies;
@@ -1427,7 +1425,6 @@ static int sge_rx(struct sge *sge, strucnetif_rx (skb);#endif } -return 0 ; }/*
@@ -1510,12 +1507,11 @@ static int process_responses(struct adapstruct sge *sge = adapter->sge;struct respQ *q = &sge->respQ;struct respQ_e *e = &q->entries[q->cidx]; -int budget_left = budget; +int done =0 ;unsigned int flags =0 ;unsigned int cmdq_processed[SGE_CMDQ_N ] = {0 ,0 }; - -while (likely (budget_left && e->GenerationBit == q->genbit)) { +while (done < budget && e->GenerationBit == q->genbit) { flags |= e->Qsleeping; cmdq_processed[0 ] += e->Cmdq0CreditReturn;
@@ -1525,14 +1521,16 @@ static int process_responses(struct adap* ping-pong of TX state information on MP where the sender * might run on a different CPU than this function... */ -if (unlikely (flags &F_CMDQ0_ENABLE || cmdq_processed[0 ] >64 )) { +if (unlikely ((flags &F_CMDQ0_ENABLE ) || cmdq_processed[0 ] >64 )) { flags =update_tx_info (adapter, flags, cmdq_processed[0 ]); cmdq_processed[0 ] =0 ; } +if (unlikely (cmdq_processed[1 ] >16 )) { sge->cmdQ[1 ].processed += cmdq_processed[1 ]; cmdq_processed[1 ] =0 ; } +if (likely (e->DataValid)) {struct freelQ *fl = &sge->freelQ[e->FreelistQid];
@@ -1542,12 +1540,16 @@ static int process_responses(struct adapelse sge_rx (sge, fl, e->BufferLength); + ++done; +/* * Note: this depends on each packet consuming a * single free-list buffer; cf. the BUG above. */ if (++fl->cidx == fl->size) fl->cidx =0 ; +prefetch (fl->centries[fl->cidx].skb); +if (unlikely (--fl->credits < fl->size -SGE_FREEL_REFILL_THRESH ))refill_free_list (sge, fl);
@@ -1566,14 +1568,12 @@ static int process_responses(struct adapwritel (q->credits, adapter->regs +A_SG_RSPQUEUECREDIT ); q->credits =0 ; } - --budget_left; } flags =update_tx_info (adapter, flags, cmdq_processed[0 ]); sge->cmdQ[1 ].processed += cmdq_processed[1 ]; - budget -= budget_left; -return budget; +return done; }static inline int responses_pending (const struct adapter *adapter)
@@ -1598,11 +1598,14 @@ static int process_pure_responses(structstruct sge *sge = adapter->sge;struct respQ *q = &sge->respQ;struct respQ_e *e = &q->entries[q->cidx]; +const struct freelQ *fl = &sge->freelQ[e->FreelistQid];unsigned int flags =0 ;unsigned int cmdq_processed[SGE_CMDQ_N ] = {0 ,0 }; +prefetch (fl->centries[fl->cidx].skb);if (e->DataValid)return 1 ; + do { flags |= e->Qsleeping;
Re: [PATCH 3/3] chelsio: more receive cleanup
Stephen Hemminger <[email protected]> :
[...]
Quoted 2 lines
drivers/net/chelsio/sge.c +0
-0
[...]
Quoted 7 lines
drivers/net/chelsio/sge.c +0
-3@@ -1059,37 +1062,33 @@ static void recycle_fl_buf(struct freelQ* threshold and the packet is too big to copy, or (b) the packet should * be copied but there is no memory for the copy. */ -static inline struct sk_buff *get_packet(struct pci_dev *pdev, - struct freelQ *fl, unsigned int len, - int dma_pad, int skb_pad,
drivers/net/chelsio/sge.c static void recycle_fl_buf(struct freelQ · line 1071
Quoted 10 lines
drivers/net/chelsio/sge.c +5
-3-unsigned int copy_thres, - unsigned int drop_thres) +static inline struct sk_buff *get_packet (struct pci_dev *pdev,struct freelQ *fl, +unsigned int len) {struct sk_buff *skb; -struct freelQ_ce *ce = &fl->centries[fl->cidx]; +const struct freelQ_ce *ce = &fl->centries[fl->cidx]; + +if (len < copybreak) {
If you are into cleanups, maybe add likely/unlikely (and remove the inline) ?
drivers/net/chelsio/sge.c static void recycle_fl_buf(struct freelQ · line 1076
Quoted 11 lines
drivers/net/chelsio/sge.c +4
-6+ skb =alloc_skb (len +2 ,GFP_ATOMIC ); +if (!skb) + goto use_orig_buf; -if (len < copy_thres) { - skb =alloc_skb (len + skb_pad,GFP_ATOMIC ); -if (likely (skb !=NULL )) { -skb_reserve (skb, skb_pad); -skb_put (skb, len); - pci_dma_sync_single_for_cpu(pdev, +skb_reserve (skb,2 );/* align IP header */
s/2/NET_IP_ALIGN/
Btw, since the driver supports netpoll, its blind enabling of interrupts in t1_poll() seems old-fashoined (see thread starting at
http://lkml.org/lkml/2006/12/12/86)
Signature · 1 line
Ueimor
Re: [PATCH 3/3] chelsio: more receive cleanup
Quoted 16 lines · On Fri, 15 Dec 2006 21:10:19 +0100 Francois Romieu <[email protected]> wrote:
Stephen Hemminger <[email protected]> :
[...]Quoted 2 lines
drivers/net/chelsio/sge.c +0
-0[...]
Quoted 11 lines
drivers/net/chelsio/sge.c +2
-5@@ -1059,37 +1062,33 @@ static void recycle_fl_buf(struct freelQ* threshold and the packet is too big to copy, or (b) the packet should * be copied but there is no memory for the copy. */ -static inline struct sk_buff *get_packet (struct pci_dev *pdev, -struct freelQ *fl,unsigned int len, -int dma_pad,int skb_pad, -unsigned int copy_thres, -unsigned int drop_thres) +static inline struct sk_buff *get_packet (struct pci_dev *pdev,struct freelQ *fl, +unsigned int len)
Quoted 8 lines
drivers/net/chelsio/sge.c static void recycle_fl_buf(struct freelQ · line 1071
Quoted 6 lines
drivers/net/chelsio/sge.c +3
-1{struct sk_buff *skb; -struct freelQ_ce *ce = &fl->centries[fl->cidx]; +const struct freelQ_ce *ce = &fl->centries[fl->cidx]; + +if (len < copybreak) {If you are into cleanups, maybe add likely/unlikely (and remove the inline) ?
Why, you can't predict copybreak on off
Quoted 5 lines
Quoted 5 lines
drivers/net/chelsio/sge.c +3
-0+ skb =alloc_skb (len +2 ,GFP_ATOMIC ); +if (!skb) + goto use_orig_buf;
Quoted 9 lines
drivers/net/chelsio/sge.c static void recycle_fl_buf(struct freelQ · line 1076
Quoted 7 lines
drivers/net/chelsio/sge.c +1
-6-if (len < copy_thres) { - skb =alloc_skb (len + skb_pad,GFP_ATOMIC ); -if (likely (skb !=NULL )) { -skb_reserve (skb, skb_pad); -skb_put (skb, len); - pci_dma_sync_single_for_cpu(pdev, +skb_reserve (skb,2 );/* align IP header */ s/2/NET_IP_ALIGN/
This is wrong, NET_IP_ALIGN is for DMA buffers only. It is defined as 0 for those platforms where DMA alignment is important. In this case we are copying, so we want to force 2.
Quoted 4 lines
Btw, since the driver supports netpoll, its blind enabling of interrupts in
t1_poll()seems old-fashoined (see thread starting at
http://lkml.org/lkml/2006/12/12/86)
Signature · 1 line
Stephen Hemminger <[email protected]>
Re: [PATCH 3/3] chelsio: more receive cleanup
Stephen Hemminger <[email protected]> :
Quoted 1 line
Francois Romieu <[email protected]> wrote:
[...]
Quoted 1 line
Why, you can't predict copybreak on off
Plain gut feeling. I'd rather spend extra cycles on small packets (whose allocation is already optimized) and keep the normal packets less expensive on a 10Gb/s network card.
[...]
Quoted 3 lines
This is wrong,
NET_IP_ALIGNis for DMA buffers only. It is defined as 0 for those platforms where DMA alignment is important. In this case we are copying, so we want to force 2.
Ok, ok.
Signature · 1 line
Ueimor
Re: [PATCH 3/3] chelsio: more receive cleanup
Quoted 9 lines · On Sat, 16 Dec 2006 01:26:04 +0100 Francois Romieu <[email protected]> wrote:
Stephen Hemminger <[email protected]> :
Quoted 1 line
Francois Romieu <[email protected]> wrote:
[...]
Quoted 1 line
Why, you can't predict copybreak on off
Plain gut feeling. I'd rather spend extra cycles on small packets (whose allocation is already optimized) and keep the normal packets less expensive on a 10Gb/s network card.
Likely/unlikely is an optimization that doesn't make a big difference and I would rather the compiler choose unless it is something obvious like an error path or external interrupt (like PHY).
There was some discussion on LKML, which I mostly forgot, that hinted that on some architectures using likely/unlikely incorrectly could have a major hit.
Signature · 1 line
Stephen Hemminger <[email protected]>
Quoted 3 lines · Stephen Hemminger
One error path bug fix, and a couple of changes to improve receive performance. The performance is now up to 5+ Gbits/sec which is about the limit of the memory/bus on the test systems.
I request a rediff+resend against netdev-2.6.git#upstream, giving sufficient time [hours :(] for it to mirror out.
Jeff
PATCH 1/3 chelsio: error path fix
Fix handling of allocation failure.
drivers/net/chelsio/my3126.c +3 -2
@@ -170,9 +170,10 @@ static struct cphy *my3126_phy_create(ad {struct cphy *cphy =kzalloc (sizeof (*cphy),GFP_KERNEL ); -if (cphy) -cphy_init (cphy, adapter, phy_addr, &my3126_ops, mdio_ops); +if (!cphy) +return NULL ; +cphy_init (cphy, adapter, phy_addr, &my3126_ops, mdio_ops);INIT_DELAYED_WORK (&cphy->phy_update, my3216_poll); cphy->bmsr =0 ;
A message that is not in the archive (sent off-list or not yet received): [email protected]
PATCH 3/3 chelsio: more rx speedup
Cleanup receive processing some more:
* do the reserve padding of skb during setup
* don't pass constants to get_packet
* do smart prefetch of skb
* make copybreak a module parameter
--- drivers/net/chelsio/sge.c | 87 +++++++++++++++++++++++----------------------- 1 file changed, 45 insertions(+), 42 deletions(-)
drivers/net/chelsio/sge.c +45 -42
@@ -71,12 +71,9 @@#define SGE_FREEL_REFILL_THRESH 16#define SGE_RESPQ_E_N 1024#define SGE_INTRTIMER_NRES 1000 -#define SGE_RX_COPY_THRES 256#define SGE_RX_SM_BUF_SIZE 1536#define SGE_TX_DESC_MAX_PLEN 16384 -# define SGE_RX_DROP_THRES 2 -#define SGE_RESPQ_REPLENISH_THRES (SGE_RESPQ_E_N / 4)/*
@@ -846,6 +843,8 @@ static void refill_free_list(struct sgeskb_reserve (skb, q->dma_offset); mapping =pci_map_single (pdev, skb->data, dma_len,PCI_DMA_FROMDEVICE ); +skb_reserve (skb, sge->rx_pkt_pad); + ce->skb = skb;pci_unmap_addr_set (ce, dma_addr, mapping);pci_unmap_len_set (ce, dma_len, dma_len);
@@ -1024,6 +1023,10 @@ static void recycle_fl_buf(struct freelQ } } +static int copybreak__read_mostly =256 ; +module_param (copybreak, int,0 ); +MODULE_PARM_DESC (copybreak,"Receive copy threshold" ); +/** * get_packet - return the next ingress packet buffer * @pdev: the PCI device that received the packet
@@ -1043,45 +1046,42 @@ static void recycle_fl_buf(struct freelQ* be copied but there is no memory for the copy. */ static inline struct sk_buff *get_packet (struct pci_dev *pdev, -struct freelQ *fl,unsigned int len, -int dma_pad,int skb_pad, -unsigned int copy_thres, -unsigned int drop_thres) +struct freelQ *fl,unsigned int len) {struct sk_buff *skb; -struct freelQ_ce *ce = &fl->centries[fl->cidx]; +const struct freelQ_ce *ce = &fl->centries[fl->cidx]; -if (len < copy_thres) { - skb =alloc_skb (len + skb_pad,GFP_ATOMIC ); -if (likely (skb !=NULL )) { -skb_reserve (skb, skb_pad); -skb_put (skb, len); -pci_dma_sync_single_for_cpu (pdev, -pci_unmap_addr (ce, dma_addr), -pci_unmap_len (ce, dma_len), -PCI_DMA_FROMDEVICE ); -memcpy (skb->data, ce->skb->data + dma_pad, len); -pci_dma_sync_single_for_device (pdev, +if (len < copybreak) { + skb =alloc_skb (len +2 ,GFP_ATOMIC ); +if (!skb) + goto use_orig_buf; + +skb_reserve (skb,2 );/* align IP header */ +skb_put (skb, len); +pci_dma_sync_single_for_cpu (pdev,pci_unmap_addr (ce, dma_addr),pci_unmap_len (ce, dma_len),PCI_DMA_FROMDEVICE ); - }else if (!drop_thres) - goto use_orig_buf; - +memcpy (skb->data, ce->skb->data, len); +pci_dma_sync_single_for_device (pdev, +pci_unmap_addr (ce, dma_addr), +pci_unmap_len (ce, dma_len), +PCI_DMA_FROMDEVICE );recycle_fl_buf (fl, fl->cidx);return skb; } -if (fl->credits < drop_thres) { +use_orig_buf: +if (fl->credits <2 ) {recycle_fl_buf (fl, fl->cidx);return NULL ; } -use_orig_buf:pci_unmap_single (pdev,pci_unmap_addr (ce, dma_addr),pci_unmap_len (ce, dma_len),PCI_DMA_FROMDEVICE ); skb = ce->skb; -skb_reserve (skb, dma_pad); +prefetch (skb->data); +skb_put (skb, len);return skb; }
@@ -1359,27 +1359,25 @@ static void restart_sched(unsigned long* * Process an ingress ethernet pakcet and deliver it to the stack. */ -static int sge_rx (struct sge *sge,struct freelQ *fl,unsigned int len) +static void sge_rx (struct sge *sge,struct freelQ *fl,unsigned int len) {struct sk_buff *skb; -struct cpl_rx_pkt *p; +const struct cpl_rx_pkt *p;struct adapter *adapter = sge->adapter;struct sge_port_stats *st; - skb =get_packet (adapter->pdev, fl, len - sge->rx_pkt_pad, - sge->rx_pkt_pad,2 ,SGE_RX_COPY_THRES , -SGE_RX_DROP_THRES ); + skb =get_packet (adapter->pdev, fl, len - sge->rx_pkt_pad);if (unlikely (!skb)) { sge->stats.rx_drops++; -return 0 ; +return ; } - p = (struct cpl_rx_pkt *)skb->data; -skb_pull (skb,sizeof (*p)); + p = (const struct cpl_rx_pkt *) skb->data;if (p->iff >= adapter->params.nports) {kfree_skb (skb); -return 0 ; +return ; } +__skb_pull (skb,sizeof (*p)); skb->dev = adapter->port[p->iff].dev; skb->dev->last_rx = jiffies;
@@ -1411,7 +1409,6 @@ static int sge_rx(struct sge *sge, strucnetif_rx (skb);#endif } -return 0 ; }/*
@@ -1493,12 +1490,11 @@ static int process_responses(struct adapstruct sge *sge = adapter->sge;struct respQ *q = &sge->respQ;struct respQ_e *e = &q->entries[q->cidx]; -int budget_left = budget; +int done =0 ;unsigned int flags =0 ;unsigned int cmdq_processed[SGE_CMDQ_N ] = {0 ,0 }; - -while (likely (budget_left && e->GenerationBit == q->genbit)) { +while (done < budget && e->GenerationBit == q->genbit) { flags |= e->Qsleeping; cmdq_processed[0 ] += e->Cmdq0CreditReturn;
@@ -1508,14 +1504,16 @@ static int process_responses(struct adap* ping-pong of TX state information on MP where the sender * might run on a different CPU than this function... */ -if (unlikely (flags &F_CMDQ0_ENABLE || cmdq_processed[0 ] >64 )) { +if (unlikely ((flags &F_CMDQ0_ENABLE ) || cmdq_processed[0 ] >64 )) { flags =update_tx_info (adapter, flags, cmdq_processed[0 ]); cmdq_processed[0 ] =0 ; } +if (unlikely (cmdq_processed[1 ] >16 )) { sge->cmdQ[1 ].processed += cmdq_processed[1 ]; cmdq_processed[1 ] =0 ; } +if (likely (e->DataValid)) {struct freelQ *fl = &sge->freelQ[e->FreelistQid];
@@ -1525,12 +1523,16 @@ static int process_responses(struct adapelse sge_rx (sge, fl, e->BufferLength); + ++done; +/* * Note: this depends on each packet consuming a * single free-list buffer; cf. the BUG above. */ if (++fl->cidx == fl->size) fl->cidx =0 ; +prefetch (fl->centries[fl->cidx].skb); +if (unlikely (--fl->credits < fl->size -SGE_FREEL_REFILL_THRESH ))refill_free_list (sge, fl);
@@ -1549,14 +1551,12 @@ static int process_responses(struct adapwritel (q->credits, adapter->regs +A_SG_RSPQUEUECREDIT ); q->credits =0 ; } - --budget_left; } flags =update_tx_info (adapter, flags, cmdq_processed[0 ]); sge->cmdQ[1 ].processed += cmdq_processed[1 ]; - budget -= budget_left; -return budget; +return done; }static inline int responses_pending (const struct adapter *adapter)
@@ -1581,11 +1581,14 @@ static int process_pure_responses(structstruct sge *sge = adapter->sge;struct respQ *q = &sge->respQ;struct respQ_e *e = &q->entries[q->cidx]; +const struct freelQ *fl = &sge->freelQ[e->FreelistQid];unsigned int flags =0 ;unsigned int cmdq_processed[SGE_CMDQ_N ] = {0 ,0 }; +prefetch (fl->centries[fl->cidx].skb);if (e->DataValid)return 1 ; + do { flags |= e->Qsleeping;
Re: [PATCH 3/3] chelsio: more rx speedup
Hi Stephen,
Quoted 2 lines · Stephen Hemminger
drivers/net/chelsio/sge.c +0
-0
[...]
Quoted 21 lines
drivers/net/chelsio/sge.c +2
-12@@ -1043,45 +1046,42 @@ static void recycle_fl_buf(struct freelQ* be copied but there is no memory for the copy. */ static inline struct sk_buff *get_packet (struct pci_dev *pdev, -struct freelQ *fl,unsigned int len, -int dma_pad,int skb_pad, -unsigned int copy_thres, -unsigned int drop_thres) +struct freelQ *fl,unsigned int len) {struct sk_buff *skb; -struct freelQ_ce *ce = &fl->centries[fl->cidx]; +const struct freelQ_ce *ce = &fl->centries[fl->cidx]; -if (len < copy_thres) { - skb =alloc_skb (len + skb_pad,GFP_ATOMIC ); -if (likely (skb !=NULL )) { -skb_reserve (skb, skb_pad); -skb_put (skb, len); - pci_dma_sync_single_for_cpu(pdev, - pci_unmap_addr(ce, dma_addr),
drivers/net/chelsio/sge.c static void recycle_fl_buf(struct freelQ · line 1059
Quoted 10 lines
drivers/net/chelsio/sge.c +6
-4-pci_unmap_len (ce, dma_len), - PCI_DMA_FROMDEVICE); -memcpy (skb->data, ce->skb->data + dma_pad, len); - pci_dma_sync_single_for_device(pdev, +if (len < copybreak) { + skb =alloc_skb (len +2 ,GFP_ATOMIC ); +if (!skb) + goto use_orig_buf; + +skb_reserve (skb,2 );/* align IP header */
Please use NET_IP_ALIGN here:
+ skb = alloc_skb(len + NET_IP_ALIGN, GFP_ATOMIC);
+ if (!skb)
+ goto use_orig_buf;
+
+ skb_reserve(skb, NET_IP_ALIGN);
Quoted 10 lines
drivers/net/chelsio/sge.c +4
-3+skb_put (skb, len); +pci_dma_sync_single_for_cpu (pdev,pci_unmap_addr (ce, dma_addr),pci_unmap_len (ce, dma_len),PCI_DMA_FROMDEVICE ); - }else if (!drop_thres) - goto use_orig_buf; - +memcpy (skb->data, ce->skb->data, len); + pci_dma_sync_single_for_device(pdev,
drivers/net/chelsio/sge.c static void recycle_fl_buf(struct freelQ · line 1075
Quoted 10 lines
drivers/net/chelsio/sge.c +5
-1+pci_unmap_addr (ce, dma_addr), +pci_unmap_len (ce, dma_len), + PCI_DMA_FROMDEVICE);recycle_fl_buf (fl, fl->cidx);return skb; } -if (fl->credits < drop_thres) { +use_orig_buf: +if (fl->credits <2 ) {
Why 2? What does this magic number mean?
Regards
Ingo Oeser
Re: [PATCH 3/3] chelsio: more rx speedup
Quoted 31 lines · On Tue, 9 Jan 2007 09:42:03 +0100 Ingo Oeser <[email protected]> wrote:
Hi Stephen,
Quoted 2 lines · Stephen Hemminger
drivers/net/chelsio/sge.c +0
-0[...]
Quoted 25 lines
drivers/net/chelsio/sge.c +2
-16@@ -1043,45 +1046,42 @@ static void recycle_fl_buf(struct freelQ* be copied but there is no memory for the copy. */ static inline struct sk_buff *get_packet (struct pci_dev *pdev, -struct freelQ *fl,unsigned int len, -int dma_pad,int skb_pad, -unsigned int copy_thres, -unsigned int drop_thres) +struct freelQ *fl,unsigned int len) {struct sk_buff *skb; -struct freelQ_ce *ce = &fl->centries[fl->cidx]; +const struct freelQ_ce *ce = &fl->centries[fl->cidx]; -if (len < copy_thres) { - skb =alloc_skb (len + skb_pad,GFP_ATOMIC ); -if (likely (skb !=NULL )) { -skb_reserve (skb, skb_pad); -skb_put (skb, len); -pci_dma_sync_single_for_cpu (pdev, -pci_unmap_addr (ce, dma_addr), -pci_unmap_len (ce, dma_len), -PCI_DMA_FROMDEVICE ); -memcpy (skb->data, ce->skb->data + dma_pad, len); - pci_dma_sync_single_for_device(pdev,
Quoted 8 lines
drivers/net/chelsio/sge.c static void recycle_fl_buf(struct freelQ · line 1059
Quoted 6 lines
drivers/net/chelsio/sge.c +6
-0+if (len < copybreak) { + skb =alloc_skb (len +2 ,GFP_ATOMIC ); +if (!skb) + goto use_orig_buf; + +skb_reserve (skb,2 );/* align IP header */ Please use
NET_IP_ALIGNhere:
Wrong, NET_IP_ALIGN is intended to deal with platforms where alignment of DMA is more important of alignment of structures. Therefore if data is copied, it should always be 2.
Quoted 24 lines
drivers/net/chelsio/sge.c +5
-0+ skb =alloc_skb (len +NET_IP_ALIGN ,GFP_ATOMIC ); +if (!skb) + goto use_orig_buf; + +skb_reserve (skb,NET_IP_ALIGN );Quoted 18 lines
drivers/net/chelsio/sge.c +7
-3+skb_put (skb, len); +pci_dma_sync_single_for_cpu (pdev,pci_unmap_addr (ce, dma_addr),pci_unmap_len (ce, dma_len),PCI_DMA_FROMDEVICE ); - }else if (!drop_thres) - goto use_orig_buf; - +memcpy (skb->data, ce->skb->data, len); +pci_dma_sync_single_for_device (pdev, +pci_unmap_addr (ce, dma_addr), +pci_unmap_len (ce, dma_len), +PCI_DMA_FROMDEVICE );recycle_fl_buf (fl, fl->cidx);return skb; }
Quoted 5 lines
drivers/net/chelsio/sge.c static void recycle_fl_buf(struct freelQ · line 1075
Quoted 3 lines
drivers/net/chelsio/sge.c +2
-1-if (fl->credits < drop_thres) { +use_orig_buf: +if (fl->credits <2 ) {Why 2? What does this magic number mean?
No idea, it was there in the original. (as a parameter).
Signature · 1 line
Stephen Hemminger <[email protected]>
Re: [PATCH 3/3] chelsio: more rx speedup
Quoted 77 lines · Stephen Hemminger
Quoted 34 lines · On Tue, 9 Jan 2007 09:42:03 +0100 Ingo Oeser <[email protected]> wrote:
Hi Stephen,
Quoted 3 lines · Stephen Hemminger
drivers/net/chelsio/sge.c +0
-0[...]
Quoted 25 lines
drivers/net/chelsio/sge.c +2
-16@@ -1043,45 +1046,42 @@ static void recycle_fl_buf(struct freelQ* be copied but there is no memory for the copy. */ static inline struct sk_buff *get_packet (struct pci_dev *pdev, -struct freelQ *fl,unsigned int len, -int dma_pad,int skb_pad, -unsigned int copy_thres, -unsigned int drop_thres) +struct freelQ *fl,unsigned int len) {struct sk_buff *skb; -struct freelQ_ce *ce = &fl->centries[fl->cidx]; +const struct freelQ_ce *ce = &fl->centries[fl->cidx]; -if (len < copy_thres) { - skb =alloc_skb (len + skb_pad,GFP_ATOMIC ); -if (likely (skb !=NULL )) { -skb_reserve (skb, skb_pad); -skb_put (skb, len); -pci_dma_sync_single_for_cpu (pdev, -pci_unmap_addr (ce, dma_addr), -pci_unmap_len (ce, dma_len), -PCI_DMA_FROMDEVICE ); -memcpy (skb->data, ce->skb->data + dma_pad, len); - pci_dma_sync_single_for_device(pdev,Quoted 9 lines
drivers/net/chelsio/sge.c static void recycle_fl_buf(struct freelQ · line 1073
Quoted 7 lines
drivers/net/chelsio/sge.c +6
-0+if (len < copybreak) { + skb =alloc_skb (len +2 ,GFP_ATOMIC ); +if (!skb) + goto use_orig_buf; + +skb_reserve (skb,2 );/* align IP header */ Please use
NET_IP_ALIGNhere:Wrong,
NET_IP_ALIGNis intended to deal with platforms where alignment of DMA is more important of alignment of structures. Therefore if data is copied, it should always be 2.Quoted 24 lines
drivers/net/chelsio/sge.c +5
-0+ skb =alloc_skb (len +NET_IP_ALIGN ,GFP_ATOMIC ); +if (!skb) + goto use_orig_buf; + +skb_reserve (skb,NET_IP_ALIGN );Quoted 19 lines
drivers/net/chelsio/sge.c +7
-3+skb_put (skb, len); +pci_dma_sync_single_for_cpu (pdev,pci_unmap_addr (ce, dma_addr),pci_unmap_len (ce, dma_len),PCI_DMA_FROMDEVICE ); - }else if (!drop_thres) - goto use_orig_buf; - +memcpy (skb->data, ce->skb->data, len); +pci_dma_sync_single_for_device (pdev, +pci_unmap_addr (ce, dma_addr), +pci_unmap_len (ce, dma_len), +PCI_DMA_FROMDEVICE );recycle_fl_buf (fl, fl->cidx);return skb; }
Quoted 10 lines
Quoted 6 lines
drivers/net/chelsio/sge.c static void recycle_fl_buf(struct freelQ · line 1079
Quoted 4 lines
drivers/net/chelsio/sge.c +2
-1-if (fl->credits < drop_thres) { +use_orig_buf: +if (fl->credits <2 ) {Why 2? What does this magic number mean?
No idea, it was there in the original. (as a parameter).
The T2 HW behaves nicely when it is guaranteed to have 2 available entries in the rx free list.
Cheers,
Divy
Re: [PATCH 3/3] chelsio: more rx speedup
Quoted 4 lines · Divy Le Ray
Quoted 3 lines · Stephen Hemminger
Quoted 1 line · On Tue, 9 Jan 2007 09:42:03 +0100 Ingo Oeser <[email protected]> wrote:
Stephen Hemminger schrieb:
Quoted 8 lines
Quoted 6 lines
Quoted 5 lines
drivers/net/chelsio/sge.c static void recycle_fl_buf(struct freelQ · line 1075
Quoted 4 lines
drivers/net/chelsio/sge.c +2
-1-if (fl->credits < drop_thres) { +use_orig_buf: +if (fl->credits <2 ) {Why 2? What does this magic number mean?
No idea, it was there in the original. (as a parameter).
The T2 HW behaves nicely when it is guaranteed to have 2 available entries in the rx free list.
Can we have a #define for this?
That would help people understand this issue more.
And don't be shy about errata, all hardware and software out there has them like the humans that made them :-)
Regards
Ingo Oeser
Re: [PATCH 3/3] chelsio: more rx speedup
In reply to Stephen HemmingerQuoted 10 lines · Stephen Hemminger
Quoted 4 lines · On Tue, 9 Jan 2007 09:42:03 +0100 Ingo Oeser <[email protected]> wrote:
drivers/net/chelsio/sge.c
Quoted 2 lines · Stephen Hemminger
drivers/net/chelsio/sge.c +0
-0Please use
NET_IP_ALIGNhere:Wrong,
NET_IP_ALIGNis intended to deal with platforms where alignment of DMA is more important of alignment of structures. Therefore if data is copied, it should always be 2.
Ah! Thanks for clearing this up. These magic numbers always make my head spin so I tried to come up with sth. useful :-)
If this pattern is present in many drivers, we might have a define or helper for this.
Regards
Ingo Oeser
Jeff Garzik
AppliedRe: [PATCH 3/3] chelsio: more rx speedup
In reply to Stephen HemmingerQuoted 7 lines · Stephen Hemminger
Cleanup receive processing some more:
* do the reserve padding of skb during setup
* don't pass constants toget_packet
* do smart prefetch of skb
* make copybreak a module parameterSigned-off-by Stephen Hemminger <[email protected]>
applied
PATCH 2/3 chelsio: NAPI speed improvement
In reply to Stephen HemmingerSpeedup and cleanup the receive processing by eliminating the mmio read and a lock round trip.
--- drivers/net/chelsio/sge.c | 77 ++++++++++++++++++++-------------------------- 1 file changed, 35 insertions(+), 42 deletions(-)
drivers/net/chelsio/sge.c +35 -42
@@ -1559,6 +1559,14 @@ static int process_responses(struct adapreturn budget; } +static inline int responses_pending (const struct adapter *adapter) +{ +const struct respQ *Q = &adapter->sge->respQ; +const struct respQ_e *e = &Q ->entries[Q ->cidx]; + +return (e->GenerationBit ==Q ->genbit); +} +#ifdef CONFIG_CHELSIO_T1_NAPI /* * A simpler version of process_responses() that handles only pure (i.e.,
@@ -1568,13 +1576,16 @@ static int process_responses(struct adap* which the caller must ensure is a valid pure response. Returns 1 if it * encounters a valid data-carrying response, 0 otherwise. */ -static int process_pure_responses (struct adapter *adapter,struct respQ_e *e) +static int process_pure_responses (struct adapter *adapter) {struct sge *sge = adapter->sge;struct respQ *q = &sge->respQ; +struct respQ_e *e = &q->entries[q->cidx];unsigned int flags =0 ;unsigned int cmdq_processed[SGE_CMDQ_N ] = {0 ,0 }; +if (e->DataValid) +return 1 ; do { flags |= e->Qsleeping;
@@ -1610,23 +1621,20 @@ static int process_pure_responses(structint t1_poll (struct net_device *dev,int *budget) {struct adapter *adapter = dev->priv; -int effective_budget =min (*budget, dev->quota); -int work_done =process_responses (adapter, effective_budget); +int work_done; + work_done =process_responses (adapter,min (*budget, dev->quota)); *budget -= work_done; dev->quota -= work_done; -if (work_done >= effective_budget) +if (unlikely (responses_pending (adapter)))return 1 ; -spin_lock_irq (&adapter->async_lock); -__netif_rx_complete (dev); +netif_rx_complete (dev);writel (adapter->sge->respQ.cidx, adapter->regs +A_SG_SLEEPING ); -writel (adapter->slow_intr_mask |F_PL_INTR_SGE_DATA , - adapter->regs +A_PL_ENABLE ); -spin_unlock_irq (&adapter->async_lock);return 0 ; + }/*
@@ -1635,44 +1643,33 @@ int t1_poll(struct net_device *dev, intirqreturn_t t1_interrupt (int irq,void *data) {struct adapter *adapter = data; -struct net_device *dev = adapter->sge->netdev;struct sge *sge = adapter->sge; -u32 cause; -int handled =0 ; +int handled; - cause =readl (adapter->regs +A_PL_CAUSE ); -if (cause ==0 || cause == ~0 ) -return IRQ_NONE ; - -spin_lock (&adapter->async_lock); -if (cause &F_PL_INTR_SGE_DATA ) { -struct respQ *q = &adapter->sge->respQ; -struct respQ_e *e = &q->entries[q->cidx]; +if (likely (responses_pending (adapter))) { +struct net_device *dev = sge->netdev; - handled =1 ;writel (F_PL_INTR_SGE_DATA , adapter->regs +A_PL_CAUSE ); -if (e->GenerationBit == q->genbit && -__netif_rx_schedule_prep (dev)) { -if (e->DataValid ||process_pure_responses (adapter, e)) { -/* mask off data IRQ */ -writel (adapter->slow_intr_mask, - adapter->regs +A_PL_ENABLE ); -__netif_rx_schedule (sge->netdev); - goto unlock; +if (__netif_rx_schedule_prep (dev)) { +if (process_pure_responses (adapter)) +__netif_rx_schedule (dev); +else { +/* no data, no NAPI needed */ +writel (sge->respQ.cidx, adapter->regs +A_SG_SLEEPING ); +netif_poll_enable (dev);/* undo schedule_prep */ } -/* no data, no NAPI needed */ -netif_poll_enable (dev); - } -writel (q->cidx, adapter->regs +A_SG_SLEEPING ); - }else - handled =t1_slow_intr_handler (adapter); +return IRQ_HANDLED ; + } + +spin_lock (&adapter->async_lock); + handled =t1_slow_intr_handler (adapter); +spin_unlock (&adapter->async_lock);if (!handled) sge->stats.unhandled_irqs++; -unlock: -spin_unlock (&adapter->async_lock); +return IRQ_RETVAL (handled !=0 ); }
@@ -1695,17 +1692,13 @@ unlock:irqreturn_t t1_interrupt (int irq,void *cookie) {int work_done; -struct respQ_e *e;struct adapter *adapter = cookie; -struct respQ *Q = &adapter->sge->respQ;spin_lock (&adapter->async_lock); - e = &Q ->entries[Q ->cidx]; -prefetch (e);writel (F_PL_INTR_SGE_DATA , adapter->regs +A_PL_CAUSE ); -if (likely (e->GenerationBit ==Q ->genbit)) + if (likely(responses_pending(adapter)) work_done =process_responses (adapter,-1 );else work_done =t1_slow_intr_handler (adapter);
Jeff Garzik
AppliedRe: [PATCH 2/3] chelsio: NAPI speed improvement
Quoted 4 lines · Stephen Hemminger
Speedup and cleanup the receive processing by eliminating the mmio read and a lock round trip.
Signed-off-by Stephen Hemminger <[email protected]>
applied