cxgb3: Replace LRO with GRO
Subsystems (3)
Hi Herbert,
I have tried the following patch as an attempt to eliminate the memcpy seen on the previous oprofile. I'm now getting about 5.5 Gbs. After that, I went through the output of opreport -d to figure out the most expensive ops witnessed in my profiling.
Here is the patch:
net/core/dev.c +10 -2
@@ -2554,6 +2554,8 @@ struct sk_buff *napi_fraginfo_skb(struct napi_struct *napi,struct net_device *dev = napi->dev;struct sk_buff *skb = napi->skb;struct ethhdr *eth; +skb_frag_t *frag; +int i; napi->skb =NULL ;
@@ -2566,9 +2568,15 @@ struct sk_buff *napi_fraginfo_skb(struct napi_struct *napi, }BUG_ON (info->nr_frags >MAX_SKB_FRAGS ); -skb_shinfo (skb)->nr_frags = info->nr_frags; -memcpy (skb_shinfo (skb)->frags, info->frags,sizeof (info->frags)); + frag = &info->frags[info->nr_frags -1 ]; +for (i =skb_shinfo (skb)->nr_frags; i < info->nr_frags; i++) { +skb_fill_page_desc (skb, i, frag->page, frag->page_offset, + frag->size); + frag++; + } +skb_shinfo (skb)->nr_frags = info->nr_frags; + skb->data_len = info->len; skb->len += info->len; skb->truesize += info->len;
Here is the non detailed opreport output for the CPU managing the reception of netperf traffic:
38.815300 copy_user_generic_unrolled vmlinux
6.373900 process_responses cxgb3.ko
4.957800 inet_gro_receive vmlinux
4.908800 put_page vmlinux
4.862100 refill_fl cxgb3.ko
3.774900 dev_gro_receive vmlinux
3.096000 tcp_gro_receive vmlinux
2.764700 napi_fraginfo_skb vmlinux
2.174400 free_hot_cold_page vmlinux
2.006400 skb_copy_datagram_iovec vmlinux
1.511800 tcp_recvmsg vmlinux
1.488500 get_page_from_freelist vmlinux
1.455800 irq_entries_start vmlinux
1.453500 skb_gro_header vmlinux
0.877200 get_pageblock_flags_group vmlinux
0.863200 memcpy_toiovec vmlinux
0.856200 _raw_spin_lock vmlinux
0.720900 memcpy vmlinux
0.711600 skb_gro_receive vmlinux
0.683600 kfree vmlinuxHere is a list of more detailed info sorted per GRO function as seen above:
- Relative % for the most expensive instructions
- gdb dissass'output for these instructions
- gdb list's output.
inet_gro_receive 4.9578 ffffffff805468c0
ffffffff80546a49 11.1059%
0xffffffff80546a49 <inet_gro_receive+393>: jne 0xffffffff805469e5 <inet_gro_receive+293>
0xffffffff80546a49 is in inet_gro_receive (/mnt/net-2.6/net/ipv4/af_inet.c:1285).
1280 if (!NAPI_GRO_CB(p)->same_flow)
1281 continue;
1282
1283 iph2 = ip_hdr(p);
1284
1285 if (iph->protocol != iph2->protocol ||
1286 iph->tos != iph2->tos ||
1287 memcmp(&iph->saddr, &iph2->saddr, 8)) {
1288 NAPI_GRO_CB(p)->same_flow = 0;
1289 continue; ffffffff80546a61 10.4000%
0xffffffff80546a61 <inet_gro_receive+417>: je 0xffffffff80546abb <inet_gro_receive+507>
0xffffffff80546a61 is in inet_gro_receive (/mnt/net-2.6/net/ipv4/af_inet.c:1293).
1288 NAPI_GRO_CB(p)->same_flow = 0;
1289 continue;
1290 }
1291
1292 /* All fields must match except length and checksum. */
1293 NAPI_GRO_CB(p)->flush |=
1294 memcmp(&iph->frag_off, &iph2->frag_off, 4) ||
1295 (u16)(ntohs(iph2->id) + NAPI_GRO_CB(p)->count) != id; ffffffff80546a58 8.2353%
0xffffffff80546a58 <inet_gro_receive+408>: mov %rdx,%rcx
0xffffffff80546a58 is in inet_gro_receive (/mnt/net-2.6/net/ipv4/af_inet.c:1293).
1288 NAPI_GRO_CB(p)->same_flow = 0;
1289 continue;
1290 }
1291
1292 /* All fields must match except length and checksum. */
1293 NAPI_GRO_CB(p)->flush |=
1294 memcmp(&iph->frag_off, &iph2->frag_off, 4) ||
1295 (u16)(ntohs(iph2->id) + NAPI_GRO_CB(p)->count) != id;
1296
1297 NAPI_GRO_CB(p)->flush |= flush;
ffffffff80546abb 8.2353%
0xffffffff80546abb <inet_gro_receive+507>: movzwl 0x4(%r10),%eax
(gdb) list *(0xffffffff80546abb)
0xffffffff80546abb is in inet_gro_receive (/mnt/net-2.6/include/linux/swab.h:51).
46 static inline __attribute_const__ __u16 __fswab16(__u16 val)
47 {
48 #ifdef __arch_swab16
49 return __arch_swab16(val);
50 #else
51 return ___constant_swab16(val);
52 #endif
53 }
54
55 static inline __attribute_const__ __u32 __fswab32(__u32 val)
ffffffff80546a4b 8.1882%
0xffffffff80546a4b is in inet_gro_receive (/mnt/net-2.6/net/ipv4/af_inet.c:1293).
1288 NAPI_GRO_CB(p)->same_flow = 0;
1289 continue;
1290 }
1291
1292 /* All fields must match except length and checksum. */
1293 NAPI_GRO_CB(p)->flush |=
1294 memcmp(&iph->frag_off, &iph2->frag_off, 4) ||
1295 (u16)(ntohs(iph2->id) + NAPI_GRO_CB(p)->count) != id;
1296
1297 NAPI_GRO_CB(p)->flush |= flush;
ffffffff80546a47 7.5765%
0xffffffff80546a47 <inet_gro_receive+391>: repz cmpsb %es:(%rdi),%ds:(%rsi)
0xffffffff80546a47 is in inet_gro_receive (/mnt/net-2.6/net/ipv4/af_inet.c:1285).
1280 if (!NAPI_GRO_CB(p)->same_flow)
1281 continue;
1282
1283 iph2 = ip_hdr(p);
1284
1285 if (iph->protocol != iph2->protocol ||
1286 iph->tos != iph2->tos ||
1287 memcmp(&iph->saddr, &iph2->saddr, 8)) {
1288 NAPI_GRO_CB(p)->same_flow = 0;
1289 continue;
ffffffff80546a44 7.1529%
0xffffffff80546a44 is in inet_gro_receive (/mnt/net-2.6/net/ipv4/af_inet.c:1285).
1280 if (!NAPI_GRO_CB(p)->same_flow)
1281 continue;
1282
1283 iph2 = ip_hdr(p);
1284
1285 if (iph->protocol != iph2->protocol ||
1286 iph->tos != iph2->tos ||
1287 memcmp(&iph->saddr, &iph2->saddr, 8)) {
1288 NAPI_GRO_CB(p)->same_flow = 0;
1289 continue;dev_gro_receive 3.7749 ffffffff805024b0
ffffffff805026a2 18.7268%
0xffffffff805026a2 <dev_gro_receive+498>: repz cmpsb %es:(%rdi),%ds:(%rsi)
0xffffffff805026a2 is in dev_gro_receive (/mnt/net-2.6/net/core/dev.c:2450).
2445 count++;
2446
2447 if (!NAPI_GRO_CB(p)->same_flow)
2448 continue;
2449
2450 if (p->mac_len != mac_len ||
2451 memcmp(skb_mac_header(p), mac, mac_len))
2452 NAPI_GRO_CB(p)->same_flow = 0;
2453 }
2454 ffffffff805026a4 13.4734%
0xffffffff805026a4 <dev_gro_receive+500>: je 0xffffffff805025c8 <dev_gro_receive+280>
(gdb) list *(0xffffffff805026a4)
0xffffffff805026a4 is in dev_gro_receive (/mnt/net-2.6/net/core/dev.c:2450).
2445 count++;
2446
2447 if (!NAPI_GRO_CB(p)->same_flow)
2448 continue;
2449
2450 if (p->mac_len != mac_len ||
2451 memcmp(skb_mac_header(p), mac, mac_len))
2452 NAPI_GRO_CB(p)->same_flow = 0; ffffffff805025c8 9.3943%
0xffffffff805025c8 <dev_gro_receive+280>: mov (%r9),%r9
0xffffffff805025c8 is in dev_gro_receive (/mnt/net-2.6/net/core/dev.c:2444).
2439 skb->mac_len = mac_len;
2440 NAPI_GRO_CB(skb)->same_flow = 0;
2441 NAPI_GRO_CB(skb)->flush = 0;
2442 NAPI_GRO_CB(skb)->free = 0;
2443
2444 for (p = napi->gro_list; p; p = p->next) {
2445 count++;
2446
2447 if (!NAPI_GRO_CB(p)->same_flow)
2448 continue;
ffffffff805025f9 7.3548%
0xffffffff805025f9 <dev_gro_receive+329>: je 0xffffffff80502614 <dev_gro_receive+356>
0xffffffff805025f9 is in dev_gro_receive (/mnt/net-2.6/net/core/dev.c:2466).
2461 goto normal;
2462
2463 same_flow = NAPI_GRO_CB(skb)->same_flow;
2464 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
2465
2466 if (pp) {
2467 struct sk_buff *nskb = *pp;
2468
2469 *pp = nskb->next;
2470 nskb->next = NULL;tcp_gro_receive 3.0960 ffffffff80528df0
ffffffff80528f2b 16.3527%
0xffffffff80528f2b <tcp_gro_receive+315>: repz cmpsb %es:(%rdi),%ds:(%rsi)
0xffffffff80528f2b is in tcp_gro_receive (/mnt/net-2.6/net/ipv4/tcp.c:2521).
2516 flush = NAPI_GRO_CB(p)->flush;
2517 flush |= flags & TCP_FLAG_CWR;
2518 flush |= (flags ^ tcp_flag_word(th2)) &
2519 ~(TCP_FLAG_CWR | TCP_FLAG_FIN | TCP_FLAG_PSH);
2520 flush |= th->ack_seq != th2->ack_seq || th->window != th2->window;
2521 flush |= memcmp(th + 1, th2 + 1, thlen - sizeof(*th));
2522
2523 total = skb_gro_len(p);
2524 mss = skb_shinfo(p)->gso_size; ffffffff80528f2d 15.9759%
0xffffffff80528f2d <tcp_gro_receive+317>: mov 0x60(%r8),%edi
0xffffffff80528f2d is in tcp_gro_receive (/mnt/net-2.6/include/linux/netdevice.h:1101).
1096 return NAPI_GRO_CB(skb)->data_offset;
1097 }
1098
1099 static inline unsigned int skb_gro_len(const struct sk_buff *skb)
1100 {
1101 return skb->len - NAPI_GRO_CB(skb)->data_offset;
1102 }
1103
1104 static inline void skb_gro_pull(struct sk_buff *skb, unsigned int len)
1105 {
ffffffff80528f31 13.7905%
0xffffffff80528f31 <tcp_gro_receive+321>: setb %al
0xffffffff80528f31 is in tcp_gro_receive (/mnt/net-2.6/net/ipv4/tcp.c:2521).
2516 flush = NAPI_GRO_CB(p)->flush;
2517 flush |= flags & TCP_FLAG_CWR;
2518 flush |= (flags ^ tcp_flag_word(th2)) &
2519 ~(TCP_FLAG_CWR | TCP_FLAG_FIN | TCP_FLAG_PSH);
2520 flush |= th->ack_seq != th2->ack_seq || th->window != th2->window;
2521 flush |= memcmp(th + 1, th2 + 1, thlen - sizeof(*th));
2522
2523 total = skb_gro_len(p);
2524 mss = skb_shinfo(p)->gso_size;napi_fraginfo_skb 2.7647 ffffffff80501dd0
ffffffff80501f16 65.2321%
0xffffffff80501f16 <napi_fraginfo_skb+326>: mov %eax,0x6c(%rbx)
0xffffffff80501f16 is in napi_fraginfo_skb (/mnt/net-2.6/net/core/dev.c:2606).
2601 * special handling. We'll fix it up properly at the end.
2602 */
2603 skb->protocol = eth->h_proto;
2604
2605 skb->ip_summed = info->ip_summed;
2606 skb->csum = info->csum;
2607
2608 out:
2609 return skb;
2610 }Cheers,
Divy
Quoted 2 lines · Divy Le Ray
Here is the patch:
Thanks, I'll add it to my series and push it later.
Quoted 2 lines
Here is the non detailed opreport output for the CPU managing the reception of netperf traffic:
This is very helpful! Now some of the extra cost of GRO is due to extra checks which are unavoidable. However, there is a lot of microoptimisations that we can perform to minimise the cost and also reduce the cost of some existing checks.
Please try the following patch (compile tested only since I'm at LCA) and let me know if it improves things.
include/linux/etherdevice.h +21 -0
@@ -184,4 +184,25 @@ static inline unsigned compare_ether_addr_64bits(const u8 addr1[6+2], }#endif /* __KERNEL__ */ +/** +* compare_ether_header - Compare two Ethernet headers +* @a: Pointer to Ethernet header +* @b: Pointer to Ethernet header +* +* Compare two ethernet headers, returns 0 if equal. +* This assumes that the network header (i.e., IP header) is 4-byte +* aligned OR the platform can handle unaligned access. This is the +* case for all packets coming into netif_receive_skb or similar +* entry points. +*/ + +static inline int compare_ether_header (const void *a,const void *b) +{ +u32 *a32 = (u32 *)((u8 *)a +2 ); +u32 *b32 = (u32 *)((u8 *)b +2 ); + +return (*(u16 *)a ^ *(u16 *)b) | (a32[0 ] ^ b32[0 ]) | + (a32[1 ] ^ b32[1 ]) | (a32[2 ] ^ b32[2 ]); +} +#endif /* _LINUX_ETHERDEVICE_H */
include/linux/netdevice.h +10 -0
@@ -314,6 +314,9 @@ struct napi_struct {spinlock_t poll_lock;int poll_owner;#endif + +unsigned int gro_count; +struct net_device *dev;struct list_head dev_list;struct sk_buff *gro_list;
@@ -1111,6 +1114,13 @@ static inline void skb_gro_reset_offset(struct sk_buff *skb)NAPI_GRO_CB (skb)->data_offset =0 ; } +static inline void *skb_gro_mac_header (struct sk_buff *skb) +{ +return skb_headlen (skb) ?skb_mac_header (skb) : +page_address (skb_shinfo (skb)->frags[0 ].page) + +skb_shinfo (skb)->frags[0 ].page_offset; +} + static inline int dev_hard_header(struct sk_buff *skb, struct net_device *dev, unsigned short type, const void *daddr, const void *saddr,
net/8021q/vlan_core.c +3 -1
@@ -85,7 +85,9 @@ static int vlan_gro_common(struct napi_struct *napi, struct vlan_group *grp, goto drop;for (p = napi->gro_list; p; p = p->next) { -NAPI_GRO_CB (p)->same_flow = p->dev == skb->dev; +NAPI_GRO_CB (p)->same_flow = + p->dev == skb->dev && !compare_ether_header ( +skb_mac_header (p),skb_gro_mac_header (skb));NAPI_GRO_CB (p)->flush =0 ; }
net/core/dev.c +9 -24
@@ -215,13 +215,6 @@ static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)return &net->dev_index_head[ifindex & ((1 <<NETDEV_HASHBITS ) -1 )]; } -static inline void *skb_gro_mac_header (struct sk_buff *skb) -{ -return skb_headlen (skb) ?skb_mac_header (skb) : -page_address (skb_shinfo (skb)->frags[0 ].page) + -skb_shinfo (skb)->frags[0 ].page_offset; -} -/* Device list insertion */ static int list_netdevice (struct net_device *dev) {
@@ -2378,6 +2371,7 @@ void napi_gro_flush(struct napi_struct *napi)napi_gro_complete (skb); } + napi->gro_count =0 ; napi->gro_list =NULL ; }EXPORT_SYMBOL (napi_gro_flush);
@@ -2407,7 +2401,6 @@ int dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)struct packet_type *ptype;__be16 type = skb->protocol;struct list_head *head = &ptype_base[ntohs (type) &PTYPE_HASH_MASK ]; -int count =0 ;int same_flow;int mac_len;int ret;
@@ -2421,30 +2414,17 @@ int dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)rcu_read_lock ();list_for_each_entry_rcu (ptype, head, list) {struct sk_buff *p; -void *mac;if (ptype->type != type || ptype->dev || !ptype->gro_receive)continue ;skb_set_network_header (skb,skb_gro_offset (skb)); - mac =skb_gro_mac_header (skb); mac_len = skb->network_header - skb->mac_header; skb->mac_len = mac_len;NAPI_GRO_CB (skb)->same_flow =0 ;NAPI_GRO_CB (skb)->flush =0 ;NAPI_GRO_CB (skb)->free =0 ; -for (p = napi->gro_list; p; p = p->next) { - count++; - -if (!NAPI_GRO_CB (p)->same_flow) -continue ; - -if (p->mac_len != mac_len || -memcmp (skb_mac_header (p), mac, mac_len)) -NAPI_GRO_CB (p)->same_flow =0 ; - } - pp = ptype->gro_receive (&napi->gro_list, skb);break ; }
@@ -2462,15 +2442,16 @@ int dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb) *pp = nskb->next; nskb->next =NULL ;napi_gro_complete (nskb); - count--; + napi->gro_count--; }if (same_flow) goto ok; -if (NAPI_GRO_CB (skb)->flush || count >=MAX_GRO_SKBS ) +if (NAPI_GRO_CB (skb)->flush || napi->gro_count >=MAX_GRO_SKBS ) goto normal; + napi->gro_count++;NAPI_GRO_CB (skb)->count =1 ;skb_shinfo (skb)->gso_size =skb_gro_len (skb); skb->next = napi->gro_list;
@@ -2490,7 +2471,8 @@ static int __napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)struct sk_buff *p;for (p = napi->gro_list; p; p = p->next) { -NAPI_GRO_CB (p)->same_flow =1 ; +NAPI_GRO_CB (p)->same_flow = !compare_ether_header ( +skb_mac_header (p),skb_gro_mac_header (skb));NAPI_GRO_CB (p)->flush =0 ; }
@@ -2714,6 +2696,7 @@ void netif_napi_add(struct net_device *dev, struct napi_struct *napi, int (*poll)(struct napi_struct *, int), int weight) {INIT_LIST_HEAD (&napi->poll_list); + napi->gro_count =0 ; napi->gro_list =NULL ; napi->skb =NULL ; napi->poll = poll;
@@ -2742,6 +2725,7 @@ void netif_napi_del(struct napi_struct *napi) } napi->gro_list =NULL ; + napi->gro_count =0 ; }EXPORT_SYMBOL (netif_napi_del);
@@ -5208,6 +5192,7 @@ static int __init net_dev_init(void) queue->backlog.poll = process_backlog; queue->backlog.weight = weight_p; queue->backlog.gro_list =NULL ; + queue->backlog.gro_count =0 ; } dev_boot_phase =0 ;
net/ipv4/af_inet.c +7 -6
@@ -1264,7 +1264,7 @@ static struct sk_buff **inet_gro_receive(struct sk_buff **head,if (!ops || !ops->gro_receive) goto out_unlock; -if (iph->version !=4 || iph->ihl !=5 ) +if (*(u8 *)iph !=0x45 ) goto out_unlock;if (unlikely (ip_fast_csum ((u8 *)iph, iph->ihl)))
@@ -1282,17 +1282,18 @@ static struct sk_buff **inet_gro_receive(struct sk_buff **head, iph2 =ip_hdr (p); -if (iph->protocol != iph2->protocol || - iph->tos != iph2->tos || -memcmp (&iph->saddr, &iph2->saddr,8 )) { +if ((iph->protocol ^ iph2->protocol) | + (iph->tos ^ iph2->tos) | + (iph->saddr ^ iph2->saddr) | + (iph->daddr ^ iph2->daddr)) {NAPI_GRO_CB (p)->same_flow =0 ;continue ; }/* All fields must match except length and checksum. */ NAPI_GRO_CB (p)->flush |= -memcmp (&iph->frag_off, &iph2->frag_off,4 ) || - (u16)(ntohs (iph2->id) +NAPI_GRO_CB (p)->count) != id; + (iph->ttl ^ iph2->ttl) | + ((u16)(ntohs (iph2->id) +NAPI_GRO_CB (p)->count) ^ id);NAPI_GRO_CB (p)->flush |= flush; }
net/ipv4/tcp.c +8 -7
@@ -2469,9 +2469,9 @@ struct sk_buff **tcp_gro_receive(struct sk_buff **head, struct sk_buff *skb)struct tcphdr *th2;unsigned int thlen;unsigned int flags; -unsigned int total;unsigned int mss =1 ;int flush =1 ; +int i; th =skb_gro_header (skb,sizeof (*th));if (unlikely (!th))
@@ -2495,7 +2495,7 @@ struct sk_buff **tcp_gro_receive(struct sk_buff **head, struct sk_buff *skb) th2 =tcp_hdr (p); -if (th->source != th2->source || th->dest != th2->dest) { +if ((th->source ^ th2->source) | (th->dest ^ th2->dest)) {NAPI_GRO_CB (p)->same_flow =0 ;continue ; }
@@ -2510,14 +2510,15 @@ found: flush |= flags &TCP_FLAG_CWR ; flush |= (flags ^tcp_flag_word (th2)) & ~(TCP_FLAG_CWR |TCP_FLAG_FIN |TCP_FLAG_PSH ); - flush |= th->ack_seq != th2->ack_seq || th->window != th2->window; - flush |=memcmp (th +1 , th2 +1 , thlen -sizeof (*th)); + flush |= (th->ack_seq ^ th2->ack_seq) | (th->window ^ th2->window); +for (i =sizeof (*th); !flush && i < thlen; i +=4 ) + flush |= *(u32 *)((u8 *)th + i) ^ + *(u32 *)((u8 *)th2 + i); - total =skb_gro_len (p); mss =skb_shinfo (p)->gso_size; - flush |=skb_gro_len (skb) > mss || !skb_gro_len (skb); - flush |=ntohl (th2->seq) + total !=ntohl (th->seq); + flush |= (skb_gro_len (skb) > mss) | !skb_gro_len (skb); + flush |= (ntohl (th2->seq) +skb_gro_len (p)) ^ntohl (th->seq);if (flush ||skb_gro_receive (head, skb)) { mss =1 ;
Thanks,
Signature · 4 lines
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <[email protected]>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txtQuoted 7 lines · Herbert Xu
there is a lot
of microoptimisations that we can perform to minimise the cost and also reduce the cost of some existing checks.Please try the following patch (compile tested only since I'm at LCA) and let me know if it improves things.
I'm now getting about an average 5.7Gbs, oprofile typically showing:
37.294500 copy_user_generic_unrolled vmlinux
8.733800 process_responses cxgb3.ko
6.123700 refill_fl cxgb3.ko
5.006900 put_page vmlinux
3.400700 napi_fraginfo_skb vmlinux
2.484600 tcp_gro_receive vmlinux
2.308900 inet_gro_receive vmlinux
2.196000 free_hot_cold_page vmlinux
2.032900 skb_gro_header vmlinux
1.970100 get_page_from_freelist vmlinux
1.869700 skb_copy_datagram_iovec vmlinux
1.380300 dev_gro_receive vmlinux
1.242300 tcp_recvmsg vmlinux
1.079200 irq_entries_start vmlinux
1.003900 get_pageblock_flags_group vmlinux
0.991300 skb_gro_receive vmlinux
0.878400 _raw_spin_lock vmlinux
0.878400 memcpy vmlinux
0.840800 kfree vmlinux
0.828200 list_del vmlinuxCheers,
Divy
Quoted 13 lines · Divy Le Ray
I'm now getting about an average 5.7Gbs, oprofile typically showing:
37.294500 copy_user_generic_unrolled vmlinux 8.733800 process_responses cxgb3.ko 6.123700 refill_fl cxgb3.ko 5.006900 put_page vmlinux 3.400700 napi_fraginfo_skb vmlinux 2.484600 tcp_gro_receive vmlinux 2.308900 inet_gro_receive vmlinux 2.196000 free_hot_cold_page vmlinux 2.032900 skb_gro_header vmlinux 1.970100 get_page_from_freelist vmlinux
Quoted 8 lines
1.869700 skb_copy_datagram_iovec vmlinux
1.380300 dev_gro_receive vmlinux
1.242300 tcp_recvmsg vmlinux
1.079200 irq_entries_start vmlinux
1.003900 get_pageblock_flags_group vmlinux
0.991300 skb_gro_receive vmlinux
0.878400 _raw_spin_lock vmlinux
0.878400 memcpy vmlinuxWhen you can get a chance can you see if this patch makes any difference at all?
net/core/skbuff.c +5 -3
@@ -2586,8 +2586,10 @@ int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb) {struct sk_buff *p = *head;struct sk_buff *nskb; +skb_frag_t *frag;unsigned int headroom;unsigned int len =skb_gro_len (skb); +int i;if (p->len + len >=65536 )return -E2BIG ;
@@ -2604,9 +2606,9 @@ int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb)skb_shinfo (skb)->frags[0 ].size -=skb_gro_offset (skb) -skb_headlen (skb); -memcpy (skb_shinfo (p)->frags +skb_shinfo (p)->nr_frags, -skb_shinfo (skb)->frags, -skb_shinfo (skb)->nr_frags *sizeof (skb_frag_t)); + frag =skb_shinfo (p)->frags +skb_shinfo (p)->nr_frags; +for (i =0 ; i <skb_shinfo (skb)->nr_frags; i++) + *frag++ =skb_shinfo (skb)->frags[i];skb_shinfo (p)->nr_frags +=skb_shinfo (skb)->nr_frags;skb_shinfo (skb)->nr_frags =0 ;
Thanks,
Signature · 4 lines
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <[email protected]>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txtQuoted 2 lines · Herbert Xu
When you can get a chance can you see if this patch makes any difference at all?
Hi Herbert,
Thanks for the patch.
I should be able to test within the next few days.
cheers,
Divy
Quoted 3 lines
When you can get a chance can you see if this patch makes any difference at all?
Hi Herbert,
Sorry for the delay.
I had to switch to different development platforms.
I've not seen much perf change with this patch, it looks good though.
Cheers,
Divy
Quoted 31 lines
net/core/skbuff.c +5
-3@@ -2586,8 +2586,10 @@ int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb) {struct sk_buff *p = *head;struct sk_buff *nskb; +skb_frag_t *frag;unsigned int headroom;unsigned int len =skb_gro_len (skb); +int i;if (p->len + len >=65536 )return -E2BIG ;@@ -2604,9 +2606,9 @@ int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb)skb_shinfo (skb)->frags[0 ].size -=skb_gro_offset (skb) -skb_headlen (skb); -memcpy (skb_shinfo (p)->frags +skb_shinfo (p)->nr_frags, -skb_shinfo (skb)->frags, -skb_shinfo (skb)->nr_frags *sizeof (skb_frag_t)); + frag =skb_shinfo (p)->frags +skb_shinfo (p)->nr_frags; +for (i =0 ; i <skb_shinfo (skb)->nr_frags; i++) + *frag++ =skb_shinfo (skb)->frags[i];skb_shinfo (p)->nr_frags +=skb_shinfo (skb)->nr_frags;skb_shinfo (skb)->nr_frags =0 ;Thanks,
Quoted 2 lines · Divy Le Ray
I've not seen much perf change with this patch, it looks good though.
Thanks. I decided to get rid of the copy altogether after I got my hands on a cxgb3 machine (well, remotely anyway). It seems to do the trick for me as in both LRO and GRO now produces about the same performance with the CPU 100% maxed out.
Here's the patch. Let me know how it fares on your machine.
gro: New frags interface to avoid copying shinfo
It turns out that copying a 16-byte area at ~800k times a second can be really expensive :) This patch redesigns the frags GRO interface to avoid copying that area twice.
The two disciples of the frags interface have been converted.
drivers/net/cxgb3/adapter.h +1 -1
@@ -195,7 +195,7 @@ struct sge_qset { /* an SGE queue set */struct sge_rspq rspq;struct sge_fl fl[SGE_RXQ_PER_SET ];struct sge_txq txq[SGE_TXQ_PER_SET ]; -struct napi_gro_fraginfo lro_frag_tbl; +int nomem;int lro_enabled;void *lro_va;struct net_device *netdev;
drivers/net/cxgb3/sge.c +32 -21
103 lines not on this page · raw message
drivers/net/sfc/rx.c +18 -8
@@ -450,17 +450,27 @@ static void efx_rx_packet_lro(struct efx_channel *channel,/* Pass the skb/page into the LRO engine */ if (rx_buf->page) { -struct napi_gro_fraginfo info; +struct sk_buff *skb =napi_get_frags (napi); - info.frags[0 ].page = rx_buf->page; - info.frags[0 ].page_offset =efx_rx_buf_offset (rx_buf); - info.frags[0 ].size = rx_buf->len; - info.nr_frags =1 ; - info.ip_summed =CHECKSUM_UNNECESSARY ; - info.len = rx_buf->len; +if (!skb) { +put_page (rx_buf->page); + goto out; + } + +skb_shinfo (skb)->frags[0 ].page = rx_buf->page; +skb_shinfo (skb)->frags[0 ].page_offset = +efx_rx_buf_offset (rx_buf); +skb_shinfo (skb)->frags[0 ].size = rx_buf->len; +skb_shinfo (skb)->nr_frags =1 ; + + skb->len = rx_buf->len; + skb->data_len = rx_buf->len; + skb->truesize += rx_buf->len; + skb->ip_summed =CHECKSUM_UNNECESSARY ; -napi_gro_frags (napi, &info); +napi_gro_frags (napi); +out:EFX_BUG_ON_PARANOID (rx_buf->skb); rx_buf->page =NULL ; }else {
include/linux/if_vlan.h +2 -4
@@ -118,8 +118,7 @@ extern int vlan_hwaccel_do_receive(struct sk_buff *skb);extern int vlan_gro_receive (struct napi_struct *napi,struct vlan_group *grp,unsigned int vlan_tci,struct sk_buff *skb);extern int vlan_gro_frags (struct napi_struct *napi,struct vlan_group *grp, -unsigned int vlan_tci, -struct napi_gro_fraginfo *info); +unsigned int vlan_tci);#else static inline struct net_device *vlan_dev_real_dev (const struct net_device *dev)
@@ -154,8 +153,7 @@ static inline int vlan_gro_receive(struct napi_struct *napi, }static inline int vlan_gro_frags (struct napi_struct *napi, -struct vlan_group *grp,unsigned int vlan_tci, -struct napi_gro_fraginfo *info) +struct vlan_group *grp,unsigned int vlan_tci) {return NET_RX_DROP ; }
include/linux/netdevice.h +10 -12
@@ -1047,14 +1047,6 @@ struct packet_type {struct list_head list; }; -struct napi_gro_fraginfo { -skb_frag_t frags[MAX_SKB_FRAGS ]; -unsigned int nr_frags; -unsigned int ip_summed; -unsigned int len; -__wsum csum; -}; -#include <linux/interrupt.h> #include <linux/notifier.h>
@@ -1442,12 +1434,18 @@ extern int napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb);extern void napi_reuse_skb (struct napi_struct *napi,struct sk_buff *skb); -extern struct sk_buff *napi_fraginfo_skb (struct napi_struct *napi, -struct napi_gro_fraginfo *info); +extern struct sk_buff *napi_get_frags (struct napi_struct *napi);extern int napi_frags_finish (struct napi_struct *napi,struct sk_buff *skb,int ret); -extern int napi_gro_frags (struct napi_struct *napi, -struct napi_gro_fraginfo *info); +extern struct sk_buff *napi_frags_skb (struct napi_struct *napi); +extern int napi_gro_frags (struct napi_struct *napi); + +static inline void napi_free_frags (struct napi_struct *napi) +{ +kfree_skb (napi->skb); + napi->skb =NULL ; +} +extern void netif_nit_deliver (struct sk_buff *skb);extern int dev_valid_name (const char *name);extern int dev_ioctl (struct net *net,unsigned int cmd,void __user *);
net/8021q/vlan_core.c +2 -2
@@ -114,9 +114,9 @@ int vlan_gro_receive(struct napi_struct *napi, struct vlan_group *grp,EXPORT_SYMBOL (vlan_gro_receive);int vlan_gro_frags (struct napi_struct *napi,struct vlan_group *grp, -unsigned int vlan_tci,struct napi_gro_fraginfo *info) +unsigned int vlan_tci) { -struct sk_buff *skb =napi_fraginfo_skb (napi, info); +struct sk_buff *skb =napi_frags_skb (napi);if (!skb)return NET_RX_DROP ;
net/core/dev.c +36 -45
115 lines not on this page · raw message
Cheers,
Signature · 4 lines
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <[email protected]>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt