Skip to contentSkip to conversation

net: Optimize flush calculation in inet_gro_receive()

PATCH In review Helge Deller 5 participants 6 messages Apr 10 – Apr 14

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 net: Optimize flush calculation in inet_gro_receive()

For the calculation of the flush variable, use the get_unaligned_xxx() helpers to access only relevant bits of the IP header.

Note: Since I don't know the network details, I'm not sure if "& ~IP_DF" (& ~0x4000) is correct, or if "& IP_OFFSET" (& 0x1FFF) should be used instead (which I believe would be more correct). Instead of possibly breaking things I left it as is, but maybe some expert can check?

net/ipv4/af_inet.c +3 -2
index c7731e300a44..58cad2687c2c 100644
@@ -1479,7 +1479,7 @@ struct sk_buff *inet_gro_receive(struct list_head *head, struct sk_buff *skb)
 	struct sk_buff *p;
 	unsigned int hlen;
 	unsigned int off;
-	int flush = 1;
+	u16 flush = 1;
 	int proto;
 
 	off = skb_gro_offset(skb);
@@ -1504,7 +1504,8 @@ struct sk_buff *inet_gro_receive(struct list_head *head, struct sk_buff *skb)
 		goto out;
 
 	NAPI_GRO_CB(skb)->proto = proto;
-	flush = (u16)((ntohl(*(__be32 *)iph) ^ skb_gro_len(skb)) | (ntohl(*(__be32 *)&iph->id) & ~IP_DF));
+	flush = (get_unaligned_be16(&iph->tot_len) ^ skb_gro_len(skb)) |
+	        (get_unaligned_be16(&iph->frag_off) & ~IP_DF);
 
 	list_for_each_entry(p, head, list) {
 		struct iphdr *iph2;
#1 lore raw

From: Helge Deller <[email protected]>
Date: Fri, 10 Apr 2026 16:43:54 +0200

Quoted 5 lines

For the calculation of the flush variable, use the get_unaligned_xxx() helpers to access only relevant bits of the IP header.

Note: Since I don't know the network details, I'm not sure if "& ~IP_DF" (& ~0x4000) is correct, or if "& IP_OFFSET" (& 0x1FFF) should be used instead

~IP_DF is correct (MF bit needs to be checked), see

commit db8caf3dbc77599dc90f4ea0a803cd1d97116f30

Author: Eric Dumazet <[email protected]>
Date:   Fri May 31 11:18:10 2013

gro: should aggregate frames without DF

Quoted 19 lines

(which I believe would be more correct). Instead of possibly breaking things I left it as is, but maybe some expert can check?

net/ipv4/af_inet.c +1 -1
index c7731e300a44..58cad2687c2c 100644
@@ -1479,7 +1479,7 @@ struct sk_buff *inet_gro_receive(struct list_head *head, struct sk_buff *skb)
 	struct sk_buff *p;
 	unsigned int hlen;
 	unsigned int off;
-	int flush = 1;
+	u16 flush = 1;
 	int proto;
 
 	off = skb_gro_offset(skb);

net/ipv4/af_inet.c struct sk_buff *inet_gro_receive(struct list_head *head, struct sk_buff *skb) · line 1508

Quoted 6 lines
net/ipv4/af_inet.c +2 -1
@@ -1504,7 +1504,8 @@ struct sk_buff *inet_gro_receive(struct list_head *head, struct sk_buff *skb)
 		goto out;
 
 	NAPI_GRO_CB(skb)->proto = proto;
-	flush = (u16)((ntohl(*(__be32 *)iph) ^ skb_gro_len(skb)) | (ntohl(*(__be32 *)&iph->id) & ~IP_DF));
+	flush = (get_unaligned_be16(&iph->tot_len) ^ skb_gro_len(skb)) |
+	        (get_unaligned_be16(&iph->frag_off) & ~IP_DF);

I think here we intentionally use 32-bit loads:

commit 1075f3f65d0e0f49351b7d4310e9f94483972a51

Author: Herbert Xu <[email protected]>
Date:   Tue May 26 18:50:29 2009

ipv4: Use 32-bit loads for ID and length in GRO

Before your patch, 32-bit load + bswap are used while 16-bit load + rol 8 after the change.

I feel the 4-byte aligned load + bswap is faster than misaligned access + 8 times shift (Is this internally optimised like xchg for a single word size ?)

Do you have some numbers ?

Before:

	flush = (u16)((ntohl(*(__be32 *)iph) ^ skb_gro_len(skb))
mov    edx,DWORD PTR [rcx]
bswap  edx
	return skb->len - NAPI_GRO_CB(skb)->data_offset;
mov    r8d,DWORD PTR [rsi+0x38]
mov    r9d,DWORD PTR [rsi+0x70]
sub    r9d,r8d
xor    r9d,edx
	| (ntohl(*(__be32 *)&iph->id) & ~IP_DF));
mov    ebp,0xffbfffff
and    ebp,DWORD PTR [rcx+0x4]
bswap  ebp
or     ebp,r9d

After:

	flush = (get_unaligned_be16(&iph->tot_len) ^ skb_gro_len(skb))
movzx  edx,WORD PTR [rcx+0x2]
rol    dx,0x8
	return skb->len - NAPI_GRO_CB(skb)->data_offset;
mov    r8d,DWORD PTR [rsi+0x38]
mov    r9d,DWORD PTR [rsi+0x70]
sub    r9d,r8d
xor    r9d,edx
	| (get_unaligned_be16(&iph->frag_off) & ~IP_DF);
movzx  ebp,WORD PTR [rcx+0x6]
and    ebp,0xffffffbf
rol    bp,0x8
or     ebp,r9d
#2 lore raw
Quoted 52 lines · On Sat, 11 Apr 2026 05:19:35 +0000 Kuniyuki Iwashima <[email protected]> wrote:

From: Helge Deller <[email protected]>
Date: Fri, 10 Apr 2026 16:43:54 +0200

Quoted 5 lines

For the calculation of the flush variable, use the get_unaligned_xxx() helpers to access only relevant bits of the IP header.

Note: Since I don't know the network details, I'm not sure if "& ~IP_DF" (& ~0x4000) is correct, or if "& IP_OFFSET" (& 0x1FFF) should be used instead

~IP_DF is correct (MF bit needs to be checked), see

commit db8caf3dbc77599dc90f4ea0a803cd1d97116f30

Author: Eric Dumazet <[email protected]>
Date:   Fri May 31 11:18:10 2013

gro: should aggregate frames without DF

Quoted 19 lines

(which I believe would be more correct). Instead of possibly breaking things I left it as is, but maybe some expert can check?

net/ipv4/af_inet.c +1 -1
index c7731e300a44..58cad2687c2c 100644
@@ -1479,7 +1479,7 @@ struct sk_buff *inet_gro_receive(struct list_head *head, struct sk_buff *skb)
 	struct sk_buff *p;
 	unsigned int hlen;
 	unsigned int off;
-	int flush = 1;
+	u16 flush = 1;
 	int proto;
 
 	off = skb_gro_offset(skb);

net/ipv4/af_inet.c struct sk_buff *inet_gro_receive(struct list_head *head, struct sk_buff *skb) · line 1508

Quoted 6 lines
net/ipv4/af_inet.c +2 -1
@@ -1504,7 +1504,8 @@ struct sk_buff *inet_gro_receive(struct list_head *head, struct sk_buff *skb)
 		goto out;
 
 	NAPI_GRO_CB(skb)->proto = proto;
-	flush = (u16)((ntohl(*(__be32 *)iph) ^ skb_gro_len(skb)) | (ntohl(*(__be32 *)&iph->id) & ~IP_DF));
+	flush = (get_unaligned_be16(&iph->tot_len) ^ skb_gro_len(skb)) |
+	        (get_unaligned_be16(&iph->frag_off) & ~IP_DF);  

I think here we intentionally use 32-bit loads:

commit 1075f3f65d0e0f49351b7d4310e9f94483972a51

Author: Herbert Xu <[email protected]>
Date:   Tue May 26 18:50:29 2009

ipv4: Use 32-bit loads for ID and length in GRO

Quoted 8 lines

Before your patch, 32-bit load + bswap are used while 16-bit load + rol 8 after the change.

I feel the 4-byte aligned load + bswap is faster than misaligned access + 8 times shift (Is this internally optimised like xchg for a single word size ?)

Do you have some numbers ?

Check on some architecture that doesn't support misaligned loads. Actually, aren't the accesses aligned??
Also on ones without 32bit byteswap (some do have byteswapping memory reads).

Also you may not want to change 'flush' to u16.
On non-x86 it may force the compiler add extra masking instructions.

David

Quoted 33 lines

Before:

	flush = (u16)((ntohl(*(__be32 *)iph) ^ skb_gro_len(skb))
mov    edx,DWORD PTR [rcx]
bswap  edx
	return skb->len - NAPI_GRO_CB(skb)->data_offset;
mov    r8d,DWORD PTR [rsi+0x38]
mov    r9d,DWORD PTR [rsi+0x70]
sub    r9d,r8d
xor    r9d,edx
	| (ntohl(*(__be32 *)&iph->id) & ~IP_DF));
mov    ebp,0xffbfffff
and    ebp,DWORD PTR [rcx+0x4]
bswap  ebp
or     ebp,r9d

After:

	flush = (get_unaligned_be16(&iph->tot_len) ^ skb_gro_len(skb))
movzx  edx,WORD PTR [rcx+0x2]
rol    dx,0x8
	return skb->len - NAPI_GRO_CB(skb)->data_offset;
mov    r8d,DWORD PTR [rsi+0x38]
mov    r9d,DWORD PTR [rsi+0x70]
sub    r9d,r8d
xor    r9d,edx
	| (get_unaligned_be16(&iph->frag_off) & ~IP_DF);
movzx  ebp,WORD PTR [rcx+0x6]
and    ebp,0xffffffbf
rol    bp,0x8
or     ebp,r9d
#3 lore raw

Hi Kikuyu and David,

Quoted 5 lines · David Laight
Quoted 2 lines · On Sat, 11 Apr 2026 05:19:35 +0000 Kikuyu Iwashima <[email protected]> wrote:

From: Helge Deller <[email protected]>
Date: Fri, 10 Apr 2026 16:43:54 +0200

Quoted 7 lines
Quoted 7 lines
Quoted 5 lines

For the calculation of the flush variable, use the get_unaligned_xxx() helpers to access only relevant bits of the IP header.

Note: Since I don't know the network details, I'm not sure if "& ~IP_DF" (& ~0x4000) is correct, or if "& IP_OFFSET" (& 0x1FFF) should be used instead

~IP_DF is correct (MF bit needs to be checked), see

Ok, Thanks for checking!

Quoted 35 lines
Quoted 35 lines

commit db8caf3dbc77599dc90f4ea0a803cd1d97116f30

Author: Eric Dumazet <[email protected]>
Date:   Fri May 31 11:18:10 2013

gro: should aggregate frames without DF

Quoted 19 lines

(which I believe would be more correct). Instead of possibly breaking things I left it as is, but maybe some expert can check?

net/ipv4/af_inet.c +1 -1
index c7731e300a44..58cad2687c2c 100644
@@ -1479,7 +1479,7 @@ struct sk_buff *inet_gro_receive(struct list_head *head, struct sk_buff *skb)
  	struct sk_buff *p;
  	unsigned int hlen;
  	unsigned int off;
-	int flush = 1;
+	u16 flush = 1;
  	int proto;
 
  	off = skb_gro_offset(skb);

net/ipv4/af_inet.c struct sk_buff *inet_gro_receive(struct list_head *head, struct sk_buff *skb) · line 1508

Quoted 6 lines
net/ipv4/af_inet.c +2 -1
@@ -1504,7 +1504,8 @@ struct sk_buff *inet_gro_receive(struct list_head *head, struct sk_buff *skb)
  		goto out;
 
  	NAPI_GRO_CB(skb)->proto = proto;
-	flush = (u16)((ntohl(*(__be32 *)iph) ^ skb_gro_len(skb)) | (ntohl(*(__be32 *)&iph->id) & ~IP_DF));
+	flush = (get_unaligned_be16(&iph->tot_len) ^ skb_gro_len(skb)) |
+	        (get_unaligned_be16(&iph->frag_off) & ~IP_DF);

I think here we intentionally use 32-bit loads:

Quoted 4 lines
Quoted 4 lines

commit Author: Herbert Xu <[email protected]>
Date: Tue May 26 18:50:29 2009

ipv4: Use 32-bit loads for ID and length in GRO

I see, this patch is exactly the opposite of mine.

Quoted 8 lines
Quoted 8 lines

Before your patch, 32-bit load + bswap are used while 16-bit load + rol 8 after the change.

I feel the 4-byte aligned load + bswap is faster than misaligned access + 8 times shift (Is this internally optimised like xchg for a single word size ?)

Do you have some numbers ?

No, I don't have.
In the end it's very platform specific anyway.

Quoted 2 lines

Check on some architecture that doesn't support misaligned loads. Actually, aren't the accesses aligned??

The reason why I touched this code at all, is because I got unaligned accesses in that function on parisc.
But those unaligned accesses were triggered by parisc-specific inline assembly, and not by this code here.

So, I believe those accesses here are aligned, and the get_unaligned_XX() helpers make the code more readable, but are NOT necessary.

That said, I suggest to drop my patch.
It makes the code more readable, but probably will not improve speed.

Thanks for your help!
Helge

Quoted 42 lines

Also on ones without 32bit byteswap (some do have byteswapping memory reads).

Also you may not want to change 'flush' to u16.
On non-x86 it may force the compiler add extra masking instructions.

David

Quoted 33 lines

Before:

	flush = (u16)((ntohl(*(__be32 *)iph) ^ skb_gro_len(skb))
mov    edx,DWORD PTR [rcx]
bswap  edx
	return skb->len - NAPI_GRO_CB(skb)->data_offset;
mov    r8d,DWORD PTR [rsi+0x38]
mov    r9d,DWORD PTR [rsi+0x70]
sub    r9d,r8d
xor    r9d,edx
	| (ntohl(*(__be32 *)&iph->id) & ~IP_DF));
mov    ebp,0xffbfffff
and    ebp,DWORD PTR [rcx+0x4]
bswap  ebp
or     ebp,r9d

After:

	flush = (get_unaligned_be16(&iph->tot_len) ^ skb_gro_len(skb))
movzx  edx,WORD PTR [rcx+0x2]
rol    dx,0x8
	return skb->len - NAPI_GRO_CB(skb)->data_offset;
mov    r8d,DWORD PTR [rsi+0x38]
mov    r9d,DWORD PTR [rsi+0x70]
sub    r9d,r8d
xor    r9d,edx
	| (get_unaligned_be16(&iph->frag_off) & ~IP_DF);
movzx  ebp,WORD PTR [rcx+0x6]
and    ebp,0xffffffbf
rol    bp,0x8
or     ebp,r9d
#4 lore raw
Quoted 1 line · On Tue, 14 Apr 2026 09:46:55 +0200 Helge Deller <[email protected]> wrote:

Hi Kikuyu and David,

...

Quoted 43 lines
Quoted 23 lines
Quoted 23 lines
Quoted 14 lines
net/ipv4/af_inet.c +1 -1
index c7731e300a44..58cad2687c2c 100644
@@ -1479,7 +1479,7 @@ struct sk_buff *inet_gro_receive(struct list_head *head, struct sk_buff *skb)
  	struct sk_buff *p;
  	unsigned int hlen;
  	unsigned int off;
-	int flush = 1;
+	u16 flush = 1;
  	int proto;
 
  	off = skb_gro_offset(skb);

net/ipv4/af_inet.c struct sk_buff *inet_gro_receive(struct list_head *head, struct sk_buff *skb) · line 1508

Quoted 6 lines
net/ipv4/af_inet.c +2 -1
@@ -1504,7 +1504,8 @@ struct sk_buff *inet_gro_receive(struct list_head *head, struct sk_buff *skb)
  		goto out;
 
  	NAPI_GRO_CB(skb)->proto = proto;
-	flush = (u16)((ntohl(*(__be32 *)iph) ^ skb_gro_len(skb)) | (ntohl(*(__be32 *)&iph->id) & ~IP_DF));
+	flush = (get_unaligned_be16(&iph->tot_len) ^ skb_gro_len(skb)) |
+	        (get_unaligned_be16(&iph->frag_off) & ~IP_DF);  

I think here we intentionally use 32-bit loads:

Quoted 5 lines
Quoted 5 lines

commit

Author: Herbert Xu <[email protected]>
Date:   Tue May 26 18:50:29 2009

ipv4: Use 32-bit loads for ID and length in GRO

I see, this patch is exactly the opposite of mine.

Quoted 8 lines
Quoted 8 lines

Before your patch, 32-bit load + bswap are used while 16-bit load + rol 8 after the change.

I feel the 4-byte aligned load + bswap is faster than misaligned access + 8 times shift (Is this internally optimised like xchg for a single word size ?)

Do you have some numbers ?

No, I don't have.
In the end it's very platform specific anyway.

Quoted 7 lines
Quoted 2 lines

Check on some architecture that doesn't support misaligned loads. Actually, aren't the accesses aligned??

The reason why I touched this code at all, is because I got unaligned accesses in that function on parisc.
But those unaligned accesses were triggered by parisc-specific inline assembly, and not by this code here.

The network stack is supposed to ensure that all receive packets are aligned to that IP header is on a 4-byte boundary.
This typically requires the ethernet receive buffer be 4n+2 aligned. Unfortunately there is some ethernet hardware that requires 4n aligned buffers (often on SoC devices with cpu that fault misaligned accesses). (Just writing two bytes of garbage before the frame solves the issue.)

Quoted 5 lines

So, I believe those accesses here are aligned, and the get_unaligned_XX() helpers make the code more readable, but are NOT necessary.

That said, I suggest to drop my patch.
It makes the code more readable, but probably will not improve speed.

I think the purpose of the change was to use the hardware's 32bit byte-swapping memory loads rather than software swapping of the 16-bit items.
That shaves off a few instructions - and they can be measurable in some of the network paths with specific workloads.

Remember, save 0.1% 100 times and the code runs 10% faster. Every little bit can make a difference.

David

Quoted 47 lines

Thanks for your help!
Helge

Quoted 42 lines

Also on ones without 32bit byteswap (some do have byteswapping memory reads).

Also you may not want to change 'flush' to u16.
On non-x86 it may force the compiler add extra masking instructions.

David

Quoted 33 lines

Before:

	flush = (u16)((ntohl(*(__be32 *)iph) ^ skb_gro_len(skb))
mov    edx,DWORD PTR [rcx]
bswap  edx
	return skb->len - NAPI_GRO_CB(skb)->data_offset;
mov    r8d,DWORD PTR [rsi+0x38]
mov    r9d,DWORD PTR [rsi+0x70]
sub    r9d,r8d
xor    r9d,edx
	| (ntohl(*(__be32 *)&iph->id) & ~IP_DF));
mov    ebp,0xffbfffff
and    ebp,DWORD PTR [rcx+0x4]
bswap  ebp
or     ebp,r9d

After:

	flush = (get_unaligned_be16(&iph->tot_len) ^ skb_gro_len(skb))
movzx  edx,WORD PTR [rcx+0x2]
rol    dx,0x8
	return skb->len - NAPI_GRO_CB(skb)->data_offset;
mov    r8d,DWORD PTR [rsi+0x38]
mov    r9d,DWORD PTR [rsi+0x70]
sub    r9d,r8d
xor    r9d,edx
	| (get_unaligned_be16(&iph->frag_off) & ~IP_DF);
movzx  ebp,WORD PTR [rcx+0x6]
and    ebp,0xffffffbf
rol    bp,0x8
or     ebp,r9d
#5 lore raw
Quoted 6 lines · David Laight

The network stack is supposed to ensure that all receive packets are aligned to that IP header is on a 4-byte boundary.
This typically requires the ethernet receive buffer be 4n+2 aligned. Unfortunately there is some ethernet hardware that requires 4n aligned buffers (often on SoC devices with cpu that fault misaligned accesses). (Just writing two bytes of garbage before the frame solves the issue.)

This is mostly true, but depends on NET_IP_ALIGN,

On x86, this constant was changed to 0 back in 2010.

commit ea812ca1b06113597adcd8e70c0f84a413d97544

Author: Alexander Duyck <[email protected]>
Date:   Tue Jun 29 18:38:00 2010 +0000

x86: Align skb w/ start of cacheline on newer core 2/Xeon Arch