udp:nat:vxlan tx after nat should recsum if vxlan tx offload on
Subsystems (4)
PATCH 1/1 udp:nat:vxlan tx after nat should recsum if vxlan tx offload on
From: "chenwei.0515" <[email protected]>
If vxlan-dev enable tx csum offload, there are two case of CHECKSUM_PARTIAL, but udp->check donot have the both meanings.
1. vxlan-dev disable tx csum offload, udp->check is just pseudo hdr.
2. vxlan-dev enable tx csum offload, udp->check is pseudo hdr and csum from outter l4 to innner l4.
Unfortunately if there is a nat process after vxlan tx,udp_manip_pkt just use CSUM_PARTIAL to re csum PKT, which is just right on vxlan tx csum disable offload.
This patch use skb->csum_local flag to identify two case, which will csum lco_csum if valid.
--- include/linux/skbuff.h | 1 + net/ipv4/udp.c | 1 + net/netfilter/nf_nat_proto.c | 9 +++++++++ 3 files changed, 11 insertions(+)
include/linux/skbuff.h +1 -0
@@ -990,6 +990,7 @@ struct sk_buff { __u8 slow_gro:1; __u8 csum_not_inet:1; __u8 scm_io_uring:1; + __u8 csum_local:1;#ifdef CONFIG_NET_SCHED __u16 tc_index;/* traffic control index */
net/ipv4/udp.c +1 -0
@@ -889,6 +889,7 @@ void udp_set_csum(bool nocheck, struct sk_buff *skb, uh->check =udp_v4_check (len, saddr, daddr,lco_csum (skb));if (uh->check ==0 ) uh->check =CSUM_MANGLED_0 ; + skb->csum_local =1 ; }else { skb->ip_summed =CHECKSUM_PARTIAL ; skb->csum_start =skb_transport_header (skb) - skb->head;
net/netfilter/nf_nat_proto.c +9 -0
@@ -25,6 +25,7 @@#include <net/ip6_route.h> #include <net/xfrm.h> #include <net/ipv6.h> +#include <net/udp.h> #include <net/netfilter/nf_conntrack_core.h> #include <net/netfilter/nf_conntrack.h>
@@ -75,6 +76,14 @@ static bool udp_manip_pkt(struct sk_buff *skb, hdr = (struct udphdr *)(skb->data + hdroff);__udp_manip_pkt (skb, iphdroff, hdr, tuple, maniptype, !!hdr->check); +if (skb->csum_local) { + hdr->check =0 ; + hdr->check =udp_v4_check (htons (hdr->len), tuple->src.u3.ip, tuple->dst.u3.ip, +lco_csum (skb)); +if (hdr->check ==0 ) + hdr->check =CSUM_MANGLED_0 ; + } +return true; }
Signature · 1 line
2.11.0
Willem de Bruijn
maintainerQuoted 6 lines · Fei Cheng
From: "chenwei.0515" <[email protected]>
If vxlan-dev enable tx csum offload, there are two case of
CHECKSUM_PARTIAL, butudp->checkdonot have the both meanings.
Quoted 6 lines
1. vxlan-dev disable tx csum offload,
udp->checkis just pseudo hdr.
2. vxlan-dev enable tx csum offload,udp->checkis pseudo hdr and csum from outter l4 to innner l4.Unfortunately if there is a nat process after vxlan tx,udp_manip_pkt just use
CSUM_PARTIALto re csum PKT, which is just right on vxlan tx csum disable offload.
The issue is that for encapsulated traffic with local checksum offload, netfilter incorrectly recomputes the outer UDP checksum as if it is an unencapsulated CHECKSUM_PARTIAL packet, correct?
So the underlying issue is that the two types of packets are indistinguishable after udp_set_csum:
} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
uh->check = 0;
uh->check = udp_v4_check(len, saddr, daddr, lco_csum(skb));
if (uh->check == 0)
uh->check = CSUM_MANGLED_0;
} else {
skb->ip_summed = CHECKSUM_PARTIAL;
skb->csum_start = skb_transport_header(skb) - skb->head;
skb->csum_offset = offsetof(struct udphdr, check);
uh->check = ~udp_v4_check(len, saddr, daddr, 0);
}Clearly their ip_summed will be the same.
Quoted 15 lines
This patch use skb->csum_local flag to identify two case, which will csum lco_csum if valid.Signed-off-by chenwei.0515 <[email protected]>--- include/linux/skbuff.h | 1 + net/ipv4/udp.c | 1 + net/netfilter/nf_nat_proto.c | 9 +++++++++ 3 files changed, 11 insertions(+)include/linux/skbuff.h +0
-0
include/linux/skbuff.h struct sk_buff { · line 993
Quoted 4 lines
include/linux/skbuff.h +1
-0@@ -990,6 +990,7 @@ struct sk_buff { __u8 slow_gro:1; __u8 csum_not_inet:1; __u8 scm_io_uring:1; + __u8 csum_local:1;
sk_buff are in space constrained.
I hope we can disambiguate the packets somehow without having to resort to a new flag.
Quoted 45 lines
include/linux/skbuff.h +0
-0#ifdef CONFIG_NET_SCHED __u16 tc_index;/* traffic control index */ net/ipv4/udp.c +1
-0@@ -889,6 +889,7 @@ void udp_set_csum(bool nocheck, struct sk_buff *skb, uh->check =udp_v4_check (len, saddr, daddr,lco_csum (skb));if (uh->check ==0 ) uh->check =CSUM_MANGLED_0 ; + skb->csum_local =1 ; }else { skb->ip_summed =CHECKSUM_PARTIAL ; skb->csum_start =skb_transport_header (skb) - skb->head;net/netfilter/nf_nat_proto.c +9
-1@@ -25,6 +25,7 @@#include <net/ip6_route.h> #include <net/xfrm.h> #include <net/ipv6.h> +#include <net/udp.h> #include <net/netfilter/nf_conntrack_core.h> #include <net/netfilter/nf_conntrack.h> @@ -75,6 +76,14 @@ static bool udp_manip_pkt(struct sk_buff *skb, hdr = (struct udphdr *)(skb->data + hdroff);__udp_manip_pkt (skb, iphdroff, hdr, tuple, maniptype, !!hdr->check); +if (skb->csum_local) { + hdr->check =0 ; + hdr->check =udp_v4_check (htons (hdr->len), tuple->src.u3.ip, tuple->dst.u3.ip, +lco_csum (skb)); +if (hdr->check ==0 ) + hdr->check =CSUM_MANGLED_0 ; + } +return true; } --2.11.0
Quoted 7 lines · Willem de Bruijn
Quoted 6 lines · Fei Cheng
From: "chenwei.0515" <[email protected]>
If vxlan-dev enable tx csum offload, there are two case of
CHECKSUM_PARTIAL, butudp->checkdonot have the both meanings.
Quoted 6 lines
Quoted 6 lines
1. vxlan-dev disable tx csum offload,
udp->checkis just pseudo hdr.
2. vxlan-dev enable tx csum offload,udp->checkis pseudo hdr and csum from outter l4 to innner l4.Unfortunately if there is a nat process after vxlan tx,udp_manip_pkt just use
CSUM_PARTIALto re csum PKT, which is just right on vxlan tx csum disable offload.
In case 1 csum_start should point to the (outer) UDP header, whereas in
case 2 csum_start should point to the inner L4 header (because in the normal TX path w/o NAT, nothing else will ever need to touch the outer csum after this point).
Quoted 3 lines
The issue is that for encapsulated traffic with local checksum offload, netfilter incorrectly recomputes the outer UDP checksum as if it is an unencapsulated
CHECKSUM_PARTIALpacket, correct?
So if netfilter sees a packet with CHECKSUM_PARTIAL whose csum_start
doesn't point to the header nf NAT is editing, that's exactly the case where it needs to use lco_csum to calculate the new outer sum. No?
-ed
PS. Fei, your emails aren't reaching the netdev mailing list, probably
because you're sending as HTML. Please switch to plain text.
Thank you for remind plain text.
Use csum_start to seperate these two cases, maybe a good idea. 1.Disable tx csumskb->ip_summed == CHECKSUM_PARTIAL && skb_transport_header == udp 2.Enable tx csumskb->ip_summed == CHECKSUM_PARTIAL && skb_transport_header != udp
Correct?
Quoted 32 lines · Edward Cree
Quoted 7 lines · Willem de Bruijn
Quoted 6 lines · Fei Cheng
From: "chenwei.0515" <[email protected]>
If vxlan-dev enable tx csum offload, there are two case of
CHECKSUM_PARTIAL, butudp->checkdonot have the both meanings.Quoted 6 lines
Quoted 6 lines
1. vxlan-dev disable tx csum offload,
udp->checkis just pseudo hdr.
2. vxlan-dev enable tx csum offload,udp->checkis pseudo hdr and csum from outter l4 to innner l4.Unfortunately if there is a nat process after vxlan tx,udp_manip_pkt just use
CSUM_PARTIALto re csum PKT, which is just right on vxlan tx csum disable offload.In case 1
csum_startshould point to the (outer) UDP header, whereas in
case 2csum_startshould point to the inner L4 header (because in the normal TX path w/o NAT, nothing else will ever need to touch the outer csum after this point).Quoted 3 lines
The issue is that for encapsulated traffic with local checksum offload, netfilter incorrectly recomputes the outer UDP checksum as if it is an unencapsulated
CHECKSUM_PARTIALpacket, correct?So if netfilter sees a packet with
CHECKSUM_PARTIALwhosecsum_start
doesn't point to the header nf NAT is editing, that's exactly the case where it needs to uselco_csumto calculate the new outer sum. No?-ed
PS. Fei, your emails aren't reaching the netdev mailing list, probably
because you're sending as HTML. Please switch to plain text.
Quoted 8 lines · Fei Cheng
Thank you for remind plain text.
Usecsum_startto seperate these two cases, maybe a good idea. 1.Disable tx csumskb->ip_summed==CHECKSUM_PARTIAL&&skb_transport_header== udp 2.Enable tx csumskb->ip_summed==CHECKSUM_PARTIAL&&skb_transport_header!= udpCorrect?
What do you mean by "skb_transport_header == udp"? That it is a UDP
header? Or that it is at the offset of the UDP header? The inner L4 packet could be UDP as well.
And why are you even looking at skb_transport_header when it's
csum_start that determines what the hardware will do?
AFAICT nothing in nf_nat_proto.c looks at skb_transport_header anyway;
indeed in nf_nat_icmp_reply_translation() we can call into this code with hdroff ending up pointing way deeper in the packet.
In any case, after digging deeper into the netfilter code, it looks to
me like there's no issue in the first place: netfilter doesn't 'recompute' the UDP checksum, it just accumulates delta into it to account for the changes it made to the packet, on the assumption that the existing checksum is correct.
Which AIUI will do the right thing whether the checksum is
* a correct checksum for an unencapsulated packet
* a pseudohdr sum for a CHECKSUM_PARTIAL (unencapsulated) packet
* a correct (LCO) checksum for an encapsulated packet, whose inner L4 checksum may or may not be CHECKSUM_PARTIAL offloaded.
Do you have a test case that breaks with the current code?
-ed
test case
1. create vxlan tunnel with tx csum offload on.
2. iptables nat the packet out from vxlan
3. vxlan remote will see the packet udp csum error.
I used to use skb->csum_valid to distinguish the two cases. If donot add extra skb flags, maybe csum_start can seperate too.
Quoted 33 lines · Edward Cree
Quoted 8 lines · Fei Cheng
Thank you for remind plain text.
Usecsum_startto seperate these two cases, maybe a good idea. 1.Disable tx csumskb->ip_summed==CHECKSUM_PARTIAL&&skb_transport_header== udp 2.Enable tx csumskb->ip_summed==CHECKSUM_PARTIAL&&skb_transport_header!= udpCorrect?
What do you mean by "
skb_transport_header== udp"? That it is a UDP
header? Or that it is at the offset of the UDP header? The inner L4 packet could be UDP as well.
And why are you even looking atskb_transport_headerwhen it's
csum_startthat determines what the hardware will do?
AFAICT nothing innf_nat_proto.clooks atskb_transport_headeranyway;
indeed innf_nat_icmp_reply_translation()we can call into this code with hdroff ending up pointing way deeper in the packet.In any case, after digging deeper into the netfilter code, it looks to
me like there's no issue in the first place: netfilter doesn't 'recompute' the UDP checksum, it just accumulates delta into it to account for the changes it made to the packet, on the assumption that the existing checksum is correct.
Which AIUI will do the right thing whether the checksum is
* a correct checksum for an unencapsulated packet
* a pseudohdr sum for aCHECKSUM_PARTIAL(unencapsulated) packet
* a correct (LCO) checksum for an encapsulated packet, whose inner L4 checksum may or may not beCHECKSUM_PARTIALoffloaded.Do you have a test case that breaks with the current code?
-ed
kernel test robot
build botThank you for the patch! Perhaps something to improve:
Hi Fei,
Thank you for the patch! Perhaps something to improve:
[auto build test WARNING on linus/master]
[also build test WARNING on v6.3-rc5]
[cannot apply to next-20230331]
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Fei-Cheng/udp-nat-vxlan-tx-after-nat-should-recsum-if-vxlan-tx-offload-on/20230401-103127 patch link: https://lore.kernel.org/r/20230401023029.967357-1-chenwei.0515%40bytedance.com patch subject: [PATCH] udp:nat:vxlan tx after nat should recsum if vxlan tx offload on config: microblaze-randconfig-s052-20230402 (https://download.01.org/0day-ci/archive/20230403/[email protected]/config) compiler: microblaze-linux-gcc (GCC) 12.1.0
reproduce:
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# apt-get install sparse
# sparse version: v0.6.4-39-gce1a6720-dirty
# https://github.com/intel-lab-lkp/linux/commit/58c404cd43734d67706e3dc48c04a3e11e9dd8e8
git remote add linux-review https://github.com/intel-lab-lkp/linux
git fetch --no-tags linux-review Fei-Cheng/udp-nat-vxlan-tx-after-nat-should-recsum-if-vxlan-tx-offload-on/20230401-103127
git checkout 58c404cd43734d67706e3dc48c04a3e11e9dd8e8
# save the config file
mkdir build_dir && cp config build_dir/.config
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-12.1.0 make.cross C=1 CF='-fdiagnostic-prefix -D__CHECK_ENDIAN__' O=build_dir ARCH=microblaze olddefconfig COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-12.1.0 make.cross C=1 CF='-fdiagnostic-prefix -D__CHECK_ENDIAN__' O=build_dir ARCH=microblaze SHELL=/bin/bash net/netfilter/
If you fix the issue, kindly add following tag where applicable
| Reported-by: kernel test robot <[email protected]>
| Link: https://lore.kernel.org/oe-kbuild-all/[email protected]/
sparse warnings: (new ones prefixed by >>)
>> net/netfilter/nf_nat_proto.c:81:43: sparse: sparse: cast from restricted __be16 >> net/netfilter/nf_nat_proto.c:81:43: sparse: sparse: incorrect type in argument 1 (different base types) @@ expected int len @@ got restricted __be16 [usertype] @@
net/netfilter/nf_nat_proto.c:81:43: sparse: expected int len
net/netfilter/nf_nat_proto.c:81:43: sparse: got restricted __be16 [usertype]
vim +81 net/netfilter/nf_nat_proto.c
65
66 static bool udp_manip_pkt(struct sk_buff *skb,
67 unsigned int iphdroff, unsigned int hdroff,
68 const struct nf_conntrack_tuple *tuple,
69 enum nf_nat_manip_type maniptype)
70 {
71 struct udphdr *hdr;
72
73 if (skb_ensure_writable(skb, hdroff + sizeof(*hdr)))
74 return false;
75
76 hdr = (struct udphdr *)(skb->data + hdroff);
77 __udp_manip_pkt(skb, iphdroff, hdr, tuple, maniptype, !!hdr->check);
78
79 if (skb->csum_local) {
80 hdr->check = 0;
> 81 hdr->check = udp_v4_check(htons(hdr->len), tuple->src.u3.ip, tuple->dst.u3.ip,
82 lco_csum(skb));
83 if (hdr->check == 0)
84 hdr->check = CSUM_MANGLED_0;
85 }
86
87 return true;
88 }
89 Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests