udp_tunnel: GRO optimizations

Most UDP tunnels bind a socket to a local port, with ANY address, no peer and no interface index specified.
Additionally it's quite common to have a single tunnel device per namespace.

Track in each namespace the UDP tunnel socket respecting the above. When only a single one is present, store a reference in the netns.

When such reference is not NULL, UDP tunnel GRO lookup just need to match the incoming packet destination port vs the socket local port.

The tunnel socket never set the reuse[port] flag[s], when bound to no address and interface, no other socket can exist in the same netns matching the specified local port.

Signed-off-by Paolo Abeni <[email protected]>

---

net/ipv4/udp.c +2 -0

+       }
+#endif
        udp_sysctl_init(net);
        udp_set_table(net);

net/ipv4/udp_offload.c +37 -0

index c1a85b300ee87..ac6dd2703190e 100644
@@ -12,6 +12,38 @@
 #include <net/udp.h>
 #include <net/protocol.h>
 #include <net/inet_common.h>
+#include <net/udp_tunnel.h>
+
+#if IS_ENABLED(CONFIG_NET_UDP_TUNNEL)
+static DEFINE_SPINLOCK(udp_tunnel_gro_lock);
+
+void udp_tunnel_update_gro_lookup(struct net *net, struct sock *sk, bool add)
+{
+       bool is_ipv6 = sk->sk_family == AF_INET6;
+       struct udp_sock *tup, *up = udp_sk(sk);
+       struct udp_tunnel_gro *udp_tunnel_gro;
+
+       spin_lock(&udp_tunnel_gro_lock);
+       udp_tunnel_gro = &net->ipv4.udp_tunnel_gro[is_ipv6];
+       if (add)
+               hlist_add_head(&up->tunnel_list, &udp_tunnel_gro->list);
+       else
+               hlist_del_init(&up->tunnel_list);
+
+       if (udp_tunnel_gro->list.first &&
+           !udp_tunnel_gro->list.first->next) {
+               tup = hlist_entry(udp_tunnel_gro->list.first, struct udp_sock,
+                                 tunnel_list);
+
+               rcu_assign_pointer(udp_tunnel_gro->sk, (struct sock *)tup);
+       } else {
+               rcu_assign_pointer(udp_tunnel_gro->sk, NULL);
+       }
+
+       spin_unlock(&udp_tunnel_gro_lock);
+}
+EXPORT_SYMBOL_GPL(udp_tunnel_update_gro_lookup);
+#endif
 
 static struct sk_buff *__skb_udp_tunnel_segment(struct sk_buff *skb,
        netdev_features_t features,

@@ -631,8 +663,13 @@ static struct sock *udp4_gro_lookup_skb(struct sk_buff *skb, __be16 sport,
 {
        const struct iphdr *iph = skb_gro_network_header(skb);
        struct net *net = dev_net_rcu(skb->dev);
+       struct sock *sk;
        int iif, sdif;
 
+       sk = udp_tunnel_sk(net, false);
+       if (sk && dport == htons(sk->sk_num))
+               return sk;
+
        inet_get_iif_sdif(skb, &iif, &sdif);
 
        return __udp4_lib_lookup(net, iph->saddr, sport,

net/ipv4/udp_tunnel_core.c +12 -0

index 619a53eb672da..b969c997c89c7 100644
@@ -58,6 +58,15 @@ int udp_sock_create4(struct net *net, struct udp_port_cfg *cfg,
 }
 EXPORT_SYMBOL(udp_sock_create4);
 
+static inline bool sk_saddr_any(struct sock *sk)
+{
+#if IS_ENABLED(CONFIG_IPV6)
+       return ipv6_addr_any(&sk->sk_v6_rcv_saddr);
+#else
+       return !sk->sk_rcv_saddr;
+#endif
+}
+
 void setup_udp_tunnel_sock(struct net *net, struct socket *sock,
                           struct udp_tunnel_sock_cfg *cfg)
 {

@@ -80,6 +89,9 @@ void setup_udp_tunnel_sock(struct net *net, struct socket *sock,
        udp_sk(sk)->gro_complete = cfg->gro_complete;
 
        udp_tunnel_encap_enable(sk);
+
+       if (!sk->sk_dport && !sk->sk_bound_dev_if && sk_saddr_any(sock->sk))
+               udp_tunnel_update_gro_lookup(net, sock->sk, true);
 }
 EXPORT_SYMBOL_GPL(setup_udp_tunnel_sock);
 

net/ipv6/udp.c +1 -0

index 3a0d6c5a8286b..3087022d18a55 100644
@@ -46,6 +46,7 @@
 #include <net/tcp_states.h>
 #include <net/ip6_checksum.h>
 #include <net/ip6_tunnel.h>
+#include <net/udp_tunnel.h>
 #include <net/xfrm.h>
 #include <net/inet_hashtables.h>
 #include <net/inet6_hashtables.h>

net/ipv6/udp.c +1 -0

+                       udp_tunnel_cleanup_gro(sk);
                        udp_encap_disable();
                }
        }

net/ipv6/udp_offload.c +5 -1

index 404212dfc99ab..d8445ac1b2e43 100644
@@ -118,8 +118,13 @@ static struct sock *udp6_gro_lookup_skb(struct sk_buff *skb, __be16 sport,
 {
        const struct ipv6hdr *iph = skb_gro_network_header(skb);
        struct net *net = dev_net_rcu(skb->dev);
+       struct sock *sk;
        int iif, sdif;
 
+       sk = udp_tunnel_sk(net, true);
+       if (sk && dport == htons(sk->sk_num))
+               return sk;
+
        inet6_get_iif_sdif(skb, &iif, &sdif);
 
        return __udp6_lib_lookup(net, &iph->saddr, sport,
--

2.48.1

net/ipv4/udp.c static void __net_init udp_set_table(struct net *net) · line 3815

Quoted 9 lines

static int __net_init udp_pernet_init(struct net *net) {

net/ipv4/udp.c +7 -0

+#if IS_ENABLED(CONFIG_NET_UDP_TUNNEL)
+       int i;
+
+       /* No tunnel is configured */
+       for (i = 0; i < ARRAY_SIZE(net->ipv4.udp_tunnel_gro); ++i) {
+               INIT_HLIST_HEAD(&net->ipv4.udp_tunnel_gro[i].list);
+               rcu_assign_pointer(net->ipv4.udp_tunnel_gro[1].sk, NULL);

typo : [i] is what you meant (instead of [1])

net/ipv6/udp.c void udpv6_destroy_sock(struct sock *sk) · line 1827

Quoted 4 lines

net/ipv6/udp.c +0 -0

@@ -1824,6 +1825,7 @@ void udpv6_destroy_sock(struct sock *sk)
                }
                if (udp_test_bit(ENCAP_ENABLED, sk)) {
                        static_branch_dec(&udpv6_encap_needed_key);

In ipv4, you removed the static_branch_dec(&udp_encap_needed_key);

Most UDP tunnels bind a socket to a local port, with ANY address, no peer and no interface index specified.
Additionally it's quite common to have a single tunnel device per namespace.

Track in each namespace the UDP tunnel socket respecting the above. When only a single one is present, store a reference in the netns.

When such reference is not NULL, UDP tunnel GRO lookup just need to match the incoming packet destination port vs the socket local port.

The tunnel socket never set the reuse[port] flag[s], when bound to no address and interface, no other socket can exist in the same netns matching the specified local port.

Signed-off-by Paolo Abeni <[email protected]>

net/ipv4/udp_offload.c +3 -0

index c1a85b300ee87..ac6dd2703190e 100644
@@ -12,6 +12,38 @@
 #include <net/udp.h>
 #include <net/protocol.h>
 #include <net/inet_common.h>
+#include <net/udp_tunnel.h>
+
+#if IS_ENABLED(CONFIG_NET_UDP_TUNNEL)

net/ipv4/udp_offload.c +19 -0

+	if (add)
+		hlist_add_head(&up->tunnel_list, &udp_tunnel_gro->list);
+	else
+		hlist_del_init(&up->tunnel_list);
+
+	if (udp_tunnel_gro->list.first &&
+	    !udp_tunnel_gro->list.first->next) {
+		tup = hlist_entry(udp_tunnel_gro->list.first, struct udp_sock,
+				  tunnel_list);
+
+		rcu_assign_pointer(udp_tunnel_gro->sk, (struct sock *)tup);
+	} else {
+		rcu_assign_pointer(udp_tunnel_gro->sk, NULL);
+	}
+
+	spin_unlock(&udp_tunnel_gro_lock);
+}
+EXPORT_SYMBOL_GPL(udp_tunnel_update_gro_lookup);
+#endif
 
 static struct sk_buff *__skb_udp_tunnel_segment(struct sk_buff *skb,
 	netdev_features_t features,

net/ipv4/udp_offload.c +0 -0

 	inet_get_iif_sdif(skb, &iif, &sdif);
 
 	return __udp4_lib_lookup(net, iph->saddr, sport,

Quoted 33 lines · Paolo Abeni

Most UDP tunnels bind a socket to a local port, with ANY address, no peer and no interface index specified.
Additionally it's quite common to have a single tunnel device per namespace.

Track in each namespace the UDP tunnel socket respecting the above. When only a single one is present, store a reference in the netns.

When such reference is not NULL, UDP tunnel GRO lookup just need to match the incoming packet destination port vs the socket local port.

The tunnel socket never set the reuse[port] flag[s], when bound to no address and interface, no other socket can exist in the same netns matching the specified local port.

Signed-off-by Paolo Abeni <[email protected]>

net/ipv4/udp_offload.c +8 -0

index c1a85b300ee87..ac6dd2703190e 100644
@@ -12,6 +12,38 @@
 #include <net/udp.h>
 #include <net/protocol.h>
 #include <net/inet_common.h>
+#include <net/udp_tunnel.h>
+
+#if IS_ENABLED(CONFIG_NET_UDP_TUNNEL)
+static DEFINE_SPINLOCK(udp_tunnel_gro_lock);
+
+void udp_tunnel_update_gro_lookup(struct net *net, struct sock *sk, bool add)
+{
+	bool is_ipv6 = sk->sk_family == AF_INET6;

net/ipv4/udp_offload.c line 27

Quoted 5 lines

net/ipv4/udp_offload.c +5 -0

+	struct udp_sock *tup, *up = udp_sk(sk);
+	struct udp_tunnel_gro *udp_tunnel_gro;
+
+	spin_lock(&udp_tunnel_gro_lock);
+	udp_tunnel_gro = &net->ipv4.udp_tunnel_gro[is_ipv6];

It's a bit odd to have an ipv6 member in netns.ipv4. Does it significantly simplify the code vs a separate entry in netns.ipv6?

net/ipv4/udp_offload.c static struct sock *udp4_gro_lookup_skb(struct sk_buff *skb, __be16 sport, · line 672

Quoted 11 lines

net/ipv4/udp_offload.c +5 -0

@@ -631,8 +663,13 @@ static struct sock *udp4_gro_lookup_skb(struct sk_buff *skb, __be16 sport,
 {
 	const struct iphdr *iph = skb_gro_network_header(skb);
 	struct net *net = dev_net_rcu(skb->dev);
+	struct sock *sk;
 	int iif, sdif;
 
+	sk = udp_tunnel_sk(net, false);
+	if (sk && dport == htons(sk->sk_num))
+		return sk;
+

This improves tunnel performance at a slight cost to everything else, by having the tunnel check before the normal socket path.

Does a 5% best case gain warrant that? Not snark, I don't have a good answer.

Quoted 34 lines · Willem de Bruijn

Quoted 33 lines · Paolo Abeni

Most UDP tunnels bind a socket to a local port, with ANY address, no peer and no interface index specified.
Additionally it's quite common to have a single tunnel device per namespace.

Track in each namespace the UDP tunnel socket respecting the above. When only a single one is present, store a reference in the netns.

When such reference is not NULL, UDP tunnel GRO lookup just need to match the incoming packet destination port vs the socket local port.

The tunnel socket never set the reuse[port] flag[s], when bound to no address and interface, no other socket can exist in the same netns matching the specified local port.

Signed-off-by Paolo Abeni <[email protected]>

net/ipv4/udp_offload.c +8 -0

index c1a85b300ee87..ac6dd2703190e 100644
@@ -12,6 +12,38 @@
 #include <net/udp.h>
 #include <net/protocol.h>
 #include <net/inet_common.h>
+#include <net/udp_tunnel.h>
+
+#if IS_ENABLED(CONFIG_NET_UDP_TUNNEL)
+static DEFINE_SPINLOCK(udp_tunnel_gro_lock);
+
+void udp_tunnel_update_gro_lookup(struct net *net, struct sock *sk, bool add)
+{
+    bool is_ipv6 = sk->sk_family == AF_INET6;

Quoted 8 lines

net/ipv4/udp_offload.c line 27

Quoted 5 lines

net/ipv4/udp_offload.c +5 -0

+    struct udp_sock *tup, *up = udp_sk(sk);
+    struct udp_tunnel_gro *udp_tunnel_gro;
+
+    spin_lock(&udp_tunnel_gro_lock);
+    udp_tunnel_gro = &net->ipv4.udp_tunnel_gro[is_ipv6];

It's a bit odd to have an ipv6 member in netns.ipv4. Does it significantly simplify the code vs a separate entry in netns.ipv6?

The code complexity should not change much. I place both the ipv4 and ipv6 data there to allow cache-line based optimization, as all the netns fast-path fields are under struct netns_ipv4.

Currently the UDP tunnel related fields share the same cache-line of udp_table.

Quoted 15 lines

net/ipv4/udp_offload.c static struct sock *udp4_gro_lookup_skb(struct sk_buff *skb, __be16 sport, · line 672

Quoted 11 lines

net/ipv4/udp_offload.c +5 -0

@@ -631,8 +663,13 @@ static struct sock *udp4_gro_lookup_skb(struct sk_buff *skb, __be16 sport,
 {
     const struct iphdr *iph = skb_gro_network_header(skb);
     struct net *net = dev_net_rcu(skb->dev);
+    struct sock *sk;
     int iif, sdif;
 
+    sk = udp_tunnel_sk(net, false);
+    if (sk && dport == htons(sk->sk_num))
+            return sk;
+

This improves tunnel performance at a slight cost to everything else, by having the tunnel check before the normal socket path.

Quoted 2 lines

Does a 5% best case gain warrant that? Not snark, I don't have a good answer.

We enter this function only when udp_encap_needed_key is true: ~an UDP tunnel has been configured[1].

When tunnels are enabled, AFAIK the single tunnel device is the most common and most relevant scenario, and in such setup this gives measurable performance improvement. Other tunnel-based scenarios will see the additional cost of a single conditional (using data on an already hot cacheline, due to the above layout).

If you are concerned about such cost, I can add an additional static branch protecting the above code chunk, so that the conditional will be performed only when there is a single UDP tunnel configured. Please, let me know.

Thanks,

Paolo

[1] to be more accurate: an UDP tunnel or an UDP socket with GRO enabled

The UDP tunnel GRO stage is source of measurable overhead for workload based on UDP-encapsulated traffic: each incoming packets requires a full UDP socket lookup and an indirect call.

In the most common setups a single UDP tunnel device is used. In such case we can optimize both the lookup and the indirect call.

Patch 1 tracks per netns the active UDP tunnels and replaces the socket lookup with a single destination port comparison when possible.

Patch 2 tracks the different types of UDP tunnels and replaces the indirect call with a static one when there is a single UDP tunnel type active.

I measure ~5% performance improvement in TCP over UDP tunnel stream tests on top of this series.

Quoted 10 lines · On Thu, 6 Mar 2025 16:56:51 +0100 Paolo Abeni wrote:

The UDP tunnel GRO stage is source of measurable overhead for workload based on UDP-encapsulated traffic: each incoming packets requires a full UDP socket lookup and an indirect call.

In the most common setups a single UDP tunnel device is used. In such case we can optimize both the lookup and the indirect call.

Patch 1 tracks per netns the active UDP tunnels and replaces the socket lookup with a single destination port comparison when possible.

Quoted 6 lines

Patch 2 tracks the different types of UDP tunnels and replaces the indirect call with a static one when there is a single UDP tunnel type active.

I measure ~5% performance improvement in TCP over UDP tunnel stream tests on top of this series.

Breaks the build with NET_UDP_TUNNEL=n (in contest) :(

net/ipv4/udp_offload.c: In function ‘udp_tunnel_gro_rcv’:
net/ipv4/udp_offload.c:172:16: error: returning ‘struct sk_buff *’ from a function with incompatible return type ‘struct skbuff *’ [-Werror=incompatible-pointer-types]

  172 |         return call_gro_receive_sk(udp_sk(sk)->gro_receive, sk, head, skb);
      |                ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

net/ipv4/udp_offload.c: In function ‘udp_gro_receive’:
net/ipv4/udp_offload.c:786:12: error: assignment to ‘struct sk_buff *’ from incompatible pointer type ‘struct skbuff *’ [-Werror=incompatible-pointer-types]
786 | pp = udp_tunnel_gro_rcv(sk, head, skb);
| ^
In file included from ./include/linux/seqlock.h:19,

                 from ./include/linux/dcache.h:11,
                 from ./include/linux/fs.h:8,
                 from ./include/linux/highmem.h:5,
                 from ./include/linux/bvec.h:10,
                 from ./include/linux/skbuff.h:17,
                 from net/ipv4/udp_offload.c:9:
net/ipv4/udp_offload.c: In function ‘udpv4_offload_init’:
net/ipv4/udp_offload.c:936:21: error: ‘udp_tunnel_gro_type_lock’ undeclared (first use in this function); did you mean ‘udp_tunnel_gro_rcv’?
  936 |         mutex_init(&udp_tunnel_gro_type_lock);
      |                     ^~~~~~~~~~~~~~~~~~~~~~~~

./include/linux/mutex.h:64:23: note: in definition of macro ‘mutex_init’
64 | __mutex_init((mutex), #mutex, &__key); \
| ^~~~~
net/ipv4/udp_offload.c:936:21: note: each undeclared identifier is reported only once for each function it appears in

  936 |         mutex_init(&udp_tunnel_gro_type_lock);
      |                     ^~~~~~~~~~~~~~~~~~~~~~~~

./include/linux/mutex.h:64:23: note: in definition of macro ‘mutex_init’
64 | __mutex_init((mutex), #mutex, &__key); \
| ^~~~~
cc1: all warnings being treated as errors