Skip to contentSkip to conversation

ipv6: remove max_size check inline with ipv4

PATCH Applied Jon Maxwell 4 participants 6 messages Jan 12, 2023 – Jan 14, 2023

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 ipv6: remove max_size check inline with ipv4

v2: Correct syntax error in net/ipv6/route.c

In ip6_dst_gc() replace:

if (entries > gc_thresh)

With:

if (entries > ops->gc_thresh)

Sending Ipv6 packets in a loop via a raw socket triggers an issue where a route is cloned by ip6_rt_cache_alloc() for each packet sent. This quickly consumes the Ipv6 max_size threshold which defaults to 4096 resulting in these warnings:

[1] 99.187805] dst_alloc: 7728 callbacks suppressed
[2] Route cache is full: consider increasing sysctl net.ipv6.route.max_size. .
.
[300] Route cache is full: consider increasing sysctl net.ipv6.route.max_size.

When this happens the packet is dropped and sendto() gets a network is unreachable error:

# ./a.out -s 

remaining pkt 200557 errno 101
remaining pkt 196462 errno 101
.
.
remaining pkt 126821 errno 101

Implement David Aherns suggestion to remove max_size check seeing that Ipv6 has a GC to manage memory usage. Ipv4 already does not check max_size.

Here are some memory comparisons for Ipv4 vs Ipv6 with the patch:

Test by running 5 instances of a program that sends UDP packets to a raw socket 5000000 times. Compare Ipv4 and Ipv6 performance with a similar program.

Ipv4:

Before test:

# grep -e Slab -e Free /proc/meminfo
MemFree:        29427108 kB
Slab:             237612 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache       1912   2528    256   32    2 : tunables    0    0    0 
xfrm_dst_cache         0      0    320   25    2 : tunables    0    0    0 
ip_dst_cache        2881   3990    192   42    2 : tunables    0    0    0 

During test:

# grep -e Slab -e Free /proc/meminfo
MemFree:        29417608 kB
Slab:             247712 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache       1912   2528    256   32    2 : tunables    0    0    0 
xfrm_dst_cache         0      0    320   25    2 : tunables    0    0    0 
ip_dst_cache       44394  44394    192   42    2 : tunables    0    0    0 

After test:

# grep -e Slab -e Free /proc/meminfo
MemFree:        29422308 kB
Slab:             238104 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache       1912   2528    256   32    2 : tunables    0    0    0 
xfrm_dst_cache         0      0    320   25    2 : tunables    0    0    0 
ip_dst_cache        3048   4116    192   42    2 : tunables    0    0    0 

Ipv6 with patch:

Errno 101 errors are not observed anymore with the patch.

Before test:

# grep -e Slab -e Free /proc/meminfo
MemFree:        29422308 kB
Slab:             238104 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache       1912   2528    256   32    2 : tunables    0    0    0 
xfrm_dst_cache         0      0    320   25    2 : tunables    0    0    0 
ip_dst_cache        3048   4116    192   42    2 : tunables    0    0    0 

During Test:

# grep -e Slab -e Free /proc/meminfo
MemFree:        29431516 kB
Slab:             240940 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache      11980  12064    256   32    2 : tunables    0    0    0
xfrm_dst_cache         0      0    320   25    2 : tunables    0    0    0
ip_dst_cache        3048   4116    192   42    2 : tunables    0    0    0

After Test:

# grep -e Slab -e Free /proc/meminfo
MemFree:        29441816 kB
Slab:             238132 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache       1902   2432    256   32    2 : tunables    0    0    0
xfrm_dst_cache         0      0    320   25    2 : tunables    0    0    0
ip_dst_cache        3048   4116    192   42    2 : tunables    0    0    0
---
 include/net/dst_ops.h |  2 +-
 net/core/dst.c        |  8 ++------
 net/ipv6/route.c      | 13 +++++--------
 3 files changed, 8 insertions(+), 15 deletions(-)
include/net/dst_ops.h +1 -1
index 88ff7bb2bb9b..632086b2f644 100644
@@ -16,7 +16,7 @@ struct dst_ops {
 	unsigned short		family;
 	unsigned int		gc_thresh;
 
-	int			(*gc)(struct dst_ops *ops);
+	void			(*gc)(struct dst_ops *ops);
 	struct dst_entry *	(*check)(struct dst_entry *, __u32 cookie);
 	unsigned int		(*default_advmss)(const struct dst_entry *);
 	unsigned int		(*mtu)(const struct dst_entry *);
net/core/dst.c +2 -6
index 6d2dd03dafa8..31c08a3386d3 100644
@@ -82,12 +82,8 @@ void *dst_alloc(struct dst_ops *ops, struct net_device *dev,
 
 	if (ops->gc &&
 	    !(flags & DST_NOCOUNT) &&
-	    dst_entries_get_fast(ops) > ops->gc_thresh) {
-		if (ops->gc(ops)) {
-			pr_notice_ratelimited("Route cache is full: consider increasing sysctl net.ipv6.route.max_size.\n");
-			return NULL;
-		}
-	}
+	    dst_entries_get_fast(ops) > ops->gc_thresh)
+		ops->gc(ops);
 
 	dst = kmem_cache_alloc(ops->kmem_cachep, GFP_ATOMIC);
 	if (!dst)
net/ipv6/route.c +5 -8
index e74e0361fd92..b643dda68d31 100644
@@ -91,7 +91,7 @@ static struct dst_entry *ip6_negative_advice(struct dst_entry *);
 static void		ip6_dst_destroy(struct dst_entry *);
 static void		ip6_dst_ifdown(struct dst_entry *,
 				       struct net_device *dev, int how);
-static int		 ip6_dst_gc(struct dst_ops *ops);
+static void		 ip6_dst_gc(struct dst_ops *ops);
 
 static int		ip6_pkt_discard(struct sk_buff *skb);
 static int		ip6_pkt_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb);
@@ -3284,11 +3284,10 @@ struct dst_entry *icmp6_dst_alloc(struct net_device *dev,
 	return dst;
 }
 
-static int ip6_dst_gc(struct dst_ops *ops)
+static void ip6_dst_gc(struct dst_ops *ops)
 {
 	struct net *net = container_of(ops, struct net, ipv6.ip6_dst_ops);
 	int rt_min_interval = net->ipv6.sysctl.ip6_rt_gc_min_interval;
-	int rt_max_size = net->ipv6.sysctl.ip6_rt_max_size;
 	int rt_elasticity = net->ipv6.sysctl.ip6_rt_gc_elasticity;
 	int rt_gc_timeout = net->ipv6.sysctl.ip6_rt_gc_timeout;
 	unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc;
@@ -3296,11 +3295,10 @@ static int ip6_dst_gc(struct dst_ops *ops)
 	int entries;
 
 	entries = dst_entries_get_fast(ops);
-	if (entries > rt_max_size)
+	if (entries > ops->gc_thresh)
 		entries = dst_entries_get_slow(ops);
 
-	if (time_after(rt_last_gc + rt_min_interval, jiffies) &&
-	    entries <= rt_max_size)
+	if (time_after(rt_last_gc + rt_min_interval, jiffies))
 		goto out;
 
 	fib6_run_gc(atomic_inc_return(&net->ipv6.ip6_rt_gc_expire), net, true);
@@ -3310,7 +3308,6 @@ static int ip6_dst_gc(struct dst_ops *ops)
 out:
 	val = atomic_read(&net->ipv6.ip6_rt_gc_expire);
 	atomic_set(&net->ipv6.ip6_rt_gc_expire, val - (val >> rt_elasticity));
-	return entries > rt_max_size;
 }
 
 static int ip6_nh_lookup_table(struct net *net, struct fib6_config *cfg,
@@ -6512,7 +6509,7 @@ static int __net_init ip6_route_net_init(struct net *net)
 #endif
 
 	net->ipv6.sysctl.flush_delay = 0;
-	net->ipv6.sysctl.ip6_rt_max_size = 4096;
+	net->ipv6.sysctl.ip6_rt_max_size = INT_MAX;
 	net->ipv6.sysctl.ip6_rt_gc_min_interval = HZ / 2;
 	net->ipv6.sysctl.ip6_rt_gc_timeout = 60*HZ;
 	net->ipv6.sysctl.ip6_rt_gc_interval = 30*HZ;
Signature · 1 line
2.31.1
#1 lore raw

Hi Jon,

Quoted 35 lines · On Thu, 12 Jan 2023 12:25:32 +1100 Jon Maxwell <[email protected]> wrote:

v2: Correct syntax error in net/ipv6/route.c

In ip6_dst_gc() replace:

if (entries > gc_thresh)

With:

if (entries > ops->gc_thresh)

Sending Ipv6 packets in a loop via a raw socket triggers an issue where a route is cloned by ip6_rt_cache_alloc() for each packet sent. This quickly consumes the Ipv6 max_size threshold which defaults to 4096 resulting in these warnings:

[1] 99.187805] dst_alloc: 7728 callbacks suppressed
[2] Route cache is full: consider increasing sysctl net.ipv6.route.max_size. .
.
[300] Route cache is full: consider increasing sysctl net.ipv6.route.max_size.

When this happens the packet is dropped and sendto() gets a network is unreachable error:

# ./a.out -s 

remaining pkt 200557 errno 101
remaining pkt 196462 errno 101
.
.
remaining pkt 126821 errno 101

Implement David Aherns suggestion to remove max_size check seeing that Ipv6 has a GC to manage memory usage. Ipv4 already does not check max_size.

Quoted 8 lines

Here are some memory comparisons for Ipv4 vs Ipv6 with the patch:

Test by running 5 instances of a program that sends UDP packets to a raw socket 5000000 times. Compare Ipv4 and Ipv6 performance with a similar program.

Ipv4:

is it supposed to be Ipv6, right?

Ciao,
Andrea

Quoted 169 lines

Before test:

# grep -e Slab -e Free /proc/meminfo
MemFree:        29427108 kB
Slab:             237612 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache       1912   2528    256   32    2 : tunables    0    0    0 
xfrm_dst_cache         0      0    320   25    2 : tunables    0    0    0 
ip_dst_cache        2881   3990    192   42    2 : tunables    0    0    0 

During test:

# grep -e Slab -e Free /proc/meminfo
MemFree:        29417608 kB
Slab:             247712 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache       1912   2528    256   32    2 : tunables    0    0    0 
xfrm_dst_cache         0      0    320   25    2 : tunables    0    0    0 
ip_dst_cache       44394  44394    192   42    2 : tunables    0    0    0 

After test:

# grep -e Slab -e Free /proc/meminfo
MemFree:        29422308 kB
Slab:             238104 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache       1912   2528    256   32    2 : tunables    0    0    0 
xfrm_dst_cache         0      0    320   25    2 : tunables    0    0    0 
ip_dst_cache        3048   4116    192   42    2 : tunables    0    0    0 

Ipv6 with patch:

Errno 101 errors are not observed anymore with the patch.

Before test:

# grep -e Slab -e Free /proc/meminfo
MemFree:        29422308 kB
Slab:             238104 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache       1912   2528    256   32    2 : tunables    0    0    0 
xfrm_dst_cache         0      0    320   25    2 : tunables    0    0    0 
ip_dst_cache        3048   4116    192   42    2 : tunables    0    0    0 

During Test:

# grep -e Slab -e Free /proc/meminfo
MemFree:        29431516 kB
Slab:             240940 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache      11980  12064    256   32    2 : tunables    0    0    0
xfrm_dst_cache         0      0    320   25    2 : tunables    0    0    0
ip_dst_cache        3048   4116    192   42    2 : tunables    0    0    0

After Test:

# grep -e Slab -e Free /proc/meminfo
MemFree:        29441816 kB
Slab:             238132 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache       1902   2432    256   32    2 : tunables    0    0    0
xfrm_dst_cache         0      0    320   25    2 : tunables    0    0    0
ip_dst_cache        3048   4116    192   42    2 : tunables    0    0    0
---
 include/net/dst_ops.h |  2 +-
 net/core/dst.c        |  8 ++------
 net/ipv6/route.c      | 13 +++++--------
 3 files changed, 8 insertions(+), 15 deletions(-)
include/net/dst_ops.h +1 -1
index 88ff7bb2bb9b..632086b2f644 100644
@@ -16,7 +16,7 @@ struct dst_ops {
 	unsigned short		family;
 	unsigned int		gc_thresh;
 
-	int			(*gc)(struct dst_ops *ops);
+	void			(*gc)(struct dst_ops *ops);
 	struct dst_entry *	(*check)(struct dst_entry *, __u32 cookie);
 	unsigned int		(*default_advmss)(const struct dst_entry *);
 	unsigned int		(*mtu)(const struct dst_entry *);
net/core/dst.c +2 -6
index 6d2dd03dafa8..31c08a3386d3 100644
@@ -82,12 +82,8 @@ void *dst_alloc(struct dst_ops *ops, struct net_device *dev,
 
 	if (ops->gc &&
 	    !(flags & DST_NOCOUNT) &&
-	    dst_entries_get_fast(ops) > ops->gc_thresh) {
-		if (ops->gc(ops)) {
-			pr_notice_ratelimited("Route cache is full: consider increasing sysctl net.ipv6.route.max_size.\n");
-			return NULL;
-		}
-	}
+	    dst_entries_get_fast(ops) > ops->gc_thresh)
+		ops->gc(ops);
 
 	dst = kmem_cache_alloc(ops->kmem_cachep, GFP_ATOMIC);
 	if (!dst)
net/ipv6/route.c +5 -8
index e74e0361fd92..b643dda68d31 100644
@@ -91,7 +91,7 @@ static struct dst_entry *ip6_negative_advice(struct dst_entry *);
 static void		ip6_dst_destroy(struct dst_entry *);
 static void		ip6_dst_ifdown(struct dst_entry *,
 				       struct net_device *dev, int how);
-static int		 ip6_dst_gc(struct dst_ops *ops);
+static void		 ip6_dst_gc(struct dst_ops *ops);
 
 static int		ip6_pkt_discard(struct sk_buff *skb);
 static int		ip6_pkt_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb);
@@ -3284,11 +3284,10 @@ struct dst_entry *icmp6_dst_alloc(struct net_device *dev,
 	return dst;
 }
 
-static int ip6_dst_gc(struct dst_ops *ops)
+static void ip6_dst_gc(struct dst_ops *ops)
 {
 	struct net *net = container_of(ops, struct net, ipv6.ip6_dst_ops);
 	int rt_min_interval = net->ipv6.sysctl.ip6_rt_gc_min_interval;
-	int rt_max_size = net->ipv6.sysctl.ip6_rt_max_size;
 	int rt_elasticity = net->ipv6.sysctl.ip6_rt_gc_elasticity;
 	int rt_gc_timeout = net->ipv6.sysctl.ip6_rt_gc_timeout;
 	unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc;
@@ -3296,11 +3295,10 @@ static int ip6_dst_gc(struct dst_ops *ops)
 	int entries;
 
 	entries = dst_entries_get_fast(ops);
-	if (entries > rt_max_size)
+	if (entries > ops->gc_thresh)
 		entries = dst_entries_get_slow(ops);
 
-	if (time_after(rt_last_gc + rt_min_interval, jiffies) &&
-	    entries <= rt_max_size)
+	if (time_after(rt_last_gc + rt_min_interval, jiffies))
 		goto out;
 
 	fib6_run_gc(atomic_inc_return(&net->ipv6.ip6_rt_gc_expire), net, true);
@@ -3310,7 +3308,6 @@ static int ip6_dst_gc(struct dst_ops *ops)
 out:
 	val = atomic_read(&net->ipv6.ip6_rt_gc_expire);
 	atomic_set(&net->ipv6.ip6_rt_gc_expire, val - (val >> rt_elasticity));
-	return entries > rt_max_size;
 }
 
 static int ip6_nh_lookup_table(struct net *net, struct fib6_config *cfg,
@@ -6512,7 +6509,7 @@ static int __net_init ip6_route_net_init(struct net *net)
 #endif
 
 	net->ipv6.sysctl.flush_delay = 0;
-	net->ipv6.sysctl.ip6_rt_max_size = 4096;
+	net->ipv6.sysctl.ip6_rt_max_size = INT_MAX;
 	net->ipv6.sysctl.ip6_rt_gc_min_interval = HZ / 2;
 	net->ipv6.sysctl.ip6_rt_gc_timeout = 60*HZ;
 	net->ipv6.sysctl.ip6_rt_gc_interval = 30*HZ;
Signature · 1 line
2.31.1
#2 lore raw

Hi Andrea,

Quoted 43 lines · Andrea Mayer

Hi Jon,

Quoted 37 lines · On Thu, 12 Jan 2023 12:25:32 +1100 Jon Maxwell <[email protected]> wrote:

v2: Correct syntax error in net/ipv6/route.c

In ip6_dst_gc() replace:

if (entries > gc_thresh)

With:

if (entries > ops->gc_thresh)

Sending Ipv6 packets in a loop via a raw socket triggers an issue where a route is cloned by ip6_rt_cache_alloc() for each packet sent. This quickly consumes the Ipv6 max_size threshold which defaults to 4096 resulting in these warnings:

[1] 99.187805] dst_alloc: 7728 callbacks suppressed
[2] Route cache is full: consider increasing sysctl net.ipv6.route.max_size. .
.
[300] Route cache is full: consider increasing sysctl net.ipv6.route.max_size.

When this happens the packet is dropped and sendto() gets a network is unreachable error:

# ./a.out -s

remaining pkt 200557 errno 101
remaining pkt 196462 errno 101
.
.
remaining pkt 126821 errno 101

Implement David Aherns suggestion to remove max_size check seeing that Ipv6 has a GC to manage memory usage. Ipv4 already does not check max_size.

Here are some memory comparisons for Ipv4 vs Ipv6 with the patch:

Quoted 9 lines
Quoted 6 lines

Test by running 5 instances of a program that sends UDP packets to a raw socket 5000000 times. Compare Ipv4 and Ipv6 performance with a similar program.

Ipv4:

is it supposed to be Ipv6, right?

No, That is for Ipv4, DavidA asked me to include a comparison between Ipv4 and Ipv6. So that is taken for a test running 5 processes that sends 5000000 packets on an ipv4 raw socket. If you scroll down, Ipv6 memory statistics are under "Ipv6 with patch:".

Regards

Jon

Quoted 172 lines

Ciao,
Andrea

Quoted 169 lines

Before test:

# grep -e Slab -e Free /proc/meminfo
MemFree:        29427108 kB
Slab:             237612 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache       1912   2528    256   32    2 : tunables    0    0    0
xfrm_dst_cache         0      0    320   25    2 : tunables    0    0    0
ip_dst_cache        2881   3990    192   42    2 : tunables    0    0    0

During test:

# grep -e Slab -e Free /proc/meminfo
MemFree:        29417608 kB
Slab:             247712 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache       1912   2528    256   32    2 : tunables    0    0    0
xfrm_dst_cache         0      0    320   25    2 : tunables    0    0    0
ip_dst_cache       44394  44394    192   42    2 : tunables    0    0    0

After test:

# grep -e Slab -e Free /proc/meminfo
MemFree:        29422308 kB
Slab:             238104 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache       1912   2528    256   32    2 : tunables    0    0    0
xfrm_dst_cache         0      0    320   25    2 : tunables    0    0    0
ip_dst_cache        3048   4116    192   42    2 : tunables    0    0    0

Ipv6 with patch:

Errno 101 errors are not observed anymore with the patch.

Before test:

# grep -e Slab -e Free /proc/meminfo
MemFree:        29422308 kB
Slab:             238104 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache       1912   2528    256   32    2 : tunables    0    0    0
xfrm_dst_cache         0      0    320   25    2 : tunables    0    0    0
ip_dst_cache        3048   4116    192   42    2 : tunables    0    0    0

During Test:

# grep -e Slab -e Free /proc/meminfo
MemFree:        29431516 kB
Slab:             240940 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache      11980  12064    256   32    2 : tunables    0    0    0
xfrm_dst_cache         0      0    320   25    2 : tunables    0    0    0
ip_dst_cache        3048   4116    192   42    2 : tunables    0    0    0

After Test:

# grep -e Slab -e Free /proc/meminfo
MemFree:        29441816 kB
Slab:             238132 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache       1902   2432    256   32    2 : tunables    0    0    0
xfrm_dst_cache         0      0    320   25    2 : tunables    0    0    0
ip_dst_cache        3048   4116    192   42    2 : tunables    0    0    0
---
 include/net/dst_ops.h |  2 +-
 net/core/dst.c        |  8 ++------
 net/ipv6/route.c      | 13 +++++--------
 3 files changed, 8 insertions(+), 15 deletions(-)
include/net/dst_ops.h +1 -1
index 88ff7bb2bb9b..632086b2f644 100644
@@ -16,7 +16,7 @@ struct dst_ops {
      unsigned short          family;
      unsigned int            gc_thresh;
 
-     int                     (*gc)(struct dst_ops *ops);
+     void                    (*gc)(struct dst_ops *ops);
      struct dst_entry *      (*check)(struct dst_entry *, __u32 cookie);
      unsigned int            (*default_advmss)(const struct dst_entry *);
      unsigned int            (*mtu)(const struct dst_entry *);
net/core/dst.c +2 -6
index 6d2dd03dafa8..31c08a3386d3 100644
@@ -82,12 +82,8 @@ void *dst_alloc(struct dst_ops *ops, struct net_device *dev,
 
      if (ops->gc &&
          !(flags & DST_NOCOUNT) &&
-         dst_entries_get_fast(ops) > ops->gc_thresh) {
-             if (ops->gc(ops)) {
-                     pr_notice_ratelimited("Route cache is full: consider increasing sysctl net.ipv6.route.max_size.\n");
-                     return NULL;
-             }
-     }
+         dst_entries_get_fast(ops) > ops->gc_thresh)
+             ops->gc(ops);
 
      dst = kmem_cache_alloc(ops->kmem_cachep, GFP_ATOMIC);
      if (!dst)
net/ipv6/route.c +5 -9
index e74e0361fd92..b643dda68d31 100644
@@ -91,7 +91,7 @@ static struct dst_entry *ip6_negative_advice(struct dst_entry *);
 static void          ip6_dst_destroy(struct dst_entry *);
 static void          ip6_dst_ifdown(struct dst_entry *,
                                     struct net_device *dev, int how);
-static int            ip6_dst_gc(struct dst_ops *ops);
+static void           ip6_dst_gc(struct dst_ops *ops);
 
 static int           ip6_pkt_discard(struct sk_buff *skb);
 static int           ip6_pkt_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb);
@@ -3284,11 +3284,10 @@ struct dst_entry *icmp6_dst_alloc(struct net_device *dev,
      return dst;
 }
 
-static int ip6_dst_gc(struct dst_ops *ops)
+static void ip6_dst_gc(struct dst_ops *ops)
 {
      struct net *net = container_of(ops, struct net, ipv6.ip6_dst_ops);
      int rt_min_interval = net->ipv6.sysctl.ip6_rt_gc_min_interval;
-     int rt_max_size = net->ipv6.sysctl.ip6_rt_max_size;
      int rt_elasticity = net->ipv6.sysctl.ip6_rt_gc_elasticity;
      int rt_gc_timeout = net->ipv6.sysctl.ip6_rt_gc_timeout;
      unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc;
@@ -3296,11 +3295,10 @@ static int ip6_dst_gc(struct dst_ops *ops)
      int entries;
 
      entries = dst_entries_get_fast(ops);
-     if (entries > rt_max_size)
+     if (entries > ops->gc_thresh)
              entries = dst_entries_get_slow(ops);
 
-     if (time_after(rt_last_gc + rt_min_interval, jiffies) &&
-         entries <= rt_max_size)
+     if (time_after(rt_last_gc + rt_min_interval, jiffies))
              goto out;
 
      fib6_run_gc(atomic_inc_return(&net->ipv6.ip6_rt_gc_expire), net, true);
@@ -3310,7 +3308,6 @@ static int ip6_dst_gc(struct dst_ops *ops)
 out:
      val = atomic_read(&net->ipv6.ip6_rt_gc_expire);
      atomic_set(&net->ipv6.ip6_rt_gc_expire, val - (val >> rt_elasticity));
-     return entries > rt_max_size;
 }
 
 static int ip6_nh_lookup_table(struct net *net, struct fib6_config *cfg,
@@ -6512,7 +6509,7 @@ static int __net_init ip6_route_net_init(struct net *net)
 #endif
 
      net->ipv6.sysctl.flush_delay = 0;
-     net->ipv6.sysctl.ip6_rt_max_size = 4096;
+     net->ipv6.sysctl.ip6_rt_max_size = INT_MAX;
      net->ipv6.sysctl.ip6_rt_gc_min_interval = HZ / 2;
      net->ipv6.sysctl.ip6_rt_gc_timeout = 60*HZ;
      net->ipv6.sysctl.ip6_rt_gc_interval = 30*HZ;
--

2.31.1

#3 lore raw
Quoted 53 lines · On Fri, 13 Jan 2023 08:48:24 +1100 Jonathan Maxwell <[email protected]>…

Hi Andrea,

Quoted 50 lines · Andrea Mayer

Hi Jon,

Quoted 43 lines · On Thu, 12 Jan 2023 12:25:32 +1100 Jon Maxwell <[email protected]> wrote:

v2: Correct syntax error in net/ipv6/route.c

In ip6_dst_gc() replace:

if (entries > gc_thresh)

With:

if (entries > ops->gc_thresh)

Sending Ipv6 packets in a loop via a raw socket triggers an issue where a route is cloned by ip6_rt_cache_alloc() for each packet sent. This quickly consumes the Ipv6 max_size threshold which defaults to 4096 resulting in these warnings:

[1] 99.187805] dst_alloc: 7728 callbacks suppressed
[2] Route cache is full: consider increasing sysctl net.ipv6.route.max_size. .
.
[300] Route cache is full: consider increasing sysctl net.ipv6.route.max_size.

When this happens the packet is dropped and sendto() gets a network is unreachable error:

# ./a.out -s

remaining pkt 200557 errno 101
remaining pkt 196462 errno 101
.
.
remaining pkt 126821 errno 101

Implement David Aherns suggestion to remove max_size check seeing that Ipv6 has a GC to manage memory usage. Ipv4 already does not check max_size.

Here are some memory comparisons for Ipv4 vs Ipv6 with the patch:

Test by running 5 instances of a program that sends UDP packets to a raw socket 5000000 times. Compare Ipv4 and Ipv6 performance with a similar program.

Ipv4:

Quoted 8 lines
Quoted 2 lines

is it supposed to be Ipv6, right?

No, That is for Ipv4, DavidA asked me to include a comparison between Ipv4 and Ipv6. So that is taken for a test running 5 processes that sends 5000000 packets on an ipv4 raw socket. If you scroll down, Ipv6 memory statistics are under "Ipv6 with patch:".

I asked because these tests also report ip6_dst_cache counters that I would not have expected to find in IPv4 cases (and viceversa).

Thanks for the clarification.

Ciao,
Andrea

Quoted 176 lines

Regards

Jon

Quoted 172 lines

Ciao,
Andrea

Quoted 169 lines

Before test:

# grep -e Slab -e Free /proc/meminfo
MemFree:        29427108 kB
Slab:             237612 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache       1912   2528    256   32    2 : tunables    0    0    0
xfrm_dst_cache         0      0    320   25    2 : tunables    0    0    0
ip_dst_cache        2881   3990    192   42    2 : tunables    0    0    0

During test:

# grep -e Slab -e Free /proc/meminfo
MemFree:        29417608 kB
Slab:             247712 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache       1912   2528    256   32    2 : tunables    0    0    0
xfrm_dst_cache         0      0    320   25    2 : tunables    0    0    0
ip_dst_cache       44394  44394    192   42    2 : tunables    0    0    0

After test:

# grep -e Slab -e Free /proc/meminfo
MemFree:        29422308 kB
Slab:             238104 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache       1912   2528    256   32    2 : tunables    0    0    0
xfrm_dst_cache         0      0    320   25    2 : tunables    0    0    0
ip_dst_cache        3048   4116    192   42    2 : tunables    0    0    0

Ipv6 with patch:

Errno 101 errors are not observed anymore with the patch.

Before test:

# grep -e Slab -e Free /proc/meminfo
MemFree:        29422308 kB
Slab:             238104 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache       1912   2528    256   32    2 : tunables    0    0    0
xfrm_dst_cache         0      0    320   25    2 : tunables    0    0    0
ip_dst_cache        3048   4116    192   42    2 : tunables    0    0    0

During Test:

# grep -e Slab -e Free /proc/meminfo
MemFree:        29431516 kB
Slab:             240940 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache      11980  12064    256   32    2 : tunables    0    0    0
xfrm_dst_cache         0      0    320   25    2 : tunables    0    0    0
ip_dst_cache        3048   4116    192   42    2 : tunables    0    0    0

After Test:

# grep -e Slab -e Free /proc/meminfo
MemFree:        29441816 kB
Slab:             238132 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache       1902   2432    256   32    2 : tunables    0    0    0
xfrm_dst_cache         0      0    320   25    2 : tunables    0    0    0
ip_dst_cache        3048   4116    192   42    2 : tunables    0    0    0
---
 include/net/dst_ops.h |  2 +-
 net/core/dst.c        |  8 ++------
 net/ipv6/route.c      | 13 +++++--------
 3 files changed, 8 insertions(+), 15 deletions(-)
include/net/dst_ops.h +1 -1
index 88ff7bb2bb9b..632086b2f644 100644
@@ -16,7 +16,7 @@ struct dst_ops {
      unsigned short          family;
      unsigned int            gc_thresh;
 
-     int                     (*gc)(struct dst_ops *ops);
+     void                    (*gc)(struct dst_ops *ops);
      struct dst_entry *      (*check)(struct dst_entry *, __u32 cookie);
      unsigned int            (*default_advmss)(const struct dst_entry *);
      unsigned int            (*mtu)(const struct dst_entry *);
net/core/dst.c +2 -6
index 6d2dd03dafa8..31c08a3386d3 100644
@@ -82,12 +82,8 @@ void *dst_alloc(struct dst_ops *ops, struct net_device *dev,
 
      if (ops->gc &&
          !(flags & DST_NOCOUNT) &&
-         dst_entries_get_fast(ops) > ops->gc_thresh) {
-             if (ops->gc(ops)) {
-                     pr_notice_ratelimited("Route cache is full: consider increasing sysctl net.ipv6.route.max_size.\n");
-                     return NULL;
-             }
-     }
+         dst_entries_get_fast(ops) > ops->gc_thresh)
+             ops->gc(ops);
 
      dst = kmem_cache_alloc(ops->kmem_cachep, GFP_ATOMIC);
      if (!dst)
net/ipv6/route.c +5 -9
index e74e0361fd92..b643dda68d31 100644
@@ -91,7 +91,7 @@ static struct dst_entry *ip6_negative_advice(struct dst_entry *);
 static void          ip6_dst_destroy(struct dst_entry *);
 static void          ip6_dst_ifdown(struct dst_entry *,
                                     struct net_device *dev, int how);
-static int            ip6_dst_gc(struct dst_ops *ops);
+static void           ip6_dst_gc(struct dst_ops *ops);
 
 static int           ip6_pkt_discard(struct sk_buff *skb);
 static int           ip6_pkt_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb);
@@ -3284,11 +3284,10 @@ struct dst_entry *icmp6_dst_alloc(struct net_device *dev,
      return dst;
 }
 
-static int ip6_dst_gc(struct dst_ops *ops)
+static void ip6_dst_gc(struct dst_ops *ops)
 {
      struct net *net = container_of(ops, struct net, ipv6.ip6_dst_ops);
      int rt_min_interval = net->ipv6.sysctl.ip6_rt_gc_min_interval;
-     int rt_max_size = net->ipv6.sysctl.ip6_rt_max_size;
      int rt_elasticity = net->ipv6.sysctl.ip6_rt_gc_elasticity;
      int rt_gc_timeout = net->ipv6.sysctl.ip6_rt_gc_timeout;
      unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc;
@@ -3296,11 +3295,10 @@ static int ip6_dst_gc(struct dst_ops *ops)
      int entries;
 
      entries = dst_entries_get_fast(ops);
-     if (entries > rt_max_size)
+     if (entries > ops->gc_thresh)
              entries = dst_entries_get_slow(ops);
 
-     if (time_after(rt_last_gc + rt_min_interval, jiffies) &&
-         entries <= rt_max_size)
+     if (time_after(rt_last_gc + rt_min_interval, jiffies))
              goto out;
 
      fib6_run_gc(atomic_inc_return(&net->ipv6.ip6_rt_gc_expire), net, true);
@@ -3310,7 +3308,6 @@ static int ip6_dst_gc(struct dst_ops *ops)
 out:
      val = atomic_read(&net->ipv6.ip6_rt_gc_expire);
      atomic_set(&net->ipv6.ip6_rt_gc_expire, val - (val >> rt_elasticity));
-     return entries > rt_max_size;
 }
 
 static int ip6_nh_lookup_table(struct net *net, struct fib6_config *cfg,
@@ -6512,7 +6509,7 @@ static int __net_init ip6_route_net_init(struct net *net)
 #endif
 
      net->ipv6.sysctl.flush_delay = 0;
-     net->ipv6.sysctl.ip6_rt_max_size = 4096;
+     net->ipv6.sysctl.ip6_rt_max_size = INT_MAX;
      net->ipv6.sysctl.ip6_rt_gc_min_interval = HZ / 2;
      net->ipv6.sysctl.ip6_rt_gc_timeout = 60*HZ;
      net->ipv6.sysctl.ip6_rt_gc_interval = 30*HZ;
--

2.31.1

Signature · 1 line
Andrea Mayer <[email protected]>
maintainer Reviewed-by on v2 1/1
#4 lore raw
In reply to Jon Maxwell
Quoted 113 lines · Jon Maxwell

v2: Correct syntax error in net/ipv6/route.c

In ip6_dst_gc() replace:

if (entries > gc_thresh)

With:

if (entries > ops->gc_thresh)

Sending Ipv6 packets in a loop via a raw socket triggers an issue where a route is cloned by ip6_rt_cache_alloc() for each packet sent. This quickly consumes the Ipv6 max_size threshold which defaults to 4096 resulting in these warnings:

[1] 99.187805] dst_alloc: 7728 callbacks suppressed
[2] Route cache is full: consider increasing sysctl net.ipv6.route.max_size. .
.
[300] Route cache is full: consider increasing sysctl net.ipv6.route.max_size.

When this happens the packet is dropped and sendto() gets a network is unreachable error:

# ./a.out -s 

remaining pkt 200557 errno 101
remaining pkt 196462 errno 101
.
.
remaining pkt 126821 errno 101

Implement David Aherns suggestion to remove max_size check seeing that Ipv6 has a GC to manage memory usage. Ipv4 already does not check max_size.

Here are some memory comparisons for Ipv4 vs Ipv6 with the patch:

Test by running 5 instances of a program that sends UDP packets to a raw socket 5000000 times. Compare Ipv4 and Ipv6 performance with a similar program.

Ipv4:

Before test:

# grep -e Slab -e Free /proc/meminfo
MemFree:        29427108 kB
Slab:             237612 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache       1912   2528    256   32    2 : tunables    0    0    0 
xfrm_dst_cache         0      0    320   25    2 : tunables    0    0    0 
ip_dst_cache        2881   3990    192   42    2 : tunables    0    0    0 

During test:

# grep -e Slab -e Free /proc/meminfo
MemFree:        29417608 kB
Slab:             247712 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache       1912   2528    256   32    2 : tunables    0    0    0 
xfrm_dst_cache         0      0    320   25    2 : tunables    0    0    0 
ip_dst_cache       44394  44394    192   42    2 : tunables    0    0    0 

After test:

# grep -e Slab -e Free /proc/meminfo
MemFree:        29422308 kB
Slab:             238104 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache       1912   2528    256   32    2 : tunables    0    0    0 
xfrm_dst_cache         0      0    320   25    2 : tunables    0    0    0 
ip_dst_cache        3048   4116    192   42    2 : tunables    0    0    0 

Ipv6 with patch:

Errno 101 errors are not observed anymore with the patch.

Before test:

# grep -e Slab -e Free /proc/meminfo
MemFree:        29422308 kB
Slab:             238104 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache       1912   2528    256   32    2 : tunables    0    0    0 
xfrm_dst_cache         0      0    320   25    2 : tunables    0    0    0 
ip_dst_cache        3048   4116    192   42    2 : tunables    0    0    0 

During Test:

# grep -e Slab -e Free /proc/meminfo
MemFree:        29431516 kB
Slab:             240940 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache      11980  12064    256   32    2 : tunables    0    0    0
xfrm_dst_cache         0      0    320   25    2 : tunables    0    0    0
ip_dst_cache        3048   4116    192   42    2 : tunables    0    0    0

After Test:

# grep -e Slab -e Free /proc/meminfo
MemFree:        29441816 kB
Slab:             238132 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache       1902   2432    256   32    2 : tunables    0    0    0
xfrm_dst_cache         0      0    320   25    2 : tunables    0    0    0
ip_dst_cache        3048   4116    192   42    2 : tunables    0    0    0
Quoted 8 lines
---
 include/net/dst_ops.h |  2 +-
 net/core/dst.c        |  8 ++------
 net/ipv6/route.c      | 13 +++++--------
 3 files changed, 8 insertions(+), 15 deletions(-)

Thanks for the data in the commit message.

#5 lore raw
In reply to Jon Maxwell
This patch was applied to netdev/net-next.git (master) by Jakub Kicinski <[email protected]>:

Hello:

This patch was applied to netdev/net-next.git (master) by Jakub Kicinski <[email protected]>:

Quoted 9 lines · On Thu, 12 Jan 2023 12:25:32 +1100 you wrote:

v2: Correct syntax error in net/ipv6/route.c

In ip6_dst_gc() replace:

if (entries > gc_thresh)

With:

[...]

Here is the summary with links:
- [net-next,v2] ipv6: remove max_size check inline with ipv4

    https://git.kernel.org/netdev/net-next/c/af6d10345ca7

You are awesome, thank you!

Signature · 2 lines
Deet-doot-dot, I am a bot.
https://korg.docs.kernel.org/patchwork/pwbot.html