ipv6: remove max_size check inline with ipv4
Subsystems (3)
PATCH v2 1/1 ipv6: remove max_size check inline with ipv4
v2: Correct syntax error in net/ipv6/route.c
In ip6_dst_gc() replace:
if (entries > gc_thresh)
With:
if (entries > ops->gc_thresh)
Sending Ipv6 packets in a loop via a raw socket triggers an issue where a route is cloned by ip6_rt_cache_alloc() for each packet sent. This quickly consumes the Ipv6 max_size threshold which defaults to 4096 resulting in these warnings:
[1] 99.187805] dst_alloc: 7728 callbacks suppressed
[2] Route cache is full: consider increasing sysctl net.ipv6.route.max_size. .
.
[300] Route cache is full: consider increasing sysctl net.ipv6.route.max_size.
When this happens the packet is dropped and sendto() gets a network is unreachable error:
# ./a.out -s
remaining pkt 200557 errno 101
remaining pkt 196462 errno 101
.
.
remaining pkt 126821 errno 101
Implement David Aherns suggestion to remove max_size check seeing that Ipv6 has a GC to manage memory usage. Ipv4 already does not check max_size.
Here are some memory comparisons for Ipv4 vs Ipv6 with the patch:
Test by running 5 instances of a program that sends UDP packets to a raw socket 5000000 times. Compare Ipv4 and Ipv6 performance with a similar program.
Ipv4:
Before test:
# grep -e Slab -e Free /proc/meminfo MemFree: 29427108 kB Slab: 237612 kB
# grep dst_cache /proc/slabinfo ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0 xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0 ip_dst_cache 2881 3990 192 42 2 : tunables 0 0 0
During test:
# grep -e Slab -e Free /proc/meminfo MemFree: 29417608 kB Slab: 247712 kB
# grep dst_cache /proc/slabinfo ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0 xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0 ip_dst_cache 44394 44394 192 42 2 : tunables 0 0 0
After test:
# grep -e Slab -e Free /proc/meminfo MemFree: 29422308 kB Slab: 238104 kB
# grep dst_cache /proc/slabinfo ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0 xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0 ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0
Ipv6 with patch:
Errno 101 errors are not observed anymore with the patch.
Before test:
# grep -e Slab -e Free /proc/meminfo MemFree: 29422308 kB Slab: 238104 kB
# grep dst_cache /proc/slabinfo ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0 xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0 ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0
During Test:
# grep -e Slab -e Free /proc/meminfo MemFree: 29431516 kB Slab: 240940 kB
# grep dst_cache /proc/slabinfo ip6_dst_cache 11980 12064 256 32 2 : tunables 0 0 0 xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0 ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0
After Test:
# grep -e Slab -e Free /proc/meminfo MemFree: 29441816 kB Slab: 238132 kB
# grep dst_cache /proc/slabinfo ip6_dst_cache 1902 2432 256 32 2 : tunables 0 0 0 xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0 ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0
--- include/net/dst_ops.h | 2 +- net/core/dst.c | 8 ++------ net/ipv6/route.c | 13 +++++-------- 3 files changed, 8 insertions(+), 15 deletions(-)
include/net/dst_ops.h +1 -1
@@ -16,7 +16,7 @@ struct dst_ops {unsigned short family;unsigned int gc_thresh; -int (*gc)(struct dst_ops *ops); +void (*gc)(struct dst_ops *ops);struct dst_entry * (*check)(struct dst_entry *,__u32 cookie);unsigned int (*default_advmss)(const struct dst_entry *);unsigned int (*mtu)(const struct dst_entry *);
net/core/dst.c +2 -6
@@ -82,12 +82,8 @@ void *dst_alloc(struct dst_ops *ops, struct net_device *dev,if (ops->gc && !(flags &DST_NOCOUNT ) && -dst_entries_get_fast (ops) > ops->gc_thresh) { -if (ops->gc (ops)) { -pr_notice_ratelimited ("Route cache is full: consider increasing sysctl net.ipv6.route.max_size.\n" ); -return NULL ; - } - } +dst_entries_get_fast (ops) > ops->gc_thresh) + ops->gc (ops); dst =kmem_cache_alloc (ops->kmem_cachep,GFP_ATOMIC );if (!dst)
net/ipv6/route.c +5 -8
@@ -91,7 +91,7 @@ static struct dst_entry *ip6_negative_advice(struct dst_entry *);static void ip6_dst_destroy (struct dst_entry *);static void ip6_dst_ifdown (struct dst_entry *,struct net_device *dev,int how); -static int ip6_dst_gc (struct dst_ops *ops); +static void ip6_dst_gc (struct dst_ops *ops);static int ip6_pkt_discard (struct sk_buff *skb);static int ip6_pkt_discard_out (struct net *net,struct sock *sk,struct sk_buff *skb);
@@ -3284,11 +3284,10 @@ struct dst_entry *icmp6_dst_alloc(struct net_device *dev,return dst; } -static int ip6_dst_gc (struct dst_ops *ops) +static void ip6_dst_gc (struct dst_ops *ops) { struct net *net = container_of(ops, struct net, ipv6.ip6_dst_ops);int rt_min_interval = net->ipv6.sysctl.ip6_rt_gc_min_interval; -int rt_max_size = net->ipv6.sysctl.ip6_rt_max_size;int rt_elasticity = net->ipv6.sysctl.ip6_rt_gc_elasticity;int rt_gc_timeout = net->ipv6.sysctl.ip6_rt_gc_timeout;unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc;
@@ -3296,11 +3295,10 @@ static int ip6_dst_gc(struct dst_ops *ops)int entries; entries =dst_entries_get_fast (ops); -if (entries > rt_max_size) +if (entries > ops->gc_thresh) entries =dst_entries_get_slow (ops); -if (time_after (rt_last_gc + rt_min_interval, jiffies) && - entries <= rt_max_size) +if (time_after (rt_last_gc + rt_min_interval, jiffies)) goto out;fib6_run_gc (atomic_inc_return (&net->ipv6.ip6_rt_gc_expire), net, true);
@@ -3310,7 +3308,6 @@ static int ip6_dst_gc(struct dst_ops *ops) out: val =atomic_read (&net->ipv6.ip6_rt_gc_expire);atomic_set (&net->ipv6.ip6_rt_gc_expire, val - (val >> rt_elasticity)); -return entries > rt_max_size; } static int ip6_nh_lookup_table(struct net *net, struct fib6_config *cfg,
@@ -6512,7 +6509,7 @@ static int __net_init ip6_route_net_init(struct net *net)#endif net->ipv6.sysctl.flush_delay =0 ; - net->ipv6.sysctl.ip6_rt_max_size =4096 ; + net->ipv6.sysctl.ip6_rt_max_size =INT_MAX ; net->ipv6.sysctl.ip6_rt_gc_min_interval =HZ /2 ; net->ipv6.sysctl.ip6_rt_gc_timeout =60 *HZ ; net->ipv6.sysctl.ip6_rt_gc_interval =30 *HZ ;
Signature · 1 line
2.31.1
Hi Jon,
Quoted 35 lines · On Thu, 12 Jan 2023 12:25:32 +1100 Jon Maxwell <[email protected]> wrote:
v2: Correct syntax error in
net/ipv6/route.cIn
ip6_dst_gc()replace:if (entries >
gc_thresh)With:
if (entries >
ops->gc_thresh)Sending Ipv6 packets in a loop via a raw socket triggers an issue where a route is cloned by
ip6_rt_cache_alloc()for each packet sent. This quickly consumes the Ipv6max_sizethreshold which defaults to 4096 resulting in these warnings:[1] 99.187805]
dst_alloc: 7728 callbacks suppressed
[2] Route cache is full: consider increasing sysctl net.ipv6.route.max_size. .
.
[300] Route cache is full: consider increasing sysctl net.ipv6.route.max_size.When this happens the packet is dropped and
sendto()gets a network is unreachable error:# ./a.out -sremaining pkt 200557 errno 101
remaining pkt 196462 errno 101
.
.
remaining pkt 126821 errno 101Implement David Aherns suggestion to remove
max_sizecheck seeing that Ipv6 has a GC to manage memory usage. Ipv4 already does not checkmax_size.
Quoted 8 lines
Here are some memory comparisons for Ipv4 vs Ipv6 with the patch:
Test by running 5 instances of a program that sends UDP packets to a raw socket 5000000 times. Compare Ipv4 and Ipv6 performance with a similar program.
Ipv4:
is it supposed to be Ipv6, right?
Ciao,
Andrea
Quoted 169 lines
Before test:
# grep -e Slab -e Free /proc/meminfo MemFree: 29427108 kB Slab: 237612 kB# grep dst_cache /proc/slabinfo ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0 xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0 ip_dst_cache 2881 3990 192 42 2 : tunables 0 0 0During test:
# grep -e Slab -e Free /proc/meminfo MemFree: 29417608 kB Slab: 247712 kB# grep dst_cache /proc/slabinfo ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0 xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0 ip_dst_cache 44394 44394 192 42 2 : tunables 0 0 0After test:
# grep -e Slab -e Free /proc/meminfo MemFree: 29422308 kB Slab: 238104 kB# grep dst_cache /proc/slabinfo ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0 xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0 ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0Ipv6 with patch:
Errno 101 errors are not observed anymore with the patch.
Before test:
# grep -e Slab -e Free /proc/meminfo MemFree: 29422308 kB Slab: 238104 kB# grep dst_cache /proc/slabinfo ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0 xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0 ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0During Test:
# grep -e Slab -e Free /proc/meminfo MemFree: 29431516 kB Slab: 240940 kB# grep dst_cache /proc/slabinfo ip6_dst_cache 11980 12064 256 32 2 : tunables 0 0 0 xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0 ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0After Test:
# grep -e Slab -e Free /proc/meminfo MemFree: 29441816 kB Slab: 238132 kB# grep dst_cache /proc/slabinfo ip6_dst_cache 1902 2432 256 32 2 : tunables 0 0 0 xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0 ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0Tested-by Andrea Mayer <[email protected]>Signed-off-by Jon Maxwell <[email protected]>--- include/net/dst_ops.h | 2 +- net/core/dst.c | 8 ++------ net/ipv6/route.c | 13 +++++-------- 3 files changed, 8 insertions(+), 15 deletions(-)include/net/dst_ops.h +1
-1@@ -16,7 +16,7 @@ struct dst_ops {unsigned short family;unsigned int gc_thresh; -int (*gc)(struct dst_ops *ops); +void (*gc)(struct dst_ops *ops);struct dst_entry * (*check)(struct dst_entry *,__u32 cookie);unsigned int (*default_advmss)(const struct dst_entry *);unsigned int (*mtu)(const struct dst_entry *);net/core/dst.c +2
-6@@ -82,12 +82,8 @@ void *dst_alloc(struct dst_ops *ops, struct net_device *dev,if (ops->gc && !(flags &DST_NOCOUNT ) && -dst_entries_get_fast (ops) > ops->gc_thresh) { -if (ops->gc (ops)) { -pr_notice_ratelimited ("Route cache is full: consider increasing sysctl net.ipv6.route.max_size.\n" ); -return NULL ; - } - } +dst_entries_get_fast (ops) > ops->gc_thresh) + ops->gc (ops); dst =kmem_cache_alloc (ops->kmem_cachep,GFP_ATOMIC );if (!dst)net/ipv6/route.c +5
-8@@ -91,7 +91,7 @@ static struct dst_entry *ip6_negative_advice(struct dst_entry *);static void ip6_dst_destroy (struct dst_entry *);static void ip6_dst_ifdown (struct dst_entry *,struct net_device *dev,int how); -static int ip6_dst_gc (struct dst_ops *ops); +static void ip6_dst_gc (struct dst_ops *ops);static int ip6_pkt_discard (struct sk_buff *skb);static int ip6_pkt_discard_out (struct net *net,struct sock *sk,struct sk_buff *skb);@@ -3284,11 +3284,10 @@ struct dst_entry *icmp6_dst_alloc(struct net_device *dev,return dst; } -static int ip6_dst_gc (struct dst_ops *ops) +static void ip6_dst_gc (struct dst_ops *ops) { struct net *net = container_of(ops, struct net, ipv6.ip6_dst_ops);int rt_min_interval = net->ipv6.sysctl.ip6_rt_gc_min_interval; -int rt_max_size = net->ipv6.sysctl.ip6_rt_max_size;int rt_elasticity = net->ipv6.sysctl.ip6_rt_gc_elasticity;int rt_gc_timeout = net->ipv6.sysctl.ip6_rt_gc_timeout;unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc;@@ -3296,11 +3295,10 @@ static int ip6_dst_gc(struct dst_ops *ops)int entries; entries =dst_entries_get_fast (ops); -if (entries > rt_max_size) +if (entries > ops->gc_thresh) entries =dst_entries_get_slow (ops); -if (time_after (rt_last_gc + rt_min_interval, jiffies) && - entries <= rt_max_size) +if (time_after (rt_last_gc + rt_min_interval, jiffies)) goto out;fib6_run_gc (atomic_inc_return (&net->ipv6.ip6_rt_gc_expire), net, true);@@ -3310,7 +3308,6 @@ static int ip6_dst_gc(struct dst_ops *ops) out: val =atomic_read (&net->ipv6.ip6_rt_gc_expire);atomic_set (&net->ipv6.ip6_rt_gc_expire, val - (val >> rt_elasticity)); -return entries > rt_max_size; } static int ip6_nh_lookup_table(struct net *net, struct fib6_config *cfg,@@ -6512,7 +6509,7 @@ static int __net_init ip6_route_net_init(struct net *net)#endif net->ipv6.sysctl.flush_delay =0 ; - net->ipv6.sysctl.ip6_rt_max_size =4096 ; + net->ipv6.sysctl.ip6_rt_max_size =INT_MAX ; net->ipv6.sysctl.ip6_rt_gc_min_interval =HZ /2 ; net->ipv6.sysctl.ip6_rt_gc_timeout =60 *HZ ; net->ipv6.sysctl.ip6_rt_gc_interval =30 *HZ ;Signature · 1 line
2.31.1
Hi Andrea,
Quoted 43 lines · Andrea Mayer
Hi Jon,
Quoted 37 lines · On Thu, 12 Jan 2023 12:25:32 +1100 Jon Maxwell <[email protected]> wrote:
v2: Correct syntax error in
net/ipv6/route.cIn
ip6_dst_gc()replace:if (entries >
gc_thresh)With:
if (entries >
ops->gc_thresh)Sending Ipv6 packets in a loop via a raw socket triggers an issue where a route is cloned by
ip6_rt_cache_alloc()for each packet sent. This quickly consumes the Ipv6max_sizethreshold which defaults to 4096 resulting in these warnings:[1] 99.187805]
dst_alloc: 7728 callbacks suppressed
[2] Route cache is full: consider increasing sysctl net.ipv6.route.max_size. .
.
[300] Route cache is full: consider increasing sysctl net.ipv6.route.max_size.When this happens the packet is dropped and
sendto()gets a network is unreachable error:# ./a.out -sremaining pkt 200557 errno 101
remaining pkt 196462 errno 101
.
.
remaining pkt 126821 errno 101Implement David Aherns suggestion to remove
max_sizecheck seeing that Ipv6 has a GC to manage memory usage. Ipv4 already does not checkmax_size.Here are some memory comparisons for Ipv4 vs Ipv6 with the patch:
Quoted 9 lines
Quoted 6 lines
Test by running 5 instances of a program that sends UDP packets to a raw socket 5000000 times. Compare Ipv4 and Ipv6 performance with a similar program.
Ipv4:
is it supposed to be Ipv6, right?
No, That is for Ipv4, DavidA asked me to include a comparison between Ipv4 and Ipv6. So that is taken for a test running 5 processes that sends 5000000 packets on an ipv4 raw socket. If you scroll down, Ipv6 memory statistics are under "Ipv6 with patch:".
Regards
Jon
Quoted 172 lines
Ciao,
AndreaQuoted 169 lines
Before test:
# grep -e Slab -e Free /proc/meminfo MemFree: 29427108 kB Slab: 237612 kB# grep dst_cache /proc/slabinfo ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0 xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0 ip_dst_cache 2881 3990 192 42 2 : tunables 0 0 0During test:
# grep -e Slab -e Free /proc/meminfo MemFree: 29417608 kB Slab: 247712 kB# grep dst_cache /proc/slabinfo ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0 xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0 ip_dst_cache 44394 44394 192 42 2 : tunables 0 0 0After test:
# grep -e Slab -e Free /proc/meminfo MemFree: 29422308 kB Slab: 238104 kB# grep dst_cache /proc/slabinfo ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0 xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0 ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0Ipv6 with patch:
Errno 101 errors are not observed anymore with the patch.
Before test:
# grep -e Slab -e Free /proc/meminfo MemFree: 29422308 kB Slab: 238104 kB# grep dst_cache /proc/slabinfo ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0 xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0 ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0During Test:
# grep -e Slab -e Free /proc/meminfo MemFree: 29431516 kB Slab: 240940 kB# grep dst_cache /proc/slabinfo ip6_dst_cache 11980 12064 256 32 2 : tunables 0 0 0 xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0 ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0After Test:
# grep -e Slab -e Free /proc/meminfo MemFree: 29441816 kB Slab: 238132 kB# grep dst_cache /proc/slabinfo ip6_dst_cache 1902 2432 256 32 2 : tunables 0 0 0 xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0 ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0Tested-by Andrea Mayer <[email protected]>Signed-off-by Jon Maxwell <[email protected]>--- include/net/dst_ops.h | 2 +- net/core/dst.c | 8 ++------ net/ipv6/route.c | 13 +++++-------- 3 files changed, 8 insertions(+), 15 deletions(-)include/net/dst_ops.h +1
-1@@ -16,7 +16,7 @@ struct dst_ops {unsigned short family;unsigned int gc_thresh; -int (*gc)(struct dst_ops *ops); +void (*gc)(struct dst_ops *ops);struct dst_entry * (*check)(struct dst_entry *,__u32 cookie);unsigned int (*default_advmss)(const struct dst_entry *);unsigned int (*mtu)(const struct dst_entry *);net/core/dst.c +2
-6@@ -82,12 +82,8 @@ void *dst_alloc(struct dst_ops *ops, struct net_device *dev,if (ops->gc && !(flags &DST_NOCOUNT ) && -dst_entries_get_fast (ops) > ops->gc_thresh) { -if (ops->gc (ops)) { -pr_notice_ratelimited ("Route cache is full: consider increasing sysctl net.ipv6.route.max_size.\n" ); -return NULL ; - } - } +dst_entries_get_fast (ops) > ops->gc_thresh) + ops->gc (ops); dst =kmem_cache_alloc (ops->kmem_cachep,GFP_ATOMIC );if (!dst)net/ipv6/route.c +5
-9@@ -91,7 +91,7 @@ static struct dst_entry *ip6_negative_advice(struct dst_entry *);static void ip6_dst_destroy (struct dst_entry *);static void ip6_dst_ifdown (struct dst_entry *,struct net_device *dev,int how); -static int ip6_dst_gc (struct dst_ops *ops); +static void ip6_dst_gc (struct dst_ops *ops);static int ip6_pkt_discard (struct sk_buff *skb);static int ip6_pkt_discard_out (struct net *net,struct sock *sk,struct sk_buff *skb);@@ -3284,11 +3284,10 @@ struct dst_entry *icmp6_dst_alloc(struct net_device *dev,return dst; } -static int ip6_dst_gc (struct dst_ops *ops) +static void ip6_dst_gc (struct dst_ops *ops) { struct net *net = container_of(ops, struct net, ipv6.ip6_dst_ops);int rt_min_interval = net->ipv6.sysctl.ip6_rt_gc_min_interval; -int rt_max_size = net->ipv6.sysctl.ip6_rt_max_size;int rt_elasticity = net->ipv6.sysctl.ip6_rt_gc_elasticity;int rt_gc_timeout = net->ipv6.sysctl.ip6_rt_gc_timeout;unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc;@@ -3296,11 +3295,10 @@ static int ip6_dst_gc(struct dst_ops *ops)int entries; entries =dst_entries_get_fast (ops); -if (entries > rt_max_size) +if (entries > ops->gc_thresh) entries =dst_entries_get_slow (ops); -if (time_after (rt_last_gc + rt_min_interval, jiffies) && - entries <= rt_max_size) +if (time_after (rt_last_gc + rt_min_interval, jiffies)) goto out;fib6_run_gc (atomic_inc_return (&net->ipv6.ip6_rt_gc_expire), net, true);@@ -3310,7 +3308,6 @@ static int ip6_dst_gc(struct dst_ops *ops) out: val =atomic_read (&net->ipv6.ip6_rt_gc_expire);atomic_set (&net->ipv6.ip6_rt_gc_expire, val - (val >> rt_elasticity)); -return entries > rt_max_size; } static int ip6_nh_lookup_table(struct net *net, struct fib6_config *cfg,@@ -6512,7 +6509,7 @@ static int __net_init ip6_route_net_init(struct net *net)#endif net->ipv6.sysctl.flush_delay =0 ; - net->ipv6.sysctl.ip6_rt_max_size =4096 ; + net->ipv6.sysctl.ip6_rt_max_size =INT_MAX ; net->ipv6.sysctl.ip6_rt_gc_min_interval =HZ /2 ; net->ipv6.sysctl.ip6_rt_gc_timeout =60 *HZ ; net->ipv6.sysctl.ip6_rt_gc_interval =30 *HZ ; --2.31.1
Quoted 53 lines · On Fri, 13 Jan 2023 08:48:24 +1100 Jonathan Maxwell <[email protected]>…
Hi Andrea,
Quoted 50 lines · Andrea Mayer
Hi Jon,
Quoted 43 lines · On Thu, 12 Jan 2023 12:25:32 +1100 Jon Maxwell <[email protected]> wrote:
v2: Correct syntax error in
net/ipv6/route.cIn
ip6_dst_gc()replace:if (entries >
gc_thresh)With:
if (entries >
ops->gc_thresh)Sending Ipv6 packets in a loop via a raw socket triggers an issue where a route is cloned by
ip6_rt_cache_alloc()for each packet sent. This quickly consumes the Ipv6max_sizethreshold which defaults to 4096 resulting in these warnings:[1] 99.187805]
dst_alloc: 7728 callbacks suppressed
[2] Route cache is full: consider increasing sysctl net.ipv6.route.max_size. .
.
[300] Route cache is full: consider increasing sysctl net.ipv6.route.max_size.When this happens the packet is dropped and
sendto()gets a network is unreachable error:# ./a.out -sremaining pkt 200557 errno 101
remaining pkt 196462 errno 101
.
.
remaining pkt 126821 errno 101Implement David Aherns suggestion to remove
max_sizecheck seeing that Ipv6 has a GC to manage memory usage. Ipv4 already does not checkmax_size.Here are some memory comparisons for Ipv4 vs Ipv6 with the patch:
Test by running 5 instances of a program that sends UDP packets to a raw socket 5000000 times. Compare Ipv4 and Ipv6 performance with a similar program.
Ipv4:
Quoted 8 lines
Quoted 2 lines
is it supposed to be Ipv6, right?
No, That is for Ipv4, DavidA asked me to include a comparison between Ipv4 and Ipv6. So that is taken for a test running 5 processes that sends 5000000 packets on an ipv4 raw socket. If you scroll down, Ipv6 memory statistics are under "Ipv6 with patch:".
I asked because these tests also report ip6_dst_cache counters that I would not have expected to find in IPv4 cases (and viceversa).
Thanks for the clarification.
Ciao,
Andrea
Quoted 176 lines
Regards
Jon
Quoted 172 lines
Ciao,
AndreaQuoted 169 lines
Before test:
# grep -e Slab -e Free /proc/meminfo MemFree: 29427108 kB Slab: 237612 kB# grep dst_cache /proc/slabinfo ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0 xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0 ip_dst_cache 2881 3990 192 42 2 : tunables 0 0 0During test:
# grep -e Slab -e Free /proc/meminfo MemFree: 29417608 kB Slab: 247712 kB# grep dst_cache /proc/slabinfo ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0 xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0 ip_dst_cache 44394 44394 192 42 2 : tunables 0 0 0After test:
# grep -e Slab -e Free /proc/meminfo MemFree: 29422308 kB Slab: 238104 kB# grep dst_cache /proc/slabinfo ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0 xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0 ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0Ipv6 with patch:
Errno 101 errors are not observed anymore with the patch.
Before test:
# grep -e Slab -e Free /proc/meminfo MemFree: 29422308 kB Slab: 238104 kB# grep dst_cache /proc/slabinfo ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0 xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0 ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0During Test:
# grep -e Slab -e Free /proc/meminfo MemFree: 29431516 kB Slab: 240940 kB# grep dst_cache /proc/slabinfo ip6_dst_cache 11980 12064 256 32 2 : tunables 0 0 0 xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0 ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0After Test:
# grep -e Slab -e Free /proc/meminfo MemFree: 29441816 kB Slab: 238132 kB# grep dst_cache /proc/slabinfo ip6_dst_cache 1902 2432 256 32 2 : tunables 0 0 0 xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0 ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0Tested-by Andrea Mayer <[email protected]>Signed-off-by Jon Maxwell <[email protected]>--- include/net/dst_ops.h | 2 +- net/core/dst.c | 8 ++------ net/ipv6/route.c | 13 +++++-------- 3 files changed, 8 insertions(+), 15 deletions(-)include/net/dst_ops.h +1
-1@@ -16,7 +16,7 @@ struct dst_ops {unsigned short family;unsigned int gc_thresh; -int (*gc)(struct dst_ops *ops); +void (*gc)(struct dst_ops *ops);struct dst_entry * (*check)(struct dst_entry *,__u32 cookie);unsigned int (*default_advmss)(const struct dst_entry *);unsigned int (*mtu)(const struct dst_entry *);net/core/dst.c +2
-6@@ -82,12 +82,8 @@ void *dst_alloc(struct dst_ops *ops, struct net_device *dev,if (ops->gc && !(flags &DST_NOCOUNT ) && -dst_entries_get_fast (ops) > ops->gc_thresh) { -if (ops->gc (ops)) { -pr_notice_ratelimited ("Route cache is full: consider increasing sysctl net.ipv6.route.max_size.\n" ); -return NULL ; - } - } +dst_entries_get_fast (ops) > ops->gc_thresh) + ops->gc (ops); dst =kmem_cache_alloc (ops->kmem_cachep,GFP_ATOMIC );if (!dst)net/ipv6/route.c +5
-9@@ -91,7 +91,7 @@ static struct dst_entry *ip6_negative_advice(struct dst_entry *);static void ip6_dst_destroy (struct dst_entry *);static void ip6_dst_ifdown (struct dst_entry *,struct net_device *dev,int how); -static int ip6_dst_gc (struct dst_ops *ops); +static void ip6_dst_gc (struct dst_ops *ops);static int ip6_pkt_discard (struct sk_buff *skb);static int ip6_pkt_discard_out (struct net *net,struct sock *sk,struct sk_buff *skb);@@ -3284,11 +3284,10 @@ struct dst_entry *icmp6_dst_alloc(struct net_device *dev,return dst; } -static int ip6_dst_gc (struct dst_ops *ops) +static void ip6_dst_gc (struct dst_ops *ops) { struct net *net = container_of(ops, struct net, ipv6.ip6_dst_ops);int rt_min_interval = net->ipv6.sysctl.ip6_rt_gc_min_interval; -int rt_max_size = net->ipv6.sysctl.ip6_rt_max_size;int rt_elasticity = net->ipv6.sysctl.ip6_rt_gc_elasticity;int rt_gc_timeout = net->ipv6.sysctl.ip6_rt_gc_timeout;unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc;@@ -3296,11 +3295,10 @@ static int ip6_dst_gc(struct dst_ops *ops)int entries; entries =dst_entries_get_fast (ops); -if (entries > rt_max_size) +if (entries > ops->gc_thresh) entries =dst_entries_get_slow (ops); -if (time_after (rt_last_gc + rt_min_interval, jiffies) && - entries <= rt_max_size) +if (time_after (rt_last_gc + rt_min_interval, jiffies)) goto out;fib6_run_gc (atomic_inc_return (&net->ipv6.ip6_rt_gc_expire), net, true);@@ -3310,7 +3308,6 @@ static int ip6_dst_gc(struct dst_ops *ops) out: val =atomic_read (&net->ipv6.ip6_rt_gc_expire);atomic_set (&net->ipv6.ip6_rt_gc_expire, val - (val >> rt_elasticity)); -return entries > rt_max_size; } static int ip6_nh_lookup_table(struct net *net, struct fib6_config *cfg,@@ -6512,7 +6509,7 @@ static int __net_init ip6_route_net_init(struct net *net)#endif net->ipv6.sysctl.flush_delay =0 ; - net->ipv6.sysctl.ip6_rt_max_size =4096 ; + net->ipv6.sysctl.ip6_rt_max_size =INT_MAX ; net->ipv6.sysctl.ip6_rt_gc_min_interval =HZ /2 ; net->ipv6.sysctl.ip6_rt_gc_timeout =60 *HZ ; net->ipv6.sysctl.ip6_rt_gc_interval =30 *HZ ; --2.31.1
Signature · 1 line
Andrea Mayer <[email protected]>
Quoted 113 lines · Jon Maxwell
v2: Correct syntax error in
net/ipv6/route.cIn
ip6_dst_gc()replace:if (entries >
gc_thresh)With:
if (entries >
ops->gc_thresh)Sending Ipv6 packets in a loop via a raw socket triggers an issue where a route is cloned by
ip6_rt_cache_alloc()for each packet sent. This quickly consumes the Ipv6max_sizethreshold which defaults to 4096 resulting in these warnings:[1] 99.187805]
dst_alloc: 7728 callbacks suppressed
[2] Route cache is full: consider increasing sysctl net.ipv6.route.max_size. .
.
[300] Route cache is full: consider increasing sysctl net.ipv6.route.max_size.When this happens the packet is dropped and
sendto()gets a network is unreachable error:# ./a.out -sremaining pkt 200557 errno 101
remaining pkt 196462 errno 101
.
.
remaining pkt 126821 errno 101Implement David Aherns suggestion to remove
max_sizecheck seeing that Ipv6 has a GC to manage memory usage. Ipv4 already does not checkmax_size.Here are some memory comparisons for Ipv4 vs Ipv6 with the patch:
Test by running 5 instances of a program that sends UDP packets to a raw socket 5000000 times. Compare Ipv4 and Ipv6 performance with a similar program.
Ipv4:
Before test:
# grep -e Slab -e Free /proc/meminfo MemFree: 29427108 kB Slab: 237612 kB# grep dst_cache /proc/slabinfo ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0 xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0 ip_dst_cache 2881 3990 192 42 2 : tunables 0 0 0During test:
# grep -e Slab -e Free /proc/meminfo MemFree: 29417608 kB Slab: 247712 kB# grep dst_cache /proc/slabinfo ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0 xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0 ip_dst_cache 44394 44394 192 42 2 : tunables 0 0 0After test:
# grep -e Slab -e Free /proc/meminfo MemFree: 29422308 kB Slab: 238104 kB# grep dst_cache /proc/slabinfo ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0 xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0 ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0Ipv6 with patch:
Errno 101 errors are not observed anymore with the patch.
Before test:
# grep -e Slab -e Free /proc/meminfo MemFree: 29422308 kB Slab: 238104 kB# grep dst_cache /proc/slabinfo ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0 xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0 ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0During Test:
# grep -e Slab -e Free /proc/meminfo MemFree: 29431516 kB Slab: 240940 kB# grep dst_cache /proc/slabinfo ip6_dst_cache 11980 12064 256 32 2 : tunables 0 0 0 xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0 ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0After Test:
# grep -e Slab -e Free /proc/meminfo MemFree: 29441816 kB Slab: 238132 kB# grep dst_cache /proc/slabinfo ip6_dst_cache 1902 2432 256 32 2 : tunables 0 0 0 xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0 ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0
Quoted 8 lines
Tested-by Andrea Mayer <[email protected]>Signed-off-by Jon Maxwell <[email protected]>--- include/net/dst_ops.h | 2 +- net/core/dst.c | 8 ++------ net/ipv6/route.c | 13 +++++-------- 3 files changed, 8 insertions(+), 15 deletions(-)
Thanks for the data in the commit message.
This patch was applied to netdev/net-next.git (master) by Jakub Kicinski <[email protected]>:
Hello:
This patch was applied to netdev/net-next.git (master) by Jakub Kicinski <[email protected]>:
Quoted 9 lines · On Thu, 12 Jan 2023 12:25:32 +1100 you wrote:
v2: Correct syntax error in
net/ipv6/route.cIn
ip6_dst_gc()replace:if (entries >
gc_thresh)With:
[...]
Here is the summary with links:
- [net-next,v2] ipv6: remove max_size check inline with ipv4
https://git.kernel.org/netdev/net-next/c/af6d10345ca7
You are awesome, thank you!
Signature · 2 lines
Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/patchwork/pwbot.html