net/dst: use a smaller percpu_counter batch for dst entries accounting
Subsystems (3)
PATCH 1/1 net/dst: use a smaller percpu_counter batch for dst entries accounting
percpu_counter_add() uses a default batch size which is quite big on platforms with 256 cpus. (2*256 -> 512)
This means dst_entries_get_fast() can be off by +/- 2*(nr_cpus^2) (131072 on servers with 256 cpus)
Reduce the batch size to something more reasonable, and add logic to ip6_dst_gc() to call dst_entries_get_slow() before calling the _very_ expensive fib6_run_gc() function.
--- include/net/dst_ops.h | 4 +++- net/core/dst.c | 8 ++++---- net/ipv6/route.c | 3 +++ 3 files changed, 10 insertions(+), 5 deletions(-)
include/net/dst_ops.h +3 -1
@@ -53,9 +53,11 @@ static inline int dst_entries_get_slow(struct dst_ops *dst)return percpu_counter_sum_positive (&dst->pcpuc_entries); } +#define DST_PERCPU_COUNTER_BATCH 32static inline void dst_entries_add (struct dst_ops *dst,int val) { -percpu_counter_add (&dst->pcpuc_entries, val); +percpu_counter_add_batch (&dst->pcpuc_entries, val, +DST_PERCPU_COUNTER_BATCH ); }static inline int dst_entries_init (struct dst_ops *dst)
net/core/dst.c +4 -4
@@ -81,11 +81,11 @@ void *dst_alloc(struct dst_ops *ops, struct net_device *dev, {struct dst_entry *dst; -if (ops->gc &&dst_entries_get_fast (ops) > ops->gc_thresh) { +if (ops->gc && + !(flags &DST_NOCOUNT ) && +dst_entries_get_fast (ops) > ops->gc_thresh) {if (ops->gc (ops)) { -printk_ratelimited (KERN_NOTICE "Route cache is full: " -"consider increasing sysctl " -"net.ipv[4|6].route.max_size.\n" ); +pr_notice_ratelimited ("Route cache is full: consider increasing sysctl net.ipv6.route.max_size.\n" );return NULL ; } }
net/ipv6/route.c +3 -0
@@ -3195,6 +3195,9 @@ static int ip6_dst_gc(struct dst_ops *ops)int entries; entries =dst_entries_get_fast (ops); +if (entries > rt_max_size) + entries =dst_entries_get_slow (ops); +if (time_after (rt_last_gc + rt_min_interval, jiffies) && entries <= rt_max_size) goto out;
Signature · 1 line
2.26.2.645.ge9eca65c58-goog
Quoted 60 lines · Eric Dumazet
percpu_counter_add()uses a default batch size which is quite big on platforms with 256 cpus. (2*256 -> 512)This means
dst_entries_get_fast()can be off by +/- 2*(nr_cpus^2) (131072 on servers with 256 cpus)Reduce the batch size to something more reasonable, and add logic to
ip6_dst_gc()to calldst_entries_get_slow()before calling the _very_ expensivefib6_run_gc()function.Signed-off-by Eric Dumazet <[email protected]>--- include/net/dst_ops.h | 4 +++- net/core/dst.c | 8 ++++---- net/ipv6/route.c | 3 +++ 3 files changed, 10 insertions(+), 5 deletions(-)include/net/dst_ops.h +3
-1@@ -53,9 +53,11 @@ static inline int dst_entries_get_slow(struct dst_ops *dst)return percpu_counter_sum_positive (&dst->pcpuc_entries); } +#define DST_PERCPU_COUNTER_BATCH 32static inline void dst_entries_add (struct dst_ops *dst,int val) { -percpu_counter_add (&dst->pcpuc_entries, val); +percpu_counter_add_batch (&dst->pcpuc_entries, val, +DST_PERCPU_COUNTER_BATCH ); }static inline int dst_entries_init (struct dst_ops *dst)net/core/dst.c +4
-4@@ -81,11 +81,11 @@ void *dst_alloc(struct dst_ops *ops, struct net_device *dev, {struct dst_entry *dst; -if (ops->gc &&dst_entries_get_fast (ops) > ops->gc_thresh) { +if (ops->gc && + !(flags &DST_NOCOUNT ) && +dst_entries_get_fast (ops) > ops->gc_thresh) {if (ops->gc (ops)) { -printk_ratelimited (KERN_NOTICE "Route cache is full: " -"consider increasing sysctl " -"net.ipv[4|6].route.max_size.\n" ); +pr_notice_ratelimited ("Route cache is full: consider increasing sysctl net.ipv6.route.max_size.\n" );return NULL ; } }net/ipv6/route.c +0
-0
net/ipv6/route.c static int ip6_dst_gc(struct dst_ops *ops) · line 3201
Quoted 7 lines
net/ipv6/route.c +3
-0@@ -3195,6 +3195,9 @@ static int ip6_dst_gc(struct dst_ops *ops)int entries; entries =dst_entries_get_fast (ops); +if (entries > rt_max_size) + entries =dst_entries_get_slow (ops); + if (time_after(rt_last_gc + rt_min_interval, jiffies) &&
if this part of the condition is not satisfied, you are going to call fib6_run_gc anyways and after that you will update the entries. So I was wondering if code here could be something like:
net/ipv6/route.c +7 -2
@@ -3197,11 +3197,16 @@ static int ip6_dst_gc(struct dst_ops *ops)unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc;int entries; + if (time_before(rt_last_gc + rt_min_interval, jiffies) + goto run_gc; + entries =dst_entries_get_fast (ops); -if (time_after (rt_last_gc + rt_min_interval, jiffies) && - entries <= rt_max_size) +if (entries > rt_max_size) + entries =dst_entries_get_slow (ops); +if (entries <= rt_max_size) goto out; +run_gc: net->ipv6.ip6_rt_gc_expire++;fib6_run_gc (net->ipv6.ip6_rt_gc_expire, net, true); entries =dst_entries_get_slow (ops);
That way you could potentially avoid an extra call to dst_entries_get_slow when you know for sure that fib6_run_gc will be run. WDYT?
Quoted 5 lines
net/ipv6/route.c +0
-1entries <= rt_max_size) goto out; --2.26.2.645.ge9eca65c58-goog
Quoted 94 lines · Brian Vazquez
Quoted 60 lines · Eric Dumazet
percpu_counter_add()uses a default batch size which is quite big on platforms with 256 cpus. (2*256 -> 512)This means
dst_entries_get_fast()can be off by +/- 2*(nr_cpus^2) (131072 on servers with 256 cpus)Reduce the batch size to something more reasonable, and add logic to
ip6_dst_gc()to calldst_entries_get_slow()before calling the _very_ expensivefib6_run_gc()function.Signed-off-by Eric Dumazet <[email protected]>--- include/net/dst_ops.h | 4 +++- net/core/dst.c | 8 ++++---- net/ipv6/route.c | 3 +++ 3 files changed, 10 insertions(+), 5 deletions(-)include/net/dst_ops.h +3
-1@@ -53,9 +53,11 @@ static inline int dst_entries_get_slow(struct dst_ops *dst)return percpu_counter_sum_positive (&dst->pcpuc_entries); } +#define DST_PERCPU_COUNTER_BATCH 32static inline void dst_entries_add (struct dst_ops *dst,int val) { -percpu_counter_add (&dst->pcpuc_entries, val); +percpu_counter_add_batch (&dst->pcpuc_entries, val, +DST_PERCPU_COUNTER_BATCH ); }static inline int dst_entries_init (struct dst_ops *dst)net/core/dst.c +4
-4@@ -81,11 +81,11 @@ void *dst_alloc(struct dst_ops *ops, struct net_device *dev, {struct dst_entry *dst; -if (ops->gc &&dst_entries_get_fast (ops) > ops->gc_thresh) { +if (ops->gc && + !(flags &DST_NOCOUNT ) && +dst_entries_get_fast (ops) > ops->gc_thresh) {if (ops->gc (ops)) { -printk_ratelimited (KERN_NOTICE "Route cache is full: " -"consider increasing sysctl " -"net.ipv[4|6].route.max_size.\n" ); +pr_notice_ratelimited ("Route cache is full: consider increasing sysctl net.ipv6.route.max_size.\n" );return NULL ; } }net/ipv6/route.c +0
-0net/ipv6/route.c static int ip6_dst_gc(struct dst_ops *ops) · line 3201
Quoted 7 lines
net/ipv6/route.c +3
-0@@ -3195,6 +3195,9 @@ static int ip6_dst_gc(struct dst_ops *ops)int entries; entries =dst_entries_get_fast (ops); +if (entries > rt_max_size) + entries =dst_entries_get_slow (ops); + if (time_after(rt_last_gc + rt_min_interval, jiffies) &&if this part of the condition is not satisfied, you are going to call
fib6_run_gcanyways and after that you will update the entries. So I was wondering if code here could be something like:net/ipv6/route.c +7
-2@@ -3197,11 +3197,16 @@ static int ip6_dst_gc(struct dst_ops *ops)unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc;int entries; + if (time_before(rt_last_gc + rt_min_interval, jiffies) + goto run_gc; + entries =dst_entries_get_fast (ops); -if (time_after (rt_last_gc + rt_min_interval, jiffies) && - entries <= rt_max_size) +if (entries > rt_max_size) + entries =dst_entries_get_slow (ops); +if (entries <= rt_max_size) goto out; +run_gc: net->ipv6.ip6_rt_gc_expire++;fib6_run_gc (net->ipv6.ip6_rt_gc_expire, net, true); entries =dst_entries_get_slow (ops);
Quoted 3 lines
That way you could potentially avoid an extra call to
dst_entries_get_slowwhen you know for sure thatfib6_run_gcwill be run. WDYT?
The problem is that you might still return a wrong status in the final :
return entries > rt_max_size;
If we are in ip6_dst_gc(), we know for sure entries might be wrong, if it holds dst_entries_get_fast(ops)
If you prefer, the patch is really (since the caller calls us only if dst_entries_get_fast(ops) was suspect)
net/ipv6/route.c +0 -0
100644
net/ipv6/route.c +1 -1
@@ -3196,7 +3196,7 @@ static int ip6_dst_gc(struct dst_ops *ops)unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc;int entries; - entries =dst_entries_get_fast (ops); + entries =dst_entries_get_slow (ops);if (time_after (rt_last_gc + rt_min_interval, jiffies) && entries <= rt_max_size) goto out;
Quoted 116 lines · Eric Dumazet
Quoted 94 lines · Brian Vazquez
Quoted 60 lines · Eric Dumazet
percpu_counter_add()uses a default batch size which is quite big on platforms with 256 cpus. (2*256 -> 512)This means
dst_entries_get_fast()can be off by +/- 2*(nr_cpus^2) (131072 on servers with 256 cpus)Reduce the batch size to something more reasonable, and add logic to
ip6_dst_gc()to calldst_entries_get_slow()before calling the _very_ expensivefib6_run_gc()function.Signed-off-by Eric Dumazet <[email protected]>--- include/net/dst_ops.h | 4 +++- net/core/dst.c | 8 ++++---- net/ipv6/route.c | 3 +++ 3 files changed, 10 insertions(+), 5 deletions(-)include/net/dst_ops.h +3
-1@@ -53,9 +53,11 @@ static inline int dst_entries_get_slow(struct dst_ops *dst)return percpu_counter_sum_positive (&dst->pcpuc_entries); } +#define DST_PERCPU_COUNTER_BATCH 32static inline void dst_entries_add (struct dst_ops *dst,int val) { -percpu_counter_add (&dst->pcpuc_entries, val); +percpu_counter_add_batch (&dst->pcpuc_entries, val, +DST_PERCPU_COUNTER_BATCH ); }static inline int dst_entries_init (struct dst_ops *dst)net/core/dst.c +4
-4@@ -81,11 +81,11 @@ void *dst_alloc(struct dst_ops *ops, struct net_device *dev, {struct dst_entry *dst; -if (ops->gc &&dst_entries_get_fast (ops) > ops->gc_thresh) { +if (ops->gc && + !(flags &DST_NOCOUNT ) && +dst_entries_get_fast (ops) > ops->gc_thresh) {if (ops->gc (ops)) { -printk_ratelimited (KERN_NOTICE "Route cache is full: " -"consider increasing sysctl " -"net.ipv[4|6].route.max_size.\n" ); +pr_notice_ratelimited ("Route cache is full: consider increasing sysctl net.ipv6.route.max_size.\n" );return NULL ; } }net/ipv6/route.c +0
-0net/ipv6/route.c static int ip6_dst_gc(struct dst_ops *ops) · line 3201
Quoted 7 lines
net/ipv6/route.c +3
-0@@ -3195,6 +3195,9 @@ static int ip6_dst_gc(struct dst_ops *ops)int entries; entries =dst_entries_get_fast (ops); +if (entries > rt_max_size) + entries =dst_entries_get_slow (ops); + if (time_after(rt_last_gc + rt_min_interval, jiffies) &&if this part of the condition is not satisfied, you are going to call
fib6_run_gcanyways and after that you will update the entries. So I was wondering if code here could be something like:net/ipv6/route.c +7
-2@@ -3197,11 +3197,16 @@ static int ip6_dst_gc(struct dst_ops *ops)unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc;int entries; + if (time_before(rt_last_gc + rt_min_interval, jiffies) + goto run_gc; + entries =dst_entries_get_fast (ops); -if (time_after (rt_last_gc + rt_min_interval, jiffies) && - entries <= rt_max_size) +if (entries > rt_max_size) + entries =dst_entries_get_slow (ops); +if (entries <= rt_max_size) goto out; +run_gc: net->ipv6.ip6_rt_gc_expire++;fib6_run_gc (net->ipv6.ip6_rt_gc_expire, net, true); entries =dst_entries_get_slow (ops);Quoted 3 lines
That way you could potentially avoid an extra call to
dst_entries_get_slowwhen you know for sure thatfib6_run_gcwill be run. WDYT?The problem is that you might still return a wrong status in the final :
return entries >
rt_max_size;If we are in
ip6_dst_gc(), we know for sure entries might be wrong, if it holdsdst_entries_get_fast(ops)If you prefer, the patch is really (since the caller calls us only if
dst_entries_get_fast(ops)was suspect)net/ipv6/route.c +0
-0100644
net/ipv6/route.c static int ip6_dst_gc(struct dst_ops *ops) · line 3202
Quoted 8 lines
net/ipv6/route.c +1
-1@@ -3196,7 +3196,7 @@ static int ip6_dst_gc(struct dst_ops *ops)unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc;int entries; - entries =dst_entries_get_fast (ops); + entries =dst_entries_get_slow (ops);if (time_after (rt_last_gc + rt_min_interval, jiffies) && entries <= rt_max_size) goto out;
BTW, we do not _have_ to force a gc if entries (the more accurate value) is below gc_thresh
That would be a separate patch :
net/ipv6/route.c +0 -0
100644
net/ipv6/route.c +3 -4
@@ -3194,10 +3194,9 @@ static int ip6_dst_gc(struct dst_ops *ops)unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc;int entries; - entries =dst_entries_get_fast (ops); -if (entries > rt_max_size) - entries =dst_entries_get_slow (ops); - + entries =dst_entries_get_slow (ops); +if (entries < ops->gc_thresh) +return 0 ;if (time_after (rt_last_gc + rt_min_interval, jiffies) && entries <= rt_max_size) goto out;
Quoted 98 lines · Eric Dumazet
Quoted 96 lines · Eric Dumazet
Quoted 94 lines · Brian Vazquez
Quoted 60 lines · Eric Dumazet
percpu_counter_add()uses a default batch size which is quite big on platforms with 256 cpus. (2*256 -> 512)This means
dst_entries_get_fast()can be off by +/- 2*(nr_cpus^2) (131072 on servers with 256 cpus)Reduce the batch size to something more reasonable, and add logic to
ip6_dst_gc()to calldst_entries_get_slow()before calling the _very_ expensivefib6_run_gc()function.Signed-off-by Eric Dumazet <[email protected]>--- include/net/dst_ops.h | 4 +++- net/core/dst.c | 8 ++++---- net/ipv6/route.c | 3 +++ 3 files changed, 10 insertions(+), 5 deletions(-)include/net/dst_ops.h +3
-1@@ -53,9 +53,11 @@ static inline int dst_entries_get_slow(struct dst_ops *dst)return percpu_counter_sum_positive (&dst->pcpuc_entries); } +#define DST_PERCPU_COUNTER_BATCH 32static inline void dst_entries_add (struct dst_ops *dst,int val) { -percpu_counter_add (&dst->pcpuc_entries, val); +percpu_counter_add_batch (&dst->pcpuc_entries, val, +DST_PERCPU_COUNTER_BATCH ); }static inline int dst_entries_init (struct dst_ops *dst)net/core/dst.c +4
-4@@ -81,11 +81,11 @@ void *dst_alloc(struct dst_ops *ops, struct net_device *dev, {struct dst_entry *dst; -if (ops->gc &&dst_entries_get_fast (ops) > ops->gc_thresh) { +if (ops->gc && + !(flags &DST_NOCOUNT ) && +dst_entries_get_fast (ops) > ops->gc_thresh) {if (ops->gc (ops)) { -printk_ratelimited (KERN_NOTICE "Route cache is full: " -"consider increasing sysctl " -"net.ipv[4|6].route.max_size.\n" ); +pr_notice_ratelimited ("Route cache is full: consider increasing sysctl net.ipv6.route.max_size.\n" );return NULL ; } }net/ipv6/route.c +0
-0net/ipv6/route.c static int ip6_dst_gc(struct dst_ops *ops) · line 3201
Quoted 7 lines
net/ipv6/route.c +3
-0@@ -3195,6 +3195,9 @@ static int ip6_dst_gc(struct dst_ops *ops)int entries; entries =dst_entries_get_fast (ops); +if (entries > rt_max_size) + entries =dst_entries_get_slow (ops); + if (time_after(rt_last_gc + rt_min_interval, jiffies) &&if this part of the condition is not satisfied, you are going to call
fib6_run_gcanyways and after that you will update the entries. So I was wondering if code here could be something like:net/ipv6/route.c +7
-2@@ -3197,11 +3197,16 @@ static int ip6_dst_gc(struct dst_ops *ops)unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc;int entries; + if (time_before(rt_last_gc + rt_min_interval, jiffies) + goto run_gc; + entries =dst_entries_get_fast (ops); -if (time_after (rt_last_gc + rt_min_interval, jiffies) && - entries <= rt_max_size) +if (entries > rt_max_size) + entries =dst_entries_get_slow (ops); +if (entries <= rt_max_size) goto out; +run_gc: net->ipv6.ip6_rt_gc_expire++;fib6_run_gc (net->ipv6.ip6_rt_gc_expire, net, true); entries =dst_entries_get_slow (ops);
Quoted 7 lines
Quoted 7 lines
Quoted 3 lines
That way you could potentially avoid an extra call to
dst_entries_get_slowwhen you know for sure thatfib6_run_gcwill be run. WDYT?The problem is that you might still return a wrong status in the final :
return entries >
rt_max_size;
Oh that's right, thanks for explaining!
Quoted 36 lines
Quoted 13 lines
If we are in
ip6_dst_gc(), we know for sure entries might be wrong, if it holdsdst_entries_get_fast(ops)If you prefer, the patch is really (since the caller calls us only if
dst_entries_get_fast(ops)was suspect)net/ipv6/route.c +0
-0100644
net/ipv6/route.c static int ip6_dst_gc(struct dst_ops *ops) · line 3202
Quoted 8 lines
net/ipv6/route.c +1
-1@@ -3196,7 +3196,7 @@ static int ip6_dst_gc(struct dst_ops *ops)unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc;int entries; - entries =dst_entries_get_fast (ops); + entries =dst_entries_get_slow (ops);if (time_after (rt_last_gc + rt_min_interval, jiffies) && entries <= rt_max_size) goto out;BTW, we do not _have_ to force a gc if entries (the more accurate value) is below
gc_threshThat would be a separate patch :
net/ipv6/route.c +0
-0100644
net/ipv6/route.c +0
-0@@ -3194,10 +3194,9 @@ static int ip6_dst_gc(struct dst_ops *ops)unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc;int entries;
net/ipv6/route.c static int ip6_dst_gc(struct dst_ops *ops) · line 3202
Quoted 10 lines
net/ipv6/route.c +3
-4- entries =dst_entries_get_fast (ops); -if (entries > rt_max_size) - entries =dst_entries_get_slow (ops); - + entries =dst_entries_get_slow (ops); +if (entries < ops->gc_thresh) +return 0 ;if (time_after (rt_last_gc + rt_min_interval, jiffies) && entries <= rt_max_size) goto out;
This makes sense, thanks!
Quoted 11 lines · On Thu, 7 May 2020 18:58:10 -0700 Eric Dumazet wrote:
percpu_counter_add()uses a default batch size which is quite big on platforms with 256 cpus. (2*256 -> 512)This means
dst_entries_get_fast()can be off by +/- 2*(nr_cpus^2) (131072 on servers with 256 cpus)Reduce the batch size to something more reasonable, and add logic to
ip6_dst_gc()to calldst_entries_get_slow()before calling the _very_ expensivefib6_run_gc()function.Signed-off-by Eric Dumazet <[email protected]>
Applied, thank you!