net: ipvs: add sysctl_run_estimation to support disable estimation
Subsystems (6)
PATCH v3 1/1 net: ipvs: add sysctl_run_estimation to support disable estimation
estimation_timer will iterater the est_list to do estimation for each ipvs stats. When there are lots of services, the list can be very large.
We observiced estimation_timer() run for more then 200ms on a machine with 104 CPU and 50K services.
yunhong-cgl jiang report the same phenomenon before:
https://www.spinics.net/lists/lvs-devel/msg05426.html
In some cases(for example a large K8S cluster with many ipvs services), ipvs estimation may not be needed. So adding a sysctl blob to allow users to disable this completely.
Default is: 1 (enable)
---
v2: Use common sysctl facilities
v3: Fix sysctl_run_estimation() redefine when CONFIG_SYSCTL not enabled
--- Documentation/networking/ipvs-sysctl.rst | 17 +++++++++++++++++ include/net/ip_vs.h | 12 ++++++++++++ net/netfilter/ipvs/ip_vs_ctl.c | 8 ++++++++ net/netfilter/ipvs/ip_vs_est.c | 5 +++++ 4 files changed, 42 insertions(+)
Documentation/networking/ipvs-sysctl.rst +17 -0
@@ -300,3 +300,20 @@ sync_version - INTEGER Kernels with this sync_version entry are able to receive messages of both version 1 and version 2 of the synchronisation protocol. + +run_estimation - BOOLEAN + 0 - disabled + not 0 - enabled (default) + + If disabled, the estimation will be stop, and you can't see + any update on speed estimation data. + + For example + 'Conns/s Pkts/s Pkts/s Bytes/s Bytes/s' + those data in /proc/net/ip_vs_stats will always be zero. + Note, this only affect the speed estimation, the total data + will still be updated. + + You can always re-enable estimation by setting this value to 1. + But be carefull, the first estimation after re-enable is not + accurate.
include/net/ip_vs.h +12 -0
@@ -931,6 +931,7 @@ struct netns_ipvs {int sysctl_conn_reuse_mode;int sysctl_schedule_icmp;int sysctl_ignore_tunneled; +int sysctl_run_estimation;/* ip_vs_lblc */ int sysctl_lblc_expiration;
@@ -1071,6 +1072,11 @@ static inline int sysctl_cache_bypass(struct netns_ipvs *ipvs) return ipvs->sysctl_cache_bypass; } +static inline int sysctl_run_estimation (struct netns_ipvs *ipvs) +{ +return ipvs->sysctl_run_estimation; +} +#else static inline int sysctl_sync_threshold (struct netns_ipvs *ipvs)
@@ -1163,6 +1169,11 @@ static inline int sysctl_cache_bypass(struct netns_ipvs *ipvs) return 0; } +static inline int sysctl_run_estimation (struct netns_ipvs *ipvs) +{ +return 1 ; +} +#endif /* IPVS core functions
@@ -1650,6 +1661,7 @@ static inline int ip_vs_confirm_conntrack(struct sk_buff *skb)static inline void ip_vs_conn_drop_conntrack (struct ip_vs_conn *cp) { } +#endif /* CONFIG_IP_VS_NFCT */ /* Using old conntrack that can not be redirected to another real server? */
net/netfilter/ipvs/ip_vs_ctl.c +8 -0
@@ -2017,6 +2017,12 @@ static struct ctl_table vs_vars[] = { .mode = 0644, .proc_handler = proc_dointvec, }, + { + .procname = "run_estimation", + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, #ifdef CONFIG_IP_VS_DEBUG { .procname = "debug_level",
@@ -4090,6 +4096,8 @@ static int __net_init ip_vs_control_net_init_sysctl(struct netns_ipvs *ipvs) tbl[idx++].data = &ipvs->sysctl_conn_reuse_mode; tbl[idx++].data = &ipvs->sysctl_schedule_icmp; tbl[idx++].data = &ipvs->sysctl_ignore_tunneled; + ipvs->sysctl_run_estimation =1 ; + tbl[idx++].data = &ipvs->sysctl_run_estimation; ipvs->sysctl_hdr =register_net_sysctl (net,"net/ipv4/vs" , tbl);if (ipvs->sysctl_hdr ==NULL ) {
net/netfilter/ipvs/ip_vs_est.c +5 -0
@@ -100,6 +100,9 @@ static void estimation_timer(struct timer_list *t)u64 rate;struct netns_ipvs *ipvs =from_timer (ipvs, t, est_timer); +if (!sysctl_run_estimation (ipvs)) + goto skip; +spin_lock (&ipvs->est_lock);list_for_each_entry (e, &ipvs->est_list, list) { s = container_of(e, struct ip_vs_stats, est);
@@ -131,6 +134,8 @@ static void estimation_timer(struct timer_list *t)spin_unlock (&s->lock); }spin_unlock (&ipvs->est_lock); + +skip:mod_timer (&ipvs->est_timer, jiffies +2 *HZ ); }
Signature · 1 line
2.19.1.3.ge56e4f7
Julian Anastasov
maintainerHello,
Quoted 1 line · Dust Li
estimation_timerwill iterater theest_listto do estimation
...will iterate...
Quoted 3 lines
for each ipvs stats. When there are lots of services, the list can be very large.
We observicedestimation_timer()run for more then 200ms on
...We found that estimation_timer() runs for...
Quoted 9 lines
a machine with 104 CPU and 50K services.
yunhong-cgl jiang report the same phenomenon before:
https://www.spinics.net/lists/lvs-devel/msg05426.htmlIn some cases(for example a large K8S cluster with many ipvs services), ipvs estimation may not be needed. So adding a sysctl blob to allow users to disable this completely.
Quoted 8 lines
Default is: 1 (enable)
Cc yunhong-cgl jiang <[email protected]>Signed-off-by Dust Li <[email protected]>---
v2: Use common sysctl facilities
v3: Fixsysctl_run_estimation()redefine whenCONFIG_SYSCTLnot enabled
---
Replace above "---" after v3 with empty line,
avoid multiple "---" lines.
Quoted 17 lines
Documentation/networking/ipvs-sysctl.rst | 17 +++++++++++++++++ include/net/ip_vs.h | 12 ++++++++++++ net/netfilter/ipvs/ip_vs_ctl.c | 8 ++++++++ net/netfilter/ipvs/ip_vs_est.c | 5 +++++ 4 files changed, 42 insertions(+)Documentation/networking/ipvs-sysctl.rst +3
-0@@ -300,3 +300,20 @@ sync_version - INTEGER Kernels with this sync_version entry are able to receive messages of both version 1 and version 2 of the synchronisation protocol. + +run_estimation - BOOLEAN + 0 - disabled
Documentation/networking/ipvs-sysctl.rst sync_version - INTEGER · line 315
Quoted 10 lines
Documentation/networking/ipvs-sysctl.rst +10
-0+ not 0 - enabled (default) + + If disabled, the estimation will be stop, and you can't see + any update on speed estimation data. + + For example + 'Conns/s Pkts/s Pkts/s Bytes/s Bytes/s' + those data in /proc/net/ip_vs_stats will always be zero. + Note, this only affect the speed estimation, the total data + will still be updated.
We can remove the above example
Documentation/networking/ipvs-sysctl.rst sync_version - INTEGER · line 318
Quoted 3 lines
Documentation/networking/ipvs-sysctl.rst +3
-0+ + You can always re-enable estimation by setting this value to 1. + But be carefull, the first estimation after re-enable is not
...careful...
include/net/ip_vs.h struct netns_ipvs { · line 934
Quoted 10 lines
Documentation/networking/ipvs-sysctl.rst +1
-0+ accurate.include/net/ip_vs.h +1
-0@@ -931,6 +931,7 @@ struct netns_ipvs {int sysctl_conn_reuse_mode;int sysctl_schedule_icmp;int sysctl_ignore_tunneled; +int sysctl_run_estimation;
scripts/checkpatch.pl --strict /tmp/file.patch
reports for extra space just after 'int'.
Quoted 28 lines
include/net/ip_vs.h +10
-0/* ip_vs_lblc */ int sysctl_lblc_expiration;@@ -1071,6 +1072,11 @@ static inline int sysctl_cache_bypass(struct netns_ipvs *ipvs) return ipvs->sysctl_cache_bypass; } +static inline int sysctl_run_estimation (struct netns_ipvs *ipvs) +{ +return ipvs->sysctl_run_estimation; +} +#else static inline int sysctl_sync_threshold (struct netns_ipvs *ipvs)@@ -1163,6 +1169,11 @@ static inline int sysctl_cache_bypass(struct netns_ipvs *ipvs) return 0; } +static inline int sysctl_run_estimation (struct netns_ipvs *ipvs) +{ +return 1 ; +} +#endif /* IPVS core functions
include/net/ip_vs.h static inline int ip_vs_confirm_conntrack(struct sk_buff *skb) · line 1664
Quoted 4 lines
include/net/ip_vs.h +1
-0@@ -1650,6 +1661,7 @@ static inline int ip_vs_confirm_conntrack(struct sk_buff *skb)static inline void ip_vs_conn_drop_conntrack (struct ip_vs_conn *cp) { } +
Remove this irrelevant hunk.
Quoted 54 lines
include/net/ip_vs.h +0
-0#endif /* CONFIG_IP_VS_NFCT */ /* Using old conntrack that can not be redirected to another real server? */ net/netfilter/ipvs/ip_vs_ctl.c +8
-0@@ -2017,6 +2017,12 @@ static struct ctl_table vs_vars[] = { .mode = 0644, .proc_handler = proc_dointvec, }, + { + .procname = "run_estimation", + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, #ifdef CONFIG_IP_VS_DEBUG { .procname = "debug_level",@@ -4090,6 +4096,8 @@ static int __net_init ip_vs_control_net_init_sysctl(struct netns_ipvs *ipvs) tbl[idx++].data = &ipvs->sysctl_conn_reuse_mode; tbl[idx++].data = &ipvs->sysctl_schedule_icmp; tbl[idx++].data = &ipvs->sysctl_ignore_tunneled; + ipvs->sysctl_run_estimation =1 ; + tbl[idx++].data = &ipvs->sysctl_run_estimation; ipvs->sysctl_hdr =register_net_sysctl (net,"net/ipv4/vs" , tbl);if (ipvs->sysctl_hdr ==NULL ) {net/netfilter/ipvs/ip_vs_est.c +5
-0@@ -100,6 +100,9 @@ static void estimation_timer(struct timer_list *t)u64 rate;struct netns_ipvs *ipvs =from_timer (ipvs, t, est_timer); +if (!sysctl_run_estimation (ipvs)) + goto skip; +spin_lock (&ipvs->est_lock);list_for_each_entry (e, &ipvs->est_list, list) { s = container_of(e, struct ip_vs_stats, est);@@ -131,6 +134,8 @@ static void estimation_timer(struct timer_list *t)spin_unlock (&s->lock); }spin_unlock (&ipvs->est_lock); + +skip:mod_timer (&ipvs->est_timer, jiffies +2 *HZ ); }Signature · 1 line
2.19.1.3.ge56e4f7
Regards
--
Julian Anastasov <[email protected]>
Quoted 7 lines · Julian Anastasov
Hello,
Quoted 1 line · Dust Li
estimation_timerwill iterater theest_listto do estimation
Quoted 7 lines
...will iterate...Quoted 3 lines
for each ipvs stats. When there are lots of services, the list can be very large.
We observicedestimation_timer()run for more then 200ms on...We found that
estimation_timer()runs for...
OK, thanks for you carefull review, I will fix those details and send a v4.
Thanks again !
Quoted 166 lines
Quoted 10 lines
a machine with 104 CPU and 50K services.
yunhong-cgl jiang report the same phenomenon before:
https://www.spinics.net/lists/lvs-devel/msg05426.htmlIn some cases(for example a large K8S cluster with many ipvs services), ipvs estimation may not be needed. So adding a sysctl blob to allow users to disable this completely.
Quoted 8 lines
Default is: 1 (enable)
Cc yunhong-cgl jiang <[email protected]>Signed-off-by Dust Li <[email protected]>---
v2: Use common sysctl facilities
v3: Fixsysctl_run_estimation()redefine whenCONFIG_SYSCTLnot enabled
---Replace above "---" after v3 with empty line,
avoid multiple "---" lines.Quoted 17 lines
Documentation/networking/ipvs-sysctl.rst | 17 +++++++++++++++++ include/net/ip_vs.h | 12 ++++++++++++ net/netfilter/ipvs/ip_vs_ctl.c | 8 ++++++++ net/netfilter/ipvs/ip_vs_est.c | 5 +++++ 4 files changed, 42 insertions(+)Documentation/networking/ipvs-sysctl.rst +3
-0@@ -300,3 +300,20 @@ sync_version - INTEGER Kernels with this sync_version entry are able to receive messages of both version 1 and version 2 of the synchronisation protocol. + +run_estimation - BOOLEAN + 0 - disabledDocumentation/networking/ipvs-sysctl.rst sync_version - INTEGER · line 315
Quoted 10 lines
Documentation/networking/ipvs-sysctl.rst +10
-0+ not 0 - enabled (default) + + If disabled, the estimation will be stop, and you can't see + any update on speed estimation data. + + For example + 'Conns/s Pkts/s Pkts/s Bytes/s Bytes/s' + those data in /proc/net/ip_vs_stats will always be zero. + Note, this only affect the speed estimation, the total data + will still be updated.We can remove the above example
Documentation/networking/ipvs-sysctl.rst sync_version - INTEGER · line 318
Quoted 3 lines
Documentation/networking/ipvs-sysctl.rst +3
-0+ + You can always re-enable estimation by setting this value to 1. + But be carefull, the first estimation after re-enable is not...careful...include/net/ip_vs.h struct netns_ipvs { · line 934
Quoted 10 lines
Documentation/networking/ipvs-sysctl.rst +1
-0+ accurate.include/net/ip_vs.h +1
-0@@ -931,6 +931,7 @@ struct netns_ipvs {int sysctl_conn_reuse_mode;int sysctl_schedule_icmp;int sysctl_ignore_tunneled; +int sysctl_run_estimation;scripts/checkpatch.pl --strict /tmp/file.patchreports for extra space just after 'int'.
Quoted 28 lines
include/net/ip_vs.h +10
-0/* ip_vs_lblc */ int sysctl_lblc_expiration;@@ -1071,6 +1072,11 @@ static inline int sysctl_cache_bypass(struct netns_ipvs *ipvs) return ipvs->sysctl_cache_bypass; } +static inline int sysctl_run_estimation (struct netns_ipvs *ipvs) +{ +return ipvs->sysctl_run_estimation; +} +#else static inline int sysctl_sync_threshold (struct netns_ipvs *ipvs)@@ -1163,6 +1169,11 @@ static inline int sysctl_cache_bypass(struct netns_ipvs *ipvs) return 0; } +static inline int sysctl_run_estimation (struct netns_ipvs *ipvs) +{ +return 1 ; +} +#endif /* IPVS core functions include/net/ip_vs.h static inline int ip_vs_confirm_conntrack(struct sk_buff *skb) · line 1664
Quoted 4 lines
include/net/ip_vs.h +1
-0@@ -1650,6 +1661,7 @@ static inline int ip_vs_confirm_conntrack(struct sk_buff *skb)static inline void ip_vs_conn_drop_conntrack (struct ip_vs_conn *cp) { } +Remove this irrelevant hunk.
Quoted 54 lines
include/net/ip_vs.h +0
-0#endif /* CONFIG_IP_VS_NFCT */ /* Using old conntrack that can not be redirected to another real server? */ net/netfilter/ipvs/ip_vs_ctl.c +8
-0@@ -2017,6 +2017,12 @@ static struct ctl_table vs_vars[] = { .mode = 0644, .proc_handler = proc_dointvec, }, + { + .procname = "run_estimation", + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, #ifdef CONFIG_IP_VS_DEBUG { .procname = "debug_level",@@ -4090,6 +4096,8 @@ static int __net_init ip_vs_control_net_init_sysctl(struct netns_ipvs *ipvs) tbl[idx++].data = &ipvs->sysctl_conn_reuse_mode; tbl[idx++].data = &ipvs->sysctl_schedule_icmp; tbl[idx++].data = &ipvs->sysctl_ignore_tunneled; + ipvs->sysctl_run_estimation =1 ; + tbl[idx++].data = &ipvs->sysctl_run_estimation; ipvs->sysctl_hdr =register_net_sysctl (net,"net/ipv4/vs" , tbl);if (ipvs->sysctl_hdr ==NULL ) {net/netfilter/ipvs/ip_vs_est.c +5
-0@@ -100,6 +100,9 @@ static void estimation_timer(struct timer_list *t)u64 rate;struct netns_ipvs *ipvs =from_timer (ipvs, t, est_timer); +if (!sysctl_run_estimation (ipvs)) + goto skip; +spin_lock (&ipvs->est_lock);list_for_each_entry (e, &ipvs->est_list, list) { s = container_of(e, struct ip_vs_stats, est);@@ -131,6 +134,8 @@ static void estimation_timer(struct timer_list *t)spin_unlock (&s->lock); }spin_unlock (&ipvs->est_lock); + +skip:mod_timer (&ipvs->est_timer, jiffies +2 *HZ ); }Signature · 1 line
2.19.1.3.ge56e4f7Regards
--
Julian Anastasov <[email protected]>