Skip to contentSkip to conversation

ipvs: add a sysctl switch to control ipvs to bypass OUTPUT chain or not

PATCH In review longguang.yue 2 participants 3 messages Aug 19, 2022 – Aug 25, 2022 1 message not in the archive

Subsystems (4)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 ipvs: add a sysctl switch to control ipvs to bypass OUTPUT chain or not

Netfilter's rules are matched in sequence, more rules worse performance. IPVS is a special system, its traffic is clear and definite, for better performance, should better not be interfered heavily. This patch adds a sysctl switch and enable ipvs to control traffic to pass netfilter OUTPUT chain or not.

---
 include/net/ip_vs.h             | 11 +++++++++++
 net/netfilter/ipvs/ip_vs_ctl.c  |  8 ++++++++
 net/netfilter/ipvs/ip_vs_xmit.c | 31 ++++++++++++++++++++++++-------
 3 files changed, 43 insertions(+), 7 deletions(-)
include/net/ip_vs.h +11 -0
index ff1804a0c469..c1232ef3a1b5 100644
@@ -932,6 +932,7 @@ struct netns_ipvs {
 	int			sysctl_schedule_icmp;
 	int			sysctl_ignore_tunneled;
 	int			sysctl_run_estimation;
+	int			sysctl_output_bypass;
 
 	/* ip_vs_lblc */
 	int			sysctl_lblc_expiration;
@@ -1077,6 +1078,11 @@ static inline int sysctl_run_estimation(struct netns_ipvs *ipvs)
 	return ipvs->sysctl_run_estimation;
 }
 
+static inline int sysctl_output_bypass(struct netns_ipvs *ipvs)
+{
+	return ipvs->sysctl_output_bypass;
+}
+
 #else
 
 static inline int sysctl_sync_threshold(struct netns_ipvs *ipvs)
@@ -1174,6 +1180,11 @@ static inline int sysctl_run_estimation(struct netns_ipvs *ipvs)
 	return 1;
 }
 
+static inline int sysctl_output_bypass(struct netns_ipvs *ipvs)
+{
+	return 0;
+}
+
 #endif
 
 /* IPVS core functions
net/netfilter/ipvs/ip_vs_ctl.c +8 -0
index efab2b06d373..8a08a783e85e 100644
@@ -2019,6 +2019,12 @@ static struct ctl_table vs_vars[] = {
 		.mode		= 0644,
 		.proc_handler	= proc_dointvec,
 	},
+	{
+		.procname	= "output_bypass",
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec,
+	},
 #ifdef CONFIG_IP_VS_DEBUG
 	{
 		.procname	= "debug_level",
@@ -4094,6 +4100,8 @@ static int __net_init ip_vs_control_net_init_sysctl(struct netns_ipvs *ipvs)
 	tbl[idx++].data = &ipvs->sysctl_ignore_tunneled;
 	ipvs->sysctl_run_estimation = 1;
 	tbl[idx++].data = &ipvs->sysctl_run_estimation;
+	ipvs->sysctl_output_bypass = 1;
+	tbl[idx++].data = &ipvs->sysctl_output_bypass;
 #ifdef CONFIG_IP_VS_DEBUG
 	/* Global sysctls must be ro in non-init netns */
 	if (!net_eq(net, &init_net))
net/netfilter/ipvs/ip_vs_xmit.c +24 -7
index 029171379884..46a34dd2555e 100644
@@ -653,8 +653,12 @@ static inline int ip_vs_nat_send_or_cont(int pf, struct sk_buff *skb,
 		skb_forward_csum(skb);
 		if (skb->dev)
 			skb_clear_tstamp(skb);
-		NF_HOOK(pf, NF_INET_LOCAL_OUT, cp->ipvs->net, NULL, skb,
-			NULL, skb_dst(skb)->dev, dst_output);
+		if (sysctl_output_bypass(cp->ipvs)) {
+			dst_output(cp->ipvs->net, NULL, skb);
+		} else {
+			NF_HOOK(pf, NF_INET_LOCAL_OUT, cp->ipvs->net, NULL, skb,
+				NULL, skb_dst(skb)->dev, dst_output);
+		}
 	} else
 		ret = NF_ACCEPT;
 
@@ -675,8 +679,12 @@ static inline int ip_vs_send_or_cont(int pf, struct sk_buff *skb,
 		skb_forward_csum(skb);
 		if (skb->dev)
 			skb_clear_tstamp(skb);
-		NF_HOOK(pf, NF_INET_LOCAL_OUT, cp->ipvs->net, NULL, skb,
-			NULL, skb_dst(skb)->dev, dst_output);
+		if (sysctl_output_bypass(cp->ipvs)) {
+			dst_output(cp->ipvs->net, NULL, skb);
+		} else {
+			NF_HOOK(pf, NF_INET_LOCAL_OUT, cp->ipvs->net, NULL, skb,
+				NULL, skb_dst(skb)->dev, dst_output);
+		}
 	} else
 		ret = NF_ACCEPT;
 	return ret;
@@ -1262,10 +1270,19 @@ ip_vs_tunnel_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
 	skb->ignore_df = 1;
 
 	ret = ip_vs_tunnel_xmit_prepare(skb, cp);
-	if (ret == NF_ACCEPT)
-		ip_local_out(net, skb->sk, skb);
-	else if (ret == NF_DROP)
+	if (ret == NF_ACCEPT) {
+		if (sysctl_output_bypass(cp->ipvs)) {
+			struct iphdr *iph = ip_hdr(skb);
+
+			iph->tot_len = htons(skb->len);
+			ip_send_check(iph);
+			dst_output(cp->ipvs->net, NULL, skb);
+		} else {
+			ip_local_out(net, skb->sk, skb);
+		}
+	} else if (ret == NF_DROP) {
 		kfree_skb(skb);
+	}
 
 	LeaveFunction(10);
 
Signature · 1 line
2.34.1
maintainer
#1 lore raw

Hello,

Quoted 67 lines · longguang.yue

Netfilter's rules are matched in sequence, more rules worse performance. IPVS is a special system, its traffic is clear and definite, for better performance, should better not be interfered heavily. This patch adds a sysctl switch and enable ipvs to control traffic to pass netfilter OUTPUT chain or not.

---
 include/net/ip_vs.h             | 11 +++++++++++
 net/netfilter/ipvs/ip_vs_ctl.c  |  8 ++++++++
 net/netfilter/ipvs/ip_vs_xmit.c | 31 ++++++++++++++++++++++++-------
 3 files changed, 43 insertions(+), 7 deletions(-)
include/net/ip_vs.h +11 -0
index ff1804a0c469..c1232ef3a1b5 100644
@@ -932,6 +932,7 @@ struct netns_ipvs {
 	int			sysctl_schedule_icmp;
 	int			sysctl_ignore_tunneled;
 	int			sysctl_run_estimation;
+	int			sysctl_output_bypass;
 
 	/* ip_vs_lblc */
 	int			sysctl_lblc_expiration;
@@ -1077,6 +1078,11 @@ static inline int sysctl_run_estimation(struct netns_ipvs *ipvs)
 	return ipvs->sysctl_run_estimation;
 }
 
+static inline int sysctl_output_bypass(struct netns_ipvs *ipvs)
+{
+	return ipvs->sysctl_output_bypass;
+}
+
 #else
 
 static inline int sysctl_sync_threshold(struct netns_ipvs *ipvs)
@@ -1174,6 +1180,11 @@ static inline int sysctl_run_estimation(struct netns_ipvs *ipvs)
 	return 1;
 }
 
+static inline int sysctl_output_bypass(struct netns_ipvs *ipvs)
+{
+	return 0;
+}
+
 #endif
 
 /* IPVS core functions
net/netfilter/ipvs/ip_vs_ctl.c +6 -0
index efab2b06d373..8a08a783e85e 100644
@@ -2019,6 +2019,12 @@ static struct ctl_table vs_vars[] = {
 		.mode		= 0644,
 		.proc_handler	= proc_dointvec,
 	},
+	{
+		.procname	= "output_bypass",
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec,
+	},
 #ifdef CONFIG_IP_VS_DEBUG
 	{
 		.procname	= "debug_level",

net/netfilter/ipvs/ip_vs_ctl.c static int __net_init ip_vs_control_net_init_sysctl(struct netns_ipvs *ipvs) · line 4103

Quoted 4 lines
net/netfilter/ipvs/ip_vs_ctl.c +1 -0
@@ -4094,6 +4100,8 @@ static int __net_init ip_vs_control_net_init_sysctl(struct netns_ipvs *ipvs)
 	tbl[idx++].data = &ipvs->sysctl_ignore_tunneled;
 	ipvs->sysctl_run_estimation = 1;
 	tbl[idx++].data = &ipvs->sysctl_run_estimation;
+	ipvs->sysctl_output_bypass = 1;

Default should be 0, so without above line.

Quoted 42 lines
net/netfilter/ipvs/ip_vs_ctl.c +1 -0
+	tbl[idx++].data = &ipvs->sysctl_output_bypass;
 #ifdef CONFIG_IP_VS_DEBUG
 	/* Global sysctls must be ro in non-init netns */
 	if (!net_eq(net, &init_net))
net/netfilter/ipvs/ip_vs_xmit.c +12 -4
index 029171379884..46a34dd2555e 100644
@@ -653,8 +653,12 @@ static inline int ip_vs_nat_send_or_cont(int pf, struct sk_buff *skb,
 		skb_forward_csum(skb);
 		if (skb->dev)
 			skb_clear_tstamp(skb);
-		NF_HOOK(pf, NF_INET_LOCAL_OUT, cp->ipvs->net, NULL, skb,
-			NULL, skb_dst(skb)->dev, dst_output);
+		if (sysctl_output_bypass(cp->ipvs)) {
+			dst_output(cp->ipvs->net, NULL, skb);
+		} else {
+			NF_HOOK(pf, NF_INET_LOCAL_OUT, cp->ipvs->net, NULL, skb,
+				NULL, skb_dst(skb)->dev, dst_output);
+		}
 	} else
 		ret = NF_ACCEPT;
 
@@ -675,8 +679,12 @@ static inline int ip_vs_send_or_cont(int pf, struct sk_buff *skb,
 		skb_forward_csum(skb);
 		if (skb->dev)
 			skb_clear_tstamp(skb);
-		NF_HOOK(pf, NF_INET_LOCAL_OUT, cp->ipvs->net, NULL, skb,
-			NULL, skb_dst(skb)->dev, dst_output);
+		if (sysctl_output_bypass(cp->ipvs)) {
+			dst_output(cp->ipvs->net, NULL, skb);
+		} else {
+			NF_HOOK(pf, NF_INET_LOCAL_OUT, cp->ipvs->net, NULL, skb,
+				NULL, skb_dst(skb)->dev, dst_output);
+		}
 	} else
 		ret = NF_ACCEPT;
 	return ret;
@@ -1262,10 +1270,19 @@ ip_vs_tunnel_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
 	skb->ignore_df = 1;
 
 	ret = ip_vs_tunnel_xmit_prepare(skb, cp);

net/netfilter/ipvs/ip_vs_xmit.c ip_vs_tunnel_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, · line 1279

Quoted 10 lines
net/netfilter/ipvs/ip_vs_xmit.c +7 -3
-	if (ret == NF_ACCEPT)
-		ip_local_out(net, skb->sk, skb);
-	else if (ret == NF_DROP)
+	if (ret == NF_ACCEPT) {
+		if (sysctl_output_bypass(cp->ipvs)) {
+			struct iphdr *iph = ip_hdr(skb);
+
+			iph->tot_len = htons(skb->len);
+			ip_send_check(iph);
+			dst_output(cp->ipvs->net, NULL, skb);

After our last discussion I still think this is a hack.
If you expand also __ip6_local_out you'll see what maintenance cost adds such flag. Your patch also warns how much ip_local_out changed in the years and our NF_HOOK calls look as hacks too. They were used to avoid duplicate calls like ip_send_check which we should not have anymore, ip_send_check was added to the ip_defrag flow in 2015 with commit 0848f6428ba3. So, I think we should be moving to ip_local_out/ip6_local_out calls which are risky to duplicate here in the IPVS code.

net/netfilter/ipvs/ip_vs_xmit.c ip_vs_tunnel_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, · line 1287

Quoted 8 lines
net/netfilter/ipvs/ip_vs_xmit.c +5 -0
+		} else {
+			ip_local_out(net, skb->sk, skb);
+		}
+	} else if (ret == NF_DROP) {
 		kfree_skb(skb);
+	}
 
 	LeaveFunction(10);

Regards

--
Julian Anastasov <[email protected]>

A message that is not in the archive (sent off-list or not yet received): [email protected]

maintainer
#3 lore raw

Hello,

Quoted 3 lines · longguang.yue

I see.
I hope we could find a maintainable and decoupled way to keep ipvs high performance. especially for kubernetes environment, there are from dozens up to one hundred rules in OUTPUT chain.

May be some rules can help the bunch of rules to
be applied only for first packet, not for every packet in connection, such as:

iptables -t filter -A OUTPUT -m state --state RELATED,ESTABLISHED -j ACCEPT iptables -t filter -A OUTPUT -m ipvs --ipvs -j ACCEPT

Regards

--
Julian Anastasov <[email protected]>