ipvs: drop first packet to dead server
Subsystems (5)
PATCH 1/1 ipvs: drop first packet to dead server
Since commit dc7b3eb9 (ipvs: Fix reuse connection if real server is dead), new connections to dead servers are redistributed immediately to new servers. The old connection is expired using ip_vs_conn_expire_now() which sets the connection timer to expire immediately.
However, before the timer callback, ip_vs_conn_expire(), is run to clean the connection's conntrack entry, the new redistributed connection may already be established and its conntrack removed instead.
Fix this by dropping the first packet of the new connection instead, like we do when the destination server is not available. The timer will have deleted the old conntrack entry long before the first packet of the new connection is retransmitted.
net/netfilter/ipvs/ip_vs_core.c +1 -1
@@ -1657,7 +1657,7 @@ ip_vs_in(unsigned int hooknum, struct sk_buff *skb, int af) is_new_conn(skb, &iph)) {ip_vs_conn_expire_now (cp);__ip_vs_conn_put (cp); - cp =NULL ; +return NF_DROP ; }if (unlikely (!cp) && !iph.fragoffs) {
Signature · 2 lines
Jiri Bohac <[email protected]> SUSE Labs, SUSE CZ
PATCH v2 1/1 ipvs: drop first packet to dead server
[ original diff was against and older tree, sorry;
rediffed against net-next ]
Since commit dc7b3eb9 (ipvs: Fix reuse connection if real server is dead), new connections to dead servers are redistributed immediately to new servers. The old connection is expired using ip_vs_conn_expire_now() which sets the connection timer to expire immediately.
However, before the timer callback, ip_vs_conn_expire(), is run to clean the connection's conntrack entry, the new redistributed connection may already be established and its conntrack removed instead.
Fix this by dropping the first packet of the new connection instead, like we do when the destination server is not available. The timer will have deleted the old conntrack entry long before the first packet of the new connection is retransmitted.
net/netfilter/ipvs/ip_vs_core.c +1 -1
@@ -1785,7 +1785,7 @@ ip_vs_in(unsigned int hooknum, struct sk_buff *skb, int af)if (!atomic_read (&cp->n_control))ip_vs_conn_expire_now (cp);__ip_vs_conn_put (cp); - cp =NULL ; +return NF_DROP ; }if (unlikely (!cp)) {
Julian Anastasov
maintainerHello,
Review
Quoted 5 lines · Jiri Bohac
(quoted diff) +1
-1if (!atomic_read(&cp->n_control)) ip_vs_conn_expire_now(cp); __ip_vs_conn_put(cp); - cp = NULL; + return NF_DROP;
So, at this point we do not know whether we have
one or many real servers, with same or different forwarding method. For example, if we know that old real server is DR and the new real server is again DR we can reuse the conntrack.
Without such info we have to drop the connection
_only_ when conntrack is used. But if the connection is referenced by others, eg. FTP:21 referenced by data connection, it is not going to expire soon. We have to reuse it, otherwise client will retransmit for long time. See below...
Also, according to RFC 5961, 4.2. Mitigation, client can
send RST during its SYN retransmissions. So, may be we have to allow rescheduling by adding IP_VS_TCP_S_CLOSE check after the IP_VS_TCP_S_TIME_WAIT check in is_new_conn_expected(). This should allow restarted client to be rescheduled. But this can be in separate patch.
Here is a RFC patch (not for latest tree, not tested):
include/net/ip_vs.h +17 -0
@@ -1591,6 +1591,23 @@ static inline void ip_vs_conn_drop_conntrack(struct ip_vs_conn *cp) }#endif /* CONFIG_IP_VS_NFCT */ +/* Really using conntrack? */ +static inline bool ip_vs_conn_uses_conntrack (struct ip_vs_conn *cp, +struct sk_buff *skb) +{ +#ifdef CONFIG_IP_VS_NFCT +enum ip_conntrack_info ctinfo; +struct nf_conn *ct; + +if (!(cp->flags &IP_VS_CONN_F_NFCT )) +return false; + ct =nf_ct_get (skb, &ctinfo); +if (ct && !nf_ct_is_untracked (ct)) +return true; +#endif +return false; +} +static inline int ip_vs_dest_conn_overhead (struct ip_vs_dest *dest) {
net/netfilter/ipvs/ip_vs_core.c +28 -9
@@ -1689,15 +1689,34 @@ ip_vs_in(unsigned int hooknum, struct sk_buff *skb, int af) cp = pp->conn_in_get (af, skb, &iph,0 ); conn_reuse_mode =sysctl_conn_reuse_mode (ipvs); -if (conn_reuse_mode && !iph.fragoffs && -is_new_conn (skb, &iph) && cp && - ((unlikely (sysctl_expire_nodest_conn (ipvs)) && cp->dest && -unlikely (!atomic_read (&cp->dest->weight))) || -unlikely (is_new_conn_expected (cp, conn_reuse_mode)))) { -if (!atomic_read (&cp->n_control)) -ip_vs_conn_expire_now (cp); -__ip_vs_conn_put (cp); - cp =NULL ; +if (conn_reuse_mode && !iph.fragoffs &&is_new_conn (skb, &iph) && cp) { +bool uses_ct = false, resched = false; + +if (unlikely (sysctl_expire_nodest_conn (ipvs)) && cp->dest && +unlikely (!atomic_read (&cp->dest->weight))) { + resched = true; + uses_ct =ip_vs_conn_uses_conntrack (cp, skb); + }else if (is_new_conn_expected (cp, conn_reuse_mode)) { + uses_ct =ip_vs_conn_uses_conntrack (cp, skb); +if (!atomic_read (&cp->n_control)) { + resched = true; + }else { +/* Do not reschedule controlling connection +* that uses conntrack while it is still +* referenced by controlled connection(s). +*/ + resched = !uses_ct; + } + } + +if (resched) { +if (!atomic_read (&cp->n_control)) +ip_vs_conn_expire_now (cp); +__ip_vs_conn_put (cp); +if (uses_ct) +return NF_DROP ; + cp =NULL ; + } }if (unlikely (!cp) && !iph.fragoffs) {
Regards
--
Julian Anastasov <[email protected]>
Hi,
Quoted 13 lines · Julian Anastasov
Review
Quoted 5 lines · Jiri Bohac
(quoted diff) +1
-1if (!atomic_read(&cp->n_control)) ip_vs_conn_expire_now(cp); __ip_vs_conn_put(cp); - cp = NULL; + return NF_DROP;So, at this point we do not know whether we have
one or many real servers, with same or different forwarding method. For example, if we know that old real server is DR and the new real server is again DR we can reuse the conntrack.
Quoted 2 lines
Without such info we have to drop the connection
_only_ when conntrack is used.
right, good point!
Quoted 6 lines
include/net/ip_vs.h +6
-0+static inline bool ip_vs_conn_uses_conntrack (struct ip_vs_conn *cp, +struct sk_buff *skb) +{ +#ifdef CONFIG_IP_VS_NFCT +enum ip_conntrack_info ctinfo; +struct nf_conn *ct;
include/net/ip_vs.h static inline void ip_vs_conn_drop_conntrack(struct ip_vs_conn *cp) · line 1610
Quoted 10 lines
include/net/ip_vs.h +10
-0+ +if (!(cp->flags &IP_VS_CONN_F_NFCT )) +return false; + ct =nf_ct_get (skb, &ctinfo); +if (ct && !nf_ct_is_untracked (ct)) +return true; +#endif +return false; +} +
I tested this part; we found the problem on an old (3.12) kernel, we're missing the parts dealing with rescheduling on port reuse - only dealing with the "weight == 0" case.
Quoted 18 lines
net/netfilter/ipvs/ip_vs_core.c +18
-0+if (conn_reuse_mode && !iph.fragoffs &&is_new_conn (skb, &iph) && cp) { +bool uses_ct = false, resched = false; + +if (unlikely (sysctl_expire_nodest_conn (ipvs)) && cp->dest && +unlikely (!atomic_read (&cp->dest->weight))) { + resched = true; + uses_ct =ip_vs_conn_uses_conntrack (cp, skb); + }else if (is_new_conn_expected (cp, conn_reuse_mode)) { + uses_ct =ip_vs_conn_uses_conntrack (cp, skb); +if (!atomic_read (&cp->n_control)) { + resched = true; + }else { +/* Do not reschedule controlling connection +* that uses conntrack while it is still +* referenced by controlled connection(s). +*/ + resched = !uses_ct; + }
net/netfilter/ipvs/ip_vs_core.c ip_vs_in(unsigned int hooknum, struct sk_buff *skb, int af) · line 1719
Quoted 10 lines
net/netfilter/ipvs/ip_vs_core.c +10
-0+ } + +if (resched) { +if (!atomic_read (&cp->n_control)) +ip_vs_conn_expire_now (cp); +__ip_vs_conn_put (cp); +if (uses_ct) +return NF_DROP ; + cp =NULL ; + }
Looks good, but I can't easily test this.
Thanks,
Signature · 2 lines
Jiri Bohac <[email protected]> SUSE Labs, SUSE CZ