Skip to contentSkip to conversation

ipv4: introduce new IP_MTU_DISCOVER mode IP_PMTUDISC_INTERFACE

PATCH In review Hannes Frederic Sowa 3 participants 14 messages Oct 26, 2013 – Nov 5, 2013

Subsystems (4)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 ipv4: introduce new IP_MTU_DISCOVER mode IP_PMTUDISC_INTERFACE

Sockets marked with IP_PMTUDISC_INTERFACE won't do path mtu discovery, their sockets won't accept and install new path mtu information and they will always use the interface mtu for outgoing packets. It is guaranteed that the packet is not fragmented locally. But we won't set the DF-Flag on the outgoing frames.

Florian Weimer had the idea to use this flag to ensure DNS servers are never generating outgoing fragments. They may well be fragmented on the path, but the server never stores or usees path mtu values, which could well be forged in an attack.

(The root of the problem with path MTU discovery is that there is no reliable way to authenticate ICMP Fragmentation Needed But DF Set messages because they are sent from intermediate routers with their source addresses, and the IMCP payload will not always contain sufficient information to identify a flow.)

Recent research in the DNS community showed that it is possible to implement an attack where DNS cache poisoning is feasible by spoofing fragments. This work was done by Amir Herzberg and Haya Shulman:
<https://sites.google.com/site/hayashulman/files/fragmentation-poisoning.pdf>

This issue was previously discussed among the DNS community, e.g.
<http://www.ietf.org/mail-archive/web/dnsext/current/msg01204.html>,
without leading to fixes.

This patch depends on the patch "ipv4: fix DO and PROBE pmtu mode regarding local fragmentation with UFO/CORK" for the enforcement of the non-fragmentable checks. If other users than ip_append_page/data should use this semantic too, we have to add a new flag to IPCB(skb)->flags to suppress local fragmentation and check for this in ip_finish_output.

Many thanks to Florian Weimer for the idea and feedback while implementing this patch.

---
 include/net/route.h     | 16 ++++++++++++----
 include/uapi/linux/in.h |  5 +++++
 net/dccp/ipv4.c         |  1 +
 net/ipv4/ip_output.c    |  8 ++++----
 net/ipv4/ip_sockglue.c  |  2 +-
 net/ipv4/route.c        |  4 ++++
 net/ipv4/tcp_ipv4.c     |  1 +
 7 files changed, 28 insertions(+), 9 deletions(-)
include/net/route.h +12 -4
index dd4ae00..f68c167 100644
@@ -313,12 +313,20 @@ static inline int ip4_dst_hoplimit(const struct dst_entry *dst)
 	return hoplimit;
 }
 
-static inline int ip_skb_dst_mtu(struct sk_buff *skb)
+static inline bool ip_sk_accept_pmtu(const struct sock *sk)
 {
-	struct inet_sock *inet = skb->sk ? inet_sk(skb->sk) : NULL;
+	return inet_sk(sk)->pmtudisc != IP_PMTUDISC_INTERFACE;
+}
 
-	return (inet && inet->pmtudisc == IP_PMTUDISC_PROBE) ?
-	       skb_dst(skb)->dev->mtu : dst_mtu(skb_dst(skb));
+static inline bool ip_sk_use_pmtu(const struct sock *sk)
+{
+	return inet_sk(sk)->pmtudisc < IP_PMTUDISC_PROBE;
+}
+
+static inline int ip_skb_dst_mtu(const struct sk_buff *skb)
+{
+	return (!skb->sk || ip_sk_use_pmtu(skb->sk)) ?
+	       dst_mtu(skb_dst(skb)) : skb_dst(skb)->dev->mtu;
 }
 
 #endif	/* _ROUTE_H */
include/uapi/linux/in.h +5 -0
index f9e8e49..393c5de 100644
@@ -115,6 +115,11 @@ struct in_addr {
 #define IP_PMTUDISC_WANT		1	/* Use per route hints	*/
 #define IP_PMTUDISC_DO			2	/* Always DF		*/
 #define IP_PMTUDISC_PROBE		3       /* Ignore dst pmtu      */
+/* Always use interface mtu (ignores dst pmtu) but don't set DF flag.
+ * Also incoming ICMP frag_needed notifications will be ignored on
+ * this socket to prevent accepting spoofed ones.
+ */
+#define IP_PMTUDISC_INTERFACE		4
 
 #define IP_MULTICAST_IF			32
 #define IP_MULTICAST_TTL 		33
net/dccp/ipv4.c +1 -0
index 720c362..d9f65fc 100644
@@ -174,6 +174,7 @@ static inline void dccp_do_pmtu_discovery(struct sock *sk,
 	mtu = dst_mtu(dst);
 
 	if (inet->pmtudisc != IP_PMTUDISC_DONT &&
+	    ip_sk_accept_pmtu(sk) &&
 	    inet_csk(sk)->icsk_pmtu_cookie > mtu) {
 		dccp_sync_mss(sk, mtu);
 
net/ipv4/ip_output.c +4 -4
index 27dc1b3..88c270a 100644
@@ -1037,7 +1037,6 @@ error:
 static int ip_setup_cork(struct sock *sk, struct inet_cork *cork,
 			 struct ipcm_cookie *ipc, struct rtable **rtp)
 {
-	struct inet_sock *inet = inet_sk(sk);
 	struct ip_options_rcu *opt;
 	struct rtable *rt;
 
@@ -1063,8 +1062,8 @@ static int ip_setup_cork(struct sock *sk, struct inet_cork *cork,
 	 * We steal reference to this route, caller should not release it
 	 */
 	*rtp = NULL;
-	cork->fragsize = inet->pmtudisc == IP_PMTUDISC_PROBE ?
-			 rt->dst.dev->mtu : dst_mtu(&rt->dst);
+	cork->fragsize = ip_sk_use_pmtu(sk) ?
+			 dst_mtu(&rt->dst) : rt->dst.dev->mtu;
 	cork->dst = &rt->dst;
 	cork->length = 0;
 	cork->ttl = ipc->ttl;
@@ -1315,7 +1314,8 @@ struct sk_buff *__ip_make_skb(struct sock *sk,
 	/* DF bit is set when we want to see DF on outgoing frames.
 	 * If local_df is set too, we still allow to fragment this frame
 	 * locally. */
-	if (inet->pmtudisc >= IP_PMTUDISC_DO ||
+	if (inet->pmtudisc == IP_PMTUDISC_DO ||
+	    inet->pmtudisc == IP_PMTUDISC_PROBE ||
 	    (skb->len <= dst_mtu(&rt->dst) &&
 	     ip_dont_fragment(sk, &rt->dst)))
 		df = htons(IP_DF);
net/ipv4/ip_sockglue.c +1 -1
index 0626f2c..3f85826 100644
@@ -627,7 +627,7 @@ static int do_ip_setsockopt(struct sock *sk, int level,
 		inet->nodefrag = val ? 1 : 0;
 		break;
 	case IP_MTU_DISCOVER:
-		if (val < IP_PMTUDISC_DONT || val > IP_PMTUDISC_PROBE)
+		if (val < IP_PMTUDISC_DONT || val > IP_PMTUDISC_INTERFACE)
 			goto e_inval;
 		inet->pmtudisc = val;
 		break;
net/ipv4/route.c +4 -0
index d2d3253..f428935 100644
@@ -1036,6 +1036,10 @@ void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
 	bool new = false;
 
 	bh_lock_sock(sk);
+
+	if (!ip_sk_accept_pmtu(sk))
+		goto out;
+
 	rt = (struct rtable *) __sk_dst_get(sk);
 
 	if (sock_owned_by_user(sk) || !rt) {
net/ipv4/tcp_ipv4.c +1 -0
index 300ab2c..14bba8a 100644
@@ -288,6 +288,7 @@ static void tcp_v4_mtu_reduced(struct sock *sk)
 	mtu = dst_mtu(dst);
 
 	if (inet->pmtudisc != IP_PMTUDISC_DONT &&
+	    ip_sk_accept_pmtu(sk) &&
 	    inet_csk(sk)->icsk_pmtu_cookie > mtu) {
 		tcp_sync_mss(sk, mtu);
 
Signature · 1 line
1.8.3.1
maintainer
#1 lore raw

From: Hannes Frederic Sowa <[email protected]>
Date: Sat, 26 Oct 2013 22:11:58 +0200

Quoted 11 lines

Sockets marked with IP_PMTUDISC_INTERFACE won't do path mtu discovery, their sockets won't accept and install new path mtu information and they will always use the interface mtu for outgoing packets. It is guaranteed that the packet is not fragmented locally. But we won't set the DF-Flag on the outgoing frames.

Florian Weimer had the idea to use this flag to ensure DNS servers are never generating outgoing fragments. They may well be fragmented on the path, but the server never stores or usees path mtu values, which could well be forged in an attack.

Quoted 5 lines

(The root of the problem with path MTU discovery is that there is no reliable way to authenticate ICMP Fragmentation Needed But DF Set messages because they are sent from intermediate routers with their source addresses, and the IMCP payload will not always contain sufficient information to identify a flow.)

I do not like this reasoning. You have several more acceptable paths to take to resolve this problem:

1) "I don't trust path MTU information at all"

Just turn it off globally, end of story. It has the same effect as your new per-application mode.

2) "I don't trust path MTU information unless the full socket ID is available in the ICMP packets quoted headers"

Then simply implement a policy as such and submit it to me.

#2 lore raw
Quoted 7 lines · David Miller

I do not like this reasoning. You have several more acceptable paths to take to resolve this problem:

1) "I don't trust path MTU information at all"

Just turn it off globally, end of story. It has the same effect as your new per-application mode.

We can't push this as a security update. We could tell everyone running DNS servers to reconfigure their systems in this way, but I always consider this a bit of a cop-out.

A new knob to turn IP_PMTUDISC_DONT into something that behaves like IP_PMTUDISC_INTERFACE would be more conservative and easier to deploy, I think.

Quoted 4 lines

2) "I don't trust path MTU information unless the full socket ID is available in the ICMP packets quoted headers"

Then simply implement a policy as such and submit it to me.

There are IP protocols where these bits aren't readily available and where we don't want the kernel (outside the Netfilter code) to be aware of the payload structure. Netfilter isn't a solution because it requires state and doesn't work well with request-response UDP protocols like DNS (even before source port randomization).

You could make the path MTU dependent on the protocol (which would even be the correct solution from a technical point of view) and use validation for TCP and UDP, but that's a fairly invasive change for such relatively minor functionality.

Signature · 1 line
Florian Weimer / Red Hat Product Security Team
#4 lore raw
In reply to David Miller
Quoted 14 lines · David Miller

From: Hannes Frederic Sowa <[email protected]>
Date: Sat, 26 Oct 2013 22:11:58 +0200

Quoted 11 lines

Sockets marked with IP_PMTUDISC_INTERFACE won't do path mtu discovery, their sockets won't accept and install new path mtu information and they will always use the interface mtu for outgoing packets. It is guaranteed that the packet is not fragmented locally. But we won't set the DF-Flag on the outgoing frames.

Florian Weimer had the idea to use this flag to ensure DNS servers are never generating outgoing fragments. They may well be fragmented on the path, but the server never stores or usees path mtu values, which could well be forged in an attack.

Quoted 8 lines
Quoted 5 lines

(The root of the problem with path MTU discovery is that there is no reliable way to authenticate ICMP Fragmentation Needed But DF Set messages because they are sent from intermediate routers with their source addresses, and the IMCP payload will not always contain sufficient information to identify a flow.)

I do not like this reasoning. You have several more acceptable paths to take to resolve this problem:

So, I hope I can convince you to merge this patch. ;-)

I really tried hard to find alternatives or even a way to enable the protection automatically given that at least unbound does apply IP_PMTUDISC_DONT to its sockets already. These are the reasons why I came up with the new IP_PMTUDISC_INTERFACE value:

Quoted 4 lines

1) "I don't trust path MTU information at all"

Just turn it off globally, end of story. It has the same effect as your new per-application mode.

I think you are referring to ipv4/ip_no_pmtu_disc?

This setting actually worsens the situation:

0.000 echo 1 > /proc/sys/net/ipv4/ip_no_pmtu_disc 0.500 ethtool -K tun0 ufo off

/* check current mtu with connected socket */ 1.000 socket(..., SOCK_DGRAM, IPPROTO_UDP) = 3 1.000 connect(3, ..., ...) = 0
1.000 getsockopt(3, IPPROTO_IP, IP_MTU, [1500], [4]) = 0 1.000 close(3) = 0

/* inject frag_needed to unconnected socket */
2.000 socket(..., SOCK_DGRAM, IPPROTO_UDP) = 3
2.000 setsockopt(3, IPPROTO_IP, IP_MTU_DISCOVER, [IP_PMTUDISC_DONT], 4) = 0 2.000 sendto(3, ..., 1000, 0, ..., ...) = 1000
2.000 > udp (1000)
/* this simulates a forged icmp packet which tries to lower the mtu */ 2.100 < [udp (1000)] icmp unreachable frag_needed mtu 1400

2.5 sendto(3, ..., 1000, 0, ..., ...) = 1000
/* first fragment splitted at 522 bytes */
2.5 > udp(522)
/* didn't find a way to match for the second fragment - so error out here
* comment out this check to check for the mtu a IP_PMTUDISC_DONT socket now has *
* it really is used, otherwise we would not have seen this fragment */

3.000 close(3) = 0

/* verify mtu again */
3.000 socket(..., SOCK_DGRAM, IPPROTO_UDP) = 3
3.000 connect(3, ..., ...) = 0
/* we actually have 522 as the minimal mtu - so error out here */ 3.000 getsockopt(3, IPPROTO_IP, IP_MTU, [1400], [4]) = 0 3.000 close(3) = 0

5.000 echo 0 > /proc/sys/net/ipv4/ip_no_pmtu_disc

Given this packetdrill snippet we see it is actually very counterproductive to tell people to globally disable pmtu discovery as it will certainly generate more fragments.

The reason for this is, that __ip_rt_update_pmtu is called with zero mtu and will get clamped to min_pmtu. It is possible to tell people to also increase min_pmtu but I don't want to take the risk of breaking other applications on a DNS server given how the current semantics are.

If we receive a pmtu event on a TCP or DCCP socket we actually check for the pmtudisc setting and won't apply it if it is set to IP_PMTUDISC_DONT. This is not checked on UDP and RAW sockets and it was very hard to think about the consequences to introduce such checks now (at least for me).

Adding a new sysctl would be a viable option but it is harder to use by DNS server implementations and could be fiddled with by users or scripts etc. I dislike a globel (or even per namespace) setting. Also Fedora e.g. installs an unbound server on Desktops if one installs the dnssec-trigger package. Maybe we will see more DNS servers on workstations where we don't want to globally disable pmtu.

Quoted 4 lines

2) "I don't trust path MTU information unless the full socket ID is available in the ICMP packets quoted headers"

Then simply implement a policy as such and submit it to me.

We don't trust the MTU information in any case for that socket.

It seems easy to inject wrong path mtu information on a socket which will be installed as a globel fnhe for 600 seconds. This is a very viable target if an attacker is able to do cache poisoning afterwards. There is not much protection even if we match the full socket ID on an UDP socket. As this socket is more exposed to attacks we drop the mtu information there. Of course, other ports could also install wrong pmtu information causing packets being fragmented so we always use the path mtu and don't fragment locally. Dropping the pmtu information on these sockets just makes it more symmetrical and causes less stress on the fnhe hash tables if people do these attacks (they could well be running these attacks in parallel). If the DNS port is the only one reachable externally we won't do any pmtu processing on that server from untrusted peers, which is a nice side effect.

All that said, I hope dns servers provide a setting to apply this option on sockets and fail to start up if it could not be applied. So an administrator is not in some sense of false hope that this protection is enabled when running on an older kernel. This could only be done if we provide a new value for IP_MTU_DISCOVER.

Greetings,

Hannes

#5 lore raw
Quoted 4 lines · Hannes Frederic Sowa

The reason for this is, that __ip_rt_update_pmtu is called with zero mtu and will get clamped to min_pmtu. It is possible to tell people to also increase min_pmtu but I don't want to take the risk of breaking other applications on a DNS server given how the current semantics are.

It is actually difficult to select a reasonable mtu here given more than two interfaces with different mtus.

Greetings,

Hannes

maintainer
#3 lore raw
In reply to Florian Weimer

From: Florian Weimer <[email protected]>
Date: Tue, 29 Oct 2013 08:36:35 +0100

Quoted 4 lines

You could make the path MTU dependent on the protocol (which would even be the correct solution from a technical point of view) and use validation for TCP and UDP, but that's a fairly invasive change for such relatively minor functionality.

The code pretty much already does this, we cannot even find the exact route without being able to look up the socket right now. We need the full flow info.

There is a backup path, but you could make the new (non-default) behavior be to never trust that path for protocols like UDP and TCP.

I don't buy any of your arguments against my suggestions so far.

#6 lore raw
In reply to Hannes Frederic Sowa

Hi!

Quoted 4 lines · Hannes Frederic Sowa

I really tried hard to find alternatives or even a way to enable the protection automatically given that at least unbound does apply IP_PMTUDISC_DONT to its sockets already. These are the reasons why I came up with the new IP_PMTUDISC_INTERFACE value:

Sorry to bother you but I would really love to hear your feedback on my reasoning so I can try to come up with a solution you would be happy with.

Thank you,

Hannes

maintainer
#7 lore raw

From: Hannes Frederic Sowa <[email protected]>
Date: Wed, 30 Oct 2013 21:07:25 +0100

Quoted 8 lines · Hannes Frederic Sowa
Quoted 4 lines

I really tried hard to find alternatives or even a way to enable the protection automatically given that at least unbound does apply IP_PMTUDISC_DONT to its sockets already. These are the reasons why I came up with the new IP_PMTUDISC_INTERFACE value:

Sorry to bother you but I would really love to hear your feedback on my reasoning so I can try to come up with a solution you would be happy with.

All I've read is that administrators cannot be relied upon to configure their systems properly for the requirements they have.

And that is a non-argument for adding this new socket option as far as I'm concerned.

"I strongly do not trust path MTU information" has a scope as small as a route or an interface, it doesn't go down to the socket or application level at all.

Please stop pretending that it does.

#8 lore raw
Quoted 11 lines · David Miller

From: Hannes Frederic Sowa <[email protected]>
Date: Wed, 30 Oct 2013 21:07:25 +0100

Quoted 8 lines · Hannes Frederic Sowa
Quoted 4 lines

I really tried hard to find alternatives or even a way to enable the protection automatically given that at least unbound does apply IP_PMTUDISC_DONT to its sockets already. These are the reasons why I came up with the new IP_PMTUDISC_INTERFACE value:

Sorry to bother you but I would really love to hear your feedback on my reasoning so I can try to come up with a solution you would be happy with.

Thanks for the answer but I still tend to disagree:

Quoted 5 lines

All I've read is that administrators cannot be relied upon to configure their systems properly for the requirements they have.

And that is a non-argument for adding this new socket option as far as I'm concerned.

First of, it is pretty hard to do it correct. Before writing this patch I thought that ip_no_pmtu_disc does not honour pmtu updates at all. But as soon as one fragmentation needed ICMP enters the box the mtu is reduced to pmin_mtu. This was very counterintuitive for me (I don't know if this is actually intended). Also we honour per application settings for TCP and DCCP sockets, but I totally understand if you take this as a non-argument.

But that is not that important. Documentation can fix this.

Quoted 3 lines

"I strongly do not trust path MTU information" has a scope as small as a route or an interface, it doesn't go down to the socket or application level at all.

I still disagree here. We can prevent generating UDP fragments if we lock the mtu on the route, I agree and somehow missed that before (even though I already used that myself once).

DNS resolver can fallback to TCP for querying where we can honour the path MTU because it won't do any harm and ensures connectivity.

Also for TCP the socket is matched on the whole 4-tuple and we may fallback to a 2-tuple lookup on unconnected UDP sockets. The new socket option would let an application programmer choose to do path mtu discovery because it knows it will only use a connected socket. On unconnected sockets one can specify IP_PMTUDISC_INTERFACE to suppress the path MTU updates and always use the interface MTU without the DF-bit set.

I really am open to suggestions. The socket option could be renamed or we can transform IP_PMTUDISC_DONT to what I think it actually should do. Per route or interface settings to enable the acceptance of path MTU updates per protocol would be ok, too, but more complex.

Greetings,

Hannes

maintainer
#9 lore raw

From: Hannes Frederic Sowa <[email protected]>
Date: Wed, 30 Oct 2013 23:58:51 +0100

Quoted 9 lines

DNS resolver can fallback to TCP for querying where we can honour the path MTU because it won't do any harm and ensures connectivity.

Also for TCP the socket is matched on the whole 4-tuple and we may fallback to a 2-tuple lookup on unconnected UDP sockets. The new socket option would let an application programmer choose to do path mtu discovery because it knows it will only use a connected socket. On unconnected sockets one can specify IP_PMTUDISC_INTERFACE to suppress the path MTU updates and always use the interface MTU without the DF-bit set.

This still sounds like a route scope policy with two binary states, one for connected sockets and one for unconnected ones.

#10 lore raw
Quoted 13 lines · David Miller

From: Hannes Frederic Sowa <[email protected]>
Date: Wed, 30 Oct 2013 23:58:51 +0100

Quoted 9 lines

DNS resolver can fallback to TCP for querying where we can honour the path MTU because it won't do any harm and ensures connectivity.

Also for TCP the socket is matched on the whole 4-tuple and we may fallback to a 2-tuple lookup on unconnected UDP sockets. The new socket option would let an application programmer choose to do path mtu discovery because it knows it will only use a connected socket. On unconnected sockets one can specify IP_PMTUDISC_INTERFACE to suppress the path MTU updates and always use the interface MTU without the DF-bit set.

Quoted 2 lines

This still sounds like a route scope policy with two binary states, one for connected sockets and one for unconnected ones.

Orthogonally a user may want to decide per protocol. Path MTU discovery may happen on protocols where I have protection against fragment poisoning because of the TCP 3-WHS and want to drop path mtu discovery information on simple UDP request-response protocols even if they are used in connected state (this is the specific case I want to protect in DNS).

Do we accept path MTU information if we just send a spoofed TCP SYN request and then just fire a spoofed ICMP path mtu packet afterwards? I guess (I really have not checked), yes. If the UDP stack would use this information we again generate fragments and are vulnerable to UDP based cache poisoning attacks. So when can we really say we can match and trust the full socket ID?

Please note, dropping the path MTU on those sockets is merely an optimization. All I care about is that we don't fragment packets locally and have the possibility to not set the DF-bit (so IP_PMTUDISC_PROBE without DF). I could very well remove the ip_sk_accept_pmtu logic from the patch.

I still think this patch is the reasonable way to solve this problem. If you still think it should be on a per-route basis I will look down on how this can be achieved but would focus on multiplexing the MTU per protocol somehow.

Please let me know!

Thank you,

Hannes

#11 lore raw

Hi David!

Quoted 36 lines · Hannes Frederic Sowa
Quoted 13 lines · David Miller

From: Hannes Frederic Sowa <[email protected]>
Date: Wed, 30 Oct 2013 23:58:51 +0100

Quoted 9 lines

DNS resolver can fallback to TCP for querying where we can honour the path MTU because it won't do any harm and ensures connectivity.

Also for TCP the socket is matched on the whole 4-tuple and we may fallback to a 2-tuple lookup on unconnected UDP sockets. The new socket option would let an application programmer choose to do path mtu discovery because it knows it will only use a connected socket. On unconnected sockets one can specify IP_PMTUDISC_INTERFACE to suppress the path MTU updates and always use the interface MTU without the DF-bit set.

Quoted 2 lines

This still sounds like a route scope policy with two binary states, one for connected sockets and one for unconnected ones.

Orthogonally a user may want to decide per protocol. Path MTU discovery may happen on protocols where I have protection against fragment poisoning because of the TCP 3-WHS and want to drop path mtu discovery information on simple UDP request-response protocols even if they are used in connected state (this is the specific case I want to protect in DNS).

Do we accept path MTU information if we just send a spoofed TCP SYN request and then just fire a spoofed ICMP path mtu packet afterwards? I guess (I really have not checked), yes. If the UDP stack would use this information we again generate fragments and are vulnerable to UDP based cache poisoning attacks. So when can we really say we can match and trust the full socket ID?

Please note, dropping the path MTU on those sockets is merely an optimization. All I care about is that we don't fragment packets locally and have the possibility to not set the DF-bit (so IP_PMTUDISC_PROBE without DF). I could very well remove the ip_sk_accept_pmtu logic from the patch.

Quoted 4 lines

I still think this patch is the reasonable way to solve this problem. If you still think it should be on a per-route basis I will look down on how this can be achieved but would focus on multiplexing the MTU per protocol somehow.

Sorry for being so pushy about this patch! :|

I did a proof of concept of a per-route setting to suppress local datagram fragmentation. It does work and would solve the problem. I did not find a clean way to deal with sockets already using IP_MTU_DISCOVER, so I just overwrite the setting and fallback to the safe mode and don't allow local fragmentation (but I don't touch the DF-flag). This could have implications to applications which already set a policy for their sockets and expect a certain behaviour (e.g. traceroute in some circumstances). I don't like the approach, but I would like to see this problem solved. If you would accept this patch I would retest and do a clean submission (with the iproute2 bits) then. UI currently looks like this:

# ip r change default via 10.0.0.254 dgram_dont_local_frag 1

After all, I still think this should be a per-application/per-socket setting because the application must deal with the fact that no local fragmentation is possible so it has to be more sensitive to EMSGSIZE errors. DNS has to negotiate the UDP buffer size with its peers already (with the help of EDNS), so there is already logic to deal with the size of fragments and so it should not be done transparently IMHO.

With the per-route setting DNS implementations can work around this by querying the IP_MTU or handle the socket errors but I don't know about other UDP applications on the same host. Everything is ok as long as the DNS server is the only application on that system.

This is not the same as e.g. TCP_QUICKACK which can transparently be dealt with by applications and thus makes sense to be a per-route setting. Applications will see different behaviour when using dgram_dont_frag. This really is no tuning parameter.

I also tried per-protocol path MTUs and abandoned the patch earlier today. It got too complex and had problems with how to emulate a common mtu for old iproutes.

Also, I would be happy to support this patch and the per-socket approach.

I would love to hear your feedback.

include/net/route.h +13 -2
index dd4ae00..3f6b04b 100644
@@ -313,12 +313,23 @@ static inline int ip4_dst_hoplimit(const struct dst_entry *dst)
 	return hoplimit;
 }
 
+static inline bool ip_dgram_local_fragment(const struct dst_entry *dst,
+				      const struct sock *sk)
+{
+	if (sk->sk_protocol == IPPROTO_UDP &&
+	    dst_metric(dst, RTAX_DGRAM_DONT_LOCAL_FRAG))
+		return false;
+	return true;
+}
+
 static inline int ip_skb_dst_mtu(struct sk_buff *skb)
 {
 	struct inet_sock *inet = skb->sk ? inet_sk(skb->sk) : NULL;
+	struct dst_entry *dst = skb_dst(skb);
 
-	return (inet && inet->pmtudisc == IP_PMTUDISC_PROBE) ?
-	       skb_dst(skb)->dev->mtu : dst_mtu(skb_dst(skb));
+	return (inet && (inet->pmtudisc == IP_PMTUDISC_PROBE ||
+			 !ip_dgram_local_fragment(dst, skb->sk))) ?
+	       dst->dev->mtu : dst_mtu(dst);
 }
 
 #endif	/* _ROUTE_H */
include/uapi/linux/rtnetlink.h +2 -0
index eb0f1a5..2e450f3 100644
@@ -388,6 +388,8 @@ enum {
 #define RTAX_INITRWND RTAX_INITRWND
 	RTAX_QUICKACK,
 #define RTAX_QUICKACK RTAX_QUICKACK
+	RTAX_DGRAM_DONT_LOCAL_FRAG,
+#define RTAX_DGRAM_DONT_LOCAL_FRAG RTAX_DGRAM_DONT_LOCAL_FRAG
 	__RTAX_MAX
 };
 
net/ipv4/ip_output.c +9 -5
index 51be64e..661c618 100644
@@ -823,7 +823,8 @@ static int __ip_append_data(struct sock *sk,
 
 	fragheaderlen = sizeof(struct iphdr) + (opt ? opt->optlen : 0);
 	maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen;
-	maxnonfragsize = (inet->pmtudisc >= IP_PMTUDISC_DO) ?
+	maxnonfragsize = ((inet->pmtudisc >= IP_PMTUDISC_DO) ||
+			  !ip_dgram_local_fragment(&rt->dst, sk)) ?
 			 mtu : 0xFFFF;
 
 	if (cork->length + length > maxnonfragsize - fragheaderlen) {
@@ -1063,7 +1064,8 @@ static int ip_setup_cork(struct sock *sk, struct inet_cork *cork,
 	 * We steal reference to this route, caller should not release it
 	 */
 	*rtp = NULL;
-	cork->fragsize = inet->pmtudisc == IP_PMTUDISC_PROBE ?
+	cork->fragsize = ((inet->pmtudisc == IP_PMTUDISC_PROBE) ||
+			  !ip_dgram_local_fragment(&rt->dst, sk)) ?
 			 rt->dst.dev->mtu : dst_mtu(&rt->dst);
 	cork->dst = &rt->dst;
 	cork->length = 0;
@@ -1148,8 +1150,9 @@ ssize_t	ip_append_page(struct sock *sk, struct flowi4 *fl4, struct page *page,
 
 	fragheaderlen = sizeof(struct iphdr) + (opt ? opt->optlen : 0);
 	maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen;
-	maxnonfragsize = (inet->pmtudisc >= IP_PMTUDISC_DO) ?
-			 mtu : 0xFFFF;
+	maxnonfragsize = ((inet->pmtudisc >= IP_PMTUDISC_DO) ||
+			  !ip_dgram_local_fragment(&rt->dst, sk)) ?
+		         mtu : 0xFFFF;
 
 	if (cork->length + size > maxnonfragsize - fragheaderlen) {
 		ip_local_error(sk, EMSGSIZE, fl4->daddr, inet->inet_dport, mtu);
@@ -1309,7 +1312,8 @@ struct sk_buff *__ip_make_skb(struct sock *sk,
 	 * to fragment the frame generated here. No matter, what transforms
 	 * how transforms change size of the packet, it will come out.
 	 */
-	if (inet->pmtudisc < IP_PMTUDISC_DO)
+	if (inet->pmtudisc < IP_PMTUDISC_DO &&
+	    ip_dgram_local_fragment(&rt->dst, sk))
 		skb->local_df = 1;
 
 	/* DF bit is set when we want to see DF on outgoing frames.

Thank you,

Hannes

maintainer
#12 lore raw

From: Hannes Frederic Sowa <[email protected]>
Date: Tue, 5 Nov 2013 00:25:41 +0100

Quoted 1 line

Sorry for being so pushy about this patch! :|

I haven't ruled out applying your original patch, could you please just repost it so that it gets queued up anew in patchwork?

I'll get to resolving this, please be patient.

Thanks.

#13 lore raw
Quoted 7 lines · David Miller

From: Hannes Frederic Sowa <[email protected]>
Date: Tue, 5 Nov 2013 00:25:41 +0100

Quoted 1 line

Sorry for being so pushy about this patch! :|

I haven't ruled out applying your original patch, could you please just repost it so that it gets queued up anew in patchwork?

Ok, will do, thanks!

Quoted 1 line

I'll get to resolving this, please be patient.

Sorry, I did not want to rush you.

Greetings,

Hannes