Skip to contentSkip to conversation

: FreeBSD DCO support

PATCH In review Kristof Provost 3 participants 22 messages Aug 12, 2022 – Oct 17, 2022

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 : FreeBSD DCO support

Hi,

Here's the updated version of the FreeBSD DCO patch. It should address all remarks on the userspace side of things.

I'm still trying to reproduce the fragmentation issue. I've gotten Gert's additional information, but I've not yet been able to act on it. I'm fairly certain that any bug is going to be on the kernel side, so it makes sense to me to continue these changes while I dig into that.

Best regards,
Kristof

#1 lore raw

PATCH 1/2 ovpn-dco: introduce FreeBSD data-channel offload support

From: Kristof Provost <[email protected]...>

Implement data-channel offload for FreeBSD. The implementation and flow is very similar to that of the Linux DCO support.

---
 configure.ac                   |   5 +
 src/openvpn/Makefile.am        |   1 +
 src/openvpn/dco.c              |   8 +
 src/openvpn/dco_freebsd.c      | 645 +++++++++++++++++++++++++++++++++
 src/openvpn/dco_freebsd.h      |  59 +++
 src/openvpn/dco_internal.h     |   1 +
 src/openvpn/forward.c          |   8 +-
 src/openvpn/mtcp.c             |   4 +-
 src/openvpn/mudp.c             |   2 +-
 src/openvpn/multi.c            |   4 +-
 src/openvpn/options.c          |   8 +-
 src/openvpn/options.h          |   4 +-
 src/openvpn/ovpn_dco_freebsd.h |  64 ++++
 src/openvpn/tun.c              |  39 +-
 src/openvpn/tun.h              |   6 +
 15 files changed, 827 insertions(+), 31 deletions(-)
 create mode 100644 src/openvpn/dco_freebsd.c
 create mode 100644 src/openvpn/dco_freebsd.h
 create mode 100644 src/openvpn/ovpn_dco_freebsd.h
configure.ac +5 -0
index 9466fe15..f715b404 100644
@@ -787,6 +787,11 @@ dnl
 			AC_DEFINE(ENABLE_DCO, 1, [Enable shared data channel offload])
 			AC_MSG_NOTICE([Enabled ovpn-dco support for Linux])
 			;;
+		*-*-freebsd*)
+			LIBS="${LIBS} -lnv"
+			AC_DEFINE(ENABLE_DCO, 1, [Enable data channel offload for FreeBSD])
+			AC_MSG_NOTICE([Enabled ovpn-dco support for FreeBSD])
+			;;
 		*)
 			AC_MSG_NOTICE([Ignoring --enable-dco on non Linux platform])
 			;;
src/openvpn/Makefile.am +1 -0
index aaa1dbce..2a139b23 100644
@@ -54,6 +54,7 @@ openvpn_SOURCES = \
 	crypto_openssl.c crypto_openssl.h \
 	crypto_mbedtls.c crypto_mbedtls.h \
 	dco.c dco.h dco_internal.h \
+	dco_freebsd.c dco_freebsd.h \
 	dco_linux.c dco_linux.h \
 	dhcp.c dhcp.h \
 	dns.c dns.h \
src/openvpn/dco.c +8 -0
index 4f40255e..07dc1087 100644
@@ -271,6 +271,14 @@ dco_check_option_conflict_ce(const struct connection_entry *ce, int msglevel)
         return false;
     }
 
+#if defined(TARGET_FREEBSD)
+    if (! proto_is_udp(ce->proto))
+    {
+        msg(msglevel, "TCP is not supported.");
+        return false;
+    }
+#endif
+
     return true;
 }
 
src/openvpn/dco_freebsd.c new file +645 -0

645 lines not on this page · raw message

src/openvpn/dco_freebsd.h new file +59 -0

59 lines not on this page · raw message

src/openvpn/dco_internal.h +1 -0
index 3ceb26d6..728e3092 100644
@@ -27,6 +27,7 @@
 
 #if defined(ENABLE_DCO)
 
+#include "dco_freebsd.h"
 #include "dco_linux.h"
 
 /**
src/openvpn/forward.c +4 -4
index 55c939c4..14ad24fa 100644
@@ -1113,7 +1113,7 @@ process_incoming_link(struct context *c)
 static void
 process_incoming_dco(struct context *c)
 {
-#if defined(ENABLE_DCO) && defined(TARGET_LINUX)
+#if defined(ENABLE_DCO) && (defined(TARGET_LINUX) || defined(TARGET_FREEBSD))
     struct link_socket_info *lsi = get_link_socket_info(c);
     dco_context_t *dco = &c->c1.tuntap->dco;
 
@@ -1140,7 +1140,7 @@ process_incoming_dco(struct context *c)
 
     c->c2.buf = orig_buff;
     buf_init(&dco->dco_packet_in, 0);
-#endif /* if defined(ENABLE_DCO) && defined(TARGET_LINUX) */
+#endif /* if defined(ENABLE_DCO) && (defined(TARGET_LINUX) || defined(TARGET_FREEBSD)) */
 }
 
 /*
@@ -1946,7 +1946,7 @@ io_wait_dowork(struct context *c, const unsigned int flags)
 #ifdef ENABLE_ASYNC_PUSH
     static int file_shift = FILE_SHIFT;
 #endif
-#ifdef TARGET_LINUX
+#if defined(TARGET_LINUX) || defined(TARGET_FREEBSD)
     static int dco_shift = DCO_SHIFT;    /* Event from DCO linux kernel module */
 #endif
 
@@ -2056,7 +2056,7 @@ io_wait_dowork(struct context *c, const unsigned int flags)
      */
     socket_set(c->c2.link_socket, c->c2.event_set, socket, (void *)&socket_shift, NULL);
     tun_set(c->c1.tuntap, c->c2.event_set, tuntap, (void *)&tun_shift, NULL);
-#if defined(TARGET_LINUX)
+#if defined(TARGET_LINUX) || defined(TARGET_FREEBSD)
     if (socket & EVENT_READ && c->c2.did_open_tun)
     {
         dco_event_set(&c->c1.tuntap->dco, c->c2.event_set, (void *)&dco_shift);
src/openvpn/mtcp.c +2 -2
index eb88a56a..1abb903f 100644
@@ -283,7 +283,7 @@ multi_tcp_wait(const struct context *c,
     }
 #endif
     tun_set(c->c1.tuntap, mtcp->es, EVENT_READ, MTCP_TUN, persistent);
-#if defined(TARGET_LINUX)
+#if defined(TARGET_LINUX) || defined(TARGET_FREEBSD)
     dco_event_set(&c->c1.tuntap->dco, mtcp->es, MTCP_DCO);
 #endif
 
@@ -763,7 +763,7 @@ multi_tcp_process_io(struct multi_context *m)
                     multi_tcp_action(m, mi, TA_INITIAL, false);
                 }
             }
-#if defined(ENABLE_DCO) && defined(TARGET_LINUX)
+#if defined(ENABLE_DCO) && (defined(TARGET_LINUX) || defined(TARGET_FREEBSD))
             /* incoming data on DCO? */
             else if (e->arg == MTCP_DCO)
             {
src/openvpn/mudp.c +1 -1
index ddb1efc9..4ab18b72 100644
@@ -381,7 +381,7 @@ multi_process_io_udp(struct multi_context *m)
         multi_process_file_closed(m, mpp_flags);
     }
 #endif
-#if defined(ENABLE_DCO) && defined(TARGET_LINUX)
+#if defined(ENABLE_DCO) && (defined(TARGET_LINUX) || defined(TARGET_FREEBSD))
     else if (status & DCO_READ)
     {
         if (!IS_SIG(&m->top))
src/openvpn/multi.c +2 -2
index dcf4438d..53ee3e1a 100644
@@ -3154,7 +3154,7 @@ multi_close_instance_on_signal(struct multi_context *m, struct multi_instance *m
     multi_close_instance(m, mi, false);
 }
 
-#if (defined(ENABLE_DCO) && defined(TARGET_LINUX)) || defined(ENABLE_MANAGEMENT)
+#if (defined(ENABLE_DCO) && (defined(TARGET_LINUX) || defined(TARGET_FREEBSD))) || defined(ENABLE_MANAGEMENT)
 static void
 multi_signal_instance(struct multi_context *m, struct multi_instance *mi, const int sig)
 {
@@ -3163,7 +3163,7 @@ multi_signal_instance(struct multi_context *m, struct multi_instance *mi, const
 }
 #endif
 
-#if defined(ENABLE_DCO) && defined(TARGET_LINUX)
+#if defined(ENABLE_DCO) && (defined(TARGET_LINUX) || defined(TARGET_FREEBSD))
 static void
 process_incoming_dco_packet(struct multi_context *m, struct multi_instance *mi,
                             dco_context_t *dco)
src/openvpn/options.c +4 -4
index 0ce3158b..14cb4cc4 100644
@@ -183,7 +183,7 @@ static const char usage_message[] =
     "                  does not begin with \"tun\" or \"tap\".\n"
     "--dev-node node : Explicitly set the device node rather than using\n"
     "                  /dev/net/tun, /dev/tun, /dev/tap, etc.\n"
-#if defined(ENABLE_DCO) && defined(TARGET_LINUX)
+#if defined(ENABLE_DCO) && (defined(TARGET_LINUX) || defined(TARGET_FREEBSD))
     "--disable-dco   : Do not attempt using Data Channel Offload.\n"
 #endif
     "--lladdr hw     : Set the link layer address of the tap device.\n"
@@ -1794,7 +1794,7 @@ show_settings(const struct options *o)
     SHOW_STR(dev);
     SHOW_STR(dev_type);
     SHOW_STR(dev_node);
-#if defined(ENABLE_DCO) && defined(TARGET_LINUX)
+#if defined(ENABLE_DCO) && (defined(TARGET_LINUX) || defined(TARGET_FREEBSD))
     SHOW_BOOL(tuntap_options.disable_dco);
 #endif
     SHOW_STR(lladdr);
@@ -3670,7 +3670,7 @@ options_postprocess_mutate(struct options *o, struct env_set *es)
     }
 
     /* check if any option should force disabling DCO */
-#if defined(TARGET_LINUX)
+#if defined(TARGET_LINUX) || defined(TARGET_FREEBSD)
     o->tuntap_options.disable_dco = !dco_check_option_conflict(D_DCO, o);
 #endif
 
@@ -5872,7 +5872,7 @@ add_option(struct options *options,
 #endif
     else if (streq(p[0], "disable-dco"))
     {
-#if defined(TARGET_LINUX)
+#if defined(TARGET_LINUX) || defined(TARGET_FREEBSD)
         options->tuntap_options.disable_dco = true;
 #endif
     }
src/openvpn/options.h +2 -2
index ec3c44b1..212f4b05 100644
@@ -876,7 +876,7 @@ void options_string_import(struct options *options,
 
 bool key_is_external(const struct options *options);
 
-#if defined(ENABLE_DCO) && defined(TARGET_LINUX)
+#if defined(ENABLE_DCO) && (defined(TARGET_LINUX) || defined(TARGET_FREEBSD))
 
 /**
  * Returns whether the current configuration has dco enabled.
@@ -887,7 +887,7 @@ dco_enabled(const struct options *o)
     return !o->tuntap_options.disable_dco;
 }
 
-#else /* if defined(ENABLE_DCO) && defined(TARGET_LINUX) */
+#else /* if defined(ENABLE_DCO) && (defined(TARGET_LINUX) || defined(TARGET_FREEBSD))*/
 
 static inline bool
 dco_enabled(const struct options *o)
src/openvpn/ovpn_dco_freebsd.h new file +64 -0

64 lines not on this page · raw message

src/openvpn/tun.c +23 -16
index f3152a52..11025267 100644
@@ -1722,7 +1722,7 @@ tun_name_is_fixed(const char *dev)
     return has_digit(dev);
 }
 
-#if defined(TARGET_LINUX)
+#if defined(TARGET_LINUX) || defined(TARGET_FREEBSD)
 static bool
 tun_dco_enabled(struct tuntap *tt)
 {
@@ -1836,9 +1836,9 @@ open_tun_generic(const char *dev, const char *dev_type, const char *dev_node,
         tt->actual_name = string_alloc(dynamic_opened ? dynamic_name : dev, NULL);
     }
 }
-#endif /* !_WIN32 && !TARGET_LINUX */
+#endif /* !_WIN32 && !TARGET_LINUX && !TARGET_FREEBSD*/
 
-#if defined(TARGET_LINUX)
+#if defined(TARGET_LINUX) || defined(TARGET_FREEBSD)
 static void
 open_tun_dco_generic(const char *dev, const char *dev_type,
                      struct tuntap *tt, openvpn_net_ctx_t *ctx)
@@ -1911,7 +1911,7 @@ open_tun_dco_generic(const char *dev, const char *dev_type,
         tt->actual_name = string_alloc(dev, NULL);
     }
 }
-#endif /* TARGET_LINUX */
+#endif /* TARGET_LINUX || TARGET_FREEBSD*/
 
 #if !defined(_WIN32)
 static void
@@ -2294,7 +2294,7 @@ close_tun(struct tuntap *tt, openvpn_net_ctx_t *ctx)
         net_ctx_reset(ctx);
     }
 
-#ifdef TARGET_LINUX
+#if defined(TARGET_LINUX) || defined(TARGET_FREEBSD)
     if (tun_dco_enabled(tt))
     {
         close_tun_dco(tt, ctx);
@@ -2915,20 +2915,27 @@ void
 open_tun(const char *dev, const char *dev_type, const char *dev_node, struct tuntap *tt,
          openvpn_net_ctx_t *ctx)
 {
-    open_tun_generic(dev, dev_type, dev_node, tt);
-
-    if (tt->fd >= 0 && tt->type == DEV_TYPE_TUN)
+    if (tun_dco_enabled(tt))
     {
-        int i = IFF_POINTOPOINT | IFF_MULTICAST;
+        open_tun_dco_generic(dev, dev_type, tt, ctx);
+    }
+    else
+    {
+        open_tun_generic(dev, dev_type, dev_node, tt);
 
-        if (ioctl(tt->fd, TUNSIFMODE, &i) < 0)
+        if (tt->fd >= 0 && tt->type == DEV_TYPE_TUN)
         {
-            msg(M_WARN | M_ERRNO, "ioctl(TUNSIFMODE)");
-        }
-        i = 1;
-        if (ioctl(tt->fd, TUNSIFHEAD, &i) < 0)
-        {
-            msg(M_WARN | M_ERRNO, "ioctl(TUNSIFHEAD)");
+            int i = IFF_POINTOPOINT | IFF_MULTICAST;
+
+            if (ioctl(tt->fd, TUNSIFMODE, &i) < 0)
+            {
+                msg(M_WARN | M_ERRNO, "ioctl(TUNSIFMODE)");
+            }
+            i = 1;
+            if (ioctl(tt->fd, TUNSIFHEAD, &i) < 0)
+            {
+                msg(M_WARN | M_ERRNO, "ioctl(TUNSIFHEAD)");
+            }
         }
     }
 }
src/openvpn/tun.h +6 -0
index 8ec8f51f..ea4946e9 100644
@@ -142,6 +142,12 @@ struct tuntap_options {
     bool disable_dco;
 };
 
+#elif defined(TARGET_FREEBSD)
+
+struct tuntap_options {
+    bool disable_dco;
+};
+
 #else  /* if defined(_WIN32) || defined(TARGET_ANDROID) */
 
 struct tuntap_options {
Signature · 1 line
2.37.1
Acked-by on v1 1/2
#2 lore raw

[Openvpn-devel] [PATCH applied] Re: ovpn-dco: introduce FreeBSD data-channel offload support

Stared at the code, stared at the diff, the changes are what I asked for (thanks :-) ). I'm sure we'll find more stuff to polish, but I want this to proceed so the merge conflict with dco-win can be fixed by rebasing that other tree... (which is needed anyway).

Uncrustify complained about a few tab-vs-space things, which I adjusted (mostly in ovpn_dco_freebsd.h).

I have also adjusted the "TCP is bah" message to be more in line with the other "does not work with DCO" messages:

+ msg(msglevel, "NOTE: TCP transport disables data channel offload on FreeBSD.");

(and indeed, this is what it does -> tests 1* succeed now)

I have tested this on Linux and FreeBSD "without DCO" (full client and server test, though there is no actual new code that would be compiled for Linux or for non-DCO FreeBSD), Linux "with DCO" (works), and FreeBSD 14 with DCO enabled, which looks good, besides the "double fragment fails" issue - which is not a userland thing.

So far I have only tested the client side (p2p), the server side needs the iroute patch in 2/2 for full test coverage - "soon".

Your patch has been applied to the master branch.

commit f08fcc2f1eb15941292d6e4e520642a4e474fd1e

Author: Kristof Provost
Date:   Fri Aug 12 15:41:53 2022 +0200
     ovpn-dco: introduce FreeBSD data-channel offload support

     Signed-off-by: Kristof Provost <[email protected]...>
     Acked-by: Gert Doering <[email protected]...>
     Message-Id: <[email protected]...>
     URL: https://www.mail-archive.com/[email protected]/msg24894.html
     Signed-off-by: Gert Doering <[email protected]...>

--
kind regards,

Gert Doering

#3 lore raw

Re: [Openvpn-devel] [PATCH applied] Re: ovpn-dco: introduce FreeBSD data-channel offload support

Hi,

Quoted 2 lines · Gert Doering

Uncrustify complained about a few tab-vs-space things, which I adjusted (mostly in ovpn_dco_freebsd.h).

And promptly forgot to do "git commit --amend" on *both* files. So here comes a whitespace correction commit again...

commit 702a4a2c237842bb4adef5de98d82746e5715f78 (HEAD -> master)

Author: Gert Doering <[email protected]...>
Date:   Sat Aug 13 14:44:38 2022 +0200

Apply uncrustify changes that were forgotten in the FreeBSD DCO 1/2 patch.

*sigh*

I need to be away from the keyboard now, and bake a cake for a change :-)

gert

Signature · 6 lines
"If was one thing all people took for granted, was conviction that if you 
 feed honest figures into a computer, honest figures come out. Never doubted 
 it myself till I met a computer with a sense of humor."
                             Robert A. Heinlein, The Moon is a Harsh Mistress

Gert Doering - Munich, Germany                             [email protected]...
#4 lore raw

PATCH 2/2 Support creating iroute route entries on FreeBSD

In reply to Kristof Provost

From: Kristof Provost <[email protected]...>

---
 src/openvpn/Makefile.am          |   1 +
 src/openvpn/dco.c                |   8 +--
 src/openvpn/dco_freebsd.h        |   2 +
 src/openvpn/networking.h         |   9 +++
 src/openvpn/networking_freebsd.c | 101 +++++++++++++++++++++++++++++++
 5 files changed, 117 insertions(+), 4 deletions(-)
 create mode 100644 src/openvpn/networking_freebsd.c
src/openvpn/Makefile.am +1 -0
index 2a139b23..5155a180 100644
@@ -88,6 +88,7 @@ openvpn_SOURCES = \
 	mtu.c mtu.h \
 	mudp.c mudp.h \
 	multi.c multi.h \
+	networking_freebsd.c \
 	networking_iproute2.c networking_iproute2.h \
 	networking_sitnl.c networking_sitnl.h \
 	networking.h \
src/openvpn/dco.c +4 -4
index 07dc1087..3ffc56d1 100644
@@ -599,7 +599,7 @@ void
 dco_install_iroute(struct multi_context *m, struct multi_instance *mi,
                    struct mroute_addr *addr)
 {
-#if defined(TARGET_LINUX)
+#if defined(TARGET_LINUX) || defined(TARGET_FREEBSD)
     if (!dco_enabled(&m->top.options))
     {
         return;
@@ -642,13 +642,13 @@ dco_install_iroute(struct multi_context *m, struct multi_instance *mi,
                          &mi->context.c2.push_ifconfig_local, dev, 0,
                          DCO_IROUTE_METRIC);
     }
-#endif /* if defined(TARGET_LINUX) */
+#endif /* if defined(TARGET_LINUX) || defined(TARGET_FREEBSD) */
 }
 
 void
 dco_delete_iroutes(struct multi_context *m, struct multi_instance *mi)
 {
-#if defined(TARGET_LINUX)
+#if defined(TARGET_LINUX) || defined(TARGET_FREEBSD)
     if (!dco_enabled(&m->top.options))
     {
         return;
@@ -681,7 +681,7 @@ dco_delete_iroutes(struct multi_context *m, struct multi_instance *mi)
                              0, DCO_IROUTE_METRIC);
         }
     }
-#endif /* if defined(TARGET_LINUX) */
+#endif /* if defined(TARGET_LINUX) || defined(TARGET_FREEBSD) */
 }
 
 #endif /* defined(ENABLE_DCO) */
src/openvpn/dco_freebsd.h +2 -0
index 3594f229..7de11697 100644
@@ -27,6 +27,8 @@
 
 #include "ovpn_dco_freebsd.h"
 
+#define DCO_IROUTE_METRIC   100
+
 typedef enum ovpn_key_slot dco_key_slot_t;
 typedef enum ovpn_key_cipher dco_cipher_t;
 
src/openvpn/networking.h +9 -0
index cf6d39ac..b0b31ea1 100644
@@ -31,6 +31,9 @@ struct context;
 #include "networking_sitnl.h"
 #elif ENABLE_IPROUTE
 #include "networking_iproute2.h"
+#elif defined(TARGET_FREEBSD)
+typedef void *openvpn_net_ctx_t;
+typedef char openvpn_net_iface_t;
 #else
 /* define mock types to ensure code builds on any platform */
 typedef void *openvpn_net_ctx_t;
@@ -238,7 +241,9 @@ int net_addr_ptp_v4_del(openvpn_net_ctx_t *ctx,
                         const openvpn_net_iface_t *iface,
                         const in_addr_t *local, const in_addr_t *remote);
 
+#endif /* ENABLE_SITNL || ENABLE_IPROUTE */
 
+#if defined(ENABLE_SITNL) || defined(ENABLE_IPROUTE) || defined(TARGET_FREEBSD)
 /**
  * Add a route for an IPv4 address/network
  *
@@ -315,6 +320,10 @@ int net_route_v6_del(openvpn_net_ctx_t *ctx, const struct in6_addr *dst,
                      const openvpn_net_iface_t *iface,
                      uint32_t table, int metric);
 
+#endif /* ENABLE_SITNL || ENABLE_IPROUTE || TARGET_FREEBSD */
+
+#if defined(ENABLE_SITNL) || defined(ENABLE_IPROUTE)
+
 /**
  * Retrieve the gateway and outgoing interface for the specified IPv4
  * address/network
src/openvpn/networking_freebsd.c new file +101 -0

101 lines not on this page · raw message

Signature · 1 line
2.37.1
Acked-by on v1 2/2
#5 lore raw

[Openvpn-devel] [PATCH applied] Re: Support creating iroute route entries on FreeBSD

Stared-at-code, tested on FreeBSD 14 with and without DCO as client, and with DCO as server, with a number of interesting iroute problems (--route /24, --iroute /32, --route /24 and --iroute same /24, no --route at all, but --iroute /24). See below.

The "big packet / fragment" thing still haunts us, but that's not something this patch would address, so out of scope wrt "does iroute work".

Some notes

- uncrustify had some remarks on #else/#endif with comments -> applied

- inet_ntop(3) says one shouldn't use magic numbers, but "INET_ADDRSTRLEN and INET6_ADDRSTRLEN" - this could be addressed in a followup patch

- "the rest of the code" uses print_in_addr_t() and print_in6_addr() today, to get a formatted IPv4/IPv6 address out of an in*_addr
- but these excel if there is a &gc around, and adding extra gc_new()/gc_free() around the argv_printf() did not really sound like "the resulting code is much nicer".

- any particular reason you used argv_printf() + argv_printf_cat(), instead of just putting all into a single argv_printf() call?

- the "openvpn_execve_check()" messages state "route *add* command failed", while it could be "del" as well. Making this "proper" might not be worth it, as it would need string manipulation (or "duplicate whole message").

- iroute installation works for the easy cases (--route in server.conf, --iroute with a more-specific of that in ccd/). It does not work for the nasty cases (--route and --iroute with same netbits).

I will send a followup e-mail with more details on this soon.

- route *deletion* does not work for IPv4 host routes, because of "other code stupidity" - this is not a problem of your code, but of an omission in dco.c (and I do wonder why this works for Linux)

	2022-08-20 14:06:56 us=764185 /sbin/route del -net 10.114.200.74/-1 10.114.2.8 - fib 0
	route: bad mask length: -1

our internal routes carry a flag structure that says "this is a host route!!" and if it is, "netbits" is not valid and needs to be set to 32 / 128 explicitly. This is pending a larger refactoring, but for the time being, I'll send a patch to fix this in dco.c (interesting enough, my code for IPv6 does this correctly, only the old IPv4 cruft...)

- in one case I saw something that looks like a race condition between DCO, "ifconfig route propagation", and "route add"

	2022-08-20 15:14:58 us=882323 DCO device tun1 opened
	2022-08-20 15:14:58 us=882357 do_ifconfig, ipv4=1, ipv6=1
	2022-08-20 15:14:58 us=882393 /sbin/ifconfig tun1 10.114.2.1 10.114.2.2 mtu 1500 netmask 255.255.255.0 up
	2022-08-20 15:14:58 us=887080 /sbin/route add -net 10.114.2.0 10.114.2.2 255.255.255.0
	add net 10.114.2.0: gateway 10.114.2.2
	2022-08-20 15:14:58 us=890690 /sbin/ifconfig tun1 inet6 fd00:abcd:114:2::1/64 mtu 1500 up
	2022-08-20 15:14:59 us=902853 /sbin/ifconfig tun1 inet6 -ifdisabled
	2022-08-20 15:15:02 us=57619 freebsd-74-amd64/2001:608:0:814::f000:3 /sbin/route add -net 10.114.202.74/32 10.114.2.8 -fib 0
	route: writing to routing socket: Network is unreachable
	add net 10.114.202.74: gateway 10.114.2.8 fib 0: Network is unreachable

... repeating the "route add" call manually (copy-paste) worked without error, so I'm not sure what upset things here. 3 Seconds between interface init and route addition should be more than enough for cooldown as well.

Your patch has been applied to the master branch.

commit 3433577a99270845cbce7fe738534e0b0ffd5780

Author: Kristof Provost
Date:   Fri Aug 12 15:41:54 2022 +0200
     Support creating iroute route entries on FreeBSD

     Signed-off-by: Kristof Provost <[email protected]...>
     Acked-by: Gert Doering <[email protected]...>
     Message-Id: <[email protected]...>
     URL: https://www.mail-archive.com/[email protected]/msg24895.html
     Signed-off-by: Gert Doering <[email protected]...>

--
kind regards,

Gert Doering

#6 lore raw

Re: [Openvpn-devel] route/iroute handling on FreeBSD

Hi,

so, as promised, more details on the route/iroute issues I saw... with a new Subject: line, to keep the threads halfway orderly.

Quoted 3 lines · Gert Doering

- route *deletion* does not work for IPv4 host routes, because of "other code stupidity" - this is not a problem of your code, but of an omission in dco.c (and I do wonder why this works for Linux)

A patch for *this* has been sent, so this is of no concern anymore (just needs someone bold enough to ACK it).

Quoted 2 lines

- in one case I saw something that looks like a race condition between DCO, "ifconfig route propagation", and "route add"

This is something I need to test further, to see if this was a one-off issue, or can be reproduced with good timing.

Quoted 5 lines

- iroute installation works for the easy cases (--route in server.conf, --iroute with a more-specific of that in ccd/). It does not work for the nasty cases (--route and --iroute with same netbits).

I will send a followup e-mail with more details on this soon.

*This* is the bit I want to cover separately, because this e-mail is going to be long :-)

So, here's my test cases, to see if I could get DCO to misbehave.

Server config has:

# grep ^route server.conf
route 10.114.200.0 255.255.255.0
route-ipv6 fd00:abcd:114:200::/64
route 10.114.201.0 255.255.255.0
route-ipv6 fd00:abcd:114:201::/64

("pull two /24 and two /64 towards the tun interface")

There is one particular client which has many addresses configured on its loopback

lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384

        inet 10.114.200.74 netmask 0xffffffff 
        inet 10.114.201.74 netmask 0xffffffff 
        inet 10.114.202.74 netmask 0xffffffff 
        inet6 fd00:abcd:114:200::74 prefixlen 128 
        inet6 fd00:abcd:114:201::74 prefixlen 128 
        inet6 fd00:abcd:114:202::74 prefixlen 128 

... and when that client connects, the expectation is that *other* clients can ping these 6 addresses (note it's 200, 201, 202 for v4 and v6).

Here's the ccd file for this client:

# cat /root/t_server/tun-udp-p2mp/ccd/freebsd-74-amd64
# Host iroute
iroute 10.114.200.74 255.255.255.255
iroute-ipv6 fd00:abcd:114:200::74/128
# Subnet-iroute
iroute 10.114.201.0 255.255.255.0
iroute-ipv6 fd00:abcd:114:201::/64
# iroute-without-route
# 10.114.202, fd00:abcd:114:202::/64
iroute 10.114.202.74 255.255.255.255
iroute-ipv6 fd00:abcd:114:202::74/128

So the "200" routes are "install a host route for IPv4 or IPv6 that is part of an aggregate". This works fine.

The "202" routes are "install a host route for IPv4 or IPv6 where there is no covering --route" - this is basically something OpenVPN could not do before DCO (update host routes on-demand on client connect), but is something I've long waited for :-) - this also works fine.

Now, the 201 routes are "there whole network specified in --route is routed to a single client". This is not such an unusual setup in cases like "there is a central office OpenVPN server, and one of the branch offices wants it 192.168.77.0/24 reachable from the other clients". This *fails*.

For IPv4, we end up with "netstat -rn" looking like this:

10.114.201.0/24    10.114.2.2         UGS        tun1
10.114.201.0/24    10.114.2.8         UGS        tun1

and "route get" telling us the the first one is preferred

$ route get 10.114.201.74
   route to: 10.114.201.74
destination: 10.114.201.0
       mask: 255.255.255.0
    gateway: 10.114.2.2
        fib: 0
  interface: tun1

so packets loop... (10.114.2.2 is not an address that belongs to an actual client, but "the tun interface peer") the client is 10.114.2.8.

$ traceroute 10.114.201.74
traceroute to 10.114.201.74 (10.114.201.74), 64 hops max, 40 byte packets
 1  10.114.2.2 (10.114.2.2)  1.706 ms  1.240 ms  0.921 ms
 2  localhost (127.0.0.1)  0.865 ms  0.809 ms  0.807 ms
 3  10.114.2.2 (10.114.2.2)  1.603 ms  1.605 ms  1.595 ms
 4  localhost (127.0.0.1)  1.533 ms  1.383 ms  1.768 ms

For IPv6, things look different - route installation fails

2022-08-20 15:47:24 freebsd-74-amd64/2001:608:0:814::f000:3 /sbin/route -6 add -net fd00:abcd:114:201::/64 fd00:abcd:114:2::1006 -fib 0

add net fd00:abcd:114:201::/64: gateway fd00:abcd:114:2::1006 fib 0: route already in table

so, the iroute never makes it to kernel, "netstat -rn" remains at

fd00:abcd:114:201::/64 link#14 US tun1

and packets loop as well

$ traceroute6 fd00:abcd:114:201::74
traceroute6 to fd00:abcd:114:201::74 (fd00:abcd:114:201::74) from fd00:abcd:114:2::1, 64 hops max, 28 byte packets
 1  fd00:abcd:114:2::1000  1.749 ms  1.246 ms  0.930 ms
 2  fd00:abcd:114:2::1  0.836 ms  1.225 ms  1.067 ms
 3  fd00:abcd:114:2::1000  1.555 ms  1.631 ms  1.575 ms
 4  fd00:abcd:114:2::1  1.472 ms  1.531 ms  1.677 ms

So, what we do on *Linux* to handle this, is to work with route metrics (10.220.* = "same instances, different host, so different address range")

linux$ ip route |grep tun1
10.220.200.0/24 via 10.220.2.2 dev tun1 metric 200
10.220.200.74 via 10.220.2.8 dev tun1 metric 100
10.220.201.0/24 via 10.220.2.8 dev tun1 metric 100
10.220.201.0/24 via 10.220.2.2 dev tun1 metric 200
10.220.202.74 via 10.220.2.8 dev tun1 metric 100
linux$ ip -6 route |grep tun1
fd00:abcd:220:200::74 via fd00:abcd:220:2::1006 dev tun1 metric 100 pref medium fd00:abcd:220:200::/64 dev tun1 metric 200 pref medium fd00:abcd:220:201::/64 via fd00:abcd:220:2::1006 dev tun1 metric 100 pref medium fd00:abcd:220:201::/64 dev tun1 metric 200 pref medium fd00:abcd:220:202::74 via fd00:abcd:220:2::1006 dev tun1 metric 100 pref medium

so there's always 5 routes in the active table - 2 "--route" routes (according to server.conf) with *metric 200*, and then 3 "--iroute" routes (from ccd/) with metric DCO_IROUTE_METRIC = 100.

Linux handles this as would a "Cisco router" do - if there are two identical routes with different metric, the lower-metric route is used, and the other one is ignored. So this works very nicely.

Now, back to FreeBSD.

- our code does not try to set metrics on FreeBSD
- my reading of route(8) does not show me anything in that direction (metric, preference, administrative distance, ...)
- I do not understand FreeBSD internals well enough to know whether it can be done at all, or not.

If it can not be done, we need to document this as "one of the issues with FreeBSD DCO", and it can of course be easily workarounded in most cases

--route 10.114.201.0/24

    --iroute 10.114.201.0/25
    --iroute 10.114.201.128/25

will get the same thing done "when client is online route the whole /25 down there", but it's harder to get into people's heads :-) (adding code to auto-split such a prefix is technically possible but would be quite a complication, so I'd rather solve this "with documentation").

So, what shall we do?

gert

Signature · 6 lines
"If was one thing all people took for granted, was conviction that if you 
 feed honest figures into a computer, honest figures come out. Never doubted 
 it myself till I met a computer with a sense of humor."
                             Robert A. Heinlein, The Moon is a Harsh Mistress

Gert Doering - Munich, Germany                             [email protected]...
#7 lore raw

Re: [Openvpn-devel] route/iroute handling on FreeBSD

Quoted 138 lines · Gert Doering
Quoted 6 lines

- iroute installation works for the easy cases (--route in
server.conf,
--iroute with a more-specific of that in ccd/). It does not work for the nasty cases (--route and --iroute with same netbits).

I will send a followup e-mail with more details on this soon.

*This* is the bit I want to cover separately, because this e-mail is going to be long :-)

So, here's my test cases, to see if I could get DCO to misbehave.

Server config has:

# grep ^route server.conf
route 10.114.200.0 255.255.255.0
route-ipv6 fd00:abcd:114:200::/64
route 10.114.201.0 255.255.255.0
route-ipv6 fd00:abcd:114:201::/64

("pull two /24 and two /64 towards the tun interface")

There is one particular client which has many addresses configured on its loopback

lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384

        inet 10.114.200.74 netmask 0xffffffff
        inet 10.114.201.74 netmask 0xffffffff
        inet 10.114.202.74 netmask 0xffffffff
        inet6 fd00:abcd:114:200::74 prefixlen 128
        inet6 fd00:abcd:114:201::74 prefixlen 128
        inet6 fd00:abcd:114:202::74 prefixlen 128

... and when that client connects, the expectation is that *other* clients can ping these 6 addresses (note it's 200, 201, 202 for v4 and v6).

Here's the ccd file for this client:

# cat /root/t_server/tun-udp-p2mp/ccd/freebsd-74-amd64
# Host iroute
iroute 10.114.200.74 255.255.255.255
iroute-ipv6 fd00:abcd:114:200::74/128
# Subnet-iroute
iroute 10.114.201.0 255.255.255.0
iroute-ipv6 fd00:abcd:114:201::/64
# iroute-without-route
# 10.114.202, fd00:abcd:114:202::/64
iroute 10.114.202.74 255.255.255.255
iroute-ipv6 fd00:abcd:114:202::74/128

So the "200" routes are "install a host route for IPv4 or IPv6 that is part of an aggregate". This works fine.

The "202" routes are "install a host route for IPv4 or IPv6 where there is no covering --route" - this is basically something OpenVPN could not do before DCO (update host routes on-demand on client connect), but is something I've long waited for :-) - this also works fine.

Now, the 201 routes are "there whole network specified in --route is routed to a single client". This is not such an unusual setup in cases like "there is a central office OpenVPN server, and one of the branch offices wants it 192.168.77.0/24 reachable from the other clients". This *fails*.

For IPv4, we end up with "netstat -rn" looking like this:

10.114.201.0/24    10.114.2.2         UGS        tun1
10.114.201.0/24    10.114.2.8         UGS        tun1

and "route get" telling us the the first one is preferred

$ route get 10.114.201.74
   route to: 10.114.201.74
destination: 10.114.201.0
       mask: 255.255.255.0
    gateway: 10.114.2.2
        fib: 0
  interface: tun1

so packets loop... (10.114.2.2 is not an address that belongs to an actual client, but "the tun interface peer") the client is 10.114.2.8.

$ traceroute 10.114.201.74
traceroute to 10.114.201.74 (10.114.201.74), 64 hops max, 40 byte packets
 1  10.114.2.2 (10.114.2.2)  1.706 ms  1.240 ms  0.921 ms
 2  localhost (127.0.0.1)  0.865 ms  0.809 ms  0.807 ms
 3  10.114.2.2 (10.114.2.2)  1.603 ms  1.605 ms  1.595 ms
 4  localhost (127.0.0.1)  1.533 ms  1.383 ms  1.768 ms

For IPv6, things look different - route installation fails

2022-08-20 15:47:24 freebsd-74-amd64/2001:608:0:814::f000:3 /sbin/route -6 add -net fd00:abcd:114:201::/64 fd00:abcd:114:2::1006 

-fib 0
add net fd00:abcd:114:201::/64: gateway fd00:abcd:114:2::1006 fib 0: route already in table

so, the iroute never makes it to kernel, "netstat -rn" remains at

fd00:abcd:114:201::/64 link#14 US
tun1

and packets loop as well

$ traceroute6 fd00:abcd:114:201::74
traceroute6 to fd00:abcd:114:201::74 (fd00:abcd:114:201::74) from fd00:abcd:114:2::1, 64 hops max, 28 byte packets
 1  fd00:abcd:114:2::1000  1.749 ms  1.246 ms  0.930 ms
 2  fd00:abcd:114:2::1  0.836 ms  1.225 ms  1.067 ms
 3  fd00:abcd:114:2::1000  1.555 ms  1.631 ms  1.575 ms
 4  fd00:abcd:114:2::1  1.472 ms  1.531 ms  1.677 ms

So, what we do on *Linux* to handle this, is to work with route metrics (10.220.* = "same instances, different host, so different address range")

linux$ ip route |grep tun1
10.220.200.0/24 via 10.220.2.2 dev tun1 metric 200
10.220.200.74 via 10.220.2.8 dev tun1 metric 100
10.220.201.0/24 via 10.220.2.8 dev tun1 metric 100
10.220.201.0/24 via 10.220.2.2 dev tun1 metric 200
10.220.202.74 via 10.220.2.8 dev tun1 metric 100
linux$ ip -6 route |grep tun1
fd00:abcd:220:200::74 via fd00:abcd:220:2::1006 dev tun1 metric 100 pref medium fd00:abcd:220:200::/64 dev tun1 metric 200 pref medium fd00:abcd:220:201::/64 via fd00:abcd:220:2::1006 dev tun1 metric 100 pref medium fd00:abcd:220:201::/64 dev tun1 metric 200 pref medium fd00:abcd:220:202::74 via fd00:abcd:220:2::1006 dev tun1 metric 100 pref medium

so there's always 5 routes in the active table - 2 "--route" routes (according to server.conf) with *metric 200*, and then 3 "--iroute" routes (from ccd/) with metric DCO_IROUTE_METRIC = 100.

Linux handles this as would a "Cisco router" do - if there are two identical routes with different metric, the lower-metric route is used, and the other one is ignored. So this works very nicely.

Quoted 8 lines

Now, back to FreeBSD.

- our code does not try to set metrics on FreeBSD
- my reading of route(8) does not show me anything in that direction (metric, preference, administrative distance, ...)
- I do not understand FreeBSD internals well enough to know whether it can be done at all, or not.

It’s rather poorly (i.e. not) documented, but it is possible to set a route metric. It’s called a ‘weight’, and not actually mentioned in the man page. Or shown in netstat output.

But:

	$ sudo route add 172.16.2.0/24 10.0.2.1
	add net 172.16.2.0: gateway 10.0.2.1
	$ sudo route add 172.16.2.0/24 -weight 2 10.0.2.254
	add net 172.16.2.0: gateway 10.0.2.254

So we can add multiple routes for the same network, as long as they have a different gateway.

The usual netstat -rn doesn’t show the weight. You have to use the json/xml output to get it:

	$ netstat -rn --libxo json | jq
	…
              {
                "destination": "172.16.2.0/24",
                "gateway": "10.0.2.1",
                "flags": "UGS",
                "flags_pretty": [
                  "up",
                  "gateway",
                  "static"
                ],
                "weight": 1,
                "interface-name": "bnxt0"
              },
              {
                "destination": "172.16.2.0/24",
                "gateway": "10.0.2.254",
                "flags": "UGS",
                "flags_pretty": [
                  "up",
                  "gateway",
                  "static"
                ],
                "weight": 2,
                "interface-name": "bnxt0"
              },

A higher weight gives a higher priority, so that’s the opposite of what Linux does with metric, I believe.

Quoted 8 lines

If it can not be done, we need to document this as "one of the issues with FreeBSD DCO", and it can of course be easily workarounded in most cases

--route 10.114.201.0/24

    --iroute 10.114.201.0/25
    --iroute 10.114.201.128/25
Quoted 7 lines

will get the same thing done "when client is online route the whole /25 down there", but it's harder to get into people's heads :-) (adding code to auto-split such a prefix is technically possible but would be quite a complication, so I'd rather solve this "with documentation").

So, what shall we do?

If I’m understanding everything correctly it should be relatively simple to extend networking_freebsd.c to also set a weight for each route it installs. We’d have to convert the metric into a weight, which I think we could do as weight = RT_MAX_WEIGHT - metric (RT_MAX_WEIGHT is 16777215 /* 3 bytes */).

Can you try this?

	diff --git a/src/openvpn/networking_freebsd.c 
b/src/openvpn/networking_freebsd.c
	index 0633dce7..e79af2bf 100644
	--- a/src/openvpn/networking_freebsd.c
	+++ b/src/openvpn/networking_freebsd.c
	@@ -10,6 +10,8 @@

	 #if defined(TARGET_FREEBSD)

	+#include <net/route.h>
	+
	 static int
	 net_route_v4(const char *op, const in_addr_t *dst, int prefixlen,
	              const in_addr_t *gw, const char *iface, uint32_t table,
	@@ -23,12 +25,13 @@ net_route_v4(const char *op, const in_addr_t *dst, 
int prefixlen,
	     _dst = ntohl(*dst);
	     _gw = ntohl(*gw);

	-    argv_printf(&argv, "%s %s -net %s/%d %s -fib %d",
	+    argv_printf(&argv, "%s %s -net %s/%d %s -fib %d -weight %d",
	                 ROUTE_PATH, op,
	                 inet_ntop(AF_INET, &_dst, buf1, sizeof(buf1)),
	                 prefixlen,
	                 inet_ntop(AF_INET, &_gw, buf2, sizeof(buf2)),
	-                table);
	+                table,
	+                RT_MAX_WEIGHT - metric);

	     argv_msg(M_INFO, &argv);
	     status = openvpn_execve_check(&argv, NULL, 0,
	@@ -48,12 +51,13 @@ net_route_v6(const char *op, const struct in6_addr 

*dst,
struct argv argv = argv_new();
bool status;

- argv_printf(&argv, "%s -6 %s -net %s/%d %s -fib %d",
+ argv_printf(&argv, "%s -6 %s -net %s/%d %s -fib %d -weight %d",
ROUTE_PATH, op, inet_ntop(AF_INET6, dst, buf1, sizeof(buf1)), prefixlen,
inet_ntop(AF_INET6, gw, buf2, sizeof(buf2)),
- table);
+ table,
+ RT_MAX_WEIGHT - metric);

argv_msg(M_INFO, &argv);
status = openvpn_execve_check(&argv, NULL, 0,

Kristof

#8 lore raw

Re: [Openvpn-devel] route/iroute handling on FreeBSD

Hi,

Quoted 12 lines · Kristof Provost
Quoted 8 lines

Now, back to FreeBSD.

- our code does not try to set metrics on FreeBSD
- my reading of route(8) does not show me anything in that direction (metric, preference, administrative distance, ...)
- I do not understand FreeBSD internals well enough to know whether it can be done at all, or not.

It???s rather poorly (i.e. not) documented, but it is possible to set a route metric. It???s called a ???weight???, and not actually mentioned in the man page. Or shown in netstat output.

Quoted 6 lines

But:

	$ sudo route add 172.16.2.0/24 10.0.2.1
	add net 172.16.2.0: gateway 10.0.2.1
	$ sudo route add 172.16.2.0/24 -weight 2 10.0.2.254
	add net 172.16.2.0: gateway 10.0.2.254

Oh, this is interesting, and good to know.

What is the default weight? "1"?

Quoted 6 lines

So we can add multiple routes for the same network, as long as they have a different gateway.

The usual netstat -rn doesn???t show the weight. You have to use the json/xml output to get it:

Quoted 14 lines
	$ netstat -rn --libxo json | jq
	???
               {
                 "destination": "172.16.2.0/24",
                 "gateway": "10.0.2.1",
                 "flags": "UGS",
                 "flags_pretty": [
                   "up",
                   "gateway",
                   "static"
                 ],
                 "weight": 1,
                 "interface-name": "bnxt0"
               },

Interesting :-)

[..]

Quoted 2 lines

A higher weight gives a higher priority, so that???s the opposite of what Linux does with metric, I believe.

Right. Linux metrics are "lower is better", like a routing protocol cost would do.

[..]

Quoted 8 lines

If I???m understanding everything correctly it should be relatively simple to extend networking_freebsd.c to also set a weight for each route it installs.
We???d have to convert the metric into a weight, which I think we could do as weight = RT_MAX_WEIGHT - metric (RT_MAX_WEIGHT is 16777215 /* 3 bytes */).

Can you try this?

Looks good, might work :-).

Will give this a try as soon as time permits - next 2-3 weeks might not work out.

gert

Signature · 6 lines
"If was one thing all people took for granted, was conviction that if you 
 feed honest figures into a computer, honest figures come out. Never doubted 
 it myself till I met a computer with a sense of humor."
                             Robert A. Heinlein, The Moon is a Harsh Mistress

Gert Doering - Munich, Germany                             [email protected]...
#9 lore raw

Re: [Openvpn-devel] route/iroute handling on FreeBSD

Quoted 15 lines · Gert Doering
Quoted 14 lines · Kristof Provost
Quoted 8 lines

Now, back to FreeBSD.

- our code does not try to set metrics on FreeBSD
- my reading of route(8) does not show me anything in that direction (metric, preference, administrative distance, ...)
- I do not understand FreeBSD internals well enough to know whether it can be done at all, or not.

It???s rather poorly (i.e. not) documented, but it is possible to set a route metric. It???s called a ???weight???, and not actually mentioned in the man page. Or shown in netstat output.

But:

Quoted 8 lines
Quoted 4 lines
	$ sudo route add 172.16.2.0/24 10.0.2.1
	add net 172.16.2.0: gateway 10.0.2.1
	$ sudo route add 172.16.2.0/24 -weight 2 10.0.2.254
	add net 172.16.2.0: gateway 10.0.2.254

Oh, this is interesting, and good to know.

What is the default weight? "1"?

Yes, the default is RT_DEFAULT_WEIGHT:

	/usr/include/net/route.h
	105:#define	RT_DEFAULT_WEIGHT	1

Best regards,
Kristof

#10 lore raw

Re: [Openvpn-devel] route/iroute handling on FreeBSD

In reply to Kristof Provost

Hi,

FreeBSD DCO has the open issue of "iroute with the same netmask as route, so we want metric/weight/... to differenciate"

Quoted 6 lines · Kristof Provost

But:

	$ sudo route add 172.16.2.0/24 10.0.2.1
	add net 172.16.2.0: gateway 10.0.2.1
	$ sudo route add 172.16.2.0/24 -weight 2 10.0.2.254
	add net 172.16.2.0: gateway 10.0.2.254

I tried patching networking_freebsd.c the way you suggested, and it seems to work "halfway". That is...

2022-10-06 17:26:35 us=684854 freebsd-74-amd64/2001:608:0:814::f000:3 /sbin/route add -net 10.114.201.0/24 10.114.2.2 -fib 0 -weight 16777115

add net 10.114.201.0: gateway 10.114.2.2 fib 0: route already in table

2022-10-06 17:26:35 us=695547 freebsd-74-amd64/2001:608:0:814::f000:3 /sbin/route -6 add -net fd00:abcd:114:201::/64 fd00:abcd:114:2::1000 -fib 0 -weight 16777115

add net fd00:abcd:114:201::/64: gateway fd00:abcd:114:2::1000 fib 0: route already in table

... it takes the command, but still refuses to create the routes.

$ netstat -rn --libxo json | jq

tells me...

              {
                "destination": "10.114.201.0/24",
                "gateway": "10.114.2.2",
                "flags": "UGS",
                "flags_pretty": [
                  "up",
                  "gateway",
                  "static"
                ],
                "weight": 1,
                "interface-name": "tun1"
              },
              {
                "destination": "fd00:abcd:114:201::/64",
                "gateway": "link#5",
                "flags": "US",
                "flags_pretty": [
                  "up",
                  "static"
                ],
                "weight": 1,
                "interface-name": "tun1"
              },

For IPv6, we definitely have "different weight, different gateway", so I'd expect this to work.

For IPv4, this is a bit of a nuisance - the client in question happens to be the *first* client in ipp.txt (= .2), and that's what OpenVPN picks for the "route" gateway too... so, "different weight, same gateway".

Now, bumping the "first client" via ipp.txt to the .4 address gives me a whole new sleigh of interesting problems...

2022-10-06 17:36:05 us=447754 freebsd-74-amd64/2001:608:0:814::f000:3 /sbin/route add -net 10.114.200.74/32 10.114.2.4 -fib 0 -weight 16777115

route: writing to routing socket: Network is unreachable
add net 10.114.200.74: gateway 10.114.2.4 fib 0: Network is unreachable

... which, I think, is because the interface is flagged as "pointtopoint", and .4 is off-net...

$ ifconfig tun1
tun1: flags=8051<UP,POINTOPOINT,RUNNING,MULTICAST> metric 0 mtu 1500
        options=80000<LINKSTATE>
        inet 10.114.2.1 --> 10.114.2.2 netmask 0xffffff00
        inet6 fd00:abcd:114:2::1 prefixlen 64
        inet6 fe80::250:56ff:fe9c:41b%tun1 prefixlen 64 scopeid 0x5
        groups: openvpn

So, started out with one solution, now I have two new problems...

gert

Signature · 6 lines
"If was one thing all people took for granted, was conviction that if you 
 feed honest figures into a computer, honest figures come out. Never doubted 
 it myself till I met a computer with a sense of humor."
                             Robert A. Heinlein, The Moon is a Harsh Mistress

Gert Doering - Munich, Germany                             [email protected]...
#11 lore raw

Re: [Openvpn-devel] route/iroute handling on FreeBSD

Hi Gert,

Quoted 3 lines · Gert Doering

FreeBSD DCO has the open issue of "iroute with the same netmask as route, so we want metric/weight/... to differenciate"

I’ll try to dig into that, but it’ll be late next week at the earliest.

Best regards,
Kristof

#12 lore raw

Re: [Openvpn-devel] route/iroute handling on FreeBSD

Hi,

Quoted 4 lines · Kristof Provost
Quoted 2 lines · Gert Doering

FreeBSD DCO has the open issue of "iroute with the same netmask as route, so we want metric/weight/... to differenciate"

I???ll try to dig into that, but it???ll be late next week at the earliest.

Thanks.

I'll dig into the POINTTOPOINT issue... configure a real MULTIPOINT interface for --topology subnet (which is something I've always avoided so far, because "we know the other stuff works")

gert

Signature · 6 lines
"If was one thing all people took for granted, was conviction that if you 
 feed honest figures into a computer, honest figures come out. Never doubted 
 it myself till I met a computer with a sense of humor."
                             Robert A. Heinlein, The Moon is a Harsh Mistress

Gert Doering - Munich, Germany                             [email protected]...
#13 lore raw

Re: [Openvpn-devel] route/iroute handling on FreeBSD

Hi,

Quoted 3 lines · Gert Doering

I'll dig into the POINTTOPOINT issue... configure a real MULTIPOINT interface for --topology subnet (which is something I've always avoided so far, because "we know the other stuff works")

This is... an interesting problem.

Changing the tun(4) stuff to do IFF_BROADCAST for "topology subnet" is straightforward, and makes the whole code much simpler actually (and you can do "ifconfig tun7 1.2.3.4/24" and have it actually do what you expect "here is a subnet, with 255 different next-hop IPs available for routing"). Patch is attached so you can see what I'm talking about :-)

Butbut... ovpn(4) does not support the TUNSIFMODE ioctl() to set something that is not IFF_POINTOPOINT, so we fail with

    2022-10-11 20:58:28 ioctl(TUNSIFMODE): Can't assign requested address (errno=49)
    2022-10-11 20:58:28 DCO device tun14 opened
    2022-10-11 20:58:28 /sbin/ifconfig tun14 10.194.3.56/24 mtu 1500 up
    ifconfig: ioctl (SIOCAIFADDR): Destination address required
    2022-10-11 20:58:28 FreeBSD ifconfig failed: external program exited with error status: 1
    2022-10-11 20:58:28 Exiting due to fatal error

@kp, not sure how to move forward. What we need is a "true on-link subnet", so installation of iroutes to "peer #4" works (which it doesn't today), and I think that means "ovpn(4) needs to learn IFF_BROADCAST".

OTOH, in "not peer2peer mode", we only support topology subnet anyway, so maybe it should autoconfigure that, depending on the mode OpenVPN requests (p2p mode -> IFF_POINTOPOINT, p2mp mode -> IFF_BROADCAST), so we don't even need TUNSIFMODE.

Looking into if_ovpn.c, this is a bit beyond my understanding of the code, though...

gert

Signature · 6 lines
"If was one thing all people took for granted, was conviction that if you 
 feed honest figures into a computer, honest figures come out. Never doubted 
 it myself till I met a computer with a sense of humor."
                             Robert A. Heinlein, The Moon is a Harsh Mistress

Gert Doering - Munich, Germany                             [email protected]...

From 02f74941b29532f977be58205202eb2ccb21b12e Mon Sep 17 00:00:00 2001
From: Gert Doering <[email protected]...>
Date: Tue, 11 Oct 2022 19:54:23 +0200
Subject: [PATCH] FreeBSD: for topology subnet, put tun interface into
IFF_BROADCAST mode

For reasons unknown, OpenVPN has always put FreeBSD tun(4) interfaces into point-to-point mode (IFF_POINTOPOINT), which means "local and remote address, no on-link subnet".

"--topology subnet" was emulated by adding a subnet-route to the "remote" (which was just picking a free address from the subnet).

This works well enough for classic tun(4) interfaces that have no next-hop resolution, and routes pointing to "that fake remote" only (because all routing is done inside OpenVPN and it does not matter how packets get there) - but for ovpn(4) interfaces, it breaks iroute setup, where the route next-hop must be an on-link address.

Thus, change interface to IFF_BROADCAST mode, and get rid of all the special code needed to "fake" subnet mode.

Tested with tun(4) and ovpn(4) on FreeBSD 14, client and server, and with tun(4) on FreeBSD 12 and 7.4

---
 Changes.rst       |  5 +++++
 src/openvpn/tun.c | 28 ++++++++++------------------
 2 files changed, 15 insertions(+), 18 deletions(-)
Changes.rst +5 -0
index df56f76a..0397cb94 100644
@@ -163,6 +163,11 @@ User-visible Changes
 - :code:`link_mtu` parameter is removed from environment or replaced with 0 when scripts are
   called with parameters. This parameter is unreliable and no longer internally calculated.
 
+- FreeBSD tun interfaces with ``--topology subnet`` are now put into real
+  subnet mode (IFF_BROADCAST instead of IFF_POINTOPOINT) - this might upset
+  software that enumerates interfaces, looking for "broadcast capable?" and
+  expecting certain results.  Normal uses should not see any difference.
+
 Overview of changes in 2.5
 ==========================
 
src/openvpn/tun.c +10 -18
index ddee49f9..ede65cbe 100644
@@ -1479,8 +1479,6 @@ do_ifconfig_ipv4(struct tuntap *tt, const char *ifname, int tun_mtu,
 
 #elif defined(TARGET_FREEBSD) || defined(TARGET_DRAGONFLY)
 
-    in_addr_t remote_end;           /* for "virtual" subnet topology */
-
     /* example: ifconfig tun2 10.2.0.2 10.2.0.1 mtu 1450 netmask 255.255.255.255 up */
     if (tun)
     {
@@ -1490,10 +1488,9 @@ do_ifconfig_ipv4(struct tuntap *tt, const char *ifname, int tun_mtu,
     }
     else if (tt->type == DEV_TYPE_TUN && tt->topology == TOP_SUBNET)
     {
-        remote_end = create_arbitrary_remote( tt );
-        argv_printf(&argv, "%s %s %s %s mtu %d netmask %s up", IFCONFIG_PATH,
-                    ifname, ifconfig_local, print_in_addr_t(remote_end, 0, &gc),
-                    tun_mtu, ifconfig_remote_netmask);
+        int netbits = netmask_to_netbits2(tt->remote_netmask);
+        argv_printf(&argv, "%s %s %s/%d mtu %d up", IFCONFIG_PATH,
+                    ifname, ifconfig_local, netbits, tun_mtu );
     }
     else
     {
@@ -1504,18 +1501,6 @@ do_ifconfig_ipv4(struct tuntap *tt, const char *ifname, int tun_mtu,
     argv_msg(M_INFO, &argv);
     openvpn_execve_check(&argv, es, S_FATAL, "FreeBSD ifconfig failed");
 
-    /* Add a network route for the local tun interface */
-    if (!tun && tt->type == DEV_TYPE_TUN && tt->topology == TOP_SUBNET)
-    {
-        struct route_ipv4 r;
-        CLEAR(r);
-        r.flags = RT_DEFINED;
-        r.network = tt->local & tt->remote_netmask;
-        r.netmask = tt->remote_netmask;
-        r.gateway = remote_end;
-        add_route(&r, tt, 0, NULL, es, NULL);
-    }
-
 #elif defined(TARGET_AIX)
     {
         /* AIX ifconfig will complain if it can't find ODM path in env */
@@ -2949,12 +2934,19 @@ open_tun(const char *dev, const char *dev_type, const char *dev_node, struct tun
 
         if (tt->fd >= 0 && tt->type == DEV_TYPE_TUN)
         {
+            /* see "Interface Flags" in ifnet(9) */
             int i = IFF_POINTOPOINT | IFF_MULTICAST;
+            if (tt->topology == TOP_SUBNET)
+            {
+                i = IFF_BROADCAST | IFF_MULTICAST;
+            }
 
             if (ioctl(tt->fd, TUNSIFMODE, &i) < 0)
             {
                 msg(M_WARN | M_ERRNO, "ioctl(TUNSIFMODE)");
             }
+
+            /* multi_af mode for v4+v6, see "tun(4)" */
             i = 1;
             if (ioctl(tt->fd, TUNSIFHEAD, &i) < 0)
             {
Signature · 1 line
2.37.3
#14 lore raw

Re: [Openvpn-devel] route/iroute handling on FreeBSD

Hi,

Quoted 4 lines · Gert Doering

OTOH, in "not peer2peer mode", we only support topology subnet anyway, so maybe it should autoconfigure that, depending on the mode OpenVPN requests (p2p mode -> IFF_POINTOPOINT, p2mp mode -> IFF_BROADCAST), so we don't even need TUNSIFMODE.

It's not that easy - there are two factors coming into play here, which are orthogonal.

Factor 1: single-peer (client or p2p) vs. multi-peer

single-peer -> DCO has only 1 peer, all packets that go into the
tun/dco interface are sent out to the single peer ("dumb pipe mode") - exactly like tun(4) behaves

If a subnet is configured on the interface, packets to ALL IPs (!= local) in that subnet are sent to the other side. No next-hop lookup is done.

multi-peer -> DCO has 0..N peers. Packets that go into the tun/dco
interface get mapped according to route next-hop <-> peer[]->vpn_v4 (or v6) address and sent to that peer.

If a subnet is configured on the interface (only supported mode :-) ), packets to IP addresses in that subnet that have no mapped-to peer are DROPPED.

On Linux this is controlled by openvpn_dco_init() setting dco->ifmode to OVPN_MODE_MP/OVPN_MODE_P2P, and net_iface_new() passing this parameter to kernel DCO.

On FreeBSD, OpenVPN does not signal the p2p/p2mp mode to kernel yet (if I am not misreading the code), and if I am reading ovpn_route_peer() in if_ovpn.c correctly, "single-peer mode" will drop packets where a corresponding peer can neither be found by ovpn_find_peer_by_ip, using target IP or Route->Gateway lookup.

(*) ovpn_find_peer_by_ip() is using a sequential search, for every single packet, which will perform poorly on servers with 1000s of clients... room for optimization.

Factor 2: IFF_POINTOPOINT vs. IFF_BROADCAST

This seems to be a *BSD-specific thing, aka "there is nothing in the Linux specific code that seems to bother with this".

An Interface can be put to point-to-point mode (by ioctl(TUNSIFMODE) with the IFF_POINTOPOINT flag set). In that case, there is no "on-link" subnet, just "us" and "them", and ifconfig wants two IPv4 addresses as parameter

     ifconfig tun0 10.2.3.77 10.2.3.1

... that do not need to be adjacent. It takes a "netmask" argument, but that does not have a real effect. To simulate "subnet" mode, OpenVPN will add a 10.2.3.0/24 -> 10.2.3.1 route.

In "dumb pipe mode", this works nicely - the /24 route shoves all packets towards the tun0 interface, and the tun(4) sends them over, without looking.

In "openvpn non-dco p2mp server mode" this *also* works nicely, because the tun(4) interface is still in "dumb pipe mode" and OpenVPN will do an iroute lookup in the p2mp server forwarding path, sending to the proper client.

In DCO mode with iroutes, this does not work at all anymore. Say, there is a client on 10.2.3.50, using iroute 10.22.33.0/24, so OpenVPN tries to install a route via

    route add 10.22.33.0/24 10.2.3.50

and FreeBSD *refuses* to install this route, because "10.2.3.50" is not adjacent. In my initial tests I did not notice this, because the only client with iroute was assigned the .2 address,

One option that sounds like it would work: always use "remote_addr" (= second parameter to IFF_POINTOPOINT ifconfig) as next-hop for all iroute-originated routes - this would make FreeBSD accept the route, and send packets towards the ovpn(4) interface. Alas, the next-hop would be "remote_addr", not "the right p2mp peer", so the "what peer do I need to send this packet to?" search in ovpn(4) would find the peer on the "remote_addr" IP, not the correct one (= lookup happens in normal routing table). So, it wouldn't work [AND it would add more ugly bits to OpenVPN].

Option #2: set the interface to IFF_BROADCAST, which means "true multi-IP interface, with subnet, and Kernel considers all IPs of that subnet to be adjacent, and acceptable route next-hops".

This is what the patch I sent yesterday does to tun(4) interfaces, and the result is amazingly well-behaved - "it just works" and gets *rid* of quite a bit of special-case hack-around code. Want.

Unfortunately, this breaks for two reasons with FreeBSD DCO as of today:

- no TUNSIFMODE support in ovpn(4), so stuck to IFF_POINTOPOINT mode (I think I can add that, copy-pasting code from if_tuntap.c)

- no single-peer mode in ovpn(4)
-> this will break "I am on client .5 and want to send packets to client .6" because vpn_v4 peer lookup won't find ".6", and no route towards .1 (server) either, so packet drop.
(I'm not sure I can make this happen, but I'll give it a try,
following the Linux DCO approach described above)

(There is another option here that I think I could mention - we can, of course, do IFF_BROADCAST only on the openvpn p2mp server, and stick to IFF_POINTOPOINT on client/p2p instances. BUT that implies we'd get yet another case for FreeBSD ifconfig - p2p, p2p-simulated subnet, subnet
- so the OpenVPN code will get more complex, not simpler - and we still need IFF_BROADCAST support in ovpn(4) - so we can do it properly all the way, I'd say)

Note that Aspect 2 is only about IPv4. IPv6 always uses true subnet mode, and all these considerations are of no concern...

Phew, what a long mail :-) - hope it clarifies what we're missing today and how to move forward.

gert

Signature · 6 lines
"If was one thing all people took for granted, was conviction that if you 
 feed honest figures into a computer, honest figures come out. Never doubted 
 it myself till I met a computer with a sense of humor."
                             Robert A. Heinlein, The Moon is a Harsh Mistress

Gert Doering - Munich, Germany                             [email protected]...
#15 lore raw

Re: [Openvpn-devel] route/iroute handling on FreeBSD

Hi,

people have alreadycomplained at me that I write so long e-mails today, so I can write more...

Quoted 9 lines · Gert Doering

Factor 1: single-peer (client or p2p) vs. multi-peer

single-peer -> DCO has only 1 peer, all packets that go into the
tun/dco interface are sent out to the single peer ("dumb pipe mode") - exactly like tun(4) behaves

If a subnet is configured on the interface, packets to ALL IPs (!= local) in that subnet are sent to the other side. No next-hop lookup is done.

This is "sort of" handled in if_ovpn.c today

ovpn_route_peer(struct ovpn_softc *sc, struct mbuf **m0,
const struct sockaddr *dst)
{
...

        /* Shortcut if we're a client (or are a server and have only one client). */
        if (sc->peercount == 1)
                return (ovpn_find_only_peer(sc));

... so this works for the client, but has one interesting drawback on the server - if there is only a single client connected, the server will send ALL to-be-tunneled packets to that client. As soon as client #2 connects, packets are properly sorted.

[..]

Quoted 4 lines

Factor 2: IFF_POINTOPOINT vs. IFF_BROADCAST

This seems to be a *BSD-specific thing, aka "there is nothing in the Linux specific code that seems to bother with this".

I've whacked at if_ovpn.c and dco_freebsd.c a bit now, and I seem to have working code for both ends. I am not a FreeBSD kernel coder, so I have no idea how many behavioural standards I am violating, but it makes "real subnet mode in OpenVPN" work for me, with DCO.

Kernel patch attached, OpenVPN patches will follow soonish (outside of this e-mail thread).

gert

Signature · 6 lines
"If was one thing all people took for granted, was conviction that if you 
 feed honest figures into a computer, honest figures come out. Never doubted 
 it myself till I met a computer with a sense of humor."
                             Robert A. Heinlein, The Moon is a Harsh Mistress

Gert Doering - Munich, Germany                             [email protected]...

From ddb95b7f57d78498003f3212b23e6adb0b5a9828 Mon Sep 17 00:00:00 2001
From: Gert Doering <[email protected]...>
Date: Wed, 12 Oct 2022 15:30:07 +0200
Subject: [PATCH] if_ovpn(4): implement ioctl() to set if_flags

Fully working openvpn(8) --iroute support needs real subnet config on ovpn(4) interfaces (IFF_BROADCAST), while client-side/p2p configs need IFF_POINTOPOINT setting. So make this configurable.

---
 sys/net/if_ovpn.c | 41 +++++++++++++++++++++++++++++++++++++++++
 sys/net/if_ovpn.h |  1 +
 2 files changed, 42 insertions(+)
sys/net/if_ovpn.c +41 -0
index ed0ff178972..dfb475e3072 100644
@@ -1081,6 +1081,44 @@ ovpn_set_peer(struct ifnet *ifp, const nvlist_t *nvl)
 	return (0);
 }
 
+static int
+ovpn_set_ifmode(struct ifnet *ifp, const nvlist_t *nvl)
+{
+	struct ovpn_softc *sc = ifp->if_softc;
+
+	if (nvl == NULL)
+		return (EINVAL);
+
+	if (! nvlist_exists_number(nvl, "ifmode") )
+		return (EINVAL);
+
+        int ifmode = nvlist_get_number(nvl, "ifmode");
+
+	OVPN_WLOCK(sc);
+
+	/* deny this if UP */
+	if (ifp->if_flags & IFF_UP) {
+		OVPN_WUNLOCK(sc);
+		return (EBUSY);
+	}
+
+	switch (ifmode & ~IFF_MULTICAST) {
+	case IFF_POINTOPOINT:
+	case IFF_BROADCAST:
+		ifp->if_flags &=
+		    ~(IFF_BROADCAST|IFF_POINTOPOINT|IFF_MULTICAST);
+		ifp->if_flags |= ifmode;
+		break;
+	default:
+		OVPN_WUNLOCK(sc);
+		return (EINVAL);
+	}
+
+	OVPN_WUNLOCK(sc);
+
+	return (0);
+}
+
 static int
 ovpn_ioctl_set(struct ifnet *ifp, struct ifdrv *ifd)
 {
@@ -1135,6 +1173,9 @@ ovpn_ioctl_set(struct ifnet *ifp, struct ifdrv *ifd)
 	case OVPN_SET_PEER:
 		ret = ovpn_set_peer(ifp, nvl);
 		break;
+	case OVPN_SET_IFMODE:
+		ret = ovpn_set_ifmode(ifp, nvl);
+		break;
 	default:
 		ret = ENOTSUP;
 	}
sys/net/if_ovpn.h +1 -0
index 26a9907711c..fd1c21e7043 100644
@@ -60,5 +60,6 @@ enum ovpn_key_cipher {
 #define OVPN_SEND_PKT		_IO  ('D', 9)
 #define OVPN_POLL_PKT		_IO  ('D', 10)
 #define OVPN_GET_PKT		_IO  ('D', 11)
+#define OVPN_SET_IFMODE		_IO  ('D', 12)
 
 #endif
Signature · 1 line
2.37.3
#16 lore raw

Re: [Openvpn-devel] route/iroute handling on FreeBSD

Hi Gert,

I’m still travelling today (and am due for my 5G chip update tomorrow), so it’ll be another day or two before I can look at this in any detail, but at first glance this looks sane.

Best regards,
Kristof

Quoted 54 lines · Gert Doering

Hi,

people have alreadycomplained at me that I write so long e-mails today, so I can write more...

Quoted 9 lines · Gert Doering

Factor 1: single-peer (client or p2p) vs. multi-peer

single-peer -> DCO has only 1 peer, all packets that go into the
tun/dco interface are sent out to the single peer ("dumb pipe mode") - exactly like tun(4) behaves

If a subnet is configured on the interface, packets to ALL IPs (!= local) in that subnet are sent to the other side. No next-hop lookup is done.

This is "sort of" handled in if_ovpn.c today

ovpn_route_peer(struct ovpn_softc *sc, struct mbuf **m0,
const struct sockaddr *dst)
{
...

        /* Shortcut if we're a client (or are a server and have only one client). */
        if (sc->peercount == 1)
                return (ovpn_find_only_peer(sc));

... so this works for the client, but has one interesting drawback on the server - if there is only a single client connected, the server will send ALL to-be-tunneled packets to that client. As soon as client #2 connects, packets are properly sorted.

[..]

Quoted 4 lines

Factor 2: IFF_POINTOPOINT vs. IFF_BROADCAST

This seems to be a *BSD-specific thing, aka "there is nothing in the Linux specific code that seems to bother with this".

I've whacked at if_ovpn.c and dco_freebsd.c a bit now, and I seem to have working code for both ends. I am not a FreeBSD kernel coder, so I have no idea how many behavioural standards I am violating, but it makes "real subnet mode in OpenVPN" work for me, with DCO.

Kernel patch attached, OpenVPN patches will follow soonish (outside of this e-mail thread).

gert

Signature · 6 lines
"If was one thing all people took for granted, was conviction that if you
 feed honest figures into a computer, honest figures come out. Never doubted
 it myself till I met a computer with a sense of humor."
                             Robert A. Heinlein, The Moon is a Harsh Mistress

Gert Doering - Munich, Germany                             [email protected]...
#17 lore raw

Re: [Openvpn-devel] route/iroute handling on FreeBSD

In reply to Gert Doering
Quoted 36 lines · Gert Doering

people have alreadycomplained at me that I write so long e-mails today, so I can write more...

Quoted 9 lines · Gert Doering

Factor 1: single-peer (client or p2p) vs. multi-peer

single-peer -> DCO has only 1 peer, all packets that go into the
tun/dco interface are sent out to the single peer ("dumb pipe mode") - exactly like tun(4) behaves

If a subnet is configured on the interface, packets to ALL IPs (!= local) in that subnet are sent to the other side. No next-hop lookup is done.

This is "sort of" handled in if_ovpn.c today

ovpn_route_peer(struct ovpn_softc *sc, struct mbuf **m0,
const struct sockaddr *dst)
{
...

        /* Shortcut if we're a client (or are a server and have only one client). */
        if (sc->peercount == 1)
                return (ovpn_find_only_peer(sc));

... so this works for the client, but has one interesting drawback on the server - if there is only a single client connected, the server will send ALL to-be-tunneled packets to that client. As soon as client #2 connects, packets are properly sorted.

[..]

Quoted 4 lines

Factor 2: IFF_POINTOPOINT vs. IFF_BROADCAST

This seems to be a *BSD-specific thing, aka "there is nothing in the Linux specific code that seems to bother with this".

Quoted 8 lines

I've whacked at if_ovpn.c and dco_freebsd.c a bit now, and I seem to have working code for both ends. I am not a FreeBSD kernel coder, so I have no idea how many behavioural standards I am violating, but it makes "real subnet mode in OpenVPN" work for me, with DCO.

Kernel patch attached, OpenVPN patches will follow soonish (outside of this e-mail thread).

I’m happy with that, and will commit (modulo one small style tweak) that patch in FreeBSD.

Best regards,
Kristof

#18 lore raw

PATCH 1/1 : FreeBSD networking cleanup

In reply to Gert Doering

I’ll post a patch.

Quoted 3 lines

- any particular reason you used argv_printf() + argv_printf_cat(), instead of just putting all into a single argv_printf() call?

Mostly that that’s what src/openvpn/networking_iproute2.c does too. I don’t have particularly strong feelings either way, so I’ll add that to the upcoming patch.

Quoted 5 lines

- the "openvpn_execve_check()" messages state "route *add* command failed", while it could be "del" as well. Making this "proper" might not be worth it, as it would need string manipulation (or "duplicate whole message").

It might be nice to turn openvpn_execve_check() into a variadic function so we could do this:

status = openvpn_execve_check(&argv, NULL, 0,
"ERROR: FreeBSD route %s command failed”, op);

But I think I’ll just remove the ‘add’, which makes the error message make sense for both add and del.

Best regards,
Kristof

#19 lore raw

PATCH 1/1 FreeBSD networking cleanup

From: Kristof Provost <[email protected]...>

Address a few minor code review remarks:

- use constants for the inet_ntop() buffers
- replace argv_printf() + argv_printf_cat() with a single argv_printf()
- net_route_v4/6 both add and remove, so adjust the error message to reflect that.

---
 src/openvpn/networking_freebsd.c | 34 +++++++++++++++-----------------
 1 file changed, 16 insertions(+), 18 deletions(-)
src/openvpn/networking_freebsd.c +16 -18
index 5fcb738e..0633dce7 100644
@@ -15,7 +15,7 @@ net_route_v4(const char *op, const in_addr_t *dst, int prefixlen,
              const in_addr_t *gw, const char *iface, uint32_t table,
              int metric)
 {
-    char buf1[16], buf2[16];
+    char buf1[INET_ADDRSTRLEN], buf2[INET_ADDRSTRLEN];
     in_addr_t _dst, _gw;
     struct argv argv = argv_new();
     bool status;
@@ -23,17 +23,16 @@ net_route_v4(const char *op, const in_addr_t *dst, int prefixlen,
     _dst = ntohl(*dst);
     _gw = ntohl(*gw);
 
-    argv_printf(&argv, "%s %s",
-                ROUTE_PATH, op);
-    argv_printf_cat(&argv, "-net %s/%d %s -fib %d",
-                    inet_ntop(AF_INET, &_dst, buf1, sizeof(buf1)),
-                    prefixlen,
-                    inet_ntop(AF_INET, &_gw, buf2, sizeof(buf2)),
-                    table);
+    argv_printf(&argv, "%s %s -net %s/%d %s -fib %d",
+                ROUTE_PATH, op,
+                inet_ntop(AF_INET, &_dst, buf1, sizeof(buf1)),
+                prefixlen,
+                inet_ntop(AF_INET, &_gw, buf2, sizeof(buf2)),
+                table);
 
     argv_msg(M_INFO, &argv);
     status = openvpn_execve_check(&argv, NULL, 0,
-                                  "ERROR: FreeBSD route add command failed");
+                                  "ERROR: FreeBSD route command failed");
 
     argv_free(&argv);
 
@@ -45,21 +44,20 @@ net_route_v6(const char *op, const struct in6_addr *dst,
              int prefixlen, const struct in6_addr *gw, const char *iface,
              uint32_t table, int metric)
 {
-    char buf1[64], buf2[64];
+    char buf1[INET6_ADDRSTRLEN], buf2[INET6_ADDRSTRLEN];
     struct argv argv = argv_new();
     bool status;
 
-    argv_printf(&argv, "%s -6 %s",
-                ROUTE_PATH, op);
-    argv_printf_cat(&argv, "-net %s/%d %s -fib %d",
-                    inet_ntop(AF_INET6, dst, buf1, sizeof(buf1)),
-                    prefixlen,
-                    inet_ntop(AF_INET6, gw, buf2, sizeof(buf2)),
-                    table);
+    argv_printf(&argv, "%s -6 %s -net %s/%d %s -fib %d",
+                ROUTE_PATH, op,
+                inet_ntop(AF_INET6, dst, buf1, sizeof(buf1)),
+                prefixlen,
+                inet_ntop(AF_INET6, gw, buf2, sizeof(buf2)),
+                table);
 
     argv_msg(M_INFO, &argv);
     status = openvpn_execve_check(&argv, NULL, 0,
-                                  "ERROR: FreeBSD route add command failed");
+                                  "ERROR: FreeBSD route command failed");
 
     argv_free(&argv);
 
Signature · 1 line
2.37.2
Acked-by on v1 1/1
#20 lore raw

[Openvpn-devel] [PATCH applied] Re: FreeBSD networking cleanup

Thanks. Tested on FreeBSD with and without DCO (the latter is more for good measure, as the changes only influence the FreeBSD-iroute part).

Your patch has been applied to the master branch.

commit b5b132c1ba36e6d2ca261d15a9d70648890021f7

Author: Kristof Provost
Date:   Mon Aug 22 11:28:34 2022 +0200
     FreeBSD networking cleanup

     Signed-off-by: Kristof Provost <[email protected]...>
     Acked-by: Gert Doering <[email protected]...>
     Message-Id: <[email protected]...>
     URL: https://www.mail-archive.com/[email protected]/msg25054.html
     Signed-off-by: Gert Doering <[email protected]...>

--
kind regards,

Gert Doering

#21 lore raw
In reply to Kristof Provost

Hi,

Quoted 2 lines · Kristof Provost via Openvpn-devel

Here's the updated version of the FreeBSD DCO patch. It should address all remarks on the userspace side of things.

As I said, I only tested the client side of things so far. Now I started building a server test rig, to see what I could break :-) - and then see if the iroute patch does what it should (looks generally good).

Before getting to routing, I discovered something that "upstream changes" might have broken in your patch:

Namely, if I run multiple instances with --dev tun, the first one grabs "tun0", and the next one will *fail*

2022-08-19 19:21:08 us=399955 OpenVPN 2.6_git [git:vw/master/d7f16eea8e939b42] amd64-unknown-freebsd14.0 [SSL (OpenSSL)] [LZO] [LZ4] [MH/RECVDA] [AEAD] [DCO] built on Aug 13 2022
2022-08-19 19:21:08 us=399974 library versions: OpenSSL 1.1.1q-freebsd  5 Jul 2022, LZO 2.10
[...]
2022-08-19 19:21:08 us=403141 GDG6: remote_host_ipv6=n/a
2022-08-19 19:21:08 us=403225 ROUTE6_GATEWAY 2001:608:0:814::ffff IFACE=em0
2022-08-19 19:21:08 us=414627 Failed to create interface tun0: File exists (errno=17)
2022-08-19 19:21:08 us=414703 Exiting due to fatal error

This is because of a misunderstanding between tun.c:open_tun_dco_generic() and FreeBSD's open_tun_dco() / create_interface() - tun.c expects the return code to be "-errno", and gives up on -EPERM - which happens to be "-1" on FreeBSD ("if we can not create a single interface due to *permission*, break the loop").

Changing the flow to return -errno makes it succeed. I will send a patch for that in a few minutes (feed family first).

gert

Signature · 6 lines
"If was one thing all people took for granted, was conviction that if you 
 feed honest figures into a computer, honest figures come out. Never doubted 
 it myself till I met a computer with a sense of humor."
                             Robert A. Heinlein, The Moon is a Harsh Mistress

Gert Doering - Munich, Germany                             [email protected]...