Skip to contentSkip to conversation

: FreeBSD DCO updates

PATCH New Kristof Provost 5 participants 18 messages Dec 5, 2022 – Dec 14, 2022

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 : FreeBSD DCO updates

Hi,

This patch series is version two of the "Read DCO traffic stats from the kernel" patch. This version should build for FreeBSD, Linux and Windows now (although I've not tested Windows builds, because I lack Windows machines).

It also adds the ability for DCO drivers to include counters in the peer deletion notification.

It also includes extensions to read the reason a peer went away from the kernel (which the FreeBSD kernel started including from
https://cgit.freebsd.org/src/commit/?id=da69782bf06645f38852a8b23afc965fc30d0e08)
Happily this will still work even when the kernel doesn't include that data.

Finally there's a small cleanup patch to remove reading packets through the ioctl interface.

If anyone wants to play around with the peer counters on FreeBSD you'll also want these patches for FreeBSD:

* https://reviews.freebsd.org/D37602 (not related, but required to have the next patches apply cleanly)
* https://reviews.freebsd.org/D37603
* https://reviews.freebsd.org/D37604
* https://reviews.freebsd.org/D37605 (not related, but required to have the next one apply cleanly)
* https://reviews.freebsd.org/D37606

Best regards,
Kristof

#1 lore raw

PATCH 1/4 Read DCO traffic stats from the kernel

From: Kristof Provost <[email protected]...>

When DCO is active userspace doesn't see all of the traffic, so when we access these stats we must update them.

Retrieve kernel statistics every time we access the link_(read|write)_bytes values.

Introduce a dco_(read|write)_bytes so that we don't clobber the existing statistics, which still count control packets, sent or received directly through the socket.

---
 src/openvpn/dco.h              |  8 ++++
 src/openvpn/dco_freebsd.c      | 78 ++++++++++++++++++++++++++++++++++
 src/openvpn/dco_linux.c        |  7 +++
 src/openvpn/dco_win.c          |  7 +++
 src/openvpn/multi.c            | 30 +++++++------
 src/openvpn/openvpn.h          |  2 +
 src/openvpn/ovpn_dco_freebsd.h |  1 +
 7 files changed, 120 insertions(+), 13 deletions(-)
src/openvpn/dco.h +8 -0
index e051db06..e5d89358 100644
@@ -225,6 +225,14 @@ void dco_install_iroute(struct multi_context *m, struct multi_instance *mi,
  */
 void dco_delete_iroutes(struct multi_context *m, struct multi_instance *mi);
 
+/**
+ * Update traffic statistics for all peers
+ *
+ * @param dco	DCO device context
+ * @param m	the server context
+ **/
+int dco_get_peer_stats(dco_context_t *dco, struct multi_context *m);
+
 /**
  * Retrieve the list of ciphers supported by the current platform
  *
src/openvpn/dco_freebsd.c +78 -0

90 lines not on this page · raw message

src/openvpn/dco_linux.c +7 -0
index 10935820..0306cec3 100644
@@ -911,6 +911,13 @@ nla_put_failure:
     return ret;
 }
 
+int
+dco_get_peer_stats(dco_context_t *dco, struct multi_context *m)
+{
+    /* Not implemented. */
+    return 0;
+}
+
 bool
 dco_available(int msglevel)
 {
src/openvpn/dco_win.c +7 -0
index 48a1755a..68ec931c 100644
@@ -399,6 +399,13 @@ dco_do_write(dco_context_t *dco, int peer_id, struct buffer *buf)
     return 0;
 }
 
+int
+dco_get_peer_stats(dco_context_t *dco, struct multi_context *m)
+{
+    /* Not implemented. */
+    return 0;
+}
+
 void
 dco_event_set(dco_context_t *dco, struct event_set *es, void *arg)
 {
src/openvpn/multi.c +17 -13
index 0a23c2bc..38da87b8 100644
@@ -538,29 +538,31 @@ multi_del_iroutes(struct multi_context *m,
 }
 
 static void
-setenv_stats(struct context *c)
+setenv_stats(struct multi_context *m, struct context *c)
 {
-    setenv_counter(c->c2.es, "bytes_received", c->c2.link_read_bytes);
-    setenv_counter(c->c2.es, "bytes_sent", c->c2.link_write_bytes);
+    dco_get_peer_stats(&m->top.c1.tuntap->dco, m);
+
+    setenv_counter(c->c2.es, "bytes_received", c->c2.link_read_bytes + c->c2.dco_read_bytes);
+    setenv_counter(c->c2.es, "bytes_sent", c->c2.link_write_bytes + c->c2.dco_write_bytes);
 }
 
 static void
-multi_client_disconnect_setenv(struct multi_instance *mi)
+multi_client_disconnect_setenv(struct multi_context *m, struct multi_instance *mi)
 {
     /* setenv client real IP address */
     setenv_trusted(mi->context.c2.es, get_link_socket_info(&mi->context));
 
     /* setenv stats */
-    setenv_stats(&mi->context);
+    setenv_stats(m, &mi->context);
 
     /* setenv connection duration */
     setenv_long_long(mi->context.c2.es, "time_duration", now - mi->created);
 }
 
 static void
-multi_client_disconnect_script(struct multi_instance *mi)
+multi_client_disconnect_script(struct multi_context *m, struct multi_instance *mi)
 {
-    multi_client_disconnect_setenv(mi);
+    multi_client_disconnect_setenv(m, mi);
 
     if (plugin_defined(mi->context.plugins, OPENVPN_PLUGIN_CLIENT_DISCONNECT))
     {
@@ -667,7 +669,7 @@ multi_close_instance(struct multi_context *m,
 
     if (mi->context.c2.tls_multi->multi_state >= CAS_CONNECT_DONE)
     {
-        multi_client_disconnect_script(mi);
+        multi_client_disconnect_script(m, mi);
     }
 
     close_context(&mi->context, SIGTERM, CC_GC_FREE);
@@ -837,6 +839,8 @@ multi_print_status(struct multi_context *m, struct status_output *so, const int
 
         status_reset(so);
 
+        dco_get_peer_stats(&m->top.c1.tuntap->dco, m);
+
         if (version == 1)
         {
             /*
@@ -856,8 +860,8 @@ multi_print_status(struct multi_context *m, struct status_output *so, const int
                     status_printf(so, "%s,%s," counter_format "," counter_format ",%s",
                                   tls_common_name(mi->context.c2.tls_multi, false),
                                   mroute_addr_print(&mi->real, &gc),
-                                  mi->context.c2.link_read_bytes,
-                                  mi->context.c2.link_write_bytes,
+                                  mi->context.c2.link_read_bytes + mi->context.c2.dco_read_bytes,
+                                  mi->context.c2.link_write_bytes + mi->context.c2.dco_write_bytes,
                                   time_string(mi->created, 0, false, &gc));
                 }
                 gc_free(&gc);
@@ -932,8 +936,8 @@ multi_print_status(struct multi_context *m, struct status_output *so, const int
                                   sep, mroute_addr_print(&mi->real, &gc),
                                   sep, print_in_addr_t(mi->reporting_addr, IA_EMPTY_IF_UNDEF, &gc),
                                   sep, print_in6_addr(mi->reporting_addr_ipv6, IA_EMPTY_IF_UNDEF, &gc),
-                                  sep, mi->context.c2.link_read_bytes,
-                                  sep, mi->context.c2.link_write_bytes,
+                                  sep, mi->context.c2.link_read_bytes + mi->context.c2.dco_read_bytes,
+                                  sep, mi->context.c2.link_write_bytes + mi->context.c2.dco_write_bytes,
                                   sep, time_string(mi->created, 0, false, &gc),
                                   sep, (unsigned int)mi->created,
                                   sep, tls_username(mi->context.c2.tls_multi, false),
@@ -2752,7 +2756,7 @@ multi_connection_established(struct multi_context *m, struct multi_instance *mi)
          * did not fail */
         if (mi->context.c2.tls_multi->multi_state == CAS_PENDING_DEFERRED_PARTIAL)
         {
-            multi_client_disconnect_script(mi);
+            multi_client_disconnect_script(m, mi);
         }
 
         mi->context.c2.tls_multi->multi_state = CAS_FAILED;
src/openvpn/openvpn.h +2 -0
index c543cbf6..5981e4d5 100644
@@ -267,8 +267,10 @@ struct context_2
     counter_type tun_read_bytes;
     counter_type tun_write_bytes;
     counter_type link_read_bytes;
+    counter_type dco_read_bytes;
     counter_type link_read_bytes_auth;
     counter_type link_write_bytes;
+    counter_type dco_write_bytes;
 #ifdef PACKET_TRUNCATION_CHECK
     counter_type n_trunc_tun_read;
     counter_type n_trunc_tun_write;
src/openvpn/ovpn_dco_freebsd.h +1 -0
index cf92d597..cc90111e 100644
@@ -61,5 +61,6 @@ enum ovpn_key_cipher {
 #define OVPN_POLL_PKT           _IO('D', 10)
 #define OVPN_GET_PKT            _IO('D', 11)
 #define OVPN_SET_IFMODE         _IO('D', 12)
+#define OVPN_GET_PEER_STATS     _IO('D', 13)
 
 #endif /* ifndef _NET_IF_OVPN_H_ */
Signature · 1 line
2.38.1
#2 lore raw

Re: [Openvpn-devel] [PATCH 1/4] Read DCO traffic stats from the kernel

Hi,

This is good - I need an API to get stats to make openvpn-gui show those (via the management interface).

I am not too much into FreeBSD parts, but

src/openvpn/dco_freebsd.c dco_event_set(dco_context_t *dco, struct event_set *es, void *arg) · line 658

Quoted 8 lines
src/openvpn/dco_freebsd.c +8 -0
+    hash_iterator_init(m->hash, &hi);
+
+    while ((he = hash_iterator_next(&hi)))
+    {
+        struct multi_instance *mi = (struct multi_instance *) he->value;
+
+        if (mi->context.c2.tls_multi->peer_id != peerid)
+            continue;

Shouldn't we use m->instances[peerid] instead of iterating over m->hash?

from multi.h:

struct multi_context {
struct multi_instance **instances; /**< Array of multi_instances.
An instance can be
* accessed using peer-id as
an index. */

Signature · 1 line
-Lev
#3 lore raw

Re: [Openvpn-devel] [PATCH 1/4] Read DCO traffic stats from the kernel

Am 12.12.22 um 13:34 schrieb Lev Stipakov:

Quoted 7 lines

Hi,

This is good - I need an API to get stats to make openvpn-gui show those (via the management interface).

I am not too much into FreeBSD parts, but

Quoted 11 lines

src/openvpn/dco_freebsd.c dco_event_set(dco_context_t *dco, struct event_set *es, void *arg) · line 658

Quoted 8 lines
src/openvpn/dco_freebsd.c +8 -0
+    hash_iterator_init(m->hash, &hi);
+
+    while ((he = hash_iterator_next(&hi)))
+    {
+        struct multi_instance *mi = (struct multi_instance *) he->value;
+
+        if (mi->context.c2.tls_multi->peer_id != peerid)
+            continue;

Shouldn't we use m->instances[peerid] instead of iterating over m->hash?

Yes and a check that kernel does not give something > max_peer

Arne

#4 lore raw

Re: [Openvpn-devel] [PATCH 1/4] Read DCO traffic stats from the kernel

Hi,

Quoted 13 lines · Arne Schwabe
Quoted 12 lines

I am not too much into FreeBSD parts, but

src/openvpn/dco_freebsd.c dco_event_set(dco_context_t *dco, struct event_set *es, void *arg) · line 658

Quoted 8 lines
src/openvpn/dco_freebsd.c +8 -0
+    hash_iterator_init(m->hash, &hi);
+
+    while ((he = hash_iterator_next(&hi)))
+    {
+        struct multi_instance *mi = (struct multi_instance *) he->value;
+
+        if (mi->context.c2.tls_multi->peer_id != peerid)
+            continue;

Shouldn't we use m->instances[peerid] instead of iterating over m->hash?

Quoted 1 line

Yes and a check that kernel does not give something > max_peer

I agree, doing the iterator here for every single active peer when we already have the array. But then, of course, p2p peer-id will be "something random", not constrained by max-clients.

Anyway, we have a procedural problem here - kp@ is on vacation for a few weeks now, so we won't see a v2 of this patch any time soon.

Do you think the patch is ready to be merged, "as is", with not-so-good efficiency? One of you could then send a followup patch changing this to use the array accessor.

gert

Signature · 6 lines
"If was one thing all people took for granted, was conviction that if you 
 feed honest figures into a computer, honest figures come out. Never doubted 
 it myself till I met a computer with a sense of humor."
                             Robert A. Heinlein, The Moon is a Harsh Mistress

Gert Doering - Munich, Germany                             [email protected]...
#5 lore raw

Re: [Openvpn-devel] [PATCH 1/4] Read DCO traffic stats from the kernel

Am 12.12.22 um 14:30 schrieb Gert Doering:

Quoted 16 lines

Hi,

Quoted 13 lines · Arne Schwabe
Quoted 12 lines

I am not too much into FreeBSD parts, but

src/openvpn/dco_freebsd.c dco_event_set(dco_context_t *dco, struct event_set *es, void *arg) · line 658

Quoted 8 lines
src/openvpn/dco_freebsd.c +8 -0
+    hash_iterator_init(m->hash, &hi);
+
+    while ((he = hash_iterator_next(&hi)))
+    {
+        struct multi_instance *mi = (struct multi_instance *) he->value;
+
+        if (mi->context.c2.tls_multi->peer_id != peerid)
+            continue;

Shouldn't we use m->instances[peerid] instead of iterating over m->hash?

Quoted 5 lines
Quoted 1 line

Yes and a check that kernel does not give something > max_peer

I agree, doing the iterator here for every single active peer when we already have the array. But then, of course, p2p peer-id will be "something random", not constrained by max-clients.

on p2mp code, it will be contrainst by max-clients. The random peerid only happens in p2p mode where do not have multi_instance anyway.

Quoted 7 lines

Anyway, we have a procedural problem here - kp@ is on vacation for a few weeks now, so we won't see a v2 of this patch any time soon.

Do you think the patch is ready to be merged, "as is", with not-so-good efficiency? One of you could then send a followup patch changing this to use the array accessor.

Fine by me.

Arne

#6 lore raw

Re: [Openvpn-devel] [PATCH 1/4] Read DCO traffic stats from the kernel

In reply to Kristof Provost

Hi,

On 05/12/2022 17:41, Kristof Provost via Openvpn-devel wrote:
[cut]

Quoted 10 lines
src/openvpn/dco_freebsd.c +10 -0
+
+int
+dco_get_peer_stats(dco_context_t *dco, struct multi_context *m)
+{
+
+    struct ifdrv drv;
+    uint8_t buf[4096];
+    nvlist_t *nvl;
+    const nvlist_t *const *nvpeers;
+    size_t npeers;

src/openvpn/dco_freebsd.c dco_event_set(dco_context_t *dco, struct event_set *es, void *arg) · line 687

Quoted 10 lines
src/openvpn/dco_freebsd.c +10 -0
+    int ret;
+
+    if (!dco || !dco->open)
+    {
+        return 0;
+    }
+
+    CLEAR(drv);
+    snprintf(drv.ifd_name, IFNAMSIZ, "%s", dco->ifname);
+    drv.ifd_cmd = OVPN_GET_PEER_STATS;

What speaks against having a simple GET_PEER here which returns the full peer object with all possible attributes?

This way, if we want to retrieve another attribute in the future, this attribute will already be delivered by the same API, without the need to implement a new command each time.

Cheers,

Signature · 1 line
Antonio Quartulli
#7 lore raw

Re: [Openvpn-devel] [PATCH 1/4] Read DCO traffic stats from the kernel

Hi,

Quoted 25 lines · Antonio Quartulli

On 05/12/2022 17:41, Kristof Provost via Openvpn-devel wrote:
[cut]

Quoted 9 lines
src/openvpn/dco_freebsd.c +9 -0
+int
+dco_get_peer_stats(dco_context_t *dco, struct multi_context *m)
+{
+
+    struct ifdrv drv;
+    uint8_t buf[4096];
+    nvlist_t *nvl;
+    const nvlist_t *const *nvpeers;
+    size_t npeers;

src/openvpn/dco_freebsd.c dco_event_set(dco_context_t *dco, struct event_set *es, void *arg) · line 687

Quoted 10 lines
src/openvpn/dco_freebsd.c +10 -0
+    int ret;
+
+    if (!dco || !dco->open)
+    {
+        return 0;
+    }
+
+    CLEAR(drv);
+    snprintf(drv.ifd_name, IFNAMSIZ, "%s", dco->ifname);
+    drv.ifd_cmd = OVPN_GET_PEER_STATS;

What speaks against having a simple GET_PEER here which returns the full peer object with all possible attributes?

Quoted 3 lines

This way, if we want to retrieve another attribute in the future, this attribute will already be delivered by the same API, without the need to implement a new command each time.

One counter-argument would be "the statistics are polled more often than 'all information about a peer object', so it should be less costly" (also, when do we ever poll 'all attributes'? Since OpenVPN *sets* these, we should know, except for the counters...)

gert

Signature · 6 lines
"If was one thing all people took for granted, was conviction that if you 
 feed honest figures into a computer, honest figures come out. Never doubted 
 it myself till I met a computer with a sense of humor."
                             Robert A. Heinlein, The Moon is a Harsh Mistress

Gert Doering - Munich, Germany                             [email protected]...
#8 lore raw

Re: [Openvpn-devel] [PATCH 1/4] Read DCO traffic stats from the kernel

Hi,

Quoted 28 lines · Gert Doering

Hi,

Quoted 25 lines · Antonio Quartulli

On 05/12/2022 17:41, Kristof Provost via Openvpn-devel wrote:
[cut]

Quoted 9 lines
src/openvpn/dco_freebsd.c +9 -0
+int
+dco_get_peer_stats(dco_context_t *dco, struct multi_context *m)
+{
+
+    struct ifdrv drv;
+    uint8_t buf[4096];
+    nvlist_t *nvl;
+    const nvlist_t *const *nvpeers;
+    size_t npeers;

src/openvpn/dco_freebsd.c dco_event_set(dco_context_t *dco, struct event_set *es, void *arg) · line 687

Quoted 10 lines
src/openvpn/dco_freebsd.c +10 -0
+    int ret;
+
+    if (!dco || !dco->open)
+    {
+        return 0;
+    }
+
+    CLEAR(drv);
+    snprintf(drv.ifd_name, IFNAMSIZ, "%s", dco->ifname);
+    drv.ifd_cmd = OVPN_GET_PEER_STATS;

What speaks against having a simple GET_PEER here which returns the full peer object with all possible attributes?

Quoted 6 lines
Quoted 3 lines

This way, if we want to retrieve another attribute in the future, this attribute will already be delivered by the same API, without the need to implement a new command each time.

One counter-argument would be "the statistics are polled more often than 'all information about a peer object', so it should be less costly"

It's a netlink/ioctl API: it's already costly and out of any fast-path, therefore I don't think this argument really plays an important role here.

What is more important is designing an API that can last long, without the need for extra cluttering when we decide to implement something new.

Ideally a GET_PEER command is pretty standard and can also be used for any kind of state inspection (i.e. even for 'debugging', although it's not the primary usage)

Quoted 2 lines

(also, when do we ever poll 'all attributes'? Since OpenVPN *sets* these, we should know, except for the counters...)

Like I said above, being this an API imho it is important to implement it in a way that can serve a reasonably broad set of use cases. (OpenVPN is the only user now, but may not be the case later)

Cheers,

Signature · 1 line
Antonio Quartulli
#9 lore raw

Re: [Openvpn-devel] [PATCH 1/4] Read DCO traffic stats from the kernel

Hi,

Quoted 3 lines · Antonio Quartulli

Ideally a GET_PEER command is pretty standard and can also be used for any kind of state inspection (i.e. even for 'debugging', although it's not the primary usage)

[..]

Quoted 3 lines

Like I said above, being this an API imho it is important to implement it in a way that can serve a reasonably broad set of use cases. (OpenVPN is the only user now, but may not be the case later)

I think there is nothing that stops us from doing GET_PEER on Linux, if that's already there and a proper "people agree that this is the way" API, and using the API we have for FreeBSD for the time being.

The Linux counter support patch might want rename the OS-independent functions accordingly then :-)

gert

Signature · 6 lines
"If was one thing all people took for granted, was conviction that if you 
 feed honest figures into a computer, honest figures come out. Never doubted 
 it myself till I met a computer with a sense of humor."
                             Robert A. Heinlein, The Moon is a Harsh Mistress

Gert Doering - Munich, Germany                             [email protected]...
Acked-by on v1 1/4
#10 lore raw

Re: [Openvpn-devel] [PATCH 1/4] Read DCO traffic stats from the kernel

In reply to Kristof Provost

Hi,

Quoted 5 lines · Kristof Provost via Openvpn-devel

From: Kristof Provost <[email protected]...>

When DCO is active userspace doesn't see all of the traffic, so when we access these stats we must update them.

Quoted 8 lines

Retrieve kernel statistics every time we access the link_(read|write)_bytes values.

Introduce a dco_(read|write)_bytes so that we don't clobber the existing statistics, which still count control packets, sent or received directly through the socket.

Quoted 279 lines
---
  src/openvpn/dco.h              |  8 ++++
  src/openvpn/dco_freebsd.c      | 78 ++++++++++++++++++++++++++++++++++
  src/openvpn/dco_linux.c        |  7 +++
  src/openvpn/dco_win.c          |  7 +++
  src/openvpn/multi.c            | 30 +++++++------
  src/openvpn/openvpn.h          |  2 +
  src/openvpn/ovpn_dco_freebsd.h |  1 +
  7 files changed, 120 insertions(+), 13 deletions(-)
src/openvpn/dco.h +8 -0
index e051db06..e5d89358 100644
@@ -225,6 +225,14 @@ void dco_install_iroute(struct multi_context *m, struct multi_instance *mi,
   */
  void dco_delete_iroutes(struct multi_context *m, struct multi_instance *mi);
 
+/**
+ * Update traffic statistics for all peers
+ *
+ * @param dco	DCO device context
+ * @param m	the server context
+ **/
+int dco_get_peer_stats(dco_context_t *dco, struct multi_context *m);
+
  /**
   * Retrieve the list of ciphers supported by the current platform
   *
src/openvpn/dco_freebsd.c +78 -0
index a52ac8c1..5b352859 100644
@@ -37,6 +37,7 @@
  #include "dco.h"
  #include "tun.h"
  #include "crypto.h"
+#include "multi.h"
  #include "ssl_common.h"
 
  static nvlist_t *
@@ -641,6 +642,83 @@ dco_event_set(dco_context_t *dco, struct event_set *es, void *arg)
      nvlist_destroy(nvl);
  }
 
+static void
+dco_update_peer_stat(struct multi_context *m, uint32_t peerid, const nvlist_t *nvl)
+{
+    struct hash_element *he;
+    struct hash_iterator hi;
+
+    hash_iterator_init(m->hash, &hi);
+
+    while ((he = hash_iterator_next(&hi)))
+    {
+        struct multi_instance *mi = (struct multi_instance *) he->value;
+
+        if (mi->context.c2.tls_multi->peer_id != peerid)
+            continue;
+
+        mi->context.c2.dco_read_bytes = nvlist_get_number(nvl, "in");
+        mi->context.c2.dco_write_bytes = nvlist_get_number(nvl, "out");
+
+        return;
+    }
+
+    msg(M_INFO, "Peer %d returned by kernel, but not found locally", peerid);
+}
+
+int
+dco_get_peer_stats(dco_context_t *dco, struct multi_context *m)
+{
+
+    struct ifdrv drv;
+    uint8_t buf[4096];
+    nvlist_t *nvl;
+    const nvlist_t *const *nvpeers;
+    size_t npeers;
+    int ret;
+
+    if (!dco || !dco->open)
+    {
+        return 0;
+    }
+
+    CLEAR(drv);
+    snprintf(drv.ifd_name, IFNAMSIZ, "%s", dco->ifname);
+    drv.ifd_cmd = OVPN_GET_PEER_STATS;
+    drv.ifd_len = sizeof(buf);
+    drv.ifd_data = buf;
+
+    ret = ioctl(dco->fd, SIOCGDRVSPEC, &drv);
+    if (ret)
+    {
+        msg(M_WARN | M_ERRNO, "Failed to get peer stats");
+        return -EINVAL;
+    }
+
+    nvl = nvlist_unpack(buf, drv.ifd_len, 0);
+    if (! nvl)
+    {
+        msg(M_WARN, "Failed to unpack nvlist");
+        return -EINVAL;
+    }
+
+    if (! nvlist_exists_nvlist_array(nvl, "peers")) {
+        /* no peers */
+        return 0;
+    }
+
+    nvpeers = nvlist_get_nvlist_array(nvl, "peers", &npeers);
+    for (size_t i = 0; i < npeers; i++)
+    {
+        const nvlist_t *peer = nvpeers[i];
+        uint32_t peerid = nvlist_get_number(peer, "peerid");
+
+        dco_update_peer_stat(m, peerid, nvlist_get_nvlist(peer, "bytes"));
+    }
+
+    return 0;
+}
+
  const char *
  dco_get_supported_ciphers()
  {
src/openvpn/dco_linux.c +7 -0
index 10935820..0306cec3 100644
@@ -911,6 +911,13 @@ nla_put_failure:
      return ret;
  }
 
+int
+dco_get_peer_stats(dco_context_t *dco, struct multi_context *m)
+{
+    /* Not implemented. */
+    return 0;
+}
+
  bool
  dco_available(int msglevel)
  {
src/openvpn/dco_win.c +7 -0
index 48a1755a..68ec931c 100644
@@ -399,6 +399,13 @@ dco_do_write(dco_context_t *dco, int peer_id, struct buffer *buf)
      return 0;
  }
 
+int
+dco_get_peer_stats(dco_context_t *dco, struct multi_context *m)
+{
+    /* Not implemented. */
+    return 0;
+}
+
  void
  dco_event_set(dco_context_t *dco, struct event_set *es, void *arg)
  {
src/openvpn/multi.c +17 -13
index 0a23c2bc..38da87b8 100644
@@ -538,29 +538,31 @@ multi_del_iroutes(struct multi_context *m,
  }
 
  static void
-setenv_stats(struct context *c)
+setenv_stats(struct multi_context *m, struct context *c)
  {
-    setenv_counter(c->c2.es, "bytes_received", c->c2.link_read_bytes);
-    setenv_counter(c->c2.es, "bytes_sent", c->c2.link_write_bytes);
+    dco_get_peer_stats(&m->top.c1.tuntap->dco, m);
+
+    setenv_counter(c->c2.es, "bytes_received", c->c2.link_read_bytes + c->c2.dco_read_bytes);
+    setenv_counter(c->c2.es, "bytes_sent", c->c2.link_write_bytes + c->c2.dco_write_bytes);
  }
 
  static void
-multi_client_disconnect_setenv(struct multi_instance *mi)
+multi_client_disconnect_setenv(struct multi_context *m, struct multi_instance *mi)
  {
      /* setenv client real IP address */
      setenv_trusted(mi->context.c2.es, get_link_socket_info(&mi->context));
 
      /* setenv stats */
-    setenv_stats(&mi->context);
+    setenv_stats(m, &mi->context);
 
      /* setenv connection duration */
      setenv_long_long(mi->context.c2.es, "time_duration", now - mi->created);
  }
 
  static void
-multi_client_disconnect_script(struct multi_instance *mi)
+multi_client_disconnect_script(struct multi_context *m, struct multi_instance *mi)
  {
-    multi_client_disconnect_setenv(mi);
+    multi_client_disconnect_setenv(m, mi);
 
      if (plugin_defined(mi->context.plugins, OPENVPN_PLUGIN_CLIENT_DISCONNECT))
      {
@@ -667,7 +669,7 @@ multi_close_instance(struct multi_context *m,
 
      if (mi->context.c2.tls_multi->multi_state >= CAS_CONNECT_DONE)
      {
-        multi_client_disconnect_script(mi);
+        multi_client_disconnect_script(m, mi);
      }
 
      close_context(&mi->context, SIGTERM, CC_GC_FREE);
@@ -837,6 +839,8 @@ multi_print_status(struct multi_context *m, struct status_output *so, const int
 
          status_reset(so);
 
+        dco_get_peer_stats(&m->top.c1.tuntap->dco, m);
+
          if (version == 1)
          {
              /*
@@ -856,8 +860,8 @@ multi_print_status(struct multi_context *m, struct status_output *so, const int
                      status_printf(so, "%s,%s," counter_format "," counter_format ",%s",
                                    tls_common_name(mi->context.c2.tls_multi, false),
                                    mroute_addr_print(&mi->real, &gc),
-                                  mi->context.c2.link_read_bytes,
-                                  mi->context.c2.link_write_bytes,
+                                  mi->context.c2.link_read_bytes + mi->context.c2.dco_read_bytes,
+                                  mi->context.c2.link_write_bytes + mi->context.c2.dco_write_bytes,
                                    time_string(mi->created, 0, false, &gc));
                  }
                  gc_free(&gc);
@@ -932,8 +936,8 @@ multi_print_status(struct multi_context *m, struct status_output *so, const int
                                    sep, mroute_addr_print(&mi->real, &gc),
                                    sep, print_in_addr_t(mi->reporting_addr, IA_EMPTY_IF_UNDEF, &gc),
                                    sep, print_in6_addr(mi->reporting_addr_ipv6, IA_EMPTY_IF_UNDEF, &gc),
-                                  sep, mi->context.c2.link_read_bytes,
-                                  sep, mi->context.c2.link_write_bytes,
+                                  sep, mi->context.c2.link_read_bytes + mi->context.c2.dco_read_bytes,
+                                  sep, mi->context.c2.link_write_bytes + mi->context.c2.dco_write_bytes,
                                    sep, time_string(mi->created, 0, false, &gc),
                                    sep, (unsigned int)mi->created,
                                    sep, tls_username(mi->context.c2.tls_multi, false),
@@ -2752,7 +2756,7 @@ multi_connection_established(struct multi_context *m, struct multi_instance *mi)
           * did not fail */
          if (mi->context.c2.tls_multi->multi_state == CAS_PENDING_DEFERRED_PARTIAL)
          {
-            multi_client_disconnect_script(mi);
+            multi_client_disconnect_script(m, mi);
          }
 
          mi->context.c2.tls_multi->multi_state = CAS_FAILED;
src/openvpn/openvpn.h +2 -0
index c543cbf6..5981e4d5 100644
@@ -267,8 +267,10 @@ struct context_2
      counter_type tun_read_bytes;
      counter_type tun_write_bytes;
      counter_type link_read_bytes;
+    counter_type dco_read_bytes;
      counter_type link_read_bytes_auth;
      counter_type link_write_bytes;
+    counter_type dco_write_bytes;
  #ifdef PACKET_TRUNCATION_CHECK
      counter_type n_trunc_tun_read;
      counter_type n_trunc_tun_write;
src/openvpn/ovpn_dco_freebsd.h +1 -0
index cf92d597..cc90111e 100644
@@ -61,5 +61,6 @@ enum ovpn_key_cipher {
  #define OVPN_POLL_PKT           _IO('D', 10)
  #define OVPN_GET_PKT            _IO('D', 11)
  #define OVPN_SET_IFMODE         _IO('D', 12)
+#define OVPN_GET_PEER_STATS     _IO('D', 13)
 
  #endif /* ifndef _NET_IF_OVPN_H_ */
Signature · 1 line
Antonio Quartulli
#11 lore raw

[Openvpn-devel] [PATCH applied] Re: Read DCO traffic stats from the kernel

In reply to Kristof Provost

I've stared at the code for a bit (reasonable, with the enhancements to come from Arne / Lev), and tested on Linux without DCO (client/server), Linux with DCO (client/server) and FreeBSD with DCO (client/server), and everything worked.

To make this new code work for the "configure --disable-dco" case, I had to add an empty stub to dco.h

+static inline int
+dco_get_peer_stats(dco_context_t *dco, struct multi_context *m) +{
+ return 0;
+}
+

.. which is not something I normally do, but this is the same "non function" as for the "no linux and no windows implementation yet", and we want this to proceed now. It's quite obviously no code that does anything.

I have not tested the actual counter functionality, as the kernel side bits are not merged yet (in FreeBSD review process, but they all look quite straightforward). It will log warnings about not being able to get the stats...

Dec 13 22:40:03 fbsd14 tun-udp-p2mp-topology-subnet[19846]: Failed to get peer stats: Operation not supported (errno=45)

.. but this is "just cosmetic", if a bit annoying. Will go away as soon as the kernel bits are there :-)

Uncrustify demanded a few whitespace amendments, which I've done.

Your patch has been applied to the master and release/2.6 branch.

commit ce2b459dabc29d071be28b8ddaa0512f8c8143ec (master) commit 7695a55be06ccdcfae070b0cb3f0dfddca605003 (release/2.6)

Author: Kristof Provost
Date:   Mon Dec 5 17:41:00 2022 +0100
     Read DCO traffic stats from the kernel

     Signed-off-by: Kristof Provost <[email protected]...>
     Acked-by: Antonio Quartulli <[email protected]...>
     Message-Id: <[email protected]...>
     URL: https://www.mail-archive.com/[email protected]/msg25618.html
     Signed-off-by: Gert Doering <[email protected]...>

--
kind regards,

Gert Doering

#12 lore raw

PATCH 2/4 dco: Update counters when a client disconnects

In reply to Kristof Provost

From: Kristof Provost <[email protected]...>

When the kernel module (Linux or FreeBSD) notifies us that a peer has disconnected we'd like to get a final count of the in/out bytes for that peer.
We can't request that information any more, because the kernel has already removed the peer at that point.

Have the kernel send that information as part of the "delete peer" notification, and update the counters a final time.

This implements the FreeBSD-specific DCO code, but not the Linux-specific code. It will simply add 0 to the count on Linux.

---
 src/openvpn/dco_freebsd.c | 9 +++++++++
 src/openvpn/dco_freebsd.h | 2 ++
 src/openvpn/dco_linux.h   | 2 ++
 src/openvpn/multi.c       | 4 ++++
 4 files changed, 17 insertions(+)
src/openvpn/dco_freebsd.c +9 -0
index 5b352859..2ae46589 100644
@@ -528,6 +528,15 @@ dco_do_read(dco_context_t *dco)
     else
     {
         dco->dco_del_peer_reason = OVPN_DEL_PEER_REASON_EXPIRED;
+
+        if (nvlist_exists_nvlist(nvl, "bytes"))
+        {
+            const nvlist_t *bytes = nvlist_get_nvlist(nvl, "bytes");
+
+            dco->dco_read_bytes = nvlist_get_number(bytes, "in");
+            dco->dco_write_bytes = nvlist_get_number(bytes, "out");
+        }
+
         dco->dco_message_type = OVPN_CMD_DEL_PEER;
     }
 
src/openvpn/dco_freebsd.h +2 -0
index 7de11697..0d059dda 100644
@@ -55,6 +55,8 @@ typedef struct dco_context {
     int dco_message_type;
     int dco_message_peer_id;
     int dco_del_peer_reason;
+    uint64_t dco_read_bytes;
+    uint64_t dco_write_bytes;
 } dco_context_t;
 
 #endif /* defined(ENABLE_DCO) && defined(TARGET_FREEBSD) */
src/openvpn/dco_linux.h +2 -0
index 416ea30a..7d56308b 100644
@@ -53,6 +53,8 @@ typedef struct
     int dco_message_type;
     int dco_message_peer_id;
     int dco_del_peer_reason;
+    uint64_t dco_read_bytes;
+    uint64_t dco_write_bytes;
 } dco_context_t;
 
 #endif /* defined(ENABLE_DCO) && defined(TARGET_LINUX) */
src/openvpn/multi.c +4 -0
index 38da87b8..74671303 100644
@@ -3245,6 +3245,8 @@ process_incoming_del_peer(struct multi_context *m, struct multi_instance *mi,
      * installed, and we do not need to clean up the state in the kernel */
     mi->context.c2.tls_multi->dco_peer_id = -1;
     mi->context.sig->signal_text = reason;
+    mi->context.c2.dco_read_bytes = dco->dco_read_bytes;
+    mi->context.c2.dco_write_bytes = dco->dco_write_bytes;
     multi_signal_instance(m, mi, SIGTERM);
 }
 
@@ -3278,6 +3280,8 @@ multi_process_incoming_dco(struct multi_context *m)
 
     dco->dco_message_type = 0;
     dco->dco_message_peer_id = -1;
+    dco->dco_read_bytes = 0;
+    dco->dco_write_bytes = 0;
     return ret > 0;
 }
 #endif /* if defined(ENABLE_DCO) && defined(TARGET_LINUX) */
Signature · 1 line
2.38.1
Acked-by on v1 2/4
#13 lore raw

[Openvpn-devel] [PATCH applied] Re: dco: Update counters when a client disconnects

This is indeed an important missing piece for correct stats (... to be handed to --client-disconnect scripts, etc).

I have not tested the feature as such (kernel side support has landed but I have not yet rebuilt that system). I *have* tested the tree on Linux/FreeBSD with/without DCO, and everything works as before.

Looking at the code: storing these in dco_context "to be able to transport to the caller" sounds like an reasonable approach, because the dco_do_read() functions know "nothing else" - we could introduce call-by-ref variables, but that won't make the code easier to read.

The nested nvlists (bytes->{in,out}) here add a level of trust to the running kernel - should "in" be missing, nvlist_get_number(9) documents "the program will be aborted". The existance of "bytes" is checked, so old/new kernel compat is fine (OTOH if we do not trust the kernel module here, we're lost anyway). Just sayin'.

Your patch has been applied to the master and release/2.6 branch.

commit 6674963debfb88c0dd3dd4eae4533010ffc319b1 (master) commit f05c7a8a11329cc579d3751469b597e0eab1317f (HEAD -> release/2.6)

Author: Kristof Provost
Date:   Mon Dec 5 17:41:01 2022 +0100
     dco: Update counters when a client disconnects

     Signed-off-by: Kristof Provost <[email protected]...>
     Acked-by: Gert Doering <[email protected]...>
     Message-Id: <[email protected]...>
     URL: https://www.mail-archive.com/[email protected]/msg25614.html
     Signed-off-by: Gert Doering <[email protected]...>

--
kind regards,

Gert Doering

#14 lore raw

PATCH 3/4 Read the peer deletion reason from the kernel

In reply to Kristof Provost

From: Kristof Provost <[email protected]...>

Recent FreeBSD kernels supply a reason for the OVPN_NOTIF_DEL_PEER notification. Parse this from the nvlist so we can distinguish user-requested removals from timeouts.

---
 src/openvpn/dco_freebsd.c      | 13 +++++++++++++
 src/openvpn/ovpn_dco_freebsd.h |  5 +++++
 2 files changed, 18 insertions(+)
src/openvpn/dco_freebsd.c +13 -0
index 2ae46589..8d7ceb70 100644
@@ -529,6 +529,19 @@ dco_do_read(dco_context_t *dco)
     {
         dco->dco_del_peer_reason = OVPN_DEL_PEER_REASON_EXPIRED;
 
+        if (nvlist_exists_number(nvl, "del_reason"))
+        {
+            uint32_t reason = nvlist_get_number(nvl, "del_reason");
+            if (reason == OVPN_DEL_REASON_TIMEOUT)
+            {
+                dco->dco_del_peer_reason = OVPN_DEL_PEER_REASON_EXPIRED;
+            }
+            else
+            {
+                dco->dco_del_peer_reason = OVPN_DEL_PEER_REASON_USERSPACE;
+            }
+        }
+
         if (nvlist_exists_nvlist(nvl, "bytes"))
         {
             const nvlist_t *bytes = nvlist_get_nvlist(nvl, "bytes");
src/openvpn/ovpn_dco_freebsd.h +5 -0
index cc90111e..fec33835 100644
@@ -38,6 +38,11 @@ enum ovpn_notif_type {
     OVPN_NOTIF_DEL_PEER,
 };
 
+enum ovpn_del_reason {
+    OVPN_DEL_REASON_REQUESTED       = 0,
+    OVPN_DEL_REASON_TIMEOUT         = 1
+};
+
 enum ovpn_key_slot {
     OVPN_KEY_SLOT_PRIMARY   = 0,
     OVPN_KEY_SLOT_SECONDARY = 1
Signature · 1 line
2.38.1
Acked-by on v1 3/4
#15 lore raw

[Openvpn-devel] [PATCH applied] Re: Read the peer deletion reason from the kernel

Makes sense & goes hand in hand with kernel change "commit da69782bf06", which has landed in the official tree now. So I've tested this, of course :-)

As the change only touches dco_freebsd.c I have only tested this on FreeBSD (but client/server, with/without DCO). Logging is a bit thin there, so I added a msg() to my local tree print out the reason given (just to verify that "yes, it's coming up").

I can see counters and disconnect reason coming out of the kernel - which is very nice - but after a few connect/disconnects the box crashes hard on me. So something is not good there... OTOH, this is a kernel side thing, doesn't stop me from shipping this userland patch.

Your patch has been applied to the master and release/2.6 branch.

commit 10ea19092ded38ff25a445f987ad948aa9eac49f (master) commit 3186566e6967f0afb31e0426f5ec3490b0a363e9 (release/2.6)

Author: Kristof Provost
Date:   Mon Dec 5 17:41:02 2022 +0100
     Read the peer deletion reason from the kernel

     Signed-off-by: Kristof Provost <[email protected]...>
     Acked-by: Gert Doering <[email protected]...>
     Message-Id: <[email protected]...>
     URL: https://www.mail-archive.com/[email protected]/msg25617.html
     Signed-off-by: Gert Doering <[email protected]...>

--
kind regards,

Gert Doering

#16 lore raw

PATCH 4/4 dco: cleanup FreeBSD dco_do_read()

In reply to Kristof Provost

From: Kristof Provost <[email protected]...>

Remove support for reading packets through the control interface. FreeBSD no longer does this, so there's no point in keeping the code for it.

While here also check that we know what type of notification we're getting. There's currently only one, but we should check anyway.

---
 src/openvpn/dco_freebsd.c | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)
src/openvpn/dco_freebsd.c +8 -11
index 8d7ceb70..b6d869b0 100644
@@ -489,8 +489,7 @@ dco_do_read(dco_context_t *dco)
     struct ifdrv drv;
     uint8_t buf[4096];
     nvlist_t *nvl;
-    const uint8_t *pkt;
-    size_t pktlen;
+    enum ovpn_notif_type type;
     int ret;
 
     /* Flush any pending data from the pipe. */
@@ -518,15 +517,9 @@ dco_do_read(dco_context_t *dco)
 
     dco->dco_message_peer_id = nvlist_get_number(nvl, "peerid");
 
-    if (nvlist_exists_binary(nvl, "packet"))
-    {
-        pkt = nvlist_get_binary(nvl, "packet", &pktlen);
-        memcpy(BPTR(&dco->dco_packet_in), pkt, pktlen);
-        dco->dco_packet_in.len = pktlen;
-        dco->dco_message_type = OVPN_CMD_PACKET;
-    }
-    else
-    {
+    type = nvlist_get_number(nvl, "notification");
+    switch (type) {
+    case OVPN_NOTIF_DEL_PEER:
         dco->dco_del_peer_reason = OVPN_DEL_PEER_REASON_EXPIRED;
 
         if (nvlist_exists_number(nvl, "del_reason"))
@@ -551,6 +544,10 @@ dco_do_read(dco_context_t *dco)
         }
 
         dco->dco_message_type = OVPN_CMD_DEL_PEER;
+        break;
+    default:
+        msg(M_WARN, "Unknown kernel notification %d", type);
+        break;
     }
 
     nvlist_destroy(nvl);
Signature · 1 line
2.38.1
Acked-by on v1 4/4
#17 lore raw

[Openvpn-devel] [PATCH applied] Re: dco: cleanup FreeBSD dco_do_read()

Tested on FreeBSD with/without DCO, client/server, all works as before.

The formatting of the switch/case bits was not what uncrustify wanted to see, so I've adjusted this (only indenting). This makes "git show" a bit hard to read, but "git show -w" shows that most of the patch is very straightforward.

Your patch has been applied to the master and release/2.6 branch.

commit 235161cd2bcd5403c807e66432c421114c896b74 (master) commit 67e7846134bd8119737c3f80d958f401b5128cef (release/2.6)

Author: Kristof Provost
Date:   Mon Dec 5 17:41:03 2022 +0100
     dco: cleanup FreeBSD dco_do_read()

     Signed-off-by: Kristof Provost <[email protected]...>
     Acked-by: Gert Doering <[email protected]...>
     Message-Id: <[email protected]...>
     URL: https://www.mail-archive.com/[email protected]/msg25616.html
     Signed-off-by: Gert Doering <[email protected]...>

--
kind regards,

Gert Doering