Skip to contentSkip to conversation

Improve debug logging of DCO swap key message and Linux dco_new_peer

PATCH In review Arne Schwabe 3 participants 9 messages Dec 13, 2022 – Dec 14, 2022

lore
Share & tools
lore
#0 lore raw

PATCH 1/3 Improve debug logging of DCO swap key message and Linux dco_new_peer

---
 src/openvpn/dco.c       | 18 ++++++++++++++----
 src/openvpn/dco_linux.c | 10 ++++++++--
 2 files changed, 22 insertions(+), 6 deletions(-)
src/openvpn/dco.c +14 -4
index feb38cd02..2396bcbf0 100644
@@ -55,8 +55,8 @@ dco_install_key(struct tls_multi *multi, struct key_state *ks,
                 const char *ciphername)
 
 {
-    msg(D_DCO_DEBUG, "%s: peer_id=%d keyid=%d", __func__, multi->dco_peer_id,
-        ks->key_id);
+    msg(D_DCO_DEBUG, "%s: peer_id=%d keyid=%d, currently installed %d",
+        __func__, multi->dco_peer_id, ks->key_id, multi->dco_keys_installed);
 
     /* Install a key in the PRIMARY slot only when no other key exist.
      * From that moment on, any new key will be installed in the SECONDARY
@@ -181,8 +181,18 @@ dco_update_keys(dco_context_t *dco, struct tls_multi *multi)
      */
     if (primary->dco_status == DCO_INSTALLED_SECONDARY)
     {
-        msg(D_DCO_DEBUG, "Swapping primary and secondary keys, now: id1=%d id2=%d",
-            primary->key_id, secondary ? secondary->key_id : -1);
+        if (secondary)
+        {
+            msg(D_DCO_DEBUG, "Swapping primary and secondary keys to "
+                "primary-id=%d secondary-id=%d",
+                primary->key_id, secondary->key_id);
+        }
+        else
+        {
+            msg(D_DCO_DEBUG, "Swapping primary and secondary keys to"
+                "primary-id=%d secondary-id=(to be deleted)",
+                primary->key_id);
+        }
 
         int ret = dco_swap_keys(dco, multi->dco_peer_id);
         if (ret < 0)
src/openvpn/dco_linux.c +8 -2
index 109358205..fbd940c28 100644
@@ -216,9 +216,15 @@ dco_new_peer(dco_context_t *dco, unsigned int peerid, int sd,
              struct sockaddr *localaddr, struct sockaddr *remoteaddr,
              struct in_addr *remote_in4, struct in6_addr *remote_in6)
 {
-    msg(D_DCO_DEBUG, "%s: peer-id %d, fd %d", __func__, peerid, sd);
-
     struct gc_arena gc = gc_new();
+    const char *remotestr = "[undefined]";
+    if (remoteaddr)
+    {
+        remotestr = print_sockaddr(remoteaddr, &gc);
+    }
+    msg(D_DCO_DEBUG, "%s: peer-id %d, fd %d, remote addr: %s", __func__,
+        peerid, sd, remotestr);
+
     struct nl_msg *nl_msg = ovpn_dco_nlmsg_create(dco, OVPN_CMD_NEW_PEER);
     struct nlattr *attr = nla_nest_start(nl_msg, OVPN_ATTR_NEW_PEER);
     int ret = -EMSGSIZE;
Signature · 1 line
2.37.1 (Apple Git-137.1)
#1 lore raw

PATCH 2/3 Trigger a USR1 if dco_update_keys fails

When dco_update_keys fails, we are in some weird state that we are unlikely to recover since what userspace and kernel space think of the keys is very likely to not in sync anymore. So abandon the connection if this happens.

---
 src/openvpn/dco.c     | 15 ++++++++-------
 src/openvpn/dco.h     |  9 ++++++---
 src/openvpn/forward.c |  7 ++++++-
 3 files changed, 20 insertions(+), 11 deletions(-)
src/openvpn/dco.c +8 -7
index 2396bcbf0..36bfbf10a 100644
@@ -130,7 +130,7 @@ dco_get_secondary_key(struct tls_multi *multi, const struct key_state *primary)
     return NULL;
 }
 
-void
+bool
 dco_update_keys(dco_context_t *dco, struct tls_multi *multi)
 {
     msg(D_DCO_DEBUG, "%s: peer_id=%d", __func__, multi->dco_peer_id);
@@ -140,7 +140,7 @@ dco_update_keys(dco_context_t *dco, struct tls_multi *multi)
      */
     if (multi->dco_keys_installed == 0)
     {
-        return;
+        return true;
     }
 
     struct key_state *primary = tls_select_encryption_key(multi);
@@ -155,18 +155,18 @@ dco_update_keys(dco_context_t *dco, struct tls_multi *multi)
         if (ret < 0)
         {
             msg(D_DCO, "Cannot delete primary key during wipe: %s (%d)", strerror(-ret), ret);
-            return;
+            return false;
         }
 
         ret = dco_del_key(dco, multi->dco_peer_id, OVPN_KEY_SLOT_SECONDARY);
         if (ret < 0)
         {
             msg(D_DCO, "Cannot delete secondary key during wipe: %s (%d)", strerror(-ret), ret);
-            return;
+            return false;
         }
 
         multi->dco_keys_installed = 0;
-        return;
+        return true;
     }
 
     /* if we have a primary key, it must have been installed already (keys
@@ -198,7 +198,7 @@ dco_update_keys(dco_context_t *dco, struct tls_multi *multi)
         if (ret < 0)
         {
             msg(D_DCO, "Cannot swap keys: %s (%d)", strerror(-ret), ret);
-            return;
+            return false;
         }
 
         primary->dco_status = DCO_INSTALLED_PRIMARY;
@@ -216,7 +216,7 @@ dco_update_keys(dco_context_t *dco, struct tls_multi *multi)
         if (ret < 0)
         {
             msg(D_DCO, "Cannot delete secondary key: %s (%d)", strerror(-ret), ret);
-            return;
+            return false;
         }
         multi->dco_keys_installed = 1;
     }
@@ -230,6 +230,7 @@ dco_update_keys(dco_context_t *dco, struct tls_multi *multi)
             ks->dco_status = DCO_NOT_INSTALLED;
         }
     }
+    return true;
 }
 
 static bool
src/openvpn/dco.h +6 -3
index e051db068..7e1febaa3 100644
@@ -164,9 +164,11 @@ int init_key_dco_bi(struct tls_multi *multi, struct key_state *ks,
  *
  * @param dco           DCO device context
  * @param multi         TLS multi instance
+ *
+ * @return              returns false if an error occurred that is not
+ *                      recoverable and should reset the connection
  */
-void dco_update_keys(dco_context_t *dco, struct tls_multi *multi);
-
+bool dco_update_keys(dco_context_t *dco, struct tls_multi *multi);
 /**
  * Install a new peer in DCO - to be called by a CLIENT (or P2P) instance
  *
@@ -304,10 +306,11 @@ init_key_dco_bi(struct tls_multi *multi, struct key_state *ks,
     return 0;
 }
 
-static inline void
+static inline bool
 dco_update_keys(dco_context_t *dco, struct tls_multi *multi)
 {
     ASSERT(false);
+    return false;
 }
 
 static inline int
src/openvpn/forward.c +6 -1
index 5cd7eaa6e..8c1e49a34 100644
@@ -151,7 +151,12 @@ check_dco_key_status(struct context *c)
         return;
     }
 
-    dco_update_keys(&c->c1.tuntap->dco, c->c2.tls_multi);
+    if (!dco_update_keys(&c->c1.tuntap->dco, c->c2.tls_multi))
+    {
+        /* Something bad happened. Kill the connection to
+         * be able to recover. */
+        register_signal(c, SIGUSR1, "dco update keys error");
+    }
 }
 
 /*
Signature · 1 line
2.37.1 (Apple Git-137.1)
#4 lore raw

PATCH 3/3 Set DCO_NOT_INSTALLED also for keys not in the get_key_scan range

In reply to Arne Schwabe

We have 6 key slots but normally only consider 3 of them to be active/valid keys. Especially the secondary key of TM_LAME_DUCK can in rare corner cases have a key that is still installed in the kernel.

While this should not cause any issues since I do not see way for this key to become active ever again, it is better to keep the state correctly.

---
 src/openvpn/dco.c | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)
src/openvpn/dco.c +9 -5
index 36bfbf10a..20196fe5d 100644
@@ -221,13 +221,17 @@ dco_update_keys(dco_context_t *dco, struct tls_multi *multi)
         multi->dco_keys_installed = 1;
     }
 
-    /* all keys that are not installed are set to NOT installed */
-    for (int i = 0; i < KEY_SCAN_SIZE; ++i)
+    /* all keys that are not installed are set to NOT installed. Include also
+     * keys that might even be considered as active keys to be sure*/
+    for (int i = 0; i < TM_SIZE; ++i)
     {
-        struct key_state *ks = get_key_scan(multi, i);
-        if (ks != primary && ks != secondary)
+        for (int j = 0; j < KS_SIZE; j++)
         {
-            ks->dco_status = DCO_NOT_INSTALLED;
+            struct key_state *ks = &multi->session[i].key[j];
+            if (ks != primary && ks != secondary)
+            {
+                ks->dco_status = DCO_NOT_INSTALLED;
+            }
         }
     }
     return true;
Signature · 1 line
2.37.1 (Apple Git-137.1)
Acked-by on v1 2/3
#2 lore raw

Re: [Openvpn-devel] [PATCH 2/3] Trigger a USR1 if dco_update_keys fails

In reply to Arne Schwabe

Hi,

Quoted 6 lines · Arne Schwabe

When dco_update_keys fails, we are in some weird state that we are unlikely to recover since what userspace and kernel space think of the keys is very likely to not in sync anymore. So abandon the connection if this happens.

This makes sense to me.
We didn't do that earlier because we weren't sure about what to doing this case, but issuing USR1 and bailing out is actually sensible.

Quoted 125 lines
---
  src/openvpn/dco.c     | 15 ++++++++-------
  src/openvpn/dco.h     |  9 ++++++---
  src/openvpn/forward.c |  7 ++++++-
  3 files changed, 20 insertions(+), 11 deletions(-)
src/openvpn/dco.c +8 -7
index 2396bcbf0..36bfbf10a 100644
@@ -130,7 +130,7 @@ dco_get_secondary_key(struct tls_multi *multi, const struct key_state *primary)
      return NULL;
  }
 
-void
+bool
  dco_update_keys(dco_context_t *dco, struct tls_multi *multi)
  {
      msg(D_DCO_DEBUG, "%s: peer_id=%d", __func__, multi->dco_peer_id);
@@ -140,7 +140,7 @@ dco_update_keys(dco_context_t *dco, struct tls_multi *multi)
       */
      if (multi->dco_keys_installed == 0)
      {
-        return;
+        return true;
      }
 
      struct key_state *primary = tls_select_encryption_key(multi);
@@ -155,18 +155,18 @@ dco_update_keys(dco_context_t *dco, struct tls_multi *multi)
          if (ret < 0)
          {
              msg(D_DCO, "Cannot delete primary key during wipe: %s (%d)", strerror(-ret), ret);
-            return;
+            return false;
          }
 
          ret = dco_del_key(dco, multi->dco_peer_id, OVPN_KEY_SLOT_SECONDARY);
          if (ret < 0)
          {
              msg(D_DCO, "Cannot delete secondary key during wipe: %s (%d)", strerror(-ret), ret);
-            return;
+            return false;
          }
 
          multi->dco_keys_installed = 0;
-        return;
+        return true;
      }
 
      /* if we have a primary key, it must have been installed already (keys
@@ -198,7 +198,7 @@ dco_update_keys(dco_context_t *dco, struct tls_multi *multi)
          if (ret < 0)
          {
              msg(D_DCO, "Cannot swap keys: %s (%d)", strerror(-ret), ret);
-            return;
+            return false;
          }
 
          primary->dco_status = DCO_INSTALLED_PRIMARY;
@@ -216,7 +216,7 @@ dco_update_keys(dco_context_t *dco, struct tls_multi *multi)
          if (ret < 0)
          {
              msg(D_DCO, "Cannot delete secondary key: %s (%d)", strerror(-ret), ret);
-            return;
+            return false;
          }
          multi->dco_keys_installed = 1;
      }
@@ -230,6 +230,7 @@ dco_update_keys(dco_context_t *dco, struct tls_multi *multi)
              ks->dco_status = DCO_NOT_INSTALLED;
          }
      }
+    return true;
  }
 
  static bool
src/openvpn/dco.h +6 -3
index e051db068..7e1febaa3 100644
@@ -164,9 +164,11 @@ int init_key_dco_bi(struct tls_multi *multi, struct key_state *ks,
   *
   * @param dco           DCO device context
   * @param multi         TLS multi instance
+ *
+ * @return              returns false if an error occurred that is not
+ *                      recoverable and should reset the connection
   */
-void dco_update_keys(dco_context_t *dco, struct tls_multi *multi);
-
+bool dco_update_keys(dco_context_t *dco, struct tls_multi *multi);
  /**
   * Install a new peer in DCO - to be called by a CLIENT (or P2P) instance
   *
@@ -304,10 +306,11 @@ init_key_dco_bi(struct tls_multi *multi, struct key_state *ks,
      return 0;
  }
 
-static inline void
+static inline bool
  dco_update_keys(dco_context_t *dco, struct tls_multi *multi)
  {
      ASSERT(false);
+    return false;
  }
 
  static inline int
src/openvpn/forward.c +6 -1
index 5cd7eaa6e..8c1e49a34 100644
@@ -151,7 +151,12 @@ check_dco_key_status(struct context *c)
          return;
      }
 
-    dco_update_keys(&c->c1.tuntap->dco, c->c2.tls_multi);
+    if (!dco_update_keys(&c->c1.tuntap->dco, c->c2.tls_multi))
+    {
+        /* Something bad happened. Kill the connection to
+         * be able to recover. */
+        register_signal(c, SIGUSR1, "dco update keys error");
+    }
  }
 
  /*
Signature · 1 line
Antonio Quartulli
Acked-by on v1 3/3
#5 lore raw

Re: [Openvpn-devel] [PATCH 3/3] Set DCO_NOT_INSTALLED also for keys not in the get_key_scan range

In reply to Arne Schwabe

Hi,

Quoted 8 lines · Arne Schwabe

We have 6 key slots but normally only consider 3 of them to be active/valid keys. Especially the secondary key of TM_LAME_DUCK can in rare corner cases have a key that is still installed in the kernel.

While this should not cause any issues since I do not see way for this key to become active ever again, it is better to keep the state correctly.

I am fine with this because it is just "safer".
But I truly dislike that we have to "Defend ourselves" from conditions which we don't even know when they may happen.... Still, this is a rant for another patch/cleanup.

This patch makes sense

Quoted 31 lines
---
  src/openvpn/dco.c | 14 +++++++++-----
  1 file changed, 9 insertions(+), 5 deletions(-)
src/openvpn/dco.c +9 -5
index 36bfbf10a..20196fe5d 100644
@@ -221,13 +221,17 @@ dco_update_keys(dco_context_t *dco, struct tls_multi *multi)
          multi->dco_keys_installed = 1;
      }
 
-    /* all keys that are not installed are set to NOT installed */
-    for (int i = 0; i < KEY_SCAN_SIZE; ++i)
+    /* all keys that are not installed are set to NOT installed. Include also
+     * keys that might even be considered as active keys to be sure*/
+    for (int i = 0; i < TM_SIZE; ++i)
      {
-        struct key_state *ks = get_key_scan(multi, i);
-        if (ks != primary && ks != secondary)
+        for (int j = 0; j < KS_SIZE; j++)
          {
-            ks->dco_status = DCO_NOT_INSTALLED;
+            struct key_state *ks = &multi->session[i].key[j];
+            if (ks != primary && ks != secondary)
+            {
+                ks->dco_status = DCO_NOT_INSTALLED;
+            }
          }
      }
      return true;
Signature · 1 line
Antonio Quartulli
Acked-by on v1 1/3
#7 lore raw
In reply to Arne Schwabe
Quoted 11 lines · Arne Schwabe
---
  src/openvpn/dco.c       | 18 ++++++++++++++----
  src/openvpn/dco_linux.c | 10 ++++++++--
  2 files changed, 22 insertions(+), 6 deletions(-)
src/openvpn/dco.c +0 -0
index feb38cd02..2396bcbf0 100644

src/openvpn/dco.c dco_install_key(struct tls_multi *multi, struct key_state *ks, · line 58

Quoted 6 lines
src/openvpn/dco.c +1 -2
@@ -55,8 +55,8 @@ dco_install_key(struct tls_multi *multi, struct key_state *ks,
                  const char *ciphername)
 
  {
-    msg(D_DCO_DEBUG, "%s: peer_id=%d keyid=%d", __func__, multi->dco_peer_id,
-        ks->key_id);
+    msg(D_DCO_DEBUG, "%s: peer_id=%d keyid=%d, currently installed %d",

I'd make this "num. installed" because "currently installed" makes me think as if what follows is the ID of something that is installed. While we should make it explicit that this is the number of installed keys.

Quoted 37 lines
src/openvpn/dco.c +13 -2
+        __func__, multi->dco_peer_id, ks->key_id, multi->dco_keys_installed);
 
      /* Install a key in the PRIMARY slot only when no other key exist.
       * From that moment on, any new key will be installed in the SECONDARY
@@ -181,8 +181,18 @@ dco_update_keys(dco_context_t *dco, struct tls_multi *multi)
       */
      if (primary->dco_status == DCO_INSTALLED_SECONDARY)
      {
-        msg(D_DCO_DEBUG, "Swapping primary and secondary keys, now: id1=%d id2=%d",
-            primary->key_id, secondary ? secondary->key_id : -1);
+        if (secondary)
+        {
+            msg(D_DCO_DEBUG, "Swapping primary and secondary keys to "
+                "primary-id=%d secondary-id=%d",
+                primary->key_id, secondary->key_id);
+        }
+        else
+        {
+            msg(D_DCO_DEBUG, "Swapping primary and secondary keys to"
+                "primary-id=%d secondary-id=(to be deleted)",
+                primary->key_id);
+        }
 
          int ret = dco_swap_keys(dco, multi->dco_peer_id);
          if (ret < 0)
src/openvpn/dco_linux.c +1 -2
index 109358205..fbd940c28 100644
@@ -216,9 +216,15 @@ dco_new_peer(dco_context_t *dco, unsigned int peerid, int sd,
               struct sockaddr *localaddr, struct sockaddr *remoteaddr,
               struct in_addr *remote_in4, struct in6_addr *remote_in6)
  {
-    msg(D_DCO_DEBUG, "%s: peer-id %d, fd %d", __func__, peerid, sd);
-
      struct gc_arena gc = gc_new();
+    const char *remotestr = "[undefined]";

src/openvpn/dco_linux.c dco_new_peer(dco_context_t *dco, unsigned int peerid, int sd, · line 230

Quoted 10 lines
src/openvpn/dco_linux.c +7 -0
+    if (remoteaddr)
+    {
+        remotestr = print_sockaddr(remoteaddr, &gc);
+    }
+    msg(D_DCO_DEBUG, "%s: peer-id %d, fd %d, remote addr: %s", __func__,
+        peerid, sd, remotestr);
+
      struct nl_msg *nl_msg = ovpn_dco_nlmsg_create(dco, OVPN_CMD_NEW_PEER);
      struct nlattr *attr = nla_nest_start(nl_msg, OVPN_ATTR_NEW_PEER);
      int ret = -EMSGSIZE;

The rest makes sense:

Signature · 1 line
Antonio Quartulli
#8 lore raw
In reply to Arne Schwabe

As instructed I have changed the wording a bit, it's now

"... currently %d keys installed"

which should make it clear that we are talking about the number of keys, and not a key ID. "currently installed %d" is ambiguous here.

I have not tested this beyond "does it compile?", but since this is all just messages (and a bit of string formatting with an existing gc) the risk for "it will explode on platform X or with compile-time option Y" is minimal.

Your patch has been applied to the master and release/2.6 branch.

commit 63f838e384df3fb644bdeae6ede777b54dc968db (master) commit 8910b5b6028ee0feb9c324b1baf137cae5b4d9bd (release/2.6)

Author: Arne Schwabe
Date:   Tue Dec 13 23:54:28 2022 +0100
     Improve debug logging of DCO swap key message and Linux dco_new_peer

     Signed-off-by: Arne Schwabe <[email protected]...>
     Acked-by: Antonio Quartulli <[email protected]...>
     Message-Id: <[email protected]...>
     URL: https://www.mail-archive.com/[email protected]/msg25680.html
     Signed-off-by: Gert Doering <[email protected]...>

--
kind regards,

Gert Doering

#3 lore raw

[Openvpn-devel] [PATCH applied] Re: Trigger a USR1 if dco_update_keys fails

In reply to Arne Schwabe

This looks like a good way to recover out of "we are all confused about state" situations. We should never get there, but then, well- defined recovery is certainly a plus.

So far I've only stared-at-code, and done compile tests, but the change is sufficiently platform-independent (also taking --disable-dco into account) that I expect no breakage.

Your patch has been applied to the master and release/2.6 branch.

commit 419051c96e9fb1f3202fd67733aa3b6a4bbc3181 (master) commit 238ac1785f08e4cde4c095629c94f2b9bcc977fa (release/2.6)

Author: Arne Schwabe
Date:   Tue Dec 13 23:54:29 2022 +0100
     Trigger a USR1 if dco_update_keys fails

     Signed-off-by: Arne Schwabe <[email protected]...>
     Acked-by: Antonio Quartulli <[email protected]...>
     Message-Id: <[email protected]...>
     URL: https://www.mail-archive.com/[email protected]/msg25679.html
     Signed-off-by: Gert Doering <[email protected]...>

--
kind regards,

Gert Doering

#6 lore raw

[Openvpn-devel] [PATCH applied] Re: Set DCO_NOT_INSTALLED also for keys not in the get_key_scan range

In reply to Arne Schwabe

I've done a quick read over ssl_common.h to be sure the indexes are valid, and fed this to a light test run (client-side only, for the start). Looks good, passes.

I can see why Antonio grumbles, but then, ensuring stuff is in a really well-known state if we suspect it might not always be is a valid line of defense.

Your patch has been applied to the master and release/2.6 branch.

commit 4cf7409e82580f2890c391372d60ed713ba4650c (master) commit 5c26918f482f21484527f4b065818863c459b226 (release/2.6)

Author: Arne Schwabe
Date:   Tue Dec 13 23:54:30 2022 +0100
     Set DCO_NOT_INSTALLED also for keys not in the get_key_scan range

     Signed-off-by: Arne Schwabe <[email protected]...>
     Acked-by: Antonio Quartulli <[email protected]...>
     Message-Id: <[email protected]...>
     URL: https://www.mail-archive.com/[email protected]/msg25681.html
     Signed-off-by: Gert Doering <[email protected]...>

--
kind regards,

Gert Doering