kni: fix rtnl deadlocks and race conditions

This patch leverages on Stephen Hemminger's 64106 patch from Dec 2019, and fixes the issues reported by Ferruh and Igor:

A. KNI sync lock is being locked while rtnl is held.
If two threads are calling kni_net_process_request() ,
then the first one wil take the sync lock, release rtnl lock then sleep. The second thread will try to lock sync lock while holding rtnl. The first thread will wake, and try to lock rtnl, resulting in a deadlock. The remedy is to release rtnl before locking the KNI sync lock. Since in between nothing is accessing Linux network-wise,
no rtnl locking is needed.

B. There is a race condition in __dev_close_many() processing the close_list while the application terminates.
It looks like if two vEth devices are terminating,
and one releases the rtnl lock, the other takes it, updating the close_list in an unstable state,
causing the close_list to become a circular linked list, hence list_for_each_entry() will endlessly loop inside __dev_close_many() .
Since the description for the original patch indicate the original motivation was bringing the device up,
I have changed kni_net_process_request() to hold the rtnl mutex in case of bringing the device down since this is the path called from __dev_close_many() , causing the corruption of the close_list.

Signed-off-by Elad Nachman <[email protected]>

---
  kernel/linux/kni/kni_net.c | 47 +++++++++++++++++++++++++-------------
  1 file changed, 31 insertions(+), 16 deletions(-)

kernel/linux/kni/kni_net.c +14 -5

index 4b752083d..cf5b0845d 100644
@@ -17,6 +17,7 @@
  #include <linux/skbuff.h>
  #include <linux/kthread.h>
  #include <linux/delay.h>
+#include <linux/rtnetlink.h>
 
  #include <rte_kni_common.h>
  #include <kni_fifo.h>

@@ -102,18 +103,26 @@ get_data_kva(struct kni_dev *kni, void *pkt_kva)
   * It can be called to process the request.
   */
  static int
-kni_net_process_request(struct kni_dev *kni, struct rte_kni_request *req)
+kni_net_process_request(struct net_device *dev, struct rte_kni_request *req)
  {
+	struct kni_dev *kni = netdev_priv(dev);
  	int ret = -1;
  	void *resp_va;
  	uint32_t num;
  	int ret_val;
+	int req_is_dev_stop = 0;
 
-	if (!kni || !req) {
-		pr_err("No kni instance or request\n");
-		return -EINVAL;
-	}
+	if (req->req_id == RTE_KNI_REQ_CFG_NETWORK_IF &&
+			req->if_up == 0)
+		req_is_dev_stop = 1;
 
+	ASSERT_RTNL();
+
+	if (!req_is_dev_stop) {
+		dev_hold(dev);
+		rtnl_unlock();
+	}
+
  	mutex_lock(&kni->sync_lock);
 
  	/* Construct data */

kernel/linux/kni/kni_net.c +5 -1

+	if (!req_is_dev_stop) {
+		rtnl_lock();
+		dev_put(dev);
+	}
+
  	return ret;
  }
 

@@ -155,7 +176,6 @@ kni_net_open(struct net_device *dev)
  {
  	int ret;
  	struct rte_kni_request req;
-	struct kni_dev *kni = netdev_priv(dev);
 
  	netif_start_queue(dev);
  	if (kni_dflt_carrier == 1)

Quoted 11 lines · Elad Nachman

This patch leverages on Stephen Hemminger's 64106 patch from Dec 2019, and fixes the issues reported by Ferruh and Igor:

A. KNI sync lock is being locked while rtnl is held.
If two threads are calling kni_net_process_request() ,
then the first one wil take the sync lock, release rtnl lock then sleep. The second thread will try to lock sync lock while holding rtnl. The first thread will wake, and try to lock rtnl, resulting in a deadlock. The remedy is to release rtnl before locking the KNI sync lock. Since in between nothing is accessing Linux network-wise,
no rtnl locking is needed.

Hi Elad,

Thanks for explanation, that clarifies the issue.
Also I confirm I don't see the hang, at least as much as I test.

Quoted 15 lines

B. There is a race condition in __dev_close_many() processing the close_list while the application terminates.
It looks like if two vEth devices are terminating,
and one releases the rtnl lock, the other takes it, updating the close_list in an unstable state,
causing the close_list to become a circular linked list, hence list_for_each_entry() will endlessly loop inside __dev_close_many() .
Since the description for the original patch indicate the original motivation was bringing the device up,
I have changed kni_net_process_request() to hold the rtnl mutex in case of bringing the device down since this is the path called from __dev_close_many() , causing the corruption of the close_list.

I can't reproduce this case, I see the protection in the code, but better to get confirmation from Igor.

Overall the issue seems calling a function pointed by 'rte_kni_ops' which requires to acquire the rtnl lock.
So I wonder if this can't be handled in the ops function, by processing the request asynchronously,
like recording the request, return from 'rte_kni_ops', and process the request afterwards?

I assume the application we mention is not kni sample application.

Quoted 52 lines

Signed-off-by Elad Nachman <[email protected]>

---
  kernel/linux/kni/kni_net.c | 47 +++++++++++++++++++++++++-------------
  1 file changed, 31 insertions(+), 16 deletions(-)

kernel/linux/kni/kni_net.c +14 -5

index 4b752083d..cf5b0845d 100644
@@ -17,6 +17,7 @@
  #include <linux/skbuff.h>
  #include <linux/kthread.h>
  #include <linux/delay.h>
+#include <linux/rtnetlink.h>
 
  #include <rte_kni_common.h>
  #include <kni_fifo.h>

@@ -102,18 +103,26 @@ get_data_kva(struct kni_dev *kni, void *pkt_kva)
   * It can be called to process the request.
   */
  static int
-kni_net_process_request(struct kni_dev *kni, struct rte_kni_request *req)
+kni_net_process_request(struct net_device *dev, struct rte_kni_request *req)
  {
+     struct kni_dev *kni = netdev_priv(dev);
      int ret = -1;
      void *resp_va;
      uint32_t num;
      int ret_val;
+     int req_is_dev_stop = 0;
 
-     if (!kni || !req) {
-             pr_err("No kni instance or request\n");
-             return -EINVAL;
-     }
+     if (req->req_id == RTE_KNI_REQ_CFG_NETWORK_IF &&
+                     req->if_up == 0)
+             req_is_dev_stop = 1;
 
+     ASSERT_RTNL();
+
+     if (!req_is_dev_stop) {
+             dev_hold(dev);
+             rtnl_unlock();
+     }
+
      mutex_lock(&kni->sync_lock);
 
      /* Construct data */

kernel/linux/kni/kni_net.c kni_net_process_request(struct kni_dev *kni, struct rte_kni_request *req) · line 140

Quoted 7 lines

kernel/linux/kni/kni_net.c +4 -0

@@ -125,8 +134,13 @@ kni_net_process_request(struct kni_dev *kni, struct rte_kni_request *req)
              goto fail;
      }
 
+     /* Since we need to wait and RTNL mutex is held
+      * drop the mutex and hold refernce to keep device
+      */
+

Comment seems left here, need to go up. s/refernce/reference

kernel/linux/kni/kni_net.c kni_net_process_request(struct kni_dev *kni, struct rte_kni_request *req) · line 162

Quoted 12 lines

kernel/linux/kni/kni_net.c +3 -0

      ret_val = wait_event_interruptible_timeout(kni->wq,
                      kni_fifo_count(kni->resp_q), 3 * HZ);
+
      if (signal_pending(current) || ret_val <= 0) {
              ret = -ETIME;
              goto fail;

@@ -144,6 +158,13 @@ kni_net_process_request(struct kni_dev *kni, struct rte_kni_request *req)
 
  fail:
      mutex_unlock(&kni->sync_lock);
+
+

extra empty line

Quoted 17 lines

kernel/linux/kni/kni_net.c +5 -1

+     if (!req_is_dev_stop) {
+             rtnl_lock();
+             dev_put(dev);
+     }
+
      return ret;
  }
 

@@ -155,7 +176,6 @@ kni_net_open(struct net_device *dev)
  {
      int ret;
      struct rte_kni_request req;
-     struct kni_dev *kni = netdev_priv(dev);
 
      netif_start_queue(dev);
      if (kni_dflt_carrier == 1)

kernel/linux/kni/kni_net.c kni_net_open(struct net_device *dev) · line 192

Quoted 6 lines

kernel/linux/kni/kni_net.c +1 -1

@@ -168,7 +188,7 @@ kni_net_open(struct net_device *dev)
 
      /* Setting if_up to non-zero means up */
      req.if_up = 1;
-     ret = kni_net_process_request(kni, &req);
+     ret = kni_net_process_request(dev, &req);
 

Althoug it is not soo confusing, these lines and following ones are noise for this patch, they are just for 'kni_net_process_request' paramter change.

What do you think do the 'kni_net_process_request' parameter change in first patch, and fix the issue in second, this way second patch can contain only the actual changes required for fix.

Hi,

Regarding the asynchronous call - thought about it, but then the request will always return OK to user-space and I will have no way to return failure error codes back to user-space.

If the above explanation is acceptable, per your other comments - I can send a new patch without the parameter change , without the empty line, and with the comment moved to the proper place in the code.

Waiting for your decision,

Elad.

Quoted 156 lines · Ferruh Yigit

Quoted 11 lines · Elad Nachman

This patch leverages on Stephen Hemminger's 64106 patch from Dec 2019, and fixes the issues reported by Ferruh and Igor:

A. KNI sync lock is being locked while rtnl is held.
If two threads are calling kni_net_process_request() ,
then the first one wil take the sync lock, release rtnl lock then sleep. The second thread will try to lock sync lock while holding rtnl. The first thread will wake, and try to lock rtnl, resulting in a deadlock. The remedy is to release rtnl before locking the KNI sync lock. Since in between nothing is accessing Linux network-wise,
no rtnl locking is needed.

Hi Elad,

Thanks for explanation, that clarifies the issue.
Also I confirm I don't see the hang, at least as much as I test.

Quoted 15 lines

B. There is a race condition in __dev_close_many() processing the close_list while the application terminates.
It looks like if two vEth devices are terminating,
and one releases the rtnl lock, the other takes it, updating the close_list in an unstable state,
causing the close_list to become a circular linked list, hence list_for_each_entry() will endlessly loop inside __dev_close_many() .
Since the description for the original patch indicate the original motivation was bringing the device up,
I have changed kni_net_process_request() to hold the rtnl mutex in case of bringing the device down since this is the path called from __dev_close_many() , causing the corruption of the close_list.

I can't reproduce this case, I see the protection in the code, but better to get confirmation from Igor.

Overall the issue seems calling a function pointed by 'rte_kni_ops' which requires to acquire the rtnl lock.
So I wonder if this can't be handled in the ops function, by processing the request asynchronously,
like recording the request, return from 'rte_kni_ops', and process the request afterwards?

I assume the application we mention is not kni sample application.

Quoted 52 lines

Signed-off-by Elad Nachman <[email protected]>

---
   kernel/linux/kni/kni_net.c | 47 +++++++++++++++++++++++++-------------
   1 file changed, 31 insertions(+), 16 deletions(-)

kernel/linux/kni/kni_net.c +14 -5

index 4b752083d..cf5b0845d 100644
@@ -17,6 +17,7 @@
   #include <linux/skbuff.h>
   #include <linux/kthread.h>
   #include <linux/delay.h>
+#include <linux/rtnetlink.h>
 
   #include <rte_kni_common.h>
   #include <kni_fifo.h>

@@ -102,18 +103,26 @@ get_data_kva(struct kni_dev *kni, void *pkt_kva)
    * It can be called to process the request.
    */
   static int
-kni_net_process_request(struct kni_dev *kni, struct rte_kni_request *req)
+kni_net_process_request(struct net_device *dev, struct rte_kni_request *req)
   {
+     struct kni_dev *kni = netdev_priv(dev);
       int ret = -1;
       void *resp_va;
       uint32_t num;
       int ret_val;
+     int req_is_dev_stop = 0;
 
-     if (!kni || !req) {
-             pr_err("No kni instance or request\n");
-             return -EINVAL;
-     }
+     if (req->req_id == RTE_KNI_REQ_CFG_NETWORK_IF &&
+                     req->if_up == 0)
+             req_is_dev_stop = 1;
 
+     ASSERT_RTNL();
+
+     if (!req_is_dev_stop) {
+             dev_hold(dev);
+             rtnl_unlock();
+     }
+
       mutex_lock(&kni->sync_lock);
 
       /* Construct data */

kernel/linux/kni/kni_net.c kni_net_process_request(struct kni_dev *kni, struct rte_kni_request *req) · line 140

Quoted 7 lines

kernel/linux/kni/kni_net.c +4 -0

@@ -125,8 +134,13 @@ kni_net_process_request(struct kni_dev *kni, struct rte_kni_request *req)
               goto fail;
       }
 
+     /* Since we need to wait and RTNL mutex is held
+      * drop the mutex and hold refernce to keep device
+      */
+

Comment seems left here, need to go up. s/refernce/reference

kernel/linux/kni/kni_net.c kni_net_process_request(struct kni_dev *kni, struct rte_kni_request *req) · line 162

Quoted 12 lines

kernel/linux/kni/kni_net.c +3 -0

       ret_val = wait_event_interruptible_timeout(kni->wq,
                       kni_fifo_count(kni->resp_q), 3 * HZ);
+
       if (signal_pending(current) || ret_val <= 0) {
               ret = -ETIME;
               goto fail;

@@ -144,6 +158,13 @@ kni_net_process_request(struct kni_dev *kni, struct rte_kni_request *req)
 
   fail:
       mutex_unlock(&kni->sync_lock);
+
+

extra empty line

Quoted 17 lines

kernel/linux/kni/kni_net.c +5 -1

+     if (!req_is_dev_stop) {
+             rtnl_lock();
+             dev_put(dev);
+     }
+
       return ret;
   }
 

@@ -155,7 +176,6 @@ kni_net_open(struct net_device *dev)
   {
       int ret;
       struct rte_kni_request req;
-     struct kni_dev *kni = netdev_priv(dev);
 
       netif_start_queue(dev);
       if (kni_dflt_carrier == 1)

kernel/linux/kni/kni_net.c kni_net_open(struct net_device *dev) · line 192

Quoted 6 lines

kernel/linux/kni/kni_net.c +1 -1

@@ -168,7 +188,7 @@ kni_net_open(struct net_device *dev)
 
       /* Setting if_up to non-zero means up */
       req.if_up = 1;
-     ret = kni_net_process_request(kni, &req);
+     ret = kni_net_process_request(dev, &req);
 

Althoug it is not soo confusing, these lines and following ones are noise for this patch, they are just for 'kni_net_process_request' paramter change.

What do you think do the 'kni_net_process_request' parameter change in first patch, and fix the issue in second, this way second patch can contain only the actual changes required for fix.

This version 2 of the patch leverages on Stephen Hemminger's 64106 patch from Dec 2019,
and fixes the issues reported by Ferruh and Igor:

A. KNI sync lock is being locked while rtnl is held.
If two threads are calling kni_net_process_request() ,
then the first one will take the sync lock, release rtnl lock then sleep. The second thread will try to lock sync lock while holding rtnl. The first thread will wake, and try to lock rtnl, resulting in a deadlock. The remedy is to release rtnl before locking the KNI sync lock. Since in between nothing is accessing Linux network-wise,
no rtnl locking is needed.

B. There is a race condition in __dev_close_many() processing the close_list while the application terminates.
It looks like if two vEth devices are terminating,
and one releases the rtnl lock, the other takes it, updating the close_list in an unstable state,
causing the close_list to become a circular linked list, hence list_for_each_entry() will endlessly loop inside __dev_close_many() .
Since the description for the original patch indicate the original motivation was bringing the device up,
I have changed kni_net_process_request() to hold the rtnl mutex in case of bringing the device down since this is the path called from __dev_close_many() , causing the corruption of the close_list.

Depends-on: patch-64106 ("kni: fix kernel deadlock when using mlx devices")

Signed-off-by Elad Nachman <[email protected]>

---
V2:
* rebuild the patch as increment from patch 64106
* fix comment and blank lines

---
  kernel/linux/kni/kni_net.c | 25 +++++++++++++++++--------
  1 file changed, 17 insertions(+), 8 deletions(-)

kernel/linux/kni/kni_net.c +0 -0

index f0b6e9a8d..b41360220 100644

This part of the series includes my fixes for the issues reported by Ferruh and Igor (and Igor comments for v3 of the patch) on top of part 1 of the patch series:

A. KNI sync lock is being locked while rtnl is held.
If two threads are calling kni_net_process_request() ,
then the first one will take the sync lock, release rtnl lock then sleep. The second thread will try to lock sync lock while holding rtnl. The first thread will wake, and try to lock rtnl, resulting in a deadlock. The remedy is to release rtnl before locking the KNI sync lock. Since in between nothing is accessing Linux network-wise,
no rtnl locking is needed.

B. There is a race condition in __dev_close_many() processing the close_list while the application terminates.
It looks like if two vEth devices are terminating,
and one releases the rtnl lock, the other takes it, updating the close_list in an unstable state,
causing the close_list to become a circular linked list, hence list_for_each_entry() will endlessly loop inside __dev_close_many() .
Since the description for the original patch indicate the original motivation was bringing the device up,
I have changed kni_net_process_request() to hold the rtnl mutex in case of bringing the device down since this is the path called from __dev_close_many() , causing the corruption of the close_list. In order to prevent deadlock in Mellanox device in this case, the code has been modified not to wait for user-space while holding the rtnl lock.
Instead, after the request has been sent, all locks are relinquished and the function exits immediately with return code of zero (success).

To summarize:
request != interface down : unlock rtnl, send request to user-space, wait for response, send the response error code to caller in user-space.

request == interface down: send request to user-space, return immediately with error code of 0 (success) to user-space.

Signed-off-by Elad Nachman <[email protected]>

---
v4:
* for if down case, send asynchronously with rtnl locked and without wait, returning immediately to avoid both kernel race conditions and deadlock in user-space
v3:
* Include original patch and new patch as a series of patch, added a comment to the new patch
v2:
* rebuild the patch as increment from patch 64106
* fix comment and blank lines

---
 kernel/linux/kni/kni_net.c      | 41 +++++++++++++++++++++++++++------
 lib/librte_kni/rte_kni.c        |  7 ++++--
 lib/librte_kni/rte_kni_common.h |  1 +
 3 files changed, 40 insertions(+), 9 deletions(-)

kernel/linux/kni/kni_net.c +23 -1

index f0b6e9a8d..ba991802b 100644
@@ -110,12 +110,34 @@ kni_net_process_request(struct net_device *dev,
 struct rte_kni_request *req)
        void *resp_va;
        uint32_t num;
        int ret_val;
+       int req_is_dev_stop = 0;
+
+       /* For configuring the interface to down,
+        * rtnl must be held all the way to prevent race condition
+        * inside __dev_close_many() between two netdev instances of KNI
+        */
+       if (req->req_id == RTE_KNI_REQ_CFG_NETWORK_IF &&
+                       req->if_up == 0)
+               req_is_dev_stop = 1;
 
        ASSERT_RTNL();
 
+       /* Since we need to wait and RTNL mutex is held
+        * drop the mutex and hold reference to keep device
+        */
+       if (!req_is_dev_stop) {
+               dev_hold(dev);
+               rtnl_unlock();
+       }
+
        mutex_lock(&kni->sync_lock);
 
-       /* Construct data */
+       /* Construct data, for dev stop send asynchronously
+        * so instruct user-space not to sent response as no
+        * one will be waiting for it.
+        */
+       if (req_is_dev_stop)
+               req->skip_post_resp_to_q = 1;
        memcpy(kni->sync_kva, req, sizeof(struct rte_kni_request));
        num = kni_fifo_put(kni->req_q, &kni->sync_va, 1);

        if (num < 1) {

kernel/linux/kni/kni_net.c +6 -6

@@ -124,16 +146,16 @@ kni_net_process_request(struct net_device *dev,
 struct rte_kni_request *req)
                goto fail;
        }
 
-       /* Since we need to wait and RTNL mutex is held
-        * drop the mutex and hold refernce to keep device
+       /* No result available since request is handled
+        * asynchronously. set response to success.
         */
-       dev_hold(dev);
-       rtnl_unlock();
+       if (req_is_dev_stop) {
+               req->result = 0;
+               goto async;
+       }
 
        ret_val = wait_event_interruptible_timeout(kni->wq,
                        kni_fifo_count(kni->resp_q), 3 * HZ);
-       rtnl_lock();
-       dev_put(dev);
 
        if (signal_pending(current) || ret_val <= 0) {

                ret = -ETIME;

kernel/linux/kni/kni_net.c +5 -0

@@ -148,10 +170,15 @@ kni_net_process_request(struct net_device *dev,
 struct rte_kni_request *req)
        }
 
        memcpy(req, kni->sync_kva, sizeof(struct rte_kni_request));
+async:
        ret = 0;
 
 fail:
        mutex_unlock(&kni->sync_lock);
+       if (!req_is_dev_stop) {
+               rtnl_lock();
+               dev_put(dev);
+       }
        return ret;
 }

lib/librte_kni/rte_kni.c +5 -2

index 837d0217d..6d777266d 100644
@@ -591,8 +591,11 @@ rte_kni_handle_request(struct rte_kni *kni)
                break;
        }
 
-       /* Construct response mbuf and put it back to resp_q */
-       ret = kni_fifo_put(kni->resp_q, (void **)&req, 1);
+       /* if needed, construct response mbuf and put it back to resp_q */
+       if (!req->skip_post_resp_to_q)
+               ret = kni_fifo_put(kni->resp_q, (void **)&req, 1);
+       else
+               ret = 1;
        if (ret != 1) {
                RTE_LOG(ERR, KNI, "Fail to put the muf back to resp_q\n");
                return -1; /* It is an error of can't putting the mbuf

back */

lib/librte_kni/rte_kni_common.h +0 -0

index ffb318273..3b5d06850 100644
@@ -48,6 +48,7 @@ struct rte_kni_request {
                uint8_t promiscusity;/**< 1: promisc mode enable, 0:

disable */
uint8_t allmulti; /**< 1: all-multicast mode enable, 0:
disable */

        };

+ int32_t skip_post_resp_to_q; /**< 1: skip queue response 0:
disable */
int32_t result; /**< Result for processing request */
} __attribute__((__packed__));

--
2.17.1

Hi Elad,

Thanks for the patch, but this is still NACK from me.

The only real advantage of KNI over other exceptional-path techniques like virtio-user is the ability to configure DPDK-managed interfaces directly
from the kernel using well-known utils like iproute2. A very important part of this is getting responses from the DPDK app and knowing the actual result of command execution.
If you're making async requests to the application and you don't know the result, then what's the point of using KNI at all?

Igor

Quoted 13 lines · On Fri, 26 Feb 2021 00:01:01 +0300 Igor Ryzhov <[email protected]> wrote:

Hi Elad,

Thanks for the patch, but this is still NACK from me.

The only real advantage of KNI over other exceptional-path techniques like virtio-user is the ability to configure DPDK-managed interfaces directly
from the kernel using well-known utils like iproute2. A very important part of this is getting responses from the DPDK app and knowing the actual result of command execution.
If you're making async requests to the application and you don't know the result, then what's the point of using KNI at all?

Quoted 1 line

Igor

Do you have a better proposal that keeps the request result but does not call userspace with lock held.

PS: I also have strong dislike of KNI, as designed it would have been rejected by Linux kernel developers. A better solution would be userspace version of something like devlink devices. But doing control operations by proxy is a locking nightmare.

The way the kernel handles its locks and lists for the dev close many path, there is no way you can go around this with rtnl unlocked :
"

There is a race condition in __dev_close_many() processing the close_list while the application terminates.
It looks like if two vEth devices are terminating, and one releases the rtnl lock, the other takes it, updating the close_list in an unstable state, causing the close_list to become a circular linked list, hence list_for_each_entry() will endlessly loop inside __dev_close_many() .

"
And I don't expect David Miller will bend the kernel networking for DPDK or KNI.

But - Stephen - if you can personally convince David to accept a kernel patch which will separate the close_list locking mechanism to a separate (RCU?) lock, then I can introduce first a patch to the kernel which will add a lock for the close_list, this way rtnl can be unlocked for the if down case.

After that kernel patch, your original patch + relocation of the sync mutex locking will do the job .

Otherwise, rtnl has to be kept locked all of the way for the if down case in order to prevent corruption causing a circular linked list out of the close_list, causing a hang in the kernel.

Currently, the rtnl lock is the only thing keeping the close_list from corruption.

If you doubt rtnl cannot be unlocked for dev close path, you can consult David for his opinion, as I think it is critical to understand what the kernel can or cannot do, or expects to be done before we can unlock its locks as we wish inside rte_kni.ko .

Otherwise, if we are still in disagreement on how to patch this set of problems, I think the responsible way around it is to completely remove kni from the main dpdk tree and move it to dpdk-kmods repository.

I know BSD style open-source does not carry legal responsibility from the developers, but I think when a bunch of developers know a piece of code is highly buggy, they should not leave it for countless new users to bounce their head desperately against, if they cannot agree on a correct way to solve the bunch of problems, of which I think we all agree exist (we just do not agree on the proper solution or patch)...

That's my two cents,

Elad.

Quoted 4 lines · Stephen Hemminger

On Fri, 26 Feb 2021 00:01:01 +0300
Igor Ryzhov <[email protected]> wrote:

Quoted 8 lines

Quoted 8 lines

Hi Elad,

Thanks for the patch, but this is still NACK from me.

The only real advantage of KNI over other exceptional-path techniques like virtio-user is the ability to configure DPDK-managed interfaces directly
from the kernel using well-known utils like iproute2. A very important

part

Quoted 11 lines

Quoted 6 lines

of this is getting responses from the DPDK app and knowing the actual result of command execution.
If you're making async requests to the application and you don't know the result, then what's the point of using KNI at all?

Igor

Do you have a better proposal that keeps the request result but does not call userspace with lock held.

PS: I also have strong dislike of KNI, as designed it would have been

rejected

Quoted 1 line

by Linux kernel developers. A better solution would be userspace

version of

Quoted 2 lines

something like devlink devices. But doing control operations by proxy is a locking nightmare.

Stephen,

No, I don't have a better proposal, but I think it is not correct to change the behavior of KNI (making link down without a real response). Even though we know that communicating with userspace under rtnl_lock is a bad idea, it works as it is for many years already.

Elad,

I agree with you that KNI should be removed from the main tree if it is not possible to fix this __dev_close_many issue.
There were discussions about this multiple times already, but no one is working on this AFAIK.
Last time the discussion was a month ago:
https://www.mail-archive.com/[email protected]/msg196033.html

Igor

Quoted 16 lines · On Mon, 1 Mar 2021 11:10:01 +0300 Igor Ryzhov <[email protected]> wrote:

Stephen,

No, I don't have a better proposal, but I think it is not correct to change the behavior of KNI (making link down without a real response). Even though we know that communicating with userspace under rtnl_lock is a bad idea, it works as it is for many years already.

Elad,

I agree with you that KNI should be removed from the main tree if it is not possible to fix this __dev_close_many issue.
There were discussions about this multiple times already, but no one is working on this AFAIK.
Last time the discussion was a month ago:
https://www.mail-archive.com/[email protected]/msg196033.html

Quoted 1 line

Igor

The better proposal would be to make DPDK virtio smarter. There already is virtio devices that must handle this (VDPA) etc. And when you can control link through virtio, then put a big warning in KNI that says "Don't use this"

Stephen,

No, I don't have a better proposal, but I think it is not correct to change the behavior of KNI (making link down without a real response). Even though we know that communicating with userspace under rtnl_lock is a bad idea, it works as it is for many years already.

Elad,

I agree with you that KNI should be removed from the main tree if it is not possible to fix this __dev_close_many issue.
There were discussions about this multiple times already, but no one is working on this AFAIK.
Last time the discussion was a month ago:
https://www.mail-archive.com/[email protected]/msg196033.html

Igor

Quoted 86 lines · Elad Nachman

The way the kernel handles its locks and lists for the dev close many path, there is no way you can go around this with rtnl unlocked :
"

There is a race condition in __dev_close_many() processing the close_list while the application terminates.
It looks like if two vEth devices are terminating, and one releases the rtnl lock, the other takes it, updating the close_list in an unstable state, causing the close_list to become a circular linked list, hence list_for_each_entry() will endlessly loop inside __dev_close_many() .

"
And I don't expect David Miller will bend the kernel networking for DPDK or KNI.

But - Stephen - if you can personally convince David to accept a kernel patch which will separate the close_list locking mechanism to a separate (RCU?) lock, then I can introduce first a patch to the kernel which will add a lock for the close_list, this way rtnl can be unlocked for the if down case.

After that kernel patch, your original patch + relocation of the sync mutex locking will do the job .
lockups
Otherwise, rtnl has to be kept locked all of the way for the if down case in order to prevent corruption causing a circular linked list out of the close_list, causing a hang in the kernel.
lockups
Currently, the rtnl lock is the only thing keeping the close_list from corruption.

If you doubt rtnl cannot be unlocked for dev close path, you can consult David for his opinion, as I think it is critical to understand what the kernel can or cannot do, or expects to be done before we can unlock its locks as we wish inside rte_kni.ko .

Otherwise, if we are still in disagreement on how to patch this set of problems, I think the responsible way around it is to completely remove kni from the main dpdk tree and move it to dpdk-kmods repository.

I know BSD style open-source does not carry legal responsibility from the developers, but I think when a bunch of developers know a piece of code is highly buggy, they should not leave it for countless new users to bounce their head desperately against, if they cannot agree on a correct way to solve the bunch of problems, of which I think we all agree exist (we just do not agree on the proper solution or patch)...

That's my two cents,

Elad.

Quoted 4 lines · Stephen Hemminger

On Fri, 26 Feb 2021 00:01:01 +0300
Igor Ryzhov <[email protected]> wrote:

Quoted 8 lines

Quoted 8 lines

Hi Elad,

Thanks for the patch, but this is still NACK from me.

The only real advantage of KNI over other exceptional-path techniques like virtio-user is the ability to configure DPDK-managed interfaces directly
from the kernel using well-known utils like iproute2. A very important

part

Quoted 11 lines

Quoted 6 lines

of this is getting responses from the DPDK app and knowing the actual result of command execution.
If you're making async requests to the application and you don't know the result, then what's the point of using KNI at all?

Igor

Do you have a better proposal that keeps the request result but does not call userspace with lock held.

PS: I also have strong dislike of KNI, as designed it would have been

rejected

Quoted 1 line

by Linux kernel developers. A better solution would be userspace

version of

Quoted 2 lines

something like devlink devices. But doing control operations by proxy is a locking nightmare.

Hi All,

Sorry to butt in on this, but I fixed this same issue about 3 years ago in my application, but I was never able to get the changes integrated and eventually just gave up trying.

The rule with KNI is:
1) The app should have a separate control thread per rte_kni which just spins calling rte_kni_handle_request(). This ensures that other threads calling rte_kni_XXX functions will always get a response.

2) In order to deal with lockups and timeouts when closing the device, I sent patches which separated the closing process into two steps:
rte_kni_release() which would unregister the underlying netdev, then rte_kni_free() which would free the KNI portions of the KNI device. When rte_kni_release() is called the kernel netdev is unregistered and a response is sent back to the application, the control thread calling rte_kni_handle_request() is still running, so the application will still get a response back from the kernel and not lock up, the application then kills the control thread so that rte_kni_handle_request() is not called again, then the application calls rte_kni_free() which frees all of the FIFOs and closes the device.

If anyone is interested the patches are probably still floating around patchwork. If not you can check them out here:

https://github.com/danielgora/dpdk.git

thanks-
dan

Quoted 107 lines · Igor Ryzhov

Stephen,

No, I don't have a better proposal, but I think it is not correct to change the behavior of KNI (making link down without a real response). Even though we know that communicating with userspace under rtnl_lock is a bad idea, it works as it is for many years already.

Elad,

I agree with you that KNI should be removed from the main tree if it is not possible to fix this __dev_close_many issue.
There were discussions about this multiple times already, but no one is working on this AFAIK.
Last time the discussion was a month ago:
https://www.mail-archive.com/[email protected]/msg196033.html

Igor

Quoted 86 lines · Elad Nachman

The way the kernel handles its locks and lists for the dev close many path, there is no way you can go around this with rtnl unlocked :
"

There is a race condition in __dev_close_many() processing the close_list while the application terminates.
It looks like if two vEth devices are terminating, and one releases the rtnl lock, the other takes it, updating the close_list in an unstable state, causing the close_list to become a circular linked list, hence list_for_each_entry() will endlessly loop inside __dev_close_many() .

"
And I don't expect David Miller will bend the kernel networking for DPDK or KNI.

But - Stephen - if you can personally convince David to accept a kernel patch which will separate the close_list locking mechanism to a separate (RCU?) lock, then I can introduce first a patch to the kernel which will add a lock for the close_list, this way rtnl can be unlocked for the if down case.

After that kernel patch, your original patch + relocation of the sync mutex locking will do the job .
lockups
Otherwise, rtnl has to be kept locked all of the way for the if down case in order to prevent corruption causing a circular linked list out of the close_list, causing a hang in the kernel.
lockups
Currently, the rtnl lock is the only thing keeping the close_list from corruption.

If you doubt rtnl cannot be unlocked for dev close path, you can consult David for his opinion, as I think it is critical to understand what the kernel can or cannot do, or expects to be done before we can unlock its locks as we wish inside rte_kni.ko .

Otherwise, if we are still in disagreement on how to patch this set of problems, I think the responsible way around it is to completely remove kni from the main dpdk tree and move it to dpdk-kmods repository.

I know BSD style open-source does not carry legal responsibility from the developers, but I think when a bunch of developers know a piece of code is highly buggy, they should not leave it for countless new users to bounce their head desperately against, if they cannot agree on a correct way to solve the bunch of problems, of which I think we all agree exist (we just do not agree on the proper solution or patch)...

That's my two cents,

Elad.

Quoted 4 lines · Stephen Hemminger

On Fri, 26 Feb 2021 00:01:01 +0300
Igor Ryzhov <[email protected]> wrote:

Quoted 8 lines

Quoted 8 lines

Hi Elad,

Thanks for the patch, but this is still NACK from me.

The only real advantage of KNI over other exceptional-path techniques like virtio-user is the ability to configure DPDK-managed interfaces directly
from the kernel using well-known utils like iproute2. A very important

part

Quoted 11 lines

Quoted 6 lines

of this is getting responses from the DPDK app and knowing the actual result of command execution.
If you're making async requests to the application and you don't know the result, then what's the point of using KNI at all?

Igor

Do you have a better proposal that keeps the request result but does not call userspace with lock held.

PS: I also have strong dislike of KNI, as designed it would have been

rejected

Quoted 1 line

by Linux kernel developers. A better solution would be userspace

version of

Quoted 2 lines

something like devlink devices. But doing control operations by proxy is a locking nightmare.

This is from my git commit fixing this:

kni: separate releasing netdev from freeing KNI interface

Currently the rte_kni kernel driver suffers from a problem where when the interface is released, it generates a callback to the DPDK application to change the interface state to Down. However, after the DPDK application handles the callback and generates a response back to the kernel, the rte_kni driver cannot wake the thread which is asleep waiting for the response, because it is holding the kni_link_lock semaphore and it has already removed the 'struct kni_dev' from the list of interfaces to poll for responses.

This means that if the KNI interface is in the Up state when rte_kni_release() is called, it will always sleep for three seconds until kni_net_release gives up waiting for a response from the DPDK application.

To fix this, we must separate the step to release the kernel network interface from the steps to remove the KNI interface from the list of interfaces to poll.

When the kernel network interface is removed with unregister_netdev(), if the interface is up, it will generate a callback to mark the interface down, which calls kni_net_release(). kni_net_release() will block waiting for the DPDK application to call rte_kni_handle_request() to handle the callback, but it also needs the thread in the KNI driver (either the per-dev thread for multi-thread or the per-driver thread) to call kni_net_poll_resp() in order to wake the thread sleeping in kni_net_release (actually kni_net_process_request()).

So now, KNI interfaces should be removed as such:

1) The user calls rte_kni_release(). This only unregisters the netdev in the kernel, but touches nothing else. This allows all the threads to run which are necessary to handle the callback into the DPDK application to mark the interface down.

2) The user stops the thread running rte_kni_handle_request(). After rte_kni_release() has been called, there will be no more callbacks for that interface so it is not necessary. It cannot be running at the same time that rte_kni_free() frees all of the FIFOs and DPDK memory for that KNI interface.

3) The user calls rte_kni_free(). This performs the RTE_KNI_IOCTL_FREE ioctl which calls kni_ioctl_free(). This function removes the struct kni_dev from the list of interfaces to poll (and kills the per-dev kthread, if configured for multi-thread), then frees the memory in the FIFOs.

Signed-off-by Dan Gora <[email protected]>

I'm not sure that this is exactly the problem that you're seeing, but it sounds like it to me.

thanks
dan

Quoted 7 lines · Dan Gora

Hi All,

Sorry to butt in on this, but I fixed this same issue about 3 years ago in my application, but I was never able to get the changes integrated and eventually just gave up trying.

Quoted 6 lines

The rule with KNI is:
1) The app should have a separate control thread per rte_kni which just spins calling rte_kni_handle_request(). This ensures that other threads calling rte_kni_XXX functions will always get a response.

2) In order to deal with lockups and timeouts when closing the device, I

sent

Quoted 15 lines

patches which separated the closing process into two steps:
rte_kni_release() which would unregister the underlying netdev, then rte_kni_free() which would free the KNI portions of the KNI device. When rte_kni_release() is called the kernel netdev is unregistered and a response is sent back to the application, the control thread calling rte_kni_handle_request() is still running, so the application will still get a response back from the kernel and not lock up, the application then kills the control thread so that rte_kni_handle_request() is not called again, then the application calls rte_kni_free() which frees all of the FIFOs and closes the device.

If anyone is interested the patches are probably still floating around patchwork. If not you can check them out here:

Quoted 10 lines

https://github.com/danielgora/dpdk.git

thanks-
dan

Quoted 4 lines · Igor Ryzhov

Stephen,

No, I don't have a better proposal, but I think it is not correct to

change

Quoted 2 lines

Quoted 2 lines

the behavior of KNI (making link down without a real response). Even though we know that communicating with userspace under rtnl_lock

is a

Quoted 5 lines

Quoted 5 lines

bad idea, it works as it is for many years already.

Elad,

I agree with you that KNI should be removed from the main tree if it

is not

Quoted 23 lines

Quoted 23 lines

possible to fix this __dev_close_many issue.
There were discussions about this multiple times already, but no one is working on this AFAIK.
Last time the discussion was a month ago:
https://www.mail-archive.com/[email protected]/msg196033.html

Igor

Quoted 13 lines · Elad Nachman

The way the kernel handles its locks and lists for the dev close many path, there is no way you can go around this with rtnl unlocked :
"

There is a race condition in __dev_close_many() processing the close_list while the application terminates.
It looks like if two vEth devices are terminating, and one releases the rtnl lock, the other takes it, updating the close_list in an unstable state, causing the close_list to become a circular linked list, hence list_for_each_entry() will endlessly loop inside __dev_close_many() .

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

"
And I don't expect David Miller will bend the kernel networking for

DPDK

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

or KNI.

But - Stephen - if you can personally convince David to accept a kernel patch which will separate the close_list locking mechanism to

a

Quoted 1 line

Quoted 1 line

Quoted 1 line

separate (RCU?) lock, then I can introduce first a patch to the

kernel

Quoted 8 lines

Quoted 8 lines

Quoted 8 lines

which will add a lock for the close_list, this way rtnl can be unlocked for the if down case.

After that kernel patch, your original patch + relocation of the sync mutex locking will do the job .
lockups
Otherwise, rtnl has to be kept locked all of the way for the if down case in order to prevent corruption causing a circular linked list

out

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

of the close_list, causing a hang in the kernel. lockups
Currently, the rtnl lock is the only thing keeping the close_list

from

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

corruption.

If you doubt rtnl cannot be unlocked for dev close path, you can consult David for his opinion, as I think it is critical to

understand

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

what the kernel can or cannot do, or expects to be done before we can unlock its locks as we wish inside rte_kni.ko .

Otherwise, if we are still in disagreement on how to patch this set

of

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

problems, I think the responsible way around it is to completely remove kni from the main dpdk tree and move it to dpdk-kmods repository.

I know BSD style open-source does not carry legal responsibility from the developers, but I think when a bunch of developers know a piece

of

Quoted 1 line

Quoted 1 line

Quoted 1 line

code is highly buggy, they should not leave it for countless new

users

Quoted 11 lines

Quoted 11 lines

Quoted 11 lines

to bounce their head desperately against, if they cannot agree on a correct way to solve the bunch of problems, of which I think we all agree exist (we just do not agree on the proper solution or patch)...

That's my two cents,

Elad.

Quoted 1 line · Stephen Hemminger

Quoted 8 lines

Quoted 8 lines

Quoted 8 lines

Quoted 8 lines

Quoted 5 lines · On Fri, 26 Feb 2021 00:01:01 +0300 Igor Ryzhov <[email protected]> wrote:

Hi Elad,

Thanks for the patch, but this is still NACK from me.

The only real advantage of KNI over other exceptional-path

techniques

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

like virtio-user is the ability to configure DPDK-managed

interfaces

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

directly
from the kernel using well-known utils like iproute2. A very

important

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

part

Quoted 1 line

Quoted 1 line

of this is getting responses from the DPDK app and knowing the

actual

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

result of command execution.
If you're making async requests to the application and you don't

know

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Quoted 3 lines

the result, then what's the point of using KNI at all?

Igor

Do you have a better proposal that keeps the request result but

does not

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

call userspace with lock held.

PS: I also have strong dislike of KNI, as designed it would have

been

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

rejected

Quoted 1 line

by Linux kernel developers. A better solution would be userspace

version of

Quoted 1 line

something like devlink devices. But doing control operations by

proxy is

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 1 line

a locking nightmare.

--
Dan Gora
Software Engineer

Adax, Inc.
Rua Dona Maria Alves, 1070 Casa 5
Centro
Ubatuba, SP
CEP 11680-000
Brasil

Tel: +55 (12) 3833-1021 (Brazil and outside of US)
: +1 (510) 859-4801 (Inside of US)
: dan_gora (Skype)

email: [email protected]

This part of the series includes my fixes for the issues reported by Ferruh and Igor (and Igor comments for v3 of the patch) on top of part 1 of the patch series:

A. KNI sync lock is being locked while rtnl is held.
If two threads are calling kni_net_process_request() ,
then the first one will take the sync lock, release rtnl lock then sleep. The second thread will try to lock sync lock while holding rtnl. The first thread will wake, and try to lock rtnl, resulting in a deadlock. The remedy is to release rtnl before locking the KNI sync lock. Since in between nothing is accessing Linux network-wise,
no rtnl locking is needed.

B. There is a race condition in __dev_close_many() processing the close_list while the application terminates.
It looks like if two vEth devices are terminating,
and one releases the rtnl lock, the other takes it, updating the close_list in an unstable state,
causing the close_list to become a circular linked list, hence list_for_each_entry() will endlessly loop inside __dev_close_many() .
Since the description for the original patch indicate the original motivation was bringing the device up,
I have changed kni_net_process_request() to hold the rtnl mutex in case of bringing the device down since this is the path called from __dev_close_many() , causing the corruption of the close_list. In order to prevent deadlock in Mellanox device in this case, the code has been modified not to wait for user-space while holding the rtnl lock.
Instead, after the request has been sent, all locks are relinquished and the function exits immediately with return code of zero (success).

To summarize:
request != interface down : unlock rtnl, send request to user-space, wait for response, send the response error code to caller in user-space.

request == interface down: send request to user-space, return immediately with error code of 0 (success) to user-space.

Signed-off-by Elad Nachman <[email protected]>

---
v4:
* for if down case, send asynchronously with rtnl locked and without wait, returning immediately to avoid both kernel race conditions and deadlock in user-space
v3:
* Include original patch and new patch as a series of patch, added a comment to the new patch
v2:
* rebuild the patch as increment from patch 64106
* fix comment and blank lines

---
  kernel/linux/kni/kni_net.c      | 41 +++++++++++++++++++++++++++------
  lib/librte_kni/rte_kni.c        |  7 ++++--
  lib/librte_kni/rte_kni_common.h |  1 +
  3 files changed, 40 insertions(+), 9 deletions(-)

kernel/linux/kni/kni_net.c +0 -0

index f0b6e9a8d..ba991802b 100644
@@ -110,12 +110,34 @@ kni_net_process_request(struct net_device *dev, struct rte_kni_request *req)
  	void *resp_va;
  	uint32_t num;

Quoted 9 lines

This part of the series includes my fixes for the issues reported by Ferruh and Igor (and Igor comments for v3 of the patch) on top of part 1 of the patch series:

A. KNI sync lock is being locked while rtnl is held.
If two threads are calling kni_net_process_request() ,
then the first one will take the sync lock, release rtnl lock then sleep. The second thread will try to lock sync lock while holding rtnl. The first thread will wake, and try to lock rtnl, resulting in a

deadlock.

kernel/linux/kni/kni_net.c kni_net_process_request(struct net_device *dev,

Quoted 55 lines

The remedy is to release rtnl before locking the KNI sync lock. Since in between nothing is accessing Linux network-wise, no rtnl locking is needed.

B. There is a race condition in __dev_close_many() processing the close_list while the application terminates.
It looks like if two vEth devices are terminating,
and one releases the rtnl lock, the other takes it, updating the close_list in an unstable state,
causing the close_list to become a circular linked list, hence list_for_each_entry() will endlessly loop inside __dev_close_many() .
Since the description for the original patch indicate the original motivation was bringing the device up,
I have changed kni_net_process_request() to hold the rtnl mutex in case of bringing the device down since this is the path called from __dev_close_many() , causing the corruption of the close_list. In order to prevent deadlock in Mellanox device in this case, the code has been modified not to wait for user-space while holding the rtnl lock.
Instead, after the request has been sent, all locks are relinquished and the function exits immediately with return code of zero (success).

To summarize:
request != interface down : unlock rtnl, send request to user-space, wait for response, send the response error code to caller in user-space.

request == interface down: send request to user-space, return immediately with error code of 0 (success) to user-space.

Signed-off-by Elad Nachman <[email protected]>

---
v4:
* for if down case, send asynchronously with rtnl locked and without wait, returning immediately to avoid both kernel race conditions and deadlock in user-space
v3:
* Include original patch and new patch as a series of patch, added a comment to the new patch
v2:
* rebuild the patch as increment from patch 64106
* fix comment and blank lines

---
  kernel/linux/kni/kni_net.c      | 41 +++++++++++++++++++++++++++------
  lib/librte_kni/rte_kni.c        |  7 ++++--
  lib/librte_kni/rte_kni_common.h |  1 +
  3 files changed, 40 insertions(+), 9 deletions(-)

kernel/linux/kni/kni_net.c +0 -0

index f0b6e9a8d..ba991802b 100644
@@ -110,12 +110,34 @@ kni_net_process_request(struct net_device *dev,

struct rte_kni_request *req)

kernel/linux/kni/kni_net.c kni_net_process_request(struct net_device *dev, struct rte_kni_request *req) · line 121

Quoted 12 lines

kernel/linux/kni/kni_net.c +9 -0

      void *resp_va;
      uint32_t num;
      int ret_val;
+     int req_is_dev_stop = 0;
+
+     /* For configuring the interface to down,
+      * rtnl must be held all the way to prevent race condition
+      * inside __dev_close_many() between two netdev instances of KNI
+      */
+     if (req->req_id == RTE_KNI_REQ_CFG_NETWORK_IF &&
+                     req->if_up == 0)
+             req_is_dev_stop = 1;

Having this request type checks in the 'kni_net_process_request()' function
looks like hack.
Since adding a new field into the "struct rte_kni_request", that can be a more
generic 'asnyc' field, and the requested function, like 'kni_net_release()' can
set it to support async requests.

And can you please separate the function to add a more generic async request
support on its patch, which should do:
- add new 'asnyc' field to "struct rte_kni_request"
- in 'kni_net_process_request()', if 'req->async' set, do not wait for response
- in library, 'lib/librte_kni/rte_kni.c', in 'rte_kni_handle_request()' function, if the request is async don't put the response (These are already done in this patch)

Overall it can be three patch set:
1) Function parameter change
2) Add generic async request support (with documentation update)
3) rtnl unlock and make 'kni_net_release()' request async (actual fix) (We can discuss more if to make 'kni_net_release()' async with a kernel parameter or not)

What do you think, does it make sense?

Hi,

Owing to my current development schedule and obligations, I see no opportunity to make this set of changes in the near future.

Sorry,

Elad.

בתאריך יום ב׳, 15 במרץ 2021, 19:17, מאת Ferruh Yigit ‏<[email protected] <mailto:[email protected]>>:

On 2/25/2021 2:32 PM, Elad Nachman wrote:
> This part of the series includes my fixes for the issues reported > by Ferruh and Igor (and Igor comments for v3 of the patch)
> on top of part 1 of the patch series:
>
> A. KNI sync lock is being locked while rtnl is held.
> If two threads are calling kni_net_process_request() ,
> then the first one will take the sync lock, release rtnl lock then sleep. > The second thread will try to lock sync lock while holding rtnl. > The first thread will wake, and try to lock rtnl, resulting in a deadlock. > The remedy is to release rtnl before locking the KNI sync lock. > Since in between nothing is accessing Linux network-wise,
> no rtnl locking is needed.
>
> B. There is a race condition in __dev_close_many() processing the > close_list while the application terminates.
> It looks like if two vEth devices are terminating,
> and one releases the rtnl lock, the other takes it,
> updating the close_list in an unstable state,
> causing the close_list to become a circular linked list,
> hence list_for_each_entry() will endlessly loop inside
> __dev_close_many() .
> Since the description for the original patch indicate the
> original motivation was bringing the device up,
> I have changed kni_net_process_request() to hold the rtnl mutex > in case of bringing the device down since this is the path called > from __dev_close_many() , causing the corruption of the close_list. > In order to prevent deadlock in Mellanox device in this case, the > code has been modified not to wait for user-space while holding > the rtnl lock.
> Instead, after the request has been sent, all locks are relinquished > and the function exits immediately with return code of zero (success). >
> To summarize:
> request != interface down : unlock rtnl, send request to user-space, > wait for response, send the response error code to caller in user-space. >
> request == interface down: send request to user-space, return immediately > with error code of 0 (success) to user-space.
>
> Signed-off-by: Elad Nachman <[email protected] <mailto:[email protected]>> >
>
> ---
> v4:
> * for if down case, send asynchronously with rtnl locked and without >    wait, returning immediately to avoid both kernel race conditions >    and deadlock in user-space
> v3:
> * Include original patch and new patch as a series of patch, added a >    comment to the new patch
> v2:
> * rebuild the patch as increment from patch 64106
> * fix comment and blank lines
> ---
>   kernel/linux/kni/kni_net.c      | 41 +++++++++++++++++++++++++++------ >   lib/librte_kni/rte_kni.c        |  7 ++++--
>   lib/librte_kni/rte_kni_common.h |  1 +
>   3 files changed, 40 insertions(+), 9 deletions(-)
>
> diff --git a/kernel/linux/kni/kni_net.c b/kernel/linux/kni/kni_net.c > index f0b6e9a8d..ba991802b 100644
> --- a/kernel/linux/kni/kni_net.c
> +++ b/kernel/linux/kni/kni_net.c
> @@ -110,12 +110,34 @@ kni_net_process_request(struct net_device *dev,
struct rte_kni_request *req)
>       void *resp_va;
>       uint32_t num;
>       int ret_val;
> +     int req_is_dev_stop = 0;
> +
> +     /* For configuring the interface to down,
> +      * rtnl must be held all the way to prevent race condition > +      * inside __dev_close_many() between two netdev instances of KNI > +      */
> +     if (req->req_id == RTE_KNI_REQ_CFG_NETWORK_IF &&
> +                     req->if_up == 0)
> +             req_is_dev_stop = 1;

Having this request type checks in the 'kni_net_process_request()' function looks like hack.
Since adding a new field into the "struct rte_kni_request", that can be a more generic 'asnyc' field, and the requested function, like 'kni_net_release()' can set it to support async requests.

And can you please separate the function to add a more generic async request support on its patch, which should do:
- add new 'asnyc' field to "struct rte_kni_request"
- in 'kni_net_process_request()', if 'req->async' set, do not wait for response
- in library, 'lib/librte_kni/rte_kni.c', in 'rte_kni_handle_request()' function, if the request is async don't put the response
(These are already done in this patch)

Overall it can be three patch set:
1) Function parameter change
2) Add generic async request support (with documentation update)
3) rtnl unlock and make 'kni_net_release()' request async (actual fix) (We can discuss more if to make 'kni_net_release()' async with a kernel parameter or not)

What do you think, does it make sense?

This first part of v4 of the patch re-introduces Stephen Hemminger's patch 64106 . This part changes the parameter kni_net_process_request() gets and introduces the initial rtnl unlocking mechanism.

Signed-off-by Elad Nachman <[email protected]>

---
v4:
* for if down case, send asynchronously with rtnl locked and without wait, returning immediately to avoid both kernel race conditions and deadlock in user-space
v3:
* Include original patch and new patch as a series of patch, added a comment to the new patch
v2:
* rebuild the patch as increment from patch 64106
* fix comment and blank lines

This part of the series includes my fixes for the issues reported by Ferruh and Igor on top of part 1 of the patch series:

A. KNI sync lock is being locked while rtnl is held.
If two threads are calling kni_net_process_request() ,
then the first one will take the sync lock, release rtnl lock then sleep. The second thread will try to lock sync lock while holding rtnl. The first thread will wake, and try to lock rtnl, resulting in a deadlock. The remedy is to release rtnl before locking the KNI sync lock. Since in between nothing is accessing Linux network-wise,
no rtnl locking is needed.

B. There is a race condition in __dev_close_many() processing the close_list while the application terminates.
It looks like if two vEth devices are terminating,
and one releases the rtnl lock, the other takes it, updating the close_list in an unstable state,
causing the close_list to become a circular linked list, hence list_for_each_entry() will endlessly loop inside __dev_close_many() .
Since the description for the original patch indicate the original motivation was bringing the device up,
I have changed kni_net_process_request() to hold the rtnl mutex in case of bringing the device down since this is the path called from __dev_close_many() , causing the corruption of the close_list.

Signed-off-by Elad Nachman <[email protected]>

---
v3:
* Include original patch and new patch as a series of patch, added a comment to the new patch
v2:
* rebuild the patch as increment from patch 64106
* fix comment and blank lines

---
 kernel/linux/kni/kni_net.c | 29 +++++++++++++++++++++--------
 1 file changed, 21 insertions(+), 8 deletions(-)

kernel/linux/kni/kni_net.c +17 -0

index f0b6e9a8d..017e44812 100644
@@ -110,9 +110,26 @@ kni_net_process_request(struct net_device *dev,
 struct rte_kni_request *req)
        void *resp_va;
        uint32_t num;
        int ret_val;
+       int req_is_dev_stop = 0;
+
+       /* For configuring the interface to down,
+        * rtnl must be held all the way to prevent race condition
+        * inside __dev_close_many() between two netdev instances of KNI
+        */
+       if (req->req_id == RTE_KNI_REQ_CFG_NETWORK_IF &&
+                       req->if_up == 0)
+               req_is_dev_stop = 1;
 
        ASSERT_RTNL();
 
+       /* Since we need to wait and RTNL mutex is held
+        * drop the mutex and hold reference to keep device
+        */
+       if (!req_is_dev_stop) {
+               dev_hold(dev);
+               rtnl_unlock();
+       }
+
        mutex_lock(&kni->sync_lock);
 

        /* Construct data */

kernel/linux/kni/kni_net.c +0 -8

@@ -124,16 +141,8 @@ kni_net_process_request(struct net_device *dev,
 struct rte_kni_request *req)
                goto fail;
        }
 
-       /* Since we need to wait and RTNL mutex is held
-        * drop the mutex and hold refernce to keep device
-        */
-       dev_hold(dev);
-       rtnl_unlock();
-
        ret_val = wait_event_interruptible_timeout(kni->wq,
                        kni_fifo_count(kni->resp_q), 3 * HZ);
-       rtnl_lock();
-       dev_put(dev);
 
        if (signal_pending(current) || ret_val <= 0) {

                ret = -ETIME;

kernel/linux/kni/kni_net.c +4 -1

@@ -152,6 +161,10 @@ kni_net_process_request(struct net_device *dev,
 struct rte_kni_request *req)
 
 fail:
        mutex_unlock(&kni->sync_lock);
+       if (!req_is_dev_stop) {
+               rtnl_lock();
+               dev_put(dev);
+       }
        return ret;
 }
 
--

2.17.1

This looks more like a hack than an actual fix to me.

After this commit:
"ip link set up" is sent to the userspace with unlocked rtnl_lock "ip link set down" is sent to the userspace with locked rtnl_lock

How is this really fixing anything? IMHO it only complicates the code.
If talking with userspace under rtnl_lock is a problem, then we should fix all such requests, not only part of them. If it is not a problem, then I don't see any point in merging this.

Quoted 101 lines · Elad Nachman

This part of the series includes my fixes for the issues reported by Ferruh and Igor on top of part 1 of the patch series:

A. KNI sync lock is being locked while rtnl is held.
If two threads are calling kni_net_process_request() ,
then the first one will take the sync lock, release rtnl lock then sleep. The second thread will try to lock sync lock while holding rtnl. The first thread will wake, and try to lock rtnl, resulting in a deadlock. The remedy is to release rtnl before locking the KNI sync lock. Since in between nothing is accessing Linux network-wise,
no rtnl locking is needed.

B. There is a race condition in __dev_close_many() processing the close_list while the application terminates.
It looks like if two vEth devices are terminating,
and one releases the rtnl lock, the other takes it, updating the close_list in an unstable state,
causing the close_list to become a circular linked list, hence list_for_each_entry() will endlessly loop inside __dev_close_many() .
Since the description for the original patch indicate the original motivation was bringing the device up,
I have changed kni_net_process_request() to hold the rtnl mutex in case of bringing the device down since this is the path called from __dev_close_many() , causing the corruption of the close_list.

Signed-off-by Elad Nachman <[email protected]>

---
v3:
* Include original patch and new patch as a series of patch, added a comment to the new patch
v2:
* rebuild the patch as increment from patch 64106
* fix comment and blank lines

---
 kernel/linux/kni/kni_net.c | 29 +++++++++++++++++++++--------
 1 file changed, 21 insertions(+), 8 deletions(-)

kernel/linux/kni/kni_net.c +21 -9

index f0b6e9a8d..017e44812 100644
@@ -110,9 +110,26 @@ kni_net_process_request(struct net_device *dev, struct rte_kni_request *req)
        void *resp_va;
        uint32_t num;
        int ret_val;
+       int req_is_dev_stop = 0;
+
+       /* For configuring the interface to down,
+        * rtnl must be held all the way to prevent race condition
+        * inside __dev_close_many() between two netdev instances of KNI
+        */
+       if (req->req_id == RTE_KNI_REQ_CFG_NETWORK_IF &&
+                       req->if_up == 0)
+               req_is_dev_stop = 1;
 
        ASSERT_RTNL();
 
+       /* Since we need to wait and RTNL mutex is held
+        * drop the mutex and hold reference to keep device
+        */
+       if (!req_is_dev_stop) {
+               dev_hold(dev);
+               rtnl_unlock();
+       }
+
        mutex_lock(&kni->sync_lock);
 
        /* Construct data */

@@ -124,16 +141,8 @@ kni_net_process_request(struct net_device *dev, struct rte_kni_request *req)
                goto fail;
        }
 
-       /* Since we need to wait and RTNL mutex is held
-        * drop the mutex and hold refernce to keep device
-        */
-       dev_hold(dev);
-       rtnl_unlock();
-
        ret_val = wait_event_interruptible_timeout(kni->wq,
                        kni_fifo_count(kni->resp_q), 3 * HZ);
-       rtnl_lock();
-       dev_put(dev);
 
        if (signal_pending(current) || ret_val <= 0) {
                ret = -ETIME;

@@ -152,6 +161,10 @@ kni_net_process_request(struct net_device *dev, struct rte_kni_request *req)
 
 fail:
        mutex_unlock(&kni->sync_lock);
+       if (!req_is_dev_stop) {
+               rtnl_lock();
+               dev_put(dev);
+       }
        return ret;
 }
 
--

2.17.1

Currently KNI has a lot of issues with deadlocks locking the code, after this commit, they are gone, and the code runs properly without crashing.
That was tested with over 100 restarts of the application, which previously required a hard reset of the board.

I think this benefit overweights the complication of the code.

The function is called with rtnl locked because this is how the Linux kernel is designed to work - it is not designed to work with deferral to user-space mid-function.

To fix all such requests you need to reach an agreement with Linux netdev, which is unlikely.

Calling user-space can be done asynchronously, as Ferruh asked, but then you will always have to return success, even on failure, as Linux kernel does not have a mechanism to asynchronously report on failure for such system calls.

IMHO - weighting the non-reporting of failure versus how the code looks (as it functions perfectly OK), I decided to go with functionality.

FYI,

Elad.

Quoted 9 lines · Igor Ryzhov

This looks more like a hack than an actual fix to me.

After this commit:
"ip link set up" is sent to the userspace with unlocked rtnl_lock "ip link set down" is sent to the userspace with locked rtnl_lock

How is this really fixing anything? IMHO it only complicates the code. If talking with userspace under rtnl_lock is a problem, then we should

fix all such requests, not only part of them.

Quoted 10 lines

If it is not a problem, then I don't see any point in merging this.

Quoted 7 lines · Elad Nachman

This part of the series includes my fixes for the issues reported by Ferruh and Igor on top of part 1 of the patch series:

A. KNI sync lock is being locked while rtnl is held.
If two threads are calling kni_net_process_request() , then the first one will take the sync lock, release rtnl lock then

sleep.

Quoted 2 lines

Quoted 2 lines

The second thread will try to lock sync lock while holding rtnl. The first thread will wake, and try to lock rtnl, resulting in a

deadlock.

Quoted 30 lines

Quoted 30 lines

The remedy is to release rtnl before locking the KNI sync lock. Since in between nothing is accessing Linux network-wise, no rtnl locking is needed.

B. There is a race condition in __dev_close_many() processing the close_list while the application terminates.
It looks like if two vEth devices are terminating,
and one releases the rtnl lock, the other takes it, updating the close_list in an unstable state,
causing the close_list to become a circular linked list, hence list_for_each_entry() will endlessly loop inside __dev_close_many() .
Since the description for the original patch indicate the original motivation was bringing the device up,
I have changed kni_net_process_request() to hold the rtnl mutex in case of bringing the device down since this is the path called from __dev_close_many() , causing the corruption of the close_list.

Signed-off-by Elad Nachman <[email protected]>

---
v3:
* Include original patch and new patch as a series of patch, added a comment to the new patch
v2:
* rebuild the patch as increment from patch 64106
* fix comment and blank lines

---
 kernel/linux/kni/kni_net.c | 29 +++++++++++++++++++++--------
 1 file changed, 21 insertions(+), 8 deletions(-)

Quoted 5 lines

kernel/linux/kni/kni_net.c kni_net_process_request(struct net_device *dev,

Quoted 5 lines

kernel/linux/kni/kni_net.c +0 -0

index f0b6e9a8d..017e44812 100644
@@ -110,9 +110,26 @@ kni_net_process_request(struct net_device *dev,

struct rte_kni_request *req)

Quoted 25 lines

Quoted 25 lines

kernel/linux/kni/kni_net.c +17 -0

        void *resp_va;
        uint32_t num;
        int ret_val;
+       int req_is_dev_stop = 0;
+
+       /* For configuring the interface to down,
+        * rtnl must be held all the way to prevent race condition
+        * inside __dev_close_many() between two netdev instances of KNI
+        */
+       if (req->req_id == RTE_KNI_REQ_CFG_NETWORK_IF &&
+                       req->if_up == 0)
+               req_is_dev_stop = 1;
 
        ASSERT_RTNL();
 
+       /* Since we need to wait and RTNL mutex is held
+        * drop the mutex and hold reference to keep device
+        */
+       if (!req_is_dev_stop) {
+               dev_hold(dev);
+               rtnl_unlock();
+       }
+
        mutex_lock(&kni->sync_lock);
 

Quoted 2 lines

kernel/linux/kni/kni_net.c kni_net_process_request(struct net_device *dev,

Quoted 2 lines

kernel/linux/kni/kni_net.c +0 -0

        /* Construct data */

@@ -124,16 +141,8 @@ kni_net_process_request(struct net_device *dev,

struct rte_kni_request *req)

Quoted 14 lines

Quoted 14 lines

kernel/linux/kni/kni_net.c +0 -8

                goto fail;
        }
 
-       /* Since we need to wait and RTNL mutex is held
-        * drop the mutex and hold refernce to keep device
-        */
-       dev_hold(dev);
-       rtnl_unlock();
-
        ret_val = wait_event_interruptible_timeout(kni->wq,
                        kni_fifo_count(kni->resp_q), 3 * HZ);
-       rtnl_lock();
-       dev_put(dev);
 

Quoted 3 lines

kernel/linux/kni/kni_net.c kni_net_process_request(struct net_device *dev,

Quoted 3 lines

kernel/linux/kni/kni_net.c +0 -0

        if (signal_pending(current) || ret_val <= 0) {
                ret = -ETIME;

@@ -152,6 +161,10 @@ kni_net_process_request(struct net_device *dev,

struct rte_kni_request *req)

Quoted 13 lines

Quoted 13 lines

kernel/linux/kni/kni_net.c +4 -1

 fail:
        mutex_unlock(&kni->sync_lock);
+       if (!req_is_dev_stop) {
+               rtnl_lock();
+               dev_put(dev);
+       }
        return ret;
 }
 
--

2.17.1

Elad,

I understand your point.
But the fact that this fix works for you doesn't mean that it will work for all DPDK users.

For example, I provided two simple commands: "ip link set up" and "ip link set down". Your fix works for only one of them. For me, this is not a proper fix.
It may work for you because you don't disable interfaces, but it will fail for users who do.

Quoted 143 lines · Elad Nachman

Currently KNI has a lot of issues with deadlocks locking the code, after this commit, they are gone, and the code runs properly without crashing.
That was tested with over 100 restarts of the application, which previously required a hard reset of the board.

I think this benefit overweights the complication of the code.

The function is called with rtnl locked because this is how the Linux kernel is designed to work - it is not designed to work with deferral to user-space mid-function.

To fix all such requests you need to reach an agreement with Linux netdev, which is unlikely.

Calling user-space can be done asynchronously, as Ferruh asked, but then you will always have to return success, even on failure, as Linux kernel does not have a mechanism to asynchronously report on failure for such system calls.

IMHO - weighting the non-reporting of failure versus how the code looks (as it functions perfectly OK), I decided to go with functionality.

FYI,

Elad.

Quoted 113 lines · Igor Ryzhov

This looks more like a hack than an actual fix to me.

After this commit:
"ip link set up" is sent to the userspace with unlocked rtnl_lock "ip link set down" is sent to the userspace with locked rtnl_lock

How is this really fixing anything? IMHO it only complicates the code.
If talking with userspace under rtnl_lock is a problem, then we should fix all such requests, not only part of them. If it is not a problem, then I don't see any point in merging this.

Quoted 101 lines · Elad Nachman

This part of the series includes my fixes for the issues reported by Ferruh and Igor on top of part 1 of the patch series:

A. KNI sync lock is being locked while rtnl is held.
If two threads are calling kni_net_process_request() ,
then the first one will take the sync lock, release rtnl lock then sleep. The second thread will try to lock sync lock while holding rtnl. The first thread will wake, and try to lock rtnl, resulting in a deadlock. The remedy is to release rtnl before locking the KNI sync lock. Since in between nothing is accessing Linux network-wise,
no rtnl locking is needed.

B. There is a race condition in __dev_close_many() processing the close_list while the application terminates.
It looks like if two vEth devices are terminating,
and one releases the rtnl lock, the other takes it, updating the close_list in an unstable state,
causing the close_list to become a circular linked list, hence list_for_each_entry() will endlessly loop inside __dev_close_many() .
Since the description for the original patch indicate the original motivation was bringing the device up,
I have changed kni_net_process_request() to hold the rtnl mutex in case of bringing the device down since this is the path called from __dev_close_many() , causing the corruption of the close_list.

Signed-off-by Elad Nachman <[email protected]>

---
v3:
* Include original patch and new patch as a series of patch, added a comment to the new patch
v2:
* rebuild the patch as increment from patch 64106
* fix comment and blank lines

---
 kernel/linux/kni/kni_net.c | 29 +++++++++++++++++++++--------
 1 file changed, 21 insertions(+), 8 deletions(-)

kernel/linux/kni/kni_net.c +21 -9

index f0b6e9a8d..017e44812 100644
@@ -110,9 +110,26 @@ kni_net_process_request(struct net_device *dev, struct rte_kni_request *req)
        void *resp_va;
        uint32_t num;
        int ret_val;
+       int req_is_dev_stop = 0;
+
+       /* For configuring the interface to down,
+        * rtnl must be held all the way to prevent race condition
+        * inside __dev_close_many() between two netdev instances of KNI
+        */
+       if (req->req_id == RTE_KNI_REQ_CFG_NETWORK_IF &&
+                       req->if_up == 0)
+               req_is_dev_stop = 1;
 
        ASSERT_RTNL();
 
+       /* Since we need to wait and RTNL mutex is held
+        * drop the mutex and hold reference to keep device
+        */
+       if (!req_is_dev_stop) {
+               dev_hold(dev);
+               rtnl_unlock();
+       }
+
        mutex_lock(&kni->sync_lock);
 
        /* Construct data */

@@ -124,16 +141,8 @@ kni_net_process_request(struct net_device *dev, struct rte_kni_request *req)
                goto fail;
        }
 
-       /* Since we need to wait and RTNL mutex is held
-        * drop the mutex and hold refernce to keep device
-        */
-       dev_hold(dev);
-       rtnl_unlock();
-
        ret_val = wait_event_interruptible_timeout(kni->wq,
                        kni_fifo_count(kni->resp_q), 3 * HZ);
-       rtnl_lock();
-       dev_put(dev);
 
        if (signal_pending(current) || ret_val <= 0) {
                ret = -ETIME;

@@ -152,6 +161,10 @@ kni_net_process_request(struct net_device *dev, struct rte_kni_request *req)
 
 fail:
        mutex_unlock(&kni->sync_lock);
+       if (!req_is_dev_stop) {
+               rtnl_lock();
+               dev_put(dev);
+       }
        return ret;
 }
 
--

2.17.1

I tested both link up and link down many times without any problems on 100 restarts of the application.

Having KNI deadlock frequently for real life applications is far worst, IMHO.

FYI

Elad.

Quoted 5 lines · Igor Ryzhov

Elad,

I understand your point.
But the fact that this fix works for you doesn't mean that it will work

for all DPDK users.

Quoted 2 lines

For example, I provided two simple commands: "ip link set up" and "ip

link set down".

Quoted 2 lines

Your fix works for only one of them. For me, this is not a proper fix. It may work for you because you don't disable interfaces, but it will

fail for users who do.

Quoted 33 lines · Elad Nachman

Quoted 31 lines

Currently KNI has a lot of issues with deadlocks locking the code, after this commit, they are gone, and the code runs properly without crashing.
That was tested with over 100 restarts of the application, which previously required a hard reset of the board.

I think this benefit overweights the complication of the code.

The function is called with rtnl locked because this is how the Linux kernel is designed to work - it is not designed to work with deferral to user-space mid-function.

To fix all such requests you need to reach an agreement with Linux netdev, which is unlikely.

Calling user-space can be done asynchronously, as Ferruh asked, but then you will always have to return success, even on failure, as Linux kernel does not have a mechanism to asynchronously report on failure for such system calls.

IMHO - weighting the non-reporting of failure versus how the code looks (as it functions perfectly OK), I decided to go with functionality.

FYI,

Elad.

Quoted 1 line · Igor Ryzhov

Quoted 8 lines

Quoted 8 lines

Quoted 8 lines

This looks more like a hack than an actual fix to me.

After this commit:
"ip link set up" is sent to the userspace with unlocked rtnl_lock "ip link set down" is sent to the userspace with locked rtnl_lock

How is this really fixing anything? IMHO it only complicates the code. If talking with userspace under rtnl_lock is a problem, then we

should fix all such requests, not only part of them.

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

If it is not a problem, then I don't see any point in merging this.

On Tue, Feb 23, 2021 at 4:45 PM Elad Nachman <[email protected]>

Quoted 7 lines · wrote:

Quoted 7 lines

Quoted 7 lines

Quoted 7 lines

This part of the series includes my fixes for the issues reported by Ferruh and Igor on top of part 1 of the patch series:

A. KNI sync lock is being locked while rtnl is held.
If two threads are calling kni_net_process_request() , then the first one will take the sync lock, release rtnl lock then

sleep.

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

The second thread will try to lock sync lock while holding rtnl. The first thread will wake, and try to lock rtnl, resulting in a

deadlock.

Quoted 30 lines

Quoted 30 lines

Quoted 30 lines

Quoted 30 lines

The remedy is to release rtnl before locking the KNI sync lock. Since in between nothing is accessing Linux network-wise, no rtnl locking is needed.

B. There is a race condition in __dev_close_many() processing the close_list while the application terminates.
It looks like if two vEth devices are terminating,
and one releases the rtnl lock, the other takes it, updating the close_list in an unstable state,
causing the close_list to become a circular linked list, hence list_for_each_entry() will endlessly loop inside __dev_close_many() .
Since the description for the original patch indicate the original motivation was bringing the device up,
I have changed kni_net_process_request() to hold the rtnl mutex in case of bringing the device down since this is the path called from __dev_close_many() , causing the corruption of the close_list.

Signed-off-by Elad Nachman <[email protected]>

---
v3:
* Include original patch and new patch as a series of patch, added a comment to the new patch
v2:
* rebuild the patch as increment from patch 64106
* fix comment and blank lines

---
 kernel/linux/kni/kni_net.c | 29 +++++++++++++++++++++--------
 1 file changed, 21 insertions(+), 8 deletions(-)

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

kernel/linux/kni/kni_net.c kni_net_process_request(struct net_device *dev,

Quoted 5 lines

kernel/linux/kni/kni_net.c +0 -0

index f0b6e9a8d..017e44812 100644
@@ -110,9 +110,26 @@ kni_net_process_request(struct net_device *dev,

struct rte_kni_request *req)

Quoted 8 lines

Quoted 8 lines

Quoted 8 lines

kernel/linux/kni/kni_net.c kni_net_process_request(struct net_device *dev, struct rte_kni_request *req)

Quoted 8 lines

kernel/linux/kni/kni_net.c +5 -0

        void *resp_va;
        uint32_t num;
        int ret_val;
+       int req_is_dev_stop = 0;
+
+       /* For configuring the interface to down,
+        * rtnl must be held all the way to prevent race condition
+        * inside __dev_close_many() between two netdev instances of

KNI

Quoted 17 lines

Quoted 17 lines

Quoted 17 lines

Quoted 17 lines

kernel/linux/kni/kni_net.c +12 -0

+        */
+       if (req->req_id == RTE_KNI_REQ_CFG_NETWORK_IF &&
+                       req->if_up == 0)
+               req_is_dev_stop = 1;
 
        ASSERT_RTNL();
 
+       /* Since we need to wait and RTNL mutex is held
+        * drop the mutex and hold reference to keep device
+        */
+       if (!req_is_dev_stop) {
+               dev_hold(dev);
+               rtnl_unlock();
+       }
+
        mutex_lock(&kni->sync_lock);
 

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

kernel/linux/kni/kni_net.c kni_net_process_request(struct net_device *dev,

Quoted 2 lines

kernel/linux/kni/kni_net.c +0 -0

        /* Construct data */

@@ -124,16 +141,8 @@ kni_net_process_request(struct net_device *dev,

struct rte_kni_request *req)

Quoted 14 lines

Quoted 14 lines

Quoted 14 lines

Quoted 14 lines

kernel/linux/kni/kni_net.c +0 -8

                goto fail;
        }
 
-       /* Since we need to wait and RTNL mutex is held
-        * drop the mutex and hold refernce to keep device
-        */
-       dev_hold(dev);
-       rtnl_unlock();
-
        ret_val = wait_event_interruptible_timeout(kni->wq,
                        kni_fifo_count(kni->resp_q), 3 * HZ);
-       rtnl_lock();
-       dev_put(dev);
 

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

kernel/linux/kni/kni_net.c kni_net_process_request(struct net_device *dev,

Quoted 3 lines

kernel/linux/kni/kni_net.c +0 -0

        if (signal_pending(current) || ret_val <= 0) {
                ret = -ETIME;

@@ -152,6 +161,10 @@ kni_net_process_request(struct net_device *dev,

struct rte_kni_request *req)

Quoted 13 lines

Quoted 13 lines

Quoted 13 lines

Quoted 13 lines

kernel/linux/kni/kni_net.c +4 -1

 fail:
        mutex_unlock(&kni->sync_lock);
+       if (!req_is_dev_stop) {
+               rtnl_lock();
+               dev_put(dev);
+       }
        return ret;
 }
 
--

2.17.1

Both link up and link down also work for me without this patch. So what's the point in merging it?

Just to clarify - I am not against the idea of this patch. Talking to userspace under rtnl_lock is a bad idea. I just think that any patch should fix some specified problem.

If this patch is trying to solve the overall "userspace request under rtnl_lock" problem, then it doesn't solve it correctly, because we still send link down requests under the lock.

If this patch is trying to solve some other issue, for example, some "KNI deadlocks" you're talking about, then you should explain what these deadlocks are, how to reproduce them and why this patch solves the issue.

Quoted 165 lines · Elad Nachman

I tested both link up and link down many times without any problems on 100 restarts of the application.

Having KNI deadlock frequently for real life applications is far worst, IMHO.

FYI

Elad.

Quoted 154 lines · Igor Ryzhov

Elad,

I understand your point.
But the fact that this fix works for you doesn't mean that it will work for all DPDK users.

For example, I provided two simple commands: "ip link set up" and "ip link set down". Your fix works for only one of them. For me, this is not a proper fix.
It may work for you because you don't disable interfaces, but it will fail for users who do.

Quoted 143 lines · Elad Nachman

Currently KNI has a lot of issues with deadlocks locking the code, after this commit, they are gone, and the code runs properly without crashing.
That was tested with over 100 restarts of the application, which previously required a hard reset of the board.

I think this benefit overweights the complication of the code.

The function is called with rtnl locked because this is how the Linux kernel is designed to work - it is not designed to work with deferral to user-space mid-function.

To fix all such requests you need to reach an agreement with Linux netdev, which is unlikely.

Calling user-space can be done asynchronously, as Ferruh asked, but then you will always have to return success, even on failure, as Linux kernel does not have a mechanism to asynchronously report on failure for such system calls.

IMHO - weighting the non-reporting of failure versus how the code looks (as it functions perfectly OK), I decided to go with functionality.

FYI,

Elad.

Quoted 113 lines · Igor Ryzhov

This looks more like a hack than an actual fix to me.

After this commit:
"ip link set up" is sent to the userspace with unlocked rtnl_lock "ip link set down" is sent to the userspace with locked rtnl_lock

How is this really fixing anything? IMHO it only complicates the code.
If talking with userspace under rtnl_lock is a problem, then we should fix all such requests, not only part of them. If it is not a problem, then I don't see any point in merging this.

Quoted 101 lines · Elad Nachman

This part of the series includes my fixes for the issues reported by Ferruh and Igor on top of part 1 of the patch series:

A. KNI sync lock is being locked while rtnl is held.
If two threads are calling kni_net_process_request() ,
then the first one will take the sync lock, release rtnl lock then sleep. The second thread will try to lock sync lock while holding rtnl. The first thread will wake, and try to lock rtnl, resulting in a deadlock. The remedy is to release rtnl before locking the KNI sync lock. Since in between nothing is accessing Linux network-wise,
no rtnl locking is needed.

B. There is a race condition in __dev_close_many() processing the close_list while the application terminates.
It looks like if two vEth devices are terminating,
and one releases the rtnl lock, the other takes it, updating the close_list in an unstable state,
causing the close_list to become a circular linked list, hence list_for_each_entry() will endlessly loop inside __dev_close_many() .
Since the description for the original patch indicate the original motivation was bringing the device up,
I have changed kni_net_process_request() to hold the rtnl mutex in case of bringing the device down since this is the path called from __dev_close_many() , causing the corruption of the close_list.

Signed-off-by Elad Nachman <[email protected]>

---
v3:
* Include original patch and new patch as a series of patch, added a comment to the new patch
v2:
* rebuild the patch as increment from patch 64106
* fix comment and blank lines

---
 kernel/linux/kni/kni_net.c | 29 +++++++++++++++++++++--------
 1 file changed, 21 insertions(+), 8 deletions(-)

kernel/linux/kni/kni_net.c +21 -9

index f0b6e9a8d..017e44812 100644
@@ -110,9 +110,26 @@ kni_net_process_request(struct net_device *dev, struct rte_kni_request *req)
        void *resp_va;
        uint32_t num;
        int ret_val;
+       int req_is_dev_stop = 0;
+
+       /* For configuring the interface to down,
+        * rtnl must be held all the way to prevent race condition
+        * inside __dev_close_many() between two netdev instances of KNI
+        */
+       if (req->req_id == RTE_KNI_REQ_CFG_NETWORK_IF &&
+                       req->if_up == 0)
+               req_is_dev_stop = 1;
 
        ASSERT_RTNL();
 
+       /* Since we need to wait and RTNL mutex is held
+        * drop the mutex and hold reference to keep device
+        */
+       if (!req_is_dev_stop) {
+               dev_hold(dev);
+               rtnl_unlock();
+       }
+
        mutex_lock(&kni->sync_lock);
 
        /* Construct data */

@@ -124,16 +141,8 @@ kni_net_process_request(struct net_device *dev, struct rte_kni_request *req)
                goto fail;
        }
 
-       /* Since we need to wait and RTNL mutex is held
-        * drop the mutex and hold refernce to keep device
-        */
-       dev_hold(dev);
-       rtnl_unlock();
-
        ret_val = wait_event_interruptible_timeout(kni->wq,
                        kni_fifo_count(kni->resp_q), 3 * HZ);
-       rtnl_lock();
-       dev_put(dev);
 
        if (signal_pending(current) || ret_val <= 0) {
                ret = -ETIME;

@@ -152,6 +161,10 @@ kni_net_process_request(struct net_device *dev, struct rte_kni_request *req)
 
 fail:
        mutex_unlock(&kni->sync_lock);
+       if (!req_is_dev_stop) {
+               rtnl_lock();
+               dev_put(dev);
+       }
        return ret;
 }
 
--

2.17.1

The deadlock scenarios are explained below:

It is described in Stephen Hemminger's original patch:

"

This fixes a deadlock when using KNI with bifurcated drivers. Bringing kni device up always times out when using Mellanox devices.

The kernel KNI driver sends message to userspace to complete the request. For the case of bifurcated driver, this may involve an additional request to kernel to change state. This request would deadlock because KNI was holding the RTNL mutex.

"

And also in my patch:

"
KNI sync lock is being locked while rtnl is held.
If two threads are calling kni_net_process_request() ,
then the first one will take the sync lock, release rtnl lock then sleep. The second thread will try to lock sync lock while holding rtnl. The first thread will wake, and try to lock rtnl, resulting in a deadlock. The remedy is to release rtnl before locking the KNI sync lock. Since in between nothing is accessing Linux network-wise,
no rtnl locking is needed.
"

FYI,

Elad.

Quoted 9 lines · Igor Ryzhov

Both link up and link down also work for me without this patch. So what's the point in merging it?

Just to clarify - I am not against the idea of this patch. Talking to userspace under rtnl_lock is a bad idea. I just think that any patch should fix some specified problem.

If this patch is trying to solve the overall "userspace request under

rtnl_lock" problem,

Quoted 1 line

then it doesn't solve it correctly, because we still send link down

requests under the lock.

Quoted 2 lines

If this patch is trying to solve some other issue, for example, some

"KNI deadlocks"

Quoted 1 line

you're talking about, then you should explain what these deadlocks are,

how to reproduce

Quoted 8 lines

them and why this patch solves the issue.

Quoted 5 lines · Elad Nachman

I tested both link up and link down many times without any problems on 100 restarts of the application.

Having KNI deadlock frequently for real life applications is far worst,

IMHO.

Quoted 11 lines

Quoted 11 lines

FYI

Elad.

Quoted 5 lines · Igor Ryzhov

Elad,

I understand your point.
But the fact that this fix works for you doesn't mean that it will

work for all DPDK users.

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

For example, I provided two simple commands: "ip link set up" and "ip

link set down".

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Your fix works for only one of them. For me, this is not a proper fix. It may work for you because you don't disable interfaces, but it will

fail for users who do.

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

On Wed, Feb 24, 2021 at 4:33 PM Elad Nachman <[email protected]>

Quoted 13 lines · wrote:

Quoted 13 lines

Quoted 13 lines

Quoted 13 lines

Currently KNI has a lot of issues with deadlocks locking the code, after this commit, they are gone, and the code runs properly without crashing.
That was tested with over 100 restarts of the application, which previously required a hard reset of the board.

I think this benefit overweights the complication of the code.

The function is called with rtnl locked because this is how the Linux kernel is designed to work - it is not designed to work with deferral to user-space mid-function.

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

To fix all such requests you need to reach an agreement with Linux netdev, which is unlikely.

Calling user-space can be done asynchronously, as Ferruh asked, but then you will always have to return success, even on failure, as

Linux

Quoted 12 lines

Quoted 12 lines

Quoted 12 lines

Quoted 12 lines

kernel does not have a mechanism to asynchronously report on failure for such system calls.

IMHO - weighting the non-reporting of failure versus how the code looks (as it functions perfectly OK), I decided to go with functionality.

FYI,

Elad.

On Wed, Feb 24, 2021 at 2:50 PM Igor Ryzhov <[email protected]>

Quoted 8 lines · wrote:

Quoted 8 lines

Quoted 8 lines

Quoted 8 lines

Quoted 8 lines

This looks more like a hack than an actual fix to me.

After this commit:
"ip link set up" is sent to the userspace with unlocked rtnl_lock "ip link set down" is sent to the userspace with locked rtnl_lock

How is this really fixing anything? IMHO it only complicates the

code.

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

If talking with userspace under rtnl_lock is a problem, then we

should fix all such requests, not only part of them.

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

If it is not a problem, then I don't see any point in merging this.

On Tue, Feb 23, 2021 at 4:45 PM Elad Nachman <[email protected]>

Quoted 7 lines · wrote:

Quoted 7 lines

Quoted 7 lines

Quoted 7 lines

Quoted 7 lines

Quoted 7 lines

This part of the series includes my fixes for the issues reported by Ferruh and Igor on top of part 1 of the patch series:

A. KNI sync lock is being locked while rtnl is held. If two threads are calling kni_net_process_request() , then the first one will take the sync lock, release rtnl lock

then sleep.

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

The second thread will try to lock sync lock while holding rtnl. The first thread will wake, and try to lock rtnl, resulting in a

deadlock.

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

The remedy is to release rtnl before locking the KNI sync lock. Since in between nothing is accessing Linux network-wise, no rtnl locking is needed.

Quoted 13 lines

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

B. There is a race condition in __dev_close_many() processing the close_list while the application terminates.
It looks like if two vEth devices are terminating,
and one releases the rtnl lock, the other takes it, updating the close_list in an unstable state,

Quoted 8 lines

Quoted 8 lines

Quoted 8 lines

Quoted 8 lines

Quoted 8 lines

causing the close_list to become a circular linked list, hence list_for_each_entry() will endlessly loop inside __dev_close_many() .
Since the description for the original patch indicate the original motivation was bringing the device up,
I have changed kni_net_process_request() to hold the rtnl mutex in case of bringing the device down since this is the path called from __dev_close_many() , causing the corruption of the

close_list.

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Signed-off-by Elad Nachman <[email protected]>

---
v3:
* Include original patch and new patch as a series of patch,

added a

Quoted 9 lines

Quoted 9 lines

Quoted 9 lines

Quoted 9 lines

Quoted 9 lines

Review

Quoted 9 lines

comment to the new patch
v2:
* rebuild the patch as increment from patch 64106
* fix comment and blank lines

---
 kernel/linux/kni/kni_net.c | 29 +++++++++++++++++++++--------
 1 file changed, 21 insertions(+), 8 deletions(-)

(quoted diff) +0 -0

b/kernel/linux/kni/kni_net.c

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

kernel/linux/kni/kni_net.c kni_net_process_request(struct net_device

Quoted 4 lines

index f0b6e9a8d..017e44812 100644

kernel/linux/kni/kni_net.c +0 -0

@@ -110,9 +110,26 @@ kni_net_process_request(struct net_device

*dev, struct rte_kni_request *req)

Quoted 8 lines

Quoted 8 lines

Quoted 8 lines

Quoted 8 lines

Quoted 8 lines

kernel/linux/kni/kni_net.c kni_net_process_request(struct net_device · line 117

Quoted 8 lines

kernel/linux/kni/kni_net.c +5 -0

        void *resp_va;
        uint32_t num;
        int ret_val;
+       int req_is_dev_stop = 0;
+
+       /* For configuring the interface to down,
+        * rtnl must be held all the way to prevent race condition
+        * inside __dev_close_many() between two netdev instances

of KNI

Quoted 17 lines

Quoted 17 lines

Quoted 17 lines

Quoted 17 lines

Quoted 17 lines

Quoted 17 lines

kernel/linux/kni/kni_net.c +12 -0

+        */
+       if (req->req_id == RTE_KNI_REQ_CFG_NETWORK_IF &&
+                       req->if_up == 0)
+               req_is_dev_stop = 1;
 
        ASSERT_RTNL();
 
+       /* Since we need to wait and RTNL mutex is held
+        * drop the mutex and hold reference to keep device
+        */
+       if (!req_is_dev_stop) {
+               dev_hold(dev);
+               rtnl_unlock();
+       }
+
        mutex_lock(&kni->sync_lock);
 

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

kernel/linux/kni/kni_net.c kni_net_process_request(struct net_device

Quoted 2 lines

kernel/linux/kni/kni_net.c +0 -0

        /* Construct data */

@@ -124,16 +141,8 @@ kni_net_process_request(struct net_device

*dev, struct rte_kni_request *req)

Quoted 14 lines

Quoted 14 lines

Quoted 14 lines

Quoted 14 lines

Quoted 14 lines

Quoted 14 lines

kernel/linux/kni/kni_net.c +0 -8

                goto fail;
        }
 
-       /* Since we need to wait and RTNL mutex is held
-        * drop the mutex and hold refernce to keep device
-        */
-       dev_hold(dev);
-       rtnl_unlock();
-
        ret_val = wait_event_interruptible_timeout(kni->wq,
                        kni_fifo_count(kni->resp_q), 3 * HZ);
-       rtnl_lock();
-       dev_put(dev);
 

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

kernel/linux/kni/kni_net.c kni_net_process_request(struct net_device

Quoted 3 lines

kernel/linux/kni/kni_net.c +0 -0

        if (signal_pending(current) || ret_val <= 0) {
                ret = -ETIME;

@@ -152,6 +161,10 @@ kni_net_process_request(struct net_device

*dev, struct rte_kni_request *req)

Quoted 13 lines

Quoted 13 lines

Quoted 13 lines

Quoted 13 lines

Quoted 13 lines

Quoted 13 lines

kernel/linux/kni/kni_net.c +4 -1

 fail:
        mutex_unlock(&kni->sync_lock);
+       if (!req_is_dev_stop) {
+               rtnl_lock();
+               dev_put(dev);
+       }
        return ret;
 }
 
--

2.17.1

Sorry, don't have Mellanox NIC currently. Will have one in 8-12 weeks. Will be happy to test it remotely if anyone can provide remote HW or VM (Azure, for example).

Elad.

Quoted 2 lines · Igor Ryzhov

Stephen's idea was to fix the deadlock when working with the bifurcated

driver.

Quoted 1 line

Your rework breaks this because you still send link down requests under

rtnl_lock.

Quoted 27 lines

Did you test your patch with Mellanox devices?

Quoted 24 lines · Elad Nachman

The deadlock scenarios are explained below:

It is described in Stephen Hemminger's original patch:

"

This fixes a deadlock when using KNI with bifurcated drivers. Bringing kni device up always times out when using Mellanox devices.

The kernel KNI driver sends message to userspace to complete the request. For the case of bifurcated driver, this may involve an additional request to kernel to change state. This request would deadlock because KNI was holding the RTNL mutex.

"

And also in my patch:

"
KNI sync lock is being locked while rtnl is held.
If two threads are calling kni_net_process_request() , then the first one will take the sync lock, release rtnl lock then

sleep.

Quoted 2 lines

Quoted 2 lines

The second thread will try to lock sync lock while holding rtnl. The first thread will wake, and try to lock rtnl, resulting in a

deadlock.

Quoted 11 lines

Quoted 11 lines

The remedy is to release rtnl before locking the KNI sync lock. Since in between nothing is accessing Linux network-wise, no rtnl locking is needed.
"

FYI,

Elad.

Quoted 1 line · Igor Ryzhov

Quoted 8 lines

Quoted 8 lines

Quoted 8 lines

Both link up and link down also work for me without this patch. So what's the point in merging it?

Just to clarify - I am not against the idea of this patch. Talking to userspace under rtnl_lock is a bad idea. I just think that any patch should fix some specified problem.

If this patch is trying to solve the overall "userspace request under

rtnl_lock" problem,

Quoted 1 line

Quoted 1 line

Quoted 1 line

then it doesn't solve it correctly, because we still send link down

requests under the lock.

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

If this patch is trying to solve some other issue, for example, some

"KNI deadlocks"

Quoted 1 line

Quoted 1 line

Quoted 1 line

you're talking about, then you should explain what these deadlocks

are, how to reproduce

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

them and why this patch solves the issue.

On Wed, Feb 24, 2021 at 5:07 PM Elad Nachman <[email protected]>

Quoted 2 lines · wrote:

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

I tested both link up and link down many times without any problems

on

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

100 restarts of the application.

Having KNI deadlock frequently for real life applications is far

worst, IMHO.

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

FYI

Elad.

On Wed, Feb 24, 2021 at 4:04 PM Igor Ryzhov <[email protected]>

Quoted 5 lines · wrote:

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Elad,

I understand your point.
But the fact that this fix works for you doesn't mean that it will

work for all DPDK users.

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

For example, I provided two simple commands: "ip link set up" and

"ip link set down".

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Your fix works for only one of them. For me, this is not a proper

fix.

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

It may work for you because you don't disable interfaces, but it

will fail for users who do.

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

On Wed, Feb 24, 2021 at 4:33 PM Elad Nachman <[email protected]>

Quoted 3 lines · wrote:

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

Currently KNI has a lot of issues with deadlocks locking the code, after this commit, they are gone, and the code runs properly

without

Quoted 7 lines

Quoted 7 lines

Quoted 7 lines

Quoted 7 lines

Quoted 7 lines

Quoted 7 lines

crashing.
That was tested with over 100 restarts of the application, which previously required a hard reset of the board.

I think this benefit overweights the complication of the code.

The function is called with rtnl locked because this is how the

Linux

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

kernel is designed to work - it is not designed to work with

deferral

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

to user-space mid-function.

To fix all such requests you need to reach an agreement with Linux netdev, which is unlikely.

Calling user-space can be done asynchronously, as Ferruh asked,

but

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

then you will always have to return success, even on failure, as

Linux

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

kernel does not have a mechanism to asynchronously report on

failure

Quoted 11 lines

Quoted 11 lines

Quoted 11 lines

Quoted 11 lines

Quoted 11 lines

Quoted 11 lines

for such system calls.

IMHO - weighting the non-reporting of failure versus how the code looks (as it functions perfectly OK), I decided to go with functionality.

FYI,

Elad.

On Wed, Feb 24, 2021 at 2:50 PM Igor Ryzhov <[email protected]>

Quoted 5 lines · wrote:

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

This looks more like a hack than an actual fix to me.

After this commit:
"ip link set up" is sent to the userspace with unlocked

rtnl_lock

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

"ip link set down" is sent to the userspace with locked

rtnl_lock

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

How is this really fixing anything? IMHO it only complicates

the code.

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

If talking with userspace under rtnl_lock is a problem, then we

should fix all such requests, not only part of them.

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

If it is not a problem, then I don't see any point in merging

this.

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

On Tue, Feb 23, 2021 at 4:45 PM Elad Nachman <[email protected]>

Quoted 2 lines · wrote:

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

This part of the series includes my fixes for the issues

reported

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

by Ferruh and Igor on top of part 1 of the patch series:

A. KNI sync lock is being locked while rtnl is held. If two threads are calling kni_net_process_request() , then the first one will take the sync lock, release rtnl lock

then sleep.

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

The second thread will try to lock sync lock while holding

rtnl.

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

The first thread will wake, and try to lock rtnl, resulting in

a deadlock.

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

The remedy is to release rtnl before locking the KNI sync lock. Since in between nothing is accessing Linux network-wise, no rtnl locking is needed.

B. There is a race condition in __dev_close_many() processing

the

Quoted 11 lines

Quoted 11 lines

Quoted 11 lines

Quoted 11 lines

Quoted 11 lines

Quoted 11 lines

Quoted 11 lines

Quoted 11 lines

close_list while the application terminates.
It looks like if two vEth devices are terminating, and one releases the rtnl lock, the other takes it, updating the close_list in an unstable state,
causing the close_list to become a circular linked list, hence list_for_each_entry() will endlessly loop inside __dev_close_many() .
Since the description for the original patch indicate the original motivation was bringing the device up,
I have changed kni_net_process_request() to hold the rtnl mutex in case of bringing the device down since this is the path

called

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

from __dev_close_many() , causing the corruption of the

close_list.

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Signed-off-by Elad Nachman <[email protected]>

---
v3:
* Include original patch and new patch as a series of patch,

added a

Quoted 9 lines

Quoted 9 lines

Quoted 9 lines

Quoted 9 lines

Quoted 9 lines

Quoted 9 lines

Quoted 9 lines

Review

Quoted 9 lines

comment to the new patch
v2:
* rebuild the patch as increment from patch 64106
* fix comment and blank lines

---
 kernel/linux/kni/kni_net.c | 29 +++++++++++++++++++++--------
 1 file changed, 21 insertions(+), 8 deletions(-)

(quoted diff) +0 -0

b/kernel/linux/kni/kni_net.c

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

kernel/linux/kni/kni_net.c kni_net_process_request(struct net_device

Quoted 4 lines

index f0b6e9a8d..017e44812 100644

kernel/linux/kni/kni_net.c +0 -0

@@ -110,9 +110,26 @@ kni_net_process_request(struct net_device

*dev, struct rte_kni_request *req)

Quoted 7 lines

Quoted 7 lines

Quoted 7 lines

Quoted 7 lines

Quoted 7 lines

Quoted 7 lines

Quoted 7 lines

kernel/linux/kni/kni_net.c kni_net_process_request(struct net_device · line 116

Quoted 7 lines

kernel/linux/kni/kni_net.c +4 -0

        void *resp_va;
        uint32_t num;
        int ret_val;
+       int req_is_dev_stop = 0;
+
+       /* For configuring the interface to down,
+        * rtnl must be held all the way to prevent race

condition

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

kernel/linux/kni/kni_net.c kni_net_process_request(struct net_device · line 117

Quoted 1 line

kernel/linux/kni/kni_net.c +1 -0

+        * inside __dev_close_many() between two netdev

instances of KNI

Quoted 17 lines

Quoted 17 lines

Quoted 17 lines

Quoted 17 lines

Quoted 17 lines

Quoted 17 lines

Quoted 17 lines

Quoted 17 lines

kernel/linux/kni/kni_net.c +12 -0

+        */
+       if (req->req_id == RTE_KNI_REQ_CFG_NETWORK_IF &&
+                       req->if_up == 0)
+               req_is_dev_stop = 1;
 
        ASSERT_RTNL();
 
+       /* Since we need to wait and RTNL mutex is held
+        * drop the mutex and hold reference to keep device
+        */
+       if (!req_is_dev_stop) {
+               dev_hold(dev);
+               rtnl_unlock();
+       }
+
        mutex_lock(&kni->sync_lock);
 

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

kernel/linux/kni/kni_net.c kni_net_process_request(struct net_device

Quoted 2 lines

kernel/linux/kni/kni_net.c +0 -0

        /* Construct data */

@@ -124,16 +141,8 @@ kni_net_process_request(struct net_device

*dev, struct rte_kni_request *req)

Quoted 14 lines

Quoted 14 lines

Quoted 14 lines

Quoted 14 lines

Quoted 14 lines

Quoted 14 lines

Quoted 14 lines

Quoted 14 lines

kernel/linux/kni/kni_net.c +0 -8

                goto fail;
        }
 
-       /* Since we need to wait and RTNL mutex is held
-        * drop the mutex and hold refernce to keep device
-        */
-       dev_hold(dev);
-       rtnl_unlock();
-
        ret_val = wait_event_interruptible_timeout(kni->wq,
                        kni_fifo_count(kni->resp_q), 3 * HZ);
-       rtnl_lock();
-       dev_put(dev);
 

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

kernel/linux/kni/kni_net.c kni_net_process_request(struct net_device

Quoted 3 lines

kernel/linux/kni/kni_net.c +0 -0

        if (signal_pending(current) || ret_val <= 0) {
                ret = -ETIME;

@@ -152,6 +161,10 @@ kni_net_process_request(struct net_device

*dev, struct rte_kni_request *req)

Quoted 13 lines

Quoted 13 lines

Quoted 13 lines

Quoted 13 lines

Quoted 13 lines

Quoted 13 lines

Quoted 13 lines

Quoted 13 lines

kernel/linux/kni/kni_net.c +4 -1

 fail:
        mutex_unlock(&kni->sync_lock);
+       if (!req_is_dev_stop) {
+               rtnl_lock();
+               dev_put(dev);
+       }
        return ret;
 }
 
--

2.17.1

This looks more like a hack than an actual fix to me.

After this commit:
"ip link set up" is sent to the userspace with unlocked rtnl_lock "ip link set down" is sent to the userspace with locked rtnl_lock