Skip to contentSkip to conversation

RDMA/rxe: Fix rnr retry behavior

PATCH In review Bob Pearson 3 participants 12 messages May 12, 2022 – May 14, 2022

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 RDMA/rxe: Fix rnr retry behavior

Currently the completer tasklet when it sets the retransmit timer or the nak timer sets the same flag (qp->req.need_retry) so that if either timer fires it will attempt to perform a retry flow on the send queue. This has the effect of responding to an RNR NAK at the first retransmit timer event which does not allow for the requested rnr timeout.

This patch adds a new flag (qp->req.need_rnr_timeout) which, if set, prevents a retry flow until the rnr nak timer fires.

This patch fixes rnr retry errors which can be observed by running the pyverbs test suite 50-100X. With this patch applied they do not occur.

Fixes 8700e3e7c485 ("Soft RoCE (RXE) - The software RoCE driver")
---
 drivers/infiniband/sw/rxe/rxe_comp.c  | 4 +---
 drivers/infiniband/sw/rxe/rxe_qp.c    | 1 +
 drivers/infiniband/sw/rxe/rxe_req.c   | 6 ++++--
 drivers/infiniband/sw/rxe/rxe_verbs.h | 1 +
 4 files changed, 7 insertions(+), 5 deletions(-)
drivers/infiniband/sw/rxe/rxe_comp.c +1 -3
index 138b3e7d3a5f..bc668cb211b1 100644
@@ -733,9 +733,7 @@ int rxe_completer(void *arg)
 				if (qp->comp.rnr_retry != 7)
 					qp->comp.rnr_retry--;
 
-				qp->req.need_retry = 1;
-				pr_debug("qp#%d set rnr nak timer\n",
-					 qp_num(qp));
+				qp->req.need_rnr_timeout = 1;
 				mod_timer(&qp->rnr_nak_timer,
 					  jiffies + rnrnak_jiffies(aeth_syn(pkt)
 						& ~AETH_TYPE_MASK));
drivers/infiniband/sw/rxe/rxe_qp.c +1 -0
index 62acf890af6c..1c962468714e 100644
@@ -513,6 +513,7 @@ static void rxe_qp_reset(struct rxe_qp *qp)
 	atomic_set(&qp->ssn, 0);
 	qp->req.opcode = -1;
 	qp->req.need_retry = 0;
+	qp->req.need_rnr_timeout = 0;
 	qp->req.noack_pkts = 0;
 	qp->resp.msn = 0;
 	qp->resp.opcode = -1;
drivers/infiniband/sw/rxe/rxe_req.c +4 -2
index ae5fbc79dd5c..770ae4279f73 100644
@@ -103,7 +103,8 @@ void rnr_nak_timer(struct timer_list *t)
 {
 	struct rxe_qp *qp = from_timer(qp, t, rnr_nak_timer);
 
-	pr_debug("qp#%d rnr nak timer fired\n", qp_num(qp));
+	qp->req.need_retry = 1;
+	qp->req.need_rnr_timeout = 0;
 	rxe_run_task(&qp->req.task, 1);
 }
 
@@ -624,10 +625,11 @@ int rxe_requester(void *arg)
 		qp->req.need_rd_atomic = 0;
 		qp->req.wait_psn = 0;
 		qp->req.need_retry = 0;
+		qp->req.need_rnr_timeout = 0;
 		goto exit;
 	}
 
-	if (unlikely(qp->req.need_retry)) {
+	if (unlikely(qp->req.need_retry && !qp->req.need_rnr_timeout)) {
 		req_retry(qp);
 		qp->req.need_retry = 0;
 	}
drivers/infiniband/sw/rxe/rxe_verbs.h +1 -0
index e7eff1ca75e9..ab3186478c3f 100644
@@ -123,6 +123,7 @@ struct rxe_req_info {
 	int			need_rd_atomic;
 	int			wait_psn;
 	int			need_retry;
+	int			need_rnr_timeout;
 	int			noack_pkts;
 	struct rxe_task		task;
 };

base-commit: c5eb0a61238dd6faf37f58c9ce61c9980aaffd7a

Signature · 1 line
2.34.1
#1 lore raw
Quoted 11 lines · Bob Pearson

Currently the completer tasklet when it sets the retransmit timer or the nak timer sets the same flag (qp->req.need_retry) so that if either timer fires it will attempt to perform a retry flow on the send queue. This has the effect of responding to an RNR NAK at the first retransmit timer event which does not allow for the requested rnr timeout.

This patch adds a new flag (qp->req.need_rnr_timeout) which, if set, prevents a retry flow until the rnr nak timer fires.

This patch fixes rnr retry errors which can be observed by running the pyverbs test suite 50-100X. With this patch applied they do not occur.

Do you mean that running run_tests.py for 50-100times can reproduce this bug? I want to reproduce this problem.

Thanks a lot.
Zhu Yanjun

Quoted 79 lines
Fixes 8700e3e7c485 ("Soft RoCE (RXE) - The software RoCE driver")
---
  drivers/infiniband/sw/rxe/rxe_comp.c  | 4 +---
  drivers/infiniband/sw/rxe/rxe_qp.c    | 1 +
  drivers/infiniband/sw/rxe/rxe_req.c   | 6 ++++--
  drivers/infiniband/sw/rxe/rxe_verbs.h | 1 +
  4 files changed, 7 insertions(+), 5 deletions(-)
drivers/infiniband/sw/rxe/rxe_comp.c +1 -3
index 138b3e7d3a5f..bc668cb211b1 100644
@@ -733,9 +733,7 @@ int rxe_completer(void *arg)
  				if (qp->comp.rnr_retry != 7)
  					qp->comp.rnr_retry--;
 
-				qp->req.need_retry = 1;
-				pr_debug("qp#%d set rnr nak timer\n",
-					 qp_num(qp));
+				qp->req.need_rnr_timeout = 1;
  				mod_timer(&qp->rnr_nak_timer,
  					  jiffies + rnrnak_jiffies(aeth_syn(pkt)
  						& ~AETH_TYPE_MASK));
drivers/infiniband/sw/rxe/rxe_qp.c +1 -0
index 62acf890af6c..1c962468714e 100644
@@ -513,6 +513,7 @@ static void rxe_qp_reset(struct rxe_qp *qp)
  	atomic_set(&qp->ssn, 0);
  	qp->req.opcode = -1;
  	qp->req.need_retry = 0;
+	qp->req.need_rnr_timeout = 0;
  	qp->req.noack_pkts = 0;
  	qp->resp.msn = 0;
  	qp->resp.opcode = -1;
drivers/infiniband/sw/rxe/rxe_req.c +4 -2
index ae5fbc79dd5c..770ae4279f73 100644
@@ -103,7 +103,8 @@ void rnr_nak_timer(struct timer_list *t)
  {
  	struct rxe_qp *qp = from_timer(qp, t, rnr_nak_timer);
 
-	pr_debug("qp#%d rnr nak timer fired\n", qp_num(qp));
+	qp->req.need_retry = 1;
+	qp->req.need_rnr_timeout = 0;
  	rxe_run_task(&qp->req.task, 1);
  }
 
@@ -624,10 +625,11 @@ int rxe_requester(void *arg)
  		qp->req.need_rd_atomic = 0;
  		qp->req.wait_psn = 0;
  		qp->req.need_retry = 0;
+		qp->req.need_rnr_timeout = 0;
  		goto exit;
  	}
 
-	if (unlikely(qp->req.need_retry)) {
+	if (unlikely(qp->req.need_retry && !qp->req.need_rnr_timeout)) {
  		req_retry(qp);
  		qp->req.need_retry = 0;
  	}
drivers/infiniband/sw/rxe/rxe_verbs.h +1 -0
index e7eff1ca75e9..ab3186478c3f 100644
@@ -123,6 +123,7 @@ struct rxe_req_info {
  	int			need_rd_atomic;
  	int			wait_psn;
  	int			need_retry;
+	int			need_rnr_timeout;
  	int			noack_pkts;
  	struct rxe_task		task;
  };

base-commit: c5eb0a61238dd6faf37f58c9ce61c9980aaffd7a

#2 lore raw
Quoted 7 lines · Yanjun Zhu
Quoted 6 lines · Bob Pearson

Currently the completer tasklet when it sets the retransmit timer or the nak timer sets the same flag (qp->req.need_retry) so that if either timer fires it will attempt to perform a retry flow on the send queue. This has the effect of responding to an RNR NAK at the first retransmit timer event which does not allow for the requested rnr timeout.

Quoted 7 lines
Quoted 5 lines

This patch adds a new flag (qp->req.need_rnr_timeout) which, if set, prevents a retry flow until the rnr nak timer fires.

This patch fixes rnr retry errors which can be observed by running the pyverbs test suite 50-100X. With this patch applied they do not occur.

Do you mean that running run_tests.py for 50-100times can reproduce this bug? I want to reproduce this problem.

Running run_tests.py for 50-100times, I can not reproduce this problem

Zhu Yanjun

Quoted 81 lines

Thanks a lot.
Zhu Yanjun

Quoted 76 lines
Fixes 8700e3e7c485 ("Soft RoCE (RXE) - The software RoCE driver")

---
  drivers/infiniband/sw/rxe/rxe_comp.c  | 4 +--- drivers/infiniband/sw/rxe/rxe_qp.c    | 1 + drivers/infiniband/sw/rxe/rxe_req.c   | 6 ++++-- drivers/infiniband/sw/rxe/rxe_verbs.h | 1 +

  4 files changed, 7 insertions(+), 5 deletions(-)
drivers/infiniband/sw/rxe/rxe_comp.c +1 -3
index 138b3e7d3a5f..bc668cb211b1 100644
@@ -733,9 +733,7 @@ int rxe_completer(void *arg)
                   if (qp->comp.rnr_retry != 7)
                       qp->comp.rnr_retry--;
-                qp->req.need_retry = 1;
-                pr_debug("qp#%d set rnr nak timer\n",
-                     qp_num(qp));
+                qp->req.need_rnr_timeout = 1;
                   mod_timer(&qp->rnr_nak_timer,
                         jiffies + rnrnak_jiffies(aeth_syn(pkt)
                           & ~AETH_TYPE_MASK));
drivers/infiniband/sw/rxe/rxe_qp.c +1 -0
index 62acf890af6c..1c962468714e 100644
@@ -513,6 +513,7 @@ static void rxe_qp_reset(struct rxe_qp *qp)
       atomic_set(&qp->ssn, 0);
       qp->req.opcode = -1;
       qp->req.need_retry = 0;
+    qp->req.need_rnr_timeout = 0;
       qp->req.noack_pkts = 0;
       qp->resp.msn = 0;
       qp->resp.opcode = -1;
drivers/infiniband/sw/rxe/rxe_req.c +4 -2
index ae5fbc79dd5c..770ae4279f73 100644
@@ -103,7 +103,8 @@ void rnr_nak_timer(struct timer_list *t)
   {
       struct rxe_qp *qp = from_timer(qp, t, rnr_nak_timer);
-    pr_debug("qp#%d rnr nak timer fired\n", qp_num(qp));
+    qp->req.need_retry = 1;
+    qp->req.need_rnr_timeout = 0;
       rxe_run_task(&qp->req.task, 1);
   }
@@ -624,10 +625,11 @@ int rxe_requester(void *arg)
           qp->req.need_rd_atomic = 0;
           qp->req.wait_psn = 0;
           qp->req.need_retry = 0;
+        qp->req.need_rnr_timeout = 0;
           goto exit;
       }
-    if (unlikely(qp->req.need_retry)) {
+    if (unlikely(qp->req.need_retry && !qp->req.need_rnr_timeout)) {
           req_retry(qp);
           qp->req.need_retry = 0;
       }
drivers/infiniband/sw/rxe/rxe_verbs.h +1 -0
index e7eff1ca75e9..ab3186478c3f 100644
@@ -123,6 +123,7 @@ struct rxe_req_info {
       int            need_rd_atomic;
       int            wait_psn;
       int            need_retry;
+    int            need_rnr_timeout;
       int            noack_pkts;
       struct rxe_task        task;
   };

base-commit: c5eb0a61238dd6faf37f58c9ce61c9980aaffd7a

#3 lore raw
Quoted 13 lines · Yanjun Zhu
Quoted 10 lines · Yanjun Zhu
Quoted 9 lines · Bob Pearson

Currently the completer tasklet when it sets the retransmit timer or the nak timer sets the same flag (qp->req.need_retry) so that if either timer fires it will attempt to perform a retry flow on the send queue. This has the effect of responding to an RNR NAK at the first retransmit timer event which does not allow for the requested rnr timeout.

This patch adds a new flag (qp->req.need_rnr_timeout) which, if set, prevents a retry flow until the rnr nak timer fires.

Quoted 6 lines
Quoted 4 lines
Quoted 2 lines

This patch fixes rnr retry errors which can be observed by running the pyverbs test suite 50-100X. With this patch applied they do not occur.

Do you mean that running run_tests.py for 50-100times can reproduce this bug? I want to reproduce this problem.

Running run_tests.py for 50-100times, I can not reproduce this problem

I see them consistently. It is just luck of timing. If you add print statements to show when the retry timer fires and when the rnr timer fires you should see that all the retry sequences are a result of the retry timer and not the rnr timer. This means that you are not allowing the requested rnr delay for the responder. In my traces most of the time the retry timer fires and the test ends and the qp is torn down deleting the rnr timer before it ever times out. Occasionally the retry timer fires too early and the test fails. This is still a bug and it should be fixed.

Bob

Quoted 83 lines

Zhu Yanjun

Quoted 80 lines

Thanks a lot.
Zhu Yanjun

Quoted 75 lines
Fixes 8700e3e7c485 ("Soft RoCE (RXE) - The software RoCE driver")

---
  drivers/infiniband/sw/rxe/rxe_comp.c  | 4 +--- drivers/infiniband/sw/rxe/rxe_qp.c    | 1 + drivers/infiniband/sw/rxe/rxe_req.c   | 6 ++++-- drivers/infiniband/sw/rxe/rxe_verbs.h | 1 +

  4 files changed, 7 insertions(+), 5 deletions(-)
drivers/infiniband/sw/rxe/rxe_comp.c +1 -3
index 138b3e7d3a5f..bc668cb211b1 100644
@@ -733,9 +733,7 @@ int rxe_completer(void *arg)
                   if (qp->comp.rnr_retry != 7)
                       qp->comp.rnr_retry--;
-                qp->req.need_retry = 1;
-                pr_debug("qp#%d set rnr nak timer\n",
-                     qp_num(qp));
+                qp->req.need_rnr_timeout = 1;
                   mod_timer(&qp->rnr_nak_timer,
                         jiffies + rnrnak_jiffies(aeth_syn(pkt)
                           & ~AETH_TYPE_MASK));
drivers/infiniband/sw/rxe/rxe_qp.c +1 -0
index 62acf890af6c..1c962468714e 100644
@@ -513,6 +513,7 @@ static void rxe_qp_reset(struct rxe_qp *qp)
       atomic_set(&qp->ssn, 0);
       qp->req.opcode = -1;
       qp->req.need_retry = 0;
+    qp->req.need_rnr_timeout = 0;
       qp->req.noack_pkts = 0;
       qp->resp.msn = 0;
       qp->resp.opcode = -1;
drivers/infiniband/sw/rxe/rxe_req.c +4 -2
index ae5fbc79dd5c..770ae4279f73 100644
@@ -103,7 +103,8 @@ void rnr_nak_timer(struct timer_list *t)
   {
       struct rxe_qp *qp = from_timer(qp, t, rnr_nak_timer);
-    pr_debug("qp#%d rnr nak timer fired\n", qp_num(qp));
+    qp->req.need_retry = 1;
+    qp->req.need_rnr_timeout = 0;
       rxe_run_task(&qp->req.task, 1);
   }
@@ -624,10 +625,11 @@ int rxe_requester(void *arg)
           qp->req.need_rd_atomic = 0;
           qp->req.wait_psn = 0;
           qp->req.need_retry = 0;
+        qp->req.need_rnr_timeout = 0;
           goto exit;
       }
-    if (unlikely(qp->req.need_retry)) {
+    if (unlikely(qp->req.need_retry && !qp->req.need_rnr_timeout)) {
           req_retry(qp);
           qp->req.need_retry = 0;
       }
drivers/infiniband/sw/rxe/rxe_verbs.h +1 -0
index e7eff1ca75e9..ab3186478c3f 100644
@@ -123,6 +123,7 @@ struct rxe_req_info {
       int            need_rd_atomic;
       int            wait_psn;
       int            need_retry;
+    int            need_rnr_timeout;
       int            noack_pkts;
       struct rxe_task        task;
   };

base-commit: c5eb0a61238dd6faf37f58c9ce61c9980aaffd7a

#4 lore raw
In reply to Yanjun Zhu
Quoted 13 lines · Yanjun Zhu
Quoted 10 lines · Yanjun Zhu
Quoted 9 lines · Bob Pearson

Currently the completer tasklet when it sets the retransmit timer or the nak timer sets the same flag (qp->req.need_retry) so that if either timer fires it will attempt to perform a retry flow on the send queue. This has the effect of responding to an RNR NAK at the first retransmit timer event which does not allow for the requested rnr timeout.

This patch adds a new flag (qp->req.need_rnr_timeout) which, if set, prevents a retry flow until the rnr nak timer fires.

Quoted 6 lines
Quoted 4 lines
Quoted 2 lines

This patch fixes rnr retry errors which can be observed by running the pyverbs test suite 50-100X. With this patch applied they do not occur.

Do you mean that running run_tests.py for 50-100times can reproduce this bug? I want to reproduce this problem.

Running run_tests.py for 50-100times, I can not reproduce this problem

I went back and looked. The occasionally failing test case is:

test_rdmacm_async_traffic_external_qp

This test is normally skipped for rxe because of a hack in <path to rdma-core>/tests/base.py. I sent you a patch that enables these tests a few days ago.

It randomly causes an rnr retry about 1/3 of the time. Most of these are repaired by the retry timer going off. A few ~1-2% are not.

Bob

Quoted 83 lines

Zhu Yanjun

Quoted 80 lines

Thanks a lot.
Zhu Yanjun

Quoted 75 lines
Fixes 8700e3e7c485 ("Soft RoCE (RXE) - The software RoCE driver")

---
  drivers/infiniband/sw/rxe/rxe_comp.c  | 4 +--- drivers/infiniband/sw/rxe/rxe_qp.c    | 1 + drivers/infiniband/sw/rxe/rxe_req.c   | 6 ++++-- drivers/infiniband/sw/rxe/rxe_verbs.h | 1 +

  4 files changed, 7 insertions(+), 5 deletions(-)
drivers/infiniband/sw/rxe/rxe_comp.c +1 -3
index 138b3e7d3a5f..bc668cb211b1 100644
@@ -733,9 +733,7 @@ int rxe_completer(void *arg)
                   if (qp->comp.rnr_retry != 7)
                       qp->comp.rnr_retry--;
-                qp->req.need_retry = 1;
-                pr_debug("qp#%d set rnr nak timer\n",
-                     qp_num(qp));
+                qp->req.need_rnr_timeout = 1;
                   mod_timer(&qp->rnr_nak_timer,
                         jiffies + rnrnak_jiffies(aeth_syn(pkt)
                           & ~AETH_TYPE_MASK));
drivers/infiniband/sw/rxe/rxe_qp.c +1 -0
index 62acf890af6c..1c962468714e 100644
@@ -513,6 +513,7 @@ static void rxe_qp_reset(struct rxe_qp *qp)
       atomic_set(&qp->ssn, 0);
       qp->req.opcode = -1;
       qp->req.need_retry = 0;
+    qp->req.need_rnr_timeout = 0;
       qp->req.noack_pkts = 0;
       qp->resp.msn = 0;
       qp->resp.opcode = -1;
drivers/infiniband/sw/rxe/rxe_req.c +4 -2
index ae5fbc79dd5c..770ae4279f73 100644
@@ -103,7 +103,8 @@ void rnr_nak_timer(struct timer_list *t)
   {
       struct rxe_qp *qp = from_timer(qp, t, rnr_nak_timer);
-    pr_debug("qp#%d rnr nak timer fired\n", qp_num(qp));
+    qp->req.need_retry = 1;
+    qp->req.need_rnr_timeout = 0;
       rxe_run_task(&qp->req.task, 1);
   }
@@ -624,10 +625,11 @@ int rxe_requester(void *arg)
           qp->req.need_rd_atomic = 0;
           qp->req.wait_psn = 0;
           qp->req.need_retry = 0;
+        qp->req.need_rnr_timeout = 0;
           goto exit;
       }
-    if (unlikely(qp->req.need_retry)) {
+    if (unlikely(qp->req.need_retry && !qp->req.need_rnr_timeout)) {
           req_retry(qp);
           qp->req.need_retry = 0;
       }
drivers/infiniband/sw/rxe/rxe_verbs.h +1 -0
index e7eff1ca75e9..ab3186478c3f 100644
@@ -123,6 +123,7 @@ struct rxe_req_info {
       int            need_rd_atomic;
       int            wait_psn;
       int            need_retry;
+    int            need_rnr_timeout;
       int            noack_pkts;
       struct rxe_task        task;
   };

base-commit: c5eb0a61238dd6faf37f58c9ce61c9980aaffd7a

#5 lore raw
Quoted 15 lines · Bob Pearson
Quoted 14 lines · Yanjun Zhu
Quoted 12 lines · Yanjun Zhu
Quoted 11 lines · Bob Pearson

Currently the completer tasklet when it sets the retransmit timer or the nak timer sets the same flag (qp->req.need_retry) so that if either timer fires it will attempt to perform a retry flow on the send queue. This has the effect of responding to an RNR NAK at the first retransmit timer event which does not allow for the requested rnr timeout.

This patch adds a new flag (qp->req.need_rnr_timeout) which, if set, prevents a retry flow until the rnr nak timer fires.

This patch fixes rnr retry errors which can be observed by running the pyverbs test suite 50-100X. With this patch applied they do not occur.

Quoted 8 lines
Quoted 2 lines
Quoted 1 line

Do you mean that running run_tests.py for 50-100times can reproduce this bug? I want to reproduce this problem.

Running run_tests.py for 50-100times, I can not reproduce this problem

I went back and looked. The occasionally failing test case is:

test_rdmacm_async_traffic_external_qp

This test is normally skipped for rxe because of a hack in <path to rdma-core>/tests/base.py. I sent you a patch that enables these tests a few days ago.

What is the patch? Can you send me it again?

And what is the symptoms when this problem occurs?

I want to check if I can reproduce this problem.

Thanks and Regards,

Zhu Yanjun

Quoted 84 lines

It randomly causes an rnr retry about 1/3 of the time. Most of these are repaired by the retry timer going off. A few ~1-2% are not.

Bob

Quoted 79 lines

Zhu Yanjun

Quoted 77 lines

Thanks a lot.
Zhu Yanjun

Quoted 74 lines
Fixes 8700e3e7c485 ("Soft RoCE (RXE) - The software RoCE driver")
---
   drivers/infiniband/sw/rxe/rxe_comp.c  | 4 +---
   drivers/infiniband/sw/rxe/rxe_qp.c    | 1 +
   drivers/infiniband/sw/rxe/rxe_req.c   | 6 ++++--
   drivers/infiniband/sw/rxe/rxe_verbs.h | 1 +
   4 files changed, 7 insertions(+), 5 deletions(-)
drivers/infiniband/sw/rxe/rxe_comp.c +1 -3
index 138b3e7d3a5f..bc668cb211b1 100644
@@ -733,9 +733,7 @@ int rxe_completer(void *arg)
                   if (qp->comp.rnr_retry != 7)
                       qp->comp.rnr_retry--;
-                qp->req.need_retry = 1;
-                pr_debug("qp#%d set rnr nak timer\n",
-                     qp_num(qp));
+                qp->req.need_rnr_timeout = 1;
                   mod_timer(&qp->rnr_nak_timer,
                         jiffies + rnrnak_jiffies(aeth_syn(pkt)
                           & ~AETH_TYPE_MASK));
drivers/infiniband/sw/rxe/rxe_qp.c +1 -0
index 62acf890af6c..1c962468714e 100644
@@ -513,6 +513,7 @@ static void rxe_qp_reset(struct rxe_qp *qp)
       atomic_set(&qp->ssn, 0);
       qp->req.opcode = -1;
       qp->req.need_retry = 0;
+    qp->req.need_rnr_timeout = 0;
       qp->req.noack_pkts = 0;
       qp->resp.msn = 0;
       qp->resp.opcode = -1;
drivers/infiniband/sw/rxe/rxe_req.c +4 -2
index ae5fbc79dd5c..770ae4279f73 100644
@@ -103,7 +103,8 @@ void rnr_nak_timer(struct timer_list *t)
   {
       struct rxe_qp *qp = from_timer(qp, t, rnr_nak_timer);
-    pr_debug("qp#%d rnr nak timer fired\n", qp_num(qp));
+    qp->req.need_retry = 1;
+    qp->req.need_rnr_timeout = 0;
       rxe_run_task(&qp->req.task, 1);
   }
@@ -624,10 +625,11 @@ int rxe_requester(void *arg)
           qp->req.need_rd_atomic = 0;
           qp->req.wait_psn = 0;
           qp->req.need_retry = 0;
+        qp->req.need_rnr_timeout = 0;
           goto exit;
       }
-    if (unlikely(qp->req.need_retry)) {
+    if (unlikely(qp->req.need_retry && !qp->req.need_rnr_timeout)) {
           req_retry(qp);
           qp->req.need_retry = 0;
       }
drivers/infiniband/sw/rxe/rxe_verbs.h +1 -0
index e7eff1ca75e9..ab3186478c3f 100644
@@ -123,6 +123,7 @@ struct rxe_req_info {
       int            need_rd_atomic;
       int            wait_psn;
       int            need_retry;
+    int            need_rnr_timeout;
       int            noack_pkts;
       struct rxe_task        task;
   };

base-commit: c5eb0a61238dd6faf37f58c9ce61c9980aaffd7a

#6 lore raw
Quoted 19 lines · Yanjun Zhu
Quoted 17 lines · Bob Pearson
Quoted 16 lines · Yanjun Zhu
Quoted 7 lines · Yanjun Zhu
Quoted 6 lines · Bob Pearson

Currently the completer tasklet when it sets the retransmit timer or the nak timer sets the same flag (qp->req.need_retry) so that if either timer fires it will attempt to perform a retry flow on the send queue. This has the effect of responding to an RNR NAK at the first retransmit timer event which does not allow for the requested rnr timeout.

Quoted 6 lines
Quoted 5 lines

This patch adds a new flag (qp->req.need_rnr_timeout) which, if set, prevents a retry flow until the rnr nak timer fires.

This patch fixes rnr retry errors which can be observed by running the pyverbs test suite 50-100X. With this patch applied they do not occur.

Do you mean that running run_tests.py for 50-100times can reproduce this bug? I want to reproduce this problem.

Running run_tests.py for 50-100times, I can not reproduce this problem

Quoted 8 lines
Quoted 6 lines

I went back and looked. The occasionally failing test case is:

    test_rdmacm_async_traffic_external_qp

This test is normally skipped for rxe because of a hack in <path to rdma-core>/tests/base.py. I sent you a patch that enables these tests a few days ago.

What is the patch? Can you send me it again?

tests/base.py +0 -0

index 93568c7f..03b5601e 100644

--- a/tests/base.py

+++ b/tests/base.py

@@ -245,10 +245,11 @@ class RDMATestCase(unittest.TestCase):

self._add_gids_per_port(ctx, dev, self.ib_port)

def _get_ip_mac(self, dev, port, idx):

- if not os.path.exists('/sys/class/infiniband/{}/device/net/'.format(dev)):

- self.args.append([dev, port, idx, None, None])

- return

- net_name = self.get_net_name(dev)

+ #if not os.path.exists('/sys/class/infiniband/{}/device/net/'.format(dev)):

+ #self.args.append([dev, port, idx, None, None])

+ #return

+ #net_name = self.get_net_name(dev)

+ net_name = "enp0s3"

try:

ip_addr, mac_addr = self.get_ip_mac_address(net_name)

except (KeyError, IndexError):

Save this to a file (e.g. net_name.patch) and type git apply net_name.patch.

Before doing this change "enp0s3" to the correct name for the Ethernet adapter you are using in your system. tests/base.py assumes that all roce devices have a file with a path

/sys/class/infiniband/rxe0/device/net/

But this is not always the case and rxe does not have that path. This breaks all the rdmacm test cases in pyverbs.

Quoted 2 lines

And what is the symptoms when this problem occurs?

When I run

# sudo ./build/bin/run_tests -k test_rdmacm_async_traffic_external_qp several times

I will eventually see an rnr timeout error. You may not have this experience but you can add print statements to the retransmit_timer and rnr_timer routines in rxe_comp.c and rxe_req.c respectively. You should see that without the below patch you will never see the rnr_timer fire and you will see lots of retransmit_timer fires.

The current code responds to the retry timer by initiating a send queue retry and the test will usually pass if it has sent an RNR NAK and tear down the rnr timer before it have a chance to fire.

Bob

Quoted 7 lines

I want to check if I can reproduce this problem.

Thanks and Regards,

Zhu Yanjun

Thanks

Bob

Quoted 85 lines
Quoted 85 lines

It randomly causes an rnr retry about 1/3 of the time. Most of these are repaired by the retry timer going off. A few ~1-2% are not.

Bob

Quoted 79 lines

Zhu Yanjun

Quoted 77 lines

Thanks a lot.
Zhu Yanjun

Quoted 74 lines
Fixes 8700e3e7c485 ("Soft RoCE (RXE) - The software RoCE driver")

---
   drivers/infiniband/sw/rxe/rxe_comp.c  | 4 +--- drivers/infiniband/sw/rxe/rxe_qp.c    | 1 + drivers/infiniband/sw/rxe/rxe_req.c   | 6 ++++-- drivers/infiniband/sw/rxe/rxe_verbs.h | 1 +

   4 files changed, 7 insertions(+), 5 deletions(-)
drivers/infiniband/sw/rxe/rxe_comp.c +1 -3
index 138b3e7d3a5f..bc668cb211b1 100644
@@ -733,9 +733,7 @@ int rxe_completer(void *arg)
                    if (qp->comp.rnr_retry != 7)
                        qp->comp.rnr_retry--;
-                qp->req.need_retry = 1;
-                pr_debug("qp#%d set rnr nak timer\n",
-                     qp_num(qp));
+                qp->req.need_rnr_timeout = 1;
                    mod_timer(&qp->rnr_nak_timer,
                          jiffies + rnrnak_jiffies(aeth_syn(pkt)
                            & ~AETH_TYPE_MASK));
drivers/infiniband/sw/rxe/rxe_qp.c +1 -0
index 62acf890af6c..1c962468714e 100644
@@ -513,6 +513,7 @@ static void rxe_qp_reset(struct rxe_qp *qp)
        atomic_set(&qp->ssn, 0);
        qp->req.opcode = -1;
        qp->req.need_retry = 0;
+    qp->req.need_rnr_timeout = 0;
        qp->req.noack_pkts = 0;
        qp->resp.msn = 0;
        qp->resp.opcode = -1;
drivers/infiniband/sw/rxe/rxe_req.c +4 -2
index ae5fbc79dd5c..770ae4279f73 100644
@@ -103,7 +103,8 @@ void rnr_nak_timer(struct timer_list *t)
    {
        struct rxe_qp *qp = from_timer(qp, t, rnr_nak_timer);
-    pr_debug("qp#%d rnr nak timer fired\n", qp_num(qp));
+    qp->req.need_retry = 1;
+    qp->req.need_rnr_timeout = 0;
        rxe_run_task(&qp->req.task, 1);
    }
@@ -624,10 +625,11 @@ int rxe_requester(void *arg)
            qp->req.need_rd_atomic = 0;
            qp->req.wait_psn = 0;
            qp->req.need_retry = 0;
+        qp->req.need_rnr_timeout = 0;
            goto exit;
        }
-    if (unlikely(qp->req.need_retry)) {
+    if (unlikely(qp->req.need_retry && !qp->req.need_rnr_timeout)) {
            req_retry(qp);
            qp->req.need_retry = 0;
        }
drivers/infiniband/sw/rxe/rxe_verbs.h +1 -0
index e7eff1ca75e9..ab3186478c3f 100644
@@ -123,6 +123,7 @@ struct rxe_req_info {
        int            need_rd_atomic;
        int            wait_psn;
        int            need_retry;
+    int            need_rnr_timeout;
        int            noack_pkts;
        struct rxe_task        task;
    };

base-commit: c5eb0a61238dd6faf37f58c9ce61c9980aaffd7a

#7 lore raw
In reply to Bob Pearson
Quoted 8 lines · Bob Pearson

Currently the completer tasklet when it sets the retransmit timer or the nak timer sets the same flag (qp->req.need_retry) so that if either timer fires it will attempt to perform a retry flow on the send queue. This has the effect of responding to an RNR NAK at the first retransmit timer event which does not allow for the requested rnr timeout.

This patch adds a new flag (qp->req.need_rnr_timeout) which, if set, prevents a retry flow until the rnr nak timer fires.

The new name is a little confusing, nobody "needs" an RNR timeout. :)
But it seems really odd (and maybe fragile) to have two flags. To me,
setting need_retry to "-1", or anything but 0 or 1, would be better.
After all, if the RNR NAK timer goes off, the code will generally retry, right? So it seems more logical to merge these.

Quoted 18 lines

This patch fixes rnr retry errors which can be observed by running the pyverbs test suite 50-100X. With this patch applied they do not occur.

Fixes 8700e3e7c485 ("Soft RoCE (RXE) - The software RoCE driver")
---
  drivers/infiniband/sw/rxe/rxe_comp.c  | 4 +---
  drivers/infiniband/sw/rxe/rxe_qp.c    | 1 +
  drivers/infiniband/sw/rxe/rxe_req.c   | 6 ++++--
  drivers/infiniband/sw/rxe/rxe_verbs.h | 1 +
  4 files changed, 7 insertions(+), 5 deletions(-)
drivers/infiniband/sw/rxe/rxe_comp.c +0 -0
index 138b3e7d3a5f..bc668cb211b1 100644

drivers/infiniband/sw/rxe/rxe_comp.c int rxe_completer(void *arg) · line 736

Quoted 7 lines
drivers/infiniband/sw/rxe/rxe_comp.c +1 -3
@@ -733,9 +733,7 @@ int rxe_completer(void *arg)
  				if (qp->comp.rnr_retry != 7)
  					qp->comp.rnr_retry--;
 
-				qp->req.need_retry = 1;
-				pr_debug("qp#%d set rnr nak timer\n",
-					 qp_num(qp));
+				qp->req.need_rnr_timeout = 1;

Suggest req.need_rnr_retry = -1 (and keep the useful pr_debug!)

Quoted 20 lines
drivers/infiniband/sw/rxe/rxe_comp.c +0 -0
  				mod_timer(&qp->rnr_nak_timer,
  					  jiffies + rnrnak_jiffies(aeth_syn(pkt)
  						& ~AETH_TYPE_MASK));
drivers/infiniband/sw/rxe/rxe_qp.c +1 -0
index 62acf890af6c..1c962468714e 100644
@@ -513,6 +513,7 @@ static void rxe_qp_reset(struct rxe_qp *qp)
  	atomic_set(&qp->ssn, 0);
  	qp->req.opcode = -1;
  	qp->req.need_retry = 0;
+	qp->req.need_rnr_timeout = 0;
  	qp->req.noack_pkts = 0;
  	qp->resp.msn = 0;
  	qp->resp.opcode = -1;
drivers/infiniband/sw/rxe/rxe_req.c +0 -0
index ae5fbc79dd5c..770ae4279f73 100644

drivers/infiniband/sw/rxe/rxe_req.c void rnr_nak_timer(struct timer_list *t) · line 107

Quoted 6 lines
drivers/infiniband/sw/rxe/rxe_req.c +2 -1
@@ -103,7 +103,8 @@ void rnr_nak_timer(struct timer_list *t)
  {
  	struct rxe_qp *qp = from_timer(qp, t, rnr_nak_timer);
 
-	pr_debug("qp#%d rnr nak timer fired\n", qp_num(qp));
+	qp->req.need_retry = 1;
+	qp->req.need_rnr_timeout = 0;

Simply setting need_retry = 1 would suffice, if suggestion accepted.

drivers/infiniband/sw/rxe/rxe_req.c int rxe_requester(void *arg) · line 632

Quoted 13 lines
drivers/infiniband/sw/rxe/rxe_req.c +2 -1
  	rxe_run_task(&qp->req.task, 1);
  }
 
@@ -624,10 +625,11 @@ int rxe_requester(void *arg)
  		qp->req.need_rd_atomic = 0;
  		qp->req.wait_psn = 0;
  		qp->req.need_retry = 0;
+		qp->req.need_rnr_timeout = 0;
  		goto exit;
  	}
 
-	if (unlikely(qp->req.need_retry)) {
+	if (unlikely(qp->req.need_retry && !qp->req.need_rnr_timeout)) {

This would become (unlikely (qp->req.rnr_retry > 0)) ...

drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_req_info { · line 126

Quoted 12 lines
drivers/infiniband/sw/rxe/rxe_req.c +0 -0
  		req_retry(qp);
  		qp->req.need_retry = 0;
  	}
drivers/infiniband/sw/rxe/rxe_verbs.h +1 -0
index e7eff1ca75e9..ab3186478c3f 100644
@@ -123,6 +123,7 @@ struct rxe_req_info {
  	int			need_rd_atomic;
  	int			wait_psn;
  	int			need_retry;
+	int			need_rnr_timeout;

Drop

Quoted 5 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +0 -0
  	int			noack_pkts;
  	struct rxe_task		task;
  };

base-commit: c5eb0a61238dd6faf37f58c9ce61c9980aaffd7a

Tom.

#8 lore raw
Quoted 8 lines · Tom Talpey
Quoted 6 lines · Bob Pearson

Currently the completer tasklet when it sets the retransmit timer or the nak timer sets the same flag (qp->req.need_retry) so that if either timer fires it will attempt to perform a retry flow on the send queue. This has the effect of responding to an RNR NAK at the first retransmit timer event which does not allow for the requested rnr timeout.

Quoted 7 lines
Quoted 2 lines

This patch adds a new flag (qp->req.need_rnr_timeout) which, if set, prevents a retry flow until the rnr nak timer fires.

The new name is a little confusing, nobody "needs" an RNR timeout. :)
But it seems really odd (and maybe fragile) to have two flags. To me,
setting need_retry to "-1", or anything but 0 or 1, would be better.
After all, if the RNR NAK timer goes off, the code will generally retry, right? So it seems more logical to merge these.

I am trying to cleanup pyverbs runs which sometimes fail with rnr retry errors. In this specific case the rnr timeout value is set a lot longer than the retry timeout. As discussed earlier the retry timer is never cleared so it continuously fires at about 40 times a second. The failing test is intentionally setting up a situation where the receiver is not ready and then it is. The retry timeout usually acts as though it were the rnr retry and 'fixes' the problem and the test passes. Since the retry timer is just randomly firing it sometimes happens too soon and the receiver isn't ready yet which leads to the failed test.

Logic says that if you receive an rnr nak the target is waiting for that specific send packet to be resent and no other so we shouldn't be responding to a spurious retry timeout.

You are correct that the retry sequence can be shared but it shouldn't start until the rnr timer has fired once an rnr nak is seen. This patch does exactly that by blocking the retry sequence until the rnr timer fires.

If you don't like 'need_rnr_timeout' perhaps you would accept 'wait_rnr_timeout' instead. Overloading need_retry is less clear IMHO. You don't need the retry until the rnr timer has fired then you do. If you really want to just use one variable we are basically implementing a state machine with 3 states and it should get an enum to define the states.

Once this issue is resolved I will fix the spurious retries to make them a lot less likely.

Quoted 93 lines
Quoted 19 lines

This patch fixes rnr retry errors which can be observed by running the pyverbs test suite 50-100X. With this patch applied they do not occur.

Fixes 8700e3e7c485 ("Soft RoCE (RXE) - The software RoCE driver")

---
  drivers/infiniband/sw/rxe/rxe_comp.c  | 4 +--- drivers/infiniband/sw/rxe/rxe_qp.c    | 1 + drivers/infiniband/sw/rxe/rxe_req.c   | 6 ++++-- drivers/infiniband/sw/rxe/rxe_verbs.h | 1 +

  4 files changed, 7 insertions(+), 5 deletions(-)
drivers/infiniband/sw/rxe/rxe_comp.c +0 -0
index 138b3e7d3a5f..bc668cb211b1 100644

drivers/infiniband/sw/rxe/rxe_comp.c int rxe_completer(void *arg) · line 736

Quoted 6 lines
drivers/infiniband/sw/rxe/rxe_comp.c +1 -2
@@ -733,9 +733,7 @@ int rxe_completer(void *arg)
                   if (qp->comp.rnr_retry != 7)
                       qp->comp.rnr_retry--;
   -                qp->req.need_retry = 1;
-                pr_debug("qp#%d set rnr nak timer\n",
-                     qp_num(qp));
+                qp->req.need_rnr_timeout = 1;

Suggest req.need_rnr_retry = -1  (and keep the useful pr_debug!)

Quoted 20 lines
drivers/infiniband/sw/rxe/rxe_comp.c +0 -0
                   mod_timer(&qp->rnr_nak_timer,
                         jiffies + rnrnak_jiffies(aeth_syn(pkt)
                           & ~AETH_TYPE_MASK));
drivers/infiniband/sw/rxe/rxe_qp.c +1 -0
index 62acf890af6c..1c962468714e 100644
@@ -513,6 +513,7 @@ static void rxe_qp_reset(struct rxe_qp *qp)
       atomic_set(&qp->ssn, 0);
       qp->req.opcode = -1;
       qp->req.need_retry = 0;
+    qp->req.need_rnr_timeout = 0;
       qp->req.noack_pkts = 0;
       qp->resp.msn = 0;
       qp->resp.opcode = -1;
drivers/infiniband/sw/rxe/rxe_req.c +0 -0
index ae5fbc79dd5c..770ae4279f73 100644

drivers/infiniband/sw/rxe/rxe_req.c void rnr_nak_timer(struct timer_list *t) · line 107

Quoted 5 lines
drivers/infiniband/sw/rxe/rxe_req.c +2 -0
@@ -103,7 +103,8 @@ void rnr_nak_timer(struct timer_list *t)
   {
       struct rxe_qp *qp = from_timer(qp, t, rnr_nak_timer);
   -    pr_debug("qp#%d rnr nak timer fired\n", qp_num(qp));
+    qp->req.need_retry = 1;
+    qp->req.need_rnr_timeout = 0;

Simply setting need_retry = 1 would suffice, if suggestion accepted.

Quoted 11 lines
drivers/infiniband/sw/rxe/rxe_req.c +0 -0
       rxe_run_task(&qp->req.task, 1);
   }
   @@ -624,10 +625,11 @@ int rxe_requester(void *arg)

          qp->req.need_rd_atomic = 0;
          qp->req.wait_psn = 0;
          qp->req.need_retry = 0;
+        qp->req.need_rnr_timeout = 0;
          goto exit;
      }
  -    if (unlikely(qp->req.need_retry)) {
+    if (unlikely(qp->req.need_retry && !qp->req.need_rnr_timeout)) {

This would become (unlikely (qp->req.rnr_retry > 0)) ...

drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_req_info { · line 126

Quoted 12 lines

          req_retry(qp);
          qp->req.need_retry = 0;
      }

drivers/infiniband/sw/rxe/rxe_verbs.h +1 -0
index e7eff1ca75e9..ab3186478c3f 100644
@@ -123,6 +123,7 @@ struct rxe_req_info {
       int            need_rd_atomic;
       int            wait_psn;
       int            need_retry;
+    int            need_rnr_timeout;

Drop

Quoted 5 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +0 -0
       int            noack_pkts;
       struct rxe_task        task;
   };

base-commit: c5eb0a61238dd6faf37f58c9ce61c9980aaffd7a

Tom.

#9 lore raw
Quoted 12 lines · Bob Pearson
Quoted 11 lines · Tom Talpey
Quoted 8 lines · Bob Pearson

Currently the completer tasklet when it sets the retransmit timer or the nak timer sets the same flag (qp->req.need_retry) so that if either timer fires it will attempt to perform a retry flow on the send queue. This has the effect of responding to an RNR NAK at the first retransmit timer event which does not allow for the requested rnr timeout.

This patch adds a new flag (qp->req.need_rnr_timeout) which, if set, prevents a retry flow until the rnr nak timer fires.

Quoted 6 lines
Quoted 4 lines

The new name is a little confusing, nobody "needs" an RNR timeout. :)
But it seems really odd (and maybe fragile) to have two flags. To me,
setting need_retry to "-1", or anything but 0 or 1, would be better.
After all, if the RNR NAK timer goes off, the code will generally retry, right? So it seems more logical to merge these.

I am trying to cleanup pyverbs runs which sometimes fail with rnr retry errors. In this specific case the rnr timeout value is set a lot longer than the retry timeout.

Interesting. I was not aware that was possible, and I'm skeptical. But perhaps that's another matter.

Quoted 11 lines

As discussed earlier the retry timer is never cleared so it continuously fires at about 40 times a second. The failing test is intentionally setting up a situation where the receiver is not ready and then it is. The retry timeout usually acts as though it were the rnr retry and 'fixes' the problem and the test passes. Since the retry timer is just randomly firing it sometimes happens too soon and the receiver isn't ready yet which leads to the failed test.

Logic says that if you receive an rnr nak the target is waiting for that specific send packet to be resent and no other so we shouldn't be responding to a spurious retry timeout.

Quoted 8 lines

You are correct that the retry sequence can be shared but it shouldn't start until the rnr timer has fired once an rnr nak is seen. This patch does exactly that by blocking the retry sequence until the rnr timer fires.

If you don't like 'need_rnr_timeout' perhaps you would accept 'wait_rnr_timeout' instead. Overloading need_retry is less clear IMHO. You don't need the retry until the rnr timer has fired then you do. If you really want to just use one variable we are basically implementing a state machine with 3 states and it should get an enum to define the states.

I guess I do prefer the latter enum approach. To elaborate on what I meant by fragile, it appears that with the two-field approach, only three states are valid. It can never be the case that both are set, yet here is your hunk in rxe_requester:

  if (unlikely(qp->req.need_retry && !qp->req.need_rnr_timeout)) {

The part after the && will always evaluate to true, right? The code clears need_rnr_timeout wherever it sets need_retry. So this is testing for an impossible case.

That said, if you don't agree, the name needs to be less confusing. I guess I'd suggest "rnr_timer_running".

Quoted 1 line

Once this issue is resolved I will fix the spurious retries to make them a lot less likely.

Excellent. RoCE suffers from congestion enough without pouring gasoline on the fire.

Tom.

Quoted 95 lines
Quoted 93 lines
Quoted 19 lines

This patch fixes rnr retry errors which can be observed by running the pyverbs test suite 50-100X. With this patch applied they do not occur.

Fixes 8700e3e7c485 ("Soft RoCE (RXE) - The software RoCE driver")
---
   drivers/infiniband/sw/rxe/rxe_comp.c  | 4 +---
   drivers/infiniband/sw/rxe/rxe_qp.c    | 1 +
   drivers/infiniband/sw/rxe/rxe_req.c   | 6 ++++--
   drivers/infiniband/sw/rxe/rxe_verbs.h | 1 +
   4 files changed, 7 insertions(+), 5 deletions(-)
drivers/infiniband/sw/rxe/rxe_comp.c +0 -0
index 138b3e7d3a5f..bc668cb211b1 100644

drivers/infiniband/sw/rxe/rxe_comp.c int rxe_completer(void *arg) · line 736

Quoted 6 lines
drivers/infiniband/sw/rxe/rxe_comp.c +1 -2
@@ -733,9 +733,7 @@ int rxe_completer(void *arg)
                   if (qp->comp.rnr_retry != 7)
                       qp->comp.rnr_retry--;
   -                qp->req.need_retry = 1;
-                pr_debug("qp#%d set rnr nak timer\n",
-                     qp_num(qp));
+                qp->req.need_rnr_timeout = 1;

Suggest req.need_rnr_retry = -1  (and keep the useful pr_debug!)

Quoted 20 lines
drivers/infiniband/sw/rxe/rxe_comp.c +0 -0
                   mod_timer(&qp->rnr_nak_timer,
                         jiffies + rnrnak_jiffies(aeth_syn(pkt)
                           & ~AETH_TYPE_MASK));
drivers/infiniband/sw/rxe/rxe_qp.c +1 -0
index 62acf890af6c..1c962468714e 100644
@@ -513,6 +513,7 @@ static void rxe_qp_reset(struct rxe_qp *qp)
       atomic_set(&qp->ssn, 0);
       qp->req.opcode = -1;
       qp->req.need_retry = 0;
+    qp->req.need_rnr_timeout = 0;
       qp->req.noack_pkts = 0;
       qp->resp.msn = 0;
       qp->resp.opcode = -1;
drivers/infiniband/sw/rxe/rxe_req.c +0 -0
index ae5fbc79dd5c..770ae4279f73 100644

drivers/infiniband/sw/rxe/rxe_req.c void rnr_nak_timer(struct timer_list *t) · line 107

Quoted 5 lines
drivers/infiniband/sw/rxe/rxe_req.c +2 -0
@@ -103,7 +103,8 @@ void rnr_nak_timer(struct timer_list *t)
   {
       struct rxe_qp *qp = from_timer(qp, t, rnr_nak_timer);
   -    pr_debug("qp#%d rnr nak timer fired\n", qp_num(qp));
+    qp->req.need_retry = 1;
+    qp->req.need_rnr_timeout = 0;

Simply setting need_retry = 1 would suffice, if suggestion accepted.

drivers/infiniband/sw/rxe/rxe_req.c int rxe_requester(void *arg) · line 632

Quoted 11 lines
drivers/infiniband/sw/rxe/rxe_req.c +2 -0
       rxe_run_task(&qp->req.task, 1);
   }
   @@ -624,10 +625,11 @@ int rxe_requester(void *arg)
           qp->req.need_rd_atomic = 0;
           qp->req.wait_psn = 0;
           qp->req.need_retry = 0;
+        qp->req.need_rnr_timeout = 0;
           goto exit;
       }
   -    if (unlikely(qp->req.need_retry)) {
+    if (unlikely(qp->req.need_retry && !qp->req.need_rnr_timeout)) {

This would become (unlikely (qp->req.rnr_retry > 0)) ...

drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_req_info { · line 126

Quoted 12 lines

          req_retry(qp);
          qp->req.need_retry = 0;
      }

drivers/infiniband/sw/rxe/rxe_verbs.h +1 -0
index e7eff1ca75e9..ab3186478c3f 100644
@@ -123,6 +123,7 @@ struct rxe_req_info {
       int            need_rd_atomic;
       int            wait_psn;
       int            need_retry;
+    int            need_rnr_timeout;

Drop

Quoted 5 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +0 -0
       int            noack_pkts;
       struct rxe_task        task;
   };

base-commit: c5eb0a61238dd6faf37f58c9ce61c9980aaffd7a

Tom.

#10 lore raw
Quoted 17 lines · Tom Talpey
Quoted 16 lines · Bob Pearson
Quoted 15 lines · Tom Talpey
Quoted 8 lines · Bob Pearson

Currently the completer tasklet when it sets the retransmit timer or the nak timer sets the same flag (qp->req.need_retry) so that if either timer fires it will attempt to perform a retry flow on the send queue. This has the effect of responding to an RNR NAK at the first retransmit timer event which does not allow for the requested rnr timeout.

This patch adds a new flag (qp->req.need_rnr_timeout) which, if set, prevents a retry flow until the rnr nak timer fires.

The new name is a little confusing, nobody "needs" an RNR timeout. :)
But it seems really odd (and maybe fragile) to have two flags. To me,
setting need_retry to "-1", or anything but 0 or 1, would be better.
After all, if the RNR NAK timer goes off, the code will generally retry, right? So it seems more logical to merge these.

Quoted 5 lines
Quoted 2 lines

I am trying to cleanup pyverbs runs which sometimes fail with rnr retry errors. In this specific case the rnr timeout value is set a lot longer than the retry timeout.

Interesting. I was not aware that was possible, and I'm skeptical. But perhaps that's another matter.

It would make sense. The retry timer handles NIC to NIC failures which are fast. Basically a network transit time or two. RNR recovery requires software to intervene and post a recv WR.

Quoted 26 lines
Quoted 12 lines

As discussed earlier the retry timer is never cleared so it continuously fires at about 40 times a second. The failing test is intentionally setting up a situation where the receiver is not ready and then it is. The retry timeout usually acts as though it were the rnr retry and 'fixes' the problem and the test passes. Since the retry timer is just randomly firing it sometimes happens too soon and the receiver isn't ready yet which leads to the failed test.

Logic says that if you receive an rnr nak the target is waiting for that specific send packet to be resent and no other so we shouldn't be responding to a spurious retry timeout.

Quoted 8 lines

You are correct that the retry sequence can be shared but it shouldn't start until the rnr timer has fired once an rnr nak is seen. This patch does exactly that by blocking the retry sequence until the rnr timer fires.

If you don't like 'need_rnr_timeout' perhaps you would accept 'wait_rnr_timeout' instead. Overloading need_retry is less clear IMHO. You don't need the retry until the rnr timer has fired then you do. If you really want to just use one variable we are basically implementing a state machine with 3 states and it should get an enum to define the states.

I guess I do prefer the latter enum approach. To elaborate on what I meant by fragile, it appears that with the two-field approach, only three states are valid. It can never be the case that both are set, yet here is your hunk in rxe_requester:

Quoted 5 lines

  if (unlikely(qp->req.need_retry && !qp->req.need_rnr_timeout)) {

The part after the && will always evaluate to true, right? The code clears need_rnr_timeout wherever it sets need_retry. So this is testing for an impossible case.

Wrong. In the RNR path things are as you say but if the retry timer fires, independently of receiving an RNR NAK, need_retry is set and need_rnr_timeout may or may not be true. If we did receive an RNR NAK then we should ignore the state of the retry timer until the rnr timer goes off.

Bob

Quoted 106 lines

That said, if you don't agree, the name needs to be less confusing. I guess I'd suggest "rnr_timer_running".

Quoted 1 line

Once this issue is resolved I will fix the spurious retries to make them a lot less likely.

Excellent. RoCE suffers from congestion enough without pouring gasoline on the fire.

Tom.

Quoted 95 lines
Quoted 93 lines
Quoted 19 lines

This patch fixes rnr retry errors which can be observed by running the pyverbs test suite 50-100X. With this patch applied they do not occur.

Fixes 8700e3e7c485 ("Soft RoCE (RXE) - The software RoCE driver")

---
   drivers/infiniband/sw/rxe/rxe_comp.c  | 4 +--- drivers/infiniband/sw/rxe/rxe_qp.c    | 1 + drivers/infiniband/sw/rxe/rxe_req.c   | 6 ++++-- drivers/infiniband/sw/rxe/rxe_verbs.h | 1 +

   4 files changed, 7 insertions(+), 5 deletions(-)
drivers/infiniband/sw/rxe/rxe_comp.c +0 -0
index 138b3e7d3a5f..bc668cb211b1 100644

drivers/infiniband/sw/rxe/rxe_comp.c int rxe_completer(void *arg) · line 736

Quoted 6 lines
drivers/infiniband/sw/rxe/rxe_comp.c +1 -2
@@ -733,9 +733,7 @@ int rxe_completer(void *arg)
                    if (qp->comp.rnr_retry != 7)
                        qp->comp.rnr_retry--;
    -                qp->req.need_retry = 1;
-                pr_debug("qp#%d set rnr nak timer\n",
-                     qp_num(qp));
+                qp->req.need_rnr_timeout = 1;

Suggest req.need_rnr_retry = -1  (and keep the useful pr_debug!)

Quoted 20 lines
drivers/infiniband/sw/rxe/rxe_comp.c +0 -0
                    mod_timer(&qp->rnr_nak_timer,
                          jiffies + rnrnak_jiffies(aeth_syn(pkt)
                            & ~AETH_TYPE_MASK));
drivers/infiniband/sw/rxe/rxe_qp.c +1 -0
index 62acf890af6c..1c962468714e 100644
@@ -513,6 +513,7 @@ static void rxe_qp_reset(struct rxe_qp *qp)
        atomic_set(&qp->ssn, 0);
        qp->req.opcode = -1;
        qp->req.need_retry = 0;
+    qp->req.need_rnr_timeout = 0;
        qp->req.noack_pkts = 0;
        qp->resp.msn = 0;
        qp->resp.opcode = -1;
drivers/infiniband/sw/rxe/rxe_req.c +0 -0
index ae5fbc79dd5c..770ae4279f73 100644

drivers/infiniband/sw/rxe/rxe_req.c void rnr_nak_timer(struct timer_list *t) · line 107

Quoted 5 lines
drivers/infiniband/sw/rxe/rxe_req.c +2 -0
@@ -103,7 +103,8 @@ void rnr_nak_timer(struct timer_list *t)
    {
        struct rxe_qp *qp = from_timer(qp, t, rnr_nak_timer);
    -    pr_debug("qp#%d rnr nak timer fired\n", qp_num(qp));
+    qp->req.need_retry = 1;
+    qp->req.need_rnr_timeout = 0;

Simply setting need_retry = 1 would suffice, if suggestion accepted.

Quoted 11 lines
drivers/infiniband/sw/rxe/rxe_req.c +0 -0
        rxe_run_task(&qp->req.task, 1);
    }
    @@ -624,10 +625,11 @@ int rxe_requester(void *arg)

           qp->req.need_rd_atomic = 0;
           qp->req.wait_psn = 0;
           qp->req.need_retry = 0;
+        qp->req.need_rnr_timeout = 0;
           goto exit;
       }
   -    if (unlikely(qp->req.need_retry)) {
+    if (unlikely(qp->req.need_retry && !qp->req.need_rnr_timeout)) {

This would become (unlikely (qp->req.rnr_retry > 0)) ...

drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_req_info { · line 126

Quoted 12 lines

           req_retry(qp);
           qp->req.need_retry = 0;
       }

drivers/infiniband/sw/rxe/rxe_verbs.h +1 -0
index e7eff1ca75e9..ab3186478c3f 100644
@@ -123,6 +123,7 @@ struct rxe_req_info {
        int            need_rd_atomic;
        int            wait_psn;
        int            need_retry;
+    int            need_rnr_timeout;

Drop

Quoted 5 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +0 -0
        int            noack_pkts;
        struct rxe_task        task;
    };

base-commit: c5eb0a61238dd6faf37f58c9ce61c9980aaffd7a

Tom.

#11 lore raw
Quoted 56 lines · Bob Pearson
Quoted 17 lines · Tom Talpey
Quoted 16 lines · Bob Pearson
Quoted 15 lines · Tom Talpey
Quoted 8 lines · Bob Pearson

Currently the completer tasklet when it sets the retransmit timer or the nak timer sets the same flag (qp->req.need_retry) so that if either timer fires it will attempt to perform a retry flow on the send queue. This has the effect of responding to an RNR NAK at the first retransmit timer event which does not allow for the requested rnr timeout.

This patch adds a new flag (qp->req.need_rnr_timeout) which, if set, prevents a retry flow until the rnr nak timer fires.

The new name is a little confusing, nobody "needs" an RNR timeout. :)
But it seems really odd (and maybe fragile) to have two flags. To me,
setting need_retry to "-1", or anything but 0 or 1, would be better.
After all, if the RNR NAK timer goes off, the code will generally retry, right? So it seems more logical to merge these.

Quoted 5 lines
Quoted 2 lines

I am trying to cleanup pyverbs runs which sometimes fail with rnr retry errors. In this specific case the rnr timeout value is set a lot longer than the retry timeout.

Interesting. I was not aware that was possible, and I'm skeptical. But perhaps that's another matter.

It would make sense. The retry timer handles NIC to NIC failures which are fast. Basically a network transit time or two. RNR recovery requires software to intervene and post a recv WR.

Quoted 28 lines
Quoted 12 lines

As discussed earlier the retry timer is never cleared so it continuously fires at about 40 times a second. The failing test is intentionally setting up a situation where the receiver is not ready and then it is. The retry timeout usually acts as though it were the rnr retry and 'fixes' the problem and the test passes. Since the retry timer is just randomly firing it sometimes happens too soon and the receiver isn't ready yet which leads to the failed test.

Logic says that if you receive an rnr nak the target is waiting for that specific send packet to be resent and no other so we shouldn't be responding to a spurious retry timeout.

Quoted 8 lines

You are correct that the retry sequence can be shared but it shouldn't start until the rnr timer has fired once an rnr nak is seen. This patch does exactly that by blocking the retry sequence until the rnr timer fires.

If you don't like 'need_rnr_timeout' perhaps you would accept 'wait_rnr_timeout' instead. Overloading need_retry is less clear IMHO. You don't need the retry until the rnr timer has fired then you do. If you really want to just use one variable we are basically implementing a state machine with 3 states and it should get an enum to define the states.

I guess I do prefer the latter enum approach. To elaborate on what I meant by fragile, it appears that with the two-field approach, only three states are valid. It can never be the case that both are set, yet here is your hunk in rxe_requester:

  if (unlikely(qp->req.need_retry && !qp->req.need_rnr_timeout)) {

Quoted 8 lines
Quoted 3 lines

The part after the && will always evaluate to true, right? The code clears need_rnr_timeout wherever it sets need_retry. So this is testing for an impossible case.

Wrong. In the RNR path things are as you say but if the retry timer fires, independently of receiving an RNR NAK, need_retry is set and need_rnr_timeout may or may not be true. If we did receive an RNR NAK then we should ignore the state of the retry timer until the rnr timer goes off.

Ok, you're correct that it's possible that need_retry may be determined, and at just that moment before the retry happens, an RNR NAK arrives and starts the second timer. These fields aren't protected by locks so it all seems super-racy. But anyway...

This really screams for clarity in the code, and again, I'll suggest that the new field name is confusing.

Another name suggestion might be "need_retry_after_delay", which rhymes with the existing "need_retry".

Tom.

Quoted 109 lines

Bob

Quoted 106 lines

That said, if you don't agree, the name needs to be less confusing. I guess I'd suggest "rnr_timer_running".

Quoted 1 line

Once this issue is resolved I will fix the spurious retries to make them a lot less likely.

Excellent. RoCE suffers from congestion enough without pouring gasoline on the fire.

Tom.

Quoted 95 lines
Quoted 93 lines
Quoted 19 lines

This patch fixes rnr retry errors which can be observed by running the pyverbs test suite 50-100X. With this patch applied they do not occur.

Fixes 8700e3e7c485 ("Soft RoCE (RXE) - The software RoCE driver")
---
    drivers/infiniband/sw/rxe/rxe_comp.c  | 4 +---
    drivers/infiniband/sw/rxe/rxe_qp.c    | 1 +
    drivers/infiniband/sw/rxe/rxe_req.c   | 6 ++++--
    drivers/infiniband/sw/rxe/rxe_verbs.h | 1 +
    4 files changed, 7 insertions(+), 5 deletions(-)
drivers/infiniband/sw/rxe/rxe_comp.c +0 -0
index 138b3e7d3a5f..bc668cb211b1 100644

drivers/infiniband/sw/rxe/rxe_comp.c int rxe_completer(void *arg) · line 736

Quoted 6 lines
drivers/infiniband/sw/rxe/rxe_comp.c +1 -2
@@ -733,9 +733,7 @@ int rxe_completer(void *arg)
                    if (qp->comp.rnr_retry != 7)
                        qp->comp.rnr_retry--;
    -                qp->req.need_retry = 1;
-                pr_debug("qp#%d set rnr nak timer\n",
-                     qp_num(qp));
+                qp->req.need_rnr_timeout = 1;

Suggest req.need_rnr_retry = -1  (and keep the useful pr_debug!)

Quoted 20 lines
drivers/infiniband/sw/rxe/rxe_comp.c +0 -0
                    mod_timer(&qp->rnr_nak_timer,
                          jiffies + rnrnak_jiffies(aeth_syn(pkt)
                            & ~AETH_TYPE_MASK));
drivers/infiniband/sw/rxe/rxe_qp.c +1 -0
index 62acf890af6c..1c962468714e 100644
@@ -513,6 +513,7 @@ static void rxe_qp_reset(struct rxe_qp *qp)
        atomic_set(&qp->ssn, 0);
        qp->req.opcode = -1;
        qp->req.need_retry = 0;
+    qp->req.need_rnr_timeout = 0;
        qp->req.noack_pkts = 0;
        qp->resp.msn = 0;
        qp->resp.opcode = -1;
drivers/infiniband/sw/rxe/rxe_req.c +0 -0
index ae5fbc79dd5c..770ae4279f73 100644

drivers/infiniband/sw/rxe/rxe_req.c void rnr_nak_timer(struct timer_list *t) · line 107

Quoted 5 lines
drivers/infiniband/sw/rxe/rxe_req.c +2 -0
@@ -103,7 +103,8 @@ void rnr_nak_timer(struct timer_list *t)
    {
        struct rxe_qp *qp = from_timer(qp, t, rnr_nak_timer);
    -    pr_debug("qp#%d rnr nak timer fired\n", qp_num(qp));
+    qp->req.need_retry = 1;
+    qp->req.need_rnr_timeout = 0;

Simply setting need_retry = 1 would suffice, if suggestion accepted.

drivers/infiniband/sw/rxe/rxe_req.c int rxe_requester(void *arg) · line 632

Quoted 11 lines
drivers/infiniband/sw/rxe/rxe_req.c +2 -0
        rxe_run_task(&qp->req.task, 1);
    }
    @@ -624,10 +625,11 @@ int rxe_requester(void *arg)
            qp->req.need_rd_atomic = 0;
            qp->req.wait_psn = 0;
            qp->req.need_retry = 0;
+        qp->req.need_rnr_timeout = 0;
            goto exit;
        }
    -    if (unlikely(qp->req.need_retry)) {
+    if (unlikely(qp->req.need_retry && !qp->req.need_rnr_timeout)) {

This would become (unlikely (qp->req.rnr_retry > 0)) ...

drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_req_info { · line 126

Quoted 12 lines

           req_retry(qp);
           qp->req.need_retry = 0;
       }

drivers/infiniband/sw/rxe/rxe_verbs.h +1 -0
index e7eff1ca75e9..ab3186478c3f 100644
@@ -123,6 +123,7 @@ struct rxe_req_info {
        int            need_rd_atomic;
        int            wait_psn;
        int            need_retry;
+    int            need_rnr_timeout;

Drop

Quoted 5 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +0 -0
        int            noack_pkts;
        struct rxe_task        task;
    };

base-commit: c5eb0a61238dd6faf37f58c9ce61c9980aaffd7a

Tom.