Skip to contentSkip to conversation

RDMA/rxe: Fix a dead lock problem

PATCH In review [email protected] 3 participants 6 messages Apr 16, 2022 – Apr 21, 2022

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH v5 1/2 RDMA/rxe: Fix a dead lock problem

From: Zhu Yanjun <[email protected]>

This is a dead lock problem.
The ah_pool xa_lock first is acquired in this:

{SOFTIRQ-ON-W} state was registered at:

  lock_acquire+0x1d2/0x5a0
  _raw_spin_lock+0x33/0x80
  __rxe_add_to_pool+0x183/0x230 [rdma_rxe]

Then ah_pool xa_lock is acquired in this:

{IN-SOFTIRQ-W}:

Call Trace:
 <TASK>
  dump_stack_lvl+0x44/0x57
  mark_lock.part.52.cold.79+0x3c/0x46
  __lock_acquire+0x1565/0x34a0
  lock_acquire+0x1d2/0x5a0
  _raw_spin_lock_irqsave+0x42/0x90
  rxe_pool_get_index+0x72/0x1d0 [rdma_rxe]
  rxe_get_av+0x168/0x2a0 [rdma_rxe]
</TASK>

From the above, in the function __rxe_add_to_pool, xa_lock is acquired. Then the function __rxe_add_to_pool is interrupted by softirq. The function rxe_pool_get_index will also acquire xa_lock.

Finally, the dead lock appears.

[  296.806097]        CPU0
[  296.808550]        ----
[  296.811003]   lock(&xa->xa_lock#15);  <----- __rxe_add_to_pool
[  296.814583]   <Interrupt>
[  296.817209]     lock(&xa->xa_lock#15); <---- rxe_pool_get_index
[  296.820961]
                 *** DEADLOCK ***
Fixes 3225717f6dfa ("RDMA/rxe: Replace red-black trees by carrays")
Reported-and-tested-by Yi Zhang <[email protected]>

---
V4->V5: Commit logs are changed to avoid confusion. V3->V4: xa_lock_irq locks are used.
V2->V3: __rxe_add_to_pool is between spin_lock and spin_unlock, so
GFP_ATOMIC is used in __rxe_add_to_pool.
V1->V2: Replace GFP_KERNEL with GFP_ATOMIC

---
 drivers/infiniband/sw/rxe/rxe_pool.c | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)
drivers/infiniband/sw/rxe/rxe_pool.c +14 -6
index 87066d04ed18..f1f06dc7e64f 100644
@@ -106,7 +106,7 @@ void rxe_pool_init(struct rxe_dev *rxe, struct rxe_pool *pool,
 
 	atomic_set(&pool->num_elem, 0);
 
-	xa_init_flags(&pool->xa, XA_FLAGS_ALLOC);
+	xa_init_flags(&pool->xa, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ);
 	pool->limit.min = info->min_index;
 	pool->limit.max = info->max_index;
 }
@@ -138,8 +138,10 @@ void *rxe_alloc(struct rxe_pool *pool)
 	elem->obj = obj;
 	kref_init(&elem->ref_cnt);
 
-	err = xa_alloc_cyclic(&pool->xa, &elem->index, elem, pool->limit,
-			      &pool->next, GFP_KERNEL);
+	xa_lock_irq(&pool->xa);
+	err = __xa_alloc_cyclic(&pool->xa, &elem->index, elem, pool->limit,
+				&pool->next, GFP_KERNEL);
+	xa_unlock_irq(&pool->xa);
 	if (err)
 		goto err_free;
 
@@ -155,6 +157,7 @@ void *rxe_alloc(struct rxe_pool *pool)
 int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem)
 {
 	int err;
+	unsigned long flags;
 
 	if (WARN_ON(pool->flags & RXE_POOL_ALLOC))
 		return -EINVAL;
@@ -166,8 +169,10 @@ int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem)
 	elem->obj = (u8 *)elem - pool->elem_offset;
 	kref_init(&elem->ref_cnt);
 
-	err = xa_alloc_cyclic(&pool->xa, &elem->index, elem, pool->limit,
-			      &pool->next, GFP_KERNEL);
+	xa_lock_irqsave(&pool->xa, flags);
+	err = __xa_alloc_cyclic(&pool->xa, &elem->index, elem, pool->limit,
+				&pool->next, GFP_ATOMIC);
+	xa_unlock_irqrestore(&pool->xa, flags);
 	if (err)
 		goto err_cnt;
 
@@ -200,8 +205,11 @@ static void rxe_elem_release(struct kref *kref)
 {
 	struct rxe_pool_elem *elem = container_of(kref, typeof(*elem), ref_cnt);
 	struct rxe_pool *pool = elem->pool;
+	unsigned long flags;
 
-	xa_erase(&pool->xa, elem->index);
+	xa_lock_irqsave(&pool->xa, flags);
+	__xa_erase(&pool->xa, elem->index);
+	xa_unlock_irqrestore(&pool->xa, flags);
 
 	if (pool->cleanup)
 		pool->cleanup(elem);
Signature · 1 line
2.27.0
#1 lore raw

PATCH v2 2/2 RDMA/rxe: Use different xa locks on different path

From: Zhu Yanjun <[email protected]>

The function __rxe_add_to_pool is called on different path, and the requirement of the locks is different. The function rxe_create_ah requires xa_lock_irqsave/irqrestore while others only require xa_lock_irq.

---
V1->V2: Use pool type instead of gfp_t;

---
 drivers/infiniband/sw/rxe/rxe_pool.c | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)
drivers/infiniband/sw/rxe/rxe_pool.c +13 -5
index f1f06dc7e64f..5b097d6666eb 100644
@@ -157,7 +157,6 @@ void *rxe_alloc(struct rxe_pool *pool)
 int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem)
 {
 	int err;
-	unsigned long flags;
 
 	if (WARN_ON(pool->flags & RXE_POOL_ALLOC))
 		return -EINVAL;
@@ -169,10 +168,19 @@ int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem)
 	elem->obj = (u8 *)elem - pool->elem_offset;
 	kref_init(&elem->ref_cnt);
 
-	xa_lock_irqsave(&pool->xa, flags);
-	err = __xa_alloc_cyclic(&pool->xa, &elem->index, elem, pool->limit,
-				&pool->next, GFP_ATOMIC);
-	xa_unlock_irqrestore(&pool->xa, flags);
+	if (pool->type == RXE_TYPE_AH) {
+		unsigned long flags;
+
+		xa_lock_irqsave(&pool->xa, flags);
+		err = __xa_alloc_cyclic(&pool->xa, &elem->index, elem, pool->limit,
+					&pool->next, GFP_ATOMIC);
+		xa_unlock_irqrestore(&pool->xa, flags);
+	} else {
+		xa_lock_irq(&pool->xa);
+		err = __xa_alloc_cyclic(&pool->xa, &elem->index, elem, pool->limit,
+					&pool->next, GFP_KERNEL);
+		xa_unlock_irq(&pool->xa);
+	}
 	if (err)
 		goto err_cnt;
 
Signature · 1 line
2.27.0
#2 lore raw

Re: [PATCHv2 2/2] RDMA/rxe: Use different xa locks on different path

Quoted 27 lines · On Sat, Apr 16, 2022 at 10:43:43PM -0400, [email protected] wrote:

From: Zhu Yanjun <[email protected]>

The function __rxe_add_to_pool is called on different path, and the requirement of the locks is different. The function rxe_create_ah requires xa_lock_irqsave/irqrestore while others only require xa_lock_irq.

V1->V2: Use pool type instead of gfp_t;

 drivers/infiniband/sw/rxe/rxe_pool.c | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)
drivers/infiniband/sw/rxe/rxe_pool.c +1 -2
index f1f06dc7e64f..5b097d6666eb 100644
+++ b/drivers/infiniband/sw/rxe/rxe_pool.c
@@ -157,7 +157,6 @@ void *rxe_alloc(struct rxe_pool *pool)
 int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem)
 {
 	int err;
-	unsigned long flags;
 
 	if (WARN_ON(pool->flags & RXE_POOL_ALLOC))
 		return -EINVAL;
@@ -169,10 +168,19 @@ int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem)
 	elem->obj = (u8 *)elem - pool->elem_offset;
 	kref_init(&elem->ref_cnt);
 
-	xa_lock_irqsave(&pool->xa, flags);

drivers/infiniband/sw/rxe/rxe_pool.c int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem) · line 177

Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_pool.c +7 -3
-	err = __xa_alloc_cyclic(&pool->xa, &elem->index, elem, pool->limit,
-				&pool->next, GFP_ATOMIC);
-	xa_unlock_irqrestore(&pool->xa, flags);
+	if (pool->type == RXE_TYPE_AH) {
+		unsigned long flags;
+
+		xa_lock_irqsave(&pool->xa, flags);
+		err = __xa_alloc_cyclic(&pool->xa, &elem->index, elem, pool->limit,
+					&pool->next, GFP_ATOMIC);
+		xa_unlock_irqrestore(&pool->xa, flags);

This is still not right - you have to test RDMA_CREATE_AH_SLEEPABLE for the AH path as well.

Jason

#3 lore raw
In reply to [email protected]
Quoted 68 lines · On Sat, Apr 16, 2022 at 10:43:42PM -0400, [email protected] wrote:

From: Zhu Yanjun <[email protected]>

This is a dead lock problem.
The ah_pool xa_lock first is acquired in this:

{SOFTIRQ-ON-W} state was registered at:

  lock_acquire+0x1d2/0x5a0
  _raw_spin_lock+0x33/0x80
  __rxe_add_to_pool+0x183/0x230 [rdma_rxe]

Then ah_pool xa_lock is acquired in this:

{IN-SOFTIRQ-W}:

Call Trace:
 <TASK>
  dump_stack_lvl+0x44/0x57
  mark_lock.part.52.cold.79+0x3c/0x46
  __lock_acquire+0x1565/0x34a0
  lock_acquire+0x1d2/0x5a0
  _raw_spin_lock_irqsave+0x42/0x90
  rxe_pool_get_index+0x72/0x1d0 [rdma_rxe]
  rxe_get_av+0x168/0x2a0 [rdma_rxe]
</TASK>

From the above, in the function __rxe_add_to_pool, xa_lock is acquired. Then the function __rxe_add_to_pool is interrupted by softirq. The function rxe_pool_get_index will also acquire xa_lock.

Finally, the dead lock appears.

[  296.806097]        CPU0
[  296.808550]        ----
[  296.811003]   lock(&xa->xa_lock#15);  <----- __rxe_add_to_pool
[  296.814583]   <Interrupt>
[  296.817209]     lock(&xa->xa_lock#15); <---- rxe_pool_get_index
[  296.820961]
                 *** DEADLOCK ***
Fixes 3225717f6dfa ("RDMA/rxe: Replace red-black trees by carrays")
Reported-and-tested-by Yi Zhang <[email protected]>

---
V4->V5: Commit logs are changed to avoid confusion. V3->V4: xa_lock_irq locks are used.
V2->V3: __rxe_add_to_pool is between spin_lock and spin_unlock, so
GFP_ATOMIC is used in __rxe_add_to_pool.
V1->V2: Replace GFP_KERNEL with GFP_ATOMIC

---
 drivers/infiniband/sw/rxe/rxe_pool.c | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)
drivers/infiniband/sw/rxe/rxe_pool.c +1 -1
index 87066d04ed18..f1f06dc7e64f 100644
@@ -106,7 +106,7 @@ void rxe_pool_init(struct rxe_dev *rxe, struct rxe_pool *pool,
 
 	atomic_set(&pool->num_elem, 0);
 
-	xa_init_flags(&pool->xa, XA_FLAGS_ALLOC);
+	xa_init_flags(&pool->xa, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ);
 	pool->limit.min = info->min_index;
 	pool->limit.max = info->max_index;
 }

drivers/infiniband/sw/rxe/rxe_pool.c void *rxe_alloc(struct rxe_pool *pool) · line 144

Quoted 9 lines
drivers/infiniband/sw/rxe/rxe_pool.c +4 -2
@@ -138,8 +138,10 @@ void *rxe_alloc(struct rxe_pool *pool)
 	elem->obj = obj;
 	kref_init(&elem->ref_cnt);
 
-	err = xa_alloc_cyclic(&pool->xa, &elem->index, elem, pool->limit,
-			      &pool->next, GFP_KERNEL);
+	xa_lock_irq(&pool->xa);
+	err = __xa_alloc_cyclic(&pool->xa, &elem->index, elem, pool->limit,
+				&pool->next, GFP_KERNEL);
+	xa_unlock_irq(&pool->xa);

I may admit that I didn't follow your previous discussions, so maybe you already explained it. But why do you need xa_lock_irq() here?

Thanks

Quoted 40 lines
drivers/infiniband/sw/rxe/rxe_pool.c +9 -3
 	if (err)
 		goto err_free;
 
@@ -155,6 +157,7 @@ void *rxe_alloc(struct rxe_pool *pool)
 int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem)
 {
 	int err;
+	unsigned long flags;
 
 	if (WARN_ON(pool->flags & RXE_POOL_ALLOC))
 		return -EINVAL;
@@ -166,8 +169,10 @@ int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem)
 	elem->obj = (u8 *)elem - pool->elem_offset;
 	kref_init(&elem->ref_cnt);
 
-	err = xa_alloc_cyclic(&pool->xa, &elem->index, elem, pool->limit,
-			      &pool->next, GFP_KERNEL);
+	xa_lock_irqsave(&pool->xa, flags);
+	err = __xa_alloc_cyclic(&pool->xa, &elem->index, elem, pool->limit,
+				&pool->next, GFP_ATOMIC);
+	xa_unlock_irqrestore(&pool->xa, flags);
 	if (err)
 		goto err_cnt;
 
@@ -200,8 +205,11 @@ static void rxe_elem_release(struct kref *kref)
 {
 	struct rxe_pool_elem *elem = container_of(kref, typeof(*elem), ref_cnt);
 	struct rxe_pool *pool = elem->pool;
+	unsigned long flags;
 
-	xa_erase(&pool->xa, elem->index);
+	xa_lock_irqsave(&pool->xa, flags);
+	__xa_erase(&pool->xa, elem->index);
+	xa_unlock_irqrestore(&pool->xa, flags);
 
 	if (pool->cleanup)
 		pool->cleanup(elem);
Signature · 1 line
2.27.0
#4 lore raw
Quoted 69 lines · Leon Romanovsky
Quoted 68 lines · On Sat, Apr 16, 2022 at 10:43:42PM -0400, [email protected] wrote:

From: Zhu Yanjun <[email protected]>

This is a dead lock problem.
The ah_pool xa_lock first is acquired in this:

{SOFTIRQ-ON-W} state was registered at:

  lock_acquire+0x1d2/0x5a0
  _raw_spin_lock+0x33/0x80
  __rxe_add_to_pool+0x183/0x230 [rdma_rxe]

Then ah_pool xa_lock is acquired in this:

{IN-SOFTIRQ-W}:

Call Trace:
 <TASK>
  dump_stack_lvl+0x44/0x57
  mark_lock.part.52.cold.79+0x3c/0x46
  __lock_acquire+0x1565/0x34a0
  lock_acquire+0x1d2/0x5a0
  _raw_spin_lock_irqsave+0x42/0x90
  rxe_pool_get_index+0x72/0x1d0 [rdma_rxe]
  rxe_get_av+0x168/0x2a0 [rdma_rxe]
</TASK>

From the above, in the function __rxe_add_to_pool, xa_lock is acquired. Then the function __rxe_add_to_pool is interrupted by softirq. The function rxe_pool_get_index will also acquire xa_lock.

Finally, the dead lock appears.

[  296.806097]        CPU0
[  296.808550]        ----
[  296.811003]   lock(&xa->xa_lock#15);  <----- __rxe_add_to_pool
[  296.814583]   <Interrupt>
[  296.817209]     lock(&xa->xa_lock#15); <---- rxe_pool_get_index
[  296.820961]
                 *** DEADLOCK ***
Fixes 3225717f6dfa ("RDMA/rxe: Replace red-black trees by carrays")
Reported-and-tested-by Yi Zhang <[email protected]>

V4->V5: Commit logs are changed to avoid confusion. V3->V4: xa_lock_irq locks are used.
V2->V3: __rxe_add_to_pool is between spin_lock and spin_unlock, so
GFP_ATOMIC is used in __rxe_add_to_pool.
V1->V2: Replace GFP_KERNEL with GFP_ATOMIC

 drivers/infiniband/sw/rxe/rxe_pool.c | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)
drivers/infiniband/sw/rxe/rxe_pool.c +2 -1
index 87066d04ed18..f1f06dc7e64f 100644
+++ b/drivers/infiniband/sw/rxe/rxe_pool.c
@@ -106,7 +106,7 @@ void rxe_pool_init(struct rxe_dev *rxe, struct rxe_pool *pool,
 
 	atomic_set(&pool->num_elem, 0);
 
-	xa_init_flags(&pool->xa, XA_FLAGS_ALLOC);
+	xa_init_flags(&pool->xa, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ);
 	pool->limit.min = info->min_index;
 	pool->limit.max = info->max_index;
 }
@@ -138,8 +138,10 @@ void *rxe_alloc(struct rxe_pool *pool)
 	elem->obj = obj;
 	kref_init(&elem->ref_cnt);
 
Quoted 6 lines

drivers/infiniband/sw/rxe/rxe_pool.c void *rxe_alloc(struct rxe_pool *pool) · line 144

Quoted 6 lines
drivers/infiniband/sw/rxe/rxe_pool.c +4 -2
-	err = xa_alloc_cyclic(&pool->xa, &elem->index, elem, pool->limit,
-			      &pool->next, GFP_KERNEL);
+	xa_lock_irq(&pool->xa);
+	err = __xa_alloc_cyclic(&pool->xa, &elem->index, elem, pool->limit,
+				&pool->next, GFP_KERNEL);
+	xa_unlock_irq(&pool->xa);

It should just use xa_alloc_cyclic_irq() and xa_erase_irq(). Don't open code the lock.

Quoted 2 lines

I may admit that I didn't follow your previous discussions, so maybe you already explained it. But why do you need xa_lock_irq() here?

The spinlock type needs to be consistent in all users.

You can only use the naked version if the spinlock is always obtained from a process context.

You can only use bh version if the spinlock is always obtained from a process context or bh/softirq

You can always use the irq version

What I don't understand is why IRQ and not BH? AFAIK there is no case where rxe is called from a real IRQ, right? Or is it because you can't nest BH under the IRQ spinlock from CM?

Jason

author
#5 lore raw
Quoted 70 lines · Jason Gunthorpe
Quoted 69 lines · Leon Romanovsky
Quoted 68 lines · On Sat, Apr 16, 2022 at 10:43:42PM -0400, [email protected] wrote:

From: Zhu Yanjun <[email protected]>

This is a dead lock problem.
The ah_pool xa_lock first is acquired in this:

{SOFTIRQ-ON-W} state was registered at:

   lock_acquire+0x1d2/0x5a0
   _raw_spin_lock+0x33/0x80
   __rxe_add_to_pool+0x183/0x230 [rdma_rxe]

Then ah_pool xa_lock is acquired in this:

{IN-SOFTIRQ-W}:

Call Trace:
  <TASK>
   dump_stack_lvl+0x44/0x57
   mark_lock.part.52.cold.79+0x3c/0x46
   __lock_acquire+0x1565/0x34a0
   lock_acquire+0x1d2/0x5a0
   _raw_spin_lock_irqsave+0x42/0x90
   rxe_pool_get_index+0x72/0x1d0 [rdma_rxe]
   rxe_get_av+0x168/0x2a0 [rdma_rxe]
</TASK>

From the above, in the function __rxe_add_to_pool,
xa_lock is acquired. Then the function __rxe_add_to_pool is interrupted by softirq. The function rxe_pool_get_index will also acquire xa_lock.

Finally, the dead lock appears.

[  296.806097]        CPU0
[  296.808550]        ----
[  296.811003]   lock(&xa->xa_lock#15);  <----- __rxe_add_to_pool
[  296.814583]   <Interrupt>
[  296.817209]     lock(&xa->xa_lock#15); <---- rxe_pool_get_index
[  296.820961]
                  *** DEADLOCK ***
Fixes 3225717f6dfa ("RDMA/rxe: Replace red-black trees by carrays")
Reported-and-tested-by Yi Zhang <[email protected]>

V4->V5: Commit logs are changed to avoid confusion. V3->V4: xa_lock_irq locks are used.
V2->V3: __rxe_add_to_pool is between spin_lock and spin_unlock, so
GFP_ATOMIC is used in __rxe_add_to_pool.
V1->V2: Replace GFP_KERNEL with GFP_ATOMIC

  drivers/infiniband/sw/rxe/rxe_pool.c | 20 ++++++++++++++------
  1 file changed, 14 insertions(+), 6 deletions(-)
drivers/infiniband/sw/rxe/rxe_pool.c +2 -1
index 87066d04ed18..f1f06dc7e64f 100644
+++ b/drivers/infiniband/sw/rxe/rxe_pool.c
@@ -106,7 +106,7 @@ void rxe_pool_init(struct rxe_dev *rxe, struct rxe_pool *pool,
 
  	atomic_set(&pool->num_elem, 0);
 
-	xa_init_flags(&pool->xa, XA_FLAGS_ALLOC);
+	xa_init_flags(&pool->xa, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ);
  	pool->limit.min = info->min_index;
  	pool->limit.max = info->max_index;
  }
@@ -138,8 +138,10 @@ void *rxe_alloc(struct rxe_pool *pool)
  	elem->obj = obj;
  	kref_init(&elem->ref_cnt);
 
Quoted 9 lines
Quoted 6 lines

drivers/infiniband/sw/rxe/rxe_pool.c void *rxe_alloc(struct rxe_pool *pool) · line 144

Quoted 6 lines
drivers/infiniband/sw/rxe/rxe_pool.c +4 -2
-	err = xa_alloc_cyclic(&pool->xa, &elem->index, elem, pool->limit,
-			      &pool->next, GFP_KERNEL);
+	xa_lock_irq(&pool->xa);
+	err = __xa_alloc_cyclic(&pool->xa, &elem->index, elem, pool->limit,
+				&pool->next, GFP_KERNEL);
+	xa_unlock_irq(&pool->xa);

It should just use xa_alloc_cyclic_irq() and xa_erase_irq(). Don't open code the lock.

Got it. I will use the above functions.

Quoted 9 lines
Quoted 3 lines

I may admit that I didn't follow your previous discussions, so maybe you already explained it. But why do you need xa_lock_irq() here?

The spinlock type needs to be consistent in all users.

You can only use the naked version if the spinlock is always obtained from a process context.

Quoted 8 lines

You can only use bh version if the spinlock is always obtained from a process context or bh/softirq

You can always use the irq version

What I don't understand is why IRQ and not BH? AFAIK there is no case where rxe is called from a real IRQ, right? Or is it because you can't nest BH under the IRQ spinlock from CM?

Sure. I will use IRQ spinlock. The reason is as below:

1. https://patchwork.kernel.org/project/linux-rdma/patch/[email protected]/

2. https://patchwork.kernel.org/project/linux-rdma/patch/[email protected]/

The above 2 links are why I used IRQ.

Zhu Yanjun

Quoted 2 lines

Jason